diff options
author | Peng Hui Jiang <jiangphcn@apache.org> | 2018-08-22 00:04:43 +0800 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-08-22 00:04:43 +0800 |
commit | c02f6d294d962d18c22d8179f43ee07add5437eb (patch) | |
tree | 7edb822ce2189c22f0ad1692275220a24043cdda | |
parent | 8819a27864e4f4dbe0f4544986749b51d4a1d7eb (diff) | |
parent | e0c7ad386803dea20573201f4e6468bdae8294e1 (diff) | |
download | couchdb-c02f6d294d962d18c22d8179f43ee07add5437eb.tar.gz |
Merge pull request #1368 from apache/COUCHDB-3326-clustered-purge-pr3-refactor-pse-tests
[3/5] Clustered Purge - Rewrite pluggable storage engine tests
21 files changed, 1817 insertions, 1735 deletions
diff --git a/rebar.config.script b/rebar.config.script index b4de2d962..22dde7cde 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -31,6 +31,7 @@ SubDirs = [ "src/couch_mrview", "src/couch_replicator", "src/couch_plugins", + "src/couch_pse_tests", "src/couch_event", "src/couch_stats", "src/couch_peruser", diff --git a/src/couch/src/test_engine_compaction.erl b/src/couch/src/test_engine_compaction.erl deleted file mode 100644 index 09a1e4ea5..000000000 --- a/src/couch/src/test_engine_compaction.erl +++ /dev/null @@ -1,185 +0,0 @@ -% Licensed under the Apache License, Version 2.0 (the "License"); you may not -% use this file except in compliance with the License. You may obtain a copy of -% the License at -% -% http://www.apache.org/licenses/LICENSE-2.0 -% -% Unless required by applicable law or agreed to in writing, software -% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -% License for the specific language governing permissions and limitations under -% the License. - --module(test_engine_compaction). --compile(export_all). - - --include_lib("eunit/include/eunit.hrl"). --include_lib("couch/include/couch_db.hrl"). - - -cet_compact_empty() -> - {ok, Engine, Path, St1} = test_engine_util:init_engine(dbpath), - Db1 = test_engine_util:db_as_term(Engine, St1), - {ok, St2, DbName, _, Term} = test_engine_util:compact(Engine, St1, Path), - {ok, St3, undefined} = Engine:finish_compaction(St2, DbName, [], Term), - Db2 = test_engine_util:db_as_term(Engine, St3), - Diff = test_engine_util:term_diff(Db1, Db2), - ?assertEqual(nodiff, Diff). - - -cet_compact_doc() -> - {ok, Engine, Path, St1} = test_engine_util:init_engine(dbpath), - Actions = [{create, {<<"foo">>, []}}], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), - Db1 = test_engine_util:db_as_term(Engine, St2), - {ok, St3, DbName, _, Term} = test_engine_util:compact(Engine, St2, Path), - {ok, St4, undefined} = Engine:finish_compaction(St3, DbName, [], Term), - Db2 = test_engine_util:db_as_term(Engine, St4), - Diff = test_engine_util:term_diff(Db1, Db2), - ?assertEqual(nodiff, Diff). - - -cet_compact_local_doc() -> - {ok, Engine, Path, St1} = test_engine_util:init_engine(dbpath), - Actions = [{create, {<<"_local/foo">>, []}}], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), - Db1 = test_engine_util:db_as_term(Engine, St2), - {ok, St3, DbName, _, Term} = test_engine_util:compact(Engine, St2, Path), - {ok, St4, undefined} = Engine:finish_compaction(St3, DbName, [], Term), - Db2 = test_engine_util:db_as_term(Engine, St4), - Diff = test_engine_util:term_diff(Db1, Db2), - ?assertEqual(nodiff, Diff). - - -cet_compact_with_everything() -> - {ok, Engine, Path, St1} = test_engine_util:init_engine(dbpath), - - % Add a whole bunch of docs - DocActions = lists:map(fun(Seq) -> - {create, {docid(Seq), [{<<"int">>, Seq}]}} - end, lists:seq(1, 1000)), - - LocalActions = lists:map(fun(I) -> - {create, {local_docid(I), [{<<"int">>, I}]}} - end, lists:seq(1, 25)), - - Actions1 = DocActions ++ LocalActions, - - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1), - {ok, St3} = Engine:set_security(St2, [{<<"readers">>, <<"ohai">>}]), - {ok, St4} = Engine:set_revs_limit(St3, 500), - - Actions2 = [ - {create, {<<"foo">>, []}}, - {create, {<<"bar">>, [{<<"hooray">>, <<"purple">>}]}}, - {conflict, {<<"bar">>, [{<<"booo">>, false}]}} - ], - - {ok, St5} = test_engine_util:apply_actions(Engine, St4, Actions2), - - [FooFDI, BarFDI] = Engine:open_docs(St5, [<<"foo">>, <<"bar">>]), - - FooRev = test_engine_util:prev_rev(FooFDI), - BarRev = test_engine_util:prev_rev(BarFDI), - - Actions3 = [ - {batch, [ - {purge, {<<"foo">>, FooRev#rev_info.rev}}, - {purge, {<<"bar">>, BarRev#rev_info.rev}} - ]} - ], - - {ok, St6} = test_engine_util:apply_actions(Engine, St5, Actions3), - - PurgedIdRevs = [ - {<<"bar">>, [BarRev#rev_info.rev]}, - {<<"foo">>, [FooRev#rev_info.rev]} - ], - - ?assertEqual(PurgedIdRevs, lists:sort(Engine:get_last_purged(St6))), - - {ok, St7} = try - [Att0, Att1, Att2, Att3, Att4] = test_engine_util:prep_atts(Engine, St6, [ - {<<"ohai.txt">>, crypto:strong_rand_bytes(2048)}, - {<<"stuff.py">>, crypto:strong_rand_bytes(32768)}, - {<<"a.erl">>, crypto:strong_rand_bytes(29)}, - {<<"a.hrl">>, crypto:strong_rand_bytes(5000)}, - {<<"a.app">>, crypto:strong_rand_bytes(400)} - ]), - - Actions4 = [ - {create, {<<"small_att">>, [], [Att0]}}, - {create, {<<"large_att">>, [], [Att1]}}, - {create, {<<"multi_att">>, [], [Att2, Att3, Att4]}} - ], - test_engine_util:apply_actions(Engine, St6, Actions4) - catch throw:not_supported -> - {ok, St6} - end, - {ok, St8} = Engine:commit_data(St7), - - Db1 = test_engine_util:db_as_term(Engine, St8), - - Config = [ - {"database_compaction", "doc_buffer_size", "1024"}, - {"database_compaction", "checkpoint_after", "2048"} - ], - - {ok, St9, DbName, _, Term} = test_engine_util:with_config(Config, fun() -> - test_engine_util:compact(Engine, St8, Path) - end), - - {ok, St10, undefined} = Engine:finish_compaction(St9, DbName, [], Term), - Db2 = test_engine_util:db_as_term(Engine, St10), - Diff = test_engine_util:term_diff(Db1, Db2), - ?assertEqual(nodiff, Diff). - - -cet_recompact_updates() -> - {ok, Engine, Path, St1} = test_engine_util:init_engine(dbpath), - - Actions1 = [ - {create, {<<"foo">>, []}}, - {create, {<<"bar">>, []}} - ], - - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1), - {ok, St3, DbName, _, Term} = test_engine_util:compact(Engine, St2, Path), - - Actions2 = [ - {update, {<<"foo">>, [{<<"updated">>, true}]}}, - {create, {<<"baz">>, []}} - ], - - {ok, St4} = test_engine_util:apply_actions(Engine, St3, Actions2), - Db1 = test_engine_util:db_as_term(Engine, St4), - - {ok, St5, NewPid} = Engine:finish_compaction(St4, DbName, [], Term), - - ?assertEqual(true, is_pid(NewPid)), - Ref = erlang:monitor(process, NewPid), - - NewTerm = receive - {'$gen_cast', {compact_done, Engine, Term0}} -> - Term0; - {'DOWN', Ref, _, _, Reason} -> - erlang:error({compactor_died, Reason}) - after 10000 -> - erlang:error(compactor_timed_out) - end, - - {ok, St6, undefined} = Engine:finish_compaction(St5, DbName, [], NewTerm), - Db2 = test_engine_util:db_as_term(Engine, St6), - Diff = test_engine_util:term_diff(Db1, Db2), - ?assertEqual(nodiff, Diff). - - -docid(I) -> - Str = io_lib:format("~4..0b", [I]), - iolist_to_binary(Str). - - -local_docid(I) -> - Str = io_lib:format("_local/~4..0b", [I]), - iolist_to_binary(Str). diff --git a/src/couch/src/test_engine_fold_changes.erl b/src/couch/src/test_engine_fold_changes.erl deleted file mode 100644 index 6e97fda9b..000000000 --- a/src/couch/src/test_engine_fold_changes.erl +++ /dev/null @@ -1,190 +0,0 @@ -% Licensed under the Apache License, Version 2.0 (the "License"); you may not -% use this file except in compliance with the License. You may obtain a copy of -% the License at -% -% http://www.apache.org/licenses/LICENSE-2.0 -% -% Unless required by applicable law or agreed to in writing, software -% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -% License for the specific language governing permissions and limitations under -% the License. - --module(test_engine_fold_changes). --compile(export_all). - - --include_lib("eunit/include/eunit.hrl"). --include_lib("couch/include/couch_db.hrl"). - - --define(NUM_DOCS, 100). - - -cet_empty_changes() -> - {ok, Engine, St} = test_engine_util:init_engine(), - - ?assertEqual(0, Engine:count_changes_since(St, 0)), - ?assertEqual({ok, []}, Engine:fold_changes(St, 0, fun fold_fun/2, [], [])). - - -cet_single_change() -> - {ok, Engine, St1} = test_engine_util:init_engine(), - Actions = [{create, {<<"a">>, []}}], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), - - ?assertEqual(1, Engine:count_changes_since(St2, 0)), - ?assertEqual({ok, [{<<"a">>, 1}]}, - Engine:fold_changes(St2, 0, fun fold_fun/2, [], [])). - - -cet_two_changes() -> - {ok, Engine, St1} = test_engine_util:init_engine(), - Actions = [ - {create, {<<"a">>, []}}, - {create, {<<"b">>, []}} - ], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), - - ?assertEqual(2, Engine:count_changes_since(St2, 0)), - {ok, Changes} = Engine:fold_changes(St2, 0, fun fold_fun/2, [], []), - ?assertEqual([{<<"a">>, 1}, {<<"b">>, 2}], lists:reverse(Changes)). - - -cet_two_changes_batch() -> - {ok, Engine, St1} = test_engine_util:init_engine(), - Actions1 = [ - {batch, [ - {create, {<<"a">>, []}}, - {create, {<<"b">>, []}} - ]} - ], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1), - - ?assertEqual(2, Engine:count_changes_since(St2, 0)), - {ok, Changes1} = Engine:fold_changes(St2, 0, fun fold_fun/2, [], []), - ?assertEqual([{<<"a">>, 1}, {<<"b">>, 2}], lists:reverse(Changes1)), - - {ok, Engine, St3} = test_engine_util:init_engine(), - Actions2 = [ - {batch, [ - {create, {<<"b">>, []}}, - {create, {<<"a">>, []}} - ]} - ], - {ok, St4} = test_engine_util:apply_actions(Engine, St3, Actions2), - - ?assertEqual(2, Engine:count_changes_since(St4, 0)), - {ok, Changes2} = Engine:fold_changes(St4, 0, fun fold_fun/2, [], []), - ?assertEqual([{<<"b">>, 1}, {<<"a">>, 2}], lists:reverse(Changes2)). - - -cet_update_one() -> - {ok, Engine, St1} = test_engine_util:init_engine(), - Actions = [ - {create, {<<"a">>, []}}, - {update, {<<"a">>, []}} - ], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), - - ?assertEqual(1, Engine:count_changes_since(St2, 0)), - ?assertEqual({ok, [{<<"a">>, 2}]}, - Engine:fold_changes(St2, 0, fun fold_fun/2, [], [])). - - -cet_update_first_of_two() -> - {ok, Engine, St1} = test_engine_util:init_engine(), - Actions = [ - {create, {<<"a">>, []}}, - {create, {<<"b">>, []}}, - {update, {<<"a">>, []}} - ], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), - - ?assertEqual(2, Engine:count_changes_since(St2, 0)), - {ok, Changes} = Engine:fold_changes(St2, 0, fun fold_fun/2, [], []), - ?assertEqual([{<<"b">>, 2}, {<<"a">>, 3}], lists:reverse(Changes)). - - -cet_update_second_of_two() -> - {ok, Engine, St1} = test_engine_util:init_engine(), - Actions = [ - {create, {<<"a">>, []}}, - {create, {<<"b">>, []}}, - {update, {<<"b">>, []}} - ], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), - - ?assertEqual(2, Engine:count_changes_since(St2, 0)), - {ok, Changes} = Engine:fold_changes(St2, 0, fun fold_fun/2, [], []), - ?assertEqual([{<<"a">>, 1}, {<<"b">>, 3}], lists:reverse(Changes)). - - -cet_check_mutation_ordering() -> - Actions = shuffle(lists:map(fun(Seq) -> - {create, {docid(Seq), []}} - end, lists:seq(1, ?NUM_DOCS))), - - DocIdOrder = [DocId || {_, {DocId, _}} <- Actions], - DocSeqs = lists:zip(DocIdOrder, lists:seq(1, ?NUM_DOCS)), - - {ok, Engine, St1} = test_engine_util:init_engine(), - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), - - % First lets see that we can get the correct - % suffix/prefix starting at every update sequence - lists:foreach(fun(Seq) -> - {ok, Suffix} = Engine:fold_changes(St2, Seq, fun fold_fun/2, [], []), - ?assertEqual(lists:nthtail(Seq, DocSeqs), lists:reverse(Suffix)), - - {ok, Prefix} = Engine:fold_changes(St2, Seq, fun fold_fun/2, [], [ - {dir, rev} - ]), - ?assertEqual(lists:sublist(DocSeqs, Seq + 1), Prefix) - end, lists:seq(0, ?NUM_DOCS)), - - ok = do_mutation_ordering(Engine, St2, ?NUM_DOCS + 1, DocSeqs, []). - - -do_mutation_ordering(Engine, St, _Seq, [], FinalDocSeqs) -> - {ok, RevOrder} = Engine:fold_changes(St, 0, fun fold_fun/2, [], []), - ?assertEqual(FinalDocSeqs, lists:reverse(RevOrder)), - ok; - -do_mutation_ordering(Engine, St, Seq, [{DocId, _OldSeq} | Rest], DocSeqAcc) -> - Actions = [{update, {DocId, []}}], - {ok, NewSt} = test_engine_util:apply_actions(Engine, St, Actions), - NewAcc = DocSeqAcc ++ [{DocId, Seq}], - Expected = Rest ++ NewAcc, - {ok, RevOrder} = Engine:fold_changes(NewSt, 0, fun fold_fun/2, [], []), - ?assertEqual(Expected, lists:reverse(RevOrder)), - do_mutation_ordering(Engine, NewSt, Seq + 1, Rest, NewAcc). - - -shuffle(List) -> - random:seed(os:timestamp()), - Paired = [{random:uniform(), I} || I <- List], - Sorted = lists:sort(Paired), - [I || {_, I} <- Sorted]. - - -remove_random(List) -> - Pos = random:uniform(length(List)), - remove_random(Pos, List). - - -remove_random(1, [Item | Rest]) -> - {Item, Rest}; - -remove_random(N, [Skip | Rest]) when N > 1 -> - {Item, Tail} = remove_random(N - 1, Rest), - {Item, [Skip | Tail]}. - - -fold_fun(#full_doc_info{id=Id, update_seq=Seq}, Acc) -> - {ok, [{Id, Seq} | Acc]}. - - -docid(I) -> - Str = io_lib:format("~4..0b", [I]), - iolist_to_binary(Str). diff --git a/src/couch/src/test_engine_get_set_props.erl b/src/couch/src/test_engine_get_set_props.erl deleted file mode 100644 index 6d2a44779..000000000 --- a/src/couch/src/test_engine_get_set_props.erl +++ /dev/null @@ -1,70 +0,0 @@ -% Licensed under the Apache License, Version 2.0 (the "License"); you may not -% use this file except in compliance with the License. You may obtain a copy of -% the License at -% -% http://www.apache.org/licenses/LICENSE-2.0 -% -% Unless required by applicable law or agreed to in writing, software -% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -% License for the specific language governing permissions and limitations under -% the License. - --module(test_engine_get_set_props). --compile(export_all). - - --include_lib("eunit/include/eunit.hrl"). - - -cet_default_props() -> - Engine = test_engine_util:get_engine(), - DbPath = test_engine_util:dbpath(), - - {ok, St} = Engine:init(DbPath, [ - create, - {default_security_object, dso} - ]), - - Node = node(), - - ?assertEqual(0, Engine:get_doc_count(St)), - ?assertEqual(0, Engine:get_del_doc_count(St)), - ?assertEqual(true, is_list(Engine:get_size_info(St))), - ?assertEqual(true, is_integer(Engine:get_disk_version(St))), - ?assertEqual(0, Engine:get_update_seq(St)), - ?assertEqual(0, Engine:get_purge_seq(St)), - ?assertEqual([], Engine:get_last_purged(St)), - ?assertEqual(dso, Engine:get_security(St)), - ?assertEqual(1000, Engine:get_revs_limit(St)), - ?assertMatch(<<_:32/binary>>, Engine:get_uuid(St)), - ?assertEqual([{Node, 0}], Engine:get_epochs(St)), - ?assertEqual(0, Engine:get_compacted_seq(St)). - - -cet_set_security() -> - check_prop_set(get_security, set_security, dso, [{<<"readers">>, []}]). - - -cet_set_revs_limit() -> - check_prop_set(get_revs_limit, set_revs_limit, 1000, 50). - - -check_prop_set(GetFun, SetFun, Default, Value) -> - Engine = test_engine_util:get_engine(), - DbPath = test_engine_util:dbpath(), - - {ok, St0} = Engine:init(DbPath, [ - create, - {default_security_object, dso} - ]), - ?assertEqual(Default, Engine:GetFun(St0)), - - {ok, St1} = Engine:SetFun(St0, Value), - ?assertEqual(Value, Engine:GetFun(St1)), - - {ok, St2} = Engine:commit_data(St1), - Engine:terminate(normal, St2), - - {ok, St3} = Engine:init(DbPath, []), - ?assertEqual(Value, Engine:GetFun(St3)). diff --git a/src/couch/src/test_engine_open_close_delete.erl b/src/couch/src/test_engine_open_close_delete.erl deleted file mode 100644 index b099d9fb0..000000000 --- a/src/couch/src/test_engine_open_close_delete.erl +++ /dev/null @@ -1,81 +0,0 @@ -% Licensed under the Apache License, Version 2.0 (the "License"); you may not -% use this file except in compliance with the License. You may obtain a copy of -% the License at -% -% http://www.apache.org/licenses/LICENSE-2.0 -% -% Unless required by applicable law or agreed to in writing, software -% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -% License for the specific language governing permissions and limitations under -% the License. - --module(test_engine_open_close_delete). --compile(export_all). - - --include_lib("eunit/include/eunit.hrl"). - - -cet_open_non_existent() -> - Engine = test_engine_util:get_engine(), - DbPath = test_engine_util:dbpath(), - - ?assertEqual(false, Engine:exists(DbPath)), - ?assertThrow({not_found, no_db_file}, Engine:init(DbPath, [])), - ?assertEqual(false, Engine:exists(DbPath)). - - -cet_open_create() -> - process_flag(trap_exit, true), - Engine = test_engine_util:get_engine(), - DbPath = test_engine_util:dbpath(), - - ?assertEqual(false, Engine:exists(DbPath)), - ?assertMatch({ok, _}, Engine:init(DbPath, [create])), - ?assertEqual(true, Engine:exists(DbPath)). - - -cet_open_when_exists() -> - Engine = test_engine_util:get_engine(), - DbPath = test_engine_util:dbpath(), - - ?assertEqual(false, Engine:exists(DbPath)), - ?assertMatch({ok, _}, Engine:init(DbPath, [create])), - ?assertThrow({error, eexist}, Engine:init(DbPath, [create])). - - -cet_terminate() -> - Engine = test_engine_util:get_engine(), - DbPath = test_engine_util:dbpath(), - - ?assertEqual(false, Engine:exists(DbPath)), - {ok, St} = Engine:init(DbPath, [create]), - Engine:terminate(normal, St), - ?assertEqual(true, Engine:exists(DbPath)). - - -cet_rapid_recycle() -> - Engine = test_engine_util:get_engine(), - DbPath = test_engine_util:dbpath(), - - {ok, St0} = Engine:init(DbPath, [create]), - Engine:terminate(normal, St0), - - lists:foreach(fun(_) -> - {ok, St1} = Engine:init(DbPath, []), - Engine:terminate(normal, St1) - end, lists:seq(1, 100)). - - -cet_delete() -> - Engine = test_engine_util:get_engine(), - RootDir = test_engine_util:rootdir(), - DbPath = test_engine_util:dbpath(), - - ?assertEqual(false, Engine:exists(DbPath)), - {ok, St} = Engine:init(DbPath, [create]), - Engine:terminate(normal, St), - ?assertEqual(true, Engine:exists(DbPath)), - ?assertEqual(ok, Engine:delete(RootDir, DbPath, [async])), - ?assertEqual(false, Engine:exists(DbPath)). diff --git a/src/couch/src/test_engine_purge_docs.erl b/src/couch/src/test_engine_purge_docs.erl deleted file mode 100644 index e5bf249eb..000000000 --- a/src/couch/src/test_engine_purge_docs.erl +++ /dev/null @@ -1,158 +0,0 @@ -% Licensed under the Apache License, Version 2.0 (the "License"); you may not -% use this file except in compliance with the License. You may obtain a copy of -% the License at -% -% http://www.apache.org/licenses/LICENSE-2.0 -% -% Unless required by applicable law or agreed to in writing, software -% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -% License for the specific language governing permissions and limitations under -% the License. - --module(test_engine_purge_docs). --compile(export_all). - - --include_lib("eunit/include/eunit.hrl"). --include_lib("couch/include/couch_db.hrl"). - - -cet_purge_simple() -> - {ok, Engine, St1} = test_engine_util:init_engine(), - - Actions1 = [ - {create, {<<"foo">>, [{<<"vsn">>, 1}]}} - ], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1), - - ?assertEqual(1, Engine:get_doc_count(St2)), - ?assertEqual(0, Engine:get_del_doc_count(St2)), - ?assertEqual(1, Engine:get_update_seq(St2)), - ?assertEqual(0, Engine:get_purge_seq(St2)), - ?assertEqual([], Engine:get_last_purged(St2)), - - [FDI] = Engine:open_docs(St2, [<<"foo">>]), - PrevRev = test_engine_util:prev_rev(FDI), - Rev = PrevRev#rev_info.rev, - - Actions2 = [ - {purge, {<<"foo">>, Rev}} - ], - {ok, St3} = test_engine_util:apply_actions(Engine, St2, Actions2), - - ?assertEqual(0, Engine:get_doc_count(St3)), - ?assertEqual(0, Engine:get_del_doc_count(St3)), - ?assertEqual(2, Engine:get_update_seq(St3)), - ?assertEqual(1, Engine:get_purge_seq(St3)), - ?assertEqual([{<<"foo">>, [Rev]}], Engine:get_last_purged(St3)). - - -cet_purge_conflicts() -> - {ok, Engine, St1} = test_engine_util:init_engine(), - - Actions1 = [ - {create, {<<"foo">>, [{<<"vsn">>, 1}]}}, - {conflict, {<<"foo">>, [{<<"vsn">>, 2}]}} - ], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1), - - ?assertEqual(1, Engine:get_doc_count(St2)), - ?assertEqual(0, Engine:get_del_doc_count(St2)), - ?assertEqual(2, Engine:get_update_seq(St2)), - ?assertEqual(0, Engine:get_purge_seq(St2)), - ?assertEqual([], Engine:get_last_purged(St2)), - - [FDI1] = Engine:open_docs(St2, [<<"foo">>]), - PrevRev1 = test_engine_util:prev_rev(FDI1), - Rev1 = PrevRev1#rev_info.rev, - - Actions2 = [ - {purge, {<<"foo">>, Rev1}} - ], - {ok, St3} = test_engine_util:apply_actions(Engine, St2, Actions2), - - ?assertEqual(1, Engine:get_doc_count(St3)), - ?assertEqual(0, Engine:get_del_doc_count(St3)), - ?assertEqual(4, Engine:get_update_seq(St3)), - ?assertEqual(1, Engine:get_purge_seq(St3)), - ?assertEqual([{<<"foo">>, [Rev1]}], Engine:get_last_purged(St3)), - - [FDI2] = Engine:open_docs(St3, [<<"foo">>]), - PrevRev2 = test_engine_util:prev_rev(FDI2), - Rev2 = PrevRev2#rev_info.rev, - - Actions3 = [ - {purge, {<<"foo">>, Rev2}} - ], - {ok, St4} = test_engine_util:apply_actions(Engine, St3, Actions3), - - ?assertEqual(0, Engine:get_doc_count(St4)), - ?assertEqual(0, Engine:get_del_doc_count(St4)), - ?assertEqual(5, Engine:get_update_seq(St4)), - ?assertEqual(2, Engine:get_purge_seq(St4)), - ?assertEqual([{<<"foo">>, [Rev2]}], Engine:get_last_purged(St4)). - - -cet_add_delete_purge() -> - {ok, Engine, St1} = test_engine_util:init_engine(), - - Actions1 = [ - {create, {<<"foo">>, [{<<"vsn">>, 1}]}}, - {delete, {<<"foo">>, [{<<"vsn">>, 2}]}} - ], - - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1), - - ?assertEqual(0, Engine:get_doc_count(St2)), - ?assertEqual(1, Engine:get_del_doc_count(St2)), - ?assertEqual(2, Engine:get_update_seq(St2)), - ?assertEqual(0, Engine:get_purge_seq(St2)), - ?assertEqual([], Engine:get_last_purged(St2)), - - [FDI] = Engine:open_docs(St2, [<<"foo">>]), - PrevRev = test_engine_util:prev_rev(FDI), - Rev = PrevRev#rev_info.rev, - - Actions2 = [ - {purge, {<<"foo">>, Rev}} - ], - {ok, St3} = test_engine_util:apply_actions(Engine, St2, Actions2), - - ?assertEqual(0, Engine:get_doc_count(St3)), - ?assertEqual(0, Engine:get_del_doc_count(St3)), - ?assertEqual(3, Engine:get_update_seq(St3)), - ?assertEqual(1, Engine:get_purge_seq(St3)), - ?assertEqual([{<<"foo">>, [Rev]}], Engine:get_last_purged(St3)). - - -cet_add_two_purge_one() -> - {ok, Engine, St1} = test_engine_util:init_engine(), - - Actions1 = [ - {create, {<<"foo">>, [{<<"vsn">>, 1}]}}, - {create, {<<"bar">>, []}} - ], - - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1), - - ?assertEqual(2, Engine:get_doc_count(St2)), - ?assertEqual(0, Engine:get_del_doc_count(St2)), - ?assertEqual(2, Engine:get_update_seq(St2)), - ?assertEqual(0, Engine:get_purge_seq(St2)), - ?assertEqual([], Engine:get_last_purged(St2)), - - [FDI] = Engine:open_docs(St2, [<<"foo">>]), - PrevRev = test_engine_util:prev_rev(FDI), - Rev = PrevRev#rev_info.rev, - - Actions2 = [ - {purge, {<<"foo">>, Rev}} - ], - {ok, St3} = test_engine_util:apply_actions(Engine, St2, Actions2), - - ?assertEqual(1, Engine:get_doc_count(St3)), - ?assertEqual(0, Engine:get_del_doc_count(St3)), - ?assertEqual(3, Engine:get_update_seq(St3)), - ?assertEqual(1, Engine:get_purge_seq(St3)), - ?assertEqual([{<<"foo">>, [Rev]}], Engine:get_last_purged(St3)). diff --git a/src/couch/src/test_engine_read_write_docs.erl b/src/couch/src/test_engine_read_write_docs.erl deleted file mode 100644 index 4307702d4..000000000 --- a/src/couch/src/test_engine_read_write_docs.erl +++ /dev/null @@ -1,317 +0,0 @@ -% Licensed under the Apache License, Version 2.0 (the "License"); you may not -% use this file except in compliance with the License. You may obtain a copy of -% the License at -% -% http://www.apache.org/licenses/LICENSE-2.0 -% -% Unless required by applicable law or agreed to in writing, software -% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -% License for the specific language governing permissions and limitations under -% the License. - --module(test_engine_read_write_docs). --compile(export_all). - - --include_lib("eunit/include/eunit.hrl"). --include_lib("couch/include/couch_db.hrl"). - - -cet_read_empty_docs() -> - {ok, Engine, St} = test_engine_util:init_engine(), - - ?assertEqual([not_found], Engine:open_docs(St, [<<"foo">>])), - ?assertEqual( - [not_found, not_found], - Engine:open_docs(St, [<<"a">>, <<"b">>]) - ). - - -cet_read_empty_local_docs() -> - {ok, Engine, St} = test_engine_util:init_engine(), - - ?assertEqual([not_found], Engine:open_local_docs(St, [<<"_local/foo">>])), - ?assertEqual( - [not_found, not_found], - Engine:open_local_docs(St, [<<"_local/a">>, <<"_local/b">>]) - ). - - -cet_write_one_doc() -> - {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), - - ?assertEqual(0, Engine:get_doc_count(St1)), - ?assertEqual(0, Engine:get_del_doc_count(St1)), - ?assertEqual(0, Engine:get_update_seq(St1)), - - Actions = [ - {create, {<<"foo">>, [{<<"vsn">>, 1}]}} - ], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), - {ok, St3} = Engine:commit_data(St2), - Engine:terminate(normal, St3), - {ok, St4} = Engine:init(DbPath, []), - - ?assertEqual(1, Engine:get_doc_count(St4)), - ?assertEqual(0, Engine:get_del_doc_count(St4)), - ?assertEqual(1, Engine:get_update_seq(St4)), - - [FDI] = Engine:open_docs(St4, [<<"foo">>]), - #rev_info{ - rev = {RevPos, PrevRevId}, - deleted = Deleted, - body_sp = DocPtr - } = test_engine_util:prev_rev(FDI), - - Doc0 = #doc{ - id = <<"foo">>, - revs = {RevPos, [PrevRevId]}, - deleted = Deleted, - body = DocPtr - }, - - Doc1 = Engine:read_doc_body(St4, Doc0), - Body1 = if not is_binary(Doc1#doc.body) -> Doc1#doc.body; true -> - couch_compress:decompress(Doc1#doc.body) - end, - ?assertEqual([{<<"vsn">>, 1}], Body1). - - -cet_write_two_docs() -> - {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), - - ?assertEqual(0, Engine:get_doc_count(St1)), - ?assertEqual(0, Engine:get_del_doc_count(St1)), - ?assertEqual(0, Engine:get_update_seq(St1)), - - Actions = [ - {create, {<<"foo">>, [{<<"vsn">>, 1}]}}, - {create, {<<"bar">>, [{<<"stuff">>, true}]}} - ], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), - {ok, St3} = Engine:commit_data(St2), - Engine:terminate(normal, St3), - {ok, St4} = Engine:init(DbPath, []), - - ?assertEqual(2, Engine:get_doc_count(St4)), - ?assertEqual(0, Engine:get_del_doc_count(St4)), - ?assertEqual(2, Engine:get_update_seq(St4)), - - Resps = Engine:open_docs(St4, [<<"foo">>, <<"bar">>]), - ?assertEqual(false, lists:member(not_found, Resps)). - - -cet_write_three_doc_batch() -> - {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), - - ?assertEqual(0, Engine:get_doc_count(St1)), - ?assertEqual(0, Engine:get_del_doc_count(St1)), - ?assertEqual(0, Engine:get_update_seq(St1)), - - Actions = [ - {batch, [ - {create, {<<"foo">>, [{<<"vsn">>, 1}]}}, - {create, {<<"bar">>, [{<<"stuff">>, true}]}}, - {create, {<<"baz">>, []}} - ]} - ], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), - {ok, St3} = Engine:commit_data(St2), - Engine:terminate(normal, St3), - {ok, St4} = Engine:init(DbPath, []), - - ?assertEqual(3, Engine:get_doc_count(St4)), - ?assertEqual(0, Engine:get_del_doc_count(St4)), - ?assertEqual(3, Engine:get_update_seq(St4)), - - Resps = Engine:open_docs(St4, [<<"foo">>, <<"bar">>, <<"baz">>]), - ?assertEqual(false, lists:member(not_found, Resps)). - - -cet_update_doc() -> - {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), - - ?assertEqual(0, Engine:get_doc_count(St1)), - ?assertEqual(0, Engine:get_del_doc_count(St1)), - ?assertEqual(0, Engine:get_update_seq(St1)), - - Actions = [ - {create, {<<"foo">>, [{<<"vsn">>, 1}]}}, - {update, {<<"foo">>, [{<<"vsn">>, 2}]}} - ], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), - {ok, St3} = Engine:commit_data(St2), - Engine:terminate(normal, St3), - {ok, St4} = Engine:init(DbPath, []), - - ?assertEqual(1, Engine:get_doc_count(St4)), - ?assertEqual(0, Engine:get_del_doc_count(St4)), - ?assertEqual(2, Engine:get_update_seq(St4)), - - [FDI] = Engine:open_docs(St4, [<<"foo">>]), - - #rev_info{ - rev = {RevPos, PrevRevId}, - deleted = Deleted, - body_sp = DocPtr - } = test_engine_util:prev_rev(FDI), - - Doc0 = #doc{ - id = <<"foo">>, - revs = {RevPos, [PrevRevId]}, - deleted = Deleted, - body = DocPtr - }, - - Doc1 = Engine:read_doc_body(St4, Doc0), - Body1 = if not is_binary(Doc1#doc.body) -> Doc1#doc.body; true -> - couch_compress:decompress(Doc1#doc.body) - end, - - ?assertEqual([{<<"vsn">>, 2}], Body1). - - -cet_delete_doc() -> - {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), - - ?assertEqual(0, Engine:get_doc_count(St1)), - ?assertEqual(0, Engine:get_del_doc_count(St1)), - ?assertEqual(0, Engine:get_update_seq(St1)), - - Actions = [ - {create, {<<"foo">>, [{<<"vsn">>, 1}]}}, - {delete, {<<"foo">>, []}} - ], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), - {ok, St3} = Engine:commit_data(St2), - Engine:terminate(normal, St3), - {ok, St4} = Engine:init(DbPath, []), - - ?assertEqual(0, Engine:get_doc_count(St4)), - ?assertEqual(1, Engine:get_del_doc_count(St4)), - ?assertEqual(2, Engine:get_update_seq(St4)), - - [FDI] = Engine:open_docs(St4, [<<"foo">>]), - - #rev_info{ - rev = {RevPos, PrevRevId}, - deleted = Deleted, - body_sp = DocPtr - } = test_engine_util:prev_rev(FDI), - - Doc0 = #doc{ - id = <<"foo">>, - revs = {RevPos, [PrevRevId]}, - deleted = Deleted, - body = DocPtr - }, - - Doc1 = Engine:read_doc_body(St4, Doc0), - Body1 = if not is_binary(Doc1#doc.body) -> Doc1#doc.body; true -> - couch_compress:decompress(Doc1#doc.body) - end, - - ?assertEqual([], Body1). - - -cet_write_local_doc() -> - {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), - - ?assertEqual(0, Engine:get_doc_count(St1)), - ?assertEqual(0, Engine:get_del_doc_count(St1)), - ?assertEqual(0, Engine:get_update_seq(St1)), - - Actions = [ - {create, {<<"_local/foo">>, [{<<"yay">>, false}]}} - ], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), - {ok, St3} = Engine:commit_data(St2), - Engine:terminate(normal, St3), - {ok, St4} = Engine:init(DbPath, []), - - ?assertEqual(0, Engine:get_doc_count(St4)), - ?assertEqual(0, Engine:get_del_doc_count(St4)), - ?assertEqual(0, Engine:get_update_seq(St4)), - - [not_found] = Engine:open_docs(St4, [<<"_local/foo">>]), - [#doc{} = Doc] = Engine:open_local_docs(St4, [<<"_local/foo">>]), - ?assertEqual([{<<"yay">>, false}], Doc#doc.body). - - -cet_write_mixed_batch() -> - {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), - - ?assertEqual(0, Engine:get_doc_count(St1)), - ?assertEqual(0, Engine:get_del_doc_count(St1)), - ?assertEqual(0, Engine:get_update_seq(St1)), - - Actions = [ - {batch, [ - {create, {<<"bar">>, []}}, - {create, {<<"_local/foo">>, [{<<"yay">>, false}]}} - ]} - ], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), - {ok, St3} = Engine:commit_data(St2), - Engine:terminate(normal, St3), - {ok, St4} = Engine:init(DbPath, []), - - ?assertEqual(1, Engine:get_doc_count(St4)), - ?assertEqual(0, Engine:get_del_doc_count(St4)), - ?assertEqual(1, Engine:get_update_seq(St4)), - - [#full_doc_info{}] = Engine:open_docs(St4, [<<"bar">>]), - [not_found] = Engine:open_docs(St4, [<<"_local/foo">>]), - - [not_found] = Engine:open_local_docs(St4, [<<"bar">>]), - [#doc{}] = Engine:open_local_docs(St4, [<<"_local/foo">>]). - - -cet_update_local_doc() -> - {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), - - ?assertEqual(0, Engine:get_doc_count(St1)), - ?assertEqual(0, Engine:get_del_doc_count(St1)), - ?assertEqual(0, Engine:get_update_seq(St1)), - - Actions = [ - {create, {<<"_local/foo">>, []}}, - {update, {<<"_local/foo">>, [{<<"stuff">>, null}]}} - ], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), - {ok, St3} = Engine:commit_data(St2), - Engine:terminate(normal, St3), - {ok, St4} = Engine:init(DbPath, []), - - ?assertEqual(0, Engine:get_doc_count(St4)), - ?assertEqual(0, Engine:get_del_doc_count(St4)), - ?assertEqual(0, Engine:get_update_seq(St4)), - - [not_found] = Engine:open_docs(St4, [<<"_local/foo">>]), - [#doc{} = Doc] = Engine:open_local_docs(St4, [<<"_local/foo">>]), - ?assertEqual([{<<"stuff">>, null}], Doc#doc.body). - - -cet_delete_local_doc() -> - {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), - - ?assertEqual(0, Engine:get_doc_count(St1)), - ?assertEqual(0, Engine:get_del_doc_count(St1)), - ?assertEqual(0, Engine:get_update_seq(St1)), - - Actions = [ - {create, {<<"_local/foo">>, []}}, - {delete, {<<"_local/foo">>, []}} - ], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), - {ok, St3} = Engine:commit_data(St2), - Engine:terminate(normal, St3), - {ok, St4} = Engine:init(DbPath, []), - - ?assertEqual(0, Engine:get_doc_count(St4)), - ?assertEqual(0, Engine:get_del_doc_count(St4)), - ?assertEqual(0, Engine:get_update_seq(St4)), - - [not_found] = Engine:open_docs(St4, [<<"_local/foo">>]), - ?assertEqual([not_found], Engine:open_local_docs(St4, [<<"_local/foo">>])). diff --git a/src/couch/src/test_engine_util.erl b/src/couch/src/test_engine_util.erl deleted file mode 100644 index 6cc6bccdc..000000000 --- a/src/couch/src/test_engine_util.erl +++ /dev/null @@ -1,608 +0,0 @@ -% Licensed under the Apache License, Version 2.0 (the "License"); you may not -% use this file except in compliance with the License. You may obtain a copy of -% the License at -% -% http://www.apache.org/licenses/LICENSE-2.0 -% -% Unless required by applicable law or agreed to in writing, software -% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -% License for the specific language governing permissions and limitations under -% the License. - --module(test_engine_util). --compile(export_all). - - --include_lib("couch/include/couch_db.hrl"). - - --define(TEST_MODULES, [ - test_engine_open_close_delete, - test_engine_get_set_props, - test_engine_read_write_docs, - test_engine_attachments, - test_engine_fold_docs, - test_engine_fold_changes, - test_engine_purge_docs, - test_engine_compaction, - test_engine_ref_counting -]). - --define(COMPACTOR_TIMEOUT, 50000). --define(ATTACHMENT_WRITE_TIMEOUT, 10000). --define(MAKE_DOC_SUMMARY_TIMEOUT, 5000). - -create_tests(EngineApp) -> - create_tests(EngineApp, EngineApp). - - -create_tests(EngineApp, EngineModule) -> - application:set_env(couch, test_engine, {EngineApp, EngineModule}), - Tests = lists:map(fun(TestMod) -> - {atom_to_list(TestMod), gather(TestMod)} - end, ?TEST_MODULES), - Setup = fun() -> - Ctx = test_util:start_couch(), - config:set("log", "include_sasl", "false", false), - Ctx - end, - { - setup, - Setup, - fun test_util:stop_couch/1, - fun(_) -> Tests end - }. - - -gather(Module) -> - Exports = Module:module_info(exports), - Tests = lists:foldl(fun({Fun, Arity}, Acc) -> - case {atom_to_list(Fun), Arity} of - {[$c, $e, $t, $_ | _], 0} -> - TestFun = make_test_fun(Module, Fun), - [{timeout, 60, {spawn, TestFun}} | Acc]; - _ -> - Acc - end - end, [], Exports), - lists:reverse(Tests). - - -make_test_fun(Module, Fun) -> - Name = lists:flatten(io_lib:format("~s:~s", [Module, Fun])), - Wrapper = fun() -> - process_flag(trap_exit, true), - Module:Fun() - end, - {Name, Wrapper}. - -rootdir() -> - config:get("couchdb", "database_dir", "."). - - -dbpath() -> - binary_to_list(filename:join(rootdir(), couch_uuids:random())). - - -get_engine() -> - case application:get_env(couch, test_engine) of - {ok, {_, Engine}} -> - Engine; - _ -> - couch_bt_engine - end. - - -init_engine() -> - init_engine(default). - - -init_engine(default) -> - Engine = get_engine(), - DbPath = dbpath(), - {ok, St} = Engine:init(DbPath, [ - create, - {default_security_object, []} - ]), - {ok, Engine, St}; - -init_engine(dbpath) -> - Engine = get_engine(), - DbPath = dbpath(), - {ok, St} = Engine:init(DbPath, [ - create, - {default_security_object, []} - ]), - {ok, Engine, DbPath, St}. - - -apply_actions(_Engine, St, []) -> - {ok, St}; - -apply_actions(Engine, St, [Action | Rest]) -> - NewSt = apply_action(Engine, St, Action), - apply_actions(Engine, NewSt, Rest). - - -apply_action(Engine, St, {batch, BatchActions}) -> - apply_batch(Engine, St, BatchActions); - -apply_action(Engine, St, Action) -> - apply_batch(Engine, St, [Action]). - - -apply_batch(Engine, St, Actions) -> - UpdateSeq = Engine:get_update_seq(St) + 1, - AccIn = {UpdateSeq, [], [], []}, - AccOut = lists:foldl(fun(Action, Acc) -> - {SeqAcc, DocAcc, LDocAcc, PurgeAcc} = Acc, - case Action of - {_, {<<"_local/", _/binary>>, _}} -> - LDoc = gen_local_write(Engine, St, Action), - {SeqAcc, DocAcc, [LDoc | LDocAcc], PurgeAcc}; - _ -> - case gen_write(Engine, St, Action, SeqAcc) of - {_OldFDI, _NewFDI} = Pair -> - {SeqAcc + 1, [Pair | DocAcc], LDocAcc, PurgeAcc}; - {Pair, NewSeqAcc, NewPurgeInfo} -> - NewPurgeAcc = [NewPurgeInfo | PurgeAcc], - {NewSeqAcc, [Pair | DocAcc], LDocAcc, NewPurgeAcc} - end - end - end, AccIn, Actions), - {_, Docs0, LDocs, PurgeIdRevs} = AccOut, - Docs = lists:reverse(Docs0), - {ok, NewSt} = Engine:write_doc_infos(St, Docs, LDocs, PurgeIdRevs), - NewSt. - - -gen_local_write(Engine, St, {Action, {DocId, Body}}) -> - PrevRev = case Engine:open_local_docs(St, [DocId]) of - [not_found] -> - 0; - [#doc{revs = {0, []}}] -> - 0; - [#doc{revs = {0, [RevStr | _]}}] -> - binary_to_integer(RevStr) - end, - {RevId, Deleted} = case Action of - Action when Action == create; Action == update -> - {PrevRev + 1, false}; - delete -> - {0, true} - end, - #doc{ - id = DocId, - revs = {0, [RevId]}, - body = Body, - deleted = Deleted - }. - -gen_write(Engine, St, {Action, {DocId, Body}}, UpdateSeq) -> - gen_write(Engine, St, {Action, {DocId, Body, []}}, UpdateSeq); - -gen_write(Engine, St, {create, {DocId, Body, Atts0}}, UpdateSeq) -> - [not_found] = Engine:open_docs(St, [DocId]), - Atts = [couch_att:to_disk_term(Att) || Att <- Atts0], - - Rev = couch_hash:md5_hash(term_to_binary({DocId, Body, Atts})), - - Doc0 = #doc{ - id = DocId, - revs = {0, [Rev]}, - deleted = false, - body = Body, - atts = Atts - }, - - Doc1 = make_doc_summary(Engine, St, Doc0), - {ok, Doc2, Len} = Engine:write_doc_body(St, Doc1), - - Sizes = #size_info{ - active = Len, - external = erlang:external_size(Doc1#doc.body) - }, - - Leaf = #leaf{ - deleted = false, - ptr = Doc2#doc.body, - seq = UpdateSeq, - sizes = Sizes, - atts = Atts - }, - - {not_found, #full_doc_info{ - id = DocId, - deleted = false, - update_seq = UpdateSeq, - rev_tree = [{0, {Rev, Leaf, []}}], - sizes = Sizes - }}; - -gen_write(Engine, St, {purge, {DocId, PrevRevs0, _}}, UpdateSeq) -> - [#full_doc_info{} = PrevFDI] = Engine:open_docs(St, [DocId]), - PrevRevs = if is_list(PrevRevs0) -> PrevRevs0; true -> [PrevRevs0] end, - - #full_doc_info{ - rev_tree = PrevTree - } = PrevFDI, - - {NewTree, RemRevs} = couch_key_tree:remove_leafs(PrevTree, PrevRevs), - RemovedAll = lists:sort(RemRevs) == lists:sort(PrevRevs), - if RemovedAll -> ok; true -> - % If we didn't purge all the requested revisions - % then its a bug in the test. - erlang:error({invalid_purge_test_revs, PrevRevs}) - end, - - case NewTree of - [] -> - % We've completely purged the document - {{PrevFDI, not_found}, UpdateSeq, {DocId, RemRevs}}; - _ -> - % We have to relabel the update_seq of all - % leaves. See couch_db_updater for details. - {NewNewTree, NewUpdateSeq} = couch_key_tree:mapfold(fun - (_RevId, Leaf, leaf, InnerSeqAcc) -> - {Leaf#leaf{seq = InnerSeqAcc}, InnerSeqAcc + 1}; - (_RevId, Value, _Type, InnerSeqAcc) -> - {Value, InnerSeqAcc} - end, UpdateSeq, NewTree), - NewFDI = PrevFDI#full_doc_info{ - update_seq = NewUpdateSeq - 1, - rev_tree = NewNewTree - }, - {{PrevFDI, NewFDI}, NewUpdateSeq, {DocId, RemRevs}} - end; - -gen_write(Engine, St, {Action, {DocId, Body, Atts0}}, UpdateSeq) -> - [#full_doc_info{} = PrevFDI] = Engine:open_docs(St, [DocId]), - Atts = [couch_att:to_disk_term(Att) || Att <- Atts0], - - #full_doc_info{ - id = DocId, - rev_tree = PrevRevTree - } = PrevFDI, - - #rev_info{ - rev = PrevRev - } = prev_rev(PrevFDI), - - {RevPos, PrevRevId} = PrevRev, - - Rev = gen_revision(Action, DocId, PrevRev, Body, Atts), - - Doc0 = #doc{ - id = DocId, - revs = {RevPos + 1, [Rev, PrevRevId]}, - deleted = false, - body = Body, - atts = Atts - }, - - Doc1 = make_doc_summary(Engine, St, Doc0), - {ok, Doc2, Len} = Engine:write_doc_body(St, Doc1), - - Deleted = case Action of - update -> false; - conflict -> false; - delete -> true - end, - - Sizes = #size_info{ - active = Len, - external = erlang:external_size(Doc1#doc.body) - }, - - Leaf = #leaf{ - deleted = Deleted, - ptr = Doc2#doc.body, - seq = UpdateSeq, - sizes = Sizes, - atts = Atts - }, - - Path = gen_path(Action, RevPos, PrevRevId, Rev, Leaf), - RevsLimit = Engine:get_revs_limit(St), - NodeType = case Action of - conflict -> new_branch; - _ -> new_leaf - end, - {MergedTree, NodeType} = couch_key_tree:merge(PrevRevTree, Path), - NewTree = couch_key_tree:stem(MergedTree, RevsLimit), - - NewFDI = PrevFDI#full_doc_info{ - deleted = couch_doc:is_deleted(NewTree), - update_seq = UpdateSeq, - rev_tree = NewTree, - sizes = Sizes - }, - - {PrevFDI, NewFDI}. - - -gen_revision(conflict, DocId, _PrevRev, Body, Atts) -> - couch_hash:md5_hash(term_to_binary({DocId, Body, Atts})); -gen_revision(delete, DocId, PrevRev, Body, Atts) -> - gen_revision(update, DocId, PrevRev, Body, Atts); -gen_revision(update, DocId, PrevRev, Body, Atts) -> - couch_hash:md5_hash(term_to_binary({DocId, PrevRev, Body, Atts})). - - -gen_path(conflict, _RevPos, _PrevRevId, Rev, Leaf) -> - {0, {Rev, Leaf, []}}; -gen_path(delete, RevPos, PrevRevId, Rev, Leaf) -> - gen_path(update, RevPos, PrevRevId, Rev, Leaf); -gen_path(update, RevPos, PrevRevId, Rev, Leaf) -> - {RevPos, {PrevRevId, ?REV_MISSING, [{Rev, Leaf, []}]}}. - - -make_doc_summary(Engine, St, DocData) -> - {_, Ref} = spawn_monitor(fun() -> - exit({result, Engine:serialize_doc(St, DocData)}) - end), - receive - {'DOWN', Ref, _, _, {result, Summary}} -> - Summary; - {'DOWN', Ref, _, _, Error} -> - erlang:error({make_doc_summary_error, Error}) - after ?MAKE_DOC_SUMMARY_TIMEOUT -> - erlang:error(make_doc_summary_timeout) - end. - - -prep_atts(_Engine, _St, []) -> - []; - -prep_atts(Engine, St, [{FileName, Data} | Rest]) -> - {_, Ref} = spawn_monitor(fun() -> - {ok, Stream} = Engine:open_write_stream(St, []), - exit(write_att(Stream, FileName, Data, Data)) - end), - Att = receive - {'DOWN', Ref, _, _, {{no_catch, not_supported}, _}} -> - throw(not_supported); - {'DOWN', Ref, _, _, Resp} -> - Resp - after ?ATTACHMENT_WRITE_TIMEOUT -> - erlang:error(attachment_write_timeout) - end, - [Att | prep_atts(Engine, St, Rest)]. - - -write_att(Stream, FileName, OrigData, <<>>) -> - {StreamEngine, Len, Len, Md5, Md5} = couch_stream:close(Stream), - couch_util:check_md5(Md5, couch_hash:md5_hash(OrigData)), - Len = size(OrigData), - couch_att:new([ - {name, FileName}, - {type, <<"application/octet-stream">>}, - {data, {stream, StreamEngine}}, - {att_len, Len}, - {disk_len, Len}, - {md5, Md5}, - {encoding, identity} - ]); - -write_att(Stream, FileName, OrigData, Data) -> - {Chunk, Rest} = case size(Data) > 4096 of - true -> - <<Head:4096/binary, Tail/binary>> = Data, - {Head, Tail}; - false -> - {Data, <<>>} - end, - ok = couch_stream:write(Stream, Chunk), - write_att(Stream, FileName, OrigData, Rest). - - -prev_rev(#full_doc_info{} = FDI) -> - #doc_info{ - revs = [#rev_info{} = PrevRev | _] - } = couch_doc:to_doc_info(FDI), - PrevRev. - - -db_as_term(Engine, St) -> - [ - {props, db_props_as_term(Engine, St)}, - {docs, db_docs_as_term(Engine, St)}, - {local_docs, db_local_docs_as_term(Engine, St)}, - {changes, db_changes_as_term(Engine, St)} - ]. - - -db_props_as_term(Engine, St) -> - Props = [ - get_doc_count, - get_del_doc_count, - get_disk_version, - get_update_seq, - get_purge_seq, - get_last_purged, - get_security, - get_revs_limit, - get_uuid, - get_epochs - ], - lists:map(fun(Fun) -> - {Fun, Engine:Fun(St)} - end, Props). - - -db_docs_as_term(Engine, St) -> - FoldFun = fun(FDI, Acc) -> {ok, [FDI | Acc]} end, - {ok, FDIs} = Engine:fold_docs(St, FoldFun, [], []), - lists:reverse(lists:map(fun(FDI) -> - fdi_to_term(Engine, St, FDI) - end, FDIs)). - - -db_local_docs_as_term(Engine, St) -> - FoldFun = fun(Doc, Acc) -> {ok, [Doc | Acc]} end, - {ok, LDocs} = Engine:fold_local_docs(St, FoldFun, [], []), - lists:reverse(LDocs). - - -db_changes_as_term(Engine, St) -> - FoldFun = fun(FDI, Acc) -> {ok, [FDI | Acc]} end, - {ok, Changes} = Engine:fold_changes(St, 0, FoldFun, [], []), - lists:reverse(lists:map(fun(FDI) -> - fdi_to_term(Engine, St, FDI) - end, Changes)). - - -fdi_to_term(Engine, St, FDI) -> - #full_doc_info{ - id = DocId, - rev_tree = OldTree - } = FDI, - {NewRevTree, _} = couch_key_tree:mapfold(fun(Rev, Node, Type, Acc) -> - tree_to_term(Rev, Node, Type, Acc, DocId) - end, {Engine, St}, OldTree), - FDI#full_doc_info{ - rev_tree = NewRevTree, - % Blank out sizes because we allow storage - % engines to handle this with their own - % definition until further notice. - sizes = #size_info{ - active = -1, - external = -1 - } - }. - - -tree_to_term(_Rev, _Leaf, branch, Acc, _DocId) -> - {?REV_MISSING, Acc}; - -tree_to_term({Pos, RevId}, #leaf{} = Leaf, leaf, {Engine, St}, DocId) -> - #leaf{ - deleted = Deleted, - ptr = Ptr - } = Leaf, - - Doc0 = #doc{ - id = DocId, - revs = {Pos, [RevId]}, - deleted = Deleted, - body = Ptr - }, - - Doc1 = Engine:read_doc_body(St, Doc0), - - Body = if not is_binary(Doc1#doc.body) -> Doc1#doc.body; true -> - couch_compress:decompress(Doc1#doc.body) - end, - - Atts1 = if not is_binary(Doc1#doc.atts) -> Doc1#doc.atts; true -> - couch_compress:decompress(Doc1#doc.atts) - end, - - StreamSrc = fun(Sp) -> Engine:open_read_stream(St, Sp) end, - Atts2 = [couch_att:from_disk_term(StreamSrc, Att) || Att <- Atts1], - Atts = [att_to_term(Att) || Att <- Atts2], - - NewLeaf = Leaf#leaf{ - ptr = Body, - sizes = #size_info{active = -1, external = -1}, - atts = Atts - }, - {NewLeaf, {Engine, St}}. - - -att_to_term(Att) -> - Bin = couch_att:to_binary(Att), - couch_att:store(data, Bin, Att). - - -term_diff(T1, T2) when is_tuple(T1), is_tuple(T2) -> - tuple_diff(tuple_to_list(T1), tuple_to_list(T2)); - -term_diff(L1, L2) when is_list(L1), is_list(L2) -> - list_diff(L1, L2); - -term_diff(V1, V2) when V1 == V2 -> - nodiff; - -term_diff(V1, V2) -> - {V1, V2}. - - -tuple_diff([], []) -> - nodiff; - -tuple_diff([T1 | _], []) -> - {longer, T1}; - -tuple_diff([], [T2 | _]) -> - {shorter, T2}; - -tuple_diff([T1 | R1], [T2 | R2]) -> - case term_diff(T1, T2) of - nodiff -> - tuple_diff(R1, R2); - Else -> - {T1, Else} - end. - - -list_diff([], []) -> - nodiff; - -list_diff([T1 | _], []) -> - {longer, T1}; - -list_diff([], [T2 | _]) -> - {shorter, T2}; - -list_diff([T1 | R1], [T2 | R2]) -> - case term_diff(T1, T2) of - nodiff -> - list_diff(R1, R2); - Else -> - {T1, Else} - end. - - -compact(Engine, St1, DbPath) -> - DbName = filename:basename(DbPath), - {ok, St2, Pid} = Engine:start_compaction(St1, DbName, [], self()), - Ref = erlang:monitor(process, Pid), - - % Ideally I'd assert that Pid is linked to us - % at this point but its technically possible - % that it could have finished compacting by - % the time we check... Quite the quandry. - - Term = receive - {'$gen_cast', {compact_done, Engine, Term0}} -> - Term0; - {'DOWN', Ref, _, _, Reason} -> - erlang:error({compactor_died, Reason}) - after ?COMPACTOR_TIMEOUT -> - erlang:error(compactor_timed_out) - end, - - {ok, St2, DbName, Pid, Term}. - - -with_config(Config, Fun) -> - OldConfig = apply_config(Config), - try - Fun() - after - apply_config(OldConfig) - end. - - -apply_config([]) -> - []; - -apply_config([{Section, Key, Value} | Rest]) -> - Orig = config:get(Section, Key), - case Value of - undefined -> config:delete(Section, Key); - _ -> config:set(Section, Key, Value) - end, - [{Section, Key, Orig} | apply_config(Rest)]. diff --git a/src/couch/test/couch_bt_engine_tests.erl b/src/couch/test/couch_bt_engine_tests.erl index df200df47..3e3ecbf25 100644 --- a/src/couch/test/couch_bt_engine_tests.erl +++ b/src/couch/test/couch_bt_engine_tests.erl @@ -17,4 +17,4 @@ couch_bt_engine_test_()-> - test_engine_util:create_tests(couch, couch_bt_engine). + cpse_util:create_tests(couch, couch_bt_engine, "couch"). diff --git a/src/couch_pse_tests/src/couch_pse_tests.app.src b/src/couch_pse_tests/src/couch_pse_tests.app.src new file mode 100644 index 000000000..83f3875a0 --- /dev/null +++ b/src/couch_pse_tests/src/couch_pse_tests.app.src @@ -0,0 +1,20 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +{application, couch_pse_tests, [ + {description, "Apache CouchDB Pluggable Storage Engine Test Suite"}, + {vsn, git}, + {applications, [ + kernel, + stdlib + ]} +]}. diff --git a/src/couch_pse_tests/src/cpse_gather.erl b/src/couch_pse_tests/src/cpse_gather.erl new file mode 100644 index 000000000..7804d419e --- /dev/null +++ b/src/couch_pse_tests/src/cpse_gather.erl @@ -0,0 +1,95 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(cpse_gather). + + +-export([ + module/1 +]). + + +module(ModName) -> + Exports = ModName:module_info(exports), + + SetupMod = get_setup_all(ModName, Exports), + TeardownMod = get_teardown_all(ModName, Exports), + SetupTest = get_fun(ModName, setup_each, 0, Exports), + TeardownTest = get_fun(ModName, teardown_each, 1, Exports), + + RevTests = lists:foldl(fun({Fun, Arity}, Acc) -> + case {atom_to_list(Fun), Arity} of + {[$c, $p, $s, $e, $_ | _], Arity} when Arity == 0; Arity == 1 -> + TestFun = make_test_fun(ModName, Fun, Arity), + [TestFun | Acc]; + _ -> + Acc + end + end, [], Exports), + Tests = lists:reverse(RevTests), + + { + setup, + spawn, + SetupMod, + TeardownMod, + [ + { + foreach, + SetupTest, + TeardownTest, + Tests + } + ] + }. + + +get_setup_all(ModName, Exports) -> + case lists:member({setup_all, 0}, Exports) of + true -> fun ModName:setup_all/0; + false -> fun cpse_util:setup_all/0 + end. + + +get_teardown_all(ModName, Exports) -> + case lists:member({teardown_all, 1}, Exports) of + true -> fun ModName:teardown_all/1; + false -> fun cpse_util:teardown_all/1 + end. + + +get_fun(ModName, FunName, Arity, Exports) -> + case lists:member({FunName, Arity}, Exports) of + true -> fun ModName:FunName/Arity; + false when Arity == 0 -> fun() -> ok end; + false when Arity == 1 -> fun(_) -> ok end + end. + + +make_test_fun(Module, Fun, Arity) -> + Name = atom_to_list(Fun), + case Arity of + 0 -> + fun(_) -> + {timeout, 60, {Name, fun() -> + process_flag(trap_exit, true), + Module:Fun() + end}} + end; + 1 -> + fun(Arg) -> + {timeout, 60, {Name, fun() -> + process_flag(trap_exit, true), + Module:Fun(Arg) + end}} + end + end. diff --git a/src/couch/src/test_engine_attachments.erl b/src/couch_pse_tests/src/cpse_test_attachments.erl index 691d4bd3c..61ada38c4 100644 --- a/src/couch/src/test_engine_attachments.erl +++ b/src/couch_pse_tests/src/cpse_test_attachments.erl @@ -10,7 +10,7 @@ % License for the specific language governing permissions and limitations under % the License. --module(test_engine_attachments). +-module(cpse_test_attachments). -compile(export_all). @@ -18,32 +18,40 @@ -include_lib("couch/include/couch_db.hrl"). -cet_write_attachment() -> - {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), +setup_each() -> + {ok, Db} = cpse_util:create_db(), + Db. + +teardown_each(Db) -> + ok = couch_server:delete(couch_db:name(Db), []). + + +cpse_write_attachment(Db1) -> AttBin = crypto:strong_rand_bytes(32768), try - [Att0] = test_engine_util:prep_atts(Engine, St1, [ + [Att0] = cpse_util:prep_atts(Db1, [ {<<"ohai.txt">>, AttBin} ]), {stream, Stream} = couch_att:fetch(data, Att0), - ?assertEqual(true, Engine:is_active_stream(St1, Stream)), + ?assertEqual(true, couch_db_engine:is_active_stream(Db1, Stream)), - Actions = [{create, {<<"first">>, [], [Att0]}}], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), - {ok, St3} = Engine:commit_data(St2), - Engine:terminate(normal, St3), + Actions = [{create, {<<"first">>, {[]}, [Att0]}}], + {ok, Db2} = cpse_util:apply_actions(Db1, Actions), + {ok, _} = couch_db:ensure_full_commit(Db2), + cpse_util:shutdown_db(Db2), - {ok, St4} = Engine:init(DbPath, []), - [FDI] = Engine:open_docs(St4, [<<"first">>]), + {ok, Db3} = couch_db:reopen(Db2), + + [FDI] = couch_db_engine:open_docs(Db3, [<<"first">>]), #rev_info{ rev = {RevPos, PrevRevId}, deleted = Deleted, body_sp = DocPtr - } = test_engine_util:prev_rev(FDI), + } = cpse_util:prev_rev(FDI), Doc0 = #doc{ id = <<"foo">>, @@ -52,12 +60,12 @@ cet_write_attachment() -> body = DocPtr }, - Doc1 = Engine:read_doc_body(St4, Doc0), + Doc1 = couch_db_engine:read_doc_body(Db3, Doc0), Atts1 = if not is_binary(Doc1#doc.atts) -> Doc1#doc.atts; true -> couch_compress:decompress(Doc1#doc.atts) end, - StreamSrc = fun(Sp) -> Engine:open_read_stream(St4, Sp) end, + StreamSrc = fun(Sp) -> couch_db_engine:open_read_stream(Db3, Sp) end, [Att1] = [couch_att:from_disk_term(StreamSrc, T) || T <- Atts1], ReadBin = couch_att:to_binary(Att1), ?assertEqual(AttBin, ReadBin) @@ -71,23 +79,21 @@ cet_write_attachment() -> % attachments streams when restarting (for instance if % we ever have something that stores attachemnts in % an external object store) -cet_inactive_stream() -> - {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), - +cpse_inactive_stream(Db1) -> AttBin = crypto:strong_rand_bytes(32768), try - [Att0] = test_engine_util:prep_atts(Engine, St1, [ + [Att0] = cpse_util:prep_atts(Db1, [ {<<"ohai.txt">>, AttBin} ]), {stream, Stream} = couch_att:fetch(data, Att0), - ?assertEqual(true, Engine:is_active_stream(St1, Stream)), + ?assertEqual(true, couch_db_engine:is_active_stream(Db1, Stream)), - Engine:terminate(normal, St1), - {ok, St2} = Engine:init(DbPath, []), + cpse_util:shutdown_db(Db1), + {ok, Db2} = couch_db:reopen(Db1), - ?assertEqual(false, Engine:is_active_stream(St2, Stream)) + ?assertEqual(false, couch_db_engine:is_active_stream(Db2, Stream)) catch throw:not_supported -> ok end. diff --git a/src/couch_pse_tests/src/cpse_test_compaction.erl b/src/couch_pse_tests/src/cpse_test_compaction.erl new file mode 100644 index 000000000..11bf106d2 --- /dev/null +++ b/src/couch_pse_tests/src/cpse_test_compaction.erl @@ -0,0 +1,191 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(cpse_test_compaction). +-compile(export_all). + + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("couch/include/couch_db.hrl"). + + +setup_each() -> + {ok, Db} = cpse_util:create_db(), + Db. + + +teardown_each(Db) -> + ok = couch_server:delete(couch_db:name(Db), []). + + +cpse_compact_empty(Db1) -> + Term1 = cpse_util:db_as_term(Db1), + + cpse_util:compact(Db1), + + {ok, Db2} = couch_db:reopen(Db1), + Term2 = cpse_util:db_as_term(Db2), + + Diff = cpse_util:term_diff(Term1, Term2), + ?assertEqual(nodiff, Diff). + + +cpse_compact_doc(Db1) -> + Actions = [{create, {<<"foo">>, {[]}}}], + {ok, Db2} = cpse_util:apply_actions(Db1, Actions), + Term1 = cpse_util:db_as_term(Db2), + + cpse_util:compact(Db2), + + {ok, Db3} = couch_db:reopen(Db2), + Term2 = cpse_util:db_as_term(Db3), + + Diff = cpse_util:term_diff(Term1, Term2), + ?assertEqual(nodiff, Diff). + + +cpse_compact_local_doc(Db1) -> + Actions = [{create, {<<"_local/foo">>, {[]}}}], + {ok, Db2} = cpse_util:apply_actions(Db1, Actions), + Term1 = cpse_util:db_as_term(Db2), + + cpse_util:compact(Db2), + + {ok, Db3} = couch_db:reopen(Db2), + Term2 = cpse_util:db_as_term(Db3), + + Diff = cpse_util:term_diff(Term1, Term2), + ?assertEqual(nodiff, Diff). + + +cpse_compact_with_everything(Db1) -> + % Add a whole bunch of docs + DocActions = lists:map(fun(Seq) -> + {create, {docid(Seq), {[{<<"int">>, Seq}]}}} + end, lists:seq(1, 1000)), + + LocalActions = lists:map(fun(I) -> + {create, {local_docid(I), {[{<<"int">>, I}]}}} + end, lists:seq(1, 25)), + + Actions1 = DocActions ++ LocalActions, + + {ok, Db2} = cpse_util:apply_batch(Db1, Actions1), + ok = couch_db:set_security(Db1, {[{<<"foo">>, <<"bar">>}]}), + ok = couch_db:set_revs_limit(Db1, 500), + + Actions2 = [ + {create, {<<"foo">>, {[]}}}, + {create, {<<"bar">>, {[{<<"hooray">>, <<"purple">>}]}}}, + {conflict, {<<"bar">>, {[{<<"booo">>, false}]}}} + ], + + {ok, Db3} = cpse_util:apply_actions(Db2, Actions2), + + [FooFDI, BarFDI] = couch_db_engine:open_docs(Db3, [<<"foo">>, <<"bar">>]), + + FooRev = cpse_util:prev_rev(FooFDI), + BarRev = cpse_util:prev_rev(BarFDI), + + Actions3 = [ + {batch, [ + {purge, {<<"foo">>, FooRev#rev_info.rev}}, + {purge, {<<"bar">>, BarRev#rev_info.rev}} + ]} + ], + + {ok, Db4} = cpse_util:apply_actions(Db3, Actions3), + + PurgedIdRevs = [ + {<<"bar">>, [BarRev#rev_info.rev]}, + {<<"foo">>, [FooRev#rev_info.rev]} + ], + + ?assertEqual( + PurgedIdRevs, + lists:sort(couch_db_engine:get_last_purged(Db4)) + ), + + {ok, Db5} = try + [Att0, Att1, Att2, Att3, Att4] = cpse_util:prep_atts(Db4, [ + {<<"ohai.txt">>, crypto:strong_rand_bytes(2048)}, + {<<"stuff.py">>, crypto:strong_rand_bytes(32768)}, + {<<"a.erl">>, crypto:strong_rand_bytes(29)}, + {<<"a.hrl">>, crypto:strong_rand_bytes(5000)}, + {<<"a.app">>, crypto:strong_rand_bytes(400)} + ]), + + Actions4 = [ + {create, {<<"small_att">>, {[]}, [Att0]}}, + {create, {<<"large_att">>, {[]}, [Att1]}}, + {create, {<<"multi_att">>, {[]}, [Att2, Att3, Att4]}} + ], + cpse_util:apply_actions(Db4, Actions4) + catch throw:not_supported -> + {ok, Db4} + end, + {ok, _} = couch_db:ensure_full_commit(Db5), + {ok, Db6} = couch_db:reopen(Db5), + + Term1 = cpse_util:db_as_term(Db6), + + Config = [ + {"database_compaction", "doc_buffer_size", "1024"}, + {"database_compaction", "checkpoint_after", "2048"} + ], + + cpse_util:with_config(Config, fun() -> + cpse_util:compact(Db6) + end), + + {ok, Db7} = couch_db:reopen(Db6), + Term2 = cpse_util:db_as_term(Db7), + + Diff = cpse_util:term_diff(Term1, Term2), + ?assertEqual(nodiff, Diff). + + +cpse_recompact_updates(Db1) -> + Actions1 = lists:map(fun(Seq) -> + {create, {docid(Seq), {[{<<"int">>, Seq}]}}} + end, lists:seq(1, 1000)), + {ok, Db2} = cpse_util:apply_batch(Db1, Actions1), + + {ok, Compactor} = couch_db:start_compact(Db2), + catch erlang:suspend_process(Compactor), + + Actions2 = [ + {update, {<<"0001">>, {[{<<"updated">>, true}]}}}, + {create, {<<"boop">>, {[]}}} + ], + + {ok, Db3} = cpse_util:apply_actions(Db2, Actions2), + Term1 = cpse_util:db_as_term(Db3), + + catch erlang:resume_process(Compactor), + cpse_util:compact(Db3), + + {ok, Db4} = couch_db:reopen(Db3), + Term2 = cpse_util:db_as_term(Db4), + + Diff = cpse_util:term_diff(Term1, Term2), + ?assertEqual(nodiff, Diff). + + +docid(I) -> + Str = io_lib:format("~4..0b", [I]), + iolist_to_binary(Str). + + +local_docid(I) -> + Str = io_lib:format("_local/~4..0b", [I]), + iolist_to_binary(Str). diff --git a/src/couch_pse_tests/src/cpse_test_fold_changes.erl b/src/couch_pse_tests/src/cpse_test_fold_changes.erl new file mode 100644 index 000000000..8ee74f041 --- /dev/null +++ b/src/couch_pse_tests/src/cpse_test_fold_changes.erl @@ -0,0 +1,198 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(cpse_test_fold_changes). +-compile(export_all). + + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("couch/include/couch_db.hrl"). + + +-define(NUM_DOCS, 25). + + +setup_each() -> + {ok, Db} = cpse_util:create_db(), + Db. + + +teardown_each(Db) -> + ok = couch_server:delete(couch_db:name(Db), []). + + +cpse_empty_changes(Db) -> + ?assertEqual(0, couch_db_engine:count_changes_since(Db, 0)), + ?assertEqual({ok, []}, + couch_db_engine:fold_changes(Db, 0, fun fold_fun/2, [], [])). + + +cpse_single_change(Db1) -> + Actions = [{create, {<<"a">>, {[]}}}], + {ok, Db2} = cpse_util:apply_actions(Db1, Actions), + + ?assertEqual(1, couch_db_engine:count_changes_since(Db2, 0)), + ?assertEqual({ok, [{<<"a">>, 1}]}, + couch_db_engine:fold_changes(Db2, 0, fun fold_fun/2, [], [])). + + +cpse_two_changes(Db1) -> + Actions = [ + {create, {<<"a">>, {[]}}}, + {create, {<<"b">>, {[]}}} + ], + {ok, Db2} = cpse_util:apply_actions(Db1, Actions), + + ?assertEqual(2, couch_db_engine:count_changes_since(Db2, 0)), + {ok, Changes} = + couch_db_engine:fold_changes(Db2, 0, fun fold_fun/2, [], []), + ?assertEqual([{<<"a">>, 1}, {<<"b">>, 2}], lists:reverse(Changes)). + + +cpse_two_changes_batch(Db1) -> + Actions = [ + {batch, [ + {create, {<<"a">>, {[]}}}, + {create, {<<"b">>, {[]}}} + ]} + ], + {ok, Db2} = cpse_util:apply_actions(Db1, Actions), + + ?assertEqual(2, couch_db_engine:count_changes_since(Db2, 0)), + {ok, Changes} = + couch_db_engine:fold_changes(Db2, 0, fun fold_fun/2, [], []), + ?assertEqual([{<<"a">>, 1}, {<<"b">>, 2}], lists:reverse(Changes)). + + +cpse_two_changes_batch_sorted(Db1) -> + Actions = [ + {batch, [ + {create, {<<"b">>, {[]}}}, + {create, {<<"a">>, {[]}}} + ]} + ], + {ok, Db2} = cpse_util:apply_actions(Db1, Actions), + + ?assertEqual(2, couch_db_engine:count_changes_since(Db2, 0)), + {ok, Changes} = + couch_db_engine:fold_changes(Db2, 0, fun fold_fun/2, [], []), + ?assertEqual([{<<"a">>, 1}, {<<"b">>, 2}], lists:reverse(Changes)). + + +cpse_update_one(Db1) -> + Actions = [ + {create, {<<"a">>, {[]}}}, + {update, {<<"a">>, {[]}}} + ], + {ok, Db2} = cpse_util:apply_actions(Db1, Actions), + + ?assertEqual(1, couch_db_engine:count_changes_since(Db2, 0)), + ?assertEqual({ok, [{<<"a">>, 2}]}, + couch_db_engine:fold_changes(Db2, 0, fun fold_fun/2, [], [])). + + +cpse_update_first_of_two(Db1) -> + Actions = [ + {create, {<<"a">>, {[]}}}, + {create, {<<"b">>, {[]}}}, + {update, {<<"a">>, {[]}}} + ], + {ok, Db2} = cpse_util:apply_actions(Db1, Actions), + + ?assertEqual(2, couch_db_engine:count_changes_since(Db2, 0)), + {ok, Changes} = + couch_db_engine:fold_changes(Db2, 0, fun fold_fun/2, [], []), + ?assertEqual([{<<"b">>, 2}, {<<"a">>, 3}], lists:reverse(Changes)). + + +cpse_update_second_of_two(Db1) -> + Actions = [ + {create, {<<"a">>, {[]}}}, + {create, {<<"b">>, {[]}}}, + {update, {<<"b">>, {[]}}} + ], + {ok, Db2} = cpse_util:apply_actions(Db1, Actions), + + ?assertEqual(2, couch_db_engine:count_changes_since(Db2, 0)), + {ok, Changes} = + couch_db_engine:fold_changes(Db2, 0, fun fold_fun/2, [], []), + ?assertEqual([{<<"a">>, 1}, {<<"b">>, 3}], lists:reverse(Changes)). + + +cpse_check_mutation_ordering(Db1) -> + Actions = shuffle(lists:map(fun(Seq) -> + {create, {docid(Seq), {[]}}} + end, lists:seq(1, ?NUM_DOCS))), + + DocIdOrder = [DocId || {_, {DocId, _}} <- Actions], + DocSeqs = lists:zip(DocIdOrder, lists:seq(1, ?NUM_DOCS)), + + {ok, Db2} = cpse_util:apply_actions(Db1, Actions), + + % First lets see that we can get the correct + % suffix/prefix starting at every update sequence + lists:foreach(fun(Seq) -> + {ok, Suffix} = + couch_db_engine:fold_changes(Db2, Seq, fun fold_fun/2, [], []), + ?assertEqual(lists:nthtail(Seq, DocSeqs), lists:reverse(Suffix)), + + {ok, Prefix} = couch_db_engine:fold_changes( + Db2, Seq, fun fold_fun/2, [], [{dir, rev}]), + ?assertEqual(lists:sublist(DocSeqs, Seq + 1), Prefix) + end, lists:seq(0, ?NUM_DOCS)), + + ok = do_mutation_ordering(Db2, ?NUM_DOCS + 1, DocSeqs, []). + + +do_mutation_ordering(Db, _Seq, [], FinalDocSeqs) -> + {ok, RevOrder} = couch_db_engine:fold_changes(Db, 0, fun fold_fun/2, [], []), + ?assertEqual(FinalDocSeqs, lists:reverse(RevOrder)), + ok; + +do_mutation_ordering(Db, Seq, [{DocId, _OldSeq} | Rest], DocSeqAcc) -> + Actions = [{update, {DocId, {[]}}}], + {ok, NewDb} = cpse_util:apply_actions(Db, Actions), + NewAcc = DocSeqAcc ++ [{DocId, Seq}], + Expected = Rest ++ NewAcc, + {ok, RevOrder} = + couch_db_engine:fold_changes(NewDb, 0, fun fold_fun/2, [], []), + ?assertEqual(Expected, lists:reverse(RevOrder)), + do_mutation_ordering(NewDb, Seq + 1, Rest, NewAcc). + + +shuffle(List) -> + random:seed(os:timestamp()), + Paired = [{random:uniform(), I} || I <- List], + Sorted = lists:sort(Paired), + [I || {_, I} <- Sorted]. + + +remove_random(List) -> + Pos = random:uniform(length(List)), + remove_random(Pos, List). + + +remove_random(1, [Item | Rest]) -> + {Item, Rest}; + +remove_random(N, [Skip | Rest]) when N > 1 -> + {Item, Tail} = remove_random(N - 1, Rest), + {Item, [Skip | Tail]}. + + +fold_fun(#full_doc_info{id=Id, update_seq=Seq}, Acc) -> + {ok, [{Id, Seq} | Acc]}. + + +docid(I) -> + Str = io_lib:format("~4..0b", [I]), + iolist_to_binary(Str). diff --git a/src/couch/src/test_engine_fold_docs.erl b/src/couch_pse_tests/src/cpse_test_fold_docs.erl index 458878d97..09fbd26d0 100644 --- a/src/couch/src/test_engine_fold_docs.erl +++ b/src/couch_pse_tests/src/cpse_test_fold_docs.erl @@ -10,7 +10,7 @@ % License for the specific language governing permissions and limitations under % the License. --module(test_engine_fold_docs). +-module(cpse_test_fold_docs). -compile(export_all). @@ -21,80 +21,90 @@ -define(NUM_DOCS, 100). -cet_fold_all() -> - fold_all(fold_docs, fun docid/1). +setup_each() -> + cpse_util:dbname(). -cet_fold_all_local() -> - fold_all(fold_local_docs, fun local_docid/1). +teardown_each(DbName) -> + ok = couch_server:delete(DbName, []). -cet_fold_start_key() -> - fold_start_key(fold_docs, fun docid/1). +cpse_fold_all(DbName) -> + fold_all(DbName, fold_docs, fun docid/1). -cet_fold_start_key_local() -> - fold_start_key(fold_local_docs, fun local_docid/1). +cpse_fold_all_local(DbName) -> + fold_all(DbName, fold_local_docs, fun local_docid/1). -cet_fold_end_key() -> - fold_end_key(fold_docs, fun docid/1). +cpse_fold_start_key(DbName) -> + fold_start_key(DbName, fold_docs, fun docid/1). -cet_fold_end_key_local() -> - fold_end_key(fold_local_docs, fun local_docid/1). +cpse_fold_start_key_local(DbName) -> + fold_start_key(DbName, fold_local_docs, fun local_docid/1). -cet_fold_end_key_gt() -> - fold_end_key_gt(fold_docs, fun docid/1). +cpse_fold_end_key(DbName) -> + fold_end_key(DbName, fold_docs, fun docid/1). -cet_fold_end_key_gt_local() -> - fold_end_key_gt(fold_local_docs, fun local_docid/1). +cpse_fold_end_key_local(DbName) -> + fold_end_key(DbName, fold_local_docs, fun local_docid/1). -cet_fold_range() -> - fold_range(fold_docs, fun docid/1). +cpse_fold_end_key_gt(DbName) -> + fold_end_key_gt(DbName, fold_docs, fun docid/1). -cet_fold_range_local() -> - fold_range(fold_local_docs, fun local_docid/1). +cpse_fold_end_key_gt_local(DbName) -> + fold_end_key_gt(DbName, fold_local_docs, fun local_docid/1). -cet_fold_stop() -> - fold_stop(fold_docs, fun docid/1). +cpse_fold_range(DbName) -> + fold_range(DbName, fold_docs, fun docid/1). -cet_fold_stop_local() -> - fold_stop(fold_local_docs, fun local_docid/1). +cpse_fold_range_local(DbName) -> + fold_range(DbName, fold_local_docs, fun local_docid/1). + + +cpse_fold_stop(DbName) -> + fold_user_fun_stop(DbName, fold_docs, fun docid/1). + + +cpse_fold_stop_local(DbName) -> + fold_user_fun_stop(DbName, fold_local_docs, fun local_docid/1). % This is a loose test but we have to have this until % I figure out what to do about the total_rows/offset % meta data included in _all_docs -cet_fold_include_reductions() -> - {ok, Engine, St} = init_st(fun docid/1), +cpse_fold_include_reductions(DbName) -> + {ok, Db} = init_db(DbName, fun docid/1), FoldFun = fun(_, _, nil) -> {ok, nil} end, - {ok, Count, nil} = Engine:fold_docs(St, FoldFun, nil, [include_reductions]), + Opts = [include_reductions], + {ok, Count, nil} = couch_db_engine:fold_docs(Db, FoldFun, nil, Opts), ?assert(is_integer(Count)), ?assert(Count >= 0). -fold_all(FoldFun, DocIdFun) -> +fold_all(DbName, FoldFun, DocIdFun) -> DocIds = [DocIdFun(I) || I <- lists:seq(1, ?NUM_DOCS)], - {ok, Engine, St} = init_st(DocIdFun), + {ok, Db} = init_db(DbName, DocIdFun), - {ok, DocIdAccFwd} = Engine:FoldFun(St, fun fold_fun/2, [], []), + {ok, DocIdAccFwd} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], []), ?assertEqual(?NUM_DOCS, length(DocIdAccFwd)), ?assertEqual(DocIds, lists:reverse(DocIdAccFwd)), - {ok, DocIdAccRev} = Engine:FoldFun(St, fun fold_fun/2, [], [{dir, rev}]), + Opts = [{dir, rev}], + {ok, DocIdAccRev} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], Opts), ?assertEqual(?NUM_DOCS, length(DocIdAccRev)), ?assertEqual(DocIds, DocIdAccRev). -fold_start_key(FoldFun, DocIdFun) -> - {ok, Engine, St} = init_st(DocIdFun), +fold_start_key(DbName, FoldFun, DocIdFun) -> + {ok, Db} = init_db(DbName, DocIdFun), StartKeyNum = ?NUM_DOCS div 4, StartKey = DocIdFun(StartKeyNum), @@ -103,35 +113,35 @@ fold_start_key(FoldFun, DocIdFun) -> DocIdsFwd = [DocIdFun(I) || I <- lists:seq(StartKeyNum, ?NUM_DOCS)], DocIdsRev = [DocIdFun(I) || I <- lists:seq(1, StartKeyNum)], - ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun/2, [], [ + ?assertEqual({ok, []}, couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {start_key, <<255>>} ])), - ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun/2, [], [ + ?assertEqual({ok, []}, couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {dir, rev}, {start_key, <<"">>} ])), - {ok, AllDocIdAccFwd} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {ok, AllDocIdAccFwd} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {start_key, <<"">>} ]), ?assertEqual(length(AllDocIds), length(AllDocIdAccFwd)), ?assertEqual(AllDocIds, lists:reverse(AllDocIdAccFwd)), - {ok, AllDocIdAccRev} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {ok, AllDocIdAccRev} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {dir, rev}, {start_key, <<255>>} ]), ?assertEqual(length(AllDocIds), length(AllDocIdAccRev)), ?assertEqual(AllDocIds, AllDocIdAccRev), - {ok, DocIdAccFwd} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {ok, DocIdAccFwd} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {start_key, StartKey} ]), ?assertEqual(length(DocIdsFwd), length(DocIdAccFwd)), ?assertEqual(DocIdsFwd, lists:reverse(DocIdAccFwd)), - {ok, DocIdAccRev} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {ok, DocIdAccRev} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {dir, rev}, {start_key, StartKey} ]), @@ -139,30 +149,30 @@ fold_start_key(FoldFun, DocIdFun) -> ?assertEqual(DocIdsRev, DocIdAccRev). -fold_end_key(FoldFun, DocIdFun) -> - {ok, Engine, St} = init_st(DocIdFun), +fold_end_key(DbName, FoldFun, DocIdFun) -> + {ok, Db} = init_db(DbName, DocIdFun), EndKeyNum = ?NUM_DOCS div 4, EndKey = DocIdFun(EndKeyNum), - ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun/2, [], [ + ?assertEqual({ok, []}, couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {end_key, <<"">>} ])), - ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun/2, [], [ + ?assertEqual({ok, []}, couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {dir, rev}, {end_key, <<255>>} ])), AllDocIds = [DocIdFun(I) || I <- lists:seq(1, ?NUM_DOCS)], - {ok, AllDocIdAccFwd} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {ok, AllDocIdAccFwd} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {end_key, <<255>>} ]), ?assertEqual(length(AllDocIds), length(AllDocIdAccFwd)), ?assertEqual(AllDocIds, lists:reverse(AllDocIdAccFwd)), - {ok, AllDocIdAccRev} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {ok, AllDocIdAccRev} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {dir, rev}, {end_key, <<"">>} ]), @@ -171,7 +181,7 @@ fold_end_key(FoldFun, DocIdFun) -> DocIdsFwd = [DocIdFun(I) || I <- lists:seq(1, EndKeyNum)], - {ok, DocIdAccFwd} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {ok, DocIdAccFwd} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {end_key, EndKey} ]), ?assertEqual(length(DocIdsFwd), length(DocIdAccFwd)), @@ -179,7 +189,7 @@ fold_end_key(FoldFun, DocIdFun) -> DocIdsRev = [DocIdFun(I) || I <- lists:seq(EndKeyNum, ?NUM_DOCS)], - {ok, DocIdAccRev} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {ok, DocIdAccRev} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {dir, rev}, {end_key, EndKey} ]), @@ -187,30 +197,30 @@ fold_end_key(FoldFun, DocIdFun) -> ?assertEqual(DocIdsRev, DocIdAccRev). -fold_end_key_gt(FoldFun, DocIdFun) -> - {ok, Engine, St} = init_st(DocIdFun), +fold_end_key_gt(DbName, FoldFun, DocIdFun) -> + {ok, Db} = init_db(DbName, DocIdFun), EndKeyNum = ?NUM_DOCS div 4, EndKey = DocIdFun(EndKeyNum), - ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun/2, [], [ + ?assertEqual({ok, []}, couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {end_key_gt, <<"">>} ])), - ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun/2, [], [ + ?assertEqual({ok, []}, couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {dir, rev}, {end_key_gt, <<255>>} ])), AllDocIds = [DocIdFun(I) || I <- lists:seq(1, ?NUM_DOCS)], - {ok, AllDocIdAccFwd} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {ok, AllDocIdAccFwd} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {end_key_gt, <<255>>} ]), ?assertEqual(length(AllDocIds), length(AllDocIdAccFwd)), ?assertEqual(AllDocIds, lists:reverse(AllDocIdAccFwd)), - {ok, AllDocIdAccRev} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {ok, AllDocIdAccRev} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {dir, rev}, {end_key_gt, <<"">>} ]), @@ -219,7 +229,7 @@ fold_end_key_gt(FoldFun, DocIdFun) -> DocIdsFwd = [DocIdFun(I) || I <- lists:seq(1, EndKeyNum - 1)], - {ok, DocIdAccFwd} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {ok, DocIdAccFwd} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {end_key_gt, EndKey} ]), ?assertEqual(length(DocIdsFwd), length(DocIdAccFwd)), @@ -227,7 +237,7 @@ fold_end_key_gt(FoldFun, DocIdFun) -> DocIdsRev = [DocIdFun(I) || I <- lists:seq(EndKeyNum + 1, ?NUM_DOCS)], - {ok, DocIdAccRev} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {ok, DocIdAccRev} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {dir, rev}, {end_key_gt, EndKey} ]), @@ -235,8 +245,8 @@ fold_end_key_gt(FoldFun, DocIdFun) -> ?assertEqual(DocIdsRev, DocIdAccRev). -fold_range(FoldFun, DocIdFun) -> - {ok, Engine, St} = init_st(DocIdFun), +fold_range(DbName, FoldFun, DocIdFun) -> + {ok, Db} = init_db(DbName, DocIdFun), StartKeyNum = ?NUM_DOCS div 4, EndKeyNum = StartKeyNum * 3, @@ -244,12 +254,12 @@ fold_range(FoldFun, DocIdFun) -> StartKey = DocIdFun(StartKeyNum), EndKey = DocIdFun(EndKeyNum), - ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun/2, [], [ + ?assertEqual({ok, []}, couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {start_key, <<"">>}, {end_key, <<"">>} ])), - ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun/2, [], [ + ?assertEqual({ok, []}, couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {dir, rev}, {start_key, <<"">>}, {end_key, <<255>>} @@ -257,14 +267,14 @@ fold_range(FoldFun, DocIdFun) -> AllDocIds = [DocIdFun(I) || I <- lists:seq(1, ?NUM_DOCS)], - {ok, AllDocIdAccFwd} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {ok, AllDocIdAccFwd} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {start_key, <<"">>}, {end_key, <<255>>} ]), ?assertEqual(length(AllDocIds), length(AllDocIdAccFwd)), ?assertEqual(AllDocIds, lists:reverse(AllDocIdAccFwd)), - {ok, AllDocIdAccRev} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {ok, AllDocIdAccRev} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {dir, rev}, {start_key, <<255>>}, {end_key_gt, <<"">>} @@ -274,7 +284,7 @@ fold_range(FoldFun, DocIdFun) -> DocIdsFwd = [DocIdFun(I) || I <- lists:seq(StartKeyNum, EndKeyNum)], - {ok, DocIdAccFwd} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {ok, DocIdAccFwd} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {start_key, StartKey}, {end_key, EndKey} ]), @@ -283,13 +293,13 @@ fold_range(FoldFun, DocIdFun) -> DocIdsRev = [DocIdFun(I) || I <- lists:seq(StartKeyNum, EndKeyNum)], - ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun/2, [], [ + ?assertEqual({ok, []}, couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {dir, rev}, {start_key, StartKey}, {end_key, EndKey} ])), - {ok, DocIdAccRev} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {ok, DocIdAccRev} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {dir, rev}, {start_key, EndKey}, {end_key, StartKey} @@ -298,24 +308,24 @@ fold_range(FoldFun, DocIdFun) -> ?assertEqual(DocIdsRev, DocIdAccRev). -fold_stop(FoldFun, DocIdFun) -> - {ok, Engine, St} = init_st(DocIdFun), +fold_user_fun_stop(DbName, FoldFun, DocIdFun) -> + {ok, Db} = init_db(DbName, DocIdFun), StartKeyNum = ?NUM_DOCS div 4, StartKey = DocIdFun(StartKeyNum), - ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun_stop/2, [], [ + ?assertEqual({ok, []}, couch_db_engine:FoldFun(Db, fun fold_stop/2, [], [ {start_key, <<255>>} ])), - ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun_stop/2, [], [ + ?assertEqual({ok, []}, couch_db_engine:FoldFun(Db, fun fold_stop/2, [], [ {dir, rev}, {start_key, <<"">>} ])), SuffixDocIds = [DocIdFun(I) || I <- lists:seq(?NUM_DOCS - 3, ?NUM_DOCS)], - {ok, SuffixDocIdAcc} = Engine:FoldFun(St, fun fold_fun_stop/2, [], [ + {ok, SuffixDocIdAcc} = couch_db_engine:FoldFun(Db, fun fold_stop/2, [], [ {start_key, DocIdFun(?NUM_DOCS - 3)} ]), ?assertEqual(length(SuffixDocIds), length(SuffixDocIdAcc)), @@ -323,7 +333,7 @@ fold_stop(FoldFun, DocIdFun) -> PrefixDocIds = [DocIdFun(I) || I <- lists:seq(1, 3)], - {ok, PrefixDocIdAcc} = Engine:FoldFun(St, fun fold_fun_stop/2, [], [ + {ok, PrefixDocIdAcc} = couch_db_engine:FoldFun(Db, fun fold_stop/2, [], [ {dir, rev}, {start_key, DocIdFun(3)} ]), @@ -333,7 +343,7 @@ fold_stop(FoldFun, DocIdFun) -> FiveDocIdsFwd = [DocIdFun(I) || I <- lists:seq(StartKeyNum, StartKeyNum + 5)], - {ok, FiveDocIdAccFwd} = Engine:FoldFun(St, fun fold_fun_stop/2, [], [ + {ok, FiveDocIdAccFwd} = couch_db_engine:FoldFun(Db, fun fold_stop/2, [], [ {start_key, StartKey} ]), ?assertEqual(length(FiveDocIdsFwd), length(FiveDocIdAccFwd)), @@ -342,7 +352,7 @@ fold_stop(FoldFun, DocIdFun) -> FiveDocIdsRev = [DocIdFun(I) || I <- lists:seq(StartKeyNum - 5, StartKeyNum)], - {ok, FiveDocIdAccRev} = Engine:FoldFun(St, fun fold_fun_stop/2, [], [ + {ok, FiveDocIdAccRev} = couch_db_engine:FoldFun(Db, fun fold_stop/2, [], [ {dir, rev}, {start_key, StartKey} ]), @@ -350,13 +360,12 @@ fold_stop(FoldFun, DocIdFun) -> ?assertEqual(FiveDocIdsRev, FiveDocIdAccRev). -init_st(DocIdFun) -> - {ok, Engine, St1} = test_engine_util:init_engine(), +init_db(DbName, DocIdFun) -> + {ok, Db1} = cpse_util:create_db(DbName), Actions = lists:map(fun(Id) -> - {create, {DocIdFun(Id), [{<<"int">>, Id}]}} + {create, {DocIdFun(Id), {[{<<"int">>, Id}]}}} end, lists:seq(1, ?NUM_DOCS)), - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), - {ok, Engine, St2}. + cpse_util:apply_actions(Db1, [{batch, Actions}]). fold_fun(Doc, Acc) -> @@ -367,7 +376,7 @@ fold_fun(Doc, Acc) -> {ok, [Id | Acc]}. -fold_fun_stop(Doc, Acc) -> +fold_stop(Doc, Acc) -> Id = case Doc of #doc{id = Id0} -> Id0; #full_doc_info{id = Id0} -> Id0 diff --git a/src/couch_pse_tests/src/cpse_test_get_set_props.erl b/src/couch_pse_tests/src/cpse_test_get_set_props.erl new file mode 100644 index 000000000..97f164bf8 --- /dev/null +++ b/src/couch_pse_tests/src/cpse_test_get_set_props.erl @@ -0,0 +1,94 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(cpse_test_get_set_props). +-compile(export_all). + + +-include_lib("eunit/include/eunit.hrl"). + + +setup_each() -> + cpse_util:dbname(). + + +teardown_each(DbName) -> + ok = couch_server:delete(DbName, []). + + +cpse_default_props(DbName) -> + {ok, {_App, Engine, _Extension}} = application:get_env(couch, test_engine), + {ok, Db} = cpse_util:create_db(DbName), + Node = node(), + + ?assertEqual(Engine, couch_db_engine:get_engine(Db)), + ?assertEqual(0, couch_db_engine:get_doc_count(Db)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db)), + ?assertEqual(true, is_list(couch_db_engine:get_size_info(Db))), + ?assertEqual(true, is_integer(couch_db_engine:get_disk_version(Db))), + ?assertEqual(0, couch_db_engine:get_update_seq(Db)), + ?assertEqual(0, couch_db_engine:get_purge_seq(Db)), + ?assertEqual([], couch_db_engine:get_last_purged(Db)), + ?assertEqual([], couch_db_engine:get_security(Db)), + ?assertEqual(1000, couch_db_engine:get_revs_limit(Db)), + ?assertMatch(<<_:32/binary>>, couch_db_engine:get_uuid(Db)), + ?assertEqual([{Node, 0}], couch_db_engine:get_epochs(Db)), + ?assertEqual(0, couch_db_engine:get_compacted_seq(Db)). + + +-define(ADMIN_ONLY_SEC_PROPS, {[ + {<<"members">>, {[ + {<<"roles">>, [<<"_admin">>]} + ]}}, + {<<"admins">>, {[ + {<<"roles">>, [<<"_admin">>]} + ]}} +]}). + + +cpse_admin_only_security(DbName) -> + Config = [{"couchdb", "default_security", "admin_only"}], + {ok, Db1} = cpse_util:with_config(Config, fun() -> + cpse_util:create_db(DbName) + end), + + ?assertEqual(?ADMIN_ONLY_SEC_PROPS, couch_db:get_security(Db1)), + cpse_util:shutdown_db(Db1), + + {ok, Db2} = couch_db:reopen(Db1), + couch_log:error("~n~n~n~n~s -> ~s~n~n", [couch_db:name(Db1), couch_db:name(Db2)]), + ?assertEqual(?ADMIN_ONLY_SEC_PROPS, couch_db:get_security(Db2)). + + +cpse_set_security(DbName) -> + SecProps = {[{<<"foo">>, <<"bar">>}]}, + check_prop_set(DbName, get_security, set_security, {[]}, SecProps). + + +cpse_set_revs_limit(DbName) -> + check_prop_set(DbName, get_revs_limit, set_revs_limit, 1000, 50). + + +check_prop_set(DbName, GetFun, SetFun, Default, Value) -> + {ok, Db0} = cpse_util:create_db(DbName), + + ?assertEqual(Default, couch_db:GetFun(Db0)), + ?assertMatch(ok, couch_db:SetFun(Db0, Value)), + + {ok, Db1} = couch_db:reopen(Db0), + ?assertEqual(Value, couch_db:GetFun(Db1)), + + ?assertMatch({ok, _}, couch_db:ensure_full_commit(Db1)), + cpse_util:shutdown_db(Db1), + + {ok, Db2} = couch_db:reopen(Db1), + ?assertEqual(Value, couch_db:GetFun(Db2)). diff --git a/src/couch_pse_tests/src/cpse_test_open_close_delete.erl b/src/couch_pse_tests/src/cpse_test_open_close_delete.erl new file mode 100644 index 000000000..c19d0ee4c --- /dev/null +++ b/src/couch_pse_tests/src/cpse_test_open_close_delete.erl @@ -0,0 +1,76 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(cpse_test_open_close_delete). +-compile(export_all). + + +-include_lib("eunit/include/eunit.hrl"). + + +setup_each() -> + cpse_util:dbname(). + + +teardown_each(DbName) -> + case couch_server:exists(DbName) of + true -> ok = couch_server:delete(DbName, []); + false -> ok + end. + + +cpse_open_non_existent(DbName) -> + % Try twice to check that a failed open doesn't create + % the database for some reason. + ?assertEqual({not_found, no_db_file}, cpse_util:open_db(DbName)), + ?assertEqual({not_found, no_db_file}, cpse_util:open_db(DbName)). + + +cpse_open_create(DbName) -> + ?assertEqual(false, couch_server:exists(DbName)), + ?assertEqual({not_found, no_db_file}, cpse_util:open_db(DbName)), + ?assertMatch({ok, _}, cpse_util:create_db(DbName)), + ?assertEqual(true, couch_server:exists(DbName)). + + +cpse_open_when_exists(DbName) -> + ?assertEqual(false, couch_server:exists(DbName)), + ?assertEqual({not_found, no_db_file}, cpse_util:open_db(DbName)), + ?assertMatch({ok, _}, cpse_util:create_db(DbName)), + ?assertEqual(file_exists, cpse_util:create_db(DbName)). + + +cpse_terminate(DbName) -> + ?assertEqual(false, couch_server:exists(DbName)), + ?assertEqual({not_found, no_db_file}, cpse_util:open_db(DbName)), + ?assertEqual(ok, cycle_db(DbName, create_db)), + ?assertEqual(true, couch_server:exists(DbName)). + + +cpse_rapid_recycle(DbName) -> + ?assertEqual(ok, cycle_db(DbName, create_db)), + lists:foreach(fun(_) -> + ?assertEqual(ok, cycle_db(DbName, open_db)) + end, lists:seq(1, 100)). + + +cpse_delete(DbName) -> + ?assertEqual(false, couch_server:exists(DbName)), + ?assertMatch(ok, cycle_db(DbName, create_db)), + ?assertEqual(true, couch_server:exists(DbName)), + ?assertEqual(ok, couch_server:delete(DbName, [])), + ?assertEqual(false, couch_server:exists(DbName)). + + +cycle_db(DbName, Type) -> + {ok, Db} = cpse_util:Type(DbName), + cpse_util:shutdown_db(Db). diff --git a/src/couch_pse_tests/src/cpse_test_purge_docs.erl b/src/couch_pse_tests/src/cpse_test_purge_docs.erl new file mode 100644 index 000000000..435226899 --- /dev/null +++ b/src/couch_pse_tests/src/cpse_test_purge_docs.erl @@ -0,0 +1,159 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(cpse_test_purge_docs). +-compile(export_all). + + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("couch/include/couch_db.hrl"). + + +setup_each() -> + {ok, Db} = cpse_util:create_db(), + Db. + + +teardown_each(Db) -> + ok = couch_server:delete(couch_db:name(Db), []). + + +cpse_purge_simple(Db1) -> + Actions1 = [ + {create, {<<"foo">>, {[{<<"vsn">>, 1}]}}} + ], + {ok, Db2} = cpse_util:apply_actions(Db1, Actions1), + + ?assertEqual(1, couch_db_engine:get_doc_count(Db2)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db2)), + ?assertEqual(1, couch_db_engine:get_update_seq(Db2)), + ?assertEqual(0, couch_db_engine:get_purge_seq(Db2)), + ?assertEqual([], couch_db_engine:get_last_purged(Db2)), + + [FDI] = couch_db_engine:open_docs(Db2, [<<"foo">>]), + PrevRev = cpse_util:prev_rev(FDI), + Rev = PrevRev#rev_info.rev, + + Actions2 = [ + {purge, {<<"foo">>, Rev}} + ], + {ok, Db3} = cpse_util:apply_actions(Db2, Actions2), + + ?assertEqual(0, couch_db_engine:get_doc_count(Db3)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db3)), + ?assertEqual(2, couch_db_engine:get_update_seq(Db3)), + ?assertEqual(1, couch_db_engine:get_purge_seq(Db3)), + ?assertEqual([{<<"foo">>, [Rev]}], couch_db_engine:get_last_purged(Db3)). + + +cpse_purge_conflicts(Db1) -> + Actions1 = [ + {create, {<<"foo">>, {[{<<"vsn">>, 1}]}}}, + {conflict, {<<"foo">>, {[{<<"vsn">>, 2}]}}} + ], + {ok, Db2} = cpse_util:apply_actions(Db1, Actions1), + + ?assertEqual(1, couch_db_engine:get_doc_count(Db2)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db2)), + ?assertEqual(2, couch_db_engine:get_update_seq(Db2)), + ?assertEqual(0, couch_db_engine:get_purge_seq(Db2)), + ?assertEqual([], couch_db_engine:get_last_purged(Db2)), + + [FDI1] = couch_db_engine:open_docs(Db2, [<<"foo">>]), + PrevRev1 = cpse_util:prev_rev(FDI1), + Rev1 = PrevRev1#rev_info.rev, + + Actions2 = [ + {purge, {<<"foo">>, Rev1}} + ], + {ok, Db3} = cpse_util:apply_actions(Db2, Actions2), + + ?assertEqual(1, couch_db_engine:get_doc_count(Db3)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db3)), + ?assertEqual(4, couch_db_engine:get_update_seq(Db3)), + ?assertEqual(1, couch_db_engine:get_purge_seq(Db3)), + ?assertEqual([{<<"foo">>, [Rev1]}], couch_db_engine:get_last_purged(Db3)), + + [FDI2] = couch_db_engine:open_docs(Db3, [<<"foo">>]), + PrevRev2 = cpse_util:prev_rev(FDI2), + Rev2 = PrevRev2#rev_info.rev, + + Actions3 = [ + {purge, {<<"foo">>, Rev2}} + ], + {ok, Db4} = cpse_util:apply_actions(Db3, Actions3), + + ?assertEqual(0, couch_db_engine:get_doc_count(Db4)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db4)), + ?assertEqual(5, couch_db_engine:get_update_seq(Db4)), + ?assertEqual(2, couch_db_engine:get_purge_seq(Db4)), + ?assertEqual([{<<"foo">>, [Rev2]}], couch_db_engine:get_last_purged(Db4)). + + +cpse_add_delete_purge(Db1) -> + Actions1 = [ + {create, {<<"foo">>, {[{<<"vsn">>, 1}]}}}, + {delete, {<<"foo">>, {[{<<"vsn">>, 2}]}}} + ], + + {ok, Db2} = cpse_util:apply_actions(Db1, Actions1), + + ?assertEqual(0, couch_db_engine:get_doc_count(Db2)), + ?assertEqual(1, couch_db_engine:get_del_doc_count(Db2)), + ?assertEqual(2, couch_db_engine:get_update_seq(Db2)), + ?assertEqual(0, couch_db_engine:get_purge_seq(Db2)), + ?assertEqual([], couch_db_engine:get_last_purged(Db2)), + + [FDI] = couch_db_engine:open_docs(Db2, [<<"foo">>]), + PrevRev = cpse_util:prev_rev(FDI), + Rev = PrevRev#rev_info.rev, + + Actions2 = [ + {purge, {<<"foo">>, Rev}} + ], + {ok, Db3} = cpse_util:apply_actions(Db2, Actions2), + + ?assertEqual(0, couch_db_engine:get_doc_count(Db3)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db3)), + ?assertEqual(3, couch_db_engine:get_update_seq(Db3)), + ?assertEqual(1, couch_db_engine:get_purge_seq(Db3)), + ?assertEqual([{<<"foo">>, [Rev]}], couch_db_engine:get_last_purged(Db3)). + + +cpse_add_two_purge_one(Db1) -> + Actions1 = [ + {create, {<<"foo">>, {[{<<"vsn">>, 1}]}}}, + {create, {<<"bar">>, {[]}}} + ], + + {ok, Db2} = cpse_util:apply_actions(Db1, Actions1), + + ?assertEqual(2, couch_db_engine:get_doc_count(Db2)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db2)), + ?assertEqual(2, couch_db_engine:get_update_seq(Db2)), + ?assertEqual(0, couch_db_engine:get_purge_seq(Db2)), + ?assertEqual([], couch_db_engine:get_last_purged(Db2)), + + [FDI] = couch_db_engine:open_docs(Db2, [<<"foo">>]), + PrevRev = cpse_util:prev_rev(FDI), + Rev = PrevRev#rev_info.rev, + + Actions2 = [ + {purge, {<<"foo">>, Rev}} + ], + {ok, Db3} = cpse_util:apply_actions(Db2, Actions2), + + ?assertEqual(1, couch_db_engine:get_doc_count(Db3)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db3)), + ?assertEqual(3, couch_db_engine:get_update_seq(Db3)), + ?assertEqual(1, couch_db_engine:get_purge_seq(Db3)), + ?assertEqual([{<<"foo">>, [Rev]}], couch_db_engine:get_last_purged(Db3)). diff --git a/src/couch_pse_tests/src/cpse_test_read_write_docs.erl b/src/couch_pse_tests/src/cpse_test_read_write_docs.erl new file mode 100644 index 000000000..84bf9f323 --- /dev/null +++ b/src/couch_pse_tests/src/cpse_test_read_write_docs.erl @@ -0,0 +1,318 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(cpse_test_read_write_docs). +-compile(export_all). + + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("couch/include/couch_db.hrl"). + + +setup_each() -> + {ok, Db} = cpse_util:create_db(), + Db. + + +teardown_each(Db) -> + ok = couch_server:delete(couch_db:name(Db), []). + + +cpse_read_docs_from_empty_db(Db) -> + ?assertEqual([not_found], couch_db_engine:open_docs(Db, [<<"foo">>])), + ?assertEqual( + [not_found, not_found], + couch_db_engine:open_docs(Db, [<<"a">>, <<"b">>]) + ). + + +cpse_read_empty_local_docs(Db) -> + {LocalA, LocalB} = {<<"_local/a">>, <<"_local/b">>}, + ?assertEqual([not_found], couch_db_engine:open_local_docs(Db, [LocalA])), + ?assertEqual( + [not_found, not_found], + couch_db_engine:open_local_docs(Db, [LocalA, LocalB]) + ). + + +cpse_write_one_doc(Db1) -> + ?assertEqual(0, couch_db_engine:get_doc_count(Db1)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db1)), + ?assertEqual(0, couch_db_engine:get_update_seq(Db1)), + + Actions = [ + {create, {<<"foo">>, {[{<<"vsn">>, 1}]}}} + ], + {ok, Db2} = cpse_util:apply_actions(Db1, Actions), + ?assertEqual(1, couch_db_engine:get_doc_count(Db2)), + + {ok, _} = couch_db:ensure_full_commit(Db2), + cpse_util:shutdown_db(Db2), + + {ok, Db3} = couch_db:reopen(Db2), + + ?assertEqual(1, couch_db_engine:get_doc_count(Db3)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db3)), + ?assertEqual(1, couch_db_engine:get_update_seq(Db3)), + + [FDI] = couch_db_engine:open_docs(Db3, [<<"foo">>]), + #rev_info{ + rev = {RevPos, PrevRevId}, + deleted = Deleted, + body_sp = DocPtr + } = cpse_util:prev_rev(FDI), + + Doc0 = #doc{ + id = <<"foo">>, + revs = {RevPos, [PrevRevId]}, + deleted = Deleted, + body = DocPtr + }, + + Doc1 = couch_db_engine:read_doc_body(Db3, Doc0), + Body1 = if not is_binary(Doc1#doc.body) -> Doc1#doc.body; true -> + couch_compress:decompress(Doc1#doc.body) + end, + ?assertEqual({[{<<"vsn">>, 1}]}, Body1). + + +cpse_write_two_docs(Db1) -> + ?assertEqual(0, couch_db_engine:get_doc_count(Db1)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db1)), + ?assertEqual(0, couch_db_engine:get_update_seq(Db1)), + + Actions = [ + {create, {<<"foo">>, {[{<<"vsn">>, 1}]}}}, + {create, {<<"bar">>, {[{<<"stuff">>, true}]}}} + ], + {ok, Db2} = cpse_util:apply_actions(Db1, Actions), + {ok, _} = couch_db:ensure_full_commit(Db2), + cpse_util:shutdown_db(Db2), + + {ok, Db3} = couch_db:reopen(Db2), + + ?assertEqual(2, couch_db_engine:get_doc_count(Db3)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db3)), + ?assertEqual(2, couch_db_engine:get_update_seq(Db3)), + + Resps = couch_db_engine:open_docs(Db3, [<<"foo">>, <<"bar">>]), + ?assertEqual(false, lists:member(not_found, Resps)). + + +cpse_write_three_doc_batch(Db1) -> + ?assertEqual(0, couch_db_engine:get_doc_count(Db1)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db1)), + ?assertEqual(0, couch_db_engine:get_update_seq(Db1)), + + Actions = [ + {batch, [ + {create, {<<"foo">>, {[{<<"vsn">>, 1}]}}}, + {create, {<<"bar">>, {[{<<"stuff">>, true}]}}}, + {create, {<<"baz">>, {[]}}} + ]} + ], + {ok, Db2} = cpse_util:apply_actions(Db1, Actions), + {ok, _} = couch_db:ensure_full_commit(Db2), + cpse_util:shutdown_db(Db2), + + {ok, Db3} = couch_db:reopen(Db2), + + ?assertEqual(3, couch_db_engine:get_doc_count(Db3)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db3)), + ?assertEqual(3, couch_db_engine:get_update_seq(Db3)), + + Resps = couch_db_engine:open_docs(Db3, [<<"foo">>, <<"bar">>, <<"baz">>]), + ?assertEqual(false, lists:member(not_found, Resps)). + + +cpse_update_doc(Db1) -> + ?assertEqual(0, couch_db_engine:get_doc_count(Db1)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db1)), + ?assertEqual(0, couch_db_engine:get_update_seq(Db1)), + + Actions = [ + {create, {<<"foo">>, {[{<<"vsn">>, 1}]}}}, + {update, {<<"foo">>, {[{<<"vsn">>, 2}]}}} + ], + {ok, Db2} = cpse_util:apply_actions(Db1, Actions), + {ok, _} = couch_db:ensure_full_commit(Db2), + cpse_util:shutdown_db(Db2), + + {ok, Db3} = couch_db:reopen(Db2), + + ?assertEqual(1, couch_db_engine:get_doc_count(Db3)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db3)), + ?assertEqual(2, couch_db_engine:get_update_seq(Db3)), + + [FDI] = couch_db_engine:open_docs(Db3, [<<"foo">>]), + + #rev_info{ + rev = {RevPos, PrevRevId}, + deleted = Deleted, + body_sp = DocPtr + } = cpse_util:prev_rev(FDI), + + Doc0 = #doc{ + id = <<"foo">>, + revs = {RevPos, [PrevRevId]}, + deleted = Deleted, + body = DocPtr + }, + + Doc1 = couch_db_engine:read_doc_body(Db3, Doc0), + Body1 = if not is_binary(Doc1#doc.body) -> Doc1#doc.body; true -> + couch_compress:decompress(Doc1#doc.body) + end, + + ?assertEqual({[{<<"vsn">>, 2}]}, Body1). + + +cpse_delete_doc(Db1) -> + ?assertEqual(0, couch_db_engine:get_doc_count(Db1)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db1)), + ?assertEqual(0, couch_db_engine:get_update_seq(Db1)), + + Actions = [ + {create, {<<"foo">>, {[{<<"vsn">>, 1}]}}}, + {delete, {<<"foo">>, {[]}}} + ], + {ok, Db2} = cpse_util:apply_actions(Db1, Actions), + {ok, _} = couch_db:ensure_full_commit(Db2), + cpse_util:shutdown_db(Db2), + + {ok, Db3} = couch_db:reopen(Db2), + ?assertEqual(0, couch_db_engine:get_doc_count(Db3)), + ?assertEqual(1, couch_db_engine:get_del_doc_count(Db3)), + ?assertEqual(2, couch_db_engine:get_update_seq(Db3)), + + [FDI] = couch_db_engine:open_docs(Db3, [<<"foo">>]), + + #rev_info{ + rev = {RevPos, PrevRevId}, + deleted = Deleted, + body_sp = DocPtr + } = cpse_util:prev_rev(FDI), + + Doc0 = #doc{ + id = <<"foo">>, + revs = {RevPos, [PrevRevId]}, + deleted = Deleted, + body = DocPtr + }, + + Doc1 = couch_db_engine:read_doc_body(Db3, Doc0), + Body1 = if not is_binary(Doc1#doc.body) -> Doc1#doc.body; true -> + couch_compress:decompress(Doc1#doc.body) + end, + + ?assertEqual({[]}, Body1). + + +cpse_write_local_doc(Db1) -> + ?assertEqual(0, couch_db_engine:get_doc_count(Db1)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db1)), + ?assertEqual(0, couch_db_engine:get_update_seq(Db1)), + + Actions = [ + {create, {<<"_local/foo">>, {[{<<"yay">>, false}]}}} + ], + {ok, Db2} = cpse_util:apply_actions(Db1, Actions), + {ok, _} = couch_db:ensure_full_commit(Db2), + cpse_util:shutdown_db(Db2), + + {ok, Db3} = couch_db:reopen(Db2), + + ?assertEqual(0, couch_db_engine:get_doc_count(Db3)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db3)), + ?assertEqual(0, couch_db_engine:get_update_seq(Db3)), + + [not_found] = couch_db_engine:open_docs(Db3, [<<"_local/foo">>]), + [#doc{} = Doc] = couch_db_engine:open_local_docs(Db3, [<<"_local/foo">>]), + ?assertEqual({[{<<"yay">>, false}]}, Doc#doc.body). + + +cpse_write_mixed_batch(Db1) -> + ?assertEqual(0, couch_db_engine:get_doc_count(Db1)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db1)), + ?assertEqual(0, couch_db_engine:get_update_seq(Db1)), + + Actions = [ + {batch, [ + {create, {<<"bar">>, {[]}}}, + {create, {<<"_local/foo">>, {[{<<"yay">>, false}]}}} + ]} + ], + {ok, Db2} = cpse_util:apply_actions(Db1, Actions), + {ok, _} = couch_db:ensure_full_commit(Db2), + cpse_util:shutdown_db(Db2), + + {ok, Db3} = couch_db:reopen(Db2), + + ?assertEqual(1, couch_db_engine:get_doc_count(Db3)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db3)), + ?assertEqual(1, couch_db_engine:get_update_seq(Db3)), + + [#full_doc_info{}] = couch_db_engine:open_docs(Db3, [<<"bar">>]), + [not_found] = couch_db_engine:open_docs(Db3, [<<"_local/foo">>]), + + [not_found] = couch_db_engine:open_local_docs(Db3, [<<"bar">>]), + [#doc{}] = couch_db_engine:open_local_docs(Db3, [<<"_local/foo">>]). + + +cpse_update_local_doc(Db1) -> + ?assertEqual(0, couch_db_engine:get_doc_count(Db1)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db1)), + ?assertEqual(0, couch_db_engine:get_update_seq(Db1)), + + Actions = [ + {create, {<<"_local/foo">>, {[]}}}, + {update, {<<"_local/foo">>, {[{<<"stuff">>, null}]}}} + ], + {ok, Db2} = cpse_util:apply_actions(Db1, Actions), + {ok, _} = couch_db:ensure_full_commit(Db1), + cpse_util:shutdown_db(Db2), + + {ok, Db3} = couch_db:reopen(Db2), + + ?assertEqual(0, couch_db_engine:get_doc_count(Db3)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db3)), + ?assertEqual(0, couch_db_engine:get_update_seq(Db3)), + + [not_found] = couch_db_engine:open_docs(Db3, [<<"_local/foo">>]), + [#doc{} = Doc] = couch_db_engine:open_local_docs(Db3, [<<"_local/foo">>]), + ?assertEqual({[{<<"stuff">>, null}]}, Doc#doc.body). + + +cpse_delete_local_doc(Db1) -> + ?assertEqual(0, couch_db_engine:get_doc_count(Db1)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db1)), + ?assertEqual(0, couch_db_engine:get_update_seq(Db1)), + + Actions = [ + {create, {<<"_local/foo">>, []}}, + {delete, {<<"_local/foo">>, []}} + ], + {ok, Db2} = cpse_util:apply_actions(Db1, Actions), + {ok, _} = couch_db:ensure_full_commit(Db2), + cpse_util:shutdown_db(Db2), + + {ok, Db3} = couch_db:reopen(Db2), + + ?assertEqual(0, couch_db_engine:get_doc_count(Db3)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db3)), + ?assertEqual(0, couch_db_engine:get_update_seq(Db3)), + + [not_found] = couch_db_engine:open_docs(Db3, [<<"_local/foo">>]), + ?assertEqual( + [not_found], + couch_db_engine:open_local_docs(Db3, [<<"_local/foo">>]) + ). diff --git a/src/couch/src/test_engine_ref_counting.erl b/src/couch_pse_tests/src/cpse_test_ref_counting.erl index 18e75fb5a..2a0e4c202 100644 --- a/src/couch/src/test_engine_ref_counting.erl +++ b/src/couch_pse_tests/src/cpse_test_ref_counting.erl @@ -10,7 +10,7 @@ % License for the specific language governing permissions and limitations under % the License. --module(test_engine_ref_counting). +-module(cpse_test_ref_counting). -compile(export_all). @@ -21,49 +21,59 @@ -define(NUM_CLIENTS, 1000). -cet_empty_monitors() -> - {ok, Engine, St} = test_engine_util:init_engine(), - Pids = Engine:monitored_by(St), - ?assert(is_list(Pids)), - ?assertEqual([], Pids -- [self(), whereis(couch_stats_process_tracker)]). +setup_each() -> + {ok, Db} = cpse_util:create_db(), + {Db, self()}. + +teardown_each({Db, _}) -> + ok = couch_server:delete(couch_db:name(Db), []). -cet_incref_decref() -> - {ok, Engine, St} = test_engine_util:init_engine(), - {Pid, _} = Client = start_client(Engine, St), +cpse_empty_monitors({Db, Pid}) -> + Pids = couch_db_engine:monitored_by(Db), + ?assert(is_list(Pids)), + Expected = [ + Pid, + couch_db:get_pid(Db), + whereis(couch_stats_process_tracker) + ], + ?assertEqual([], Pids -- Expected). + + +cpse_incref_decref({Db, _}) -> + {Pid, _} = Client = start_client(Db), wait_client(Client), - Pids1 = Engine:monitored_by(St), + Pids1 = couch_db_engine:monitored_by(Db), ?assert(lists:member(Pid, Pids1)), close_client(Client), - Pids2 = Engine:monitored_by(St), + Pids2 = couch_db_engine:monitored_by(Db), ?assert(not lists:member(Pid, Pids2)). -cet_incref_decref_many() -> - {ok, Engine, St} = test_engine_util:init_engine(), +cpse_incref_decref_many({Db, _}) -> Clients = lists:map(fun(_) -> - start_client(Engine, St) + start_client(Db) end, lists:seq(1, ?NUM_CLIENTS)), lists:foreach(fun(C) -> wait_client(C) end, Clients), - Pids1 = Engine:monitored_by(St), - % +2 for db pid and process tracker - ?assertEqual(?NUM_CLIENTS + 2, length(Pids1)), + Pids1 = couch_db_engine:monitored_by(Db), + % +3 for self, db pid, and process tracker + ?assertEqual(?NUM_CLIENTS + 3, length(Pids1)), lists:foreach(fun(C) -> close_client(C) end, Clients), - Pids2 = Engine:monitored_by(St), - ?assertEqual(2, length(Pids2)). + Pids2 = couch_db_engine:monitored_by(Db), + ?assertEqual(3, length(Pids2)). -start_client(Engine, St1) -> +start_client(Db0) -> spawn_monitor(fun() -> - {ok, St2} = Engine:incref(St1), + {ok, Db1} = couch_db:open_int(couch_db:name(Db0), []), receive {waiting, Pid} -> @@ -74,12 +84,11 @@ start_client(Engine, St1) -> receive close -> + couch_db:close(Db1), ok after 1000 -> erlang:error(timeout) - end, - - Engine:decref(St2) + end end). diff --git a/src/couch_pse_tests/src/cpse_util.erl b/src/couch_pse_tests/src/cpse_util.erl new file mode 100644 index 000000000..ff119519d --- /dev/null +++ b/src/couch_pse_tests/src/cpse_util.erl @@ -0,0 +1,515 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(cpse_util). +-compile(export_all). + + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("couch/include/couch_db.hrl"). + + +-define(TEST_MODULES, [ + cpse_test_open_close_delete, + cpse_test_get_set_props, + cpse_test_read_write_docs, + cpse_test_attachments, + cpse_test_fold_docs, + cpse_test_fold_changes, + cpse_test_purge_docs, + cpse_test_compaction, + cpse_test_ref_counting +]). + + +-define(SHUTDOWN_TIMEOUT, 5000). +-define(COMPACTOR_TIMEOUT, 50000). +-define(ATTACHMENT_WRITE_TIMEOUT, 10000). +-define(MAKE_DOC_SUMMARY_TIMEOUT, 5000). + + +create_tests(EngineApp, Extension) -> + create_tests(EngineApp, EngineApp, Extension). + + +create_tests(EngineApp, EngineModule, Extension) -> + TestEngine = {EngineApp, EngineModule, Extension}, + application:set_env(couch, test_engine, TestEngine), + lists:map(fun(TestMod) -> + {atom_to_list(TestMod), cpse_gather:module(TestMod)} + end, ?TEST_MODULES). + + +setup_all() -> + setup_all([]). + + +setup_all(ExtraApps) -> + Ctx = test_util:start_couch(ExtraApps), + {ok, {_, EngineMod, Extension}} = application:get_env(couch, test_engine), + EngineModStr = atom_to_list(EngineMod), + config:set("couchdb_engines", Extension, EngineModStr, false), + config:set("log", "include_sasl", "false", false), + Ctx. + + +teardown_all(Ctx) -> + test_util:stop_couch(Ctx). + + +rootdir() -> + config:get("couchdb", "database_dir", "."). + + +dbname() -> + UUID = couch_uuids:random(), + <<"db-", UUID/binary>>. + + +get_engine() -> + case application:get_env(couch, test_engine) of + {ok, {_App, _Mod, Extension}} -> + list_to_binary(Extension); + _ -> + <<"couch">> + end. + + +create_db() -> + create_db(dbname()). + + +create_db(DbName) -> + Engine = get_engine(), + couch_db:create(DbName, [{engine, Engine}, ?ADMIN_CTX]). + + +open_db(DbName) -> + Engine = get_engine(), + couch_db:open_int(DbName, [{engine, Engine}, ?ADMIN_CTX]). + + +shutdown_db(Db) -> + Pid = couch_db:get_pid(Db), + Ref = erlang:monitor(process, Pid), + exit(Pid, kill), + receive + {'DOWN', Ref, _, _, _} -> + ok + after ?SHUTDOWN_TIMEOUT -> + erlang:error(database_shutdown_timeout) + end, + test_util:wait(fun() -> + case ets:member(couch_dbs, couch_db:name(Db)) of + true -> wait; + false -> ok + end + end). + + +apply_actions(Db, []) -> + {ok, Db}; + +apply_actions(Db, [Action | Rest]) -> + {ok, NewDb} = apply_action(Db, Action), + apply_actions(NewDb, Rest). + + +apply_action(Db, {batch, BatchActions}) -> + apply_batch(Db, BatchActions); + +apply_action(Db, Action) -> + apply_batch(Db, [Action]). + + +apply_batch(Db, Actions) -> + AccIn = {[], [], [], []}, + AccOut = lists:foldl(fun(Action, Acc) -> + {DocAcc, ConfAcc, LDocAcc, PurgeAcc} = Acc, + case gen_write(Db, Action) of + {update, Doc} -> + {[Doc | DocAcc], ConfAcc, LDocAcc, PurgeAcc}; + {conflict, Doc} -> + {DocAcc, [Doc | ConfAcc], LDocAcc, PurgeAcc}; + {local, Doc} -> + {DocAcc, ConfAcc, [Doc | LDocAcc], PurgeAcc}; + {purge, PurgeInfo} -> + {DocAcc, ConfAcc, LDocAcc, [PurgeInfo | PurgeAcc]} + end + end, AccIn, Actions), + + {Docs0, Conflicts0, LDocs0, PurgeInfos0} = AccOut, + Docs = lists:reverse(Docs0), + Conflicts = lists:reverse(Conflicts0), + LDocs = lists:reverse(LDocs0), + PurgeInfos = lists:reverse(PurgeInfos0), + + {ok, Resp} = couch_db:update_docs(Db, Docs ++ LDocs), + false = lists:member(conflict, Resp), + {ok, Db1} = couch_db:reopen(Db), + + {ok, []} = couch_db:update_docs(Db, Conflicts, [], replicated_changes), + {ok, Db2} = couch_db:reopen(Db1), + + if PurgeInfos == [] -> ok; true -> + {ok, _, _} = couch_db:purge_docs(Db2, PurgeInfos) + end, + couch_db:reopen(Db2). + + +gen_write(Db, {Action, {<<"_local/", _/binary>> = DocId, Body}}) -> + PrevRev = case couch_db:open_doc(Db, DocId) of + {not_found, _} -> + 0; + {ok, #doc{revs = {0, []}}} -> + 0; + {ok, #doc{revs = {0, [RevStr | _]}}} -> + binary_to_integer(RevStr) + end, + {RevId, Deleted} = case Action of + Action when Action == create; Action == update -> + {PrevRev + 1, false}; + delete -> + {0, true} + end, + {local, #doc{ + id = DocId, + revs = {0, [list_to_binary(integer_to_list(RevId))]}, + body = Body, + deleted = Deleted + }}; + +gen_write(Db, {Action, {DocId, Body}}) -> + gen_write(Db, {Action, {DocId, Body, []}}); + +gen_write(Db, {create, {DocId, Body, Atts}}) -> + {not_found, _} = couch_db:open_doc(Db, DocId), + {update, #doc{ + id = DocId, + revs = {0, []}, + deleted = false, + body = Body, + atts = Atts + }}; + +gen_write(_Db, {purge, {DocId, PrevRevs0, _}}) -> + PrevRevs = if is_list(PrevRevs0) -> PrevRevs0; true -> [PrevRevs0] end, + {purge, {DocId, PrevRevs}}; + +gen_write(Db, {Action, {DocId, Body, Atts}}) -> + #full_doc_info{} = PrevFDI = couch_db:get_full_doc_info(Db, DocId), + + #full_doc_info{ + id = DocId + } = PrevFDI, + + #rev_info{ + rev = PrevRev + } = prev_rev(PrevFDI), + + NewRev = gen_rev(Action, DocId, PrevRev, Body, Atts), + + Deleted = case Action of + update -> false; + conflict -> false; + delete -> true + end, + + Type = case Action of + conflict -> conflict; + _ -> update + end, + + {Type, #doc{ + id = DocId, + revs = NewRev, + deleted = Deleted, + body = Body, + atts = Atts + }}. + + +gen_rev(A, DocId, {Pos, Rev}, Body, Atts) when A == update; A == delete -> + NewRev = crypto:hash(md5, term_to_binary({DocId, Rev, Body, Atts})), + {Pos + 1, [NewRev, Rev]}; +gen_rev(conflict, DocId, _, Body, Atts) -> + UUID = couch_uuids:random(), + NewRev = crypto:hash(md5, term_to_binary({DocId, UUID, Body, Atts})), + {1, [NewRev]}. + + +prep_atts(_Db, []) -> + []; + +prep_atts(Db, [{FileName, Data} | Rest]) -> + {_, Ref} = spawn_monitor(fun() -> + {ok, Stream} = couch_db:open_write_stream(Db, []), + exit(write_att(Stream, FileName, Data, Data)) + end), + Att = receive + {'DOWN', Ref, _, _, {{no_catch, not_supported}, _}} -> + throw(not_supported); + {'DOWN', Ref, _, _, Resp} -> + Resp + after ?ATTACHMENT_WRITE_TIMEOUT -> + erlang:error(attachment_write_timeout) + end, + [Att | prep_atts(Db, Rest)]. + + +write_att(Stream, FileName, OrigData, <<>>) -> + {StreamEngine, Len, Len, Md5, Md5} = couch_stream:close(Stream), + couch_util:check_md5(Md5, couch_hash:md5_hash(OrigData)), + Len = size(OrigData), + couch_att:new([ + {name, FileName}, + {type, <<"application/octet-stream">>}, + {data, {stream, StreamEngine}}, + {att_len, Len}, + {disk_len, Len}, + {md5, Md5}, + {encoding, identity} + ]); + +write_att(Stream, FileName, OrigData, Data) -> + {Chunk, Rest} = case size(Data) > 4096 of + true -> + <<Head:4096/binary, Tail/binary>> = Data, + {Head, Tail}; + false -> + {Data, <<>>} + end, + ok = couch_stream:write(Stream, Chunk), + write_att(Stream, FileName, OrigData, Rest). + + +prev_rev(#full_doc_info{} = FDI) -> + #doc_info{ + revs = [#rev_info{} = PrevRev | _] + } = couch_doc:to_doc_info(FDI), + PrevRev. + + +db_as_term(Db) -> + [ + {props, db_props_as_term(Db)}, + {docs, db_docs_as_term(Db)}, + {local_docs, db_local_docs_as_term(Db)}, + {changes, db_changes_as_term(Db)} + ]. + + +db_props_as_term(Db) -> + Props = [ + get_doc_count, + get_del_doc_count, + get_disk_version, + get_update_seq, + get_purge_seq, + get_last_purged, + get_security, + get_revs_limit, + get_uuid, + get_epochs + ], + lists:map(fun(Fun) -> + {Fun, couch_db_engine:Fun(Db)} + end, Props). + + +db_docs_as_term(Db) -> + FoldFun = fun(FDI, Acc) -> {ok, [FDI | Acc]} end, + {ok, FDIs} = couch_db:fold_docs(Db, FoldFun, [], []), + lists:reverse(lists:map(fun(FDI) -> + fdi_to_term(Db, FDI) + end, FDIs)). + + +db_local_docs_as_term(Db) -> + FoldFun = fun(Doc, Acc) -> {ok, [Doc | Acc]} end, + {ok, LDocs} = couch_db:fold_local_docs(Db, FoldFun, [], []), + lists:reverse(LDocs). + + +db_changes_as_term(Db) -> + FoldFun = fun(FDI, Acc) -> {ok, [FDI | Acc]} end, + {ok, Changes} = couch_db:fold_changes(Db, 0, FoldFun, [], []), + lists:reverse(lists:map(fun(FDI) -> + fdi_to_term(Db, FDI) + end, Changes)). + + +fdi_to_term(Db, FDI) -> + #full_doc_info{ + id = DocId, + rev_tree = OldTree + } = FDI, + {NewRevTree, _} = couch_key_tree:mapfold(fun(Rev, Node, Type, Acc) -> + tree_to_term(Rev, Node, Type, Acc, DocId) + end, Db, OldTree), + FDI#full_doc_info{ + rev_tree = NewRevTree, + % Blank out sizes because we allow storage + % engines to handle this with their own + % definition until further notice. + sizes = #size_info{ + active = -1, + external = -1 + } + }. + + +tree_to_term(_Rev, _Leaf, branch, Acc, _DocId) -> + {?REV_MISSING, Acc}; + +tree_to_term({Pos, RevId}, #leaf{} = Leaf, leaf, Db, DocId) -> + #leaf{ + deleted = Deleted, + ptr = Ptr + } = Leaf, + + Doc0 = #doc{ + id = DocId, + revs = {Pos, [RevId]}, + deleted = Deleted, + body = Ptr + }, + + Doc1 = couch_db_engine:read_doc_body(Db, Doc0), + + Body = if not is_binary(Doc1#doc.body) -> Doc1#doc.body; true -> + couch_compress:decompress(Doc1#doc.body) + end, + + Atts1 = if not is_binary(Doc1#doc.atts) -> Doc1#doc.atts; true -> + couch_compress:decompress(Doc1#doc.atts) + end, + + StreamSrc = fun(Sp) -> couch_db:open_read_stream(Db, Sp) end, + Atts2 = [couch_att:from_disk_term(StreamSrc, Att) || Att <- Atts1], + Atts = [att_to_term(Att) || Att <- Atts2], + + NewLeaf = Leaf#leaf{ + ptr = Body, + sizes = #size_info{active = -1, external = -1}, + atts = Atts + }, + {NewLeaf, Db}. + + +att_to_term(Att) -> + Bin = couch_att:to_binary(Att), + couch_att:store(data, Bin, Att). + + +term_diff(T1, T2) when is_tuple(T1), is_tuple(T2) -> + tuple_diff(tuple_to_list(T1), tuple_to_list(T2)); + +term_diff(L1, L2) when is_list(L1), is_list(L2) -> + list_diff(L1, L2); + +term_diff(V1, V2) when V1 == V2 -> + nodiff; + +term_diff(V1, V2) -> + {V1, V2}. + + +tuple_diff([], []) -> + nodiff; + +tuple_diff([T1 | _], []) -> + {longer, T1}; + +tuple_diff([], [T2 | _]) -> + {shorter, T2}; + +tuple_diff([T1 | R1], [T2 | R2]) -> + case term_diff(T1, T2) of + nodiff -> + tuple_diff(R1, R2); + Else -> + {T1, Else} + end. + + +list_diff([], []) -> + nodiff; + +list_diff([T1 | _], []) -> + {longer, T1}; + +list_diff([], [T2 | _]) -> + {shorter, T2}; + +list_diff([T1 | R1], [T2 | R2]) -> + case term_diff(T1, T2) of + nodiff -> + list_diff(R1, R2); + Else -> + {T1, Else} + end. + + +compact(Db) -> + {ok, Pid} = couch_db:start_compact(Db), + Ref = erlang:monitor(process, Pid), + + % Ideally I'd assert that Pid is linked to us + % at this point but its technically possible + % that it could have finished compacting by + % the time we check... Quite the quandry. + + receive + {'DOWN', Ref, _, _, normal} -> + ok; + {'DOWN', Ref, _, _, noproc} -> + ok; + {'DOWN', Ref, _, _, Reason} -> + erlang:error({compactor_died, Reason}) + after ?COMPACTOR_TIMEOUT -> + erlang:error(compactor_timed_out) + end, + + test_util:wait(fun() -> + {ok, Db2} = couch_db:open_int(couch_db:name(Db), []), + try + CPid = couch_db:get_compactor_pid(Db2), + case is_pid(CPid) of + true -> wait; + false -> ok + end + after + couch_db:close(Db2) + end + end). + + +with_config(Config, Fun) -> + OldConfig = apply_config(Config), + try + Fun() + after + apply_config(OldConfig) + end. + + +apply_config([]) -> + []; + +apply_config([{Section, Key, Value} | Rest]) -> + Orig = config:get(Section, Key), + case Value of + undefined -> config:delete(Section, Key, false); + _ -> config:set(Section, Key, Value, false) + end, + [{Section, Key, Orig} | apply_config(Rest)]. |