diff options
author | jiangphcn <jiangph@cn.ibm.com> | 2018-05-02 16:52:46 +0800 |
---|---|---|
committer | jiangphcn <jiangph@cn.ibm.com> | 2018-05-02 16:52:46 +0800 |
commit | e402b0c27239e81f4e2ddc1de07fb4356216f96e (patch) | |
tree | 3b6a8f00656e1c4ba3b1b369c0001cf7bba7a003 | |
parent | 6184bfbbe577de4cf09eb65f7406ccf26e9efa8a (diff) | |
download | couchdb-e402b0c27239e81f4e2ddc1de07fb4356216f96e.tar.gz |
Merge from COUCHDB-3326-clustered-purge-davisp-refactor-2
COUCHDB-3326
24 files changed, 2085 insertions, 1811 deletions
diff --git a/src/chttpd/test/chttpd_purge_tests.erl b/src/chttpd/test/chttpd_purge_tests.erl index ca4eddabe..be60db851 100644 --- a/src/chttpd/test/chttpd_purge_tests.erl +++ b/src/chttpd/test/chttpd_purge_tests.erl @@ -65,6 +65,9 @@ purge_test_() -> fun teardown/1, [ fun test_empty_purge_request/1, + fun test_chttpd_status_code_201/1, + %fun test_chttpd_status_code_202/1, + %fun test_chttpd_status_code_500/1, fun test_ok_purge_request/1, fun test_exceed_limits_on_purge_infos/1, fun should_error_set_purged_docs_limit_to0/1 @@ -91,6 +94,23 @@ test_empty_purge_request(Url) -> end). +test_chttpd_status_code_201(Url) -> + ?_test(begin + IdsRevs = "{}", + {ok, Status, _, ResultBody} = test_request:post(Url ++ "/_purge/", + [?CONTENT_JSON, ?AUTH], IdsRevs), + ResultJson = ?JSON_DECODE(ResultBody), + ?assert(Status =:= 201 orelse Status =:= 202), + ?assertEqual( + {[ + {<<"purge_seq">>, null}, + {<<"purged">>,{[]}} + ]}, + ResultJson + ) + end). + + test_ok_purge_request(Url) -> ?_test(begin {ok, _, _, Body} = create_doc(Url, "doc1"), diff --git a/src/couch/src/couch_bt_engine.erl b/src/couch/src/couch_bt_engine.erl index bfecca0b5..f4d37aa88 100644 --- a/src/couch/src/couch_bt_engine.erl +++ b/src/couch/src/couch_bt_engine.erl @@ -850,7 +850,7 @@ init_state(FilePath, Fd, Header0, Options) -> % to be written to disk. case Header /= Header0 of true -> - {ok, NewSt} = commit_data(St), + {ok, NewSt} = commit_data(St#st{needs_commit = true}), NewSt; false -> St diff --git a/src/couch/src/couch_bt_engine_compactor.erl b/src/couch/src/couch_bt_engine_compactor.erl index fe78f4319..370cae26b 100644 --- a/src/couch/src/couch_bt_engine_compactor.erl +++ b/src/couch/src/couch_bt_engine_compactor.erl @@ -104,10 +104,9 @@ open_compaction_files(SrcHdr, DbFilePath, Options) -> copy_purge_info(DbName, OldSt, NewSt, Retry) -> - %MinPurgeSeq = couch_util:with_db(DbName, fun(Db) -> - % couch_db:get_minimum_purge_seq(Db) - %end), - MinPurgeSeq = 0, + MinPurgeSeq = couch_util:with_db(DbName, fun(Db) -> + couch_db:get_minimum_purge_seq(Db) + end), OldPSTree = OldSt#st.purge_seq_tree, StartSeq = couch_bt_engine:get_purge_seq(NewSt) + 1, BufferSize = config:get_integer( @@ -140,28 +139,32 @@ copy_purge_info(DbName, OldSt, NewSt, Retry) -> copy_purge_infos(OldSt, NewStAcc, Infos, MinPurgeSeq, Retry). -copy_purge_infos(OldSt, NewSt, Infos, MinPurgeSeq, Retry) -> +copy_purge_infos(OldSt, NewSt0, Infos, MinPurgeSeq, Retry) -> #st{ id_tree = OldIdTree } = OldSt, + + % Re-bind our id_tree to the backing btree + NewIdTreeState = couch_bt_engine_header:id_tree_state(NewSt0#st.header), + MetaFd = couch_emsort:get_fd(NewSt0#st.id_tree), + MetaState = couch_emsort:get_state(NewSt0#st.id_tree), + NewSt1 = bind_id_tree(NewSt0, NewSt0#st.fd, NewIdTreeState), + #st{ id_tree = NewIdTree0, seq_tree = NewSeqTree0, purge_tree = NewPurgeTree0, purge_seq_tree = NewPurgeSeqTree0 - } = copy_meta_data(NewSt), + } = NewSt1, % Copy over the purge infos - InfosToAdd = lists:takewhile(fun({PSeq, _, _, _}) -> + InfosToAdd = lists:filter(fun({PSeq, _, _, _}) -> PSeq > MinPurgeSeq end, Infos), {ok, NewPurgeTree1} = couch_btree:add(NewPurgeTree0, InfosToAdd), {ok, NewPurgeSeqTree1} = couch_btree:add(NewPurgeSeqTree0, InfosToAdd), - NewIdTreeState = couch_bt_engine_header:id_tree_state(NewSt#st.header), - MetaFd = couch_emsort:get_fd(NewSt#st.id_tree), - MetaState = couch_emsort:get_state(NewSt#st.id_tree), - NewSt0 = bind_id_tree(NewSt, NewSt#st.fd, NewIdTreeState), - NewSt1 = NewSt0#st{ + + NewSt2 = NewSt1#st{ purge_tree = NewPurgeTree1, purge_seq_tree = NewPurgeSeqTree1 }, @@ -170,7 +173,7 @@ copy_purge_infos(OldSt, NewSt, Infos, MinPurgeSeq, Retry) -> % any of the referenced docs have been completely purged % from the database. Any doc that has been completely purged % must then be removed from our partially compacted database. - NewSt2 = if Retry == nil -> NewSt1; true -> + NewSt3 = if Retry == nil -> NewSt2; true -> AllDocIds = [DocId || {_PurgeSeq, _UUID, DocId, _Revs} <- Infos], UniqDocIds = lists:usort(AllDocIds), OldIdResults = couch_btree:lookup(OldIdTree, UniqDocIds), @@ -194,17 +197,17 @@ copy_purge_infos(OldSt, NewSt, Infos, MinPurgeSeq, Retry) -> {ok, NewIdTree1} = couch_btree:add_remove(NewIdTree0, [], RemIds), {ok, NewSeqTree1} = couch_btree:add_remove(NewSeqTree0, [], RemSeqs), - NewSt1#st{ + NewSt2#st{ id_tree = NewIdTree1, seq_tree = NewSeqTree1 } end, - Header = couch_bt_engine:update_header(NewSt2, NewSt2#st.header), - NewSt3 = NewSt2#st{ + Header = couch_bt_engine:update_header(NewSt3, NewSt3#st.header), + NewSt4 = NewSt3#st{ header = Header }, - bind_emsort(NewSt3, MetaFd, MetaState). + bind_emsort(NewSt4, MetaFd, MetaState). copy_compact(DbName, St, NewSt0, Retry) -> diff --git a/src/couch/src/couch_db.erl b/src/couch/src/couch_db.erl index 34c1d2a90..cefe993f3 100644 --- a/src/couch/src/couch_db.erl +++ b/src/couch/src/couch_db.erl @@ -171,8 +171,11 @@ reopen(#db{} = Db) -> % We could have just swapped out the storage engine % for this database during a compaction so we just % reimplement this as a close/open pair now. - close(Db), - open(Db#db.name, [{user_ctx, Db#db.user_ctx} | Db#db.options]). + try + open(Db#db.name, [{user_ctx, Db#db.user_ctx} | Db#db.options]) + after + close(Db) + end. % You shouldn't call this. Its part of the ref counting between @@ -386,9 +389,22 @@ purge_docs(Db, IdRevs) -> Rev :: {non_neg_integer(), binary()}, PurgeOption :: interactive_edit | replicated_changes, Reply :: {ok, []} | {ok, [Rev]}. -purge_docs(#db{main_pid = Pid} = Db, UUIdsIdsRevs, Options) -> +purge_docs(#db{main_pid = Pid} = Db, UUIDsIdsRevs, Options) -> + % Check here if any UUIDs already exist when + % we're not replicating purge infos + IsRepl = lists:member(replicated_changes, Options), + if IsRepl -> ok; true -> + UUIDs = [UUID || {UUID, _, _} <- UUIDsIdsRevs], + lists:foreach(fun(Resp) -> + if Resp == not_found -> ok; true -> + Fmt = "Duplicate purge info UIUD: ~s", + Reason = io_lib:format(Fmt, [element(2, Resp)]), + throw({badreq, Reason}) + end + end, get_purge_infos(Db, UUIDs)) + end, increment_stat(Db, [couchdb, database_purges]), - gen_server:call(Pid, {purge_docs, UUIdsIdsRevs, Options}). + gen_server:call(Pid, {purge_docs, UUIDsIdsRevs, Options}). -spec get_purge_infos(#db{}, [UUId]) -> [PurgeInfo] when UUId :: binary(), diff --git a/src/couch/src/couch_db_engine.erl b/src/couch/src/couch_db_engine.erl index 5ac986554..ad527e829 100644 --- a/src/couch/src/couch_db_engine.erl +++ b/src/couch/src/couch_db_engine.erl @@ -881,7 +881,7 @@ read_doc_body(#db{} = Db, RawDoc) -> load_purge_infos(#db{} = Db, UUIDs) -> #db{engine = {Engine, EngineState}} = Db, - Engine:open_purged_docs(EngineState, UUIDs). + Engine:load_purge_infos(EngineState, UUIDs). serialize_doc(#db{} = Db, #doc{} = Doc) -> diff --git a/src/couch/src/couch_db_updater.erl b/src/couch/src/couch_db_updater.erl index 307e41773..31a36ad6d 100644 --- a/src/couch/src/couch_db_updater.erl +++ b/src/couch/src/couch_db_updater.erl @@ -97,7 +97,10 @@ handle_call({set_revs_limit, Limit}, _From, Db) -> handle_call({set_purge_infos_limit, Limit}, _From, Db) -> {ok, Db2} = couch_db_engine:set_purge_infos_limit(Db, Limit), ok = gen_server:call(couch_server, {db_updated, Db2}, infinity), - {reply, ok, Db2}; + {reply, ok, Db2, idle_limit()}; + +handle_call({purge_docs, [], _}, _From, Db) -> + {reply, {ok, []}, Db, idle_limit()}; handle_call({purge_docs, PurgeReqs0, Options}, _From, Db) -> % Filter out any previously applied updates during @@ -105,29 +108,47 @@ handle_call({purge_docs, PurgeReqs0, Options}, _From, Db) -> IsRepl = lists:member(replicated_changes, Options), PurgeReqs = if not IsRepl -> PurgeReqs0; true -> UUIDs = [UUID || {UUID, _Id, _Revs} <- PurgeReqs0], - {ok, PurgeInfos} = couch_db:load_purge_infos(Db, UUIDs), + PurgeInfos = couch_db_engine:load_purge_infos(Db, UUIDs), lists:flatmap(fun ({not_found, PReq}) -> [PReq]; ({{_, _, _, _}, _}) -> [] end, lists:zip(PurgeInfos, PurgeReqs0)) end, - Ids = [Id || {_UUID, Id, _Revs} <- PurgeReqs], - DocInfos = couch_db_engine:open_docs(Db, Ids), - UpdateSeq = couch_db_engine:get_update_seq(Db), - PurgeSeq = couch_db_engine:get_purge_seq(Db), + Ids = lists:usort(lists:map(fun({_UUID, Id, _Revs}) -> Id end, PurgeReqs)), + FDIs = couch_db_engine:open_docs(Db, Ids), + USeq = couch_db_engine:get_update_seq(Db), + + IdFDIs = lists:zip(Ids, FDIs), + {NewIdFDIs, Replies} = purge_docs(PurgeReqs, IdFDIs, USeq, []), + + Pairs = lists:flatmap(fun({DocId, OldFDI}) -> + {DocId, NewFDI} = lists:keyfind(DocId, 1, NewIdFDIs), + %io:format(standard_error, "~nPAIR: ~p~n", [{OldFDI, NewFDI}]), + case {OldFDI, NewFDI} of + {not_found, not_found} -> + []; + {#full_doc_info{} = A, #full_doc_info{} = A} -> + []; + {#full_doc_info{}, _} -> + [{OldFDI, NewFDI}] + end + end, IdFDIs), - InitAcc = {[], [], []}, - {Pairs, PInfos, Replies} = purge_docs( - PurgeReqs, DocInfos, UpdateSeq, PurgeSeq, InitAcc), + PSeq = couch_db_engine:get_purge_seq(Db), + {RevPInfos, _} = lists:foldl(fun({UUID, DocId, Revs}, {PIAcc, PSeqAcc}) -> + Info = {PSeqAcc + 1, UUID, DocId, Revs}, + {[Info | PIAcc], PSeqAcc + 1} + end, {[], PSeq}, PurgeReqs), + PInfos = lists:reverse(RevPInfos), - Db2 = if Pairs == [] -> Db; true -> - {ok, Db1} = couch_db_engine:purge_docs(Db, Pairs, PInfos), - ok = gen_server:call(couch_server, {db_updated, Db1}, infinity), - couch_event:notify(Db1#db.name, updated), - Db1 - end, - {reply, {ok, Replies}, Db2}; + %io:format(standard_error, "~n~nPAIRS: ~p~n~n", [Pairs]), + + {ok, Db1} = couch_db_engine:purge_docs(Db, Pairs, PInfos), + Db2 = commit_data(Db1), + ok = gen_server:call(couch_server, {db_updated, Db2}, infinity), + couch_event:notify(Db2#db.name, updated), + {reply, {ok, Replies}, Db2, idle_limit()}; handle_call(Msg, From, Db) -> case couch_db_engine:handle_db_updater_call(Msg, From, Db) of @@ -653,20 +674,21 @@ update_local_doc_revs(Docs) -> end, Docs). -purge_docs([], [], _USeq, _PSeq, {Pairs, PInfos, Replies}) -> - {lists:reverse(Pairs), lists:reverse(PInfos), lists:reverse(Replies)}; +purge_docs([], IdFDIs, _USeq, Replies) -> + {IdFDIs, lists:reverse(Replies)}; -purge_docs([Req | RestReqs], [FDI | RestInfos], USeq, PSeq, Acc) -> - {UUID, DocId, Revs} = Req, - {Pair, RemovedRevs, NewUSeq} = case FDI of +purge_docs([Req | RestReqs], IdFDIs, USeq, Replies) -> + {_UUID, DocId, Revs} = Req, + {value, {_, FDI0}, RestIdFDIs} = lists:keytake(DocId, 1, IdFDIs), + {NewFDI, RemovedRevs, NewUSeq} = case FDI0 of #full_doc_info{rev_tree = Tree} -> case couch_key_tree:remove_leafs(Tree, Revs) of {_, []} -> % No change - {no_change, [], USeq}; + {FDI0, [], USeq}; {[], Removed} -> % Completely purged - {{FDI, not_found}, Removed, USeq}; + {not_found, Removed, USeq}; {NewTree, Removed} -> % Its possible to purge the #leaf{} that contains % the update_seq where this doc sits in the @@ -681,27 +703,18 @@ purge_docs([Req | RestReqs], [FDI | RestInfos], USeq, PSeq, Acc) -> {Value, SeqAcc} end, USeq, NewTree), - NewFDI = FDI#full_doc_info{ + FDI1 = FDI0#full_doc_info{ update_seq = NewUpdateSeq, rev_tree = NewTree2 }, - {{FDI, NewFDI}, Removed, NewUpdateSeq} + {FDI1, Removed, NewUpdateSeq} end; not_found -> % Not found means nothing to change - {no_change, [], USeq} + {not_found, [], USeq} end, - {Pairs, PInfos, Replies} = Acc, - NewPairs = case Pair of - no_change -> Pairs; - _ -> [Pair | Pairs] - end, - NewAcc = { - NewPairs, - [{PSeq + 1, UUID, DocId, Revs} | PInfos], - [{ok, RemovedRevs} | Replies] - }, - purge_docs(RestReqs, RestInfos, NewUSeq, PSeq + 1, NewAcc). + NewReplies = [{ok, RemovedRevs} | Replies], + purge_docs(RestReqs, [{DocId, NewFDI} | RestIdFDIs], NewUSeq, NewReplies). commit_data(Db) -> diff --git a/src/couch/src/test_engine_attachments.erl b/src/couch/src/test_engine_attachments.erl index 691d4bd3c..9763ef545 100644 --- a/src/couch/src/test_engine_attachments.erl +++ b/src/couch/src/test_engine_attachments.erl @@ -19,25 +19,26 @@ cet_write_attachment() -> - {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), + {ok, Db1} = test_engine_util:create_db(), AttBin = crypto:strong_rand_bytes(32768), try - [Att0] = test_engine_util:prep_atts(Engine, St1, [ + [Att0] = test_engine_util:prep_atts(Db1, [ {<<"ohai.txt">>, AttBin} ]), {stream, Stream} = couch_att:fetch(data, Att0), - ?assertEqual(true, Engine:is_active_stream(St1, Stream)), + ?assertEqual(true, couch_db_engine:is_active_stream(Db1, Stream)), - Actions = [{create, {<<"first">>, [], [Att0]}}], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), - {ok, St3} = Engine:commit_data(St2), - Engine:terminate(normal, St3), + Actions = [{create, {<<"first">>, {[]}, [Att0]}}], + {ok, Db2} = test_engine_util:apply_actions(Db1, Actions), + {ok, _} = couch_db:ensure_full_commit(Db2), + test_engine_util:shutdown_db(Db2), - {ok, St4} = Engine:init(DbPath, []), - [FDI] = Engine:open_docs(St4, [<<"first">>]), + {ok, Db3} = couch_db:reopen(Db2), + + [FDI] = couch_db_engine:open_docs(Db3, [<<"first">>]), #rev_info{ rev = {RevPos, PrevRevId}, @@ -52,12 +53,12 @@ cet_write_attachment() -> body = DocPtr }, - Doc1 = Engine:read_doc_body(St4, Doc0), + Doc1 = couch_db_engine:read_doc_body(Db3, Doc0), Atts1 = if not is_binary(Doc1#doc.atts) -> Doc1#doc.atts; true -> couch_compress:decompress(Doc1#doc.atts) end, - StreamSrc = fun(Sp) -> Engine:open_read_stream(St4, Sp) end, + StreamSrc = fun(Sp) -> couch_db_engine:open_read_stream(Db3, Sp) end, [Att1] = [couch_att:from_disk_term(StreamSrc, T) || T <- Atts1], ReadBin = couch_att:to_binary(Att1), ?assertEqual(AttBin, ReadBin) @@ -72,22 +73,22 @@ cet_write_attachment() -> % we ever have something that stores attachemnts in % an external object store) cet_inactive_stream() -> - {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), + {ok, Db1} = test_engine_util:create_db(), AttBin = crypto:strong_rand_bytes(32768), try - [Att0] = test_engine_util:prep_atts(Engine, St1, [ + [Att0] = test_engine_util:prep_atts(Db1, [ {<<"ohai.txt">>, AttBin} ]), {stream, Stream} = couch_att:fetch(data, Att0), - ?assertEqual(true, Engine:is_active_stream(St1, Stream)), + ?assertEqual(true, couch_db_engine:is_active_stream(Db1, Stream)), - Engine:terminate(normal, St1), - {ok, St2} = Engine:init(DbPath, []), + test_engine_util:shutdown_db(Db1), + {ok, Db2} = couch_db:reopen(Db1), - ?assertEqual(false, Engine:is_active_stream(St2, Stream)) + ?assertEqual(false, couch_db_engine:is_active_stream(Db2, Stream)) catch throw:not_supported -> ok end. diff --git a/src/couch/src/test_engine_compaction.erl b/src/couch/src/test_engine_compaction.erl index 389298d03..26714a1ca 100644 --- a/src/couch/src/test_engine_compaction.erl +++ b/src/couch/src/test_engine_compaction.erl @@ -19,66 +19,75 @@ cet_compact_empty() -> - {ok, Engine, Path, St1} = test_engine_util:init_engine(dbpath), - Db1 = test_engine_util:db_as_term(Engine, St1), - {ok, St2, DbName, _, Term} = test_engine_util:compact(Engine, St1, Path), - {ok, St3, undefined} = Engine:finish_compaction(St2, DbName, [], Term), - Db2 = test_engine_util:db_as_term(Engine, St3), - Diff = test_engine_util:term_diff(Db1, Db2), + {ok, Db1} = test_engine_util:create_db(), + Term1 = test_engine_util:db_as_term(Db1), + + test_engine_util:compact(Db1), + + {ok, Db2} = couch_db:reopen(Db1), + Term2 = test_engine_util:db_as_term(Db2), + + Diff = test_engine_util:term_diff(Term1, Term2), ?assertEqual(nodiff, Diff). cet_compact_doc() -> - {ok, Engine, Path, St1} = test_engine_util:init_engine(dbpath), - Actions = [{create, {<<"foo">>, []}}], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), - Db1 = test_engine_util:db_as_term(Engine, St2), - {ok, St3, DbName, _, Term} = test_engine_util:compact(Engine, St2, Path), - {ok, St4, undefined} = Engine:finish_compaction(St3, DbName, [], Term), - Db2 = test_engine_util:db_as_term(Engine, St4), - Diff = test_engine_util:term_diff(Db1, Db2), + {ok, Db1} = test_engine_util:create_db(), + Actions = [{create, {<<"foo">>, {[]}}}], + {ok, Db2} = test_engine_util:apply_actions(Db1, Actions), + Term1 = test_engine_util:db_as_term(Db2), + + test_engine_util:compact(Db2), + + {ok, Db3} = couch_db:reopen(Db2), + Term2 = test_engine_util:db_as_term(Db3), + + Diff = test_engine_util:term_diff(Term1, Term2), ?assertEqual(nodiff, Diff). cet_compact_local_doc() -> - {ok, Engine, Path, St1} = test_engine_util:init_engine(dbpath), - Actions = [{create, {<<"_local/foo">>, []}}], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), - Db1 = test_engine_util:db_as_term(Engine, St2), - {ok, St3, DbName, _, Term} = test_engine_util:compact(Engine, St2, Path), - {ok, St4, undefined} = Engine:finish_compaction(St3, DbName, [], Term), - Db2 = test_engine_util:db_as_term(Engine, St4), - Diff = test_engine_util:term_diff(Db1, Db2), + {ok, Db1} = test_engine_util:create_db(), + Actions = [{create, {<<"_local/foo">>, {[]}}}], + {ok, Db2} = test_engine_util:apply_actions(Db1, Actions), + Term1 = test_engine_util:db_as_term(Db2), + + test_engine_util:compact(Db2), + + {ok, Db3} = couch_db:reopen(Db2), + Term2 = test_engine_util:db_as_term(Db3), + + Diff = test_engine_util:term_diff(Term1, Term2), ?assertEqual(nodiff, Diff). cet_compact_with_everything() -> - {ok, Engine, Path, St1} = test_engine_util:init_engine(dbpath), + {ok, Db1} = test_engine_util:create_db(), % Add a whole bunch of docs DocActions = lists:map(fun(Seq) -> - {create, {docid(Seq), [{<<"int">>, Seq}]}} + {create, {docid(Seq), {[{<<"int">>, Seq}]}}} end, lists:seq(1, 1000)), LocalActions = lists:map(fun(I) -> - {create, {local_docid(I), [{<<"int">>, I}]}} + {create, {local_docid(I), {[{<<"int">>, I}]}}} end, lists:seq(1, 25)), Actions1 = DocActions ++ LocalActions, - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1), - {ok, St3} = Engine:set_security(St2, [{<<"readers">>, <<"ohai">>}]), - {ok, St4} = Engine:set_revs_limit(St3, 500), + {ok, Db2} = test_engine_util:apply_batch(Db1, Actions1), + ok = couch_db:set_security(Db1, {[{<<"foo">>, <<"bar">>}]}), + ok = couch_db:set_revs_limit(Db1, 500), Actions2 = [ - {create, {<<"foo">>, []}}, - {create, {<<"bar">>, [{<<"hooray">>, <<"purple">>}]}}, - {conflict, {<<"bar">>, [{<<"booo">>, false}]}} + {create, {<<"foo">>, {[]}}}, + {create, {<<"bar">>, {[{<<"hooray">>, <<"purple">>}]}}}, + {conflict, {<<"bar">>, {[{<<"booo">>, false}]}}} ], - {ok, St5} = test_engine_util:apply_actions(Engine, St4, Actions2), + {ok, Db3} = test_engine_util:apply_actions(Db2, Actions2), - [FooFDI, BarFDI] = Engine:open_docs(St5, [<<"foo">>, <<"bar">>]), + [FooFDI, BarFDI] = couch_db_engine:open_docs(Db3, [<<"foo">>, <<"bar">>]), FooRev = test_engine_util:prev_rev(FooFDI), BarRev = test_engine_util:prev_rev(BarFDI), @@ -88,18 +97,19 @@ cet_compact_with_everything() -> {purge, {<<"bar">>, BarRev#rev_info.rev}} ], - {ok, St6} = test_engine_util:apply_actions(Engine, St5, Actions3), + {ok, Db4} = test_engine_util:apply_actions(Db3, Actions3), PurgedIdRevs = [ {<<"bar">>, [BarRev#rev_info.rev]}, {<<"foo">>, [FooRev#rev_info.rev]} ], - {ok, PIdRevs6} = Engine:fold_purge_infos(St6, 0, fun fold_fun/2, [], []), - ?assertEqual(PurgedIdRevs, PIdRevs6), + {ok, PIdRevs4} = couch_db_engine:fold_purge_infos( + Db4, 0, fun fold_fun/2, [], []), + ?assertEqual(PurgedIdRevs, PIdRevs4), - {ok, St7} = try - [Att0, Att1, Att2, Att3, Att4] = test_engine_util:prep_atts(Engine, St6, [ + {ok, Db5} = try + [Att0, Att1, Att2, Att3, Att4] = test_engine_util:prep_atts(Db4, [ {<<"ohai.txt">>, crypto:strong_rand_bytes(2048)}, {<<"stuff.py">>, crypto:strong_rand_bytes(32768)}, {<<"a.erl">>, crypto:strong_rand_bytes(29)}, @@ -108,250 +118,190 @@ cet_compact_with_everything() -> ]), Actions4 = [ - {create, {<<"small_att">>, [], [Att0]}}, - {create, {<<"large_att">>, [], [Att1]}}, - {create, {<<"multi_att">>, [], [Att2, Att3, Att4]}} + {create, {<<"small_att">>, {[]}, [Att0]}}, + {create, {<<"large_att">>, {[]}, [Att1]}}, + {create, {<<"multi_att">>, {[]}, [Att2, Att3, Att4]}} ], - test_engine_util:apply_actions(Engine, St6, Actions4) + test_engine_util:apply_actions(Db4, Actions4) catch throw:not_supported -> - {ok, St6} + {ok, Db4} end, - {ok, St8} = Engine:commit_data(St7), + {ok, _} = couch_db:ensure_full_commit(Db5), + {ok, Db6} = couch_db:reopen(Db5), - Db1 = test_engine_util:db_as_term(Engine, St8), + Term1 = test_engine_util:db_as_term(Db6), Config = [ {"database_compaction", "doc_buffer_size", "1024"}, {"database_compaction", "checkpoint_after", "2048"} ], - {ok, St9, DbName, _, Term} = test_engine_util:with_config(Config, fun() -> - test_engine_util:compact(Engine, St8, Path) + test_engine_util:with_config(Config, fun() -> + test_engine_util:compact(Db6) end), - {ok, St10, undefined} = Engine:finish_compaction(St9, DbName, [], Term), - {ok, PIdRevs11} = Engine:fold_purge_infos(St10, 0, fun fold_fun/2, [], []), - ?assertEqual(PurgedIdRevs, PIdRevs11), + {ok, Db7} = couch_db:reopen(Db6), + Term2 = test_engine_util:db_as_term(Db7), - Db2 = test_engine_util:db_as_term(Engine, St10), - Diff = test_engine_util:term_diff(Db1, Db2), + Diff = test_engine_util:term_diff(Term1, Term2), ?assertEqual(nodiff, Diff). cet_recompact_updates() -> - {ok, Engine, Path, St1} = test_engine_util:init_engine(dbpath), + {ok, Db1} = test_engine_util:create_db(), - Actions1 = [ - {create, {<<"foo">>, []}}, - {create, {<<"bar">>, []}} - ], + Actions1 = lists:map(fun(Seq) -> + {create, {docid(Seq), {[{<<"int">>, Seq}]}}} + end, lists:seq(1, 1000)), + {ok, Db2} = test_engine_util:apply_batch(Db1, Actions1), - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1), - {ok, St3, DbName, _, Term} = test_engine_util:compact(Engine, St2, Path), + {ok, Compactor} = couch_db:start_compact(Db2), + catch erlang:suspend_process(Compactor), Actions2 = [ - {update, {<<"foo">>, [{<<"updated">>, true}]}}, - {create, {<<"baz">>, []}} + {update, {<<"0001">>, {[{<<"updated">>, true}]}}}, + {create, {<<"boop">>, {[]}}} ], - {ok, St4} = test_engine_util:apply_actions(Engine, St3, Actions2), - Db1 = test_engine_util:db_as_term(Engine, St4), - - {ok, St5, NewPid} = Engine:finish_compaction(St4, DbName, [], Term), + {ok, Db3} = test_engine_util:apply_actions(Db2, Actions2), + Term1 = test_engine_util:db_as_term(Db3), - ?assertEqual(true, is_pid(NewPid)), - Ref = erlang:monitor(process, NewPid), + catch erlang:resume_process(Compactor), + test_engine_util:compact(Db3), - NewTerm = receive - {'$gen_cast', {compact_done, Engine, Term0}} -> - Term0; - {'DOWN', Ref, _, _, Reason} -> - erlang:error({compactor_died, Reason}) - after 10000 -> - erlang:error(compactor_timed_out) - end, + {ok, Db4} = couch_db:reopen(Db3), + Term2 = test_engine_util:db_as_term(Db4), - {ok, St6, undefined} = Engine:finish_compaction(St5, DbName, [], NewTerm), - Db2 = test_engine_util:db_as_term(Engine, St6), - Diff = test_engine_util:term_diff(Db1, Db2), + Diff = test_engine_util:term_diff(Term1, Term2), ?assertEqual(nodiff, Diff). cet_purge_during_compact() -> - {ok, Engine, Path, St1} = test_engine_util:init_engine(dbpath), + {ok, Db1} = test_engine_util:create_db(), - Actions1 = [ - {create, {<<"foo">>, []}}, - {create, {<<"bar">>, []}}, - {conflict, {<<"bar">>, [{<<"vsn">>, 2}]}}, - {create, {<<"baz">>, []}} + Actions1 = lists:map(fun(Seq) -> + {create, {docid(Seq), {[{<<"int">>, Seq}]}}} + end, lists:seq(1, 1000)), + Actions2 = [ + {create, {<<"foo">>, {[]}}}, + {create, {<<"bar">>, {[]}}}, + {create, {<<"baz">>, {[]}}} ], + {ok, Db2} = test_engine_util:apply_batch(Db1, Actions1 ++ Actions2), + Actions3 = [ + {conflict, {<<"bar">>, {[{<<"vsn">>, 2}]}}} + ], + {ok, Db3} = test_engine_util:apply_actions(Db2, Actions3), - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1), - {ok, St3, DbName, _, Term} = test_engine_util:compact(Engine, St2, Path), + {ok, Pid} = couch_db:start_compact(Db3), + catch erlang:suspend_process(Pid), - [BarFDI, BazFDI] = Engine:open_docs(St3, [<<"bar">>, <<"baz">>]), + [BarFDI, BazFDI] = couch_db_engine:open_docs(Db3, [<<"bar">>, <<"baz">>]), BarRev = test_engine_util:prev_rev(BarFDI), BazRev = test_engine_util:prev_rev(BazFDI), - Actions2 = [ + Actions4 = [ {purge, {<<"bar">>, BarRev#rev_info.rev}}, {purge, {<<"baz">>, BazRev#rev_info.rev}} ], - {ok, St4} = test_engine_util:apply_actions(Engine, St3, Actions2), - Db1 = test_engine_util:db_as_term(Engine, St4), - {ok, St5, NewPid} = Engine:finish_compaction(St4, DbName, [], Term), + {ok, Db4} = test_engine_util:apply_actions(Db3, Actions4), + Term1 = test_engine_util:db_as_term(Db4), - ?assertEqual(true, is_pid(NewPid)), - Ref = erlang:monitor(process, NewPid), + catch erlang:resume_process(Pid), + test_engine_util:compact(Db4), - NewTerm = receive - {'$gen_cast', {compact_done, Engine, Term0}} -> - Term0; - {'DOWN', Ref, _, _, Reason} -> - erlang:error({compactor_died, Reason}) - after 10000 -> - erlang:error(compactor_timed_out) - end, + {ok, Db5} = couch_db:reopen(Db4), + Term2 = test_engine_util:db_as_term(Db5), - {ok, St6, undefined} = Engine:finish_compaction(St5, DbName, [], NewTerm), - Db2 = test_engine_util:db_as_term(Engine, St6), - Diff = test_engine_util:term_diff(Db1, Db2), + Diff = test_engine_util:term_diff(Term1, Term2), ?assertEqual(nodiff, Diff). cet_multiple_purge_during_compact() -> - {ok, Engine, Path, St1} = test_engine_util:init_engine(dbpath), - - Actions1 = [ - {create, {<<"foo">>, []}}, - {create, {<<"bar">>, []}}, - {conflict, {<<"bar">>, [{<<"vsn">>, 2}]}}, - {create, {<<"baz">>, []}} - ], - - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1), - {ok, St3, DbName, _, Term} = test_engine_util:compact(Engine, St2, Path), + {ok, Db1} = test_engine_util:create_db(), - [BarFDI, BazFDI] = Engine:open_docs(St3, [<<"bar">>, <<"baz">>]), - BarRev = test_engine_util:prev_rev(BarFDI), + Actions1 = lists:map(fun(Seq) -> + {create, {docid(Seq), {[{<<"int">>, Seq}]}}} + end, lists:seq(1, 1000)), Actions2 = [ - {purge, {<<"bar">>, BarRev#rev_info.rev}} + {create, {<<"foo">>, {[]}}}, + {create, {<<"bar">>, {[]}}}, + {create, {<<"baz">>, {[]}}} ], - {ok, St4} = test_engine_util:apply_actions(Engine, St3, Actions2), + {ok, Db2} = test_engine_util:apply_batch(Db1, Actions1 ++ Actions2), - BazRev = test_engine_util:prev_rev(BazFDI), Actions3 = [ - {purge, {<<"baz">>, BazRev#rev_info.rev}} + {conflict, {<<"bar">>, {[{<<"vsn">>, 2}]}}} ], - {ok, St5} = test_engine_util:apply_actions(Engine, St4, Actions3), - - Db1 = test_engine_util:db_as_term(Engine, St5), - {ok, St6, NewPid} = Engine:finish_compaction(St5, DbName, [], Term), - - ?assertEqual(true, is_pid(NewPid)), - Ref = erlang:monitor(process, NewPid), - - NewTerm = receive - {'$gen_cast', {compact_done, Engine, Term0}} -> - Term0; - {'DOWN', Ref, _, _, Reason} -> - erlang:error({compactor_died, Reason}) - after 10000 -> - erlang:error(compactor_timed_out) - end, - - {ok, St7, undefined} = Engine:finish_compaction(St6, DbName, [], NewTerm), - Db2 = test_engine_util:db_as_term(Engine, St7), - Diff = test_engine_util:term_diff(Db1, Db2), - ?assertEqual(nodiff, Diff). + {ok, Db3} = test_engine_util:apply_actions(Db2, Actions3), -cet_recompact_purge() -> - {ok, Engine, Path, St1} = test_engine_util:init_engine(dbpath), + {ok, Pid} = couch_db:start_compact(Db3), + catch erlang:suspend_process(Pid), - Actions1 = [ - {create, {<<"foo">>, []}}, - {create, {<<"bar">>, []}}, - {conflict, {<<"bar">>, [{<<"vsn">>, 2}]}}, - {create, {<<"baz">>, []}} + [BarFDI, BazFDI] = couch_db_engine:open_docs(Db3, [<<"bar">>, <<"baz">>]), + BarRev = test_engine_util:prev_rev(BarFDI), + Actions4 = [ + {purge, {<<"bar">>, BarRev#rev_info.rev}} ], + {ok, Db4} = test_engine_util:apply_actions(Db3, Actions4), - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1), - {ok, St3, DbName, _, Term} = test_engine_util:compact(Engine, St2, Path), - - [BarFDI, BazFDI] = Engine:open_docs(St3, [<<"bar">>, <<"baz">>]), - BarRev = test_engine_util:prev_rev(BarFDI), BazRev = test_engine_util:prev_rev(BazFDI), - Actions2 = [ - {purge, {<<"bar">>, BarRev#rev_info.rev}}, + Actions5 = [ {purge, {<<"baz">>, BazRev#rev_info.rev}} ], - {ok, St4} = test_engine_util:apply_actions(Engine, St3, Actions2), - Db1 = test_engine_util:db_as_term(Engine, St4), - {ok, St5, NewPid} = Engine:finish_compaction(St4, DbName, [], Term), + {ok, Db5} = test_engine_util:apply_actions(Db4, Actions5), + Term1 = test_engine_util:db_as_term(Db5), - ?assertEqual(true, is_pid(NewPid)), - Ref = erlang:monitor(process, NewPid), + catch erlang:resume_process(Pid), + test_engine_util:compact(Db5), - NewTerm = receive - {'$gen_cast', {compact_done, Engine, Term0}} -> - Term0; - {'DOWN', Ref, _, _, Reason} -> - erlang:error({compactor_died, Reason}) - after 10000 -> - erlang:error(compactor_timed_out) - end, + {ok, Db6} = couch_db:reopen(Db5), + Term2 = test_engine_util:db_as_term(Db6), - {ok, St6, undefined} = Engine:finish_compaction(St5, DbName, [], NewTerm), - Db2 = test_engine_util:db_as_term(Engine, St6), - Diff = test_engine_util:term_diff(Db1, Db2), + Diff = test_engine_util:term_diff(Term1, Term2), ?assertEqual(nodiff, Diff). -% temporary ignoring this test as it times out -ignore_cet_compact_purged_docs_limit() -> - {ok, Engine, Path, St1} = test_engine_util:init_engine(dbpath), - % create NumDocs docs +cet_compact_purged_docs_limit() -> + {ok, Db1} = test_engine_util:create_db(), NumDocs = 1200, {RActions, RIds} = lists:foldl(fun(Id, {CActions, CIds}) -> Id1 = docid(Id), - Action = {create, {Id1, [{<<"int">>, Id}]}}, + Action = {create, {Id1, {[{<<"int">>, Id}]}}}, {[Action| CActions], [Id1| CIds]} end, {[], []}, lists:seq(1, NumDocs)), Ids = lists:reverse(RIds), - {ok, St2} = test_engine_util:apply_actions(Engine, St1, - lists:reverse(RActions)), + {ok, Db2} = test_engine_util:apply_batch(Db1, lists:reverse(RActions)), - % purge NumDocs docs - FDIs = Engine:open_docs(St2, Ids), - RevActions2 = lists:foldl(fun(FDI, CActions) -> + FDIs = couch_db_engine:open_docs(Db2, Ids), + RActions2 = lists:foldl(fun(FDI, CActions) -> Id = FDI#full_doc_info.id, PrevRev = test_engine_util:prev_rev(FDI), Rev = PrevRev#rev_info.rev, [{purge, {Id, Rev}}| CActions] end, [], FDIs), - {ok, St3} = test_engine_util:apply_actions(Engine, St2, - lists:reverse(RevActions2)), + {ok, Db3} = test_engine_util:apply_batch(Db2, lists:reverse(RActions2)), % check that before compaction all NumDocs of purge_requests % are in purge_tree, % even if NumDocs=1200 is greater than purged_docs_limit=1000 - {ok, PurgedIdRevs} = Engine:fold_purge_infos(St3, 0, fun fold_fun/2, [], []), - ?assertEqual(1, Engine:get_oldest_purge_seq(St3)), + {ok, PurgedIdRevs} = couch_db_engine:fold_purge_infos( + Db3, 0, fun fold_fun/2, [], []), + ?assertEqual(1, couch_db_engine:get_oldest_purge_seq(Db3)), ?assertEqual(NumDocs, length(PurgedIdRevs)), % compact db - {ok, St4, DbName, _, Term} = test_engine_util:compact(Engine, St3, Path), - {ok, St5, undefined} = Engine:finish_compaction(St4, DbName, [], Term), + test_engine_util:compact(Db3), + {ok, Db4} = couch_db:reopen(Db3), % check that after compaction only purged_docs_limit purge_requests % are in purge_tree - PurgedDocsLimit = Engine:get_purge_infos_limit(St5), - OldestPSeq = Engine:get_oldest_purge_seq(St5), - {ok, PurgedIdRevs2} = Engine:fold_purge_infos( - St5, OldestPSeq - 1, fun fold_fun/2, [], []), + PurgedDocsLimit = couch_db_engine:get_purge_infos_limit(Db4), + OldestPSeq = couch_db_engine:get_oldest_purge_seq(Db4), + {ok, PurgedIdRevs2} = couch_db_engine:fold_purge_infos( + Db4, OldestPSeq - 1, fun fold_fun/2, [], []), ExpectedOldestPSeq = NumDocs - PurgedDocsLimit + 1, ?assertEqual(ExpectedOldestPSeq, OldestPSeq), ?assertEqual(PurgedDocsLimit, length(PurgedIdRevs2)). diff --git a/src/couch/src/test_engine_fold_changes.erl b/src/couch/src/test_engine_fold_changes.erl index 6e97fda9b..4ca09a863 100644 --- a/src/couch/src/test_engine_fold_changes.erl +++ b/src/couch/src/test_engine_fold_changes.erl @@ -18,147 +18,153 @@ -include_lib("couch/include/couch_db.hrl"). --define(NUM_DOCS, 100). +-define(NUM_DOCS, 25). cet_empty_changes() -> - {ok, Engine, St} = test_engine_util:init_engine(), - - ?assertEqual(0, Engine:count_changes_since(St, 0)), - ?assertEqual({ok, []}, Engine:fold_changes(St, 0, fun fold_fun/2, [], [])). + {ok, Db} = test_engine_util:create_db(), + ?assertEqual(0, couch_db_engine:count_changes_since(Db, 0)), + ?assertEqual({ok, []}, + couch_db_engine:fold_changes(Db, 0, fun fold_fun/2, [], [])). cet_single_change() -> - {ok, Engine, St1} = test_engine_util:init_engine(), - Actions = [{create, {<<"a">>, []}}], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + {ok, Db1} = test_engine_util:create_db(), + Actions = [{create, {<<"a">>, {[]}}}], + {ok, Db2} = test_engine_util:apply_actions(Db1, Actions), - ?assertEqual(1, Engine:count_changes_since(St2, 0)), + ?assertEqual(1, couch_db_engine:count_changes_since(Db2, 0)), ?assertEqual({ok, [{<<"a">>, 1}]}, - Engine:fold_changes(St2, 0, fun fold_fun/2, [], [])). + couch_db_engine:fold_changes(Db2, 0, fun fold_fun/2, [], [])). cet_two_changes() -> - {ok, Engine, St1} = test_engine_util:init_engine(), + {ok, Db1} = test_engine_util:create_db(), Actions = [ - {create, {<<"a">>, []}}, - {create, {<<"b">>, []}} + {create, {<<"a">>, {[]}}}, + {create, {<<"b">>, {[]}}} ], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + {ok, Db2} = test_engine_util:apply_actions(Db1, Actions), - ?assertEqual(2, Engine:count_changes_since(St2, 0)), - {ok, Changes} = Engine:fold_changes(St2, 0, fun fold_fun/2, [], []), + ?assertEqual(2, couch_db_engine:count_changes_since(Db2, 0)), + {ok, Changes} = + couch_db_engine:fold_changes(Db2, 0, fun fold_fun/2, [], []), ?assertEqual([{<<"a">>, 1}, {<<"b">>, 2}], lists:reverse(Changes)). cet_two_changes_batch() -> - {ok, Engine, St1} = test_engine_util:init_engine(), + {ok, Db1} = test_engine_util:create_db(), Actions1 = [ {batch, [ - {create, {<<"a">>, []}}, - {create, {<<"b">>, []}} + {create, {<<"a">>, {[]}}}, + {create, {<<"b">>, {[]}}} ]} ], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1), + {ok, Db2} = test_engine_util:apply_actions(Db1, Actions1), - ?assertEqual(2, Engine:count_changes_since(St2, 0)), - {ok, Changes1} = Engine:fold_changes(St2, 0, fun fold_fun/2, [], []), + ?assertEqual(2, couch_db_engine:count_changes_since(Db2, 0)), + {ok, Changes1} = + couch_db_engine:fold_changes(Db2, 0, fun fold_fun/2, [], []), ?assertEqual([{<<"a">>, 1}, {<<"b">>, 2}], lists:reverse(Changes1)), - {ok, Engine, St3} = test_engine_util:init_engine(), + {ok, Db3} = test_engine_util:create_db(), Actions2 = [ {batch, [ - {create, {<<"b">>, []}}, - {create, {<<"a">>, []}} + {create, {<<"b">>, {[]}}}, + {create, {<<"a">>, {[]}}} ]} ], - {ok, St4} = test_engine_util:apply_actions(Engine, St3, Actions2), + {ok, Db4} = test_engine_util:apply_actions(Db3, Actions2), - ?assertEqual(2, Engine:count_changes_since(St4, 0)), - {ok, Changes2} = Engine:fold_changes(St4, 0, fun fold_fun/2, [], []), - ?assertEqual([{<<"b">>, 1}, {<<"a">>, 2}], lists:reverse(Changes2)). + ?assertEqual(2, couch_db_engine:count_changes_since(Db4, 0)), + {ok, Changes2} = + couch_db_engine:fold_changes(Db4, 0, fun fold_fun/2, [], []), + ?assertEqual([{<<"a">>, 1}, {<<"b">>, 2}], lists:reverse(Changes2)). cet_update_one() -> - {ok, Engine, St1} = test_engine_util:init_engine(), + {ok, Db1} = test_engine_util:create_db(), Actions = [ - {create, {<<"a">>, []}}, - {update, {<<"a">>, []}} + {create, {<<"a">>, {[]}}}, + {update, {<<"a">>, {[]}}} ], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + {ok, Db2} = test_engine_util:apply_actions(Db1, Actions), - ?assertEqual(1, Engine:count_changes_since(St2, 0)), + ?assertEqual(1, couch_db_engine:count_changes_since(Db2, 0)), ?assertEqual({ok, [{<<"a">>, 2}]}, - Engine:fold_changes(St2, 0, fun fold_fun/2, [], [])). + couch_db_engine:fold_changes(Db2, 0, fun fold_fun/2, [], [])). cet_update_first_of_two() -> - {ok, Engine, St1} = test_engine_util:init_engine(), + {ok, Db1} = test_engine_util:create_db(), Actions = [ - {create, {<<"a">>, []}}, - {create, {<<"b">>, []}}, - {update, {<<"a">>, []}} + {create, {<<"a">>, {[]}}}, + {create, {<<"b">>, {[]}}}, + {update, {<<"a">>, {[]}}} ], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + {ok, Db2} = test_engine_util:apply_actions(Db1, Actions), - ?assertEqual(2, Engine:count_changes_since(St2, 0)), - {ok, Changes} = Engine:fold_changes(St2, 0, fun fold_fun/2, [], []), + ?assertEqual(2, couch_db_engine:count_changes_since(Db2, 0)), + {ok, Changes} = + couch_db_engine:fold_changes(Db2, 0, fun fold_fun/2, [], []), ?assertEqual([{<<"b">>, 2}, {<<"a">>, 3}], lists:reverse(Changes)). cet_update_second_of_two() -> - {ok, Engine, St1} = test_engine_util:init_engine(), + {ok, Db1} = test_engine_util:create_db(), Actions = [ - {create, {<<"a">>, []}}, - {create, {<<"b">>, []}}, - {update, {<<"b">>, []}} + {create, {<<"a">>, {[]}}}, + {create, {<<"b">>, {[]}}}, + {update, {<<"b">>, {[]}}} ], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + {ok, Db2} = test_engine_util:apply_actions(Db1, Actions), - ?assertEqual(2, Engine:count_changes_since(St2, 0)), - {ok, Changes} = Engine:fold_changes(St2, 0, fun fold_fun/2, [], []), + ?assertEqual(2, couch_db_engine:count_changes_since(Db2, 0)), + {ok, Changes} = + couch_db_engine:fold_changes(Db2, 0, fun fold_fun/2, [], []), ?assertEqual([{<<"a">>, 1}, {<<"b">>, 3}], lists:reverse(Changes)). cet_check_mutation_ordering() -> Actions = shuffle(lists:map(fun(Seq) -> - {create, {docid(Seq), []}} + {create, {docid(Seq), {[]}}} end, lists:seq(1, ?NUM_DOCS))), DocIdOrder = [DocId || {_, {DocId, _}} <- Actions], DocSeqs = lists:zip(DocIdOrder, lists:seq(1, ?NUM_DOCS)), - {ok, Engine, St1} = test_engine_util:init_engine(), - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + {ok, Db1} = test_engine_util:create_db(), + {ok, Db2} = test_engine_util:apply_actions(Db1, Actions), % First lets see that we can get the correct % suffix/prefix starting at every update sequence lists:foreach(fun(Seq) -> - {ok, Suffix} = Engine:fold_changes(St2, Seq, fun fold_fun/2, [], []), + {ok, Suffix} = + couch_db_engine:fold_changes(Db2, Seq, fun fold_fun/2, [], []), ?assertEqual(lists:nthtail(Seq, DocSeqs), lists:reverse(Suffix)), - {ok, Prefix} = Engine:fold_changes(St2, Seq, fun fold_fun/2, [], [ - {dir, rev} - ]), + {ok, Prefix} = couch_db_engine:fold_changes( + Db2, Seq, fun fold_fun/2, [], [{dir, rev}]), ?assertEqual(lists:sublist(DocSeqs, Seq + 1), Prefix) end, lists:seq(0, ?NUM_DOCS)), - ok = do_mutation_ordering(Engine, St2, ?NUM_DOCS + 1, DocSeqs, []). + ok = do_mutation_ordering(Db2, ?NUM_DOCS + 1, DocSeqs, []). -do_mutation_ordering(Engine, St, _Seq, [], FinalDocSeqs) -> - {ok, RevOrder} = Engine:fold_changes(St, 0, fun fold_fun/2, [], []), +do_mutation_ordering(Db, _Seq, [], FinalDocSeqs) -> + {ok, RevOrder} = couch_db_engine:fold_changes(Db, 0, fun fold_fun/2, [], []), ?assertEqual(FinalDocSeqs, lists:reverse(RevOrder)), ok; -do_mutation_ordering(Engine, St, Seq, [{DocId, _OldSeq} | Rest], DocSeqAcc) -> - Actions = [{update, {DocId, []}}], - {ok, NewSt} = test_engine_util:apply_actions(Engine, St, Actions), +do_mutation_ordering(Db, Seq, [{DocId, _OldSeq} | Rest], DocSeqAcc) -> + Actions = [{update, {DocId, {[]}}}], + {ok, NewDb} = test_engine_util:apply_actions(Db, Actions), NewAcc = DocSeqAcc ++ [{DocId, Seq}], Expected = Rest ++ NewAcc, - {ok, RevOrder} = Engine:fold_changes(NewSt, 0, fun fold_fun/2, [], []), + {ok, RevOrder} = + couch_db_engine:fold_changes(NewDb, 0, fun fold_fun/2, [], []), ?assertEqual(Expected, lists:reverse(RevOrder)), - do_mutation_ordering(Engine, NewSt, Seq + 1, Rest, NewAcc). + do_mutation_ordering(NewDb, Seq + 1, Rest, NewAcc). shuffle(List) -> diff --git a/src/couch/src/test_engine_fold_docs.erl b/src/couch/src/test_engine_fold_docs.erl index 458878d97..3ed068f0c 100644 --- a/src/couch/src/test_engine_fold_docs.erl +++ b/src/couch/src/test_engine_fold_docs.erl @@ -62,39 +62,41 @@ cet_fold_range_local() -> cet_fold_stop() -> - fold_stop(fold_docs, fun docid/1). + fold_user_fun_stop(fold_docs, fun docid/1). cet_fold_stop_local() -> - fold_stop(fold_local_docs, fun local_docid/1). + fold_user_fun_stop(fold_local_docs, fun local_docid/1). % This is a loose test but we have to have this until % I figure out what to do about the total_rows/offset % meta data included in _all_docs cet_fold_include_reductions() -> - {ok, Engine, St} = init_st(fun docid/1), + {ok, Db} = init_st(fun docid/1), FoldFun = fun(_, _, nil) -> {ok, nil} end, - {ok, Count, nil} = Engine:fold_docs(St, FoldFun, nil, [include_reductions]), + Opts = [include_reductions], + {ok, Count, nil} = couch_db_engine:fold_docs(Db, FoldFun, nil, Opts), ?assert(is_integer(Count)), ?assert(Count >= 0). fold_all(FoldFun, DocIdFun) -> DocIds = [DocIdFun(I) || I <- lists:seq(1, ?NUM_DOCS)], - {ok, Engine, St} = init_st(DocIdFun), + {ok, Db} = init_st(DocIdFun), - {ok, DocIdAccFwd} = Engine:FoldFun(St, fun fold_fun/2, [], []), + {ok, DocIdAccFwd} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], []), ?assertEqual(?NUM_DOCS, length(DocIdAccFwd)), ?assertEqual(DocIds, lists:reverse(DocIdAccFwd)), - {ok, DocIdAccRev} = Engine:FoldFun(St, fun fold_fun/2, [], [{dir, rev}]), + Opts = [{dir, rev}], + {ok, DocIdAccRev} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], Opts), ?assertEqual(?NUM_DOCS, length(DocIdAccRev)), ?assertEqual(DocIds, DocIdAccRev). fold_start_key(FoldFun, DocIdFun) -> - {ok, Engine, St} = init_st(DocIdFun), + {ok, Db} = init_st(DocIdFun), StartKeyNum = ?NUM_DOCS div 4, StartKey = DocIdFun(StartKeyNum), @@ -103,35 +105,35 @@ fold_start_key(FoldFun, DocIdFun) -> DocIdsFwd = [DocIdFun(I) || I <- lists:seq(StartKeyNum, ?NUM_DOCS)], DocIdsRev = [DocIdFun(I) || I <- lists:seq(1, StartKeyNum)], - ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun/2, [], [ + ?assertEqual({ok, []}, couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {start_key, <<255>>} ])), - ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun/2, [], [ + ?assertEqual({ok, []}, couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {dir, rev}, {start_key, <<"">>} ])), - {ok, AllDocIdAccFwd} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {ok, AllDocIdAccFwd} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {start_key, <<"">>} ]), ?assertEqual(length(AllDocIds), length(AllDocIdAccFwd)), ?assertEqual(AllDocIds, lists:reverse(AllDocIdAccFwd)), - {ok, AllDocIdAccRev} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {ok, AllDocIdAccRev} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {dir, rev}, {start_key, <<255>>} ]), ?assertEqual(length(AllDocIds), length(AllDocIdAccRev)), ?assertEqual(AllDocIds, AllDocIdAccRev), - {ok, DocIdAccFwd} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {ok, DocIdAccFwd} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {start_key, StartKey} ]), ?assertEqual(length(DocIdsFwd), length(DocIdAccFwd)), ?assertEqual(DocIdsFwd, lists:reverse(DocIdAccFwd)), - {ok, DocIdAccRev} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {ok, DocIdAccRev} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {dir, rev}, {start_key, StartKey} ]), @@ -140,29 +142,29 @@ fold_start_key(FoldFun, DocIdFun) -> fold_end_key(FoldFun, DocIdFun) -> - {ok, Engine, St} = init_st(DocIdFun), + {ok, Db} = init_st(DocIdFun), EndKeyNum = ?NUM_DOCS div 4, EndKey = DocIdFun(EndKeyNum), - ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun/2, [], [ + ?assertEqual({ok, []}, couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {end_key, <<"">>} ])), - ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun/2, [], [ + ?assertEqual({ok, []}, couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {dir, rev}, {end_key, <<255>>} ])), AllDocIds = [DocIdFun(I) || I <- lists:seq(1, ?NUM_DOCS)], - {ok, AllDocIdAccFwd} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {ok, AllDocIdAccFwd} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {end_key, <<255>>} ]), ?assertEqual(length(AllDocIds), length(AllDocIdAccFwd)), ?assertEqual(AllDocIds, lists:reverse(AllDocIdAccFwd)), - {ok, AllDocIdAccRev} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {ok, AllDocIdAccRev} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {dir, rev}, {end_key, <<"">>} ]), @@ -171,7 +173,7 @@ fold_end_key(FoldFun, DocIdFun) -> DocIdsFwd = [DocIdFun(I) || I <- lists:seq(1, EndKeyNum)], - {ok, DocIdAccFwd} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {ok, DocIdAccFwd} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {end_key, EndKey} ]), ?assertEqual(length(DocIdsFwd), length(DocIdAccFwd)), @@ -179,7 +181,7 @@ fold_end_key(FoldFun, DocIdFun) -> DocIdsRev = [DocIdFun(I) || I <- lists:seq(EndKeyNum, ?NUM_DOCS)], - {ok, DocIdAccRev} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {ok, DocIdAccRev} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {dir, rev}, {end_key, EndKey} ]), @@ -188,29 +190,29 @@ fold_end_key(FoldFun, DocIdFun) -> fold_end_key_gt(FoldFun, DocIdFun) -> - {ok, Engine, St} = init_st(DocIdFun), + {ok, Db} = init_st(DocIdFun), EndKeyNum = ?NUM_DOCS div 4, EndKey = DocIdFun(EndKeyNum), - ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun/2, [], [ + ?assertEqual({ok, []}, couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {end_key_gt, <<"">>} ])), - ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun/2, [], [ + ?assertEqual({ok, []}, couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {dir, rev}, {end_key_gt, <<255>>} ])), AllDocIds = [DocIdFun(I) || I <- lists:seq(1, ?NUM_DOCS)], - {ok, AllDocIdAccFwd} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {ok, AllDocIdAccFwd} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {end_key_gt, <<255>>} ]), ?assertEqual(length(AllDocIds), length(AllDocIdAccFwd)), ?assertEqual(AllDocIds, lists:reverse(AllDocIdAccFwd)), - {ok, AllDocIdAccRev} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {ok, AllDocIdAccRev} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {dir, rev}, {end_key_gt, <<"">>} ]), @@ -219,7 +221,7 @@ fold_end_key_gt(FoldFun, DocIdFun) -> DocIdsFwd = [DocIdFun(I) || I <- lists:seq(1, EndKeyNum - 1)], - {ok, DocIdAccFwd} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {ok, DocIdAccFwd} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {end_key_gt, EndKey} ]), ?assertEqual(length(DocIdsFwd), length(DocIdAccFwd)), @@ -227,7 +229,7 @@ fold_end_key_gt(FoldFun, DocIdFun) -> DocIdsRev = [DocIdFun(I) || I <- lists:seq(EndKeyNum + 1, ?NUM_DOCS)], - {ok, DocIdAccRev} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {ok, DocIdAccRev} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {dir, rev}, {end_key_gt, EndKey} ]), @@ -236,7 +238,7 @@ fold_end_key_gt(FoldFun, DocIdFun) -> fold_range(FoldFun, DocIdFun) -> - {ok, Engine, St} = init_st(DocIdFun), + {ok, Db} = init_st(DocIdFun), StartKeyNum = ?NUM_DOCS div 4, EndKeyNum = StartKeyNum * 3, @@ -244,12 +246,12 @@ fold_range(FoldFun, DocIdFun) -> StartKey = DocIdFun(StartKeyNum), EndKey = DocIdFun(EndKeyNum), - ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun/2, [], [ + ?assertEqual({ok, []}, couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {start_key, <<"">>}, {end_key, <<"">>} ])), - ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun/2, [], [ + ?assertEqual({ok, []}, couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {dir, rev}, {start_key, <<"">>}, {end_key, <<255>>} @@ -257,14 +259,14 @@ fold_range(FoldFun, DocIdFun) -> AllDocIds = [DocIdFun(I) || I <- lists:seq(1, ?NUM_DOCS)], - {ok, AllDocIdAccFwd} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {ok, AllDocIdAccFwd} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {start_key, <<"">>}, {end_key, <<255>>} ]), ?assertEqual(length(AllDocIds), length(AllDocIdAccFwd)), ?assertEqual(AllDocIds, lists:reverse(AllDocIdAccFwd)), - {ok, AllDocIdAccRev} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {ok, AllDocIdAccRev} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {dir, rev}, {start_key, <<255>>}, {end_key_gt, <<"">>} @@ -274,7 +276,7 @@ fold_range(FoldFun, DocIdFun) -> DocIdsFwd = [DocIdFun(I) || I <- lists:seq(StartKeyNum, EndKeyNum)], - {ok, DocIdAccFwd} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {ok, DocIdAccFwd} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {start_key, StartKey}, {end_key, EndKey} ]), @@ -283,13 +285,13 @@ fold_range(FoldFun, DocIdFun) -> DocIdsRev = [DocIdFun(I) || I <- lists:seq(StartKeyNum, EndKeyNum)], - ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun/2, [], [ + ?assertEqual({ok, []}, couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {dir, rev}, {start_key, StartKey}, {end_key, EndKey} ])), - {ok, DocIdAccRev} = Engine:FoldFun(St, fun fold_fun/2, [], [ + {ok, DocIdAccRev} = couch_db_engine:FoldFun(Db, fun fold_fun/2, [], [ {dir, rev}, {start_key, EndKey}, {end_key, StartKey} @@ -298,24 +300,24 @@ fold_range(FoldFun, DocIdFun) -> ?assertEqual(DocIdsRev, DocIdAccRev). -fold_stop(FoldFun, DocIdFun) -> - {ok, Engine, St} = init_st(DocIdFun), +fold_user_fun_stop(FoldFun, DocIdFun) -> + {ok, Db} = init_st(DocIdFun), StartKeyNum = ?NUM_DOCS div 4, StartKey = DocIdFun(StartKeyNum), - ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun_stop/2, [], [ + ?assertEqual({ok, []}, couch_db_engine:FoldFun(Db, fun fold_stop/2, [], [ {start_key, <<255>>} ])), - ?assertEqual({ok, []}, Engine:FoldFun(St, fun fold_fun_stop/2, [], [ + ?assertEqual({ok, []}, couch_db_engine:FoldFun(Db, fun fold_stop/2, [], [ {dir, rev}, {start_key, <<"">>} ])), SuffixDocIds = [DocIdFun(I) || I <- lists:seq(?NUM_DOCS - 3, ?NUM_DOCS)], - {ok, SuffixDocIdAcc} = Engine:FoldFun(St, fun fold_fun_stop/2, [], [ + {ok, SuffixDocIdAcc} = couch_db_engine:FoldFun(Db, fun fold_stop/2, [], [ {start_key, DocIdFun(?NUM_DOCS - 3)} ]), ?assertEqual(length(SuffixDocIds), length(SuffixDocIdAcc)), @@ -323,7 +325,7 @@ fold_stop(FoldFun, DocIdFun) -> PrefixDocIds = [DocIdFun(I) || I <- lists:seq(1, 3)], - {ok, PrefixDocIdAcc} = Engine:FoldFun(St, fun fold_fun_stop/2, [], [ + {ok, PrefixDocIdAcc} = couch_db_engine:FoldFun(Db, fun fold_stop/2, [], [ {dir, rev}, {start_key, DocIdFun(3)} ]), @@ -333,7 +335,7 @@ fold_stop(FoldFun, DocIdFun) -> FiveDocIdsFwd = [DocIdFun(I) || I <- lists:seq(StartKeyNum, StartKeyNum + 5)], - {ok, FiveDocIdAccFwd} = Engine:FoldFun(St, fun fold_fun_stop/2, [], [ + {ok, FiveDocIdAccFwd} = couch_db_engine:FoldFun(Db, fun fold_stop/2, [], [ {start_key, StartKey} ]), ?assertEqual(length(FiveDocIdsFwd), length(FiveDocIdAccFwd)), @@ -342,7 +344,7 @@ fold_stop(FoldFun, DocIdFun) -> FiveDocIdsRev = [DocIdFun(I) || I <- lists:seq(StartKeyNum - 5, StartKeyNum)], - {ok, FiveDocIdAccRev} = Engine:FoldFun(St, fun fold_fun_stop/2, [], [ + {ok, FiveDocIdAccRev} = couch_db_engine:FoldFun(Db, fun fold_stop/2, [], [ {dir, rev}, {start_key, StartKey} ]), @@ -351,12 +353,11 @@ fold_stop(FoldFun, DocIdFun) -> init_st(DocIdFun) -> - {ok, Engine, St1} = test_engine_util:init_engine(), + {ok, Db1} = test_engine_util:create_db(), Actions = lists:map(fun(Id) -> - {create, {DocIdFun(Id), [{<<"int">>, Id}]}} + {create, {DocIdFun(Id), {[{<<"int">>, Id}]}}} end, lists:seq(1, ?NUM_DOCS)), - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), - {ok, Engine, St2}. + test_engine_util:apply_batch(Db1, Actions). fold_fun(Doc, Acc) -> @@ -367,7 +368,7 @@ fold_fun(Doc, Acc) -> {ok, [Id | Acc]}. -fold_fun_stop(Doc, Acc) -> +fold_stop(Doc, Acc) -> Id = case Doc of #doc{id = Id0} -> Id0; #full_doc_info{id = Id0} -> Id0 diff --git a/src/couch/src/test_engine_fold_purge_infos.erl b/src/couch/src/test_engine_fold_purge_infos.erl index 74556c2ca..3422e0581 100644 --- a/src/couch/src/test_engine_fold_purge_infos.erl +++ b/src/couch/src/test_engine_fold_purge_infos.erl @@ -22,23 +22,24 @@ cet_empty_purged_docs() -> - {ok, Engine, St} = test_engine_util:init_engine(), - ?assertEqual({ok, []}, Engine:fold_purge_infos(St, 0, fun fold_fun/2, [], [])). + {ok, Db} = test_engine_util:create_db(), + ?assertEqual({ok, []}, couch_db_engine:fold_purge_infos( + Db, 0, fun fold_fun/2, [], [])). cet_all_purged_docs() -> - {ok, Engine, St1} = test_engine_util:init_engine(), + {ok, Db1} = test_engine_util:create_db(), {RActions, RIds} = lists:foldl(fun(Id, {CActions, CIds}) -> Id1 = docid(Id), - Action = {create, {Id1, [{<<"int">>, Id}]}}, + Action = {create, {Id1, {[{<<"int">>, Id}]}}}, {[Action| CActions], [Id1| CIds]} end, {[], []}, lists:seq(1, ?NUM_DOCS)), Actions = lists:reverse(RActions), Ids = lists:reverse(RIds), - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), + {ok, Db2} = test_engine_util:apply_batch(Db1, Actions), - FDIs = Engine:open_docs(St2, Ids), + FDIs = couch_db_engine:open_docs(Db2, Ids), {RevActions2, RevIdRevs} = lists:foldl(fun(FDI, {CActions, CIdRevs}) -> Id = FDI#full_doc_info.id, PrevRev = test_engine_util:prev_rev(FDI), @@ -48,24 +49,25 @@ cet_all_purged_docs() -> end, {[], []}, FDIs), {Actions2, IdsRevs} = {lists:reverse(RevActions2), lists:reverse(RevIdRevs)}, - {ok, St3} = test_engine_util:apply_actions(Engine, St2, Actions2), - {ok, PurgedIdRevs} = Engine:fold_purge_infos(St3, 0, fun fold_fun/2, [], []), + {ok, Db3} = test_engine_util:apply_batch(Db2, Actions2), + {ok, PurgedIdRevs} = couch_db_engine:fold_purge_infos( + Db3, 0, fun fold_fun/2, [], []), ?assertEqual(IdsRevs, lists:reverse(PurgedIdRevs)). cet_start_seq() -> - {ok, Engine, St1} = test_engine_util:init_engine(), + {ok, Db1} = test_engine_util:create_db(), Actions1 = [ - {create, {docid(1), [{<<"int">>, 1}]}}, - {create, {docid(2), [{<<"int">>, 2}]}}, - {create, {docid(3), [{<<"int">>, 3}]}}, - {create, {docid(4), [{<<"int">>, 4}]}}, - {create, {docid(5), [{<<"int">>, 5}]}} + {create, {docid(1), {[{<<"int">>, 1}]}}}, + {create, {docid(2), {[{<<"int">>, 2}]}}}, + {create, {docid(3), {[{<<"int">>, 3}]}}}, + {create, {docid(4), {[{<<"int">>, 4}]}}}, + {create, {docid(5), {[{<<"int">>, 5}]}}} ], Ids = [docid(1), docid(2), docid(3), docid(4), docid(5)], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1), + {ok, Db2} = test_engine_util:apply_actions(Db1, Actions1), - FDIs = Engine:open_docs(St2, Ids), + FDIs = couch_db_engine:open_docs(Db2, Ids), {RActions2, RIdRevs} = lists:foldl(fun(FDI, {CActions, CIdRevs}) -> Id = FDI#full_doc_info.id, PrevRev = test_engine_util:prev_rev(FDI), @@ -73,55 +75,83 @@ cet_start_seq() -> Action = {purge, {Id, Rev}}, {[Action| CActions], [{Id, [Rev]}| CIdRevs]} end, {[], []}, FDIs), - {ok, St3} = test_engine_util:apply_actions(Engine, St2, lists:reverse(RActions2)), + {ok, Db3} = test_engine_util:apply_actions(Db2, lists:reverse(RActions2)), StartSeq = 3, StartSeqIdRevs = lists:nthtail(StartSeq, lists:reverse(RIdRevs)), - {ok, PurgedIdRevs} = Engine:fold_purge_infos(St3, StartSeq, fun fold_fun/2, [], []), + {ok, PurgedIdRevs} = couch_db_engine:fold_purge_infos( + Db3, StartSeq, fun fold_fun/2, [], []), ?assertEqual(StartSeqIdRevs, lists:reverse(PurgedIdRevs)). cet_id_rev_repeated() -> - {ok, Engine, St1} = test_engine_util:init_engine(), + {ok, Db1} = test_engine_util:create_db(), Actions1 = [ - {create, {<<"foo">>, [{<<"vsn">>, 1}]}}, - {conflict, {<<"foo">>, [{<<"vsn">>, 2}]}} + {create, {<<"foo">>, {[{<<"vsn">>, 1}]}}}, + {conflict, {<<"foo">>, {[{<<"vsn">>, 2}]}}} ], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1), + {ok, Db2} = test_engine_util:apply_actions(Db1, Actions1), - [FDI1] = Engine:open_docs(St2, [<<"foo">>]), + [FDI1] = couch_db_engine:open_docs(Db2, [<<"foo">>]), PrevRev1 = test_engine_util:prev_rev(FDI1), Rev1 = PrevRev1#rev_info.rev, Actions2 = [ {purge, {<<"foo">>, Rev1}} ], - {ok, St3} = test_engine_util:apply_actions(Engine, St2, Actions2), - PurgedIdRevs0 = [{<<"foo">>, [Rev1]}], - {ok, PurgedIdRevs1} = Engine:fold_purge_infos(St3, 0, fun fold_fun/2, [], []), - ?assertEqual(PurgedIdRevs0, PurgedIdRevs1), - ?assertEqual(1, Engine:get_purge_seq(St3)), + + {ok, Db3} = test_engine_util:apply_actions(Db2, Actions2), + {ok, PurgedIdRevs1} = couch_db_engine:fold_purge_infos( + Db3, 0, fun fold_fun/2, [], []), + ExpectedPurgedIdRevs1 = [ + {<<"foo">>, [Rev1]} + ], + + ?assertEqual(ExpectedPurgedIdRevs1, lists:reverse(PurgedIdRevs1)), + ?assertEqual(1, couch_db_engine:get_purge_seq(Db3)), % purge the same Id,Rev when the doc still exists - {ok, St4} = test_engine_util:apply_actions(Engine, St3, Actions2), - {ok, PurgedIdRevs2} = Engine:fold_purge_infos(St4, 0, fun fold_fun/2, [], []), - ?assertEqual(PurgedIdRevs0, PurgedIdRevs2), - ?assertEqual(1, Engine:get_purge_seq(St4)), + {ok, Db4} = test_engine_util:apply_actions(Db3, Actions2), + {ok, PurgedIdRevs2} = couch_db_engine:fold_purge_infos( + Db4, 0, fun fold_fun/2, [], []), + ExpectedPurgedIdRevs2 = [ + {<<"foo">>, [Rev1]}, + {<<"foo">>, [Rev1]} + ], + ?assertEqual(ExpectedPurgedIdRevs2, lists:reverse(PurgedIdRevs2)), + ?assertEqual(2, couch_db_engine:get_purge_seq(Db4)), - [FDI2] = Engine:open_docs(St4, [<<"foo">>]), + [FDI2] = couch_db_engine:open_docs(Db4, [<<"foo">>]), PrevRev2 = test_engine_util:prev_rev(FDI2), Rev2 = PrevRev2#rev_info.rev, Actions3 = [ {purge, {<<"foo">>, Rev2}} ], - {ok, St5} = test_engine_util:apply_actions(Engine, St4, Actions3), - PurgedIdRevs00 = [{<<"foo">>, [Rev1]}, {<<"foo">>, [Rev2]}], + {ok, Db5} = test_engine_util:apply_actions(Db4, Actions3), + + {ok, PurgedIdRevs3} = couch_db_engine:fold_purge_infos( + Db5, 0, fun fold_fun/2, [], []), + ExpectedPurgedIdRevs3 = [ + {<<"foo">>, [Rev1]}, + {<<"foo">>, [Rev1]}, + {<<"foo">>, [Rev2]} + ], + ?assertEqual(ExpectedPurgedIdRevs3, lists:reverse(PurgedIdRevs3)), + ?assertEqual(3, couch_db_engine:get_purge_seq(Db5)), % purge the same Id,Rev when the doc was completely purged - {ok, St6} = test_engine_util:apply_actions(Engine, St5, Actions3), - {ok, PurgedIdRevs3} = Engine:fold_purge_infos(St6, 0, fun fold_fun/2, [], []), - ?assertEqual(PurgedIdRevs00, lists:reverse(PurgedIdRevs3)), - ?assertEqual(2, Engine:get_purge_seq(St6)). + {ok, Db6} = test_engine_util:apply_actions(Db5, Actions3), + + {ok, PurgedIdRevs4} = couch_db_engine:fold_purge_infos( + Db6, 0, fun fold_fun/2, [], []), + ExpectedPurgedIdRevs4 = [ + {<<"foo">>, [Rev1]}, + {<<"foo">>, [Rev1]}, + {<<"foo">>, [Rev2]}, + {<<"foo">>, [Rev2]} + ], + ?assertEqual(ExpectedPurgedIdRevs4, lists:reverse(PurgedIdRevs4)), + ?assertEqual(4, couch_db_engine:get_purge_seq(Db6)). fold_fun({_PSeq, _UUID, Id, Revs}, Acc) -> diff --git a/src/couch/src/test_engine_get_set_props.erl b/src/couch/src/test_engine_get_set_props.erl index 74cbd4400..07a3932fd 100644 --- a/src/couch/src/test_engine_get_set_props.erl +++ b/src/couch/src/test_engine_get_set_props.erl @@ -18,33 +18,53 @@ cet_default_props() -> - Engine = test_engine_util:get_engine(), - DbPath = test_engine_util:dbpath(), - - {ok, St} = Engine:init(DbPath, [ - create, - {default_security_object, dso} - ]), - + {ok, {_App, Engine, _Extension}} = application:get_env(couch, test_engine), + {ok, Db} = test_engine_util:create_db(), Node = node(), - ?assertEqual(0, Engine:get_doc_count(St)), - ?assertEqual(0, Engine:get_del_doc_count(St)), - ?assertEqual(true, is_list(Engine:get_size_info(St))), - ?assertEqual(true, is_integer(Engine:get_disk_version(St))), - ?assertEqual(0, Engine:get_update_seq(St)), - ?assertEqual(0, Engine:get_purge_seq(St)), - ?assertEqual(true, is_integer(Engine:get_purge_infos_limit(St))), - ?assertEqual(true, Engine:get_purge_infos_limit(St) > 0), - ?assertEqual(dso, Engine:get_security(St)), - ?assertEqual(1000, Engine:get_revs_limit(St)), - ?assertMatch(<<_:32/binary>>, Engine:get_uuid(St)), - ?assertEqual([{Node, 0}], Engine:get_epochs(St)), - ?assertEqual(0, Engine:get_compacted_seq(St)). + ?assertEqual(Engine, couch_db_engine:get_engine(Db)), + ?assertEqual(0, couch_db_engine:get_doc_count(Db)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db)), + ?assertEqual(true, is_list(couch_db_engine:get_size_info(Db))), + ?assertEqual(true, is_integer(couch_db_engine:get_disk_version(Db))), + ?assertEqual(0, couch_db_engine:get_update_seq(Db)), + ?assertEqual(0, couch_db_engine:get_purge_seq(Db)), + ?assertEqual(true, is_integer(couch_db_engine:get_purge_infos_limit(Db))), + ?assertEqual(true, couch_db_engine:get_purge_infos_limit(Db) > 0), + ?assertEqual([], couch_db_engine:get_security(Db)), + ?assertEqual(1000, couch_db_engine:get_revs_limit(Db)), + ?assertMatch(<<_:32/binary>>, couch_db_engine:get_uuid(Db)), + ?assertEqual([{Node, 0}], couch_db_engine:get_epochs(Db)), + ?assertEqual(0, couch_db_engine:get_compacted_seq(Db)). + + +-define(ADMIN_ONLY_SEC_PROPS, {[ + {<<"members">>, {[ + {<<"roles">>, [<<"_admin">>]} + ]}}, + {<<"admins">>, {[ + {<<"roles">>, [<<"_admin">>]} + ]}} +]}). + + +cet_admin_only_security() -> + Config = [{"couchdb", "default_security", "admin_only"}], + {ok, Db1} = test_engine_util:with_config(Config, fun() -> + test_engine_util:create_db() + end), + + ?assertEqual(?ADMIN_ONLY_SEC_PROPS, couch_db:get_security(Db1)), + test_engine_util:shutdown_db(Db1), + + {ok, Db2} = couch_db:reopen(Db1), + couch_log:error("~n~n~n~n~s -> ~s~n~n", [couch_db:name(Db1), couch_db:name(Db2)]), + ?assertEqual(?ADMIN_ONLY_SEC_PROPS, couch_db:get_security(Db2)). cet_set_security() -> - check_prop_set(get_security, set_security, dso, [{<<"readers">>, []}]). + SecProps = {[{<<"foo">>, <<"bar">>}]}, + check_prop_set(get_security, set_security, {[]}, SecProps). cet_set_revs_limit() -> @@ -52,20 +72,16 @@ cet_set_revs_limit() -> check_prop_set(GetFun, SetFun, Default, Value) -> - Engine = test_engine_util:get_engine(), - DbPath = test_engine_util:dbpath(), + {ok, Db0} = test_engine_util:create_db(), - {ok, St0} = Engine:init(DbPath, [ - create, - {default_security_object, dso} - ]), - ?assertEqual(Default, Engine:GetFun(St0)), + ?assertEqual(Default, couch_db:GetFun(Db0)), + ?assertMatch(ok, couch_db:SetFun(Db0, Value)), - {ok, St1} = Engine:SetFun(St0, Value), - ?assertEqual(Value, Engine:GetFun(St1)), + {ok, Db1} = couch_db:reopen(Db0), + ?assertEqual(Value, couch_db:GetFun(Db1)), - {ok, St2} = Engine:commit_data(St1), - Engine:terminate(normal, St2), + ?assertMatch({ok, _}, couch_db:ensure_full_commit(Db1)), + test_engine_util:shutdown_db(Db1), - {ok, St3} = Engine:init(DbPath, []), - ?assertEqual(Value, Engine:GetFun(St3)). + {ok, Db2} = couch_db:reopen(Db1), + ?assertEqual(Value, couch_db:GetFun(Db2)). diff --git a/src/couch/src/test_engine_open_close_delete.erl b/src/couch/src/test_engine_open_close_delete.erl index b099d9fb0..ce0187ef9 100644 --- a/src/couch/src/test_engine_open_close_delete.erl +++ b/src/couch/src/test_engine_open_close_delete.erl @@ -18,64 +18,59 @@ cet_open_non_existent() -> - Engine = test_engine_util:get_engine(), - DbPath = test_engine_util:dbpath(), - - ?assertEqual(false, Engine:exists(DbPath)), - ?assertThrow({not_found, no_db_file}, Engine:init(DbPath, [])), - ?assertEqual(false, Engine:exists(DbPath)). + % Try twice to check that a failed open doesn't create + % the database for some reason. + DbName = test_engine_util:dbname(), + ?assertEqual({not_found, no_db_file}, test_engine_util:open_db(DbName)), + ?assertEqual({not_found, no_db_file}, test_engine_util:open_db(DbName)). cet_open_create() -> - process_flag(trap_exit, true), - Engine = test_engine_util:get_engine(), - DbPath = test_engine_util:dbpath(), + DbName = test_engine_util:dbname(), - ?assertEqual(false, Engine:exists(DbPath)), - ?assertMatch({ok, _}, Engine:init(DbPath, [create])), - ?assertEqual(true, Engine:exists(DbPath)). + ?assertEqual(false, couch_server:exists(DbName)), + ?assertEqual({not_found, no_db_file}, test_engine_util:open_db(DbName)), + ?assertMatch({ok, _}, test_engine_util:create_db(DbName)), + ?assertEqual(true, couch_server:exists(DbName)). cet_open_when_exists() -> - Engine = test_engine_util:get_engine(), - DbPath = test_engine_util:dbpath(), + DbName = test_engine_util:dbname(), - ?assertEqual(false, Engine:exists(DbPath)), - ?assertMatch({ok, _}, Engine:init(DbPath, [create])), - ?assertThrow({error, eexist}, Engine:init(DbPath, [create])). + ?assertEqual(false, couch_server:exists(DbName)), + ?assertEqual({not_found, no_db_file}, test_engine_util:open_db(DbName)), + ?assertMatch({ok, _}, test_engine_util:create_db(DbName)), + ?assertEqual(file_exists, test_engine_util:create_db(DbName)). cet_terminate() -> - Engine = test_engine_util:get_engine(), - DbPath = test_engine_util:dbpath(), + DbName = test_engine_util:dbname(), - ?assertEqual(false, Engine:exists(DbPath)), - {ok, St} = Engine:init(DbPath, [create]), - Engine:terminate(normal, St), - ?assertEqual(true, Engine:exists(DbPath)). + ?assertEqual(false, couch_server:exists(DbName)), + ?assertEqual({not_found, no_db_file}, test_engine_util:open_db(DbName)), + ?assertEqual(ok, cycle_db(DbName, create_db)), + ?assertEqual(true, couch_server:exists(DbName)). cet_rapid_recycle() -> - Engine = test_engine_util:get_engine(), - DbPath = test_engine_util:dbpath(), - - {ok, St0} = Engine:init(DbPath, [create]), - Engine:terminate(normal, St0), + DbName = test_engine_util:dbname(), + ?assertEqual(ok, cycle_db(DbName, create_db)), lists:foreach(fun(_) -> - {ok, St1} = Engine:init(DbPath, []), - Engine:terminate(normal, St1) + ?assertEqual(ok, cycle_db(DbName, open_db)) end, lists:seq(1, 100)). cet_delete() -> - Engine = test_engine_util:get_engine(), - RootDir = test_engine_util:rootdir(), - DbPath = test_engine_util:dbpath(), - - ?assertEqual(false, Engine:exists(DbPath)), - {ok, St} = Engine:init(DbPath, [create]), - Engine:terminate(normal, St), - ?assertEqual(true, Engine:exists(DbPath)), - ?assertEqual(ok, Engine:delete(RootDir, DbPath, [async])), - ?assertEqual(false, Engine:exists(DbPath)). + DbName = test_engine_util:dbname(), + + ?assertEqual(false, couch_server:exists(DbName)), + ?assertMatch(ok, cycle_db(DbName, create_db)), + ?assertEqual(true, couch_server:exists(DbName)), + ?assertEqual(ok, couch_server:delete(DbName, [])), + ?assertEqual(false, couch_server:exists(DbName)). + + +cycle_db(DbName, Type) -> + {ok, Db} = test_engine_util:Type(DbName), + test_engine_util:shutdown_db(Db). diff --git a/src/couch/src/test_engine_purge_docs.erl b/src/couch/src/test_engine_purge_docs.erl index 58b605554..8e6dbc0c3 100644 --- a/src/couch/src/test_engine_purge_docs.erl +++ b/src/couch/src/test_engine_purge_docs.erl @@ -16,198 +16,208 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("couch/include/couch_db.hrl"). --include("couch_bt_engine.hrl"). cet_purge_simple() -> - {ok, Engine, St1} = test_engine_util:init_engine(), + {ok, Db1} = test_engine_util:create_db(), Actions1 = [ - {create, {<<"foo">>, [{<<"vsn">>, 1}]}} + {create, {<<"foo">>, {[{<<"vsn">>, 1}]}}} ], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1), - {ok, PIdRevs2} = Engine:fold_purge_infos(St2, 0, fun fold_fun/2, [], []), - - ?assertEqual(1, Engine:get_doc_count(St2)), - ?assertEqual(0, Engine:get_del_doc_count(St2)), - ?assertEqual(1, Engine:get_update_seq(St2)), - ?assertEqual(0, Engine:get_purge_seq(St2)), + {ok, Db2} = test_engine_util:apply_actions(Db1, Actions1), + {ok, PIdRevs2} = couch_db_engine:fold_purge_infos( + Db2, 0, fun fold_fun/2, [], []), + + ?assertEqual(1, couch_db_engine:get_doc_count(Db2)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db2)), + ?assertEqual(1, couch_db_engine:get_update_seq(Db2)), + ?assertEqual(0, couch_db_engine:get_purge_seq(Db2)), ?assertEqual([], PIdRevs2), - [FDI] = Engine:open_docs(St2, [<<"foo">>]), + [FDI] = couch_db_engine:open_docs(Db2, [<<"foo">>]), PrevRev = test_engine_util:prev_rev(FDI), Rev = PrevRev#rev_info.rev, Actions2 = [ {purge, {<<"foo">>, Rev}} ], - {ok, St3} = test_engine_util:apply_actions(Engine, St2, Actions2), - {ok, PIdRevs3} = Engine:fold_purge_infos(St3, 0, fun fold_fun/2, [], []), - - ?assertEqual(0, Engine:get_doc_count(St3)), - ?assertEqual(0, Engine:get_del_doc_count(St3)), - ?assertEqual(2, Engine:get_update_seq(St3)), - ?assertEqual(1, Engine:get_purge_seq(St3)), + {ok, Db3} = test_engine_util:apply_actions(Db2, Actions2), + {ok, PIdRevs3} = couch_db_engine:fold_purge_infos( + Db3, 0, fun fold_fun/2, [], []), + + ?assertEqual(0, couch_db_engine:get_doc_count(Db3)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db3)), + ?assertEqual(2, couch_db_engine:get_update_seq(Db3)), + ?assertEqual(1, couch_db_engine:get_purge_seq(Db3)), ?assertEqual([{<<"foo">>, [Rev]}], PIdRevs3). cet_purge_UUID() -> - {ok, Engine, St1} = test_engine_util:init_engine(), + {ok, Db1} = test_engine_util:create_db(), Actions1 = [ - {create, {<<"foo">>, [{<<"vsn">>, 1}]}} + {create, {<<"foo">>, {[{<<"vsn">>, 1}]}}} ], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1), - {ok, PIdRevs2} = Engine:fold_purge_infos(St2, 0, fun fold_fun/2, [], []), - - ?assertEqual(1, Engine:get_doc_count(St2)), - ?assertEqual(0, Engine:get_del_doc_count(St2)), - ?assertEqual(1, Engine:get_update_seq(St2)), - ?assertEqual(0, Engine:get_purge_seq(St2)), + {ok, Db2} = test_engine_util:apply_actions(Db1, Actions1), + {ok, PIdRevs2} = couch_db_engine:fold_purge_infos( + Db2, 0, fun fold_fun/2, [], []), + + ?assertEqual(1, couch_db_engine:get_doc_count(Db2)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db2)), + ?assertEqual(1, couch_db_engine:get_update_seq(Db2)), + ?assertEqual(0, couch_db_engine:get_purge_seq(Db2)), ?assertEqual([], PIdRevs2), - [FDI] = Engine:open_docs(St2, [<<"foo">>]), + [FDI] = couch_db_engine:open_docs(Db2, [<<"foo">>]), PrevRev = test_engine_util:prev_rev(FDI), Rev = PrevRev#rev_info.rev, Actions2 = [ {purge, {<<"foo">>, Rev}} ], - {ok, St3} = test_engine_util:apply_actions(Engine, St2, Actions2), - {ok, _PIdRevs3} = Engine:fold_purge_infos(St3, 0, fun fold_fun/2, [], []), - - ?assertEqual(0, Engine:get_doc_count(St3)), - ?assertEqual(0, Engine:get_del_doc_count(St3)), - ?assertEqual(2, Engine:get_update_seq(St3)), - ?assertEqual(1, Engine:get_purge_seq(St3)), - - PurgeSeqTree = St3#st.purge_seq_tree, - - Fun = fun({PurgeSeq, UUID, _, _}, _Reds, _Acc) -> - {stop, {PurgeSeq, UUID}} - end, - {ok, _, {_, UUID}} = couch_btree:fold( - PurgeSeqTree, Fun, 0, [{dir, rev}] - ), + {ok, Db3} = test_engine_util:apply_actions(Db2, Actions2), + {ok, PIdRevs3} = couch_db_engine:fold_purge_infos( + Db3, 0, fun fold_fun/2, [], []), + + ?assertEqual(0, couch_db_engine:get_doc_count(Db3)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db3)), + ?assertEqual(2, couch_db_engine:get_update_seq(Db3)), + ?assertEqual(1, couch_db_engine:get_purge_seq(Db3)), + ?assertEqual([{<<"foo">>, [Rev]}], PIdRevs3), + + {ok, {PSeq, UUID}} = couch_db_engine:fold_purge_infos( + Db3, 0, fun first_uuid/2, [], []), + ?assertEqual(1, PSeq), ?assert(is_binary(UUID)). cet_purge_conflicts() -> - {ok, Engine, St1} = test_engine_util:init_engine(), + {ok, Db1} = test_engine_util:create_db(), Actions1 = [ - {create, {<<"foo">>, [{<<"vsn">>, 1}]}}, - {conflict, {<<"foo">>, [{<<"vsn">>, 2}]}} + {create, {<<"foo">>, {[{<<"vsn">>, 1}]}}}, + {conflict, {<<"foo">>, {[{<<"vsn">>, 2}]}}} ], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1), - {ok, PIdRevs2} = Engine:fold_purge_infos(St2, 0, fun fold_fun/2, [], []), - - ?assertEqual(1, Engine:get_doc_count(St2)), - ?assertEqual(0, Engine:get_del_doc_count(St2)), - ?assertEqual(2, Engine:get_update_seq(St2)), - ?assertEqual(0, Engine:get_purge_seq(St2)), + {ok, Db2} = test_engine_util:apply_actions(Db1, Actions1), + {ok, PIdRevs2} = couch_db_engine:fold_purge_infos( + Db2, 0, fun fold_fun/2, [], []), + + ?assertEqual(1, couch_db_engine:get_doc_count(Db2)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db2)), + ?assertEqual(2, couch_db_engine:get_update_seq(Db2)), + ?assertEqual(0, couch_db_engine:get_purge_seq(Db2)), ?assertEqual([], PIdRevs2), - [FDI1] = Engine:open_docs(St2, [<<"foo">>]), + [FDI1] = couch_db_engine:open_docs(Db2, [<<"foo">>]), PrevRev1 = test_engine_util:prev_rev(FDI1), Rev1 = PrevRev1#rev_info.rev, Actions2 = [ {purge, {<<"foo">>, Rev1}} ], - {ok, St3} = test_engine_util:apply_actions(Engine, St2, Actions2), - {ok, PIdRevs3} = Engine:fold_purge_infos(St3, 0, fun fold_fun/2, [], []), - - ?assertEqual(1, Engine:get_doc_count(St3)), - ?assertEqual(0, Engine:get_del_doc_count(St3)), - ?assertEqual(3, Engine:get_update_seq(St3)), - ?assertEqual(1, Engine:get_purge_seq(St3)), + {ok, Db3} = test_engine_util:apply_actions(Db2, Actions2), + {ok, PIdRevs3} = couch_db_engine:fold_purge_infos( + Db3, 0, fun fold_fun/2, [], []), + + ?assertEqual(1, couch_db_engine:get_doc_count(Db3)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db3)), + ?assertEqual(3, couch_db_engine:get_update_seq(Db3)), + ?assertEqual(1, couch_db_engine:get_purge_seq(Db3)), ?assertEqual([{<<"foo">>, [Rev1]}], PIdRevs3), - [FDI2] = Engine:open_docs(St3, [<<"foo">>]), + [FDI2] = couch_db_engine:open_docs(Db3, [<<"foo">>]), PrevRev2 = test_engine_util:prev_rev(FDI2), Rev2 = PrevRev2#rev_info.rev, Actions3 = [ {purge, {<<"foo">>, Rev2}} ], - {ok, St4} = test_engine_util:apply_actions(Engine, St3, Actions3), - {ok, PIdRevs4} = Engine:fold_purge_infos(St4, 0, fun fold_fun/2, [], []), - - ?assertEqual(0, Engine:get_doc_count(St4)), - ?assertEqual(0, Engine:get_del_doc_count(St4)), - ?assertEqual(4, Engine:get_update_seq(St4)), - ?assertEqual(2, Engine:get_purge_seq(St4)), + {ok, Db4} = test_engine_util:apply_actions(Db3, Actions3), + {ok, PIdRevs4} = couch_db_engine:fold_purge_infos( + Db4, 0, fun fold_fun/2, [], []), + + ?assertEqual(0, couch_db_engine:get_doc_count(Db4)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db4)), + ?assertEqual(4, couch_db_engine:get_update_seq(Db4)), + ?assertEqual(2, couch_db_engine:get_purge_seq(Db4)), ?assertEqual([{<<"foo">>, [Rev2]}, {<<"foo">>, [Rev1]}], PIdRevs4). cet_add_delete_purge() -> - {ok, Engine, St1} = test_engine_util:init_engine(), + {ok, Db1} = test_engine_util:create_db(), Actions1 = [ - {create, {<<"foo">>, [{<<"vsn">>, 1}]}}, - {delete, {<<"foo">>, [{<<"vsn">>, 2}]}} + {create, {<<"foo">>, {[{<<"vsn">>, 1}]}}}, + {delete, {<<"foo">>, {[{<<"vsn">>, 2}]}}} ], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1), - {ok, PIdRevs2} = Engine:fold_purge_infos(St2, 0, fun fold_fun/2, [], []), + {ok, Db2} = test_engine_util:apply_actions(Db1, Actions1), + {ok, PIdRevs2} = couch_db_engine:fold_purge_infos( + Db2, 0, fun fold_fun/2, [], []), - ?assertEqual(0, Engine:get_doc_count(St2)), - ?assertEqual(1, Engine:get_del_doc_count(St2)), - ?assertEqual(2, Engine:get_update_seq(St2)), - ?assertEqual(0, Engine:get_purge_seq(St2)), + ?assertEqual(0, couch_db_engine:get_doc_count(Db2)), + ?assertEqual(1, couch_db_engine:get_del_doc_count(Db2)), + ?assertEqual(2, couch_db_engine:get_update_seq(Db2)), + ?assertEqual(0, couch_db_engine:get_purge_seq(Db2)), ?assertEqual([], PIdRevs2), - [FDI] = Engine:open_docs(St2, [<<"foo">>]), + [FDI] = couch_db_engine:open_docs(Db2, [<<"foo">>]), PrevRev = test_engine_util:prev_rev(FDI), Rev = PrevRev#rev_info.rev, Actions2 = [ {purge, {<<"foo">>, Rev}} ], - {ok, St3} = test_engine_util:apply_actions(Engine, St2, Actions2), - {ok, PIdRevs3} = Engine:fold_purge_infos(St3, 0, fun fold_fun/2, [], []), - - ?assertEqual(0, Engine:get_doc_count(St3)), - ?assertEqual(0, Engine:get_del_doc_count(St3)), - ?assertEqual(3, Engine:get_update_seq(St3)), - ?assertEqual(1, Engine:get_purge_seq(St3)), + {ok, Db3} = test_engine_util:apply_actions(Db2, Actions2), + {ok, PIdRevs3} = couch_db_engine:fold_purge_infos( + Db3, 0, fun fold_fun/2, [], []), + + ?assertEqual(0, couch_db_engine:get_doc_count(Db3)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db3)), + ?assertEqual(3, couch_db_engine:get_update_seq(Db3)), + ?assertEqual(1, couch_db_engine:get_purge_seq(Db3)), ?assertEqual([{<<"foo">>, [Rev]}], PIdRevs3). cet_add_two_purge_one() -> - {ok, Engine, St1} = test_engine_util:init_engine(), + {ok, Db1} = test_engine_util:create_db(), Actions1 = [ - {create, {<<"foo">>, [{<<"vsn">>, 1}]}}, - {create, {<<"bar">>, []}} + {create, {<<"foo">>, {[{<<"vsn">>, 1}]}}}, + {create, {<<"bar">>, {[]}}} ], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions1), - {ok, PIdRevs2} = Engine:fold_purge_infos(St2, 0, fun fold_fun/2, [], []), + {ok, Db2} = test_engine_util:apply_actions(Db1, Actions1), + {ok, PIdRevs2} = couch_db_engine:fold_purge_infos( + Db2, 0, fun fold_fun/2, [], []), - ?assertEqual(2, Engine:get_doc_count(St2)), - ?assertEqual(0, Engine:get_del_doc_count(St2)), - ?assertEqual(2, Engine:get_update_seq(St2)), - ?assertEqual(0, Engine:get_purge_seq(St2)), + ?assertEqual(2, couch_db_engine:get_doc_count(Db2)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db2)), + ?assertEqual(2, couch_db_engine:get_update_seq(Db2)), + ?assertEqual(0, couch_db_engine:get_purge_seq(Db2)), ?assertEqual([], PIdRevs2), - [FDI] = Engine:open_docs(St2, [<<"foo">>]), + [FDI] = couch_db_engine:open_docs(Db2, [<<"foo">>]), PrevRev = test_engine_util:prev_rev(FDI), Rev = PrevRev#rev_info.rev, Actions2 = [ {purge, {<<"foo">>, Rev}} ], - {ok, St3} = test_engine_util:apply_actions(Engine, St2, Actions2), - {ok, PIdRevs3} = Engine:fold_purge_infos(St3, 0, fun fold_fun/2, [], []), - - ?assertEqual(1, Engine:get_doc_count(St3)), - ?assertEqual(0, Engine:get_del_doc_count(St3)), - ?assertEqual(3, Engine:get_update_seq(St3)), - ?assertEqual(1, Engine:get_purge_seq(St3)), + {ok, Db3} = test_engine_util:apply_actions(Db2, Actions2), + {ok, PIdRevs3} = couch_db_engine:fold_purge_infos( + Db3, 0, fun fold_fun/2, [], []), + + ?assertEqual(1, couch_db_engine:get_doc_count(Db3)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db3)), + ?assertEqual(3, couch_db_engine:get_update_seq(Db3)), + ?assertEqual(1, couch_db_engine:get_purge_seq(Db3)), ?assertEqual([{<<"foo">>, [Rev]}], PIdRevs3). fold_fun({_Pseq, _UUID, Id, Revs}, Acc) -> - {ok, [{Id, Revs} | Acc]}.
\ No newline at end of file + {ok, [{Id, Revs} | Acc]}. + + +first_uuid({PSeq, UUID, _, _}, _) -> + {stop, {PSeq, UUID}}. diff --git a/src/couch/src/test_engine_read_write_docs.erl b/src/couch/src/test_engine_read_write_docs.erl index 4307702d4..2eeeab58e 100644 --- a/src/couch/src/test_engine_read_write_docs.erl +++ b/src/couch/src/test_engine_read_write_docs.erl @@ -19,45 +19,49 @@ cet_read_empty_docs() -> - {ok, Engine, St} = test_engine_util:init_engine(), + {ok, Db} = test_engine_util:create_db(), - ?assertEqual([not_found], Engine:open_docs(St, [<<"foo">>])), + ?assertEqual([not_found], couch_db_engine:open_docs(Db, [<<"foo">>])), ?assertEqual( [not_found, not_found], - Engine:open_docs(St, [<<"a">>, <<"b">>]) + couch_db_engine:open_docs(Db, [<<"a">>, <<"b">>]) ). cet_read_empty_local_docs() -> - {ok, Engine, St} = test_engine_util:init_engine(), + {ok, Db} = test_engine_util:create_db(), - ?assertEqual([not_found], Engine:open_local_docs(St, [<<"_local/foo">>])), + {LocalA, LocalB} = {<<"_local/a">>, <<"_local/b">>}, + ?assertEqual([not_found], couch_db_engine:open_local_docs(Db, [LocalA])), ?assertEqual( [not_found, not_found], - Engine:open_local_docs(St, [<<"_local/a">>, <<"_local/b">>]) + couch_db_engine:open_local_docs(Db, [LocalA, LocalB]) ). cet_write_one_doc() -> - {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), + {ok, Db1} = test_engine_util:create_db(), - ?assertEqual(0, Engine:get_doc_count(St1)), - ?assertEqual(0, Engine:get_del_doc_count(St1)), - ?assertEqual(0, Engine:get_update_seq(St1)), + ?assertEqual(0, couch_db_engine:get_doc_count(Db1)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db1)), + ?assertEqual(0, couch_db_engine:get_update_seq(Db1)), Actions = [ - {create, {<<"foo">>, [{<<"vsn">>, 1}]}} + {create, {<<"foo">>, {[{<<"vsn">>, 1}]}}} ], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), - {ok, St3} = Engine:commit_data(St2), - Engine:terminate(normal, St3), - {ok, St4} = Engine:init(DbPath, []), + {ok, Db2} = test_engine_util:apply_actions(Db1, Actions), + ?assertEqual(1, couch_db_engine:get_doc_count(Db2)), - ?assertEqual(1, Engine:get_doc_count(St4)), - ?assertEqual(0, Engine:get_del_doc_count(St4)), - ?assertEqual(1, Engine:get_update_seq(St4)), + {ok, _} = couch_db:ensure_full_commit(Db2), + test_engine_util:shutdown_db(Db2), - [FDI] = Engine:open_docs(St4, [<<"foo">>]), + {ok, Db3} = couch_db:reopen(Db2), + + ?assertEqual(1, couch_db_engine:get_doc_count(Db3)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db3)), + ?assertEqual(1, couch_db_engine:get_update_seq(Db3)), + + [FDI] = couch_db_engine:open_docs(Db3, [<<"foo">>]), #rev_info{ rev = {RevPos, PrevRevId}, deleted = Deleted, @@ -71,85 +75,88 @@ cet_write_one_doc() -> body = DocPtr }, - Doc1 = Engine:read_doc_body(St4, Doc0), + Doc1 = couch_db_engine:read_doc_body(Db3, Doc0), Body1 = if not is_binary(Doc1#doc.body) -> Doc1#doc.body; true -> couch_compress:decompress(Doc1#doc.body) end, - ?assertEqual([{<<"vsn">>, 1}], Body1). + ?assertEqual({[{<<"vsn">>, 1}]}, Body1). cet_write_two_docs() -> - {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), + {ok, Db1} = test_engine_util:create_db(), - ?assertEqual(0, Engine:get_doc_count(St1)), - ?assertEqual(0, Engine:get_del_doc_count(St1)), - ?assertEqual(0, Engine:get_update_seq(St1)), + ?assertEqual(0, couch_db_engine:get_doc_count(Db1)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db1)), + ?assertEqual(0, couch_db_engine:get_update_seq(Db1)), Actions = [ - {create, {<<"foo">>, [{<<"vsn">>, 1}]}}, - {create, {<<"bar">>, [{<<"stuff">>, true}]}} + {create, {<<"foo">>, {[{<<"vsn">>, 1}]}}}, + {create, {<<"bar">>, {[{<<"stuff">>, true}]}}} ], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), - {ok, St3} = Engine:commit_data(St2), - Engine:terminate(normal, St3), - {ok, St4} = Engine:init(DbPath, []), + {ok, Db2} = test_engine_util:apply_actions(Db1, Actions), + {ok, _} = couch_db:ensure_full_commit(Db2), + test_engine_util:shutdown_db(Db2), + + {ok, Db3} = couch_db:reopen(Db2), - ?assertEqual(2, Engine:get_doc_count(St4)), - ?assertEqual(0, Engine:get_del_doc_count(St4)), - ?assertEqual(2, Engine:get_update_seq(St4)), + ?assertEqual(2, couch_db_engine:get_doc_count(Db3)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db3)), + ?assertEqual(2, couch_db_engine:get_update_seq(Db3)), - Resps = Engine:open_docs(St4, [<<"foo">>, <<"bar">>]), + Resps = couch_db_engine:open_docs(Db3, [<<"foo">>, <<"bar">>]), ?assertEqual(false, lists:member(not_found, Resps)). cet_write_three_doc_batch() -> - {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), + {ok, Db1} = test_engine_util:create_db(), - ?assertEqual(0, Engine:get_doc_count(St1)), - ?assertEqual(0, Engine:get_del_doc_count(St1)), - ?assertEqual(0, Engine:get_update_seq(St1)), + ?assertEqual(0, couch_db_engine:get_doc_count(Db1)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db1)), + ?assertEqual(0, couch_db_engine:get_update_seq(Db1)), Actions = [ {batch, [ - {create, {<<"foo">>, [{<<"vsn">>, 1}]}}, - {create, {<<"bar">>, [{<<"stuff">>, true}]}}, - {create, {<<"baz">>, []}} + {create, {<<"foo">>, {[{<<"vsn">>, 1}]}}}, + {create, {<<"bar">>, {[{<<"stuff">>, true}]}}}, + {create, {<<"baz">>, {[]}}} ]} ], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), - {ok, St3} = Engine:commit_data(St2), - Engine:terminate(normal, St3), - {ok, St4} = Engine:init(DbPath, []), + {ok, Db2} = test_engine_util:apply_actions(Db1, Actions), + {ok, _} = couch_db:ensure_full_commit(Db2), + test_engine_util:shutdown_db(Db2), - ?assertEqual(3, Engine:get_doc_count(St4)), - ?assertEqual(0, Engine:get_del_doc_count(St4)), - ?assertEqual(3, Engine:get_update_seq(St4)), + {ok, Db3} = couch_db:reopen(Db2), - Resps = Engine:open_docs(St4, [<<"foo">>, <<"bar">>, <<"baz">>]), + ?assertEqual(3, couch_db_engine:get_doc_count(Db3)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db3)), + ?assertEqual(3, couch_db_engine:get_update_seq(Db3)), + + Resps = couch_db_engine:open_docs(Db3, [<<"foo">>, <<"bar">>, <<"baz">>]), ?assertEqual(false, lists:member(not_found, Resps)). cet_update_doc() -> - {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), + {ok, Db1} = test_engine_util:create_db(), - ?assertEqual(0, Engine:get_doc_count(St1)), - ?assertEqual(0, Engine:get_del_doc_count(St1)), - ?assertEqual(0, Engine:get_update_seq(St1)), + ?assertEqual(0, couch_db_engine:get_doc_count(Db1)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db1)), + ?assertEqual(0, couch_db_engine:get_update_seq(Db1)), Actions = [ - {create, {<<"foo">>, [{<<"vsn">>, 1}]}}, - {update, {<<"foo">>, [{<<"vsn">>, 2}]}} + {create, {<<"foo">>, {[{<<"vsn">>, 1}]}}}, + {update, {<<"foo">>, {[{<<"vsn">>, 2}]}}} ], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), - {ok, St3} = Engine:commit_data(St2), - Engine:terminate(normal, St3), - {ok, St4} = Engine:init(DbPath, []), + {ok, Db2} = test_engine_util:apply_actions(Db1, Actions), + {ok, _} = couch_db:ensure_full_commit(Db2), + test_engine_util:shutdown_db(Db2), + + {ok, Db3} = couch_db:reopen(Db2), - ?assertEqual(1, Engine:get_doc_count(St4)), - ?assertEqual(0, Engine:get_del_doc_count(St4)), - ?assertEqual(2, Engine:get_update_seq(St4)), + ?assertEqual(1, couch_db_engine:get_doc_count(Db3)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db3)), + ?assertEqual(2, couch_db_engine:get_update_seq(Db3)), - [FDI] = Engine:open_docs(St4, [<<"foo">>]), + [FDI] = couch_db_engine:open_docs(Db3, [<<"foo">>]), #rev_info{ rev = {RevPos, PrevRevId}, @@ -164,35 +171,35 @@ cet_update_doc() -> body = DocPtr }, - Doc1 = Engine:read_doc_body(St4, Doc0), + Doc1 = couch_db_engine:read_doc_body(Db3, Doc0), Body1 = if not is_binary(Doc1#doc.body) -> Doc1#doc.body; true -> couch_compress:decompress(Doc1#doc.body) end, - ?assertEqual([{<<"vsn">>, 2}], Body1). + ?assertEqual({[{<<"vsn">>, 2}]}, Body1). cet_delete_doc() -> - {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), + {ok, Db1} = test_engine_util:create_db(), - ?assertEqual(0, Engine:get_doc_count(St1)), - ?assertEqual(0, Engine:get_del_doc_count(St1)), - ?assertEqual(0, Engine:get_update_seq(St1)), + ?assertEqual(0, couch_db_engine:get_doc_count(Db1)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db1)), + ?assertEqual(0, couch_db_engine:get_update_seq(Db1)), Actions = [ - {create, {<<"foo">>, [{<<"vsn">>, 1}]}}, - {delete, {<<"foo">>, []}} + {create, {<<"foo">>, {[{<<"vsn">>, 1}]}}}, + {delete, {<<"foo">>, {[]}}} ], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), - {ok, St3} = Engine:commit_data(St2), - Engine:terminate(normal, St3), - {ok, St4} = Engine:init(DbPath, []), + {ok, Db2} = test_engine_util:apply_actions(Db1, Actions), + {ok, _} = couch_db:ensure_full_commit(Db2), + test_engine_util:shutdown_db(Db2), - ?assertEqual(0, Engine:get_doc_count(St4)), - ?assertEqual(1, Engine:get_del_doc_count(St4)), - ?assertEqual(2, Engine:get_update_seq(St4)), + {ok, Db3} = couch_db:reopen(Db2), + ?assertEqual(0, couch_db_engine:get_doc_count(Db3)), + ?assertEqual(1, couch_db_engine:get_del_doc_count(Db3)), + ?assertEqual(2, couch_db_engine:get_update_seq(Db3)), - [FDI] = Engine:open_docs(St4, [<<"foo">>]), + [FDI] = couch_db_engine:open_docs(Db3, [<<"foo">>]), #rev_info{ rev = {RevPos, PrevRevId}, @@ -207,111 +214,118 @@ cet_delete_doc() -> body = DocPtr }, - Doc1 = Engine:read_doc_body(St4, Doc0), + Doc1 = couch_db_engine:read_doc_body(Db3, Doc0), Body1 = if not is_binary(Doc1#doc.body) -> Doc1#doc.body; true -> couch_compress:decompress(Doc1#doc.body) end, - ?assertEqual([], Body1). + ?assertEqual({[]}, Body1). cet_write_local_doc() -> - {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), + {ok, Db1} = test_engine_util:create_db(), - ?assertEqual(0, Engine:get_doc_count(St1)), - ?assertEqual(0, Engine:get_del_doc_count(St1)), - ?assertEqual(0, Engine:get_update_seq(St1)), + ?assertEqual(0, couch_db_engine:get_doc_count(Db1)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db1)), + ?assertEqual(0, couch_db_engine:get_update_seq(Db1)), Actions = [ - {create, {<<"_local/foo">>, [{<<"yay">>, false}]}} + {create, {<<"_local/foo">>, {[{<<"yay">>, false}]}}} ], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), - {ok, St3} = Engine:commit_data(St2), - Engine:terminate(normal, St3), - {ok, St4} = Engine:init(DbPath, []), + {ok, Db2} = test_engine_util:apply_actions(Db1, Actions), + {ok, _} = couch_db:ensure_full_commit(Db2), + test_engine_util:shutdown_db(Db2), + + {ok, Db3} = couch_db:reopen(Db2), - ?assertEqual(0, Engine:get_doc_count(St4)), - ?assertEqual(0, Engine:get_del_doc_count(St4)), - ?assertEqual(0, Engine:get_update_seq(St4)), + ?assertEqual(0, couch_db_engine:get_doc_count(Db3)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db3)), + ?assertEqual(0, couch_db_engine:get_update_seq(Db3)), - [not_found] = Engine:open_docs(St4, [<<"_local/foo">>]), - [#doc{} = Doc] = Engine:open_local_docs(St4, [<<"_local/foo">>]), - ?assertEqual([{<<"yay">>, false}], Doc#doc.body). + [not_found] = couch_db_engine:open_docs(Db3, [<<"_local/foo">>]), + [#doc{} = Doc] = couch_db_engine:open_local_docs(Db3, [<<"_local/foo">>]), + ?assertEqual({[{<<"yay">>, false}]}, Doc#doc.body). cet_write_mixed_batch() -> - {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), + {ok, Db1} = test_engine_util:create_db(), - ?assertEqual(0, Engine:get_doc_count(St1)), - ?assertEqual(0, Engine:get_del_doc_count(St1)), - ?assertEqual(0, Engine:get_update_seq(St1)), + ?assertEqual(0, couch_db_engine:get_doc_count(Db1)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db1)), + ?assertEqual(0, couch_db_engine:get_update_seq(Db1)), Actions = [ {batch, [ - {create, {<<"bar">>, []}}, - {create, {<<"_local/foo">>, [{<<"yay">>, false}]}} + {create, {<<"bar">>, {[]}}}, + {create, {<<"_local/foo">>, {[{<<"yay">>, false}]}}} ]} ], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), - {ok, St3} = Engine:commit_data(St2), - Engine:terminate(normal, St3), - {ok, St4} = Engine:init(DbPath, []), + {ok, Db2} = test_engine_util:apply_actions(Db1, Actions), + {ok, _} = couch_db:ensure_full_commit(Db2), + test_engine_util:shutdown_db(Db2), - ?assertEqual(1, Engine:get_doc_count(St4)), - ?assertEqual(0, Engine:get_del_doc_count(St4)), - ?assertEqual(1, Engine:get_update_seq(St4)), + {ok, Db3} = couch_db:reopen(Db2), - [#full_doc_info{}] = Engine:open_docs(St4, [<<"bar">>]), - [not_found] = Engine:open_docs(St4, [<<"_local/foo">>]), + ?assertEqual(1, couch_db_engine:get_doc_count(Db3)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db3)), + ?assertEqual(1, couch_db_engine:get_update_seq(Db3)), - [not_found] = Engine:open_local_docs(St4, [<<"bar">>]), - [#doc{}] = Engine:open_local_docs(St4, [<<"_local/foo">>]). + [#full_doc_info{}] = couch_db_engine:open_docs(Db3, [<<"bar">>]), + [not_found] = couch_db_engine:open_docs(Db3, [<<"_local/foo">>]), + + [not_found] = couch_db_engine:open_local_docs(Db3, [<<"bar">>]), + [#doc{}] = couch_db_engine:open_local_docs(Db3, [<<"_local/foo">>]). cet_update_local_doc() -> - {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), + {ok, Db1} = test_engine_util:create_db(), - ?assertEqual(0, Engine:get_doc_count(St1)), - ?assertEqual(0, Engine:get_del_doc_count(St1)), - ?assertEqual(0, Engine:get_update_seq(St1)), + ?assertEqual(0, couch_db_engine:get_doc_count(Db1)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db1)), + ?assertEqual(0, couch_db_engine:get_update_seq(Db1)), Actions = [ - {create, {<<"_local/foo">>, []}}, - {update, {<<"_local/foo">>, [{<<"stuff">>, null}]}} + {create, {<<"_local/foo">>, {[]}}}, + {update, {<<"_local/foo">>, {[{<<"stuff">>, null}]}}} ], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), - {ok, St3} = Engine:commit_data(St2), - Engine:terminate(normal, St3), - {ok, St4} = Engine:init(DbPath, []), + {ok, Db2} = test_engine_util:apply_actions(Db1, Actions), + {ok, _} = couch_db:ensure_full_commit(Db1), + test_engine_util:shutdown_db(Db2), + + {ok, Db3} = couch_db:reopen(Db2), - ?assertEqual(0, Engine:get_doc_count(St4)), - ?assertEqual(0, Engine:get_del_doc_count(St4)), - ?assertEqual(0, Engine:get_update_seq(St4)), + ?assertEqual(0, couch_db_engine:get_doc_count(Db3)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db3)), + ?assertEqual(0, couch_db_engine:get_update_seq(Db3)), - [not_found] = Engine:open_docs(St4, [<<"_local/foo">>]), - [#doc{} = Doc] = Engine:open_local_docs(St4, [<<"_local/foo">>]), - ?assertEqual([{<<"stuff">>, null}], Doc#doc.body). + [not_found] = couch_db_engine:open_docs(Db3, [<<"_local/foo">>]), + [#doc{} = Doc] = couch_db_engine:open_local_docs(Db3, [<<"_local/foo">>]), + ?assertEqual({[{<<"stuff">>, null}]}, Doc#doc.body). cet_delete_local_doc() -> - {ok, Engine, DbPath, St1} = test_engine_util:init_engine(dbpath), + {ok, Db1} = test_engine_util:create_db(), - ?assertEqual(0, Engine:get_doc_count(St1)), - ?assertEqual(0, Engine:get_del_doc_count(St1)), - ?assertEqual(0, Engine:get_update_seq(St1)), + ?assertEqual(0, couch_db_engine:get_doc_count(Db1)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db1)), + ?assertEqual(0, couch_db_engine:get_update_seq(Db1)), Actions = [ {create, {<<"_local/foo">>, []}}, {delete, {<<"_local/foo">>, []}} ], - {ok, St2} = test_engine_util:apply_actions(Engine, St1, Actions), - {ok, St3} = Engine:commit_data(St2), - Engine:terminate(normal, St3), - {ok, St4} = Engine:init(DbPath, []), + {ok, Db2} = test_engine_util:apply_actions(Db1, Actions), + {ok, _} = couch_db:ensure_full_commit(Db2), + test_engine_util:shutdown_db(Db2), - ?assertEqual(0, Engine:get_doc_count(St4)), - ?assertEqual(0, Engine:get_del_doc_count(St4)), - ?assertEqual(0, Engine:get_update_seq(St4)), + {ok, Db3} = couch_db:reopen(Db2), - [not_found] = Engine:open_docs(St4, [<<"_local/foo">>]), - ?assertEqual([not_found], Engine:open_local_docs(St4, [<<"_local/foo">>])). + ?assertEqual(0, couch_db_engine:get_doc_count(Db3)), + ?assertEqual(0, couch_db_engine:get_del_doc_count(Db3)), + ?assertEqual(0, couch_db_engine:get_update_seq(Db3)), + + [not_found] = couch_db_engine:open_docs(Db3, [<<"_local/foo">>]), + ?assertEqual( + [not_found], + couch_db_engine:open_local_docs(Db3, [<<"_local/foo">>]) + ). diff --git a/src/couch/src/test_engine_ref_counting.erl b/src/couch/src/test_engine_ref_counting.erl index 18e75fb5a..f0a2489f6 100644 --- a/src/couch/src/test_engine_ref_counting.erl +++ b/src/couch/src/test_engine_ref_counting.erl @@ -22,48 +22,53 @@ cet_empty_monitors() -> - {ok, Engine, St} = test_engine_util:init_engine(), - Pids = Engine:monitored_by(St), + {ok, Db} = test_engine_util:create_db(), + Pids = couch_db_engine:monitored_by(Db), ?assert(is_list(Pids)), - ?assertEqual([], Pids -- [self(), whereis(couch_stats_process_tracker)]). + Expected = [ + self(), + couch_db:get_pid(Db), + whereis(couch_stats_process_tracker) + ], + ?assertEqual([], Pids -- Expected). cet_incref_decref() -> - {ok, Engine, St} = test_engine_util:init_engine(), + {ok, Db} = test_engine_util:create_db(), - {Pid, _} = Client = start_client(Engine, St), + {Pid, _} = Client = start_client(Db), wait_client(Client), - Pids1 = Engine:monitored_by(St), + Pids1 = couch_db_engine:monitored_by(Db), ?assert(lists:member(Pid, Pids1)), close_client(Client), - Pids2 = Engine:monitored_by(St), + Pids2 = couch_db_engine:monitored_by(Db), ?assert(not lists:member(Pid, Pids2)). cet_incref_decref_many() -> - {ok, Engine, St} = test_engine_util:init_engine(), + {ok, Db} = test_engine_util:create_db(), Clients = lists:map(fun(_) -> - start_client(Engine, St) + start_client(Db) end, lists:seq(1, ?NUM_CLIENTS)), lists:foreach(fun(C) -> wait_client(C) end, Clients), - Pids1 = Engine:monitored_by(St), - % +2 for db pid and process tracker - ?assertEqual(?NUM_CLIENTS + 2, length(Pids1)), + Pids1 = couch_db_engine:monitored_by(Db), + % +3 for self, db pid, and process tracker + ?assertEqual(?NUM_CLIENTS + 3, length(Pids1)), lists:foreach(fun(C) -> close_client(C) end, Clients), - Pids2 = Engine:monitored_by(St), - ?assertEqual(2, length(Pids2)). + Pids2 = couch_db_engine:monitored_by(Db), + ?assertEqual(3, length(Pids2)). -start_client(Engine, St1) -> +start_client(Db0) -> spawn_monitor(fun() -> - {ok, St2} = Engine:incref(St1), + {ok, Db1} = couch_db:open_int(couch_db:name(Db0), []), receive {waiting, Pid} -> @@ -74,12 +79,11 @@ start_client(Engine, St1) -> receive close -> + couch_db:close(Db1), ok after 1000 -> erlang:error(timeout) - end, - - Engine:decref(St2) + end end). diff --git a/src/couch/src/test_engine_util.erl b/src/couch/src/test_engine_util.erl index 9efc6bcc1..b9c41c8d0 100644 --- a/src/couch/src/test_engine_util.erl +++ b/src/couch/src/test_engine_util.erl @@ -30,21 +30,27 @@ test_engine_ref_counting ]). + +-define(SHUTDOWN_TIMEOUT, 5000). -define(COMPACTOR_TIMEOUT, 50000). -define(ATTACHMENT_WRITE_TIMEOUT, 10000). -define(MAKE_DOC_SUMMARY_TIMEOUT, 5000). -create_tests(EngineApp) -> - create_tests(EngineApp, EngineApp). + +create_tests(EngineApp, Extension) -> + create_tests(EngineApp, EngineApp, Extension). -create_tests(EngineApp, EngineModule) -> - application:set_env(couch, test_engine, {EngineApp, EngineModule}), +create_tests(EngineApp, EngineModule, Extension) -> + TestEngine = {EngineApp, EngineModule, Extension}, + application:set_env(couch, test_engine, TestEngine), Tests = lists:map(fun(TestMod) -> {atom_to_list(TestMod), gather(TestMod)} end, ?TEST_MODULES), Setup = fun() -> Ctx = test_util:start_couch(), + EngineModStr = atom_to_list(EngineModule), + config:set("couchdb_engines", Extension, EngineModStr, false), config:set("log", "include_sasl", "false", false), Ctx end, @@ -78,102 +84,114 @@ make_test_fun(Module, Fun) -> end, {Name, Wrapper}. + rootdir() -> config:get("couchdb", "database_dir", "."). -dbpath() -> - binary_to_list(filename:join(rootdir(), - list_to_binary("a" ++ binary_to_list(couch_uuids:random()) ++ ".couch"))). +dbname() -> + UUID = couch_uuids:random(), + <<"db-", UUID/binary>>. get_engine() -> case application:get_env(couch, test_engine) of - {ok, {_, Engine}} -> - Engine; + {ok, {_App, _Mod, Extension}} -> + list_to_binary(Extension); _ -> - couch_bt_engine + <<"couch">> end. -init_engine() -> - init_engine(default). +create_db() -> + create_db(dbname()). -init_engine(default) -> +create_db(DbName) -> Engine = get_engine(), - DbPath = dbpath(), - {ok, St} = Engine:init(DbPath, [ - create, - {default_security_object, []} - ]), - {ok, Engine, St}; - -init_engine(dbpath) -> + couch_db:create(DbName, [{engine, Engine}, ?ADMIN_CTX]). + + +open_db(DbName) -> Engine = get_engine(), - DbPath = dbpath(), - {ok, St} = Engine:init(DbPath, [ - create, - {default_security_object, []} - ]), - {ok, Engine, DbPath, St}. + couch_db:open_int(DbName, [{engine, Engine}, ?ADMIN_CTX]). -apply_actions(_Engine, St, []) -> - {ok, St}; +shutdown_db(Db) -> + Pid = couch_db:get_pid(Db), + Ref = erlang:monitor(process, Pid), + exit(Pid, kill), + receive + {'DOWN', Ref, _, _, _} -> + ok + after ?SHUTDOWN_TIMEOUT -> + erlang:error(database_shutdown_timeout) + end, + test_util:wait(fun() -> + case ets:member(couch_dbs, couch_db:name(Db)) of + true -> wait; + false -> ok + end + end). -apply_actions(Engine, St, [Action | Rest]) -> - NewSt = apply_action(Engine, St, Action), - apply_actions(Engine, NewSt, Rest). +apply_actions(Db, []) -> + {ok, Db}; -apply_action(Engine, St, {batch, BatchActions}) -> - apply_batch(Engine, St, BatchActions); +apply_actions(Db, [Action | Rest]) -> + {ok, NewDb} = apply_action(Db, Action), + apply_actions(NewDb, Rest). -apply_action(Engine, St, Action) -> - apply_batch(Engine, St, [Action]). +apply_action(Db, {batch, BatchActions}) -> + apply_batch(Db, BatchActions); -apply_batch(Engine, St, [{purge, {Id, Revs}}]) -> - UpdateSeq = Engine:get_update_seq(St) + 1, - PurgeSeq = Engine:get_purge_seq(St) + 1, - case gen_write(Engine, St, {purge, {Id, Revs}}, UpdateSeq) of - {_, _, purged_before}-> - St; - {Pair, _, {Id, PRevs}} -> - UUID = couch_uuids:new(), - {ok, NewSt} = Engine:purge_docs( - St, [Pair], [{PurgeSeq, UUID, Id, PRevs}]), - NewSt - end; +apply_action(Db, Action) -> + apply_batch(Db, [Action]). -apply_batch(Engine, St, Actions) -> - UpdateSeq = Engine:get_update_seq(St) + 1, - AccIn = {UpdateSeq, [], []}, + +apply_batch(Db, Actions) -> + AccIn = {[], [], [], []}, AccOut = lists:foldl(fun(Action, Acc) -> - {SeqAcc, DocAcc, LDocAcc} = Acc, - case Action of - {_, {<<"_local/", _/binary>>, _}} -> - LDoc = gen_local_write(Engine, St, Action), - {SeqAcc, DocAcc, [LDoc | LDocAcc]}; - _ -> - {OldFDI, NewFDI} = gen_write(Engine, St, Action, SeqAcc), - {SeqAcc + 1, [{OldFDI, NewFDI} | DocAcc], LDocAcc} + {DocAcc, ConfAcc, LDocAcc, PurgeAcc} = Acc, + case gen_write(Db, Action) of + {update, Doc} -> + {[Doc | DocAcc], ConfAcc, LDocAcc, PurgeAcc}; + {conflict, Doc} -> + {DocAcc, [Doc | ConfAcc], LDocAcc, PurgeAcc}; + {local, Doc} -> + {DocAcc, ConfAcc, [Doc | LDocAcc], PurgeAcc}; + {purge, PurgeInfo} -> + {DocAcc, ConfAcc, LDocAcc, [PurgeInfo | PurgeAcc]} end end, AccIn, Actions), - {_, Docs0, LDocs} = AccOut, + + {Docs0, Conflicts0, LDocs0, PurgeInfos0} = AccOut, Docs = lists:reverse(Docs0), - {ok, NewSt} = Engine:write_doc_infos(St, Docs, LDocs), - NewSt. + Conflicts = lists:reverse(Conflicts0), + LDocs = lists:reverse(LDocs0), + PurgeInfos = lists:reverse(PurgeInfos0), + + {ok, Resp} = couch_db:update_docs(Db, Docs ++ LDocs), + false = lists:member(conflict, Resp), + {ok, Db1} = couch_db:reopen(Db), + {ok, []} = couch_db:update_docs(Db, Conflicts, [], replicated_changes), + {ok, Db2} = couch_db:reopen(Db1), -gen_local_write(Engine, St, {Action, {DocId, Body}}) -> - PrevRev = case Engine:open_local_docs(St, [DocId]) of - [not_found] -> + if PurgeInfos == [] -> ok; true -> + {ok, _} = couch_db:purge_docs(Db2, PurgeInfos) + end, + couch_db:reopen(Db2). + + +gen_write(Db, {Action, {<<"_local/", _/binary>> = DocId, Body}}) -> + PrevRev = case couch_db:open_doc(Db, DocId) of + {not_found, _} -> 0; - [#doc{revs = {0, []}}] -> + {ok, #doc{revs = {0, []}}} -> 0; - [#doc{revs = {0, [RevStr | _]}}] -> + {ok, #doc{revs = {0, [RevStr | _]}}} -> binary_to_integer(RevStr) end, {RevId, Deleted} = case Action of @@ -182,149 +200,42 @@ gen_local_write(Engine, St, {Action, {DocId, Body}}) -> delete -> {0, true} end, - #doc{ + {local, #doc{ id = DocId, - revs = {0, [RevId]}, + revs = {0, [list_to_binary(integer_to_list(RevId))]}, body = Body, deleted = Deleted - }. - -gen_write(Engine, St, {Action, {DocId, Body}}, UpdateSeq) -> - gen_write(Engine, St, {Action, {DocId, Body, []}}, UpdateSeq); - -gen_write(Engine, St, {create, {DocId, Body, Atts0}}, UpdateSeq) -> - [not_found] = Engine:open_docs(St, [DocId]), - Atts = [couch_att:to_disk_term(Att) || Att <- Atts0], + }}; - Rev = crypto:hash(md5, term_to_binary({DocId, Body, Atts})), +gen_write(Db, {Action, {DocId, Body}}) -> + gen_write(Db, {Action, {DocId, Body, []}}); - Doc0 = #doc{ +gen_write(Db, {create, {DocId, Body, Atts}}) -> + {not_found, _} = couch_db:open_doc(Db, DocId), + {update, #doc{ id = DocId, - revs = {0, [Rev]}, + revs = {0, []}, deleted = false, body = Body, atts = Atts - }, - - Doc1 = make_doc_summary(Engine, St, Doc0), - {ok, Doc2, Len} = Engine:write_doc_body(St, Doc1), - - Sizes = #size_info{ - active = Len, - external = erlang:external_size(Doc1#doc.body) - }, - - Leaf = #leaf{ - deleted = false, - ptr = Doc2#doc.body, - seq = UpdateSeq, - sizes = Sizes, - atts = Atts - }, - - {not_found, #full_doc_info{ - id = DocId, - deleted = false, - update_seq = UpdateSeq, - rev_tree = [{0, {Rev, Leaf, []}}], - sizes = Sizes }}; -gen_write(Engine, St, {purge, {DocId, PrevRevs0, _}}, UpdateSeq) -> - case Engine:open_docs(St, [DocId]) of - [not_found] -> - % Check if this doc has been purged before - FoldFun = fun({_PSeq, _UUID, Id, _Revs}, _Acc) -> - case Id of - DocId -> {stop, true}; - _ -> {ok, false} - end - end, - {ok, IsPurgedBefore} = Engine:fold_purge_infos( - St, 0, FoldFun, false, []), - case IsPurgedBefore of - true -> {{}, UpdateSeq, purged_before}; - false -> erlang:error({invalid_purge_test_id, DocId}) - end; - [#full_doc_info{} = PrevFDI] -> - PrevRevs = if is_list(PrevRevs0) -> PrevRevs0; true -> [PrevRevs0] end, - - #full_doc_info{ - rev_tree = PrevTree - } = PrevFDI, - - {NewTree, RemRevs0} = couch_key_tree:remove_leafs(PrevTree, PrevRevs), - {RemRevs, NotRemRevs} = lists:partition(fun(R) -> - lists:member(R, RemRevs0) end, PrevRevs), - - if NotRemRevs == [] -> ok; true -> - % Check if these Revs have been purged before - FoldFun = fun({_Pseq, _UUID, Id, Revs}, Acc) -> - case Id of - DocId -> {ok, Acc ++ Revs}; - _ -> {ok, Acc} - end - end, - {ok, PurgedRevs} = Engine:fold_purge_infos(St, 0, FoldFun, [], []), - case lists:subtract(PrevRevs, PurgedRevs) of [] -> ok; _ -> - % If we didn't purge all the requested revisions - % and they haven't been purged before - % then its a bug in the test. - erlang:error({invalid_purge_test_revs, PrevRevs}) - end - end, - - case {RemRevs, NewTree} of - {[], _} -> - {{PrevFDI, PrevFDI}, UpdateSeq, purged_before}; - {_, []} -> - % We've completely purged the document - {{PrevFDI, not_found}, UpdateSeq, {DocId, RemRevs}}; - _ -> - % We have to relabel the update_seq of all - % leaves. See couch_db_updater for details. - {NewNewTree, NewUpdateSeq} = couch_key_tree:mapfold(fun - (_RevId, Leaf, leaf, InnerSeqAcc) -> - {Leaf#leaf{seq = InnerSeqAcc}, InnerSeqAcc + 1}; - (_RevId, Value, _Type, InnerSeqAcc) -> - {Value, InnerSeqAcc} - end, UpdateSeq, NewTree), - NewFDI = PrevFDI#full_doc_info{ - update_seq = NewUpdateSeq - 1, - rev_tree = NewNewTree - }, - {{PrevFDI, NewFDI}, NewUpdateSeq, {DocId, RemRevs}} - - end - end; +gen_write(_Db, {purge, {DocId, PrevRevs0, _}}) -> + PrevRevs = if is_list(PrevRevs0) -> PrevRevs0; true -> [PrevRevs0] end, + {purge, {couch_uuids:random(), DocId, PrevRevs}}; -gen_write(Engine, St, {Action, {DocId, Body, Atts0}}, UpdateSeq) -> - [#full_doc_info{} = PrevFDI] = Engine:open_docs(St, [DocId]), - Atts = [couch_att:to_disk_term(Att) || Att <- Atts0], +gen_write(Db, {Action, {DocId, Body, Atts}}) -> + #full_doc_info{} = PrevFDI = couch_db:get_full_doc_info(Db, DocId), #full_doc_info{ - id = DocId, - rev_tree = PrevRevTree + id = DocId } = PrevFDI, #rev_info{ rev = PrevRev } = prev_rev(PrevFDI), - {RevPos, PrevRevId} = PrevRev, - - Rev = gen_revision(Action, DocId, PrevRev, Body, Atts), - - Doc0 = #doc{ - id = DocId, - revs = {RevPos + 1, [Rev, PrevRevId]}, - deleted = false, - body = Body, - atts = Atts - }, - - Doc1 = make_doc_summary(Engine, St, Doc0), - {ok, Doc2, Len} = Engine:write_doc_body(St, Doc1), + NewRev = gen_rev(Action, DocId, PrevRev, Body, Atts), Deleted = case Action of update -> false; @@ -332,73 +243,35 @@ gen_write(Engine, St, {Action, {DocId, Body, Atts0}}, UpdateSeq) -> delete -> true end, - Sizes = #size_info{ - active = Len, - external = erlang:external_size(Doc1#doc.body) - }, + Type = case Action of + conflict -> conflict; + _ -> update + end, - Leaf = #leaf{ + {Type, #doc{ + id = DocId, + revs = NewRev, deleted = Deleted, - ptr = Doc2#doc.body, - seq = UpdateSeq, - sizes = Sizes, + body = Body, atts = Atts - }, - - Path = gen_path(Action, RevPos, PrevRevId, Rev, Leaf), - RevsLimit = Engine:get_revs_limit(St), - NodeType = case Action of - conflict -> new_branch; - _ -> new_leaf - end, - {NewTree, NodeType} = couch_key_tree:merge(PrevRevTree, Path, RevsLimit), - - NewFDI = PrevFDI#full_doc_info{ - deleted = couch_doc:is_deleted(NewTree), - update_seq = UpdateSeq, - rev_tree = NewTree, - sizes = Sizes - }, + }}. - {PrevFDI, NewFDI}. +gen_rev(A, DocId, {Pos, Rev}, Body, Atts) when A == update; A == delete -> + NewRev = crypto:hash(md5, term_to_binary({DocId, Rev, Body, Atts})), + {Pos + 1, [NewRev, Rev]}; +gen_rev(conflict, DocId, _, Body, Atts) -> + UUID = couch_uuids:random(), + NewRev = crypto:hash(md5, term_to_binary({DocId, UUID, Body, Atts})), + {1, [NewRev]}. -gen_revision(conflict, DocId, _PrevRev, Body, Atts) -> - crypto:hash(md5, term_to_binary({DocId, Body, Atts})); -gen_revision(delete, DocId, PrevRev, Body, Atts) -> - gen_revision(update, DocId, PrevRev, Body, Atts); -gen_revision(update, DocId, PrevRev, Body, Atts) -> - crypto:hash(md5, term_to_binary({DocId, PrevRev, Body, Atts})). - -gen_path(conflict, _RevPos, _PrevRevId, Rev, Leaf) -> - {0, {Rev, Leaf, []}}; -gen_path(delete, RevPos, PrevRevId, Rev, Leaf) -> - gen_path(update, RevPos, PrevRevId, Rev, Leaf); -gen_path(update, RevPos, PrevRevId, Rev, Leaf) -> - {RevPos, {PrevRevId, ?REV_MISSING, [{Rev, Leaf, []}]}}. - - -make_doc_summary(Engine, St, DocData) -> - {_, Ref} = spawn_monitor(fun() -> - exit({result, Engine:serialize_doc(St, DocData)}) - end), - receive - {'DOWN', Ref, _, _, {result, Summary}} -> - Summary; - {'DOWN', Ref, _, _, Error} -> - erlang:error({make_doc_summary_error, Error}) - after ?MAKE_DOC_SUMMARY_TIMEOUT -> - erlang:error(make_doc_summary_timeout) - end. - - -prep_atts(_Engine, _St, []) -> +prep_atts(_Db, []) -> []; -prep_atts(Engine, St, [{FileName, Data} | Rest]) -> +prep_atts(Db, [{FileName, Data} | Rest]) -> {_, Ref} = spawn_monitor(fun() -> - {ok, Stream} = Engine:open_write_stream(St, []), + {ok, Stream} = couch_db:open_write_stream(Db, []), exit(write_att(Stream, FileName, Data, Data)) end), Att = receive @@ -409,7 +282,7 @@ prep_atts(Engine, St, [{FileName, Data} | Rest]) -> after ?ATTACHMENT_WRITE_TIMEOUT -> erlang:error(attachment_write_timeout) end, - [Att | prep_atts(Engine, St, Rest)]. + [Att | prep_atts(Db, Rest)]. write_att(Stream, FileName, OrigData, <<>>) -> @@ -445,17 +318,17 @@ prev_rev(#full_doc_info{} = FDI) -> PrevRev. -db_as_term(Engine, St) -> +db_as_term(Db) -> [ - {props, db_props_as_term(Engine, St)}, - {docs, db_docs_as_term(Engine, St)}, - {local_docs, db_local_docs_as_term(Engine, St)}, - {changes, db_changes_as_term(Engine, St)}, - {purged_docs, db_purged_docs_as_term(Engine, St)} + {props, db_props_as_term(Db)}, + {docs, db_docs_as_term(Db)}, + {local_docs, db_local_docs_as_term(Db)}, + {changes, db_changes_as_term(Db)}, + {purged_docs, db_purged_docs_as_term(Db)} ]. -db_props_as_term(Engine, St) -> +db_props_as_term(Db) -> Props = [ get_doc_count, get_del_doc_count, @@ -469,49 +342,50 @@ db_props_as_term(Engine, St) -> get_epochs ], lists:map(fun(Fun) -> - {Fun, Engine:Fun(St)} + {Fun, couch_db_engine:Fun(Db)} end, Props). -db_docs_as_term(Engine, St) -> +db_docs_as_term(Db) -> FoldFun = fun(FDI, Acc) -> {ok, [FDI | Acc]} end, - {ok, FDIs} = Engine:fold_docs(St, FoldFun, [], []), + {ok, FDIs} = couch_db:fold_docs(Db, FoldFun, [], []), lists:reverse(lists:map(fun(FDI) -> - fdi_to_term(Engine, St, FDI) + fdi_to_term(Db, FDI) end, FDIs)). -db_local_docs_as_term(Engine, St) -> +db_local_docs_as_term(Db) -> FoldFun = fun(Doc, Acc) -> {ok, [Doc | Acc]} end, - {ok, LDocs} = Engine:fold_local_docs(St, FoldFun, [], []), + {ok, LDocs} = couch_db:fold_local_docs(Db, FoldFun, [], []), lists:reverse(LDocs). -db_changes_as_term(Engine, St) -> +db_changes_as_term(Db) -> FoldFun = fun(FDI, Acc) -> {ok, [FDI | Acc]} end, - {ok, Changes} = Engine:fold_changes(St, 0, FoldFun, [], []), + {ok, Changes} = couch_db:fold_changes(Db, 0, FoldFun, [], []), lists:reverse(lists:map(fun(FDI) -> - fdi_to_term(Engine, St, FDI) + fdi_to_term(Db, FDI) end, Changes)). -db_purged_docs_as_term(Engine, St) -> - StartPSeq = Engine:get_oldest_purge_seq(St) - 1, +db_purged_docs_as_term(Db) -> + InitPSeq = couch_db_engine:get_oldest_purge_seq(Db) - 1, FoldFun = fun({PSeq, UUID, Id, Revs}, Acc) -> {ok, [{PSeq, UUID, Id, Revs} | Acc]} end, - {ok, PDocs} = Engine:fold_purge_infos(St, StartPSeq, FoldFun, [], []), + {ok, PDocs} = couch_db_engine:fold_purge_infos( + Db, InitPSeq, FoldFun, [], []), lists:reverse(PDocs). -fdi_to_term(Engine, St, FDI) -> +fdi_to_term(Db, FDI) -> #full_doc_info{ id = DocId, rev_tree = OldTree } = FDI, {NewRevTree, _} = couch_key_tree:mapfold(fun(Rev, Node, Type, Acc) -> tree_to_term(Rev, Node, Type, Acc, DocId) - end, {Engine, St}, OldTree), + end, Db, OldTree), FDI#full_doc_info{ rev_tree = NewRevTree, % Blank out sizes because we allow storage @@ -527,7 +401,7 @@ fdi_to_term(Engine, St, FDI) -> tree_to_term(_Rev, _Leaf, branch, Acc, _DocId) -> {?REV_MISSING, Acc}; -tree_to_term({Pos, RevId}, #leaf{} = Leaf, leaf, {Engine, St}, DocId) -> +tree_to_term({Pos, RevId}, #leaf{} = Leaf, leaf, Db, DocId) -> #leaf{ deleted = Deleted, ptr = Ptr @@ -540,7 +414,7 @@ tree_to_term({Pos, RevId}, #leaf{} = Leaf, leaf, {Engine, St}, DocId) -> body = Ptr }, - Doc1 = Engine:read_doc_body(St, Doc0), + Doc1 = couch_db_engine:read_doc_body(Db, Doc0), Body = if not is_binary(Doc1#doc.body) -> Doc1#doc.body; true -> couch_compress:decompress(Doc1#doc.body) @@ -550,7 +424,7 @@ tree_to_term({Pos, RevId}, #leaf{} = Leaf, leaf, {Engine, St}, DocId) -> couch_compress:decompress(Doc1#doc.atts) end, - StreamSrc = fun(Sp) -> Engine:open_read_stream(St, Sp) end, + StreamSrc = fun(Sp) -> couch_db:open_read_stream(Db, Sp) end, Atts2 = [couch_att:from_disk_term(StreamSrc, Att) || Att <- Atts1], Atts = [att_to_term(Att) || Att <- Atts2], @@ -559,7 +433,7 @@ tree_to_term({Pos, RevId}, #leaf{} = Leaf, leaf, {Engine, St}, DocId) -> sizes = #size_info{active = -1, external = -1}, atts = Atts }, - {NewLeaf, {Engine, St}}. + {NewLeaf, Db}. att_to_term(Att) -> @@ -616,10 +490,8 @@ list_diff([T1 | R1], [T2 | R2]) -> end. -compact(Engine, St1, DbPath) -> - DbName1 = filename:basename(DbPath), - DbName2 = filename:rootname(DbName1), - {ok, St2, Pid} = Engine:start_compaction(St1, ?l2b(DbName2), [], self()), +compact(Db) -> + {ok, Pid} = couch_db:start_compact(Db), Ref = erlang:monitor(process, Pid), % Ideally I'd assert that Pid is linked to us @@ -627,16 +499,29 @@ compact(Engine, St1, DbPath) -> % that it could have finished compacting by % the time we check... Quite the quandry. - Term = receive - {'$gen_cast', {compact_done, Engine, Term0}} -> - Term0; + receive + {'DOWN', Ref, _, _, normal} -> + ok; + {'DOWN', Ref, _, _, noproc} -> + ok; {'DOWN', Ref, _, _, Reason} -> erlang:error({compactor_died, Reason}) after ?COMPACTOR_TIMEOUT -> erlang:error(compactor_timed_out) end, - {ok, St2, DbName2, Pid, Term}. + test_util:wait(fun() -> + {ok, Db2} = couch_db:open_int(couch_db:name(Db), []), + try + CPid = couch_db:get_compactor_pid(Db2), + case is_pid(CPid) of + true -> wait; + false -> ok + end + after + couch_db:close(Db2) + end + end). with_config(Config, Fun) -> @@ -654,7 +539,7 @@ apply_config([]) -> apply_config([{Section, Key, Value} | Rest]) -> Orig = config:get(Section, Key), case Value of - undefined -> config:delete(Section, Key); - _ -> config:set(Section, Key, Value) + undefined -> config:delete(Section, Key, false); + _ -> config:set(Section, Key, Value, false) end, [{Section, Key, Orig} | apply_config(Rest)]. diff --git a/src/couch/test/couch_bt_engine_tests.erl b/src/couch/test/couch_bt_engine_tests.erl index df200df47..c4055d0f0 100644 --- a/src/couch/test/couch_bt_engine_tests.erl +++ b/src/couch/test/couch_bt_engine_tests.erl @@ -17,4 +17,4 @@ couch_bt_engine_test_()-> - test_engine_util:create_tests(couch, couch_bt_engine). + test_engine_util:create_tests(couch, couch_bt_engine, "couch"). diff --git a/src/couch/test/couch_db_purge_docs_tests.erl b/src/couch/test/couch_db_purge_docs_tests.erl index 7b56c84f0..1ac42a64c 100644 --- a/src/couch/test/couch_db_purge_docs_tests.erl +++ b/src/couch/test/couch_db_purge_docs_tests.erl @@ -16,14 +16,8 @@ -include_lib("couch/include/couch_db.hrl"). -setup() -> - DbName = ?tempdb(), - {ok, _Db} = create_db(DbName), - DbName. +-define(REV_DEPTH, 100). -teardown(DbName) -> - delete_db(DbName), - ok. couch_db_purge_docs_test_() -> { @@ -31,566 +25,722 @@ couch_db_purge_docs_test_() -> [ { setup, - fun test_util:start_couch/0, fun test_util:stop_couch/1, - [couch_db_purge_docs()] + fun test_util:start_couch/0, + fun test_util:stop_couch/1, + [ + couch_db_purge_docs() + ] }, - purge_with_replication() + { + setup, + fun start_with_replicator/0, + fun test_util:stop_couch/1, + [ + couch_db_purge_with_replication() + ] + } ] - }. couch_db_purge_docs() -> { - foreach, - fun setup/0, fun teardown/1, - [ - fun test_purge_2_to_purge_3/1, - fun test_purge_all/1, - fun test_purge_some/1, - fun test_purge_none/1, - fun test_purge_missing_docid/1, - fun test_purge_repeated_docid/1, - %fun test_purge_repeated_rev/1, % improving - fun test_purge_partial/1, - fun test_all_removal_purges/1, - fun test_purge_invalid_rev/1, - fun test_purge_duplicate_UUID/1, - fun purge_id_not_exist/1, - fun purge_non_leaf_rev/1, - fun purge_deep_tree/1 - ] + foreach, + fun setup/0, + fun teardown/1, + [ + fun test_purge_2_to_purge_3/1, + fun test_purge_all/1, + fun test_purge_some/1, + fun test_purge_none/1, + fun test_purge_missing_docid/1, + fun test_purge_repeated_docid/1, + fun test_purge_repeated_rev/1, + fun test_purge_partial/1, + fun test_all_removal_purges/1, + fun test_purge_invalid_rev/1, + fun test_purge_duplicate_UUID/1, + fun test_purge_id_not_exist/1, + fun test_purge_non_leaf_rev/1, + fun test_purge_deep_tree/1 + ] + }. + + +couch_db_purge_with_replication() -> + { + foreach, + fun setup_replicator/0, + fun teardown_replicator/1, + [ + fun test_purge_with_replication/1 + ] }. +start_with_replicator() -> + test_util:start_couch([couch_replicator]). + + +setup() -> + DbName = ?tempdb(), + {ok, _Db} = create_db(DbName), + DbName. + + +teardown(DbName) -> + delete_db(DbName), + ok. + + +setup_replicator() -> + Source = ?tempdb(), + Target = ?tempdb(), + {ok, _} = create_db(Source), + {ok, _} = create_db(Target), + {Source, Target}. + + +teardown_replicator({Source, Target}) -> + delete_db(Source), + delete_db(Target), + ok. + + test_purge_2_to_purge_3(DbName) -> - ?_test( - begin - {ok, Db} = couch_db:open_int(DbName, []), - Doc1 = {[{<<"_id">>, <<"foo1">>}, {<<"vsn">>, 1.1}]}, - {ok, Rev} = save_doc(Db, Doc1), - couch_db:ensure_full_commit(Db), - {ok, Db2} = couch_db:reopen(Db), - UUID = couch_uuids:new(), - {ok, [{ok, PRevs}]} = couch_db:purge_docs( - Db2, [{UUID, <<"foo1">>, [Rev]}] - ), - ?assertEqual([Rev], PRevs), - {ok, Db3} = couch_db:reopen(Db2), - {ok, _PIdsRevs} = couch_db:fold_purge_infos( - Db3, 0, fun fold_fun/2, [], []), - ?assertEqual(0, couch_db_engine:get_doc_count(Db3)), - ?assertEqual(1, couch_db_engine:get_purge_seq(Db3)) - end). + ?_test(begin + {ok, Rev} = save_doc(DbName, {[{'_id', foo1}, {vsn, 1.1}]}), + + PurgeInfos = [ + {uuid(), <<"foo1">>, [Rev]} + ], + + {ok, [{ok, PRevs}]} = purge(DbName, PurgeInfos), + ?assertEqual([Rev], PRevs), + + assertProps(DbName, [ + {doc_count, 0}, + {del_doc_count, 0}, + {update_seq, 2}, + {changes, 0}, + {purge_seq, 1}, + {purge_infos, PurgeInfos} + ]) + end). test_purge_all(DbName) -> - ?_test( - begin - {ok, Db} = couch_db:open_int(DbName, []), - Doc1 = {[{<<"_id">>, <<"foo1">>}, {<<"vsn">>, 1.1}]}, - Doc2 = {[{<<"_id">>, <<"foo2">>}, {<<"vsn">>, 1.2}]}, - {ok, Rev} = save_doc(Db, Doc1), - {ok, Rev2} = save_doc(Db, Doc2), - couch_db:ensure_full_commit(Db), - - {ok, Db2} = couch_db:reopen(Db), - ?assertEqual(2, couch_db_engine:get_doc_count(Db2)), - ?assertEqual(0, couch_db_engine:get_del_doc_count(Db2)), - ?assertEqual(2, couch_db_engine:get_update_seq(Db2)), - ?assertEqual(0, couch_db_engine:get_purge_seq(Db2)), - - UUID = couch_uuids:new(), UUID2 = couch_uuids:new(), - {ok, [{ok, PRevs}, {ok, PRevs2}]} = couch_db:purge_docs( - Db2, [{UUID, <<"foo1">>, [Rev]}, {UUID2, <<"foo2">>, [Rev2]}] - ), - - ?assertEqual([Rev], PRevs), - ?assertEqual([Rev2], PRevs2), - - {ok, Db3} = couch_db:reopen(Db2), - {ok, PIdsRevs} = couch_db:fold_purge_infos( - Db3, 0, fun fold_fun/2, [], []), - ?assertEqual(0, couch_db_engine:get_doc_count(Db3)), - ?assertEqual(0, couch_db_engine:get_del_doc_count(Db3)), - ?assertEqual(3, couch_db_engine:get_update_seq(Db3)), - ?assertEqual(2, couch_db_engine:get_purge_seq(Db3)), - ?assertEqual([{<<"foo2">>, [Rev2]}, {<<"foo1">>, [Rev]}], PIdsRevs) - end). + ?_test(begin + {ok, [Rev1, Rev2]} = save_docs(DbName, [ + {[{'_id', foo1}, {vsn, 1.1}]}, + {[{'_id', foo2}, {vsn, 1.2}]} + ]), + + assertProps(DbName, [ + {doc_count, 2}, + {del_doc_count, 0}, + {update_seq, 2}, + {changes, 2}, + {purge_seq, 0}, + {purge_infos, []} + ]), + + PurgeInfos = [ + {uuid(), <<"foo1">>, [Rev1]}, + {uuid(), <<"foo2">>, [Rev2]} + ], + + {ok, [{ok, PRevs1}, {ok, PRevs2}]} = purge(DbName, PurgeInfos), + + ?assertEqual([Rev1], PRevs1), + ?assertEqual([Rev2], PRevs2), + + assertProps(DbName, [ + {doc_count, 0}, + {del_doc_count, 0}, + {update_seq, 3}, + {changes, 0}, + {purge_seq, 2}, + {purge_infos, PurgeInfos} + ]) + end). test_all_removal_purges(DbName) -> - ?_test( - begin - {ok, Db0} = couch_db:open_int(DbName, []), - Doc0 = {[{<<"_id">>,<<"foo">>}, {<<"vsn">>, 1}]}, - {ok, Rev} = save_doc(Db0, Doc0), - couch_db:ensure_full_commit(Db0), - {ok, Db1} = couch_db:reopen(Db0), - - Doc1 = {[ - {<<"_id">>, <<"foo">>}, {<<"vsn">>, 2}, - {<<"_rev">>, couch_doc:rev_to_str(Rev)}, - {<<"_deleted">>, true}] - }, - {ok, Rev2} = save_doc(Db1, Doc1), - couch_db:ensure_full_commit(Db1), - - {ok, Db2} = couch_db:reopen(Db1), - {ok, PIdsRevs1} = couch_db:fold_purge_infos( - Db2, 0, fun fold_fun/2, [], []), - ?assertEqual(0, couch_db_engine:get_doc_count(Db2)), - ?assertEqual(1, couch_db_engine:get_del_doc_count(Db2)), - ?assertEqual(2, couch_db_engine:get_update_seq(Db2)), - ?assertEqual(0, couch_db_engine:get_purge_seq(Db2)), - ?assertEqual([], PIdsRevs1), - - UUID = couch_uuids:new(), - {ok, [{ok, PRevs}]} = couch_db:purge_docs( - Db2, [{UUID, <<"foo">>, [Rev2]}]), - ?assertEqual([Rev2], PRevs), - - {ok, Db3} = couch_db:reopen(Db2), - {ok, PIdsRevs2} = couch_db:fold_purge_infos( - Db3, 0, fun fold_fun/2, [], []), - ?assertEqual(0, couch_db_engine:get_doc_count(Db3)), - ?assertEqual(0, couch_db_engine:get_del_doc_count(Db3)), - ?assertEqual(3, couch_db_engine:get_update_seq(Db3)), - ?assertEqual(1, couch_db_engine:get_purge_seq(Db3)), - ?assertEqual([{<<"foo">>, [Rev2]}], PIdsRevs2) - end). + ?_test(begin + {ok, Rev1} = save_doc(DbName, {[{'_id', foo}, {vsn, 1}]}), + Update = {[ + {<<"_id">>, <<"foo">>}, + {<<"_rev">>, couch_doc:rev_to_str(Rev1)}, + {<<"_deleted">>, true}, + {<<"vsn">>, 2} + ]}, + {ok, Rev2} = save_doc(DbName, Update), + + assertProps(DbName, [ + {doc_count, 0}, + {del_doc_count, 1}, + {update_seq, 2}, + {changes, 1}, + {purge_seq, 0}, + {purge_infos, []} + ]), + + PurgeInfos = [ + {uuid(), <<"foo">>, [Rev2]} + ], + + {ok, [{ok, PRevs}]} = purge(DbName, PurgeInfos), + + ?assertEqual([Rev2], PRevs), + + assertProps(DbName, [ + {doc_count, 0}, + {del_doc_count, 0}, + {update_seq, 3}, + {changes, 0}, + {purge_seq, 1}, + {purge_infos, PurgeInfos} + ]) + end). test_purge_some(DbName) -> - ?_test( - begin - {ok, Db} = couch_db:open_int(DbName, []), - Doc1 = {[{<<"_id">>, <<"foo1">>}, {<<"vsn">>, 1}]}, - Doc2 = {[{<<"_id">>, <<"foo2">>}, {<<"vsn">>, 2}]}, - {ok, Rev} = save_doc(Db, Doc1), - {ok, _Rev2} = save_doc(Db, Doc2), - couch_db:ensure_full_commit(Db), - - {ok, Db2} = couch_db:reopen(Db), - ?assertEqual(2, couch_db_engine:get_doc_count(Db2)), - ?assertEqual(0, couch_db_engine:get_del_doc_count(Db2)), - ?assertEqual(2, couch_db_engine:get_update_seq(Db2)), - ?assertEqual(0, couch_db_engine:get_purge_seq(Db2)), - - UUID = couch_uuids:new(), - {ok, [{ok, PRevs}]} = couch_db:purge_docs(Db2, - [{UUID, <<"foo1">>, [Rev]}]), - ?assertEqual([Rev], PRevs), - - {ok, Db3} = couch_db:reopen(Db2), - {ok, PIdsRevs} = couch_db:fold_purge_infos( - Db3, 0, fun fold_fun/2, [], []), - ?assertEqual(1, couch_db_engine:get_doc_count(Db3)), - ?assertEqual(0, couch_db_engine:get_del_doc_count(Db3)), - ?assertEqual(3, couch_db_engine:get_update_seq(Db3)), - ?assertEqual(1, couch_db_engine:get_purge_seq(Db3)), - ?assertEqual([{<<"foo1">>, [Rev]}], PIdsRevs) - end). + ?_test(begin + {ok, [Rev1, _Rev2]} = save_docs(DbName, [ + {[{'_id', foo1}, {vsn, 1}]}, + {[{'_id', foo2}, {vsn, 2}]} + ]), + + assertProps(DbName, [ + {doc_count, 2}, + {del_doc_count, 0}, + {update_seq, 2}, + {changes, 2}, + {purge_seq, 0}, + {purge_infos, []} + ]), + + PurgeInfos = [ + {uuid(), <<"foo1">>, [Rev1]} + ], + {ok, [{ok, PRevs}]} = purge(DbName, PurgeInfos), + + ?assertEqual([Rev1], PRevs), + + assertProps(DbName, [ + {doc_count, 1}, + {del_doc_count, 0}, + {update_seq, 3}, + {changes, 1}, + {purge_seq, 1}, + {purge_infos, PurgeInfos} + ]) + end). test_purge_none(DbName) -> - ?_test( - begin - {ok, Db} = couch_db:open_int(DbName, []), - Doc1 = {[{<<"_id">>, <<"foo1">>}, {<<"vsn">>, 1}]}, - Doc2 = {[{<<"_id">>, <<"foo2">>}, {<<"vsn">>, 2}]}, - {ok, _Rev} = save_doc(Db, Doc1), - {ok, _Rev2} = save_doc(Db, Doc2), - couch_db:ensure_full_commit(Db), - - {ok, Db2} = couch_db:reopen(Db), - ?assertEqual(2, couch_db_engine:get_doc_count(Db2)), - ?assertEqual(0, couch_db_engine:get_del_doc_count(Db2)), - ?assertEqual(2, couch_db_engine:get_update_seq(Db2)), - ?assertEqual(0, couch_db_engine:get_purge_seq(Db2)), - - {ok, []} = couch_db:purge_docs(Db2, []), - - {ok, Db3} = couch_db:reopen(Db2), - {ok, PIdsRevs} = couch_db:fold_purge_infos( - Db3, 0, fun fold_fun/2, [], []), - ?assertEqual(2, couch_db_engine:get_doc_count(Db3)), - ?assertEqual(0, couch_db_engine:get_del_doc_count(Db3)), - ?assertEqual(2, couch_db_engine:get_update_seq(Db3)), - ?assertEqual(0, couch_db_engine:get_purge_seq(Db3)), - ?assertEqual([], PIdsRevs) - end). + ?_test(begin + {ok, [_Rev1, _Rev2]} = save_docs(DbName, [ + {[{'_id', foo1}, {vsn, 1}]}, + {[{'_id', foo2}, {vsn, 2}]} + ]), + + assertProps(DbName, [ + {doc_count, 2}, + {del_doc_count, 0}, + {update_seq, 2}, + {changes, 2}, + {purge_seq, 0}, + {purge_infos, []} + ]), + + {ok, []} = purge(DbName, []), + + assertProps(DbName, [ + {doc_count, 2}, + {del_doc_count, 0}, + {update_seq, 2}, + {changes, 2}, + {purge_seq, 0}, + {purge_infos, []} + ]) + end). test_purge_missing_docid(DbName) -> - ?_test( - begin - {ok, Db} = couch_db:open_int(DbName, []), - Doc1 = {[{<<"_id">>, <<"foo1">>}, {<<"vsn">>, 1}]}, - Doc2 = {[{<<"_id">>, <<"foo2">>}, {<<"vsn">>, 2}]}, - {ok, Rev} = save_doc(Db, Doc1), - {ok, _Rev2} = save_doc(Db, Doc2), - couch_db:ensure_full_commit(Db), - - {ok, Db2} = couch_db:reopen(Db), - ?assertEqual(2, couch_db_engine:get_doc_count(Db2)), - ?assertEqual(0, couch_db_engine:get_del_doc_count(Db2)), - ?assertEqual(2, couch_db_engine:get_update_seq(Db2)), - ?assertEqual(0, couch_db_engine:get_purge_seq(Db2)), - - UUID = couch_uuids:new(), - {ok, [{ok, PRevs}]} = couch_db:purge_docs(Db2, - [{UUID, <<"">>, [Rev]}]), - ?assertEqual([], PRevs), - - {ok, Db3} = couch_db:reopen(Db2), - {ok, _PIdsRevs} = couch_db:fold_purge_infos( - Db3, 0, fun fold_fun/2, [], []), - ?assertEqual(2, couch_db_engine:get_doc_count(Db3)), - ?assertEqual(0, couch_db_engine:get_del_doc_count(Db3)), - ?assertEqual(2, couch_db_engine:get_update_seq(Db3)), - ?assertEqual(0, couch_db_engine:get_purge_seq(Db3)) - end). + ?_test(begin + {ok, [Rev1, _Rev2]} = save_docs(DbName, [ + {[{'_id', foo1}, {vsn, 1}]}, + {[{'_id', foo2}, {vsn, 2}]} + ]), + + assertProps(DbName, [ + {doc_count, 2}, + {del_doc_count, 0}, + {update_seq, 2}, + {changes, 2}, + {purge_seq, 0}, + {purge_infos, []} + ]), + + PurgeInfos = [ + {uuid(), <<"baz">>, [Rev1]} + ], + + {ok, [{ok, []}]} = purge(DbName, PurgeInfos), + + assertProps(DbName, [ + {doc_count, 2}, + {del_doc_count, 0}, + {update_seq, 3}, + {changes, 2}, + {purge_seq, 1}, + {purge_infos, PurgeInfos} + ]) + end). test_purge_repeated_docid(DbName) -> - ?_test( - begin - {ok, Db} = couch_db:open_int(DbName, []), - Doc1 = {[{<<"_id">>, <<"foo1">>}, {<<"vsn">>, 1}]}, - Doc2 = {[{<<"_id">>, <<"foo2">>}, {<<"vsn">>, 2}]}, - {ok, Rev} = save_doc(Db, Doc1), - {ok, _Rev2} = save_doc(Db, Doc2), - couch_db:ensure_full_commit(Db), - - {ok, Db2} = couch_db:reopen(Db), - ?assertEqual(2, couch_db_engine:get_doc_count(Db2)), - ?assertEqual(0, couch_db_engine:get_del_doc_count(Db2)), - ?assertEqual(2, couch_db_engine:get_update_seq(Db2)), - ?assertEqual(0, couch_db_engine:get_purge_seq(Db2)), - - UUID = couch_uuids:new(), - UUID2 = couch_uuids:new(), - {ok, [{ok, PRevs}, {ok, PRevs}]} = couch_db:purge_docs(Db2, - [{UUID, <<"foo1">>, [Rev]}, {UUID2, <<"foo1">>, [Rev]}] - ), - ?assertEqual([Rev], PRevs), - - {ok, Db3} = couch_db:reopen(Db2), - {ok, _PIdsRevs} = couch_db:fold_purge_infos( - Db3, 0, fun fold_fun/2, [], []), - ?assertEqual(1, couch_db_engine:get_doc_count(Db3)), - ?assertEqual(0, couch_db_engine:get_del_doc_count(Db3)), - ?assertEqual(3, couch_db_engine:get_update_seq(Db3)), - ?assertEqual(2, couch_db_engine:get_purge_seq(Db3)) - end). - - -purge_id_not_exist(DbName) -> - ?_test( - begin - {ok, Db} = couch_db:open_int(DbName, []), - UUID = couch_uuids:new(), - {ok, [{ok, PRevs}]} = couch_db:purge_docs(Db, - [{UUID, <<"foo">>, [{0, <<0>>}]}]), - ?assertEqual([], PRevs), - - {ok, Db2} = couch_db:reopen(Db), - {ok, PIdsRevs} = couch_db:fold_purge_infos( - Db2, 0, fun fold_fun/2, [], []), - ?assertEqual(0, couch_db_engine:get_doc_count(Db2)), - ?assertEqual(0, couch_db_engine:get_del_doc_count(Db2)), - ?assertEqual(0, couch_db_engine:get_update_seq(Db2)), - ?assertEqual(0, couch_db_engine:get_purge_seq(Db2)), - ?assertEqual([], PIdsRevs) - end). - - -purge_non_leaf_rev(DbName) -> - ?_test( - begin - {ok, Db} = couch_db:open_int(DbName, []), - Doc0 = {[{<<"_id">>, <<"foo">>}, {<<"vsn">>, 1}]}, - {ok, Rev} = save_doc(Db, Doc0), - couch_db:ensure_full_commit(Db), - {ok, Db2} = couch_db:reopen(Db), - - Doc1 = {[ - {<<"_id">>, <<"foo">>}, {<<"vsn">>, 2}, - {<<"_rev">>, couch_doc:rev_to_str(Rev)} - ]}, - {ok, _Rev2} = save_doc(Db2, Doc1), - couch_db:ensure_full_commit(Db2), - {ok, Db3} = couch_db:reopen(Db2), - - UUID = couch_uuids:new(), - {ok, [{ok, PRevs}]} = couch_db:purge_docs(Db3, - [{UUID, <<"foo">>, [Rev]}]), - ?assertEqual([], PRevs), - - {ok, Db4} = couch_db:reopen(Db3), - {ok, PIdsRevs} = couch_db:fold_purge_infos( - Db4, 0, fun fold_fun/2, [], [] - ), - ?assertEqual(1, couch_db_engine:get_doc_count(Db4)), - ?assertEqual(2, couch_db_engine:get_update_seq(Db4)), - ?assertEqual(0, couch_db_engine:get_purge_seq(Db4)), - ?assertEqual([], PIdsRevs) - end). + ?_test(begin + {ok, [Rev1, _Rev2]} = save_docs(DbName, [ + {[{'_id', foo1}, {vsn, 1}]}, + {[{'_id', foo2}, {vsn, 2}]} + ]), + + assertProps(DbName, [ + {doc_count, 2}, + {del_doc_count, 0}, + {update_seq, 2}, + {purge_seq, 0}, + {changes, 2}, + {purge_infos, []} + ]), + + PurgeInfos = [ + {uuid(), <<"foo1">>, [Rev1]}, + {uuid(), <<"foo1">>, [Rev1]} + ], + + {ok, Resp} = purge(DbName, PurgeInfos), + ?assertEqual([{ok, [Rev1]}, {ok, []}], Resp), + + assertProps(DbName, [ + {doc_count, 1}, + {del_doc_count, 0}, + {update_seq, 3}, + {purge_seq, 2}, + {changes, 1}, + {purge_infos, PurgeInfos} + ]) + end). + + +test_purge_id_not_exist(DbName) -> + ?_test(begin + PurgeInfos = [ + {uuid(), <<"foo">>, [{0, <<0>>}]} + ], + + {ok, [{ok, PRevs}]} = purge(DbName, PurgeInfos), + ?assertEqual([], PRevs), + + assertProps(DbName, [ + {doc_count, 0}, + {del_doc_count, 0}, + {update_seq, 1}, + {changes, 0}, + {purge_seq, 1}, + {purge_infos, PurgeInfos} + ]) + end). + + +test_purge_non_leaf_rev(DbName) -> + ?_test(begin + {ok, Rev1} = save_doc(DbName, {[{'_id', foo}, {vsn, 1}]}), + Update = {[ + {'_id', foo}, + {'_rev', couch_doc:rev_to_str(Rev1)}, + {vsn, 2} + ]}, + {ok, _Rev2} = save_doc(DbName, Update), + + PurgeInfos = [ + {couch_uuids:new(), <<"foo">>, [Rev1]} + ], + + {ok, [{ok, PRevs}]} = purge(DbName, PurgeInfos), + ?assertEqual([], PRevs), + + assertProps(DbName, [ + {doc_count, 1}, + {del_doc_count, 0}, + {update_seq, 3}, + {changes, 1}, + {purge_seq, 1}, + {purge_infos, PurgeInfos} + ]) + end). test_purge_invalid_rev(DbName) -> - ?_test( - begin - {ok, Db} = couch_db:open_int(DbName, []), - Doc1 = {[{<<"_id">>, <<"foo1">>}, {<<"vsn">>, 1}]}, - Doc2 = {[{<<"_id">>, <<"foo2">>}, {<<"vsn">>, 2}]}, - {ok, _Rev} = save_doc(Db, Doc1), - {ok, Rev2} = save_doc(Db, Doc2), - couch_db:ensure_full_commit(Db), - {ok, Db2} = couch_db:reopen(Db), - - UUID = couch_uuids:new(), - {ok, [{ok, PRevs}]} = couch_db:purge_docs(Db2, - [{UUID, <<"foo1">>, [Rev2]}]), - ?assertEqual([], PRevs), - - {ok, Db3} = couch_db:reopen(Db2), - {ok, PIdsRevs} = couch_db:fold_purge_infos( - Db3, 0, fun fold_fun/2, [], [] - ), - ?assertEqual(2, couch_db_engine:get_doc_count(Db3)), - ?assertEqual(2, couch_db_engine:get_update_seq(Db3)), - ?assertEqual(0, couch_db_engine:get_purge_seq(Db3)), - ?assertEqual([], PIdsRevs) - end). + ?_test(begin + {ok, [_Rev1, Rev2]} = save_docs(DbName, [ + {[{'_id', foo1}, {vsn, 1}]}, + {[{'_id', foo2}, {vsn, 2}]} + ]), + + PurgeInfos = [ + {uuid(), <<"foo1">>, [Rev2]} + ], + + {ok, [{ok, PRevs}]} = purge(DbName, PurgeInfos), + ?assertEqual([], PRevs), + + assertProps(DbName, [ + {doc_count, 2}, + {del_doc_count, 0}, + {update_seq, 3}, + {changes, 2}, + {purge_seq, 1}, + {purge_infos, PurgeInfos} + ]) + end). test_purge_partial(DbName) -> - ?_test( - begin - {ok, Db} = couch_db:open_int(DbName, []), - Doc = {[{<<"_id">>, <<"foo">>}, {<<"vsn">>, <<"v1.1">>}]}, - {ok, Rev} = save_doc(Db, Doc), - couch_db:ensure_full_commit(Db), - {ok, Db2} = couch_db:reopen(Db), - - % create a conflict - DocConflict = #doc{ - id = <<"foo">>, - revs = {1, [crypto:hash(md5, <<"v1.2">>)]}, - body = {[ {<<"vsn">>, <<"v1.2">>}]} - }, - {ok, _} = couch_db:update_doc( - Db2, DocConflict, [], replicated_changes - ), - couch_db:ensure_full_commit(Db2), - {ok, Db3} = couch_db:reopen(Db2), - - UUID = couch_uuids:new(), - {ok, [{ok, PRevs}]} = couch_db:purge_docs(Db3, - [{UUID, <<"foo">>, [Rev]}]), - ?assertEqual([Rev], PRevs), - - {ok, Db4} = couch_db:reopen(Db3), - {ok, PIdsRevs} = couch_db:fold_purge_infos( - Db4, 0, fun fold_fun/2, [], [] - ), - % still has one doc - ?assertEqual(1, couch_db_engine:get_doc_count(Db4)), - ?assertEqual(0, couch_db_engine:get_del_doc_count(Db4)), - ?assertEqual(3, couch_db_engine:get_update_seq(Db4)), - ?assertEqual(1, couch_db_engine:get_purge_seq(Db4)), - ?assertEqual([{<<"foo">>, [Rev]}], PIdsRevs) - end). + ?_test(begin + {ok, Rev1} = save_doc(DbName, {[{'_id', foo}, {vsn, <<"v1.1">>}]}), + Update = {[ + {'_id', foo}, + {'_rev', couch_doc:rev_to_str({1, [crypto:hash(md5, <<"v1.2">>)]})}, + {vsn, <<"v1.2">>} + ]}, + {ok, [_Rev2]} = save_docs(DbName, [Update], [replicated_changes]), + + PurgeInfos = [ + {uuid(), <<"foo">>, [Rev1]} + ], + + {ok, [{ok, PRevs}]} = purge(DbName, PurgeInfos), + ?assertEqual([Rev1], PRevs), + + assertProps(DbName, [ + {doc_count, 1}, + {del_doc_count, 0}, + {update_seq, 3}, + {changes, 1}, + {purge_seq, 1}, + {purge_infos, PurgeInfos} + ]) + end). test_purge_repeated_rev(DbName) -> - ?_test( - begin - {ok, Db} = couch_db:open_int(DbName, []), - Doc = {[{<<"_id">>, <<"foo">>}, {<<"vsn">>, <<"v1.1">>}]}, - {ok, Rev} = save_doc(Db, Doc), - couch_db:ensure_full_commit(Db), - {ok, Db2} = couch_db:reopen(Db), - - % create a conflict - DocConflict = #doc{ - id = <<"foo">>, - revs = {1, [crypto:hash(md5, <<"v1.2">>)]}, - body = {[ {<<"vsn">>, <<"v1.2">>}]} - }, - {ok, _} = couch_db:update_doc( - Db2, DocConflict, [], replicated_changes - ), - couch_db:ensure_full_commit(Db2), - {ok, Db3} = couch_db:reopen(Db2), - - ?assertEqual(1, couch_db_engine:get_doc_count(Db3)), - ?assertEqual(0, couch_db_engine:get_del_doc_count(Db3)), - ?assertEqual(2, couch_db_engine:get_update_seq(Db3)), - ?assertEqual(0, couch_db_engine:get_purge_seq(Db3)), - - UUID = couch_uuids:new(), - UUID2 = couch_uuids:new(), - - FDI = couch_db:get_full_doc_info(Db3, <<"foo">>), - Revs2 = FDI#full_doc_info.rev_tree, - - [{1, {_Rev1, _Leaf1, []}}, {1, {Rev2, _Leaf2, []}}] = Revs2, - {ok, [ {ok, _PRevs2}]} = couch_db:purge_docs(Db3, - [{UUID2, <<"foo">>, [{1, Rev2}]}] - ), - - {ok, Db4} = couch_db:reopen(Db3), - {ok, PIdsRevs} = couch_db:fold_purge_infos( - Db4, 0, fun fold_fun/2, [], []), - % still has one doc - ?assertEqual(1, couch_db_engine:get_doc_count(Db4)), - ?assertEqual(0, couch_db_engine:get_del_doc_count(Db4)), - ?assertEqual(3, couch_db_engine:get_update_seq(Db4)), - ?assertEqual(2, couch_db_engine:get_purge_seq(Db4)) - end). - - -purge_deep_tree(DbName) -> - ?_test( - begin - NRevs = 100, - {ok, Db0} = couch_db:open_int(DbName, []), - Doc0 = {[{<<"_id">>, <<"bar">>}, {<<"vsn">>, 0}]}, - {ok, InitRev} = save_doc(Db0, Doc0), - ok = couch_db:close(Db0), - LastRev = lists:foldl(fun(V, PrevRev) -> - {ok, Db} = couch_db:open_int(DbName, []), - {ok, Rev} = save_doc(Db, - {[{<<"_id">>, <<"bar">>}, - {<<"vsn">>, V}, - {<<"_rev">>, couch_doc:rev_to_str(PrevRev)}]} - ), - ok = couch_db:close(Db), - Rev - end, InitRev, lists:seq(2, NRevs)), - {ok, Db1} = couch_db:open_int(DbName, []), - - % purge doc - UUID = couch_uuids:new(), - {ok, [{ok, PRevs}]} = couch_db:purge_docs(Db1, - [{UUID, <<"bar">>, [LastRev]}]), - ?assertEqual([LastRev], PRevs), - - {ok, Db2} = couch_db:reopen(Db1), - % no docs left - ?assertEqual(0, couch_db_engine:get_doc_count(Db2)), - ?assertEqual(0, couch_db_engine:get_del_doc_count(Db2)), - ?assertEqual(1, couch_db_engine:get_purge_seq(Db2)), - ?assertEqual(NRevs + 1 , couch_db_engine:get_update_seq(Db2)) - end). + ?_test(begin + {ok, Rev1} = save_doc(DbName, {[{'_id', foo}, {vsn, <<"v1.1">>}]}), + Update = {[ + {'_id', foo}, + {'_rev', couch_doc:rev_to_str({1, [crypto:hash(md5, <<"v1.2">>)]})}, + {vsn, <<"v1.2">>} + ]}, + {ok, [Rev2]} = save_docs(DbName, [Update], [replicated_changes]), + + assertProps(DbName, [ + {doc_count, 1}, + {del_doc_count, 0}, + {update_seq, 2}, + {changes, 1}, + {purge_seq, 0}, + {purge_infos, []} + ]), + + PurgeInfos1 = [ + {uuid(), <<"foo">>, [Rev1]}, + {uuid(), <<"foo">>, [Rev1, Rev2]} + ], + + {ok, [{ok, PRevs1}, {ok, PRevs2}]} = purge(DbName, PurgeInfos1), + ?assertEqual([Rev1], PRevs1), + ?assertEqual([Rev2], PRevs2), + + assertProps(DbName, [ + {doc_count, 0}, + {del_doc_count, 0}, + {update_seq, 3}, + {changes, 0}, + {purge_seq, 2}, + {purge_infos, PurgeInfos1} + ]) + end). + + +test_purge_deep_tree(DbName) -> + ?_test(begin + {ok, InitRev} = save_doc(DbName, {[{'_id', bar}, {vsn, 0}]}), + LastRev = lists:foldl(fun(Count, PrevRev) -> + Update = {[ + {'_id', bar}, + {'_rev', couch_doc:rev_to_str(PrevRev)}, + {vsn, Count} + ]}, + {ok, NewRev} = save_doc(DbName, Update), + NewRev + end, InitRev, lists:seq(1, ?REV_DEPTH)), + PurgeInfos = [ + {uuid(), <<"bar">>, [LastRev]} + ], -test_purge_duplicate_UUID(DbName) -> - ?_test( - begin - {ok, Db} = couch_db:open_int(DbName, []), - Doc1 = {[{<<"_id">>, <<"foo1">>}, {<<"vsn">>, 1.1}]}, - {ok, Rev} = save_doc(Db, Doc1), - couch_db:ensure_full_commit(Db), - - {ok, Db2} = couch_db:reopen(Db), - ?assertEqual(1, couch_db_engine:get_doc_count(Db2)), - ?assertEqual(0, couch_db_engine:get_del_doc_count(Db2)), - ?assertEqual(1, couch_db_engine:get_update_seq(Db2)), - ?assertEqual(0, couch_db_engine:get_purge_seq(Db2)), - - {ok, [{ok, PRevs}]} = couch_db:purge_docs( - Db2, [{uuid, <<"foo1">>, [Rev]}] - ), - {ok, [{ok, PRevs2}]} = couch_db:purge_docs( - Db2, [{uuid, <<"foo1">>, [Rev]}] - ), - ?assertEqual([Rev], PRevs), - ?assertEqual([], PRevs2), - - {ok, Db3} = couch_db:reopen(Db2), - {ok, _PIdsRevs} = couch_db:fold_purge_infos( - Db3, 0, fun fold_fun/2, [], []), - ?assertEqual(0, couch_db_engine:get_doc_count(Db3)), - ?assertEqual(0, couch_db_engine:get_del_doc_count(Db3)), - ?assertEqual(2, couch_db_engine:get_update_seq(Db3)), - ?assertEqual(1, couch_db_engine:get_purge_seq(Db3)) - end). - - -purge_with_replication() -> - ?_test( - begin - Ctx = test_util:start_couch([couch_replicator]), - Source = ?tempdb(), - {ok, SourceDb} = create_db(Source), - Target = ?tempdb(), - {ok, _Db} = create_db(Target), - - % create Doc and do replication to Target - {ok, Rev} = save_doc(SourceDb, - {[{<<"_id">>, <<"foo">>}, {<<"vsn">>, 1}]}), - couch_db:ensure_full_commit(SourceDb), - {ok, SourceDb2} = couch_db:reopen(SourceDb), - RepObject = {[ - {<<"source">>, Source}, - {<<"target">>, Target} - ]}, - {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER), - {ok, TargetDb} = couch_db:open_int(Target, []), - {ok, Doc} = couch_db:get_doc_info(TargetDb, <<"foo">>), - - % purge Doc on Source and do replication to Target - % assert purges don't get replicated to Target - UUID = couch_uuids:new(), - {ok, _} = couch_db:purge_docs( - SourceDb2, [{UUID, <<"foo">>, [Rev]}] - ), - {ok, SourceDb3} = couch_db:reopen(SourceDb2), - {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER), - {ok, TargetDb2} = couch_db:open_int(Target, []), - {ok, Doc2} = couch_db:get_doc_info(TargetDb2, <<"foo">>), - [Rev2] = Doc2#doc_info.revs, - ?assertEqual(Rev, Rev2#rev_info.rev), - ?assertEqual(Doc, Doc2), - ?assertEqual(0, couch_db_engine:get_doc_count(SourceDb3)), - ?assertEqual(1, couch_db_engine:get_purge_seq(SourceDb3)), - ?assertEqual(1, couch_db_engine:get_doc_count(TargetDb2)), - ?assertEqual(0, couch_db_engine:get_purge_seq(TargetDb2)), - - % replicate from Target to Source - % assert that Doc reappears on Source - RepObject2 = {[ - {<<"source">>, Target}, - {<<"target">>, Source} - ]}, - {ok, _} = couch_replicator:replicate(RepObject2, ?ADMIN_USER), - {ok, SourceDb4} = couch_db:reopen(SourceDb3), - {ok, Doc3} = couch_db:get_doc_info(SourceDb4, <<"foo">>), - [Rev3] = Doc3#doc_info.revs, - ?assertEqual(Rev, Rev3#rev_info.rev), - ?assertEqual(1, couch_db_engine:get_doc_count(SourceDb4)), - ?assertEqual(1, couch_db_engine:get_purge_seq(SourceDb4)), + {ok, [{ok, PRevs}]} = purge(DbName, PurgeInfos), + ?assertEqual([LastRev], PRevs), + + assertProps(DbName, [ + {doc_count, 0}, + {del_doc_count, 0}, + {update_seq, ?REV_DEPTH + 2}, + {changes, 0}, + {purge_seq, 1}, + {purge_infos, PurgeInfos} + ]) + end). - delete_db(Source), - delete_db(Target), - ok = application:stop(couch_replicator), - ok = test_util:stop_couch(Ctx) - end). + +test_purge_duplicate_UUID(DbName) -> + ?_test(begin + {ok, Rev} = save_doc(DbName, {[{'_id', foo1}, {vsn, 1.1}]}), + + assertProps(DbName, [ + {doc_count, 1}, + {del_doc_count, 0}, + {update_seq, 1}, + {changes, 1}, + {purge_seq, 0}, + {purge_infos, []} + ]), + + PurgeInfos = [ + {uuid(), <<"foo1">>, [Rev]} + ], + + {ok, [{ok, PRevs1}]} = purge(DbName, PurgeInfos), + ?assertEqual([Rev], PRevs1), + + % Attempting to purge a repeated UUID is an error + ?assertThrow({badreq, _}, purge(DbName, PurgeInfos)), + + % Although we can replicate it in + {ok, []} = purge(DbName, PurgeInfos, [replicated_changes]), + + assertProps(DbName, [ + {doc_count, 0}, + {del_doc_count, 0}, + {update_seq, 3}, + {changes, 0}, + {purge_seq, 1}, + {purge_infos, PurgeInfos} + ]) + end). + + +test_purge_with_replication({Source, Target}) -> + ?_test(begin + {ok, Rev1} = save_doc(Source, {[{'_id', foo}, {vsn, 1}]}), + + assertProps(Source, [ + {doc_count, 1}, + {del_doc_count, 0}, + {update_seq, 1}, + {changes, 1}, + {purge_seq, 0}, + {purge_infos, []} + ]), + + RepObject = {[ + {<<"source">>, Source}, + {<<"target">>, Target} + ]}, + + {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER), + {ok, Doc1} = open_doc(Target, foo), + + assertProps(Target, [ + {doc_count, 1}, + {del_doc_count, 0}, + {update_seq, 1}, + {changes, 1}, + {purge_seq, 0}, + {purge_infos, []} + ]), + + PurgeInfos = [ + {uuid(), <<"foo">>, [Rev1]} + ], + + {ok, [{ok, PRevs}]} = purge(Source, PurgeInfos), + ?assertEqual([Rev1], PRevs), + + assertProps(Source, [ + {doc_count, 0}, + {del_doc_count, 0}, + {update_seq, 2}, + {changes, 0}, + {purge_seq, 1}, + {purge_infos, PurgeInfos} + ]), + + % Show that a purge on the source is + % not replicated to the target + {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER), + {ok, Doc2} = open_doc(Target, foo), + [Rev2] = Doc2#doc_info.revs, + ?assertEqual(Rev1, Rev2#rev_info.rev), + ?assertEqual(Doc1, Doc2), + + assertProps(Target, [ + {doc_count, 1}, + {del_doc_count, 0}, + {update_seq, 1}, + {changes, 1}, + {purge_seq, 0}, + {purge_infos, []} + ]), + + % Show that replicating from the target + % back to the source reintroduces the doc + RepObject2 = {[ + {<<"source">>, Target}, + {<<"target">>, Source} + ]}, + + {ok, _} = couch_replicator:replicate(RepObject2, ?ADMIN_USER), + {ok, Doc3} = open_doc(Source, foo), + [Revs3] = Doc3#doc_info.revs, + ?assertEqual(Rev1, Revs3#rev_info.rev), + + assertProps(Source, [ + {doc_count, 1}, + {del_doc_count, 0}, + {update_seq, 3}, + {changes, 1}, + {purge_seq, 1}, + {purge_infos, PurgeInfos} + ]) + end). create_db(DbName) -> couch_db:create(DbName, [?ADMIN_CTX, overwrite]). + delete_db(DbName) -> couch_server:delete(DbName, [?ADMIN_CTX]). -save_doc(Db, Json) -> - Doc = couch_doc:from_json_obj(Json), - couch_db:update_doc(Db, Doc, []). -fold_fun({_PSeq, _UUID, Id, Revs}, Acc) -> - {ok, [{Id, Revs} | Acc]}.
\ No newline at end of file +save_doc(DbName, Json) -> + {ok, [Rev]} = save_docs(DbName, [Json], []), + {ok, Rev}. + + +save_docs(DbName, JsonDocs) -> + save_docs(DbName, JsonDocs, []). + + +save_docs(DbName, JsonDocs, Options) -> + Docs = lists:map(fun(JDoc) -> + couch_doc:from_json_obj(?JSON_DECODE(?JSON_ENCODE(JDoc))) + end, JsonDocs), + Opts = [full_commit | Options], + {ok, Db} = couch_db:open_int(DbName, []), + try + case lists:member(replicated_changes, Options) of + true -> + {ok, []} = couch_db:update_docs( + Db, Docs, Opts, replicated_changes), + {ok, lists:map(fun(Doc) -> + {Pos, [RevId | _]} = Doc#doc.revs, + {Pos, RevId} + end, Docs)}; + false -> + {ok, Resp} = couch_db:update_docs(Db, Docs, Opts), + {ok, [Rev || {ok, Rev} <- Resp]} + end + after + couch_db:close(Db) + end. + + +open_doc(DbName, DocId0) -> + DocId = ?JSON_DECODE(?JSON_ENCODE(DocId0)), + {ok, Db} = couch_db:open_int(DbName, []), + try + couch_db:get_doc_info(Db, DocId) + after + couch_db:close(Db) + end. + + +purge(DbName, PurgeInfos) -> + purge(DbName, PurgeInfos, []). + + +purge(DbName, PurgeInfos0, Options) when is_list(PurgeInfos0) -> + PurgeInfos = lists:map(fun({UUID, DocIdJson, Revs}) -> + {UUID, ?JSON_DECODE(?JSON_ENCODE(DocIdJson)), Revs} + end, PurgeInfos0), + {ok, Db} = couch_db:open_int(DbName, []), + try + couch_db:purge_docs(Db, PurgeInfos, Options) + after + couch_db:close(Db) + end. + + +assertProps(DbName, Props) when is_binary(DbName) -> + {ok, Db} = couch_db:open_int(DbName, []), + try + assertEachProp(Db, Props) + after + couch_db:close(Db) + end. + + +assertEachProp(_Db, []) -> + ok; +assertEachProp(Db, [{doc_count, Expect} | Rest]) -> + {ok, DocCount} = couch_db:get_doc_count(Db), + ?assertEqual(Expect, DocCount), + assertEachProp(Db, Rest); +assertEachProp(Db, [{del_doc_count, Expect} | Rest]) -> + {ok, DelDocCount} = couch_db:get_del_doc_count(Db), + ?assertEqual(Expect, DelDocCount), + assertEachProp(Db, Rest); +assertEachProp(Db, [{update_seq, Expect} | Rest]) -> + UpdateSeq = couch_db:get_update_seq(Db), + ?assertEqual(Expect, UpdateSeq), + assertEachProp(Db, Rest); +assertEachProp(Db, [{changes, Expect} | Rest]) -> + {ok, NumChanges} = couch_db:fold_changes(Db, 0, fun fold_changes/2, 0, []), + ?assertEqual(Expect, NumChanges), + assertEachProp(Db, Rest); +assertEachProp(Db, [{purge_seq, Expect} | Rest]) -> + {ok, PurgeSeq} = couch_db:get_purge_seq(Db), + ?assertEqual(Expect, PurgeSeq), + assertEachProp(Db, Rest); +assertEachProp(Db, [{purge_infos, Expect} | Rest]) -> + {ok, PurgeInfos} = couch_db:fold_purge_infos(Db, 0, fun fold_fun/2, [], []), + ?assertEqual(Expect, lists:reverse(PurgeInfos)), + assertEachProp(Db, Rest). + + +fold_fun({_PSeq, UUID, Id, Revs}, Acc) -> + {ok, [{UUID, Id, Revs} | Acc]}. + + +fold_changes(_A, Acc) -> + %io:format(standard_error, "~nCHANGE: ~p~n~n", [_A]), + {ok, Acc + 1}. + + +uuid() -> couch_uuids:random(). diff --git a/src/couch/test/couch_db_purge_seqs_tests.erl b/src/couch/test/couch_db_purge_seqs_tests.erl index 214d5b1db..dda23f300 100644 --- a/src/couch/test/couch_db_purge_seqs_tests.erl +++ b/src/couch/test/couch_db_purge_seqs_tests.erl @@ -21,10 +21,12 @@ setup() -> {ok, _Db} = create_db(DbName), DbName. + teardown(DbName) -> delete_db(DbName), ok. + couch_db_purge_seqs_test_() -> { "Couch_db purge_seqs", @@ -51,6 +53,7 @@ couch_db_purge_seqs() -> ] }. + test_update_seq_bounce(DbName) -> ?_test( begin @@ -99,7 +102,7 @@ test_update_seq_inc_on_complete_purge(DbName) -> ), {ok, Db3} = couch_db:reopen(Db2), - ?assertEqual(2, couch_db_engine:get_update_seq(Db3)), + ?assertEqual(3, couch_db_engine:get_update_seq(Db3)), ?assertEqual([], PRevs), @@ -111,7 +114,7 @@ test_update_seq_inc_on_complete_purge(DbName) -> ?assertEqual([Rev], PRevs2), {ok, Db4} = couch_db:reopen(Db3), - ?assertEqual(3, couch_db_engine:get_update_seq(Db4)) + ?assertEqual(4, couch_db_engine:get_update_seq(Db4)) end). diff --git a/src/couch/test/couchdb_file_compression_tests.erl b/src/couch/test/couchdb_file_compression_tests.erl index e5ef74c61..8f0fe5bf1 100644 --- a/src/couch/test/couchdb_file_compression_tests.erl +++ b/src/couch/test/couchdb_file_compression_tests.erl @@ -57,8 +57,8 @@ couch_file_compression_test_() -> fun should_use_none/1, fun should_use_deflate_1/1, fun should_use_deflate_9/1, - fun should_use_snappy/1 - %fun should_compare_compression_methods/1 + fun should_use_snappy/1, + fun should_compare_compression_methods/1 ] } } diff --git a/src/couch_mrview/test/couch_mrview_purge_docs_tests.erl b/src/couch_mrview/test/couch_mrview_purge_docs_tests.erl index 1c62c3f2a..bce91fd3d 100644 --- a/src/couch_mrview/test/couch_mrview_purge_docs_tests.erl +++ b/src/couch_mrview/test/couch_mrview_purge_docs_tests.erl @@ -40,8 +40,8 @@ view_purge_test_() -> [ fun test_purge_single/1, fun test_purge_multiple/1, - fun test_purge_with_compact1/1 - %fun test_purge_with_compact2/1 + fun test_purge_with_compact1/1, + fun test_purge_with_compact2/1 ] } } diff --git a/src/fabric/src/fabric_doc_purge.erl b/src/fabric/src/fabric_doc_purge.erl index b51f28eec..61b84d81b 100644 --- a/src/fabric/src/fabric_doc_purge.erl +++ b/src/fabric/src/fabric_doc_purge.erl @@ -24,8 +24,8 @@ -record(acc, { worker_uuids, - req_count, resps, + uuid_counts, w }). @@ -34,7 +34,7 @@ go(_, [], _) -> {ok, []}; go(DbName, IdsRevs, Options) -> % Generate our purge requests of {UUID, DocId, Revs} - {UUIDs, Reqs, Count} = create_reqs(IdsRevs, [], [], 0), + {UUIDs, Reqs} = create_reqs(IdsRevs, [], []), % Fire off rexi workers for each shard. {Workers, WorkerUUIDs} = dict:fold(fun(Shard, ShardReqs, {Ws, WUUIDs}) -> @@ -46,12 +46,18 @@ go(DbName, IdsRevs, Options) -> {[Worker | Ws], [{Worker, ShardUUIDs} | WUUIDs]} end, {[], []}, group_reqs_by_shard(DbName, Reqs)), + UUIDCounts = lists:foldl(fun({_Worker, WUUIDs}, CountAcc) -> + lists:foldl(fun(UUID, InnerCountAcc) -> + dict:update_counter(UUID, 1, InnerCountAcc) + end, CountAcc, WUUIDs) + end, dict:new(), WorkerUUIDs), + RexiMon = fabric_util:create_monitors(Workers), Timeout = fabric_util:request_timeout(), Acc0 = #acc{ worker_uuids = WorkerUUIDs, - req_count = Count, resps = dict:from_list([{UUID, []} || UUID <- UUIDs]), + uuid_counts = UUIDCounts, w = w(DbName, Options) }, Acc2 = try rexi_utils:recv(Workers, #shard.ref, @@ -73,7 +79,8 @@ go(DbName, IdsRevs, Options) -> rexi_monitor:stop(RexiMon) end, - {ok, format_resps(UUIDs, Acc2)}. + FinalResps = format_resps(UUIDs, Acc2), + {resp_health(FinalResps), FinalResps}. handle_message({rexi_DOWN, _, {_, Node}, _}, _Worker, Acc) -> @@ -101,7 +108,6 @@ handle_message({ok, Replies}, Worker, Acc) -> resps = Resps } = Acc, {value, {_W, UUIDs}, Rest} = lists:keytake(Worker, 1, WorkerUUIDs), - couch_log:error("XKCD: ~p ~p :: ~p ~p", [length(UUIDs), length(Replies), UUIDs, Replies]), NewResps = append_resps(UUIDs, Replies, Resps), maybe_stop(Acc#acc{worker_uuids = Rest, resps = NewResps}); @@ -109,14 +115,14 @@ handle_message({bad_request, Msg}, _, _) -> throw({bad_request, Msg}). -create_reqs([], UUIDs, Reqs, Count) -> - {lists:reverse(UUIDs), lists:reverse(Reqs), Count}; +create_reqs([], UUIDs, Reqs) -> + {lists:reverse(UUIDs), lists:reverse(Reqs)}; -create_reqs([{Id, Revs} | RestIdsRevs], UUIDs, Reqs, Count) -> +create_reqs([{Id, Revs} | RestIdsRevs], UUIDs, Reqs) -> UUID = couch_uuids:new(), NewUUIDs = [UUID | UUIDs], NewReqs = [{UUID, Id, Revs} | Reqs], - create_reqs(RestIdsRevs, NewUUIDs, NewReqs, Count + 1). + create_reqs(RestIdsRevs, NewUUIDs, NewReqs). group_reqs_by_shard(DbName, Reqs) -> @@ -151,11 +157,11 @@ append_resps([UUID | RestUUIDs], [Reply | RestReplies], Resps) -> maybe_stop(#acc{worker_uuids = []} = Acc) -> {stop, Acc}; -maybe_stop(#acc{resps = Resps, w = W} = Acc) -> +maybe_stop(#acc{resps = Resps, uuid_counts = Counts, w = W} = Acc) -> try - dict:fold(fun(_UUID, UUIDResps, _) -> - couch_log:error("XKCD: ~p ~p", [UUIDResps, W]), - case has_quorum(UUIDResps, W) of + dict:fold(fun(UUID, UUIDResps, _) -> + UUIDCount = dict:fetch(UUID, Counts), + case has_quorum(UUIDResps, UUIDCount, W) of true -> ok; false -> throw(keep_going) end @@ -179,7 +185,9 @@ format_resps(UUIDs, #acc{} = Acc) -> [{UUID, Error} | ReplyAcc]; _ -> AllRevs = lists:usort(lists:flatten(OkReplies)), - Health = if length(OkReplies) >= W -> ok; true -> accepted end, + IsOk = length(OkReplies) >= W + andalso length(lists:usort(OkReplies)) == 1, + Health = if IsOk -> ok; true -> accepted end, [{UUID, {Health, AllRevs}} | ReplyAcc] end end, @@ -190,224 +198,375 @@ format_resps(_UUIDs, Else) -> Else. -has_quorum([], W) when W > 0 -> - false; -has_quorum(_, W) when W =< 0 -> - true; -has_quorum([{ok, _} | Rest], W) when W > 0 -> - has_quorum(Rest, W - 1). - - -%% % eunits -%% doc_purge_ok_test() -> -%% meck:new(couch_log), -%% meck:expect(couch_log, warning, fun(_,_) -> ok end), -%% meck:expect(couch_log, notice, fun(_,_) -> ok end), -%% -%% Revs1 = [{1, <<"rev11">>}], UUID1 = <<"3de03c5f4c2cd34cc515a9d1ea000abd">>, -%% UUIDIdRevs1 = {UUID1, <<"id1">>, Revs1}, -%% Revs2 = [{1, <<"rev12">>}], UUID2 = <<"4de03c5f4c2cd34cc515a9d1ea000abc">>, -%% UUIDIdRevs2 = {UUID2, <<"id2">>, Revs2}, -%% UUIDsIDdsRevs = [UUIDIdRevs1, UUIDIdRevs2], -%% Shards = -%% mem3_util:create_partition_map("foo",3,1,["node1","node2","node3"]), -%% Counters = dict:to_list( -%% group_idrevs_by_shard_hack(<<"foo">>, Shards, UUIDsIDdsRevs)), -%% DocsDict = dict:new(), -%% -%% % ***test for W = 2 -%% AccW2 = {length(Shards), length(UUIDsIDdsRevs), list_to_integer("2"), -%% Counters, DocsDict}, -%% {ok, {WaitingCountW2_1,_,_,_,_} = AccW2_1} = -%% handle_message({ok,[{ok, Revs1}, {ok, Revs2}]}, hd(Shards), AccW2), -%% ?assertEqual(2, WaitingCountW2_1), -%% {stop, FinalReplyW2 } = -%% handle_message({ok, [{ok, Revs1}, {ok, Revs2}]}, -%% lists:nth(2,Shards), AccW2_1), -%% ?assertEqual( -%% {ok, [{UUID1, {ok, Revs1}}, {UUID2, {ok, Revs2}}]}, -%% FinalReplyW2 -%% ), -%% -%% % ***test for W = 3 -%% AccW3 = {length(Shards), length(UUIDsIDdsRevs), list_to_integer("3"), -%% Counters, DocsDict}, -%% {ok, {WaitingCountW3_1,_,_,_,_} = AccW3_1} = -%% handle_message({ok, [{ok, Revs1}, {ok, Revs2}]}, hd(Shards), AccW3), -%% ?assertEqual(2, WaitingCountW3_1), -%% {ok, {WaitingCountW3_2,_,_,_,_} = AccW3_2} = -%% handle_message({ok,[{ok, Revs1}, {ok, Revs2}]}, -%% lists:nth(2,Shards), AccW3_1), -%% ?assertEqual(1, WaitingCountW3_2), -%% {stop, FinalReplyW3 } = -%% handle_message({ok, [{ok, Revs1}, {ok, Revs2}]}, -%% lists:nth(3,Shards), AccW3_2), -%% ?assertEqual( -%% {ok, [{UUID1, {ok, Revs1}}, {UUID2, {ok, Revs2}}]}, -%% FinalReplyW3 -%% ), -%% -%% % *** test rexi_exit on 1 node -%% Acc0 = {length(Shards), length(UUIDsIDdsRevs), list_to_integer("2"), -%% Counters, DocsDict}, -%% {ok, {WaitingCount1,_,_,_,_} = Acc1} = -%% handle_message({ok, [{ok, Revs1}, {ok, Revs2}]}, hd(Shards), Acc0), -%% ?assertEqual(2, WaitingCount1), -%% {ok, {WaitingCount2,_,_,_,_} = Acc2} = -%% handle_message({rexi_EXIT, nil}, lists:nth(2,Shards), Acc1), -%% ?assertEqual(1, WaitingCount2), -%% {stop, Reply} = -%% handle_message({ok, [{ok, Revs1}, {ok, Revs2}]}, -%% lists:nth(3,Shards), Acc2), -%% ?assertEqual( -%% {ok,[{UUID1, {ok, Revs1}}, {UUID2, {ok, Revs2}}]}, -%% Reply -%% ), -%% -%% % *** test {error, purge_during_compaction_exceeded_limit} on all nodes -%% % *** still should return ok reply for the request -%% ErrPDCEL = {error, purge_during_compaction_exceeded_limit}, -%% Acc20 = {length(Shards), length(UUIDsIDdsRevs), list_to_integer("3"), -%% Counters, DocsDict}, -%% {ok, {WaitingCount21,_,_,_,_} = Acc21} = -%% handle_message({ok, [ErrPDCEL, ErrPDCEL]}, hd(Shards), Acc20), -%% ?assertEqual(2, WaitingCount21), -%% {ok, {WaitingCount22,_,_,_,_} = Acc22} = -%% handle_message({ok, [ErrPDCEL, ErrPDCEL]}, lists:nth(2,Shards), Acc21), -%% ?assertEqual(1, WaitingCount22), -%% {stop, Reply2 } = -%% handle_message({ok, [ErrPDCEL, ErrPDCEL]}, lists:nth(3,Shards), Acc22), -%% ?assertEqual( -%% {ok, [{UUID1, ErrPDCEL}, {UUID2, ErrPDCEL}]}, -%% Reply2 -%% ), -%% -%% % *** test {error, purged_docs_limit_exceeded} on all nodes -%% % *** still should return ok reply for the request -%% ErrPDLE = {error, purged_docs_limit_exceeded}, -%% Acc30 = {length(Shards), length(UUIDsIDdsRevs), list_to_integer("3"), -%% Counters, DocsDict}, -%% {ok, {WaitingCount31,_,_,_,_} = Acc31} = -%% handle_message({ok, [ErrPDLE, ErrPDLE]}, hd(Shards), Acc30), -%% ?assertEqual(2, WaitingCount31), -%% {ok, {WaitingCount32,_,_,_,_} = Acc32} = -%% handle_message({ok, [ErrPDLE, ErrPDLE]}, lists:nth(2,Shards), Acc31), -%% ?assertEqual(1, WaitingCount32), -%% {stop, Reply3 } = -%% handle_message({ok, [ErrPDLE, ErrPDLE]},lists:nth(3,Shards), Acc32), -%% ?assertEqual( -%% {ok, [{UUID1, ErrPDLE}, {UUID2, ErrPDLE}]}, -%% Reply3 -%% ), -%% meck:unload(couch_log). -%% -%% -%% doc_purge_accepted_test() -> -%% meck:new(couch_log), -%% meck:expect(couch_log, warning, fun(_,_) -> ok end), -%% meck:expect(couch_log, notice, fun(_,_) -> ok end), -%% -%% Revs1 = [{1, <<"rev11">>}], UUID1 = <<"3de03c5f4c2cd34cc515a9d1ea000abd">>, -%% UUIDIdRevs1 = {UUID1, <<"id1">>, Revs1}, -%% Revs2 = [{1, <<"rev12">>}], UUID2 = <<"4de03c5f4c2cd34cc515a9d1ea000abc">>, -%% UUIDIdRevs2 = {UUID2, <<"id2">>, Revs2}, -%% UUIDsIDdsRevs = [UUIDIdRevs1, UUIDIdRevs2], -%% Shards = -%% mem3_util:create_partition_map("foo",3,1,["node1","node2","node3"]), -%% Counters = dict:to_list( -%% group_idrevs_by_shard_hack(<<"foo">>, Shards, UUIDsIDdsRevs)), -%% DocsDict = dict:new(), -%% -%% % *** test rexi_exit on 2 nodes -%% Acc0 = {length(Shards), length(UUIDsIDdsRevs), list_to_integer("2"), -%% Counters, DocsDict}, -%% {ok, {WaitingCount1,_,_,_,_} = Acc1} = -%% handle_message({ok, [{ok, Revs1}, {ok, Revs2}]}, hd(Shards), Acc0), -%% ?assertEqual(2, WaitingCount1), -%% {ok, {WaitingCount2,_,_,_,_} = Acc2} = -%% handle_message({rexi_EXIT, nil}, lists:nth(2, Shards), Acc1), -%% ?assertEqual(1, WaitingCount2), -%% {stop, Reply} = -%% handle_message({rexi_EXIT, nil}, lists:nth(3, Shards), Acc2), -%% ?assertEqual( -%% {accepted, [{UUID1, {accepted, Revs1}}, {UUID2, {accepted, Revs2}}]}, -%% Reply -%% ), -%% meck:unload(couch_log). -%% -%% -%% doc_purge_error_test() -> -%% meck:new(couch_log), -%% meck:expect(couch_log, warning, fun(_,_) -> ok end), -%% meck:expect(couch_log, notice, fun(_,_) -> ok end), -%% -%% Revs1 = [{1, <<"rev11">>}], UUID1 = <<"3de03c5f4c2cd34cc515a9d1ea000abd">>, -%% UUIDIdRevs1 = {UUID1, <<"id1">>, Revs1}, -%% Revs2 = [{1, <<"rev12">>}], UUID2 = <<"4de03c5f4c2cd34cc515a9d1ea000abc">>, -%% UUIDIdRevs2 = {UUID2, <<"id2">>, Revs2}, -%% UUIDsIDdsRevs = [UUIDIdRevs1, UUIDIdRevs2], -%% Shards = -%% mem3_util:create_partition_map("foo",3,1,["node1","node2","node3"]), -%% Counters = dict:to_list( -%% group_idrevs_by_shard_hack(<<"foo">>, Shards, UUIDsIDdsRevs)), -%% DocsDict = dict:new(), -%% -%% % *** test rexi_exit on all 3 nodes -%% Acc0 = {length(Shards), length(UUIDsIDdsRevs), list_to_integer("2"), -%% Counters, DocsDict}, -%% {ok, {WaitingCount1,_,_,_,_} = Acc1} = -%% handle_message({rexi_EXIT, nil}, hd(Shards), Acc0), -%% ?assertEqual(2, WaitingCount1), -%% {ok, {WaitingCount2,_,_,_,_} = Acc2} = -%% handle_message({rexi_EXIT, nil}, lists:nth(2,Shards), Acc1), -%% ?assertEqual(1, WaitingCount2), -%% {stop, Reply} = -%% handle_message({rexi_EXIT, nil}, lists:nth(3,Shards), Acc2), -%% ?assertEqual( -%% {error, [{UUID1, {error, internal_server_error}}, -%% {UUID2, {error, internal_server_error}}]}, -%% Reply -%% ), -%% -%% % ***test w quorum > # shards, which should fail immediately -%% Shards2 = mem3_util:create_partition_map("foo",1,1,["node1"]), -%% Counters2 = dict:to_list( -%% group_idrevs_by_shard_hack(<<"foo">>, Shards2, UUIDsIDdsRevs)), -%% AccW4 = {length(Shards2), length(UUIDsIDdsRevs), list_to_integer("2"), -%% Counters2, DocsDict}, -%% Bool = -%% case handle_message({ok, [{ok, Revs1}, {ok, Revs2}]}, -%% hd(Shards), AccW4) of -%% {stop, _Reply} -> -%% true; -%% _ -> false -%% end, -%% ?assertEqual(true, Bool), -%% -%% % *** test Docs with no replies should end up as {error, internal_server_error} -%% SA1 = #shard{node = a, range = [1]}, -%% SA2 = #shard{node = a, range = [2]}, -%% SB1 = #shard{node = b, range = [1]}, -%% SB2 = #shard{node = b, range = [2]}, -%% Counters3 = [{SA1,[UUID1]}, {SB1,[UUID1]}, -%% {SA2,[UUID2]}, {SB2,[UUID2]}], -%% Acc30 = {length(Counters3), length(UUIDsIDdsRevs), 2, Counters3, DocsDict}, -%% {ok, Acc31} = handle_message({ok, [{ok, Revs1}]}, SA1, Acc30), -%% {ok, Acc32} = handle_message({rexi_EXIT, nil}, SB1, Acc31), -%% {ok, Acc33} = handle_message({rexi_EXIT, nil}, SA2, Acc32), -%% {stop, Acc34} = handle_message({rexi_EXIT, nil}, SB2, Acc33), -%% ?assertEqual( -%% {error, [{UUID1, {accepted, Revs1}}, -%% {UUID2, {error, internal_server_error}}]}, -%% Acc34 -%% ), -%% meck:unload(couch_log). -%% -%% -%% % needed for testing to avoid having to start the mem3 application -%% group_idrevs_by_shard_hack(_DbName, Shards, UUIDsIdsRevs) -> -%% lists:foldl(fun({UUID, _Id, _Revs}, Dict0) -> -%% lists:foldl(fun(Shard, Dict1) -> -%% dict:append(Shard, UUID, Dict1) -%% end, Dict0, Shards) -%% end, dict:new(), UUIDsIdsRevs). +resp_health(Resps) -> + Healths = lists:usort([H || {H, _} <- Resps]), + HasError = lists:member(error, Healths), + HasAccepted = lists:member(accepted, Healths), + AllOk = Healths == [ok], + if + HasError -> error; + HasAccepted -> accepted; + AllOk -> ok; + true -> error + end. + + +has_quorum(Resps, Count, W) -> + OkResps = [R || {ok, _} = R <- Resps], + OkCounts = lists:foldl(fun(R, Acc) -> + dict:update_counter(R, 1, Acc) + end, dict:new(), OkResps), + MaxOk = lists:max([0 | element(2, lists:unzip(dict:to_list(OkCounts)))]), + if + MaxOk >= W -> true; + length(Resps) >= Count -> true; + true -> false + end. + + +-ifdef(TEST). + +-include_lib("eunit/include/eunit.hrl"). + +purge_test_() -> + { + foreach, + fun setup/0, + fun teardown/1, + [ + t_w2_ok(), + t_w3_ok(), + + t_w2_mixed_accepted(), + t_w3_mixed_accepted(), + + t_w2_exit1_ok(), + t_w2_exit2_accepted(), + t_w2_exit3_error(), + + t_w4_accepted(), + + t_mixed_ok_accepted(), + t_mixed_errors() + ] + }. + + +setup() -> + meck:new(couch_log), + meck:expect(couch_log, warning, fun(_, _) -> ok end), + meck:expect(couch_log, notice, fun(_, _) -> ok end). + + +teardown(_) -> + meck:unload(). + + +t_w2_ok() -> + ?_test(begin + Acc0 = create_init_acc(2), + Msg = {ok, [{ok, [{1, <<"foo">>}]}, {ok, [{2, <<"bar">>}]}]}, + + {ok, Acc1} = handle_message(Msg, worker(1, Acc0), Acc0), + ?assertEqual(2, length(Acc1#acc.worker_uuids)), + check_quorum(Acc1, false), + + {stop, Acc2} = handle_message(Msg, worker(2, Acc0), Acc1), + ?assertEqual(1, length(Acc2#acc.worker_uuids)), + check_quorum(Acc2, true), + + Expect = [{ok, [{1, <<"foo">>}]}, {ok, [{2, <<"bar">>}]}], + Resps = format_resps([<<"uuid1">>, <<"uuid2">>], Acc2), + ?assertEqual(Expect, Resps), + ?assertEqual(ok, resp_health(Resps)) + end). + + +t_w3_ok() -> + ?_test(begin + Acc0 = create_init_acc(3), + Msg = {ok, [{ok, [{1, <<"foo">>}]}, {ok, [{2, <<"bar">>}]}]}, + + {ok, Acc1} = handle_message(Msg, worker(1, Acc0), Acc0), + check_quorum(Acc1, false), + + {ok, Acc2} = handle_message(Msg, worker(2, Acc0), Acc1), + ?assertEqual(1, length(Acc2#acc.worker_uuids)), + check_quorum(Acc2, false), + + {stop, Acc3} = handle_message(Msg, worker(3, Acc0), Acc2), + ?assertEqual(0, length(Acc3#acc.worker_uuids)), + check_quorum(Acc3, true), + + Expect = [{ok, [{1, <<"foo">>}]}, {ok, [{2, <<"bar">>}]}], + Resps = format_resps([<<"uuid1">>, <<"uuid2">>], Acc3), + ?assertEqual(Expect, Resps), + ?assertEqual(ok, resp_health(Resps)) + end). + + +t_w2_mixed_accepted() -> + ?_test(begin + Acc0 = create_init_acc(2), + Msg1 = {ok, [{ok, [{1, <<"foo1">>}]}, {ok, [{2, <<"bar1">>}]}]}, + Msg2 = {ok, [{ok, [{1, <<"foo2">>}]}, {ok, [{2, <<"bar2">>}]}]}, + + {ok, Acc1} = handle_message(Msg1, worker(1, Acc0), Acc0), + ?assertEqual(2, length(Acc1#acc.worker_uuids)), + check_quorum(Acc1, false), + + {ok, Acc2} = handle_message(Msg2, worker(2, Acc0), Acc1), + ?assertEqual(1, length(Acc2#acc.worker_uuids)), + check_quorum(Acc2, false), + + {stop, Acc3} = handle_message(Msg1, worker(3, Acc0), Acc2), + ?assertEqual(0, length(Acc3#acc.worker_uuids)), + check_quorum(Acc3, true), + + Expect = [ + {accepted, [{1, <<"foo1">>}, {1, <<"foo2">>}]}, + {accepted, [{2, <<"bar1">>}, {2, <<"bar2">>}]} + ], + Resps = format_resps([<<"uuid1">>, <<"uuid2">>], Acc2), + ?assertEqual(Expect, Resps), + ?assertEqual(accepted, resp_health(Resps)) + end). + + +t_w3_mixed_accepted() -> + ?_test(begin + Acc0 = create_init_acc(3), + Msg1 = {ok, [{ok, [{1, <<"foo1">>}]}, {ok, [{2, <<"bar1">>}]}]}, + Msg2 = {ok, [{ok, [{1, <<"foo2">>}]}, {ok, [{2, <<"bar2">>}]}]}, + + {ok, Acc1} = handle_message(Msg1, worker(1, Acc0), Acc0), + ?assertEqual(2, length(Acc1#acc.worker_uuids)), + check_quorum(Acc1, false), + + {ok, Acc2} = handle_message(Msg2, worker(2, Acc0), Acc1), + ?assertEqual(1, length(Acc2#acc.worker_uuids)), + check_quorum(Acc2, false), + + {stop, Acc3} = handle_message(Msg2, worker(3, Acc0), Acc2), + ?assertEqual(0, length(Acc3#acc.worker_uuids)), + check_quorum(Acc3, true), + + Expect = [ + {accepted, [{1, <<"foo1">>}, {1, <<"foo2">>}]}, + {accepted, [{2, <<"bar1">>}, {2, <<"bar2">>}]} + ], + Resps = format_resps([<<"uuid1">>, <<"uuid2">>], Acc2), + ?assertEqual(Expect, Resps), + ?assertEqual(accepted, resp_health(Resps)) + end). + + +t_w2_exit1_ok() -> + ?_test(begin + Acc0 = create_init_acc(2), + Msg = {ok, [{ok, [{1, <<"foo">>}]}, {ok, [{2, <<"bar">>}]}]}, + ExitMsg = {rexi_EXIT, blargh}, + + {ok, Acc1} = handle_message(Msg, worker(1, Acc0), Acc0), + ?assertEqual(2, length(Acc1#acc.worker_uuids)), + check_quorum(Acc1, false), + + {ok, Acc2} = handle_message(ExitMsg, worker(2, Acc0), Acc1), + ?assertEqual(1, length(Acc2#acc.worker_uuids)), + check_quorum(Acc2, false), + + {stop, Acc3} = handle_message(Msg, worker(3, Acc0), Acc2), + ?assertEqual(0, length(Acc3#acc.worker_uuids)), + check_quorum(Acc3, true), + + Expect = [{ok, [{1, <<"foo">>}]}, {ok, [{2, <<"bar">>}]}], + Resps = format_resps([<<"uuid1">>, <<"uuid2">>], Acc3), + ?assertEqual(Expect, Resps), + ?assertEqual(ok, resp_health(Resps)) + end). + + +t_w2_exit2_accepted() -> + ?_test(begin + Acc0 = create_init_acc(2), + Msg = {ok, [{ok, [{1, <<"foo">>}]}, {ok, [{2, <<"bar">>}]}]}, + ExitMsg = {rexi_EXIT, blargh}, + + {ok, Acc1} = handle_message(Msg, worker(1, Acc0), Acc0), + ?assertEqual(2, length(Acc1#acc.worker_uuids)), + check_quorum(Acc1, false), + + {ok, Acc2} = handle_message(ExitMsg, worker(2, Acc0), Acc1), + ?assertEqual(1, length(Acc2#acc.worker_uuids)), + check_quorum(Acc2, false), + + {stop, Acc3} = handle_message(ExitMsg, worker(3, Acc0), Acc2), + ?assertEqual(0, length(Acc3#acc.worker_uuids)), + check_quorum(Acc3, true), + + Expect = [{accepted, [{1, <<"foo">>}]}, {accepted, [{2, <<"bar">>}]}], + Resps = format_resps([<<"uuid1">>, <<"uuid2">>], Acc3), + ?assertEqual(Expect, Resps), + ?assertEqual(accepted, resp_health(Resps)) + end). + + +t_w2_exit3_error() -> + ?_test(begin + Acc0 = create_init_acc(2), + ExitMsg = {rexi_EXIT, blargh}, + + {ok, Acc1} = handle_message(ExitMsg, worker(1, Acc0), Acc0), + ?assertEqual(2, length(Acc1#acc.worker_uuids)), + check_quorum(Acc1, false), + + {ok, Acc2} = handle_message(ExitMsg, worker(2, Acc0), Acc1), + ?assertEqual(1, length(Acc2#acc.worker_uuids)), + check_quorum(Acc2, false), + + {stop, Acc3} = handle_message(ExitMsg, worker(3, Acc0), Acc2), + ?assertEqual(0, length(Acc3#acc.worker_uuids)), + check_quorum(Acc3, true), + + Expect = [ + {error, internal_server_error}, + {error, internal_server_error} + ], + Resps = format_resps([<<"uuid1">>, <<"uuid2">>], Acc3), + ?assertEqual(Expect, Resps), + ?assertEqual(error, resp_health(Resps)) + end). + + +t_w4_accepted() -> + % Make sure we return when all workers have responded + % rather than wait around for a timeout if a user asks + % for a qourum with more than the available number of + % shards. + ?_test(begin + Acc0 = create_init_acc(4), + Msg = {ok, [{ok, [{1, <<"foo">>}]}, {ok, [{2, <<"bar">>}]}]}, + + {ok, Acc1} = handle_message(Msg, worker(1, Acc0), Acc0), + ?assertEqual(2, length(Acc1#acc.worker_uuids)), + check_quorum(Acc1, false), + + {ok, Acc2} = handle_message(Msg, worker(2, Acc0), Acc1), + ?assertEqual(1, length(Acc2#acc.worker_uuids)), + check_quorum(Acc2, false), + + {stop, Acc3} = handle_message(Msg, worker(3, Acc0), Acc2), + ?assertEqual(0, length(Acc3#acc.worker_uuids)), + check_quorum(Acc3, true), + + Expect = [{accepted, [{1, <<"foo">>}]}, {accepted, [{2, <<"bar">>}]}], + Resps = format_resps([<<"uuid1">>, <<"uuid2">>], Acc3), + ?assertEqual(Expect, Resps), + ?assertEqual(accepted, resp_health(Resps)) + end). + + +t_mixed_ok_accepted() -> + ?_test(begin + WorkerUUIDs = [ + {#shard{node = a, range = [1, 2]}, [<<"uuid1">>]}, + {#shard{node = b, range = [1, 2]}, [<<"uuid1">>]}, + {#shard{node = c, range = [1, 2]}, [<<"uuid1">>]}, + + {#shard{node = a, range = [3, 4]}, [<<"uuid2">>]}, + {#shard{node = b, range = [3, 4]}, [<<"uuid2">>]}, + {#shard{node = c, range = [3, 4]}, [<<"uuid2">>]} + ], + + Acc0 = #acc{ + worker_uuids = WorkerUUIDs, + resps = dict:from_list([{<<"uuid1">>, []}, {<<"uuid2">>, []}]), + uuid_counts = dict:from_list([{<<"uuid1">>, 3}, {<<"uuid2">>, 3}]), + w = 2 + }, + + Msg1 = {ok, [{ok, [{1, <<"foo">>}]}]}, + Msg2 = {ok, [{ok, [{2, <<"bar">>}]}]}, + ExitMsg = {rexi_EXIT, blargh}, + + {ok, Acc1} = handle_message(Msg1, worker(1, Acc0), Acc0), + {ok, Acc2} = handle_message(Msg1, worker(2, Acc0), Acc1), + {ok, Acc3} = handle_message(ExitMsg, worker(4, Acc0), Acc2), + {ok, Acc4} = handle_message(ExitMsg, worker(5, Acc0), Acc3), + {stop, Acc5} = handle_message(Msg2, worker(6, Acc0), Acc4), + + Expect = [{ok, [{1, <<"foo">>}]}, {accepted, [{2, <<"bar">>}]}], + Resps = format_resps([<<"uuid1">>, <<"uuid2">>], Acc5), + ?assertEqual(Expect, Resps), + ?assertEqual(accepted, resp_health(Resps)) + end). + + +t_mixed_errors() -> + ?_test(begin + WorkerUUIDs = [ + {#shard{node = a, range = [1, 2]}, [<<"uuid1">>]}, + {#shard{node = b, range = [1, 2]}, [<<"uuid1">>]}, + {#shard{node = c, range = [1, 2]}, [<<"uuid1">>]}, + + {#shard{node = a, range = [3, 4]}, [<<"uuid2">>]}, + {#shard{node = b, range = [3, 4]}, [<<"uuid2">>]}, + {#shard{node = c, range = [3, 4]}, [<<"uuid2">>]} + ], + + Acc0 = #acc{ + worker_uuids = WorkerUUIDs, + resps = dict:from_list([{<<"uuid1">>, []}, {<<"uuid2">>, []}]), + uuid_counts = dict:from_list([{<<"uuid1">>, 3}, {<<"uuid2">>, 3}]), + w = 2 + }, + + Msg = {ok, [{ok, [{1, <<"foo">>}]}]}, + ExitMsg = {rexi_EXIT, blargh}, + + {ok, Acc1} = handle_message(Msg, worker(1, Acc0), Acc0), + {ok, Acc2} = handle_message(Msg, worker(2, Acc0), Acc1), + {ok, Acc3} = handle_message(ExitMsg, worker(4, Acc0), Acc2), + {ok, Acc4} = handle_message(ExitMsg, worker(5, Acc0), Acc3), + {stop, Acc5} = handle_message(ExitMsg, worker(6, Acc0), Acc4), + + Expect = [{ok, [{1, <<"foo">>}]}, {error, internal_server_error}], + Resps = format_resps([<<"uuid1">>, <<"uuid2">>], Acc5), + ?assertEqual(Expect, Resps), + ?assertEqual(error, resp_health(Resps)) + end). + + +create_init_acc(W) -> + UUID1 = <<"uuid1">>, + UUID2 = <<"uuid2">>, + + Nodes = [node1, node2, node3], + Shards = mem3_util:create_partition_map(<<"foo">>, 3, 1, Nodes), + + % Create our worker_uuids. We're relying on the fact that + % we're using a fake Q=1 db so we don't have to worry + % about any hashing here. + WorkerUUIDs = lists:map(fun(Shard) -> + {Shard#shard{ref = erlang:make_ref()}, [UUID1, UUID2]} + end, Shards), + + #acc{ + worker_uuids = WorkerUUIDs, + resps = dict:from_list([{UUID1, []}, {UUID2, []}]), + uuid_counts = dict:from_list([{UUID1, 3}, {UUID2, 3}]), + w = W + }. + + +worker(N, #acc{worker_uuids = WorkerUUIDs}) -> + {Worker, _} = lists:nth(N, WorkerUUIDs), + Worker. + + +check_quorum(Acc, Expect) -> + dict:fold(fun(_Shard, Resps, _) -> + ?assertEqual(Expect, has_quorum(Resps, 3, Acc#acc.w)) + end, nil, Acc#acc.resps). + +-endif. diff --git a/src/fabric/src/fabric_util.erl b/src/fabric/src/fabric_util.erl index dd4b80da6..49f4c8913 100644 --- a/src/fabric/src/fabric_util.erl +++ b/src/fabric/src/fabric_util.erl @@ -194,11 +194,9 @@ get_shard([#shard{node = Node, name = Name} | Rest], Opts, Timeout, Factor) -> throw(Error); {Ref, {'rexi_EXIT', {{forbidden, _} = Error, _}}} -> throw(Error); - {Ref, Reason} -> - couch_log:debug("Failed to open shard ~p because: ~p", [Name, Reason]), + {Ref, _Else} -> get_shard(Rest, Opts, Timeout, Factor) after Timeout -> - couch_log:debug("Failed to open shard ~p after: ~p", [Name, Timeout]), get_shard(Rest, Opts, Factor * Timeout, Factor) end after |