diff options
author | Nick Vatamaniuc <vatamane@apache.org> | 2020-03-17 12:50:20 -0400 |
---|---|---|
committer | Nick Vatamaniuc <nickva@users.noreply.github.com> | 2020-03-17 13:12:27 -0400 |
commit | 28ffba4611934cabcf68929abcf32656d60b91c0 (patch) | |
tree | 3fbfb2986c070e504dc7f6631f83129514743b5b | |
parent | 0cafb178488e77e94b29607d278aae22ffb98187 (diff) | |
download | couchdb-28ffba4611934cabcf68929abcf32656d60b91c0.tar.gz |
Handle transaction timeouts in _all_docs
Previously transactions could time-out, retry and re-emit the same data. Use
the same mechanism as the _list_dbs and _changes feeds to fix it. Additional
detail in the mailing list discussion:
https://lists.apache.org/thread.html/r02cee7045cac4722e1682bb69ba0ec791f5cce025597d0099fb34033%40%3Cdev.couchdb.apache.org%3E
-rw-r--r-- | src/chttpd/src/chttpd_db.erl | 3 | ||||
-rw-r--r-- | src/fabric/test/fabric2_doc_fold_tests.erl | 117 |
2 files changed, 90 insertions, 30 deletions
diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl index 50a9effdb..b7a149b09 100644 --- a/src/chttpd/src/chttpd_db.erl +++ b/src/chttpd/src/chttpd_db.erl @@ -867,7 +867,8 @@ all_docs_view(Req, Db, Keys, OP) -> send_all_docs(Db, #mrargs{keys = undefined} = Args, VAcc0) -> - Opts = all_docs_view_opts(Args), + Opts0 = all_docs_view_opts(Args), + Opts = Opts0 ++ [{restart_tx, true}], NS = couch_util:get_value(namespace, Opts), FoldFun = case NS of <<"_all_docs">> -> fold_docs; diff --git a/src/fabric/test/fabric2_doc_fold_tests.erl b/src/fabric/test/fabric2_doc_fold_tests.erl index 6262a10fe..7c95dd372 100644 --- a/src/fabric/test/fabric2_doc_fold_tests.erl +++ b/src/fabric/test/fabric2_doc_fold_tests.erl @@ -27,25 +27,38 @@ doc_fold_test_() -> "Test document fold operations", { setup, - fun setup/0, - fun cleanup/1, - with([ - ?TDEF(fold_docs_basic), - ?TDEF(fold_docs_rev), - ?TDEF(fold_docs_with_start_key), - ?TDEF(fold_docs_with_end_key), - ?TDEF(fold_docs_with_both_keys_the_same), - ?TDEF(fold_docs_with_different_keys, 10000), - ?TDEF(fold_docs_with_limit), - ?TDEF(fold_docs_with_skip), - ?TDEF(fold_docs_with_skip_and_limit) - ]) + fun setup_all/0, + fun teardown_all/1, + { + foreach, + fun setup/0, + fun cleanup/1, + [ + ?TDEF_FE(fold_docs_basic), + ?TDEF_FE(fold_docs_rev), + ?TDEF_FE(fold_docs_with_start_key), + ?TDEF_FE(fold_docs_with_end_key), + ?TDEF_FE(fold_docs_with_both_keys_the_same), + ?TDEF_FE(fold_docs_with_different_keys, 10000), + ?TDEF_FE(fold_docs_with_limit), + ?TDEF_FE(fold_docs_with_skip), + ?TDEF_FE(fold_docs_with_skip_and_limit), + ?TDEF_FE(fold_docs_tx_too_old) + ] + } } }. +setup_all() -> + test_util:start_couch([fabric]). + + +teardown_all(Ctx) -> + test_util:stop_couch(Ctx). + + setup() -> - Ctx = test_util:start_couch([fabric]), {ok, Db} = fabric2_db:create(?tempdb(), [{user_ctx, ?ADMIN_USER}]), DocIdRevs = lists:map(fun(Val) -> DocId = fabric2_util:uuid(), @@ -56,38 +69,39 @@ setup() -> {ok, Rev} = fabric2_db:update_doc(Db, Doc, []), {DocId, {[{rev, couch_doc:rev_to_str(Rev)}]}} end, lists:seq(1, ?DOC_COUNT)), - {Db, lists:sort(DocIdRevs), Ctx}. + fabric2_test_util:tx_too_old_mock_erlfdb(), + {Db, lists:sort(DocIdRevs)}. -cleanup({Db, _DocIdRevs, Ctx}) -> - ok = fabric2_db:delete(fabric2_db:name(Db), []), - test_util:stop_couch(Ctx). +cleanup({Db, _DocIdRevs}) -> + fabric2_test_util:tx_too_old_reset_errors(), + ok = fabric2_db:delete(fabric2_db:name(Db), []). -fold_docs_basic({Db, DocIdRevs, _}) -> +fold_docs_basic({Db, DocIdRevs}) -> {ok, {?DOC_COUNT, Rows}} = fabric2_db:fold_docs(Db, fun fold_fun/2, []), ?assertEqual(DocIdRevs, lists:reverse(Rows)). -fold_docs_rev({Db, DocIdRevs, _}) -> +fold_docs_rev({Db, DocIdRevs}) -> Opts = [{dir, rev}], {ok, {?DOC_COUNT, Rows}} = fabric2_db:fold_docs(Db, fun fold_fun/2, [], Opts), ?assertEqual(DocIdRevs, Rows). -fold_docs_with_start_key({Db, DocIdRevs, _}) -> +fold_docs_with_start_key({Db, DocIdRevs}) -> {StartKey, _} = hd(DocIdRevs), Opts = [{start_key, StartKey}], {ok, {?DOC_COUNT, Rows}} = fabric2_db:fold_docs(Db, fun fold_fun/2, [], Opts), ?assertEqual(DocIdRevs, lists:reverse(Rows)), if length(DocIdRevs) == 1 -> ok; true -> - fold_docs_with_start_key({Db, tl(DocIdRevs), nil}) + fold_docs_with_start_key({Db, tl(DocIdRevs)}) end. -fold_docs_with_end_key({Db, DocIdRevs, _}) -> +fold_docs_with_end_key({Db, DocIdRevs}) -> RevDocIdRevs = lists:reverse(DocIdRevs), {EndKey, _} = hd(RevDocIdRevs), Opts = [{end_key, EndKey}], @@ -95,24 +109,24 @@ fold_docs_with_end_key({Db, DocIdRevs, _}) -> fabric2_db:fold_docs(Db, fun fold_fun/2, [], Opts), ?assertEqual(RevDocIdRevs, Rows), if length(DocIdRevs) == 1 -> ok; true -> - fold_docs_with_end_key({Db, lists:reverse(tl(RevDocIdRevs)), nil}) + fold_docs_with_end_key({Db, lists:reverse(tl(RevDocIdRevs))}) end. -fold_docs_with_both_keys_the_same({Db, DocIdRevs, _}) -> +fold_docs_with_both_keys_the_same({Db, DocIdRevs}) -> lists:foreach(fun({DocId, _} = Row) -> check_all_combos(Db, DocId, DocId, [Row]) end, DocIdRevs). -fold_docs_with_different_keys({Db, DocIdRevs, _}) -> +fold_docs_with_different_keys({Db, DocIdRevs}) -> lists:foreach(fun(_) -> {StartKey, EndKey, Rows} = pick_range(DocIdRevs), check_all_combos(Db, StartKey, EndKey, Rows) end, lists:seq(1, 500)). -fold_docs_with_limit({Db, DocIdRevs, _}) -> +fold_docs_with_limit({Db, DocIdRevs}) -> lists:foreach(fun(Limit) -> Opts1 = [{limit, Limit}], {ok, {?DOC_COUNT, Rows1}} = @@ -129,7 +143,7 @@ fold_docs_with_limit({Db, DocIdRevs, _}) -> end, lists:seq(0, 51)). -fold_docs_with_skip({Db, DocIdRevs, _}) -> +fold_docs_with_skip({Db, DocIdRevs}) -> lists:foreach(fun(Skip) -> Opts1 = [{skip, Skip}], {ok, {?DOC_COUNT, Rows1}} = @@ -151,13 +165,57 @@ fold_docs_with_skip({Db, DocIdRevs, _}) -> end, lists:seq(0, 51)). -fold_docs_with_skip_and_limit({Db, DocIdRevs, _}) -> +fold_docs_with_skip_and_limit({Db, DocIdRevs}) -> lists:foreach(fun(_) -> check_skip_and_limit(Db, [], DocIdRevs), check_skip_and_limit(Db, [{dir, rev}], lists:reverse(DocIdRevs)) end, lists:seq(1, 100)). +fold_docs_tx_too_old({Db, _DocIdRevs}) -> + {ok, Expected} = fabric2_db:fold_docs(Db, fun fold_fun/2, []), + + FoldDocsFun = fun() -> + fabric2_db:fold_docs(Db, fun fold_fun/2, [], [{restart_tx, true}]) + end, + + % Blow up in fold range on the first call + fabric2_test_util:tx_too_old_setup_errors(0, 1), + ?assertEqual({ok, Expected}, FoldDocsFun()), + + % Blow up in fold_range after emitting one row + fabric2_test_util:tx_too_old_setup_errors(0, {1, 1}), + ?assertEqual({ok, Expected}, FoldDocsFun()), + + % Blow up in fold_range after emitting 48 rows + fabric2_test_util:tx_too_old_setup_errors(0, {?DOC_COUNT - 2, 1}), + ?assertEqual({ok, Expected}, FoldDocsFun()), + + % Blow up in fold_range after emitting 49 rows + fabric2_test_util:tx_too_old_setup_errors(0, {?DOC_COUNT - 1, 1}), + ?assertEqual({ok, Expected}, FoldDocsFun()), + + % Blow up in user fun + fabric2_test_util:tx_too_old_setup_errors(1, 0), + ?assertEqual({ok, Expected}, FoldDocsFun()), + + % Blow up in user fun after emitting one row + fabric2_test_util:tx_too_old_setup_errors({1, 1}, 0), + ?assertEqual({ok, Expected}, FoldDocsFun()), + + % Blow up in user fun after emitting 48 rows + fabric2_test_util:tx_too_old_setup_errors({?DOC_COUNT - 2, 1}, 0), + ?assertEqual({ok, Expected}, FoldDocsFun()), + + % Blow up in user fun after emitting 49 rows + fabric2_test_util:tx_too_old_setup_errors({?DOC_COUNT - 1, 1}, 0), + ?assertEqual({ok, Expected}, FoldDocsFun()), + + % Blow up in in user fun and fold range + fabric2_test_util:tx_too_old_setup_errors(1, {1, 1}), + ?assertEqual({ok, Expected}, FoldDocsFun()). + + check_all_combos(Db, StartKey, EndKey, Rows) -> Opts1 = make_opts(fwd, StartKey, EndKey, true), {ok, {?DOC_COUNT, Rows1}} = @@ -280,6 +338,7 @@ fold_fun({meta, Meta}, _Acc) -> Total = fabric2_util:get_value(total, Meta), {ok, {Total, []}}; fold_fun({row, Row}, {Total, Rows}) -> + fabric2_test_util:tx_too_old_raise_in_user_fun(), RowId = fabric2_util:get_value(id, Row), RowId = fabric2_util:get_value(key, Row), RowRev = fabric2_util:get_value(value, Row), |