diff options
author | Ronny Berndt <ronny@apache.org> | 2023-01-20 09:43:12 +0100 |
---|---|---|
committer | Ronny Berndt <ronny@apache.org> | 2023-01-20 10:00:37 +0100 |
commit | cb259f93fdc75d5aa72b48503cbe5afcff579e7c (patch) | |
tree | 13e5e224e9e46aea6188475d427e2156ae2cd09e | |
parent | d3de330ff9667d21e79e2f00aa7ba4ff249ce256 (diff) | |
parent | 3b6ebdeb3b5d10f03677b5f2c38c21c6c144a2bd (diff) | |
download | couchdb-jenkins-add-windows-pipeline.tar.gz |
Merge branch 'main' into jenkins-add-windows-pipelinejenkins-add-windows-pipeline
-rw-r--r-- | Makefile.win | 5 | ||||
-rw-r--r-- | build-aux/Jenkinsfile.full | 2 | ||||
-rw-r--r-- | rebar.config.script | 2 | ||||
-rw-r--r-- | src/couch_replicator/src/couch_replicator_worker.erl | 22 | ||||
-rw-r--r-- | src/couch_replicator/test/eunit/couch_replicator_bulk_get_tests.erl | 79 | ||||
-rw-r--r-- | src/docs/src/api/database/find.rst | 4 | ||||
-rw-r--r-- | src/mango/README.md | 2 |
7 files changed, 103 insertions, 13 deletions
diff --git a/Makefile.win b/Makefile.win index 0a2f6513c..d24634ce7 100644 --- a/Makefile.win +++ b/Makefile.win @@ -250,6 +250,11 @@ elixir-check-formatted: elixir-init elixir-credo: elixir-init @mix credo +.PHONY: build-report +# target: build-report - Generate a build report +build-report: + @$(PYTHON) build-aux/show-test-results.py --suites=10 --tests=10 > test-results.log + .PHONY: check-qs # target: check-qs - Run query server tests (ruby and rspec required!) check-qs: diff --git a/build-aux/Jenkinsfile.full b/build-aux/Jenkinsfile.full index 59270c5eb..8c0e7cb4d 100644 --- a/build-aux/Jenkinsfile.full +++ b/build-aux/Jenkinsfile.full @@ -188,7 +188,7 @@ def generateNativeStage(platform) { dir( "${platform}/build" ) { powershell( script: 'Get-ChildItem') powershell( script: '& ..\\..\\couchdb-glazier\\bin\\shell.ps1; make -f Makefile.win build-report') - powershell( script: 'Get-Content .\test-results.log') + powershell( script: 'Get-Content test-results.log') } error("Build step failed with error: ${err.getMessage()}") } diff --git a/rebar.config.script b/rebar.config.script index 028aabe8f..2bdb17a49 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -156,7 +156,7 @@ DepDescs = [ {jiffy, "jiffy", {tag, "CouchDB-1.0.9-2"}}, {mochiweb, "mochiweb", {tag, "v3.1.1"}}, {meck, "meck", {tag, "0.9.2"}}, -{recon, "recon", {tag, "2.5.2"}} +{recon, "recon", {tag, "2.5.3"}} ]. WithProper = lists:keyfind(with_proper, 1, CouchConfig) == {with_proper, true}. diff --git a/src/couch_replicator/src/couch_replicator_worker.erl b/src/couch_replicator/src/couch_replicator_worker.erl index d8f872388..46e4a6e94 100644 --- a/src/couch_replicator/src/couch_replicator_worker.erl +++ b/src/couch_replicator/src/couch_replicator_worker.erl @@ -297,17 +297,25 @@ queue_fetch_loop(#fetch_st{} = St) -> {changes, ChangesManager, Changes, ReportSeq} -> % Find missing revisions (POST to _revs_diff) {IdRevs, RdSt1} = find_missing(Changes, Target, Parent, RdSt), - {Docs, BgSt1} = bulk_get(UseBulkGet, Source, IdRevs, Parent, BgSt), - % Documents without attachments can be uploaded right away - BatchFun = fun({_, #doc{} = Doc}) -> - ok = gen_server:call(Parent, {batch_doc, Doc}, infinity) + % Filter out and handle design docs individually + DDocFilter = fun + ({<<?DESIGN_DOC_PREFIX, _/binary>>, _Rev}, _PAs) -> true; + ({_Id, _Rev}, _PAs) -> false end, - lists:foreach(BatchFun, lists:sort(maps:to_list(Docs))), - % Fetch individually if _bulk_get failed or there are attachments + DDocIdRevs = maps:filter(DDocFilter, IdRevs), FetchFun = fun({Id, Rev}, PAs) -> ok = gen_server:call(Parent, {fetch_doc, {Id, [Rev], PAs}}, infinity) end, - maps:map(FetchFun, maps:without(maps:keys(Docs), IdRevs)), + maps:map(FetchFun, DDocIdRevs), + % IdRevs1 is all the docs without design docs. Bulk get those. + IdRevs1 = maps:without(maps:keys(DDocIdRevs), IdRevs), + {Docs, BgSt1} = bulk_get(UseBulkGet, Source, IdRevs1, Parent, BgSt), + BatchFun = fun({_, #doc{} = Doc}) -> + ok = gen_server:call(Parent, {batch_doc, Doc}, infinity) + end, + lists:foreach(BatchFun, lists:sort(maps:to_list(Docs))), + % Invidually upload docs with attachments. + maps:map(FetchFun, maps:without(maps:keys(Docs), IdRevs1)), {ok, Stats} = gen_server:call(Parent, flush, infinity), ok = report_seq_done(Cp, ReportSeq, Stats), couch_log:debug("Worker reported completion of seq ~p", [ReportSeq]), diff --git a/src/couch_replicator/test/eunit/couch_replicator_bulk_get_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_bulk_get_tests.erl index 2ecd0f4ee..f0d9569db 100644 --- a/src/couch_replicator/test/eunit/couch_replicator_bulk_get_tests.erl +++ b/src/couch_replicator/test/eunit/couch_replicator_bulk_get_tests.erl @@ -26,7 +26,11 @@ bulk_get_test_() -> fun couch_replicator_test_helper:test_teardown/1, [ ?TDEF_FE(use_bulk_get), + ?TDEF_FE(use_bulk_get_with_ddocs), + ?TDEF_FE(use_bulk_get_with_attachments), ?TDEF_FE(dont_use_bulk_get), + ?TDEF_FE(dont_use_bulk_get_ddocs), + ?TDEF_FE(dont_use_bulk_get_attachments), ?TDEF_FE(job_enable_overrides_global_disable), ?TDEF_FE(global_disable_works) ] @@ -39,7 +43,33 @@ use_bulk_get({_Ctx, {Source, Target}}) -> replicate(Source, Target, true), BulkGets = meck:num_calls(couch_replicator_api_wrap, bulk_get, 3), JustGets = meck:num_calls(couch_replicator_api_wrap, open_doc_revs, 6), + DocUpdates = meck:num_calls(couch_replicator_api_wrap, update_doc, 4), ?assertEqual(0, JustGets), + ?assertEqual(0, DocUpdates), + ?assert(BulkGets >= 1), + compare_dbs(Source, Target). + +use_bulk_get_with_ddocs({_Ctx, {Source, Target}}) -> + populate_db_ddocs(Source, ?DOC_COUNT), + meck:new(couch_replicator_api_wrap, [passthrough]), + replicate(Source, Target, true), + BulkGets = meck:num_calls(couch_replicator_api_wrap, bulk_get, 3), + JustGets = meck:num_calls(couch_replicator_api_wrap, open_doc_revs, 6), + DocUpdates = meck:num_calls(couch_replicator_api_wrap, update_doc, 4), + ?assertEqual(?DOC_COUNT, JustGets), + ?assertEqual(?DOC_COUNT, DocUpdates), + ?assert(BulkGets >= 1), + compare_dbs(Source, Target). + +use_bulk_get_with_attachments({_Ctx, {Source, Target}}) -> + populate_db_atts(Source, ?DOC_COUNT), + meck:new(couch_replicator_api_wrap, [passthrough]), + replicate(Source, Target, true), + BulkGets = meck:num_calls(couch_replicator_api_wrap, bulk_get, 3), + JustGets = meck:num_calls(couch_replicator_api_wrap, open_doc_revs, 6), + DocUpdates = meck:num_calls(couch_replicator_api_wrap, update_doc, 4), + ?assertEqual(?DOC_COUNT, JustGets), + ?assertEqual(?DOC_COUNT, DocUpdates), ?assert(BulkGets >= 1), compare_dbs(Source, Target). @@ -49,10 +79,36 @@ dont_use_bulk_get({_Ctx, {Source, Target}}) -> replicate(Source, Target, false), BulkGets = meck:num_calls(couch_replicator_api_wrap, bulk_get, 3), JustGets = meck:num_calls(couch_replicator_api_wrap, open_doc_revs, 6), + DocUpdates = meck:num_calls(couch_replicator_api_wrap, update_doc, 4), ?assertEqual(0, BulkGets), + ?assertEqual(0, DocUpdates), ?assertEqual(?DOC_COUNT, JustGets), compare_dbs(Source, Target). +dont_use_bulk_get_ddocs({_Ctx, {Source, Target}}) -> + populate_db_ddocs(Source, ?DOC_COUNT), + meck:new(couch_replicator_api_wrap, [passthrough]), + replicate(Source, Target, false), + BulkGets = meck:num_calls(couch_replicator_api_wrap, bulk_get, 3), + JustGets = meck:num_calls(couch_replicator_api_wrap, open_doc_revs, 6), + DocUpdates = meck:num_calls(couch_replicator_api_wrap, update_doc, 4), + ?assertEqual(0, BulkGets), + ?assertEqual(?DOC_COUNT, JustGets), + ?assertEqual(?DOC_COUNT, DocUpdates), + compare_dbs(Source, Target). + +dont_use_bulk_get_attachments({_Ctx, {Source, Target}}) -> + populate_db_atts(Source, ?DOC_COUNT), + meck:new(couch_replicator_api_wrap, [passthrough]), + replicate(Source, Target, false), + BulkGets = meck:num_calls(couch_replicator_api_wrap, bulk_get, 3), + JustGets = meck:num_calls(couch_replicator_api_wrap, open_doc_revs, 6), + DocUpdates = meck:num_calls(couch_replicator_api_wrap, update_doc, 4), + ?assertEqual(0, BulkGets), + ?assertEqual(?DOC_COUNT, JustGets), + ?assertEqual(?DOC_COUNT, DocUpdates), + compare_dbs(Source, Target). + job_enable_overrides_global_disable({_Ctx, {Source, Target}}) -> populate_db(Source, ?DOC_COUNT), Persist = false, @@ -78,10 +134,31 @@ global_disable_works({_Ctx, {Source, Target}}) -> compare_dbs(Source, Target). populate_db(DbName, DocCount) -> - Fun = fun(Id, Acc) -> [#doc{id = integer_to_binary(Id)} | Acc] end, + IdFun = fun(Id) -> integer_to_binary(Id) end, + Fun = fun(Id, Acc) -> [#doc{id = IdFun(Id)} | Acc] end, + Docs = lists:foldl(Fun, [], lists:seq(1, DocCount)), + {ok, _} = fabric:update_docs(DbName, Docs, [?ADMIN_CTX]). + +populate_db_ddocs(DbName, DocCount) -> + IdFun = fun(Id) -> <<"_design/", (integer_to_binary(Id))/binary>> end, + Fun = fun(Id, Acc) -> [#doc{id = IdFun(Id)} | Acc] end, Docs = lists:foldl(Fun, [], lists:seq(1, DocCount)), {ok, _} = fabric:update_docs(DbName, Docs, [?ADMIN_CTX]). +populate_db_atts(DbName, DocCount) -> + IdFun = fun(Id) -> integer_to_binary(Id) end, + Fun = fun(Id, Acc) -> [#doc{id = IdFun(Id), atts = [att(<<"a">>)]} | Acc] end, + Docs = lists:foldl(Fun, [], lists:seq(1, DocCount)), + {ok, _} = fabric:update_docs(DbName, Docs, [?ADMIN_CTX]). + +att(Name) when is_binary(Name) -> + couch_att:new([ + {name, Name}, + {att_len, 1}, + {type, <<"app/binary">>}, + {data, <<"x">>} + ]). + compare_dbs(Source, Target) -> couch_replicator_test_helper:cluster_compare_dbs(Source, Target). diff --git a/src/docs/src/api/database/find.rst b/src/docs/src/api/database/find.rst index b5a0bd460..0e511e6ad 100644 --- a/src/docs/src/api/database/find.rst +++ b/src/docs/src/api/database/find.rst @@ -705,8 +705,8 @@ operators require the argument to be in a specific JSON format. | | | | document. Non-array fields cannot | | | | | match this condition. | +---------------+-------------+------------+-----------------------------------+ -| Miscellaneous | ``$mod`` | [Divisor, | Divisor and Remainder are both | -| | | Remainder] | positive or negative integers. | +| Miscellaneous | ``$mod`` | [Divisor, | Divisor is a non-zero integer, | +| | | Remainder] | Remainder is any integer. | | | | | Non-integer values result in a | | | | | 404. Matches documents where | | | | | ``field % Divisor == Remainder`` | diff --git a/src/mango/README.md b/src/mango/README.md index 4c4bb60a6..9fa273b15 100644 --- a/src/mango/README.md +++ b/src/mango/README.md @@ -302,7 +302,7 @@ Array related operators Misc related operators -* "$mod" - [Divisor, Remainder], where Divisor and Remainder are both positive integers (ie, greater than 0). Matches documents where (field % Divisor == Remainder) is true. This is false for any non-integer field +* "$mod" - [Divisor, Remainder], where Divisor is a non-zero integer and Remainder is any integer. Matches documents where (field % Divisor == Remainder) is true. This is false for any non-integer field * "$regex" - string, a regular expression pattern to match against the document field. Only matches when the field is a string value and matches the supplied matches |