diff options
author | Eric Avdey <eiri@eiri.ca> | 2018-11-21 19:09:01 -0400 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-11-21 19:09:01 -0400 |
commit | 4fc292bde6097a11942dbd408264c8301b2853b1 (patch) | |
tree | f7cb998618aaf7804413077c715fe39ad7f056d1 | |
parent | 854f3d9fcbd8db91ab208e27038c252d91841fcf (diff) | |
parent | 1f055f0a8f3e1cb59a40b5399b69ff0a1480dc4f (diff) | |
download | couchdb-4fc292bde6097a11942dbd408264c8301b2853b1.tar.gz |
Merge pull request #1744 from cloudant/fix-_design_docs_total_rows
Fix total_rows value for queries on `_design_docs` handler
-rw-r--r-- | src/couch/src/couch_db.erl | 5 | ||||
-rw-r--r-- | src/couch_mrview/src/couch_mrview.erl | 3 | ||||
-rw-r--r-- | src/couch_mrview/src/couch_mrview_test_util.erl | 13 | ||||
-rw-r--r-- | src/couch_mrview/test/couch_mrview_design_docs_tests.erl | 136 | ||||
-rw-r--r-- | src/fabric/src/fabric.erl | 24 | ||||
-rw-r--r-- | src/fabric/src/fabric_design_doc_count.erl | 69 | ||||
-rw-r--r-- | src/fabric/src/fabric_rpc.erl | 15 | ||||
-rw-r--r-- | src/fabric/src/fabric_view_all_docs.erl | 36 | ||||
-rw-r--r-- | test/javascript/tests/design_docs_query.js | 38 |
9 files changed, 289 insertions, 50 deletions
diff --git a/src/couch/src/couch_db.erl b/src/couch/src/couch_db.erl index 9d6a5dc45..60a395fd7 100644 --- a/src/couch/src/couch_db.erl +++ b/src/couch/src/couch_db.erl @@ -78,6 +78,7 @@ get_full_doc_infos/2, get_missing_revs/2, get_design_docs/1, + get_design_doc_count/1, get_purge_infos/2, get_minimum_purge_seq/1, @@ -618,6 +619,10 @@ get_design_docs(#db{} = Db) -> {ok, Docs} = fold_design_docs(Db, FoldFun, [], []), {ok, lists:reverse(Docs)}. +get_design_doc_count(#db{} = Db) -> + FoldFun = fun(_, Acc) -> {ok, Acc + 1} end, + fold_design_docs(Db, FoldFun, 0, []). + check_is_admin(#db{user_ctx=UserCtx}=Db) -> case is_admin(Db) of true -> ok; diff --git a/src/couch_mrview/src/couch_mrview.erl b/src/couch_mrview/src/couch_mrview.erl index 533dd2de9..f6462e156 100644 --- a/src/couch_mrview/src/couch_mrview.erl +++ b/src/couch_mrview/src/couch_mrview.erl @@ -712,6 +712,9 @@ get_total_rows(Db, #mrargs{extra = Extra}) -> case couch_util:get_value(namespace, Extra) of <<"_local">> -> null; + <<"_design">> -> + {ok, N} = couch_db:get_design_doc_count(Db), + N; _ -> {ok, Info} = couch_db:get_db_info(Db), couch_util:get_value(doc_count, Info) diff --git a/src/couch_mrview/src/couch_mrview_test_util.erl b/src/couch_mrview/src/couch_mrview_test_util.erl index 35ab6c673..ac298263d 100644 --- a/src/couch_mrview/src/couch_mrview_test_util.erl +++ b/src/couch_mrview/src/couch_mrview_test_util.erl @@ -28,7 +28,7 @@ init_db(Name, Type, Count) -> save_docs(Db, Docs). -new_db(Name, local) -> +new_db(Name, Type) when Type == local; Type == design -> couch_server:delete(Name, [?ADMIN_CTX]), couch_db:create(Name, [?ADMIN_CTX]); new_db(Name, Type) -> @@ -46,13 +46,17 @@ save_docs(Db, Docs) -> make_docs(local, Count) -> [local_doc(I) || I <- lists:seq(1, Count)]; +make_docs(design, Count) -> + lists:foldl(fun(I, Acc) -> + [doc(I), ddoc(I) | Acc] + end, [], lists:seq(1, Count)); make_docs(_, Count) -> [doc(I) || I <- lists:seq(1, Count)]. make_docs(_, Since, Count) -> [doc(I) || I <- lists:seq(Since, Count)]. - + ddoc({changes, Opts}) -> ViewOpts = case Opts of @@ -120,6 +124,11 @@ ddoc(red) -> {<<"reduce">>, <<"_count">>} ]}} ]}} + ]}); +ddoc(Id) -> + couch_doc:from_json_obj({[ + {<<"_id">>, list_to_binary(io_lib:format("_design/bar~2..0b", [Id]))}, + {<<"views">>, {[]}} ]}). diff --git a/src/couch_mrview/test/couch_mrview_design_docs_tests.erl b/src/couch_mrview/test/couch_mrview_design_docs_tests.erl new file mode 100644 index 000000000..aedd42865 --- /dev/null +++ b/src/couch_mrview/test/couch_mrview_design_docs_tests.erl @@ -0,0 +1,136 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(couch_mrview_design_docs_tests). + +-include_lib("couch/include/couch_eunit.hrl"). +-include_lib("couch/include/couch_db.hrl"). + +-define(TIMEOUT, 1000). + + + +setup() -> + {ok, Db} = couch_mrview_test_util:init_db(?tempdb(), design), + Db. + +teardown(Db) -> + couch_db:close(Db), + couch_server:delete(couch_db:name(Db), [?ADMIN_CTX]), + ok. + + +design_docs_test_() -> + { + "_design_docs view tests", + { + setup, + fun test_util:start_couch/0, fun test_util:stop_couch/1, + { + foreach, + fun setup/0, fun teardown/1, + [ + fun should_query/1, + fun should_query_with_range/1, + fun should_query_with_range_rev/1, + fun should_query_with_limit_and_skip/1, + fun should_query_with_include_docs/1 + ] + } + } + }. + + +should_query(Db) -> + Result = run_query(Db, []), + Expect = {ok, [ + {meta, [{total, 10}, {offset, 10}]}, + mk_row(<<"_design/bar01">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), + mk_row(<<"_design/bar02">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), + mk_row(<<"_design/bar03">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), + mk_row(<<"_design/bar04">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), + mk_row(<<"_design/bar05">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), + mk_row(<<"_design/bar06">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), + mk_row(<<"_design/bar07">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), + mk_row(<<"_design/bar08">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), + mk_row(<<"_design/bar09">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), + mk_row(<<"_design/bar10">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>) + ]}, + ?_assertEqual(Expect, Result). + +should_query_with_range(Db) -> + Result = run_query(Db, [ + {start_key, <<"_design/bar03">>}, + {end_key, <<"_design/bar05">>} + ]), + Expect = {ok, [ + {meta, [{total, 10}, {offset, 12}]}, + mk_row(<<"_design/bar03">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), + mk_row(<<"_design/bar04">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), + mk_row(<<"_design/bar05">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>) + ]}, + ?_assertEqual(Expect, Result). + +should_query_with_range_rev(Db) -> + Result = run_query(Db, [ + {direction, rev}, + {start_key, <<"_design/bar05">>}, {end_key, <<"_design/bar03">>}, + {inclusive_end, true} + ]), + Expect = {ok, [ + {meta, [{total, 10}, {offset, 5}]}, + mk_row(<<"_design/bar05">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), + mk_row(<<"_design/bar04">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), + mk_row(<<"_design/bar03">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>) + ]}, + ?_assertEqual(Expect, Result). + +should_query_with_limit_and_skip(Db) -> + Result = run_query(Db, [ + {start_key, <<"_design/bar02">>}, + {limit, 3}, + {skip, 3} + ]), + Expect = {ok, [ + {meta, [{total, 10}, {offset, 14}]}, + mk_row(<<"_design/bar05">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), + mk_row(<<"_design/bar06">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>), + mk_row(<<"_design/bar07">>, <<"1-0b24e44a44af45e51e562fd124ce3007">>) + ]}, + ?_assertEqual(Expect, Result). + +should_query_with_include_docs(Db) -> + Result = run_query(Db, [ + {start_key, <<"_design/bar08">>}, + {end_key, <<"_design/bar08">>}, + {include_docs, true} + ]), + Doc = {[ + {<<"_id">>,<<"_design/bar08">>}, + {<<"_rev">>,<<"1-0b24e44a44af45e51e562fd124ce3007">>}, + {<<"views">>,{[]}} + ]}, + Val = {[{rev, <<"1-0b24e44a44af45e51e562fd124ce3007">>}]}, + Expect = {ok, [ + {meta, [{total, 10}, {offset, 17}]}, + {row, [{id, <<"_design/bar08">>}, {key, <<"_design/bar08">>}, + {value, Val}, {doc, Doc}]} + ]}, + ?_assertEqual(Expect, Result). + + +mk_row(Id, Rev) -> + {row, [{id, Id}, {key, Id}, {value, {[{rev, Rev}]}}]}. + +run_query(Db, Opts0) -> + Opts = [{extra, [{namespace, <<"_design">>}]} | Opts0], + couch_mrview:query_all_docs(Db, Opts). diff --git a/src/fabric/src/fabric.erl b/src/fabric/src/fabric.erl index 36ed67154..9bc99c265 100644 --- a/src/fabric/src/fabric.erl +++ b/src/fabric/src/fabric.erl @@ -18,9 +18,10 @@ % DBs -export([all_dbs/0, all_dbs/1, create_db/1, create_db/2, delete_db/1, - delete_db/2, get_db_info/1, get_doc_count/1, set_revs_limit/3, - set_security/2, set_security/3, get_revs_limit/1, get_security/1, - get_security/2, get_all_security/1, get_all_security/2, + delete_db/2, get_db_info/1, get_doc_count/1, get_doc_count/2, + set_revs_limit/3, set_security/2, set_security/3, + get_revs_limit/1, get_security/1, get_security/2, + get_all_security/1, get_all_security/2, get_purge_infos_limit/1, set_purge_infos_limit/3, compact/1, compact/2]). @@ -86,12 +87,21 @@ get_db_info(DbName) -> fabric_db_info:go(dbname(DbName)). %% @doc the number of docs in a database --spec get_doc_count(dbname()) -> - {ok, non_neg_integer()} | +%% @equiv get_doc_count(DbName, <<"_all_docs">>) +get_doc_count(DbName) -> + get_doc_count(DbName, <<"_all_docs">>). + +%% @doc the number of design docs in a database +-spec get_doc_count(dbname(), Namespace::binary()) -> + {ok, non_neg_integer() | null} | {error, atom()} | {error, atom(), any()}. -get_doc_count(DbName) -> - fabric_db_doc_count:go(dbname(DbName)). +get_doc_count(DbName, <<"_all_docs">>) -> + fabric_db_doc_count:go(dbname(DbName)); +get_doc_count(DbName, <<"_design">>) -> + fabric_design_doc_count:go(dbname(DbName)); +get_doc_count(_DbName, <<"_local">>) -> + {ok, null}. %% @equiv create_db(DbName, []) create_db(DbName) -> diff --git a/src/fabric/src/fabric_design_doc_count.erl b/src/fabric/src/fabric_design_doc_count.erl new file mode 100644 index 000000000..22d03c5d4 --- /dev/null +++ b/src/fabric/src/fabric_design_doc_count.erl @@ -0,0 +1,69 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(fabric_design_doc_count). + +-export([go/1]). + +-include_lib("fabric/include/fabric.hrl"). +-include_lib("mem3/include/mem3.hrl"). +-include_lib("couch/include/couch_db.hrl"). + +go(DbName) -> + Shards = mem3:shards(DbName), + Workers = fabric_util:submit_jobs(Shards, get_design_doc_count, []), + RexiMon = fabric_util:create_monitors(Shards), + Acc0 = {fabric_dict:init(Workers, nil), 0}, + try fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0) of + {timeout, {WorkersDict, _}} -> + DefunctWorkers = fabric_util:remove_done_workers(WorkersDict, nil), + fabric_util:log_timeout(DefunctWorkers, "get_design_doc_count"), + {error, timeout}; + Else -> + Else + after + rexi_monitor:stop(RexiMon) + end. + +handle_message({rexi_DOWN, _, {_,NodeRef},_}, _Shard, {Counters, Acc}) -> + case fabric_util:remove_down_workers(Counters, NodeRef) of + {ok, NewCounters} -> + {ok, {NewCounters, Acc}}; + error -> + {error, {nodedown, <<"progress not possible">>}} + end; + +handle_message({rexi_EXIT, Reason}, Shard, {Counters, Acc}) -> + NewCounters = lists:keydelete(Shard, #shard.ref, Counters), + case fabric_view:is_progress_possible(NewCounters) of + true -> + {ok, {NewCounters, Acc}}; + false -> + {error, Reason} + end; + +handle_message({ok, Count}, Shard, {Counters, Acc}) -> + case fabric_dict:lookup_element(Shard, Counters) of + undefined -> + {ok, {Counters, Acc}}; + nil -> + C1 = fabric_dict:store(Shard, ok, Counters), + C2 = fabric_view:remove_overlapping_shards(Shard, C1), + case fabric_dict:any(nil, C2) of + true -> + {ok, {C2, Count+Acc}}; + false -> + {stop, Count+Acc} + end + end; +handle_message(_, _, Acc) -> + {ok, Acc}. diff --git a/src/fabric/src/fabric_rpc.erl b/src/fabric/src/fabric_rpc.erl index 11e675464..c8aa19e0f 100644 --- a/src/fabric/src/fabric_rpc.erl +++ b/src/fabric/src/fabric_rpc.erl @@ -12,7 +12,8 @@ -module(fabric_rpc). --export([get_db_info/1, get_doc_count/1, get_update_seq/1]). +-export([get_db_info/1, get_doc_count/1, get_design_doc_count/1, + get_update_seq/1]). -export([open_doc/3, open_revs/4, get_doc_info/3, get_full_doc_info/3, get_missing_revs/2, get_missing_revs/3, update_docs/3]). -export([all_docs/3, changes/3, map_view/4, reduce_view/4, group_info/2]). @@ -23,8 +24,9 @@ -export([compact/1, compact/2]). -export([get_purge_seq/2, purge_docs/3, set_purge_infos_limit/3]). --export([get_db_info/2, get_doc_count/2, get_update_seq/2, - changes/4, map_view/5, reduce_view/5, group_info/3, update_mrview/4]). +-export([get_db_info/2, get_doc_count/2, get_design_doc_count/2, + get_update_seq/2, changes/4, map_view/5, reduce_view/5, + group_info/3, update_mrview/4]). -include_lib("fabric/include/fabric.hrl"). -include_lib("couch/include/couch_db.hrl"). @@ -181,6 +183,13 @@ get_doc_count(DbName) -> get_doc_count(DbName, DbOptions) -> with_db(DbName, DbOptions, {couch_db, get_doc_count, []}). +%% equiv get_design_doc_count(DbName, []) +get_design_doc_count(DbName) -> + get_design_doc_count(DbName, []). + +get_design_doc_count(DbName, DbOptions) -> + with_db(DbName, DbOptions, {couch_db, get_design_doc_count, []}). + %% equiv get_update_seq(DbName, []) get_update_seq(DbName) -> get_update_seq(DbName, []). diff --git a/src/fabric/src/fabric_view_all_docs.erl b/src/fabric/src/fabric_view_all_docs.erl index ac16dac52..30c8e8d51 100644 --- a/src/fabric/src/fabric_view_all_docs.erl +++ b/src/fabric/src/fabric_view_all_docs.erl @@ -82,24 +82,22 @@ go(DbName, Options, QueryArgs, Callback, Acc0) -> true -> lists:sublist(Keys2, Limit); false -> Keys2 end, - Resp = case couch_util:get_value(namespace, Extra, <<"_all_docs">>) of - <<"_local">> -> - {ok, null}; - _ -> - Timeout = fabric_util:all_docs_timeout(), - {_, Ref0} = spawn_monitor(fun() -> - exit(fabric:get_doc_count(DbName)) - end), - receive {'DOWN', Ref0, _, _, Result} -> - Result - after Timeout -> - timeout - end + %% namespace can be _set_ to `undefined`, so we want simulate enum here + Namespace = case couch_util:get_value(namespace, Extra) of + <<"_all_docs">> -> <<"_all_docs">>; + <<"_design">> -> <<"_design">>; + <<"_local">> -> <<"_local">>; + _ -> <<"_all_docs">> end, - case Resp of - {ok, TotalRows} -> + Timeout = fabric_util:all_docs_timeout(), + {_, Ref} = spawn_monitor(fun() -> + exit(fabric:get_doc_count(DbName, Namespace)) + end), + receive + {'DOWN', Ref, _, _, {ok, TotalRows}} -> Meta = case UpdateSeq of - false -> [{total, TotalRows}, {offset, null}]; + false -> + [{total, TotalRows}, {offset, null}]; true -> [{total, TotalRows}, {offset, null}, {update_seq, null}] end, @@ -108,10 +106,10 @@ go(DbName, Options, QueryArgs, Callback, Acc0) -> Keys3, queue:new(), SpawnFun, MaxJobs, Callback, Acc1 ), Callback(complete, Acc2); - timeout -> - Callback(timeout, Acc0); - Error -> + {'DOWN', Ref, _, _, Error} -> Callback({error, Error}, Acc0) + after Timeout -> + Callback(timeout, Acc0) end. go(DbName, _Options, Workers, QueryArgs, Callback, Acc0) -> diff --git a/test/javascript/tests/design_docs_query.js b/test/javascript/tests/design_docs_query.js index 8fc8da5f8..07e6577ab 100644 --- a/test/javascript/tests/design_docs_query.js +++ b/test/javascript/tests/design_docs_query.js @@ -15,7 +15,7 @@ couchTests.design_docs_query = function(debug) { var db = new CouchDB(db_name, {"X-Couch-Full-Commit":"false"}); db.createDb(); if (debug) debugger; - + var docs = makeDocs(5); // create the docs @@ -24,7 +24,7 @@ couchTests.design_docs_query = function(debug) { for (var i = 0; i < 5; i++) { T(results[i].id == docs[i]._id); } - + // create the ddocs for (var i = 0; i < 5; i++) { T(db.save({ @@ -35,16 +35,16 @@ couchTests.design_docs_query = function(debug) { } } }).ok); - } - - // test design_docs + } + + // test design_docs var path = "/" + db_name + "/_design_docs?"; var xhr_AllDDocs = CouchDB.request("GET", path); T(xhr_AllDDocs.status == 200, "standard get should be 200"); var allDDocs = JSON.parse(xhr_AllDDocs.responseText); - TEquals(10, allDDocs.total_rows, "total_rows mismatch"); + TEquals(5, allDDocs.total_rows, "total_rows mismatch"); TEquals(5, allDDocs.rows.length, "amount of rows mismatch"); - + // test key="_design/ddoc03" var xhr = CouchDB.request("GET", path + "key=\"_design/ddoc03\""); T(xhr.status = 200, "standard get should be 200"); @@ -72,21 +72,21 @@ couchTests.design_docs_query = function(debug) { var result = JSON.parse(xhr.responseText); TEquals(3, result.rows.length, "amount of rows mismatch"); TEquals("_design/ddoc03", result.rows[2].key, "end_key test"); - + // test endkey="_design/ddoc03" var xhr = CouchDB.request("GET", path + "endkey=\"_design/ddoc03\""); T(xhr.status = 200, "standard get should be 200"); var result = JSON.parse(xhr.responseText); TEquals(3, result.rows.length, "amount of rows mismatch"); TEquals("_design/ddoc03", result.rows[2].key, "endkey test"); - + // test start_key="_design/ddoc03" var xhr = CouchDB.request("GET", path + "start_key=\"_design/ddoc03\""); T(xhr.status = 200, "standard get should be 200"); var result = JSON.parse(xhr.responseText); TEquals(3, result.rows.length, "amount of rows mismatch"); TEquals("_design/ddoc03", result.rows[0].key, "start_key test"); - + // test startkey="_design/ddoc03" var xhr = CouchDB.request("GET", path + "startkey=\"_design/ddoc03\""); T(xhr.status = 200, "standard get should be 200"); @@ -107,39 +107,39 @@ couchTests.design_docs_query = function(debug) { var result = JSON.parse(xhr.responseText); TEquals(2, result.rows.length, "amount of rows mismatch"); TEquals("_design/ddoc02", result.rows[1].key, "end_key and inclusive_end test"); - + // test end_key="_design/ddoc03"&inclusive_end=false&descending=true - var xhr = CouchDB.request("GET", path + + var xhr = CouchDB.request("GET", path + "end_key=\"_design/ddoc03\"&inclusive_end=false&descending=true"); T(xhr.status = 200, "standard get should be 200"); var result = JSON.parse(xhr.responseText); TEquals(2, result.rows.length, "amount of rows mismatch"); TEquals("_design/ddoc04", result.rows[1].key, "end_key, inclusive_end and descending test"); - + // test end_key="_design/ddoc05"&limit=2 - var xhr = CouchDB.request("GET", path + + var xhr = CouchDB.request("GET", path + "end_key=\"_design/ddoc05\"&limit=2"); T(xhr.status = 200, "standard get should be 200"); var result = JSON.parse(xhr.responseText); TEquals(2, result.rows.length, "amount of rows mismatch"); TEquals("_design/ddoc02", result.rows[1].key, "end_key and limit test"); - + // test end_key="_design/ddoc05"&skip=2 - var xhr = CouchDB.request("GET", path + + var xhr = CouchDB.request("GET", path + "end_key=\"_design/ddoc05\"&skip=2"); T(xhr.status = 200, "standard get should be 200"); var result = JSON.parse(xhr.responseText); TEquals(3, result.rows.length, "amount of rows mismatch"); TEquals("_design/ddoc03", result.rows[0].key, "end_key and skip test"); TEquals("_design/ddoc05", result.rows[2].key, "end_key and skip test"); - + // test end_key="_design/ddoc05"&update_seq=true - var xhr = CouchDB.request("GET", path + + var xhr = CouchDB.request("GET", path + "end_key=\"_design/ddoc05\"&update_seq=true"); T(xhr.status = 200, "standard get should be 200"); var result = JSON.parse(xhr.responseText); T(result.update_seq); - + // test POST with keys var xhr = CouchDB.request("POST", path, { headers: {"Content-Type": "application/json"}, |