Make _all_docs and view query limits configurable

This allows us to set a maximun allowed number of documents
to be returned for a global or a partitioned view query and
_all_docs query.

Co-authored-by: Paul J. Davis <paul.joseph.davis@gmail.com>

5 files changed, 155 insertions, 22 deletions

diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini
index 9a332acdd..b9d51af71 100644
--- a/rel/overlay/etc/default.ini
+++ b/rel/overlay/etc/default.ini
@@ -293,6 +293,8 @@ os_process_limit = 100
 ; Timeout for how long a response from a busy view group server can take.
 ; "infinity" is also a valid configuration value.
 ;group_info_timeout = 5000
+;query_limit = 268435456
+;partition_query_limit = 268435456
 
 [mango]
 ; Set to true to disable the "index all fields" text index, which can lead
diff --git a/src/couch_mrview/include/couch_mrview.hrl b/src/couch_mrview/include/couch_mrview.hrl
index e17aaba93..29fe52bdb 100644
--- a/src/couch_mrview/include/couch_mrview.hrl
+++ b/src/couch_mrview/include/couch_mrview.hrl
@@ -60,6 +60,7 @@
     view_states=nil
 }).
 
+-define(MAX_VIEW_LIMIT, 16#10000000).
 
 -record(mrargs, {
     view_type,
@@ -74,7 +75,7 @@
     keys,
 
     direction = fwd,
-    limit = 16#10000000,
+    limit = ?MAX_VIEW_LIMIT,
     skip = 0,
     group_level = 0,
     group = undefined,
diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl
index b879d1242..eb68124a0 100644
--- a/src/couch_mrview/src/couch_mrview_util.erl
+++ b/src/couch_mrview/src/couch_mrview_util.erl
@@ -496,9 +496,10 @@ fold_reduce({NthRed, Lang, View}, Fun,  Acc, Options) ->
     couch_btree:fold_reduce(Bt, WrapperFun, Acc, Options).
 
 
-validate_args(Db, DDoc, Args) ->
+validate_args(Db, DDoc, Args0) ->
     {ok, State} = couch_mrview_index:init(Db, DDoc),
-    validate_args(State, Args).
+    Args1 = apply_limit(State#mrst.partitioned, Args0),
+    validate_args(State, Args1).
 
 
 validate_args(#mrst{} = State, Args0) ->
@@ -523,6 +524,28 @@ validate_args(#mrst{} = State, Args0) ->
     end.
 
 
+apply_limit(ViewPartitioned, Args) ->
+    LimitType = case ViewPartitioned of
+        true -> "partition_query_limit";
+        false -> "query_limit"
+    end,
+
+    MaxLimit = config:get_integer("query_server_config",
+        LimitType, ?MAX_VIEW_LIMIT),
+
+    % Set the highest limit possible if a user has not
+    % specified a limit
+    Args1 = case Args#mrargs.limit == ?MAX_VIEW_LIMIT of
+        true -> Args#mrargs{limit = MaxLimit};
+        false -> Args
+    end,
+
+    if Args1#mrargs.limit =< MaxLimit -> Args1; true ->
+        Fmt = "Limit is too large, must not exceed ~p",
+        mrverror(io_lib:format(Fmt, [MaxLimit]))
+    end.
+
+
 validate_all_docs_args(Db, Args0) ->
     Args = validate_args(Args0),
 
@@ -533,7 +556,8 @@ validate_all_docs_args(Db, Args0) ->
         {false, <<_/binary>>} ->
             mrverror(<<"`partition` parameter is not supported on this db">>);
         {_, <<_/binary>>} ->
-            apply_all_docs_partition(Args, Partition);
+            Args1 = apply_limit(true, Args),
+            apply_all_docs_partition(Args1, Partition);
         _ ->
             Args
     end.
diff --git a/test/elixir/test/partition_all_docs_test.exs b/test/elixir/test/partition_all_docs_test.exs
index 308c1b439..87bab341c 100644
--- a/test/elixir/test/partition_all_docs_test.exs
+++ b/test/elixir/test/partition_all_docs_test.exs
@@ -116,36 +116,82 @@ defmodule PartitionAllDocsTest do
     assert ids == ["foo:22"]
   end
 
+  test "partition all docs can set query limits", context do
+    set_config({"query_server_config", "partition_query_limit", "2000"})
+
+    db_name = context[:db_name]
+    create_partition_docs(db_name)
+    create_partition_ddoc(db_name)
+
+    url = "/#{db_name}/_partition/foo/_all_docs"
+
+    resp =
+      Couch.get(url,
+        query: %{
+          limit: 20
+        }
+      )
+
+    assert resp.status_code == 200
+    ids = get_ids(resp)
+    assert length(ids) == 20
+
+    resp = Couch.get(url)
+    assert resp.status_code == 200
+    ids = get_ids(resp)
+    assert length(ids) == 50
+
+    resp =
+      Couch.get(url,
+        query: %{
+          limit: 2000
+        }
+      )
+
+    assert resp.status_code == 200
+    ids = get_ids(resp)
+    assert length(ids) == 50
+
+    resp =
+      Couch.get(url,
+        query: %{
+          limit: 2001
+        }
+      )
+
+    assert resp.status_code == 400
+    %{:body => %{"reason" => reason}} = resp
+    assert Regex.match?(~r/Limit is too large/, reason)
+
+    resp =
+      Couch.get(url,
+        query: %{
+          limit: 2000,
+          skip: 25
+        }
+      )
+
+    assert resp.status_code == 200
+    ids = get_ids(resp)
+    assert length(ids) == 25
+  end
+
   # This test is timing based so it could be a little flaky.
   # If that turns out to be the case we should probably just skip it
   test "partition _all_docs with timeout", context do
-    on_exit(fn ->
-      resp = Couch.get("/_membership")
-      %{:body => body} = resp
-
-      Enum.each(body["all_nodes"], fn node ->
-        resp = Couch.put("/_node/#{node}/_config/fabric/partition_view_timeout", body: "\"3600000\"")
-        assert resp.status_code == 200
-      end)
-    end)
-
-    resp = Couch.get("/_membership")
-    %{:body => body} = resp
-
-     Enum.each(body["all_nodes"], fn node ->
-      resp = Couch.put("/_node/#{node}/_config/fabric/partition_view_timeout", body: "\"1\"")
-      assert resp.status_code == 200
-    end)
+    set_config({"fabric", "partition_view_timeout", "1"})
 
     db_name = context[:db_name]
     create_partition_docs(db_name)
 
     retry_until(fn ->
       url = "/#{db_name}/_partition/foo/_all_docs"
+
       case Couch.get(url) do
         %{:body => %{"reason" => reason}} ->
           Regex.match?(~r/not be processed in a reasonable amount of time./, reason)
-        _ -> 
+
+        _ ->
           false
       end
     end)
diff --git a/test/elixir/test/partition_view_test.exs b/test/elixir/test/partition_view_test.exs
index a25539172..890cb8858 100644
--- a/test/elixir/test/partition_view_test.exs
+++ b/test/elixir/test/partition_view_test.exs
@@ -286,6 +286,66 @@ defmodule ViewPartitionTest do
            ]
   end
 
+  test "partition query can set query limits", context do
+    set_config({"query_server_config", "partition_query_limit", "2000"})
+
+    db_name = context[:db_name]
+    create_partition_docs(db_name)
+    create_partition_ddoc(db_name)
+
+    url = "/#{db_name}/_partition/foo/_design/mrtest/_view/some"
+
+    resp =
+      Couch.get(url,
+        query: %{
+          limit: 20
+        }
+      )
+
+    assert resp.status_code == 200
+    ids = get_ids(resp)
+    assert length(ids) == 20
+
+    resp = Couch.get(url)
+    assert resp.status_code == 200
+    ids = get_ids(resp)
+    assert length(ids) == 50
+
+    resp =
+      Couch.get(url,
+        query: %{
+          limit: 2000
+        }
+      )
+
+    assert resp.status_code == 200
+    ids = get_ids(resp)
+    assert length(ids) == 50
+
+    resp =
+      Couch.get(url,
+        query: %{
+          limit: 2001
+        }
+      )
+
+    assert resp.status_code == 400
+    %{:body => %{"reason" => reason}} = resp
+    assert Regex.match?(~r/Limit is too large/, reason)
+
+    resp =
+      Couch.get(url,
+        query: %{
+          limit: 2000,
+          skip: 25
+        }
+      )
+
+    assert resp.status_code == 200
+    ids = get_ids(resp)
+    assert length(ids) == 25
+  end
+
   test "include_design works correctly", context do
     db_name = context[:db_name]