summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorWill Holley <willholley@gmail.com>2020-01-06 13:22:07 +0000
committerWill Holley <willholley@gmail.com>2020-01-17 13:54:34 +0000
commit5d55e289760959eb2fc228acb2ed00d57ddd4dee (patch)
tree2918556fee71003641ddcb96a5c8c5ee59c812a7
parentb5fe3d69271813ac5aa7966b60eb2e94947a0606 (diff)
downloadcouchdb-5d55e289760959eb2fc228acb2ed00d57ddd4dee.tar.gz
Warn on mango index scan
Adds a warning to the _find endpoint if the ratio of docs scanned: results returned is higher than a configurable threshold (default 10). This warning was previously generated in Fauxton; moving it to the server side allows us to expose it via _stats as well.
-rw-r--r--rel/overlay/etc/default.ini4
-rw-r--r--src/couch/priv/stats_descriptions.cfg4
-rw-r--r--src/mango/src/mango_cursor.erl83
-rw-r--r--src/mango/src/mango_cursor_text.erl2
-rw-r--r--src/mango/src/mango_cursor_view.erl2
5 files changed, 65 insertions, 30 deletions
diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini
index a301987b2..d4945e1a1 100644
--- a/rel/overlay/etc/default.ini
+++ b/rel/overlay/etc/default.ini
@@ -317,6 +317,10 @@ os_process_limit = 100
;index_all_disabled = false
; Default limit value for mango _find queries.
;default_limit = 25
+; Ratio between documents scanned and results matched that will
+; generate a warning in the _find response. Setting this to 0 disables
+; the warning.
+;index_scan_warning_threshold = 10
[indexers]
couch_mrview = true
diff --git a/src/couch/priv/stats_descriptions.cfg b/src/couch/priv/stats_descriptions.cfg
index 0d81ea31b..557ac36b8 100644
--- a/src/couch/priv/stats_descriptions.cfg
+++ b/src/couch/priv/stats_descriptions.cfg
@@ -306,3 +306,7 @@
{type, counter},
{desc, <<"number of mango queries that generated an invalid index warning">>}
]}.
+{[mango, too_many_docs_scanned], [
+ {type, counter},
+ {desc, <<"number of mango queries that generated an index scan warning">>}
+]}.
diff --git a/src/mango/src/mango_cursor.erl b/src/mango/src/mango_cursor.erl
index a16e0d5e4..29be49490 100644
--- a/src/mango/src/mango_cursor.erl
+++ b/src/mango/src/mango_cursor.erl
@@ -19,7 +19,7 @@
execute/3,
maybe_filter_indexes_by_ddoc/2,
remove_indexes_with_partial_filter_selector/1,
- maybe_add_warning/3,
+ maybe_add_warning/4,
maybe_noop_range/2
]).
@@ -124,6 +124,22 @@ remove_indexes_with_partial_filter_selector(Indexes) ->
lists:filter(FiltFun, Indexes).
+maybe_add_warning(UserFun, #cursor{index = Index, opts = Opts}, Stats, UserAcc) ->
+ W0 = invalid_index_warning(Index, Opts),
+ W1 = no_index_warning(Index),
+ W2 = index_scan_warning(Stats),
+ Warnings = lists:append([W0, W1, W2]),
+ case Warnings of
+ [] ->
+ UserAcc;
+ _ ->
+ WarningStr = lists:join(<<"\n">>, Warnings),
+ Arg = {add_key, warning, WarningStr},
+ {_Go, UserAcc1} = UserFun(Arg, UserAcc),
+ UserAcc1
+ end.
+
+
create_cursor(Db, Indexes, Selector, Opts) ->
[{CursorMod, CursorModIndexes} | _] = group_indexes_by_type(Indexes),
CursorMod:create(Db, CursorModIndexes, Selector, Opts).
@@ -151,6 +167,7 @@ group_indexes_by_type(Indexes) ->
no_index_warning(#idx{type = Type}) when Type =:= <<"special">> ->
couch_stats:increment_counter([mango, unindexed_queries]),
[<<"No matching index found, create an index to optimize query time.">>];
+
no_index_warning(_) ->
[].
@@ -164,41 +181,44 @@ invalid_index_warning(Index, Opts) ->
invalid_index_warning_int(Index, {use_index, [DesignId]}) ->
- case filter_indexes([Index], DesignId) of
- [] ->
- couch_stats:increment_counter([mango, query_invalid_index]),
- Reason = fmt("_design/~s was not used because it does not contain a valid index for this query.",
+ Filtered = filter_indexes([Index], DesignId),
+ if Filtered /= [] -> []; true ->
+ couch_stats:increment_counter([mango, query_invalid_index]),
+ Reason = fmt("_design/~s was not used because it does not contain a valid index for this query.",
[ddoc_name(DesignId)]),
- [Reason];
- _ ->
- []
+ [Reason]
end;
+
invalid_index_warning_int(Index, {use_index, [DesignId, ViewName]}) ->
- case filter_indexes([Index], DesignId, ViewName) of
- [] ->
- couch_stats:increment_counter([mango, query_invalid_index]),
- Reason = fmt("_design/~s, ~s was not used because it is not a valid index for this query.",
- [ddoc_name(DesignId), ViewName]),
- [Reason];
- _ ->
- []
+ Filtered = filter_indexes([Index], DesignId, ViewName),
+ if Filtered /= [] -> []; true ->
+ couch_stats:increment_counter([mango, query_invalid_index]),
+ Reason = fmt("_design/~s, ~s was not used because it is not a valid index for this query.",
+ [ddoc_name(DesignId), ViewName]),
+ [Reason]
end;
+
invalid_index_warning_int(_, _) ->
[].
-maybe_add_warning(UserFun, #cursor{index = Index, opts = Opts}, UserAcc) ->
- W0 = invalid_index_warning(Index, Opts),
- W1 = no_index_warning(Index),
- Warnings = lists:append([W0, W1]),
- case Warnings of
- [] ->
- UserAcc;
- _ ->
- WarningStr = lists:join(<<"\n">>, Warnings),
- Arg = {add_key, warning, WarningStr},
- {_Go, UserAcc1} = UserFun(Arg, UserAcc),
- UserAcc1
+% warn if a large number of documents needed to be scanned per result
+% returned, implying a lot of in-memory filtering
+index_scan_warning(#execution_stats {
+ totalDocsExamined = Docs,
+ totalQuorumDocsExamined = DocsQuorum,
+ resultsReturned = ResultCount
+ }) ->
+ % Docs and DocsQuorum are mutually exclusive so it's safe to sum them
+ DocsScanned = Docs + DocsQuorum,
+ Ratio = calculate_index_scan_ratio(DocsScanned, ResultCount),
+ Threshold = config:get_integer("mango", "index_scan_warning_threshold", 10),
+ case Threshold > 0 andalso Ratio > Threshold of
+ true ->
+ couch_stats:increment_counter([mango, too_many_docs_scanned]),
+ Reason = <<"The number of documents examined is high in proportion to the number of results returned. Consider adding a more specific index to improve this.">>,
+ [Reason];
+ false -> []
end.
% When there is an empty array for certain operators, we don't actually
@@ -218,6 +238,13 @@ maybe_noop_range(_, IndexRanges) ->
IndexRanges.
+calculate_index_scan_ratio(DocsScanned, 0) ->
+ DocsScanned;
+
+calculate_index_scan_ratio(DocsScanned, ResultCount) ->
+ DocsScanned / ResultCount.
+
+
fmt(Format, Args) ->
iolist_to_binary(io_lib:format(Format, Args)).
diff --git a/src/mango/src/mango_cursor_text.erl b/src/mango/src/mango_cursor_text.erl
index e05ab0765..6b202daab 100644
--- a/src/mango/src/mango_cursor_text.erl
+++ b/src/mango/src/mango_cursor_text.erl
@@ -132,7 +132,7 @@ execute(Cursor, UserFun, UserAcc) ->
Arg = {add_key, bookmark, JsonBM},
{_Go, FinalUserAcc} = UserFun(Arg, LastUserAcc),
FinalUserAcc0 = mango_execution_stats:maybe_add_stats(Opts, UserFun, Stats0, FinalUserAcc),
- FinalUserAcc1 = mango_cursor:maybe_add_warning(UserFun, Cursor, FinalUserAcc0),
+ FinalUserAcc1 = mango_cursor:maybe_add_warning(UserFun, Cursor, Stats0, FinalUserAcc0),
{ok, FinalUserAcc1}
end.
diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index 921b1fde1..256946900 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -155,7 +155,7 @@ execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFu
{_Go, FinalUserAcc} = UserFun(Arg, LastCursor#cursor.user_acc),
Stats0 = LastCursor#cursor.execution_stats,
FinalUserAcc0 = mango_execution_stats:maybe_add_stats(Opts, UserFun, Stats0, FinalUserAcc),
- FinalUserAcc1 = mango_cursor:maybe_add_warning(UserFun, Cursor, FinalUserAcc0),
+ FinalUserAcc1 = mango_cursor:maybe_add_warning(UserFun, Cursor, Stats0, FinalUserAcc0),
{ok, FinalUserAcc1};
{error, Reason} ->
{error, Reason}