diff options
author | Will Holley <willholley@gmail.com> | 2020-01-06 13:22:07 +0000 |
---|---|---|
committer | Will Holley <willholley@gmail.com> | 2020-01-17 13:54:34 +0000 |
commit | 5d55e289760959eb2fc228acb2ed00d57ddd4dee (patch) | |
tree | 2918556fee71003641ddcb96a5c8c5ee59c812a7 | |
parent | b5fe3d69271813ac5aa7966b60eb2e94947a0606 (diff) | |
download | couchdb-5d55e289760959eb2fc228acb2ed00d57ddd4dee.tar.gz |
Warn on mango index scan
Adds a warning to the _find endpoint if the ratio of docs scanned:
results returned is higher than a configurable threshold (default
10). This warning was previously generated in Fauxton; moving
it to the server side allows us to expose it via _stats as well.
-rw-r--r-- | rel/overlay/etc/default.ini | 4 | ||||
-rw-r--r-- | src/couch/priv/stats_descriptions.cfg | 4 | ||||
-rw-r--r-- | src/mango/src/mango_cursor.erl | 83 | ||||
-rw-r--r-- | src/mango/src/mango_cursor_text.erl | 2 | ||||
-rw-r--r-- | src/mango/src/mango_cursor_view.erl | 2 |
5 files changed, 65 insertions, 30 deletions
diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini index a301987b2..d4945e1a1 100644 --- a/rel/overlay/etc/default.ini +++ b/rel/overlay/etc/default.ini @@ -317,6 +317,10 @@ os_process_limit = 100 ;index_all_disabled = false ; Default limit value for mango _find queries. ;default_limit = 25 +; Ratio between documents scanned and results matched that will +; generate a warning in the _find response. Setting this to 0 disables +; the warning. +;index_scan_warning_threshold = 10 [indexers] couch_mrview = true diff --git a/src/couch/priv/stats_descriptions.cfg b/src/couch/priv/stats_descriptions.cfg index 0d81ea31b..557ac36b8 100644 --- a/src/couch/priv/stats_descriptions.cfg +++ b/src/couch/priv/stats_descriptions.cfg @@ -306,3 +306,7 @@ {type, counter}, {desc, <<"number of mango queries that generated an invalid index warning">>} ]}. +{[mango, too_many_docs_scanned], [ + {type, counter}, + {desc, <<"number of mango queries that generated an index scan warning">>} +]}. diff --git a/src/mango/src/mango_cursor.erl b/src/mango/src/mango_cursor.erl index a16e0d5e4..29be49490 100644 --- a/src/mango/src/mango_cursor.erl +++ b/src/mango/src/mango_cursor.erl @@ -19,7 +19,7 @@ execute/3, maybe_filter_indexes_by_ddoc/2, remove_indexes_with_partial_filter_selector/1, - maybe_add_warning/3, + maybe_add_warning/4, maybe_noop_range/2 ]). @@ -124,6 +124,22 @@ remove_indexes_with_partial_filter_selector(Indexes) -> lists:filter(FiltFun, Indexes). +maybe_add_warning(UserFun, #cursor{index = Index, opts = Opts}, Stats, UserAcc) -> + W0 = invalid_index_warning(Index, Opts), + W1 = no_index_warning(Index), + W2 = index_scan_warning(Stats), + Warnings = lists:append([W0, W1, W2]), + case Warnings of + [] -> + UserAcc; + _ -> + WarningStr = lists:join(<<"\n">>, Warnings), + Arg = {add_key, warning, WarningStr}, + {_Go, UserAcc1} = UserFun(Arg, UserAcc), + UserAcc1 + end. + + create_cursor(Db, Indexes, Selector, Opts) -> [{CursorMod, CursorModIndexes} | _] = group_indexes_by_type(Indexes), CursorMod:create(Db, CursorModIndexes, Selector, Opts). @@ -151,6 +167,7 @@ group_indexes_by_type(Indexes) -> no_index_warning(#idx{type = Type}) when Type =:= <<"special">> -> couch_stats:increment_counter([mango, unindexed_queries]), [<<"No matching index found, create an index to optimize query time.">>]; + no_index_warning(_) -> []. @@ -164,41 +181,44 @@ invalid_index_warning(Index, Opts) -> invalid_index_warning_int(Index, {use_index, [DesignId]}) -> - case filter_indexes([Index], DesignId) of - [] -> - couch_stats:increment_counter([mango, query_invalid_index]), - Reason = fmt("_design/~s was not used because it does not contain a valid index for this query.", + Filtered = filter_indexes([Index], DesignId), + if Filtered /= [] -> []; true -> + couch_stats:increment_counter([mango, query_invalid_index]), + Reason = fmt("_design/~s was not used because it does not contain a valid index for this query.", [ddoc_name(DesignId)]), - [Reason]; - _ -> - [] + [Reason] end; + invalid_index_warning_int(Index, {use_index, [DesignId, ViewName]}) -> - case filter_indexes([Index], DesignId, ViewName) of - [] -> - couch_stats:increment_counter([mango, query_invalid_index]), - Reason = fmt("_design/~s, ~s was not used because it is not a valid index for this query.", - [ddoc_name(DesignId), ViewName]), - [Reason]; - _ -> - [] + Filtered = filter_indexes([Index], DesignId, ViewName), + if Filtered /= [] -> []; true -> + couch_stats:increment_counter([mango, query_invalid_index]), + Reason = fmt("_design/~s, ~s was not used because it is not a valid index for this query.", + [ddoc_name(DesignId), ViewName]), + [Reason] end; + invalid_index_warning_int(_, _) -> []. -maybe_add_warning(UserFun, #cursor{index = Index, opts = Opts}, UserAcc) -> - W0 = invalid_index_warning(Index, Opts), - W1 = no_index_warning(Index), - Warnings = lists:append([W0, W1]), - case Warnings of - [] -> - UserAcc; - _ -> - WarningStr = lists:join(<<"\n">>, Warnings), - Arg = {add_key, warning, WarningStr}, - {_Go, UserAcc1} = UserFun(Arg, UserAcc), - UserAcc1 +% warn if a large number of documents needed to be scanned per result +% returned, implying a lot of in-memory filtering +index_scan_warning(#execution_stats { + totalDocsExamined = Docs, + totalQuorumDocsExamined = DocsQuorum, + resultsReturned = ResultCount + }) -> + % Docs and DocsQuorum are mutually exclusive so it's safe to sum them + DocsScanned = Docs + DocsQuorum, + Ratio = calculate_index_scan_ratio(DocsScanned, ResultCount), + Threshold = config:get_integer("mango", "index_scan_warning_threshold", 10), + case Threshold > 0 andalso Ratio > Threshold of + true -> + couch_stats:increment_counter([mango, too_many_docs_scanned]), + Reason = <<"The number of documents examined is high in proportion to the number of results returned. Consider adding a more specific index to improve this.">>, + [Reason]; + false -> [] end. % When there is an empty array for certain operators, we don't actually @@ -218,6 +238,13 @@ maybe_noop_range(_, IndexRanges) -> IndexRanges. +calculate_index_scan_ratio(DocsScanned, 0) -> + DocsScanned; + +calculate_index_scan_ratio(DocsScanned, ResultCount) -> + DocsScanned / ResultCount. + + fmt(Format, Args) -> iolist_to_binary(io_lib:format(Format, Args)). diff --git a/src/mango/src/mango_cursor_text.erl b/src/mango/src/mango_cursor_text.erl index e05ab0765..6b202daab 100644 --- a/src/mango/src/mango_cursor_text.erl +++ b/src/mango/src/mango_cursor_text.erl @@ -132,7 +132,7 @@ execute(Cursor, UserFun, UserAcc) -> Arg = {add_key, bookmark, JsonBM}, {_Go, FinalUserAcc} = UserFun(Arg, LastUserAcc), FinalUserAcc0 = mango_execution_stats:maybe_add_stats(Opts, UserFun, Stats0, FinalUserAcc), - FinalUserAcc1 = mango_cursor:maybe_add_warning(UserFun, Cursor, FinalUserAcc0), + FinalUserAcc1 = mango_cursor:maybe_add_warning(UserFun, Cursor, Stats0, FinalUserAcc0), {ok, FinalUserAcc1} end. diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl index 921b1fde1..256946900 100644 --- a/src/mango/src/mango_cursor_view.erl +++ b/src/mango/src/mango_cursor_view.erl @@ -155,7 +155,7 @@ execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFu {_Go, FinalUserAcc} = UserFun(Arg, LastCursor#cursor.user_acc), Stats0 = LastCursor#cursor.execution_stats, FinalUserAcc0 = mango_execution_stats:maybe_add_stats(Opts, UserFun, Stats0, FinalUserAcc), - FinalUserAcc1 = mango_cursor:maybe_add_warning(UserFun, Cursor, FinalUserAcc0), + FinalUserAcc1 = mango_cursor:maybe_add_warning(UserFun, Cursor, Stats0, FinalUserAcc0), {ok, FinalUserAcc1}; {error, Reason} -> {error, Reason} |