summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--rel/overlay/etc/default.ini20
-rw-r--r--src/couch/priv/stats_descriptions.cfg28
-rw-r--r--src/mango/src/mango_cursor.erl118
-rw-r--r--src/mango/src/mango_cursor_text.erl3
-rw-r--r--src/mango/src/mango_cursor_view.erl4
-rw-r--r--src/mango/src/mango_execution_stats.erl9
-rw-r--r--src/mango/src/mango_selector.erl18
7 files changed, 141 insertions, 59 deletions
diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini
index a301987b2..506d88784 100644
--- a/rel/overlay/etc/default.ini
+++ b/rel/overlay/etc/default.ini
@@ -168,8 +168,8 @@ enable_xframe_options = false
; CouchDB can optionally enforce a maximum uri length;
; max_uri_length = 8000
; changes_timeout = 60000
-; config_whitelist =
-; max_uri_length =
+; config_whitelist =
+; max_uri_length =
; rewrite_limit = 100
; x_forwarded_host = X-Forwarded-Host
; x_forwarded_proto = X-Forwarded-Proto
@@ -178,7 +178,7 @@ enable_xframe_options = false
max_http_request_size = 4294967296 ; 4GB
; [httpd_design_handlers]
-; _view =
+; _view =
; [ioq]
; concurrency = 10
@@ -192,7 +192,7 @@ port = 6984
; [chttpd_auth_cache]
; max_lifetime = 600000
-; max_objects =
+; max_objects =
; max_size = 104857600
; [mem3]
@@ -203,7 +203,7 @@ port = 6984
; [fabric]
; all_docs_concurrency = 10
-; changes_duration =
+; changes_duration =
; shard_timeout_factor = 2
; uuid_prefix_len = 7
; request_timeout = 60000
@@ -242,7 +242,7 @@ iterations = 10 ; iterations for password hashing
; proxy_use_secret = false
; comma-separated list of public fields, 404 if empty
; public_fields =
-; secret =
+; secret =
; users_db_public = false
; cookie_domain = example.com
; Set the SameSite cookie property for the auth cookie. If empty, the SameSite property is not set.
@@ -317,12 +317,16 @@ os_process_limit = 100
;index_all_disabled = false
; Default limit value for mango _find queries.
;default_limit = 25
+; Ratio between documents scanned and results matched that will
+; generate a warning in the _find response. Setting this to 0 disables
+; the warning.
+;index_scan_warning_threshold = 10
[indexers]
couch_mrview = true
[feature_flags]
-; This enables any database to be created as a partitioned databases (except system db's).
+; This enables any database to be created as a partitioned databases (except system db's).
; Setting this to false will stop the creation of paritioned databases.
; paritioned||allowed* = true will scope the creation of partitioned databases
; to databases with 'allowed' prefix.
@@ -565,7 +569,7 @@ compaction = false
; The default number of results returned from a search on a partition
; of a database.
; limit_partitions = 2000
-
+
; The maximum number of results that can be returned from a global
; search query (or any search query on a database without user-defined
; partitions). Attempts to set ?limit=N higher than this value will
diff --git a/src/couch/priv/stats_descriptions.cfg b/src/couch/priv/stats_descriptions.cfg
index ae203bb21..7c8fd94cb 100644
--- a/src/couch/priv/stats_descriptions.cfg
+++ b/src/couch/priv/stats_descriptions.cfg
@@ -302,3 +302,31 @@
{type, counter},
{desc, <<"number of mango queries that could not use an index">>}
]}.
+{[mango, query_invalid_index], [
+ {type, counter},
+ {desc, <<"number of mango queries that generated an invalid index warning">>}
+]}.
+{[mango, too_many_docs_scanned], [
+ {type, counter},
+ {desc, <<"number of mango queries that generated an index scan warning">>}
+]}.
+{[mango, docs_examined], [
+ {type, counter},
+ {desc, <<"number of documents examined by mango queries coordinated by this node">>}
+]}.
+{[mango, quorum_docs_examined], [
+ {type, counter},
+ {desc, <<"number of documents examined by mango queries, using cluster quorum">>}
+]}.
+{[mango, results_returned], [
+ {type, counter},
+ {desc, <<"number of rows returned by mango queries">>}
+]}.
+{[mango, query_time], [
+ {type, histogram},
+ {desc, <<"length of time processing a mango query">>}
+]}.
+{[mango, evaluate_selector], [
+ {type, counter},
+ {desc, <<"number of mango selector evaluations">>}
+]}.
diff --git a/src/mango/src/mango_cursor.erl b/src/mango/src/mango_cursor.erl
index dc2ee74c7..e84a65991 100644
--- a/src/mango/src/mango_cursor.erl
+++ b/src/mango/src/mango_cursor.erl
@@ -19,7 +19,7 @@
execute/3,
maybe_filter_indexes_by_ddoc/2,
remove_indexes_with_partial_filter_selector/1,
- maybe_add_warning/3
+ maybe_add_warning/4
]).
@@ -114,7 +114,7 @@ filter_indexes(Indexes0, DesignId, ViewName) ->
remove_indexes_with_partial_filter_selector(Indexes) ->
- FiltFun = fun(Idx) ->
+ FiltFun = fun(Idx) ->
case mango_idx:get_partial_filter_selector(Idx) of
undefined -> true;
_ -> false
@@ -123,6 +123,22 @@ remove_indexes_with_partial_filter_selector(Indexes) ->
lists:filter(FiltFun, Indexes).
+maybe_add_warning(UserFun, #cursor{index = Index, opts = Opts}, Stats, UserAcc) ->
+ W0 = invalid_index_warning(Index, Opts),
+ W1 = no_index_warning(Index),
+ W2 = index_scan_warning(Stats),
+ Warnings = lists:append([W0, W1, W2]),
+ case Warnings of
+ [] ->
+ UserAcc;
+ _ ->
+ WarningStr = lists:join(<<"\n">>, Warnings),
+ Arg = {add_key, warning, WarningStr},
+ {_Go, UserAcc1} = UserFun(Arg, UserAcc),
+ UserAcc1
+ end.
+
+
create_cursor(Db, Indexes, Selector, Opts) ->
[{CursorMod, CursorModIndexes} | _] = group_indexes_by_type(Indexes),
CursorMod:create(Db, CursorModIndexes, Selector, Opts).
@@ -146,46 +162,70 @@ group_indexes_by_type(Indexes) ->
end, ?CURSOR_MODULES).
-maybe_add_warning(UserFun, #cursor{index = Index, opts = Opts}, UserAcc) ->
- NoIndexWarning = case Index#idx.type of
- <<"special">> ->
- <<"no matching index found, create an index to optimize query time">>;
- _ ->
- ok
- end,
-
- UseIndexInvalidWarning = case lists:keyfind(use_index, 1, Opts) of
- {use_index, []} ->
- NoIndexWarning;
- {use_index, [DesignId]} ->
- case filter_indexes([Index], DesignId) of
- [] ->
- fmt("_design/~s was not used because it does not contain a valid index for this query.",
- [ddoc_name(DesignId)]);
- _ ->
- NoIndexWarning
- end;
- {use_index, [DesignId, ViewName]} ->
- case filter_indexes([Index], DesignId, ViewName) of
- [] ->
- fmt("_design/~s, ~s was not used because it is not a valid index for this query.",
- [ddoc_name(DesignId), ViewName]);
- _ ->
- NoIndexWarning
- end
- end,
-
- maybe_add_warning_int(UseIndexInvalidWarning, UserFun, UserAcc).
+% warn if the _all_docs index was used to fulfil a query
+no_index_warning(#idx{type = Type}) when Type =:= <<"special">> ->
+ couch_stats:increment_counter([mango, unindexed_queries]),
+ [<<"No matching index found, create an index to optimize query time.">>];
+
+no_index_warning(_) ->
+ [].
+
+
+% warn if user specified an index which doesn't exist or isn't valid
+% for the selector.
+% In this scenario, Mango will ignore the index hint and auto-select an index.
+invalid_index_warning(Index, Opts) ->
+ UseIndex = lists:keyfind(use_index, 1, Opts),
+ invalid_index_warning_int(Index, UseIndex).
+
+
+invalid_index_warning_int(Index, {use_index, [DesignId]}) ->
+ Filtered = filter_indexes([Index], DesignId),
+ if Filtered /= [] -> []; true ->
+ couch_stats:increment_counter([mango, query_invalid_index]),
+ Reason = fmt("_design/~s was not used because it does not contain a valid index for this query.",
+ [ddoc_name(DesignId)]),
+ [Reason]
+ end;
+
+invalid_index_warning_int(Index, {use_index, [DesignId, ViewName]}) ->
+ Filtered = filter_indexes([Index], DesignId, ViewName),
+ if Filtered /= [] -> []; true ->
+ couch_stats:increment_counter([mango, query_invalid_index]),
+ Reason = fmt("_design/~s, ~s was not used because it is not a valid index for this query.",
+ [ddoc_name(DesignId), ViewName]),
+ [Reason]
+ end;
+
+invalid_index_warning_int(_, _) ->
+ [].
+
+
+% warn if a large number of documents needed to be scanned per result
+% returned, implying a lot of in-memory filtering
+index_scan_warning(#execution_stats {
+ totalDocsExamined = Docs,
+ totalQuorumDocsExamined = DocsQuorum,
+ resultsReturned = ResultCount
+ }) ->
+ % Docs and DocsQuorum are mutually exclusive so it's safe to sum them
+ DocsScanned = Docs + DocsQuorum,
+ Ratio = calculate_index_scan_ratio(DocsScanned, ResultCount),
+ Threshold = config:get_integer("mango", "index_scan_warning_threshold", 10),
+ case Threshold > 0 andalso Ratio > Threshold of
+ true ->
+ couch_stats:increment_counter([mango, too_many_docs_scanned]),
+ Reason = <<"The number of documents examined is high in proportion to the number of results returned. Consider adding a more specific index to improve this.">>,
+ [Reason];
+ false -> []
+ end.
-maybe_add_warning_int(ok, _, UserAcc) ->
- UserAcc;
+calculate_index_scan_ratio(DocsScanned, 0) ->
+ DocsScanned;
-maybe_add_warning_int(Warning, UserFun, UserAcc) ->
- couch_stats:increment_counter([mango, unindexed_queries]),
- Arg = {add_key, warning, Warning},
- {_Go, UserAcc0} = UserFun(Arg, UserAcc),
- UserAcc0.
+calculate_index_scan_ratio(DocsScanned, ResultCount) ->
+ DocsScanned / ResultCount.
fmt(Format, Args) ->
diff --git a/src/mango/src/mango_cursor_text.erl b/src/mango/src/mango_cursor_text.erl
index 8938f3557..8b437e813 100644
--- a/src/mango/src/mango_cursor_text.erl
+++ b/src/mango/src/mango_cursor_text.erl
@@ -126,7 +126,7 @@ execute(Cursor, UserFun, UserAcc) ->
Arg = {add_key, bookmark, JsonBM},
{_Go, FinalUserAcc} = UserFun(Arg, LastUserAcc),
FinalUserAcc0 = mango_execution_stats:maybe_add_stats(Opts, UserFun, Stats0, FinalUserAcc),
- FinalUserAcc1 = mango_cursor:maybe_add_warning(UserFun, Cursor, FinalUserAcc0),
+ FinalUserAcc1 = mango_cursor:maybe_add_warning(UserFun, Cursor, Stats0, FinalUserAcc0),
{ok, FinalUserAcc1}
end.
@@ -178,6 +178,7 @@ handle_hit(CAcc0, Sort, Doc) ->
} = CAcc0,
CAcc1 = update_bookmark(CAcc0, Sort),
Stats1 = mango_execution_stats:incr_docs_examined(Stats),
+ couch_stats:increment_counter([mango, docs_examined]),
CAcc2 = CAcc1#cacc{execution_stats = Stats1},
case mango_selector:match(CAcc2#cacc.selector, Doc) of
true when Skip > 0 ->
diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index f1b753bd7..2266e9936 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -145,7 +145,7 @@ execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFu
{_Go, FinalUserAcc} = UserFun(Arg, LastCursor#cursor.user_acc),
Stats0 = LastCursor#cursor.execution_stats,
FinalUserAcc0 = mango_execution_stats:maybe_add_stats(Opts, UserFun, Stats0, FinalUserAcc),
- FinalUserAcc1 = mango_cursor:maybe_add_warning(UserFun, Cursor, FinalUserAcc0),
+ FinalUserAcc1 = mango_cursor:maybe_add_warning(UserFun, Cursor, Stats0, FinalUserAcc0),
{ok, FinalUserAcc1};
{error, Reason} ->
{error, Reason}
@@ -239,6 +239,7 @@ view_cb({row, Row}, #mrargs{extra = Options} = Acc) ->
Doc ->
put(mango_docs_examined, get(mango_docs_examined) + 1),
Selector = couch_util:get_value(selector, Options),
+ couch_stats:increment_counter([mango, docs_examined]),
case mango_selector:match(Selector, Doc) of
true ->
ok = rexi:stream2(ViewRow),
@@ -423,6 +424,7 @@ doc_member(Cursor, RowProps) ->
% an undefined doc was returned, indicating we should
% perform a quorum fetch
ExecutionStats1 = mango_execution_stats:incr_quorum_docs_examined(ExecutionStats),
+ couch_stats:increment_counter([mango, quorum_docs_examined]),
Id = couch_util:get_value(id, RowProps),
case mango_util:defer(fabric, open_doc, [Db, Id, Opts]) of
{ok, #doc{}=DocProps} ->
diff --git a/src/mango/src/mango_execution_stats.erl b/src/mango/src/mango_execution_stats.erl
index 7e8afd782..5878a3190 100644
--- a/src/mango/src/mango_execution_stats.erl
+++ b/src/mango/src/mango_execution_stats.erl
@@ -62,6 +62,7 @@ incr_quorum_docs_examined(Stats) ->
incr_results_returned(Stats) ->
+ couch_stats:increment_counter([mango, results_returned]),
Stats#execution_stats {
resultsReturned = Stats#execution_stats.resultsReturned + 1
}.
@@ -81,11 +82,13 @@ log_end(Stats) ->
}.
-maybe_add_stats(Opts, UserFun, Stats, UserAcc) ->
+maybe_add_stats(Opts, UserFun, Stats0, UserAcc) ->
+ Stats1 = log_end(Stats0),
+ couch_stats:update_histogram([mango, query_time], Stats1#execution_stats.executionTimeMs),
+
case couch_util:get_value(execution_stats, Opts) of
true ->
- Stats0 = log_end(Stats),
- JSONValue = to_json(Stats0),
+ JSONValue = to_json(Stats1),
Arg = {add_key, execution_stats, JSONValue},
{_Go, FinalUserAcc} = UserFun(Arg, UserAcc),
FinalUserAcc;
diff --git a/src/mango/src/mango_selector.erl b/src/mango/src/mango_selector.erl
index fffadcd20..f19c7e462 100644
--- a/src/mango/src/mango_selector.erl
+++ b/src/mango/src/mango_selector.erl
@@ -52,15 +52,19 @@ normalize(Selector) ->
% Match a selector against a #doc{} or EJSON value.
% This assumes that the Selector has been normalized.
% Returns true or false.
+match(Selector, D) ->
+ couch_stats:increment_counter([mango, evaluate_selector]),
+ match_int(Selector, D).
+
% An empty selector matches any value.
-match({[]}, _) ->
+match_int({[]}, _) ->
true;
-match(Selector, #doc{body=Body}) ->
+match_int(Selector, #doc{body=Body}) ->
match(Selector, Body, fun mango_json:cmp/2);
-match(Selector, {Props}) ->
+match_int(Selector, {Props}) ->
match(Selector, {Props}, fun mango_json:cmp/2).
% Convert each operator into a normalized version as well
@@ -570,7 +574,7 @@ match({[_, _ | _] = _Props} = Sel, _Value, _Cmp) ->
erlang:error({unnormalized_selector, Sel}).
-% Returns true if Selector requires all
+% Returns true if Selector requires all
% fields in RequiredFields to exist in any matching documents.
% For each condition in the selector, check
@@ -600,13 +604,13 @@ has_required_fields_int(Selector, RequiredFields) when not is_list(Selector) ->
% We can "see" through $and operator. Iterate
% through the list of child operators.
-has_required_fields_int([{[{<<"$and">>, Args}]}], RequiredFields)
+has_required_fields_int([{[{<<"$and">>, Args}]}], RequiredFields)
when is_list(Args) ->
has_required_fields_int(Args, RequiredFields);
% We can "see" through $or operator. Required fields
% must be covered by all children.
-has_required_fields_int([{[{<<"$or">>, Args}]} | Rest], RequiredFields)
+has_required_fields_int([{[{<<"$or">>, Args}]} | Rest], RequiredFields)
when is_list(Args) ->
Remainder0 = lists:foldl(fun(Arg, Acc) ->
% for each child test coverage against the full
@@ -623,7 +627,7 @@ has_required_fields_int([{[{<<"$or">>, Args}]} | Rest], RequiredFields)
% Handle $and operator where it has peers. Required fields
% can be covered by any child.
-has_required_fields_int([{[{<<"$and">>, Args}]} | Rest], RequiredFields)
+has_required_fields_int([{[{<<"$and">>, Args}]} | Rest], RequiredFields)
when is_list(Args) ->
Remainder = has_required_fields_int(Args, RequiredFields),
has_required_fields_int(Rest, Remainder);
View Lorry Controller Status