diff options
author | garren smith <garren.smith@gmail.com> | 2017-09-14 14:28:42 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2017-09-14 14:28:42 +0200 |
commit | ed6ec66972f268b6d6d86b55730e73cc4ab54ceb (patch) | |
tree | dce5c8dfd0a28de8618bc79cf1c653725c4d315f | |
parent | c622e17ad1b5b2d8bc8c6f22ca17a7168a07dcc3 (diff) | |
download | couchdb-ed6ec66972f268b6d6d86b55730e73cc4ab54ceb.tar.gz |
Add selector support for json indexes (#808)
* Add selector support for json indexes
Adds selector support to json indexes. The selector can be used to
filter what documents are added to the index. When executing a query
the index will only be used if the index is specified in the use_index
field.
-rw-r--r-- | src/mango/src/mango_cursor.erl | 18 | ||||
-rw-r--r-- | src/mango/src/mango_idx.erl | 15 | ||||
-rw-r--r-- | src/mango/src/mango_idx_view.erl | 6 | ||||
-rw-r--r-- | src/mango/src/mango_native_proc.erl | 38 | ||||
-rw-r--r-- | src/mango/test/16-index-selectors.py | 156 | ||||
-rw-r--r-- | src/mango/test/mango.py | 6 |
6 files changed, 219 insertions, 20 deletions
diff --git a/src/mango/src/mango_cursor.erl b/src/mango/src/mango_cursor.erl index 12ce2c93b..f36febdfc 100644 --- a/src/mango/src/mango_cursor.erl +++ b/src/mango/src/mango_cursor.erl @@ -17,7 +17,7 @@ create/3, explain/1, execute/3, - maybe_filter_indexes/2, + maybe_filter_indexes_by_ddoc/2, maybe_add_warning/3 ]). @@ -87,10 +87,12 @@ execute(#cursor{index=Idx}=Cursor, UserFun, UserAcc) -> Mod:execute(Cursor, UserFun, UserAcc). -maybe_filter_indexes(Indexes, Opts) -> +maybe_filter_indexes_by_ddoc(Indexes, Opts) -> case lists:keyfind(use_index, 1, Opts) of {use_index, []} -> - Indexes; + %We remove any indexes that have a selector + % since they are only used when specified via use_index + remove_indexes_with_selector(Indexes); {use_index, [DesignId]} -> filter_indexes(Indexes, DesignId); {use_index, [DesignId, ViewName]} -> @@ -115,6 +117,16 @@ filter_indexes(Indexes0, DesignId, ViewName) -> lists:filter(FiltFun, Indexes). +remove_indexes_with_selector(Indexes) -> + FiltFun = fun(Idx) -> + case mango_idx:get_idx_selector(Idx) of + undefined -> true; + _ -> false + end + end, + lists:filter(FiltFun, Indexes). + + create_cursor(Db, Indexes, Selector, Opts) -> [{CursorMod, CursorModIndexes} | _] = group_indexes_by_type(Indexes), CursorMod:create(Db, CursorModIndexes, Selector, Opts). diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl index c330702b1..b8122517d 100644 --- a/src/mango/src/mango_idx.erl +++ b/src/mango/src/mango_idx.erl @@ -43,7 +43,8 @@ idx_mod/1, to_json/1, delete/4, - get_usable_indexes/3 + get_usable_indexes/3, + get_idx_selector/1 ]). @@ -64,7 +65,7 @@ get_usable_indexes(Db, Selector0, Opts) -> ?MANGO_ERROR({no_usable_index, no_indexes_defined}) end, - FilteredIndexes = mango_cursor:maybe_filter_indexes(ExistingIndexes, Opts), + FilteredIndexes = mango_cursor:maybe_filter_indexes_by_ddoc(ExistingIndexes, Opts), if FilteredIndexes /= [] -> ok; true -> ?MANGO_ERROR({no_usable_index, no_index_matching_name}) end, @@ -367,3 +368,13 @@ filter_opts([Opt | Rest]) -> [Opt | filter_opts(Rest)]. +get_idx_selector(#idx{def = Def}) when Def =:= all_docs; Def =:= undefined -> + undefined; +get_idx_selector(#idx{def = {Def}}) -> + case proplists:get_value(<<"selector">>, Def) of + undefined -> undefined; + {[]} -> undefined; + Selector -> Selector + end. + + diff --git a/src/mango/src/mango_idx_view.erl b/src/mango/src/mango_idx_view.erl index 8bad34cca..d5dcd0c07 100644 --- a/src/mango/src/mango_idx_view.erl +++ b/src/mango/src/mango_idx_view.erl @@ -197,6 +197,12 @@ opts() -> {<<"fields">>, [ {tag, fields}, {validator, fun mango_opts:validate_sort/1} + ]}, + {<<"selector">>, [ + {tag, selector}, + {optional, true}, + {default, {[]}}, + {validator, fun mango_opts:validate_selector/1} ]} ]. diff --git a/src/mango/src/mango_native_proc.erl b/src/mango/src/mango_native_proc.erl index ba17c4867..82081a976 100644 --- a/src/mango/src/mango_native_proc.erl +++ b/src/mango/src/mango_native_proc.erl @@ -135,26 +135,31 @@ index_doc(#st{indexes=Indexes}, Doc) -> get_index_entries({IdxProps}, Doc) -> {Fields} = couch_util:get_value(<<"fields">>, IdxProps), - Values = lists:map(fun({Field, _Dir}) -> + Selector = get_index_selector(IdxProps), + case should_index(Selector, Doc) of + false -> + []; + true -> + Values = get_index_values(Fields, Doc), + case lists:member(not_found, Values) of + true -> []; + false -> [[Values, null]] + end + end. + + +get_index_values(Fields, Doc) -> + lists:map(fun({Field, _Dir}) -> case mango_doc:get_field(Doc, Field) of not_found -> not_found; bad_path -> not_found; - Else -> Else + Value -> Value end - end, Fields), - case lists:member(not_found, Values) of - true -> - []; - false -> - [[Values, null]] - end. + end, Fields). get_text_entries({IdxProps}, Doc) -> - Selector = case couch_util:get_value(<<"selector">>, IdxProps) of - [] -> {[]}; - Else -> Else - end, + Selector = get_index_selector(IdxProps), case should_index(Selector, Doc) of true -> get_text_entries0(IdxProps, Doc); @@ -163,6 +168,13 @@ get_text_entries({IdxProps}, Doc) -> end. +get_index_selector(IdxProps) -> + case couch_util:get_value(<<"selector">>, IdxProps) of + [] -> {[]}; + Else -> Else + end. + + get_text_entries0(IdxProps, Doc) -> DefaultEnabled = get_default_enabled(IdxProps), IndexArrayLengths = get_index_array_lengths(IdxProps), diff --git a/src/mango/test/16-index-selectors.py b/src/mango/test/16-index-selectors.py new file mode 100644 index 000000000..b18945609 --- /dev/null +++ b/src/mango/test/16-index-selectors.py @@ -0,0 +1,156 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +import copy +import mango +import unittest + +DOCS = [ + { + "_id": "100", + "name": "Jimi", + "location": "AUS", + "user_id": 1, + "same": "value" + }, + { + "_id": "200", + "name": "Eddie", + "location": "BRA", + "user_id": 2, + "same": "value" + }, + { + "_id": "300", + "name": "Harry", + "location": "CAN", + "user_id":3, + "same": "value" + }, + { + "_id": "400", + "name": "Eddie", + "location": "DEN", + "user_id":4, + "same": "value" + }, + { + "_id": "500", + "name": "Jones", + "location": "ETH", + "user_id":5, + "same": "value" + }, + { + "_id": "600", + "name": "Winnifried", + "location": "FRA", + "user_id":6, + "same": "value" + }, + { + "_id": "700", + "name": "Marilyn", + "location": "GHA", + "user_id":7, + "same": "value" + }, + { + "_id": "800", + "name": "Sandra", + "location": "ZAR", + "user_id":8, + "same": "value" + }, +] + +class IndexSelectorJson(mango.DbPerClass): + def setUp(self): + self.db.recreate() + self.db.save_docs(copy.deepcopy(DOCS)) + + def test_saves_selector_in_index(self): + selector = {"location": {"$gte": "FRA"}} + self.db.create_index(["location"], selector=selector) + indexes = self.db.list_indexes() + self.assertEqual(indexes[1]["def"]["selector"], selector) + + def test_uses_partial_index_for_query_selector(self): + selector = {"location": {"$gte": "FRA"}} + self.db.create_index(["location"], selector=selector, ddoc="Selected", name="Selected") + resp = self.db.find(selector, explain=True, use_index='Selected') + self.assertEqual(resp["index"]["name"], "Selected") + docs = self.db.find(selector, use_index='Selected') + self.assertEqual(len(docs), 3) + + def test_uses_partial_index_with_different_selector(self): + selector = {"location": {"$gte": "FRA"}} + selector2 = {"location": {"$gte": "A"}} + self.db.create_index(["location"], selector=selector, ddoc="Selected", name="Selected") + resp = self.db.find(selector2, explain=True, use_index='Selected') + self.assertEqual(resp["index"]["name"], "Selected") + docs = self.db.find(selector2, use_index='Selected') + self.assertEqual(len(docs), 3) + + def test_doesnot_use_selector_when_not_specified(self): + selector = {"location": {"$gte": "FRA"}} + self.db.create_index(["location"], selector=selector, ddoc="Selected", name="Selected") + resp = self.db.find(selector, explain=True) + self.assertEqual(resp["index"]["name"], "_all_docs") + + def test_doesnot_use_selector_when_not_specified_with_index(self): + selector = {"location": {"$gte": "FRA"}} + self.db.create_index(["location"], selector=selector, ddoc="Selected", name="Selected") + self.db.create_index(["location"], name="NotSelected") + resp = self.db.find(selector, explain=True) + self.assertEqual(resp["index"]["name"], "NotSelected") + + @unittest.skipUnless(mango.has_text_service(), "requires text service") + def test_text_saves_selector_in_index(self): + selector = {"location": {"$gte": "FRA"}} + self.db.create_text_index(fields=[{"name":"location", "type":"string"}], selector=selector) + indexes = self.db.list_indexes() + self.assertEqual(indexes[1]["def"]["selector"], selector) + + @unittest.skipUnless(mango.has_text_service(), "requires text service") + def test_text_uses_partial_index_for_query_selector(self): + selector = {"location": {"$gte": "FRA"}} + self.db.create_text_index(fields=[{"name":"location", "type":"string"}], selector=selector, ddoc="Selected", name="Selected") + resp = self.db.find(selector, explain=True, use_index='Selected') + self.assertEqual(resp["index"]["name"], "Selected") + docs = self.db.find(selector, use_index='Selected', fields=['_id', 'location']) + self.assertEqual(len(docs), 3) + + @unittest.skipUnless(mango.has_text_service(), "requires text service") + def test_text_uses_partial_index_with_different_selector(self): + selector = {"location": {"$gte": "FRA"}} + selector2 = {"location": {"$gte": "A"}} + self.db.create_text_index(fields=[{"name":"location", "type":"string"}], selector=selector, ddoc="Selected", name="Selected") + resp = self.db.find(selector2, explain=True, use_index='Selected') + self.assertEqual(resp["index"]["name"], "Selected") + docs = self.db.find(selector2, use_index='Selected') + self.assertEqual(len(docs), 3) + + @unittest.skipUnless(mango.has_text_service(), "requires text service") + def test_text_doesnot_use_selector_when_not_specified(self): + selector = {"location": {"$gte": "FRA"}} + self.db.create_text_index(fields=[{"name":"location", "type":"string"}], selector=selector, ddoc="Selected", name="Selected") + resp = self.db.find(selector, explain=True) + self.assertEqual(resp["index"]["name"], "_all_docs") + + @unittest.skipUnless(mango.has_text_service(), "requires text service") + def test_text_doesnot_use_selector_when_not_specified_with_index(self): + selector = {"location": {"$gte": "FRA"}} + self.db.create_text_index(fields=[{"name":"location", "type":"string"}], selector=selector, ddoc="Selected", name="Selected") + self.db.create_text_index(fields=[{"name":"location", "type":"string"}], name="NotSelected") + resp = self.db.find(selector, explain=True) + self.assertEqual(resp["index"]["name"], "NotSelected")
\ No newline at end of file diff --git a/src/mango/test/mango.py b/src/mango/test/mango.py index 69b3649ee..2c8971485 100644 --- a/src/mango/test/mango.py +++ b/src/mango/test/mango.py @@ -84,7 +84,7 @@ class Database(object): r.raise_for_status() return r.json() - def create_index(self, fields, idx_type="json", name=None, ddoc=None): + def create_index(self, fields, idx_type="json", name=None, ddoc=None, selector=None): body = { "index": { "fields": fields @@ -96,6 +96,8 @@ class Database(object): body["name"] = name if ddoc is not None: body["ddoc"] = ddoc + if selector is not None: + body["index"]["selector"] = selector body = json.dumps(body) r = self.sess.post(self.path("_index"), data=body) r.raise_for_status() @@ -120,7 +122,7 @@ class Database(object): if index_array_lengths is not None: body["index"]["index_array_lengths"] = index_array_lengths if selector is not None: - body["selector"] = selector + body["index"]["selector"] = selector if fields is not None: body["index"]["fields"] = fields if ddoc is not None: |