Add selector support for json indexes (#808)

* Add selector support for json indexes

Adds selector support to json indexes. The selector can be used to
filter what documents are added to the index. When executing a query
the index will only be used if the index is specified in the use_index
field.

6 files changed, 219 insertions, 20 deletions

diff --git a/src/mango/src/mango_cursor.erl b/src/mango/src/mango_cursor.erl
index 12ce2c93b..f36febdfc 100644
--- a/src/mango/src/mango_cursor.erl
+++ b/src/mango/src/mango_cursor.erl
@@ -17,7 +17,7 @@
     create/3,
     explain/1,
     execute/3,
-    maybe_filter_indexes/2,
+    maybe_filter_indexes_by_ddoc/2,
     maybe_add_warning/3
 ]).
 
@@ -87,10 +87,12 @@ execute(#cursor{index=Idx}=Cursor, UserFun, UserAcc) ->
     Mod:execute(Cursor, UserFun, UserAcc).
 
 
-maybe_filter_indexes(Indexes, Opts) ->
+maybe_filter_indexes_by_ddoc(Indexes, Opts) ->
     case lists:keyfind(use_index, 1, Opts) of
         {use_index, []} ->
-            Indexes;
+            %We remove any indexes that have a selector 
+            % since they are only used when specified via use_index
+            remove_indexes_with_selector(Indexes);
         {use_index, [DesignId]} ->
             filter_indexes(Indexes, DesignId);
         {use_index, [DesignId, ViewName]} ->
@@ -115,6 +117,16 @@ filter_indexes(Indexes0, DesignId, ViewName) ->
     lists:filter(FiltFun, Indexes).
 
 
+remove_indexes_with_selector(Indexes) ->
+    FiltFun = fun(Idx) -> 
+        case mango_idx:get_idx_selector(Idx) of
+            undefined -> true;
+            _ -> false
+        end
+    end,
+    lists:filter(FiltFun, Indexes).
+
+
 create_cursor(Db, Indexes, Selector, Opts) ->
     [{CursorMod, CursorModIndexes} | _] = group_indexes_by_type(Indexes),
     CursorMod:create(Db, CursorModIndexes, Selector, Opts).
diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl
index c330702b1..b8122517d 100644
--- a/src/mango/src/mango_idx.erl
+++ b/src/mango/src/mango_idx.erl
@@ -43,7 +43,8 @@
     idx_mod/1,
     to_json/1,
     delete/4,
-    get_usable_indexes/3
+    get_usable_indexes/3,
+    get_idx_selector/1
 ]).
 
 
@@ -64,7 +65,7 @@ get_usable_indexes(Db, Selector0, Opts) ->
         ?MANGO_ERROR({no_usable_index, no_indexes_defined})
     end,
 
-    FilteredIndexes = mango_cursor:maybe_filter_indexes(ExistingIndexes, Opts),
+    FilteredIndexes = mango_cursor:maybe_filter_indexes_by_ddoc(ExistingIndexes, Opts),
     if FilteredIndexes /= [] -> ok; true ->
         ?MANGO_ERROR({no_usable_index, no_index_matching_name})
     end,
@@ -367,3 +368,13 @@ filter_opts([Opt | Rest]) ->
     [Opt | filter_opts(Rest)].
 
 
+get_idx_selector(#idx{def = Def}) when Def =:= all_docs; Def =:= undefined ->
+    undefined;
+get_idx_selector(#idx{def = {Def}}) ->
+    case proplists:get_value(<<"selector">>, Def) of
+        undefined -> undefined;
+        {[]} -> undefined;
+        Selector -> Selector
+    end.
+
+
diff --git a/src/mango/src/mango_idx_view.erl b/src/mango/src/mango_idx_view.erl
index 8bad34cca..d5dcd0c07 100644
--- a/src/mango/src/mango_idx_view.erl
+++ b/src/mango/src/mango_idx_view.erl
@@ -197,6 +197,12 @@ opts() ->
         {<<"fields">>, [
             {tag, fields},
             {validator, fun mango_opts:validate_sort/1}
+        ]},
+        {<<"selector">>, [
+            {tag, selector},
+            {optional, true},
+            {default, {[]}},
+            {validator, fun mango_opts:validate_selector/1}
         ]}
     ].
 
diff --git a/src/mango/src/mango_native_proc.erl b/src/mango/src/mango_native_proc.erl
index ba17c4867..82081a976 100644
--- a/src/mango/src/mango_native_proc.erl
+++ b/src/mango/src/mango_native_proc.erl
@@ -135,26 +135,31 @@ index_doc(#st{indexes=Indexes}, Doc) ->
 
 get_index_entries({IdxProps}, Doc) ->
     {Fields} = couch_util:get_value(<<"fields">>, IdxProps),
-    Values = lists:map(fun({Field, _Dir}) ->
+    Selector = get_index_selector(IdxProps),
+    case should_index(Selector, Doc) of
+        false -> 
+            [];
+        true -> 
+            Values = get_index_values(Fields, Doc),
+            case lists:member(not_found, Values) of
+                true -> [];
+                false -> [[Values, null]]
+            end
+    end.
+
+
+get_index_values(Fields, Doc) ->
+    lists:map(fun({Field, _Dir}) ->
         case mango_doc:get_field(Doc, Field) of
             not_found -> not_found;
             bad_path -> not_found;
-            Else -> Else
+            Value -> Value
         end
-    end, Fields),
-    case lists:member(not_found, Values) of
-        true ->
-            [];
-        false ->
-            [[Values, null]]
-    end.
+    end, Fields).
 
 
 get_text_entries({IdxProps}, Doc) ->
-    Selector = case couch_util:get_value(<<"selector">>, IdxProps) of
-        [] -> {[]};
-        Else -> Else
-    end,
+    Selector = get_index_selector(IdxProps),
     case should_index(Selector, Doc) of
         true ->
             get_text_entries0(IdxProps, Doc);
@@ -163,6 +168,13 @@ get_text_entries({IdxProps}, Doc) ->
     end.
 
 
+get_index_selector(IdxProps) ->
+    case couch_util:get_value(<<"selector">>, IdxProps) of
+        [] -> {[]};
+        Else -> Else
+    end.
+
+
 get_text_entries0(IdxProps, Doc) ->
     DefaultEnabled = get_default_enabled(IdxProps),
     IndexArrayLengths = get_index_array_lengths(IdxProps),
diff --git a/src/mango/test/16-index-selectors.py b/src/mango/test/16-index-selectors.py
new file mode 100644
index 000000000..b18945609
--- /dev/null
+++ b/src/mango/test/16-index-selectors.py
@@ -0,0 +1,156 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+import copy
+import mango
+import unittest
+
+DOCS = [
+    {
+        "_id": "100",
+        "name": "Jimi",
+        "location": "AUS",
+        "user_id": 1,
+        "same": "value"
+    },
+    {
+        "_id": "200",
+        "name": "Eddie",
+        "location": "BRA",
+        "user_id": 2,
+        "same": "value"
+    },
+    {
+        "_id": "300",
+        "name": "Harry",
+        "location": "CAN",
+        "user_id":3,
+        "same": "value"
+    },
+    {
+        "_id": "400",
+        "name": "Eddie",
+        "location": "DEN",
+        "user_id":4,
+        "same": "value"
+    },
+    {
+        "_id": "500",
+        "name": "Jones",
+        "location": "ETH",
+        "user_id":5,
+        "same": "value"
+    },
+    {
+        "_id": "600",
+        "name": "Winnifried",
+        "location": "FRA",
+        "user_id":6,
+        "same": "value"
+    },
+    {
+        "_id": "700",
+        "name": "Marilyn",
+        "location": "GHA",
+        "user_id":7,
+        "same": "value"
+    },
+    {
+        "_id": "800",
+        "name": "Sandra",
+        "location": "ZAR",
+        "user_id":8,
+        "same": "value"
+    },
+]
+
+class IndexSelectorJson(mango.DbPerClass):
+    def setUp(self):
+        self.db.recreate()
+        self.db.save_docs(copy.deepcopy(DOCS))
+
+    def test_saves_selector_in_index(self):
+        selector = {"location": {"$gte": "FRA"}}
+        self.db.create_index(["location"], selector=selector)
+        indexes = self.db.list_indexes()
+        self.assertEqual(indexes[1]["def"]["selector"], selector)
+
+    def test_uses_partial_index_for_query_selector(self):
+        selector = {"location": {"$gte": "FRA"}}
+        self.db.create_index(["location"], selector=selector, ddoc="Selected", name="Selected")
+        resp = self.db.find(selector, explain=True, use_index='Selected')
+        self.assertEqual(resp["index"]["name"], "Selected")
+        docs = self.db.find(selector, use_index='Selected')
+        self.assertEqual(len(docs), 3)
+
+    def test_uses_partial_index_with_different_selector(self):
+        selector = {"location": {"$gte": "FRA"}}
+        selector2 = {"location": {"$gte": "A"}}
+        self.db.create_index(["location"], selector=selector, ddoc="Selected", name="Selected")
+        resp = self.db.find(selector2, explain=True, use_index='Selected')
+        self.assertEqual(resp["index"]["name"], "Selected")
+        docs = self.db.find(selector2, use_index='Selected')
+        self.assertEqual(len(docs), 3)
+
+    def test_doesnot_use_selector_when_not_specified(self):
+        selector = {"location": {"$gte": "FRA"}}
+        self.db.create_index(["location"], selector=selector, ddoc="Selected", name="Selected")
+        resp = self.db.find(selector, explain=True)
+        self.assertEqual(resp["index"]["name"], "_all_docs")
+
+    def test_doesnot_use_selector_when_not_specified_with_index(self):
+        selector = {"location": {"$gte": "FRA"}}
+        self.db.create_index(["location"], selector=selector, ddoc="Selected", name="Selected")
+        self.db.create_index(["location"], name="NotSelected")
+        resp = self.db.find(selector, explain=True)
+        self.assertEqual(resp["index"]["name"], "NotSelected")
+
+    @unittest.skipUnless(mango.has_text_service(), "requires text service")
+    def test_text_saves_selector_in_index(self):
+        selector = {"location": {"$gte": "FRA"}}
+        self.db.create_text_index(fields=[{"name":"location", "type":"string"}], selector=selector)
+        indexes = self.db.list_indexes()
+        self.assertEqual(indexes[1]["def"]["selector"], selector)
+
+    @unittest.skipUnless(mango.has_text_service(), "requires text service")
+    def test_text_uses_partial_index_for_query_selector(self):
+        selector = {"location": {"$gte": "FRA"}}
+        self.db.create_text_index(fields=[{"name":"location", "type":"string"}], selector=selector, ddoc="Selected", name="Selected")
+        resp = self.db.find(selector, explain=True, use_index='Selected')
+        self.assertEqual(resp["index"]["name"], "Selected")
+        docs = self.db.find(selector, use_index='Selected', fields=['_id', 'location'])
+        self.assertEqual(len(docs), 3)
+
+    @unittest.skipUnless(mango.has_text_service(), "requires text service")
+    def test_text_uses_partial_index_with_different_selector(self):
+        selector = {"location": {"$gte": "FRA"}}
+        selector2 = {"location": {"$gte": "A"}}
+        self.db.create_text_index(fields=[{"name":"location", "type":"string"}], selector=selector, ddoc="Selected", name="Selected")
+        resp = self.db.find(selector2, explain=True, use_index='Selected')
+        self.assertEqual(resp["index"]["name"], "Selected")
+        docs = self.db.find(selector2, use_index='Selected')
+        self.assertEqual(len(docs), 3)
+
+    @unittest.skipUnless(mango.has_text_service(), "requires text service")
+    def test_text_doesnot_use_selector_when_not_specified(self):
+        selector = {"location": {"$gte": "FRA"}}
+        self.db.create_text_index(fields=[{"name":"location", "type":"string"}], selector=selector, ddoc="Selected", name="Selected")
+        resp = self.db.find(selector, explain=True)
+        self.assertEqual(resp["index"]["name"], "_all_docs")
+
+    @unittest.skipUnless(mango.has_text_service(), "requires text service")
+    def test_text_doesnot_use_selector_when_not_specified_with_index(self):
+        selector = {"location": {"$gte": "FRA"}}
+        self.db.create_text_index(fields=[{"name":"location", "type":"string"}], selector=selector, ddoc="Selected", name="Selected")
+        self.db.create_text_index(fields=[{"name":"location", "type":"string"}], name="NotSelected")
+        resp = self.db.find(selector, explain=True)
+        self.assertEqual(resp["index"]["name"], "NotSelected")
\ No newline at end of file
diff --git a/src/mango/test/mango.py b/src/mango/test/mango.py
index 69b3649ee..2c8971485 100644
--- a/src/mango/test/mango.py
+++ b/src/mango/test/mango.py
@@ -84,7 +84,7 @@ class Database(object):
         r.raise_for_status()
         return r.json()
 
-    def create_index(self, fields, idx_type="json", name=None, ddoc=None):
+    def create_index(self, fields, idx_type="json", name=None, ddoc=None, selector=None):
         body = {
             "index": {
                 "fields": fields
@@ -96,6 +96,8 @@ class Database(object):
             body["name"] = name
         if ddoc is not None:
             body["ddoc"] = ddoc
+        if selector is not None:
+            body["index"]["selector"] = selector
         body = json.dumps(body)
         r = self.sess.post(self.path("_index"), data=body)
         r.raise_for_status()
@@ -120,7 +122,7 @@ class Database(object):
         if index_array_lengths is not None:
             body["index"]["index_array_lengths"] = index_array_lengths
         if selector is not None:
-            body["selector"] = selector
+            body["index"]["selector"] = selector
         if fields is not None:
             body["index"]["fields"] = fields
         if ddoc is not None: