Fix maximum key value when using JSON indexes (#881)

Mango previously constrained range queries against JSON indexes
(map/reduce views) to startkey=[]&endkey=[{}]. In Mango, JSON
index keys are always compound (i.e. always arrays), but this
restriction resulted in Mango failing to match documents where
the indexed value was an object.

For example, an index with keys:

[1],
[2],
[{"foo": 3}]

would be restricted such that only [1] and [2] were returned
if a range query was issued.

On its own, this behaviour isn't necessarily unintuitive, but
it is different from the behaviour of a non-indexed Mango
query, so the query results would change in the presence of an
index.

Additonally, it prevented operators or selectors which explicitly
depend on a full index scan (such as $exists) from returning a
complete result set.

This commit changes the maximum range boundary from {} to a 
value that collates higher than any JSON object, so all 
array/compound keys will be included.

Note that this uses an invalid UTF-8 character, so we depend
on the view engine not barfing when this is passed as a
parameter. In addition, we can't represent the value in JSON
so we need to subtitute is when returning a query plan
in the _explain endpoint.

5 files changed, 128 insertions, 5 deletions

diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
index 59dd52226..3fcec07be 100644
--- a/src/mango/src/mango_cursor_view.erl
+++ b/src/mango/src/mango_cursor_view.erl
@@ -29,7 +29,7 @@
 -include_lib("couch/include/couch_db.hrl").
 -include_lib("couch_mrview/include/couch_mrview.hrl").
 -include("mango_cursor.hrl").
-
+-include("mango_idx_view.hrl").
 
 create(Db, Indexes, Selector, Opts) ->
     FieldRanges = mango_idx_view:field_ranges(Selector),
@@ -61,18 +61,37 @@ explain(Cursor) ->
 
     BaseArgs = base_args(Cursor),
     Args = apply_opts(Opts, BaseArgs),
+
     [{mrargs, {[
         {include_docs, Args#mrargs.include_docs},
         {view_type, Args#mrargs.view_type},
         {reduce, Args#mrargs.reduce},
         {start_key, Args#mrargs.start_key},
-        {end_key, Args#mrargs.end_key},
+        {end_key, maybe_replace_max_json(Args#mrargs.end_key)},
         {direction, Args#mrargs.direction},
         {stable, Args#mrargs.stable},
         {update, Args#mrargs.update}
     ]}}].
 
 
+% replace internal values that cannot
+% be represented as a valid UTF-8 string
+% with a token for JSON serialization
+maybe_replace_max_json([]) ->
+    [];
+
+maybe_replace_max_json(?MAX_STR) ->
+    <<"<MAX>">>;
+
+maybe_replace_max_json([H | T] = EndKey) when is_list(EndKey) ->
+    H1 = if H == ?MAX_JSON_OBJ -> <<"<MAX>">>;
+            true -> H
+    end,
+    [H1 | maybe_replace_max_json(T)];
+
+maybe_replace_max_json(EndKey) ->
+    EndKey.
+
 base_args(#cursor{index = Idx} = Cursor) ->
     #mrargs{
         view_type = map,
diff --git a/src/mango/src/mango_idx_view.erl b/src/mango/src/mango_idx_view.erl
index 8331683a0..f1041bbaf 100644
--- a/src/mango/src/mango_idx_view.erl
+++ b/src/mango/src/mango_idx_view.erl
@@ -34,6 +34,7 @@
 -include_lib("couch/include/couch_db.hrl").
 -include("mango.hrl").
 -include("mango_idx.hrl").
+-include("mango_idx_view.hrl").
 
 
 validate_new(#idx{}=Idx, _Db) ->
@@ -163,11 +164,11 @@ start_key([{'$eq', Key, '$eq', Key} | Rest]) ->
 
 
 end_key([]) ->
-    [{[]}];
+    [?MAX_JSON_OBJ];
 end_key([{_, _, '$lt', Key} | Rest]) ->
     case mango_json:special(Key) of
         true ->
-            [{[]}];
+            [?MAX_JSON_OBJ];
         false ->
             [Key | end_key(Rest)]
     end;
diff --git a/src/mango/src/mango_idx_view.hrl b/src/mango/src/mango_idx_view.hrl
new file mode 100644
index 000000000..0d213e56e
--- /dev/null
+++ b/src/mango/src/mango_idx_view.hrl
@@ -0,0 +1,13 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-define(MAX_JSON_OBJ, {<<255, 255, 255, 255>>}).
\ No newline at end of file
diff --git a/src/mango/test/02-basic-find-test.py b/src/mango/test/02-basic-find-test.py
index 72a4e3feb..82554a112 100644
--- a/src/mango/test/02-basic-find-test.py
+++ b/src/mango/test/02-basic-find-test.py
@@ -319,5 +319,5 @@ class BasicFindTests(mango.UserDocsTests):
         assert explain["mrargs"]["update"] == True
         assert explain["mrargs"]["reduce"] == False
         assert explain["mrargs"]["start_key"] == [0]
-        assert explain["mrargs"]["end_key"] == [{}]
+        assert explain["mrargs"]["end_key"] == ["<MAX>"]
         assert explain["mrargs"]["include_docs"] == True
diff --git a/src/mango/test/17-multi-type-value-test.py b/src/mango/test/17-multi-type-value-test.py
new file mode 100644
index 000000000..d838447d5
--- /dev/null
+++ b/src/mango/test/17-multi-type-value-test.py
@@ -0,0 +1,90 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+import copy
+import mango
+import unittest
+
+DOCS = [
+    {
+        "_id": "1",
+        "name": "Jimi",
+        "age": 10
+    },
+    {
+        "_id": "2",
+        "name": {"forename":"Eddie"},
+        "age": 20
+    },
+    {
+        "_id": "3",
+        "name": None,
+        "age": 30
+    },
+    {
+        "_id": "4",
+        "name": 1,
+        "age": 40
+    },
+    {
+        "_id": "5",
+        "forename": "Sam",
+        "age": 50
+    }
+]
+
+
+class MultiValueFieldTests:
+
+    def test_can_query_with_name(self):
+        docs = self.db.find({"name": {"$exists": True}})
+        self.assertEqual(len(docs), 4)
+        for d in docs:
+            self.assertIn("name", d)
+
+    def test_can_query_with_name_subfield(self):
+        docs = self.db.find({"name.forename": {"$exists": True}})
+        self.assertEqual(len(docs), 1)
+        self.assertEqual(docs[0]["_id"], "2")
+
+    def test_can_query_with_name_range(self):
+        docs = self.db.find({"name": {"$gte": 0}})
+        # expect to include "Jimi", 1 and {"forename":"Eddie"}
+        self.assertEqual(len(docs), 3)
+        for d in docs:
+            self.assertIn("name", d)
+
+    def test_can_query_with_age_and_name_range(self):
+        docs = self.db.find({"age": {"$gte": 0, "$lt": 40}, "name": {"$gte": 0}})
+        # expect to include "Jimi", 1 and {"forename":"Eddie"}
+        self.assertEqual(len(docs), 2)
+        for d in docs:
+            self.assertIn("name", d)
+
+
+
+class MultiValueFieldJSONTests(mango.DbPerClass, MultiValueFieldTests):
+    def setUp(self):
+        self.db.recreate()
+        self.db.save_docs(copy.deepcopy(DOCS))
+        self.db.create_index(["name"])
+        self.db.create_index(["age", "name"])
+
+# @unittest.skipUnless(mango.has_text_service(), "requires text service")
+# class MultiValueFieldTextTests(MultiValueFieldDocsNoIndexes, OperatorTests):
+#     pass
+
+
+class MultiValueFieldAllDocsTests(mango.DbPerClass, MultiValueFieldTests):
+    def setUp(self):
+        self.db.recreate()
+        self.db.save_docs(copy.deepcopy(DOCS))