summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJan Lehnardt <jan@apache.org>2017-04-01 19:38:52 +0200
committerJan Lehnardt <jan@apache.org>2017-04-01 19:38:52 +0200
commit8f39d4191040e7519a573ce1b3cdf5b674bf8e4f (patch)
tree774f85e6fcfa9d2fc91b4e28ec5120c73097d2a8
parentc525ad3cb175afed68af7da2c37b9505abccc5b2 (diff)
parent312e2c45535913c190cdef51f6ea65066ccd89dc (diff)
downloadcouchdb-8f39d4191040e7519a573ce1b3cdf5b674bf8e4f.tar.gz
Add 'src/mango/' from commit '312e2c45535913c190cdef51f6ea65066ccd89dc'
git-subtree-dir: src/mango git-subtree-mainline: c525ad3cb175afed68af7da2c37b9505abccc5b2 git-subtree-split: 312e2c45535913c190cdef51f6ea65066ccd89dc
-rw-r--r--src/mango/.gitignore5
-rw-r--r--src/mango/.travis.yml29
-rw-r--r--src/mango/LICENSE.txt202
-rw-r--r--src/mango/Makefile56
-rw-r--r--src/mango/README.md372
-rw-r--r--src/mango/TODO.md9
-rw-r--r--src/mango/rebar.config.script27
-rw-r--r--src/mango/src/mango.app.src26
-rw-r--r--src/mango/src/mango.hrl13
-rw-r--r--src/mango/src/mango_app.erl21
-rw-r--r--src/mango/src/mango_crud.erl177
-rw-r--r--src/mango/src/mango_cursor.erl129
-rw-r--r--src/mango/src/mango_cursor.hrl24
-rw-r--r--src/mango/src/mango_cursor_special.erl61
-rw-r--r--src/mango/src/mango_cursor_text.erl306
-rw-r--r--src/mango/src/mango_cursor_view.erl273
-rw-r--r--src/mango/src/mango_doc.erl537
-rw-r--r--src/mango/src/mango_epi.erl48
-rw-r--r--src/mango/src/mango_error.erl372
-rw-r--r--src/mango/src/mango_fields.erl55
-rw-r--r--src/mango/src/mango_httpd.erl305
-rw-r--r--src/mango/src/mango_httpd_handlers.erl24
-rw-r--r--src/mango/src/mango_idx.erl369
-rw-r--r--src/mango/src/mango_idx.hrl20
-rw-r--r--src/mango/src/mango_idx_special.erl98
-rw-r--r--src/mango/src/mango_idx_text.erl422
-rw-r--r--src/mango/src/mango_idx_view.erl490
-rw-r--r--src/mango/src/mango_json.erl121
-rw-r--r--src/mango/src/mango_native_proc.erl347
-rw-r--r--src/mango/src/mango_opts.erl314
-rw-r--r--src/mango/src/mango_selector.erl568
-rw-r--r--src/mango/src/mango_selector_text.erl416
-rw-r--r--src/mango/src/mango_sort.erl75
-rw-r--r--src/mango/src/mango_sup.erl24
-rw-r--r--src/mango/src/mango_util.erl423
-rw-r--r--src/mango/test/01-index-crud-test.py302
-rw-r--r--src/mango/test/02-basic-find-test.py266
-rw-r--r--src/mango/test/03-operator-test.py143
-rw-r--r--src/mango/test/04-key-tests.py151
-rw-r--r--src/mango/test/05-index-selection-test.py178
-rw-r--r--src/mango/test/06-basic-text-test.py653
-rw-r--r--src/mango/test/06-text-default-field-test.py73
-rw-r--r--src/mango/test/07-text-custom-field-list-test.py158
-rw-r--r--src/mango/test/08-text-limit-test.py137
-rw-r--r--src/mango/test/09-text-sort-test.py101
-rw-r--r--src/mango/test/10-disable-array-length-field-test.py42
-rw-r--r--src/mango/test/11-ignore-design-docs.py39
-rw-r--r--src/mango/test/README.md12
-rw-r--r--src/mango/test/friend_docs.py604
-rw-r--r--src/mango/test/limit_docs.py408
-rw-r--r--src/mango/test/mango.py245
-rw-r--r--src/mango/test/user_docs.py490
52 files changed, 10760 insertions, 0 deletions
diff --git a/src/mango/.gitignore b/src/mango/.gitignore
new file mode 100644
index 000000000..446945396
--- /dev/null
+++ b/src/mango/.gitignore
@@ -0,0 +1,5 @@
+.rebar/
+ebin/
+test/*.pyc
+venv/
+.eunit
diff --git a/src/mango/.travis.yml b/src/mango/.travis.yml
new file mode 100644
index 000000000..d6130128b
--- /dev/null
+++ b/src/mango/.travis.yml
@@ -0,0 +1,29 @@
+language: erlang
+
+before_install:
+ - sudo apt-get update -qq
+ - sudo apt-get -y install libmozjs-dev python-virtualenv
+ - git clone --depth=1 https://github.com/apache/couchdb
+ - cd couchdb
+ - ./configure --disable-docs --disable-fauxton
+ - cp -R ../src ./src/mango
+ - make
+ - cd ..
+ - couchdb/dev/run -n 1 --admin=testuser:testpass &
+ - sleep 10
+
+before_script:
+ - make venv
+ - source venv/bin/activate
+ - make pip-install
+
+matrix:
+ include:
+ - otp_release: 18.1
+ python: 2.7
+ - otp_release: 17.5
+ python: 2.7
+ - otp_release: R16B03-1
+ python: 2.7
+
+cache: apt
diff --git a/src/mango/LICENSE.txt b/src/mango/LICENSE.txt
new file mode 100644
index 000000000..b47557aaf
--- /dev/null
+++ b/src/mango/LICENSE.txt
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright 2014 IBM Corporation
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/src/mango/Makefile b/src/mango/Makefile
new file mode 100644
index 000000000..1b2a50452
--- /dev/null
+++ b/src/mango/Makefile
@@ -0,0 +1,56 @@
+REBAR?=rebar
+
+
+.PHONY: all
+# target: all - Makes everything
+all: build
+
+
+.PHONY: build
+# target: build - Builds the project
+build:
+ $(REBAR) compile
+
+
+.PHONY: check
+# target: check - Checks if project builds and passes all the tests
+check: build test
+
+
+.PHONY: clean
+# target: clean - Prints this help
+clean:
+ $(REBAR) clean
+ rm -f test/*.pyc
+
+
+.PHONY: distclean
+# target: distclean - Removes all unversioned files
+distclean: clean
+ git clean -fxd
+
+
+.PHONY: help
+# target: help - Prints this help
+help:
+ @egrep "^# target:" Makefile | sed -e 's/^# target: //g' | sort
+
+
+.PHONY: test
+# target: test - Runs test suite
+test:
+ nosetests
+
+
+.PHONY: pip-install
+# target: pip-install - Installs requires Python packages
+pip-install:
+ pip install nose requests
+ pip install hypothesis
+
+
+.PHONY: venv
+# target: venv - Initializes virtual environment (requires virtualenv)
+venv:
+ virtualenv --python=python2.7 venv
+ @echo "VirtualEnv has been created. Don't forget to run . venv/bin/active"
diff --git a/src/mango/README.md b/src/mango/README.md
new file mode 100644
index 000000000..4c4bb60a6
--- /dev/null
+++ b/src/mango/README.md
@@ -0,0 +1,372 @@
+Mango
+=====
+
+A MongoDB inspired query language interface for Apache CouchDB.
+
+
+Motivation
+----------
+
+Mango provides a single HTTP API endpoint that accepts JSON bodies via HTTP POST. These bodies provide a set of instructions that will be handled with the results being returned to the client in the same order as they were specified. The general principle of this API is to be simple to implement on the client side while providing users a more natural conversion to Apache CouchDB than would otherwise exist using the standard RESTful HTTP interface that already exists.
+
+
+Actions
+-------
+
+The general API exposes a set of actions that are similar to what MongoDB exposes (although not all of MongoDB's API is supported). These are meant to be loosely and obviously inspired by MongoDB but without too much attention to maintaining the exact behavior.
+
+Each action is specified as a JSON object with a number of keys that affect the behavior. Each action object has at least one field named "action" which must
+have a string value indicating the action to be performed. For each action there are zero or more fields that will affect behavior. Some of these fields are required and some are optional.
+
+For convenience, the HTTP API will accept a JSON body that is either a single JSON object which specifies a single action or a JSON array that specifies a list of actions that will then be invoked serially. While multiple commands can be batched into a single HTTP request, there are no guarantees about atomicity or isolation for a batch of commands.
+
+Activating Query on a cluster
+--------------------------------------------
+
+Query can be enabled by setting the following config:
+
+```
+rpc:multicall(config, set, ["native_query_servers", "query", "{mango_native_proc, start_link, []}"]).
+```
+
+HTTP API
+========
+
+This API adds a single URI endpoint to the existing CouchDB HTTP API. Creating databases, authentication, Map/Reduce views, etc are all still supported exactly as currently document. No existing behavior is changed.
+
+The endpoint added is for the URL pattern `/dbname/_query` and has the following characteristics:
+
+* The only HTTP method supported is `POST`.
+* The request `Content-Type` must be `application/json`.
+* The response status code will either be `200`, `4XX`, or `5XX`
+* The response `Content-Type` will be `application/json`
+* The response `Transfer-Encoding` will be `chunked`.
+* The response is a single JSON object or array that matches to the single command or list of commands that exist in the request.
+
+This is intended to be a significantly simpler use of HTTP than the current APIs. This is motivated by the fact that this entire API is aimed at customers who are not as savvy at HTTP or non-relational document stores. Once a customer is comfortable using this API we hope to expose any other "power features" through the existing HTTP API and its adherence to HTTP semantics.
+
+
+Supported Actions
+=================
+
+This is a list of supported actions that Mango understands. For the time being it is limited to the four normal CRUD actions plus one meta action to create indices on the database.
+
+insert
+------
+
+Insert a document or documents into the database.
+
+Keys:
+
+* action - "insert"
+* docs - The JSON document to insert
+* w (optional) (default: 2) - An integer > 0 for the write quorum size
+
+If the provided document or documents do not contain an "\_id" field one will be added using an automatically generated UUID.
+
+It is more performant to specify multiple documents in the "docs" field than it is to specify multiple independent insert actions. Each insert action is submitted as a single bulk update (ie, \_bulk\_docs in CouchDB terminology). This, however, does not make any guarantees on the isolation or atomicity of the bulk operation. It is merely a performance benefit.
+
+
+find
+----
+
+Retrieve documents from the database.
+
+Keys:
+
+* action - "find"
+* selector - JSON object following selector syntax, described below
+* limit (optional) (default: 25) - integer >= 0, Limit the number of rows returned
+* skip (optional) (default: 0) - integer >= 0, Skip the specified number of rows
+* sort (optional) (default: []) - JSON array following sort syntax, described below
+* fields (optional) (default: null) - JSON array following the field syntax, described below
+* r (optional) (default: 1) - By default a find will return the document that was found when traversing the index. Optionally there can be a quorum read for each document using `r` as the read quorum. This is obviously less performant than using the document local to the index.
+* conflicts (optional) (default: false) - boolean, whether or not to include information about any existing conflicts for the document.
+
+The important thing to note about the find command is that it must execute over a generated index. If a selector is provided that cannot be satisfied using an existing index the list of basic indices that could be used will be returned.
+
+For the most part, indices are generated in response to the "create\_index" action (described below) although there are two special indices that can be used as well. The "\_id" is automatically indexed and is similar to every other index. There is also a special "\_seq" index to retrieve documents in the order of their update sequence.
+
+Its also quite possible to generate a query that can't be satisfied by any index. In this case an error will be returned stating that fact. Generally speaking the easiest way to stumble onto this is to attempt to OR two separate fields which would require a complete table scan. In the future I expect to support these more complicated queries using an extended indexing API (which deviates from the current MongoDB model a bit).
+
+
+update
+------
+
+Update an existing document in the database
+
+Keys:
+
+* action - "update"
+* selector - JSON object following selector syntax, described below
+* update - JSON object following update syntax, described below
+* upsert - (optional) (default: false) - boolean, Whether or not to create a new document if the selector does not match any documents in the database
+* limit (optional) (default: 1) - integer > 0, How many documents returned from the selector should be modified. Currently has a maximum value of 100
+* sort - (optional) (default: []) - JSON array following sort syntax, described below
+* r (optional) (default: 1) - integer > 0, read quorum constant
+* w (optional) (default: 2) - integer > 0, write quorum constant
+
+Updates are fairly straightforward other than to mention that the selector (like find) must be satisifiable using an existing index.
+
+On the update field, if the provided JSON object has one or more update operator (described below) then the operation is applied onto the existing document (if one exists) else the entire contents are replaced with exactly the value of the `update` field.
+
+
+delete
+------
+
+Remove a document from the database.
+
+Keys:
+
+* action - "delete"
+* selector - JSON object following selector syntax, described below
+* force (optional) (default: false) - Delete all conflicted versions of the document as well
+* limit - (optional) (default: 1) - integer > 0, How many documents to delete from the database. Currently has a maximum value of 100
+* sort - (optional) (default: []) - JSON array following sort syntax, described below
+* r (optional) (default: 1) - integer > 1, read quorum constant
+* w (optional) (default: 2) - integer > 0, write quorum constant
+
+Deletes behave quite similarly to update except they attempt to remove documents from the database. Its important to note that if a document has conflicts it may "appear" that delete's aren't having an effect. This is because the delete operation by default only removes a single revision. Specify `"force":true` if you would like to attempt to delete all live revisions.
+
+If you wish to delete a specific revision of the document, you can specify it in the selector using the special "\_rev" field.
+
+
+create\_index
+-------------
+
+Create an index on the database
+
+Keys:
+
+* action - "create\_index"
+* index - JSON array following sort syntax, described below
+* type (optional) (default: "json") - string, specifying the index type to create. Currently only "json" indexes are supported but in the future we will provide full-text indexes as well as Geo spatial indexes
+* name (optional) - string, optionally specify a name for the index. If a name is not provided one will be automatically generated
+* ddoc (optional) - Indexes can be grouped into design documents underneath the hood for efficiency. This is an advanced feature. Don't specify a design document here unless you know the consequences of index invalidation. By default each index is placed in its own separate design document for isolation.
+
+Anytime an operation is required to locate a document in the database it is required that an index must exist that can be used to locate it. By default the only two indices that exist are for the document "\_id" and the special "\_seq" index.
+
+Indices are created in the background. If you attempt to create an index on a large database and then immediately utilize it, the request may block for a considerable amount of time before the request completes.
+
+Indices can specify multiple fields to index simultaneously. This is roughly analogous to a compound index in SQL with the corresponding tradeoffs. For instance, an index may contain the (ordered set of) fields "foo", "bar", and "baz". If a selector specifying "bar" is received, it can not be answered. Although if a selector specifying "foo" and "bar" is received, it can be answered more efficiently than if there were only an index on "foo" and "bar" independently.
+
+NB: while the index allows the ability to specify sort directions these are currently not supported. The sort direction must currently be specified as "asc" in the JSON. [INTERNAL]: This will require that we patch the view engine as well as the cluster coordinators in Fabric to follow the specified sort orders. The concepts are straightforward but the implementation may need some thought to fit into the current shape of things.
+
+
+list\_indexes
+-------------
+
+List the indexes that exist in a given database.
+
+Keys:
+
+* action - "list\_indexes"
+
+
+delete\_index
+-------------
+
+Delete the specified index from the database.
+
+Keys:
+
+* action - "delete\_index"
+* name - string, the index to delete
+* design\_doc - string, the design doc id from which to delete the index. For auto-generated index names and design docs, you can retrieve this information from the `list\_indexes` action
+
+Indexes require resources to maintain. If you find that an index is no longer necessary then it can be beneficial to remove it from the database.
+
+
+describe\_selector
+------------------
+
+Shows debugging information for a given selector
+
+Keys:
+
+* action - "describe\_selector"
+* selector - JSON object in selector syntax, described below
+* extended (optional) (default: false) - Show information on what existing indexes could be used with this selector
+
+This is a useful debugging utility that will show how a given selector is normalized before execution as well as information on what indexes could be used to satisfy it.
+
+If `"extended": true` is included then the list of existing indices that could be used for this selector are also returned.
+
+
+
+JSON Syntax Descriptions
+========================
+
+This API uses a few defined JSON structures for various operations. Here we'll describe each in detail.
+
+
+Selector Syntax
+---------------
+
+The Mango query language is expressed as a JSON object describing documents of interest. Within this structure it is also possible to express conditional logic using specially named fields. This is inspired by and intended to maintain a fairly close parity to the existing MongoDB behavior.
+
+As an example, the simplest selector for Mango might look something like such:
+
+ {"_id": "Paul"}
+
+Which would match the document named "Paul" (if one exists). Extending this example using other fields might look like such:
+
+ {"_id": "Paul", "location": "Boston"}
+
+This would match a document named "Paul" *AND* having a "location" value of "Boston". Seeing as though I'm sitting in my basement in Omaha, this is unlikely.
+
+There are two special syntax elements for the object keys in a selector. The first is that the period (full stop, or simply `.`) character denotes subfields in a document. For instance, here are two equivalent examples:
+
+ {"location": {"city": "Omaha"}}
+ {"location.city": "Omaha"}
+
+If the object's key contains the period it could be escaped with backslash, i.e.
+
+ {"location\\.city": "Omaha"}
+
+Note that the double backslash here is necessary to encode an actual single backslash.
+
+The second important syntax element is the use of a dollar sign (`$`) prefix to denote operators. For example:
+
+ {"age": {"$gt": 21}}
+
+In this example, we have created the boolean expression `age > 21`.
+
+There are two core types of operators in the selector syntax: combination operators and condition operators. In general, combination operators contain groups of condition operators. We'll describe the list of each below.
+
+### Implicit Operators
+
+For the most part every operator must be of the form `{"$operator": argument}`. Though there are two implicit operators for selectors.
+
+First, any JSON object that is not the argument to a condition operator is an implicit `$and` operator on each field. For instance, these two examples are identical:
+
+ {"foo": "bar", "baz": true}
+ {"$and": [{"foo": {"$eq": "bar"}}, {"baz": {"$eq": true}}]}
+
+And as shown, any field that contains a JSON value that has no operators in it is an equality condition. For instance, these are equivalent:
+
+ {"foo": "bar"}
+ {"foo": {"$eq": "bar"}}
+
+And to be clear, these are also equivalent:
+
+ {"foo": {"bar": "baz"}}
+ {"foo": {"$eq": {"bar": "baz"}}}
+
+Although, the previous example would actually be normalized internally to this:
+
+ {"foo.bar": {"$eq": "baz"}}
+
+
+### Combination Operators
+
+These operators are responsible for combining groups of condition operators. Most familiar are the standard boolean operators plus a few extra for working with JSON arrays.
+
+Each of the combining operators take a single argument that is either a condition operator or an array of condition operators.
+
+The list of combining characters:
+
+* "$and" - array argument
+* "$or" - array argument
+* "$not" - single argument
+* "$nor" - array argument
+* "$all" - array argument (special operator for array values)
+* "$elemMatch" - single argument (special operator for array values)
+* "$allMatch" - single argument (special operator for array values)
+
+### Condition Operators
+
+Condition operators are specified on a per field basis and apply to the value indexed for that field. For instance, the basic "$eq" operator matches when the indexed field is equal to its argument. There is currently support for the basic equality and inequality operators as well as a number of meta operators. Some of these operators will accept any JSON argument while some require a specific JSON formatted argument. Each is noted below.
+
+The list of conditional arguments:
+
+(In)equality operators
+
+* "$lt" - any JSON
+* "$lte" - any JSON
+* "$eq" - any JSON
+* "$ne" - any JSON
+* "$gte" - any JSON
+* "$gt" - any JSON
+
+Object related operators
+
+* "$exists" - boolean, check whether the field exists or not regardless of its value
+* "$type" - string, check the document field's type
+
+Array related operators
+
+* "$in" - array of JSON values, the document field must exist in the list provided
+* "$nin" - array of JSON values, the document field must not exist in the list provided
+* "$size" - integer, special condition to match the length of an array field in a document. Non-array fields cannot match this condition.
+
+Misc related operators
+
+* "$mod" - [Divisor, Remainder], where Divisor and Remainder are both positive integers (ie, greater than 0). Matches documents where (field % Divisor == Remainder) is true. This is false for any non-integer field
+* "$regex" - string, a regular expression pattern to match against the document field. Only matches when the field is a string value and matches the supplied matches
+
+
+Update Syntax
+-------------
+
+Need to describe the syntax for update operators.
+
+
+Sort Syntax
+-----------
+
+The sort syntax is a basic array of field name and direction pairs. It looks like such:
+
+ [{field1: dir1} | ...]
+
+Where field1 can be any field (dotted notation is available for sub-document fields) and dir1 can be "asc" or "desc".
+
+Note that it is highly recommended that you specify a single key per object in your sort ordering so that the order is not dependent on the combination of JSON libraries between your application and the internals of Mango's indexing engine.
+
+
+Fields Syntax
+-------------
+
+When retrieving documents from the database you can specify that only a subset of the fields are returned. This allows you to limit your results strictly to the parts of the document that are interesting for the local application logic. The fields returned are specified as an array. Unlike MongoDB only the fields specified are included, there is no automatic inclusion of the "\_id" or other metadata fields when a field list is included.
+
+A trivial example:
+
+ ["foo", "bar", "baz"]
+
+
+HTTP API
+========
+
+Short summary until the full documentation can be brought over.
+
+POST /dbname/\_find
+-------------------------
+
+Issue a query.
+
+Request body is a JSON object that has the selector and the various options like limit/skip etc. Or we could post the selector and put the other options into the query string. Though I'd probably prefer to have it all in the body for consistency.
+
+Response is streamed out like a view.
+
+POST /dbname/\_index
+--------------------------
+
+Request body contains the index definition.
+
+Response body is empty and the result is returned as the status code (200 OK -> created, 3something for exists).
+
+GET /dbname/\_index
+-------------------------
+
+Request body is empty.
+
+Response body is all of the indexes that are available for use by find.
+
+DELETE /dbname/\_index/ddocid/viewname
+--------------------------------------------
+
+Remove the specified index.
+
+Request body is empty.
+
+Response body is empty. The status code gives enough information.
diff --git a/src/mango/TODO.md b/src/mango/TODO.md
new file mode 100644
index 000000000..ce2d85f3d
--- /dev/null
+++ b/src/mango/TODO.md
@@ -0,0 +1,9 @@
+
+* Patch the view engine to do alternative sorts. This will include both the lower level couch\_view* modules as well as the fabric coordinators.
+
+* Patch the view engine so we can specify options when returning docs from cursors. We'll want this so that we can delete specific revisions from a document.
+
+* Need to figure out how to do raw collation on some indices because at
+least the _id index uses it forcefully.
+
+* Add lots more to the update API. Mongo appears to be missing some pretty obvious easy functionality here. Things like managing values doing things like multiplying numbers, or common string mutations would be obvious examples. Also it could be interesting to add to the language so that you can do conditional updates based on other document attributes. Definitely not a V1 endeavor. \ No newline at end of file
diff --git a/src/mango/rebar.config.script b/src/mango/rebar.config.script
new file mode 100644
index 000000000..02604f965
--- /dev/null
+++ b/src/mango/rebar.config.script
@@ -0,0 +1,27 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+DreyfusAppFile = filename:join(filename:dirname(SCRIPT),
+ "../dreyfus/src/dreyfus.app.src"),
+CursorFile = filename:join(filename:dirname(SCRIPT),
+ "src/mango_cursor_text.erl"),
+RenameFile = filename:join(filename:dirname(SCRIPT),
+ "src/mango_cursor_text.nocompile"),
+
+case filelib:is_regular(DreyfusAppFile) of
+ true ->
+ ok;
+ false ->
+ file:rename(CursorFile, RenameFile)
+end,
+
+CONFIG. \ No newline at end of file
diff --git a/src/mango/src/mango.app.src b/src/mango/src/mango.app.src
new file mode 100644
index 000000000..a63f036e0
--- /dev/null
+++ b/src/mango/src/mango.app.src
@@ -0,0 +1,26 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+{application, mango, [
+ {description, "MongoDB API compatibility layer for CouchDB"},
+ {vsn, git},
+ {registered, []},
+ {applications, [
+ kernel,
+ stdlib,
+ couch_epi,
+ config,
+ couch_log,
+ fabric
+ ]},
+ {mod, {mango_app, []}}
+]}.
diff --git a/src/mango/src/mango.hrl b/src/mango/src/mango.hrl
new file mode 100644
index 000000000..26a9d43b9
--- /dev/null
+++ b/src/mango/src/mango.hrl
@@ -0,0 +1,13 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-define(MANGO_ERROR(R), throw({mango_error, ?MODULE, R})).
diff --git a/src/mango/src/mango_app.erl b/src/mango/src/mango_app.erl
new file mode 100644
index 000000000..7a0c39db7
--- /dev/null
+++ b/src/mango/src/mango_app.erl
@@ -0,0 +1,21 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(mango_app).
+-behaviour(application).
+-export([start/2, stop/1]).
+
+start(_Type, StartArgs) ->
+ mango_sup:start_link(StartArgs).
+
+stop(_State) ->
+ ok.
diff --git a/src/mango/src/mango_crud.erl b/src/mango/src/mango_crud.erl
new file mode 100644
index 000000000..68c9d6cc4
--- /dev/null
+++ b/src/mango/src/mango_crud.erl
@@ -0,0 +1,177 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(mango_crud).
+
+-export([
+ insert/3,
+ find/5,
+ update/4,
+ delete/3,
+ explain/3
+]).
+
+-export([
+ collect_cb/2
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include("mango.hrl").
+
+
+insert(Db, #doc{}=Doc, Opts) ->
+ insert(Db, [Doc], Opts);
+insert(Db, {_}=Doc, Opts) ->
+ insert(Db, [Doc], Opts);
+insert(Db, Docs, Opts0) when is_list(Docs) ->
+ Opts1 = maybe_add_user_ctx(Db, Opts0),
+ Opts2 = maybe_int_to_str(w, Opts1),
+ case fabric:update_docs(Db, Docs, Opts2) of
+ {ok, Results0} ->
+ {ok, lists:zipwith(fun result_to_json/2, Docs, Results0)};
+ {accepted, Results0} ->
+ {ok, lists:zipwith(fun result_to_json/2, Docs, Results0)};
+ {aborted, Errors} ->
+ {error, lists:map(fun result_to_json/1, Errors)}
+ end.
+
+
+find(Db, Selector, Callback, UserAcc, Opts0) ->
+ Opts1 = maybe_add_user_ctx(Db, Opts0),
+ Opts2 = maybe_int_to_str(r, Opts1),
+ {ok, Cursor} = mango_cursor:create(Db, Selector, Opts2),
+ mango_cursor:execute(Cursor, Callback, UserAcc).
+
+
+update(Db, Selector, Update, Options) ->
+ Upsert = proplists:get_value(upsert, Options),
+ case collect_docs(Db, Selector, Options) of
+ {ok, []} when Upsert ->
+ InitDoc = mango_doc:update_as_insert(Update),
+ case mango_doc:has_operators(InitDoc) of
+ true ->
+ ?MANGO_ERROR(invalid_upsert_with_operators);
+ false ->
+ % Probably need to catch and rethrow errors from
+ % this function.
+ Doc = couch_doc:from_json_obj(InitDoc),
+ NewDoc = case Doc#doc.id of
+ <<"">> ->
+ Doc#doc{id=couch_uuids:new(), revs={0, []}};
+ _ ->
+ Doc
+ end,
+ insert(Db, NewDoc, Options)
+ end;
+ {ok, Docs} ->
+ NewDocs = lists:map(fun(Doc) ->
+ mango_doc:apply_update(Doc, Update)
+ end, Docs),
+ insert(Db, NewDocs, Options);
+ Else ->
+ Else
+ end.
+
+
+delete(Db, Selector, Options) ->
+ case collect_docs(Db, Selector, Options) of
+ {ok, Docs} ->
+ NewDocs = lists:map(fun({Props}) ->
+ {[
+ {<<"_id">>, proplists:get_value(<<"_id">>, Props)},
+ {<<"_rev">>, proplists:get_value(<<"_rev">>, Props)},
+ {<<"_deleted">>, true}
+ ]}
+ end, Docs),
+ insert(Db, NewDocs, Options);
+ Else ->
+ Else
+ end.
+
+
+explain(Db, Selector, Opts0) ->
+ Opts1 = maybe_add_user_ctx(Db, Opts0),
+ Opts2 = maybe_int_to_str(r, Opts1),
+ {ok, Cursor} = mango_cursor:create(Db, Selector, Opts2),
+ mango_cursor:explain(Cursor).
+
+
+maybe_add_user_ctx(Db, Opts) ->
+ case lists:keyfind(user_ctx, 1, Opts) of
+ {user_ctx, _} ->
+ Opts;
+ false ->
+ [{user_ctx, Db#db.user_ctx} | Opts]
+ end.
+
+
+maybe_int_to_str(_Key, []) ->
+ [];
+maybe_int_to_str(Key, [{Key, Val} | Rest]) when is_integer(Val) ->
+ [{Key, integer_to_list(Val)} | maybe_int_to_str(Key, Rest)];
+maybe_int_to_str(Key, [KV | Rest]) ->
+ [KV | maybe_int_to_str(Key, Rest)].
+
+
+result_to_json(#doc{id=Id}, Result) ->
+ result_to_json(Id, Result);
+result_to_json({Props}, Result) ->
+ Id = couch_util:get_value(<<"_id">>, Props),
+ result_to_json(Id, Result);
+result_to_json(DocId, {ok, NewRev}) ->
+ {[
+ {id, DocId},
+ {rev, couch_doc:rev_to_str(NewRev)}
+ ]};
+result_to_json(DocId, {accepted, NewRev}) ->
+ {[
+ {id, DocId},
+ {rev, couch_doc:rev_to_str(NewRev)},
+ {accepted, true}
+ ]};
+result_to_json(DocId, Error) ->
+ % chttpd:error_info/1 because this is coming from fabric
+ % and not internal mango operations.
+ {_Code, ErrorStr, Reason} = chttpd:error_info(Error),
+ {[
+ {id, DocId},
+ {error, ErrorStr},
+ {reason, Reason}
+ ]}.
+
+
+% This is for errors because for some reason we
+% need a different return value for errors? Blargh.
+result_to_json({{Id, Rev}, Error}) ->
+ {_Code, ErrorStr, Reason} = chttpd:error_info(Error),
+ {[
+ {id, Id},
+ {rev, couch_doc:rev_to_str(Rev)},
+ {error, ErrorStr},
+ {reason, Reason}
+ ]}.
+
+
+collect_docs(Db, Selector, Options) ->
+ Cb = fun ?MODULE:collect_cb/2,
+ case find(Db, Selector, Cb, [], Options) of
+ {ok, Docs} ->
+ {ok, lists:reverse(Docs)};
+ Else ->
+ Else
+ end.
+
+
+collect_cb({row, Doc}, Acc) ->
+ {ok, [Doc | Acc]}.
+
diff --git a/src/mango/src/mango_cursor.erl b/src/mango/src/mango_cursor.erl
new file mode 100644
index 000000000..911ecdb3e
--- /dev/null
+++ b/src/mango/src/mango_cursor.erl
@@ -0,0 +1,129 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(mango_cursor).
+
+
+-export([
+ create/3,
+ explain/1,
+ execute/3,
+ maybe_filter_indexes/2
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include("mango.hrl").
+-include("mango_cursor.hrl").
+
+
+-define(SUPERVISOR, mango_cursor_sup).
+
+
+create(Db, Selector0, Opts) ->
+ Selector = mango_selector:normalize(Selector0),
+ UsableIndexes = mango_idx:get_usable_indexes(Db, Selector0, Opts),
+
+ {use_index, IndexSpecified} = proplists:lookup(use_index, Opts),
+ case {length(UsableIndexes), length(IndexSpecified)} of
+ {0, 1} ->
+ ?MANGO_ERROR({no_usable_index, selector_unsupported});
+ {0, 0} ->
+ AllDocs = mango_idx:special(Db),
+ create_cursor(Db, AllDocs, Selector, Opts);
+ _ ->
+ create_cursor(Db, UsableIndexes, Selector, Opts)
+ end.
+
+
+explain(#cursor{}=Cursor) ->
+ #cursor{
+ index = Idx,
+ selector = Selector,
+ opts = Opts0,
+ limit = Limit,
+ skip = Skip,
+ fields = Fields
+ } = Cursor,
+ Mod = mango_idx:cursor_mod(Idx),
+ Opts = lists:keydelete(user_ctx, 1, Opts0),
+ {[
+ {dbname, mango_idx:dbname(Idx)},
+ {index, mango_idx:to_json(Idx)},
+ {selector, Selector},
+ {opts, {Opts}},
+ {limit, Limit},
+ {skip, Skip},
+ {fields, Fields}
+ ] ++ Mod:explain(Cursor)}.
+
+
+execute(#cursor{index=Idx}=Cursor, UserFun, UserAcc) ->
+ Mod = mango_idx:cursor_mod(Idx),
+ Mod:execute(Cursor, UserFun, UserAcc).
+
+
+maybe_filter_indexes(Indexes, Opts) ->
+ case lists:keyfind(use_index, 1, Opts) of
+ {use_index, []} ->
+ Indexes;
+ {use_index, [DesignId]} ->
+ filter_indexes(Indexes, DesignId);
+ {use_index, [DesignId, ViewName]} ->
+ filter_indexes(Indexes, DesignId, ViewName)
+ end.
+
+
+filter_indexes(Indexes, DesignId0) ->
+ DesignId = case DesignId0 of
+ <<"_design/", _/binary>> ->
+ DesignId0;
+ Else ->
+ <<"_design/", Else/binary>>
+ end,
+ FiltFun = fun(I) -> mango_idx:ddoc(I) == DesignId end,
+ lists:filter(FiltFun, Indexes).
+
+
+filter_indexes(Indexes0, DesignId, ViewName) ->
+ Indexes = filter_indexes(Indexes0, DesignId),
+ FiltFun = fun(I) -> mango_idx:name(I) == ViewName end,
+ lists:filter(FiltFun, Indexes).
+
+
+create_cursor(Db, Indexes, Selector, Opts) ->
+ [{CursorMod, CursorModIndexes} | _] = group_indexes_by_type(Indexes),
+ CursorMod:create(Db, CursorModIndexes, Selector, Opts).
+
+
+group_indexes_by_type(Indexes) ->
+ IdxDict = lists:foldl(fun(I, D) ->
+ dict:append(mango_idx:cursor_mod(I), I, D)
+ end, dict:new(), Indexes),
+ % The first cursor module that has indexes will be
+ % used to service this query. This is so that we
+ % don't suddenly switch indexes for existing client
+ % queries.
+ CursorModules = case module_loaded(dreyfus_index) of
+ true ->
+ [mango_cursor_view, mango_cursor_text, mango_cursor_special];
+ false ->
+ [mango_cursor_view, mango_cursor_special]
+ end,
+ lists:flatmap(fun(CMod) ->
+ case dict:find(CMod, IdxDict) of
+ {ok, CModIndexes} ->
+ [{CMod, CModIndexes}];
+ error ->
+ []
+ end
+ end, CursorModules).
diff --git a/src/mango/src/mango_cursor.hrl b/src/mango/src/mango_cursor.hrl
new file mode 100644
index 000000000..58782e5f8
--- /dev/null
+++ b/src/mango/src/mango_cursor.hrl
@@ -0,0 +1,24 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-record(cursor, {
+ db,
+ index,
+ ranges,
+ selector,
+ opts,
+ limit,
+ skip = 0,
+ fields = undefined,
+ user_fun,
+ user_acc
+}).
diff --git a/src/mango/src/mango_cursor_special.erl b/src/mango/src/mango_cursor_special.erl
new file mode 100644
index 000000000..8404bc04b
--- /dev/null
+++ b/src/mango/src/mango_cursor_special.erl
@@ -0,0 +1,61 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(mango_cursor_special).
+
+-export([
+ create/4,
+ explain/1,
+ execute/3
+]).
+
+-export([
+ handle_message/2
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("couch_mrview/include/couch_mrview.hrl").
+-include("mango_cursor.hrl").
+
+
+create(Db, Indexes, Selector, Opts) ->
+ InitialRange = mango_idx_view:field_ranges(Selector),
+ CatchAll = [{<<"_id">>, {'$gt', null, '$lt', mango_json_max}}],
+ FieldRanges = lists:append(CatchAll, InitialRange),
+ Composited = mango_cursor_view:composite_indexes(Indexes, FieldRanges),
+ {Index, IndexRanges} = mango_cursor_view:choose_best_index(Db, Composited),
+
+ Limit = couch_util:get_value(limit, Opts, mango_opts:default_limit()),
+ Skip = couch_util:get_value(skip, Opts, 0),
+ Fields = couch_util:get_value(fields, Opts, all_fields),
+
+ {ok, #cursor{
+ db = Db,
+ index = Index,
+ ranges = IndexRanges,
+ selector = Selector,
+ opts = Opts,
+ limit = Limit,
+ skip = Skip,
+ fields = Fields
+ }}.
+
+
+explain(Cursor) ->
+ mango_cursor_view:explain(Cursor).
+
+execute(Cursor0, UserFun, UserAcc) ->
+ mango_cursor_view:execute(Cursor0, UserFun, UserAcc).
+
+handle_message(Msg, Cursor) ->
+ mango_cursor_view:handle_message(Msg, Cursor).
diff --git a/src/mango/src/mango_cursor_text.erl b/src/mango/src/mango_cursor_text.erl
new file mode 100644
index 000000000..a094b5528
--- /dev/null
+++ b/src/mango/src/mango_cursor_text.erl
@@ -0,0 +1,306 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(mango_cursor_text).
+
+-export([
+ create/4,
+ explain/1,
+ execute/3
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("dreyfus/include/dreyfus.hrl").
+-include("mango_cursor.hrl").
+-include("mango.hrl").
+
+
+-record(cacc, {
+ selector,
+ dbname,
+ ddocid,
+ idx_name,
+ query_args,
+ bookmark,
+ limit,
+ skip,
+ user_fun,
+ user_acc,
+ fields
+}).
+
+
+create(Db, Indexes, Selector, Opts0) ->
+ Index = case Indexes of
+ [Index0] ->
+ Index0;
+ _ ->
+ ?MANGO_ERROR(multiple_text_indexes)
+ end,
+
+ Opts = unpack_bookmark(Db#db.name, Opts0),
+
+ DreyfusLimit = get_dreyfus_limit(),
+ Limit = erlang:min(DreyfusLimit, couch_util:get_value(limit, Opts, mango_opts:default_limit())),
+ Skip = couch_util:get_value(skip, Opts, 0),
+ Fields = couch_util:get_value(fields, Opts, all_fields),
+
+ {ok, #cursor{
+ db = Db,
+ index = Index,
+ ranges = null,
+ selector = Selector,
+ opts = Opts,
+ limit = Limit,
+ skip = Skip,
+ fields = Fields
+ }}.
+
+
+explain(Cursor) ->
+ #cursor{
+ selector = Selector,
+ opts = Opts
+ } = Cursor,
+ [
+ {'query', mango_selector_text:convert(Selector)},
+ {sort, sort_query(Opts, Selector)}
+ ].
+
+
+execute(Cursor, UserFun, UserAcc) ->
+ #cursor{
+ db = Db,
+ index = Idx,
+ limit = Limit,
+ skip = Skip,
+ selector = Selector,
+ opts = Opts
+ } = Cursor,
+ QueryArgs = #index_query_args{
+ q = mango_selector_text:convert(Selector),
+ sort = sort_query(Opts, Selector),
+ raw_bookmark = true
+ },
+ CAcc = #cacc{
+ selector = Selector,
+ dbname = Db#db.name,
+ ddocid = ddocid(Idx),
+ idx_name = mango_idx:name(Idx),
+ bookmark = get_bookmark(Opts),
+ limit = Limit,
+ skip = Skip,
+ query_args = QueryArgs,
+ user_fun = UserFun,
+ user_acc = UserAcc,
+ fields = Cursor#cursor.fields
+ },
+ try
+ execute(CAcc)
+ catch
+ throw:{stop, FinalCAcc} ->
+ #cacc{
+ bookmark = FinalBM,
+ user_fun = UserFun,
+ user_acc = LastUserAcc
+ } = FinalCAcc,
+ JsonBM = dreyfus_bookmark:pack(FinalBM),
+ Arg = {add_key, bookmark, JsonBM},
+ {_Go, FinalUserAcc} = UserFun(Arg, LastUserAcc),
+ {ok, FinalUserAcc}
+ end.
+
+
+execute(CAcc) ->
+ case search_docs(CAcc) of
+ {ok, Bookmark, []} ->
+ % If we don't have any results from the
+ % query it means the request has paged through
+ % all possible results and the request is over.
+ NewCAcc = CAcc#cacc{bookmark = Bookmark},
+ throw({stop, NewCAcc});
+ {ok, Bookmark, Hits} ->
+ NewCAcc = CAcc#cacc{bookmark = Bookmark},
+ HitDocs = get_json_docs(CAcc#cacc.dbname, Hits),
+ {ok, FinalCAcc} = handle_hits(NewCAcc, HitDocs),
+ execute(FinalCAcc)
+ end.
+
+
+search_docs(CAcc) ->
+ #cacc{
+ dbname = DbName,
+ ddocid = DDocId,
+ idx_name = IdxName
+ } = CAcc,
+ QueryArgs = update_query_args(CAcc),
+ case dreyfus_fabric_search:go(DbName, DDocId, IdxName, QueryArgs) of
+ {ok, Bookmark, _, Hits, _, _} ->
+ {ok, Bookmark, Hits};
+ {error, Reason} ->
+ ?MANGO_ERROR({text_search_error, {error, Reason}})
+ end.
+
+
+handle_hits(CAcc, []) ->
+ {ok, CAcc};
+
+handle_hits(CAcc0, [{Sort, Doc} | Rest]) ->
+ CAcc1 = handle_hit(CAcc0, Sort, Doc),
+ handle_hits(CAcc1, Rest).
+
+
+handle_hit(CAcc0, Sort, Doc) ->
+ #cacc{
+ limit = Limit,
+ skip = Skip
+ } = CAcc0,
+ CAcc1 = update_bookmark(CAcc0, Sort),
+ case mango_selector:match(CAcc1#cacc.selector, Doc) of
+ true when Skip > 0 ->
+ CAcc1#cacc{skip = Skip - 1};
+ true when Limit == 0 ->
+ % We hit this case if the user spcified with a
+ % zero limit. Notice that in this case we need
+ % to return the bookmark from before this match
+ throw({stop, CAcc0});
+ true when Limit == 1 ->
+ NewCAcc = apply_user_fun(CAcc1, Doc),
+ throw({stop, NewCAcc});
+ true when Limit > 1 ->
+ NewCAcc = apply_user_fun(CAcc1, Doc),
+ NewCAcc#cacc{limit = Limit - 1};
+ false ->
+ CAcc1
+ end.
+
+
+apply_user_fun(CAcc, Doc) ->
+ FinalDoc = mango_fields:extract(Doc, CAcc#cacc.fields),
+ #cacc{
+ user_fun = UserFun,
+ user_acc = UserAcc
+ } = CAcc,
+ case UserFun({row, FinalDoc}, UserAcc) of
+ {ok, NewUserAcc} ->
+ CAcc#cacc{user_acc = NewUserAcc};
+ {stop, NewUserAcc} ->
+ throw({stop, CAcc#cacc{user_acc = NewUserAcc}})
+ end.
+
+
+%% Convert Query to Dreyfus sort specifications
+%% Covert <<"Field">>, <<"desc">> to <<"-Field">>
+%% and append to the dreyfus query
+sort_query(Opts, Selector) ->
+ {sort, {Sort}} = lists:keyfind(sort, 1, Opts),
+ SortList = lists:map(fun(SortField) ->
+ {Dir, RawSortField} = case SortField of
+ {Field, <<"asc">>} -> {asc, Field};
+ {Field, <<"desc">>} -> {desc, Field};
+ Field when is_binary(Field) -> {asc, Field}
+ end,
+ SField = mango_selector_text:append_sort_type(RawSortField, Selector),
+ case Dir of
+ asc ->
+ SField;
+ desc ->
+ <<"-", SField/binary>>
+ end
+ end, Sort),
+ case SortList of
+ [] -> relevance;
+ _ -> SortList
+ end.
+
+
+get_bookmark(Opts) ->
+ case lists:keyfind(bookmark, 1, Opts) of
+ {_, BM} when is_list(BM), BM /= [] ->
+ BM;
+ _ ->
+ nil
+ end.
+
+
+update_bookmark(CAcc, Sortable) ->
+ BM = CAcc#cacc.bookmark,
+ QueryArgs = CAcc#cacc.query_args,
+ Sort = QueryArgs#index_query_args.sort,
+ NewBM = dreyfus_bookmark:update(Sort, BM, [Sortable]),
+ CAcc#cacc{bookmark = NewBM}.
+
+
+pack_bookmark(Bookmark) ->
+ case dreyfus_bookmark:pack(Bookmark) of
+ null -> nil;
+ Enc -> Enc
+ end.
+
+
+unpack_bookmark(DbName, Opts) ->
+ NewBM = case lists:keyfind(bookmark, 1, Opts) of
+ {_, nil} ->
+ [];
+ {_, Bin} ->
+ try
+ dreyfus_bookmark:unpack(DbName, Bin)
+ catch _:_ ->
+ ?MANGO_ERROR({invalid_bookmark, Bin})
+ end
+ end,
+ lists:keystore(bookmark, 1, Opts, {bookmark, NewBM}).
+
+
+ddocid(Idx) ->
+ case mango_idx:ddoc(Idx) of
+ <<"_design/", Rest/binary>> ->
+ Rest;
+ Else ->
+ Else
+ end.
+
+
+update_query_args(CAcc) ->
+ #cacc{
+ bookmark = Bookmark,
+ query_args = QueryArgs
+ } = CAcc,
+ QueryArgs#index_query_args{
+ bookmark = pack_bookmark(Bookmark),
+ limit = get_limit(CAcc)
+ }.
+
+
+get_limit(CAcc) ->
+ erlang:min(get_dreyfus_limit(), CAcc#cacc.limit + CAcc#cacc.skip).
+
+
+get_dreyfus_limit() ->
+ config:get_integer("dreyfus", "max_limit", 200).
+
+
+get_json_docs(DbName, Hits) ->
+ Ids = lists:map(fun(#sortable{item = Item}) ->
+ couch_util:get_value(<<"_id">>, Item#hit.fields)
+ end, Hits),
+ {ok, IdDocs} = dreyfus_fabric:get_json_docs(DbName, Ids),
+ lists:map(fun(#sortable{item = Item} = Sort) ->
+ Id = couch_util:get_value(<<"_id">>, Item#hit.fields),
+ case lists:keyfind(Id, 1, IdDocs) of
+ {Id, {doc, Doc}} ->
+ {Sort, Doc};
+ false ->
+ {Sort, not_found}
+ end
+ end, Hits).
diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl
new file mode 100644
index 000000000..2918a2d08
--- /dev/null
+++ b/src/mango/src/mango_cursor_view.erl
@@ -0,0 +1,273 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(mango_cursor_view).
+
+-export([
+ create/4,
+ explain/1,
+ execute/3
+]).
+
+-export([
+ handle_message/2,
+ handle_all_docs_message/2,
+ composite_indexes/2,
+ choose_best_index/2
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("couch_mrview/include/couch_mrview.hrl").
+-include("mango_cursor.hrl").
+
+
+create(Db, Indexes, Selector, Opts) ->
+ FieldRanges = mango_idx_view:field_ranges(Selector),
+ Composited = composite_indexes(Indexes, FieldRanges),
+ {Index, IndexRanges} = choose_best_index(Db, Composited),
+
+ Limit = couch_util:get_value(limit, Opts, mango_opts:default_limit()),
+ Skip = couch_util:get_value(skip, Opts, 0),
+ Fields = couch_util:get_value(fields, Opts, all_fields),
+
+ {ok, #cursor{
+ db = Db,
+ index = Index,
+ ranges = IndexRanges,
+ selector = Selector,
+ opts = Opts,
+ limit = Limit,
+ skip = Skip,
+ fields = Fields
+ }}.
+
+
+explain(Cursor) ->
+ #cursor{
+ index = Idx,
+ ranges = Ranges
+ } = Cursor,
+ case Ranges of
+ [empty] ->
+ [{range, empty}];
+ _ ->
+ [{range, {[
+ {start_key, mango_idx:start_key(Idx, Ranges)},
+ {end_key, mango_idx:end_key(Idx, Ranges)}
+ ]}}]
+ end.
+
+
+execute(#cursor{db = Db, index = Idx} = Cursor0, UserFun, UserAcc) ->
+ Cursor = Cursor0#cursor{
+ user_fun = UserFun,
+ user_acc = UserAcc
+ },
+ case Cursor#cursor.ranges of
+ [empty] ->
+ % empty indicates unsatisfiable ranges, so don't perform search
+ {ok, UserAcc};
+ _ ->
+ BaseArgs = #mrargs{
+ view_type = map,
+ reduce = false,
+ start_key = mango_idx:start_key(Idx, Cursor#cursor.ranges),
+ end_key = mango_idx:end_key(Idx, Cursor#cursor.ranges),
+ include_docs = true
+ },
+ Args = apply_opts(Cursor#cursor.opts, BaseArgs),
+ {ok, LastCursor} = case mango_idx:def(Idx) of
+ all_docs ->
+ CB = fun ?MODULE:handle_all_docs_message/2,
+ fabric:all_docs(Db, CB, Cursor, Args);
+ _ ->
+ CB = fun ?MODULE:handle_message/2,
+ % Normal view
+ DDoc = ddocid(Idx),
+ Name = mango_idx:name(Idx),
+ fabric:query_view(Db, DDoc, Name, CB, Cursor, Args)
+ end,
+ {ok, LastCursor#cursor.user_acc}
+ end.
+
+
+% Any of these indexes may be a composite index. For each
+% index find the most specific set of fields for each
+% index. Ie, if an index has columns a, b, c, d, then
+% check FieldRanges for a, b, c, and d and return
+% the longest prefix of columns found.
+composite_indexes(Indexes, FieldRanges) ->
+ lists:foldl(fun(Idx, Acc) ->
+ Cols = mango_idx:columns(Idx),
+ Prefix = composite_prefix(Cols, FieldRanges),
+ [{Idx, Prefix} | Acc]
+ end, [], Indexes).
+
+
+composite_prefix([], _) ->
+ [];
+composite_prefix([Col | Rest], Ranges) ->
+ case lists:keyfind(Col, 1, Ranges) of
+ {Col, Range} ->
+ [Range | composite_prefix(Rest, Ranges)];
+ false ->
+ []
+ end.
+
+
+% Low and behold our query planner. Or something.
+% So stupid, but we can fix this up later. First
+% pass: Sort the IndexRanges by (num_columns, idx_name)
+% and return the first element. Yes. Its going to
+% be that dumb for now.
+%
+% In the future we can look into doing a cached parallel
+% reduce view read on each index with the ranges to find
+% the one that has the fewest number of rows or something.
+choose_best_index(_DbName, IndexRanges) ->
+ Cmp = fun({A1, A2}, {B1, B2}) ->
+ case length(A2) - length(B2) of
+ N when N < 0 -> true;
+ N when N == 0 ->
+ % This is a really bad sort and will end
+ % up preferring indices based on the
+ % (dbname, ddocid, view_name) triple
+ A1 =< B1;
+ _ ->
+ false
+ end
+ end,
+ hd(lists:sort(Cmp, IndexRanges)).
+
+
+handle_message({meta, _}, Cursor) ->
+ {ok, Cursor};
+handle_message({row, Props}, Cursor) ->
+ case doc_member(Cursor#cursor.db, Props, Cursor#cursor.opts) of
+ {ok, Doc} ->
+ case mango_selector:match(Cursor#cursor.selector, Doc) of
+ true ->
+ FinalDoc = mango_fields:extract(Doc, Cursor#cursor.fields),
+ handle_doc(Cursor, FinalDoc);
+ false ->
+ {ok, Cursor}
+ end;
+ Error ->
+ couch_log:error("~s :: Error loading doc: ~p", [?MODULE, Error]),
+ {ok, Cursor}
+ end;
+handle_message(complete, Cursor) ->
+ {ok, Cursor};
+handle_message({error, Reason}, _Cursor) ->
+ {error, Reason}.
+
+
+handle_all_docs_message({row, Props}, Cursor) ->
+ case is_design_doc(Props) of
+ true -> {ok, Cursor};
+ false -> handle_message({row, Props}, Cursor)
+ end;
+handle_all_docs_message(Message, Cursor) ->
+ handle_message(Message, Cursor).
+
+
+handle_doc(#cursor{skip = S} = C, _) when S > 0 ->
+ {ok, C#cursor{skip = S - 1}};
+handle_doc(#cursor{limit = L} = C, Doc) when L > 0 ->
+ UserFun = C#cursor.user_fun,
+ UserAcc = C#cursor.user_acc,
+ {Go, NewAcc} = UserFun({row, Doc}, UserAcc),
+ {Go, C#cursor{
+ user_acc = NewAcc,
+ limit = L - 1
+ }};
+handle_doc(C, _Doc) ->
+ {stop, C}.
+
+
+ddocid(Idx) ->
+ case mango_idx:ddoc(Idx) of
+ <<"_design/", Rest/binary>> ->
+ Rest;
+ Else ->
+ Else
+ end.
+
+
+apply_opts([], Args) ->
+ Args;
+apply_opts([{r, RStr} | Rest], Args) ->
+ IncludeDocs = case list_to_integer(RStr) of
+ 1 ->
+ true;
+ R when R > 1 ->
+ % We don't load the doc in the view query because
+ % we have to do a quorum read in the coordinator
+ % so there's no point.
+ false
+ end,
+ NewArgs = Args#mrargs{include_docs = IncludeDocs},
+ apply_opts(Rest, NewArgs);
+apply_opts([{conflicts, true} | Rest], Args) ->
+ % I need to patch things so that views can specify
+ % parameters when loading the docs from disk
+ apply_opts(Rest, Args);
+apply_opts([{conflicts, false} | Rest], Args) ->
+ % Ignored cause default
+ apply_opts(Rest, Args);
+apply_opts([{sort, Sort} | Rest], Args) ->
+ % We only support single direction sorts
+ % so nothing fancy here.
+ case mango_sort:directions(Sort) of
+ [] ->
+ apply_opts(Rest, Args);
+ [<<"asc">> | _] ->
+ apply_opts(Rest, Args);
+ [<<"desc">> | _] ->
+ SK = Args#mrargs.start_key,
+ SKDI = Args#mrargs.start_key_docid,
+ EK = Args#mrargs.end_key,
+ EKDI = Args#mrargs.end_key_docid,
+ NewArgs = Args#mrargs{
+ direction = rev,
+ start_key = EK,
+ start_key_docid = EKDI,
+ end_key = SK,
+ end_key_docid = SKDI
+ },
+ apply_opts(Rest, NewArgs)
+ end;
+apply_opts([{_, _} | Rest], Args) ->
+ % Ignore unknown options
+ apply_opts(Rest, Args).
+
+
+doc_member(Db, RowProps, Opts) ->
+ case couch_util:get_value(doc, RowProps) of
+ {DocProps} ->
+ {ok, {DocProps}};
+ undefined ->
+ Id = couch_util:get_value(id, RowProps),
+ case mango_util:defer(fabric, open_doc, [Db, Id, Opts]) of
+ {ok, #doc{}=Doc} ->
+ {ok, couch_doc:to_json_obj(Doc, [])};
+ Else ->
+ Else
+ end
+ end.
+
+is_design_doc(RowProps) ->
+ case couch_util:get_value(id, RowProps) of
+ <<"_design/", _/binary>> -> true;
+ _ -> false
+ end. \ No newline at end of file
diff --git a/src/mango/src/mango_doc.erl b/src/mango/src/mango_doc.erl
new file mode 100644
index 000000000..c22b15544
--- /dev/null
+++ b/src/mango/src/mango_doc.erl
@@ -0,0 +1,537 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(mango_doc).
+
+
+-export([
+ from_bson/1,
+
+ apply_update/2,
+ update_as_insert/1,
+ has_operators/1,
+
+ get_field/2,
+ get_field/3,
+ rem_field/2,
+ set_field/3
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include("mango.hrl").
+
+
+from_bson({Props}) ->
+ DocProps = case lists:keytake(<<"_id">>, 1, Props) of
+ {value, {<<"_id">>, DocId0}, RestProps} ->
+ DocId = case DocId0 of
+ {[{<<"$id">>, Id}]} ->
+ Id;
+ Else ->
+ Else
+ end,
+ [{<<"_id">>, DocId} | RestProps];
+ false ->
+ Props
+ end,
+ Doc = couch_doc:from_json_obj({DocProps}),
+ case Doc#doc.id of
+ <<"">> ->
+ Doc#doc{id=couch_uuids:new(), revs={0, []}};
+ _ ->
+ Doc
+ end.
+
+
+apply_update(#doc{body={Props}}=Doc, Update) ->
+ NewProps = apply_update(Props, Update),
+ Doc#doc{body={NewProps}};
+apply_update({Props}, {Update}) ->
+ Result = do_update({Props}, Update),
+ case has_operators(Result) of
+ true ->
+ ?MANGO_ERROR(update_leaves_operators);
+ false ->
+ ok
+ end,
+ Result.
+
+
+update_as_insert({Update}) ->
+ NewProps = do_update_to_insert(Update, {[]}),
+ apply_update(NewProps, {Update}).
+
+
+has_operators(#doc{body=Body}) ->
+ has_operators(Body);
+has_operators({Props}) when is_list(Props) ->
+ has_operators_obj(Props);
+has_operators(Arr) when is_list(Arr) ->
+ has_operators_arr(Arr);
+has_operators(Val) when is_atom(Val) ->
+ false;
+has_operators(Val) when is_number(Val) ->
+ false;
+has_operators(Val) when is_binary(Val) ->
+ false.
+
+
+has_operators_obj([]) ->
+ false;
+has_operators_obj([{K, V} | Rest]) ->
+ case K of
+ <<"$", _/binary>> ->
+ true;
+ _ ->
+ case has_operators(V) of
+ true ->
+ true;
+ false ->
+ has_operators_obj(Rest)
+ end
+ end.
+
+
+has_operators_arr([]) ->
+ false;
+has_operators_arr([V | Rest]) ->
+ case has_operators(V) of
+ true ->
+ true;
+ false ->
+ has_operators_arr(Rest)
+ end.
+
+
+do_update(Props, []) ->
+ Props;
+do_update(Props, [{Op, Value} | Rest]) ->
+ UpdateFun = update_operator_fun(Op),
+ NewProps = case UpdateFun of
+ undefined ->
+ lists:keystore(Op, 1, Props, {Op, Value});
+ Fun when is_function(Fun, 2) ->
+ case Value of
+ {ValueProps} ->
+ Fun(Props, ValueProps);
+ _ ->
+ ?MANGO_ERROR({invalid_operand, Op, Value})
+ end
+ end,
+ do_update(NewProps, Rest).
+
+
+update_operator_fun(<<"$", _/binary>> = Op) ->
+ OperatorFuns = [
+ % Object operators
+ {<<"$inc">>, fun do_update_inc/2},
+ {<<"$rename">>, fun do_update_rename/2},
+ {<<"$setOnInsert">>, fun do_update_set_on_insert/2},
+ {<<"$set">>, fun do_update_set/2},
+ {<<"$unset">>, fun do_update_unset/2},
+
+ % Array opereators
+ {<<"$addToSet">>, fun do_update_add_to_set/2},
+ {<<"$pop">>, fun do_update_pop/2},
+ {<<"$pullAll">>, fun do_update_pull_all/2},
+ {<<"$pull">>, fun do_update_pull/2},
+ {<<"$pushAll">>, fun do_update_push_all/2},
+ {<<"$push">>, fun do_update_push/2},
+
+ % Bitwise Operators
+ {<<"$bit">>, fun do_update_bitwise/2}
+ ],
+ case lists:keyfind(Op, 1, OperatorFuns) of
+ {Op, Fun} ->
+ Fun;
+ false ->
+ ?MANGO_ERROR({update_operator_not_supported, Op})
+ end;
+update_operator_fun(_) ->
+ undefined.
+
+
+do_update_inc(Props, []) ->
+ Props;
+do_update_inc(Props, [{Field, Incr} | Rest]) ->
+ if is_number(Incr) -> ok; true ->
+ ?MANGO_ERROR({invalid_increment, Incr})
+ end,
+ NewProps = case get_field(Props, Field, fun is_number/1) of
+ Value when is_number(Value) ->
+ set_field(Props, Field, Value + Incr);
+ not_found ->
+ set_field(Props, Field, Incr);
+ _ ->
+ Props
+ end,
+ do_update_inc(NewProps, Rest).
+
+
+do_update_rename(Props, []) ->
+ Props;
+do_update_rename(Props, [{OldField, NewField} | Rest]) ->
+ NewProps = case rem_field(Props, OldField) of
+ {RemProps, OldValue} ->
+ set_field(RemProps, NewField, OldValue);
+ _ ->
+ Props
+ end,
+ do_update_rename(NewProps, Rest).
+
+
+do_update_set_on_insert(Props, _) ->
+ % This is only called during calls to apply_update/2
+ % which means this isn't an insert, so drop it on
+ % the floor.
+ Props.
+
+
+do_update_set(Props, []) ->
+ Props;
+do_update_set(Props, [{Field, Value} | Rest]) ->
+ NewProps = set_field(Props, Field, Value),
+ do_update_set(NewProps, Rest).
+
+
+do_update_unset(Props, []) ->
+ Props;
+do_update_unset(Props, [{Field, _} | Rest]) ->
+ NewProps = case rem_field(Props, Field) of
+ {RemProps, _} ->
+ RemProps;
+ _ ->
+ Props
+ end,
+ do_update_unset(NewProps, Rest).
+
+
+do_update_add_to_set(Props, []) ->
+ Props;
+do_update_add_to_set(Props, [{Field, NewValue} | Rest]) ->
+ ToAdd = case NewValue of
+ {[{<<"$each">>, NewValues}]} when is_list(NewValues) ->
+ NewValues;
+ {[{<<"$each">>, NewValue}]} ->
+ [NewValue];
+ Else ->
+ [Else]
+ end,
+ NewProps = case get_field(Props, Field) of
+ OldValues when is_list(OldValues) ->
+ FinalValues = lists:foldl(fun(V, Acc) ->
+ lists:append(Acc, [V])
+ end, OldValues, ToAdd),
+ set_field(Props, Field, FinalValues);
+ _ ->
+ Props
+ end,
+ do_update_add_to_set(NewProps, Rest).
+
+
+do_update_pop(Props, []) ->
+ Props;
+do_update_pop(Props, [{Field, Pos} | Rest]) ->
+ NewProps = case get_field(Props, Field) of
+ OldValues when is_list(OldValues) ->
+ NewValues = case Pos > 0 of
+ true ->
+ lists:sublist(OldValues, 1, length(OldValues) - 1);
+ false ->
+ lists:sublist(OldValues, 2, length(OldValues) - 1)
+ end,
+ set_field(Props, Field, NewValues);
+ _ ->
+ Props
+ end,
+ do_update_pop(NewProps, Rest).
+
+
+do_update_pull_all(Props, []) ->
+ Props;
+do_update_pull_all(Props, [{Field, Values} | Rest]) ->
+ ToRem = case is_list(Values) of
+ true -> Values;
+ false -> [Values]
+ end,
+ NewProps = case get_field(Props, Field) of
+ OldValues when is_list(OldValues) ->
+ NewValues = lists:foldl(fun(ValToRem, Acc) ->
+ % The logic in these filter functions is a bit
+ % subtle. The way to think of this is that we
+ % return true for all elements we want to keep.
+ FilterFun = case has_operators(ValToRem) of
+ true ->
+ fun(A) ->
+ Sel = mango_selector:normalize(ValToRem),
+ not mango_selector:match(A, Sel)
+ end;
+ false ->
+ fun(A) -> A /= ValToRem end
+ end,
+ lists:filter(FilterFun, Acc)
+ end, OldValues, ToRem),
+ set_field(Props, Field, NewValues);
+ _ ->
+ Props
+ end,
+ do_update_add_to_set(NewProps, Rest).
+
+
+do_update_pull(Props, []) ->
+ Props;
+do_update_pull(Props, [{Field, Value} | Rest]) ->
+ ToRem = case Value of
+ {[{<<"$each">>, Values}]} when is_list(Values) ->
+ Values;
+ {[{<<"$each">>, Value}]} ->
+ [Value];
+ Else ->
+ [Else]
+ end,
+ NewProps = do_update_pull_all(Props, [{Field, ToRem}]),
+ do_update_pull(NewProps, Rest).
+
+
+do_update_push_all(_, []) ->
+ [];
+do_update_push_all(Props, [{Field, Values} | Rest]) ->
+ ToAdd = case is_list(Values) of
+ true -> Values;
+ false -> [Values]
+ end,
+ NewProps = case get_field(Props, Field) of
+ OldValues when is_list(OldValues) ->
+ NewValues = OldValues ++ ToAdd,
+ set_field(Props, Field, NewValues);
+ _ ->
+ Props
+ end,
+ do_update_push_all(NewProps, Rest).
+
+
+do_update_push(Props, []) ->
+ Props;
+do_update_push(Props, [{Field, Value} | Rest]) ->
+ ToAdd = case Value of
+ {[{<<"$each">>, Values}]} when is_list(Values) ->
+ Values;
+ {[{<<"$each">>, Value}]} ->
+ [Value];
+ Else ->
+ [Else]
+ end,
+ NewProps = do_update_push_all(Props, [{Field, ToAdd}]),
+ do_update_push(NewProps, Rest).
+
+
+
+do_update_bitwise(Props, []) ->
+ Props;
+do_update_bitwise(Props, [{Field, Value} | Rest]) ->
+ DoOp = case Value of
+ {[{<<"and">>, Val}]} when is_integer(Val) ->
+ fun(V) -> V band Val end;
+ {[{<<"or">>, Val}]} when is_integer(Val) ->
+ fun(V) -> V bor Val end;
+ _ ->
+ fun(V) -> V end
+ end,
+ NewProps = case get_field(Props, Field, fun is_number/1) of
+ Value when is_number(Value) ->
+ NewValue = DoOp(Value),
+ set_field(Props, Field, NewValue);
+ _ ->
+ Props
+ end,
+ do_update_bitwise(NewProps, Rest).
+
+
+do_update_to_insert([], Doc) ->
+ Doc;
+do_update_to_insert([{<<"$setOnInsert">>, {FieldProps}}], Doc) ->
+ lists:foldl(fun({Field, Value}, DocAcc) ->
+ set_field(DocAcc, Field, Value)
+ end, Doc, FieldProps);
+do_update_to_insert([{_, _} | Rest], Doc) ->
+ do_update_to_insert(Rest, Doc).
+
+
+get_field(Props, Field) ->
+ get_field(Props, Field, no_validation).
+
+
+get_field(Props, Field, Validator) when is_binary(Field) ->
+ {ok, Path} = mango_util:parse_field(Field),
+ get_field(Props, Path, Validator);
+get_field(Props, [], no_validation) ->
+ Props;
+get_field(Props, [], Validator) ->
+ case (catch Validator(Props)) of
+ true ->
+ Props;
+ _ ->
+ invalid_value
+ end;
+get_field({Props}, [Name | Rest], Validator) ->
+ case lists:keyfind(Name, 1, Props) of
+ {Name, Value} ->
+ get_field(Value, Rest, Validator);
+ false ->
+ not_found
+ end;
+get_field(Values, [Name | Rest], Validator) when is_list(Values) ->
+ % Name might be an integer index into an array
+ try
+ Pos = list_to_integer(binary_to_list(Name)),
+ case Pos >= 0 andalso Pos < length(Values) of
+ true ->
+ % +1 because Erlang uses 1 based list indices
+ Value = lists:nth(Pos + 1, Values),
+ get_field(Value, Rest, Validator);
+ false ->
+ bad_path
+ end
+ catch error:badarg ->
+ bad_path
+ end;
+get_field(_, [_|_], _) ->
+ bad_path.
+
+
+rem_field(Props, Field) when is_binary(Field) ->
+ {ok, Path} = mango_util:parse_field(Field),
+ rem_field(Props, Path);
+rem_field({Props}, [Name]) ->
+ case lists:keytake(Name, 1, Props) of
+ {value, Value, NewProps} ->
+ {NewProps, Value};
+ false ->
+ not_found
+ end;
+rem_field({Props}, [Name | Rest]) ->
+ case lists:keyfind(Name, 1, Props) of
+ {Name, Value} ->
+ case rem_field(Value, Rest) of
+ {NewValue, Ret} ->
+ NewObj = {lists:keystore(Name, 1, Props, {Name, NewValue})},
+ {NewObj, Ret};
+ Else ->
+ Else
+ end;
+ false ->
+ not_found
+ end;
+rem_field(Values, [Name]) when is_list(Values) ->
+ % Name might be an integer index into an array
+ try
+ Pos = list_to_integer(binary_to_list(Name)),
+ case Pos >= 0 andalso Pos < length(Values) of
+ true ->
+ % +1 because Erlang uses 1 based list indices
+ rem_elem(Pos + 1, Values);
+ false ->
+ bad_path
+ end
+ catch error:badarg ->
+ bad_path
+ end;
+rem_field(Values, [Name | Rest]) when is_list(Values) ->
+ % Name might be an integer index into an array
+ try
+ Pos = list_to_integer(binary_to_list(Name)),
+ case Pos >= 0 andalso Pos < length(Values) of
+ true ->
+ % +1 because Erlang uses 1 based list indices
+ Value = lists:nth(Pos + 1, Values),
+ case rem_field(Value, Rest) of
+ {NewValue, Ret} ->
+ {set_elem(Pos + 1, Values, NewValue), Ret};
+ Else ->
+ Else
+ end;
+ false ->
+ bad_path
+ end
+ catch error:badarg ->
+ bad_path
+ end;
+rem_field(_, [_|_]) ->
+ bad_path.
+
+
+set_field(Props, Field, Value) when is_binary(Field) ->
+ {ok, Path} = mango_util:parse_field(Field),
+ set_field(Props, Path, Value);
+set_field({Props}, [Name], Value) ->
+ {lists:keystore(Name, 1, Props, {Name, Value})};
+set_field({Props}, [Name | Rest], Value) ->
+ case lists:keyfind(Name, 1, Props) of
+ {Name, Elem} ->
+ Result = set_field(Elem, Rest, Value),
+ {lists:keystore(Name, 1, Props, {Name, Result})};
+ false ->
+ Nested = make_nested(Rest, Value),
+ {lists:keystore(Name, 1, Props, {Name, Nested})}
+ end;
+set_field(Values, [Name], Value) when is_list(Values) ->
+ % Name might be an integer index into an array
+ try
+ Pos = list_to_integer(binary_to_list(Name)),
+ case Pos >= 0 andalso Pos < length(Values) of
+ true ->
+ % +1 because Erlang uses 1 based list indices
+ set_elem(Pos, Values, Value);
+ false ->
+ Values
+ end
+ catch error:badarg ->
+ Values
+ end;
+set_field(Values, [Name | Rest], Value) when is_list(Values) ->
+ % Name might be an integer index into an array
+ try
+ Pos = list_to_integer(binary_to_list(Name)),
+ case Pos >= 0 andalso Pos < length(Values) of
+ true ->
+ % +1 because Erlang uses 1 based list indices
+ Elem = lists:nth(Pos + 1, Values),
+ Result = set_field(Elem, Rest, Value),
+ set_elem(Pos, Values, Result);
+ false ->
+ Values
+ end
+ catch error:badarg ->
+ Values
+ end;
+set_field(Value, [_|_], _) ->
+ Value.
+
+
+make_nested([], Value) ->
+ Value;
+make_nested([Name | Rest], Value) ->
+ {[{Name, make_nested(Rest, Value)}]}.
+
+
+rem_elem(1, [Value | Rest]) ->
+ {Rest, Value};
+rem_elem(I, [Item | Rest]) when I > 1 ->
+ {Tail, Value} = rem_elem(I+1, Rest),
+ {[Item | Tail], Value}.
+
+
+set_elem(1, [_ | Rest], Value) ->
+ [Value | Rest];
+set_elem(I, [Item | Rest], Value) when I > 1 ->
+ [Item | set_elem(I-1, Rest, Value)].
diff --git a/src/mango/src/mango_epi.erl b/src/mango/src/mango_epi.erl
new file mode 100644
index 000000000..1fcd05b7f
--- /dev/null
+++ b/src/mango/src/mango_epi.erl
@@ -0,0 +1,48 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(mango_epi).
+
+-behaviour(couch_epi_plugin).
+
+-export([
+ app/0,
+ providers/0,
+ services/0,
+ data_subscriptions/0,
+ data_providers/0,
+ processes/0,
+ notify/3
+]).
+
+app() ->
+ mango.
+
+providers() ->
+ [
+ {chttpd_handlers, mango_httpd_handlers}
+ ].
+
+services() ->
+ [].
+
+data_subscriptions() ->
+ [].
+
+data_providers() ->
+ [].
+
+processes() ->
+ [].
+
+notify(_Key, _Old, _New) ->
+ ok.
diff --git a/src/mango/src/mango_error.erl b/src/mango/src/mango_error.erl
new file mode 100644
index 000000000..7d77b5e9a
--- /dev/null
+++ b/src/mango/src/mango_error.erl
@@ -0,0 +1,372 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(mango_error).
+
+
+-include_lib("couch/include/couch_db.hrl").
+
+
+-export([
+ info/2
+]).
+
+
+info(mango_cursor, {no_usable_index, no_indexes_defined}) ->
+ {
+ 400,
+ <<"no_usable_index">>,
+ <<"There are no indexes defined in this database.">>
+ };
+info(mango_cursor, {no_usable_index, no_index_matching_name}) ->
+ {
+ 400,
+ <<"no_usable_index">>,
+ <<"No index matches the index specified with \"use_index\"">>
+ };
+info(mango_cursor, {no_usable_index, missing_sort_index}) ->
+ {
+ 400,
+ <<"no_usable_index">>,
+ <<"No index exists for this sort, try indexing by the sort fields.">>
+ };
+info(mango_cursor, {no_usable_index, selector_unsupported}) ->
+ {
+ 400,
+ <<"no_usable_index">>,
+ <<"There is no index available for this selector.">>
+ };
+
+info(mango_cursor_text, {invalid_bookmark, BadBookmark}) ->
+ {
+ 400,
+ <<"invalid_bookmark">>,
+ fmt("Invalid boomkark value: ~s", [?JSON_ENCODE(BadBookmark)])
+ };
+info(mango_cursor_text, multiple_text_indexes) ->
+ {
+ 400,
+ <<"multiple_text_indexes">>,
+ <<"You must specify an index with the `use_index` parameter.">>
+ };
+info(mango_cursor_text, {text_search_error, {error, {bad_request, Msg}}})
+ when is_binary(Msg) ->
+ {
+ 400,
+ <<"text_search_error">>,
+ Msg
+ };
+info(mango_cursor_text, {text_search_error, {error, Error}}) ->
+ {
+ 400,
+ <<"text_search_error">>,
+ fmt("~p", [Error])
+ };
+
+info(mango_fields, {invalid_fields_json, BadFields}) ->
+ {
+ 400,
+ <<"invalid_fields">>,
+ fmt("Fields must be an array of strings, not: ~w", [BadFields])
+ };
+info(mango_fields, {invalid_field_json, BadField}) ->
+ {
+ 400,
+ <<"invalid_field">>,
+ fmt("Invalid JSON for field spec: ~w", [BadField])
+ };
+
+info(mango_httpd, error_saving_ddoc) ->
+ {
+ 500,
+ <<"error_saving_ddoc">>,
+ <<"Unknown error while saving the design document.">>
+ };
+info(mango_httpd, {error_saving_ddoc, <<"conflict">>}) ->
+ {
+ 500,
+ <<"error_saving_ddoc">>,
+ <<"Encountered a conflict while saving the design document.">>
+ };
+info(mango_httpd, {error_saving_ddoc, Reason}) ->
+ {
+ 500,
+ <<"error_saving_ddoc">>,
+ fmt("Unknown error while saving the design document: ~s", [Reason])
+ };
+info(mango_httpd, invalid_list_index_params) ->
+ {
+ 500,
+ <<"invalid_list_index_params">>,
+ <<"Index parameter ranges: limit > 1, skip > 0" >>
+ };
+
+info(mango_idx, {invalid_index_type, BadType}) ->
+ {
+ 400,
+ <<"invalid_index">>,
+ fmt("Invalid type for index: ~s", [BadType])
+ };
+info(mango_idx, invalid_query_ddoc_language) ->
+ {
+ 400,
+ <<"invalid_index">>,
+ <<"Invalid design document query language.">>
+ };
+info(mango_idx, no_index_definition) ->
+ {
+ 400,
+ <<"invalid_index">>,
+ <<"Index is missing its definition.">>
+ };
+info(mango_idx, {index_not_implemented, IndexName}) ->
+ {
+ 501,
+ <<"index_not_implemented">>,
+ fmt("~s", [IndexName])
+ };
+info(mango_idx, {index_service_unavailable, IndexName}) ->
+ {
+ 503,
+ <<"required index service unavailable">>,
+ fmt("~s", [IndexName])
+ };
+
+info(mango_idx_view, {invalid_index_json, BadIdx}) ->
+ {
+ 400,
+ <<"invalid_index">>,
+ fmt("JSON indexes must be an object, not: ~w", [BadIdx])
+ };
+info(mango_idx_text, {invalid_index_fields_definition, Def}) ->
+ {
+ 400,
+ <<"invalid_index_fields_definition">>,
+ fmt("Text Index field definitions must be of the form
+ {\"name\": \"non-empty fieldname\", \"type\":
+ \"boolean,number, or string\"}. Def: ~p", [Def])
+ };
+info(mango_idx_view, {index_not_found, BadIdx}) ->
+ {
+ 404,
+ <<"invalid_index">>,
+ fmt("JSON index ~s not found in this design doc.", [BadIdx])
+ };
+
+info(mango_idx_text, {invalid_index_text, BadIdx}) ->
+ {
+ 400,
+ <<"invalid_index">>,
+ fmt("Text indexes must be an object, not: ~w", [BadIdx])
+ };
+info(mango_idx_text, {index_not_found, BadIdx}) ->
+ {
+ 404,
+ <<"index_not_found">>,
+ fmt("Text index ~s not found in this design doc.", [BadIdx])
+ };
+info(mango_idx_text, index_all_disabled) ->
+ {
+ 403,
+ <<"index_all_disabled">>,
+ <<"New text indexes are forbidden to index all fields.">>
+ };
+
+info(mango_opts, {invalid_bulk_docs, Val}) ->
+ {
+ 400,
+ <<"invalid_bulk_docs">>,
+ fmt("Bulk Delete requires an array of non-null docids. Docids: ~w",
+ [Val])
+ };
+info(mango_opts, {invalid_ejson, Val}) ->
+ {
+ 400,
+ <<"invalid_ejson">>,
+ fmt("Invalid JSON value: ~w", [Val])
+ };
+info(mango_opts, {invalid_key, Key}) ->
+ {
+ 400,
+ <<"invalid_key">>,
+ fmt("Invalid key ~s for this request.", [Key])
+ };
+info(mango_opts, {missing_required_key, Key}) ->
+ {
+ 400,
+ <<"missing_required_key">>,
+ fmt("Missing required key: ~s", [Key])
+ };
+info(mango_opts, {invalid_value, Name, Expect, Found}) ->
+ {
+ 400,
+ <<"invalid_value">>,
+ fmt("Value for ~s is ~w, should be ~w", [Name, Found, Expect])
+ };
+info(mango_opts, {invalid_value, Name, Value}) ->
+ {
+ 400,
+ <<"invalid_value">>,
+ fmt("Invalid value for ~s: ~w", [Name, Value])
+ };
+info(mango_opts, {invalid_string, Val}) ->
+ {
+ 400,
+ <<"invalid_string">>,
+ fmt("Invalid string: ~w", [Val])
+ };
+info(mango_opts, {invalid_boolean, Val}) ->
+ {
+ 400,
+ <<"invalid_boolean">>,
+ fmt("Invalid boolean value: ~w", [Val])
+ };
+info(mango_opts, {invalid_pos_integer, Val}) ->
+ {
+ 400,
+ <<"invalid_pos_integer">>,
+ fmt("~w is not an integer greater than zero", [Val])
+ };
+info(mango_opts, {invalid_non_neg_integer, Val}) ->
+ {
+ 400,
+ <<"invalid_non_neg_integer">>,
+ fmt("~w is not an integer greater than or equal to zero", [Val])
+ };
+info(mango_opts, {invalid_object, BadObj}) ->
+ {
+ 400,
+ <<"invalid_object">>,
+ fmt("~w is not a JSON object", [BadObj])
+ };
+info(mango_opts, {invalid_selector_json, BadSel}) ->
+ {
+ 400,
+ <<"invalid_selector_json">>,
+ fmt("Selector must be a JSON object, not: ~w", [BadSel])
+ };
+info(mango_opts, {invalid_index_name, BadName}) ->
+ {
+ 400,
+ <<"invalid_index_name">>,
+ fmt("Invalid index name: ~w", [BadName])
+ };
+
+info(mango_opts, {multiple_text_operator, {invalid_selector, BadSel}}) ->
+ {
+ 400,
+ <<"multiple_text_selector">>,
+ fmt("Selector cannot contain more than one $text operator: ~w",
+ [BadSel])
+ };
+
+info(mango_selector, {invalid_selector, missing_field_name}) ->
+ {
+ 400,
+ <<"invalid_selector">>,
+ <<"One or more conditions is missing a field name.">>
+ };
+info(mango_selector, {bad_arg, Op, Arg}) ->
+ {
+ 400,
+ <<"bad_arg">>,
+ fmt("Bad argument for operator ~s: ~w", [Op, Arg])
+ };
+info(mango_selector, {not_supported, Op}) ->
+ {
+ 400,
+ <<"not_supported">>,
+ fmt("Unsupported operator: ~s", [Op])
+ };
+info(mango_selector, {invalid_operator, Op}) ->
+ {
+ 400,
+ <<"invalid_operator">>,
+ fmt("Invalid operator: ~s", [Op])
+ };
+info(mango_selector, {bad_field, BadSel}) ->
+ {
+ 400,
+ <<"bad_field">>,
+ fmt("Invalid field normalization on selector: ~w", [BadSel])
+ };
+
+info(mango_selector_text, {invalid_operator, Op}) ->
+ {
+ 400,
+ <<"invalid_operator">>,
+ fmt("Invalid text operator: ~s", [Op])
+ };
+info(mango_selector_text, {text_sort_error, Field}) ->
+ S = binary_to_list(Field),
+ Msg = "Unspecified or ambiguous sort type. Try appending :number or"
+ " :string to the sort field. ~s",
+ {
+ 400,
+ <<"text_sort_error">>,
+ fmt(Msg, [S])
+ };
+
+info(mango_sort, {invalid_sort_json, BadSort}) ->
+ {
+ 400,
+ <<"invalid_sort_json">>,
+ fmt("Sort must be an array of sort specs, not: ~w", [BadSort])
+ };
+info(mango_sort, {invalid_sort_dir, BadSpec}) ->
+ {
+ 400,
+ <<"invalid_sort_dir">>,
+ fmt("Invalid sort direction: ~w", BadSpec)
+ };
+info(mango_sort, {invalid_sort_field, BadField}) ->
+ {
+ 400,
+ <<"invalid_sort_field">>,
+ fmt("Invalid sort field: ~w", [BadField])
+ };
+info(mango_sort, {unsupported, mixed_sort}) ->
+ {
+ 400,
+ <<"unsupported_mixed_sort">>,
+ <<"Sorts currently only support a single direction for all fields.">>
+ };
+
+info(mango_util, {error_loading_doc, DocId}) ->
+ {
+ 500,
+ <<"internal_error">>,
+ fmt("Error loading doc: ~s", [DocId])
+ };
+info(mango_util, error_loading_ddocs) ->
+ {
+ 500,
+ <<"internal_error">>,
+ <<"Error loading design documents">>
+ };
+info(mango_util, {invalid_ddoc_lang, Lang}) ->
+ {
+ 400,
+ <<"invalid_ddoc_lang">>,
+ fmt("Existing design doc has an invalid language: ~w", [Lang])
+ };
+
+info(Module, Reason) ->
+ {
+ 500,
+ <<"unknown_error">>,
+ fmt("Unknown Error: ~s :: ~w", [Module, Reason])
+ }.
+
+
+fmt(Format, Args) ->
+ iolist_to_binary(io_lib:format(Format, Args)).
diff --git a/src/mango/src/mango_fields.erl b/src/mango/src/mango_fields.erl
new file mode 100644
index 000000000..273256025
--- /dev/null
+++ b/src/mango/src/mango_fields.erl
@@ -0,0 +1,55 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(mango_fields).
+
+-export([
+ new/1,
+ extract/2
+]).
+
+
+-include("mango.hrl").
+
+
+new([]) ->
+ {ok, all_fields};
+new(Fields) when is_list(Fields) ->
+ {ok, [field(F) || F <- Fields]};
+new(Else) ->
+ ?MANGO_ERROR({invalid_fields_json, Else}).
+
+
+extract(Doc, undefined) ->
+ Doc;
+extract(Doc, all_fields) ->
+ Doc;
+extract(Doc, Fields) ->
+ lists:foldl(fun(F, NewDoc) ->
+ {ok, Path} = mango_util:parse_field(F),
+ case mango_doc:get_field(Doc, Path) of
+ not_found ->
+ NewDoc;
+ bad_path ->
+ NewDoc;
+ Value ->
+ mango_doc:set_field(NewDoc, Path, Value)
+ end
+ end, {[]}, Fields).
+
+
+field(Val) when is_binary(Val) ->
+ Val;
+field({Val}) when is_list(Val) ->
+ {Val};
+field(Else) ->
+ ?MANGO_ERROR({invalid_field_json, Else}).
diff --git a/src/mango/src/mango_httpd.erl b/src/mango/src/mango_httpd.erl
new file mode 100644
index 000000000..a08827649
--- /dev/null
+++ b/src/mango/src/mango_httpd.erl
@@ -0,0 +1,305 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(mango_httpd).
+
+
+-export([
+ handle_req/2
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include("mango.hrl").
+-include("mango_idx.hrl").
+
+-record(vacc, {
+ resp,
+ prepend,
+ kvs,
+ buffer = [],
+ bufsize = 0,
+ threshold = 1490
+}).
+
+handle_req(#httpd{} = Req, Db0) ->
+ try
+ Db = set_user_ctx(Req, Db0),
+ handle_req_int(Req, Db)
+ catch
+ throw:{mango_error, Module, Reason} ->
+ %Stack = erlang:get_stacktrace(),
+ {Code, ErrorStr, ReasonStr} = mango_error:info(Module, Reason),
+ Resp = {[
+ {<<"error">>, ErrorStr},
+ {<<"reason">>, ReasonStr}
+ ]},
+ chttpd:send_json(Req, Code, Resp)
+ end.
+
+
+handle_req_int(#httpd{path_parts=[_, <<"_index">> | _]} = Req, Db) ->
+ handle_index_req(Req, Db);
+handle_req_int(#httpd{path_parts=[_, <<"_explain">> | _]} = Req, Db) ->
+ handle_explain_req(Req, Db);
+handle_req_int(#httpd{path_parts=[_, <<"_find">> | _]} = Req, Db) ->
+ handle_find_req(Req, Db);
+handle_req_int(_, _) ->
+ throw({not_found, missing}).
+
+
+handle_index_req(#httpd{method='GET', path_parts=[_, _]}=Req, Db) ->
+ Params = lists:flatmap(fun({K, V}) -> parse_index_param(K, V) end,
+ chttpd:qs(Req)),
+ Idxs = lists:sort(mango_idx:list(Db)),
+ JsonIdxs0 = lists:map(fun mango_idx:to_json/1, Idxs),
+ TotalRows = length(JsonIdxs0),
+ Limit = case couch_util:get_value(limit, Params, TotalRows) of
+ Limit0 when Limit0 < 1 ->
+ ?MANGO_ERROR(invalid_list_index_params);
+ Limit0 ->
+ Limit0
+ end,
+ Skip = case couch_util:get_value(skip, Params, 0) of
+ Skip0 when Skip0 < 0 ->
+ ?MANGO_ERROR(invalid_list_index_params);
+ Skip0 when Skip0 > TotalRows ->
+ TotalRows;
+ Skip0 ->
+ Skip0
+ end,
+ JsonIdxs = lists:sublist(JsonIdxs0, Skip+1, Limit),
+ chttpd:send_json(Req, {[{total_rows, TotalRows}, {indexes, JsonIdxs}]});
+
+handle_index_req(#httpd{method='POST', path_parts=[_, _]}=Req, Db) ->
+ chttpd:validate_ctype(Req, "application/json"),
+ {ok, Opts} = mango_opts:validate_idx_create(chttpd:json_body_obj(Req)),
+ {ok, Idx0} = mango_idx:new(Db, Opts),
+ {ok, Idx} = mango_idx:validate_new(Idx0, Db),
+ {ok, DDoc} = mango_util:load_ddoc(Db, mango_idx:ddoc(Idx)),
+ Id = Idx#idx.ddoc,
+ Name = Idx#idx.name,
+ Status = case mango_idx:add(DDoc, Idx) of
+ {ok, DDoc} ->
+ <<"exists">>;
+ {ok, NewDDoc} ->
+ CreateOpts = get_idx_w_opts(Opts),
+ case mango_crud:insert(Db, NewDDoc, CreateOpts) of
+ {ok, [{RespProps}]} ->
+ case lists:keyfind(error, 1, RespProps) of
+ {error, Reason} ->
+ ?MANGO_ERROR({error_saving_ddoc, Reason});
+ _ ->
+ <<"created">>
+ end;
+ _ ->
+ ?MANGO_ERROR(error_saving_ddoc)
+ end
+ end,
+ chttpd:send_json(Req, {[{result, Status}, {id, Id}, {name, Name}]});
+
+handle_index_req(#httpd{path_parts=[_, _]}=Req, _Db) ->
+ chttpd:send_method_not_allowed(Req, "GET,POST");
+
+%% Essentially we just iterate through the list of ddoc ids passed in and
+%% delete one by one. If an error occurs, all previous documents will be
+%% deleted, but an error will be thrown for the current ddoc id.
+handle_index_req(#httpd{method='POST', path_parts=[_, <<"_index">>,
+ <<"_bulk_delete">>]}=Req, Db) ->
+ chttpd:validate_ctype(Req, "application/json"),
+ {ok, Opts} = mango_opts:validate_bulk_delete(chttpd:json_body_obj(Req)),
+ Idxs = mango_idx:list(Db),
+ DDocs = get_bulk_delete_ddocs(Opts),
+ DelOpts = get_idx_w_opts(Opts),
+ {Success, Fail} = lists:foldl(fun(DDocId0, {Success0, Fail0}) ->
+ DDocId = convert_to_design_id(DDocId0),
+ Filt = fun(Idx) -> mango_idx:ddoc(Idx) == DDocId end,
+ Id = {<<"id">>, DDocId},
+ case mango_idx:delete(Filt, Db, Idxs, DelOpts) of
+ {ok, true} ->
+ {[{[Id, {<<"ok">>, true}]} | Success0], Fail0};
+ {error, Error} ->
+ {Success0, [{[Id, {<<"error">>, Error}]} | Fail0]}
+ end
+ end, {[], []}, DDocs),
+ chttpd:send_json(Req, {[{<<"success">>, Success}, {<<"fail">>, Fail}]});
+
+handle_index_req(#httpd{path_parts=[_, <<"_index">>,
+ <<"_bulk_delete">>]}=Req, _Db) ->
+ chttpd:send_method_not_allowed(Req, "POST");
+
+handle_index_req(#httpd{method='DELETE',
+ path_parts=[A, B, <<"_design">>, DDocId0, Type, Name]}=Req, Db) ->
+ PathParts = [A, B, <<"_design/", DDocId0/binary>>, Type, Name],
+ handle_index_req(Req#httpd{path_parts=PathParts}, Db);
+
+handle_index_req(#httpd{method='DELETE',
+ path_parts=[_, _, DDocId0, Type, Name]}=Req, Db) ->
+ Idxs = mango_idx:list(Db),
+ DDocId = convert_to_design_id(DDocId0),
+ DelOpts = get_idx_del_opts(Req),
+ Filt = fun(Idx) ->
+ IsDDoc = mango_idx:ddoc(Idx) == DDocId,
+ IsType = mango_idx:type(Idx) == Type,
+ IsName = mango_idx:name(Idx) == Name,
+ IsDDoc andalso IsType andalso IsName
+ end,
+ case mango_idx:delete(Filt, Db, Idxs, DelOpts) of
+ {ok, true} ->
+ chttpd:send_json(Req, {[{ok, true}]});
+ {error, not_found} ->
+ throw({not_found, missing});
+ {error, Error} ->
+ ?MANGO_ERROR({error_saving_ddoc, Error})
+ end;
+
+handle_index_req(#httpd{path_parts=[_, _, _DDocId0, _Type, _Name]}=Req, _Db) ->
+ chttpd:send_method_not_allowed(Req, "DELETE").
+
+
+handle_explain_req(#httpd{method='POST'}=Req, Db) ->
+ chttpd:validate_ctype(Req, "application/json"),
+ {ok, Opts0} = mango_opts:validate_find(chttpd:json_body_obj(Req)),
+ {value, {selector, Sel}, Opts} = lists:keytake(selector, 1, Opts0),
+ Resp = mango_crud:explain(Db, Sel, Opts),
+ chttpd:send_json(Req, Resp);
+
+handle_explain_req(Req, _Db) ->
+ chttpd:send_method_not_allowed(Req, "POST").
+
+
+handle_find_req(#httpd{method='POST'}=Req, Db) ->
+ chttpd:validate_ctype(Req, "application/json"),
+ {ok, Opts0} = mango_opts:validate_find(chttpd:json_body_obj(Req)),
+ {value, {selector, Sel}, Opts} = lists:keytake(selector, 1, Opts0),
+ {ok, Resp0} = start_find_resp(Req, Db, Sel, Opts),
+ {ok, AccOut} = run_find(Resp0, Db, Sel, Opts),
+ end_find_resp(AccOut);
+
+handle_find_req(Req, _Db) ->
+ chttpd:send_method_not_allowed(Req, "POST").
+
+
+set_user_ctx(#httpd{user_ctx=Ctx}, Db) ->
+ Db#db{user_ctx=Ctx}.
+
+
+get_idx_w_opts(Opts) ->
+ case lists:keyfind(w, 1, Opts) of
+ {w, N} when is_integer(N), N > 0 ->
+ [{w, integer_to_list(N)}];
+ _ ->
+ [{w, "2"}]
+ end.
+
+
+get_bulk_delete_ddocs(Opts) ->
+ case lists:keyfind(docids, 1, Opts) of
+ {docids, DDocs} when is_list(DDocs) ->
+ DDocs;
+ _ ->
+ []
+ end.
+
+
+get_idx_del_opts(Req) ->
+ try
+ WStr = chttpd:qs_value(Req, "w", "2"),
+ _ = list_to_integer(WStr),
+ [{w, WStr}]
+ catch _:_ ->
+ [{w, "2"}]
+ end.
+
+
+convert_to_design_id(DDocId) ->
+ case DDocId of
+ <<"_design/", _/binary>> -> DDocId;
+ _ -> <<"_design/", DDocId/binary>>
+ end.
+
+
+start_find_resp(Req, Db, Sel, Opts) ->
+ chttpd:start_delayed_json_response(Req, 200, [], maybe_add_warning(Db, Sel, Opts)).
+
+
+maybe_add_warning(Db, Selector, Opts) ->
+ UsableIndexes = mango_idx:get_usable_indexes(Db, Selector, Opts),
+ case length(UsableIndexes) of
+ 0 ->
+ "{\"warning\":\"no matching index found, create an index to optimize query time\",\r\n\"docs\":[";
+ _ ->
+ "{\"docs\":["
+ end.
+
+
+end_find_resp(Acc0) ->
+ #vacc{resp=Resp00, buffer=Buf, kvs=KVs, threshold=Max} = Acc0,
+ {ok, Resp0} = chttpd:close_delayed_json_object(Resp00, Buf, "\r\n]", Max),
+ FinalAcc = lists:foldl(fun({K, V}, Acc) ->
+ JK = ?JSON_ENCODE(K),
+ JV = ?JSON_ENCODE(V),
+ [JV, ": ", JK, ",\r\n" | Acc]
+ end, [], KVs),
+ Chunk = lists:reverse(FinalAcc, ["}\r\n"]),
+ {ok, Resp1} = chttpd:send_delayed_chunk(Resp0, Chunk),
+ chttpd:end_delayed_json_response(Resp1).
+
+
+run_find(Resp, Db, Sel, Opts) ->
+ Acc0 = #vacc{
+ resp = Resp,
+ prepend = "\r\n",
+ kvs = [],
+ threshold = chttpd:chunked_response_buffer_size()
+ },
+ mango_crud:find(Db, Sel, fun handle_doc/2, Acc0, Opts).
+
+
+handle_doc({add_key, Key, Value}, Acc0) ->
+ #vacc{kvs=KVs} = Acc0,
+ NewKVs = lists:keystore(Key, 1, KVs, {Key, Value}),
+ {ok, Acc0#vacc{kvs = NewKVs}};
+handle_doc({row, Doc}, Acc0) ->
+ #vacc{prepend=Prepend} = Acc0,
+ Chunk = [Prepend, ?JSON_ENCODE(Doc)],
+ maybe_flush_response(Acc0, Chunk, iolist_size(Chunk)).
+
+maybe_flush_response(#vacc{bufsize=Size, threshold=Max} = Acc, Data, Len)
+ when Size > 0 andalso (Size + Len) > Max ->
+ #vacc{buffer = Buffer, resp = Resp} = Acc,
+ {ok, R1} = chttpd:send_delayed_chunk(Resp, Buffer),
+ {ok, Acc#vacc{prepend = ",\r\n", buffer = Data, bufsize = Len, resp = R1}};
+maybe_flush_response(Acc0, Data, Len) ->
+ #vacc{buffer = Buf, bufsize = Size} = Acc0,
+ Acc = Acc0#vacc{
+ prepend = ",\r\n",
+ buffer = [Buf | Data],
+ bufsize = Size + Len
+ },
+ {ok, Acc}.
+
+
+parse_index_param("limit", Value) ->
+ [{limit, parse_val(Value)}];
+parse_index_param("skip", Value) ->
+ [{skip, parse_val(Value)}];
+parse_index_param(_Key, _Value) ->
+ [].
+
+parse_val(Value) ->
+ case (catch list_to_integer(Value)) of
+ IntVal when is_integer(IntVal) ->
+ IntVal;
+ _ ->
+ ?MANGO_ERROR(invalid_list_index_params)
+ end.
diff --git a/src/mango/src/mango_httpd_handlers.erl b/src/mango/src/mango_httpd_handlers.erl
new file mode 100644
index 000000000..80e5e277e
--- /dev/null
+++ b/src/mango/src/mango_httpd_handlers.erl
@@ -0,0 +1,24 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(mango_httpd_handlers).
+
+-export([url_handler/1, db_handler/1, design_handler/1]).
+
+url_handler(_) -> no_match.
+
+db_handler(<<"_index">>) -> fun mango_httpd:handle_req/2;
+db_handler(<<"_explain">>) -> fun mango_httpd:handle_req/2;
+db_handler(<<"_find">>) -> fun mango_httpd:handle_req/2;
+db_handler(_) -> no_match.
+
+design_handler(_) -> no_match.
diff --git a/src/mango/src/mango_idx.erl b/src/mango/src/mango_idx.erl
new file mode 100644
index 000000000..bc88b970c
--- /dev/null
+++ b/src/mango/src/mango_idx.erl
@@ -0,0 +1,369 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% This module is for the "index object" as in, the data structure
+% representing an index. Not to be confused with mango_index which
+% contains APIs for managing indexes.
+
+-module(mango_idx).
+
+
+-export([
+ list/1,
+ recover/1,
+ for_sort/2,
+
+ new/2,
+ validate_new/2,
+ add/2,
+ remove/2,
+ from_ddoc/2,
+ special/1,
+
+ dbname/1,
+ ddoc/1,
+ name/1,
+ type/1,
+ def/1,
+ opts/1,
+ columns/1,
+ is_usable/2,
+ start_key/2,
+ end_key/2,
+ cursor_mod/1,
+ idx_mod/1,
+ to_json/1,
+ delete/4,
+ get_usable_indexes/3
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include("mango.hrl").
+-include("mango_idx.hrl").
+
+
+list(Db) ->
+ {ok, Indexes} = ddoc_cache:open(db_to_name(Db), ?MODULE),
+ Indexes.
+
+get_usable_indexes(Db, Selector0, Opts) ->
+ Selector = mango_selector:normalize(Selector0),
+
+ ExistingIndexes = mango_idx:list(Db),
+ if ExistingIndexes /= [] -> ok; true ->
+ ?MANGO_ERROR({no_usable_index, no_indexes_defined})
+ end,
+
+ FilteredIndexes = mango_cursor:maybe_filter_indexes(ExistingIndexes, Opts),
+ if FilteredIndexes /= [] -> ok; true ->
+ ?MANGO_ERROR({no_usable_index, no_index_matching_name})
+ end,
+
+ SortIndexes = mango_idx:for_sort(FilteredIndexes, Opts),
+ if SortIndexes /= [] -> ok; true ->
+ ?MANGO_ERROR({no_usable_index, missing_sort_index})
+ end,
+
+ UsableFilter = fun(I) -> mango_idx:is_usable(I, Selector) end,
+ lists:filter(UsableFilter, SortIndexes).
+
+recover(Db) ->
+ {ok, DDocs0} = mango_util:open_ddocs(Db),
+ Pred = fun({Props}) ->
+ case proplists:get_value(<<"language">>, Props) of
+ <<"query">> -> true;
+ _ -> false
+ end
+ end,
+ DDocs = lists:filter(Pred, DDocs0),
+ Special = special(Db),
+ {ok, Special ++ lists:flatmap(fun(Doc) ->
+ from_ddoc(Db, Doc)
+ end, DDocs)}.
+
+
+for_sort(Indexes, Opts) ->
+ % If a sort was specified we have to find an index that
+ % can satisfy the request.
+ case lists:keyfind(sort, 1, Opts) of
+ {sort, {SProps}} when is_list(SProps) ->
+ for_sort_int(Indexes, {SProps});
+ _ ->
+ Indexes
+ end.
+
+
+for_sort_int(Indexes, Sort) ->
+ Fields = mango_sort:fields(Sort),
+ FilterFun = fun(Idx) ->
+ Cols = mango_idx:columns(Idx),
+ case {mango_idx:type(Idx), Cols} of
+ {_, all_fields} ->
+ true;
+ {<<"text">>, _} ->
+ sets:is_subset(sets:from_list(Fields), sets:from_list(Cols));
+ {<<"json">>, _} ->
+ lists:prefix(Fields, Cols);
+ {<<"special">>, _} ->
+ lists:prefix(Fields, Cols)
+ end
+ end,
+ lists:filter(FilterFun, Indexes).
+
+
+new(Db, Opts) ->
+ Def = get_idx_def(Opts),
+ Type = get_idx_type(Opts),
+ IdxName = get_idx_name(Def, Opts),
+ DDoc = get_idx_ddoc(Def, Opts),
+ {ok, #idx{
+ dbname = db_to_name(Db),
+ ddoc = DDoc,
+ name = IdxName,
+ type = Type,
+ def = Def,
+ opts = filter_opts(Opts)
+ }}.
+
+
+validate_new(Idx, Db) ->
+ Mod = idx_mod(Idx),
+ Mod:validate_new(Idx, Db).
+
+
+add(DDoc, Idx) ->
+ Mod = idx_mod(Idx),
+ {ok, NewDDoc} = Mod:add(DDoc, Idx),
+ % Round trip through JSON for normalization
+ Body = ?JSON_DECODE(?JSON_ENCODE(NewDDoc#doc.body)),
+ {ok, NewDDoc#doc{body = Body}}.
+
+
+remove(DDoc, Idx) ->
+ Mod = idx_mod(Idx),
+ {ok, NewDDoc} = Mod:remove(DDoc, Idx),
+ % Round trip through JSON for normalization
+ Body = ?JSON_DECODE(?JSON_ENCODE(NewDDoc#doc.body)),
+ {ok, NewDDoc#doc{body = Body}}.
+
+
+delete(Filt, Db, Indexes, DelOpts) ->
+ case lists:filter(Filt, Indexes) of
+ [Idx] ->
+ {ok, DDoc} = mango_util:load_ddoc(Db, mango_idx:ddoc(Idx)),
+ {ok, NewDDoc} = mango_idx:remove(DDoc, Idx),
+ FinalDDoc = case NewDDoc#doc.body of
+ {[{<<"language">>, <<"query">>}]} ->
+ NewDDoc#doc{deleted = true, body = {[]}};
+ _ ->
+ NewDDoc
+ end,
+ case mango_crud:insert(Db, FinalDDoc, DelOpts) of
+ {ok, _} ->
+ {ok, true};
+ Error ->
+ {error, Error}
+ end;
+ [] ->
+ {error, not_found}
+ end.
+
+
+from_ddoc(Db, {Props}) ->
+ DbName = db_to_name(Db),
+ DDoc = proplists:get_value(<<"_id">>, Props),
+
+ case proplists:get_value(<<"language">>, Props) of
+ <<"query">> -> ok;
+ _ ->
+ ?MANGO_ERROR(invalid_query_ddoc_language)
+ end,
+ IdxMods = case module_loaded(dreyfus_index) of
+ true ->
+ [mango_idx_view, mango_idx_text];
+ false ->
+ [mango_idx_view]
+ end,
+ Idxs = lists:flatmap(fun(Mod) -> Mod:from_ddoc({Props}) end, IdxMods),
+ lists:map(fun(Idx) ->
+ Idx#idx{
+ dbname = DbName,
+ ddoc = DDoc
+ }
+ end, Idxs).
+
+
+special(Db) ->
+ AllDocs = #idx{
+ dbname = db_to_name(Db),
+ name = <<"_all_docs">>,
+ type = <<"special">>,
+ def = all_docs,
+ opts = []
+ },
+ % Add one for _update_seq
+ [AllDocs].
+
+
+dbname(#idx{dbname=DbName}) ->
+ DbName.
+
+
+ddoc(#idx{ddoc=DDoc}) ->
+ DDoc.
+
+
+name(#idx{name=Name}) ->
+ Name.
+
+
+type(#idx{type=Type}) ->
+ Type.
+
+
+def(#idx{def=Def}) ->
+ Def.
+
+
+opts(#idx{opts=Opts}) ->
+ Opts.
+
+
+to_json(#idx{}=Idx) ->
+ Mod = idx_mod(Idx),
+ Mod:to_json(Idx).
+
+
+columns(#idx{}=Idx) ->
+ Mod = idx_mod(Idx),
+ Mod:columns(Idx).
+
+
+is_usable(#idx{}=Idx, Selector) ->
+ Mod = idx_mod(Idx),
+ Mod:is_usable(Idx, Selector).
+
+
+start_key(#idx{}=Idx, Ranges) ->
+ Mod = idx_mod(Idx),
+ Mod:start_key(Ranges).
+
+
+end_key(#idx{}=Idx, Ranges) ->
+ Mod = idx_mod(Idx),
+ Mod:end_key(Ranges).
+
+
+cursor_mod(#idx{type = <<"json">>}) ->
+ mango_cursor_view;
+cursor_mod(#idx{def = all_docs, type= <<"special">>}) ->
+ mango_cursor_special;
+cursor_mod(#idx{type = <<"text">>}) ->
+ case module_loaded(dreyfus_index) of
+ true ->
+ mango_cursor_text;
+ false ->
+ ?MANGO_ERROR({index_service_unavailable, <<"text">>})
+ end.
+
+
+idx_mod(#idx{type = <<"json">>}) ->
+ mango_idx_view;
+idx_mod(#idx{type = <<"special">>}) ->
+ mango_idx_special;
+idx_mod(#idx{type = <<"text">>}) ->
+ case module_loaded(dreyfus_index) of
+ true ->
+ mango_idx_text;
+ false ->
+ ?MANGO_ERROR({index_service_unavailable, <<"text">>})
+ end.
+
+
+db_to_name(#db{name=Name}) ->
+ Name;
+db_to_name(Name) when is_binary(Name) ->
+ Name;
+db_to_name(Name) when is_list(Name) ->
+ iolist_to_binary(Name).
+
+
+get_idx_def(Opts) ->
+ case proplists:get_value(def, Opts) of
+ undefined ->
+ ?MANGO_ERROR(no_index_definition);
+ Def ->
+ Def
+ end.
+
+
+get_idx_type(Opts) ->
+ case proplists:get_value(type, Opts) of
+ <<"json">> -> <<"json">>;
+ <<"text">> -> case module_loaded(dreyfus_index) of
+ true ->
+ <<"text">>;
+ false ->
+ ?MANGO_ERROR({index_service_unavailable, <<"text">>})
+ end;
+ %<<"geo">> -> <<"geo">>;
+ undefined -> <<"json">>;
+ BadType ->
+ ?MANGO_ERROR({invalid_index_type, BadType})
+ end.
+
+
+get_idx_ddoc(Idx, Opts) ->
+ case proplists:get_value(ddoc, Opts) of
+ <<"_design/", _Rest>> = Name ->
+ Name;
+ Name when is_binary(Name) ->
+ <<"_design/", Name/binary>>;
+ _ ->
+ Bin = gen_name(Idx, Opts),
+ <<"_design/", Bin/binary>>
+ end.
+
+
+get_idx_name(Idx, Opts) ->
+ case proplists:get_value(name, Opts) of
+ Name when is_binary(Name) ->
+ Name;
+ _ ->
+ gen_name(Idx, Opts)
+ end.
+
+
+gen_name(Idx, Opts0) ->
+ Opts = lists:usort(Opts0),
+ TermBin = term_to_binary({Idx, Opts}),
+ Sha = couch_crypto:hash(sha, TermBin),
+ mango_util:enc_hex(Sha).
+
+
+filter_opts([]) ->
+ [];
+filter_opts([{user_ctx, _} | Rest]) ->
+ filter_opts(Rest);
+filter_opts([{ddoc, _} | Rest]) ->
+ filter_opts(Rest);
+filter_opts([{name, _} | Rest]) ->
+ filter_opts(Rest);
+filter_opts([{type, _} | Rest]) ->
+ filter_opts(Rest);
+filter_opts([{w, _} | Rest]) ->
+ filter_opts(Rest);
+filter_opts([Opt | Rest]) ->
+ [Opt | filter_opts(Rest)].
+
+
diff --git a/src/mango/src/mango_idx.hrl b/src/mango/src/mango_idx.hrl
new file mode 100644
index 000000000..712031b75
--- /dev/null
+++ b/src/mango/src/mango_idx.hrl
@@ -0,0 +1,20 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-record(idx, {
+ dbname,
+ ddoc,
+ name,
+ type,
+ def,
+ opts
+}).
diff --git a/src/mango/src/mango_idx_special.erl b/src/mango/src/mango_idx_special.erl
new file mode 100644
index 000000000..a8f94002b
--- /dev/null
+++ b/src/mango/src/mango_idx_special.erl
@@ -0,0 +1,98 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(mango_idx_special).
+
+
+-export([
+ validate/1,
+ add/2,
+ remove/2,
+ from_ddoc/1,
+ to_json/1,
+ columns/1,
+ is_usable/2,
+ start_key/1,
+ end_key/1
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include("mango_idx.hrl").
+
+
+validate(_) ->
+ erlang:exit(invalid_call).
+
+
+add(_, _) ->
+ erlang:exit(invalid_call).
+
+
+remove(_, _) ->
+ erlang:exit(invalid_call).
+
+
+from_ddoc(_) ->
+ erlang:exit(invalid_call).
+
+
+to_json(#idx{def=all_docs}) ->
+ {[
+ {ddoc, null},
+ {name, <<"_all_docs">>},
+ {type, <<"special">>},
+ {def, {[
+ {<<"fields">>, [{[
+ {<<"_id">>, <<"asc">>}
+ ]}]}
+ ]}}
+ ]}.
+
+
+columns(#idx{def=all_docs}) ->
+ [<<"_id">>].
+
+
+is_usable(#idx{def=all_docs}, Selector) ->
+ Fields = mango_idx_view:indexable_fields(Selector),
+ lists:member(<<"_id">>, Fields).
+
+
+start_key([{'$gt', Key, _, _}]) ->
+ case mango_json:special(Key) of
+ true ->
+ ?MIN_STR;
+ false ->
+ Key
+ end;
+start_key([{'$gte', Key, _, _}]) ->
+ false = mango_json:special(Key),
+ Key;
+start_key([{'$eq', Key, '$eq', Key}]) ->
+ false = mango_json:special(Key),
+ Key.
+
+
+end_key([{_, _, '$lt', Key}]) ->
+ case mango_json:special(Key) of
+ true ->
+ ?MAX_STR;
+ false ->
+ Key
+ end;
+end_key([{_, _, '$lte', Key}]) ->
+ false = mango_json:special(Key),
+ Key;
+end_key([{'$eq', Key, '$eq', Key}]) ->
+ false = mango_json:special(Key),
+ Key.
diff --git a/src/mango/src/mango_idx_text.erl b/src/mango/src/mango_idx_text.erl
new file mode 100644
index 000000000..ad9d2e8d7
--- /dev/null
+++ b/src/mango/src/mango_idx_text.erl
@@ -0,0 +1,422 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(mango_idx_text).
+
+
+-export([
+ validate_new/2,
+ validate_fields/1,
+ validate_index_def/1,
+ add/2,
+ remove/2,
+ from_ddoc/1,
+ to_json/1,
+ columns/1,
+ is_usable/2,
+ get_default_field_options/1
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include("mango.hrl").
+-include("mango_idx.hrl").
+
+
+validate_new(#idx{}=Idx, Db) ->
+ {ok, Def} = do_validate(Idx#idx.def),
+ maybe_reject_index_all_req(Def, Db),
+ {ok, Idx#idx{def=Def}}.
+
+
+validate_index_def(IndexInfo) ->
+ do_validate(IndexInfo).
+
+
+add(#doc{body={Props0}}=DDoc, Idx) ->
+ Texts1 = case proplists:get_value(<<"indexes">>, Props0) of
+ {Texts0} -> Texts0;
+ _ -> []
+ end,
+ NewText = make_text(Idx),
+ Texts2 = lists:keystore(element(1, NewText), 1, Texts1, NewText),
+ Props1 = lists:keystore(<<"indexes">>, 1, Props0, {<<"indexes">>,
+ {Texts2}}),
+ {ok, DDoc#doc{body={Props1}}}.
+
+
+remove(#doc{body={Props0}}=DDoc, Idx) ->
+ Texts1 = case proplists:get_value(<<"indexes">>, Props0) of
+ {Texts0} ->
+ Texts0;
+ _ ->
+ ?MANGO_ERROR({index_not_found, Idx#idx.name})
+ end,
+ Texts2 = lists:keydelete(Idx#idx.name, 1, Texts1),
+ if Texts2 /= Texts1 -> ok; true ->
+ ?MANGO_ERROR({index_not_found, Idx#idx.name})
+ end,
+ Props1 = case Texts2 of
+ [] ->
+ lists:keydelete(<<"indexes">>, 1, Props0);
+ _ ->
+ lists:keystore(<<"indexes">>, 1, Props0, {<<"indexes">>, {Texts2}})
+ end,
+ {ok, DDoc#doc{body={Props1}}}.
+
+
+from_ddoc({Props}) ->
+ case lists:keyfind(<<"indexes">>, 1, Props) of
+ {<<"indexes">>, {Texts}} when is_list(Texts) ->
+ lists:flatmap(fun({Name, {VProps}}) ->
+ case validate_ddoc(VProps) of
+ invalid_ddoc ->
+ [];
+ Def ->
+ I = #idx{
+ type = <<"text">>,
+ name = Name,
+ def = Def
+ },
+ [I]
+ end
+ end, Texts);
+ _ ->
+ []
+ end.
+
+
+to_json(Idx) ->
+ {[
+ {ddoc, Idx#idx.ddoc},
+ {name, Idx#idx.name},
+ {type, Idx#idx.type},
+ {def, {def_to_json(Idx#idx.def)}}
+ ]}.
+
+
+columns(Idx) ->
+ {Props} = Idx#idx.def,
+ {<<"fields">>, Fields} = lists:keyfind(<<"fields">>, 1, Props),
+ case Fields of
+ <<"all_fields">> ->
+ all_fields;
+ _ ->
+ {DFProps} = couch_util:get_value(<<"default_field">>, Props, {[]}),
+ Enabled = couch_util:get_value(<<"enabled">>, DFProps, true),
+ Default = case Enabled of
+ true -> [<<"$default">>];
+ false -> []
+ end,
+ Default ++ lists:map(fun({FProps}) ->
+ {_, Name} = lists:keyfind(<<"name">>, 1, FProps),
+ {_, Type} = lists:keyfind(<<"type">>, 1, FProps),
+ iolist_to_binary([Name, ":", Type])
+ end, Fields)
+ end.
+
+
+is_usable(Idx, Selector) ->
+ case columns(Idx) of
+ all_fields ->
+ true;
+ Cols ->
+ Fields = indexable_fields(Selector),
+ sets:is_subset(sets:from_list(Fields), sets:from_list(Cols))
+ end.
+
+
+do_validate({Props}) ->
+ {ok, Opts} = mango_opts:validate(Props, opts()),
+ {ok, {Opts}};
+do_validate(Else) ->
+ ?MANGO_ERROR({invalid_index_text, Else}).
+
+
+def_to_json({Props}) ->
+ def_to_json(Props);
+def_to_json([]) ->
+ [];
+def_to_json([{<<"fields">>, <<"all_fields">>} | Rest]) ->
+ [{<<"fields">>, []} | def_to_json(Rest)];
+def_to_json([{fields, Fields} | Rest]) ->
+ [{<<"fields">>, fields_to_json(Fields)} | def_to_json(Rest)];
+def_to_json([{<<"fields">>, Fields} | Rest]) ->
+ [{<<"fields">>, fields_to_json(Fields)} | def_to_json(Rest)];
+def_to_json([{Key, Value} | Rest]) ->
+ [{Key, Value} | def_to_json(Rest)].
+
+
+fields_to_json([]) ->
+ [];
+fields_to_json([{[{<<"name">>, Name}, {<<"type">>, Type0}]} | Rest]) ->
+ ok = validate_field_name(Name),
+ Type = validate_field_type(Type0),
+ [{[{Name, Type}]} | fields_to_json(Rest)];
+fields_to_json([{[{<<"type">>, Type0}, {<<"name">>, Name}]} | Rest]) ->
+ ok = validate_field_name(Name),
+ Type = validate_field_type(Type0),
+ [{[{Name, Type}]} | fields_to_json(Rest)].
+
+
+%% In the future, we can possibly add more restrictive validation.
+%% For now, let's make sure the field name is not blank.
+validate_field_name(<<"">>) ->
+ throw(invalid_field_name);
+validate_field_name(Else) when is_binary(Else)->
+ ok;
+validate_field_name(_) ->
+ throw(invalid_field_name).
+
+
+validate_field_type(<<"string">>) ->
+ <<"string">>;
+validate_field_type(<<"number">>) ->
+ <<"number">>;
+validate_field_type(<<"boolean">>) ->
+ <<"boolean">>.
+
+
+validate_fields(<<"all_fields">>) ->
+ {ok, all_fields};
+validate_fields(Fields) ->
+ try fields_to_json(Fields) of
+ _ ->
+ mango_fields:new(Fields)
+ catch error:function_clause ->
+ ?MANGO_ERROR({invalid_index_fields_definition, Fields});
+ throw:invalid_field_name ->
+ ?MANGO_ERROR({invalid_index_fields_definition, Fields})
+ end.
+
+
+validate_ddoc(VProps) ->
+ try
+ Def = proplists:get_value(<<"index">>, VProps),
+ validate_index_def(Def),
+ Def
+ catch Error:Reason ->
+ couch_log:error("Invalid Index Def ~p: Error. ~p, Reason: ~p",
+ [VProps, Error, Reason]),
+ invalid_ddoc
+ end.
+
+
+opts() ->
+ [
+ {<<"default_analyzer">>, [
+ {tag, default_analyzer},
+ {optional, true},
+ {default, <<"keyword">>}
+ ]},
+ {<<"default_field">>, [
+ {tag, default_field},
+ {optional, true},
+ {default, {[]}}
+ ]},
+ {<<"selector">>, [
+ {tag, selector},
+ {optional, true},
+ {default, {[]}},
+ {validator, fun mango_opts:validate_selector/1}
+ ]},
+ {<<"fields">>, [
+ {tag, fields},
+ {optional, true},
+ {default, []},
+ {validator, fun ?MODULE:validate_fields/1}
+ ]},
+ {<<"index_array_lengths">>, [
+ {tag, index_array_lengths},
+ {optional, true},
+ {default, true},
+ {validator, fun mango_opts:is_boolean/1}
+ ]}
+ ].
+
+
+make_text(Idx) ->
+ Text= {[
+ {<<"index">>, Idx#idx.def},
+ {<<"analyzer">>, construct_analyzer(Idx#idx.def)}
+ ]},
+ {Idx#idx.name, Text}.
+
+
+get_default_field_options(Props) ->
+ Default = couch_util:get_value(default_field, Props, {[]}),
+ case Default of
+ Bool when is_boolean(Bool) ->
+ {Bool, <<"standard">>};
+ {[]} ->
+ {true, <<"standard">>};
+ {Opts}->
+ Enabled = couch_util:get_value(<<"enabled">>, Opts, true),
+ Analyzer = couch_util:get_value(<<"analyzer">>, Opts,
+ <<"standard">>),
+ {Enabled, Analyzer}
+ end.
+
+
+construct_analyzer({Props}) ->
+ DefaultAnalyzer = couch_util:get_value(default_analyzer, Props,
+ <<"keyword">>),
+ {DefaultField, DefaultFieldAnalyzer} = get_default_field_options(Props),
+ DefaultAnalyzerDef = case DefaultField of
+ true ->
+ [{<<"$default">>, DefaultFieldAnalyzer}];
+ _ ->
+ []
+ end,
+ case DefaultAnalyzerDef of
+ [] ->
+ <<"keyword">>;
+ _ ->
+ {[
+ {<<"name">>, <<"perfield">>},
+ {<<"default">>, DefaultAnalyzer},
+ {<<"fields">>, {DefaultAnalyzerDef}}
+ ]}
+ end.
+
+
+indexable_fields(Selector) ->
+ TupleTree = mango_selector_text:convert([], Selector),
+ indexable_fields([], TupleTree).
+
+
+indexable_fields(Fields, {op_and, Args}) when is_list(Args) ->
+ lists:foldl(fun(Arg, Fields0) -> indexable_fields(Fields0, Arg) end,
+ Fields, Args);
+
+%% For queries that use array element access or $in operations, two
+%% fields get generated by mango_selector_text:convert. At index
+%% definition time, only one field gets defined. In this situation, we
+%% remove the extra generated field so that the index can be used. For
+%% all other situations, we include the fields as normal.
+indexable_fields(Fields, {op_or, [{op_field, Field0},
+ {op_field, {[Name | _], _}} = Field1]}) ->
+ case lists:member(<<"[]">>, Name) of
+ true ->
+ indexable_fields(Fields, Field1);
+ false ->
+ Fields1 = indexable_fields(Fields, {op_field, Field0}),
+ indexable_fields(Fields1, Field1)
+ end;
+indexable_fields(Fields, {op_or, Args}) when is_list(Args) ->
+ lists:foldl(fun(Arg, Fields0) -> indexable_fields(Fields0, Arg) end,
+ Fields, Args);
+
+indexable_fields(Fields, {op_not, {ExistsQuery, Arg}}) when is_tuple(Arg) ->
+ Fields0 = indexable_fields(Fields, ExistsQuery),
+ indexable_fields(Fields0, Arg);
+
+indexable_fields(Fields, {op_insert, Arg}) when is_binary(Arg) ->
+ Fields;
+
+%% fieldname.[]:length is not a user defined field.
+indexable_fields(Fields, {op_field, {[_, <<":length">>], _}}) ->
+ Fields;
+indexable_fields(Fields, {op_field, {Name, _}}) ->
+ [iolist_to_binary(Name) | Fields];
+
+%% In this particular case, the lucene index is doing a field_exists query
+%% so it is looking at all sorts of combinations of field:* and field.*
+%% We don't add the field because we cannot pre-determine what field will exist.
+%% Hence we just return Fields and make it less restrictive.
+indexable_fields(Fields, {op_fieldname, {_, _}}) ->
+ Fields;
+
+%% Similar idea to op_fieldname but with fieldname:null
+indexable_fields(Fields, {op_null, {_, _}}) ->
+ Fields;
+
+indexable_fields(Fields, {op_default, _}) ->
+ [<<"$default">> | Fields].
+
+
+maybe_reject_index_all_req({Def}, #db{name=DbName, user_ctx=Ctx}) ->
+ User = Ctx#user_ctx.name,
+ Fields = couch_util:get_value(fields, Def),
+ case {Fields, forbid_index_all()} of
+ {all_fields, "true"} ->
+ ?MANGO_ERROR(index_all_disabled);
+ {all_fields, "warn"} ->
+ couch_log:warning("User ~p is indexing all fields in db ~p",
+ [User, DbName]);
+ _ ->
+ ok
+ end.
+
+
+forbid_index_all() ->
+ config:get("mango", "index_all_disabled", "false").
+
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+
+setup() ->
+ test_util:start_couch(),
+ meck:expect(couch_log, warning, 2,
+ fun(_,_) ->
+ throw({test_error, logged_warning})
+ end),
+ %default index all def that generates {fields, all_fields}
+ Index = #idx{def={[]}},
+ Db = #db{name = <<"testdb">>, user_ctx=#user_ctx{name = <<"u1">>}},
+ {Index, Db}.
+
+
+teardown(_) ->
+ ok = config:delete("mango", "index_all_disabled"),
+ test_util:stop_couch().
+
+
+index_all_test_() ->
+ {
+ foreach,
+ fun setup/0,
+ fun teardown/1,
+ [
+ fun forbid_index_all/1,
+ fun default_and_false_index_all/1,
+ fun warn_index_all/1
+ ]
+
+ }.
+
+
+forbid_index_all({Idx, Db}) ->
+ ok = config:set("mango", "index_all_disabled", "true"),
+ ?_assertThrow({mango_error, ?MODULE, index_all_disabled},
+ validate_new(Idx, Db)
+ ).
+
+
+default_and_false_index_all({Idx, Db}) ->
+ {ok, #idx{def={Def}}} = validate_new(Idx, Db),
+ Fields = couch_util:get_value(fields, Def),
+ ?_assertEqual(all_fields, Fields),
+ ok = config:set("mango", "index_all_disabled", "false"),
+ {ok, #idx{def={Def2}}} = validate_new(Idx, Db),
+ Fields2 = couch_util:get_value(fields, Def2),
+ ?_assertEqual(all_fields, Fields2).
+
+
+warn_index_all({Idx, Db}) ->
+ ok = config:set("mango", "index_all_disabled", "warn"),
+ ?_assertThrow({test_error, logged_warning}, validate_new(Idx, Db)).
+
+
+-endif.
diff --git a/src/mango/src/mango_idx_view.erl b/src/mango/src/mango_idx_view.erl
new file mode 100644
index 000000000..8bad34cca
--- /dev/null
+++ b/src/mango/src/mango_idx_view.erl
@@ -0,0 +1,490 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(mango_idx_view).
+
+
+-export([
+ validate_new/2,
+ validate_index_def/1,
+ add/2,
+ remove/2,
+ from_ddoc/1,
+ to_json/1,
+ is_usable/2,
+ columns/1,
+ start_key/1,
+ end_key/1,
+
+ indexable_fields/1,
+ field_ranges/1,
+ field_ranges/2
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include("mango.hrl").
+-include("mango_idx.hrl").
+
+
+validate_new(#idx{}=Idx, _Db) ->
+ {ok, Def} = do_validate(Idx#idx.def),
+ {ok, Idx#idx{def=Def}}.
+
+
+validate_index_def(Def) ->
+ def_to_json(Def).
+
+
+add(#doc{body={Props0}}=DDoc, Idx) ->
+ Views1 = case proplists:get_value(<<"views">>, Props0) of
+ {Views0} -> Views0;
+ _ -> []
+ end,
+ NewView = make_view(Idx),
+ Views2 = lists:keystore(element(1, NewView), 1, Views1, NewView),
+ Props1 = lists:keystore(<<"views">>, 1, Props0, {<<"views">>, {Views2}}),
+ {ok, DDoc#doc{body={Props1}}}.
+
+
+remove(#doc{body={Props0}}=DDoc, Idx) ->
+ Views1 = case proplists:get_value(<<"views">>, Props0) of
+ {Views0} ->
+ Views0;
+ _ ->
+ ?MANGO_ERROR({index_not_found, Idx#idx.name})
+ end,
+ Views2 = lists:keydelete(Idx#idx.name, 1, Views1),
+ if Views2 /= Views1 -> ok; true ->
+ ?MANGO_ERROR({index_not_found, Idx#idx.name})
+ end,
+ Props1 = case Views2 of
+ [] ->
+ lists:keydelete(<<"views">>, 1, Props0);
+ _ ->
+ lists:keystore(<<"views">>, 1, Props0, {<<"views">>, {Views2}})
+ end,
+ {ok, DDoc#doc{body={Props1}}}.
+
+
+from_ddoc({Props}) ->
+ case lists:keyfind(<<"views">>, 1, Props) of
+ {<<"views">>, {Views}} when is_list(Views) ->
+ lists:flatmap(fun({Name, {VProps}}) ->
+ case validate_ddoc(VProps) of
+ invalid_view ->
+ [];
+ {Def, Opts} ->
+ I = #idx{
+ type = <<"json">>,
+ name = Name,
+ def = Def,
+ opts = Opts
+ },
+ [I]
+ end
+ end, Views);
+ _ ->
+ []
+ end.
+
+
+to_json(Idx) ->
+ {[
+ {ddoc, Idx#idx.ddoc},
+ {name, Idx#idx.name},
+ {type, Idx#idx.type},
+ {def, {def_to_json(Idx#idx.def)}}
+ ]}.
+
+
+columns(Idx) ->
+ {Props} = Idx#idx.def,
+ {<<"fields">>, {Fields}} = lists:keyfind(<<"fields">>, 1, Props),
+ [Key || {Key, _} <- Fields].
+
+
+is_usable(Idx, Selector) ->
+ % This index is usable if at least the first column is
+ % a member of the indexable fields of the selector.
+ Columns = columns(Idx),
+ Fields = indexable_fields(Selector),
+ lists:member(hd(Columns), Fields) andalso not is_text_search(Selector).
+
+
+is_text_search({[]}) ->
+ false;
+is_text_search({[{<<"$default">>, _}]}) ->
+ true;
+is_text_search({[{_Field, Cond}]}) when is_list(Cond) ->
+ lists:foldl(fun(C, Exists) ->
+ Exists orelse is_text_search(C)
+ end, false, Cond);
+is_text_search({[{_Field, Cond}]}) when is_tuple(Cond) ->
+ is_text_search(Cond);
+is_text_search({[{_Field, _Cond}]}) ->
+ false;
+%% we reached values, which should always be false
+is_text_search(Val)
+ when is_number(Val); is_boolean(Val); is_binary(Val)->
+ false.
+
+
+start_key([]) ->
+ [];
+start_key([{'$gt', Key, _, _} | Rest]) ->
+ case mango_json:special(Key) of
+ true ->
+ [];
+ false ->
+ [Key | start_key(Rest)]
+ end;
+start_key([{'$gte', Key, _, _} | Rest]) ->
+ false = mango_json:special(Key),
+ [Key | start_key(Rest)];
+start_key([{'$eq', Key, '$eq', Key} | Rest]) ->
+ false = mango_json:special(Key),
+ [Key | start_key(Rest)].
+
+
+end_key([]) ->
+ [{[]}];
+end_key([{_, _, '$lt', Key} | Rest]) ->
+ case mango_json:special(Key) of
+ true ->
+ [{[]}];
+ false ->
+ [Key | end_key(Rest)]
+ end;
+end_key([{_, _, '$lte', Key} | Rest]) ->
+ false = mango_json:special(Key),
+ [Key | end_key(Rest)];
+end_key([{'$eq', Key, '$eq', Key} | Rest]) ->
+ false = mango_json:special(Key),
+ [Key | end_key(Rest)].
+
+
+do_validate({Props}) ->
+ {ok, Opts} = mango_opts:validate(Props, opts()),
+ {ok, {Opts}};
+do_validate(Else) ->
+ ?MANGO_ERROR({invalid_index_json, Else}).
+
+
+def_to_json({Props}) ->
+ def_to_json(Props);
+def_to_json([]) ->
+ [];
+def_to_json([{fields, Fields} | Rest]) ->
+ [{<<"fields">>, mango_sort:to_json(Fields)} | def_to_json(Rest)];
+def_to_json([{<<"fields">>, Fields} | Rest]) ->
+ [{<<"fields">>, mango_sort:to_json(Fields)} | def_to_json(Rest)];
+def_to_json([{Key, Value} | Rest]) ->
+ [{Key, Value} | def_to_json(Rest)].
+
+
+opts() ->
+ [
+ {<<"fields">>, [
+ {tag, fields},
+ {validator, fun mango_opts:validate_sort/1}
+ ]}
+ ].
+
+
+make_view(Idx) ->
+ View = {[
+ {<<"map">>, Idx#idx.def},
+ {<<"reduce">>, <<"_count">>},
+ {<<"options">>, {Idx#idx.opts}}
+ ]},
+ {Idx#idx.name, View}.
+
+
+validate_ddoc(VProps) ->
+ try
+ Def = proplists:get_value(<<"map">>, VProps),
+ validate_index_def(Def),
+ {Opts0} = proplists:get_value(<<"options">>, VProps),
+ Opts = lists:keydelete(<<"sort">>, 1, Opts0),
+ {Def, Opts}
+ catch Error:Reason ->
+ couch_log:error("Invalid Index Def ~p. Error: ~p, Reason: ~p",
+ [VProps, Error, Reason]),
+ invalid_view
+ end.
+
+
+% This function returns a list of indexes that
+% can be used to restrict this query. This works by
+% searching the selector looking for field names that
+% can be "seen".
+%
+% Operators that can be seen through are '$and' and any of
+% the logical comparisons ('$lt', '$eq', etc). Things like
+% '$regex', '$in', '$nin', and '$or' can't be serviced by
+% a single index scan so we disallow them. In the future
+% we may become more clever and increase our ken such that
+% we will be able to see through these with crafty indexes
+% or new uses for existing indexes. For instance, I could
+% see an '$or' between comparisons on the same field becoming
+% the equivalent of a multi-query. But that's for another
+% day.
+
+% We can see through '$and' trivially
+indexable_fields({[{<<"$and">>, Args}]}) ->
+ lists:usort(lists:flatten([indexable_fields(A) || A <- Args]));
+
+% So far we can't see through any other operator
+indexable_fields({[{<<"$", _/binary>>, _}]}) ->
+ [];
+
+% If we have a field with a terminator that is locatable
+% using an index then the field is a possible index
+indexable_fields({[{Field, Cond}]}) ->
+ case indexable(Cond) of
+ true ->
+ [Field];
+ false ->
+ []
+ end;
+
+% An empty selector
+indexable_fields({[]}) ->
+ [].
+
+
+% Check if a condition is indexable. The logical
+% comparisons are mostly straight forward. We
+% currently don't understand '$in' which is
+% theoretically supportable. '$nin' and '$ne'
+% aren't currently supported because they require
+% multiple index scans.
+indexable({[{<<"$lt">>, _}]}) ->
+ true;
+indexable({[{<<"$lte">>, _}]}) ->
+ true;
+indexable({[{<<"$eq">>, _}]}) ->
+ true;
+indexable({[{<<"$gt">>, _}]}) ->
+ true;
+indexable({[{<<"$gte">>, _}]}) ->
+ true;
+
+% All other operators are currently not indexable.
+% This is also a subtle assertion that we don't
+% call indexable/1 on a field name.
+indexable({[{<<"$", _/binary>>, _}]}) ->
+ false.
+
+
+% For each field, return {Field, Range}
+field_ranges(Selector) ->
+ Fields = indexable_fields(Selector),
+ field_ranges(Selector, Fields).
+
+
+field_ranges(Selector, Fields) ->
+ field_ranges(Selector, Fields, []).
+
+
+field_ranges(_Selector, [], Acc) ->
+ lists:reverse(Acc);
+field_ranges(Selector, [Field | Rest], Acc) ->
+ case range(Selector, Field) of
+ empty ->
+ [{Field, empty}];
+ Range ->
+ field_ranges(Selector, Rest, [{Field, Range} | Acc])
+ end.
+
+
+% Find the complete range for a given index in this
+% selector. This works by AND'ing logical comparisons
+% together so that we can define the start and end
+% keys for a given index.
+%
+% Selector must have been normalized before calling
+% this function.
+range(Selector, Index) ->
+ range(Selector, Index, '$gt', mango_json:min(), '$lt', mango_json:max()).
+
+
+% Adjust Low and High based on values found for the
+% givend Index in Selector.
+range({[{<<"$and">>, Args}]}, Index, LCmp, Low, HCmp, High) ->
+ lists:foldl(fun
+ (Arg, {LC, L, HC, H}) ->
+ range(Arg, Index, LC, L, HC, H);
+ (_Arg, empty) ->
+ empty
+ end, {LCmp, Low, HCmp, High}, Args);
+
+% We can currently only traverse '$and' operators
+range({[{<<"$", _/binary>>}]}, _Index, LCmp, Low, HCmp, High) ->
+ {LCmp, Low, HCmp, High};
+
+% If the field name matches the index see if we can narrow
+% the acceptable range.
+range({[{Index, Cond}]}, Index, LCmp, Low, HCmp, High) ->
+ range(Cond, LCmp, Low, HCmp, High);
+
+% Else we have a field unrelated to this index so just
+% return the current values.
+range(_, _, LCmp, Low, HCmp, High) ->
+ {LCmp, Low, HCmp, High}.
+
+
+% The comments below are a bit cryptic at first but they show
+% where the Arg cand land in the current range.
+%
+% For instance, given:
+%
+% {$lt: N}
+% Low = 1
+% High = 5
+%
+% Depending on the value of N we can have one of five locations
+% in regards to a given Low/High pair:
+%
+% min low mid high max
+%
+% That is:
+% min = (N < Low)
+% low = (N == Low)
+% mid = (Low < N < High)
+% high = (N == High)
+% max = (High < N)
+%
+% If N < 1, (min) then the effective range is empty.
+%
+% If N == 1, (low) then we have to set the range to empty because
+% N < 1 && N >= 1 is an empty set. If the operator had been '$lte'
+% and LCmp was '$gte' or '$eq' then we could keep around the equality
+% check on Arg by setting LCmp == HCmp = '$eq' and Low == High == Arg.
+%
+% If 1 < N < 5 (mid), then we set High to Arg and Arg has just
+% narrowed our range. HCmp is set the the '$lt' operator that was
+% part of the input.
+%
+% If N == 5 (high), We just set HCmp to '$lt' since its guaranteed
+% to be equally or more restrictive than the current possible values
+% of '$lt' or '$lte'.
+%
+% If N > 5 (max), nothing changes as our current range is already
+% more narrow than the current condition.
+%
+% Obviously all of that logic gets tweaked for the other logical
+% operators but its all straight forward once you figure out how
+% we're basically just narrowing our logical ranges.
+
+range({[{<<"$lt">>, Arg}]}, LCmp, Low, HCmp, High) ->
+ case range_pos(Low, Arg, High) of
+ min ->
+ empty;
+ low ->
+ empty;
+ mid ->
+ {LCmp, Low, '$lt', Arg};
+ high ->
+ {LCmp, Low, '$lt', Arg};
+ max ->
+ {LCmp, Low, HCmp, High}
+ end;
+
+range({[{<<"$lte">>, Arg}]}, LCmp, Low, HCmp, High) ->
+ case range_pos(Low, Arg, High) of
+ min ->
+ empty;
+ low when LCmp == '$gte'; LCmp == '$eq' ->
+ {'$eq', Arg, '$eq', Arg};
+ low ->
+ empty;
+ mid ->
+ {LCmp, Low, '$lte', Arg};
+ high ->
+ {LCmp, Low, HCmp, High};
+ max ->
+ {LCmp, Low, HCmp, High}
+ end;
+
+range({[{<<"$eq">>, Arg}]}, LCmp, Low, HCmp, High) ->
+ case range_pos(Low, Arg, High) of
+ min ->
+ empty;
+ low when LCmp == '$gte'; LCmp == '$eq' ->
+ {'$eq', Arg, '$eq', Arg};
+ low ->
+ empty;
+ mid ->
+ {'$eq', Arg, '$eq', Arg};
+ high when HCmp == '$lte'; HCmp == '$eq' ->
+ {'$eq', Arg, '$eq', Arg};
+ high ->
+ empty;
+ max ->
+ empty
+ end;
+
+range({[{<<"$gte">>, Arg}]}, LCmp, Low, HCmp, High) ->
+ case range_pos(Low, Arg, High) of
+ min ->
+ {LCmp, Low, HCmp, High};
+ low ->
+ {LCmp, Low, HCmp, High};
+ mid ->
+ {'$gte', Arg, HCmp, High};
+ high when HCmp == '$lte'; HCmp == '$eq' ->
+ {'$eq', Arg, '$eq', Arg};
+ high ->
+ empty;
+ max ->
+ empty
+ end;
+
+range({[{<<"$gt">>, Arg}]}, LCmp, Low, HCmp, High) ->
+ case range_pos(Low, Arg, High) of
+ min ->
+ {LCmp, Low, HCmp, High};
+ low ->
+ {'$gt', Arg, HCmp, High};
+ mid ->
+ {'$gt', Arg, HCmp, High};
+ high ->
+ empty;
+ max ->
+ empty
+ end;
+
+% There's some other un-indexable restriction on the index
+% that will be applied as a post-filter. Ignore it and
+% carry on our merry way.
+range({[{<<"$", _/binary>>, _}]}, LCmp, Low, HCmp, High) ->
+ {LCmp, Low, HCmp, High}.
+
+
+% Returns the value min | low | mid | high | max depending
+% on how Arg compares to Low and High.
+range_pos(Low, Arg, High) ->
+ case mango_json:cmp(Arg, Low) of
+ N when N < 0 -> min;
+ N when N == 0 -> low;
+ _ ->
+ case mango_json:cmp(Arg, High) of
+ X when X < 0 ->
+ mid;
+ X when X == 0 ->
+ high;
+ _ ->
+ max
+ end
+ end.
diff --git a/src/mango/src/mango_json.erl b/src/mango/src/mango_json.erl
new file mode 100644
index 000000000..9584c2d7e
--- /dev/null
+++ b/src/mango/src/mango_json.erl
@@ -0,0 +1,121 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(mango_json).
+
+
+-export([
+ min/0,
+ max/0,
+ cmp/2,
+ cmp_raw/2,
+ type/1,
+ special/1,
+ to_binary/1
+]).
+
+
+-define(MIN_VAL, mango_json_min).
+-define(MAX_VAL, mango_json_max).
+
+
+min() ->
+ ?MIN_VAL.
+
+
+max() ->
+ ?MAX_VAL.
+
+
+cmp(?MIN_VAL, ?MIN_VAL) ->
+ 0;
+cmp(?MIN_VAL, _) ->
+ -1;
+cmp(_, ?MIN_VAL) ->
+ 1;
+cmp(?MAX_VAL, ?MAX_VAL) ->
+ 0;
+cmp(?MAX_VAL, _) ->
+ 1;
+cmp(_, ?MAX_VAL) ->
+ -1;
+cmp(A, B) ->
+ couch_ejson_compare:less(A, B).
+
+
+cmp_raw(?MIN_VAL, ?MIN_VAL) ->
+ 0;
+cmp_raw(?MIN_VAL, _) ->
+ -1;
+cmp_raw(_, ?MIN_VAL) ->
+ 1;
+cmp_raw(?MAX_VAL, ?MAX_VAL) ->
+ 0;
+cmp_raw(?MAX_VAL, _) ->
+ 1;
+cmp_raw(_, ?MAX_VAL) ->
+ -1;
+cmp_raw(A, B) ->
+ case A < B of
+ true ->
+ -1;
+ false ->
+ case A > B of
+ true ->
+ 1;
+ false ->
+ 0
+ end
+ end.
+
+
+type(null) ->
+ <<"null">>;
+type(Bool) when is_boolean(Bool) ->
+ <<"boolean">>;
+type(Num) when is_number(Num) ->
+ <<"number">>;
+type(Str) when is_binary(Str) ->
+ <<"string">>;
+type({Props}) when is_list(Props) ->
+ <<"object">>;
+type(Vals) when is_list(Vals) ->
+ <<"array">>.
+
+
+special(?MIN_VAL) ->
+ true;
+special(?MAX_VAL) ->
+ true;
+special(_) ->
+ false.
+
+
+to_binary({Props}) ->
+ Pred = fun({Key, Value}) ->
+ {to_binary(Key), to_binary(Value)}
+ end,
+ {lists:map(Pred, Props)};
+to_binary(Data) when is_list(Data) ->
+ [to_binary(D) || D <- Data];
+to_binary(null) ->
+ null;
+to_binary(true) ->
+ true;
+to_binary(false) ->
+ false;
+to_binary(Data) when is_atom(Data) ->
+ list_to_binary(atom_to_list(Data));
+to_binary(Data) when is_number(Data) ->
+ Data;
+to_binary(Data) when is_binary(Data) ->
+ Data. \ No newline at end of file
diff --git a/src/mango/src/mango_native_proc.erl b/src/mango/src/mango_native_proc.erl
new file mode 100644
index 000000000..6d0fb2400
--- /dev/null
+++ b/src/mango/src/mango_native_proc.erl
@@ -0,0 +1,347 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(mango_native_proc).
+-behavior(gen_server).
+
+
+-include("mango_idx.hrl").
+
+
+-export([
+ start_link/0,
+ set_timeout/2,
+ prompt/2
+]).
+
+-export([
+ init/1,
+ terminate/2,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ code_change/3
+]).
+
+
+-record(st, {
+ indexes = [],
+ timeout = 5000
+}).
+
+
+-record(tacc, {
+ index_array_lengths = true,
+ fields = all_fields,
+ path = []
+}).
+
+
+start_link() ->
+ gen_server:start_link(?MODULE, [], []).
+
+
+set_timeout(Pid, TimeOut) when is_integer(TimeOut), TimeOut > 0 ->
+ gen_server:call(Pid, {set_timeout, TimeOut}).
+
+
+prompt(Pid, Data) ->
+ gen_server:call(Pid, {prompt, Data}).
+
+
+init(_) ->
+ {ok, #st{}}.
+
+
+terminate(_Reason, _St) ->
+ ok.
+
+
+handle_call({set_timeout, TimeOut}, _From, St) ->
+ {reply, ok, St#st{timeout=TimeOut}};
+
+handle_call({prompt, [<<"reset">>]}, _From, St) ->
+ {reply, true, St#st{indexes=[]}};
+
+handle_call({prompt, [<<"reset">>, _QueryConfig]}, _From, St) ->
+ {reply, true, St#st{indexes=[]}};
+
+handle_call({prompt, [<<"add_fun">>, IndexInfo]}, _From, St) ->
+ Indexes = case validate_index_info(IndexInfo) of
+ true ->
+ St#st.indexes ++ [IndexInfo];
+ false ->
+ couch_log:error("No Valid Indexes For: ~p", [IndexInfo]),
+ St#st.indexes
+ end,
+ NewSt = St#st{indexes = Indexes},
+ {reply, true, NewSt};
+
+handle_call({prompt, [<<"map_doc">>, Doc]}, _From, St) ->
+ {reply, map_doc(St, mango_json:to_binary(Doc)), St};
+
+handle_call({prompt, [<<"reduce">>, _, _]}, _From, St) ->
+ {reply, null, St};
+
+handle_call({prompt, [<<"rereduce">>, _, _]}, _From, St) ->
+ {reply, null, St};
+
+handle_call({prompt, [<<"index_doc">>, Doc]}, _From, St) ->
+ Vals = case index_doc(St, mango_json:to_binary(Doc)) of
+ [] ->
+ [[]];
+ Else ->
+ Else
+ end,
+ {reply, Vals, St};
+
+
+handle_call(Msg, _From, St) ->
+ {stop, {invalid_call, Msg}, {invalid_call, Msg}, St}.
+
+
+handle_cast(garbage_collect, St) ->
+ erlang:garbage_collect(),
+ {noreply, St};
+
+handle_cast(Msg, St) ->
+ {stop, {invalid_cast, Msg}, St}.
+
+
+handle_info(Msg, St) ->
+ {stop, {invalid_info, Msg}, St}.
+
+
+code_change(_OldVsn, St, _Extra) ->
+ {ok, St}.
+
+
+map_doc(#st{indexes=Indexes}, Doc) ->
+ lists:map(fun(Idx) -> get_index_entries(Idx, Doc) end, Indexes).
+
+
+index_doc(#st{indexes=Indexes}, Doc) ->
+ lists:map(fun(Idx) -> get_text_entries(Idx, Doc) end, Indexes).
+
+
+get_index_entries({IdxProps}, Doc) ->
+ {Fields} = couch_util:get_value(<<"fields">>, IdxProps),
+ Values = lists:map(fun({Field, _Dir}) ->
+ case mango_doc:get_field(Doc, Field) of
+ not_found -> not_found;
+ bad_path -> not_found;
+ Else -> Else
+ end
+ end, Fields),
+ case lists:member(not_found, Values) of
+ true ->
+ [];
+ false ->
+ [[Values, null]]
+ end.
+
+
+get_text_entries({IdxProps}, Doc) ->
+ Selector = case couch_util:get_value(<<"selector">>, IdxProps) of
+ [] -> {[]};
+ Else -> Else
+ end,
+ case should_index(Selector, Doc) of
+ true ->
+ get_text_entries0(IdxProps, Doc);
+ false ->
+ []
+ end.
+
+
+get_text_entries0(IdxProps, Doc) ->
+ DefaultEnabled = get_default_enabled(IdxProps),
+ IndexArrayLengths = get_index_array_lengths(IdxProps),
+ FieldsList = get_text_field_list(IdxProps),
+ TAcc = #tacc{
+ index_array_lengths = IndexArrayLengths,
+ fields = FieldsList
+ },
+ Fields0 = get_text_field_values(Doc, TAcc),
+ Fields = if not DefaultEnabled -> Fields0; true ->
+ add_default_text_field(Fields0)
+ end,
+ FieldNames = get_field_names(Fields, []),
+ Converted = convert_text_fields(Fields),
+ FieldNames ++ Converted.
+
+
+get_text_field_values({Props}, TAcc) when is_list(Props) ->
+ get_text_field_values_obj(Props, TAcc, []);
+
+get_text_field_values(Values, TAcc) when is_list(Values) ->
+ IndexArrayLengths = TAcc#tacc.index_array_lengths,
+ NewPath = ["[]" | TAcc#tacc.path],
+ NewTAcc = TAcc#tacc{path = NewPath},
+ case IndexArrayLengths of
+ true ->
+ % We bypass make_text_field and directly call make_text_field_name
+ % because the length field name is not part of the path.
+ LengthFieldName = make_text_field_name(NewTAcc#tacc.path, <<"length">>),
+ LengthField = [{LengthFieldName, <<"length">>, length(Values)}],
+ get_text_field_values_arr(Values, NewTAcc, LengthField);
+ _ ->
+ get_text_field_values_arr(Values, NewTAcc, [])
+ end;
+
+get_text_field_values(Bin, TAcc) when is_binary(Bin) ->
+ make_text_field(TAcc, <<"string">>, Bin);
+
+get_text_field_values(Num, TAcc) when is_number(Num) ->
+ make_text_field(TAcc, <<"number">>, Num);
+
+get_text_field_values(Bool, TAcc) when is_boolean(Bool) ->
+ make_text_field(TAcc, <<"boolean">>, Bool);
+
+get_text_field_values(null, TAcc) ->
+ make_text_field(TAcc, <<"null">>, true).
+
+
+get_text_field_values_obj([], _, FAcc) ->
+ FAcc;
+get_text_field_values_obj([{Key, Val} | Rest], TAcc, FAcc) ->
+ NewPath = [Key | TAcc#tacc.path],
+ NewTAcc = TAcc#tacc{path = NewPath},
+ Fields = get_text_field_values(Val, NewTAcc),
+ get_text_field_values_obj(Rest, TAcc, Fields ++ FAcc).
+
+
+get_text_field_values_arr([], _, FAcc) ->
+ FAcc;
+get_text_field_values_arr([Value | Rest], TAcc, FAcc) ->
+ Fields = get_text_field_values(Value, TAcc),
+ get_text_field_values_arr(Rest, TAcc, Fields ++ FAcc).
+
+
+get_default_enabled(Props) ->
+ case couch_util:get_value(<<"default_field">>, Props, {[]}) of
+ Bool when is_boolean(Bool) ->
+ Bool;
+ {[]} ->
+ true;
+ {Opts}->
+ couch_util:get_value(<<"enabled">>, Opts, true)
+ end.
+
+
+get_index_array_lengths(Props) ->
+ couch_util:get_value(<<"index_array_lengths">>, Props, true).
+
+
+add_default_text_field(Fields) ->
+ DefaultFields = add_default_text_field(Fields, []),
+ DefaultFields ++ Fields.
+
+
+add_default_text_field([], Acc) ->
+ Acc;
+add_default_text_field([{_Name, <<"string">>, Value} | Rest], Acc) ->
+ NewAcc = [{<<"$default">>, <<"string">>, Value} | Acc],
+ add_default_text_field(Rest, NewAcc);
+add_default_text_field([_ | Rest], Acc) ->
+ add_default_text_field(Rest, Acc).
+
+
+%% index of all field names
+get_field_names([], FAcc) ->
+ FAcc;
+get_field_names([{Name, _Type, _Value} | Rest], FAcc) ->
+ case lists:member([<<"$fieldnames">>, Name, []], FAcc) of
+ true ->
+ get_field_names(Rest, FAcc);
+ false ->
+ get_field_names(Rest, [[<<"$fieldnames">>, Name, []] | FAcc])
+ end.
+
+
+convert_text_fields([]) ->
+ [];
+convert_text_fields([{Name, _Type, Value} | Rest]) ->
+ [[Name, Value, []] | convert_text_fields(Rest)].
+
+
+should_index(Selector, Doc) ->
+ % We should do this
+ NormSelector = mango_selector:normalize(Selector),
+ Matches = mango_selector:match(NormSelector, Doc),
+ IsDesign = case mango_doc:get_field(Doc, <<"_id">>) of
+ <<"_design/", _/binary>> -> true;
+ _ -> false
+ end,
+ Matches and not IsDesign.
+
+
+get_text_field_list(IdxProps) ->
+ case couch_util:get_value(<<"fields">>, IdxProps) of
+ Fields when is_list(Fields) ->
+ RawList = lists:flatmap(fun get_text_field_info/1, Fields),
+ [mango_util:lucene_escape_user(Field) || Field <- RawList];
+ _ ->
+ all_fields
+ end.
+
+
+get_text_field_info({Props}) ->
+ Name = couch_util:get_value(<<"name">>, Props),
+ Type0 = couch_util:get_value(<<"type">>, Props),
+ if not is_binary(Name) -> []; true ->
+ Type = get_text_field_type(Type0),
+ [iolist_to_binary([Name, ":", Type])]
+ end.
+
+
+get_text_field_type(<<"number">>) ->
+ <<"number">>;
+get_text_field_type(<<"boolean">>) ->
+ <<"boolean">>;
+get_text_field_type(_) ->
+ <<"string">>.
+
+
+make_text_field(TAcc, Type, Value) ->
+ FieldName = make_text_field_name(TAcc#tacc.path, Type),
+ Fields = TAcc#tacc.fields,
+ case Fields == all_fields orelse lists:member(FieldName, Fields) of
+ true ->
+ [{FieldName, Type, Value}];
+ false ->
+ []
+ end.
+
+
+make_text_field_name([P | Rest], Type) ->
+ Parts = lists:reverse(Rest, [iolist_to_binary([P, ":", Type])]),
+ Escaped = [mango_util:lucene_escape_field(N) || N <- Parts],
+ iolist_to_binary(mango_util:join(".", Escaped)).
+
+
+validate_index_info(IndexInfo) ->
+ IdxTypes = case module_loaded(dreyfus_index) of
+ true ->
+ [mango_idx_view, mango_idx_text];
+ false ->
+ [mango_idx_view]
+ end,
+ Results = lists:foldl(fun(IdxType, Results0) ->
+ try
+ IdxType:validate_index_def(IndexInfo),
+ [valid_index | Results0]
+ catch _:_ ->
+ [invalid_index | Results0]
+ end
+ end, [], IdxTypes),
+ lists:member(valid_index, Results). \ No newline at end of file
diff --git a/src/mango/src/mango_opts.erl b/src/mango/src/mango_opts.erl
new file mode 100644
index 000000000..af318d238
--- /dev/null
+++ b/src/mango/src/mango_opts.erl
@@ -0,0 +1,314 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(mango_opts).
+
+-export([
+ validate_idx_create/1,
+ validate_find/1
+]).
+
+-export([
+ validate/2,
+
+ is_string/1,
+ is_boolean/1,
+ is_pos_integer/1,
+ is_non_neg_integer/1,
+ is_object/1,
+
+ validate_idx_name/1,
+ validate_selector/1,
+ validate_use_index/1,
+ validate_bookmark/1,
+ validate_sort/1,
+ validate_fields/1,
+ validate_bulk_delete/1,
+
+ default_limit/0
+]).
+
+
+-include("mango.hrl").
+
+
+validate_idx_create({Props}) ->
+ Opts = [
+ {<<"index">>, [
+ {tag, def}
+ ]},
+ {<<"type">>, [
+ {tag, type},
+ {optional, true},
+ {default, <<"json">>},
+ {validator, fun is_string/1}
+ ]},
+ {<<"name">>, [
+ {tag, name},
+ {optional, true},
+ {default, auto_name},
+ {validator, fun validate_idx_name/1}
+ ]},
+ {<<"ddoc">>, [
+ {tag, ddoc},
+ {optional, true},
+ {default, auto_name},
+ {validator, fun validate_idx_name/1}
+ ]},
+ {<<"w">>, [
+ {tag, w},
+ {optional, true},
+ {default, 2},
+ {validator, fun is_pos_integer/1}
+ ]}
+ ],
+ validate(Props, Opts).
+
+
+validate_find({Props}) ->
+ Opts = [
+ {<<"selector">>, [
+ {tag, selector},
+ {validator, fun validate_selector/1}
+ ]},
+ {<<"use_index">>, [
+ {tag, use_index},
+ {optional, true},
+ {default, []},
+ {validator, fun validate_use_index/1}
+ ]},
+ {<<"bookmark">>, [
+ {tag, bookmark},
+ {optional, true},
+ {default, <<>>},
+ {validator, fun validate_bookmark/1}
+ ]},
+ {<<"limit">>, [
+ {tag, limit},
+ {optional, true},
+ {default, default_limit()},
+ {validator, fun is_non_neg_integer/1}
+ ]},
+ {<<"skip">>, [
+ {tag, skip},
+ {optional, true},
+ {default, 0},
+ {validator, fun is_non_neg_integer/1}
+ ]},
+ {<<"sort">>, [
+ {tag, sort},
+ {optional, true},
+ {default, []},
+ {validator, fun validate_sort/1}
+ ]},
+ {<<"fields">>, [
+ {tag, fields},
+ {optional, true},
+ {default, []},
+ {validator, fun validate_fields/1}
+ ]},
+ {<<"r">>, [
+ {tag, r},
+ {optional, true},
+ {default, 1},
+ {validator, fun mango_opts:is_pos_integer/1}
+ ]},
+ {<<"conflicts">>, [
+ {tag, conflicts},
+ {optional, true},
+ {default, false},
+ {validator, fun mango_opts:is_boolean/1}
+ ]}
+ ],
+ validate(Props, Opts).
+
+
+validate_bulk_delete({Props}) ->
+ Opts = [
+ {<<"docids">>, [
+ {tag, docids},
+ {validator, fun validate_bulk_docs/1}
+ ]},
+ {<<"w">>, [
+ {tag, w},
+ {optional, true},
+ {default, 2},
+ {validator, fun is_pos_integer/1}
+ ]}
+ ],
+ validate(Props, Opts).
+
+
+validate(Props, Opts) ->
+ case mango_util:assert_ejson({Props}) of
+ true ->
+ ok;
+ false ->
+ ?MANGO_ERROR({invalid_ejson, {Props}})
+ end,
+ {Rest, Acc} = validate_opts(Opts, Props, []),
+ case Rest of
+ [] ->
+ ok;
+ [{BadKey, _} | _] ->
+ ?MANGO_ERROR({invalid_key, BadKey})
+ end,
+ {ok, Acc}.
+
+
+is_string(Val) when is_binary(Val) ->
+ {ok, Val};
+is_string(Else) ->
+ ?MANGO_ERROR({invalid_string, Else}).
+
+
+is_boolean(true) ->
+ {ok, true};
+is_boolean(false) ->
+ {ok, false};
+is_boolean(Else) ->
+ ?MANGO_ERROR({invalid_boolean, Else}).
+
+
+is_pos_integer(V) when is_integer(V), V > 0 ->
+ {ok, V};
+is_pos_integer(Else) ->
+ ?MANGO_ERROR({invalid_pos_integer, Else}).
+
+
+is_non_neg_integer(V) when is_integer(V), V >= 0 ->
+ {ok, V};
+is_non_neg_integer(Else) ->
+ ?MANGO_ERROR({invalid_non_neg_integer, Else}).
+
+
+is_object({Props}) ->
+ true = mango_util:assert_ejson({Props}),
+ {ok, {Props}};
+is_object(Else) ->
+ ?MANGO_ERROR({invalid_object, Else}).
+
+
+validate_idx_name(auto_name) ->
+ {ok, auto_name};
+validate_idx_name(Else) ->
+ is_string(Else).
+
+
+validate_selector({Props}) ->
+ Norm = mango_selector:normalize({Props}),
+ {ok, Norm};
+validate_selector(Else) ->
+ ?MANGO_ERROR({invalid_selector_json, Else}).
+
+
+%% We re-use validate_use_index to make sure the index names are valid
+validate_bulk_docs(Docs) when is_list(Docs) ->
+ lists:foreach(fun validate_use_index/1, Docs),
+ {ok, Docs};
+validate_bulk_docs(Else) ->
+ ?MANGO_ERROR({invalid_bulk_docs, Else}).
+
+
+validate_use_index(IndexName) when is_binary(IndexName) ->
+ case binary:split(IndexName, <<"/">>) of
+ [DesignId] ->
+ {ok, [DesignId]};
+ [<<"_design">>, DesignId] ->
+ {ok, [DesignId]};
+ [DesignId, ViewName] ->
+ {ok, [DesignId, ViewName]};
+ [<<"_design">>, DesignId, ViewName] ->
+ {ok, [DesignId, ViewName]};
+ _ ->
+ ?MANGO_ERROR({invalid_index_name, IndexName})
+ end;
+validate_use_index(null) ->
+ {ok, []};
+validate_use_index([]) ->
+ {ok, []};
+validate_use_index([DesignId]) when is_binary(DesignId) ->
+ {ok, [DesignId]};
+validate_use_index([DesignId, ViewName])
+ when is_binary(DesignId), is_binary(ViewName) ->
+ {ok, [DesignId, ViewName]};
+validate_use_index(Else) ->
+ ?MANGO_ERROR({invalid_index_name, Else}).
+
+
+validate_bookmark(null) ->
+ {ok, nil};
+validate_bookmark(<<>>) ->
+ {ok, nil};
+validate_bookmark(Bin) when is_binary(Bin) ->
+ {ok, Bin};
+validate_bookmark(Else) ->
+ ?MANGO_ERROR({invalid_bookmark, Else}).
+
+
+validate_sort(Value) ->
+ mango_sort:new(Value).
+
+
+validate_fields(Value) ->
+ mango_fields:new(Value).
+
+
+validate_opts([], Props, Acc) ->
+ {Props, lists:reverse(Acc)};
+validate_opts([{Name, Desc} | Rest], Props, Acc) ->
+ {tag, Tag} = lists:keyfind(tag, 1, Desc),
+ case lists:keytake(Name, 1, Props) of
+ {value, {Name, Prop}, RestProps} ->
+ NewAcc = [{Tag, validate_opt(Name, Desc, Prop)} | Acc],
+ validate_opts(Rest, RestProps, NewAcc);
+ false ->
+ NewAcc = [{Tag, validate_opt(Name, Desc, undefined)} | Acc],
+ validate_opts(Rest, Props, NewAcc)
+ end.
+
+
+validate_opt(_Name, [], Value) ->
+ Value;
+validate_opt(Name, Desc0, undefined) ->
+ case lists:keytake(optional, 1, Desc0) of
+ {value, {optional, true}, Desc1} ->
+ {value, {default, Value}, Desc2} = lists:keytake(default, 1, Desc1),
+ false = (Value == undefined),
+ validate_opt(Name, Desc2, Value);
+ _ ->
+ ?MANGO_ERROR({missing_required_key, Name})
+ end;
+validate_opt(Name, [{tag, _} | Rest], Value) ->
+ % Tags aren't really validated
+ validate_opt(Name, Rest, Value);
+validate_opt(Name, [{optional, _} | Rest], Value) ->
+ % A value was specified for an optional value
+ validate_opt(Name, Rest, Value);
+validate_opt(Name, [{default, _} | Rest], Value) ->
+ % A value was specified for an optional value
+ validate_opt(Name, Rest, Value);
+validate_opt(Name, [{assert, Value} | Rest], Value) ->
+ validate_opt(Name, Rest, Value);
+validate_opt(Name, [{assert, Expect} | _], Found) ->
+ ?MANGO_ERROR({invalid_value, Name, Expect, Found});
+validate_opt(Name, [{validator, Fun} | Rest], Value) ->
+ case Fun(Value) of
+ {ok, Validated} ->
+ validate_opt(Name, Rest, Validated);
+ false ->
+ ?MANGO_ERROR({invalid_value, Name, Value})
+ end.
+
+
+default_limit() ->
+ config:get_integer("mango", "default_limit", 25).
diff --git a/src/mango/src/mango_selector.erl b/src/mango/src/mango_selector.erl
new file mode 100644
index 000000000..691aac7ed
--- /dev/null
+++ b/src/mango/src/mango_selector.erl
@@ -0,0 +1,568 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(mango_selector).
+
+
+-export([
+ normalize/1,
+ match/2
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include("mango.hrl").
+
+
+% Validate and normalize each operator. This translates
+% every selector operator into a consistent version that
+% we can then rely on for all other selector functions.
+% See the definition of each step below for more information
+% on what each one does.
+normalize({[]}) ->
+ {[]};
+normalize(Selector) ->
+ Steps = [
+ fun norm_ops/1,
+ fun norm_fields/1,
+ fun norm_negations/1
+ ],
+ {NProps} = lists:foldl(fun(Step, Sel) -> Step(Sel) end, Selector, Steps),
+ FieldNames = [Name || {Name, _} <- NProps],
+ case lists:member(<<>>, FieldNames) of
+ true ->
+ ?MANGO_ERROR({invalid_selector, missing_field_name});
+ false ->
+ ok
+ end,
+ {NProps}.
+
+
+% Match a selector against a #doc{} or EJSON value.
+% This assumes that the Selector has been normalized.
+% Returns true or false.
+
+% An empty selector matches any value.
+match({[]}, _) ->
+ true;
+
+match(Selector, #doc{body=Body}) ->
+ match(Selector, Body, fun mango_json:cmp/2);
+
+match(Selector, {Props}) ->
+ match(Selector, {Props}, fun mango_json:cmp/2).
+
+% Convert each operator into a normalized version as well
+% as convert an implict operators into their explicit
+% versions.
+norm_ops({[{<<"$and">>, Args}]}) when is_list(Args) ->
+ {[{<<"$and">>, [norm_ops(A) || A <- Args]}]};
+norm_ops({[{<<"$and">>, Arg}]}) ->
+ ?MANGO_ERROR({bad_arg, '$and', Arg});
+
+norm_ops({[{<<"$or">>, Args}]}) when is_list(Args) ->
+ {[{<<"$or">>, [norm_ops(A) || A <- Args]}]};
+norm_ops({[{<<"$or">>, Arg}]}) ->
+ ?MANGO_ERROR({bad_arg, '$or', Arg});
+
+norm_ops({[{<<"$not">>, {_}=Arg}]}) ->
+ {[{<<"$not">>, norm_ops(Arg)}]};
+norm_ops({[{<<"$not">>, Arg}]}) ->
+ ?MANGO_ERROR({bad_arg, '$not', Arg});
+
+norm_ops({[{<<"$nor">>, Args}]}) when is_list(Args) ->
+ {[{<<"$nor">>, [norm_ops(A) || A <- Args]}]};
+norm_ops({[{<<"$nor">>, Arg}]}) ->
+ ?MANGO_ERROR({bad_arg, '$nor', Arg});
+
+norm_ops({[{<<"$in">>, Args}]} = Cond) when is_list(Args) ->
+ Cond;
+norm_ops({[{<<"$in">>, Arg}]}) ->
+ ?MANGO_ERROR({bad_arg, '$in', Arg});
+
+norm_ops({[{<<"$nin">>, Args}]} = Cond) when is_list(Args) ->
+ Cond;
+norm_ops({[{<<"$nin">>, Arg}]}) ->
+ ?MANGO_ERROR({bad_arg, '$nin', Arg});
+
+norm_ops({[{<<"$exists">>, Arg}]} = Cond) when is_boolean(Arg) ->
+ Cond;
+norm_ops({[{<<"$exists">>, Arg}]}) ->
+ ?MANGO_ERROR({bad_arg, '$exists', Arg});
+
+norm_ops({[{<<"$type">>, Arg}]} = Cond) when is_binary(Arg) ->
+ Cond;
+norm_ops({[{<<"$type">>, Arg}]}) ->
+ ?MANGO_ERROR({bad_arg, '$type', Arg});
+
+norm_ops({[{<<"$mod">>, [D, R]}]} = Cond) when is_integer(D), is_integer(R) ->
+ Cond;
+norm_ops({[{<<"$mod">>, Arg}]}) ->
+ ?MANGO_ERROR({bad_arg, '$mod', Arg});
+
+norm_ops({[{<<"$regex">>, Regex}]} = Cond) when is_binary(Regex) ->
+ case re:compile(Regex) of
+ {ok, _} ->
+ Cond;
+ _ ->
+ ?MANGO_ERROR({bad_arg, '$regex', Regex})
+ end;
+
+norm_ops({[{<<"$all">>, Args}]}) when is_list(Args) ->
+ {[{<<"$all">>, Args}]};
+norm_ops({[{<<"$all">>, Arg}]}) ->
+ ?MANGO_ERROR({bad_arg, '$all', Arg});
+
+norm_ops({[{<<"$elemMatch">>, {_}=Arg}]}) ->
+ {[{<<"$elemMatch">>, norm_ops(Arg)}]};
+norm_ops({[{<<"$elemMatch">>, Arg}]}) ->
+ ?MANGO_ERROR({bad_arg, '$elemMatch', Arg});
+
+norm_ops({[{<<"$allMatch">>, {_}=Arg}]}) ->
+ {[{<<"$allMatch">>, norm_ops(Arg)}]};
+norm_ops({[{<<"$allMatch">>, Arg}]}) ->
+ ?MANGO_ERROR({bad_arg, '$allMatch', Arg});
+
+norm_ops({[{<<"$size">>, Arg}]}) when is_integer(Arg), Arg >= 0 ->
+ {[{<<"$size">>, Arg}]};
+norm_ops({[{<<"$size">>, Arg}]}) ->
+ ?MANGO_ERROR({bad_arg, '$size', Arg});
+
+norm_ops({[{<<"$text">>, Arg}]}) when is_binary(Arg); is_number(Arg);
+ is_boolean(Arg) ->
+ {[{<<"$default">>, {[{<<"$text">>, Arg}]}}]};
+norm_ops({[{<<"$text">>, Arg}]}) ->
+ ?MANGO_ERROR({bad_arg, '$text', Arg});
+
+% Not technically an operator but we pass it through here
+% so that this function accepts its own output. This exists
+% so that $text can have a field name value which simplifies
+% logic elsewhere.
+norm_ops({[{<<"$default">>, _}]} = Selector) ->
+ Selector;
+
+% Terminals where we can't perform any validation
+% on the value because any value is acceptable.
+norm_ops({[{<<"$lt">>, _}]} = Cond) ->
+ Cond;
+norm_ops({[{<<"$lte">>, _}]} = Cond) ->
+ Cond;
+norm_ops({[{<<"$eq">>, _}]} = Cond) ->
+ Cond;
+norm_ops({[{<<"$ne">>, _}]} = Cond) ->
+ Cond;
+norm_ops({[{<<"$gte">>, _}]} = Cond) ->
+ Cond;
+norm_ops({[{<<"$gt">>, _}]} = Cond) ->
+ Cond;
+
+% Known but unsupported operators
+norm_ops({[{<<"$where">>, _}]}) ->
+ ?MANGO_ERROR({not_supported, '$where'});
+norm_ops({[{<<"$geoWithin">>, _}]}) ->
+ ?MANGO_ERROR({not_supported, '$geoWithin'});
+norm_ops({[{<<"$geoIntersects">>, _}]}) ->
+ ?MANGO_ERROR({not_supported, '$geoIntersects'});
+norm_ops({[{<<"$near">>, _}]}) ->
+ ?MANGO_ERROR({not_supported, '$near'});
+norm_ops({[{<<"$nearSphere">>, _}]}) ->
+ ?MANGO_ERROR({not_supported, '$nearSphere'});
+
+% Unknown operator
+norm_ops({[{<<"$", _/binary>>=Op, _}]}) ->
+ ?MANGO_ERROR({invalid_operator, Op});
+
+% A {Field: Cond} pair
+norm_ops({[{Field, Cond}]}) ->
+ {[{Field, norm_ops(Cond)}]};
+
+% An implicit $and
+norm_ops({Props}) when length(Props) > 1 ->
+ {[{<<"$and">>, [norm_ops({[P]}) || P <- Props]}]};
+
+% A bare value condition means equality
+norm_ops(Value) ->
+ {[{<<"$eq">>, Value}]}.
+
+
+% This takes a selector and normalizes all of the
+% field names as far as possible. For instance:
+%
+% Unnormalized:
+% {foo: {$and: [{$gt: 5}, {$lt: 10}]}}
+%
+% Normalized:
+% {$and: [{foo: {$gt: 5}}, {foo: {$lt: 10}}]}
+%
+% And another example:
+%
+% Unnormalized:
+% {foo: {bar: {$gt: 10}}}
+%
+% Normalized:
+% {"foo.bar": {$gt: 10}}
+%
+% Its important to note that we can only normalize
+% field names like this through boolean operators where
+% we can gaurantee commutativity. We can't necessarily
+% do the same through the '$elemMatch' or '$allMatch'
+% operators but we can apply the same algorithm to its
+% arguments.
+norm_fields({[]}) ->
+ {[]};
+norm_fields(Selector) ->
+ norm_fields(Selector, <<>>).
+
+
+% Operators where we can push the field names further
+% down the operator tree
+norm_fields({[{<<"$and">>, Args}]}, Path) ->
+ {[{<<"$and">>, [norm_fields(A, Path) || A <- Args]}]};
+
+norm_fields({[{<<"$or">>, Args}]}, Path) ->
+ {[{<<"$or">>, [norm_fields(A, Path) || A <- Args]}]};
+
+norm_fields({[{<<"$not">>, Arg}]}, Path) ->
+ {[{<<"$not">>, norm_fields(Arg, Path)}]};
+
+norm_fields({[{<<"$nor">>, Args}]}, Path) ->
+ {[{<<"$nor">>, [norm_fields(A, Path) || A <- Args]}]};
+
+% Fields where we can normalize fields in the
+% operator arguments independently.
+norm_fields({[{<<"$elemMatch">>, Arg}]}, Path) ->
+ Cond = {[{<<"$elemMatch">>, norm_fields(Arg)}]},
+ {[{Path, Cond}]};
+
+norm_fields({[{<<"$allMatch">>, Arg}]}, Path) ->
+ Cond = {[{<<"$allMatch">>, norm_fields(Arg)}]},
+ {[{Path, Cond}]};
+
+
+% The text operator operates against the internal
+% $default field. This also asserts that the $default
+% field is at the root as well as that it only has
+% a $text operator applied.
+norm_fields({[{<<"$default">>, {[{<<"$text">>, _Arg}]}}]}=Sel, <<>>) ->
+ Sel;
+norm_fields({[{<<"$default">>, _}]} = Selector, _) ->
+ ?MANGO_ERROR({bad_field, Selector});
+
+
+% Any other operator is a terminal below which no
+% field names should exist. Set the path to this
+% terminal and return it.
+norm_fields({[{<<"$", _/binary>>, _}]} = Cond, Path) ->
+ {[{Path, Cond}]};
+
+% We've found a field name. Append it to the path
+% and skip this node as we unroll the stack as
+% the full path will be further down the branch.
+norm_fields({[{Field, Cond}]}, <<>>) ->
+ % Don't include the '.' for the first element of
+ % the path.
+ norm_fields(Cond, Field);
+norm_fields({[{Field, Cond}]}, Path) ->
+ norm_fields(Cond, <<Path/binary, ".", Field/binary>>);
+
+% An empty selector
+norm_fields({[]}, Path) ->
+ {Path, {[]}};
+
+% Else we have an invalid selector
+norm_fields(BadSelector, _) ->
+ ?MANGO_ERROR({bad_field, BadSelector}).
+
+
+% Take all the negation operators and move the logic
+% as far down the branch as possible. This does things
+% like:
+%
+% Unnormalized:
+% {$not: {foo: {$gt: 10}}}
+%
+% Normalized:
+% {foo: {$lte: 10}}
+%
+% And we also apply DeMorgan's laws
+%
+% Unnormalized:
+% {$not: {$and: [{foo: {$gt: 10}}, {foo: {$lt: 5}}]}}
+%
+% Normalized:
+% {$or: [{foo: {$lte: 10}}, {foo: {$gte: 5}}]}
+%
+% This logic is important because we can't "see" through
+% a '$not' operator to be able to locate indices that may
+% service a specific query. Though if we move the negations
+% down to the terminals we may be able to negate specific
+% operators which allows us to find usable indices.
+
+% Operators that cause a negation
+norm_negations({[{<<"$not">>, Arg}]}) ->
+ negate(Arg);
+
+norm_negations({[{<<"$nor">>, Args}]}) ->
+ {[{<<"$and">>, [negate(A) || A <- Args]}]};
+
+% Operators that we merely seek through as we look for
+% negations.
+norm_negations({[{<<"$and">>, Args}]}) ->
+ {[{<<"$and">>, [norm_negations(A) || A <- Args]}]};
+
+norm_negations({[{<<"$or">>, Args}]}) ->
+ {[{<<"$or">>, [norm_negations(A) || A <- Args]}]};
+
+norm_negations({[{<<"$elemMatch">>, Arg}]}) ->
+ {[{<<"$elemMatch">>, norm_negations(Arg)}]};
+
+norm_negations({[{<<"$allMatch">>, Arg}]}) ->
+ {[{<<"$allMatch">>, norm_negations(Arg)}]};
+
+% All other conditions can't introduce negations anywhere
+% further down the operator tree.
+norm_negations(Cond) ->
+ Cond.
+
+
+% Actually negate an expression. Make sure and read up
+% on DeMorgan's laws if you're trying to read this, but
+% in a nutshell:
+%
+% NOT(a AND b) == NOT(a) OR NOT(b)
+% NOT(a OR b) == NOT(a) AND NOT(b)
+%
+% Also notice that if a negation hits another negation
+% operator that we just nullify the combination. Its
+% possible that below the nullification we have more
+% negations so we have to recurse back to norm_negations/1.
+
+% Negating negation, nullify but recurse to
+% norm_negations/1
+negate({[{<<"$not">>, Arg}]}) ->
+ norm_negations(Arg);
+
+negate({[{<<"$nor">>, Args}]}) ->
+ {[{<<"$or">>, [norm_negations(A) || A <- Args]}]};
+
+% DeMorgan Negations
+negate({[{<<"$and">>, Args}]}) ->
+ {[{<<"$or">>, [negate(A) || A <- Args]}]};
+
+negate({[{<<"$or">>, Args}]}) ->
+ {[{<<"$and">>, [negate(A) || A <- Args]}]};
+
+negate({[{<<"$default">>, _}]} = Arg) ->
+ ?MANGO_ERROR({bad_arg, '$not', Arg});
+
+% Negating comparison operators is straight forward
+negate({[{<<"$lt">>, Arg}]}) ->
+ {[{<<"$gte">>, Arg}]};
+negate({[{<<"$lte">>, Arg}]}) ->
+ {[{<<"$gt">>, Arg}]};
+negate({[{<<"$eq">>, Arg}]}) ->
+ {[{<<"$ne">>, Arg}]};
+negate({[{<<"$ne">>, Arg}]}) ->
+ {[{<<"$eq">>, Arg}]};
+negate({[{<<"$gte">>, Arg}]}) ->
+ {[{<<"$lt">>, Arg}]};
+negate({[{<<"$gt">>, Arg}]}) ->
+ {[{<<"$lte">>, Arg}]};
+negate({[{<<"$in">>, Args}]}) ->
+ {[{<<"$nin">>, Args}]};
+negate({[{<<"$nin">>, Args}]}) ->
+ {[{<<"$in">>, Args}]};
+
+% We can also trivially negate the exists operator
+negate({[{<<"$exists">>, Arg}]}) ->
+ {[{<<"$exists">>, not Arg}]};
+
+% Anything else we have to just terminate the
+% negation by reinserting the negation operator
+negate({[{<<"$", _/binary>>, _}]} = Cond) ->
+ {[{<<"$not">>, Cond}]};
+
+% Finally, negating a field just means we negate its
+% condition.
+negate({[{Field, Cond}]}) ->
+ {[{Field, negate(Cond)}]}.
+
+
+match({[{<<"$and">>, Args}]}, Value, Cmp) ->
+ Pred = fun(SubSel) -> match(SubSel, Value, Cmp) end,
+ lists:all(Pred, Args);
+
+match({[{<<"$or">>, Args}]}, Value, Cmp) ->
+ Pred = fun(SubSel) -> match(SubSel, Value, Cmp) end,
+ lists:any(Pred, Args);
+
+match({[{<<"$not">>, Arg}]}, Value, Cmp) ->
+ not match(Arg, Value, Cmp);
+
+% All of the values in Args must exist in Values or
+% Values == hd(Args) if Args is a single element list
+% that contains a list.
+match({[{<<"$all">>, Args}]}, Values, _Cmp) when is_list(Values) ->
+ Pred = fun(A) -> lists:member(A, Values) end,
+ HasArgs = lists:all(Pred, Args),
+ IsArgs = case Args of
+ [A] when is_list(A) ->
+ A == Values;
+ _ ->
+ false
+ end,
+ HasArgs orelse IsArgs;
+match({[{<<"$all">>, _Args}]}, _Values, _Cmp) ->
+ false;
+
+%% This is for $elemMatch, $allMatch, and possibly $in because of our normalizer.
+%% A selector such as {"field_name": {"$elemMatch": {"$gte": 80, "$lt": 85}}}
+%% gets normalized to:
+%% {[{<<"field_name">>,
+%% {[{<<"$elemMatch">>,
+%% {[{<<"$and">>, [
+%% {[{<<>>,{[{<<"$gte">>,80}]}}]},
+%% {[{<<>>,{[{<<"$lt">>,85}]}}]}
+%% ]}]}
+%% }]}
+%% }]}.
+%% So we filter out the <<>>.
+match({[{<<>>, Arg}]}, Values, Cmp) ->
+ match(Arg, Values, Cmp);
+
+% Matches when any element in values matches the
+% sub-selector Arg.
+match({[{<<"$elemMatch">>, Arg}]}, Values, Cmp) when is_list(Values) ->
+ try
+ lists:foreach(fun(V) ->
+ case match(Arg, V, Cmp) of
+ true -> throw(matched);
+ _ -> ok
+ end
+ end, Values),
+ false
+ catch
+ throw:matched ->
+ true;
+ _:_ ->
+ false
+ end;
+match({[{<<"$elemMatch">>, _Arg}]}, _Value, _Cmp) ->
+ false;
+
+% Matches when all elements in values match the
+% sub-selector Arg.
+match({[{<<"$allMatch">>, Arg}]}, Values, Cmp) when is_list(Values) ->
+ try
+ lists:foreach(fun(V) ->
+ case match(Arg, V, Cmp) of
+ false -> throw(unmatched);
+ _ -> ok
+ end
+ end, Values),
+ true
+ catch
+ _:_ ->
+ false
+ end;
+match({[{<<"$allMatch">>, _Arg}]}, _Value, _Cmp) ->
+ false;
+
+% Our comparison operators are fairly straight forward
+match({[{<<"$lt">>, Arg}]}, Value, Cmp) ->
+ Cmp(Value, Arg) < 0;
+match({[{<<"$lte">>, Arg}]}, Value, Cmp) ->
+ Cmp(Value, Arg) =< 0;
+match({[{<<"$eq">>, Arg}]}, Value, Cmp) ->
+ Cmp(Value, Arg) == 0;
+match({[{<<"$ne">>, Arg}]}, Value, Cmp) ->
+ Cmp(Value, Arg) /= 0;
+match({[{<<"$gte">>, Arg}]}, Value, Cmp) ->
+ Cmp(Value, Arg) >= 0;
+match({[{<<"$gt">>, Arg}]}, Value, Cmp) ->
+ Cmp(Value, Arg) > 0;
+
+match({[{<<"$in">>, Args}]}, Values, Cmp) when is_list(Values)->
+ Pred = fun(Arg) ->
+ lists:foldl(fun(Value,Match) ->
+ (Cmp(Value, Arg) == 0) or Match
+ end, false, Values)
+ end,
+ lists:any(Pred, Args);
+match({[{<<"$in">>, Args}]}, Value, Cmp) ->
+ Pred = fun(Arg) -> Cmp(Value, Arg) == 0 end,
+ lists:any(Pred, Args);
+
+match({[{<<"$nin">>, Args}]}, Values, Cmp) when is_list(Values)->
+ not match({[{<<"$in">>, Args}]}, Values, Cmp);
+match({[{<<"$nin">>, Args}]}, Value, Cmp) ->
+ Pred = fun(Arg) -> Cmp(Value, Arg) /= 0 end,
+ lists:all(Pred, Args);
+
+% This logic is a bit subtle. Basically, if value is
+% not undefined, then it exists.
+match({[{<<"$exists">>, ShouldExist}]}, Value, _Cmp) ->
+ Exists = Value /= undefined,
+ ShouldExist andalso Exists;
+
+match({[{<<"$type">>, Arg}]}, Value, _Cmp) when is_binary(Arg) ->
+ Arg == mango_json:type(Value);
+
+match({[{<<"$mod">>, [D, R]}]}, Value, _Cmp) when is_integer(Value) ->
+ Value rem D == R;
+match({[{<<"$mod">>, _}]}, _Value, _Cmp) ->
+ false;
+
+match({[{<<"$regex">>, Regex}]}, Value, _Cmp) when is_binary(Value) ->
+ try
+ match == re:run(Value, Regex, [{capture, none}])
+ catch _:_ ->
+ false
+ end;
+match({[{<<"$regex">>, _}]}, _Value, _Cmp) ->
+ false;
+
+match({[{<<"$size">>, Arg}]}, Values, _Cmp) when is_list(Values) ->
+ length(Values) == Arg;
+match({[{<<"$size">>, _}]}, _Value, _Cmp) ->
+ false;
+
+% We don't have any choice but to believe that the text
+% index returned valid matches
+match({[{<<"$default">>, _}]}, _Value, _Cmp) ->
+ true;
+
+% All other operators are internal assertion errors for
+% matching because we either should've removed them during
+% normalization or something else broke.
+match({[{<<"$", _/binary>>=Op, _}]}, _, _) ->
+ ?MANGO_ERROR({invalid_operator, Op});
+
+% We need to traverse value to find field. The call to
+% mango_doc:get_field/2 may return either not_found or
+% bad_path in which case matching fails.
+match({[{Field, Cond}]}, Value, Cmp) ->
+ case mango_doc:get_field(Value, Field) of
+ not_found when Cond == {[{<<"$exists">>, false}]} ->
+ true;
+ not_found ->
+ false;
+ bad_path ->
+ false;
+ SubValue when Field == <<"_id">> ->
+ match(Cond, SubValue, fun mango_json:cmp_raw/2);
+ SubValue ->
+ match(Cond, SubValue, Cmp)
+ end;
+
+match({Props} = Sel, _Value, _Cmp) when length(Props) > 1 ->
+ erlang:error({unnormalized_selector, Sel}).
diff --git a/src/mango/src/mango_selector_text.erl b/src/mango/src/mango_selector_text.erl
new file mode 100644
index 000000000..cfa3baf6d
--- /dev/null
+++ b/src/mango/src/mango_selector_text.erl
@@ -0,0 +1,416 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(mango_selector_text).
+
+
+-export([
+ convert/1,
+ convert/2,
+
+ append_sort_type/2
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include("mango.hrl").
+
+
+%% Regex for <<"\\.">>
+-define(PERIOD, "\\.").
+
+
+convert(Object) ->
+ TupleTree = convert([], Object),
+ iolist_to_binary(to_query(TupleTree)).
+
+
+convert(Path, {[{<<"$and">>, Args}]}) ->
+ Parts = [convert(Path, Arg) || Arg <- Args],
+ {op_and, Parts};
+convert(Path, {[{<<"$or">>, Args}]}) ->
+ Parts = [convert(Path, Arg) || Arg <- Args],
+ {op_or, Parts};
+convert(Path, {[{<<"$not">>, Arg}]}) ->
+ {op_not, {field_exists_query(Path), convert(Path, Arg)}};
+convert(Path, {[{<<"$default">>, Arg}]}) ->
+ {op_field, {_, Query}} = convert(Path, Arg),
+ {op_default, Query};
+
+% The $text operator specifies a Lucene syntax query
+% so we just pull it in directly.
+convert(Path, {[{<<"$text">>, Query}]}) when is_binary(Query) ->
+ {op_field, {make_field(Path, Query), value_str(Query)}};
+
+% The MongoDB docs for $all are super confusing and read more
+% like they screwed up the implementation of this operator
+% and then just documented it as a feature.
+%
+% This implementation will match the behavior as closely as
+% possible based on the available docs but we'll need to have
+% the testing team validate how MongoDB handles edge conditions
+convert(Path, {[{<<"$all">>, Args}]}) ->
+ case Args of
+ [Values] when is_list(Values) ->
+ % If Args is a single element array then we have to
+ % either match if Path is that array or if it contains
+ % the array as an element of an array (which isn't at all
+ % confusing). For Lucene to return us all possible matches
+ % that means we just need to search for each value in
+ % Path.[] and Path.[].[] and rely on our filtering to limit
+ % the results properly.
+ Fields1 = convert(Path, {[{<<"$eq">> , Values}]}),
+ Fields2 = convert([<<"[]">>| Path], {[{<<"$eq">> , Values}]}),
+ {op_or, [Fields1, Fields2]};
+ _ ->
+ % Otherwise the $all operator is equivalent to an $and
+ % operator so we treat it as such.
+ convert([<<"[]">> | Path], {[{<<"$and">>, Args}]})
+ end;
+
+% The $elemMatch Lucene query is not an exact translation
+% as we can't enforce that the matches are all for the same
+% item in an array. We just rely on the final selector match
+% to filter out anything that doesn't match. The only trick
+% is that we have to add the `[]` path element since the docs
+% say this has to match against an array.
+convert(Path, {[{<<"$elemMatch">>, Arg}]}) ->
+ convert([<<"[]">> | Path], Arg);
+
+convert(Path, {[{<<"$allMatch">>, Arg}]}) ->
+ convert([<<"[]">> | Path], Arg);
+
+% Our comparison operators are fairly straight forward
+convert(Path, {[{<<"$lt">>, Arg}]}) when is_list(Arg); is_tuple(Arg);
+ Arg =:= null ->
+ field_exists_query(Path);
+convert(Path, {[{<<"$lt">>, Arg}]}) ->
+ {op_field, {make_field(Path, Arg), range(lt, Arg)}};
+convert(Path, {[{<<"$lte">>, Arg}]}) when is_list(Arg); is_tuple(Arg);
+ Arg =:= null->
+ field_exists_query(Path);
+convert(Path, {[{<<"$lte">>, Arg}]}) ->
+ {op_field, {make_field(Path, Arg), range(lte, Arg)}};
+%% This is for indexable_fields
+convert(Path, {[{<<"$eq">>, Arg}]}) when Arg =:= null ->
+ {op_null, {make_field(Path, Arg), value_str(Arg)}};
+convert(Path, {[{<<"$eq">>, Args}]}) when is_list(Args) ->
+ Path0 = [<<"[]">> | Path],
+ LPart = {op_field, {make_field(Path0, length), value_str(length(Args))}},
+ Parts0 = [convert(Path0, {[{<<"$eq">>, Arg}]}) || Arg <- Args],
+ Parts = [LPart | Parts0],
+ {op_and, Parts};
+convert(Path, {[{<<"$eq">>, {_} = Arg}]}) ->
+ convert(Path, Arg);
+convert(Path, {[{<<"$eq">>, Arg}]}) ->
+ {op_field, {make_field(Path, Arg), value_str(Arg)}};
+convert(Path, {[{<<"$ne">>, Arg}]}) ->
+ {op_not, {field_exists_query(Path), convert(Path, {[{<<"$eq">>, Arg}]})}};
+convert(Path, {[{<<"$gte">>, Arg}]}) when is_list(Arg); is_tuple(Arg);
+ Arg =:= null ->
+ field_exists_query(Path);
+convert(Path, {[{<<"$gte">>, Arg}]}) ->
+ {op_field, {make_field(Path, Arg), range(gte, Arg)}};
+convert(Path, {[{<<"$gt">>, Arg}]}) when is_list(Arg); is_tuple(Arg);
+ Arg =:= null->
+ field_exists_query(Path);
+convert(Path, {[{<<"$gt">>, Arg}]}) ->
+ {op_field, {make_field(Path, Arg), range(gt, Arg)}};
+
+convert(Path, {[{<<"$in">>, Args}]}) ->
+ {op_or, convert_in(Path, Args)};
+
+convert(Path, {[{<<"$nin">>, Args}]}) ->
+ {op_not, {field_exists_query(Path), convert(Path, {[{<<"$in">>, Args}]})}};
+
+convert(Path, {[{<<"$exists">>, ShouldExist}]}) ->
+ FieldExists = field_exists_query(Path),
+ case ShouldExist of
+ true -> FieldExists;
+ false -> {op_not, {FieldExists, false}}
+ end;
+
+% We're not checking the actual type here, just looking for
+% anything that has a possibility of matching by checking
+% for the field name. We use the same logic for $exists on
+% the actual query.
+convert(Path, {[{<<"$type">>, _}]}) ->
+ field_exists_query(Path);
+
+convert(Path, {[{<<"$mod">>, _}]}) ->
+ field_exists_query(Path, "number");
+
+% The lucene regular expression engine does not use java's regex engine but
+% instead a custom implementation. The syntax is therefore different, so we do
+% would get different behavior than our view indexes. To be consistent, we will
+% simply return docs for fields that exist and then run our match filter.
+convert(Path, {[{<<"$regex">>, _}]}) ->
+ field_exists_query(Path, "string");
+
+convert(Path, {[{<<"$size">>, Arg}]}) ->
+ {op_field, {make_field([<<"[]">> | Path], length), value_str(Arg)}};
+
+% All other operators are internal assertion errors for
+% matching because we either should've removed them during
+% normalization or something else broke.
+convert(_Path, {[{<<"$", _/binary>>=Op, _}]}) ->
+ ?MANGO_ERROR({invalid_operator, Op});
+
+% We've hit a field name specifier. Check if the field name is accessing
+% arrays. Convert occurrences of element position references to .[]. Then we
+% need to break the name into path parts and continue our conversion.
+convert(Path, {[{Field0, Cond}]}) ->
+ {ok, PP0} = case Field0 of
+ <<>> ->
+ {ok, []};
+ _ ->
+ mango_util:parse_field(Field0)
+ end,
+ % Later on, we perform a lucene_escape_user call on the
+ % final Path, which calls parse_field again. Calling the function
+ % twice converts <<"a\\.b">> to [<<"a">>,<<"b">>]. This leads to
+ % an incorrect query since we need [<<"a.b">>]. Without breaking
+ % our escaping mechanism, we simply revert this first parse_field
+ % effect and replace instances of "." to "\\.".
+ MP = mango_util:cached_re(mango_period, ?PERIOD),
+ PP1 = [re:replace(P, MP, <<"\\\\.">>,
+ [global,{return,binary}]) || P <- PP0],
+ {PP2, HasInteger} = replace_array_indexes(PP1, [], false),
+ NewPath = PP2 ++ Path,
+ case HasInteger of
+ true ->
+ OldPath = lists:reverse(PP1, Path),
+ OldParts = convert(OldPath, Cond),
+ NewParts = convert(NewPath, Cond),
+ {op_or, [OldParts, NewParts]};
+ false ->
+ convert(NewPath, Cond)
+ end;
+
+%% For $in
+convert(Path, Val) when is_binary(Val); is_number(Val); is_boolean(Val) ->
+ {op_field, {make_field(Path, Val), value_str(Val)}};
+
+% Anything else is a bad selector.
+convert(_Path, {Props} = Sel) when length(Props) > 1 ->
+ erlang:error({unnormalized_selector, Sel}).
+
+
+to_query({op_and, Args}) when is_list(Args) ->
+ QueryArgs = lists:map(fun to_query/1, Args),
+ ["(", mango_util:join(<<" AND ">>, QueryArgs), ")"];
+
+to_query({op_or, Args}) when is_list(Args) ->
+ ["(", mango_util:join(" OR ", lists:map(fun to_query/1, Args)), ")"];
+
+to_query({op_not, {ExistsQuery, Arg}}) when is_tuple(Arg) ->
+ ["(", to_query(ExistsQuery), " AND NOT (", to_query(Arg), "))"];
+
+%% For $exists:false
+to_query({op_not, {ExistsQuery, false}}) ->
+ ["($fieldnames:/.*/ ", " AND NOT (", to_query(ExistsQuery), "))"];
+
+to_query({op_insert, Arg}) when is_binary(Arg) ->
+ ["(", Arg, ")"];
+
+%% We escape : and / for now for values and all lucene chars for fieldnames
+%% This needs to be resolved.
+to_query({op_field, {Name, Value}}) ->
+ NameBin = iolist_to_binary(Name),
+ ["(", mango_util:lucene_escape_user(NameBin), ":", Value, ")"];
+
+%% This is for indexable_fields
+to_query({op_null, {Name, Value}}) ->
+ NameBin = iolist_to_binary(Name),
+ ["(", mango_util:lucene_escape_user(NameBin), ":", Value, ")"];
+
+to_query({op_fieldname, {Name, Wildcard}}) ->
+ NameBin = iolist_to_binary(Name),
+ ["($fieldnames:", mango_util:lucene_escape_user(NameBin), Wildcard, ")"];
+
+to_query({op_default, Value}) ->
+ ["($default:", Value, ")"].
+
+
+%% We match on fieldname and fieldname.[]
+convert_in(Path, Args) ->
+ Path0 = [<<"[]">> | Path],
+ lists:map(fun(Arg) ->
+ case Arg of
+ {Object} ->
+ Parts = lists:map(fun (SubObject) ->
+ Fields1 = convert(Path, {[SubObject]}),
+ Fields2 = convert(Path0, {[SubObject]}),
+ {op_or, [Fields1, Fields2]}
+ end, Object),
+ {op_or, Parts};
+ SingleVal ->
+ Fields1 = {op_field, {make_field(Path, SingleVal),
+ value_str(SingleVal)}},
+ Fields2 = {op_field, {make_field(Path0, SingleVal),
+ value_str(SingleVal)}},
+ {op_or, [Fields1, Fields2]}
+ end
+ end, Args).
+
+
+make_field(Path, length) ->
+ [path_str(Path), <<":length">>];
+make_field(Path, Arg) ->
+ [path_str(Path), <<":">>, type_str(Arg)].
+
+
+range(lt, Arg) ->
+ Min = get_range(min, Arg),
+ [<<"[", Min/binary, " TO ">>, value_str(Arg), <<"}">>];
+range(lte, Arg) ->
+ Min = get_range(min, Arg),
+ [<<"[", Min/binary, " TO ">>, value_str(Arg), <<"]">>];
+range(gte, Arg) ->
+ Max = get_range(max, Arg),
+ [<<"[">>, value_str(Arg), <<" TO ", Max/binary, "]">>];
+range(gt, Arg) ->
+ Max = get_range(max, Arg),
+ [<<"{">>, value_str(Arg), <<" TO ", Max/binary, "]">>].
+
+get_range(min, Arg) when is_number(Arg) ->
+ <<"-Infinity">>;
+get_range(min, _Arg) ->
+ <<"\"\"">>;
+get_range(max, Arg) when is_number(Arg) ->
+ <<"Infinity">>;
+get_range(max, _Arg) ->
+ <<"\u0x10FFFF">>.
+
+
+field_exists_query(Path) ->
+ % We specify two here for :* and .* so that we don't incorrectly
+ % match a path foo.name against foo.name_first (if were to just
+ % appened * isntead).
+ Parts = [
+ % We need to remove the period from the path list to indicate that it is
+ % a path separator. We escape the colon because it is not used as a
+ % separator and we escape colons in field names.
+ {op_fieldname, {[path_str(Path), ":"], "*"}},
+ {op_fieldname, {[path_str(Path)], ".*"}}
+ ],
+ {op_or, Parts}.
+
+
+field_exists_query(Path, Type) ->
+ {op_fieldname, {[path_str(Path), ":"], Type}}.
+
+
+path_str(Path) ->
+ path_str(Path, []).
+
+
+path_str([], Acc) ->
+ Acc;
+path_str([Part], Acc) ->
+ % No reverse because Path is backwards
+ % during recursion of convert.
+ [Part | Acc];
+path_str([Part | Rest], Acc) ->
+ case Part of
+ % do not append a period if Part is blank
+ <<>> ->
+ path_str(Rest, [Acc]);
+ _ ->
+ path_str(Rest, [<<".">>, Part | Acc])
+ end.
+
+
+type_str(Value) when is_number(Value) ->
+ <<"number">>;
+type_str(Value) when is_boolean(Value) ->
+ <<"boolean">>;
+type_str(Value) when is_binary(Value) ->
+ <<"string">>;
+type_str(null) ->
+ <<"null">>.
+
+
+value_str(Value) when is_binary(Value) ->
+ case mango_util:is_number_string(Value) of
+ true ->
+ <<"\"", Value/binary, "\"">>;
+ false ->
+ mango_util:lucene_escape_query_value(Value)
+ end;
+value_str(Value) when is_integer(Value) ->
+ list_to_binary(integer_to_list(Value));
+value_str(Value) when is_float(Value) ->
+ list_to_binary(float_to_list(Value));
+value_str(true) ->
+ <<"true">>;
+value_str(false) ->
+ <<"false">>;
+value_str(null) ->
+ <<"true">>.
+
+
+append_sort_type(RawSortField, Selector) ->
+ EncodeField = mango_util:lucene_escape_user(RawSortField),
+ String = mango_util:has_suffix(EncodeField, <<"_3astring">>),
+ Number = mango_util:has_suffix(EncodeField, <<"_3anumber">>),
+ case {String, Number} of
+ {true, _} ->
+ <<EncodeField/binary, "<string>">>;
+ {_, true} ->
+ <<EncodeField/binary, "<number>">>;
+ _ ->
+ Type = get_sort_type(RawSortField, Selector),
+ <<EncodeField/binary, Type/binary>>
+ end.
+
+
+get_sort_type(Field, Selector) ->
+ Types = get_sort_types(Field, Selector, []),
+ case lists:usort(Types) of
+ [str] -> <<"_3astring<string>">>;
+ [num] -> <<"_3anumber<number>">>;
+ _ -> ?MANGO_ERROR({text_sort_error, Field})
+ end.
+
+
+get_sort_types(Field, {[{Field, {[{<<"$", _/binary>>, Cond}]}}]}, Acc)
+ when is_binary(Cond) ->
+ [str | Acc];
+
+get_sort_types(Field, {[{Field, {[{<<"$", _/binary>>, Cond}]}}]}, Acc)
+ when is_number(Cond) ->
+ [num | Acc];
+
+get_sort_types(Field, {[{_, Cond}]}, Acc) when is_list(Cond) ->
+ lists:foldl(fun(Arg, InnerAcc) ->
+ get_sort_types(Field, Arg, InnerAcc)
+ end, Acc, Cond);
+
+get_sort_types(Field, {[{_, Cond}]}, Acc) when is_tuple(Cond)->
+ get_sort_types(Field, Cond, Acc);
+
+get_sort_types(_Field, _, Acc) ->
+ Acc.
+
+
+replace_array_indexes([], NewPartsAcc, HasIntAcc) ->
+ {NewPartsAcc, HasIntAcc};
+replace_array_indexes([Part | Rest], NewPartsAcc, HasIntAcc) ->
+ {NewPart, HasInt} = try
+ _ = list_to_integer(binary_to_list(Part)),
+ {<<"[]">>, true}
+ catch _:_ ->
+ {Part, false}
+ end,
+ replace_array_indexes(Rest, [NewPart | NewPartsAcc],
+ HasInt or HasIntAcc).
diff --git a/src/mango/src/mango_sort.erl b/src/mango/src/mango_sort.erl
new file mode 100644
index 000000000..17249c297
--- /dev/null
+++ b/src/mango/src/mango_sort.erl
@@ -0,0 +1,75 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(mango_sort).
+
+-export([
+ new/1,
+ to_json/1,
+ fields/1,
+ directions/1
+]).
+
+
+-include("mango.hrl").
+
+
+new(Fields) when is_list(Fields) ->
+ Sort = {[sort_field(Field) || Field <- Fields]},
+ validate(Sort),
+ {ok, Sort};
+new(Else) ->
+ ?MANGO_ERROR({invalid_sort_json, Else}).
+
+
+to_json({Fields}) ->
+ to_json(Fields);
+to_json([]) ->
+ [];
+to_json([{Name, Dir} | Rest]) ->
+ [{[{Name, Dir}]} | to_json(Rest)].
+
+
+fields({Props}) ->
+ [Name || {Name, _Dir} <- Props].
+
+
+directions({Props}) ->
+ [Dir || {_Name, Dir} <- Props].
+
+
+sort_field(<<"">>) ->
+ ?MANGO_ERROR({invalid_sort_field, <<"">>});
+sort_field(Field) when is_binary(Field) ->
+ {Field, <<"asc">>};
+sort_field({[{Name, <<"asc">>}]}) when is_binary(Name) ->
+ {Name, <<"asc">>};
+sort_field({[{Name, <<"desc">>}]}) when is_binary(Name) ->
+ {Name, <<"desc">>};
+sort_field({Name, BadDir}) when is_binary(Name) ->
+ ?MANGO_ERROR({invalid_sort_dir, BadDir});
+sort_field(Else) ->
+ ?MANGO_ERROR({invalid_sort_field, Else}).
+
+
+validate({Props}) ->
+ % Assert each field is in the same direction
+ % until we support mixed direction sorts.
+ Dirs = [D || {_, D} <- Props],
+ case lists:usort(Dirs) of
+ [] ->
+ ok;
+ [_] ->
+ ok;
+ _ ->
+ ?MANGO_ERROR({unsupported, mixed_sort})
+ end.
diff --git a/src/mango/src/mango_sup.erl b/src/mango/src/mango_sup.erl
new file mode 100644
index 000000000..b0dedf125
--- /dev/null
+++ b/src/mango/src/mango_sup.erl
@@ -0,0 +1,24 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(mango_sup).
+-behaviour(supervisor).
+-export([init/1]).
+
+-export([start_link/1]).
+
+
+start_link(Args) ->
+ supervisor:start_link({local,?MODULE}, ?MODULE, Args).
+
+init([]) ->
+ {ok, {{one_for_one, 3, 10}, couch_epi:register_service(mango_epi, [])}}.
diff --git a/src/mango/src/mango_util.erl b/src/mango/src/mango_util.erl
new file mode 100644
index 000000000..c3513dced
--- /dev/null
+++ b/src/mango/src/mango_util.erl
@@ -0,0 +1,423 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(mango_util).
+
+
+-export([
+ open_doc/2,
+ open_ddocs/1,
+ load_ddoc/2,
+
+ defer/3,
+ do_defer/3,
+
+ assert_ejson/1,
+
+ to_lower/1,
+
+ enc_dbname/1,
+ dec_dbname/1,
+
+ enc_hex/1,
+ dec_hex/1,
+
+ lucene_escape_field/1,
+ lucene_escape_query_value/1,
+ lucene_escape_user/1,
+ is_number_string/1,
+
+ has_suffix/2,
+
+ join/2,
+
+ parse_field/1,
+
+ cached_re/2
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include("mango.hrl").
+
+
+-define(DIGITS, "(\\p{N}+)").
+-define(HEXDIGITS, "([0-9a-fA-F]+)").
+-define(EXP, "[eE][+-]?" ++ ?DIGITS).
+-define(NUMSTRING,
+"[\\x00-\\x20]*" ++ "[+-]?(" ++ "NaN|"
+ ++ "Infinity|" ++ "((("
+ ++ ?DIGITS
+ ++ "(\\.)?("
+ ++ ?DIGITS
+ ++ "?)("
+ ++ ?EXP
+ ++ ")?)|"
+ ++ "(\\.("
+ ++ ?DIGITS
+ ++ ")("
+ ++ ?EXP
+ ++ ")?)|"
+ ++ "(("
+ ++ "(0[xX]"
+ ++ ?HEXDIGITS
+ ++ "(\\.)?)|"
+ ++ "(0[xX]"
+ ++ ?HEXDIGITS
+ ++ "?(\\.)"
+ ++ ?HEXDIGITS
+ ++ ")"
+ ++ ")[pP][+-]?" ++ ?DIGITS ++ "))" ++ "[fFdD]?))" ++ "[\\x00-\\x20]*").
+
+
+open_doc(Db, DocId) ->
+ open_doc(Db, DocId, [deleted, ejson_body]).
+
+
+open_doc(Db, DocId, Options) ->
+ case mango_util:defer(fabric, open_doc, [Db, DocId, Options]) of
+ {ok, Doc} ->
+ {ok, Doc};
+ {not_found, _} ->
+ not_found;
+ _ ->
+ ?MANGO_ERROR({error_loading_doc, DocId})
+ end.
+
+
+open_ddocs(Db) ->
+ case mango_util:defer(fabric, design_docs, [Db]) of
+ {ok, Docs} ->
+ {ok, Docs};
+ _ ->
+ ?MANGO_ERROR(error_loading_ddocs)
+ end.
+
+
+load_ddoc(Db, DDocId) ->
+ case open_doc(Db, DDocId, [deleted, ejson_body]) of
+ {ok, Doc} ->
+ {ok, check_lang(Doc)};
+ not_found ->
+ Body = {[
+ {<<"language">>, <<"query">>}
+ ]},
+ {ok, #doc{id = DDocId, body = Body}}
+ end.
+
+
+defer(Mod, Fun, Args) ->
+ {Pid, Ref} = erlang:spawn_monitor(?MODULE, do_defer, [Mod, Fun, Args]),
+ receive
+ {'DOWN', Ref, process, Pid, {mango_defer_ok, Value}} ->
+ Value;
+ {'DOWN', Ref, process, Pid, {mango_defer_throw, Value}} ->
+ erlang:throw(Value);
+ {'DOWN', Ref, process, Pid, {mango_defer_error, Value}} ->
+ erlang:error(Value);
+ {'DOWN', Ref, process, Pid, {mango_defer_exit, Value}} ->
+ erlang:exit(Value)
+ end.
+
+
+do_defer(Mod, Fun, Args) ->
+ try erlang:apply(Mod, Fun, Args) of
+ Resp ->
+ erlang:exit({mango_defer_ok, Resp})
+ catch
+ throw:Error ->
+ Stack = erlang:get_stacktrace(),
+ couch_log:error("Defered error: ~w~n ~p", [{throw, Error}, Stack]),
+ erlang:exit({mango_defer_throw, Error});
+ error:Error ->
+ Stack = erlang:get_stacktrace(),
+ couch_log:error("Defered error: ~w~n ~p", [{error, Error}, Stack]),
+ erlang:exit({mango_defer_error, Error});
+ exit:Error ->
+ Stack = erlang:get_stacktrace(),
+ couch_log:error("Defered error: ~w~n ~p", [{exit, Error}, Stack]),
+ erlang:exit({mango_defer_exit, Error})
+ end.
+
+
+assert_ejson({Props}) ->
+ assert_ejson_obj(Props);
+assert_ejson(Vals) when is_list(Vals) ->
+ assert_ejson_arr(Vals);
+assert_ejson(null) ->
+ true;
+assert_ejson(true) ->
+ true;
+assert_ejson(false) ->
+ true;
+assert_ejson(String) when is_binary(String) ->
+ true;
+assert_ejson(Number) when is_number(Number) ->
+ true;
+assert_ejson(_Else) ->
+ false.
+
+
+assert_ejson_obj([]) ->
+ true;
+assert_ejson_obj([{Key, Val} | Rest]) when is_binary(Key) ->
+ case assert_ejson(Val) of
+ true ->
+ assert_ejson_obj(Rest);
+ false ->
+ false
+ end;
+assert_ejson_obj(_Else) ->
+ false.
+
+
+assert_ejson_arr([]) ->
+ true;
+assert_ejson_arr([Val | Rest]) ->
+ case assert_ejson(Val) of
+ true ->
+ assert_ejson_arr(Rest);
+ false ->
+ false
+ end.
+
+
+check_lang(#doc{id = Id, deleted = true}) ->
+ Body = {[
+ {<<"language">>, <<"query">>}
+ ]},
+ #doc{id = Id, body = Body};
+check_lang(#doc{body = {Props}} = Doc) ->
+ case lists:keyfind(<<"language">>, 1, Props) of
+ {<<"language">>, <<"query">>} ->
+ Doc;
+ Else ->
+ ?MANGO_ERROR({invalid_ddoc_lang, Else})
+ end.
+
+
+to_lower(Key) when is_binary(Key) ->
+ KStr = binary_to_list(Key),
+ KLower = string:to_lower(KStr),
+ list_to_binary(KLower).
+
+
+enc_dbname(<<>>) ->
+ <<>>;
+enc_dbname(<<A:8/integer, Rest/binary>>) ->
+ Bytes = enc_db_byte(A),
+ Tail = enc_dbname(Rest),
+ <<Bytes/binary, Tail/binary>>.
+
+
+enc_db_byte(N) when N >= $a, N =< $z -> <<N>>;
+enc_db_byte(N) when N >= $0, N =< $9 -> <<N>>;
+enc_db_byte(N) when N == $/; N == $_; N == $- -> <<N>>;
+enc_db_byte(N) ->
+ H = enc_hex_byte(N div 16),
+ L = enc_hex_byte(N rem 16),
+ <<$$, H:8/integer, L:8/integer>>.
+
+
+dec_dbname(<<>>) ->
+ <<>>;
+dec_dbname(<<$$, _:8/integer>>) ->
+ throw(invalid_dbname_encoding);
+dec_dbname(<<$$, H:8/integer, L:8/integer, Rest/binary>>) ->
+ Byte = (dec_hex_byte(H) bsl 4) bor dec_hex_byte(L),
+ Tail = dec_dbname(Rest),
+ <<Byte:8/integer, Tail/binary>>;
+dec_dbname(<<N:8/integer, Rest/binary>>) ->
+ Tail = dec_dbname(Rest),
+ <<N:8/integer, Tail/binary>>.
+
+
+enc_hex(<<>>) ->
+ <<>>;
+enc_hex(<<V:8/integer, Rest/binary>>) ->
+ H = enc_hex_byte(V div 16),
+ L = enc_hex_byte(V rem 16),
+ Tail = enc_hex(Rest),
+ <<H:8/integer, L:8/integer, Tail/binary>>.
+
+
+enc_hex_byte(N) when N >= 0, N < 10 -> $0 + N;
+enc_hex_byte(N) when N >= 10, N < 16 -> $a + (N - 10);
+enc_hex_byte(N) -> throw({invalid_hex_value, N}).
+
+
+dec_hex(<<>>) ->
+ <<>>;
+dec_hex(<<_:8/integer>>) ->
+ throw(invalid_hex_string);
+dec_hex(<<H:8/integer, L:8/integer, Rest/binary>>) ->
+ Byte = (dec_hex_byte(H) bsl 4) bor dec_hex_byte(L),
+ Tail = dec_hex(Rest),
+ <<Byte:8/integer, Tail/binary>>.
+
+
+dec_hex_byte(N) when N >= $0, N =< $9 -> (N - $0);
+dec_hex_byte(N) when N >= $a, N =< $f -> (N - $a) + 10;
+dec_hex_byte(N) when N >= $A, N =< $F -> (N - $A) + 10;
+dec_hex_byte(N) -> throw({invalid_hex_character, N}).
+
+
+
+lucene_escape_field(Bin) when is_binary(Bin) ->
+ Str = binary_to_list(Bin),
+ Enc = lucene_escape_field(Str),
+ iolist_to_binary(Enc);
+lucene_escape_field([H | T]) when is_number(H), H >= 0, H =< 255 ->
+ if
+ H >= $a, $z >= H ->
+ [H | lucene_escape_field(T)];
+ H >= $A, $Z >= H ->
+ [H | lucene_escape_field(T)];
+ H >= $0, $9 >= H ->
+ [H | lucene_escape_field(T)];
+ true ->
+ Hi = enc_hex_byte(H div 16),
+ Lo = enc_hex_byte(H rem 16),
+ [$_, Hi, Lo | lucene_escape_field(T)]
+ end;
+lucene_escape_field([]) ->
+ [].
+
+
+lucene_escape_query_value(IoList) when is_list(IoList) ->
+ lucene_escape_query_value(iolist_to_binary(IoList));
+lucene_escape_query_value(Bin) when is_binary(Bin) ->
+ IoList = lucene_escape_qv(Bin),
+ iolist_to_binary(IoList).
+
+
+% This escapes the special Lucene query characters
+% listed below as well as any whitespace.
+%
+% + - && || ! ( ) { } [ ] ^ ~ * ? : \ " /
+%
+
+lucene_escape_qv(<<>>) -> [];
+lucene_escape_qv(<<"&&", Rest/binary>>) ->
+ ["\\&&" | lucene_escape_qv(Rest)];
+lucene_escape_qv(<<"||", Rest/binary>>) ->
+ ["\\||" | lucene_escape_qv(Rest)];
+lucene_escape_qv(<<C, Rest/binary>>) ->
+ NeedsEscape = "+-(){}[]!^~*?:/\\\" \t\r\n",
+ Out = case lists:member(C, NeedsEscape) of
+ true -> ["\\", C];
+ false -> [C]
+ end,
+ Out ++ lucene_escape_qv(Rest).
+
+
+lucene_escape_user(Field) ->
+ {ok, Path} = parse_field(Field),
+ Escaped = [mango_util:lucene_escape_field(P) || P <- Path],
+ iolist_to_binary(join(".", Escaped)).
+
+
+has_suffix(Bin, Suffix) when is_binary(Bin), is_binary(Suffix) ->
+ SBin = size(Bin),
+ SSuffix = size(Suffix),
+ if SBin < SSuffix -> false; true ->
+ PSize = SBin - SSuffix,
+ case Bin of
+ <<_:PSize/binary, Suffix/binary>> ->
+ true;
+ _ ->
+ false
+ end
+ end.
+
+
+join(_Sep, [Item]) ->
+ [Item];
+join(Sep, [Item | Rest]) ->
+ [Item, Sep | join(Sep, Rest)].
+
+
+is_number_string(Value) when is_binary(Value) ->
+ is_number_string(binary_to_list(Value));
+is_number_string(Value) when is_list(Value)->
+ MP = cached_re(mango_numstring_re, ?NUMSTRING),
+ case re:run(Value, MP) of
+ nomatch ->
+ false;
+ _ ->
+ true
+ end.
+
+
+cached_re(Name, RE) ->
+ case mochiglobal:get(Name) of
+ undefined ->
+ {ok, MP} = re:compile(RE),
+ ok = mochiglobal:put(Name, MP),
+ MP;
+ MP ->
+ MP
+ end.
+
+
+parse_field(Field) ->
+ case binary:match(Field, <<"\\">>, []) of
+ nomatch ->
+ % Fast path, no regex required
+ {ok, check_non_empty(Field, binary:split(Field, <<".">>, [global]))};
+ _ ->
+ parse_field_slow(Field)
+ end.
+
+parse_field_slow(Field) ->
+ Path = lists:map(fun
+ (P) when P =:= <<>> ->
+ ?MANGO_ERROR({invalid_field_name, Field});
+ (P) ->
+ re:replace(P, <<"\\\\">>, <<>>, [global, {return, binary}])
+ end, re:split(Field, <<"(?<!\\\\)\\.">>)),
+ {ok, Path}.
+
+check_non_empty(Field, Parts) ->
+ case lists:member(<<>>, Parts) of
+ true ->
+ ?MANGO_ERROR({invalid_field_name, Field});
+ false ->
+ Parts
+ end.
+
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+parse_field_test() ->
+ ?assertEqual({ok, [<<"ab">>]}, parse_field(<<"ab">>)),
+ ?assertEqual({ok, [<<"a">>, <<"b">>]}, parse_field(<<"a.b">>)),
+ ?assertEqual({ok, [<<"a.b">>]}, parse_field(<<"a\\.b">>)),
+ ?assertEqual({ok, [<<"a">>, <<"b">>, <<"c">>]}, parse_field(<<"a.b.c">>)),
+ ?assertEqual({ok, [<<"a">>, <<"b.c">>]}, parse_field(<<"a.b\\.c">>)),
+ Exception = {mango_error, ?MODULE, {invalid_field_name, <<"a..b">>}},
+ ?assertThrow(Exception, parse_field(<<"a..b">>)).
+
+is_number_string_test() ->
+ ?assert(is_number_string("0")),
+ ?assert(is_number_string("1")),
+ ?assert(is_number_string("1.0")),
+ ?assert(is_number_string("1.0E10")),
+ ?assert(is_number_string("0d")),
+ ?assert(is_number_string("-1")),
+ ?assert(is_number_string("-1.0")),
+ ?assertNot(is_number_string("hello")),
+ ?assertNot(is_number_string("")),
+ ?assertMatch({match, _}, re:run("1.0", mochiglobal:get(mango_numstring_re))).
+
+-endif.
diff --git a/src/mango/test/01-index-crud-test.py b/src/mango/test/01-index-crud-test.py
new file mode 100644
index 000000000..342c94f9b
--- /dev/null
+++ b/src/mango/test/01-index-crud-test.py
@@ -0,0 +1,302 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+import random
+
+import mango
+import unittest
+
+class IndexCrudTests(mango.DbPerClass):
+ def test_bad_fields(self):
+ bad_fields = [
+ None,
+ True,
+ False,
+ "bing",
+ 2.0,
+ {"foo": "bar"},
+ [{"foo": 2}],
+ [{"foo": "asc", "bar": "desc"}],
+ [{"foo": "asc"}, {"bar": "desc"}],
+ [""]
+ ]
+ for fields in bad_fields:
+ try:
+ self.db.create_index(fields)
+ except Exception, e:
+ assert e.response.status_code == 400
+ else:
+ raise AssertionError("bad create index")
+
+ def test_bad_types(self):
+ bad_types = [
+ None,
+ True,
+ False,
+ 1.5,
+ "foo", # Future support
+ "geo", # Future support
+ {"foo": "bar"},
+ ["baz", 3.0]
+ ]
+ for bt in bad_types:
+ try:
+ self.db.create_index(["foo"], idx_type=bt)
+ except Exception, e:
+ assert e.response.status_code == 400, (bt, e.response.status_code)
+ else:
+ raise AssertionError("bad create index")
+
+ def test_bad_names(self):
+ bad_names = [
+ True,
+ False,
+ 1.5,
+ {"foo": "bar"},
+ [None, False]
+ ]
+ for bn in bad_names:
+ try:
+ self.db.create_index(["foo"], name=bn)
+ except Exception, e:
+ assert e.response.status_code == 400
+ else:
+ raise AssertionError("bad create index")
+ try:
+ self.db.create_index(["foo"], ddoc=bn)
+ except Exception, e:
+ assert e.response.status_code == 400
+ else:
+ raise AssertionError("bad create index")
+
+ def test_create_idx_01(self):
+ fields = ["foo", "bar"]
+ ret = self.db.create_index(fields, name="idx_01")
+ assert ret is True
+ for idx in self.db.list_indexes():
+ if idx["name"] != "idx_01":
+ continue
+ assert idx["def"]["fields"] == [{"foo": "asc"}, {"bar": "asc"}]
+ return
+ raise AssertionError("index not created")
+
+ def test_create_idx_01_exists(self):
+ fields = ["foo", "bar"]
+ ret = self.db.create_index(fields, name="idx_01")
+ assert ret is False
+
+ def test_create_idx_02(self):
+ fields = ["baz", "foo"]
+ ret = self.db.create_index(fields, name="idx_02")
+ assert ret is True
+ for idx in self.db.list_indexes():
+ if idx["name"] != "idx_02":
+ continue
+ assert idx["def"]["fields"] == [{"baz": "asc"}, {"foo": "asc"}]
+ return
+ raise AssertionError("index not created")
+
+ def test_read_idx_doc(self):
+ for idx in self.db.list_indexes():
+ if idx["type"] == "special":
+ continue
+ ddocid = idx["ddoc"]
+ doc = self.db.open_doc(ddocid)
+ assert doc["_id"] == ddocid
+ info = self.db.ddoc_info(ddocid)
+ assert info["name"] == ddocid.split('_design/')[-1]
+
+ def test_delete_idx_escaped(self):
+ pre_indexes = self.db.list_indexes()
+ ret = self.db.create_index(["bing"], name="idx_del_1")
+ assert ret is True
+ for idx in self.db.list_indexes():
+ if idx["name"] != "idx_del_1":
+ continue
+ assert idx["def"]["fields"] == [{"bing": "asc"}]
+ self.db.delete_index(idx["ddoc"].replace("/", "%2F"), idx["name"])
+ post_indexes = self.db.list_indexes()
+ assert pre_indexes == post_indexes
+
+ def test_delete_idx_unescaped(self):
+ pre_indexes = self.db.list_indexes()
+ ret = self.db.create_index(["bing"], name="idx_del_2")
+ assert ret is True
+ for idx in self.db.list_indexes():
+ if idx["name"] != "idx_del_2":
+ continue
+ assert idx["def"]["fields"] == [{"bing": "asc"}]
+ self.db.delete_index(idx["ddoc"], idx["name"])
+ post_indexes = self.db.list_indexes()
+ assert pre_indexes == post_indexes
+
+ def test_delete_idx_no_design(self):
+ pre_indexes = self.db.list_indexes()
+ ret = self.db.create_index(["bing"], name="idx_del_3")
+ assert ret is True
+ for idx in self.db.list_indexes():
+ if idx["name"] != "idx_del_3":
+ continue
+ assert idx["def"]["fields"] == [{"bing": "asc"}]
+ self.db.delete_index(idx["ddoc"].split("/")[-1], idx["name"])
+ post_indexes = self.db.list_indexes()
+ assert pre_indexes == post_indexes
+
+ def test_bulk_delete(self):
+ fields = ["field1"]
+ ret = self.db.create_index(fields, name="idx_01")
+ assert ret is True
+
+ fields = ["field2"]
+ ret = self.db.create_index(fields, name="idx_02")
+ assert ret is True
+
+ fields = ["field3"]
+ ret = self.db.create_index(fields, name="idx_03")
+ assert ret is True
+
+ docids = []
+
+ for idx in self.db.list_indexes():
+ if idx["ddoc"] is not None:
+ docids.append(idx["ddoc"])
+
+ docids.append("_design/this_is_not_an_index_name")
+
+ ret = self.db.bulk_delete(docids)
+
+ assert ret["fail"][0]["id"] == "_design/this_is_not_an_index_name"
+ assert len(ret["success"]) == 3
+
+ for idx in self.db.list_indexes():
+ assert idx["type"] != "json"
+ assert idx["type"] != "text"
+
+ def test_recreate_index(self):
+ pre_indexes = self.db.list_indexes()
+ for i in range(5):
+ ret = self.db.create_index(["bing"], name="idx_recreate")
+ assert ret is True
+ for idx in self.db.list_indexes():
+ if idx["name"] != "idx_recreate":
+ continue
+ assert idx["def"]["fields"] == [{"bing": "asc"}]
+ self.db.delete_index(idx["ddoc"], idx["name"])
+ break
+ post_indexes = self.db.list_indexes()
+ assert pre_indexes == post_indexes
+
+ def test_delete_misisng(self):
+ # Missing design doc
+ try:
+ self.db.delete_index("this_is_not_a_design_doc_id", "foo")
+ except Exception, e:
+ assert e.response.status_code == 404
+ else:
+ raise AssertionError("bad index delete")
+
+ # Missing view name
+ indexes = self.db.list_indexes()
+ not_special = [idx for idx in indexes if idx["type"] != "special"]
+ idx = random.choice(not_special)
+ ddocid = idx["ddoc"].split("/")[-1]
+ try:
+ self.db.delete_index(ddocid, "this_is_not_an_index_name")
+ except Exception, e:
+ assert e.response.status_code == 404
+ else:
+ raise AssertionError("bad index delete")
+
+ # Bad view type
+ try:
+ self.db.delete_index(ddocid, idx["name"], idx_type="not_a_real_type")
+ except Exception, e:
+ assert e.response.status_code == 404
+ else:
+ raise AssertionError("bad index delete")
+
+ @unittest.skipUnless(mango.has_text_service(), "requires text service")
+ def test_create_text_idx(self):
+ fields = [
+ {"name":"stringidx", "type" : "string"},
+ {"name":"booleanidx", "type": "boolean"}
+ ]
+ ret = self.db.create_text_index(fields=fields, name="text_idx_01")
+ assert ret is True
+ for idx in self.db.list_indexes():
+ if idx["name"] != "text_idx_01":
+ continue
+ print idx["def"]
+ assert idx["def"]["fields"] == [
+ {"stringidx": "string"},
+ {"booleanidx": "boolean"}
+ ]
+ return
+ raise AssertionError("index not created")
+
+ @unittest.skipUnless(mango.has_text_service(), "requires text service")
+ def test_create_bad_text_idx(self):
+ bad_fields = [
+ True,
+ False,
+ "bing",
+ 2.0,
+ ["foo", "bar"],
+ [{"name": "foo2"}],
+ [{"name": "foo3", "type": "garbage"}],
+ [{"type": "number"}],
+ [{"name": "age", "type": "number"} , {"name": "bad"}],
+ [{"name": "age", "type": "number"} , "bla"],
+ [{"name": "", "type": "number"} , "bla"]
+ ]
+ for fields in bad_fields:
+ try:
+ self.db.create_text_index(fields=fields)
+ except Exception, e:
+ assert e.response.status_code == 400
+ else:
+ raise AssertionError("bad create text index")
+
+ def test_limit_skip_index(self):
+ fields = ["field1"]
+ ret = self.db.create_index(fields, name="idx_01")
+ assert ret is True
+
+ fields = ["field2"]
+ ret = self.db.create_index(fields, name="idx_02")
+ assert ret is True
+
+ fields = ["field3"]
+ ret = self.db.create_index(fields, name="idx_03")
+ assert ret is True
+
+ skip_add = 0
+
+ if mango.has_text_service():
+ skip_add = 1
+
+ assert len(self.db.list_indexes(limit=2)) == 2
+ assert len(self.db.list_indexes(limit=5,skip=4)) == 2 + skip_add
+ assert len(self.db.list_indexes(skip=5)) == 1 + skip_add
+ assert len(self.db.list_indexes(skip=6)) == 0 + skip_add
+ assert len(self.db.list_indexes(skip=100)) == 0
+ assert len(self.db.list_indexes(limit=10000000)) == 6 + skip_add
+
+ try:
+ self.db.list_indexes(skip=-1)
+ except Exception, e:
+ assert e.response.status_code == 500
+
+ try:
+ self.db.list_indexes(limit=0)
+ except Exception, e:
+ assert e.response.status_code == 500
diff --git a/src/mango/test/02-basic-find-test.py b/src/mango/test/02-basic-find-test.py
new file mode 100644
index 000000000..e634ce9fe
--- /dev/null
+++ b/src/mango/test/02-basic-find-test.py
@@ -0,0 +1,266 @@
+# -*- coding: latin-1 -*-
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+
+import mango
+
+
+class BasicFindTests(mango.UserDocsTests):
+
+ def test_bad_selector(self):
+ bad_selectors = [
+ None,
+ True,
+ False,
+ 1.0,
+ "foobarbaz",
+ {"foo":{"$not_an_op": 2}},
+ {"$gt":2},
+ [None, "bing"]
+ ]
+ for bs in bad_selectors:
+ try:
+ self.db.find(bs)
+ except Exception, e:
+ assert e.response.status_code == 400
+ else:
+ raise AssertionError("bad find")
+
+ def test_bad_limit(self):
+ bad_limits = [
+ None,
+ True,
+ False,
+ -1,
+ 1.2,
+ "no limit!",
+ {"foo": "bar"},
+ [2]
+ ],
+ for bl in bad_limits:
+ try:
+ self.db.find({"int":{"$gt":2}}, limit=bl)
+ except Exception, e:
+ assert e.response.status_code == 400
+ else:
+ raise AssertionError("bad find")
+
+ def test_bad_skip(self):
+ bad_skips = [
+ None,
+ True,
+ False,
+ -3,
+ 1.2,
+ "no limit!",
+ {"foo": "bar"},
+ [2]
+ ],
+ for bs in bad_skips:
+ try:
+ self.db.find({"int":{"$gt":2}}, skip=bs)
+ except Exception, e:
+ assert e.response.status_code == 400
+ else:
+ raise AssertionError("bad find")
+
+ def test_bad_sort(self):
+ bad_sorts = [
+ None,
+ True,
+ False,
+ 1.2,
+ "no limit!",
+ {"foo": "bar"},
+ [2],
+ [{"foo":"asc", "bar": "asc"}],
+ [{"foo":"asc"}, {"bar":"desc"}],
+ ],
+ for bs in bad_sorts:
+ try:
+ self.db.find({"int":{"$gt":2}}, sort=bs)
+ except Exception, e:
+ assert e.response.status_code == 400
+ else:
+ raise AssertionError("bad find")
+
+ def test_bad_fields(self):
+ bad_fields = [
+ None,
+ True,
+ False,
+ 1.2,
+ "no limit!",
+ {"foo": "bar"},
+ [2],
+ [[]],
+ ["foo", 2.0],
+ ],
+ for bf in bad_fields:
+ try:
+ self.db.find({"int":{"$gt":2}}, fields=bf)
+ except Exception, e:
+ assert e.response.status_code == 400
+ else:
+ raise AssertionError("bad find")
+
+ def test_bad_r(self):
+ bad_rs = [
+ None,
+ True,
+ False,
+ 1.2,
+ "no limit!",
+ {"foo": "bar"},
+ [2],
+ ],
+ for br in bad_rs:
+ try:
+ self.db.find({"int":{"$gt":2}}, r=br)
+ except Exception, e:
+ assert e.response.status_code == 400
+ else:
+ raise AssertionError("bad find")
+
+ def test_bad_conflicts(self):
+ bad_conflicts = [
+ None,
+ 1.2,
+ "no limit!",
+ {"foo": "bar"},
+ [2],
+ ],
+ for bc in bad_conflicts:
+ try:
+ self.db.find({"int":{"$gt":2}}, conflicts=bc)
+ except Exception, e:
+ assert e.response.status_code == 400
+ else:
+ raise AssertionError("bad find")
+
+ def test_simple_find(self):
+ docs = self.db.find({"age": {"$lt": 35}})
+ assert len(docs) == 3
+ assert docs[0]["user_id"] == 9
+ assert docs[1]["user_id"] == 1
+ assert docs[2]["user_id"] == 7
+
+ def test_multi_cond_and(self):
+ docs = self.db.find({"manager": True, "location.city": "Longbranch"})
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == 7
+
+ def test_multi_cond_or(self):
+ docs = self.db.find({
+ "$and":[
+ {"age":{"$gte": 75}},
+ {"$or": [
+ {"name.first": "Mathis"},
+ {"name.first": "Whitley"}
+ ]}
+ ]
+ })
+ assert len(docs) == 2
+ assert docs[0]["user_id"] == 11
+ assert docs[1]["user_id"] == 13
+
+ def test_multi_col_idx(self):
+ docs = self.db.find({
+ "location.state": {"$and": [
+ {"$gt": "Hawaii"},
+ {"$lt": "Maine"}
+ ]},
+ "location.city": {"$lt": "Longbranch"}
+ })
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == 6
+
+ def test_missing_not_indexed(self):
+ docs = self.db.find({"favorites.3": "C"})
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == 6
+
+ docs = self.db.find({"favorites.3": None})
+ assert len(docs) == 0
+
+ docs = self.db.find({"twitter": {"$gt": None}})
+ assert len(docs) == 4
+ assert docs[0]["user_id"] == 1
+ assert docs[1]["user_id"] == 4
+ assert docs[2]["user_id"] == 0
+ assert docs[3]["user_id"] == 13
+
+ def test_limit(self):
+ docs = self.db.find({"age": {"$gt": 0}})
+ assert len(docs) == 15
+ for l in [0, 1, 5, 14]:
+ docs = self.db.find({"age": {"$gt": 0}}, limit=l)
+ assert len(docs) == l
+
+ def test_skip(self):
+ docs = self.db.find({"age": {"$gt": 0}})
+ assert len(docs) == 15
+ for s in [0, 1, 5, 14]:
+ docs = self.db.find({"age": {"$gt": 0}}, skip=s)
+ assert len(docs) == (15 - s)
+
+ def test_sort(self):
+ docs1 = self.db.find({"age": {"$gt": 0}}, sort=[{"age":"asc"}])
+ docs2 = list(sorted(docs1, key=lambda d: d["age"]))
+ assert docs1 is not docs2 and docs1 == docs2
+
+ docs1 = self.db.find({"age": {"$gt": 0}}, sort=[{"age":"desc"}])
+ docs2 = list(reversed(sorted(docs1, key=lambda d: d["age"])))
+ assert docs1 is not docs2 and docs1 == docs2
+
+ def test_fields(self):
+ selector = {"age": {"$gt": 0}}
+ docs = self.db.find(selector, fields=["user_id", "location.address"])
+ for d in docs:
+ assert sorted(d.keys()) == ["location", "user_id"]
+ assert sorted(d["location"].keys()) == ["address"]
+
+ def test_r(self):
+ for r in [1, 2, 3]:
+ docs = self.db.find({"age": {"$gt": 0}}, r=r)
+ assert len(docs) == 15
+
+ def test_empty(self):
+ docs = self.db.find({})
+ # 15 users
+ assert len(docs) == 15
+
+ def test_empty_subsel(self):
+ docs = self.db.find({
+ "_id": {"$gt": None},
+ "location": {}
+ })
+ assert len(docs) == 0
+
+ def test_empty_subsel_match(self):
+ self.db.save_docs([{"user_id": "eo", "empty_obj": {}}])
+ docs = self.db.find({
+ "_id": {"$gt": None},
+ "empty_obj": {}
+ })
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == "eo"
+
+ def test_unsatisfiable_range(self):
+ docs = self.db.find({
+ "$and":[
+ {"age":{"$gt": 0}},
+ {"age":{"$lt": 0}}
+ ]
+ })
+ assert len(docs) == 0
diff --git a/src/mango/test/03-operator-test.py b/src/mango/test/03-operator-test.py
new file mode 100644
index 000000000..56c286227
--- /dev/null
+++ b/src/mango/test/03-operator-test.py
@@ -0,0 +1,143 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+import mango
+
+
+class OperatorTests(mango.UserDocsTests):
+
+ def test_all(self):
+ docs = self.db.find({
+ "manager": True,
+ "favorites": {"$all": ["Lisp", "Python"]}
+ })
+ print docs
+ assert len(docs) == 4
+ assert docs[0]["user_id"] == 2
+ assert docs[1]["user_id"] == 12
+ assert docs[2]["user_id"] == 9
+ assert docs[3]["user_id"] == 14
+
+ def test_all_non_array(self):
+ docs = self.db.find({
+ "manager": True,
+ "location": {"$all": ["Ohai"]}
+ })
+ assert len(docs) == 0
+
+ def test_elem_match(self):
+ emdocs = [
+ {
+ "user_id": "a",
+ "bang": [{
+ "foo": 1,
+ "bar": 2
+ }]
+ },
+ {
+ "user_id": "b",
+ "bang": [{
+ "foo": 2,
+ "bam": True
+ }]
+ }
+ ]
+ self.db.save_docs(emdocs, w=3)
+ docs = self.db.find({
+ "_id": {"$gt": None},
+ "bang": {"$elemMatch": {
+ "foo": {"$gte": 1},
+ "bam": True
+ }}
+ })
+ print docs
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == "b"
+
+ def test_all_match(self):
+ amdocs = [
+ {
+ "user_id": "a",
+ "bang": [
+ {
+ "foo": 1,
+ "bar": 2
+ },
+ {
+ "foo": 3,
+ "bar": 4
+ }
+ ]
+ },
+ {
+ "user_id": "b",
+ "bang": [
+ {
+ "foo": 1,
+ "bar": 2
+ },
+ {
+ "foo": 4,
+ "bar": 4
+ }
+ ]
+ }
+ ]
+ self.db.save_docs(amdocs, w=3)
+ docs = self.db.find({
+ "_id": {"$gt": None},
+ "bang": {"$allMatch": {
+ "foo": {"$mod": [2,1]},
+ "bar": {"$mod": [2,0]}
+ }}
+ })
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == "a"
+
+ def test_in_operator_array(self):
+ docs = self.db.find({
+ "manager": True,
+ "favorites": {"$in": ["Ruby", "Python"]}
+ })
+ assert len(docs) == 7
+ assert docs[0]["user_id"] == 2
+ assert docs[1]["user_id"] == 12
+
+ def test_nin_operator_array(self):
+ docs = self.db.find({
+ "manager": True,
+ "favorites": {"$nin": ["Erlang", "Python"]}
+ })
+ assert len(docs) == 4
+ for doc in docs:
+ if isinstance(doc["favorites"], list):
+ assert "Erlang" not in doc["favorites"]
+ assert "Python" not in doc["favorites"]
+
+ def test_regex(self):
+ docs = self.db.find({
+ "age": {"$gt": 40},
+ "location.state": {"$regex": "(?i)new.*"}
+ })
+ assert len(docs) == 2
+ assert docs[0]["user_id"] == 2
+ assert docs[1]["user_id"] == 10
+
+ def test_exists_false(self):
+ docs = self.db.find({
+ "age": {"$gt": 0},
+ "twitter": {"$exists": False}
+ })
+ user_ids = [2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 14]
+ assert len(docs) == len(user_ids)
+ for doc in docs:
+ assert doc["user_id"] in user_ids
diff --git a/src/mango/test/04-key-tests.py b/src/mango/test/04-key-tests.py
new file mode 100644
index 000000000..4956d4689
--- /dev/null
+++ b/src/mango/test/04-key-tests.py
@@ -0,0 +1,151 @@
+# -*- coding: latin-1 -*-
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+
+import mango
+import unittest
+
+TEST_DOCS = [
+ {
+ "type": "complex_key",
+ "title": "normal key"
+ },
+ {
+ "type": "complex_key",
+ "title": "key with dot",
+ "dot.key": "dot's value",
+ "none": {
+ "dot": "none dot's value"
+ },
+ "name.first" : "Kvothe"
+ },
+ {
+ "type": "complex_key",
+ "title": "key with peso",
+ "$key": "peso",
+ "deep": {
+ "$key": "deep peso"
+ },
+ "name": {"first" : "Master Elodin"}
+ },
+ {
+ "type": "complex_key",
+ "title": "unicode key",
+ "": "apple"
+ },
+ {
+ "title": "internal_fields_format",
+ "utf8-1[]:string" : "string",
+ "utf8-2[]:boolean[]" : True,
+ "utf8-3[]:number" : 9,
+ "utf8-3[]:null" : None
+ }
+]
+
+
+@unittest.skipUnless(mango.has_text_service(), "requires text service")
+class KeyTests(mango.DbPerClass):
+ @classmethod
+ def setUpClass(klass):
+ super(KeyTests, klass).setUpClass()
+ klass.db.save_docs(TEST_DOCS, w=3)
+ klass.db.create_index(["type"], ddoc="view")
+ if mango.has_text_service():
+ klass.db.create_text_index(ddoc="text")
+
+ def run_check(self, query, check, fields=None, indexes=None):
+ if indexes is None:
+ indexes = ["view", "text"]
+ for idx in indexes:
+ docs = self.db.find(query, fields=fields, use_index=idx)
+ check(docs)
+
+ def test_dot_key(self):
+ query = {"type": "complex_key"}
+ fields = ["title", "dot\\.key", "none.dot"]
+ def check(docs):
+ assert len(docs) == 4
+ assert docs[1].has_key("dot.key")
+ assert docs[1]["dot.key"] == "dot's value"
+ assert docs[1].has_key("none")
+ assert docs[1]["none"]["dot"] == "none dot's value"
+ self.run_check(query, check, fields=fields)
+
+ def test_peso_key(self):
+ query = {"type": "complex_key"}
+ fields = ["title", "$key", "deep.$key"]
+ def check(docs):
+ assert len(docs) == 4
+ assert docs[2].has_key("$key")
+ assert docs[2]["$key"] == "peso"
+ assert docs[2].has_key("deep")
+ assert docs[2]["deep"]["$key"] == "deep peso"
+ self.run_check(query, check, fields=fields)
+
+ def test_unicode_in_fieldname(self):
+ query = {"type": "complex_key"}
+ fields = ["title", ""]
+ def check(docs):
+ assert len(docs) == 4
+ # note:  == \uf8ff
+ assert docs[3].has_key(u'\uf8ff')
+ assert docs[3][u'\uf8ff'] == "apple"
+ self.run_check(query, check, fields=fields)
+
+ # The rest of these tests are only run against the text
+ # indexes because view indexes don't have to worry about
+ # field *name* escaping in the index.
+
+ def test_unicode_in_selector_field(self):
+ query = {"" : "apple"}
+ def check(docs):
+ assert len(docs) == 1
+ assert docs[0][u"\uf8ff"] == "apple"
+ self.run_check(query, check, indexes=["text"])
+
+ def test_internal_field_tests(self):
+ queries = [
+ {"utf8-1[]:string" : "string"},
+ {"utf8-2[]:boolean[]" : True},
+ {"utf8-3[]:number" : 9},
+ {"utf8-3[]:null" : None}
+ ]
+ def check(docs):
+ assert len(docs) == 1
+ assert docs[0]["title"] == "internal_fields_format"
+ for query in queries:
+ self.run_check(query, check, indexes=["text"])
+
+ def test_escape_period(self):
+ query = {"name\\.first" : "Kvothe"}
+ def check(docs):
+ assert len(docs) == 1
+ assert docs[0]["name.first"] == "Kvothe"
+ self.run_check(query, check, indexes=["text"])
+
+ query = {"name.first" : "Kvothe"}
+ def check_empty(docs):
+ assert len(docs) == 0
+ self.run_check(query, check_empty, indexes=["text"])
+
+ def test_object_period(self):
+ query = {"name.first" : "Master Elodin"}
+ def check(docs):
+ assert len(docs) == 1
+ assert docs[0]["title"] == "key with peso"
+ self.run_check(query, check, indexes=["text"])
+
+ query = {"name\\.first" : "Master Elodin"}
+ def check_empty(docs):
+ assert len(docs) == 0
+ self.run_check(query, check_empty, indexes=["text"])
diff --git a/src/mango/test/05-index-selection-test.py b/src/mango/test/05-index-selection-test.py
new file mode 100644
index 000000000..bbd3aa7f2
--- /dev/null
+++ b/src/mango/test/05-index-selection-test.py
@@ -0,0 +1,178 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+import mango
+import user_docs
+import unittest
+
+
+class IndexSelectionTests(mango.UserDocsTests):
+ @classmethod
+ def setUpClass(klass):
+ super(IndexSelectionTests, klass).setUpClass()
+ if mango.has_text_service():
+ user_docs.add_text_indexes(klass.db, {})
+
+ def test_basic(self):
+ resp = self.db.find({"name.last": "A last name"}, explain=True)
+ assert resp["index"]["type"] == "json"
+
+ def test_with_and(self):
+ resp = self.db.find({
+ "name.first": "Stephanie",
+ "name.last": "This doesn't have to match anything."
+ }, explain=True)
+ assert resp["index"]["type"] == "json"
+
+ @unittest.skipUnless(mango.has_text_service(), "requires text service")
+ def test_with_text(self):
+ resp = self.db.find({
+ "$text" : "Stephanie",
+ "name.first": "Stephanie",
+ "name.last": "This doesn't have to match anything."
+ }, explain=True)
+ assert resp["index"]["type"] == "text"
+
+ @unittest.skipUnless(mango.has_text_service(), "requires text service")
+ def test_no_view_index(self):
+ resp = self.db.find({"name.first": "Ohai!"}, explain=True)
+ assert resp["index"]["type"] == "text"
+
+ @unittest.skipUnless(mango.has_text_service(), "requires text service")
+ def test_with_or(self):
+ resp = self.db.find({
+ "$or": [
+ {"name.first": "Stephanie"},
+ {"name.last": "This doesn't have to match anything."}
+ ]
+ }, explain=True)
+ assert resp["index"]["type"] == "text"
+
+ def test_use_most_columns(self):
+ # ddoc id for the age index
+ ddocid = "_design/ad3d537c03cd7c6a43cf8dff66ef70ea54c2b40f"
+ resp = self.db.find({
+ "name.first": "Stephanie",
+ "name.last": "Something or other",
+ "age": {"$gt": 1}
+ }, explain=True)
+ assert resp["index"]["ddoc"] != "_design/" + ddocid
+
+ resp = self.db.find({
+ "name.first": "Stephanie",
+ "name.last": "Something or other",
+ "age": {"$gt": 1}
+ }, use_index=ddocid, explain=True)
+ assert resp["index"]["ddoc"] == ddocid
+
+ def test_use_most_columns(self):
+ # ddoc id for the age index
+ ddocid = "_design/ad3d537c03cd7c6a43cf8dff66ef70ea54c2b40f"
+ try:
+ self.db.find({}, use_index=ddocid)
+ except Exception, e:
+ assert e.response.status_code == 400
+ else:
+ raise AssertionError("bad find")
+
+ # This doc will not be saved given the new ddoc validation code
+ # in couch_mrview
+ def test_manual_bad_view_idx01(self):
+ design_doc = {
+ "_id": "_design/bad_view_index",
+ "language": "query",
+ "views": {
+ "queryidx1": {
+ "map": {
+ "fields": {
+ "age": "asc"
+ }
+ },
+ "reduce": "_count",
+ "options": {
+ "def": {
+ "fields": [
+ {
+ "age": "asc"
+ }
+ ]
+ },
+ "w": 2
+ }
+ }
+ },
+ "views" : {
+ "views001" : {
+ "map" : "function(employee){if(employee.training)"
+ + "{emit(employee.number, employee.training);}}"
+ }
+ }
+ }
+ with self.assertRaises(KeyError):
+ self.db.save_doc(design_doc)
+
+ @unittest.skipUnless(mango.has_text_service(), "requires text service")
+ def test_manual_bad_text_idx(self):
+ design_doc = {
+ "_id": "_design/bad_text_index",
+ "language": "query",
+ "indexes": {
+ "text_index": {
+ "default_analyzer": "keyword",
+ "default_field": {},
+ "selector": {},
+ "fields": "all_fields",
+ "analyzer": {
+ "name": "perfield",
+ "default": "keyword",
+ "fields": {
+ "$default": "standard"
+ }
+ }
+ }
+ },
+ "indexes": {
+ "st_index": {
+ "analyzer": "standard",
+ "index": "function(doc){\n index(\"st_index\", doc.geometry);\n}"
+ }
+ }
+ }
+ self.db.save_doc(design_doc)
+ docs= self.db.find({"age" : 48})
+ assert len(docs) == 1
+ assert docs[0]["name"]["first"] == "Stephanie"
+ assert docs[0]["age"] == 48
+
+
+@unittest.skipUnless(mango.has_text_service(), "requires text service")
+class MultiTextIndexSelectionTests(mango.UserDocsTests):
+ @classmethod
+ def setUpClass(klass):
+ super(MultiTextIndexSelectionTests, klass).setUpClass()
+ if mango.has_text_service():
+ klass.db.create_text_index(ddoc="foo", analyzer="keyword")
+ klass.db.create_text_index(ddoc="bar", analyzer="email")
+
+ def test_view_ok_with_multi_text(self):
+ resp = self.db.find({"name.last": "A last name"}, explain=True)
+ assert resp["index"]["type"] == "json"
+
+ def test_multi_text_index_is_error(self):
+ try:
+ self.db.find({"$text": "a query"}, explain=True)
+ except Exception, e:
+ assert e.response.status_code == 400
+
+ def test_use_index_works(self):
+ resp = self.db.find({"$text": "a query"}, use_index="foo", explain=True)
+ assert resp["index"]["ddoc"] == "_design/foo"
diff --git a/src/mango/test/06-basic-text-test.py b/src/mango/test/06-basic-text-test.py
new file mode 100644
index 000000000..7f5ce6345
--- /dev/null
+++ b/src/mango/test/06-basic-text-test.py
@@ -0,0 +1,653 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+import json
+import mango
+import unittest
+import user_docs
+import math
+from hypothesis import given, assume, example
+import hypothesis.strategies as st
+
+@unittest.skipIf(mango.has_text_service(), "text service exists")
+class TextIndexCheckTests(mango.DbPerClass):
+
+ def test_create_text_index(self):
+ body = json.dumps({
+ 'index': {
+ },
+ 'type': 'text'
+ })
+ resp = self.db.sess.post(self.db.path("_index"), data=body)
+ assert resp.status_code == 503, resp
+
+@unittest.skipUnless(mango.has_text_service(), "requires text service")
+class BasicTextTests(mango.UserDocsTextTests):
+
+ def test_simple(self):
+ docs = self.db.find({"$text": "Stephanie"})
+ assert len(docs) == 1
+ assert docs[0]["name"]["first"] == "Stephanie"
+
+ def test_with_integer(self):
+ docs = self.db.find({"name.first": "Stephanie", "age": 48})
+ assert len(docs) == 1
+ assert docs[0]["name"]["first"] == "Stephanie"
+ assert docs[0]["age"] == 48
+
+ def test_with_boolean(self):
+ docs = self.db.find({"name.first": "Stephanie", "manager": False})
+ assert len(docs) == 1
+ assert docs[0]["name"]["first"] == "Stephanie"
+ assert docs[0]["manager"] == False
+
+ def test_with_array(self):
+ faves = ["Ruby", "C", "Python"]
+ docs = self.db.find({"name.first": "Stephanie", "favorites": faves})
+ assert docs[0]["name"]["first"] == "Stephanie"
+ assert docs[0]["favorites"] == faves
+
+ def test_array_ref(self):
+ docs = self.db.find({"favorites.1": "Python"})
+ assert len(docs) == 4
+ for d in docs:
+ assert "Python" in d["favorites"]
+
+ # Nested Level
+ docs = self.db.find({"favorites.0.2": "Python"})
+ print len(docs)
+ assert len(docs) == 1
+ for d in docs:
+ assert "Python" in d["favorites"][0][2]
+
+ def test_number_ref(self):
+ docs = self.db.find({"11111": "number_field"})
+ assert len(docs) == 1
+ assert docs[0]["11111"] == "number_field"
+
+ docs = self.db.find({"22222.33333": "nested_number_field"})
+ assert len(docs) == 1
+ assert docs[0]["22222"]["33333"] == "nested_number_field"
+
+ def test_lt(self):
+ docs = self.db.find({"age": {"$lt": 22}})
+ assert len(docs) == 0
+
+ docs = self.db.find({"age": {"$lt": 23}})
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == 9
+
+ docs = self.db.find({"age": {"$lt": 33}})
+ assert len(docs) == 2
+ for d in docs:
+ assert d["user_id"] in (1, 9)
+
+ docs = self.db.find({"age": {"$lt": 34}})
+ assert len(docs) == 3
+ for d in docs:
+ assert d["user_id"] in (1, 7, 9)
+
+ docs = self.db.find({"company": {"$lt": "Dreamia"}})
+ assert len(docs) == 1
+ assert docs[0]["company"] == "Affluex"
+
+ def test_lte(self):
+ docs = self.db.find({"age": {"$lte": 21}})
+ assert len(docs) == 0
+
+ docs = self.db.find({"age": {"$lte": 22}})
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == 9
+
+ docs = self.db.find({"age": {"$lte": 33}})
+ assert len(docs) == 3
+ for d in docs:
+ assert d["user_id"] in (1, 7, 9)
+
+ docs = self.db.find({"company": {"$lte": "Dreamia"}})
+ assert len(docs) == 2
+ for d in docs:
+ assert d["user_id"] in (0, 11)
+
+ def test_eq(self):
+ docs = self.db.find({"age": 21})
+ assert len(docs) == 0
+
+ docs = self.db.find({"age": 22})
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == 9
+
+ docs = self.db.find({"age": {"$eq": 22}})
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == 9
+
+ docs = self.db.find({"age": 33})
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == 7
+
+ def test_ne(self):
+ docs = self.db.find({"age": {"$ne": 22}})
+ assert len(docs) == len(user_docs.DOCS) - 1
+ for d in docs:
+ assert d["age"] != 22
+
+ docs = self.db.find({"$not": {"age": 22}})
+ assert len(docs) == len(user_docs.DOCS) - 1
+ for d in docs:
+ assert d["age"] != 22
+
+ def test_gt(self):
+ docs = self.db.find({"age": {"$gt": 77}})
+ assert len(docs) == 2
+ for d in docs:
+ assert d["user_id"] in (3, 13)
+
+ docs = self.db.find({"age": {"$gt": 78}})
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == 3
+
+ docs = self.db.find({"age": {"$gt": 79}})
+ assert len(docs) == 0
+
+ docs = self.db.find({"company": {"$gt": "Zialactic"}})
+ assert len(docs) == 0
+
+ def test_gte(self):
+ docs = self.db.find({"age": {"$gte": 77}})
+ assert len(docs) == 2
+ for d in docs:
+ assert d["user_id"] in (3, 13)
+
+ docs = self.db.find({"age": {"$gte": 78}})
+ assert len(docs) == 2
+ for d in docs:
+ assert d["user_id"] in (3, 13)
+
+ docs = self.db.find({"age": {"$gte": 79}})
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == 3
+
+ docs = self.db.find({"age": {"$gte": 80}})
+ assert len(docs) == 0
+
+ docs = self.db.find({"company": {"$gte": "Zialactic"}})
+ assert len(docs) == 1
+ assert docs[0]["company"] == "Zialactic"
+
+ def test_and(self):
+ docs = self.db.find({"age": 22, "manager": True})
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == 9
+
+ docs = self.db.find({"age": 22, "manager": False})
+ assert len(docs) == 0
+
+ docs = self.db.find({"$and": [{"age": 22}, {"manager": True}]})
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == 9
+
+ docs = self.db.find({"$and": [{"age": 22}, {"manager": False}]})
+ assert len(docs) == 0
+
+ docs = self.db.find({"$text": "Ramona", "age": 22})
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == 9
+
+ docs = self.db.find({"$and": [{"$text": "Ramona"}, {"age": 22}]})
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == 9
+
+ docs = self.db.find({"$and": [{"$text": "Ramona"}, {"$text": "Floyd"}]})
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == 9
+
+ def test_or(self):
+ docs = self.db.find({"$or": [{"age": 22}, {"age": 33}]})
+ assert len(docs) == 2
+ for d in docs:
+ assert d["user_id"] in (7, 9)
+
+ q = {"$or": [{"$text": "Ramona"}, {"$text": "Stephanie"}]}
+ docs = self.db.find(q)
+ assert len(docs) == 2
+ for d in docs:
+ assert d["user_id"] in (0, 9)
+
+ q = {"$or": [{"$text": "Ramona"}, {"age": 22}]}
+ docs = self.db.find(q)
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == 9
+
+ def test_and_or(self):
+ q = {
+ "age": 22,
+ "$or": [
+ {"manager": False},
+ {"location.state": "Missouri"}
+ ]
+ }
+ docs = self.db.find(q)
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == 9
+
+ q = {
+ "$or": [
+ {"age": 22},
+ {"age": 43, "manager": True}
+ ]
+ }
+ docs = self.db.find(q)
+ assert len(docs) == 2
+ for d in docs:
+ assert d["user_id"] in (9, 10)
+
+ q = {
+ "$or": [
+ {"$text": "Ramona"},
+ {"age": 43, "manager": True}
+ ]
+ }
+ docs = self.db.find(q)
+ assert len(docs) == 2
+ for d in docs:
+ assert d["user_id"] in (9, 10)
+
+ def test_nor(self):
+ docs = self.db.find({"$nor": [{"age": 22}, {"age": 33}]})
+ assert len(docs) == 13
+ for d in docs:
+ assert d["user_id"] not in (7, 9)
+
+ def test_in_with_value(self):
+ docs = self.db.find({"age": {"$in": [1, 5]}})
+ assert len(docs) == 0
+
+ docs = self.db.find({"age": {"$in": [1, 5, 22]}})
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == 9
+
+ docs = self.db.find({"age": {"$in": [1, 5, 22, 31]}})
+ assert len(docs) == 2
+ for d in docs:
+ assert d["user_id"] in (1, 9)
+
+ docs = self.db.find({"age": {"$in": [22, 31]}})
+ assert len(docs) == 2
+ for d in docs:
+ assert d["user_id"] in (1, 9)
+
+ # Limits on boolean clauses?
+ docs = self.db.find({"age": {"$in": range(1000)}})
+ assert len(docs) == 15
+
+ def test_in_with_array(self):
+ vals = ["Random Garbage", 52, {"Versions": {"Alpha": "Beta"}}]
+ docs = self.db.find({"favorites": {"$in": vals}})
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == 1
+
+ vals = ["Lisp", "Python"]
+ docs = self.db.find({"favorites": {"$in": vals}})
+ assert len(docs) == 10
+
+ vals = [{"val1": 1, "val2": "val2"}]
+ docs = self.db.find({"test_in": {"$in": vals}})
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == 2
+
+ def test_nin_with_value(self):
+ docs = self.db.find({"age": {"$nin": [1, 5]}})
+ assert len(docs) == len(user_docs.DOCS)
+
+ docs = self.db.find({"age": {"$nin": [1, 5, 22]}})
+ assert len(docs) == len(user_docs.DOCS) - 1
+ for d in docs:
+ assert d["user_id"] != 9
+
+ docs = self.db.find({"age": {"$nin": [1, 5, 22, 31]}})
+ assert len(docs) == len(user_docs.DOCS) - 2
+ for d in docs:
+ assert d["user_id"] not in (1, 9)
+
+ docs = self.db.find({"age": {"$nin": [22, 31]}})
+ assert len(docs) == len(user_docs.DOCS) - 2
+ for d in docs:
+ assert d["user_id"] not in (1, 9)
+
+ # Limits on boolean clauses?
+ docs = self.db.find({"age": {"$nin": range(1000)}})
+ assert len(docs) == 0
+
+ def test_nin_with_array(self):
+ vals = ["Random Garbage", 52, {"Versions": {"Alpha": "Beta"}}]
+ docs = self.db.find({"favorites": {"$nin": vals}})
+ assert len(docs) == len(user_docs.DOCS) - 1
+ for d in docs:
+ assert d["user_id"] != 1
+
+ vals = ["Lisp", "Python"]
+ docs = self.db.find({"favorites": {"$nin": vals}})
+ assert len(docs) == 5
+
+ vals = [{"val1": 1, "val2": "val2"}]
+ docs = self.db.find({"test_in": {"$nin": vals}})
+ assert len(docs) == 0
+
+ def test_all(self):
+ vals = ["Ruby", "C", "Python", {"Versions": {"Alpha": "Beta"}}]
+ docs = self.db.find({"favorites": {"$all": vals}})
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == 1
+
+ # This matches where favorites either contains
+ # the nested array, or is the nested array. This is
+ # notably different than the non-nested array in that
+ # it does not match a re-ordered version of the array.
+ # The fact that user_id 14 isn't included demonstrates
+ # this behavior.
+ vals = [["Lisp", "Erlang", "Python"]]
+ docs = self.db.find({"favorites": {"$all": vals}})
+ assert len(docs) == 2
+ for d in docs:
+ assert d["user_id"] in (3, 9)
+
+ def test_exists_field(self):
+ docs = self.db.find({"exists_field": {"$exists": True}})
+ assert len(docs) == 2
+ for d in docs:
+ assert d["user_id"] in (7, 8)
+
+ docs = self.db.find({"exists_field": {"$exists": False}})
+ assert len(docs) == len(user_docs.DOCS) - 2
+ for d in docs:
+ assert d["user_id"] not in (7, 8)
+
+ def test_exists_array(self):
+ docs = self.db.find({"exists_array": {"$exists": True}})
+ assert len(docs) == 2
+ for d in docs:
+ assert d["user_id"] in (9, 10)
+
+ docs = self.db.find({"exists_array": {"$exists": False}})
+ assert len(docs) == len(user_docs.DOCS) - 2
+ for d in docs:
+ assert d["user_id"] not in (9, 10)
+
+ def test_exists_object(self):
+ docs = self.db.find({"exists_object": {"$exists": True}})
+ assert len(docs) == 2
+ for d in docs:
+ assert d["user_id"] in (11, 12)
+
+ docs = self.db.find({"exists_object": {"$exists": False}})
+ assert len(docs) == len(user_docs.DOCS) - 2
+ for d in docs:
+ assert d["user_id"] not in (11, 12)
+
+ def test_exists_object_member(self):
+ docs = self.db.find({"exists_object.should": {"$exists": True}})
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == 11
+
+ docs = self.db.find({"exists_object.should": {"$exists": False}})
+ assert len(docs) == len(user_docs.DOCS) - 1
+ for d in docs:
+ assert d["user_id"] != 11
+
+ def test_exists_and(self):
+ q = {"$and": [
+ {"manager": {"$exists": True}},
+ {"exists_object.should": {"$exists": True}}
+ ]}
+ docs = self.db.find(q)
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == 11
+
+ q = {"$and": [
+ {"manager": {"$exists": False}},
+ {"exists_object.should": {"$exists": True}}
+ ]}
+ docs = self.db.find(q)
+ assert len(docs) == 0
+
+ # Translates to manager exists or exists_object.should doesn't
+ # exist, which will match all docs
+ q = {"$not": q}
+ docs = self.db.find(q)
+ assert len(docs) == len(user_docs.DOCS)
+
+ def test_value_chars(self):
+ q = {"complex_field_value": "+-(){}[]^~&&*||\"\\/?:!"}
+ docs = self.db.find(q)
+ assert len(docs) == 1
+
+ def test_regex(self):
+ docs = self.db.find({
+ "age": {"$gt": 40},
+ "location.state": {"$regex": "(?i)new.*"}
+ })
+ assert len(docs) == 2
+ assert docs[0]["user_id"] == 2
+ assert docs[1]["user_id"] == 10
+
+ # test lucene syntax in $text
+
+@unittest.skipUnless(mango.has_text_service(), "requires text service")
+class ElemMatchTests(mango.FriendDocsTextTests):
+
+ def test_elem_match_non_object(self):
+ q = {"bestfriends":{
+ "$elemMatch":
+ {"$eq":"Wolverine", "$eq":"Cyclops"}
+ }
+ }
+ docs = self.db.find(q)
+ print len(docs)
+ assert len(docs) == 1
+ assert docs[0]["bestfriends"] == ["Wolverine", "Cyclops"]
+
+ q = {"results": {"$elemMatch": {"$gte": 80, "$lt": 85}}}
+
+ docs = self.db.find(q)
+ print len(docs)
+ assert len(docs) == 1
+ assert docs[0]["results"] == [82, 85, 88]
+
+ def test_elem_match(self):
+ q = {"friends": {
+ "$elemMatch":
+ {"name.first": "Vargas"}
+ }
+ }
+ docs = self.db.find(q)
+ assert len(docs) == 2
+ for d in docs:
+ assert d["user_id"] in (0, 1)
+
+ q = {
+ "friends": {
+ "$elemMatch": {
+ "name.first": "Ochoa",
+ "name.last": "Burch"
+ }
+ }
+ }
+ docs = self.db.find(q)
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == 4
+
+
+ # Check that we can do logic in elemMatch
+ q = {
+ "friends": {"$elemMatch": {
+ "name.first": "Ochoa", "type": "work"
+ }}
+ }
+ docs = self.db.find(q)
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == 1
+
+ q = {
+ "friends": {
+ "$elemMatch": {
+ "name.first": "Ochoa",
+ "$or": [
+ {"type": "work"},
+ {"type": "personal"}
+ ]
+ }
+ }
+ }
+ docs = self.db.find(q)
+ assert len(docs) == 2
+ for d in docs:
+ assert d["user_id"] in (1, 4)
+
+ # Same as last, but using $in
+ q = {
+ "friends": {
+ "$elemMatch": {
+ "name.first": "Ochoa",
+ "type": {"$in": ["work", "personal"]}
+ }
+ }
+ }
+ docs = self.db.find(q)
+ assert len(docs) == 2
+ for d in docs:
+ assert d["user_id"] in (1, 4)
+
+ q = {
+ "$and": [{
+ "friends": {
+ "$elemMatch": {
+ "id": 0,
+ "name": {
+ "$exists": True
+ }
+ }
+ }
+ },
+ {
+ "friends": {
+ "$elemMatch": {
+ "$or": [
+ {
+ "name": {
+ "first": "Campos",
+ "last": "Freeman"
+ }
+ },
+ {
+ "name": {
+ "$in": [{
+ "first": "Gibbs",
+ "last": "Mccarty"
+ },
+ {
+ "first": "Wilkins",
+ "last": "Chang"
+ }
+ ]
+ }
+ }
+ ]
+ }
+ }
+ }
+ ]
+ }
+ docs = self.db.find(q)
+ assert len(docs) == 3
+ for d in docs:
+ assert d["user_id"] in (10, 11,12)
+
+@unittest.skipUnless(mango.has_text_service(), "requires text service")
+class AllMatchTests(mango.FriendDocsTextTests):
+
+ def test_all_match(self):
+ q = {"friends": {
+ "$allMatch":
+ {"type": "personal"}
+ }
+ }
+ docs = self.db.find(q)
+ assert len(docs) == 2
+ for d in docs:
+ assert d["user_id"] in (8, 5)
+
+ # Check that we can do logic in allMatch
+ q = {
+ "friends": {
+ "$allMatch": {
+ "name.first": "Ochoa",
+ "$or": [
+ {"type": "work"},
+ {"type": "personal"}
+ ]
+ }
+ }
+ }
+ docs = self.db.find(q)
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == 15
+
+ # Same as last, but using $in
+ q = {
+ "friends": {
+ "$allMatch": {
+ "name.first": "Ochoa",
+ "type": {"$in": ["work", "personal"]}
+ }
+ }
+ }
+ docs = self.db.find(q)
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == 15
+
+
+# Test numeric strings for $text
+@unittest.skipUnless(mango.has_text_service(), "requires text service")
+class NumStringTests(mango.DbPerClass):
+
+ @classmethod
+ def setUpClass(klass):
+ super(NumStringTests, klass).setUpClass()
+ klass.db.recreate()
+ if mango.has_text_service():
+ klass.db.create_text_index()
+
+ # not available for python 2.7.x
+ def isFinite(num):
+ not (math.isinf(num) or math.isnan(num))
+
+ @given(f=st.floats().filter(isFinite).map(str)
+ | st.floats().map(lambda f: f.hex()))
+ @example('NaN')
+ @example('Infinity')
+ def test_floating_point_val(self,f):
+ doc = {"number_string": f}
+ self.db.save_doc(doc)
+ q = {"$text": f}
+ docs = self.db.find(q)
+ if len(docs) == 1:
+ assert docs[0]["number_string"] == f
+ if len(docs) == 2:
+ if docs[0]["number_string"] != f:
+ assert docs[1]["number_string"] == f
+ q = {"number_string": f}
+ docs = self.db.find(q)
+ if len(docs) == 1:
+ assert docs[0]["number_string"] == f
+ if len(docs) == 2:
+ if docs[0]["number_string"] != f:
+ assert docs[1]["number_string"] == f
diff --git a/src/mango/test/06-text-default-field-test.py b/src/mango/test/06-text-default-field-test.py
new file mode 100644
index 000000000..3f86f0e41
--- /dev/null
+++ b/src/mango/test/06-text-default-field-test.py
@@ -0,0 +1,73 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+import mango
+import unittest
+
+
+@unittest.skipUnless(mango.has_text_service(), "requires text service")
+class NoDefaultFieldTest(mango.UserDocsTextTests):
+
+ DEFAULT_FIELD = False
+
+ def test_basic(self):
+ docs = self.db.find({"$text": "Ramona"})
+ # Or should this throw an error?
+ assert len(docs) == 0
+
+ def test_other_fields_exist(self):
+ docs = self.db.find({"age": 22})
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == 9
+
+
+@unittest.skipUnless(mango.has_text_service(), "requires text service")
+class NoDefaultFieldWithAnalyzer(mango.UserDocsTextTests):
+
+ DEFAULT_FIELD = {
+ "enabled": False,
+ "analyzer": "keyword"
+ }
+
+ def test_basic(self):
+ docs = self.db.find({"$text": "Ramona"})
+ assert len(docs) == 0
+
+ def test_other_fields_exist(self):
+ docs = self.db.find({"age": 22})
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == 9
+
+
+@unittest.skipUnless(mango.has_text_service(), "requires text service")
+class DefaultFieldWithCustomAnalyzer(mango.UserDocsTextTests):
+
+ DEFAULT_FIELD = {
+ "enabled": True,
+ "analyzer": "keyword"
+ }
+
+ def test_basic(self):
+ docs = self.db.find({"$text": "Ramona"})
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == 9
+
+ def test_not_analyzed(self):
+ docs = self.db.find({"$text": "Lott Place"})
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == 9
+
+ docs = self.db.find({"$text": "Lott"})
+ assert len(docs) == 0
+
+ docs = self.db.find({"$text": "Place"})
+ assert len(docs) == 0
diff --git a/src/mango/test/07-text-custom-field-list-test.py b/src/mango/test/07-text-custom-field-list-test.py
new file mode 100644
index 000000000..4db11a5af
--- /dev/null
+++ b/src/mango/test/07-text-custom-field-list-test.py
@@ -0,0 +1,158 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+import mango
+import unittest
+
+
+@unittest.skipUnless(mango.has_text_service(), "requires text service")
+class CustomFieldsTest(mango.UserDocsTextTests):
+
+ FIELDS = [
+ {"name": "favorites.[]", "type": "string"},
+ {"name": "manager", "type": "boolean"},
+ {"name": "age", "type": "number"},
+ # These two are to test the default analyzer for
+ # each field.
+ {"name": "location.state", "type": "string"},
+ {
+ "name": "location.address.street",
+ "type": "string"
+ },
+ {"name": "name\\.first", "type": "string"}
+ ]
+
+ def test_basic(self):
+ docs = self.db.find({"age": 22})
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == 9
+
+ def test_multi_field(self):
+ docs = self.db.find({"age": 22, "manager": True})
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == 9
+
+ docs = self.db.find({"age": 22, "manager": False})
+ assert len(docs) == 0
+
+ def test_element_acess(self):
+ docs = self.db.find({"favorites.0": "Ruby"})
+ assert len(docs) == 3
+ for d in docs:
+ assert "Ruby" in d["favorites"]
+
+ # This should throw an exception because we only index the array
+ # favorites.[], and not the string field favorites
+ def test_index_selection(self):
+ try:
+ self.db.find({"selector": {"$or": [{"favorites": "Ruby"},
+ {"favorites.0":"Ruby"}]}})
+ except Exception, e:
+ assert e.response.status_code == 400
+
+ def test_in_with_array(self):
+ vals = ["Lisp", "Python"]
+ docs = self.db.find({"favorites": {"$in": vals}})
+ assert len(docs) == 10
+
+ # This should also throw an error because we only indexed
+ # favorites.[] of type string. For the following query to work, the
+ # user has to index favorites.[] of type number, and also
+ # favorites.[].Versions.Alpha of type string.
+ def test_in_different_types(self):
+ vals = ["Random Garbage", 52, {"Versions": {"Alpha": "Beta"}}]
+ try:
+ self.db.find({"favorites": {"$in": vals}})
+ except Exception, e:
+ assert e.response.status_code == 400
+
+ # This test differs from the situation where we index everything.
+ # When we index everything the actual number of docs that gets
+ # returned is 5. That's because of the special situation where we
+ # have an array of an array, i.e: [["Lisp"]], because we're indexing
+ # specifically favorites.[] of type string. So it does not count
+ # the example and we only get 4 back.
+ def test_nin_with_array(self):
+ vals = ["Lisp", "Python"]
+ docs = self.db.find({"favorites": {"$nin": vals}})
+ assert len(docs) == 4
+
+ def test_missing(self):
+ self.db.find({"location.state": "Nevada"})
+
+ def test_missing_type(self):
+ # Raises an exception
+ try:
+ self.db.find({"age": "foo"})
+ raise Exception("Should have thrown an HTTPError")
+ except:
+ return
+
+ def test_field_analyzer_is_keyword(self):
+ docs = self.db.find({"location.state": "New"})
+ assert len(docs) == 0
+
+ docs = self.db.find({"location.state": "New Hampshire"})
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == 10
+
+ # Since our FIELDS list only includes "name\\.first", we should
+ # get an error when we try to search for "name.first", since the index
+ # for that field does not exist.
+ def test_escaped_field(self):
+ docs = self.db.find({"name\\.first": "name dot first"})
+ assert len(docs) == 1
+ assert docs[0]["name.first"] == "name dot first"
+
+ try:
+ self.db.find({"name.first": "name dot first"})
+ raise Exception("Should have thrown an HTTPError")
+ except:
+ return
+
+ def test_filtered_search_fields(self):
+ docs = self.db.find({"age": 22}, fields = ["age", "location.state"])
+ assert len(docs) == 1
+ assert docs == [{"age": 22, "location": {"state": "Missouri"}}]
+
+ docs = self.db.find({"age": 22}, fields = ["age", "Random Garbage"])
+ assert len(docs) == 1
+ assert docs == [{"age": 22}]
+
+ docs = self.db.find({"age": 22}, fields = ["favorites"])
+ assert len(docs) == 1
+ assert docs == [{"favorites": ["Lisp", "Erlang", "Python"]}]
+
+ docs = self.db.find({"age": 22}, fields = ["favorites.[]"])
+ assert len(docs) == 1
+ assert docs == [{}]
+
+ docs = self.db.find({"age": 22}, fields = ["all_fields"])
+ assert len(docs) == 1
+ assert docs == [{}]
+
+ def test_two_or(self):
+ docs = self.db.find({"$or": [{"location.state": "New Hampshire"},
+ {"location.state": "Don't Exist"}]})
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == 10
+
+ def test_all_match(self):
+ docs = self.db.find({
+ "favorites": {
+ "$allMatch": {
+ "$eq": "Erlang"
+ }
+ }
+ })
+ assert len(docs) == 1
+ assert docs[0]["user_id"] == 10
diff --git a/src/mango/test/08-text-limit-test.py b/src/mango/test/08-text-limit-test.py
new file mode 100644
index 000000000..191a1108a
--- /dev/null
+++ b/src/mango/test/08-text-limit-test.py
@@ -0,0 +1,137 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+import mango
+import limit_docs
+import unittest
+
+@unittest.skipUnless(mango.has_text_service(), "requires text service")
+class LimitTests(mango.LimitDocsTextTests):
+
+ def test_limit_field(self):
+ q = {"$or": [{"user_id" : {"$lt" : 10}}, {"filtered_array.[]": 1}]}
+ docs = self.db.find(q, limit=10)
+ assert len(docs) == 8
+ for d in docs:
+ assert d["user_id"] < 10
+
+ def test_limit_field2(self):
+ q = {"$or": [{"user_id" : {"$lt" : 20}}, {"filtered_array.[]": 1}]}
+ docs = self.db.find(q, limit=10)
+ assert len(docs) == 10
+ for d in docs:
+ assert d["user_id"] < 20
+
+ def test_limit_field3(self):
+ q = {"$or": [{"user_id" : {"$lt" : 100}}, {"filtered_array.[]": 1}]}
+ docs = self.db.find(q, limit=1)
+ assert len(docs) == 1
+ for d in docs:
+ assert d["user_id"] < 100
+
+ def test_limit_field4(self):
+ q = {"$or": [{"user_id" : {"$lt" : 0}}, {"filtered_array.[]": 1}]}
+ docs = self.db.find(q, limit=35)
+ assert len(docs) == 0
+
+ # We reach our cap here of 50
+ def test_limit_field5(self):
+ q = {"age": {"$exists": True}}
+ docs = self.db.find(q, limit=250)
+ print len(docs)
+ assert len(docs) == 75
+ for d in docs:
+ assert d["age"] < 100
+
+ def test_limit_skip_field1(self):
+ q = {"$or": [{"user_id" : {"$lt" : 100}}, {"filtered_array.[]": 1}]}
+ docs = self.db.find(q, limit=10, skip=20)
+ assert len(docs) == 10
+ for d in docs:
+ assert d["user_id"] > 20
+
+ def test_limit_skip_field2(self):
+ q = {"$or": [{"user_id" : {"$lt" : 100}}, {"filtered_array.[]": 1}]}
+ docs = self.db.find(q, limit=100, skip=100)
+ assert len(docs) == 0
+
+ def test_limit_skip_field3(self):
+ q = {"$or": [{"user_id" : {"$lt" : 20}}, {"filtered_array.[]": 1}]}
+ docs = self.db.find(q, limit=1, skip=30)
+ assert len(docs) == 0
+
+ def test_limit_skip_field4(self):
+ q = {"$or": [{"user_id" : {"$lt" : 100}}, {"filtered_array.[]": 1}]}
+ docs = self.db.find(q, limit=0, skip=0)
+ assert len(docs) == 0
+
+ def test_limit_skip_field5(self):
+ q = {"$or": [{"user_id" : {"$lt" : 100}}, {"filtered_array.[]": 1}]}
+ try:
+ self.db.find(q, limit=-1)
+ except Exception, e:
+ assert e.response.status_code == 400
+ else:
+ raise AssertionError("Should have thrown error for negative limit")
+
+ def test_limit_skip_field6(self):
+ q = {"$or": [{"user_id" : {"$lt" : 100}}, {"filtered_array.[]": 1}]}
+ try:
+ self.db.find(q, skip=-1)
+ except Exception, e:
+ assert e.response.status_code == 400
+ else:
+ raise AssertionError("Should have thrown error for negative skip")
+
+ # Basic test to ensure we can iterate through documents with a bookmark
+ def test_limit_bookmark(self):
+ for i in range(1, len(limit_docs.DOCS), 5):
+ self.run_bookmark_check(i)
+
+ for i in range(1, len(limit_docs.DOCS), 5):
+ self.run_bookmark_sort_check(i)
+
+
+ def run_bookmark_check(self, size):
+ print size
+ q = {"age": {"$gt": 0}}
+ seen_docs = set()
+ bm = None
+ while True:
+ json = self.db.find(q, limit=size, bookmark=bm, return_raw=True)
+ for doc in json["docs"]:
+ assert doc["_id"] not in seen_docs
+ seen_docs.add(doc["_id"])
+ if not len(json["docs"]):
+ break
+ assert json["bookmark"] != bm
+ bm = json["bookmark"]
+ assert len(seen_docs) == len(limit_docs.DOCS)
+
+ def run_bookmark_sort_check(self, size):
+ q = {"age": {"$gt": 0}}
+ seen_docs = set()
+ bm = None
+ age = 0
+ while True:
+ json = self.db.find(q, limit=size, bookmark=bm, sort=["age"],
+ return_raw=True)
+ for doc in json["docs"]:
+ assert doc["_id"] not in seen_docs
+ assert doc["age"] >= age
+ age = doc["age"]
+ seen_docs.add(doc["_id"])
+ if not len(json["docs"]):
+ break
+ assert json["bookmark"] != bm
+ bm = json["bookmark"]
+ assert len(seen_docs) == len(limit_docs.DOCS)
diff --git a/src/mango/test/09-text-sort-test.py b/src/mango/test/09-text-sort-test.py
new file mode 100644
index 000000000..ae36a6a33
--- /dev/null
+++ b/src/mango/test/09-text-sort-test.py
@@ -0,0 +1,101 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+import mango
+import unittest
+
+@unittest.skipUnless(mango.has_text_service(), "requires text service")
+class SortTests(mango.UserDocsTextTests):
+
+ def test_number_sort(self):
+ q = {"age": {"$gt": 0}}
+ docs = self.db.find(q, sort=["age:number"])
+ assert len(docs) == 15
+ assert docs[0]["age"] == 22
+
+ def test_number_sort_desc(self):
+ q = {"age": {"$gt": 0}}
+ docs = self.db.find(q, sort=[{"age": "desc"}])
+ assert len(docs) == 15
+ assert docs[0]["age"] == 79
+
+ q = {"manager": True}
+ docs = self.db.find(q, sort=[{"age:number": "desc"}])
+ assert len(docs) == 11
+ assert docs[0]["age"] == 79
+
+ def test_string_sort(self):
+ q = {"email": {"$gt": None}}
+ docs = self.db.find(q, sort=["email:string"])
+ assert len(docs) == 15
+ assert docs[0]["email"] == "abbottwatson@talkola.com"
+
+ def test_notype_sort(self):
+ q = {"email": {"$gt": None}}
+ try:
+ self.db.find(q, sort=["email"])
+ except Exception, e:
+ assert e.response.status_code == 400
+ else:
+ raise AssertionError("Should have thrown error for sort")
+
+ def test_array_sort(self):
+ q = {"favorites": {"$exists": True}}
+ docs = self.db.find(q, sort=["favorites.[]:string"])
+ assert len(docs) == 15
+ assert docs[0]["user_id"] == 8
+
+ def test_multi_sort(self):
+ q = {"name": {"$exists": True}}
+ docs = self.db.find(q, sort=["name.last:string", "age:number"])
+ assert len(docs) == 15
+ assert docs[0]["name"] == {"last":"Ewing","first":"Shelly"}
+ assert docs[1]["age"] == 22
+
+ def test_guess_type_sort(self):
+ q = {"$or": [{"age":{"$gt": 0}}, {"email": {"$gt": None}}]}
+ docs = self.db.find(q, sort=["age"])
+ assert len(docs) == 15
+ assert docs[0]["age"] == 22
+
+ def test_guess_dup_type_sort(self):
+ q = {"$and": [{"age":{"$gt": 0}}, {"email": {"$gt": None}},
+ {"age":{"$lte": 100}}]}
+ docs = self.db.find(q, sort=["age"])
+ assert len(docs) == 15
+ assert docs[0]["age"] == 22
+
+ def test_ambiguous_type_sort(self):
+ q = {"$or": [{"age":{"$gt": 0}}, {"email": {"$gt": None}},
+ {"age": "34"}]}
+ try:
+ self.db.find(q, sort=["age"])
+ except Exception, e:
+ assert e.response.status_code == 400
+ else:
+ raise AssertionError("Should have thrown error for sort")
+
+ def test_guess_multi_sort(self):
+ q = {"$or": [{"age":{"$gt": 0}}, {"email": {"$gt": None}},
+ {"name.last": "Harvey"}]}
+ docs = self.db.find(q, sort=["name.last", "age"])
+ assert len(docs) == 15
+ assert docs[0]["name"] == {"last":"Ewing","first":"Shelly"}
+ assert docs[1]["age"] == 22
+
+ def test_guess_mix_sort(self):
+ q = {"$or": [{"age":{"$gt": 0}}, {"email": {"$gt": None}},
+ {"name.last": "Harvey"}]}
+ docs = self.db.find(q, sort=["name.last:string", "age"])
+ assert len(docs) == 15
+ assert docs[0]["name"] == {"last":"Ewing","first":"Shelly"}
+ assert docs[1]["age"] == 22
diff --git a/src/mango/test/10-disable-array-length-field-test.py b/src/mango/test/10-disable-array-length-field-test.py
new file mode 100644
index 000000000..0715f1db9
--- /dev/null
+++ b/src/mango/test/10-disable-array-length-field-test.py
@@ -0,0 +1,42 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+import mango
+import unittest
+
+
+class DisableIndexArrayLengthsTest(mango.UserDocsTextTests):
+
+ @classmethod
+ def setUpClass(klass):
+ super(DisableIndexArrayLengthsTest, klass).setUpClass()
+ if mango.has_text_service():
+ klass.db.create_text_index(ddoc="disable_index_array_lengths",
+ analyzer="keyword",
+ index_array_lengths=False)
+ klass.db.create_text_index(ddoc="explicit_enable_index_array_lengths",
+ analyzer="keyword",
+ index_array_lengths=True)
+
+ @unittest.skipUnless(mango.has_text_service(), "requires text service")
+ def test_disable_index_array_length(self):
+ docs = self.db.find({"favorites": {"$size": 4}},
+ use_index="disable_index_array_lengths")
+ for d in docs:
+ assert len(d["favorites"]) == 0
+
+ @unittest.skipUnless(mango.has_text_service(), "requires text service")
+ def test_enable_index_array_length(self):
+ docs = self.db.find({"favorites": {"$size": 4}},
+ use_index="explicit_enable_index_array_lengths")
+ for d in docs:
+ assert len(d["favorites"]) == 4
diff --git a/src/mango/test/11-ignore-design-docs.py b/src/mango/test/11-ignore-design-docs.py
new file mode 100644
index 000000000..ea7165e3f
--- /dev/null
+++ b/src/mango/test/11-ignore-design-docs.py
@@ -0,0 +1,39 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+import mango
+import unittest
+
+DOCS = [
+ {
+ "_id": "_design/my-design-doc",
+ },
+ {
+ "_id": "54af50626de419f5109c962f",
+ "user_id": 0,
+ "age": 10,
+ "name": "Jimi"
+ },
+ {
+ "_id": "54af50622071121b25402dc3",
+ "user_id": 1,
+ "age": 11,
+ "name": "Eddie"
+ }
+]
+
+class IgnoreDesignDocsForAllDocsIndexTests(mango.DbPerClass):
+ def test_should_not_return_design_docs(self):
+ self.db.save_docs(DOCS)
+ docs = self.db.find({"_id": {"$gte": None}})
+ assert len(docs) == 2
+
diff --git a/src/mango/test/README.md b/src/mango/test/README.md
new file mode 100644
index 000000000..fc2cd62e5
--- /dev/null
+++ b/src/mango/test/README.md
@@ -0,0 +1,12 @@
+Mango Tests
+===========
+
+CouchDB should be started with `./dev/run -a testuser:testpass`.
+
+To run these, do this in the Mango top level directory:
+
+ $ virtualenv venv
+ $ source venv/bin/activate
+ $ pip install nose requests
+ $ pip install hypothesis
+ $ nosetests
diff --git a/src/mango/test/friend_docs.py b/src/mango/test/friend_docs.py
new file mode 100644
index 000000000..075796138
--- /dev/null
+++ b/src/mango/test/friend_docs.py
@@ -0,0 +1,604 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+"""
+Generated with http://www.json-generator.com/
+
+With this pattern:
+
+[
+ '{{repeat(15)}}',
+ {
+ _id: '{{index()}}',
+ name: {
+ first: '{{firstName()}}',
+ last: '{{surname()}}'
+ },
+ friends: [
+ '{{repeat(3)}}',
+ {
+ id: '{{index()}}',
+ name: {
+ first: '{{firstName()}}',
+ last: '{{surname()}}'
+ },
+ type: '{{random("personal", "work")}}'
+ }
+ ]
+ }
+]
+"""
+
+import copy
+
+
+def setup(db, index_type="view"):
+ db.recreate()
+ db.save_docs(copy.deepcopy(DOCS))
+ if index_type == "view":
+ add_view_indexes(db)
+ elif index_type == "text":
+ add_text_indexes(db)
+
+
+def add_text_indexes(db):
+ db.create_text_index()
+
+
+DOCS = [
+ {
+ "_id": "54a43171d37ae5e81bff5ae0",
+ "user_id": 0,
+ "name": {
+ "first": "Ochoa",
+ "last": "Fox"
+ },
+ "friends": [
+ {
+ "id": 0,
+ "name": {
+ "first": "Sherman",
+ "last": "Davidson"
+ },
+ "type": "personal"
+ },
+ {
+ "id": 1,
+ "name": {
+ "first": "Vargas",
+ "last": "Mendez"
+ },
+ "type": "personal"
+ },
+ {
+ "id": 2,
+ "name": {
+ "first": "Sheppard",
+ "last": "Cotton"
+ },
+ "type": "work"
+ }
+ ]
+ },
+ {
+ "_id": "54a43171958485dc32917c50",
+ "user_id": 1,
+ "name": {
+ "first": "Sheppard",
+ "last": "Cotton"
+ },
+ "friends": [
+ {
+ "id": 0,
+ "name": {
+ "first": "Ochoa",
+ "last": "Fox"
+ },
+ "type": "work"
+ },
+ {
+ "id": 1,
+ "name": {
+ "first": "Vargas",
+ "last": "Mendez"
+ },
+ "type": "personal"
+ },
+ {
+ "id": 2,
+ "name": {
+ "first": "Kendra",
+ "last": "Burns"
+ },
+ "type": "work"
+ }
+ ]
+ },
+ {
+ "_id": "54a431711cf025ba74bea899",
+ "user_id": 2,
+ "name": {
+ "first": "Hunter",
+ "last": "Wells"
+ },
+ "friends": [
+ {
+ "id": 0,
+ "name": {
+ "first": "Estes",
+ "last": "Fischer"
+ },
+ "type": "work"
+ },
+ {
+ "id": 1,
+ "name": {
+ "first": "Farrell",
+ "last": "Maddox"
+ },
+ "type": "personal"
+ },
+ {
+ "id": 2,
+ "name": {
+ "first": "Kendra",
+ "last": "Burns"
+ },
+ "type": "work"
+ }
+ ]
+ },
+ {
+ "_id": "54a4317151a70a9881ac28a4",
+ "user_id": 3,
+ "name": {
+ "first": "Millicent",
+ "last": "Guy"
+ },
+ "friends": [
+ {
+ "id": 0,
+ "name": {
+ "first": "Luella",
+ "last": "Mendoza"
+ },
+ "type": "work"
+ },
+ {
+ "id": 1,
+ "name": {
+ "first": "Melanie",
+ "last": "Foster"
+ },
+ "type": "personal"
+ },
+ {
+ "id": 2,
+ "name": {
+ "first": "Hopkins",
+ "last": "Scott"
+ },
+ "type": "work"
+ }
+ ]
+ },
+ {
+ "_id": "54a43171d946b78703a0e076",
+ "user_id": 4,
+ "name": {
+ "first": "Elisabeth",
+ "last": "Brady"
+ },
+ "friends": [
+ {
+ "id": 0,
+ "name": {
+ "first": "Sofia",
+ "last": "Workman"
+ },
+ "type": "work"
+ },
+ {
+ "id": 1,
+ "name": {
+ "first": "Alisha",
+ "last": "Reilly"
+ },
+ "type": "work"
+ },
+ {
+ "id": 2,
+ "name": {
+ "first": "Ochoa",
+ "last": "Burch"
+ },
+ "type": "personal"
+ }
+ ]
+ },
+ {
+ "_id": "54a4317118abd7f1992464ee",
+ "user_id": 5,
+ "name": {
+ "first": "Pollard",
+ "last": "French"
+ },
+ "friends": [
+ {
+ "id": 0,
+ "name": {
+ "first": "Hollie",
+ "last": "Juarez"
+ },
+ "type": "personal"
+ },
+ {
+ "id": 1,
+ "name": {
+ "first": "Nelda",
+ "last": "Newton"
+ },
+ "type": "personal"
+ },
+ {
+ "id": 2,
+ "name": {
+ "first": "Yang",
+ "last": "Pace"
+ },
+ "type": "personal"
+ }
+ ]
+ },
+ {
+ "_id": "54a43171f139e63d6579121e",
+ "user_id": 6,
+ "name": {
+ "first": "Acevedo",
+ "last": "Morales"
+ },
+ "friends": [
+ {
+ "id": 0,
+ "name": {
+ "first": "Payne",
+ "last": "Berry"
+ },
+ "type": "personal"
+ },
+ {
+ "id": 1,
+ "name": {
+ "first": "Rene",
+ "last": "Valenzuela"
+ },
+ "type": "personal"
+ },
+ {
+ "id": 2,
+ "name": {
+ "first": "Dora",
+ "last": "Gallegos"
+ },
+ "type": "work"
+ }
+ ]
+ },
+ {
+ "_id": "54a431719783cef80876dde8",
+ "user_id": 7,
+ "name": {
+ "first": "Cervantes",
+ "last": "Marquez"
+ },
+ "friends": [
+ {
+ "id": 0,
+ "name": {
+ "first": "Maxwell",
+ "last": "Norman"
+ },
+ "type": "personal"
+ },
+ {
+ "id": 1,
+ "name": {
+ "first": "Shields",
+ "last": "Bass"
+ },
+ "type": "personal"
+ },
+ {
+ "id": 2,
+ "name": {
+ "first": "Luz",
+ "last": "Jacobson"
+ },
+ "type": "work"
+ }
+ ]
+ },
+ {
+ "_id": "54a43171ecc7540d1f7aceae",
+ "user_id": 8,
+ "name": {
+ "first": "West",
+ "last": "Morrow"
+ },
+ "friends": [
+ {
+ "id": 0,
+ "name": {
+ "first": "Townsend",
+ "last": "Dixon"
+ },
+ "type": "personal"
+ },
+ {
+ "id": 1,
+ "name": {
+ "first": "Callahan",
+ "last": "Buck"
+ },
+ "type": "personal"
+ },
+ {
+ "id": 2,
+ "name": {
+ "first": "Rachel",
+ "last": "Fletcher"
+ },
+ "type": "personal"
+ }
+ ]
+ },
+ {
+ "_id": "54a4317113e831f4af041a0a",
+ "user_id": 9,
+ "name": {
+ "first": "Cotton",
+ "last": "House"
+ },
+ "friends": [
+ {
+ "id": 0,
+ "name": {
+ "first": "Mckenzie",
+ "last": "Medina"
+ },
+ "type": "personal"
+ },
+ {
+ "id": 1,
+ "name": {
+ "first": "Cecilia",
+ "last": "Miles"
+ },
+ "type": "work"
+ },
+ {
+ "id": 2,
+ "name": {
+ "first": "Guerra",
+ "last": "Cervantes"
+ },
+ "type": "work"
+ }
+ ]
+ },
+ {
+ "_id": "54a43171686eb1f48ebcbe01",
+ "user_id": 10,
+ "name": {
+ "first": "Wright",
+ "last": "Rivas"
+ },
+ "friends": [
+ {
+ "id": 0,
+ "name": {
+ "first": "Campos",
+ "last": "Freeman"
+ },
+ "type": "personal"
+ },
+ {
+ "id": 1,
+ "name": {
+ "first": "Christian",
+ "last": "Ferguson"
+ },
+ "type": "personal"
+ },
+ {
+ "id": 2,
+ "name": {
+ "first": "Doreen",
+ "last": "Wilder"
+ },
+ "type": "work"
+ }
+ ]
+ },
+ {
+ "_id": "54a43171a4f3d5638c162f4f",
+ "user_id": 11,
+ "name": {
+ "first": "Lorene",
+ "last": "Dorsey"
+ },
+ "friends": [
+ {
+ "id": 0,
+ "name": {
+ "first": "Gibbs",
+ "last": "Mccarty"
+ },
+ "type": "personal"
+ },
+ {
+ "id": 1,
+ "name": {
+ "first": "Neal",
+ "last": "Franklin"
+ },
+ "type": "work"
+ },
+ {
+ "id": 2,
+ "name": {
+ "first": "Kristy",
+ "last": "Head"
+ },
+ "type": "personal"
+ }
+ ],
+ "bestfriends" : ["Wolverine", "Cyclops"]
+ },
+ {
+ "_id": "54a431719faa420a5b4fbeb0",
+ "user_id": 12,
+ "name": {
+ "first": "Juanita",
+ "last": "Cook"
+ },
+ "friends": [
+ {
+ "id": 0,
+ "name": {
+ "first": "Wilkins",
+ "last": "Chang"
+ },
+ "type": "work"
+ },
+ {
+ "id": 1,
+ "name": {
+ "first": "Haney",
+ "last": "Rivera"
+ },
+ "type": "work"
+ },
+ {
+ "id": 2,
+ "name": {
+ "first": "Lauren",
+ "last": "Manning"
+ },
+ "type": "work"
+ }
+ ]
+ },
+ {
+ "_id": "54a43171e65d35f9ee8c53c0",
+ "user_id": 13,
+ "name": {
+ "first": "Levy",
+ "last": "Osborn"
+ },
+ "friends": [
+ {
+ "id": 0,
+ "name": {
+ "first": "Vinson",
+ "last": "Vargas"
+ },
+ "type": "work"
+ },
+ {
+ "id": 1,
+ "name": {
+ "first": "Felicia",
+ "last": "Beach"
+ },
+ "type": "work"
+ },
+ {
+ "id": 2,
+ "name": {
+ "first": "Nadine",
+ "last": "Kemp"
+ },
+ "type": "work"
+ }
+ ],
+ "results": [ 82, 85, 88 ]
+ },
+ {
+ "_id": "54a4317132f2c81561833259",
+ "user_id": 14,
+ "name": {
+ "first": "Christina",
+ "last": "Raymond"
+ },
+ "friends": [
+ {
+ "id": 0,
+ "name": {
+ "first": "Herrera",
+ "last": "Walton"
+ },
+ "type": "work"
+ },
+ {
+ "id": 1,
+ "name": {
+ "first": "Hahn",
+ "last": "Rutledge"
+ },
+ "type": "work"
+ },
+ {
+ "id": 2,
+ "name": {
+ "first": "Stacie",
+ "last": "Harding"
+ },
+ "type": "work"
+ }
+ ]
+ },
+ {
+ "_id": "589f32af493145f890e1b051",
+ "user_id": 15,
+ "name": {
+ "first": "Tanisha",
+ "last": "Bowers"
+ },
+ "friends": [
+ {
+ "id": 0,
+ "name": {
+ "first": "Ochoa",
+ "last": "Pratt"
+ },
+ "type": "personal"
+ },
+ {
+ "id": 1,
+ "name": {
+ "first": "Ochoa",
+ "last": "Romero"
+ },
+ "type": "personal"
+ },
+ {
+ "id": 2,
+ "name": {
+ "first": "Ochoa",
+ "last": "Bowman"
+ },
+ "type": "work"
+ }
+ ]
+ }
+]
diff --git a/src/mango/test/limit_docs.py b/src/mango/test/limit_docs.py
new file mode 100644
index 000000000..53ab5232d
--- /dev/null
+++ b/src/mango/test/limit_docs.py
@@ -0,0 +1,408 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+import copy
+
+
+def setup(db, index_type="view"):
+ db.recreate()
+ db.save_docs(copy.deepcopy(DOCS))
+ if index_type == "view":
+ add_view_indexes(db)
+ elif index_type == "text":
+ add_text_indexes(db)
+
+
+def add_text_indexes(db):
+ db.create_text_index()
+
+
+DOCS = [
+ {
+ "_id": "54af50626de419f5109c962f",
+ "user_id": 0,
+ "age": 10
+ },
+ {
+ "_id": "54af50622071121b25402dc3",
+ "user_id": 1,
+ "age": 11
+
+ },
+ {
+ "_id": "54af50623809e19159a3cdd0",
+ "user_id": 2,
+ "age": 12
+ },
+ {
+ "_id": "54af50629f45a0f49a441d01",
+ "user_id": 3,
+ "age": 13
+
+ },
+ {
+ "_id": "54af50620f1755c22359a362",
+ "user_id": 4,
+ "age": 14
+ },
+ {
+ "_id": "54af5062dd6f6c689ad2ca23",
+ "user_id": 5,
+ "age": 15
+ },
+ {
+ "_id": "54af50623e89b432be1187b8",
+ "user_id": 6,
+ "age": 16
+ },
+ {
+ "_id": "54af5062932a00270a3b5ab0",
+ "user_id": 7,
+ "age": 17
+
+ },
+ {
+ "_id": "54af5062df773d69174e3345",
+ "filtered_array" : [1, 2, 3],
+ "age": 18
+ },
+ {
+ "_id": "54af50629c1153b9e21e346d",
+ "filtered_array" : [1, 2, 3],
+ "age": 19
+ },
+ {
+ "_id": "54af5062dabb7cc4b60e0c95",
+ "user_id": 10,
+ "age": 20
+ },
+ {
+ "_id": "54af5062204996970a4439a2",
+ "user_id": 11,
+ "age": 21
+ },
+ {
+ "_id": "54af50629cea39e8ea52bfac",
+ "user_id": 12,
+ "age": 22
+ },
+ {
+ "_id": "54af50620597c094f75db2a1",
+ "user_id": 13,
+ "age": 23
+ },
+ {
+ "_id": "54af50628d4048de0010723c",
+ "user_id": 14,
+ "age": 24
+ },
+ {
+ "_id": "54af5062f339b6f44f52faf6",
+ "user_id": 15,
+ "age": 25
+ },
+ {
+ "_id": "54af5062a893f17ea4402031",
+ "user_id": 16,
+ "age": 26
+ },
+ {
+ "_id": "54af5062323dbc7077deb60a",
+ "user_id": 17,
+ "age": 27
+ },
+ {
+ "_id": "54af506224db85bd7fcd0243",
+ "filtered_array" : [1, 2, 3],
+ "age": 28
+ },
+ {
+ "_id": "54af506255bb551c9cc251bf",
+ "filtered_array" : [1, 2, 3],
+ "age": 29
+ },
+ {
+ "_id": "54af50625a97394e07d718a1",
+ "filtered_array" : [1, 2, 3],
+ "age": 30
+ },
+ {
+ "_id": "54af506223f51d586b4ef529",
+ "user_id": 21,
+ "age": 31
+ },
+ {
+ "_id": "54af50622740dede7d6117b7",
+ "user_id": 22,
+ "age": 32
+ },
+ {
+ "_id": "54af50624efc87684a52e8fb",
+ "user_id": 23,
+ "age": 33
+ },
+ {
+ "_id": "54af5062f40932760347799c",
+ "user_id": 24,
+ "age": 34
+ },
+ {
+ "_id": "54af5062d9f7361951ac645d",
+ "user_id": 25,
+ "age": 35
+ },
+ {
+ "_id": "54af5062f89aef302b37c3bc",
+ "filtered_array" : [1, 2, 3],
+ "age": 36
+ },
+ {
+ "_id": "54af5062498ec905dcb351f8",
+ "filtered_array" : [1, 2, 3],
+ "age": 37
+ },
+ {
+ "_id": "54af5062b1d2f2c5a85bdd7e",
+ "user_id": 28,
+ "age": 38
+ },
+ {
+ "_id": "54af50625061029c0dd942b5",
+ "filtered_array" : [1, 2, 3],
+ "age": 39
+ },
+ {
+ "_id": "54af50628b0d08a1d23c030a",
+ "user_id": 30,
+ "age": 40
+ },
+ {
+ "_id": "54af506271b6e3119eb31d46",
+ "filtered_array" : [1, 2, 3],
+ "age": 41
+ },
+ {
+ "_id": "54af5062b69f46424dfcf3e5",
+ "user_id": 32,
+ "age": 42
+ },
+ {
+ "_id": "54af5062ed00c7dbe4d1bdcf",
+ "user_id": 33,
+ "age": 43
+ },
+ {
+ "_id": "54af5062fb64e45180c9a90d",
+ "user_id": 34,
+ "age": 44
+ },
+ {
+ "_id": "54af5062241c72b067127b09",
+ "user_id": 35,
+ "age": 45
+ },
+ {
+ "_id": "54af50626a467d8b781a6d06",
+ "user_id": 36,
+ "age": 46
+ },
+ {
+ "_id": "54af50620e992d60af03bf86",
+ "filtered_array" : [1, 2, 3],
+ "age": 47
+ },
+ {
+ "_id": "54af506254f992aa3c51532f",
+ "user_id": 38,
+ "age": 48
+ },
+ {
+ "_id": "54af5062e99b20f301de39b9",
+ "user_id": 39,
+ "age": 49
+ },
+ {
+ "_id": "54af50624fbade6b11505b5d",
+ "user_id": 40,
+ "age": 50
+ },
+ {
+ "_id": "54af506278ad79b21e807ae4",
+ "user_id": 41,
+ "age": 51
+ },
+ {
+ "_id": "54af5062fc7a1dcb33f31d08",
+ "user_id": 42,
+ "age": 52
+ },
+ {
+ "_id": "54af5062ea2c954c650009cf",
+ "user_id": 43,
+ "age": 53
+ },
+ {
+ "_id": "54af506213576c2f09858266",
+ "user_id": 44,
+ "age": 54
+ },
+ {
+ "_id": "54af50624a05ac34c994b1c0",
+ "user_id": 45,
+ "age": 55
+ },
+ {
+ "_id": "54af50625a624983edf2087e",
+ "user_id": 46,
+ "age": 56
+ },
+ {
+ "_id": "54af50623de488c49d064355",
+ "user_id": 47,
+ "age": 57
+ },
+ {
+ "_id": "54af5062628b5df08661a9d5",
+ "user_id": 48,
+ "age": 58
+ },
+ {
+ "_id": "54af50620c706fc23032ae62",
+ "user_id": 49,
+ "age": 59
+ },
+ {
+ "_id": "54af5062509f1e2371fe1da4",
+ "user_id": 50,
+ "age": 60
+ },
+ {
+ "_id": "54af50625e96b22436791653",
+ "user_id": 51,
+ "age": 61
+ },
+ {
+ "_id": "54af5062a9cb71463bb9577f",
+ "user_id": 52,
+ "age": 62
+ },
+ {
+ "_id": "54af50624fea77a4221a4baf",
+ "user_id": 53,
+ "age": 63
+ },
+ {
+ "_id": "54af5062c63df0a147d2417e",
+ "user_id": 54,
+ "age": 64
+ },
+ {
+ "_id": "54af50623c56d78029316c9f",
+ "user_id": 55,
+ "age": 65
+ },
+ {
+ "_id": "54af5062167f6e13aa0dd014",
+ "user_id": 56,
+ "age": 66
+ },
+ {
+ "_id": "54af50621558abe77797d137",
+ "filtered_array" : [1, 2, 3],
+ "age": 67
+ },
+ {
+ "_id": "54af50624d5b36aa7cb5fa77",
+ "user_id": 58,
+ "age": 68
+ },
+ {
+ "_id": "54af50620d79118184ae66bd",
+ "user_id": 59,
+ "age": 69
+ },
+ {
+ "_id": "54af5062d18aafa5c4ca4935",
+ "user_id": 60,
+ "age": 71
+ },
+ {
+ "_id": "54af5062fd22a409649962f4",
+ "filtered_array" : [1, 2, 3],
+ "age": 72
+ },
+ {
+ "_id": "54af5062e31045a1908e89f9",
+ "user_id": 62,
+ "age": 73
+ },
+ {
+ "_id": "54af50624c062fcb4c59398b",
+ "user_id": 63,
+ "age": 74
+ },
+ {
+ "_id": "54af506241ec83430a15957f",
+ "user_id": 64,
+ "age": 75
+ },
+ {
+ "_id": "54af506224d0f888ae411101",
+ "user_id": 65,
+ "age": 76
+ },
+ {
+ "_id": "54af506272a971c6cf3ab6b8",
+ "user_id": 66,
+ "age": 77
+ },
+ {
+ "_id": "54af506221e25b485c95355b",
+ "user_id": 67,
+ "age": 78
+ },
+ {
+ "_id": "54af5062800f7f2ca73e9623",
+ "user_id": 68,
+ "age": 79
+ },
+ {
+ "_id": "54af5062bc962da30740534a",
+ "user_id": 69,
+ "age": 80
+ },
+ {
+ "_id": "54af50625102d6e210fc2efd",
+ "filtered_array" : [1, 2, 3],
+ "age": 81
+ },
+ {
+ "_id": "54af5062e014b9d039f02c5e",
+ "user_id": 71,
+ "age": 82
+ },
+ {
+ "_id": "54af5062fbd5e801dd217515",
+ "user_id": 72,
+ "age": 83
+ },
+ {
+ "_id": "54af50629971992b658fcb88",
+ "user_id": 73,
+ "age": 84
+ },
+ {
+ "_id": "54af5062607d53416c30bafd",
+ "filtered_array" : [1, 2, 3],
+ "age": 85
+ }
+]
diff --git a/src/mango/test/mango.py b/src/mango/test/mango.py
new file mode 100644
index 000000000..da51180b1
--- /dev/null
+++ b/src/mango/test/mango.py
@@ -0,0 +1,245 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+import json
+import time
+import unittest
+import uuid
+import os
+
+import requests
+
+import friend_docs
+import user_docs
+import limit_docs
+
+
+def random_db_name():
+ return "mango_test_" + uuid.uuid4().hex
+
+def has_text_service():
+ return os.path.isfile(os.getcwd() + "/../src/mango_cursor_text.erl")
+
+
+class Database(object):
+ def __init__(self, host, port, dbname, auth=None):
+ self.host = host
+ self.port = port
+ self.dbname = dbname
+ self.sess = requests.session()
+ self.sess.auth = ('testuser', 'testpass')
+ self.sess.headers["Content-Type"] = "application/json"
+
+ @property
+ def url(self):
+ return "http://{}:{}/{}".format(self.host, self.port, self.dbname)
+
+ def path(self, parts):
+ if isinstance(parts, (str, unicode)):
+ parts = [parts]
+ return "/".join([self.url] + parts)
+
+ def create(self, q=1, n=3):
+ r = self.sess.get(self.url)
+ if r.status_code == 404:
+ r = self.sess.put(self.url, params={"q":q, "n": n})
+ r.raise_for_status()
+
+ def delete(self):
+ r = self.sess.delete(self.url)
+
+ def recreate(self):
+ self.delete()
+ time.sleep(1)
+ self.create()
+ time.sleep(1)
+
+ def save_doc(self, doc):
+ self.save_docs([doc])
+
+ def save_docs(self, docs, **kwargs):
+ body = json.dumps({"docs": docs})
+ r = self.sess.post(self.path("_bulk_docs"), data=body, params=kwargs)
+ r.raise_for_status()
+ for doc, result in zip(docs, r.json()):
+ doc["_id"] = result["id"]
+ doc["_rev"] = result["rev"]
+
+ def open_doc(self, docid):
+ r = self.sess.get(self.path(docid))
+ r.raise_for_status()
+ return r.json()
+
+ def ddoc_info(self, ddocid):
+ r = self.sess.get(self.path([ddocid, "_info"]))
+ r.raise_for_status()
+ return r.json()
+
+ def create_index(self, fields, idx_type="json", name=None, ddoc=None):
+ body = {
+ "index": {
+ "fields": fields
+ },
+ "type": idx_type,
+ "w": 3
+ }
+ if name is not None:
+ body["name"] = name
+ if ddoc is not None:
+ body["ddoc"] = ddoc
+ body = json.dumps(body)
+ r = self.sess.post(self.path("_index"), data=body)
+ r.raise_for_status()
+ assert r.json()["id"] is not None
+ assert r.json()["name"] is not None
+ return r.json()["result"] == "created"
+
+ def create_text_index(self, analyzer=None, selector=None, idx_type="text",
+ default_field=None, fields=None, name=None, ddoc=None,index_array_lengths=None):
+ body = {
+ "index": {
+ },
+ "type": idx_type,
+ "w": 3,
+ }
+ if name is not None:
+ body["name"] = name
+ if analyzer is not None:
+ body["index"]["default_analyzer"] = analyzer
+ if default_field is not None:
+ body["index"]["default_field"] = default_field
+ if index_array_lengths is not None:
+ body["index"]["index_array_lengths"] = index_array_lengths
+ if selector is not None:
+ body["selector"] = selector
+ if fields is not None:
+ body["index"]["fields"] = fields
+ if ddoc is not None:
+ body["ddoc"] = ddoc
+ body = json.dumps(body)
+ r = self.sess.post(self.path("_index"), data=body)
+ r.raise_for_status()
+ return r.json()["result"] == "created"
+
+ def list_indexes(self, limit="", skip=""):
+ if limit != "":
+ limit = "limit=" + str(limit)
+ if skip != "":
+ skip = "skip=" + str(skip)
+ r = self.sess.get(self.path("_index?"+limit+";"+skip))
+ r.raise_for_status()
+ return r.json()["indexes"]
+
+ def delete_index(self, ddocid, name, idx_type="json"):
+ path = ["_index", ddocid, idx_type, name]
+ r = self.sess.delete(self.path(path), params={"w":"3"})
+ r.raise_for_status()
+
+ def bulk_delete(self, docs):
+ body = {
+ "docids" : docs,
+ "w": 3
+ }
+ body = json.dumps(body)
+ r = self.sess.post(self.path("_index/_bulk_delete"), data=body)
+ return r.json()
+
+ def find(self, selector, limit=25, skip=0, sort=None, fields=None,
+ r=1, conflicts=False, use_index=None, explain=False,
+ bookmark=None, return_raw=False):
+ body = {
+ "selector": selector,
+ "use_index": use_index,
+ "limit": limit,
+ "skip": skip,
+ "r": r,
+ "conflicts": conflicts
+ }
+ if sort is not None:
+ body["sort"] = sort
+ if fields is not None:
+ body["fields"] = fields
+ if bookmark is not None:
+ body["bookmark"] = bookmark
+ body = json.dumps(body)
+ if explain:
+ path = self.path("_explain")
+ else:
+ path = self.path("_find")
+ r = self.sess.post(path, data=body)
+ r.raise_for_status()
+ if explain or return_raw:
+ return r.json()
+ else:
+ return r.json()["docs"]
+
+ def find_one(self, *args, **kwargs):
+ results = self.find(*args, **kwargs)
+ if len(results) > 1:
+ raise RuntimeError("Multiple results for Database.find_one")
+ if len(results):
+ return results[0]
+ else:
+ return None
+
+
+class DbPerClass(unittest.TestCase):
+
+ @classmethod
+ def setUpClass(klass):
+ klass.db = Database("127.0.0.1", "15984", random_db_name())
+ klass.db.create(q=1, n=3)
+
+ def setUp(self):
+ self.db = self.__class__.db
+
+
+class UserDocsTests(DbPerClass):
+
+ @classmethod
+ def setUpClass(klass):
+ super(UserDocsTests, klass).setUpClass()
+ user_docs.setup(klass.db)
+
+
+class UserDocsTextTests(DbPerClass):
+
+ DEFAULT_FIELD = None
+ FIELDS = None
+
+ @classmethod
+ def setUpClass(klass):
+ super(UserDocsTextTests, klass).setUpClass()
+ if has_text_service():
+ user_docs.setup(
+ klass.db,
+ index_type="text",
+ default_field=klass.DEFAULT_FIELD,
+ fields=klass.FIELDS
+ )
+
+
+class FriendDocsTextTests(DbPerClass):
+
+ @classmethod
+ def setUpClass(klass):
+ super(FriendDocsTextTests, klass).setUpClass()
+ if has_text_service():
+ friend_docs.setup(klass.db, index_type="text")
+
+class LimitDocsTextTests(DbPerClass):
+
+ @classmethod
+ def setUpClass(klass):
+ super(LimitDocsTextTests, klass).setUpClass()
+ if has_text_service():
+ limit_docs.setup(klass.db, index_type="text")
diff --git a/src/mango/test/user_docs.py b/src/mango/test/user_docs.py
new file mode 100644
index 000000000..e2f1705b0
--- /dev/null
+++ b/src/mango/test/user_docs.py
@@ -0,0 +1,490 @@
+# -*- coding: utf-8 -*-
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+"""
+Generated with http://www.json-generator.com/
+
+With this pattern:
+
+[
+ '{{repeat(20)}}',
+ {
+ _id: '{{guid()}}',
+ user_id: "{{index()}}",
+ name: {
+ first: "{{firstName()}}",
+ last: "{{surname()}}"
+ },
+ age: "{{integer(18,90)}}",
+ location: {
+ state: "{{state()}}",
+ city: "{{city()}}",
+ address: {
+ street: "{{street()}}",
+ number: "{{integer(10, 10000)}}"
+ }
+ },
+ company: "{{company()}}",
+ email: "{{email()}}",
+ manager: "{{bool()}}",
+ twitter: function(tags) {
+ if(this.manager)
+ return;
+ return "@" + this.email.split("@")[0];
+ },
+ favorites: [
+ "{{repeat(2,5)}}",
+ "{{random('C', 'C++', 'Python', 'Ruby', 'Erlang', 'Lisp')}}"
+ ]
+ }
+]
+"""
+
+
+import copy
+
+
+def setup(db, index_type="view", **kwargs):
+ db.recreate()
+ db.save_docs(copy.deepcopy(DOCS))
+ if index_type == "view":
+ add_view_indexes(db, kwargs)
+ elif index_type == "text":
+ add_text_indexes(db, kwargs)
+
+
+def add_view_indexes(db, kwargs):
+ indexes = [
+ ["user_id"],
+ ["name.last", "name.first"],
+ ["age"],
+ [
+ "location.state",
+ "location.city",
+ "location.address.street",
+ "location.address.number"
+ ],
+ ["company", "manager"],
+ ["manager"],
+ ["favorites"],
+ ["favorites.3"],
+ ["twitter"]
+ ]
+ for idx in indexes:
+ assert db.create_index(idx) is True
+
+
+def add_text_indexes(db, kwargs):
+ db.create_text_index(**kwargs)
+
+
+DOCS = [
+ {
+ "_id": "71562648-6acb-42bc-a182-df6b1f005b09",
+ "user_id": 0,
+ "name": {
+ "first": "Stephanie",
+ "last": "Kirkland"
+ },
+ "age": 48,
+ "location": {
+ "state": "Nevada",
+ "city": "Ronco",
+ "address": {
+ "street": "Evergreen Avenue",
+ "number": 347
+ }
+ },
+ "company": "Dreamia",
+ "email": "stephaniekirkland@dreamia.com",
+ "manager": False,
+ "twitter": "@stephaniekirkland",
+ "favorites": [
+ "Ruby",
+ "C",
+ "Python"
+ ],
+ "test" : [{"a":1}, {"b":2}]
+ },
+ {
+ "_id": "12a2800c-4fe2-45a8-8d78-c084f4e242a9",
+ "user_id": 1,
+ "name": {
+ "first": "Abbott",
+ "last": "Watson"
+ },
+ "age": 31,
+ "location": {
+ "state": "Connecticut",
+ "city": "Gerber",
+ "address": {
+ "street": "Huntington Street",
+ "number": 8987
+ }
+ },
+ "company": "Talkola",
+ "email": "abbottwatson@talkola.com",
+ "manager": False,
+ "twitter": "@abbottwatson",
+ "favorites": [
+ "Ruby",
+ "Python",
+ "C",
+ {"Versions": {"Alpha": "Beta"}}
+ ],
+ "test" : [{"a":1, "b":2}]
+ },
+ {
+ "_id": "48ca0455-8bd0-473f-9ae2-459e42e3edd1",
+ "user_id": 2,
+ "name": {
+ "first": "Shelly",
+ "last": "Ewing"
+ },
+ "age": 42,
+ "location": {
+ "state": "New Mexico",
+ "city": "Thornport",
+ "address": {
+ "street": "Miller Avenue",
+ "number": 7100
+ }
+ },
+ "company": "Zialactic",
+ "email": "shellyewing@zialactic.com",
+ "manager": True,
+ "favorites": [
+ "Lisp",
+ "Python",
+ "Erlang"
+ ],
+ "test_in": {"val1" : 1, "val2": "val2"}
+ },
+ {
+ "_id": "0461444c-e60a-457d-a4bb-b8d811853f21",
+ "user_id": 3,
+ "name": {
+ "first": "Madelyn",
+ "last": "Soto"
+ },
+ "age": 79,
+ "location": {
+ "state": "Utah",
+ "city": "Albany",
+ "address": {
+ "street": "Stockholm Street",
+ "number": 710
+ }
+ },
+ "company": "Tasmania",
+ "email": "madelynsoto@tasmania.com",
+ "manager": True,
+ "favorites": [[
+ "Lisp",
+ "Erlang",
+ "Python"
+ ],
+ "Erlang",
+ "C",
+ "Erlang"
+ ],
+ "11111": "number_field",
+ "22222": {"33333" : "nested_number_field"}
+ },
+ {
+ "_id": "8e1c90c0-ac18-4832-8081-40d14325bde0",
+ "user_id": 4,
+ "name": {
+ "first": "Nona",
+ "last": "Horton"
+ },
+ "age": 61,
+ "location": {
+ "state": "Georgia",
+ "city": "Corinne",
+ "address": {
+ "street": "Woodhull Street",
+ "number": 6845
+ }
+ },
+ "company": "Signidyne",
+ "email": "nonahorton@signidyne.com",
+ "manager": False,
+ "twitter": "@nonahorton",
+ "favorites": [
+ "Lisp",
+ "C",
+ "Ruby",
+ "Ruby"
+ ],
+ "name.first" : "name dot first"
+ },
+ {
+ "_id": "a33d5457-741a-4dce-a217-3eab28b24e3e",
+ "user_id": 5,
+ "name": {
+ "first": "Sheri",
+ "last": "Perkins"
+ },
+ "age": 73,
+ "location": {
+ "state": "Michigan",
+ "city": "Nutrioso",
+ "address": {
+ "street": "Bassett Avenue",
+ "number": 5648
+ }
+ },
+ "company": "Myopium",
+ "email": "sheriperkins@myopium.com",
+ "manager": True,
+ "favorites": [
+ "Lisp",
+ "Lisp"
+ ]
+ },
+ {
+ "_id": "b31dad3f-ae8b-4f86-8327-dfe8770beb27",
+ "user_id": 6,
+ "name": {
+ "first": "Tate",
+ "last": "Guy"
+ },
+ "age": 47,
+ "location": {
+ "state": "Illinois",
+ "city": "Helen",
+ "address": {
+ "street": "Schenck Court",
+ "number": 7392
+ }
+ },
+ "company": "Prosely",
+ "email": "tateguy@prosely.com",
+ "manager": True,
+ "favorites": [
+ "C",
+ "Lisp",
+ "Ruby",
+ "C"
+ ]
+ },
+ {
+ "_id": "659d0430-b1f4-413a-a6b7-9ea1ef071325",
+ "user_id": 7,
+ "name": {
+ "first": "Jewell",
+ "last": "Stafford"
+ },
+ "age": 33,
+ "location": {
+ "state": "Iowa",
+ "city": "Longbranch",
+ "address": {
+ "street": "Dodworth Street",
+ "number": 3949
+ }
+ },
+ "company": "Niquent",
+ "email": "jewellstafford@niquent.com",
+ "manager": True,
+ "favorites": [
+ "C",
+ "C",
+ "Ruby",
+ "Ruby",
+ "Erlang"
+ ],
+ "exists_field" : "should_exist1"
+
+ },
+ {
+ "_id": "6c0afcf1-e57e-421d-a03d-0c0717ebf843",
+ "user_id": 8,
+ "name": {
+ "first": "James",
+ "last": "Mcdaniel"
+ },
+ "age": 68,
+ "location": {
+ "state": "Maine",
+ "city": "Craig",
+ "address": {
+ "street": "Greene Avenue",
+ "number": 8776
+ }
+ },
+ "company": "Globoil",
+ "email": "jamesmcdaniel@globoil.com",
+ "manager": True,
+ "favorites": None,
+ "exists_field" : "should_exist2"
+ },
+ {
+ "_id": "954272af-d5ed-4039-a5eb-8ed57e9def01",
+ "user_id": 9,
+ "name": {
+ "first": "Ramona",
+ "last": "Floyd"
+ },
+ "age": 22,
+ "location": {
+ "state": "Missouri",
+ "city": "Foxworth",
+ "address": {
+ "street": "Lott Place",
+ "number": 1697
+ }
+ },
+ "company": "Manglo",
+ "email": "ramonafloyd@manglo.com",
+ "manager": True,
+ "favorites": [
+ "Lisp",
+ "Erlang",
+ "Python"
+ ],
+ "exists_array" : ["should", "exist", "array1"],
+ "complex_field_value" : "+-(){}[]^~&&*||\"\\/?:!"
+ },
+ {
+ "_id": "e900001d-bc48-48a6-9b1a-ac9a1f5d1a03",
+ "user_id": 10,
+ "name": {
+ "first": "Charmaine",
+ "last": "Mills"
+ },
+ "age": 43,
+ "location": {
+ "state": "New Hampshire",
+ "city": "Kiskimere",
+ "address": {
+ "street": "Nostrand Avenue",
+ "number": 4503
+ }
+ },
+ "company": "Lyria",
+ "email": "charmainemills@lyria.com",
+ "manager": True,
+ "favorites": [
+ "Erlang",
+ "Erlang"
+ ],
+ "exists_array" : ["should", "exist", "array2"]
+ },
+ {
+ "_id": "b06aadcf-cd0f-4ca6-9f7e-2c993e48d4c4",
+ "user_id": 11,
+ "name": {
+ "first": "Mathis",
+ "last": "Hernandez"
+ },
+ "age": 75,
+ "location": {
+ "state": "Hawaii",
+ "city": "Dupuyer",
+ "address": {
+ "street": "Bancroft Place",
+ "number": 2741
+ }
+ },
+ "company": "Affluex",
+ "email": "mathishernandez@affluex.com",
+ "manager": True,
+ "favorites": [
+ "Ruby",
+ "Lisp",
+ "C",
+ "C++",
+ "C++"
+ ],
+ "exists_object" : {"should": "object"}
+ },
+ {
+ "_id": "5b61abc1-a3d3-4092-b9d7-ced90e675536",
+ "user_id": 12,
+ "name": {
+ "first": "Patti",
+ "last": "Rosales"
+ },
+ "age": 71,
+ "location": {
+ "state": "Pennsylvania",
+ "city": "Juntura",
+ "address": {
+ "street": "Hunterfly Place",
+ "number": 7683
+ }
+ },
+ "company": "Oulu",
+ "email": "pattirosales@oulu.com",
+ "manager": True,
+ "favorites": [
+ "C",
+ "Python",
+ "Lisp"
+ ],
+ "exists_object" : {"another": "object"}
+ },
+ {
+ "_id": "b1e70402-8add-4068-af8f-b4f3d0feb049",
+ "user_id": 13,
+ "name": {
+ "first": "Whitley",
+ "last": "Harvey"
+ },
+ "age": 78,
+ "location": {
+ "state": "Minnesota",
+ "city": "Trail",
+ "address": {
+ "street": "Pleasant Place",
+ "number": 8766
+ }
+ },
+ "company": None,
+ "email": "whitleyharvey@fangold.com",
+ "manager": False,
+ "twitter": "@whitleyharvey",
+ "favorites": [
+ "C",
+ "Ruby",
+ "Ruby"
+ ]
+ },
+ {
+ "_id": "c78c529f-0b07-4947-90a6-d6b7ca81da62",
+ "user_id": 14,
+ "name": {
+ "first": "Faith",
+ "last": "Hess"
+ },
+ "age": 51,
+ "location": {
+ "state": "North Dakota",
+ "city": "Axis",
+ "address": {
+ "street": "Brightwater Avenue",
+ "number": 1106
+ }
+ },
+ "company": "Pharmex",
+ "email": "faithhess@pharmex.com",
+ "manager": True,
+ "favorites": [
+ "Erlang",
+ "Python",
+ "Lisp"
+ ]
+ }
+]