Initial in browser MR using PouchDB.mapreduce.js

5 files changed, 435 insertions, 1 deletions

diff --git a/src/fauxton/app/modules/pouchdb/base.js b/src/fauxton/app/modules/pouchdb/base.js
new file mode 100644
index 000000000..abc37937c
--- /dev/null
+++ b/src/fauxton/app/modules/pouchdb/base.js
@@ -0,0 +1,49 @@
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+define([
+  "app",
+
+  "api",
+
+  // Modules
+  "modules/pouchdb/pouchdb.mapreduce.js"
+],
+
+function(app, FauxtonAPI, MapReduce) {
+  var Pouch = {};
+  Pouch.MapReduce = MapReduce;
+
+  Pouch.runViewQuery = function(fun, docs) {
+    docs = [
+      {_id: 'test_doc_1', foo: 'bar-1'},
+      {_id: 'test_doc_2', foo: 'bar-2'},
+      {_id: 'test_doc_3', foo: 'bar-3'},
+      {_id: 'test_doc_4', foo: 'bar-4'},
+      {_id: 'test_doc_5', foo: 'bar-5'},
+      {_id: 'test_doc_6', foo: 'bar-6'},
+      {_id: 'test_doc_7', foo: 'bar-7'},
+      {_id: 'test_doc_8', foo: 'bar-8'},
+      {_id: 'test_doc_9', foo: 'bar-9'},
+      {_id: 'test_doc_10', foo: 'bar-10'}
+    ];
+
+    var deferred = FauxtonAPI.Deferred();
+    var complete = function(resp) {
+      console.log("COMPLETE TRIGGERED", arguments);
+    };
+
+    return Pouch.MapReduce.query(fun, {docs: docs, complete:complete});
+  };
+
+  return Pouch;
+});
diff --git a/src/fauxton/app/modules/pouchdb/pouch.collate.js b/src/fauxton/app/modules/pouchdb/pouch.collate.js
new file mode 100644
index 000000000..f930b2780
--- /dev/null
+++ b/src/fauxton/app/modules/pouchdb/pouch.collate.js
@@ -0,0 +1,106 @@
+/*
+(function() {
+  // a few hacks to get things in the right place for node.js
+  if (typeof module !== 'undefined' && module.exports) {
+    module.exports = Pouch;
+  }
+*/
+
+define([
+  "app",
+
+  "api",
+
+  // Modules
+  "modules/pouchdb/pouch.collate.js"
+],
+
+function(app, FauxtonAPI, Collate) {
+  var Pouch = {};
+
+  Pouch.collate = function(a, b) {
+    var ai = collationIndex(a);
+    var bi = collationIndex(b);
+    if ((ai - bi) !== 0) {
+      return ai - bi;
+    }
+    if (a === null) {
+      return 0;
+    }
+    if (typeof a === 'number') {
+      return a - b;
+    }
+    if (typeof a === 'boolean') {
+      return a < b ? -1 : 1;
+    }
+    if (typeof a === 'string') {
+      return stringCollate(a, b);
+    }
+    if (Array.isArray(a)) {
+      return arrayCollate(a, b);
+    }
+    if (typeof a === 'object') {
+      return objectCollate(a, b);
+    }
+  };
+
+  var stringCollate = function(a, b) {
+    // See: https://github.com/daleharvey/pouchdb/issues/40
+    // This is incompatible with the CouchDB implementation, but its the
+    // best we can do for now
+    return (a === b) ? 0 : ((a > b) ? 1 : -1);
+  };
+
+  var objectCollate = function(a, b) {
+    var ak = Object.keys(a), bk = Object.keys(b);
+    var len = Math.min(ak.length, bk.length);
+    for (var i = 0; i < len; i++) {
+      // First sort the keys
+      var sort = Pouch.collate(ak[i], bk[i]);
+      if (sort !== 0) {
+        return sort;
+      }
+      // if the keys are equal sort the values
+      sort = Pouch.collate(a[ak[i]], b[bk[i]]);
+      if (sort !== 0) {
+        return sort;
+      }
+
+    }
+    return (ak.length === bk.length) ? 0 :
+      (ak.length > bk.length) ? 1 : -1;
+  };
+
+  var arrayCollate = function(a, b) {
+    var len = Math.min(a.length, b.length);
+    for (var i = 0; i < len; i++) {
+      var sort = Pouch.collate(a[i], b[i]);
+      if (sort !== 0) {
+        return sort;
+      }
+    }
+    return (a.length === b.length) ? 0 :
+      (a.length > b.length) ? 1 : -1;
+  };
+
+  // The collation is defined by erlangs ordered terms
+  // the atoms null, true, false come first, then numbers, strings,
+  // arrays, then objects
+  var collationIndex = function(x) {
+    var id = ['boolean', 'number', 'string', 'object'];
+    if (id.indexOf(typeof x) !== -1) {
+      if (x === null) {
+        return 1;
+      }
+      return id.indexOf(typeof x) + 2;
+    }
+    if (Array.isArray(x)) {
+      return 4.5;
+    }
+  };
+
+  return Pouch;
+
+//}).call(this);
+
+});
diff --git a/src/fauxton/app/modules/pouchdb/pouchdb.mapreduce.js b/src/fauxton/app/modules/pouchdb/pouchdb.mapreduce.js
new file mode 100644
index 000000000..a1a47b4d7
--- /dev/null
+++ b/src/fauxton/app/modules/pouchdb/pouchdb.mapreduce.js
@@ -0,0 +1,277 @@
+/*global Pouch: true */
+
+//"use strict";
+
+// This is the first implementation of a basic plugin, we register the
+// plugin object with pouch and it is mixin'd to each database created
+// (regardless of adapter), adapters can override plugins by providing
+// their own implementation. functions on the plugin object that start
+// with _ are reserved function that are called by pouchdb for special
+// notifications.
+
+// If we wanted to store incremental views we can do it here by listening
+// to the changes feed (keeping track of our last update_seq between page loads)
+// and storing the result of the map function (possibly using the upcoming
+// extracted adapter functions)
+
+define([
+  "app",
+
+  "api",
+
+  // Modules
+  "modules/pouchdb/pouch.collate.js"
+],
+
+function(app, FauxtonAPI, Collate) {
+  var Pouch = {};
+  Pouch.collate = Collate.collate;
+
+  //var MapReduce = function(db) {
+  var MapReduce = function() {
+
+    function viewQuery(fun, options) {
+      console.log("IN VIEW QUERY");
+      if (!options.complete) {
+        return;
+      }
+
+      function sum(values) {
+        return values.reduce(function(a, b) { return a + b; }, 0);
+      }
+
+      var results = [];
+      var current = null;
+      var num_started= 0;
+      var completed= false;
+
+      var emit = function(key, val) {
+        console.log("IN EMIT: ", key, val, current);
+        var viewRow = {
+          id: current.doc._id,
+          key: key,
+          value: val
+        }; 
+        console.log("VIEW ROW: ", viewRow);
+
+        if (options.startkey && Pouch.collate(key, options.startkey) < 0) return;
+        if (options.endkey && Pouch.collate(key, options.endkey) > 0) return;
+        if (options.key && Pouch.collate(key, options.key) !== 0) return;
+        num_started++;
+        if (options.include_docs) {
+          // TODO:: FIX
+          throw({error: "Include Docs not supported"});
+          /*
+
+          //in this special case, join on _id (issue #106)
+          if (val && typeof val === 'object' && val._id){
+            db.get(val._id,
+                function(_, joined_doc){
+                  if (joined_doc) {
+                    viewRow.doc = joined_doc;
+                  }
+                  results.push(viewRow);
+                  checkComplete();
+                });
+            return;
+          } else {
+            viewRow.doc = current.doc;
+          }
+          */
+        }
+        console.log("EMITTING: ", viewRow);
+        results.push(viewRow);
+      };
+
+      // ugly way to make sure references to 'emit' in map/reduce bind to the
+      // above emit
+      eval('fun.map = ' + fun.map.toString() + ';');
+      if (fun.reduce) {
+        eval('fun.reduce = ' + fun.reduce.toString() + ';');
+      }
+
+      // exclude  _conflicts key by default
+      // or to use options.conflicts if it's set when called by db.query
+      var conflicts = ('conflicts' in options ? options.conflicts : false);
+
+      //only proceed once all documents are mapped and joined
+      var checkComplete= function(){
+        if (completed && results.length == num_started){
+          results.sort(function(a, b) {
+            return Pouch.collate(a.key, b.key);
+          });
+          if (options.descending) {
+            results.reverse();
+          }
+          if (options.reduce === false) {
+            return options.complete(null, {rows: results});
+          }
+
+          var groups = [];
+          results.forEach(function(e) {
+            var last = groups[groups.length-1] || null;
+            if (last && Pouch.collate(last.key[0][0], e.key) === 0) {
+              last.key.push([e.key, e.id]);
+              last.value.push(e.value);
+              return;
+            }
+            groups.push({key: [[e.key, e.id]], value: [e.value]});
+          });
+          groups.forEach(function(e) {
+            e.value = fun.reduce(e.key, e.value) || null;
+            e.key = e.key[0][0];
+          });
+          options.complete(null, {rows: groups});
+        }
+      };
+
+      if (options.docs) {
+        console.log("RUNNING MR ON DOCS: ", options.docs);
+        _.each(options.docs, function(doc) {
+          current = {doc: doc};
+          fun.map.call(this, doc);
+        }, this);
+        return options.complete(null, {rows: results});
+      } else {
+        console.log("COULD NOT FIND DOCS");
+        return false;
+      }
+
+      /*
+      db.changes({
+        conflicts: conflicts,
+        include_docs: true,
+        onChange: function(doc) {
+          if (!('deleted' in doc)) {
+            current = {doc: doc.doc};
+            fun.map.call(this, doc.doc);
+          }
+        },
+        complete: function() {
+          completed= true;
+          checkComplete();
+        }
+      });
+      */
+    }
+
+    /*
+    function httpQuery(fun, opts, callback) {
+
+      // List of parameters to add to the PUT request
+      var params = [];
+      var body = undefined;
+      var method = 'GET';
+
+      // If opts.reduce exists and is defined, then add it to the list
+      // of parameters.
+      // If reduce=false then the results are that of only the map function
+      // not the final result of map and reduce.
+      if (typeof opts.reduce !== 'undefined') {
+        params.push('reduce=' + opts.reduce);
+      }
+      if (typeof opts.include_docs !== 'undefined') {
+        params.push('include_docs=' + opts.include_docs);
+      }
+      if (typeof opts.limit !== 'undefined') {
+        params.push('limit=' + opts.limit);
+      }
+      if (typeof opts.descending !== 'undefined') {
+        params.push('descending=' + opts.descending);
+      }
+      if (typeof opts.startkey !== 'undefined') {
+        params.push('startkey=' + encodeURIComponent(JSON.stringify(opts.startkey)));
+      }
+      if (typeof opts.endkey !== 'undefined') {
+        params.push('endkey=' + encodeURIComponent(JSON.stringify(opts.endkey)));
+      }
+      if (typeof opts.key !== 'undefined') {
+        params.push('key=' + encodeURIComponent(JSON.stringify(opts.key)));
+      }
+
+      // If keys are supplied, issue a POST request to circumvent GET query string limits
+      // see http://wiki.apache.org/couchdb/HTTP_view_API#Querying_Options
+      if (typeof opts.keys !== 'undefined') {
+        method = 'POST';
+        body = JSON.stringify({keys:opts.keys});
+      }
+
+      // Format the list of parameters into a valid URI query string
+      params = params.join('&');
+      params = params === '' ? '' : '?' + params;
+
+      // We are referencing a query defined in the design doc
+      if (typeof fun === 'string') {
+        var parts = fun.split('/');
+        db.request({
+          method: method,
+          url: '_design/' + parts[0] + '/_view/' + parts[1] + params,
+          body: body
+        }, callback);
+        return;
+      }
+
+      // We are using a temporary view, terrible for performance but good for testing
+      var queryObject = JSON.parse(JSON.stringify(fun, function(key, val) {
+        if (typeof val === 'function') {
+          return val + ''; // implicitly `toString` it
+        }
+        return val;
+      }));
+
+      db.request({
+        method:'POST',
+        url: '_temp_view' + params,
+        body: queryObject
+      }, callback);
+    }
+    */
+
+    function query(fun, opts, callback) {
+      if (typeof opts === 'function') {
+        callback = opts;
+        opts = {};
+      }
+
+      if (callback) {
+        opts.complete = callback;
+      }
+
+      /*
+      if (db.type() === 'http') {
+        return httpQuery(fun, opts, callback);
+      }
+      */
+
+      if (typeof fun === 'object') {
+        console.log("RUNNING VIEW QUERY", fun, opts, arguments);
+        return viewQuery(fun, opts);
+      }
+
+      throw({error: "Shouldn't have gotten here"});
+
+      /*
+      var parts = fun.split('/');
+      db.get('_design/' + parts[0], function(err, doc) {
+        if (err) {
+          if (callback) callback(err);
+          return;
+        }
+        viewQuery({
+          map: doc.views[parts[1]].map,
+          reduce: doc.views[parts[1]].reduce
+        }, opts);
+      });
+      */
+    }
+
+    return {'query': query};
+  };
+
+  // Deletion is a noop since we dont store the results of the view
+  MapReduce._delete = function() { };
+
+  //Pouch.plugin('mapreduce', MapReduce);
+
+  return MapReduce();
+});
diff --git a/src/fauxton/app/router.js b/src/fauxton/app/router.js
index 3b74afc4d..dea55d52c 100644
--- a/src/fauxton/app/router.js
+++ b/src/fauxton/app/router.js
@@ -32,6 +32,7 @@ define([
   // Routes return the module that they define routes for
   "modules/databases/base",
   "modules/documents/base",
+  "modules/pouchdb/base",
 
 
   // this needs to be added as a plugin later
diff --git a/src/fauxton/grunt.js b/src/fauxton/grunt.js
index 428ec144a..b8a0643c4 100644
--- a/src/fauxton/grunt.js
+++ b/src/fauxton/grunt.js
@@ -63,7 +63,8 @@ module.exports = function(grunt) {
     // route.
     jshint: {
       options: {
-        scripturl: true
+        scripturl: true,
+        evil: true
       }
     },