able to add/delete/update mango fdb indexes

7 files changed, 252 insertions, 166 deletions

diff --git a/src/fabric/src/fabric2_fdb.erl b/src/fabric/src/fabric2_fdb.erl
index 723f14d1c..9771acb63 100644
--- a/src/fabric/src/fabric2_fdb.erl
+++ b/src/fabric/src/fabric2_fdb.erl
@@ -588,7 +588,10 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
     } = Doc,
 
     % Doc body
-
+    % Fetch the old doc body for the mango hooks later
+    PrevDoc = if OldWinner == not_found -> not_found; true ->
+        get_doc_body(Db, DocId, OldWinner)
+    end,
     ok = write_doc_body(Db, Doc),
 
     % Attachment bookkeeping
@@ -716,11 +719,9 @@ write_doc(#{} = Db0, Doc, NewWinner0, OldWinner, ToUpdate, ToRemove) ->
             end,
             incr_stat(Db, <<"doc_count">>, -1),
             incr_stat(Db, <<"doc_del_count">>, 1),
-            OldDoc = get_doc_body(Db, DocId, OldWinner),
-            mango_indexer:update(Db, deleted, not_found, OldDoc);
+            mango_indexer:update(Db, deleted, not_found, PrevDoc);
         updated ->
-            OldDoc = get_doc_body(Db, DocId, OldWinner),
-            mango_indexer:update(Db, updated, Doc, OldDoc)
+            mango_indexer:update(Db, updated, Doc, PrevDoc)
     end,
 
     ok.
diff --git a/src/mango/src/mango_fdb.erl b/src/mango/src/mango_fdb.erl
index dbd22fa32..def942fa3 100644
--- a/src/mango/src/mango_fdb.erl
+++ b/src/mango/src/mango_fdb.erl
@@ -23,6 +23,7 @@
 
 -export([
     query_all_docs/4,
+    remove_doc/3,
     write_doc/3,
     query/4
 ]).
@@ -143,6 +144,17 @@ fold_cb({Key, _}, Acc) ->
     end.
 
 
+remove_doc(TxDb, DocId, IdxResults) ->
+    lists:foreach(fun (IdxResult) ->
+        #{
+            ddoc_id := DDocId,
+            results := Results
+        } = IdxResult,
+        MangoIdxPrefix = mango_idx_prefix(TxDb, DDocId),
+        clear_key(TxDb, MangoIdxPrefix, Results, DocId)
+    end, IdxResults).
+
+
 write_doc(TxDb, DocId, IdxResults) ->
     lists:foreach(fun (IdxResult) ->
         #{
@@ -162,11 +174,23 @@ mango_idx_prefix(TxDb, Id) ->
     erlfdb_tuple:pack(Key, DbPrefix).
 
 
+create_key(MangoIdxPrefix, Results, DocId) ->
+    EncodedResults = couch_views_encoding:encode(Results, key),
+    erlfdb_tuple:pack({{EncodedResults, DocId}}, MangoIdxPrefix).
+
+
+clear_key(TxDb, MangoIdxPrefix, Results, DocId) ->
+    #{
+        tx := Tx
+    } = TxDb,
+    Key = create_key(MangoIdxPrefix, Results, DocId),
+    erlfdb:clear(Tx, Key).
+
+
 add_key(TxDb, MangoIdxPrefix, Results, DocId) ->
     #{
         tx := Tx
     } = TxDb,
-    EncodedResults = couch_views_encoding:encode(Results, key),
-    Key = erlfdb_tuple:pack({{EncodedResults, DocId}}, MangoIdxPrefix),
+    Key = create_key(MangoIdxPrefix, Results, DocId),
     erlfdb:set(Tx, Key, <<0>>).
 
diff --git a/src/mango/src/mango_indexer.erl b/src/mango/src/mango_indexer.erl
index 20af5bd58..0cb15f76c 100644
--- a/src/mango/src/mango_indexer.erl
+++ b/src/mango/src/mango_indexer.erl
@@ -22,37 +22,69 @@
 -include_lib("couch/include/couch_db.hrl").
 -include("mango_idx.hrl").
 
+
+update(Db, State, Doc, PrevDoc) ->
+    try
+        update_int(Db, State, Doc, PrevDoc)
+    catch
+        Error:Reason ->
+            io:format("ERROR ~p ~p ~p ~n", [Error, Reason, erlang:display(erlang:get_stacktrace())]),
+            #{
+                name := DbName
+            } = Db,
+
+            Id = case Doc of
+                not_found when is_record(PrevDoc, doc) ->
+                    #doc{id = DocId} = PrevDoc,
+                    DocId;
+                not_found ->
+                    <<"unknown_doc_id">>;
+                #doc{} ->
+                    #doc{id = DocId} = Doc,
+                    DocId
+            end,
+
+            couch_log:error("Mango index error for Db ~s Doc ~p ~p ~p",
+                [DbName, Id, Error, Reason])
+    end,
+    ok.
+
 % Design doc
 % Todo: Check if design doc is mango index and kick off background worker
 % to build new index
-update(Db, Change, #doc{id = <<?DESIGN_DOC_PREFIX, _/binary>>} = Doc, OldDoc) ->
+update_int(Db, State, #doc{id = <<?DESIGN_DOC_PREFIX, _/binary>>} = Doc, PrevDoc) ->
     io:format("DESIGN DOC SAVED ~p ~n", [Doc]),
     ok;
 
-update(Db, deleted, _, OldDoc)  ->
-    ok;
+update_int(Db, deleted, _, PrevDoc)  ->
+    Indexes = mango_idx:list(Db),
+    Indexes1 = filter_json_indexes(Indexes),
+    remove_doc(Db, PrevDoc, Indexes1);
 
-update(Db, updated, Doc, OldDoc) ->
-    ok;
+update_int(Db, updated, Doc, PrevDoc) ->
+    Indexes = mango_idx:list(Db),
+    Indexes1 = filter_json_indexes(Indexes),
+    remove_doc(Db, PrevDoc, Indexes1),
+    write_doc(Db, Doc, Indexes1);
 
-update(Db, created, Doc, _) ->
-    try
-        io:format("CREATED ~p ~n", [Doc]),
-        #doc{id = DocId} = Doc,
-        Indexes = mango_idx:list(Db),
-        Indexes1 = filter_json_indexes(Indexes),
-        io:format("UPDATE INDEXES ~p ~n filtered ~p ~n", [Indexes, Indexes1]),
-        JSONDoc = mango_json:to_binary(couch_doc:to_json_obj(Doc, [])),
-        io:format("DOC ~p ~n", [Doc]),
-        Results = index_doc(Indexes1, JSONDoc),
-        io:format("Update ~p ~n, ~p ~n Results ~p ~n", [Doc, JSONDoc, Results]),
-        mango_fdb:write_doc(Db, DocId, Results)
-    catch
-        Error:Reason ->
-            io:format("ERROR ~p ~p ~p ~n", [Error, Reason, erlang:display(erlang:get_stacktrace())]),
-            ok
-    end,
-    ok.
+update_int(Db, created, Doc, _) ->
+    Indexes = mango_idx:list(Db),
+    Indexes1 = filter_json_indexes(Indexes),
+    write_doc(Db, Doc, Indexes1).
+
+
+remove_doc(Db, #doc{} = Doc, Indexes) ->
+    #doc{id = DocId} = Doc,
+    PrevJSONDoc = mango_json:to_binary(couch_doc:to_json_obj(Doc, [])),
+    PrevResults = index_doc(Indexes, PrevJSONDoc),
+    mango_fdb:remove_doc(Db, DocId, PrevResults).
+
+
+write_doc(Db, #doc{} = Doc, Indexes) ->
+    #doc{id = DocId} = Doc,
+    JSONDoc = mango_json:to_binary(couch_doc:to_json_obj(Doc, [])),
+    Results = index_doc(Indexes, JSONDoc),
+    mango_fdb:write_doc(Db, DocId, Results).
 
 
 filter_json_indexes(Indexes) ->
diff --git a/src/mango/test/21-fdb-indexing.py b/src/mango/test/21-fdb-indexing.py
deleted file mode 100644
index e1cfd9030..000000000
--- a/src/mango/test/21-fdb-indexing.py
+++ /dev/null
@@ -1,48 +0,0 @@
-# -*- coding: latin-1 -*-
-# Licensed under the Apache License, Version 2.0 (the "License"); you may not
-# use this file except in compliance with the License. You may obtain a copy of
-# the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations under
-# the License.
-
-import mango
-import copy
-
-DOCS = [
-    {"_id": "100", "name": "Jimi", "location": "AUS", "user_id": 1, "same": "value"},
-    {"_id": "200", "name": "Eddie", "location": "BRA", "user_id": 2, "same": "value"},
-    {"_id": "300", "name": "Harry", "location": "CAN", "user_id": 3, "same": "value"},
-    {"_id": "400", "name": "Eddie", "location": "DEN", "user_id": 4, "same": "value"},
-    {"_id": "500", "name": "Jones", "location": "ETH", "user_id": 5, "same": "value"}
-]
-
-class FdbIndexingTests(mango.DbPerClass):
-    def setUp(self):
-        self.db.recreate()
-        self.db.create_index(["name"], name="name")
-        self.db.save_docs(copy.deepcopy(DOCS))
-
-    def test_doc_update(self):
-        docs = self.db.find({"name": "Eddie"})
-        self.assertEqual(len(docs), 2)
-        self.assertEqual(docs[0]["_id"], "200")
-        self.assertEqual(docs[1]["_id"], "400")
-
-        doc = self.db.open_doc("400")
-        doc["name"] = "NotEddie"
-        self.db.save_doc(doc)
-
-        docs = self.db.find({"name": "Eddie"})
-        print("DD")
-        print(docs)
-        self.assertEqual(len(docs), 1)
-        self.assertEqual(docs[0]["_id"], "200")
-
-
-
diff --git a/src/mango/test/eunit/mango_indexer_test.erl b/src/mango/test/eunit/mango_indexer_test.erl
new file mode 100644
index 000000000..778caea50
--- /dev/null
+++ b/src/mango/test/eunit/mango_indexer_test.erl
@@ -0,0 +1,165 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(mango_indexer_test).
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("mango/src/mango_cursor.hrl").
+-include_lib("fabric/test/fabric2_test.hrl").
+
+
+indexer_test_() ->
+    {
+        "Test indexing",
+        {
+            setup,
+            fun setup/0,
+            fun cleanup/1,
+            {
+                foreach,
+                fun foreach_setup/0,
+                fun foreach_teardown/1,
+                [with([
+                    ?TDEF(index_docs),
+                    ?TDEF(update_doc),
+                    ?TDEF(delete_doc)
+                ])]
+            }
+        }
+    }.
+
+
+setup() ->
+    Ctx = test_util:start_couch([
+        fabric,
+        couch_jobs,
+        couch_js,
+        couch_views
+    ]),
+    Ctx.
+
+
+cleanup(Ctx) ->
+    test_util:stop_couch(Ctx).
+
+
+foreach_setup() ->
+    {ok, Db} = fabric2_db:create(?tempdb(), [{user_ctx, ?ADMIN_USER}]),
+
+    DDoc = create_idx_ddoc(Db),
+    fabric2_db:update_docs(Db, [DDoc]),
+
+    Docs = make_docs(3),
+    fabric2_db:update_docs(Db, Docs),
+    {Db, couch_doc:to_json_obj(DDoc, [])}.
+
+
+foreach_teardown({Db, _}) ->
+    ok = fabric2_db:delete(fabric2_db:name(Db), []).
+
+
+index_docs({Db, DDoc}) ->
+    Docs = run_query(Db, DDoc),
+    ?assertEqual([
+        [{id, <<"1">>}, {value, 1}],
+        [{id, <<"2">>}, {value, 2}],
+        [{id, <<"3">>}, {value, 3}]
+    ], Docs).
+
+update_doc({Db, DDoc}) ->
+    {ok, Doc} = fabric2_db:open_doc(Db, <<"2">>),
+    JsonDoc = couch_doc:to_json_obj(Doc, []),
+    JsonDoc2 = couch_util:json_apply_field({<<"value">>, 4}, JsonDoc),
+    Doc2 = couch_doc:from_json_obj(JsonDoc2),
+    fabric2_db:update_doc(Db, Doc2),
+
+    Docs = run_query(Db, DDoc),
+    ?assertEqual([
+        [{id, <<"1">>}, {value, 1}],
+        [{id, <<"3">>}, {value, 3}],
+        [{id, <<"2">>}, {value, 4}]
+    ], Docs).
+
+
+delete_doc({Db, DDoc}) ->
+    {ok, Doc} = fabric2_db:open_doc(Db, <<"2">>),
+    JsonDoc = couch_doc:to_json_obj(Doc, []),
+    JsonDoc2 = couch_util:json_apply_field({<<"_deleted">>, true}, JsonDoc),
+    Doc2 = couch_doc:from_json_obj(JsonDoc2),
+    fabric2_db:update_doc(Db, Doc2),
+
+    Docs = run_query(Db, DDoc),
+    ?assertEqual([
+        [{id, <<"1">>}, {value, 1}],
+        [{id, <<"3">>}, {value, 3}]
+    ], Docs).
+
+
+run_query(Db, DDoc) ->
+    Args = #{
+        start_key => [],
+        start_key_docid => <<>>,
+        end_key => [],
+        end_key_docid => <<255>>,
+        dir => fwd,
+        skip => 0
+    },
+    [Idx] = mango_idx:from_ddoc(Db, DDoc),
+    Cursor = #cursor{
+        db = Db,
+        index = Idx,
+        user_acc = []
+    },
+    {ok, Cursor1} = mango_fdb:query(Db, fun query_cb/2, Cursor, Args),
+    Acc = Cursor1#cursor.user_acc,
+    lists:map(fun ({Props}) ->
+        [
+            {id, couch_util:get_value(<<"_id">>, Props)},
+            {value, couch_util:get_value(<<"value">>, Props)}
+        ]
+
+    end, Acc).
+
+
+create_idx_ddoc(Db) ->
+    Opts = [
+        {def, {[{<<"fields">>,{[{<<"value">>,<<"asc">>}]}}]}},
+        {type, <<"json">>},
+        {name, <<"idx_01">>},
+        {ddoc, auto_name},
+        {w, 3},
+        {partitioned, db_default}
+    ],
+
+    {ok, Idx} = mango_idx:new(Db, Opts),
+    {ok, DDoc} = mango_util:load_ddoc(Db, mango_idx:ddoc(Idx), []),
+    {ok, NewDDoc} = mango_idx:add(DDoc, Idx),
+    NewDDoc.
+
+
+make_docs(Count) ->
+    [doc(I) || I <- lists:seq(1, Count)].
+
+
+doc(Id) ->
+    couch_doc:from_json_obj({[
+        {<<"_id">>, list_to_binary(integer_to_list(Id))},
+        {<<"value">>, Id}
+    ]}).
+
+
+query_cb({doc, Doc}, #cursor{user_acc = Acc} = Cursor) ->
+    {ok, Cursor#cursor{
+        user_acc =  Acc ++ [Doc]
+    }}.
+
diff --git a/src/mango/test/exunit/mango_indexer_test.exs b/src/mango/test/exunit/mango_indexer_test.exs
deleted file mode 100644
index f62f47ea9..000000000
--- a/src/mango/test/exunit/mango_indexer_test.exs
+++ /dev/null
@@ -1,86 +0,0 @@
-defmodule MangoIndexerTest do
-  use Couch.Test.ExUnit.Case
-
-  alias Couch.Test.Utils
-  alias Couch.Test.Setup
-  alias Couch.Test.Setup.Step
-
-  setup_all do
-    test_ctx = :test_util.start_couch([:couch_log, :fabric, :couch_js, :couch_jobs])
-
-    on_exit(fn ->
-      :test_util.stop_couch(test_ctx)
-    end)
-  end
-
-  setup do
-    db_name = Utils.random_name("db")
-
-    admin_ctx =
-      {:user_ctx,
-       Utils.erlang_record(:user_ctx, "couch/include/couch_db.hrl", roles: ["_admin"])}
-
-    {:ok, db} = :fabric2_db.create(db_name, [admin_ctx])
-
-    ddocs = create_ddocs()
-    idx_ddocs = create_indexes(db)
-    docs = create_docs()
-
-    IO.inspect(idx_ddocs)
-    {ok, _} = :fabric2_db.update_docs(db, ddocs ++ idx_ddocs)
-    {ok, _} = :fabric2_db.update_docs(db, docs)
-
-    on_exit(fn ->
-      :fabric2_db.delete(db_name, [admin_ctx])
-    end)
-
-    %{
-      db_name: db_name,
-      db: db,
-      ddoc: ddocs,
-      idx: idx_ddocs
-    }
-  end
-
-  test "update doc", context do
-    db = context[:db]
-  end
-
-  defp create_indexes(db) do
-    opts = [
-      {:def, {[{"fields", ["group", "value"]}]}},
-      {:type, "json"},
-      {:name, "idx_01"},
-      {:ddoc, :auto_name},
-      {:w, 3},
-      {:partitioned, :db_default}
-    ]
-
-    {:ok, idx} = :mango_idx.new(db, opts)
-    db_opts = [{:user_ctx, db["user_ctx"]}, :deleted, :ejson_body]
-    {:ok, ddoc} = :mango_util.load_ddoc(db, :mango_idx.ddoc(idx), db_opts)
-    {:ok, new_ddoc} = :mango_idx.add(ddoc, idx)
-    [new_ddoc]
-  end
-
-  defp create_docs() do
-    for i <- 1..1 do
-      group =
-        if rem(i, 3) == 0 do
-          "first"
-        else
-          "second"
-        end
-
-      :couch_doc.from_json_obj(
-        {[
-           {"_id", "doc-id-#{i}"},
-           {"value", i},
-           {"val_str", Integer.to_string(i, 8)},
-           {"some", "field"},
-           {"group", group}
-         ]}
-      )
-    end
-  end
-end
diff --git a/src/mango/test/exunit/test_helper.exs b/src/mango/test/exunit/test_helper.exs
deleted file mode 100644
index 314050085..000000000
--- a/src/mango/test/exunit/test_helper.exs
+++ /dev/null
@@ -1,2 +0,0 @@
-ExUnit.configure(formatters: [JUnitFormatter, ExUnit.CLIFormatter])
-ExUnit.start()