Add ability to black list indexes (#27)

We add the ability to black list search indexes. The implementation requires
couch_epi that generates a dynamic module to check for configuration values. 
A new config section, dreyfus_blacklist is added. Each key in the section will
be of the form "<db>.<groupid>. <indexname>". Setting this value, either via
remsh or via the _node/<node>/_config endpoint,  to "true", will disable the index.
Search requests will throw a 400 error, and indexing will not start. Index processes
that have already begun will be allowed to finish.

13 files changed, 333 insertions, 13 deletions

diff --git a/src/dreyfus_config.erl b/src/dreyfus_config.erl
new file mode 100644
index 000000000..8dbd62430
--- /dev/null
+++ b/src/dreyfus_config.erl
@@ -0,0 +1,10 @@
+ -module(dreyfus_config).
+
+ -export([data/0, get/1]).
+
+data() ->
+    config:get("dreyfus_blacklist").
+
+get(Key) ->
+    Handle = couch_epi:get_handle({dreyfus, black_list}),
+    couch_epi:get_value(Handle, dreyfus, Key).
diff --git a/src/dreyfus_epi.erl b/src/dreyfus_epi.erl
index 1c7a9f0b8..3cda975ea 100644
--- a/src/dreyfus_epi.erl
+++ b/src/dreyfus_epi.erl
@@ -12,6 +12,8 @@
     notify/3
 ]).
 
+-define(DATA_INTERVAL, 1000).
+
 app() ->
     dreyfus.
 
@@ -25,13 +27,19 @@ services() ->
     [].
 
 data_subscriptions() ->
-    [].
+    [{dreyfus, black_list}].
 
 data_providers() ->
-    [].
+    [
+        {{dreyfus, black_list}, {callback_module, dreyfus_config},
+            [{interval, ?DATA_INTERVAL}]}
+    ].
 
 processes() ->
     [].
 
 notify(_Key, _Old, _New) ->
-    ok.
+    Listeners = application:get_env(dreyfus, config_listeners, []),
+    lists:foreach(fun(L) ->
+        L ! dreyfus_config_change_finished
+    end, Listeners).
diff --git a/src/dreyfus_fabric_group1.erl b/src/dreyfus_fabric_group1.erl
index 79e041c00..a0b488737 100644
--- a/src/dreyfus_fabric_group1.erl
+++ b/src/dreyfus_fabric_group1.erl
@@ -32,6 +32,7 @@
 
 go(DbName, GroupId, IndexName, QueryArgs) when is_binary(GroupId) ->
     {ok, DDoc} = fabric:open_doc(DbName, <<"_design/", GroupId/binary>>, []),
+    dreyfus_util:maybe_deny_index(DbName, GroupId, IndexName),
     go(DbName, DDoc, IndexName, QueryArgs);
 
 go(DbName, DDoc, IndexName, #index_query_args{}=QueryArgs) ->
diff --git a/src/dreyfus_fabric_group2.erl b/src/dreyfus_fabric_group2.erl
index 6c2765163..33e099281 100644
--- a/src/dreyfus_fabric_group2.erl
+++ b/src/dreyfus_fabric_group2.erl
@@ -34,6 +34,7 @@
 
 go(DbName, GroupId, IndexName, QueryArgs) when is_binary(GroupId) ->
     {ok, DDoc} = fabric:open_doc(DbName, <<"_design/", GroupId/binary>>, []),
+    dreyfus_util:maybe_deny_index(DbName, GroupId, IndexName),
     go(DbName, DDoc, IndexName, QueryArgs);
 
 go(DbName, DDoc, IndexName, #index_query_args{}=QueryArgs) ->
diff --git a/src/dreyfus_fabric_info.erl b/src/dreyfus_fabric_info.erl
index 8d3877c66..6c5dd9af3 100644
--- a/src/dreyfus_fabric_info.erl
+++ b/src/dreyfus_fabric_info.erl
@@ -23,6 +23,7 @@
 
 go(DbName, DDocId, IndexName, InfoLevel) when is_binary(DDocId) ->
     {ok, DDoc} = fabric:open_doc(DbName, <<"_design/", DDocId/binary>>, []),
+    dreyfus_util:maybe_deny_index(DbName, DDocId, IndexName),
     go(DbName, DDoc, IndexName, InfoLevel);
 
 go(DbName, DDoc, IndexName, InfoLevel) ->
diff --git a/src/dreyfus_fabric_search.erl b/src/dreyfus_fabric_search.erl
index 2e1541c19..7a208012d 100644
--- a/src/dreyfus_fabric_search.erl
+++ b/src/dreyfus_fabric_search.erl
@@ -31,7 +31,9 @@
 }).
 
 go(DbName, GroupId, IndexName, QueryArgs) when is_binary(GroupId) ->
-    {ok, DDoc} = fabric:open_doc(DbName, <<"_design/", GroupId/binary>>, [ejson_body]),
+    {ok, DDoc} = fabric:open_doc(DbName, <<"_design/", GroupId/binary>>,
+        [ejson_body]),
+    dreyfus_util:maybe_deny_index(DbName, GroupId, IndexName),
     go(DbName, DDoc, IndexName, QueryArgs);
 
 go(DbName, DDoc, IndexName, #index_query_args{bookmark=nil}=QueryArgs) ->
diff --git a/src/dreyfus_index.erl b/src/dreyfus_index.erl
index 19dce4f5c..5f87ab5b6 100644
--- a/src/dreyfus_index.erl
+++ b/src/dreyfus_index.erl
@@ -120,21 +120,33 @@ init({DbName, Index}) ->
 
 handle_call({await, RequestSeq}, From,
             #state{
-                index=#index{current_seq=Seq}=Index,
+                index=#index{dbname=DbName,name=IdxName,ddoc_id=DDocId,current_seq=Seq}=Index,
                 index_pid=IndexPid,
                 updater_pid=nil,
                 waiting_list=WaitList
             }=State) when RequestSeq > Seq ->
-    UpPid = spawn_link(fun() -> dreyfus_index_updater:update(IndexPid, Index) end),
-    {noreply, State#state{
-        updater_pid=UpPid,
-        waiting_list=[{From,RequestSeq}|WaitList]
-    }};
+    DbName2 = mem3:dbname(DbName),
+    <<"_design/", GroupId/binary>> = DDocId,
+    NewState = case dreyfus_util:in_black_list(DbName2, GroupId, IdxName) of
+        false ->
+            UpPid = spawn_link(fun() ->
+                dreyfus_index_updater:update(IndexPid,Index)
+            end),
+            State#state{
+                updater_pid=UpPid,
+                waiting_list=[{From,RequestSeq}|WaitList]
+            };
+        _ ->
+            couch_log:notice("Index Blocked from Updating - db: ~p,"
+                " ddocid: ~p name: ~p", [DbName, DDocId, IdxName]),
+            State
+    end,
+    {noreply, NewState};
 handle_call({await, RequestSeq}, _From,
             #state{index=#index{current_seq=Seq}}=State) when RequestSeq =< Seq ->
     {reply, {ok, State#state.index_pid, Seq}, State};
 handle_call({await, RequestSeq}, From, #state{waiting_list=WaitList}=State) ->
-    {noreply, State#state{
+    {no_reply, State#state{
         waiting_list=[{From,RequestSeq}|WaitList]
     }};
 
@@ -162,7 +174,7 @@ handle_cast(_Msg, State) ->
 
 handle_info({'EXIT', FromPid, {updated, NewSeq}},
             #state{
-              index=Index0,
+              index=#index{dbname=DbName,name=IdxName,ddoc_id=DDocId}=Index0,
               index_pid=IndexPid,
               updater_pid=UpPid,
               waiting_list=WaitList
@@ -175,7 +187,18 @@ handle_info({'EXIT', FromPid, {updated, NewSeq}},
                               waiting_list=[]
                              }};
     StillWaiting ->
-        Pid = spawn_link(fun() -> dreyfus_index_updater:update(IndexPid, Index) end),
+        DbName2 = mem3:dbname(DbName),
+        <<"_design/", GroupId/binary>> = DDocId,
+        Pid = case dreyfus_util:in_black_list(DbName2, GroupId, IdxName) of
+            true ->
+                couch_log:notice("Index Blocked from Updating - db: ~p, ddocid: ~p"
+                    " name: ~p", [DbName, GroupId, IdxName]),
+                nil;
+            false ->
+                spawn_link(fun() ->
+                    dreyfus_index_updater:update(IndexPid, Index) 
+                end)
+        end,
         {noreply, State#state{index=Index,
                               updater_pid=Pid,
                               waiting_list=StillWaiting
diff --git a/src/dreyfus_sup.erl b/src/dreyfus_sup.erl
index f97e284d7..d855a822e 100644
--- a/src/dreyfus_sup.erl
+++ b/src/dreyfus_sup.erl
@@ -15,6 +15,7 @@
 
 -module(dreyfus_sup).
 -behaviour(supervisor).
+
 -export([start_link/0, init/1]).
 
 start_link() ->
diff --git a/src/dreyfus_util.erl b/src/dreyfus_util.erl
index b6f22679b..d6dffa9a6 100644
--- a/src/dreyfus_util.erl
+++ b/src/dreyfus_util.erl
@@ -20,6 +20,7 @@
 -include_lib("couch/include/couch_db.hrl").
 
 -export([get_shards/2, sort/2, upgrade/1, export/1, time/2]).
+-export([in_black_list/1, in_black_list/3, maybe_deny_index/3]).
 
 get_shards(DbName, #index_query_args{stale=ok}) ->
     mem3:ushards(DbName);
@@ -166,6 +167,33 @@ time(Metric, {M, F, A}) when is_list(Metric) ->
         couch_stats:update_histogram([dreyfus | Metric],  Length)
     end.
 
+in_black_list(DbName, GroupId, IndexName) when is_binary(DbName),
+        is_binary(GroupId), is_binary(IndexName) ->
+    in_black_list(?b2l(DbName), ?b2l(GroupId), ?b2l(IndexName));
+in_black_list(DbName, GroupId, IndexName) when is_list(DbName),
+        is_list(GroupId), is_list(IndexName) ->
+    in_black_list(lists:flatten([DbName, ".", GroupId, ".", IndexName]));
+in_black_list(_DbName, _GroupId, _IndexName) ->
+    false.
+
+in_black_list(IndexEntry) when is_list(IndexEntry) ->
+    case dreyfus_config:get(IndexEntry) of
+        undefined -> false;
+        _ -> true
+    end;
+in_black_list(_IndexEntry) ->
+    false.
+
+maybe_deny_index(DbName, GroupId, IndexName) ->
+    case in_black_list(DbName, GroupId, IndexName) of
+        true ->
+            Reason = ?l2b(io_lib:format("Index <~s, ~s, ~s>, is BlackListed",
+                [?b2l(DbName), ?b2l(GroupId), ?b2l(IndexName)])),
+            throw ({bad_request, Reason});
+        _ ->
+            ok
+    end.
+
 -ifdef(TEST).
 -include_lib("eunit/include/eunit.hrl").
 
diff --git a/test/dreyfus_blacklist_await_test.erl b/test/dreyfus_blacklist_await_test.erl
new file mode 100644
index 000000000..28a5e7f30
--- /dev/null
+++ b/test/dreyfus_blacklist_await_test.erl
@@ -0,0 +1,76 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(dreyfus_blacklist_await_test).
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("dreyfus/include/dreyfus.hrl").
+-include_lib("eunit/include/eunit.hrl").
+
+-define(DDOC_ID, <<"_design/black_list_doc">>).
+-define(INDEX_NAME, <<"my_index">>).
+-define(DBNAME, <<"mydb">>).
+-define(TIMEOUT, 1000).
+
+start() ->
+    test_util:start_couch([dreyfus]).
+
+stop(_) ->
+    test_util:stop_couch([dreyfus]).
+
+setup() ->
+    ok = meck:new(couch_log),
+    ok = meck:expect(couch_log, notice, fun(_Fmt, _Args) ->
+        ?debugFmt(_Fmt, _Args)
+    end).
+
+teardown(_) ->
+    ok = meck:unload(couch_log).
+
+dreyfus_blacklist_await_test_() ->
+    {
+        "dreyfus black_list_doc await tests",
+        {
+            setup,
+            fun start/0, fun stop/1,
+            {
+                foreach,
+                fun setup/0, fun teardown/1,
+                [
+                    fun do_not_await_1/0
+                ]
+            }
+        }
+    }.
+
+do_not_await_1() ->
+    ok = meck:new(dreyfus_index, [passthrough]),
+    Denied = lists:flatten([?b2l(?DBNAME), ".", "black_list_doc", ".",
+        "my_index"]),
+    config:set("dreyfus_blacklist", Denied, "true"),
+    dreyfus_test_util:wait_config_change(Denied, "true"),
+    Index = #index{dbname=?DBNAME, name=?INDEX_NAME, ddoc_id=?DDOC_ID},
+    State = create_state(?DBNAME, Index, nil, nil, []),
+    Msg = "Index Blocked from Updating - db: ~p, ddocid: ~p name: ~p",
+    Return = wait_log_message(Msg, fun() ->
+        {noreply, NewState} = dreyfus_index:handle_call({await, 1},
+        self(), State)
+    end),
+    ?assertEqual(Return, ok).
+
+wait_log_message(Fmt, Fun) ->
+    ok = meck:reset(couch_log),
+    Fun(),
+    ok = meck:wait(couch_log, '_', [Fmt, '_'], 5000).
+
+create_state(DbName, Index, UPid, IPid, WList) ->
+    {state, DbName, Index, UPid, IPid, WList}.
diff --git a/test/dreyfus_blacklist_request_test.erl b/test/dreyfus_blacklist_request_test.erl
new file mode 100644
index 000000000..faf8b747f
--- /dev/null
+++ b/test/dreyfus_blacklist_request_test.erl
@@ -0,0 +1,85 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(dreyfus_blacklist_request_test).
+
+-include_lib("couch_log/include/couch_log.hrl").
+-include_lib("dreyfus/include/dreyfus.hrl").
+-include_lib("eunit/include/eunit.hrl").
+
+-define(TIMEOUT, 1000).
+
+start() ->
+    test_util:start_couch([dreyfus]),
+    ok = meck:new(fabric, [passthrough]),
+    ok = meck:expect(fabric, open_doc, fun(_, _, _) ->
+        {ok, ddoc}
+    end).
+
+stop(_) ->
+    ok = meck:unload(fabric),
+    test_util:stop_couch([dreyfus]).
+
+setup() ->
+    ok.
+
+teardown(_) ->
+    ok.
+
+dreyfus_blacklist_request_test_() ->
+    {
+        "dreyfus blacklist request tests",
+        {
+            setup,
+            fun start/0, fun stop/1,
+            {
+                foreach,
+                fun setup/0, fun teardown/1,
+                [
+                    fun deny_fabric_requests/0,
+                    fun allow_fabric_request/0
+                ]
+            }
+        }
+    }.
+
+deny_fabric_requests() ->
+    Reason = <<"Index <mydb, myddocid, myindexname>, is BlackListed">>,
+    QueryArgs = #index_query_args{},
+    Denied = "mydb.myddocid.myindexname",
+    config:set("dreyfus_blacklist", Denied, "true"),
+    dreyfus_test_util:wait_config_change(Denied, "true"),
+    ?assertThrow({bad_request, Reason}, dreyfus_fabric_search:go(<<"mydb">>,
+        <<"myddocid">>,  <<"myindexname">>, QueryArgs)),
+    ?assertThrow({bad_request, Reason}, dreyfus_fabric_group1:go(<<"mydb">>,
+        <<"myddocid">>,  <<"myindexname">>, QueryArgs)),
+    ?assertThrow({bad_request, Reason}, dreyfus_fabric_group2:go(<<"mydb">>,
+        <<"myddocid">>,  <<"myindexname">>, QueryArgs)),
+    ?assertThrow({bad_request, Reason}, dreyfus_fabric_info:go(<<"mydb">>,
+        <<"myddocid">>,  <<"myindexname">>, QueryArgs)).
+
+allow_fabric_request() ->
+    ok = meck:new(dreyfus_fabric_search, [passthrough]),
+    ok = meck:expect(dreyfus_fabric_search, go,
+        fun(A, GroupId, B, C) when is_binary(GroupId) -> 
+            meck:passthrough([A, GroupId, B, C])
+    end),
+    ok = meck:expect(dreyfus_fabric_search, go, fun(_, _, _, _) ->
+        ok
+    end),
+    Denied = "mydb2.myddocid2.myindexname2",
+    QueryArgs = #index_query_args{},
+    config:set("dreyfus_blacklist", Denied, "true"),
+    dreyfus_test_util:wait_config_change(Denied, "true"),
+    ?assertEqual(ok, dreyfus_fabric_search:go(<<"mydb">>,
+        <<"myddocid">>,  <<"indexnotthere">>, QueryArgs)),
+    ok = meck:unload(dreyfus_fabric_search).
diff --git a/test/dreyfus_config_test.erl b/test/dreyfus_config_test.erl
new file mode 100644
index 000000000..775e49d7f
--- /dev/null
+++ b/test/dreyfus_config_test.erl
@@ -0,0 +1,71 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(dreyfus_config_test).
+
+
+-include_lib("couch_log/include/couch_log.hrl").
+-include_lib("eunit/include/eunit.hrl").
+
+-define(TIMEOUT, 1000).
+
+
+start() ->
+    test_util:start_couch([dreyfus]).
+
+setup() ->
+    ok.
+
+teardown(_) ->
+    ok.
+
+dreyfus_config_test_() ->
+    {
+        "dreyfus config tests",
+        {
+            setup,
+            fun start/0, fun test_util:stop_couch/1,
+            {
+                foreach,
+                fun setup/0, fun teardown/1,
+                [
+                    fun check_black_list/0,
+                    fun check_delete_from_blacklist/0
+                ]
+            }
+        }
+    }.
+
+check_black_list() ->
+    Index = "mydb.myddocid.myindexname",
+    Index2 = "mydb2.myddocid2.myindexname2",
+    Index3 = "mydb3.myddocid3.myindexname3",
+    ok = config:set("dreyfus_blacklist", Index, "true"),
+    ok = config:set("dreyfus_blacklist", Index2, "true"),
+    ok = config:set("dreyfus_blacklist", Index3, "true"),
+    dreyfus_test_util:wait_config_change(Index3, "true"),
+    FinalBl = [Index3, Index2, Index],
+    lists:foreach(fun (I) ->
+        ?assertEqual("true", dreyfus_config:get(I))
+    end, FinalBl).
+
+check_delete_from_blacklist() ->
+    Index = "mydb.myddocid.myindexname",
+    Index2 = "mydb2.myddocid2.myindexname2",
+    ok = config:set("dreyfus_blacklist", Index, "true"),
+    dreyfus_test_util:wait_config_change(Index, "true"),
+    ok = config:delete("dreyfus_blacklist", Index),
+    dreyfus_test_util:wait_config_change(Index, undefined),
+    ok = config:set("dreyfus_blacklist", Index2, "true"),
+    dreyfus_test_util:wait_config_change(Index2, "true"),
+    ?assertEqual(undefined, dreyfus_config:get(Index)),
+    ?assertEqual("true", dreyfus_config:get(Index2)).
diff --git a/test/dreyfus_test_util.erl b/test/dreyfus_test_util.erl
new file mode 100644
index 000000000..631bc1047
--- /dev/null
+++ b/test/dreyfus_test_util.erl
@@ -0,0 +1,13 @@
+-module(dreyfus_test_util).
+
+-compile(export_all).
+
+-include_lib("couch/include/couch_db.hrl").
+
+wait_config_change(Key, Value) ->
+    test_util:wait(fun() ->
+        case dreyfus_config:get(Key) of
+            Value -> ok;
+            _ -> wait
+        end
+    end).