Merge branch 'master' into python-blackpython-black

10 files changed, 106 insertions, 61 deletions

diff --git a/Makefile b/Makefile
index b2f3f04aa..39d548b8a 100644
--- a/Makefile
+++ b/Makefile
@@ -194,8 +194,7 @@ python-black-update: .venv/bin/black
 		. dev/run rel/overlay/bin/couchup test/javascript/run
 
 .PHONY: elixir
-elixir: elixir-check-formatted
-	@rm -rf dev/lib
+elixir: elixir-check-formatted devclean
 	@dev/run -a adm:pass --no-eval test/elixir/run
 
 .PHONY: elixir-check-formatted
diff --git a/Makefile.win b/Makefile.win
index fc552500c..2b74e455e 100644
--- a/Makefile.win
+++ b/Makefile.win
@@ -165,8 +165,7 @@ python-black-update: .venv/bin/black
 		. dev\run rel\overlay\bin\couchup test\javascript\run
 
 .PHONY: elixir
-elixir: elixir-check-formatted
-	@del /s/q dev\lib 2> nul
+elixir: elixir-check-formatted devclean
 	@dev\run -a adm:pass --no-eval test\elixir\run.cmd
 
 .PHONY: elixir-check-formatted
@@ -347,7 +346,7 @@ clean:
 	-@del /f/q/s src\*.dll
 	-@del /f/q src\couch\priv\*.exe
 	-@del /f/q share\server\main.js share\server\main-coffee.js
-	-@rmdir /s/q tmp 
+	-@rmdir /s/q tmp
 	-@rmdir /s/q dev\data
 	-@rmdir /s/q dev\lib
 	-@rmdir /s/q dev\logs
diff --git a/dev/run b/dev/run
index 879a31391..94f6e5d2f 100755
--- a/dev/run
+++ b/dev/run
@@ -220,7 +220,6 @@ def setup_context(opts, args):
 
 @log("Setup environment")
 def setup_dirs(ctx):
-    ensure_dir_exists(ctx["devdir"], "data")
     ensure_dir_exists(ctx["devdir"], "logs")
 
 
diff --git a/rebar.config.script b/rebar.config.script
index d35cf3de1..5f17c29e0 100644
--- a/rebar.config.script
+++ b/rebar.config.script
@@ -57,9 +57,9 @@ DepDescs = [
 
 %% Non-Erlang deps
 {docs,             {url, "https://github.com/apache/couchdb-documentation"},
-                   "c75b2331fa04b93dc73411f34c08ba773c40fc3b", [raw]},
+                   {tag, "2.3.0"}, [raw]},
 {fauxton,          {url, "https://github.com/apache/couchdb-fauxton"},
-                   {tag, "v1.1.18"}, [raw]},
+                   {tag, "v1.1.19"}, [raw]},
 %% Third party deps
 {folsom,           "folsom",           {tag, "CouchDB-0.8.2"}},
 {hyper,            "hyper",            {tag, "CouchDB-2.2.0-4"}},
diff --git a/rel/reltool.config b/rel/reltool.config
index bf4ae448b..858e935b2 100644
--- a/rel/reltool.config
+++ b/rel/reltool.config
@@ -12,7 +12,7 @@
 
 {sys, [
     {lib_dirs, ["../src"]},
-    {rel, "couchdb", "2.2.0", [
+    {rel, "couchdb", "2.3.0", [
         %% stdlib
         asn1,
         compiler,
diff --git a/src/couch/include/couch_db.hrl b/src/couch/include/couch_db.hrl
index feaa64069..801483aa1 100644
--- a/src/couch/include/couch_db.hrl
+++ b/src/couch/include/couch_db.hrl
@@ -203,6 +203,14 @@
     atts = []
 }).
 
+-record (fabric_changes_acc, {
+    db,
+    seq,
+    args,
+    options,
+    pending,
+    epochs
+}).
 
 -type doc() :: #doc{}.
 -type ddoc() :: #doc{}.
diff --git a/src/couch/src/couch_changes.erl b/src/couch/src/couch_changes.erl
index db465ee98..c5b5edf9f 100644
--- a/src/couch/src/couch_changes.erl
+++ b/src/couch/src/couch_changes.erl
@@ -24,7 +24,9 @@
     filter/3,
     handle_db_event/3,
     handle_view_event/3,
-    view_filter/3
+    view_filter/3,
+    send_changes_doc_ids/6,
+    send_changes_design_docs/6
 ]).
 
 -export([changes_enumerator/2]).
@@ -627,7 +629,14 @@ send_lookup_changes(FullDocInfos, StartSeq, Dir, Db, Fun, Acc0) ->
         {stop, Acc} -> Acc
     end,
     case Dir of
-        fwd -> {ok, FinalAcc#changes_acc{seq = couch_db:get_update_seq(Db)}};
+        fwd ->
+            FinalAcc0 = case element(1, FinalAcc) of
+                changes_acc -> % we came here via couch_http or internal call
+                    FinalAcc#changes_acc{seq = couch_db:get_update_seq(Db)};
+                fabric_changes_acc -> % we came here via chttpd / fabric / rexi
+                    FinalAcc#fabric_changes_acc{seq = couch_db:get_update_seq(Db)}
+            end,
+            {ok, FinalAcc0};
         rev -> {ok, FinalAcc}
     end.
 
diff --git a/src/couch/src/couch_proc_manager.erl b/src/couch/src/couch_proc_manager.erl
index d2a198e82..3366b2bca 100644
--- a/src/couch/src/couch_proc_manager.erl
+++ b/src/couch/src/couch_proc_manager.erl
@@ -43,6 +43,7 @@
 -define(PROCS, couch_proc_manager_procs).
 -define(WAITERS, couch_proc_manager_waiters).
 -define(OPENING, couch_proc_manager_opening).
+-define(SERVERS, couch_proc_manager_servers).
 -define(RELISTEN_DELAY, 5000).
 
 -record(state, {
@@ -104,6 +105,11 @@ init([]) ->
     ets:new(?PROCS, TableOpts ++ [{keypos, #proc_int.pid}]),
     ets:new(?WAITERS, TableOpts ++ [{keypos, #client.timestamp}]),
     ets:new(?OPENING, [public, named_table, set]),
+    ets:new(?SERVERS, [public, named_table, set]),
+    ets:insert(?SERVERS, get_servers_from_env("COUCHDB_QUERY_SERVER_")),
+    ets:insert(?SERVERS, get_servers_from_env("COUCHDB_NATIVE_QUERY_SERVER_")),
+    ets:insert(?SERVERS, [{"QUERY", {mango_native_proc, start_link, []}}]),
+    maybe_configure_erlang_native_servers(),
 
     {ok, #state{
         config = get_proc_config(),
@@ -207,6 +213,7 @@ handle_cast(reload_config, State) ->
         hard_limit = get_hard_limit(),
         soft_limit = get_soft_limit()
     },
+    maybe_configure_erlang_native_servers(),
     {noreply, flush_waiters(NewState)};
 
 handle_cast(_Msg, State) ->
@@ -267,6 +274,9 @@ handle_config_terminate(_Server, _Reason, _State) ->
     gen_server:cast(?MODULE, reload_config),
     erlang:send_after(?RELISTEN_DELAY, whereis(?MODULE), restart_config_listener).
 
+handle_config_change("native_query_servers", _, _, _, _) ->
+    gen_server:cast(?MODULE, reload_config),
+    {ok, undefined};
 handle_config_change("query_server_config", _, _, _, _) ->
     gen_server:cast(?MODULE, reload_config),
     {ok, undefined};
@@ -372,23 +382,36 @@ new_proc(Client) ->
     end,
     exit(Resp).
 
-get_env_for_spec(Spec, Target) ->
-    % loop over os:getenv(), match SPEC_TARGET
-    lists:filtermap(fun(VarName) ->
-        SpecStr = Spec ++ Target,
-        case string:tokens(VarName, "=") of
-            [SpecStr, Cmd] -> {true, Cmd};
-            _Else -> false
+split_string_if_longer(String, Pos) ->
+    case length(String) > Pos of
+        true -> lists:split(Pos, String);
+        false -> false
+    end.
+
+split_by_char(String, Char) ->
+    %% 17.5 doesn't have string:split
+    %% the function doesn't handle errors
+    %% it is designed to be used only in specific context
+    Pos = string:chr(String, Char),
+    {Key, [_Eq | Value]} = lists:split(Pos - 1, String),
+    {Key, Value}.
+
+get_servers_from_env(Spec) ->
+    SpecLen = length(Spec),
+    % loop over os:getenv(), match SPEC_
+    lists:filtermap(fun(EnvStr) ->
+        case split_string_if_longer(EnvStr, SpecLen) of
+            {Spec, Rest} ->
+                {true, split_by_char(Rest, $=)};
+            _ ->
+                false
         end
     end, os:getenv()).
 
 get_query_server(LangStr) ->
-    % look for COUCH_QUERY_SERVER_LANGSTR in env
-    % if exists, return value, else undefined
-    UpperLangString = string:to_upper(LangStr),
-    case get_env_for_spec("COUCHDB_QUERY_SERVER_", UpperLangString) of
-        [] -> undefined;
-        [Command] -> Command
+    case ets:lookup(?SERVERS, string:to_upper(LangStr)) of
+        [{_, Command}] -> Command;
+        _ -> undefined
     end.
 
 native_query_server_enabled() ->
@@ -397,36 +420,25 @@ native_query_server_enabled() ->
     NativeEnabled = config:get_boolean("native_query_servers", "enable_erlang_query_server", false),
     NativeLegacyConfig = config:get("native_query_servers", "erlang", ""),
     NativeLegacyEnabled = NativeLegacyConfig =:= "{couch_native_process, start_link, []}",
-
     NativeEnabled orelse NativeLegacyEnabled.
 
-get_native_query_server("query") -> % mango query server
-    "{mango_native_proc, start_link, []}";
-get_native_query_server("erlang") -> % erlang query server
+maybe_configure_erlang_native_servers() ->
     case native_query_server_enabled() of
-        true -> "{couch_native_process, start_link, []}";
-        _Else -> undefined
-    end;
-get_native_query_server(LangStr) ->
-    % same as above, but COUCH_NATIVE_QUERY_SERVER_LANGSTR
-    UpperLangString = string:uppercase(LangStr),
-    case get_env_for_spec("COUCHDB_NATIVE_QUERY_SERVER_", UpperLangString) of
-        [] -> undefined;
-        [Command] -> Command
+        true ->
+           ets:insert(?SERVERS, [
+               {"ERLANG", {couch_native_process, start_link, []}}]);
+        _Else ->
+           ok
     end.
 
 new_proc_int(From, Lang) when is_binary(Lang) ->
     LangStr = binary_to_list(Lang),
     case get_query_server(LangStr) of
     undefined ->
-        case get_native_query_server(LangStr) of
-        undefined ->
-            gen_server:reply(From, {unknown_query_language, Lang});
-        SpecStr ->
-            {ok, {M,F,A}} = couch_util:parse_term(SpecStr),
-            {ok, Pid} = apply(M, F, A),
-            make_proc(Pid, Lang, M)
-        end;
+        gen_server:reply(From, {unknown_query_language, Lang});
+    {M, F, A} ->
+        {ok, Pid} = apply(M, F, A),
+        make_proc(Pid, Lang, M);
     Command ->
         {ok, Pid} = couch_os_process:start_link(Command),
         make_proc(Pid, Lang, couch_os_process)
diff --git a/src/fabric/src/fabric_rpc.erl b/src/fabric/src/fabric_rpc.erl
index c8aa19e0f..2b00a3668 100644
--- a/src/fabric/src/fabric_rpc.erl
+++ b/src/fabric/src/fabric_rpc.erl
@@ -32,15 +32,6 @@
 -include_lib("couch/include/couch_db.hrl").
 -include_lib("couch_mrview/include/couch_mrview.hrl").
 
--record (cacc, {
-    db,
-    seq,
-    args,
-    options,
-    pending,
-    epochs
-}).
-
 %% rpc endpoints
 %%  call to with_db will supply your M:F with a Db instance
 %%  and then remaining args
@@ -75,7 +66,7 @@ changes(DbName, Options, StartVector, DbOptions) ->
         StartSeq = calculate_start_seq(Db, node(), StartVector),
         Enum = fun changes_enumerator/2,
         Opts = [{dir,Dir}],
-        Acc0 = #cacc{
+        Acc0 = #fabric_changes_acc{
           db = Db,
           seq = StartSeq,
           args = Args,
@@ -84,8 +75,8 @@ changes(DbName, Options, StartVector, DbOptions) ->
           epochs = couch_db:get_epochs(Db)
         },
         try
-            {ok, #cacc{seq=LastSeq, pending=Pending, epochs=Epochs}} =
-                couch_db:fold_changes(Db, StartSeq, Enum, Acc0, Opts),
+            {ok, #fabric_changes_acc{seq=LastSeq, pending=Pending, epochs=Epochs}} =
+                do_changes(Db, StartSeq, Enum, Acc0, Opts),
             rexi:stream_last({complete, [
                 {seq, {LastSeq, uuid(Db), couch_db:owner_of(Epochs, LastSeq)}},
                 {pending, Pending}
@@ -97,6 +88,34 @@ changes(DbName, Options, StartVector, DbOptions) ->
         rexi:stream_last(Error)
     end.
 
+do_changes(Db, StartSeq, Enum, Acc0, Opts) ->
+    #fabric_changes_acc {
+        args = Args
+    } = Acc0,
+    #changes_args {
+        filter = Filter
+    } = Args,
+    case Filter of
+        "_doc_ids" ->
+            % optimised code path, we’re looking up all doc_ids in the by-id instead of filtering
+            % the entire by-seq tree to find the doc_ids one by one
+            #changes_args {
+                filter_fun = {doc_ids, Style, DocIds},
+                dir = Dir
+            } = Args,
+            couch_changes:send_changes_doc_ids(Db, StartSeq, Dir, Enum, Acc0, {doc_ids, Style, DocIds});
+        "_design_docs" ->
+            % optimised code path, we’re looking up all design_docs in the by-id instead of
+            % filtering the entire by-seq tree to find the design_docs one by one
+            #changes_args {
+                filter_fun = {design_docs, Style},
+                dir = Dir
+            } = Args,
+            couch_changes:send_changes_design_docs(Db, StartSeq, Dir, Enum, Acc0, {design_docs, Style});
+        _ ->
+            couch_db:fold_changes(Db, StartSeq, Enum, Acc0, Opts)
+    end.
+
 all_docs(DbName, Options, Args0) ->
     case fabric_util:upgrade_mrargs(Args0) of
         #mrargs{keys=undefined} = Args1 ->
@@ -482,9 +501,9 @@ reduce_cb(ok, ddoc_updated) ->
 changes_enumerator(#full_doc_info{} = FDI, Acc) ->
     changes_enumerator(couch_doc:to_doc_info(FDI), Acc);
 changes_enumerator(#doc_info{id= <<"_local/", _/binary>>, high_seq=Seq}, Acc) ->
-    {ok, Acc#cacc{seq = Seq, pending = Acc#cacc.pending-1}};
+    {ok, Acc#fabric_changes_acc{seq = Seq, pending = Acc#fabric_changes_acc.pending-1}};
 changes_enumerator(DocInfo, Acc) ->
-    #cacc{
+    #fabric_changes_acc{
         db = Db,
         args = #changes_args{
             include_docs = IncludeDocs,
@@ -513,7 +532,7 @@ changes_enumerator(DocInfo, Acc) ->
         ]}
     end,
     ok = rexi:stream2(ChangesRow),
-    {ok, Acc#cacc{seq = Seq, pending = Pending-1}}.
+    {ok, Acc#fabric_changes_acc{seq = Seq, pending = Pending-1}}.
 
 doc_member(Shard, DocInfo, Opts, Filter) ->
     case couch_db:open_doc(Shard, DocInfo, [deleted | Opts]) of
diff --git a/version.mk b/version.mk
index af15abe4a..a93c9b198 100644
--- a/version.mk
+++ b/version.mk
@@ -1,3 +1,3 @@
 vsn_major=2
-vsn_minor=2
+vsn_minor=3
 vsn_patch=0