Implement customizable chttpd statisticsadd-chttpd-stats

5 files changed, 206 insertions, 18 deletions

diff --git a/src/chttpd/src/chttpd.erl b/src/chttpd/src/chttpd.erl
index 0b3349a24..631eb77c9 100644
--- a/src/chttpd/src/chttpd.erl
+++ b/src/chttpd/src/chttpd.erl
@@ -265,6 +265,7 @@ handle_request_int(MochiReq) ->
 
 before_request(HttpReq) ->
     try
+        chttpd_stats:init(),
         chttpd_plugin:before_request(HttpReq)
     catch Tag:Error ->
         {error, catch_error(HttpReq, Tag, Error)}
@@ -280,6 +281,7 @@ after_request(HttpReq, HttpResp0) ->
             {ok, HttpResp0#httpd_resp{status = aborted}}
         end,
     HttpResp2 = update_stats(HttpReq, HttpResp1),
+    chttpd_stats:report(HttpReq, HttpResp2),
     maybe_log(HttpReq, HttpResp2),
     HttpResp2.
 
diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl
index 6eda451e0..7a00d2b1b 100644
--- a/src/chttpd/src/chttpd_db.erl
+++ b/src/chttpd/src/chttpd_db.erl
@@ -136,6 +136,7 @@ changes_callback(start, #cacc{feed = continuous} = Acc) ->
     {ok, Resp} = chttpd:start_delayed_json_response(Acc#cacc.mochi, 200),
     {ok, Acc#cacc{mochi = Resp, responding = true}};
 changes_callback({change, Change}, #cacc{feed = continuous} = Acc) ->
+    chttpd_stats:incr_rows(),
     Data = [?JSON_ENCODE(Change) | "\n"],
     Len = iolist_size(Data),
     maybe_flush_changes_feed(Acc, Data, Len);
@@ -159,6 +160,7 @@ changes_callback(start, #cacc{feed = eventsource} = Acc) ->
     {ok, Resp} = chttpd:start_delayed_json_response(Req, 200, Headers),
     {ok, Acc#cacc{mochi = Resp, responding = true}};
 changes_callback({change, {ChangeProp}=Change}, #cacc{feed = eventsource} = Acc) ->
+    chttpd_stats:incr_rows(),
     Seq = proplists:get_value(seq, ChangeProp),
     Chunk = [
         "data: ", ?JSON_ENCODE(Change),
@@ -190,6 +192,7 @@ changes_callback(start, Acc) ->
     {ok, Resp} = chttpd:start_delayed_json_response(Req, 200, [], FirstChunk),
     {ok, Acc#cacc{mochi = Resp, responding = true}};
 changes_callback({change, Change}, Acc) ->
+    chttpd_stats:incr_rows(),
     Data = [Acc#cacc.prepend, ?JSON_ENCODE(Change)],
     Len = iolist_size(Data),
     maybe_flush_changes_feed(Acc, Data, Len);
@@ -409,8 +412,12 @@ db_req(#httpd{method='POST', path_parts=[DbName], user_ctx=Ctx}=Req, Db) ->
         % async_batching
         spawn(fun() ->
                 case catch(fabric:update_doc(Db, Doc2, Options)) of
-                {ok, _} -> ok;
-                {accepted, _} -> ok;
+                {ok, _} ->
+                    chttpd_stats:incr_writes(),
+                    ok;
+                {accepted, _} ->
+                    chttpd_stats:incr_writes(),
+                    ok;
                 Error ->
                     couch_log:debug("Batch doc error (~s): ~p",[DocId, Error])
                 end
@@ -426,8 +433,10 @@ db_req(#httpd{method='POST', path_parts=[DbName], user_ctx=Ctx}=Req, Db) ->
             $/, couch_util:url_encode(DocId)]),
         case fabric:update_doc(Db, Doc2, Options) of
         {ok, NewRev} ->
+            chttpd_stats:incr_writes(),
             HttpCode = 201;
         {accepted, NewRev} ->
+            chttpd_stats:incr_writes(),
             HttpCode = 202
         end,
         send_json(Req, HttpCode, [{"Location", DocUrl}], {[
@@ -500,11 +509,13 @@ db_req(#httpd{method='POST',path_parts=[_,<<"_bulk_docs">>], user_ctx=Ctx}=Req,
         case fabric:update_docs(Db, Docs, Options2) of
         {ok, Results} ->
             % output the results
+            chttpd_stats:incr_writes(length(Results)),
             DocResults = lists:zipwith(fun update_doc_result_to_json/2,
                 Docs, Results),
             send_json(Req, 201, DocResults);
         {accepted, Results} ->
             % output the results
+            chttpd_stats:incr_writes(length(Results)),
             DocResults = lists:zipwith(fun update_doc_result_to_json/2,
                 Docs, Results),
             send_json(Req, 202, DocResults);
@@ -516,9 +527,11 @@ db_req(#httpd{method='POST',path_parts=[_,<<"_bulk_docs">>], user_ctx=Ctx}=Req,
     false ->
         case fabric:update_docs(Db, Docs, [replicated_changes|Options]) of
         {ok, Errors} ->
+            chttpd_stats:incr_writes(length(Docs)),
             ErrorsJson = lists:map(fun update_doc_result_to_json/1, Errors),
             send_json(Req, 201, ErrorsJson);
         {accepted, Errors} ->
+            chttpd_stats:incr_writes(length(Docs)),
             ErrorsJson = lists:map(fun update_doc_result_to_json/1, Errors),
             send_json(Req, 202, ErrorsJson)
         end;
@@ -627,8 +640,12 @@ db_req(#httpd{method='POST',path_parts=[_,<<"_purge">>]}=Req, Db) ->
     end,
     couch_stats:increment_counter([couchdb, document_purges, total], length(IdsRevs2)),
     Results2 = case fabric:purge_docs(Db, IdsRevs2, Options) of
-        {ok, Results} -> Results;
-        {accepted, Results} -> Results
+        {ok, Results} ->
+            chttpd_stats:incr_writes(length(Results)),
+            Results;
+        {accepted, Results} ->
+            chttpd_stats:incr_writes(length(Results)),
+            Results
     end,
     {Code, Json} = purge_results_to_json(IdsRevs2, Results2),
     send_json(Req, Code, {[{<<"purge_seq">>, null}, {<<"purged">>, {Json}}]});
@@ -814,7 +831,7 @@ multi_all_docs_view(Req, Db, OP, Queries) ->
     VAcc1 = VAcc0#vacc{resp=Resp0},
     VAcc2 = lists:foldl(fun(Args, Acc0) ->
         {ok, Acc1} = fabric:all_docs(Db, Options,
-            fun couch_mrview_http:view_cb/2, Acc0, Args),
+            fun view_cb/2, Acc0, Args),
         Acc1
     end, VAcc1, ArgQueries),
     {ok, Resp1} = chttpd:send_delayed_chunk(VAcc2#vacc.resp, "\r\n]}"),
@@ -828,9 +845,20 @@ all_docs_view(Req, Db, Keys, OP) ->
     Options = [{user_ctx, Req#httpd.user_ctx}],
     Max = chttpd:chunked_response_buffer_size(),
     VAcc = #vacc{db=Db, req=Req, threshold=Max},
-    {ok, Resp} = fabric:all_docs(Db, Options, fun couch_mrview_http:view_cb/2, VAcc, Args3),
+    {ok, Resp} = fabric:all_docs(Db, Options, fun view_cb/2, VAcc, Args3),
     {ok, Resp#vacc.resp}.
 
+view_cb({row, Row} = Msg, Acc) ->
+    case lists:keymember(doc, 1, Row) of
+        true -> chttpd_stats:incr_reads();
+        false -> ok
+    end,
+    chttpd_stats:incr_rows(),
+    couch_mrview_http:view_cb(Msg, Acc);
+
+view_cb(Msg, Acc) ->
+    couch_mrview_http:view_cb(Msg, Acc).
+
 db_doc_req(#httpd{method='DELETE'}=Req, Db, DocId) ->
     % check for the existence of the doc to handle the 404 case.
     couch_doc_open(Db, DocId, nil, []),
@@ -865,6 +893,7 @@ db_doc_req(#httpd{method='GET', mochi_req=MochiReq}=Req, Db, DocId) ->
             {ok, []} when Revs == all ->
                 chttpd:send_error(Req, {not_found, missing});
             {ok, Results} ->
+                chttpd_stats:incr_reads(length(Results)),
                 case MochiReq:accepts_content_type("multipart/mixed") of
                 false ->
                     {ok, Resp} = start_json_response(Req, 200),
@@ -912,8 +941,11 @@ db_doc_req(#httpd{method='POST', user_ctx=Ctx}=Req, Db, DocId) ->
     false ->
         Rev = couch_doc:parse_rev(list_to_binary(couch_util:get_value("_rev", Form))),
         Doc = case fabric:open_revs(Db, DocId, [Rev], []) of
-            {ok, [{ok, Doc0}]} -> Doc0;
-            {error, Error} -> throw(Error)
+            {ok, [{ok, Doc0}]} ->
+                chttpd_stats:incr_reads(),
+                Doc0;
+            {error, Error} ->
+                throw(Error)
         end
     end,
     UpdatedAtts = [
@@ -939,8 +971,10 @@ db_doc_req(#httpd{method='POST', user_ctx=Ctx}=Req, Db, DocId) ->
     },
     case fabric:update_doc(Db, NewDoc, Options) of
     {ok, NewRev} ->
+        chttpd_stats:incr_writes(),
         HttpCode = 201;
     {accepted, NewRev} ->
+        chttpd_stats:incr_writes(),
         HttpCode = 202
     end,
     send_json(Req, HttpCode, [{"ETag", "\"" ++ ?b2l(couch_doc:rev_to_str(NewRev)) ++ "\""}], {[
@@ -986,8 +1020,12 @@ db_doc_req(#httpd{method='PUT', user_ctx=Ctx}=Req, Db, DocId) ->
 
             spawn(fun() ->
                     case catch(fabric:update_doc(Db, Doc, Options)) of
-                    {ok, _} -> ok;
-                    {accepted, _} -> ok;
+                    {ok, _} ->
+                        chttpd_stats:incr_writes(),
+                        ok;
+                    {accepted, _} ->
+                        chttpd_stats:incr_writes(),
+                        ok;
                     Error ->
                         couch_log:notice("Batch doc error (~s): ~p",[DocId, Error])
                     end
@@ -1018,8 +1056,10 @@ db_doc_req(#httpd{method='COPY', user_ctx=Ctx}=Req, Db, SourceDocId) ->
     case fabric:update_doc(Db,
         Doc#doc{id=TargetDocId, revs=TargetRevs}, [{user_ctx,Ctx}]) of
     {ok, NewTargetRev} ->
+        chttpd_stats:incr_writes(),
         HttpCode = 201;
     {accepted, NewTargetRev} ->
+        chttpd_stats:incr_writes(),
         HttpCode = 202
     end,
     % respond
@@ -1322,6 +1362,7 @@ couch_doc_open(Db, DocId, Rev, Options0) ->
     nil -> % open most recent rev
         case fabric:open_doc(Db, DocId, Options) of
         {ok, Doc} ->
+            chttpd_stats:incr_reads(),
             Doc;
          Error ->
              throw(Error)
@@ -1329,6 +1370,7 @@ couch_doc_open(Db, DocId, Rev, Options0) ->
     _ -> % open a specific rev (deletions come back as stubs)
         case fabric:open_revs(Db, DocId, [Rev], Options) of
         {ok, [{ok, Doc}]} ->
+            chttpd_stats:incr_reads(),
             Doc;
         {ok, [{{not_found, missing}, Rev}]} ->
             throw(not_found);
@@ -1503,9 +1545,13 @@ db_attachment_req(#httpd{method=Method, user_ctx=Ctx}=Req, Db, DocId, FileNamePa
             #doc{id=DocId};
         Rev ->
             case fabric:open_revs(Db, DocId, [Rev], [{user_ctx,Ctx}]) of
-            {ok, [{ok, Doc0}]}  -> Doc0;
-            {ok, [Error]}       -> throw(Error);
-            {error, Error}      -> throw(Error)
+            {ok, [{ok, Doc0}]} ->
+                chttpd_stats:incr_reads(),
+                Doc0;
+            {ok, [Error]} ->
+                throw(Error);
+            {error, Error} ->
+                throw(Error)
             end
     end,
 
@@ -1516,8 +1562,10 @@ db_attachment_req(#httpd{method=Method, user_ctx=Ctx}=Req, Db, DocId, FileNamePa
     W = chttpd:qs_value(Req, "w", integer_to_list(mem3:quorum(Db))),
     case fabric:update_doc(Db, DocEdited, [{user_ctx,Ctx}, {w,W}]) of
     {ok, UpdatedRev} ->
+        chttpd_stats:incr_writes(),
         HttpCode = 201;
     {accepted, UpdatedRev} ->
+        chttpd_stats:incr_writes(),
         HttpCode = 202
     end,
     erlang:put(mochiweb_request_recv, true),
@@ -1913,8 +1961,11 @@ bulk_get_open_doc_revs1(Db, Props, _, {DocId, Revs, Options}) ->
             RevStr = couch_util:get_value(<<"rev">>, Props),
             Error = {RevStr, <<"not_found">>, <<"missing">>},
             {DocId, {error, Error}, Options};
-        Results ->
-            {DocId, Results, Options}
+        {ok, Resps} = Results ->
+            chttpd_stats:incr_reads(length(Resps)),
+            {DocId, Results, Options};
+        Else ->
+            {DocId, Else, Options}
     end.
 
 
diff --git a/src/chttpd/src/chttpd_show.erl b/src/chttpd/src/chttpd_show.erl
index a724189cf..c3bf11929 100644
--- a/src/chttpd/src/chttpd_show.erl
+++ b/src/chttpd/src/chttpd_show.erl
@@ -25,6 +25,7 @@
 maybe_open_doc(Db, DocId, Options) ->
     case fabric:open_doc(Db, DocId, Options) of
     {ok, Doc} ->
+        chttpd_stats:incr_reads(),
         Doc;
     {not_found, _} ->
         nil
@@ -135,6 +136,7 @@ send_doc_update_response(Req, Db, DDoc, UpdateName, Doc, DocId) ->
             NewDoc = couch_db:doc_from_json_obj_validate(Db, {NewJsonDoc}),
             couch_doc:validate_docid(NewDoc#doc.id),
             {UpdateResult, NewRev} = fabric:update_doc(Db, NewDoc, Options),
+            chttpd_stats:incr_writes(),
             NewRevStr = couch_doc:rev_to_str(NewRev),
             case {UpdateResult, NewRev} of
             {ok, _} ->
@@ -201,7 +203,7 @@ handle_view_list(Req, Db, DDoc, LName, {ViewDesignName, ViewName}, Keys) ->
     couch_util:get_nested_json_value(DDoc#doc.body, [<<"lists">>, LName]),
     DbName = couch_db:name(Db),
     {ok, VDoc} = ddoc_cache:open(DbName, <<"_design/", ViewDesignName/binary>>),
-    CB = fun couch_mrview_show:list_cb/2,
+    CB = fun list_cb/2,
     QueryArgs = couch_mrview_http:parse_params(Req, Keys),
     Options = [{user_ctx, Req#httpd.user_ctx}],
     couch_query_servers:with_ddoc_proc(DDoc, fun(QServer) ->
@@ -220,6 +222,19 @@ handle_view_list(Req, Db, DDoc, LName, {ViewDesignName, ViewName}, Keys) ->
         end
     end).
 
+
+list_cb({row, Row} = Msg, Acc) ->
+    case lists:keymember(doc, 1, Row) of
+        true -> chttpd_stats:incr_reads();
+        false -> ok
+    end,
+    chttpd_stats:incr_rows(),
+    couch_mrview_show:list_cb(Msg, Acc);
+
+list_cb(Msg, Acc) ->
+    couch_mrview_show:list_cb(Msg, Acc).
+
+
 % Maybe this is in the proplists API
 % todo move to couch_util
 json_apply_field(H, {L}) ->
diff --git a/src/chttpd/src/chttpd_stats.erl b/src/chttpd/src/chttpd_stats.erl
new file mode 100644
index 000000000..59ec9268d
--- /dev/null
+++ b/src/chttpd/src/chttpd_stats.erl
@@ -0,0 +1,107 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License.  You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(chttpd_stats).
+
+
+-export([
+    init/0,
+    report/2,
+
+    incr_reads/0,
+    incr_reads/1,
+
+    incr_writes/0,
+    incr_writes/1,
+
+    incr_rows/0,
+    incr_rows/1
+]).
+
+
+-record(st, {
+    reads = 0,
+    writes = 0,
+    rows = 0
+}).
+
+
+-define(KEY, chttpd_stats).
+
+
+init() ->
+    put(?KEY, #st{}).
+
+
+report(HttpReq, HttpResp) ->
+    try
+        case get(?KEY) of
+            #st{} = St ->
+                report(HttpReq, HttpResp, St);
+            _ ->
+                ok
+        end
+    catch T:R ->
+        S = erlang:get_stacktrace(),
+        Fmt = "Failed to report chttpd request stats: ~p:~p ~p",
+        couch_log:error(Fmt, [T, R, S])
+    end.
+
+
+report(HttpReq, HttpResp, St) ->
+    case config:get("chttpd", "stats_reporter") of
+        undefined ->
+            ok;
+        ModStr ->
+            Mod = list_to_existing_atom(ModStr),
+            #st{
+                reads = Reads,
+                writes = Writes,
+                rows = Rows
+            } = St,
+            Mod:report(HttpReq, HttpResp, Reads, Writes, Rows)
+    end.
+
+
+incr_reads() ->
+    incr(#st.reads, 1).
+
+
+incr_reads(N) when is_integer(N), N >= 0 ->
+    incr(#st.reads, N).
+
+
+incr_writes() ->
+    incr(#st.writes, 1).
+
+
+incr_writes(N) when is_integer(N), N >= 0 ->
+    incr(#st.writes, N).
+
+
+incr_rows() ->
+    incr(#st.rows, 1).
+
+
+incr_rows(N) when is_integer(N), N >= 0 ->
+    incr(#st.rows, N).
+
+
+incr(Idx, Count) ->
+    case get(?KEY) of
+        #st{} = St ->
+            Total = element(Idx, St) + Count,
+            NewSt = setelement(Idx, St, Total),
+            put(?KEY, NewSt);
+        _ ->
+            ok
+    end.
diff --git a/src/chttpd/src/chttpd_view.erl b/src/chttpd/src/chttpd_view.erl
index 1fce165f9..26107d7c5 100644
--- a/src/chttpd/src/chttpd_view.erl
+++ b/src/chttpd/src/chttpd_view.erl
@@ -33,7 +33,7 @@ multi_query_view(Req, Db, DDoc, ViewName, Queries) ->
     VAcc1 = VAcc0#vacc{resp=Resp0},
     VAcc2 = lists:foldl(fun(Args, Acc0) ->
         {ok, Acc1} = fabric:query_view(Db, Options, DDoc, ViewName,
-            fun couch_mrview_http:view_cb/2, Acc0, Args),
+            fun view_cb/2, Acc0, Args),
         Acc1
     end, VAcc1, ArgQueries),
     {ok, Resp1} = chttpd:send_delayed_chunk(VAcc2#vacc.resp, "\r\n]}"),
@@ -46,9 +46,22 @@ design_doc_view(Req, Db, DDoc, ViewName, Keys) ->
     VAcc = #vacc{db=Db, req=Req, threshold=Max},
     Options = [{user_ctx, Req#httpd.user_ctx}],
     {ok, Resp} = fabric:query_view(Db, Options, DDoc, ViewName,
-        fun couch_mrview_http:view_cb/2, VAcc, Args),
+            fun view_cb/2, VAcc, Args),
     {ok, Resp#vacc.resp}.
 
+
+view_cb({row, Row} = Msg, Acc) ->
+    case lists:keymember(doc, 1, Row) of
+        true -> chttpd_stats:incr_reads();
+        false -> ok
+    end,
+    chttpd_stats:incr_rows(),
+    couch_mrview_http:view_cb(Msg, Acc);
+
+view_cb(Msg, Acc) ->
+    couch_mrview_http:view_cb(Msg, Acc).
+
+
 handle_view_req(#httpd{method='POST',
     path_parts=[_, _, _, _, ViewName, <<"queries">>]}=Req, Db, DDoc) ->
     chttpd:validate_ctype(Req, "application/json"),