Revert "Revert "Use Ejson Body Instead of Compressed Body for External size (#606)""

This reverts commit c8ee29505c718ed6bd9687a664dae11d984d89a7.

PR is here: https://github.com/apache/couchdb/pull/606

3 files changed, 42 insertions, 4 deletions

diff --git a/src/couch/src/couch_db.erl b/src/couch/src/couch_db.erl
index d01a3e0c4..e4e3a8b34 100644
--- a/src/couch/src/couch_db.erl
+++ b/src/couch/src/couch_db.erl
@@ -1111,7 +1111,11 @@ prepare_doc_summaries(Db, BucketList) ->
                 nil
             end,
             SummaryChunk = couch_db_updater:make_doc_summary(Db, {Body, DiskAtts}),
-            Doc#doc{body = {summary, SummaryChunk, SizeInfo, AttsFd}}
+            Meta = Doc#doc.meta,
+            Doc#doc{
+                body = {summary, SummaryChunk, SizeInfo, AttsFd},
+                meta = [{ejson_size, ?term_size(Body)} | Meta]
+            }
         end,
         Bucket) || Bucket <- BucketList].
 
diff --git a/src/couch/src/couch_db_updater.erl b/src/couch/src/couch_db_updater.erl
index 49061b2f6..277f2b535 100644
--- a/src/couch/src/couch_db_updater.erl
+++ b/src/couch/src/couch_db_updater.erl
@@ -677,6 +677,7 @@ flush_trees(#db{fd = Fd} = Db,
             case Value of
             #doc{deleted = IsDeleted, body = {summary, _, _, _} = DocSummary} ->
                 {summary, Summary, AttSizeInfo, AttsFd} = DocSummary,
+                ExternalSize = get_meta_body_size(Value#doc.meta, Summary),
                 % this node value is actually an unwritten document summary,
                 % write to disk.
                 % make sure the Fd in the written bins is the same Fd we are
@@ -695,7 +696,6 @@ flush_trees(#db{fd = Fd} = Db,
                                     " changed. Possibly retrying.", []),
                     throw(retry)
                 end,
-                ExternalSize = ?term_size(Summary),
                 {ok, NewSummaryPointer, SummarySize} =
                     couch_file:append_raw_chunk(Fd, Summary),
                 Leaf = #leaf{
@@ -1086,8 +1086,16 @@ copy_docs(Db, #db{fd = DestFd} = NewDb, MixedInfos, Retry) ->
         {NewRevTree, FinalAcc} = couch_key_tree:mapfold(fun
             (_Rev, #leaf{ptr=Sp}=Leaf, leaf, SizesAcc) ->
                 {Body, AttInfos} = copy_doc_attachments(Db, Sp, DestFd),
+                % In the future, we should figure out how to do this for
+                % upgrade purposes.
+                EJsonBody = case is_binary(Body) of
+                    true ->
+                        couch_compress:decompress(Body);
+                    false ->
+                        Body
+                end,
                 SummaryChunk = make_doc_summary(NewDb, {Body, AttInfos}),
-                ExternalSize = ?term_size(SummaryChunk),
+                ExternalSize = ?term_size(EJsonBody),
                 {ok, Pos, SummarySize} = couch_file:append_raw_chunk(
                     DestFd, SummaryChunk),
                 AttSizes = [{element(3,A), element(4,A)} || A <- AttInfos],
@@ -1467,6 +1475,16 @@ make_doc_summary(#db{compression = Comp}, {Body0, Atts0}) ->
     SummaryBin = ?term_to_bin({Body, Atts}),
     couch_file:assemble_file_chunk(SummaryBin, couch_crypto:hash(md5, SummaryBin)).
 
+
+get_meta_body_size(Meta, Summary) ->
+    case lists:keyfind(ejson_size, 1, Meta) of
+        {ejson_size, ExternalSize} ->
+            ExternalSize;
+        false ->
+            ?term_size(couch_compress:decompress(Summary))
+    end.
+
+
 default_security_object(<<"shards/", _/binary>>) ->
     case config:get("couchdb", "default_security", "everyone") of
         "admin_only" ->
diff --git a/src/couch/test/couchdb_file_compression_tests.erl b/src/couch/test/couchdb_file_compression_tests.erl
index ccfa24435..41d055624 100644
--- a/src/couch/test/couchdb_file_compression_tests.erl
+++ b/src/couch/test/couchdb_file_compression_tests.erl
@@ -119,16 +119,19 @@ should_compare_compression_methods(DbName) ->
 
 compare_compression_methods(DbName) ->
     config:set("couchdb", "file_compression", "none", false),
+    ExternalSizePreCompact = db_external_size(DbName),
     compact_db(DbName),
     compact_view(DbName),
     DbSizeNone = db_disk_size(DbName),
     ViewSizeNone = view_disk_size(DbName),
+    ExternalSizeNone = db_external_size(DbName),
 
     config:set("couchdb", "file_compression", "snappy", false),
     compact_db(DbName),
     compact_view(DbName),
     DbSizeSnappy = db_disk_size(DbName),
     ViewSizeSnappy = view_disk_size(DbName),
+    ExternalSizeSnappy = db_external_size(DbName),
 
     ?assert(DbSizeNone > DbSizeSnappy),
     ?assert(ViewSizeNone > ViewSizeSnappy),
@@ -147,9 +150,13 @@ compare_compression_methods(DbName) ->
     compact_view(DbName),
     DbSizeDeflate9 = db_disk_size(DbName),
     ViewSizeDeflate9 = view_disk_size(DbName),
+    ExternalSizeDeflate9 = db_external_size(DbName),
 
     ?assert(DbSizeDeflate1 > DbSizeDeflate9),
-    ?assert(ViewSizeDeflate1 > ViewSizeDeflate9).
+    ?assert(ViewSizeDeflate1 > ViewSizeDeflate9),
+    ?assert(ExternalSizePreCompact =:= ExternalSizeNone),
+    ?assert(ExternalSizeNone =:= ExternalSizeSnappy),
+    ?assert(ExternalSizeNone =:= ExternalSizeDeflate9).
 
 
 populate_db(_Db, NumDocs) when NumDocs =< 0 ->
@@ -194,6 +201,12 @@ db_disk_size(DbName) ->
     ok = couch_db:close(Db),
     active_size(Info).
 
+db_external_size(DbName) ->
+    {ok, Db} = couch_db:open_int(DbName, []),
+    {ok, Info} = couch_db:get_db_info(Db),
+    ok = couch_db:close(Db),
+    external_size(Info).
+
 view_disk_size(DbName) ->
     {ok, Db} = couch_db:open_int(DbName, []),
     {ok, DDoc} = couch_db:open_doc(Db, ?DDOC_ID, [ejson_body]),
@@ -204,6 +217,9 @@ view_disk_size(DbName) ->
 active_size(Info) ->
     couch_util:get_nested_json_value({Info}, [sizes, active]).
 
+external_size(Info) ->
+    couch_util:get_nested_json_value({Info}, [sizes, external]).
+
 wait_compaction(DbName, Kind, Line) ->
     WaitFun = fun() ->
        case is_compaction_running(DbName) of