Improve test 05-replication-many-leaves.t

The test now includes attachments to verify that the
incremental attachment replication code path succeeds
when there are many leaf revisions of a document with
attachments.

1 files changed, 112 insertions, 36 deletions

diff --git a/src/couch_replicator/test/05-replication-many-leaves.t b/src/couch_replicator/test/05-replication-many-leaves.t
index 52d2023e9..22c7df8a0 100755
--- a/src/couch_replicator/test/05-replication-many-leaves.t
+++ b/src/couch_replicator/test/05-replication-many-leaves.t
@@ -32,6 +32,17 @@
     meta = []
 }).
 
+-record(att, {
+    name,
+    type,
+    att_len,
+    disk_len,
+    md5= <<>>,
+    revpos=0,
+    data,
+    encoding=identity
+}).
+
 -define(b2l(B), binary_to_list(B)).
 -define(l2b(L), list_to_binary(L)).
 -define(i2l(I), integer_to_list(I)).
@@ -43,15 +54,15 @@ target_db_name() -> <<"couch_test_rep_db_b">>.
 doc_ids() ->
     [<<"doc1">>, <<"doc2">>, <<"doc3">>].
 
-doc_num_conflicts(<<"doc1">>) -> 100;
-doc_num_conflicts(<<"doc2">>) -> 200;
-doc_num_conflicts(<<"doc3">>) -> 550.
+doc_num_conflicts(<<"doc1">>) -> 10;
+doc_num_conflicts(<<"doc2">>) -> 100;
+doc_num_conflicts(<<"doc3">>) -> 286.
 
 
 main(_) ->
     test_util:init_code_path(),
 
-    etap:plan(16),
+    etap:plan(56),
     case (catch test()) of
         ok ->
             etap:end_tests();
@@ -74,13 +85,12 @@ test() ->
         {{remote, source_db_name()}, {remote, (target_db_name())}}
     ],
 
-    {ok, SourceDb} = create_db(source_db_name()),
-    etap:diag("Populating source database"),
-    {ok, DocRevs} = populate_db(SourceDb),
-    ok = couch_db:close(SourceDb),
-
     lists:foreach(
         fun({Source, Target}) ->
+            {ok, SourceDb} = create_db(source_db_name()),
+            etap:diag("Populating source database"),
+            {ok, DocRevs} = populate_db(SourceDb),
+            ok = couch_db:close(SourceDb),
             etap:diag("Creating target database"),
             {ok, TargetDb} = create_db(target_db_name()),
 
@@ -88,17 +98,31 @@ test() ->
             etap:diag("Triggering replication"),
             replicate(Source, Target),
             etap:diag("Replication finished, comparing source and target databases"),
+            {ok, SourceDb2} = couch_db:open_int(source_db_name(), []),
             {ok, TargetDb2} = couch_db:open_int(target_db_name(), []),
-            verify_target(TargetDb2, DocRevs),
+            verify_target(SourceDb2, TargetDb2, DocRevs),
+            ok = couch_db:close(SourceDb2),
             ok = couch_db:close(TargetDb2),
 
-            etap:diag("Deleting target database"),
+            {ok, SourceDb3} = couch_db:open_int(source_db_name(), []),
+            {ok, DocRevs2} = add_attachments(SourceDb3, DocRevs, 2),
+            ok = couch_db:close(SourceDb3),
+            etap:diag("Triggering replication again"),
+            replicate(Source, Target),
+            etap:diag("Replication finished, comparing source and target databases"),
+            {ok, SourceDb4} = couch_db:open_int(source_db_name(), []),
+            {ok, TargetDb4} = couch_db:open_int(target_db_name(), []),
+            verify_target(SourceDb4, TargetDb4, DocRevs2),
+            ok = couch_db:close(SourceDb4),
+            ok = couch_db:close(TargetDb4),
+
+            etap:diag("Deleting source and target databases"),
             delete_db(TargetDb),
+            delete_db(SourceDb),
             ok = timer:sleep(1000)
         end,
         Pairs),
 
-    delete_db(SourceDb),
     couch_server_sup:stop(),
     ok.
 
@@ -112,44 +136,48 @@ populate_db(Db) ->
                 body = {[ {<<"value">>, Value} ]}
             },
             {ok, Rev} = couch_db:update_doc(Db, Doc, []),
-            {ok, RevsDict} = add_doc_siblings(Db, DocId, doc_num_conflicts(DocId)),
-            RevsDict2 = dict:store(Rev, Value, RevsDict),
-            dict:store(DocId, RevsDict2, Acc)
+            {ok, DocRevs} = add_doc_siblings(Db, DocId, doc_num_conflicts(DocId)),
+            dict:store(DocId, [Rev | DocRevs], Acc)
         end,
         dict:new(), doc_ids()),
     {ok, dict:to_list(DocRevsDict)}.
 
 
 add_doc_siblings(Db, DocId, NumLeaves) when NumLeaves > 0 ->
-    add_doc_siblings(Db, DocId, NumLeaves, [], dict:new()).
+    add_doc_siblings(Db, DocId, NumLeaves, [], []).
 
 
-add_doc_siblings(Db, _DocId, 0, AccDocs, RevsDict) ->
+add_doc_siblings(Db, _DocId, 0, AccDocs, AccRevs) ->
     {ok, []} = couch_db:update_docs(Db, AccDocs, [], replicated_changes),
-    {ok, RevsDict};
+    {ok, AccRevs};
 
-add_doc_siblings(Db, DocId, NumLeaves, AccDocs, RevsDict) ->
+add_doc_siblings(Db, DocId, NumLeaves, AccDocs, AccRevs) ->
     Value = list_to_binary(integer_to_list(NumLeaves)),
     Rev = couch_util:md5(Value),
-    RevsDict2 = dict:store({1, Rev}, Value, RevsDict),
     Doc = #doc{
         id = DocId,
         revs = {1, [Rev]},
         body = {[ {<<"value">>, Value} ]}
     },
-    add_doc_siblings(Db, DocId, NumLeaves - 1, [Doc | AccDocs], RevsDict2).
+    add_doc_siblings(Db, DocId, NumLeaves - 1, [Doc | AccDocs], [{1, Rev} | AccRevs]).
 
 
-verify_target(_TargetDb, []) ->
+verify_target(_SourceDb, _TargetDb, []) ->
     ok;
 
-verify_target(TargetDb, [{DocId, RevsDict} | Rest]) ->
+verify_target(SourceDb, TargetDb, [{DocId, RevList} | Rest]) ->
     {ok, Lookups} = couch_db:open_doc_revs(
         TargetDb,
         DocId,
-        [R || {R, _} <- dict:to_list(RevsDict)],
-        [ejson_body]),
+        RevList,
+        []),
     Docs = [Doc || {ok, Doc} <- Lookups],
+    {ok, SourceLookups} = couch_db:open_doc_revs(
+        SourceDb,
+        DocId,
+        RevList,
+        []),
+    SourceDocs = [Doc || {ok, Doc} <- SourceLookups],
     Total = doc_num_conflicts(DocId) + 1,
     etap:is(
         length(Docs),
@@ -157,21 +185,69 @@ verify_target(TargetDb, [{DocId, RevsDict} | Rest]) ->
         "Target has " ++ ?i2l(Total) ++ " leaf revisions of document " ++ ?b2l(DocId)),
     etap:diag("Verifying all revisions of document " ++ ?b2l(DocId)),
     lists:foreach(
-        fun(#doc{revs = {Pos, [RevId]}, body = {Body}}) ->
-            Rev = {Pos, RevId},
-            {ok, Value} = dict:find(Rev, RevsDict),
-            case couch_util:get_value(<<"value">>, Body) of
-            Value ->
+        fun({#doc{id = Id, revs = Revs} = TgtDoc, #doc{id = Id, revs = Revs} = SrcDoc}) ->
+            SourceJson = couch_doc:to_json_obj(SrcDoc, [attachments, conflicts]),
+            TargetJson = couch_doc:to_json_obj(TgtDoc, [attachments, conflicts]),
+            case TargetJson of
+            SourceJson ->
                 ok;
-            Other ->
+            _ ->
+                {Pos, [Rev | _]} = Revs,
                 etap:bail("Wrong value for revision " ++
-                    ?b2l(couch_doc:rev_to_str(Rev)) ++ " of document " ++
-                    ?b2l(DocId) ++ ". Expected `" ++ couch_util:to_list(Value) ++
-                    "`, got `" ++ couch_util:to_list(Other) ++ "`")
+                    ?b2l(couch_doc:rev_to_str({Pos, Rev})) ++
+                    " of document " ++ ?b2l(DocId))
             end
         end,
-        Docs),
-    verify_target(TargetDb, Rest).
+        lists:zip(Docs, SourceDocs)),
+    verify_target(SourceDb, TargetDb, Rest).
+
+
+add_attachments(Source, DocIdRevs, NumAtts) ->
+    add_attachments(Source, DocIdRevs, NumAtts, []).
+
+add_attachments(_SourceDb, [], _NumAtts, Acc) ->
+    {ok, Acc};
+
+add_attachments(SourceDb, [{DocId, RevList} | Rest], NumAtts, IdRevsAcc) ->
+    {ok, SourceLookups} = couch_db:open_doc_revs(
+        SourceDb,
+        DocId,
+        RevList,
+        []),
+    SourceDocs = [Doc || {ok, Doc} <- SourceLookups],
+    Total = doc_num_conflicts(DocId) + 1,
+    etap:is(
+        length(SourceDocs),
+        Total,
+        "Source still has " ++ ?i2l(Total) ++
+            " leaf revisions of document " ++ ?b2l(DocId)),
+    etap:diag("Adding " ++ ?i2l(NumAtts) ++
+        " attachments to each revision of the document " ++ ?b2l(DocId)),
+    NewDocs = lists:foldl(
+        fun(#doc{atts = Atts, revs = {Pos, [Rev | _]}} = Doc, Acc) ->
+            NewAtts = lists:foldl(
+                fun(I, AttAcc) ->
+                    AttData = crypto:rand_bytes(100),
+                    NewAtt = #att{
+                        name = iolist_to_binary(
+                            ["att_", ?i2l(I), "_", couch_doc:rev_to_str({Pos, Rev})]),
+                        type = <<"application/foobar">>,
+                        att_len = byte_size(AttData),
+                        data = AttData
+                    },
+                    [NewAtt | AttAcc]
+                end,
+                [], lists:seq(1, NumAtts)),
+            [Doc#doc{atts = Atts ++ NewAtts} | Acc]
+        end,
+        [], SourceDocs),
+    {ok, UpdateResults} = couch_db:update_docs(SourceDb, NewDocs, []),
+    NewRevs = [R || {ok, R} <- UpdateResults],
+    etap:is(
+        length(NewRevs),
+        length(NewDocs),
+        "Document revisions updated with " ++ ?i2l(NumAtts) ++ " attachments"),
+    add_attachments(SourceDb, Rest, NumAtts, [{DocId, NewRevs} | IdRevsAcc]).
 
 
 db_url(DbName) ->