From 1f05d07c456e23c0827efbbb3e738afc9f3152e7 Mon Sep 17 00:00:00 2001 From: David Barr Date: Wed, 17 Nov 2010 23:03:51 -0600 Subject: vcs-svn: Allow simple v3 dumps (no deltas yet) Since the dumpfile version 1 days, the Subversion dump format gained some new fields: - a unique identifier for the repository (version 2 format) - whether the text and properties for a node should be interpreted as deltas - checksums for a delta's preimage - SHA-1 sums as alternatives to the existing MD5 checksums for copy source and the payload (delta). For now what is relevant to us is the Text-delta and Prop-delta fields, since not noticing these causes a dump file to be misinterpreted (see the previous commit). [jn: with tests] Signed-off-by: David Barr Signed-off-by: Jonathan Nieder Signed-off-by: Junio C Hamano --- t/t9010-svn-fe.sh | 350 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 347 insertions(+), 3 deletions(-) (limited to 't/t9010-svn-fe.sh') diff --git a/t/t9010-svn-fe.sh b/t/t9010-svn-fe.sh index e9e46ea0fe..be5372ab3b 100755 --- a/t/t9010-svn-fe.sh +++ b/t/t9010-svn-fe.sh @@ -9,6 +9,30 @@ reinit_git () { git init } +properties () { + while test "$#" -ne 0 + do + property="$1" && + value="$2" && + printf "%s\n" "K ${#property}" && + printf "%s\n" "$property" && + printf "%s\n" "V ${#value}" && + printf "%s\n" "$value" && + shift 2 || + return 1 + done +} + +text_no_props () { + text="$1 +" && + printf "%s\n" "Prop-content-length: 10" && + printf "%s\n" "Text-content-length: ${#text}" && + printf "%s\n" "Content-length: $((${#text} + 10))" && + printf "%s\n" "" "PROPS-END" && + printf "%s\n" "$text" +} + >empty test_expect_success 'empty dump' ' @@ -18,13 +42,333 @@ test_expect_success 'empty dump' ' git fast-import input && - test_must_fail test-svn-fe input >stream && + echo "SVN-fs-dump-format-version: 4" >v4.dump && + test_must_fail test-svn-fe v4.dump >stream && test_cmp empty stream ' +test_expect_failure 'empty revision' ' + reinit_git && + printf "rev : %s\n" "" "" >expect && + cat >emptyrev.dump <<-\EOF && + SVN-fs-dump-format-version: 3 + + Revision-number: 1 + Prop-content-length: 0 + Content-length: 0 + + Revision-number: 2 + Prop-content-length: 0 + Content-length: 0 + + EOF + test-svn-fe emptyrev.dump >stream && + git fast-import actual && + test_cmp expect actual +' + +test_expect_success 'empty properties' ' + reinit_git && + printf "rev : %s\n" "" "" >expect && + cat >emptyprop.dump <<-\EOF && + SVN-fs-dump-format-version: 3 + + Revision-number: 1 + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + + Revision-number: 2 + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + EOF + test-svn-fe emptyprop.dump >stream && + git fast-import actual && + test_cmp expect actual +' + +test_expect_success 'author name and commit message' ' + reinit_git && + echo "" >expect.author && + cat >message <<-\EOF && + A concise summary of the change + + A detailed description of the change, why it is needed, what + was broken and why applying this is the best course of action. + + * file.c + Details pertaining to an individual file. + EOF + { + properties \ + svn:author author@example.com \ + svn:log "$(cat message)" && + echo PROPS-END + } >props && + { + echo "SVN-fs-dump-format-version: 3" && + echo && + echo "Revision-number: 1" && + echo Prop-content-length: $(wc -c log.dump && + test-svn-fe log.dump >stream && + git fast-import actual.log && + git log --format="<%an, %ae>" >actual.author && + test_cmp message actual.log && + test_cmp expect.author actual.author +' + +test_expect_success 'unsupported properties are ignored' ' + reinit_git && + echo author >expect && + cat >extraprop.dump <<-\EOF && + SVN-fs-dump-format-version: 3 + + Revision-number: 1 + Prop-content-length: 56 + Content-length: 56 + + K 8 + nonsense + V 1 + y + K 10 + svn:author + V 6 + author + PROPS-END + EOF + test-svn-fe extraprop.dump >stream && + git fast-import actual && + test_cmp expect actual +' + +test_expect_failure 'timestamp and empty file' ' + echo author@example.com >expect.author && + echo 1999-01-01 >expect.date && + echo file >expect.files && + reinit_git && + { + properties \ + svn:author author@example.com \ + svn:date "1999-01-01T00:01:002.000000Z" \ + svn:log "add empty file" && + echo PROPS-END + } >props && + { + cat <<-EOF && + SVN-fs-dump-format-version: 3 + + Revision-number: 1 + EOF + echo Prop-content-length: $(wc -c emptyfile.dump && + test-svn-fe emptyfile.dump >stream && + git fast-import actual.author && + git log --date=short --format=%ad HEAD >actual.date && + git ls-tree -r --name-only HEAD >actual.files && + test_cmp expect.author actual.author && + test_cmp expect.date actual.date && + test_cmp expect.files actual.files && + git checkout HEAD empty-file && + test_cmp empty file +' + +test_expect_success 'directory with files' ' + reinit_git && + printf "%s\n" directory/file1 directory/file2 >expect.files && + echo hi >hi && + echo hello >hello && + { + properties \ + svn:author author@example.com \ + svn:date "1999-02-01T00:01:002.000000Z" \ + svn:log "add directory with some files in it" && + echo PROPS-END + } >props && + { + cat <<-EOF && + SVN-fs-dump-format-version: 3 + + Revision-number: 1 + EOF + echo Prop-content-length: $(wc -c directory.dump && + test-svn-fe directory.dump >stream && + git fast-import actual.files && + git checkout HEAD directory && + test_cmp expect.files actual.files && + test_cmp hello directory/file1 && + test_cmp hi directory/file2 +' + +test_expect_success 'deltas not supported' ' + { + # (old) h + (inline) ello + (old) \n + printf "SVNQ%b%b%s" "Q\003\006\005\004" "\001Q\0204\001\002" "ello" | + q_to_nul + } >delta && + { + properties \ + svn:author author@example.com \ + svn:date "1999-01-05T00:01:002.000000Z" \ + svn:log "add greeting" && + echo PROPS-END + } >props && + { + properties \ + svn:author author@example.com \ + svn:date "1999-01-06T00:01:002.000000Z" \ + svn:log "change it" && + echo PROPS-END + } >props2 && + { + echo SVN-fs-dump-format-version: 3 && + echo && + echo Revision-number: 1 && + echo Prop-content-length: $(wc -c delta.dump && + test_must_fail test-svn-fe delta.dump +' + +test_expect_success 'property deltas not supported' ' + { + properties \ + svn:author author@example.com \ + svn:date "1999-03-06T00:01:002.000000Z" \ + svn:log "make an executable, or chmod -x it" && + echo PROPS-END + } >revprops && + { + echo SVN-fs-dump-format-version: 3 && + echo && + echo Revision-number: 1 && + echo Prop-content-length: $(wc -c propdelta.dump && + test_must_fail test-svn-fe propdelta.dump +' + test_expect_success 't9135/svn.dump' ' svnadmin create simple-svn && svnadmin load simple-svn <"$TEST_DIRECTORY/t9135/svn.dump" && -- cgit v1.2.1 From 1d13e9f600986b7ced8db37a9a9c4967ee7ff9d5 Mon Sep 17 00:00:00 2001 From: Jonathan Nieder Date: Fri, 19 Nov 2010 18:46:22 -0600 Subject: vcs-svn: Eliminate node_ctx.srcRev global The srcRev variable is only used in handle_node(); its purpose is to hold the old mode for a path, to only be used if properties are not being changed. Narrow its scope to make its meaningful lifetime more obvious. No functional change intended. Add some tests as a sanity-check for the simplest case (no renames). Signed-off-by: Jonathan Nieder Signed-off-by: Junio C Hamano --- t/t9010-svn-fe.sh | 158 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) (limited to 't/t9010-svn-fe.sh') diff --git a/t/t9010-svn-fe.sh b/t/t9010-svn-fe.sh index be5372ab3b..729e73dddf 100755 --- a/t/t9010-svn-fe.sh +++ b/t/t9010-svn-fe.sh @@ -252,6 +252,164 @@ test_expect_success 'directory with files' ' test_cmp hi directory/file2 ' +test_expect_failure 'change file mode but keep old content' ' + reinit_git && + cat >expect <<-\EOF && + OBJID + :120000 100644 OBJID OBJID T greeting + OBJID + :100644 120000 OBJID OBJID T greeting + OBJID + :000000 100644 OBJID OBJID A greeting + EOF + echo "link hello" >expect.blob && + echo hello >hello && + cat >filemode.dump <<-\EOF && + SVN-fs-dump-format-version: 3 + + Revision-number: 1 + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + + Node-path: greeting + Node-kind: file + Node-action: add + Prop-content-length: 10 + Text-content-length: 11 + Content-length: 21 + + PROPS-END + link hello + + Revision-number: 2 + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + + Node-path: greeting + Node-kind: file + Node-action: change + Prop-content-length: 33 + Content-length: 33 + + K 11 + svn:special + V 1 + * + PROPS-END + + Revision-number: 3 + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + + Node-path: greeting + Node-kind: file + Node-action: change + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + EOF + test-svn-fe filemode.dump >stream && + git fast-import actual && + git show HEAD:greeting >actual.blob && + git show HEAD^:greeting >actual.target && + test_cmp expect actual && + test_cmp expect.blob actual.blob && + test_cmp hello actual.target +' + +test_expect_success 'change file mode and reiterate content' ' + reinit_git && + cat >expect <<-\EOF && + OBJID + :120000 100644 OBJID OBJID T greeting + OBJID + :100644 120000 OBJID OBJID T greeting + OBJID + :000000 100644 OBJID OBJID A greeting + EOF + echo "link hello" >expect.blob && + echo hello >hello && + cat >filemode.dump <<-\EOF && + SVN-fs-dump-format-version: 3 + + Revision-number: 1 + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + + Node-path: greeting + Node-kind: file + Node-action: add + Prop-content-length: 10 + Text-content-length: 11 + Content-length: 21 + + PROPS-END + link hello + + Revision-number: 2 + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + + Node-path: greeting + Node-kind: file + Node-action: change + Prop-content-length: 33 + Text-content-length: 11 + Content-length: 44 + + K 11 + svn:special + V 1 + * + PROPS-END + link hello + + Revision-number: 3 + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + + Node-path: greeting + Node-kind: file + Node-action: change + Prop-content-length: 10 + Text-content-length: 11 + Content-length: 21 + + PROPS-END + link hello + EOF + test-svn-fe filemode.dump >stream && + git fast-import actual && + git show HEAD:greeting >actual.blob && + git show HEAD^:greeting >actual.target && + test_cmp expect actual && + test_cmp expect.blob actual.blob && + test_cmp hello actual.target +' + test_expect_success 'deltas not supported' ' { # (old) h + (inline) ello + (old) \n -- cgit v1.2.1 From 414e569e453a49171b1f3db613f88378324104e8 Mon Sep 17 00:00:00 2001 From: Jonathan Nieder Date: Fri, 19 Nov 2010 18:52:59 -0600 Subject: vcs-svn: Reject path nodes without Node-action It would be better to flag such errors and let the import proceed anyway, but for now it is simpler not to worry about recovery from such weird cases. Signed-off-by: Jonathan Nieder Signed-off-by: Junio C Hamano --- t/t9010-svn-fe.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) (limited to 't/t9010-svn-fe.sh') diff --git a/t/t9010-svn-fe.sh b/t/t9010-svn-fe.sh index 729e73dddf..cb9a236245 100755 --- a/t/t9010-svn-fe.sh +++ b/t/t9010-svn-fe.sh @@ -252,6 +252,26 @@ test_expect_success 'directory with files' ' test_cmp hi directory/file2 ' +test_expect_success 'node without action' ' + cat >inaction.dump <<-\EOF && + SVN-fs-dump-format-version: 3 + + Revision-number: 1 + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + + Node-path: directory + Node-kind: dir + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + EOF + test_must_fail test-svn-fe inaction.dump +' + test_expect_failure 'change file mode but keep old content' ' reinit_git && cat >expect <<-\EOF && -- cgit v1.2.1 From c7dbf35e91cffbc326078d0c0470662f6422150d Mon Sep 17 00:00:00 2001 From: Jonathan Nieder Date: Fri, 19 Nov 2010 18:53:34 -0600 Subject: vcs-svn: More dump format sanity checks Node-action: change is not appropriate when switching between file and directory or adding a new file. Current svn-fe silently accepts such nodes and the resulting tree has missing files in the "changed when meant to add" case. Node-action: add requires some content (text or directory); there is no such thing as an "intent to add" node in svn dumps. Current svn-fe accepts such contentless adds but produces an invalid fast-import stream that refers to nonexistent mark :0 in response. Signed-off-by: Jonathan Nieder Signed-off-by: Junio C Hamano --- t/t9010-svn-fe.sh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) (limited to 't/t9010-svn-fe.sh') diff --git a/t/t9010-svn-fe.sh b/t/t9010-svn-fe.sh index cb9a236245..f1e8799bbd 100755 --- a/t/t9010-svn-fe.sh +++ b/t/t9010-svn-fe.sh @@ -272,6 +272,27 @@ test_expect_success 'node without action' ' test_must_fail test-svn-fe inaction.dump ' +test_expect_success 'action: add node without text' ' + cat >textless.dump <<-\EOF && + SVN-fs-dump-format-version: 3 + + Revision-number: 1 + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + + Node-path: textless + Node-kind: file + Node-action: add + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + EOF + test_must_fail test-svn-fe textless.dump +' + test_expect_failure 'change file mode but keep old content' ' reinit_git && cat >expect <<-\EOF && -- cgit v1.2.1 From 6b01b67658e2905b550739f1aee56a00911ca13c Mon Sep 17 00:00:00 2001 From: David Barr Date: Fri, 19 Nov 2010 18:57:46 -0600 Subject: vcs-svn: Implement Prop-delta handling The rules for what file is used as delta source for each file are not documented in dump-load-format.txt. Luckily, the Apache Software Foundation repository has rich enough examples to figure out most of the rules: Node-action: replace implies the empty property set and empty text as preimage for deltas. Otherwise, if a copyfrom source is given, that node is the preimage for deltas. Lastly, if none of the above applies and the node path exists in the current revision, then that version forms the basis. [jn: refactored, with tests] Signed-off-by: David Barr Signed-off-by: Jonathan Nieder Signed-off-by: Junio C Hamano --- t/t9010-svn-fe.sh | 102 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 100 insertions(+), 2 deletions(-) (limited to 't/t9010-svn-fe.sh') diff --git a/t/t9010-svn-fe.sh b/t/t9010-svn-fe.sh index f1e8799bbd..7dc06707a1 100755 --- a/t/t9010-svn-fe.sh +++ b/t/t9010-svn-fe.sh @@ -514,7 +514,12 @@ test_expect_success 'deltas not supported' ' test_must_fail test-svn-fe delta.dump ' -test_expect_success 'property deltas not supported' ' +test_expect_success 'property deltas supported' ' + reinit_git && + cat >expect <<-\EOF && + OBJID + :100755 100644 OBJID OBJID M script.sh + EOF { properties \ svn:author author@example.com \ @@ -565,7 +570,100 @@ test_expect_success 'property deltas not supported' ' PROPS-END EOF } >propdelta.dump && - test_must_fail test-svn-fe propdelta.dump + test-svn-fe propdelta.dump >stream && + git fast-import actual && + test_cmp expect actual +' + +test_expect_success 'deltas for typechange' ' + reinit_git && + cat >expect <<-\EOF && + OBJID + :120000 100644 OBJID OBJID T test-file + OBJID + :100755 120000 OBJID OBJID T test-file + OBJID + :000000 100755 OBJID OBJID A test-file + EOF + cat >deleteprop.dump <<-\EOF && + SVN-fs-dump-format-version: 3 + + Revision-number: 1 + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + + Node-path: test-file + Node-kind: file + Node-action: add + Prop-delta: true + Prop-content-length: 35 + Text-content-length: 17 + Content-length: 52 + + K 14 + svn:executable + V 0 + + PROPS-END + link testing 123 + + Revision-number: 2 + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + + Node-path: test-file + Node-kind: file + Node-action: change + Prop-delta: true + Prop-content-length: 53 + Text-content-length: 17 + Content-length: 70 + + K 11 + svn:special + V 1 + * + D 14 + svn:executable + PROPS-END + link testing 231 + + Revision-number: 3 + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + + Node-path: test-file + Node-kind: file + Node-action: change + Prop-delta: true + Prop-content-length: 27 + Text-content-length: 17 + Content-length: 44 + + D 11 + svn:special + PROPS-END + link testing 321 + EOF + test-svn-fe deleteprop.dump >stream && + git fast-import actual && + test_cmp expect actual ' test_expect_success 't9135/svn.dump' ' -- cgit v1.2.1 From 9e8c532108b9078812f23c53a2df3509e7ce71bf Mon Sep 17 00:00:00 2001 From: Jonathan Nieder Date: Mon, 6 Dec 2010 16:19:32 -0600 Subject: vcs-svn: Allow change nodes for root of tree (/) It is not uncommon for a svn repository to include change records for properties at the top level of the tracked tree: Node-path: Node-kind: dir Node-action: change Prop-delta: true Prop-content-length: 43 Content-length: 43 K 10 svn:ignore V 11 build-area PROPS-END Unfortunately a recent svn-fe change (vcs-svn: More dump format sanity checks, 2010-11-19) causes such nodes to be rejected with the error message fatal: invalid dump: path to be modified is missing The repo_tree module does not keep a dirent for the root of the tree. Add a block to the dump parser to take care of this case. Signed-off-by: Jonathan Nieder Signed-off-by: Junio C Hamano --- t/t9010-svn-fe.sh | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) (limited to 't/t9010-svn-fe.sh') diff --git a/t/t9010-svn-fe.sh b/t/t9010-svn-fe.sh index 7dc06707a1..87945073b0 100755 --- a/t/t9010-svn-fe.sh +++ b/t/t9010-svn-fe.sh @@ -580,6 +580,61 @@ test_expect_success 'property deltas supported' ' test_cmp expect actual ' +test_expect_success 'properties on /' ' + reinit_git && + cat <<-\EOF >expect && + OBJID + OBJID + :000000 100644 OBJID OBJID A greeting + EOF + sed -e "s/X$//" <<-\EOF >changeroot.dump && + SVN-fs-dump-format-version: 3 + + Revision-number: 1 + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + + Node-path: greeting + Node-kind: file + Node-action: add + Text-content-length: 0 + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + + Revision-number: 2 + Prop-content-length: 10 + Content-length: 10 + + PROPS-END + + Node-path: X + Node-kind: dir + Node-action: change + Prop-delta: true + Prop-content-length: 43 + Content-length: 43 + + K 10 + svn:ignore + V 11 + build-area + + PROPS-END + EOF + test-svn-fe changeroot.dump >stream && + git fast-import actual && + test_cmp expect actual +' + test_expect_success 'deltas for typechange' ' reinit_git && cat >expect <<-\EOF && -- cgit v1.2.1