From 41529bbce407fbf1a925cfbc7c1aa30064f66ae7 Mon Sep 17 00:00:00 2001 From: David Barr Date: Sat, 5 Mar 2011 13:30:23 +1100 Subject: vcs-svn: set up channel to read fast-import cat-blob response Set up some plumbing: teach the svndump lib to pass a file descriptor number to the fast_export lib, representing where cat-blob/ls responses can be read from, and add a get_response_line helper function to the fast_export lib to read a line from that file. Unfortunately this means that svn-fe needs file descriptor 3 to be redirected from somewhere (preferrably the cat-blob stream of a fast-import backend); otherwise it will fail: $ svndump | svn-fe fatal: cannot read from file descriptor 3: Bad file descriptor For the moment, "svn-fe 3 Signed-off-by: David Barr Signed-off-by: Jonathan Nieder --- vcs-svn/fast_export.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) (limited to 'vcs-svn/fast_export.c') diff --git a/vcs-svn/fast_export.c b/vcs-svn/fast_export.c index 260cf50e77..70bd9597e7 100644 --- a/vcs-svn/fast_export.c +++ b/vcs-svn/fast_export.c @@ -12,6 +12,24 @@ #define MAX_GITSVN_LINE_LEN 4096 static uint32_t first_commit_done; +static struct line_buffer report_buffer = LINE_BUFFER_INIT; + +void fast_export_init(int fd) +{ + if (buffer_fdinit(&report_buffer, fd)) + die_errno("cannot read from file descriptor %d", fd); +} + +void fast_export_deinit(void) +{ + if (buffer_deinit(&report_buffer)) + die_errno("error closing fast-import feedback stream"); +} + +void fast_export_reset(void) +{ + buffer_reset(&report_buffer); +} void fast_export_delete(uint32_t depth, uint32_t *path) { @@ -63,6 +81,16 @@ void fast_export_commit(uint32_t revision, uint32_t author, char *log, printf("progress Imported commit %"PRIu32".\n\n", revision); } +static const char *get_response_line(void) +{ + const char *line = buffer_read_line(&report_buffer); + if (line) + return line; + if (buffer_ferror(&report_buffer)) + die_errno("error reading from fast-import"); + die("unexpected end of fast-import feedback"); +} + void fast_export_blob(uint32_t mode, uint32_t mark, uint32_t len, struct line_buffer *input) { if (mode == REPO_MODE_LNK) { -- cgit v1.2.1 From 78e1a3ff236af3afaf1ba9db92985df42141cb0e Mon Sep 17 00:00:00 2001 From: Jonathan Nieder Date: Thu, 9 Dec 2010 18:57:13 -0600 Subject: vcs-svn: save marks for imported commits This way, a person can use svnadmin dump $path | svn-fe | git fast-import --relative-marks --export-marks=svn-revs to get a list of what commit corresponds to each svn revision (plus some irrelevant blob names) in .git/info/fast-import/svn-revs. Signed-off-by: Jonathan Nieder Signed-off-by: David Barr Signed-off-by: Jonathan Nieder --- vcs-svn/fast_export.c | 1 + 1 file changed, 1 insertion(+) (limited to 'vcs-svn/fast_export.c') diff --git a/vcs-svn/fast_export.c b/vcs-svn/fast_export.c index 70bd9597e7..0ad5382bfb 100644 --- a/vcs-svn/fast_export.c +++ b/vcs-svn/fast_export.c @@ -63,6 +63,7 @@ void fast_export_commit(uint32_t revision, uint32_t author, char *log, *gitsvnline = '\0'; } printf("commit refs/heads/master\n"); + printf("mark :%"PRIu32"\n", revision); printf("committer %s <%s@%s> %ld +0000\n", ~author ? pool_fetch(author) : "nobody", ~author ? pool_fetch(author) : "nobody", -- cgit v1.2.1 From 7e11902c995715836dec140eb55cfef1d24334bb Mon Sep 17 00:00:00 2001 From: Jonathan Nieder Date: Tue, 4 Jan 2011 21:53:33 -0600 Subject: vcs-svn: add a comment before each commit Current svn-fe produces output like this: blob mark :7382321 data 5 hello blob mark :7382322 data 5 Hello commit mark :3 [...] M 100644 :7382321 hello.c M 100644 :7382322 hello2.c This means svn-fe has to keep track of the paths modified in each commit and the corresponding marks, instead of dealing with each file as it arrives in input and then forgetting about it. A better strategy would be to use inline blobs: commit mark :3 [...] M 100644 inline hello.c data 5 hello [...] As a first step towards that, teach svn-fe to notice when the collection of blobs for each commit starts and write a comment ("# commit 3.") there. Signed-off-by: Jonathan Nieder --- vcs-svn/fast_export.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'vcs-svn/fast_export.c') diff --git a/vcs-svn/fast_export.c b/vcs-svn/fast_export.c index 0ad5382bfb..8786ed234a 100644 --- a/vcs-svn/fast_export.c +++ b/vcs-svn/fast_export.c @@ -48,6 +48,11 @@ void fast_export_modify(uint32_t depth, uint32_t *path, uint32_t mode, putchar('\n'); } +void fast_export_begin_commit(uint32_t revision) +{ + printf("# commit %"PRIu32".\n", revision); +} + static char gitsvnline[MAX_GITSVN_LINE_LEN]; void fast_export_commit(uint32_t revision, uint32_t author, char *log, uint32_t uuid, uint32_t url, -- cgit v1.2.1 From 723b7a2789d66c1365390cc9b9213e34ab8513d7 Mon Sep 17 00:00:00 2001 From: Jonathan Nieder Date: Fri, 10 Dec 2010 04:00:55 -0600 Subject: vcs-svn: eliminate repo_tree structure Rely on fast-import for information about previous revs. This requires always setting up backward flow of information, even for v2 dumps. On the plus side, it simplifies the code by quite a bit and opens the door to further simplifications. [db: adjusted to support final version of the cat-blob patch] [jn: avoiding hard-coding git's name for the empty tree for portability to other backends] Signed-off-by: Jonathan Nieder Signed-off-by: David Barr Signed-off-by: Jonathan Nieder --- vcs-svn/fast_export.c | 108 +++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 94 insertions(+), 14 deletions(-) (limited to 'vcs-svn/fast_export.c') diff --git a/vcs-svn/fast_export.c b/vcs-svn/fast_export.c index 8786ed234a..a8ce5c64b2 100644 --- a/vcs-svn/fast_export.c +++ b/vcs-svn/fast_export.c @@ -8,6 +8,7 @@ #include "line_buffer.h" #include "repo_tree.h" #include "string_pool.h" +#include "strbuf.h" #define MAX_GITSVN_LINE_LEN 4096 @@ -31,7 +32,7 @@ void fast_export_reset(void) buffer_reset(&report_buffer); } -void fast_export_delete(uint32_t depth, uint32_t *path) +void fast_export_delete(uint32_t depth, const uint32_t *path) { putchar('D'); putchar(' '); @@ -39,22 +40,27 @@ void fast_export_delete(uint32_t depth, uint32_t *path) putchar('\n'); } -void fast_export_modify(uint32_t depth, uint32_t *path, uint32_t mode, - uint32_t mark) +static void fast_export_truncate(uint32_t depth, const uint32_t *path, uint32_t mode) { - /* Mode must be 100644, 100755, 120000, or 160000. */ - printf("M %06"PRIo32" :%"PRIu32" ", mode, mark); - pool_print_seq(depth, path, '/', stdout); - putchar('\n'); + fast_export_modify(depth, path, mode, "inline"); + printf("data 0\n\n"); } -void fast_export_begin_commit(uint32_t revision) +void fast_export_modify(uint32_t depth, const uint32_t *path, uint32_t mode, + const char *dataref) { - printf("# commit %"PRIu32".\n", revision); + /* Mode must be 100644, 100755, 120000, or 160000. */ + if (!dataref) { + fast_export_truncate(depth, path, mode); + return; + } + printf("M %06"PRIo32" %s ", mode, dataref); + pool_print_seq(depth, path, '/', stdout); + putchar('\n'); } static char gitsvnline[MAX_GITSVN_LINE_LEN]; -void fast_export_commit(uint32_t revision, uint32_t author, char *log, +void fast_export_begin_commit(uint32_t revision, uint32_t author, char *log, uint32_t uuid, uint32_t url, unsigned long timestamp) { @@ -81,12 +87,31 @@ void fast_export_commit(uint32_t revision, uint32_t author, char *log, printf("from refs/heads/master^0\n"); first_commit_done = 1; } - repo_diff(revision - 1, revision); - fputc('\n', stdout); +} +void fast_export_end_commit(uint32_t revision) +{ printf("progress Imported commit %"PRIu32".\n\n", revision); } +static void ls_from_rev(uint32_t rev, uint32_t depth, const uint32_t *path) +{ + /* ls :5 path/to/old/file */ + printf("ls :%"PRIu32" ", rev); + pool_print_seq(depth, path, '/', stdout); + putchar('\n'); + fflush(stdout); +} + +static void ls_from_active_commit(uint32_t depth, const uint32_t *path) +{ + /* ls "path/to/file" */ + printf("ls \""); + pool_print_seq(depth, path, '/', stdout); + printf("\"\n"); + fflush(stdout); +} + static const char *get_response_line(void) { const char *line = buffer_read_line(&report_buffer); @@ -97,14 +122,69 @@ static const char *get_response_line(void) die("unexpected end of fast-import feedback"); } -void fast_export_blob(uint32_t mode, uint32_t mark, uint32_t len, struct line_buffer *input) +void fast_export_data(uint32_t mode, uint32_t len, struct line_buffer *input) { if (mode == REPO_MODE_LNK) { /* svn symlink blobs start with "link " */ buffer_skip_bytes(input, 5); len -= 5; } - printf("blob\nmark :%"PRIu32"\ndata %"PRIu32"\n", mark, len); + printf("data %"PRIu32"\n", len); buffer_copy_bytes(input, len); fputc('\n', stdout); } + +static int parse_ls_response(const char *response, uint32_t *mode, + struct strbuf *dataref) +{ + const char *tab; + const char *response_end; + + assert(response); + response_end = response + strlen(response); + + if (*response == 'm') { /* Missing. */ + errno = ENOENT; + return -1; + } + + /* Mode. */ + if (response_end - response < strlen("100644") || + response[strlen("100644")] != ' ') + die("invalid ls response: missing mode: %s", response); + *mode = 0; + for (; *response != ' '; response++) { + char ch = *response; + if (ch < '0' || ch > '7') + die("invalid ls response: mode is not octal: %s", response); + *mode *= 8; + *mode += ch - '0'; + } + + /* ' blob ' or ' tree ' */ + if (response_end - response < strlen(" blob ") || + (response[1] != 'b' && response[1] != 't')) + die("unexpected ls response: not a tree or blob: %s", response); + response += strlen(" blob "); + + /* Dataref. */ + tab = memchr(response, '\t', response_end - response); + if (!tab) + die("invalid ls response: missing tab: %s", response); + strbuf_add(dataref, response, tab - response); + return 0; +} + +int fast_export_ls_rev(uint32_t rev, uint32_t depth, const uint32_t *path, + uint32_t *mode, struct strbuf *dataref) +{ + ls_from_rev(rev, depth, path); + return parse_ls_response(get_response_line(), mode, dataref); +} + +int fast_export_ls(uint32_t depth, const uint32_t *path, + uint32_t *mode, struct strbuf *dataref) +{ + ls_from_active_commit(depth, path); + return parse_ls_response(get_response_line(), mode, dataref); +} -- cgit v1.2.1 From e43581120843f6f55f411af470faf806e052ad9d Mon Sep 17 00:00:00 2001 From: David Barr Date: Sun, 12 Dec 2010 03:59:31 +1100 Subject: vcs-svn: quote paths correctly for ls command This bug was found while importing rev 601865 of ASF. [jn: with test] Signed-off-by: David Barr Signed-off-by: Jonathan Nieder --- vcs-svn/fast_export.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'vcs-svn/fast_export.c') diff --git a/vcs-svn/fast_export.c b/vcs-svn/fast_export.c index a8ce5c64b2..4d57efabd5 100644 --- a/vcs-svn/fast_export.c +++ b/vcs-svn/fast_export.c @@ -107,7 +107,7 @@ static void ls_from_active_commit(uint32_t depth, const uint32_t *path) { /* ls "path/to/file" */ printf("ls \""); - pool_print_seq(depth, path, '/', stdout); + pool_print_seq_q(depth, path, '/', stdout); printf("\"\n"); fflush(stdout); } -- cgit v1.2.1 From 1ae469b06c50aade4781931ca1587453082f57eb Mon Sep 17 00:00:00 2001 From: Jonathan Nieder Date: Sat, 11 Dec 2010 17:08:51 -0600 Subject: vcs-svn: handle filenames with dq correctly Quote paths passed to fast-import so filenames with double quotes are not misinterpreted. One might imagine this could help with filenames with newlines, too, but svn does not allow those. Helped-by: David Barr Signed-off-by: Jonathan Nieder Signed-off-by: David Barr Signed-off-by: Jonathan Nieder --- vcs-svn/fast_export.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) (limited to 'vcs-svn/fast_export.c') diff --git a/vcs-svn/fast_export.c b/vcs-svn/fast_export.c index 4d57efabd5..9c03f3e16d 100644 --- a/vcs-svn/fast_export.c +++ b/vcs-svn/fast_export.c @@ -34,10 +34,9 @@ void fast_export_reset(void) void fast_export_delete(uint32_t depth, const uint32_t *path) { - putchar('D'); - putchar(' '); - pool_print_seq(depth, path, '/', stdout); - putchar('\n'); + printf("D \""); + pool_print_seq_q(depth, path, '/', stdout); + printf("\"\n"); } static void fast_export_truncate(uint32_t depth, const uint32_t *path, uint32_t mode) @@ -54,9 +53,9 @@ void fast_export_modify(uint32_t depth, const uint32_t *path, uint32_t mode, fast_export_truncate(depth, path, mode); return; } - printf("M %06"PRIo32" %s ", mode, dataref); - pool_print_seq(depth, path, '/', stdout); - putchar('\n'); + printf("M %06"PRIo32" %s \"", mode, dataref); + pool_print_seq_q(depth, path, '/', stdout); + printf("\"\n"); } static char gitsvnline[MAX_GITSVN_LINE_LEN]; @@ -97,9 +96,9 @@ void fast_export_end_commit(uint32_t revision) static void ls_from_rev(uint32_t rev, uint32_t depth, const uint32_t *path) { /* ls :5 path/to/old/file */ - printf("ls :%"PRIu32" ", rev); - pool_print_seq(depth, path, '/', stdout); - putchar('\n'); + printf("ls :%"PRIu32" \"", rev); + pool_print_seq_q(depth, path, '/', stdout); + printf("\"\n"); fflush(stdout); } -- cgit v1.2.1 From dd3f42ad793b5334d506a451addcefd0054c27bb Mon Sep 17 00:00:00 2001 From: David Barr Date: Sun, 12 Dec 2010 13:41:38 +1100 Subject: vcs-svn: use mark from previous import for parent commit With this patch, overlapping incremental imports work. Signed-off-by: David Barr Signed-off-by: Jonathan Nieder --- vcs-svn/fast_export.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'vcs-svn/fast_export.c') diff --git a/vcs-svn/fast_export.c b/vcs-svn/fast_export.c index 9c03f3e16d..f19db9ae82 100644 --- a/vcs-svn/fast_export.c +++ b/vcs-svn/fast_export.c @@ -83,7 +83,7 @@ void fast_export_begin_commit(uint32_t revision, uint32_t author, char *log, log, gitsvnline); if (!first_commit_done) { if (revision > 1) - printf("from refs/heads/master^0\n"); + printf("from :%"PRIu32"\n", revision - 1); first_commit_done = 1; } } -- cgit v1.2.1