diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | Documentation/git-remote-helpers.txt | 21 | ||||
| -rw-r--r-- | Documentation/technical/api-argv-array.txt | 8 | ||||
| -rw-r--r-- | Makefile | 5 | ||||
| -rw-r--r-- | argv-array.c | 20 | ||||
| -rw-r--r-- | argv-array.h | 2 | ||||
| -rw-r--r-- | contrib/svn-fe/svn-fe.c | 3 | ||||
| -rwxr-xr-x | contrib/svn-fe/svnrdump_sim.py | 53 | ||||
| -rw-r--r-- | remote-testsvn.c | 342 | ||||
| -rwxr-xr-x | t/t9020-remote-svn.sh | 84 | ||||
| -rw-r--r-- | test-svn-fe.c | 2 | ||||
| -rw-r--r-- | transport-helper.c | 44 | ||||
| -rw-r--r-- | vcs-svn/fast_export.c | 36 | ||||
| -rw-r--r-- | vcs-svn/fast_export.h | 8 | ||||
| -rw-r--r-- | vcs-svn/svndump.c | 61 | ||||
| -rw-r--r-- | vcs-svn/svndump.h | 3 |
16 files changed, 655 insertions, 38 deletions
diff --git a/.gitignore b/.gitignore index a188a82bb1..f702415c12 100644 --- a/.gitignore +++ b/.gitignore @@ -125,6 +125,7 @@ /git-remote-fd /git-remote-ext /git-remote-testgit +/git-remote-testsvn /git-repack /git-replace /git-repo-config diff --git a/Documentation/git-remote-helpers.txt b/Documentation/git-remote-helpers.txt index f5836e46d0..5ce4cda8e7 100644 --- a/Documentation/git-remote-helpers.txt +++ b/Documentation/git-remote-helpers.txt @@ -98,6 +98,20 @@ advertised with this capability must cover all refs reported by the list command. If no 'refspec' capability is advertised, there is an implied `refspec *:*`. +'bidi-import':: + The fast-import commands 'cat-blob' and 'ls' can be used by remote-helpers + to retrieve information about blobs and trees that already exist in + fast-import's memory. This requires a channel from fast-import to the + remote-helper. + If it is advertised in addition to "import", git establishes a pipe from + fast-import to the remote-helper's stdin. + It follows that git and fast-import are both connected to the + remote-helper's stdin. Because git can send multiple commands to + the remote-helper it is required that helpers that use 'bidi-import' + buffer all 'import' commands of a batch before sending data to fast-import. + This is to prevent mixing commands and fast-import responses on the + helper's stdin. + Capabilities for Pushing ~~~~~~~~~~~~~~~~~~~~~~~~ 'connect':: @@ -286,7 +300,12 @@ terminated with a blank line. For each batch of 'import', the remote helper should produce a fast-import stream terminated by a 'done' command. + -Supported if the helper has the "import" capability. +Note that if the 'bidi-import' capability is used the complete batch +sequence has to be buffered before starting to send data to fast-import +to prevent mixing of commands and fast-import responses on the helper's +stdin. ++ +Supported if the helper has the 'import' capability. 'connect' <service>:: Connects to given service. Standard input and standard output diff --git a/Documentation/technical/api-argv-array.txt b/Documentation/technical/api-argv-array.txt index 1a797812fb..a959517b23 100644 --- a/Documentation/technical/api-argv-array.txt +++ b/Documentation/technical/api-argv-array.txt @@ -53,3 +53,11 @@ Functions `argv_array_clear`:: Free all memory associated with the array and return it to the initial, empty state. + +`argv_array_detach`:: + Detach the argv array from the `struct argv_array`, transfering + ownership of the allocated array and strings. + +`argv_array_free_detached`:: + Free the memory allocated by a `struct argv_array` that was later + detached and is now no longer needed. @@ -495,6 +495,7 @@ PROGRAM_OBJS += sh-i18n--envsubst.o PROGRAM_OBJS += shell.o PROGRAM_OBJS += show-index.o PROGRAM_OBJS += upload-pack.o +PROGRAM_OBJS += remote-testsvn.o # Binary suffix, set to .exe for Windows builds X = @@ -2449,6 +2450,10 @@ git-http-push$X: revision.o http.o http-push.o GIT-LDFLAGS $(GITLIBS) $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \ $(LIBS) $(CURL_LIBCURL) $(EXPAT_LIBEXPAT) +git-remote-testsvn$X: remote-testsvn.o GIT-LDFLAGS $(GITLIBS) $(VCSSVN_LIB) + $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) $(LIBS) \ + $(VCSSVN_LIB) + $(REMOTE_CURL_ALIASES): $(REMOTE_CURL_PRIMARY) $(QUIET_LNCP)$(RM) $@ && \ ln $< $@ 2>/dev/null || \ diff --git a/argv-array.c b/argv-array.c index 256741d226..9e960d549c 100644 --- a/argv-array.c +++ b/argv-array.c @@ -68,3 +68,23 @@ void argv_array_clear(struct argv_array *array) } argv_array_init(array); } + +const char **argv_array_detach(struct argv_array *array, int *argc) +{ + const char **argv = + array->argv == empty_argv || array->argc == 0 ? NULL : array->argv; + if (argc) + *argc = array->argc; + argv_array_init(array); + return argv; +} + +void argv_array_free_detached(const char **argv) +{ + if (argv) { + int i; + for (i = 0; argv[i]; i++) + free((char **)argv[i]); + free(argv); + } +} diff --git a/argv-array.h b/argv-array.h index f4b98660f8..40248d424c 100644 --- a/argv-array.h +++ b/argv-array.h @@ -18,5 +18,7 @@ void argv_array_pushf(struct argv_array *, const char *fmt, ...); void argv_array_pushl(struct argv_array *, ...); void argv_array_pop(struct argv_array *); void argv_array_clear(struct argv_array *); +const char **argv_array_detach(struct argv_array *array, int *argc); +void argv_array_free_detached(const char **argv); #endif /* ARGV_ARRAY_H */ diff --git a/contrib/svn-fe/svn-fe.c b/contrib/svn-fe/svn-fe.c index 35db24f5ea..f363505abb 100644 --- a/contrib/svn-fe/svn-fe.c +++ b/contrib/svn-fe/svn-fe.c @@ -10,7 +10,8 @@ int main(int argc, char **argv) { if (svndump_init(NULL)) return 1; - svndump_read((argc > 1) ? argv[1] : NULL); + svndump_read((argc > 1) ? argv[1] : NULL, "refs/heads/master", + "refs/notes/svn/revs"); svndump_deinit(); svndump_reset(); return 0; diff --git a/contrib/svn-fe/svnrdump_sim.py b/contrib/svn-fe/svnrdump_sim.py new file mode 100755 index 0000000000..1cfac4a6f8 --- /dev/null +++ b/contrib/svn-fe/svnrdump_sim.py @@ -0,0 +1,53 @@ +#!/usr/bin/python +""" +Simulates svnrdump by replaying an existing dump from a file, taking care +of the specified revision range. +To simulate incremental imports the environment variable SVNRMAX can be set +to the highest revision that should be available. +""" +import sys, os + + +def getrevlimit(): + var = 'SVNRMAX' + if os.environ.has_key(var): + return os.environ[var] + return None + +def writedump(url, lower, upper): + if url.startswith('sim://'): + filename = url[6:] + if filename[-1] == '/': filename = filename[:-1] #remove terminating slash + else: + raise ValueError('sim:// url required') + f = open(filename, 'r'); + state = 'header' + wroterev = False + while(True): + l = f.readline() + if l == '': break + if state == 'header' and l.startswith('Revision-number: '): + state = 'prefix' + if state == 'prefix' and l == 'Revision-number: %s\n' % lower: + state = 'selection' + if not upper == 'HEAD' and state == 'selection' and l == 'Revision-number: %s\n' % upper: + break; + + if state == 'header' or state == 'selection': + if state == 'selection': wroterev = True + sys.stdout.write(l) + return wroterev + +if __name__ == "__main__": + if not (len(sys.argv) in (3, 4, 5)): + print "usage: %s dump URL -rLOWER:UPPER" + sys.exit(1) + if not sys.argv[1] == 'dump': raise NotImplementedError('only "dump" is suppported.') + url = sys.argv[2] + r = ('0', 'HEAD') + if len(sys.argv) == 4 and sys.argv[3][0:2] == '-r': + r = sys.argv[3][2:].lstrip().split(':') + if not getrevlimit() is None: r[1] = getrevlimit() + if writedump(url, r[0], r[1]): ret = 0 + else: ret = 1 + sys.exit(ret) diff --git a/remote-testsvn.c b/remote-testsvn.c new file mode 100644 index 0000000000..51fba059a2 --- /dev/null +++ b/remote-testsvn.c @@ -0,0 +1,342 @@ +#include "cache.h" +#include "remote.h" +#include "strbuf.h" +#include "url.h" +#include "exec_cmd.h" +#include "run-command.h" +#include "vcs-svn/svndump.h" +#include "notes.h" +#include "argv-array.h" + +static const char *url; +static int dump_from_file; +static const char *private_ref; +static const char *remote_ref = "refs/heads/master"; +static const char *marksfilename, *notes_ref; +struct rev_note { unsigned int rev_nr; }; + +static int cmd_capabilities(const char *line); +static int cmd_import(const char *line); +static int cmd_list(const char *line); + +typedef int (*input_command_handler)(const char *); +struct input_command_entry { + const char *name; + input_command_handler fn; + unsigned char batchable; /* whether the command starts or is part of a batch */ +}; + +static const struct input_command_entry input_command_list[] = { + { "capabilities", cmd_capabilities, 0 }, + { "import", cmd_import, 1 }, + { "list", cmd_list, 0 }, + { NULL, NULL } +}; + +static int cmd_capabilities(const char *line) +{ + printf("import\n"); + printf("bidi-import\n"); + printf("refspec %s:%s\n\n", remote_ref, private_ref); + fflush(stdout); + return 0; +} + +static void terminate_batch(void) +{ + /* terminate a current batch's fast-import stream */ + printf("done\n"); + fflush(stdout); +} + +/* NOTE: 'ref' refers to a git reference, while 'rev' refers to a svn revision. */ +static char *read_ref_note(const unsigned char sha1[20]) +{ + const unsigned char *note_sha1; + char *msg = NULL; + unsigned long msglen; + enum object_type type; + + init_notes(NULL, notes_ref, NULL, 0); + if (!(note_sha1 = get_note(NULL, sha1))) + return NULL; /* note tree not found */ + if (!(msg = read_sha1_file(note_sha1, &type, &msglen))) + error("Empty notes tree. %s", notes_ref); + else if (!msglen || type != OBJ_BLOB) { + error("Note contains unusable content. " + "Is something else using this notes tree? %s", notes_ref); + free(msg); + msg = NULL; + } + free_notes(NULL); + return msg; +} + +static int parse_rev_note(const char *msg, struct rev_note *res) +{ + const char *key, *value, *end; + size_t len; + + while (*msg) { + end = strchr(msg, '\n'); + len = end ? end - msg : strlen(msg); + + key = "Revision-number: "; + if (!prefixcmp(msg, key)) { + long i; + char *end; + value = msg + strlen(key); + i = strtol(value, &end, 0); + if (end == value || i < 0 || i > UINT32_MAX) + return -1; + res->rev_nr = i; + } + msg += len + 1; + } + return 0; +} + +static int note2mark_cb(const unsigned char *object_sha1, + const unsigned char *note_sha1, char *note_path, + void *cb_data) +{ + FILE *file = (FILE *)cb_data; + char *msg; + unsigned long msglen; + enum object_type type; + struct rev_note note; + + if (!(msg = read_sha1_file(note_sha1, &type, &msglen)) || + !msglen || type != OBJ_BLOB) { + free(msg); + return 1; + } + if (parse_rev_note(msg, ¬e)) + return 2; + if (fprintf(file, ":%d %s\n", note.rev_nr, sha1_to_hex(object_sha1)) < 1) + return 3; + return 0; +} + +static void regenerate_marks(void) +{ + int ret; + FILE *marksfile = fopen(marksfilename, "w+"); + + if (!marksfile) + die_errno("Couldn't create mark file %s.", marksfilename); + ret = for_each_note(NULL, 0, note2mark_cb, marksfile); + if (ret) + die("Regeneration of marks failed, returned %d.", ret); + fclose(marksfile); +} + +static void check_or_regenerate_marks(int latestrev) +{ + FILE *marksfile; + struct strbuf sb = STRBUF_INIT; + struct strbuf line = STRBUF_INIT; + int found = 0; + + if (latestrev < 1) + return; + + init_notes(NULL, notes_ref, NULL, 0); + marksfile = fopen(marksfilename, "r"); + if (!marksfile) { + regenerate_marks(); + marksfile = fopen(marksfilename, "r"); + if (!marksfile) + die_errno("cannot read marks file %s!", marksfilename); + fclose(marksfile); + } else { + strbuf_addf(&sb, ":%d ", latestrev); + while (strbuf_getline(&line, marksfile, '\n') != EOF) { + if (!prefixcmp(line.buf, sb.buf)) { + found++; + break; + } + } + fclose(marksfile); + if (!found) + regenerate_marks(); + } + free_notes(NULL); + strbuf_release(&sb); + strbuf_release(&line); +} + +static int cmd_import(const char *line) +{ + int code; + int dumpin_fd; + char *note_msg; + unsigned char head_sha1[20]; + unsigned int startrev; + struct argv_array svndump_argv = ARGV_ARRAY_INIT; + struct child_process svndump_proc; + + if (read_ref(private_ref, head_sha1)) + startrev = 0; + else { + note_msg = read_ref_note(head_sha1); + if(note_msg == NULL) { + warning("No note found for %s.", private_ref); + startrev = 0; + } else { + struct rev_note note = { 0 }; + if (parse_rev_note(note_msg, ¬e)) + die("Revision number couldn't be parsed from note."); + startrev = note.rev_nr + 1; + free(note_msg); + } + } + check_or_regenerate_marks(startrev - 1); + + if (dump_from_file) { + dumpin_fd = open(url, O_RDONLY); + if(dumpin_fd < 0) + die_errno("Couldn't open svn dump file %s.", url); + } else { + memset(&svndump_proc, 0, sizeof(struct child_process)); + svndump_proc.out = -1; + argv_array_push(&svndump_argv, "svnrdump"); + argv_array_push(&svndump_argv, "dump"); + argv_array_push(&svndump_argv, url); + argv_array_pushf(&svndump_argv, "-r%u:HEAD", startrev); + svndump_proc.argv = svndump_argv.argv; + + code = start_command(&svndump_proc); + if (code) + die("Unable to start %s, code %d", svndump_proc.argv[0], code); + dumpin_fd = svndump_proc.out; + } + /* setup marks file import/export */ + printf("feature import-marks-if-exists=%s\n" + "feature export-marks=%s\n", marksfilename, marksfilename); + + svndump_init_fd(dumpin_fd, STDIN_FILENO); + svndump_read(url, private_ref, notes_ref); + svndump_deinit(); + svndump_reset(); + + close(dumpin_fd); + if (!dump_from_file) { + code = finish_command(&svndump_proc); + if (code) + warning("%s, returned %d", svndump_proc.argv[0], code); + argv_array_clear(&svndump_argv); + } + + return 0; +} + +static int cmd_list(const char *line) +{ + printf("? %s\n\n", remote_ref); + fflush(stdout); + return 0; +} + +static int do_command(struct strbuf *line) +{ + const struct input_command_entry *p = input_command_list; + static struct string_list batchlines = STRING_LIST_INIT_DUP; + static const struct input_command_entry *batch_cmd; + /* + * commands can be grouped together in a batch. + * Batches are ended by \n. If no batch is active the program ends. + * During a batch all lines are buffered and passed to the handler function + * when the batch is terminated. + */ + if (line->len == 0) { + if (batch_cmd) { + struct string_list_item *item; + for_each_string_list_item(item, &batchlines) + batch_cmd->fn(item->string); + terminate_batch(); + batch_cmd = NULL; + string_list_clear(&batchlines, 0); + return 0; /* end of the batch, continue reading other commands. */ + } + return 1; /* end of command stream, quit */ + } + if (batch_cmd) { + if (prefixcmp(batch_cmd->name, line->buf)) + die("Active %s batch interrupted by %s", batch_cmd->name, line->buf); + /* buffer batch lines */ + string_list_append(&batchlines, line->buf); + return 0; + } + + for (p = input_command_list; p->name; p++) { + if (!prefixcmp(line->buf, p->name) && (strlen(p->name) == line->len || + line->buf[strlen(p->name)] == ' ')) { + if (p->batchable) { + batch_cmd = p; + string_list_append(&batchlines, line->buf); + return 0; + } + return p->fn(line->buf); + } + } + die("Unknown command '%s'\n", line->buf); + return 0; +} + +int main(int argc, const char **argv) +{ + struct strbuf buf = STRBUF_INIT, url_sb = STRBUF_INIT, + private_ref_sb = STRBUF_INIT, marksfilename_sb = STRBUF_INIT, + notes_ref_sb = STRBUF_INIT; + static struct remote *remote; + const char *url_in; + + git_extract_argv0_path(argv[0]); + setup_git_directory(); + if (argc < 2 || argc > 3) { + usage("git-remote-svn <remote-name> [<url>]"); + return 1; + } + + remote = remote_get(argv[1]); + url_in = (argc == 3) ? argv[2] : remote->url[0]; + + if (!prefixcmp(url_in, "file://")) { + dump_from_file = 1; + url = url_decode(url_in + sizeof("file://")-1); + } else { + dump_from_file = 0; + end_url_with_slash(&url_sb, url_in); + url = url_sb.buf; + } + + strbuf_addf(&private_ref_sb, "refs/svn/%s/master", remote->name); + private_ref = private_ref_sb.buf; + + strbuf_addf(¬es_ref_sb, "refs/notes/%s/revs", remote->name); + notes_ref = notes_ref_sb.buf; + + strbuf_addf(&marksfilename_sb, "%s/info/fast-import/remote-svn/%s.marks", + get_git_dir(), remote->name); + marksfilename = marksfilename_sb.buf; + + while (1) { + if (strbuf_getline(&buf, stdin, '\n') == EOF) { + if (ferror(stdin)) + die("Error reading command stream"); + else + die("Unexpected end of command stream"); + } + if (do_command(&buf)) + break; + strbuf_reset(&buf); + } + + strbuf_release(&buf); + strbuf_release(&url_sb); + strbuf_release(&private_ref_sb); + strbuf_release(¬es_ref_sb); + strbuf_release(&marksfilename_sb); + return 0; +} diff --git a/t/t9020-remote-svn.sh b/t/t9020-remote-svn.sh new file mode 100755 index 0000000000..4f2dfe0e3d --- /dev/null +++ b/t/t9020-remote-svn.sh @@ -0,0 +1,84 @@ +#!/bin/sh + +test_description='tests remote-svn' + +. ./test-lib.sh + +MARKSPATH=.git/info/fast-import/remote-svn + +if ! test_have_prereq PYTHON +then + skip_all='skipping remote-svn tests, python not available' + test_done +fi + +# We override svnrdump by placing a symlink to the svnrdump-emulator in . +export PATH="$HOME:$PATH" +ln -sf $GIT_BUILD_DIR/contrib/svn-fe/svnrdump_sim.py "$HOME/svnrdump" + +init_git () { + rm -fr .git && + git init && + #git remote add svnsim testsvn::sim:///$TEST_DIRECTORY/t9020/example.svnrdump + # let's reuse an exisiting dump file!? + git remote add svnsim testsvn::sim://$TEST_DIRECTORY/t9154/svn.dump + git remote add svnfile testsvn::file://$TEST_DIRECTORY/t9154/svn.dump +} + +if test -e "$GIT_BUILD_DIR/git-remote-testsvn" +then + test_set_prereq REMOTE_SVN +fi + +test_debug ' + git --version + which git + which svnrdump +' + +test_expect_success REMOTE_SVN 'simple fetch' ' + init_git && + git fetch svnsim && + test_cmp .git/refs/svn/svnsim/master .git/refs/remotes/svnsim/master && + cp .git/refs/remotes/svnsim/master master.good +' + +test_debug ' + cat .git/refs/svn/svnsim/master + cat .git/refs/remotes/svnsim/master +' + +test_expect_success REMOTE_SVN 'repeated fetch, nothing shall change' ' + git fetch svnsim && + test_cmp master.good .git/refs/remotes/svnsim/master +' + +test_expect_success REMOTE_SVN 'fetch from a file:// url gives the same result' ' + git fetch svnfile +' + +test_expect_failure REMOTE_SVN 'the sha1 differ because the git-svn-id line in the commit msg contains the url' ' + test_cmp .git/refs/remotes/svnfile/master .git/refs/remotes/svnsim/master +' + +test_expect_success REMOTE_SVN 'mark-file regeneration' ' + # filter out any other marks, that can not be regenerated. Only up to 3 digit revisions are allowed here + grep ":[0-9]\{1,3\} " $MARKSPATH/svnsim.marks > $MARKSPATH/svnsim.marks.old && + rm $MARKSPATH/svnsim.marks && + git fetch svnsim && + test_cmp $MARKSPATH/svnsim.marks.old $MARKSPATH/svnsim.marks +' + +test_expect_success REMOTE_SVN 'incremental imports must lead to the same head' ' + export SVNRMAX=3 && + init_git && + git fetch svnsim && + test_cmp .git/refs/svn/svnsim/master .git/refs/remotes/svnsim/master && + unset SVNRMAX && + git fetch svnsim && + test_cmp master.good .git/refs/remotes/svnsim/master +' + +test_debug 'git branch -a' + +test_done diff --git a/test-svn-fe.c b/test-svn-fe.c index 83633a21e5..0f2d9a4a3d 100644 --- a/test-svn-fe.c +++ b/test-svn-fe.c @@ -40,7 +40,7 @@ int main(int argc, char *argv[]) if (argc == 2) { if (svndump_init(argv[1])) return 1; - svndump_read(NULL); + svndump_read(NULL, "refs/heads/master", "refs/notes/svn/revs"); svndump_deinit(); svndump_reset(); return 0; diff --git a/transport-helper.c b/transport-helper.c index cfe0988490..4713b69302 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -10,6 +10,7 @@ #include "string-list.h" #include "thread-utils.h" #include "sigchain.h" +#include "argv-array.h" static int debug; @@ -19,6 +20,7 @@ struct helper_data { FILE *out; unsigned fetch : 1, import : 1, + bidi_import : 1, export : 1, option : 1, push : 1, @@ -101,6 +103,7 @@ static void do_take_over(struct transport *transport) static struct child_process *get_helper(struct transport *transport) { struct helper_data *data = transport->data; + struct argv_array argv = ARGV_ARRAY_INIT; struct strbuf buf = STRBUF_INIT; struct child_process *helper; const char **refspecs = NULL; @@ -122,11 +125,10 @@ static struct child_process *get_helper(struct transport *transport) helper->in = -1; helper->out = -1; helper->err = 0; - helper->argv = xcalloc(4, sizeof(*helper->argv)); - strbuf_addf(&buf, "git-remote-%s", data->name); - helper->argv[0] = strbuf_detach(&buf, NULL); - helper->argv[1] = transport->remote->name; - helper->argv[2] = remove_ext_force(transport->url); + argv_array_pushf(&argv, "git-remote-%s", data->name); + argv_array_push(&argv, transport->remote->name); + argv_array_push(&argv, remove_ext_force(transport->url)); + helper->argv = argv_array_detach(&argv, NULL); helper->git_cmd = 0; helper->silent_exec_failure = 1; @@ -178,6 +180,8 @@ static struct child_process *get_helper(struct transport *transport) data->push = 1; else if (!strcmp(capname, "import")) data->import = 1; + else if (!strcmp(capname, "bidi-import")) + data->bidi_import = 1; else if (!strcmp(capname, "export")) data->export = 1; else if (!data->refspecs && !prefixcmp(capname, "refspec ")) { @@ -241,8 +245,7 @@ static int disconnect_helper(struct transport *transport) close(data->helper->out); fclose(data->out); res = finish_command(data->helper); - free((char *)data->helper->argv[0]); - free(data->helper->argv); + argv_array_free_detached(data->helper->argv); free(data->helper); data->helper = NULL; } @@ -376,14 +379,23 @@ static int fetch_with_fetch(struct transport *transport, static int get_importer(struct transport *transport, struct child_process *fastimport) { struct child_process *helper = get_helper(transport); + struct helper_data *data = transport->data; + struct argv_array argv = ARGV_ARRAY_INIT; + int cat_blob_fd, code; memset(fastimport, 0, sizeof(*fastimport)); fastimport->in = helper->out; - fastimport->argv = xcalloc(5, sizeof(*fastimport->argv)); - fastimport->argv[0] = "fast-import"; - fastimport->argv[1] = "--quiet"; + argv_array_push(&argv, "fast-import"); + argv_array_push(&argv, debug ? "--stats" : "--quiet"); + if (data->bidi_import) { + cat_blob_fd = xdup(helper->in); + argv_array_pushf(&argv, "--cat-blob-fd=%d", cat_blob_fd); + } + fastimport->argv = argv.argv; fastimport->git_cmd = 1; - return start_command(fastimport); + + code = start_command(fastimport); + return code; } static int get_exporter(struct transport *transport, @@ -438,11 +450,17 @@ static int fetch_with_import(struct transport *transport, } write_constant(data->helper->in, "\n"); + /* + * remote-helpers that advertise the bidi-import capability are required to + * buffer the complete batch of import commands until this newline before + * sending data to fast-import. + * These helpers read back data from fast-import on their stdin, which could + * be mixed with import commands, otherwise. + */ if (finish_command(&fastimport)) die("Error while running fast-import"); - free(fastimport.argv); - fastimport.argv = NULL; + argv_array_free_detached(fastimport.argv); /* * The fast-import stream of a remote helper that advertises diff --git a/vcs-svn/fast_export.c b/vcs-svn/fast_export.c index 1f04697866..f2b23c81de 100644 --- a/vcs-svn/fast_export.c +++ b/vcs-svn/fast_export.c @@ -3,8 +3,7 @@ * See LICENSE for details. */ -#include "git-compat-util.h" -#include "strbuf.h" +#include "cache.h" #include "quote.h" #include "fast_export.h" #include "repo_tree.h" @@ -68,11 +67,33 @@ void fast_export_modify(const char *path, uint32_t mode, const char *dataref) putchar('\n'); } +void fast_export_begin_note(uint32_t revision, const char *author, + const char *log, unsigned long timestamp, const char *note_ref) +{ + static int firstnote = 1; + size_t loglen = strlen(log); + printf("commit %s\n", note_ref); + printf("committer %s <%s@%s> %ld +0000\n", author, author, "local", timestamp); + printf("data %"PRIuMAX"\n", (uintmax_t)loglen); + fwrite(log, loglen, 1, stdout); + if (firstnote) { + if (revision > 1) + printf("from %s^0", note_ref); + firstnote = 0; + } + fputc('\n', stdout); +} + +void fast_export_note(const char *committish, const char *dataref) +{ + printf("N %s %s\n", dataref, committish); +} + static char gitsvnline[MAX_GITSVN_LINE_LEN]; void fast_export_begin_commit(uint32_t revision, const char *author, const struct strbuf *log, const char *uuid, const char *url, - unsigned long timestamp) + unsigned long timestamp, const char *local_ref) { static const struct strbuf empty = STRBUF_INIT; if (!log) @@ -84,7 +105,7 @@ void fast_export_begin_commit(uint32_t revision, const char *author, } else { *gitsvnline = '\0'; } - printf("commit refs/heads/master\n"); + printf("commit %s\n", local_ref); printf("mark :%"PRIu32"\n", revision); printf("committer %s <%s@%s> %ld +0000\n", *author ? author : "nobody", @@ -222,6 +243,13 @@ static long apply_delta(off_t len, struct line_buffer *input, return ret; } +void fast_export_buf_to_data(const struct strbuf *data) +{ + printf("data %"PRIuMAX"\n", (uintmax_t)data->len); + fwrite(data->buf, data->len, 1, stdout); + fputc('\n', stdout); +} + void fast_export_data(uint32_t mode, off_t len, struct line_buffer *input) { assert(len >= 0); diff --git a/vcs-svn/fast_export.h b/vcs-svn/fast_export.h index 8823aca15c..c8b5adb811 100644 --- a/vcs-svn/fast_export.h +++ b/vcs-svn/fast_export.h @@ -9,11 +9,15 @@ void fast_export_deinit(void); void fast_export_delete(const char *path); void fast_export_modify(const char *path, uint32_t mode, const char *dataref); +void fast_export_note(const char *committish, const char *dataref); +void fast_export_begin_note(uint32_t revision, const char *author, + const char *log, unsigned long timestamp, const char *note_ref); void fast_export_begin_commit(uint32_t revision, const char *author, - const struct strbuf *log, const char *uuid, - const char *url, unsigned long timestamp); + const struct strbuf *log, const char *uuid,const char *url, + unsigned long timestamp, const char *local_ref); void fast_export_end_commit(uint32_t revision); void fast_export_data(uint32_t mode, off_t len, struct line_buffer *input); +void fast_export_buf_to_data(const struct strbuf *data); void fast_export_blob_delta(uint32_t mode, uint32_t old_mode, const char *old_data, off_t len, struct line_buffer *input); diff --git a/vcs-svn/svndump.c b/vcs-svn/svndump.c index 2b168aee75..31d1d83d45 100644 --- a/vcs-svn/svndump.c +++ b/vcs-svn/svndump.c @@ -48,7 +48,7 @@ static struct { static struct { uint32_t revision; unsigned long timestamp; - struct strbuf log, author; + struct strbuf log, author, note; } rev_ctx; static struct { @@ -77,6 +77,7 @@ static void reset_rev_ctx(uint32_t revision) rev_ctx.timestamp = 0; strbuf_reset(&rev_ctx.log); strbuf_reset(&rev_ctx.author); + strbuf_reset(&rev_ctx.note); } static void reset_dump_ctx(const char *url) @@ -299,22 +300,29 @@ static void handle_node(void) node_ctx.text_length, &input); } -static void begin_revision(void) +static void begin_revision(const char *remote_ref) { if (!rev_ctx.revision) /* revision 0 gets no git commit. */ return; fast_export_begin_commit(rev_ctx.revision, rev_ctx.author.buf, &rev_ctx.log, dump_ctx.uuid.buf, dump_ctx.url.buf, - rev_ctx.timestamp); + rev_ctx.timestamp, remote_ref); } -static void end_revision(void) +static void end_revision(const char *note_ref) { - if (rev_ctx.revision) + struct strbuf mark = STRBUF_INIT; + if (rev_ctx.revision) { fast_export_end_commit(rev_ctx.revision); + fast_export_begin_note(rev_ctx.revision, "remote-svn", + "Note created by remote-svn.", rev_ctx.timestamp, note_ref); + strbuf_addf(&mark, ":%"PRIu32, rev_ctx.revision); + fast_export_note(mark.buf, "inline"); + fast_export_buf_to_data(&rev_ctx.note); + } } -void svndump_read(const char *url) +void svndump_read(const char *url, const char *local_ref, const char *notes_ref) { char *val; char *t; @@ -353,11 +361,12 @@ void svndump_read(const char *url) if (active_ctx == NODE_CTX) handle_node(); if (active_ctx == REV_CTX) - begin_revision(); + begin_revision(local_ref); if (active_ctx != DUMP_CTX) - end_revision(); + end_revision(notes_ref); active_ctx = REV_CTX; reset_rev_ctx(atoi(val)); + strbuf_addf(&rev_ctx.note, "%s\n", t); break; case sizeof("Node-path"): if (constcmp(t, "Node-")) @@ -366,13 +375,15 @@ void svndump_read(const char *url) if (active_ctx == NODE_CTX) handle_node(); if (active_ctx == REV_CTX) - begin_revision(); + begin_revision(local_ref); active_ctx = NODE_CTX; reset_node_ctx(val); + strbuf_addf(&rev_ctx.note, "%s\n", t); break; } if (constcmp(t + strlen("Node-"), "kind")) continue; + strbuf_addf(&rev_ctx.note, "%s\n", t); if (!strcmp(val, "dir")) node_ctx.type = REPO_MODE_DIR; else if (!strcmp(val, "file")) @@ -383,6 +394,7 @@ void svndump_read(const char *url) case sizeof("Node-action"): if (constcmp(t, "Node-action")) continue; + strbuf_addf(&rev_ctx.note, "%s\n", t); if (!strcmp(val, "delete")) { node_ctx.action = NODEACT_DELETE; } else if (!strcmp(val, "add")) { @@ -401,11 +413,13 @@ void svndump_read(const char *url) continue; strbuf_reset(&node_ctx.src); strbuf_addstr(&node_ctx.src, val); + strbuf_addf(&rev_ctx.note, "%s\n", t); break; case sizeof("Node-copyfrom-rev"): if (constcmp(t, "Node-copyfrom-rev")) continue; node_ctx.srcRev = atoi(val); + strbuf_addf(&rev_ctx.note, "%s\n", t); break; case sizeof("Text-content-length"): if (constcmp(t, "Text") && constcmp(t, "Prop")) @@ -463,25 +477,40 @@ void svndump_read(const char *url) if (active_ctx == NODE_CTX) handle_node(); if (active_ctx == REV_CTX) - begin_revision(); + begin_revision(local_ref); if (active_ctx != DUMP_CTX) - end_revision(); + end_revision(notes_ref); } -int svndump_init(const char *filename) +static void init(int report_fd) { - if (buffer_init(&input, filename)) - return error("cannot open %s: %s", filename, strerror(errno)); - fast_export_init(REPORT_FILENO); + fast_export_init(report_fd); strbuf_init(&dump_ctx.uuid, 4096); strbuf_init(&dump_ctx.url, 4096); strbuf_init(&rev_ctx.log, 4096); strbuf_init(&rev_ctx.author, 4096); + strbuf_init(&rev_ctx.note, 4096); strbuf_init(&node_ctx.src, 4096); strbuf_init(&node_ctx.dst, 4096); reset_dump_ctx(NULL); reset_rev_ctx(0); reset_node_ctx(NULL); + return; +} + +int svndump_init(const char *filename) +{ + if (buffer_init(&input, filename)) + return error("cannot open %s: %s", filename ? filename : "NULL", strerror(errno)); + init(REPORT_FILENO); + return 0; +} + +int svndump_init_fd(int in_fd, int back_fd) +{ + if(buffer_fdinit(&input, xdup(in_fd))) + return error("cannot open fd %d: %s", in_fd, strerror(errno)); + init(xdup(back_fd)); return 0; } @@ -492,6 +521,8 @@ void svndump_deinit(void) reset_rev_ctx(0); reset_node_ctx(NULL); strbuf_release(&rev_ctx.log); + strbuf_release(&rev_ctx.author); + strbuf_release(&rev_ctx.note); strbuf_release(&node_ctx.src); strbuf_release(&node_ctx.dst); if (buffer_deinit(&input)) diff --git a/vcs-svn/svndump.h b/vcs-svn/svndump.h index df9ceb0e8d..b8eb12954e 100644 --- a/vcs-svn/svndump.h +++ b/vcs-svn/svndump.h @@ -2,7 +2,8 @@ #define SVNDUMP_H_ int svndump_init(const char *filename); -void svndump_read(const char *url); +int svndump_init_fd(int in_fd, int back_fd); +void svndump_read(const char *url, const char *local_ref, const char *notes_ref); void svndump_deinit(void); void svndump_reset(void); |