diff options
| -rw-r--r-- | NEWS | 9 | ||||
| -rwxr-xr-x | include_server/include_analyzer.py | 6 | ||||
| -rw-r--r-- | man/distcc.1 | 48 | ||||
| -rw-r--r-- | src/clirpc.c | 4 | ||||
| -rw-r--r-- | src/compile.c | 19 | ||||
| -rw-r--r-- | src/compile.h | 5 | ||||
| -rw-r--r-- | src/distcc.c | 6 | ||||
| -rw-r--r-- | src/distcc.h | 4 | ||||
| -rw-r--r-- | src/include_server_if.c | 119 | ||||
| -rw-r--r-- | src/include_server_if.h | 1 | ||||
| -rw-r--r-- | src/util.c | 2 | ||||
| -rwxr-xr-x | test/testdistcc.py | 52 |
12 files changed, 262 insertions, 13 deletions
@@ -1,3 +1,12 @@ +New since distcc-3.0: + + FEATURES: + + * New "--scan-includes" option to distcc shows which headers + it would send to the remote machine in pump mode, without + actually compiling anything. This provides a simple interface + to the include server. + distcc-3.0 "The moment has arrived" 2008-08-06 DOCUMENTATION: diff --git a/include_server/include_analyzer.py b/include_server/include_analyzer.py index c284c3f..576fa17 100755 --- a/include_server/include_analyzer.py +++ b/include_server/include_analyzer.py @@ -313,8 +313,10 @@ class IncludeAnalyzer(object): """ must_exist_dirs = self.mirror_path.MustExistDirs() - random_name = 'forcing_technique_271828' - forcing_files = [d + '/' + random_name + # Note: distcc's --scan-includes option needs to + # know about this name; see ../src/compile.c. + special_name = 'forcing_technique_271828' + forcing_files = [d + '/' + special_name for d in must_exist_dirs] for forcing_file in forcing_files: # If for extremly obscure reasons the file already exists and is useful, diff --git a/man/distcc.1 b/man/distcc.1 index 96c7ee2..904c634 100644 --- a/man/distcc.1 +++ b/man/distcc.1 @@ -229,22 +229,62 @@ causes of violations of distcc-pump mode assumptions. .SH "OPTION SUMMARY" Most options passed to distcc are interpreted as compiler options. -The following options are understood by distcc itself: -.TP +The following options are understood by distcc itself. +If any of these options are specified, distcc will not invoke the +compiler. + +.TP .B --help Displays summary instructions. + .TP .B --version Displays the distcc client version. + .TP .B --show-hosts Displays the host list that distcc would use. See the Host Specifications section. + +.TP +.B --scan-includes +Displays the list of files that distcc would send to the +remote machine, as computed by the include server. This is a conservative +(over-)approximation of the files that would be read by the C compiler. +This option only works in pump mode. See the "How Distcc-pump Mode Works" +section for details on how this is computed. + +The list output by \fBdistcc --scan-includes\fR will +contain one entry per line. Each line contains a category followed by a path. +The category is one of FILE, SYMLINK, DIRECTORY, or SYSTEMDIR: + +.RS +.B FILE +indicates a source file or header file that would +be sent to the distcc server host. + +.B SYMLINK +indicates a symbolic link that would be sent to +the distcc server host. + +.B DIRECTORY +indicates a directory that may be needed in order to +compile the source file. For example, a directory "foo" may be needed +because of an include of the form #include "foo/../bar.h". +Such directories would be created on the distcc server host. + +.B SYSTEMDIR +indicates a system include directory, i.e. a directory +which is on the compiler's default include path, such as "/usr/include"; +such directories are assumed to be present on the distcc server host, +and so would not be sent to the distcc server host. +.RE + .TP .B -j Displays distcc's concurrency level, as calculated from the host list; -it is the maximum number of outstanding jobs issued by this client -to all servers +it is the maximum number of outstanding jobs issued by this client +to all servers. By default this will be four times the number of hosts in the host list, unless the /LIMIT option was used in the host list. See the Host Specifications section. diff --git a/src/clirpc.c b/src/clirpc.c index 2156860..bacbfad 100644 --- a/src/clirpc.c +++ b/src/clirpc.c @@ -210,7 +210,7 @@ int dcc_retrieve_results(int net_fd, } /* points_to must be at least MAXPATHLEN + 1 long */ -static int dcc_read_link(const char* fname, char *points_to) +int dcc_read_link(const char* fname, char *points_to) { int len; if ((len = readlink(fname, points_to, MAXPATHLEN)) == -1) { @@ -221,7 +221,7 @@ static int dcc_read_link(const char* fname, char *points_to) return 0; } -static int dcc_is_link(const char *fname, int *is_link) +int dcc_is_link(const char *fname, int *is_link) { struct stat buf; diff --git a/src/compile.c b/src/compile.c index cca49af..2fc60b7 100644 --- a/src/compile.c +++ b/src/compile.c @@ -60,10 +60,17 @@ #include "emaillog.h" #include "dotd.h" +/** + * This boolean is true iff --scan-includes option is enabled. + * If so, distcc will just run the source file through the include server, + * and print out the list of header files that might be #included, + * rather than actually compiling the sources. + */ +int dcc_scan_includes = 0; /* TODO(klarlund)Warning: this constant should have the same value as in the * pump.in script! Make it configurable and user changeable.*/ - static const int max_discrepancies_before_demotion = 1; +static const int max_discrepancies_before_demotion = 1; static const char *const include_server_port_suffix = "/socket"; static const char *const discrepancy_suffix = "/discrepancy_counter"; @@ -544,6 +551,10 @@ dcc_build_somewhere(char *argv[], dcc_perhaps_adjust_cpp_where_and_protover(input_fname, host, discrepancy_filename); } + if (dcc_scan_includes) { + ret = dcc_approximate_includes(host, argv); + goto unlock_and_clean_up; + } if (host->cpp_where == DCC_CPP_ON_SERVER) { if ((ret = dcc_talk_to_include_server(argv, &files))) { /* Fallback to doing cpp locally */ @@ -719,10 +730,16 @@ dcc_build_somewhere(char *argv[], discrepancy_filename); } + unlock_and_clean_up: if (cpu_lock_fd != -1) { dcc_unlock(cpu_lock_fd); cpu_lock_fd = -1; /* Not really needed, just for consistency. */ } + /* For the --scan_includes case. */ + if (local_cpu_lock_fd != -1) { + dcc_unlock(local_cpu_lock_fd); + local_cpu_lock_fd = -1; /* Not really needed, just for consistency. */ + } clean_up: dcc_free_argv(argv); diff --git a/src/compile.h b/src/compile.h index ce0a5ad..5eaaf49 100644 --- a/src/compile.h +++ b/src/compile.h @@ -22,6 +22,7 @@ */ /* remote.c */ + int dcc_compile_remote(char **argv, char *input_fname, char *cpp_fname, @@ -34,6 +35,10 @@ int dcc_compile_remote(char **argv, struct dcc_hostdef *host, int *status); +/* compile.c */ + +extern int dcc_scan_includes; + int dcc_build_somewhere_timed(char *argv[], int sg_level, int *status); diff --git a/src/distcc.c b/src/distcc.c index f97b4e8..ee360d5 100644 --- a/src/distcc.c +++ b/src/distcc.c @@ -55,7 +55,6 @@ /* Name of this program, for trace.c */ const char *rs_program_name = "distcc"; - /** * @file * @@ -241,6 +240,11 @@ int main(int argc, char **argv) goto out; } + if (!strcmp(argv[1], "--scan-includes")) { + dcc_scan_includes = 1; + argv++; + } + if ((ret = dcc_find_compiler(argv, &compiler_args)) != 0) { goto out; } diff --git a/src/distcc.h b/src/distcc.h index aff2203..d4fb44b 100644 --- a/src/distcc.h +++ b/src/distcc.h @@ -157,6 +157,8 @@ int dcc_x_req_header(int fd, enum dcc_protover protover); int dcc_x_argv(int fd, char **argv); int dcc_x_cwd(int fd); +int dcc_is_link(const char *fname, int *is_link); +int dcc_read_link(const char* fname, char *points_to); /* srvrpc.c */ int dcc_r_cwd(int ifd, char **cwd); @@ -239,7 +241,7 @@ int dcc_compress_lzo1x_alloc(const char *in_buf, -/* bulk.h */ +/* bulk.c */ void dcc_calc_rate(off_t size_out, struct timeval *before, struct timeval *after, diff --git a/src/include_server_if.c b/src/include_server_if.c index fd0dbb4..f7ebadf 100644 --- a/src/include_server_if.c +++ b/src/include_server_if.c @@ -17,10 +17,11 @@ * USA. */ -/* Author: Manos Renieris */ +/* Authors: Manos Renieris, Fergus Henderson */ #include <string.h> #include <stdlib.h> +#include <stdio.h> #include <sys/stat.h> #include <sys/types.h> @@ -36,8 +37,13 @@ #include "clinet.h" #include "exitcode.h" #include "util.h" +#include "hosts.h" #include "include_server_if.h" +static int dcc_count_slashes(const char *path); +static int dcc_count_leading_dotdots(const char *path); +static int dcc_categorize_file(const char *include_server_filename); + /* The include server puts all files in its own special directory, * which is n path components long, where n = INCLUDE_SERVER_DIR_DEPTH */ @@ -162,3 +168,114 @@ int dcc_get_original_fname(const char *fname, char **original_fname) free(alloced_work); return 0; } + +/** + * This implements the --scan_includes option. + * Talks to the include server, and prints the results to stdout. + */ +int +dcc_approximate_includes(struct dcc_hostdef *host, char **argv) +{ + char **files; + int i; + int ret; + + if (host->cpp_where != DCC_CPP_ON_SERVER) { + rs_log_error("'--scan_includes' specified, " + "but distcc wouldn't have used include server " + "(make sure hosts list includes ',cpp' option?)"); + return EXIT_DISTCC_FAILED; + //return 0; + } + + if ((ret = dcc_talk_to_include_server(argv, &files))) { + rs_log_error("failed to get includes from include server"); + return ret; + } + + for (i = 0; files[i]; i++) { + if ((ret = dcc_categorize_file(files[i]))) + return ret; + } + + return 0; +} + +/* + * A subroutine of dcc_approximate_includes(). + * Take a filename output from the include server, + * convert the filename back so that it refers to the original source tree + * (as opposed to the include server's mirror tree), + * categorize it as SYSTEMDIR, DIRECTORY, SYMLINK, or FILE, + * and print the category and original name to stdout. + * For SYMLINKs, also print out what the symlink points to. + */ +static int +dcc_categorize_file(const char *include_server_filename) { + char *filename; + int is_symlink = 0; + int is_forced_directory = 0; + int is_system_include_directory = 0; + char link_target[MAXPATHLEN + 1]; + int ret; + + if ((ret = dcc_is_link(include_server_filename, &is_symlink))) + return ret; + + if (is_symlink) + if ((ret = dcc_read_link(include_server_filename, link_target))) + return ret; + + if ((ret = dcc_get_original_fname(include_server_filename, &filename))) { + rs_log_error("dcc_get_original_fname failed"); + return ret; + } + + if (str_endswith("/forcing_technique_271828", filename)) { + /* Replace "<foo>/forcing_technique_271818" with "<foo>". */ + filename[strlen(filename) - strlen("/forcing_technique_271828")] + = '\0'; + is_forced_directory = 1; + } + + if (is_symlink) { + int leading_dotdots = dcc_count_leading_dotdots(link_target); + is_system_include_directory = + leading_dotdots > 0 && + leading_dotdots > dcc_count_slashes(filename) && + strcmp(link_target + 3 * leading_dotdots - 1, filename) == 0; + } + + printf("%-9s %s\n", is_system_include_directory ? "SYSTEMDIR" : + is_forced_directory ? "DIRECTORY" : + is_symlink ? "SYMLINK" : + "FILE", + filename); + + return 0; +} + +/* Count the number of slashes in a path. */ +static int +dcc_count_slashes(const char *path) +{ + int i; + int count = 0; + for (i = 0; path[i]; i++) { + if (path[i] == '/') + count++; + } + return count; +} + +/* Count the number of leading "../" references in a path. */ +static int +dcc_count_leading_dotdots(const char *path) +{ + int count = 0; + while (str_startswith("../", path)) { + path += 3; + count++; + } + return count; +} diff --git a/src/include_server_if.h b/src/include_server_if.h index 2c3d380..409044b 100644 --- a/src/include_server_if.h +++ b/src/include_server_if.h @@ -21,3 +21,4 @@ int dcc_talk_to_include_server(char **argv, char ***files); int dcc_get_original_fname(const char *fname, char **original_fname); +int dcc_approximate_includes(struct dcc_hostdef *host, char **argv); @@ -87,7 +87,7 @@ void dcc_exit(int exitcode) int str_endswith(const char *tail, const char *tiger) { - size_t len_tail = strlen(tail); + size_t len_tail = strlen(tail); size_t len_tiger = strlen(tiger); if (len_tail > len_tiger) diff --git a/test/testdistcc.py b/test/testdistcc.py index 7a0a935..938e531 100755 --- a/test/testdistcc.py +++ b/test/testdistcc.py @@ -1521,6 +1521,58 @@ class DashWpMD_Case(CompileHello_Case): # Pump mode is treating -MD as if it was -MMD. # self.assert_re_search(r"stdio.h", deps); +class ScanIncludes_Case(CompileHello_Case): + """Test --scan-includes""" + + def createSource(self): + CompileHello_Case.createSource(self) + self.runcmd("mv testhdr.h test_header.h") + self.runcmd("ln -s test_header.h testhdr.h") + self.runcmd("mkdir test_subdir") + self.runcmd("touch test_another_header.h") + + def headerSource(self): + return """ +#define HELLO_WORLD "hello world" +#include "test_subdir/../test_another_header.h" +""" + + def compileCmd(self): + return self.distcc_without_fallback() + "--scan-includes " + \ + _gcc + " -o testtmp.o " + self.compileOpts() + \ + " -c %s" % (self.sourceFilename()) + + def runtest(self): + cmd = self.compileCmd() + rc, out, err = self.runcmd_unchecked(cmd) + log = open('distcc.log').read() + pump_mode = _server_options.find('cpp') != -1 + if pump_mode: + if err != '': + self.fail("distcc command %s produced stderr:\n%s" % (`cmd`, err)) + if rc != 0: + self.fail("distcc command %s failed:\n%s" % (`cmd`, rc)) + self.assert_re_search( + r"FILE /.*/ScanIncludes_Case/testtmp.c", out); + self.assert_re_search( + r"FILE /.*/ScanIncludes_Case/test_header\.h", out); + self.assert_re_search( + r"FILE /.*/ScanIncludes_Case/test_another_header\.h", out); + self.assert_re_search( + r"SYMLINK /.*/ScanIncludes_Case/testhdr\.h", out); + self.assert_re_search( + r"DIRECTORY /.*/ScanIncludes_Case/test_subdir", out); + self.assert_re_search( + r"SYSTEMDIR /.*", out); + else: + self.assert_re_search(r"ERROR: '--scan_includes' specified, but " + "distcc wouldn't have used include server " + ".make sure hosts list includes ',cpp' option", + log) + self.assert_equal(rc, 100) + self.assert_equal(out, '') + self.assert_equal(err, '') + class AbsSourceFilename_Case(CompileHello_Case): """Test remote compilation of files with absolute names.""" |