Provides a simple shell API to the include server:

add a new "--scan-includes" option to distcc which shows
which headers distcc would send to the remote machine in
pump mode, without actually compiling anything.



git-svn-id: http://distcc.googlecode.com/svn/trunk@613 01de4be4-8c4a-0410-9132-4925637da917

12 files changed, 262 insertions, 13 deletions

diff --git a/NEWS b/NEWS
index 8bfbdef..d4ca28e 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,12 @@
+New since distcc-3.0:
+
+  FEATURES:
+
+   * New "--scan-includes" option to distcc shows which headers
+     it would send to the remote machine in pump mode, without
+     actually compiling anything.  This provides a simple interface
+     to the include server.
+
 distcc-3.0 "The moment has arrived"   2008-08-06
 
   DOCUMENTATION:
diff --git a/include_server/include_analyzer.py b/include_server/include_analyzer.py
index c284c3f..576fa17 100755
--- a/include_server/include_analyzer.py
+++ b/include_server/include_analyzer.py
@@ -313,8 +313,10 @@ class IncludeAnalyzer(object):
     """
 
     must_exist_dirs = self.mirror_path.MustExistDirs()
-    random_name = 'forcing_technique_271828'
-    forcing_files = [d + '/' + random_name
+    # Note: distcc's --scan-includes option needs to
+    # know about this name; see ../src/compile.c.
+    special_name = 'forcing_technique_271828'
+    forcing_files = [d + '/' + special_name
                      for d in must_exist_dirs]
     for forcing_file in forcing_files:
       # If for extremly obscure reasons the file already exists and is useful,
diff --git a/man/distcc.1 b/man/distcc.1
index 96c7ee2..904c634 100644
--- a/man/distcc.1
+++ b/man/distcc.1
@@ -229,22 +229,62 @@ causes of violations of distcc-pump mode assumptions.
 
 .SH "OPTION SUMMARY"
 Most options passed to distcc are interpreted as compiler options.
-The following options are understood by distcc itself:
-.TP 
+The following options are understood by distcc itself.
+If any of these options are specified, distcc will not invoke the
+compiler.
+
+.TP
 .B --help
 Displays summary instructions.
+
 .TP
 .B --version
 Displays the distcc client version.
+
 .TP
 .B --show-hosts
 Displays the host list that distcc would use.
 See the Host Specifications section.
+
+.TP
+.B --scan-includes
+Displays the list of files that distcc would send to the
+remote machine, as computed by the include server.  This is a conservative
+(over-)approximation of the files that would be read by the C compiler.
+This option only works in pump mode.  See the "How Distcc-pump Mode Works"
+section for details on how this is computed.
+
+The list output by \fBdistcc --scan-includes\fR will
+contain one entry per line.  Each line contains a category followed by a path.
+The category is one of FILE, SYMLINK, DIRECTORY, or SYSTEMDIR:
+
+.RS
+.B FILE
+indicates a source file or header file that would
+be sent to the distcc server host.
+
+.B SYMLINK
+indicates a symbolic link that would be sent to
+the distcc server host.
+
+.B DIRECTORY
+indicates a directory that may be needed in order to
+compile the source file.  For example, a directory "foo" may be needed
+because of an include of the form #include "foo/../bar.h".
+Such directories would be created on the distcc server host.
+
+.B SYSTEMDIR
+indicates a system include directory, i.e. a directory
+which is on the compiler's default include path, such as "/usr/include";
+such directories are assumed to be present on the distcc server host,
+and so would not be sent to the distcc server host.
+.RE
+
 .TP
 .B -j
 Displays distcc's concurrency level, as calculated from the host list;
-it is the maximum number of outstanding jobs issued by this client 
-to all servers
+it is the maximum number of outstanding jobs issued by this client
+to all servers.
 By default this will be four times the number of hosts in the host list,
 unless the /LIMIT option was used in the host list.
 See the Host Specifications section.
diff --git a/src/clirpc.c b/src/clirpc.c
index 2156860..bacbfad 100644
--- a/src/clirpc.c
+++ b/src/clirpc.c
@@ -210,7 +210,7 @@ int dcc_retrieve_results(int net_fd,
 }
 
 /* points_to must be at least MAXPATHLEN + 1 long */
-static int dcc_read_link(const char* fname, char *points_to)
+int dcc_read_link(const char* fname, char *points_to)
 {
     int len;
     if ((len = readlink(fname, points_to, MAXPATHLEN)) == -1) {
@@ -221,7 +221,7 @@ static int dcc_read_link(const char* fname, char *points_to)
     return 0;
 }
 
-static int dcc_is_link(const char *fname, int *is_link)
+int dcc_is_link(const char *fname, int *is_link)
 {
     struct stat buf;
 
diff --git a/src/compile.c b/src/compile.c
index cca49af..2fc60b7 100644
--- a/src/compile.c
+++ b/src/compile.c
@@ -60,10 +60,17 @@
 #include "emaillog.h"
 #include "dotd.h"
 
+/**
+ * This boolean is true iff --scan-includes option is enabled.
+ * If so, distcc will just run the source file through the include server,
+ * and print out the list of header files that might be #included,
+ * rather than actually compiling the sources.
+ */
+int dcc_scan_includes = 0;
 
 /* TODO(klarlund)Warning: this constant should have the same value as in the
  * pump.in script! Make it configurable and user changeable.*/
- static const int max_discrepancies_before_demotion = 1;
+static const int max_discrepancies_before_demotion = 1;
 
 static const char *const include_server_port_suffix = "/socket";
 static const char *const discrepancy_suffix = "/discrepancy_counter";
@@ -544,6 +551,10 @@ dcc_build_somewhere(char *argv[],
         dcc_perhaps_adjust_cpp_where_and_protover(input_fname, host,
                                                   discrepancy_filename);
     }
+    if (dcc_scan_includes) {
+        ret = dcc_approximate_includes(host, argv);
+        goto unlock_and_clean_up;
+    }
     if (host->cpp_where == DCC_CPP_ON_SERVER) {
         if ((ret = dcc_talk_to_include_server(argv, &files))) {
             /* Fallback to doing cpp locally */
@@ -719,10 +730,16 @@ dcc_build_somewhere(char *argv[],
             discrepancy_filename);
     }
 
+  unlock_and_clean_up:
     if (cpu_lock_fd != -1) {
         dcc_unlock(cpu_lock_fd);
         cpu_lock_fd = -1; /* Not really needed, just for consistency. */
     }
+    /* For the --scan_includes case. */
+    if (local_cpu_lock_fd != -1) {
+        dcc_unlock(local_cpu_lock_fd);
+        local_cpu_lock_fd = -1; /* Not really needed, just for consistency. */
+    }
 
   clean_up:
     dcc_free_argv(argv);
diff --git a/src/compile.h b/src/compile.h
index ce0a5ad..5eaaf49 100644
--- a/src/compile.h
+++ b/src/compile.h
@@ -22,6 +22,7 @@
  */
 
 /* remote.c */
+
 int dcc_compile_remote(char **argv,
                        char *input_fname,
                        char *cpp_fname,
@@ -34,6 +35,10 @@ int dcc_compile_remote(char **argv,
                        struct dcc_hostdef *host,
                        int *status);
 
+/* compile.c */
+
+extern int dcc_scan_includes;
+
 int dcc_build_somewhere_timed(char *argv[],
                               int sg_level,
                               int *status);
diff --git a/src/distcc.c b/src/distcc.c
index f97b4e8..ee360d5 100644
--- a/src/distcc.c
+++ b/src/distcc.c
@@ -55,7 +55,6 @@
 /* Name of this program, for trace.c */
 const char *rs_program_name = "distcc";
 
-
 /**
  * @file
  *
@@ -241,6 +240,11 @@ int main(int argc, char **argv)
             goto out;
         }
 
+        if (!strcmp(argv[1], "--scan-includes")) {
+            dcc_scan_includes = 1;
+            argv++;
+        }
+
         if ((ret = dcc_find_compiler(argv, &compiler_args)) != 0) {
             goto out;
         }
diff --git a/src/distcc.h b/src/distcc.h
index aff2203..d4fb44b 100644
--- a/src/distcc.h
+++ b/src/distcc.h
@@ -157,6 +157,8 @@ int dcc_x_req_header(int fd,
                      enum dcc_protover protover);
 int dcc_x_argv(int fd, char **argv);
 int dcc_x_cwd(int fd);
+int dcc_is_link(const char *fname, int *is_link);
+int dcc_read_link(const char* fname, char *points_to);
 
 /* srvrpc.c */
 int dcc_r_cwd(int ifd, char **cwd);
@@ -239,7 +241,7 @@ int dcc_compress_lzo1x_alloc(const char *in_buf,
 
 
 
-/* bulk.h */
+/* bulk.c */
 void dcc_calc_rate(off_t size_out,
                    struct timeval *before,
                    struct timeval *after,
diff --git a/src/include_server_if.c b/src/include_server_if.c
index fd0dbb4..f7ebadf 100644
--- a/src/include_server_if.c
+++ b/src/include_server_if.c
@@ -17,10 +17,11 @@
  * USA.
 */
 
-/* Author: Manos Renieris */
+/* Authors: Manos Renieris, Fergus Henderson */
 
 #include <string.h>
 #include <stdlib.h>
+#include <stdio.h>
 
 #include <sys/stat.h>
 #include <sys/types.h>
@@ -36,8 +37,13 @@
 #include "clinet.h"
 #include "exitcode.h"
 #include "util.h"
+#include "hosts.h"
 #include "include_server_if.h"
 
+static int dcc_count_slashes(const char *path);
+static int dcc_count_leading_dotdots(const char *path);
+static int dcc_categorize_file(const char *include_server_filename);
+
 /* The include server puts all files in its own special directory,
  * which is n path components long, where n = INCLUDE_SERVER_DIR_DEPTH
  */
@@ -162,3 +168,114 @@ int dcc_get_original_fname(const char *fname, char **original_fname)
     free(alloced_work);
     return 0;
 }
+
+/**
+ * This implements the --scan_includes option.
+ * Talks to the include server, and prints the results to stdout.
+ */
+int
+dcc_approximate_includes(struct dcc_hostdef *host, char **argv)
+{
+    char **files;
+    int i;
+    int ret;
+
+    if (host->cpp_where != DCC_CPP_ON_SERVER) {
+        rs_log_error("'--scan_includes' specified, "
+                     "but distcc wouldn't have used include server "
+                     "(make sure hosts list includes ',cpp' option?)");
+        return EXIT_DISTCC_FAILED;
+        //return 0;
+    }
+
+    if ((ret = dcc_talk_to_include_server(argv, &files))) {
+        rs_log_error("failed to get includes from include server");
+        return ret;
+    }
+
+    for (i = 0; files[i]; i++) {
+        if ((ret = dcc_categorize_file(files[i])))
+            return ret;
+    }
+
+    return 0;
+}
+
+/*
+ * A subroutine of dcc_approximate_includes().
+ * Take a filename output from the include server,
+ * convert the filename back so that it refers to the original source tree
+ * (as opposed to the include server's mirror tree),
+ * categorize it as SYSTEMDIR, DIRECTORY, SYMLINK, or FILE,
+ * and print the category and original name to stdout.
+ * For SYMLINKs, also print out what the symlink points to.
+ */
+static int
+dcc_categorize_file(const char *include_server_filename) {
+    char *filename;
+    int is_symlink = 0;
+    int is_forced_directory = 0;
+    int is_system_include_directory = 0;
+    char link_target[MAXPATHLEN + 1];
+    int ret;
+
+    if ((ret = dcc_is_link(include_server_filename, &is_symlink)))
+        return ret;
+
+    if (is_symlink)
+        if ((ret = dcc_read_link(include_server_filename, link_target)))
+            return ret;
+
+    if ((ret = dcc_get_original_fname(include_server_filename, &filename))) {
+        rs_log_error("dcc_get_original_fname failed");
+        return ret;
+    }
+
+    if (str_endswith("/forcing_technique_271828", filename)) {
+        /* Replace "<foo>/forcing_technique_271818" with "<foo>". */
+        filename[strlen(filename) - strlen("/forcing_technique_271828")]
+            = '\0';
+        is_forced_directory = 1;
+    }
+
+    if (is_symlink) {
+        int leading_dotdots = dcc_count_leading_dotdots(link_target);
+        is_system_include_directory =
+            leading_dotdots > 0 &&
+            leading_dotdots > dcc_count_slashes(filename) &&
+            strcmp(link_target + 3 * leading_dotdots - 1, filename) == 0;
+    }
+
+    printf("%-9s %s\n", is_system_include_directory ? "SYSTEMDIR" :
+                        is_forced_directory         ? "DIRECTORY" :
+                        is_symlink                  ? "SYMLINK" :
+                                                      "FILE",
+                        filename);
+
+    return 0;
+}
+
+/* Count the number of slashes in a path. */
+static int
+dcc_count_slashes(const char *path)
+{
+    int i;
+    int count = 0;
+    for (i = 0; path[i]; i++) {
+        if (path[i] == '/')
+            count++;
+    }
+    return count;
+}
+
+/* Count the number of leading "../" references in a path. */
+static int
+dcc_count_leading_dotdots(const char *path)
+{
+    int count = 0;
+    while (str_startswith("../", path)) {
+        path += 3;
+        count++;
+    }
+    return count;
+}
diff --git a/src/include_server_if.h b/src/include_server_if.h
index 2c3d380..409044b 100644
--- a/src/include_server_if.h
+++ b/src/include_server_if.h
@@ -21,3 +21,4 @@
 
 int dcc_talk_to_include_server(char **argv, char ***files);
 int dcc_get_original_fname(const char *fname, char **original_fname);
+int dcc_approximate_includes(struct dcc_hostdef *host, char **argv);
diff --git a/src/util.c b/src/util.c
index 93a17ca..a4c446f 100644
--- a/src/util.c
+++ b/src/util.c
@@ -87,7 +87,7 @@ void dcc_exit(int exitcode)
 
 int str_endswith(const char *tail, const char *tiger)
 {
-        size_t len_tail = strlen(tail);
+    size_t len_tail = strlen(tail);
     size_t len_tiger = strlen(tiger);
 
     if (len_tail > len_tiger)
diff --git a/test/testdistcc.py b/test/testdistcc.py
index 7a0a935..938e531 100755
--- a/test/testdistcc.py
+++ b/test/testdistcc.py
@@ -1521,6 +1521,58 @@ class DashWpMD_Case(CompileHello_Case):
         #       Pump mode is treating -MD as if it was -MMD.
         # self.assert_re_search(r"stdio.h", deps);
 
+class ScanIncludes_Case(CompileHello_Case):
+    """Test --scan-includes"""
+
+    def createSource(self):
+      CompileHello_Case.createSource(self)
+      self.runcmd("mv testhdr.h test_header.h")
+      self.runcmd("ln -s test_header.h testhdr.h")
+      self.runcmd("mkdir test_subdir")
+      self.runcmd("touch test_another_header.h")
+
+    def headerSource(self):
+        return """
+#define HELLO_WORLD "hello world"
+#include "test_subdir/../test_another_header.h"
+"""
+
+    def compileCmd(self):
+        return self.distcc_without_fallback() + "--scan-includes " + \
+               _gcc + " -o testtmp.o " + self.compileOpts() + \
+               " -c %s" % (self.sourceFilename())
+
+    def runtest(self):
+        cmd = self.compileCmd()
+        rc, out, err = self.runcmd_unchecked(cmd)
+        log = open('distcc.log').read()
+        pump_mode = _server_options.find('cpp') != -1
+        if pump_mode:
+          if err != '':
+              self.fail("distcc command %s produced stderr:\n%s" % (`cmd`, err))
+          if rc != 0:
+              self.fail("distcc command %s failed:\n%s" % (`cmd`, rc))
+          self.assert_re_search(
+              r"FILE      /.*/ScanIncludes_Case/testtmp.c", out);
+          self.assert_re_search(
+              r"FILE      /.*/ScanIncludes_Case/test_header\.h", out);
+          self.assert_re_search(
+              r"FILE      /.*/ScanIncludes_Case/test_another_header\.h", out);
+          self.assert_re_search(
+              r"SYMLINK   /.*/ScanIncludes_Case/testhdr\.h", out);
+          self.assert_re_search(
+              r"DIRECTORY /.*/ScanIncludes_Case/test_subdir", out);
+          self.assert_re_search(
+              r"SYSTEMDIR /.*", out);
+        else:
+          self.assert_re_search(r"ERROR: '--scan_includes' specified, but "
+                                "distcc wouldn't have used include server "
+                                ".make sure hosts list includes ',cpp' option",
+                                log)
+          self.assert_equal(rc, 100)
+          self.assert_equal(out, '')
+          self.assert_equal(err, '')
+
 class AbsSourceFilename_Case(CompileHello_Case):
     """Test remote compilation of files with absolute names."""