Partial work towards git regression testing.. Comit-pushing now before

git decides to delete more of my work.

3 files changed, 178 insertions, 53 deletions

diff --git a/tools/regression/src/git.py b/tools/regression/src/git.py
index 9331e51404..1bc39fbd84 100755
--- a/tools/regression/src/git.py
+++ b/tools/regression/src/git.py
@@ -1,6 +1,6 @@
 #!/usr/bin/python
 
-# Copyright Redshift Software, Inc. 2012-2013
+# Copyright Rene Rivera 2012-2013
 #
 # Distributed under the Boost Software License, Version 1.0.
 # (See accompanying file LICENSE_1_0.txt or copy at
@@ -8,6 +8,10 @@
 
 import os
 import sys
+
+root = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
+sys.path.insert(0,os.path.join(root,'dulwich'))
+
 import shutil
 import stat
 import dulwich
@@ -18,6 +22,9 @@ from dulwich.objects import ZERO_SHA, Tree
 from dulwich.file import ensure_dir_exists
 from dulwich.index import changes_from_tree, Index, index_entry_from_stat
 
+if __name__ == '__main__':
+    print "#  Using dulwich version %s.%s.%s"%(dulwich.__version__)
+
 class RemoteRepo(Repo):
     def __init__(self, root, remote_uri, branch='develop', reset=False):
         if reset:
@@ -104,6 +111,8 @@ class RemoteRepo(Repo):
                     submodule_url = l.strip().rsplit(" ",1)[-1]
                     if submodule_url.startswith('git:'):
                         submodule_url = submodule_url.replace("git:","https:")
+                    if submodule_url.startswith('.'):
+                        submodule_url = self.remote_uri+"/"+submodule_url
     
     def _checkout_entry(self,index,path,mode,sha):
         full_path = os.path.join(self.path, path)
@@ -184,7 +193,7 @@ if __name__ == '__main__':
             'git_test_boost',
             'https://github.com/boostorg/boost.git',
             branch='master',
-            reset=True)
+            reset=False)
         g.checkout()
         g.status()
     if False:
diff --git a/tools/regression/src/regression.py b/tools/regression/src/regression.py
index fdf00d0485..7dd0923e66 100644
--- a/tools/regression/src/regression.py
+++ b/tools/regression/src/regression.py
@@ -1,7 +1,7 @@
 #!/usr/bin/python
 
 # Copyright MetaCommunications, Inc. 2003-2007
-# Copyright Redshift Software, Inc. 2007
+# Copyright Rene Rivera 2007-2013
 #
 # Distributed under the Boost Software License, Version 1.0.
 # (See accompanying file LICENSE_1_0.txt or copy at
@@ -32,6 +32,32 @@ repo_path = {
                     : 'trunk/boost-build.jam'
     }
 
+git_info = {
+    'build' : {
+        'git' : 'https://github.com/boostorg/build.git',
+        'dir' : 'tools_bb',
+        'subdir' : '',
+        },
+    'regression' : {
+        'git' : 'https://github.com/boostorg/boost.git',
+        'dir' : 'boost_root',
+        'subdir' : 'tools/regression',
+        },
+    'boost-build.jam' : {
+        'raw' : 'https://raw.github.com/boostorg/boost/%s/boost-build.jam'
+        },
+    'boost' : {
+        'git' : 'https://github.com/boostorg/boost.git',
+        'dir' : 'boost_root',
+        'subdir' : '',
+        },
+    }
+
+git_branch = {
+    'trunk' : 'develop',
+    'release' : 'master',
+    }
+
 class runner:
     
     def __init__(self,root):
@@ -79,11 +105,20 @@ class runner:
         opt.add_option( '--local',
             help="the name of the boost tarball" )
         opt.add_option( '--force-update',
-            help="do an SVN update (if applicable) instead of a clean checkout, even when performing a full run",
+            help="do an update (if applicable) instead of a clean checkout, even when performing a full run",
             action='store_true' )
         opt.add_option( '--have-source',
-            help="do neither a tarball download nor an SVN update; used primarily for testing script changes",
+            help="do neither a tarball download nor a repo update; used primarily for testing script changes",
             action='store_true' )
+        opt.add_option( '--use-svn',
+            help="use old svn repo",
+            action='store_true')
+        opt.add_option( '--use-git',
+            help="use the canonical git repo",
+            action='store_true')
+        opt.add_option( '--use-dulwich',
+            help="use dulwich git implementation for git access",
+            action='store_true')
 
         #~ Connection Options:
         opt.add_option( '--ftp',
@@ -135,21 +170,30 @@ class runner:
         self.mail=None
         self.smtp_login=None
         self.skip_tests=False
+        self.use_git=True
+        self.use_dulwich=False
         ( _opt_, self.actions ) = opt.parse_args(None,self)
         if not self.actions or self.actions == []:
             self.actions = [ 'regression' ]
         
         #~ Initialize option dependent values.
         self.regression_root = root
-        self.boost_root = os.path.join( self.regression_root, 'boost' )
+        self.boost_root = os.path.join( self.regression_root, 'boost_root' )
         self.regression_results = os.path.join( self.regression_root, 'results' )
         if self.pjl_toolset != 'python':
             self.regression_log = os.path.join( self.regression_results, 'bjam.log' )
         else:
             self.regression_log = os.path.join( self.regression_results, 'bjam.xml' )
         self.tools_bb_root = os.path.join( self.regression_root,'tools_bb' )
-        self.tools_bjam_root = os.path.join( self.regression_root,'tools_bjam' )
-        self.tools_regression_root = os.path.join( self.regression_root,'tools_regression' )
+        if self.use_git:
+            self.tools_bb_root = os.path.join( self.tools_bb_root, 'src')
+            self.tools_bjam_root = os.path.join( self.regression_root,'tools_bb', 'src', 'engine' )
+        else:
+            self.tools_bjam_root = os.path.join( self.regression_root,'tools_bb', 'v2', 'engine' )
+        if self.use_git:
+            self.tools_regression_root = os.path.join( self.regression_root,'boost_root', 'tools', 'regression' )
+        else:
+            self.tools_regression_root = os.path.join( self.regression_root,'tools_regression' )
         self.xsl_reports_dir = os.path.join( self.tools_regression_root, 'xsl_reports' )
         self.timestamp_path = os.path.join( self.regression_root, 'timestamp' )
         if sys.platform == 'win32':
@@ -228,7 +272,11 @@ class runner:
     def command_get_tools(self):
         #~ Get Boost.Build v2...
         self.log( 'Getting Boost.Build v2...' )
-        if self.user and self.user != '':
+        if self.use_git:
+            self.git_checkout(
+                git_info['build'],
+                git_branch[self.tag])
+        elif self.user and self.user != '':
             os.chdir( os.path.dirname(self.tools_bb_root) )
             self.svn_command( 'co %s %s' % (
                 self.svn_repository_url(repo_path['build']),
@@ -240,23 +288,13 @@ class runner:
             self.unpack_tarball(
                 self.tools_bb_root+".tar.bz2",
                 os.path.basename(self.tools_bb_root) )
-        #~ Get Boost.Jam...
-        self.log( 'Getting Boost.Jam...' )
-        if self.user and self.user != '':
-            os.chdir( os.path.dirname(self.tools_bjam_root) )
-            self.svn_command( 'co %s %s' % (
-                self.svn_repository_url(repo_path['jam']),
-                os.path.basename(self.tools_bjam_root) ) )
-        else:
-            self.retry( lambda: self.download_tarball(
-                os.path.basename(self.tools_bjam_root)+".tar.bz2",
-                self.tarball_url(repo_path['jam']) ) )
-            self.unpack_tarball(
-                self.tools_bjam_root+".tar.bz2",
-                os.path.basename(self.tools_bjam_root) )
         #~ Get the regression tools and utilities...
-        self.log( 'Getting regression tools an utilities...' )
-        if self.user and self.user != '':
+        self.log( 'Getting regression tools and utilities...' )
+        if self.use_git:
+            self.git_checkout(
+                git_info['regression'],
+                git_branch[self.tag])
+        elif self.user and self.user != '':
             os.chdir( os.path.dirname(self.tools_regression_root) )
             self.svn_command( 'co %s %s' % (
                 self.svn_repository_url(repo_path['regression']),
@@ -270,24 +308,33 @@ class runner:
                 os.path.basename(self.tools_regression_root) )
         
         #~ We get a boost-build.jam to make the tool build work even if there's
-        #~ and existing boost-build.jam above the testing root.
+        #~ an existing boost-build.jam above the testing root.
         self.log( 'Getting boost-build.jam...' )
-        self.http_get(
-            self.svn_repository_url(repo_path['boost-build.jam']),
-            os.path.join( self.regression_root, 'boost-build.jam' ) )
+        if self.use_git:
+            self.http_get(
+                git_info['boost-build.jam']['raw']%(git_branch[self.tag]),
+                os.path.join(self.regression_root, 'boost-build.jam') )
+        else:
+            self.http_get(
+                self.svn_repository_url(repo_path['boost-build.jam']),
+                os.path.join( self.regression_root, 'boost-build.jam' ) )
     
     def command_get_source(self):
         self.refresh_timestamp()
         self.log( 'Getting sources (%s)...' % self.timestamp() )
 
-        if self.user and self.user != '':
+        if self.use_git:
+            self.retry(self.git_source_checkout)
+        elif self.user and self.user != '':
             self.retry( self.svn_checkout )
         else:
             self.retry( self.get_tarball )
         pass
     
     def command_update_source(self):
-        if self.user and self.user != '' \
+        if self.use_git:
+            self.retry(self.git_source_checkout)
+        elif self.user and self.user != '' \
             or os.path.exists( os.path.join( self.boost_root, '.svn' ) ):
             open( self.timestamp_path, 'w' ).close()
             self.log( 'Updating sources from SVN (%s)...' % self.timestamp() )
@@ -405,23 +452,34 @@ class runner:
 
         source = 'tarball'
         revision = ''
-        svn_root_file = os.path.join( self.boost_root, '.svn' )
-        svn_info_file = os.path.join( self.boost_root, 'svn_info.txt' )
-        if os.path.exists( svn_root_file ):
-            source = 'SVN'
-            self.svn_command( 'info --xml "%s" >"%s"' % (self.boost_root,svn_info_file) )
-
-        if os.path.exists( svn_info_file ):
-            f = open( svn_info_file, 'r' )
-            svn_info = f.read()
+        
+        if self.use_git:
+            git_head_file = os.path.join(self.boost_root, '.git', 'HEAD')
+            f = open( git_head_file, 'r' )
+            git_head_info = f.read()
+            f.close()
+            git_ref_file = os.path.join(self.boost_root, '.git', git_head_info.strip())
+            f = open( git_ref_file, 'r' )
+            revision = f.read()
             f.close()
-            i = svn_info.find( 'Revision:' )
-            if i < 0: i = svn_info.find( 'revision=' )  # --xml format
-            if i >= 0:
-                i += 10
-                while svn_info[i] >= '0' and svn_info[i] <= '9':
-                  revision += svn_info[i]
-                  i += 1
+            revision = revision.strip()
+        else:
+            svn_root_file = os.path.join( self.boost_root, '.svn' )
+            svn_info_file = os.path.join( self.boost_root, 'svn_info.txt' )
+            if os.path.exists( svn_root_file ):
+                source = 'SVN'
+                self.svn_command( 'info --xml "%s" >"%s"' % (self.boost_root,svn_info_file) )
+            if os.path.exists( svn_info_file ):
+                f = open( svn_info_file, 'r' )
+                svn_info = f.read()
+                f.close()
+                i = svn_info.find( 'Revision:' )
+                if i < 0: i = svn_info.find( 'revision=' )  # --xml format
+                if i >= 0:
+                    i += 10
+                    while svn_info[i] >= '0' and svn_info[i] <= '9':
+                      revision += svn_info[i]
+                      i += 1
 
         if self.pjl_toolset != 'python':
             from collect_and_upload_logs import collect_logs
@@ -549,8 +607,8 @@ class runner:
             raise
 
     def command_show_revision(self):
-        modified = '$Date$'
-        revision = '$Revision$'
+        modified = '$Date: 2013-02-28 12:09:13 -0600 (Thu, 28 Feb 2013) $'
+        revision = '$Revision: 83205 $'
 
         import re
         re_keyword_value = re.compile( r'^\$\w+:\s+(.*)\s+\$$' )
@@ -803,6 +861,48 @@ class runner:
         else:
             return '%s%s' % (repo_root['anon'],path)
     
+    #~ Downloading source, from GIT...
+    
+    def git_command(self, command, *args):
+        git_cli = "'git' '%(command)s'" % { 'command': command }
+        for a in args:
+            git_cli += " '"+a+"'"
+        rc = os.system( git_cli )
+        if rc != 0:
+            raise Exception( 'GIT command "%s" failed with code %d' % (git_cli, rc) )
+        return rc
+    
+    def git_checkout(self, info, branch):
+        if self.use_dulwich:
+            import git;
+            git_root = os.path.join(self.regression_root, info['dir'])
+            g = git.RemoteRepo(git_root, info['git'], branch=branch, reset=False)
+            g.checkout()
+        else:
+            git_root = os.path.join(self.regression_root, info['dir'])
+            if os.path.exists( git_root ):
+                os.chdir( git_root )
+                self.git_command( 'pull' )
+                self.git_command( 'submodule', 'update')
+            else:
+                git_root_parent = os.path.dirname( git_root )
+                git_root_subdir = os.path.basename( git_root )
+                os.chdir( git_root_parent )
+                self.git_command(
+                    'clone',
+                    '--depth', '1',
+                    '--branch', branch,
+                    '--single-branch',
+                    info[ 'git' ],
+                    git_root_subdir )
+                os.chdir( git_root )
+                self.git_command(
+                    'submodule', 'update', '--init', '--merge' )
+    
+    def git_source_checkout(self):
+        os.chdir( self.regression_root )
+        self.git_checkout(git_info['boost'], git_branch[self.tag])
+    
     #~ Downloading and extracting source archives, from tarballs or zipballs...
     
     def get_tarball( self, *args ):
diff --git a/tools/regression/src/run.py b/tools/regression/src/run.py
index 5e8e0c7d01..fd3f988227 100644
--- a/tools/regression/src/run.py
+++ b/tools/regression/src/run.py
@@ -1,6 +1,6 @@
 #!/usr/bin/python
 
-# Copyright Redshift Software, Inc. 2007
+# Copyright Rene Rivera 2007-2013
 #
 # Distributed under the Boost Software License, Version 1.0.
 # (See accompanying file LICENSE_1_0.txt or copy at
@@ -19,13 +19,29 @@ no_update = no_update_argument in sys.argv
 if no_update:
     del sys.argv[sys.argv.index(no_update_argument)]
 
+use_local_argument = '--use-local'
+use_local = use_local_argument in sys.argv
+if use_local:
+    del sys.argv[sys.argv.index(use_local_argument)]
+
+use_git_argument = '--use-git'
+use_git = use_git_argument in sys.argv
+
 #~ The directory this file is in.
-root = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
+if use_local:
+    root = os.path.abspath(os.path.realpath(os.path.curdir))
+else:
+    root = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
 print '# Running regressions in %s...' % root
 
 script_sources = [ 'collect_and_upload_logs.py', 'process_jam_log.py', 'regression.py' ]
 script_local = os.path.join(root,'tools','regression','src')
-script_remote = 'http://svn.boost.org/svn/boost/trunk/tools/regression/src'
+if use_local:
+    script_remote = 'file://'+os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
+elif use_git:
+    script_remote = 'https://raw.github.com/boostorg/boost/master/tools/regression/src'
+else:
+    script_remote = 'http://svn.boost.org/svn/boost/trunk/tools/regression/src'
 script_dir = os.path.join(root,'tools_regression_src')
 
 if not no_update:
@@ -52,7 +68,7 @@ if not no_update:
             urllib.FancyURLopener(proxy).retrieve(
                 '%s/%s' % (script_remote,src), os.path.join(script_dir,src) )
 
-#~ * Make the scripts available to Python
+#~ * Make the scripts are available to Python
 sys.path.insert(0,os.path.join(root,'tools_regression_src'))
 
 #~ Launch runner.