summaryrefslogtreecommitdiff
path: root/tools/regression/src/collect_and_upload_logs.py
diff options
context:
space:
mode:
Diffstat (limited to 'tools/regression/src/collect_and_upload_logs.py')
-rw-r--r--tools/regression/src/collect_and_upload_logs.py546
1 files changed, 546 insertions, 0 deletions
diff --git a/tools/regression/src/collect_and_upload_logs.py b/tools/regression/src/collect_and_upload_logs.py
new file mode 100644
index 0000000000..7f1345bd6f
--- /dev/null
+++ b/tools/regression/src/collect_and_upload_logs.py
@@ -0,0 +1,546 @@
+
+# Copyright (c) MetaCommunications, Inc. 2003-2007
+#
+# Distributed under the Boost Software License, Version 1.0.
+# (See accompanying file LICENSE_1_0.txt or copy at
+# http://www.boost.org/LICENSE_1_0.txt)
+
+import xml.sax.saxutils
+import zipfile
+import ftplib
+import time
+import stat
+import xml.dom.minidom
+import xmlrpclib
+import httplib
+
+import os.path
+import string
+import sys
+import re
+import urlparse
+
+
+def process_xml_file( input_file, output_file ):
+ utils.log( 'Processing test log "%s"' % input_file )
+
+ f = open( input_file, 'r' )
+ xml = f.readlines()
+ f.close()
+
+ for i in range( 0, len(xml)):
+ xml[i] = string.translate( xml[i], utils.char_translation_table )
+
+ output_file.writelines( xml )
+
+
+def process_test_log_files( output_file, dir, names ):
+ for file in names:
+ if os.path.basename( file ) == 'test_log.xml':
+ process_xml_file( os.path.join( dir, file ), output_file )
+
+
+def collect_test_logs( input_dirs, test_results_writer ):
+ __log__ = 1
+ utils.log( 'Collecting test logs ...' )
+ for input_dir in input_dirs:
+ utils.log( 'Walking directory "%s" ...' % input_dir )
+ os.path.walk( input_dir, process_test_log_files, test_results_writer )
+
+dart_status_from_result = {
+ 'succeed': 'passed',
+ 'fail': 'failed',
+ 'note': 'passed',
+ '': 'notrun'
+ }
+
+dart_project = {
+ 'trunk': 'Boost_HEAD',
+ '': 'Boost_HEAD'
+ }
+
+dart_track = {
+ 'full': 'Nightly',
+ 'incremental': 'Continuous',
+ '': 'Experimental'
+ }
+
+ascii_only_table = ""
+for i in range(0,256):
+ if chr(i) == '\n' or chr(i) == '\r':
+ ascii_only_table += chr(i)
+ elif i < 32 or i >= 0x80:
+ ascii_only_table += '?'
+ else:
+ ascii_only_table += chr(i)
+
+class xmlrpcProxyTransport(xmlrpclib.Transport):
+ def __init__(self, proxy):
+ self.proxy = proxy
+ def make_connection(self, host):
+ self.realhost = host
+ return httplib.HTTP(self.proxy)
+ def send_request(self, connection, handler, request_body):
+ connection.putrequest('POST','http://%s%s' % (self.realhost,handler))
+ def send_host(self, connection, host):
+ connection.putheader('Host',self.realhost)
+
+
+def publish_test_logs(
+ input_dirs,
+ runner_id, tag, platform, comment_file, timestamp, user, source, run_type,
+ dart_server = None,
+ http_proxy = None,
+ **unused
+ ):
+ __log__ = 1
+ utils.log( 'Publishing test logs ...' )
+ dart_rpc = None
+ dart_dom = {}
+
+ def _publish_test_log_files_ ( unused, dir, names ):
+ for file in names:
+ if os.path.basename( file ) == 'test_log.xml':
+ utils.log( 'Publishing test log "%s"' % os.path.join(dir,file) )
+ if dart_server:
+ log_xml = open(os.path.join(dir,file)).read().translate(ascii_only_table)
+ #~ utils.log( '--- XML:\n%s' % log_xml)
+ #~ It seems possible to get an empty XML result file :-(
+ if log_xml == "": continue
+ log_dom = xml.dom.minidom.parseString(log_xml)
+ test = {
+ 'library': log_dom.documentElement.getAttribute('library'),
+ 'test-name': log_dom.documentElement.getAttribute('test-name'),
+ 'toolset': log_dom.documentElement.getAttribute('toolset')
+ }
+ if not test['test-name'] or test['test-name'] == '':
+ test['test-name'] = 'unknown'
+ if not test['toolset'] or test['toolset'] == '':
+ test['toolset'] = 'unknown'
+ if not dart_dom.has_key(test['toolset']):
+ dart_dom[test['toolset']] = xml.dom.minidom.parseString(
+'''<?xml version="1.0" encoding="UTF-8"?>
+<DartSubmission version="2.0" createdby="collect_and_upload_logs.py">
+ <Site>%(site)s</Site>
+ <BuildName>%(buildname)s</BuildName>
+ <Track>%(track)s</Track>
+ <DateTimeStamp>%(datetimestamp)s</DateTimeStamp>
+</DartSubmission>
+''' % {
+ 'site': runner_id,
+ 'buildname': "%s -- %s (%s)" % (platform,test['toolset'],run_type),
+ 'track': dart_track[run_type],
+ 'datetimestamp' : timestamp
+ } )
+ submission_dom = dart_dom[test['toolset']]
+ for node in log_dom.documentElement.childNodes:
+ if node.nodeType == xml.dom.Node.ELEMENT_NODE:
+ if node.firstChild:
+ log_data = xml.sax.saxutils.escape(node.firstChild.data)
+ else:
+ log_data = ''
+ test_dom = xml.dom.minidom.parseString('''<?xml version="1.0" encoding="UTF-8"?>
+<Test>
+ <Name>.Test.Boost.%(tag)s.%(library)s.%(test-name)s.%(type)s</Name>
+ <Status>%(result)s</Status>
+ <Measurement name="Toolset" type="text/string">%(toolset)s</Measurement>
+ <Measurement name="Timestamp" type="text/string">%(timestamp)s</Measurement>
+ <Measurement name="Log" type="text/text">%(log)s</Measurement>
+</Test>
+ ''' % {
+ 'tag': tag,
+ 'library': test['library'],
+ 'test-name': test['test-name'],
+ 'toolset': test['toolset'],
+ 'type': node.nodeName,
+ 'result': dart_status_from_result[node.getAttribute('result')],
+ 'timestamp': node.getAttribute('timestamp'),
+ 'log': log_data
+ })
+ submission_dom.documentElement.appendChild(
+ test_dom.documentElement.cloneNode(1) )
+
+ for input_dir in input_dirs:
+ utils.log( 'Walking directory "%s" ...' % input_dir )
+ os.path.walk( input_dir, _publish_test_log_files_, None )
+ if dart_server:
+ try:
+ rpc_transport = None
+ if http_proxy:
+ rpc_transport = xmlrpcProxyTransport(http_proxy)
+ dart_rpc = xmlrpclib.ServerProxy(
+ 'http://%s/%s/Command/' % (dart_server,dart_project[tag]),
+ rpc_transport )
+ for dom in dart_dom.values():
+ #~ utils.log('Dart XML: %s' % dom.toxml('utf-8'))
+ dart_rpc.Submit.put(xmlrpclib.Binary(dom.toxml('utf-8')))
+ except Exception, e:
+ utils.log('Dart server error: %s' % e)
+
+
+def upload_to_ftp( tag, results_file, ftp_proxy, debug_level, ftp_url ):
+
+ if not ftp_url:
+ ftp_host = 'boost.cowic.de'
+ ftp_url = ''.join(['ftp','://anonymous','@',ftp_host,'/boost/do-not-publish-this-url/results/'])
+ utils.log( 'Uploading log archive "%s" to %s/%s' % ( results_file, ftp_url, tag ) )
+
+ ftp_parts = urlparse.urlparse(ftp_url)
+ ftp_netloc = re.split('[@]',ftp_parts[1])
+ ftp_user = re.split('[:]',ftp_netloc[0])[0]
+ ftp_password = re.split('[:]',ftp_netloc[0]+':anonymous')[1]
+ ftp_site = re.split('[:]',ftp_netloc[1])[0]
+ ftp_path = ftp_parts[2]
+
+ if not ftp_proxy:
+ ftp = ftplib.FTP( ftp_site )
+ ftp.set_debuglevel( debug_level )
+ ftp.login( ftp_user, ftp_password )
+ else:
+ utils.log( ' Connecting through FTP proxy server "%s"' % ftp_proxy )
+ ftp = ftplib.FTP( ftp_proxy )
+ ftp.set_debuglevel( debug_level )
+ ftp.set_pasv (0) # turn off PASV mode
+ ftp.login( '%s@%s' % (ftp_user,ftp_site), ftp_password )
+
+ ftp.cwd( ftp_path )
+ try:
+ ftp.cwd( tag )
+ except ftplib.error_perm:
+ for dir in tag.split( '/' ):
+ ftp.mkd( dir )
+ ftp.cwd( dir )
+
+ f = open( results_file, 'rb' )
+ ftp.storbinary( 'STOR %s' % os.path.basename( results_file ), f )
+ ftp.quit()
+
+
+def copy_comments( results_xml, comment_file ):
+ results_xml.startElement( 'comment', {} )
+
+ if os.path.exists( comment_file ):
+ utils.log( 'Reading comments file "%s"...' % comment_file )
+ f = open( comment_file, 'r' )
+ try:
+ results_xml.characters( f.read() )
+ finally:
+ f.close()
+ else:
+ utils.log( 'Warning: comment file "%s" is not found.' % comment_file )
+
+ lines = ['']
+ for arg in sys.argv:
+ # Make sure that the ftp details are hidden
+ arg = re.sub( 'ftp://.*$', 'ftp://XXXXX', arg )
+
+ # Escape quotes
+ arg = re.sub( r'(\\|")', r'\\\1', arg )
+
+ # Quote arguments if needed
+ if arg.find( ' ' ) != -1:
+ arg = '"%s"' % arg
+ if len( lines[-1] ) + len( arg ) + 2 >= 80:
+ # align backslashes
+ lines[-1] += ' ' * ( 79 - len( lines[-1] ) )
+ # indent lines after the first
+ lines.append( ' ' )
+ lines[-1] += ( arg + ' ' )
+
+ results_xml.characters( '<hr>' )
+ results_xml.characters( '<dl>' )
+ results_xml.characters( '<dt>Command Line</dt>' )
+ results_xml.characters( '<dd>' )
+ results_xml.characters( '<pre>' )
+ results_xml.characters( '\\\n'.join(lines) )
+ results_xml.characters( '</pre>' )
+ results_xml.characters( '</dd>' )
+ results_xml.characters( '</dl>\n' )
+
+ results_xml.endElement( 'comment' )
+
+
+def compress_file( file_path, archive_path ):
+ utils.log( 'Compressing "%s"...' % file_path )
+
+ try:
+ z = zipfile.ZipFile( archive_path, 'w', zipfile.ZIP_DEFLATED )
+ z.write( file_path, os.path.basename( file_path ) )
+ z.close()
+ utils.log( 'Done writing "%s".'% archive_path )
+ except Exception, msg:
+ utils.log( 'Warning: Compressing falied (%s)' % msg )
+ utils.log( ' Trying to compress using a platform-specific tool...' )
+ try: import zip_cmd
+ except ImportError:
+ script_dir = os.path.dirname( os.path.abspath( sys.argv[0] ) )
+ utils.log( 'Could not find \'zip_cmd\' module in the script directory (%s).' % script_dir )
+ raise Exception( 'Compressing failed!' )
+ else:
+ if os.path.exists( archive_path ):
+ os.unlink( archive_path )
+ utils.log( 'Removing stale "%s".' % archive_path )
+
+ zip_cmd.main( file_path, archive_path )
+ utils.log( 'Done compressing "%s".' % archive_path )
+
+
+def read_timestamp( file ):
+ if not os.path.exists( file ):
+ result = time.gmtime()
+ utils.log( 'Warning: timestamp file "%s" does not exist'% file )
+ utils.log( 'Using current UTC time (%s)' % result )
+ return result
+
+ return time.gmtime( os.stat( file ).st_mtime )
+
+
+def collect_logs(
+ results_dir
+ , runner_id
+ , tag
+ , platform
+ , comment_file
+ , timestamp_file
+ , user
+ , source
+ , run_type
+ , dart_server = None
+ , http_proxy = None
+ , revision = ''
+ , **unused
+ ):
+
+ timestamp = time.strftime( '%Y-%m-%dT%H:%M:%SZ', read_timestamp( timestamp_file ) )
+
+ if dart_server:
+ publish_test_logs( [ results_dir ],
+ runner_id, tag, platform, comment_file, timestamp, user, source, run_type,
+ dart_server = dart_server,
+ http_proxy = http_proxy )
+
+ results_file = os.path.join( results_dir, '%s.xml' % runner_id )
+ results_writer = open( results_file, 'w' )
+ utils.log( 'Collecting test logs into "%s"...' % results_file )
+
+ results_xml = xml.sax.saxutils.XMLGenerator( results_writer )
+ results_xml.startDocument()
+ results_xml.startElement(
+ 'test-run'
+ , {
+ 'tag': tag
+ , 'platform': platform
+ , 'runner': runner_id
+ , 'timestamp': timestamp
+ , 'source': source
+ , 'run-type': run_type
+ , 'revision': revision
+ }
+ )
+
+ copy_comments( results_xml, comment_file )
+ collect_test_logs( [ results_dir ], results_writer )
+
+ results_xml.endElement( "test-run" )
+ results_xml.endDocument()
+ results_writer.close()
+ utils.log( 'Done writing "%s".' % results_file )
+
+ compress_file(
+ results_file
+ , os.path.join( results_dir,'%s.zip' % runner_id )
+ )
+
+
+def upload_logs(
+ results_dir
+ , runner_id
+ , tag
+ , user
+ , ftp_proxy
+ , debug_level
+ , send_bjam_log = False
+ , timestamp_file = None
+ , dart_server = None
+ , ftp_url = None
+ , **unused
+ ):
+
+ logs_archive = os.path.join( results_dir, '%s.zip' % runner_id )
+ upload_to_ftp( tag, logs_archive, ftp_proxy, debug_level, ftp_url )
+ if send_bjam_log:
+ bjam_log_path = os.path.join( results_dir, 'bjam.log' )
+ if not timestamp_file:
+ timestamp_file = bjam_log_path
+
+ timestamp = time.strftime( '%Y-%m-%d-%H-%M-%S', read_timestamp( timestamp_file ) )
+ logs_archive = os.path.join( results_dir, '%s.%s.log.zip' % ( runner_id, timestamp ) )
+ compress_file( bjam_log_path, logs_archive )
+ upload_to_ftp( '%s/logs' % tag, logs_archive, ftp_proxy, debug_level, ftp_url )
+
+
+def collect_and_upload_logs(
+ results_dir
+ , runner_id
+ , tag
+ , platform
+ , comment_file
+ , timestamp_file
+ , user
+ , source
+ , run_type
+ , revision = None
+ , ftp_proxy = None
+ , debug_level = 0
+ , send_bjam_log = False
+ , dart_server = None
+ , http_proxy = None
+ , ftp_url = None
+ , **unused
+ ):
+
+ collect_logs(
+ results_dir
+ , runner_id
+ , tag
+ , platform
+ , comment_file
+ , timestamp_file
+ , user
+ , source
+ , run_type
+ , revision = revision
+ , dart_server = dart_server
+ , http_proxy = http_proxy
+ )
+
+ upload_logs(
+ results_dir
+ , runner_id
+ , tag
+ , user
+ , ftp_proxy
+ , debug_level
+ , send_bjam_log
+ , timestamp_file
+ , dart_server = dart_server
+ , ftp_url = ftp_url
+ )
+
+
+def accept_args( args ):
+ args_spec = [
+ 'locate-root='
+ , 'runner='
+ , 'tag='
+ , 'platform='
+ , 'comment='
+ , 'timestamp='
+ , 'source='
+ , 'run-type='
+ , 'user='
+ , 'ftp-proxy='
+ , 'proxy='
+ , 'debug-level='
+ , 'send-bjam-log'
+ , 'help'
+ , 'dart-server='
+ , 'revision='
+ , 'ftp='
+ ]
+
+ options = {
+ '--tag' : 'trunk'
+ , '--platform' : sys.platform
+ , '--comment' : 'comment.html'
+ , '--timestamp' : 'timestamp'
+ , '--user' : None
+ , '--source' : 'SVN'
+ , '--run-type' : 'full'
+ , '--ftp-proxy' : None
+ , '--proxy' : None
+ , '--debug-level' : 0
+ , '--dart-server' : 'beta.boost.org:8081'
+ , '--revision' : None
+ , '--ftp' : None
+
+ }
+
+ utils.accept_args( args_spec, args, options, usage )
+
+ return {
+ 'results_dir' : options[ '--locate-root' ]
+ , 'runner_id' : options[ '--runner' ]
+ , 'tag' : options[ '--tag' ]
+ , 'platform' : options[ '--platform']
+ , 'comment_file' : options[ '--comment' ]
+ , 'timestamp_file' : options[ '--timestamp' ]
+ , 'user' : options[ '--user' ]
+ , 'source' : options[ '--source' ]
+ , 'run_type' : options[ '--run-type' ]
+ , 'ftp_proxy' : options[ '--ftp-proxy' ]
+ , 'http_proxy' : options[ '--proxy' ]
+ , 'debug_level' : int(options[ '--debug-level' ])
+ , 'send_bjam_log' : options.has_key( '--send-bjam-log' )
+ , 'dart_server' : options[ '--dart-server' ]
+ , 'revision' : options[ '--revision' ]
+ , 'ftp' : options[ '--ftp' ]
+ }
+
+
+commands = {
+ 'collect-and-upload' : collect_and_upload_logs
+ , 'collect-logs' : collect_logs
+ , 'upload-logs' : upload_logs
+ }
+
+def usage():
+ print 'Usage: %s [command] [options]' % os.path.basename( sys.argv[0] )
+ print '''
+Commands:
+\t%s
+
+Options:
+\t--locate-root directory to to scan for "test_log.xml" files
+\t--runner runner ID (e.g. "Metacomm")
+\t--timestamp path to a file which modification time will be used
+\t as a timestamp of the run ("timestamp" by default)
+\t--comment an HTML comment file to be inserted in the reports
+\t ("comment.html" by default)
+\t--tag the tag for the results ("trunk" by default)
+\t--user SourceForge user name for a shell account (optional)
+\t--source where Boost sources came from ("SVN" or "tarball";
+\t "SVN" by default)
+\t--run-type "incremental" or "full" ("full" by default)
+\t--send-bjam-log in addition to regular XML results, send in full bjam
+\t log of the regression run
+\t--proxy HTTP proxy server address and port (e.g.
+\t 'http://www.someproxy.com:3128', optional)
+\t--ftp-proxy FTP proxy server (e.g. 'ftpproxy', optional)
+\t--debug-level debugging level; controls the amount of debugging
+\t output printed; 0 by default (no debug output)
+\t--dart-server The dart server to send results to.
+\t--ftp The ftp URL to upload results to.
+''' % '\n\t'.join( commands.keys() )
+
+
+def main():
+ if len(sys.argv) > 1 and sys.argv[1] in commands:
+ command = sys.argv[1]
+ args = sys.argv[ 2: ]
+ else:
+ command = 'collect-and-upload'
+ args = sys.argv[ 1: ]
+
+ commands[ command ]( **accept_args( args ) )
+
+
+if __name__ != '__main__': import utils
+else:
+ # in absense of relative import...
+ xsl_path = os.path.abspath( os.path.dirname( sys.argv[ 0 ] ) )
+ while os.path.basename( xsl_path ) != 'xsl_reports': xsl_path = os.path.dirname( xsl_path )
+ sys.path.append( xsl_path )
+
+ import utils
+ main()
View Lorry Controller Status