diff options
Diffstat (limited to 'tools/regression/src/collect_and_upload_logs.py')
-rw-r--r-- | tools/regression/src/collect_and_upload_logs.py | 546 |
1 files changed, 546 insertions, 0 deletions
diff --git a/tools/regression/src/collect_and_upload_logs.py b/tools/regression/src/collect_and_upload_logs.py new file mode 100644 index 0000000000..7f1345bd6f --- /dev/null +++ b/tools/regression/src/collect_and_upload_logs.py @@ -0,0 +1,546 @@ + +# Copyright (c) MetaCommunications, Inc. 2003-2007 +# +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE_1_0.txt or copy at +# http://www.boost.org/LICENSE_1_0.txt) + +import xml.sax.saxutils +import zipfile +import ftplib +import time +import stat +import xml.dom.minidom +import xmlrpclib +import httplib + +import os.path +import string +import sys +import re +import urlparse + + +def process_xml_file( input_file, output_file ): + utils.log( 'Processing test log "%s"' % input_file ) + + f = open( input_file, 'r' ) + xml = f.readlines() + f.close() + + for i in range( 0, len(xml)): + xml[i] = string.translate( xml[i], utils.char_translation_table ) + + output_file.writelines( xml ) + + +def process_test_log_files( output_file, dir, names ): + for file in names: + if os.path.basename( file ) == 'test_log.xml': + process_xml_file( os.path.join( dir, file ), output_file ) + + +def collect_test_logs( input_dirs, test_results_writer ): + __log__ = 1 + utils.log( 'Collecting test logs ...' ) + for input_dir in input_dirs: + utils.log( 'Walking directory "%s" ...' % input_dir ) + os.path.walk( input_dir, process_test_log_files, test_results_writer ) + +dart_status_from_result = { + 'succeed': 'passed', + 'fail': 'failed', + 'note': 'passed', + '': 'notrun' + } + +dart_project = { + 'trunk': 'Boost_HEAD', + '': 'Boost_HEAD' + } + +dart_track = { + 'full': 'Nightly', + 'incremental': 'Continuous', + '': 'Experimental' + } + +ascii_only_table = "" +for i in range(0,256): + if chr(i) == '\n' or chr(i) == '\r': + ascii_only_table += chr(i) + elif i < 32 or i >= 0x80: + ascii_only_table += '?' + else: + ascii_only_table += chr(i) + +class xmlrpcProxyTransport(xmlrpclib.Transport): + def __init__(self, proxy): + self.proxy = proxy + def make_connection(self, host): + self.realhost = host + return httplib.HTTP(self.proxy) + def send_request(self, connection, handler, request_body): + connection.putrequest('POST','http://%s%s' % (self.realhost,handler)) + def send_host(self, connection, host): + connection.putheader('Host',self.realhost) + + +def publish_test_logs( + input_dirs, + runner_id, tag, platform, comment_file, timestamp, user, source, run_type, + dart_server = None, + http_proxy = None, + **unused + ): + __log__ = 1 + utils.log( 'Publishing test logs ...' ) + dart_rpc = None + dart_dom = {} + + def _publish_test_log_files_ ( unused, dir, names ): + for file in names: + if os.path.basename( file ) == 'test_log.xml': + utils.log( 'Publishing test log "%s"' % os.path.join(dir,file) ) + if dart_server: + log_xml = open(os.path.join(dir,file)).read().translate(ascii_only_table) + #~ utils.log( '--- XML:\n%s' % log_xml) + #~ It seems possible to get an empty XML result file :-( + if log_xml == "": continue + log_dom = xml.dom.minidom.parseString(log_xml) + test = { + 'library': log_dom.documentElement.getAttribute('library'), + 'test-name': log_dom.documentElement.getAttribute('test-name'), + 'toolset': log_dom.documentElement.getAttribute('toolset') + } + if not test['test-name'] or test['test-name'] == '': + test['test-name'] = 'unknown' + if not test['toolset'] or test['toolset'] == '': + test['toolset'] = 'unknown' + if not dart_dom.has_key(test['toolset']): + dart_dom[test['toolset']] = xml.dom.minidom.parseString( +'''<?xml version="1.0" encoding="UTF-8"?> +<DartSubmission version="2.0" createdby="collect_and_upload_logs.py"> + <Site>%(site)s</Site> + <BuildName>%(buildname)s</BuildName> + <Track>%(track)s</Track> + <DateTimeStamp>%(datetimestamp)s</DateTimeStamp> +</DartSubmission> +''' % { + 'site': runner_id, + 'buildname': "%s -- %s (%s)" % (platform,test['toolset'],run_type), + 'track': dart_track[run_type], + 'datetimestamp' : timestamp + } ) + submission_dom = dart_dom[test['toolset']] + for node in log_dom.documentElement.childNodes: + if node.nodeType == xml.dom.Node.ELEMENT_NODE: + if node.firstChild: + log_data = xml.sax.saxutils.escape(node.firstChild.data) + else: + log_data = '' + test_dom = xml.dom.minidom.parseString('''<?xml version="1.0" encoding="UTF-8"?> +<Test> + <Name>.Test.Boost.%(tag)s.%(library)s.%(test-name)s.%(type)s</Name> + <Status>%(result)s</Status> + <Measurement name="Toolset" type="text/string">%(toolset)s</Measurement> + <Measurement name="Timestamp" type="text/string">%(timestamp)s</Measurement> + <Measurement name="Log" type="text/text">%(log)s</Measurement> +</Test> + ''' % { + 'tag': tag, + 'library': test['library'], + 'test-name': test['test-name'], + 'toolset': test['toolset'], + 'type': node.nodeName, + 'result': dart_status_from_result[node.getAttribute('result')], + 'timestamp': node.getAttribute('timestamp'), + 'log': log_data + }) + submission_dom.documentElement.appendChild( + test_dom.documentElement.cloneNode(1) ) + + for input_dir in input_dirs: + utils.log( 'Walking directory "%s" ...' % input_dir ) + os.path.walk( input_dir, _publish_test_log_files_, None ) + if dart_server: + try: + rpc_transport = None + if http_proxy: + rpc_transport = xmlrpcProxyTransport(http_proxy) + dart_rpc = xmlrpclib.ServerProxy( + 'http://%s/%s/Command/' % (dart_server,dart_project[tag]), + rpc_transport ) + for dom in dart_dom.values(): + #~ utils.log('Dart XML: %s' % dom.toxml('utf-8')) + dart_rpc.Submit.put(xmlrpclib.Binary(dom.toxml('utf-8'))) + except Exception, e: + utils.log('Dart server error: %s' % e) + + +def upload_to_ftp( tag, results_file, ftp_proxy, debug_level, ftp_url ): + + if not ftp_url: + ftp_host = 'boost.cowic.de' + ftp_url = ''.join(['ftp','://anonymous','@',ftp_host,'/boost/do-not-publish-this-url/results/']) + utils.log( 'Uploading log archive "%s" to %s/%s' % ( results_file, ftp_url, tag ) ) + + ftp_parts = urlparse.urlparse(ftp_url) + ftp_netloc = re.split('[@]',ftp_parts[1]) + ftp_user = re.split('[:]',ftp_netloc[0])[0] + ftp_password = re.split('[:]',ftp_netloc[0]+':anonymous')[1] + ftp_site = re.split('[:]',ftp_netloc[1])[0] + ftp_path = ftp_parts[2] + + if not ftp_proxy: + ftp = ftplib.FTP( ftp_site ) + ftp.set_debuglevel( debug_level ) + ftp.login( ftp_user, ftp_password ) + else: + utils.log( ' Connecting through FTP proxy server "%s"' % ftp_proxy ) + ftp = ftplib.FTP( ftp_proxy ) + ftp.set_debuglevel( debug_level ) + ftp.set_pasv (0) # turn off PASV mode + ftp.login( '%s@%s' % (ftp_user,ftp_site), ftp_password ) + + ftp.cwd( ftp_path ) + try: + ftp.cwd( tag ) + except ftplib.error_perm: + for dir in tag.split( '/' ): + ftp.mkd( dir ) + ftp.cwd( dir ) + + f = open( results_file, 'rb' ) + ftp.storbinary( 'STOR %s' % os.path.basename( results_file ), f ) + ftp.quit() + + +def copy_comments( results_xml, comment_file ): + results_xml.startElement( 'comment', {} ) + + if os.path.exists( comment_file ): + utils.log( 'Reading comments file "%s"...' % comment_file ) + f = open( comment_file, 'r' ) + try: + results_xml.characters( f.read() ) + finally: + f.close() + else: + utils.log( 'Warning: comment file "%s" is not found.' % comment_file ) + + lines = [''] + for arg in sys.argv: + # Make sure that the ftp details are hidden + arg = re.sub( 'ftp://.*$', 'ftp://XXXXX', arg ) + + # Escape quotes + arg = re.sub( r'(\\|")', r'\\\1', arg ) + + # Quote arguments if needed + if arg.find( ' ' ) != -1: + arg = '"%s"' % arg + if len( lines[-1] ) + len( arg ) + 2 >= 80: + # align backslashes + lines[-1] += ' ' * ( 79 - len( lines[-1] ) ) + # indent lines after the first + lines.append( ' ' ) + lines[-1] += ( arg + ' ' ) + + results_xml.characters( '<hr>' ) + results_xml.characters( '<dl>' ) + results_xml.characters( '<dt>Command Line</dt>' ) + results_xml.characters( '<dd>' ) + results_xml.characters( '<pre>' ) + results_xml.characters( '\\\n'.join(lines) ) + results_xml.characters( '</pre>' ) + results_xml.characters( '</dd>' ) + results_xml.characters( '</dl>\n' ) + + results_xml.endElement( 'comment' ) + + +def compress_file( file_path, archive_path ): + utils.log( 'Compressing "%s"...' % file_path ) + + try: + z = zipfile.ZipFile( archive_path, 'w', zipfile.ZIP_DEFLATED ) + z.write( file_path, os.path.basename( file_path ) ) + z.close() + utils.log( 'Done writing "%s".'% archive_path ) + except Exception, msg: + utils.log( 'Warning: Compressing falied (%s)' % msg ) + utils.log( ' Trying to compress using a platform-specific tool...' ) + try: import zip_cmd + except ImportError: + script_dir = os.path.dirname( os.path.abspath( sys.argv[0] ) ) + utils.log( 'Could not find \'zip_cmd\' module in the script directory (%s).' % script_dir ) + raise Exception( 'Compressing failed!' ) + else: + if os.path.exists( archive_path ): + os.unlink( archive_path ) + utils.log( 'Removing stale "%s".' % archive_path ) + + zip_cmd.main( file_path, archive_path ) + utils.log( 'Done compressing "%s".' % archive_path ) + + +def read_timestamp( file ): + if not os.path.exists( file ): + result = time.gmtime() + utils.log( 'Warning: timestamp file "%s" does not exist'% file ) + utils.log( 'Using current UTC time (%s)' % result ) + return result + + return time.gmtime( os.stat( file ).st_mtime ) + + +def collect_logs( + results_dir + , runner_id + , tag + , platform + , comment_file + , timestamp_file + , user + , source + , run_type + , dart_server = None + , http_proxy = None + , revision = '' + , **unused + ): + + timestamp = time.strftime( '%Y-%m-%dT%H:%M:%SZ', read_timestamp( timestamp_file ) ) + + if dart_server: + publish_test_logs( [ results_dir ], + runner_id, tag, platform, comment_file, timestamp, user, source, run_type, + dart_server = dart_server, + http_proxy = http_proxy ) + + results_file = os.path.join( results_dir, '%s.xml' % runner_id ) + results_writer = open( results_file, 'w' ) + utils.log( 'Collecting test logs into "%s"...' % results_file ) + + results_xml = xml.sax.saxutils.XMLGenerator( results_writer ) + results_xml.startDocument() + results_xml.startElement( + 'test-run' + , { + 'tag': tag + , 'platform': platform + , 'runner': runner_id + , 'timestamp': timestamp + , 'source': source + , 'run-type': run_type + , 'revision': revision + } + ) + + copy_comments( results_xml, comment_file ) + collect_test_logs( [ results_dir ], results_writer ) + + results_xml.endElement( "test-run" ) + results_xml.endDocument() + results_writer.close() + utils.log( 'Done writing "%s".' % results_file ) + + compress_file( + results_file + , os.path.join( results_dir,'%s.zip' % runner_id ) + ) + + +def upload_logs( + results_dir + , runner_id + , tag + , user + , ftp_proxy + , debug_level + , send_bjam_log = False + , timestamp_file = None + , dart_server = None + , ftp_url = None + , **unused + ): + + logs_archive = os.path.join( results_dir, '%s.zip' % runner_id ) + upload_to_ftp( tag, logs_archive, ftp_proxy, debug_level, ftp_url ) + if send_bjam_log: + bjam_log_path = os.path.join( results_dir, 'bjam.log' ) + if not timestamp_file: + timestamp_file = bjam_log_path + + timestamp = time.strftime( '%Y-%m-%d-%H-%M-%S', read_timestamp( timestamp_file ) ) + logs_archive = os.path.join( results_dir, '%s.%s.log.zip' % ( runner_id, timestamp ) ) + compress_file( bjam_log_path, logs_archive ) + upload_to_ftp( '%s/logs' % tag, logs_archive, ftp_proxy, debug_level, ftp_url ) + + +def collect_and_upload_logs( + results_dir + , runner_id + , tag + , platform + , comment_file + , timestamp_file + , user + , source + , run_type + , revision = None + , ftp_proxy = None + , debug_level = 0 + , send_bjam_log = False + , dart_server = None + , http_proxy = None + , ftp_url = None + , **unused + ): + + collect_logs( + results_dir + , runner_id + , tag + , platform + , comment_file + , timestamp_file + , user + , source + , run_type + , revision = revision + , dart_server = dart_server + , http_proxy = http_proxy + ) + + upload_logs( + results_dir + , runner_id + , tag + , user + , ftp_proxy + , debug_level + , send_bjam_log + , timestamp_file + , dart_server = dart_server + , ftp_url = ftp_url + ) + + +def accept_args( args ): + args_spec = [ + 'locate-root=' + , 'runner=' + , 'tag=' + , 'platform=' + , 'comment=' + , 'timestamp=' + , 'source=' + , 'run-type=' + , 'user=' + , 'ftp-proxy=' + , 'proxy=' + , 'debug-level=' + , 'send-bjam-log' + , 'help' + , 'dart-server=' + , 'revision=' + , 'ftp=' + ] + + options = { + '--tag' : 'trunk' + , '--platform' : sys.platform + , '--comment' : 'comment.html' + , '--timestamp' : 'timestamp' + , '--user' : None + , '--source' : 'SVN' + , '--run-type' : 'full' + , '--ftp-proxy' : None + , '--proxy' : None + , '--debug-level' : 0 + , '--dart-server' : 'beta.boost.org:8081' + , '--revision' : None + , '--ftp' : None + + } + + utils.accept_args( args_spec, args, options, usage ) + + return { + 'results_dir' : options[ '--locate-root' ] + , 'runner_id' : options[ '--runner' ] + , 'tag' : options[ '--tag' ] + , 'platform' : options[ '--platform'] + , 'comment_file' : options[ '--comment' ] + , 'timestamp_file' : options[ '--timestamp' ] + , 'user' : options[ '--user' ] + , 'source' : options[ '--source' ] + , 'run_type' : options[ '--run-type' ] + , 'ftp_proxy' : options[ '--ftp-proxy' ] + , 'http_proxy' : options[ '--proxy' ] + , 'debug_level' : int(options[ '--debug-level' ]) + , 'send_bjam_log' : options.has_key( '--send-bjam-log' ) + , 'dart_server' : options[ '--dart-server' ] + , 'revision' : options[ '--revision' ] + , 'ftp' : options[ '--ftp' ] + } + + +commands = { + 'collect-and-upload' : collect_and_upload_logs + , 'collect-logs' : collect_logs + , 'upload-logs' : upload_logs + } + +def usage(): + print 'Usage: %s [command] [options]' % os.path.basename( sys.argv[0] ) + print ''' +Commands: +\t%s + +Options: +\t--locate-root directory to to scan for "test_log.xml" files +\t--runner runner ID (e.g. "Metacomm") +\t--timestamp path to a file which modification time will be used +\t as a timestamp of the run ("timestamp" by default) +\t--comment an HTML comment file to be inserted in the reports +\t ("comment.html" by default) +\t--tag the tag for the results ("trunk" by default) +\t--user SourceForge user name for a shell account (optional) +\t--source where Boost sources came from ("SVN" or "tarball"; +\t "SVN" by default) +\t--run-type "incremental" or "full" ("full" by default) +\t--send-bjam-log in addition to regular XML results, send in full bjam +\t log of the regression run +\t--proxy HTTP proxy server address and port (e.g. +\t 'http://www.someproxy.com:3128', optional) +\t--ftp-proxy FTP proxy server (e.g. 'ftpproxy', optional) +\t--debug-level debugging level; controls the amount of debugging +\t output printed; 0 by default (no debug output) +\t--dart-server The dart server to send results to. +\t--ftp The ftp URL to upload results to. +''' % '\n\t'.join( commands.keys() ) + + +def main(): + if len(sys.argv) > 1 and sys.argv[1] in commands: + command = sys.argv[1] + args = sys.argv[ 2: ] + else: + command = 'collect-and-upload' + args = sys.argv[ 1: ] + + commands[ command ]( **accept_args( args ) ) + + +if __name__ != '__main__': import utils +else: + # in absense of relative import... + xsl_path = os.path.abspath( os.path.dirname( sys.argv[ 0 ] ) ) + while os.path.basename( xsl_path ) != 'xsl_reports': xsl_path = os.path.dirname( xsl_path ) + sys.path.append( xsl_path ) + + import utils + main() |