diff options
Diffstat (limited to 'qpid/buildtools/buildCreator/buildCreator.py')
-rwxr-xr-x | qpid/buildtools/buildCreator/buildCreator.py | 1472 |
1 files changed, 1472 insertions, 0 deletions
diff --git a/qpid/buildtools/buildCreator/buildCreator.py b/qpid/buildtools/buildCreator/buildCreator.py new file mode 100755 index 0000000000..0a26ce09b0 --- /dev/null +++ b/qpid/buildtools/buildCreator/buildCreator.py @@ -0,0 +1,1472 @@ +#!/usr/bin/env python +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import re +import datetime +import urllib +import sys +import string + +from xml.dom import minidom +from optparse import OptionParser + +if map(int, string.split(string.split(sys.version)[0], ".")) < [2, 4, 0]: + print ("subprocess is required for this tool and is not present in versions prior to 2.4.0") + try: + import subprocess + except ImportError: + print ("subprocess module not found please install it locally or upgrade your python version") + sys.exit(1) + +import subprocess +from subprocess import Popen + +TOOL_NAME="buildCreator.py" + +#Default Build script +DEFAULT_BUILD="build.config" + +# Path locations +DEFAULT_ROOTDIR="builder" +SOURCE_DIR="src" +PATCH_DIR="patch" +BUILD_DIR="build" +RELEASE_DIR="release" + +# Command Binaries +SVN='svn' +SVN_BIN='svn' +HTTP='http' +FTP='ftp' +WGET_BIN='wget' +FILE='file' +CP_BIN='cp' +PATCH_BIN='patch' +FILE_BIN='file' +LS_BIN='ls' +TAR_BIN='tar' +BZIP2_BIN='bzip2' +UNZIP_BIN='unzip' +ECHO_BIN='echo' +SVNVERSION_BIN='svnversion' + + + +GZIP_DATA='gzip compressed data' +BZIP2_DATA='bzip2 compressed data' +ZIP_DATA='Zip archive data' +TAR_DATA='POSIX tar archive' +DIFF_FILE="'diff' output text" + +#Build Targets +DISTCLEAN='distclean' +CLEAN='clean' +RETRIEVE='retrieve' +PREPARE='prepare' +PATCH='patch' +SHOWBUILDS='showbuilds' +BUILD='build' +RELEASE='release' +FULL='full' +HELP='help' +DEFAULT_TARGET=FULL + +# XML Elements toplevel +BUILDER="builder" +ENVIRONMENT="environment" +SOURCES="sources" +SOURCE="source" +PATCHES="patches" +PATCH="patch" +BUILDS="builds" +INCLUDE="include" +DEPENDENCY='dependency' +TARGETS='targets' +SCRIPT='script' + +# XML Elements - Source/Patch elements +NAME="name" +TYPE="type" +URL="url" +REVISION="revision" +ROOTDIR="root" +VERSION="version" +PREFIX='prefix' +PATH='path' + +PATH_SEP=os.sep + +_source=None +_target=DEFAULT_BUILD +_log = True +_verbose = False +_debug = False +_ignoreErrors = False + +_charIndex = 0 +_waitingChars = ['-', '/' , '|', '\\'] + +def showUsage(): + print TOOL_NAME+" [-c|--configure <config file>] [-v| --verbose] [-q|--quiet] [-i|--ignore-errors] [<build target>] [options]" + print "Available Targets:" + print " distclean [source] - Remove all or specified retrieved source" + print " clean [source] - Remove all or specified source build directory" + print " retrieve [source] - Retrieve all or specified source" + print " prepare [source] - Prepare all or specified source : i.e. extract archives" + print " patch [source] - Patch all or specified source" + print " showbuilds - List all builds" + print " build [build] - Perform the build scripts for all or specified build" + print " release [build] - Perform the release scripts for all or specified source" + print " full - Perfrom clean, retrieve, prepare, patch, build, release for all builds (DEFAULT)" + +def main(): + global _log, _verbose, _debug, _rootDir, _target, _source, _baseConfiguration, _ignoreErrors + + # Load the + parser = OptionParser() + parser.add_option("-c", "--config", dest="config", + action="store", default=DEFAULT_BUILD, + help="set configuration file : default = " + DEFAULT_BUILD) + + parser.add_option("-v", "--verbose", dest="verbose", + action="store_true", default=False, help="enable verbose output") + + parser.add_option("-d", "--debug", dest="debug", + action="store_true", default=False, help="enable debug output") + + parser.add_option("-q", "--quiet", dest="quiet", + action="store_false", default=True, help="Enable quiet ouptut") + + parser.add_option("-i", "--ignore-errors", dest="ignoreErrors", + action="store_true", default=False, help="Ignore errors") + + + (options, args) = parser.parse_args() + + _verbose = options.verbose + _debug = options.debug + _log = options.quiet + _ignoreErrors = options.ignoreErrors + + log("Logging Enabled") + verbose("Verbose Output Enabled") + debug("Debug Enabled") + + if (len(args) > 2): + showUsage() + sys.exit(1) + else: + # NOTE : Would be good to be able to do builder.py clean build release + if (len(args) > 0 ): + # Override the default target + _target = checkTarget(args[0]) + # limit the comand to just the specified source + if (len(args) > 1 ): + _source = args[1] + else: + _source = None + else: + _target = FULL + + + _baseConfiguration = loadBaseConfiguration(options.config) + + debug ("Loading Environment") + prepareEnvironment(_baseConfiguration.getElementsByTagName(ENVIRONMENT)[0]) + + if _target == DISTCLEAN: + distclean() + + if _target == CLEAN or _target == FULL: + clean() + + if _target == RETRIEVE or _target == FULL: + try: + retrieve() + except KeyboardInterrupt: + log ("User Interrupted preparation") + sys.exit(0) + + if _target == PREPARE or _target == FULL: + prepare() + + if _target == PATCH or _target == FULL: + patch() + + if _target == SHOWBUILDS: + showBuilds() + + if _target == BUILD or _target == FULL: + build() + + if _target == RELEASE or _target == FULL: + release() + + log("Complete") + +def checkTarget(target): + + if target == HELP: + showUsage() + sys.exit(0) + + if target == DISTCLEAN: + return DISTCLEAN + + if target == CLEAN: + return CLEAN + + if target == RETRIEVE: + return RETRIEVE + + if target == PREPARE: + return PREPARE + + if target == PATCH: + return PATCH + + if target == SHOWBUILDS: + return SHOWBUILDS + + if target == BUILD: + return BUILD + + if target == RELEASE: + return RELEASE + + if target == FULL: + return FULL + + warn("Target: '"+target+"' not valid") + showUsage() + sys.exit(1) + + +################################################################################ +# +# Environment +# +################################################################################ +def prepareEnvironment(env): + global _rootDir + + rootdir = env.getElementsByTagName(ROOTDIR) + if (rootdir.length > 0): + _rootDir = getValue(rootdir[0]) + else: + verbose ("Using default build dir: "+DEFAULT_ROOTDIR) + _rootDir = os.getcwd() + PATH_SEP + DEFAULT_ROOTDIR + + if _rootDir == "": + verbose (ROOTDIR+" value is empty. Please specify a value for "+ ROOTDIR) + attemptExit(0) + + if (os.path.exists(_rootDir)): + verbose ("Using Existing root dir: "+_rootDir) + else: + mkdir(_rootDir) + +################################################################################ +# +# Clean Methods +# +################################################################################ +def clean(): + global _source + sources = getSourceList() + + if len(sources) > 0: + log ("Removing built code...") + performed = False + for source in sources: + if _source != None: + if getName(source) == _source: + performed = True + removeDir(source, BUILD_DIR) + else: + removeDir(source, BUILD_DIR) + + if _source == None: + deleteDir(_rootDir + PATH_SEP + BUILD_DIR) + + builds = getBuildList() + if len(builds) > 0: + log ("Removing built releases...") + for build in builds: + if _source != None: + if getName(build) == _source: + performed = True + removeDir(build, RELEASE_DIR) + else: + removeDir(build, RELEASE_DIR) + if _source == None: + deleteDir(_rootDir + PATH_SEP + RELEASE_DIR) + + if _source != None: + if not performed: + fatal("No such source:" + _source); + + + +def distclean(): + sources = getSourceList() + + if len(sources) > 0: + log ("Removing source...") + for source in sources: + if _source != None: + if getName(source) == _source: + performed = True + removeDir(source, SOURCE_DIR) + else: + removeDir(source, SOURCE_DIR) + + if _source == None: + deleteDir(_rootDir + PATH_SEP + SOURCE_DIR) + + log ("Removing built code...") + for source in sources: + if _source != None: + if getName(source) == _source: + performed = True + removeDir(source, BUILD_DIR) + else: + removeDir(source, BUILD_DIR) + if _source == None: + deleteDir(_rootDir + PATH_SEP + BUILD_DIR) + + patches =getPatchList() + if len(patches) > 0: + log ("Removing patches...") + for patch in patches: + if _source != None: + if getName(patch) == _source: + performed = True + removeDir(patch, PATCH_DIR) + else: + removeDir(patch, PATCH_DIR) + if _source == None: + deleteDir(_rootDir + PATH_SEP + PATCH_DIR) + + + builds = getBuildList() + if len(builds) > 0: + log ("Removing built releases...") + for build in builds: + if _source != None: + if getName(build) == _source: + performed = True + removeDir(build, RELEASE_DIR) + else: + removeDir(build, RELEASE_DIR) + if _source == None: + deleteDir(_rootDir + PATH_SEP + RELEASE_DIR) + + + if _source == None: + deleteDir(_rootDir) + + if _source != None: + if not performed: + fatal("No such source:" + _source); + + + +def removeDir(source, rootdir): + name = getName(source) + deleteDir(_rootDir + PATH_SEP + rootdir + PATH_SEP + name) + +################################################################################ +# +# Retrieve Methods +# +################################################################################ +def retrieve(): + global _source + sources = getSourceList() + + # Retreive Source + performed=False + if len(sources) > 0: + log ("Retrieving source...") + + mkdir(_rootDir + PATH_SEP + SOURCE_DIR) + + for source in sources: + if _source != None: + if getName(source) == _source: + performed = True + downloadSource(source, SOURCE_DIR) + else: + downloadSource(source, SOURCE_DIR) + + # Retreive Patches + patches = getPatchList() + if len(patches) > 0: + + log ("Retrieving patches...") + + mkdir(_rootDir + PATH_SEP + PATCH_DIR) + + for patch in patches: + if _source != None: + if getName(patch) == _source: + performed = True + downloadSource(patch, PATCH_DIR) + else: + downloadSource(patch, PATCH_DIR) + + if _source != None: + if not performed: + fatal("No such patch:" + _source); + + +################################################################################ +# +# Prepare Methods +# +################################################################################ + +def prepare(): + verbose("Prepare") + + mkdir(_rootDir + PATH_SEP + BUILD_DIR) + + sources = getSourceList() + performed = False + if len(sources) > 0: + log ("Preparing source...") + for source in sources: + if _source != None: + if getName(source) == _source: + log_no_newline("Preparing "+getName(source)+": ") + performed = True + postProcess(source, SOURCE_DIR) + else: + log_no_newline("Preparing "+getName(source)+": ") + postProcess(source, SOURCE_DIR) + if _source != None: + if not performed: + fatal("No such source:" + _source); + + patches = getPatchList() + if len(patches) > 0: + log ("Preparing patches...") + for patch in patches: + if _source != None: + if getName(patch) == _source: + performed = True + log("Preparing "+getName(patch)) + postProcess(patch, PATCH_DIR) + else: + log("Preparing "+getName(patch)) + postProcess(patch, PATCH_DIR) + + if _source != None: + if not performed: + fatal("No such patch:" + _source); + + +def postProcess(item, destination): + name = getName(item) + type = getType(item) + + verbose("Post Processing:"+name) + + targetdir = _rootDir + PATH_SEP + destination + PATH_SEP + name + + builddir = _rootDir + PATH_SEP + BUILD_DIR + PATH_SEP + name + + + if type == SVN: + # Do we want to perform an export? + + #extractcommand=SVN_BIN+" export "+ targetdir +" "+ builddir + # Use -v just now so we can show progress + extractcommand=CP_BIN+" -rvf "+ targetdir +" "+ builddir + + runCommand(extractcommand, False) + + else: + if type == FILE or type == HTTP or type == FTP: + + mkdir(builddir) + + # Look at all the files and see if they need unpacks + for root, dirs, files in os.walk(targetdir, topdown=False): + for file in files: + command = FILE_BIN+" "+root+PATH_SEP+file + + result = Popen(command, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + line = result.stdout.readline() + firstline=line + while (line != "" ): + # process nextline + line=result.stdout.readline() + + result.wait() + + if result.returncode != 0: + fatal("Download (" + name + ") contained unrecognized file type:"+ firstline) + + + extractcommand="" + + if firstline.find(GZIP_DATA) != -1: + extractcommand=TAR_BIN+" -vxzf "+root+PATH_SEP+file+" -C " + builddir + + if firstline.find(BZIP2_DATA) != -1: + extractcommand=TAR_BIN+" -vxjf "+root+PATH_SEP+file+" -C " + builddir + + if firstline.find(ZIP_DATA) != -1: + extractcommand=UNZIP_BIN+" -v "+root+PATH_SEP+file+" -d "+ builddir + + if firstline.find(TAR_DATA) != -1: + extractcommand=TAR_BIN+" -vxf "+root+PATH_SEP+file+" -C "+ builddir + + if firstline.find(DIFF_FILE) != -1 or firstline.find("text"): + extractcommand=CP_BIN+" -r "+root+PATH_SEP+file+" "+ builddir + + + + if not extractcommand=="": + log_no_newline("Extracting archive:" + file+": " ) + runCommand(extractcommand, False) + else: + fatal("Download (" + name + ") contained unsupported file type:"+ firstline) + + + +################################################################################ +# +# Patch Methods +# +################################################################################ +def patch(): + + # Retreive Patches + patches= getPatchList() + if len(patches) > 0: + performed = False + for patch in patches: + if _source != None: + if getName(source) == _source: + performed = True + applyPatch(patch) + else: + applyPatch(patch) + + if _source != None: + if not performed: + fatal("No such patch:" + _source); + + +def applyPatch(patch): + global _rootDir + + name = getName(patch) + type = getType(patch) + source = getValue(patch.getElementsByTagName(SOURCE)[0]) + if (patch.getElementsByTagName(PREFIX).length > 0): + prefix = getValue(patch.getElementsByTagName(PREFIX)[0]) + else: + prefix = None + + if (patch.getElementsByTagName(PATH).length > 0): + path= getValue(patch.getElementsByTagName(PATH)[0]) + else: + path = None + + + basecommand = PATCH_BIN + + if prefix != None: + basecommand = basecommand + " -p "+prefix + + basecommand = basecommand + " -E -d "+ _rootDir + PATH_SEP + BUILD_DIR + PATH_SEP + source + PATH_SEP + + if path != None: + basecommand = basecommand + path + + basecommand = basecommand + " < " + + patchSource= _rootDir + PATH_SEP + PATCH_DIR + PATH_SEP + name + + for root, dirs, files in os.walk(patchSource): + if '.svn' in dirs: + dirs.remove('.svn') + files.sort() + for patchName in files: + log("Applying patch '" + name + "'("+patchName+") to " + source) + runCommandShowError(basecommand + patchSource + PATH_SEP + patchName) + + +################################################################################ +# +# build Methods +# +################################################################################ +def showBuilds(): + builds = getNamesfromBuildList(getBuildList()) + if len(builds) > 0: + log("Available Builds:") + for build in builds: + log(" "+build) + else: + log("No builds available") + +# +# Given a list of build elements extract the Name values and return as a list +# +def getNamesfromBuildList(list): + names=[] + for item in list: + name = getName(item) + if name != None: + names.append(name) + return names + +def build(): + doBuildAction(BUILD) + + + +################################################################################ +# +# Release Methods +# +################################################################################ +def release(): + log ("Releasing...") + mkdir(_rootDir + PATH_SEP + RELEASE_DIR) + + builds = getBuildList() + + for build in builds: + if _source != None: + if getName(build) == _source: + mkdir(_rootDir + PATH_SEP + RELEASE_DIR + PATH_SEP + getName(build)) + else: + mkdir(_rootDir + PATH_SEP + RELEASE_DIR + PATH_SEP + getName(build)) + + doBuildAction(RELEASE) + + + +################################################################################ +# +# Build Helper Methods +# +################################################################################ + +def doBuildAction(action): + config = _baseConfiguration + + if len(getSourceList()) > 0: + log("Performing "+ action.title() +"...") + + builds = getBuildList() + + performed = False + for build in builds: + if _source != None: + if getName(build) == _source: + performed = True + performScript(build , action) + else: + performScript(build, action) + + if _source != None: + if not performed: + fatal("No such build:" + _source); + + +def performScript(build, scriptName): + name = getName(build) + + checkDependencies(build) + + verbose("Running "+scriptName+":"+name) + + targets = build.getElementsByTagName(TARGETS) + + if targets.length > 0: + target = targets[0].getElementsByTagName(scriptName) + + if target.length > 1: + fatal("More than one build target specified") + + if target.length == 0: + fatal("No build target specified") + + script = getValue(target[0].getElementsByTagName(SCRIPT)[0]) + + script = peformSubstitutionsInScript(build, script) + + debug(script) + + runScript(script) + + else: + fatal("Build "+name+" has no build targets") + + +def checkDependencies(build): + name = getName(build) + dependencies = build.getElementsByTagName(DEPENDENCY) + + if dependencies > 0: + for dependency in dependencies : + sources = dependency.getElementsByTagName(SOURCE) + if sources.length == 0: + fatal("No sources specified in dependency block for build:"+name) + else: + for source in sources: + sourceDependency = getValue(source) + if not (sourceDefined(sourceDependency)): + fatal("Unable to build "+name+" as specifed dependency("+sourceDependency +") is not available") + +def sourceDefined(name): + for source in getSourceList(): + sourcename = getValue(source.getElementsByTagName(NAME)[0]) + if sourcename == name: + return True + return False + + +def runScript(script): + (returncode, stdout, stderr) = runCommandWithOutput(script) + + if _debug: + for line in stdout: + debug(line) + for line in stderr: + debug(line) + + if returncode != 0: + for line in stdout: + warn(line) + for line in stderr: + warn(line) + + warn("Script Failed") + + attemptExit(1) + + +################################################################################ +# +# XML Helper Methods +# +################################################################################ + +def loadBaseConfiguration(config): + log ("Loading configuration:" + config) + full = minidom.parse(config) + return full.getElementsByTagName(BUILDER)[0] + +# +# Assumes that we have a <node>text</node> element and returns the text value. +# +def getValue(node): + if node.childNodes.length > 0: + return node.childNodes[0].data + else: + return "" + +def getEnvironment(): + env = _baseConfiguration.getElementsByTagName(ENVIRONMENT) + if env.length > 0: + return env[0] + else: + return None + +# +# Returns the value of the NAME element contained in the specified item +# +def getName(item): + name = item.getElementsByTagName(NAME) + if name.length > 0: + return getValue(name[0]) + +# +# Returns the value of the TYPE element contained in the specified item +# +def getType(item): + type = item.getElementsByTagName(TYPE) + if type.length > 0: + return getValue(type[0]) + +# +# Returns the value of the URL element contained in the specified item +# +def getURL(item): + url = item.getElementsByTagName(URL) + if url.length > 0: + return getValue(url[0]) + +# +# Return the list of sources in this build configuration +# If no sources are available then this is logged as a fatal error. +# +def getSourceList(): + config = _baseConfiguration + sourceCount = config.getElementsByTagName(SOURCES).length + if sourceCount > 0: + return config.getElementsByTagName(SOURCES)[0].getElementsByTagName(SOURCE) + else: + fatal("No source elements defined.") +# +# Return the list of patches in this build configuration +# +def getPatchList(): + config = _baseConfiguration + patchCount = config.getElementsByTagName(PATCHES).length + if patchCount > 0: + return config.getElementsByTagName(PATCHES)[0].getElementsByTagName(PATCH) + else: + return [] + +# Returns a list of build elements including any any included build files +# Currently nested build elements are not supported so all builds must be specified via the <include> tag. +# +def getBuildList(): + config = _baseConfiguration + + builds = config.getElementsByTagName(BUILDS) + buildcount = builds.length + + if buildcount > 0: + build = builds[0] + useInclude = build.getElementsByTagName(INCLUDE).length > 0 + + # If we are using includes then build a list of all the files we need to include + if useInclude: + return getIncludeList(build) + + else: + warn("Nested builds not currently supported") + else: + fatal("No Builds defined in config") + +# +# Look at all <include> values in the given element and return the list of inlcudes +# +def getIncludeList(build): + includelist=[] + for include in build.getElementsByTagName(INCLUDE): + for item in getIncludeValue(getValue(include)): + includelist.append(item) + + return includelist + +# +# Process in the given include value. +# This is done by performing `ls <include>` +# This means includes such as 'builds/*.config' will match multiple includes and return all entries +# +# Any error in performing the ls is printed and the tool exits (unless ignore errors) +# +def getIncludeValue(include): + debug("Loading Includes:"+include+" ") + + command = LS_BIN+" "+include + (returncode, stdout, stderr) = runCommandWithOutput(command) + + if returncode == 0: + values=[] + for line in stdout: + include = loadIncludeFile(line) + if not include == None: + values.append(include) + return values + else: + for line in stderr: + warn(line) + attemptExit(1) + +# +# Given a file name parse the XML. Any trailing '\n's that the ls command may have added are removed here. +# The file is checked to ensure that it is a <builds> file +# The first <build> element is returned. +# +# TODO: Allow multiple builds per file. +# +def loadIncludeFile(file): + buildFile = minidom.parse(file.rstrip('\n')) + + builds = buildFile.getElementsByTagName(BUILDS) + + if builds.length != 1: + warn("Build Configuration does not contain any <"+BUILDS+"> definitions") + else: + buildElements = builds[0].getElementsByTagName(BUILD) + if not buildElements.length > 0: + warn("Build Configuration does not contain any <"+BUILD+"> definitions") + else: + if buildElements.length > 0: + build = buildElements[0] + # getElementsByTagName is recursive so this will pick up the sub element build + # Only use the first element + namecount = build.getElementsByTagName(NAME).length + if namecount > 0: + return build + else: + return None + +# +# Given the build target and a script substitute $value entries in script for values in +# the Environment +# the Source entries <source><name> +# the build <build><name> +# the release location : _rootDir + PATH_SEP + RELEASE_DIR + PATH_SEP + buildName +# +def peformSubstitutionsInScript(build, script): + buildName = getName(build) + sources = getSourceList() + + #Replace Build name + script = script.replace("$build", buildName) + + #Replace release directory + releaseDir = _rootDir + PATH_SEP + RELEASE_DIR + PATH_SEP + buildName + script = script.replace("$release", releaseDir) + + # Replace Source varables + for source in sources: + sourceName = getName(source) + + search = "$"+sourceName + + sourcePath = source.getElementsByTagName(PATH) + + replacement = _rootDir + PATH_SEP + BUILD_DIR + PATH_SEP + sourceName + if sourcePath.length > 0: + replacement = replacement + PATH_SEP + getValue(sourcePath[0]) + + script = script.replace(search,replacement) + + # Take values from the environment script for replacement + env = getEnvironment() + if env != None: + for item in env.childNodes: + if item.nodeType == 1: + + search = "$"+item.tagName + replace = item.childNodes[0].data + + script = script.replace(search,replace) + + # Perform keyword substitution replacements + # Currently only one substitution exists so for simplisity fix it here + writeVersionSubstitution = script.find("$writeVersions") + if writeVersionSubstitution != -1: + + #Extract Filename + fileNameStart = script.find("(",writeVersionSubstitution) + fileNameEnd = script.find(")",fileNameStart) + fileName= script[fileNameStart+1:fileNameEnd] + + substitution = createVersionSubstitution(build, fileName) + + script = script.replace("$writeVersions(" + fileName + ")", substitution) + + + return script + +################################################################################ +# +# Keyword Substitutions +# +################################################################################ + +# +# Use the specified build as to lookup all associated source/patches and write out their details +# to the specified file using shell redirects. redirects are to be used as the absolute filename +# location may not be known as the name comes in via the release script +# +def createVersionSubstitution(build, filename): + substitution = "" + sources = getSourceList(); + + dependencies = build.getElementsByTagName(DEPENDENCY) + + if dependencies > 0: + substitution += "\n echo 'Source Version Information:'>> " + filename + for dependency in dependencies : + depSources = dependency.getElementsByTagName(SOURCE) + # Can assume we have dependencies as we would have failed before now + for source in depSources: + sourceDependency = getValue(source) + # We can assume source is valid. + for s in sources: + if sourceDependency == getName(s): + # provide header <source>:<type>:<revision> + substitution += "\n " + ECHO_BIN + " -n '" + sourceDependency + ":" \ + + getType(s) + ":' >> " + filename + substitution += "\n" + getVersionCommand(s) + " >>" + filename + # Add Source URL to Revisions file + url = getValue(s.getElementsByTagName(URL)[0]) + substitution += "\n" + ECHO_BIN + " \"URL:" + url + "\" >> "+filename + # Add Patches applied to this source to revisions file + substitution += addPatchVersions(s, filename) + + return substitution + +# +# Use the specified source as to lookup all associated patches and write their details out the +# the specified file using shell redirects. redirects are to be used as the absolute filename +# location may not be known as the name comes in via the release script +# +def addPatchVersions(source, filename): + substitution = "" + + patches = getPatchList() + + sourceName = getName(source) + + for patch in patches: + patchSourceName = getValue(patch.getElementsByTagName(SOURCE)[0]) + + if sourceName == patchSourceName: + type = getType(patch) + substitution += "\n" + ECHO_BIN + " \"\t"+getName(patch)+":"+type + "\" >> "+filename + url = getValue(patch.getElementsByTagName(URL)[0]) + substitution += "\n" + ECHO_BIN + " \"\t\tURL:" + url + "\" >> "+filename + if (type == SVN): + if (patch.getElementsByTagName(REVISION).length > 0): + substitution += "\n" + ECHO_BIN + " \"\t\tREVISION:"+ \ + getValue(patch.getElementsByTagName(REVISION)[0]) + "\" >> " + filename + else: + substitution += "\n" + ECHO_BIN + " -n \"\t\tREVISION: \" >> " + filename + substitution += "\n" + SVNVERSION_BIN + " " + _rootDir + PATH_SEP + PATCH_DIR + PATH_SEP + getName(patch) + " >> " + filename + + + if (patch.getElementsByTagName(PREFIX).length > 0): + substitution += "\n" + ECHO_BIN + " \"\t\tPREFIX: " + \ + getValue(patch.getElementsByTagName(PREFIX)[0]) + "\" >> " + filename + + if (patch.getElementsByTagName(PATH).length > 0): + substitution += "\n" + ECHO_BIN + " \"\t\tPATH: " + \ + getValue(patch.getElementsByTagName(PATH)[0]) + "\" >> " + filename + + global _rootDir + patchSource= _rootDir + PATH_SEP + PATCH_DIR + PATH_SEP + getName(patch) + + # + # Include the list of patches files applied + # + for root, dirs, files in os.walk(patchSource): + if '.svn' in dirs: + dirs.remove('.svn') + files.sort() + for patchName in files: + substitution += "\n" + ECHO_BIN + " \"\t\tFILE: " + patchName + "\" >> " + filename + + + + + if (substitution != ""): + return "\n" + ECHO_BIN + " \"\tPatches applied to " + sourceName + ":\" >> " + filename + substitution + else: + return "\n" + ECHO_BIN + " \"\tNo Patches\" >> " + filename + + +# +# Given a source entry return the command that will provide the current version +# of that source. +# i.e. svn source : svnversion <path to source> +# http source : echo <URL> +# +def getVersionCommand(source): + global _rootDir + type = getType(source) + + versionCommand="" + + if type == SVN: + versionCommand=SVNVERSION_BIN+" "+_rootDir + PATH_SEP + SOURCE_DIR + PATH_SEP + getName(source) + else: + if type == FILE or type == HTTP or type == FTP: + versionCommand = ECHO_BIN +" " + getURL(source) + + + return versionCommand + +################################################################################ +# +# Download Helper Methods +# +################################################################################ + +# +# Download the item specified in source to the given destintation +# +def downloadSource(source, destination): + name = getName(source) + type = getType(source) + url = getValue(source.getElementsByTagName(URL)[0]) + log ( "Retrieving "+ name + "("+ type +")") + + targetdir=_rootDir + PATH_SEP + destination + PATH_SEP + name + + command = "" + + mkdir(targetdir) + + if (os.listdir(targetdir)==[]): + + # Setup command for a fresh checkout + if (type == SVN): + command = SVN_BIN+" co "+url+" "+targetdir + if (source.getElementsByTagName(REVISION).length > 0): + revision = getValue(source.getElementsByTagName(REVISION)[0]) + command = SVN_BIN+" co -r"+revision+" "+url+" "+targetdir + else: + if (type == HTTP): + command = WGET_BIN+" --no-directories -P "+targetdir+" "+url + else: + if (type == FILE): + if url.startswith(HTTP): + command = WGET_BIN+" -P "+targetdir+" "+url + else: + if url.startswith(FTP): + command = WGET_BIN+" -P "+targetdir+" "+url + else: + command = CP_BIN+" -R "+url+" "+targetdir + else: + warn("Target directory(" + targetdir + ") is not empty please ensure contents are valid or run 'clean "+name+"'") + + verbose("Executing:"+command) + log_no_newline("Retrieving "+source.nodeName+": ") + + if (type == FILE): + runCommand(command, True) + else: + runCommand(command, False) + +################################################################################ +# +# Command Helper Methods +# +################################################################################ + +# +# Run command and print out last 20 lines of data on error +# +def runCommandShowError(command): + last20 = runCommand(command, False) + if last20 != None: + lines=last20[0] + lines=lines + 1 + current = 1 + while current != lines: + log (last20[current]) + current = current + 1 + attemptExit(1) + +# +# Runs the given command if showOutput is true then stdout/stderr is shown on screen +# other wise the last 20 lines of output is gathered: +# +# As command runs progress is shown +# +# return array [0] = no of elements in array. Array is fixed size 21 elements but not all are used. FIXME: this is poor +# +# TODO: Current mechanism for limiting to 20 lines is poor, potential to replace usages of this +# method with runCommandWithOutput below +# +def runCommand(command, showOutput): + debug("Running Command:"+command) + try: + if showOutput: + # Process that shows the output + result = Popen(command, shell=True) + else: + # consume the output ourselves + result = Popen(command, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + index=0 + last20=[""] * 21 + line = result.stdout.readline() + while (line != "" ): + logWaiting() + + #Record last 20 lines of output + index = index + 1 + if index == 20: + index = 1 + + last20[index]=line + + # process nextline + line = result.stdout.readline() + + # + # If we didn't get any standard or fill our buffers then fill the end of the buffer with any stderr + # + if index == 0 | index < 15 : + line = result.stderr.readline() + + if index != 0: + index = index + 1 + if line != "": + last20[index]="STDERR" + reset = index + while (line != "" ): + logWaiting() + + #Record last 20 lines of output + index = index + 1 + if index == 20: + index = reset + + last20[index]=line + + # process nextline + line = result.stderr.readline() + + result.wait() + + if result.returncode == 0: + logWaitingDone() + else: + logWaitingFailed("Failed") + attemptExit(1) + if not showOutput: + last20[0]=index + return last20 + + return None + + except IOError: + logWaitingFailed ("Error running command.") + attemptExit(1) + +# +# Runs the given command if showOutput is true then stdout/stderr is shown on screen +# Stdout and stderr is gathered up and returned with error code. +# +# return (result.returncode, stdout, stderr) +# +# As command runs progress is shown +# +def runCommandWithOutput(command): + + result = Popen(command, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + + # Retrieve STDOUT + stdout=[] + line = result.stdout.readline() + while (line != "" ): + logWaiting() + + stdout.append(line) + + # process nextline + line = result.stdout.readline() + + line = result.stderr.readline() + + # Retrieve STDERR + stderr=[] + while (line != "" ): + stderr.append(line) + + # process nextline + line = result.stderr.readline() + + result.wait() + + logWaitingClear() + + return (result.returncode, stdout, stderr) + + +################################################################################ +# +# OS Helper Methods +# +################################################################################ + +# +# Check _ignoreErrors value and exit if false +# +def attemptExit(code): + if not _ignoreErrors: + sys.exit(code) + else: + print ("Ignoring Errors") + +# +# Check that the required binaries are present for this tool. +# Only checks the minimum set. +# Logs warning if archive tools are missing +# +def checkSystemRequirements(): + exists = checkExists(SVN_BIN) + exists = exists & checkExists(WGET_BIN) + exists = exists & checkExists(CP_BIN) + exists = exists & checkExists(PATCH_BIN) + exists = exists & checkExists(FILE_BIN) + + if not checkExists(TAR_BIN): + warn("Unable to process tar files as tar binary does not exist:" + TAR_BIN) + if not checkExists(BZIP2_BIN): + warn("Unable to process bzip2 files as bzip2 binary does not exist:" + BZIP2_BIN) + if not checkExists(UNZIP_BIN): + warn("Unable to process zip files as unzip binary does not exist:" + UNZIP_BIN) + + if not exists: + sys.exit(1) + +# +# Helper that checks for files existence +# +def checkExists(command): + debug_no_newline("Checking for "+command+":") + command = LS_BIN+" "+command + + result = Popen(command, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + line = result.stdout.readline() + + while (line != "" ): + # process nextline + line = result.stdout.readline() + + result.wait() + + if result.returncode == 0: + debug("OK") + return True + else: + debug("Missing") + warn("Missing dependancy:"+command) + return False + + +# Delete everything reachable from the directory named in 'top', +# assuming there are no symbolic links. +# +# If an attempt to delete '/' is performed this is logged as a fatal error +# +def deleteDir(top): + if top == '/': + fatal("Exiting as attempt to delete '/' occured.") + else: + if (os.path.exists(top)): + log_no_newline("Removing:"+top+". ") + for root, dirs, files in os.walk(top, topdown=False): + logWaiting() + for name in files: + os.remove(os.path.join(root, name)) + for name in dirs: + logWaiting() + os.rmdir(os.path.join(root, name)) + + logWaiting() + os.rmdir(top) + + logWaitingDone() + +def mkdir(dir): + if not os.path.exists(dir): + os.mkdir(dir) + + +################################################################################ +# +# Logging Helper Methods +# +################################################################################ + +# +# Provide a spinning -/|\ +# +def logWaiting(): + global _charIndex, _waitingChars + + _charIndex = (_charIndex + 1) % len(_waitingChars) + + log_no_newline('\b') + log_no_newline(_waitingChars[_charIndex]) + +# +# Clear the logWaiting symbol and end the line with ' Done' +# +def logWaitingDone(): + log_no_newline('\b') + log(" Done") + +# +# Clear the logWaiting symbol +# +def logWaitingClear(): + log_no_newline('\b') + +# +# Clear the logWaiting symbol and end line with messsage +# +def logWaitingFailed(message): + log_no_newline('\b') + log(" "+message) + +def debug(string): + if _debug: + log(string) + +def verbose(string): + if _verbose: + log(string) + +def log (string): + if _log: + print string + +def warn (string): + print string + +def fatal(string): + print string + attemptExit(1) + +def log_no_newline (string): + if _log: + sys.stdout.write(string) + sys.stdout.flush() + +def verbose_no_newline (string): + if _verbose: + sys.stdout.write(string) + sys.stdout.flush() + +def debug_no_newline (string): + if _debug: + sys.stdout.write(string) + sys.stdout.flush() + +if __name__ == "__main__": + main() |