path: root/qpid/buildtools/buildCreator/buildCreator.py

#!/usr/bin/env python
#
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import os
import re
import datetime
import urllib
import sys
import string

from xml.dom import minidom
from optparse import OptionParser

if map(int, string.split(string.split(sys.version)[0], ".")) < [2, 4, 0]:
    print ("subprocess is required for this tool and is not present in versions prior to 2.4.0")
    try:
        import subprocess
    except ImportError:
        print ("subprocess module not found please install it locally or upgrade your python version")
        sys.exit(1)

import subprocess
from subprocess import Popen

TOOL_NAME="buildCreator.py"

#Default Build script
DEFAULT_BUILD="build.config"

# Path locations
DEFAULT_ROOTDIR="builder"
SOURCE_DIR="src"
PATCH_DIR="patch"
BUILD_DIR="build"
RELEASE_DIR="release"

# Command Binaries
SVN='svn'
SVN_BIN='svn'
HTTP='http'
FTP='ftp'
WGET_BIN='wget'
FILE='file'
CP_BIN='cp'
PATCH_BIN='patch'
FILE_BIN='file'
LS_BIN='ls'
TAR_BIN='tar'
BZIP2_BIN='bzip2'
UNZIP_BIN='unzip'
ECHO_BIN='echo'
SVNVERSION_BIN='svnversion'



GZIP_DATA='gzip compressed data'
BZIP2_DATA='bzip2 compressed data'
ZIP_DATA='Zip archive data'
TAR_DATA='POSIX tar archive'
DIFF_FILE="'diff' output text"

#Build Targets
DISTCLEAN='distclean'
CLEAN='clean'
RETRIEVE='retrieve'
PREPARE='prepare'
PATCH='patch'
SHOWBUILDS='showbuilds'
BUILD='build'
RELEASE='release'
FULL='full'
HELP='help'
DEFAULT_TARGET=FULL

# XML Elements toplevel
BUILDER="builder"
ENVIRONMENT="environment"
SOURCES="sources"
SOURCE="source"
PATCHES="patches"
PATCH="patch"
BUILDS="builds"
INCLUDE="include"
DEPENDENCY='dependency'
TARGETS='targets'
SCRIPT='script'

# XML Elements - Source/Patch elements
NAME="name"
TYPE="type"
URL="url"
REVISION="revision"
ROOTDIR="root"
VERSION="version"
PREFIX='prefix'
PATH='path'

PATH_SEP=os.sep

_source=None
_target=DEFAULT_BUILD
_log = True
_verbose = False
_debug = False
_ignoreErrors = False
                         
_charIndex = 0
_waitingChars = ['-', '/' , '|',  '\\']

def showUsage():
    print TOOL_NAME+" [-c|--configure <config file>] [-v| --verbose] [-q|--quiet] [-i|--ignore-errors] [<build target>] [options]"
    print "Available Targets:"
    print "     distclean [source] - Remove all or specified retrieved source"
    print "     clean [source]     - Remove all or specified source build directory"
    print "     retrieve [source]  - Retrieve all or specified source"
    print "     prepare [source]   - Prepare all or specified source : i.e. extract archives"
    print "     patch [source]     - Patch all or specified source"
    print "     showbuilds         - List all builds"
    print "     build [build]      - Perform the build scripts for all or specified build"
    print "     release [build]    - Perform the release scripts for all or specified source"
    print "     full               - Perfrom clean, retrieve, prepare,  patch, build, release for all builds (DEFAULT)"

def main():
    global _log, _verbose, _debug, _rootDir, _target, _source, _baseConfiguration, _ignoreErrors

    # Load the
    parser = OptionParser()
    parser.add_option("-c", "--config", dest="config",
                      action="store", default=DEFAULT_BUILD,
                      help="set configuration file : default = " + DEFAULT_BUILD)

    parser.add_option("-v", "--verbose", dest="verbose",
                      action="store_true", default=False, help="enable verbose output")

    parser.add_option("-d", "--debug", dest="debug",
                      action="store_true", default=False, help="enable debug output")

    parser.add_option("-q", "--quiet", dest="quiet",
                      action="store_false", default=True, help="Enable quiet ouptut")

    parser.add_option("-i", "--ignore-errors", dest="ignoreErrors",
                      action="store_true", default=False, help="Ignore errors")


    (options, args) = parser.parse_args()

    _verbose = options.verbose
    _debug = options.debug
    _log = options.quiet
    _ignoreErrors = options.ignoreErrors

    log("Logging Enabled")
    verbose("Verbose Output Enabled")
    debug("Debug Enabled")

    if (len(args) > 2):
        showUsage()
        sys.exit(1)
    else:
        # NOTE : Would be good to be able to do builder.py clean build release
        if (len(args) > 0 ):
            # Override the default target
            _target = checkTarget(args[0])
            # limit the comand to just the specified source
            if (len(args) > 1 ):
                _source = args[1]
            else:
                _source = None
        else:
            _target = FULL


    _baseConfiguration = loadBaseConfiguration(options.config)

    debug ("Loading Environment")
    prepareEnvironment(_baseConfiguration.getElementsByTagName(ENVIRONMENT)[0])

    if _target == DISTCLEAN:
        distclean()

    if _target == CLEAN or _target == FULL:
        clean()

    if _target == RETRIEVE or _target == FULL:
        try:
            retrieve()
        except KeyboardInterrupt:
            log ("User Interrupted preparation")
            sys.exit(0)

    if _target == PREPARE or _target == FULL:
        prepare()

    if _target == PATCH or _target == FULL:
        patch()

    if _target == SHOWBUILDS:
        showBuilds()

    if _target == BUILD or _target == FULL:
        build()

    if _target == RELEASE or _target == FULL:
        release()

    log("Complete")

def checkTarget(target):

    if target == HELP:
        showUsage()
        sys.exit(0)                

    if target == DISTCLEAN:
        return DISTCLEAN

    if target == CLEAN:
        return CLEAN

    if target == RETRIEVE:
        return RETRIEVE

    if target == PREPARE:
        return PREPARE

    if target == PATCH:
        return PATCH

    if target == SHOWBUILDS:
        return SHOWBUILDS

    if target == BUILD:
        return BUILD

    if target == RELEASE:
        return RELEASE

    if target == FULL:
        return FULL

    warn("Target: '"+target+"' not valid")
    showUsage()
    sys.exit(1)


################################################################################
#
# Environment
#
################################################################################
def prepareEnvironment(env):
    global _rootDir

    rootdir = env.getElementsByTagName(ROOTDIR)
    if (rootdir.length > 0):
        _rootDir = getValue(rootdir[0])
    else:
        verbose ("Using default build dir: "+DEFAULT_ROOTDIR)
        _rootDir = os.getcwd() + PATH_SEP + DEFAULT_ROOTDIR

    if _rootDir == "":
        verbose (ROOTDIR+" value is empty. Please specify a value for "+ ROOTDIR)
        attemptExit(0)

    if (os.path.exists(_rootDir)):
        verbose ("Using Existing root dir: "+_rootDir)
    else:
        mkdir(_rootDir)

################################################################################
#
# Clean Methods
#
################################################################################
def clean():
    global _source
    sources = getSourceList()

    if len(sources) > 0:
        log ("Removing built code...")
        performed = False        
        for source in sources:
            if _source != None:
                if getName(source) == _source:
                    performed = True
                    removeDir(source, BUILD_DIR)
            else:
                removeDir(source, BUILD_DIR)

        if _source == None:
            deleteDir(_rootDir + PATH_SEP + BUILD_DIR)

    builds = getBuildList()
    if len(builds) > 0:
        log ("Removing built releases...")
        for build in builds:
            if _source != None:
                if getName(build) == _source:
                    performed = True
                    removeDir(build, RELEASE_DIR)
            else:
                removeDir(build, RELEASE_DIR)
        if _source == None:
            deleteDir(_rootDir + PATH_SEP + RELEASE_DIR)

    if _source != None:
        if not performed:
            fatal("No such source:" + _source);



def distclean():
    sources  = getSourceList()

    if len(sources) > 0:
        log ("Removing source...")
        for source in sources:
            if _source != None:
                if getName(source) == _source:
                    performed = True
                    removeDir(source, SOURCE_DIR)
            else:
                removeDir(source, SOURCE_DIR)

        if _source == None:
            deleteDir(_rootDir + PATH_SEP + SOURCE_DIR)

        log ("Removing built code...")
        for source in sources:
            if _source != None:
                if getName(source) == _source:
                    performed = True
                    removeDir(source, BUILD_DIR)
            else:
                removeDir(source, BUILD_DIR)
        if _source == None:
            deleteDir(_rootDir + PATH_SEP + BUILD_DIR)

    patches =getPatchList()
    if len(patches) > 0:
        log ("Removing patches...")
        for patch in patches:
            if _source != None:
                if getName(patch) == _source:
                    performed = True
                    removeDir(patch, PATCH_DIR)
            else:
                removeDir(patch, PATCH_DIR)
        if _source == None:
            deleteDir(_rootDir + PATH_SEP + PATCH_DIR)


    builds = getBuildList()
    if len(builds) > 0:
        log ("Removing built releases...")
        for build in builds:
            if _source != None:
                if getName(build) == _source:
                    performed = True
                    removeDir(build, RELEASE_DIR)
            else:
                removeDir(build, RELEASE_DIR)
        if _source == None:
            deleteDir(_rootDir + PATH_SEP + RELEASE_DIR)


    if _source == None:
        deleteDir(_rootDir)

    if _source != None:
        if not performed:
            fatal("No such source:" + _source);



def removeDir(source, rootdir):
    name = getName(source)
    deleteDir(_rootDir + PATH_SEP + rootdir + PATH_SEP + name)

################################################################################
#
# Retrieve Methods
#
################################################################################
def retrieve():
    global _source
    sources  = getSourceList()

    # Retreive Source
    performed=False
    if len(sources) > 0:
        log ("Retrieving source...")

        mkdir(_rootDir + PATH_SEP + SOURCE_DIR)

        for source in sources:
            if _source != None:
                if getName(source) == _source:
                    performed = True
                    downloadSource(source, SOURCE_DIR)
            else:
                downloadSource(source, SOURCE_DIR)

    # Retreive Patches
    patches = getPatchList()
    if len(patches) > 0:

        log ("Retrieving patches...")

        mkdir(_rootDir + PATH_SEP + PATCH_DIR)

        for patch in patches:
            if _source != None:
                if getName(patch) == _source:
                    performed = True
                    downloadSource(patch, PATCH_DIR)
            else:
                downloadSource(patch, PATCH_DIR)

    if _source != None:
        if not performed:
            fatal("No such patch:" + _source);


################################################################################
#
# Prepare Methods
#
################################################################################

def prepare():
    verbose("Prepare")

    mkdir(_rootDir + PATH_SEP + BUILD_DIR)

    sources = getSourceList()
    performed = False
    if len(sources) > 0:
        log ("Preparing source...")
        for source in sources:
            if _source != None:
                if getName(source) == _source:
                    log_no_newline("Preparing "+getName(source)+": ")
                    performed = True
                    postProcess(source, SOURCE_DIR)
            else:
                log_no_newline("Preparing "+getName(source)+": ")
                postProcess(source, SOURCE_DIR)
        if _source != None:
            if not performed:
                fatal("No such source:" + _source);

    patches = getPatchList()
    if len(patches) > 0:
        log ("Preparing patches...")
        for patch in patches:
            if _source != None:
                if getName(patch) == _source:
                    performed = True
                    log("Preparing "+getName(patch))
                    postProcess(patch, PATCH_DIR)
            else:
                log("Preparing "+getName(patch))
                postProcess(patch, PATCH_DIR)

        if _source != None:
            if not performed:
                fatal("No such patch:" + _source);


def postProcess(item, destination):
    name = getName(item)
    type = getType(item)

    verbose("Post Processing:"+name)

    targetdir = _rootDir + PATH_SEP + destination + PATH_SEP + name

    builddir = _rootDir + PATH_SEP + BUILD_DIR + PATH_SEP + name


    if type == SVN:
       # Do we want to perform an export?

       #extractcommand=SVN_BIN+" export "+ targetdir +" "+ builddir
       # Use -v just now so we can show progress 
       extractcommand=CP_BIN+" -rvf "+ targetdir +" "+ builddir

       runCommand(extractcommand, False)

    else:
        if type == FILE or type == HTTP or type == FTP:

            mkdir(builddir)

            # Look at all the files and see if they need unpacks
            for root, dirs, files in os.walk(targetdir, topdown=False):
                for file in files:
                    command = FILE_BIN+" "+root+PATH_SEP+file

                    result = Popen(command, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                                            stderr=subprocess.PIPE)
                    line = result.stdout.readline()
                    firstline=line
                    while (line != "" ):
                        # process nextline
                        line=result.stdout.readline()

                    result.wait()

                    if result.returncode != 0:
                        fatal("Download (" + name + ") contained unrecognized file type:"+ firstline)


                    extractcommand=""

                    if firstline.find(GZIP_DATA) != -1:
                        extractcommand=TAR_BIN+" -vxzf "+root+PATH_SEP+file+" -C " + builddir

                    if firstline.find(BZIP2_DATA) != -1:
                        extractcommand=TAR_BIN+" -vxjf "+root+PATH_SEP+file+" -C " + builddir

                    if firstline.find(ZIP_DATA) != -1:
                       extractcommand=UNZIP_BIN+" -v "+root+PATH_SEP+file+" -d "+ builddir

                    if firstline.find(TAR_DATA) != -1:
                       extractcommand=TAR_BIN+" -vxf "+root+PATH_SEP+file+" -C "+ builddir

                    if firstline.find(DIFF_FILE) != -1 or firstline.find("text"):
                       extractcommand=CP_BIN+" -r "+root+PATH_SEP+file+" "+ builddir



                    if not extractcommand=="":
                        log_no_newline("Extracting archive:" + file+": " )
                        runCommand(extractcommand, False)
                    else:
                        fatal("Download (" + name + ") contained unsupported file type:"+ firstline)



################################################################################
#
# Patch Methods
#
################################################################################
def patch():

    # Retreive Patches
    patches= getPatchList()
    if len(patches) > 0:
        performed = False
        for patch in patches:
            if _source != None:
                if getName(source) == _source:
                    performed = True
                    applyPatch(patch)
            else:
                applyPatch(patch)

        if _source != None:
            if not performed:
                fatal("No such patch:" + _source);


def applyPatch(patch):
    global _rootDir

    name = getName(patch)
    type = getType(patch)
    source = getValue(patch.getElementsByTagName(SOURCE)[0])
    if (patch.getElementsByTagName(PREFIX).length > 0):
        prefix = getValue(patch.getElementsByTagName(PREFIX)[0])
    else:
        prefix = None

    if (patch.getElementsByTagName(PATH).length > 0):
        path= getValue(patch.getElementsByTagName(PATH)[0])
    else:
        path = None


    basecommand = PATCH_BIN

    if prefix != None:
        basecommand = basecommand + " -p "+prefix

    basecommand = basecommand + " -E -d "+ _rootDir + PATH_SEP + BUILD_DIR + PATH_SEP + source + PATH_SEP

    if path != None:
        basecommand = basecommand + path

    basecommand = basecommand + " < "

    patchSource= _rootDir + PATH_SEP + PATCH_DIR + PATH_SEP + name

    for root, dirs, files in os.walk(patchSource):
    	if '.svn' in dirs:
		dirs.remove('.svn')
        files.sort()	
        for patchName in files:
	                log("Applying patch '" + name + "'("+patchName+") to " + source)
        	        runCommandShowError(basecommand + patchSource + PATH_SEP + patchName)


################################################################################
#
# build Methods
#
################################################################################
def showBuilds():
    builds = getNamesfromBuildList(getBuildList())
    if len(builds) > 0:
        log("Available Builds:")
        for build in builds:
            log(" "+build)
    else:
        log("No builds available")

#
# Given a list of build elements extract the Name values and return as a list
#
def getNamesfromBuildList(list):
    names=[]
    for item in list:
        name = getName(item)
        if name != None:
            names.append(name)
    return names

def build():
    doBuildAction(BUILD)



################################################################################
#
# Release Methods
#
################################################################################
def release():
    log ("Releasing...")
    mkdir(_rootDir + PATH_SEP + RELEASE_DIR)

    builds = getBuildList()

    for build in builds:
        if _source != None:
            if getName(build) == _source:
                mkdir(_rootDir + PATH_SEP + RELEASE_DIR + PATH_SEP + getName(build))
        else:
            mkdir(_rootDir + PATH_SEP + RELEASE_DIR + PATH_SEP + getName(build))
                    
    doBuildAction(RELEASE)



################################################################################
#
# Build Helper Methods
#
################################################################################

def doBuildAction(action):
    config = _baseConfiguration

    if len(getSourceList()) > 0:
        log("Performing "+ action.title() +"...")

        builds = getBuildList()

        performed = False
        for build in builds:
            if _source != None:
                if getName(build) == _source:
                    performed = True
                    performScript(build , action)
            else:
                performScript(build, action)

        if _source != None:
            if not performed:
                fatal("No such build:" + _source);


def performScript(build, scriptName):
    name = getName(build)

    checkDependencies(build)

    verbose("Running "+scriptName+":"+name)

    targets = build.getElementsByTagName(TARGETS)

    if targets.length > 0:
        target = targets[0].getElementsByTagName(scriptName)

        if target.length > 1:
            fatal("More than one build target specified")

        if target.length == 0:
            fatal("No build target specified")

        script = getValue(target[0].getElementsByTagName(SCRIPT)[0])

        script = peformSubstitutionsInScript(build, script)

        debug(script)

        runScript(script)

    else:
        fatal("Build "+name+" has no build targets")


def checkDependencies(build):
    name = getName(build)
    dependencies = build.getElementsByTagName(DEPENDENCY)

    if dependencies > 0:
        for dependency in dependencies :
            sources = dependency.getElementsByTagName(SOURCE)
            if sources.length == 0:
                fatal("No sources specified in dependency block for build:"+name)
            else:
                for source in sources:
                    sourceDependency = getValue(source)
                    if not (sourceDefined(sourceDependency)):
                        fatal("Unable to build "+name+" as specifed dependency("+sourceDependency +") is not available")

def sourceDefined(name):
    for source in getSourceList():
        sourcename = getValue(source.getElementsByTagName(NAME)[0])
        if sourcename == name:
            return True
    return False


def runScript(script):
    (returncode, stdout, stderr) = runCommandWithOutput(script)

    if _debug:
        for line in stdout:            
            debug(line)
        for line in stderr:            
            debug(line)

    if returncode != 0:
        for line in stdout:            
                warn(line)
        for line in stderr:            
                warn(line)

        warn("Script Failed")

        attemptExit(1)
        

################################################################################
#
# XML Helper Methods
#
################################################################################

def loadBaseConfiguration(config):
    log ("Loading configuration:" + config)
    full = minidom.parse(config)
    return full.getElementsByTagName(BUILDER)[0]

#
# Assumes that we have a <node>text</node> element and returns the text value.
#
def getValue(node):
    if node.childNodes.length > 0:
        return node.childNodes[0].data
    else:
        return ""

def getEnvironment():
    env = _baseConfiguration.getElementsByTagName(ENVIRONMENT)
    if env.length > 0:
        return env[0]
    else:
        return None

#
# Returns the value of the NAME element contained in the specified item
#
def getName(item):
    name = item.getElementsByTagName(NAME)
    if name.length > 0:
        return getValue(name[0])
	
#
# Returns the value of the TYPE element contained in the specified item
#
def getType(item):
    type = item.getElementsByTagName(TYPE)	
    if type.length > 0:
        return getValue(type[0])
	    
#
# Returns the value of the URL element contained in the specified item
#
def getURL(item):
    url = item.getElementsByTagName(URL)	
    if url.length > 0:
        return getValue(url[0])

#
# Return the list of sources in this build configuration
# If no sources are available then this is logged as a fatal error.
#
def getSourceList():
    config = _baseConfiguration
    sourceCount = config.getElementsByTagName(SOURCES).length
    if sourceCount > 0:
        return config.getElementsByTagName(SOURCES)[0].getElementsByTagName(SOURCE)
    else:
        fatal("No source elements defined.")
#
# Return the list of patches in this build configuration
#
def getPatchList():
    config = _baseConfiguration
    patchCount = config.getElementsByTagName(PATCHES).length
    if patchCount > 0:
        return config.getElementsByTagName(PATCHES)[0].getElementsByTagName(PATCH)
    else:
        return []

# Returns a list of build elements including any any included build files
# Currently nested build elements are not supported so all builds must be specified via the <include> tag.
#
def getBuildList():
    config = _baseConfiguration

    builds = config.getElementsByTagName(BUILDS)
    buildcount = builds.length

    if buildcount > 0:
        build = builds[0]
        useInclude = build.getElementsByTagName(INCLUDE).length > 0

        # If we are using includes then build a list of all the files we need to include
        if useInclude:
            return  getIncludeList(build)

        else:
           warn("Nested builds not currently supported")
    else:
        fatal("No Builds defined in config")        

#
# Look at all <include> values in the given element and return the list of inlcudes
#
def getIncludeList(build):
    includelist=[]
    for include in build.getElementsByTagName(INCLUDE):
        for item in getIncludeValue(getValue(include)):
            includelist.append(item)

    return includelist

#
# Process in the given include value.
# This is done by performing `ls <include>`
# This means includes such as 'builds/*.config' will match multiple includes and return all entries
#
# Any error in performing the ls is printed and the tool exits (unless ignore errors)
#
def getIncludeValue(include):
    debug("Loading Includes:"+include+" ")    

    command = LS_BIN+" "+include
    (returncode, stdout, stderr) = runCommandWithOutput(command)

    if returncode == 0:
        values=[]
        for line in stdout:
            include = loadIncludeFile(line)
            if not include == None:
                values.append(include)
        return values
    else:
        for line in stderr:
            warn(line)
        attemptExit(1)

#
# Given a file name parse the XML. Any trailing '\n's that the ls command may have added are removed here.
# The file is checked to ensure that it is a <builds> file
# The first <build> element is returned.
#
#  TODO: Allow multiple builds per file.
#
def loadIncludeFile(file):
     buildFile = minidom.parse(file.rstrip('\n'))

     builds = buildFile.getElementsByTagName(BUILDS)

     if builds.length != 1:
        warn("Build Configuration does not contain any <"+BUILDS+"> definitions")
     else:
        buildElements = builds[0].getElementsByTagName(BUILD)
        if not buildElements.length > 0:
            warn("Build Configuration does not contain any <"+BUILD+"> definitions")
        else:
            if buildElements.length > 0:
                build = buildElements[0]
                # getElementsByTagName is recursive so this will pick up the sub element build
                # Only use the first element
                namecount = build.getElementsByTagName(NAME).length
                if namecount > 0:
                    return build
                else:
                    return None

#
# Given the build target and a script substitute $value entries in script for values in
# the Environment
# the Source entries <source><name>
# the build <build><name>
# the release location : _rootDir + PATH_SEP + RELEASE_DIR + PATH_SEP + buildName
#
def peformSubstitutionsInScript(build, script):
    buildName = getName(build)
    sources = getSourceList()

    #Replace Build name
    script = script.replace("$build", buildName)

    #Replace release directory
    releaseDir = _rootDir + PATH_SEP + RELEASE_DIR + PATH_SEP + buildName            
    script = script.replace("$release", releaseDir)

    # Replace Source varables
    for source in sources:
        sourceName = getName(source)

        search = "$"+sourceName

        sourcePath = source.getElementsByTagName(PATH)

        replacement = _rootDir + PATH_SEP + BUILD_DIR + PATH_SEP + sourceName
        if sourcePath.length > 0:
            replacement = replacement + PATH_SEP + getValue(sourcePath[0])

        script = script.replace(search,replacement)

    # Take values from the environment script for replacement
    env = getEnvironment()
    if env != None:
        for item in env.childNodes:
            if item.nodeType == 1:

                search = "$"+item.tagName
                replace = item.childNodes[0].data

                script = script.replace(search,replace)
		
    # Perform keyword substitution replacements
    # Currently only one substitution exists so for simplisity fix it here    
    writeVersionSubstitution = script.find("$writeVersions")
    if writeVersionSubstitution != -1:
    	
	#Extract Filename
        fileNameStart = script.find("(",writeVersionSubstitution)
        fileNameEnd = script.find(")",fileNameStart)	
	fileName= script[fileNameStart+1:fileNameEnd]
	
	substitution = createVersionSubstitution(build, fileName)
		
	script = script.replace("$writeVersions(" + fileName + ")", substitution)		
	

    return script     

################################################################################
#
# Keyword Substitutions
#
################################################################################
   
#
# Use the specified build as to lookup all associated source/patches and write out their details
# to the specified file using shell redirects. redirects are to be used as the absolute filename 
# location may not be known as the name comes in via the release script
#
def createVersionSubstitution(build, filename):
    substitution = ""
    sources = getSourceList();

    dependencies = build.getElementsByTagName(DEPENDENCY)
	
    if dependencies > 0:
        substitution += "\n echo 'Source Version Information:'>> " + filename	
        for dependency in dependencies :
            depSources = dependency.getElementsByTagName(SOURCE)
            # Can assume we have dependencies as we would have failed before now
            for source in depSources:
                sourceDependency = getValue(source)	
		# We can assume source is valid.		
		for s in sources:
		    if sourceDependency == getName(s):		    	
		    	# provide header <source>:<type>:<revision>
			substitution += "\n " + ECHO_BIN + " -n '" + sourceDependency + ":" \
			         + getType(s) + ":' >> " + filename
			substitution += "\n" + getVersionCommand(s) + " >>" + filename
			# Add Source URL to Revisions file
			url = getValue(s.getElementsByTagName(URL)[0])
			substitution += "\n" + ECHO_BIN + " \"URL:" + url + "\" >> "+filename
			# Add Patches applied to this source to revisions file
			substitution += addPatchVersions(s, filename)
						   	    	     
    return substitution
    
#
# Use the specified source as to lookup all associated patches and write their details out the 
# the specified file using shell redirects. redirects are to be used as the absolute filename 
# location may not be known as the name comes in via the release script
#
def addPatchVersions(source, filename):
    substitution = ""
    
    patches = getPatchList()
    
    sourceName = getName(source)
	
    for patch in patches:
    	patchSourceName = getValue(patch.getElementsByTagName(SOURCE)[0])

        if sourceName == patchSourceName:
	    type = getType(patch)
	    substitution += "\n" + ECHO_BIN + " \"\t"+getName(patch)+":"+type + "\" >> "+filename
	    url = getValue(patch.getElementsByTagName(URL)[0])
	    substitution += "\n" + ECHO_BIN + " \"\t\tURL:" + url + "\" >> "+filename
	    if (type == SVN):
		    if (patch.getElementsByTagName(REVISION).length > 0):
                        substitution += "\n" + ECHO_BIN + " \"\t\tREVISION:"+ \
			         getValue(patch.getElementsByTagName(REVISION)[0])  + "\" >> " + filename	    
		    else:			
                        substitution += "\n" + ECHO_BIN + " -n \"\t\tREVISION: \"  >> " + filename
			substitution += "\n" + SVNVERSION_BIN + " " + _rootDir + PATH_SEP + PATCH_DIR + PATH_SEP + getName(patch) + " >> " + filename	    

			
	    if (patch.getElementsByTagName(PREFIX).length > 0):
		substitution += "\n" + ECHO_BIN + " \"\t\tPREFIX: " + \
		         getValue(patch.getElementsByTagName(PREFIX)[0]) + "\" >> " + filename
			 
	    if (patch.getElementsByTagName(PATH).length > 0):
   		substitution += "\n" + ECHO_BIN + " \"\t\tPATH: " + \
		         getValue(patch.getElementsByTagName(PATH)[0]) + "\" >> " + filename 
	    
	    global _rootDir
	    patchSource= _rootDir + PATH_SEP + PATCH_DIR + PATH_SEP + getName(patch)
	
	    #
	    # Include the list of patches files applied
	    #
	    for root, dirs, files in os.walk(patchSource):
    		if '.svn' in dirs:
			dirs.remove('.svn')
	        files.sort()	
        	for patchName in files:
        	        substitution += "\n" + ECHO_BIN + " \"\t\tFILE: " + patchName + "\" >> " + filename 
 
    
	   
		
    if (substitution != ""):
	return "\n" + ECHO_BIN + " \"\tPatches applied to " + sourceName + ":\" >> " + filename + substitution 
    else:
    	return "\n" + ECHO_BIN + " \"\tNo Patches\" >> " + filename 

      
#
# Given a source entry return the command that will provide the current version 
# of that source.
# i.e. svn source : svnversion <path to source>
#     http source : echo <URL>
#      
def getVersionCommand(source):
    global _rootDir
    type = getType(source)	
	
    versionCommand=""
    
    if type == SVN:
        versionCommand=SVNVERSION_BIN+" "+_rootDir + PATH_SEP + SOURCE_DIR + PATH_SEP + getName(source)
    else:
        if type == FILE or type == HTTP or type == FTP:
	    versionCommand = ECHO_BIN +" " + getURL(source)
	
	
    return versionCommand	

################################################################################
#
# Download Helper Methods
#
################################################################################

#
# Download the item specified in source to the given destintation
#
def downloadSource(source, destination):
    name = getName(source)
    type = getType(source)
    url =  getValue(source.getElementsByTagName(URL)[0])
    log ( "Retrieving "+ name + "("+ type +")")

    targetdir=_rootDir + PATH_SEP + destination + PATH_SEP + name

    command = ""

    mkdir(targetdir)

    if (os.listdir(targetdir)==[]):

        # Setup command for a fresh checkout
        if (type == SVN):
            command = SVN_BIN+" co "+url+" "+targetdir
            if (source.getElementsByTagName(REVISION).length > 0):
                revision =  getValue(source.getElementsByTagName(REVISION)[0])
		command = SVN_BIN+" co -r"+revision+" "+url+" "+targetdir
        else:
            if (type == HTTP):
                command = WGET_BIN+" --no-directories -P "+targetdir+" "+url
            else:
                if (type == FILE):
                    if url.startswith(HTTP):
                        command = WGET_BIN+" -P "+targetdir+" "+url
                    else:
                        if url.startswith(FTP):
                            command = WGET_BIN+" -P "+targetdir+" "+url
                        else:
                            command = CP_BIN+" -R "+url+" "+targetdir
    else:
        warn("Target directory(" + targetdir + ") is not empty please ensure contents are valid or run 'clean "+name+"'")

    verbose("Executing:"+command)
    log_no_newline("Retrieving "+source.nodeName+": ")

    if (type == FILE):
        runCommand(command, True)
    else:
        runCommand(command, False)

################################################################################
#
# Command Helper Methods
#
################################################################################

#
# Run command and print out last 20 lines of data on error
#
def runCommandShowError(command):
    last20 = runCommand(command, False)
    if last20 != None:
        lines=last20[0]
        lines=lines + 1
        current = 1
        while current != lines:
            log (last20[current])
            current = current + 1
        attemptExit(1)

#
# Runs the given command if showOutput is true then stdout/stderr is shown on screen
# other wise the last 20 lines of output is gathered:
#
# As command runs progress is shown
#
# return array [0] = no of elements in array. Array is fixed size 21 elements but not all are used. FIXME: this is poor
#
# TODO: Current mechanism for limiting to 20 lines is poor, potential to replace usages of this
#       method with runCommandWithOutput below
#
def runCommand(command, showOutput):
    debug("Running Command:"+command)
    try:
        if showOutput:
            # Process that shows the output
            result = Popen(command, shell=True)
        else:
            # consume the output ourselves
            result = Popen(command, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE)

            index=0
            last20=[""] * 21
            line = result.stdout.readline()
            while (line != "" ):
                logWaiting()

                #Record last 20 lines of output
                index = index + 1
                if index == 20:
                    index = 1

                last20[index]=line

                # process nextline
                line = result.stdout.readline()

            #
            # If we didn't get any standard or fill our buffers then fill the end of the buffer with any stderr
            #
            if index == 0 | index < 15 :
                line = result.stderr.readline()

                if index != 0:
                    index = index + 1
                    if line != "":
                        last20[index]="STDERR"
                reset = index
                while (line != "" ):
                    logWaiting()

                    #Record last 20 lines of output
                    index = index + 1
                    if index == 20:
                        index = reset

                    last20[index]=line

                    # process nextline
                    line = result.stderr.readline()

        result.wait()

        if result.returncode == 0:
            logWaitingDone()
        else:
            logWaitingFailed("Failed")
	    attemptExit(1)
            if not showOutput:
                last20[0]=index
                return last20

        return None

    except IOError:
        logWaitingFailed ("Error running command.")
        attemptExit(1)

#
# Runs the given command if showOutput is true then stdout/stderr is shown on screen
# Stdout and stderr is gathered up and returned with error code.
#
# return (result.returncode, stdout, stderr)
#
# As command runs progress is shown
#
def runCommandWithOutput(command):

    result = Popen(command, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE)


    # Retrieve STDOUT
    stdout=[]
    line = result.stdout.readline()
    while (line != "" ):
        logWaiting()

        stdout.append(line)

        # process nextline
        line = result.stdout.readline()

    line = result.stderr.readline()

    # Retrieve STDERR
    stderr=[]
    while (line != "" ):
        stderr.append(line)

        # process nextline
        line = result.stderr.readline()

    result.wait()

    logWaitingClear()

    return (result.returncode, stdout, stderr)


################################################################################
#
# OS Helper Methods
#
################################################################################

#
# Check _ignoreErrors value and exit if false
#
def attemptExit(code):
    if not _ignoreErrors:
        sys.exit(code)
    else:
	print ("Ignoring Errors")

#
# Check that the required binaries are present for this tool.
# Only checks the minimum set.
# Logs warning if archive tools are missing
#
def checkSystemRequirements():
    exists = checkExists(SVN_BIN)
    exists = exists & checkExists(WGET_BIN)
    exists = exists & checkExists(CP_BIN)
    exists = exists & checkExists(PATCH_BIN)
    exists = exists & checkExists(FILE_BIN)

    if not checkExists(TAR_BIN):
        warn("Unable to process tar files as tar binary does not exist:" + TAR_BIN)
    if not checkExists(BZIP2_BIN):
        warn("Unable to process bzip2 files as bzip2 binary does not exist:" + BZIP2_BIN)
    if not checkExists(UNZIP_BIN):
        warn("Unable to process zip files as unzip binary does not exist:" + UNZIP_BIN)

    if not exists:
        sys.exit(1)

#
# Helper that checks for files existence
#
def checkExists(command):
    debug_no_newline("Checking for "+command+":")
    command = LS_BIN+" "+command

    result = Popen(command, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                            stderr=subprocess.PIPE)
    line = result.stdout.readline()

    while (line != "" ):
        # process nextline
        line = result.stdout.readline()

    result.wait()

    if result.returncode == 0:
        debug("OK")
        return True
    else:
        debug("Missing")
        warn("Missing dependancy:"+command)
        return False


# Delete everything reachable from the directory named in 'top',
# assuming there are no symbolic links.
#
# If an attempt to delete '/' is  performed this is logged as a fatal error
#
def deleteDir(top):
    if top == '/':
        fatal("Exiting as attempt to delete '/' occured.")
    else:
        if (os.path.exists(top)):
            log_no_newline("Removing:"+top+". ")
            for root, dirs, files in os.walk(top, topdown=False):
                logWaiting()
                for name in files:
                    os.remove(os.path.join(root, name))
                for name in dirs:
                    logWaiting()
                    os.rmdir(os.path.join(root, name))

            logWaiting()
            os.rmdir(top)

            logWaitingDone()

def mkdir(dir):
    if not os.path.exists(dir):
        os.mkdir(dir)


################################################################################
#
# Logging Helper Methods
#
################################################################################

#
# Provide a spinning -/|\
#
def logWaiting():
    global _charIndex, _waitingChars

    _charIndex = (_charIndex + 1) % len(_waitingChars)

    log_no_newline('\b')
    log_no_newline(_waitingChars[_charIndex])

#
# Clear the logWaiting symbol and end the line with ' Done'
#
def logWaitingDone():
    log_no_newline('\b')
    log(" Done")

#
# Clear the logWaiting symbol
#
def logWaitingClear():
        log_no_newline('\b')

#
# Clear the logWaiting symbol and end line with messsage
#
def logWaitingFailed(message):
    log_no_newline('\b')
    log(" "+message)

def debug(string):
    if _debug:
        log(string)

def verbose(string):
    if _verbose:
        log(string)

def log (string):
    if _log:
        print string

def warn (string):
    print string

def fatal(string):
    print string
    attemptExit(1)

def log_no_newline (string):
    if _log:
        sys.stdout.write(string)
        sys.stdout.flush()

def verbose_no_newline (string):
    if _verbose:
        sys.stdout.write(string)
        sys.stdout.flush()

def debug_no_newline (string):
    if _debug:
        sys.stdout.write(string)
        sys.stdout.flush()

if __name__ == "__main__":
    main()