import os import re import sys from glob import glob from collections import Mapping from cache import cache_key, get_cache, md5 from app import log, timer import assembly import time import app import sandbox import shutil import yaml import repos import requests import tempfile import utils import subprocess # Common parameters required for rpm. # NOTE: _build_name_fmt would ordinary be # %{NAME}-%{VERSION}-%{RELEASE}.%{ARCH}.rpm # but we are pulling them out of a cache with a different naming scheme now. common_rpm_args = ( '--dbpath=/var/lib/rpm ' '--define "_rpmconfigdir /usr/lib/rpm" ' '--define "_rpmlock_path /var/lib/rpm/.rpm.lock" ' '--define "_fileattrsdir /usr/lib/rpm/fileattrs" ' '--define "_build_name_fmt %{NAME}.rpm" ' '--define "_rpmfilename %{_build_name_fmt}" ' '--define "_tmppath /tmp" ' '--define "_unpackaged_files_terminate_build 0" ' '--define "__spec_clean_cmd echo" ') # foreach_def # @dn: The target definition or name # @callback: The callback to call # @user_data: Something to be passed to @callback along with the definition # # Iterates over each chunk which is to be included # in the target @dn in order of dependency, the passed # @dn definition will be the last one called. # def foreach_def(dn, callback, user_data, traversed=None, whitelist=None): if traversed is None: traversed = {} if type(dn) is not dict: dn = app.defs.get(dn) # if we can't calculate cache key, we can't create this component if cache_key(dn, mode='rpm') is False: if 'tried' not in dn: log(dn, 'No cache_key, so skipping compose') dn['tried'] = True yield False, None # if dn is already cached, we're done if not get_cache(dn): log('SMTH', '%s is not cached, try again sucker !' % dn['name'], exit=True) systems = dn.get('systems', []) for system in systems: for s in system.get('subsystems', []): subsystem = app.defs.get(s['path']) for res in foreach_def( subsystem, callback, user_data, traversed, whitelist): yield res[0], res[1] for res in foreach_def( system['path'], callback, user_data, traversed, whitelist): yield res[0], res[1] contents = dn.get('contents', []) for it in contents: item = app.defs.get(it) if traversed.get(item.get('name')): continue if item.get('build-mode', 'staging') != 'bootstrap': if not traversed.get(item.get('name')): for res in foreach_def( item, callback, user_data, traversed, whitelist): yield res[0], res[1] result = True if whitelist is None or dn.get('name') in whitelist: result = callback(dn, user_data) traversed[dn.get('name')] = True yield True, result def expand_macro(system, dn, text): # Macros are identified by use of % if r'%' not in text: return text defines = ['--define "{} {}"'.format(ent['key'], ent['value']) for ent in dn['rpm-metadata'].get('defines', [])] defines.append('--define "{} {}"'.format('name', dn['name'])) command = ('rpm ' + common_rpm_args + ' '.join(defines) + ' --eval=%s' % text) env_vars = sandbox.env_vars_for_build(system) # Force subprocess to return output, we will log after the call. _, output, _ = sandbox.run_sandboxed( system, command, env_vars, run_logged=False, print_command=False) with open(system['log'], 'a') as logfile: logfile.write(output) return output.strip() def is_main_package(system, dn, package): return expand_macro(system, dn, package) == dn['name'] def generate_spec(dn, stage_dir, metafile, output, name, system): with open(metafile, "r") as metafile_f: metadata = yaml.safe_load(metafile_f) with open(output, "w") as output_f: rpm_metadata = dn['rpm-metadata'] package_list = rpm_metadata.get('packages', []) description = 'No Description' if metadata.get('description') is not None: description = metadata.get('description') # Write out the package header first output_f.write('Name: %s\n' % name) output_f.write('Summary: %s\n' % description) output_f.write('License: %s\n' % 'Undetermined') output_f.write('Prefix: %s\n' % dn.get('prefix', '/usr')) list_fields = ('Requires', 'Conflicts', 'Provides', 'Obsoletes') # Add all the main package Provides: for package in package_list: if is_main_package(system, dn, package['name']): vendor = next((d['vendor'] for d in (package, rpm_metadata) if 'vendor' in d), None) if vendor: output_f.write('Vendor: %s\n' % vendor) version = next((d['version'] for d in (package, rpm_metadata) if 'version' in d), None) if version: output_f.write('Version: %s\n' % version) else: app.log(dn, "Main package has no version!") return False release = next((d['release'] for d in (package, rpm_metadata) if 'release' in d), None) if release: output_f.write('Release: {}git{}\n' .format(release, dn['sha'])) else: app.log(dn, "Main package has no release!") return False epoch = next((d['epoch'] for d in (package, rpm_metadata) if 'epoch' in d), None) if epoch: output_f.write('Epoch: %s\n' % epoch) for field in list_fields: field_l = field.lower() if field_l in package: for item in package[field_l]: output_f.write('%s: %s\n' % (field, item)) output_f.write('\n') output_f.write('%description\n') output_f.write('%s\n' % description) output_f.write('\n') for package in package_list: if not is_main_package(system, dn, package['name']): # Sub-Package header output_f.write('%%package -n %s\n' % package['name']) output_f.write('Summary: %s\n' % description) if 'vendor' in package: output_f.write('Vendor: %s\n' % package['vendor']) if 'version' in package: output_f.write('Version: %s\n' % package['version']) if 'release' in package: output_f.write('Release: {}git{}\n' .format(package['release'], dn['sha'])) if 'epoch' in package: output_f.write('Epoch: %s\n' % package['epoch']) for field in list_fields: field_l = field.lower() if field_l in package: for item in package[field_l]: output_f.write('%s: %s\n' % (field, item)) output_f.write('\n') output_f.write('%%description -n %s\n' % package['name']) output_f.write('%s\n' % description) output_f.write('\n') for cmd in ['pre', 'preun', 'post', 'postun']: if cmd in package: if 'interpreter' in package[cmd]: interp_str = ' -p %s' % package[cmd]['interpreter'] else: # To be explicit, this might default to /bin/sh, # but that's more verbose than is strictly needed. interp_str = '' output_f.write('%%%s -n %s%s\n' % (cmd, package['name'], interp_str)) if 'body' in package[cmd]: # Expecting it to be a single multi-line command, # since running as separate commands discards env. output_f.write(package[cmd]['body'] + '\n') output_f.write('\n') # Skip writing files if nonexistent/empty if type(package.get('files')) is not list: continue metafile = package.get('metafile', '').lstrip(os.sep) if metafile: metafile = " -f " + metafile # Sub-Package files if package['name'] == name: output_f.write('%%files%s\n' % metafile) else: output_f.write('%%files -n %s%s\n' % (package['name'], metafile)) for filename in package['files']: output_f.write( filename.replace('%manifest /', '%manifest ') + "\n") output_f.write('\n') return True def extract_defines(dn): meta = dn.get('rpm-metadata') if meta is None or isinstance(meta, list): return '' defines = meta.get('defines') if defines is None: return '' strings = [] for define in defines: key = define.get('key') value = define.get('value') strings.append(' --define "{} {}"'.format(key, value)) return ''.join(strings) def sanitise_rpm_filepath(rpm_filepath): return re.sub(r'^/?baserock/', '', rpm_filepath) def extract_metafiles(system, dn, instdir, metadir): packages = dn['rpm-metadata'].get('packages', []) for package in packages: metafile = package.get('metafile', '').lstrip(os.sep) metafile = expand_macro(system, dn, metafile) if metafile: try: shutil.move(os.path.join(instdir, metafile), metadir) except IOError: filepath = sanitise_rpm_filepath(metafile) if not os.path.exists(os.path.join(metadir, filepath)): log(dn, 'ERROR: Failed to extract metafile:', metafile) sys.exit(1) def extract_manifests(system, dn, instdir, metadir): packages = dn['rpm-metadata'].get('packages', []) manifests = {line.split()[1].lstrip(os.sep) for package in packages for line in package.get('files', []) if line.startswith('%manifest')} for manifest in manifests: manifest = expand_macro(system, dn, manifest) try: shutil.move(os.path.join(instdir, manifest), metadir) except IOError: filepath = sanitise_rpm_filepath(manifest) if not os.path.exists(os.path.join(metadir, filepath)): log(dn, 'WARNING: Failed to extract manifest:', manifest) def get_package_names_with_fields(system, dn, fields): # Ignores subpackages without files sections as those aren't generated. if 'rpm-metadata' not in dn: app.log(dn, "Tried to get package names for definition " "without rpm-metadata!") sys.exit(1) if 'packages' not in dn['rpm-metadata']: app.log(dn, "Tried to get package names for definition " "with no packages in rpm-metadata!") sys.exit(1) if any(all(key in package for key in ('path', 'files')) for package in dn['rpm-metadata']['packages']): app.log(dn, "ERROR: Cannot mix 'path' (prebuilts) and 'files' " "(generated) in rpm-metadata") sys.exit(1) return {expand_macro(system, dn, package['name']) for package in dn['rpm-metadata']['packages'] if any(field in package for field in fields)} def get_generated_package_names(system, dn): return get_package_names_with_fields(system, dn, ['files', 'path']) def get_remote_rpm(dn, pkgfilename): # Defaults to allowing rpms to be fetched from kbas. # Override kbas-upload to prevent this. if 'rpm' not in app.config.get('kbas-upload', 'rpm'): return False if 'kbas-url' not in app.config: return False try: app.log(dn, 'Try downloading', pkgfilename) url = "{}get/{}".format(app.config['kbas-url'], pkgfilename) response = requests.get(url=url, stream=True) except Exception as e: app.config.pop('kbas-url') app.log(dn, "WARNING: Failure to call artifact server: ", e) return False if response.status_code == 200: try: tempfile.tempdir = app.config['tmp'] tmpdir = tempfile.mkdtemp() cachefile = os.path.join(tmpdir, pkgfilename) with open(cachefile, 'wb') as f: f.write(response.content) cache_dst = os.path.join(app.config['artifacts'], pkgfilename) os.rename(cachefile, cache_dst) size = os.path.getsize(cache_dst) size = re.sub("(\d)(?=(\d{3})+(?!\d))", r"\1,", "%d" % size) app.log(dn, 'Cached %s bytes as' % size, pkgfilename) return True except Exception as e: app.log(dn, "WARNING: Failed to download {}: {}" .format(pkgfilename, e)) return False def get_cache_pkgfilename(pkgname, dn): return "{}.{}.rpm".format(pkgname, cache_key(dn, mode='rpm')) def all_rpms_cached(system, dn): for pkgname in get_generated_package_names(system, dn): pkgfn = get_cache_pkgfilename(pkgname, dn) cached_path = os.path.join(app.config['artifacts'], pkgfn) if (not os.path.exists(cached_path) and not get_remote_rpm(dn, pkgfn)): return False return True def stage_prebuilts(dn, system): for package in dn['rpm-metadata']['packages']: if 'path' not in package: continue path = package['path'].lstrip(os.sep) basedir = os.path.join(system['sandbox'], '%s.inst' % dn['name']) cache_path = os.path.join( system['sandbox'], 'RPMS', '%s.rpm' % package['name']) rpm_glob = sorted(glob(os.path.join(basedir, path))) if not rpm_glob: metadir = os.path.join(system['sandbox'], '%s.meta' % dn['name']) rpm_glob = sorted(glob(os.path.join( metadir, sanitise_rpm_filepath(path)))) if not rpm_glob: log(dn, 'Failed to find prebuilt rpm matching', package['path']) sys.exit(1) if len(rpm_glob) > 1: exit = app.config.get('check-definitions') == 'exit' log(dn, 'WARNING: glob pattern %s matches multiple paths' % package['path'], exit=exit) log(dn, 'WARNING: Taking %s from glob match' % rpm_glob[0]) os.rename(rpm_glob[0], cache_path) def compose_rpm(dn, userdata): # a chunk lacking in rpm-metadata is not an error. # It means this should be skipped without doing anything. if 'rpm-metadata' not in dn: return True if not all_rpms_cached(userdata['system'], dn): if not package_one_rpm(dn, userdata): return False if not cache_generated_rpms(userdata['system'], dn): return False return True def upload_generated_rpm(dn, cachefile): filename = os.path.basename(cachefile) url = app.config['kbas-url'] + 'upload' params = {"filename": filename, "password": app.config['kbas-password'], "checksum": md5(cachefile)} with open(cachefile, 'rb') as f: try: response = requests.post(url=url, data=params, files={"file": f}) if response.status_code == 201: app.log(dn, 'Uploaded %s to kbas' % filename) return if response.status_code == 777: app.log(dn, 'Reproduced %s at' % md5(cachefile), filename) app.config['reproduced'].append([md5(cachefile), filename]) return if response.status_code == 405: app.log(dn, 'Artifact server already has', filename) return app.log(dn, 'Artifact server problem:', response.status_code) except: pass app.log(dn, 'Failed to upload', filename) def cache_generated_rpms(system, dn): rpms_dir = os.path.join(system['sandbox'], 'RPMS') for pkg in get_generated_package_names(system, dn): pkgfile = "{}.rpm".format(pkg) pkgpath = os.path.join(rpms_dir, pkgfile) cachepath = os.path.join(app.config['artifacts'], get_cache_pkgfilename(pkg, dn)) if not os.path.exists(pkgpath): app.log(dn, "Can't extract rpms, {} is missing!".format(pkgpath)) return False os.rename(pkgpath, cachepath) # Upload the cached rpm, if applicable if (app.config.get('kbas-password', 'insecure') != 'insecure' and 'kbas-url' in app.config and 'rpm' in app.config.get('kbas-upload', 'rpm')): with app.timer(dn, 'Upload {}'.format(pkgfile)): upload_generated_rpm(dn, cachepath) return True def package_one_rpm(dn, userdata): system = userdata['system'] kind = dn.get('kind') name = dn.get('name') if kind == 'chunk' or kind is None: with timer(name): subdir = '%s.inst' % name fulldir = os.path.join(system['sandbox'], subdir) metadir = os.path.join(system['sandbox'], '%s.meta' % name) baserockdir = os.path.join(fulldir, 'baserock') # Install the chunk we're gonna package under subdir sandbox.install(system, dn, subdir) # Move the baserock directory out of the way, # we don't package the metadata shutil.move(baserockdir, metadir) stage_prebuilts(dn, system) # Generate the specfile in the metadir, note that we use # the metadata for the given package from the system metadata # directory, not the metadata for the specific chunk artifact # XXX Right now the chunk's individual metadata is richer, it # includes the desciption, change this to use the system metadata # for that chunk later !!! metafile = os.path.join(metadir, '%s.meta' % name) specfile = os.path.join(metadir, '%s.spec' % name) success = True if not any(to_build for to_build in dn['rpm-metadata']['packages'] if 'files' in to_build): log(dn, 'WARNING: Nothing to package in %s, skipping' % name) return success if generate_spec(dn, fulldir, metafile, specfile, name, system): defines = extract_defines(dn) extract_metafiles(system, dn, fulldir, metadir) extract_manifests(system, dn, fulldir, metadir) # XXX Now we gonna run rpmbuild in the sandbox !!! command = ('rpmbuild ' + common_rpm_args + ' --buildroot=/%s.inst' % name + ' --define "_builddir /%s.meta"' % name + ' --define "_rpmdir /RPMS"' + defines + ' --target %s' % app.config['cpu'] + ' -bb /%s.meta/%s.spec' % (name, name)) env_vars = sandbox.env_vars_for_build(system) # Keep building all the rpms we can even if one fails, # we'd rather see everything that failed in a log at once. success, _, _ = sandbox.run_sandboxed( system, command, env_vars, exit_on_error=False) else: success = False app.log(dn, "Failed to generate spec, not generating rpm") if success: app.log(dn, "Removing sandbox dir", fulldir, verbose=True) shutil.rmtree(fulldir) shutil.rmtree(metadir) else: app.log(dn, "Failed rpm build sandbox dir:", fulldir) return success return True def rpm_deployment_filename(dn, rpmpath): # Reads rpm's headers to construct its filename. out = "" qf = '--queryformat=%{name}-%{version}-%{release}.%{arch}.rpm' try: with open(os.devnull, 'w') as fnull: out = subprocess.check_output(['rpm', '-q', '-p', rpmpath, qf], stderr=fnull) except subprocess.CalledProcessError: out = "" if not out: app.log(dn, "ERROR: Failed to generate name from %s headers" % rpmpath) sys.exit(1) return out def deploy_rpm(dn, userdata): # Skip this if there's no rpm-metadata if 'rpm-metadata' not in dn: return True for pkgname in get_generated_package_names(userdata['system'], dn): pkgfn = get_cache_pkgfilename(pkgname, dn) cached_path = os.path.join(app.config['artifacts'], pkgfn) if not os.path.exists(cached_path): app.log(dn, "WARNING: Missing cached file {}".format(cached_path)) return False dstfilename = rpm_deployment_filename(dn, cached_path) dstpath = os.path.join(userdata['dstdir'], dstfilename) shutil.copyfile(cached_path, dstpath) app.log(dn, "Copied %s to deployment area" % dstfilename, verbose=True) return True def deploy_rpms(system, whitelist=None): dstdir = os.path.join(app.config['deployment'], 'RPMs', cache_key(system)) utils.makedirs(dstdir, exist_ok=True) userdata = {'system': system, 'dstdir': dstdir} deploy_results = foreach_def(system, deploy_rpm, userdata, whitelist=whitelist) errors = any(not t[1] for t in deploy_results) if errors: app.log(system, "ERROR: Failed to deploy all RPMs!") sys.exit(1) app.log(system, "Finished deploying RPMs to %s!" % dstdir) # package_rpms # @system: The system to package rpms for # @whitelist: A whitelist of chunk names to package rpms for # # This function will first stage the given @system, which # must have an rpm installation, and then it will use the # metadata in the system's baserock directory to package # each individual chunk, by staging those chunks one by one # and packaging them in a chroot. # # Care will be taken to build the rpms in the order of their # dependencies, this should allow rpm to infer package dependencies # correctly # def package_rpms(system, whitelist=None): if type(system) is not dict: system = app.defs.get(system) with sandbox.setup(system): assembly.install_contents(system, assembly.compose) if system.get('kind') != "system": assembly.install_dependencies(system, assembly.compose) # Fail now if missing `rpm` or `rpmbuild` env_vars = sandbox.env_vars_for_build(system) test_cmds = ['rpm --version', 'rpmbuild --version'] for cmd in test_cmds: success, _, _ = sandbox.run_sandboxed( system, cmd, env_vars, exit_on_error=False) if not success: log('RPM-BUILD', 'To generate rpms, appropriate rpm and ' 'rpmbuild binaries are required', exit=True) # First initialize the db rpmdb_path = os.path.join(system['sandbox'], 'var', 'lib', 'rpm') rpmdir = os.path.join(system['sandbox'], 'RPMS') command = 'rpm ' + common_rpm_args + ' --initdb' # os.path.exists turned out to be unreliable, so ensuring the # directories exist is done via catching any EEXIST errors. utils.makedirs(rpmdb_path, exist_ok=True) utils.makedirs(rpmdir, exist_ok=True) sandbox.run_sandboxed(system, command, env_vars) # Package each rpm in order of build dependency package_results = foreach_def( system, compose_rpm, {'system': system}, whitelist=whitelist) errors = any(not t[1] for t in package_results) if errors: log(system, 'ERROR: Failed to successfully generate all rpms!') sys.exit(1) deploy_rpms(system, whitelist)