Make rpms use a local and remote artifact cachestaging/jonathan/ngi-rpm-caching

This will upload RPMs to kbas if they're built, unless kbas-upload is
manually set to something that doesn't include "rpm"

1 files changed, 124 insertions, 14 deletions

diff --git a/ybd/rpm.py b/ybd/rpm.py
index a899deb..542c023 100644
--- a/ybd/rpm.py
+++ b/ybd/rpm.py
@@ -12,13 +12,15 @@ import repos
 
 
 # Because rpm is otherwise totally broken
+# NOTE: _build_name_fmt would ordinary be %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm
+# but we are pulling them out of a cache with a different naming scheme now.
 #
 common_rpm_args = (
   '--dbpath=/var/lib/rpm '
   '--define "_rpmconfigdir /usr/lib/rpm" '
   '--define "_rpmlock_path /var/lib/rpm/.rpm.lock" '
   '--define "_fileattrsdir /usr/lib/rpm/fileattrs" '
-  '--define "_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm" '
+  '--define "_build_name_fmt %%{NAME}.rpm" '
   '--define "_rpmfilename %{_build_name_fmt}" '
   '--define "_tmppath /tmp" '
   '--define "_unpackaged_files_terminate_build 0" '
@@ -218,6 +220,88 @@ def extract_defines(dn):
     return ''.join(strings)
 
 
+def get_all_package_names(system, dn):
+    if 'rpm-metadata' not in dn:
+        app.log(dn, "Tried to get package names for definition withour rpm-metadata!")
+        sys.exit(1)
+    if 'packages' not in dn['rpm-metadata']:
+        app.log(dn, "Tried to get pacakge names for definition with no packages in rpm-metadata!")
+        sys.exit(1)
+    package_names = []
+    for package in dn['rpm-metadata']['packages']:
+        package_names.append(expand_macro(system, dn, package['name']))
+    return package_names
+
+
+def get_remote_rpm(dn, pkgfilename):
+    # Defaults to allowing rpms to be fetched from kbas.
+    # Override kbas-upload to prevent this.
+    if 'rpm' not in app.config.get('kbas-upload', 'rpm'):
+        return False
+    try:
+        app.log(dn, 'Try downloading', pkgfilename)
+        url = "{}get/{}".format(app.config['kbas-url'], pkgfilename)
+        response = requests.get(url=url, stream=True)
+    except:
+        app.config.pop('kbas-url')
+        app.log(dn, 'WARNING: remote artifact server is not working')
+        return False
+
+    if response.status_code == 200:
+        try:
+            tempfile.tempdir = app.config['tmp']
+            tmpdir = tempfile.mkdtemp()
+            cachefile = os.path.join(tmpdir, pkgfilename)
+            with open(cachefile, 'wb') as f:
+                f.write(response.content)
+            cache_dst = os.path.join(app.config['artifacts'], pkgfilename)
+            os.rename(cachefile, cache_dst)
+            return True
+        except Exception as e:
+            app.log(dn, "WARNING: Failed to download {}: {}".format(pkgfilename, e))
+    return False
+
+
+def get_cache_pkgfilename(pkgname, dn):
+    return "{}.{}.rpm".format(pkgname, cache_key(dn))
+
+
+def all_rpms_cached(system, dn):
+    for pkgname in get_all_package_names(system, dn):
+        pkgfn = get_cache_pkgfilename(pkgname, dn)
+        cached_path = os.path.join(app.config['artifacts'], pkgfn)
+        if (not os.path.exists(cached_path)
+                and not get_remote_rpm(dn, pkgfn)):
+            return False
+    return True
+
+
+def compose_rpm(dn, userdata):
+    if not all_rpms_cached(userdata['system'], dn):
+        if not package_one_rpm(dn, userdata):
+            return False
+
+        if not cache_generated_rpms(userdata['system'], dn):
+            return False
+
+    return True
+
+
+def cache_generated_rpms(system, dn):
+    rpms_dir = os.path.join(system['sandbox'], 'RPMS')
+    for pkg in get_all_package_names(system, dn):
+        pkgfile = "{}.rpm".format(pkg)
+        pkgpath = os.path.join(rpms_dir, pkgfile)
+        cachepath = os.path.join(app.config['artifacts'], get_cache_pkgfilename(pkg, dn))
+        if not os.path.exists(pkgpath):
+            app.log(dn, "Can't extract rpms, {} is missing!".format(pkgpath))
+            return False
+        os.rename(pkgpath, cachepath)
+
+    return True
+        
+
+
 def package_one_rpm(dn, userdata):
 
     system = userdata['system']
@@ -285,6 +369,43 @@ def package_one_rpm(dn, userdata):
     return True
 
 
+def rpm_deployment_filename(system, dn, rpmpath):
+    # Reads rpm's headers to construct its filename.
+    env_vars = sandbox.env_vars_for_build(system)
+    command = ('rpm {} {} --eval="%{name}-%{version}-%{release}.%{arch}.rpm"'
+               .format(common_rpm_args, extract_defines(dn)))
+    _, output, _ = sandbox.run_sandboxed(system, command,
+                                         env_vars, exit_on_error=False)
+    return output
+
+
+def deploy_rpm(dn, userdata):
+    for pkgname in get_all_package_names(userdata['system'], dn):
+        pkgfn = get_cache_pkgfilename(pkgname, dn)
+        cached_path = os.path.join(app.config['artifacts'], pkgfn)
+        if not os.path.exists(cached_path):
+            app.log(dn, "WARNING: Missing cached file {}".format(cached_path))
+            return False
+        dstdir = os.path.join(app.config['deployment'],
+                              'RPMs', cache_key(userdata['system']))
+        dstfilename = rpm_deployment_filename(userdata['system'],
+                                              dn, cached_path)
+        dstpath = os.path.join(dstdir, dstfilename)
+        if not os.path.exists(dstdir):
+            os.makedirs(dstdir)
+        shutil.copyfile(cached_path, dstpath)
+        return True
+
+
+def deploy_rpms(system, whitelist=None):
+    deploy_results = foreach_def(system, deploy_rpm, {'system': system},
+                                 whitelist=whitelist)
+    errors = any(not t[1] for t in deploy_results)
+    if errors:
+        app.log(system, "ERROR: Failed to deploy all RPMs!")
+        sys.exit(1)
+
+
 # package_rpms
 # @system: The system to package rpms for
 # @time: The number of whole seconds since the epoch
@@ -329,25 +450,14 @@ def package_rpms(system, time, whitelist=None):
 
         # Package each rpm in order of build dependency
         package_results = foreach_def(
-            system, package_one_rpm, {'system': system, 'time': time},
+            system, compose_rpm, {'system': system, 'time': time},
             whitelist=whitelist)
         errors = any(not t[1] for t in package_results)
         if errors:
             log(system, 'ERROR: Failed to successfully generate all rpms!')
             sys.exit(1)
 
-        # Move the resulting RPMS directory into the deployment area
-        rpm_destdir = os.path.join(app.config['deployment'], 'RPMs',
-                                   cache_key(system))
-        if not os.path.exists(rpm_destdir):
-            os.makedirs(rpm_destdir)
-        for entry in os.listdir(rpmdir):
-            srcfile = os.path.join(rpmdir, entry)
-            dstfile = os.path.join(rpm_destdir, entry)
-            # I could not move files if they already exist,
-            # but the RPMs have already been produced at this point.
-            os.rename(os.path.join(rpmdir, entry),
-                      os.path.join(rpm_destdir, entry))
+        deploy_rpms(system, whitelist)
 
 
 #