WIP add binary input supportWIP-ps-binary-import

This adds support for taking a url as input for a binary chunk,
instead of cloning from a git repo.

In definitions, these new fields are supported:

url: specifies the place to look for the binary
sha256: if present, specifies the expected sha256 for the binary
md5: if present, specifies the expected md5 for the binary

In config, these new fields are supported:
auth-user: (optional) user name for https authentication of url
auth-password: (optional) password for https authentication of url

The auth-user and auth-password should normally be exported as
environment variables YBD_auth_user and YBD_auth_password

4 files changed, 35 insertions, 2 deletions

diff --git a/ybd/assembly.py b/ybd/assembly.py
index c818c8c..1b5c571 100644
--- a/ybd/assembly.py
+++ b/ybd/assembly.py
@@ -171,6 +171,8 @@ def run_build(dn):
     if dn.get('repo'):
         repos.checkout(dn)
         dn['SOURCE_DATE_EPOCH'] = repos.source_date_epoch(dn['checkout'])
+    elif dn.get('url'):
+        repos.get_binary(dn)
 
     get_build_commands(dn)
     env_vars = sandbox.env_vars_for_build(dn)
diff --git a/ybd/cache.py b/ybd/cache.py
index e32fc2a..d8d4df4 100644
--- a/ybd/cache.py
+++ b/ybd/cache.py
@@ -24,7 +24,8 @@ from subprocess import call
 import hashlib
 import app
 from repos import get_repo_url, get_tree
-from utils import hardlink_all_files, make_deterministic_tar_archive
+from utils import hardlink_all_files, make_deterministic_gztar_archive
+from utils import make_deterministic_tar_archive
 from utils import md5, sha256, set_mtime_recursively, sorted_ls
 import tempfile
 import yaml
diff --git a/ybd/config/defaults.conf b/ybd/config/defaults.conf
index 597eed7..e998d8d 100644
--- a/ybd/config/defaults.conf
+++ b/ybd/config/defaults.conf
@@ -11,6 +11,7 @@ morph-fields: ['arch',
                'devices',
                'kind',
                'max-jobs',
+               'md5',
                'morph',
                'name',
                'orig_ref',
@@ -20,12 +21,15 @@ morph-fields: ['arch',
                'strata',
                'ref',
                'repo',
+               'rpm-metadata',
                'sha',
+               'sha256',
                'subsystems',
                'submodules',
                'systems',
                'system-integration',
-               'unpetrify-ref']
+               'unpetrify-ref',
+               'url']
 
 # Mapping from arch to cpu defaults
 cpus:
diff --git a/ybd/repos.py b/ybd/repos.py
index 2b32713..47fa2c4 100644
--- a/ybd/repos.py
+++ b/ybd/repos.py
@@ -328,6 +328,32 @@ def checkout_submodules(dn):
             app.log(dn, "Git submodules problem", exit=True)
 
 
+def get_binary(dn):
+    with app.chdir(dn['checkout']), open(os.devnull, "w") as fnull:
+
+        url = dn['url']
+        if app.config.get('auth-user') and app.config.get('auth-password'):
+            url = re.sub('^http[s]?://', '', url)
+            auth = app.config['auth-user'] + ':' + app.config['auth-password']
+            url = 'https://' + auth + '@' + url
+
+        if call(['wget', url], stdout=fnull, stderr=fnull):
+            app.log(dn, 'running wget on url:', dn['url'], exit=True)
+
+        filename = os.path.basename(dn['url'])
+        exit = (app.config.get('check-definitions') == 'exit')
+        import hashlib
+        checksum = utils.sha256(filename)
+        if checksum != dn.get('sha256'):
+            app.log(dn, 'sha256 mismatch: %s |' % dn.get('sha256'), checksum,
+                    exit=exit)
+        checksum = utils.md5(filename)
+        if checksum != dn.get('md5'):
+            app.log(dn, 'md5 mismatch:  %s |' % dn.get('md5'), checksum,
+                    exit=exit)
+        app.log(dn, 'Downloaded binary input', filename)
+
+
 @contextlib.contextmanager
 def explore(ref):
     try: