diff options
author | Ben Hutchings <ben.hutchings@codethink.co.uk> | 2020-09-09 22:21:40 +0100 |
---|---|---|
committer | Ben Hutchings <ben.hutchings@codethink.co.uk> | 2020-09-09 23:45:16 +0100 |
commit | ef33d6bf1d92f377b83a6c8b4cbec680255ee473 (patch) | |
tree | a83119f795ebe8df6e1a5b31c93d31b7425977bb /scripts | |
parent | a089e6b3767033ee548ce6718d9c362f7f3f58c9 (diff) | |
download | lorries-ef33d6bf1d92f377b83a6c8b4cbec680255ee473.tar.gz |
Add script to check all repository/file URLsbwh/check-urls
For each repository or file (or branch, for Bazaar), try the cheapest
possible operation to verify that it is present.
Related to #4.
Diffstat (limited to 'scripts')
-rwxr-xr-x | scripts/check-lorry-urls | 145 |
1 files changed, 145 insertions, 0 deletions
diff --git a/scripts/check-lorry-urls b/scripts/check-lorry-urls new file mode 100755 index 0000000..94d659c --- /dev/null +++ b/scripts/check-lorry-urls @@ -0,0 +1,145 @@ +#!/usr/bin/python3 + +import glob +import http.client +import json +import os +import ssl +import subprocess +import sys +import time +import urllib.request +import yaml + + +def check_file(filename): + is_ok = True + repo_name = None + + def diagnostic(file, level, msg): + if repo_name is None: + print('%s: %s: %s' % (level, filename, msg), + file=file) + else: + print('%s: %s: %s: %s' % (level, filename, repo_name, msg), + file=file) + + def error(msg): + nonlocal is_ok + is_ok = False + diagnostic(sys.stderr, 'E', msg) + + def info(msg): + diagnostic(sys.stdout, 'I', msg) + + def try_command(args, msg_prefix=''): + result = subprocess.run( + args, + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, + encoding='utf-8', errors='replace') + if result.returncode != 0: + error(msg_prefix + result.stderr.rstrip()) + else: + info(msg_prefix + 'OK') + + with open(filename) as f: + try: + obj = yaml.safe_load(f) + except yaml.YAMLError: + f.seek(0) + obj = json.load(f) + + for repo_name, upstream_def in obj.items(): + upstream_type = upstream_def['type'] + upstream_url = upstream_def.get('url') # optional for bzr + check_certificates = upstream_def.get('check-certificates', True) + + # Try the cheapest possible operation to verify that the + # repository or file is present + + if upstream_type == 'bzr': + cert_options = [] + if not check_certificates: + cert_options.append('-Ossl.cert_reqs=none') + if upstream_url is None: + branch_urls = upstream_def['branches'].items() + else: + branch_urls = [('trunk', upstream_url)] + for branch_name, branch_url in branch_urls: + try_command(['bzr', 'info', '--quiet', *cert_options, + branch_url], + msg_prefix=('%s: ' % branch_name)) + + elif upstream_type == 'cvs': + try_command(['cvs', '-d', upstream_url, '-Q', 'rls', + upstream_def['module']]) + + elif upstream_type == 'git': + if check_certificates: + os.environ.pop('GIT_SSL_NO_VERIFY', None) + else: + os.environ['GIT_SSL_NO_VERIFY'] = 'true' + try_command(['git', 'ls-remote', '--symref', + upstream_url, 'HEAD']) + + elif upstream_type == 'hg': + # The hg client doesn't appear to support running any + # kind of info/check command without a local clone. + # Send a Mercurial capabilities request directly. + tls_context = ssl.create_default_context() + if not check_certificates: + tls_context.check_hostname = False + tls_context.verify_mode = ssl.CERT_NONE + try: + with urllib.request.urlopen( + urllib.request.Request( + upstream_url + '?cmd=capabilities', + headers={ + 'Accept': 'application/mercurial-0.1', + 'User-Agent': 'mercurial/proto-1.0' + }), + context=tls_context) as result: + if result.info().get('Content-Type') \ + != 'application/mercurial-0.1': + error('no Mercurial server support detected') + except (urllib.error.URLError, http.client.HTTPException) as e: + error(str(e)) + else: + info('OK') + + elif upstream_type == 'svn': + try_command(['svn', 'info', '--non-interactive', upstream_url]) + + elif upstream_type in ['gzip', 'tarball', 'zip']: + try: + with urllib.request.urlopen( + urllib.request.Request(upstream_url, + method='HEAD')) as result: + pass + except (urllib.error.URLError, http.client.HTTPException) as e: + error(str(e)) + else: + info('OK') + + # Throttle requests + time.sleep(0.1) + + return is_ok + + +def main(): + all_ok = True + + if len(sys.argv) > 1: + filenames = sys.argv[1:] + else: + filenames = sorted(glob.glob('*-lorries/*.lorry')) + for filename in filenames: + if not check_file(filename): + all_ok = False + + sys.exit(0 if all_ok else 1) + + +if __name__ == '__main__': + main() |