summaryrefslogtreecommitdiff
path: root/scripts
diff options
context:
space:
mode:
authorBen Hutchings <ben.hutchings@codethink.co.uk>2020-09-09 22:21:40 +0100
committerBen Hutchings <ben.hutchings@codethink.co.uk>2020-09-09 23:45:16 +0100
commitef33d6bf1d92f377b83a6c8b4cbec680255ee473 (patch)
treea83119f795ebe8df6e1a5b31c93d31b7425977bb /scripts
parenta089e6b3767033ee548ce6718d9c362f7f3f58c9 (diff)
downloadlorries-ef33d6bf1d92f377b83a6c8b4cbec680255ee473.tar.gz
Add script to check all repository/file URLsbwh/check-urls
For each repository or file (or branch, for Bazaar), try the cheapest possible operation to verify that it is present. Related to #4.
Diffstat (limited to 'scripts')
-rwxr-xr-xscripts/check-lorry-urls145
1 files changed, 145 insertions, 0 deletions
diff --git a/scripts/check-lorry-urls b/scripts/check-lorry-urls
new file mode 100755
index 0000000..94d659c
--- /dev/null
+++ b/scripts/check-lorry-urls
@@ -0,0 +1,145 @@
+#!/usr/bin/python3
+
+import glob
+import http.client
+import json
+import os
+import ssl
+import subprocess
+import sys
+import time
+import urllib.request
+import yaml
+
+
+def check_file(filename):
+ is_ok = True
+ repo_name = None
+
+ def diagnostic(file, level, msg):
+ if repo_name is None:
+ print('%s: %s: %s' % (level, filename, msg),
+ file=file)
+ else:
+ print('%s: %s: %s: %s' % (level, filename, repo_name, msg),
+ file=file)
+
+ def error(msg):
+ nonlocal is_ok
+ is_ok = False
+ diagnostic(sys.stderr, 'E', msg)
+
+ def info(msg):
+ diagnostic(sys.stdout, 'I', msg)
+
+ def try_command(args, msg_prefix=''):
+ result = subprocess.run(
+ args,
+ stdout=subprocess.DEVNULL, stderr=subprocess.PIPE,
+ encoding='utf-8', errors='replace')
+ if result.returncode != 0:
+ error(msg_prefix + result.stderr.rstrip())
+ else:
+ info(msg_prefix + 'OK')
+
+ with open(filename) as f:
+ try:
+ obj = yaml.safe_load(f)
+ except yaml.YAMLError:
+ f.seek(0)
+ obj = json.load(f)
+
+ for repo_name, upstream_def in obj.items():
+ upstream_type = upstream_def['type']
+ upstream_url = upstream_def.get('url') # optional for bzr
+ check_certificates = upstream_def.get('check-certificates', True)
+
+ # Try the cheapest possible operation to verify that the
+ # repository or file is present
+
+ if upstream_type == 'bzr':
+ cert_options = []
+ if not check_certificates:
+ cert_options.append('-Ossl.cert_reqs=none')
+ if upstream_url is None:
+ branch_urls = upstream_def['branches'].items()
+ else:
+ branch_urls = [('trunk', upstream_url)]
+ for branch_name, branch_url in branch_urls:
+ try_command(['bzr', 'info', '--quiet', *cert_options,
+ branch_url],
+ msg_prefix=('%s: ' % branch_name))
+
+ elif upstream_type == 'cvs':
+ try_command(['cvs', '-d', upstream_url, '-Q', 'rls',
+ upstream_def['module']])
+
+ elif upstream_type == 'git':
+ if check_certificates:
+ os.environ.pop('GIT_SSL_NO_VERIFY', None)
+ else:
+ os.environ['GIT_SSL_NO_VERIFY'] = 'true'
+ try_command(['git', 'ls-remote', '--symref',
+ upstream_url, 'HEAD'])
+
+ elif upstream_type == 'hg':
+ # The hg client doesn't appear to support running any
+ # kind of info/check command without a local clone.
+ # Send a Mercurial capabilities request directly.
+ tls_context = ssl.create_default_context()
+ if not check_certificates:
+ tls_context.check_hostname = False
+ tls_context.verify_mode = ssl.CERT_NONE
+ try:
+ with urllib.request.urlopen(
+ urllib.request.Request(
+ upstream_url + '?cmd=capabilities',
+ headers={
+ 'Accept': 'application/mercurial-0.1',
+ 'User-Agent': 'mercurial/proto-1.0'
+ }),
+ context=tls_context) as result:
+ if result.info().get('Content-Type') \
+ != 'application/mercurial-0.1':
+ error('no Mercurial server support detected')
+ except (urllib.error.URLError, http.client.HTTPException) as e:
+ error(str(e))
+ else:
+ info('OK')
+
+ elif upstream_type == 'svn':
+ try_command(['svn', 'info', '--non-interactive', upstream_url])
+
+ elif upstream_type in ['gzip', 'tarball', 'zip']:
+ try:
+ with urllib.request.urlopen(
+ urllib.request.Request(upstream_url,
+ method='HEAD')) as result:
+ pass
+ except (urllib.error.URLError, http.client.HTTPException) as e:
+ error(str(e))
+ else:
+ info('OK')
+
+ # Throttle requests
+ time.sleep(0.1)
+
+ return is_ok
+
+
+def main():
+ all_ok = True
+
+ if len(sys.argv) > 1:
+ filenames = sys.argv[1:]
+ else:
+ filenames = sorted(glob.glob('*-lorries/*.lorry'))
+ for filename in filenames:
+ if not check_file(filename):
+ all_ok = False
+
+ sys.exit(0 if all_ok else 1)
+
+
+if __name__ == '__main__':
+ main()