diff options
author | Jim MacArthur <jim.macarthur@codethink.co.uk> | 2018-11-09 15:18:51 +0000 |
---|---|---|
committer | Jim MacArthur <jim.macarthur@codethink.co.uk> | 2018-11-27 15:32:19 +0000 |
commit | d5b5538de087ce560601dc980142a86220c38b0c (patch) | |
tree | 6a5e78b108e6b2623bfe0426c8d121ddabbd4566 | |
parent | 127d332f33e0badbba3526548d9089640ee95ed1 (diff) | |
download | buildstream-d5b5538de087ce560601dc980142a86220c38b0c.tar.gz |
artifactcache: Move ArtifactCacheSpec code into CASRemoteSpec.
There is nothing in ArtifactCacheSpec that's actually specific to
artifacts, so I've made it a CAS class so we can use the same
spec for remote execution.
-rw-r--r-- | buildstream/_artifactcache/artifactcache.py | 47 | ||||
-rw-r--r-- | buildstream/_artifactcache/cascache.py | 48 |
2 files changed, 50 insertions, 45 deletions
diff --git a/buildstream/_artifactcache/artifactcache.py b/buildstream/_artifactcache/artifactcache.py index 7080f2151..2121a432d 100644 --- a/buildstream/_artifactcache/artifactcache.py +++ b/buildstream/_artifactcache/artifactcache.py @@ -21,7 +21,6 @@ import multiprocessing import os import signal import string -from collections import namedtuple from collections.abc import Mapping from ..types import _KeyStrength @@ -31,7 +30,7 @@ from .. import _signals from .. import utils from .. import _yaml -from .cascache import CASCache, CASRemote +from .cascache import CASCache, CASRemote, CASRemoteSpec CACHE_SIZE_FILE = "cache_size" @@ -45,48 +44,8 @@ CACHE_SIZE_FILE = "cache_size" # push (bool): Whether we should attempt to push artifacts to this cache, # in addition to pulling from it. # -class ArtifactCacheSpec(namedtuple('ArtifactCacheSpec', 'url push server_cert client_key client_cert')): - - # _new_from_config_node - # - # Creates an ArtifactCacheSpec() from a YAML loaded node - # - @staticmethod - def _new_from_config_node(spec_node, basedir=None): - _yaml.node_validate(spec_node, ['url', 'push', 'server-cert', 'client-key', 'client-cert']) - url = _yaml.node_get(spec_node, str, 'url') - push = _yaml.node_get(spec_node, bool, 'push', default_value=False) - if not url: - provenance = _yaml.node_get_provenance(spec_node, 'url') - raise LoadError(LoadErrorReason.INVALID_DATA, - "{}: empty artifact cache URL".format(provenance)) - - server_cert = _yaml.node_get(spec_node, str, 'server-cert', default_value=None) - if server_cert and basedir: - server_cert = os.path.join(basedir, server_cert) - - client_key = _yaml.node_get(spec_node, str, 'client-key', default_value=None) - if client_key and basedir: - client_key = os.path.join(basedir, client_key) - - client_cert = _yaml.node_get(spec_node, str, 'client-cert', default_value=None) - if client_cert and basedir: - client_cert = os.path.join(basedir, client_cert) - - if client_key and not client_cert: - provenance = _yaml.node_get_provenance(spec_node, 'client-key') - raise LoadError(LoadErrorReason.INVALID_DATA, - "{}: 'client-key' was specified without 'client-cert'".format(provenance)) - - if client_cert and not client_key: - provenance = _yaml.node_get_provenance(spec_node, 'client-cert') - raise LoadError(LoadErrorReason.INVALID_DATA, - "{}: 'client-cert' was specified without 'client-key'".format(provenance)) - - return ArtifactCacheSpec(url, push, server_cert, client_key, client_cert) - - -ArtifactCacheSpec.__new__.__defaults__ = (None, None, None) +class ArtifactCacheSpec(CASRemoteSpec): + pass # An ArtifactCache manages artifacts. diff --git a/buildstream/_artifactcache/cascache.py b/buildstream/_artifactcache/cascache.py index 04c26edfa..315aa6afa 100644 --- a/buildstream/_artifactcache/cascache.py +++ b/buildstream/_artifactcache/cascache.py @@ -17,6 +17,7 @@ # Authors: # Jürg Billeter <juerg.billeter@codethink.co.uk> +from collections import namedtuple import hashlib import itertools import io @@ -34,7 +35,8 @@ from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remo from .._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc from .. import utils -from .._exceptions import CASError +from .._exceptions import CASError, LoadError, LoadErrorReason +from .. import _yaml # The default limit for gRPC messages is 4 MiB. @@ -42,6 +44,50 @@ from .._exceptions import CASError _MAX_PAYLOAD_BYTES = 1024 * 1024 +class CASRemoteSpec(namedtuple('CASRemoteSpec', 'url push server_cert client_key client_cert')): + + # _new_from_config_node + # + # Creates an CASRemoteSpec() from a YAML loaded node + # + @staticmethod + def _new_from_config_node(spec_node, basedir=None): + _yaml.node_validate(spec_node, ['url', 'push', 'server-cert', 'client-key', 'client-cert']) + url = _yaml.node_get(spec_node, str, 'url') + push = _yaml.node_get(spec_node, bool, 'push', default_value=False) + if not url: + provenance = _yaml.node_get_provenance(spec_node, 'url') + raise LoadError(LoadErrorReason.INVALID_DATA, + "{}: empty artifact cache URL".format(provenance)) + + server_cert = _yaml.node_get(spec_node, str, 'server-cert', default_value=None) + if server_cert and basedir: + server_cert = os.path.join(basedir, server_cert) + + client_key = _yaml.node_get(spec_node, str, 'client-key', default_value=None) + if client_key and basedir: + client_key = os.path.join(basedir, client_key) + + client_cert = _yaml.node_get(spec_node, str, 'client-cert', default_value=None) + if client_cert and basedir: + client_cert = os.path.join(basedir, client_cert) + + if client_key and not client_cert: + provenance = _yaml.node_get_provenance(spec_node, 'client-key') + raise LoadError(LoadErrorReason.INVALID_DATA, + "{}: 'client-key' was specified without 'client-cert'".format(provenance)) + + if client_cert and not client_key: + provenance = _yaml.node_get_provenance(spec_node, 'client-cert') + raise LoadError(LoadErrorReason.INVALID_DATA, + "{}: 'client-cert' was specified without 'client-key'".format(provenance)) + + return CASRemoteSpec(url, push, server_cert, client_key, client_cert) + + +CASRemoteSpec.__new__.__defaults__ = (None, None, None) + + # A CASCache manages a CAS repository as specified in the Remote Execution API. # # Args: |