diff options
author | Angelos Evripiotis <jevripiotis@bloomberg.net> | 2019-02-26 15:33:44 +0000 |
---|---|---|
committer | Angelos Evripiotis <jevripiotis@bloomberg.net> | 2019-02-26 17:51:53 +0000 |
commit | fe76ac9780a0951b9d85100bbd0c06d997a8e0ad (patch) | |
tree | a34a7df7f526533182b9a58f3e95340ba57394d8 | |
parent | 5a1a5814e77152bf2195f917147d5dc46f7201d7 (diff) | |
download | buildstream-fe76ac9780a0951b9d85100bbd0c06d997a8e0ad.tar.gz |
element: use json for artifact metadata for speedaevri/json_artifact_meta
Change public.yaml files in artifacts to JSON instead of YAML.
Also bump the BST_CORE_ARTIFACT_VERSION, as we don't provide
compatability with artifacts created by an older version of BuildStream.
Note that we could have made a similar change that would not have
required a version bump - JSON is a subset of YAML. We deliberately
choose not to in order to force folks to take the feature of increased
speed for the next major version of BuildStream.
This means that the files might be less human-readable, but they are
faster and simpler for machines to read.
This 'YAML for human readability, JSON for simplicity' split is observed
by the authors of the YAML spec:
https://yaml.org/spec/1.2/spec.html#id2759572
This change saves 3-4 seconds off a 12-13 second invocation of bst shell
like this:
time bst shell --build --use-buildtree never target.bst echo hello
After the change, parsing artifact info doesn't take long enough to
really show up in a py-spy profile of the invocation.
My use-case has around 400 elements in it's build plan, as calculated
by:
bst show --deps build --format BUILDDEP target.bst | grep BUILDDEP | wc -l
-rw-r--r-- | NEWS | 3 | ||||
-rw-r--r-- | buildstream/_versions.py | 2 | ||||
-rw-r--r-- | buildstream/element.py | 8 |
3 files changed, 10 insertions, 3 deletions
@@ -11,6 +11,9 @@ buildstream 1.3.1 to this, `--tar` is no longer a flag, it is a mutually incompatible option to `--directory`. For example, `bst artifact checkout foo.bst --tar foo.tar.gz`. + o The core artifact version was increased, due to switching the format of + artifact metadata from YAML to JSON. This was for greatly improved loading + speed. o Added `bst artifact log` subcommand for viewing build logs. diff --git a/buildstream/_versions.py b/buildstream/_versions.py index 56fd95223..6b35b924f 100644 --- a/buildstream/_versions.py +++ b/buildstream/_versions.py @@ -33,4 +33,4 @@ BST_FORMAT_VERSION = 23 # or if buildstream was changed in a way which can cause # the same cache key to produce something that is no longer # the same. -BST_CORE_ARTIFACT_VERSION = 8 +BST_CORE_ARTIFACT_VERSION = 9 diff --git a/buildstream/element.py b/buildstream/element.py index 365931e27..80217858f 100644 --- a/buildstream/element.py +++ b/buildstream/element.py @@ -85,6 +85,8 @@ import tempfile import shutil import string +import ujson + from . import _yaml from ._variables import Variables from ._versions import BST_CORE_ARTIFACT_VERSION @@ -1738,7 +1740,8 @@ class Element(Plugin): shutil.copyfile(log_filename, self._build_log_path) # Store public data - _yaml.dump(_yaml.node_sanitize(self.__dynamic_public), os.path.join(metadir, 'public.yaml')) + with open(os.path.join(metadir, 'public.json'), 'w') as datafile: + ujson.dump(_yaml.node_sanitize(self.__dynamic_public), datafile) # Store result build_result_dict = {"success": self.__build_result[0], "description": self.__build_result[1]} @@ -2834,7 +2837,8 @@ class Element(Plugin): # Load the public data from the artifact artifact_base, _ = self.__extract() metadir = os.path.join(artifact_base, 'meta') - self.__dynamic_public = _yaml.load(os.path.join(metadir, 'public.yaml')) + with open(os.path.join(metadir, 'public.json')) as datafile: + self.__dynamic_public = ujson.load(datafile) def __load_build_result(self, keystrength): self.__assert_cached(keystrength=keystrength) |