summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorChandan Singh <chandan@chandansingh.net>2019-11-15 08:26:20 +0000
committerChandan Singh <chandan@chandansingh.net>2019-11-15 08:26:20 +0000
commit0eac8421e3f65833774ed1d1476ed8ceb9fd20e4 (patch)
treefd042e16517eedf7e607c8b2a6b105df9b83a2f6
parent10a5b9ffdea2977d01edd28e9a21c3bb0ecb0e12 (diff)
parent6130ea6767f91f40e84c83b038f36d3d984fe3c7 (diff)
downloadbuildstream-0eac8421e3f65833774ed1d1476ed8ceb9fd20e4.tar.gz
Merge branch 'chandan/black' into 'master'
Use Black to format code See merge request BuildStream/buildstream!1703
-rw-r--r--.gitlab-ci.yml5
-rw-r--r--.pylintrc14
-rw-r--r--doc/source/conf.py79
-rw-r--r--doc/source/hacking/coding_guidelines.rst6
-rw-r--r--doc/source/hacking/using_the_testsuite.rst36
-rw-r--r--pyproject.toml18
-rw-r--r--requirements/dev-requirements.in1
-rw-r--r--requirements/dev-requirements.txt1
-rw-r--r--setup.cfg5
-rw-r--r--src/buildstream/__init__.py4
-rw-r--r--src/buildstream/__main__.py3
-rw-r--r--src/buildstream/_artifact.py41
-rw-r--r--src/buildstream/_artifactcache.py63
-rw-r--r--src/buildstream/_artifactelement.py6
-rw-r--r--src/buildstream/_basecache.py36
-rw-r--r--src/buildstream/_cachekey.py2
-rw-r--r--src/buildstream/_cas/cascache.py159
-rw-r--r--src/buildstream/_cas/casremote.py30
-rw-r--r--src/buildstream/_cas/casserver.py140
-rw-r--r--src/buildstream/_context.py176
-rw-r--r--src/buildstream/_elementfactory.py15
-rw-r--r--src/buildstream/_exceptions.py9
-rw-r--r--src/buildstream/_frontend/app.py456
-rw-r--r--src/buildstream/_frontend/cli.py943
-rw-r--r--src/buildstream/_frontend/complete.py71
-rw-r--r--src/buildstream/_frontend/linuxapp.py7
-rw-r--r--src/buildstream/_frontend/profile.py3
-rw-r--r--src/buildstream/_frontend/status.py158
-rw-r--r--src/buildstream/_frontend/widget.py344
-rw-r--r--src/buildstream/_gitsourcebase.py425
-rw-r--r--src/buildstream/_includes.py60
-rw-r--r--src/buildstream/_loader/loader.py198
-rw-r--r--src/buildstream/_loader/metaelement.py20
-rw-r--r--src/buildstream/_loader/metasource.py2
-rw-r--r--src/buildstream/_message.py76
-rw-r--r--src/buildstream/_messenger.py47
-rw-r--r--src/buildstream/_options/option.py14
-rw-r--r--src/buildstream/_options/optionarch.py10
-rw-r--r--src/buildstream/_options/optionbool.py15
-rw-r--r--src/buildstream/_options/optioneltmask.py4
-rw-r--r--src/buildstream/_options/optionenum.py28
-rw-r--r--src/buildstream/_options/optionflags.py32
-rw-r--r--src/buildstream/_options/optionos.py3
-rw-r--r--src/buildstream/_options/optionpool.py50
-rw-r--r--src/buildstream/_pipeline.py70
-rw-r--r--src/buildstream/_platform/darwin.py6
-rw-r--r--src/buildstream/_platform/fallback.py10
-rw-r--r--src/buildstream/_platform/linux.py38
-rw-r--r--src/buildstream/_platform/platform.py69
-rw-r--r--src/buildstream/_platform/win32.py3
-rw-r--r--src/buildstream/_plugincontext.py140
-rw-r--r--src/buildstream/_profile.py48
-rw-r--r--src/buildstream/_project.py368
-rw-r--r--src/buildstream/_projectrefs.py17
-rw-r--r--src/buildstream/_remote.py66
-rw-r--r--src/buildstream/_scheduler/jobs/elementjob.py12
-rw-r--r--src/buildstream/_scheduler/jobs/job.py145
-rw-r--r--src/buildstream/_scheduler/jobs/jobpickler.py14
-rw-r--r--src/buildstream/_scheduler/queues/buildqueue.py13
-rw-r--r--src/buildstream/_scheduler/queues/queue.py54
-rw-r--r--src/buildstream/_scheduler/resources.py13
-rw-r--r--src/buildstream/_scheduler/scheduler.py83
-rw-r--r--src/buildstream/_signals.py20
-rw-r--r--src/buildstream/_site.py14
-rw-r--r--src/buildstream/_sourcecache.py40
-rw-r--r--src/buildstream/_sourcefactory.py11
-rw-r--r--src/buildstream/_state.py13
-rw-r--r--src/buildstream/_stream.py538
-rw-r--r--src/buildstream/_version.py133
-rw-r--r--src/buildstream/_workspaces.py99
-rw-r--r--src/buildstream/buildelement.py59
-rw-r--r--src/buildstream/element.py601
-rw-r--r--src/buildstream/plugin.py118
-rw-r--r--src/buildstream/plugins/elements/autotools.py3
-rw-r--r--src/buildstream/plugins/elements/compose.py37
-rw-r--r--src/buildstream/plugins/elements/filter.py59
-rw-r--r--src/buildstream/plugins/elements/import.py28
-rw-r--r--src/buildstream/plugins/elements/junction.py10
-rw-r--r--src/buildstream/plugins/elements/manual.py3
-rw-r--r--src/buildstream/plugins/elements/pip.py3
-rw-r--r--src/buildstream/plugins/elements/script.py12
-rw-r--r--src/buildstream/plugins/elements/stack.py4
-rw-r--r--src/buildstream/plugins/sources/_downloadablefilesource.py61
-rw-r--r--src/buildstream/plugins/sources/bzr.py109
-rw-r--r--src/buildstream/plugins/sources/deb.py6
-rw-r--r--src/buildstream/plugins/sources/local.py8
-rw-r--r--src/buildstream/plugins/sources/patch.py12
-rw-r--r--src/buildstream/plugins/sources/pip.py106
-rw-r--r--src/buildstream/plugins/sources/remote.py11
-rw-r--r--src/buildstream/plugins/sources/tar.py45
-rw-r--r--src/buildstream/plugins/sources/workspace.py12
-rw-r--r--src/buildstream/plugins/sources/zip.py14
-rw-r--r--src/buildstream/sandbox/_config.py11
-rw-r--r--src/buildstream/sandbox/_mount.py19
-rw-r--r--src/buildstream/sandbox/_mounter.py48
-rw-r--r--src/buildstream/sandbox/_sandboxbuildbox.py73
-rw-r--r--src/buildstream/sandbox/_sandboxbwrap.py118
-rw-r--r--src/buildstream/sandbox/_sandboxchroot.py60
-rw-r--r--src/buildstream/sandbox/_sandboxdummy.py11
-rw-r--r--src/buildstream/sandbox/_sandboxreapi.py47
-rw-r--r--src/buildstream/sandbox/_sandboxremote.py215
-rw-r--r--src/buildstream/sandbox/sandbox.py120
-rw-r--r--src/buildstream/scriptelement.py72
-rw-r--r--src/buildstream/source.py169
-rw-r--r--src/buildstream/storage/_casbaseddirectory.py111
-rw-r--r--src/buildstream/storage/_filebaseddirectory.py64
-rw-r--r--src/buildstream/storage/directory.py18
-rw-r--r--src/buildstream/testing/__init__.py9
-rw-r--r--src/buildstream/testing/_fixtures.py1
-rw-r--r--src/buildstream/testing/_sourcetests/build_checkout.py36
-rw-r--r--src/buildstream/testing/_sourcetests/fetch.py57
-rw-r--r--src/buildstream/testing/_sourcetests/mirror.py318
-rw-r--r--src/buildstream/testing/_sourcetests/source_determinism.py75
-rw-r--r--src/buildstream/testing/_sourcetests/track.py245
-rw-r--r--src/buildstream/testing/_sourcetests/track_cross_junction.py134
-rw-r--r--src/buildstream/testing/_sourcetests/utils.py15
-rw-r--r--src/buildstream/testing/_sourcetests/workspace.py85
-rw-r--r--src/buildstream/testing/_utils/junction.py41
-rw-r--r--src/buildstream/testing/_utils/site.py43
-rw-r--r--src/buildstream/testing/integration.py22
-rw-r--r--src/buildstream/testing/repo.py7
-rw-r--r--src/buildstream/testing/runcli.py282
-rw-r--r--src/buildstream/types.py2
-rw-r--r--src/buildstream/utils.py265
-rw-r--r--tests/artifactcache/artifactservice.py6
-rw-r--r--tests/artifactcache/capabilities.py25
-rw-r--r--tests/artifactcache/config.py186
-rw-r--r--tests/artifactcache/expiry.py259
-rw-r--r--tests/artifactcache/junctions.py153
-rw-r--r--tests/artifactcache/pull.py84
-rw-r--r--tests/artifactcache/push.py105
-rw-r--r--tests/cachekey/cachekey.py121
-rwxr-xr-xtests/cachekey/update.py29
-rwxr-xr-xtests/conftest.py76
-rw-r--r--tests/elements/filter.py285
-rw-r--r--tests/elements/filter/basic/element_plugins/dynamic.py6
-rw-r--r--tests/examples/autotools.py50
-rw-r--r--tests/examples/developing.py61
-rw-r--r--tests/examples/first-project.py14
-rw-r--r--tests/examples/flatpak-autotools.py43
-rw-r--r--tests/examples/integration-commands.py24
-rw-r--r--tests/examples/junctions.py36
-rw-r--r--tests/examples/running-commands.py26
-rw-r--r--tests/external_plugins.py14
-rw-r--r--tests/format/assertion.py49
-rw-r--r--tests/format/dependencies.py164
-rw-r--r--tests/format/include.py265
-rw-r--r--tests/format/include_composition.py101
-rw-r--r--tests/format/invalid_keys.py20
-rw-r--r--tests/format/junctions.py301
-rw-r--r--tests/format/listdirectiveerrors.py32
-rw-r--r--tests/format/optionarch.py88
-rw-r--r--tests/format/optionbool.py121
-rw-r--r--tests/format/optioneltmask.py82
-rw-r--r--tests/format/optionenum.py128
-rw-r--r--tests/format/optionexports.py41
-rw-r--r--tests/format/optionflags.py148
-rw-r--r--tests/format/optionos.py53
-rw-r--r--tests/format/optionoverrides.py17
-rw-r--r--tests/format/options.py281
-rw-r--r--tests/format/project.py161
-rw-r--r--tests/format/project/plugin-no-load-ref/plugins/noloadref.py1
-rw-r--r--tests/format/project/plugin-preflight-error/errorplugin/preflighterror.py4
-rw-r--r--tests/format/projectoverrides.py17
-rw-r--r--tests/format/userconfig.py6
-rw-r--r--tests/format/variables.py189
-rw-r--r--tests/frontend/__init__.py6
-rw-r--r--tests/frontend/artifact_delete.py140
-rw-r--r--tests/frontend/artifact_list_contents.py79
-rw-r--r--tests/frontend/artifact_log.py33
-rw-r--r--tests/frontend/artifact_show.py80
-rw-r--r--tests/frontend/buildcheckout.py671
-rw-r--r--tests/frontend/completions.py374
-rw-r--r--tests/frontend/compose_splits.py22
-rw-r--r--tests/frontend/configurable_warnings.py45
-rw-r--r--tests/frontend/configuredwarning/plugins/corewarn.py3
-rw-r--r--tests/frontend/consistencyerror/plugins/consistencybug.py1
-rw-r--r--tests/frontend/consistencyerror/plugins/consistencyerror.py4
-rw-r--r--tests/frontend/cross_junction_workspace.py60
-rw-r--r--tests/frontend/fetch.py116
-rw-r--r--tests/frontend/help.py23
-rw-r--r--tests/frontend/init.py108
-rw-r--r--tests/frontend/large_directory.py25
-rw-r--r--tests/frontend/logging.py81
-rw-r--r--tests/frontend/main.py8
-rw-r--r--tests/frontend/mirror.py414
-rw-r--r--tests/frontend/order.py79
-rw-r--r--tests/frontend/overlaps.py43
-rw-r--r--tests/frontend/progress.py82
-rw-r--r--tests/frontend/project/sources/fetch_source.py17
-rw-r--r--tests/frontend/pull.py340
-rw-r--r--tests/frontend/push.py483
-rw-r--r--tests/frontend/rebuild.py13
-rw-r--r--tests/frontend/remote-caches.py46
-rw-r--r--tests/frontend/show.py501
-rw-r--r--tests/frontend/source_checkout.py189
-rw-r--r--tests/frontend/track.py227
-rw-r--r--tests/frontend/version.py10
-rw-r--r--tests/frontend/workspace.py1002
-rw-r--r--tests/integration/artifact.py85
-rw-r--r--tests/integration/autotools.py73
-rw-r--r--tests/integration/build-uid.py49
-rw-r--r--tests/integration/cachedfail.py194
-rw-r--r--tests/integration/cmake.py44
-rw-r--r--tests/integration/compose-symlinks.py15
-rw-r--r--tests/integration/compose.py154
-rw-r--r--tests/integration/filter.py19
-rw-r--r--tests/integration/import.py50
-rw-r--r--tests/integration/make.py28
-rw-r--r--tests/integration/manual.py164
-rw-r--r--tests/integration/messages.py50
-rw-r--r--tests/integration/pip_element.py104
-rw-r--r--tests/integration/pip_source.py186
-rw-r--r--tests/integration/project/files/pip-source/app1.py4
-rw-r--r--tests/integration/pullbuildtrees.py122
-rw-r--r--tests/integration/sandbox-bwrap.py39
-rw-r--r--tests/integration/script.py222
-rw-r--r--tests/integration/shell.py348
-rw-r--r--tests/integration/shellbuildtrees.py309
-rw-r--r--tests/integration/sockets.py17
-rw-r--r--tests/integration/source-determinism.py69
-rw-r--r--tests/integration/stack.py19
-rw-r--r--tests/integration/symlinks.py53
-rw-r--r--tests/integration/workspace.py262
-rw-r--r--tests/internals/context.py78
-rw-r--r--tests/internals/loader.py35
-rw-r--r--tests/internals/pluginfactory.py315
-rw-r--r--tests/internals/pluginfactory/wrongtype/foo.py2
-rw-r--r--tests/internals/pluginloading.py27
-rw-r--r--tests/internals/pluginloading/customelement/pluginelements/foo.py1
-rw-r--r--tests/internals/pluginloading/customsource/pluginsources/foo.py1
-rw-r--r--tests/internals/storage.py11
-rw-r--r--tests/internals/storage_vdir_import.py127
-rw-r--r--tests/internals/utils_save_atomic.py48
-rw-r--r--tests/internals/yaml.py356
-rw-r--r--tests/plugins/deprecationwarnings/deprecationwarnings.py19
-rw-r--r--tests/remoteexecution/buildfail.py35
-rw-r--r--tests/remoteexecution/buildtree.py39
-rw-r--r--tests/remoteexecution/junction.py66
-rw-r--r--tests/remoteexecution/partial.py59
-rw-r--r--tests/remoteexecution/simple.py43
-rw-r--r--tests/sandboxes/fallback.py40
-rw-r--r--tests/sandboxes/missing-command.py11
-rw-r--r--tests/sandboxes/missing_dependencies.py73
-rw-r--r--tests/sandboxes/mounting/mount_simple.py18
-rw-r--r--tests/sandboxes/remote-exec-config.py68
-rw-r--r--tests/sandboxes/selection.py57
-rw-r--r--tests/sourcecache/cache.py53
-rw-r--r--tests/sourcecache/capabilities.py24
-rw-r--r--tests/sourcecache/config.py20
-rw-r--r--tests/sourcecache/fetch.py94
-rw-r--r--tests/sourcecache/project/plugins/elements/always_fail.py1
-rw-r--r--tests/sourcecache/push.py175
-rw-r--r--tests/sourcecache/source-checkout.py28
-rw-r--r--tests/sourcecache/staging.py58
-rw-r--r--tests/sourcecache/workspace.py61
-rw-r--r--tests/sources/bzr.py30
-rw-r--r--tests/sources/deb.py64
-rw-r--r--tests/sources/git.py990
-rw-r--r--tests/sources/keytest.py3
-rw-r--r--tests/sources/local.py165
-rw-r--r--tests/sources/no-fetch-cached/plugins/sources/always_cached.py1
-rw-r--r--tests/sources/no_fetch_cached.py25
-rw-r--r--tests/sources/patch.py93
-rw-r--r--tests/sources/pip.py46
-rw-r--r--tests/sources/previous_source_access.py31
-rw-r--r--tests/sources/previous_source_access/plugins/sources/foo_transform.py26
-rw-r--r--tests/sources/remote.py155
-rw-r--r--tests/sources/tar.py236
-rw-r--r--tests/sources/zip.py109
-rw-r--r--tests/testutils/artifactshare.py37
-rw-r--r--tests/testutils/element_generators.py24
-rw-r--r--tests/testutils/file_server.py4
-rw-r--r--tests/testutils/filetypegenerator.py2
-rw-r--r--tests/testutils/ftp_server.py6
-rw-r--r--tests/testutils/http_server.py53
-rw-r--r--tests/testutils/junction.py9
-rw-r--r--tests/testutils/patch.py10
-rw-r--r--tests/testutils/python_repo.py45
-rw-r--r--tests/testutils/repo/bzr.py32
-rw-r--r--tests/testutils/repo/git.py71
-rw-r--r--tests/testutils/repo/tar.py14
-rw-r--r--tests/testutils/repo/zip.py16
-rw-r--r--tests/testutils/setuptools.py13
-rw-r--r--tox.ini23
285 files changed, 12216 insertions, 14002 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 4f87e3c92..00270e8fa 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -212,6 +212,9 @@ mypy:
# Lint separately from testing
lint:
+ # We can't use the default debian:9 based image here since that comes with
+ # Python 3.5, and Black requires Python >= 3.6.
+ image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:30-master-93453213
stage: test
before_script:
@@ -219,7 +222,7 @@ lint:
- python3 --version
script:
- - tox -e lint
+ - tox -e format-check,lint
except:
- schedules
diff --git a/.pylintrc b/.pylintrc
index 6499644a9..422406417 100644
--- a/.pylintrc
+++ b/.pylintrc
@@ -64,7 +64,7 @@ confidence=
# no Warning level messages displayed, use"--disable=all --enable=classes
# --disable=W"
-# We have two groups of disabled messages:
+# We have three groups of disabled messages:
#
# 1) Messages that are of no use to us
# This is either because we don't follow the convention
@@ -76,6 +76,10 @@ confidence=
# some issues that just grew out of control. Resolving these would
# be nice, but too much work atm.
#
+# 3) Messages related to code formatting
+# Since we use Black to format code automatically, there's no need for
+# pylint to also check for those things.
+#
disable=#####################################
# Messages that are of no use to us #
@@ -111,6 +115,14 @@ disable=#####################################
unused-argument,
+ ##################################################
+ # Formatting-related messages, enforced by Black #
+ ##################################################
+
+ bad-continuation,
+ line-too-long,
+ superfluous-parens,
+
# Enable the message, report, category or checker with the given id(s). You can
# either give multiple identifier separated by comma (,) or put this option
diff --git a/doc/source/conf.py b/doc/source/conf.py
index 269053675..076bfde53 100644
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -21,7 +21,7 @@ import os
import sys
from buildstream import __version__
-sys.path.insert(0, os.path.abspath('..'))
+sys.path.insert(0, os.path.abspath(".."))
# -- General configuration ------------------------------------------------
@@ -32,32 +32,28 @@ sys.path.insert(0, os.path.abspath('..'))
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
-extensions = [
- 'sphinx.ext.autodoc',
- 'sphinx.ext.napoleon',
- 'sphinx_click.ext'
-]
+extensions = ["sphinx.ext.autodoc", "sphinx.ext.napoleon", "sphinx_click.ext"]
# Add any paths that contain templates here, relative to this directory.
-templates_path = ['.templates']
+templates_path = [".templates"]
# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
#
# source_suffix = ['.rst', '.md']
-source_suffix = '.rst'
+source_suffix = ".rst"
# The encoding of source files.
#
# source_encoding = 'utf-8-sig'
# The master toctree document.
-master_doc = 'index'
+master_doc = "index"
# General information about the project.
-project = 'BuildStream'
-copyright = '2017-2018, The BuildStream Contributors'
-author = 'The BuildStream Contributors'
+project = "BuildStream"
+copyright = "2017-2018, The BuildStream Contributors" # pylint: disable=redefined-builtin
+author = "The BuildStream Contributors"
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
@@ -109,10 +105,10 @@ add_module_names = False
# show_authors = False
# The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = "sphinx"
# A list of ignored prefixes for module index sorting.
-modindex_common_prefix = ['buildstream.']
+modindex_common_prefix = ["buildstream."]
# If true, keep warnings as "system message" paragraphs in the built documents.
# keep_warnings = False
@@ -126,7 +122,7 @@ todo_include_todos = False
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
-html_theme = 'sphinx_rtd_theme'
+html_theme = "sphinx_rtd_theme"
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
@@ -160,7 +156,7 @@ html_theme = 'sphinx_rtd_theme'
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['badges', 'images']
+html_static_path = ["badges", "images"]
# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
@@ -240,34 +236,30 @@ html_static_path = ['badges', 'images']
# html_search_scorer = 'scorer.js'
# Output file base name for HTML help builder.
-htmlhelp_basename = 'BuildStreamdoc'
+htmlhelp_basename = "BuildStreamdoc"
# -- Options for LaTeX output ---------------------------------------------
latex_elements = {
- # The paper size ('letterpaper' or 'a4paper').
- #
- # 'papersize': 'letterpaper',
-
- # The font size ('10pt', '11pt' or '12pt').
- #
- # 'pointsize': '10pt',
-
- # Additional stuff for the LaTeX preamble.
- #
- # 'preamble': '',
-
- # Latex figure (float) alignment
- #
- # 'figure_align': 'htbp',
+ # The paper size ('letterpaper' or 'a4paper').
+ #
+ # 'papersize': 'letterpaper',
+ # The font size ('10pt', '11pt' or '12pt').
+ #
+ # 'pointsize': '10pt',
+ # Additional stuff for the LaTeX preamble.
+ #
+ # 'preamble': '',
+ # Latex figure (float) alignment
+ #
+ # 'figure_align': 'htbp',
}
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
- (master_doc, 'BuildStream.tex', 'BuildStream Documentation',
- 'The BuildStream Contributors', 'manual'),
+ (master_doc, "BuildStream.tex", "BuildStream Documentation", "The BuildStream Contributors", "manual"),
]
# The name of an image file (relative to this directory) to place at the top of
@@ -307,10 +299,7 @@ latex_documents = [
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
-man_pages = [
- (master_doc, 'buildstream', 'BuildStream Documentation',
- [author], 1)
-]
+man_pages = [(master_doc, "buildstream", "BuildStream Documentation", [author], 1)]
# If true, show URL addresses after external links.
#
@@ -323,9 +312,15 @@ man_pages = [
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
- (master_doc, 'BuildStream', 'BuildStream Documentation',
- author, 'BuildStream', 'One line description of project.',
- 'Miscellaneous'),
+ (
+ master_doc,
+ "BuildStream",
+ "BuildStream Documentation",
+ author,
+ "BuildStream",
+ "One line description of project.",
+ "Miscellaneous",
+ ),
]
# Documents to append as an appendix to all manuals.
@@ -344,4 +339,4 @@ texinfo_documents = [
#
# texinfo_no_detailmenu = False
-autodoc_member_order = 'bysource'
+autodoc_member_order = "bysource"
diff --git a/doc/source/hacking/coding_guidelines.rst b/doc/source/hacking/coding_guidelines.rst
index 7088fc365..4ba6360b6 100644
--- a/doc/source/hacking/coding_guidelines.rst
+++ b/doc/source/hacking/coding_guidelines.rst
@@ -19,10 +19,10 @@ Approximate PEP-8 Style
~~~~~~~~~~~~~~~~~~~~~~~
Python coding style for BuildStream is approximately `pep8 <https://www.python.org/dev/peps/pep-0008/>`_.
-We have a couple of minor exceptions to this standard, we dont want to compromise
-code readability by being overly restrictive on line length for instance.
+The coding style is automatically enforced by `black <https://black.readthedocs.io/en/stable/>`_.
-The pep8 linter will run automatically when :ref:`running the test suite <contributing_testing>`.
+Formatting will be checked automatically when running the testsuite on CI. For
+details on how to format your code locally, see :ref:`formatting code <contributing_formatting_code>`.
Line lengths
diff --git a/doc/source/hacking/using_the_testsuite.rst b/doc/source/hacking/using_the_testsuite.rst
index 2bd2696bb..0e476c7de 100644
--- a/doc/source/hacking/using_the_testsuite.rst
+++ b/doc/source/hacking/using_the_testsuite.rst
@@ -54,18 +54,6 @@ the same arguments you would give `tox`::
detox -e lint,py36,py37
-Linting is performed separately from testing. In order to run the linting step which
-consists of running the ``pycodestyle`` and ``pylint`` tools, run the following::
-
- tox -e lint
-
-.. tip::
-
- The project specific pylint and pycodestyle configurations are stored in the
- toplevel buildstream directory in the ``.pylintrc`` file and ``setup.cfg`` files
- respectively. These configurations can be interesting to use with IDEs and
- other developer tooling.
-
The output of all failing tests will always be printed in the summary, but
if you want to observe the stdout and stderr generated by a passing test,
you can pass the ``-s`` option to pytest as such::
@@ -155,9 +143,31 @@ can run ``tox`` with ``-r`` or ``--recreate`` option.
tests::
tox -e venv -- <your command(s) here>
-
+
Any commands after ``--`` will be run a virtualenv managed by tox.
+Running linters
+~~~~~~~~~~~~~~~
+Linting is performed separately from testing. In order to run the linting step which
+consists of running the ``pylint`` tool, run the following::
+
+ tox -e lint
+
+.. tip::
+
+ The project specific pylint configuration is stored in the toplevel
+ buildstream directory in the ``.pylintrc`` file. This configuration can be
+ interesting to use with IDEs and other developer tooling.
+
+.. _contributing_formatting_code:
+
+Formatting code
+~~~~~~~~~~~~~~~
+Similar to linting, code formatting is also done via a ``tox`` environment. To
+format the code using the ``black`` tool, run the following::
+
+ tox -e format
+
Observing coverage
~~~~~~~~~~~~~~~~~~
Once you have run the tests using `tox` (or `detox`), some coverage reports will
diff --git a/pyproject.toml b/pyproject.toml
index 4dd02d1e5..29f5589b5 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -7,3 +7,21 @@ requires = [
"Cython"
]
build-backend = "setuptools.build_meta"
+
+[tool.black]
+line-length = 119
+exclude = '''
+(
+ /(
+ \.eggs
+ | \.git
+ | \.mypy_cache
+ | \.tox
+ | _build
+ | build
+ | dist
+ )/
+ | src/buildstream/_fuse
+ | src/buildstream/_protos
+)
+'''
diff --git a/requirements/dev-requirements.in b/requirements/dev-requirements.in
index 8b871073b..cbf44c470 100644
--- a/requirements/dev-requirements.in
+++ b/requirements/dev-requirements.in
@@ -1,5 +1,4 @@
pylint
-pycodestyle
pytest >= 3.9
pytest-datafiles >= 2.0
pytest-env
diff --git a/requirements/dev-requirements.txt b/requirements/dev-requirements.txt
index c4930181e..bd8449978 100644
--- a/requirements/dev-requirements.txt
+++ b/requirements/dev-requirements.txt
@@ -1,5 +1,4 @@
pylint==2.3.1
-pycodestyle==2.5.0
pytest==5.1.2
pytest-datafiles==2.0
pytest-env==0.6.2
diff --git a/setup.cfg b/setup.cfg
index eb61e352c..363758652 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -23,11 +23,6 @@ markers =
remoteexecution: run test only if --remote-execution option is specified
xfail_strict=True
-[pycodestyle]
-max-line-length = 119
-ignore = E129,E125,W504,W605
-exclude = .git/**,.tox/**,.eggs/**,build/**,doc/source/conf.py,src/buildstream/_fuse/fuse.py,src/buildstream/_protos/**/*py,tmp/**
-
[mypy]
files = src
warn_unused_configs = True
diff --git a/src/buildstream/__init__.py b/src/buildstream/__init__.py
index cd8d0f1cf..c78fcbbf6 100644
--- a/src/buildstream/__init__.py
+++ b/src/buildstream/__init__.py
@@ -19,11 +19,13 @@
# Plugin author facing APIs
import os
+
if "_BST_COMPLETION" not in os.environ:
# Special sauce to get the version from versioneer
from ._version import get_versions
- __version__ = get_versions()['version']
+
+ __version__ = get_versions()["version"]
del get_versions
from .utils import UtilError, ProgramNotFoundError
diff --git a/src/buildstream/__main__.py b/src/buildstream/__main__.py
index 4b0fdabfe..556a0f67e 100644
--- a/src/buildstream/__main__.py
+++ b/src/buildstream/__main__.py
@@ -11,7 +11,8 @@
# This is used when we need to run BuildStream before installing,
# like when we build documentation.
#
-if __name__ == '__main__':
+if __name__ == "__main__":
# pylint: disable=no-value-for-parameter
from ._frontend.cli import cli
+
cli()
diff --git a/src/buildstream/_artifact.py b/src/buildstream/_artifact.py
index e5174eaea..feba3898b 100644
--- a/src/buildstream/_artifact.py
+++ b/src/buildstream/_artifact.py
@@ -47,7 +47,7 @@ from .storage._casbaseddirectory import CasBasedDirectory
# strong_key (str): The elements strong cache key, dependent on context
# weak_key (str): The elements weak cache key
#
-class Artifact():
+class Artifact:
version = 0
@@ -61,11 +61,11 @@ class Artifact():
self._tmpdir = context.tmpdir
self._proto = None
- self._metadata_keys = None # Strong and weak key tuple extracted from the artifact
- self._metadata_dependencies = None # Dictionary of dependency strong keys from the artifact
- self._metadata_workspaced = None # Boolean of whether it's a workspaced artifact
+ self._metadata_keys = None # Strong and weak key tuple extracted from the artifact
+ self._metadata_dependencies = None # Dictionary of dependency strong keys from the artifact
+ self._metadata_workspaced = None # Boolean of whether it's a workspaced artifact
self._metadata_workspaced_dependencies = None # List of which dependencies are workspaced from the artifact
- self._cached = None # Boolean of whether the artifact is cached
+ self._cached = None # Boolean of whether the artifact is cached
# get_files():
#
@@ -193,12 +193,11 @@ class Artifact():
artifact.buildtree.CopyFrom(buildtreevdir._get_digest())
size += buildtreevdir.get_size()
- os.makedirs(os.path.dirname(os.path.join(
- self._artifactdir, element.get_artifact_name())), exist_ok=True)
+ os.makedirs(os.path.dirname(os.path.join(self._artifactdir, element.get_artifact_name())), exist_ok=True)
keys = utils._deduplicate([self._cache_key, self._weak_cache_key])
for key in keys:
path = os.path.join(self._artifactdir, element.get_artifact_name(key=key))
- with utils.save_file_atomic(path, mode='wb') as f:
+ with utils.save_file_atomic(path, mode="wb") as f:
f.write(artifact.SerializeToString())
return size
@@ -247,7 +246,7 @@ class Artifact():
# Load the public data from the artifact
artifact = self._get_proto()
meta_file = self._cas.objpath(artifact.public_data)
- data = _yaml.load(meta_file, shortname='public.yaml')
+ data = _yaml.load(meta_file, shortname="public.yaml")
return data
@@ -263,9 +262,7 @@ class Artifact():
def load_build_result(self):
artifact = self._get_proto()
- build_result = (artifact.build_success,
- artifact.build_error,
- artifact.build_error_details)
+ build_result = (artifact.build_success, artifact.build_error, artifact.build_error_details)
return build_result
@@ -345,8 +342,9 @@ class Artifact():
# Extract proto
artifact = self._get_proto()
- self._metadata_workspaced_dependencies = [dep.element_name for dep in artifact.build_deps
- if dep.was_workspaced]
+ self._metadata_workspaced_dependencies = [
+ dep.element_name for dep in artifact.build_deps if dep.was_workspaced
+ ]
return self._metadata_workspaced_dependencies
@@ -419,12 +417,14 @@ class Artifact():
# Determine whether directories are required
require_directories = context.require_artifact_directories
# Determine whether file contents are required as well
- require_files = (context.require_artifact_files or
- self._element._artifact_files_required())
+ require_files = context.require_artifact_files or self._element._artifact_files_required()
# Check whether 'files' subdirectory is available, with or without file contents
- if (require_directories and str(artifact.files) and
- not self._cas.contains_directory(artifact.files, with_files=require_files)):
+ if (
+ require_directories
+ and str(artifact.files)
+ and not self._cas.contains_directory(artifact.files, with_files=require_files)
+ ):
self._cached = False
return False
@@ -471,11 +471,10 @@ class Artifact():
key = self.get_extract_key()
- proto_path = os.path.join(self._artifactdir,
- self._element.get_artifact_name(key=key))
+ proto_path = os.path.join(self._artifactdir, self._element.get_artifact_name(key=key))
artifact = ArtifactProto()
try:
- with open(proto_path, mode='r+b') as f:
+ with open(proto_path, mode="r+b") as f:
artifact.ParseFromString(f.read())
except FileNotFoundError:
return None
diff --git a/src/buildstream/_artifactcache.py b/src/buildstream/_artifactcache.py
index 79d0dc50b..03c47b906 100644
--- a/src/buildstream/_artifactcache.py
+++ b/src/buildstream/_artifactcache.py
@@ -23,8 +23,7 @@ import grpc
from ._basecache import BaseCache
from ._exceptions import ArtifactError, CASError, CASCacheError, CASRemoteError, RemoteError
-from ._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc, \
- artifact_pb2, artifact_pb2_grpc
+from ._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc, artifact_pb2, artifact_pb2_grpc
from ._remote import BaseRemote
from .storage._casbaseddirectory import CasBasedDirectory
@@ -38,7 +37,6 @@ from . import utils
# artifact remotes.
#
class ArtifactRemote(BaseRemote):
-
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.artifact_service = None
@@ -78,8 +76,10 @@ class ArtifactRemote(BaseRemote):
except grpc.RpcError as e:
# Check if this remote has the artifact service
if e.code() == grpc.StatusCode.UNIMPLEMENTED:
- raise RemoteError("Configured remote does not have the BuildStream "
- "capabilities service. Please check remote configuration.")
+ raise RemoteError(
+ "Configured remote does not have the BuildStream "
+ "capabilities service. Please check remote configuration."
+ )
# Else raise exception with details
raise RemoteError("Remote initialisation failed: {}".format(e.details()))
@@ -263,9 +263,11 @@ class ArtifactCache(BaseCache):
if self._push_artifact_blobs(artifact, remote):
element.info("Pushed data from artifact {} -> {}".format(display_key, remote))
else:
- element.info("Remote ({}) already has all data of artifact {} cached".format(
- remote, element._get_brief_display_key()
- ))
+ element.info(
+ "Remote ({}) already has all data of artifact {} cached".format(
+ remote, element._get_brief_display_key()
+ )
+ )
for remote in index_remotes:
remote.init()
@@ -275,9 +277,9 @@ class ArtifactCache(BaseCache):
element.info("Pushed artifact {} -> {}".format(display_key, remote))
pushed = True
else:
- element.info("Remote ({}) already has artifact {} cached".format(
- remote, element._get_brief_display_key()
- ))
+ element.info(
+ "Remote ({}) already has artifact {} cached".format(remote, element._get_brief_display_key())
+ )
return pushed
@@ -295,7 +297,7 @@ class ArtifactCache(BaseCache):
#
def pull(self, element, key, *, pull_buildtrees=False):
artifact = None
- display_key = key[:self.context.log_key_length]
+ display_key = key[: self.context.log_key_length]
project = element._get_project()
errors = []
@@ -310,16 +312,15 @@ class ArtifactCache(BaseCache):
element.info("Pulled artifact {} <- {}".format(display_key, remote))
break
else:
- element.info("Remote ({}) does not have artifact {} cached".format(
- remote, display_key
- ))
+ element.info("Remote ({}) does not have artifact {} cached".format(remote, display_key))
except CASError as e:
element.warn("Could not pull from remote {}: {}".format(remote, e))
errors.append(e)
if errors and not artifact:
- raise ArtifactError("Failed to pull artifact {}".format(display_key),
- detail="\n".join(str(e) for e in errors))
+ raise ArtifactError(
+ "Failed to pull artifact {}".format(display_key), detail="\n".join(str(e) for e in errors)
+ )
# If we don't have an artifact, we can't exactly pull our
# artifact
@@ -337,16 +338,15 @@ class ArtifactCache(BaseCache):
element.info("Pulled data for artifact {} <- {}".format(display_key, remote))
return True
- element.info("Remote ({}) does not have artifact {} cached".format(
- remote, display_key
- ))
+ element.info("Remote ({}) does not have artifact {} cached".format(remote, display_key))
except CASError as e:
element.warn("Could not pull from remote {}: {}".format(remote, e))
errors.append(e)
if errors:
- raise ArtifactError("Failed to pull artifact {}".format(display_key),
- detail="\n".join(str(e) for e in errors))
+ raise ArtifactError(
+ "Failed to pull artifact {}".format(display_key), detail="\n".join(str(e) for e in errors)
+ )
return False
@@ -388,8 +388,9 @@ class ArtifactCache(BaseCache):
push_remotes = []
if not push_remotes:
- raise ArtifactError("push_message was called, but no remote artifact " +
- "servers are configured as push remotes.")
+ raise ArtifactError(
+ "push_message was called, but no remote artifact " + "servers are configured as push remotes."
+ )
for remote in push_remotes:
message_digest = remote.push_message(message)
@@ -410,8 +411,7 @@ class ArtifactCache(BaseCache):
newref = element.get_artifact_name(newkey)
if not os.path.exists(os.path.join(self.artifactdir, newref)):
- os.link(os.path.join(self.artifactdir, oldref),
- os.path.join(self.artifactdir, newref))
+ os.link(os.path.join(self.artifactdir, oldref), os.path.join(self.artifactdir, newref))
# get_artifact_logs():
#
@@ -425,7 +425,7 @@ class ArtifactCache(BaseCache):
#
def get_artifact_logs(self, ref):
cache_id = self.cas.resolve_ref(ref, update_mtime=True)
- vdir = CasBasedDirectory(self.cas, digest=cache_id).descend('logs')
+ vdir = CasBasedDirectory(self.cas, digest=cache_id).descend("logs")
return vdir
# fetch_missing_blobs():
@@ -517,7 +517,7 @@ class ArtifactCache(BaseCache):
for root, _, files in os.walk(self.artifactdir):
for artifact_file in files:
artifact = artifact_pb2.Artifact()
- with open(os.path.join(root, artifact_file), 'r+b') as f:
+ with open(os.path.join(root, artifact_file), "r+b") as f:
artifact.ParseFromString(f.read())
if str(artifact.files):
@@ -535,7 +535,7 @@ class ArtifactCache(BaseCache):
for root, _, files in os.walk(self.artifactdir):
for artifact_file in files:
artifact = artifact_pb2.Artifact()
- with open(os.path.join(root, artifact_file), 'r+b') as f:
+ with open(os.path.join(root, artifact_file), "r+b") as f:
artifact.ParseFromString(f.read())
if str(artifact.public_data):
@@ -620,8 +620,7 @@ class ArtifactCache(BaseCache):
remote.get_artifact(element.get_artifact_name(key=key))
except grpc.RpcError as e:
if e.code() != grpc.StatusCode.NOT_FOUND:
- raise ArtifactError("Error checking artifact cache: {}"
- .format(e.details()))
+ raise ArtifactError("Error checking artifact cache: {}".format(e.details()))
else:
return False
@@ -710,7 +709,7 @@ class ArtifactCache(BaseCache):
# Write the artifact proto to cache
artifact_path = os.path.join(self.artifactdir, artifact_name)
os.makedirs(os.path.dirname(artifact_path), exist_ok=True)
- with utils.save_file_atomic(artifact_path, mode='wb') as f:
+ with utils.save_file_atomic(artifact_path, mode="wb") as f:
f.write(artifact.SerializeToString())
return artifact
diff --git a/src/buildstream/_artifactelement.py b/src/buildstream/_artifactelement.py
index 48c3d1769..1c1c5db46 100644
--- a/src/buildstream/_artifactelement.py
+++ b/src/buildstream/_artifactelement.py
@@ -40,7 +40,7 @@ if TYPE_CHECKING:
class ArtifactElement(Element):
# A hash of ArtifactElement by ref
- __instantiated_artifacts = {} # type: Dict[str, ArtifactElement]
+ __instantiated_artifacts = {} # type: Dict[str, ArtifactElement]
# ArtifactElement's require this as the sandbox will use a normal
# directory when we checkout
@@ -138,7 +138,7 @@ class ArtifactElement(Element):
# sandbox (Sandbox)
#
def configure_sandbox(self, sandbox):
- install_root = self.get_variable('install-root')
+ install_root = self.get_variable("install-root")
# Tell the sandbox to mount the build root and install root
sandbox.mark_directory(install_root)
@@ -173,7 +173,7 @@ class ArtifactElement(Element):
#
def verify_artifact_ref(ref):
try:
- project, element, key = ref.split('/', 2) # This will raise a Value error if unable to split
+ project, element, key = ref.split("/", 2) # This will raise a Value error if unable to split
# Explicitly raise a ValueError if the key length is not as expected
if not _cachekey.is_key(key):
raise ValueError
diff --git a/src/buildstream/_basecache.py b/src/buildstream/_basecache.py
index fc2e92456..516119cd1 100644
--- a/src/buildstream/_basecache.py
+++ b/src/buildstream/_basecache.py
@@ -37,21 +37,21 @@ if TYPE_CHECKING:
# Base Cache for Caches to derive from
#
-class BaseCache():
+class BaseCache:
# None of these should ever be called in the base class, but this appeases
# pylint to some degree
- spec_name = None # type: str
- spec_error = None # type: Type[BstError]
- config_node_name = None # type: str
- index_remote_class = None # type: Type[BaseRemote]
+ spec_name = None # type: str
+ spec_error = None # type: Type[BstError]
+ config_node_name = None # type: str
+ index_remote_class = None # type: Type[BaseRemote]
storage_remote_class = CASRemote # type: Type[BaseRemote]
def __init__(self, context):
self.context = context
self.cas = context.get_cascache()
- self._remotes_setup = False # Check to prevent double-setup of remotes
+ self._remotes_setup = False # Check to prevent double-setup of remotes
# Per-project list of Remote instances.
self._storage_remotes = {}
self._index_remotes = {}
@@ -116,8 +116,12 @@ class BaseCache():
artifacts = config_node.get_sequence(cls.config_node_name, default=[])
except LoadError:
provenance = config_node.get_node(cls.config_node_name).get_provenance()
- raise _yaml.LoadError("{}: '{}' must be a single remote mapping, or a list of mappings"
- .format(provenance, cls.config_node_name), _yaml.LoadErrorReason.INVALID_DATA)
+ raise _yaml.LoadError(
+ "{}: '{}' must be a single remote mapping, or a list of mappings".format(
+ provenance, cls.config_node_name
+ ),
+ _yaml.LoadErrorReason.INVALID_DATA,
+ )
for spec_node in artifacts:
cache_specs.append(RemoteSpec.new_from_config_node(spec_node))
@@ -144,8 +148,7 @@ class BaseCache():
project_specs = getattr(project, cls.spec_name)
context_specs = getattr(context, cls.spec_name)
- return list(utils._deduplicate(
- project_extra_specs + project_specs + context_specs))
+ return list(utils._deduplicate(project_extra_specs + project_specs + context_specs))
# setup_remotes():
#
@@ -266,8 +269,9 @@ class BaseCache():
# Check whether the specified element's project has push remotes
index_remotes = self._index_remotes[plugin._get_project()]
storage_remotes = self._storage_remotes[plugin._get_project()]
- return (any(remote.spec.push for remote in index_remotes) and
- any(remote.spec.push for remote in storage_remotes))
+ return any(remote.spec.push for remote in index_remotes) and any(
+ remote.spec.push for remote in storage_remotes
+ )
################################################
# Local Private Methods #
@@ -323,8 +327,9 @@ class BaseCache():
storage_remotes[remote_spec] = storage
self._has_fetch_remotes = storage_remotes and index_remotes
- self._has_push_remotes = (any(spec.push for spec in storage_remotes) and
- any(spec.push for spec in index_remotes))
+ self._has_push_remotes = any(spec.push for spec in storage_remotes) and any(
+ spec.push for spec in index_remotes
+ )
return index_remotes, storage_remotes
@@ -366,8 +371,7 @@ class BaseCache():
#
def _message(self, message_type, message, **kwargs):
args = dict(kwargs)
- self.context.messenger.message(
- Message(message_type, message, **args))
+ self.context.messenger.message(Message(message_type, message, **args))
# _set_remotes():
#
diff --git a/src/buildstream/_cachekey.py b/src/buildstream/_cachekey.py
index 89d47671e..dd9207516 100644
--- a/src/buildstream/_cachekey.py
+++ b/src/buildstream/_cachekey.py
@@ -62,5 +62,5 @@ def is_key(key):
# (str): An sha256 hex digest of the given value
#
def generate_key(value):
- ustring = ujson.dumps(value, sort_keys=True, escape_forward_slashes=False).encode('utf-8')
+ ustring = ujson.dumps(value, sort_keys=True, escape_forward_slashes=False).encode("utf-8")
return hashlib.sha256(ustring).hexdigest()
diff --git a/src/buildstream/_cas/cascache.py b/src/buildstream/_cas/cascache.py
index 022730445..c1f2b30b0 100644
--- a/src/buildstream/_cas/cascache.py
+++ b/src/buildstream/_cas/cascache.py
@@ -68,15 +68,14 @@ class CASLogLevel(FastEnum):
# protect_session_blobs (bool): Disable expiry for blobs used in the current session
# log_level (LogLevel): Log level to give to buildbox-casd for logging
#
-class CASCache():
-
+class CASCache:
def __init__(
- self, path, *, casd=True, cache_quota=None, protect_session_blobs=True, log_level=CASLogLevel.WARNING
+ self, path, *, casd=True, cache_quota=None, protect_session_blobs=True, log_level=CASLogLevel.WARNING
):
- self.casdir = os.path.join(path, 'cas')
- self.tmpdir = os.path.join(path, 'tmp')
- os.makedirs(os.path.join(self.casdir, 'refs', 'heads'), exist_ok=True)
- os.makedirs(os.path.join(self.casdir, 'objects'), exist_ok=True)
+ self.casdir = os.path.join(path, "cas")
+ self.tmpdir = os.path.join(path, "tmp")
+ os.makedirs(os.path.join(self.casdir, "refs", "heads"), exist_ok=True)
+ os.makedirs(os.path.join(self.casdir, "objects"), exist_ok=True)
os.makedirs(self.tmpdir, exist_ok=True)
self._casd_channel = None
@@ -88,19 +87,19 @@ class CASCache():
if casd:
# Place socket in global/user temporary directory to avoid hitting
# the socket path length limit.
- self._casd_socket_tempdir = tempfile.mkdtemp(prefix='buildstream')
- self._casd_socket_path = os.path.join(self._casd_socket_tempdir, 'casd.sock')
+ self._casd_socket_tempdir = tempfile.mkdtemp(prefix="buildstream")
+ self._casd_socket_path = os.path.join(self._casd_socket_tempdir, "casd.sock")
- casd_args = [utils.get_host_tool('buildbox-casd')]
- casd_args.append('--bind=unix:' + self._casd_socket_path)
- casd_args.append('--log-level=' + log_level.value)
+ casd_args = [utils.get_host_tool("buildbox-casd")]
+ casd_args.append("--bind=unix:" + self._casd_socket_path)
+ casd_args.append("--log-level=" + log_level.value)
if cache_quota is not None:
- casd_args.append('--quota-high={}'.format(int(cache_quota)))
- casd_args.append('--quota-low={}'.format(int(cache_quota / 2)))
+ casd_args.append("--quota-high={}".format(int(cache_quota)))
+ casd_args.append("--quota-low={}".format(int(cache_quota / 2)))
if protect_session_blobs:
- casd_args.append('--protect-session-blobs')
+ casd_args.append("--protect-session-blobs")
casd_args.append(path)
@@ -112,7 +111,8 @@ class CASCache():
# The frontend will take care of it if needed
with _signals.blocked([signal.SIGINT], ignore=False):
self._casd_process = subprocess.Popen(
- casd_args, cwd=path, stdout=logfile_fp, stderr=subprocess.STDOUT)
+ casd_args, cwd=path, stdout=logfile_fp, stderr=subprocess.STDOUT
+ )
self._cache_usage_monitor = _CASCacheUsageMonitor(self)
else:
@@ -123,16 +123,16 @@ class CASCache():
# Popen objects are not pickle-able, however, child processes only
# need the information whether a casd subprocess was started or not.
- assert '_casd_process' in state
- state['_casd_process'] = bool(self._casd_process)
+ assert "_casd_process" in state
+ state["_casd_process"] = bool(self._casd_process)
# The usage monitor is not pickle-able, but we also don't need it in
# child processes currently. Make sure that if this changes, we get a
# bug report, by setting _cache_usage_monitor_forbidden.
- assert '_cache_usage_monitor' in state
- assert '_cache_usage_monitor_forbidden' in state
- state['_cache_usage_monitor'] = None
- state['_cache_usage_monitor_forbidden'] = True
+ assert "_cache_usage_monitor" in state
+ assert "_cache_usage_monitor_forbidden" in state
+ state["_cache_usage_monitor"] = None
+ state["_cache_usage_monitor_forbidden"] = True
return state
@@ -148,7 +148,7 @@ class CASCache():
time.sleep(0.01)
- self._casd_channel = grpc.insecure_channel('unix:' + self._casd_socket_path)
+ self._casd_channel = grpc.insecure_channel("unix:" + self._casd_socket_path)
self._casd_cas = remote_execution_pb2_grpc.ContentAddressableStorageStub(self._casd_channel)
self._local_cas = local_cas_pb2_grpc.LocalContentAddressableStorageStub(self._casd_channel)
@@ -179,8 +179,8 @@ class CASCache():
# Preflight check.
#
def preflight(self):
- headdir = os.path.join(self.casdir, 'refs', 'heads')
- objdir = os.path.join(self.casdir, 'objects')
+ headdir = os.path.join(self.casdir, "refs", "heads")
+ objdir = os.path.join(self.casdir, "objects")
if not (os.path.isdir(headdir) and os.path.isdir(objdir)):
raise CASCacheError("CAS repository check failed for '{}'".format(self.casdir))
@@ -285,7 +285,7 @@ class CASCache():
directory = remote_execution_pb2.Directory()
- with open(self.objpath(tree), 'rb') as f:
+ with open(self.objpath(tree), "rb") as f:
directory.ParseFromString(f.read())
for filenode in directory.files:
@@ -297,8 +297,16 @@ class CASCache():
utils.safe_copy(self.objpath(filenode.digest), fullpath)
if filenode.is_executable:
- os.chmod(fullpath, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR |
- stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
+ os.chmod(
+ fullpath,
+ stat.S_IRUSR
+ | stat.S_IWUSR
+ | stat.S_IXUSR
+ | stat.S_IRGRP
+ | stat.S_IXGRP
+ | stat.S_IROTH
+ | stat.S_IXOTH,
+ )
for dirnode in directory.directories:
fullpath = os.path.join(dest, dirnode.name)
@@ -365,7 +373,7 @@ class CASCache():
# (str): The path of the object
#
def objpath(self, digest):
- return os.path.join(self.casdir, 'objects', digest.hash[:2], digest.hash[2:])
+ return os.path.join(self.casdir, "objects", digest.hash[:2], digest.hash[2:])
# add_object():
#
@@ -450,7 +458,7 @@ class CASCache():
treepath = self.objpath(tree_response.tree_digest)
tree = remote_execution_pb2.Tree()
- with open(treepath, 'rb') as f:
+ with open(treepath, "rb") as f:
tree.ParseFromString(f.read())
root_directory = tree.root.SerializeToString()
@@ -467,7 +475,7 @@ class CASCache():
def set_ref(self, ref, tree):
refpath = self._refpath(ref)
os.makedirs(os.path.dirname(refpath), exist_ok=True)
- with utils.save_file_atomic(refpath, 'wb', tempdir=self.tmpdir) as f:
+ with utils.save_file_atomic(refpath, "wb", tempdir=self.tmpdir) as f:
f.write(tree.SerializeToString())
# resolve_ref():
@@ -485,7 +493,7 @@ class CASCache():
refpath = self._refpath(ref)
try:
- with open(refpath, 'rb') as f:
+ with open(refpath, "rb") as f:
if update_mtime:
os.utime(refpath)
@@ -521,7 +529,7 @@ class CASCache():
def remove(self, ref, *, basedir=None):
if basedir is None:
- basedir = os.path.join(self.casdir, 'refs', 'heads')
+ basedir = os.path.join(self.casdir, "refs", "heads")
# Remove cache ref
self._remove_ref(ref, basedir)
@@ -611,7 +619,7 @@ class CASCache():
directory = remote_execution_pb2.Directory()
- with open(self.objpath(directory_digest), 'rb') as f:
+ with open(self.objpath(directory_digest), "rb") as f:
directory.ParseFromString(f.read())
for filenode in directory.files:
@@ -626,21 +634,19 @@ class CASCache():
dir_b = remote_execution_pb2.Directory()
if tree_a:
- with open(self.objpath(tree_a), 'rb') as f:
+ with open(self.objpath(tree_a), "rb") as f:
dir_a.ParseFromString(f.read())
if tree_b:
- with open(self.objpath(tree_b), 'rb') as f:
+ with open(self.objpath(tree_b), "rb") as f:
dir_b.ParseFromString(f.read())
a = 0
b = 0
while a < len(dir_a.files) or b < len(dir_b.files):
- if b < len(dir_b.files) and (a >= len(dir_a.files) or
- dir_a.files[a].name > dir_b.files[b].name):
+ if b < len(dir_b.files) and (a >= len(dir_a.files) or dir_a.files[a].name > dir_b.files[b].name):
added.append(os.path.join(path, dir_b.files[b].name))
b += 1
- elif a < len(dir_a.files) and (b >= len(dir_b.files) or
- dir_b.files[b].name > dir_a.files[a].name):
+ elif a < len(dir_a.files) and (b >= len(dir_b.files) or dir_b.files[b].name > dir_a.files[a].name):
removed.append(os.path.join(path, dir_a.files[a].name))
a += 1
else:
@@ -653,24 +659,41 @@ class CASCache():
a = 0
b = 0
while a < len(dir_a.directories) or b < len(dir_b.directories):
- if b < len(dir_b.directories) and (a >= len(dir_a.directories) or
- dir_a.directories[a].name > dir_b.directories[b].name):
- self.diff_trees(None, dir_b.directories[b].digest,
- added=added, removed=removed, modified=modified,
- path=os.path.join(path, dir_b.directories[b].name))
+ if b < len(dir_b.directories) and (
+ a >= len(dir_a.directories) or dir_a.directories[a].name > dir_b.directories[b].name
+ ):
+ self.diff_trees(
+ None,
+ dir_b.directories[b].digest,
+ added=added,
+ removed=removed,
+ modified=modified,
+ path=os.path.join(path, dir_b.directories[b].name),
+ )
b += 1
- elif a < len(dir_a.directories) and (b >= len(dir_b.directories) or
- dir_b.directories[b].name > dir_a.directories[a].name):
- self.diff_trees(dir_a.directories[a].digest, None,
- added=added, removed=removed, modified=modified,
- path=os.path.join(path, dir_a.directories[a].name))
+ elif a < len(dir_a.directories) and (
+ b >= len(dir_b.directories) or dir_b.directories[b].name > dir_a.directories[a].name
+ ):
+ self.diff_trees(
+ dir_a.directories[a].digest,
+ None,
+ added=added,
+ removed=removed,
+ modified=modified,
+ path=os.path.join(path, dir_a.directories[a].name),
+ )
a += 1
else:
# Subdirectory exists in both directories
if dir_a.directories[a].digest.hash != dir_b.directories[b].digest.hash:
- self.diff_trees(dir_a.directories[a].digest, dir_b.directories[b].digest,
- added=added, removed=removed, modified=modified,
- path=os.path.join(path, dir_a.directories[a].name))
+ self.diff_trees(
+ dir_a.directories[a].digest,
+ dir_b.directories[b].digest,
+ added=added,
+ removed=removed,
+ modified=modified,
+ path=os.path.join(path, dir_a.directories[a].name),
+ )
a += 1
b += 1
@@ -703,7 +726,7 @@ class CASCache():
return os.path.join(log_dir, str(self._casd_start_time) + ".log")
def _refpath(self, ref):
- return os.path.join(self.casdir, 'refs', 'heads', ref)
+ return os.path.join(self.casdir, "refs", "heads", ref)
# _remove_ref()
#
@@ -763,7 +786,7 @@ class CASCache():
directory = remote_execution_pb2.Directory()
- with open(self.objpath(tree), 'rb') as f:
+ with open(self.objpath(tree), "rb") as f:
directory.ParseFromString(f.read())
for dirnode in directory.directories:
@@ -783,7 +806,7 @@ class CASCache():
directory = remote_execution_pb2.Directory()
- with open(self.objpath(tree), 'rb') as f:
+ with open(self.objpath(tree), "rb") as f:
directory.ParseFromString(f.read())
except FileNotFoundError:
@@ -813,8 +836,7 @@ class CASCache():
@contextlib.contextmanager
def _temporary_object(self):
with utils._tempnamedfile(dir=self.tmpdir) as f:
- os.chmod(f.name,
- stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
+ os.chmod(f.name, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
yield f
# _ensure_blob():
@@ -898,12 +920,13 @@ class CASCache():
objpath = self._ensure_blob(remote, dir_digest)
directory = remote_execution_pb2.Directory()
- with open(objpath, 'rb') as f:
+ with open(objpath, "rb") as f:
directory.ParseFromString(f.read())
for dirnode in directory.directories:
- batch = self._fetch_directory_node(remote, dirnode.digest, batch,
- fetch_queue, fetch_next_queue, recursive=True)
+ batch = self._fetch_directory_node(
+ remote, dirnode.digest, batch, fetch_queue, fetch_next_queue, recursive=True
+ )
# Fetch final batch
self._fetch_directory_batch(remote, batch, fetch_queue, fetch_next_queue)
@@ -913,7 +936,7 @@ class CASCache():
tree = remote_execution_pb2.Tree()
- with open(objpath, 'rb') as f:
+ with open(objpath, "rb") as f:
tree.ParseFromString(f.read())
tree.children.extend([tree.root])
@@ -1062,8 +1085,7 @@ class CASCache():
# used_size (int): Total size used by the local cache, in bytes.
# quota_size (int): Disk quota for the local cache, in bytes.
#
-class _CASCacheUsage():
-
+class _CASCacheUsage:
def __init__(self, used_size, quota_size):
self.used_size = used_size
self.quota_size = quota_size
@@ -1080,10 +1102,11 @@ class _CASCacheUsage():
elif self.quota_size is None:
return utils._pretty_size(self.used_size, dec_places=1)
else:
- return "{} / {} ({}%)" \
- .format(utils._pretty_size(self.used_size, dec_places=1),
- utils._pretty_size(self.quota_size, dec_places=1),
- self.used_percent)
+ return "{} / {} ({}%)".format(
+ utils._pretty_size(self.used_size, dec_places=1),
+ utils._pretty_size(self.quota_size, dec_places=1),
+ self.used_percent,
+ )
# _CASCacheUsageMonitor
diff --git a/src/buildstream/_cas/casremote.py b/src/buildstream/_cas/casremote.py
index a054b288a..ee6f4679c 100644
--- a/src/buildstream/_cas/casremote.py
+++ b/src/buildstream/_cas/casremote.py
@@ -32,7 +32,6 @@ _MAX_DIGESTS = _MAX_PAYLOAD_BYTES / 80
class BlobNotFound(CASRemoteError):
-
def __init__(self, blob, msg):
self.blob = blob
super().__init__(msg)
@@ -41,7 +40,6 @@ class BlobNotFound(CASRemoteError):
# Represents a single remote CAS cache.
#
class CASRemote(BaseRemote):
-
def __init__(self, spec, cascache, **kwargs):
super().__init__(spec, **kwargs)
@@ -90,7 +88,7 @@ class CASRemote(BaseRemote):
# Represents a batch of blobs queued for fetching.
#
-class _CASBatchRead():
+class _CASBatchRead:
def __init__(self, remote):
self._remote = remote
self._requests = []
@@ -123,22 +121,28 @@ class _CASBatchRead():
for response in batch_response.responses:
if response.status.code == code_pb2.NOT_FOUND:
if missing_blobs is None:
- raise BlobNotFound(response.digest.hash, "Failed to download blob {}: {}".format(
- response.digest.hash, response.status.code))
+ raise BlobNotFound(
+ response.digest.hash,
+ "Failed to download blob {}: {}".format(response.digest.hash, response.status.code),
+ )
missing_blobs.append(response.digest)
if response.status.code != code_pb2.OK:
- raise CASRemoteError("Failed to download blob {}: {}".format(
- response.digest.hash, response.status.code))
+ raise CASRemoteError(
+ "Failed to download blob {}: {}".format(response.digest.hash, response.status.code)
+ )
if response.digest.size_bytes != len(response.data):
- raise CASRemoteError("Failed to download blob {}: expected {} bytes, received {} bytes".format(
- response.digest.hash, response.digest.size_bytes, len(response.data)))
+ raise CASRemoteError(
+ "Failed to download blob {}: expected {} bytes, received {} bytes".format(
+ response.digest.hash, response.digest.size_bytes, len(response.data)
+ )
+ )
# Represents a batch of blobs queued for upload.
#
-class _CASBatchUpdate():
+class _CASBatchUpdate:
def __init__(self, remote):
self._remote = remote
self._requests = []
@@ -175,5 +179,7 @@ class _CASBatchUpdate():
else:
reason = None
- raise CASRemoteError("Failed to upload blob {}: {}".format(
- response.digest.hash, response.status.code), reason=reason)
+ raise CASRemoteError(
+ "Failed to upload blob {}: {}".format(response.digest.hash, response.status.code),
+ reason=reason,
+ )
diff --git a/src/buildstream/_cas/casserver.py b/src/buildstream/_cas/casserver.py
index d4241435a..a2110d8a2 100644
--- a/src/buildstream/_cas/casserver.py
+++ b/src/buildstream/_cas/casserver.py
@@ -33,8 +33,14 @@ import click
from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
from .._protos.google.bytestream import bytestream_pb2, bytestream_pb2_grpc
from .._protos.google.rpc import code_pb2
-from .._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc, \
- artifact_pb2, artifact_pb2_grpc, source_pb2, source_pb2_grpc
+from .._protos.buildstream.v2 import (
+ buildstream_pb2,
+ buildstream_pb2_grpc,
+ artifact_pb2,
+ artifact_pb2_grpc,
+ source_pb2,
+ source_pb2_grpc,
+)
from .. import utils
from .._exceptions import CASError, CASCacheError
@@ -61,8 +67,8 @@ def create_server(repo, *, enable_push, quota, index_only):
cas = CASCache(os.path.abspath(repo), cache_quota=quota, protect_session_blobs=False)
try:
- artifactdir = os.path.join(os.path.abspath(repo), 'artifacts', 'refs')
- sourcedir = os.path.join(os.path.abspath(repo), 'source_protos')
+ artifactdir = os.path.join(os.path.abspath(repo), "artifacts", "refs")
+ sourcedir = os.path.join(os.path.abspath(repo), "source_protos")
# Use max_workers default from Python 3.5+
max_workers = (os.cpu_count() or 1) * 5
@@ -70,31 +76,31 @@ def create_server(repo, *, enable_push, quota, index_only):
if not index_only:
bytestream_pb2_grpc.add_ByteStreamServicer_to_server(
- _ByteStreamServicer(cas, enable_push=enable_push), server)
+ _ByteStreamServicer(cas, enable_push=enable_push), server
+ )
remote_execution_pb2_grpc.add_ContentAddressableStorageServicer_to_server(
- _ContentAddressableStorageServicer(cas, enable_push=enable_push), server)
+ _ContentAddressableStorageServicer(cas, enable_push=enable_push), server
+ )
- remote_execution_pb2_grpc.add_CapabilitiesServicer_to_server(
- _CapabilitiesServicer(), server)
+ remote_execution_pb2_grpc.add_CapabilitiesServicer_to_server(_CapabilitiesServicer(), server)
buildstream_pb2_grpc.add_ReferenceStorageServicer_to_server(
- _ReferenceStorageServicer(cas, enable_push=enable_push), server)
+ _ReferenceStorageServicer(cas, enable_push=enable_push), server
+ )
artifact_pb2_grpc.add_ArtifactServiceServicer_to_server(
- _ArtifactServicer(cas, artifactdir, update_cas=not index_only), server)
+ _ArtifactServicer(cas, artifactdir, update_cas=not index_only), server
+ )
- source_pb2_grpc.add_SourceServiceServicer_to_server(
- _SourceServicer(sourcedir), server)
+ source_pb2_grpc.add_SourceServiceServicer_to_server(_SourceServicer(sourcedir), server)
# Create up reference storage and artifact capabilities
- artifact_capabilities = buildstream_pb2.ArtifactCapabilities(
- allow_updates=enable_push)
- source_capabilities = buildstream_pb2.SourceCapabilities(
- allow_updates=enable_push)
+ artifact_capabilities = buildstream_pb2.ArtifactCapabilities(allow_updates=enable_push)
+ source_capabilities = buildstream_pb2.SourceCapabilities(allow_updates=enable_push)
buildstream_pb2_grpc.add_CapabilitiesServicer_to_server(
- _BuildStreamCapabilitiesServicer(artifact_capabilities, source_capabilities),
- server)
+ _BuildStreamCapabilitiesServicer(artifact_capabilities, source_capabilities), server
+ )
yield server
@@ -103,28 +109,25 @@ def create_server(repo, *, enable_push, quota, index_only):
@click.command(short_help="CAS Artifact Server")
-@click.option('--port', '-p', type=click.INT, required=True, help="Port number")
-@click.option('--server-key', help="Private server key for TLS (PEM-encoded)")
-@click.option('--server-cert', help="Public server certificate for TLS (PEM-encoded)")
-@click.option('--client-certs', help="Public client certificates for TLS (PEM-encoded)")
-@click.option('--enable-push', is_flag=True,
- help="Allow clients to upload blobs and update artifact cache")
-@click.option('--quota', type=click.INT, default=10e9, show_default=True,
- help="Maximum disk usage in bytes")
-@click.option('--index-only', is_flag=True,
- help="Only provide the BuildStream artifact and source services (\"index\"), not the CAS (\"storage\")")
-@click.argument('repo')
-def server_main(repo, port, server_key, server_cert, client_certs, enable_push,
- quota, index_only):
+@click.option("--port", "-p", type=click.INT, required=True, help="Port number")
+@click.option("--server-key", help="Private server key for TLS (PEM-encoded)")
+@click.option("--server-cert", help="Public server certificate for TLS (PEM-encoded)")
+@click.option("--client-certs", help="Public client certificates for TLS (PEM-encoded)")
+@click.option("--enable-push", is_flag=True, help="Allow clients to upload blobs and update artifact cache")
+@click.option("--quota", type=click.INT, default=10e9, show_default=True, help="Maximum disk usage in bytes")
+@click.option(
+ "--index-only",
+ is_flag=True,
+ help='Only provide the BuildStream artifact and source services ("index"), not the CAS ("storage")',
+)
+@click.argument("repo")
+def server_main(repo, port, server_key, server_cert, client_certs, enable_push, quota, index_only):
# Handle SIGTERM by calling sys.exit(0), which will raise a SystemExit exception,
# properly executing cleanup code in `finally` clauses and context managers.
# This is required to terminate buildbox-casd on SIGTERM.
signal.signal(signal.SIGTERM, lambda signalnum, frame: sys.exit(0))
- with create_server(repo,
- quota=quota,
- enable_push=enable_push,
- index_only=index_only) as server:
+ with create_server(repo, quota=quota, enable_push=enable_push, index_only=index_only) as server:
use_tls = bool(server_key)
@@ -138,23 +141,25 @@ def server_main(repo, port, server_key, server_cert, client_certs, enable_push,
if use_tls:
# Read public/private key pair
- with open(server_key, 'rb') as f:
+ with open(server_key, "rb") as f:
server_key_bytes = f.read()
- with open(server_cert, 'rb') as f:
+ with open(server_cert, "rb") as f:
server_cert_bytes = f.read()
if client_certs:
- with open(client_certs, 'rb') as f:
+ with open(client_certs, "rb") as f:
client_certs_bytes = f.read()
else:
client_certs_bytes = None
- credentials = grpc.ssl_server_credentials([(server_key_bytes, server_cert_bytes)],
- root_certificates=client_certs_bytes,
- require_client_auth=bool(client_certs))
- server.add_secure_port('[::]:{}'.format(port), credentials)
+ credentials = grpc.ssl_server_credentials(
+ [(server_key_bytes, server_cert_bytes)],
+ root_certificates=client_certs_bytes,
+ require_client_auth=bool(client_certs),
+ )
+ server.add_secure_port("[::]:{}".format(port), credentials)
else:
- server.add_insecure_port('[::]:{}'.format(port))
+ server.add_insecure_port("[::]:{}".format(port))
# Run artifact server
server.start()
@@ -183,7 +188,7 @@ class _ByteStreamServicer(bytestream_pb2_grpc.ByteStreamServicer):
return
try:
- with open(self.cas.objpath(client_digest), 'rb') as f:
+ with open(self.cas.objpath(client_digest), "rb") as f:
if os.fstat(f.fileno()).st_size != client_digest.size_bytes:
context.set_code(grpc.StatusCode.NOT_FOUND)
return
@@ -317,7 +322,7 @@ class _ContentAddressableStorageServicer(remote_execution_pb2_grpc.ContentAddres
blob_response.digest.size_bytes = digest.size_bytes
try:
objpath = self.cas.objpath(digest)
- with open(objpath, 'rb') as f:
+ with open(objpath, "rb") as f:
if os.fstat(f.fileno()).st_size != digest.size_bytes:
blob_response.status.code = code_pb2.NOT_FOUND
continue
@@ -437,7 +442,6 @@ class _ReferenceStorageServicer(buildstream_pb2_grpc.ReferenceStorageServicer):
class _ArtifactServicer(artifact_pb2_grpc.ArtifactServiceServicer):
-
def __init__(self, cas, artifactdir, *, update_cas=True):
super().__init__()
self.cas = cas
@@ -451,7 +455,7 @@ class _ArtifactServicer(artifact_pb2_grpc.ArtifactServiceServicer):
context.abort(grpc.StatusCode.NOT_FOUND, "Artifact proto not found")
artifact = artifact_pb2.Artifact()
- with open(artifact_path, 'rb') as f:
+ with open(artifact_path, "rb") as f:
artifact.ParseFromString(f.read())
# Artifact-only servers will not have blobs on their system,
@@ -489,11 +493,9 @@ class _ArtifactServicer(artifact_pb2_grpc.ArtifactServiceServicer):
except FileNotFoundError:
os.unlink(artifact_path)
- context.abort(grpc.StatusCode.NOT_FOUND,
- "Artifact files incomplete")
+ context.abort(grpc.StatusCode.NOT_FOUND, "Artifact files incomplete")
except DecodeError:
- context.abort(grpc.StatusCode.NOT_FOUND,
- "Artifact files not valid")
+ context.abort(grpc.StatusCode.NOT_FOUND, "Artifact files not valid")
return artifact
@@ -516,7 +518,7 @@ class _ArtifactServicer(artifact_pb2_grpc.ArtifactServiceServicer):
# Add the artifact proto to the cas
artifact_path = os.path.join(self.artifactdir, request.cache_key)
os.makedirs(os.path.dirname(artifact_path), exist_ok=True)
- with utils.save_file_atomic(artifact_path, mode='wb') as f:
+ with utils.save_file_atomic(artifact_path, mode="wb") as f:
f.write(artifact.SerializeToString())
return artifact
@@ -527,19 +529,18 @@ class _ArtifactServicer(artifact_pb2_grpc.ArtifactServiceServicer):
def _check_directory(self, name, digest, context):
try:
directory = remote_execution_pb2.Directory()
- with open(self.cas.objpath(digest), 'rb') as f:
+ with open(self.cas.objpath(digest), "rb") as f:
directory.ParseFromString(f.read())
except FileNotFoundError:
- context.abort(grpc.StatusCode.FAILED_PRECONDITION,
- "Artifact {} specified but no files found".format(name))
+ context.abort(grpc.StatusCode.FAILED_PRECONDITION, "Artifact {} specified but no files found".format(name))
except DecodeError:
- context.abort(grpc.StatusCode.FAILED_PRECONDITION,
- "Artifact {} specified but directory not found".format(name))
+ context.abort(
+ grpc.StatusCode.FAILED_PRECONDITION, "Artifact {} specified but directory not found".format(name)
+ )
def _check_file(self, name, digest, context):
if not os.path.exists(self.cas.objpath(digest)):
- context.abort(grpc.StatusCode.FAILED_PRECONDITION,
- "Artifact {} specified but not found".format(name))
+ context.abort(grpc.StatusCode.FAILED_PRECONDITION, "Artifact {} specified but not found".format(name))
class _BuildStreamCapabilitiesServicer(buildstream_pb2_grpc.CapabilitiesServicer):
@@ -564,8 +565,7 @@ class _SourceServicer(source_pb2_grpc.SourceServiceServicer):
except FileNotFoundError:
context.abort(grpc.StatusCode.NOT_FOUND, "Source not found")
except DecodeError:
- context.abort(grpc.StatusCode.NOT_FOUND,
- "Sources gives invalid directory")
+ context.abort(grpc.StatusCode.NOT_FOUND, "Sources gives invalid directory")
return source_proto
@@ -576,7 +576,7 @@ class _SourceServicer(source_pb2_grpc.SourceServiceServicer):
def _get_source(self, cache_key):
path = os.path.join(self.sourcedir, cache_key)
source_proto = source_pb2.Source()
- with open(path, 'r+b') as f:
+ with open(path, "r+b") as f:
source_proto.ParseFromString(f.read())
os.utime(path)
return source_proto
@@ -584,18 +584,18 @@ class _SourceServicer(source_pb2_grpc.SourceServiceServicer):
def _set_source(self, cache_key, source_proto):
path = os.path.join(self.sourcedir, cache_key)
os.makedirs(os.path.dirname(path), exist_ok=True)
- with utils.save_file_atomic(path, 'w+b') as f:
+ with utils.save_file_atomic(path, "w+b") as f:
f.write(source_proto.SerializeToString())
def _digest_from_download_resource_name(resource_name):
- parts = resource_name.split('/')
+ parts = resource_name.split("/")
# Accept requests from non-conforming BuildStream 1.1.x clients
if len(parts) == 2:
- parts.insert(0, 'blobs')
+ parts.insert(0, "blobs")
- if len(parts) != 3 or parts[0] != 'blobs':
+ if len(parts) != 3 or parts[0] != "blobs":
return None
try:
@@ -608,15 +608,15 @@ def _digest_from_download_resource_name(resource_name):
def _digest_from_upload_resource_name(resource_name):
- parts = resource_name.split('/')
+ parts = resource_name.split("/")
# Accept requests from non-conforming BuildStream 1.1.x clients
if len(parts) == 2:
- parts.insert(0, 'uploads')
+ parts.insert(0, "uploads")
parts.insert(1, str(uuid.uuid4()))
- parts.insert(2, 'blobs')
+ parts.insert(2, "blobs")
- if len(parts) < 5 or parts[0] != 'uploads' or parts[2] != 'blobs':
+ if len(parts) < 5 or parts[0] != "uploads" or parts[2] != "blobs":
return None
try:
diff --git a/src/buildstream/_context.py b/src/buildstream/_context.py
index 4e1007e28..f426f4bb0 100644
--- a/src/buildstream/_context.py
+++ b/src/buildstream/_context.py
@@ -46,13 +46,12 @@ from .sandbox import SandboxRemote
# verbosity levels and basically anything pertaining to the context
# in which BuildStream was invoked.
#
-class Context():
-
+class Context:
def __init__(self, *, use_casd=True):
# Whether we are running as part of a test suite. This is only relevant
# for developing BuildStream itself.
- self.is_running_in_test_suite = 'BST_TEST_SUITE' in os.environ
+ self.is_running_in_test_suite = "BST_TEST_SUITE" in os.environ
# Filename indicating which configuration file was used, or None for the defaults
self.config_origin = None
@@ -216,8 +215,7 @@ class Context():
# a $XDG_CONFIG_HOME/buildstream.conf file
#
if not config:
- default_config = os.path.join(os.environ['XDG_CONFIG_HOME'],
- 'buildstream.conf')
+ default_config = os.path.join(os.environ["XDG_CONFIG_HOME"], "buildstream.conf")
if os.path.exists(default_config):
config = default_config
@@ -231,19 +229,32 @@ class Context():
user_config._composite(defaults)
# Give obsoletion warnings
- if 'builddir' in defaults:
+ if "builddir" in defaults:
raise LoadError("builddir is obsolete, use cachedir", LoadErrorReason.INVALID_DATA)
- if 'artifactdir' in defaults:
+ if "artifactdir" in defaults:
raise LoadError("artifactdir is obsolete", LoadErrorReason.INVALID_DATA)
- defaults.validate_keys([
- 'cachedir', 'sourcedir', 'builddir', 'logdir', 'scheduler', 'build',
- 'artifacts', 'source-caches', 'logging', 'projects', 'cache', 'prompt',
- 'workspacedir', 'remote-execution',
- ])
-
- for directory in ['cachedir', 'sourcedir', 'logdir', 'workspacedir']:
+ defaults.validate_keys(
+ [
+ "cachedir",
+ "sourcedir",
+ "builddir",
+ "logdir",
+ "scheduler",
+ "build",
+ "artifacts",
+ "source-caches",
+ "logging",
+ "projects",
+ "cache",
+ "prompt",
+ "workspacedir",
+ "remote-execution",
+ ]
+ )
+
+ for directory in ["cachedir", "sourcedir", "logdir", "workspacedir"]:
# Allow the ~ tilde expansion and any environment variables in
# path specification in the config files.
#
@@ -256,25 +267,23 @@ class Context():
# Relative paths don't make sense in user configuration. The exception is
# workspacedir where `.` is useful as it will be combined with the name
# specified on the command line.
- if not os.path.isabs(path) and not (directory == 'workspacedir' and path == '.'):
+ if not os.path.isabs(path) and not (directory == "workspacedir" and path == "."):
raise LoadError("{} must be an absolute path".format(directory), LoadErrorReason.INVALID_DATA)
# add directories not set by users
- self.tmpdir = os.path.join(self.cachedir, 'tmp')
- self.casdir = os.path.join(self.cachedir, 'cas')
- self.builddir = os.path.join(self.cachedir, 'build')
- self.artifactdir = os.path.join(self.cachedir, 'artifacts', 'refs')
+ self.tmpdir = os.path.join(self.cachedir, "tmp")
+ self.casdir = os.path.join(self.cachedir, "cas")
+ self.builddir = os.path.join(self.cachedir, "build")
+ self.artifactdir = os.path.join(self.cachedir, "artifacts", "refs")
# Move old artifact cas to cas if it exists and create symlink
- old_casdir = os.path.join(self.cachedir, 'artifacts', 'cas')
- if (os.path.exists(old_casdir) and not os.path.islink(old_casdir) and
- not os.path.exists(self.casdir)):
+ old_casdir = os.path.join(self.cachedir, "artifacts", "cas")
+ if os.path.exists(old_casdir) and not os.path.islink(old_casdir) and not os.path.exists(self.casdir):
os.rename(old_casdir, self.casdir)
os.symlink(self.casdir, old_casdir)
# Cleanup old extract directories
- old_extractdirs = [os.path.join(self.cachedir, 'artifacts', 'extract'),
- os.path.join(self.cachedir, 'extract')]
+ old_extractdirs = [os.path.join(self.cachedir, "artifacts", "extract"), os.path.join(self.cachedir, "extract")]
for old_extractdir in old_extractdirs:
if os.path.isdir(old_extractdir):
shutil.rmtree(old_extractdir, ignore_errors=True)
@@ -282,21 +291,22 @@ class Context():
# Load quota configuration
# We need to find the first existing directory in the path of our
# casdir - the casdir may not have been created yet.
- cache = defaults.get_mapping('cache')
- cache.validate_keys(['quota', 'pull-buildtrees', 'cache-buildtrees'])
+ cache = defaults.get_mapping("cache")
+ cache.validate_keys(["quota", "pull-buildtrees", "cache-buildtrees"])
cas_volume = self.casdir
while not os.path.exists(cas_volume):
cas_volume = os.path.dirname(cas_volume)
- self.config_cache_quota_string = cache.get_str('quota')
+ self.config_cache_quota_string = cache.get_str("quota")
try:
- self.config_cache_quota = utils._parse_size(self.config_cache_quota_string,
- cas_volume)
+ self.config_cache_quota = utils._parse_size(self.config_cache_quota_string, cas_volume)
except utils.UtilError as e:
- raise LoadError("{}\nPlease specify the value in bytes or as a % of full disk space.\n"
- "\nValid values are, for example: 800M 10G 1T 50%\n"
- .format(str(e)), LoadErrorReason.INVALID_DATA) from e
+ raise LoadError(
+ "{}\nPlease specify the value in bytes or as a % of full disk space.\n"
+ "\nValid values are, for example: 800M 10G 1T 50%\n".format(str(e)),
+ LoadErrorReason.INVALID_DATA,
+ ) from e
# Load artifact share configuration
self.artifact_cache_specs = ArtifactCache.specs_from_config_node(defaults)
@@ -305,73 +315,70 @@ class Context():
self.source_cache_specs = SourceCache.specs_from_config_node(defaults)
# Load remote execution config getting pull-artifact-files from it
- remote_execution = defaults.get_mapping('remote-execution', default=None)
+ remote_execution = defaults.get_mapping("remote-execution", default=None)
if remote_execution:
- self.pull_artifact_files = remote_execution.get_bool('pull-artifact-files', default=True)
+ self.pull_artifact_files = remote_execution.get_bool("pull-artifact-files", default=True)
# This stops it being used in the remote service set up
- remote_execution.safe_del('pull-artifact-files')
+ remote_execution.safe_del("pull-artifact-files")
# Don't pass the remote execution settings if that was the only option
if remote_execution.keys() == []:
- del defaults['remote-execution']
+ del defaults["remote-execution"]
else:
self.pull_artifact_files = True
self.remote_execution_specs = SandboxRemote.specs_from_config_node(defaults)
# Load pull build trees configuration
- self.pull_buildtrees = cache.get_bool('pull-buildtrees')
+ self.pull_buildtrees = cache.get_bool("pull-buildtrees")
# Load cache build trees configuration
- self.cache_buildtrees = cache.get_enum('cache-buildtrees', _CacheBuildTrees)
+ self.cache_buildtrees = cache.get_enum("cache-buildtrees", _CacheBuildTrees)
# Load logging config
- logging = defaults.get_mapping('logging')
- logging.validate_keys([
- 'key-length', 'verbose',
- 'error-lines', 'message-lines',
- 'debug', 'element-format', 'message-format'
- ])
- self.log_key_length = logging.get_int('key-length')
- self.log_debug = logging.get_bool('debug')
- self.log_verbose = logging.get_bool('verbose')
- self.log_error_lines = logging.get_int('error-lines')
- self.log_message_lines = logging.get_int('message-lines')
- self.log_element_format = logging.get_str('element-format')
- self.log_message_format = logging.get_str('message-format')
+ logging = defaults.get_mapping("logging")
+ logging.validate_keys(
+ ["key-length", "verbose", "error-lines", "message-lines", "debug", "element-format", "message-format"]
+ )
+ self.log_key_length = logging.get_int("key-length")
+ self.log_debug = logging.get_bool("debug")
+ self.log_verbose = logging.get_bool("verbose")
+ self.log_error_lines = logging.get_int("error-lines")
+ self.log_message_lines = logging.get_int("message-lines")
+ self.log_element_format = logging.get_str("element-format")
+ self.log_message_format = logging.get_str("message-format")
# Load scheduler config
- scheduler = defaults.get_mapping('scheduler')
- scheduler.validate_keys([
- 'on-error', 'fetchers', 'builders',
- 'pushers', 'network-retries'
- ])
- self.sched_error_action = scheduler.get_enum('on-error', _SchedulerErrorAction)
- self.sched_fetchers = scheduler.get_int('fetchers')
- self.sched_builders = scheduler.get_int('builders')
- self.sched_pushers = scheduler.get_int('pushers')
- self.sched_network_retries = scheduler.get_int('network-retries')
+ scheduler = defaults.get_mapping("scheduler")
+ scheduler.validate_keys(["on-error", "fetchers", "builders", "pushers", "network-retries"])
+ self.sched_error_action = scheduler.get_enum("on-error", _SchedulerErrorAction)
+ self.sched_fetchers = scheduler.get_int("fetchers")
+ self.sched_builders = scheduler.get_int("builders")
+ self.sched_pushers = scheduler.get_int("pushers")
+ self.sched_network_retries = scheduler.get_int("network-retries")
# Load build config
- build = defaults.get_mapping('build')
- build.validate_keys(['max-jobs', 'dependencies'])
- self.build_max_jobs = build.get_int('max-jobs')
-
- self.build_dependencies = build.get_str('dependencies')
- if self.build_dependencies not in ['plan', 'all']:
- provenance = build.get_scalar('dependencies').get_provenance()
- raise LoadError("{}: Invalid value for 'dependencies'. Choose 'plan' or 'all'."
- .format(provenance), LoadErrorReason.INVALID_DATA)
+ build = defaults.get_mapping("build")
+ build.validate_keys(["max-jobs", "dependencies"])
+ self.build_max_jobs = build.get_int("max-jobs")
+
+ self.build_dependencies = build.get_str("dependencies")
+ if self.build_dependencies not in ["plan", "all"]:
+ provenance = build.get_scalar("dependencies").get_provenance()
+ raise LoadError(
+ "{}: Invalid value for 'dependencies'. Choose 'plan' or 'all'.".format(provenance),
+ LoadErrorReason.INVALID_DATA,
+ )
# Load per-projects overrides
- self._project_overrides = defaults.get_mapping('projects', default={})
+ self._project_overrides = defaults.get_mapping("projects", default={})
# Shallow validation of overrides, parts of buildstream which rely
# on the overrides are expected to validate elsewhere.
for overrides_project in self._project_overrides.keys():
overrides = self._project_overrides.get_mapping(overrides_project)
- overrides.validate_keys(['artifacts', 'source-caches', 'options',
- 'strict', 'default-mirror',
- 'remote-execution'])
+ overrides.validate_keys(
+ ["artifacts", "source-caches", "options", "strict", "default-mirror", "remote-execution"]
+ )
@property
def platform(self):
@@ -474,7 +481,7 @@ class Context():
# so work out if we should be strict, and then cache the result
toplevel = self.get_toplevel_project()
overrides = self.get_overrides(toplevel.name)
- self._strict_build_plan = overrides.get_bool('strict', default=True)
+ self._strict_build_plan = overrides.get_bool("strict", default=True)
# If it was set by the CLI, it overrides any config
# Ditto if we've already computed this, then we return the computed
@@ -505,12 +512,12 @@ class Context():
# preferred locations of things from user configuration
# files.
def _init_xdg(self):
- if not os.environ.get('XDG_CACHE_HOME'):
- os.environ['XDG_CACHE_HOME'] = os.path.expanduser('~/.cache')
- if not os.environ.get('XDG_CONFIG_HOME'):
- os.environ['XDG_CONFIG_HOME'] = os.path.expanduser('~/.config')
- if not os.environ.get('XDG_DATA_HOME'):
- os.environ['XDG_DATA_HOME'] = os.path.expanduser('~/.local/share')
+ if not os.environ.get("XDG_CACHE_HOME"):
+ os.environ["XDG_CACHE_HOME"] = os.path.expanduser("~/.cache")
+ if not os.environ.get("XDG_CONFIG_HOME"):
+ os.environ["XDG_CONFIG_HOME"] = os.path.expanduser("~/.config")
+ if not os.environ.get("XDG_DATA_HOME"):
+ os.environ["XDG_DATA_HOME"] = os.path.expanduser("~/.local/share")
def get_cascache(self):
if self._cascache is None:
@@ -521,10 +528,9 @@ class Context():
else:
log_level = CASLogLevel.WARNING
- self._cascache = CASCache(self.cachedir,
- casd=self.use_casd,
- cache_quota=self.config_cache_quota,
- log_level=log_level)
+ self._cascache = CASCache(
+ self.cachedir, casd=self.use_casd, cache_quota=self.config_cache_quota, log_level=log_level
+ )
return self._cascache
# prepare_fork():
diff --git a/src/buildstream/_elementfactory.py b/src/buildstream/_elementfactory.py
index d6591bf4c..5d219c627 100644
--- a/src/buildstream/_elementfactory.py
+++ b/src/buildstream/_elementfactory.py
@@ -30,14 +30,15 @@ from .element import Element
# plugin_origins (list): Data used to search for external Element plugins
#
class ElementFactory(PluginContext):
+ def __init__(self, plugin_base, *, format_versions={}, plugin_origins=None):
- def __init__(self, plugin_base, *,
- format_versions={},
- plugin_origins=None):
-
- super().__init__(plugin_base, Element, [_site.element_plugins],
- plugin_origins=plugin_origins,
- format_versions=format_versions)
+ super().__init__(
+ plugin_base,
+ Element,
+ [_site.element_plugins],
+ plugin_origins=plugin_origins,
+ format_versions=format_versions,
+ )
# create():
#
diff --git a/src/buildstream/_exceptions.py b/src/buildstream/_exceptions.py
index 46de90796..ca17577f7 100644
--- a/src/buildstream/_exceptions.py
+++ b/src/buildstream/_exceptions.py
@@ -51,7 +51,7 @@ def get_last_exception():
# Used by regression tests
#
def get_last_task_error():
- if 'BST_TEST_SUITE' not in os.environ:
+ if "BST_TEST_SUITE" not in os.environ:
raise BstError("Getting the last task error is only supported when running tests")
global _last_task_error_domain
@@ -71,7 +71,7 @@ def get_last_task_error():
# tests about how things failed in a machine readable way
#
def set_last_task_error(domain, reason):
- if 'BST_TEST_SUITE' in os.environ:
+ if "BST_TEST_SUITE" in os.environ:
global _last_task_error_domain
global _last_task_error_reason
@@ -108,7 +108,6 @@ class ErrorDomain(Enum):
# context can then be communicated back to the main process.
#
class BstError(Exception):
-
def __init__(self, message, *, detail=None, domain=None, reason=None, temporary=False):
global _last_exception
@@ -133,7 +132,7 @@ class BstError(Exception):
self.reason = reason
# Hold on to the last raised exception for testing purposes
- if 'BST_TEST_SUITE' in os.environ:
+ if "BST_TEST_SUITE" in os.environ:
_last_exception = self
@@ -330,7 +329,6 @@ class CASCacheError(CASError):
# Raised from pipeline operations
#
class PipelineError(BstError):
-
def __init__(self, message, *, detail=None, reason=None):
super().__init__(message, detail=detail, domain=ErrorDomain.PIPELINE, reason=reason)
@@ -340,7 +338,6 @@ class PipelineError(BstError):
# Raised when a stream operation fails
#
class StreamError(BstError):
-
def __init__(self, message=None, *, detail=None, reason=None, terminated=False):
# The empty string should never appear to a user,
diff --git a/src/buildstream/_frontend/app.py b/src/buildstream/_frontend/app.py
index 99e164358..09610851f 100644
--- a/src/buildstream/_frontend/app.py
+++ b/src/buildstream/_frontend/app.py
@@ -56,19 +56,18 @@ INDENT = 4
# main_options (dict): The main CLI options of the `bst`
# command, before any subcommand
#
-class App():
-
+class App:
def __init__(self, main_options):
#
# Public members
#
- self.context = None # The Context object
- self.stream = None # The Stream object
- self.project = None # The toplevel Project object
- self.logger = None # The LogLine object
- self.interactive = None # Whether we are running in interactive mode
- self.colors = None # Whether to use colors in logging
+ self.context = None # The Context object
+ self.stream = None # The Stream object
+ self.project = None # The toplevel Project object
+ self.logger = None # The LogLine object
+ self.interactive = None # Whether we are running in interactive mode
+ self.colors = None # Whether to use colors in logging
#
# Private members
@@ -76,18 +75,18 @@ class App():
self._session_start = datetime.datetime.now()
self._session_name = None
self._main_options = main_options # Main CLI options, before any command
- self._status = None # The Status object
- self._fail_messages = {} # Failure messages by unique plugin id
+ self._status = None # The Status object
+ self._fail_messages = {} # Failure messages by unique plugin id
self._interactive_failures = None # Whether to handle failures interactively
- self._started = False # Whether a session has started
- self._set_project_dir = False # Whether -C option was used
- self._state = None # Frontend reads this and registers callbacks
+ self._started = False # Whether a session has started
+ self._set_project_dir = False # Whether -C option was used
+ self._state = None # Frontend reads this and registers callbacks
# UI Colors Profiles
- self._content_profile = Profile(fg='yellow')
- self._format_profile = Profile(fg='cyan', dim=True)
- self._success_profile = Profile(fg='green')
- self._error_profile = Profile(fg='red', dim=True)
+ self._content_profile = Profile(fg="yellow")
+ self._format_profile = Profile(fg="cyan", dim=True)
+ self._success_profile = Profile(fg="green")
+ self._error_profile = Profile(fg="red", dim=True)
self._detail_profile = Profile(dim=True)
#
@@ -96,31 +95,31 @@ class App():
is_a_tty = sys.stdout.isatty() and sys.stderr.isatty()
# Enable interactive mode if we're attached to a tty
- if main_options['no_interactive']:
+ if main_options["no_interactive"]:
self.interactive = False
else:
self.interactive = is_a_tty
# Handle errors interactively if we're in interactive mode
# and --on-error was not specified on the command line
- if main_options.get('on_error') is not None:
+ if main_options.get("on_error") is not None:
self._interactive_failures = False
else:
self._interactive_failures = self.interactive
# Use color output if we're attached to a tty, unless
# otherwise specified on the command line
- if main_options['colors'] is None:
+ if main_options["colors"] is None:
self.colors = is_a_tty
- elif main_options['colors']:
+ elif main_options["colors"]:
self.colors = True
else:
self.colors = False
- if main_options['directory']:
+ if main_options["directory"]:
self._set_project_dir = True
else:
- main_options['directory'] = os.getcwd()
+ main_options["directory"] = os.getcwd()
# create()
#
@@ -133,9 +132,10 @@ class App():
#
@classmethod
def create(cls, *args, **kwargs):
- if sys.platform.startswith('linux'):
+ if sys.platform.startswith("linux"):
# Use an App with linux specific features
from .linuxapp import LinuxApp # pylint: disable=cyclic-import
+
return LinuxApp(*args, **kwargs)
else:
# The base App() class is default
@@ -163,8 +163,8 @@ class App():
#
@contextmanager
def initialized(self, *, session_name=None):
- directory = self._main_options['directory']
- config = self._main_options['config']
+ directory = self._main_options["directory"]
+ config = self._main_options["config"]
self._session_name = session_name
@@ -184,19 +184,19 @@ class App():
# the command line when used, trumps the config files.
#
override_map = {
- 'strict': '_strict_build_plan',
- 'debug': 'log_debug',
- 'verbose': 'log_verbose',
- 'error_lines': 'log_error_lines',
- 'message_lines': 'log_message_lines',
- 'on_error': 'sched_error_action',
- 'fetchers': 'sched_fetchers',
- 'builders': 'sched_builders',
- 'pushers': 'sched_pushers',
- 'max_jobs': 'build_max_jobs',
- 'network_retries': 'sched_network_retries',
- 'pull_buildtrees': 'pull_buildtrees',
- 'cache_buildtrees': 'cache_buildtrees'
+ "strict": "_strict_build_plan",
+ "debug": "log_debug",
+ "verbose": "log_verbose",
+ "error_lines": "log_error_lines",
+ "message_lines": "log_message_lines",
+ "on_error": "sched_error_action",
+ "fetchers": "sched_fetchers",
+ "builders": "sched_builders",
+ "pushers": "sched_pushers",
+ "max_jobs": "build_max_jobs",
+ "network_retries": "sched_network_retries",
+ "pull_buildtrees": "pull_buildtrees",
+ "cache_buildtrees": "cache_buildtrees",
}
for cli_option, context_attr in override_map.items():
option_value = self._main_options.get(cli_option)
@@ -208,10 +208,13 @@ class App():
self._error_exit(e, "Error instantiating platform")
# Create the stream right away, we'll need to pass it around.
- self.stream = Stream(self.context, self._session_start,
- session_start_callback=self.session_start_cb,
- interrupt_callback=self._interrupt_handler,
- ticker_callback=self._tick)
+ self.stream = Stream(
+ self.context,
+ self._session_start,
+ session_start_callback=self.session_start_cb,
+ interrupt_callback=self._interrupt_handler,
+ ticker_callback=self._tick,
+ )
self._state = self.stream.get_state()
@@ -219,13 +222,16 @@ class App():
self._state.register_task_failed_callback(self._job_failed)
# Create the logger right before setting the message handler
- self.logger = LogLine(self.context, self._state,
- self._content_profile,
- self._format_profile,
- self._success_profile,
- self._error_profile,
- self._detail_profile,
- indent=INDENT)
+ self.logger = LogLine(
+ self.context,
+ self._state,
+ self._content_profile,
+ self._format_profile,
+ self._success_profile,
+ self._error_profile,
+ self._detail_profile,
+ indent=INDENT,
+ )
# Propagate pipeline feedback to the user
self.context.messenger.set_message_handler(self._message_handler)
@@ -248,10 +254,15 @@ class App():
self.stream.init()
# Create our status printer, only available in interactive
- self._status = Status(self.context, self._state,
- self._content_profile, self._format_profile,
- self._success_profile, self._error_profile,
- self.stream)
+ self._status = Status(
+ self.context,
+ self._state,
+ self._content_profile,
+ self._format_profile,
+ self._success_profile,
+ self._error_profile,
+ self.stream,
+ )
# Mark the beginning of the session
if session_name:
@@ -261,9 +272,13 @@ class App():
# Load the Project
#
try:
- self.project = Project(directory, self.context, cli_options=self._main_options['option'],
- default_mirror=self._main_options.get('default_mirror'),
- fetch_subprojects=self.stream.fetch_subprojects)
+ self.project = Project(
+ directory,
+ self.context,
+ cli_options=self._main_options["option"],
+ default_mirror=self._main_options.get("default_mirror"),
+ fetch_subprojects=self.stream.fetch_subprojects,
+ )
self.stream.set_project(self.project)
except LoadError as e:
@@ -291,7 +306,7 @@ class App():
elapsed = self.stream.elapsed_time
if isinstance(e, StreamError) and e.terminated: # pylint: disable=no-member
- self._message(MessageType.WARN, session_name + ' Terminated', elapsed=elapsed)
+ self._message(MessageType.WARN, session_name + " Terminated", elapsed=elapsed)
else:
self._message(MessageType.FAIL, session_name, elapsed=elapsed)
@@ -304,8 +319,9 @@ class App():
# Exit with the error
self._error_exit(e)
except RecursionError:
- click.echo("RecursionError: Dependency depth is too large. Maximum recursion depth exceeded.",
- err=True)
+ click.echo(
+ "RecursionError: Dependency depth is too large. Maximum recursion depth exceeded.", err=True
+ )
sys.exit(-1)
else:
@@ -331,41 +347,51 @@ class App():
# force (bool): Allow overwriting an existing project.conf
# target_directory (str): The target directory the project should be initialized in
#
- def init_project(self, project_name, format_version=BST_FORMAT_VERSION, element_path='elements',
- force=False, target_directory=None):
+ def init_project(
+ self,
+ project_name,
+ format_version=BST_FORMAT_VERSION,
+ element_path="elements",
+ force=False,
+ target_directory=None,
+ ):
if target_directory:
directory = os.path.abspath(target_directory)
else:
- directory = self._main_options['directory']
+ directory = self._main_options["directory"]
directory = os.path.abspath(directory)
- project_path = os.path.join(directory, 'project.conf')
+ project_path = os.path.join(directory, "project.conf")
try:
if self._set_project_dir:
- raise AppError("Attempted to use -C or --directory with init.",
- reason='init-with-set-directory',
- detail="Please use 'bst init {}' instead.".format(directory))
+ raise AppError(
+ "Attempted to use -C or --directory with init.",
+ reason="init-with-set-directory",
+ detail="Please use 'bst init {}' instead.".format(directory),
+ )
# Abort if the project.conf already exists, unless `--force` was specified in `bst init`
if not force and os.path.exists(project_path):
- raise AppError("A project.conf already exists at: {}".format(project_path),
- reason='project-exists')
+ raise AppError("A project.conf already exists at: {}".format(project_path), reason="project-exists")
if project_name:
# If project name was specified, user interaction is not desired, just
# perform some validation and write the project.conf
- node._assert_symbol_name(project_name, 'project name')
+ node._assert_symbol_name(project_name, "project name")
self._assert_format_version(format_version)
self._assert_element_path(element_path)
elif not self.interactive:
- raise AppError("Cannot initialize a new project without specifying the project name",
- reason='unspecified-project-name')
+ raise AppError(
+ "Cannot initialize a new project without specifying the project name",
+ reason="unspecified-project-name",
+ )
else:
# Collect the parameters using an interactive session
- project_name, format_version, element_path = \
- self._init_project_interactive(project_name, format_version, element_path)
+ project_name, format_version, element_path = self._init_project_interactive(
+ project_name, format_version, element_path
+ )
# Create the directory if it doesnt exist
try:
@@ -378,20 +404,21 @@ class App():
try:
os.makedirs(elements_path, exist_ok=True)
except IOError as e:
- raise AppError("Error creating elements sub-directory {}: {}"
- .format(elements_path, e)) from e
+ raise AppError("Error creating elements sub-directory {}: {}".format(elements_path, e)) from e
# Dont use ruamel.yaml here, because it doesnt let
# us programatically insert comments or whitespace at
# the toplevel.
try:
- with open(project_path, 'w') as f:
- f.write("# Unique project name\n" +
- "name: {}\n\n".format(project_name) +
- "# Required BuildStream format version\n" +
- "format-version: {}\n\n".format(format_version) +
- "# Subdirectory where elements are stored\n" +
- "element-path: {}\n".format(element_path))
+ with open(project_path, "w") as f:
+ f.write(
+ "# Unique project name\n"
+ + "name: {}\n\n".format(project_name)
+ + "# Required BuildStream format version\n"
+ + "format-version: {}\n\n".format(format_version)
+ + "# Subdirectory where elements are stored\n"
+ + "element-path: {}\n".format(element_path)
+ )
except IOError as e:
raise AppError("Error writing {}: {}".format(project_path, e)) from e
@@ -419,15 +446,18 @@ class App():
_, key, dim = element_key
if self.colors:
- prompt = self._format_profile.fmt('[') + \
- self._content_profile.fmt(key, dim=dim) + \
- self._format_profile.fmt('@') + \
- self._content_profile.fmt(element_name) + \
- self._format_profile.fmt(':') + \
- self._content_profile.fmt('$PWD') + \
- self._format_profile.fmt(']$') + ' '
+ prompt = (
+ self._format_profile.fmt("[")
+ + self._content_profile.fmt(key, dim=dim)
+ + self._format_profile.fmt("@")
+ + self._content_profile.fmt(element_name)
+ + self._format_profile.fmt(":")
+ + self._content_profile.fmt("$PWD")
+ + self._format_profile.fmt("]$")
+ + " "
+ )
else:
- prompt = '[{}@{}:${{PWD}}]$ '.format(key, element_name)
+ prompt = "[{}@{}:${{PWD}}]$ ".format(key, element_name)
return prompt
@@ -473,8 +503,7 @@ class App():
#
def _message(self, message_type, message, **kwargs):
args = dict(kwargs)
- self.context.messenger.message(
- Message(message_type, message, **args))
+ self.context.messenger.message(Message(message_type, message, **args))
# Exception handler
#
@@ -482,8 +511,7 @@ class App():
# Print the regular BUG message
formatted = "".join(traceback.format_exception(etype, value, tb))
- self._message(MessageType.BUG, str(value),
- detail=formatted)
+ self._message(MessageType.BUG, str(value), detail=formatted)
# If the scheduler has started, try to terminate all jobs gracefully,
# otherwise exit immediately.
@@ -498,8 +526,7 @@ class App():
def _maybe_render_status(self):
# If we're suspended or terminating, then dont render the status area
- if self._status and self.stream and \
- not (self.stream.suspended or self.stream.terminated):
+ if self._status and self.stream and not (self.stream.suspended or self.stream.terminated):
self._status.render()
#
@@ -518,36 +545,40 @@ class App():
# the currently ongoing tasks. We can also print something more
# intelligent, like how many tasks remain to complete overall.
with self._interrupted():
- click.echo("\nUser interrupted with ^C\n" +
- "\n"
- "Choose one of the following options:\n" +
- " (c)ontinue - Continue queueing jobs as much as possible\n" +
- " (q)uit - Exit after all ongoing jobs complete\n" +
- " (t)erminate - Terminate any ongoing jobs and exit\n" +
- "\n" +
- "Pressing ^C again will terminate jobs and exit\n",
- err=True)
+ click.echo(
+ "\nUser interrupted with ^C\n" + "\n"
+ "Choose one of the following options:\n"
+ + " (c)ontinue - Continue queueing jobs as much as possible\n"
+ + " (q)uit - Exit after all ongoing jobs complete\n"
+ + " (t)erminate - Terminate any ongoing jobs and exit\n"
+ + "\n"
+ + "Pressing ^C again will terminate jobs and exit\n",
+ err=True,
+ )
try:
- choice = click.prompt("Choice:",
- value_proc=_prefix_choice_value_proc(['continue', 'quit', 'terminate']),
- default='continue', err=True)
+ choice = click.prompt(
+ "Choice:",
+ value_proc=_prefix_choice_value_proc(["continue", "quit", "terminate"]),
+ default="continue",
+ err=True,
+ )
except (click.Abort, SystemError):
# In some cases, the readline buffer underlying the prompt gets corrupted on the second CTRL+C
# This throws a SystemError, which doesn't seem to be problematic for the rest of the program
# Ensure a newline after automatically printed '^C'
click.echo("", err=True)
- choice = 'terminate'
+ choice = "terminate"
- if choice == 'terminate':
+ if choice == "terminate":
click.echo("\nTerminating all jobs at user request\n", err=True)
self.stream.terminate()
else:
- if choice == 'quit':
+ if choice == "quit":
click.echo("\nCompleting ongoing tasks before quitting\n", err=True)
self.stream.quit()
- elif choice == 'continue':
+ elif choice == "continue":
click.echo("\nContinuing\n", err=True)
def _tick(self):
@@ -577,9 +608,11 @@ class App():
# the failure message reaches us ??
if not failure:
self._status.clear()
- click.echo("\n\n\nBUG: Message handling out of sync, " +
- "unable to retrieve failure message for element {}\n\n\n\n\n"
- .format(full_name), err=True)
+ click.echo(
+ "\n\n\nBUG: Message handling out of sync, "
+ + "unable to retrieve failure message for element {}\n\n\n\n\n".format(full_name),
+ err=True,
+ )
else:
self._handle_failure(element, action_name, failure, full_name)
@@ -604,69 +637,72 @@ class App():
# Interactive mode for element failures
with self._interrupted():
- summary = ("\n{} failure on element: {}\n".format(failure.action_name, full_name) +
- "\n" +
- "Choose one of the following options:\n" +
- " (c)ontinue - Continue queueing jobs as much as possible\n" +
- " (q)uit - Exit after all ongoing jobs complete\n" +
- " (t)erminate - Terminate any ongoing jobs and exit\n" +
- " (r)etry - Retry this job\n")
+ summary = (
+ "\n{} failure on element: {}\n".format(failure.action_name, full_name)
+ + "\n"
+ + "Choose one of the following options:\n"
+ + " (c)ontinue - Continue queueing jobs as much as possible\n"
+ + " (q)uit - Exit after all ongoing jobs complete\n"
+ + " (t)erminate - Terminate any ongoing jobs and exit\n"
+ + " (r)etry - Retry this job\n"
+ )
if failure.logfile:
summary += " (l)og - View the full log file\n"
if failure.sandbox:
summary += " (s)hell - Drop into a shell in the failed build sandbox\n"
summary += "\nPressing ^C will terminate jobs and exit\n"
- choices = ['continue', 'quit', 'terminate', 'retry']
+ choices = ["continue", "quit", "terminate", "retry"]
if failure.logfile:
- choices += ['log']
+ choices += ["log"]
if failure.sandbox:
- choices += ['shell']
+ choices += ["shell"]
- choice = ''
- while choice not in ['continue', 'quit', 'terminate', 'retry']:
+ choice = ""
+ while choice not in ["continue", "quit", "terminate", "retry"]:
click.echo(summary, err=True)
- self._notify("BuildStream failure", "{} on element {}"
- .format(failure.action_name, full_name))
+ self._notify("BuildStream failure", "{} on element {}".format(failure.action_name, full_name))
try:
- choice = click.prompt("Choice:", default='continue', err=True,
- value_proc=_prefix_choice_value_proc(choices))
+ choice = click.prompt(
+ "Choice:", default="continue", err=True, value_proc=_prefix_choice_value_proc(choices)
+ )
except (click.Abort, SystemError):
# In some cases, the readline buffer underlying the prompt gets corrupted on the second CTRL+C
# This throws a SystemError, which doesn't seem to be problematic for the rest of the program
# Ensure a newline after automatically printed '^C'
click.echo("", err=True)
- choice = 'terminate'
+ choice = "terminate"
# Handle choices which you can come back from
#
- if choice == 'shell':
+ if choice == "shell":
click.echo("\nDropping into an interactive shell in the failed build sandbox\n", err=True)
try:
unique_id, element_key = element
prompt = self.shell_prompt(full_name, element_key)
- self.stream.shell(None, Scope.BUILD, prompt, isolate=True,
- usebuildtree='always', unique_id=unique_id)
+ self.stream.shell(
+ None, Scope.BUILD, prompt, isolate=True, usebuildtree="always", unique_id=unique_id
+ )
except BstError as e:
click.echo("Error while attempting to create interactive shell: {}".format(e), err=True)
- elif choice == 'log':
- with open(failure.logfile, 'r') as logfile:
+ elif choice == "log":
+ with open(failure.logfile, "r") as logfile:
content = logfile.read()
click.echo_via_pager(content)
- if choice == 'terminate':
+ if choice == "terminate":
click.echo("\nTerminating all jobs\n", err=True)
self.stream.terminate()
else:
- if choice == 'quit':
+ if choice == "quit":
click.echo("\nCompleting ongoing tasks before quitting\n", err=True)
self.stream.quit()
- elif choice == 'continue':
+ elif choice == "continue":
click.echo("\nContinuing with other non failing elements\n", err=True)
- elif choice == 'retry':
+ elif choice == "retry":
click.echo("\nRetrying failed job\n", err=True)
unique_id = element[0]
self.stream._failure_retry(action_name, unique_id)
@@ -678,17 +714,14 @@ class App():
def session_start_cb(self):
self._started = True
if self._session_name:
- self.logger.print_heading(self.project,
- self.stream,
- log_file=self._main_options['log_file'])
+ self.logger.print_heading(self.project, self.stream, log_file=self._main_options["log_file"])
#
# Print a summary of the queues
#
def _print_summary(self):
click.echo("", err=True)
- self.logger.print_summary(self.stream,
- self._main_options['log_file'])
+ self.logger.print_summary(self.stream, self._main_options["log_file"])
# _error_exit()
#
@@ -720,7 +753,7 @@ class App():
click.echo(main_error, err=True)
if error.detail:
indent = " " * INDENT
- detail = '\n' + indent + indent.join(error.detail.splitlines(True))
+ detail = "\n" + indent + indent.join(error.detail.splitlines(True))
click.echo(detail, err=True)
sys.exit(-1)
@@ -753,8 +786,8 @@ class App():
self._maybe_render_status()
# Additionally log to a file
- if self._main_options['log_file']:
- click.echo(text, file=self._main_options['log_file'], color=False, nl=False)
+ if self._main_options["log_file"]:
+ click.echo(text, file=self._main_options["log_file"], color=False, nl=False)
@contextmanager
def _interrupted(self):
@@ -768,25 +801,26 @@ class App():
# Some validation routines for project initialization
#
def _assert_format_version(self, format_version):
- message = "The version must be supported by this " + \
- "version of buildstream (0 - {})\n".format(BST_FORMAT_VERSION)
+ message = "The version must be supported by this " + "version of buildstream (0 - {})\n".format(
+ BST_FORMAT_VERSION
+ )
# Validate that it is an integer
try:
number = int(format_version)
except ValueError as e:
- raise AppError(message, reason='invalid-format-version') from e
+ raise AppError(message, reason="invalid-format-version") from e
# Validate that the specified version is supported
if number < 0 or number > BST_FORMAT_VERSION:
- raise AppError(message, reason='invalid-format-version')
+ raise AppError(message, reason="invalid-format-version")
def _assert_element_path(self, element_path):
message = "The element path cannot be an absolute path or contain any '..' components\n"
# Validate the path is not absolute
if os.path.isabs(element_path):
- raise AppError(message, reason='invalid-element-path')
+ raise AppError(message, reason="invalid-element-path")
# Validate that the path does not contain any '..' components
path = element_path
@@ -794,8 +828,8 @@ class App():
split = os.path.split(path)
path = split[0]
basename = split[1]
- if basename == '..':
- raise AppError(message, reason='invalid-element-path')
+ if basename == "..":
+ raise AppError(message, reason="invalid-element-path")
# _init_project_interactive()
#
@@ -811,11 +845,10 @@ class App():
# format_version (int): The user selected format version
# element_path (str): The user selected element path
#
- def _init_project_interactive(self, project_name, format_version=BST_FORMAT_VERSION, element_path='elements'):
-
+ def _init_project_interactive(self, project_name, format_version=BST_FORMAT_VERSION, element_path="elements"):
def project_name_proc(user_input):
try:
- node._assert_symbol_name(user_input, 'project name')
+ node._assert_symbol_name(user_input, "project name")
except LoadError as e:
message = "{}\n\n{}\n".format(e, e.detail)
raise UsageError(message) from e
@@ -835,63 +868,101 @@ class App():
raise UsageError(str(e)) from e
return user_input
- w = TextWrapper(initial_indent=' ', subsequent_indent=' ', width=79)
+ w = TextWrapper(initial_indent=" ", subsequent_indent=" ", width=79)
# Collect project name
click.echo("", err=True)
click.echo(self._content_profile.fmt("Choose a unique name for your project"), err=True)
click.echo(self._format_profile.fmt("-------------------------------------"), err=True)
click.echo("", err=True)
- click.echo(self._detail_profile.fmt(
- w.fill("The project name is a unique symbol for your project and will be used "
- "to distinguish your project from others in user preferences, namspaceing "
- "of your project's artifacts in shared artifact caches, and in any case where "
- "BuildStream needs to distinguish between multiple projects.")), err=True)
+ click.echo(
+ self._detail_profile.fmt(
+ w.fill(
+ "The project name is a unique symbol for your project and will be used "
+ "to distinguish your project from others in user preferences, namspaceing "
+ "of your project's artifacts in shared artifact caches, and in any case where "
+ "BuildStream needs to distinguish between multiple projects."
+ )
+ ),
+ err=True,
+ )
click.echo("", err=True)
- click.echo(self._detail_profile.fmt(
- w.fill("The project name must contain only alphanumeric characters, "
- "may not start with a digit, and may contain dashes or underscores.")), err=True)
+ click.echo(
+ self._detail_profile.fmt(
+ w.fill(
+ "The project name must contain only alphanumeric characters, "
+ "may not start with a digit, and may contain dashes or underscores."
+ )
+ ),
+ err=True,
+ )
click.echo("", err=True)
- project_name = click.prompt(self._content_profile.fmt("Project name"),
- value_proc=project_name_proc, err=True)
+ project_name = click.prompt(self._content_profile.fmt("Project name"), value_proc=project_name_proc, err=True)
click.echo("", err=True)
# Collect format version
click.echo(self._content_profile.fmt("Select the minimum required format version for your project"), err=True)
click.echo(self._format_profile.fmt("-----------------------------------------------------------"), err=True)
click.echo("", err=True)
- click.echo(self._detail_profile.fmt(
- w.fill("The format version is used to provide users who build your project "
- "with a helpful error message in the case that they do not have a recent "
- "enough version of BuildStream supporting all the features which your "
- "project might use.")), err=True)
+ click.echo(
+ self._detail_profile.fmt(
+ w.fill(
+ "The format version is used to provide users who build your project "
+ "with a helpful error message in the case that they do not have a recent "
+ "enough version of BuildStream supporting all the features which your "
+ "project might use."
+ )
+ ),
+ err=True,
+ )
click.echo("", err=True)
- click.echo(self._detail_profile.fmt(
- w.fill("The lowest version allowed is 0, the currently installed version of BuildStream "
- "supports up to format version {}.".format(BST_FORMAT_VERSION))), err=True)
+ click.echo(
+ self._detail_profile.fmt(
+ w.fill(
+ "The lowest version allowed is 0, the currently installed version of BuildStream "
+ "supports up to format version {}.".format(BST_FORMAT_VERSION)
+ )
+ ),
+ err=True,
+ )
click.echo("", err=True)
- format_version = click.prompt(self._content_profile.fmt("Format version"),
- value_proc=format_version_proc,
- default=format_version, err=True)
+ format_version = click.prompt(
+ self._content_profile.fmt("Format version"),
+ value_proc=format_version_proc,
+ default=format_version,
+ err=True,
+ )
click.echo("", err=True)
# Collect element path
click.echo(self._content_profile.fmt("Select the element path"), err=True)
click.echo(self._format_profile.fmt("-----------------------"), err=True)
click.echo("", err=True)
- click.echo(self._detail_profile.fmt(
- w.fill("The element path is a project subdirectory where element .bst files are stored "
- "within your project.")), err=True)
+ click.echo(
+ self._detail_profile.fmt(
+ w.fill(
+ "The element path is a project subdirectory where element .bst files are stored "
+ "within your project."
+ )
+ ),
+ err=True,
+ )
click.echo("", err=True)
- click.echo(self._detail_profile.fmt(
- w.fill("Elements will be displayed in logs as filenames relative to "
- "the element path, and similarly, dependencies must be expressed as filenames "
- "relative to the element path.")), err=True)
+ click.echo(
+ self._detail_profile.fmt(
+ w.fill(
+ "Elements will be displayed in logs as filenames relative to "
+ "the element path, and similarly, dependencies must be expressed as filenames "
+ "relative to the element path."
+ )
+ ),
+ err=True,
+ )
click.echo("", err=True)
- element_path = click.prompt(self._content_profile.fmt("Element path"),
- value_proc=element_path_proc,
- default=element_path, err=True)
+ element_path = click.prompt(
+ self._content_profile.fmt("Element path"), value_proc=element_path_proc, default=element_path, err=True
+ )
return (project_name, format_version, element_path)
@@ -909,7 +980,6 @@ class App():
# ask for a new input.
#
def _prefix_choice_value_proc(choices):
-
def value_proc(user_input):
remaining_candidate = [choice for choice in choices if choice.startswith(user_input)]
diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py
index 5c0293589..935a492d9 100644
--- a/src/buildstream/_frontend/cli.py
+++ b/src/buildstream/_frontend/cli.py
@@ -17,8 +17,8 @@ from ..utils import _get_compression, UtilError
# Helper classes and methods for Click #
##################################################################
-class FastEnumType(click.Choice):
+class FastEnumType(click.Choice):
def __init__(self, enum):
self._enum = enum
super().__init__(enum.values())
@@ -45,7 +45,7 @@ class FastEnumType(click.Choice):
#
def search_command(args, *, context=None):
if context is None:
- context = cli.make_context('bst', args, resilient_parsing=True)
+ context = cli.make_context("bst", args, resilient_parsing=True)
# Loop into the deepest command
command = cli
@@ -54,9 +54,7 @@ def search_command(args, *, context=None):
command = command_ctx.command.get_command(command_ctx, cmd)
if command is None:
return None
- command_ctx = command.make_context(command.name, [command.name],
- parent=command_ctx,
- resilient_parsing=True)
+ command_ctx = command.make_context(command.name, [command.name], parent=command_ctx, resilient_parsing=True)
return command_ctx
@@ -65,8 +63,11 @@ def search_command(args, *, context=None):
def complete_commands(cmd, args, incomplete):
command_ctx = search_command(args[1:])
if command_ctx and command_ctx.command and isinstance(command_ctx.command, click.MultiCommand):
- return [subcommand + " " for subcommand in command_ctx.command.list_commands(command_ctx)
- if not command_ctx.command.get_command(command_ctx, subcommand).hidden]
+ return [
+ subcommand + " "
+ for subcommand in command_ctx.command.list_commands(command_ctx)
+ if not command_ctx.command.get_command(command_ctx, subcommand).hidden
+ ]
return []
@@ -80,18 +81,19 @@ def complete_target(args, incomplete):
"""
from .. import utils
- project_conf = 'project.conf'
+
+ project_conf = "project.conf"
# First resolve the directory, in case there is an
# active --directory/-C option
#
- base_directory = '.'
+ base_directory = "."
idx = -1
try:
- idx = args.index('-C')
+ idx = args.index("-C")
except ValueError:
try:
- idx = args.index('--directory')
+ idx = args.index("--directory")
except ValueError:
pass
@@ -116,7 +118,7 @@ def complete_target(args, incomplete):
return []
# The project is not required to have an element-path
- element_directory = project.get_str('element-path', default='')
+ element_directory = project.get_str("element-path", default="")
# If a project was loaded, use its element-path to
# adjust our completion's base directory
@@ -132,19 +134,20 @@ def complete_target(args, incomplete):
def complete_artifact(orig_args, args, incomplete):
from .._context import Context
+
with Context(use_casd=False) as ctx:
config = None
if orig_args:
for i, arg in enumerate(orig_args):
- if arg in ('-c', '--config'):
+ if arg in ("-c", "--config"):
try:
config = orig_args[i + 1]
except IndexError:
pass
if args:
for i, arg in enumerate(args):
- if arg in ('-c', '--config'):
+ if arg in ("-c", "--config"):
try:
config = args[i + 1]
except IndexError:
@@ -167,38 +170,40 @@ def override_completions(orig_args, cmd, cmd_param, args, incomplete):
:return: all the possible user-specified completions for the param
"""
- if cmd.name == 'help':
+ if cmd.name == "help":
return complete_commands(cmd, args, incomplete)
# We can't easily extend click's data structures without
# modifying click itself, so just do some weak special casing
# right here and select which parameters we want to handle specially.
if isinstance(cmd_param.type, click.Path):
- if (cmd_param.name == 'elements' or
- cmd_param.name == 'element' or
- cmd_param.name == 'except_' or
- cmd_param.opts == ['--track'] or
- cmd_param.opts == ['--track-except']):
+ if (
+ cmd_param.name == "elements"
+ or cmd_param.name == "element"
+ or cmd_param.name == "except_"
+ or cmd_param.opts == ["--track"]
+ or cmd_param.opts == ["--track-except"]
+ ):
return complete_target(args, incomplete)
- if cmd_param.name == 'artifacts' or cmd_param.name == 'target':
+ if cmd_param.name == "artifacts" or cmd_param.name == "target":
return complete_artifact(orig_args, args, incomplete)
raise CompleteUnhandled()
def validate_output_streams():
- if sys.platform == 'win32':
+ if sys.platform == "win32":
# Windows does not support 'fcntl', the module is unavailable there as
# of Python 3.7, therefore early-out here.
return
import fcntl
+
for stream in (sys.stdout, sys.stderr):
fileno = stream.fileno()
flags = fcntl.fcntl(fileno, fcntl.F_GETFL)
if flags & os.O_NONBLOCK:
- click.echo("{} is currently set to O_NONBLOCK, try opening a new shell"
- .format(stream.name), err=True)
+ click.echo("{} is currently set to O_NONBLOCK, try opening a new shell".format(stream.name), err=True)
sys.exit(-1)
@@ -237,8 +242,7 @@ def handle_bst_force_start_method_env():
sys.exit(-1)
-def override_main(self, args=None, prog_name=None, complete_var=None,
- standalone_mode=True, **extra):
+def override_main(self, args=None, prog_name=None, complete_var=None, standalone_mode=True, **extra):
# Hook for the Bash completion. This only activates if the Bash
# completion is actually enabled, otherwise this is quite a fast
@@ -250,7 +254,7 @@ def override_main(self, args=None, prog_name=None, complete_var=None,
#
# The below is a quicker exit path for the sake
# of making completions respond faster.
- if 'BST_TEST_SUITE' not in os.environ:
+ if "BST_TEST_SUITE" not in os.environ:
sys.stdout.flush()
sys.stderr.flush()
os._exit(0)
@@ -269,14 +273,13 @@ def override_main(self, args=None, prog_name=None, complete_var=None,
# case of testing, our tests preceed our entrypoint, so we do our best.
handle_bst_force_start_method_env()
- original_main(self, args=args, prog_name=prog_name, complete_var=None,
- standalone_mode=standalone_mode, **extra)
+ original_main(self, args=args, prog_name=prog_name, complete_var=None, standalone_mode=standalone_mode, **extra)
original_main = click.BaseCommand.main
# Disable type checking since mypy doesn't support assigning to a method.
# See https://github.com/python/mypy/issues/2427.
-click.BaseCommand.main = override_main # type: ignore
+click.BaseCommand.main = override_main # type: ignore
##################################################################
@@ -287,58 +290,78 @@ def print_version(ctx, param, value):
return
from .. import __version__
+
click.echo(__version__)
ctx.exit()
-@click.group(context_settings=dict(help_option_names=['-h', '--help']))
-@click.option('--version', is_flag=True, callback=print_version,
- expose_value=False, is_eager=True)
-@click.option('--config', '-c',
- type=click.Path(exists=True, dir_okay=False, readable=True),
- help="Configuration file to use")
-@click.option('--directory', '-C', default=None, # Set to os.getcwd() later.
- type=click.Path(file_okay=False, readable=True),
- help="Project directory (default: current directory)")
-@click.option('--on-error', default=None,
- type=FastEnumType(_SchedulerErrorAction),
- help="What to do when an error is encountered")
-@click.option('--fetchers', type=click.INT, default=None,
- help="Maximum simultaneous download tasks")
-@click.option('--builders', type=click.INT, default=None,
- help="Maximum simultaneous build tasks")
-@click.option('--pushers', type=click.INT, default=None,
- help="Maximum simultaneous upload tasks")
-@click.option('--max-jobs', type=click.INT, default=None,
- help="Number of parallel jobs allowed for a given build task")
-@click.option('--network-retries', type=click.INT, default=None,
- help="Maximum retries for network tasks")
-@click.option('--no-interactive', is_flag=True,
- help="Force non interactive mode, otherwise this is automatically decided")
-@click.option('--verbose/--no-verbose', default=None,
- help="Be extra verbose")
-@click.option('--debug/--no-debug', default=None,
- help="Print debugging output")
-@click.option('--error-lines', type=click.INT, default=None,
- help="Maximum number of lines to show from a task log")
-@click.option('--message-lines', type=click.INT, default=None,
- help="Maximum number of lines to show in a detailed message")
-@click.option('--log-file',
- type=click.File(mode='w', encoding='UTF-8'),
- help="A file to store the main log (allows storing the main log while in interactive mode)")
-@click.option('--colors/--no-colors', default=None,
- help="Force enable/disable ANSI color codes in output")
-@click.option('--strict/--no-strict', default=None, is_flag=True,
- help="Elements must be rebuilt when their dependencies have changed")
-@click.option('--option', '-o', type=click.Tuple([str, str]), multiple=True, metavar='OPTION VALUE',
- help="Specify a project option")
-@click.option('--default-mirror', default=None,
- help="The mirror to fetch from first, before attempting other mirrors")
-@click.option('--pull-buildtrees', is_flag=True, default=None,
- help="Include an element's build tree when pulling remote element artifacts")
-@click.option('--cache-buildtrees', default=None,
- type=FastEnumType(_CacheBuildTrees),
- help="Cache artifact build tree content on creation")
+@click.group(context_settings=dict(help_option_names=["-h", "--help"]))
+@click.option("--version", is_flag=True, callback=print_version, expose_value=False, is_eager=True)
+@click.option(
+ "--config", "-c", type=click.Path(exists=True, dir_okay=False, readable=True), help="Configuration file to use"
+)
+@click.option(
+ "--directory",
+ "-C",
+ default=None, # Set to os.getcwd() later.
+ type=click.Path(file_okay=False, readable=True),
+ help="Project directory (default: current directory)",
+)
+@click.option(
+ "--on-error",
+ default=None,
+ type=FastEnumType(_SchedulerErrorAction),
+ help="What to do when an error is encountered",
+)
+@click.option("--fetchers", type=click.INT, default=None, help="Maximum simultaneous download tasks")
+@click.option("--builders", type=click.INT, default=None, help="Maximum simultaneous build tasks")
+@click.option("--pushers", type=click.INT, default=None, help="Maximum simultaneous upload tasks")
+@click.option(
+ "--max-jobs", type=click.INT, default=None, help="Number of parallel jobs allowed for a given build task"
+)
+@click.option("--network-retries", type=click.INT, default=None, help="Maximum retries for network tasks")
+@click.option(
+ "--no-interactive", is_flag=True, help="Force non interactive mode, otherwise this is automatically decided"
+)
+@click.option("--verbose/--no-verbose", default=None, help="Be extra verbose")
+@click.option("--debug/--no-debug", default=None, help="Print debugging output")
+@click.option("--error-lines", type=click.INT, default=None, help="Maximum number of lines to show from a task log")
+@click.option(
+ "--message-lines", type=click.INT, default=None, help="Maximum number of lines to show in a detailed message"
+)
+@click.option(
+ "--log-file",
+ type=click.File(mode="w", encoding="UTF-8"),
+ help="A file to store the main log (allows storing the main log while in interactive mode)",
+)
+@click.option("--colors/--no-colors", default=None, help="Force enable/disable ANSI color codes in output")
+@click.option(
+ "--strict/--no-strict",
+ default=None,
+ is_flag=True,
+ help="Elements must be rebuilt when their dependencies have changed",
+)
+@click.option(
+ "--option",
+ "-o",
+ type=click.Tuple([str, str]),
+ multiple=True,
+ metavar="OPTION VALUE",
+ help="Specify a project option",
+)
+@click.option("--default-mirror", default=None, help="The mirror to fetch from first, before attempting other mirrors")
+@click.option(
+ "--pull-buildtrees",
+ is_flag=True,
+ default=None,
+ help="Include an element's build tree when pulling remote element artifacts",
+)
+@click.option(
+ "--cache-buildtrees",
+ default=None,
+ type=FastEnumType(_CacheBuildTrees),
+ help="Cache artifact build tree content on creation",
+)
@click.pass_context
def cli(context, **kwargs):
"""Build and manipulate BuildStream projects
@@ -360,17 +383,15 @@ def cli(context, **kwargs):
##################################################################
# Help Command #
##################################################################
-@cli.command(name="help", short_help="Print usage information",
- context_settings={"help_option_names": []})
-@click.argument("command", nargs=-1, metavar='COMMAND')
+@cli.command(name="help", short_help="Print usage information", context_settings={"help_option_names": []})
+@click.argument("command", nargs=-1, metavar="COMMAND")
@click.pass_context
def help_command(ctx, command):
"""Print usage information about a given command
"""
command_ctx = search_command(command, context=ctx.parent)
if not command_ctx:
- click.echo("Not a valid command: '{} {}'"
- .format(ctx.parent.info_name, " ".join(command)), err=True)
+ click.echo("Not a valid command: '{} {}'".format(ctx.parent.info_name, " ".join(command)), err=True)
sys.exit(-1)
click.echo(command_ctx.command.get_help(command_ctx), err=True)
@@ -380,24 +401,32 @@ def help_command(ctx, command):
detail = " "
if command:
detail = " {} ".format(" ".join(command))
- click.echo("\nFor usage on a specific command: {} help{}COMMAND"
- .format(ctx.parent.info_name, detail), err=True)
+ click.echo(
+ "\nFor usage on a specific command: {} help{}COMMAND".format(ctx.parent.info_name, detail), err=True
+ )
##################################################################
# Init Command #
##################################################################
@cli.command(short_help="Initialize a new BuildStream project")
-@click.option('--project-name', type=click.STRING,
- help="The project name to use")
-@click.option('--format-version', type=click.INT, default=BST_FORMAT_VERSION, show_default=True,
- help="The required format version")
-@click.option('--element-path', type=click.Path(), default="elements", show_default=True,
- help="The subdirectory to store elements in")
-@click.option('--force', '-f', is_flag=True,
- help="Allow overwriting an existing project.conf")
-@click.argument('target-directory', nargs=1, required=False,
- type=click.Path(file_okay=False, writable=True))
+@click.option("--project-name", type=click.STRING, help="The project name to use")
+@click.option(
+ "--format-version",
+ type=click.INT,
+ default=BST_FORMAT_VERSION,
+ show_default=True,
+ help="The required format version",
+)
+@click.option(
+ "--element-path",
+ type=click.Path(),
+ default="elements",
+ show_default=True,
+ help="The subdirectory to store elements in",
+)
+@click.option("--force", "-f", is_flag=True, help="Allow overwriting an existing project.conf")
+@click.argument("target-directory", nargs=1, required=False, type=click.Path(file_okay=False, writable=True))
@click.pass_obj
def init(app, project_name, format_version, element_path, force, target_directory):
"""Initialize a new BuildStream project
@@ -415,13 +444,11 @@ def init(app, project_name, format_version, element_path, force, target_director
# Build Command #
##################################################################
@cli.command(short_help="Build elements in a pipeline")
-@click.option('--deps', '-d', default=None,
- type=click.Choice(['plan', 'all']),
- help='The dependencies to build')
-@click.option('--remote', '-r', default=None,
- help="The URL of the remote cache (defaults to the first configured cache)")
-@click.argument('elements', nargs=-1,
- type=click.Path(readable=False))
+@click.option("--deps", "-d", default=None, type=click.Choice(["plan", "all"]), help="The dependencies to build")
+@click.option(
+ "--remote", "-r", default=None, help="The URL of the remote cache (defaults to the first configured cache)"
+)
+@click.argument("elements", nargs=-1, type=click.Path(readable=False))
@click.pass_obj
def build(app, elements, deps, remote):
"""Build elements in a pipeline
@@ -450,30 +477,41 @@ def build(app, elements, deps, remote):
# Junction elements cannot be built, exclude them from default targets
ignore_junction_targets = True
- app.stream.build(elements,
- selection=deps,
- ignore_junction_targets=ignore_junction_targets,
- remote=remote)
+ app.stream.build(elements, selection=deps, ignore_junction_targets=ignore_junction_targets, remote=remote)
##################################################################
# Show Command #
##################################################################
@cli.command(short_help="Show elements in the pipeline")
-@click.option('--except', 'except_', multiple=True,
- type=click.Path(readable=False),
- help="Except certain dependencies")
-@click.option('--deps', '-d', default='all', show_default=True,
- type=click.Choice(['none', 'plan', 'run', 'build', 'all']),
- help='The dependencies to show')
-@click.option('--order', default="stage", show_default=True,
- type=click.Choice(['stage', 'alpha']),
- help='Staging or alphabetic ordering of dependencies')
-@click.option('--format', '-f', 'format_', metavar='FORMAT', default=None,
- type=click.STRING,
- help='Format string for each element')
-@click.argument('elements', nargs=-1,
- type=click.Path(readable=False))
+@click.option(
+ "--except", "except_", multiple=True, type=click.Path(readable=False), help="Except certain dependencies"
+)
+@click.option(
+ "--deps",
+ "-d",
+ default="all",
+ show_default=True,
+ type=click.Choice(["none", "plan", "run", "build", "all"]),
+ help="The dependencies to show",
+)
+@click.option(
+ "--order",
+ default="stage",
+ show_default=True,
+ type=click.Choice(["stage", "alpha"]),
+ help="Staging or alphabetic ordering of dependencies",
+)
+@click.option(
+ "--format",
+ "-f",
+ "format_",
+ metavar="FORMAT",
+ default=None,
+ type=click.STRING,
+ help="Format string for each element",
+)
+@click.argument("elements", nargs=-1, type=click.Path(readable=False))
@click.pass_obj
def show(app, elements, deps, except_, order, format_):
"""Show elements in the pipeline
@@ -536,9 +574,7 @@ def show(app, elements, deps, except_, order, format_):
if not elements:
elements = app.project.get_default_targets()
- dependencies = app.stream.load_selection(elements,
- selection=deps,
- except_targets=except_)
+ dependencies = app.stream.load_selection(elements, selection=deps, except_targets=except_)
if order == "alpha":
dependencies = sorted(dependencies)
@@ -554,25 +590,34 @@ def show(app, elements, deps, except_, order, format_):
# Shell Command #
##################################################################
@cli.command(short_help="Shell into an element's sandbox environment")
-@click.option('--build', '-b', 'build_', is_flag=True,
- help='Stage dependencies and sources to build')
-@click.option('--sysroot', '-s', default=None,
- type=click.Path(exists=True, file_okay=False, readable=True),
- help="An existing sysroot")
-@click.option('--mount', type=click.Tuple([click.Path(exists=True), str]), multiple=True,
- metavar='HOSTPATH PATH',
- help="Mount a file or directory into the sandbox")
-@click.option('--isolate', is_flag=True,
- help='Create an isolated build sandbox')
-@click.option('--use-buildtree', '-t', 'cli_buildtree', type=click.Choice(['ask', 'try', 'always', 'never']),
- default='ask', show_default=True,
- help=('Use a buildtree. If `always` is set, will always fail to '
- 'build if a buildtree is not available.'))
-@click.option('--pull', 'pull_', is_flag=True,
- help='Attempt to pull missing or incomplete artifacts')
-@click.argument('element', required=False,
- type=click.Path(readable=False))
-@click.argument('command', type=click.STRING, nargs=-1)
+@click.option("--build", "-b", "build_", is_flag=True, help="Stage dependencies and sources to build")
+@click.option(
+ "--sysroot",
+ "-s",
+ default=None,
+ type=click.Path(exists=True, file_okay=False, readable=True),
+ help="An existing sysroot",
+)
+@click.option(
+ "--mount",
+ type=click.Tuple([click.Path(exists=True), str]),
+ multiple=True,
+ metavar="HOSTPATH PATH",
+ help="Mount a file or directory into the sandbox",
+)
+@click.option("--isolate", is_flag=True, help="Create an isolated build sandbox")
+@click.option(
+ "--use-buildtree",
+ "-t",
+ "cli_buildtree",
+ type=click.Choice(["ask", "try", "always", "never"]),
+ default="ask",
+ show_default=True,
+ help=("Use a buildtree. If `always` is set, will always fail to " "build if a buildtree is not available."),
+)
+@click.option("--pull", "pull_", is_flag=True, help="Attempt to pull missing or incomplete artifacts")
+@click.argument("element", required=False, type=click.Path(readable=False))
+@click.argument("command", type=click.STRING, nargs=-1)
@click.pass_obj
def shell(app, element, sysroot, mount, isolate, build_, cli_buildtree, pull_, command):
"""Run a command in the target element's sandbox environment
@@ -616,8 +661,7 @@ def shell(app, element, sysroot, mount, isolate, build_, cli_buildtree, pull_, c
if not element:
raise AppError('Missing argument "ELEMENT".')
- elements = app.stream.load_selection((element,), selection=selection,
- use_artifact_config=True)
+ elements = app.stream.load_selection((element,), selection=selection, use_artifact_config=True)
# last one will be the element we want to stage, previous ones are
# elements to try and pull
@@ -628,10 +672,7 @@ def shell(app, element, sysroot, mount, isolate, build_, cli_buildtree, pull_, c
element_key = element._get_display_key()
prompt = app.shell_prompt(element_name, element_key)
- mounts = [
- HostMount(path, host_path)
- for host_path, path in mount
- ]
+ mounts = [HostMount(path, host_path) for host_path, path in mount]
cached = element._cached_buildtree()
buildtree_exists = element._buildtree_exists()
@@ -640,27 +681,31 @@ def shell(app, element, sysroot, mount, isolate, build_, cli_buildtree, pull_, c
if buildtree_exists or pull_:
use_buildtree = cli_buildtree
if not cached and use_buildtree == "always":
- click.echo("WARNING: buildtree is not cached locally, will attempt to pull from available remotes",
- err=True)
+ click.echo(
+ "WARNING: buildtree is not cached locally, will attempt to pull from available remotes",
+ err=True,
+ )
else:
if cli_buildtree == "always":
# Exit early if it won't be possible to even fetch a buildtree with always option
raise AppError("Artifact was created without buildtree, unable to launch shell with it")
- click.echo("WARNING: Artifact created without buildtree, shell will be loaded without it",
- err=True)
+ click.echo("WARNING: Artifact created without buildtree, shell will be loaded without it", err=True)
else:
# If the value has defaulted to ask and in non interactive mode, don't consider the buildtree, this
# being the default behaviour of the command
if app.interactive and cli_buildtree == "ask":
- if cached and bool(click.confirm('Do you want to use the cached buildtree?')):
+ if cached and bool(click.confirm("Do you want to use the cached buildtree?")):
use_buildtree = "always"
elif buildtree_exists:
try:
- choice = click.prompt("Do you want to pull & use a cached buildtree?",
- type=click.Choice(['try', 'always', 'never']),
- err=True, show_choices=True)
+ choice = click.prompt(
+ "Do you want to pull & use a cached buildtree?",
+ type=click.Choice(["try", "always", "never"]),
+ err=True,
+ show_choices=True,
+ )
except click.Abort:
- click.echo('Aborting', err=True)
+ click.echo("Aborting", err=True)
sys.exit(-1)
if choice != "never":
@@ -671,13 +716,17 @@ def shell(app, element, sysroot, mount, isolate, build_, cli_buildtree, pull_, c
click.echo("WARNING: using a buildtree from a failed build.", err=True)
try:
- exitcode = app.stream.shell(element, scope, prompt,
- directory=sysroot,
- mounts=mounts,
- isolate=isolate,
- command=command,
- usebuildtree=use_buildtree,
- pull_dependencies=pull_dependencies)
+ exitcode = app.stream.shell(
+ element,
+ scope,
+ prompt,
+ directory=sysroot,
+ mounts=mounts,
+ isolate=isolate,
+ command=command,
+ usebuildtree=use_buildtree,
+ pull_dependencies=pull_dependencies,
+ )
except BstError as e:
raise AppError("Error launching shell: {}".format(e), detail=e.detail) from e
@@ -697,20 +746,27 @@ def source():
# Source Fetch Command #
##################################################################
@source.command(name="fetch", short_help="Fetch sources in a pipeline")
-@click.option('--except', 'except_', multiple=True,
- type=click.Path(readable=False),
- help="Except certain dependencies from fetching")
-@click.option('--deps', '-d', default='plan', show_default=True,
- type=click.Choice(['none', 'plan', 'all']),
- help='The dependencies to fetch')
-@click.option('--track', 'track_', is_flag=True,
- help="Track new source references before fetching")
-@click.option('--track-cross-junctions', '-J', is_flag=True,
- help="Allow tracking to cross junction boundaries")
-@click.option('--remote', '-r', default=None,
- help="The URL of the remote source cache (defaults to the first configured cache)")
-@click.argument('elements', nargs=-1,
- type=click.Path(readable=False))
+@click.option(
+ "--except",
+ "except_",
+ multiple=True,
+ type=click.Path(readable=False),
+ help="Except certain dependencies from fetching",
+)
+@click.option(
+ "--deps",
+ "-d",
+ default="plan",
+ show_default=True,
+ type=click.Choice(["none", "plan", "all"]),
+ help="The dependencies to fetch",
+)
+@click.option("--track", "track_", is_flag=True, help="Track new source references before fetching")
+@click.option("--track-cross-junctions", "-J", is_flag=True, help="Allow tracking to cross junction boundaries")
+@click.option(
+ "--remote", "-r", default=None, help="The URL of the remote source cache (defaults to the first configured cache)"
+)
+@click.argument("elements", nargs=-1, type=click.Path(readable=False))
@click.pass_obj
def source_fetch(app, elements, deps, track_, except_, track_cross_junctions, remote):
"""Fetch sources required to build the pipeline
@@ -741,36 +797,48 @@ def source_fetch(app, elements, deps, track_, except_, track_cross_junctions, re
sys.exit(-1)
if track_ and deps == PipelineSelection.PLAN:
- click.echo("WARNING: --track specified for tracking of a build plan\n\n"
- "Since tracking modifies the build plan, all elements will be tracked.", err=True)
+ click.echo(
+ "WARNING: --track specified for tracking of a build plan\n\n"
+ "Since tracking modifies the build plan, all elements will be tracked.",
+ err=True,
+ )
deps = PipelineSelection.ALL
with app.initialized(session_name="Fetch"):
if not elements:
elements = app.project.get_default_targets()
- app.stream.fetch(elements,
- selection=deps,
- except_targets=except_,
- track_targets=track_,
- track_cross_junctions=track_cross_junctions,
- remote=remote)
+ app.stream.fetch(
+ elements,
+ selection=deps,
+ except_targets=except_,
+ track_targets=track_,
+ track_cross_junctions=track_cross_junctions,
+ remote=remote,
+ )
##################################################################
# Source Track Command #
##################################################################
@source.command(name="track", short_help="Track new source references")
-@click.option('--except', 'except_', multiple=True,
- type=click.Path(readable=False),
- help="Except certain dependencies from tracking")
-@click.option('--deps', '-d', default='none', show_default=True,
- type=click.Choice(['none', 'all']),
- help='The dependencies to track')
-@click.option('--cross-junctions', '-J', is_flag=True,
- help="Allow crossing junction boundaries")
-@click.argument('elements', nargs=-1,
- type=click.Path(readable=False))
+@click.option(
+ "--except",
+ "except_",
+ multiple=True,
+ type=click.Path(readable=False),
+ help="Except certain dependencies from tracking",
+)
+@click.option(
+ "--deps",
+ "-d",
+ default="none",
+ show_default=True,
+ type=click.Choice(["none", "all"]),
+ help="The dependencies to track",
+)
+@click.option("--cross-junctions", "-J", is_flag=True, help="Allow crossing junction boundaries")
+@click.argument("elements", nargs=-1, type=click.Path(readable=False))
@click.pass_obj
def source_track(app, elements, deps, except_, cross_junctions):
"""Consults the specified tracking branches for new versions available
@@ -800,41 +868,50 @@ def source_track(app, elements, deps, except_, cross_junctions):
# Substitute 'none' for 'redirect' so that element redirections
# will be done
- if deps == 'none':
- deps = 'redirect'
- app.stream.track(elements,
- selection=deps,
- except_targets=except_,
- cross_junctions=cross_junctions)
+ if deps == "none":
+ deps = "redirect"
+ app.stream.track(elements, selection=deps, except_targets=except_, cross_junctions=cross_junctions)
##################################################################
# Source Checkout Command #
##################################################################
-@source.command(name='checkout', short_help='Checkout sources of an element')
-@click.option('--force', '-f', is_flag=True,
- help="Allow files to be overwritten")
-@click.option('--except', 'except_', multiple=True,
- type=click.Path(readable=False),
- help="Except certain dependencies")
-@click.option('--deps', '-d', default='none', show_default=True,
- type=click.Choice(['build', 'none', 'run', 'all']),
- help='The dependencies whose sources to checkout')
-@click.option('--tar', default=None, metavar='LOCATION',
- type=click.Path(),
- help="Create a tarball containing the sources instead "
- "of a file tree.")
-@click.option('--compression', default=None,
- type=click.Choice(['gz', 'xz', 'bz2']),
- help="The compression option of the tarball created.")
-@click.option('--include-build-scripts', 'build_scripts', is_flag=True)
-@click.option('--directory', default='source-checkout',
- type=click.Path(file_okay=False),
- help="The directory to checkout the sources to")
-@click.argument('element', required=False, type=click.Path(readable=False))
+@source.command(name="checkout", short_help="Checkout sources of an element")
+@click.option("--force", "-f", is_flag=True, help="Allow files to be overwritten")
+@click.option(
+ "--except", "except_", multiple=True, type=click.Path(readable=False), help="Except certain dependencies"
+)
+@click.option(
+ "--deps",
+ "-d",
+ default="none",
+ show_default=True,
+ type=click.Choice(["build", "none", "run", "all"]),
+ help="The dependencies whose sources to checkout",
+)
+@click.option(
+ "--tar",
+ default=None,
+ metavar="LOCATION",
+ type=click.Path(),
+ help="Create a tarball containing the sources instead " "of a file tree.",
+)
+@click.option(
+ "--compression",
+ default=None,
+ type=click.Choice(["gz", "xz", "bz2"]),
+ help="The compression option of the tarball created.",
+)
+@click.option("--include-build-scripts", "build_scripts", is_flag=True)
+@click.option(
+ "--directory",
+ default="source-checkout",
+ type=click.Path(file_okay=False),
+ help="The directory to checkout the sources to",
+)
+@click.argument("element", required=False, type=click.Path(readable=False))
@click.pass_obj
-def source_checkout(app, element, directory, force, deps, except_,
- tar, compression, build_scripts):
+def source_checkout(app, element, directory, force, deps, except_, tar, compression, build_scripts):
"""Checkout sources of an element to the specified location
When this command is executed from a workspace directory, the default
@@ -859,14 +936,16 @@ def source_checkout(app, element, directory, force, deps, except_,
if not element:
raise AppError('Missing argument "ELEMENT".')
- app.stream.source_checkout(element,
- location=location,
- force=force,
- deps=deps,
- except_targets=except_,
- tar=bool(tar),
- compression=compression,
- include_build_scripts=build_scripts)
+ app.stream.source_checkout(
+ element,
+ location=location,
+ force=force,
+ deps=deps,
+ except_targets=except_,
+ tar=bool(tar),
+ compression=compression,
+ include_build_scripts=build_scripts,
+ )
##################################################################
@@ -880,39 +959,42 @@ def workspace():
##################################################################
# Workspace Open Command #
##################################################################
-@workspace.command(name='open', short_help="Open a new workspace")
-@click.option('--no-checkout', is_flag=True,
- help="Do not checkout the source, only link to the given directory")
-@click.option('--force', '-f', is_flag=True,
- help="The workspace will be created even if the directory in which it will be created is not empty " +
- "or if a workspace for that element already exists")
-@click.option('--track', 'track_', is_flag=True,
- help="Track and fetch new source references before checking out the workspace")
-@click.option('--directory', type=click.Path(file_okay=False), default=None,
- help="Only for use when a single Element is given: Set the directory to use to create the workspace")
-@click.argument('elements', nargs=-1, type=click.Path(readable=False), required=True)
+@workspace.command(name="open", short_help="Open a new workspace")
+@click.option("--no-checkout", is_flag=True, help="Do not checkout the source, only link to the given directory")
+@click.option(
+ "--force",
+ "-f",
+ is_flag=True,
+ help="The workspace will be created even if the directory in which it will be created is not empty "
+ + "or if a workspace for that element already exists",
+)
+@click.option(
+ "--track", "track_", is_flag=True, help="Track and fetch new source references before checking out the workspace"
+)
+@click.option(
+ "--directory",
+ type=click.Path(file_okay=False),
+ default=None,
+ help="Only for use when a single Element is given: Set the directory to use to create the workspace",
+)
+@click.argument("elements", nargs=-1, type=click.Path(readable=False), required=True)
@click.pass_obj
def workspace_open(app, no_checkout, force, track_, directory, elements):
"""Open a workspace for manual source modification"""
with app.initialized():
- app.stream.workspace_open(elements,
- no_checkout=no_checkout,
- track_first=track_,
- force=force,
- custom_dir=directory)
+ app.stream.workspace_open(
+ elements, no_checkout=no_checkout, track_first=track_, force=force, custom_dir=directory
+ )
##################################################################
# Workspace Close Command #
##################################################################
-@workspace.command(name='close', short_help="Close workspaces")
-@click.option('--remove-dir', is_flag=True,
- help="Remove the path that contains the closed workspace")
-@click.option('--all', '-a', 'all_', is_flag=True,
- help="Close all open workspaces")
-@click.argument('elements', nargs=-1,
- type=click.Path(readable=False))
+@workspace.command(name="close", short_help="Close workspaces")
+@click.option("--remove-dir", is_flag=True, help="Remove the path that contains the closed workspace")
+@click.option("--all", "-a", "all_", is_flag=True, help="Close all open workspaces")
+@click.argument("elements", nargs=-1, type=click.Path(readable=False))
@click.pass_obj
def workspace_close(app, remove_dir, all_, elements):
"""Close a workspace"""
@@ -927,11 +1009,11 @@ def workspace_close(app, remove_dir, all_, elements):
if element:
elements = (element,)
else:
- raise AppError('No elements specified')
+ raise AppError("No elements specified")
# Early exit if we specified `all` and there are no workspaces
if all_ and not app.stream.workspace_exists():
- click.echo('No open workspaces to close', err=True)
+ click.echo("No open workspaces to close", err=True)
sys.exit(0)
if all_:
@@ -958,21 +1040,19 @@ def workspace_close(app, remove_dir, all_, elements):
if removed_required_element:
click.echo(
"Removed '{}', therefore you can no longer run BuildStream "
- "commands from the current directory.".format(element_name), err=True)
+ "commands from the current directory.".format(element_name),
+ err=True,
+ )
##################################################################
# Workspace Reset Command #
##################################################################
-@workspace.command(name='reset', short_help="Reset a workspace to its original state")
-@click.option('--soft', is_flag=True,
- help="Reset workspace state without affecting its contents")
-@click.option('--track', 'track_', is_flag=True,
- help="Track and fetch the latest source before resetting")
-@click.option('--all', '-a', 'all_', is_flag=True,
- help="Reset all open workspaces")
-@click.argument('elements', nargs=-1,
- type=click.Path(readable=False))
+@workspace.command(name="reset", short_help="Reset a workspace to its original state")
+@click.option("--soft", is_flag=True, help="Reset workspace state without affecting its contents")
+@click.option("--track", "track_", is_flag=True, help="Track and fetch the latest source before resetting")
+@click.option("--all", "-a", "all_", is_flag=True, help="Reset all open workspaces")
+@click.argument("elements", nargs=-1, type=click.Path(readable=False))
@click.pass_obj
def workspace_reset(app, soft, track_, all_, elements):
"""Reset a workspace to its original state"""
@@ -985,7 +1065,7 @@ def workspace_reset(app, soft, track_, all_, elements):
if element:
elements = (element,)
else:
- raise AppError('No elements specified to reset')
+ raise AppError("No elements specified to reset")
if all_ and not app.stream.workspace_exists():
raise AppError("No open workspaces to reset")
@@ -999,7 +1079,7 @@ def workspace_reset(app, soft, track_, all_, elements):
##################################################################
# Workspace List Command #
##################################################################
-@workspace.command(name='list', short_help="List open workspaces")
+@workspace.command(name="list", short_help="List open workspaces")
@click.pass_obj
def workspace_list(app):
"""List open workspaces"""
@@ -1044,11 +1124,16 @@ def artifact():
#############################################################
# Artifact show Command #
#############################################################
-@artifact.command(name='show', short_help="Show the cached state of artifacts")
-@click.option('--deps', '-d', default='none', show_default=True,
- type=click.Choice(['build', 'run', 'all', 'none']),
- help='The dependencies we also want to show')
-@click.argument('artifacts', type=click.Path(), nargs=-1)
+@artifact.command(name="show", short_help="Show the cached state of artifacts")
+@click.option(
+ "--deps",
+ "-d",
+ default="none",
+ show_default=True,
+ type=click.Choice(["build", "run", "all", "none"]),
+ help="The dependencies we also want to show",
+)
+@click.argument("artifacts", type=click.Path(), nargs=-1)
@click.pass_obj
def artifact_show(app, deps, artifacts):
"""show the cached state of artifacts"""
@@ -1061,31 +1146,38 @@ def artifact_show(app, deps, artifacts):
#####################################################################
# Artifact Checkout Command #
#####################################################################
-@artifact.command(name='checkout', short_help="Checkout contents of an artifact")
-@click.option('--force', '-f', is_flag=True,
- help="Allow files to be overwritten")
-@click.option('--deps', '-d', default='run', show_default=True,
- type=click.Choice(['run', 'build', 'none', 'all']),
- help='The dependencies to checkout')
-@click.option('--integrate/--no-integrate', default=None, is_flag=True,
- help="Whether to run integration commands")
-@click.option('--hardlinks', is_flag=True,
- help="Checkout hardlinks instead of copying if possible")
-@click.option('--tar', default=None, metavar='LOCATION',
- type=click.Path(),
- help="Create a tarball from the artifact contents instead "
- "of a file tree. If LOCATION is '-', the tarball "
- "will be dumped to the standard output.")
-@click.option('--compression', default=None,
- type=click.Choice(['gz', 'xz', 'bz2']),
- help="The compression option of the tarball created.")
-@click.option('--pull', 'pull_', is_flag=True,
- help="Pull the artifact if it's missing or incomplete.")
-@click.option('--directory', default=None,
- type=click.Path(file_okay=False),
- help="The directory to checkout the artifact to")
-@click.argument('target', required=False,
- type=click.Path(readable=False))
+@artifact.command(name="checkout", short_help="Checkout contents of an artifact")
+@click.option("--force", "-f", is_flag=True, help="Allow files to be overwritten")
+@click.option(
+ "--deps",
+ "-d",
+ default="run",
+ show_default=True,
+ type=click.Choice(["run", "build", "none", "all"]),
+ help="The dependencies to checkout",
+)
+@click.option("--integrate/--no-integrate", default=None, is_flag=True, help="Whether to run integration commands")
+@click.option("--hardlinks", is_flag=True, help="Checkout hardlinks instead of copying if possible")
+@click.option(
+ "--tar",
+ default=None,
+ metavar="LOCATION",
+ type=click.Path(),
+ help="Create a tarball from the artifact contents instead "
+ "of a file tree. If LOCATION is '-', the tarball "
+ "will be dumped to the standard output.",
+)
+@click.option(
+ "--compression",
+ default=None,
+ type=click.Choice(["gz", "xz", "bz2"]),
+ help="The compression option of the tarball created.",
+)
+@click.option("--pull", "pull_", is_flag=True, help="Pull the artifact if it's missing or incomplete.")
+@click.option(
+ "--directory", default=None, type=click.Path(file_okay=False), help="The directory to checkout the artifact to"
+)
+@click.argument("target", required=False, type=click.Path(readable=False))
@click.pass_obj
def artifact_checkout(app, force, deps, integrate, hardlinks, tar, compression, pull_, directory, target):
"""Checkout contents of an artifact
@@ -1110,7 +1202,7 @@ def artifact_checkout(app, force, deps, integrate, hardlinks, tar, compression,
location = os.path.abspath(os.path.join(os.getcwd(), target))
else:
location = directory
- if location[-4:] == '.bst':
+ if location[-4:] == ".bst":
location = location[:-4]
tar = False
else:
@@ -1120,9 +1212,12 @@ def artifact_checkout(app, force, deps, integrate, hardlinks, tar, compression,
except UtilError as e:
click.echo("ERROR: Invalid file extension given with '--tar': {}".format(e), err=True)
sys.exit(-1)
- if compression and inferred_compression != '' and inferred_compression != compression:
- click.echo("WARNING: File extension and compression differ."
- "File extension has been overridden by --compression", err=True)
+ if compression and inferred_compression != "" and inferred_compression != compression:
+ click.echo(
+ "WARNING: File extension and compression differ."
+ "File extension has been overridden by --compression",
+ err=True,
+ )
if not compression:
compression = inferred_compression
@@ -1132,28 +1227,35 @@ def artifact_checkout(app, force, deps, integrate, hardlinks, tar, compression,
if not target:
raise AppError('Missing argument "ELEMENT".')
- app.stream.checkout(target,
- location=location,
- force=force,
- selection=deps,
- integrate=True if integrate is None else integrate,
- hardlinks=hardlinks,
- pull=pull_,
- compression=compression,
- tar=bool(tar))
+ app.stream.checkout(
+ target,
+ location=location,
+ force=force,
+ selection=deps,
+ integrate=True if integrate is None else integrate,
+ hardlinks=hardlinks,
+ pull=pull_,
+ compression=compression,
+ tar=bool(tar),
+ )
################################################################
# Artifact Pull Command #
################################################################
@artifact.command(name="pull", short_help="Pull a built artifact")
-@click.option('--deps', '-d', default='none', show_default=True,
- type=click.Choice(['none', 'all']),
- help='The dependency artifacts to pull')
-@click.option('--remote', '-r', default=None,
- help="The URL of the remote cache (defaults to the first configured cache)")
-@click.argument('artifacts', nargs=-1,
- type=click.Path(readable=False))
+@click.option(
+ "--deps",
+ "-d",
+ default="none",
+ show_default=True,
+ type=click.Choice(["none", "all"]),
+ help="The dependency artifacts to pull",
+)
+@click.option(
+ "--remote", "-r", default=None, help="The URL of the remote cache (defaults to the first configured cache)"
+)
+@click.argument("artifacts", nargs=-1, type=click.Path(readable=False))
@click.pass_obj
def artifact_pull(app, artifacts, deps, remote):
"""Pull a built artifact from the configured remote artifact cache.
@@ -1184,21 +1286,25 @@ def artifact_pull(app, artifacts, deps, remote):
# Junction elements cannot be pulled, exclude them from default targets
ignore_junction_targets = True
- app.stream.pull(artifacts, selection=deps, remote=remote,
- ignore_junction_targets=ignore_junction_targets)
+ app.stream.pull(artifacts, selection=deps, remote=remote, ignore_junction_targets=ignore_junction_targets)
##################################################################
# Artifact Push Command #
##################################################################
@artifact.command(name="push", short_help="Push a built artifact")
-@click.option('--deps', '-d', default='none', show_default=True,
- type=click.Choice(['none', 'all']),
- help='The dependencies to push')
-@click.option('--remote', '-r', default=None,
- help="The URL of the remote cache (defaults to the first configured cache)")
-@click.argument('artifacts', nargs=-1,
- type=click.Path(readable=False))
+@click.option(
+ "--deps",
+ "-d",
+ default="none",
+ show_default=True,
+ type=click.Choice(["none", "all"]),
+ help="The dependencies to push",
+)
+@click.option(
+ "--remote", "-r", default=None, help="The URL of the remote cache (defaults to the first configured cache)"
+)
+@click.argument("artifacts", nargs=-1, type=click.Path(readable=False))
@click.pass_obj
def artifact_push(app, artifacts, deps, remote):
"""Push a built artifact to a remote artifact cache.
@@ -1231,18 +1337,19 @@ def artifact_push(app, artifacts, deps, remote):
# Junction elements cannot be pushed, exclude them from default targets
ignore_junction_targets = True
- app.stream.push(artifacts, selection=deps, remote=remote,
- ignore_junction_targets=ignore_junction_targets)
+ app.stream.push(artifacts, selection=deps, remote=remote, ignore_junction_targets=ignore_junction_targets)
################################################################
# Artifact Log Command #
################################################################
-@artifact.command(name='log', short_help="Show logs of artifacts")
-@click.option('--out',
- type=click.Path(file_okay=True, writable=True),
- help="Output logs to individual files in the specified path. If absent, logs are written to stdout.")
-@click.argument('artifacts', type=click.Path(), nargs=-1)
+@artifact.command(name="log", short_help="Show logs of artifacts")
+@click.option(
+ "--out",
+ type=click.Path(file_okay=True, writable=True),
+ help="Output logs to individual files in the specified path. If absent, logs are written to stdout.",
+)
+@click.argument("artifacts", type=click.Path(), nargs=-1)
@click.pass_obj
def artifact_log(app, artifacts, out):
"""Show build logs of artifacts"""
@@ -1252,7 +1359,7 @@ def artifact_log(app, artifacts, out):
if not out:
try:
for log in list(artifact_logs.values()):
- with open(log[0], 'r') as f:
+ with open(log[0], "r") as f:
data = f.read()
click.echo_via_pager(data)
except (OSError, FileNotFoundError):
@@ -1274,7 +1381,7 @@ def artifact_log(app, artifacts, out):
shutil.copy(log, dest)
# make a dir and write in log files
else:
- log_name = os.path.splitext(name)[0] + '.log'
+ log_name = os.path.splitext(name)[0] + ".log"
dest = os.path.join(out, log_name)
shutil.copy(log_files[0], dest)
# write a log file
@@ -1283,10 +1390,11 @@ def artifact_log(app, artifacts, out):
################################################################
# Artifact List-Contents Command #
################################################################
-@artifact.command(name='list-contents', short_help="List the contents of an artifact")
-@click.option('--long', '-l', 'long_', is_flag=True,
- help="Provide more information about the contents of the artifact.")
-@click.argument('artifacts', type=click.Path(), nargs=-1)
+@artifact.command(name="list-contents", short_help="List the contents of an artifact")
+@click.option(
+ "--long", "-l", "long_", is_flag=True, help="Provide more information about the contents of the artifact."
+)
+@click.argument("artifacts", type=click.Path(), nargs=-1)
@click.pass_obj
def artifact_list_contents(app, artifacts, long_):
"""List the contents of an artifact.
@@ -1308,11 +1416,16 @@ def artifact_list_contents(app, artifacts, long_):
###################################################################
# Artifact Delete Command #
###################################################################
-@artifact.command(name='delete', short_help="Remove artifacts from the local cache")
-@click.option('--deps', '-d', default='none', show_default=True,
- type=click.Choice(['none', 'run', 'build', 'all']),
- help="The dependencies to delete")
-@click.argument('artifacts', type=click.Path(), nargs=-1)
+@artifact.command(name="delete", short_help="Remove artifacts from the local cache")
+@click.option(
+ "--deps",
+ "-d",
+ default="none",
+ show_default=True,
+ type=click.Choice(["none", "run", "build", "all"]),
+ help="The dependencies to delete",
+)
+@click.argument("artifacts", type=click.Path(), nargs=-1)
@click.pass_obj
def artifact_delete(app, artifacts, deps):
"""Remove artifacts from the local cache"""
@@ -1333,18 +1446,24 @@ def artifact_delete(app, artifacts, deps):
# Fetch Command #
##################################################################
@cli.command(short_help="COMMAND OBSOLETE - Fetch sources in a pipeline", hidden=True)
-@click.option('--except', 'except_', multiple=True,
- type=click.Path(readable=False),
- help="Except certain dependencies from fetching")
-@click.option('--deps', '-d', default='plan', show_default=True,
- type=click.Choice(['none', 'plan', 'all']),
- help='The dependencies to fetch')
-@click.option('--track', 'track_', is_flag=True,
- help="Track new source references before fetching")
-@click.option('--track-cross-junctions', '-J', is_flag=True,
- help="Allow tracking to cross junction boundaries")
-@click.argument('elements', nargs=-1,
- type=click.Path(readable=False))
+@click.option(
+ "--except",
+ "except_",
+ multiple=True,
+ type=click.Path(readable=False),
+ help="Except certain dependencies from fetching",
+)
+@click.option(
+ "--deps",
+ "-d",
+ default="plan",
+ show_default=True,
+ type=click.Choice(["none", "plan", "all"]),
+ help="The dependencies to fetch",
+)
+@click.option("--track", "track_", is_flag=True, help="Track new source references before fetching")
+@click.option("--track-cross-junctions", "-J", is_flag=True, help="Allow tracking to cross junction boundaries")
+@click.argument("elements", nargs=-1, type=click.Path(readable=False))
@click.pass_obj
def fetch(app, elements, deps, track_, except_, track_cross_junctions):
click.echo("This command is now obsolete. Use `bst source fetch` instead.", err=True)
@@ -1355,16 +1474,23 @@ def fetch(app, elements, deps, track_, except_, track_cross_junctions):
# Track Command #
##################################################################
@cli.command(short_help="COMMAND OBSOLETE - Track new source references", hidden=True)
-@click.option('--except', 'except_', multiple=True,
- type=click.Path(readable=False),
- help="Except certain dependencies from tracking")
-@click.option('--deps', '-d', default='none', show_default=True,
- type=click.Choice(['none', 'all']),
- help='The dependencies to track')
-@click.option('--cross-junctions', '-J', is_flag=True,
- help="Allow crossing junction boundaries")
-@click.argument('elements', nargs=-1,
- type=click.Path(readable=False))
+@click.option(
+ "--except",
+ "except_",
+ multiple=True,
+ type=click.Path(readable=False),
+ help="Except certain dependencies from tracking",
+)
+@click.option(
+ "--deps",
+ "-d",
+ default="none",
+ show_default=True,
+ type=click.Choice(["none", "all"]),
+ help="The dependencies to track",
+)
+@click.option("--cross-junctions", "-J", is_flag=True, help="Allow crossing junction boundaries")
+@click.argument("elements", nargs=-1, type=click.Path(readable=False))
@click.pass_obj
def track(app, elements, deps, except_, cross_junctions):
click.echo("This command is now obsolete. Use `bst source track` instead.", err=True)
@@ -1375,26 +1501,33 @@ def track(app, elements, deps, except_, cross_junctions):
# Checkout Command #
##################################################################
@cli.command(short_help="COMMAND OBSOLETE - Checkout a built artifact", hidden=True)
-@click.option('--force', '-f', is_flag=True,
- help="Allow files to be overwritten")
-@click.option('--deps', '-d', default='run', show_default=True,
- type=click.Choice(['run', 'build', 'none']),
- help='The dependencies to checkout')
-@click.option('--integrate/--no-integrate', default=True,
- help="Run integration commands (default is to run commands)")
-@click.option('--hardlinks', is_flag=True,
- help="Checkout hardlinks instead of copies (handle with care)")
-@click.option('--tar', is_flag=True,
- help="Create a tarball from the artifact contents instead "
- "of a file tree. If LOCATION is '-', the tarball "
- "will be dumped to the standard output.")
-@click.argument('element', required=False,
- type=click.Path(readable=False))
-@click.argument('location', type=click.Path(), required=False)
+@click.option("--force", "-f", is_flag=True, help="Allow files to be overwritten")
+@click.option(
+ "--deps",
+ "-d",
+ default="run",
+ show_default=True,
+ type=click.Choice(["run", "build", "none"]),
+ help="The dependencies to checkout",
+)
+@click.option("--integrate/--no-integrate", default=True, help="Run integration commands (default is to run commands)")
+@click.option("--hardlinks", is_flag=True, help="Checkout hardlinks instead of copies (handle with care)")
+@click.option(
+ "--tar",
+ is_flag=True,
+ help="Create a tarball from the artifact contents instead "
+ "of a file tree. If LOCATION is '-', the tarball "
+ "will be dumped to the standard output.",
+)
+@click.argument("element", required=False, type=click.Path(readable=False))
+@click.argument("location", type=click.Path(), required=False)
@click.pass_obj
def checkout(app, element, location, force, deps, integrate, hardlinks, tar):
- click.echo("This command is now obsolete. Use `bst artifact checkout` instead " +
- "and use the --directory option to specify LOCATION", err=True)
+ click.echo(
+ "This command is now obsolete. Use `bst artifact checkout` instead "
+ + "and use the --directory option to specify LOCATION",
+ err=True,
+ )
sys.exit(1)
@@ -1402,13 +1535,16 @@ def checkout(app, element, location, force, deps, integrate, hardlinks, tar):
# Pull Command #
################################################################
@cli.command(short_help="COMMAND OBSOLETE - Pull a built artifact", hidden=True)
-@click.option('--deps', '-d', default='none', show_default=True,
- type=click.Choice(['none', 'all']),
- help='The dependency artifacts to pull')
-@click.option('--remote', '-r',
- help="The URL of the remote cache (defaults to the first configured cache)")
-@click.argument('elements', nargs=-1,
- type=click.Path(readable=False))
+@click.option(
+ "--deps",
+ "-d",
+ default="none",
+ show_default=True,
+ type=click.Choice(["none", "all"]),
+ help="The dependency artifacts to pull",
+)
+@click.option("--remote", "-r", help="The URL of the remote cache (defaults to the first configured cache)")
+@click.argument("elements", nargs=-1, type=click.Path(readable=False))
@click.pass_obj
def pull(app, elements, deps, remote):
click.echo("This command is now obsolete. Use `bst artifact pull` instead.", err=True)
@@ -1419,13 +1555,18 @@ def pull(app, elements, deps, remote):
# Push Command #
##################################################################
@cli.command(short_help="COMMAND OBSOLETE - Push a built artifact", hidden=True)
-@click.option('--deps', '-d', default='none', show_default=True,
- type=click.Choice(['none', 'all']),
- help='The dependencies to push')
-@click.option('--remote', '-r', default=None,
- help="The URL of the remote cache (defaults to the first configured cache)")
-@click.argument('elements', nargs=-1,
- type=click.Path(readable=False))
+@click.option(
+ "--deps",
+ "-d",
+ default="none",
+ show_default=True,
+ type=click.Choice(["none", "all"]),
+ help="The dependencies to push",
+)
+@click.option(
+ "--remote", "-r", default=None, help="The URL of the remote cache (defaults to the first configured cache)"
+)
+@click.argument("elements", nargs=-1, type=click.Path(readable=False))
@click.pass_obj
def push(app, elements, deps, remote):
click.echo("This command is now obsolete. Use `bst artifact push` instead.", err=True)
diff --git a/src/buildstream/_frontend/complete.py b/src/buildstream/_frontend/complete.py
index 06067f6cc..45e857e3e 100644
--- a/src/buildstream/_frontend/complete.py
+++ b/src/buildstream/_frontend/complete.py
@@ -39,9 +39,9 @@ import click
from click.core import MultiCommand, Option, Argument
from click.parser import split_arg_string
-WORDBREAK = '='
+WORDBREAK = "="
-COMPLETION_SCRIPT = '''
+COMPLETION_SCRIPT = """
%(complete_func)s() {
local IFS=$'\n'
COMPREPLY=( $( env COMP_WORDS="${COMP_WORDS[*]}" \\
@@ -51,7 +51,7 @@ COMPLETION_SCRIPT = '''
}
complete -F %(complete_func)s -o nospace %(script_names)s
-'''
+"""
# An exception for our custom completion handler to
@@ -62,7 +62,7 @@ class CompleteUnhandled(Exception):
pass
-def complete_path(path_type, incomplete, base_directory='.'):
+def complete_path(path_type, incomplete, base_directory="."):
"""Helper method for implementing the completions() method
for File and Path parameter types.
"""
@@ -71,7 +71,7 @@ def complete_path(path_type, incomplete, base_directory='.'):
# specified in `incomplete` minus the last path component,
# otherwise list files starting from the current working directory.
entries = []
- base_path = ''
+ base_path = ""
# This is getting a bit messy
listed_base_directory = False
@@ -128,11 +128,11 @@ def complete_path(path_type, incomplete, base_directory='.'):
return [
# Return an appropriate path for each entry
- fix_path(e) for e in sorted(entries)
-
+ fix_path(e)
+ for e in sorted(entries)
# Filter out non directory elements when searching for a directory,
# the opposite is fine, however.
- if not (path_type == 'Directory' and not entry_is_dir(e))
+ if not (path_type == "Directory" and not entry_is_dir(e))
]
@@ -183,7 +183,7 @@ def start_of_option(param_str):
:param param_str: param_str to check
:return: whether or not this is the start of an option declaration (i.e. starts "-" or "--")
"""
- return param_str and param_str[:1] == '-'
+ return param_str and param_str[:1] == "-"
def is_incomplete_option(all_args, cmd_param):
@@ -218,8 +218,11 @@ def is_incomplete_argument(current_params, cmd_param):
return True
if cmd_param.nargs == -1:
return True
- if isinstance(current_param_values, collections.abc.Iterable) \
- and cmd_param.nargs > 1 and len(current_param_values) < cmd_param.nargs:
+ if (
+ isinstance(current_param_values, collections.abc.Iterable)
+ and cmd_param.nargs > 1
+ and len(current_param_values) < cmd_param.nargs
+ ):
return True
return False
@@ -237,10 +240,7 @@ def get_user_autocompletions(args, incomplete, cmd, cmd_param, override):
# Use the type specific default completions unless it was overridden
try:
- return override(cmd=cmd,
- cmd_param=cmd_param,
- args=args,
- incomplete=incomplete)
+ return override(cmd=cmd, cmd_param=cmd_param, args=args, incomplete=incomplete)
except CompleteUnhandled:
return get_param_type_completion(cmd_param.type, incomplete) or []
@@ -269,7 +269,7 @@ def get_choices(cli, prog_name, args, incomplete, override):
all_args.append(partition_incomplete[0])
incomplete = partition_incomplete[2]
elif incomplete == WORDBREAK:
- incomplete = ''
+ incomplete = ""
choices = []
found_param = False
@@ -277,8 +277,13 @@ def get_choices(cli, prog_name, args, incomplete, override):
# completions for options
for param in ctx.command.params:
if isinstance(param, Option):
- choices.extend([param_opt + " " for param_opt in param.opts + param.secondary_opts
- if param_opt not in all_args or param.multiple])
+ choices.extend(
+ [
+ param_opt + " "
+ for param_opt in param.opts + param.secondary_opts
+ if param_opt not in all_args or param.multiple
+ ]
+ )
found_param = True
if not found_param:
# completion for option values by choices
@@ -297,14 +302,22 @@ def get_choices(cli, prog_name, args, incomplete, override):
if not found_param and isinstance(ctx.command, MultiCommand):
# completion for any subcommands
- choices.extend([cmd + " " for cmd in ctx.command.list_commands(ctx)
- if not ctx.command.get_command(ctx, cmd).hidden])
-
- if not start_of_option(incomplete) and ctx.parent is not None \
- and isinstance(ctx.parent.command, MultiCommand) and ctx.parent.command.chain:
+ choices.extend(
+ [cmd + " " for cmd in ctx.command.list_commands(ctx) if not ctx.command.get_command(ctx, cmd).hidden]
+ )
+
+ if (
+ not start_of_option(incomplete)
+ and ctx.parent is not None
+ and isinstance(ctx.parent.command, MultiCommand)
+ and ctx.parent.command.chain
+ ):
# completion for chained commands
- visible_commands = [cmd for cmd in ctx.parent.command.list_commands(ctx.parent)
- if not ctx.parent.command.get_command(ctx.parent, cmd).hidden]
+ visible_commands = [
+ cmd
+ for cmd in ctx.parent.command.list_commands(ctx.parent)
+ if not ctx.parent.command.get_command(ctx.parent, cmd).hidden
+ ]
remaining_commands = set(visible_commands) - set(ctx.parent.protected_args)
choices.extend([cmd + " " for cmd in remaining_commands])
@@ -314,13 +327,13 @@ def get_choices(cli, prog_name, args, incomplete, override):
def do_complete(cli, prog_name, override):
- cwords = split_arg_string(os.environ['COMP_WORDS'])
- cword = int(os.environ['COMP_CWORD'])
+ cwords = split_arg_string(os.environ["COMP_WORDS"])
+ cword = int(os.environ["COMP_CWORD"])
args = cwords[1:cword]
try:
incomplete = cwords[cword]
except IndexError:
- incomplete = ''
+ incomplete = ""
for item in get_choices(cli, prog_name, args, incomplete, override):
click.echo(item)
@@ -331,7 +344,7 @@ def do_complete(cli, prog_name, override):
def main_bashcomplete(cmd, prog_name, override):
"""Internal handler for the bash completion support."""
- if '_BST_COMPLETION' in os.environ:
+ if "_BST_COMPLETION" in os.environ:
do_complete(cmd, prog_name, override)
return True
diff --git a/src/buildstream/_frontend/linuxapp.py b/src/buildstream/_frontend/linuxapp.py
index 0444dc7b4..987b023bd 100644
--- a/src/buildstream/_frontend/linuxapp.py
+++ b/src/buildstream/_frontend/linuxapp.py
@@ -28,9 +28,9 @@ from .app import App
#
def _osc_777_supported():
- term = os.environ.get('TERM')
+ term = os.environ.get("TERM")
- if term and (term.startswith('xterm') or term.startswith('vte')):
+ if term and (term.startswith("xterm") or term.startswith("vte")):
# Since vte version 4600, upstream silently ignores
# the OSC 777 without printing garbage to the terminal.
@@ -39,7 +39,7 @@ def _osc_777_supported():
# will trigger a desktop notification and bring attention
# to the terminal.
#
- vte_version = os.environ.get('VTE_VERSION')
+ vte_version = os.environ.get("VTE_VERSION")
try:
vte_version_int = int(vte_version)
except (ValueError, TypeError):
@@ -54,7 +54,6 @@ def _osc_777_supported():
# A linux specific App implementation
#
class LinuxApp(App):
-
def notify(self, title, text):
# Currently we only try this notification method
diff --git a/src/buildstream/_frontend/profile.py b/src/buildstream/_frontend/profile.py
index dda0f7ffe..f49be5b0a 100644
--- a/src/buildstream/_frontend/profile.py
+++ b/src/buildstream/_frontend/profile.py
@@ -28,7 +28,7 @@ import click
# Kwargs:
# The same keyword arguments which can be used with click.style()
#
-class Profile():
+class Profile:
def __init__(self, **kwargs):
self._kwargs = dict(kwargs)
@@ -64,7 +64,6 @@ class Profile():
# arguments
#
def fmt_subst(self, text, varname, value, **kwargs):
-
def subst_callback(match):
# Extract and format the "{(varname)...}" portion of the match
inner_token = match.group(1)
diff --git a/src/buildstream/_frontend/status.py b/src/buildstream/_frontend/status.py
index 85fdf7451..a3f0d8aa7 100644
--- a/src/buildstream/_frontend/status.py
+++ b/src/buildstream/_frontend/status.py
@@ -43,19 +43,12 @@ from .widget import TimeCode
# error_profile (Profile): Formatting profile for error text
# stream (Stream): The Stream
#
-class Status():
+class Status:
# Table of the terminal capabilities we require and use
- _TERM_CAPABILITIES = {
- 'move_up': 'cuu1',
- 'move_x': 'hpa',
- 'clear_eol': 'el'
- }
+ _TERM_CAPABILITIES = {"move_up": "cuu1", "move_x": "hpa", "clear_eol": "el"}
- def __init__(self, context, state,
- content_profile, format_profile,
- success_profile, error_profile,
- stream):
+ def __init__(self, context, state, content_profile, format_profile, success_profile, error_profile, stream):
self._context = context
self._state = state
@@ -67,10 +60,9 @@ class Status():
self._jobs = OrderedDict()
self._last_lines = 0 # Number of status lines we last printed to console
self._spacing = 1
- self._header = _StatusHeader(context, state,
- content_profile, format_profile,
- success_profile, error_profile,
- stream)
+ self._header = _StatusHeader(
+ context, state, content_profile, format_profile, success_profile, error_profile, stream
+ )
self._term_width, _ = click.get_terminal_size()
self._alloc_lines = 0
@@ -131,7 +123,7 @@ class Status():
# feeds for the amount of lines we intend to print first, and
# move cursor position back to the first line
for _ in range(self._alloc_lines + self._header.lines):
- click.echo('', err=True)
+ click.echo("", err=True)
for _ in range(self._alloc_lines + self._header.lines):
self._move_up()
@@ -143,14 +135,14 @@ class Status():
# alignment of each column
n_columns = len(self._alloc_columns)
for line in self._job_lines(n_columns):
- text = ''
+ text = ""
for job in line:
column = line.index(job)
text += job.render(self._alloc_columns[column] - job.size, elapsed)
# Add spacing between columns
if column < (n_columns - 1):
- text += ' ' * self._spacing
+ text += " " * self._spacing
# Print the line
click.echo(text, err=True)
@@ -196,7 +188,7 @@ class Status():
# Initialized terminal, curses might decide it doesnt
# support this terminal
try:
- curses.setupterm(os.environ.get('TERM', 'dumb'))
+ curses.setupterm(os.environ.get("TERM", "dumb"))
except curses.error:
return None
@@ -221,7 +213,7 @@ class Status():
# as well, and should provide better compatibility with most
# terminals.
#
- term_caps[capname] = code.decode('latin1')
+ term_caps[capname] = code.decode("latin1")
return term_caps
@@ -236,19 +228,19 @@ class Status():
# Explicitly move to beginning of line, fixes things up
# when there was a ^C or ^Z printed to the terminal.
- move_x = curses.tparm(self._term_caps['move_x'].encode('latin1'), 0)
- move_x = move_x.decode('latin1')
+ move_x = curses.tparm(self._term_caps["move_x"].encode("latin1"), 0)
+ move_x = move_x.decode("latin1")
- move_up = curses.tparm(self._term_caps['move_up'].encode('latin1'))
- move_up = move_up.decode('latin1')
+ move_up = curses.tparm(self._term_caps["move_up"].encode("latin1"))
+ move_up = move_up.decode("latin1")
click.echo(move_x + move_up, nl=False, err=True)
def _clear_line(self):
assert self._term_caps is not None
- clear_eol = curses.tparm(self._term_caps['clear_eol'].encode('latin1'))
- clear_eol = clear_eol.decode('latin1')
+ clear_eol = curses.tparm(self._term_caps["clear_eol"].encode("latin1"))
+ clear_eol = clear_eol.decode("latin1")
click.echo(clear_eol, nl=False, err=True)
def _allocate(self):
@@ -277,7 +269,7 @@ class Status():
def _job_lines(self, columns):
jobs_list = list(self._jobs.values())
for i in range(0, len(self._jobs), columns):
- yield jobs_list[i:i + columns]
+ yield jobs_list[i : i + columns]
# Returns an array of integers representing the maximum
# length in characters for each column, given the current
@@ -307,9 +299,7 @@ class Status():
def _add_job(self, action_name, full_name):
task = self._state.tasks[(action_name, full_name)]
elapsed = task.elapsed_offset
- job = _StatusJob(self._context, action_name, full_name,
- self._content_profile, self._format_profile,
- elapsed)
+ job = _StatusJob(self._context, action_name, full_name, self._content_profile, self._format_profile, elapsed)
self._jobs[(action_name, full_name)] = job
self._need_alloc = True
@@ -338,12 +328,8 @@ class Status():
# error_profile (Profile): Formatting profile for error text
# stream (Stream): The Stream
#
-class _StatusHeader():
-
- def __init__(self, context, state,
- content_profile, format_profile,
- success_profile, error_profile,
- stream):
+class _StatusHeader:
+ def __init__(self, context, state, content_profile, format_profile, success_profile, error_profile, stream):
#
# Public members
@@ -375,19 +361,22 @@ class _StatusHeader():
total = str(len(self._stream.total_elements))
size = 0
- text = ''
+ text = ""
size += len(total) + len(session) + 4 # Size for (N/N) with a leading space
size += 8 # Size of time code
size += len(project.name) + 1
text += self._time_code.render_time(elapsed)
- text += ' ' + self._content_profile.fmt(project.name)
- text += ' ' + self._format_profile.fmt('(') + \
- self._content_profile.fmt(session) + \
- self._format_profile.fmt('/') + \
- self._content_profile.fmt(total) + \
- self._format_profile.fmt(')')
-
- line1 = self._centered(text, size, line_length, '=')
+ text += " " + self._content_profile.fmt(project.name)
+ text += (
+ " "
+ + self._format_profile.fmt("(")
+ + self._content_profile.fmt(session)
+ + self._format_profile.fmt("/")
+ + self._content_profile.fmt(total)
+ + self._format_profile.fmt(")")
+ )
+
+ line1 = self._centered(text, size, line_length, "=")
#
# Line 2: Dynamic list of queue status reports
@@ -395,7 +384,7 @@ class _StatusHeader():
# (Sources Fetched:0 117 0)→ (Built:4 0 0)
#
size = 0
- text = ''
+ text = ""
# Format and calculate size for each queue progress
for index, task_group in enumerate(self._state.task_groups.values()):
@@ -403,13 +392,13 @@ class _StatusHeader():
# Add spacing
if index > 0:
size += 2
- text += self._format_profile.fmt('→ ')
+ text += self._format_profile.fmt("→ ")
group_text, group_size = self._render_task_group(task_group)
size += group_size
text += group_text
- line2 = self._centered(text, size, line_length, ' ')
+ line2 = self._centered(text, size, line_length, " ")
#
# Line 3: Cache usage percentage report
@@ -423,7 +412,7 @@ class _StatusHeader():
if usage.used_size is None:
# Cache usage is unknown
size = 0
- text = ''
+ text = ""
else:
size = 21
size += len(usage_string)
@@ -434,15 +423,17 @@ class _StatusHeader():
else:
formatted_usage = self._success_profile.fmt(usage_string)
- text = self._format_profile.fmt("~~~~~~ ") + \
- self._content_profile.fmt('cache') + \
- self._format_profile.fmt(': ') + \
- formatted_usage + \
- self._format_profile.fmt(' ~~~~~~')
+ text = (
+ self._format_profile.fmt("~~~~~~ ")
+ + self._content_profile.fmt("cache")
+ + self._format_profile.fmt(": ")
+ + formatted_usage
+ + self._format_profile.fmt(" ~~~~~~")
+ )
- line3 = self._centered(text, size, line_length, ' ')
+ line3 = self._centered(text, size, line_length, " ")
- return line1 + '\n' + line2 + '\n' + line3
+ return line1 + "\n" + line2 + "\n" + line3
###################################################
# Private Methods #
@@ -455,13 +446,17 @@ class _StatusHeader():
size = 5 # Space for the formatting '[', ':', ' ', ' ' and ']'
size += len(group.complete_name)
size += len(processed) + len(skipped) + len(failed)
- text = self._format_profile.fmt("(") + \
- self._content_profile.fmt(group.complete_name) + \
- self._format_profile.fmt(":") + \
- self._success_profile.fmt(processed) + ' ' + \
- self._content_profile.fmt(skipped) + ' ' + \
- self._error_profile.fmt(failed) + \
- self._format_profile.fmt(")")
+ text = (
+ self._format_profile.fmt("(")
+ + self._content_profile.fmt(group.complete_name)
+ + self._format_profile.fmt(":")
+ + self._success_profile.fmt(processed)
+ + " "
+ + self._content_profile.fmt(skipped)
+ + " "
+ + self._error_profile.fmt(failed)
+ + self._format_profile.fmt(")")
+ )
return (text, size)
@@ -469,9 +464,9 @@ class _StatusHeader():
remaining = line_length - size
remaining -= 2
- final_text = self._format_profile.fmt(fill * (remaining // 2)) + ' '
+ final_text = self._format_profile.fmt(fill * (remaining // 2)) + " "
final_text += text
- final_text += ' ' + self._format_profile.fmt(fill * (remaining // 2))
+ final_text += " " + self._format_profile.fmt(fill * (remaining // 2))
return final_text
@@ -488,14 +483,13 @@ class _StatusHeader():
# format_profile (Profile): Formatting profile for formatting text
# elapsed (datetime): The offset into the session when this job is created
#
-class _StatusJob():
-
+class _StatusJob:
def __init__(self, context, action_name, full_name, content_profile, format_profile, elapsed):
#
# Public members
#
- self.action_name = action_name # The action name
- self.size = None # The number of characters required to render
+ self.action_name = action_name # The action name
+ self.size = None # The number of characters required to render
self.full_name = full_name
#
@@ -568,24 +562,26 @@ class _StatusJob():
# elapsed (datetime): The session elapsed time offset
#
def render(self, padding, elapsed):
- text = self._format_profile.fmt('[') + \
- self._time_code.render_time(elapsed - self._offset) + \
- self._format_profile.fmt(']')
-
- text += self._format_profile.fmt('[') + \
- self._content_profile.fmt(self.action_name) + \
- self._format_profile.fmt(':') + \
- self._content_profile.fmt(self.full_name)
+ text = (
+ self._format_profile.fmt("[")
+ + self._time_code.render_time(elapsed - self._offset)
+ + self._format_profile.fmt("]")
+ )
+
+ text += (
+ self._format_profile.fmt("[")
+ + self._content_profile.fmt(self.action_name)
+ + self._format_profile.fmt(":")
+ + self._content_profile.fmt(self.full_name)
+ )
if self._current_progress is not None:
- text += self._format_profile.fmt(':') + \
- self._content_profile.fmt(str(self._current_progress))
+ text += self._format_profile.fmt(":") + self._content_profile.fmt(str(self._current_progress))
if self._maximum_progress is not None:
- text += self._format_profile.fmt('/') + \
- self._content_profile.fmt(str(self._maximum_progress))
+ text += self._format_profile.fmt("/") + self._content_profile.fmt(str(self._maximum_progress))
# Add padding before terminating ']'
- terminator = (' ' * padding) + ']'
+ terminator = (" " * padding) + "]"
text += self._format_profile.fmt(terminator)
return text
diff --git a/src/buildstream/_frontend/widget.py b/src/buildstream/_frontend/widget.py
index 0a268b717..63fbfbb7d 100644
--- a/src/buildstream/_frontend/widget.py
+++ b/src/buildstream/_frontend/widget.py
@@ -45,8 +45,7 @@ ERROR_MESSAGES = [MessageType.FAIL, MessageType.ERROR, MessageType.BUG]
#
# An abstract class for printing output columns in our text UI.
#
-class Widget():
-
+class Widget:
def __init__(self, context, content_profile, format_profile):
# The context
@@ -74,7 +73,6 @@ class Widget():
# Used to add fixed text between columns
class FixedText(Widget):
-
def __init__(self, context, text, content_profile, format_profile):
super().__init__(context, content_profile, format_profile)
self.text = text
@@ -91,15 +89,13 @@ class WallclockTime(Widget):
def render(self, message):
- fields = [self.content_profile.fmt("{:02d}".format(x)) for x in
- [message.creation_time.hour,
- message.creation_time.minute,
- message.creation_time.second,
- ]
- ]
+ fields = [
+ self.content_profile.fmt("{:02d}".format(x))
+ for x in [message.creation_time.hour, message.creation_time.minute, message.creation_time.second,]
+ ]
text = self.format_profile.fmt(":").join(fields)
- if self._output_format == 'us':
+ if self._output_format == "us":
text += self.content_profile.fmt(".{:06d}".format(message.creation_time.microsecond))
return text
@@ -107,11 +103,10 @@ class WallclockTime(Widget):
# A widget for rendering the debugging column
class Debug(Widget):
-
def render(self, message):
element_name = "n/a" if message.element_name is None else message.element_name
- text = self.format_profile.fmt('pid:')
+ text = self.format_profile.fmt("pid:")
text += self.content_profile.fmt("{: <5}".format(message.pid))
text += self.format_profile.fmt("element name:")
text += self.content_profile.fmt("{: <30}".format(element_name))
@@ -130,19 +125,13 @@ class TimeCode(Widget):
def render_time(self, elapsed):
if elapsed is None:
- fields = [
- self.content_profile.fmt('--')
- for i in range(3)
- ]
+ fields = [self.content_profile.fmt("--") for i in range(3)]
else:
hours, remainder = divmod(int(elapsed.total_seconds()), 60 * 60)
minutes, seconds = divmod(remainder, 60)
- fields = [
- self.content_profile.fmt("{0:02d}".format(field))
- for field in [hours, minutes, seconds]
- ]
+ fields = [self.content_profile.fmt("{0:02d}".format(field)) for field in [hours, minutes, seconds]]
- text = self.format_profile.fmt(':').join(fields)
+ text = self.format_profile.fmt(":").join(fields)
if self._microseconds:
if elapsed is not None:
@@ -169,41 +158,43 @@ class TypeName(Widget):
}
def render(self, message):
- return self.content_profile.fmt("{: <7}"
- .format(message.message_type.upper()),
- bold=True, dim=True,
- fg=self._action_colors[message.message_type])
+ return self.content_profile.fmt(
+ "{: <7}".format(message.message_type.upper()),
+ bold=True,
+ dim=True,
+ fg=self._action_colors[message.message_type],
+ )
# A widget for displaying the Element name
class ElementName(Widget):
-
def render(self, message):
action_name = message.action_name
element_name = message.element_name
if element_name is not None:
- name = '{: <30}'.format(element_name)
+ name = "{: <30}".format(element_name)
else:
- name = 'core activity'
- name = '{: <30}'.format(name)
+ name = "core activity"
+ name = "{: <30}".format(name)
if not action_name:
action_name = "Main"
- return self.content_profile.fmt("{: >8}".format(action_name.lower())) + \
- self.format_profile.fmt(':') + self.content_profile.fmt(name)
+ return (
+ self.content_profile.fmt("{: >8}".format(action_name.lower()))
+ + self.format_profile.fmt(":")
+ + self.content_profile.fmt(name)
+ )
# A widget for displaying the primary message text
class MessageText(Widget):
-
def render(self, message):
return message.message
# A widget for formatting the element cache key
class CacheKey(Widget):
-
def __init__(self, context, content_profile, format_profile, err_profile):
super().__init__(context, content_profile, format_profile)
@@ -216,10 +207,10 @@ class CacheKey(Widget):
return ""
if message.element_name is None:
- return ' ' * self._key_length
+ return " " * self._key_length
missing = False
- key = ' ' * self._key_length
+ key = " " * self._key_length
if message.element_key:
_, key, missing = message.element_key
@@ -233,7 +224,6 @@ class CacheKey(Widget):
# A widget for formatting the log file
class LogFile(Widget):
-
def __init__(self, context, content_profile, format_profile, err_profile):
super().__init__(context, content_profile, format_profile)
@@ -248,7 +238,7 @@ class LogFile(Widget):
logfile = message.logfile
if abbrev and self._logdir != "" and logfile.startswith(self._logdir):
- logfile = logfile[len(self._logdir):]
+ logfile = logfile[len(self._logdir) :]
logfile = logfile.lstrip(os.sep)
if message.message_type in ERROR_MESSAGES:
@@ -256,7 +246,7 @@ class LogFile(Widget):
else:
text = self.content_profile.fmt(logfile, dim=True)
else:
- text = ''
+ text = ""
return text
@@ -273,8 +263,7 @@ class MessageOrLogFile(Widget):
def render(self, message):
# Show the log file only in the main start/success messages
- if message.logfile and message.scheduler and \
- message.message_type in [MessageType.START, MessageType.SUCCESS]:
+ if message.logfile and message.scheduler and message.message_type in [MessageType.START, MessageType.SUCCESS]:
text = self._logfile_widget.render(message)
else:
text = self._message_widget.render(message)
@@ -296,14 +285,9 @@ class MessageOrLogFile(Widget):
# indent (int): Number of spaces to use for general indentation
#
class LogLine(Widget):
-
- def __init__(self, context, state,
- content_profile,
- format_profile,
- success_profile,
- err_profile,
- detail_profile,
- indent=4):
+ def __init__(
+ self, context, state, content_profile, format_profile, success_profile, err_profile, detail_profile, indent=4
+ ):
super().__init__(context, content_profile, format_profile)
self._columns = []
@@ -311,7 +295,7 @@ class LogLine(Widget):
self._success_profile = success_profile
self._err_profile = err_profile
self._detail_profile = detail_profile
- self._indent = ' ' * indent
+ self._indent = " " * indent
self._log_lines = context.log_error_lines
self._message_lines = context.log_message_lines
self._resolved_keys = None
@@ -320,19 +304,17 @@ class LogLine(Widget):
self._logfile_widget = LogFile(context, content_profile, format_profile, err_profile)
if context.log_debug:
- self._columns.extend([
- Debug(context, content_profile, format_profile)
- ])
+ self._columns.extend([Debug(context, content_profile, format_profile)])
self.logfile_variable_names = {
"elapsed": TimeCode(context, content_profile, format_profile, microseconds=False),
"elapsed-us": TimeCode(context, content_profile, format_profile, microseconds=True),
"wallclock": WallclockTime(context, content_profile, format_profile),
- "wallclock-us": WallclockTime(context, content_profile, format_profile, output_format='us'),
+ "wallclock-us": WallclockTime(context, content_profile, format_profile, output_format="us"),
"key": CacheKey(context, content_profile, format_profile, err_profile),
"element": ElementName(context, content_profile, format_profile),
"action": TypeName(context, content_profile, format_profile),
- "message": MessageOrLogFile(context, content_profile, format_profile, err_profile)
+ "message": MessageOrLogFile(context, content_profile, format_profile, err_profile),
}
logfile_tokens = self._parse_logfile_format(context.log_message_format, content_profile, format_profile)
self._columns.extend(logfile_tokens)
@@ -352,7 +334,7 @@ class LogLine(Widget):
# (str): The formatted list of elements
#
def show_pipeline(self, dependencies, format_):
- report = ''
+ report = ""
p = Profile()
for element in dependencies:
@@ -360,57 +342,57 @@ class LogLine(Widget):
full_key, cache_key, dim_keys = element._get_display_key()
- line = p.fmt_subst(line, 'name', element._get_full_name(), fg='blue', bold=True)
- line = p.fmt_subst(line, 'key', cache_key, fg='yellow', dim=dim_keys)
- line = p.fmt_subst(line, 'full-key', full_key, fg='yellow', dim=dim_keys)
+ line = p.fmt_subst(line, "name", element._get_full_name(), fg="blue", bold=True)
+ line = p.fmt_subst(line, "key", cache_key, fg="yellow", dim=dim_keys)
+ line = p.fmt_subst(line, "full-key", full_key, fg="yellow", dim=dim_keys)
consistency = element._get_consistency()
if consistency == Consistency.INCONSISTENT:
- line = p.fmt_subst(line, 'state', "no reference", fg='red')
+ line = p.fmt_subst(line, "state", "no reference", fg="red")
else:
if element._cached_failure():
- line = p.fmt_subst(line, 'state', "failed", fg='red')
+ line = p.fmt_subst(line, "state", "failed", fg="red")
elif element._cached_success():
- line = p.fmt_subst(line, 'state', "cached", fg='magenta')
+ line = p.fmt_subst(line, "state", "cached", fg="magenta")
elif consistency == Consistency.RESOLVED and not element._source_cached():
- line = p.fmt_subst(line, 'state', "fetch needed", fg='red')
+ line = p.fmt_subst(line, "state", "fetch needed", fg="red")
elif element._buildable():
- line = p.fmt_subst(line, 'state', "buildable", fg='green')
+ line = p.fmt_subst(line, "state", "buildable", fg="green")
else:
- line = p.fmt_subst(line, 'state', "waiting", fg='blue')
+ line = p.fmt_subst(line, "state", "waiting", fg="blue")
# Element configuration
if "%{config" in format_:
line = p.fmt_subst(
- line, 'config',
- yaml.round_trip_dump(element._Element__config, default_flow_style=False, allow_unicode=True))
+ line,
+ "config",
+ yaml.round_trip_dump(element._Element__config, default_flow_style=False, allow_unicode=True),
+ )
# Variables
if "%{vars" in format_:
variables = element._Element__variables.flat
line = p.fmt_subst(
- line, 'vars',
- yaml.round_trip_dump(variables, default_flow_style=False, allow_unicode=True))
+ line, "vars", yaml.round_trip_dump(variables, default_flow_style=False, allow_unicode=True)
+ )
# Environment
if "%{env" in format_:
environment = element._Element__environment
line = p.fmt_subst(
- line, 'env',
- yaml.round_trip_dump(environment, default_flow_style=False, allow_unicode=True))
+ line, "env", yaml.round_trip_dump(environment, default_flow_style=False, allow_unicode=True)
+ )
# Public
if "%{public" in format_:
environment = element._Element__public
line = p.fmt_subst(
- line, 'public',
- yaml.round_trip_dump(environment, default_flow_style=False, allow_unicode=True))
+ line, "public", yaml.round_trip_dump(environment, default_flow_style=False, allow_unicode=True)
+ )
# Workspaced
if "%{workspaced" in format_:
- line = p.fmt_subst(
- line, 'workspaced',
- '(workspaced)' if element._get_workspace() else '', fg='yellow')
+ line = p.fmt_subst(line, "workspaced", "(workspaced)" if element._get_workspace() else "", fg="yellow")
# Workspace-dirs
if "%{workspace-dirs" in format_:
@@ -418,36 +400,31 @@ class LogLine(Widget):
if workspace is not None:
path = workspace.get_absolute_path()
if path.startswith("~/"):
- path = os.path.join(os.getenv('HOME', '/root'), path[2:])
- line = p.fmt_subst(line, 'workspace-dirs', "Workspace: {}".format(path))
+ path = os.path.join(os.getenv("HOME", "/root"), path[2:])
+ line = p.fmt_subst(line, "workspace-dirs", "Workspace: {}".format(path))
else:
- line = p.fmt_subst(
- line, 'workspace-dirs', '')
+ line = p.fmt_subst(line, "workspace-dirs", "")
# Dependencies
if "%{deps" in format_:
deps = [e.name for e in element.dependencies(Scope.ALL, recurse=False)]
- line = p.fmt_subst(
- line, 'deps',
- yaml.safe_dump(deps, default_style=None).rstrip('\n'))
+ line = p.fmt_subst(line, "deps", yaml.safe_dump(deps, default_style=None).rstrip("\n"))
# Build Dependencies
if "%{build-deps" in format_:
build_deps = [e.name for e in element.dependencies(Scope.BUILD, recurse=False)]
- line = p.fmt_subst(
- line, 'build-deps',
- yaml.safe_dump(build_deps, default_style=False).rstrip('\n'))
+ line = p.fmt_subst(line, "build-deps", yaml.safe_dump(build_deps, default_style=False).rstrip("\n"))
# Runtime Dependencies
if "%{runtime-deps" in format_:
runtime_deps = [e.name for e in element.dependencies(Scope.RUN, recurse=False)]
line = p.fmt_subst(
- line, 'runtime-deps',
- yaml.safe_dump(runtime_deps, default_style=False).rstrip('\n'))
+ line, "runtime-deps", yaml.safe_dump(runtime_deps, default_style=False).rstrip("\n")
+ )
- report += line + '\n'
+ report += line + "\n"
- return report.rstrip('\n')
+ return report.rstrip("\n")
# print_heading()
#
@@ -463,25 +440,24 @@ class LogLine(Widget):
def print_heading(self, project, stream, *, log_file):
context = self.context
starttime = datetime.datetime.now()
- text = ''
+ text = ""
self._resolved_keys = {element: element._get_cache_key() for element in stream.session_elements}
# Main invocation context
- text += '\n'
+ text += "\n"
text += self.content_profile.fmt("BuildStream Version {}\n".format(bst_version), bold=True)
values = OrderedDict()
- values["Session Start"] = starttime.strftime('%A, %d-%m-%Y at %H:%M:%S')
+ values["Session Start"] = starttime.strftime("%A, %d-%m-%Y at %H:%M:%S")
values["Project"] = "{} ({})".format(project.name, project.directory)
values["Targets"] = ", ".join([t.name for t in stream.targets])
text += self._format_values(values)
# User configurations
- text += '\n'
+ text += "\n"
text += self.content_profile.fmt("User Configuration\n", bold=True)
values = OrderedDict()
- values["Configuration File"] = \
- "Default Configuration" if not context.config_origin else context.config_origin
+ values["Configuration File"] = "Default Configuration" if not context.config_origin else context.config_origin
values["Cache Directory"] = context.cachedir
values["Log Files"] = context.logdir
values["Source Mirrors"] = context.sourcedir
@@ -492,7 +468,7 @@ class LogLine(Widget):
values["Maximum Push Tasks"] = context.sched_pushers
values["Maximum Network Retries"] = context.sched_network_retries
text += self._format_values(values)
- text += '\n'
+ text += "\n"
# Project Options
values = OrderedDict()
@@ -500,22 +476,25 @@ class LogLine(Widget):
if values:
text += self.content_profile.fmt("Project Options\n", bold=True)
text += self._format_values(values)
- text += '\n'
+ text += "\n"
# Plugins
- text += self._format_plugins(project.first_pass_config.element_factory.loaded_dependencies,
- project.first_pass_config.source_factory.loaded_dependencies)
+ text += self._format_plugins(
+ project.first_pass_config.element_factory.loaded_dependencies,
+ project.first_pass_config.source_factory.loaded_dependencies,
+ )
if project.config.element_factory and project.config.source_factory:
- text += self._format_plugins(project.config.element_factory.loaded_dependencies,
- project.config.source_factory.loaded_dependencies)
+ text += self._format_plugins(
+ project.config.element_factory.loaded_dependencies, project.config.source_factory.loaded_dependencies
+ )
# Pipeline state
text += self.content_profile.fmt("Pipeline\n", bold=True)
text += self.show_pipeline(stream.total_elements, context.log_element_format)
- text += '\n'
+ text += "\n"
# Separator line before following output
- text += self.format_profile.fmt("=" * 79 + '\n')
+ text += self.format_profile.fmt("=" * 79 + "\n")
click.echo(text, nl=False, err=True)
if log_file:
@@ -537,7 +516,7 @@ class LogLine(Widget):
if not self._state.task_groups:
return
- text = ''
+ text = ""
assert self._resolved_keys is not None
elements = sorted(e for (e, k) in self._resolved_keys.items() if k != e._get_cache_key())
@@ -554,7 +533,7 @@ class LogLine(Widget):
# Exclude the failure messages if the job didn't ultimately fail
# (e.g. succeeded on retry)
if element_name in group.failed_tasks:
- values[element_name] = ''.join(self._render(v) for v in messages)
+ values[element_name] = "".join(self._render(v) for v in messages)
if values:
text += self.content_profile.fmt("Failure Summary\n", bold=True)
@@ -563,8 +542,8 @@ class LogLine(Widget):
text += self.content_profile.fmt("Pipeline Summary\n", bold=True)
values = OrderedDict()
- values['Total'] = self.content_profile.fmt(str(len(stream.total_elements)))
- values['Session'] = self.content_profile.fmt(str(len(stream.session_elements)))
+ values["Total"] = self.content_profile.fmt(str(len(stream.total_elements)))
+ values["Session"] = self.content_profile.fmt(str(len(stream.session_elements)))
processed_maxlen = 1
skipped_maxlen = 1
@@ -579,20 +558,25 @@ class LogLine(Widget):
skipped = str(group.skipped_tasks)
failed = str(len(group.failed_tasks))
- processed_align = ' ' * (processed_maxlen - len(processed))
- skipped_align = ' ' * (skipped_maxlen - len(skipped))
- failed_align = ' ' * (failed_maxlen - len(failed))
-
- status_text = self.content_profile.fmt("processed ") + \
- self._success_profile.fmt(processed) + \
- self.format_profile.fmt(', ') + processed_align
-
- status_text += self.content_profile.fmt("skipped ") + \
- self.content_profile.fmt(skipped) + \
- self.format_profile.fmt(', ') + skipped_align
-
- status_text += self.content_profile.fmt("failed ") + \
- self._err_profile.fmt(failed) + ' ' + failed_align
+ processed_align = " " * (processed_maxlen - len(processed))
+ skipped_align = " " * (skipped_maxlen - len(skipped))
+ failed_align = " " * (failed_maxlen - len(failed))
+
+ status_text = (
+ self.content_profile.fmt("processed ")
+ + self._success_profile.fmt(processed)
+ + self.format_profile.fmt(", ")
+ + processed_align
+ )
+
+ status_text += (
+ self.content_profile.fmt("skipped ")
+ + self.content_profile.fmt(skipped)
+ + self.format_profile.fmt(", ")
+ + skipped_align
+ )
+
+ status_text += self.content_profile.fmt("failed ") + self._err_profile.fmt(failed) + " " + failed_align
values["{} Queue".format(group.name)] = status_text
text += self._format_values(values, style_value=False)
@@ -627,7 +611,7 @@ class LogLine(Widget):
m = re.search(r"^%\{([^\}]+)\}", format_string)
if m is not None:
variable = m.group(1)
- format_string = format_string[m.end(0):]
+ format_string = format_string[m.end(0) :]
if variable not in self.logfile_variable_names:
raise Exception("'{0}' is not a valid log variable name.".format(variable))
logfile_tokens.append(self.logfile_variable_names[variable])
@@ -635,7 +619,7 @@ class LogLine(Widget):
m = re.search("^[^%]+", format_string)
if m is not None:
text = FixedText(self.context, m.group(0), content_profile, format_profile)
- format_string = format_string[m.end(0):]
+ format_string = format_string[m.end(0) :]
logfile_tokens.append(text)
else:
# No idea what to do now
@@ -645,11 +629,11 @@ class LogLine(Widget):
def _render(self, message):
# Render the column widgets first
- text = ''
+ text = ""
for widget in self._columns:
text += widget.render(message)
- text += '\n'
+ text += "\n"
extra_nl = False
@@ -664,51 +648,53 @@ class LogLine(Widget):
n_lines = len(lines)
abbrev = False
- if message.message_type not in ERROR_MESSAGES \
- and not frontend_message and n_lines > self._message_lines:
- lines = lines[0:self._message_lines]
+ if message.message_type not in ERROR_MESSAGES and not frontend_message and n_lines > self._message_lines:
+ lines = lines[0 : self._message_lines]
if self._message_lines > 0:
abbrev = True
else:
- lines[n_lines - 1] = lines[n_lines - 1].rstrip('\n')
+ lines[n_lines - 1] = lines[n_lines - 1].rstrip("\n")
detail = self._indent + self._indent.join(lines)
- text += '\n'
+ text += "\n"
if message.message_type in ERROR_MESSAGES:
text += self._err_profile.fmt(detail, bold=True)
else:
text += self._detail_profile.fmt(detail)
if abbrev:
- text += self._indent + \
- self.content_profile.fmt('Message contains {} additional lines'
- .format(n_lines - self._message_lines), dim=True)
- text += '\n'
+ text += self._indent + self.content_profile.fmt(
+ "Message contains {} additional lines".format(n_lines - self._message_lines), dim=True
+ )
+ text += "\n"
extra_nl = True
if message.scheduler and message.message_type == MessageType.FAIL:
- text += '\n'
+ text += "\n"
if self.context is not None and not self.context.log_verbose:
text += self._indent + self._err_profile.fmt("Log file: ")
- text += self._indent + self._logfile_widget.render(message) + '\n'
+ text += self._indent + self._logfile_widget.render(message) + "\n"
elif self._log_lines > 0:
- text += self._indent + self._err_profile.fmt("Printing the last {} lines from log file:"
- .format(self._log_lines)) + '\n'
- text += self._indent + self._logfile_widget.render_abbrev(message, abbrev=False) + '\n'
- text += self._indent + self._err_profile.fmt("=" * 70) + '\n'
+ text += (
+ self._indent
+ + self._err_profile.fmt("Printing the last {} lines from log file:".format(self._log_lines))
+ + "\n"
+ )
+ text += self._indent + self._logfile_widget.render_abbrev(message, abbrev=False) + "\n"
+ text += self._indent + self._err_profile.fmt("=" * 70) + "\n"
log_content = self._read_last_lines(message.logfile)
log_content = textwrap.indent(log_content, self._indent)
text += self._detail_profile.fmt(log_content)
- text += '\n'
- text += self._indent + self._err_profile.fmt("=" * 70) + '\n'
+ text += "\n"
+ text += self._indent + self._err_profile.fmt("=" * 70) + "\n"
extra_nl = True
if extra_nl:
- text += '\n'
+ text += "\n"
return text
@@ -716,14 +702,14 @@ class LogLine(Widget):
with ExitStack() as stack:
# mmap handles low-level memory details, allowing for
# faster searches
- f = stack.enter_context(open(logfile, 'r+'))
+ f = stack.enter_context(open(logfile, "r+"))
log = stack.enter_context(mmap(f.fileno(), os.path.getsize(f.name)))
count = 0
end = log.size() - 1
while count < self._log_lines and end >= 0:
- location = log.rfind(b'\n', 0, end)
+ location = log.rfind(b"\n", 0, end)
count += 1
# If location is -1 (none found), this will print the
@@ -735,8 +721,8 @@ class LogLine(Widget):
# then we get the first characther. If end is a newline position,
# we discard it and only want to print the beginning of the next
# line.
- lines = log[(end + 1):].splitlines()
- return '\n'.join([line.decode('utf-8') for line in lines]).rstrip()
+ lines = log[(end + 1) :].splitlines()
+ return "\n".join([line.decode("utf-8") for line in lines]).rstrip()
def _format_plugins(self, element_plugins, source_plugins):
text = ""
@@ -756,7 +742,7 @@ class LogLine(Widget):
for plugin in source_plugins:
text += self.content_profile.fmt(" - {}\n".format(plugin))
- text += '\n'
+ text += "\n"
return text
@@ -773,23 +759,23 @@ class LogLine(Widget):
# (str): The formatted values
#
def _format_values(self, values, style_value=True):
- text = ''
+ text = ""
max_key_len = 0
for key, value in values.items():
max_key_len = max(len(key), max_key_len)
for key, value in values.items():
- if isinstance(value, str) and '\n' in value:
+ if isinstance(value, str) and "\n" in value:
text += self.format_profile.fmt(" {}:\n".format(key))
text += textwrap.indent(value, self._indent)
continue
- text += self.format_profile.fmt(" {}: {}".format(key, ' ' * (max_key_len - len(key))))
+ text += self.format_profile.fmt(" {}: {}".format(key, " " * (max_key_len - len(key))))
if style_value:
text += self.content_profile.fmt(str(value))
else:
text += str(value)
- text += '\n'
+ text += "\n"
return text
@@ -806,20 +792,20 @@ class LogLine(Widget):
# (str): The formatted values
#
def _pretty_print_dictionary(self, values, long_=False, style_value=True):
- text = ''
+ text = ""
max_key_len = 0
try:
max_key_len = max(len(key) for key in values.keys())
except ValueError:
- text = ''
+ text = ""
for key, value in values.items():
- if isinstance(value, str) and '\n' in value:
+ if isinstance(value, str) and "\n" in value:
text += self.format_profile.fmt(" {}:".format(key))
text += textwrap.indent(value, self._indent)
continue
- text += self.format_profile.fmt(" {}:{}".format(key, ' ' * (max_key_len - len(key))))
+ text += self.format_profile.fmt(" {}:{}".format(key, " " * (max_key_len - len(key))))
value_list = "\n\t" + "\n\t".join((self._get_filestats(v, list_long=long_) for v in value))
if value == []:
@@ -832,7 +818,7 @@ class LogLine(Widget):
text += self.content_profile.fmt(value_list)
else:
text += value_list
- text += '\n'
+ text += "\n"
return text
@@ -854,22 +840,22 @@ class LogLine(Widget):
# cached status of
#
def show_state_of_artifacts(self, targets):
- report = ''
+ report = ""
p = Profile()
for element in targets:
- line = '%{state: >12} %{name}'
- line = p.fmt_subst(line, 'name', element.name, fg='yellow')
+ line = "%{state: >12} %{name}"
+ line = p.fmt_subst(line, "name", element.name, fg="yellow")
if element._cached_success():
- line = p.fmt_subst(line, 'state', "cached", fg='magenta')
+ line = p.fmt_subst(line, "state", "cached", fg="magenta")
elif element._cached():
- line = p.fmt_subst(line, 'state', "failed", fg='red')
+ line = p.fmt_subst(line, "state", "failed", fg="red")
elif element._cached_remotely():
- line = p.fmt_subst(line, 'state', "available", fg='green')
+ line = p.fmt_subst(line, "state", "available", fg="green")
else:
- line = p.fmt_subst(line, 'state', "not cached", fg='bright_red')
+ line = p.fmt_subst(line, "state", "not cached", fg="bright_red")
- report += line + '\n'
+ report += line + "\n"
return report
@@ -890,15 +876,27 @@ class LogLine(Widget):
# Support files up to 99G, meaning maximum characters is 11
max_v_len = 11
if entry["type"] == _FileType.DIRECTORY:
- return "drwxr-xr-x dir {}".format(entry["size"]) +\
- "{} ".format(' ' * (max_v_len - len(size))) + "{}".format(entry["name"])
+ return (
+ "drwxr-xr-x dir {}".format(entry["size"])
+ + "{} ".format(" " * (max_v_len - len(size)))
+ + "{}".format(entry["name"])
+ )
elif entry["type"] == _FileType.SYMLINK:
- return "lrwxrwxrwx link {}".format(entry["size"]) +\
- "{} ".format(' ' * (max_v_len - len(size))) + "{} -> {}".format(entry["name"], entry["target"])
+ return (
+ "lrwxrwxrwx link {}".format(entry["size"])
+ + "{} ".format(" " * (max_v_len - len(size)))
+ + "{} -> {}".format(entry["name"], entry["target"])
+ )
elif entry["executable"]:
- return "-rwxr-xr-x exe {}".format(entry["size"]) +\
- "{} ".format(' ' * (max_v_len - len(size))) + "{}".format(entry["name"])
+ return (
+ "-rwxr-xr-x exe {}".format(entry["size"])
+ + "{} ".format(" " * (max_v_len - len(size)))
+ + "{}".format(entry["name"])
+ )
else:
- return "-rw-r--r-- reg {}".format(entry["size"]) +\
- "{} ".format(' ' * (max_v_len - len(size))) + "{}".format(entry["name"])
+ return (
+ "-rw-r--r-- reg {}".format(entry["size"])
+ + "{} ".format(" " * (max_v_len - len(size)))
+ + "{}".format(entry["name"])
+ )
return entry["name"]
diff --git a/src/buildstream/_gitsourcebase.py b/src/buildstream/_gitsourcebase.py
index 120d8c72a..4e9e59161 100644
--- a/src/buildstream/_gitsourcebase.py
+++ b/src/buildstream/_gitsourcebase.py
@@ -35,7 +35,7 @@ from . import utils
from .types import FastEnum
from .utils import move_atomic, DirectoryExistsError
-GIT_MODULES = '.gitmodules'
+GIT_MODULES = ".gitmodules"
# Warnings
WARN_INCONSISTENT_SUBMODULE = "inconsistent-submodule"
@@ -53,7 +53,6 @@ class _RefFormat(FastEnum):
# might have at a given time
#
class _GitMirror(SourceFetcher):
-
def __init__(self, source, path, url, ref, *, primary=False, tags=[]):
super().__init__()
@@ -80,59 +79,64 @@ class _GitMirror(SourceFetcher):
# system configured tmpdir is not on the same partition.
#
with self.source.tempdir() as tmpdir:
- url = self.source.translate_url(self.url, alias_override=alias_override,
- primary=self.primary)
- self.source.call([self.source.host_git, 'clone', '--mirror', '-n', url, tmpdir],
- fail="Failed to clone git repository {}".format(url),
- fail_temporarily=True)
+ url = self.source.translate_url(self.url, alias_override=alias_override, primary=self.primary)
+ self.source.call(
+ [self.source.host_git, "clone", "--mirror", "-n", url, tmpdir],
+ fail="Failed to clone git repository {}".format(url),
+ fail_temporarily=True,
+ )
try:
move_atomic(tmpdir, self.mirror)
except DirectoryExistsError:
# Another process was quicker to download this repository.
# Let's discard our own
- self.source.status("{}: Discarding duplicate clone of {}"
- .format(self.source, url))
+ self.source.status("{}: Discarding duplicate clone of {}".format(self.source, url))
except OSError as e:
- raise SourceError("{}: Failed to move cloned git repository {} from '{}' to '{}': {}"
- .format(self.source, url, tmpdir, self.mirror, e)) from e
+ raise SourceError(
+ "{}: Failed to move cloned git repository {} from '{}' to '{}': {}".format(
+ self.source, url, tmpdir, self.mirror, e
+ )
+ ) from e
def _fetch(self, alias_override=None):
- url = self.source.translate_url(self.url,
- alias_override=alias_override,
- primary=self.primary)
+ url = self.source.translate_url(self.url, alias_override=alias_override, primary=self.primary)
if alias_override:
remote_name = utils.url_directory_name(alias_override)
_, remotes = self.source.check_output(
- [self.source.host_git, 'remote'],
+ [self.source.host_git, "remote"],
fail="Failed to retrieve list of remotes in {}".format(self.mirror),
- cwd=self.mirror
+ cwd=self.mirror,
)
if remote_name not in remotes:
self.source.call(
- [self.source.host_git, 'remote', 'add', remote_name, url],
+ [self.source.host_git, "remote", "add", remote_name, url],
fail="Failed to add remote {} with url {}".format(remote_name, url),
- cwd=self.mirror
+ cwd=self.mirror,
)
else:
remote_name = "origin"
- self.source.call([self.source.host_git, 'fetch', remote_name, '--prune',
- '+refs/heads/*:refs/heads/*', '+refs/tags/*:refs/tags/*'],
- fail="Failed to fetch from remote git repository: {}".format(url),
- fail_temporarily=True,
- cwd=self.mirror)
+ self.source.call(
+ [
+ self.source.host_git,
+ "fetch",
+ remote_name,
+ "--prune",
+ "+refs/heads/*:refs/heads/*",
+ "+refs/tags/*:refs/tags/*",
+ ],
+ fail="Failed to fetch from remote git repository: {}".format(url),
+ fail_temporarily=True,
+ cwd=self.mirror,
+ )
def fetch(self, alias_override=None): # pylint: disable=arguments-differ
# Resolve the URL for the message
- resolved_url = self.source.translate_url(self.url,
- alias_override=alias_override,
- primary=self.primary)
+ resolved_url = self.source.translate_url(self.url, alias_override=alias_override, primary=self.primary)
- with self.source.timed_activity("Fetching from {}"
- .format(resolved_url),
- silent_nested=True):
+ with self.source.timed_activity("Fetching from {}".format(resolved_url), silent_nested=True):
self.ensure(alias_override)
if not self.has_ref():
self._fetch(alias_override)
@@ -147,48 +151,49 @@ class _GitMirror(SourceFetcher):
return False
# Check if the ref is really there
- rc = self.source.call([self.source.host_git, 'cat-file', '-t', self.ref], cwd=self.mirror)
+ rc = self.source.call([self.source.host_git, "cat-file", "-t", self.ref], cwd=self.mirror)
return rc == 0
def assert_ref(self):
if not self.has_ref():
- raise SourceError("{}: expected ref '{}' was not found in git repository: '{}'"
- .format(self.source, self.ref, self.url))
+ raise SourceError(
+ "{}: expected ref '{}' was not found in git repository: '{}'".format(self.source, self.ref, self.url)
+ )
def latest_commit_with_tags(self, tracking, track_tags=False):
_, output = self.source.check_output(
- [self.source.host_git, 'rev-parse', tracking],
+ [self.source.host_git, "rev-parse", tracking],
fail="Unable to find commit for specified branch name '{}'".format(tracking),
- cwd=self.mirror)
- ref = output.rstrip('\n')
+ cwd=self.mirror,
+ )
+ ref = output.rstrip("\n")
if self.source.ref_format == _RefFormat.GIT_DESCRIBE:
# Prefix the ref with the closest tag, if available,
# to make the ref human readable
exit_code, output = self.source.check_output(
- [self.source.host_git, 'describe', '--tags', '--abbrev=40', '--long', ref],
- cwd=self.mirror)
+ [self.source.host_git, "describe", "--tags", "--abbrev=40", "--long", ref], cwd=self.mirror
+ )
if exit_code == 0:
- ref = output.rstrip('\n')
+ ref = output.rstrip("\n")
if not track_tags:
return ref, []
tags = set()
- for options in [[], ['--first-parent'], ['--tags'], ['--tags', '--first-parent']]:
+ for options in [[], ["--first-parent"], ["--tags"], ["--tags", "--first-parent"]]:
exit_code, output = self.source.check_output(
- [self.source.host_git, 'describe', '--abbrev=0', ref, *options],
- cwd=self.mirror)
+ [self.source.host_git, "describe", "--abbrev=0", ref, *options], cwd=self.mirror
+ )
if exit_code == 0:
tag = output.strip()
_, commit_ref = self.source.check_output(
- [self.source.host_git, 'rev-parse', tag + '^{commit}'],
+ [self.source.host_git, "rev-parse", tag + "^{commit}"],
fail="Unable to resolve tag '{}'".format(tag),
- cwd=self.mirror)
- exit_code = self.source.call(
- [self.source.host_git, 'cat-file', 'tag', tag],
- cwd=self.mirror)
- annotated = (exit_code == 0)
+ cwd=self.mirror,
+ )
+ exit_code = self.source.call([self.source.host_git, "cat-file", "tag", tag], cwd=self.mirror)
+ annotated = exit_code == 0
tags.add((tag, commit_ref.strip(), annotated))
@@ -200,13 +205,17 @@ class _GitMirror(SourceFetcher):
# Using --shared here avoids copying the objects into the checkout, in any
# case we're just checking out a specific commit and then removing the .git/
# directory.
- self.source.call([self.source.host_git, 'clone', '--no-checkout', '--shared', self.mirror, fullpath],
- fail="Failed to create git mirror {} in directory: {}".format(self.mirror, fullpath),
- fail_temporarily=True)
-
- self.source.call([self.source.host_git, 'checkout', '--force', self.ref],
- fail="Failed to checkout git ref {}".format(self.ref),
- cwd=fullpath)
+ self.source.call(
+ [self.source.host_git, "clone", "--no-checkout", "--shared", self.mirror, fullpath],
+ fail="Failed to create git mirror {} in directory: {}".format(self.mirror, fullpath),
+ fail_temporarily=True,
+ )
+
+ self.source.call(
+ [self.source.host_git, "checkout", "--force", self.ref],
+ fail="Failed to checkout git ref {}".format(self.ref),
+ cwd=fullpath,
+ )
# Remove .git dir
shutil.rmtree(os.path.join(fullpath, ".git"))
@@ -217,34 +226,37 @@ class _GitMirror(SourceFetcher):
fullpath = os.path.join(directory, self.path)
url = self.source.translate_url(self.url)
- self.source.call([self.source.host_git, 'clone', '--no-checkout', self.mirror, fullpath],
- fail="Failed to clone git mirror {} in directory: {}".format(self.mirror, fullpath),
- fail_temporarily=True)
+ self.source.call(
+ [self.source.host_git, "clone", "--no-checkout", self.mirror, fullpath],
+ fail="Failed to clone git mirror {} in directory: {}".format(self.mirror, fullpath),
+ fail_temporarily=True,
+ )
- self.source.call([self.source.host_git, 'remote', 'set-url', 'origin', url],
- fail='Failed to add remote origin "{}"'.format(url),
- cwd=fullpath)
+ self.source.call(
+ [self.source.host_git, "remote", "set-url", "origin", url],
+ fail='Failed to add remote origin "{}"'.format(url),
+ cwd=fullpath,
+ )
- self.source.call([self.source.host_git, 'checkout', '--force', self.ref],
- fail="Failed to checkout git ref {}".format(self.ref),
- cwd=fullpath)
+ self.source.call(
+ [self.source.host_git, "checkout", "--force", self.ref],
+ fail="Failed to checkout git ref {}".format(self.ref),
+ cwd=fullpath,
+ )
# List the submodules (path/url tuples) present at the given ref of this repo
def submodule_list(self):
modules = "{}:{}".format(self.ref, GIT_MODULES)
- exit_code, output = self.source.check_output(
- [self.source.host_git, 'show', modules], cwd=self.mirror)
+ exit_code, output = self.source.check_output([self.source.host_git, "show", modules], cwd=self.mirror)
# If git show reports error code 128 here, we take it to mean there is
# no .gitmodules file to display for the given revision.
if exit_code == 128:
return
elif exit_code != 0:
- raise SourceError(
- "{plugin}: Failed to show gitmodules at ref {ref}".format(
- plugin=self, ref=self.ref))
+ raise SourceError("{plugin}: Failed to show gitmodules at ref {ref}".format(plugin=self, ref=self.ref))
- content = '\n'.join([l.strip() for l in output.splitlines()])
+ content = "\n".join([l.strip() for l in output.splitlines()])
io = StringIO(content)
parser = RawConfigParser()
@@ -253,8 +265,8 @@ class _GitMirror(SourceFetcher):
for section in parser.sections():
# validate section name against the 'submodule "foo"' pattern
if re.match(r'submodule "(.*)"', section):
- path = parser.get(section, 'path')
- url = parser.get(section, 'url')
+ path = parser.get(section, "path")
+ url = parser.get(section, "url")
yield (path, url)
@@ -266,31 +278,37 @@ class _GitMirror(SourceFetcher):
# list objects in the parent repo tree to find the commit
# object that corresponds to the submodule
- _, output = self.source.check_output([self.source.host_git, 'ls-tree', ref, submodule],
- fail="ls-tree failed for commit {} and submodule: {}".format(
- ref, submodule),
- cwd=self.mirror)
+ _, output = self.source.check_output(
+ [self.source.host_git, "ls-tree", ref, submodule],
+ fail="ls-tree failed for commit {} and submodule: {}".format(ref, submodule),
+ cwd=self.mirror,
+ )
# read the commit hash from the output
fields = output.split()
- if len(fields) >= 2 and fields[1] == 'commit':
+ if len(fields) >= 2 and fields[1] == "commit":
submodule_commit = output.split()[2]
# fail if the commit hash is invalid
if len(submodule_commit) != 40:
- raise SourceError("{}: Error reading commit information for submodule '{}'"
- .format(self.source, submodule))
+ raise SourceError(
+ "{}: Error reading commit information for submodule '{}'".format(self.source, submodule)
+ )
return submodule_commit
else:
- detail = "The submodule '{}' is defined either in the BuildStream source\n".format(submodule) + \
- "definition, or in a .gitmodules file. But the submodule was never added to the\n" + \
- "underlying git repository with `git submodule add`."
+ detail = (
+ "The submodule '{}' is defined either in the BuildStream source\n".format(submodule)
+ + "definition, or in a .gitmodules file. But the submodule was never added to the\n"
+ + "underlying git repository with `git submodule add`."
+ )
- self.source.warn("{}: Ignoring inconsistent submodule '{}'"
- .format(self.source, submodule), detail=detail,
- warning_token=WARN_INCONSISTENT_SUBMODULE)
+ self.source.warn(
+ "{}: Ignoring inconsistent submodule '{}'".format(self.source, submodule),
+ detail=detail,
+ warning_token=WARN_INCONSISTENT_SUBMODULE,
+ )
return None
@@ -307,17 +325,24 @@ class _GitMirror(SourceFetcher):
# rev-list does not work in case of same rev
shallow.add(self.ref)
else:
- _, out = self.source.check_output([self.source.host_git, 'rev-list',
- '--ancestry-path', '--boundary',
- '{}..{}'.format(commit_ref, self.ref)],
- fail="Failed to get git history {}..{} in directory: {}"
- .format(commit_ref, self.ref, fullpath),
- fail_temporarily=True,
- cwd=self.mirror)
+ _, out = self.source.check_output(
+ [
+ self.source.host_git,
+ "rev-list",
+ "--ancestry-path",
+ "--boundary",
+ "{}..{}".format(commit_ref, self.ref),
+ ],
+ fail="Failed to get git history {}..{} in directory: {}".format(
+ commit_ref, self.ref, fullpath
+ ),
+ fail_temporarily=True,
+ cwd=self.mirror,
+ )
self.source.warn("refs {}..{}: {}".format(commit_ref, self.ref, out.splitlines()))
for line in out.splitlines():
- rev = line.lstrip('-')
- if line[0] == '-':
+ rev = line.lstrip("-")
+ if line[0] == "-":
shallow.add(rev)
else:
included.add(rev)
@@ -325,52 +350,64 @@ class _GitMirror(SourceFetcher):
shallow -= included
included |= shallow
- self.source.call([self.source.host_git, 'init'],
- fail="Cannot initialize git repository: {}".format(fullpath),
- cwd=fullpath)
+ self.source.call(
+ [self.source.host_git, "init"],
+ fail="Cannot initialize git repository: {}".format(fullpath),
+ cwd=fullpath,
+ )
for rev in included:
with TemporaryFile(dir=tmpdir) as commit_file:
- self.source.call([self.source.host_git, 'cat-file', 'commit', rev],
- stdout=commit_file,
- fail="Failed to get commit {}".format(rev),
- cwd=self.mirror)
+ self.source.call(
+ [self.source.host_git, "cat-file", "commit", rev],
+ stdout=commit_file,
+ fail="Failed to get commit {}".format(rev),
+ cwd=self.mirror,
+ )
commit_file.seek(0, 0)
- self.source.call([self.source.host_git, 'hash-object', '-w', '-t', 'commit', '--stdin'],
- stdin=commit_file,
- fail="Failed to add commit object {}".format(rev),
- cwd=fullpath)
-
- with open(os.path.join(fullpath, '.git', 'shallow'), 'w') as shallow_file:
+ self.source.call(
+ [self.source.host_git, "hash-object", "-w", "-t", "commit", "--stdin"],
+ stdin=commit_file,
+ fail="Failed to add commit object {}".format(rev),
+ cwd=fullpath,
+ )
+
+ with open(os.path.join(fullpath, ".git", "shallow"), "w") as shallow_file:
for rev in shallow:
- shallow_file.write('{}\n'.format(rev))
+ shallow_file.write("{}\n".format(rev))
for tag, commit_ref, annotated in self.tags:
if annotated:
with TemporaryFile(dir=tmpdir) as tag_file:
- tag_data = 'object {}\ntype commit\ntag {}\n'.format(commit_ref, tag)
- tag_file.write(tag_data.encode('ascii'))
+ tag_data = "object {}\ntype commit\ntag {}\n".format(commit_ref, tag)
+ tag_file.write(tag_data.encode("ascii"))
tag_file.seek(0, 0)
_, tag_ref = self.source.check_output(
- [self.source.host_git, 'hash-object', '-w', '-t',
- 'tag', '--stdin'],
+ [self.source.host_git, "hash-object", "-w", "-t", "tag", "--stdin"],
stdin=tag_file,
fail="Failed to add tag object {}".format(tag),
- cwd=fullpath)
-
- self.source.call([self.source.host_git, 'tag', tag, tag_ref.strip()],
- fail="Failed to tag: {}".format(tag),
- cwd=fullpath)
+ cwd=fullpath,
+ )
+
+ self.source.call(
+ [self.source.host_git, "tag", tag, tag_ref.strip()],
+ fail="Failed to tag: {}".format(tag),
+ cwd=fullpath,
+ )
else:
- self.source.call([self.source.host_git, 'tag', tag, commit_ref],
- fail="Failed to tag: {}".format(tag),
- cwd=fullpath)
+ self.source.call(
+ [self.source.host_git, "tag", tag, commit_ref],
+ fail="Failed to tag: {}".format(tag),
+ cwd=fullpath,
+ )
- with open(os.path.join(fullpath, '.git', 'HEAD'), 'w') as head:
- self.source.call([self.source.host_git, 'rev-parse', self.ref],
- stdout=head,
- fail="Failed to parse commit {}".format(self.ref),
- cwd=self.mirror)
+ with open(os.path.join(fullpath, ".git", "HEAD"), "w") as head:
+ self.source.call(
+ [self.source.host_git, "rev-parse", self.ref],
+ stdout=head,
+ fail="Failed to parse commit {}".format(self.ref),
+ cwd=self.mirror,
+ )
class _GitSourceBase(Source):
@@ -382,58 +419,57 @@ class _GitSourceBase(Source):
BST_MIRROR_CLASS = _GitMirror
def configure(self, node):
- ref = node.get_str('ref', None)
+ ref = node.get_str("ref", None)
- config_keys = ['url', 'track', 'ref', 'submodules',
- 'checkout-submodules', 'ref-format',
- 'track-tags', 'tags']
+ config_keys = ["url", "track", "ref", "submodules", "checkout-submodules", "ref-format", "track-tags", "tags"]
node.validate_keys(config_keys + Source.COMMON_CONFIG_KEYS)
- tags_node = node.get_sequence('tags', [])
+ tags_node = node.get_sequence("tags", [])
for tag_node in tags_node:
- tag_node.validate_keys(['tag', 'commit', 'annotated'])
+ tag_node.validate_keys(["tag", "commit", "annotated"])
tags = self._load_tags(node)
- self.track_tags = node.get_bool('track-tags', default=False)
+ self.track_tags = node.get_bool("track-tags", default=False)
- self.original_url = node.get_str('url')
- self.mirror = self.BST_MIRROR_CLASS(self, '', self.original_url, ref, tags=tags, primary=True)
- self.tracking = node.get_str('track', None)
+ self.original_url = node.get_str("url")
+ self.mirror = self.BST_MIRROR_CLASS(self, "", self.original_url, ref, tags=tags, primary=True)
+ self.tracking = node.get_str("track", None)
- self.ref_format = node.get_enum('ref-format', _RefFormat, _RefFormat.SHA1)
+ self.ref_format = node.get_enum("ref-format", _RefFormat, _RefFormat.SHA1)
# At this point we now know if the source has a ref and/or a track.
# If it is missing both then we will be unable to track or build.
if self.mirror.ref is None and self.tracking is None:
- raise SourceError("{}: Git sources require a ref and/or track".format(self),
- reason="missing-track-and-ref")
+ raise SourceError(
+ "{}: Git sources require a ref and/or track".format(self), reason="missing-track-and-ref"
+ )
- self.checkout_submodules = node.get_bool('checkout-submodules', default=True)
+ self.checkout_submodules = node.get_bool("checkout-submodules", default=True)
self.submodules = []
# Parse a dict of submodule overrides, stored in the submodule_overrides
# and submodule_checkout_overrides dictionaries.
self.submodule_overrides = {}
self.submodule_checkout_overrides = {}
- modules = node.get_mapping('submodules', {})
+ modules = node.get_mapping("submodules", {})
for path in modules.keys():
submodule = modules.get_mapping(path)
- url = submodule.get_str('url', None)
+ url = submodule.get_str("url", None)
# Make sure to mark all URLs that are specified in the configuration
if url:
self.mark_download_url(url, primary=False)
self.submodule_overrides[path] = url
- if 'checkout' in submodule:
- checkout = submodule.get_bool('checkout')
+ if "checkout" in submodule:
+ checkout = submodule.get_bool("checkout")
self.submodule_checkout_overrides[path] = checkout
self.mark_download_url(self.original_url)
def preflight(self):
# Check if git is installed, get the binary at the same time
- self.host_git = utils.get_host_tool('git')
+ self.host_git = utils.get_host_tool("git")
def get_unique_key(self):
# Here we want to encode the local name of the repository and
@@ -442,7 +478,7 @@ class _GitSourceBase(Source):
key = [self.original_url, self.mirror.ref]
if self.mirror.tags:
tags = {tag: (commit, annotated) for tag, commit, annotated in self.mirror.tags}
- key.append({'tags': tags})
+ key.append({"tags": tags})
# Only modify the cache key with checkout_submodules if it's something
# other than the default behaviour.
@@ -467,7 +503,7 @@ class _GitSourceBase(Source):
return Consistency.INCONSISTENT
def load_ref(self, node):
- self.mirror.ref = node.get_str('ref', None)
+ self.mirror.ref = node.get_str("ref", None)
self.mirror.tags = self._load_tags(node)
def get_ref(self):
@@ -478,25 +514,23 @@ class _GitSourceBase(Source):
def set_ref(self, ref, node):
if not ref:
self.mirror.ref = None
- if 'ref' in node:
- del node['ref']
+ if "ref" in node:
+ del node["ref"]
self.mirror.tags = []
- if 'tags' in node:
- del node['tags']
+ if "tags" in node:
+ del node["tags"]
else:
actual_ref, tags = ref
- node['ref'] = self.mirror.ref = actual_ref
+ node["ref"] = self.mirror.ref = actual_ref
self.mirror.tags = tags
if tags:
- node['tags'] = []
+ node["tags"] = []
for tag, commit_ref, annotated in tags:
- data = {'tag': tag,
- 'commit': commit_ref,
- 'annotated': annotated}
- node['tags'].append(data)
+ data = {"tag": tag, "commit": commit_ref, "annotated": annotated}
+ node["tags"].append(data)
else:
- if 'tags' in node:
- del node['tags']
+ if "tags" in node:
+ del node["tags"]
def track(self): # pylint: disable=arguments-differ
@@ -504,17 +538,13 @@ class _GitSourceBase(Source):
if not self.tracking:
# Is there a better way to check if a ref is given.
if self.mirror.ref is None:
- detail = 'Without a tracking branch ref can not be updated. Please ' + \
- 'provide a ref or a track.'
- raise SourceError("{}: No track or ref".format(self),
- detail=detail, reason="track-attempt-no-track")
+ detail = "Without a tracking branch ref can not be updated. Please " + "provide a ref or a track."
+ raise SourceError("{}: No track or ref".format(self), detail=detail, reason="track-attempt-no-track")
return None
# Resolve the URL for the message
resolved_url = self.translate_url(self.mirror.url)
- with self.timed_activity("Tracking {} from {}"
- .format(self.tracking, resolved_url),
- silent_nested=True):
+ with self.timed_activity("Tracking {} from {}".format(self.tracking, resolved_url), silent_nested=True):
self.mirror.ensure()
self.mirror._fetch()
@@ -578,11 +608,12 @@ class _GitSourceBase(Source):
for path, url in invalid_submodules:
detail.append(" Submodule URL '{}' at path '{}'".format(url, path))
- self.warn("{}: Invalid submodules specified".format(self),
- warning_token=WARN_INVALID_SUBMODULE,
- detail="The following submodules are specified in the source "
- "description but do not exist according to the repository\n\n" +
- "\n".join(detail))
+ self.warn(
+ "{}: Invalid submodules specified".format(self),
+ warning_token=WARN_INVALID_SUBMODULE,
+ detail="The following submodules are specified in the source "
+ "description but do not exist according to the repository\n\n" + "\n".join(detail),
+ )
# Warn about submodules which exist but have not been explicitly configured
if unlisted_submodules:
@@ -590,37 +621,47 @@ class _GitSourceBase(Source):
for path, url in unlisted_submodules:
detail.append(" Submodule URL '{}' at path '{}'".format(url, path))
- self.warn("{}: Unlisted submodules exist".format(self),
- warning_token=WARN_UNLISTED_SUBMODULE,
- detail="The following submodules exist but are not specified " +
- "in the source description\n\n" +
- "\n".join(detail))
+ self.warn(
+ "{}: Unlisted submodules exist".format(self),
+ warning_token=WARN_UNLISTED_SUBMODULE,
+ detail="The following submodules exist but are not specified "
+ + "in the source description\n\n"
+ + "\n".join(detail),
+ )
# Assert that the ref exists in the track tag/branch, if track has been specified.
ref_in_track = False
if self.tracking:
- _, branch = self.check_output([self.host_git, 'branch', '--list', self.tracking,
- '--contains', self.mirror.ref],
- cwd=self.mirror.mirror)
+ _, branch = self.check_output(
+ [self.host_git, "branch", "--list", self.tracking, "--contains", self.mirror.ref],
+ cwd=self.mirror.mirror,
+ )
if branch:
ref_in_track = True
else:
- _, tag = self.check_output([self.host_git, 'tag', '--list', self.tracking,
- '--contains', self.mirror.ref],
- cwd=self.mirror.mirror)
+ _, tag = self.check_output(
+ [self.host_git, "tag", "--list", self.tracking, "--contains", self.mirror.ref],
+ cwd=self.mirror.mirror,
+ )
if tag:
ref_in_track = True
if not ref_in_track:
- detail = "The ref provided for the element does not exist locally " + \
- "in the provided track branch / tag '{}'.\n".format(self.tracking) + \
- "You may wish to track the element to update the ref from '{}' ".format(self.tracking) + \
- "with `bst source track`,\n" + \
- "or examine the upstream at '{}' for the specific ref.".format(self.mirror.url)
+ detail = (
+ "The ref provided for the element does not exist locally "
+ + "in the provided track branch / tag '{}'.\n".format(self.tracking)
+ + "You may wish to track the element to update the ref from '{}' ".format(self.tracking)
+ + "with `bst source track`,\n"
+ + "or examine the upstream at '{}' for the specific ref.".format(self.mirror.url)
+ )
- self.warn("{}: expected ref '{}' was not found in given track '{}' for staged repository: '{}'\n"
- .format(self, self.mirror.ref, self.tracking, self.mirror.url),
- detail=detail, warning_token=CoreWarnings.REF_NOT_IN_TRACK)
+ self.warn(
+ "{}: expected ref '{}' was not found in given track '{}' for staged repository: '{}'\n".format(
+ self, self.mirror.ref, self.tracking, self.mirror.url
+ ),
+ detail=detail,
+ warning_token=CoreWarnings.REF_NOT_IN_TRACK,
+ )
###########################################################
# Local Functions #
@@ -668,11 +709,11 @@ class _GitSourceBase(Source):
def _load_tags(self, node):
tags = []
- tags_node = node.get_sequence('tags', [])
+ tags_node = node.get_sequence("tags", [])
for tag_node in tags_node:
- tag = tag_node.get_str('tag')
- commit_ref = tag_node.get_str('commit')
- annotated = tag_node.get_bool('annotated')
+ tag = tag_node.get_str("tag")
+ commit_ref = tag_node.get_str("commit")
+ annotated = tag_node.get_bool("annotated")
tags.append((tag, commit_ref, annotated))
return tags
diff --git a/src/buildstream/_includes.py b/src/buildstream/_includes.py
index c04601b91..bc0d7718b 100644
--- a/src/buildstream/_includes.py
+++ b/src/buildstream/_includes.py
@@ -14,7 +14,6 @@ from ._exceptions import LoadError, LoadErrorReason
# provenance. Should be true if intended to be
# serialized.
class Includes:
-
def __init__(self, loader, *, copy_tree=False):
self._loader = loader
self._loaded = {}
@@ -29,14 +28,11 @@ class Includes:
# included (set): Fail for recursion if trying to load any files in this set
# current_loader (Loader): Use alternative loader (for junction files)
# only_local (bool): Whether to ignore junction files
- def process(self, node, *,
- included=set(),
- current_loader=None,
- only_local=False):
+ def process(self, node, *, included=set(), current_loader=None, only_local=False):
if current_loader is None:
current_loader = self._loader
- includes_node = node.get_node('(@)', allowed_types=[ScalarNode, SequenceNode], allow_none=True)
+ includes_node = node.get_node("(@)", allowed_types=[ScalarNode, SequenceNode], allow_none=True)
if includes_node:
if type(includes_node) is ScalarNode: # pylint: disable=unidiomatic-typecheck
@@ -44,23 +40,24 @@ class Includes:
else:
includes = includes_node.as_str_list()
- del node['(@)']
+ del node["(@)"]
for include in reversed(includes):
- if only_local and ':' in include:
+ if only_local and ":" in include:
continue
try:
- include_node, file_path, sub_loader = self._include_file(include,
- current_loader)
+ include_node, file_path, sub_loader = self._include_file(include, current_loader)
except LoadError as e:
include_provenance = includes_node.get_provenance()
if e.reason == LoadErrorReason.MISSING_FILE:
message = "{}: Include block references a file that could not be found: '{}'.".format(
- include_provenance, include)
+ include_provenance, include
+ )
raise LoadError(message, LoadErrorReason.MISSING_FILE) from e
if e.reason == LoadErrorReason.LOADING_DIRECTORY:
message = "{}: Include block references a directory instead of a file: '{}'.".format(
- include_provenance, include)
+ include_provenance, include
+ )
raise LoadError(message, LoadErrorReason.LOADING_DIRECTORY) from e
# Otherwise, we don't know the reason, so just raise
@@ -68,8 +65,10 @@ class Includes:
if file_path in included:
include_provenance = includes_node.get_provenance()
- raise LoadError("{}: trying to recursively include {}". format(include_provenance, file_path),
- LoadErrorReason.RECURSIVE_INCLUDE)
+ raise LoadError(
+ "{}: trying to recursively include {}".format(include_provenance, file_path),
+ LoadErrorReason.RECURSIVE_INCLUDE,
+ )
# Because the included node will be modified, we need
# to copy it so that we do not modify the toplevel
# node of the provenance.
@@ -77,19 +76,14 @@ class Includes:
try:
included.add(file_path)
- self.process(include_node, included=included,
- current_loader=sub_loader,
- only_local=only_local)
+ self.process(include_node, included=included, current_loader=sub_loader, only_local=only_local)
finally:
included.remove(file_path)
include_node._composite_under(node)
for value in node.values():
- self._process_value(value,
- included=included,
- current_loader=current_loader,
- only_local=only_local)
+ self._process_value(value, included=included, current_loader=current_loader, only_local=only_local)
# _include_file()
#
@@ -101,8 +95,8 @@ class Includes:
# loader (Loader): Loader for the current project.
def _include_file(self, include, loader):
shortname = include
- if ':' in include:
- junction, include = include.split(':', 1)
+ if ":" in include:
+ junction, include = include.split(":", 1)
junction_loader = loader._get_loader(junction)
current_loader = junction_loader
else:
@@ -112,10 +106,7 @@ class Includes:
file_path = os.path.join(directory, include)
key = (current_loader, file_path)
if key not in self._loaded:
- self._loaded[key] = _yaml.load(file_path,
- shortname=shortname,
- project=project,
- copy_tree=self._copy_tree)
+ self._loaded[key] = _yaml.load(file_path, shortname=shortname, project=project, copy_tree=self._copy_tree)
return self._loaded[key], file_path, current_loader
# _process_value()
@@ -127,20 +118,11 @@ class Includes:
# included (set): Fail for recursion if trying to load any files in this set
# current_loader (Loader): Use alternative loader (for junction files)
# only_local (bool): Whether to ignore junction files
- def _process_value(self, value, *,
- included=set(),
- current_loader=None,
- only_local=False):
+ def _process_value(self, value, *, included=set(), current_loader=None, only_local=False):
value_type = type(value)
if value_type is MappingNode:
- self.process(value,
- included=included,
- current_loader=current_loader,
- only_local=only_local)
+ self.process(value, included=included, current_loader=current_loader, only_local=only_local)
elif value_type is SequenceNode:
for v in value:
- self._process_value(v,
- included=included,
- current_loader=current_loader,
- only_local=only_local)
+ self._process_value(v, included=included, current_loader=current_loader, only_local=only_local)
diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py
index e5859e9e8..da0c0fb29 100644
--- a/src/buildstream/_loader/loader.py
+++ b/src/buildstream/_loader/loader.py
@@ -54,8 +54,7 @@ _NO_PROGRESS = object()
# fetch_subprojects (callable): A function to fetch subprojects
# parent (Loader): A parent Loader object, in the case this is a junctioned Loader
#
-class Loader():
-
+class Loader:
def __init__(self, context, project, *, fetch_subprojects, parent=None):
# Ensure we have an absolute path for the base directory
@@ -66,22 +65,22 @@ class Loader():
#
# Public members
#
- self.project = project # The associated Project
- self.loaded = None # The number of loaded Elements
+ self.project = project # The associated Project
+ self.loaded = None # The number of loaded Elements
#
# Private members
#
self._context = context
- self._options = project.options # Project options (OptionPool)
- self._basedir = basedir # Base project directory
+ self._options = project.options # Project options (OptionPool)
+ self._basedir = basedir # Base project directory
self._first_pass_options = project.first_pass_config.options # Project options (OptionPool)
- self._parent = parent # The parent loader
+ self._parent = parent # The parent loader
self._fetch_subprojects = fetch_subprojects
self._meta_elements = {} # Dict of resolved meta elements by name
- self._elements = {} # Dict of elements
- self._loaders = {} # Dict of junction loaders
+ self._elements = {} # Dict of elements
+ self._loaders = {} # Dict of junction loaders
self._includes = Includes(self, copy_tree=True)
@@ -105,9 +104,11 @@ class Loader():
if os.path.isabs(filename):
# XXX Should this just be an assertion ?
# Expect that the caller gives us the right thing at least ?
- raise LoadError("Target '{}' was not specified as a relative "
- "path to the base project directory: {}"
- .format(filename, self._basedir), LoadErrorReason.INVALID_DATA)
+ raise LoadError(
+ "Target '{}' was not specified as a relative "
+ "path to the base project directory: {}".format(filename, self._basedir),
+ LoadErrorReason.INVALID_DATA,
+ )
self._warn_invalid_elements(targets)
@@ -130,8 +131,7 @@ class Loader():
dummy_target = LoadElement(Node.from_dict({}), "", self)
# Pylint is not very happy with Cython and can't understand 'dependencies' is a list
dummy_target.dependencies.extend( # pylint: disable=no-member
- Dependency(element, Symbol.RUNTIME, False)
- for element in target_elements
+ Dependency(element, Symbol.RUNTIME, False) for element in target_elements
)
with PROFILER.profile(Topics.CIRCULAR_CHECK, "_".join(targets)):
@@ -180,12 +180,12 @@ class Loader():
# too late. The only time that seems just right is here, when preparing
# the child process' copy of the Loader.
#
- del state['_fetch_subprojects']
+ del state["_fetch_subprojects"]
# Also there's no gain in pickling over the caches, and they might
# contain things which are unpleasantly large or unable to pickle.
- del state['_elements']
- del state['_meta_elements']
+ del state["_elements"]
+ del state["_meta_elements"]
return state
@@ -230,14 +230,14 @@ class Loader():
# Load the data and process any conditional statements therein
fullpath = os.path.join(self._basedir, filename)
try:
- node = _yaml.load(fullpath, shortname=filename, copy_tree=rewritable,
- project=self.project)
+ node = _yaml.load(fullpath, shortname=filename, copy_tree=rewritable, project=self.project)
except LoadError as e:
if e.reason == LoadErrorReason.MISSING_FILE:
if self.project.junction:
- message = "Could not find element '{}' in project referred to by junction element '{}'" \
- .format(filename, self.project.junction.name)
+ message = "Could not find element '{}' in project referred to by junction element '{}'".format(
+ filename, self.project.junction.name
+ )
else:
message = "Could not find element '{}' in elements directory '{}'".format(filename, self._basedir)
@@ -262,8 +262,8 @@ class Loader():
if provenance:
message = "{}: {}".format(provenance, message)
detail = None
- if os.path.exists(os.path.join(self._basedir, filename + '.bst')):
- element_name = filename + '.bst'
+ if os.path.exists(os.path.join(self._basedir, filename + ".bst")):
+ element_name = filename + ".bst"
detail = "Did you mean '{}'?\n".format(element_name)
raise LoadError(message, LoadErrorReason.LOADING_DIRECTORY, detail=detail) from e
@@ -333,10 +333,9 @@ class Loader():
if dep.junction:
self._load_file(dep.junction, rewritable, ticker, dep.provenance)
- loader = self._get_loader(dep.junction,
- rewritable=rewritable,
- ticker=ticker,
- provenance=dep.provenance)
+ loader = self._get_loader(
+ dep.junction, rewritable=rewritable, ticker=ticker, provenance=dep.provenance
+ )
dep_element = loader._load_file(dep.name, rewritable, ticker, dep.provenance)
else:
dep_element = self._elements.get(dep.name)
@@ -350,14 +349,16 @@ class Loader():
loader_queue.append((dep_element, list(reversed(dep_deps)), []))
# Pylint is not very happy about Cython and can't understand 'node' is a 'MappingNode'
- if dep_element.node.get_str(Symbol.KIND) == 'junction': # pylint: disable=no-member
- raise LoadError("{}: Cannot depend on junction" .format(dep.provenance),
- LoadErrorReason.INVALID_DATA)
+ if dep_element.node.get_str(Symbol.KIND) == "junction": # pylint: disable=no-member
+ raise LoadError(
+ "{}: Cannot depend on junction".format(dep.provenance), LoadErrorReason.INVALID_DATA
+ )
# All is well, push the dependency onto the LoadElement
# Pylint is not very happy with Cython and can't understand 'dependencies' is a list
current_element[0].dependencies.append( # pylint: disable=no-member
- Dependency(dep_element, dep.dep_type, dep.strict))
+ Dependency(dep_element, dep.dep_type, dep.strict)
+ )
else:
# We do not have any more dependencies to load for this
# element on the queue, report any invalid dep names
@@ -397,12 +398,14 @@ class Loader():
# Create `chain`, the loop of element dependencies from this
# element back to itself, by trimming everything before this
# element from the sequence under consideration.
- chain = [element.full_name for element in sequence[sequence.index(element):]]
+ chain = [element.full_name for element in sequence[sequence.index(element) :]]
chain.append(element.full_name)
- raise LoadError(("Circular dependency detected at element: {}\n" +
- "Dependency chain: {}")
- .format(element.full_name, " -> ".join(chain)),
- LoadErrorReason.CIRCULAR_DEPENDENCY)
+ raise LoadError(
+ ("Circular dependency detected at element: {}\n" + "Dependency chain: {}").format(
+ element.full_name, " -> ".join(chain)
+ ),
+ LoadErrorReason.CIRCULAR_DEPENDENCY,
+ )
if element not in validated:
# We've not already validated this element, so let's
# descend into it to check it out
@@ -447,9 +450,9 @@ class Loader():
workspace = self._context.get_workspaces().get_workspace(element.name)
skip_workspace = True
if workspace:
- workspace_node = {'kind': 'workspace'}
- workspace_node['path'] = workspace.get_absolute_path()
- workspace_node['ref'] = str(workspace.to_dict().get('last_successful', 'ignored'))
+ workspace_node = {"kind": "workspace"}
+ workspace_node["path"] = workspace.get_absolute_path()
+ workspace_node["ref"] = str(workspace.to_dict().get("last_successful", "ignored"))
node[Symbol.SOURCES] = [workspace_node]
skip_workspace = False
@@ -457,7 +460,7 @@ class Loader():
for index, source in enumerate(sources):
kind = source.get_str(Symbol.KIND)
# the workspace source plugin cannot be used unless the element is workspaced
- if kind == 'workspace' and skip_workspace:
+ if kind == "workspace" and skip_workspace:
continue
del source[Symbol.KIND]
@@ -469,15 +472,20 @@ class Loader():
meta_source = MetaSource(element.name, index, element_kind, kind, source, directory)
meta_sources.append(meta_source)
- meta_element = MetaElement(self.project, element.name, element_kind,
- elt_provenance, meta_sources,
- node.get_mapping(Symbol.CONFIG, default={}),
- node.get_mapping(Symbol.VARIABLES, default={}),
- node.get_mapping(Symbol.ENVIRONMENT, default={}),
- node.get_str_list(Symbol.ENV_NOCACHE, default=[]),
- node.get_mapping(Symbol.PUBLIC, default={}),
- node.get_mapping(Symbol.SANDBOX, default={}),
- element_kind == 'junction')
+ meta_element = MetaElement(
+ self.project,
+ element.name,
+ element_kind,
+ elt_provenance,
+ meta_sources,
+ node.get_mapping(Symbol.CONFIG, default={}),
+ node.get_mapping(Symbol.VARIABLES, default={}),
+ node.get_mapping(Symbol.ENVIRONMENT, default={}),
+ node.get_str_list(Symbol.ENV_NOCACHE, default=[]),
+ node.get_mapping(Symbol.PUBLIC, default={}),
+ node.get_mapping(Symbol.SANDBOX, default={}),
+ element_kind == "junction",
+ )
# Cache it now, make sure it's already there before recursing
self._meta_elements[element.name] = meta_element
@@ -522,9 +530,9 @@ class Loader():
else:
meta_dep = loader._meta_elements[name]
- if dep.dep_type != 'runtime':
+ if dep.dep_type != "runtime":
meta_element.build_dependencies.append(meta_dep)
- if dep.dep_type != 'build':
+ if dep.dep_type != "build":
meta_element.dependencies.append(meta_dep)
if dep.strict:
meta_element.strict_dependencies.append(meta_dep)
@@ -543,8 +551,7 @@ class Loader():
# Raises: LoadError
#
# Returns: A Loader or None if specified junction does not exist
- def _get_loader(self, filename, *, rewritable=False, ticker=None, level=0,
- provenance=None):
+ def _get_loader(self, filename, *, rewritable=False, ticker=None, level=0, provenance=None):
provenance_str = ""
if provenance is not None:
@@ -557,17 +564,21 @@ class Loader():
if loader is None:
# do not allow junctions with the same name in different
# subprojects
- raise LoadError("{}Conflicting junction {} in subprojects, define junction in {}"
- .format(provenance_str, filename, self.project.name),
- LoadErrorReason.CONFLICTING_JUNCTION)
+ raise LoadError(
+ "{}Conflicting junction {} in subprojects, define junction in {}".format(
+ provenance_str, filename, self.project.name
+ ),
+ LoadErrorReason.CONFLICTING_JUNCTION,
+ )
return loader
if self._parent:
# junctions in the parent take precedence over junctions defined
# in subprojects
- loader = self._parent._get_loader(filename, rewritable=rewritable, ticker=ticker,
- level=level + 1, provenance=provenance)
+ loader = self._parent._get_loader(
+ filename, rewritable=rewritable, ticker=ticker, level=level + 1, provenance=provenance
+ )
if loader:
self._loaders[filename] = loader
return loader
@@ -599,10 +610,11 @@ class Loader():
# Any task counting *inside* the junction will be handled by
# its loader.
meta_element = self._collect_element_no_deps(self._elements[filename], _NO_PROGRESS)
- if meta_element.kind != 'junction':
- raise LoadError("{}{}: Expected junction but element kind is {}"
- .format(provenance_str, filename, meta_element.kind),
- LoadErrorReason.INVALID_DATA)
+ if meta_element.kind != "junction":
+ raise LoadError(
+ "{}{}: Expected junction but element kind is {}".format(provenance_str, filename, meta_element.kind),
+ LoadErrorReason.INVALID_DATA,
+ )
# We check that junctions have no dependencies a little
# early. This is cheating, since we don't technically know
@@ -618,9 +630,7 @@ class Loader():
# would be nice if this could be done for *all* element types,
# but since we haven't loaded those yet that's impossible.
if self._elements[filename].dependencies:
- raise LoadError(
- "Dependencies are forbidden for 'junction' elements",
- LoadErrorReason.INVALID_JUNCTION)
+ raise LoadError("Dependencies are forbidden for 'junction' elements", LoadErrorReason.INVALID_JUNCTION)
element = Element._new_from_meta(meta_element)
element._update_state()
@@ -628,10 +638,12 @@ class Loader():
# If this junction element points to a sub-sub-project, we need to
# find loader for that project.
if element.target:
- subproject_loader = self._get_loader(element.target_junction, rewritable=rewritable, ticker=ticker,
- level=level, provenance=provenance)
- loader = subproject_loader._get_loader(element.target_element, rewritable=rewritable, ticker=ticker,
- level=level, provenance=provenance)
+ subproject_loader = self._get_loader(
+ element.target_junction, rewritable=rewritable, ticker=ticker, level=level, provenance=provenance
+ )
+ loader = subproject_loader._get_loader(
+ element.target_element, rewritable=rewritable, ticker=ticker, level=level, provenance=provenance
+ )
self._loaders[filename] = loader
return loader
@@ -639,15 +651,18 @@ class Loader():
#
if element._get_consistency() >= Consistency.RESOLVED and not element._source_cached():
if ticker:
- ticker(filename, 'Fetching subproject')
+ ticker(filename, "Fetching subproject")
self._fetch_subprojects([element])
# Handle the case where a subproject has no ref
#
elif element._get_consistency() == Consistency.INCONSISTENT:
detail = "Try tracking the junction element with `bst source track {}`".format(filename)
- raise LoadError("{}Subproject has no ref for junction: {}".format(provenance_str, filename),
- LoadErrorReason.SUBPROJECT_INCONSISTENT, detail=detail)
+ raise LoadError(
+ "{}Subproject has no ref for junction: {}".format(provenance_str, filename),
+ LoadErrorReason.SUBPROJECT_INCONSISTENT,
+ detail=detail,
+ )
sources = list(element.sources())
if len(sources) == 1 and sources[0]._get_local_path():
@@ -656,8 +671,9 @@ class Loader():
else:
# Stage sources
element._set_required()
- basedir = os.path.join(self.project.directory, ".bst", "staged-junctions",
- filename, element._get_cache_key())
+ basedir = os.path.join(
+ self.project.directory, ".bst", "staged-junctions", filename, element._get_cache_key()
+ )
if not os.path.exists(basedir):
os.makedirs(basedir, exist_ok=True)
element._stage_sources_at(basedir)
@@ -666,9 +682,15 @@ class Loader():
project_dir = os.path.join(basedir, element.path)
try:
from .._project import Project # pylint: disable=cyclic-import
- project = Project(project_dir, self._context, junction=element,
- parent_loader=self, search_for_project=False,
- fetch_subprojects=self._fetch_subprojects)
+
+ project = Project(
+ project_dir,
+ self._context,
+ junction=element,
+ parent_loader=self,
+ search_for_project=False,
+ fetch_subprojects=self._fetch_subprojects,
+ )
except LoadError as e:
if e.reason == LoadErrorReason.MISSING_PROJECT_CONF:
message = (
@@ -706,7 +728,7 @@ class Loader():
# We allow to split only once since deep junctions names are forbidden.
# Users who want to refer to elements in sub-sub-projects are required
# to create junctions on the top level project.
- junction_path = name.rsplit(':', 1)
+ junction_path = name.rsplit(":", 1)
if len(junction_path) == 1:
return None, junction_path[-1], self
else:
@@ -760,11 +782,17 @@ class Loader():
invalid_elements[CoreWarnings.BAD_CHARACTERS_IN_NAME].append(filename)
if invalid_elements[CoreWarnings.BAD_ELEMENT_SUFFIX]:
- self._warn("Target elements '{}' do not have expected file extension `.bst` "
- "Improperly named elements will not be discoverable by commands"
- .format(invalid_elements[CoreWarnings.BAD_ELEMENT_SUFFIX]),
- warning_token=CoreWarnings.BAD_ELEMENT_SUFFIX)
+ self._warn(
+ "Target elements '{}' do not have expected file extension `.bst` "
+ "Improperly named elements will not be discoverable by commands".format(
+ invalid_elements[CoreWarnings.BAD_ELEMENT_SUFFIX]
+ ),
+ warning_token=CoreWarnings.BAD_ELEMENT_SUFFIX,
+ )
if invalid_elements[CoreWarnings.BAD_CHARACTERS_IN_NAME]:
- self._warn("Target elements '{}' have invalid characerts in their name."
- .format(invalid_elements[CoreWarnings.BAD_CHARACTERS_IN_NAME]),
- warning_token=CoreWarnings.BAD_CHARACTERS_IN_NAME)
+ self._warn(
+ "Target elements '{}' have invalid characerts in their name.".format(
+ invalid_elements[CoreWarnings.BAD_CHARACTERS_IN_NAME]
+ ),
+ warning_token=CoreWarnings.BAD_CHARACTERS_IN_NAME,
+ )
diff --git a/src/buildstream/_loader/metaelement.py b/src/buildstream/_loader/metaelement.py
index 00d8560f8..97b0de242 100644
--- a/src/buildstream/_loader/metaelement.py
+++ b/src/buildstream/_loader/metaelement.py
@@ -20,7 +20,7 @@
from ..node import Node
-class MetaElement():
+class MetaElement:
# MetaElement()
#
@@ -40,9 +40,21 @@ class MetaElement():
# sandbox: Configuration specific to the sandbox environment
# first_pass: The element is to be loaded with first pass configuration (junction)
#
- def __init__(self, project, name, kind=None, provenance=None, sources=None, config=None,
- variables=None, environment=None, env_nocache=None, public=None,
- sandbox=None, first_pass=False):
+ def __init__(
+ self,
+ project,
+ name,
+ kind=None,
+ provenance=None,
+ sources=None,
+ config=None,
+ variables=None,
+ environment=None,
+ env_nocache=None,
+ public=None,
+ sandbox=None,
+ first_pass=False,
+ ):
self.project = project
self.name = name
self.kind = kind
diff --git a/src/buildstream/_loader/metasource.py b/src/buildstream/_loader/metasource.py
index da2c0e292..5466d3aa5 100644
--- a/src/buildstream/_loader/metasource.py
+++ b/src/buildstream/_loader/metasource.py
@@ -18,7 +18,7 @@
# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
-class MetaSource():
+class MetaSource:
# MetaSource()
#
diff --git a/src/buildstream/_message.py b/src/buildstream/_message.py
index f4f342a6a..a2844ddce 100644
--- a/src/buildstream/_message.py
+++ b/src/buildstream/_message.py
@@ -23,57 +23,55 @@ import os
# Types of status messages.
#
-class MessageType():
- DEBUG = "debug" # Debugging message
- STATUS = "status" # Status message, verbose details
- INFO = "info" # Informative messages
- WARN = "warning" # Warning messages
- ERROR = "error" # Error messages
- BUG = "bug" # An unhandled exception was raised in a plugin
- LOG = "log" # Messages for log files _only_, never in the frontend
+class MessageType:
+ DEBUG = "debug" # Debugging message
+ STATUS = "status" # Status message, verbose details
+ INFO = "info" # Informative messages
+ WARN = "warning" # Warning messages
+ ERROR = "error" # Error messages
+ BUG = "bug" # An unhandled exception was raised in a plugin
+ LOG = "log" # Messages for log files _only_, never in the frontend
# Timed Messages: SUCCESS and FAIL have duration timestamps
- START = "start" # Status start message
- SUCCESS = "success" # Successful status complete message
- FAIL = "failure" # Failing status complete message
+ START = "start" # Status start message
+ SUCCESS = "success" # Successful status complete message
+ FAIL = "failure" # Failing status complete message
SKIPPED = "skipped"
# Messages which should be reported regardless of whether
# they are currently silenced or not
-unconditional_messages = [
- MessageType.INFO,
- MessageType.WARN,
- MessageType.FAIL,
- MessageType.ERROR,
- MessageType.BUG
-]
+unconditional_messages = [MessageType.INFO, MessageType.WARN, MessageType.FAIL, MessageType.ERROR, MessageType.BUG]
# Message object
#
-class Message():
-
- def __init__(self, message_type, message, *,
- element_name=None,
- element_key=None,
- detail=None,
- action_name=None,
- elapsed=None,
- logfile=None,
- sandbox=False,
- scheduler=False):
+class Message:
+ def __init__(
+ self,
+ message_type,
+ message,
+ *,
+ element_name=None,
+ element_key=None,
+ detail=None,
+ action_name=None,
+ elapsed=None,
+ logfile=None,
+ sandbox=False,
+ scheduler=False
+ ):
self.message_type = message_type # Message type
- self.message = message # The message string
- self.element_name = element_name # The instance element name of the issuing plugin
- self.element_key = element_key # The display key of the issuing plugin element
- self.detail = detail # An additional detail string
- self.action_name = action_name # Name of the task queue (fetch, refresh, build, etc)
- self.elapsed = elapsed # The elapsed time, in timed messages
- self.logfile = logfile # The log file path where commands took place
- self.sandbox = sandbox # Whether the error that caused this message used a sandbox
- self.pid = os.getpid() # The process pid
- self.scheduler = scheduler # Whether this is a scheduler level message
+ self.message = message # The message string
+ self.element_name = element_name # The instance element name of the issuing plugin
+ self.element_key = element_key # The display key of the issuing plugin element
+ self.detail = detail # An additional detail string
+ self.action_name = action_name # Name of the task queue (fetch, refresh, build, etc)
+ self.elapsed = elapsed # The elapsed time, in timed messages
+ self.logfile = logfile # The log file path where commands took place
+ self.sandbox = sandbox # Whether the error that caused this message used a sandbox
+ self.pid = os.getpid() # The process pid
+ self.scheduler = scheduler # Whether this is a scheduler level message
self.creation_time = datetime.datetime.now()
if message_type in (MessageType.SUCCESS, MessageType.FAIL):
assert elapsed is not None
diff --git a/src/buildstream/_messenger.py b/src/buildstream/_messenger.py
index 20c327728..03b2833ec 100644
--- a/src/buildstream/_messenger.py
+++ b/src/buildstream/_messenger.py
@@ -39,15 +39,14 @@ if "BST_TEST_SUITE" in os.environ:
# TimeData class to contain times in an object that can be passed around
# and updated from different places
-class _TimeData():
- __slots__ = ['start_time']
+class _TimeData:
+ __slots__ = ["start_time"]
def __init__(self, start_time):
self.start_time = start_time
-class Messenger():
-
+class Messenger:
def __init__(self):
self._message_handler = None
self._silence_scope_depth = 0
@@ -238,8 +237,9 @@ class Messenger():
detail = "{} of {} subtasks processed".format(task.current_progress, task.maximum_progress)
else:
detail = "{} subtasks processed".format(task.current_progress)
- message = Message(MessageType.SUCCESS, activity_name, elapsed=elapsed, detail=detail,
- element_name=element_name)
+ message = Message(
+ MessageType.SUCCESS, activity_name, elapsed=elapsed, detail=detail, element_name=element_name
+ )
self.message(message)
# recorded_messages()
@@ -274,14 +274,13 @@ class Messenger():
# Create the fully qualified logfile in the log directory,
# appending the pid and .log extension at the end.
- self._log_filename = os.path.join(logdir,
- '{}.{}.log'.format(filename, os.getpid()))
+ self._log_filename = os.path.join(logdir, "{}.{}.log".format(filename, os.getpid()))
# Ensure the directory exists first
directory = os.path.dirname(self._log_filename)
os.makedirs(directory, exist_ok=True)
- with open(self._log_filename, 'a') as logfile:
+ with open(self._log_filename, "a") as logfile:
# Write one last line to the log and flush it to disk
def flush_log():
@@ -291,7 +290,7 @@ class Messenger():
#
# So just try to flush as well as we can at SIGTERM time
try:
- logfile.write('\n\nForcefully terminated\n')
+ logfile.write("\n\nForcefully terminated\n")
logfile.flush()
except RuntimeError:
os.fsync(logfile.fileno())
@@ -352,26 +351,28 @@ class Messenger():
template += ": {message}"
- detail = ''
+ detail = ""
if message.detail is not None:
template += "\n\n{detail}"
- detail = message.detail.rstrip('\n')
+ detail = message.detail.rstrip("\n")
detail = INDENT + INDENT.join(detail.splitlines(True))
timecode = EMPTYTIME
if message.message_type in (MessageType.SUCCESS, MessageType.FAIL):
- hours, remainder = divmod(int(message.elapsed.total_seconds()), 60**2)
+ hours, remainder = divmod(int(message.elapsed.total_seconds()), 60 ** 2)
minutes, seconds = divmod(remainder, 60)
timecode = "{0:02d}:{1:02d}:{2:02d}".format(hours, minutes, seconds)
- text = template.format(timecode=timecode,
- element_name=element_name,
- type=message.message_type.upper(),
- message=message.message,
- detail=detail)
+ text = template.format(
+ timecode=timecode,
+ element_name=element_name,
+ type=message.message_type.upper(),
+ message=message.message,
+ detail=detail,
+ )
# Write to the open log file
- self._log_handle.write('{}\n'.format(text))
+ self._log_handle.write("{}\n".format(text))
self._log_handle.flush()
# get_state_for_child_job_pickling(self)
@@ -399,21 +400,21 @@ class Messenger():
# access to private details of Messenger, but it would open up a window
# where messagesw wouldn't be handled as expected.
#
- del state['_message_handler']
+ del state["_message_handler"]
# The render status callback is only used in the main process
#
- del state['_render_status_cb']
+ del state["_render_status_cb"]
# The "simple_task" context manager is not needed outside the main
# process. During testing we override it to something that cannot
# pickle, so just drop it when pickling to a child job. Note that it
# will only appear in 'state' if it has been overridden.
#
- state.pop('simple_task', None)
+ state.pop("simple_task", None)
# The State object is not needed outside the main process
- del state['_state']
+ del state["_state"]
return state
diff --git a/src/buildstream/_options/option.py b/src/buildstream/_options/option.py
index 51017be22..71d2f12f3 100644
--- a/src/buildstream/_options/option.py
+++ b/src/buildstream/_options/option.py
@@ -27,11 +27,7 @@ if TYPE_CHECKING:
# Shared symbols for validation purposes
#
-OPTION_SYMBOLS = [
- 'type',
- 'description',
- 'variable'
-]
+OPTION_SYMBOLS = ["type", "description", "variable"]
# Option()
@@ -42,7 +38,7 @@ OPTION_SYMBOLS = [
# the loaded project options is a collection of typed Option
# instances.
#
-class Option():
+class Option:
# Subclasses use this to specify the type name used
# for the yaml format and error messages
@@ -66,12 +62,12 @@ class Option():
def load(self, node):
# We don't use the description, but we do require that options have a
# description.
- node.get_str('description')
- self.variable = node.get_str('variable', default=None)
+ node.get_str("description")
+ self.variable = node.get_str("variable", default=None)
# Assert valid symbol name for variable name
if self.variable is not None:
- _assert_symbol_name(self.variable, 'variable name', ref_node=node.get_node('variable'))
+ _assert_symbol_name(self.variable, "variable name", ref_node=node.get_node("variable"))
# load_value()
#
diff --git a/src/buildstream/_options/optionarch.py b/src/buildstream/_options/optionarch.py
index cbe360f9e..2d663f0ef 100644
--- a/src/buildstream/_options/optionarch.py
+++ b/src/buildstream/_options/optionarch.py
@@ -36,7 +36,7 @@ from .optionenum import OptionEnum
#
class OptionArch(OptionEnum):
- OPTION_TYPE = 'arch'
+ OPTION_TYPE = "arch"
def load(self, node):
super().load_special(node, allow_default_definition=False)
@@ -54,12 +54,14 @@ class OptionArch(OptionEnum):
# Do not terminate the loop early to ensure we validate
# all values in the list.
except PlatformError as e:
- provenance = node.get_sequence('values').scalar_at(index).get_provenance()
+ provenance = node.get_sequence("values").scalar_at(index).get_provenance()
prefix = ""
if provenance:
prefix = "{}: ".format(provenance)
- raise LoadError("{}Invalid value for {} option '{}': {}"
- .format(prefix, self.OPTION_TYPE, self.name, e), LoadErrorReason.INVALID_DATA)
+ raise LoadError(
+ "{}Invalid value for {} option '{}': {}".format(prefix, self.OPTION_TYPE, self.name, e),
+ LoadErrorReason.INVALID_DATA,
+ )
if default_value is None:
# Host architecture is not supported by the project.
diff --git a/src/buildstream/_options/optionbool.py b/src/buildstream/_options/optionbool.py
index f91cb257d..c7289b936 100644
--- a/src/buildstream/_options/optionbool.py
+++ b/src/buildstream/_options/optionbool.py
@@ -27,13 +27,13 @@ from .option import Option, OPTION_SYMBOLS
#
class OptionBool(Option):
- OPTION_TYPE = 'bool'
+ OPTION_TYPE = "bool"
def load(self, node):
super().load(node)
- node.validate_keys(OPTION_SYMBOLS + ['default'])
- self.value = node.get_bool('default')
+ node.validate_keys(OPTION_SYMBOLS + ["default"])
+ self.value = node.get_bool("default")
def load_value(self, node, *, transform=None):
if transform:
@@ -42,13 +42,14 @@ class OptionBool(Option):
self.value = node.get_bool(self.name)
def set_value(self, value):
- if value in ('True', 'true'):
+ if value in ("True", "true"):
self.value = True
- elif value in ('False', 'false'):
+ elif value in ("False", "false"):
self.value = False
else:
- raise LoadError("Invalid value for boolean option {}: {}".format(self.name, value),
- LoadErrorReason.INVALID_DATA)
+ raise LoadError(
+ "Invalid value for boolean option {}: {}".format(self.name, value), LoadErrorReason.INVALID_DATA
+ )
def get_value(self):
if self.value:
diff --git a/src/buildstream/_options/optioneltmask.py b/src/buildstream/_options/optioneltmask.py
index 178999fa1..5a0d15f8e 100644
--- a/src/buildstream/_options/optioneltmask.py
+++ b/src/buildstream/_options/optioneltmask.py
@@ -28,7 +28,7 @@ from .optionflags import OptionFlags
#
class OptionEltMask(OptionFlags):
- OPTION_TYPE = 'element-mask'
+ OPTION_TYPE = "element-mask"
def load(self, node):
# Ask the parent constructor to disallow value definitions,
@@ -41,6 +41,6 @@ class OptionEltMask(OptionFlags):
def load_valid_values(self, node):
values = []
for filename in utils.list_relative_paths(self.pool.element_path):
- if filename.endswith('.bst'):
+ if filename.endswith(".bst"):
values.append(filename)
return values
diff --git a/src/buildstream/_options/optionenum.py b/src/buildstream/_options/optionenum.py
index 4a0941369..d30f45696 100644
--- a/src/buildstream/_options/optionenum.py
+++ b/src/buildstream/_options/optionenum.py
@@ -27,7 +27,7 @@ from .option import Option, OPTION_SYMBOLS
#
class OptionEnum(Option):
- OPTION_TYPE = 'enum'
+ OPTION_TYPE = "enum"
def __init__(self, name, definition, pool):
self.values = None
@@ -39,17 +39,20 @@ class OptionEnum(Option):
def load_special(self, node, allow_default_definition=True):
super().load(node)
- valid_symbols = OPTION_SYMBOLS + ['values']
+ valid_symbols = OPTION_SYMBOLS + ["values"]
if allow_default_definition:
- valid_symbols += ['default']
+ valid_symbols += ["default"]
node.validate_keys(valid_symbols)
- self.values = node.get_str_list('values', default=[])
+ self.values = node.get_str_list("values", default=[])
if not self.values:
- raise LoadError("{}: No values specified for {} option '{}'"
- .format(node.get_provenance(), self.OPTION_TYPE, self.name),
- LoadErrorReason.INVALID_DATA,)
+ raise LoadError(
+ "{}: No values specified for {} option '{}'".format(
+ node.get_provenance(), self.OPTION_TYPE, self.name
+ ),
+ LoadErrorReason.INVALID_DATA,
+ )
# Allow subclass to define the default value
self.value = self.load_default_value(node)
@@ -77,13 +80,14 @@ class OptionEnum(Option):
prefix = "{}: ".format(provenance)
else:
prefix = ""
- raise LoadError("{}Invalid value for {} option '{}': {}\n"
- .format(prefix, self.OPTION_TYPE, self.name, value) +
- "Valid values: {}".format(", ".join(self.values)),
- LoadErrorReason.INVALID_DATA)
+ raise LoadError(
+ "{}Invalid value for {} option '{}': {}\n".format(prefix, self.OPTION_TYPE, self.name, value)
+ + "Valid values: {}".format(", ".join(self.values)),
+ LoadErrorReason.INVALID_DATA,
+ )
def load_default_value(self, node):
- value_node = node.get_scalar('default')
+ value_node = node.get_scalar("default")
value = value_node.as_str()
self.validate(value, value_node)
return value
diff --git a/src/buildstream/_options/optionflags.py b/src/buildstream/_options/optionflags.py
index e5217a718..82ede5649 100644
--- a/src/buildstream/_options/optionflags.py
+++ b/src/buildstream/_options/optionflags.py
@@ -27,7 +27,7 @@ from .option import Option, OPTION_SYMBOLS
#
class OptionFlags(Option):
- OPTION_TYPE = 'flags'
+ OPTION_TYPE = "flags"
def __init__(self, name, definition, pool):
self.values = None
@@ -39,20 +39,23 @@ class OptionFlags(Option):
def load_special(self, node, allow_value_definitions=True):
super().load(node)
- valid_symbols = OPTION_SYMBOLS + ['default']
+ valid_symbols = OPTION_SYMBOLS + ["default"]
if allow_value_definitions:
- valid_symbols += ['values']
+ valid_symbols += ["values"]
node.validate_keys(valid_symbols)
# Allow subclass to define the valid values
self.values = self.load_valid_values(node)
if not self.values:
- raise LoadError("{}: No values specified for {} option '{}'"
- .format(node.get_provenance(), self.OPTION_TYPE, self.name),
- LoadErrorReason.INVALID_DATA)
-
- value_node = node.get_sequence('default', default=[])
+ raise LoadError(
+ "{}: No values specified for {} option '{}'".format(
+ node.get_provenance(), self.OPTION_TYPE, self.name
+ ),
+ LoadErrorReason.INVALID_DATA,
+ )
+
+ value_node = node.get_sequence("default", default=[])
self.value = value_node.as_str_list()
self.validate(self.value, value_node)
@@ -70,7 +73,7 @@ class OptionFlags(Option):
stripped = "".join(value.split())
# Get the comma separated values
- list_value = stripped.split(',')
+ list_value = stripped.split(",")
self.validate(list_value)
self.value = sorted(list_value)
@@ -86,12 +89,13 @@ class OptionFlags(Option):
prefix = "{}: ".format(provenance)
else:
prefix = ""
- raise LoadError("{}Invalid value for flags option '{}': {}\n"
- .format(prefix, self.name, value) +
- "Valid values: {}".format(", ".join(self.values)),
- LoadErrorReason.INVALID_DATA)
+ raise LoadError(
+ "{}Invalid value for flags option '{}': {}\n".format(prefix, self.name, value)
+ + "Valid values: {}".format(", ".join(self.values)),
+ LoadErrorReason.INVALID_DATA,
+ )
def load_valid_values(self, node):
# Allow the more descriptive error to raise when no values
# exist rather than bailing out here (by specifying default_value)
- return node.get_str_list('values', default=[])
+ return node.get_str_list("values", default=[])
diff --git a/src/buildstream/_options/optionos.py b/src/buildstream/_options/optionos.py
index fcf4552f5..3f4e902c9 100644
--- a/src/buildstream/_options/optionos.py
+++ b/src/buildstream/_options/optionos.py
@@ -1,4 +1,3 @@
-
#
# Copyright (C) 2017 Codethink Limited
#
@@ -26,7 +25,7 @@ from .optionenum import OptionEnum
#
class OptionOS(OptionEnum):
- OPTION_TYPE = 'os'
+ OPTION_TYPE = "os"
def load(self, node):
super().load_special(node, allow_default_definition=False)
diff --git a/src/buildstream/_options/optionpool.py b/src/buildstream/_options/optionpool.py
index a0730c617..f105bb12c 100644
--- a/src/buildstream/_options/optionpool.py
+++ b/src/buildstream/_options/optionpool.py
@@ -50,8 +50,7 @@ class OptionTypes(FastEnum):
OS = OptionOS.OPTION_TYPE
-class OptionPool():
-
+class OptionPool:
def __init__(self, element_path):
# We hold on to the element path for the sake of OptionEltMask
self.element_path = element_path
@@ -59,7 +58,7 @@ class OptionPool():
#
# Private members
#
- self._options = {} # The Options
+ self._options = {} # The Options
self._variables = None # The Options resolved into typed variables
self._environment = None
@@ -69,7 +68,7 @@ class OptionPool():
state = self.__dict__.copy()
# Jinja2 Environments don't appear to be serializable. It is easy
# enough for us to reconstruct this one anyway, so no need to pickle it.
- del state['_environment']
+ del state["_environment"]
return state
def __setstate__(self, state):
@@ -90,7 +89,7 @@ class OptionPool():
# Assert that the option name is a valid symbol
_assert_symbol_name(option_name, "option name", ref_node=option_definition, allow_dashes=False)
- opt_type_name = option_definition.get_enum('type', OptionTypes)
+ opt_type_name = option_definition.get_enum("type", OptionTypes)
opt_type = _OPTION_TYPES[opt_type_name.value]
option = opt_type(option_name, option_definition, self)
@@ -110,8 +109,9 @@ class OptionPool():
option = self._options[option_name]
except KeyError as e:
p = option_value.get_provenance()
- raise LoadError("{}: Unknown option '{}' specified"
- .format(p, option_name), LoadErrorReason.INVALID_DATA) from e
+ raise LoadError(
+ "{}: Unknown option '{}' specified".format(p, option_name), LoadErrorReason.INVALID_DATA
+ ) from e
option.load_value(node, transform=transform)
# load_cli_values()
@@ -129,8 +129,10 @@ class OptionPool():
option = self._options[option_name]
except KeyError as e:
if not ignore_unknown:
- raise LoadError("Unknown option '{}' specified on the command line"
- .format(option_name), LoadErrorReason.INVALID_DATA) from e
+ raise LoadError(
+ "Unknown option '{}' specified on the command line".format(option_name),
+ LoadErrorReason.INVALID_DATA,
+ ) from e
else:
option.set_value(option_value)
@@ -239,11 +241,13 @@ class OptionPool():
elif val == "False":
return False
else: # pragma: nocover
- raise LoadError("Failed to evaluate expression: {}".format(expression),
- LoadErrorReason.EXPRESSION_FAILED)
+ raise LoadError(
+ "Failed to evaluate expression: {}".format(expression), LoadErrorReason.EXPRESSION_FAILED
+ )
except jinja2.exceptions.TemplateError as e:
- raise LoadError("Failed to evaluate expression ({}): {}".format(expression, e),
- LoadErrorReason.EXPRESSION_FAILED)
+ raise LoadError(
+ "Failed to evaluate expression ({}): {}".format(expression, e), LoadErrorReason.EXPRESSION_FAILED
+ )
# Recursion assistent for lists, in case there
# are lists of lists.
@@ -262,25 +266,27 @@ class OptionPool():
# Return true if a conditional was processed.
#
def _process_one_node(self, node):
- conditions = node.get_sequence('(?)', default=None)
- assertion = node.get_str('(!)', default=None)
+ conditions = node.get_sequence("(?)", default=None)
+ assertion = node.get_str("(!)", default=None)
# Process assersions first, we want to abort on the first encountered
# assertion in a given dictionary, and not lose an assertion due to
# it being overwritten by a later assertion which might also trigger.
if assertion is not None:
- p = node.get_scalar('(!)').get_provenance()
+ p = node.get_scalar("(!)").get_provenance()
raise LoadError("{}: {}".format(p, assertion.strip()), LoadErrorReason.USER_ASSERTION)
if conditions is not None:
- del node['(?)']
+ del node["(?)"]
for condition in conditions:
tuples = list(condition.items())
if len(tuples) > 1:
provenance = condition.get_provenance()
- raise LoadError("{}: Conditional statement has more than one key".format(provenance),
- LoadErrorReason.INVALID_DATA)
+ raise LoadError(
+ "{}: Conditional statement has more than one key".format(provenance),
+ LoadErrorReason.INVALID_DATA,
+ )
expression, value = tuples[0]
try:
@@ -292,8 +298,10 @@ class OptionPool():
if type(value) is not MappingNode: # pylint: disable=unidiomatic-typecheck
provenance = condition.get_provenance()
- raise LoadError("{}: Only values of type 'dict' can be composed.".format(provenance),
- LoadErrorReason.ILLEGAL_COMPOSITE)
+ raise LoadError(
+ "{}: Only values of type 'dict' can be composed.".format(provenance),
+ LoadErrorReason.ILLEGAL_COMPOSITE,
+ )
# Apply the yaml fragment if its condition evaluates to true
if apply_fragment:
diff --git a/src/buildstream/_pipeline.py b/src/buildstream/_pipeline.py
index b9efc7826..0b9ab5f24 100644
--- a/src/buildstream/_pipeline.py
+++ b/src/buildstream/_pipeline.py
@@ -40,27 +40,27 @@ from ._project import ProjectRefStorage
#
# These values correspond to the CLI `--deps` arguments for convenience.
#
-class PipelineSelection():
+class PipelineSelection:
# Select only the target elements in the associated targets
- NONE = 'none'
+ NONE = "none"
# As NONE, but redirect elements that are capable of it
- REDIRECT = 'redirect'
+ REDIRECT = "redirect"
# Select elements which must be built for the associated targets to be built
- PLAN = 'plan'
+ PLAN = "plan"
# All dependencies of all targets, including the targets
- ALL = 'all'
+ ALL = "all"
# All direct build dependencies and their recursive runtime dependencies,
# excluding the targets
- BUILD = 'build'
+ BUILD = "build"
# All direct runtime dependencies and their recursive runtime dependencies,
# including the targets
- RUN = 'run'
+ RUN = "run"
# Pipeline()
@@ -70,12 +70,11 @@ class PipelineSelection():
# context (Context): The Context object
# artifacts (Context): The ArtifactCache object
#
-class Pipeline():
-
+class Pipeline:
def __init__(self, context, project, artifacts):
- self._context = context # The Context
- self._project = project # The toplevel project
+ self._context = context # The Context
+ self._project = project # The toplevel project
#
# Private members
@@ -108,10 +107,7 @@ class Pipeline():
# Now create element groups to match the input target groups
elt_iter = iter(elements)
- element_groups = [
- [next(elt_iter) for i in range(len(group))]
- for group in target_groups
- ]
+ element_groups = [[next(elt_iter) for i in range(len(group))] for group in target_groups]
return tuple(element_groups)
@@ -240,8 +236,7 @@ class Pipeline():
for t in targets:
new_elm = t._get_source_element()
if new_elm != t and not silent:
- self._message(MessageType.INFO, "Element '{}' redirected to '{}'"
- .format(t.name, new_elm.name))
+ self._message(MessageType.INFO, "Element '{}' redirected to '{}'".format(t.name, new_elm.name))
if new_elm not in elements:
elements.append(new_elm)
elif mode == PipelineSelection.PLAN:
@@ -296,9 +291,7 @@ class Pipeline():
# Build a list of 'intersection' elements, i.e. the set of
# elements that lie on the border closest to excepted elements
# between excepted and target elements.
- intersection = list(itertools.chain.from_iterable(
- find_intersection(element) for element in except_targets
- ))
+ intersection = list(itertools.chain.from_iterable(find_intersection(element) for element in except_targets))
# Now use this set of elements to traverse the targeted
# elements, except 'intersection' elements and their unique
@@ -354,10 +347,7 @@ class Pipeline():
#
def subtract_elements(self, elements, subtract):
subtract_set = set(subtract)
- return [
- e for e in elements
- if e not in subtract_set
- ]
+ return [e for e in elements if e not in subtract_set]
# add_elements()
#
@@ -426,14 +416,13 @@ class Pipeline():
for source in element.sources():
if source._get_consistency() == Consistency.INCONSISTENT:
detail += " {} is missing ref\n".format(source)
- detail += '\n'
+ detail += "\n"
detail += "Try tracking these elements first with `bst source track`\n"
raise PipelineError("Inconsistent pipeline", detail=detail, reason="inconsistent-pipeline")
if inconsistent_workspaced:
- detail = "Some workspaces exist but are not closed\n" + \
- "Try closing them with `bst workspace close`\n\n"
+ detail = "Some workspaces exist but are not closed\n" + "Try closing them with `bst workspace close`\n\n"
for element in inconsistent_workspaced:
detail += " " + element._get_full_name() + "\n"
raise PipelineError("Inconsistent pipeline", detail=detail, reason="inconsistent-pipeline-workspaced")
@@ -449,8 +438,7 @@ class Pipeline():
uncached = []
with self._context.messenger.timed_activity("Checking sources"):
for element in elements:
- if element._get_consistency() < Consistency.CACHED and \
- not element._source_cached():
+ if element._get_consistency() < Consistency.CACHED and not element._source_cached():
uncached.append(element)
if uncached:
@@ -460,9 +448,11 @@ class Pipeline():
for source in element.sources():
if source._get_consistency() < Consistency.CACHED:
detail += " {}\n".format(source)
- detail += '\n'
- detail += "Try fetching these elements first with `bst source fetch`,\n" + \
- "or run this command with `--fetch` option\n"
+ detail += "\n"
+ detail += (
+ "Try fetching these elements first with `bst source fetch`,\n"
+ + "or run this command with `--fetch` option\n"
+ )
raise PipelineError("Uncached sources", detail=detail, reason="uncached-sources")
@@ -483,10 +473,7 @@ class Pipeline():
# not contain any cross junction elements.
#
def _filter_cross_junctions(self, project, elements):
- return [
- element for element in elements
- if element._get_project() is project
- ]
+ return [element for element in elements if element._get_project() is project]
# _assert_junction_tracking()
#
@@ -511,8 +498,10 @@ class Pipeline():
for element in elements:
element_project = element._get_project()
if element_project is not self._project:
- detail = "Requested to track sources across junction boundaries\n" + \
- "in a project which does not use project.refs ref-storage."
+ detail = (
+ "Requested to track sources across junction boundaries\n"
+ + "in a project which does not use project.refs ref-storage."
+ )
raise PipelineError("Untrackable sources", detail=detail, reason="untrackable-sources")
@@ -522,8 +511,7 @@ class Pipeline():
#
def _message(self, message_type, message, **kwargs):
args = dict(kwargs)
- self._context.messenger.message(
- Message(message_type, message, **args))
+ self._context.messenger.message(Message(message_type, message, **args))
# _Planner()
@@ -533,7 +521,7 @@ class Pipeline():
# parts need to be built depending on build only dependencies
# being cached, and depth sorting for more efficient processing.
#
-class _Planner():
+class _Planner:
def __init__(self):
self.depth_map = OrderedDict()
self.visiting_elements = set()
diff --git a/src/buildstream/_platform/darwin.py b/src/buildstream/_platform/darwin.py
index f23535373..06491e8b4 100644
--- a/src/buildstream/_platform/darwin.py
+++ b/src/buildstream/_platform/darwin.py
@@ -59,9 +59,9 @@ class Darwin(Platform):
@staticmethod
def _create_dummy_sandbox(*args, **kwargs):
- kwargs['dummy_reason'] = \
- "OSXFUSE is not supported and there are no supported sandbox " + \
- "technologies for MacOS at this time"
+ kwargs["dummy_reason"] = (
+ "OSXFUSE is not supported and there are no supported sandbox " + "technologies for MacOS at this time"
+ )
return SandboxDummy(*args, **kwargs)
def _setup_dummy_sandbox(self):
diff --git a/src/buildstream/_platform/fallback.py b/src/buildstream/_platform/fallback.py
index 4f7ff8086..b9e9f520d 100644
--- a/src/buildstream/_platform/fallback.py
+++ b/src/buildstream/_platform/fallback.py
@@ -20,15 +20,15 @@ from .platform import Platform
class Fallback(Platform):
-
def _check_dummy_sandbox_config(self, config):
return True
def _create_dummy_sandbox(self, *args, **kwargs):
- kwargs['dummy_reason'] = \
- ("FallBack platform only implements dummy sandbox, "
- "Buildstream may be having issues correctly detecting your platform, "
- "platform can be forced with BST_FORCE_BACKEND")
+ kwargs["dummy_reason"] = (
+ "FallBack platform only implements dummy sandbox, "
+ "Buildstream may be having issues correctly detecting your platform, "
+ "platform can be forced with BST_FORCE_BACKEND"
+ )
return SandboxDummy(*args, **kwargs)
def _setup_dummy_sandbox(self):
diff --git a/src/buildstream/_platform/linux.py b/src/buildstream/_platform/linux.py
index b400bfaac..bdc2e0df1 100644
--- a/src/buildstream/_platform/linux.py
+++ b/src/buildstream/_platform/linux.py
@@ -28,17 +28,16 @@ from .._exceptions import PlatformError
class Linux(Platform):
-
def _setup_sandbox(self, force_sandbox):
sandbox_setups = {
- 'bwrap': self._setup_bwrap_sandbox,
- 'buildbox': self._setup_buildbox_sandbox,
- 'chroot': self._setup_chroot_sandbox,
- 'dummy': self._setup_dummy_sandbox,
+ "bwrap": self._setup_bwrap_sandbox,
+ "buildbox": self._setup_buildbox_sandbox,
+ "chroot": self._setup_chroot_sandbox,
+ "dummy": self._setup_dummy_sandbox,
}
preferred_sandboxes = [
- 'bwrap',
+ "bwrap",
]
self._try_sandboxes(force_sandbox, sandbox_setups, preferred_sandboxes)
@@ -54,11 +53,12 @@ class Linux(Platform):
def can_crossbuild(self, config):
host_arch = self.get_host_arch()
- if ((config.build_arch == "x86-32" and host_arch == "x86-64") or
- (config.build_arch == "aarch32" and host_arch == "aarch64")):
+ if (config.build_arch == "x86-32" and host_arch == "x86-64") or (
+ config.build_arch == "aarch32" and host_arch == "aarch64"
+ ):
if self.linux32 is None:
try:
- utils.get_host_tool('linux32')
+ utils.get_host_tool("linux32")
self.linux32 = True
except utils.ProgramNotFoundError:
self.linux32 = False
@@ -76,7 +76,7 @@ class Linux(Platform):
def _create_dummy_sandbox(self, *args, **kwargs):
dummy_reasons = " and ".join(self.dummy_reasons)
- kwargs['dummy_reason'] = dummy_reasons
+ kwargs["dummy_reason"] = dummy_reasons
return SandboxDummy(*args, **kwargs)
def _setup_dummy_sandbox(self):
@@ -87,11 +87,13 @@ class Linux(Platform):
# Bubble-wrap sandbox methods
def _check_sandbox_config_bwrap(self, config):
from ..sandbox._sandboxbwrap import SandboxBwrap
+
return SandboxBwrap.check_sandbox_config(self, config)
def _create_bwrap_sandbox(self, *args, **kwargs):
from ..sandbox._sandboxbwrap import SandboxBwrap
- kwargs['linux32'] = self.linux32
+
+ kwargs["linux32"] = self.linux32
return SandboxBwrap(*args, **kwargs)
def _setup_bwrap_sandbox(self):
@@ -110,15 +112,18 @@ class Linux(Platform):
# Chroot sandbox methods
def _check_sandbox_config_chroot(self, config):
from ..sandbox._sandboxchroot import SandboxChroot
+
return SandboxChroot.check_sandbox_config(self, config)
@staticmethod
def _create_chroot_sandbox(*args, **kwargs):
from ..sandbox._sandboxchroot import SandboxChroot
+
return SandboxChroot(*args, **kwargs)
def _setup_chroot_sandbox(self):
from ..sandbox._sandboxchroot import SandboxChroot
+
self._check_sandbox(SandboxChroot)
self.check_sandbox_config = self._check_sandbox_config_chroot
self.create_sandbox = Linux._create_chroot_sandbox
@@ -127,18 +132,23 @@ class Linux(Platform):
# Buildbox sandbox methods
def _check_sandbox_config_buildbox(self, config):
from ..sandbox._sandboxbuildbox import SandboxBuildBox
+
return SandboxBuildBox.check_sandbox_config(self, config)
@staticmethod
def _create_buildbox_sandbox(*args, **kwargs):
from ..sandbox._sandboxbuildbox import SandboxBuildBox
- if kwargs.get('allow_real_directory'):
- raise PlatformError("The BuildBox Sandbox does not support real directories.",
- reason="You are using BuildBox sandbox because BST_FORCE_SANBOX=buildbox")
+
+ if kwargs.get("allow_real_directory"):
+ raise PlatformError(
+ "The BuildBox Sandbox does not support real directories.",
+ reason="You are using BuildBox sandbox because BST_FORCE_SANBOX=buildbox",
+ )
return SandboxBuildBox(*args, **kwargs)
def _setup_buildbox_sandbox(self):
from ..sandbox._sandboxbuildbox import SandboxBuildBox
+
self._check_sandbox(SandboxBuildBox)
self.check_sandbox_config = self._check_sandbox_config_buildbox
self.create_sandbox = self._create_buildbox_sandbox
diff --git a/src/buildstream/_platform/platform.py b/src/buildstream/_platform/platform.py
index af49b9e82..1fddbe82c 100644
--- a/src/buildstream/_platform/platform.py
+++ b/src/buildstream/_platform/platform.py
@@ -29,7 +29,7 @@ from .._exceptions import PlatformError, ImplError, SandboxError
from .. import utils
-class Platform():
+class Platform:
# Platform()
#
# A class to manage platform-specific details. Currently holds the
@@ -45,7 +45,7 @@ class Platform():
self._setup_sandbox(force_sandbox)
def _setup_sandbox(self, force_sandbox):
- sandbox_setups = {'dummy': self._setup_dummy_sandbox}
+ sandbox_setups = {"dummy": self._setup_dummy_sandbox}
preferred_sandboxes = []
self._try_sandboxes(force_sandbox, sandbox_setups, preferred_sandboxes)
@@ -58,12 +58,16 @@ class Platform():
try:
sandbox_setups[force_sandbox]()
except KeyError:
- raise PlatformError("Forced Sandbox is unavailable on this platform: BST_FORCE_SANDBOX"
- " is set to {} but it is not available".format(force_sandbox))
+ raise PlatformError(
+ "Forced Sandbox is unavailable on this platform: BST_FORCE_SANDBOX"
+ " is set to {} but it is not available".format(force_sandbox)
+ )
except SandboxError as Error:
- raise PlatformError("Forced Sandbox Error: BST_FORCE_SANDBOX"
- " is set to {} but cannot be setup".format(force_sandbox),
- detail=" and ".join(self.dummy_reasons)) from Error
+ raise PlatformError(
+ "Forced Sandbox Error: BST_FORCE_SANDBOX"
+ " is set to {} but cannot be setup".format(force_sandbox),
+ detail=" and ".join(self.dummy_reasons),
+ ) from Error
else:
for good_sandbox in preferred_sandboxes:
try:
@@ -73,7 +77,7 @@ class Platform():
continue
except utils.ProgramNotFoundError:
continue
- sandbox_setups['dummy']()
+ sandbox_setups["dummy"]()
def _check_sandbox(self, Sandbox):
try:
@@ -87,29 +91,29 @@ class Platform():
# Meant for testing purposes and therefore hidden in the
# deepest corners of the source code. Try not to abuse this,
# please?
- if os.getenv('BST_FORCE_SANDBOX'):
- force_sandbox = os.getenv('BST_FORCE_SANDBOX')
+ if os.getenv("BST_FORCE_SANDBOX"):
+ force_sandbox = os.getenv("BST_FORCE_SANDBOX")
else:
force_sandbox = None
- if os.getenv('BST_FORCE_BACKEND'):
- backend = os.getenv('BST_FORCE_BACKEND')
- elif sys.platform.startswith('darwin'):
- backend = 'darwin'
- elif sys.platform.startswith('linux'):
- backend = 'linux'
- elif sys.platform == 'win32':
- backend = 'win32'
+ if os.getenv("BST_FORCE_BACKEND"):
+ backend = os.getenv("BST_FORCE_BACKEND")
+ elif sys.platform.startswith("darwin"):
+ backend = "darwin"
+ elif sys.platform.startswith("linux"):
+ backend = "linux"
+ elif sys.platform == "win32":
+ backend = "win32"
else:
- backend = 'fallback'
+ backend = "fallback"
- if backend == 'linux':
+ if backend == "linux":
from .linux import Linux as PlatformImpl # pylint: disable=cyclic-import
- elif backend == 'darwin':
+ elif backend == "darwin":
from .darwin import Darwin as PlatformImpl # pylint: disable=cyclic-import
- elif backend == 'win32':
+ elif backend == "win32":
from .win32 import Win32 as PlatformImpl # pylint: disable=cyclic-import
- elif backend == 'fallback':
+ elif backend == "fallback":
from .fallback import Fallback as PlatformImpl # pylint: disable=cyclic-import
else:
raise PlatformError("No such platform: '{}'".format(backend))
@@ -156,11 +160,11 @@ class Platform():
"sparc64": "sparc-v9",
"sparc-v9": "sparc-v9",
"x86-32": "x86-32",
- "x86-64": "x86-64"
+ "x86-64": "x86-64",
}
try:
- return aliases[arch.replace('_', '-').lower()]
+ return aliases[arch.replace("_", "-").lower()]
except KeyError:
raise PlatformError("Unknown architecture: {}".format(arch))
@@ -188,7 +192,7 @@ class Platform():
def does_multiprocessing_start_require_pickling(self):
# Note that if the start method has not been set before now, it will be
# set to the platform default by `get_start_method`.
- return multiprocessing.get_start_method() != 'fork'
+ return multiprocessing.get_start_method() != "fork"
##################################################################
# Sandbox functions #
@@ -206,12 +210,12 @@ class Platform():
# (Sandbox) A sandbox
#
def create_sandbox(self, *args, **kwargs):
- raise ImplError("Platform {platform} does not implement create_sandbox()"
- .format(platform=type(self).__name__))
+ raise ImplError("Platform {platform} does not implement create_sandbox()".format(platform=type(self).__name__))
def check_sandbox_config(self, config):
- raise ImplError("Platform {platform} does not implement check_sandbox_config()"
- .format(platform=type(self).__name__))
+ raise ImplError(
+ "Platform {platform} does not implement check_sandbox_config()".format(platform=type(self).__name__)
+ )
def maximize_open_file_limit(self):
# Need to set resources for _frontend/app.py as this is dependent on the platform
@@ -230,5 +234,6 @@ class Platform():
resource.setrlimit(resource.RLIMIT_NOFILE, (hard_limit, hard_limit))
def _setup_dummy_sandbox(self):
- raise ImplError("Platform {platform} does not implement _setup_dummy_sandbox()"
- .format(platform=type(self).__name__))
+ raise ImplError(
+ "Platform {platform} does not implement _setup_dummy_sandbox()".format(platform=type(self).__name__)
+ )
diff --git a/src/buildstream/_platform/win32.py b/src/buildstream/_platform/win32.py
index 36680019d..a2529d8f6 100644
--- a/src/buildstream/_platform/win32.py
+++ b/src/buildstream/_platform/win32.py
@@ -20,7 +20,6 @@ from .platform import Platform
class Win32(Platform):
-
def maximize_open_file_limit(self):
# Note that on Windows, we don't have the 'resource' module to help us
# configure open file limits.
@@ -50,7 +49,7 @@ class Win32(Platform):
@staticmethod
def _create_dummy_sandbox(*args, **kwargs):
- kwargs['dummy_reason'] = "There are no supported sandbox technologies for Win32 at this time."
+ kwargs["dummy_reason"] = "There are no supported sandbox technologies for Win32 at this time."
return SandboxDummy(*args, **kwargs)
def _setup_dummy_sandbox(self):
diff --git a/src/buildstream/_plugincontext.py b/src/buildstream/_plugincontext.py
index b07c2b31a..54839e16b 100644
--- a/src/buildstream/_plugincontext.py
+++ b/src/buildstream/_plugincontext.py
@@ -41,10 +41,8 @@ from . import utils
# a given BuildStream project are isolated to their respective
# Pipelines.
#
-class PluginContext():
-
- def __init__(self, plugin_base, base_type, site_plugin_path, *,
- plugin_origins=None, format_versions={}):
+class PluginContext:
+ def __init__(self, plugin_base, base_type, site_plugin_path, *, plugin_origins=None, format_versions={}):
# For pickling across processes, make sure this context has a unique
# identifier, which we prepend to the identifier of each PluginSource.
@@ -59,7 +57,7 @@ class PluginContext():
# Private members
#
self._base_type = base_type # The base class plugins derive from
- self._types = {} # Plugin type lookup table by kind
+ self._types = {} # Plugin type lookup table by kind
self._plugin_origins = plugin_origins or []
# The PluginSource object
@@ -72,8 +70,7 @@ class PluginContext():
def _init_site_source(self):
self._site_source = self._plugin_base.make_plugin_source(
- searchpath=self._site_plugin_path,
- identifier=self._identifier + 'site',
+ searchpath=self._site_plugin_path, identifier=self._identifier + "site",
)
def __getstate__(self):
@@ -93,11 +90,11 @@ class PluginContext():
# this by making sure we are not creating new members, only clearing
# existing ones.
#
- del state['_site_source']
- assert '_types' in state
- state['_types'] = {}
- assert '_alternate_sources' in state
- state['_alternate_sources'] = {}
+ del state["_site_source"]
+ assert "_types" in state
+ state["_types"] = {}
+ assert "_alternate_sources" in state
+ state["_alternate_sources"] = {}
return state
@@ -133,60 +130,51 @@ class PluginContext():
return self._types.values()
def _get_local_plugin_source(self, path):
- if ('local', path) not in self._alternate_sources:
+ if ("local", path) not in self._alternate_sources:
# key by a tuple to avoid collision
- source = self._plugin_base.make_plugin_source(
- searchpath=[path],
- identifier=self._identifier + path,
- )
+ source = self._plugin_base.make_plugin_source(searchpath=[path], identifier=self._identifier + path,)
# Ensure that sources never get garbage collected,
# as they'll take the plugins with them.
- self._alternate_sources[('local', path)] = source
+ self._alternate_sources[("local", path)] = source
else:
- source = self._alternate_sources[('local', path)]
+ source = self._alternate_sources[("local", path)]
return source
def _get_pip_plugin_source(self, package_name, kind):
defaults = None
- if ('pip', package_name) not in self._alternate_sources:
+ if ("pip", package_name) not in self._alternate_sources:
import pkg_resources
+
# key by a tuple to avoid collision
try:
- package = pkg_resources.get_entry_info(package_name,
- 'buildstream.plugins',
- kind)
+ package = pkg_resources.get_entry_info(package_name, "buildstream.plugins", kind)
except pkg_resources.DistributionNotFound as e:
- raise PluginError("Failed to load {} plugin '{}': {}"
- .format(self._base_type.__name__, kind, e)) from e
+ raise PluginError("Failed to load {} plugin '{}': {}".format(self._base_type.__name__, kind, e)) from e
if package is None:
- raise PluginError("Pip package {} does not contain a plugin named '{}'"
- .format(package_name, kind))
+ raise PluginError("Pip package {} does not contain a plugin named '{}'".format(package_name, kind))
location = package.dist.get_resource_filename(
- pkg_resources._manager,
- package.module_name.replace('.', os.sep) + '.py'
+ pkg_resources._manager, package.module_name.replace(".", os.sep) + ".py"
)
# Also load the defaults - required since setuptools
# may need to extract the file.
try:
defaults = package.dist.get_resource_filename(
- pkg_resources._manager,
- package.module_name.replace('.', os.sep) + '.yaml'
+ pkg_resources._manager, package.module_name.replace(".", os.sep) + ".yaml"
)
except KeyError:
# The plugin didn't have an accompanying YAML file
defaults = None
source = self._plugin_base.make_plugin_source(
- searchpath=[os.path.dirname(location)],
- identifier=self._identifier + os.path.dirname(location),
+ searchpath=[os.path.dirname(location)], identifier=self._identifier + os.path.dirname(location),
)
- self._alternate_sources[('pip', package_name)] = source
+ self._alternate_sources[("pip", package_name)] = source
else:
- source = self._alternate_sources[('pip', package_name)]
+ source = self._alternate_sources[("pip", package_name)]
return source, defaults
@@ -199,27 +187,27 @@ class PluginContext():
loaded_dependency = False
for origin in self._plugin_origins:
- if kind not in origin.get_str_list('plugins'):
+ if kind not in origin.get_str_list("plugins"):
continue
- if origin.get_str('origin') == 'local':
- local_path = origin.get_str('path')
+ if origin.get_str("origin") == "local":
+ local_path = origin.get_str("path")
source = self._get_local_plugin_source(local_path)
- elif origin.get_str('origin') == 'pip':
- package_name = origin.get_str('package-name')
+ elif origin.get_str("origin") == "pip":
+ package_name = origin.get_str("package-name")
source, defaults = self._get_pip_plugin_source(package_name, kind)
else:
- raise PluginError("Failed to load plugin '{}': "
- "Unexpected plugin origin '{}'"
- .format(kind, origin.get_str('origin')))
+ raise PluginError(
+ "Failed to load plugin '{}': "
+ "Unexpected plugin origin '{}'".format(kind, origin.get_str("origin"))
+ )
loaded_dependency = True
break
# Fall back to getting the source from site
if not source:
if kind not in self._site_source.list_plugins():
- raise PluginError("No {} type registered for kind '{}'"
- .format(self._base_type.__name__, kind))
+ raise PluginError("No {} type registered for kind '{}'".format(self._base_type.__name__, kind))
source = self._site_source
@@ -241,17 +229,18 @@ class PluginContext():
defaults = os.path.join(plugin_dir, plugin_conf_name)
except ImportError as e:
- raise PluginError("Failed to load {} plugin '{}': {}"
- .format(self._base_type.__name__, kind, e)) from e
+ raise PluginError("Failed to load {} plugin '{}': {}".format(self._base_type.__name__, kind, e)) from e
try:
plugin_type = plugin.setup()
except AttributeError as e:
- raise PluginError("{} plugin '{}' did not provide a setup() function"
- .format(self._base_type.__name__, kind)) from e
+ raise PluginError(
+ "{} plugin '{}' did not provide a setup() function".format(self._base_type.__name__, kind)
+ ) from e
except TypeError as e:
- raise PluginError("setup symbol in {} plugin '{}' is not a function"
- .format(self._base_type.__name__, kind)) from e
+ raise PluginError(
+ "setup symbol in {} plugin '{}' is not a function".format(self._base_type.__name__, kind)
+ ) from e
self._assert_plugin(kind, plugin_type)
self._assert_version(kind, plugin_type)
@@ -259,19 +248,23 @@ class PluginContext():
def _assert_plugin(self, kind, plugin_type):
if kind in self._types:
- raise PluginError("Tried to register {} plugin for existing kind '{}' "
- "(already registered {})"
- .format(self._base_type.__name__, kind, self._types[kind].__name__))
+ raise PluginError(
+ "Tried to register {} plugin for existing kind '{}' "
+ "(already registered {})".format(self._base_type.__name__, kind, self._types[kind].__name__)
+ )
try:
if not issubclass(plugin_type, self._base_type):
- raise PluginError("{} plugin '{}' returned type '{}', which is not a subclass of {}"
- .format(self._base_type.__name__, kind,
- plugin_type.__name__,
- self._base_type.__name__))
+ raise PluginError(
+ "{} plugin '{}' returned type '{}', which is not a subclass of {}".format(
+ self._base_type.__name__, kind, plugin_type.__name__, self._base_type.__name__
+ )
+ )
except TypeError as e:
- raise PluginError("{} plugin '{}' returned something that is not a type (expected subclass of {})"
- .format(self._base_type.__name__, kind,
- self._base_type.__name__)) from e
+ raise PluginError(
+ "{} plugin '{}' returned something that is not a type (expected subclass of {})".format(
+ self._base_type.__name__, kind, self._base_type.__name__
+ )
+ ) from e
def _assert_version(self, kind, plugin_type):
@@ -282,12 +275,16 @@ class PluginContext():
req_minor = plugin_type.BST_REQUIRED_VERSION_MINOR
if (bst_major, bst_minor) < (req_major, req_minor):
- raise PluginError("BuildStream {}.{} is too old for {} plugin '{}' (requires {}.{})"
- .format(
- bst_major, bst_minor,
- self._base_type.__name__, kind,
- plugin_type.BST_REQUIRED_VERSION_MAJOR,
- plugin_type.BST_REQUIRED_VERSION_MINOR))
+ raise PluginError(
+ "BuildStream {}.{} is too old for {} plugin '{}' (requires {}.{})".format(
+ bst_major,
+ bst_minor,
+ self._base_type.__name__,
+ kind,
+ plugin_type.BST_REQUIRED_VERSION_MAJOR,
+ plugin_type.BST_REQUIRED_VERSION_MINOR,
+ )
+ )
# _assert_plugin_format()
#
@@ -296,6 +293,9 @@ class PluginContext():
#
def _assert_plugin_format(self, plugin, version):
if plugin.BST_FORMAT_VERSION < version:
- raise LoadError("{}: Format version {} is too old for requested version {}"
- .format(plugin, plugin.BST_FORMAT_VERSION, version),
- LoadErrorReason.UNSUPPORTED_PLUGIN)
+ raise LoadError(
+ "{}: Format version {} is too old for requested version {}".format(
+ plugin, plugin.BST_FORMAT_VERSION, version
+ ),
+ LoadErrorReason.UNSUPPORTED_PLUGIN,
+ )
diff --git a/src/buildstream/_profile.py b/src/buildstream/_profile.py
index c68d058ad..fdde04ab7 100644
--- a/src/buildstream/_profile.py
+++ b/src/buildstream/_profile.py
@@ -40,15 +40,15 @@ from ._exceptions import ProfileError
# BST_PROFILE=circ-dep-check:sort-deps bst <command> <args>
#
# The special 'all' value will enable all profiles.
-class Topics():
- CIRCULAR_CHECK = 'circ-dep-check'
- SORT_DEPENDENCIES = 'sort-deps'
- LOAD_CONTEXT = 'load-context'
- LOAD_PROJECT = 'load-project'
- LOAD_PIPELINE = 'load-pipeline'
- LOAD_SELECTION = 'load-selection'
- SCHEDULER = 'scheduler'
- ALL = 'all'
+class Topics:
+ CIRCULAR_CHECK = "circ-dep-check"
+ SORT_DEPENDENCIES = "sort-deps"
+ LOAD_CONTEXT = "load-context"
+ LOAD_PROJECT = "load-project"
+ LOAD_PIPELINE = "load-pipeline"
+ LOAD_SELECTION = "load-selection"
+ SCHEDULER = "scheduler"
+ ALL = "all"
class _Profile:
@@ -64,8 +64,8 @@ class _Profile:
os.getcwd(),
"profile-{}-{}".format(
datetime.datetime.fromtimestamp(self.start_time).strftime("%Y%m%dT%H%M%S"),
- self.key.replace("/", "-").replace(".", "-")
- )
+ self.key.replace("/", "-").replace(".", "-"),
+ ),
)
self.log_filename = "{}.log".format(filename_template)
self.cprofile_filename = "{}.cprofile".format(filename_template)
@@ -87,14 +87,16 @@ class _Profile:
self.profiler.disable()
def save(self):
- heading = "\n".join([
- "-" * 64,
- "Profile for key: {}".format(self.key),
- "Started at: {}".format(self.start_time),
- "\n\t{}".format(self.message) if self.message else "",
- "-" * 64,
- "" # for a final new line
- ])
+ heading = "\n".join(
+ [
+ "-" * 64,
+ "Profile for key: {}".format(self.key),
+ "Started at: {}".format(self.start_time),
+ "\n\t{}".format(self.message) if self.message else "",
+ "-" * 64,
+ "", # for a final new line
+ ]
+ )
with open(self.log_filename, "a") as fp:
stats = pstats.Stats(self.profiler, *self._additional_pstats_files, stream=fp)
@@ -116,10 +118,7 @@ class _Profiler:
self._valid_topics = False
if settings:
- self.enabled_topics = {
- topic
- for topic in settings.split(":")
- }
+ self.enabled_topics = {topic for topic in settings.split(":")}
@contextlib.contextmanager
def profile(self, topic, key, message=None):
@@ -170,8 +169,7 @@ class _Profiler:
non_valid_topics = [topic for topic in self.enabled_topics if topic not in vars(Topics).values()]
if non_valid_topics:
- raise ProfileError("Provided BST_PROFILE topics do not exist: {}"
- .format(", ".join(non_valid_topics)))
+ raise ProfileError("Provided BST_PROFILE topics do not exist: {}".format(", ".join(non_valid_topics)))
self._valid_topics = True
diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py
index 54a011e0d..67d41a6b5 100644
--- a/src/buildstream/_project.py
+++ b/src/buildstream/_project.py
@@ -49,7 +49,7 @@ from ._workspaces import WORKSPACE_PROJECT_FILE
# Project Configuration file
-_PROJECT_CONF_FILE = 'project.conf'
+_PROJECT_CONF_FILE = "project.conf"
# List of all places plugins can come from
@@ -64,8 +64,7 @@ class PluginOrigins(FastEnum):
# A simple object describing the behavior of
# a host mount.
#
-class HostMount():
-
+class HostMount:
def __init__(self, path, host_path=None, optional=False):
# Support environment variable expansion in host mounts
@@ -73,9 +72,9 @@ class HostMount():
if host_path is not None:
host_path = os.path.expandvars(host_path)
- self.path = path # Path inside the sandbox
- self.host_path = host_path # Path on the host
- self.optional = optional # Optional mounts do not incur warnings or errors
+ self.path = path # Path inside the sandbox
+ self.host_path = host_path # Path on the host
+ self.optional = optional # Optional mounts do not incur warnings or errors
if self.host_path is None:
self.host_path = self.path
@@ -86,24 +85,32 @@ class ProjectConfig:
def __init__(self):
self.element_factory = None
self.source_factory = None
- self.options = None # OptionPool
- self.base_variables = {} # The base set of variables
- self.element_overrides = {} # Element specific configurations
- self.source_overrides = {} # Source specific configurations
- self.mirrors = OrderedDict() # contains dicts of alias-mappings to URIs.
- self.default_mirror = None # The name of the preferred mirror.
- self._aliases = None # Aliases dictionary
+ self.options = None # OptionPool
+ self.base_variables = {} # The base set of variables
+ self.element_overrides = {} # Element specific configurations
+ self.source_overrides = {} # Source specific configurations
+ self.mirrors = OrderedDict() # contains dicts of alias-mappings to URIs.
+ self.default_mirror = None # The name of the preferred mirror.
+ self._aliases = None # Aliases dictionary
# Project()
#
# The Project Configuration
#
-class Project():
-
- def __init__(self, directory, context, *, junction=None, cli_options=None,
- default_mirror=None, parent_loader=None,
- search_for_project=True, fetch_subprojects=None):
+class Project:
+ def __init__(
+ self,
+ directory,
+ context,
+ *,
+ junction=None,
+ cli_options=None,
+ default_mirror=None,
+ parent_loader=None,
+ search_for_project=True,
+ fetch_subprojects=None
+ ):
# The project name
self.name = None
@@ -125,31 +132,31 @@ class Project():
self._default_targets = None
# ProjectRefs for the main refs and also for junctions
- self.refs = ProjectRefs(self.directory, 'project.refs')
- self.junction_refs = ProjectRefs(self.directory, 'junction.refs')
+ self.refs = ProjectRefs(self.directory, "project.refs")
+ self.junction_refs = ProjectRefs(self.directory, "junction.refs")
self.config = ProjectConfig()
self.first_pass_config = ProjectConfig()
- self.junction = junction # The junction Element object, if this is a subproject
+ self.junction = junction # The junction Element object, if this is a subproject
- self.ref_storage = None # ProjectRefStorage setting
- self.base_environment = {} # The base set of environment variables
- self.base_env_nocache = None # The base nocache mask (list) for the environment
+ self.ref_storage = None # ProjectRefStorage setting
+ self.base_environment = {} # The base set of environment variables
+ self.base_env_nocache = None # The base nocache mask (list) for the environment
#
# Private Members
#
- self._default_mirror = default_mirror # The name of the preferred mirror.
+ self._default_mirror = default_mirror # The name of the preferred mirror.
self._cli_options = cli_options
- self._fatal_warnings = [] # A list of warnings which should trigger an error
+ self._fatal_warnings = [] # A list of warnings which should trigger an error
- self._shell_command = [] # The default interactive shell command
+ self._shell_command = [] # The default interactive shell command
self._shell_environment = {} # Statically set environment vars
- self._shell_host_files = [] # A list of HostMount objects
+ self._shell_host_files = [] # A list of HostMount objects
self.artifact_cache_specs = None
self.source_cache_specs = None
@@ -163,7 +170,7 @@ class Project():
self._fully_loaded = False
self._project_includes = None
- with PROFILER.profile(Topics.LOAD_PROJECT, self.directory.replace(os.sep, '-')):
+ with PROFILER.profile(Topics.LOAD_PROJECT, self.directory.replace(os.sep, "-")):
self._load(parent_loader=parent_loader, fetch_subprojects=fetch_subprojects)
self._partially_loaded = True
@@ -252,23 +259,24 @@ class Project():
# (LoadError): In case that the project path is not valid or does not
# exist
#
- def get_path_from_node(self, node, *,
- check_is_file=False, check_is_dir=False):
+ def get_path_from_node(self, node, *, check_is_file=False, check_is_dir=False):
path_str = node.as_str()
path = Path(path_str)
full_path = self._absolute_directory_path / path
if full_path.is_symlink():
provenance = node.get_provenance()
- raise LoadError("{}: Specified path '{}' must not point to "
- "symbolic links ".format(provenance, path_str),
- LoadErrorReason.PROJ_PATH_INVALID_KIND)
+ raise LoadError(
+ "{}: Specified path '{}' must not point to " "symbolic links ".format(provenance, path_str),
+ LoadErrorReason.PROJ_PATH_INVALID_KIND,
+ )
- if path.parts and path.parts[0] == '..':
+ if path.parts and path.parts[0] == "..":
provenance = node.get_provenance()
- raise LoadError("{}: Specified path '{}' first component must "
- "not be '..'".format(provenance, path_str),
- LoadErrorReason.PROJ_PATH_INVALID)
+ raise LoadError(
+ "{}: Specified path '{}' first component must " "not be '..'".format(provenance, path_str),
+ LoadErrorReason.PROJ_PATH_INVALID,
+ )
try:
if sys.version_info[0] == 3 and sys.version_info[1] < 6:
@@ -277,55 +285,81 @@ class Project():
full_resolved_path = full_path.resolve(strict=True) # pylint: disable=unexpected-keyword-arg
except FileNotFoundError:
provenance = node.get_provenance()
- raise LoadError("{}: Specified path '{}' does not exist".format(provenance, path_str),
- LoadErrorReason.MISSING_FILE)
+ raise LoadError(
+ "{}: Specified path '{}' does not exist".format(provenance, path_str), LoadErrorReason.MISSING_FILE
+ )
is_inside = self._absolute_directory_path in full_resolved_path.parents or (
- full_resolved_path == self._absolute_directory_path)
+ full_resolved_path == self._absolute_directory_path
+ )
if not is_inside:
provenance = node.get_provenance()
- raise LoadError("{}: Specified path '{}' must not lead outside of the "
- "project directory".format(provenance, path_str),
- LoadErrorReason.PROJ_PATH_INVALID)
+ raise LoadError(
+ "{}: Specified path '{}' must not lead outside of the "
+ "project directory".format(provenance, path_str),
+ LoadErrorReason.PROJ_PATH_INVALID,
+ )
if path.is_absolute():
provenance = node.get_provenance()
- raise LoadError("{}: Absolute path: '{}' invalid.\n"
- "Please specify a path relative to the project's root."
- .format(provenance, path), LoadErrorReason.PROJ_PATH_INVALID)
+ raise LoadError(
+ "{}: Absolute path: '{}' invalid.\n"
+ "Please specify a path relative to the project's root.".format(provenance, path),
+ LoadErrorReason.PROJ_PATH_INVALID,
+ )
- if full_resolved_path.is_socket() or (
- full_resolved_path.is_fifo() or
- full_resolved_path.is_block_device()):
+ if full_resolved_path.is_socket() or (full_resolved_path.is_fifo() or full_resolved_path.is_block_device()):
provenance = node.get_provenance()
- raise LoadError("{}: Specified path '{}' points to an unsupported "
- "file kind".format(provenance, path_str), LoadErrorReason.PROJ_PATH_INVALID_KIND)
+ raise LoadError(
+ "{}: Specified path '{}' points to an unsupported " "file kind".format(provenance, path_str),
+ LoadErrorReason.PROJ_PATH_INVALID_KIND,
+ )
if check_is_file and not full_resolved_path.is_file():
provenance = node.get_provenance()
- raise LoadError("{}: Specified path '{}' is not a regular file"
- .format(provenance, path_str), LoadErrorReason.PROJ_PATH_INVALID_KIND)
+ raise LoadError(
+ "{}: Specified path '{}' is not a regular file".format(provenance, path_str),
+ LoadErrorReason.PROJ_PATH_INVALID_KIND,
+ )
if check_is_dir and not full_resolved_path.is_dir():
provenance = node.get_provenance()
- raise LoadError("{}: Specified path '{}' is not a directory"
- .format(provenance, path_str), LoadErrorReason.PROJ_PATH_INVALID_KIND)
+ raise LoadError(
+ "{}: Specified path '{}' is not a directory".format(provenance, path_str),
+ LoadErrorReason.PROJ_PATH_INVALID_KIND,
+ )
return path_str
def _validate_node(self, node):
- node.validate_keys([
- 'format-version',
- 'element-path', 'variables',
- 'environment', 'environment-nocache',
- 'split-rules', 'elements', 'plugins',
- 'aliases', 'name', 'defaults',
- 'artifacts', 'options',
- 'fail-on-overlap', 'shell', 'fatal-warnings',
- 'ref-storage', 'sandbox', 'mirrors', 'remote-execution',
- 'sources', 'source-caches', '(@)'
- ])
+ node.validate_keys(
+ [
+ "format-version",
+ "element-path",
+ "variables",
+ "environment",
+ "environment-nocache",
+ "split-rules",
+ "elements",
+ "plugins",
+ "aliases",
+ "name",
+ "defaults",
+ "artifacts",
+ "options",
+ "fail-on-overlap",
+ "shell",
+ "fatal-warnings",
+ "ref-storage",
+ "sandbox",
+ "mirrors",
+ "remote-execution",
+ "sources",
+ "source-caches",
+ "(@)",
+ ]
+ )
# create_element()
#
@@ -438,10 +472,7 @@ class Project():
with self._context.messenger.simple_task("Resolving elements") as task:
if task:
task.set_maximum_progress(self.loader.loaded)
- elements = [
- Element._new_from_meta(meta, task)
- for meta in meta_elements
- ]
+ elements = [Element._new_from_meta(meta, task) for meta in meta_elements]
Element._clear_meta_elements_cache()
@@ -450,13 +481,11 @@ class Project():
redundant_refs = Element._get_redundant_source_refs()
if redundant_refs:
detail = "The following inline specified source references will be ignored:\n\n"
- lines = [
- "{}:{}".format(source._get_provenance(), ref)
- for source, ref in redundant_refs
- ]
+ lines = ["{}:{}".format(source._get_provenance(), ref) for source, ref in redundant_refs]
detail += "\n".join(lines)
self._context.messenger.message(
- Message(MessageType.WARN, "Ignoring redundant source references", detail=detail))
+ Message(MessageType.WARN, "Ignoring redundant source references", detail=detail)
+ )
return elements
@@ -590,49 +619,49 @@ class Project():
self._project_conf._composite(pre_config_node)
# Assert project's format version early, before validating toplevel keys
- format_version = pre_config_node.get_int('format-version')
+ format_version = pre_config_node.get_int("format-version")
if format_version < BST_FORMAT_VERSION_MIN:
major, minor = utils.get_bst_version()
raise LoadError(
"Project requested format version {}, but BuildStream {}.{} only supports format version {} or above."
- "Use latest 1.x release"
- .format(format_version, major, minor, BST_FORMAT_VERSION_MIN), LoadErrorReason.UNSUPPORTED_PROJECT)
+ "Use latest 1.x release".format(format_version, major, minor, BST_FORMAT_VERSION_MIN),
+ LoadErrorReason.UNSUPPORTED_PROJECT,
+ )
if BST_FORMAT_VERSION < format_version:
major, minor = utils.get_bst_version()
raise LoadError(
- "Project requested format version {}, but BuildStream {}.{} only supports up until format version {}"
- .format(format_version, major, minor, BST_FORMAT_VERSION), LoadErrorReason.UNSUPPORTED_PROJECT)
+ "Project requested format version {}, but BuildStream {}.{} only supports up until format version {}".format(
+ format_version, major, minor, BST_FORMAT_VERSION
+ ),
+ LoadErrorReason.UNSUPPORTED_PROJECT,
+ )
self._validate_node(pre_config_node)
# The project name, element path and option declarations
# are constant and cannot be overridden by option conditional statements
# FIXME: we should be keeping node information for further composition here
- self.name = self._project_conf.get_str('name')
+ self.name = self._project_conf.get_str("name")
# Validate that project name is a valid symbol name
- _assert_symbol_name(self.name, "project name",
- ref_node=pre_config_node.get_node('name'))
+ _assert_symbol_name(self.name, "project name", ref_node=pre_config_node.get_node("name"))
self.element_path = os.path.join(
- self.directory,
- self.get_path_from_node(pre_config_node.get_scalar('element-path'),
- check_is_dir=True)
+ self.directory, self.get_path_from_node(pre_config_node.get_scalar("element-path"), check_is_dir=True)
)
self.config.options = OptionPool(self.element_path)
self.first_pass_config.options = OptionPool(self.element_path)
- defaults = pre_config_node.get_mapping('defaults')
- defaults.validate_keys(['targets'])
+ defaults = pre_config_node.get_mapping("defaults")
+ defaults.validate_keys(["targets"])
self._default_targets = defaults.get_str_list("targets")
# Fatal warnings
- self._fatal_warnings = pre_config_node.get_str_list('fatal-warnings', default=[])
+ self._fatal_warnings = pre_config_node.get_str_list("fatal-warnings", default=[])
- self.loader = Loader(self._context, self,
- parent=parent_loader, fetch_subprojects=fetch_subprojects)
+ self.loader = Loader(self._context, self, parent=parent_loader, fetch_subprojects=fetch_subprojects)
self._project_includes = Includes(self.loader, copy_tree=False)
@@ -641,16 +670,17 @@ class Project():
config_no_include = self._default_config_node.clone()
project_conf_first_pass._composite(config_no_include)
- self._load_pass(config_no_include, self.first_pass_config,
- ignore_unknown=True)
+ self._load_pass(config_no_include, self.first_pass_config, ignore_unknown=True)
# Use separate file for storing source references
- ref_storage_node = pre_config_node.get_scalar('ref-storage')
+ ref_storage_node = pre_config_node.get_scalar("ref-storage")
self.ref_storage = ref_storage_node.as_str()
if self.ref_storage not in [ProjectRefStorage.INLINE, ProjectRefStorage.PROJECT_REFS]:
p = ref_storage_node.get_provenance()
- raise LoadError("{}: Invalid value '{}' specified for ref-storage"
- .format(p, self.ref_storage), LoadErrorReason.INVALID_DATA)
+ raise LoadError(
+ "{}: Invalid value '{}' specified for ref-storage".format(p, self.ref_storage),
+ LoadErrorReason.INVALID_DATA,
+ )
if self.ref_storage == ProjectRefStorage.PROJECT_REFS:
self.junction_refs.load(self.first_pass_config.options)
@@ -692,8 +722,7 @@ class Project():
# Load remote-execution configuration for this project
project_specs = SandboxRemote.specs_from_config_node(config, self.directory)
- override_specs = SandboxRemote.specs_from_config_node(
- self._context.get_overrides(self.name), self.directory)
+ override_specs = SandboxRemote.specs_from_config_node(self._context.get_overrides(self.name), self.directory)
if override_specs is not None:
self.remote_execution_specs = override_specs
@@ -703,25 +732,25 @@ class Project():
self.remote_execution_specs = self._context.remote_execution_specs
# Load sandbox environment variables
- self.base_environment = config.get_mapping('environment')
- self.base_env_nocache = config.get_str_list('environment-nocache')
+ self.base_environment = config.get_mapping("environment")
+ self.base_env_nocache = config.get_str_list("environment-nocache")
# Load sandbox configuration
- self._sandbox = config.get_mapping('sandbox')
+ self._sandbox = config.get_mapping("sandbox")
# Load project split rules
- self._splits = config.get_mapping('split-rules')
+ self._splits = config.get_mapping("split-rules")
# Support backwards compatibility for fail-on-overlap
- fail_on_overlap = config.get_scalar('fail-on-overlap', None)
+ fail_on_overlap = config.get_scalar("fail-on-overlap", None)
# Deprecation check
if not fail_on_overlap.is_none():
self._context.messenger.message(
Message(
MessageType.WARN,
- "Use of fail-on-overlap within project.conf " +
- "is deprecated. Consider using fatal-warnings instead."
+ "Use of fail-on-overlap within project.conf "
+ + "is deprecated. Consider using fatal-warnings instead.",
)
)
@@ -733,29 +762,29 @@ class Project():
self.refs.load(self.options)
# Parse shell options
- shell_options = config.get_mapping('shell')
- shell_options.validate_keys(['command', 'environment', 'host-files'])
- self._shell_command = shell_options.get_str_list('command')
+ shell_options = config.get_mapping("shell")
+ shell_options.validate_keys(["command", "environment", "host-files"])
+ self._shell_command = shell_options.get_str_list("command")
# Perform environment expansion right away
- shell_environment = shell_options.get_mapping('environment', default={})
+ shell_environment = shell_options.get_mapping("environment", default={})
for key in shell_environment.keys():
value = shell_environment.get_str(key)
self._shell_environment[key] = os.path.expandvars(value)
# Host files is parsed as a list for convenience
- host_files = shell_options.get_sequence('host-files', default=[])
+ host_files = shell_options.get_sequence("host-files", default=[])
for host_file in host_files:
if isinstance(host_file, ScalarNode):
mount = HostMount(host_file)
else:
# Some validation
- host_file.validate_keys(['path', 'host_path', 'optional'])
+ host_file.validate_keys(["path", "host_path", "optional"])
# Parse the host mount
- path = host_file.get_str('path')
- host_path = host_file.get_str('host_path', default=None)
- optional = host_file.get_bool('optional', default=False)
+ path = host_file.get_str("path")
+ host_path = host_file.get_str("host_path", default=None)
+ optional = host_file.get_bool("optional", default=False)
mount = HostMount(path, host_path, optional)
self._shell_host_files.append(mount)
@@ -770,22 +799,21 @@ class Project():
# output (ProjectConfig) - ProjectConfig to load configuration onto.
# ignore_unknown (bool) - Whether option loader shoud ignore unknown options.
#
- def _load_pass(self, config, output, *,
- ignore_unknown=False):
+ def _load_pass(self, config, output, *, ignore_unknown=False):
# Element and Source type configurations will be composited later onto
# element/source types, so we delete it from here and run our final
# assertion after.
- output.element_overrides = config.get_mapping('elements', default={})
- output.source_overrides = config.get_mapping('sources', default={})
- config.safe_del('elements')
- config.safe_del('sources')
+ output.element_overrides = config.get_mapping("elements", default={})
+ output.source_overrides = config.get_mapping("sources", default={})
+ config.safe_del("elements")
+ config.safe_del("sources")
config._assert_fully_composited()
self._load_plugin_factories(config, output)
# Load project options
- options_node = config.get_mapping('options', default={})
+ options_node = config.get_mapping("options", default={})
output.options.load(options_node)
if self.junction:
# load before user configuration
@@ -793,7 +821,7 @@ class Project():
# Collect option values specified in the user configuration
overrides = self._context.get_overrides(self.name)
- override_options = overrides.get_mapping('options', default={})
+ override_options = overrides.get_mapping("options", default={})
output.options.load_yaml_values(override_options)
if self._cli_options:
output.options.load_cli_values(self._cli_options, ignore_unknown=ignore_unknown)
@@ -812,10 +840,10 @@ class Project():
output.options.process_node(output.source_overrides)
# Load base variables
- output.base_variables = config.get_mapping('variables')
+ output.base_variables = config.get_mapping("variables")
# Add the project name as a default variable
- output.base_variables['project-name'] = self.name
+ output.base_variables["project-name"] = self.name
# Extend variables with automatic variables and option exports
# Initialize it as a string as all variables are processed as strings.
@@ -825,27 +853,24 @@ class Project():
if self._context.build_max_jobs == 0:
# User requested automatic max-jobs
platform = self._context.platform
- output.base_variables['max-jobs'] = str(platform.get_cpu_count(8))
+ output.base_variables["max-jobs"] = str(platform.get_cpu_count(8))
else:
# User requested explicit max-jobs setting
- output.base_variables['max-jobs'] = str(self._context.build_max_jobs)
+ output.base_variables["max-jobs"] = str(self._context.build_max_jobs)
# Export options into variables, if that was requested
output.options.export_variables(output.base_variables)
# Override default_mirror if not set by command-line
- output.default_mirror = self._default_mirror or overrides.get_str(
- 'default-mirror', default=None)
+ output.default_mirror = self._default_mirror or overrides.get_str("default-mirror", default=None)
- mirrors = config.get_sequence('mirrors', default=[])
+ mirrors = config.get_sequence("mirrors", default=[])
for mirror in mirrors:
- allowed_mirror_fields = [
- 'name', 'aliases'
- ]
+ allowed_mirror_fields = ["name", "aliases"]
mirror.validate_keys(allowed_mirror_fields)
- mirror_name = mirror.get_str('name')
+ mirror_name = mirror.get_str("name")
alias_mappings = {}
- for alias_mapping, uris in mirror.get_mapping('aliases').items():
+ for alias_mapping, uris in mirror.get_mapping("aliases").items():
assert type(uris) is SequenceNode # pylint: disable=unidiomatic-typecheck
alias_mappings[alias_mapping] = uris.as_str_list()
output.mirrors[mirror_name] = alias_mappings
@@ -853,7 +878,7 @@ class Project():
output.default_mirror = mirror_name
# Source url aliases
- output._aliases = config.get_mapping('aliases', default={})
+ output._aliases = config.get_mapping("aliases", default={})
# _find_project_dir()
#
@@ -873,9 +898,7 @@ class Project():
def _find_project_dir(self, directory):
workspace_element = None
config_filenames = [_PROJECT_CONF_FILE, WORKSPACE_PROJECT_FILE]
- found_directory, filename = utils._search_upward_for_files(
- directory, config_filenames
- )
+ found_directory, filename = utils._search_upward_for_files(directory, config_filenames)
if filename == _PROJECT_CONF_FILE:
project_directory = found_directory
elif filename == WORKSPACE_PROJECT_FILE:
@@ -885,57 +908,62 @@ class Project():
project_directory = workspace_project.get_default_project_path()
workspace_element = workspace_project.get_default_element()
else:
- raise LoadError("None of {names} found in '{path}' or any of its parent directories"
- .format(names=config_filenames, path=directory), LoadErrorReason.MISSING_PROJECT_CONF)
+ raise LoadError(
+ "None of {names} found in '{path}' or any of its parent directories".format(
+ names=config_filenames, path=directory
+ ),
+ LoadErrorReason.MISSING_PROJECT_CONF,
+ )
return project_directory, workspace_element
def _load_plugin_factories(self, config, output):
- plugin_source_origins = [] # Origins of custom sources
+ plugin_source_origins = [] # Origins of custom sources
plugin_element_origins = [] # Origins of custom elements
# Plugin origins and versions
- origins = config.get_sequence('plugins', default=[])
+ origins = config.get_sequence("plugins", default=[])
source_format_versions = {}
element_format_versions = {}
for origin in origins:
allowed_origin_fields = [
- 'origin', 'sources', 'elements',
- 'package-name', 'path',
+ "origin",
+ "sources",
+ "elements",
+ "package-name",
+ "path",
]
origin.validate_keys(allowed_origin_fields)
# Store source versions for checking later
- source_versions = origin.get_mapping('sources', default={})
+ source_versions = origin.get_mapping("sources", default={})
for key in source_versions.keys():
if key in source_format_versions:
- raise LoadError("Duplicate listing of source '{}'".format(key),
- LoadErrorReason.INVALID_YAML)
+ raise LoadError("Duplicate listing of source '{}'".format(key), LoadErrorReason.INVALID_YAML)
source_format_versions[key] = source_versions.get_int(key)
# Store element versions for checking later
- element_versions = origin.get_mapping('elements', default={})
+ element_versions = origin.get_mapping("elements", default={})
for key in element_versions.keys():
if key in element_format_versions:
- raise LoadError("Duplicate listing of element '{}'".format(key),
- LoadErrorReason.INVALID_YAML)
+ raise LoadError("Duplicate listing of element '{}'".format(key), LoadErrorReason.INVALID_YAML)
element_format_versions[key] = element_versions.get_int(key)
# Store the origins if they're not 'core'.
# core elements are loaded by default, so storing is unnecessary.
- origin_value = origin.get_enum('origin', PluginOrigins)
+ origin_value = origin.get_enum("origin", PluginOrigins)
if origin_value != PluginOrigins.CORE:
- self._store_origin(origin, 'sources', plugin_source_origins)
- self._store_origin(origin, 'elements', plugin_element_origins)
+ self._store_origin(origin, "sources", plugin_source_origins)
+ self._store_origin(origin, "elements", plugin_element_origins)
- pluginbase = PluginBase(package='buildstream.plugins')
- output.element_factory = ElementFactory(pluginbase,
- plugin_origins=plugin_element_origins,
- format_versions=element_format_versions)
- output.source_factory = SourceFactory(pluginbase,
- plugin_origins=plugin_source_origins,
- format_versions=source_format_versions)
+ pluginbase = PluginBase(package="buildstream.plugins")
+ output.element_factory = ElementFactory(
+ pluginbase, plugin_origins=plugin_element_origins, format_versions=element_format_versions
+ )
+ output.source_factory = SourceFactory(
+ pluginbase, plugin_origins=plugin_source_origins, format_versions=source_format_versions
+ )
# _store_origin()
#
@@ -951,25 +979,25 @@ class Project():
# Raises:
# LoadError if 'origin' is an unexpected value
def _store_origin(self, origin, plugin_group, destination):
- expected_groups = ['sources', 'elements']
+ expected_groups = ["sources", "elements"]
if plugin_group not in expected_groups:
- raise LoadError("Unexpected plugin group: {}, expecting {}"
- .format(plugin_group, expected_groups),
- LoadErrorReason.INVALID_DATA)
+ raise LoadError(
+ "Unexpected plugin group: {}, expecting {}".format(plugin_group, expected_groups),
+ LoadErrorReason.INVALID_DATA,
+ )
if plugin_group in origin.keys():
origin_node = origin.clone()
plugins = origin.get_mapping(plugin_group, default={})
- origin_node['plugins'] = plugins.keys()
+ origin_node["plugins"] = plugins.keys()
for group in expected_groups:
if group in origin_node:
del origin_node[group]
- if origin_node.get_enum('origin', PluginOrigins) == PluginOrigins.LOCAL:
- path = self.get_path_from_node(origin.get_scalar('path'),
- check_is_dir=True)
+ if origin_node.get_enum("origin", PluginOrigins) == PluginOrigins.LOCAL:
+ path = self.get_path_from_node(origin.get_scalar("path"), check_is_dir=True)
# paths are passed in relative to the project, but must be absolute
- origin_node['path'] = os.path.join(self.directory, path)
+ origin_node["path"] = os.path.join(self.directory, path)
destination.append(origin_node)
# _warning_is_fatal():
diff --git a/src/buildstream/_projectrefs.py b/src/buildstream/_projectrefs.py
index 0555488c8..aca7c6712 100644
--- a/src/buildstream/_projectrefs.py
+++ b/src/buildstream/_projectrefs.py
@@ -26,15 +26,15 @@ from ._exceptions import LoadError, LoadErrorReason
# ProjectRefStorage()
#
# Indicates the type of ref storage
-class ProjectRefStorage():
+class ProjectRefStorage:
# Source references are stored inline
#
- INLINE = 'inline'
+ INLINE = "inline"
# Source references are stored in a central project.refs file
#
- PROJECT_REFS = 'project.refs'
+ PROJECT_REFS = "project.refs"
# ProjectRefs()
@@ -45,8 +45,7 @@ class ProjectRefStorage():
# directory (str): The project directory
# base_name (str): The project.refs basename
#
-class ProjectRefs():
-
+class ProjectRefs:
def __init__(self, directory, base_name):
directory = os.path.abspath(directory)
self._fullpath = os.path.join(directory, base_name)
@@ -83,12 +82,12 @@ class ProjectRefs():
self._toplevel_node = _new_synthetic_file(self._fullpath)
self._toplevel_save = self._toplevel_node
- self._toplevel_node.validate_keys(['projects'])
+ self._toplevel_node.validate_keys(["projects"])
# Ensure we create our toplevel entry point on the fly here
for node in [self._toplevel_node, self._toplevel_save]:
- if 'projects' not in node:
- node['projects'] = {}
+ if "projects" not in node:
+ node["projects"] = {}
# lookup_ref()
#
@@ -122,7 +121,7 @@ class ProjectRefs():
# Looks up a ref node in the project.refs file, creates one if ensure is True.
#
def _lookup(self, toplevel, project, element, source_index, *, ensure=False):
- projects = toplevel.get_mapping('projects')
+ projects = toplevel.get_mapping("projects")
# Fetch the project
try:
diff --git a/src/buildstream/_remote.py b/src/buildstream/_remote.py
index ab1dc1924..78f67726a 100644
--- a/src/buildstream/_remote.py
+++ b/src/buildstream/_remote.py
@@ -35,14 +35,14 @@ class RemoteType(FastEnum):
ALL = "all"
def __str__(self):
- return self.name.lower().replace('_', '-')
+ return self.name.lower().replace("_", "-")
# RemoteSpec():
#
# Defines the basic structure of a remote specification.
#
-class RemoteSpec(namedtuple('RemoteSpec', 'url push server_cert client_key client_cert instance_name type')):
+class RemoteSpec(namedtuple("RemoteSpec", "url push server_cert client_key client_cert instance_name type")):
# new_from_config_node
#
@@ -60,15 +60,15 @@ class RemoteSpec(namedtuple('RemoteSpec', 'url push server_cert client_key clien
#
@classmethod
def new_from_config_node(cls, spec_node, basedir=None):
- spec_node.validate_keys(['url', 'push', 'server-cert', 'client-key', 'client-cert', 'instance-name', 'type'])
+ spec_node.validate_keys(["url", "push", "server-cert", "client-key", "client-cert", "instance-name", "type"])
- url = spec_node.get_str('url')
+ url = spec_node.get_str("url")
if not url:
- provenance = spec_node.get_node('url').get_provenance()
+ provenance = spec_node.get_node("url").get_provenance()
raise LoadError("{}: empty artifact cache URL".format(provenance), LoadErrorReason.INVALID_DATA)
- push = spec_node.get_bool('push', default=False)
- instance_name = spec_node.get_str('instance-name', default=None)
+ push = spec_node.get_bool("push", default=False)
+ instance_name = spec_node.get_str("instance-name", default=None)
def parse_cert(key):
cert = spec_node.get_str(key, default=None)
@@ -80,20 +80,22 @@ class RemoteSpec(namedtuple('RemoteSpec', 'url push server_cert client_key clien
return cert
- cert_keys = ('server-cert', 'client-key', 'client-cert')
+ cert_keys = ("server-cert", "client-key", "client-cert")
server_cert, client_key, client_cert = tuple(parse_cert(key) for key in cert_keys)
if client_key and not client_cert:
- provenance = spec_node.get_node('client-key').get_provenance()
- raise LoadError("{}: 'client-key' was specified without 'client-cert'".format(provenance),
- LoadErrorReason.INVALID_DATA)
+ provenance = spec_node.get_node("client-key").get_provenance()
+ raise LoadError(
+ "{}: 'client-key' was specified without 'client-cert'".format(provenance), LoadErrorReason.INVALID_DATA
+ )
if client_cert and not client_key:
- provenance = spec_node.get_node('client-cert').get_provenance()
- raise LoadError("{}: 'client-cert' was specified without 'client-key'".format(provenance),
- LoadErrorReason.INVALID_DATA)
+ provenance = spec_node.get_node("client-cert").get_provenance()
+ raise LoadError(
+ "{}: 'client-cert' was specified without 'client-key'".format(provenance), LoadErrorReason.INVALID_DATA
+ )
- type_ = spec_node.get_enum('type', RemoteType, default=RemoteType.ALL)
+ type_ = spec_node.get_enum("type", RemoteType, default=RemoteType.ALL)
return cls(url, push, server_cert, client_key, client_cert, instance_name, type_)
@@ -108,11 +110,11 @@ class RemoteSpec(namedtuple('RemoteSpec', 'url push server_cert client_key clien
RemoteSpec.__new__.__defaults__ = ( # type: ignore
# mandatory # url - The url of the remote
# mandatory # push - Whether the remote should be used for pushing
- None, # server_cert - The server certificate
- None, # client_key - The (private) client key
- None, # client_cert - The (public) client certificate
- None, # instance_name - The (grpc) instance name of the remote
- RemoteType.ALL # type - The type of the remote (index, storage, both)
+ None, # server_cert - The server certificate
+ None, # client_key - The (private) client key
+ None, # client_cert - The (public) client certificate
+ None, # instance_name - The (grpc) instance name of the remote
+ RemoteType.ALL, # type - The type of the remote (index, storage, both)
)
@@ -126,7 +128,7 @@ RemoteSpec.__new__.__defaults__ = ( # type: ignore
# Customization for the particular protocol is expected to be
# performed in children.
#
-class BaseRemote():
+class BaseRemote:
key_name = None
def __init__(self, spec):
@@ -154,25 +156,24 @@ class BaseRemote():
# Set up the communcation channel
url = urlparse(self.spec.url)
- if url.scheme == 'http':
+ if url.scheme == "http":
port = url.port or 80
- self.channel = grpc.insecure_channel('{}:{}'.format(url.hostname, port))
- elif url.scheme == 'https':
+ self.channel = grpc.insecure_channel("{}:{}".format(url.hostname, port))
+ elif url.scheme == "https":
port = url.port or 443
try:
server_cert, client_key, client_cert = _read_files(
- self.spec.server_cert,
- self.spec.client_key,
- self.spec.client_cert)
+ self.spec.server_cert, self.spec.client_key, self.spec.client_cert
+ )
except FileNotFoundError as e:
raise RemoteError("Could not read certificates: {}".format(e)) from e
self.server_cert = server_cert
self.client_key = client_key
self.client_cert = client_cert
- credentials = grpc.ssl_channel_credentials(root_certificates=self.server_cert,
- private_key=self.client_key,
- certificate_chain=self.client_cert)
- self.channel = grpc.secure_channel('{}:{}'.format(url.hostname, port), credentials)
+ credentials = grpc.ssl_channel_credentials(
+ root_certificates=self.server_cert, private_key=self.client_key, certificate_chain=self.client_cert
+ )
+ self.channel = grpc.secure_channel("{}:{}".format(url.hostname, port), credentials)
else:
raise RemoteError("Unsupported URL: {}".format(self.spec.url))
@@ -258,7 +259,8 @@ class BaseRemote():
def _read_files(*files):
def read_file(f):
if f:
- with open(f, 'rb') as data:
+ with open(f, "rb") as data:
return data.read()
return None
+
return (read_file(f) for f in files)
diff --git a/src/buildstream/_scheduler/jobs/elementjob.py b/src/buildstream/_scheduler/jobs/elementjob.py
index 246eb75c6..6e035be9c 100644
--- a/src/buildstream/_scheduler/jobs/elementjob.py
+++ b/src/buildstream/_scheduler/jobs/elementjob.py
@@ -69,9 +69,9 @@ class ElementJob(Job):
super().__init__(*args, **kwargs)
self.set_name(element._get_full_name())
self.queue = queue
- self._element = element # Set the Element pertaining to the job
- self._action_cb = action_cb # The action callable function
- self._complete_cb = complete_cb # The complete callable function
+ self._element = element # Set the Element pertaining to the job
+ self._action_cb = action_cb # The action callable function
+ self._complete_cb = complete_cb # The complete callable function
# Set the plugin element name & key for logging purposes
self.set_message_element_name(self.name)
@@ -97,9 +97,7 @@ class ChildElementJob(ChildJob):
# This should probably be omitted for non-build tasks but it's harmless here
elt_env = self._element.get_environment()
env_dump = yaml.round_trip_dump(elt_env, default_flow_style=False, allow_unicode=True)
- self.message(MessageType.LOG,
- "Build environment for element {}".format(self._element.name),
- detail=env_dump)
+ self.message(MessageType.LOG, "Build environment for element {}".format(self._element.name), detail=env_dump)
# Run the action
return self._action_cb(self._element)
@@ -109,6 +107,6 @@ class ChildElementJob(ChildJob):
workspace = self._element._get_workspace()
if workspace is not None:
- data['workspace'] = workspace.to_dict()
+ data["workspace"] = workspace.to_dict()
return data
diff --git a/src/buildstream/_scheduler/jobs/job.py b/src/buildstream/_scheduler/jobs/job.py
index 3363d7b60..e7866bcd4 100644
--- a/src/buildstream/_scheduler/jobs/job.py
+++ b/src/buildstream/_scheduler/jobs/job.py
@@ -66,7 +66,7 @@ class JobStatus(FastEnum):
# Used to distinguish between status messages and return values
-class _Envelope():
+class _Envelope:
def __init__(self, message_type, message):
self.message_type = message_type
self.message = message
@@ -115,35 +115,34 @@ class _MessageType(FastEnum):
# that should be used - should contain {pid}.
# max_retries (int): The maximum number of retries
#
-class Job():
-
+class Job:
def __init__(self, scheduler, action_name, logfile, *, max_retries=0):
#
# Public members
#
- self.name = None # The name of the job, set by the job's subclass
- self.action_name = action_name # The action name for the Queue
- self.child_data = None # Data to be sent to the main process
+ self.name = None # The name of the job, set by the job's subclass
+ self.action_name = action_name # The action name for the Queue
+ self.child_data = None # Data to be sent to the main process
#
# Private members
#
- self._scheduler = scheduler # The scheduler
- self._queue = None # A message passing queue
- self._process = None # The Process object
- self._watcher = None # Child process watcher
- self._listening = False # Whether the parent is currently listening
- self._suspended = False # Whether this job is currently suspended
- self._max_retries = max_retries # Maximum number of automatic retries
- self._result = None # Return value of child action in the parent
- self._tries = 0 # Try count, for retryable jobs
- self._terminated = False # Whether this job has been explicitly terminated
+ self._scheduler = scheduler # The scheduler
+ self._queue = None # A message passing queue
+ self._process = None # The Process object
+ self._watcher = None # Child process watcher
+ self._listening = False # Whether the parent is currently listening
+ self._suspended = False # Whether this job is currently suspended
+ self._max_retries = max_retries # Maximum number of automatic retries
+ self._result = None # Return value of child action in the parent
+ self._tries = 0 # Try count, for retryable jobs
+ self._terminated = False # Whether this job has been explicitly terminated
self._logfile = logfile
- self._message_element_name = None # The plugin instance element name for messaging
- self._message_element_key = None # The element key for messaging
- self._element = None # The Element() passed to the Job() constructor, if applicable
+ self._message_element_name = None # The plugin instance element name for messaging
+ self._message_element_key = None # The element key for messaging
+ self._element = None # The Element() passed to the Job() constructor, if applicable
# set_name()
#
@@ -170,23 +169,16 @@ class Job():
self._max_retries,
self._tries,
self._message_element_name,
- self._message_element_key
+ self._message_element_key,
)
if self._scheduler.context.platform.does_multiprocessing_start_require_pickling():
- pickled = pickle_child_job(
- child_job,
- self._scheduler.context.get_projects(),
- )
+ pickled = pickle_child_job(child_job, self._scheduler.context.get_projects(),)
self._process = _multiprocessing.AsyncioSafeProcess(
- target=do_pickled_child_job,
- args=[pickled, self._queue],
+ target=do_pickled_child_job, args=[pickled, self._queue],
)
else:
- self._process = _multiprocessing.AsyncioSafeProcess(
- target=child_job.child_action,
- args=[self._queue],
- )
+ self._process = _multiprocessing.AsyncioSafeProcess(target=child_job.child_action, args=[self._queue],)
# Block signals which are handled in the main process such that
# the child process does not inherit the parent's state, but the main
@@ -257,8 +249,7 @@ class Job():
#
def kill(self):
# Force kill
- self.message(MessageType.WARN,
- "{} did not terminate gracefully, killing".format(self.action_name))
+ self.message(MessageType.WARN, "{} did not terminate gracefully, killing".format(self.action_name))
utils._kill_process_tree(self._process.pid)
# suspend()
@@ -267,8 +258,7 @@ class Job():
#
def suspend(self):
if not self._suspended:
- self.message(MessageType.STATUS,
- "{} suspending".format(self.action_name))
+ self.message(MessageType.STATUS, "{} suspending".format(self.action_name))
try:
# Use SIGTSTP so that child processes may handle and propagate
@@ -292,8 +282,7 @@ class Job():
def resume(self, silent=False):
if self._suspended:
if not silent and not self._scheduler.terminated:
- self.message(MessageType.STATUS,
- "{} resuming".format(self.action_name))
+ self.message(MessageType.STATUS, "{} resuming".format(self.action_name))
os.kill(self._process.pid, signal.SIGCONT)
self._suspended = False
@@ -335,7 +324,7 @@ class Job():
# override 'element_name' and 'element_key' this way.
#
def message(self, message_type, message, element_name=None, element_key=None, **kwargs):
- kwargs['scheduler'] = True
+ kwargs["scheduler"] = True
# If default name & key values not provided, set as given job attributes
if element_name is None:
element_name = self._message_element_name
@@ -373,8 +362,7 @@ class Job():
# lists, dicts, numbers, but not Element instances).
#
def handle_message(self, message):
- raise ImplError("Job '{kind}' does not implement handle_message()"
- .format(kind=type(self).__name__))
+ raise ImplError("Job '{kind}' does not implement handle_message()".format(kind=type(self).__name__))
# parent_complete()
#
@@ -386,8 +374,7 @@ class Job():
# result (any): The result returned by child_process().
#
def parent_complete(self, status, result):
- raise ImplError("Job '{kind}' does not implement parent_complete()"
- .format(kind=type(self).__name__))
+ raise ImplError("Job '{kind}' does not implement parent_complete()".format(kind=type(self).__name__))
# create_child_job()
#
@@ -405,8 +392,7 @@ class Job():
# (ChildJob): An instance of a subclass of ChildJob.
#
def create_child_job(self, *args, **kwargs):
- raise ImplError("Job '{kind}' does not implement create_child_job()"
- .format(kind=type(self).__name__))
+ raise ImplError("Job '{kind}' does not implement create_child_job()".format(kind=type(self).__name__))
#######################################################
# Local Private Methods #
@@ -437,9 +423,11 @@ class Job():
returncode = _ReturnCode(returncode)
except ValueError:
# An unexpected return code was returned; fail permanently and report
- self.message(MessageType.ERROR,
- "Internal job process unexpectedly died with exit code {}".format(returncode),
- logfile=self._logfile)
+ self.message(
+ MessageType.ERROR,
+ "Internal job process unexpectedly died with exit code {}".format(returncode),
+ logfile=self._logfile,
+ )
returncode = _ReturnCode.PERM_FAIL
# We don't want to retry if we got OK or a permanent fail.
@@ -503,8 +491,7 @@ class Job():
# For regression tests only, save the last error domain / reason
# reported from a child task in the main process, this global state
# is currently managed in _exceptions.py
- set_last_task_error(envelope.message['domain'],
- envelope.message['reason'])
+ set_last_task_error(envelope.message["domain"], envelope.message["reason"])
elif envelope.message_type is _MessageType.RESULT:
assert self._result is None
self._result = envelope.message
@@ -514,8 +501,7 @@ class Job():
elif envelope.message_type is _MessageType.SUBCLASS_CUSTOM_MESSAGE:
self.handle_message(envelope.message)
else:
- assert False, "Unhandled message type '{}': {}".format(
- envelope.message_type, envelope.message)
+ assert False, "Unhandled message type '{}': {}".format(envelope.message_type, envelope.message)
# _parent_process_queue()
#
@@ -552,8 +538,7 @@ class Job():
# http://bugs.python.org/issue3831
#
if not self._listening:
- self._scheduler.loop.add_reader(
- self._queue._reader.fileno(), self._parent_recv)
+ self._scheduler.loop.add_reader(self._queue._reader.fileno(), self._parent_recv)
self._listening = True
# _parent_stop_listening()
@@ -589,11 +574,10 @@ class Job():
# message_element_key (tuple): None, or the element display key tuple
# to be supplied to the Message() constructor.
#
-class ChildJob():
-
+class ChildJob:
def __init__(
- self, action_name, messenger, logdir, logfile, max_retries, tries,
- message_element_name, message_element_key):
+ self, action_name, messenger, logdir, logfile, max_retries, tries, message_element_name, message_element_key
+ ):
self.action_name = action_name
@@ -624,14 +608,15 @@ class ChildJob():
# overriden here.
#
def message(self, message_type, message, element_name=None, element_key=None, **kwargs):
- kwargs['scheduler'] = True
+ kwargs["scheduler"] = True
# If default name & key values not provided, set as given job attributes
if element_name is None:
element_name = self._message_element_name
if element_key is None:
element_key = self._message_element_key
- self._messenger.message(Message(message_type, message, element_name=element_name,
- element_key=element_key, **kwargs))
+ self._messenger.message(
+ Message(message_type, message, element_name=element_name, element_key=element_key, **kwargs)
+ )
# send_message()
#
@@ -668,8 +653,7 @@ class ChildJob():
# the result of the Job.
#
def child_process(self):
- raise ImplError("ChildJob '{kind}' does not implement child_process()"
- .format(kind=type(self).__name__))
+ raise ImplError("ChildJob '{kind}' does not implement child_process()".format(kind=type(self).__name__))
# child_process_data()
#
@@ -723,12 +707,13 @@ class ChildJob():
def resume_time():
nonlocal stopped_time
nonlocal starttime
- starttime += (datetime.datetime.now() - stopped_time)
+ starttime += datetime.datetime.now() - stopped_time
# Time, log and and run the action function
#
- with _signals.suspendable(stop_time, resume_time), \
- self._messenger.recorded_messages(self._logfile, self._logdir) as filename:
+ with _signals.suspendable(stop_time, resume_time), self._messenger.recorded_messages(
+ self._logfile, self._logdir
+ ) as filename:
# Graciously handle sigterms.
def handle_sigterm(_signum, _sigframe):
@@ -743,8 +728,7 @@ class ChildJob():
result = self.child_process() # pylint: disable=assignment-from-no-return
except SkipJob as e:
elapsed = datetime.datetime.now() - starttime
- self.message(MessageType.SKIPPED, str(e),
- elapsed=elapsed, logfile=filename)
+ self.message(MessageType.SKIPPED, str(e), elapsed=elapsed, logfile=filename)
# Alert parent of skip by return code
self._child_shutdown(_ReturnCode.SKIPPED)
@@ -753,13 +737,16 @@ class ChildJob():
retry_flag = e.temporary
if retry_flag and (self._tries <= self._max_retries):
- self.message(MessageType.FAIL,
- "Try #{} failed, retrying".format(self._tries),
- elapsed=elapsed, logfile=filename)
+ self.message(
+ MessageType.FAIL,
+ "Try #{} failed, retrying".format(self._tries),
+ elapsed=elapsed,
+ logfile=filename,
+ )
else:
- self.message(MessageType.FAIL, str(e),
- elapsed=elapsed, detail=e.detail,
- logfile=filename, sandbox=e.sandbox)
+ self.message(
+ MessageType.FAIL, str(e), elapsed=elapsed, detail=e.detail, logfile=filename, sandbox=e.sandbox
+ )
self._send_message(_MessageType.CHILD_DATA, self.child_process_data())
@@ -770,7 +757,7 @@ class ChildJob():
#
self._child_shutdown(_ReturnCode.FAIL if retry_flag else _ReturnCode.PERM_FAIL)
- except Exception: # pylint: disable=broad-except
+ except Exception: # pylint: disable=broad-except
# If an unhandled (not normalized to BstError) occurs, that's a bug,
# send the traceback and formatted exception back to the frontend
@@ -779,9 +766,7 @@ class ChildJob():
elapsed = datetime.datetime.now() - starttime
detail = "An unhandled exception occured:\n\n{}".format(traceback.format_exc())
- self.message(MessageType.BUG, self.action_name,
- elapsed=elapsed, detail=detail,
- logfile=filename)
+ self.message(MessageType.BUG, self.action_name, elapsed=elapsed, detail=detail, logfile=filename)
# Unhandled exceptions should permenantly fail
self._child_shutdown(_ReturnCode.PERM_FAIL)
@@ -791,8 +776,7 @@ class ChildJob():
self._child_send_result(result)
elapsed = datetime.datetime.now() - starttime
- self.message(MessageType.SUCCESS, self.action_name, elapsed=elapsed,
- logfile=filename)
+ self.message(MessageType.SUCCESS, self.action_name, elapsed=elapsed, logfile=filename)
# Shutdown needs to stay outside of the above context manager,
# make sure we dont try to handle SIGTERM while the process
@@ -831,10 +815,7 @@ class ChildJob():
domain = e.domain
reason = e.reason
- self._send_message(_MessageType.ERROR, {
- 'domain': domain,
- 'reason': reason
- })
+ self._send_message(_MessageType.ERROR, {"domain": domain, "reason": reason})
# _child_send_result()
#
diff --git a/src/buildstream/_scheduler/jobs/jobpickler.py b/src/buildstream/_scheduler/jobs/jobpickler.py
index b0465ec9e..1d47f67db 100644
--- a/src/buildstream/_scheduler/jobs/jobpickler.py
+++ b/src/buildstream/_scheduler/jobs/jobpickler.py
@@ -37,9 +37,7 @@ _NAME_TO_PROTO_CLASS = {
"digest": DigestProto,
}
-_PROTO_CLASS_TO_NAME = {
- cls: name for name, cls in _NAME_TO_PROTO_CLASS.items()
-}
+_PROTO_CLASS_TO_NAME = {cls: name for name, cls in _NAME_TO_PROTO_CLASS.items()}
# pickle_child_job()
@@ -57,10 +55,7 @@ def pickle_child_job(child_job, projects):
# necessary for the job, this includes e.g. the global state of the node
# module.
node_module_state = node._get_state_for_pickling()
- return _pickle_child_job_data(
- (child_job, node_module_state),
- projects,
- )
+ return _pickle_child_job_data((child_job, node_module_state), projects,)
# do_pickled_child_job()
@@ -146,10 +141,7 @@ def _pickle_child_job_data(child_job_data, projects):
]
plugin_class_to_factory = {
- cls: factory
- for factory in factory_list
- if factory is not None
- for cls, _ in factory.all_loaded_plugins()
+ cls: factory for factory in factory_list if factory is not None for cls, _ in factory.all_loaded_plugins()
}
pickled_data = io.BytesIO()
diff --git a/src/buildstream/_scheduler/queues/buildqueue.py b/src/buildstream/_scheduler/queues/buildqueue.py
index dc33e6510..d98b49476 100644
--- a/src/buildstream/_scheduler/queues/buildqueue.py
+++ b/src/buildstream/_scheduler/queues/buildqueue.py
@@ -50,10 +50,15 @@ class BuildQueue(Queue):
self._tried.add(element)
_, description, detail = element._get_build_result()
logfile = element._get_build_log()
- self._message(element, MessageType.FAIL, description,
- detail=detail, action_name=self.action_name,
- elapsed=timedelta(seconds=0),
- logfile=logfile)
+ self._message(
+ element,
+ MessageType.FAIL,
+ description,
+ detail=detail,
+ action_name=self.action_name,
+ elapsed=timedelta(seconds=0),
+ logfile=logfile,
+ )
self._done_queue.append(element)
element_name = element._get_full_name()
self._task_group.add_failed_task(element_name)
diff --git a/src/buildstream/_scheduler/queues/queue.py b/src/buildstream/_scheduler/queues/queue.py
index 49fae5677..986ac6c0a 100644
--- a/src/buildstream/_scheduler/queues/queue.py
+++ b/src/buildstream/_scheduler/queues/queue.py
@@ -57,11 +57,11 @@ class QueueStatus(FastEnum):
# Args:
# scheduler (Scheduler): The Scheduler
#
-class Queue():
+class Queue:
# These should be overridden on class data of of concrete Queue implementations
- action_name = None # type: Optional[str]
- complete_name = None # type: Optional[str]
+ action_name = None # type: Optional[str]
+ complete_name = None # type: Optional[str]
# Resources this queues' jobs want
resources = [] # type: List[int]
@@ -72,11 +72,11 @@ class Queue():
#
self._scheduler = scheduler
self._resources = scheduler.resources # Shared resource pool
- self._ready_queue = [] # Ready elements
- self._done_queue = deque() # Processed / Skipped elements
+ self._ready_queue = [] # Ready elements
+ self._done_queue = deque() # Processed / Skipped elements
self._max_retries = 0
- self._required_element_check = False # Whether we should check that elements are required before enqueuing
+ self._required_element_check = False # Whether we should check that elements are required before enqueuing
# Assert the subclass has setup class data
assert self.action_name is not None
@@ -162,8 +162,7 @@ class Queue():
# element (Element): The element waiting to be pushed into the queue
#
def register_pending_element(self, element):
- raise ImplError("Queue type: {} does not implement register_pending_element()"
- .format(self.action_name))
+ raise ImplError("Queue type: {} does not implement register_pending_element()".format(self.action_name))
#####################################################
# Scheduler / Pipeline facing APIs #
@@ -229,12 +228,16 @@ class Queue():
ready.append(element)
return [
- ElementJob(self._scheduler, self.action_name,
- self._element_log_path(element),
- element=element, queue=self,
- action_cb=self.get_process_func(),
- complete_cb=self._job_done,
- max_retries=self._max_retries)
+ ElementJob(
+ self._scheduler,
+ self.action_name,
+ self._element_log_path(element),
+ element=element,
+ queue=self,
+ action_cb=self.get_process_func(),
+ complete_cb=self._job_done,
+ max_retries=self._max_retries,
+ )
for element in ready
]
@@ -267,7 +270,7 @@ class Queue():
def _update_workspaces(self, element, job):
workspace_dict = None
if job.child_data:
- workspace_dict = job.child_data.get('workspace', None)
+ workspace_dict = job.child_data.get("workspace", None)
# Handle any workspace modifications now
#
@@ -279,10 +282,13 @@ class Queue():
workspaces.save_config()
except BstError as e:
self._message(element, MessageType.ERROR, "Error saving workspaces", detail=str(e))
- except Exception: # pylint: disable=broad-except
- self._message(element, MessageType.BUG,
- "Unhandled exception while saving workspaces",
- detail=traceback.format_exc())
+ except Exception: # pylint: disable=broad-except
+ self._message(
+ element,
+ MessageType.BUG,
+ "Unhandled exception while saving workspaces",
+ detail=traceback.format_exc(),
+ )
# _job_done()
#
@@ -322,13 +328,13 @@ class Queue():
#
set_last_task_error(e.domain, e.reason)
- except Exception: # pylint: disable=broad-except
+ except Exception: # pylint: disable=broad-except
# Report unhandled exceptions and mark as failed
#
- self._message(element, MessageType.BUG,
- "Unhandled exception in post processing",
- detail=traceback.format_exc())
+ self._message(
+ element, MessageType.BUG, "Unhandled exception in post processing", detail=traceback.format_exc()
+ )
self._task_group.add_failed_task(element._get_full_name())
else:
# All elements get placed on the done queue for later processing.
@@ -372,7 +378,7 @@ class Queue():
if status == QueueStatus.SKIP:
# Place skipped elements into the done queue immediately
self._task_group.add_skipped_task()
- self._done_queue.append(element) # Elements to proceed to the next queue
+ self._done_queue.append(element) # Elements to proceed to the next queue
elif status == QueueStatus.READY:
# Push elements which are ready to be processed immediately into the queue
heapq.heappush(self._ready_queue, (element._depth, element))
diff --git a/src/buildstream/_scheduler/resources.py b/src/buildstream/_scheduler/resources.py
index 73bf66b4a..e76158779 100644
--- a/src/buildstream/_scheduler/resources.py
+++ b/src/buildstream/_scheduler/resources.py
@@ -1,17 +1,17 @@
-class ResourceType():
+class ResourceType:
CACHE = 0
DOWNLOAD = 1
PROCESS = 2
UPLOAD = 3
-class Resources():
+class Resources:
def __init__(self, num_builders, num_fetchers, num_pushers):
self._max_resources = {
ResourceType.CACHE: 0,
ResourceType.DOWNLOAD: num_fetchers,
ResourceType.PROCESS: num_builders,
- ResourceType.UPLOAD: num_pushers
+ ResourceType.UPLOAD: num_pushers,
}
# Resources jobs are currently using.
@@ -19,7 +19,7 @@ class Resources():
ResourceType.CACHE: 0,
ResourceType.DOWNLOAD: 0,
ResourceType.PROCESS: 0,
- ResourceType.UPLOAD: 0
+ ResourceType.UPLOAD: 0,
}
# Resources jobs currently want exclusive access to. The set
@@ -31,7 +31,7 @@ class Resources():
ResourceType.CACHE: set(),
ResourceType.DOWNLOAD: set(),
ResourceType.PROCESS: set(),
- ResourceType.UPLOAD: set()
+ ResourceType.UPLOAD: set(),
}
# reserve()
@@ -90,8 +90,7 @@ class Resources():
# available. If we don't have enough, the job cannot be
# scheduled.
for resource in resources:
- if (self._max_resources[resource] > 0 and
- self._used_resources[resource] >= self._max_resources[resource]):
+ if self._max_resources[resource] > 0 and self._used_resources[resource] >= self._max_resources[resource]:
return False
# Now we register the fact that our job is using the resources
diff --git a/src/buildstream/_scheduler/scheduler.py b/src/buildstream/_scheduler/scheduler.py
index 86e3af021..171281bd9 100644
--- a/src/buildstream/_scheduler/scheduler.py
+++ b/src/buildstream/_scheduler/scheduler.py
@@ -76,17 +76,18 @@ class NotificationType(FastEnum):
# required. NOTE: The notification object should be lightweight
# and all attributes must be picklable.
#
-class Notification():
-
- def __init__(self,
- notification_type,
- *,
- full_name=None,
- job_action=None,
- job_status=None,
- time=None,
- element=None,
- message=None):
+class Notification:
+ def __init__(
+ self,
+ notification_type,
+ *,
+ full_name=None,
+ job_action=None,
+ job_status=None,
+ time=None,
+ element=None,
+ message=None
+ ):
self.notification_type = notification_type
self.full_name = full_name
self.job_action = job_action
@@ -116,40 +117,36 @@ class Notification():
# interrupt_callback: A callback to handle ^C
# ticker_callback: A callback call once per second
#
-class Scheduler():
-
- def __init__(self, context,
- start_time, state, notification_queue, notifier):
+class Scheduler:
+ def __init__(self, context, start_time, state, notification_queue, notifier):
#
# Public members
#
- self.queues = None # Exposed for the frontend to print summaries
- self.context = context # The Context object shared with Queues
- self.terminated = False # Whether the scheduler was asked to terminate or has terminated
- self.suspended = False # Whether the scheduler is currently suspended
+ self.queues = None # Exposed for the frontend to print summaries
+ self.context = context # The Context object shared with Queues
+ self.terminated = False # Whether the scheduler was asked to terminate or has terminated
+ self.suspended = False # Whether the scheduler is currently suspended
# These are shared with the Job, but should probably be removed or made private in some way.
- self.loop = None # Shared for Job access to observe the message queue
- self.internal_stops = 0 # Amount of SIGSTP signals we've introduced, this is shared with job.py
+ self.loop = None # Shared for Job access to observe the message queue
+ self.internal_stops = 0 # Amount of SIGSTP signals we've introduced, this is shared with job.py
#
# Private members
#
- self._active_jobs = [] # Jobs currently being run in the scheduler
- self._starttime = start_time # Initial application start time
- self._suspendtime = None # Session time compensation for suspended state
- self._queue_jobs = True # Whether we should continue to queue jobs
+ self._active_jobs = [] # Jobs currently being run in the scheduler
+ self._starttime = start_time # Initial application start time
+ self._suspendtime = None # Session time compensation for suspended state
+ self._queue_jobs = True # Whether we should continue to queue jobs
self._state = state
- self._casd_process = None # handle to the casd process for monitoring purpose
+ self._casd_process = None # handle to the casd process for monitoring purpose
# Bidirectional queue to send notifications back to the Scheduler's owner
self._notification_queue = notification_queue
self._notifier = notifier
- self.resources = Resources(context.sched_builders,
- context.sched_fetchers,
- context.sched_pushers)
+ self.resources = Resources(context.sched_builders, context.sched_fetchers, context.sched_pushers)
# run()
#
@@ -310,11 +307,13 @@ class Scheduler():
element_info = None
# Now check for more jobs
- notification = Notification(NotificationType.JOB_COMPLETE,
- full_name=job.name,
- job_action=job.action_name,
- job_status=status,
- element=element_info)
+ notification = Notification(
+ NotificationType.JOB_COMPLETE,
+ full_name=job.name,
+ job_action=job.action_name,
+ job_status=status,
+ element=element_info,
+ )
self._notify(notification)
self._sched()
@@ -360,10 +359,12 @@ class Scheduler():
#
def _start_job(self, job):
self._active_jobs.append(job)
- notification = Notification(NotificationType.JOB_START,
- full_name=job.name,
- job_action=job.action_name,
- time=self._state.elapsed_time(start_time=self._starttime))
+ notification = Notification(
+ NotificationType.JOB_START,
+ full_name=job.name,
+ job_action=job.action_name,
+ time=self._state.elapsed_time(start_time=self._starttime),
+ )
self._notify(notification)
job.start()
@@ -399,9 +400,7 @@ class Scheduler():
# to fetch tasks for elements which failed to pull, and
# thus need all the pulls to complete before ever starting
# a build
- ready.extend(chain.from_iterable(
- q.harvest_jobs() for q in reversed(self.queues)
- ))
+ ready.extend(chain.from_iterable(q.harvest_jobs() for q in reversed(self.queues)))
# harvest_jobs() may have decided to skip some jobs, making
# them eligible for promotion to the next queue as a side effect.
@@ -471,7 +470,7 @@ class Scheduler():
self.suspended = False
# Notify that we're unsuspended
self._notify(Notification(NotificationType.SUSPENDED))
- self._starttime += (datetime.datetime.now() - self._suspendtime)
+ self._starttime += datetime.datetime.now() - self._suspendtime
self._notify(Notification(NotificationType.SCHED_START_TIME, time=self._starttime))
self._suspendtime = None
diff --git a/src/buildstream/_signals.py b/src/buildstream/_signals.py
index 31982c199..425a57239 100644
--- a/src/buildstream/_signals.py
+++ b/src/buildstream/_signals.py
@@ -37,8 +37,8 @@ if TYPE_CHECKING:
# typing.MutableSequence. However, that is only available in Python versions
# 3.5.4 onward and 3.6.1 onward.
# Debian 9 ships with 3.5.3.
-terminator_stack = deque() # type: MutableSequence[Callable]
-suspendable_stack = deque() # type: MutableSequence[Callable]
+terminator_stack = deque() # type: MutableSequence[Callable]
+suspendable_stack = deque() # type: MutableSequence[Callable]
# Per process SIGTERM handler
@@ -47,16 +47,18 @@ def terminator_handler(signal_, frame):
terminator_ = terminator_stack.pop()
try:
terminator_()
- except: # noqa pylint: disable=bare-except
+ except: # noqa pylint: disable=bare-except
# Ensure we print something if there's an exception raised when
# processing the handlers. Note that the default exception
# handler won't be called because we os._exit next, so we must
# catch all possible exceptions with the unqualified 'except'
# clause.
traceback.print_exc(file=sys.stderr)
- print('Error encountered in BuildStream while processing custom SIGTERM handler:',
- terminator_,
- file=sys.stderr)
+ print(
+ "Error encountered in BuildStream while processing custom SIGTERM handler:",
+ terminator_,
+ file=sys.stderr,
+ )
# Use special exit here, terminate immediately, recommended
# for precisely this situation where child processes are teminated.
@@ -79,7 +81,7 @@ def terminator_handler(signal_, frame):
#
@contextmanager
def terminator(terminate_func):
- global terminator_stack # pylint: disable=global-statement
+ global terminator_stack # pylint: disable=global-statement
# Signal handling only works in the main thread
if threading.current_thread() != threading.main_thread():
@@ -101,7 +103,7 @@ def terminator(terminate_func):
# Just a simple object for holding on to two callbacks
-class Suspender():
+class Suspender:
def __init__(self, suspend_callback, resume_callback):
self.suspend = suspend_callback
self.resume = resume_callback
@@ -144,7 +146,7 @@ def suspend_handler(sig, frame):
#
@contextmanager
def suspendable(suspend_callback, resume_callback):
- global suspendable_stack # pylint: disable=global-statement
+ global suspendable_stack # pylint: disable=global-statement
outermost = bool(not suspendable_stack)
suspender = Suspender(suspend_callback, resume_callback)
diff --git a/src/buildstream/_site.py b/src/buildstream/_site.py
index 8940fa34a..db0587120 100644
--- a/src/buildstream/_site.py
+++ b/src/buildstream/_site.py
@@ -30,22 +30,22 @@ import subprocess
root = os.path.dirname(os.path.abspath(__file__))
# The Element plugin directory
-element_plugins = os.path.join(root, 'plugins', 'elements')
+element_plugins = os.path.join(root, "plugins", "elements")
# The Source plugin directory
-source_plugins = os.path.join(root, 'plugins', 'sources')
+source_plugins = os.path.join(root, "plugins", "sources")
# Default user configuration
-default_user_config = os.path.join(root, 'data', 'userconfig.yaml')
+default_user_config = os.path.join(root, "data", "userconfig.yaml")
# Default project configuration
-default_project_config = os.path.join(root, 'data', 'projectconfig.yaml')
+default_project_config = os.path.join(root, "data", "projectconfig.yaml")
# Script template to call module building scripts
-build_all_template = os.path.join(root, 'data', 'build-all.sh.in')
+build_all_template = os.path.join(root, "data", "build-all.sh.in")
# Module building script template
-build_module_template = os.path.join(root, 'data', 'build-module.sh.in')
+build_module_template = os.path.join(root, "data", "build-module.sh.in")
def get_bwrap_version():
@@ -53,7 +53,7 @@ def get_bwrap_version():
#
# returns None if no bwrap was found
# otherwise returns a tuple of 3 int: major, minor, patch
- bwrap_path = shutil.which('bwrap')
+ bwrap_path = shutil.which("bwrap")
if not bwrap_path:
return None
diff --git a/src/buildstream/_sourcecache.py b/src/buildstream/_sourcecache.py
index 28ad82831..03e2d1830 100644
--- a/src/buildstream/_sourcecache.py
+++ b/src/buildstream/_sourcecache.py
@@ -26,12 +26,10 @@ from .storage._casbaseddirectory import CasBasedDirectory
from ._basecache import BaseCache
from ._exceptions import CASError, CASRemoteError, SourceCacheError, RemoteError
from . import utils
-from ._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc, \
- source_pb2, source_pb2_grpc
+from ._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc, source_pb2, source_pb2_grpc
class SourceRemote(BaseRemote):
-
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.source_service = None
@@ -65,8 +63,10 @@ class SourceRemote(BaseRemote):
except grpc.RpcError as e:
# Check if this remote has the artifact service
if e.code() == grpc.StatusCode.UNIMPLEMENTED:
- raise RemoteError("Configured remote does not have the BuildStream "
- "capabilities service. Please check remote configuration.")
+ raise RemoteError(
+ "Configured remote does not have the BuildStream "
+ "capabilities service. Please check remote configuration."
+ )
raise RemoteError("Remote initialisation failed: {}".format(e.details()))
if not response.source_capabilities:
@@ -129,7 +129,7 @@ class SourceCache(BaseCache):
def __init__(self, context):
super().__init__(context)
- self.sourcerefdir = os.path.join(context.cachedir, 'source_protos')
+ self.sourcerefdir = os.path.join(context.cachedir, "source_protos")
os.makedirs(self.sourcerefdir, exist_ok=True)
# list_sources()
@@ -182,7 +182,7 @@ class SourceCache(BaseCache):
vdir.import_files(self.export(previous_source))
if not source.BST_STAGE_VIRTUAL_DIRECTORY:
- with utils._tempdir(dir=self.context.tmpdir, prefix='staging-temp') as tmpdir:
+ with utils._tempdir(dir=self.context.tmpdir, prefix="staging-temp") as tmpdir:
if not vdir.is_empty():
vdir.export_files(tmpdir)
source._stage(tmpdir)
@@ -233,12 +233,12 @@ class SourceCache(BaseCache):
source_proto = self._pull_source(ref, remote)
if source_proto is None:
- source.info("Remote source service ({}) does not have source {} cached".format(
- remote, display_key))
+ source.info(
+ "Remote source service ({}) does not have source {} cached".format(remote, display_key)
+ )
continue
except CASError as e:
- raise SourceCacheError("Failed to pull source {}: {}".format(
- display_key, e)) from e
+ raise SourceCacheError("Failed to pull source {}: {}".format(display_key, e)) from e
if not source_proto:
return False
@@ -255,8 +255,7 @@ class SourceCache(BaseCache):
missing_blobs = self.cas.fetch_blobs(remote, missing_blobs)
if missing_blobs:
- source.info("Remote cas ({}) does not have source {} cached".format(
- remote, display_key))
+ source.info("Remote cas ({}) does not have source {} cached".format(remote, display_key))
continue
source.info("Pulled source {} <- {}".format(display_key, remote))
@@ -266,8 +265,7 @@ class SourceCache(BaseCache):
source.info("Remote cas ({}) does not have blob {} cached".format(remote, e.blob))
continue
except CASError as e:
- raise SourceCacheError("Failed to pull source {}: {}".format(
- display_key, e)) from e
+ raise SourceCacheError("Failed to pull source {}: {}".format(display_key, e)) from e
return False
@@ -315,8 +313,7 @@ class SourceCache(BaseCache):
# check whether cache has files already
if self._pull_source(ref, remote) is not None:
- source.info("Remote ({}) already has source {} cached"
- .format(remote, display_key))
+ source.info("Remote ({}) already has source {} cached".format(remote, display_key))
continue
if not self._push_source(ref, remote):
@@ -340,19 +337,18 @@ class SourceCache(BaseCache):
def _store_proto(self, proto, ref):
path = self._source_path(ref)
os.makedirs(os.path.dirname(path), exist_ok=True)
- with utils.save_file_atomic(path, 'w+b') as f:
+ with utils.save_file_atomic(path, "w+b") as f:
f.write(proto.SerializeToString())
def _get_source(self, ref):
path = self._source_path(ref)
source_proto = source_pb2.Source()
try:
- with open(path, 'r+b') as f:
+ with open(path, "r+b") as f:
source_proto.ParseFromString(f.read())
return source_proto
except FileNotFoundError as e:
- raise SourceCacheError("Attempted to access unavailable source: {}"
- .format(e)) from e
+ raise SourceCacheError("Attempted to access unavailable source: {}".format(e)) from e
def _source_path(self, ref):
return os.path.join(self.sourcerefdir, ref)
@@ -361,7 +357,7 @@ class SourceCache(BaseCache):
for root, _, files in os.walk(self.sourcerefdir):
for source_file in files:
source = source_pb2.Source()
- with open(os.path.join(root, source_file), 'r+b') as f:
+ with open(os.path.join(root, source_file), "r+b") as f:
source.ParseFromString(f.read())
yield source.files
diff --git a/src/buildstream/_sourcefactory.py b/src/buildstream/_sourcefactory.py
index 1d959a140..7c90042a8 100644
--- a/src/buildstream/_sourcefactory.py
+++ b/src/buildstream/_sourcefactory.py
@@ -30,14 +30,11 @@ from .source import Source
# plugin_origins (list): Data used to search for external Source plugins
#
class SourceFactory(PluginContext):
+ def __init__(self, plugin_base, *, format_versions={}, plugin_origins=None):
- def __init__(self, plugin_base, *,
- format_versions={},
- plugin_origins=None):
-
- super().__init__(plugin_base, Source, [_site.source_plugins],
- format_versions=format_versions,
- plugin_origins=plugin_origins)
+ super().__init__(
+ plugin_base, Source, [_site.source_plugins], format_versions=format_versions, plugin_origins=plugin_origins
+ )
# create():
#
diff --git a/src/buildstream/_state.py b/src/buildstream/_state.py
index 310e12a63..d85e348f2 100644
--- a/src/buildstream/_state.py
+++ b/src/buildstream/_state.py
@@ -28,7 +28,7 @@ from collections import OrderedDict
# state (State): The state object
# complete_name (str): Optional name for frontend status rendering, e.g. 'built'
#
-class TaskGroup():
+class TaskGroup:
def __init__(self, name, state, complete_name=None):
self.name = name
self.complete_name = complete_name
@@ -98,14 +98,14 @@ class TaskGroup():
# Args:
# session_start (datetime): The time the session started
#
-class State():
+class State:
def __init__(self, session_start):
self._session_start = session_start
self.task_groups = OrderedDict() # key is TaskGroup name
# Note: A Task's full_name is technically unique, but only accidentally.
- self.tasks = OrderedDict() # key is a tuple of action_name and full_name
+ self.tasks = OrderedDict() # key is a tuple of action_name and full_name
self._task_added_cbs = []
self._task_removed_cbs = []
@@ -281,8 +281,9 @@ class State():
#
def add_task(self, action_name, full_name, elapsed_offset=None):
task_key = (action_name, full_name)
- assert task_key not in self.tasks, \
- "Trying to add task '{}:{}' to '{}'".format(action_name, full_name, self.tasks)
+ assert task_key not in self.tasks, "Trying to add task '{}:{}' to '{}'".format(
+ action_name, full_name, self.tasks
+ )
if not elapsed_offset:
elapsed_offset = self.elapsed_time()
@@ -366,7 +367,7 @@ class State():
# e.g. an element's name.
# elapsed_offset (timedelta): The time the task started, relative to
# buildstream's start time.
-class _Task():
+class _Task:
def __init__(self, state, action_name, full_name, elapsed_offset):
self._state = state
self.action_name = action_name
diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py
index f09a46185..2515fadce 100644
--- a/src/buildstream/_stream.py
+++ b/src/buildstream/_stream.py
@@ -33,8 +33,19 @@ from collections import deque
from ._artifactelement import verify_artifact_ref, ArtifactElement
from ._exceptions import StreamError, ImplError, BstError, ArtifactElementError, ArtifactError
from ._message import Message, MessageType
-from ._scheduler import Scheduler, SchedStatus, TrackQueue, FetchQueue, \
- SourcePushQueue, BuildQueue, PullQueue, ArtifactPushQueue, NotificationType, Notification, JobStatus
+from ._scheduler import (
+ Scheduler,
+ SchedStatus,
+ TrackQueue,
+ FetchQueue,
+ SourcePushQueue,
+ BuildQueue,
+ PullQueue,
+ ArtifactPushQueue,
+ NotificationType,
+ Notification,
+ JobStatus,
+)
from ._pipeline import Pipeline, PipelineSelection
from ._profile import Topics, PROFILER
from ._state import State
@@ -55,20 +66,18 @@ from . import Scope, Consistency
# interrupt_callback (callable): A callback to invoke when we get interrupted
# ticker_callback (callable): Invoked every second while running the scheduler
#
-class Stream():
-
- def __init__(self, context, session_start, *,
- session_start_callback=None,
- interrupt_callback=None,
- ticker_callback=None):
+class Stream:
+ def __init__(
+ self, context, session_start, *, session_start_callback=None, interrupt_callback=None, ticker_callback=None
+ ):
#
# Public members
#
- self.targets = [] # Resolved target elements
- self.session_elements = [] # List of elements being processed this session
- self.total_elements = [] # Total list of elements based on targets
- self.queues = [] # Queue objects
+ self.targets = [] # Resolved target elements
+ self.session_elements = [] # List of elements being processed this session
+ self.total_elements = [] # Total list of elements based on targets
+ self.queues = [] # Queue objects
#
# Private members
@@ -84,8 +93,9 @@ class Stream():
context.messenger.set_state(self._state)
- self._scheduler = Scheduler(context, session_start, self._state, self._notification_queue,
- self._scheduler_notification_handler)
+ self._scheduler = Scheduler(
+ context, session_start, self._state, self._notification_queue, self._scheduler_notification_handler
+ )
self._first_non_track_queue = None
self._session_start_callback = session_start_callback
self._ticker_callback = ticker_callback
@@ -138,17 +148,24 @@ class Stream():
#
# Returns:
# (list of Element): The selected elements
- def load_selection(self, targets, *,
- selection=PipelineSelection.NONE,
- except_targets=(),
- use_artifact_config=False,
- load_refs=False):
+ def load_selection(
+ self,
+ targets,
+ *,
+ selection=PipelineSelection.NONE,
+ except_targets=(),
+ use_artifact_config=False,
+ load_refs=False
+ ):
with PROFILER.profile(Topics.LOAD_SELECTION, "_".join(t.replace(os.sep, "-") for t in targets)):
- target_objects, _ = self._load(targets, (),
- selection=selection,
- except_targets=except_targets,
- use_artifact_config=use_artifact_config,
- load_refs=load_refs)
+ target_objects, _ = self._load(
+ targets,
+ (),
+ selection=selection,
+ except_targets=except_targets,
+ use_artifact_config=use_artifact_config,
+ load_refs=load_refs,
+ )
return target_objects
@@ -171,14 +188,20 @@ class Stream():
# Returns:
# (int): The exit code of the launched shell
#
- def shell(self, element, scope, prompt, *,
- directory=None,
- mounts=None,
- isolate=False,
- command=None,
- usebuildtree=None,
- pull_dependencies=None,
- unique_id=None):
+ def shell(
+ self,
+ element,
+ scope,
+ prompt,
+ *,
+ directory=None,
+ mounts=None,
+ isolate=False,
+ command=None,
+ usebuildtree=None,
+ pull_dependencies=None,
+ unique_id=None
+ ):
# Load the Element via the unique_id if given
if unique_id and element is None:
@@ -192,18 +215,16 @@ class Stream():
if not element._source_cached():
raise StreamError(
"Sources for element {} are not cached."
- "Element must be fetched.".format(element._get_full_name()))
+ "Element must be fetched.".format(element._get_full_name())
+ )
- missing_deps = [
- dep for dep in self._pipeline.dependencies([element], scope)
- if not dep._cached()
- ]
+ missing_deps = [dep for dep in self._pipeline.dependencies([element], scope) if not dep._cached()]
if missing_deps:
if not pull_dependencies:
raise StreamError(
"Elements need to be built or downloaded before staging a shell environment",
- detail="\n"
- .join(list(map(lambda x: x._get_full_name(), missing_deps))))
+ detail="\n".join(list(map(lambda x: x._get_full_name(), missing_deps))),
+ )
self._message(MessageType.INFO, "Attempting to fetch missing or incomplete artifacts")
self._scheduler.clear_queues()
self._add_queue(PullQueue(self._scheduler))
@@ -236,8 +257,9 @@ class Stream():
else:
buildtree = True
- return element._shell(scope, directory, mounts=mounts, isolate=isolate, prompt=prompt, command=command,
- usebuildtree=buildtree)
+ return element._shell(
+ scope, directory, mounts=mounts, isolate=isolate, prompt=prompt, command=command, usebuildtree=buildtree
+ )
# build()
#
@@ -252,23 +274,22 @@ class Stream():
# If `remote` specified as None, then regular configuration will be used
# to determine where to push artifacts to.
#
- def build(self, targets, *,
- selection=PipelineSelection.PLAN,
- ignore_junction_targets=False,
- remote=None):
+ def build(self, targets, *, selection=PipelineSelection.PLAN, ignore_junction_targets=False, remote=None):
use_config = True
if remote:
use_config = False
- elements, _ = \
- self._load(targets, [],
- selection=selection,
- ignore_junction_targets=ignore_junction_targets,
- use_artifact_config=use_config,
- artifact_remote_url=remote,
- use_source_config=True,
- dynamic_plan=True)
+ elements, _ = self._load(
+ targets,
+ [],
+ selection=selection,
+ ignore_junction_targets=ignore_junction_targets,
+ use_artifact_config=use_config,
+ artifact_remote_url=remote,
+ use_source_config=True,
+ dynamic_plan=True,
+ )
# Assert that the elements are consistent
self._pipeline.assert_consistent(elements)
@@ -317,12 +338,16 @@ class Stream():
# track_cross_junctions (bool): Whether tracking should cross junction boundaries
# remote (str|None): The URL of a specific remote server to pull from.
#
- def fetch(self, targets, *,
- selection=PipelineSelection.PLAN,
- except_targets=None,
- track_targets=False,
- track_cross_junctions=False,
- remote=None):
+ def fetch(
+ self,
+ targets,
+ *,
+ selection=PipelineSelection.PLAN,
+ except_targets=None,
+ track_targets=False,
+ track_cross_junctions=False,
+ remote=None
+ ):
if track_targets:
track_targets = targets
@@ -337,14 +362,17 @@ class Stream():
if remote:
use_source_config = False
- elements, track_elements = \
- self._load(targets, track_targets,
- selection=selection, track_selection=track_selection,
- except_targets=except_targets,
- track_except_targets=track_except_targets,
- track_cross_junctions=track_cross_junctions,
- use_source_config=use_source_config,
- source_remote_url=remote)
+ elements, track_elements = self._load(
+ targets,
+ track_targets,
+ selection=selection,
+ track_selection=track_selection,
+ except_targets=except_targets,
+ track_except_targets=track_except_targets,
+ track_cross_junctions=track_cross_junctions,
+ use_source_config=use_source_config,
+ source_remote_url=remote,
+ )
# Delegated to a shared fetch method
self._fetch(elements, track_elements=track_elements)
@@ -362,20 +390,20 @@ class Stream():
# If no error is encountered while tracking, then the project files
# are rewritten inline.
#
- def track(self, targets, *,
- selection=PipelineSelection.REDIRECT,
- except_targets=None,
- cross_junctions=False):
+ def track(self, targets, *, selection=PipelineSelection.REDIRECT, except_targets=None, cross_junctions=False):
# We pass no target to build. Only to track. Passing build targets
# would fully load project configuration which might not be
# possible before tracking is done.
- _, elements = \
- self._load([], targets,
- selection=selection, track_selection=selection,
- except_targets=except_targets,
- track_except_targets=except_targets,
- track_cross_junctions=cross_junctions)
+ _, elements = self._load(
+ [],
+ targets,
+ selection=selection,
+ track_selection=selection,
+ except_targets=except_targets,
+ track_except_targets=except_targets,
+ track_cross_junctions=cross_junctions,
+ )
# FIXME: this can be refactored after element._update_state is simplified/removed
elements = [element for element in elements if element._schedule_tracking()]
@@ -400,21 +428,21 @@ class Stream():
# If `remote` specified as None, then regular configuration will be used
# to determine where to pull artifacts from.
#
- def pull(self, targets, *,
- selection=PipelineSelection.NONE,
- ignore_junction_targets=False,
- remote=None):
+ def pull(self, targets, *, selection=PipelineSelection.NONE, ignore_junction_targets=False, remote=None):
use_config = True
if remote:
use_config = False
- elements, _ = self._load(targets, (),
- selection=selection,
- ignore_junction_targets=ignore_junction_targets,
- use_artifact_config=use_config,
- artifact_remote_url=remote,
- load_refs=True)
+ elements, _ = self._load(
+ targets,
+ (),
+ selection=selection,
+ ignore_junction_targets=ignore_junction_targets,
+ use_artifact_config=use_config,
+ artifact_remote_url=remote,
+ load_refs=True,
+ )
if not self._artifacts.has_fetch_remotes():
raise StreamError("No artifact caches available for pulling artifacts")
@@ -442,21 +470,21 @@ class Stream():
# a pull queue will be created if user context and available remotes allow for
# attempting to fetch them.
#
- def push(self, targets, *,
- selection=PipelineSelection.NONE,
- ignore_junction_targets=False,
- remote=None):
+ def push(self, targets, *, selection=PipelineSelection.NONE, ignore_junction_targets=False, remote=None):
use_config = True
if remote:
use_config = False
- elements, _ = self._load(targets, (),
- selection=selection,
- ignore_junction_targets=ignore_junction_targets,
- use_artifact_config=use_config,
- artifact_remote_url=remote,
- load_refs=True)
+ elements, _ = self._load(
+ targets,
+ (),
+ selection=selection,
+ ignore_junction_targets=ignore_junction_targets,
+ use_artifact_config=use_config,
+ artifact_remote_url=remote,
+ load_refs=True,
+ )
if not self._artifacts.has_push_remotes():
raise StreamError("No artifact caches available for pushing artifacts")
@@ -500,8 +528,10 @@ class Stream():
# ready an uncached element in the PushQueue.
if self._context.sched_error_action == _SchedulerErrorAction.CONTINUE and uncached_elements:
names = [element.name for element in uncached_elements]
- fail_str = "Error while pushing. The following elements were not pushed as they are " \
+ fail_str = (
+ "Error while pushing. The following elements were not pushed as they are "
"not yet cached:\n\n\t{}\n".format("\n\t".join(names))
+ )
raise StreamError(fail_str)
@@ -525,15 +555,19 @@ class Stream():
# pull (bool): If true will attempt to pull any missing or incomplete
# artifacts.
#
- def checkout(self, target, *,
- location=None,
- force=False,
- selection=PipelineSelection.RUN,
- integrate=True,
- hardlinks=False,
- compression='',
- pull=False,
- tar=False):
+ def checkout(
+ self,
+ target,
+ *,
+ location=None,
+ force=False,
+ selection=PipelineSelection.RUN,
+ integrate=True,
+ hardlinks=False,
+ compression="",
+ pull=False,
+ tar=False
+ ):
elements, _ = self._load((target,), (), selection=selection, use_artifact_config=True, load_refs=True)
@@ -554,15 +588,15 @@ class Stream():
self._run()
try:
- scope = {'run': Scope.RUN, 'build': Scope.BUILD, 'none': Scope.NONE, 'all': Scope.ALL}
- with target._prepare_sandbox(scope=scope[selection], directory=None,
- integrate=integrate) as sandbox:
+ scope = {"run": Scope.RUN, "build": Scope.BUILD, "none": Scope.NONE, "all": Scope.ALL}
+ with target._prepare_sandbox(scope=scope[selection], directory=None, integrate=integrate) as sandbox:
# Copy or move the sandbox to the target directory
virdir = sandbox.get_virtual_directory()
self._export_artifact(tar, location, compression, target, hardlinks, virdir)
except BstError as e:
- raise StreamError("Error while staging dependencies into a sandbox"
- ": '{}'".format(e), detail=e.detail, reason=e.reason) from e
+ raise StreamError(
+ "Error while staging dependencies into a sandbox" ": '{}'".format(e), detail=e.detail, reason=e.reason
+ ) from e
# _export_artifact()
#
@@ -578,34 +612,32 @@ class Stream():
#
def _export_artifact(self, tar, location, compression, target, hardlinks, virdir):
if not tar:
- with target.timed_activity("Checking out files in '{}'"
- .format(location)):
+ with target.timed_activity("Checking out files in '{}'".format(location)):
try:
if hardlinks:
self._checkout_hardlinks(virdir, location)
else:
virdir.export_files(location)
except OSError as e:
- raise StreamError("Failed to checkout files: '{}'"
- .format(e)) from e
+ raise StreamError("Failed to checkout files: '{}'".format(e)) from e
else:
- to_stdout = location == '-'
+ to_stdout = location == "-"
mode = _handle_compression(compression, to_stream=to_stdout)
with target.timed_activity("Creating tarball"):
if to_stdout:
# Save the stdout FD to restore later
saved_fd = os.dup(sys.stdout.fileno())
try:
- with os.fdopen(sys.stdout.fileno(), 'wb') as fo:
+ with os.fdopen(sys.stdout.fileno(), "wb") as fo:
with tarfile.open(fileobj=fo, mode=mode) as tf:
- virdir.export_to_tar(tf, '.')
+ virdir.export_to_tar(tf, ".")
finally:
# No matter what, restore stdout for further use
os.dup2(saved_fd, sys.stdout.fileno())
os.close(saved_fd)
else:
with tarfile.open(location, mode=mode) as tf:
- virdir.export_to_tar(tf, '.')
+ virdir.export_to_tar(tf, ".")
# artifact_show()
#
@@ -614,13 +646,9 @@ class Stream():
# Args:
# targets (str): Targets to show the cached state of
#
- def artifact_show(self, targets, *,
- selection=PipelineSelection.NONE):
+ def artifact_show(self, targets, *, selection=PipelineSelection.NONE):
# Obtain list of Element and/or ArtifactElement objects
- target_objects = self.load_selection(targets,
- selection=selection,
- use_artifact_config=True,
- load_refs=True)
+ target_objects = self.load_selection(targets, selection=selection, use_artifact_config=True, load_refs=True)
if self._artifacts.has_fetch_remotes():
self._pipeline.check_remotes(target_objects)
@@ -695,8 +723,7 @@ class Stream():
# Args:
# targets (str): Targets to remove
#
- def artifact_delete(self, targets, *,
- selection=PipelineSelection.NONE):
+ def artifact_delete(self, targets, *, selection=PipelineSelection.NONE):
# Return list of Element and/or ArtifactElement objects
target_objects = self.load_selection(targets, selection=selection, load_refs=True)
@@ -736,20 +763,22 @@ class Stream():
# compression (str): The type of compression for tarball
# include_build_scripts (bool): Whether to include build scripts in the checkout
#
- def source_checkout(self, target, *,
- location=None,
- force=False,
- deps='none',
- except_targets=(),
- tar=False,
- compression=None,
- include_build_scripts=False):
+ def source_checkout(
+ self,
+ target,
+ *,
+ location=None,
+ force=False,
+ deps="none",
+ except_targets=(),
+ tar=False,
+ compression=None,
+ include_build_scripts=False
+ ):
self._check_location_writable(location, force=force, tar=tar)
- elements, _ = self._load((target,), (),
- selection=deps,
- except_targets=except_targets)
+ elements, _ = self._load((target,), (), selection=deps, except_targets=except_targets)
# Assert all sources are cached in the source dir
self._fetch(elements)
@@ -757,11 +786,11 @@ class Stream():
# Stage all sources determined by scope
try:
- self._source_checkout(elements, location, force, deps,
- tar, compression, include_build_scripts)
+ self._source_checkout(elements, location, force, deps, tar, compression, include_build_scripts)
except BstError as e:
- raise StreamError("Error while writing sources"
- ": '{}'".format(e), detail=e.detail, reason=e.reason) from e
+ raise StreamError(
+ "Error while writing sources" ": '{}'".format(e), detail=e.detail, reason=e.reason
+ ) from e
self._message(MessageType.INFO, "Checked out sources to '{}'".format(location))
@@ -776,11 +805,7 @@ class Stream():
# force (bool): Whether to ignore contents in an existing directory
# custom_dir (str): Custom location to create a workspace or false to use default location.
#
- def workspace_open(self, targets, *,
- no_checkout,
- track_first,
- force,
- custom_dir):
+ def workspace_open(self, targets, *, no_checkout, track_first, force, custom_dir):
# This function is a little funny but it is trying to be as atomic as possible.
if track_first:
@@ -788,9 +813,9 @@ class Stream():
else:
track_targets = ()
- elements, track_elements = self._load(targets, track_targets,
- selection=PipelineSelection.REDIRECT,
- track_selection=PipelineSelection.REDIRECT)
+ elements, track_elements = self._load(
+ targets, track_targets, selection=PipelineSelection.REDIRECT, track_selection=PipelineSelection.REDIRECT
+ )
workspaces = self._context.get_workspaces()
@@ -819,33 +844,44 @@ class Stream():
workspace = workspaces.get_workspace(target._get_full_name())
if workspace:
if not force:
- raise StreamError("Element '{}' already has an open workspace defined at: {}"
- .format(target.name, workspace.get_absolute_path()))
+ raise StreamError(
+ "Element '{}' already has an open workspace defined at: {}".format(
+ target.name, workspace.get_absolute_path()
+ )
+ )
if not no_checkout:
- target.warn("Replacing existing workspace for element '{}' defined at: {}"
- .format(target.name, workspace.get_absolute_path()))
+ target.warn(
+ "Replacing existing workspace for element '{}' defined at: {}".format(
+ target.name, workspace.get_absolute_path()
+ )
+ )
self.workspace_close(target._get_full_name(), remove_dir=not no_checkout)
target_consistency = target._get_consistency()
- if not no_checkout and target_consistency < Consistency.CACHED and \
- target_consistency._source_cached():
- raise StreamError("Could not stage uncached source. For {} ".format(target.name) +
- "Use `--track` to track and " +
- "fetch the latest version of the " +
- "source.")
+ if not no_checkout and target_consistency < Consistency.CACHED and target_consistency._source_cached():
+ raise StreamError(
+ "Could not stage uncached source. For {} ".format(target.name)
+ + "Use `--track` to track and "
+ + "fetch the latest version of the "
+ + "source."
+ )
if not custom_dir:
directory = os.path.abspath(os.path.join(self._context.workspacedir, target.name))
- if directory[-4:] == '.bst':
+ if directory[-4:] == ".bst":
directory = directory[:-4]
expanded_directories.append(directory)
if custom_dir:
if len(elements) != 1:
- raise StreamError("Exactly one element can be given if --directory is used",
- reason='directory-with-multiple-elements')
+ raise StreamError(
+ "Exactly one element can be given if --directory is used",
+ reason="directory-with-multiple-elements",
+ )
directory = os.path.abspath(custom_dir)
- expanded_directories = [directory, ]
+ expanded_directories = [
+ directory,
+ ]
else:
# If this fails it is a bug in what ever calls this, usually cli.py and so can not be tested for via the
# run bst test mechanism.
@@ -854,12 +890,16 @@ class Stream():
for target, directory in zip(elements, expanded_directories):
if os.path.exists(directory):
if not os.path.isdir(directory):
- raise StreamError("For element '{}', Directory path is not a directory: {}"
- .format(target.name, directory), reason='bad-directory')
+ raise StreamError(
+ "For element '{}', Directory path is not a directory: {}".format(target.name, directory),
+ reason="bad-directory",
+ )
if not (no_checkout or force) and os.listdir(directory):
- raise StreamError("For element '{}', Directory path is not empty: {}"
- .format(target.name, directory), reason='bad-directory')
+ raise StreamError(
+ "For element '{}', Directory path is not empty: {}".format(target.name, directory),
+ reason="bad-directory",
+ )
if os.listdir(directory):
if force and not no_checkout:
utils._force_rmtree(directory)
@@ -868,8 +908,7 @@ class Stream():
# Now it does the bits that can not be made atomic.
targetGenerator = zip(elements, expanded_directories)
for target, directory in targetGenerator:
- self._message(MessageType.INFO, "Creating workspace for element {}"
- .format(target.name))
+ self._message(MessageType.INFO, "Creating workspace for element {}".format(target.name))
workspace = workspaces.get_workspace(target._get_full_name())
if workspace and not no_checkout:
@@ -886,8 +925,7 @@ class Stream():
raise StreamError("Failed to create workspace directory: {}".format(e) + todo_elements) from e
workspaces.create_workspace(target, directory, checkout=not no_checkout)
- self._message(MessageType.INFO, "Created a workspace for element: {}"
- .format(target._get_full_name()))
+ self._message(MessageType.INFO, "Created a workspace for element: {}".format(target._get_full_name()))
# workspace_close
#
@@ -903,13 +941,13 @@ class Stream():
# Remove workspace directory if prompted
if remove_dir:
- with self._context.messenger.timed_activity("Removing workspace directory {}"
- .format(workspace.get_absolute_path())):
+ with self._context.messenger.timed_activity(
+ "Removing workspace directory {}".format(workspace.get_absolute_path())
+ ):
try:
shutil.rmtree(workspace.get_absolute_path())
except OSError as e:
- raise StreamError("Could not remove '{}': {}"
- .format(workspace.get_absolute_path(), e)) from e
+ raise StreamError("Could not remove '{}': {}".format(workspace.get_absolute_path(), e)) from e
# Delete the workspace and save the configuration
workspaces.delete_workspace(element_name)
@@ -928,9 +966,9 @@ class Stream():
#
def workspace_reset(self, targets, *, soft, track_first):
- elements, _ = self._load(targets, [],
- selection=PipelineSelection.REDIRECT,
- track_selection=PipelineSelection.REDIRECT)
+ elements, _ = self._load(
+ targets, [], selection=PipelineSelection.REDIRECT, track_selection=PipelineSelection.REDIRECT
+ )
nonexisting = []
for element in elements:
@@ -946,14 +984,20 @@ class Stream():
if soft:
workspace.prepared = False
- self._message(MessageType.INFO, "Reset workspace state for {} at: {}"
- .format(element.name, workspace_path))
+ self._message(
+ MessageType.INFO, "Reset workspace state for {} at: {}".format(element.name, workspace_path)
+ )
continue
self.workspace_close(element._get_full_name(), remove_dir=True)
workspaces.save_config()
- self.workspace_open([element._get_full_name()],
- no_checkout=False, track_first=track_first, force=True, custom_dir=workspace_path)
+ self.workspace_open(
+ [element._get_full_name()],
+ no_checkout=False,
+ track_first=track_first,
+ force=True,
+ custom_dir=workspace_path,
+ )
# workspace_exists
#
@@ -1001,14 +1045,12 @@ class Stream():
workspaces = []
for element_name, workspace_ in self._context.get_workspaces().list():
workspace_detail = {
- 'element': element_name,
- 'directory': workspace_.get_absolute_path(),
+ "element": element_name,
+ "directory": workspace_.get_absolute_path(),
}
workspaces.append(workspace_detail)
- _yaml.roundtrip_dump({
- 'workspaces': workspaces
- })
+ _yaml.roundtrip_dump({"workspaces": workspaces})
# redirect_element_names()
#
@@ -1034,9 +1076,9 @@ class Stream():
else:
output_elements.add(e)
if load_elements:
- loaded_elements, _ = self._load(load_elements, (),
- selection=PipelineSelection.REDIRECT,
- track_selection=PipelineSelection.REDIRECT)
+ loaded_elements, _ = self._load(
+ load_elements, (), selection=PipelineSelection.REDIRECT, track_selection=PipelineSelection.REDIRECT
+ )
for e in loaded_elements:
output_elements.add(e.name)
@@ -1166,26 +1208,31 @@ class Stream():
# (list of Element): The primary element selection
# (list of Element): The tracking element selection
#
- def _load(self, targets, track_targets, *,
- selection=PipelineSelection.NONE,
- track_selection=PipelineSelection.NONE,
- except_targets=(),
- track_except_targets=(),
- track_cross_junctions=False,
- ignore_junction_targets=False,
- use_artifact_config=False,
- use_source_config=False,
- artifact_remote_url=None,
- source_remote_url=None,
- dynamic_plan=False,
- load_refs=False):
+ def _load(
+ self,
+ targets,
+ track_targets,
+ *,
+ selection=PipelineSelection.NONE,
+ track_selection=PipelineSelection.NONE,
+ except_targets=(),
+ track_except_targets=(),
+ track_cross_junctions=False,
+ ignore_junction_targets=False,
+ use_artifact_config=False,
+ use_source_config=False,
+ artifact_remote_url=None,
+ source_remote_url=None,
+ dynamic_plan=False,
+ load_refs=False
+ ):
# Classify element and artifact strings
target_elements, target_artifacts = self._classify_artifacts(targets)
if target_artifacts:
if not load_refs:
- detail = '\n'.join(target_artifacts)
+ detail = "\n".join(target_artifacts)
raise ArtifactElementError("Cannot perform this operation with artifact refs:", detail=detail)
if selection in (PipelineSelection.ALL, PipelineSelection.RUN):
raise StreamError("Error: '--deps {}' is not supported for artifact refs".format(selection))
@@ -1198,8 +1245,9 @@ class Stream():
# Load all target elements
loadable = [target_elements, except_targets, track_targets, track_except_targets]
if any(loadable):
- elements, except_elements, track_elements, track_except_elements = \
- self._pipeline.load(loadable, rewritable=rewritable)
+ elements, except_elements, track_elements, track_except_elements = self._pipeline.load(
+ loadable, rewritable=rewritable
+ )
else:
elements, except_elements, track_elements, track_except_elements = [], [], [], []
@@ -1208,7 +1256,7 @@ class Stream():
# Optionally filter out junction elements
if ignore_junction_targets:
- elements = [e for e in elements if e.get_kind() != 'junction']
+ elements = [e for e in elements if e.get_kind() != "junction"]
# Hold on to the targets
self.targets = elements + artifacts
@@ -1233,14 +1281,10 @@ class Stream():
for project, project_elements in track_projects.items():
selected = self._pipeline.get_selection(project_elements, track_selection)
- selected = self._pipeline.track_cross_junction_filter(project,
- selected,
- track_cross_junctions)
+ selected = self._pipeline.track_cross_junction_filter(project, selected, track_cross_junctions)
track_selected.extend(selected)
- track_selected = self._pipeline.except_elements(track_elements,
- track_selected,
- track_except_elements)
+ track_selected = self._pipeline.except_elements(track_elements, track_selected, track_except_elements)
if not targets:
return [], track_selected
@@ -1257,9 +1301,7 @@ class Stream():
#
self._pipeline.resolve_elements(self.targets)
selected = self._pipeline.get_selection(self.targets, selection, silent=False)
- selected = self._pipeline.except_elements(self.targets,
- selected,
- except_elements)
+ selected = self._pipeline.except_elements(self.targets, selected, except_elements)
if selection == PipelineSelection.PLAN and dynamic_plan:
# We use a dynamic build plan, only request artifacts of top-level targets,
@@ -1279,8 +1321,7 @@ class Stream():
#
def _message(self, message_type, message, **kwargs):
args = dict(kwargs)
- self._context.messenger.message(
- Message(message_type, message, **args))
+ self._context.messenger.message(Message(message_type, message, **args))
# _add_queue()
#
@@ -1321,9 +1362,7 @@ class Stream():
# unique_id (str): A unique_id to load an Element instance
#
def _failure_retry(self, action_name, unique_id):
- notification = Notification(NotificationType.RETRY,
- job_action=action_name,
- element=unique_id)
+ notification = Notification(NotificationType.RETRY, job_action=action_name, element=unique_id)
self._notify(notification)
# _run()
@@ -1370,8 +1409,7 @@ class Stream():
# Filter out elements with cached sources, only from the fetch plan
# let the track plan resolve new refs.
- cached = [elt for elt in fetch_plan
- if not elt._should_fetch(fetch_original)]
+ cached = [elt for elt in fetch_plan if not elt._should_fetch(fetch_original)]
fetch_plan = self._pipeline.subtract_elements(fetch_plan, cached)
# Construct queues, enqueue and run
@@ -1406,21 +1444,16 @@ class Stream():
try:
os.makedirs(location, exist_ok=True)
except OSError as e:
- raise StreamError("Failed to create destination directory: '{}'"
- .format(e)) from e
+ raise StreamError("Failed to create destination directory: '{}'".format(e)) from e
if not os.access(location, os.W_OK):
- raise StreamError("Destination directory '{}' not writable"
- .format(location))
+ raise StreamError("Destination directory '{}' not writable".format(location))
if not force and os.listdir(location):
- raise StreamError("Destination directory '{}' not empty"
- .format(location))
- elif os.path.exists(location) and location != '-':
+ raise StreamError("Destination directory '{}' not empty".format(location))
+ elif os.path.exists(location) and location != "-":
if not os.access(location, os.W_OK):
- raise StreamError("Output file '{}' not writable"
- .format(location))
+ raise StreamError("Output file '{}' not writable".format(location))
if not force and os.path.exists(location):
- raise StreamError("Output file '{}' already exists"
- .format(location))
+ raise StreamError("Output file '{}' already exists".format(location))
# Helper function for checkout()
#
@@ -1433,13 +1466,16 @@ class Stream():
sandbox_vroot.export_files(directory, can_link=True, can_destroy=True)
# Helper function for source_checkout()
- def _source_checkout(self, elements,
- location=None,
- force=False,
- deps='none',
- tar=False,
- compression=None,
- include_build_scripts=False):
+ def _source_checkout(
+ self,
+ elements,
+ location=None,
+ force=False,
+ deps="none",
+ tar=False,
+ compression=None,
+ include_build_scripts=False,
+ ):
location = os.path.abspath(location)
# Stage all our sources in a temporary directory. The this
@@ -1455,8 +1491,7 @@ class Stream():
else:
self._move_directory(temp_source_dir.name, location, force)
except OSError as e:
- raise StreamError("Failed to checkout sources to {}: {}"
- .format(location, e)) from e
+ raise StreamError("Failed to checkout sources to {}: {}".format(location, e)) from e
finally:
with suppress(FileNotFoundError):
temp_source_dir.cleanup()
@@ -1498,10 +1533,10 @@ class Stream():
# Create a tarball from the content of directory
def _create_tarball(self, directory, tar_name, compression):
if compression is None:
- compression = ''
+ compression = ""
mode = _handle_compression(compression)
try:
- with utils.save_file_atomic(tar_name, mode='wb') as f:
+ with utils.save_file_atomic(tar_name, mode="wb") as f:
tarball = tarfile.open(fileobj=f, mode=mode)
for item in os.listdir(str(directory)):
file_to_add = os.path.join(directory, item)
@@ -1598,7 +1633,7 @@ class Stream():
artifact_globs = []
for target in targets:
- if target.endswith('.bst'):
+ if target.endswith(".bst"):
if any(c in "*?[" for c in target):
element_globs.append(target)
else:
@@ -1628,7 +1663,7 @@ class Stream():
for glob in artifact_globs:
artifact_refs.extend(self._artifacts.list_artifacts(glob=glob))
if not artifact_refs:
- self._message(MessageType.WARN, "No artifacts found for globs: {}".format(', '.join(artifact_globs)))
+ self._message(MessageType.WARN, "No artifacts found for globs: {}".format(", ".join(artifact_globs)))
return element_targets, artifact_refs
@@ -1648,8 +1683,7 @@ class Stream():
elif notification.notification_type == NotificationType.JOB_COMPLETE:
self._state.remove_task(notification.job_action, notification.full_name)
if notification.job_status == JobStatus.FAIL:
- self._state.fail_task(notification.job_action, notification.full_name,
- notification.element)
+ self._state.fail_task(notification.job_action, notification.full_name, notification.element)
elif notification.notification_type == NotificationType.SCHED_START_TIME:
self._starttime = notification.time
elif notification.notification_type == NotificationType.RUNNING:
@@ -1694,5 +1728,5 @@ class Stream():
# (str): The tarfile mode string
#
def _handle_compression(compression, *, to_stream=False):
- mode_prefix = 'w|' if to_stream else 'w:'
+ mode_prefix = "w|" if to_stream else "w:"
return mode_prefix + compression
diff --git a/src/buildstream/_version.py b/src/buildstream/_version.py
index 03f946cb8..10905c4ea 100644
--- a/src/buildstream/_version.py
+++ b/src/buildstream/_version.py
@@ -60,17 +60,18 @@ HANDLERS = {}
def register_vcs_handler(vcs, method): # decorator
"""Decorator to mark a method as the handler for a particular VCS."""
+
def decorate(f):
"""Store f in HANDLERS[vcs][method]."""
if vcs not in HANDLERS:
HANDLERS[vcs] = {}
HANDLERS[vcs][method] = f
return f
+
return decorate
-def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False,
- env=None):
+def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None):
"""Call the given command(s)."""
assert isinstance(commands, list)
p = None
@@ -78,10 +79,9 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False,
try:
dispcmd = str([c] + args)
# remember shell=False, so use git.cmd on windows, not just git
- p = subprocess.Popen([c] + args, cwd=cwd, env=env,
- stdout=subprocess.PIPE,
- stderr=(subprocess.PIPE if hide_stderr
- else None))
+ p = subprocess.Popen(
+ [c] + args, cwd=cwd, env=env, stdout=subprocess.PIPE, stderr=(subprocess.PIPE if hide_stderr else None)
+ )
break
except EnvironmentError:
e = sys.exc_info()[1]
@@ -118,16 +118,19 @@ def versions_from_parentdir(parentdir_prefix, root, verbose):
for i in range(3):
dirname = os.path.basename(root)
if dirname.startswith(parentdir_prefix):
- return {"version": dirname[len(parentdir_prefix):],
- "full-revisionid": None,
- "dirty": False, "error": None, "date": None}
+ return {
+ "version": dirname[len(parentdir_prefix) :],
+ "full-revisionid": None,
+ "dirty": False,
+ "error": None,
+ "date": None,
+ }
else:
rootdirs.append(root)
root = os.path.dirname(root) # up a level
if verbose:
- print("Tried directories %s but none started with prefix %s" %
- (str(rootdirs), parentdir_prefix))
+ print("Tried directories %s but none started with prefix %s" % (str(rootdirs), parentdir_prefix))
raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
@@ -183,7 +186,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
# starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
# just "foo-1.0". If we see a "tag: " prefix, prefer those.
TAG = "tag: "
- tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)])
+ tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)])
if not tags:
# Either we're using git < 1.8.3, or there really are no tags. We use
# a heuristic: assume all version tags have a digit. The old git %d
@@ -192,7 +195,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
# between branches and tags. By ignoring refnames without digits, we
# filter out many common branch names like "release" and
# "stabilization", as well as "HEAD" and "master".
- tags = set([r for r in refs if re.search(r'\d', r)])
+ tags = set([r for r in refs if re.search(r"\d", r)])
if verbose:
print("discarding '%s', no digits" % ",".join(refs - tags))
if verbose:
@@ -200,19 +203,26 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
for ref in sorted(tags):
# sorting will prefer e.g. "2.0" over "2.0rc1"
if ref.startswith(tag_prefix):
- r = ref[len(tag_prefix):]
+ r = ref[len(tag_prefix) :]
if verbose:
print("picking %s" % r)
- return {"version": r,
- "full-revisionid": keywords["full"].strip(),
- "dirty": False, "error": None,
- "date": date}
+ return {
+ "version": r,
+ "full-revisionid": keywords["full"].strip(),
+ "dirty": False,
+ "error": None,
+ "date": date,
+ }
# no suitable tags, so version is "0+unknown", but full hex is still there
if verbose:
print("no suitable tags, using unknown + full revision id")
- return {"version": "0+unknown",
- "full-revisionid": keywords["full"].strip(),
- "dirty": False, "error": "no suitable tags", "date": None}
+ return {
+ "version": "0+unknown",
+ "full-revisionid": keywords["full"].strip(),
+ "dirty": False,
+ "error": "no suitable tags",
+ "date": None,
+ }
@register_vcs_handler("git", "pieces_from_vcs")
@@ -227,8 +237,7 @@ def git_pieces_from_vcs(tag_prefix, tag_regex, root, verbose, run_command=run_co
if sys.platform == "win32":
GITS = ["git.cmd", "git.exe"]
- out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root,
- hide_stderr=True)
+ out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True)
if rc != 0:
if verbose:
print("Directory %s not under git control" % root)
@@ -236,10 +245,11 @@ def git_pieces_from_vcs(tag_prefix, tag_regex, root, verbose, run_command=run_co
# if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
# if there isn't one, this yields HEX[-dirty] (no NUM)
- describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty",
- "--always", "--long",
- "--match", "%s%s" % (tag_prefix, tag_regex)],
- cwd=root)
+ describe_out, rc = run_command(
+ GITS,
+ ["describe", "--tags", "--dirty", "--always", "--long", "--match", "%s%s" % (tag_prefix, tag_regex)],
+ cwd=root,
+ )
# --long was added in git-1.5.5
if describe_out is None:
raise NotThisMethod("'git describe' failed")
@@ -262,17 +272,16 @@ def git_pieces_from_vcs(tag_prefix, tag_regex, root, verbose, run_command=run_co
dirty = git_describe.endswith("-dirty")
pieces["dirty"] = dirty
if dirty:
- git_describe = git_describe[:git_describe.rindex("-dirty")]
+ git_describe = git_describe[: git_describe.rindex("-dirty")]
# now we have TAG-NUM-gHEX or HEX
if "-" in git_describe:
# TAG-NUM-gHEX
- mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
+ mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe)
if not mo:
# unparseable. Maybe git-describe is misbehaving?
- pieces["error"] = ("unable to parse git-describe output: '%s'"
- % describe_out)
+ pieces["error"] = "unable to parse git-describe output: '%s'" % describe_out
return pieces
# tag
@@ -281,10 +290,9 @@ def git_pieces_from_vcs(tag_prefix, tag_regex, root, verbose, run_command=run_co
if verbose:
fmt = "tag '%s' doesn't start with prefix '%s'"
print(fmt % (full_tag, tag_prefix))
- pieces["error"] = ("tag '%s' doesn't start with prefix '%s'"
- % (full_tag, tag_prefix))
+ pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % (full_tag, tag_prefix)
return pieces
- pieces["closest-tag"] = full_tag[len(tag_prefix):]
+ pieces["closest-tag"] = full_tag[len(tag_prefix) :]
# distance: number of commits since tag
pieces["distance"] = int(mo.group(2))
@@ -295,13 +303,11 @@ def git_pieces_from_vcs(tag_prefix, tag_regex, root, verbose, run_command=run_co
else:
# HEX: no tags
pieces["closest-tag"] = None
- count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"],
- cwd=root)
+ count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], cwd=root)
pieces["distance"] = int(count_out) # total number of commits
# commit date: see ISO-8601 comment in git_versions_from_keywords()
- date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"],
- cwd=root)[0].strip()
+ date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[0].strip()
pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
return pieces
@@ -332,8 +338,7 @@ def render_pep440(pieces):
rendered += ".dirty"
else:
# exception #1
- rendered = "0+untagged.%d.g%s" % (pieces["distance"],
- pieces["short"])
+ rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"])
if pieces["dirty"]:
rendered += ".dirty"
return rendered
@@ -447,11 +452,13 @@ def render_git_describe_long(pieces):
def render(pieces, style):
"""Render the given version pieces into the requested style."""
if pieces["error"]:
- return {"version": "unknown",
- "full-revisionid": pieces.get("long"),
- "dirty": None,
- "error": pieces["error"],
- "date": None}
+ return {
+ "version": "unknown",
+ "full-revisionid": pieces.get("long"),
+ "dirty": None,
+ "error": pieces["error"],
+ "date": None,
+ }
if not style or style == "default":
style = "pep440" # the default
@@ -471,9 +478,13 @@ def render(pieces, style):
else:
raise ValueError("unknown style '%s'" % style)
- return {"version": rendered, "full-revisionid": pieces["long"],
- "dirty": pieces["dirty"], "error": None,
- "date": pieces.get("date")}
+ return {
+ "version": rendered,
+ "full-revisionid": pieces["long"],
+ "dirty": pieces["dirty"],
+ "error": None,
+ "date": pieces.get("date"),
+ }
def get_versions():
@@ -487,8 +498,7 @@ def get_versions():
verbose = cfg.verbose
try:
- return git_versions_from_keywords(get_keywords(), cfg.tag_prefix,
- verbose)
+ return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, verbose)
except NotThisMethod:
pass
@@ -497,13 +507,16 @@ def get_versions():
# versionfile_source is the relative path from the top of the source
# tree (where the .git directory might live) to this file. Invert
# this to find the root from __file__.
- for i in cfg.versionfile_source.split('/'):
+ for i in cfg.versionfile_source.split("/"):
root = os.path.dirname(root)
except NameError:
- return {"version": "0+unknown", "full-revisionid": None,
- "dirty": None,
- "error": "unable to find root of source tree",
- "date": None}
+ return {
+ "version": "0+unknown",
+ "full-revisionid": None,
+ "dirty": None,
+ "error": "unable to find root of source tree",
+ "date": None,
+ }
try:
pieces = git_pieces_from_vcs(cfg.tag_prefix, cfg.tag_regex, root, verbose)
@@ -517,6 +530,10 @@ def get_versions():
except NotThisMethod:
pass
- return {"version": "0+unknown", "full-revisionid": None,
- "dirty": None,
- "error": "unable to compute version", "date": None}
+ return {
+ "version": "0+unknown",
+ "full-revisionid": None,
+ "dirty": None,
+ "error": "unable to compute version",
+ "date": None,
+ }
diff --git a/src/buildstream/_workspaces.py b/src/buildstream/_workspaces.py
index f9023dc54..3d50fd9c0 100644
--- a/src/buildstream/_workspaces.py
+++ b/src/buildstream/_workspaces.py
@@ -38,7 +38,7 @@ WORKSPACE_PROJECT_FILE = ".bstproject.yaml"
# Args:
# directory (str): The directory that the workspace exists in.
#
-class WorkspaceProject():
+class WorkspaceProject:
def __init__(self, directory):
self._projects = []
self._directory = directory
@@ -51,7 +51,7 @@ class WorkspaceProject():
# (str): The path to a project
#
def get_default_project_path(self):
- return self._projects[0]['project-path']
+ return self._projects[0]["project-path"]
# get_default_element()
#
@@ -61,7 +61,7 @@ class WorkspaceProject():
# (str): The name of an element
#
def get_default_element(self):
- return self._projects[0]['element-name']
+ return self._projects[0]["element-name"]
# to_dict()
#
@@ -72,8 +72,8 @@ class WorkspaceProject():
#
def to_dict(self):
ret = {
- 'projects': self._projects,
- 'format-version': BST_WORKSPACE_PROJECT_FORMAT_VERSION,
+ "projects": self._projects,
+ "format-version": BST_WORKSPACE_PROJECT_FORMAT_VERSION,
}
return ret
@@ -91,13 +91,14 @@ class WorkspaceProject():
@classmethod
def from_dict(cls, directory, dictionary):
# Only know how to handle one format-version at the moment.
- format_version = int(dictionary['format-version'])
- assert format_version == BST_WORKSPACE_PROJECT_FORMAT_VERSION, \
- "Format version {} not found in {}".format(BST_WORKSPACE_PROJECT_FORMAT_VERSION, dictionary)
+ format_version = int(dictionary["format-version"])
+ assert format_version == BST_WORKSPACE_PROJECT_FORMAT_VERSION, "Format version {} not found in {}".format(
+ BST_WORKSPACE_PROJECT_FORMAT_VERSION, dictionary
+ )
workspace_project = cls(directory)
- for item in dictionary['projects']:
- workspace_project.add_project(item['project-path'], item['element-name'])
+ for item in dictionary["projects"]:
+ workspace_project.add_project(item["project-path"], item["element-name"])
return workspace_project
@@ -145,15 +146,15 @@ class WorkspaceProject():
# element_name (str): The name of the element that the workspace belongs to.
#
def add_project(self, project_path, element_name):
- assert (project_path and element_name)
- self._projects.append({'project-path': project_path, 'element-name': element_name})
+ assert project_path and element_name
+ self._projects.append({"project-path": project_path, "element-name": element_name})
# WorkspaceProjectCache()
#
# A class to manage workspace project data for multiple workspaces.
#
-class WorkspaceProjectCache():
+class WorkspaceProjectCache:
def __init__(self):
self._projects = {} # Mapping of a workspace directory to its WorkspaceProject
@@ -216,8 +217,9 @@ class WorkspaceProjectCache():
def remove(self, directory):
workspace_project = self.get(directory)
if not workspace_project:
- raise LoadError("Failed to find a {} file to remove".format(WORKSPACE_PROJECT_FILE),
- LoadErrorReason.MISSING_FILE)
+ raise LoadError(
+ "Failed to find a {} file to remove".format(WORKSPACE_PROJECT_FILE), LoadErrorReason.MISSING_FILE
+ )
path = workspace_project.get_filename()
try:
os.unlink(path)
@@ -242,7 +244,7 @@ class WorkspaceProjectCache():
# changed between failed builds. Should be
# made obsolete with failed build artifacts.
#
-class Workspace():
+class Workspace:
def __init__(self, toplevel_project, *, last_successful=None, path=None, prepared=False, running_files=None):
self.prepared = prepared
self.last_successful = last_successful
@@ -260,11 +262,7 @@ class Workspace():
# (dict) A dict representation of the workspace
#
def to_dict(self):
- ret = {
- 'prepared': self.prepared,
- 'path': self._path,
- 'running_files': self.running_files
- }
+ ret = {"prepared": self.prepared, "path": self._path, "running_files": self.running_files}
if self.last_successful is not None:
ret["last_successful"] = self.last_successful
return ret
@@ -363,8 +361,7 @@ class Workspace():
try:
stat = os.lstat(filename)
except OSError as e:
- raise LoadError("Failed to stat file in workspace: {}".format(e),
- LoadErrorReason.MISSING_FILE)
+ raise LoadError("Failed to stat file in workspace: {}".format(e), LoadErrorReason.MISSING_FILE)
# Use the mtime of any file with sub second precision
return stat.st_mtime_ns
@@ -378,8 +375,7 @@ class Workspace():
if os.path.isdir(fullpath):
filelist = utils.list_relative_paths(fullpath)
filelist = [
- (relpath, os.path.join(fullpath, relpath)) for relpath in filelist
- if relpath not in excluded_files
+ (relpath, os.path.join(fullpath, relpath)) for relpath in filelist if relpath not in excluded_files
]
else:
filelist = [(self.get_absolute_path(), fullpath)]
@@ -404,7 +400,7 @@ class Workspace():
# toplevel_project (Project): Top project used to resolve paths.
# workspace_project_cache (WorkspaceProjectCache): The cache of WorkspaceProjects
#
-class Workspaces():
+class Workspaces:
def __init__(self, toplevel_project, workspace_project_cache):
self._toplevel_project = toplevel_project
self._bst_directory = os.path.join(toplevel_project.directory, ".bst")
@@ -525,11 +521,8 @@ class Workspaces():
assert utils._is_main_process()
config = {
- 'format-version': BST_WORKSPACE_FORMAT_VERSION,
- 'workspaces': {
- element: workspace.to_dict()
- for element, workspace in self._workspaces.items()
- }
+ "format-version": BST_WORKSPACE_FORMAT_VERSION,
+ "workspaces": {element: workspace.to_dict() for element, workspace in self._workspaces.items()},
}
os.makedirs(self._bst_directory, exist_ok=True)
_yaml.roundtrip_dump(config, self._get_filename())
@@ -572,10 +565,11 @@ class Workspaces():
#
def _parse_workspace_config(self, workspaces):
try:
- version = workspaces.get_int('format-version', default=0)
+ version = workspaces.get_int("format-version", default=0)
except ValueError:
- raise LoadError("Format version is not an integer in workspace configuration",
- LoadErrorReason.INVALID_DATA)
+ raise LoadError(
+ "Format version is not an integer in workspace configuration", LoadErrorReason.INVALID_DATA
+ )
if version == 0:
# Pre-versioning format can be of two forms
@@ -588,17 +582,17 @@ class Workspaces():
elif config_type is MappingNode:
sources = list(config.values())
if len(sources) > 1:
- detail = "There are multiple workspaces open for '{}'.\n" + \
- "This is not supported anymore.\n" + \
- "Please remove this element from '{}'."
- raise LoadError(detail.format(element, self._get_filename()),
- LoadErrorReason.INVALID_DATA)
+ detail = (
+ "There are multiple workspaces open for '{}'.\n"
+ + "This is not supported anymore.\n"
+ + "Please remove this element from '{}'."
+ )
+ raise LoadError(detail.format(element, self._get_filename()), LoadErrorReason.INVALID_DATA)
workspaces[element] = sources[0]
else:
- raise LoadError("Workspace config is in unexpected format.",
- LoadErrorReason.INVALID_DATA)
+ raise LoadError("Workspace config is in unexpected format.", LoadErrorReason.INVALID_DATA)
res = {
element: Workspace(self._toplevel_project, path=config.as_str())
@@ -607,13 +601,16 @@ class Workspaces():
elif 1 <= version <= BST_WORKSPACE_FORMAT_VERSION:
workspaces = workspaces.get_mapping("workspaces", default={})
- res = {element: self._load_workspace(node)
- for element, node in workspaces.items()}
+ res = {element: self._load_workspace(node) for element, node in workspaces.items()}
else:
- raise LoadError("Workspace configuration format version {} not supported."
- "Your version of buildstream may be too old. Max supported version: {}"
- .format(version, BST_WORKSPACE_FORMAT_VERSION), LoadErrorReason.INVALID_DATA)
+ raise LoadError(
+ "Workspace configuration format version {} not supported."
+ "Your version of buildstream may be too old. Max supported version: {}".format(
+ version, BST_WORKSPACE_FORMAT_VERSION
+ ),
+ LoadErrorReason.INVALID_DATA,
+ )
return res
@@ -628,15 +625,15 @@ class Workspaces():
# (Workspace): A newly instantiated Workspace
#
def _load_workspace(self, node):
- running_files = node.get_mapping('running_files', default=None)
+ running_files = node.get_mapping("running_files", default=None)
if running_files:
running_files = running_files.strip_node_info()
dictionary = {
- 'prepared': node.get_bool('prepared', default=False),
- 'path': node.get_str('path'),
- 'last_successful': node.get_str('last_successful', default=None),
- 'running_files': running_files,
+ "prepared": node.get_bool("prepared", default=False),
+ "path": node.get_str("path"),
+ "last_successful": node.get_str("last_successful", default=None),
+ "running_files": running_files,
}
return Workspace.from_dict(self._toplevel_project, dictionary)
diff --git a/src/buildstream/buildelement.py b/src/buildstream/buildelement.py
index 7fe97c168..f04d3b0dc 100644
--- a/src/buildstream/buildelement.py
+++ b/src/buildstream/buildelement.py
@@ -144,17 +144,16 @@ from .types import Scope
# This list is preserved because of an unfortunate situation, we
# need to remove these older commands which were secret and never
# documented, but without breaking the cache keys.
-_legacy_command_steps = ['bootstrap-commands',
- 'configure-commands',
- 'build-commands',
- 'test-commands',
- 'install-commands',
- 'strip-commands']
+_legacy_command_steps = [
+ "bootstrap-commands",
+ "configure-commands",
+ "build-commands",
+ "test-commands",
+ "install-commands",
+ "strip-commands",
+]
-_command_steps = ['configure-commands',
- 'build-commands',
- 'install-commands',
- 'strip-commands']
+_command_steps = ["configure-commands", "build-commands", "install-commands", "strip-commands"]
class BuildElement(Element):
@@ -190,21 +189,21 @@ class BuildElement(Element):
# cache key, while having the side effect of setting max-jobs to 1,
# which is normally automatically resolved and does not affect
# the cache key.
- if self.get_variable('notparallel'):
- dictionary['notparallel'] = True
+ if self.get_variable("notparallel"):
+ dictionary["notparallel"] = True
return dictionary
def configure_sandbox(self, sandbox):
- build_root = self.get_variable('build-root')
- install_root = self.get_variable('install-root')
+ build_root = self.get_variable("build-root")
+ install_root = self.get_variable("install-root")
# Tell the sandbox to mount the build root and install root
sandbox.mark_directory(build_root)
sandbox.mark_directory(install_root)
# Allow running all commands in a specified subdirectory
- command_subdir = self.get_variable('command-subdir')
+ command_subdir = self.get_variable("command-subdir")
if command_subdir:
command_dir = os.path.join(build_root, command_subdir)
else:
@@ -230,13 +229,13 @@ class BuildElement(Element):
dep.integrate(sandbox)
# Stage sources in the build root
- self.stage_sources(sandbox, self.get_variable('build-root'))
+ self.stage_sources(sandbox, self.get_variable("build-root"))
def assemble(self, sandbox):
# Run commands
for command_name in _command_steps:
commands = self.__commands[command_name]
- if not commands or command_name == 'configure-commands':
+ if not commands or command_name == "configure-commands":
continue
with sandbox.batch(SandboxFlags.ROOT_READ_ONLY, label="Running {}".format(command_name)):
@@ -247,21 +246,22 @@ class BuildElement(Element):
# to - if an element later attempts to stage to a location
# that is not empty, we abort the build - in this case this
# will almost certainly happen.
- staged_build = os.path.join(self.get_variable('install-root'),
- self.get_variable('build-root'))
+ staged_build = os.path.join(self.get_variable("install-root"), self.get_variable("build-root"))
if os.path.isdir(staged_build) and os.listdir(staged_build):
- self.warn("Writing to %{install-root}/%{build-root}.",
- detail="Writing to this directory will almost " +
- "certainly cause an error, since later elements " +
- "will not be allowed to stage to %{build-root}.")
+ self.warn(
+ "Writing to %{install-root}/%{build-root}.",
+ detail="Writing to this directory will almost "
+ + "certainly cause an error, since later elements "
+ + "will not be allowed to stage to %{build-root}.",
+ )
# Return the payload, this is configurable but is generally
# always the /buildstream-install directory
- return self.get_variable('install-root')
+ return self.get_variable("install-root")
def prepare(self, sandbox):
- commands = self.__commands['configure-commands']
+ commands = self.__commands["configure-commands"]
if commands:
with sandbox.batch(SandboxFlags.ROOT_READ_ONLY, label="Running configure-commands"):
for cmd in commands:
@@ -282,15 +282,10 @@ class BuildElement(Element):
#############################################################
def __get_commands(self, node, name):
raw_commands = node.get_sequence(name, [])
- return [
- self.node_subst_vars(command)
- for command in raw_commands
- ]
+ return [self.node_subst_vars(command) for command in raw_commands]
def __run_command(self, sandbox, cmd):
# Note the -e switch to 'sh' means to exit with an error
# if any untested command fails.
#
- sandbox.run(['sh', '-c', '-e', cmd + '\n'],
- SandboxFlags.ROOT_READ_ONLY,
- label=cmd)
+ sandbox.run(["sh", "-c", "-e", cmd + "\n"], SandboxFlags.ROOT_READ_ONLY, label=cmd)
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 5fa8f14df..5028cc5fa 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -90,8 +90,7 @@ from pyroaring import BitMap # pylint: disable=no-name-in-module
from . import _yaml
from ._variables import Variables
from ._versions import BST_CORE_ARTIFACT_VERSION
-from ._exceptions import BstError, LoadError, LoadErrorReason, ImplError, \
- ErrorDomain, SourceCacheError
+from ._exceptions import BstError, LoadError, LoadErrorReason, ImplError, ErrorDomain, SourceCacheError
from .utils import FileListResult
from . import utils
from . import _cachekey
@@ -122,6 +121,7 @@ if TYPE_CHECKING:
from ._context import Context
from ._loader.metaelement import MetaElement
from ._project import Project
+
# pylint: enable=cyclic-import
@@ -136,13 +136,10 @@ class ElementError(BstError):
collect: An optional directory containing partial install contents
temporary: An indicator to whether the error may occur if the operation was run again. (*Since: 1.2*)
"""
- def __init__(self,
- message: str,
- *,
- detail: str = None,
- reason: str = None,
- collect: str = None,
- temporary: bool = False):
+
+ def __init__(
+ self, message: str, *, detail: str = None, reason: str = None, collect: str = None, temporary: bool = False
+ ):
super().__init__(message, detail=detail, domain=ErrorDomain.ELEMENT, reason=reason, temporary=temporary)
self.collect = collect
@@ -156,12 +153,13 @@ class Element(Plugin):
All elements derive from this class, this interface defines how
the core will be interacting with Elements.
"""
+
# The defaults from the yaml file and project
__defaults = None
# A hash of Element by MetaElement
- __instantiated_elements = {} # type: Dict[MetaElement, Element]
+ __instantiated_elements = {} # type: Dict[MetaElement, Element]
# A list of (source, ref) tuples which were redundantly specified
- __redundant_source_refs = [] # type: List[Tuple[Source, SourceRef]]
+ __redundant_source_refs = [] # type: List[Tuple[Source, SourceRef]]
BST_ARTIFACT_VERSION = 0
"""The element plugin's artifact version
@@ -215,10 +213,10 @@ class Element(Plugin):
*Since: 1.90*
"""
- def __init__(self, context: 'Context', project: 'Project', meta: 'MetaElement', plugin_conf: Dict[str, Any]):
+ def __init__(self, context: "Context", project: "Project", meta: "MetaElement", plugin_conf: Dict[str, Any]):
- self.__cache_key_dict = None # Dict for cache key calculation
- self.__cache_key = None # Our cached cache key
+ self.__cache_key_dict = None # Dict for cache key calculation
+ self.__cache_key = None # Our cached cache key
super().__init__(meta.name, context, project, meta.provenance, "element")
@@ -236,75 +234,75 @@ class Element(Plugin):
"""
# Direct runtime dependency Elements
- self.__runtime_dependencies = [] # type: List[Element]
+ self.__runtime_dependencies = [] # type: List[Element]
# Direct build dependency Elements
- self.__build_dependencies = [] # type: List[Element]
+ self.__build_dependencies = [] # type: List[Element]
# Direct build dependency subset which require strict rebuilds
- self.__strict_dependencies = [] # type: List[Element]
+ self.__strict_dependencies = [] # type: List[Element]
# Direct reverse build dependency Elements
- self.__reverse_build_deps = set() # type: Set[Element]
+ self.__reverse_build_deps = set() # type: Set[Element]
# Direct reverse runtime dependency Elements
- self.__reverse_runtime_deps = set() # type: Set[Element]
- self.__build_deps_without_strict_cache_key = None # Number of build dependencies without a strict key
+ self.__reverse_runtime_deps = set() # type: Set[Element]
+ self.__build_deps_without_strict_cache_key = None # Number of build dependencies without a strict key
self.__runtime_deps_without_strict_cache_key = None # Number of runtime dependencies without a strict key
- self.__build_deps_without_cache_key = None # Number of build dependencies without a cache key
+ self.__build_deps_without_cache_key = None # Number of build dependencies without a cache key
self.__runtime_deps_without_cache_key = None # Number of runtime dependencies without a cache key
- self.__build_deps_uncached = None # Build dependencies which are not yet cached
+ self.__build_deps_uncached = None # Build dependencies which are not yet cached
self.__runtime_deps_uncached = None # Runtime dependencies which are not yet cached
self.__updated_strict_cache_keys_of_rdeps = False # Whether we've updated strict cache keys of rdeps
- self.__ready_for_runtime = False # Whether the element and its runtime dependencies have cache keys
+ self.__ready_for_runtime = False # Whether the element and its runtime dependencies have cache keys
self.__ready_for_runtime_and_cached = False # Whether all runtime deps are cached, as well as the element
- self.__cached_remotely = None # Whether the element is cached remotely
+ self.__cached_remotely = None # Whether the element is cached remotely
# List of Sources
- self.__sources = [] # type: List[Source]
- self.__weak_cache_key = None # Our cached weak cache key
- self.__strict_cache_key = None # Our cached cache key for strict builds
+ self.__sources = [] # type: List[Source]
+ self.__weak_cache_key = None # Our cached weak cache key
+ self.__strict_cache_key = None # Our cached cache key for strict builds
self.__artifacts = context.artifactcache # Artifact cache
self.__sourcecache = context.sourcecache # Source cache
self.__consistency = Consistency.INCONSISTENT # Cached overall consistency state
- self.__assemble_scheduled = False # Element is scheduled to be assembled
- self.__assemble_done = False # Element is assembled
- self.__tracking_scheduled = False # Sources are scheduled to be tracked
- self.__pull_done = False # Whether pull was attempted
- self.__cached_successfully = None # If the Element is known to be successfully cached
- self.__source_cached = None # If the sources are known to be successfully cached
- self.__splits = None # Resolved regex objects for computing split domains
- self.__whitelist_regex = None # Resolved regex object to check if file is allowed to overlap
+ self.__assemble_scheduled = False # Element is scheduled to be assembled
+ self.__assemble_done = False # Element is assembled
+ self.__tracking_scheduled = False # Sources are scheduled to be tracked
+ self.__pull_done = False # Whether pull was attempted
+ self.__cached_successfully = None # If the Element is known to be successfully cached
+ self.__source_cached = None # If the sources are known to be successfully cached
+ self.__splits = None # Resolved regex objects for computing split domains
+ self.__whitelist_regex = None # Resolved regex object to check if file is allowed to overlap
# Location where Element.stage_sources() was called
self.__staged_sources_directory = None # type: Optional[str]
- self.__tainted = None # Whether the artifact is tainted and should not be shared
- self.__required = False # Whether the artifact is required in the current session
+ self.__tainted = None # Whether the artifact is tainted and should not be shared
+ self.__required = False # Whether the artifact is required in the current session
self.__artifact_files_required = False # Whether artifact files are required in the local cache
- self.__build_result = None # The result of assembling this Element (success, description, detail)
- self._build_log_path = None # The path of the build log for this Element
+ self.__build_result = None # The result of assembling this Element (success, description, detail)
+ self._build_log_path = None # The path of the build log for this Element
# Artifact class for direct artifact composite interaction
- self.__artifact = None # type: Optional[Artifact]
- self.__strict_artifact = None # Artifact for strict cache key
- self.__meta_kind = meta.kind # The kind of this source, required for unpickling
+ self.__artifact = None # type: Optional[Artifact]
+ self.__strict_artifact = None # Artifact for strict cache key
+ self.__meta_kind = meta.kind # The kind of this source, required for unpickling
# the index of the last source in this element that requires previous
# sources for staging
self.__last_source_requires_previous_ix = None
- self.__batch_prepare_assemble = False # Whether batching across prepare()/assemble() is configured
- self.__batch_prepare_assemble_flags = 0 # Sandbox flags for batching across prepare()/assemble()
+ self.__batch_prepare_assemble = False # Whether batching across prepare()/assemble() is configured
+ self.__batch_prepare_assemble_flags = 0 # Sandbox flags for batching across prepare()/assemble()
# Collect dir for batching across prepare()/assemble()
self.__batch_prepare_assemble_collect = None # type: Optional[str]
# Callbacks
- self.__required_callback = None # Callback to Queues
- self.__can_query_cache_callback = None # Callback to PullQueue/FetchQueue
- self.__buildable_callback = None # Callback to BuildQueue
+ self.__required_callback = None # Callback to Queues
+ self.__can_query_cache_callback = None # Callback to PullQueue/FetchQueue
+ self.__buildable_callback = None # Callback to BuildQueue
- self._depth = None # Depth of Element in its current dependency graph
- self._resolved_initial_state = False # Whether the initial state of the Element has been resolved
+ self._depth = None # Depth of Element in its current dependency graph
+ self._resolved_initial_state = False # Whether the initial state of the Element has been resolved
# Ensure we have loaded this class's defaults
self.__init_defaults(project, plugin_conf, meta.kind, meta.is_junction)
# Collect the composited variables and resolve them
variables = self.__extract_variables(project, meta)
- variables['element-name'] = self.name
+ variables["element-name"] = self.name
self.__variables = Variables(variables)
# Collect the composited environment now that we have variables
@@ -348,7 +346,7 @@ class Element(Plugin):
#############################################################
# Abstract Methods #
#############################################################
- def configure_sandbox(self, sandbox: 'Sandbox') -> None:
+ def configure_sandbox(self, sandbox: "Sandbox") -> None:
"""Configures the the sandbox for execution
Args:
@@ -360,10 +358,9 @@ class Element(Plugin):
Elements must implement this method to configure the sandbox object
for execution.
"""
- raise ImplError("element plugin '{kind}' does not implement configure_sandbox()".format(
- kind=self.get_kind()))
+ raise ImplError("element plugin '{kind}' does not implement configure_sandbox()".format(kind=self.get_kind()))
- def stage(self, sandbox: 'Sandbox') -> None:
+ def stage(self, sandbox: "Sandbox") -> None:
"""Stage inputs into the sandbox directories
Args:
@@ -377,10 +374,9 @@ class Element(Plugin):
objects, by staging the artifacts of the elements this element depends
on, or both.
"""
- raise ImplError("element plugin '{kind}' does not implement stage()".format(
- kind=self.get_kind()))
+ raise ImplError("element plugin '{kind}' does not implement stage()".format(kind=self.get_kind()))
- def prepare(self, sandbox: 'Sandbox') -> None:
+ def prepare(self, sandbox: "Sandbox") -> None:
"""Run one-off preparation commands.
This is run before assemble(), but is guaranteed to run only
@@ -400,7 +396,7 @@ class Element(Plugin):
*Since: 1.2*
"""
- def assemble(self, sandbox: 'Sandbox') -> str:
+ def assemble(self, sandbox: "Sandbox") -> str:
"""Assemble the output artifact
Args:
@@ -415,8 +411,7 @@ class Element(Plugin):
Elements must implement this method to create an output
artifact from its sources and dependencies.
"""
- raise ImplError("element plugin '{kind}' does not implement assemble()".format(
- kind=self.get_kind()))
+ raise ImplError("element plugin '{kind}' does not implement assemble()".format(kind=self.get_kind()))
def generate_script(self) -> str:
"""Generate a build (sh) script to build this element
@@ -437,13 +432,12 @@ class Element(Plugin):
If the script fails, it is expected to return with an exit
code != 0.
"""
- raise ImplError("element plugin '{kind}' does not implement write_script()".format(
- kind=self.get_kind()))
+ raise ImplError("element plugin '{kind}' does not implement write_script()".format(kind=self.get_kind()))
#############################################################
# Public Methods #
#############################################################
- def sources(self) -> Iterator['Source']:
+ def sources(self) -> Iterator["Source"]:
"""A generator function to enumerate the element sources
Yields:
@@ -452,7 +446,7 @@ class Element(Plugin):
for source in self.__sources:
yield source
- def dependencies(self, scope: Scope, *, recurse: bool = True, visited=None) -> Iterator['Element']:
+ def dependencies(self, scope: Scope, *, recurse: bool = True, visited=None) -> Iterator["Element"]:
"""dependencies(scope, *, recurse=True)
A generator function which yields the dependencies of the given element.
@@ -479,6 +473,7 @@ class Element(Plugin):
if scope in (Scope.RUN, Scope.ALL):
yield from self.__runtime_dependencies
else:
+
def visit(element, scope, visited):
if scope == Scope.ALL:
visited[0].add(element._unique_id)
@@ -519,7 +514,7 @@ class Element(Plugin):
yield from visit(self, scope, visited)
- def search(self, scope: Scope, name: str) -> Optional['Element']:
+ def search(self, scope: Scope, name: str) -> Optional["Element"]:
"""Search for a dependency by name
Args:
@@ -535,7 +530,7 @@ class Element(Plugin):
return None
- def node_subst_vars(self, node: 'ScalarNode') -> str:
+ def node_subst_vars(self, node: "ScalarNode") -> str:
"""Replace any variables in the string contained in the node and returns it.
Args:
@@ -559,9 +554,9 @@ class Element(Plugin):
return self.__variables.subst(node.as_str())
except LoadError as e:
provenance = node.get_provenance()
- raise LoadError('{}: {}'.format(provenance, e), e.reason, detail=e.detail) from e
+ raise LoadError("{}: {}".format(provenance, e), e.reason, detail=e.detail) from e
- def node_subst_sequence_vars(self, node: 'SequenceNode[ScalarNode]') -> List[str]:
+ def node_subst_sequence_vars(self, node: "SequenceNode[ScalarNode]") -> List[str]:
"""Substitute any variables in the given sequence
Args:
@@ -580,14 +575,12 @@ class Element(Plugin):
ret.append(self.__variables.subst(value.as_str()))
except LoadError as e:
provenance = value.get_provenance()
- raise LoadError('{}: {}'.format(provenance, e), e.reason, detail=e.detail) from e
+ raise LoadError("{}: {}".format(provenance, e), e.reason, detail=e.detail) from e
return ret
- def compute_manifest(self,
- *,
- include: Optional[List[str]] = None,
- exclude: Optional[List[str]] = None,
- orphans: bool = True) -> str:
+ def compute_manifest(
+ self, *, include: Optional[List[str]] = None, exclude: Optional[List[str]] = None, orphans: bool = True
+ ) -> str:
"""Compute and return this element's selective manifest
The manifest consists on the list of file paths in the
@@ -630,14 +623,16 @@ class Element(Plugin):
return _compose_artifact_name(self.project_name, self.normal_name, key)
- def stage_artifact(self,
- sandbox: 'Sandbox',
- *,
- path: str = None,
- include: Optional[List[str]] = None,
- exclude: Optional[List[str]] = None,
- orphans: bool = True,
- update_mtimes: Optional[List[str]] = None) -> FileListResult:
+ def stage_artifact(
+ self,
+ sandbox: "Sandbox",
+ *,
+ path: str = None,
+ include: Optional[List[str]] = None,
+ exclude: Optional[List[str]] = None,
+ orphans: bool = True,
+ update_mtimes: Optional[List[str]] = None
+ ) -> FileListResult:
"""Stage this element's output artifact in the sandbox
This will stage the files from the artifact to the sandbox at specified location.
@@ -675,10 +670,11 @@ class Element(Plugin):
"""
if not self._cached():
- detail = "No artifacts have been cached yet for that element\n" + \
- "Try building the element first with `bst build`\n"
- raise ElementError("No artifacts to stage",
- detail=detail, reason="uncached-checkout-attempt")
+ detail = (
+ "No artifacts have been cached yet for that element\n"
+ + "Try building the element first with `bst build`\n"
+ )
+ raise ElementError("No artifacts to stage", detail=detail, reason="uncached-checkout-attempt")
if update_mtimes is None:
update_mtimes = []
@@ -689,47 +685,49 @@ class Element(Plugin):
with self.timed_activity("Staging {}/{}".format(self.name, self._get_brief_display_key())):
# Disable type checking since we can't easily tell mypy that
# `self.__artifact` can't be None at this stage.
- files_vdir = self.__artifact.get_files() # type: ignore
+ files_vdir = self.__artifact.get_files() # type: ignore
# Hard link it into the staging area
#
vbasedir = sandbox.get_virtual_directory()
- vstagedir = vbasedir \
- if path is None \
- else vbasedir.descend(*path.lstrip(os.sep).split(os.sep))
+ vstagedir = vbasedir if path is None else vbasedir.descend(*path.lstrip(os.sep).split(os.sep))
split_filter = self.__split_filter_func(include, exclude, orphans)
# We must not hardlink files whose mtimes we want to update
if update_mtimes:
+
def link_filter(path):
- return ((split_filter is None or split_filter(path)) and
- path not in update_mtimes)
+ return (split_filter is None or split_filter(path)) and path not in update_mtimes
def copy_filter(path):
- return ((split_filter is None or split_filter(path)) and
- path in update_mtimes)
+ return (split_filter is None or split_filter(path)) and path in update_mtimes
+
else:
link_filter = split_filter
- result = vstagedir.import_files(files_vdir, filter_callback=link_filter,
- report_written=True, can_link=True)
+ result = vstagedir.import_files(
+ files_vdir, filter_callback=link_filter, report_written=True, can_link=True
+ )
if update_mtimes:
- copy_result = vstagedir.import_files(files_vdir, filter_callback=copy_filter,
- report_written=True, update_mtime=True)
+ copy_result = vstagedir.import_files(
+ files_vdir, filter_callback=copy_filter, report_written=True, update_mtime=True
+ )
result = result.combine(copy_result)
return result
- def stage_dependency_artifacts(self,
- sandbox: 'Sandbox',
- scope: Scope,
- *,
- path: str = None,
- include: Optional[List[str]] = None,
- exclude: Optional[List[str]] = None,
- orphans: bool = True) -> None:
+ def stage_dependency_artifacts(
+ self,
+ sandbox: "Sandbox",
+ scope: Scope,
+ *,
+ path: str = None,
+ include: Optional[List[str]] = None,
+ exclude: Optional[List[str]] = None,
+ orphans: bool = True
+ ) -> None:
"""Stage element dependencies in scope
This is primarily a convenience wrapper around
@@ -751,8 +749,8 @@ class Element(Plugin):
occur.
"""
ignored = {}
- overlaps = OrderedDict() # type: OrderedDict[str, List[str]]
- files_written = {} # type: Dict[str, List[str]]
+ overlaps = OrderedDict() # type: OrderedDict[str, List[str]]
+ files_written = {} # type: Dict[str, List[str]]
old_dep_keys = None
workspace = self._get_workspace()
context = self._get_context()
@@ -803,12 +801,9 @@ class Element(Plugin):
if utils._is_main_process():
context.get_workspaces().save_config()
- result = dep.stage_artifact(sandbox,
- path=path,
- include=include,
- exclude=exclude,
- orphans=orphans,
- update_mtimes=to_update)
+ result = dep.stage_artifact(
+ sandbox, path=path, include=include, exclude=exclude, orphans=orphans, update_mtimes=to_update
+ )
if result.overwritten:
for overwrite in result.overwritten:
# Completely new overwrite
@@ -841,8 +836,9 @@ class Element(Plugin):
warning_detail += _overlap_error_detail(f, overlap_warning_elements, elements)
if overlap_warning:
- self.warn("Non-whitelisted overlaps detected", detail=warning_detail,
- warning_token=CoreWarnings.OVERLAPS)
+ self.warn(
+ "Non-whitelisted overlaps detected", detail=warning_detail, warning_token=CoreWarnings.OVERLAPS
+ )
if ignored:
detail = "Not staging files which would replace non-empty directories:\n"
@@ -851,7 +847,7 @@ class Element(Plugin):
detail += " " + " ".join(["/" + f + "\n" for f in value])
self.warn("Ignored files", detail=detail)
- def integrate(self, sandbox: 'Sandbox') -> None:
+ def integrate(self, sandbox: "Sandbox") -> None:
"""Integrate currently staged filesystem against this artifact.
Args:
@@ -863,19 +859,18 @@ class Element(Plugin):
commands will create and update important system cache files
required for running the installed software (such as the ld.so.cache).
"""
- bstdata = self.get_public_data('bst')
+ bstdata = self.get_public_data("bst")
environment = self.get_environment()
if bstdata is not None:
with sandbox.batch(SandboxFlags.NONE):
- commands = bstdata.get_sequence('integration-commands', [])
+ commands = bstdata.get_sequence("integration-commands", [])
for command in commands:
cmd = self.node_subst_vars(command)
- sandbox.run(['sh', '-e', '-c', cmd], 0, env=environment, cwd='/',
- label=cmd)
+ sandbox.run(["sh", "-e", "-c", cmd], 0, env=environment, cwd="/", label=cmd)
- def stage_sources(self, sandbox: 'Sandbox', directory: str) -> None:
+ def stage_sources(self, sandbox: "Sandbox", directory: str) -> None:
"""Stage this element's sources to a directory in the sandbox
Args:
@@ -892,7 +887,7 @@ class Element(Plugin):
self._stage_sources_in_sandbox(sandbox, directory)
- def get_public_data(self, domain: str) -> 'MappingNode[Any, Any]':
+ def get_public_data(self, domain: str) -> "MappingNode[Any, Any]":
"""Fetch public data on this element
Args:
@@ -911,13 +906,13 @@ class Element(Plugin):
# Disable type-checking since we can't easily tell mypy that
# `self.__dynamic_public` can't be None here.
- data = self.__dynamic_public.get_mapping(domain, default=None) # type: ignore
+ data = self.__dynamic_public.get_mapping(domain, default=None) # type: ignore
if data is not None:
data = data.clone()
return data
- def set_public_data(self, domain: str, data: 'MappingNode[Any, Any]') -> None:
+ def set_public_data(self, domain: str, data: "MappingNode[Any, Any]") -> None:
"""Set public data on this element
Args:
@@ -935,7 +930,7 @@ class Element(Plugin):
if data is not None:
data = data.clone()
- self.__dynamic_public[domain] = data # type: ignore
+ self.__dynamic_public[domain] = data # type: ignore
def get_environment(self) -> Dict[str, str]:
"""Fetch the environment suitable for running in the sandbox
@@ -1016,8 +1011,7 @@ class Element(Plugin):
# Instantiate sources and generate their keys
for meta_source in meta.sources:
meta_source.first_pass = meta.is_junction
- source = meta.project.create_source(meta_source,
- first_pass=meta.first_pass)
+ source = meta.project.create_source(meta_source, first_pass=meta.first_pass)
redundant_ref = source._load_ref()
@@ -1190,8 +1184,7 @@ class Element(Plugin):
# (bool): Whether this element can currently be built
#
def _buildable(self):
- if self._get_consistency() < Consistency.CACHED and \
- not self._source_cached():
+ if self._get_consistency() < Consistency.CACHED and not self._source_cached():
return False
if not self.__assemble_scheduled:
@@ -1261,11 +1254,14 @@ class Element(Plugin):
# If the element wasn't assembled and isn't scheduled to be assemble,
# or cached, or waiting to be pulled but has an artifact then schedule
# the assembly.
- if (not self.__assemble_scheduled and not self.__assemble_done and
- self.__artifact and
- self._is_required() and
- not self._cached() and
- not self._pull_pending()):
+ if (
+ not self.__assemble_scheduled
+ and not self.__assemble_done
+ and self.__artifact
+ and self._is_required()
+ and not self._cached()
+ and not self._pull_pending()
+ ):
self._schedule_assemble()
# If a build has been scheduled, we know that the element
@@ -1298,7 +1294,7 @@ class Element(Plugin):
cache_key = self._get_cache_key()
if not cache_key:
- cache_key = "{:?<64}".format('')
+ cache_key = "{:?<64}".format("")
elif cache_key == self.__strict_cache_key:
# Strong cache key used in this session matches cache key
# that would be used in strict build mode
@@ -1378,8 +1374,10 @@ class Element(Plugin):
# Complimentary warning that the new ref will be unused.
if old_ref != new_ref and self._get_workspace():
- detail = "This source has an open workspace.\n" \
+ detail = (
+ "This source has an open workspace.\n"
+ "To start using the new reference, please close the existing workspace."
+ )
source.warn("Updated reference will be ignored as source has open workspace", detail=detail)
return refs
@@ -1393,8 +1391,9 @@ class Element(Plugin):
def _prepare_sandbox(self, scope, directory, shell=False, integrate=True, usebuildtree=False):
# bst shell and bst artifact checkout require a local sandbox.
bare_directory = bool(directory)
- with self.__sandbox(directory, config=self.__sandbox_config, allow_remote=False,
- bare_directory=bare_directory) as sandbox:
+ with self.__sandbox(
+ directory, config=self.__sandbox_config, allow_remote=False, bare_directory=bare_directory
+ ) as sandbox:
sandbox._usebuildtree = usebuildtree
# Configure always comes first, and we need it.
@@ -1452,8 +1451,9 @@ class Element(Plugin):
# It's advantageous to have this temporary directory on
# the same file system as the rest of our cache.
- with self.timed_activity("Staging sources", silent_nested=True), \
- utils._tempdir(dir=context.tmpdir, prefix='staging-temp') as temp_staging_directory:
+ with self.timed_activity("Staging sources", silent_nested=True), utils._tempdir(
+ dir=context.tmpdir, prefix="staging-temp"
+ ) as temp_staging_directory:
import_dir = temp_staging_directory
@@ -1488,12 +1488,12 @@ class Element(Plugin):
import_dir.import_files(source_dir)
except SourceCacheError as e:
- raise ElementError("Error trying to export source for {}: {}"
- .format(self.name, e))
+ raise ElementError("Error trying to export source for {}: {}".format(self.name, e))
except VirtualDirectoryError as e:
- raise ElementError("Error trying to import sources together for {}: {}"
- .format(self.name, e),
- reason="import-source-files-fail")
+ raise ElementError(
+ "Error trying to import sources together for {}: {}".format(self.name, e),
+ reason="import-source-files-fail",
+ )
with utils._deterministic_umask():
vdirectory.import_files(import_dir)
@@ -1601,8 +1601,7 @@ class Element(Plugin):
self._update_ready_for_runtime_and_cached()
if self._get_workspace() and self._cached_success():
- assert utils._is_main_process(), \
- "Attempted to save workspace configuration from child process"
+ assert utils._is_main_process(), "Attempted to save workspace configuration from child process"
#
# Note that this block can only happen in the
# main process, since `self._cached_success()` cannot
@@ -1638,9 +1637,12 @@ class Element(Plugin):
with self._output_file() as output_file:
if not self.__sandbox_config_supported:
- self.warn("Sandbox configuration is not supported by the platform.",
- detail="Falling back to UID {} GID {}. Artifact will not be pushed."
- .format(self.__sandbox_config.build_uid, self.__sandbox_config.build_gid))
+ self.warn(
+ "Sandbox configuration is not supported by the platform.",
+ detail="Falling back to UID {} GID {}. Artifact will not be pushed.".format(
+ self.__sandbox_config.build_uid, self.__sandbox_config.build_gid
+ ),
+ )
# Explicitly clean it up, keep the build dir around if exceptions are raised
os.makedirs(context.builddir, exist_ok=True)
@@ -1650,13 +1652,14 @@ class Element(Plugin):
def cleanup_rootdir():
utils._force_rmtree(rootdir)
- with _signals.terminator(cleanup_rootdir), \
- self.__sandbox(rootdir, output_file, output_file, self.__sandbox_config) as sandbox: # noqa
+ with _signals.terminator(cleanup_rootdir), self.__sandbox(
+ rootdir, output_file, output_file, self.__sandbox_config
+ ) as sandbox: # noqa
# Let the sandbox know whether the buildtree will be required.
# This allows the remote execution sandbox to skip buildtree
# download when it's not needed.
- buildroot = self.get_variable('build-root')
+ buildroot = self.get_variable("build-root")
cache_buildtrees = context.cache_buildtrees
if cache_buildtrees != _CacheBuildTrees.NEVER:
always_cache_buildtrees = cache_buildtrees == _CacheBuildTrees.ALWAYS
@@ -1681,8 +1684,9 @@ class Element(Plugin):
self.stage(sandbox)
try:
if self.__batch_prepare_assemble:
- cm = sandbox.batch(self.__batch_prepare_assemble_flags,
- collect=self.__batch_prepare_assemble_collect)
+ cm = sandbox.batch(
+ self.__batch_prepare_assemble_flags, collect=self.__batch_prepare_assemble_collect
+ )
else:
cm = contextlib.suppress()
@@ -1724,11 +1728,13 @@ class Element(Plugin):
# result. Element types without a build-root dir will be cached
# with an empty buildtreedir regardless of this configuration.
- if cache_buildtrees == _CacheBuildTrees.ALWAYS or \
- (cache_buildtrees == _CacheBuildTrees.AUTO and not build_success):
+ if cache_buildtrees == _CacheBuildTrees.ALWAYS or (
+ cache_buildtrees == _CacheBuildTrees.AUTO and not build_success
+ ):
try:
sandbox_build_dir = sandbox_vroot.descend(
- *self.get_variable('build-root').lstrip(os.sep).split(os.sep))
+ *self.get_variable("build-root").lstrip(os.sep).split(os.sep)
+ )
sandbox._fetch_missing_blobs(sandbox_build_dir)
except VirtualDirectoryError:
# Directory could not be found. Pre-virtual
@@ -1747,14 +1753,13 @@ class Element(Plugin):
self._assemble_done()
with self.timed_activity("Caching artifact"):
- artifact_size = self.__artifact.cache(rootdir, sandbox_build_dir, collectvdir,
- buildresult, publicdata)
+ artifact_size = self.__artifact.cache(rootdir, sandbox_build_dir, collectvdir, buildresult, publicdata)
if collect is not None and collectvdir is None:
raise ElementError(
"Directory '{}' was not found inside the sandbox, "
- "unable to collect artifact contents"
- .format(collect))
+ "unable to collect artifact contents".format(collect)
+ )
return artifact_size
@@ -1855,8 +1860,7 @@ class Element(Plugin):
def _skip_source_push(self):
if not self.__sources or self._get_workspace():
return True
- return not (self.__sourcecache.has_push_remotes(plugin=self) and
- self._source_cached())
+ return not (self.__sourcecache.has_push_remotes(plugin=self) and self._source_cached())
def _source_push(self):
# try and push sources if we've got them
@@ -1931,8 +1935,9 @@ class Element(Plugin):
# Returns: Exit code
#
# If directory is not specified, one will be staged using scope
- def _shell(self, scope=None, directory=None, *, mounts=None, isolate=False, prompt=None, command=None,
- usebuildtree=False):
+ def _shell(
+ self, scope=None, directory=None, *, mounts=None, isolate=False, prompt=None, command=None, usebuildtree=False
+ ):
with self._prepare_sandbox(scope, directory, shell=True, usebuildtree=usebuildtree) as sandbox:
environment = self.get_environment()
@@ -1946,7 +1951,7 @@ class Element(Plugin):
shell_command, shell_environment, shell_host_files = project.get_shell_config()
if prompt is not None:
- environment['PS1'] = prompt
+ environment["PS1"] = prompt
# Special configurations for non-isolated sandboxes
if not isolate:
@@ -2002,8 +2007,7 @@ class Element(Plugin):
# additional support from Source implementations.
#
os.makedirs(context.builddir, exist_ok=True)
- with utils._tempdir(dir=context.builddir, prefix='workspace-{}'
- .format(self.normal_name)) as temp:
+ with utils._tempdir(dir=context.builddir, prefix="workspace-{}".format(self.normal_name)) as temp:
for source in self.sources():
source._init_workspace(temp)
@@ -2032,10 +2036,10 @@ class Element(Plugin):
script = script_template.format(
name=self.normal_name,
- build_root=self.get_variable('build-root'),
- install_root=self.get_variable('install-root'),
+ build_root=self.get_variable("build-root"),
+ install_root=self.get_variable("install-root"),
variables=variable_string,
- commands=self.generate_script()
+ commands=self.generate_script(),
)
os.makedirs(directory, exist_ok=True)
@@ -2120,8 +2124,7 @@ class Element(Plugin):
continue
# try and fetch from source cache
- if source._get_consistency() < Consistency.CACHED and \
- self.__sourcecache.has_fetch_remotes():
+ if source._get_consistency() < Consistency.CACHED and self.__sourcecache.has_fetch_remotes():
if self.__sourcecache.pull(source):
continue
@@ -2154,35 +2157,31 @@ class Element(Plugin):
# Generate dict that is used as base for all cache keys
if self.__cache_key_dict is None:
# Filter out nocache variables from the element's environment
- cache_env = {
- key: value
- for key, value in self.__environment.items()
- if key not in self.__env_nocache
- }
+ cache_env = {key: value for key, value in self.__environment.items() if key not in self.__env_nocache}
project = self._get_project()
self.__cache_key_dict = {
- 'core-artifact-version': BST_CORE_ARTIFACT_VERSION,
- 'element-plugin-key': self.get_unique_key(),
- 'element-plugin-name': self.get_kind(),
- 'element-plugin-version': self.BST_ARTIFACT_VERSION,
- 'sandbox': self.__sandbox_config.get_unique_key(),
- 'environment': cache_env,
- 'public': self.__public.strip_node_info()
+ "core-artifact-version": BST_CORE_ARTIFACT_VERSION,
+ "element-plugin-key": self.get_unique_key(),
+ "element-plugin-name": self.get_kind(),
+ "element-plugin-version": self.BST_ARTIFACT_VERSION,
+ "sandbox": self.__sandbox_config.get_unique_key(),
+ "environment": cache_env,
+ "public": self.__public.strip_node_info(),
}
- self.__cache_key_dict['sources'] = []
+ self.__cache_key_dict["sources"] = []
for source in self.__sources:
- self.__cache_key_dict['sources'].append(
- {'key': source._get_unique_key(),
- 'name': source._get_source_name()})
+ self.__cache_key_dict["sources"].append(
+ {"key": source._get_unique_key(), "name": source._get_source_name()}
+ )
- self.__cache_key_dict['fatal-warnings'] = sorted(project._fatal_warnings)
+ self.__cache_key_dict["fatal-warnings"] = sorted(project._fatal_warnings)
cache_key_dict = self.__cache_key_dict.copy()
- cache_key_dict['dependencies'] = dependencies
+ cache_key_dict["dependencies"] = dependencies
return _cachekey.generate_key(cache_key_dict)
@@ -2216,8 +2215,9 @@ class Element(Plugin):
Args:
fetch_original (bool): whether we need to original unstaged source
"""
- if (self._get_consistency() == Consistency.CACHED and fetch_original) or \
- (self._source_cached() and not fetch_original):
+ if (self._get_consistency() == Consistency.CACHED and fetch_original) or (
+ self._source_cached() and not fetch_original
+ ):
return False
else:
return True
@@ -2299,8 +2299,7 @@ class Element(Plugin):
#
def _update_ready_for_runtime_and_cached(self):
if not self.__ready_for_runtime_and_cached:
- if self.__runtime_deps_uncached == 0 and self._cached_success() and \
- self.__cache_key:
+ if self.__runtime_deps_uncached == 0 and self._cached_success() and self.__cache_key:
self.__ready_for_runtime_and_cached = True
# Notify reverse dependencies
@@ -2450,6 +2449,7 @@ class Element(Plugin):
self.prepare(sandbox)
if workspace:
+
def mark_workspace_prepared():
workspace.prepared = True
@@ -2466,23 +2466,31 @@ class Element(Plugin):
if self.BST_FORBID_RDEPENDS and self.BST_FORBID_BDEPENDS:
if any(self.dependencies(Scope.RUN, recurse=False)) or any(self.dependencies(Scope.BUILD, recurse=False)):
- raise ElementError("{}: Dependencies are forbidden for '{}' elements"
- .format(self, self.get_kind()), reason="element-forbidden-depends")
+ raise ElementError(
+ "{}: Dependencies are forbidden for '{}' elements".format(self, self.get_kind()),
+ reason="element-forbidden-depends",
+ )
if self.BST_FORBID_RDEPENDS:
if any(self.dependencies(Scope.RUN, recurse=False)):
- raise ElementError("{}: Runtime dependencies are forbidden for '{}' elements"
- .format(self, self.get_kind()), reason="element-forbidden-rdepends")
+ raise ElementError(
+ "{}: Runtime dependencies are forbidden for '{}' elements".format(self, self.get_kind()),
+ reason="element-forbidden-rdepends",
+ )
if self.BST_FORBID_BDEPENDS:
if any(self.dependencies(Scope.BUILD, recurse=False)):
- raise ElementError("{}: Build dependencies are forbidden for '{}' elements"
- .format(self, self.get_kind()), reason="element-forbidden-bdepends")
+ raise ElementError(
+ "{}: Build dependencies are forbidden for '{}' elements".format(self, self.get_kind()),
+ reason="element-forbidden-bdepends",
+ )
if self.BST_FORBID_SOURCES:
if any(self.sources()):
- raise ElementError("{}: Sources are forbidden for '{}' elements"
- .format(self, self.get_kind()), reason="element-forbidden-sources")
+ raise ElementError(
+ "{}: Sources are forbidden for '{}' elements".format(self, self.get_kind()),
+ reason="element-forbidden-sources",
+ )
try:
self.preflight()
@@ -2492,9 +2500,10 @@ class Element(Plugin):
# Ensure that the first source does not need access to previous soruces
if self.__sources and self.__sources[0]._requires_previous_sources():
- raise ElementError("{}: {} cannot be the first source of an element "
- "as it requires access to previous sources"
- .format(self, self.__sources[0]))
+ raise ElementError(
+ "{}: {} cannot be the first source of an element "
+ "as it requires access to previous sources".format(self, self.__sources[0])
+ )
# Preflight the sources
for source in self.sources():
@@ -2505,8 +2514,7 @@ class Element(Plugin):
# Raises an error if the artifact is not cached.
#
def __assert_cached(self):
- assert self._cached(), "{}: Missing artifact {}".format(
- self, self._get_brief_display_key())
+ assert self._cached(), "{}: Missing artifact {}".format(self, self._get_brief_display_key())
# __get_tainted():
#
@@ -2532,8 +2540,7 @@ class Element(Plugin):
workspaced_dependencies = self.__artifact.get_metadata_workspaced_dependencies()
# Other conditions should be or-ed
- self.__tainted = (workspaced or workspaced_dependencies or
- not self.__sandbox_config_supported)
+ self.__tainted = workspaced or workspaced_dependencies or not self.__sandbox_config_supported
return self.__tainted
@@ -2572,36 +2579,45 @@ class Element(Plugin):
if directory is not None and allow_remote and self.__use_remote_execution():
if not self.BST_VIRTUAL_DIRECTORY:
- raise ElementError("Element {} is configured to use remote execution but plugin does not support it."
- .format(self.name), detail="Plugin '{kind}' does not support virtual directories."
- .format(kind=self.get_kind()))
+ raise ElementError(
+ "Element {} is configured to use remote execution but plugin does not support it.".format(
+ self.name
+ ),
+ detail="Plugin '{kind}' does not support virtual directories.".format(kind=self.get_kind()),
+ )
self.info("Using a remote sandbox for artifact {} with directory '{}'".format(self.name, directory))
output_files_required = context.require_artifact_files or self._artifact_files_required()
- sandbox = SandboxRemote(context, project,
- directory,
- plugin=self,
- stdout=stdout,
- stderr=stderr,
- config=config,
- specs=self.__remote_execution_specs,
- bare_directory=bare_directory,
- allow_real_directory=False,
- output_files_required=output_files_required)
+ sandbox = SandboxRemote(
+ context,
+ project,
+ directory,
+ plugin=self,
+ stdout=stdout,
+ stderr=stderr,
+ config=config,
+ specs=self.__remote_execution_specs,
+ bare_directory=bare_directory,
+ allow_real_directory=False,
+ output_files_required=output_files_required,
+ )
yield sandbox
elif directory is not None and os.path.exists(directory):
- sandbox = platform.create_sandbox(context, project,
- directory,
- plugin=self,
- stdout=stdout,
- stderr=stderr,
- config=config,
- bare_directory=bare_directory,
- allow_real_directory=not self.BST_VIRTUAL_DIRECTORY)
+ sandbox = platform.create_sandbox(
+ context,
+ project,
+ directory,
+ plugin=self,
+ stdout=stdout,
+ stderr=stderr,
+ config=config,
+ bare_directory=bare_directory,
+ allow_real_directory=not self.BST_VIRTUAL_DIRECTORY,
+ )
yield sandbox
else:
@@ -2609,8 +2625,9 @@ class Element(Plugin):
rootdir = tempfile.mkdtemp(prefix="{}-".format(self.normal_name), dir=context.builddir)
# Recursive contextmanager...
- with self.__sandbox(rootdir, stdout=stdout, stderr=stderr, config=config,
- allow_remote=allow_remote, bare_directory=False) as sandbox:
+ with self.__sandbox(
+ rootdir, stdout=stdout, stderr=stderr, config=config, allow_remote=allow_remote, bare_directory=False
+ ) as sandbox:
yield sandbox
# Cleanup the build dir
@@ -2632,9 +2649,9 @@ class Element(Plugin):
# Extend project wide split rules with any split rules defined by the element
element_splits._composite(splits)
- element_bst['split-rules'] = splits
- element_public['bst'] = element_bst
- defaults['public'] = element_public
+ element_bst["split-rules"] = splits
+ element_public["bst"] = element_bst
+ defaults["public"] = element_public
@classmethod
def __init_defaults(cls, project, plugin_conf, kind, is_junction):
@@ -2704,7 +2721,7 @@ class Element(Plugin):
else:
project_nocache = project.base_env_nocache
- default_nocache = cls.__defaults.get_str_list('environment-nocache', default=[])
+ default_nocache = cls.__defaults.get_str_list("environment-nocache", default=[])
element_nocache = meta.env_nocache
# Accumulate values from the element default, the project and the element
@@ -2719,7 +2736,7 @@ class Element(Plugin):
#
@classmethod
def __extract_variables(cls, project, meta):
- default_vars = cls.__defaults.get_mapping('variables', default={})
+ default_vars = cls.__defaults.get_mapping("variables", default={})
if meta.is_junction:
variables = project.first_pass_config.base_variables.clone()
@@ -2730,7 +2747,7 @@ class Element(Plugin):
meta.variables._composite(variables)
variables._assert_fully_composited()
- for var in ('project-name', 'element-name', 'max-jobs'):
+ for var in ("project-name", "element-name", "max-jobs"):
node = variables.get_node(var, allow_none=True)
if node is None:
@@ -2738,8 +2755,10 @@ class Element(Plugin):
provenance = node.get_provenance()
if not provenance._is_synthetic:
- raise LoadError("{}: invalid redefinition of protected variable '{}'"
- .format(provenance, var), LoadErrorReason.PROTECTED_VARIABLE_REDEFINED)
+ raise LoadError(
+ "{}: invalid redefinition of protected variable '{}'".format(provenance, var),
+ LoadErrorReason.PROTECTED_VARIABLE_REDEFINED,
+ )
return variables
@@ -2750,7 +2769,7 @@ class Element(Plugin):
def __extract_config(cls, meta):
# The default config is already composited with the project overrides
- config = cls.__defaults.get_mapping('config', default={})
+ config = cls.__defaults.get_mapping("config", default={})
config = config.clone()
meta.config._composite(config)
@@ -2763,10 +2782,7 @@ class Element(Plugin):
@classmethod
def __extract_sandbox_config(cls, context, project, meta):
if meta.is_junction:
- sandbox_config = Node.from_dict({
- 'build-uid': 0,
- 'build-gid': 0
- })
+ sandbox_config = Node.from_dict({"build-uid": 0, "build-gid": 0})
else:
sandbox_config = project._sandbox.clone()
@@ -2776,7 +2792,7 @@ class Element(Plugin):
host_os = platform.get_host_os()
# The default config is already composited with the project overrides
- sandbox_defaults = cls.__defaults.get_mapping('sandbox', default={})
+ sandbox_defaults = cls.__defaults.get_mapping("sandbox", default={})
sandbox_defaults = sandbox_defaults.clone()
sandbox_defaults._composite(sandbox_config)
@@ -2784,41 +2800,42 @@ class Element(Plugin):
sandbox_config._assert_fully_composited()
# Sandbox config, unlike others, has fixed members so we should validate them
- sandbox_config.validate_keys(['build-uid', 'build-gid', 'build-os', 'build-arch'])
+ sandbox_config.validate_keys(["build-uid", "build-gid", "build-os", "build-arch"])
- build_arch = sandbox_config.get_str('build-arch', default=None)
+ build_arch = sandbox_config.get_str("build-arch", default=None)
if build_arch:
build_arch = Platform.canonicalize_arch(build_arch)
else:
build_arch = host_arch
return SandboxConfig(
- sandbox_config.get_int('build-uid'),
- sandbox_config.get_int('build-gid'),
- sandbox_config.get_str('build-os', default=host_os),
- build_arch)
+ sandbox_config.get_int("build-uid"),
+ sandbox_config.get_int("build-gid"),
+ sandbox_config.get_str("build-os", default=host_os),
+ build_arch,
+ )
# This makes a special exception for the split rules, which
# elements may extend but whos defaults are defined in the project.
#
@classmethod
def __extract_public(cls, meta):
- base_public = cls.__defaults.get_mapping('public', default={})
+ base_public = cls.__defaults.get_mapping("public", default={})
base_public = base_public.clone()
- base_bst = base_public.get_mapping('bst', default={})
- base_splits = base_bst.get_mapping('split-rules', default={})
+ base_bst = base_public.get_mapping("bst", default={})
+ base_splits = base_bst.get_mapping("split-rules", default={})
element_public = meta.public.clone()
- element_bst = element_public.get_mapping('bst', default={})
- element_splits = element_bst.get_mapping('split-rules', default={})
+ element_bst = element_public.get_mapping("bst", default={})
+ element_splits = element_bst.get_mapping("split-rules", default={})
# Allow elements to extend the default splits defined in their project or
# element specific defaults
element_splits._composite(base_splits)
- element_bst['split-rules'] = base_splits
- element_public['bst'] = element_bst
+ element_bst["split-rules"] = base_splits
+ element_public["bst"] = element_bst
element_public._assert_fully_composited()
@@ -2826,24 +2843,21 @@ class Element(Plugin):
# Expand the splits in the public data using the Variables in the element
def __expand_splits(self, element_public):
- element_bst = element_public.get_mapping('bst', default={})
- element_splits = element_bst.get_mapping('split-rules', default={})
+ element_bst = element_public.get_mapping("bst", default={})
+ element_splits = element_bst.get_mapping("split-rules", default={})
# Resolve any variables in the public split rules directly
for domain, splits in element_splits.items():
- splits = [
- self.__variables.subst(split.strip())
- for split in splits.as_str_list()
- ]
+ splits = [self.__variables.subst(split.strip()) for split in splits.as_str_list()]
element_splits[domain] = splits
return element_public
def __init_splits(self):
- bstdata = self.get_public_data('bst')
- splits = bstdata.get_mapping('split-rules')
+ bstdata = self.get_public_data("bst")
+ splits = bstdata.get_mapping("split-rules")
self.__splits = {
- domain: re.compile('^(?:' + '|'.join([utils._glob2re(r) for r in rules.as_str_list()]) + ')$')
+ domain: re.compile("^(?:" + "|".join([utils._glob2re(r) for r in rules.as_str_list()]) + ")$")
for domain, rules in splits.items()
}
@@ -2944,10 +2958,10 @@ class Element(Plugin):
# the build, but I can think of no reason to change it mid-build.
# If this ever changes, things will go wrong unexpectedly.
if not self.__whitelist_regex:
- bstdata = self.get_public_data('bst')
- whitelist = bstdata.get_str_list('overlap-whitelist', default=[])
+ bstdata = self.get_public_data("bst")
+ whitelist = bstdata.get_str_list("overlap-whitelist", default=[])
whitelist_expressions = [utils._glob2re(self.__variables.subst(exp.strip())) for exp in whitelist]
- expression = ('^(?:' + '|'.join(whitelist_expressions) + ')$')
+ expression = "^(?:" + "|".join(whitelist_expressions) + ")$"
self.__whitelist_regex = re.compile(expression)
return self.__whitelist_regex.match(os.path.join(os.sep, path))
@@ -3005,8 +3019,7 @@ class Element(Plugin):
#
def __pull_weak(self, *, pull_buildtrees):
weak_key = self._get_cache_key(strength=_KeyStrength.WEAK)
- if not self.__artifacts.pull(self, weak_key,
- pull_buildtrees=pull_buildtrees):
+ if not self.__artifacts.pull(self, weak_key, pull_buildtrees=pull_buildtrees):
return False
# extract strong cache key from this newly fetched artifact
@@ -3159,8 +3172,9 @@ class Element(Plugin):
return
if not self.__strict_artifact:
- self.__strict_artifact = Artifact(self, context, strong_key=self.__strict_cache_key,
- weak_key=self.__weak_cache_key)
+ self.__strict_artifact = Artifact(
+ self, context, strong_key=self.__strict_cache_key, weak_key=self.__weak_cache_key
+ )
if context.get_strict():
self.__artifact = self.__strict_artifact
@@ -3192,9 +3206,7 @@ class Element(Plugin):
self.__cache_key = strong_key
elif self.__assemble_scheduled or self.__assemble_done:
# Artifact will or has been built, not downloaded
- dependencies = [
- e._get_cache_key() for e in self.dependencies(Scope.BUILD)
- ]
+ dependencies = [e._get_cache_key() for e in self.dependencies(Scope.BUILD)]
self.__cache_key = self._calculate_cache_key(dependencies)
if self.__cache_key is None:
@@ -3216,8 +3228,7 @@ class Element(Plugin):
#
def __update_strict_cache_key_of_rdeps(self):
if not self.__updated_strict_cache_keys_of_rdeps:
- if self.__runtime_deps_without_strict_cache_key == 0 and \
- self.__strict_cache_key is not None:
+ if self.__runtime_deps_without_strict_cache_key == 0 and self.__strict_cache_key is not None:
self.__updated_strict_cache_keys_of_rdeps = True
# Notify reverse dependencies
@@ -3251,8 +3262,7 @@ class Element(Plugin):
#
def __update_ready_for_runtime(self):
if not self.__ready_for_runtime:
- if self.__runtime_deps_without_cache_key == 0 and \
- self.__cache_key is not None:
+ if self.__runtime_deps_without_cache_key == 0 and self.__cache_key is not None:
self.__ready_for_runtime = True
# Notify reverse dependencies
@@ -3279,10 +3289,12 @@ class Element(Plugin):
def _overlap_error_detail(f, forbidden_overlap_elements, elements):
if forbidden_overlap_elements:
- return ("/{}: {} {} not permitted to overlap other elements, order {} \n"
- .format(f, " and ".join(forbidden_overlap_elements),
- "is" if len(forbidden_overlap_elements) == 1 else "are",
- " above ".join(reversed(elements))))
+ return "/{}: {} {} not permitted to overlap other elements, order {} \n".format(
+ f,
+ " and ".join(forbidden_overlap_elements),
+ "is" if len(forbidden_overlap_elements) == 1 else "are",
+ " above ".join(reversed(elements)),
+ )
else:
return ""
@@ -3299,7 +3311,7 @@ def _overlap_error_detail(f, forbidden_overlap_elements, elements):
# (str): The normalised element name
#
def _get_normal_name(element_name):
- return os.path.splitext(element_name.replace(os.sep, '-'))[0]
+ return os.path.splitext(element_name.replace(os.sep, "-"))[0]
# _compose_artifact_name():
@@ -3315,12 +3327,9 @@ def _get_normal_name(element_name):
# (str): The constructed artifact name path
#
def _compose_artifact_name(project_name, normal_name, cache_key):
- valid_chars = string.digits + string.ascii_letters + '-._'
- normal_name = ''.join([
- x if x in valid_chars else '_'
- for x in normal_name
- ])
+ valid_chars = string.digits + string.ascii_letters + "-._"
+ normal_name = "".join([x if x in valid_chars else "_" for x in normal_name])
# Note that project names are not allowed to contain slashes. Element names containing
# a '/' will have this replaced with a '-' upon Element object instantiation.
- return '{0}/{1}/{2}'.format(project_name, normal_name, cache_key)
+ return "{0}/{1}/{2}".format(project_name, normal_name, cache_key)
diff --git a/src/buildstream/plugin.py b/src/buildstream/plugin.py
index c1ee333f7..0cbd72e27 100644
--- a/src/buildstream/plugin.py
+++ b/src/buildstream/plugin.py
@@ -127,10 +127,11 @@ if TYPE_CHECKING:
# pylint: disable=cyclic-import
from ._context import Context
from ._project import Project
+
# pylint: enable=cyclic-import
-class Plugin():
+class Plugin:
"""Plugin()
Base Plugin class.
@@ -210,15 +211,17 @@ class Plugin():
#
# Note that Plugins can only be instantiated in the main process before
# scheduling tasks.
- __TABLE = WeakValueDictionary() # type: WeakValueDictionary[int, Plugin]
-
- def __init__(self,
- name: str,
- context: 'Context',
- project: 'Project',
- provenance: ProvenanceInformation,
- type_tag: str,
- unique_id: Optional[int] = None):
+ __TABLE = WeakValueDictionary() # type: WeakValueDictionary[int, Plugin]
+
+ def __init__(
+ self,
+ name: str,
+ context: "Context",
+ project: "Project",
+ provenance: ProvenanceInformation,
+ type_tag: str,
+ unique_id: Optional[int] = None,
+ ):
self.name = name
"""The plugin name
@@ -248,30 +251,29 @@ class Plugin():
# plugin in a subprocess and should use the same ID.
self._unique_id = unique_id
- self.__context = context # The Context object
+ self.__context = context # The Context object
# Note that when pickling jobs over to a child process, we rely on this
# reference to the Project, it keeps the plugin factory alive. If the
# factory were to be GC'd then we would see undefined behaviour. Make
# sure to test plugin pickling if this reference is to be removed.
- self.__project = project # The Project object
+ self.__project = project # The Project object
self.__provenance = provenance # The Provenance information
- self.__type_tag = type_tag # The type of plugin (element or source)
- self.__configuring = False # Whether we are currently configuring
+ self.__type_tag = type_tag # The type of plugin (element or source)
+ self.__configuring = False # Whether we are currently configuring
# Get the full_name as project & type_tag are resolved
self.__full_name = self.__get_full_name()
# Infer the kind identifier
modulename = type(self).__module__
- self.__kind = modulename.split('.')[-1]
+ self.__kind = modulename.split(".")[-1]
self.debug("Created: {}".format(self))
# If this plugin has been deprecated, emit a warning.
if self.BST_PLUGIN_DEPRECATED and not self.__deprecation_warning_silenced():
- detail = "Using deprecated plugin {}: {}".format(self.__kind,
- self.BST_PLUGIN_DEPRECATION_MESSAGE)
+ detail = "Using deprecated plugin {}: {}".format(self.__kind, self.BST_PLUGIN_DEPRECATION_MESSAGE)
self.__message(MessageType.WARN, detail)
def __del__(self):
@@ -282,9 +284,8 @@ class Plugin():
def __str__(self):
return "{kind} {typetag} at {provenance}".format(
- kind=self.__kind,
- typetag=self.__type_tag,
- provenance=self.__provenance)
+ kind=self.__kind, typetag=self.__type_tag, provenance=self.__provenance
+ )
#############################################################
# Abstract Methods #
@@ -312,8 +313,9 @@ class Plugin():
:func:`Element.node_subst_member() <buildstream.element.Element.node_subst_member>`
method can be used.
"""
- raise ImplError("{tag} plugin '{kind}' does not implement configure()".format(
- tag=self.__type_tag, kind=self.get_kind()))
+ raise ImplError(
+ "{tag} plugin '{kind}' does not implement configure()".format(tag=self.__type_tag, kind=self.get_kind())
+ )
def preflight(self) -> None:
"""Preflight Check
@@ -333,8 +335,9 @@ class Plugin():
them with :func:`utils.get_host_tool() <buildstream.utils.get_host_tool>` which
will raise an error automatically informing the user that a host tool is needed.
"""
- raise ImplError("{tag} plugin '{kind}' does not implement preflight()".format(
- tag=self.__type_tag, kind=self.get_kind()))
+ raise ImplError(
+ "{tag} plugin '{kind}' does not implement preflight()".format(tag=self.__type_tag, kind=self.get_kind())
+ )
def get_unique_key(self) -> SourceRef:
"""Return something which uniquely identifies the plugin input
@@ -355,8 +358,11 @@ class Plugin():
which is to say that the Source is expected to have an exact *ref* indicating
exactly what source is going to be staged.
"""
- raise ImplError("{tag} plugin '{kind}' does not implement get_unique_key()".format(
- tag=self.__type_tag, kind=self.get_kind()))
+ raise ImplError(
+ "{tag} plugin '{kind}' does not implement get_unique_key()".format(
+ tag=self.__type_tag, kind=self.get_kind()
+ )
+ )
#############################################################
# Public Methods #
@@ -369,8 +375,7 @@ class Plugin():
"""
return self.__kind
- def node_get_project_path(self, node, *,
- check_is_file=False, check_is_dir=False):
+ def node_get_project_path(self, node, *, check_is_file=False, check_is_dir=False):
"""Fetches a project path from a dictionary node and validates it
Paths are asserted to never lead to a directory outside of the
@@ -408,9 +413,7 @@ class Plugin():
"""
- return self.__project.get_path_from_node(node,
- check_is_file=check_is_file,
- check_is_dir=check_is_dir)
+ return self.__project.get_path_from_node(node, check_is_file=check_is_file, check_is_dir=check_is_dir)
def debug(self, brief: str, *, detail: Optional[str] = None) -> None:
"""Print a debugging message
@@ -485,11 +488,9 @@ class Plugin():
self.__message(MessageType.LOG, brief, detail=detail)
@contextmanager
- def timed_activity(self,
- activity_name: str,
- *,
- detail: Optional[str] = None,
- silent_nested: bool = False) -> Generator[None, None, None]:
+ def timed_activity(
+ self, activity_name: str, *, detail: Optional[str] = None, silent_nested: bool = False
+ ) -> Generator[None, None, None]:
"""Context manager for performing timed activities in plugins
Args:
@@ -511,10 +512,9 @@ class Plugin():
# This will raise SourceError on its own
self.call(... command which takes time ...)
"""
- with self.__context.messenger.timed_activity(activity_name,
- element_name=self._get_full_name(),
- detail=detail,
- silent_nested=silent_nested):
+ with self.__context.messenger.timed_activity(
+ activity_name, element_name=self._get_full_name(), detail=detail, silent_nested=silent_nested
+ ):
yield
def call(self, *popenargs, fail: Optional[str] = None, fail_temporarily: bool = False, **kwargs) -> int:
@@ -722,8 +722,11 @@ class Plugin():
# so it's not an ImplError - those apply to custom plugins. Direct
# descendants of Plugin must implement this, e.g. Element and Source.
# Raise NotImplementedError as this would be an internal bug.
- raise NotImplementedError("{tag} plugin '{kind}' does not implement _get_args_for_child_job_pickling()".format(
- tag=self.__type_tag, kind=self.get_kind()))
+ raise NotImplementedError(
+ "{tag} plugin '{kind}' does not implement _get_args_for_child_job_pickling()".format(
+ tag=self.__type_tag, kind=self.get_kind()
+ )
+ )
#############################################################
# Local Private Methods #
@@ -734,20 +737,19 @@ class Plugin():
def __call(self, *popenargs, collect_stdout=False, fail=None, fail_temporarily=False, **kwargs):
with self._output_file() as output_file:
- if 'stdout' not in kwargs:
- kwargs['stdout'] = output_file
- if 'stderr' not in kwargs:
- kwargs['stderr'] = output_file
+ if "stdout" not in kwargs:
+ kwargs["stdout"] = output_file
+ if "stderr" not in kwargs:
+ kwargs["stderr"] = output_file
if collect_stdout:
- kwargs['stdout'] = subprocess.PIPE
+ kwargs["stdout"] = subprocess.PIPE
self.__note_command(output_file, *popenargs, **kwargs)
exit_code, output = utils._call(*popenargs, **kwargs)
if fail and exit_code:
- raise PluginError("{plugin}: {message}".format(plugin=self, message=fail),
- temporary=fail_temporarily)
+ raise PluginError("{plugin}: {message}".format(plugin=self, message=fail), temporary=fail_temporarily)
return (exit_code, output)
@@ -756,11 +758,11 @@ class Plugin():
self.__context.messenger.message(message)
def __note_command(self, output, *popenargs, **kwargs):
- workdir = kwargs.get('cwd', os.getcwd())
+ workdir = kwargs.get("cwd", os.getcwd())
command = " ".join(popenargs[0])
- output.write('Running host command {}: {}\n'.format(workdir, command))
+ output.write("Running host command {}: {}\n".format(workdir, command))
output.flush()
- self.status('Running host command', detail=command)
+ self.status("Running host command", detail=command)
def __deprecation_warning_silenced(self):
if not self.BST_PLUGIN_DEPRECATED:
@@ -770,10 +772,10 @@ class Plugin():
project = self.__project
for key, value in project.element_overrides.items():
- if value.get_bool('suppress-deprecation-warnings', default=False):
+ if value.get_bool("suppress-deprecation-warnings", default=False):
silenced_warnings.add(key)
for key, value in project.source_overrides.items():
- if value.get_bool('suppress-deprecation-warnings', default=False):
+ if value.get_bool("suppress-deprecation-warnings", default=False):
silenced_warnings.add(key)
return self.get_kind() in silenced_warnings
@@ -783,18 +785,14 @@ class Plugin():
# Set the name, depending on element or source plugin type
name = self._element_name if self.__type_tag == "source" else self.name # pylint: disable=no-member
if project.junction:
- return '{}:{}'.format(project.junction.name, name)
+ return "{}:{}".format(project.junction.name, name)
else:
return name
# A local table for _prefix_warning()
#
-__CORE_WARNINGS = [
- value
- for name, value in CoreWarnings.__dict__.items()
- if not name.startswith("__")
-]
+__CORE_WARNINGS = [value for name, value in CoreWarnings.__dict__.items() if not name.startswith("__")]
# _prefix_warning():
diff --git a/src/buildstream/plugins/elements/autotools.py b/src/buildstream/plugins/elements/autotools.py
index 7a05336b7..089c9bca0 100644
--- a/src/buildstream/plugins/elements/autotools.py
+++ b/src/buildstream/plugins/elements/autotools.py
@@ -66,8 +66,7 @@ class AutotoolsElement(BuildElement):
# Enable command batching across prepare() and assemble()
def configure_sandbox(self, sandbox):
super().configure_sandbox(sandbox)
- self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY,
- collect=self.get_variable('install-root'))
+ self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY, collect=self.get_variable("install-root"))
# Plugin entry point
diff --git a/src/buildstream/plugins/elements/compose.py b/src/buildstream/plugins/elements/compose.py
index 511925731..063c5d44f 100644
--- a/src/buildstream/plugins/elements/compose.py
+++ b/src/buildstream/plugins/elements/compose.py
@@ -62,27 +62,23 @@ class ComposeElement(Element):
BST_RUN_COMMANDS = False
def configure(self, node):
- node.validate_keys([
- 'integrate', 'include', 'exclude', 'include-orphans'
- ])
+ node.validate_keys(["integrate", "include", "exclude", "include-orphans"])
# We name this variable 'integration' only to avoid
# collision with the Element.integrate() method.
- self.integration = node.get_bool('integrate')
- self.include = node.get_str_list('include')
- self.exclude = node.get_str_list('exclude')
- self.include_orphans = node.get_bool('include-orphans')
+ self.integration = node.get_bool("integrate")
+ self.include = node.get_str_list("include")
+ self.exclude = node.get_str_list("exclude")
+ self.include_orphans = node.get_bool("include-orphans")
def preflight(self):
pass
def get_unique_key(self):
- key = {'integrate': self.integration,
- 'include': sorted(self.include),
- 'orphans': self.include_orphans}
+ key = {"integrate": self.integration, "include": sorted(self.include), "orphans": self.include_orphans}
if self.exclude:
- key['exclude'] = sorted(self.exclude)
+ key["exclude"] = sorted(self.exclude)
return key
@@ -104,9 +100,9 @@ class ComposeElement(Element):
if require_split:
with self.timed_activity("Computing split", silent_nested=True):
for dep in self.dependencies(Scope.BUILD):
- files = dep.compute_manifest(include=self.include,
- exclude=self.exclude,
- orphans=self.include_orphans)
+ files = dep.compute_manifest(
+ include=self.include, exclude=self.exclude, orphans=self.include_orphans
+ )
manifest.update(files)
# Make a snapshot of all the files.
@@ -141,13 +137,16 @@ class ComposeElement(Element):
for path in basedir_contents:
if path not in snapshot:
added_files.add(path)
- self.info("Integration modified {}, added {} and removed {} files"
- .format(len(modified_files), len(added_files), len(removed_files)))
+ self.info(
+ "Integration modified {}, added {} and removed {} files".format(
+ len(modified_files), len(added_files), len(removed_files)
+ )
+ )
# The remainder of this is expensive, make an early exit if
# we're not being selective about what is to be included.
if not require_split:
- return '/'
+ return "/"
# Do we want to force include files which were modified by
# the integration commands, even if they were not added ?
@@ -159,7 +158,7 @@ class ComposeElement(Element):
# instead of into a subdir. The element assemble() method should
# support this in some way.
#
- installdir = vbasedir.descend('buildstream', 'install', create=True)
+ installdir = vbasedir.descend("buildstream", "install", create=True)
# We already saved the manifest for created files in the integration phase,
# now collect the rest of the manifest.
@@ -189,7 +188,7 @@ class ComposeElement(Element):
installdir.import_files(vbasedir, filter_callback=import_filter, can_link=True)
# And we're done
- return os.path.join(os.sep, 'buildstream', 'install')
+ return os.path.join(os.sep, "buildstream", "install")
# Plugin entry point
diff --git a/src/buildstream/plugins/elements/filter.py b/src/buildstream/plugins/elements/filter.py
index d808c9e5a..71ed1f6cb 100644
--- a/src/buildstream/plugins/elements/filter.py
+++ b/src/buildstream/plugins/elements/filter.py
@@ -167,17 +167,15 @@ class FilterElement(Element):
BST_RUN_COMMANDS = False
def configure(self, node):
- node.validate_keys([
- 'include', 'exclude', 'include-orphans', 'pass-integration'
- ])
+ node.validate_keys(["include", "exclude", "include-orphans", "pass-integration"])
- self.include_node = node.get_sequence('include')
- self.exclude_node = node.get_sequence('exclude')
+ self.include_node = node.get_sequence("include")
+ self.exclude_node = node.get_sequence("exclude")
self.include = self.include_node.as_str_list()
self.exclude = self.exclude_node.as_str_list()
- self.include_orphans = node.get_bool('include-orphans')
- self.pass_integration = node.get_bool('pass-integration', False)
+ self.include_orphans = node.get_bool("include-orphans")
+ self.pass_integration = node.get_bool("pass-integration", False)
def preflight(self):
# Exactly one build-depend is permitted
@@ -186,9 +184,13 @@ class FilterElement(Element):
detail = "Full list of build-depends:\n"
deps_list = " \n".join([x.name for x in build_deps])
detail += deps_list
- raise ElementError("{}: {} element must have exactly 1 build-dependency, actually have {}"
- .format(self, type(self).__name__, len(build_deps)),
- detail=detail, reason="filter-bdepend-wrong-count")
+ raise ElementError(
+ "{}: {} element must have exactly 1 build-dependency, actually have {}".format(
+ self, type(self).__name__, len(build_deps)
+ ),
+ detail=detail,
+ reason="filter-bdepend-wrong-count",
+ )
# That build-depend must not also be a runtime-depend
runtime_deps = list(self.dependencies(Scope.RUN, recurse=False))
@@ -196,23 +198,29 @@ class FilterElement(Element):
detail = "Full list of runtime depends:\n"
deps_list = " \n".join([x.name for x in runtime_deps])
detail += deps_list
- raise ElementError("{}: {} element's build dependency must not also be a runtime dependency"
- .format(self, type(self).__name__),
- detail=detail, reason="filter-bdepend-also-rdepend")
+ raise ElementError(
+ "{}: {} element's build dependency must not also be a runtime dependency".format(
+ self, type(self).__name__
+ ),
+ detail=detail,
+ reason="filter-bdepend-also-rdepend",
+ )
# If a parent does not produce an artifact, fail and inform user that the dependency
# must produce artifacts
if not build_deps[0].BST_ELEMENT_HAS_ARTIFACT:
detail = "{} does not produce an artifact, so there is nothing to filter".format(build_deps[0].name)
- raise ElementError("{}: {} element's build dependency must produce an artifact"
- .format(self, type(self).__name__),
- detail=detail, reason="filter-bdepend-no-artifact")
+ raise ElementError(
+ "{}: {} element's build dependency must produce an artifact".format(self, type(self).__name__),
+ detail=detail,
+ reason="filter-bdepend-no-artifact",
+ )
def get_unique_key(self):
key = {
- 'include': sorted(self.include),
- 'exclude': sorted(self.exclude),
- 'orphans': self.include_orphans,
+ "include": sorted(self.include),
+ "exclude": sorted(self.exclude),
+ "orphans": self.include_orphans,
}
return key
@@ -226,8 +234,8 @@ class FilterElement(Element):
with self.timed_activity("Staging artifact", silent_nested=True):
for dep in self.dependencies(Scope.BUILD, recurse=False):
# Check that all the included/excluded domains exist
- pub_data = dep.get_public_data('bst')
- split_rules = pub_data.get_mapping('split-rules', {})
+ pub_data = dep.get_public_data("bst")
+ split_rules = pub_data.get_mapping("split-rules", {})
unfound_includes = []
for domain in self.include:
if domain not in split_rules:
@@ -240,18 +248,17 @@ class FilterElement(Element):
detail = []
if unfound_includes:
detail.append("Unknown domains were used in {}".format(self.include_node.get_provenance()))
- detail.extend([' - {}'.format(domain) for domain in unfound_includes])
+ detail.extend([" - {}".format(domain) for domain in unfound_includes])
if unfound_excludes:
detail.append("Unknown domains were used in {}".format(self.exclude_node.get_provenance()))
- detail.extend([' - {}'.format(domain) for domain in unfound_excludes])
+ detail.extend([" - {}".format(domain) for domain in unfound_excludes])
if detail:
- detail = '\n'.join(detail)
+ detail = "\n".join(detail)
raise ElementError("Unknown domains declared.", detail=detail)
- dep.stage_artifact(sandbox, include=self.include,
- exclude=self.exclude, orphans=self.include_orphans)
+ dep.stage_artifact(sandbox, include=self.include, exclude=self.exclude, orphans=self.include_orphans)
return ""
def _get_source_element(self):
diff --git a/src/buildstream/plugins/elements/import.py b/src/buildstream/plugins/elements/import.py
index 404a0f4ee..2b68197a7 100644
--- a/src/buildstream/plugins/elements/import.py
+++ b/src/buildstream/plugins/elements/import.py
@@ -45,12 +45,10 @@ class ImportElement(Element):
BST_RUN_COMMANDS = False
def configure(self, node):
- node.validate_keys([
- 'source', 'target'
- ])
+ node.validate_keys(["source", "target"])
- self.source = self.node_subst_vars(node.get_scalar('source'))
- self.target = self.node_subst_vars(node.get_scalar('target'))
+ self.source = self.node_subst_vars(node.get_scalar("source"))
+ self.target = self.node_subst_vars(node.get_scalar("target"))
def preflight(self):
# Assert that we have at least one source to fetch.
@@ -60,10 +58,7 @@ class ImportElement(Element):
raise ElementError("{}: An import element must have at least one source.".format(self))
def get_unique_key(self):
- return {
- 'source': self.source,
- 'target': self.target
- }
+ return {"source": self.source, "target": self.target}
def configure_sandbox(self, sandbox):
pass
@@ -74,11 +69,11 @@ class ImportElement(Element):
def assemble(self, sandbox):
# Stage sources into the input directory
- self.stage_sources(sandbox, 'input')
+ self.stage_sources(sandbox, "input")
rootdir = sandbox.get_virtual_directory()
- inputdir = rootdir.descend('input')
- outputdir = rootdir.descend('output', create=True)
+ inputdir = rootdir.descend("input")
+ outputdir = rootdir.descend("output", create=True)
# The directory to grab
inputdir = inputdir.descend(*self.source.strip(os.sep).split(os.sep))
@@ -87,18 +82,17 @@ class ImportElement(Element):
outputdir = outputdir.descend(*self.target.strip(os.sep).split(os.sep), create=True)
if inputdir.is_empty():
- raise ElementError("{}: No files were found inside directory '{}'"
- .format(self, self.source))
+ raise ElementError("{}: No files were found inside directory '{}'".format(self, self.source))
# Move it over
outputdir.import_files(inputdir)
# And we're done
- return '/output'
+ return "/output"
def generate_script(self):
- build_root = self.get_variable('build-root')
- install_root = self.get_variable('install-root')
+ build_root = self.get_variable("build-root")
+ install_root = self.get_variable("install-root")
commands = []
# The directory to grab
diff --git a/src/buildstream/plugins/elements/junction.py b/src/buildstream/plugins/elements/junction.py
index aec32516b..42b9ef08e 100644
--- a/src/buildstream/plugins/elements/junction.py
+++ b/src/buildstream/plugins/elements/junction.py
@@ -187,13 +187,13 @@ class JunctionElement(Element):
BST_FORBID_RDEPENDS = True
def configure(self, node):
- self.path = node.get_str('path', default='')
- self.options = node.get_mapping('options', default={})
- self.target = node.get_str('target', default=None)
+ self.path = node.get_str("path", default="")
+ self.options = node.get_mapping("options", default={})
+ self.target = node.get_str("target", default=None)
self.target_element = None
self.target_junction = None
- self.cache_junction_elements = node.get_bool('cache-junction-elements', default=False)
- self.ignore_junction_remotes = node.get_bool('ignore-junction-remotes', default=False)
+ self.cache_junction_elements = node.get_bool("cache-junction-elements", default=False)
+ self.ignore_junction_remotes = node.get_bool("ignore-junction-remotes", default=False)
def preflight(self):
# "target" cannot be used in conjunction with:
diff --git a/src/buildstream/plugins/elements/manual.py b/src/buildstream/plugins/elements/manual.py
index bbda65312..97da41615 100644
--- a/src/buildstream/plugins/elements/manual.py
+++ b/src/buildstream/plugins/elements/manual.py
@@ -42,8 +42,7 @@ class ManualElement(BuildElement):
# Enable command batching across prepare() and assemble()
def configure_sandbox(self, sandbox):
super().configure_sandbox(sandbox)
- self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY,
- collect=self.get_variable('install-root'))
+ self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY, collect=self.get_variable("install-root"))
# Plugin entry point
diff --git a/src/buildstream/plugins/elements/pip.py b/src/buildstream/plugins/elements/pip.py
index 4a9eefde1..93303748d 100644
--- a/src/buildstream/plugins/elements/pip.py
+++ b/src/buildstream/plugins/elements/pip.py
@@ -42,8 +42,7 @@ class PipElement(BuildElement):
# Enable command batching across prepare() and assemble()
def configure_sandbox(self, sandbox):
super().configure_sandbox(sandbox)
- self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY,
- collect=self.get_variable('install-root'))
+ self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY, collect=self.get_variable("install-root"))
# Plugin entry point
diff --git a/src/buildstream/plugins/elements/script.py b/src/buildstream/plugins/elements/script.py
index f3f0a2f7a..abfb7b3b0 100644
--- a/src/buildstream/plugins/elements/script.py
+++ b/src/buildstream/plugins/elements/script.py
@@ -46,21 +46,19 @@ class ScriptElement(buildstream.ScriptElement):
BST_VIRTUAL_DIRECTORY = True
def configure(self, node):
- for n in node.get_sequence('layout', []):
- dst = self.node_subst_vars(n.get_scalar('destination'))
- elm = self.node_subst_vars(n.get_scalar('element', None))
+ for n in node.get_sequence("layout", []):
+ dst = self.node_subst_vars(n.get_scalar("destination"))
+ elm = self.node_subst_vars(n.get_scalar("element", None))
self.layout_add(elm, dst)
- node.validate_keys([
- 'commands', 'root-read-only', 'layout'
- ])
+ node.validate_keys(["commands", "root-read-only", "layout"])
cmds = self.node_subst_sequence_vars(node.get_sequence("commands"))
self.add_commands("commands", cmds)
self.set_work_dir()
self.set_install_root()
- self.set_root_read_only(node.get_bool('root-read-only', default=False))
+ self.set_root_read_only(node.get_bool("root-read-only", default=False))
# Plugin entry point
diff --git a/src/buildstream/plugins/elements/stack.py b/src/buildstream/plugins/elements/stack.py
index ae15af63a..f569199b6 100644
--- a/src/buildstream/plugins/elements/stack.py
+++ b/src/buildstream/plugins/elements/stack.py
@@ -64,10 +64,10 @@ class StackElement(Element):
# Just create a dummy empty artifact, its existence is a statement
# that all this stack's dependencies are built.
vrootdir = sandbox.get_virtual_directory()
- vrootdir.descend('output', create=True)
+ vrootdir.descend("output", create=True)
# And we're done
- return '/output'
+ return "/output"
# Plugin entry point
diff --git a/src/buildstream/plugins/sources/_downloadablefilesource.py b/src/buildstream/plugins/sources/_downloadablefilesource.py
index 1e759b94f..4e43ee3e3 100644
--- a/src/buildstream/plugins/sources/_downloadablefilesource.py
+++ b/src/buildstream/plugins/sources/_downloadablefilesource.py
@@ -12,7 +12,6 @@ from buildstream import utils
class _NetrcFTPOpener(urllib.request.FTPHandler):
-
def __init__(self, netrc_config):
self.netrc = netrc_config
@@ -28,11 +27,11 @@ class _NetrcFTPOpener(urllib.request.FTPHandler):
def _unsplit(self, host, port, user, passwd):
if port:
- host = '{}:{}'.format(host, port)
+ host = "{}:{}".format(host, port)
if user:
if passwd:
- user = '{}:{}'.format(user, passwd)
- host = '{}@{}'.format(user, host)
+ user = "{}:{}".format(user, passwd)
+ host = "{}@{}".format(user, host)
return host
@@ -50,7 +49,6 @@ class _NetrcFTPOpener(urllib.request.FTPHandler):
class _NetrcPasswordManager:
-
def __init__(self, netrc_config):
self.netrc = netrc_config
@@ -72,17 +70,16 @@ class _NetrcPasswordManager:
class DownloadableFileSource(Source):
# pylint: disable=attribute-defined-outside-init
- COMMON_CONFIG_KEYS = Source.COMMON_CONFIG_KEYS + ['url', 'ref', 'etag']
+ COMMON_CONFIG_KEYS = Source.COMMON_CONFIG_KEYS + ["url", "ref", "etag"]
__urlopener = None
__default_mirror_file = None
def configure(self, node):
- self.original_url = node.get_str('url')
- self.ref = node.get_str('ref', None)
+ self.original_url = node.get_str("url")
+ self.ref = node.get_str("ref", None)
self.url = self.translate_url(self.original_url)
- self._mirror_dir = os.path.join(self.get_mirror_directory(),
- utils.url_directory_name(self.original_url))
+ self._mirror_dir = os.path.join(self.get_mirror_directory(), utils.url_directory_name(self.original_url))
self._warn_deprecated_etag(node)
def preflight(self):
@@ -102,28 +99,29 @@ class DownloadableFileSource(Source):
return Consistency.RESOLVED
def load_ref(self, node):
- self.ref = node.get_str('ref', None)
+ self.ref = node.get_str("ref", None)
self._warn_deprecated_etag(node)
def get_ref(self):
return self.ref
def set_ref(self, ref, node):
- node['ref'] = self.ref = ref
+ node["ref"] = self.ref = ref
def track(self): # pylint: disable=arguments-differ
# there is no 'track' field in the source to determine what/whether
# or not to update refs, because tracking a ref is always a conscious
# decision by the user.
- with self.timed_activity("Tracking {}".format(self.url),
- silent_nested=True):
+ with self.timed_activity("Tracking {}".format(self.url), silent_nested=True):
new_ref = self._ensure_mirror()
if self.ref and self.ref != new_ref:
- detail = "When tracking, new ref differs from current ref:\n" \
- + " Tracked URL: {}\n".format(self.url) \
- + " Current ref: {}\n".format(self.ref) \
+ detail = (
+ "When tracking, new ref differs from current ref:\n"
+ + " Tracked URL: {}\n".format(self.url)
+ + " Current ref: {}\n".format(self.ref)
+ " New ref: {}\n".format(new_ref)
+ )
self.warn("Potential man-in-the-middle attack!", detail=detail)
return new_ref
@@ -142,25 +140,26 @@ class DownloadableFileSource(Source):
with self.timed_activity("Fetching {}".format(self.url), silent_nested=True):
sha256 = self._ensure_mirror()
if sha256 != self.ref:
- raise SourceError("File downloaded from {} has sha256sum '{}', not '{}'!"
- .format(self.url, sha256, self.ref))
+ raise SourceError(
+ "File downloaded from {} has sha256sum '{}', not '{}'!".format(self.url, sha256, self.ref)
+ )
def _warn_deprecated_etag(self, node):
- etag = node.get_str('etag', None)
+ etag = node.get_str("etag", None)
if etag:
provenance = node.get_scalar(etag).get_provenance()
self.warn('{} "etag" is deprecated and ignored.'.format(provenance))
def _get_etag(self, ref):
- etagfilename = os.path.join(self._mirror_dir, '{}.etag'.format(ref))
+ etagfilename = os.path.join(self._mirror_dir, "{}.etag".format(ref))
if os.path.exists(etagfilename):
- with open(etagfilename, 'r') as etagfile:
+ with open(etagfilename, "r") as etagfile:
return etagfile.read()
return None
def _store_etag(self, ref, etag):
- etagfilename = os.path.join(self._mirror_dir, '{}.etag'.format(ref))
+ etagfilename = os.path.join(self._mirror_dir, "{}.etag".format(ref))
with utils.save_file_atomic(etagfilename) as etagfile:
etagfile.write(etag)
@@ -170,7 +169,7 @@ class DownloadableFileSource(Source):
with self.tempdir() as td:
default_name = os.path.basename(self.url)
request = urllib.request.Request(self.url)
- request.add_header('Accept', '*/*')
+ request.add_header("Accept", "*/*")
# We do not use etag in case what we have in cache is
# not matching ref in order to be able to recover from
@@ -180,18 +179,18 @@ class DownloadableFileSource(Source):
# Do not re-download the file if the ETag matches.
if etag and self.get_consistency() == Consistency.CACHED:
- request.add_header('If-None-Match', etag)
+ request.add_header("If-None-Match", etag)
opener = self.__get_urlopener()
with contextlib.closing(opener.open(request)) as response:
info = response.info()
- etag = info['ETag'] if 'ETag' in info else None
+ etag = info["ETag"] if "ETag" in info else None
filename = info.get_filename(default_name)
filename = os.path.basename(filename)
local_file = os.path.join(td, filename)
- with open(local_file, 'wb') as dest:
+ with open(local_file, "wb") as dest:
shutil.copyfileobj(response, dest)
# Make sure url-specific mirror dir exists.
@@ -214,14 +213,12 @@ class DownloadableFileSource(Source):
# Because we use etag only for matching ref, currently specified ref is what
# we would have downloaded.
return self.ref
- raise SourceError("{}: Error mirroring {}: {}"
- .format(self, self.url, e), temporary=True) from e
+ raise SourceError("{}: Error mirroring {}: {}".format(self, self.url, e), temporary=True) from e
except (urllib.error.URLError, urllib.error.ContentTooShortError, OSError, ValueError) as e:
# Note that urllib.request.Request in the try block may throw a
# ValueError for unknown url types, so we handle it here.
- raise SourceError("{}: Error mirroring {}: {}"
- .format(self, self.url, e), temporary=True) from e
+ raise SourceError("{}: Error mirroring {}: {}".format(self, self.url, e), temporary=True) from e
def _get_mirror_file(self, sha=None):
if sha is not None:
@@ -245,7 +242,7 @@ class DownloadableFileSource(Source):
#
DownloadableFileSource.__urlopener = urllib.request.build_opener()
except netrc.NetrcParseError as e:
- self.warn('{}: While reading .netrc: {}'.format(self, e))
+ self.warn("{}: While reading .netrc: {}".format(self, e))
return urllib.request.build_opener()
else:
netrc_pw_mgr = _NetrcPasswordManager(netrc_config)
diff --git a/src/buildstream/plugins/sources/bzr.py b/src/buildstream/plugins/sources/bzr.py
index 88dba7dc2..30ce55585 100644
--- a/src/buildstream/plugins/sources/bzr.py
+++ b/src/buildstream/plugins/sources/bzr.py
@@ -67,16 +67,16 @@ class BzrSource(Source):
# pylint: disable=attribute-defined-outside-init
def configure(self, node):
- node.validate_keys(['url', 'track', 'ref', *Source.COMMON_CONFIG_KEYS])
+ node.validate_keys(["url", "track", "ref", *Source.COMMON_CONFIG_KEYS])
- self.original_url = node.get_str('url')
- self.tracking = node.get_str('track')
- self.ref = node.get_str('ref', None)
+ self.original_url = node.get_str("url")
+ self.tracking = node.get_str("track")
+ self.ref = node.get_str("ref", None)
self.url = self.translate_url(self.original_url)
def preflight(self):
# Check if bzr is installed, get the binary at the same time.
- self.host_bzr = utils.get_host_tool('bzr')
+ self.host_bzr = utils.get_host_tool("bzr")
def get_unique_key(self):
return [self.original_url, self.tracking, self.ref]
@@ -93,39 +93,44 @@ class BzrSource(Source):
return Consistency.RESOLVED
def load_ref(self, node):
- self.ref = node.get_str('ref', None)
+ self.ref = node.get_str("ref", None)
def get_ref(self):
return self.ref
def set_ref(self, ref, node):
- node['ref'] = self.ref = ref
+ node["ref"] = self.ref = ref
def track(self): # pylint: disable=arguments-differ
- with self.timed_activity("Tracking {}".format(self.url),
- silent_nested=True), self._locked():
+ with self.timed_activity("Tracking {}".format(self.url), silent_nested=True), self._locked():
self._ensure_mirror(skip_ref_check=True)
- ret, out = self.check_output([self.host_bzr, "version-info",
- "--custom", "--template={revno}",
- self._get_branch_dir()],
- fail="Failed to read the revision number at '{}'"
- .format(self._get_branch_dir()))
+ ret, out = self.check_output(
+ [self.host_bzr, "version-info", "--custom", "--template={revno}", self._get_branch_dir()],
+ fail="Failed to read the revision number at '{}'".format(self._get_branch_dir()),
+ )
if ret != 0:
raise SourceError("{}: Failed to get ref for tracking {}".format(self, self.tracking))
return out
def fetch(self): # pylint: disable=arguments-differ
- with self.timed_activity("Fetching {}".format(self.url),
- silent_nested=True), self._locked():
+ with self.timed_activity("Fetching {}".format(self.url), silent_nested=True), self._locked():
self._ensure_mirror()
def stage(self, directory):
- self.call([self.host_bzr, "checkout", "--lightweight",
- "--revision=revno:{}".format(self.ref),
- self._get_branch_dir(), directory],
- fail="Failed to checkout revision {} from branch {} to {}"
- .format(self.ref, self._get_branch_dir(), directory))
+ self.call(
+ [
+ self.host_bzr,
+ "checkout",
+ "--lightweight",
+ "--revision=revno:{}".format(self.ref),
+ self._get_branch_dir(),
+ directory,
+ ],
+ fail="Failed to checkout revision {} from branch {} to {}".format(
+ self.ref, self._get_branch_dir(), directory
+ ),
+ )
# Remove .bzr dir
shutil.rmtree(os.path.join(directory, ".bzr"))
@@ -133,16 +138,24 @@ class BzrSource(Source):
url = os.path.join(self.url, self.tracking)
with self.timed_activity('Setting up workspace "{}"'.format(directory), silent_nested=True):
# Checkout from the cache
- self.call([self.host_bzr, "branch",
- "--use-existing-dir",
- "--revision=revno:{}".format(self.ref),
- self._get_branch_dir(), directory],
- fail="Failed to branch revision {} from branch {} to {}"
- .format(self.ref, self._get_branch_dir(), directory))
+ self.call(
+ [
+ self.host_bzr,
+ "branch",
+ "--use-existing-dir",
+ "--revision=revno:{}".format(self.ref),
+ self._get_branch_dir(),
+ directory,
+ ],
+ fail="Failed to branch revision {} from branch {} to {}".format(
+ self.ref, self._get_branch_dir(), directory
+ ),
+ )
# Switch the parent branch to the source's origin
- self.call([self.host_bzr, "switch",
- "--directory={}".format(directory), url],
- fail="Failed to switch workspace's parent branch to {}".format(url))
+ self.call(
+ [self.host_bzr, "switch", "--directory={}".format(directory), url],
+ fail="Failed to switch workspace's parent branch to {}".format(url),
+ )
# _locked()
#
@@ -151,13 +164,10 @@ class BzrSource(Source):
#
@contextmanager
def _locked(self):
- lockdir = os.path.join(self.get_mirror_directory(), 'locks')
- lockfile = os.path.join(
- lockdir,
- utils.url_directory_name(self.original_url) + '.lock'
- )
+ lockdir = os.path.join(self.get_mirror_directory(), "locks")
+ lockfile = os.path.join(lockdir, utils.url_directory_name(self.original_url) + ".lock")
os.makedirs(lockdir, exist_ok=True)
- with open(lockfile, 'w') as lock:
+ with open(lockfile, "w") as lock:
fcntl.flock(lock, fcntl.LOCK_EX)
try:
yield
@@ -169,41 +179,42 @@ class BzrSource(Source):
if not os.path.exists(self._get_branch_dir()):
return False
- return self.call([self.host_bzr, "revno",
- "--revision=revno:{}".format(self.ref),
- self._get_branch_dir()]) == 0
+ return self.call([self.host_bzr, "revno", "--revision=revno:{}".format(self.ref), self._get_branch_dir()]) == 0
def _get_branch_dir(self):
return os.path.join(self._get_mirror_dir(), self.tracking)
def _get_mirror_dir(self):
- return os.path.join(self.get_mirror_directory(),
- utils.url_directory_name(self.original_url))
+ return os.path.join(self.get_mirror_directory(), utils.url_directory_name(self.original_url))
def _ensure_mirror(self, skip_ref_check=False):
mirror_dir = self._get_mirror_dir()
bzr_metadata_dir = os.path.join(mirror_dir, ".bzr")
if not os.path.exists(bzr_metadata_dir):
- self.call([self.host_bzr, "init-repo", "--no-trees", mirror_dir],
- fail="Failed to initialize bzr repository")
+ self.call(
+ [self.host_bzr, "init-repo", "--no-trees", mirror_dir], fail="Failed to initialize bzr repository"
+ )
branch_dir = os.path.join(mirror_dir, self.tracking)
branch_url = self.url + "/" + self.tracking
if not os.path.exists(branch_dir):
# `bzr branch` the branch if it doesn't exist
# to get the upstream code
- self.call([self.host_bzr, "branch", branch_url, branch_dir],
- fail="Failed to branch from {} to {}".format(branch_url, branch_dir))
+ self.call(
+ [self.host_bzr, "branch", branch_url, branch_dir],
+ fail="Failed to branch from {} to {}".format(branch_url, branch_dir),
+ )
else:
# `bzr pull` the branch if it does exist
# to get any changes to the upstream code
- self.call([self.host_bzr, "pull", "--directory={}".format(branch_dir), branch_url],
- fail="Failed to pull new changes for {}".format(branch_dir))
+ self.call(
+ [self.host_bzr, "pull", "--directory={}".format(branch_dir), branch_url],
+ fail="Failed to pull new changes for {}".format(branch_dir),
+ )
if not skip_ref_check and not self._check_ref():
- raise SourceError("Failed to ensure ref '{}' was mirrored".format(self.ref),
- reason="ref-not-mirrored")
+ raise SourceError("Failed to ensure ref '{}' was mirrored".format(self.ref), reason="ref-not-mirrored")
def setup():
diff --git a/src/buildstream/plugins/sources/deb.py b/src/buildstream/plugins/sources/deb.py
index cc88cf53c..a7437b150 100644
--- a/src/buildstream/plugins/sources/deb.py
+++ b/src/buildstream/plugins/sources/deb.py
@@ -50,7 +50,7 @@ details on common configuration options for sources.
import tarfile
from contextlib import contextmanager
-import arpy # pylint: disable=import-error
+import arpy # pylint: disable=import-error
from .tar import TarSource
@@ -61,14 +61,14 @@ class DebSource(TarSource):
def configure(self, node):
super().configure(node)
- self.base_dir = node.get_str('base-dir', None)
+ self.base_dir = node.get_str("base-dir", None)
def preflight(self):
return
@contextmanager
def _get_tar(self):
- with open(self._get_mirror_file(), 'rb') as deb_file:
+ with open(self._get_mirror_file(), "rb") as deb_file:
arpy_archive = arpy.Archive(fileobj=deb_file)
arpy_archive.read_all_headers()
data_tar_arpy = [v for k, v in arpy_archive.archived_files.items() if b"data.tar" in k][0]
diff --git a/src/buildstream/plugins/sources/local.py b/src/buildstream/plugins/sources/local.py
index f40fd79c0..90d8a8f6f 100644
--- a/src/buildstream/plugins/sources/local.py
+++ b/src/buildstream/plugins/sources/local.py
@@ -54,8 +54,8 @@ class LocalSource(Source):
self.__unique_key = None
def configure(self, node):
- node.validate_keys(['path', *Source.COMMON_CONFIG_KEYS])
- self.path = self.node_get_project_path(node.get_scalar('path'))
+ node.validate_keys(["path", *Source.COMMON_CONFIG_KEYS])
+ self.path = self.node_get_project_path(node.get_scalar("path"))
self.fullpath = os.path.join(self.get_project_directory(), self.path)
def preflight(self):
@@ -89,8 +89,8 @@ class LocalSource(Source):
if result.overwritten or result.ignored:
raise SourceError(
- "Failed to stage source: files clash with existing directory",
- reason='ensure-stage-dir-fail')
+ "Failed to stage source: files clash with existing directory", reason="ensure-stage-dir-fail"
+ )
def _get_local_path(self):
return self.fullpath
diff --git a/src/buildstream/plugins/sources/patch.py b/src/buildstream/plugins/sources/patch.py
index 86811cb4d..082983023 100644
--- a/src/buildstream/plugins/sources/patch.py
+++ b/src/buildstream/plugins/sources/patch.py
@@ -56,8 +56,7 @@ class PatchSource(Source):
def configure(self, node):
node.validate_keys(["path", "strip-level", *Source.COMMON_CONFIG_KEYS])
- self.path = self.node_get_project_path(node.get_scalar('path'),
- check_is_file=True)
+ self.path = self.node_get_project_path(node.get_scalar("path"), check_is_file=True)
self.strip_level = node.get_int("strip-level", default=1)
self.fullpath = os.path.join(self.get_project_directory(), self.path)
@@ -89,12 +88,13 @@ class PatchSource(Source):
# Bail out with a comprehensive message if the target directory is empty
if not os.listdir(directory):
- raise SourceError("Nothing to patch in directory '{}'".format(directory),
- reason="patch-no-files")
+ raise SourceError("Nothing to patch in directory '{}'".format(directory), reason="patch-no-files")
strip_level_option = "-p{}".format(self.strip_level)
- self.call([self.host_patch, strip_level_option, "-i", self.fullpath, "-d", directory],
- fail="Failed to apply patch {}".format(self.path))
+ self.call(
+ [self.host_patch, strip_level_option, "-i", self.fullpath, "-d", directory],
+ fail="Failed to apply patch {}".format(self.path),
+ )
# Plugin entry point
diff --git a/src/buildstream/plugins/sources/pip.py b/src/buildstream/plugins/sources/pip.py
index 758ef665f..2c9773787 100644
--- a/src/buildstream/plugins/sources/pip.py
+++ b/src/buildstream/plugins/sources/pip.py
@@ -74,30 +74,28 @@ import re
from buildstream import Consistency, Source, SourceError, utils
-_OUTPUT_DIRNAME = '.bst_pip_downloads'
-_PYPI_INDEX_URL = 'https://pypi.org/simple/'
+_OUTPUT_DIRNAME = ".bst_pip_downloads"
+_PYPI_INDEX_URL = "https://pypi.org/simple/"
# Used only for finding pip command
_PYTHON_VERSIONS = [
- 'python', # when running in a venv, we might not have the exact version
- 'python2.7',
- 'python3.0',
- 'python3.1',
- 'python3.2',
- 'python3.3',
- 'python3.4',
- 'python3.5',
- 'python3.6',
- 'python3.7',
+ "python", # when running in a venv, we might not have the exact version
+ "python2.7",
+ "python3.0",
+ "python3.1",
+ "python3.2",
+ "python3.3",
+ "python3.4",
+ "python3.5",
+ "python3.6",
+ "python3.7",
]
# List of allowed extensions taken from
# https://docs.python.org/3/distutils/sourcedist.html.
# Names of source distribution archives must be of the form
# '%{package-name}-%{version}.%{extension}'.
-_SDIST_RE = re.compile(
- r'^([\w.-]+?)-((?:[\d.]+){2,})\.(?:tar|tar.bz2|tar.gz|tar.xz|tar.Z|zip)$',
- re.IGNORECASE)
+_SDIST_RE = re.compile(r"^([\w.-]+?)-((?:[\d.]+){2,})\.(?:tar|tar.bz2|tar.gz|tar.xz|tar.Z|zip)$", re.IGNORECASE)
class PipSource(Source):
@@ -109,16 +107,15 @@ class PipSource(Source):
BST_REQUIRES_PREVIOUS_SOURCES_TRACK = True
def configure(self, node):
- node.validate_keys(['url', 'packages', 'ref', 'requirements-files'] +
- Source.COMMON_CONFIG_KEYS)
- self.ref = node.get_str('ref', None)
- self.original_url = node.get_str('url', _PYPI_INDEX_URL)
+ node.validate_keys(["url", "packages", "ref", "requirements-files"] + Source.COMMON_CONFIG_KEYS)
+ self.ref = node.get_str("ref", None)
+ self.original_url = node.get_str("url", _PYPI_INDEX_URL)
self.index_url = self.translate_url(self.original_url)
- self.packages = node.get_str_list('packages', [])
- self.requirements_files = node.get_str_list('requirements-files', [])
+ self.packages = node.get_str_list("packages", [])
+ self.requirements_files = node.get_str_list("requirements-files", [])
if not (self.packages or self.requirements_files):
- raise SourceError("{}: Either 'packages' or 'requirements-files' must be specified". format(self))
+ raise SourceError("{}: Either 'packages' or 'requirements-files' must be specified".format(self))
def preflight(self):
# Try to find a pip version that spports download command
@@ -126,9 +123,9 @@ class PipSource(Source):
for python in reversed(_PYTHON_VERSIONS):
try:
host_python = utils.get_host_tool(python)
- rc = self.call([host_python, '-m', 'pip', 'download', '--help'])
+ rc = self.call([host_python, "-m", "pip", "download", "--help"])
if rc == 0:
- self.host_pip = [host_python, '-m', 'pip']
+ self.host_pip = [host_python, "-m", "pip"]
break
except utils.ProgramNotFoundError:
pass
@@ -150,10 +147,10 @@ class PipSource(Source):
return self.ref
def load_ref(self, node):
- self.ref = node.get_str('ref', None)
+ self.ref = node.get_str("ref", None)
def set_ref(self, ref, node):
- node['ref'] = self.ref = ref
+ node["ref"] = self.ref = ref
def track(self, previous_sources_dir): # pylint: disable=arguments-differ
# XXX pip does not offer any public API other than the CLI tool so it
@@ -163,32 +160,44 @@ class PipSource(Source):
# for details.
# As a result, we have to wastefully install the packages during track.
with self.tempdir() as tmpdir:
- install_args = self.host_pip + ['download',
- '--no-binary', ':all:',
- '--index-url', self.index_url,
- '--dest', tmpdir]
+ install_args = self.host_pip + [
+ "download",
+ "--no-binary",
+ ":all:",
+ "--index-url",
+ self.index_url,
+ "--dest",
+ tmpdir,
+ ]
for requirement_file in self.requirements_files:
fpath = os.path.join(previous_sources_dir, requirement_file)
- install_args += ['-r', fpath]
+ install_args += ["-r", fpath]
install_args += self.packages
self.call(install_args, fail="Failed to install python packages")
reqs = self._parse_sdist_names(tmpdir)
- return '\n'.join(["{}=={}".format(pkg, ver) for pkg, ver in reqs])
+ return "\n".join(["{}=={}".format(pkg, ver) for pkg, ver in reqs])
def fetch(self): # pylint: disable=arguments-differ
with self.tempdir() as tmpdir:
- packages = self.ref.strip().split('\n')
- package_dir = os.path.join(tmpdir, 'packages')
+ packages = self.ref.strip().split("\n")
+ package_dir = os.path.join(tmpdir, "packages")
os.makedirs(package_dir)
- self.call([*self.host_pip,
- 'download',
- '--no-binary', ':all:',
- '--index-url', self.index_url,
- '--dest', package_dir,
- *packages],
- fail="Failed to install python packages: {}".format(packages))
+ self.call(
+ [
+ *self.host_pip,
+ "download",
+ "--no-binary",
+ ":all:",
+ "--index-url",
+ self.index_url,
+ "--dest",
+ package_dir,
+ *packages,
+ ],
+ fail="Failed to install python packages: {}".format(packages),
+ )
# If the mirror directory already exists, assume that some other
# process has fetched the sources before us and ensure that we do
@@ -200,8 +209,11 @@ class PipSource(Source):
# before us.
pass
except OSError as e:
- raise SourceError("{}: Failed to move downloaded pip packages from '{}' to '{}': {}"
- .format(self, package_dir, self._mirror, e)) from e
+ raise SourceError(
+ "{}: Failed to move downloaded pip packages from '{}' to '{}': {}".format(
+ self, package_dir, self._mirror, e
+ )
+ ) from e
def stage(self, directory):
with self.timed_activity("Staging Python packages", silent_nested=True):
@@ -213,9 +225,11 @@ class PipSource(Source):
def _mirror(self):
if not self.ref:
return None
- return os.path.join(self.get_mirror_directory(),
- utils.url_directory_name(self.original_url),
- hashlib.sha256(self.ref.encode()).hexdigest())
+ return os.path.join(
+ self.get_mirror_directory(),
+ utils.url_directory_name(self.original_url),
+ hashlib.sha256(self.ref.encode()).hexdigest(),
+ )
# Parse names of downloaded source distributions
#
diff --git a/src/buildstream/plugins/sources/remote.py b/src/buildstream/plugins/sources/remote.py
index 68aa577fc..da1a1f964 100644
--- a/src/buildstream/plugins/sources/remote.py
+++ b/src/buildstream/plugins/sources/remote.py
@@ -62,13 +62,14 @@ class RemoteSource(DownloadableFileSource):
def configure(self, node):
super().configure(node)
- self.filename = node.get_str('filename', os.path.basename(self.url))
- self.executable = node.get_bool('executable', default=False)
+ self.filename = node.get_str("filename", os.path.basename(self.url))
+ self.executable = node.get_bool("executable", default=False)
if os.sep in self.filename:
- raise SourceError('{}: filename parameter cannot contain directories'.format(self),
- reason="filename-contains-directory")
- node.validate_keys(DownloadableFileSource.COMMON_CONFIG_KEYS + ['filename', 'executable'])
+ raise SourceError(
+ "{}: filename parameter cannot contain directories".format(self), reason="filename-contains-directory"
+ )
+ node.validate_keys(DownloadableFileSource.COMMON_CONFIG_KEYS + ["filename", "executable"])
def get_unique_key(self):
return super().get_unique_key() + [self.filename, self.executable]
diff --git a/src/buildstream/plugins/sources/tar.py b/src/buildstream/plugins/sources/tar.py
index 60d464457..658cc2735 100644
--- a/src/buildstream/plugins/sources/tar.py
+++ b/src/buildstream/plugins/sources/tar.py
@@ -73,6 +73,7 @@ class ReadableTarInfo(tarfile.TarInfo):
`mode` attribute in `TarInfo`, the class that encapsulates the internal meta-data of the tarball,
so that the owner-read bit is always set.
"""
+
@property
def mode(self):
# ensure file is readable by owner
@@ -89,13 +90,13 @@ class TarSource(DownloadableFileSource):
def configure(self, node):
super().configure(node)
- self.base_dir = node.get_str('base-dir', '*')
- node.validate_keys(DownloadableFileSource.COMMON_CONFIG_KEYS + ['base-dir'])
+ self.base_dir = node.get_str("base-dir", "*")
+ node.validate_keys(DownloadableFileSource.COMMON_CONFIG_KEYS + ["base-dir"])
def preflight(self):
self.host_lzip = None
- if self.url.endswith('.lz'):
- self.host_lzip = utils.get_host_tool('lzip')
+ if self.url.endswith(".lz"):
+ self.host_lzip = utils.get_host_tool("lzip")
def get_unique_key(self):
return super().get_unique_key() + [self.base_dir]
@@ -104,19 +105,17 @@ class TarSource(DownloadableFileSource):
def _run_lzip(self):
assert self.host_lzip
with TemporaryFile() as lzip_stdout:
- with open(self._get_mirror_file(), 'r') as lzip_file:
- self.call([self.host_lzip, '-d'],
- stdin=lzip_file,
- stdout=lzip_stdout)
+ with open(self._get_mirror_file(), "r") as lzip_file:
+ self.call([self.host_lzip, "-d"], stdin=lzip_file, stdout=lzip_stdout)
lzip_stdout.seek(0, 0)
yield lzip_stdout
@contextmanager
def _get_tar(self):
- if self.url.endswith('.lz'):
+ if self.url.endswith(".lz"):
with self._run_lzip() as lzip_dec:
- with tarfile.open(fileobj=lzip_dec, mode='r:', tarinfo=ReadableTarInfo) as tar:
+ with tarfile.open(fileobj=lzip_dec, mode="r:", tarinfo=ReadableTarInfo) as tar:
yield tar
else:
with tarfile.open(self._get_mirror_file(), tarinfo=ReadableTarInfo) as tar:
@@ -147,14 +146,18 @@ class TarSource(DownloadableFileSource):
def assert_safe(member):
final_path = os.path.abspath(os.path.join(target_dir, member.path))
if not final_path.startswith(target_dir):
- raise SourceError("{}: Tarfile attempts to extract outside the staging area: "
- "{} -> {}".format(self, member.path, final_path))
+ raise SourceError(
+ "{}: Tarfile attempts to extract outside the staging area: "
+ "{} -> {}".format(self, member.path, final_path)
+ )
if member.islnk():
linked_path = os.path.abspath(os.path.join(target_dir, member.linkname))
if not linked_path.startswith(target_dir):
- raise SourceError("{}: Tarfile attempts to hardlink outside the staging area: "
- "{} -> {}".format(self, member.path, final_path))
+ raise SourceError(
+ "{}: Tarfile attempts to hardlink outside the staging area: "
+ "{} -> {}".format(self, member.path, final_path)
+ )
# Don't need to worry about symlinks because they're just
# files here and won't be able to do much harm once we are
@@ -167,9 +170,9 @@ class TarSource(DownloadableFileSource):
for member in tar.getmembers():
# First, ensure that a member never starts with `./`
- if member.path.startswith('./'):
+ if member.path.startswith("./"):
member.path = member.path[2:]
- if member.islnk() and member.linkname.startswith('./'):
+ if member.islnk() and member.linkname.startswith("./"):
member.linkname = member.linkname[2:]
# Now extract only the paths which match the normalized path
@@ -202,16 +205,16 @@ class TarSource(DownloadableFileSource):
# Remove any possible leading './', offer more consistent behavior
# across tarballs encoded with or without a leading '.'
- member_name = member.name.lstrip('./')
+ member_name = member.name.lstrip("./")
if not member.isdir():
# Loop over the components of a path, for a path of a/b/c/d
# we will first visit 'a', then 'a/b' and then 'a/b/c', excluding
# the final component
- components = member_name.split('/')
+ components = member_name.split("/")
for i in range(len(components) - 1):
- dir_component = '/'.join([components[j] for j in range(i + 1)])
+ dir_component = "/".join([components[j] for j in range(i + 1)])
if dir_component not in visited:
visited.add(dir_component)
try:
@@ -219,7 +222,7 @@ class TarSource(DownloadableFileSource):
# exist in the archive
_ = tar.getmember(dir_component)
except KeyError:
- if dir_component != '.':
+ if dir_component != ".":
yield dir_component
continue
@@ -227,7 +230,7 @@ class TarSource(DownloadableFileSource):
# Avoid considering the '.' directory, if any is included in the archive
# this is to avoid the default 'base-dir: *' value behaving differently
# depending on whether the tarball was encoded with a leading '.' or not
- elif member_name == '.':
+ elif member_name == ".":
continue
yield member_name
diff --git a/src/buildstream/plugins/sources/workspace.py b/src/buildstream/plugins/sources/workspace.py
index f40f5fae8..a845fd440 100644
--- a/src/buildstream/plugins/sources/workspace.py
+++ b/src/buildstream/plugins/sources/workspace.py
@@ -59,9 +59,9 @@ class WorkspaceSource(Source):
return None
def configure(self, node: MappingNode) -> None:
- node.validate_keys(['path', 'ref', 'kind'])
- self.path = node.get_str('path')
- self.__digest = node.get_str('ref')
+ node.validate_keys(["path", "ref", "kind"])
+ self.path = node.get_str("path")
+ self.__digest = node.get_str("ref")
def preflight(self) -> None:
pass # pragma: nocover
@@ -79,7 +79,7 @@ class WorkspaceSource(Source):
#
# Raises AssertionError: existing workspaces should not be reinitialized
def init_workspace(self, directory: Directory) -> None:
- raise AssertionError('Attempting to re-open an existing workspace')
+ raise AssertionError("Attempting to re-open an existing workspace")
def get_consistency(self):
# always return cached state
@@ -95,8 +95,8 @@ class WorkspaceSource(Source):
if result.overwritten or result.ignored:
raise SourceError(
- "Failed to stage source: files clash with existing directory",
- reason='ensure-stage-dir-fail')
+ "Failed to stage source: files clash with existing directory", reason="ensure-stage-dir-fail"
+ )
def _get_local_path(self) -> str:
return self.path
diff --git a/src/buildstream/plugins/sources/zip.py b/src/buildstream/plugins/sources/zip.py
index 322be58d7..47933c8eb 100644
--- a/src/buildstream/plugins/sources/zip.py
+++ b/src/buildstream/plugins/sources/zip.py
@@ -72,8 +72,8 @@ class ZipSource(DownloadableFileSource):
def configure(self, node):
super().configure(node)
- self.base_dir = node.get_str('base-dir', '*')
- node.validate_keys(DownloadableFileSource.COMMON_CONFIG_KEYS + ['base-dir'])
+ self.base_dir = node.get_str("base-dir", "*")
+ node.validate_keys(DownloadableFileSource.COMMON_CONFIG_KEYS + ["base-dir"])
def get_unique_key(self):
return super().get_unique_key() + [self.base_dir]
@@ -139,14 +139,14 @@ class ZipSource(DownloadableFileSource):
# ZipInfo.is_dir() is only available in python >= 3.6, but all
# it does is check for a trailing '/' in the name
#
- if not member.filename.endswith('/'):
+ if not member.filename.endswith("/"):
# Loop over the components of a path, for a path of a/b/c/d
# we will first visit 'a', then 'a/b' and then 'a/b/c', excluding
# the final component
- components = member.filename.split('/')
+ components = member.filename.split("/")
for i in range(len(components) - 1):
- dir_component = '/'.join([components[j] for j in range(i + 1)])
+ dir_component = "/".join([components[j] for j in range(i + 1)])
if dir_component not in visited:
visited[dir_component] = True
try:
@@ -154,7 +154,7 @@ class ZipSource(DownloadableFileSource):
# exist in the archive
_ = archive.getinfo(dir_component)
except KeyError:
- if dir_component != '.':
+ if dir_component != ".":
yield dir_component
continue
@@ -162,7 +162,7 @@ class ZipSource(DownloadableFileSource):
# Avoid considering the '.' directory, if any is included in the archive
# this is to avoid the default 'base-dir: *' value behaving differently
# depending on whether the archive was encoded with a leading '.' or not
- elif member.filename == '.' or member.filename == './':
+ elif member.filename == "." or member.filename == "./":
continue
yield member.filename
diff --git a/src/buildstream/sandbox/_config.py b/src/buildstream/sandbox/_config.py
index 457f92b3c..614f22063 100644
--- a/src/buildstream/sandbox/_config.py
+++ b/src/buildstream/sandbox/_config.py
@@ -22,7 +22,7 @@
#
# A container for sandbox configuration data. We want the internals
# of this to be opaque, hence putting it in its own private file.
-class SandboxConfig():
+class SandboxConfig:
def __init__(self, build_uid, build_gid, build_os=None, build_arch=None):
self.build_uid = build_uid
self.build_gid = build_gid
@@ -46,17 +46,14 @@ class SandboxConfig():
# However this should be the right place to support
# such configurations in the future.
#
- unique_key = {
- 'os': self.build_os,
- 'arch': self.build_arch
- }
+ unique_key = {"os": self.build_os, "arch": self.build_arch}
# Avoid breaking cache key calculation with
# the addition of configurabuild build uid/gid
if self.build_uid != 0:
- unique_key['build-uid'] = self.build_uid
+ unique_key["build-uid"] = self.build_uid
if self.build_gid != 0:
- unique_key['build-gid'] = self.build_gid
+ unique_key["build-gid"] = self.build_gid
return unique_key
diff --git a/src/buildstream/sandbox/_mount.py b/src/buildstream/sandbox/_mount.py
index c0f26c8d7..18751dde5 100644
--- a/src/buildstream/sandbox/_mount.py
+++ b/src/buildstream/sandbox/_mount.py
@@ -29,7 +29,7 @@ from .._fuse import SafeHardlinks
#
# Helper data object representing a single mount point in the mount map
#
-class Mount():
+class Mount:
def __init__(self, sandbox, mount_point, safe_hardlinks, fuse_mount_options=None):
# Getting _get_underlying_directory() here is acceptable as
# we're part of the sandbox code. This will fail if our
@@ -54,8 +54,8 @@ class Mount():
# Redirected mount
self.mount_origin = os.path.join(root_directory, mount_point.lstrip(os.sep))
self.mount_base = os.path.join(scratch_directory, utils.url_directory_name(mount_point))
- self.mount_source = os.path.join(self.mount_base, 'mount')
- self.mount_tempdir = os.path.join(self.mount_base, 'temp')
+ self.mount_source = os.path.join(self.mount_base, "mount")
+ self.mount_tempdir = os.path.join(self.mount_base, "temp")
os.makedirs(self.mount_origin, exist_ok=True)
os.makedirs(self.mount_tempdir, exist_ok=True)
else:
@@ -74,10 +74,10 @@ class Mount():
# When mounting a regular file, ensure the parent
# directory exists in the sandbox; and that an empty
# file is created at the mount location.
- parent_dir = os.path.dirname(self.mount_source.rstrip('/'))
+ parent_dir = os.path.dirname(self.mount_source.rstrip("/"))
os.makedirs(parent_dir, exist_ok=True)
if not os.path.exists(self.mount_source):
- with open(self.mount_source, 'w'):
+ with open(self.mount_source, "w"):
pass
@contextmanager
@@ -99,8 +99,7 @@ class Mount():
# sandbox (Sandbox): The sandbox object
# root_readonly (bool): Whether the sandbox root is readonly
#
-class MountMap():
-
+class MountMap:
def __init__(self, sandbox, root_readonly, fuse_mount_options=None):
# We will be doing the mounts in the order in which they were declared.
self.mounts = OrderedDict()
@@ -109,11 +108,11 @@ class MountMap():
fuse_mount_options = {}
# We want safe hardlinks on rootfs whenever root is not readonly
- self.mounts['/'] = Mount(sandbox, '/', not root_readonly, fuse_mount_options)
+ self.mounts["/"] = Mount(sandbox, "/", not root_readonly, fuse_mount_options)
for mark in sandbox._get_marked_directories():
- directory = mark['directory']
- artifact = mark['artifact']
+ directory = mark["directory"]
+ artifact = mark["artifact"]
# We want safe hardlinks for any non-root directory where
# artifacts will be staged to
diff --git a/src/buildstream/sandbox/_mounter.py b/src/buildstream/sandbox/_mounter.py
index 4e31ef67a..3adf8ff5b 100644
--- a/src/buildstream/sandbox/_mounter.py
+++ b/src/buildstream/sandbox/_mounter.py
@@ -25,22 +25,20 @@ from .. import utils, _signals
# A class to wrap the `mount` and `umount` system commands
-class Mounter():
+class Mounter:
@classmethod
- def _mount(cls, dest, src=None, mount_type=None,
- stdout=None, stderr=None, options=None,
- flags=None):
+ def _mount(cls, dest, src=None, mount_type=None, stdout=None, stderr=None, options=None, flags=None):
if stdout is None:
stdout = sys.stdout
if stderr is None:
stderr = sys.stderr
- argv = [utils.get_host_tool('mount')]
+ argv = [utils.get_host_tool("mount")]
if mount_type:
- argv.extend(['-t', mount_type])
+ argv.extend(["-t", mount_type])
if options:
- argv.extend(['-o', options])
+ argv.extend(["-o", options])
if flags:
argv.extend(flags)
@@ -48,16 +46,10 @@ class Mounter():
argv += [src]
argv += [dest]
- status, _ = utils._call(
- argv,
- terminate=True,
- stdout=stdout,
- stderr=stderr
- )
+ status, _ = utils._call(argv, terminate=True, stdout=stdout, stderr=stderr)
if status != 0:
- raise SandboxError('`{}` failed with exit code {}'
- .format(' '.join(argv), status))
+ raise SandboxError("`{}` failed with exit code {}".format(" ".join(argv), status))
return dest
@@ -68,17 +60,11 @@ class Mounter():
if stderr is None:
stderr = sys.stderr
- cmd = [utils.get_host_tool('umount'), '-R', path]
- status, _ = utils._call(
- cmd,
- terminate=True,
- stdout=stdout,
- stderr=stderr
- )
+ cmd = [utils.get_host_tool("umount"), "-R", path]
+ status, _ = utils._call(cmd, terminate=True, stdout=stdout, stderr=stderr)
if status != 0:
- raise SandboxError('`{}` failed with exit code {}'
- .format(' '.join(cmd), status))
+ raise SandboxError("`{}` failed with exit code {}".format(" ".join(cmd), status))
# mount()
#
@@ -98,8 +84,7 @@ class Mounter():
#
@classmethod
@contextmanager
- def mount(cls, dest, src=None, stdout=None,
- stderr=None, mount_type=None, **kwargs):
+ def mount(cls, dest, src=None, stdout=None, stderr=None, mount_type=None, **kwargs):
if stdout is None:
stdout = sys.stdout
if stderr is None:
@@ -108,7 +93,7 @@ class Mounter():
def kill_proc():
cls._umount(dest, stdout, stderr)
- options = ','.join([key for key, val in kwargs.items() if val])
+ options = ",".join([key for key, val in kwargs.items() if val])
path = cls._mount(dest, src, mount_type, stdout=stdout, stderr=stderr, options=options)
try:
@@ -139,8 +124,7 @@ class Mounter():
#
@classmethod
@contextmanager
- def bind_mount(cls, dest, src=None, stdout=None,
- stderr=None, **kwargs):
+ def bind_mount(cls, dest, src=None, stdout=None, stderr=None, **kwargs):
if stdout is None:
stdout = sys.stdout
if stderr is None:
@@ -149,8 +133,8 @@ class Mounter():
def kill_proc():
cls._umount(dest, stdout, stderr)
- kwargs['rbind'] = True
- options = ','.join([key for key, val in kwargs.items() if val])
+ kwargs["rbind"] = True
+ options = ",".join([key for key, val in kwargs.items() if val])
path = cls._mount(dest, src, None, stdout, stderr, options)
@@ -158,7 +142,7 @@ class Mounter():
with _signals.terminator(kill_proc):
# Make the rbind a slave to avoid unmounting vital devices in
# /proc
- cls._mount(dest, flags=['--make-rslave'])
+ cls._mount(dest, flags=["--make-rslave"])
yield path
finally:
cls._umount(dest, stdout, stderr)
diff --git a/src/buildstream/sandbox/_sandboxbuildbox.py b/src/buildstream/sandbox/_sandboxbuildbox.py
index 4258ee26d..c34d95223 100644
--- a/src/buildstream/sandbox/_sandboxbuildbox.py
+++ b/src/buildstream/sandbox/_sandboxbuildbox.py
@@ -34,22 +34,20 @@ from .._exceptions import SandboxError
# BuildBox-based sandbox implementation.
#
class SandboxBuildBox(Sandbox):
-
def __init__(self, context, project, directory, **kwargs):
- if kwargs.get('allow_real_directory'):
+ if kwargs.get("allow_real_directory"):
raise SandboxError("BuildBox does not support real directories")
- kwargs['allow_real_directory'] = False
+ kwargs["allow_real_directory"] = False
super().__init__(context, project, directory, **kwargs)
@classmethod
def check_available(cls):
try:
- utils.get_host_tool('buildbox')
+ utils.get_host_tool("buildbox")
except utils.ProgramNotFoundError as Error:
cls._dummy_reasons += ["buildbox not found"]
- raise SandboxError(" and ".join(cls._dummy_reasons),
- reason="unavailable-local-sandbox") from Error
+ raise SandboxError(" and ".join(cls._dummy_reasons), reason="unavailable-local-sandbox") from Error
@classmethod
def check_sandbox_config(cls, platform, config):
@@ -73,42 +71,42 @@ class SandboxBuildBox(Sandbox):
scratch_directory = self._get_scratch_directory()
if not self._has_command(command[0], env):
- raise SandboxCommandError("Staged artifacts do not provide command "
- "'{}'".format(command[0]),
- reason='missing-command')
+ raise SandboxCommandError(
+ "Staged artifacts do not provide command " "'{}'".format(command[0]), reason="missing-command"
+ )
# Grab the full path of the buildbox binary
try:
- buildbox_command = [utils.get_host_tool('buildbox')]
+ buildbox_command = [utils.get_host_tool("buildbox")]
except ProgramNotFoundError as Err:
- raise SandboxError(("BuildBox not on path, you are using the BuildBox sandbox because "
- "BST_FORCE_SANDBOX=buildbox")) from Err
+ raise SandboxError(
+ ("BuildBox not on path, you are using the BuildBox sandbox because " "BST_FORCE_SANDBOX=buildbox")
+ ) from Err
for mark in self._get_marked_directories():
- path = mark['directory']
- assert path.startswith('/') and len(path) > 1
+ path = mark["directory"]
+ assert path.startswith("/") and len(path) > 1
root_directory.descend(*path[1:].split(os.path.sep), create=True)
digest = root_directory._get_digest()
- with open(os.path.join(scratch_directory, 'in'), 'wb') as input_digest_file:
+ with open(os.path.join(scratch_directory, "in"), "wb") as input_digest_file:
input_digest_file.write(digest.SerializeToString())
buildbox_command += ["--local=" + root_directory.cas_cache.casdir]
buildbox_command += ["--input-digest=in"]
buildbox_command += ["--output-digest=out"]
- common_details = ("BuildBox is a experimental sandbox and does not support the requested feature.\n"
- "You are using this feature because BST_FORCE_SANDBOX=buildbox.")
+ common_details = (
+ "BuildBox is a experimental sandbox and does not support the requested feature.\n"
+ "You are using this feature because BST_FORCE_SANDBOX=buildbox."
+ )
if not flags & SandboxFlags.NETWORK_ENABLED:
# TODO
- self._issue_warning(
- "BuildBox sandbox does not have Networking yet",
- detail=common_details
- )
+ self._issue_warning("BuildBox sandbox does not have Networking yet", detail=common_details)
if cwd is not None:
- buildbox_command += ['--chdir=' + cwd]
+ buildbox_command += ["--chdir=" + cwd]
# In interactive mode, we want a complete devpts inside
# the container, so there is a /dev/console and such. In
@@ -118,27 +116,24 @@ class SandboxBuildBox(Sandbox):
if flags & SandboxFlags.INTERACTIVE:
# TODO
self._issue_warning(
- "BuildBox sandbox does not fully support BuildStream shells yet",
- detail=common_details
+ "BuildBox sandbox does not fully support BuildStream shells yet", detail=common_details
)
if flags & SandboxFlags.ROOT_READ_ONLY:
# TODO
self._issue_warning(
- "BuildBox sandbox does not fully support BuildStream `Read only Root`",
- detail=common_details
+ "BuildBox sandbox does not fully support BuildStream `Read only Root`", detail=common_details
)
# Set UID and GID
if not flags & SandboxFlags.INHERIT_UID:
# TODO
self._issue_warning(
- "BuildBox sandbox does not fully support BuildStream Inherit UID",
- detail=common_details
+ "BuildBox sandbox does not fully support BuildStream Inherit UID", detail=common_details
)
- os.makedirs(os.path.join(scratch_directory, 'mnt'), exist_ok=True)
- buildbox_command += ['mnt']
+ os.makedirs(os.path.join(scratch_directory, "mnt"), exist_ok=True)
+ buildbox_command += ["mnt"]
# Add the command
buildbox_command += command
@@ -150,7 +145,7 @@ class SandboxBuildBox(Sandbox):
with ExitStack() as stack:
# Ensure the cwd exists
if cwd is not None and len(cwd) > 1:
- assert cwd.startswith('/')
+ assert cwd.startswith("/")
root_directory.descend(*cwd[1:].split(os.path.sep), create=True)
# If we're interactive, we want to inherit our stdin,
@@ -162,12 +157,18 @@ class SandboxBuildBox(Sandbox):
stdin = stack.enter_context(open(os.devnull, "r"))
# Run buildbox !
- exit_code = self.run_buildbox(buildbox_command, stdin, stdout, stderr, env,
- interactive=(flags & SandboxFlags.INTERACTIVE),
- cwd=scratch_directory)
+ exit_code = self.run_buildbox(
+ buildbox_command,
+ stdin,
+ stdout,
+ stderr,
+ env,
+ interactive=(flags & SandboxFlags.INTERACTIVE),
+ cwd=scratch_directory,
+ )
if exit_code == 0:
- with open(os.path.join(scratch_directory, 'out'), 'rb') as output_digest_file:
+ with open(os.path.join(scratch_directory, "out"), "rb") as output_digest_file:
output_digest = remote_execution_pb2.Digest()
output_digest.ParseFromString(output_digest_file.read())
self._vdir = CasBasedDirectory(root_directory.cas_cache, digest=output_digest)
@@ -203,7 +204,7 @@ class SandboxBuildBox(Sandbox):
stdout=stdout,
stderr=stderr,
cwd=cwd,
- start_new_session=interactive
+ start_new_session=interactive,
)
# Wait for the child process to finish, ensuring that
diff --git a/src/buildstream/sandbox/_sandboxbwrap.py b/src/buildstream/sandbox/_sandboxbwrap.py
index 5c4b9a295..1405611bc 100644
--- a/src/buildstream/sandbox/_sandboxbwrap.py
+++ b/src/buildstream/sandbox/_sandboxbwrap.py
@@ -48,34 +48,27 @@ class SandboxBwrap(Sandbox):
_have_good_bwrap = None
# Minimal set of devices for the sandbox
- DEVICES = [
- '/dev/full',
- '/dev/null',
- '/dev/urandom',
- '/dev/random',
- '/dev/zero'
- ]
+ DEVICES = ["/dev/full", "/dev/null", "/dev/urandom", "/dev/random", "/dev/zero"]
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
- self.linux32 = kwargs['linux32']
+ self.linux32 = kwargs["linux32"]
@classmethod
def check_available(cls):
cls._have_fuse = os.path.exists("/dev/fuse")
if not cls._have_fuse:
- cls._dummy_reasons += ['Fuse is unavailable']
+ cls._dummy_reasons += ["Fuse is unavailable"]
try:
- utils.get_host_tool('bwrap')
+ utils.get_host_tool("bwrap")
except utils.ProgramNotFoundError as Error:
cls._bwrap_exists = False
cls._have_good_bwrap = False
cls._die_with_parent_available = False
cls._json_status_available = False
- cls._dummy_reasons += ['Bubblewrap not found']
- raise SandboxError(" and ".join(cls._dummy_reasons),
- reason="unavailable-local-sandbox") from Error
+ cls._dummy_reasons += ["Bubblewrap not found"]
+ raise SandboxError(" and ".join(cls._dummy_reasons), reason="unavailable-local-sandbox") from Error
bwrap_version = _site.get_bwrap_version()
@@ -84,7 +77,7 @@ class SandboxBwrap(Sandbox):
cls._die_with_parent_available = (0, 1, 8) <= bwrap_version
cls._json_status_available = (0, 3, 2) <= bwrap_version
if not cls._have_good_bwrap:
- cls._dummy_reasons += ['Bubblewrap is too old']
+ cls._dummy_reasons += ["Bubblewrap is too old"]
raise SandboxError(" and ".join(cls._dummy_reasons))
cls._uid = os.geteuid()
@@ -98,29 +91,26 @@ class SandboxBwrap(Sandbox):
# issue a warning if it's not available, and save the state
# locally so that we can inform the sandbox to not try it
# later on.
- bwrap = utils.get_host_tool('bwrap')
+ bwrap = utils.get_host_tool("bwrap")
try:
- whoami = utils.get_host_tool('whoami')
- output = subprocess.check_output([
- bwrap,
- '--ro-bind', '/', '/',
- '--unshare-user',
- '--uid', '0', '--gid', '0',
- whoami,
- ], universal_newlines=True).strip()
+ whoami = utils.get_host_tool("whoami")
+ output = subprocess.check_output(
+ [bwrap, "--ro-bind", "/", "/", "--unshare-user", "--uid", "0", "--gid", "0", whoami,],
+ universal_newlines=True,
+ ).strip()
except subprocess.CalledProcessError:
- output = ''
+ output = ""
except utils.ProgramNotFoundError:
- output = ''
+ output = ""
- return output == 'root'
+ return output == "root"
@classmethod
def check_sandbox_config(cls, local_platform, config):
if cls.user_ns_available:
# User namespace support allows arbitrary build UID/GID settings.
pass
- elif (config.build_uid != local_platform._uid or config.build_gid != local_platform._gid):
+ elif config.build_uid != local_platform._uid or config.build_gid != local_platform._gid:
# Without user namespace support, the UID/GID in the sandbox
# will match the host UID/GID.
return False
@@ -141,9 +131,9 @@ class SandboxBwrap(Sandbox):
root_directory = self.get_virtual_directory()._get_underlying_directory()
if not self._has_command(command[0], env):
- raise SandboxCommandError("Staged artifacts do not provide command "
- "'{}'".format(command[0]),
- reason='missing-command')
+ raise SandboxCommandError(
+ "Staged artifacts do not provide command " "'{}'".format(command[0]), reason="missing-command"
+ )
# NOTE: MountMap transitively imports `_fuse/fuse.py` which raises an
# EnvironmentError when fuse is not found. Since this module is
@@ -154,29 +144,29 @@ class SandboxBwrap(Sandbox):
# Create the mount map, this will tell us where
# each mount point needs to be mounted from and to
mount_map = MountMap(self, flags & SandboxFlags.ROOT_READ_ONLY)
- root_mount_source = mount_map.get_mount_source('/')
+ root_mount_source = mount_map.get_mount_source("/")
# start command with linux32 if needed
if self.linux32:
- bwrap_command = [utils.get_host_tool('linux32')]
+ bwrap_command = [utils.get_host_tool("linux32")]
else:
bwrap_command = []
# Grab the full path of the bwrap binary
- bwrap_command += [utils.get_host_tool('bwrap')]
+ bwrap_command += [utils.get_host_tool("bwrap")]
for k, v in env.items():
- bwrap_command += ['--setenv', k, v]
+ bwrap_command += ["--setenv", k, v]
for k in os.environ.keys() - env.keys():
- bwrap_command += ['--unsetenv', k]
+ bwrap_command += ["--unsetenv", k]
# Create a new pid namespace, this also ensures that any subprocesses
# are cleaned up when the bwrap process exits.
- bwrap_command += ['--unshare-pid']
+ bwrap_command += ["--unshare-pid"]
# Ensure subprocesses are cleaned up when the bwrap parent dies.
if self._die_with_parent_available:
- bwrap_command += ['--die-with-parent']
+ bwrap_command += ["--die-with-parent"]
# Add in the root filesystem stuff first.
#
@@ -186,15 +176,12 @@ class SandboxBwrap(Sandbox):
bwrap_command += ["--bind", root_mount_source, "/"]
if not flags & SandboxFlags.NETWORK_ENABLED:
- bwrap_command += ['--unshare-net']
- bwrap_command += ['--unshare-uts', '--hostname', 'buildstream']
- bwrap_command += ['--unshare-ipc']
+ bwrap_command += ["--unshare-net"]
+ bwrap_command += ["--unshare-uts", "--hostname", "buildstream"]
+ bwrap_command += ["--unshare-ipc"]
# Give it a proc and tmpfs
- bwrap_command += [
- '--proc', '/proc',
- '--tmpfs', '/tmp'
- ]
+ bwrap_command += ["--proc", "/proc", "--tmpfs", "/tmp"]
# In interactive mode, we want a complete devpts inside
# the container, so there is a /dev/console and such. In
@@ -202,21 +189,21 @@ class SandboxBwrap(Sandbox):
# a minimal set of devices to expose to the sandbox.
#
if flags & SandboxFlags.INTERACTIVE:
- bwrap_command += ['--dev', '/dev']
+ bwrap_command += ["--dev", "/dev"]
else:
for device in self.DEVICES:
- bwrap_command += ['--dev-bind', device, device]
+ bwrap_command += ["--dev-bind", device, device]
# Create a tmpfs for /dev/shm, if we're in interactive this
# is handled by `--dev /dev`
#
- bwrap_command += ['--tmpfs', '/dev/shm']
+ bwrap_command += ["--tmpfs", "/dev/shm"]
# Add bind mounts to any marked directories
marked_directories = self._get_marked_directories()
mount_source_overrides = self._get_mount_sources()
for mark in marked_directories:
- mount_point = mark['directory']
+ mount_point = mark["directory"]
if mount_point in mount_source_overrides: # pylint: disable=consider-using-get
mount_source = mount_source_overrides[mount_point]
else:
@@ -230,22 +217,22 @@ class SandboxBwrap(Sandbox):
# harmless to do in a build environment where the directories
# we mount just never contain device files.
#
- bwrap_command += ['--dev-bind', mount_source, mount_point]
+ bwrap_command += ["--dev-bind", mount_source, mount_point]
if flags & SandboxFlags.ROOT_READ_ONLY:
bwrap_command += ["--remount-ro", "/"]
if cwd is not None:
- bwrap_command += ['--dir', cwd]
- bwrap_command += ['--chdir', cwd]
+ bwrap_command += ["--dir", cwd]
+ bwrap_command += ["--chdir", cwd]
# Set UID and GUI
if self.user_ns_available:
- bwrap_command += ['--unshare-user']
+ bwrap_command += ["--unshare-user"]
if not flags & SandboxFlags.INHERIT_UID:
uid = self._get_config().build_uid
gid = self._get_config().build_gid
- bwrap_command += ['--uid', str(uid), '--gid', str(gid)]
+ bwrap_command += ["--uid", str(uid), "--gid", str(gid)]
with ExitStack() as stack:
pass_fds = ()
@@ -253,7 +240,7 @@ class SandboxBwrap(Sandbox):
if self._json_status_available:
json_status_file = stack.enter_context(TemporaryFile())
pass_fds = (json_status_file.fileno(),)
- bwrap_command += ['--json-status-fd', str(json_status_file.fileno())]
+ bwrap_command += ["--json-status-fd", str(json_status_file.fileno())]
# Add the command
bwrap_command += command
@@ -265,7 +252,7 @@ class SandboxBwrap(Sandbox):
#
existing_basedirs = {
directory: os.path.exists(os.path.join(root_directory, directory))
- for directory in ['dev/shm', 'tmp', 'dev', 'proc']
+ for directory in ["dev/shm", "tmp", "dev", "proc"]
}
# Use the MountMap context manager to ensure that any redirected
@@ -283,15 +270,16 @@ class SandboxBwrap(Sandbox):
stdin = stack.enter_context(open(os.devnull, "r"))
# Run bubblewrap !
- exit_code = self.run_bwrap(bwrap_command, stdin, stdout, stderr,
- (flags & SandboxFlags.INTERACTIVE), pass_fds)
+ exit_code = self.run_bwrap(
+ bwrap_command, stdin, stdout, stderr, (flags & SandboxFlags.INTERACTIVE), pass_fds
+ )
# Cleanup things which bwrap might have left behind, while
# everything is still mounted because bwrap can be creating
# the devices on the fuse mount, so we should remove it there.
if not flags & SandboxFlags.INTERACTIVE:
for device in self.DEVICES:
- device_path = os.path.join(root_mount_source, device.lstrip('/'))
+ device_path = os.path.join(root_mount_source, device.lstrip("/"))
# This will remove the device in a loop, allowing some
# retries in case the device file leaked by bubblewrap is still busy
@@ -299,7 +287,7 @@ class SandboxBwrap(Sandbox):
# Remove /tmp, this is a bwrap owned thing we want to be sure
# never ends up in an artifact
- for basedir in ['dev/shm', 'tmp', 'dev', 'proc']:
+ for basedir in ["dev/shm", "tmp", "dev", "proc"]:
# Skip removal of directories which already existed before
# launching bwrap
@@ -341,12 +329,14 @@ class SandboxBwrap(Sandbox):
for line in json_status_file:
with suppress(json.decoder.JSONDecodeError):
o = json.loads(line.decode())
- if isinstance(o, collections.abc.Mapping) and 'exit-code' in o:
- child_exit_code = o['exit-code']
+ if isinstance(o, collections.abc.Mapping) and "exit-code" in o:
+ child_exit_code = o["exit-code"]
break
if child_exit_code is None:
- raise SandboxError("`bwrap' terminated during sandbox setup with exitcode {}".format(exit_code),
- reason="bwrap-sandbox-fail")
+ raise SandboxError(
+ "`bwrap' terminated during sandbox setup with exitcode {}".format(exit_code),
+ reason="bwrap-sandbox-fail",
+ )
exit_code = child_exit_code
self._vdir._mark_changed()
@@ -432,7 +422,7 @@ class SandboxBwrap(Sandbox):
stdin=stdin,
stdout=stdout,
stderr=stderr,
- start_new_session=new_session
+ start_new_session=new_session,
)
# Wait for the child process to finish, ensuring that
diff --git a/src/buildstream/sandbox/_sandboxchroot.py b/src/buildstream/sandbox/_sandboxchroot.py
index 8d4c54058..1805131b1 100644
--- a/src/buildstream/sandbox/_sandboxchroot.py
+++ b/src/buildstream/sandbox/_sandboxchroot.py
@@ -35,7 +35,7 @@ from . import Sandbox, SandboxFlags, SandboxCommandError
class SandboxChroot(Sandbox):
- _FUSE_MOUNT_OPTIONS = {'dev': True}
+ _FUSE_MOUNT_OPTIONS = {"dev": True}
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@@ -43,8 +43,10 @@ class SandboxChroot(Sandbox):
uid = self._get_config().build_uid
gid = self._get_config().build_gid
if uid != 0 or gid != 0:
- raise SandboxError("Chroot sandboxes cannot specify a non-root uid/gid "
- "({},{} were supplied via config)".format(uid, gid))
+ raise SandboxError(
+ "Chroot sandboxes cannot specify a non-root uid/gid "
+ "({},{} were supplied via config)".format(uid, gid)
+ )
self.mount_map = None
@@ -78,20 +80,19 @@ class SandboxChroot(Sandbox):
def _run(self, command, flags, *, cwd, env):
if not self._has_command(command[0], env):
- raise SandboxCommandError("Staged artifacts do not provide command "
- "'{}'".format(command[0]),
- reason='missing-command')
+ raise SandboxCommandError(
+ "Staged artifacts do not provide command " "'{}'".format(command[0]), reason="missing-command"
+ )
stdout, stderr = self._get_output()
# Create the mount map, this will tell us where
# each mount point needs to be mounted from and to
- self.mount_map = MountMap(self, flags & SandboxFlags.ROOT_READ_ONLY,
- self._FUSE_MOUNT_OPTIONS)
+ self.mount_map = MountMap(self, flags & SandboxFlags.ROOT_READ_ONLY, self._FUSE_MOUNT_OPTIONS)
# Create a sysroot and run the command inside it
with ExitStack() as stack:
- os.makedirs('/var/run/buildstream', exist_ok=True)
+ os.makedirs("/var/run/buildstream", exist_ok=True)
# FIXME: While we do not currently do anything to prevent
# network access, we also don't copy /etc/resolv.conf to
@@ -104,21 +105,20 @@ class SandboxChroot(Sandbox):
#
# Nonetheless a better solution could perhaps be found.
- rootfs = stack.enter_context(utils._tempdir(dir='/var/run/buildstream'))
+ rootfs = stack.enter_context(utils._tempdir(dir="/var/run/buildstream"))
stack.enter_context(self.create_devices(self._root, flags))
stack.enter_context(self.mount_dirs(rootfs, flags, stdout, stderr))
if flags & SandboxFlags.INTERACTIVE:
stdin = sys.stdin
else:
- stdin = stack.enter_context(open(os.devnull, 'r'))
+ stdin = stack.enter_context(open(os.devnull, "r"))
# Ensure the cwd exists
if cwd is not None:
workdir = os.path.join(rootfs, cwd.lstrip(os.sep))
os.makedirs(workdir, exist_ok=True)
- status = self.chroot(rootfs, command, stdin, stdout,
- stderr, cwd, env, flags)
+ status = self.chroot(rootfs, command, stdin, stdout, stderr, cwd, env, flags)
self._vdir._mark_changed()
return status
@@ -173,7 +173,7 @@ class SandboxChroot(Sandbox):
# If you try to put gtk dialogs here Tristan (either)
# will personally scald you
preexec_fn=lambda: (os.chroot(rootfs), os.chdir(cwd)),
- start_new_session=flags & SandboxFlags.INTERACTIVE
+ start_new_session=flags & SandboxFlags.INTERACTIVE,
)
# Wait for the child process to finish, ensuring that
@@ -214,13 +214,14 @@ class SandboxChroot(Sandbox):
# Exceptions in preexec_fn are simply reported as
# 'Exception occurred in preexec_fn', turn these into
# a more readable message.
- if str(e) == 'Exception occurred in preexec_fn.':
- raise SandboxError('Could not chroot into {} or chdir into {}. '
- 'Ensure you are root and that the relevant directory exists.'
- .format(rootfs, cwd)) from e
+ if str(e) == "Exception occurred in preexec_fn.":
+ raise SandboxError(
+ "Could not chroot into {} or chdir into {}. "
+ "Ensure you are root and that the relevant directory exists.".format(rootfs, cwd)
+ ) from e
# Otherwise, raise a more general error
- raise SandboxError('Could not run command {}: {}'.format(command, e)) from e
+ raise SandboxError("Could not run command {}: {}".format(command, e)) from e
return code
@@ -251,8 +252,10 @@ class SandboxChroot(Sandbox):
devices.append(self.mknod(device, location))
except OSError as err:
if err.errno == 1:
- raise SandboxError("Permission denied while creating device node: {}.".format(err) +
- "BuildStream reqiures root permissions for these setttings.")
+ raise SandboxError(
+ "Permission denied while creating device node: {}.".format(err)
+ + "BuildStream reqiures root permissions for these setttings."
+ )
raise
@@ -300,16 +303,16 @@ class SandboxChroot(Sandbox):
with ExitStack() as stack:
stack.enter_context(self.mount_map.mounted(self))
- stack.enter_context(mount_point('/'))
+ stack.enter_context(mount_point("/"))
if flags & SandboxFlags.INTERACTIVE:
- stack.enter_context(mount_src('/dev'))
+ stack.enter_context(mount_src("/dev"))
- stack.enter_context(mount_src('/tmp'))
- stack.enter_context(mount_src('/proc'))
+ stack.enter_context(mount_src("/tmp"))
+ stack.enter_context(mount_src("/proc"))
for mark in self._get_marked_directories():
- stack.enter_context(mount_point(mark['directory']))
+ stack.enter_context(mount_point(mark["directory"]))
# Remount root RO if necessary
if flags & flags & SandboxFlags.ROOT_READ_ONLY:
@@ -343,10 +346,9 @@ class SandboxChroot(Sandbox):
os.mknod(target, mode=stat.S_IFCHR | dev.st_mode, device=target_dev)
except PermissionError as e:
- raise SandboxError('Could not create device {}, ensure that you have root permissions: {}')
+ raise SandboxError("Could not create device {}, ensure that you have root permissions: {}")
except OSError as e:
- raise SandboxError('Could not create device {}: {}'
- .format(target, e)) from e
+ raise SandboxError("Could not create device {}: {}".format(target, e)) from e
return target
diff --git a/src/buildstream/sandbox/_sandboxdummy.py b/src/buildstream/sandbox/_sandboxdummy.py
index ae3d5e512..0cae2b6a9 100644
--- a/src/buildstream/sandbox/_sandboxdummy.py
+++ b/src/buildstream/sandbox/_sandboxdummy.py
@@ -28,9 +28,10 @@ class SandboxDummy(Sandbox):
def _run(self, command, flags, *, cwd, env):
if not self._has_command(command[0], env):
- raise SandboxCommandError("Staged artifacts do not provide command "
- "'{}'".format(command[0]),
- reason='missing-command')
+ raise SandboxCommandError(
+ "Staged artifacts do not provide command " "'{}'".format(command[0]), reason="missing-command"
+ )
- raise SandboxError("This platform does not support local builds: {}".format(self._reason),
- reason="unavailable-local-sandbox")
+ raise SandboxError(
+ "This platform does not support local builds: {}".format(self._reason), reason="unavailable-local-sandbox"
+ )
diff --git a/src/buildstream/sandbox/_sandboxreapi.py b/src/buildstream/sandbox/_sandboxreapi.py
index 31c1c9674..2d661c893 100644
--- a/src/buildstream/sandbox/_sandboxreapi.py
+++ b/src/buildstream/sandbox/_sandboxreapi.py
@@ -30,7 +30,6 @@ from ..storage._casbaseddirectory import CasBasedDirectory
# the Remote Execution API.
#
class SandboxREAPI(Sandbox):
-
def _use_cas_based_directory(self):
# Always use CasBasedDirectory for REAPI
return True
@@ -46,14 +45,14 @@ class SandboxREAPI(Sandbox):
# Ensure working directory exists
if len(cwd) > 1:
- assert cwd.startswith('/')
+ assert cwd.startswith("/")
vdir.descend(*cwd[1:].split(os.path.sep), create=True)
# Create directories for all marked directories. This emulates
# some of the behaviour of other sandboxes, which create these
# to use as mount points.
for mark in self._get_marked_directories():
- directory = mark['directory']
+ directory = mark["directory"]
# Create each marked directory
vdir.descend(*directory.split(os.path.sep), create=True)
@@ -61,21 +60,21 @@ class SandboxREAPI(Sandbox):
input_root_digest = vdir._get_digest()
command_proto = self._create_command(command, cwd, env)
command_digest = cascache.add_object(buffer=command_proto.SerializeToString())
- action = remote_execution_pb2.Action(command_digest=command_digest,
- input_root_digest=input_root_digest)
+ action = remote_execution_pb2.Action(command_digest=command_digest, input_root_digest=input_root_digest)
action_result = self._execute_action(action) # pylint: disable=assignment-from-no-return
# Get output of build
- self._process_job_output(action_result.output_directories, action_result.output_files,
- failure=action_result.exit_code != 0)
+ self._process_job_output(
+ action_result.output_directories, action_result.output_files, failure=action_result.exit_code != 0
+ )
if stdout:
if action_result.stdout_raw:
- stdout.write(str(action_result.stdout_raw, 'utf-8', errors='ignore'))
+ stdout.write(str(action_result.stdout_raw, "utf-8", errors="ignore"))
if stderr:
if action_result.stderr_raw:
- stderr.write(str(action_result.stderr_raw, 'utf-8', errors='ignore'))
+ stderr.write(str(action_result.stderr_raw, "utf-8", errors="ignore"))
# Non-zero exit code means a normal error during the build:
# the remote execution system has worked correctly but the command failed.
@@ -83,19 +82,21 @@ class SandboxREAPI(Sandbox):
def _create_command(self, command, working_directory, environment):
# Creates a command proto
- environment_variables = [remote_execution_pb2.Command.
- EnvironmentVariable(name=k, value=v)
- for (k, v) in environment.items()]
+ environment_variables = [
+ remote_execution_pb2.Command.EnvironmentVariable(name=k, value=v) for (k, v) in environment.items()
+ ]
# Request the whole directory tree as output
output_directory = os.path.relpath(os.path.sep, start=working_directory)
- return remote_execution_pb2.Command(arguments=command,
- working_directory=working_directory,
- environment_variables=environment_variables,
- output_files=[],
- output_directories=[output_directory],
- platform=None)
+ return remote_execution_pb2.Command(
+ arguments=command,
+ working_directory=working_directory,
+ environment_variables=environment_variables,
+ output_files=[],
+ output_directories=[output_directory],
+ platform=None,
+ )
def _process_job_output(self, output_directories, output_files, *, failure):
# Reads the remote execution server response to an execution request.
@@ -124,7 +125,7 @@ class SandboxREAPI(Sandbox):
# Get digest of root directory from tree digest
tree = remote_execution_pb2.Tree()
- with open(cascache.objpath(tree_digest), 'rb') as f:
+ with open(cascache.objpath(tree_digest), "rb") as f:
tree.ParseFromString(f.read())
root_directory = tree.root.SerializeToString()
dir_digest = utils._message_digest(root_directory)
@@ -140,8 +141,7 @@ class SandboxREAPI(Sandbox):
return _SandboxREAPIBatch(self, main_group, flags, collect=collect)
def _execute_action(self, action):
- raise ImplError("Sandbox of type '{}' does not implement _execute_action()"
- .format(type(self).__name__))
+ raise ImplError("Sandbox of type '{}' does not implement _execute_action()".format(type(self).__name__))
# _SandboxREAPIBatch()
@@ -149,7 +149,6 @@ class SandboxREAPI(Sandbox):
# Command batching by shell script generation.
#
class _SandboxREAPIBatch(_SandboxBatch):
-
def __init__(self, sandbox, main_group, flags, *, collect=None):
super().__init__(sandbox, main_group, flags, collect=collect)
@@ -164,7 +163,7 @@ class _SandboxREAPIBatch(_SandboxBatch):
self.main_group.execute(self)
first = self.first_command
- if first and self.sandbox.run(['sh', '-c', '-e', self.script], self.flags, cwd=first.cwd, env=first.env) != 0:
+ if first and self.sandbox.run(["sh", "-c", "-e", self.script], self.flags, cwd=first.cwd, env=first.env) != 0:
raise SandboxCommandError("Command execution failed", collect=self.collect)
def execute_group(self, group):
@@ -195,7 +194,7 @@ class _SandboxREAPIBatch(_SandboxBatch):
self.env = command.env
# Actual command execution
- cmdline = ' '.join(shlex.quote(cmd) for cmd in command.command)
+ cmdline = " ".join(shlex.quote(cmd) for cmd in command.command)
self.script += "(set -ex; {})".format(cmdline)
# Error handling
diff --git a/src/buildstream/sandbox/_sandboxremote.py b/src/buildstream/sandbox/_sandboxremote.py
index fa7cc9f90..d4ffd64a1 100644
--- a/src/buildstream/sandbox/_sandboxremote.py
+++ b/src/buildstream/sandbox/_sandboxremote.py
@@ -39,7 +39,7 @@ from .._cas import CASRemote
from .._remote import RemoteSpec
-class RemoteExecutionSpec(namedtuple('RemoteExecutionSpec', 'exec_service storage_service action_service')):
+class RemoteExecutionSpec(namedtuple("RemoteExecutionSpec", "exec_service storage_service action_service")):
pass
@@ -49,59 +49,63 @@ class RemoteExecutionSpec(namedtuple('RemoteExecutionSpec', 'exec_service storag
# commands to a remote server and retrieves the results from it.
#
class SandboxRemote(SandboxREAPI):
-
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
- self._output_files_required = kwargs.get('output_files_required', True)
+ self._output_files_required = kwargs.get("output_files_required", True)
- config = kwargs['specs'] # This should be a RemoteExecutionSpec
+ config = kwargs["specs"] # This should be a RemoteExecutionSpec
if config is None:
return
# gRPC doesn't support fork without exec, which is used in the main process.
assert not utils._is_main_process()
- self.storage_url = config.storage_service['url']
- self.exec_url = config.exec_service['url']
+ self.storage_url = config.storage_service["url"]
+ self.exec_url = config.exec_service["url"]
exec_certs = {}
- for key in ['client-cert', 'client-key', 'server-cert']:
+ for key in ["client-cert", "client-key", "server-cert"]:
if key in config.exec_service:
- with open(config.exec_service[key], 'rb') as f:
+ with open(config.exec_service[key], "rb") as f:
exec_certs[key] = f.read()
self.exec_credentials = grpc.ssl_channel_credentials(
- root_certificates=exec_certs.get('server-cert'),
- private_key=exec_certs.get('client-key'),
- certificate_chain=exec_certs.get('client-cert'))
+ root_certificates=exec_certs.get("server-cert"),
+ private_key=exec_certs.get("client-key"),
+ certificate_chain=exec_certs.get("client-cert"),
+ )
action_certs = {}
- for key in ['client-cert', 'client-key', 'server-cert']:
+ for key in ["client-cert", "client-key", "server-cert"]:
if key in config.action_service:
- with open(config.action_service[key], 'rb') as f:
+ with open(config.action_service[key], "rb") as f:
action_certs[key] = f.read()
if config.action_service:
- self.action_url = config.action_service['url']
- self.action_instance = config.action_service.get('instance-name', None)
+ self.action_url = config.action_service["url"]
+ self.action_instance = config.action_service.get("instance-name", None)
self.action_credentials = grpc.ssl_channel_credentials(
- root_certificates=action_certs.get('server-cert'),
- private_key=action_certs.get('client-key'),
- certificate_chain=action_certs.get('client-cert'))
+ root_certificates=action_certs.get("server-cert"),
+ private_key=action_certs.get("client-key"),
+ certificate_chain=action_certs.get("client-cert"),
+ )
else:
self.action_url = None
self.action_instance = None
self.action_credentials = None
- self.exec_instance = config.exec_service.get('instance-name', None)
- self.storage_instance = config.storage_service.get('instance-name', None)
-
- self.storage_remote_spec = RemoteSpec(self.storage_url, push=True,
- server_cert=config.storage_service.get('server-cert'),
- client_key=config.storage_service.get('client-key'),
- client_cert=config.storage_service.get('client-cert'),
- instance_name=self.storage_instance)
+ self.exec_instance = config.exec_service.get("instance-name", None)
+ self.storage_instance = config.storage_service.get("instance-name", None)
+
+ self.storage_remote_spec = RemoteSpec(
+ self.storage_url,
+ push=True,
+ server_cert=config.storage_service.get("server-cert"),
+ client_key=config.storage_service.get("client-key"),
+ client_cert=config.storage_service.get("client-cert"),
+ instance_name=self.storage_instance,
+ )
self.operation_name = None
def info(self, msg):
@@ -109,47 +113,49 @@ class SandboxRemote(SandboxREAPI):
@staticmethod
def specs_from_config_node(config_node, basedir=None):
-
def require_node(config, keyname):
val = config.get_mapping(keyname, default=None)
if val is None:
provenance = remote_config.get_provenance()
- raise _yaml.LoadError("{}: '{}' was not present in the remote "
- "execution configuration (remote-execution). "
- .format(str(provenance), keyname),
- _yaml.LoadErrorReason.INVALID_DATA)
+ raise _yaml.LoadError(
+ "{}: '{}' was not present in the remote "
+ "execution configuration (remote-execution). ".format(str(provenance), keyname),
+ _yaml.LoadErrorReason.INVALID_DATA,
+ )
return val
- remote_config = config_node.get_mapping('remote-execution', default=None)
+ remote_config = config_node.get_mapping("remote-execution", default=None)
if remote_config is None:
return None
- service_keys = ['execution-service', 'storage-service', 'action-cache-service']
+ service_keys = ["execution-service", "storage-service", "action-cache-service"]
- remote_config.validate_keys(['url', *service_keys])
+ remote_config.validate_keys(["url", *service_keys])
- exec_config = require_node(remote_config, 'execution-service')
- storage_config = require_node(remote_config, 'storage-service')
- action_config = remote_config.get_mapping('action-cache-service', default={})
+ exec_config = require_node(remote_config, "execution-service")
+ storage_config = require_node(remote_config, "storage-service")
+ action_config = remote_config.get_mapping("action-cache-service", default={})
- tls_keys = ['client-key', 'client-cert', 'server-cert']
+ tls_keys = ["client-key", "client-cert", "server-cert"]
- exec_config.validate_keys(['url', 'instance-name', *tls_keys])
- storage_config.validate_keys(['url', 'instance-name', *tls_keys])
+ exec_config.validate_keys(["url", "instance-name", *tls_keys])
+ storage_config.validate_keys(["url", "instance-name", *tls_keys])
if action_config:
- action_config.validate_keys(['url', 'instance-name', *tls_keys])
+ action_config.validate_keys(["url", "instance-name", *tls_keys])
# Maintain some backwards compatibility with older configs, in which
# 'url' was the only valid key for remote-execution:
- if 'url' in remote_config:
- if 'execution-service' not in remote_config:
- exec_config = Node.from_dict({'url': remote_config['url']})
+ if "url" in remote_config:
+ if "execution-service" not in remote_config:
+ exec_config = Node.from_dict({"url": remote_config["url"]})
else:
- provenance = remote_config.get_node('url').get_provenance()
- raise _yaml.LoadError("{}: 'url' and 'execution-service' keys were found in the remote "
- "execution configuration (remote-execution). "
- "You can only specify one of these."
- .format(str(provenance)), _yaml.LoadErrorReason.INVALID_DATA)
+ provenance = remote_config.get_node("url").get_provenance()
+ raise _yaml.LoadError(
+ "{}: 'url' and 'execution-service' keys were found in the remote "
+ "execution configuration (remote-execution). "
+ "You can only specify one of these.".format(str(provenance)),
+ _yaml.LoadErrorReason.INVALID_DATA,
+ )
service_configs = [exec_config, storage_config, action_config]
@@ -161,12 +167,14 @@ class SandboxRemote(SandboxREAPI):
for config_key, config in zip(service_keys, service_configs):
# Either both or none of the TLS client key/cert pair must be specified:
- if ('client-key' in config) != ('client-cert' in config):
+ if ("client-key" in config) != ("client-cert" in config):
provenance = remote_config.get_node(config_key).get_provenance()
- raise _yaml.LoadError("{}: TLS client key/cert pair is incomplete. "
- "You must specify both 'client-key' and 'client-cert' "
- "for authenticated HTTPS connections."
- .format(str(provenance)), _yaml.LoadErrorReason.INVALID_DATA)
+ raise _yaml.LoadError(
+ "{}: TLS client key/cert pair is incomplete. "
+ "You must specify both 'client-key' and 'client-cert' "
+ "for authenticated HTTPS connections.".format(str(provenance)),
+ _yaml.LoadErrorReason.INVALID_DATA,
+ )
for tls_key in tls_keys:
if tls_key in config:
@@ -182,9 +190,9 @@ class SandboxRemote(SandboxREAPI):
# Try to create a communication channel to the BuildGrid server.
stub = remote_execution_pb2_grpc.ExecutionStub(channel)
- request = remote_execution_pb2.ExecuteRequest(instance_name=self.exec_instance,
- action_digest=action_digest,
- skip_cache_lookup=False)
+ request = remote_execution_pb2.ExecuteRequest(
+ instance_name=self.exec_instance, action_digest=action_digest, skip_cache_lookup=False
+ )
def __run_remote_command(stub, execute_request=None, running_operation=None):
try:
@@ -206,26 +214,30 @@ class SandboxRemote(SandboxREAPI):
except grpc.RpcError as e:
status_code = e.code()
if status_code == grpc.StatusCode.UNAVAILABLE:
- raise SandboxError("Failed contacting remote execution server at {}."
- .format(self.exec_url))
-
- if status_code in (grpc.StatusCode.INVALID_ARGUMENT,
- grpc.StatusCode.FAILED_PRECONDITION,
- grpc.StatusCode.RESOURCE_EXHAUSTED,
- grpc.StatusCode.INTERNAL,
- grpc.StatusCode.DEADLINE_EXCEEDED):
+ raise SandboxError("Failed contacting remote execution server at {}.".format(self.exec_url))
+
+ if status_code in (
+ grpc.StatusCode.INVALID_ARGUMENT,
+ grpc.StatusCode.FAILED_PRECONDITION,
+ grpc.StatusCode.RESOURCE_EXHAUSTED,
+ grpc.StatusCode.INTERNAL,
+ grpc.StatusCode.DEADLINE_EXCEEDED,
+ ):
raise SandboxError("{} ({}).".format(e.details(), status_code.name))
if running_operation and status_code == grpc.StatusCode.UNIMPLEMENTED:
- raise SandboxError("Failed trying to recover from connection loss: "
- "server does not support operation status polling recovery.")
+ raise SandboxError(
+ "Failed trying to recover from connection loss: "
+ "server does not support operation status polling recovery."
+ )
return last_operation
# Set up signal handler to trigger cancel_operation on SIGTERM
operation = None
- with self._get_context().messenger.timed_activity("Waiting for the remote build to complete"), \
- _signals.terminator(partial(self.cancel_operation, channel)):
+ with self._get_context().messenger.timed_activity(
+ "Waiting for the remote build to complete"
+ ), _signals.terminator(partial(self.cancel_operation, channel)):
operation = __run_remote_command(stub, execute_request=request)
if operation is None:
return None
@@ -242,18 +254,17 @@ class SandboxRemote(SandboxREAPI):
return
stub = operations_pb2_grpc.OperationsStub(channel)
- request = operations_pb2.CancelOperationRequest(
- name=str(self.operation_name))
+ request = operations_pb2.CancelOperationRequest(name=str(self.operation_name))
try:
stub.CancelOperation(request)
except grpc.RpcError as e:
- if (e.code() == grpc.StatusCode.UNIMPLEMENTED or
- e.code() == grpc.StatusCode.INVALID_ARGUMENT):
+ if e.code() == grpc.StatusCode.UNIMPLEMENTED or e.code() == grpc.StatusCode.INVALID_ARGUMENT:
pass
else:
- raise SandboxError("Failed trying to send CancelOperation request: "
- "{} ({})".format(e.details(), e.code().name))
+ raise SandboxError(
+ "Failed trying to send CancelOperation request: " "{} ({})".format(e.details(), e.code().name)
+ )
def _fetch_missing_blobs(self, vdir):
context = self._get_context()
@@ -282,8 +293,9 @@ class SandboxRemote(SandboxREAPI):
remote_missing_blobs = cascache.fetch_blobs(casremote, blobs_to_fetch)
if remote_missing_blobs:
- raise SandboxError("{} output files are missing on the CAS server"
- .format(len(remote_missing_blobs)))
+ raise SandboxError(
+ "{} output files are missing on the CAS server".format(len(remote_missing_blobs))
+ )
def _execute_action(self, action):
context = self._get_context()
@@ -301,8 +313,9 @@ class SandboxRemote(SandboxREAPI):
try:
casremote.init()
except grpc.RpcError as e:
- raise SandboxError("Failed to contact remote execution CAS endpoint at {}: {}"
- .format(self.storage_url, e)) from e
+ raise SandboxError(
+ "Failed to contact remote execution CAS endpoint at {}: {}".format(self.storage_url, e)
+ ) from e
# Determine blobs missing on remote
try:
@@ -333,15 +346,19 @@ class SandboxRemote(SandboxREAPI):
# Next, try to create a communication channel to the BuildGrid server.
url = urlparse(self.exec_url)
if not url.port:
- raise SandboxError("You must supply a protocol and port number in the execution-service url, "
- "for example: http://buildservice:50051.")
- if url.scheme == 'http':
- channel = grpc.insecure_channel('{}:{}'.format(url.hostname, url.port))
- elif url.scheme == 'https':
- channel = grpc.secure_channel('{}:{}'.format(url.hostname, url.port), self.exec_credentials)
+ raise SandboxError(
+ "You must supply a protocol and port number in the execution-service url, "
+ "for example: http://buildservice:50051."
+ )
+ if url.scheme == "http":
+ channel = grpc.insecure_channel("{}:{}".format(url.hostname, url.port))
+ elif url.scheme == "https":
+ channel = grpc.secure_channel("{}:{}".format(url.hostname, url.port), self.exec_credentials)
else:
- raise SandboxError("Remote execution currently only supports the 'http' protocol "
- "and '{}' was supplied.".format(url.scheme))
+ raise SandboxError(
+ "Remote execution currently only supports the 'http' protocol "
+ "and '{}' was supplied.".format(url.scheme)
+ )
# Now request to execute the action
with channel:
@@ -369,23 +386,25 @@ class SandboxRemote(SandboxREAPI):
return None
url = urlparse(self.action_url)
if not url.port:
- raise SandboxError("You must supply a protocol and port number in the action-cache-service url, "
- "for example: http://buildservice:50051.")
- if url.scheme == 'http':
- channel = grpc.insecure_channel('{}:{}'.format(url.hostname, url.port))
- elif url.scheme == 'https':
- channel = grpc.secure_channel('{}:{}'.format(url.hostname, url.port), self.action_credentials)
+ raise SandboxError(
+ "You must supply a protocol and port number in the action-cache-service url, "
+ "for example: http://buildservice:50051."
+ )
+ if url.scheme == "http":
+ channel = grpc.insecure_channel("{}:{}".format(url.hostname, url.port))
+ elif url.scheme == "https":
+ channel = grpc.secure_channel("{}:{}".format(url.hostname, url.port), self.action_credentials)
with channel:
- request = remote_execution_pb2.GetActionResultRequest(instance_name=self.action_instance,
- action_digest=action_digest)
+ request = remote_execution_pb2.GetActionResultRequest(
+ instance_name=self.action_instance, action_digest=action_digest
+ )
stub = remote_execution_pb2_grpc.ActionCacheStub(channel)
try:
result = stub.GetActionResult(request)
except grpc.RpcError as e:
if e.code() != grpc.StatusCode.NOT_FOUND:
- raise SandboxError("Failed to query action cache: {} ({})"
- .format(e.code(), e.details()))
+ raise SandboxError("Failed to query action cache: {} ({})".format(e.code(), e.details()))
return None
else:
self.info("Action result found in action cache")
@@ -397,7 +416,7 @@ class SandboxRemote(SandboxREAPI):
# Failure of remote execution, usually due to an error in BuildStream
raise SandboxError("No response returned from server")
- assert not operation.HasField('error') and operation.HasField('response')
+ assert not operation.HasField("error") and operation.HasField("response")
execution_response = remote_execution_pb2.ExecuteResponse()
# The response is expected to be an ExecutionResponse message
diff --git a/src/buildstream/sandbox/sandbox.py b/src/buildstream/sandbox/sandbox.py
index b4691fe3f..c88cbf977 100644
--- a/src/buildstream/sandbox/sandbox.py
+++ b/src/buildstream/sandbox/sandbox.py
@@ -47,10 +47,11 @@ if TYPE_CHECKING:
# pylint: disable=cyclic-import
from .._context import Context
from .._project import Project
+
# pylint: enable=cyclic-import
-class SandboxFlags():
+class SandboxFlags:
"""Flags indicating how the sandbox should be run.
"""
@@ -100,49 +101,45 @@ class SandboxCommandError(SandboxError):
collect (str): An optional directory containing partial install contents
reason (str): An optional reason string (defaults to 'command-failed')
"""
- def __init__(self, message, *, detail=None, collect=None, reason='command-failed'):
+
+ def __init__(self, message, *, detail=None, collect=None, reason="command-failed"):
super().__init__(message, detail=detail, reason=reason)
self.collect = collect
-class Sandbox():
+class Sandbox:
"""Sandbox()
Sandbox programming interface for :class:`.Element` plugins.
"""
# Minimal set of devices for the sandbox
- DEVICES = [
- '/dev/urandom',
- '/dev/random',
- '/dev/zero',
- '/dev/null'
- ]
- _dummy_reasons = [] # type: List[str]
-
- def __init__(self, context: 'Context', project: 'Project', directory: str, **kwargs):
+ DEVICES = ["/dev/urandom", "/dev/random", "/dev/zero", "/dev/null"]
+ _dummy_reasons = [] # type: List[str]
+
+ def __init__(self, context: "Context", project: "Project", directory: str, **kwargs):
self.__context = context
self.__project = project
- self.__directories = [] # type: List[Dict[str, Union[int, str]]]
- self.__cwd = None # type: Optional[str]
- self.__env = None # type: Optional[Dict[str, str]]
- self.__mount_sources = {} # type: Dict[str, str]
- self.__allow_real_directory = kwargs['allow_real_directory']
+ self.__directories = [] # type: List[Dict[str, Union[int, str]]]
+ self.__cwd = None # type: Optional[str]
+ self.__env = None # type: Optional[Dict[str, str]]
+ self.__mount_sources = {} # type: Dict[str, str]
+ self.__allow_real_directory = kwargs["allow_real_directory"]
self.__allow_run = True
# Plugin element full name for logging
- plugin = kwargs.get('plugin', None)
+ plugin = kwargs.get("plugin", None)
if plugin:
self.__element_name = plugin._get_full_name()
else:
self.__element_name = None
# Configuration from kwargs common to all subclasses
- self.__config = kwargs['config']
- self.__stdout = kwargs['stdout']
- self.__stderr = kwargs['stderr']
- self.__bare_directory = kwargs['bare_directory']
+ self.__config = kwargs["config"]
+ self.__stdout = kwargs["stdout"]
+ self.__stderr = kwargs["stderr"]
+ self.__bare_directory = kwargs["bare_directory"]
# Setup the directories. Root and output_directory should be
# available to subclasses, hence being single-underscore. The
@@ -153,15 +150,15 @@ class Sandbox():
self.__scratch = None
os.makedirs(self._root, exist_ok=True)
else:
- self._root = os.path.join(directory, 'root')
- self.__scratch = os.path.join(directory, 'scratch')
+ self._root = os.path.join(directory, "root")
+ self.__scratch = os.path.join(directory, "scratch")
for directory_ in [self._root, self.__scratch]:
os.makedirs(directory_, exist_ok=True)
- self._output_directory = None # type: Optional[str]
+ self._output_directory = None # type: Optional[str]
self._build_directory = None
self._build_directory_always = None
- self._vdir = None # type: Optional[Directory]
+ self._vdir = None # type: Optional[Directory]
self._usebuildtree = False
# This is set if anyone requests access to the underlying
@@ -255,18 +252,17 @@ class Sandbox():
Any marked directories will be read-write in the sandboxed
environment, only the root directory is allowed to be readonly.
"""
- self.__directories.append({
- 'directory': directory,
- 'artifact': artifact
- })
-
- def run(self,
- command: List[str],
- flags: int,
- *,
- cwd: Optional[str] = None,
- env: Optional[Dict[str, str]] = None,
- label: str = None) -> Optional[int]:
+ self.__directories.append({"directory": directory, "artifact": artifact})
+
+ def run(
+ self,
+ command: List[str],
+ flags: int,
+ *,
+ cwd: Optional[str] = None,
+ env: Optional[Dict[str, str]] = None,
+ label: str = None
+ ) -> Optional[int]:
"""Run a command in the sandbox.
If this is called outside a batch context, the command is immediately
@@ -314,8 +310,7 @@ class Sandbox():
command = [command]
if self.__batch:
- assert flags == self.__batch.flags, \
- "Inconsistent sandbox flags in single command batch"
+ assert flags == self.__batch.flags, "Inconsistent sandbox flags in single command batch"
batch_command = _SandboxBatchCommand(command, cwd=cwd, env=env, label=label)
@@ -352,8 +347,7 @@ class Sandbox():
if self.__batch:
# Nested batch
- assert flags == self.__batch.flags, \
- "Inconsistent sandbox flags in single command batch"
+ assert flags == self.__batch.flags, "Inconsistent sandbox flags in single command batch"
parent_group = self.__batch.current_group
parent_group.append(group)
@@ -394,8 +388,7 @@ class Sandbox():
# (int): The program exit code.
#
def _run(self, command, flags, *, cwd, env):
- raise ImplError("Sandbox of type '{}' does not implement _run()"
- .format(type(self).__name__))
+ raise ImplError("Sandbox of type '{}' does not implement _run()".format(type(self).__name__))
# _create_batch()
#
@@ -425,7 +418,7 @@ class Sandbox():
if not self.__allow_real_directory and not self.__allow_run:
return True
- return 'BST_CAS_DIRECTORIES' in os.environ
+ return "BST_CAS_DIRECTORIES" in os.environ
# _fetch_missing_blobs()
#
@@ -513,7 +506,7 @@ class Sandbox():
# what directory it is in makes it unnecessary to call the faulty
# getcwd.
env = dict(env)
- env['PWD'] = cwd
+ env["PWD"] = cwd
return env
@@ -528,7 +521,7 @@ class Sandbox():
# Returns:
# (str): The sandbox work directory
def _get_work_directory(self, *, cwd=None):
- return cwd or self.__cwd or '/'
+ return cwd or self.__cwd or "/"
# _get_scratch_directory()
#
@@ -584,7 +577,7 @@ class Sandbox():
if len(command_as_parts) > 1:
return False
- for path in env.get('PATH').split(':'):
+ for path in env.get("PATH").split(":"):
path_as_parts = path.lstrip(os.sep).split(os.sep)
if vroot._exists(*path_as_parts, command, follow_symlinks=True):
return True
@@ -649,20 +642,14 @@ class Sandbox():
# message (str): A message to issue
# details (str): optional, more detatils
def _issue_warning(self, message, detail=None):
- self.__context.messenger.message(
- Message(MessageType.WARN,
- message,
- detail=detail
- )
- )
+ self.__context.messenger.message(Message(MessageType.WARN, message, detail=detail))
# _SandboxBatch()
#
# A batch of sandbox commands.
#
-class _SandboxBatch():
-
+class _SandboxBatch:
def __init__(self, sandbox, main_group, flags, *, collect=None):
self.sandbox = sandbox
self.main_group = main_group
@@ -686,16 +673,21 @@ class _SandboxBatch():
def execute_command(self, command):
if command.label:
context = self.sandbox._get_context()
- message = Message(MessageType.STATUS, 'Running command', detail=command.label,
- element_name=self.sandbox._get_element_name())
+ message = Message(
+ MessageType.STATUS,
+ "Running command",
+ detail=command.label,
+ element_name=self.sandbox._get_element_name(),
+ )
context.messenger.message(message)
exitcode = self.sandbox._run(command.command, self.flags, cwd=command.cwd, env=command.env)
if exitcode != 0:
- cmdline = ' '.join(shlex.quote(cmd) for cmd in command.command)
+ cmdline = " ".join(shlex.quote(cmd) for cmd in command.command)
label = command.label or cmdline
- raise SandboxCommandError("Command failed with exitcode {}".format(exitcode),
- detail=label, collect=self.collect)
+ raise SandboxCommandError(
+ "Command failed with exitcode {}".format(exitcode), detail=label, collect=self.collect
+ )
def execute_call(self, call):
call.callback()
@@ -705,8 +697,7 @@ class _SandboxBatch():
#
# An item in a command batch.
#
-class _SandboxBatchItem():
-
+class _SandboxBatchItem:
def __init__(self, *, label=None):
self.label = label
@@ -716,7 +707,6 @@ class _SandboxBatchItem():
# A command item in a command batch.
#
class _SandboxBatchCommand(_SandboxBatchItem):
-
def __init__(self, command, *, cwd, env, label=None):
super().__init__(label=label)
@@ -733,7 +723,6 @@ class _SandboxBatchCommand(_SandboxBatchItem):
# A group in a command batch.
#
class _SandboxBatchGroup(_SandboxBatchItem):
-
def __init__(self, *, label=None):
super().__init__(label=label)
@@ -755,7 +744,6 @@ class _SandboxBatchGroup(_SandboxBatchItem):
# A call item in a command batch.
#
class _SandboxBatchCall(_SandboxBatchItem):
-
def __init__(self, callback):
super().__init__()
diff --git a/src/buildstream/scriptelement.py b/src/buildstream/scriptelement.py
index e78049b4a..f8deff28e 100644
--- a/src/buildstream/scriptelement.py
+++ b/src/buildstream/scriptelement.py
@@ -48,8 +48,8 @@ class ScriptElement(Element):
__install_root = "/"
__cwd = "/"
__root_read_only = False
- __commands = None # type: OrderedDict[str, List[str]]
- __layout = [] # type: List[Dict[str, Optional[str]]]
+ __commands = None # type: OrderedDict[str, List[str]]
+ __layout = [] # type: List[Dict[str, Optional[str]]]
# The compose element's output is its dependencies, so
# we must rebuild if the dependencies change even when
@@ -149,8 +149,7 @@ class ScriptElement(Element):
#
if not self.__layout:
self.__layout = []
- self.__layout.append({"element": element,
- "destination": destination})
+ self.__layout.append({"element": element, "destination": destination})
def add_commands(self, group_name: str, command_list: List[str]) -> None:
"""Adds a list of commands under the group-name.
@@ -183,11 +182,11 @@ class ScriptElement(Element):
def get_unique_key(self):
return {
- 'commands': self.__commands,
- 'cwd': self.__cwd,
- 'install-root': self.__install_root,
- 'layout': self.__layout,
- 'root-read-only': self.__root_read_only
+ "commands": self.__commands,
+ "cwd": self.__cwd,
+ "install-root": self.__install_root,
+ "layout": self.__layout,
+ "root-read-only": self.__root_read_only,
}
def configure_sandbox(self, sandbox):
@@ -206,14 +205,14 @@ class ScriptElement(Element):
# Mark the artifact directories in the layout
for item in self.__layout:
- destination = item['destination']
+ destination = item["destination"]
was_artifact = directories.get(destination, False)
- directories[destination] = item['element'] or was_artifact
+ directories[destination] = item["element"] or was_artifact
for directory, artifact in directories.items():
# Root does not need to be marked as it is always mounted
# with artifact (unless explicitly marked non-artifact)
- if directory != '/':
+ if directory != "/":
sandbox.mark_directory(directory, artifact=artifact)
def stage(self, sandbox):
@@ -222,8 +221,7 @@ class ScriptElement(Element):
if not self.__layout:
# if no layout set, stage all dependencies into /
for build_dep in self.dependencies(Scope.BUILD, recurse=False):
- with self.timed_activity("Staging {} at /"
- .format(build_dep.name), silent_nested=True):
+ with self.timed_activity("Staging {} at /".format(build_dep.name), silent_nested=True):
build_dep.stage_dependency_artifacts(sandbox, Scope.RUN, path="/")
with sandbox.batch(SandboxFlags.NONE):
@@ -236,35 +234,33 @@ class ScriptElement(Element):
for item in self.__layout:
# Skip layout members which dont stage an element
- if not item['element']:
+ if not item["element"]:
continue
- element = self.search(Scope.BUILD, item['element'])
- if item['destination'] == '/':
- with self.timed_activity("Staging {} at /".format(element.name),
- silent_nested=True):
+ element = self.search(Scope.BUILD, item["element"])
+ if item["destination"] == "/":
+ with self.timed_activity("Staging {} at /".format(element.name), silent_nested=True):
element.stage_dependency_artifacts(sandbox, Scope.RUN)
else:
- with self.timed_activity("Staging {} at {}"
- .format(element.name, item['destination']),
- silent_nested=True):
+ with self.timed_activity(
+ "Staging {} at {}".format(element.name, item["destination"]), silent_nested=True
+ ):
virtual_dstdir = sandbox.get_virtual_directory()
- virtual_dstdir.descend(*item['destination'].lstrip(os.sep).split(os.sep), create=True)
- element.stage_dependency_artifacts(sandbox, Scope.RUN, path=item['destination'])
+ virtual_dstdir.descend(*item["destination"].lstrip(os.sep).split(os.sep), create=True)
+ element.stage_dependency_artifacts(sandbox, Scope.RUN, path=item["destination"])
with sandbox.batch(SandboxFlags.NONE):
for item in self.__layout:
# Skip layout members which dont stage an element
- if not item['element']:
+ if not item["element"]:
continue
- element = self.search(Scope.BUILD, item['element'])
+ element = self.search(Scope.BUILD, item["element"])
# Integration commands can only be run for elements staged to /
- if item['destination'] == '/':
- with self.timed_activity("Integrating {}".format(element.name),
- silent_nested=True):
+ if item["destination"] == "/":
+ with self.timed_activity("Integrating {}".format(element.name), silent_nested=True):
for dep in element.dependencies(Scope.RUN):
dep.integrate(sandbox)
@@ -283,9 +279,7 @@ class ScriptElement(Element):
for cmd in commands:
# Note the -e switch to 'sh' means to exit with an error
# if any untested command fails.
- sandbox.run(['sh', '-c', '-e', cmd + '\n'],
- flags,
- label=cmd)
+ sandbox.run(["sh", "-c", "-e", cmd + "\n"], flags, label=cmd)
# Return where the result can be collected from
return self.__install_root
@@ -297,18 +291,18 @@ class ScriptElement(Element):
def __validate_layout(self):
if self.__layout:
# Cannot proceeed if layout is used, but none are for "/"
- root_defined = any([(entry['destination'] == '/') for entry in self.__layout])
+ root_defined = any([(entry["destination"] == "/") for entry in self.__layout])
if not root_defined:
- raise ElementError("{}: Using layout, but none are staged as '/'"
- .format(self))
+ raise ElementError("{}: Using layout, but none are staged as '/'".format(self))
# Cannot proceed if layout specifies an element that isn't part
# of the dependencies.
for item in self.__layout:
- if item['element']:
- if not self.search(Scope.BUILD, item['element']):
- raise ElementError("{}: '{}' in layout not found in dependencies"
- .format(self, item['element']))
+ if item["element"]:
+ if not self.search(Scope.BUILD, item["element"]):
+ raise ElementError(
+ "{}: '{}' in layout not found in dependencies".format(self, item["element"])
+ )
def setup():
diff --git a/src/buildstream/source.py b/src/buildstream/source.py
index 05a1ae464..2e7460439 100644
--- a/src/buildstream/source.py
+++ b/src/buildstream/source.py
@@ -184,6 +184,7 @@ if TYPE_CHECKING:
# pylint: disable=cyclic-import
from ._context import Context
from ._project import Project
+
# pylint: enable=cyclic-import
@@ -197,16 +198,14 @@ class SourceError(BstError):
reason: An optional machine readable reason string, used for test cases
temporary: An indicator to whether the error may occur if the operation was run again. (*Since: 1.2*)
"""
- def __init__(self,
- message: str,
- *,
- detail: Optional[str] = None,
- reason: Optional[str] = None,
- temporary: bool = False):
+
+ def __init__(
+ self, message: str, *, detail: Optional[str] = None, reason: Optional[str] = None, temporary: bool = False
+ ):
super().__init__(message, detail=detail, domain=ErrorDomain.SOURCE, reason=reason, temporary=temporary)
-class SourceFetcher():
+class SourceFetcher:
"""SourceFetcher()
This interface exists so that a source that downloads from multiple
@@ -222,6 +221,7 @@ class SourceFetcher():
for every URL found in the configuration data at
:func:`Plugin.configure() <buildstream.plugin.Plugin.configure>` time.
"""
+
def __init__(self):
self.__alias = None
@@ -275,8 +275,9 @@ class Source(Plugin):
All Sources derive from this class, this interface defines how
the core will be interacting with Sources.
"""
+
# The defaults from the project
- __defaults = None # type: Optional[Dict[str, Any]]
+ __defaults = None # type: Optional[Dict[str, Any]]
BST_REQUIRES_PREVIOUS_SOURCES_TRACK = False
"""Whether access to previous sources is required during track
@@ -331,32 +332,40 @@ class Source(Plugin):
*Since: 1.91.2*
"""
- def __init__(self,
- context: 'Context',
- project: 'Project',
- meta: MetaSource,
- *,
- alias_override: Optional[Tuple[str, str]] = None,
- unique_id: Optional[int] = None):
+ def __init__(
+ self,
+ context: "Context",
+ project: "Project",
+ meta: MetaSource,
+ *,
+ alias_override: Optional[Tuple[str, str]] = None,
+ unique_id: Optional[int] = None
+ ):
provenance = meta.config.get_provenance()
# Set element_name member before parent init, as needed for debug messaging
- self.__element_name = meta.element_name # The name of the element owning this source
- super().__init__("{}-{}".format(meta.element_name, meta.element_index),
- context, project, provenance, "source", unique_id=unique_id)
+ self.__element_name = meta.element_name # The name of the element owning this source
+ super().__init__(
+ "{}-{}".format(meta.element_name, meta.element_index),
+ context,
+ project,
+ provenance,
+ "source",
+ unique_id=unique_id,
+ )
- self.__element_index = meta.element_index # The index of the source in the owning element's source list
- self.__element_kind = meta.element_kind # The kind of the element owning this source
- self.__directory = meta.directory # Staging relative directory
- self.__consistency = Consistency.INCONSISTENT # Cached consistency state
- self.__meta_kind = meta.kind # The kind of this source, required for unpickling
+ self.__element_index = meta.element_index # The index of the source in the owning element's source list
+ self.__element_kind = meta.element_kind # The kind of the element owning this source
+ self.__directory = meta.directory # Staging relative directory
+ self.__consistency = Consistency.INCONSISTENT # Cached consistency state
+ self.__meta_kind = meta.kind # The kind of this source, required for unpickling
- self.__key = None # Cache key for source
+ self.__key = None # Cache key for source
# The alias_override is only set on a re-instantiated Source
- self.__alias_override = alias_override # Tuple of alias and its override to use instead
- self.__expected_alias = None # The primary alias
+ self.__alias_override = alias_override # Tuple of alias and its override to use instead
+ self.__expected_alias = None # The primary alias
# Set of marked download URLs
- self.__marked_urls = set() # type: Set[str]
+ self.__marked_urls = set() # type: Set[str]
# Collect the composited element configuration and
# ask the element to configure itself.
@@ -365,12 +374,12 @@ class Source(Plugin):
self.__first_pass = meta.first_pass
# cached values for commonly access values on the source
- self.__mirror_directory = None # type: Optional[str]
+ self.__mirror_directory = None # type: Optional[str]
self._configure(self.__config)
self.__digest = None
- COMMON_CONFIG_KEYS = ['kind', 'directory']
+ COMMON_CONFIG_KEYS = ["kind", "directory"]
"""Common source config keys
Source config keys that must not be accessed in configure(), and
@@ -611,8 +620,8 @@ class Source(Plugin):
# specific alias, so that sources that fetch from multiple
# URLs and use different aliases default to only overriding
# one alias, rather than getting confused.
- override_alias = self.__alias_override[0] # type: ignore
- override_url = self.__alias_override[1] # type: ignore
+ override_alias = self.__alias_override[0] # type: ignore
+ override_url = self.__alias_override[1] # type: ignore
if url_alias == override_alias:
url = override_url + url_body
return url
@@ -642,9 +651,9 @@ class Source(Plugin):
if primary:
expected_alias = _extract_alias(url)
- assert (self.__expected_alias is None or
- self.__expected_alias == expected_alias), \
- "Primary URL marked twice with different URLs"
+ assert (
+ self.__expected_alias is None or self.__expected_alias == expected_alias
+ ), "Primary URL marked twice with different URLs"
self.__expected_alias = expected_alias
@@ -664,8 +673,9 @@ class Source(Plugin):
# the case for git submodules which might be automatically
# discovered.
#
- assert (url in self.__marked_urls or not _extract_alias(url)), \
- "URL was not seen at configure time: {}".format(url)
+ assert url in self.__marked_urls or not _extract_alias(
+ url
+ ), "URL was not seen at configure time: {}".format(url)
def get_project_directory(self) -> str:
"""Fetch the project base directory
@@ -790,8 +800,7 @@ class Source(Plugin):
if self.BST_KEY_REQUIRES_STAGE:
# _get_unique_key should be called before _stage
assert self.__digest is not None
- cas_dir = CasBasedDirectory(self._get_context().get_cascache(),
- digest=self.__digest)
+ cas_dir = CasBasedDirectory(self._get_context().get_cascache(), digest=self.__digest)
directory.import_files(cas_dir)
else:
self.stage(directory)
@@ -811,11 +820,11 @@ class Source(Plugin):
#
def _get_unique_key(self):
key = {}
- key['directory'] = self.__directory
+ key["directory"] = self.__directory
if self.BST_KEY_REQUIRES_STAGE:
- key['unique'] = self._stage_into_cas()
+ key["unique"] = self._stage_into_cas()
else:
- key['unique'] = self.get_unique_key() # pylint: disable=assignment-from-no-return
+ key["unique"] = self.get_unique_key() # pylint: disable=assignment-from-no-return
return key
# _project_refs():
@@ -828,7 +837,7 @@ class Source(Plugin):
#
def _project_refs(self, project):
element_kind = self.__element_kind
- if element_kind == 'junction':
+ if element_kind == "junction":
return project.junction_refs
return project.refs
@@ -863,9 +872,10 @@ class Source(Plugin):
try:
self.load_ref(ref_node)
except ImplError as e:
- raise SourceError("{}: Storing refs in project.refs is not supported by '{}' sources"
- .format(self, self.get_kind()),
- reason="unsupported-load-ref") from e
+ raise SourceError(
+ "{}: Storing refs in project.refs is not supported by '{}' sources".format(self, self.get_kind()),
+ reason="unsupported-load-ref",
+ ) from e
# If the main project overrides the ref, use the override
if project is not toplevel and toplevel.ref_storage == ProjectRefStorage.PROJECT_REFS:
@@ -938,12 +948,12 @@ class Source(Plugin):
elif provenance._project is None:
assert provenance._filename == ""
assert provenance._shortname == ""
- raise SourceError("{}: Error saving source reference to synthetic node."
- .format(self))
+ raise SourceError("{}: Error saving source reference to synthetic node.".format(self))
else:
- raise SourceError("{}: Cannot track source in a fragment from a junction"
- .format(provenance._shortname),
- reason="tracking-junction-fragment")
+ raise SourceError(
+ "{}: Cannot track source in a fragment from a junction".format(provenance._shortname),
+ reason="tracking-junction-fragment",
+ )
#
# Step 2 - Set the ref in memory, and determine changed state
@@ -968,13 +978,13 @@ class Source(Plugin):
actions = {}
for k, v in clean.items():
if k not in to_modify:
- actions[k] = 'del'
+ actions[k] = "del"
else:
if v != to_modify[k]:
- actions[k] = 'mod'
+ actions[k] = "mod"
for k in to_modify.keys():
if k not in clean:
- actions[k] = 'add'
+ actions[k] = "add"
def walk_container(container, path):
# For each step along path, synthesise if we need to.
@@ -1002,20 +1012,19 @@ class Source(Plugin):
def process_value(action, container, path, key, new_value):
container = walk_container(container, path)
- if action == 'del':
+ if action == "del":
del container[key]
- elif action == 'mod':
+ elif action == "mod":
container[key] = new_value
- elif action == 'add':
+ elif action == "add":
container[key] = new_value
else:
- assert False, \
- "BUG: Unknown action: {}".format(action)
+ assert False, "BUG: Unknown action: {}".format(action)
roundtrip_cache = {}
for key, action in actions.items():
# Obtain the top level node and its file
- if action == 'add':
+ if action == "add":
provenance = node.get_provenance()
else:
provenance = node.get_node(key).get_provenance()
@@ -1023,7 +1032,7 @@ class Source(Plugin):
toplevel_node = provenance._toplevel
# Get the path to whatever changed
- if action == 'add':
+ if action == "add":
path = toplevel_node._find(node)
else:
full_path = toplevel_node._find(node.get_node(key))
@@ -1033,8 +1042,7 @@ class Source(Plugin):
roundtrip_file = roundtrip_cache.get(provenance._filename)
if not roundtrip_file:
roundtrip_file = roundtrip_cache[provenance._filename] = _yaml.roundtrip_load(
- provenance._filename,
- allow_missing=True
+ provenance._filename, allow_missing=True
)
# Get the value of the round trip file that we need to change
@@ -1048,9 +1056,9 @@ class Source(Plugin):
try:
_yaml.roundtrip_dump(data, filename)
except OSError as e:
- raise SourceError("{}: Error saving source reference to '{}': {}"
- .format(self, filename, e),
- reason="save-ref-error") from e
+ raise SourceError(
+ "{}: Error saving source reference to '{}': {}".format(self, filename, e), reason="save-ref-error"
+ ) from e
return True
@@ -1059,7 +1067,7 @@ class Source(Plugin):
# Args:
# previous_sources (list): List of Sources listed prior to this source
#
- def _track(self, previous_sources: List['Source']) -> SourceRef:
+ def _track(self, previous_sources: List["Source"]) -> SourceRef:
if self.BST_KEY_REQUIRES_STAGE:
# ensure that these sources have a key after tracking
self._get_unique_key()
@@ -1067,8 +1075,7 @@ class Source(Plugin):
if self.BST_REQUIRES_PREVIOUS_SOURCES_TRACK:
self.__ensure_previous_sources(previous_sources)
- with self.__stage_previous_sources(previous_sources) \
- as staging_directory:
+ with self.__stage_previous_sources(previous_sources) as staging_directory:
new_ref = self.__do_track(previous_sources_dir=self.__ensure_directory(staging_directory))
else:
new_ref = self.__do_track()
@@ -1135,9 +1142,7 @@ class Source(Plugin):
# Gives a ref path that points to where sources are kept in the CAS
def _get_source_name(self):
# @ is used to prevent conflicts with project names
- return "{}/{}".format(
- self.get_kind(),
- self._key)
+ return "{}/{}".format(self.get_kind(), self._key)
def _get_brief_display_key(self):
context = self._get_context()
@@ -1210,9 +1215,7 @@ class Source(Plugin):
meta.first_pass = self.__first_pass
- clone = source_kind(context, project, meta,
- alias_override=(alias, uri),
- unique_id=self._unique_id)
+ clone = source_kind(context, project, meta, alias_override=(alias, uri), unique_id=self._unique_id)
# Do the necessary post instantiation routines here
#
@@ -1352,20 +1355,18 @@ class Source(Plugin):
try:
os.makedirs(directory, exist_ok=True)
except OSError as e:
- raise SourceError("Failed to create staging directory: {}"
- .format(e),
- reason="ensure-stage-dir-fail") from e
+ raise SourceError(
+ "Failed to create staging directory: {}".format(e), reason="ensure-stage-dir-fail"
+ ) from e
else:
if self.__directory is not None:
try:
- directory = directory.descend(
- *self.__directory.lstrip(os.sep).split(os.sep),
- create=True)
+ directory = directory.descend(*self.__directory.lstrip(os.sep).split(os.sep), create=True)
except VirtualDirectoryError as e:
- raise SourceError("Failed to descend into staging directory: {}"
- .format(e),
- reason="ensure-stage-dir-fail") from e
+ raise SourceError(
+ "Failed to descend into staging directory: {}".format(e), reason="ensure-stage-dir-fail"
+ ) from e
return directory
@@ -1383,7 +1384,7 @@ class Source(Plugin):
#
@classmethod
def __extract_config(cls, meta):
- config = cls.__defaults.get_mapping('config', default={})
+ config = cls.__defaults.get_mapping("config", default={})
config = config.clone()
meta.config._composite(config)
diff --git a/src/buildstream/storage/_casbaseddirectory.py b/src/buildstream/storage/_casbaseddirectory.py
index 3786f25b6..df28dc591 100644
--- a/src/buildstream/storage/_casbaseddirectory.py
+++ b/src/buildstream/storage/_casbaseddirectory.py
@@ -38,10 +38,20 @@ from ._filebaseddirectory import FileBasedDirectory
from ..utils import FileListResult, BST_ARBITRARY_TIMESTAMP
-class IndexEntry():
+class IndexEntry:
""" Directory entry used in CasBasedDirectory.index """
- def __init__(self, name, entrytype, *, digest=None, target=None, is_executable=False,
- buildstream_object=None, modified=False):
+
+ def __init__(
+ self,
+ name,
+ entrytype,
+ *,
+ digest=None,
+ target=None,
+ is_executable=False,
+ buildstream_object=None,
+ modified=False
+ ):
self.name = name
self.type = entrytype
self.digest = digest
@@ -52,8 +62,9 @@ class IndexEntry():
def get_directory(self, parent):
if not self.buildstream_object:
- self.buildstream_object = CasBasedDirectory(parent.cas_cache, digest=self.digest,
- parent=parent, filename=self.name)
+ self.buildstream_object = CasBasedDirectory(
+ parent.cas_cache, digest=self.digest, parent=parent, filename=self.name
+ )
self.digest = None
return self.buildstream_object
@@ -69,6 +80,7 @@ class IndexEntry():
# which is meant to be unimplemented.
# pylint: disable=super-init-not-called
+
class CasBasedDirectory(Directory):
"""
CAS-based directories can have two names; one is a 'common name' which has no effect
@@ -100,21 +112,19 @@ class CasBasedDirectory(Directory):
def _populate_index(self, digest):
try:
pb2_directory = remote_execution_pb2.Directory()
- with open(self.cas_cache.objpath(digest), 'rb') as f:
+ with open(self.cas_cache.objpath(digest), "rb") as f:
pb2_directory.ParseFromString(f.read())
except FileNotFoundError as e:
raise VirtualDirectoryError("Directory not found in local cache: {}".format(e)) from e
for entry in pb2_directory.directories:
- self.index[entry.name] = IndexEntry(entry.name, _FileType.DIRECTORY,
- digest=entry.digest)
+ self.index[entry.name] = IndexEntry(entry.name, _FileType.DIRECTORY, digest=entry.digest)
for entry in pb2_directory.files:
- self.index[entry.name] = IndexEntry(entry.name, _FileType.REGULAR_FILE,
- digest=entry.digest,
- is_executable=entry.is_executable)
+ self.index[entry.name] = IndexEntry(
+ entry.name, _FileType.REGULAR_FILE, digest=entry.digest, is_executable=entry.is_executable
+ )
for entry in pb2_directory.symlinks:
- self.index[entry.name] = IndexEntry(entry.name, _FileType.SYMLINK,
- target=entry.target)
+ self.index[entry.name] = IndexEntry(entry.name, _FileType.SYMLINK, target=entry.target)
def _find_self_in_parent(self):
assert self.parent is not None
@@ -136,8 +146,7 @@ class CasBasedDirectory(Directory):
return newdir
def _add_file(self, basename, filename, modified=False, can_link=False):
- entry = IndexEntry(filename, _FileType.REGULAR_FILE,
- modified=modified or filename in self.index)
+ entry = IndexEntry(filename, _FileType.REGULAR_FILE, modified=modified or filename in self.index)
path = os.path.join(basename, filename)
entry.digest = self.cas_cache.add_object(path=path, link_directly=can_link)
entry.is_executable = os.access(path, os.X_OK)
@@ -206,14 +215,13 @@ class CasBasedDirectory(Directory):
current_dir = current_dir.descend(*newpaths, follow_symlinks=True)
else:
error = "Cannot descend into {}, which is a '{}' in the directory {}"
- raise VirtualDirectoryError(error.format(path,
- current_dir.index[path].type,
- current_dir),
- reason="not-a-directory")
+ raise VirtualDirectoryError(
+ error.format(path, current_dir.index[path].type, current_dir), reason="not-a-directory"
+ )
else:
- if path == '.':
+ if path == ".":
continue
- elif path == '..':
+ elif path == "..":
if current_dir.parent is not None:
current_dir = current_dir.parent
# In POSIX /.. == / so just stay at the root dir
@@ -222,8 +230,7 @@ class CasBasedDirectory(Directory):
current_dir = current_dir._add_directory(path)
else:
error = "'{}' not found in {}"
- raise VirtualDirectoryError(error.format(path, str(current_dir)),
- reason="directory-not-found")
+ raise VirtualDirectoryError(error.format(path, str(current_dir)), reason="directory-not-found")
return current_dir
@@ -299,12 +306,13 @@ class CasBasedDirectory(Directory):
dest_subdir = self.descend(name, create=create_subdir)
except VirtualDirectoryError:
filetype = self.index[name].type
- raise VirtualDirectoryError('Destination is a {}, not a directory: /{}'
- .format(filetype, relative_pathname))
+ raise VirtualDirectoryError(
+ "Destination is a {}, not a directory: /{}".format(filetype, relative_pathname)
+ )
- dest_subdir._partial_import_cas_into_cas(src_subdir, filter_callback,
- path_prefix=relative_pathname,
- origin=origin, result=result)
+ dest_subdir._partial_import_cas_into_cas(
+ src_subdir, filter_callback, path_prefix=relative_pathname, origin=origin, result=result
+ )
if filter_callback and not filter_callback(relative_pathname):
if is_dir and create_subdir and dest_subdir.is_empty():
@@ -317,20 +325,22 @@ class CasBasedDirectory(Directory):
if not is_dir:
if self._check_replacement(name, relative_pathname, result):
if entry.type == _FileType.REGULAR_FILE:
- self.index[name] = IndexEntry(name, _FileType.REGULAR_FILE,
- digest=entry.digest,
- is_executable=entry.is_executable,
- modified=True)
+ self.index[name] = IndexEntry(
+ name,
+ _FileType.REGULAR_FILE,
+ digest=entry.digest,
+ is_executable=entry.is_executable,
+ modified=True,
+ )
self.__invalidate_digest()
else:
assert entry.type == _FileType.SYMLINK
self._add_new_link_direct(name=name, target=entry.target)
result.files_written.append(relative_pathname)
- def import_files(self, external_pathspec, *,
- filter_callback=None,
- report_written=True, update_mtime=False,
- can_link=False):
+ def import_files(
+ self, external_pathspec, *, filter_callback=None, report_written=True, update_mtime=False, can_link=False
+ ):
""" See superclass Directory for arguments """
result = FileListResult()
@@ -358,13 +368,12 @@ class CasBasedDirectory(Directory):
def import_single_file(self, external_pathspec):
result = FileListResult()
- if self._check_replacement(os.path.basename(external_pathspec),
- os.path.dirname(external_pathspec),
- result):
- self._add_file(os.path.dirname(external_pathspec),
- os.path.basename(external_pathspec),
- modified=os.path.basename(external_pathspec)
- in result.overwritten)
+ if self._check_replacement(os.path.basename(external_pathspec), os.path.dirname(external_pathspec), result):
+ self._add_file(
+ os.path.dirname(external_pathspec),
+ os.path.basename(external_pathspec),
+ modified=os.path.basename(external_pathspec) in result.overwritten,
+ )
result.files_written.append(external_pathspec)
return result
@@ -516,10 +525,8 @@ class CasBasedDirectory(Directory):
"""
- file_list = list(filter(lambda i: i[1].type != _FileType.DIRECTORY,
- self.index.items()))
- directory_list = filter(lambda i: i[1].type == _FileType.DIRECTORY,
- self.index.items())
+ file_list = list(filter(lambda i: i[1].type != _FileType.DIRECTORY, self.index.items()))
+ directory_list = filter(lambda i: i[1].type == _FileType.DIRECTORY, self.index.items())
if prefix != "":
yield prefix
@@ -553,10 +560,7 @@ class CasBasedDirectory(Directory):
"""
for leaf in sorted(self.index.keys()):
entry = self.index[leaf]
- info = {
- "name": os.path.join(prefix, leaf),
- "type": entry.type
- }
+ info = {"name": os.path.join(prefix, leaf), "type": entry.type}
if entry.type == _FileType.REGULAR_FILE:
info["executable"] = entry.is_executable
info["size"] = self.get_size()
@@ -599,8 +603,9 @@ class CasBasedDirectory(Directory):
def _get_underlying_directory(self):
""" There is no underlying directory for a CAS-backed directory, so
throw an exception. """
- raise VirtualDirectoryError("_get_underlying_directory was called on a CAS-backed directory," +
- " which has no underlying directory.")
+ raise VirtualDirectoryError(
+ "_get_underlying_directory was called on a CAS-backed directory," + " which has no underlying directory."
+ )
# _get_digest():
#
diff --git a/src/buildstream/storage/_filebaseddirectory.py b/src/buildstream/storage/_filebaseddirectory.py
index 07c23c192..222b47979 100644
--- a/src/buildstream/storage/_filebaseddirectory.py
+++ b/src/buildstream/storage/_filebaseddirectory.py
@@ -65,23 +65,22 @@ class FileBasedDirectory(Directory):
try:
st = os.lstat(new_path)
if not stat.S_ISDIR(st.st_mode):
- raise VirtualDirectoryError("Cannot descend into '{}': '{}' is not a directory"
- .format(path, new_path))
+ raise VirtualDirectoryError(
+ "Cannot descend into '{}': '{}' is not a directory".format(path, new_path)
+ )
except FileNotFoundError:
if create:
os.mkdir(new_path)
else:
- raise VirtualDirectoryError("Cannot descend into '{}': '{}' does not exist"
- .format(path, new_path))
+ raise VirtualDirectoryError("Cannot descend into '{}': '{}' does not exist".format(path, new_path))
current_dir = FileBasedDirectory(new_path)
return current_dir
- def import_files(self, external_pathspec, *,
- filter_callback=None,
- report_written=True, update_mtime=False,
- can_link=False):
+ def import_files(
+ self, external_pathspec, *, filter_callback=None, report_written=True, update_mtime=False, can_link=False
+ ):
""" See superclass Directory for arguments """
from ._casbaseddirectory import CasBasedDirectory # pylint: disable=cyclic-import
@@ -101,13 +100,21 @@ class FileBasedDirectory(Directory):
source_directory = external_pathspec
if can_link and not update_mtime:
- import_result = link_files(source_directory, self.external_directory,
- filter_callback=filter_callback,
- ignore_missing=False, report_written=report_written)
+ import_result = link_files(
+ source_directory,
+ self.external_directory,
+ filter_callback=filter_callback,
+ ignore_missing=False,
+ report_written=report_written,
+ )
else:
- import_result = copy_files(source_directory, self.external_directory,
- filter_callback=filter_callback,
- ignore_missing=False, report_written=report_written)
+ import_result = copy_files(
+ source_directory,
+ self.external_directory,
+ filter_callback=filter_callback,
+ ignore_missing=False,
+ report_written=report_written,
+ )
if update_mtime:
cur_time = time.time()
@@ -190,8 +197,11 @@ class FileBasedDirectory(Directory):
Return value: List(str) - list of modified paths
"""
- return [f for f in list_relative_paths(self.external_directory)
- if _get_link_mtime(os.path.join(self.external_directory, f)) != BST_ARBITRARY_TIMESTAMP]
+ return [
+ f
+ for f in list_relative_paths(self.external_directory)
+ if _get_link_mtime(os.path.join(self.external_directory, f)) != BST_ARBITRARY_TIMESTAMP
+ ]
def list_relative_paths(self):
"""Provide a list of all relative paths.
@@ -251,11 +261,13 @@ class FileBasedDirectory(Directory):
dest_subdir = self.descend(name, create=create_subdir)
except VirtualDirectoryError:
filetype = self._get_filetype(name)
- raise VirtualDirectoryError('Destination is a {}, not a directory: /{}'
- .format(filetype, relative_pathname))
+ raise VirtualDirectoryError(
+ "Destination is a {}, not a directory: /{}".format(filetype, relative_pathname)
+ )
- dest_subdir._import_files_from_cas(src_subdir, actionfunc, filter_callback,
- path_prefix=relative_pathname, result=result)
+ dest_subdir._import_files_from_cas(
+ src_subdir, actionfunc, filter_callback, path_prefix=relative_pathname, result=result
+ )
if filter_callback and not filter_callback(relative_pathname):
if is_dir and create_subdir and dest_subdir.is_empty():
@@ -279,8 +291,16 @@ class FileBasedDirectory(Directory):
src_path = source_directory.cas_cache.objpath(entry.digest)
actionfunc(src_path, dest_path, result=result)
if entry.is_executable:
- os.chmod(dest_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR |
- stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
+ os.chmod(
+ dest_path,
+ stat.S_IRUSR
+ | stat.S_IWUSR
+ | stat.S_IXUSR
+ | stat.S_IRGRP
+ | stat.S_IXGRP
+ | stat.S_IROTH
+ | stat.S_IXOTH,
+ )
else:
assert entry.type == _FileType.SYMLINK
os.symlink(entry.target, dest_path)
diff --git a/src/buildstream/storage/directory.py b/src/buildstream/storage/directory.py
index 29cbb53f2..89d20c433 100644
--- a/src/buildstream/storage/directory.py
+++ b/src/buildstream/storage/directory.py
@@ -46,11 +46,12 @@ class VirtualDirectoryError(BstError):
or either of the :class:`.ElementError` or :class:`.SourceError`
exceptions should be raised from this error.
"""
+
def __init__(self, message, reason=None):
super().__init__(message, domain=ErrorDomain.VIRTUAL_FS, reason=reason)
-class Directory():
+class Directory:
def __init__(self, external_directory=None):
raise NotImplementedError()
@@ -74,10 +75,15 @@ class Directory():
raise NotImplementedError()
# Import and export of files and links
- def import_files(self, external_pathspec: Union['Directory', str], *,
- filter_callback: Optional[Callable[[str], bool]] = None,
- report_written: bool = True, update_mtime: bool = False,
- can_link: bool = False) -> FileListResult:
+ def import_files(
+ self,
+ external_pathspec: Union["Directory", str],
+ *,
+ filter_callback: Optional[Callable[[str], bool]] = None,
+ report_written: bool = True,
+ update_mtime: bool = False,
+ can_link: bool = False
+ ) -> FileListResult:
"""Imports some or all files from external_path into this directory.
Args:
@@ -214,4 +220,4 @@ class _FileType(FastEnum):
def __str__(self):
# https://github.com/PyCQA/pylint/issues/2062
- return self.name.lower().replace('_', ' ') # pylint: disable=no-member
+ return self.name.lower().replace("_", " ") # pylint: disable=no-member
diff --git a/src/buildstream/testing/__init__.py b/src/buildstream/testing/__init__.py
index 3926b4eab..67e96885a 100644
--- a/src/buildstream/testing/__init__.py
+++ b/src/buildstream/testing/__init__.py
@@ -31,9 +31,8 @@ from .integration import integration_cache
try:
import pytest
except ImportError:
- module_name = globals()['__name__']
- msg = "Could not import pytest:\n" \
- "To use the {} module, you must have pytest installed.".format(module_name)
+ module_name = globals()["__name__"]
+ msg = "Could not import pytest:\n" "To use the {} module, you must have pytest installed.".format(module_name)
raise ImportError(msg)
@@ -41,7 +40,7 @@ except ImportError:
ALL_REPO_KINDS = OrderedDict() # type: OrderedDict[Repo, str]
-def create_repo(kind, directory, subdir='repo'):
+def create_repo(kind, directory, subdir="repo"):
"""Convenience method for creating a Repo
Args:
@@ -92,6 +91,7 @@ def sourcetests_collection_hook(session):
Args:
session (pytest.Session): The current pytest session
"""
+
def should_collect_tests(config):
args = config.args
rootdir = config.rootdir
@@ -112,6 +112,7 @@ def sourcetests_collection_hook(session):
return True
from . import _sourcetests
+
source_test_path = os.path.dirname(_sourcetests.__file__)
# Add the location of the source tests to the session's
# python_files config. Without this, pytest may filter out these
diff --git a/src/buildstream/testing/_fixtures.py b/src/buildstream/testing/_fixtures.py
index 2684782a1..5da51bb45 100644
--- a/src/buildstream/testing/_fixtures.py
+++ b/src/buildstream/testing/_fixtures.py
@@ -30,6 +30,7 @@ def thread_check():
yield
assert utils._is_single_threaded()
+
# Reset global state in node.pyx to improve test isolation
@pytest.fixture(autouse=True)
def reset_global_node_state():
diff --git a/src/buildstream/testing/_sourcetests/build_checkout.py b/src/buildstream/testing/_sourcetests/build_checkout.py
index 4d4bcf0e2..782d99814 100644
--- a/src/buildstream/testing/_sourcetests/build_checkout.py
+++ b/src/buildstream/testing/_sourcetests/build_checkout.py
@@ -29,23 +29,23 @@ from .utils import kind # pylint: disable=unused-import
# Project directory
TOP_DIR = os.path.dirname(os.path.realpath(__file__))
-DATA_DIR = os.path.join(TOP_DIR, 'project')
+DATA_DIR = os.path.join(TOP_DIR, "project")
def strict_args(args, strict):
if strict != "strict":
- return ['--no-strict', *args]
+ return ["--no-strict", *args]
return args
@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.parametrize("strict", ["strict", "non-strict"])
def test_fetch_build_checkout(cli, tmpdir, datafiles, strict, kind):
- checkout = os.path.join(cli.directory, 'checkout')
+ checkout = os.path.join(cli.directory, "checkout")
project = str(datafiles)
- dev_files_path = os.path.join(project, 'files', 'dev-files')
- element_path = os.path.join(project, 'elements')
- element_name = 'build-test-{}.bst'.format(kind)
+ dev_files_path = os.path.join(project, "files", "dev-files")
+ element_path = os.path.join(project, "elements")
+ element_name = "build-test-{}.bst".format(kind)
# Create our repo object of the given source type with
# the dev files, and then collect the initial ref.
@@ -54,26 +54,20 @@ def test_fetch_build_checkout(cli, tmpdir, datafiles, strict, kind):
ref = repo.create(dev_files_path)
# Write out our test target
- element = {
- 'kind': 'import',
- 'sources': [
- repo.source_config(ref=ref)
- ]
- }
- _yaml.roundtrip_dump(element,
- os.path.join(element_path, element_name))
+ element = {"kind": "import", "sources": [repo.source_config(ref=ref)]}
+ _yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
- assert cli.get_element_state(project, element_name) == 'fetch needed'
- result = cli.run(project=project, args=strict_args(['build', element_name], strict))
+ assert cli.get_element_state(project, element_name) == "fetch needed"
+ result = cli.run(project=project, args=strict_args(["build", element_name], strict))
result.assert_success()
- assert cli.get_element_state(project, element_name) == 'cached'
+ assert cli.get_element_state(project, element_name) == "cached"
# Now check it out
- result = cli.run(project=project, args=strict_args([
- 'artifact', 'checkout', element_name, '--directory', checkout
- ], strict))
+ result = cli.run(
+ project=project, args=strict_args(["artifact", "checkout", element_name, "--directory", checkout], strict)
+ )
result.assert_success()
# Check that the pony.h include from files/dev-files exists
- filename = os.path.join(checkout, 'usr', 'include', 'pony.h')
+ filename = os.path.join(checkout, "usr", "include", "pony.h")
assert os.path.exists(filename)
diff --git a/src/buildstream/testing/_sourcetests/fetch.py b/src/buildstream/testing/_sourcetests/fetch.py
index 897752297..05b43d793 100644
--- a/src/buildstream/testing/_sourcetests/fetch.py
+++ b/src/buildstream/testing/_sourcetests/fetch.py
@@ -32,15 +32,15 @@ from .utils import kind # pylint: disable=unused-import
# Project directory
TOP_DIR = os.path.dirname(os.path.realpath(__file__))
-DATA_DIR = os.path.join(TOP_DIR, 'project')
+DATA_DIR = os.path.join(TOP_DIR, "project")
@pytest.mark.datafiles(DATA_DIR)
def test_fetch(cli, tmpdir, datafiles, kind):
project = str(datafiles)
- bin_files_path = os.path.join(project, 'files', 'bin-files')
- element_path = os.path.join(project, 'elements')
- element_name = 'fetch-test-{}.bst'.format(kind)
+ bin_files_path = os.path.join(project, "files", "bin-files")
+ element_path = os.path.join(project, "elements")
+ element_name = "fetch-test-{}.bst".format(kind)
# Create our repo object of the given source type with
# the bin files, and then collect the initial ref.
@@ -49,59 +49,46 @@ def test_fetch(cli, tmpdir, datafiles, kind):
ref = repo.create(bin_files_path)
# Write out our test target
- element = {
- 'kind': 'import',
- 'sources': [
- repo.source_config(ref=ref)
- ]
- }
- _yaml.roundtrip_dump(element,
- os.path.join(element_path, element_name))
+ element = {"kind": "import", "sources": [repo.source_config(ref=ref)]}
+ _yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
# Assert that a fetch is needed
- assert cli.get_element_state(project, element_name) == 'fetch needed'
+ assert cli.get_element_state(project, element_name) == "fetch needed"
# Now try to fetch it
- result = cli.run(project=project, args=['source', 'fetch', element_name])
+ result = cli.run(project=project, args=["source", "fetch", element_name])
result.assert_success()
# Assert that we are now buildable because the source is
# now cached.
- assert cli.get_element_state(project, element_name) == 'buildable'
+ assert cli.get_element_state(project, element_name) == "buildable"
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("ref_storage", ['inline', 'project.refs'])
+@pytest.mark.parametrize("ref_storage", ["inline", "project.refs"])
def test_fetch_cross_junction(cli, tmpdir, datafiles, ref_storage, kind):
project = str(datafiles)
- subproject_path = os.path.join(project, 'files', 'sub-project')
- junction_path = os.path.join(project, 'elements', 'junction.bst')
+ subproject_path = os.path.join(project, "files", "sub-project")
+ junction_path = os.path.join(project, "elements", "junction.bst")
- import_etc_path = os.path.join(subproject_path, 'elements', 'import-etc-repo.bst')
- etc_files_path = os.path.join(subproject_path, 'files', 'etc-files')
+ import_etc_path = os.path.join(subproject_path, "elements", "import-etc-repo.bst")
+ etc_files_path = os.path.join(subproject_path, "files", "etc-files")
- repo = create_repo(kind, str(tmpdir.join('import-etc')))
+ repo = create_repo(kind, str(tmpdir.join("import-etc")))
ref = repo.create(etc_files_path)
- element = {
- 'kind': 'import',
- 'sources': [
- repo.source_config(ref=(ref if ref_storage == 'inline' else None))
- ]
- }
+ element = {"kind": "import", "sources": [repo.source_config(ref=(ref if ref_storage == "inline" else None))]}
_yaml.roundtrip_dump(element, import_etc_path)
- update_project_configuration(project, {
- 'ref-storage': ref_storage
- })
+ update_project_configuration(project, {"ref-storage": ref_storage})
- generate_junction(tmpdir, subproject_path, junction_path, store_ref=(ref_storage == 'inline'))
+ generate_junction(tmpdir, subproject_path, junction_path, store_ref=(ref_storage == "inline"))
- if ref_storage == 'project.refs':
- result = cli.run(project=project, args=['source', 'track', 'junction.bst'])
+ if ref_storage == "project.refs":
+ result = cli.run(project=project, args=["source", "track", "junction.bst"])
result.assert_success()
- result = cli.run(project=project, args=['source', 'track', 'junction.bst:import-etc.bst'])
+ result = cli.run(project=project, args=["source", "track", "junction.bst:import-etc.bst"])
result.assert_success()
- result = cli.run(project=project, args=['source', 'fetch', 'junction.bst:import-etc.bst'])
+ result = cli.run(project=project, args=["source", "fetch", "junction.bst:import-etc.bst"])
result.assert_success()
diff --git a/src/buildstream/testing/_sourcetests/mirror.py b/src/buildstream/testing/_sourcetests/mirror.py
index b6316045d..3ff3fb981 100644
--- a/src/buildstream/testing/_sourcetests/mirror.py
+++ b/src/buildstream/testing/_sourcetests/mirror.py
@@ -31,25 +31,25 @@ from .utils import kind # pylint: disable=unused-import
# Project directory
TOP_DIR = os.path.dirname(os.path.realpath(__file__))
-DATA_DIR = os.path.join(TOP_DIR, 'project')
+DATA_DIR = os.path.join(TOP_DIR, "project")
def _set_project_mirrors_and_aliases(project_path, mirrors, aliases):
- project_conf_path = os.path.join(project_path, 'project.conf')
+ project_conf_path = os.path.join(project_path, "project.conf")
project_conf = _yaml.roundtrip_load(project_conf_path)
- project_conf['mirrors'] = mirrors
- project_conf['aliases'].update(aliases)
+ project_conf["mirrors"] = mirrors
+ project_conf["aliases"].update(aliases)
_yaml.roundtrip_dump(project_conf, project_conf_path)
def _set_project_includes_and_aliases(project_path, includes, aliases):
- project_conf_path = os.path.join(project_path, 'project.conf')
+ project_conf_path = os.path.join(project_path, "project.conf")
project_conf = _yaml.roundtrip_load(project_conf_path)
- project_conf['aliases'].update(aliases)
- project_conf['(@)'] = includes
+ project_conf["aliases"].update(aliases)
+ project_conf["(@)"] = includes
_yaml.roundtrip_dump(project_conf, project_conf_path)
@@ -57,11 +57,11 @@ def _set_project_includes_and_aliases(project_path, includes, aliases):
@pytest.mark.datafiles(DATA_DIR)
def test_mirror_fetch(cli, tmpdir, datafiles, kind):
project_dir = str(datafiles)
- bin_files_path = os.path.join(project_dir, 'files', 'bin-files', 'usr')
- dev_files_path = os.path.join(project_dir, 'files', 'dev-files', 'usr')
- upstream_repodir = os.path.join(str(tmpdir), 'upstream')
- mirror_repodir = os.path.join(str(tmpdir), 'mirror')
- element_dir = os.path.join(project_dir, 'elements')
+ bin_files_path = os.path.join(project_dir, "files", "bin-files", "usr")
+ dev_files_path = os.path.join(project_dir, "files", "dev-files", "usr")
+ upstream_repodir = os.path.join(str(tmpdir), "upstream")
+ mirror_repodir = os.path.join(str(tmpdir), "mirror")
+ element_dir = os.path.join(project_dir, "elements")
# Create repo objects of the upstream and mirror
upstream_repo = create_repo(kind, upstream_repodir)
@@ -69,228 +69,162 @@ def test_mirror_fetch(cli, tmpdir, datafiles, kind):
mirror_repo = upstream_repo.copy(mirror_repodir)
upstream_ref = upstream_repo.create(dev_files_path)
- element = {
- 'kind': 'import',
- 'sources': [
- upstream_repo.source_config(ref=upstream_ref)
- ]
- }
- element_name = 'test.bst'
+ element = {"kind": "import", "sources": [upstream_repo.source_config(ref=upstream_ref)]}
+ element_name = "test.bst"
element_path = os.path.join(element_dir, element_name)
- full_repo = element['sources'][0]['url']
+ full_repo = element["sources"][0]["url"]
upstream_map, repo_name = os.path.split(full_repo)
- alias = 'foo-' + kind
- aliased_repo = alias + ':' + repo_name
- element['sources'][0]['url'] = aliased_repo
- full_mirror = mirror_repo.source_config()['url']
+ alias = "foo-" + kind
+ aliased_repo = alias + ":" + repo_name
+ element["sources"][0]["url"] = aliased_repo
+ full_mirror = mirror_repo.source_config()["url"]
mirror_map, _ = os.path.split(full_mirror)
_yaml.roundtrip_dump(element, element_path)
_set_project_mirrors_and_aliases(
project_dir,
- [
- {
- 'name': 'middle-earth',
- 'aliases': {
- alias: [mirror_map + '/'],
- },
- },
- ],
- {alias: upstream_map + '/'},
+ [{"name": "middle-earth", "aliases": {alias: [mirror_map + "/"],},},],
+ {alias: upstream_map + "/"},
)
# No obvious ways of checking that the mirror has been fetched
# But at least we can be sure it succeeds
- result = cli.run(project=project_dir, args=['source', 'fetch', element_name])
+ result = cli.run(project=project_dir, args=["source", "fetch", element_name])
result.assert_success()
@pytest.mark.datafiles(DATA_DIR)
def test_mirror_fetch_upstream_absent(cli, tmpdir, datafiles, kind):
project_dir = str(datafiles)
- dev_files_path = os.path.join(project_dir, 'files', 'dev-files', 'usr')
- upstream_repodir = os.path.join(project_dir, 'upstream')
- mirror_repodir = os.path.join(str(tmpdir), 'mirror')
- element_dir = os.path.join(project_dir, 'elements')
+ dev_files_path = os.path.join(project_dir, "files", "dev-files", "usr")
+ upstream_repodir = os.path.join(project_dir, "upstream")
+ mirror_repodir = os.path.join(str(tmpdir), "mirror")
+ element_dir = os.path.join(project_dir, "elements")
# Create repo objects of the upstream and mirror
upstream_repo = create_repo(kind, upstream_repodir)
ref = upstream_repo.create(dev_files_path)
mirror_repo = upstream_repo.copy(mirror_repodir)
- element = {
- 'kind': 'import',
- 'sources': [
- upstream_repo.source_config(ref=ref)
- ]
- }
+ element = {"kind": "import", "sources": [upstream_repo.source_config(ref=ref)]}
- element_name = 'test.bst'
+ element_name = "test.bst"
element_path = os.path.join(element_dir, element_name)
- full_repo = element['sources'][0]['url']
+ full_repo = element["sources"][0]["url"]
_, repo_name = os.path.split(full_repo)
- alias = 'foo-' + kind
- aliased_repo = alias + ':' + repo_name
- element['sources'][0]['url'] = aliased_repo
- full_mirror = mirror_repo.source_config()['url']
+ alias = "foo-" + kind
+ aliased_repo = alias + ":" + repo_name
+ element["sources"][0]["url"] = aliased_repo
+ full_mirror = mirror_repo.source_config()["url"]
mirror_map, _ = os.path.split(full_mirror)
_yaml.roundtrip_dump(element, element_path)
_set_project_mirrors_and_aliases(
project_dir,
- [
- {
- 'name': 'middle-earth',
- 'aliases': {
- alias: [mirror_map + "/"]
- },
- },
- ],
- {alias: 'http://www.example.com'},
+ [{"name": "middle-earth", "aliases": {alias: [mirror_map + "/"]},},],
+ {alias: "http://www.example.com"},
)
- result = cli.run(project=project_dir, args=['source', 'fetch', element_name])
+ result = cli.run(project=project_dir, args=["source", "fetch", element_name])
result.assert_success()
@pytest.mark.datafiles(DATA_DIR)
def test_mirror_from_includes(cli, tmpdir, datafiles, kind):
project_dir = str(datafiles)
- bin_files_path = os.path.join(project_dir, 'files', 'bin-files', 'usr')
- upstream_repodir = os.path.join(str(tmpdir), 'upstream')
- mirror_repodir = os.path.join(str(tmpdir), 'mirror')
- element_dir = os.path.join(project_dir, 'elements')
+ bin_files_path = os.path.join(project_dir, "files", "bin-files", "usr")
+ upstream_repodir = os.path.join(str(tmpdir), "upstream")
+ mirror_repodir = os.path.join(str(tmpdir), "mirror")
+ element_dir = os.path.join(project_dir, "elements")
# Create repo objects of the upstream and mirror
upstream_repo = create_repo(kind, upstream_repodir)
upstream_ref = upstream_repo.create(bin_files_path)
mirror_repo = upstream_repo.copy(mirror_repodir)
- element = {
- 'kind': 'import',
- 'sources': [
- upstream_repo.source_config(ref=upstream_ref)
- ]
- }
- element_name = 'test.bst'
+ element = {"kind": "import", "sources": [upstream_repo.source_config(ref=upstream_ref)]}
+ element_name = "test.bst"
element_path = os.path.join(element_dir, element_name)
- full_repo = element['sources'][0]['url']
+ full_repo = element["sources"][0]["url"]
upstream_map, repo_name = os.path.split(full_repo)
- alias = 'foo-' + kind
- aliased_repo = alias + ':' + repo_name
- element['sources'][0]['url'] = aliased_repo
- full_mirror = mirror_repo.source_config()['url']
+ alias = "foo-" + kind
+ aliased_repo = alias + ":" + repo_name
+ element["sources"][0]["url"] = aliased_repo
+ full_mirror = mirror_repo.source_config()["url"]
mirror_map, _ = os.path.split(full_mirror)
_yaml.roundtrip_dump(element, element_path)
- config_project_dir = str(tmpdir.join('config'))
+ config_project_dir = str(tmpdir.join("config"))
os.makedirs(config_project_dir, exist_ok=True)
- config_project = {
- 'name': 'config'
- }
- _yaml.roundtrip_dump(config_project, os.path.join(config_project_dir, 'project.conf'))
- extra_mirrors = {
- 'mirrors': [
- {
- 'name': 'middle-earth',
- 'aliases': {
- alias: [mirror_map + "/"],
- }
- }
- ]
- }
- _yaml.roundtrip_dump(extra_mirrors, os.path.join(config_project_dir, 'mirrors.yml'))
- generate_junction(str(tmpdir.join('config_repo')),
- config_project_dir,
- os.path.join(element_dir, 'config.bst'))
+ config_project = {"name": "config"}
+ _yaml.roundtrip_dump(config_project, os.path.join(config_project_dir, "project.conf"))
+ extra_mirrors = {"mirrors": [{"name": "middle-earth", "aliases": {alias: [mirror_map + "/"],}}]}
+ _yaml.roundtrip_dump(extra_mirrors, os.path.join(config_project_dir, "mirrors.yml"))
+ generate_junction(str(tmpdir.join("config_repo")), config_project_dir, os.path.join(element_dir, "config.bst"))
_set_project_includes_and_aliases(
- project_dir,
- ['config.bst:mirrors.yml'],
- {alias: upstream_map + '/'},
+ project_dir, ["config.bst:mirrors.yml"], {alias: upstream_map + "/"},
)
# Now make the upstream unavailable.
- os.rename(upstream_repo.repo, '{}.bak'.format(upstream_repo.repo))
- result = cli.run(project=project_dir, args=['source', 'fetch', element_name])
+ os.rename(upstream_repo.repo, "{}.bak".format(upstream_repo.repo))
+ result = cli.run(project=project_dir, args=["source", "fetch", element_name])
result.assert_success()
@pytest.mark.datafiles(DATA_DIR)
def test_mirror_junction_from_includes(cli, tmpdir, datafiles, kind):
project_dir = str(datafiles)
- bin_files_path = os.path.join(project_dir, 'files', 'bin-files', 'usr')
- upstream_repodir = os.path.join(str(tmpdir), 'upstream')
- mirror_repodir = os.path.join(str(tmpdir), 'mirror')
- element_dir = os.path.join(project_dir, 'elements')
+ bin_files_path = os.path.join(project_dir, "files", "bin-files", "usr")
+ upstream_repodir = os.path.join(str(tmpdir), "upstream")
+ mirror_repodir = os.path.join(str(tmpdir), "mirror")
+ element_dir = os.path.join(project_dir, "elements")
# Create repo objects of the upstream and mirror
upstream_repo = create_repo(kind, upstream_repodir)
upstream_ref = upstream_repo.create(bin_files_path)
mirror_repo = upstream_repo.copy(mirror_repodir)
- element = {
- 'kind': 'junction',
- 'sources': [
- upstream_repo.source_config(ref=upstream_ref)
- ]
- }
- element_name = 'test.bst'
+ element = {"kind": "junction", "sources": [upstream_repo.source_config(ref=upstream_ref)]}
+ element_name = "test.bst"
element_path = os.path.join(element_dir, element_name)
- full_repo = element['sources'][0]['url']
+ full_repo = element["sources"][0]["url"]
upstream_map, repo_name = os.path.split(full_repo)
- alias = 'foo-' + kind
- aliased_repo = alias + ':' + repo_name
- element['sources'][0]['url'] = aliased_repo
- full_mirror = mirror_repo.source_config()['url']
+ alias = "foo-" + kind
+ aliased_repo = alias + ":" + repo_name
+ element["sources"][0]["url"] = aliased_repo
+ full_mirror = mirror_repo.source_config()["url"]
mirror_map, _ = os.path.split(full_mirror)
_yaml.roundtrip_dump(element, element_path)
- config_project_dir = str(tmpdir.join('config'))
+ config_project_dir = str(tmpdir.join("config"))
os.makedirs(config_project_dir, exist_ok=True)
- config_project = {
- 'name': 'config'
- }
- _yaml.roundtrip_dump(config_project, os.path.join(config_project_dir, 'project.conf'))
- extra_mirrors = {
- 'mirrors': [
- {
- 'name': 'middle-earth',
- 'aliases': {
- alias: [mirror_map + "/"],
- }
- }
- ]
- }
- _yaml.roundtrip_dump(extra_mirrors, os.path.join(config_project_dir, 'mirrors.yml'))
- generate_junction(str(tmpdir.join('config_repo')),
- config_project_dir,
- os.path.join(element_dir, 'config.bst'))
+ config_project = {"name": "config"}
+ _yaml.roundtrip_dump(config_project, os.path.join(config_project_dir, "project.conf"))
+ extra_mirrors = {"mirrors": [{"name": "middle-earth", "aliases": {alias: [mirror_map + "/"],}}]}
+ _yaml.roundtrip_dump(extra_mirrors, os.path.join(config_project_dir, "mirrors.yml"))
+ generate_junction(str(tmpdir.join("config_repo")), config_project_dir, os.path.join(element_dir, "config.bst"))
- _set_project_includes_and_aliases(
- project_dir,
- ['config.bst:mirrors.yml'],
- {alias: upstream_map + '/'}
- )
+ _set_project_includes_and_aliases(project_dir, ["config.bst:mirrors.yml"], {alias: upstream_map + "/"})
# Now make the upstream unavailable.
- os.rename(upstream_repo.repo, '{}.bak'.format(upstream_repo.repo))
- result = cli.run(project=project_dir, args=['source', 'fetch', element_name])
+ os.rename(upstream_repo.repo, "{}.bak".format(upstream_repo.repo))
+ result = cli.run(project=project_dir, args=["source", "fetch", element_name])
result.assert_main_error(ErrorDomain.STREAM, None)
# Now make the upstream available again.
- os.rename('{}.bak'.format(upstream_repo.repo), upstream_repo.repo)
- result = cli.run(project=project_dir, args=['source', 'fetch', element_name])
+ os.rename("{}.bak".format(upstream_repo.repo), upstream_repo.repo)
+ result = cli.run(project=project_dir, args=["source", "fetch", element_name])
result.assert_success()
@pytest.mark.datafiles(DATA_DIR)
def test_mirror_track_upstream_present(cli, tmpdir, datafiles, kind):
project_dir = str(datafiles)
- bin_files_path = os.path.join(project_dir, 'files', 'bin-files', 'usr')
- dev_files_path = os.path.join(project_dir, 'files', 'dev-files', 'usr')
- upstream_repodir = os.path.join(str(tmpdir), 'upstream')
- mirror_repodir = os.path.join(str(tmpdir), 'mirror')
- element_dir = os.path.join(project_dir, 'elements')
+ bin_files_path = os.path.join(project_dir, "files", "bin-files", "usr")
+ dev_files_path = os.path.join(project_dir, "files", "dev-files", "usr")
+ upstream_repodir = os.path.join(str(tmpdir), "upstream")
+ mirror_repodir = os.path.join(str(tmpdir), "mirror")
+ element_dir = os.path.join(project_dir, "elements")
# Create repo objects of the upstream and mirror
upstream_repo = create_repo(kind, upstream_repodir)
@@ -298,55 +232,43 @@ def test_mirror_track_upstream_present(cli, tmpdir, datafiles, kind):
mirror_repo = upstream_repo.copy(mirror_repodir)
upstream_ref = upstream_repo.create(dev_files_path)
- element = {
- 'kind': 'import',
- 'sources': [
- upstream_repo.source_config(ref=upstream_ref)
- ]
- }
+ element = {"kind": "import", "sources": [upstream_repo.source_config(ref=upstream_ref)]}
- element_name = 'test.bst'
+ element_name = "test.bst"
element_path = os.path.join(element_dir, element_name)
- full_repo = element['sources'][0]['url']
+ full_repo = element["sources"][0]["url"]
upstream_map, repo_name = os.path.split(full_repo)
- alias = 'foo-' + kind
- aliased_repo = alias + ':' + repo_name
- element['sources'][0]['url'] = aliased_repo
- full_mirror = mirror_repo.source_config()['url']
+ alias = "foo-" + kind
+ aliased_repo = alias + ":" + repo_name
+ element["sources"][0]["url"] = aliased_repo
+ full_mirror = mirror_repo.source_config()["url"]
mirror_map, _ = os.path.split(full_mirror)
_yaml.roundtrip_dump(element, element_path)
_set_project_mirrors_and_aliases(
project_dir,
- [
- {
- 'name': 'middle-earth',
- 'aliases': {
- alias: [mirror_map + '/'],
- },
- },
- ],
- {alias: upstream_map + '/'},
+ [{"name": "middle-earth", "aliases": {alias: [mirror_map + "/"],},},],
+ {alias: upstream_map + "/"},
)
- result = cli.run(project=project_dir, args=['source', 'track', element_name])
+ result = cli.run(project=project_dir, args=["source", "track", element_name])
result.assert_success()
# Tracking tries upstream first. Check the ref is from upstream.
new_element = _yaml.load(element_path)
- source = new_element.get_sequence('sources').mapping_at(0)
- if 'ref' in source:
- assert source.get_str('ref') == upstream_ref
+ source = new_element.get_sequence("sources").mapping_at(0)
+ if "ref" in source:
+ assert source.get_str("ref") == upstream_ref
@pytest.mark.datafiles(DATA_DIR)
def test_mirror_track_upstream_absent(cli, tmpdir, datafiles, kind):
project_dir = str(datafiles)
- bin_files_path = os.path.join(project_dir, 'files', 'bin-files', 'usr')
- dev_files_path = os.path.join(project_dir, 'files', 'dev-files', 'usr')
- upstream_repodir = os.path.join(str(tmpdir), 'upstream')
- mirror_repodir = os.path.join(str(tmpdir), 'mirror')
- element_dir = os.path.join(project_dir, 'elements')
+ bin_files_path = os.path.join(project_dir, "files", "bin-files", "usr")
+ dev_files_path = os.path.join(project_dir, "files", "dev-files", "usr")
+ upstream_repodir = os.path.join(str(tmpdir), "upstream")
+ mirror_repodir = os.path.join(str(tmpdir), "mirror")
+ element_dir = os.path.join(project_dir, "elements")
# Create repo objects of the upstream and mirror
upstream_repo = create_repo(kind, upstream_repodir)
@@ -355,42 +277,30 @@ def test_mirror_track_upstream_absent(cli, tmpdir, datafiles, kind):
mirror_ref = upstream_ref
upstream_ref = upstream_repo.create(dev_files_path)
- element = {
- 'kind': 'import',
- 'sources': [
- upstream_repo.source_config(ref=upstream_ref)
- ]
- }
+ element = {"kind": "import", "sources": [upstream_repo.source_config(ref=upstream_ref)]}
- element_name = 'test.bst'
+ element_name = "test.bst"
element_path = os.path.join(element_dir, element_name)
- full_repo = element['sources'][0]['url']
+ full_repo = element["sources"][0]["url"]
_, repo_name = os.path.split(full_repo)
- alias = 'foo-' + kind
- aliased_repo = alias + ':' + repo_name
- element['sources'][0]['url'] = aliased_repo
- full_mirror = mirror_repo.source_config()['url']
+ alias = "foo-" + kind
+ aliased_repo = alias + ":" + repo_name
+ element["sources"][0]["url"] = aliased_repo
+ full_mirror = mirror_repo.source_config()["url"]
mirror_map, _ = os.path.split(full_mirror)
_yaml.roundtrip_dump(element, element_path)
_set_project_mirrors_and_aliases(
project_dir,
- [
- {
- 'name': 'middle-earth',
- 'aliases': {
- alias: [mirror_map + '/'],
- },
- },
- ],
- {alias: 'http://www.example.com'},
+ [{"name": "middle-earth", "aliases": {alias: [mirror_map + "/"],},},],
+ {alias: "http://www.example.com"},
)
- result = cli.run(project=project_dir, args=['source', 'track', element_name])
+ result = cli.run(project=project_dir, args=["source", "track", element_name])
result.assert_success()
# Check that tracking fell back to the mirror
new_element = _yaml.load(element_path)
- source = new_element.get_sequence('sources').mapping_at(0)
- if 'ref' in source:
- assert source.get_str('ref') == mirror_ref
+ source = new_element.get_sequence("sources").mapping_at(0)
+ if "ref" in source:
+ assert source.get_str("ref") == mirror_ref
diff --git a/src/buildstream/testing/_sourcetests/source_determinism.py b/src/buildstream/testing/_sourcetests/source_determinism.py
index fc0e4618c..27664e0c2 100644
--- a/src/buildstream/testing/_sourcetests/source_determinism.py
+++ b/src/buildstream/testing/_sourcetests/source_determinism.py
@@ -30,83 +30,72 @@ from .utils import kind # pylint: disable=unused-import
# Project directory
TOP_DIR = os.path.dirname(os.path.realpath(__file__))
-DATA_DIR = os.path.join(TOP_DIR, 'project')
+DATA_DIR = os.path.join(TOP_DIR, "project")
-def create_test_file(*path, mode=0o644, content='content\n'):
+def create_test_file(*path, mode=0o644, content="content\n"):
path = os.path.join(*path)
os.makedirs(os.path.dirname(path), exist_ok=True)
- with open(path, 'w') as f:
+ with open(path, "w") as f:
f.write(content)
os.fchmod(f.fileno(), mode)
def create_test_directory(*path, mode=0o644):
- create_test_file(*path, '.keep', content='')
+ create_test_file(*path, ".keep", content="")
path = os.path.join(*path)
os.chmod(path, mode)
@pytest.mark.integration
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.skipif(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox, Must Fix')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
+@pytest.mark.skipif(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox, Must Fix")
def test_deterministic_source_umask(cli, tmpdir, datafiles, kind):
project = str(datafiles)
- element_name = 'list.bst'
- element_path = os.path.join(project, 'elements', element_name)
- repodir = os.path.join(str(tmpdir), 'repo')
- sourcedir = os.path.join(project, 'source')
-
- create_test_file(sourcedir, 'a.txt', mode=0o700)
- create_test_file(sourcedir, 'b.txt', mode=0o755)
- create_test_file(sourcedir, 'c.txt', mode=0o600)
- create_test_file(sourcedir, 'd.txt', mode=0o400)
- create_test_file(sourcedir, 'e.txt', mode=0o644)
- create_test_file(sourcedir, 'f.txt', mode=0o4755)
- create_test_file(sourcedir, 'g.txt', mode=0o2755)
- create_test_file(sourcedir, 'h.txt', mode=0o1755)
- create_test_directory(sourcedir, 'dir-a', mode=0o0700)
- create_test_directory(sourcedir, 'dir-c', mode=0o0755)
- create_test_directory(sourcedir, 'dir-d', mode=0o4755)
- create_test_directory(sourcedir, 'dir-e', mode=0o2755)
- create_test_directory(sourcedir, 'dir-f', mode=0o1755)
+ element_name = "list.bst"
+ element_path = os.path.join(project, "elements", element_name)
+ repodir = os.path.join(str(tmpdir), "repo")
+ sourcedir = os.path.join(project, "source")
+
+ create_test_file(sourcedir, "a.txt", mode=0o700)
+ create_test_file(sourcedir, "b.txt", mode=0o755)
+ create_test_file(sourcedir, "c.txt", mode=0o600)
+ create_test_file(sourcedir, "d.txt", mode=0o400)
+ create_test_file(sourcedir, "e.txt", mode=0o644)
+ create_test_file(sourcedir, "f.txt", mode=0o4755)
+ create_test_file(sourcedir, "g.txt", mode=0o2755)
+ create_test_file(sourcedir, "h.txt", mode=0o1755)
+ create_test_directory(sourcedir, "dir-a", mode=0o0700)
+ create_test_directory(sourcedir, "dir-c", mode=0o0755)
+ create_test_directory(sourcedir, "dir-d", mode=0o4755)
+ create_test_directory(sourcedir, "dir-e", mode=0o2755)
+ create_test_directory(sourcedir, "dir-f", mode=0o1755)
repo = create_repo(kind, repodir)
ref = repo.create(sourcedir)
source = repo.source_config(ref=ref)
element = {
- 'kind': 'manual',
- 'depends': [
- {
- 'filename': 'base.bst',
- 'type': 'build'
- }
- ],
- 'sources': [
- source
- ],
- 'config': {
- 'install-commands': [
- 'ls -l >"%{install-root}/ls-l"'
- ]
- }
+ "kind": "manual",
+ "depends": [{"filename": "base.bst", "type": "build"}],
+ "sources": [source],
+ "config": {"install-commands": ['ls -l >"%{install-root}/ls-l"']},
}
_yaml.roundtrip_dump(element, element_path)
def get_value_for_umask(umask):
- checkoutdir = os.path.join(str(tmpdir), 'checkout-{}'.format(umask))
+ checkoutdir = os.path.join(str(tmpdir), "checkout-{}".format(umask))
old_umask = os.umask(umask)
try:
- result = cli.run(project=project, args=['build', element_name])
+ result = cli.run(project=project, args=["build", element_name])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkoutdir])
result.assert_success()
- with open(os.path.join(checkoutdir, 'ls-l'), 'r') as f:
+ with open(os.path.join(checkoutdir, "ls-l"), "r") as f:
return f.read()
finally:
os.umask(old_umask)
diff --git a/src/buildstream/testing/_sourcetests/track.py b/src/buildstream/testing/_sourcetests/track.py
index 48856b351..623045cd9 100644
--- a/src/buildstream/testing/_sourcetests/track.py
+++ b/src/buildstream/testing/_sourcetests/track.py
@@ -33,33 +33,26 @@ from .utils import kind # pylint: disable=unused-import
# Project directory
TOP_DIR = os.path.dirname(os.path.realpath(__file__))
-DATA_DIR = os.path.join(TOP_DIR, 'project')
+DATA_DIR = os.path.join(TOP_DIR, "project")
def generate_element(repo, element_path, dep_name=None):
- element = {
- 'kind': 'import',
- 'sources': [
- repo.source_config()
- ]
- }
+ element = {"kind": "import", "sources": [repo.source_config()]}
if dep_name:
- element['depends'] = [dep_name]
+ element["depends"] = [dep_name]
_yaml.roundtrip_dump(element, element_path)
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("ref_storage", ['inline', 'project.refs'])
+@pytest.mark.parametrize("ref_storage", ["inline", "project.refs"])
def test_track(cli, tmpdir, datafiles, ref_storage, kind):
project = str(datafiles)
- dev_files_path = os.path.join(project, 'files', 'dev-files')
- element_path = os.path.join(project, 'elements')
- element_name = 'track-test-{}.bst'.format(kind)
+ dev_files_path = os.path.join(project, "files", "dev-files")
+ element_path = os.path.join(project, "elements")
+ element_name = "track-test-{}.bst".format(kind)
- update_project_configuration(project, {
- 'ref-storage': ref_storage
- })
+ update_project_configuration(project, {"ref-storage": ref_storage})
# Create our repo object of the given source type with
# the dev files, and then collect the initial ref.
@@ -71,28 +64,28 @@ def test_track(cli, tmpdir, datafiles, ref_storage, kind):
generate_element(repo, os.path.join(element_path, element_name))
# Assert that a fetch is needed
- assert cli.get_element_state(project, element_name) == 'no reference'
+ assert cli.get_element_state(project, element_name) == "no reference"
# Now first try to track it
- result = cli.run(project=project, args=['source', 'track', element_name])
+ result = cli.run(project=project, args=["source", "track", element_name])
result.assert_success()
# And now fetch it: The Source has probably already cached the
# latest ref locally, but it is not required to have cached
# the associated content of the latest ref at track time, that
# is the job of fetch.
- result = cli.run(project=project, args=['source', 'fetch', element_name])
+ result = cli.run(project=project, args=["source", "fetch", element_name])
result.assert_success()
# Assert that we are now buildable because the source is
# now cached.
- assert cli.get_element_state(project, element_name) == 'buildable'
+ assert cli.get_element_state(project, element_name) == "buildable"
# Assert there was a project.refs created, depending on the configuration
- if ref_storage == 'project.refs':
- assert os.path.exists(os.path.join(project, 'project.refs'))
+ if ref_storage == "project.refs":
+ assert os.path.exists(os.path.join(project, "project.refs"))
else:
- assert not os.path.exists(os.path.join(project, 'project.refs'))
+ assert not os.path.exists(os.path.join(project, "project.refs"))
# NOTE:
@@ -112,18 +105,14 @@ def test_track(cli, tmpdir, datafiles, ref_storage, kind):
@pytest.mark.parametrize("amount", [1, 10])
def test_track_recurse(cli, tmpdir, datafiles, kind, amount):
project = str(datafiles)
- dev_files_path = os.path.join(project, 'files', 'dev-files')
- element_path = os.path.join(project, 'elements')
+ dev_files_path = os.path.join(project, "files", "dev-files")
+ element_path = os.path.join(project, "elements")
# Try to actually launch as many fetch jobs as possible at the same time
#
# This stresses the Source plugins and helps to ensure that
# they handle concurrent access to the store correctly.
- cli.configure({
- 'scheduler': {
- 'fetchers': amount,
- }
- })
+ cli.configure({"scheduler": {"fetchers": amount,}})
# Create our repo object of the given source type with
# the dev files, and then collect the initial ref.
@@ -135,7 +124,7 @@ def test_track_recurse(cli, tmpdir, datafiles, kind, amount):
element_names = []
last_element_name = None
for i in range(amount + 1):
- element_name = 'track-test-{}-{}.bst'.format(kind, i + 1)
+ element_name = "track-test-{}-{}.bst".format(kind, i + 1)
filename = os.path.join(element_path, element_name)
element_names.append(element_name)
@@ -146,39 +135,35 @@ def test_track_recurse(cli, tmpdir, datafiles, kind, amount):
# Assert that a fetch is needed
states = cli.get_element_states(project, [last_element_name])
for element_name in element_names:
- assert states[element_name] == 'no reference'
+ assert states[element_name] == "no reference"
# Now first try to track it
- result = cli.run(project=project, args=[
- 'source', 'track', '--deps', 'all',
- last_element_name])
+ result = cli.run(project=project, args=["source", "track", "--deps", "all", last_element_name])
result.assert_success()
# And now fetch it: The Source has probably already cached the
# latest ref locally, but it is not required to have cached
# the associated content of the latest ref at track time, that
# is the job of fetch.
- result = cli.run(project=project, args=[
- 'source', 'fetch', '--deps', 'all',
- last_element_name])
+ result = cli.run(project=project, args=["source", "fetch", "--deps", "all", last_element_name])
result.assert_success()
# Assert that the base is buildable and the rest are waiting
states = cli.get_element_states(project, [last_element_name])
for element_name in element_names:
if element_name == element_names[0]:
- assert states[element_name] == 'buildable'
+ assert states[element_name] == "buildable"
else:
- assert states[element_name] == 'waiting'
+ assert states[element_name] == "waiting"
@pytest.mark.datafiles(DATA_DIR)
def test_track_recurse_except(cli, tmpdir, datafiles, kind):
project = str(datafiles)
- dev_files_path = os.path.join(project, 'files', 'dev-files')
- element_path = os.path.join(project, 'elements')
- element_dep_name = 'track-test-dep-{}.bst'.format(kind)
- element_target_name = 'track-test-target-{}.bst'.format(kind)
+ dev_files_path = os.path.join(project, "files", "dev-files")
+ element_path = os.path.join(project, "elements")
+ element_dep_name = "track-test-dep-{}.bst".format(kind)
+ element_target_name = "track-test-target-{}.bst".format(kind)
# Create our repo object of the given source type with
# the dev files, and then collect the initial ref.
@@ -188,88 +173,79 @@ def test_track_recurse_except(cli, tmpdir, datafiles, kind):
# Write out our test targets
generate_element(repo, os.path.join(element_path, element_dep_name))
- generate_element(repo, os.path.join(element_path, element_target_name),
- dep_name=element_dep_name)
+ generate_element(repo, os.path.join(element_path, element_target_name), dep_name=element_dep_name)
# Assert that a fetch is needed
states = cli.get_element_states(project, [element_target_name])
- assert states[element_dep_name] == 'no reference'
- assert states[element_target_name] == 'no reference'
+ assert states[element_dep_name] == "no reference"
+ assert states[element_target_name] == "no reference"
# Now first try to track it
- result = cli.run(project=project, args=[
- 'source', 'track', '--deps', 'all', '--except', element_dep_name,
- element_target_name])
+ result = cli.run(
+ project=project, args=["source", "track", "--deps", "all", "--except", element_dep_name, element_target_name]
+ )
result.assert_success()
# And now fetch it: The Source has probably already cached the
# latest ref locally, but it is not required to have cached
# the associated content of the latest ref at track time, that
# is the job of fetch.
- result = cli.run(project=project, args=[
- 'source', 'fetch', '--deps', 'none',
- element_target_name])
+ result = cli.run(project=project, args=["source", "fetch", "--deps", "none", element_target_name])
result.assert_success()
# Assert that the dependency is buildable and the target is waiting
states = cli.get_element_states(project, [element_target_name])
- assert states[element_dep_name] == 'no reference'
- assert states[element_target_name] == 'waiting'
+ assert states[element_dep_name] == "no reference"
+ assert states[element_target_name] == "waiting"
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("ref_storage", ['inline', 'project.refs'])
+@pytest.mark.parametrize("ref_storage", ["inline", "project.refs"])
def test_cross_junction(cli, tmpdir, datafiles, ref_storage, kind):
project = str(datafiles)
- subproject_path = os.path.join(project, 'files', 'sub-project')
- junction_path = os.path.join(project, 'elements', 'junction.bst')
- etc_files = os.path.join(subproject_path, 'files', 'etc-files')
- repo_element_path = os.path.join(subproject_path, 'elements',
- 'import-etc-repo.bst')
+ subproject_path = os.path.join(project, "files", "sub-project")
+ junction_path = os.path.join(project, "elements", "junction.bst")
+ etc_files = os.path.join(subproject_path, "files", "etc-files")
+ repo_element_path = os.path.join(subproject_path, "elements", "import-etc-repo.bst")
- update_project_configuration(project, {
- 'ref-storage': ref_storage
- })
+ update_project_configuration(project, {"ref-storage": ref_storage})
- repo = create_repo(kind, str(tmpdir.join('element_repo')))
+ repo = create_repo(kind, str(tmpdir.join("element_repo")))
repo.create(etc_files)
generate_element(repo, repo_element_path)
- generate_junction(str(tmpdir.join('junction_repo')),
- subproject_path, junction_path, store_ref=False)
+ generate_junction(str(tmpdir.join("junction_repo")), subproject_path, junction_path, store_ref=False)
# Track the junction itself first.
- result = cli.run(project=project, args=['source', 'track', 'junction.bst'])
+ result = cli.run(project=project, args=["source", "track", "junction.bst"])
result.assert_success()
- assert cli.get_element_state(project, 'junction.bst:import-etc-repo.bst') == 'no reference'
+ assert cli.get_element_state(project, "junction.bst:import-etc-repo.bst") == "no reference"
# Track the cross junction element. -J is not given, it is implied.
- result = cli.run(project=project, args=['source', 'track', 'junction.bst:import-etc-repo.bst'])
+ result = cli.run(project=project, args=["source", "track", "junction.bst:import-etc-repo.bst"])
- if ref_storage == 'inline':
+ if ref_storage == "inline":
# This is not allowed to track cross junction without project.refs.
- result.assert_main_error(ErrorDomain.PIPELINE, 'untrackable-sources')
+ result.assert_main_error(ErrorDomain.PIPELINE, "untrackable-sources")
else:
result.assert_success()
- assert cli.get_element_state(project, 'junction.bst:import-etc-repo.bst') == 'buildable'
+ assert cli.get_element_state(project, "junction.bst:import-etc-repo.bst") == "buildable"
- assert os.path.exists(os.path.join(project, 'project.refs'))
+ assert os.path.exists(os.path.join(project, "project.refs"))
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("ref_storage", ['inline', 'project.refs'])
+@pytest.mark.parametrize("ref_storage", ["inline", "project.refs"])
def test_track_include(cli, tmpdir, datafiles, ref_storage, kind):
project = str(datafiles)
- dev_files_path = os.path.join(project, 'files', 'dev-files')
- element_path = os.path.join(project, 'elements')
- element_name = 'track-test-{}.bst'.format(kind)
+ dev_files_path = os.path.join(project, "files", "dev-files")
+ element_path = os.path.join(project, "elements")
+ element_name = "track-test-{}.bst".format(kind)
- update_project_configuration(project, {
- 'ref-storage': ref_storage
- })
+ update_project_configuration(project, {"ref-storage": ref_storage})
# Create our repo object of the given source type with
# the dev files, and then collect the initial ref.
@@ -278,139 +254,118 @@ def test_track_include(cli, tmpdir, datafiles, ref_storage, kind):
ref = repo.create(dev_files_path)
# Generate the element
- element = {
- 'kind': 'import',
- '(@)': ['elements/sources.yml']
- }
- sources = {
- 'sources': [
- repo.source_config()
- ]
- }
+ element = {"kind": "import", "(@)": ["elements/sources.yml"]}
+ sources = {"sources": [repo.source_config()]}
_yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
- _yaml.roundtrip_dump(sources, os.path.join(element_path, 'sources.yml'))
+ _yaml.roundtrip_dump(sources, os.path.join(element_path, "sources.yml"))
# Assert that a fetch is needed
- assert cli.get_element_state(project, element_name) == 'no reference'
+ assert cli.get_element_state(project, element_name) == "no reference"
# Now first try to track it
- result = cli.run(project=project, args=['source', 'track', element_name])
+ result = cli.run(project=project, args=["source", "track", element_name])
result.assert_success()
# And now fetch it: The Source has probably already cached the
# latest ref locally, but it is not required to have cached
# the associated content of the latest ref at track time, that
# is the job of fetch.
- result = cli.run(project=project, args=['source', 'fetch', element_name])
+ result = cli.run(project=project, args=["source", "fetch", element_name])
result.assert_success()
# Assert that we are now buildable because the source is
# now cached.
- assert cli.get_element_state(project, element_name) == 'buildable'
+ assert cli.get_element_state(project, element_name) == "buildable"
# Assert there was a project.refs created, depending on the configuration
- if ref_storage == 'project.refs':
- assert os.path.exists(os.path.join(project, 'project.refs'))
+ if ref_storage == "project.refs":
+ assert os.path.exists(os.path.join(project, "project.refs"))
else:
- assert not os.path.exists(os.path.join(project, 'project.refs'))
+ assert not os.path.exists(os.path.join(project, "project.refs"))
- new_sources = _yaml.load(os.path.join(element_path, 'sources.yml'))
+ new_sources = _yaml.load(os.path.join(element_path, "sources.yml"))
# Get all of the sources
- assert 'sources' in new_sources
- sources_list = new_sources.get_sequence('sources')
+ assert "sources" in new_sources
+ sources_list = new_sources.get_sequence("sources")
assert len(sources_list) == 1
# Get the first source from the sources list
new_source = sources_list.mapping_at(0)
- assert 'ref' in new_source
- assert ref == new_source.get_str('ref')
+ assert "ref" in new_source
+ assert ref == new_source.get_str("ref")
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("ref_storage", ['inline', 'project.refs'])
+@pytest.mark.parametrize("ref_storage", ["inline", "project.refs"])
def test_track_include_junction(cli, tmpdir, datafiles, ref_storage, kind):
project = str(datafiles)
- dev_files_path = os.path.join(project, 'files', 'dev-files')
- element_path = os.path.join(project, 'elements')
- element_name = 'track-test-{}.bst'.format(kind)
- subproject_path = os.path.join(project, 'files', 'sub-project')
- sub_element_path = os.path.join(subproject_path, 'elements')
- junction_path = os.path.join(element_path, 'junction.bst')
+ dev_files_path = os.path.join(project, "files", "dev-files")
+ element_path = os.path.join(project, "elements")
+ element_name = "track-test-{}.bst".format(kind)
+ subproject_path = os.path.join(project, "files", "sub-project")
+ sub_element_path = os.path.join(subproject_path, "elements")
+ junction_path = os.path.join(element_path, "junction.bst")
- update_project_configuration(project, {
- 'ref-storage': ref_storage
- })
+ update_project_configuration(project, {"ref-storage": ref_storage})
# Create our repo object of the given source type with
# the dev files, and then collect the initial ref.
#
- repo = create_repo(kind, str(tmpdir.join('element_repo')))
+ repo = create_repo(kind, str(tmpdir.join("element_repo")))
repo.create(dev_files_path)
# Generate the element
- element = {
- 'kind': 'import',
- '(@)': ['junction.bst:elements/sources.yml']
- }
- sources = {
- 'sources': [
- repo.source_config()
- ]
- }
+ element = {"kind": "import", "(@)": ["junction.bst:elements/sources.yml"]}
+ sources = {"sources": [repo.source_config()]}
_yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
- _yaml.roundtrip_dump(sources, os.path.join(sub_element_path, 'sources.yml'))
+ _yaml.roundtrip_dump(sources, os.path.join(sub_element_path, "sources.yml"))
- generate_junction(str(tmpdir.join('junction_repo')),
- subproject_path, junction_path, store_ref=True)
+ generate_junction(str(tmpdir.join("junction_repo")), subproject_path, junction_path, store_ref=True)
- result = cli.run(project=project, args=['source', 'track', 'junction.bst'])
+ result = cli.run(project=project, args=["source", "track", "junction.bst"])
result.assert_success()
# Assert that a fetch is needed
- assert cli.get_element_state(project, element_name) == 'no reference'
+ assert cli.get_element_state(project, element_name) == "no reference"
# Now first try to track it
- result = cli.run(project=project, args=['source', 'track', element_name])
+ result = cli.run(project=project, args=["source", "track", element_name])
# Assert there was a project.refs created, depending on the configuration
- if ref_storage == 'inline':
+ if ref_storage == "inline":
# FIXME: We should expect an error. But only a warning is emitted
# result.assert_main_error(ErrorDomain.SOURCE, 'tracking-junction-fragment')
- assert 'junction.bst:elements/sources.yml: Cannot track source in a fragment from a junction' in result.stderr
+ assert "junction.bst:elements/sources.yml: Cannot track source in a fragment from a junction" in result.stderr
else:
- assert os.path.exists(os.path.join(project, 'project.refs'))
+ assert os.path.exists(os.path.join(project, "project.refs"))
# And now fetch it: The Source has probably already cached the
# latest ref locally, but it is not required to have cached
# the associated content of the latest ref at track time, that
# is the job of fetch.
- result = cli.run(project=project, args=['source', 'fetch', element_name])
+ result = cli.run(project=project, args=["source", "fetch", element_name])
result.assert_success()
# Assert that we are now buildable because the source is
# now cached.
- assert cli.get_element_state(project, element_name) == 'buildable'
+ assert cli.get_element_state(project, element_name) == "buildable"
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("ref_storage", ['inline', 'project.refs'])
+@pytest.mark.parametrize("ref_storage", ["inline", "project.refs"])
def test_track_junction_included(cli, tmpdir, datafiles, ref_storage, kind):
project = str(datafiles)
- element_path = os.path.join(project, 'elements')
- subproject_path = os.path.join(project, 'files', 'sub-project')
- junction_path = os.path.join(element_path, 'junction.bst')
+ element_path = os.path.join(project, "elements")
+ subproject_path = os.path.join(project, "files", "sub-project")
+ junction_path = os.path.join(element_path, "junction.bst")
- update_project_configuration(project, {
- 'ref-storage': ref_storage,
- '(@)': ['junction.bst:test.yml']
- })
+ update_project_configuration(project, {"ref-storage": ref_storage, "(@)": ["junction.bst:test.yml"]})
- generate_junction(str(tmpdir.join('junction_repo')),
- subproject_path, junction_path, store_ref=False)
+ generate_junction(str(tmpdir.join("junction_repo")), subproject_path, junction_path, store_ref=False)
- result = cli.run(project=project, args=['source', 'track', 'junction.bst'])
+ result = cli.run(project=project, args=["source", "track", "junction.bst"])
result.assert_success()
diff --git a/src/buildstream/testing/_sourcetests/track_cross_junction.py b/src/buildstream/testing/_sourcetests/track_cross_junction.py
index 550f57faf..2477b37ee 100644
--- a/src/buildstream/testing/_sourcetests/track_cross_junction.py
+++ b/src/buildstream/testing/_sourcetests/track_cross_junction.py
@@ -32,32 +32,27 @@ from .utils import add_plugins_conf
# Project directory
TOP_DIR = os.path.dirname(os.path.realpath(__file__))
-DATA_DIR = os.path.join(TOP_DIR, 'project')
+DATA_DIR = os.path.join(TOP_DIR, "project")
def generate_element(repo, element_path, dep_name=None):
- element = {
- 'kind': 'import',
- 'sources': [
- repo.source_config()
- ]
- }
+ element = {"kind": "import", "sources": [repo.source_config()]}
if dep_name:
- element['depends'] = [dep_name]
+ element["depends"] = [dep_name]
_yaml.roundtrip_dump(element, element_path)
def generate_import_element(tmpdir, kind, project, name):
- element_name = 'import-{}.bst'.format(name)
- repo_element_path = os.path.join(project, 'elements', element_name)
+ element_name = "import-{}.bst".format(name)
+ repo_element_path = os.path.join(project, "elements", element_name)
files = str(tmpdir.join("imported_files_{}".format(name)))
os.makedirs(files)
- with open(os.path.join(files, '{}.txt'.format(name)), 'w') as f:
+ with open(os.path.join(files, "{}.txt".format(name)), "w") as f:
f.write(name)
- repo = create_repo(kind, str(tmpdir.join('element_{}_repo'.format(name))))
+ repo = create_repo(kind, str(tmpdir.join("element_{}_repo".format(name))))
repo.create(files)
generate_element(repo, repo_element_path)
@@ -69,28 +64,22 @@ def generate_project(tmpdir, name, kind, config=None):
if config is None:
config = {}
- project_name = 'project-{}'.format(name)
+ project_name = "project-{}".format(name)
subproject_path = os.path.join(str(tmpdir.join(project_name)))
- os.makedirs(os.path.join(subproject_path, 'elements'))
+ os.makedirs(os.path.join(subproject_path, "elements"))
- project_conf = {
- 'name': name,
- 'element-path': 'elements'
- }
+ project_conf = {"name": name, "element-path": "elements"}
project_conf.update(config)
- _yaml.roundtrip_dump(project_conf, os.path.join(subproject_path, 'project.conf'))
+ _yaml.roundtrip_dump(project_conf, os.path.join(subproject_path, "project.conf"))
add_plugins_conf(subproject_path, kind)
return project_name, subproject_path
def generate_simple_stack(project, name, dependencies):
- element_name = '{}.bst'.format(name)
- element_path = os.path.join(project, 'elements', element_name)
- element = {
- 'kind': 'stack',
- 'depends': dependencies
- }
+ element_name = "{}.bst".format(name)
+ element_path = os.path.join(project, "elements", element_name)
+ element = {"kind": "stack", "depends": dependencies}
_yaml.roundtrip_dump(element, element_path)
return element_name
@@ -98,11 +87,11 @@ def generate_simple_stack(project, name, dependencies):
def generate_cross_element(project, subproject_name, import_name):
basename, _ = os.path.splitext(import_name)
- return generate_simple_stack(project, 'import-{}-{}'.format(subproject_name, basename),
- [{
- 'junction': '{}.bst'.format(subproject_name),
- 'filename': import_name
- }])
+ return generate_simple_stack(
+ project,
+ "import-{}-{}".format(subproject_name, basename),
+ [{"junction": "{}.bst".format(subproject_name), "filename": import_name}],
+ )
@pytest.mark.parametrize("kind", ALL_REPO_KINDS.keys())
@@ -110,30 +99,30 @@ def test_cross_junction_multiple_projects(cli, tmpdir, kind):
tmpdir = tmpdir.join(kind)
# Generate 3 projects: main, a, b
- _, project = generate_project(tmpdir, 'main', kind, {'ref-storage': 'project.refs'})
- project_a, project_a_path = generate_project(tmpdir, 'a', kind)
- project_b, project_b_path = generate_project(tmpdir, 'b', kind)
+ _, project = generate_project(tmpdir, "main", kind, {"ref-storage": "project.refs"})
+ project_a, project_a_path = generate_project(tmpdir, "a", kind)
+ project_b, project_b_path = generate_project(tmpdir, "b", kind)
# Generate an element with a trackable source for each project
- element_a = generate_import_element(tmpdir, kind, project_a_path, 'a')
- element_b = generate_import_element(tmpdir, kind, project_b_path, 'b')
- element_c = generate_import_element(tmpdir, kind, project, 'c')
+ element_a = generate_import_element(tmpdir, kind, project_a_path, "a")
+ element_b = generate_import_element(tmpdir, kind, project_b_path, "b")
+ element_c = generate_import_element(tmpdir, kind, project, "c")
# Create some indirections to the elements with dependencies to test --deps
- stack_a = generate_simple_stack(project_a_path, 'stack-a', [element_a])
- stack_b = generate_simple_stack(project_b_path, 'stack-b', [element_b])
+ stack_a = generate_simple_stack(project_a_path, "stack-a", [element_a])
+ stack_b = generate_simple_stack(project_b_path, "stack-b", [element_b])
# Create junctions for projects a and b in main.
- junction_a = '{}.bst'.format(project_a)
- junction_a_path = os.path.join(project, 'elements', junction_a)
- generate_junction(tmpdir.join('repo_a'), project_a_path, junction_a_path, store_ref=False)
+ junction_a = "{}.bst".format(project_a)
+ junction_a_path = os.path.join(project, "elements", junction_a)
+ generate_junction(tmpdir.join("repo_a"), project_a_path, junction_a_path, store_ref=False)
- junction_b = '{}.bst'.format(project_b)
- junction_b_path = os.path.join(project, 'elements', junction_b)
- generate_junction(tmpdir.join('repo_b'), project_b_path, junction_b_path, store_ref=False)
+ junction_b = "{}.bst".format(project_b)
+ junction_b_path = os.path.join(project, "elements", junction_b)
+ generate_junction(tmpdir.join("repo_b"), project_b_path, junction_b_path, store_ref=False)
# Track the junctions.
- result = cli.run(project=project, args=['source', 'track', junction_a, junction_b])
+ result = cli.run(project=project, args=["source", "track", junction_a, junction_b])
result.assert_success()
# Import elements from a and b in to main.
@@ -141,18 +130,16 @@ def test_cross_junction_multiple_projects(cli, tmpdir, kind):
imported_b = generate_cross_element(project, project_b, stack_b)
# Generate a top level stack depending on everything
- all_bst = generate_simple_stack(project, 'all', [imported_a, imported_b, element_c])
+ all_bst = generate_simple_stack(project, "all", [imported_a, imported_b, element_c])
# Track without following junctions. But explicitly also track the elements in project a.
- result = cli.run(project=project, args=['source', 'track',
- '--deps', 'all',
- all_bst,
- '{}:{}'.format(junction_a, stack_a)])
+ result = cli.run(
+ project=project, args=["source", "track", "--deps", "all", all_bst, "{}:{}".format(junction_a, stack_a)]
+ )
result.assert_success()
# Elements in project b should not be tracked. But elements in project a and main should.
- expected = [element_c,
- '{}:{}'.format(junction_a, element_a)]
+ expected = [element_c, "{}:{}".format(junction_a, element_a)]
assert set(result.get_tracked_elements()) == set(expected)
@@ -160,31 +147,38 @@ def test_cross_junction_multiple_projects(cli, tmpdir, kind):
def test_track_exceptions(cli, tmpdir, kind):
tmpdir = tmpdir.join(kind)
- _, project = generate_project(tmpdir, 'main', kind, {'ref-storage': 'project.refs'})
- project_a, project_a_path = generate_project(tmpdir, 'a', kind)
+ _, project = generate_project(tmpdir, "main", kind, {"ref-storage": "project.refs"})
+ project_a, project_a_path = generate_project(tmpdir, "a", kind)
- element_a = generate_import_element(tmpdir, kind, project_a_path, 'a')
- element_b = generate_import_element(tmpdir, kind, project_a_path, 'b')
+ element_a = generate_import_element(tmpdir, kind, project_a_path, "a")
+ element_b = generate_import_element(tmpdir, kind, project_a_path, "b")
- all_bst = generate_simple_stack(project_a_path, 'all', [element_a,
- element_b])
+ all_bst = generate_simple_stack(project_a_path, "all", [element_a, element_b])
- junction_a = '{}.bst'.format(project_a)
- junction_a_path = os.path.join(project, 'elements', junction_a)
- generate_junction(tmpdir.join('repo_a'), project_a_path, junction_a_path, store_ref=False)
+ junction_a = "{}.bst".format(project_a)
+ junction_a_path = os.path.join(project, "elements", junction_a)
+ generate_junction(tmpdir.join("repo_a"), project_a_path, junction_a_path, store_ref=False)
- result = cli.run(project=project, args=['source', 'track', junction_a])
+ result = cli.run(project=project, args=["source", "track", junction_a])
result.assert_success()
imported_b = generate_cross_element(project, project_a, element_b)
- indirection = generate_simple_stack(project, 'indirection', [imported_b])
-
- result = cli.run(project=project,
- args=['source', 'track', '--deps', 'all',
- '--except', indirection,
- '{}:{}'.format(junction_a, all_bst), imported_b])
+ indirection = generate_simple_stack(project, "indirection", [imported_b])
+
+ result = cli.run(
+ project=project,
+ args=[
+ "source",
+ "track",
+ "--deps",
+ "all",
+ "--except",
+ indirection,
+ "{}:{}".format(junction_a, all_bst),
+ imported_b,
+ ],
+ )
result.assert_success()
- expected = ['{}:{}'.format(junction_a, element_a),
- '{}:{}'.format(junction_a, element_b)]
+ expected = ["{}:{}".format(junction_a, element_a), "{}:{}".format(junction_a, element_b)]
assert set(result.get_tracked_elements()) == set(expected)
diff --git a/src/buildstream/testing/_sourcetests/utils.py b/src/buildstream/testing/_sourcetests/utils.py
index a0e65b4f4..116506807 100644
--- a/src/buildstream/testing/_sourcetests/utils.py
+++ b/src/buildstream/testing/_sourcetests/utils.py
@@ -27,9 +27,8 @@ import os
try:
import pytest
except ImportError:
- module_name = globals()['__name__']
- msg = "Could not import pytest:\n" \
- "To use the {} module, you must have pytest installed.".format(module_name)
+ module_name = globals()["__name__"]
+ msg = "Could not import pytest:\n" "To use the {} module, you must have pytest installed.".format(module_name)
raise ImportError(msg)
from buildstream import _yaml
@@ -72,13 +71,7 @@ def add_plugins_conf(project, plugin_kind):
if plugin_package is not None:
project_conf["plugins"] = [
- {
- "origin": "pip",
- "package-name": plugin_package,
- "sources": {
- plugin_kind: 0,
- },
- },
+ {"origin": "pip", "package-name": plugin_package, "sources": {plugin_kind: 0,},},
]
_yaml.roundtrip_dump(project_conf, project_conf_file)
@@ -96,7 +89,7 @@ def add_plugins_conf(project, plugin_kind):
# updated_configuration (dict): configuration to merge into the existing one
#
def update_project_configuration(project_path, updated_configuration):
- project_conf_path = os.path.join(project_path, 'project.conf')
+ project_conf_path = os.path.join(project_path, "project.conf")
project_conf = _yaml.roundtrip_load(project_conf_path)
project_conf.update(updated_configuration)
diff --git a/src/buildstream/testing/_sourcetests/workspace.py b/src/buildstream/testing/_sourcetests/workspace.py
index dd7977e76..34e2247ea 100644
--- a/src/buildstream/testing/_sourcetests/workspace.py
+++ b/src/buildstream/testing/_sourcetests/workspace.py
@@ -30,10 +30,10 @@ from .utils import kind # pylint: disable=unused-import
# Project directory
TOP_DIR = os.path.dirname(os.path.realpath(__file__))
-DATA_DIR = os.path.join(TOP_DIR, 'project')
+DATA_DIR = os.path.join(TOP_DIR, "project")
-class WorkspaceCreator():
+class WorkspaceCreator:
def __init__(self, cli, tmpdir, datafiles, project_path=None):
self.cli = cli
self.tmpdir = tmpdir
@@ -45,17 +45,16 @@ class WorkspaceCreator():
shutil.copytree(str(datafiles), project_path)
self.project_path = project_path
- self.bin_files_path = os.path.join(project_path, 'files', 'bin-files')
+ self.bin_files_path = os.path.join(project_path, "files", "bin-files")
- self.workspace_cmd = os.path.join(self.project_path, 'workspace_cmd')
+ self.workspace_cmd = os.path.join(self.project_path, "workspace_cmd")
- def create_workspace_element(self, kind, track, suffix='', workspace_dir=None,
- element_attrs=None):
- element_name = 'workspace-test-{}{}.bst'.format(kind, suffix)
- element_path = os.path.join(self.project_path, 'elements')
+ def create_workspace_element(self, kind, track, suffix="", workspace_dir=None, element_attrs=None):
+ element_name = "workspace-test-{}{}.bst".format(kind, suffix)
+ element_path = os.path.join(self.project_path, "elements")
if not workspace_dir:
workspace_dir = os.path.join(self.workspace_cmd, element_name)
- if workspace_dir[-4:] == '.bst':
+ if workspace_dir[-4:] == ".bst":
workspace_dir = workspace_dir[:-4]
# Create our repo object of the given source type with
@@ -66,64 +65,53 @@ class WorkspaceCreator():
ref = None
# Write out our test target
- element = {
- 'kind': 'import',
- 'sources': [
- repo.source_config(ref=ref)
- ]
- }
+ element = {"kind": "import", "sources": [repo.source_config(ref=ref)]}
if element_attrs:
element = {**element, **element_attrs}
- _yaml.roundtrip_dump(element,
- os.path.join(element_path, element_name))
+ _yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
return element_name, element_path, workspace_dir
- def create_workspace_elements(self, kinds, track, suffixs=None, workspace_dir_usr=None,
- element_attrs=None):
+ def create_workspace_elements(self, kinds, track, suffixs=None, workspace_dir_usr=None, element_attrs=None):
element_tuples = []
if suffixs is None:
- suffixs = ['', ] * len(kinds)
+ suffixs = ["",] * len(kinds)
else:
if len(suffixs) != len(kinds):
raise "terable error"
for suffix, kind in zip(suffixs, kinds):
- element_name, _, workspace_dir = \
- self.create_workspace_element(kind, track, suffix, workspace_dir_usr,
- element_attrs)
+ element_name, _, workspace_dir = self.create_workspace_element(
+ kind, track, suffix, workspace_dir_usr, element_attrs
+ )
element_tuples.append((element_name, workspace_dir))
# Assert that there is no reference, a track & fetch is needed
- states = self.cli.get_element_states(self.project_path, [
- e for e, _ in element_tuples
- ])
+ states = self.cli.get_element_states(self.project_path, [e for e, _ in element_tuples])
if track:
- assert not any(states[e] != 'no reference' for e, _ in element_tuples)
+ assert not any(states[e] != "no reference" for e, _ in element_tuples)
else:
- assert not any(states[e] != 'fetch needed' for e, _ in element_tuples)
+ assert not any(states[e] != "fetch needed" for e, _ in element_tuples)
return element_tuples
- def open_workspaces(self, kinds, track, suffixs=None, workspace_dir=None,
- element_attrs=None, no_checkout=False):
+ def open_workspaces(self, kinds, track, suffixs=None, workspace_dir=None, element_attrs=None, no_checkout=False):
- element_tuples = self.create_workspace_elements(kinds, track, suffixs, workspace_dir,
- element_attrs)
+ element_tuples = self.create_workspace_elements(kinds, track, suffixs, workspace_dir, element_attrs)
os.makedirs(self.workspace_cmd, exist_ok=True)
# Now open the workspace, this should have the effect of automatically
# tracking & fetching the source from the repo.
- args = ['workspace', 'open']
+ args = ["workspace", "open"]
if track:
- args.append('--track')
+ args.append("--track")
if no_checkout:
- args.append('--no-checkout')
+ args.append("--no-checkout")
if workspace_dir is not None:
assert len(element_tuples) == 1, "test logic error"
_, workspace_dir = element_tuples[0]
- args.extend(['--directory', workspace_dir])
+ args.extend(["--directory", workspace_dir])
args.extend([element_name for element_name, workspace_dir_suffix in element_tuples])
result = self.cli.run(cwd=self.workspace_cmd, project=self.project_path, args=args)
@@ -132,24 +120,31 @@ class WorkspaceCreator():
if not no_checkout:
# Assert that we are now buildable because the source is now cached.
- states = self.cli.get_element_states(self.project_path, [
- e for e, _ in element_tuples
- ])
- assert not any(states[e] != 'buildable' for e, _ in element_tuples)
+ states = self.cli.get_element_states(self.project_path, [e for e, _ in element_tuples])
+ assert not any(states[e] != "buildable" for e, _ in element_tuples)
# Check that the executable hello file is found in each workspace
for _, workspace in element_tuples:
- filename = os.path.join(workspace, 'usr', 'bin', 'hello')
+ filename = os.path.join(workspace, "usr", "bin", "hello")
assert os.path.exists(filename)
return element_tuples
-def open_workspace(cli, tmpdir, datafiles, kind, track, suffix='', workspace_dir=None,
- project_path=None, element_attrs=None, no_checkout=False):
+def open_workspace(
+ cli,
+ tmpdir,
+ datafiles,
+ kind,
+ track,
+ suffix="",
+ workspace_dir=None,
+ project_path=None,
+ element_attrs=None,
+ no_checkout=False,
+):
workspace_object = WorkspaceCreator(cli, tmpdir, datafiles, project_path)
- workspaces = workspace_object.open_workspaces((kind, ), track, (suffix, ), workspace_dir,
- element_attrs, no_checkout)
+ workspaces = workspace_object.open_workspaces((kind,), track, (suffix,), workspace_dir, element_attrs, no_checkout)
assert len(workspaces) == 1
element_name, workspace = workspaces[0]
return element_name, workspace_object.project_path, workspace
diff --git a/src/buildstream/testing/_utils/junction.py b/src/buildstream/testing/_utils/junction.py
index 98d23b0a2..cfc5898a9 100644
--- a/src/buildstream/testing/_utils/junction.py
+++ b/src/buildstream/testing/_utils/junction.py
@@ -28,12 +28,7 @@ def generate_junction(tmpdir, subproject_path, junction_path, *, store_ref=True)
if not store_ref:
source_ref = None
- element = {
- 'kind': 'junction',
- 'sources': [
- repo.source_config(ref=source_ref)
- ]
- }
+ element = {"kind": "junction", "sources": [repo.source_config(ref=source_ref)]}
_yaml.roundtrip_dump(element, junction_path)
return ref
@@ -41,46 +36,38 @@ def generate_junction(tmpdir, subproject_path, junction_path, *, store_ref=True)
# A barebones Git Repo class to use for generating junctions
class _SimpleGit(Repo):
- def __init__(self, directory, subdir='repo'):
+ def __init__(self, directory, subdir="repo"):
if not HAVE_GIT:
- pytest.skip('git is not available')
+ pytest.skip("git is not available")
super().__init__(directory, subdir)
def create(self, directory):
self.copy_directory(directory, self.repo)
- self._run_git('init', '.')
- self._run_git('add', '.')
- self._run_git('commit', '-m', 'Initial commit')
+ self._run_git("init", ".")
+ self._run_git("add", ".")
+ self._run_git("commit", "-m", "Initial commit")
return self.latest_commit()
def latest_commit(self):
- return self._run_git(
- 'rev-parse', 'HEAD',
- stdout=subprocess.PIPE,
- universal_newlines=True,
- ).stdout.strip()
+ return self._run_git("rev-parse", "HEAD", stdout=subprocess.PIPE, universal_newlines=True,).stdout.strip()
def source_config(self, ref=None):
return self.source_config_extra(ref)
def source_config_extra(self, ref=None, checkout_submodules=None):
- config = {
- 'kind': 'git',
- 'url': 'file://' + self.repo,
- 'track': 'master'
- }
+ config = {"kind": "git", "url": "file://" + self.repo, "track": "master"}
if ref is not None:
- config['ref'] = ref
+ config["ref"] = ref
if checkout_submodules is not None:
- config['checkout-submodules'] = checkout_submodules
+ config["checkout-submodules"] = checkout_submodules
return config
def _run_git(self, *args, **kwargs):
argv = [GIT]
argv.extend(args)
- if 'env' not in kwargs:
- kwargs['env'] = dict(GIT_ENV, PWD=self.repo)
- kwargs.setdefault('cwd', self.repo)
- kwargs.setdefault('check', True)
+ if "env" not in kwargs:
+ kwargs["env"] = dict(GIT_ENV, PWD=self.repo)
+ kwargs.setdefault("cwd", self.repo)
+ kwargs.setdefault("check", True)
return subprocess.run(argv, **kwargs)
diff --git a/src/buildstream/testing/_utils/site.py b/src/buildstream/testing/_utils/site.py
index ca74d9505..953d21607 100644
--- a/src/buildstream/testing/_utils/site.py
+++ b/src/buildstream/testing/_utils/site.py
@@ -5,29 +5,29 @@ import os
import subprocess
import sys
import platform
-from typing import Optional # pylint: disable=unused-import
+from typing import Optional # pylint: disable=unused-import
from buildstream import _site, utils, ProgramNotFoundError
from buildstream._platform import Platform
try:
- GIT = utils.get_host_tool('git') # type: Optional[str]
+ GIT = utils.get_host_tool("git") # type: Optional[str]
HAVE_GIT = True
- out = str(subprocess.check_output(['git', '--version']), "utf-8")
+ out = str(subprocess.check_output(["git", "--version"]), "utf-8")
# e.g. on Git for Windows we get "git version 2.21.0.windows.1".
# e.g. on Mac via Homebrew we get "git version 2.19.0".
- version = tuple(int(x) for x in out.split(' ')[2].split('.')[:3])
+ version = tuple(int(x) for x in out.split(" ")[2].split(".")[:3])
HAVE_OLD_GIT = version < (1, 8, 5)
GIT_ENV = {
- 'GIT_AUTHOR_DATE': '1320966000 +0200',
- 'GIT_AUTHOR_NAME': 'tomjon',
- 'GIT_AUTHOR_EMAIL': 'tom@jon.com',
- 'GIT_COMMITTER_DATE': '1320966000 +0200',
- 'GIT_COMMITTER_NAME': 'tomjon',
- 'GIT_COMMITTER_EMAIL': 'tom@jon.com'
+ "GIT_AUTHOR_DATE": "1320966000 +0200",
+ "GIT_AUTHOR_NAME": "tomjon",
+ "GIT_AUTHOR_EMAIL": "tom@jon.com",
+ "GIT_COMMITTER_DATE": "1320966000 +0200",
+ "GIT_COMMITTER_NAME": "tomjon",
+ "GIT_COMMITTER_EMAIL": "tom@jon.com",
}
except ProgramNotFoundError:
GIT = None
@@ -36,18 +36,16 @@ except ProgramNotFoundError:
GIT_ENV = dict()
try:
- BZR = utils.get_host_tool('bzr') # type: Optional[str]
+ BZR = utils.get_host_tool("bzr") # type: Optional[str]
HAVE_BZR = True
- BZR_ENV = {
- "BZR_EMAIL": "Testy McTesterson <testy.mctesterson@example.com>"
- }
+ BZR_ENV = {"BZR_EMAIL": "Testy McTesterson <testy.mctesterson@example.com>"}
except ProgramNotFoundError:
BZR = None
HAVE_BZR = False
BZR_ENV = {}
try:
- utils.get_host_tool('bwrap')
+ utils.get_host_tool("bwrap")
HAVE_BWRAP = True
HAVE_BWRAP_JSON_STATUS = _site.get_bwrap_version() >= (0, 3, 2)
except ProgramNotFoundError:
@@ -55,32 +53,33 @@ except ProgramNotFoundError:
HAVE_BWRAP_JSON_STATUS = False
try:
- utils.get_host_tool('lzip')
+ utils.get_host_tool("lzip")
HAVE_LZIP = True
except ProgramNotFoundError:
HAVE_LZIP = False
try:
import arpy # pylint: disable=unused-import
+
HAVE_ARPY = True
except ImportError:
HAVE_ARPY = False
try:
- utils.get_host_tool('buildbox')
+ utils.get_host_tool("buildbox")
HAVE_BUILDBOX = True
except ProgramNotFoundError:
HAVE_BUILDBOX = False
-IS_LINUX = os.getenv('BST_FORCE_BACKEND', sys.platform).startswith('linux')
-IS_WSL = (IS_LINUX and 'Microsoft' in platform.uname().release)
-IS_WINDOWS = (os.name == 'nt')
+IS_LINUX = os.getenv("BST_FORCE_BACKEND", sys.platform).startswith("linux")
+IS_WSL = IS_LINUX and "Microsoft" in platform.uname().release
+IS_WINDOWS = os.name == "nt"
MACHINE_ARCH = Platform.get_host_arch()
-HAVE_SANDBOX = os.getenv('BST_FORCE_SANDBOX')
+HAVE_SANDBOX = os.getenv("BST_FORCE_SANDBOX")
if HAVE_SANDBOX is not None:
pass
elif IS_LINUX and HAVE_BWRAP and (not IS_WSL):
- HAVE_SANDBOX = 'bwrap'
+ HAVE_SANDBOX = "bwrap"
diff --git a/src/buildstream/testing/integration.py b/src/buildstream/testing/integration.py
index 01635de74..584d7da1b 100644
--- a/src/buildstream/testing/integration.py
+++ b/src/buildstream/testing/integration.py
@@ -39,11 +39,11 @@ def walk_dir(root):
# print path to all subdirectories first.
for subdirname in dirnames:
- yield os.path.join(dirname, subdirname)[len(root):]
+ yield os.path.join(dirname, subdirname)[len(root) :]
# print path to all filenames.
for filename in filenames:
- yield os.path.join(dirname, filename)[len(root):]
+ yield os.path.join(dirname, filename)[len(root) :]
# Ensure that a directory contains the given filenames.
@@ -51,35 +51,33 @@ def assert_contains(directory, expected):
missing = set(expected)
missing.difference_update(walk_dir(directory))
if missing:
- raise AssertionError("Missing {} expected elements from list: {}"
- .format(len(missing), missing))
+ raise AssertionError("Missing {} expected elements from list: {}".format(len(missing), missing))
class IntegrationCache:
-
def __init__(self, cache):
self.root = os.path.abspath(cache)
os.makedirs(cache, exist_ok=True)
# Use the same sources every time
- self.sources = os.path.join(self.root, 'sources')
+ self.sources = os.path.join(self.root, "sources")
# Create a temp directory for the duration of the test for
# the artifacts directory
try:
- self.cachedir = tempfile.mkdtemp(dir=self.root, prefix='cache-')
+ self.cachedir = tempfile.mkdtemp(dir=self.root, prefix="cache-")
except OSError as e:
raise AssertionError("Unable to create test directory !") from e
-@pytest.fixture(scope='session')
+@pytest.fixture(scope="session")
def integration_cache(request):
# Set the cache dir to the INTEGRATION_CACHE variable, or the
# default if that is not set.
- if 'INTEGRATION_CACHE' in os.environ:
- cache_dir = os.environ['INTEGRATION_CACHE']
+ if "INTEGRATION_CACHE" in os.environ:
+ cache_dir = os.environ["INTEGRATION_CACHE"]
else:
- cache_dir = os.path.abspath('./integration-cache')
+ cache_dir = os.path.abspath("./integration-cache")
cache = IntegrationCache(cache_dir)
@@ -92,6 +90,6 @@ def integration_cache(request):
except FileNotFoundError:
pass
try:
- shutil.rmtree(os.path.join(cache.root, 'cas'))
+ shutil.rmtree(os.path.join(cache.root, "cas"))
except FileNotFoundError:
pass
diff --git a/src/buildstream/testing/repo.py b/src/buildstream/testing/repo.py
index c1538685d..1b46ec806 100644
--- a/src/buildstream/testing/repo.py
+++ b/src/buildstream/testing/repo.py
@@ -25,7 +25,7 @@ import os
import shutil
-class Repo():
+class Repo:
"""Repo()
Abstract class providing scaffolding for generating data to be
@@ -38,7 +38,8 @@ class Repo():
subdir (str): The subdir for the repo, in case there is more than one
"""
- def __init__(self, directory, subdir='repo'):
+
+ def __init__(self, directory, subdir="repo"):
# The working directory for the repo object
#
@@ -100,7 +101,7 @@ class Repo():
Returns:
(Repo): A Repo object for the new repository.
"""
- subdir = self.repo[len(self.directory):].lstrip(os.sep)
+ subdir = self.repo[len(self.directory) :].lstrip(os.sep)
new_dir = os.path.join(dest, subdir)
os.makedirs(new_dir, exist_ok=True)
self.copy_directory(self.repo, new_dir)
diff --git a/src/buildstream/testing/runcli.py b/src/buildstream/testing/runcli.py
index 36426c8af..d4f7e2ffd 100644
--- a/src/buildstream/testing/runcli.py
+++ b/src/buildstream/testing/runcli.py
@@ -61,14 +61,8 @@ from buildstream._protos.buildstream.v2 import artifact_pb2
# Wrapper for the click.testing result
-class Result():
-
- def __init__(self,
- exit_code=None,
- exception=None,
- exc_info=None,
- output=None,
- stderr=None):
+class Result:
+ def __init__(self, exit_code=None, exception=None, exc_info=None, output=None, stderr=None):
self.exit_code = exit_code
self.exc = exception
self.exc_info = exc_info
@@ -94,8 +88,7 @@ class Result():
self.unhandled_exception = True
self.exception = get_last_exception()
- self.task_error_domain, \
- self.task_error_reason = get_last_task_error()
+ self.task_error_domain, self.task_error_reason = get_last_task_error()
else:
self.exception = None
self.task_error_domain = None
@@ -111,7 +104,7 @@ class Result():
# Raises:
# (AssertionError): If the session did not complete successfully
#
- def assert_success(self, fail_message=''):
+ def assert_success(self, fail_message=""):
assert self.exit_code == 0, fail_message
assert self.exc is None, fail_message
assert self.exception is None, fail_message
@@ -131,11 +124,7 @@ class Result():
# Raises:
# (AssertionError): If any of the assertions fail
#
- def assert_main_error(self,
- error_domain,
- error_reason,
- fail_message='',
- *, debug=False):
+ def assert_main_error(self, error_domain, error_reason, fail_message="", *, debug=False):
if debug:
print(
"""
@@ -144,11 +133,9 @@ class Result():
Domain: {}
Reason: {}
""".format(
- self.exit_code,
- self.exception,
- self.exception.domain,
- self.exception.reason
- ))
+ self.exit_code, self.exception, self.exception.domain, self.exception.reason
+ )
+ )
assert self.exit_code == -1, fail_message
assert self.exc is not None, fail_message
assert self.exception is not None, fail_message
@@ -172,10 +159,7 @@ class Result():
# Raises:
# (AssertionError): If any of the assertions fail
#
- def assert_task_error(self,
- error_domain,
- error_reason,
- fail_message=''):
+ def assert_task_error(self, error_domain, error_reason, fail_message=""):
assert self.exit_code == -1, fail_message
assert self.exc is not None, fail_message
@@ -197,7 +181,7 @@ class Result():
# Raises:
# (AssertionError): If any of the assertions fail
#
- def assert_shell_error(self, fail_message=''):
+ def assert_shell_error(self, fail_message=""):
assert self.exit_code == 1, fail_message
# get_start_order()
@@ -212,7 +196,7 @@ class Result():
# (list): A list of element names in the order which they first appeared in the result
#
def get_start_order(self, activity):
- results = re.findall(r'\[\s*{}:(\S+)\s*\]\s*START\s*.*\.log'.format(activity), self.stderr)
+ results = re.findall(r"\[\s*{}:(\S+)\s*\]\s*START\s*.*\.log".format(activity), self.stderr)
if results is None:
return []
return list(results)
@@ -228,29 +212,28 @@ class Result():
# (list): A list of element names
#
def get_tracked_elements(self):
- tracked = re.findall(r'\[\s*track:(\S+)\s*]', self.stderr)
+ tracked = re.findall(r"\[\s*track:(\S+)\s*]", self.stderr)
if tracked is None:
return []
return list(tracked)
def get_pushed_elements(self):
- pushed = re.findall(r'\[\s*push:(\S+)\s*\]\s*INFO\s*Pushed artifact', self.stderr)
+ pushed = re.findall(r"\[\s*push:(\S+)\s*\]\s*INFO\s*Pushed artifact", self.stderr)
if pushed is None:
return []
return list(pushed)
def get_pulled_elements(self):
- pulled = re.findall(r'\[\s*pull:(\S+)\s*\]\s*INFO\s*Pulled artifact', self.stderr)
+ pulled = re.findall(r"\[\s*pull:(\S+)\s*\]\s*INFO\s*Pulled artifact", self.stderr)
if pulled is None:
return []
return list(pulled)
-class Cli():
-
+class Cli:
def __init__(self, directory, verbose=True, default_options=None):
self.directory = directory
self.config = None
@@ -286,14 +269,13 @@ class Cli():
# element_name (str): The name of the element artifact
# cache_dir (str): Specific cache dir to remove artifact from
#
- def remove_artifact_from_cache(self, project, element_name,
- *, cache_dir=None):
+ def remove_artifact_from_cache(self, project, element_name, *, cache_dir=None):
# Read configuration to figure out where artifacts are stored
if not cache_dir:
- default = os.path.join(project, 'cache')
+ default = os.path.join(project, "cache")
if self.config is not None:
- cache_dir = self.config.get('cachedir', default)
+ cache_dir = self.config.get("cachedir", default)
else:
cache_dir = default
@@ -313,8 +295,17 @@ class Cli():
# args (list): A list of arguments to pass buildstream
# binary_capture (bool): Whether to capture the stdout/stderr as binary
#
- def run(self, configure=True, project=None, silent=False, env=None,
- cwd=None, options=None, args=None, binary_capture=False):
+ def run(
+ self,
+ configure=True,
+ project=None,
+ silent=False,
+ env=None,
+ cwd=None,
+ options=None,
+ args=None,
+ binary_capture=False,
+ ):
# We don't want to carry the state of one bst invocation into another
# bst invocation. Since node _FileInfo objects hold onto BuildStream
@@ -335,22 +326,20 @@ class Cli():
options = self.default_options + options
with ExitStack() as stack:
- bst_args = ['--no-colors']
+ bst_args = ["--no-colors"]
if silent:
- bst_args += ['--no-verbose']
+ bst_args += ["--no-verbose"]
if configure:
- config_file = stack.enter_context(
- configured(self.directory, self.config)
- )
- bst_args += ['--config', config_file]
+ config_file = stack.enter_context(configured(self.directory, self.config))
+ bst_args += ["--config", config_file]
if project:
- bst_args += ['--directory', str(project)]
+ bst_args += ["--directory", str(project)]
for option, value in options:
- bst_args += ['--option', option, value]
+ bst_args += ["--option", option, value]
bst_args += args
@@ -366,15 +355,14 @@ class Cli():
try:
sys.__stdout__.fileno()
except ValueError:
- sys.__stdout__ = open('/dev/stdout', 'w')
+ sys.__stdout__ = open("/dev/stdout", "w")
result = self._invoke(bst_cli, bst_args, binary_capture=binary_capture)
# Some informative stdout we can observe when anything fails
if self.verbose:
command = "bst " + " ".join(bst_args)
- print("BuildStream exited with code {} for invocation:\n\t{}"
- .format(result.exit_code, command))
+ print("BuildStream exited with code {} for invocation:\n\t{}".format(result.exit_code, command))
if result.output:
print("Program output was:\n{}".format(result.output))
if result.stderr:
@@ -409,9 +397,9 @@ class Cli():
exit_code = e.code
if not isinstance(exit_code, int):
- sys.stdout.write('Program exit code was not an integer: ')
+ sys.stdout.write("Program exit code was not an integer: ")
sys.stdout.write(str(exit_code))
- sys.stdout.write('\n')
+ sys.stdout.write("\n")
exit_code = 1
except Exception as e: # pylint: disable=broad-except
exception = e
@@ -424,11 +412,7 @@ class Cli():
out, err = capture.readouterr()
capture.stop_capturing()
- return Result(exit_code=exit_code,
- exception=exception,
- exc_info=exc_info,
- output=out,
- stderr=err)
+ return Result(exit_code=exit_code, exception=exception, exc_info=exc_info, output=out, stderr=err)
# Fetch an element state by name by
# invoking bst show on the project with the CLI
@@ -437,12 +421,9 @@ class Cli():
# then use get_element_states(s) instead.
#
def get_element_state(self, project, element_name):
- result = self.run(project=project, silent=True, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{state}',
- element_name
- ])
+ result = self.run(
+ project=project, silent=True, args=["show", "--deps", "none", "--format", "%{state}", element_name]
+ )
result.assert_success()
return result.output.strip()
@@ -450,18 +431,15 @@ class Cli():
#
# Returns a dictionary with the element names as keys
#
- def get_element_states(self, project, targets, deps='all'):
- result = self.run(project=project, silent=True, args=[
- 'show',
- '--deps', deps,
- '--format', '%{name}||%{state}',
- *targets
- ])
+ def get_element_states(self, project, targets, deps="all"):
+ result = self.run(
+ project=project, silent=True, args=["show", "--deps", deps, "--format", "%{name}||%{state}", *targets]
+ )
result.assert_success()
lines = result.output.splitlines()
states = {}
for line in lines:
- split = line.split(sep='||')
+ split = line.split(sep="||")
states[split[0]] = split[1]
return states
@@ -469,24 +447,18 @@ class Cli():
# on the project with the CLI
#
def get_element_key(self, project, element_name):
- result = self.run(project=project, silent=True, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{full-key}',
- element_name
- ])
+ result = self.run(
+ project=project, silent=True, args=["show", "--deps", "none", "--format", "%{full-key}", element_name]
+ )
result.assert_success()
return result.output.strip()
# Get the decoded config of an element.
#
def get_element_config(self, project, element_name):
- result = self.run(project=project, silent=True, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{config}',
- element_name
- ])
+ result = self.run(
+ project=project, silent=True, args=["show", "--deps", "none", "--format", "%{config}", element_name]
+ )
result.assert_success()
return yaml.safe_load(result.output)
@@ -494,12 +466,12 @@ class Cli():
# Fetch the elements that would be in the pipeline with the given
# arguments.
#
- def get_pipeline(self, project, elements, except_=None, scope='plan'):
+ def get_pipeline(self, project, elements, except_=None, scope="plan"):
if except_ is None:
except_ = []
- args = ['show', '--deps', scope, '--format', '%{name}']
- args += list(itertools.chain.from_iterable(zip(itertools.repeat('--except'), except_)))
+ args = ["show", "--deps", scope, "--format", "%{name}"]
+ args += list(itertools.chain.from_iterable(zip(itertools.repeat("--except"), except_)))
result = self.run(project=project, silent=True, args=args + elements)
result.assert_success()
@@ -523,11 +495,27 @@ class CliIntegration(Cli):
#
# This supports the same arguments as Cli.run(), see run_project_config().
#
- def run(self, configure=True, project=None, silent=False, env=None,
- cwd=None, options=None, args=None, binary_capture=False):
+ def run(
+ self,
+ configure=True,
+ project=None,
+ silent=False,
+ env=None,
+ cwd=None,
+ options=None,
+ args=None,
+ binary_capture=False,
+ ):
return self.run_project_config(
- configure=configure, project=project, silent=silent, env=env,
- cwd=cwd, options=options, args=args, binary_capture=binary_capture)
+ configure=configure,
+ project=project,
+ silent=silent,
+ env=env,
+ cwd=cwd,
+ options=options,
+ args=args,
+ binary_capture=binary_capture,
+ )
# run_project_config()
#
@@ -549,9 +537,9 @@ class CliIntegration(Cli):
# Save the original project.conf, because we will run more than
# once in the same temp directory
#
- project_directory = kwargs['project']
- project_filename = os.path.join(project_directory, 'project.conf')
- project_backup = os.path.join(project_directory, 'project.conf.backup')
+ project_directory = kwargs["project"]
+ project_filename = os.path.join(project_directory, "project.conf")
+ project_backup = os.path.join(project_directory, "project.conf.backup")
project_load_filename = project_filename
if not os.path.exists(project_backup):
@@ -576,8 +564,8 @@ class CliIntegration(Cli):
#
with tempfile.TemporaryDirectory(dir=project_directory) as scratchdir:
- temp_project = os.path.join(scratchdir, 'project.conf')
- with open(temp_project, 'w') as f:
+ temp_project = os.path.join(scratchdir, "project.conf")
+ with open(temp_project, "w") as f:
yaml.safe_dump(project_config, f)
project_config = _yaml.load(temp_project)
@@ -589,7 +577,7 @@ class CliIntegration(Cli):
else:
# Otherwise, just dump it as is
- with open(project_filename, 'w') as f:
+ with open(project_filename, "w") as f:
f.write(config)
return super().run(**kwargs)
@@ -611,50 +599,49 @@ class CliRemote(CliIntegration):
#
# Returns a list of configured services (by names).
#
- def ensure_services(self, actions=True, execution=True, storage=True,
- artifacts=False, sources=False):
+ def ensure_services(self, actions=True, execution=True, storage=True, artifacts=False, sources=False):
# Build a list of configured services by name:
configured_services = []
if not self.config:
return configured_services
- if 'remote-execution' in self.config:
- rexec_config = self.config['remote-execution']
+ if "remote-execution" in self.config:
+ rexec_config = self.config["remote-execution"]
- if 'action-cache-service' in rexec_config:
+ if "action-cache-service" in rexec_config:
if actions:
- configured_services.append('action-cache')
+ configured_services.append("action-cache")
else:
- rexec_config.pop('action-cache-service')
+ rexec_config.pop("action-cache-service")
- if 'execution-service' in rexec_config:
+ if "execution-service" in rexec_config:
if execution:
- configured_services.append('execution')
+ configured_services.append("execution")
else:
- rexec_config.pop('execution-service')
+ rexec_config.pop("execution-service")
- if 'storage-service' in rexec_config:
+ if "storage-service" in rexec_config:
if storage:
- configured_services.append('storage')
+ configured_services.append("storage")
else:
- rexec_config.pop('storage-service')
+ rexec_config.pop("storage-service")
- if 'artifacts' in self.config:
+ if "artifacts" in self.config:
if artifacts:
- configured_services.append('artifact-cache')
+ configured_services.append("artifact-cache")
else:
- self.config.pop('artifacts')
+ self.config.pop("artifacts")
- if 'source-caches' in self.config:
+ if "source-caches" in self.config:
if sources:
- configured_services.append('source-cache')
+ configured_services.append("source-cache")
else:
- self.config.pop('source-caches')
+ self.config.pop("source-caches")
return configured_services
-class TestArtifact():
+class TestArtifact:
# remove_artifact_from_cache():
#
@@ -666,10 +653,10 @@ class TestArtifact():
#
def remove_artifact_from_cache(self, cache_dir, element_name):
- cache_dir = os.path.join(cache_dir, 'artifacts', 'refs')
+ cache_dir = os.path.join(cache_dir, "artifacts", "refs")
- normal_name = element_name.replace(os.sep, '-')
- cache_dir = os.path.splitext(os.path.join(cache_dir, 'test', normal_name))[0]
+ normal_name = element_name.replace(os.sep, "-")
+ cache_dir = os.path.splitext(os.path.join(cache_dir, "test", normal_name))[0]
shutil.rmtree(cache_dir)
# is_cached():
@@ -688,7 +675,7 @@ class TestArtifact():
# cas = CASCache(str(cache_dir))
artifact_ref = element.get_artifact_name(element_key)
- return os.path.exists(os.path.join(cache_dir, 'artifacts', 'refs', artifact_ref))
+ return os.path.exists(os.path.join(cache_dir, "artifacts", "refs", artifact_ref))
# get_digest():
#
@@ -705,9 +692,9 @@ class TestArtifact():
def get_digest(self, cache_dir, element, element_key):
artifact_ref = element.get_artifact_name(element_key)
- artifact_dir = os.path.join(cache_dir, 'artifacts', 'refs')
+ artifact_dir = os.path.join(cache_dir, "artifacts", "refs")
artifact_proto = artifact_pb2.Artifact()
- with open(os.path.join(artifact_dir, artifact_ref), 'rb') as f:
+ with open(os.path.join(artifact_dir, artifact_ref), "rb") as f:
artifact_proto.ParseFromString(f.read())
return artifact_proto.files
@@ -727,7 +714,7 @@ class TestArtifact():
def extract_buildtree(self, cache_dir, tmpdir, ref):
artifact = artifact_pb2.Artifact()
try:
- with open(os.path.join(cache_dir, 'artifacts', 'refs', ref), 'rb') as f:
+ with open(os.path.join(cache_dir, "artifacts", "refs", ref), "rb") as f:
artifact.ParseFromString(f.read())
except FileNotFoundError:
yield None
@@ -768,7 +755,7 @@ class TestArtifact():
#
@pytest.fixture()
def cli(tmpdir):
- directory = os.path.join(str(tmpdir), 'cache')
+ directory = os.path.join(str(tmpdir), "cache")
os.makedirs(directory)
return Cli(directory)
@@ -781,27 +768,26 @@ def cli(tmpdir):
# stacktraces.
@pytest.fixture()
def cli_integration(tmpdir, integration_cache):
- directory = os.path.join(str(tmpdir), 'cache')
+ directory = os.path.join(str(tmpdir), "cache")
os.makedirs(directory)
fixture = CliIntegration(directory)
# We want to cache sources for integration tests more permanently,
# to avoid downloading the huge base-sdk repeatedly
- fixture.configure({
- 'cachedir': integration_cache.cachedir,
- 'sourcedir': integration_cache.sources,
- })
+ fixture.configure(
+ {"cachedir": integration_cache.cachedir, "sourcedir": integration_cache.sources,}
+ )
yield fixture
# remove following folders if necessary
try:
- shutil.rmtree(os.path.join(integration_cache.cachedir, 'build'))
+ shutil.rmtree(os.path.join(integration_cache.cachedir, "build"))
except FileNotFoundError:
pass
try:
- shutil.rmtree(os.path.join(integration_cache.cachedir, 'tmp'))
+ shutil.rmtree(os.path.join(integration_cache.cachedir, "tmp"))
except FileNotFoundError:
pass
@@ -813,36 +799,32 @@ def cli_integration(tmpdir, integration_cache):
# stacktraces.
@pytest.fixture()
def cli_remote_execution(tmpdir, remote_services):
- directory = os.path.join(str(tmpdir), 'cache')
+ directory = os.path.join(str(tmpdir), "cache")
os.makedirs(directory)
fixture = CliRemote(directory)
if remote_services.artifact_service:
- fixture.configure({'artifacts': [{
- 'url': remote_services.artifact_service,
- }]})
+ fixture.configure({"artifacts": [{"url": remote_services.artifact_service,}]})
remote_execution = {}
if remote_services.action_service:
- remote_execution['action-cache-service'] = {
- 'url': remote_services.action_service,
+ remote_execution["action-cache-service"] = {
+ "url": remote_services.action_service,
}
if remote_services.exec_service:
- remote_execution['execution-service'] = {
- 'url': remote_services.exec_service,
+ remote_execution["execution-service"] = {
+ "url": remote_services.exec_service,
}
if remote_services.storage_service:
- remote_execution['storage-service'] = {
- 'url': remote_services.storage_service,
+ remote_execution["storage-service"] = {
+ "url": remote_services.storage_service,
}
if remote_execution:
- fixture.configure({'remote-execution': remote_execution})
+ fixture.configure({"remote-execution": remote_execution})
if remote_services.source_service:
- fixture.configure({'source-caches': [{
- 'url': remote_services.source_service,
- }]})
+ fixture.configure({"source-caches": [{"url": remote_services.source_service,}]})
return fixture
@@ -882,12 +864,12 @@ def configured(directory, config=None):
if not config:
config = {}
- if not config.get('sourcedir', False):
- config['sourcedir'] = os.path.join(directory, 'sources')
- if not config.get('cachedir', False):
- config['cachedir'] = directory
- if not config.get('logdir', False):
- config['logdir'] = os.path.join(directory, 'logs')
+ if not config.get("sourcedir", False):
+ config["sourcedir"] = os.path.join(directory, "sources")
+ if not config.get("cachedir", False):
+ config["cachedir"] = directory
+ if not config.get("logdir", False):
+ config["logdir"] = os.path.join(directory, "logs")
# Dump it and yield the filename for test scripts to feed it
# to buildstream as an artument
diff --git a/src/buildstream/types.py b/src/buildstream/types.py
index 5688bf393..180044dbd 100644
--- a/src/buildstream/types.py
+++ b/src/buildstream/types.py
@@ -151,7 +151,7 @@ class Consistency(FastEnum):
return self.value < other.value
-class CoreWarnings():
+class CoreWarnings:
"""CoreWarnings()
Some common warnings which are raised by core functionalities within BuildStream are found in this class.
diff --git a/src/buildstream/utils.py b/src/buildstream/utils.py
index de7c14b70..181ea1df9 100644
--- a/src/buildstream/utils.py
+++ b/src/buildstream/utils.py
@@ -52,7 +52,7 @@ from ._utils import url_directory_name # pylint: disable=unused-import
BST_ARBITRARY_TIMESTAMP = calendar.timegm((2011, 11, 11, 11, 11, 11))
# The separator we use for user specified aliases
-_ALIAS_SEPARATOR = ':'
+_ALIAS_SEPARATOR = ":"
_URI_SCHEMES = ["http", "https", "ftp", "file", "git", "sftp", "ssh"]
# Main process pid
@@ -74,6 +74,7 @@ class UtilError(BstError):
or either of the :class:`.ElementError` or :class:`.SourceError`
exceptions should be raised from this error.
"""
+
def __init__(self, message, reason=None):
super().__init__(message, domain=ErrorDomain.UTIL, reason=reason)
@@ -83,6 +84,7 @@ class ProgramNotFoundError(BstError):
It is normally unneeded to handle this exception from plugin code.
"""
+
def __init__(self, message, reason=None):
super().__init__(message, domain=ErrorDomain.PROG_NOT_FOUND, reason=reason)
@@ -92,7 +94,7 @@ class DirectoryExistsError(OSError):
"""
-class FileListResult():
+class FileListResult:
"""An object which stores the result of one of the operations
which run on a list of files.
"""
@@ -112,7 +114,7 @@ class FileListResult():
self.files_written = []
"""List of files that were written."""
- def combine(self, other: 'FileListResult') -> 'FileListResult':
+ def combine(self, other: "FileListResult") -> "FileListResult":
"""Create a new FileListResult that contains the results of both.
"""
ret = FileListResult()
@@ -165,10 +167,10 @@ def list_relative_paths(directory: str) -> Iterator[str]:
# We don't want "./" pre-pended to all the entries in the root of
# `directory`, prefer to have no prefix in that case.
- basepath = relpath if relpath != '.' and dirpath != directory else ''
+ basepath = relpath if relpath != "." and dirpath != directory else ""
# First yield the walked directory itself, except for the root
- if basepath != '':
+ if basepath != "":
yield basepath
# List the filenames in the walked directory
@@ -248,8 +250,7 @@ def sha256sum(filename: str) -> str:
h.update(chunk)
except OSError as e:
- raise UtilError("Failed to get a checksum of file '{}': {}"
- .format(filename, e)) from e
+ raise UtilError("Failed to get a checksum of file '{}': {}".format(filename, e)) from e
return h.hexdigest()
@@ -274,8 +275,7 @@ def safe_copy(src: str, dest: str, *, result: Optional[FileListResult] = None) -
os.unlink(dest)
except OSError as e:
if e.errno != errno.ENOENT:
- raise UtilError("Failed to remove destination file '{}': {}"
- .format(dest, e)) from e
+ raise UtilError("Failed to remove destination file '{}': {}".format(dest, e)) from e
shutil.copyfile(src, dest)
try:
@@ -291,8 +291,7 @@ def safe_copy(src: str, dest: str, *, result: Optional[FileListResult] = None) -
result.failed_attributes.append(dest)
except shutil.Error as e:
- raise UtilError("Failed to copy '{} -> {}': {}"
- .format(src, dest, e)) from e
+ raise UtilError("Failed to copy '{} -> {}': {}".format(src, dest, e)) from e
def safe_link(src: str, dest: str, *, result: Optional[FileListResult] = None, _unlink=False) -> None:
@@ -313,8 +312,7 @@ def safe_link(src: str, dest: str, *, result: Optional[FileListResult] = None, _
os.unlink(dest)
except OSError as e:
if e.errno != errno.ENOENT:
- raise UtilError("Failed to remove destination file '{}': {}"
- .format(dest, e)) from e
+ raise UtilError("Failed to remove destination file '{}': {}".format(dest, e)) from e
# If we can't link it due to cross-device hardlink, copy
try:
@@ -326,8 +324,7 @@ def safe_link(src: str, dest: str, *, result: Optional[FileListResult] = None, _
elif e.errno == errno.EXDEV:
safe_copy(src, dest)
else:
- raise UtilError("Failed to link '{} -> {}': {}"
- .format(src, dest, e)) from e
+ raise UtilError("Failed to link '{} -> {}': {}".format(src, dest, e)) from e
def safe_remove(path: str) -> bool:
@@ -363,16 +360,17 @@ def safe_remove(path: str) -> bool:
# Path does not exist
return True
- raise UtilError("Failed to remove '{}': {}"
- .format(path, e))
+ raise UtilError("Failed to remove '{}': {}".format(path, e))
-def copy_files(src: str,
- dest: str,
- *,
- filter_callback: Optional[Callable[[str], bool]] = None,
- ignore_missing: bool = False,
- report_written: bool = False) -> FileListResult:
+def copy_files(
+ src: str,
+ dest: str,
+ *,
+ filter_callback: Optional[Callable[[str], bool]] = None,
+ ignore_missing: bool = False,
+ report_written: bool = False
+) -> FileListResult:
"""Copy files from source to destination.
Args:
@@ -401,22 +399,28 @@ def copy_files(src: str,
"""
result = FileListResult()
try:
- _process_list(src, dest, safe_copy, result,
- filter_callback=filter_callback,
- ignore_missing=ignore_missing,
- report_written=report_written)
+ _process_list(
+ src,
+ dest,
+ safe_copy,
+ result,
+ filter_callback=filter_callback,
+ ignore_missing=ignore_missing,
+ report_written=report_written,
+ )
except OSError as e:
- raise UtilError("Failed to copy '{} -> {}': {}"
- .format(src, dest, e))
+ raise UtilError("Failed to copy '{} -> {}': {}".format(src, dest, e))
return result
-def link_files(src: str,
- dest: str,
- *,
- filter_callback: Optional[Callable[[str], bool]] = None,
- ignore_missing: bool = False,
- report_written: bool = False) -> FileListResult:
+def link_files(
+ src: str,
+ dest: str,
+ *,
+ filter_callback: Optional[Callable[[str], bool]] = None,
+ ignore_missing: bool = False,
+ report_written: bool = False
+) -> FileListResult:
"""Hardlink files from source to destination.
Args:
@@ -450,13 +454,17 @@ def link_files(src: str,
"""
result = FileListResult()
try:
- _process_list(src, dest, safe_link, result,
- filter_callback=filter_callback,
- ignore_missing=ignore_missing,
- report_written=report_written)
+ _process_list(
+ src,
+ dest,
+ safe_link,
+ result,
+ filter_callback=filter_callback,
+ ignore_missing=ignore_missing,
+ report_written=report_written,
+ )
except OSError as e:
- raise UtilError("Failed to link '{} -> {}': {}"
- .format(src, dest, e))
+ raise UtilError("Failed to link '{} -> {}': {}".format(src, dest, e))
return result
@@ -473,7 +481,7 @@ def get_host_tool(name: str) -> str:
Raises:
:class:`.ProgramNotFoundError`
"""
- search_path = os.environ.get('PATH')
+ search_path = os.environ.get("PATH")
program_path = shutil.which(name, path=search_path)
if not program_path:
@@ -491,22 +499,27 @@ def get_bst_version() -> Tuple[int, int]:
"""
# Import this only conditionally, it's not resolved at bash complete time
from . import __version__ # pylint: disable=cyclic-import
- versions = __version__.split('.')[:2]
- if versions[0] == '0+untagged':
- raise UtilError("Your git repository has no tags - BuildStream can't "
- "determine its version. Please run `git fetch --tags`.")
+ versions = __version__.split(".")[:2]
+
+ if versions[0] == "0+untagged":
+ raise UtilError(
+ "Your git repository has no tags - BuildStream can't "
+ "determine its version. Please run `git fetch --tags`."
+ )
try:
return (int(versions[0]), int(versions[1]))
except IndexError:
- raise UtilError("Cannot detect Major and Minor parts of the version\n"
- "Version: {} not in XX.YY.whatever format"
- .format(__version__))
+ raise UtilError(
+ "Cannot detect Major and Minor parts of the version\n"
+ "Version: {} not in XX.YY.whatever format".format(__version__)
+ )
except ValueError:
- raise UtilError("Cannot convert version to integer numbers\n"
- "Version: {} not in Integer.Integer.whatever format"
- .format(__version__))
+ raise UtilError(
+ "Cannot convert version to integer numbers\n"
+ "Version: {} not in Integer.Integer.whatever format".format(__version__)
+ )
def move_atomic(source: Union[Path, str], destination: Union[Path, str], *, ensure_parents: bool = True) -> None:
@@ -548,16 +561,18 @@ def move_atomic(source: Union[Path, str], destination: Union[Path, str], *, ensu
@contextmanager
-def save_file_atomic(filename: str,
- mode: str = 'w',
- *,
- buffering: int = -1,
- encoding: Optional[str] = None,
- errors: Optional[str] = None,
- newline: Optional[str] = None,
- closefd: bool = True,
- opener: Optional[Callable[[str, int], int]] = None,
- tempdir: Optional[str] = None) -> Iterator[IO]:
+def save_file_atomic(
+ filename: str,
+ mode: str = "w",
+ *,
+ buffering: int = -1,
+ encoding: Optional[str] = None,
+ errors: Optional[str] = None,
+ newline: Optional[str] = None,
+ closefd: bool = True,
+ opener: Optional[Callable[[str, int], int]] = None,
+ tempdir: Optional[str] = None
+) -> Iterator[IO]:
"""Save a file with a temporary name and rename it into place when ready.
This is a context manager which is meant for saving data to files.
@@ -589,8 +604,16 @@ def save_file_atomic(filename: str,
fd, tempname = tempfile.mkstemp(dir=tempdir)
os.close(fd)
- f = open(tempname, mode=mode, buffering=buffering, encoding=encoding,
- errors=errors, newline=newline, closefd=closefd, opener=opener)
+ f = open(
+ tempname,
+ mode=mode,
+ buffering=buffering,
+ encoding=encoding,
+ errors=errors,
+ newline=newline,
+ closefd=closefd,
+ opener=opener,
+ )
def cleanup_tempfile():
f.close()
@@ -604,7 +627,7 @@ def save_file_atomic(filename: str,
try:
with _signals.terminator(cleanup_tempfile):
# Disable type-checking since "IO[Any]" has no attribute "real_filename"
- f.real_filename = filename # type: ignore
+ f.real_filename = filename # type: ignore
yield f
f.close()
# This operation is atomic, at least on platforms we care about:
@@ -660,8 +683,7 @@ def _get_volume_size(path):
try:
usage = shutil.disk_usage(path)
except OSError as e:
- raise UtilError("Failed to retrieve stats on volume for path '{}': {}"
- .format(path, e)) from e
+ raise UtilError("Failed to retrieve stats on volume for path '{}': {}".format(path, e)) from e
return usage.total, usage.free
@@ -685,16 +707,16 @@ def _get_volume_size(path):
# UtilError if the string is not a valid data size.
#
def _parse_size(size, volume):
- if size == 'infinity':
+ if size == "infinity":
return None
- matches = re.fullmatch(r'([0-9]+\.?[0-9]*)([KMGT%]?)', size)
+ matches = re.fullmatch(r"([0-9]+\.?[0-9]*)([KMGT%]?)", size)
if matches is None:
raise UtilError("{} is not a valid data size.".format(size))
num, unit = matches.groups()
- if unit == '%':
+ if unit == "%":
num = float(num)
if num > 100:
raise UtilError("{}% is not a valid percentage value.".format(num))
@@ -703,8 +725,8 @@ def _parse_size(size, volume):
return disk_size * (num / 100)
- units = ('', 'K', 'M', 'G', 'T')
- return int(num) * 1024**units.index(unit)
+ units = ("", "K", "M", "G", "T")
+ return int(num) * 1024 ** units.index(unit)
# _pretty_size()
@@ -720,8 +742,8 @@ def _parse_size(size, volume):
# (str): The string representation of the number of bytes in the largest
def _pretty_size(size, dec_places=0):
psize = size
- unit = 'B'
- units = ('B', 'K', 'M', 'G', 'T')
+ unit = "B"
+ units = ("B", "K", "M", "G", "T")
for unit in units:
if psize < 1024:
break
@@ -746,19 +768,17 @@ def _force_rmtree(rootpath, **kwargs):
os.chmod(rootpath, 0o755)
for root, dirs, _ in os.walk(rootpath):
for d in dirs:
- path = os.path.join(root, d.lstrip('/'))
+ path = os.path.join(root, d.lstrip("/"))
if os.path.exists(path) and not os.path.islink(path):
try:
os.chmod(path, 0o755)
except OSError as e:
- raise UtilError("Failed to ensure write permission on file '{}': {}"
- .format(path, e))
+ raise UtilError("Failed to ensure write permission on file '{}': {}".format(path, e))
try:
shutil.rmtree(rootpath, **kwargs)
except OSError as e:
- raise UtilError("Failed to remove cache directory '{}': {}"
- .format(rootpath, e))
+ raise UtilError("Failed to remove cache directory '{}': {}".format(rootpath, e))
# Recursively make directories in target area
@@ -779,8 +799,7 @@ def _copy_directories(srcdir, destdir, target):
os.makedirs(new_dir)
yield (new_dir, mode)
else:
- raise UtilError('Source directory tree has file where '
- 'directory expected: {}'.format(old_dir))
+ raise UtilError("Source directory tree has file where " "directory expected: {}".format(old_dir))
else:
if not os.access(new_dir, os.W_OK):
# If the destination directory is not writable, change permissions to make it
@@ -806,16 +825,16 @@ def _ensure_real_directory(root, path):
try:
deststat = os.lstat(destpath)
if not stat.S_ISDIR(deststat.st_mode):
- relpath = destpath[len(root):]
+ relpath = destpath[len(root) :]
if stat.S_ISLNK(deststat.st_mode):
- filetype = 'symlink'
+ filetype = "symlink"
elif stat.S_ISREG(deststat.st_mode):
- filetype = 'regular file'
+ filetype = "regular file"
else:
- filetype = 'special file'
+ filetype = "special file"
- raise UtilError('Destination is a {}, not a directory: {}'.format(filetype, relpath))
+ raise UtilError("Destination is a {}, not a directory: {}".format(filetype, relpath))
except FileNotFoundError:
os.makedirs(destpath)
@@ -836,9 +855,9 @@ def _ensure_real_directory(root, path):
# ignore_missing: Dont raise any error if a source file is missing
#
#
-def _process_list(srcdir, destdir, actionfunc, result,
- filter_callback=None,
- ignore_missing=False, report_written=False):
+def _process_list(
+ srcdir, destdir, actionfunc, result, filter_callback=None, ignore_missing=False, report_written=False
+):
# Keep track of directory permissions, since these need to be set
# *after* files have been written.
@@ -921,7 +940,7 @@ def _process_list(srcdir, destdir, actionfunc, result,
else:
# Unsupported type.
- raise UtilError('Cannot extract {} into staging-area. Unsupported type.'.format(srcpath))
+ raise UtilError("Cannot extract {} into staging-area. Unsupported type.".format(srcpath))
# Write directory permissions now that all files have been written
for d, perms in permissions:
@@ -1035,8 +1054,9 @@ def _tempnamedfile(suffix="", prefix="tmp", dir=None): # pylint: disable=redefi
if temp is not None:
temp.close()
- with _signals.terminator(close_tempfile), \
- tempfile.NamedTemporaryFile(suffix=suffix, prefix=prefix, dir=dir) as temp:
+ with _signals.terminator(close_tempfile), tempfile.NamedTemporaryFile(
+ suffix=suffix, prefix=prefix, dir=dir
+ ) as temp:
yield temp
@@ -1145,13 +1165,13 @@ def _kill_process_tree(pid):
#
def _call(*popenargs, terminate=False, **kwargs):
- kwargs['start_new_session'] = True
+ kwargs["start_new_session"] = True
process = None
- old_preexec_fn = kwargs.get('preexec_fn')
- if 'preexec_fn' in kwargs:
- del kwargs['preexec_fn']
+ old_preexec_fn = kwargs.get("preexec_fn")
+ if "preexec_fn" in kwargs:
+ del kwargs["preexec_fn"]
def preexec_fn():
os.umask(stat.S_IWGRP | stat.S_IWOTH)
@@ -1203,7 +1223,8 @@ def _call(*popenargs, terminate=False, **kwargs):
with _signals.suspendable(suspend_proc, resume_proc), _signals.terminator(kill_proc):
process = subprocess.Popen( # pylint: disable=subprocess-popen-preexec-fn
- *popenargs, preexec_fn=preexec_fn, universal_newlines=True, **kwargs)
+ *popenargs, preexec_fn=preexec_fn, universal_newlines=True, **kwargs
+ )
output, _ = process.communicate()
exit_code = process.poll()
@@ -1237,44 +1258,44 @@ def _call(*popenargs, terminate=False, **kwargs):
#
def _glob2re(pat):
i, n = 0, len(pat)
- res = '(?ms)'
+ res = "(?ms)"
while i < n:
c = pat[i]
i = i + 1
- if c == '*':
+ if c == "*":
# fnmatch.translate() simply uses the '.*' separator here,
# we only want that for double asterisk (bash 'globstar' behavior)
#
- if i < n and pat[i] == '*':
- res = res + '.*'
+ if i < n and pat[i] == "*":
+ res = res + ".*"
i = i + 1
else:
- res = res + '[^/]*'
- elif c == '?':
+ res = res + "[^/]*"
+ elif c == "?":
# fnmatch.translate() simply uses the '.' wildcard here, but
# we dont want to match path separators here
- res = res + '[^/]'
- elif c == '[':
+ res = res + "[^/]"
+ elif c == "[":
j = i
- if j < n and pat[j] == '!':
+ if j < n and pat[j] == "!":
j = j + 1
- if j < n and pat[j] == ']':
+ if j < n and pat[j] == "]":
j = j + 1
- while j < n and pat[j] != ']':
+ while j < n and pat[j] != "]":
j = j + 1
if j >= n:
- res = res + '\\['
+ res = res + "\\["
else:
- stuff = pat[i:j].replace('\\', '\\\\')
+ stuff = pat[i:j].replace("\\", "\\\\")
i = j + 1
- if stuff[0] == '!':
- stuff = '^' + stuff[1:]
- elif stuff[0] == '^':
- stuff = '\\' + stuff
- res = '{}[{}]'.format(res, stuff)
+ if stuff[0] == "!":
+ stuff = "^" + stuff[1:]
+ elif stuff[0] == "^":
+ stuff = "\\" + stuff
+ res = "{}[{}]".format(res, stuff)
else:
res = res + re.escape(c)
- return res + r'\Z'
+ return res + r"\Z"
# _deduplicate()
@@ -1392,7 +1413,7 @@ def _deterministic_umask():
#
#
def _get_compression(tar):
- mapped_extensions = {'.tar': '', '.gz': 'gz', '.xz': 'xz', '.bz2': 'bz2'}
+ mapped_extensions = {".tar": "", ".gz": "gz", ".xz": "xz", ".bz2": "bz2"}
name, ext = os.path.splitext(tar)
@@ -1403,12 +1424,14 @@ def _get_compression(tar):
# If so, we assume we have been given an unsupported extension,
# which expects compression. Raise an error
_, suffix = os.path.splitext(name)
- if suffix == '.tar':
- raise UtilError("Expected compression with unknown file extension ('{}'), "
- "supported extensions are ('.tar'), ('.gz'), ('.xz'), ('.bz2')".format(ext))
+ if suffix == ".tar":
+ raise UtilError(
+ "Expected compression with unknown file extension ('{}'), "
+ "supported extensions are ('.tar'), ('.gz'), ('.xz'), ('.bz2')".format(ext)
+ )
# Assume just an unconventional name was provided, default to uncompressed
- return ''
+ return ""
# _is_single_threaded()
diff --git a/tests/artifactcache/artifactservice.py b/tests/artifactcache/artifactservice.py
index 51f128c32..c640665a3 100644
--- a/tests/artifactcache/artifactservice.py
+++ b/tests/artifactcache/artifactservice.py
@@ -22,11 +22,9 @@ from urllib.parse import urlparse
import grpc
import pytest
-from buildstream._protos.buildstream.v2.artifact_pb2 \
- import Artifact, GetArtifactRequest, UpdateArtifactRequest
+from buildstream._protos.buildstream.v2.artifact_pb2 import Artifact, GetArtifactRequest, UpdateArtifactRequest
from buildstream._protos.buildstream.v2.artifact_pb2_grpc import ArtifactServiceStub
-from buildstream._protos.build.bazel.remote.execution.v2 \
- import remote_execution_pb2 as re_pb2
+from buildstream._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 as re_pb2
from buildstream import utils
from tests.testutils.artifactshare import create_artifact_share
diff --git a/tests/artifactcache/capabilities.py b/tests/artifactcache/capabilities.py
index a28516aea..c8a49f9b5 100644
--- a/tests/artifactcache/capabilities.py
+++ b/tests/artifactcache/capabilities.py
@@ -13,10 +13,7 @@ from tests.testutils import dummy_context
from tests.testutils.artifactshare import create_dummy_artifact_share
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project",
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",)
@pytest.mark.datafiles(DATA_DIR)
@@ -26,17 +23,12 @@ def test_artifact_cache_with_missing_capabilities_is_skipped(cli, tmpdir, datafi
# Set up an artifact cache.
with create_dummy_artifact_share() as share:
# Configure artifact share
- cache_dir = os.path.join(str(tmpdir), 'cache')
- user_config_file = str(tmpdir.join('buildstream.conf'))
+ cache_dir = os.path.join(str(tmpdir), "cache")
+ user_config_file = str(tmpdir.join("buildstream.conf"))
user_config = {
- 'scheduler': {
- 'pushers': 1
- },
- 'artifacts': {
- 'url': share.repo,
- 'push': True,
- },
- 'cachedir': cache_dir
+ "scheduler": {"pushers": 1},
+ "artifacts": {"url": share.repo, "push": True,},
+ "cachedir": cache_dir,
}
_yaml.roundtrip_dump(user_config, file=user_config_file)
@@ -51,5 +43,6 @@ def test_artifact_cache_with_missing_capabilities_is_skipped(cli, tmpdir, datafi
# Manually setup the CAS remote
artifactcache.setup_remotes(use_config=True)
- assert not artifactcache.has_fetch_remotes(), \
- "System didn't realize the artifact cache didn't support BuildStream"
+ assert (
+ not artifactcache.has_fetch_remotes()
+ ), "System didn't realize the artifact cache didn't support BuildStream"
diff --git a/tests/artifactcache/config.py b/tests/artifactcache/config.py
index 75cce267c..e5e2213eb 100644
--- a/tests/artifactcache/config.py
+++ b/tests/artifactcache/config.py
@@ -19,32 +19,20 @@ from tests.testutils import dummy_context
DATA_DIR = os.path.dirname(os.path.realpath(__file__))
-cache1 = RemoteSpec(url='https://example.com/cache1', push=True)
-cache2 = RemoteSpec(url='https://example.com/cache2', push=False)
-cache3 = RemoteSpec(url='https://example.com/cache3', push=False)
-cache4 = RemoteSpec(url='https://example.com/cache4', push=False)
-cache5 = RemoteSpec(url='https://example.com/cache5', push=False)
-cache6 = RemoteSpec(url='https://example.com/cache6',
- push=True,
- type=RemoteType.ALL)
-cache7 = RemoteSpec(url='https://index.example.com/cache1',
- push=True,
- type=RemoteType.INDEX)
-cache8 = RemoteSpec(url='https://storage.example.com/cache1',
- push=True,
- type=RemoteType.STORAGE)
+cache1 = RemoteSpec(url="https://example.com/cache1", push=True)
+cache2 = RemoteSpec(url="https://example.com/cache2", push=False)
+cache3 = RemoteSpec(url="https://example.com/cache3", push=False)
+cache4 = RemoteSpec(url="https://example.com/cache4", push=False)
+cache5 = RemoteSpec(url="https://example.com/cache5", push=False)
+cache6 = RemoteSpec(url="https://example.com/cache6", push=True, type=RemoteType.ALL)
+cache7 = RemoteSpec(url="https://index.example.com/cache1", push=True, type=RemoteType.INDEX)
+cache8 = RemoteSpec(url="https://storage.example.com/cache1", push=True, type=RemoteType.STORAGE)
# Generate cache configuration fragments for the user config and project config files.
#
-def configure_remote_caches(override_caches,
- project_caches=None,
- user_caches=None):
- type_strings = {
- RemoteType.INDEX: 'index',
- RemoteType.STORAGE: 'storage',
- RemoteType.ALL: 'all'
- }
+def configure_remote_caches(override_caches, project_caches=None, user_caches=None):
+ type_strings = {RemoteType.INDEX: "index", RemoteType.STORAGE: "storage", RemoteType.ALL: "all"}
if project_caches is None:
project_caches = []
@@ -54,39 +42,32 @@ def configure_remote_caches(override_caches,
user_config = {}
if len(user_caches) == 1:
- user_config['artifacts'] = {
- 'url': user_caches[0].url,
- 'push': user_caches[0].push,
- 'type': type_strings[user_caches[0].type]
+ user_config["artifacts"] = {
+ "url": user_caches[0].url,
+ "push": user_caches[0].push,
+ "type": type_strings[user_caches[0].type],
}
elif len(user_caches) > 1:
- user_config['artifacts'] = [
- {
- 'url': cache.url,
- 'push': cache.push,
- 'type': type_strings[cache.type]
- } for cache in user_caches
+ user_config["artifacts"] = [
+ {"url": cache.url, "push": cache.push, "type": type_strings[cache.type]} for cache in user_caches
]
if len(override_caches) == 1:
- user_config['projects'] = {
- 'test': {
- 'artifacts': {
- 'url': override_caches[0].url,
- 'push': override_caches[0].push,
- 'type': type_strings[override_caches[0].type]
+ user_config["projects"] = {
+ "test": {
+ "artifacts": {
+ "url": override_caches[0].url,
+ "push": override_caches[0].push,
+ "type": type_strings[override_caches[0].type],
}
}
}
elif len(override_caches) > 1:
- user_config['projects'] = {
- 'test': {
- 'artifacts': [
- {
- 'url': cache.url,
- 'push': cache.push,
- 'type': type_strings[cache.type]
- } for cache in override_caches
+ user_config["projects"] = {
+ "test": {
+ "artifacts": [
+ {"url": cache.url, "push": cache.push, "type": type_strings[cache.type]}
+ for cache in override_caches
]
}
}
@@ -94,23 +75,24 @@ def configure_remote_caches(override_caches,
project_config = {}
if project_caches:
if len(project_caches) == 1:
- project_config.update({
- 'artifacts': {
- 'url': project_caches[0].url,
- 'push': project_caches[0].push,
- 'type': type_strings[project_caches[0].type],
+ project_config.update(
+ {
+ "artifacts": {
+ "url": project_caches[0].url,
+ "push": project_caches[0].push,
+ "type": type_strings[project_caches[0].type],
+ }
}
- })
+ )
elif len(project_caches) > 1:
- project_config.update({
- 'artifacts': [
- {
- 'url': cache.url,
- 'push': cache.push,
- 'type': type_strings[cache.type]
- } for cache in project_caches
- ]
- })
+ project_config.update(
+ {
+ "artifacts": [
+ {"url": cache.url, "push": cache.push, "type": type_strings[cache.type]}
+ for cache in project_caches
+ ]
+ }
+ )
return user_config, project_config
@@ -118,27 +100,28 @@ def configure_remote_caches(override_caches,
# Test that parsing the remote artifact cache locations produces the
# expected results.
@pytest.mark.parametrize(
- 'override_caches, project_caches, user_caches',
+ "override_caches, project_caches, user_caches",
[
# The leftmost cache is the highest priority one in all cases here.
- pytest.param([], [], [], id='empty-config'),
- pytest.param([], [], [cache1, cache2], id='user-config'),
- pytest.param([], [cache1, cache2], [cache3], id='project-config'),
- pytest.param([cache1], [cache2], [cache3], id='project-override-in-user-config'),
- pytest.param([cache1, cache2], [cache3, cache4], [cache5, cache6], id='list-order'),
- pytest.param([cache1, cache2, cache1], [cache2], [cache2, cache1], id='duplicates'),
- pytest.param([cache7, cache8], [], [cache1], id='split-caches')
- ])
+ pytest.param([], [], [], id="empty-config"),
+ pytest.param([], [], [cache1, cache2], id="user-config"),
+ pytest.param([], [cache1, cache2], [cache3], id="project-config"),
+ pytest.param([cache1], [cache2], [cache3], id="project-override-in-user-config"),
+ pytest.param([cache1, cache2], [cache3, cache4], [cache5, cache6], id="list-order"),
+ pytest.param([cache1, cache2, cache1], [cache2], [cache2, cache1], id="duplicates"),
+ pytest.param([cache7, cache8], [], [cache1], id="split-caches"),
+ ],
+)
def test_artifact_cache_precedence(tmpdir, override_caches, project_caches, user_caches):
# Produce a fake user and project config with the cache configuration.
user_config, project_config = configure_remote_caches(override_caches, project_caches, user_caches)
- project_config['name'] = 'test'
+ project_config["name"] = "test"
- user_config_file = str(tmpdir.join('buildstream.conf'))
+ user_config_file = str(tmpdir.join("buildstream.conf"))
_yaml.roundtrip_dump(user_config, file=user_config_file)
- project_dir = tmpdir.mkdir('project')
- project_config_file = str(project_dir.join('project.conf'))
+ project_dir = tmpdir.mkdir("project")
+ project_config_file = str(project_dir.join("project.conf"))
_yaml.roundtrip_dump(project_config, file=project_config_file)
with dummy_context(config=user_config_file) as context:
@@ -157,29 +140,21 @@ def test_artifact_cache_precedence(tmpdir, override_caches, project_caches, user
# without specifying its counterpart, we get a comprehensive LoadError
# instead of an unhandled exception.
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize('config_key, config_value', [
- ('client-cert', 'client.crt'),
- ('client-key', 'client.key')
-])
+@pytest.mark.parametrize("config_key, config_value", [("client-cert", "client.crt"), ("client-key", "client.key")])
def test_missing_certs(cli, datafiles, config_key, config_value):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'missing-certs')
+ project = os.path.join(datafiles.dirname, datafiles.basename, "missing-certs")
project_conf = {
- 'name': 'test',
-
- 'artifacts': {
- 'url': 'https://cache.example.com:12345',
- 'push': 'true',
- config_key: config_value
- }
+ "name": "test",
+ "artifacts": {"url": "https://cache.example.com:12345", "push": "true", config_key: config_value},
}
- project_conf_file = os.path.join(project, 'project.conf')
+ project_conf_file = os.path.join(project, "project.conf")
_yaml.roundtrip_dump(project_conf, project_conf_file)
# Use `pull` here to ensure we try to initialize the remotes, triggering the error
#
# This does not happen for a simple `bst show`.
- result = cli.run(project=project, args=['artifact', 'pull', 'element.bst'])
+ result = cli.run(project=project, args=["artifact", "pull", "element.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
@@ -187,32 +162,33 @@ def test_missing_certs(cli, datafiles, config_key, config_value):
# only one type of storage.
@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.parametrize(
- 'override_caches, project_caches, user_caches',
+ "override_caches, project_caches, user_caches",
[
# The leftmost cache is the highest priority one in all cases here.
- pytest.param([], [], [cache7], id='index-user'),
- pytest.param([], [], [cache8], id='storage-user'),
- pytest.param([], [cache7], [], id='index-project'),
- pytest.param([], [cache8], [], id='storage-project'),
- pytest.param([cache7], [], [], id='index-override'),
- pytest.param([cache8], [], [], id='storage-override'),
- ])
+ pytest.param([], [], [cache7], id="index-user"),
+ pytest.param([], [], [cache8], id="storage-user"),
+ pytest.param([], [cache7], [], id="index-project"),
+ pytest.param([], [cache8], [], id="storage-project"),
+ pytest.param([cache7], [], [], id="index-override"),
+ pytest.param([cache8], [], [], id="storage-override"),
+ ],
+)
def test_only_one(cli, datafiles, override_caches, project_caches, user_caches):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'only-one')
+ project = os.path.join(datafiles.dirname, datafiles.basename, "only-one")
# Produce a fake user and project config with the cache configuration.
user_config, project_config = configure_remote_caches(override_caches, project_caches, user_caches)
- project_config['name'] = 'test'
+ project_config["name"] = "test"
cli.configure(user_config)
- project_config_file = os.path.join(project, 'project.conf')
+ project_config_file = os.path.join(project, "project.conf")
_yaml.roundtrip_dump(project_config, file=project_config_file)
# Use `pull` here to ensure we try to initialize the remotes, triggering the error
#
# This does not happen for a simple `bst show`.
- result = cli.run(project=project, args=['artifact', 'pull', 'element.bst'])
+ result = cli.run(project=project, args=["artifact", "pull", "element.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
@@ -239,8 +215,8 @@ def test_only_one(cli, datafiles, override_caches, project_caches, user_caches):
"client-cert": "~/client2.crt",
"client-key": "~/client2.key",
},
- ]
- )
+ ],
+ ),
)
@pytest.mark.parametrize("in_user_config", [True, False])
def test_paths_for_artifact_config_are_expanded(tmpdir, monkeypatch, artifacts_config, in_user_config):
@@ -260,11 +236,11 @@ def test_paths_for_artifact_config_are_expanded(tmpdir, monkeypatch, artifacts_c
"artifacts": artifacts_config,
}
- user_config_file = str(tmpdir.join('buildstream.conf'))
+ user_config_file = str(tmpdir.join("buildstream.conf"))
_yaml.roundtrip_dump(user_config, file=user_config_file)
- project_dir = tmpdir.mkdir('project')
- project_config_file = str(project_dir.join('project.conf'))
+ project_dir = tmpdir.mkdir("project")
+ project_config_file = str(project_dir.join("project.conf"))
_yaml.roundtrip_dump(project_config, file=project_config_file)
with dummy_context(config=user_config_file) as context:
diff --git a/tests/artifactcache/expiry.py b/tests/artifactcache/expiry.py
index 9ede1a8d3..83577f0c6 100644
--- a/tests/artifactcache/expiry.py
+++ b/tests/artifactcache/expiry.py
@@ -32,10 +32,7 @@ from buildstream.testing import cli # pylint: disable=unused-import
from tests.testutils import create_element_size, wait_for_cache_granularity
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "expiry"
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "expiry")
def get_cache_usage(directory):
@@ -59,115 +56,109 @@ def get_cache_usage(directory):
@pytest.mark.datafiles(DATA_DIR)
def test_artifact_expires(cli, datafiles):
project = str(datafiles)
- element_path = 'elements'
+ element_path = "elements"
- cli.configure({
- 'cache': {
- 'quota': 10000000,
- }
- })
+ cli.configure({"cache": {"quota": 10000000,}})
# Create an element that uses almost the entire cache (an empty
# ostree cache starts at about ~10KiB, so we need a bit of a
# buffer)
- create_element_size('target.bst', project, element_path, [], 6000000)
- res = cli.run(project=project, args=['build', 'target.bst'])
+ create_element_size("target.bst", project, element_path, [], 6000000)
+ res = cli.run(project=project, args=["build", "target.bst"])
res.assert_success()
- assert cli.get_element_state(project, 'target.bst') == 'cached'
+ assert cli.get_element_state(project, "target.bst") == "cached"
# Our cache should now be almost full. Let's create another
# artifact and see if we can cause buildstream to delete the old
# one.
- create_element_size('target2.bst', project, element_path, [], 6000000)
- res = cli.run(project=project, args=['build', 'target2.bst'])
+ create_element_size("target2.bst", project, element_path, [], 6000000)
+ res = cli.run(project=project, args=["build", "target2.bst"])
res.assert_success()
# Check that the correct element remains in the cache
- states = cli.get_element_states(project, ['target.bst', 'target2.bst'])
- assert states['target.bst'] != 'cached'
- assert states['target2.bst'] == 'cached'
+ states = cli.get_element_states(project, ["target.bst", "target2.bst"])
+ assert states["target.bst"] != "cached"
+ assert states["target2.bst"] == "cached"
# Ensure that we don't end up deleting the whole cache (or worse) if
# we try to store an artifact that is too large to fit in the quota.
-@pytest.mark.parametrize('size', [
- # Test an artifact that is obviously too large
- (500000),
- # Test an artifact that might be too large due to slight overhead
- # of storing stuff in ostree
- (399999)
-])
+@pytest.mark.parametrize(
+ "size",
+ [
+ # Test an artifact that is obviously too large
+ (500000),
+ # Test an artifact that might be too large due to slight overhead
+ # of storing stuff in ostree
+ (399999),
+ ],
+)
@pytest.mark.datafiles(DATA_DIR)
def test_artifact_too_large(cli, datafiles, size):
project = str(datafiles)
- element_path = 'elements'
+ element_path = "elements"
- cli.configure({
- 'cache': {
- 'quota': 400000
- }
- })
+ cli.configure({"cache": {"quota": 400000}})
# Create an element whose artifact is too large
- create_element_size('target.bst', project, element_path, [], size)
- res = cli.run(project=project, args=['build', 'target.bst'])
+ create_element_size("target.bst", project, element_path, [], size)
+ res = cli.run(project=project, args=["build", "target.bst"])
res.assert_main_error(ErrorDomain.STREAM, None)
- res.assert_task_error(ErrorDomain.CAS, 'cache-too-full')
+ res.assert_task_error(ErrorDomain.CAS, "cache-too-full")
@pytest.mark.datafiles(DATA_DIR)
def test_expiry_order(cli, datafiles):
project = str(datafiles)
- element_path = 'elements'
- checkout = os.path.join(project, 'workspace')
+ element_path = "elements"
+ checkout = os.path.join(project, "workspace")
- cli.configure({
- 'cache': {
- 'quota': 9000000
- }
- })
+ cli.configure({"cache": {"quota": 9000000}})
# Create an artifact
- create_element_size('dep.bst', project, element_path, [], 2000000)
- res = cli.run(project=project, args=['build', 'dep.bst'])
+ create_element_size("dep.bst", project, element_path, [], 2000000)
+ res = cli.run(project=project, args=["build", "dep.bst"])
res.assert_success()
# Create another artifact
- create_element_size('unrelated.bst', project, element_path, [], 2000000)
- res = cli.run(project=project, args=['build', 'unrelated.bst'])
+ create_element_size("unrelated.bst", project, element_path, [], 2000000)
+ res = cli.run(project=project, args=["build", "unrelated.bst"])
res.assert_success()
# And build something else
- create_element_size('target.bst', project, element_path, [], 2000000)
- res = cli.run(project=project, args=['build', 'target.bst'])
+ create_element_size("target.bst", project, element_path, [], 2000000)
+ res = cli.run(project=project, args=["build", "target.bst"])
res.assert_success()
- create_element_size('target2.bst', project, element_path, [], 2000000)
- res = cli.run(project=project, args=['build', 'target2.bst'])
+ create_element_size("target2.bst", project, element_path, [], 2000000)
+ res = cli.run(project=project, args=["build", "target2.bst"])
res.assert_success()
wait_for_cache_granularity()
# Now extract dep.bst
- res = cli.run(project=project, args=['artifact', 'checkout', 'dep.bst', '--directory', checkout])
+ res = cli.run(project=project, args=["artifact", "checkout", "dep.bst", "--directory", checkout])
res.assert_success()
# Finally, build something that will cause the cache to overflow
- create_element_size('expire.bst', project, element_path, [], 2000000)
- res = cli.run(project=project, args=['build', 'expire.bst'])
+ create_element_size("expire.bst", project, element_path, [], 2000000)
+ res = cli.run(project=project, args=["build", "expire.bst"])
res.assert_success()
# While dep.bst was the first element to be created, it should not
# have been removed.
# Note that buildstream will reduce the cache to 50% of the
# original size - we therefore remove multiple elements.
- check_elements = [
- 'unrelated.bst', 'target.bst', 'target2.bst', 'dep.bst', 'expire.bst'
- ]
+ check_elements = ["unrelated.bst", "target.bst", "target2.bst", "dep.bst", "expire.bst"]
states = cli.get_element_states(project, check_elements)
- assert (tuple(states[element] for element in check_elements) ==
- ('buildable', 'buildable', 'buildable', 'cached', 'cached', ))
+ assert tuple(states[element] for element in check_elements) == (
+ "buildable",
+ "buildable",
+ "buildable",
+ "cached",
+ "cached",
+ )
# Ensure that we don't accidentally remove an artifact from something
@@ -176,28 +167,24 @@ def test_expiry_order(cli, datafiles):
@pytest.mark.datafiles(DATA_DIR)
def test_keep_dependencies(cli, datafiles):
project = str(datafiles)
- element_path = 'elements'
+ element_path = "elements"
- cli.configure({
- 'cache': {
- 'quota': 10000000
- }
- })
+ cli.configure({"cache": {"quota": 10000000}})
# Create a pretty big dependency
- create_element_size('dependency.bst', project, element_path, [], 5000000)
- res = cli.run(project=project, args=['build', 'dependency.bst'])
+ create_element_size("dependency.bst", project, element_path, [], 5000000)
+ res = cli.run(project=project, args=["build", "dependency.bst"])
res.assert_success()
# Now create some other unrelated artifact
- create_element_size('unrelated.bst', project, element_path, [], 4000000)
- res = cli.run(project=project, args=['build', 'unrelated.bst'])
+ create_element_size("unrelated.bst", project, element_path, [], 4000000)
+ res = cli.run(project=project, args=["build", "unrelated.bst"])
res.assert_success()
# Check that the correct element remains in the cache
- states = cli.get_element_states(project, ['dependency.bst', 'unrelated.bst'])
- assert states['dependency.bst'] == 'cached'
- assert states['unrelated.bst'] == 'cached'
+ states = cli.get_element_states(project, ["dependency.bst", "unrelated.bst"])
+ assert states["dependency.bst"] == "cached"
+ assert states["unrelated.bst"] == "cached"
# We try to build an element which depends on the LRU artifact,
# and could therefore fail if we didn't make sure dependencies
@@ -207,54 +194,45 @@ def test_keep_dependencies(cli, datafiles):
# duplicating artifacts (bad!) we need to make this equal in size
# or smaller than half the size of its dependencies.
#
- create_element_size('target.bst', project,
- element_path, ['dependency.bst'], 2000000)
- res = cli.run(project=project, args=['build', 'target.bst'])
+ create_element_size("target.bst", project, element_path, ["dependency.bst"], 2000000)
+ res = cli.run(project=project, args=["build", "target.bst"])
res.assert_success()
- states = cli.get_element_states(project, ['target.bst', 'unrelated.bst'])
- assert states['target.bst'] == 'cached'
- assert states['dependency.bst'] == 'cached'
- assert states['unrelated.bst'] != 'cached'
+ states = cli.get_element_states(project, ["target.bst", "unrelated.bst"])
+ assert states["target.bst"] == "cached"
+ assert states["dependency.bst"] == "cached"
+ assert states["unrelated.bst"] != "cached"
# Assert that we never delete a dependency required for a build tree
@pytest.mark.datafiles(DATA_DIR)
def test_never_delete_required(cli, datafiles):
project = str(datafiles)
- element_path = 'elements'
-
- cli.configure({
- 'cache': {
- 'quota': 10000000
- },
- 'scheduler': {
- 'fetchers': 1,
- 'builders': 1
- }
- })
+ element_path = "elements"
+
+ cli.configure({"cache": {"quota": 10000000}, "scheduler": {"fetchers": 1, "builders": 1}})
# Create a linear build tree
- create_element_size('dep1.bst', project, element_path, [], 8000000)
- create_element_size('dep2.bst', project, element_path, ['dep1.bst'], 8000000)
- create_element_size('dep3.bst', project, element_path, ['dep2.bst'], 8000000)
- create_element_size('target.bst', project, element_path, ['dep3.bst'], 8000000)
+ create_element_size("dep1.bst", project, element_path, [], 8000000)
+ create_element_size("dep2.bst", project, element_path, ["dep1.bst"], 8000000)
+ create_element_size("dep3.bst", project, element_path, ["dep2.bst"], 8000000)
+ create_element_size("target.bst", project, element_path, ["dep3.bst"], 8000000)
# Build dep1.bst, which should fit into the cache.
- res = cli.run(project=project, args=['build', 'dep1.bst'])
+ res = cli.run(project=project, args=["build", "dep1.bst"])
res.assert_success()
# We try to build this pipeline, but it's too big for the
# cache. Since all elements are required, the build should fail.
- res = cli.run(project=project, args=['build', 'target.bst'])
+ res = cli.run(project=project, args=["build", "target.bst"])
res.assert_main_error(ErrorDomain.STREAM, None)
- res.assert_task_error(ErrorDomain.CAS, 'cache-too-full')
+ res.assert_task_error(ErrorDomain.CAS, "cache-too-full")
- states = cli.get_element_states(project, ['target.bst'])
- assert states['dep1.bst'] == 'cached'
- assert states['dep2.bst'] != 'cached'
- assert states['dep3.bst'] != 'cached'
- assert states['target.bst'] != 'cached'
+ states = cli.get_element_states(project, ["target.bst"])
+ assert states["dep1.bst"] == "cached"
+ assert states["dep2.bst"] != "cached"
+ assert states["dep3.bst"] != "cached"
+ assert states["target.bst"] != "cached"
# Ensure that only valid cache quotas make it through the loading
@@ -267,32 +245,33 @@ def test_never_delete_required(cli, datafiles):
#
# If err_domain is 'success', then err_reason is unused.
#
-@pytest.mark.parametrize("quota,err_domain,err_reason", [
- # Valid configurations
- ("1", 'success', None),
- ("1K", 'success', None),
- ("50%", 'success', None),
- ("infinity", 'success', None),
- ("0", 'success', None),
- # Invalid configurations
- ("-1", ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA),
- ("pony", ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA),
- ("200%", ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA),
-])
+@pytest.mark.parametrize(
+ "quota,err_domain,err_reason",
+ [
+ # Valid configurations
+ ("1", "success", None),
+ ("1K", "success", None),
+ ("50%", "success", None),
+ ("infinity", "success", None),
+ ("0", "success", None),
+ # Invalid configurations
+ ("-1", ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA),
+ ("pony", ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA),
+ ("200%", ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA),
+ ],
+)
@pytest.mark.datafiles(DATA_DIR)
def test_invalid_cache_quota(cli, datafiles, quota, err_domain, err_reason):
project = str(datafiles)
- os.makedirs(os.path.join(project, 'elements'))
+ os.makedirs(os.path.join(project, "elements"))
- cli.configure({
- 'cache': {
- 'quota': quota,
- },
- })
+ cli.configure(
+ {"cache": {"quota": quota,},}
+ )
- res = cli.run(project=project, args=['workspace', 'list'])
+ res = cli.run(project=project, args=["workspace", "list"])
- if err_domain == 'success':
+ if err_domain == "success":
res.assert_success()
else:
res.assert_main_error(err_domain, err_reason)
@@ -304,59 +283,47 @@ def test_invalid_cache_quota(cli, datafiles, quota, err_domain, err_reason):
@pytest.mark.datafiles(DATA_DIR)
def test_cleanup_first(cli, datafiles):
project = str(datafiles)
- element_path = 'elements'
+ element_path = "elements"
- cli.configure({
- 'cache': {
- 'quota': 10000000,
- }
- })
+ cli.configure({"cache": {"quota": 10000000,}})
# Create an element that uses almost the entire cache (an empty
# ostree cache starts at about ~10KiB, so we need a bit of a
# buffer)
- create_element_size('target.bst', project, element_path, [], 8000000)
- res = cli.run(project=project, args=['build', 'target.bst'])
+ create_element_size("target.bst", project, element_path, [], 8000000)
+ res = cli.run(project=project, args=["build", "target.bst"])
res.assert_success()
- assert cli.get_element_state(project, 'target.bst') == 'cached'
+ assert cli.get_element_state(project, "target.bst") == "cached"
# Now configure with a smaller quota, create a situation
# where the cache must be cleaned up before building anything else.
#
# Fix the fetchers and builders just to ensure a predictable
# sequence of events (although it does not effect this test)
- cli.configure({
- 'cache': {
- 'quota': 5000000,
- },
- 'scheduler': {
- 'fetchers': 1,
- 'builders': 1
- }
- })
+ cli.configure({"cache": {"quota": 5000000,}, "scheduler": {"fetchers": 1, "builders": 1}})
# Our cache is now more than full, BuildStream
- create_element_size('target2.bst', project, element_path, [], 4000000)
- res = cli.run(project=project, args=['build', 'target2.bst'])
+ create_element_size("target2.bst", project, element_path, [], 4000000)
+ res = cli.run(project=project, args=["build", "target2.bst"])
res.assert_success()
# Check that the correct element remains in the cache
- states = cli.get_element_states(project, ['target.bst', 'target2.bst'])
- assert states['target.bst'] != 'cached'
- assert states['target2.bst'] == 'cached'
+ states = cli.get_element_states(project, ["target.bst", "target2.bst"])
+ assert states["target.bst"] != "cached"
+ assert states["target2.bst"] == "cached"
@pytest.mark.datafiles(DATA_DIR)
def test_cache_usage_monitor(cli, tmpdir, datafiles):
project = str(datafiles)
- element_path = 'elements'
+ element_path = "elements"
assert get_cache_usage(cli.directory) == 0
ELEMENT_SIZE = 1000000
- create_element_size('target.bst', project, element_path, [], ELEMENT_SIZE)
- res = cli.run(project=project, args=['build', 'target.bst'])
+ create_element_size("target.bst", project, element_path, [], ELEMENT_SIZE)
+ res = cli.run(project=project, args=["build", "target.bst"])
res.assert_success()
assert get_cache_usage(cli.directory) >= ELEMENT_SIZE
diff --git a/tests/artifactcache/junctions.py b/tests/artifactcache/junctions.py
index dab69ea8d..76ba85fb5 100644
--- a/tests/artifactcache/junctions.py
+++ b/tests/artifactcache/junctions.py
@@ -11,187 +11,184 @@ from buildstream.testing import cli # pylint: disable=unused-import
from tests.testutils import create_artifact_share, assert_shared, assert_not_shared
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "junctions",
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "junctions",)
def project_set_artifacts(project, url):
- project_conf_file = os.path.join(project, 'project.conf')
+ project_conf_file = os.path.join(project, "project.conf")
project_config = _yaml.load(project_conf_file)
- project_config['artifacts'] = {
- 'url': url,
- 'push': True
- }
+ project_config["artifacts"] = {"url": url, "push": True}
_yaml.roundtrip_dump(project_config.strip_node_info(), file=project_conf_file)
@pytest.mark.datafiles(DATA_DIR)
def test_push_pull(cli, tmpdir, datafiles):
- project = os.path.join(str(datafiles), 'parent')
- base_project = os.path.join(str(project), 'base')
+ project = os.path.join(str(datafiles), "parent")
+ base_project = os.path.join(str(project), "base")
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare-parent')) as share,\
- create_artifact_share(os.path.join(str(tmpdir), 'artifactshare-base')) as base_share:
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare-parent")) as share, create_artifact_share(
+ os.path.join(str(tmpdir), "artifactshare-base")
+ ) as base_share:
# First build it without the artifact cache configured
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
assert result.exit_code == 0
# Assert that we are now cached locally
- state = cli.get_element_state(project, 'target.bst')
- assert state == 'cached'
- state = cli.get_element_state(base_project, 'base-element.bst')
- assert state == 'cached'
+ state = cli.get_element_state(project, "target.bst")
+ assert state == "cached"
+ state = cli.get_element_state(base_project, "base-element.bst")
+ assert state == "cached"
project_set_artifacts(project, share.repo)
project_set_artifacts(base_project, base_share.repo)
# Now try bst artifact push
- result = cli.run(project=project, args=['artifact', 'push', '--deps', 'all', 'target.bst'])
+ result = cli.run(project=project, args=["artifact", "push", "--deps", "all", "target.bst"])
assert result.exit_code == 0
# And finally assert that the artifacts are in the right shares
#
# In the parent project's cache
- assert_shared(cli, share, project, 'target.bst', project_name='parent')
- assert_shared(cli, share, project, 'app.bst', project_name='parent')
- assert_not_shared(cli, share, base_project, 'base-element.bst', project_name='base')
+ assert_shared(cli, share, project, "target.bst", project_name="parent")
+ assert_shared(cli, share, project, "app.bst", project_name="parent")
+ assert_not_shared(cli, share, base_project, "base-element.bst", project_name="base")
# In the junction project's cache
- assert_not_shared(cli, base_share, project, 'target.bst', project_name='parent')
- assert_not_shared(cli, base_share, project, 'app.bst', project_name='parent')
- assert_shared(cli, base_share, base_project, 'base-element.bst', project_name='base')
+ assert_not_shared(cli, base_share, project, "target.bst", project_name="parent")
+ assert_not_shared(cli, base_share, project, "app.bst", project_name="parent")
+ assert_shared(cli, base_share, base_project, "base-element.bst", project_name="base")
# Now we've pushed, delete the user's local artifact cache
# directory and try to redownload it from the share
#
- cas = os.path.join(cli.directory, 'cas')
+ cas = os.path.join(cli.directory, "cas")
shutil.rmtree(cas)
- artifact_dir = os.path.join(cli.directory, 'artifacts')
+ artifact_dir = os.path.join(cli.directory, "artifacts")
shutil.rmtree(artifact_dir)
# Assert that nothing is cached locally anymore
- state = cli.get_element_state(project, 'target.bst')
- assert state != 'cached'
- state = cli.get_element_state(base_project, 'base-element.bst')
- assert state != 'cached'
+ state = cli.get_element_state(project, "target.bst")
+ assert state != "cached"
+ state = cli.get_element_state(base_project, "base-element.bst")
+ assert state != "cached"
# Now try bst artifact pull
- result = cli.run(project=project, args=['artifact', 'pull', '--deps', 'all', 'target.bst'])
+ result = cli.run(project=project, args=["artifact", "pull", "--deps", "all", "target.bst"])
assert result.exit_code == 0
# And assert that they are again in the local cache, without having built
- state = cli.get_element_state(project, 'target.bst')
- assert state == 'cached'
- state = cli.get_element_state(base_project, 'base-element.bst')
- assert state == 'cached'
+ state = cli.get_element_state(project, "target.bst")
+ assert state == "cached"
+ state = cli.get_element_state(base_project, "base-element.bst")
+ assert state == "cached"
@pytest.mark.datafiles(DATA_DIR)
def test_caching_junction_elements(cli, tmpdir, datafiles):
- project = os.path.join(str(datafiles), 'parent')
- base_project = os.path.join(str(project), 'base')
+ project = os.path.join(str(datafiles), "parent")
+ base_project = os.path.join(str(project), "base")
# Load the junction element
- junction_element = os.path.join(project, 'base.bst')
+ junction_element = os.path.join(project, "base.bst")
junction_data = _yaml.roundtrip_load(junction_element)
# Add the "cache-junction-elements" boolean to the junction Element
- junction_data['config'] = {"cache-junction-elements": True}
+ junction_data["config"] = {"cache-junction-elements": True}
_yaml.roundtrip_dump(junction_data, junction_element)
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare-parent')) as share,\
- create_artifact_share(os.path.join(str(tmpdir), 'artifactshare-base')) as base_share:
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare-parent")) as share, create_artifact_share(
+ os.path.join(str(tmpdir), "artifactshare-base")
+ ) as base_share:
# First build it without the artifact cache configured
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
assert result.exit_code == 0
# Assert that we are now cached locally
- state = cli.get_element_state(project, 'target.bst')
- assert state == 'cached'
- state = cli.get_element_state(base_project, 'base-element.bst')
- assert state == 'cached'
+ state = cli.get_element_state(project, "target.bst")
+ assert state == "cached"
+ state = cli.get_element_state(base_project, "base-element.bst")
+ assert state == "cached"
project_set_artifacts(project, share.repo)
project_set_artifacts(base_project, base_share.repo)
# Now try bst artifact push
- result = cli.run(project=project, args=['artifact', 'push', '--deps', 'all', 'target.bst'])
+ result = cli.run(project=project, args=["artifact", "push", "--deps", "all", "target.bst"])
assert result.exit_code == 0
# And finally assert that the artifacts are in the right shares
#
# The parent project's cache should *also* contain elements from the junction
- assert_shared(cli, share, project, 'target.bst', project_name='parent')
- assert_shared(cli, share, project, 'app.bst', project_name='parent')
- assert_shared(cli, share, base_project, 'base-element.bst', project_name='base')
+ assert_shared(cli, share, project, "target.bst", project_name="parent")
+ assert_shared(cli, share, project, "app.bst", project_name="parent")
+ assert_shared(cli, share, base_project, "base-element.bst", project_name="base")
# The junction project's cache should only contain elements in the junction project
- assert_not_shared(cli, base_share, project, 'target.bst', project_name='parent')
- assert_not_shared(cli, base_share, project, 'app.bst', project_name='parent')
- assert_shared(cli, base_share, base_project, 'base-element.bst', project_name='base')
+ assert_not_shared(cli, base_share, project, "target.bst", project_name="parent")
+ assert_not_shared(cli, base_share, project, "app.bst", project_name="parent")
+ assert_shared(cli, base_share, base_project, "base-element.bst", project_name="base")
@pytest.mark.datafiles(DATA_DIR)
def test_ignore_junction_remotes(cli, tmpdir, datafiles):
- project = os.path.join(str(datafiles), 'parent')
- base_project = os.path.join(str(project), 'base')
+ project = os.path.join(str(datafiles), "parent")
+ base_project = os.path.join(str(project), "base")
# Load the junction element
- junction_element = os.path.join(project, 'base.bst')
+ junction_element = os.path.join(project, "base.bst")
junction_data = _yaml.roundtrip_load(junction_element)
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare-parent')) as share,\
- create_artifact_share(os.path.join(str(tmpdir), 'artifactshare-base')) as base_share:
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare-parent")) as share, create_artifact_share(
+ os.path.join(str(tmpdir), "artifactshare-base")
+ ) as base_share:
# Immediately declare the artifact caches in the appropriate project configs
project_set_artifacts(project, share.repo)
project_set_artifacts(base_project, base_share.repo)
# Build and populate the project remotes with their respective elements
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
assert result.exit_code == 0
# And finally assert that the artifacts are in the right shares
#
# The parent project's cache should only contain project elements
- assert_shared(cli, share, project, 'target.bst', project_name='parent')
- assert_shared(cli, share, project, 'app.bst', project_name='parent')
- assert_not_shared(cli, share, base_project, 'base-element.bst', project_name='base')
+ assert_shared(cli, share, project, "target.bst", project_name="parent")
+ assert_shared(cli, share, project, "app.bst", project_name="parent")
+ assert_not_shared(cli, share, base_project, "base-element.bst", project_name="base")
# The junction project's cache should only contain elements in the junction project
- assert_not_shared(cli, base_share, project, 'target.bst', project_name='parent')
- assert_not_shared(cli, base_share, project, 'app.bst', project_name='parent')
- assert_shared(cli, base_share, base_project, 'base-element.bst', project_name='base')
+ assert_not_shared(cli, base_share, project, "target.bst", project_name="parent")
+ assert_not_shared(cli, base_share, project, "app.bst", project_name="parent")
+ assert_shared(cli, base_share, base_project, "base-element.bst", project_name="base")
# Ensure that, from now on, we ignore junction element remotes
- junction_data['config'] = {"ignore-junction-remotes": True}
+ junction_data["config"] = {"ignore-junction-remotes": True}
_yaml.roundtrip_dump(junction_data, junction_element)
# Now delete everything from the local cache and try to
# redownload from the shares.
#
- cas = os.path.join(cli.directory, 'cas')
+ cas = os.path.join(cli.directory, "cas")
shutil.rmtree(cas)
- artifact_dir = os.path.join(cli.directory, 'artifacts')
+ artifact_dir = os.path.join(cli.directory, "artifacts")
shutil.rmtree(artifact_dir)
# Assert that nothing is cached locally anymore
- state = cli.get_element_state(project, 'target.bst')
- assert state != 'cached'
- state = cli.get_element_state(base_project, 'base-element.bst')
- assert state != 'cached'
+ state = cli.get_element_state(project, "target.bst")
+ assert state != "cached"
+ state = cli.get_element_state(base_project, "base-element.bst")
+ assert state != "cached"
# Now try bst artifact pull
- result = cli.run(project=project, args=['artifact', 'pull', '--deps', 'all', 'target.bst'])
+ result = cli.run(project=project, args=["artifact", "pull", "--deps", "all", "target.bst"])
assert result.exit_code == 0
# And assert that they are again in the local cache, without having built
- state = cli.get_element_state(project, 'target.bst')
- assert state == 'cached'
+ state = cli.get_element_state(project, "target.bst")
+ assert state == "cached"
# We shouldn't be able to download base-element!
- state = cli.get_element_state(base_project, 'base-element.bst')
- assert state != 'cached'
+ state = cli.get_element_state(base_project, "base-element.bst")
+ assert state != "cached"
diff --git a/tests/artifactcache/pull.py b/tests/artifactcache/pull.py
index a42a01bf7..e6eaec960 100644
--- a/tests/artifactcache/pull.py
+++ b/tests/artifactcache/pull.py
@@ -14,10 +14,7 @@ from tests.testutils import create_artifact_share, dummy_context
# Project directory
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project",
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",)
def tree_maker(cas, tree, directory):
@@ -27,7 +24,7 @@ def tree_maker(cas, tree, directory):
for directory_node in directory.directories:
child_directory = tree.children.add()
- with open(cas.objpath(directory_node.digest), 'rb') as f:
+ with open(cas.objpath(directory_node.digest), "rb") as f:
child_directory.ParseFromString(f.read())
tree_maker(cas, tree, child_directory)
@@ -38,19 +35,14 @@ def test_pull(cli, tmpdir, datafiles):
project_dir = str(datafiles)
# Set up an artifact cache.
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
# Configure artifact share
- cache_dir = os.path.join(str(tmpdir), 'cache')
- user_config_file = str(tmpdir.join('buildstream.conf'))
+ cache_dir = os.path.join(str(tmpdir), "cache")
+ user_config_file = str(tmpdir.join("buildstream.conf"))
user_config = {
- 'scheduler': {
- 'pushers': 1
- },
- 'artifacts': {
- 'url': share.repo,
- 'push': True,
- },
- 'cachedir': cache_dir
+ "scheduler": {"pushers": 1},
+ "artifacts": {"url": share.repo, "push": True,},
+ "cachedir": cache_dir,
}
# Write down the user configuration file
@@ -59,19 +51,19 @@ def test_pull(cli, tmpdir, datafiles):
cli.configure(user_config)
# First build the project with the artifact cache configured
- result = cli.run(project=project_dir, args=['build', 'target.bst'])
+ result = cli.run(project=project_dir, args=["build", "target.bst"])
result.assert_success()
# Assert that we are now cached locally
- assert cli.get_element_state(project_dir, 'target.bst') == 'cached'
+ assert cli.get_element_state(project_dir, "target.bst") == "cached"
# Assert that we shared/pushed the cached artifact
- assert share.get_artifact(cli.get_artifact_name(project_dir, 'test', 'target.bst'))
+ assert share.get_artifact(cli.get_artifact_name(project_dir, "test", "target.bst"))
# Delete the artifact locally
- cli.remove_artifact_from_cache(project_dir, 'target.bst')
+ cli.remove_artifact_from_cache(project_dir, "target.bst")
# Assert that we are not cached locally anymore
- assert cli.get_element_state(project_dir, 'target.bst') != 'cached'
+ assert cli.get_element_state(project_dir, "target.bst") != "cached"
with dummy_context(config=user_config_file) as context:
# Load the project
@@ -79,13 +71,13 @@ def test_pull(cli, tmpdir, datafiles):
project.ensure_fully_loaded()
# Assert that the element's artifact is **not** cached
- element = project.load_elements(['target.bst'])[0]
- element_key = cli.get_element_key(project_dir, 'target.bst')
+ element = project.load_elements(["target.bst"])[0]
+ element_key = cli.get_element_key(project_dir, "target.bst")
assert not cli.artifact.is_cached(cache_dir, element, element_key)
context.cachedir = cache_dir
- context.casdir = os.path.join(cache_dir, 'cas')
- context.tmpdir = os.path.join(cache_dir, 'tmp')
+ context.casdir = os.path.join(cache_dir, "cas")
+ context.tmpdir = os.path.join(cache_dir, "tmp")
# Load the project manually
project = Project(project_dir, context)
@@ -97,8 +89,7 @@ def test_pull(cli, tmpdir, datafiles):
# Manually setup the CAS remote
artifactcache.setup_remotes(use_config=True)
- assert artifactcache.has_push_remotes(plugin=element), \
- "No remote configured for element target.bst"
+ assert artifactcache.has_push_remotes(plugin=element), "No remote configured for element target.bst"
assert artifactcache.pull(element, element_key), "Pull operation failed"
assert cli.artifact.is_cached(cache_dir, element, element_key)
@@ -109,19 +100,14 @@ def test_pull_tree(cli, tmpdir, datafiles):
project_dir = str(datafiles)
# Set up an artifact cache.
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
# Configure artifact share
- rootcache_dir = os.path.join(str(tmpdir), 'cache')
- user_config_file = str(tmpdir.join('buildstream.conf'))
+ rootcache_dir = os.path.join(str(tmpdir), "cache")
+ user_config_file = str(tmpdir.join("buildstream.conf"))
user_config = {
- 'scheduler': {
- 'pushers': 1
- },
- 'artifacts': {
- 'url': share.repo,
- 'push': True,
- },
- 'cachedir': rootcache_dir
+ "scheduler": {"pushers": 1},
+ "artifacts": {"url": share.repo, "push": True,},
+ "cachedir": rootcache_dir,
}
# Write down the user configuration file
@@ -130,13 +116,13 @@ def test_pull_tree(cli, tmpdir, datafiles):
cli.configure(user_config)
# First build the project with the artifact cache configured
- result = cli.run(project=project_dir, args=['build', 'target.bst'])
+ result = cli.run(project=project_dir, args=["build", "target.bst"])
result.assert_success()
# Assert that we are now cached locally
- assert cli.get_element_state(project_dir, 'target.bst') == 'cached'
+ assert cli.get_element_state(project_dir, "target.bst") == "cached"
# Assert that we shared/pushed the cached artifact
- assert share.get_artifact(cli.get_artifact_name(project_dir, 'test', 'target.bst'))
+ assert share.get_artifact(cli.get_artifact_name(project_dir, "test", "target.bst"))
with dummy_context(config=user_config_file) as context:
# Load the project and CAS cache
@@ -145,8 +131,8 @@ def test_pull_tree(cli, tmpdir, datafiles):
cas = context.get_cascache()
# Assert that the element's artifact is cached
- element = project.load_elements(['target.bst'])[0]
- element_key = cli.get_element_key(project_dir, 'target.bst')
+ element = project.load_elements(["target.bst"])[0]
+ element_key = cli.get_element_key(project_dir, "target.bst")
assert cli.artifact.is_cached(rootcache_dir, element, element_key)
# Retrieve the Directory object from the cached artifact
@@ -159,7 +145,7 @@ def test_pull_tree(cli, tmpdir, datafiles):
directory = remote_execution_pb2.Directory()
- with open(cas.objpath(artifact_digest), 'rb') as f:
+ with open(cas.objpath(artifact_digest), "rb") as f:
directory.ParseFromString(f.read())
# Build the Tree object while we are still cached
@@ -172,15 +158,14 @@ def test_pull_tree(cli, tmpdir, datafiles):
assert tree_hash and tree_size
# Now delete the artifact locally
- cli.remove_artifact_from_cache(project_dir, 'target.bst')
+ cli.remove_artifact_from_cache(project_dir, "target.bst")
# Assert that we are not cached locally anymore
artifactcache.close_grpc_channels()
cas.close_grpc_channels()
- assert cli.get_element_state(project_dir, 'target.bst') != 'cached'
+ assert cli.get_element_state(project_dir, "target.bst") != "cached"
- tree_digest = remote_execution_pb2.Digest(hash=tree_hash,
- size_bytes=tree_size)
+ tree_digest = remote_execution_pb2.Digest(hash=tree_hash, size_bytes=tree_size)
# Pull the artifact using the Tree object
directory_digest = artifactcache.pull_tree(project, artifact_digest)
@@ -189,8 +174,7 @@ def test_pull_tree(cli, tmpdir, datafiles):
# Directory size now zero with AaaP and stack element commit #1cbc5e63dc
assert directory_hash and not directory_size
- directory_digest = remote_execution_pb2.Digest(hash=directory_hash,
- size_bytes=directory_size)
+ directory_digest = remote_execution_pb2.Digest(hash=directory_hash, size_bytes=directory_size)
# Ensure the entire Tree stucture has been pulled
assert os.path.exists(cas.objpath(directory_digest))
diff --git a/tests/artifactcache/push.py b/tests/artifactcache/push.py
index 62c443d61..238d5f7ef 100644
--- a/tests/artifactcache/push.py
+++ b/tests/artifactcache/push.py
@@ -14,10 +14,7 @@ from tests.testutils import create_artifact_share, create_split_share, dummy_con
# Project directory
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project",
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",)
# Push the given element and return its artifact key for assertions.
@@ -28,8 +25,8 @@ def _push(cli, cache_dir, project_dir, config_file, target):
project.ensure_fully_loaded()
# Assert that the element's artifact is cached
- element = project.load_elements(['target.bst'])[0]
- element_key = cli.get_element_key(project_dir, 'target.bst')
+ element = project.load_elements(["target.bst"])[0]
+ element_key = cli.get_element_key(project_dir, "target.bst")
assert cli.artifact.is_cached(cache_dir, element, element_key)
# Create a local artifact cache handle
@@ -46,8 +43,7 @@ def _push(cli, cache_dir, project_dir, config_file, target):
artifactcache.setup_remotes(use_config=True)
artifactcache.initialize_remotes()
- assert artifactcache.has_push_remotes(plugin=element), \
- "No remote configured for element target.bst"
+ assert artifactcache.has_push_remotes(plugin=element), "No remote configured for element target.bst"
assert element._push(), "Push operation failed"
return element_key
@@ -58,32 +54,27 @@ def test_push(cli, tmpdir, datafiles):
project_dir = str(datafiles)
# First build the project without the artifact cache configured
- result = cli.run(project=project_dir, args=['build', 'target.bst'])
+ result = cli.run(project=project_dir, args=["build", "target.bst"])
result.assert_success()
# Assert that we are now cached locally
- assert cli.get_element_state(project_dir, 'target.bst') == 'cached'
+ assert cli.get_element_state(project_dir, "target.bst") == "cached"
# Set up an artifact cache.
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
# Configure artifact share
- rootcache_dir = os.path.join(str(tmpdir), 'cache')
- user_config_file = str(tmpdir.join('buildstream.conf'))
+ rootcache_dir = os.path.join(str(tmpdir), "cache")
+ user_config_file = str(tmpdir.join("buildstream.conf"))
user_config = {
- 'scheduler': {
- 'pushers': 1
- },
- 'artifacts': {
- 'url': share.repo,
- 'push': True,
- },
- 'cachedir': rootcache_dir
+ "scheduler": {"pushers": 1},
+ "artifacts": {"url": share.repo, "push": True,},
+ "cachedir": rootcache_dir,
}
# Write down the user configuration file
_yaml.roundtrip_dump(user_config, file=user_config_file)
- element_key = _push(cli, rootcache_dir, project_dir, user_config_file, 'target.bst')
- assert share.get_artifact(cli.get_artifact_name(project_dir, 'test', 'target.bst', cache_key=element_key))
+ element_key = _push(cli, rootcache_dir, project_dir, user_config_file, "target.bst")
+ assert share.get_artifact(cli.get_artifact_name(project_dir, "test", "target.bst", cache_key=element_key))
@pytest.mark.datafiles(DATA_DIR)
@@ -91,41 +82,33 @@ def test_push_split(cli, tmpdir, datafiles):
project_dir = str(datafiles)
# First build the project without the artifact cache configured
- result = cli.run(project=project_dir, args=['build', 'target.bst'])
+ result = cli.run(project=project_dir, args=["build", "target.bst"])
result.assert_success()
# Assert that we are now cached locally
- assert cli.get_element_state(project_dir, 'target.bst') == 'cached'
+ assert cli.get_element_state(project_dir, "target.bst") == "cached"
- indexshare = os.path.join(str(tmpdir), 'indexshare')
- storageshare = os.path.join(str(tmpdir), 'storageshare')
+ indexshare = os.path.join(str(tmpdir), "indexshare")
+ storageshare = os.path.join(str(tmpdir), "storageshare")
# Set up an artifact cache.
with create_split_share(indexshare, storageshare) as (index, storage):
- rootcache_dir = os.path.join(str(tmpdir), 'cache')
+ rootcache_dir = os.path.join(str(tmpdir), "cache")
user_config = {
- 'scheduler': {
- 'pushers': 1
- },
- 'artifacts': [{
- 'url': index.repo,
- 'push': True,
- 'type': 'index'
- }, {
- 'url': storage.repo,
- 'push': True,
- 'type': 'storage'
- }],
- 'cachedir': rootcache_dir
+ "scheduler": {"pushers": 1},
+ "artifacts": [
+ {"url": index.repo, "push": True, "type": "index"},
+ {"url": storage.repo, "push": True, "type": "storage"},
+ ],
+ "cachedir": rootcache_dir,
}
- config_path = str(tmpdir.join('buildstream.conf'))
+ config_path = str(tmpdir.join("buildstream.conf"))
_yaml.roundtrip_dump(user_config, file=config_path)
- element_key = _push(cli, rootcache_dir, project_dir, config_path, 'target.bst')
- proto = index.get_artifact_proto(cli.get_artifact_name(project_dir,
- 'test',
- 'target.bst',
- cache_key=element_key))
+ element_key = _push(cli, rootcache_dir, project_dir, config_path, "target.bst")
+ proto = index.get_artifact_proto(
+ cli.get_artifact_name(project_dir, "test", "target.bst", cache_key=element_key)
+ )
assert storage.get_cas_files(proto) is not None
@@ -134,20 +117,15 @@ def test_push_message(tmpdir, datafiles):
project_dir = str(datafiles)
# Set up an artifact cache.
- artifactshare = os.path.join(str(tmpdir), 'artifactshare')
+ artifactshare = os.path.join(str(tmpdir), "artifactshare")
with create_artifact_share(artifactshare) as share:
# Configure artifact share
- rootcache_dir = os.path.join(str(tmpdir), 'cache')
- user_config_file = str(tmpdir.join('buildstream.conf'))
+ rootcache_dir = os.path.join(str(tmpdir), "cache")
+ user_config_file = str(tmpdir.join("buildstream.conf"))
user_config = {
- 'scheduler': {
- 'pushers': 1
- },
- 'artifacts': {
- 'url': share.repo,
- 'push': True,
- },
- 'cachedir': rootcache_dir
+ "scheduler": {"pushers": 1},
+ "artifacts": {"url": share.repo, "push": True,},
+ "cachedir": rootcache_dir,
}
# Write down the user configuration file
@@ -166,15 +144,16 @@ def test_push_message(tmpdir, datafiles):
artifactcache.initialize_remotes()
assert artifactcache.has_push_remotes()
- command = remote_execution_pb2.Command(arguments=['/usr/bin/gcc', '--help'],
- working_directory='/buildstream-build',
- output_directories=['/buildstream-install'])
+ command = remote_execution_pb2.Command(
+ arguments=["/usr/bin/gcc", "--help"],
+ working_directory="/buildstream-build",
+ output_directories=["/buildstream-install"],
+ )
# Push the message object
command_digest = artifactcache.push_message(project, command)
message_hash, message_size = command_digest.hash, command_digest.size_bytes
assert message_hash and message_size
- message_digest = remote_execution_pb2.Digest(hash=message_hash,
- size_bytes=message_size)
+ message_digest = remote_execution_pb2.Digest(hash=message_hash, size_bytes=message_size)
assert share.has_object(message_digest)
diff --git a/tests/cachekey/cachekey.py b/tests/cachekey/cachekey.py
index fa93f5746..b669bacaa 100644
--- a/tests/cachekey/cachekey.py
+++ b/tests/cachekey/cachekey.py
@@ -62,7 +62,7 @@ def element_filename(project_dir, element_name, alt_suffix=None):
if alt_suffix:
# Just in case...
- assert element_name.endswith('.bst')
+ assert element_name.endswith(".bst")
# Chop off the 'bst' in '.bst' and add the new suffix
element_name = element_name[:-3]
@@ -93,18 +93,20 @@ def load_expected_keys(project_dir, actual_keys, raise_error=True):
expected_keys = OrderedDict()
for element_name in actual_keys:
- expected = element_filename(project_dir, element_name, 'expected')
+ expected = element_filename(project_dir, element_name, "expected")
try:
- with open(expected, 'r') as f:
+ with open(expected, "r") as f:
expected_key = f.read()
expected_key = expected_key.strip()
except FileNotFoundError:
expected_key = None
if raise_error:
- raise Exception("Cache key test needs update, " +
- "expected file {} not found.\n\n".format(expected) +
- "Use tests/cachekey/update.py to automatically " +
- "update this test case")
+ raise Exception(
+ "Cache key test needs update, "
+ + "expected file {} not found.\n\n".format(expected)
+ + "Use tests/cachekey/update.py to automatically "
+ + "update this test case"
+ )
expected_keys[element_name] = expected_key
@@ -127,13 +129,17 @@ def assert_cache_keys(project_dir, output):
if mismatches:
info = ""
for element_name in mismatches:
- info += " Element: {}\n".format(element_name) + \
- " Expected: {}\n".format(expected_keys[element_name]) + \
- " Actual: {}\n".format(actual_keys[element_name])
+ info += (
+ " Element: {}\n".format(element_name)
+ + " Expected: {}\n".format(expected_keys[element_name])
+ + " Actual: {}\n".format(actual_keys[element_name])
+ )
- raise AssertionError("Cache key mismatches occurred:\n{}\n".format(info) +
- "Use tests/cachekey/update.py to automatically " +
- "update this test case")
+ raise AssertionError(
+ "Cache key mismatches occurred:\n{}\n".format(info)
+ + "Use tests/cachekey/update.py to automatically "
+ + "update this test case"
+ )
##############################################
@@ -141,18 +147,14 @@ def assert_cache_keys(project_dir, output):
##############################################
# Project directory
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project",
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",)
# The cache key test uses a project which exercises all plugins,
# so we cant run it at all if we dont have them installed.
#
-@pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
- reason='Cache keys depend on architecture')
-@pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
+@pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Cache keys depend on architecture")
+@pytest.mark.skipif(not IS_LINUX, reason="Only available on linux")
@pytest.mark.skipif(HAVE_BZR is False, reason="bzr is not available")
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
@pytest.mark.datafiles(DATA_DIR)
@@ -163,55 +165,48 @@ def test_cache_key(datafiles, cli):
# versions of setuptools fail to preserve symbolic links
# when creating a source distribution, causing this test
# to fail from a dist tarball.
- goodbye_link = os.path.join(project, 'files', 'local',
- 'usr', 'bin', 'goodbye')
+ goodbye_link = os.path.join(project, "files", "local", "usr", "bin", "goodbye")
os.unlink(goodbye_link)
- os.symlink('hello', goodbye_link)
+ os.symlink("hello", goodbye_link)
# pytest-datafiles does not copy mode bits
# https://github.com/omarkohl/pytest-datafiles/issues/11
os.chmod(goodbye_link, 0o755)
- result = cli.run(project=project, silent=True, args=[
- 'show',
- '--format', '%{name}::%{full-key}',
- 'target.bst'
- ])
+ result = cli.run(project=project, silent=True, args=["show", "--format", "%{name}::%{full-key}", "target.bst"])
result.assert_success()
assert_cache_keys(project, result.output)
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("first_warnings, second_warnings, identical_keys", [
- [[], [], True],
- [[], [CoreWarnings.REF_NOT_IN_TRACK], False],
- [[CoreWarnings.REF_NOT_IN_TRACK], [], False],
- [[CoreWarnings.REF_NOT_IN_TRACK], [CoreWarnings.REF_NOT_IN_TRACK], True],
- [[CoreWarnings.REF_NOT_IN_TRACK, CoreWarnings.OVERLAPS],
- [CoreWarnings.OVERLAPS, CoreWarnings.REF_NOT_IN_TRACK], True],
-])
+@pytest.mark.parametrize(
+ "first_warnings, second_warnings, identical_keys",
+ [
+ [[], [], True],
+ [[], [CoreWarnings.REF_NOT_IN_TRACK], False],
+ [[CoreWarnings.REF_NOT_IN_TRACK], [], False],
+ [[CoreWarnings.REF_NOT_IN_TRACK], [CoreWarnings.REF_NOT_IN_TRACK], True],
+ [
+ [CoreWarnings.REF_NOT_IN_TRACK, CoreWarnings.OVERLAPS],
+ [CoreWarnings.OVERLAPS, CoreWarnings.REF_NOT_IN_TRACK],
+ True,
+ ],
+ ],
+)
def test_cache_key_fatal_warnings(cli, tmpdir, first_warnings, second_warnings, identical_keys):
# Builds project, Runs bst show, gathers cache keys
def run_get_cache_key(project_name, warnings):
- config = {
- 'name': project_name,
- 'element-path': 'elements',
- 'fatal-warnings': warnings
- }
+ config = {"name": project_name, "element-path": "elements", "fatal-warnings": warnings}
project_dir = tmpdir.mkdir(project_name)
- project_config_file = str(project_dir.join('project.conf'))
+ project_config_file = str(project_dir.join("project.conf"))
_yaml.roundtrip_dump(config, file=project_config_file)
- elem_dir = project_dir.mkdir('elements')
- element_file = str(elem_dir.join('stack.bst'))
- _yaml.roundtrip_dump({'kind': 'stack'}, file=element_file)
+ elem_dir = project_dir.mkdir("elements")
+ element_file = str(elem_dir.join("stack.bst"))
+ _yaml.roundtrip_dump({"kind": "stack"}, file=element_file)
- result = cli.run(project=str(project_dir), args=[
- 'show',
- '--format', '%{name}::%{full-key}',
- 'stack.bst'
- ])
+ result = cli.run(project=str(project_dir), args=["show", "--format", "%{name}::%{full-key}", "stack.bst"])
return result.output
# Returns true if all keys are identical
@@ -226,34 +221,20 @@ def test_cache_key_fatal_warnings(cli, tmpdir, first_warnings, second_warnings,
@pytest.mark.datafiles(DATA_DIR)
def test_keys_stable_over_targets(cli, datafiles):
- root_element = 'elements/key-stability/top-level.bst'
- target1 = 'elements/key-stability/t1.bst'
- target2 = 'elements/key-stability/t2.bst'
+ root_element = "elements/key-stability/top-level.bst"
+ target1 = "elements/key-stability/t1.bst"
+ target2 = "elements/key-stability/t2.bst"
project = str(datafiles)
- full_graph_result = cli.run(project=project, args=[
- 'show',
- '--format', '%{name}::%{full-key}',
- root_element
- ])
+ full_graph_result = cli.run(project=project, args=["show", "--format", "%{name}::%{full-key}", root_element])
full_graph_result.assert_success()
all_cache_keys = parse_output_keys(full_graph_result.output)
- ordering1_result = cli.run(project=project, args=[
- 'show',
- '--format', '%{name}::%{full-key}',
- target1,
- target2
- ])
+ ordering1_result = cli.run(project=project, args=["show", "--format", "%{name}::%{full-key}", target1, target2])
ordering1_result.assert_success()
ordering1_cache_keys = parse_output_keys(ordering1_result.output)
- ordering2_result = cli.run(project=project, args=[
- 'show',
- '--format', '%{name}::%{full-key}',
- target2,
- target1
- ])
+ ordering2_result = cli.run(project=project, args=["show", "--format", "%{name}::%{full-key}", target2, target1])
ordering2_result.assert_success()
ordering2_cache_keys = parse_output_keys(ordering2_result.output)
diff --git a/tests/cachekey/update.py b/tests/cachekey/update.py
index feda5dbde..49af39fe1 100755
--- a/tests/cachekey/update.py
+++ b/tests/cachekey/update.py
@@ -25,31 +25,28 @@ except ImportError:
from .cachekey import element_filename, parse_output_keys, load_expected_keys
# Project directory
-PROJECT_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project",
-)
+PROJECT_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",)
def write_expected_key(element_name, actual_key):
- expected_file = element_filename(PROJECT_DIR, element_name, 'expected')
- with open(expected_file, 'w') as f:
+ expected_file = element_filename(PROJECT_DIR, element_name, "expected")
+ with open(expected_file, "w") as f:
f.write(actual_key)
def update_keys():
with tempfile.TemporaryDirectory(dir=PROJECT_DIR) as tmpdir:
- directory = os.path.join(tmpdir, 'cache')
+ directory = os.path.join(tmpdir, "cache")
os.makedirs(directory)
cli = Cli(directory, verbose=True)
# Run bst show
- result = cli.run(project=PROJECT_DIR, silent=True, args=[
- '--no-colors',
- 'show', '--format', '%{name}::%{full-key}',
- 'target.bst'
- ])
+ result = cli.run(
+ project=PROJECT_DIR,
+ silent=True,
+ args=["--no-colors", "show", "--format", "%{name}::%{full-key}", "target.bst"],
+ )
# Load the actual keys, and the expected ones if they exist
if not result.output:
@@ -59,7 +56,7 @@ def update_keys():
expected_keys = load_expected_keys(PROJECT_DIR, actual_keys, raise_error=False)
for element_name in actual_keys:
- expected = element_filename(PROJECT_DIR, element_name, 'expected')
+ expected = element_filename(PROJECT_DIR, element_name, "expected")
if actual_keys[element_name] != expected_keys[element_name]:
if not expected_keys[element_name]:
@@ -70,10 +67,10 @@ def update_keys():
write_expected_key(element_name, actual_keys[element_name])
-if __name__ == '__main__':
+if __name__ == "__main__":
# patch the environment BST_TEST_SUITE value to something if it's not
# present. This avoids an exception thrown at the cli level
- bst = 'BST_TEST_SUITE'
- mock_bst = os.environ.get(bst, 'True')
+ bst = "BST_TEST_SUITE"
+ mock_bst = os.environ.get(bst, "True")
with mock.patch.dict(os.environ, {**os.environ, bst: mock_bst}):
update_keys()
diff --git a/tests/conftest.py b/tests/conftest.py
index 216c83893..cfca4ad06 100755
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -45,28 +45,26 @@ from tests.testutils.repo.zip import Zip
# Implement pytest option #
#################################################
def pytest_addoption(parser):
- parser.addoption('--integration', action='store_true', default=False,
- help='Run integration tests')
+ parser.addoption("--integration", action="store_true", default=False, help="Run integration tests")
- parser.addoption('--remote-execution', action='store_true', default=False,
- help='Run remote-execution tests only')
+ parser.addoption("--remote-execution", action="store_true", default=False, help="Run remote-execution tests only")
def pytest_runtest_setup(item):
# Without --integration: skip tests not marked with 'integration'
- if not item.config.getvalue('integration'):
- if item.get_closest_marker('integration'):
- pytest.skip('skipping integration test')
+ if not item.config.getvalue("integration"):
+ if item.get_closest_marker("integration"):
+ pytest.skip("skipping integration test")
# With --remote-execution: only run tests marked with 'remoteexecution'
- if item.config.getvalue('remote_execution'):
- if not item.get_closest_marker('remoteexecution'):
- pytest.skip('skipping non remote-execution test')
+ if item.config.getvalue("remote_execution"):
+ if not item.get_closest_marker("remoteexecution"):
+ pytest.skip("skipping non remote-execution test")
# Without --remote-execution: skip tests marked with 'remoteexecution'
else:
- if item.get_closest_marker('remoteexecution'):
- pytest.skip('skipping remote-execution test')
+ if item.get_closest_marker("remoteexecution"):
+ pytest.skip("skipping remote-execution test")
#################################################
@@ -75,30 +73,29 @@ def pytest_runtest_setup(item):
#
# This is returned by the `remote_services` fixture
#
-class RemoteServices():
-
+class RemoteServices:
def __init__(self, **kwargs):
- self.action_service = kwargs.get('action_service')
- self.artifact_service = kwargs.get('artifact_service')
- self.exec_service = kwargs.get('exec_service')
- self.source_service = kwargs.get('source_service')
- self.storage_service = kwargs.get('storage_service')
+ self.action_service = kwargs.get("action_service")
+ self.artifact_service = kwargs.get("artifact_service")
+ self.exec_service = kwargs.get("exec_service")
+ self.source_service = kwargs.get("source_service")
+ self.storage_service = kwargs.get("storage_service")
-@pytest.fixture(scope='session')
+@pytest.fixture(scope="session")
def remote_services(request):
kwargs = {}
# Look for remote services configuration in environment.
- if 'ARTIFACT_CACHE_SERVICE' in os.environ:
- kwargs['artifact_service'] = os.environ.get('ARTIFACT_CACHE_SERVICE')
+ if "ARTIFACT_CACHE_SERVICE" in os.environ:
+ kwargs["artifact_service"] = os.environ.get("ARTIFACT_CACHE_SERVICE")
- if 'REMOTE_EXECUTION_SERVICE' in os.environ:
- kwargs['action_service'] = os.environ.get('REMOTE_EXECUTION_SERVICE')
- kwargs['exec_service'] = os.environ.get('REMOTE_EXECUTION_SERVICE')
- kwargs['storage_service'] = os.environ.get('REMOTE_EXECUTION_SERVICE')
+ if "REMOTE_EXECUTION_SERVICE" in os.environ:
+ kwargs["action_service"] = os.environ.get("REMOTE_EXECUTION_SERVICE")
+ kwargs["exec_service"] = os.environ.get("REMOTE_EXECUTION_SERVICE")
+ kwargs["storage_service"] = os.environ.get("REMOTE_EXECUTION_SERVICE")
- if 'SOURCE_CACHE_SERVICE' in os.environ:
- kwargs['source_service'] = os.environ.get('SOURCE_CACHE_SERVICE')
+ if "SOURCE_CACHE_SERVICE" in os.environ:
+ kwargs["source_service"] = os.environ.get("SOURCE_CACHE_SERVICE")
return RemoteServices(**kwargs)
@@ -106,10 +103,10 @@ def remote_services(request):
#################################################
# Setup for templated source tests #
#################################################
-register_repo_kind('git', Git, None)
-register_repo_kind('bzr', Bzr, None)
-register_repo_kind('tar', Tar, None)
-register_repo_kind('zip', Zip, None)
+register_repo_kind("git", Git, None)
+register_repo_kind("bzr", Bzr, None)
+register_repo_kind("tar", Tar, None)
+register_repo_kind("zip", Zip, None)
# This hook enables pytest to collect the templated source tests from
@@ -124,10 +121,10 @@ def pytest_sessionstart(session):
@pytest.fixture(scope="session", autouse=True)
def set_xdg_paths(pytestconfig):
for env_var, default in [
- ("HOME", "tmp"),
- ("XDG_CACHE_HOME", "tmp/cache"),
- ("XDG_CONFIG_HOME", "tmp/config"),
- ("XDG_DATA_HOME", "tmp/share"),
+ ("HOME", "tmp"),
+ ("XDG_CACHE_HOME", "tmp/cache"),
+ ("XDG_CONFIG_HOME", "tmp/config"),
+ ("XDG_DATA_HOME", "tmp/share"),
]:
value = os.environ.get("BST_TEST_{}".format(env_var))
if value is None:
@@ -141,10 +138,9 @@ def pytest_configure(config):
# possible. Note that some tests implicitly set the start method by using
# multiprocessing. If we wait for bst to do it, it will already be too
# late.
- if 'BST_FORCE_START_METHOD' in os.environ:
- start_method = os.environ['BST_FORCE_START_METHOD']
+ if "BST_FORCE_START_METHOD" in os.environ:
+ start_method = os.environ["BST_FORCE_START_METHOD"]
multiprocessing.set_start_method(start_method)
print(
- "Multiprocessing method set to:",
- start_method,
+ "Multiprocessing method set to:", start_method,
)
diff --git a/tests/elements/filter.py b/tests/elements/filter.py
index 99370052e..0befa46f8 100644
--- a/tests/elements/filter.py
+++ b/tests/elements/filter.py
@@ -12,188 +12,186 @@ from buildstream.testing._utils.site import HAVE_SANDBOX
from buildstream._exceptions import ErrorDomain
from buildstream import _yaml
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- 'filter',
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "filter",)
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_filter_include(datafiles, cli, tmpdir):
project = str(datafiles)
- result = cli.run(project=project, args=['build', 'output-include.bst'])
+ result = cli.run(project=project, args=["build", "output-include.bst"])
result.assert_success()
- checkout = os.path.join(tmpdir.dirname, tmpdir.basename, 'checkout')
- result = cli.run(project=project, args=['artifact', 'checkout', 'output-include.bst', '--directory', checkout])
+ checkout = os.path.join(tmpdir.dirname, tmpdir.basename, "checkout")
+ result = cli.run(project=project, args=["artifact", "checkout", "output-include.bst", "--directory", checkout])
result.assert_success()
assert os.path.exists(os.path.join(checkout, "foo"))
assert not os.path.exists(os.path.join(checkout, "bar"))
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox")
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_filter_include_dynamic(datafiles, cli, tmpdir):
project = str(datafiles)
- result = cli.run(project=project, args=['build', 'output-dynamic-include.bst'])
+ result = cli.run(project=project, args=["build", "output-dynamic-include.bst"])
result.assert_success()
- checkout = os.path.join(tmpdir.dirname, tmpdir.basename, 'checkout')
- result = cli.run(project=project, args=['artifact', 'checkout', 'output-dynamic-include.bst',
- '--directory', checkout])
+ checkout = os.path.join(tmpdir.dirname, tmpdir.basename, "checkout")
+ result = cli.run(
+ project=project, args=["artifact", "checkout", "output-dynamic-include.bst", "--directory", checkout]
+ )
result.assert_success()
assert os.path.exists(os.path.join(checkout, "foo"))
assert not os.path.exists(os.path.join(checkout, "bar"))
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_filter_exclude(datafiles, cli, tmpdir):
project = str(datafiles)
- result = cli.run(project=project, args=['build', 'output-exclude.bst'])
+ result = cli.run(project=project, args=["build", "output-exclude.bst"])
result.assert_success()
- checkout = os.path.join(tmpdir.dirname, tmpdir.basename, 'checkout')
- result = cli.run(project=project, args=['artifact', 'checkout', 'output-exclude.bst', '--directory', checkout])
+ checkout = os.path.join(tmpdir.dirname, tmpdir.basename, "checkout")
+ result = cli.run(project=project, args=["artifact", "checkout", "output-exclude.bst", "--directory", checkout])
result.assert_success()
assert not os.path.exists(os.path.join(checkout, "foo"))
assert os.path.exists(os.path.join(checkout, "bar"))
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_filter_orphans(datafiles, cli, tmpdir):
project = str(datafiles)
- result = cli.run(project=project, args=['build', 'output-orphans.bst'])
+ result = cli.run(project=project, args=["build", "output-orphans.bst"])
result.assert_success()
- checkout = os.path.join(tmpdir.dirname, tmpdir.basename, 'checkout')
- result = cli.run(project=project, args=['artifact', 'checkout', 'output-orphans.bst', '--directory', checkout])
+ checkout = os.path.join(tmpdir.dirname, tmpdir.basename, "checkout")
+ result = cli.run(project=project, args=["artifact", "checkout", "output-orphans.bst", "--directory", checkout])
result.assert_success()
assert os.path.exists(os.path.join(checkout, "baz"))
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_filter_deps_ok(datafiles, cli):
project = str(datafiles)
- result = cli.run(project=project, args=['build', 'deps-permitted.bst'])
+ result = cli.run(project=project, args=["build", "deps-permitted.bst"])
result.assert_success()
- result = cli.run(project=project,
- args=['show', '--deps=run', "--format='%{name}'", 'deps-permitted.bst'])
+ result = cli.run(project=project, args=["show", "--deps=run", "--format='%{name}'", "deps-permitted.bst"])
result.assert_success()
- assert 'output-exclude.bst' in result.output
- assert 'output-orphans.bst' in result.output
+ assert "output-exclude.bst" in result.output
+ assert "output-orphans.bst" in result.output
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_filter_forbid_sources(datafiles, cli):
project = str(datafiles)
- result = cli.run(project=project, args=['build', 'forbidden-source.bst'])
- result.assert_main_error(ErrorDomain.ELEMENT, 'element-forbidden-sources')
+ result = cli.run(project=project, args=["build", "forbidden-source.bst"])
+ result.assert_main_error(ErrorDomain.ELEMENT, "element-forbidden-sources")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_filter_forbid_multi_bdep(datafiles, cli):
project = str(datafiles)
- result = cli.run(project=project, args=['build', 'forbidden-multi-bdep.bst'])
- result.assert_main_error(ErrorDomain.ELEMENT, 'filter-bdepend-wrong-count')
+ result = cli.run(project=project, args=["build", "forbidden-multi-bdep.bst"])
+ result.assert_main_error(ErrorDomain.ELEMENT, "filter-bdepend-wrong-count")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_filter_forbid_no_bdep(datafiles, cli):
project = str(datafiles)
- result = cli.run(project=project, args=['build', 'forbidden-no-bdep.bst'])
- result.assert_main_error(ErrorDomain.ELEMENT, 'filter-bdepend-wrong-count')
+ result = cli.run(project=project, args=["build", "forbidden-no-bdep.bst"])
+ result.assert_main_error(ErrorDomain.ELEMENT, "filter-bdepend-wrong-count")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_filter_forbid_also_rdep(datafiles, cli):
project = str(datafiles)
- result = cli.run(project=project, args=['build', 'forbidden-also-rdep.bst'])
- result.assert_main_error(ErrorDomain.ELEMENT, 'filter-bdepend-also-rdepend')
+ result = cli.run(project=project, args=["build", "forbidden-also-rdep.bst"])
+ result.assert_main_error(ErrorDomain.ELEMENT, "filter-bdepend-also-rdepend")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_filter_workspace_open(datafiles, cli, tmpdir):
project = str(datafiles)
workspace_dir = os.path.join(tmpdir.dirname, tmpdir.basename, "workspace")
- result = cli.run(project=project, args=['workspace', 'open', '--directory', workspace_dir, 'deps-permitted.bst'])
+ result = cli.run(project=project, args=["workspace", "open", "--directory", workspace_dir, "deps-permitted.bst"])
result.assert_success()
assert os.path.exists(os.path.join(workspace_dir, "foo"))
assert os.path.exists(os.path.join(workspace_dir, "bar"))
assert os.path.exists(os.path.join(workspace_dir, "baz"))
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_filter_workspace_open_multi(datafiles, cli):
project = str(datafiles)
- result = cli.run(cwd=project, project=project, args=['workspace', 'open', 'deps-permitted.bst',
- 'output-orphans.bst'])
+ result = cli.run(
+ cwd=project, project=project, args=["workspace", "open", "deps-permitted.bst", "output-orphans.bst"]
+ )
result.assert_success()
assert os.path.exists(os.path.join(project, "input"))
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_filter_workspace_build(datafiles, cli, tmpdir):
project = str(datafiles)
tempdir = os.path.join(tmpdir.dirname, tmpdir.basename)
workspace_dir = os.path.join(tempdir, "workspace")
- result = cli.run(project=project, args=['workspace', 'open', '--directory', workspace_dir, 'output-orphans.bst'])
+ result = cli.run(project=project, args=["workspace", "open", "--directory", workspace_dir, "output-orphans.bst"])
result.assert_success()
src = os.path.join(workspace_dir, "foo")
dst = os.path.join(workspace_dir, "quux")
shutil.copyfile(src, dst)
- result = cli.run(project=project, args=['build', 'output-orphans.bst'])
+ result = cli.run(project=project, args=["build", "output-orphans.bst"])
result.assert_success()
checkout_dir = os.path.join(tempdir, "checkout")
- result = cli.run(project=project, args=['artifact', 'checkout', 'output-orphans.bst', '--directory', checkout_dir])
+ result = cli.run(project=project, args=["artifact", "checkout", "output-orphans.bst", "--directory", checkout_dir])
result.assert_success()
assert os.path.exists(os.path.join(checkout_dir, "quux"))
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_filter_workspace_close(datafiles, cli, tmpdir):
project = str(datafiles)
tempdir = os.path.join(tmpdir.dirname, tmpdir.basename)
workspace_dir = os.path.join(tempdir, "workspace")
- result = cli.run(project=project, args=['workspace', 'open', '--directory', workspace_dir, 'output-orphans.bst'])
+ result = cli.run(project=project, args=["workspace", "open", "--directory", workspace_dir, "output-orphans.bst"])
result.assert_success()
src = os.path.join(workspace_dir, "foo")
dst = os.path.join(workspace_dir, "quux")
shutil.copyfile(src, dst)
- result = cli.run(project=project, args=['workspace', 'close', 'deps-permitted.bst'])
+ result = cli.run(project=project, args=["workspace", "close", "deps-permitted.bst"])
result.assert_success()
- result = cli.run(project=project, args=['build', 'output-orphans.bst'])
+ result = cli.run(project=project, args=["build", "output-orphans.bst"])
result.assert_success()
checkout_dir = os.path.join(tempdir, "checkout")
- result = cli.run(project=project, args=['artifact', 'checkout', 'output-orphans.bst', '--directory', checkout_dir])
+ result = cli.run(project=project, args=["artifact", "checkout", "output-orphans.bst", "--directory", checkout_dir])
result.assert_success()
assert not os.path.exists(os.path.join(checkout_dir, "quux"))
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_filter_workspace_reset(datafiles, cli, tmpdir):
project = str(datafiles)
tempdir = os.path.join(tmpdir.dirname, tmpdir.basename)
workspace_dir = os.path.join(tempdir, "workspace")
- result = cli.run(project=project, args=['workspace', 'open', '--directory', workspace_dir, 'output-orphans.bst'])
+ result = cli.run(project=project, args=["workspace", "open", "--directory", workspace_dir, "output-orphans.bst"])
result.assert_success()
src = os.path.join(workspace_dir, "foo")
dst = os.path.join(workspace_dir, "quux")
shutil.copyfile(src, dst)
- result = cli.run(project=project, args=['workspace', 'reset', 'deps-permitted.bst'])
+ result = cli.run(project=project, args=["workspace", "reset", "deps-permitted.bst"])
result.assert_success()
- result = cli.run(project=project, args=['build', 'output-orphans.bst'])
+ result = cli.run(project=project, args=["build", "output-orphans.bst"])
result.assert_success()
checkout_dir = os.path.join(tempdir, "checkout")
- result = cli.run(project=project, args=['artifact', 'checkout', 'output-orphans.bst', '--directory', checkout_dir])
+ result = cli.run(project=project, args=["artifact", "checkout", "output-orphans.bst", "--directory", checkout_dir])
result.assert_success()
assert not os.path.exists(os.path.join(checkout_dir, "quux"))
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_filter_track(datafiles, cli, tmpdir):
- repo = create_repo('git', str(tmpdir))
+ repo = create_repo("git", str(tmpdir))
ref = repo.create(os.path.join(str(datafiles), "files"))
elements_dir = os.path.join(str(tmpdir), "elements")
project = str(tmpdir)
@@ -214,26 +212,16 @@ def test_filter_track(datafiles, cli, tmpdir):
input_file = os.path.join(elements_dir, input_name)
_yaml.roundtrip_dump(input_config, input_file)
- filter1_config = {
- "kind": "filter",
- "depends": [
- {"filename": input_name, "type": "build"}
- ]
- }
+ filter1_config = {"kind": "filter", "depends": [{"filename": input_name, "type": "build"}]}
filter1_file = os.path.join(elements_dir, "filter1.bst")
_yaml.roundtrip_dump(filter1_config, filter1_file)
- filter2_config = {
- "kind": "filter",
- "depends": [
- {"filename": "filter1.bst", "type": "build"}
- ]
- }
+ filter2_config = {"kind": "filter", "depends": [{"filename": "filter1.bst", "type": "build"}]}
filter2_file = os.path.join(elements_dir, "filter2.bst")
_yaml.roundtrip_dump(filter2_config, filter2_file)
# Assert that a fetch is needed
- assert cli.get_element_state(project, input_name) == 'no reference'
+ assert cli.get_element_state(project, input_name) == "no reference"
# Now try to track it
result = cli.run(project=project, args=["source", "track", "filter2.bst"])
@@ -241,14 +229,14 @@ def test_filter_track(datafiles, cli, tmpdir):
# Now check that a ref field exists
new_input = _yaml.load(input_file)
- source_node = new_input.get_sequence('sources').mapping_at(0)
- new_input_ref = source_node.get_str('ref')
+ source_node = new_input.get_sequence("sources").mapping_at(0)
+ new_input_ref = source_node.get_str("ref")
assert new_input_ref == ref
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_filter_track_excepted(datafiles, cli, tmpdir):
- repo = create_repo('git', str(tmpdir))
+ repo = create_repo("git", str(tmpdir))
repo.create(os.path.join(str(datafiles), "files"))
elements_dir = os.path.join(str(tmpdir), "elements")
project = str(tmpdir)
@@ -269,26 +257,16 @@ def test_filter_track_excepted(datafiles, cli, tmpdir):
input_file = os.path.join(elements_dir, input_name)
_yaml.roundtrip_dump(input_config, input_file)
- filter1_config = {
- "kind": "filter",
- "depends": [
- {"filename": input_name, "type": "build"}
- ]
- }
+ filter1_config = {"kind": "filter", "depends": [{"filename": input_name, "type": "build"}]}
filter1_file = os.path.join(elements_dir, "filter1.bst")
_yaml.roundtrip_dump(filter1_config, filter1_file)
- filter2_config = {
- "kind": "filter",
- "depends": [
- {"filename": "filter1.bst", "type": "build"}
- ]
- }
+ filter2_config = {"kind": "filter", "depends": [{"filename": "filter1.bst", "type": "build"}]}
filter2_file = os.path.join(elements_dir, "filter2.bst")
_yaml.roundtrip_dump(filter2_config, filter2_file)
# Assert that a fetch is needed
- assert cli.get_element_state(project, input_name) == 'no reference'
+ assert cli.get_element_state(project, input_name) == "no reference"
# Now try to track it
result = cli.run(project=project, args=["source", "track", "filter2.bst", "--except", "input.bst"])
@@ -296,13 +274,13 @@ def test_filter_track_excepted(datafiles, cli, tmpdir):
# Now check that a ref field exists
new_input = _yaml.load(input_file)
- source_node = new_input.get_sequence('sources').mapping_at(0)
- assert 'ref' not in source_node
+ source_node = new_input.get_sequence("sources").mapping_at(0)
+ assert "ref" not in source_node
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_filter_track_multi_to_one(datafiles, cli, tmpdir):
- repo = create_repo('git', str(tmpdir))
+ repo = create_repo("git", str(tmpdir))
ref = repo.create(os.path.join(str(datafiles), "files"))
elements_dir = os.path.join(str(tmpdir), "elements")
project = str(tmpdir)
@@ -323,26 +301,16 @@ def test_filter_track_multi_to_one(datafiles, cli, tmpdir):
input_file = os.path.join(elements_dir, input_name)
_yaml.roundtrip_dump(input_config, input_file)
- filter1_config = {
- "kind": "filter",
- "depends": [
- {"filename": input_name, "type": "build"}
- ]
- }
+ filter1_config = {"kind": "filter", "depends": [{"filename": input_name, "type": "build"}]}
filter1_file = os.path.join(elements_dir, "filter1.bst")
_yaml.roundtrip_dump(filter1_config, filter1_file)
- filter2_config = {
- "kind": "filter",
- "depends": [
- {"filename": input_name, "type": "build"}
- ]
- }
+ filter2_config = {"kind": "filter", "depends": [{"filename": input_name, "type": "build"}]}
filter2_file = os.path.join(elements_dir, "filter2.bst")
_yaml.roundtrip_dump(filter2_config, filter2_file)
# Assert that a fetch is needed
- assert cli.get_element_state(project, input_name) == 'no reference'
+ assert cli.get_element_state(project, input_name) == "no reference"
# Now try to track it
result = cli.run(project=project, args=["source", "track", "filter1.bst", "filter2.bst"])
@@ -350,14 +318,14 @@ def test_filter_track_multi_to_one(datafiles, cli, tmpdir):
# Now check that a ref field exists
new_input = _yaml.load(input_file)
- source_node = new_input.get_sequence('sources').mapping_at(0)
- new_ref = source_node.get_str('ref')
+ source_node = new_input.get_sequence("sources").mapping_at(0)
+ new_ref = source_node.get_str("ref")
assert new_ref == ref
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_filter_track_multi(datafiles, cli, tmpdir):
- repo = create_repo('git', str(tmpdir))
+ repo = create_repo("git", str(tmpdir))
ref = repo.create(os.path.join(str(datafiles), "files"))
elements_dir = os.path.join(str(tmpdir), "elements")
project = str(tmpdir)
@@ -383,21 +351,11 @@ def test_filter_track_multi(datafiles, cli, tmpdir):
input2_file = os.path.join(elements_dir, input2_name)
_yaml.roundtrip_dump(input2_config, input2_file)
- filter1_config = {
- "kind": "filter",
- "depends": [
- {"filename": input_name, "type": "build"}
- ]
- }
+ filter1_config = {"kind": "filter", "depends": [{"filename": input_name, "type": "build"}]}
filter1_file = os.path.join(elements_dir, "filter1.bst")
_yaml.roundtrip_dump(filter1_config, filter1_file)
- filter2_config = {
- "kind": "filter",
- "depends": [
- {"filename": input2_name, "type": "build"}
- ]
- }
+ filter2_config = {"kind": "filter", "depends": [{"filename": input2_name, "type": "build"}]}
filter2_file = os.path.join(elements_dir, "filter2.bst")
_yaml.roundtrip_dump(filter2_config, filter2_file)
@@ -415,19 +373,19 @@ def test_filter_track_multi(datafiles, cli, tmpdir):
# Now check that a ref field exists
new_input = _yaml.load(input_file)
- source_node = new_input.get_sequence('sources').mapping_at(0)
- new_ref = source_node.get_str('ref')
+ source_node = new_input.get_sequence("sources").mapping_at(0)
+ new_ref = source_node.get_str("ref")
assert new_ref == ref
new_input2 = _yaml.load(input2_file)
- source_node2 = new_input2.get_sequence('sources').mapping_at(0)
- new_ref2 = source_node2.get_str('ref')
+ source_node2 = new_input2.get_sequence("sources").mapping_at(0)
+ new_ref2 = source_node2.get_str("ref")
assert new_ref2 == ref
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_filter_track_multi_exclude(datafiles, cli, tmpdir):
- repo = create_repo('git', str(tmpdir))
+ repo = create_repo("git", str(tmpdir))
ref = repo.create(os.path.join(str(datafiles), "files"))
elements_dir = os.path.join(str(tmpdir), "elements")
project = str(tmpdir)
@@ -453,21 +411,11 @@ def test_filter_track_multi_exclude(datafiles, cli, tmpdir):
input2_file = os.path.join(elements_dir, input2_name)
_yaml.roundtrip_dump(input2_config, input2_file)
- filter1_config = {
- "kind": "filter",
- "depends": [
- {"filename": input_name, "type": "build"}
- ]
- }
+ filter1_config = {"kind": "filter", "depends": [{"filename": input_name, "type": "build"}]}
filter1_file = os.path.join(elements_dir, "filter1.bst")
_yaml.roundtrip_dump(filter1_config, filter1_file)
- filter2_config = {
- "kind": "filter",
- "depends": [
- {"filename": input2_name, "type": "build"}
- ]
- }
+ filter2_config = {"kind": "filter", "depends": [{"filename": input2_name, "type": "build"}]}
filter2_file = os.path.join(elements_dir, "filter2.bst")
_yaml.roundtrip_dump(filter2_config, filter2_file)
@@ -484,25 +432,26 @@ def test_filter_track_multi_exclude(datafiles, cli, tmpdir):
# Now check that a ref field exists
new_input = _yaml.load(input_file)
- source_node = new_input.get_sequence('sources').mapping_at(0)
- assert 'ref' not in source_node
+ source_node = new_input.get_sequence("sources").mapping_at(0)
+ assert "ref" not in source_node
new_input2 = _yaml.load(input2_file)
- source_node2 = new_input2.get_sequence('sources').mapping_at(0)
- new_ref2 = source_node2.get_str('ref')
+ source_node2 = new_input2.get_sequence("sources").mapping_at(0)
+ new_ref2 = source_node2.get_str("ref")
assert new_ref2 == ref
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_filter_include_with_indirect_deps(datafiles, cli, tmpdir):
project = str(datafiles)
- result = cli.run(project=project, args=[
- 'build', 'output-include-with-indirect-deps.bst'])
+ result = cli.run(project=project, args=["build", "output-include-with-indirect-deps.bst"])
result.assert_success()
- checkout = os.path.join(tmpdir.dirname, tmpdir.basename, 'checkout')
- result = cli.run(project=project, args=[
- 'artifact', 'checkout', 'output-include-with-indirect-deps.bst', '--directory', checkout])
+ checkout = os.path.join(tmpdir.dirname, tmpdir.basename, "checkout")
+ result = cli.run(
+ project=project,
+ args=["artifact", "checkout", "output-include-with-indirect-deps.bst", "--directory", checkout],
+ )
result.assert_success()
# direct dependencies should be staged and filtered
@@ -513,46 +462,50 @@ def test_filter_include_with_indirect_deps(datafiles, cli, tmpdir):
assert not os.path.exists(os.path.join(checkout, "bar"))
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_filter_fails_for_nonexisting_domain(datafiles, cli):
project = str(datafiles)
- result = cli.run(project=project, args=['build', 'output-include-nonexistent-domain.bst'])
+ result = cli.run(project=project, args=["build", "output-include-nonexistent-domain.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
error = "Unknown domains were used in output-include-nonexistent-domain.bst [line 7 column 2]"
assert error in result.stderr
- assert '- unknown_file' in result.stderr
+ assert "- unknown_file" in result.stderr
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_filter_pass_integration(datafiles, cli):
project = str(datafiles)
# Explicitly not passing integration commands should be fine
- result = cli.run(project=project, args=['build', 'no-pass-integration.bst'])
+ result = cli.run(project=project, args=["build", "no-pass-integration.bst"])
result.assert_success()
# Passing integration commands should build nicely
- result = cli.run(project=project, args=['build', 'pass-integration.bst'])
+ result = cli.run(project=project, args=["build", "pass-integration.bst"])
result.assert_success()
# Checking out elements which don't pass integration commands should still work
- checkout_dir = os.path.join(project, 'no-pass')
- result = cli.run(project=project, args=['artifact', 'checkout', '--integrate',
- '--directory', checkout_dir, 'no-pass-integration.bst'])
+ checkout_dir = os.path.join(project, "no-pass")
+ result = cli.run(
+ project=project,
+ args=["artifact", "checkout", "--integrate", "--directory", checkout_dir, "no-pass-integration.bst"],
+ )
result.assert_success()
# Checking out the artifact should fail if we run integration commands, as
# the staged artifacts don't have a shell
- checkout_dir = os.path.join(project, 'pass')
- result = cli.run(project=project, args=['artifact', 'checkout', '--integrate',
- '--directory', checkout_dir, 'pass-integration.bst'])
+ checkout_dir = os.path.join(project, "pass")
+ result = cli.run(
+ project=project,
+ args=["artifact", "checkout", "--integrate", "--directory", checkout_dir, "pass-integration.bst"],
+ )
result.assert_main_error(ErrorDomain.STREAM, "missing-command")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_filter_stack_depend_failure(datafiles, cli):
project = str(datafiles)
- result = cli.run(project=project, args=['build', 'forbidden-stack-dep.bst'])
+ result = cli.run(project=project, args=["build", "forbidden-stack-dep.bst"])
result.assert_main_error(ErrorDomain.ELEMENT, "filter-bdepend-no-artifact")
diff --git a/tests/elements/filter/basic/element_plugins/dynamic.py b/tests/elements/filter/basic/element_plugins/dynamic.py
index fe83d7295..bf079111f 100644
--- a/tests/elements/filter/basic/element_plugins/dynamic.py
+++ b/tests/elements/filter/basic/element_plugins/dynamic.py
@@ -4,14 +4,14 @@ from buildstream import Element, Scope
# Copies files from the dependent element but inserts split-rules using dynamic data
class DynamicElement(Element):
def configure(self, node):
- node.validate_keys(['split-rules'])
- self.split_rules = {key: value.as_str_list() for key, value in node.get_mapping('split-rules').items()}
+ node.validate_keys(["split-rules"])
+ self.split_rules = {key: value.as_str_list() for key, value in node.get_mapping("split-rules").items()}
def preflight(self):
pass
def get_unique_key(self):
- return {'split-rules': self.split_rules}
+ return {"split-rules": self.split_rules}
def configure_sandbox(self, sandbox):
pass
diff --git a/tests/examples/autotools.py b/tests/examples/autotools.py
index 45783b30c..d4c12ecac 100644
--- a/tests/examples/autotools.py
+++ b/tests/examples/autotools.py
@@ -10,48 +10,52 @@ from buildstream.testing._utils.site import IS_LINUX, MACHINE_ARCH, HAVE_SANDBOX
pytestmark = pytest.mark.integration
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)), '..', '..', 'doc', 'examples', 'autotools'
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "doc", "examples", "autotools")
# Tests a build of the autotools amhello project on a alpine-linux base runtime
-@pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
- reason='Examples are written for x86-64')
-@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with sandbox')
-@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
+@pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64")
+@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with sandbox")
+@pytest.mark.skipif(HAVE_SANDBOX == "chroot", reason="This test is not meant to work with chroot sandbox")
@pytest.mark.datafiles(DATA_DIR)
def test_autotools_build(cli, datafiles):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout')
+ checkout = os.path.join(cli.directory, "checkout")
# Check that the project can be built correctly.
- result = cli.run(project=project, args=['build', 'hello.bst'])
+ result = cli.run(project=project, args=["build", "hello.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'hello.bst', '--directory', checkout])
+ result = cli.run(project=project, args=["artifact", "checkout", "hello.bst", "--directory", checkout])
result.assert_success()
- assert_contains(checkout, ['/usr', '/usr/lib', '/usr/bin',
- '/usr/share',
- '/usr/bin/hello',
- '/usr/share/doc', '/usr/share/doc/amhello',
- '/usr/share/doc/amhello/README'])
+ assert_contains(
+ checkout,
+ [
+ "/usr",
+ "/usr/lib",
+ "/usr/bin",
+ "/usr/share",
+ "/usr/bin/hello",
+ "/usr/share/doc",
+ "/usr/share/doc/amhello",
+ "/usr/share/doc/amhello/README",
+ ],
+ )
# Test running an executable built with autotools.
-@pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
- reason='Examples are written for x86-64')
-@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
-@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
+@pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64")
+@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with sandbox")
+@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox")
+@pytest.mark.skipif(HAVE_SANDBOX == "chroot", reason="This test is not meant to work with chroot sandbox")
@pytest.mark.datafiles(DATA_DIR)
def test_autotools_run(cli, datafiles):
project = str(datafiles)
- result = cli.run(project=project, args=['build', 'hello.bst'])
+ result = cli.run(project=project, args=["build", "hello.bst"])
result.assert_success()
- result = cli.run(project=project, args=['shell', 'hello.bst', 'hello'])
+ result = cli.run(project=project, args=["shell", "hello.bst", "hello"])
result.assert_success()
- assert result.output == 'Hello World!\nThis is amhello 1.0.\n'
+ assert result.output == "Hello World!\nThis is amhello 1.0.\n"
diff --git a/tests/examples/developing.py b/tests/examples/developing.py
index 53a554b86..d33a26425 100644
--- a/tests/examples/developing.py
+++ b/tests/examples/developing.py
@@ -11,93 +11,86 @@ import tests.testutils.patch as patch
pytestmark = pytest.mark.integration
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)), '..', '..', 'doc', 'examples', 'developing'
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "doc", "examples", "developing")
# Test that the project builds successfully
-@pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
- reason='Examples are written for x86-64')
-@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with SANDBOX')
-@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This is not meant to work with chroot')
+@pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64")
+@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with SANDBOX")
+@pytest.mark.skipif(HAVE_SANDBOX == "chroot", reason="This is not meant to work with chroot")
@pytest.mark.datafiles(DATA_DIR)
def test_autotools_build(cli, datafiles):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout')
+ checkout = os.path.join(cli.directory, "checkout")
# Check that the project can be built correctly.
- result = cli.run(project=project, args=['build', 'hello.bst'])
+ result = cli.run(project=project, args=["build", "hello.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'hello.bst', '--directory', checkout])
+ result = cli.run(project=project, args=["artifact", "checkout", "hello.bst", "--directory", checkout])
result.assert_success()
- assert_contains(checkout, ['/usr', '/usr/lib', '/usr/bin',
- '/usr/share',
- '/usr/bin/hello'])
+ assert_contains(checkout, ["/usr", "/usr/lib", "/usr/bin", "/usr/share", "/usr/bin/hello"])
# Test the unmodified hello command works as expected.
-@pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
- reason='Examples are written for x86-64')
-@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with SANDBOX')
-@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This is not meant to work with chroot')
+@pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64")
+@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with SANDBOX")
+@pytest.mark.skipif(HAVE_SANDBOX == "chroot", reason="This is not meant to work with chroot")
@pytest.mark.datafiles(DATA_DIR)
def test_run_unmodified_hello(cli, datafiles):
project = str(datafiles)
- result = cli.run(project=project, args=['build', 'hello.bst'])
+ result = cli.run(project=project, args=["build", "hello.bst"])
result.assert_success()
- result = cli.run(project=project, args=['shell', 'hello.bst', 'hello'])
+ result = cli.run(project=project, args=["shell", "hello.bst", "hello"])
result.assert_success()
- assert result.output == 'Hello World\n'
+ assert result.output == "Hello World\n"
# Test opening a workspace
-@pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
+@pytest.mark.skipif(not IS_LINUX, reason="Only available on linux")
@pytest.mark.datafiles(DATA_DIR)
def test_open_workspace(cli, tmpdir, datafiles):
project = str(datafiles)
workspace_dir = os.path.join(str(tmpdir), "workspace_hello")
- result = cli.run(project=project, args=['workspace', 'open', '-f', '--directory', workspace_dir, 'hello.bst', ])
+ result = cli.run(project=project, args=["workspace", "open", "-f", "--directory", workspace_dir, "hello.bst",])
result.assert_success()
- result = cli.run(project=project, args=['workspace', 'list'])
+ result = cli.run(project=project, args=["workspace", "list"])
result.assert_success()
- result = cli.run(project=project, args=['workspace', 'close', '--remove-dir', 'hello.bst'])
+ result = cli.run(project=project, args=["workspace", "close", "--remove-dir", "hello.bst"])
result.assert_success()
# Test making a change using the workspace
-@pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
- reason='Examples are written for x86-64')
-@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with SANDBOX')
-@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This is not meant to work with chroot')
+@pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64")
+@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with SANDBOX")
+@pytest.mark.skipif(HAVE_SANDBOX == "chroot", reason="This is not meant to work with chroot")
@pytest.mark.datafiles(DATA_DIR)
def test_make_change_in_workspace(cli, tmpdir, datafiles):
project = str(datafiles)
workspace_dir = os.path.join(str(tmpdir), "workspace_hello")
- result = cli.run(project=project, args=['workspace', 'open', '-f', '--directory', workspace_dir, 'hello.bst'])
+ result = cli.run(project=project, args=["workspace", "open", "-f", "--directory", workspace_dir, "hello.bst"])
result.assert_success()
- result = cli.run(project=project, args=['workspace', 'list'])
+ result = cli.run(project=project, args=["workspace", "list"])
result.assert_success()
patch_target = os.path.join(workspace_dir, "hello.c")
patch_source = os.path.join(project, "update.patch")
patch.apply(patch_target, patch_source)
- result = cli.run(project=project, args=['build', 'hello.bst'])
+ result = cli.run(project=project, args=["build", "hello.bst"])
result.assert_success()
- result = cli.run(project=project, args=['shell', 'hello.bst', '--', 'hello'])
+ result = cli.run(project=project, args=["shell", "hello.bst", "--", "hello"])
result.assert_success()
- assert result.output == 'Hello World\nWe can use workspaces!\n'
+ assert result.output == "Hello World\nWe can use workspaces!\n"
- result = cli.run(project=project, args=['workspace', 'close', '--remove-dir', 'hello.bst'])
+ result = cli.run(project=project, args=["workspace", "close", "--remove-dir", "hello.bst"])
result.assert_success()
diff --git a/tests/examples/first-project.py b/tests/examples/first-project.py
index 84ab7aa61..8783c4ae3 100644
--- a/tests/examples/first-project.py
+++ b/tests/examples/first-project.py
@@ -12,21 +12,19 @@ from buildstream.testing._utils.site import IS_LINUX
pytestmark = pytest.mark.integration
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)), '..', '..', 'doc', 'examples', 'first-project'
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "doc", "examples", "first-project")
-@pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
+@pytest.mark.skipif(not IS_LINUX, reason="Only available on linux")
@pytest.mark.datafiles(DATA_DIR)
def test_first_project_build_checkout(cli, datafiles):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout')
+ checkout = os.path.join(cli.directory, "checkout")
- result = cli.run(project=project, args=['build', 'hello.bst'])
+ result = cli.run(project=project, args=["build", "hello.bst"])
assert result.exit_code == 0
- result = cli.run(project=project, args=['artifact', 'checkout', 'hello.bst', '--directory', checkout])
+ result = cli.run(project=project, args=["artifact", "checkout", "hello.bst", "--directory", checkout])
assert result.exit_code == 0
- assert_contains(checkout, ['/hello.world'])
+ assert_contains(checkout, ["/hello.world"])
diff --git a/tests/examples/flatpak-autotools.py b/tests/examples/flatpak-autotools.py
index 2418807c0..d20c7e136 100644
--- a/tests/examples/flatpak-autotools.py
+++ b/tests/examples/flatpak-autotools.py
@@ -13,12 +13,13 @@ pytestmark = pytest.mark.integration
DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)), '..', '..', 'doc', 'examples', 'flatpak-autotools'
+ os.path.dirname(os.path.realpath(__file__)), "..", "..", "doc", "examples", "flatpak-autotools"
)
try:
from bst_plugins_experimental.sources import _ostree # pylint: disable=unused-import
+
# Even when we have the plugin, it might be missing dependencies. This requires
# bst_plugins_experimantal to be fully installed, with host ostree dependencies
HAVE_OSTREE_PLUGIN = True
@@ -44,40 +45,46 @@ def workaround_setuptools_bug(project):
# Test that a build upon flatpak runtime 'works' - we use the autotools sample
# amhello project for this.
-@pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
- reason='Examples are written for x86-64')
-@pytest.mark.skipif(not IS_LINUX or not HAVE_OSTREE_PLUGIN, reason='Only available on linux with ostree')
+@pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64")
+@pytest.mark.skipif(not IS_LINUX or not HAVE_OSTREE_PLUGIN, reason="Only available on linux with ostree")
@pytest.mark.datafiles(DATA_DIR)
def test_autotools_build(cli, datafiles):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout')
+ checkout = os.path.join(cli.directory, "checkout")
workaround_setuptools_bug(project)
- result = cli.run(project=project, args=['build', 'hello.bst'])
+ result = cli.run(project=project, args=["build", "hello.bst"])
assert result.exit_code == 0
- result = cli.run(project=project, args=['artifact', 'checkout', 'hello.bst', '--directory', checkout])
+ result = cli.run(project=project, args=["artifact", "checkout", "hello.bst", "--directory", checkout])
assert result.exit_code == 0
- assert_contains(checkout, ['/usr', '/usr/lib', '/usr/bin',
- '/usr/share',
- '/usr/bin/hello', '/usr/share/doc',
- '/usr/share/doc/amhello',
- '/usr/share/doc/amhello/README'])
+ assert_contains(
+ checkout,
+ [
+ "/usr",
+ "/usr/lib",
+ "/usr/bin",
+ "/usr/share",
+ "/usr/bin/hello",
+ "/usr/share/doc",
+ "/usr/share/doc/amhello",
+ "/usr/share/doc/amhello/README",
+ ],
+ )
# Test running an executable built with autotools
-@pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
- reason='Examples are written for x86-64')
-@pytest.mark.skipif(not IS_LINUX or not HAVE_OSTREE_PLUGIN, reason='Only available on linux with ostree')
+@pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64")
+@pytest.mark.skipif(not IS_LINUX or not HAVE_OSTREE_PLUGIN, reason="Only available on linux with ostree")
@pytest.mark.datafiles(DATA_DIR)
def test_autotools_run(cli, datafiles):
project = str(datafiles)
workaround_setuptools_bug(project)
- result = cli.run(project=project, args=['build', 'hello.bst'])
+ result = cli.run(project=project, args=["build", "hello.bst"])
assert result.exit_code == 0
- result = cli.run(project=project, args=['shell', 'hello.bst', '/usr/bin/hello'])
+ result = cli.run(project=project, args=["shell", "hello.bst", "/usr/bin/hello"])
assert result.exit_code == 0
- assert result.output == 'Hello World!\nThis is amhello 1.0.\n'
+ assert result.output == "Hello World!\nThis is amhello 1.0.\n"
diff --git a/tests/examples/integration-commands.py b/tests/examples/integration-commands.py
index 1ed888b5d..2bdbfb3ec 100644
--- a/tests/examples/integration-commands.py
+++ b/tests/examples/integration-commands.py
@@ -10,34 +10,32 @@ from buildstream.testing._utils.site import IS_LINUX, MACHINE_ARCH, HAVE_SANDBOX
pytestmark = pytest.mark.integration
DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)), '..', '..', 'doc', 'examples', 'integration-commands'
+ os.path.dirname(os.path.realpath(__file__)), "..", "..", "doc", "examples", "integration-commands"
)
-@pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
- reason='Examples are written for x86-64')
-@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with sandbox')
-@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
+@pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64")
+@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with sandbox")
+@pytest.mark.skipif(HAVE_SANDBOX == "chroot", reason="This test is not meant to work with chroot sandbox")
@pytest.mark.datafiles(DATA_DIR)
def test_integration_commands_build(cli, datafiles):
project = str(datafiles)
- result = cli.run(project=project, args=['build', 'hello.bst'])
+ result = cli.run(project=project, args=["build", "hello.bst"])
assert result.exit_code == 0
# Test running the executable
-@pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
- reason='Examples are written for x86-64')
-@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with sandbox')
-@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
+@pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64")
+@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with sandbox")
+@pytest.mark.skipif(HAVE_SANDBOX == "chroot", reason="This test is not meant to work with chroot sandbox")
@pytest.mark.datafiles(DATA_DIR)
def test_integration_commands_run(cli, datafiles):
project = str(datafiles)
- result = cli.run(project=project, args=['build', 'hello.bst'])
+ result = cli.run(project=project, args=["build", "hello.bst"])
assert result.exit_code == 0
- result = cli.run(project=project, args=['shell', 'hello.bst', '--', 'hello', 'pony'])
+ result = cli.run(project=project, args=["shell", "hello.bst", "--", "hello", "pony"])
assert result.exit_code == 0
- assert result.output == 'Hello pony\n'
+ assert result.output == "Hello pony\n"
diff --git a/tests/examples/junctions.py b/tests/examples/junctions.py
index 18bf4da4f..28b7a6108 100644
--- a/tests/examples/junctions.py
+++ b/tests/examples/junctions.py
@@ -9,52 +9,48 @@ from buildstream.testing._utils.site import IS_LINUX, MACHINE_ARCH, HAVE_SANDBOX
pytestmark = pytest.mark.integration
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)), '..', '..', 'doc', 'examples', 'junctions'
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "doc", "examples", "junctions")
# Test that the project builds successfully
-@pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
- reason='Examples are written for x86-64')
-@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with bubblewrap')
-@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
+@pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64")
+@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with bubblewrap")
+@pytest.mark.skipif(HAVE_SANDBOX == "chroot", reason="This test is not meant to work with chroot sandbox")
@pytest.mark.datafiles(DATA_DIR)
def test_build(cli, datafiles):
project = str(datafiles)
- result = cli.run(project=project, args=['build', 'callHello.bst'])
+ result = cli.run(project=project, args=["build", "callHello.bst"])
result.assert_success()
# Test the callHello script works as expected.
-@pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
- reason='Examples are written for x86-64')
-@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with bubblewrap')
-@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
+@pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64")
+@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with bubblewrap")
+@pytest.mark.skipif(HAVE_SANDBOX == "chroot", reason="This test is not meant to work with chroot sandbox")
@pytest.mark.datafiles(DATA_DIR)
def test_shell_call_hello(cli, datafiles):
project = str(datafiles)
- result = cli.run(project=project, args=['build', 'callHello.bst'])
+ result = cli.run(project=project, args=["build", "callHello.bst"])
result.assert_success()
- result = cli.run(project=project, args=['shell', 'callHello.bst', '--', '/bin/sh', 'callHello.sh'])
+ result = cli.run(project=project, args=["shell", "callHello.bst", "--", "/bin/sh", "callHello.sh"])
result.assert_success()
- assert result.output == 'Calling hello:\nHello World!\nThis is amhello 1.0.\n'
+ assert result.output == "Calling hello:\nHello World!\nThis is amhello 1.0.\n"
# Test opening a cross-junction workspace
-@pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
+@pytest.mark.skipif(not IS_LINUX, reason="Only available on linux")
@pytest.mark.datafiles(DATA_DIR)
def test_open_cross_junction_workspace(cli, tmpdir, datafiles):
project = str(datafiles)
workspace_dir = os.path.join(str(tmpdir), "workspace_hello_junction")
- result = cli.run(project=project,
- args=['workspace', 'open', '--directory', workspace_dir, 'hello-junction.bst:hello.bst'])
+ result = cli.run(
+ project=project, args=["workspace", "open", "--directory", workspace_dir, "hello-junction.bst:hello.bst"]
+ )
result.assert_success()
- result = cli.run(project=project,
- args=['workspace', 'close', '--remove-dir', 'hello-junction.bst:hello.bst'])
+ result = cli.run(project=project, args=["workspace", "close", "--remove-dir", "hello-junction.bst:hello.bst"])
result.assert_success()
diff --git a/tests/examples/running-commands.py b/tests/examples/running-commands.py
index 1c419d524..bf8a9c002 100644
--- a/tests/examples/running-commands.py
+++ b/tests/examples/running-commands.py
@@ -9,35 +9,31 @@ from buildstream.testing._utils.site import IS_LINUX, MACHINE_ARCH, HAVE_SANDBOX
pytestmark = pytest.mark.integration
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)), '..', '..', 'doc', 'examples', 'running-commands'
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "doc", "examples", "running-commands")
-@pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
- reason='Examples are written for x86-64')
+@pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64")
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with sandbox')
-@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
+@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with sandbox")
+@pytest.mark.skipif(HAVE_SANDBOX == "chroot", reason="This test is not meant to work with chroot sandbox")
def test_running_commands_build(cli, datafiles):
project = str(datafiles)
- result = cli.run(project=project, args=['build', 'hello.bst'])
+ result = cli.run(project=project, args=["build", "hello.bst"])
assert result.exit_code == 0
# Test running the executable
-@pytest.mark.skipif(MACHINE_ARCH != 'x86-64',
- reason='Examples are written for x86-64')
-@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with sandbox')
-@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox')
+@pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64")
+@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with sandbox")
+@pytest.mark.skipif(HAVE_SANDBOX == "chroot", reason="This test is not meant to work with chroot sandbox")
@pytest.mark.datafiles(DATA_DIR)
def test_running_commands_run(cli, datafiles):
project = str(datafiles)
- result = cli.run(project=project, args=['build', 'hello.bst'])
+ result = cli.run(project=project, args=["build", "hello.bst"])
assert result.exit_code == 0
- result = cli.run(project=project, args=['shell', 'hello.bst', '--', 'hello'])
+ result = cli.run(project=project, args=["shell", "hello.bst", "--", "hello"])
assert result.exit_code == 0
- assert result.output == 'Hello World\n'
+ assert result.output == "Hello World\n"
diff --git a/tests/external_plugins.py b/tests/external_plugins.py
index f6f67707b..2123b846b 100644
--- a/tests/external_plugins.py
+++ b/tests/external_plugins.py
@@ -16,7 +16,7 @@ import pytest
# test_match_patterns (list[str]): A list of shell style globs which may be
# used to specify a subset of test files from the repository to run.
# These must be specified relative to the root of the repository.
-class ExternalPluginRepo():
+class ExternalPluginRepo:
def __init__(self, name, url, ref, test_match_patterns=None):
self.name = name
self.url = url
@@ -30,17 +30,13 @@ class ExternalPluginRepo():
def clone(self, location):
self._clone_location = os.path.join(location, self.name)
- subprocess.run(['git', 'clone',
- '--single-branch',
- '--branch', self.ref,
- '--depth', '1',
- self.url,
- self._clone_location,
- ])
+ subprocess.run(
+ ["git", "clone", "--single-branch", "--branch", self.ref, "--depth", "1", self.url, self._clone_location,]
+ )
return self._clone_location
def install(self):
- subprocess.run(['pip3', 'install', self._clone_location])
+ subprocess.run(["pip3", "install", self._clone_location])
def test(self, args):
test_files = self._match_test_patterns()
diff --git a/tests/format/assertion.py b/tests/format/assertion.py
index 7e87977cb..67436250a 100644
--- a/tests/format/assertion.py
+++ b/tests/format/assertion.py
@@ -7,30 +7,41 @@ from buildstream._exceptions import ErrorDomain, LoadErrorReason
from buildstream.testing.runcli import cli # pylint: disable=unused-import
# Project directory
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- 'assertion'
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "assertion")
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("target,opt_pony,opt_horsy,assertion", [
- # Test an unconditional (!) directly in the element
- ('raw-assertion.bst', 'False', 'False', 'Raw assertion boogey'),
- # Test an assertion in a conditional
- ('conditional-assertion.bst', 'True', 'False', "It's not pony time yet"),
- # Test that we get the first composited assertion
- ('ordered-assertion.bst', 'True', 'True', "It's not horsy time yet"),
-])
+@pytest.mark.parametrize(
+ "target,opt_pony,opt_horsy,assertion",
+ [
+ # Test an unconditional (!) directly in the element
+ ("raw-assertion.bst", "False", "False", "Raw assertion boogey"),
+ # Test an assertion in a conditional
+ ("conditional-assertion.bst", "True", "False", "It's not pony time yet"),
+ # Test that we get the first composited assertion
+ ("ordered-assertion.bst", "True", "True", "It's not horsy time yet"),
+ ],
+)
def test_assertion_cli(cli, datafiles, target, opt_pony, opt_horsy, assertion):
project = str(datafiles)
- result = cli.run(project=project, silent=True, args=[
- '--option', 'pony', opt_pony,
- '--option', 'horsy', opt_horsy,
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- target])
+ result = cli.run(
+ project=project,
+ silent=True,
+ args=[
+ "--option",
+ "pony",
+ opt_pony,
+ "--option",
+ "horsy",
+ opt_horsy,
+ "show",
+ "--deps",
+ "none",
+ "--format",
+ "%{vars}",
+ target,
+ ],
+ )
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.USER_ASSERTION)
# Assert that the assertion text provided by the user
diff --git a/tests/format/dependencies.py b/tests/format/dependencies.py
index f92b89afa..e54b9b2d5 100644
--- a/tests/format/dependencies.py
+++ b/tests/format/dependencies.py
@@ -15,125 +15,125 @@ DATA_DIR = os.path.dirname(os.path.realpath(__file__))
#
@pytest.mark.datafiles(DATA_DIR)
def test_two_files(cli, datafiles):
- project = os.path.join(str(datafiles), 'dependencies1')
+ project = os.path.join(str(datafiles), "dependencies1")
- elements = cli.get_pipeline(project, ['target.bst'])
- assert elements == ['firstdep.bst', 'target.bst']
+ elements = cli.get_pipeline(project, ["target.bst"])
+ assert elements == ["firstdep.bst", "target.bst"]
@pytest.mark.datafiles(DATA_DIR)
def test_shared_dependency(cli, datafiles):
- project = os.path.join(str(datafiles), 'dependencies1')
+ project = os.path.join(str(datafiles), "dependencies1")
- elements = cli.get_pipeline(project, ['shareddeptarget.bst'])
- assert elements == ['firstdep.bst', 'shareddep.bst', 'shareddeptarget.bst']
+ elements = cli.get_pipeline(project, ["shareddeptarget.bst"])
+ assert elements == ["firstdep.bst", "shareddep.bst", "shareddeptarget.bst"]
@pytest.mark.datafiles(DATA_DIR)
def test_dependency_dict(cli, datafiles):
- project = os.path.join(str(datafiles), 'dependencies1')
- elements = cli.get_pipeline(project, ['target-depdict.bst'])
- assert elements == ['firstdep.bst', 'target-depdict.bst']
+ project = os.path.join(str(datafiles), "dependencies1")
+ elements = cli.get_pipeline(project, ["target-depdict.bst"])
+ assert elements == ["firstdep.bst", "target-depdict.bst"]
@pytest.mark.datafiles(DATA_DIR)
def test_invalid_dependency_declaration(cli, datafiles):
- project = os.path.join(str(datafiles), 'dependencies1')
- result = cli.run(project=project, args=['show', 'invaliddep.bst'])
+ project = os.path.join(str(datafiles), "dependencies1")
+ result = cli.run(project=project, args=["show", "invaliddep.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
@pytest.mark.datafiles(DATA_DIR)
def test_invalid_dependency_type(cli, datafiles):
- project = os.path.join(str(datafiles), 'dependencies1')
- result = cli.run(project=project, args=['show', 'invaliddeptype.bst'])
+ project = os.path.join(str(datafiles), "dependencies1")
+ result = cli.run(project=project, args=["show", "invaliddeptype.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
@pytest.mark.datafiles(DATA_DIR)
def test_invalid_strict_dependency(cli, datafiles):
- project = os.path.join(str(datafiles), 'dependencies1')
- result = cli.run(project=project, args=['show', 'invalidstrict.bst'])
+ project = os.path.join(str(datafiles), "dependencies1")
+ result = cli.run(project=project, args=["show", "invalidstrict.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
@pytest.mark.datafiles(DATA_DIR)
def test_invalid_non_strict_dependency(cli, datafiles):
- project = os.path.join(str(datafiles), 'dependencies1')
- result = cli.run(project=project, args=['show', 'invalidnonstrict.bst'])
+ project = os.path.join(str(datafiles), "dependencies1")
+ result = cli.run(project=project, args=["show", "invalidnonstrict.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
@pytest.mark.datafiles(DATA_DIR)
def test_circular_dependency(cli, datafiles):
- project = os.path.join(str(datafiles), 'dependencies1')
- result = cli.run(project=project, args=['show', 'circulartarget.bst'])
+ project = os.path.join(str(datafiles), "dependencies1")
+ result = cli.run(project=project, args=["show", "circulartarget.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.CIRCULAR_DEPENDENCY)
@pytest.mark.datafiles(DATA_DIR)
def test_build_dependency(cli, datafiles):
- project = os.path.join(str(datafiles), 'dependencies1')
+ project = os.path.join(str(datafiles), "dependencies1")
- elements = cli.get_pipeline(project, ['builddep.bst'], scope='run')
- assert elements == ['builddep.bst']
+ elements = cli.get_pipeline(project, ["builddep.bst"], scope="run")
+ assert elements == ["builddep.bst"]
- elements = cli.get_pipeline(project, ['builddep.bst'], scope='build')
- assert elements == ['firstdep.bst']
+ elements = cli.get_pipeline(project, ["builddep.bst"], scope="build")
+ assert elements == ["firstdep.bst"]
@pytest.mark.datafiles(DATA_DIR)
def test_runtime_dependency(cli, datafiles):
- project = os.path.join(str(datafiles), 'dependencies1')
- elements = cli.get_pipeline(project, ['runtimedep.bst'], scope='build')
+ project = os.path.join(str(datafiles), "dependencies1")
+ elements = cli.get_pipeline(project, ["runtimedep.bst"], scope="build")
# FIXME: The empty line should probably never happen here when there are no results.
- assert elements == ['']
- elements = cli.get_pipeline(project, ['runtimedep.bst'], scope='run')
- assert elements == ['firstdep.bst', 'runtimedep.bst']
+ assert elements == [""]
+ elements = cli.get_pipeline(project, ["runtimedep.bst"], scope="run")
+ assert elements == ["firstdep.bst", "runtimedep.bst"]
@pytest.mark.datafiles(DATA_DIR)
def test_all_dependency(cli, datafiles):
- project = os.path.join(str(datafiles), 'dependencies1')
+ project = os.path.join(str(datafiles), "dependencies1")
- elements = cli.get_pipeline(project, ['alldep.bst'], scope='build')
- assert elements == ['firstdep.bst']
+ elements = cli.get_pipeline(project, ["alldep.bst"], scope="build")
+ assert elements == ["firstdep.bst"]
- elements = cli.get_pipeline(project, ['alldep.bst'], scope='run')
- assert elements == ['firstdep.bst', 'alldep.bst']
+ elements = cli.get_pipeline(project, ["alldep.bst"], scope="run")
+ assert elements == ["firstdep.bst", "alldep.bst"]
@pytest.mark.datafiles(DATA_DIR)
def test_list_build_dependency(cli, datafiles):
- project = os.path.join(str(datafiles), 'dependencies1')
+ project = os.path.join(str(datafiles), "dependencies1")
# Check that the pipeline includes the build dependency
- deps = cli.get_pipeline(project, ['builddep-list.bst'], scope="build")
+ deps = cli.get_pipeline(project, ["builddep-list.bst"], scope="build")
assert "firstdep.bst" in deps
@pytest.mark.datafiles(DATA_DIR)
def test_list_runtime_dependency(cli, datafiles):
- project = os.path.join(str(datafiles), 'dependencies1')
+ project = os.path.join(str(datafiles), "dependencies1")
# Check that the pipeline includes the runtime dependency
- deps = cli.get_pipeline(project, ['runtimedep-list.bst'], scope="run")
+ deps = cli.get_pipeline(project, ["runtimedep-list.bst"], scope="run")
assert "firstdep.bst" in deps
@pytest.mark.datafiles(DATA_DIR)
def test_list_dependencies_combined(cli, datafiles):
- project = os.path.join(str(datafiles), 'dependencies1')
+ project = os.path.join(str(datafiles), "dependencies1")
# Check that runtime deps get combined
- rundeps = cli.get_pipeline(project, ['list-combine.bst'], scope="run")
+ rundeps = cli.get_pipeline(project, ["list-combine.bst"], scope="run")
assert "firstdep.bst" not in rundeps
assert "seconddep.bst" in rundeps
assert "thirddep.bst" in rundeps
# Check that build deps get combined
- builddeps = cli.get_pipeline(project, ['list-combine.bst'], scope="build")
+ builddeps = cli.get_pipeline(project, ["list-combine.bst"], scope="build")
assert "firstdep.bst" in builddeps
assert "seconddep.bst" not in builddeps
assert "thirddep.bst" in builddeps
@@ -141,12 +141,12 @@ def test_list_dependencies_combined(cli, datafiles):
@pytest.mark.datafiles(DATA_DIR)
def test_list_overlap(cli, datafiles):
- project = os.path.join(str(datafiles), 'dependencies1')
+ project = os.path.join(str(datafiles), "dependencies1")
# Check that dependencies get merged
- rundeps = cli.get_pipeline(project, ['list-overlap.bst'], scope="run")
+ rundeps = cli.get_pipeline(project, ["list-overlap.bst"], scope="run")
assert "firstdep.bst" in rundeps
- builddeps = cli.get_pipeline(project, ['list-overlap.bst'], scope="build")
+ builddeps = cli.get_pipeline(project, ["list-overlap.bst"], scope="build")
assert "firstdep.bst" in builddeps
@@ -156,10 +156,10 @@ def test_list_overlap(cli, datafiles):
#
@pytest.mark.datafiles(DATA_DIR)
def test_scope_all(cli, datafiles):
- project = os.path.join(str(datafiles), 'dependencies2')
- elements = ['target.bst']
+ project = os.path.join(str(datafiles), "dependencies2")
+ elements = ["target.bst"]
- element_list = cli.get_pipeline(project, elements, scope='all')
+ element_list = cli.get_pipeline(project, elements, scope="all")
assert element_list == [
"build-build.bst",
@@ -174,10 +174,10 @@ def test_scope_all(cli, datafiles):
@pytest.mark.datafiles(DATA_DIR)
def test_scope_run(cli, datafiles):
- project = os.path.join(str(datafiles), 'dependencies2')
- elements = ['target.bst']
+ project = os.path.join(str(datafiles), "dependencies2")
+ elements = ["target.bst"]
- element_list = cli.get_pipeline(project, elements, scope='run')
+ element_list = cli.get_pipeline(project, elements, scope="run")
assert element_list == [
"dep-one.bst",
@@ -189,69 +189,71 @@ def test_scope_run(cli, datafiles):
@pytest.mark.datafiles(DATA_DIR)
def test_scope_build(cli, datafiles):
- project = os.path.join(str(datafiles), 'dependencies2')
- elements = ['target.bst']
+ project = os.path.join(str(datafiles), "dependencies2")
+ elements = ["target.bst"]
- element_list = cli.get_pipeline(project, elements, scope='build')
+ element_list = cli.get_pipeline(project, elements, scope="build")
assert element_list == ["dep-one.bst", "run.bst", "dep-two.bst"]
@pytest.mark.datafiles(DATA_DIR)
def test_scope_build_of_child(cli, datafiles):
- project = os.path.join(str(datafiles), 'dependencies2')
- elements = ['target.bst']
+ project = os.path.join(str(datafiles), "dependencies2")
+ elements = ["target.bst"]
- element_list = cli.get_pipeline(project, elements, scope='build')
+ element_list = cli.get_pipeline(project, elements, scope="build")
# First pass, lets check dep-two
element = element_list[2]
# Pass two, let's look at these
- element_list = cli.get_pipeline(project, [element], scope='build')
+ element_list = cli.get_pipeline(project, [element], scope="build")
assert element_list == ["run-build.bst", "build.bst"]
@pytest.mark.datafiles(DATA_DIR)
def test_no_recurse(cli, datafiles):
- project = os.path.join(str(datafiles), 'dependencies2')
- elements = ['target.bst']
+ project = os.path.join(str(datafiles), "dependencies2")
+ elements = ["target.bst"]
# We abuse the 'plan' scope here to ensure that we call
# element.dependencies() with recurse=False - currently, no `bst
# show` option does this directly.
- element_list = cli.get_pipeline(project, elements, scope='plan')
+ element_list = cli.get_pipeline(project, elements, scope="plan")
assert element_list == [
- 'build-build.bst',
- 'run-build.bst',
- 'build.bst',
- 'dep-one.bst',
- 'run.bst',
- 'dep-two.bst',
- 'target.bst',
+ "build-build.bst",
+ "run-build.bst",
+ "build.bst",
+ "dep-one.bst",
+ "run.bst",
+ "dep-two.bst",
+ "target.bst",
]
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize(("element", "asserts"), [
- ('build-runtime', False),
- ('build-build', True),
- ('build-all', True),
- ('runtime-runtime', True),
- ('runtime-all', True),
- ('all-all', True),
-])
+@pytest.mark.parametrize(
+ ("element", "asserts"),
+ [
+ ("build-runtime", False),
+ ("build-build", True),
+ ("build-all", True),
+ ("runtime-runtime", True),
+ ("runtime-all", True),
+ ("all-all", True),
+ ],
+)
def test_duplicate_deps(cli, datafiles, element, asserts):
- project = os.path.join(str(datafiles), 'dependencies3')
+ project = os.path.join(str(datafiles), "dependencies3")
- result = cli.run(project=project, args=['show', '{}.bst'.format(element)])
+ result = cli.run(project=project, args=["show", "{}.bst".format(element)])
if asserts:
- result.assert_main_error(ErrorDomain.LOAD,
- LoadErrorReason.DUPLICATE_DEPENDENCY)
- assert '[line 10 column 2]' in result.stderr
- assert '[line 8 column 2]' in result.stderr
+ result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.DUPLICATE_DEPENDENCY)
+ assert "[line 10 column 2]" in result.stderr
+ assert "[line 8 column 2]" in result.stderr
else:
result.assert_success()
diff --git a/tests/format/include.py b/tests/format/include.py
index 434a94d1f..5efbc62a9 100644
--- a/tests/format/include.py
+++ b/tests/format/include.py
@@ -12,97 +12,90 @@ from tests.testutils import generate_junction
# Project directory
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- 'include'
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "include")
@pytest.mark.datafiles(DATA_DIR)
def test_include_project_file(cli, datafiles):
- project = os.path.join(str(datafiles), 'file')
- result = cli.run(project=project, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'])
+ project = os.path.join(str(datafiles), "file")
+ result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"])
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert loaded.get_bool('included')
+ assert loaded.get_bool("included")
def test_include_missing_file(cli, tmpdir):
- tmpdir.join('project.conf').write('{"name": "test"}')
- element = tmpdir.join('include_missing_file.bst')
+ tmpdir.join("project.conf").write('{"name": "test"}')
+ element = tmpdir.join("include_missing_file.bst")
# Normally we would use dicts and _yaml.roundtrip_dump to write such things, but here
# we want to be sure of a stable line and column number.
- element.write(textwrap.dedent("""
+ element.write(
+ textwrap.dedent(
+ """
kind: manual
"(@)":
- nosuch.yaml
- """).strip())
+ """
+ ).strip()
+ )
- result = cli.run(project=str(tmpdir), args=['show', str(element.basename)])
+ result = cli.run(project=str(tmpdir), args=["show", str(element.basename)])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE)
# Make sure the root cause provenance is in the output.
- assert 'line 4 column 2' in result.stderr
+ assert "line 4 column 2" in result.stderr
def test_include_dir(cli, tmpdir):
- tmpdir.join('project.conf').write('{"name": "test"}')
- tmpdir.mkdir('subdir')
- element = tmpdir.join('include_dir.bst')
+ tmpdir.join("project.conf").write('{"name": "test"}')
+ tmpdir.mkdir("subdir")
+ element = tmpdir.join("include_dir.bst")
# Normally we would use dicts and _yaml.roundtrip_dump to write such things, but here
# we want to be sure of a stable line and column number.
- element.write(textwrap.dedent("""
+ element.write(
+ textwrap.dedent(
+ """
kind: manual
"(@)":
- subdir/
- """).strip())
+ """
+ ).strip()
+ )
- result = cli.run(project=str(tmpdir), args=['show', str(element.basename)])
- result.assert_main_error(
- ErrorDomain.LOAD, LoadErrorReason.LOADING_DIRECTORY)
+ result = cli.run(project=str(tmpdir), args=["show", str(element.basename)])
+ result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.LOADING_DIRECTORY)
# Make sure the root cause provenance is in the output.
- assert 'line 4 column 2' in result.stderr
+ assert "line 4 column 2" in result.stderr
@pytest.mark.datafiles(DATA_DIR)
def test_include_junction_file(cli, tmpdir, datafiles):
- project = os.path.join(str(datafiles), 'junction')
-
- generate_junction(tmpdir,
- os.path.join(project, 'subproject'),
- os.path.join(project, 'junction.bst'),
- store_ref=True)
-
- result = cli.run(project=project, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'])
+ project = os.path.join(str(datafiles), "junction")
+
+ generate_junction(
+ tmpdir, os.path.join(project, "subproject"), os.path.join(project, "junction.bst"), store_ref=True
+ )
+
+ result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"])
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert loaded.get_bool('included')
+ assert loaded.get_bool("included")
@pytest.mark.datafiles(DATA_DIR)
def test_include_junction_options(cli, datafiles):
- project = os.path.join(str(datafiles), 'options')
-
- result = cli.run(project=project, args=[
- '-o', 'build_arch', 'x86_64',
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'])
+ project = os.path.join(str(datafiles), "options")
+
+ result = cli.run(
+ project=project,
+ args=["-o", "build_arch", "x86_64", "show", "--deps", "none", "--format", "%{vars}", "element.bst"],
+ )
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert loaded.get_str('build_arch') == 'x86_64'
+ assert loaded.get_str("build_arch") == "x86_64"
@pytest.mark.datafiles(DATA_DIR)
@@ -111,31 +104,22 @@ def test_junction_element_partial_project_project(cli, tmpdir, datafiles):
Junction elements never depend on fully include processed project.
"""
- project = os.path.join(str(datafiles), 'junction')
+ project = os.path.join(str(datafiles), "junction")
- subproject_path = os.path.join(project, 'subproject')
- junction_path = os.path.join(project, 'junction.bst')
+ subproject_path = os.path.join(project, "subproject")
+ junction_path = os.path.join(project, "junction.bst")
- repo = create_repo('git', str(tmpdir))
+ repo = create_repo("git", str(tmpdir))
ref = repo.create(subproject_path)
- element = {
- 'kind': 'junction',
- 'sources': [
- repo.source_config(ref=ref)
- ]
- }
+ element = {"kind": "junction", "sources": [repo.source_config(ref=ref)]}
_yaml.roundtrip_dump(element, junction_path)
- result = cli.run(project=project, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'junction.bst'])
+ result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{vars}", "junction.bst"])
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert loaded.get_str('included', default=None) is None
+ assert loaded.get_str("included", default=None) is None
@pytest.mark.datafiles(DATA_DIR)
@@ -144,170 +128,127 @@ def test_junction_element_not_partial_project_file(cli, tmpdir, datafiles):
Junction elements never depend on fully include processed project.
"""
- project = os.path.join(str(datafiles), 'file_with_subproject')
+ project = os.path.join(str(datafiles), "file_with_subproject")
- subproject_path = os.path.join(project, 'subproject')
- junction_path = os.path.join(project, 'junction.bst')
+ subproject_path = os.path.join(project, "subproject")
+ junction_path = os.path.join(project, "junction.bst")
- repo = create_repo('git', str(tmpdir))
+ repo = create_repo("git", str(tmpdir))
ref = repo.create(subproject_path)
- element = {
- 'kind': 'junction',
- 'sources': [
- repo.source_config(ref=ref)
- ]
- }
+ element = {"kind": "junction", "sources": [repo.source_config(ref=ref)]}
_yaml.roundtrip_dump(element, junction_path)
- result = cli.run(project=project, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'junction.bst'])
+ result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{vars}", "junction.bst"])
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert loaded.get_str('included', default=None) is not None
+ assert loaded.get_str("included", default=None) is not None
@pytest.mark.datafiles(DATA_DIR)
def test_include_element_overrides(cli, datafiles):
- project = os.path.join(str(datafiles), 'overrides')
+ project = os.path.join(str(datafiles), "overrides")
- result = cli.run(project=project, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'])
+ result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"])
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert loaded.get_str('manual_main_override', default=None) is not None
- assert loaded.get_str('manual_included_override', default=None) is not None
+ assert loaded.get_str("manual_main_override", default=None) is not None
+ assert loaded.get_str("manual_included_override", default=None) is not None
@pytest.mark.datafiles(DATA_DIR)
def test_include_element_overrides_composition(cli, datafiles):
- project = os.path.join(str(datafiles), 'overrides')
+ project = os.path.join(str(datafiles), "overrides")
- result = cli.run(project=project, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{config}',
- 'element.bst'])
+ result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{config}", "element.bst"])
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert loaded.get_str_list('build-commands') == ['first', 'second']
+ assert loaded.get_str_list("build-commands") == ["first", "second"]
@pytest.mark.datafiles(DATA_DIR)
def test_list_overide_does_not_fail_upon_first_composition(cli, datafiles):
- project = os.path.join(str(datafiles), 'eventual_overrides')
+ project = os.path.join(str(datafiles), "eventual_overrides")
- result = cli.run(project=project, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{public}',
- 'element.bst'])
+ result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{public}", "element.bst"])
result.assert_success()
loaded = _yaml.load_data(result.output)
# Assert that the explicitly overwritten public data is present
- bst = loaded.get_mapping('bst')
- assert 'foo-commands' in bst
- assert bst.get_str_list('foo-commands') == ['need', 'this']
+ bst = loaded.get_mapping("bst")
+ assert "foo-commands" in bst
+ assert bst.get_str_list("foo-commands") == ["need", "this"]
@pytest.mark.datafiles(DATA_DIR)
def test_include_element_overrides_sub_include(cli, datafiles):
- project = os.path.join(str(datafiles), 'sub-include')
+ project = os.path.join(str(datafiles), "sub-include")
- result = cli.run(project=project, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'])
+ result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"])
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert loaded.get_str('included', default=None) is not None
+ assert loaded.get_str("included", default=None) is not None
@pytest.mark.datafiles(DATA_DIR)
def test_junction_do_not_use_included_overrides(cli, tmpdir, datafiles):
- project = os.path.join(str(datafiles), 'overrides-junction')
-
- generate_junction(tmpdir,
- os.path.join(project, 'subproject'),
- os.path.join(project, 'junction.bst'),
- store_ref=True)
-
- result = cli.run(project=project, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'junction.bst'])
+ project = os.path.join(str(datafiles), "overrides-junction")
+
+ generate_junction(
+ tmpdir, os.path.join(project, "subproject"), os.path.join(project, "junction.bst"), store_ref=True
+ )
+
+ result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{vars}", "junction.bst"])
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert loaded.get_str('main_override', default=None) is not None
- assert loaded.get_str('included_override', default=None) is None
+ assert loaded.get_str("main_override", default=None) is not None
+ assert loaded.get_str("included_override", default=None) is None
@pytest.mark.datafiles(DATA_DIR)
def test_conditional_in_fragment(cli, datafiles):
- project = os.path.join(str(datafiles), 'conditional')
-
- result = cli.run(project=project, args=[
- '-o', 'build_arch', 'x86_64',
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'])
+ project = os.path.join(str(datafiles), "conditional")
+
+ result = cli.run(
+ project=project,
+ args=["-o", "build_arch", "x86_64", "show", "--deps", "none", "--format", "%{vars}", "element.bst"],
+ )
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert loaded.get_str('size') == '8'
+ assert loaded.get_str("size") == "8"
@pytest.mark.datafiles(DATA_DIR)
def test_inner(cli, datafiles):
- project = os.path.join(str(datafiles), 'inner')
- result = cli.run(project=project, args=[
- '-o', 'build_arch', 'x86_64',
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'])
+ project = os.path.join(str(datafiles), "inner")
+ result = cli.run(
+ project=project,
+ args=["-o", "build_arch", "x86_64", "show", "--deps", "none", "--format", "%{vars}", "element.bst"],
+ )
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert loaded.get_str('build_arch') == 'x86_64'
+ assert loaded.get_str("build_arch") == "x86_64"
@pytest.mark.datafiles(DATA_DIR)
def test_recursive_include(cli, datafiles):
- project = os.path.join(str(datafiles), 'recursive')
+ project = os.path.join(str(datafiles), "recursive")
- result = cli.run(project=project, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'])
+ result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.RECURSIVE_INCLUDE)
- assert 'line 2 column 2' in result.stderr
+ assert "line 2 column 2" in result.stderr
@pytest.mark.datafiles(DATA_DIR)
def test_local_to_junction(cli, tmpdir, datafiles):
- project = os.path.join(str(datafiles), 'local_to_junction')
-
- generate_junction(tmpdir,
- os.path.join(project, 'subproject'),
- os.path.join(project, 'junction.bst'),
- store_ref=True)
-
- result = cli.run(project=project, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'])
+ project = os.path.join(str(datafiles), "local_to_junction")
+
+ generate_junction(
+ tmpdir, os.path.join(project, "subproject"), os.path.join(project, "junction.bst"), store_ref=True
+ )
+
+ result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"])
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert loaded.get_bool('included')
+ assert loaded.get_bool("included")
diff --git a/tests/format/include_composition.py b/tests/format/include_composition.py
index e10e28bc0..4af6862fa 100644
--- a/tests/format/include_composition.py
+++ b/tests/format/include_composition.py
@@ -11,7 +11,7 @@ from tests.testutils import dummy_context
@contextmanager
def make_includes(basedir):
- _yaml.roundtrip_dump({'name': 'test'}, os.path.join(basedir, 'project.conf'))
+ _yaml.roundtrip_dump({"name": "test"}, os.path.join(basedir, "project.conf"))
with dummy_context() as context:
project = Project(basedir, context)
loader = project.loader
@@ -21,131 +21,118 @@ def make_includes(basedir):
def test_main_has_priority(tmpdir):
with make_includes(str(tmpdir)) as includes:
- _yaml.roundtrip_dump({'(@)': ['a.yml'], 'test': ['main']},
- str(tmpdir.join('main.yml')))
+ _yaml.roundtrip_dump({"(@)": ["a.yml"], "test": ["main"]}, str(tmpdir.join("main.yml")))
- main = _yaml.load(str(tmpdir.join('main.yml')))
+ main = _yaml.load(str(tmpdir.join("main.yml")))
- _yaml.roundtrip_dump({'test': ['a']}, str(tmpdir.join('a.yml')))
+ _yaml.roundtrip_dump({"test": ["a"]}, str(tmpdir.join("a.yml")))
includes.process(main)
- assert main.get_str_list('test') == ['main']
+ assert main.get_str_list("test") == ["main"]
def test_include_cannot_append(tmpdir):
with make_includes(str(tmpdir)) as includes:
- _yaml.roundtrip_dump({'(@)': ['a.yml'], 'test': ['main']},
- str(tmpdir.join('main.yml')))
- main = _yaml.load(str(tmpdir.join('main.yml')))
+ _yaml.roundtrip_dump({"(@)": ["a.yml"], "test": ["main"]}, str(tmpdir.join("main.yml")))
+ main = _yaml.load(str(tmpdir.join("main.yml")))
- _yaml.roundtrip_dump({'test': {'(>)': ['a']}},
- str(tmpdir.join('a.yml')))
+ _yaml.roundtrip_dump({"test": {"(>)": ["a"]}}, str(tmpdir.join("a.yml")))
includes.process(main)
- assert main.get_str_list('test') == ['main']
+ assert main.get_str_list("test") == ["main"]
def test_main_can_append(tmpdir):
with make_includes(str(tmpdir)) as includes:
- _yaml.roundtrip_dump({'(@)': ['a.yml'], 'test': {'(>)': ['main']}},
- str(tmpdir.join('main.yml')))
- main = _yaml.load(str(tmpdir.join('main.yml')))
+ _yaml.roundtrip_dump({"(@)": ["a.yml"], "test": {"(>)": ["main"]}}, str(tmpdir.join("main.yml")))
+ main = _yaml.load(str(tmpdir.join("main.yml")))
- _yaml.roundtrip_dump({'test': ['a']}, str(tmpdir.join('a.yml')))
+ _yaml.roundtrip_dump({"test": ["a"]}, str(tmpdir.join("a.yml")))
includes.process(main)
- assert main.get_str_list('test') == ['a', 'main']
+ assert main.get_str_list("test") == ["a", "main"]
def test_sibling_cannot_append_backward(tmpdir):
with make_includes(str(tmpdir)) as includes:
- _yaml.roundtrip_dump({'(@)': ['a.yml', 'b.yml']},
- str(tmpdir.join('main.yml')))
- main = _yaml.load(str(tmpdir.join('main.yml')))
+ _yaml.roundtrip_dump({"(@)": ["a.yml", "b.yml"]}, str(tmpdir.join("main.yml")))
+ main = _yaml.load(str(tmpdir.join("main.yml")))
- _yaml.roundtrip_dump({'test': {'(>)': ['a']}},
- str(tmpdir.join('a.yml')))
- _yaml.roundtrip_dump({'test': ['b']},
- str(tmpdir.join('b.yml')))
+ _yaml.roundtrip_dump({"test": {"(>)": ["a"]}}, str(tmpdir.join("a.yml")))
+ _yaml.roundtrip_dump({"test": ["b"]}, str(tmpdir.join("b.yml")))
includes.process(main)
- assert main.get_str_list('test') == ['b']
+ assert main.get_str_list("test") == ["b"]
def test_sibling_can_append_forward(tmpdir):
with make_includes(str(tmpdir)) as includes:
- _yaml.roundtrip_dump({'(@)': ['a.yml', 'b.yml']},
- str(tmpdir.join('main.yml')))
- main = _yaml.load(str(tmpdir.join('main.yml')))
+ _yaml.roundtrip_dump({"(@)": ["a.yml", "b.yml"]}, str(tmpdir.join("main.yml")))
+ main = _yaml.load(str(tmpdir.join("main.yml")))
- _yaml.roundtrip_dump({'test': ['a']},
- str(tmpdir.join('a.yml')))
- _yaml.roundtrip_dump({'test': {'(>)': ['b']}},
- str(tmpdir.join('b.yml')))
+ _yaml.roundtrip_dump({"test": ["a"]}, str(tmpdir.join("a.yml")))
+ _yaml.roundtrip_dump({"test": {"(>)": ["b"]}}, str(tmpdir.join("b.yml")))
includes.process(main)
- assert main.get_str_list('test') == ['a', 'b']
+ assert main.get_str_list("test") == ["a", "b"]
def test_lastest_sibling_has_priority(tmpdir):
with make_includes(str(tmpdir)) as includes:
- _yaml.roundtrip_dump({'(@)': ['a.yml', 'b.yml']},
- str(tmpdir.join('main.yml')))
- main = _yaml.load(str(tmpdir.join('main.yml')))
+ _yaml.roundtrip_dump({"(@)": ["a.yml", "b.yml"]}, str(tmpdir.join("main.yml")))
+ main = _yaml.load(str(tmpdir.join("main.yml")))
- _yaml.roundtrip_dump({'test': ['a']},
- str(tmpdir.join('a.yml')))
- _yaml.roundtrip_dump({'test': ['b']},
- str(tmpdir.join('b.yml')))
+ _yaml.roundtrip_dump({"test": ["a"]}, str(tmpdir.join("a.yml")))
+ _yaml.roundtrip_dump({"test": ["b"]}, str(tmpdir.join("b.yml")))
includes.process(main)
- assert main.get_str_list('test') == ['b']
+ assert main.get_str_list("test") == ["b"]
def test_main_keeps_keys(tmpdir):
with make_includes(str(tmpdir)) as includes:
- _yaml.roundtrip_dump({'(@)': ['a.yml'], 'something': 'else'},
- str(tmpdir.join('main.yml')))
- main = _yaml.load(str(tmpdir.join('main.yml')))
+ _yaml.roundtrip_dump({"(@)": ["a.yml"], "something": "else"}, str(tmpdir.join("main.yml")))
+ main = _yaml.load(str(tmpdir.join("main.yml")))
- _yaml.roundtrip_dump({'test': ['a']}, str(tmpdir.join('a.yml')))
+ _yaml.roundtrip_dump({"test": ["a"]}, str(tmpdir.join("a.yml")))
includes.process(main)
- assert main.get_str_list('test') == ['a']
- assert main.get_str('something') == 'else'
+ assert main.get_str_list("test") == ["a"]
+ assert main.get_str("something") == "else"
def test_overwrite_directive_on_later_composite(tmpdir):
with make_includes(str(tmpdir)) as includes:
- _yaml.roundtrip_dump({'(@)': ['a.yml', 'b.yml'], 'test': {'(=)': ['Overwritten']}},
- str(tmpdir.join('main.yml')))
+ _yaml.roundtrip_dump(
+ {"(@)": ["a.yml", "b.yml"], "test": {"(=)": ["Overwritten"]}}, str(tmpdir.join("main.yml"))
+ )
- main = _yaml.load(str(tmpdir.join('main.yml')))
+ main = _yaml.load(str(tmpdir.join("main.yml")))
# a.yml
- _yaml.roundtrip_dump({'test': ['some useless', 'list', 'to be overwritten'],
- 'foo': 'should not be present'},
- str(tmpdir.join('a.yml')))
+ _yaml.roundtrip_dump(
+ {"test": ["some useless", "list", "to be overwritten"], "foo": "should not be present"},
+ str(tmpdir.join("a.yml")),
+ )
# b.yaml isn't going to have a 'test' node to overwrite
- _yaml.roundtrip_dump({'foo': 'should be present'},
- str(tmpdir.join('b.yml')))
+ _yaml.roundtrip_dump({"foo": "should be present"}, str(tmpdir.join("b.yml")))
includes.process(main)
- assert main.get_str_list('test') == ['Overwritten']
- assert main.get_str('foo') == 'should be present'
+ assert main.get_str_list("test") == ["Overwritten"]
+ assert main.get_str("foo") == "should be present"
diff --git a/tests/format/invalid_keys.py b/tests/format/invalid_keys.py
index 861cfeabd..ce1e2e487 100644
--- a/tests/format/invalid_keys.py
+++ b/tests/format/invalid_keys.py
@@ -7,21 +7,21 @@ from buildstream._exceptions import ErrorDomain, LoadErrorReason
from buildstream.testing.runcli import cli # pylint: disable=unused-import
# Project directory
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- 'invalid-keys'
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "invalid-keys")
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize(("element", "location"), [
- ("no-path-specified.bst", "line 4 column 4"),
- ("optional-source.bst", "line 6 column 10"),
- ("included-source.bst", "line 4 column 4"),
-])
+@pytest.mark.parametrize(
+ ("element", "location"),
+ [
+ ("no-path-specified.bst", "line 4 column 4"),
+ ("optional-source.bst", "line 6 column 10"),
+ ("included-source.bst", "line 4 column 4"),
+ ],
+)
def test_compositied_node_fails_usefully(cli, datafiles, element, location):
project = str(datafiles)
- result = cli.run(project=project, args=['show', element])
+ result = cli.run(project=project, args=["show", element])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
assert "synthetic node" not in result.stderr
diff --git a/tests/format/junctions.py b/tests/format/junctions.py
index a0af521a2..45ccff222 100644
--- a/tests/format/junctions.py
+++ b/tests/format/junctions.py
@@ -13,10 +13,7 @@ from buildstream.testing import create_repo
from buildstream.testing._utils.site import HAVE_GIT
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- 'junctions',
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "junctions",)
def copy_subprojects(project, datafiles, subprojects):
@@ -26,41 +23,41 @@ def copy_subprojects(project, datafiles, subprojects):
@pytest.mark.datafiles(DATA_DIR)
def test_simple_pipeline(cli, datafiles):
- project = os.path.join(str(datafiles), 'foo')
- copy_subprojects(project, datafiles, ['base'])
+ project = os.path.join(str(datafiles), "foo")
+ copy_subprojects(project, datafiles, ["base"])
# Check that the pipeline includes the subproject element
- element_list = cli.get_pipeline(project, ['target.bst'])
- assert 'base.bst:target.bst' in element_list
+ element_list = cli.get_pipeline(project, ["target.bst"])
+ assert "base.bst:target.bst" in element_list
@pytest.mark.datafiles(DATA_DIR)
def test_simple_build(cli, tmpdir, datafiles):
- project = os.path.join(str(datafiles), 'foo')
- copy_subprojects(project, datafiles, ['base'])
+ project = os.path.join(str(datafiles), "foo")
+ copy_subprojects(project, datafiles, ["base"])
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Build, checkout
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
# Check that the checkout contains the expected files from both projects
- assert os.path.exists(os.path.join(checkoutdir, 'base.txt'))
- assert os.path.exists(os.path.join(checkoutdir, 'foo.txt'))
+ assert os.path.exists(os.path.join(checkoutdir, "base.txt"))
+ assert os.path.exists(os.path.join(checkoutdir, "foo.txt"))
@pytest.mark.datafiles(DATA_DIR)
def test_junction_missing_project_conf(cli, datafiles):
- project = datafiles / 'foo'
- copy_subprojects(project, datafiles, ['base'])
+ project = datafiles / "foo"
+ copy_subprojects(project, datafiles, ["base"])
# TODO: see if datafiles can tidy this concat up
# py3.5 requires this str conversion.
- os.remove(str(project / 'base' / 'project.conf'))
+ os.remove(str(project / "base" / "project.conf"))
# Note that both 'foo' and 'base' projects have a 'target.bst'. The
# 'app.bst' in 'foo' depends on the 'target.bst' in 'base', i.e.:
@@ -78,7 +75,7 @@ def test_junction_missing_project_conf(cli, datafiles):
# That would lead to a 'circular dependency error' in this setup, when we
# expect an 'invalid junction'.
#
- result = cli.run(project=project, args=['build', 'app.bst'])
+ result = cli.run(project=project, args=["build", "app.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_JUNCTION)
# Assert that we have the expected provenance encoded into the error
@@ -89,20 +86,18 @@ def test_junction_missing_project_conf(cli, datafiles):
def test_workspaced_junction_missing_project_conf(cli, datafiles):
# See test_junction_missing_project_conf for some more background.
- project = datafiles / 'foo'
- workspace_dir = project / 'base_workspace'
- copy_subprojects(project, datafiles, ['base'])
+ project = datafiles / "foo"
+ workspace_dir = project / "base_workspace"
+ copy_subprojects(project, datafiles, ["base"])
- result = cli.run(
- project=project,
- args=['workspace', 'open', 'base.bst', '--directory', workspace_dir])
+ result = cli.run(project=project, args=["workspace", "open", "base.bst", "--directory", workspace_dir])
print(result)
result.assert_success()
# py3.5 requires this str conversion.
- os.remove(str(workspace_dir / 'project.conf'))
+ os.remove(str(workspace_dir / "project.conf"))
- result = cli.run(project=project, args=['build', 'app.bst'])
+ result = cli.run(project=project, args=["build", "app.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_JUNCTION)
# Assert that we have the expected provenance encoded into the error
@@ -111,18 +106,18 @@ def test_workspaced_junction_missing_project_conf(cli, datafiles):
@pytest.mark.datafiles(DATA_DIR)
def test_build_of_same_junction_used_twice(cli, datafiles):
- project = os.path.join(str(datafiles), 'inconsistent-names')
+ project = os.path.join(str(datafiles), "inconsistent-names")
# Check we can build a project that contains the same junction
# that is used twice, but named differently
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
@pytest.mark.datafiles(DATA_DIR)
def test_missing_file_in_subproject(cli, datafiles):
- project = os.path.join(str(datafiles), 'missing-element')
- result = cli.run(project=project, args=['show', 'target.bst'])
+ project = os.path.join(str(datafiles), "missing-element")
+ result = cli.run(project=project, args=["show", "target.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE)
# Assert that we have the expected provenance encoded into the error
@@ -131,8 +126,8 @@ def test_missing_file_in_subproject(cli, datafiles):
@pytest.mark.datafiles(DATA_DIR)
def test_missing_file_in_subsubproject(cli, datafiles):
- project = os.path.join(str(datafiles), 'missing-element')
- result = cli.run(project=project, args=['show', 'sub-target.bst'])
+ project = os.path.join(str(datafiles), "missing-element")
+ result = cli.run(project=project, args=["show", "sub-target.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE)
# Assert that we have the expected provenance encoded into the error
@@ -141,8 +136,8 @@ def test_missing_file_in_subsubproject(cli, datafiles):
@pytest.mark.datafiles(DATA_DIR)
def test_missing_junction_in_subproject(cli, datafiles):
- project = os.path.join(str(datafiles), 'missing-element')
- result = cli.run(project=project, args=['show', 'sub-target-bad-junction.bst'])
+ project = os.path.join(str(datafiles), "missing-element")
+ result = cli.run(project=project, args=["show", "sub-target-bad-junction.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE)
# Assert that we have the expected provenance encoded into the error
@@ -151,62 +146,62 @@ def test_missing_junction_in_subproject(cli, datafiles):
@pytest.mark.datafiles(DATA_DIR)
def test_nested_simple(cli, tmpdir, datafiles):
- project = os.path.join(str(datafiles), 'foo')
- copy_subprojects(project, datafiles, ['base'])
+ project = os.path.join(str(datafiles), "foo")
+ copy_subprojects(project, datafiles, ["base"])
- project = os.path.join(str(datafiles), 'nested')
- copy_subprojects(project, datafiles, ['foo'])
+ project = os.path.join(str(datafiles), "nested")
+ copy_subprojects(project, datafiles, ["foo"])
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Build, checkout
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
# Check that the checkout contains the expected files from all subprojects
- assert os.path.exists(os.path.join(checkoutdir, 'base.txt'))
- assert os.path.exists(os.path.join(checkoutdir, 'foo.txt'))
+ assert os.path.exists(os.path.join(checkoutdir, "base.txt"))
+ assert os.path.exists(os.path.join(checkoutdir, "foo.txt"))
@pytest.mark.datafiles(DATA_DIR)
def test_nested_double(cli, tmpdir, datafiles):
- project_foo = os.path.join(str(datafiles), 'foo')
- copy_subprojects(project_foo, datafiles, ['base'])
+ project_foo = os.path.join(str(datafiles), "foo")
+ copy_subprojects(project_foo, datafiles, ["base"])
- project_bar = os.path.join(str(datafiles), 'bar')
- copy_subprojects(project_bar, datafiles, ['base'])
+ project_bar = os.path.join(str(datafiles), "bar")
+ copy_subprojects(project_bar, datafiles, ["base"])
- project = os.path.join(str(datafiles), 'toplevel')
- copy_subprojects(project, datafiles, ['base', 'foo', 'bar'])
+ project = os.path.join(str(datafiles), "toplevel")
+ copy_subprojects(project, datafiles, ["base", "foo", "bar"])
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Build, checkout
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
# Check that the checkout contains the expected files from all subprojects
- assert os.path.exists(os.path.join(checkoutdir, 'base.txt'))
- assert os.path.exists(os.path.join(checkoutdir, 'foo.txt'))
- assert os.path.exists(os.path.join(checkoutdir, 'bar.txt'))
+ assert os.path.exists(os.path.join(checkoutdir, "base.txt"))
+ assert os.path.exists(os.path.join(checkoutdir, "foo.txt"))
+ assert os.path.exists(os.path.join(checkoutdir, "bar.txt"))
@pytest.mark.datafiles(DATA_DIR)
def test_nested_conflict(cli, datafiles):
- project_foo = os.path.join(str(datafiles), 'foo')
- copy_subprojects(project_foo, datafiles, ['base'])
+ project_foo = os.path.join(str(datafiles), "foo")
+ copy_subprojects(project_foo, datafiles, ["base"])
- project_bar = os.path.join(str(datafiles), 'bar')
- copy_subprojects(project_bar, datafiles, ['base'])
+ project_bar = os.path.join(str(datafiles), "bar")
+ copy_subprojects(project_bar, datafiles, ["base"])
- project = os.path.join(str(datafiles), 'conflict')
- copy_subprojects(project, datafiles, ['foo', 'bar'])
+ project = os.path.join(str(datafiles), "conflict")
+ copy_subprojects(project, datafiles, ["foo", "bar"])
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.CONFLICTING_JUNCTION)
assert "bar.bst:target.bst [line 3 column 2]" in result.stderr
@@ -215,49 +210,49 @@ def test_nested_conflict(cli, datafiles):
# Test that we error correctly when the junction element itself is missing
@pytest.mark.datafiles(DATA_DIR)
def test_missing_junction(cli, datafiles):
- project = os.path.join(str(datafiles), 'invalid')
+ project = os.path.join(str(datafiles), "invalid")
- result = cli.run(project=project, args=['build', 'missing.bst'])
+ result = cli.run(project=project, args=["build", "missing.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE)
# Test that we error correctly when an element is not found in the subproject
@pytest.mark.datafiles(DATA_DIR)
def test_missing_subproject_element(cli, datafiles):
- project = os.path.join(str(datafiles), 'invalid')
- copy_subprojects(project, datafiles, ['base'])
+ project = os.path.join(str(datafiles), "invalid")
+ copy_subprojects(project, datafiles, ["base"])
- result = cli.run(project=project, args=['build', 'missing-element.bst'])
+ result = cli.run(project=project, args=["build", "missing-element.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE)
# Test that we error correctly when a junction itself has dependencies
@pytest.mark.datafiles(DATA_DIR)
def test_invalid_with_deps(cli, datafiles):
- project = os.path.join(str(datafiles), 'invalid')
- copy_subprojects(project, datafiles, ['base'])
+ project = os.path.join(str(datafiles), "invalid")
+ copy_subprojects(project, datafiles, ["base"])
- result = cli.run(project=project, args=['build', 'junction-with-deps.bst'])
+ result = cli.run(project=project, args=["build", "junction-with-deps.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_JUNCTION)
# Test that we error correctly when a junction is directly depended on
@pytest.mark.datafiles(DATA_DIR)
def test_invalid_junction_dep(cli, datafiles):
- project = os.path.join(str(datafiles), 'invalid')
- copy_subprojects(project, datafiles, ['base'])
+ project = os.path.join(str(datafiles), "invalid")
+ copy_subprojects(project, datafiles, ["base"])
- result = cli.run(project=project, args=['build', 'junction-dep.bst'])
+ result = cli.run(project=project, args=["build", "junction-dep.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
# Test that we error correctly when we junction-depend on a non-junction
@pytest.mark.datafiles(DATA_DIR)
def test_invalid_junctiondep_not_a_junction(cli, datafiles):
- project = os.path.join(str(datafiles), 'invalid')
- copy_subprojects(project, datafiles, ['base'])
+ project = os.path.join(str(datafiles), "invalid")
+ copy_subprojects(project, datafiles, ["base"])
- result = cli.run(project=project, args=['build', 'junctiondep-not-a-junction.bst'])
+ result = cli.run(project=project, args=["build", "junctiondep-not-a-junction.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
# Assert that we have the expected provenance encoded into the error
@@ -266,132 +261,117 @@ def test_invalid_junctiondep_not_a_junction(cli, datafiles):
@pytest.mark.datafiles(DATA_DIR)
def test_options_default(cli, tmpdir, datafiles):
- project = os.path.join(str(datafiles), 'options-default')
- copy_subprojects(project, datafiles, ['options-base'])
+ project = os.path.join(str(datafiles), "options-default")
+ copy_subprojects(project, datafiles, ["options-base"])
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Build, checkout
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
- assert os.path.exists(os.path.join(checkoutdir, 'pony.txt'))
- assert not os.path.exists(os.path.join(checkoutdir, 'horsy.txt'))
+ assert os.path.exists(os.path.join(checkoutdir, "pony.txt"))
+ assert not os.path.exists(os.path.join(checkoutdir, "horsy.txt"))
@pytest.mark.datafiles(DATA_DIR)
def test_options(cli, tmpdir, datafiles):
- project = os.path.join(str(datafiles), 'options')
- copy_subprojects(project, datafiles, ['options-base'])
+ project = os.path.join(str(datafiles), "options")
+ copy_subprojects(project, datafiles, ["options-base"])
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Build, checkout
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
- assert not os.path.exists(os.path.join(checkoutdir, 'pony.txt'))
- assert os.path.exists(os.path.join(checkoutdir, 'horsy.txt'))
+ assert not os.path.exists(os.path.join(checkoutdir, "pony.txt"))
+ assert os.path.exists(os.path.join(checkoutdir, "horsy.txt"))
@pytest.mark.datafiles(DATA_DIR)
def test_options_inherit(cli, tmpdir, datafiles):
- project = os.path.join(str(datafiles), 'options-inherit')
- copy_subprojects(project, datafiles, ['options-base'])
+ project = os.path.join(str(datafiles), "options-inherit")
+ copy_subprojects(project, datafiles, ["options-base"])
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Build, checkout
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
- assert not os.path.exists(os.path.join(checkoutdir, 'pony.txt'))
- assert os.path.exists(os.path.join(checkoutdir, 'horsy.txt'))
+ assert not os.path.exists(os.path.join(checkoutdir, "pony.txt"))
+ assert os.path.exists(os.path.join(checkoutdir, "horsy.txt"))
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
@pytest.mark.datafiles(DATA_DIR)
def test_git_show(cli, tmpdir, datafiles):
- project = os.path.join(str(datafiles), 'foo')
+ project = os.path.join(str(datafiles), "foo")
# Create the repo from 'base' subdir
- repo = create_repo('git', str(tmpdir))
- ref = repo.create(os.path.join(str(datafiles), 'base'))
+ repo = create_repo("git", str(tmpdir))
+ ref = repo.create(os.path.join(str(datafiles), "base"))
# Write out junction element with git source
- element = {
- 'kind': 'junction',
- 'sources': [
- repo.source_config(ref=ref)
- ]
- }
- _yaml.roundtrip_dump(element, os.path.join(project, 'base.bst'))
+ element = {"kind": "junction", "sources": [repo.source_config(ref=ref)]}
+ _yaml.roundtrip_dump(element, os.path.join(project, "base.bst"))
# Check that bst show succeeds with implicit subproject fetching and the
# pipeline includes the subproject element
- element_list = cli.get_pipeline(project, ['target.bst'])
- assert 'base.bst:target.bst' in element_list
+ element_list = cli.get_pipeline(project, ["target.bst"])
+ assert "base.bst:target.bst" in element_list
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
@pytest.mark.datafiles(DATA_DIR)
def test_git_build(cli, tmpdir, datafiles):
- project = os.path.join(str(datafiles), 'foo')
+ project = os.path.join(str(datafiles), "foo")
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Create the repo from 'base' subdir
- repo = create_repo('git', str(tmpdir))
- ref = repo.create(os.path.join(str(datafiles), 'base'))
+ repo = create_repo("git", str(tmpdir))
+ ref = repo.create(os.path.join(str(datafiles), "base"))
# Write out junction element with git source
- element = {
- 'kind': 'junction',
- 'sources': [
- repo.source_config(ref=ref)
- ]
- }
- _yaml.roundtrip_dump(element, os.path.join(project, 'base.bst'))
+ element = {"kind": "junction", "sources": [repo.source_config(ref=ref)]}
+ _yaml.roundtrip_dump(element, os.path.join(project, "base.bst"))
# Build (with implicit fetch of subproject), checkout
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
# Check that the checkout contains the expected files from both projects
- assert os.path.exists(os.path.join(checkoutdir, 'base.txt'))
- assert os.path.exists(os.path.join(checkoutdir, 'foo.txt'))
+ assert os.path.exists(os.path.join(checkoutdir, "base.txt"))
+ assert os.path.exists(os.path.join(checkoutdir, "foo.txt"))
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
@pytest.mark.datafiles(DATA_DIR)
def test_git_missing_project_conf(cli, tmpdir, datafiles):
- project = datafiles / 'foo'
+ project = datafiles / "foo"
# See test_junction_missing_project_conf for some more background.
# py3.5 requires this str conversion.
- os.remove(str(datafiles / 'base' / 'project.conf'))
+ os.remove(str(datafiles / "base" / "project.conf"))
# Create the repo from 'base' subdir
- repo = create_repo('git', str(tmpdir))
- ref = repo.create(os.path.join(str(datafiles), 'base'))
+ repo = create_repo("git", str(tmpdir))
+ ref = repo.create(os.path.join(str(datafiles), "base"))
# Write out junction element with git source
- element = {
- 'kind': 'junction',
- 'sources': [
- repo.source_config(ref=ref)
- ]
- }
- _yaml.roundtrip_dump(element, str(project / 'base.bst'))
-
- result = cli.run(project=project, args=['build', 'app.bst'])
+ element = {"kind": "junction", "sources": [repo.source_config(ref=ref)]}
+ _yaml.roundtrip_dump(element, str(project / "base.bst"))
+
+ result = cli.run(project=project, args=["build", "app.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_JUNCTION)
# Assert that we have the expected provenance encoded into the error
@@ -400,64 +380,59 @@ def test_git_missing_project_conf(cli, tmpdir, datafiles):
@pytest.mark.datafiles(DATA_DIR)
def test_cross_junction_names(cli, datafiles):
- project = os.path.join(str(datafiles), 'foo')
- copy_subprojects(project, datafiles, ['base'])
+ project = os.path.join(str(datafiles), "foo")
+ copy_subprojects(project, datafiles, ["base"])
- element_list = cli.get_pipeline(project, ['base.bst:target.bst'])
- assert 'base.bst:target.bst' in element_list
+ element_list = cli.get_pipeline(project, ["base.bst:target.bst"])
+ assert "base.bst:target.bst" in element_list
@pytest.mark.datafiles(DATA_DIR)
def test_build_git_cross_junction_names(cli, tmpdir, datafiles):
- project = os.path.join(str(datafiles), 'foo')
+ project = os.path.join(str(datafiles), "foo")
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Create the repo from 'base' subdir
- repo = create_repo('git', str(tmpdir))
- ref = repo.create(os.path.join(str(datafiles), 'base'))
+ repo = create_repo("git", str(tmpdir))
+ ref = repo.create(os.path.join(str(datafiles), "base"))
# Write out junction element with git source
- element = {
- 'kind': 'junction',
- 'sources': [
- repo.source_config(ref=ref)
- ]
- }
- _yaml.roundtrip_dump(element, os.path.join(project, 'base.bst'))
+ element = {"kind": "junction", "sources": [repo.source_config(ref=ref)]}
+ _yaml.roundtrip_dump(element, os.path.join(project, "base.bst"))
print(element)
- print(cli.get_pipeline(project, ['base.bst']))
+ print(cli.get_pipeline(project, ["base.bst"]))
# Build (with implicit fetch of subproject), checkout
- result = cli.run(project=project, args=['build', 'base.bst:target.bst'])
+ result = cli.run(project=project, args=["build", "base.bst:target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'base.bst:target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "base.bst:target.bst", "--directory", checkoutdir])
result.assert_success()
# Check that the checkout contains the expected files from both projects
- assert os.path.exists(os.path.join(checkoutdir, 'base.txt'))
+ assert os.path.exists(os.path.join(checkoutdir, "base.txt"))
@pytest.mark.datafiles(DATA_DIR)
def test_config_target(cli, tmpdir, datafiles):
- project = os.path.join(str(datafiles), 'config-target')
- checkoutdir = os.path.join(str(tmpdir), 'checkout')
+ project = os.path.join(str(datafiles), "config-target")
+ checkoutdir = os.path.join(str(tmpdir), "checkout")
# Build, checkout
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
# Check that the checkout contains the expected files from sub-sub-project
- assert os.path.exists(os.path.join(checkoutdir, 'hello.txt'))
+ assert os.path.exists(os.path.join(checkoutdir, "hello.txt"))
@pytest.mark.datafiles(DATA_DIR)
def test_invalid_sources_and_target(cli, tmpdir, datafiles):
- project = os.path.join(str(datafiles), 'config-target')
+ project = os.path.join(str(datafiles), "config-target")
- result = cli.run(project=project, args=['show', 'invalid-source-target.bst'])
+ result = cli.run(project=project, args=["show", "invalid-source-target.bst"])
result.assert_main_error(ErrorDomain.ELEMENT, None)
assert "junction elements cannot define both 'sources' and 'target' config option" in result.stderr
@@ -465,15 +440,15 @@ def test_invalid_sources_and_target(cli, tmpdir, datafiles):
@pytest.mark.datafiles(DATA_DIR)
def test_invalid_target_name(cli, tmpdir, datafiles):
- project = os.path.join(str(datafiles), 'config-target')
+ project = os.path.join(str(datafiles), "config-target")
# Rename our junction element to the same name as its target
- old_path = os.path.join(project, 'elements/subsubproject.bst')
- new_path = os.path.join(project, 'elements/subsubproject-junction.bst')
+ old_path = os.path.join(project, "elements/subsubproject.bst")
+ new_path = os.path.join(project, "elements/subsubproject-junction.bst")
os.rename(old_path, new_path)
# This should fail now
- result = cli.run(project=project, args=['show', 'subsubproject-junction.bst'])
+ result = cli.run(project=project, args=["show", "subsubproject-junction.bst"])
result.assert_main_error(ErrorDomain.ELEMENT, None)
assert "junction elements cannot target an element with the same name" in result.stderr
@@ -481,12 +456,12 @@ def test_invalid_target_name(cli, tmpdir, datafiles):
# We cannot exhaustively test all possible ways in which this can go wrong, so
# test a couple of common ways in which we expect this to go wrong.
-@pytest.mark.parametrize('target', ['no-junction.bst', 'nested-junction-target.bst'])
+@pytest.mark.parametrize("target", ["no-junction.bst", "nested-junction-target.bst"])
@pytest.mark.datafiles(DATA_DIR)
def test_invalid_target_format(cli, tmpdir, datafiles, target):
- project = os.path.join(str(datafiles), 'config-target')
+ project = os.path.join(str(datafiles), "config-target")
- result = cli.run(project=project, args=['show', target])
+ result = cli.run(project=project, args=["show", target])
result.assert_main_error(ErrorDomain.ELEMENT, None)
assert "'target' option must be in format '{junction-name}:{element-name}'" in result.stderr
diff --git a/tests/format/listdirectiveerrors.py b/tests/format/listdirectiveerrors.py
index 269b521a4..79102cb02 100644
--- a/tests/format/listdirectiveerrors.py
+++ b/tests/format/listdirectiveerrors.py
@@ -12,38 +12,28 @@ DATA_DIR = os.path.dirname(os.path.realpath(__file__))
@pytest.mark.datafiles(DATA_DIR)
def test_project_error(cli, datafiles):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'list-directive-error-project')
- result = cli.run(project=project, silent=True, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "list-directive-error-project")
+ result = cli.run(
+ project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"]
+ )
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.TRAILING_LIST_DIRECTIVE)
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("target", [
- ('variables.bst'), ('environment.bst'), ('config.bst'), ('public.bst')
-])
+@pytest.mark.parametrize("target", [("variables.bst"), ("environment.bst"), ("config.bst"), ("public.bst")])
def test_element_error(cli, datafiles, target):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'list-directive-error-element')
- result = cli.run(project=project, silent=True, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- target])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "list-directive-error-element")
+ result = cli.run(project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", target])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.TRAILING_LIST_DIRECTIVE)
@pytest.mark.datafiles(DATA_DIR)
def test_project_composite_error(cli, datafiles):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'list-directive-type-error')
- result = cli.run(project=project, silent=True, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "list-directive-type-error")
+ result = cli.run(
+ project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"]
+ )
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.ILLEGAL_COMPOSITE)
diff --git a/tests/format/optionarch.py b/tests/format/optionarch.py
index f347e27ae..75cae9abe 100644
--- a/tests/format/optionarch.py
+++ b/tests/format/optionarch.py
@@ -16,52 +16,45 @@ DATA_DIR = os.path.dirname(os.path.realpath(__file__))
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("machine,value,expected", [
- # Test explicitly provided arches
- ('arm', 'aarch32', 'Army'),
- ('arm', 'aarch64', 'Aarchy'),
-
- # Test automatically derived arches
- ('arm', None, 'Army'),
- ('aarch64', None, 'Aarchy'),
-
- # Test that explicitly provided arches dont error out
- # when the `uname` reported arch is not supported
- ('i386', 'aarch32', 'Army'),
- ('x86_64', 'aarch64', 'Aarchy'),
-])
+@pytest.mark.parametrize(
+ "machine,value,expected",
+ [
+ # Test explicitly provided arches
+ ("arm", "aarch32", "Army"),
+ ("arm", "aarch64", "Aarchy"),
+ # Test automatically derived arches
+ ("arm", None, "Army"),
+ ("aarch64", None, "Aarchy"),
+ # Test that explicitly provided arches dont error out
+ # when the `uname` reported arch is not supported
+ ("i386", "aarch32", "Army"),
+ ("x86_64", "aarch64", "Aarchy"),
+ ],
+)
def test_conditional(cli, datafiles, machine, value, expected):
with override_platform_uname(machine=machine):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'option-arch')
+ project = os.path.join(datafiles.dirname, datafiles.basename, "option-arch")
bst_args = []
if value is not None:
- bst_args += ['--option', 'machine_arch', value]
-
- bst_args += [
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'
- ]
+ bst_args += ["--option", "machine_arch", value]
+
+ bst_args += ["show", "--deps", "none", "--format", "%{vars}", "element.bst"]
result = cli.run(project=project, silent=True, args=bst_args)
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert loaded.get_str('result') == expected
+ assert loaded.get_str("result") == expected
@pytest.mark.datafiles(DATA_DIR)
def test_unsupported_arch(cli, datafiles):
with override_platform_uname(machine="x86_64"):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'option-arch')
- result = cli.run(project=project, silent=True, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'
- ])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "option-arch")
+ result = cli.run(
+ project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"]
+ )
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
@@ -70,13 +63,10 @@ def test_unsupported_arch(cli, datafiles):
def test_alias(cli, datafiles):
with override_platform_uname(machine="arm"):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'option-arch-alias')
- result = cli.run(project=project, silent=True, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'
- ])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "option-arch-alias")
+ result = cli.run(
+ project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"]
+ )
result.assert_success()
@@ -85,13 +75,10 @@ def test_alias(cli, datafiles):
def test_unknown_host_arch(cli, datafiles):
with override_platform_uname(machine="x86_128"):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'option-arch')
- result = cli.run(project=project, silent=True, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'
- ])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "option-arch")
+ result = cli.run(
+ project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"]
+ )
result.assert_main_error(ErrorDomain.PLATFORM, None)
@@ -99,12 +86,9 @@ def test_unknown_host_arch(cli, datafiles):
@pytest.mark.datafiles(DATA_DIR)
def test_unknown_project_arch(cli, datafiles):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'option-arch-unknown')
- result = cli.run(project=project, silent=True, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'
- ])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "option-arch-unknown")
+ result = cli.run(
+ project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"]
+ )
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
diff --git a/tests/format/optionbool.py b/tests/format/optionbool.py
index d772b483c..3b05aafa1 100644
--- a/tests/format/optionbool.py
+++ b/tests/format/optionbool.py
@@ -12,101 +12,72 @@ DATA_DIR = os.path.dirname(os.path.realpath(__file__))
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("target,option,expected", [
- # Test 'foo' syntax, and valid values of 'True' / 'False'
- ('element.bst', 'True', 'a pony'),
- ('element.bst', 'true', 'a pony'),
- ('element.bst', 'False', 'not pony'),
- ('element.bst', 'false', 'not pony'),
-
- # Test 'not foo' syntax
- ('element-not.bst', 'False', 'not pony'),
- ('element-not.bst', 'True', 'a pony'),
-
- # Test 'foo == True' syntax
- ('element-equals.bst', 'False', 'not pony'),
- ('element-equals.bst', 'True', 'a pony'),
-
- # Test 'foo != True' syntax
- ('element-not-equals.bst', 'False', 'not pony'),
- ('element-not-equals.bst', 'True', 'a pony'),
-])
+@pytest.mark.parametrize(
+ "target,option,expected",
+ [
+ # Test 'foo' syntax, and valid values of 'True' / 'False'
+ ("element.bst", "True", "a pony"),
+ ("element.bst", "true", "a pony"),
+ ("element.bst", "False", "not pony"),
+ ("element.bst", "false", "not pony"),
+ # Test 'not foo' syntax
+ ("element-not.bst", "False", "not pony"),
+ ("element-not.bst", "True", "a pony"),
+ # Test 'foo == True' syntax
+ ("element-equals.bst", "False", "not pony"),
+ ("element-equals.bst", "True", "a pony"),
+ # Test 'foo != True' syntax
+ ("element-not-equals.bst", "False", "not pony"),
+ ("element-not-equals.bst", "True", "a pony"),
+ ],
+)
def test_conditional_cli(cli, datafiles, target, option, expected):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'option-bool')
- result = cli.run(project=project, silent=True, args=[
- '--option', 'pony', option,
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- target])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "option-bool")
+ result = cli.run(
+ project=project,
+ silent=True,
+ args=["--option", "pony", option, "show", "--deps", "none", "--format", "%{vars}", target],
+ )
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert loaded.get_str('thepony') == expected
+ assert loaded.get_str("thepony") == expected
# Test configuration of boolean option in the config file
#
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("target,option,expected", [
- ('element.bst', True, 'a pony'),
- ('element.bst', False, 'not pony'),
-])
+@pytest.mark.parametrize(
+ "target,option,expected", [("element.bst", True, "a pony"), ("element.bst", False, "not pony"),]
+)
def test_conditional_config(cli, datafiles, target, option, expected):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'option-bool')
- cli.configure({
- 'projects': {
- 'test': {
- 'options': {
- 'pony': option
- }
- }
- }
- })
- result = cli.run(project=project, silent=True, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- target])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "option-bool")
+ cli.configure({"projects": {"test": {"options": {"pony": option}}}})
+ result = cli.run(project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", target])
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert loaded.get_str('thepony') == expected
+ assert loaded.get_str("thepony") == expected
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("cli_option", [
- ('falsey'), ('pony'), ('trUE')
-])
+@pytest.mark.parametrize("cli_option", [("falsey"), ("pony"), ("trUE")])
def test_invalid_value_cli(cli, datafiles, cli_option):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'option-bool')
- result = cli.run(project=project, silent=True, args=[
- '--option', 'pony', cli_option,
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "option-bool")
+ result = cli.run(
+ project=project,
+ silent=True,
+ args=["--option", "pony", cli_option, "show", "--deps", "none", "--format", "%{vars}", "element.bst"],
+ )
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("config_option", [
- ('pony'), (['its', 'a', 'list']), ({'dic': 'tionary'})
-])
+@pytest.mark.parametrize("config_option", [("pony"), (["its", "a", "list"]), ({"dic": "tionary"})])
def test_invalid_value_config(cli, datafiles, config_option):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'option-bool')
- cli.configure({
- 'projects': {
- 'test': {
- 'options': {
- 'pony': config_option
- }
- }
- }
- })
- result = cli.run(project=project, silent=True, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "option-bool")
+ cli.configure({"projects": {"test": {"options": {"pony": config_option}}}})
+ result = cli.run(
+ project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"]
+ )
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
diff --git a/tests/format/optioneltmask.py b/tests/format/optioneltmask.py
index 75265fdd7..399b37b97 100644
--- a/tests/format/optioneltmask.py
+++ b/tests/format/optioneltmask.py
@@ -12,73 +12,61 @@ DATA_DIR = os.path.dirname(os.path.realpath(__file__))
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("target,value,expected", [
- ('pony.bst', 'pony.bst', 'True'),
- ('horsy.bst', 'pony.bst, horsy.bst', 'True'),
- ('zebry.bst', 'pony.bst, horsy.bst', 'False'),
-])
+@pytest.mark.parametrize(
+ "target,value,expected",
+ [
+ ("pony.bst", "pony.bst", "True"),
+ ("horsy.bst", "pony.bst, horsy.bst", "True"),
+ ("zebry.bst", "pony.bst, horsy.bst", "False"),
+ ],
+)
def test_conditional_cli(cli, datafiles, target, value, expected):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'option-element-mask')
- result = cli.run(project=project, silent=True, args=[
- '--option', 'debug_elements', value,
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- target])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "option-element-mask")
+ result = cli.run(
+ project=project,
+ silent=True,
+ args=["--option", "debug_elements", value, "show", "--deps", "none", "--format", "%{vars}", target],
+ )
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert loaded.get_str('debug') == expected
+ assert loaded.get_str("debug") == expected
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("target,value,expected", [
- ('pony.bst', ['pony.bst'], 'True'),
- ('horsy.bst', ['pony.bst', 'horsy.bst'], 'True'),
- ('zebry.bst', ['pony.bst', 'horsy.bst'], 'False'),
-])
+@pytest.mark.parametrize(
+ "target,value,expected",
+ [
+ ("pony.bst", ["pony.bst"], "True"),
+ ("horsy.bst", ["pony.bst", "horsy.bst"], "True"),
+ ("zebry.bst", ["pony.bst", "horsy.bst"], "False"),
+ ],
+)
def test_conditional_config(cli, datafiles, target, value, expected):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'option-element-mask')
- cli.configure({
- 'projects': {
- 'test': {
- 'options': {
- 'debug_elements': value
- }
- }
- }
- })
- result = cli.run(project=project, silent=True, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- target])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "option-element-mask")
+ cli.configure({"projects": {"test": {"options": {"debug_elements": value}}}})
+ result = cli.run(project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", target])
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert loaded.get_str('debug') == expected
+ assert loaded.get_str("debug") == expected
@pytest.mark.datafiles(DATA_DIR)
def test_invalid_declaration(cli, datafiles):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'option-element-mask-invalid')
- result = cli.run(project=project, silent=True, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'pony.bst'])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "option-element-mask-invalid")
+ result = cli.run(project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "pony.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
@pytest.mark.datafiles(DATA_DIR)
def test_invalid_value(cli, datafiles):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'option-element-mask')
- result = cli.run(project=project, silent=True, args=[
- '--option', 'debug_elements', 'kitten.bst',
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'pony.bst'])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "option-element-mask")
+ result = cli.run(
+ project=project,
+ silent=True,
+ args=["--option", "debug_elements", "kitten.bst", "show", "--deps", "none", "--format", "%{vars}", "pony.bst"],
+ )
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
diff --git a/tests/format/optionenum.py b/tests/format/optionenum.py
index f9aff503f..ba79bea79 100644
--- a/tests/format/optionenum.py
+++ b/tests/format/optionenum.py
@@ -12,104 +12,82 @@ DATA_DIR = os.path.dirname(os.path.realpath(__file__))
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("target,option,value,expected", [
- # Test 'var == "foo"' syntax
- ('element.bst', 'brother', 'pony', 'a pony'),
- ('element.bst', 'brother', 'zebry', 'a zebry'),
- ('element.bst', 'brother', 'horsy', 'a horsy'),
- # Test 'var1 == var2' syntax
- ('element-compare.bst', 'brother', 'horsy', 'different'),
- ('element-compare.bst', 'brother', 'zebry', 'same'),
- ('element-compare.bst', 'sister', 'pony', 'same'),
-])
+@pytest.mark.parametrize(
+ "target,option,value,expected",
+ [
+ # Test 'var == "foo"' syntax
+ ("element.bst", "brother", "pony", "a pony"),
+ ("element.bst", "brother", "zebry", "a zebry"),
+ ("element.bst", "brother", "horsy", "a horsy"),
+ # Test 'var1 == var2' syntax
+ ("element-compare.bst", "brother", "horsy", "different"),
+ ("element-compare.bst", "brother", "zebry", "same"),
+ ("element-compare.bst", "sister", "pony", "same"),
+ ],
+)
def test_conditional_cli(cli, datafiles, target, option, value, expected):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'option-enum')
- result = cli.run(project=project, silent=True, args=[
- '--option', option, value,
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- target])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "option-enum")
+ result = cli.run(
+ project=project,
+ silent=True,
+ args=["--option", option, value, "show", "--deps", "none", "--format", "%{vars}", target],
+ )
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert loaded.get_str('result') == expected
+ assert loaded.get_str("result") == expected
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("target,option,value,expected", [
- # Test 'var == "foo"' syntax
- ('element.bst', 'brother', 'pony', 'a pony'),
- ('element.bst', 'brother', 'zebry', 'a zebry'),
- ('element.bst', 'brother', 'horsy', 'a horsy'),
- # Test 'var1 == var2' syntax
- ('element-compare.bst', 'brother', 'horsy', 'different'),
- ('element-compare.bst', 'brother', 'zebry', 'same'),
- ('element-compare.bst', 'sister', 'pony', 'same'),
-])
+@pytest.mark.parametrize(
+ "target,option,value,expected",
+ [
+ # Test 'var == "foo"' syntax
+ ("element.bst", "brother", "pony", "a pony"),
+ ("element.bst", "brother", "zebry", "a zebry"),
+ ("element.bst", "brother", "horsy", "a horsy"),
+ # Test 'var1 == var2' syntax
+ ("element-compare.bst", "brother", "horsy", "different"),
+ ("element-compare.bst", "brother", "zebry", "same"),
+ ("element-compare.bst", "sister", "pony", "same"),
+ ],
+)
def test_conditional_config(cli, datafiles, target, option, value, expected):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'option-enum')
- cli.configure({
- 'projects': {
- 'test': {
- 'options': {
- option: value
- }
- }
- }
- })
- result = cli.run(project=project, silent=True, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- target])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "option-enum")
+ cli.configure({"projects": {"test": {"options": {option: value}}}})
+ result = cli.run(project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", target])
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert loaded.get_str('result') == expected
+ assert loaded.get_str("result") == expected
@pytest.mark.datafiles(DATA_DIR)
def test_invalid_value_cli(cli, datafiles):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'option-enum')
- result = cli.run(project=project, silent=True, args=[
- '--option', 'brother', 'giraffy',
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "option-enum")
+ result = cli.run(
+ project=project,
+ silent=True,
+ args=["--option", "brother", "giraffy", "show", "--deps", "none", "--format", "%{vars}", "element.bst"],
+ )
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("config_option", [
- ('giraffy'), (['its', 'a', 'list']), ({'dic': 'tionary'})
-])
+@pytest.mark.parametrize("config_option", [("giraffy"), (["its", "a", "list"]), ({"dic": "tionary"})])
def test_invalid_value_config(cli, datafiles, config_option):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'option-enum')
- cli.configure({
- 'projects': {
- 'test': {
- 'options': {
- 'brother': config_option
- }
- }
- }
- })
- result = cli.run(project=project, silent=True, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "option-enum")
+ cli.configure({"projects": {"test": {"options": {"brother": config_option}}}})
+ result = cli.run(
+ project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"]
+ )
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
@pytest.mark.datafiles(DATA_DIR)
def test_missing_values(cli, datafiles):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'option-enum-missing')
- result = cli.run(project=project, silent=True, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "option-enum-missing")
+ result = cli.run(
+ project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"]
+ )
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
diff --git a/tests/format/optionexports.py b/tests/format/optionexports.py
index 104abcf83..4f95dc109 100644
--- a/tests/format/optionexports.py
+++ b/tests/format/optionexports.py
@@ -11,28 +11,27 @@ DATA_DIR = os.path.dirname(os.path.realpath(__file__))
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("option_name,option_value,var_name,var_value", [
-
- # Test boolean
- ('bool_export', 'False', 'exported-bool', '0'),
- ('bool_export', 'True', 'exported-bool', '1'),
-
- # Enum
- ('enum_export', 'pony', 'exported-enum', 'pony'),
- ('enum_export', 'horsy', 'exported-enum', 'horsy'),
-
- # Flags
- ('flags_export', 'pony', 'exported-flags', 'pony'),
- ('flags_export', 'pony, horsy', 'exported-flags', 'horsy,pony'),
-])
+@pytest.mark.parametrize(
+ "option_name,option_value,var_name,var_value",
+ [
+ # Test boolean
+ ("bool_export", "False", "exported-bool", "0"),
+ ("bool_export", "True", "exported-bool", "1"),
+ # Enum
+ ("enum_export", "pony", "exported-enum", "pony"),
+ ("enum_export", "horsy", "exported-enum", "horsy"),
+ # Flags
+ ("flags_export", "pony", "exported-flags", "pony"),
+ ("flags_export", "pony, horsy", "exported-flags", "horsy,pony"),
+ ],
+)
def test_export(cli, datafiles, option_name, option_value, var_name, var_value):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'option-exports')
- result = cli.run(project=project, silent=True, args=[
- '--option', option_name, option_value,
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "option-exports")
+ result = cli.run(
+ project=project,
+ silent=True,
+ args=["--option", option_name, option_value, "show", "--deps", "none", "--format", "%{vars}", "element.bst"],
+ )
result.assert_success()
loaded = _yaml.load_data(result.output)
diff --git a/tests/format/optionflags.py b/tests/format/optionflags.py
index 29bb7ec2c..53e674919 100644
--- a/tests/format/optionflags.py
+++ b/tests/format/optionflags.py
@@ -12,113 +12,93 @@ DATA_DIR = os.path.dirname(os.path.realpath(__file__))
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("target,option,value,expected", [
- # Test (var == [ "foo" ]) syntax
- ('element.bst', 'farm', 'pony', 'a pony'),
- ('element.bst', 'farm', 'zebry', 'a zebry'),
- ('element.bst', 'farm', 'pony, horsy', 'a pony and a horsy'),
- ('element.bst', 'farm', 'zebry,horsy , pony', 'all the animals'),
-
- # Test ("literal" in var) syntax
- ('element-in.bst', 'farm', 'zebry, horsy, pony', 'a zebry'),
-
- # Test ("literal" not in var) syntax
- ('element-in.bst', 'farm', 'zebry, horsy', 'no pony'),
-
- # Test (var1 not in var2) syntax (where var1 is enum and var2 is flags)
- ('element-in.bst', 'farm', 'zebry, pony', 'no horsy'),
-])
+@pytest.mark.parametrize(
+ "target,option,value,expected",
+ [
+ # Test (var == [ "foo" ]) syntax
+ ("element.bst", "farm", "pony", "a pony"),
+ ("element.bst", "farm", "zebry", "a zebry"),
+ ("element.bst", "farm", "pony, horsy", "a pony and a horsy"),
+ ("element.bst", "farm", "zebry,horsy , pony", "all the animals"),
+ # Test ("literal" in var) syntax
+ ("element-in.bst", "farm", "zebry, horsy, pony", "a zebry"),
+ # Test ("literal" not in var) syntax
+ ("element-in.bst", "farm", "zebry, horsy", "no pony"),
+ # Test (var1 not in var2) syntax (where var1 is enum and var2 is flags)
+ ("element-in.bst", "farm", "zebry, pony", "no horsy"),
+ ],
+)
def test_conditional_cli(cli, datafiles, target, option, value, expected):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'option-flags')
- result = cli.run(project=project, silent=True, args=[
- '--option', option, value,
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- target])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "option-flags")
+ result = cli.run(
+ project=project,
+ silent=True,
+ args=["--option", option, value, "show", "--deps", "none", "--format", "%{vars}", target],
+ )
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert loaded.get_str('result') == expected
+ assert loaded.get_str("result") == expected
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("target,option,value,expected", [
- # Test 'var == [ "foo" ]' syntax
- ('element.bst', 'farm', ['pony'], 'a pony'),
- ('element.bst', 'farm', ['zebry'], 'a zebry'),
- ('element.bst', 'farm', ['pony', 'horsy'], 'a pony and a horsy'),
- ('element.bst', 'farm', ['zebry', 'horsy', 'pony'], 'all the animals'),
-])
+@pytest.mark.parametrize(
+ "target,option,value,expected",
+ [
+ # Test 'var == [ "foo" ]' syntax
+ ("element.bst", "farm", ["pony"], "a pony"),
+ ("element.bst", "farm", ["zebry"], "a zebry"),
+ ("element.bst", "farm", ["pony", "horsy"], "a pony and a horsy"),
+ ("element.bst", "farm", ["zebry", "horsy", "pony"], "all the animals"),
+ ],
+)
def test_conditional_config(cli, datafiles, target, option, value, expected):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'option-flags')
- cli.configure({
- 'projects': {
- 'test': {
- 'options': {
- option: value
- }
- }
- }
- })
- result = cli.run(project=project, silent=True, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- target])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "option-flags")
+ cli.configure({"projects": {"test": {"options": {option: value}}}})
+ result = cli.run(project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", target])
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert loaded.get_str('result') == expected
+ assert loaded.get_str("result") == expected
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("cli_option", [
- ('giraffy'), # Not a valid animal for the farm option
- ('horsy pony') # Does not include comma separators
-])
+@pytest.mark.parametrize(
+ "cli_option",
+ [("giraffy"), ("horsy pony")], # Not a valid animal for the farm option # Does not include comma separators
+)
def test_invalid_value_cli(cli, datafiles, cli_option):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'option-flags')
- result = cli.run(project=project, silent=True, args=[
- '--option', 'farm', cli_option,
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "option-flags")
+ result = cli.run(
+ project=project,
+ silent=True,
+ args=["--option", "farm", cli_option, "show", "--deps", "none", "--format", "%{vars}", "element.bst"],
+ )
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("config_option", [
- ('pony'), # Not specified as a list
- (['horsy', 'pony', 'giraffy']), # Invalid giraffy animal for farm option
- ({'dic': 'tionary'}) # Dicts also dont make sense in the config for flags
-])
+@pytest.mark.parametrize(
+ "config_option",
+ [
+ ("pony"), # Not specified as a list
+ (["horsy", "pony", "giraffy"]), # Invalid giraffy animal for farm option
+ ({"dic": "tionary"}), # Dicts also dont make sense in the config for flags
+ ],
+)
def test_invalid_value_config(cli, datafiles, config_option):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'option-flags')
- cli.configure({
- 'projects': {
- 'test': {
- 'options': {
- 'farm': config_option
- }
- }
- }
- })
- result = cli.run(project=project, silent=True, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "option-flags")
+ cli.configure({"projects": {"test": {"options": {"farm": config_option}}}})
+ result = cli.run(
+ project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"]
+ )
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
@pytest.mark.datafiles(DATA_DIR)
def test_missing_values(cli, datafiles):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'option-flags-missing')
- result = cli.run(project=project, silent=True, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "option-flags-missing")
+ result = cli.run(
+ project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"]
+ )
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
diff --git a/tests/format/optionos.py b/tests/format/optionos.py
index f915d889e..92486104e 100644
--- a/tests/format/optionos.py
+++ b/tests/format/optionos.py
@@ -15,51 +15,44 @@ DATA_DIR = os.path.dirname(os.path.realpath(__file__))
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("system,value,expected", [
- # Test explicitly provided arches
- ('Darwin', 'Linux', 'Linuxy'),
- ('SunOS', 'FreeBSD', 'FreeBSDy'),
-
- # Test automatically derived arches
- ('Linux', None, 'Linuxy'),
- ('Darwin', None, 'Darwiny'),
-
- # Test that explicitly provided arches dont error out
- # when the `uname` reported arch is not supported
- ('AIX', 'Linux', 'Linuxy'),
- ('HaikuOS', 'SunOS', 'SunOSy'),
-])
+@pytest.mark.parametrize(
+ "system,value,expected",
+ [
+ # Test explicitly provided arches
+ ("Darwin", "Linux", "Linuxy"),
+ ("SunOS", "FreeBSD", "FreeBSDy"),
+ # Test automatically derived arches
+ ("Linux", None, "Linuxy"),
+ ("Darwin", None, "Darwiny"),
+ # Test that explicitly provided arches dont error out
+ # when the `uname` reported arch is not supported
+ ("AIX", "Linux", "Linuxy"),
+ ("HaikuOS", "SunOS", "SunOSy"),
+ ],
+)
def test_conditionals(cli, datafiles, system, value, expected):
with override_platform_uname(system=system):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'option-os')
+ project = os.path.join(datafiles.dirname, datafiles.basename, "option-os")
bst_args = []
if value is not None:
- bst_args += ['--option', 'machine_os', value]
+ bst_args += ["--option", "machine_os", value]
- bst_args += [
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'
- ]
+ bst_args += ["show", "--deps", "none", "--format", "%{vars}", "element.bst"]
result = cli.run(project=project, silent=True, args=bst_args)
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert loaded.get_str('result') == expected
+ assert loaded.get_str("result") == expected
@pytest.mark.datafiles(DATA_DIR)
def test_unsupported_arch(cli, datafiles):
with override_platform_uname(system="AIX"):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'option-os')
- result = cli.run(project=project, silent=True, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'
- ])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "option-os")
+ result = cli.run(
+ project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"]
+ )
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
diff --git a/tests/format/optionoverrides.py b/tests/format/optionoverrides.py
index d4ed257dd..ba12e751f 100644
--- a/tests/format/optionoverrides.py
+++ b/tests/format/optionoverrides.py
@@ -11,22 +11,17 @@ DATA_DIR = os.path.dirname(os.path.realpath(__file__))
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("arch", [('i686'), ('x86_64')])
+@pytest.mark.parametrize("arch", [("i686"), ("x86_64")])
def test_override(cli, datafiles, arch):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'option-overrides')
+ project = os.path.join(datafiles.dirname, datafiles.basename, "option-overrides")
- bst_args = ['--option', 'arch', arch]
- bst_args += [
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'
- ]
+ bst_args = ["--option", "arch", arch]
+ bst_args += ["show", "--deps", "none", "--format", "%{vars}", "element.bst"]
result = cli.run(project=project, silent=True, args=bst_args)
result.assert_success()
# See the associated project.conf for the expected values
- expected_value = '--host={}-unknown-linux-gnu'.format(arch)
+ expected_value = "--host={}-unknown-linux-gnu".format(arch)
loaded = _yaml.load_data(result.output)
- assert loaded.get_str('conf-global') == expected_value
+ assert loaded.get_str("conf-global") == expected_value
diff --git a/tests/format/options.py b/tests/format/options.py
index 9c0e043f9..aa0854e96 100644
--- a/tests/format/options.py
+++ b/tests/format/options.py
@@ -8,244 +8,233 @@ from buildstream._exceptions import ErrorDomain, LoadErrorReason
from buildstream.testing.runcli import cli # pylint: disable=unused-import
# Project directory
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- 'options'
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "options")
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("project_dir", [
- ('invalid-name-spaces'),
- ('invalid-name-dashes'),
- ('invalid-name-plus'),
- ('invalid-name-leading-number'),
-])
+@pytest.mark.parametrize(
+ "project_dir",
+ [("invalid-name-spaces"), ("invalid-name-dashes"), ("invalid-name-plus"), ("invalid-name-leading-number"),],
+)
def test_invalid_option_name(cli, datafiles, project_dir):
project = os.path.join(datafiles.dirname, datafiles.basename, project_dir)
- result = cli.run(project=project, silent=True, args=['show', 'element.bst'])
+ result = cli.run(project=project, silent=True, args=["show", "element.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_SYMBOL_NAME)
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("project_dir", [
- ('invalid-variable-name-spaces'),
- ('invalid-variable-name-plus'),
-])
+@pytest.mark.parametrize("project_dir", [("invalid-variable-name-spaces"), ("invalid-variable-name-plus"),])
def test_invalid_variable_name(cli, datafiles, project_dir):
project = os.path.join(datafiles.dirname, datafiles.basename, project_dir)
- result = cli.run(project=project, silent=True, args=['show', 'element.bst'])
+ result = cli.run(project=project, silent=True, args=["show", "element.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_SYMBOL_NAME)
@pytest.mark.datafiles(DATA_DIR)
def test_invalid_option_type(cli, datafiles):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'invalid-type')
+ project = os.path.join(datafiles.dirname, datafiles.basename, "invalid-type")
# Test with the opt option set
- result = cli.run(project=project, silent=True, args=[
- '--option', 'opt', 'funny',
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'])
+ result = cli.run(
+ project=project,
+ silent=True,
+ args=["--option", "opt", "funny", "show", "--deps", "none", "--format", "%{vars}", "element.bst"],
+ )
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
@pytest.mark.datafiles(DATA_DIR)
def test_invalid_option_cli(cli, datafiles):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'simple-condition')
+ project = os.path.join(datafiles.dirname, datafiles.basename, "simple-condition")
# Test with the opt option set
- result = cli.run(project=project, silent=True, args=[
- '--option', 'fart', 'funny',
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'])
+ result = cli.run(
+ project=project,
+ silent=True,
+ args=["--option", "fart", "funny", "show", "--deps", "none", "--format", "%{vars}", "element.bst"],
+ )
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
@pytest.mark.datafiles(DATA_DIR)
def test_invalid_option_config(cli, datafiles):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'simple-condition')
- cli.configure({
- 'projects': {
- 'test': {
- 'options': {
- 'fart': 'Hello'
- }
- }
- }
- })
- result = cli.run(project=project, silent=True, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "simple-condition")
+ cli.configure({"projects": {"test": {"options": {"fart": "Hello"}}}})
+ result = cli.run(
+ project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"]
+ )
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
@pytest.mark.datafiles(DATA_DIR)
def test_invalid_expression(cli, datafiles):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'invalid-expression')
- result = cli.run(project=project, silent=True, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "invalid-expression")
+ result = cli.run(
+ project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"]
+ )
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.EXPRESSION_FAILED)
@pytest.mark.datafiles(DATA_DIR)
def test_undefined(cli, datafiles):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'undefined-variable')
- result = cli.run(project=project, silent=True, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "undefined-variable")
+ result = cli.run(
+ project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"]
+ )
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.EXPRESSION_FAILED)
@pytest.mark.datafiles(DATA_DIR)
def test_invalid_condition(cli, datafiles):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'invalid-condition')
- result = cli.run(project=project, silent=True, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "invalid-condition")
+ result = cli.run(
+ project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"]
+ )
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("opt_option,expected_prefix", [
- ('False', '/usr'),
- ('True', '/opt'),
-])
+@pytest.mark.parametrize("opt_option,expected_prefix", [("False", "/usr"), ("True", "/opt"),])
def test_simple_conditional(cli, datafiles, opt_option, expected_prefix):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'simple-condition')
+ project = os.path.join(datafiles.dirname, datafiles.basename, "simple-condition")
# Test with the opt option set
- result = cli.run(project=project, silent=True, args=[
- '--option', 'opt', opt_option,
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'])
+ result = cli.run(
+ project=project,
+ silent=True,
+ args=["--option", "opt", opt_option, "show", "--deps", "none", "--format", "%{vars}", "element.bst"],
+ )
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert loaded.get_str('prefix') == expected_prefix
+ assert loaded.get_str("prefix") == expected_prefix
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("debug,logging,expected", [
- ('False', 'False', 'False'),
- ('True', 'False', 'False'),
- ('False', 'True', 'False'),
- ('True', 'True', 'True'),
-])
+@pytest.mark.parametrize(
+ "debug,logging,expected",
+ [("False", "False", "False"), ("True", "False", "False"), ("False", "True", "False"), ("True", "True", "True"),],
+)
def test_nested_conditional(cli, datafiles, debug, logging, expected):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'nested-condition')
+ project = os.path.join(datafiles.dirname, datafiles.basename, "nested-condition")
# Test with the opt option set
- result = cli.run(project=project, silent=True, args=[
- '--option', 'debug', debug,
- '--option', 'logging', logging,
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'])
+ result = cli.run(
+ project=project,
+ silent=True,
+ args=[
+ "--option",
+ "debug",
+ debug,
+ "--option",
+ "logging",
+ logging,
+ "show",
+ "--deps",
+ "none",
+ "--format",
+ "%{vars}",
+ "element.bst",
+ ],
+ )
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert loaded.get_str('debug') == expected
+ assert loaded.get_str("debug") == expected
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("debug,logging,expected", [
- ('False', 'False', 'False'),
- ('True', 'False', 'False'),
- ('False', 'True', 'False'),
- ('True', 'True', 'True'),
-])
+@pytest.mark.parametrize(
+ "debug,logging,expected",
+ [("False", "False", "False"), ("True", "False", "False"), ("False", "True", "False"), ("True", "True", "True"),],
+)
def test_compound_and_conditional(cli, datafiles, debug, logging, expected):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'compound-and-condition')
+ project = os.path.join(datafiles.dirname, datafiles.basename, "compound-and-condition")
# Test with the opt option set
- result = cli.run(project=project, silent=True, args=[
- '--option', 'debug', debug,
- '--option', 'logging', logging,
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'])
+ result = cli.run(
+ project=project,
+ silent=True,
+ args=[
+ "--option",
+ "debug",
+ debug,
+ "--option",
+ "logging",
+ logging,
+ "show",
+ "--deps",
+ "none",
+ "--format",
+ "%{vars}",
+ "element.bst",
+ ],
+ )
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert loaded.get_str('debug') == expected
+ assert loaded.get_str("debug") == expected
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("debug,logging,expected", [
- ('False', 'False', 'False'),
- ('True', 'False', 'True'),
- ('False', 'True', 'True'),
- ('True', 'True', 'True'),
-])
+@pytest.mark.parametrize(
+ "debug,logging,expected",
+ [("False", "False", "False"), ("True", "False", "True"), ("False", "True", "True"), ("True", "True", "True"),],
+)
def test_compound_or_conditional(cli, datafiles, debug, logging, expected):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'compound-or-condition')
+ project = os.path.join(datafiles.dirname, datafiles.basename, "compound-or-condition")
# Test with the opt option set
- result = cli.run(project=project, silent=True, args=[
- '--option', 'debug', debug,
- '--option', 'logging', logging,
- 'show',
- '--deps', 'none',
- '--format', '%{vars}',
- 'element.bst'])
+ result = cli.run(
+ project=project,
+ silent=True,
+ args=[
+ "--option",
+ "debug",
+ debug,
+ "--option",
+ "logging",
+ logging,
+ "show",
+ "--deps",
+ "none",
+ "--format",
+ "%{vars}",
+ "element.bst",
+ ],
+ )
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert loaded.get_str('logging') == expected
+ assert loaded.get_str("logging") == expected
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("option,expected", [
- ('False', 'horsy'),
- ('True', 'pony'),
-])
+@pytest.mark.parametrize("option,expected", [("False", "horsy"), ("True", "pony"),])
def test_deep_nesting_level1(cli, datafiles, option, expected):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'deep-nesting')
- result = cli.run(project=project, silent=True, args=[
- '--option', 'pony', option,
- 'show',
- '--deps', 'none',
- '--format', '%{public}',
- 'element.bst'])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "deep-nesting")
+ result = cli.run(
+ project=project,
+ silent=True,
+ args=["--option", "pony", option, "show", "--deps", "none", "--format", "%{public}", "element.bst"],
+ )
result.assert_success()
loaded = _yaml.load_data(result.output)
- shallow_list = loaded.get_sequence('shallow-nest')
+ shallow_list = loaded.get_sequence("shallow-nest")
first_dict = shallow_list.mapping_at(0)
- assert first_dict.get_str('animal') == expected
+ assert first_dict.get_str("animal") == expected
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("option,expected", [
- ('False', 'horsy'),
- ('True', 'pony'),
-])
+@pytest.mark.parametrize("option,expected", [("False", "horsy"), ("True", "pony"),])
def test_deep_nesting_level2(cli, datafiles, option, expected):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'deep-nesting')
- result = cli.run(project=project, silent=True, args=[
- '--option', 'pony', option,
- 'show',
- '--deps', 'none',
- '--format', '%{public}',
- 'element-deeper.bst'])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "deep-nesting")
+ result = cli.run(
+ project=project,
+ silent=True,
+ args=["--option", "pony", option, "show", "--deps", "none", "--format", "%{public}", "element-deeper.bst"],
+ )
result.assert_success()
loaded = _yaml.load_data(result.output)
- shallow_list = loaded.get_sequence('deep-nest')
+ shallow_list = loaded.get_sequence("deep-nest")
deeper_list = shallow_list.sequence_at(0)
first_dict = deeper_list.mapping_at(0)
- assert first_dict.get_str('animal') == expected
+ assert first_dict.get_str("animal") == expected
diff --git a/tests/format/project.py b/tests/format/project.py
index 2e0a729dc..e6bc6a5cd 100644
--- a/tests/format/project.py
+++ b/tests/format/project.py
@@ -11,30 +11,27 @@ from tests.testutils import filetypegenerator
# Project directory
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project"
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_missing_project_conf(cli, datafiles):
project = str(datafiles)
- result = cli.run(project=project, args=['workspace', 'list'])
+ result = cli.run(project=project, args=["workspace", "list"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_PROJECT_CONF)
@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_missing_project_name(cli, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename, "missingname")
- result = cli.run(project=project, args=['workspace', 'list'])
+ result = cli.run(project=project, args=["workspace", "list"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_missing_element(cli, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename, "missing-element")
- result = cli.run(project=project, args=['show', 'manual.bst'])
+ result = cli.run(project=project, args=["show", "manual.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE)
# Assert that we have the expected provenance encoded into the error
@@ -44,7 +41,7 @@ def test_missing_element(cli, datafiles):
@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_missing_junction(cli, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename, "missing-junction")
- result = cli.run(project=project, args=['show', 'manual.bst'])
+ result = cli.run(project=project, args=["show", "manual.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE)
# Assert that we have the expected provenance encoded into the error
@@ -54,176 +51,156 @@ def test_missing_junction(cli, datafiles):
@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_empty_project_name(cli, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename, "emptyname")
- result = cli.run(project=project, args=['workspace', 'list'])
+ result = cli.run(project=project, args=["workspace", "list"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_SYMBOL_NAME)
@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_invalid_project_name(cli, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename, "invalidname")
- result = cli.run(project=project, args=['workspace', 'list'])
+ result = cli.run(project=project, args=["workspace", "list"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_SYMBOL_NAME)
@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_invalid_yaml(cli, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename, "invalid-yaml")
- result = cli.run(project=project, args=['workspace', 'list'])
+ result = cli.run(project=project, args=["workspace", "list"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_YAML)
@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_load_default_project(cli, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename, "default")
- result = cli.run(project=project, args=[
- 'show', '--format', '%{env}', 'manual.bst'
- ])
+ result = cli.run(project=project, args=["show", "--format", "%{env}", "manual.bst"])
result.assert_success()
# Read back some of our project defaults from the env
env = _yaml.load_data(result.output)
- assert env.get_str('USER') == "tomjon"
- assert env.get_str('TERM') == "dumb"
+ assert env.get_str("USER") == "tomjon"
+ assert env.get_str("TERM") == "dumb"
@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_load_project_from_subdir(cli, datafiles):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'project-from-subdir')
+ project = os.path.join(datafiles.dirname, datafiles.basename, "project-from-subdir")
result = cli.run(
- project=project,
- cwd=os.path.join(project, 'subdirectory'),
- args=['show', '--format', '%{env}', 'manual.bst'])
+ project=project, cwd=os.path.join(project, "subdirectory"), args=["show", "--format", "%{env}", "manual.bst"]
+ )
result.assert_success()
# Read back some of our project defaults from the env
env = _yaml.load_data(result.output)
- assert env.get_str('USER') == "tomjon"
- assert env.get_str('TERM') == "dumb"
+ assert env.get_str("USER") == "tomjon"
+ assert env.get_str("TERM") == "dumb"
@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_override_project_path(cli, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename, "overridepath")
- result = cli.run(project=project, args=[
- 'show', '--format', '%{env}', 'manual.bst'
- ])
+ result = cli.run(project=project, args=["show", "--format", "%{env}", "manual.bst"])
result.assert_success()
# Read back the overridden path
env = _yaml.load_data(result.output)
- assert env.get_str('PATH') == "/bin:/sbin"
+ assert env.get_str("PATH") == "/bin:/sbin"
@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_project_unsupported(cli, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename, "unsupported")
- result = cli.run(project=project, args=['workspace', 'list'])
+ result = cli.run(project=project, args=["workspace", "list"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.UNSUPPORTED_PROJECT)
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'element-path'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "element-path"))
def test_missing_element_path_directory(cli, datafiles):
project = str(datafiles)
- result = cli.run(project=project, args=['workspace', 'list'])
- result.assert_main_error(ErrorDomain.LOAD,
- LoadErrorReason.MISSING_FILE)
+ result = cli.run(project=project, args=["workspace", "list"])
+ result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE)
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'element-path'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "element-path"))
def test_element_path_not_a_directory(cli, datafiles):
project = str(datafiles)
- path = os.path.join(project, 'elements')
+ path = os.path.join(project, "elements")
for _file_type in filetypegenerator.generate_file_types(path):
- result = cli.run(project=project, args=['workspace', 'list'])
+ result = cli.run(project=project, args=["workspace", "list"])
if not os.path.isdir(path):
- result.assert_main_error(ErrorDomain.LOAD,
- LoadErrorReason.PROJ_PATH_INVALID_KIND)
+ result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.PROJ_PATH_INVALID_KIND)
else:
result.assert_success()
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'local-plugin'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "local-plugin"))
def test_missing_local_plugin_directory(cli, datafiles):
project = str(datafiles)
- result = cli.run(project=project, args=['workspace', 'list'])
- result.assert_main_error(ErrorDomain.LOAD,
- LoadErrorReason.MISSING_FILE)
+ result = cli.run(project=project, args=["workspace", "list"])
+ result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE)
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'local-plugin'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "local-plugin"))
def test_local_plugin_not_directory(cli, datafiles):
project = str(datafiles)
- path = os.path.join(project, 'plugins')
+ path = os.path.join(project, "plugins")
for _file_type in filetypegenerator.generate_file_types(path):
- result = cli.run(project=project, args=['workspace', 'list'])
+ result = cli.run(project=project, args=["workspace", "list"])
if not os.path.isdir(path):
- result.assert_main_error(ErrorDomain.LOAD,
- LoadErrorReason.PROJ_PATH_INVALID_KIND)
+ result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.PROJ_PATH_INVALID_KIND)
else:
result.assert_success()
@pytest.mark.datafiles(DATA_DIR)
def test_plugin_load_allowed(cli, datafiles):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'plugin-allowed')
- result = cli.run(project=project, silent=True, args=[
- 'show', 'element.bst'])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "plugin-allowed")
+ result = cli.run(project=project, silent=True, args=["show", "element.bst"])
result.assert_success()
@pytest.mark.datafiles(DATA_DIR)
def test_plugin_load_forbidden(cli, datafiles):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'plugin-forbidden')
- result = cli.run(project=project, silent=True, args=[
- 'show', 'element.bst'])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "plugin-forbidden")
+ result = cli.run(project=project, silent=True, args=["show", "element.bst"])
result.assert_main_error(ErrorDomain.PLUGIN, None)
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
+@pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")])
def test_plugin_no_load_ref(cli, datafiles, ref_storage):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'plugin-no-load-ref')
+ project = os.path.join(datafiles.dirname, datafiles.basename, "plugin-no-load-ref")
# Generate project with access to the noloadref plugin and project.refs enabled
#
config = {
- 'name': 'test',
- 'ref-storage': ref_storage,
- 'plugins': [
- {
- 'origin': 'local',
- 'path': 'plugins',
- 'sources': {
- 'noloadref': 0
- }
- }
- ]
+ "name": "test",
+ "ref-storage": ref_storage,
+ "plugins": [{"origin": "local", "path": "plugins", "sources": {"noloadref": 0}}],
}
- _yaml.roundtrip_dump(config, os.path.join(project, 'project.conf'))
+ _yaml.roundtrip_dump(config, os.path.join(project, "project.conf"))
- result = cli.run(project=project, silent=True, args=['show', 'noloadref.bst'])
+ result = cli.run(project=project, silent=True, args=["show", "noloadref.bst"])
# There is no error if project.refs is not in use, otherwise we
# assert our graceful failure
- if ref_storage == 'inline':
+ if ref_storage == "inline":
result.assert_success()
else:
- result.assert_main_error(ErrorDomain.SOURCE, 'unsupported-load-ref')
+ result.assert_main_error(ErrorDomain.SOURCE, "unsupported-load-ref")
@pytest.mark.datafiles(DATA_DIR)
def test_plugin_preflight_error(cli, datafiles):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'plugin-preflight-error')
- result = cli.run(project=project, args=['source', 'fetch', 'error.bst'])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "plugin-preflight-error")
+ result = cli.run(project=project, args=["source", "fetch", "error.bst"])
result.assert_main_error(ErrorDomain.SOURCE, "the-preflight-error")
@pytest.mark.datafiles(DATA_DIR)
def test_duplicate_plugins(cli, datafiles):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'duplicate-plugins')
- result = cli.run(project=project, silent=True, args=[
- 'show', 'element.bst'])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "duplicate-plugins")
+ result = cli.run(project=project, silent=True, args=["show", "element.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_YAML)
@@ -232,42 +209,40 @@ def test_duplicate_plugins(cli, datafiles):
#
@pytest.mark.datafiles(DATA_DIR)
def test_project_refs_options(cli, datafiles):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'refs-options')
-
- result1 = cli.run(project=project, silent=True, args=[
- '--option', 'test', 'True',
- 'show',
- '--deps', 'none',
- '--format', '%{key}',
- 'target.bst'])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "refs-options")
+
+ result1 = cli.run(
+ project=project,
+ silent=True,
+ args=["--option", "test", "True", "show", "--deps", "none", "--format", "%{key}", "target.bst"],
+ )
result1.assert_success()
- result2 = cli.run(project=project, silent=True, args=[
- '--option', 'test', 'False',
- 'show',
- '--deps', 'none',
- '--format', '%{key}',
- 'target.bst'])
+ result2 = cli.run(
+ project=project,
+ silent=True,
+ args=["--option", "test", "False", "show", "--deps", "none", "--format", "%{key}", "target.bst"],
+ )
result2.assert_success()
# Assert that the cache keys are different
assert result1.output != result2.output
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'element-path'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "element-path"))
def test_element_path_project_path_contains_symlinks(cli, datafiles, tmpdir):
real_project = str(datafiles)
- linked_project = os.path.join(str(tmpdir), 'linked')
+ linked_project = os.path.join(str(tmpdir), "linked")
os.symlink(real_project, linked_project)
- os.makedirs(os.path.join(real_project, 'elements'), exist_ok=True)
- with open(os.path.join(real_project, 'elements', 'element.bst'), 'w') as f:
+ os.makedirs(os.path.join(real_project, "elements"), exist_ok=True)
+ with open(os.path.join(real_project, "elements", "element.bst"), "w") as f:
f.write("kind: manual\n")
- result = cli.run(project=linked_project, args=['show', 'element.bst'])
+ result = cli.run(project=linked_project, args=["show", "element.bst"])
result.assert_success()
@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_empty_depends(cli, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename, "empty-depends")
- result = cli.run(project=project, args=['show', 'manual.bst'])
+ result = cli.run(project=project, args=["show", "manual.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
diff --git a/tests/format/project/plugin-no-load-ref/plugins/noloadref.py b/tests/format/project/plugin-no-load-ref/plugins/noloadref.py
index 0cc457f07..5d44cd5c7 100644
--- a/tests/format/project/plugin-no-load-ref/plugins/noloadref.py
+++ b/tests/format/project/plugin-no-load-ref/plugins/noloadref.py
@@ -6,7 +6,6 @@ from buildstream import Source, Consistency
# Use this to test that the core behaves as expected with such plugins.
#
class NoLoadRefSource(Source):
-
def configure(self, node):
pass
diff --git a/tests/format/project/plugin-preflight-error/errorplugin/preflighterror.py b/tests/format/project/plugin-preflight-error/errorplugin/preflighterror.py
index 0eee463ad..3fa8afb93 100644
--- a/tests/format/project/plugin-preflight-error/errorplugin/preflighterror.py
+++ b/tests/format/project/plugin-preflight-error/errorplugin/preflighterror.py
@@ -2,15 +2,13 @@ from buildstream import Source, SourceError, Consistency
class PreflightErrorSource(Source):
-
def configure(self, node):
pass
def preflight(self):
# Raise a preflight error unconditionally
- raise SourceError("Unsatisfied requirements in preflight, raising this error",
- reason="the-preflight-error")
+ raise SourceError("Unsatisfied requirements in preflight, raising this error", reason="the-preflight-error")
def get_unique_key(self):
return {}
diff --git a/tests/format/projectoverrides.py b/tests/format/projectoverrides.py
index 7932ffb4a..16aad2cf8 100644
--- a/tests/format/projectoverrides.py
+++ b/tests/format/projectoverrides.py
@@ -7,23 +7,18 @@ from buildstream import _yaml
from buildstream.testing.runcli import cli # pylint: disable=unused-import
# Project directory
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project-overrides"
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project-overrides")
@pytest.mark.datafiles(DATA_DIR)
def test_prepend_configure_commands(cli, datafiles):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'prepend-configure-commands')
- result = cli.run(project=project, silent=True, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{config}',
- 'element.bst'])
+ project = os.path.join(datafiles.dirname, datafiles.basename, "prepend-configure-commands")
+ result = cli.run(
+ project=project, silent=True, args=["show", "--deps", "none", "--format", "%{config}", "element.bst"]
+ )
result.assert_success()
loaded = _yaml.load_data(result.output)
- config_commands = loaded.get_str_list('configure-commands')
+ config_commands = loaded.get_str_list("configure-commands")
assert len(config_commands) == 3
assert config_commands[0] == 'echo "Hello World!"'
diff --git a/tests/format/userconfig.py b/tests/format/userconfig.py
index 72172a24d..9b514cc3d 100644
--- a/tests/format/userconfig.py
+++ b/tests/format/userconfig.py
@@ -14,11 +14,7 @@ DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
@pytest.mark.datafiles(DATA_DIR)
def test_ensure_misformed_project_overrides_give_sensible_errors(cli, datafiles):
- userconfig = {
- "projects": {
- "test": []
- }
- }
+ userconfig = {"projects": {"test": []}}
cli.configure(userconfig)
result = cli.run(project=datafiles, args=["show"])
diff --git a/tests/format/variables.py b/tests/format/variables.py
index 4610e039f..35b105f66 100644
--- a/tests/format/variables.py
+++ b/tests/format/variables.py
@@ -12,13 +12,10 @@ from buildstream.testing.runcli import cli # pylint: disable=unused-import
# Project directory
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "variables"
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "variables")
# List of BuildStream protected variables
-PROTECTED_VARIABLES = [('project-name'), ('element-name'), ('max-jobs')]
+PROTECTED_VARIABLES = [("project-name"), ("element-name"), ("max-jobs")]
def print_warning(msg):
@@ -29,24 +26,31 @@ def print_warning(msg):
###############################################################
# Test proper loading of some default commands from plugins #
###############################################################
-@pytest.mark.parametrize("target,varname,expected", [
- ('autotools.bst', 'make-install', "make -j1 DESTDIR=\"/buildstream-install\" install"),
- ('cmake.bst', 'cmake',
- "cmake -B_builddir -H\".\" -G\"Unix Makefiles\" " + "-DCMAKE_INSTALL_PREFIX:PATH=\"/usr\" \\\n" +
- "-DCMAKE_INSTALL_LIBDIR:PATH=\"lib\""),
- ('distutils.bst', 'python-install',
- "python3 ./setup.py install --prefix \"/usr\" \\\n" +
- "--root \"/buildstream-install\""),
- ('makemaker.bst', 'configure', "perl Makefile.PL PREFIX=/buildstream-install/usr"),
- ('modulebuild.bst', 'configure', "perl Build.PL --prefix \"/buildstream-install/usr\""),
- ('qmake.bst', 'make-install', "make -j1 INSTALL_ROOT=\"/buildstream-install\" install"),
-])
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'defaults'))
+@pytest.mark.parametrize(
+ "target,varname,expected",
+ [
+ ("autotools.bst", "make-install", 'make -j1 DESTDIR="/buildstream-install" install'),
+ (
+ "cmake.bst",
+ "cmake",
+ 'cmake -B_builddir -H"." -G"Unix Makefiles" '
+ + '-DCMAKE_INSTALL_PREFIX:PATH="/usr" \\\n'
+ + '-DCMAKE_INSTALL_LIBDIR:PATH="lib"',
+ ),
+ (
+ "distutils.bst",
+ "python-install",
+ 'python3 ./setup.py install --prefix "/usr" \\\n' + '--root "/buildstream-install"',
+ ),
+ ("makemaker.bst", "configure", "perl Makefile.PL PREFIX=/buildstream-install/usr"),
+ ("modulebuild.bst", "configure", 'perl Build.PL --prefix "/buildstream-install/usr"'),
+ ("qmake.bst", "make-install", 'make -j1 INSTALL_ROOT="/buildstream-install" install'),
+ ],
+)
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "defaults"))
def test_defaults(cli, datafiles, target, varname, expected):
project = str(datafiles)
- result = cli.run(project=project, silent=True, args=[
- 'show', '--deps', 'none', '--format', '%{vars}', target
- ])
+ result = cli.run(project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", target])
result.assert_success()
result_vars = _yaml.load_data(result.output)
assert result_vars.get_str(varname) == expected
@@ -55,130 +59,97 @@ def test_defaults(cli, datafiles, target, varname, expected):
################################################################
# Test overriding of variables to produce different commands #
################################################################
-@pytest.mark.parametrize("target,varname,expected", [
- ('autotools.bst', 'make-install', "make -j1 DESTDIR=\"/custom/install/root\" install"),
- ('cmake.bst', 'cmake',
- "cmake -B_builddir -H\".\" -G\"Ninja\" " + "-DCMAKE_INSTALL_PREFIX:PATH=\"/opt\" \\\n" +
- "-DCMAKE_INSTALL_LIBDIR:PATH=\"lib\""),
- ('distutils.bst', 'python-install',
- "python3 ./setup.py install --prefix \"/opt\" \\\n" +
- "--root \"/custom/install/root\""),
- ('makemaker.bst', 'configure', "perl Makefile.PL PREFIX=/custom/install/root/opt"),
- ('modulebuild.bst', 'configure', "perl Build.PL --prefix \"/custom/install/root/opt\""),
- ('qmake.bst', 'make-install', "make -j1 INSTALL_ROOT=\"/custom/install/root\" install"),
-])
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'overrides'))
+@pytest.mark.parametrize(
+ "target,varname,expected",
+ [
+ ("autotools.bst", "make-install", 'make -j1 DESTDIR="/custom/install/root" install'),
+ (
+ "cmake.bst",
+ "cmake",
+ 'cmake -B_builddir -H"." -G"Ninja" '
+ + '-DCMAKE_INSTALL_PREFIX:PATH="/opt" \\\n'
+ + '-DCMAKE_INSTALL_LIBDIR:PATH="lib"',
+ ),
+ (
+ "distutils.bst",
+ "python-install",
+ 'python3 ./setup.py install --prefix "/opt" \\\n' + '--root "/custom/install/root"',
+ ),
+ ("makemaker.bst", "configure", "perl Makefile.PL PREFIX=/custom/install/root/opt"),
+ ("modulebuild.bst", "configure", 'perl Build.PL --prefix "/custom/install/root/opt"'),
+ ("qmake.bst", "make-install", 'make -j1 INSTALL_ROOT="/custom/install/root" install'),
+ ],
+)
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "overrides"))
def test_overrides(cli, datafiles, target, varname, expected):
project = str(datafiles)
- result = cli.run(project=project, silent=True, args=[
- 'show', '--deps', 'none', '--format', '%{vars}', target
- ])
+ result = cli.run(project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", target])
result.assert_success()
result_vars = _yaml.load_data(result.output)
assert result_vars.get_str(varname) == expected
@pytest.mark.parametrize("element", ["manual.bst", "manual2.bst"])
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'missing_variables'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "missing_variables"))
def test_missing_variable(cli, datafiles, element):
project = str(datafiles)
- result = cli.run(project=project, silent=True, args=[
- 'show', '--deps', 'none', '--format', '%{config}', element
- ])
- result.assert_main_error(ErrorDomain.LOAD,
- LoadErrorReason.UNRESOLVED_VARIABLE)
+ result = cli.run(project=project, silent=True, args=["show", "--deps", "none", "--format", "%{config}", element])
+ result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.UNRESOLVED_VARIABLE)
@pytest.mark.timeout(3, method="signal")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'cyclic_variables'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "cyclic_variables"))
def test_cyclic_variables(cli, datafiles):
- print_warning("Performing cyclic test, if this test times out it will " +
- "exit the test sequence")
+ print_warning("Performing cyclic test, if this test times out it will " + "exit the test sequence")
project = str(datafiles)
- result = cli.run(project=project, silent=True, args=[
- "build", "cyclic.bst"
- ])
+ result = cli.run(project=project, silent=True, args=["build", "cyclic.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.RECURSIVE_VARIABLE)
-@pytest.mark.parametrize('protected_var', PROTECTED_VARIABLES)
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'protected-vars'))
+@pytest.mark.parametrize("protected_var", PROTECTED_VARIABLES)
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "protected-vars"))
def test_use_of_protected_var_project_conf(cli, datafiles, protected_var):
project = str(datafiles)
- conf = {
- 'name': 'test',
- 'variables': {
- protected_var: 'some-value'
- }
- }
- _yaml.roundtrip_dump(conf, os.path.join(project, 'project.conf'))
+ conf = {"name": "test", "variables": {protected_var: "some-value"}}
+ _yaml.roundtrip_dump(conf, os.path.join(project, "project.conf"))
element = {
- 'kind': 'import',
- 'sources': [
- {
- 'kind': 'local',
- 'path': 'foo.txt'
- }
- ],
+ "kind": "import",
+ "sources": [{"kind": "local", "path": "foo.txt"}],
}
- _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst'))
+ _yaml.roundtrip_dump(element, os.path.join(project, "target.bst"))
- result = cli.run(project=project, args=['build', 'target.bst'])
- result.assert_main_error(ErrorDomain.LOAD,
- LoadErrorReason.PROTECTED_VARIABLE_REDEFINED)
+ result = cli.run(project=project, args=["build", "target.bst"])
+ result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.PROTECTED_VARIABLE_REDEFINED)
-@pytest.mark.parametrize('protected_var', PROTECTED_VARIABLES)
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'protected-vars'))
+@pytest.mark.parametrize("protected_var", PROTECTED_VARIABLES)
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "protected-vars"))
def test_use_of_protected_var_element_overrides(cli, datafiles, protected_var):
project = str(datafiles)
- conf = {
- 'name': 'test',
- 'elements': {
- 'manual': {
- 'variables': {
- protected_var: 'some-value'
- }
- }
- }
- }
- _yaml.roundtrip_dump(conf, os.path.join(project, 'project.conf'))
+ conf = {"name": "test", "elements": {"manual": {"variables": {protected_var: "some-value"}}}}
+ _yaml.roundtrip_dump(conf, os.path.join(project, "project.conf"))
element = {
- 'kind': 'manual',
- 'sources': [
- {
- 'kind': 'local',
- 'path': 'foo.txt'
- }
- ],
+ "kind": "manual",
+ "sources": [{"kind": "local", "path": "foo.txt"}],
}
- _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst'))
+ _yaml.roundtrip_dump(element, os.path.join(project, "target.bst"))
- result = cli.run(project=project, args=['build', 'target.bst'])
- result.assert_main_error(ErrorDomain.LOAD,
- LoadErrorReason.PROTECTED_VARIABLE_REDEFINED)
+ result = cli.run(project=project, args=["build", "target.bst"])
+ result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.PROTECTED_VARIABLE_REDEFINED)
-@pytest.mark.parametrize('protected_var', PROTECTED_VARIABLES)
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'protected-vars'))
+@pytest.mark.parametrize("protected_var", PROTECTED_VARIABLES)
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "protected-vars"))
def test_use_of_protected_var_in_element(cli, datafiles, protected_var):
project = str(datafiles)
element = {
- 'kind': 'import',
- 'sources': [
- {
- 'kind': 'local',
- 'path': 'foo.txt'
- }
- ],
- 'variables': {
- protected_var: 'some-value'
- }
+ "kind": "import",
+ "sources": [{"kind": "local", "path": "foo.txt"}],
+ "variables": {protected_var: "some-value"},
}
- _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst'))
+ _yaml.roundtrip_dump(element, os.path.join(project, "target.bst"))
- result = cli.run(project=project, args=['build', 'target.bst'])
- result.assert_main_error(ErrorDomain.LOAD,
- LoadErrorReason.PROTECTED_VARIABLE_REDEFINED)
+ result = cli.run(project=project, args=["build", "target.bst"])
+ result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.PROTECTED_VARIABLE_REDEFINED)
diff --git a/tests/frontend/__init__.py b/tests/frontend/__init__.py
index f1c8c41b8..ad1f9ea92 100644
--- a/tests/frontend/__init__.py
+++ b/tests/frontend/__init__.py
@@ -5,6 +5,6 @@ from buildstream import _yaml
# Shared function to configure the project.conf inline
#
def configure_project(path, config):
- config['name'] = 'test'
- config['element-path'] = 'elements'
- _yaml.roundtrip_dump(config, os.path.join(path, 'project.conf'))
+ config["name"] = "test"
+ config["element-path"] = "elements"
+ _yaml.roundtrip_dump(config, os.path.join(path, "project.conf"))
diff --git a/tests/frontend/artifact_delete.py b/tests/frontend/artifact_delete.py
index a9f5ec6da..a93d99ef6 100644
--- a/tests/frontend/artifact_delete.py
+++ b/tests/frontend/artifact_delete.py
@@ -28,10 +28,7 @@ from tests.testutils import create_artifact_share
# Project directory
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project",
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",)
# Test that we can delete the artifact of the element which corresponds
@@ -39,60 +36,60 @@ DATA_DIR = os.path.join(
@pytest.mark.datafiles(DATA_DIR)
def test_artifact_delete_element(cli, tmpdir, datafiles):
project = str(datafiles)
- element = 'target.bst'
+ element = "target.bst"
# Build the element and ensure it's cached
- result = cli.run(project=project, args=['build', element])
+ result = cli.run(project=project, args=["build", element])
result.assert_success()
- assert cli.get_element_state(project, element) == 'cached'
+ assert cli.get_element_state(project, element) == "cached"
- result = cli.run(project=project, args=['artifact', 'delete', element])
+ result = cli.run(project=project, args=["artifact", "delete", element])
result.assert_success()
- assert cli.get_element_state(project, element) != 'cached'
+ assert cli.get_element_state(project, element) != "cached"
# Test that we can delete an artifact by specifying its ref.
@pytest.mark.datafiles(DATA_DIR)
def test_artifact_delete_artifact(cli, tmpdir, datafiles):
project = str(datafiles)
- element = 'target.bst'
+ element = "target.bst"
# Configure a local cache
- local_cache = os.path.join(str(tmpdir), 'cache')
- cli.configure({'cachedir': local_cache})
+ local_cache = os.path.join(str(tmpdir), "cache")
+ cli.configure({"cachedir": local_cache})
# First build an element so that we can find its artifact
- result = cli.run(project=project, args=['build', element])
+ result = cli.run(project=project, args=["build", element])
result.assert_success()
# Obtain the artifact ref
cache_key = cli.get_element_key(project, element)
- artifact = os.path.join('test', os.path.splitext(element)[0], cache_key)
+ artifact = os.path.join("test", os.path.splitext(element)[0], cache_key)
# Explicitly check that the ARTIFACT exists in the cache
- assert os.path.exists(os.path.join(local_cache, 'artifacts', 'refs', artifact))
+ assert os.path.exists(os.path.join(local_cache, "artifacts", "refs", artifact))
# Delete the artifact
- result = cli.run(project=project, args=['artifact', 'delete', artifact])
+ result = cli.run(project=project, args=["artifact", "delete", artifact])
result.assert_success()
# Check that the ARTIFACT is no longer in the cache
- assert not os.path.exists(os.path.join(local_cache, 'cas', 'refs', 'heads', artifact))
+ assert not os.path.exists(os.path.join(local_cache, "cas", "refs", "heads", artifact))
# Test the `bst artifact delete` command with multiple, different arguments.
@pytest.mark.datafiles(DATA_DIR)
def test_artifact_delete_element_and_artifact(cli, tmpdir, datafiles):
project = str(datafiles)
- element = 'target.bst'
- dep = 'compose-all.bst'
+ element = "target.bst"
+ dep = "compose-all.bst"
# Configure a local cache
- local_cache = os.path.join(str(tmpdir), 'cache')
- cli.configure({'cachedir': local_cache})
+ local_cache = os.path.join(str(tmpdir), "cache")
+ cli.configure({"cachedir": local_cache})
# First build an element so that we can find its artifact
- result = cli.run(project=project, args=['build', element])
+ result = cli.run(project=project, args=["build", element])
result.assert_success()
assert cli.get_element_states(project, [element, dep], deps="none") == {
element: "cached",
@@ -101,20 +98,20 @@ def test_artifact_delete_element_and_artifact(cli, tmpdir, datafiles):
# Obtain the artifact ref
cache_key = cli.get_element_key(project, element)
- artifact = os.path.join('test', os.path.splitext(element)[0], cache_key)
+ artifact = os.path.join("test", os.path.splitext(element)[0], cache_key)
# Explicitly check that the ARTIFACT exists in the cache
- assert os.path.exists(os.path.join(local_cache, 'artifacts', 'refs', artifact))
+ assert os.path.exists(os.path.join(local_cache, "artifacts", "refs", artifact))
# Delete the artifact
- result = cli.run(project=project, args=['artifact', 'delete', artifact, dep])
+ result = cli.run(project=project, args=["artifact", "delete", artifact, dep])
result.assert_success()
# Check that the ARTIFACT is no longer in the cache
- assert not os.path.exists(os.path.join(local_cache, 'artifacts', artifact))
+ assert not os.path.exists(os.path.join(local_cache, "artifacts", artifact))
# Check that the dependency ELEMENT is no longer cached
- assert cli.get_element_state(project, dep) != 'cached'
+ assert cli.get_element_state(project, dep) != "cached"
# Test that we receive the appropriate stderr when we try to delete an artifact
@@ -122,19 +119,19 @@ def test_artifact_delete_element_and_artifact(cli, tmpdir, datafiles):
@pytest.mark.datafiles(DATA_DIR)
def test_artifact_delete_unbuilt_artifact(cli, tmpdir, datafiles):
project = str(datafiles)
- element = 'target.bst'
+ element = "target.bst"
# delete it, just in case it's there
- _ = cli.run(project=project, args=['artifact', 'delete', element])
+ _ = cli.run(project=project, args=["artifact", "delete", element])
# Ensure the element is not cached
- assert cli.get_element_state(project, element) != 'cached'
+ assert cli.get_element_state(project, element) != "cached"
# Now try and remove it again (now we know its not there)
- result = cli.run(project=project, args=['artifact', 'delete', element])
+ result = cli.run(project=project, args=["artifact", "delete", element])
cache_key = cli.get_element_key(project, element)
- artifact = os.path.join('test', os.path.splitext(element)[0], cache_key)
+ artifact = os.path.join("test", os.path.splitext(element)[0], cache_key)
expected_err = "WARNING Could not find ref '{}'".format(artifact)
assert expected_err in result.stderr
@@ -144,122 +141,121 @@ def test_artifact_delete_unbuilt_artifact(cli, tmpdir, datafiles):
@pytest.mark.datafiles(DATA_DIR)
def test_artifact_delete_pulled_artifact_without_buildtree(cli, tmpdir, datafiles):
project = str(datafiles)
- element = 'target.bst'
+ element = "target.bst"
# Set up remote and local shares
- local_cache = os.path.join(str(tmpdir), 'artifacts')
- with create_artifact_share(os.path.join(str(tmpdir), 'remote')) as remote:
- cli.configure({
- 'artifacts': {'url': remote.repo, 'push': True},
- 'cachedir': local_cache,
- })
+ local_cache = os.path.join(str(tmpdir), "artifacts")
+ with create_artifact_share(os.path.join(str(tmpdir), "remote")) as remote:
+ cli.configure(
+ {"artifacts": {"url": remote.repo, "push": True}, "cachedir": local_cache,}
+ )
# Build the element
- result = cli.run(project=project, args=['build', element])
+ result = cli.run(project=project, args=["build", element])
result.assert_success()
# Make sure it's in the share
- assert remote.get_artifact(cli.get_artifact_name(project, 'test', element))
+ assert remote.get_artifact(cli.get_artifact_name(project, "test", element))
# Delete and then pull the artifact (without its buildtree)
- result = cli.run(project=project, args=['artifact', 'delete', element])
+ result = cli.run(project=project, args=["artifact", "delete", element])
result.assert_success()
- assert cli.get_element_state(project, element) != 'cached'
- result = cli.run(project=project, args=['artifact', 'pull', element])
+ assert cli.get_element_state(project, element) != "cached"
+ result = cli.run(project=project, args=["artifact", "pull", element])
result.assert_success()
- assert cli.get_element_state(project, element) == 'cached'
+ assert cli.get_element_state(project, element) == "cached"
# Now delete it again (it should have been pulled without the buildtree, but
# a digest of the buildtree is pointed to in the artifact's metadata
- result = cli.run(project=project, args=['artifact', 'delete', element])
+ result = cli.run(project=project, args=["artifact", "delete", element])
result.assert_success()
- assert cli.get_element_state(project, element) != 'cached'
+ assert cli.get_element_state(project, element) != "cached"
# Test that we can delete the build deps of an element
@pytest.mark.datafiles(DATA_DIR)
def test_artifact_delete_elements_build_deps(cli, tmpdir, datafiles):
project = str(datafiles)
- element = 'target.bst'
+ element = "target.bst"
# Build the element and ensure it's cached
- result = cli.run(project=project, args=['build', element])
+ result = cli.run(project=project, args=["build", element])
result.assert_success()
# Assert element and build deps are cached
- assert cli.get_element_state(project, element) == 'cached'
- bdep_states = cli.get_element_states(project, [element], deps='build')
+ assert cli.get_element_state(project, element) == "cached"
+ bdep_states = cli.get_element_states(project, [element], deps="build")
for state in bdep_states.values():
- assert state == 'cached'
+ assert state == "cached"
- result = cli.run(project=project, args=['artifact', 'delete', '--deps', 'build', element])
+ result = cli.run(project=project, args=["artifact", "delete", "--deps", "build", element])
result.assert_success()
# Assert that the build deps have been deleted and that the artifact remains cached
- assert cli.get_element_state(project, element) == 'cached'
- bdep_states = cli.get_element_states(project, [element], deps='build')
+ assert cli.get_element_state(project, element) == "cached"
+ bdep_states = cli.get_element_states(project, [element], deps="build")
for state in bdep_states.values():
- assert state != 'cached'
+ assert state != "cached"
# Test that we can delete the build deps of an artifact by providing an artifact ref
@pytest.mark.datafiles(DATA_DIR)
def test_artifact_delete_artifacts_build_deps(cli, tmpdir, datafiles):
project = str(datafiles)
- element = 'target.bst'
+ element = "target.bst"
# Configure a local cache
- local_cache = os.path.join(str(tmpdir), 'cache')
- cli.configure({'cachedir': local_cache})
+ local_cache = os.path.join(str(tmpdir), "cache")
+ cli.configure({"cachedir": local_cache})
# First build an element so that we can find its artifact
- result = cli.run(project=project, args=['build', element])
+ result = cli.run(project=project, args=["build", element])
result.assert_success()
# Obtain the artifact ref
cache_key = cli.get_element_key(project, element)
- artifact = os.path.join('test', os.path.splitext(element)[0], cache_key)
+ artifact = os.path.join("test", os.path.splitext(element)[0], cache_key)
# Explicitly check that the ARTIFACT exists in the cache
- assert os.path.exists(os.path.join(local_cache, 'artifacts', 'refs', artifact))
+ assert os.path.exists(os.path.join(local_cache, "artifacts", "refs", artifact))
# get the artifact refs of the build dependencies
bdep_refs = []
- bdep_states = cli.get_element_states(project, [element], deps='build')
+ bdep_states = cli.get_element_states(project, [element], deps="build")
for bdep in bdep_states.keys():
- bdep_refs.append(os.path.join('test', _get_normal_name(bdep), cli.get_element_key(project, bdep)))
+ bdep_refs.append(os.path.join("test", _get_normal_name(bdep), cli.get_element_key(project, bdep)))
# Assert build dependencies are cached
for ref in bdep_refs:
- assert os.path.exists(os.path.join(local_cache, 'artifacts', 'refs', ref))
+ assert os.path.exists(os.path.join(local_cache, "artifacts", "refs", ref))
# Delete the artifact
- result = cli.run(project=project, args=['artifact', 'delete', '--deps', 'build', artifact])
+ result = cli.run(project=project, args=["artifact", "delete", "--deps", "build", artifact])
result.assert_success()
# Check that the artifact's build deps are no longer in the cache
# Assert build dependencies have been deleted and that the artifact remains
for ref in bdep_refs:
- assert not os.path.exists(os.path.join(local_cache, 'artifacts', 'refs', ref))
- assert os.path.exists(os.path.join(local_cache, 'artifacts', 'refs', artifact))
+ assert not os.path.exists(os.path.join(local_cache, "artifacts", "refs", ref))
+ assert os.path.exists(os.path.join(local_cache, "artifacts", "refs", artifact))
# Test that `--deps all` option fails if an artifact ref is specified
@pytest.mark.datafiles(DATA_DIR)
def test_artifact_delete_artifact_with_deps_all_fails(cli, tmpdir, datafiles):
project = str(datafiles)
- element = 'target.bst'
+ element = "target.bst"
# First build an element so that we can find its artifact
- result = cli.run(project=project, args=['build', element])
+ result = cli.run(project=project, args=["build", element])
result.assert_success()
# Obtain the artifact ref
cache_key = cli.get_element_key(project, element)
- artifact = os.path.join('test', os.path.splitext(element)[0], cache_key)
+ artifact = os.path.join("test", os.path.splitext(element)[0], cache_key)
# Try to delete the artifact with all of its dependencies
- result = cli.run(project=project, args=['artifact', 'delete', '--deps', 'all', artifact])
+ result = cli.run(project=project, args=["artifact", "delete", "--deps", "all", artifact])
result.assert_main_error(ErrorDomain.STREAM, None)
assert "Error: '--deps all' is not supported for artifact refs" in result.stderr
diff --git a/tests/frontend/artifact_list_contents.py b/tests/frontend/artifact_list_contents.py
index 626eb3fa7..a1473c806 100644
--- a/tests/frontend/artifact_list_contents.py
+++ b/tests/frontend/artifact_list_contents.py
@@ -25,10 +25,7 @@ from buildstream.testing import cli # pylint: disable=unused-import
# Project directory
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project",
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",)
@pytest.mark.datafiles(DATA_DIR)
@@ -36,16 +33,13 @@ def test_artifact_list_exact_contents_element(cli, datafiles):
project = str(datafiles)
# Ensure we have an artifact to read
- result = cli.run(project=project, args=['build', 'import-bin.bst'])
+ result = cli.run(project=project, args=["build", "import-bin.bst"])
assert result.exit_code == 0
# List the contents via the element name
- result = cli.run(project=project, args=['artifact', 'list-contents', 'import-bin.bst'])
+ result = cli.run(project=project, args=["artifact", "list-contents", "import-bin.bst"])
assert result.exit_code == 0
- expected_output = ("import-bin.bst:\n"
- "\tusr\n"
- "\tusr/bin\n"
- "\tusr/bin/hello\n\n")
+ expected_output = "import-bin.bst:\n" "\tusr\n" "\tusr/bin\n" "\tusr/bin/hello\n\n"
assert expected_output in result.output
@@ -54,20 +48,17 @@ def test_artifact_list_exact_contents_ref(cli, datafiles):
project = str(datafiles)
# Get the cache key of our test element
- key = cli.get_element_key(project, 'import-bin.bst')
+ key = cli.get_element_key(project, "import-bin.bst")
# Ensure we have an artifact to read
- result = cli.run(project=project, args=['build', 'import-bin.bst'])
+ result = cli.run(project=project, args=["build", "import-bin.bst"])
assert result.exit_code == 0
# List the contents via the key
- result = cli.run(project=project, args=['artifact', 'list-contents', 'test/import-bin/' + key])
+ result = cli.run(project=project, args=["artifact", "list-contents", "test/import-bin/" + key])
assert result.exit_code == 0
- expected_output = ("test/import-bin/" + key + ":\n"
- "\tusr\n"
- "\tusr/bin\n"
- "\tusr/bin/hello\n\n")
+ expected_output = "test/import-bin/" + key + ":\n" "\tusr\n" "\tusr/bin\n" "\tusr/bin/hello\n\n"
assert expected_output in result.output
@@ -76,23 +67,25 @@ def test_artifact_list_exact_contents_glob(cli, datafiles):
project = str(datafiles)
# Ensure we have an artifact to read
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
assert result.exit_code == 0
# List the contents via glob
- result = cli.run(project=project, args=['artifact', 'list-contents', 'test/*'])
+ result = cli.run(project=project, args=["artifact", "list-contents", "test/*"])
assert result.exit_code == 0
# get the cahe keys for each element in the glob
- import_bin_key = cli.get_element_key(project, 'import-bin.bst')
- import_dev_key = cli.get_element_key(project, 'import-dev.bst')
- compose_all_key = cli.get_element_key(project, 'compose-all.bst')
- target_key = cli.get_element_key(project, 'target.bst')
-
- expected_artifacts = ["test/import-bin/" + import_bin_key,
- "test/import-dev/" + import_dev_key,
- "test/compose-all/" + compose_all_key,
- "test/target/" + target_key]
+ import_bin_key = cli.get_element_key(project, "import-bin.bst")
+ import_dev_key = cli.get_element_key(project, "import-dev.bst")
+ compose_all_key = cli.get_element_key(project, "compose-all.bst")
+ target_key = cli.get_element_key(project, "target.bst")
+
+ expected_artifacts = [
+ "test/import-bin/" + import_bin_key,
+ "test/import-dev/" + import_dev_key,
+ "test/compose-all/" + compose_all_key,
+ "test/target/" + target_key,
+ ]
for artifact in expected_artifacts:
assert artifact in result.output
@@ -103,16 +96,18 @@ def test_artifact_list_exact_contents_element_long(cli, datafiles):
project = str(datafiles)
# Ensure we have an artifact to read
- result = cli.run(project=project, args=['build', 'import-bin.bst'])
+ result = cli.run(project=project, args=["build", "import-bin.bst"])
assert result.exit_code == 0
# List the contents via the element name
- result = cli.run(project=project, args=['artifact', 'list-contents', '--long', 'import-bin.bst'])
+ result = cli.run(project=project, args=["artifact", "list-contents", "--long", "import-bin.bst"])
assert result.exit_code == 0
- expected_output = ("import-bin.bst:\n"
- "\tdrwxr-xr-x dir 1 usr\n"
- "\tdrwxr-xr-x dir 1 usr/bin\n"
- "\t-rw-r--r-- reg 107 usr/bin/hello\n\n")
+ expected_output = (
+ "import-bin.bst:\n"
+ "\tdrwxr-xr-x dir 1 usr\n"
+ "\tdrwxr-xr-x dir 1 usr/bin\n"
+ "\t-rw-r--r-- reg 107 usr/bin/hello\n\n"
+ )
assert expected_output in result.output
@@ -122,19 +117,21 @@ def test_artifact_list_exact_contents_ref_long(cli, datafiles):
project = str(datafiles)
# Get the cache key of our test element
- key = cli.get_element_key(project, 'import-bin.bst')
+ key = cli.get_element_key(project, "import-bin.bst")
# Ensure we have an artifact to read
- result = cli.run(project=project, args=['build', 'import-bin.bst'])
+ result = cli.run(project=project, args=["build", "import-bin.bst"])
assert result.exit_code == 0
# List the contents via the key
- result = cli.run(project=project, args=['artifact', 'list-contents', '-l', 'test/import-bin/' + key])
+ result = cli.run(project=project, args=["artifact", "list-contents", "-l", "test/import-bin/" + key])
assert result.exit_code == 0
- expected_output = (" test/import-bin/" + key + ":\n"
- "\tdrwxr-xr-x dir 1 usr\n"
- "\tdrwxr-xr-x dir 1 usr/bin\n"
- "\t-rw-r--r-- reg 107 usr/bin/hello\n\n")
+ expected_output = (
+ " test/import-bin/" + key + ":\n"
+ "\tdrwxr-xr-x dir 1 usr\n"
+ "\tdrwxr-xr-x dir 1 usr/bin\n"
+ "\t-rw-r--r-- reg 107 usr/bin/hello\n\n"
+ )
assert expected_output in result.output
diff --git a/tests/frontend/artifact_log.py b/tests/frontend/artifact_log.py
index 39c9458ad..6ea610a25 100644
--- a/tests/frontend/artifact_log.py
+++ b/tests/frontend/artifact_log.py
@@ -25,10 +25,7 @@ from buildstream.testing import cli # pylint: disable=unused-import
# Project directory
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project",
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",)
@pytest.mark.datafiles(DATA_DIR)
@@ -36,32 +33,32 @@ def test_artifact_log(cli, datafiles):
project = str(datafiles)
# Get the cache key of our test element
- result = cli.run(project=project, silent=True, args=[
- '--no-colors',
- 'show', '--deps', 'none', '--format', '%{full-key}',
- 'target.bst'
- ])
+ result = cli.run(
+ project=project,
+ silent=True,
+ args=["--no-colors", "show", "--deps", "none", "--format", "%{full-key}", "target.bst"],
+ )
key = result.output.strip()
# Ensure we have an artifact to read
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
assert result.exit_code == 0
# Read the log via the element name
- result = cli.run(project=project, args=['artifact', 'log', 'target.bst'])
+ result = cli.run(project=project, args=["artifact", "log", "target.bst"])
assert result.exit_code == 0
log = result.output
# Assert that there actually was a log file
- assert log != ''
+ assert log != ""
# Read the log via the key
- result = cli.run(project=project, args=['artifact', 'log', 'test/target/' + key])
+ result = cli.run(project=project, args=["artifact", "log", "test/target/" + key])
assert result.exit_code == 0
assert log == result.output
# Read the log via glob
- result = cli.run(project=project, args=['artifact', 'log', 'test/target/*'])
+ result = cli.run(project=project, args=["artifact", "log", "test/target/*"])
assert result.exit_code == 0
# The artifact is cached under both a strong key and a weak key
assert log == result.output
@@ -72,7 +69,7 @@ def test_artifact_log_files(cli, datafiles):
project = str(datafiles)
# Ensure we have an artifact to read
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
assert result.exit_code == 0
logfiles = os.path.join(project, "logfiles")
@@ -84,16 +81,16 @@ def test_artifact_log_files(cli, datafiles):
assert not os.path.exists(import_bin)
# Run the command and ensure the file now exists
- result = cli.run(project=project, args=['artifact', 'log', '--out', logfiles, 'target.bst', 'import-bin.bst'])
+ result = cli.run(project=project, args=["artifact", "log", "--out", logfiles, "target.bst", "import-bin.bst"])
assert result.exit_code == 0
assert os.path.exists(logfiles)
assert os.path.exists(target)
assert os.path.exists(import_bin)
# Ensure the file contains the logs by checking for the LOG line
- with open(target, 'r') as f:
+ with open(target, "r") as f:
data = f.read()
assert "LOG target.bst" in data
- with open(import_bin, 'r') as f:
+ with open(import_bin, "r") as f:
data = f.read()
assert "LOG import-bin.bst" in data
diff --git a/tests/frontend/artifact_show.py b/tests/frontend/artifact_show.py
index 76ea93d63..6f824c0e4 100644
--- a/tests/frontend/artifact_show.py
+++ b/tests/frontend/artifact_show.py
@@ -27,110 +27,106 @@ from tests.testutils import create_artifact_share
# Project directory
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project",
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",)
# Test artifact show
@pytest.mark.datafiles(DATA_DIR)
def test_artifact_show_element_name(cli, tmpdir, datafiles):
project = str(datafiles)
- element = 'target.bst'
+ element = "target.bst"
- result = cli.run(project=project, args=['artifact', 'show', element])
+ result = cli.run(project=project, args=["artifact", "show", element])
result.assert_success()
- assert 'not cached {}'.format(element) in result.output
+ assert "not cached {}".format(element) in result.output
- result = cli.run(project=project, args=['build', element])
+ result = cli.run(project=project, args=["build", element])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'show', element])
+ result = cli.run(project=project, args=["artifact", "show", element])
result.assert_success()
- assert 'cached {}'.format(element) in result.output
+ assert "cached {}".format(element) in result.output
# Test artifact show on a failed element
@pytest.mark.datafiles(DATA_DIR)
def test_artifact_show_failed_element(cli, tmpdir, datafiles):
project = str(datafiles)
- element = 'manual.bst'
+ element = "manual.bst"
- result = cli.run(project=project, args=['artifact', 'show', element])
+ result = cli.run(project=project, args=["artifact", "show", element])
result.assert_success()
- assert 'not cached {}'.format(element) in result.output
+ assert "not cached {}".format(element) in result.output
- result = cli.run(project=project, args=['build', element])
- result.assert_task_error(ErrorDomain.SANDBOX, 'missing-command')
+ result = cli.run(project=project, args=["build", element])
+ result.assert_task_error(ErrorDomain.SANDBOX, "missing-command")
- result = cli.run(project=project, args=['artifact', 'show', element])
+ result = cli.run(project=project, args=["artifact", "show", element])
result.assert_success()
- assert 'failed {}'.format(element) in result.output
+ assert "failed {}".format(element) in result.output
# Test artifact show with a deleted dependency
@pytest.mark.datafiles(DATA_DIR)
def test_artifact_show_element_missing_deps(cli, tmpdir, datafiles):
project = str(datafiles)
- element = 'target.bst'
- dependency = 'import-bin.bst'
+ element = "target.bst"
+ dependency = "import-bin.bst"
- result = cli.run(project=project, args=['build', element])
+ result = cli.run(project=project, args=["build", element])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'delete', dependency])
+ result = cli.run(project=project, args=["artifact", "delete", dependency])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'show', '--deps', 'all', element])
+ result = cli.run(project=project, args=["artifact", "show", "--deps", "all", element])
result.assert_success()
- assert 'not cached {}'.format(dependency) in result.output
- assert 'cached {}'.format(element) in result.output
+ assert "not cached {}".format(dependency) in result.output
+ assert "cached {}".format(element) in result.output
# Test artifact show with artifact ref
@pytest.mark.datafiles(DATA_DIR)
def test_artifact_show_artifact_ref(cli, tmpdir, datafiles):
project = str(datafiles)
- element = 'target.bst'
+ element = "target.bst"
- result = cli.run(project=project, args=['build', element])
+ result = cli.run(project=project, args=["build", element])
result.assert_success()
cache_key = cli.get_element_key(project, element)
- artifact_ref = 'test/target/' + cache_key
+ artifact_ref = "test/target/" + cache_key
- result = cli.run(project=project, args=['artifact', 'show', artifact_ref])
+ result = cli.run(project=project, args=["artifact", "show", artifact_ref])
result.assert_success()
- assert 'cached {}'.format(artifact_ref) in result.output
+ assert "cached {}".format(artifact_ref) in result.output
# Test artifact show artifact in remote
@pytest.mark.datafiles(DATA_DIR)
def test_artifact_show_element_available_remotely(cli, tmpdir, datafiles):
project = str(datafiles)
- element = 'target.bst'
+ element = "target.bst"
# Set up remote and local shares
- local_cache = os.path.join(str(tmpdir), 'artifacts')
- with create_artifact_share(os.path.join(str(tmpdir), 'remote')) as remote:
- cli.configure({
- 'artifacts': {'url': remote.repo, 'push': True},
- 'cachedir': local_cache,
- })
+ local_cache = os.path.join(str(tmpdir), "artifacts")
+ with create_artifact_share(os.path.join(str(tmpdir), "remote")) as remote:
+ cli.configure(
+ {"artifacts": {"url": remote.repo, "push": True}, "cachedir": local_cache,}
+ )
# Build the element
- result = cli.run(project=project, args=['build', element])
+ result = cli.run(project=project, args=["build", element])
result.assert_success()
# Make sure it's in the share
- assert remote.get_artifact(cli.get_artifact_name(project, 'test', element))
+ assert remote.get_artifact(cli.get_artifact_name(project, "test", element))
# Delete the artifact from the local cache
- result = cli.run(project=project, args=['artifact', 'delete', element])
+ result = cli.run(project=project, args=["artifact", "delete", element])
result.assert_success()
- assert cli.get_element_state(project, element) != 'cached'
+ assert cli.get_element_state(project, element) != "cached"
- result = cli.run(project=project, args=['artifact', 'show', element])
+ result = cli.run(project=project, args=["artifact", "show", element])
result.assert_success()
- assert 'available {}'.format(element) in result.output
+ assert "available {}".format(element) in result.output
diff --git a/tests/frontend/buildcheckout.py b/tests/frontend/buildcheckout.py
index a24446d61..364ff98d1 100644
--- a/tests/frontend/buildcheckout.py
+++ b/tests/frontend/buildcheckout.py
@@ -21,69 +21,64 @@ from tests.testutils import generate_junction, create_artifact_share
from . import configure_project
# Project directory
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project",
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",)
def strict_args(args, strict):
if strict != "strict":
- return ['--no-strict', *args]
+ return ["--no-strict", *args]
return args
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("strict,hardlinks", [
- ("strict", "copies"),
- ("strict", "hardlinks"),
- ("non-strict", "copies"),
- ("non-strict", "hardlinks"),
-])
+@pytest.mark.parametrize(
+ "strict,hardlinks",
+ [("strict", "copies"), ("strict", "hardlinks"), ("non-strict", "copies"), ("non-strict", "hardlinks"),],
+)
def test_build_checkout(datafiles, cli, strict, hardlinks):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout')
+ checkout = os.path.join(cli.directory, "checkout")
# First build it
- result = cli.run(project=project, args=strict_args(['build', 'target.bst'], strict))
+ result = cli.run(project=project, args=strict_args(["build", "target.bst"], strict))
result.assert_success()
# Assert that after a successful build, the builddir is empty
- builddir = os.path.join(cli.directory, 'build')
+ builddir = os.path.join(cli.directory, "build")
assert os.path.isdir(builddir)
assert not os.listdir(builddir)
# Prepare checkout args
- checkout_args = strict_args(['artifact', 'checkout'], strict)
+ checkout_args = strict_args(["artifact", "checkout"], strict)
if hardlinks == "hardlinks":
- checkout_args += ['--hardlinks']
- checkout_args += ['target.bst', '--directory', checkout]
+ checkout_args += ["--hardlinks"]
+ checkout_args += ["target.bst", "--directory", checkout]
# Now check it out
result = cli.run(project=project, args=checkout_args)
result.assert_success()
# Check that the executable hello file is found in the checkout
- filename = os.path.join(checkout, 'usr', 'bin', 'hello')
+ filename = os.path.join(checkout, "usr", "bin", "hello")
assert os.path.exists(filename)
- filename = os.path.join(checkout, 'usr', 'include', 'pony.h')
+ filename = os.path.join(checkout, "usr", "include", "pony.h")
assert os.path.exists(filename)
@pytest.mark.datafiles(DATA_DIR + "_world")
def test_build_default_all(datafiles, cli):
project = str(datafiles)
- result = cli.run(project=project, silent=True, args=['build'])
+ result = cli.run(project=project, silent=True, args=["build"])
result.assert_success()
target_dir = os.path.join(cli.directory, DATA_DIR + "_world", "elements")
output_dir = os.path.join(cli.directory, "logs", "test")
- expected = subprocess.Popen(('ls', target_dir), stdout=subprocess.PIPE)
+ expected = subprocess.Popen(("ls", target_dir), stdout=subprocess.PIPE)
expected = subprocess.check_output(("wc", "-w"), stdin=expected.stdout)
- results = subprocess.Popen(('ls', output_dir), stdout=subprocess.PIPE)
+ results = subprocess.Popen(("ls", output_dir), stdout=subprocess.PIPE)
results = subprocess.check_output(("wc", "-w"), stdin=results.stdout)
assert results == expected
@@ -92,7 +87,7 @@ def test_build_default_all(datafiles, cli):
@pytest.mark.datafiles(DATA_DIR + "_default")
def test_build_default(cli, datafiles):
project = str(datafiles)
- result = cli.run(project=project, silent=True, args=['build'])
+ result = cli.run(project=project, silent=True, args=["build"])
result.assert_success()
results = cli.get_element_state(project, "target2.bst")
@@ -101,60 +96,56 @@ def test_build_default(cli, datafiles):
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("strict,hardlinks", [
- ("non-strict", "hardlinks"),
-])
+@pytest.mark.parametrize("strict,hardlinks", [("non-strict", "hardlinks"),])
def test_build_invalid_suffix(datafiles, cli, strict, hardlinks):
project = str(datafiles)
- result = cli.run(project=project, args=strict_args(['build', 'target.foo'], strict))
+ result = cli.run(project=project, args=strict_args(["build", "target.foo"], strict))
result.assert_main_error(ErrorDomain.LOAD, "bad-element-suffix")
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("strict,hardlinks", [
- ("non-strict", "hardlinks"),
-])
+@pytest.mark.parametrize("strict,hardlinks", [("non-strict", "hardlinks"),])
def test_build_invalid_suffix_dep(datafiles, cli, strict, hardlinks):
project = str(datafiles)
# target2.bst depends on an element called target.foo
- result = cli.run(project=project, args=strict_args(['build', 'target2.bst'], strict))
+ result = cli.run(project=project, args=strict_args(["build", "target2.bst"], strict))
result.assert_main_error(ErrorDomain.LOAD, "bad-element-suffix")
-@pytest.mark.skipif(IS_WINDOWS, reason='Not available on Windows')
+@pytest.mark.skipif(IS_WINDOWS, reason="Not available on Windows")
@pytest.mark.datafiles(DATA_DIR)
def test_build_invalid_filename_chars(datafiles, cli):
project = str(datafiles)
- element_name = 'invalid-chars|<>-in-name.bst'
+ element_name = "invalid-chars|<>-in-name.bst"
# The name of this file contains characters that are not allowed by
# BuildStream, using it should raise a warning.
element = {
- 'kind': 'stack',
+ "kind": "stack",
}
- _yaml.roundtrip_dump(element, os.path.join(project, 'elements', element_name))
+ _yaml.roundtrip_dump(element, os.path.join(project, "elements", element_name))
- result = cli.run(project=project, args=strict_args(['build', element_name], 'non-strict'))
+ result = cli.run(project=project, args=strict_args(["build", element_name], "non-strict"))
result.assert_main_error(ErrorDomain.LOAD, "bad-characters-in-name")
-@pytest.mark.skipif(IS_WINDOWS, reason='Not available on Windows')
+@pytest.mark.skipif(IS_WINDOWS, reason="Not available on Windows")
@pytest.mark.datafiles(DATA_DIR)
def test_build_invalid_filename_chars_dep(datafiles, cli):
project = str(datafiles)
- element_name = 'invalid-chars|<>-in-name.bst'
+ element_name = "invalid-chars|<>-in-name.bst"
# The name of this file contains characters that are not allowed by
# BuildStream, and is listed as a dependency of 'invalid-chars-in-dep.bst'.
# This should also raise a warning.
element = {
- 'kind': 'stack',
+ "kind": "stack",
}
- _yaml.roundtrip_dump(element, os.path.join(project, 'elements', element_name))
+ _yaml.roundtrip_dump(element, os.path.join(project, "elements", element_name))
- result = cli.run(project=project, args=strict_args(['build', 'invalid-chars-in-dep.bst'], 'non-strict'))
+ result = cli.run(project=project, args=strict_args(["build", "invalid-chars-in-dep.bst"], "non-strict"))
result.assert_main_error(ErrorDomain.LOAD, "bad-characters-in-name")
@@ -162,39 +153,40 @@ def test_build_invalid_filename_chars_dep(datafiles, cli):
@pytest.mark.parametrize("deps", [("run"), ("none"), ("build"), ("all")])
def test_build_checkout_deps(datafiles, cli, deps):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout')
+ checkout = os.path.join(cli.directory, "checkout")
element_name = "checkout-deps.bst"
# First build it
- result = cli.run(project=project, args=['build', element_name])
+ result = cli.run(project=project, args=["build", element_name])
result.assert_success()
# Assert that after a successful build, the builddir is empty
- builddir = os.path.join(cli.directory, 'build')
+ builddir = os.path.join(cli.directory, "build")
assert os.path.isdir(builddir)
assert not os.listdir(builddir)
# Now check it out
- result = cli.run(project=project, args=['artifact', 'checkout', element_name,
- '--deps', deps, '--directory', checkout])
+ result = cli.run(
+ project=project, args=["artifact", "checkout", element_name, "--deps", deps, "--directory", checkout]
+ )
result.assert_success()
# Verify output of this element
- filename = os.path.join(checkout, 'etc', 'buildstream', 'config')
+ filename = os.path.join(checkout, "etc", "buildstream", "config")
if deps == "build":
assert not os.path.exists(filename)
else:
assert os.path.exists(filename)
# Verify output of this element's build dependencies
- filename = os.path.join(checkout, 'usr', 'include', 'pony.h')
+ filename = os.path.join(checkout, "usr", "include", "pony.h")
if deps in ["build", "all"]:
assert os.path.exists(filename)
else:
assert not os.path.exists(filename)
# Verify output of this element's runtime dependencies
- filename = os.path.join(checkout, 'usr', 'bin', 'hello')
+ filename = os.path.join(checkout, "usr", "bin", "hello")
if deps in ["run", "all"]:
assert os.path.exists(filename)
else:
@@ -204,162 +196,165 @@ def test_build_checkout_deps(datafiles, cli, deps):
@pytest.mark.datafiles(DATA_DIR)
def test_build_checkout_unbuilt(datafiles, cli):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout')
+ checkout = os.path.join(cli.directory, "checkout")
# Check that checking out an unbuilt element fails nicely
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkout])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkout])
result.assert_main_error(ErrorDomain.STREAM, "uncached-checkout-attempt")
@pytest.mark.datafiles(DATA_DIR)
def test_build_checkout_compression_no_tar(datafiles, cli):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout.tar')
+ checkout = os.path.join(cli.directory, "checkout.tar")
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- checkout_args = ['artifact', 'checkout', '--directory', checkout, '--compression', 'gz', 'target.bst']
+ checkout_args = ["artifact", "checkout", "--directory", checkout, "--compression", "gz", "target.bst"]
result = cli.run(project=project, args=checkout_args)
assert "ERROR: --compression can only be provided if --tar is provided" in result.stderr
assert result.exit_code != 0
+
# If we don't support the extension, we default to an uncompressed tarball
@pytest.mark.datafiles(DATA_DIR)
def test_build_checkout_tar_with_unconventional_name(datafiles, cli):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout.foo')
+ checkout = os.path.join(cli.directory, "checkout.foo")
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- checkout_args = ['artifact', 'checkout', '--tar', checkout, 'target.bst']
+ checkout_args = ["artifact", "checkout", "--tar", checkout, "target.bst"]
result = cli.run(project=project, args=checkout_args)
result.assert_success()
- tar = tarfile.open(name=checkout, mode='r')
- assert os.path.join('.', 'usr', 'bin', 'hello') in tar.getnames()
- assert os.path.join('.', 'usr', 'include', 'pony.h') in tar.getnames()
+ tar = tarfile.open(name=checkout, mode="r")
+ assert os.path.join(".", "usr", "bin", "hello") in tar.getnames()
+ assert os.path.join(".", "usr", "include", "pony.h") in tar.getnames()
@pytest.mark.datafiles(DATA_DIR)
def test_build_checkout_tar_with_unsupported_ext(datafiles, cli):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout.tar.foo')
+ checkout = os.path.join(cli.directory, "checkout.tar.foo")
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- checkout_args = ['artifact', 'checkout', '--tar', checkout, 'target.bst']
+ checkout_args = ["artifact", "checkout", "--tar", checkout, "target.bst"]
result = cli.run(project=project, args=checkout_args)
- assert "Invalid file extension given with '--tar': Expected compression with unknown file extension ('.foo'), " \
- "supported extensions are ('.tar'), ('.gz'), ('.xz'), ('.bz2')" in result.stderr
+ assert (
+ "Invalid file extension given with '--tar': Expected compression with unknown file extension ('.foo'), "
+ "supported extensions are ('.tar'), ('.gz'), ('.xz'), ('.bz2')" in result.stderr
+ )
@pytest.mark.datafiles(DATA_DIR)
def test_build_checkout_tar_no_compression(datafiles, cli):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout.tar.gz')
+ checkout = os.path.join(cli.directory, "checkout.tar.gz")
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- builddir = os.path.join(cli.directory, 'build')
+ builddir = os.path.join(cli.directory, "build")
assert os.path.isdir(builddir)
assert not os.listdir(builddir)
- checkout_args = ['artifact', 'checkout', '--tar', checkout, 'target.bst']
+ checkout_args = ["artifact", "checkout", "--tar", checkout, "target.bst"]
result = cli.run(project=project, args=checkout_args)
result.assert_success()
- tar = tarfile.open(name=checkout, mode='r:gz')
- assert os.path.join('.', 'usr', 'bin', 'hello') in tar.getnames()
- assert os.path.join('.', 'usr', 'include', 'pony.h') in tar.getnames()
+ tar = tarfile.open(name=checkout, mode="r:gz")
+ assert os.path.join(".", "usr", "bin", "hello") in tar.getnames()
+ assert os.path.join(".", "usr", "include", "pony.h") in tar.getnames()
@pytest.mark.datafiles(DATA_DIR)
def test_build_checkout_tarball(datafiles, cli):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout.tar')
+ checkout = os.path.join(cli.directory, "checkout.tar")
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- builddir = os.path.join(cli.directory, 'build')
+ builddir = os.path.join(cli.directory, "build")
assert os.path.isdir(builddir)
assert not os.listdir(builddir)
- checkout_args = ['artifact', 'checkout', '--tar', checkout, 'target.bst']
+ checkout_args = ["artifact", "checkout", "--tar", checkout, "target.bst"]
result = cli.run(project=project, args=checkout_args)
result.assert_success()
tar = tarfile.TarFile(checkout)
- assert os.path.join('.', 'usr', 'bin', 'hello') in tar.getnames()
- assert os.path.join('.', 'usr', 'include', 'pony.h') in tar.getnames()
+ assert os.path.join(".", "usr", "bin", "hello") in tar.getnames()
+ assert os.path.join(".", "usr", "include", "pony.h") in tar.getnames()
@pytest.mark.datafiles(DATA_DIR)
def test_build_checkout_using_ref(datafiles, cli):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout')
+ checkout = os.path.join(cli.directory, "checkout")
- result = cli.run(project=project, args=['build', 'checkout-deps.bst'])
+ result = cli.run(project=project, args=["build", "checkout-deps.bst"])
result.assert_success()
- key = cli.get_element_key(project, 'checkout-deps.bst')
- checkout_args = ['artifact', 'checkout', '--directory', checkout, '--deps', 'none', 'test/checkout-deps/' + key]
+ key = cli.get_element_key(project, "checkout-deps.bst")
+ checkout_args = ["artifact", "checkout", "--directory", checkout, "--deps", "none", "test/checkout-deps/" + key]
result = cli.run(project=project, args=checkout_args)
result.assert_success()
- filename = os.path.join(checkout, 'etc', 'buildstream', 'config')
+ filename = os.path.join(checkout, "etc", "buildstream", "config")
assert os.path.exists(filename)
@pytest.mark.datafiles(DATA_DIR)
def test_build_checkout_tarball_using_ref(datafiles, cli):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout.tar')
+ checkout = os.path.join(cli.directory, "checkout.tar")
- result = cli.run(project=project, args=['build', 'checkout-deps.bst'])
+ result = cli.run(project=project, args=["build", "checkout-deps.bst"])
result.assert_success()
- builddir = os.path.join(cli.directory, 'build')
+ builddir = os.path.join(cli.directory, "build")
assert os.path.isdir(builddir)
assert not os.listdir(builddir)
- key = cli.get_element_key(project, 'checkout-deps.bst')
- checkout_args = ['artifact', 'checkout', '--deps', 'none', '--tar', checkout, 'test/checkout-deps/' + key]
+ key = cli.get_element_key(project, "checkout-deps.bst")
+ checkout_args = ["artifact", "checkout", "--deps", "none", "--tar", checkout, "test/checkout-deps/" + key]
result = cli.run(project=project, args=checkout_args)
result.assert_success()
tar = tarfile.TarFile(checkout)
- assert os.path.join('.', 'etc', 'buildstream', 'config') in tar.getnames()
+ assert os.path.join(".", "etc", "buildstream", "config") in tar.getnames()
@pytest.mark.datafiles(DATA_DIR)
def test_build_checkout_build_deps_using_ref(datafiles, cli):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout')
+ checkout = os.path.join(cli.directory, "checkout")
- result = cli.run(project=project, args=['build', 'checkout-deps.bst'])
+ result = cli.run(project=project, args=["build", "checkout-deps.bst"])
result.assert_success()
- key = cli.get_element_key(project, 'checkout-deps.bst')
- checkout_args = ['artifact', 'checkout', '--directory', checkout, '--deps', 'build', 'test/checkout-deps/' + key]
+ key = cli.get_element_key(project, "checkout-deps.bst")
+ checkout_args = ["artifact", "checkout", "--directory", checkout, "--deps", "build", "test/checkout-deps/" + key]
result = cli.run(project=project, args=checkout_args)
result.assert_success()
- build_dep_files = os.path.join(checkout, 'usr', 'include', 'pony.h')
- runtime_dep_files = os.path.join(checkout, 'usr', 'bin', 'hello')
- target_files = os.path.join(checkout, 'etc', 'buildstream', 'config')
+ build_dep_files = os.path.join(checkout, "usr", "include", "pony.h")
+ runtime_dep_files = os.path.join(checkout, "usr", "bin", "hello")
+ target_files = os.path.join(checkout, "etc", "buildstream", "config")
assert os.path.exists(build_dep_files)
assert not os.path.exists(runtime_dep_files)
assert not os.path.exists(target_files)
@@ -368,13 +363,13 @@ def test_build_checkout_build_deps_using_ref(datafiles, cli):
@pytest.mark.datafiles(DATA_DIR)
def test_build_checkout_runtime_deps_using_ref_fails(datafiles, cli):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout')
+ checkout = os.path.join(cli.directory, "checkout")
- result = cli.run(project=project, args=['build', 'checkout-deps.bst'])
+ result = cli.run(project=project, args=["build", "checkout-deps.bst"])
result.assert_success()
- key = cli.get_element_key(project, 'checkout-deps.bst')
- checkout_args = ['artifact', 'checkout', '--directory', checkout, '--deps', 'run', 'test/checkout-deps/' + key]
+ key = cli.get_element_key(project, "checkout-deps.bst")
+ checkout_args = ["artifact", "checkout", "--directory", checkout, "--deps", "run", "test/checkout-deps/" + key]
result = cli.run(project=project, args=checkout_args)
result.assert_main_error(ErrorDomain.STREAM, None)
@@ -383,17 +378,17 @@ def test_build_checkout_runtime_deps_using_ref_fails(datafiles, cli):
@pytest.mark.datafiles(DATA_DIR)
def test_build_checkout_invalid_ref(datafiles, cli):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout.tar')
+ checkout = os.path.join(cli.directory, "checkout.tar")
- result = cli.run(project=project, args=['build', 'checkout-deps.bst'])
+ result = cli.run(project=project, args=["build", "checkout-deps.bst"])
result.assert_success()
- builddir = os.path.join(cli.directory, 'build')
+ builddir = os.path.join(cli.directory, "build")
assert os.path.isdir(builddir)
assert not os.listdir(builddir)
- non_existent_artifact = 'test/checkout-deps/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
- checkout_args = ['artifact', 'checkout', '--deps', 'none', '--tar', checkout, non_existent_artifact]
+ non_existent_artifact = "test/checkout-deps/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+ checkout_args = ["artifact", "checkout", "--deps", "none", "--tar", checkout, non_existent_artifact]
result = cli.run(project=project, args=checkout_args)
assert "Error while staging dependencies into a sandbox: 'No artifacts to stage'" in result.stderr
@@ -404,16 +399,16 @@ def test_build_checkout_no_tar_no_directory(datafiles, cli, tmpdir):
project = str(datafiles)
runtestdir = str(tmpdir)
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- checkout_args = ['artifact', 'checkout', 'target.bst']
+ checkout_args = ["artifact", "checkout", "target.bst"]
result = cli.run(cwd=runtestdir, project=project, args=checkout_args)
result.assert_success()
- filename = os.path.join(runtestdir, 'target', 'usr', 'bin', 'hello')
+ filename = os.path.join(runtestdir, "target", "usr", "bin", "hello")
assert os.path.exists(filename)
- filename = os.path.join(runtestdir, 'target', 'usr', 'include', 'pony.h')
+ filename = os.path.join(runtestdir, "target", "usr", "include", "pony.h")
assert os.path.exists(filename)
@@ -421,61 +416,61 @@ def test_build_checkout_no_tar_no_directory(datafiles, cli, tmpdir):
@pytest.mark.parametrize("compression", [("gz"), ("xz"), ("bz2")])
def test_build_checkout_tarball_compression(datafiles, cli, compression):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout.tar')
+ checkout = os.path.join(cli.directory, "checkout.tar")
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- builddir = os.path.join(cli.directory, 'build')
+ builddir = os.path.join(cli.directory, "build")
assert os.path.isdir(builddir)
assert not os.listdir(builddir)
- checkout_args = ['artifact', 'checkout', '--tar', checkout, '--compression', compression, 'target.bst']
+ checkout_args = ["artifact", "checkout", "--tar", checkout, "--compression", compression, "target.bst"]
result = cli.run(project=project, args=checkout_args)
result.assert_success()
# Docs say not to use TarFile class directly, using .open seems to work.
# https://docs.python.org/3/library/tarfile.html#tarfile.TarFile
- tar = tarfile.open(name=checkout, mode='r:' + compression)
- assert os.path.join('.', 'usr', 'bin', 'hello') in tar.getnames()
- assert os.path.join('.', 'usr', 'include', 'pony.h') in tar.getnames()
+ tar = tarfile.open(name=checkout, mode="r:" + compression)
+ assert os.path.join(".", "usr", "bin", "hello") in tar.getnames()
+ assert os.path.join(".", "usr", "include", "pony.h") in tar.getnames()
@pytest.mark.datafiles(DATA_DIR)
def test_build_checkout_tarball_stdout(datafiles, cli):
project = str(datafiles)
- tarball = os.path.join(cli.directory, 'tarball.tar')
+ tarball = os.path.join(cli.directory, "tarball.tar")
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- builddir = os.path.join(cli.directory, 'build')
+ builddir = os.path.join(cli.directory, "build")
assert os.path.isdir(builddir)
assert not os.listdir(builddir)
- checkout_args = ['artifact', 'checkout', '--tar', '-', 'target.bst']
+ checkout_args = ["artifact", "checkout", "--tar", "-", "target.bst"]
result = cli.run(project=project, args=checkout_args, binary_capture=True)
result.assert_success()
- with open(tarball, 'wb') as f:
+ with open(tarball, "wb") as f:
f.write(result.output)
tar = tarfile.TarFile(tarball)
- assert os.path.join('.', 'usr', 'bin', 'hello') in tar.getnames()
- assert os.path.join('.', 'usr', 'include', 'pony.h') in tar.getnames()
+ assert os.path.join(".", "usr", "bin", "hello") in tar.getnames()
+ assert os.path.join(".", "usr", "include", "pony.h") in tar.getnames()
@pytest.mark.datafiles(DATA_DIR)
def test_build_checkout_tarball_mtime_nonzero(datafiles, cli):
project = str(datafiles)
- tarpath = os.path.join(cli.directory, 'mtime_tar.tar')
+ tarpath = os.path.join(cli.directory, "mtime_tar.tar")
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- checkout_args = ['artifact', 'checkout', '--tar', tarpath, 'target.bst']
+ checkout_args = ["artifact", "checkout", "--tar", tarpath, "target.bst"]
result = cli.run(project=project, args=checkout_args)
result.assert_success()
@@ -490,31 +485,31 @@ def test_build_checkout_tarball_mtime_nonzero(datafiles, cli):
@pytest.mark.datafiles(DATA_DIR)
def test_build_checkout_tarball_is_deterministic(datafiles, cli):
project = str(datafiles)
- tarball1 = os.path.join(cli.directory, 'tarball1.tar')
- tarball2 = os.path.join(cli.directory, 'tarball2.tar')
+ tarball1 = os.path.join(cli.directory, "tarball1.tar")
+ tarball2 = os.path.join(cli.directory, "tarball2.tar")
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- builddir = os.path.join(cli.directory, 'build')
+ builddir = os.path.join(cli.directory, "build")
assert os.path.isdir(builddir)
assert not os.listdir(builddir)
- checkout_args = ['artifact', 'checkout', '--force', 'target.bst']
+ checkout_args = ["artifact", "checkout", "--force", "target.bst"]
- checkout_args1 = checkout_args + ['--tar', tarball1]
+ checkout_args1 = checkout_args + ["--tar", tarball1]
result = cli.run(project=project, args=checkout_args1)
result.assert_success()
- checkout_args2 = checkout_args + ['--tar', tarball2]
+ checkout_args2 = checkout_args + ["--tar", tarball2]
result = cli.run(project=project, args=checkout_args2)
result.assert_success()
- with open(tarball1, 'rb') as f:
+ with open(tarball1, "rb") as f:
contents = f.read()
hash1 = hashlib.sha1(contents).hexdigest()
- with open(tarball2, 'rb') as f:
+ with open(tarball2, "rb") as f:
contents = f.read()
hash2 = hashlib.sha1(contents).hexdigest()
@@ -524,75 +519,75 @@ def test_build_checkout_tarball_is_deterministic(datafiles, cli):
@pytest.mark.datafiles(DATA_DIR)
def test_build_checkout_tarball_links(datafiles, cli):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout.tar')
- extract = os.path.join(cli.directory, 'extract')
+ checkout = os.path.join(cli.directory, "checkout.tar")
+ extract = os.path.join(cli.directory, "extract")
# Create the link before running the tests.
# This is needed for users working on Windows, git checks out symlinks as files which content is the name
# of the symlink and the test therefore doesn't have the correct content
os.symlink(
os.path.join("..", "basicfile"),
- os.path.join(project, "files", "files-and-links", "basicfolder", "basicsymlink")
+ os.path.join(project, "files", "files-and-links", "basicfolder", "basicsymlink"),
)
- result = cli.run(project=project, args=['build', 'import-links.bst'])
+ result = cli.run(project=project, args=["build", "import-links.bst"])
result.assert_success()
- builddir = os.path.join(cli.directory, 'build')
+ builddir = os.path.join(cli.directory, "build")
assert os.path.isdir(builddir)
assert not os.listdir(builddir)
- checkout_args = ['artifact', 'checkout', '--tar', checkout, 'import-links.bst']
+ checkout_args = ["artifact", "checkout", "--tar", checkout, "import-links.bst"]
result = cli.run(project=project, args=checkout_args)
result.assert_success()
tar = tarfile.open(name=checkout, mode="r:")
tar.extractall(extract)
- assert open(os.path.join(extract, 'basicfolder', 'basicsymlink')).read() == "file contents\n"
+ assert open(os.path.join(extract, "basicfolder", "basicsymlink")).read() == "file contents\n"
@pytest.mark.datafiles(DATA_DIR)
def test_build_checkout_links(datafiles, cli):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout')
+ checkout = os.path.join(cli.directory, "checkout")
# Create the link before running the tests.
# This is needed for users working on Windows, git checks out symlinks as files which content is the name
# of the symlink and the test therefore doesn't have the correct content
os.symlink(
os.path.join("..", "basicfile"),
- os.path.join(project, "files", "files-and-links", "basicfolder", "basicsymlink")
+ os.path.join(project, "files", "files-and-links", "basicfolder", "basicsymlink"),
)
- result = cli.run(project=project, args=['build', 'import-links.bst'])
+ result = cli.run(project=project, args=["build", "import-links.bst"])
result.assert_success()
- builddir = os.path.join(cli.directory, 'build')
+ builddir = os.path.join(cli.directory, "build")
assert os.path.isdir(builddir)
assert not os.listdir(builddir)
- checkout_args = ['artifact', 'checkout', '--directory', checkout, 'import-links.bst']
+ checkout_args = ["artifact", "checkout", "--directory", checkout, "import-links.bst"]
result = cli.run(project=project, args=checkout_args)
result.assert_success()
- assert open(os.path.join(checkout, 'basicfolder', 'basicsymlink')).read() == "file contents\n"
+ assert open(os.path.join(checkout, "basicfolder", "basicsymlink")).read() == "file contents\n"
@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.parametrize("hardlinks", [("copies"), ("hardlinks")])
def test_build_checkout_nonempty(datafiles, cli, hardlinks):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout')
+ checkout = os.path.join(cli.directory, "checkout")
filename = os.path.join(checkout, "file.txt")
# First build it
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
# Assert that after a successful build, the builddir is empty
- builddir = os.path.join(cli.directory, 'build')
+ builddir = os.path.join(cli.directory, "build")
assert os.path.isdir(builddir)
assert not os.listdir(builddir)
@@ -602,10 +597,10 @@ def test_build_checkout_nonempty(datafiles, cli, hardlinks):
f.write("Hello")
# Prepare checkout args
- checkout_args = ['artifact', 'checkout']
+ checkout_args = ["artifact", "checkout"]
if hardlinks == "hardlinks":
- checkout_args += ['--hardlinks']
- checkout_args += ['target.bst', '--directory', checkout]
+ checkout_args += ["--hardlinks"]
+ checkout_args += ["target.bst", "--directory", checkout]
# Now check it out
result = cli.run(project=project, args=checkout_args)
@@ -616,15 +611,15 @@ def test_build_checkout_nonempty(datafiles, cli, hardlinks):
@pytest.mark.parametrize("hardlinks", [("copies"), ("hardlinks")])
def test_build_checkout_force(datafiles, cli, hardlinks):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout')
+ checkout = os.path.join(cli.directory, "checkout")
filename = os.path.join(checkout, "file.txt")
# First build it
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
# Assert that after a successful build, the builddir is empty
- builddir = os.path.join(cli.directory, 'build')
+ builddir = os.path.join(cli.directory, "build")
assert os.path.isdir(builddir)
assert not os.listdir(builddir)
@@ -634,466 +629,388 @@ def test_build_checkout_force(datafiles, cli, hardlinks):
f.write("Hello")
# Prepare checkout args
- checkout_args = ['artifact', 'checkout', '--force']
+ checkout_args = ["artifact", "checkout", "--force"]
if hardlinks == "hardlinks":
- checkout_args += ['--hardlinks']
- checkout_args += ['target.bst', '--directory', checkout]
+ checkout_args += ["--hardlinks"]
+ checkout_args += ["target.bst", "--directory", checkout]
# Now check it out
result = cli.run(project=project, args=checkout_args)
result.assert_success()
# Check that the file we added is still there
- filename = os.path.join(checkout, 'file.txt')
+ filename = os.path.join(checkout, "file.txt")
assert os.path.exists(filename)
# Check that the executable hello file is found in the checkout
- filename = os.path.join(checkout, 'usr', 'bin', 'hello')
+ filename = os.path.join(checkout, "usr", "bin", "hello")
assert os.path.exists(filename)
# Check that the executable hello file is found in the checkout
- filename = os.path.join(checkout, 'usr', 'include', 'pony.h')
+ filename = os.path.join(checkout, "usr", "include", "pony.h")
assert os.path.exists(filename)
@pytest.mark.datafiles(DATA_DIR)
def test_build_checkout_force_tarball(datafiles, cli):
project = str(datafiles)
- tarball = os.path.join(cli.directory, 'tarball.tar')
+ tarball = os.path.join(cli.directory, "tarball.tar")
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- builddir = os.path.join(cli.directory, 'build')
+ builddir = os.path.join(cli.directory, "build")
assert os.path.isdir(builddir)
assert not os.listdir(builddir)
with open(tarball, "w") as f:
f.write("Hello")
- checkout_args = ['artifact', 'checkout', '--force', '--tar', tarball, 'target.bst']
+ checkout_args = ["artifact", "checkout", "--force", "--tar", tarball, "target.bst"]
result = cli.run(project=project, args=checkout_args)
result.assert_success()
tar = tarfile.TarFile(tarball)
- assert os.path.join('.', 'usr', 'bin', 'hello') in tar.getnames()
- assert os.path.join('.', 'usr', 'include', 'pony.h') in tar.getnames()
+ assert os.path.join(".", "usr", "bin", "hello") in tar.getnames()
+ assert os.path.join(".", "usr", "include", "pony.h") in tar.getnames()
@pytest.mark.datafiles(DATA_DIR)
def test_install_to_build(cli, datafiles):
project = str(datafiles)
- element = 'installed-to-build.bst'
+ element = "installed-to-build.bst"
# Attempt building the element
# We expect this to throw an ElementError, since the element will
# attempt to stage into /buildstream/build, which is not allowed.
- result = cli.run(project=project, args=strict_args(['build', element], True))
+ result = cli.run(project=project, args=strict_args(["build", element], True))
result.assert_main_error(ErrorDomain.STREAM, None)
result.assert_task_error(ErrorDomain.ELEMENT, None)
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
+@pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")])
def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage):
project = str(datafiles)
- subproject_path = os.path.join(project, 'files', 'sub-project')
- junction_path = os.path.join(project, 'elements', 'junction.bst')
- element_path = os.path.join(project, 'elements', 'junction-dep.bst')
+ subproject_path = os.path.join(project, "files", "sub-project")
+ junction_path = os.path.join(project, "elements", "junction.bst")
+ element_path = os.path.join(project, "elements", "junction-dep.bst")
- configure_project(project, {
- 'ref-storage': ref_storage
- })
+ configure_project(project, {"ref-storage": ref_storage})
# Create a repo to hold the subproject and generate a junction element for it
generate_junction(tmpdir, subproject_path, junction_path, store_ref=False)
# Create a stack element to depend on a cross junction element
#
- element = {
- 'kind': 'stack',
- 'depends': [
- {
- 'junction': 'junction.bst',
- 'filename': 'import-etc.bst'
- }
- ]
- }
+ element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]}
_yaml.roundtrip_dump(element, element_path)
# Now try to track it, this will bail with the appropriate error
# informing the user to track the junction first
- result = cli.run(project=project, args=['build', 'junction-dep.bst'])
+ result = cli.run(project=project, args=["build", "junction-dep.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT)
# Assert that we have the expected provenance encoded into the error
- element_node = _yaml.load(element_path, shortname='junction-dep.bst')
- ref_node = element_node.get_sequence('depends').mapping_at(0)
+ element_node = _yaml.load(element_path, shortname="junction-dep.bst")
+ ref_node = element_node.get_sequence("depends").mapping_at(0)
provenance = ref_node.get_provenance()
assert str(provenance) in result.stderr
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
+@pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")])
def test_unfetched_junction(cli, tmpdir, datafiles, ref_storage):
project = str(datafiles)
- subproject_path = os.path.join(project, 'files', 'sub-project')
- junction_path = os.path.join(project, 'elements', 'junction.bst')
- element_path = os.path.join(project, 'elements', 'junction-dep.bst')
+ subproject_path = os.path.join(project, "files", "sub-project")
+ junction_path = os.path.join(project, "elements", "junction.bst")
+ element_path = os.path.join(project, "elements", "junction-dep.bst")
- configure_project(project, {
- 'ref-storage': ref_storage
- })
+ configure_project(project, {"ref-storage": ref_storage})
# Create a repo to hold the subproject and generate a junction element for it
- ref = generate_junction(tmpdir, subproject_path, junction_path, store_ref=(ref_storage == 'inline'))
+ ref = generate_junction(tmpdir, subproject_path, junction_path, store_ref=(ref_storage == "inline"))
# Create a stack element to depend on a cross junction element
#
- element = {
- 'kind': 'stack',
- 'depends': [
- {
- 'junction': 'junction.bst',
- 'filename': 'import-etc.bst'
- }
- ]
- }
+ element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]}
_yaml.roundtrip_dump(element, element_path)
# Dump a project.refs if we're using project.refs storage
#
- if ref_storage == 'project.refs':
- project_refs = {
- 'projects': {
- 'test': {
- 'junction.bst': [
- {
- 'ref': ref
- }
- ]
- }
- }
- }
- _yaml.roundtrip_dump(project_refs, os.path.join(project, 'junction.refs'))
+ if ref_storage == "project.refs":
+ project_refs = {"projects": {"test": {"junction.bst": [{"ref": ref}]}}}
+ _yaml.roundtrip_dump(project_refs, os.path.join(project, "junction.refs"))
# Now try to build it, this should automatically result in fetching
# the junction itself at load time.
- result = cli.run(project=project, args=['build', 'junction-dep.bst'])
+ result = cli.run(project=project, args=["build", "junction-dep.bst"])
result.assert_success()
# Assert that it's cached now
- assert cli.get_element_state(project, 'junction-dep.bst') == 'cached'
+ assert cli.get_element_state(project, "junction-dep.bst") == "cached"
@pytest.mark.datafiles(DATA_DIR)
def test_build_checkout_junction(cli, tmpdir, datafiles):
project = str(datafiles)
- subproject_path = os.path.join(project, 'files', 'sub-project')
- junction_path = os.path.join(project, 'elements', 'junction.bst')
- element_path = os.path.join(project, 'elements', 'junction-dep.bst')
- checkout = os.path.join(cli.directory, 'checkout')
+ subproject_path = os.path.join(project, "files", "sub-project")
+ junction_path = os.path.join(project, "elements", "junction.bst")
+ element_path = os.path.join(project, "elements", "junction-dep.bst")
+ checkout = os.path.join(cli.directory, "checkout")
# Create a repo to hold the subproject and generate a junction element for it
generate_junction(tmpdir, subproject_path, junction_path)
# Create a stack element to depend on a cross junction element
#
- element = {
- 'kind': 'stack',
- 'depends': [
- {
- 'junction': 'junction.bst',
- 'filename': 'import-etc.bst'
- }
- ]
- }
+ element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]}
_yaml.roundtrip_dump(element, element_path)
# Now try to build it, this should automatically result in fetching
# the junction itself at load time.
- result = cli.run(project=project, args=['build', 'junction-dep.bst'])
+ result = cli.run(project=project, args=["build", "junction-dep.bst"])
result.assert_success()
# Assert that it's cached now
- assert cli.get_element_state(project, 'junction-dep.bst') == 'cached'
+ assert cli.get_element_state(project, "junction-dep.bst") == "cached"
# Now check it out
- result = cli.run(project=project, args=[
- 'artifact', 'checkout', 'junction-dep.bst', '--directory', checkout
- ])
+ result = cli.run(project=project, args=["artifact", "checkout", "junction-dep.bst", "--directory", checkout])
result.assert_success()
# Assert the content of /etc/animal.conf
- filename = os.path.join(checkout, 'etc', 'animal.conf')
+ filename = os.path.join(checkout, "etc", "animal.conf")
assert os.path.exists(filename)
- with open(filename, 'r') as f:
+ with open(filename, "r") as f:
contents = f.read()
- assert contents == 'animal=Pony\n'
+ assert contents == "animal=Pony\n"
# Test that default targets work with projects with junctions
@pytest.mark.datafiles(DATA_DIR + "_world")
def test_build_checkout_junction_default_targets(cli, tmpdir, datafiles):
project = str(datafiles)
- subproject_path = os.path.join(project, 'files', 'sub-project')
- junction_path = os.path.join(project, 'elements', 'junction.bst')
- element_path = os.path.join(project, 'elements', 'junction-dep.bst')
- checkout = os.path.join(cli.directory, 'checkout')
+ subproject_path = os.path.join(project, "files", "sub-project")
+ junction_path = os.path.join(project, "elements", "junction.bst")
+ element_path = os.path.join(project, "elements", "junction-dep.bst")
+ checkout = os.path.join(cli.directory, "checkout")
# Create a repo to hold the subproject and generate a junction element for it
generate_junction(tmpdir, subproject_path, junction_path)
# Create a stack element to depend on a cross junction element
#
- element = {
- 'kind': 'stack',
- 'depends': [
- {
- 'junction': 'junction.bst',
- 'filename': 'import-etc.bst'
- }
- ]
- }
+ element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]}
_yaml.roundtrip_dump(element, element_path)
# Now try to build it, this should automatically result in fetching
# the junction itself at load time.
- result = cli.run(project=project, args=['build'])
+ result = cli.run(project=project, args=["build"])
result.assert_success()
# Assert that it's cached now
- assert cli.get_element_state(project, 'junction-dep.bst') == 'cached'
+ assert cli.get_element_state(project, "junction-dep.bst") == "cached"
# Now check it out
- result = cli.run(project=project, args=[
- 'artifact', 'checkout', 'junction-dep.bst', '--directory', checkout
- ])
+ result = cli.run(project=project, args=["artifact", "checkout", "junction-dep.bst", "--directory", checkout])
result.assert_success()
# Assert the content of /etc/animal.conf
- filename = os.path.join(checkout, 'etc', 'animal.conf')
+ filename = os.path.join(checkout, "etc", "animal.conf")
assert os.path.exists(filename)
- with open(filename, 'r') as f:
+ with open(filename, "r") as f:
contents = f.read()
- assert contents == 'animal=Pony\n'
+ assert contents == "animal=Pony\n"
@pytest.mark.datafiles(DATA_DIR)
def test_build_checkout_workspaced_junction(cli, tmpdir, datafiles):
project = str(datafiles)
- subproject_path = os.path.join(project, 'files', 'sub-project')
- junction_path = os.path.join(project, 'elements', 'junction.bst')
- element_path = os.path.join(project, 'elements', 'junction-dep.bst')
- workspace = os.path.join(cli.directory, 'workspace')
- checkout = os.path.join(cli.directory, 'checkout')
+ subproject_path = os.path.join(project, "files", "sub-project")
+ junction_path = os.path.join(project, "elements", "junction.bst")
+ element_path = os.path.join(project, "elements", "junction-dep.bst")
+ workspace = os.path.join(cli.directory, "workspace")
+ checkout = os.path.join(cli.directory, "checkout")
# Create a repo to hold the subproject and generate a junction element for it
generate_junction(tmpdir, subproject_path, junction_path)
# Create a stack element to depend on a cross junction element
#
- element = {
- 'kind': 'stack',
- 'depends': [
- {
- 'junction': 'junction.bst',
- 'filename': 'import-etc.bst'
- }
- ]
- }
+ element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]}
_yaml.roundtrip_dump(element, element_path)
# Now open a workspace on the junction
#
- result = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, 'junction.bst'])
+ result = cli.run(project=project, args=["workspace", "open", "--directory", workspace, "junction.bst"])
result.assert_success()
- filename = os.path.join(workspace, 'files', 'etc-files', 'etc', 'animal.conf')
+ filename = os.path.join(workspace, "files", "etc-files", "etc", "animal.conf")
# Assert the content of /etc/animal.conf in the workspace
assert os.path.exists(filename)
- with open(filename, 'r') as f:
+ with open(filename, "r") as f:
contents = f.read()
- assert contents == 'animal=Pony\n'
+ assert contents == "animal=Pony\n"
# Modify the content of the animal.conf in the workspace
- with open(filename, 'w') as f:
- f.write('animal=Horsy\n')
+ with open(filename, "w") as f:
+ f.write("animal=Horsy\n")
# Now try to build it, this should automatically result in fetching
# the junction itself at load time.
- result = cli.run(project=project, args=['build', 'junction-dep.bst'])
+ result = cli.run(project=project, args=["build", "junction-dep.bst"])
result.assert_success()
# Assert that it's cached now
- assert cli.get_element_state(project, 'junction-dep.bst') == 'cached'
+ assert cli.get_element_state(project, "junction-dep.bst") == "cached"
# Now check it out
- result = cli.run(project=project, args=[
- 'artifact', 'checkout', 'junction-dep.bst', '--directory', checkout
- ])
+ result = cli.run(project=project, args=["artifact", "checkout", "junction-dep.bst", "--directory", checkout])
result.assert_success()
# Assert the workspace modified content of /etc/animal.conf
- filename = os.path.join(checkout, 'etc', 'animal.conf')
+ filename = os.path.join(checkout, "etc", "animal.conf")
assert os.path.exists(filename)
- with open(filename, 'r') as f:
+ with open(filename, "r") as f:
contents = f.read()
- assert contents == 'animal=Horsy\n'
+ assert contents == "animal=Horsy\n"
@pytest.mark.datafiles(DATA_DIR)
def test_build_checkout_cross_junction(datafiles, cli, tmpdir):
project = str(datafiles)
- subproject_path = os.path.join(project, 'files', 'sub-project')
- junction_path = os.path.join(project, 'elements', 'junction.bst')
- checkout = os.path.join(cli.directory, 'checkout')
+ subproject_path = os.path.join(project, "files", "sub-project")
+ junction_path = os.path.join(project, "elements", "junction.bst")
+ checkout = os.path.join(cli.directory, "checkout")
generate_junction(tmpdir, subproject_path, junction_path)
- result = cli.run(project=project, args=['build', 'junction.bst:import-etc.bst'])
+ result = cli.run(project=project, args=["build", "junction.bst:import-etc.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'junction.bst:import-etc.bst',
- '--directory', checkout])
+ result = cli.run(
+ project=project, args=["artifact", "checkout", "junction.bst:import-etc.bst", "--directory", checkout]
+ )
result.assert_success()
- filename = os.path.join(checkout, 'etc', 'animal.conf')
+ filename = os.path.join(checkout, "etc", "animal.conf")
assert os.path.exists(filename)
@pytest.mark.datafiles(DATA_DIR)
def test_build_junction_short_notation(cli, tmpdir, datafiles):
project = str(datafiles)
- subproject_path = os.path.join(project, 'files', 'sub-project')
- junction_path = os.path.join(project, 'elements', 'junction.bst')
- element_path = os.path.join(project, 'elements', 'junction-dep.bst')
- checkout = os.path.join(cli.directory, 'checkout')
+ subproject_path = os.path.join(project, "files", "sub-project")
+ junction_path = os.path.join(project, "elements", "junction.bst")
+ element_path = os.path.join(project, "elements", "junction-dep.bst")
+ checkout = os.path.join(cli.directory, "checkout")
# Create a repo to hold the subproject and generate a junction element for it
generate_junction(tmpdir, subproject_path, junction_path)
# Create a stack element to depend on a cross junction element, using
# colon (:) as the separator
- element = {
- 'kind': 'stack',
- 'depends': ['junction.bst:import-etc.bst']
- }
+ element = {"kind": "stack", "depends": ["junction.bst:import-etc.bst"]}
_yaml.roundtrip_dump(element, element_path)
# Now try to build it, this should automatically result in fetching
# the junction itself at load time.
- result = cli.run(project=project, args=['build', 'junction-dep.bst'])
+ result = cli.run(project=project, args=["build", "junction-dep.bst"])
result.assert_success()
# Assert that it's cached now
- assert cli.get_element_state(project, 'junction-dep.bst') == 'cached'
+ assert cli.get_element_state(project, "junction-dep.bst") == "cached"
# Now check it out
- result = cli.run(project=project, args=[
- 'artifact', 'checkout', 'junction-dep.bst', '--directory', checkout
- ])
+ result = cli.run(project=project, args=["artifact", "checkout", "junction-dep.bst", "--directory", checkout])
result.assert_success()
# Assert the content of /etc/animal.conf
- filename = os.path.join(checkout, 'etc', 'animal.conf')
+ filename = os.path.join(checkout, "etc", "animal.conf")
assert os.path.exists(filename)
- with open(filename, 'r') as f:
+ with open(filename, "r") as f:
contents = f.read()
- assert contents == 'animal=Pony\n'
+ assert contents == "animal=Pony\n"
@pytest.mark.datafiles(DATA_DIR)
def test_build_junction_short_notation_filename(cli, tmpdir, datafiles):
project = str(datafiles)
- subproject_path = os.path.join(project, 'files', 'sub-project')
- junction_path = os.path.join(project, 'elements', 'junction.bst')
- element_path = os.path.join(project, 'elements', 'junction-dep.bst')
- checkout = os.path.join(cli.directory, 'checkout')
+ subproject_path = os.path.join(project, "files", "sub-project")
+ junction_path = os.path.join(project, "elements", "junction.bst")
+ element_path = os.path.join(project, "elements", "junction-dep.bst")
+ checkout = os.path.join(cli.directory, "checkout")
# Create a repo to hold the subproject and generate a junction element for it
generate_junction(tmpdir, subproject_path, junction_path)
# Create a stack element to depend on a cross junction element, using
# colon (:) as the separator
- element = {
- 'kind': 'stack',
- 'depends': [{'filename': 'junction.bst:import-etc.bst'}]
- }
+ element = {"kind": "stack", "depends": [{"filename": "junction.bst:import-etc.bst"}]}
_yaml.roundtrip_dump(element, element_path)
# Now try to build it, this should automatically result in fetching
# the junction itself at load time.
- result = cli.run(project=project, args=['build', 'junction-dep.bst'])
+ result = cli.run(project=project, args=["build", "junction-dep.bst"])
result.assert_success()
# Assert that it's cached now
- assert cli.get_element_state(project, 'junction-dep.bst') == 'cached'
+ assert cli.get_element_state(project, "junction-dep.bst") == "cached"
# Now check it out
- result = cli.run(project=project, args=[
- 'artifact', 'checkout', 'junction-dep.bst', '--directory', checkout
- ])
+ result = cli.run(project=project, args=["artifact", "checkout", "junction-dep.bst", "--directory", checkout])
result.assert_success()
# Assert the content of /etc/animal.conf
- filename = os.path.join(checkout, 'etc', 'animal.conf')
+ filename = os.path.join(checkout, "etc", "animal.conf")
assert os.path.exists(filename)
- with open(filename, 'r') as f:
+ with open(filename, "r") as f:
contents = f.read()
- assert contents == 'animal=Pony\n'
+ assert contents == "animal=Pony\n"
@pytest.mark.datafiles(DATA_DIR)
def test_build_junction_short_notation_with_junction(cli, tmpdir, datafiles):
project = str(datafiles)
- subproject_path = os.path.join(project, 'files', 'sub-project')
- junction_path = os.path.join(project, 'elements', 'junction.bst')
- element_path = os.path.join(project, 'elements', 'junction-dep.bst')
+ subproject_path = os.path.join(project, "files", "sub-project")
+ junction_path = os.path.join(project, "elements", "junction.bst")
+ element_path = os.path.join(project, "elements", "junction-dep.bst")
# Create a repo to hold the subproject and generate a junction element for it
generate_junction(tmpdir, subproject_path, junction_path)
# Create a stack element to depend on a cross junction element, using
# colon (:) as the separator
- element = {
- 'kind': 'stack',
- 'depends': [{
- 'filename': 'junction.bst:import-etc.bst',
- 'junction': 'junction.bst',
- }]
- }
+ element = {"kind": "stack", "depends": [{"filename": "junction.bst:import-etc.bst", "junction": "junction.bst",}]}
_yaml.roundtrip_dump(element, element_path)
# Now try to build it, this should fail as filenames should not contain
# `:` when junction is explicity specified
- result = cli.run(project=project, args=['build', 'junction-dep.bst'])
+ result = cli.run(project=project, args=["build", "junction-dep.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
@pytest.mark.datafiles(DATA_DIR)
def test_build_junction_transitive_short_notation_with_junction(cli, tmpdir, datafiles):
project = str(datafiles)
- subproject_path = os.path.join(project, 'files', 'sub-project')
- junction_path = os.path.join(project, 'elements', 'junction.bst')
- element_path = os.path.join(project, 'elements', 'junction-dep.bst')
+ subproject_path = os.path.join(project, "files", "sub-project")
+ junction_path = os.path.join(project, "elements", "junction.bst")
+ element_path = os.path.join(project, "elements", "junction-dep.bst")
# Create a repo to hold the subproject and generate a junction element for it
generate_junction(tmpdir, subproject_path, junction_path)
# Create a stack element to depend on a cross junction element, using
# colon (:) as the separator
- element = {
- 'kind': 'stack',
- 'depends': ['junction.bst:import-etc.bst:foo.bst']
- }
+ element = {"kind": "stack", "depends": ["junction.bst:import-etc.bst:foo.bst"]}
_yaml.roundtrip_dump(element, element_path)
# Now try to build it, this should fail as recursive lookups for
# cross-junction elements is not allowed.
- result = cli.run(project=project, args=['build', 'junction-dep.bst'])
+ result = cli.run(project=project, args=["build", "junction-dep.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
@@ -1102,11 +1019,11 @@ def test_build_junction_transitive_short_notation_with_junction(cli, tmpdir, dat
@pytest.mark.datafiles(DATA_DIR)
def test_partial_artifact_checkout_fetch(cli, datafiles, tmpdir):
project = str(datafiles)
- checkout_dir = os.path.join(str(tmpdir), 'checkout')
+ checkout_dir = os.path.join(str(tmpdir), "checkout")
- repo = create_repo('git', str(tmpdir))
+ repo = create_repo("git", str(tmpdir))
repo.create(os.path.join(str(datafiles), "files"))
- element_dir = os.path.join(str(tmpdir), 'elements')
+ element_dir = os.path.join(str(tmpdir), "elements")
project = str(tmpdir)
project_config = {
"name": "partial-artifact-checkout-fetch",
@@ -1118,38 +1035,33 @@ def test_partial_artifact_checkout_fetch(cli, datafiles, tmpdir):
"kind": "import",
"sources": [repo.source_config()],
}
- input_name = 'input.bst'
+ input_name = "input.bst"
input_file = os.path.join(element_dir, input_name)
_yaml.roundtrip_dump(input_config, input_file)
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
- cli.configure({'artifacts': {
- 'url': share.repo,
- 'push': True
- }})
+ cli.configure({"artifacts": {"url": share.repo, "push": True}})
- result = cli.run(project=project, args=['source', 'track', input_name])
+ result = cli.run(project=project, args=["source", "track", input_name])
result.assert_success()
- result = cli.run(project=project, args=['build', input_name])
+ result = cli.run(project=project, args=["build", input_name])
result.assert_success()
# A push artifact cache means we have to pull to push to them, so
# delete some blobs from that CAS such that we have to fetch
- digest = utils.sha256sum(os.path.join(project, 'files', 'bin-files', 'usr', 'bin', 'hello'))
- objpath = os.path.join(cli.directory, 'cas', 'objects', digest[:2], digest[2:])
+ digest = utils.sha256sum(os.path.join(project, "files", "bin-files", "usr", "bin", "hello"))
+ objpath = os.path.join(cli.directory, "cas", "objects", digest[:2], digest[2:])
os.unlink(objpath)
# Verify that the build-only dependency is not (complete) in the local cache
- result = cli.run(project=project, args=[
- 'artifact', 'checkout', input_name,
- '--directory', checkout_dir])
- result.assert_main_error(ErrorDomain.STREAM, 'uncached-checkout-attempt')
+ result = cli.run(project=project, args=["artifact", "checkout", input_name, "--directory", checkout_dir])
+ result.assert_main_error(ErrorDomain.STREAM, "uncached-checkout-attempt")
# Verify that the pull method fetches relevant artifacts in order to stage
- result = cli.run(project=project, args=[
- 'artifact', 'checkout', '--pull', input_name,
- '--directory', checkout_dir])
+ result = cli.run(
+ project=project, args=["artifact", "checkout", "--pull", input_name, "--directory", checkout_dir]
+ )
result.assert_success()
# should have pulled whatever was deleted previous
@@ -1159,18 +1071,13 @@ def test_partial_artifact_checkout_fetch(cli, datafiles, tmpdir):
@pytest.mark.datafiles(DATA_DIR)
def test_partial_checkout_fail(tmpdir, datafiles, cli):
project = str(datafiles)
- build_elt = 'import-bin.bst'
- checkout_dir = os.path.join(str(tmpdir), 'checkout')
+ build_elt = "import-bin.bst"
+ checkout_dir = os.path.join(str(tmpdir), "checkout")
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
- cli.configure({'artifacts': {
- 'url': share.repo,
- 'push': True
- }})
+ cli.configure({"artifacts": {"url": share.repo, "push": True}})
- res = cli.run(project=project, args=[
- 'artifact', 'checkout', '--pull', build_elt, '--directory',
- checkout_dir])
- res.assert_main_error(ErrorDomain.STREAM, 'uncached-checkout-attempt')
- assert re.findall(r'Remote \((\S+)\) does not have artifact (\S+) cached', res.stderr)
+ res = cli.run(project=project, args=["artifact", "checkout", "--pull", build_elt, "--directory", checkout_dir])
+ res.assert_main_error(ErrorDomain.STREAM, "uncached-checkout-attempt")
+ assert re.findall(r"Remote \((\S+)\) does not have artifact (\S+) cached", res.stderr)
diff --git a/tests/frontend/completions.py b/tests/frontend/completions.py
index 3352ddcc9..a5e3c8ed3 100644
--- a/tests/frontend/completions.py
+++ b/tests/frontend/completions.py
@@ -6,21 +6,9 @@ import pytest
from buildstream.testing import cli # pylint: disable=unused-import
# Project directory
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- 'completions'
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "completions")
-MAIN_COMMANDS = [
- 'artifact ',
- 'build ',
- 'help ',
- 'init ',
- 'shell ',
- 'show ',
- 'source ',
- 'workspace '
-]
+MAIN_COMMANDS = ["artifact ", "build ", "help ", "init ", "shell ", "show ", "source ", "workspace "]
MAIN_OPTIONS = [
"--builders ",
@@ -54,27 +42,22 @@ MAIN_OPTIONS = [
]
SOURCE_COMMANDS = [
- 'checkout ',
- 'fetch ',
- 'track ',
+ "checkout ",
+ "fetch ",
+ "track ",
]
ARTIFACT_COMMANDS = [
- 'checkout ',
- 'delete ',
- 'push ',
- 'pull ',
- 'log ',
- 'list-contents ',
- 'show ',
+ "checkout ",
+ "delete ",
+ "push ",
+ "pull ",
+ "log ",
+ "list-contents ",
+ "show ",
]
-WORKSPACE_COMMANDS = [
- 'close ',
- 'list ',
- 'open ',
- 'reset '
-]
+WORKSPACE_COMMANDS = ["close ", "list ", "open ", "reset "]
PROJECT_ELEMENTS = [
"compose-all.bst",
@@ -82,7 +65,7 @@ PROJECT_ELEMENTS = [
"compose-include-bin.bst",
"import-bin.bst",
"import-dev.bst",
- "target.bst"
+ "target.bst",
]
INVALID_ELEMENTS = [
@@ -94,11 +77,9 @@ MIXED_ELEMENTS = PROJECT_ELEMENTS + INVALID_ELEMENTS
def assert_completion(cli, cmd, word_idx, expected, cwd=None):
- result = cli.run(project='.', cwd=cwd, env={
- '_BST_COMPLETION': 'complete',
- 'COMP_WORDS': cmd,
- 'COMP_CWORD': str(word_idx)
- })
+ result = cli.run(
+ project=".", cwd=cwd, env={"_BST_COMPLETION": "complete", "COMP_WORDS": cmd, "COMP_CWORD": str(word_idx)}
+ )
words = []
if result.output:
words = result.output.splitlines()
@@ -112,11 +93,7 @@ def assert_completion(cli, cmd, word_idx, expected, cwd=None):
def assert_completion_failed(cli, cmd, word_idx, expected, cwd=None):
- result = cli.run(cwd=cwd, env={
- '_BST_COMPLETION': 'complete',
- 'COMP_WORDS': cmd,
- 'COMP_CWORD': str(word_idx)
- })
+ result = cli.run(cwd=cwd, env={"_BST_COMPLETION": "complete", "COMP_WORDS": cmd, "COMP_CWORD": str(word_idx)})
words = []
if result.output:
words = result.output.splitlines()
@@ -129,67 +106,76 @@ def assert_completion_failed(cli, cmd, word_idx, expected, cwd=None):
assert words != expected
-@pytest.mark.parametrize("cmd,word_idx,expected", [
- ('bst', 0, []),
- ('bst ', 1, MAIN_COMMANDS),
- ('bst artifact ', 2, ARTIFACT_COMMANDS),
- ('bst source ', 2, SOURCE_COMMANDS),
- ('bst w ', 1, ['workspace ']),
- ('bst workspace ', 2, WORKSPACE_COMMANDS),
-])
+@pytest.mark.parametrize(
+ "cmd,word_idx,expected",
+ [
+ ("bst", 0, []),
+ ("bst ", 1, MAIN_COMMANDS),
+ ("bst artifact ", 2, ARTIFACT_COMMANDS),
+ ("bst source ", 2, SOURCE_COMMANDS),
+ ("bst w ", 1, ["workspace "]),
+ ("bst workspace ", 2, WORKSPACE_COMMANDS),
+ ],
+)
def test_commands(cli, cmd, word_idx, expected):
assert_completion(cli, cmd, word_idx, expected)
-@pytest.mark.parametrize("cmd,word_idx,expected", [
- ('bst -', 1, MAIN_OPTIONS),
- ('bst --l', 1, ['--log-file ']),
-
- # Test that options of subcommands also complete
- ('bst --no-colors build -', 3, ['--deps ', '-d ',
- '--remote ', '-r ']),
-
- # Test the behavior of completing after an option that has a
- # parameter that cannot be completed, vs an option that has
- # no parameter
- ('bst --fetchers ', 2, []),
- ('bst --no-colors ', 2, MAIN_COMMANDS),
-])
+@pytest.mark.parametrize(
+ "cmd,word_idx,expected",
+ [
+ ("bst -", 1, MAIN_OPTIONS),
+ ("bst --l", 1, ["--log-file "]),
+ # Test that options of subcommands also complete
+ ("bst --no-colors build -", 3, ["--deps ", "-d ", "--remote ", "-r "]),
+ # Test the behavior of completing after an option that has a
+ # parameter that cannot be completed, vs an option that has
+ # no parameter
+ ("bst --fetchers ", 2, []),
+ ("bst --no-colors ", 2, MAIN_COMMANDS),
+ ],
+)
def test_options(cli, cmd, word_idx, expected):
assert_completion(cli, cmd, word_idx, expected)
-@pytest.mark.parametrize("cmd,word_idx,expected", [
- ('bst --on-error ', 2, ['continue ', 'quit ', 'terminate ']),
- ('bst --cache-buildtrees ', 2, ['always ', 'auto ', 'never ']),
- ('bst show --deps ', 3, ['all ', 'build ', 'none ', 'plan ', 'run ']),
- ('bst show --deps=', 2, ['all ', 'build ', 'none ', 'plan ', 'run ']),
- ('bst show --deps b', 3, ['build ']),
- ('bst show --deps=b', 2, ['build ']),
- ('bst show --deps r', 3, ['run ']),
- ('bst source track --deps ', 4, ['all ', 'none ']),
-])
+@pytest.mark.parametrize(
+ "cmd,word_idx,expected",
+ [
+ ("bst --on-error ", 2, ["continue ", "quit ", "terminate "]),
+ ("bst --cache-buildtrees ", 2, ["always ", "auto ", "never "]),
+ ("bst show --deps ", 3, ["all ", "build ", "none ", "plan ", "run "]),
+ ("bst show --deps=", 2, ["all ", "build ", "none ", "plan ", "run "]),
+ ("bst show --deps b", 3, ["build "]),
+ ("bst show --deps=b", 2, ["build "]),
+ ("bst show --deps r", 3, ["run "]),
+ ("bst source track --deps ", 4, ["all ", "none "]),
+ ],
+)
def test_option_choice(cli, cmd, word_idx, expected):
assert_completion(cli, cmd, word_idx, expected)
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project'))
-@pytest.mark.parametrize("cmd,word_idx,expected,subdir", [
- # Note that elements/ and files/ are partial completions and
- # as such do not come with trailing whitespace
- ('bst --config ', 2, ['cache/', 'elements/', 'files/', 'project.conf '], None),
- ('bst --log-file ', 2, ['cache/', 'elements/', 'files/', 'project.conf '], None),
- ('bst --config f', 2, ['files/'], None),
- ('bst --log-file f', 2, ['files/'], None),
- ('bst --config files', 2, ['files/bin-files/', 'files/dev-files/'], None),
- ('bst --log-file files', 2, ['files/bin-files/', 'files/dev-files/'], None),
- ('bst --config files/', 2, ['files/bin-files/', 'files/dev-files/'], None),
- ('bst --log-file elements/', 2, [os.path.join('elements', e) + ' ' for e in PROJECT_ELEMENTS], None),
- ('bst --config ../', 2, ['../cache/', '../elements/', '../files/', '../project.conf '], 'files'),
- ('bst --config ../elements/', 2, [os.path.join('..', 'elements', e) + ' ' for e in PROJECT_ELEMENTS], 'files'),
- ('bst --config ../nofile', 2, [], 'files'),
- ('bst --config /pony/rainbow/nobodyhas/this/file', 2, [], 'files'),
-])
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "project"))
+@pytest.mark.parametrize(
+ "cmd,word_idx,expected,subdir",
+ [
+ # Note that elements/ and files/ are partial completions and
+ # as such do not come with trailing whitespace
+ ("bst --config ", 2, ["cache/", "elements/", "files/", "project.conf "], None),
+ ("bst --log-file ", 2, ["cache/", "elements/", "files/", "project.conf "], None),
+ ("bst --config f", 2, ["files/"], None),
+ ("bst --log-file f", 2, ["files/"], None),
+ ("bst --config files", 2, ["files/bin-files/", "files/dev-files/"], None),
+ ("bst --log-file files", 2, ["files/bin-files/", "files/dev-files/"], None),
+ ("bst --config files/", 2, ["files/bin-files/", "files/dev-files/"], None),
+ ("bst --log-file elements/", 2, [os.path.join("elements", e) + " " for e in PROJECT_ELEMENTS], None),
+ ("bst --config ../", 2, ["../cache/", "../elements/", "../files/", "../project.conf "], "files"),
+ ("bst --config ../elements/", 2, [os.path.join("..", "elements", e) + " " for e in PROJECT_ELEMENTS], "files"),
+ ("bst --config ../nofile", 2, [], "files"),
+ ("bst --config /pony/rainbow/nobodyhas/this/file", 2, [], "files"),
+ ],
+)
def test_option_file(datafiles, cli, cmd, word_idx, expected, subdir):
cwd = str(datafiles)
if subdir:
@@ -197,15 +183,18 @@ def test_option_file(datafiles, cli, cmd, word_idx, expected, subdir):
assert_completion(cli, cmd, word_idx, expected, cwd=cwd)
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project'))
-@pytest.mark.parametrize("cmd,word_idx,expected,subdir", [
- # Note that regular files like project.conf are not returned when
- # completing for a directory
- ('bst --directory ', 2, ['cache/', 'elements/', 'files/'], None),
- ('bst --directory elements/', 2, [], None),
- ('bst --directory ', 2, ['dev-files/', 'bin-files/'], 'files'),
- ('bst --directory ../', 2, ['../cache/', '../elements/', '../files/'], 'files'),
-])
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "project"))
+@pytest.mark.parametrize(
+ "cmd,word_idx,expected,subdir",
+ [
+ # Note that regular files like project.conf are not returned when
+ # completing for a directory
+ ("bst --directory ", 2, ["cache/", "elements/", "files/"], None),
+ ("bst --directory elements/", 2, [], None),
+ ("bst --directory ", 2, ["dev-files/", "bin-files/"], "files"),
+ ("bst --directory ../", 2, ["../cache/", "../elements/", "../files/"], "files"),
+ ],
+)
def test_option_directory(datafiles, cli, cmd, word_idx, expected, subdir):
cwd = str(datafiles)
if subdir:
@@ -214,56 +203,82 @@ def test_option_directory(datafiles, cli, cmd, word_idx, expected, subdir):
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("project,cmd,word_idx,expected,subdir", [
- # When running in the project directory
- ('project', 'bst show ', 2, [e + ' ' for e in PROJECT_ELEMENTS], None),
- ('project', 'bst build com', 2,
- ['compose-all.bst ', 'compose-include-bin.bst ', 'compose-exclude-dev.bst '], None),
-
- # When running from the files subdir
- ('project', 'bst show ', 2, [e + ' ' for e in PROJECT_ELEMENTS], 'files'),
- ('project', 'bst build com', 2,
- ['compose-all.bst ', 'compose-include-bin.bst ', 'compose-exclude-dev.bst '], 'files'),
-
- # When passing the project directory
- ('project', 'bst --directory ../ show ', 4, [e + ' ' for e in PROJECT_ELEMENTS], 'files'),
- ('project', 'bst --directory ../ build com', 4,
- ['compose-all.bst ', 'compose-include-bin.bst ', 'compose-exclude-dev.bst '], 'files'),
-
- # Also try multi arguments together
- ('project', 'bst --directory ../ artifact checkout t ', 5, ['target.bst '], 'files'),
- ('project', 'bst --directory ../ artifact checkout --directory ', 6,
- ['bin-files/', 'dev-files/'], 'files'),
-
- # When running in the project directory
- ('no-element-path', 'bst show ', 2,
- [e + ' ' for e in PROJECT_ELEMENTS] + ['files/'], None),
- ('no-element-path', 'bst build com', 2,
- ['compose-all.bst ', 'compose-include-bin.bst ', 'compose-exclude-dev.bst '], None),
-
- # When running from the files subdir
- ('no-element-path', 'bst show ', 2,
- [e + ' ' for e in PROJECT_ELEMENTS] + ['files/'], 'files'),
- ('no-element-path', 'bst build com', 2,
- ['compose-all.bst ', 'compose-include-bin.bst ', 'compose-exclude-dev.bst '], 'files'),
-
- # When passing the project directory
- ('no-element-path', 'bst --directory ../ show ', 4,
- [e + ' ' for e in PROJECT_ELEMENTS] + ['files/'], 'files'),
- ('no-element-path', 'bst --directory ../ show f', 4, ['files/'], 'files'),
- ('no-element-path', 'bst --directory ../ show files/', 4, ['files/bin-files/', 'files/dev-files/'], 'files'),
- ('no-element-path', 'bst --directory ../ build com', 4,
- ['compose-all.bst ', 'compose-include-bin.bst ', 'compose-exclude-dev.bst '], 'files'),
-
- # Also try multi arguments together
- ('no-element-path', 'bst --directory ../ artifact checkout t ', 5, ['target.bst '], 'files'),
- ('no-element-path', 'bst --directory ../ artifact checkout --directory ', 6,
- ['bin-files/', 'dev-files/'], 'files'),
-
- # When element-path have sub-folders
- ('sub-folders', 'bst show base', 2, ['base/wanted.bst '], None),
- ('sub-folders', 'bst show base/', 2, ['base/wanted.bst '], None),
-])
+@pytest.mark.parametrize(
+ "project,cmd,word_idx,expected,subdir",
+ [
+ # When running in the project directory
+ ("project", "bst show ", 2, [e + " " for e in PROJECT_ELEMENTS], None),
+ (
+ "project",
+ "bst build com",
+ 2,
+ ["compose-all.bst ", "compose-include-bin.bst ", "compose-exclude-dev.bst "],
+ None,
+ ),
+ # When running from the files subdir
+ ("project", "bst show ", 2, [e + " " for e in PROJECT_ELEMENTS], "files"),
+ (
+ "project",
+ "bst build com",
+ 2,
+ ["compose-all.bst ", "compose-include-bin.bst ", "compose-exclude-dev.bst "],
+ "files",
+ ),
+ # When passing the project directory
+ ("project", "bst --directory ../ show ", 4, [e + " " for e in PROJECT_ELEMENTS], "files"),
+ (
+ "project",
+ "bst --directory ../ build com",
+ 4,
+ ["compose-all.bst ", "compose-include-bin.bst ", "compose-exclude-dev.bst "],
+ "files",
+ ),
+ # Also try multi arguments together
+ ("project", "bst --directory ../ artifact checkout t ", 5, ["target.bst "], "files"),
+ ("project", "bst --directory ../ artifact checkout --directory ", 6, ["bin-files/", "dev-files/"], "files"),
+ # When running in the project directory
+ ("no-element-path", "bst show ", 2, [e + " " for e in PROJECT_ELEMENTS] + ["files/"], None),
+ (
+ "no-element-path",
+ "bst build com",
+ 2,
+ ["compose-all.bst ", "compose-include-bin.bst ", "compose-exclude-dev.bst "],
+ None,
+ ),
+ # When running from the files subdir
+ ("no-element-path", "bst show ", 2, [e + " " for e in PROJECT_ELEMENTS] + ["files/"], "files"),
+ (
+ "no-element-path",
+ "bst build com",
+ 2,
+ ["compose-all.bst ", "compose-include-bin.bst ", "compose-exclude-dev.bst "],
+ "files",
+ ),
+ # When passing the project directory
+ ("no-element-path", "bst --directory ../ show ", 4, [e + " " for e in PROJECT_ELEMENTS] + ["files/"], "files"),
+ ("no-element-path", "bst --directory ../ show f", 4, ["files/"], "files"),
+ ("no-element-path", "bst --directory ../ show files/", 4, ["files/bin-files/", "files/dev-files/"], "files"),
+ (
+ "no-element-path",
+ "bst --directory ../ build com",
+ 4,
+ ["compose-all.bst ", "compose-include-bin.bst ", "compose-exclude-dev.bst "],
+ "files",
+ ),
+ # Also try multi arguments together
+ ("no-element-path", "bst --directory ../ artifact checkout t ", 5, ["target.bst "], "files"),
+ (
+ "no-element-path",
+ "bst --directory ../ artifact checkout --directory ",
+ 6,
+ ["bin-files/", "dev-files/"],
+ "files",
+ ),
+ # When element-path have sub-folders
+ ("sub-folders", "bst show base", 2, ["base/wanted.bst "], None),
+ ("sub-folders", "bst show base/", 2, ["base/wanted.bst "], None),
+ ],
+)
def test_argument_element(datafiles, cli, project, cmd, word_idx, expected, subdir):
cwd = os.path.join(str(datafiles), project)
if subdir:
@@ -272,11 +287,13 @@ def test_argument_element(datafiles, cli, project, cmd, word_idx, expected, subd
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("project,cmd,word_idx,expected,subdir", [
-
- # When element has invalid suffix
- ('project', 'bst --directory ../ show ', 4, [e + ' ' for e in MIXED_ELEMENTS], 'files')
-])
+@pytest.mark.parametrize(
+ "project,cmd,word_idx,expected,subdir",
+ [
+ # When element has invalid suffix
+ ("project", "bst --directory ../ show ", 4, [e + " " for e in MIXED_ELEMENTS], "files")
+ ],
+)
def test_argument_element_invalid(datafiles, cli, project, cmd, word_idx, expected, subdir):
cwd = os.path.join(str(datafiles), project)
if subdir:
@@ -284,46 +301,45 @@ def test_argument_element_invalid(datafiles, cli, project, cmd, word_idx, expect
assert_completion_failed(cli, cmd, word_idx, expected, cwd=cwd)
-@pytest.mark.parametrize("cmd,word_idx,expected", [
- ('bst he', 1, ['help ']),
- ('bst help ', 2, MAIN_COMMANDS),
- ('bst help artifact ', 3, ARTIFACT_COMMANDS),
- ('bst help in', 2, ['init ']),
- ('bst help source ', 3, SOURCE_COMMANDS),
- ('bst help artifact ', 3, ARTIFACT_COMMANDS),
- ('bst help w', 2, ['workspace ']),
- ('bst help workspace ', 3, WORKSPACE_COMMANDS),
-])
+@pytest.mark.parametrize(
+ "cmd,word_idx,expected",
+ [
+ ("bst he", 1, ["help "]),
+ ("bst help ", 2, MAIN_COMMANDS),
+ ("bst help artifact ", 3, ARTIFACT_COMMANDS),
+ ("bst help in", 2, ["init "]),
+ ("bst help source ", 3, SOURCE_COMMANDS),
+ ("bst help artifact ", 3, ARTIFACT_COMMANDS),
+ ("bst help w", 2, ["workspace "]),
+ ("bst help workspace ", 3, WORKSPACE_COMMANDS),
+ ],
+)
def test_help_commands(cli, cmd, word_idx, expected):
assert_completion(cli, cmd, word_idx, expected)
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "project"))
def test_argument_artifact(cli, datafiles):
project = str(datafiles)
# Build an import element with no dependencies (as there will only be ONE cache key)
- result = cli.run(project=project, args=['build', 'import-bin.bst']) # Has no dependencies
+ result = cli.run(project=project, args=["build", "import-bin.bst"]) # Has no dependencies
result.assert_success()
# Get the key and the artifact ref ($project/$element_name/$key)
- key = cli.get_element_key(project, 'import-bin.bst')
- artifact = os.path.join('test', 'import-bin', key)
+ key = cli.get_element_key(project, "import-bin.bst")
+ artifact = os.path.join("test", "import-bin", key)
# Test autocompletion of the artifact
- cmds = [
- 'bst artifact log ',
- 'bst artifact log t',
- 'bst artifact log test/'
- ]
+ cmds = ["bst artifact log ", "bst artifact log t", "bst artifact log test/"]
for i, cmd in enumerate(cmds):
word_idx = 3
- result = cli.run(project=project, cwd=project, env={
- '_BST_COMPLETION': 'complete',
- 'COMP_WORDS': cmd,
- 'COMP_CWORD': str(word_idx)
- })
+ result = cli.run(
+ project=project,
+ cwd=project,
+ env={"_BST_COMPLETION": "complete", "COMP_WORDS": cmd, "COMP_CWORD": str(word_idx)},
+ )
if result.output:
words = result.output.splitlines() # This leaves an extra space on each e.g. ['foo.bst ']
@@ -332,7 +348,7 @@ def test_argument_artifact(cli, datafiles):
if i == 0:
expected = PROJECT_ELEMENTS + [artifact] # We should now be able to see the artifact
elif i == 1:
- expected = ['target.bst', artifact]
+ expected = ["target.bst", artifact]
elif i == 2:
expected = [artifact]
diff --git a/tests/frontend/compose_splits.py b/tests/frontend/compose_splits.py
index f1f9b73be..d333b031e 100644
--- a/tests/frontend/compose_splits.py
+++ b/tests/frontend/compose_splits.py
@@ -6,35 +6,27 @@ import pytest
from buildstream.testing.runcli import cli # pylint: disable=unused-import
# Project directory
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project",
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",)
-@pytest.mark.parametrize("target", [
- ('compose-include-bin.bst'),
- ('compose-exclude-dev.bst')
-])
+@pytest.mark.parametrize("target", [("compose-include-bin.bst"), ("compose-exclude-dev.bst")])
@pytest.mark.datafiles(DATA_DIR)
def test_compose_splits(datafiles, cli, target):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout')
+ checkout = os.path.join(cli.directory, "checkout")
# First build it
- result = cli.run(project=project, args=['build', target])
+ result = cli.run(project=project, args=["build", target])
result.assert_success()
# Now check it out
- result = cli.run(project=project, args=[
- 'artifact', 'checkout', target, '--directory', checkout
- ])
+ result = cli.run(project=project, args=["artifact", "checkout", target, "--directory", checkout])
result.assert_success()
# Check that the executable hello file is found in the checkout
- filename = os.path.join(checkout, 'usr', 'bin', 'hello')
+ filename = os.path.join(checkout, "usr", "bin", "hello")
assert os.path.exists(filename)
# Check that the executable hello file is found in the checkout
- filename = os.path.join(checkout, 'usr', 'include', 'pony.h')
+ filename = os.path.join(checkout, "usr", "include", "pony.h")
assert not os.path.exists(filename)
diff --git a/tests/frontend/configurable_warnings.py b/tests/frontend/configurable_warnings.py
index caa91bb61..472df3eea 100644
--- a/tests/frontend/configurable_warnings.py
+++ b/tests/frontend/configurable_warnings.py
@@ -11,10 +11,7 @@ from buildstream import _yaml
from buildstream.testing.runcli import cli # pylint: disable=unused-import
from buildstream.testing._utils.site import HAVE_SANDBOX
-TOP_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "configuredwarning"
-)
+TOP_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "configuredwarning")
def get_project(fatal_warnings):
@@ -22,17 +19,9 @@ def get_project(fatal_warnings):
"name": "test",
"element-path": "elements",
"plugins": [
- {
- "origin": "local",
- "path": "plugins",
- "elements": {
- "warninga": 0,
- "warningb": 0,
- "corewarn": 0,
- }
- }
+ {"origin": "local", "path": "plugins", "elements": {"warninga": 0, "warningb": 0, "corewarn": 0,}}
],
- "fatal-warnings": fatal_warnings
+ "fatal-warnings": fatal_warnings,
}
@@ -47,19 +36,21 @@ def build_project(datafiles, fatal_warnings):
@pytest.mark.datafiles(TOP_DIR)
-@pytest.mark.parametrize("element_name, fatal_warnings, expect_fatal, error_domain", [
- ("corewarn.bst", [CoreWarnings.OVERLAPS], True, ErrorDomain.STREAM),
- ("warninga.bst", ["warninga:warning-a"], True, ErrorDomain.STREAM),
- ("warningb.bst", ["warningb:warning-b"], True, ErrorDomain.STREAM),
- ("corewarn.bst", [], False, None),
- ("warninga.bst", [], False, None),
- ("warningb.bst", [], False, None),
- ("warninga.bst", [CoreWarnings.OVERLAPS], False, None),
- ("warningb.bst", [CoreWarnings.OVERLAPS], False, None),
-])
-def test_fatal_warnings(cli, datafiles, element_name,
- fatal_warnings, expect_fatal, error_domain):
- if HAVE_SANDBOX == 'buildbox' and error_domain != ErrorDomain.STREAM:
+@pytest.mark.parametrize(
+ "element_name, fatal_warnings, expect_fatal, error_domain",
+ [
+ ("corewarn.bst", [CoreWarnings.OVERLAPS], True, ErrorDomain.STREAM),
+ ("warninga.bst", ["warninga:warning-a"], True, ErrorDomain.STREAM),
+ ("warningb.bst", ["warningb:warning-b"], True, ErrorDomain.STREAM),
+ ("corewarn.bst", [], False, None),
+ ("warninga.bst", [], False, None),
+ ("warningb.bst", [], False, None),
+ ("warninga.bst", [CoreWarnings.OVERLAPS], False, None),
+ ("warningb.bst", [CoreWarnings.OVERLAPS], False, None),
+ ],
+)
+def test_fatal_warnings(cli, datafiles, element_name, fatal_warnings, expect_fatal, error_domain):
+ if HAVE_SANDBOX == "buildbox" and error_domain != ErrorDomain.STREAM:
pytest.xfail()
project_path = build_project(datafiles, fatal_warnings)
diff --git a/tests/frontend/configuredwarning/plugins/corewarn.py b/tests/frontend/configuredwarning/plugins/corewarn.py
index 1f263a0ce..bef0a4904 100644
--- a/tests/frontend/configuredwarning/plugins/corewarn.py
+++ b/tests/frontend/configuredwarning/plugins/corewarn.py
@@ -19,8 +19,7 @@ class CoreWarn(Element):
pass
def assemble(self, sandbox):
- self.warn("Testing: CoreWarning produced during assemble",
- warning_token=CoreWarnings.OVERLAPS)
+ self.warn("Testing: CoreWarning produced during assemble", warning_token=CoreWarnings.OVERLAPS)
def setup():
diff --git a/tests/frontend/consistencyerror/plugins/consistencybug.py b/tests/frontend/consistencyerror/plugins/consistencybug.py
index c60d81ea0..c442d883a 100644
--- a/tests/frontend/consistencyerror/plugins/consistencybug.py
+++ b/tests/frontend/consistencyerror/plugins/consistencybug.py
@@ -2,7 +2,6 @@ from buildstream import Source
class ConsistencyBugSource(Source):
-
def configure(self, node):
pass
diff --git a/tests/frontend/consistencyerror/plugins/consistencyerror.py b/tests/frontend/consistencyerror/plugins/consistencyerror.py
index bcbd1b5e9..125baf39c 100644
--- a/tests/frontend/consistencyerror/plugins/consistencyerror.py
+++ b/tests/frontend/consistencyerror/plugins/consistencyerror.py
@@ -2,7 +2,6 @@ from buildstream import Source, SourceError
class ConsistencyErrorSource(Source):
-
def configure(self, node):
pass
@@ -15,8 +14,7 @@ class ConsistencyErrorSource(Source):
def get_consistency(self):
# Raise an error unconditionally
- raise SourceError("Something went terribly wrong",
- reason="the-consistency-error")
+ raise SourceError("Something went terribly wrong", reason="the-consistency-error")
def get_ref(self):
return None
diff --git a/tests/frontend/cross_junction_workspace.py b/tests/frontend/cross_junction_workspace.py
index ca21e7548..574de2bed 100644
--- a/tests/frontend/cross_junction_workspace.py
+++ b/tests/frontend/cross_junction_workspace.py
@@ -13,8 +13,8 @@ def prepare_junction_project(cli, tmpdir):
os.makedirs(str(main_project))
os.makedirs(str(sub_project))
- _yaml.roundtrip_dump({'name': 'main'}, str(main_project.join("project.conf")))
- _yaml.roundtrip_dump({'name': 'sub'}, str(sub_project.join("project.conf")))
+ _yaml.roundtrip_dump({"name": "main"}, str(main_project.join("project.conf")))
+ _yaml.roundtrip_dump({"name": "sub"}, str(sub_project.join("project.conf")))
import_dir = tmpdir.join("import")
os.makedirs(str(import_dir))
@@ -26,20 +26,20 @@ def prepare_junction_project(cli, tmpdir):
import_repo = create_repo("git", str(import_repo_dir))
import_ref = import_repo.create(str(import_dir))
- _yaml.roundtrip_dump({'kind': 'import',
- 'sources': [import_repo.source_config(ref=import_ref)]},
- str(sub_project.join("data.bst")))
+ _yaml.roundtrip_dump(
+ {"kind": "import", "sources": [import_repo.source_config(ref=import_ref)]}, str(sub_project.join("data.bst"))
+ )
sub_repo_dir = tmpdir.join("sub_repo")
os.makedirs(str(sub_repo_dir))
sub_repo = create_repo("git", str(sub_repo_dir))
sub_ref = sub_repo.create(str(sub_project))
- _yaml.roundtrip_dump({'kind': 'junction',
- 'sources': [sub_repo.source_config(ref=sub_ref)]},
- str(main_project.join("sub.bst")))
+ _yaml.roundtrip_dump(
+ {"kind": "junction", "sources": [sub_repo.source_config(ref=sub_ref)]}, str(main_project.join("sub.bst"))
+ )
- args = ['source', 'fetch', 'sub.bst']
+ args = ["source", "fetch", "sub.bst"]
result = cli.run(project=str(main_project), args=args)
result.assert_success()
@@ -50,13 +50,13 @@ def open_cross_junction(cli, tmpdir):
project = prepare_junction_project(cli, tmpdir)
workspace = tmpdir.join("workspace")
- element = 'sub.bst:data.bst'
- args = ['workspace', 'open', '--directory', str(workspace), element]
+ element = "sub.bst:data.bst"
+ args = ["workspace", "open", "--directory", str(workspace), element]
result = cli.run(project=project, args=args)
result.assert_success()
- assert cli.get_element_state(project, element) == 'buildable'
- assert os.path.exists(str(workspace.join('hello.txt')))
+ assert cli.get_element_state(project, element) == "buildable"
+ assert os.path.exists(str(workspace.join("hello.txt")))
return project, workspace
@@ -68,55 +68,55 @@ def test_open_cross_junction(cli, tmpdir):
def test_list_cross_junction(cli, tmpdir):
project, _ = open_cross_junction(cli, tmpdir)
- element = 'sub.bst:data.bst'
+ element = "sub.bst:data.bst"
- args = ['workspace', 'list']
+ args = ["workspace", "list"]
result = cli.run(project=project, args=args)
result.assert_success()
loaded = _yaml.load_data(result.output)
- workspaces = loaded.get_sequence('workspaces')
+ workspaces = loaded.get_sequence("workspaces")
assert len(workspaces) == 1
first_workspace = workspaces.mapping_at(0)
- assert 'element' in first_workspace
- assert first_workspace.get_str('element') == element
+ assert "element" in first_workspace
+ assert first_workspace.get_str("element") == element
def test_close_cross_junction(cli, tmpdir):
project, workspace = open_cross_junction(cli, tmpdir)
- element = 'sub.bst:data.bst'
- args = ['workspace', 'close', '--remove-dir', element]
+ element = "sub.bst:data.bst"
+ args = ["workspace", "close", "--remove-dir", element]
result = cli.run(project=project, args=args)
result.assert_success()
assert not os.path.exists(str(workspace))
- args = ['workspace', 'list']
+ args = ["workspace", "list"]
result = cli.run(project=project, args=args)
result.assert_success()
loaded = _yaml.load_data(result.output)
- workspaces = loaded.get_sequence('workspaces')
+ workspaces = loaded.get_sequence("workspaces")
assert not workspaces
def test_close_all_cross_junction(cli, tmpdir):
project, workspace = open_cross_junction(cli, tmpdir)
- args = ['workspace', 'close', '--remove-dir', '--all']
+ args = ["workspace", "close", "--remove-dir", "--all"]
result = cli.run(project=project, args=args)
result.assert_success()
assert not os.path.exists(str(workspace))
- args = ['workspace', 'list']
+ args = ["workspace", "list"]
result = cli.run(project=project, args=args)
result.assert_success()
loaded = _yaml.load_data(result.output)
- workspaces = loaded.get_sequence('workspaces')
+ workspaces = loaded.get_sequence("workspaces")
assert not workspaces
@@ -124,17 +124,17 @@ def test_subdir_command_cross_junction(cli, tmpdir):
# i.e. commands can be run successfully from a subdirectory of the
# junction's workspace, in case project loading logic has gone wrong
project = prepare_junction_project(cli, tmpdir)
- workspace = os.path.join(str(tmpdir), 'workspace')
- junction_element = 'sub.bst'
+ workspace = os.path.join(str(tmpdir), "workspace")
+ junction_element = "sub.bst"
# Open the junction as a workspace
- args = ['workspace', 'open', '--directory', workspace, junction_element]
+ args = ["workspace", "open", "--directory", workspace, junction_element]
result = cli.run(project=project, args=args)
result.assert_success()
# Run commands from a subdirectory of the workspace
newdir = os.path.join(str(workspace), "newdir")
- element_name = 'data.bst'
+ element_name = "data.bst"
os.makedirs(newdir)
- result = cli.run(project=str(workspace), args=['-C', newdir, 'show', element_name])
+ result = cli.run(project=str(workspace), args=["-C", newdir, "show", element_name])
result.assert_success()
diff --git a/tests/frontend/fetch.py b/tests/frontend/fetch.py
index 7ea357ac2..ff3667707 100644
--- a/tests/frontend/fetch.py
+++ b/tests/frontend/fetch.py
@@ -15,54 +15,48 @@ from . import configure_project
# Project directory
TOP_DIR = os.path.dirname(os.path.realpath(__file__))
-DATA_DIR = os.path.join(TOP_DIR, 'project')
+DATA_DIR = os.path.join(TOP_DIR, "project")
-@pytest.mark.datafiles(os.path.join(TOP_DIR, 'project_world'))
+@pytest.mark.datafiles(os.path.join(TOP_DIR, "project_world"))
def test_fetch_default_targets(cli, tmpdir, datafiles):
project = str(datafiles)
- element_path = os.path.join(project, 'elements')
- element_name = 'fetch-test.bst'
+ element_path = os.path.join(project, "elements")
+ element_name = "fetch-test.bst"
# Create our repo object of the given source type with
# the bin files, and then collect the initial ref.
#
- repo = create_repo('git', str(tmpdir))
+ repo = create_repo("git", str(tmpdir))
ref = repo.create(project)
# Write out our test target
- element = {
- 'kind': 'import',
- 'sources': [
- repo.source_config(ref=ref)
- ]
- }
- _yaml.roundtrip_dump(element,
- os.path.join(element_path, element_name))
+ element = {"kind": "import", "sources": [repo.source_config(ref=ref)]}
+ _yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
# Assert that a fetch is needed
- assert cli.get_element_state(project, element_name) == 'fetch needed'
+ assert cli.get_element_state(project, element_name) == "fetch needed"
# Now try to fetch it, using the default target feature
- result = cli.run(project=project, args=['source', 'fetch'])
+ result = cli.run(project=project, args=["source", "fetch"])
result.assert_success()
# Assert that we are now buildable because the source is
# now cached.
- assert cli.get_element_state(project, element_name) == 'buildable'
+ assert cli.get_element_state(project, element_name) == "buildable"
-@pytest.mark.datafiles(os.path.join(TOP_DIR, 'consistencyerror'))
+@pytest.mark.datafiles(os.path.join(TOP_DIR, "consistencyerror"))
def test_fetch_consistency_error(cli, datafiles):
project = str(datafiles)
# When the error occurs outside of the scheduler at load time,
# then the SourceError is reported directly as the main error.
- result = cli.run(project=project, args=['source', 'fetch', 'error.bst'])
- result.assert_main_error(ErrorDomain.SOURCE, 'the-consistency-error')
+ result = cli.run(project=project, args=["source", "fetch", "error.bst"])
+ result.assert_main_error(ErrorDomain.SOURCE, "the-consistency-error")
-@pytest.mark.datafiles(os.path.join(TOP_DIR, 'consistencyerror'))
+@pytest.mark.datafiles(os.path.join(TOP_DIR, "consistencyerror"))
def test_fetch_consistency_bug(cli, datafiles):
project = str(datafiles)
@@ -73,104 +67,68 @@ def test_fetch_consistency_bug(cli, datafiles):
# for a fetch command, we could report this to the user
# more gracefully as a BUG message.
#
- result = cli.run(project=project, args=['source', 'fetch', 'bug.bst'])
+ result = cli.run(project=project, args=["source", "fetch", "bug.bst"])
assert result.exc is not None
assert str(result.exc) == "Something went terribly wrong"
@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.parametrize("strict", [True, False], ids=["strict", "no-strict"])
-@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
+@pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")])
def test_unfetched_junction(cli, tmpdir, datafiles, strict, ref_storage):
project = str(datafiles)
- subproject_path = os.path.join(project, 'files', 'sub-project')
- junction_path = os.path.join(project, 'elements', 'junction.bst')
- element_path = os.path.join(project, 'elements', 'junction-dep.bst')
-
- configure_project(project, {
- 'ref-storage': ref_storage
- })
- cli.configure({
- 'projects': {
- 'test': {
- 'strict': strict
- }
- }
- })
+ subproject_path = os.path.join(project, "files", "sub-project")
+ junction_path = os.path.join(project, "elements", "junction.bst")
+ element_path = os.path.join(project, "elements", "junction-dep.bst")
+
+ configure_project(project, {"ref-storage": ref_storage})
+ cli.configure({"projects": {"test": {"strict": strict}}})
# Create a repo to hold the subproject and generate a junction element for it
- ref = generate_junction(tmpdir, subproject_path, junction_path, store_ref=(ref_storage == 'inline'))
+ ref = generate_junction(tmpdir, subproject_path, junction_path, store_ref=(ref_storage == "inline"))
# Create a stack element to depend on a cross junction element
#
- element = {
- 'kind': 'stack',
- 'depends': [
- {
- 'junction': 'junction.bst',
- 'filename': 'import-etc.bst'
- }
- ]
- }
+ element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]}
_yaml.roundtrip_dump(element, element_path)
# Dump a project.refs if we're using project.refs storage
#
- if ref_storage == 'project.refs':
- project_refs = {
- 'projects': {
- 'test': {
- 'junction.bst': [
- {
- 'ref': ref
- }
- ]
- }
- }
- }
- _yaml.roundtrip_dump(project_refs, os.path.join(project, 'junction.refs'))
+ if ref_storage == "project.refs":
+ project_refs = {"projects": {"test": {"junction.bst": [{"ref": ref}]}}}
+ _yaml.roundtrip_dump(project_refs, os.path.join(project, "junction.refs"))
# Now try to fetch it, this should automatically result in fetching
# the junction itself.
- result = cli.run(project=project, args=['source', 'fetch', 'junction-dep.bst'])
+ result = cli.run(project=project, args=["source", "fetch", "junction-dep.bst"])
result.assert_success()
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
+@pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")])
def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage):
project = str(datafiles)
- subproject_path = os.path.join(project, 'files', 'sub-project')
- junction_path = os.path.join(project, 'elements', 'junction.bst')
- element_path = os.path.join(project, 'elements', 'junction-dep.bst')
+ subproject_path = os.path.join(project, "files", "sub-project")
+ junction_path = os.path.join(project, "elements", "junction.bst")
+ element_path = os.path.join(project, "elements", "junction-dep.bst")
- configure_project(project, {
- 'ref-storage': ref_storage
- })
+ configure_project(project, {"ref-storage": ref_storage})
# Create a repo to hold the subproject and generate a junction element for it
generate_junction(tmpdir, subproject_path, junction_path, store_ref=False)
# Create a stack element to depend on a cross junction element
#
- element = {
- 'kind': 'stack',
- 'depends': [
- {
- 'junction': 'junction.bst',
- 'filename': 'import-etc.bst'
- }
- ]
- }
+ element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]}
_yaml.roundtrip_dump(element, element_path)
# Now try to fetch it, this will bail with the appropriate error
# informing the user to track the junction first
- result = cli.run(project=project, args=['source', 'fetch', 'junction-dep.bst'])
+ result = cli.run(project=project, args=["source", "fetch", "junction-dep.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT)
# Assert that we have the expected provenance encoded into the error
- element_node = _yaml.load(element_path, shortname='junction-dep.bst')
- ref_node = element_node.get_sequence('depends').mapping_at(0)
+ element_node = _yaml.load(element_path, shortname="junction-dep.bst")
+ ref_node = element_node.get_sequence("depends").mapping_at(0)
provenance = ref_node.get_provenance()
assert str(provenance) in result.stderr
diff --git a/tests/frontend/help.py b/tests/frontend/help.py
index 3bc18499b..20a93160f 100644
--- a/tests/frontend/help.py
+++ b/tests/frontend/help.py
@@ -8,28 +8,21 @@ from buildstream.testing.runcli import cli # pylint: disable=unused-import
def assert_help(cli_output):
expected_start = "Usage: "
if not cli_output.startswith(expected_start):
- raise AssertionError("Help output expected to begin with '{}',"
- .format(expected_start) +
- " output was: {}"
- .format(cli_output))
+ raise AssertionError(
+ "Help output expected to begin with '{}',".format(expected_start) + " output was: {}".format(cli_output)
+ )
def test_help_main(cli):
- result = cli.run(args=['--help'])
+ result = cli.run(args=["--help"])
result.assert_success()
assert_help(result.output)
-@pytest.mark.parametrize("command", [
- ('artifact'),
- ('build'),
- ('checkout'),
- ('shell'),
- ('show'),
- ('source'),
- ('workspace')
-])
+@pytest.mark.parametrize(
+ "command", [("artifact"), ("build"), ("checkout"), ("shell"), ("show"), ("source"), ("workspace")]
+)
def test_help(cli, command):
- result = cli.run(args=[command, '--help'])
+ result = cli.run(args=[command, "--help"])
result.assert_success()
assert_help(result.output)
diff --git a/tests/frontend/init.py b/tests/frontend/init.py
index 0fdc0eda5..3b9a95c34 100644
--- a/tests/frontend/init.py
+++ b/tests/frontend/init.py
@@ -13,114 +13,106 @@ from buildstream._versions import BST_FORMAT_VERSION
def test_defaults(cli, tmpdir):
project = str(tmpdir)
- project_path = os.path.join(project, 'project.conf')
+ project_path = os.path.join(project, "project.conf")
- result = cli.run(args=['init', '--project-name', 'foo', project])
+ result = cli.run(args=["init", "--project-name", "foo", project])
result.assert_success()
project_conf = _yaml.load(project_path)
- assert project_conf.get_str('name') == 'foo'
- assert project_conf.get_str('format-version') == str(BST_FORMAT_VERSION)
- assert project_conf.get_str('element-path') == 'elements'
+ assert project_conf.get_str("name") == "foo"
+ assert project_conf.get_str("format-version") == str(BST_FORMAT_VERSION)
+ assert project_conf.get_str("element-path") == "elements"
def test_all_options(cli, tmpdir):
project = str(tmpdir)
- project_path = os.path.join(project, 'project.conf')
-
- result = cli.run(args=[
- 'init',
- '--project-name', 'foo',
- '--format-version', '2',
- '--element-path', 'ponies',
- project
- ])
+ project_path = os.path.join(project, "project.conf")
+
+ result = cli.run(
+ args=["init", "--project-name", "foo", "--format-version", "2", "--element-path", "ponies", project]
+ )
result.assert_success()
project_conf = _yaml.load(project_path)
- assert project_conf.get_str('name') == 'foo'
- assert project_conf.get_str('format-version') == str(2)
- assert project_conf.get_str('element-path') == 'ponies'
+ assert project_conf.get_str("name") == "foo"
+ assert project_conf.get_str("format-version") == str(2)
+ assert project_conf.get_str("element-path") == "ponies"
- elements_dir = os.path.join(project, 'ponies')
+ elements_dir = os.path.join(project, "ponies")
assert os.path.isdir(elements_dir)
def test_no_project_name(cli, tmpdir):
- result = cli.run(args=['init', str(tmpdir)])
- result.assert_main_error(ErrorDomain.APP, 'unspecified-project-name')
+ result = cli.run(args=["init", str(tmpdir)])
+ result.assert_main_error(ErrorDomain.APP, "unspecified-project-name")
def test_project_exists(cli, tmpdir):
project = str(tmpdir)
- project_path = os.path.join(project, 'project.conf')
- with open(project_path, 'w') as f:
- f.write('name: pony\n')
+ project_path = os.path.join(project, "project.conf")
+ with open(project_path, "w") as f:
+ f.write("name: pony\n")
- result = cli.run(args=['init', '--project-name', 'foo', project])
- result.assert_main_error(ErrorDomain.APP, 'project-exists')
+ result = cli.run(args=["init", "--project-name", "foo", project])
+ result.assert_main_error(ErrorDomain.APP, "project-exists")
def test_force_overwrite_project(cli, tmpdir):
project = str(tmpdir)
- project_path = os.path.join(project, 'project.conf')
- with open(project_path, 'w') as f:
- f.write('name: pony\n')
+ project_path = os.path.join(project, "project.conf")
+ with open(project_path, "w") as f:
+ f.write("name: pony\n")
- result = cli.run(args=['init', '--project-name', 'foo', '--force', project])
+ result = cli.run(args=["init", "--project-name", "foo", "--force", project])
result.assert_success()
project_conf = _yaml.load(project_path)
- assert project_conf.get_str('name') == 'foo'
- assert project_conf.get_str('format-version') == str(BST_FORMAT_VERSION)
+ assert project_conf.get_str("name") == "foo"
+ assert project_conf.get_str("format-version") == str(BST_FORMAT_VERSION)
def test_relative_path_directory_as_argument(cli, tmpdir):
- project = os.path.join(str(tmpdir), 'child-directory')
+ project = os.path.join(str(tmpdir), "child-directory")
os.makedirs(project, exist_ok=True)
- project_path = os.path.join(project, 'project.conf')
+ project_path = os.path.join(project, "project.conf")
rel_path = os.path.relpath(project)
- result = cli.run(args=['init', '--project-name', 'foo', rel_path])
+ result = cli.run(args=["init", "--project-name", "foo", rel_path])
result.assert_success()
project_conf = _yaml.load(project_path)
- assert project_conf.get_str('name') == 'foo'
- assert project_conf.get_int('format-version') == BST_FORMAT_VERSION
- assert project_conf.get_str('element-path') == 'elements'
+ assert project_conf.get_str("name") == "foo"
+ assert project_conf.get_int("format-version") == BST_FORMAT_VERSION
+ assert project_conf.get_str("element-path") == "elements"
def test_set_directory_and_directory_as_argument(cli, tmpdir):
- result = cli.run(args=['-C', '/foo/bar', 'init', '--project-name', 'foo', '/boo/far'])
- result.assert_main_error(ErrorDomain.APP, 'init-with-set-directory')
+ result = cli.run(args=["-C", "/foo/bar", "init", "--project-name", "foo", "/boo/far"])
+ result.assert_main_error(ErrorDomain.APP, "init-with-set-directory")
-@pytest.mark.parametrize("project_name", [('Micheal Jackson'), ('one+one')])
+@pytest.mark.parametrize("project_name", [("Micheal Jackson"), ("one+one")])
def test_bad_project_name(cli, tmpdir, project_name):
- result = cli.run(args=['init', '--project-name', str(tmpdir)])
+ result = cli.run(args=["init", "--project-name", str(tmpdir)])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_SYMBOL_NAME)
@pytest.mark.parametrize("format_version", [(str(-1)), (str(BST_FORMAT_VERSION + 1))])
def test_bad_format_version(cli, tmpdir, format_version):
- result = cli.run(args=[
- 'init', '--project-name', 'foo', '--format-version', format_version, str(tmpdir)
- ])
- result.assert_main_error(ErrorDomain.APP, 'invalid-format-version')
+ result = cli.run(args=["init", "--project-name", "foo", "--format-version", format_version, str(tmpdir)])
+ result.assert_main_error(ErrorDomain.APP, "invalid-format-version")
-@pytest.mark.parametrize("element_path", [('/absolute/path'), ('../outside/of/project')])
+@pytest.mark.parametrize("element_path", [("/absolute/path"), ("../outside/of/project")])
def test_bad_element_path(cli, tmpdir, element_path):
- result = cli.run(args=[
- 'init', '--project-name', 'foo', '--element-path', element_path, str(tmpdir)
- ])
- result.assert_main_error(ErrorDomain.APP, 'invalid-element-path')
+ result = cli.run(args=["init", "--project-name", "foo", "--element-path", element_path, str(tmpdir)])
+ result.assert_main_error(ErrorDomain.APP, "invalid-element-path")
-@pytest.mark.parametrize("element_path", [('foo'), ('foo/bar')])
+@pytest.mark.parametrize("element_path", [("foo"), ("foo/bar")])
def test_element_path_interactive(cli, tmp_path, monkeypatch, element_path):
project = tmp_path
- project_conf_path = project.joinpath('project.conf')
+ project_conf_path = project.joinpath("project.conf")
class DummyInteractiveApp(App):
def __init__(self, *args, **kwargs):
@@ -132,17 +124,17 @@ def test_element_path_interactive(cli, tmp_path, monkeypatch, element_path):
return DummyInteractiveApp(*args, **kwargs)
def _init_project_interactive(self, *args, **kwargs): # pylint: disable=arguments-differ
- return ('project_name', '0', element_path)
+ return ("project_name", "0", element_path)
- monkeypatch.setattr(App, 'create', DummyInteractiveApp.create)
+ monkeypatch.setattr(App, "create", DummyInteractiveApp.create)
- result = cli.run(args=['init', str(project)])
+ result = cli.run(args=["init", str(project)])
result.assert_success()
full_element_path = project.joinpath(element_path)
assert full_element_path.exists()
project_conf = _yaml.load(str(project_conf_path))
- assert project_conf.get_str('name') == 'project_name'
- assert project_conf.get_str('format-version') == '0'
- assert project_conf.get_str('element-path') == element_path
+ assert project_conf.get_str("name") == "project_name"
+ assert project_conf.get_str("format-version") == "0"
+ assert project_conf.get_str("element-path") == element_path
diff --git a/tests/frontend/large_directory.py b/tests/frontend/large_directory.py
index 921e2ddbe..ea29fd1ca 100644
--- a/tests/frontend/large_directory.py
+++ b/tests/frontend/large_directory.py
@@ -29,10 +29,7 @@ from tests.testutils import create_artifact_share, assert_shared
# Project directory
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project",
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",)
@contextmanager
@@ -40,7 +37,7 @@ def limit_grpc_message_length(limit):
orig_insecure_channel = grpc.insecure_channel
def new_insecure_channel(target):
- return orig_insecure_channel(target, options=(('grpc.max_send_message_length', limit),))
+ return orig_insecure_channel(target, options=(("grpc.max_send_message_length", limit),))
grpc.insecure_channel = new_insecure_channel
try:
@@ -58,29 +55,25 @@ def test_large_directory(cli, tmpdir, datafiles):
MAX_MESSAGE_LENGTH = 1024 * 1024
NUM_FILES = MAX_MESSAGE_LENGTH // 64 + 1
- large_directory_dir = os.path.join(project, 'files', 'large-directory')
+ large_directory_dir = os.path.join(project, "files", "large-directory")
os.mkdir(large_directory_dir)
for i in range(NUM_FILES):
- with open(os.path.join(large_directory_dir, str(i)), 'w') as f:
+ with open(os.path.join(large_directory_dir, str(i)), "w") as f:
# The files need to have different content as we want different digests.
f.write(str(i))
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
# Configure bst to push to the artifact share
- cli.configure({
- 'artifacts': [
- {'url': share.repo, 'push': True},
- ]
- })
+ cli.configure({"artifacts": [{"url": share.repo, "push": True},]})
# Enforce 1 MB gRPC message limit
with limit_grpc_message_length(MAX_MESSAGE_LENGTH):
# Build and push
- result = cli.run(project=project, args=['build', 'import-large-directory.bst'])
+ result = cli.run(project=project, args=["build", "import-large-directory.bst"])
result.assert_success()
# Assert that we are now cached locally
- assert cli.get_element_state(project, 'import-large-directory.bst') == 'cached'
+ assert cli.get_element_state(project, "import-large-directory.bst") == "cached"
# Assert that the push was successful
- assert_shared(cli, share, project, 'import-large-directory.bst')
+ assert_shared(cli, share, project, "import-large-directory.bst")
diff --git a/tests/frontend/logging.py b/tests/frontend/logging.py
index 462af821f..27ff88352 100644
--- a/tests/frontend/logging.py
+++ b/tests/frontend/logging.py
@@ -13,37 +13,28 @@ from buildstream._exceptions import ErrorDomain
from buildstream.testing import cli # pylint: disable=unused-import
# Project directory
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project",
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",)
@pytest.mark.datafiles(DATA_DIR)
def test_default_logging(cli, tmpdir, datafiles):
project = str(datafiles)
- bin_files_path = os.path.join(project, 'files', 'bin-files')
- element_path = os.path.join(project, 'elements')
- element_name = 'fetch-test-git.bst'
+ bin_files_path = os.path.join(project, "files", "bin-files")
+ element_path = os.path.join(project, "elements")
+ element_name = "fetch-test-git.bst"
# Create our repo object of the given source type with
# the bin files, and then collect the initial ref.
#
- repo = create_repo('git', str(tmpdir))
+ repo = create_repo("git", str(tmpdir))
ref = repo.create(bin_files_path)
# Write out our test target
- element = {
- 'kind': 'import',
- 'sources': [
- repo.source_config(ref=ref)
- ]
- }
- _yaml.roundtrip_dump(element,
- os.path.join(element_path, element_name))
+ element = {"kind": "import", "sources": [repo.source_config(ref=ref)]}
+ _yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
# Now try to fetch it
- result = cli.run(project=project, args=['source', 'fetch', element_name])
+ result = cli.run(project=project, args=["source", "fetch", element_name])
result.assert_success()
m = re.search(r"\[\d\d:\d\d:\d\d\]\[\s*\]\[.*\] SUCCESS Checking sources", result.stderr)
@@ -53,37 +44,34 @@ def test_default_logging(cli, tmpdir, datafiles):
@pytest.mark.datafiles(DATA_DIR)
def test_custom_logging(cli, tmpdir, datafiles):
project = str(datafiles)
- bin_files_path = os.path.join(project, 'files', 'bin-files')
- element_path = os.path.join(project, 'elements')
- element_name = 'fetch-test-git.bst'
-
- custom_log_format = ('%{elapsed},%{elapsed-us},%{wallclock},%{wallclock-us},'
- '%{key},%{element},%{action},%{message}')
- user_config = {'logging': {'message-format': custom_log_format}}
+ bin_files_path = os.path.join(project, "files", "bin-files")
+ element_path = os.path.join(project, "elements")
+ element_name = "fetch-test-git.bst"
+
+ custom_log_format = (
+ "%{elapsed},%{elapsed-us},%{wallclock},%{wallclock-us}," "%{key},%{element},%{action},%{message}"
+ )
+ user_config = {"logging": {"message-format": custom_log_format}}
cli.configure(user_config)
# Create our repo object of the given source type with
# the bin files, and then collect the initial ref.
#
- repo = create_repo('git', str(tmpdir))
+ repo = create_repo("git", str(tmpdir))
ref = repo.create(bin_files_path)
# Write out our test target
- element = {
- 'kind': 'import',
- 'sources': [
- repo.source_config(ref=ref)
- ]
- }
- _yaml.roundtrip_dump(element,
- os.path.join(element_path, element_name))
+ element = {"kind": "import", "sources": [repo.source_config(ref=ref)]}
+ _yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
# Now try to fetch it
- result = cli.run(project=project, args=['source', 'fetch', element_name])
+ result = cli.run(project=project, args=["source", "fetch", element_name])
result.assert_success()
- m = re.search(r"\d\d:\d\d:\d\d,\d\d:\d\d:\d\d.\d{6},\d\d:\d\d:\d\d,\d\d:\d\d:\d\d.\d{6}\s*,.*"
- r",SUCCESS,Checking sources", result.stderr)
+ m = re.search(
+ r"\d\d:\d\d:\d\d,\d\d:\d\d:\d\d.\d{6},\d\d:\d\d:\d\d,\d\d:\d\d:\d\d.\d{6}\s*,.*" r",SUCCESS,Checking sources",
+ result.stderr,
+ )
assert m is not None
@@ -92,19 +80,12 @@ def test_failed_build_listing(cli, datafiles):
project = str(datafiles)
element_names = []
for i in range(3):
- element_name = 'testfail-{}.bst'.format(i)
- element_path = os.path.join('elements', element_name)
- element = {
- 'kind': 'script',
- 'config': {
- 'commands': [
- 'false'
- ]
- }
- }
+ element_name = "testfail-{}.bst".format(i)
+ element_path = os.path.join("elements", element_name)
+ element = {"kind": "script", "config": {"commands": ["false"]}}
_yaml.roundtrip_dump(element, os.path.join(project, element_path))
element_names.append(element_name)
- result = cli.run(project=project, args=['--on-error=continue', 'build', *element_names])
+ result = cli.run(project=project, args=["--on-error=continue", "build", *element_names])
result.assert_main_error(ErrorDomain.STREAM, None)
# Check that we re-print the failure summaries only in the "Failure Summary"
@@ -115,10 +96,10 @@ def test_failed_build_listing(cli, datafiles):
# testfail-0.bst:
# [00:00:00][44f1b8c3][ build:testfail-0.bst ] FAILURE Running 'commands'
#
- failure_heading_pos = re.search(r'^Failure Summary$', result.stderr, re.MULTILINE).start()
- pipeline_heading_pos = re.search(r'^Pipeline Summary$', result.stderr, re.MULTILINE).start()
+ failure_heading_pos = re.search(r"^Failure Summary$", result.stderr, re.MULTILINE).start()
+ pipeline_heading_pos = re.search(r"^Pipeline Summary$", result.stderr, re.MULTILINE).start()
failure_summary_range = range(failure_heading_pos, pipeline_heading_pos)
- matches = tuple(re.finditer(r'^\s+testfail-.\.bst:$', result.stderr, re.MULTILINE))
+ matches = tuple(re.finditer(r"^\s+testfail-.\.bst:$", result.stderr, re.MULTILINE))
for m in matches:
assert m.start() in failure_summary_range
assert m.end() in failure_summary_range
diff --git a/tests/frontend/main.py b/tests/frontend/main.py
index 0df52e2c9..d864a0b1f 100644
--- a/tests/frontend/main.py
+++ b/tests/frontend/main.py
@@ -5,7 +5,7 @@ from buildstream._frontend.app import _prefix_choice_value_proc
def test_prefix_choice_value_proc_full_match():
- value_proc = _prefix_choice_value_proc(['foo', 'bar', 'baz'])
+ value_proc = _prefix_choice_value_proc(["foo", "bar", "baz"])
assert value_proc("foo") == "foo"
assert value_proc("bar") == "bar"
@@ -13,13 +13,13 @@ def test_prefix_choice_value_proc_full_match():
def test_prefix_choice_value_proc_prefix_match():
- value_proc = _prefix_choice_value_proc(['foo'])
+ value_proc = _prefix_choice_value_proc(["foo"])
assert value_proc("f") == "foo"
def test_prefix_choice_value_proc_ambigous_match():
- value_proc = _prefix_choice_value_proc(['bar', 'baz'])
+ value_proc = _prefix_choice_value_proc(["bar", "baz"])
assert value_proc("bar") == "bar"
assert value_proc("baz") == "baz"
@@ -28,7 +28,7 @@ def test_prefix_choice_value_proc_ambigous_match():
def test_prefix_choice_value_proc_value_not_in_choices():
- value_proc = _prefix_choice_value_proc(['bar', 'baz'])
+ value_proc = _prefix_choice_value_proc(["bar", "baz"])
with pytest.raises(click.UsageError):
value_proc("foo")
diff --git a/tests/frontend/mirror.py b/tests/frontend/mirror.py
index 855155785..f71cb554d 100644
--- a/tests/frontend/mirror.py
+++ b/tests/frontend/mirror.py
@@ -11,15 +11,15 @@ from buildstream.testing import cli # pylint: disable=unused-import
# Project directory
TOP_DIR = os.path.dirname(os.path.realpath(__file__))
-DATA_DIR = os.path.join(TOP_DIR, 'project')
+DATA_DIR = os.path.join(TOP_DIR, "project")
def generate_element(output_file):
element = {
- 'kind': 'import',
- 'sources': [
+ "kind": "import",
+ "sources": [
{
- 'kind': 'fetch_source',
+ "kind": "fetch_source",
"output-text": output_file,
"urls": ["foo:repo1", "bar:repo2"],
"fetch-succeeds": {
@@ -31,53 +31,24 @@ def generate_element(output_file):
"RBA/repo2": False,
"ooF/repo1": False,
"raB/repo2": False,
- }
+ },
}
- ]
+ ],
}
return element
def generate_project():
project = {
- 'name': 'test',
- 'element-path': 'elements',
- 'aliases': {
- 'foo': 'FOO/',
- 'bar': 'BAR/',
- },
- 'mirrors': [
- {
- 'name': 'middle-earth',
- 'aliases': {
- 'foo': ['OOF/'],
- 'bar': ['RAB/'],
- },
- },
- {
- 'name': 'arrakis',
- 'aliases': {
- 'foo': ['OFO/'],
- 'bar': ['RBA/'],
- },
- },
- {
- 'name': 'oz',
- 'aliases': {
- 'foo': ['ooF/'],
- 'bar': ['raB/'],
- }
- },
+ "name": "test",
+ "element-path": "elements",
+ "aliases": {"foo": "FOO/", "bar": "BAR/",},
+ "mirrors": [
+ {"name": "middle-earth", "aliases": {"foo": ["OOF/"], "bar": ["RAB/"],},},
+ {"name": "arrakis", "aliases": {"foo": ["OFO/"], "bar": ["RBA/"],},},
+ {"name": "oz", "aliases": {"foo": ["ooF/"], "bar": ["raB/"],}},
],
- 'plugins': [
- {
- 'origin': 'local',
- 'path': 'sources',
- 'sources': {
- 'fetch_source': 0
- }
- }
- ]
+ "plugins": [{"origin": "local", "path": "sources", "sources": {"fetch_source": 0}}],
}
return project
@@ -86,74 +57,58 @@ def generate_project():
@pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")])
@pytest.mark.parametrize("mirror", [("no-mirror"), ("mirror"), ("unrelated-mirror")])
def test_mirror_fetch_ref_storage(cli, tmpdir, datafiles, ref_storage, mirror):
- bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
- dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
- upstream_repodir = os.path.join(str(tmpdir), 'upstream')
- mirror_repodir = os.path.join(str(tmpdir), 'mirror')
- project_dir = os.path.join(str(tmpdir), 'project')
+ bin_files_path = os.path.join(str(datafiles), "files", "bin-files", "usr")
+ dev_files_path = os.path.join(str(datafiles), "files", "dev-files", "usr")
+ upstream_repodir = os.path.join(str(tmpdir), "upstream")
+ mirror_repodir = os.path.join(str(tmpdir), "mirror")
+ project_dir = os.path.join(str(tmpdir), "project")
os.makedirs(project_dir)
- element_dir = os.path.join(project_dir, 'elements')
+ element_dir = os.path.join(project_dir, "elements")
# Create repo objects of the upstream and mirror
- upstream_repo = create_repo('tar', upstream_repodir)
+ upstream_repo = create_repo("tar", upstream_repodir)
upstream_repo.create(bin_files_path)
mirror_repo = upstream_repo.copy(mirror_repodir)
upstream_ref = upstream_repo.create(dev_files_path)
element = {
- 'kind': 'import',
- 'sources': [
- upstream_repo.source_config(ref=upstream_ref if ref_storage == 'inline' else None)
- ]
+ "kind": "import",
+ "sources": [upstream_repo.source_config(ref=upstream_ref if ref_storage == "inline" else None)],
}
- element_name = 'test.bst'
+ element_name = "test.bst"
element_path = os.path.join(element_dir, element_name)
- full_repo = element['sources'][0]['url']
+ full_repo = element["sources"][0]["url"]
upstream_map, repo_name = os.path.split(full_repo)
- alias = 'foo'
- aliased_repo = alias + ':' + repo_name
- element['sources'][0]['url'] = aliased_repo
- full_mirror = mirror_repo.source_config()['url']
+ alias = "foo"
+ aliased_repo = alias + ":" + repo_name
+ element["sources"][0]["url"] = aliased_repo
+ full_mirror = mirror_repo.source_config()["url"]
mirror_map, _ = os.path.split(full_mirror)
os.makedirs(element_dir)
_yaml.roundtrip_dump(element, element_path)
- if ref_storage == 'project.refs':
+ if ref_storage == "project.refs":
# Manually set project.refs to avoid caching the repo prematurely
- project_refs = {'projects': {
- 'test': {
- element_name: [
- {'ref': upstream_ref}
- ]
- }
- }}
- project_refs_path = os.path.join(project_dir, 'project.refs')
+ project_refs = {"projects": {"test": {element_name: [{"ref": upstream_ref}]}}}
+ project_refs_path = os.path.join(project_dir, "project.refs")
_yaml.roundtrip_dump(project_refs, project_refs_path)
project = {
- 'name': 'test',
- 'element-path': 'elements',
- 'aliases': {
- alias: upstream_map + "/"
- },
- 'ref-storage': ref_storage
+ "name": "test",
+ "element-path": "elements",
+ "aliases": {alias: upstream_map + "/"},
+ "ref-storage": ref_storage,
}
- if mirror != 'no-mirror':
- mirror_data = [{
- 'name': 'middle-earth',
- 'aliases': {alias: [mirror_map + '/']}
- }]
- if mirror == 'unrelated-mirror':
- mirror_data.insert(0, {
- 'name': 'narnia',
- 'aliases': {'frob': ['http://www.example.com/repo']}
- })
- project['mirrors'] = mirror_data
-
- project_file = os.path.join(project_dir, 'project.conf')
+ if mirror != "no-mirror":
+ mirror_data = [{"name": "middle-earth", "aliases": {alias: [mirror_map + "/"]}}]
+ if mirror == "unrelated-mirror":
+ mirror_data.insert(0, {"name": "narnia", "aliases": {"frob": ["http://www.example.com/repo"]}})
+ project["mirrors"] = mirror_data
+
+ project_file = os.path.join(project_dir, "project.conf")
_yaml.roundtrip_dump(project, project_file)
- result = cli.run(project=project_dir, args=['source', 'fetch', element_name])
+ result = cli.run(project=project_dir, args=["source", "fetch", element_name])
result.assert_success()
@@ -162,18 +117,18 @@ def test_mirror_fetch_ref_storage(cli, tmpdir, datafiles, ref_storage, mirror):
def test_mirror_fetch_multi(cli, tmpdir):
output_file = os.path.join(str(tmpdir), "output.txt")
project_dir = str(tmpdir)
- element_dir = os.path.join(project_dir, 'elements')
+ element_dir = os.path.join(project_dir, "elements")
os.makedirs(element_dir, exist_ok=True)
element_name = "test.bst"
element_path = os.path.join(element_dir, element_name)
element = generate_element(output_file)
_yaml.roundtrip_dump(element, element_path)
- project_file = os.path.join(project_dir, 'project.conf')
+ project_file = os.path.join(project_dir, "project.conf")
project = generate_project()
_yaml.roundtrip_dump(project, project_file)
- result = cli.run(project=project_dir, args=['source', 'fetch', element_name])
+ result = cli.run(project=project_dir, args=["source", "fetch", element_name])
result.assert_success()
with open(output_file) as f:
contents = f.read()
@@ -186,18 +141,18 @@ def test_mirror_fetch_multi(cli, tmpdir):
def test_mirror_fetch_default_cmdline(cli, tmpdir):
output_file = os.path.join(str(tmpdir), "output.txt")
project_dir = str(tmpdir)
- element_dir = os.path.join(project_dir, 'elements')
+ element_dir = os.path.join(project_dir, "elements")
os.makedirs(element_dir, exist_ok=True)
element_name = "test.bst"
element_path = os.path.join(element_dir, element_name)
element = generate_element(output_file)
_yaml.roundtrip_dump(element, element_path)
- project_file = os.path.join(project_dir, 'project.conf')
+ project_file = os.path.join(project_dir, "project.conf")
project = generate_project()
_yaml.roundtrip_dump(project, project_file)
- result = cli.run(project=project_dir, args=['--default-mirror', 'arrakis', 'source', 'fetch', element_name])
+ result = cli.run(project=project_dir, args=["--default-mirror", "arrakis", "source", "fetch", element_name])
result.assert_success()
with open(output_file) as f:
contents = f.read()
@@ -217,27 +172,21 @@ def test_mirror_fetch_default_cmdline(cli, tmpdir):
def test_mirror_fetch_default_userconfig(cli, tmpdir):
output_file = os.path.join(str(tmpdir), "output.txt")
project_dir = str(tmpdir)
- element_dir = os.path.join(project_dir, 'elements')
+ element_dir = os.path.join(project_dir, "elements")
os.makedirs(element_dir, exist_ok=True)
element_name = "test.bst"
element_path = os.path.join(element_dir, element_name)
element = generate_element(output_file)
_yaml.roundtrip_dump(element, element_path)
- project_file = os.path.join(project_dir, 'project.conf')
+ project_file = os.path.join(project_dir, "project.conf")
project = generate_project()
_yaml.roundtrip_dump(project, project_file)
- userconfig = {
- 'projects': {
- 'test': {
- 'default-mirror': 'oz'
- }
- }
- }
+ userconfig = {"projects": {"test": {"default-mirror": "oz"}}}
cli.configure(userconfig)
- result = cli.run(project=project_dir, args=['source', 'fetch', element_name])
+ result = cli.run(project=project_dir, args=["source", "fetch", element_name])
result.assert_success()
with open(output_file) as f:
contents = f.read()
@@ -257,27 +206,21 @@ def test_mirror_fetch_default_userconfig(cli, tmpdir):
def test_mirror_fetch_default_cmdline_overrides_config(cli, tmpdir):
output_file = os.path.join(str(tmpdir), "output.txt")
project_dir = str(tmpdir)
- element_dir = os.path.join(project_dir, 'elements')
+ element_dir = os.path.join(project_dir, "elements")
os.makedirs(element_dir, exist_ok=True)
element_name = "test.bst"
element_path = os.path.join(element_dir, element_name)
element = generate_element(output_file)
_yaml.roundtrip_dump(element, element_path)
- project_file = os.path.join(project_dir, 'project.conf')
+ project_file = os.path.join(project_dir, "project.conf")
project = generate_project()
_yaml.roundtrip_dump(project, project_file)
- userconfig = {
- 'projects': {
- 'test': {
- 'default-mirror': 'oz'
- }
- }
- }
+ userconfig = {"projects": {"test": {"default-mirror": "oz"}}}
cli.configure(userconfig)
- result = cli.run(project=project_dir, args=['--default-mirror', 'arrakis', 'source', 'fetch', element_name])
+ result = cli.run(project=project_dir, args=["--default-mirror", "arrakis", "source", "fetch", element_name])
result.assert_success()
with open(output_file) as f:
contents = f.read()
@@ -296,79 +239,65 @@ def test_mirror_fetch_default_cmdline_overrides_config(cli, tmpdir):
def test_mirror_git_submodule_fetch(cli, tmpdir, datafiles):
# Test that it behaves as expected with submodules, both defined in config
# and discovered when fetching.
- foo_file = os.path.join(str(datafiles), 'files', 'foo')
- bar_file = os.path.join(str(datafiles), 'files', 'bar')
- bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
- dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
- mirror_dir = os.path.join(str(datafiles), 'mirror')
+ foo_file = os.path.join(str(datafiles), "files", "foo")
+ bar_file = os.path.join(str(datafiles), "files", "bar")
+ bin_files_path = os.path.join(str(datafiles), "files", "bin-files", "usr")
+ dev_files_path = os.path.join(str(datafiles), "files", "dev-files", "usr")
+ mirror_dir = os.path.join(str(datafiles), "mirror")
- defined_subrepo = create_repo('git', str(tmpdir), 'defined_subrepo')
+ defined_subrepo = create_repo("git", str(tmpdir), "defined_subrepo")
defined_subrepo.create(bin_files_path)
defined_subrepo.copy(mirror_dir)
defined_subrepo.add_file(foo_file)
- found_subrepo = create_repo('git', str(tmpdir), 'found_subrepo')
+ found_subrepo = create_repo("git", str(tmpdir), "found_subrepo")
found_subrepo.create(dev_files_path)
- main_repo = create_repo('git', str(tmpdir))
+ main_repo = create_repo("git", str(tmpdir))
main_mirror_ref = main_repo.create(bin_files_path)
- main_repo.add_submodule('defined', 'file://' + defined_subrepo.repo)
- main_repo.add_submodule('found', 'file://' + found_subrepo.repo)
+ main_repo.add_submodule("defined", "file://" + defined_subrepo.repo)
+ main_repo.add_submodule("found", "file://" + found_subrepo.repo)
main_mirror = main_repo.copy(mirror_dir)
main_repo.add_file(bar_file)
- project_dir = os.path.join(str(tmpdir), 'project')
+ project_dir = os.path.join(str(tmpdir), "project")
os.makedirs(project_dir)
- element_dir = os.path.join(project_dir, 'elements')
+ element_dir = os.path.join(project_dir, "elements")
os.makedirs(element_dir)
- element = {
- 'kind': 'import',
- 'sources': [
- main_repo.source_config(ref=main_mirror_ref)
- ]
- }
- element_name = 'test.bst'
+ element = {"kind": "import", "sources": [main_repo.source_config(ref=main_mirror_ref)]}
+ element_name = "test.bst"
element_path = os.path.join(element_dir, element_name)
# Alias the main repo
- full_repo = element['sources'][0]['url']
+ full_repo = element["sources"][0]["url"]
_, repo_name = os.path.split(full_repo)
- alias = 'foo'
- aliased_repo = alias + ':' + repo_name
- element['sources'][0]['url'] = aliased_repo
+ alias = "foo"
+ aliased_repo = alias + ":" + repo_name
+ element["sources"][0]["url"] = aliased_repo
# Hide the found subrepo
- del element['sources'][0]['submodules']['found']
+ del element["sources"][0]["submodules"]["found"]
# Alias the defined subrepo
- subrepo = element['sources'][0]['submodules']['defined']['url']
+ subrepo = element["sources"][0]["submodules"]["defined"]["url"]
_, repo_name = os.path.split(subrepo)
- aliased_repo = alias + ':' + repo_name
- element['sources'][0]['submodules']['defined']['url'] = aliased_repo
+ aliased_repo = alias + ":" + repo_name
+ element["sources"][0]["submodules"]["defined"]["url"] = aliased_repo
_yaml.roundtrip_dump(element, element_path)
- full_mirror = main_mirror.source_config()['url']
+ full_mirror = main_mirror.source_config()["url"]
mirror_map, _ = os.path.split(full_mirror)
project = {
- 'name': 'test',
- 'element-path': 'elements',
- 'aliases': {
- alias: 'http://www.example.com/'
- },
- 'mirrors': [
- {
- 'name': 'middle-earth',
- 'aliases': {
- alias: [mirror_map + "/"],
- },
- },
- ]
+ "name": "test",
+ "element-path": "elements",
+ "aliases": {alias: "http://www.example.com/"},
+ "mirrors": [{"name": "middle-earth", "aliases": {alias: [mirror_map + "/"],},},],
}
- project_file = os.path.join(project_dir, 'project.conf')
+ project_file = os.path.join(project_dir, "project.conf")
_yaml.roundtrip_dump(project, project_file)
- result = cli.run(project=project_dir, args=['source', 'fetch', element_name])
+ result = cli.run(project=project_dir, args=["source", "fetch", element_name])
result.assert_success()
@@ -383,90 +312,76 @@ def test_mirror_fallback_git_only_submodules(cli, tmpdir, datafiles):
# - overriden submodule is fetched from mirror.
# - other submodule is fetched.
- bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
- dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
+ bin_files_path = os.path.join(str(datafiles), "files", "bin-files", "usr")
+ dev_files_path = os.path.join(str(datafiles), "files", "dev-files", "usr")
- upstream_bin_repodir = os.path.join(str(tmpdir), 'bin-upstream')
- mirror_bin_repodir = os.path.join(str(tmpdir), 'bin-mirror')
- upstream_bin_repo = create_repo('git', upstream_bin_repodir)
+ upstream_bin_repodir = os.path.join(str(tmpdir), "bin-upstream")
+ mirror_bin_repodir = os.path.join(str(tmpdir), "bin-mirror")
+ upstream_bin_repo = create_repo("git", upstream_bin_repodir)
upstream_bin_repo.create(bin_files_path)
mirror_bin_repo = upstream_bin_repo.copy(mirror_bin_repodir)
- dev_repodir = os.path.join(str(tmpdir), 'dev-upstream')
- dev_repo = create_repo('git', dev_repodir)
+ dev_repodir = os.path.join(str(tmpdir), "dev-upstream")
+ dev_repo = create_repo("git", dev_repodir)
dev_repo.create(dev_files_path)
- main_files = os.path.join(str(tmpdir), 'main-files')
+ main_files = os.path.join(str(tmpdir), "main-files")
os.makedirs(main_files)
- with open(os.path.join(main_files, 'README'), 'w') as f:
+ with open(os.path.join(main_files, "README"), "w") as f:
f.write("TEST\n")
- main_repodir = os.path.join(str(tmpdir), 'main-upstream')
- main_repo = create_repo('git', main_repodir)
+ main_repodir = os.path.join(str(tmpdir), "main-upstream")
+ main_repo = create_repo("git", main_repodir)
main_repo.create(main_files)
- upstream_url = 'file://{}'.format(upstream_bin_repo.repo)
- main_repo.add_submodule('bin', url=upstream_url)
- main_repo.add_submodule('dev', url='file://{}'.format(dev_repo.repo))
+ upstream_url = "file://{}".format(upstream_bin_repo.repo)
+ main_repo.add_submodule("bin", url=upstream_url)
+ main_repo.add_submodule("dev", url="file://{}".format(dev_repo.repo))
# Unlist 'dev'.
- del main_repo.submodules['dev']
+ del main_repo.submodules["dev"]
main_ref = main_repo.latest_commit()
upstream_map, repo_name = os.path.split(upstream_url)
- alias = 'foo'
- aliased_repo = '{}:{}'.format(alias, repo_name)
- main_repo.submodules['bin']['url'] = aliased_repo
+ alias = "foo"
+ aliased_repo = "{}:{}".format(alias, repo_name)
+ main_repo.submodules["bin"]["url"] = aliased_repo
- full_mirror = mirror_bin_repo.source_config()['url']
+ full_mirror = mirror_bin_repo.source_config()["url"]
mirror_map, _ = os.path.split(full_mirror)
- project_dir = os.path.join(str(tmpdir), 'project')
+ project_dir = os.path.join(str(tmpdir), "project")
os.makedirs(project_dir)
- element_dir = os.path.join(project_dir, 'elements')
+ element_dir = os.path.join(project_dir, "elements")
- element = {
- 'kind': 'import',
- 'sources': [
- main_repo.source_config_extra(ref=main_ref, checkout_submodules=True)
- ]
- }
- element_name = 'test.bst'
+ element = {"kind": "import", "sources": [main_repo.source_config_extra(ref=main_ref, checkout_submodules=True)]}
+ element_name = "test.bst"
element_path = os.path.join(element_dir, element_name)
os.makedirs(element_dir)
_yaml.roundtrip_dump(element, element_path)
project = {
- 'name': 'test',
- 'element-path': 'elements',
- 'aliases': {
- alias: upstream_map + "/"
- },
- 'mirrors': [
- {
- 'name': 'middle-earth',
- 'aliases': {
- alias: [mirror_map + "/"],
- }
- }
- ]
+ "name": "test",
+ "element-path": "elements",
+ "aliases": {alias: upstream_map + "/"},
+ "mirrors": [{"name": "middle-earth", "aliases": {alias: [mirror_map + "/"],}}],
}
- project_file = os.path.join(project_dir, 'project.conf')
+ project_file = os.path.join(project_dir, "project.conf")
_yaml.roundtrip_dump(project, project_file)
# Now make the upstream unavailable.
- os.rename(upstream_bin_repo.repo, '{}.bak'.format(upstream_bin_repo.repo))
- result = cli.run(project=project_dir, args=['source', 'fetch', element_name])
+ os.rename(upstream_bin_repo.repo, "{}.bak".format(upstream_bin_repo.repo))
+ result = cli.run(project=project_dir, args=["source", "fetch", element_name])
result.assert_success()
- result = cli.run(project=project_dir, args=['build', element_name])
+ result = cli.run(project=project_dir, args=["build", element_name])
result.assert_success()
- checkout = os.path.join(str(tmpdir), 'checkout')
- result = cli.run(project=project_dir, args=['artifact', 'checkout', element_name, '--directory', checkout])
+ checkout = os.path.join(str(tmpdir), "checkout")
+ result = cli.run(project=project_dir, args=["artifact", "checkout", element_name, "--directory", checkout])
result.assert_success()
- assert os.path.exists(os.path.join(checkout, 'bin', 'bin', 'hello'))
- assert os.path.exists(os.path.join(checkout, 'dev', 'include', 'pony.h'))
+ assert os.path.exists(os.path.join(checkout, "bin", "bin", "hello"))
+ assert os.path.exists(os.path.join(checkout, "dev", "include", "pony.h"))
@pytest.mark.datafiles(DATA_DIR)
@@ -476,90 +391,79 @@ def test_mirror_fallback_git_with_submodules(cli, tmpdir, datafiles):
# We expect:
# - we will fetch submodules anyway
- bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
- dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
+ bin_files_path = os.path.join(str(datafiles), "files", "bin-files", "usr")
+ dev_files_path = os.path.join(str(datafiles), "files", "dev-files", "usr")
- bin_repodir = os.path.join(str(tmpdir), 'bin-repo')
- bin_repo = create_repo('git', bin_repodir)
+ bin_repodir = os.path.join(str(tmpdir), "bin-repo")
+ bin_repo = create_repo("git", bin_repodir)
bin_repo.create(bin_files_path)
- dev_repodir = os.path.join(str(tmpdir), 'dev-repo')
- dev_repo = create_repo('git', dev_repodir)
+ dev_repodir = os.path.join(str(tmpdir), "dev-repo")
+ dev_repo = create_repo("git", dev_repodir)
dev_repo.create(dev_files_path)
- main_files = os.path.join(str(tmpdir), 'main-files')
+ main_files = os.path.join(str(tmpdir), "main-files")
os.makedirs(main_files)
- with open(os.path.join(main_files, 'README'), 'w') as f:
+ with open(os.path.join(main_files, "README"), "w") as f:
f.write("TEST\n")
- upstream_main_repodir = os.path.join(str(tmpdir), 'main-upstream')
- upstream_main_repo = create_repo('git', upstream_main_repodir)
+ upstream_main_repodir = os.path.join(str(tmpdir), "main-upstream")
+ upstream_main_repo = create_repo("git", upstream_main_repodir)
upstream_main_repo.create(main_files)
- upstream_main_repo.add_submodule('bin', url='file://{}'.format(bin_repo.repo))
- upstream_main_repo.add_submodule('dev', url='file://{}'.format(dev_repo.repo))
+ upstream_main_repo.add_submodule("bin", url="file://{}".format(bin_repo.repo))
+ upstream_main_repo.add_submodule("dev", url="file://{}".format(dev_repo.repo))
# Unlist submodules.
- del upstream_main_repo.submodules['bin']
- del upstream_main_repo.submodules['dev']
+ del upstream_main_repo.submodules["bin"]
+ del upstream_main_repo.submodules["dev"]
upstream_main_ref = upstream_main_repo.latest_commit()
- mirror_main_repodir = os.path.join(str(tmpdir), 'main-mirror')
+ mirror_main_repodir = os.path.join(str(tmpdir), "main-mirror")
mirror_main_repo = upstream_main_repo.copy(mirror_main_repodir)
- upstream_url = mirror_main_repo.source_config()['url']
+ upstream_url = mirror_main_repo.source_config()["url"]
upstream_map, repo_name = os.path.split(upstream_url)
- alias = 'foo'
- aliased_repo = '{}:{}'.format(alias, repo_name)
+ alias = "foo"
+ aliased_repo = "{}:{}".format(alias, repo_name)
- full_mirror = mirror_main_repo.source_config()['url']
+ full_mirror = mirror_main_repo.source_config()["url"]
mirror_map, _ = os.path.split(full_mirror)
- project_dir = os.path.join(str(tmpdir), 'project')
+ project_dir = os.path.join(str(tmpdir), "project")
os.makedirs(project_dir)
- element_dir = os.path.join(project_dir, 'elements')
+ element_dir = os.path.join(project_dir, "elements")
element = {
- 'kind': 'import',
- 'sources': [
- upstream_main_repo.source_config_extra(ref=upstream_main_ref, checkout_submodules=True)
- ]
+ "kind": "import",
+ "sources": [upstream_main_repo.source_config_extra(ref=upstream_main_ref, checkout_submodules=True)],
}
- element['sources'][0]['url'] = aliased_repo
- element_name = 'test.bst'
+ element["sources"][0]["url"] = aliased_repo
+ element_name = "test.bst"
element_path = os.path.join(element_dir, element_name)
os.makedirs(element_dir)
_yaml.roundtrip_dump(element, element_path)
project = {
- 'name': 'test',
- 'element-path': 'elements',
- 'aliases': {
- alias: upstream_map + "/"
- },
- 'mirrors': [
- {
- 'name': 'middle-earth',
- 'aliases': {
- alias: [mirror_map + "/"],
- }
- }
- ]
+ "name": "test",
+ "element-path": "elements",
+ "aliases": {alias: upstream_map + "/"},
+ "mirrors": [{"name": "middle-earth", "aliases": {alias: [mirror_map + "/"],}}],
}
- project_file = os.path.join(project_dir, 'project.conf')
+ project_file = os.path.join(project_dir, "project.conf")
_yaml.roundtrip_dump(project, project_file)
# Now make the upstream unavailable.
- os.rename(upstream_main_repo.repo, '{}.bak'.format(upstream_main_repo.repo))
- result = cli.run(project=project_dir, args=['source', 'fetch', element_name])
+ os.rename(upstream_main_repo.repo, "{}.bak".format(upstream_main_repo.repo))
+ result = cli.run(project=project_dir, args=["source", "fetch", element_name])
result.assert_success()
- result = cli.run(project=project_dir, args=['build', element_name])
+ result = cli.run(project=project_dir, args=["build", element_name])
result.assert_success()
- checkout = os.path.join(str(tmpdir), 'checkout')
- result = cli.run(project=project_dir, args=['artifact', 'checkout', element_name, '--directory', checkout])
+ checkout = os.path.join(str(tmpdir), "checkout")
+ result = cli.run(project=project_dir, args=["artifact", "checkout", element_name, "--directory", checkout])
result.assert_success()
- assert os.path.exists(os.path.join(checkout, 'bin', 'bin', 'hello'))
- assert os.path.exists(os.path.join(checkout, 'dev', 'include', 'pony.h'))
+ assert os.path.exists(os.path.join(checkout, "bin", "bin", "hello"))
+ assert os.path.exists(os.path.join(checkout, "dev", "include", "pony.h"))
diff --git a/tests/frontend/order.py b/tests/frontend/order.py
index c62377419..fbeb7c398 100644
--- a/tests/frontend/order.py
+++ b/tests/frontend/order.py
@@ -9,10 +9,7 @@ from buildstream.testing import cli # pylint: disable=unused-import
from buildstream import _yaml
# Project directory
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project",
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",)
# create_element()
@@ -25,18 +22,12 @@ DATA_DIR = os.path.join(
# Returns:
# (Repo): The corresponding git repository created for the element
def create_element(project, name, dependencies):
- dev_files_path = os.path.join(project, 'files', 'dev-files')
- element_path = os.path.join(project, 'elements')
- repo = create_repo('git', project, "{}-repo".format(name))
+ dev_files_path = os.path.join(project, "files", "dev-files")
+ element_path = os.path.join(project, "elements")
+ repo = create_repo("git", project, "{}-repo".format(name))
ref = repo.create(dev_files_path)
- element = {
- 'kind': 'import',
- 'sources': [
- repo.source_config(ref=ref)
- ],
- 'depends': dependencies
- }
+ element = {"kind": "import", "sources": [repo.source_config(ref=ref)], "depends": dependencies}
_yaml.roundtrip_dump(element, os.path.join(element_path, name))
return repo
@@ -56,37 +47,37 @@ def create_element(project, name, dependencies):
# expected (list): A list of element names in the expected order
#
@pytest.mark.datafiles(os.path.join(DATA_DIR))
-@pytest.mark.parametrize("target,template,expected", [
- # First simple test
- ('3.bst', {
- '0.bst': ['1.bst'],
- '1.bst': [],
- '2.bst': ['0.bst'],
- '3.bst': ['0.bst', '1.bst', '2.bst']
- }, ['1.bst', '0.bst', '2.bst', '3.bst']),
-
- # A more complicated test with build of build dependencies
- ('target.bst', {
- 'a.bst': [],
- 'base.bst': [],
- 'timezones.bst': [],
- 'middleware.bst': [{'filename': 'base.bst', 'type': 'build'}],
- 'app.bst': [{'filename': 'middleware.bst', 'type': 'build'}],
- 'target.bst': ['a.bst', 'base.bst', 'middleware.bst', 'app.bst', 'timezones.bst']
- }, ['base.bst', 'middleware.bst', 'a.bst', 'app.bst', 'timezones.bst', 'target.bst']),
-])
-@pytest.mark.parametrize("operation", [('show'), ('fetch'), ('build')])
+@pytest.mark.parametrize(
+ "target,template,expected",
+ [
+ # First simple test
+ (
+ "3.bst",
+ {"0.bst": ["1.bst"], "1.bst": [], "2.bst": ["0.bst"], "3.bst": ["0.bst", "1.bst", "2.bst"]},
+ ["1.bst", "0.bst", "2.bst", "3.bst"],
+ ),
+ # A more complicated test with build of build dependencies
+ (
+ "target.bst",
+ {
+ "a.bst": [],
+ "base.bst": [],
+ "timezones.bst": [],
+ "middleware.bst": [{"filename": "base.bst", "type": "build"}],
+ "app.bst": [{"filename": "middleware.bst", "type": "build"}],
+ "target.bst": ["a.bst", "base.bst", "middleware.bst", "app.bst", "timezones.bst"],
+ },
+ ["base.bst", "middleware.bst", "a.bst", "app.bst", "timezones.bst", "target.bst"],
+ ),
+ ],
+)
+@pytest.mark.parametrize("operation", [("show"), ("fetch"), ("build")])
def test_order(cli, datafiles, operation, target, template, expected):
project = str(datafiles)
# Configure to only allow one fetcher at a time, make it easy to
# determine what is being planned in what order.
- cli.configure({
- 'scheduler': {
- 'fetchers': 1,
- 'builders': 1
- }
- })
+ cli.configure({"scheduler": {"fetchers": 1, "builders": 1}})
# Build the project from the template, make import elements
# all with the same repo
@@ -95,13 +86,13 @@ def test_order(cli, datafiles, operation, target, template, expected):
create_element(project, element, dependencies)
# Run test and collect results
- if operation == 'show':
- result = cli.run(args=['show', '--deps', 'plan', '--format', '%{name}', target], project=project, silent=True)
+ if operation == "show":
+ result = cli.run(args=["show", "--deps", "plan", "--format", "%{name}", target], project=project, silent=True)
result.assert_success()
results = result.output.splitlines()
else:
- if operation == 'fetch':
- result = cli.run(args=['source', 'fetch', target], project=project, silent=True)
+ if operation == "fetch":
+ result = cli.run(args=["source", "fetch", target], project=project, silent=True)
else:
result = cli.run(args=[operation, target], project=project, silent=True)
result.assert_success()
diff --git a/tests/frontend/overlaps.py b/tests/frontend/overlaps.py
index eb2cd4a86..e1c5a7c94 100644
--- a/tests/frontend/overlaps.py
+++ b/tests/frontend/overlaps.py
@@ -10,16 +10,11 @@ from buildstream.plugin import CoreWarnings
from tests.testutils import generate_junction
# Project directory
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "overlaps"
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "overlaps")
def gen_project(project_dir, fail_on_overlap, use_fatal_warnings=True, project_name="test"):
- template = {
- "name": project_name
- }
+ template = {"name": project_name}
if use_fatal_warnings:
template["fatal-warnings"] = [CoreWarnings.OVERLAPS] if fail_on_overlap else []
else:
@@ -33,8 +28,7 @@ def gen_project(project_dir, fail_on_overlap, use_fatal_warnings=True, project_n
def test_overlaps(cli, datafiles, use_fatal_warnings):
project_dir = str(datafiles)
gen_project(project_dir, False, use_fatal_warnings)
- result = cli.run(project=project_dir, silent=True, args=[
- 'build', 'collect.bst'])
+ result = cli.run(project=project_dir, silent=True, args=["build", "collect.bst"])
result.assert_success()
@@ -43,8 +37,7 @@ def test_overlaps(cli, datafiles, use_fatal_warnings):
def test_overlaps_error(cli, datafiles, use_fatal_warnings):
project_dir = str(datafiles)
gen_project(project_dir, True, use_fatal_warnings)
- result = cli.run(project=project_dir, silent=True, args=[
- 'build', 'collect.bst'])
+ result = cli.run(project=project_dir, silent=True, args=["build", "collect.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.OVERLAPS)
@@ -53,8 +46,7 @@ def test_overlaps_error(cli, datafiles, use_fatal_warnings):
def test_overlaps_whitelist(cli, datafiles):
project_dir = str(datafiles)
gen_project(project_dir, True)
- result = cli.run(project=project_dir, silent=True, args=[
- 'build', 'collect-whitelisted.bst'])
+ result = cli.run(project=project_dir, silent=True, args=["build", "collect-whitelisted.bst"])
result.assert_success()
@@ -62,8 +54,7 @@ def test_overlaps_whitelist(cli, datafiles):
def test_overlaps_whitelist_ignored(cli, datafiles):
project_dir = str(datafiles)
gen_project(project_dir, False)
- result = cli.run(project=project_dir, silent=True, args=[
- 'build', 'collect-whitelisted.bst'])
+ result = cli.run(project=project_dir, silent=True, args=["build", "collect-whitelisted.bst"])
result.assert_success()
@@ -74,8 +65,7 @@ def test_overlaps_whitelist_on_overlapper(cli, datafiles):
# it'll still fail because A doesn't permit overlaps.
project_dir = str(datafiles)
gen_project(project_dir, True)
- result = cli.run(project=project_dir, silent=True, args=[
- 'build', 'collect-partially-whitelisted.bst'])
+ result = cli.run(project=project_dir, silent=True, args=["build", "collect-partially-whitelisted.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.OVERLAPS)
@@ -87,21 +77,20 @@ def test_overlaps_script(cli, datafiles, use_fatal_warnings):
# Element.stage_dependency_artifacts() with Scope.RUN
project_dir = str(datafiles)
gen_project(project_dir, False, use_fatal_warnings)
- result = cli.run(project=project_dir, silent=True, args=[
- 'build', 'script.bst'])
+ result = cli.run(project=project_dir, silent=True, args=["build", "script.bst"])
result.assert_success()
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("project_policy", [('fail'), ('warn')])
-@pytest.mark.parametrize("subproject_policy", [('fail'), ('warn')])
+@pytest.mark.parametrize("project_policy", [("fail"), ("warn")])
+@pytest.mark.parametrize("subproject_policy", [("fail"), ("warn")])
def test_overlap_subproject(cli, tmpdir, datafiles, project_policy, subproject_policy):
project_dir = str(datafiles)
- subproject_dir = os.path.join(project_dir, 'sub-project')
- junction_path = os.path.join(project_dir, 'sub-project.bst')
+ subproject_dir = os.path.join(project_dir, "sub-project")
+ junction_path = os.path.join(project_dir, "sub-project.bst")
- gen_project(project_dir, bool(project_policy == 'fail'), project_name='test')
- gen_project(subproject_dir, bool(subproject_policy == 'fail'), project_name='subtest')
+ gen_project(project_dir, bool(project_policy == "fail"), project_name="test")
+ gen_project(subproject_dir, bool(subproject_policy == "fail"), project_name="subtest")
generate_junction(tmpdir, subproject_dir, junction_path)
# Here we have a dependency chain where the project element
@@ -110,8 +99,8 @@ def test_overlap_subproject(cli, tmpdir, datafiles, project_policy, subproject_p
# Test that overlap error vs warning policy for this overlap
# is always controlled by the project and not the subproject.
#
- result = cli.run(project=project_dir, silent=True, args=['build', 'sub-collect.bst'])
- if project_policy == 'fail':
+ result = cli.run(project=project_dir, silent=True, args=["build", "sub-collect.bst"])
+ if project_policy == "fail":
result.assert_main_error(ErrorDomain.STREAM, None)
result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.OVERLAPS)
else:
diff --git a/tests/frontend/progress.py b/tests/frontend/progress.py
index e3b127f3b..45cba0b50 100644
--- a/tests/frontend/progress.py
+++ b/tests/frontend/progress.py
@@ -11,125 +11,97 @@ from buildstream._exceptions import ErrorDomain, LoadErrorReason
from tests.testutils import generate_junction
# Project directory
-DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), )
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)),)
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "project"))
def test_show_progress_tally(cli, datafiles):
# Check that the progress reporting messages give correct tallies
project = str(datafiles)
- result = cli.run(project=project, args=['show', 'compose-all.bst'])
+ result = cli.run(project=project, args=["show", "compose-all.bst"])
result.assert_success()
assert " 3 subtasks processed" in result.stderr
assert "3 of 3 subtasks processed" in result.stderr
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "project"))
def test_junction_tally(cli, tmpdir, datafiles):
# Check that the progress reporting messages count elements in junctions
project = str(datafiles)
- subproject_path = os.path.join(project, 'files', 'sub-project')
- junction_path = os.path.join(project, 'elements', 'junction.bst')
- element_path = os.path.join(project, 'elements', 'junction-dep.bst')
+ subproject_path = os.path.join(project, "files", "sub-project")
+ junction_path = os.path.join(project, "elements", "junction.bst")
+ element_path = os.path.join(project, "elements", "junction-dep.bst")
# Create a repo to hold the subproject and generate a junction element for it
generate_junction(tmpdir, subproject_path, junction_path, store_ref=True)
# Create a stack element to depend on a cross junction element
#
- element = {
- 'kind': 'stack',
- 'depends': [{
- 'junction': 'junction.bst',
- 'filename': 'import-etc.bst'
- }]
- }
+ element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]}
_yaml.roundtrip_dump(element, element_path)
- result = cli.run(project=project,
- silent=True,
- args=['source', 'fetch', 'junction.bst'])
+ result = cli.run(project=project, silent=True, args=["source", "fetch", "junction.bst"])
result.assert_success()
# Assert the correct progress tallies are in the logging
- result = cli.run(project=project, args=['show', 'junction-dep.bst'])
+ result = cli.run(project=project, args=["show", "junction-dep.bst"])
assert " 2 subtasks processed" in result.stderr
assert "2 of 2 subtasks processed" in result.stderr
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "project"))
def test_nested_junction_tally(cli, tmpdir, datafiles):
# Check that the progress reporting messages count elements in
# junctions of junctions
project = str(datafiles)
- sub1_path = os.path.join(project, 'files', 'sub-project')
- sub2_path = os.path.join(project, 'files', 'sub2-project')
+ sub1_path = os.path.join(project, "files", "sub-project")
+ sub2_path = os.path.join(project, "files", "sub2-project")
# A junction element which pulls sub1 into sub2
- sub1_element = os.path.join(project, 'files', 'sub2-project', 'elements', 'sub-junction.bst')
+ sub1_element = os.path.join(project, "files", "sub2-project", "elements", "sub-junction.bst")
# A junction element which pulls sub2 into the main project
- sub2_element = os.path.join(project, 'elements', 'junction.bst')
- element_path = os.path.join(project, 'elements', 'junction-dep.bst')
+ sub2_element = os.path.join(project, "elements", "junction.bst")
+ element_path = os.path.join(project, "elements", "junction-dep.bst")
generate_junction(tmpdir / "sub-project", sub1_path, sub1_element, store_ref=True)
generate_junction(tmpdir / "sub2-project", sub2_path, sub2_element, store_ref=True)
# Create a stack element to depend on a cross junction element
#
- element = {
- 'kind': 'stack',
- 'depends': [{
- 'junction': 'junction.bst',
- 'filename': 'import-sub.bst'
- }]
- }
+ element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-sub.bst"}]}
_yaml.roundtrip_dump(element, element_path)
- result = cli.run(project=project,
- silent=True,
- args=['source', 'fetch', 'junction.bst'])
+ result = cli.run(project=project, silent=True, args=["source", "fetch", "junction.bst"])
result.assert_success()
# Assert the correct progress tallies are in the logging
- result = cli.run(project=project, args=['show', 'junction-dep.bst'])
+ result = cli.run(project=project, args=["show", "junction-dep.bst"])
assert " 3 subtasks processed" in result.stderr
assert "3 of 3 subtasks processed" in result.stderr
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "project"))
def test_junction_dep_tally(cli, tmpdir, datafiles):
# Check that the progress reporting messages count elements in junctions
project = str(datafiles)
- subproject_path = os.path.join(project, 'files', 'sub-project')
- junction_path = os.path.join(project, 'elements', 'junction.bst')
- element_path = os.path.join(project, 'elements', 'junction-dep.bst')
+ subproject_path = os.path.join(project, "files", "sub-project")
+ junction_path = os.path.join(project, "elements", "junction.bst")
+ element_path = os.path.join(project, "elements", "junction-dep.bst")
# Create a repo to hold the subproject and generate a junction element for it
generate_junction(tmpdir, subproject_path, junction_path, store_ref=True)
# Add dependencies to the junction (not allowed, but let's do it
# anyway)
- with open(junction_path, 'a') as f:
- deps = {
- 'depends': [
- 'manual.bst'
- ]
- }
+ with open(junction_path, "a") as f:
+ deps = {"depends": ["manual.bst"]}
_yaml.roundtrip_dump(deps, f)
# Create a stack element to depend on a cross junction element
#
- element = {
- 'kind': 'stack',
- 'depends': [{
- 'junction': 'junction.bst',
- 'filename': 'import-etc.bst'
- }]
- }
+ element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]}
_yaml.roundtrip_dump(element, element_path)
- result = cli.run(project=project,
- silent=True,
- args=['source', 'fetch', 'junction-dep.bst'])
+ result = cli.run(project=project, silent=True, args=["source", "fetch", "junction-dep.bst"])
# Since we aren't allowed to specify any dependencies on a
# junction, we should fail
diff --git a/tests/frontend/project/sources/fetch_source.py b/tests/frontend/project/sources/fetch_source.py
index ac3020ec2..51bfe1049 100644
--- a/tests/frontend/project/sources/fetch_source.py
+++ b/tests/frontend/project/sources/fetch_source.py
@@ -22,14 +22,10 @@ class FetchFetcher(SourceFetcher):
self.mark_download_url(url)
def fetch(self, alias_override=None):
- url = self.source.translate_url(self.original_url,
- alias_override=alias_override,
- primary=self.primary)
+ url = self.source.translate_url(self.original_url, alias_override=alias_override, primary=self.primary)
with open(self.source.output_file, "a") as f:
success = url in self.source.fetch_succeeds and self.source.fetch_succeeds[url]
- message = "Fetch {} {} from {}\n".format(self.original_url,
- "succeeded" if success else "failed",
- url)
+ message = "Fetch {} {} from {}\n".format(self.original_url, "succeeded" if success else "failed", url)
f.write(message)
if not success:
raise SourceError("Failed to fetch {}".format(url))
@@ -38,12 +34,9 @@ class FetchFetcher(SourceFetcher):
class FetchSource(Source):
# Read config to know which URLs to fetch
def configure(self, node):
- self.original_urls = node.get_str_list('urls')
- self.output_file = node.get_str('output-text')
- self.fetch_succeeds = {
- key: value.as_bool()
- for key, value in node.get_mapping('fetch-succeeds', {}).items()
- }
+ self.original_urls = node.get_str_list("urls")
+ self.output_file = node.get_str("output-text")
+ self.fetch_succeeds = {key: value.as_bool() for key, value in node.get_mapping("fetch-succeeds", {}).items()}
# First URL is the primary one for this test
#
diff --git a/tests/frontend/pull.py b/tests/frontend/pull.py
index 234f1133d..3e726ffcb 100644
--- a/tests/frontend/pull.py
+++ b/tests/frontend/pull.py
@@ -12,10 +12,7 @@ from tests.testutils import create_artifact_share, generate_junction, assert_sha
# Project directory
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project",
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",)
# Tests that:
@@ -27,40 +24,38 @@ DATA_DIR = os.path.join(
def test_push_pull_all(cli, tmpdir, datafiles):
project = str(datafiles)
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
# First build the target element and push to the remote.
- cli.configure({
- 'artifacts': {'url': share.repo, 'push': True}
- })
- result = cli.run(project=project, args=['build', 'target.bst'])
+ cli.configure({"artifacts": {"url": share.repo, "push": True}})
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- assert cli.get_element_state(project, 'target.bst') == 'cached'
+ assert cli.get_element_state(project, "target.bst") == "cached"
# Assert that everything is now cached in the remote.
- all_elements = ['target.bst', 'import-bin.bst', 'import-dev.bst', 'compose-all.bst']
+ all_elements = ["target.bst", "import-bin.bst", "import-dev.bst", "compose-all.bst"]
for element_name in all_elements:
assert_shared(cli, share, project, element_name)
# Now we've pushed, delete the user's local artifact cache
# directory and try to redownload it from the share
#
- casdir = os.path.join(cli.directory, 'cas')
+ casdir = os.path.join(cli.directory, "cas")
shutil.rmtree(casdir)
- artifactdir = os.path.join(cli.directory, 'artifacts')
+ artifactdir = os.path.join(cli.directory, "artifacts")
shutil.rmtree(artifactdir)
# Assert that nothing is cached locally anymore
states = cli.get_element_states(project, all_elements)
- assert not any(states[e] == 'cached' for e in all_elements)
+ assert not any(states[e] == "cached" for e in all_elements)
# Now try bst artifact pull
- result = cli.run(project=project, args=['artifact', 'pull', '--deps', 'all', 'target.bst'])
+ result = cli.run(project=project, args=["artifact", "pull", "--deps", "all", "target.bst"])
result.assert_success()
# And assert that it's again in the local cache, without having built
states = cli.get_element_states(project, all_elements)
- assert not any(states[e] != 'cached' for e in all_elements)
+ assert not any(states[e] != "cached" for e in all_elements)
# Tests that:
@@ -68,51 +63,47 @@ def test_push_pull_all(cli, tmpdir, datafiles):
# * `bst artifact push` (default targets) pushes all built elements to configured 'push' cache
# * `bst artifact pull` (default targets) downloads everything from cache after local deletion
#
-@pytest.mark.datafiles(DATA_DIR + '_world')
+@pytest.mark.datafiles(DATA_DIR + "_world")
def test_push_pull_default_targets(cli, tmpdir, datafiles):
project = str(datafiles)
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
# First build the target elements
- cli.configure({
- 'artifacts': {'url': share.repo}
- })
- result = cli.run(project=project, args=['build'])
+ cli.configure({"artifacts": {"url": share.repo}})
+ result = cli.run(project=project, args=["build"])
result.assert_success()
- assert cli.get_element_state(project, 'target.bst') == 'cached'
+ assert cli.get_element_state(project, "target.bst") == "cached"
# Push all elements
- cli.configure({
- 'artifacts': {'url': share.repo, 'push': True}
- })
- result = cli.run(project=project, args=['artifact', 'push'])
+ cli.configure({"artifacts": {"url": share.repo, "push": True}})
+ result = cli.run(project=project, args=["artifact", "push"])
result.assert_success()
# Assert that everything is now cached in the remote.
- all_elements = ['target.bst', 'import-bin.bst', 'import-dev.bst', 'compose-all.bst']
+ all_elements = ["target.bst", "import-bin.bst", "import-dev.bst", "compose-all.bst"]
for element_name in all_elements:
assert_shared(cli, share, project, element_name)
# Now we've pushed, delete the user's local artifact cache
# directory and try to redownload it from the share
#
- casdir = os.path.join(cli.directory, 'cas')
+ casdir = os.path.join(cli.directory, "cas")
shutil.rmtree(casdir)
- artifactdir = os.path.join(cli.directory, 'artifacts')
+ artifactdir = os.path.join(cli.directory, "artifacts")
shutil.rmtree(artifactdir)
# Assert that nothing is cached locally anymore
states = cli.get_element_states(project, all_elements)
- assert not any(states[e] == 'cached' for e in all_elements)
+ assert not any(states[e] == "cached" for e in all_elements)
# Now try bst artifact pull
- result = cli.run(project=project, args=['artifact', 'pull'])
+ result = cli.run(project=project, args=["artifact", "pull"])
result.assert_success()
# And assert that it's again in the local cache, without having built
states = cli.get_element_states(project, all_elements)
- assert not any(states[e] != 'cached' for e in all_elements)
+ assert not any(states[e] != "cached" for e in all_elements)
# Tests that:
@@ -124,38 +115,34 @@ def test_push_pull_default_targets(cli, tmpdir, datafiles):
def test_pull_secondary_cache(cli, tmpdir, datafiles):
project = str(datafiles)
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare1')) as share1,\
- create_artifact_share(os.path.join(str(tmpdir), 'artifactshare2')) as share2:
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare1")) as share1, create_artifact_share(
+ os.path.join(str(tmpdir), "artifactshare2")
+ ) as share2:
# Build the target and push it to share2 only.
- cli.configure({
- 'artifacts': [
- {'url': share1.repo, 'push': False},
- {'url': share2.repo, 'push': True},
- ]
- })
- result = cli.run(project=project, args=['build', 'target.bst'])
+ cli.configure({"artifacts": [{"url": share1.repo, "push": False}, {"url": share2.repo, "push": True},]})
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- assert_not_shared(cli, share1, project, 'target.bst')
- assert_shared(cli, share2, project, 'target.bst')
+ assert_not_shared(cli, share1, project, "target.bst")
+ assert_shared(cli, share2, project, "target.bst")
# Delete the user's local artifact cache.
- casdir = os.path.join(cli.directory, 'cas')
+ casdir = os.path.join(cli.directory, "cas")
shutil.rmtree(casdir)
- artifactdir = os.path.join(cli.directory, 'artifacts')
+ artifactdir = os.path.join(cli.directory, "artifacts")
shutil.rmtree(artifactdir)
# Assert that the element is not cached anymore.
- assert cli.get_element_state(project, 'target.bst') != 'cached'
+ assert cli.get_element_state(project, "target.bst") != "cached"
# Now try bst artifact pull
- result = cli.run(project=project, args=['artifact', 'pull', 'target.bst'])
+ result = cli.run(project=project, args=["artifact", "pull", "target.bst"])
result.assert_success()
# And assert that it's again in the local cache, without having built,
# i.e. we found it in share2.
- assert cli.get_element_state(project, 'target.bst') == 'cached'
+ assert cli.get_element_state(project, "target.bst") == "cached"
# Tests that:
@@ -167,47 +154,45 @@ def test_pull_secondary_cache(cli, tmpdir, datafiles):
def test_push_pull_specific_remote(cli, tmpdir, datafiles):
project = str(datafiles)
- with create_artifact_share(os.path.join(str(tmpdir), 'goodartifactshare')) as good_share,\
- create_artifact_share(os.path.join(str(tmpdir), 'badartifactshare')) as bad_share:
+ with create_artifact_share(os.path.join(str(tmpdir), "goodartifactshare")) as good_share, create_artifact_share(
+ os.path.join(str(tmpdir), "badartifactshare")
+ ) as bad_share:
# Build the target so we have it cached locally only.
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- state = cli.get_element_state(project, 'target.bst')
- assert state == 'cached'
+ state = cli.get_element_state(project, "target.bst")
+ assert state == "cached"
# Configure the default push location to be bad_share; we will assert that
# nothing actually gets pushed there.
- cli.configure({
- 'artifacts': {'url': bad_share.repo, 'push': True},
- })
+ cli.configure(
+ {"artifacts": {"url": bad_share.repo, "push": True},}
+ )
# Now try `bst artifact push` to the good_share.
- result = cli.run(project=project, args=[
- 'artifact', 'push', 'target.bst', '--remote', good_share.repo
- ])
+ result = cli.run(project=project, args=["artifact", "push", "target.bst", "--remote", good_share.repo])
result.assert_success()
# Assert that all the artifacts are in the share we pushed
# to, and not the other.
- assert_shared(cli, good_share, project, 'target.bst')
- assert_not_shared(cli, bad_share, project, 'target.bst')
+ assert_shared(cli, good_share, project, "target.bst")
+ assert_not_shared(cli, bad_share, project, "target.bst")
# Now we've pushed, delete the user's local artifact cache
# directory and try to redownload it from the good_share.
#
- casdir = os.path.join(cli.directory, 'cas')
+ casdir = os.path.join(cli.directory, "cas")
shutil.rmtree(casdir)
- artifactdir = os.path.join(cli.directory, 'artifacts')
+ artifactdir = os.path.join(cli.directory, "artifacts")
shutil.rmtree(artifactdir)
- result = cli.run(project=project, args=['artifact', 'pull', 'target.bst', '--remote',
- good_share.repo])
+ result = cli.run(project=project, args=["artifact", "pull", "target.bst", "--remote", good_share.repo])
result.assert_success()
# And assert that it's again in the local cache, without having built
- assert cli.get_element_state(project, 'target.bst') == 'cached'
+ assert cli.get_element_state(project, "target.bst") == "cached"
# Tests that:
@@ -218,123 +203,114 @@ def test_push_pull_specific_remote(cli, tmpdir, datafiles):
def test_push_pull_non_strict(cli, tmpdir, datafiles):
project = str(datafiles)
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
# First build the target element and push to the remote.
- cli.configure({
- 'artifacts': {'url': share.repo, 'push': True},
- 'projects': {
- 'test': {'strict': False}
- }
- })
- result = cli.run(project=project, args=['build', 'target.bst'])
+ cli.configure({"artifacts": {"url": share.repo, "push": True}, "projects": {"test": {"strict": False}}})
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- assert cli.get_element_state(project, 'target.bst') == 'cached'
+ assert cli.get_element_state(project, "target.bst") == "cached"
# Assert that everything is now cached in the remote.
- all_elements = ['target.bst', 'import-bin.bst', 'import-dev.bst', 'compose-all.bst']
+ all_elements = ["target.bst", "import-bin.bst", "import-dev.bst", "compose-all.bst"]
for element_name in all_elements:
assert_shared(cli, share, project, element_name)
# Now we've pushed, delete the user's local artifact cache
# directory and try to redownload it from the share
#
- casdir = os.path.join(cli.directory, 'cas')
+ casdir = os.path.join(cli.directory, "cas")
shutil.rmtree(casdir)
- artifactdir = os.path.join(cli.directory, 'artifacts')
+ artifactdir = os.path.join(cli.directory, "artifacts")
shutil.rmtree(artifactdir)
# Assert that nothing is cached locally anymore
for element_name in all_elements:
- assert cli.get_element_state(project, element_name) != 'cached'
+ assert cli.get_element_state(project, element_name) != "cached"
# Add a file to force change in strict cache key of import-bin.bst
- with open(os.path.join(str(project), 'files', 'bin-files', 'usr', 'bin', 'world'), 'w') as f:
- f.write('world')
+ with open(os.path.join(str(project), "files", "bin-files", "usr", "bin", "world"), "w") as f:
+ f.write("world")
# Assert that the workspaced element requires a rebuild
- assert cli.get_element_state(project, 'import-bin.bst') == 'buildable'
+ assert cli.get_element_state(project, "import-bin.bst") == "buildable"
# Assert that the target is still waiting due to --no-strict
- assert cli.get_element_state(project, 'target.bst') == 'waiting'
+ assert cli.get_element_state(project, "target.bst") == "waiting"
# Now try bst artifact pull
- result = cli.run(project=project, args=['artifact', 'pull', '--deps', 'all', 'target.bst'])
+ result = cli.run(project=project, args=["artifact", "pull", "--deps", "all", "target.bst"])
result.assert_success()
# And assert that the target is again in the local cache, without having built
- assert cli.get_element_state(project, 'target.bst') == 'cached'
+ assert cli.get_element_state(project, "target.bst") == "cached"
@pytest.mark.datafiles(DATA_DIR)
def test_push_pull_cross_junction(cli, tmpdir, datafiles):
project = str(datafiles)
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
- subproject_path = os.path.join(project, 'files', 'sub-project')
- junction_path = os.path.join(project, 'elements', 'junction.bst')
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
+ subproject_path = os.path.join(project, "files", "sub-project")
+ junction_path = os.path.join(project, "elements", "junction.bst")
generate_junction(tmpdir, subproject_path, junction_path, store_ref=True)
# First build the target element and push to the remote.
- cli.configure({
- 'artifacts': {'url': share.repo, 'push': True}
- })
- result = cli.run(project=project, args=['build', 'junction.bst:import-etc.bst'])
+ cli.configure({"artifacts": {"url": share.repo, "push": True}})
+ result = cli.run(project=project, args=["build", "junction.bst:import-etc.bst"])
result.assert_success()
- assert cli.get_element_state(project, 'junction.bst:import-etc.bst') == 'cached'
+ assert cli.get_element_state(project, "junction.bst:import-etc.bst") == "cached"
- cache_dir = os.path.join(project, 'cache', 'cas')
+ cache_dir = os.path.join(project, "cache", "cas")
shutil.rmtree(cache_dir)
- artifact_dir = os.path.join(project, 'cache', 'artifacts')
+ artifact_dir = os.path.join(project, "cache", "artifacts")
shutil.rmtree(artifact_dir)
- assert cli.get_element_state(project, 'junction.bst:import-etc.bst') == 'buildable'
+ assert cli.get_element_state(project, "junction.bst:import-etc.bst") == "buildable"
# Now try bst artifact pull
- result = cli.run(project=project, args=['artifact', 'pull', 'junction.bst:import-etc.bst'])
+ result = cli.run(project=project, args=["artifact", "pull", "junction.bst:import-etc.bst"])
result.assert_success()
# And assert that it's again in the local cache, without having built
- assert cli.get_element_state(project, 'junction.bst:import-etc.bst') == 'cached'
+ assert cli.get_element_state(project, "junction.bst:import-etc.bst") == "cached"
@pytest.mark.datafiles(DATA_DIR)
def test_pull_missing_blob(cli, tmpdir, datafiles):
project = str(datafiles)
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
# First build the target element and push to the remote.
- cli.configure({
- 'artifacts': {'url': share.repo, 'push': True}
- })
- result = cli.run(project=project, args=['build', 'target.bst'])
+ cli.configure({"artifacts": {"url": share.repo, "push": True}})
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- assert cli.get_element_state(project, 'target.bst') == 'cached'
+ assert cli.get_element_state(project, "target.bst") == "cached"
# Assert that everything is now cached in the remote.
- all_elements = ['target.bst', 'import-bin.bst', 'import-dev.bst', 'compose-all.bst']
+ all_elements = ["target.bst", "import-bin.bst", "import-dev.bst", "compose-all.bst"]
for element_name in all_elements:
assert_shared(cli, share, project, element_name)
# Now we've pushed, delete the user's local artifact cache
# directory and try to redownload it from the share
#
- casdir = os.path.join(cli.directory, 'cas')
+ casdir = os.path.join(cli.directory, "cas")
shutil.rmtree(casdir)
- artifactdir = os.path.join(cli.directory, 'artifacts')
+ artifactdir = os.path.join(cli.directory, "artifacts")
shutil.rmtree(artifactdir)
# Assert that nothing is cached locally anymore
for element_name in all_elements:
- assert cli.get_element_state(project, element_name) != 'cached'
+ assert cli.get_element_state(project, element_name) != "cached"
# Now delete blobs in the remote without deleting the artifact ref.
# This simulates scenarios with concurrent artifact expiry.
- remote_objdir = os.path.join(share.repodir, 'cas', 'objects')
+ remote_objdir = os.path.join(share.repodir, "cas", "objects")
shutil.rmtree(remote_objdir)
# Now try bst build
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
# Assert that no artifacts were pulled
@@ -344,9 +320,9 @@ def test_pull_missing_blob(cli, tmpdir, datafiles):
@pytest.mark.datafiles(DATA_DIR)
def test_pull_missing_local_blob(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
- repo = create_repo('git', str(tmpdir))
+ repo = create_repo("git", str(tmpdir))
repo.create(os.path.join(str(datafiles), "files"))
- element_dir = os.path.join(str(tmpdir), 'elements')
+ element_dir = os.path.join(str(tmpdir), "elements")
project = str(tmpdir)
project_config = {
"name": "pull-missing-local-blob",
@@ -358,43 +334,36 @@ def test_pull_missing_local_blob(cli, tmpdir, datafiles):
"kind": "import",
"sources": [repo.source_config()],
}
- input_name = 'input.bst'
+ input_name = "input.bst"
input_file = os.path.join(element_dir, input_name)
_yaml.roundtrip_dump(input_config, input_file)
- depends_name = 'depends.bst'
- depends_config = {
- "kind": "stack",
- "depends": [
- {"filename": input_name, "type": "build"}
- ]
- }
+ depends_name = "depends.bst"
+ depends_config = {"kind": "stack", "depends": [{"filename": input_name, "type": "build"}]}
depends_file = os.path.join(element_dir, depends_name)
_yaml.roundtrip_dump(depends_config, depends_file)
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
# First build the import-bin element and push to the remote.
- cli.configure({
- 'artifacts': {'url': share.repo, 'push': True}
- })
+ cli.configure({"artifacts": {"url": share.repo, "push": True}})
- result = cli.run(project=project, args=['source', 'track', input_name])
+ result = cli.run(project=project, args=["source", "track", input_name])
result.assert_success()
- result = cli.run(project=project, args=['build', input_name])
+ result = cli.run(project=project, args=["build", input_name])
result.assert_success()
- assert cli.get_element_state(project, input_name) == 'cached'
+ assert cli.get_element_state(project, input_name) == "cached"
# Delete a file blob from the local cache.
# This is a placeholder to test partial CAS handling until we support
# partial artifact pulling (or blob-based CAS expiry).
#
- digest = utils.sha256sum(os.path.join(project, 'files', 'bin-files', 'usr', 'bin', 'hello'))
- objpath = os.path.join(cli.directory, 'cas', 'objects', digest[:2], digest[2:])
+ digest = utils.sha256sum(os.path.join(project, "files", "bin-files", "usr", "bin", "hello"))
+ objpath = os.path.join(cli.directory, "cas", "objects", digest[:2], digest[2:])
os.unlink(objpath)
# Now try bst build
- result = cli.run(project=project, args=['build', depends_name])
+ result = cli.run(project=project, args=["build", depends_name])
result.assert_success()
# Assert that the import-bin artifact was pulled (completing the partial artifact)
@@ -406,16 +375,13 @@ def test_pull_missing_notifies_user(caplog, cli, tmpdir, datafiles):
project = str(datafiles)
caplog.set_level(1)
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
- cli.configure({
- 'artifacts': {'url': share.repo}
- })
- result = cli.run(project=project, args=['build', 'target.bst'])
+ cli.configure({"artifacts": {"url": share.repo}})
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- assert not result.get_pulled_elements(), \
- "No elements should have been pulled since the cache was empty"
+ assert not result.get_pulled_elements(), "No elements should have been pulled since the cache was empty"
assert "INFO Remote ({}) does not have".format(share.repo) in result.stderr
assert "SKIPPED Pull" in result.stderr
@@ -426,25 +392,23 @@ def test_build_remote_option(caplog, cli, tmpdir, datafiles):
project = str(datafiles)
caplog.set_level(1)
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare1')) as shareuser,\
- create_artifact_share(os.path.join(str(tmpdir), 'artifactshare2')) as shareproject,\
- create_artifact_share(os.path.join(str(tmpdir), 'artifactshare3')) as sharecli:
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare1")) as shareuser, create_artifact_share(
+ os.path.join(str(tmpdir), "artifactshare2")
+ ) as shareproject, create_artifact_share(os.path.join(str(tmpdir), "artifactshare3")) as sharecli:
# Add shareproject repo url to project.conf
with open(os.path.join(project, "project.conf"), "a") as projconf:
projconf.write("artifacts:\n url: {}\n push: True".format(shareproject.repo))
# Configure shareuser remote in user conf
- cli.configure({
- 'artifacts': {'url': shareuser.repo, 'push': True}
- })
+ cli.configure({"artifacts": {"url": shareuser.repo, "push": True}})
# Push the artifacts to the shareuser and shareproject remotes.
# Assert that shareuser and shareproject have the artfifacts cached,
# but sharecli doesn't, then delete locally cached elements
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- all_elements = ['target.bst', 'import-bin.bst', 'compose-all.bst']
+ all_elements = ["target.bst", "import-bin.bst", "compose-all.bst"]
for element_name in all_elements:
assert element_name in result.get_pushed_elements()
assert_not_shared(cli, sharecli, project, element_name)
@@ -455,7 +419,7 @@ def test_build_remote_option(caplog, cli, tmpdir, datafiles):
# Now check that a build with cli set as sharecli results in nothing being pulled,
# as it doesn't have them cached and shareuser/shareproject should be ignored. This
# will however result in the artifacts being built and pushed to it
- result = cli.run(project=project, args=['build', '--remote', sharecli.repo, 'target.bst'])
+ result = cli.run(project=project, args=["build", "--remote", sharecli.repo, "target.bst"])
result.assert_success()
for element_name in all_elements:
assert element_name not in result.get_pulled_elements()
@@ -464,10 +428,10 @@ def test_build_remote_option(caplog, cli, tmpdir, datafiles):
# Now check that a clean build with cli set as sharecli should result in artifacts only
# being pulled from it, as that was provided via the cli and is populated
- result = cli.run(project=project, args=['build', '--remote', sharecli.repo, 'target.bst'])
+ result = cli.run(project=project, args=["build", "--remote", sharecli.repo, "target.bst"])
result.assert_success()
for element_name in all_elements:
- assert cli.get_element_state(project, element_name) == 'cached'
+ assert cli.get_element_state(project, element_name) == "cached"
assert element_name in result.get_pulled_elements()
assert shareproject.repo not in result.stderr
assert shareuser.repo not in result.stderr
@@ -477,70 +441,66 @@ def test_build_remote_option(caplog, cli, tmpdir, datafiles):
@pytest.mark.datafiles(DATA_DIR)
def test_pull_access_rights(cli, tmpdir, datafiles):
project = str(datafiles)
- checkout = os.path.join(str(tmpdir), 'checkout')
+ checkout = os.path.join(str(tmpdir), "checkout")
# Work-around datafiles not preserving mode
- os.chmod(os.path.join(project, 'files/bin-files/usr/bin/hello'), 0o0755)
+ os.chmod(os.path.join(project, "files/bin-files/usr/bin/hello"), 0o0755)
# We need a big file that does not go into a batch to test a different
# code path
- os.makedirs(os.path.join(project, 'files/dev-files/usr/share'), exist_ok=True)
- with open(os.path.join(project, 'files/dev-files/usr/share/big-file'), 'w') as f:
- buf = ' ' * 4096
+ os.makedirs(os.path.join(project, "files/dev-files/usr/share"), exist_ok=True)
+ with open(os.path.join(project, "files/dev-files/usr/share/big-file"), "w") as f:
+ buf = " " * 4096
for _ in range(1024):
f.write(buf)
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
- cli.configure({
- 'artifacts': {'url': share.repo, 'push': True}
- })
- result = cli.run(project=project, args=['build', 'compose-all.bst'])
+ cli.configure({"artifacts": {"url": share.repo, "push": True}})
+ result = cli.run(project=project, args=["build", "compose-all.bst"])
result.assert_success()
- result = cli.run(project=project,
- args=['artifact', 'checkout',
- '--hardlinks', '--no-integrate',
- 'compose-all.bst',
- '--directory', checkout])
+ result = cli.run(
+ project=project,
+ args=["artifact", "checkout", "--hardlinks", "--no-integrate", "compose-all.bst", "--directory", checkout],
+ )
result.assert_success()
- st = os.lstat(os.path.join(checkout, 'usr/include/pony.h'))
+ st = os.lstat(os.path.join(checkout, "usr/include/pony.h"))
assert stat.S_ISREG(st.st_mode)
assert stat.S_IMODE(st.st_mode) == 0o0644
- st = os.lstat(os.path.join(checkout, 'usr/bin/hello'))
+ st = os.lstat(os.path.join(checkout, "usr/bin/hello"))
assert stat.S_ISREG(st.st_mode)
assert stat.S_IMODE(st.st_mode) == 0o0755
- st = os.lstat(os.path.join(checkout, 'usr/share/big-file'))
+ st = os.lstat(os.path.join(checkout, "usr/share/big-file"))
assert stat.S_ISREG(st.st_mode)
assert stat.S_IMODE(st.st_mode) == 0o0644
shutil.rmtree(checkout)
- casdir = os.path.join(cli.directory, 'cas')
+ casdir = os.path.join(cli.directory, "cas")
shutil.rmtree(casdir)
- result = cli.run(project=project, args=['artifact', 'pull', 'compose-all.bst'])
+ result = cli.run(project=project, args=["artifact", "pull", "compose-all.bst"])
result.assert_success()
- result = cli.run(project=project,
- args=['artifact', 'checkout',
- '--hardlinks', '--no-integrate',
- 'compose-all.bst',
- '--directory', checkout])
+ result = cli.run(
+ project=project,
+ args=["artifact", "checkout", "--hardlinks", "--no-integrate", "compose-all.bst", "--directory", checkout],
+ )
result.assert_success()
- st = os.lstat(os.path.join(checkout, 'usr/include/pony.h'))
+ st = os.lstat(os.path.join(checkout, "usr/include/pony.h"))
assert stat.S_ISREG(st.st_mode)
assert stat.S_IMODE(st.st_mode) == 0o0644
- st = os.lstat(os.path.join(checkout, 'usr/bin/hello'))
+ st = os.lstat(os.path.join(checkout, "usr/bin/hello"))
assert stat.S_ISREG(st.st_mode)
assert stat.S_IMODE(st.st_mode) == 0o0755
- st = os.lstat(os.path.join(checkout, 'usr/share/big-file'))
+ st = os.lstat(os.path.join(checkout, "usr/share/big-file"))
assert stat.S_ISREG(st.st_mode)
assert stat.S_IMODE(st.st_mode) == 0o0644
@@ -549,39 +509,37 @@ def test_pull_access_rights(cli, tmpdir, datafiles):
@pytest.mark.datafiles(DATA_DIR)
def test_pull_artifact(cli, tmpdir, datafiles):
project = str(datafiles)
- element = 'target.bst'
+ element = "target.bst"
# Configure a local cache
- local_cache = os.path.join(str(tmpdir), 'cache')
- cli.configure({'cachedir': local_cache})
+ local_cache = os.path.join(str(tmpdir), "cache")
+ cli.configure({"cachedir": local_cache})
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
# First build the target element and push to the remote.
- cli.configure({
- 'artifacts': {'url': share.repo, 'push': True}
- })
+ cli.configure({"artifacts": {"url": share.repo, "push": True}})
- result = cli.run(project=project, args=['build', element])
+ result = cli.run(project=project, args=["build", element])
result.assert_success()
# Assert that the *artifact* is cached locally
cache_key = cli.get_element_key(project, element)
- artifact_ref = os.path.join('test', os.path.splitext(element)[0], cache_key)
- assert os.path.exists(os.path.join(local_cache, 'artifacts', 'refs', artifact_ref))
+ artifact_ref = os.path.join("test", os.path.splitext(element)[0], cache_key)
+ assert os.path.exists(os.path.join(local_cache, "artifacts", "refs", artifact_ref))
# Assert that the target is shared (note that assert shared will use the artifact name)
assert_shared(cli, share, project, element)
# Now we've pushed, remove the local cache
- shutil.rmtree(os.path.join(local_cache, 'artifacts'))
+ shutil.rmtree(os.path.join(local_cache, "artifacts"))
# Assert that nothing is cached locally anymore
- assert not os.path.exists(os.path.join(local_cache, 'artifacts', 'refs', artifact_ref))
+ assert not os.path.exists(os.path.join(local_cache, "artifacts", "refs", artifact_ref))
# Now try bst artifact pull
- result = cli.run(project=project, args=['artifact', 'pull', artifact_ref])
+ result = cli.run(project=project, args=["artifact", "pull", artifact_ref])
result.assert_success()
# And assert that it's again in the local cache, without having built
- assert os.path.exists(os.path.join(local_cache, 'artifacts', 'refs', artifact_ref))
+ assert os.path.exists(os.path.join(local_cache, "artifacts", "refs", artifact_ref))
diff --git a/tests/frontend/push.py b/tests/frontend/push.py
index 31f96cbdf..583b57399 100644
--- a/tests/frontend/push.py
+++ b/tests/frontend/push.py
@@ -28,15 +28,18 @@ import pytest
from buildstream._exceptions import ErrorDomain
from buildstream.testing import cli # pylint: disable=unused-import
-from tests.testutils import create_artifact_share, create_element_size, generate_junction, \
- wait_for_cache_granularity, assert_shared, assert_not_shared
+from tests.testutils import (
+ create_artifact_share,
+ create_element_size,
+ generate_junction,
+ wait_for_cache_granularity,
+ assert_shared,
+ assert_not_shared,
+)
# Project directory
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project",
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",)
# Tests that:
@@ -49,100 +52,85 @@ def test_push(cli, tmpdir, datafiles):
project = str(datafiles)
# First build the project without the artifact cache configured
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
# Assert that we are now cached locally
- assert cli.get_element_state(project, 'target.bst') == 'cached'
+ assert cli.get_element_state(project, "target.bst") == "cached"
# Set up two artifact shares.
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare1')) as share1:
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare1")) as share1:
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare2')) as share2:
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare2")) as share2:
# Try pushing with no remotes configured. This should fail.
- result = cli.run(project=project, args=['artifact', 'push', 'target.bst'])
+ result = cli.run(project=project, args=["artifact", "push", "target.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
# Configure bst to pull but not push from a cache and run `bst artifact push`.
# This should also fail.
- cli.configure({
- 'artifacts': {'url': share1.repo, 'push': False},
- })
- result = cli.run(project=project, args=['artifact', 'push', 'target.bst'])
+ cli.configure(
+ {"artifacts": {"url": share1.repo, "push": False},}
+ )
+ result = cli.run(project=project, args=["artifact", "push", "target.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
# Configure bst to push to one of the caches and run `bst artifact push`. This works.
- cli.configure({
- 'artifacts': [
- {'url': share1.repo, 'push': False},
- {'url': share2.repo, 'push': True},
- ]
- })
- cli.run(project=project, args=['artifact', 'push', 'target.bst'])
+ cli.configure({"artifacts": [{"url": share1.repo, "push": False}, {"url": share2.repo, "push": True},]})
+ cli.run(project=project, args=["artifact", "push", "target.bst"])
- assert_not_shared(cli, share1, project, 'target.bst')
- assert_shared(cli, share2, project, 'target.bst')
+ assert_not_shared(cli, share1, project, "target.bst")
+ assert_shared(cli, share2, project, "target.bst")
# Now try pushing to both
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare2')) as share2:
- cli.configure({
- 'artifacts': [
- {'url': share1.repo, 'push': True},
- {'url': share2.repo, 'push': True},
- ]
- })
- cli.run(project=project, args=['artifact', 'push', 'target.bst'])
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare2")) as share2:
+ cli.configure({"artifacts": [{"url": share1.repo, "push": True}, {"url": share2.repo, "push": True},]})
+ cli.run(project=project, args=["artifact", "push", "target.bst"])
- assert_shared(cli, share1, project, 'target.bst')
- assert_shared(cli, share2, project, 'target.bst')
+ assert_shared(cli, share1, project, "target.bst")
+ assert_shared(cli, share2, project, "target.bst")
# Tests `bst artifact push $artifact_ref`
@pytest.mark.datafiles(DATA_DIR)
def test_push_artifact(cli, tmpdir, datafiles):
project = str(datafiles)
- element = 'target.bst'
+ element = "target.bst"
# Configure a local cache
- local_cache = os.path.join(str(tmpdir), 'cache')
- cli.configure({'cachedir': local_cache})
+ local_cache = os.path.join(str(tmpdir), "cache")
+ cli.configure({"cachedir": local_cache})
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
# First build it without the artifact cache configured
- result = cli.run(project=project, args=['build', element])
+ result = cli.run(project=project, args=["build", element])
result.assert_success()
# Assert that the *artifact* is cached locally
cache_key = cli.get_element_key(project, element)
- artifact_ref = os.path.join('test', os.path.splitext(element)[0], cache_key)
- assert os.path.exists(os.path.join(local_cache, 'artifacts', 'refs', artifact_ref))
+ artifact_ref = os.path.join("test", os.path.splitext(element)[0], cache_key)
+ assert os.path.exists(os.path.join(local_cache, "artifacts", "refs", artifact_ref))
# Configure artifact share
- cli.configure({
- #
- # FIXME: This test hangs "sometimes" if we allow
- # concurrent push.
- #
- # It's not too bad to ignore since we're
- # using the local artifact cache functionality
- # only, but it should probably be fixed.
- #
- 'scheduler': {
- 'pushers': 1
- },
- 'artifacts': {
- 'url': share.repo,
- 'push': True,
+ cli.configure(
+ {
+ #
+ # FIXME: This test hangs "sometimes" if we allow
+ # concurrent push.
+ #
+ # It's not too bad to ignore since we're
+ # using the local artifact cache functionality
+ # only, but it should probably be fixed.
+ #
+ "scheduler": {"pushers": 1},
+ "artifacts": {"url": share.repo, "push": True,},
}
- })
+ )
# Now try bst artifact push all the deps
- result = cli.run(project=project, args=[
- 'artifact', 'push', artifact_ref
- ])
+ result = cli.run(project=project, args=["artifact", "push", artifact_ref])
result.assert_success()
# And finally assert that all the artifacts are in the share
@@ -162,27 +150,23 @@ def test_push_fails(cli, tmpdir, datafiles):
project = str(datafiles)
# Set up the share
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
# Configure bst to be able to push to the share
- cli.configure({
- 'artifacts': [
- {'url': share.repo, 'push': True},
- ]
- })
+ cli.configure({"artifacts": [{"url": share.repo, "push": True},]})
# First ensure that the target is *NOT* cache
- assert cli.get_element_state(project, 'target.bst') != 'cached'
+ assert cli.get_element_state(project, "target.bst") != "cached"
# Now try and push the target
- result = cli.run(project=project, args=['artifact', 'push', 'target.bst'])
+ result = cli.run(project=project, args=["artifact", "push", "target.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
assert "Push failed: target.bst is not cached" in result.stderr
# Now ensure that deps are also not cached
- assert cli.get_element_state(project, 'import-bin.bst') != 'cached'
- assert cli.get_element_state(project, 'import-dev.bst') != 'cached'
- assert cli.get_element_state(project, 'compose-all.bst') != 'cached'
+ assert cli.get_element_state(project, "import-bin.bst") != "cached"
+ assert cli.get_element_state(project, "import-dev.bst") != "cached"
+ assert cli.get_element_state(project, "compose-all.bst") != "cached"
# Tests that:
@@ -194,45 +178,42 @@ def test_push_fails_with_on_error_continue(cli, tmpdir, datafiles):
project = str(datafiles)
# Set up the share
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
# First build the target (and its deps)
- result = cli.run(project=project, args=['build', 'target.bst'])
- assert cli.get_element_state(project, 'target.bst') == 'cached'
- assert cli.get_element_state(project, 'import-dev.bst') == 'cached'
+ result = cli.run(project=project, args=["build", "target.bst"])
+ assert cli.get_element_state(project, "target.bst") == "cached"
+ assert cli.get_element_state(project, "import-dev.bst") == "cached"
# Now delete the artifact of a dependency and ensure it is not in the cache
- result = cli.run(project=project, args=['artifact', 'delete', 'import-dev.bst'])
- assert cli.get_element_state(project, 'import-dev.bst') != 'cached'
+ result = cli.run(project=project, args=["artifact", "delete", "import-dev.bst"])
+ assert cli.get_element_state(project, "import-dev.bst") != "cached"
# Configure bst to be able to push to the share
- cli.configure({
- 'artifacts': [
- {'url': share.repo, 'push': True},
- ]
- })
+ cli.configure({"artifacts": [{"url": share.repo, "push": True},]})
# Now try and push the target with its deps using --on-error continue
# and assert that push failed, but what could be pushed was pushed
- result = cli.run(project=project,
- args=['--on-error=continue', 'artifact', 'push', '--deps', 'all', 'target.bst'])
+ result = cli.run(
+ project=project, args=["--on-error=continue", "artifact", "push", "--deps", "all", "target.bst"]
+ )
# The overall process should return as failed
result.assert_main_error(ErrorDomain.STREAM, None)
# We should still have pushed what we could
- assert_shared(cli, share, project, 'import-bin.bst')
- assert_shared(cli, share, project, 'compose-all.bst')
- assert_shared(cli, share, project, 'target.bst')
+ assert_shared(cli, share, project, "import-bin.bst")
+ assert_shared(cli, share, project, "compose-all.bst")
+ assert_shared(cli, share, project, "target.bst")
- assert_not_shared(cli, share, project, 'import-dev.bst')
+ assert_not_shared(cli, share, project, "import-dev.bst")
errors = [
"import-dev.bst is not cached",
(
"Error while pushing. The following elements were not pushed as they are not yet cached:\n"
"\n"
"\timport-dev.bst\n"
- )
+ ),
]
for error in errors:
assert error in result.stderr
@@ -244,91 +225,81 @@ def test_push_fails_with_on_error_continue(cli, tmpdir, datafiles):
def test_push_all(cli, tmpdir, datafiles):
project = str(datafiles)
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
# First build it without the artifact cache configured
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
# Assert that we are now cached locally
- assert cli.get_element_state(project, 'target.bst') == 'cached'
+ assert cli.get_element_state(project, "target.bst") == "cached"
# Configure artifact share
- cli.configure({
- #
- # FIXME: This test hangs "sometimes" if we allow
- # concurrent push.
- #
- # It's not too bad to ignore since we're
- # using the local artifact cache functionality
- # only, but it should probably be fixed.
- #
- 'scheduler': {
- 'pushers': 1
- },
- 'artifacts': {
- 'url': share.repo,
- 'push': True,
+ cli.configure(
+ {
+ #
+ # FIXME: This test hangs "sometimes" if we allow
+ # concurrent push.
+ #
+ # It's not too bad to ignore since we're
+ # using the local artifact cache functionality
+ # only, but it should probably be fixed.
+ #
+ "scheduler": {"pushers": 1},
+ "artifacts": {"url": share.repo, "push": True,},
}
- })
+ )
# Now try bst artifact push all the deps
- result = cli.run(project=project, args=[
- 'artifact', 'push', 'target.bst',
- '--deps', 'all'
- ])
+ result = cli.run(project=project, args=["artifact", "push", "target.bst", "--deps", "all"])
result.assert_success()
# And finally assert that all the artifacts are in the share
- assert_shared(cli, share, project, 'target.bst')
- assert_shared(cli, share, project, 'import-bin.bst')
- assert_shared(cli, share, project, 'import-dev.bst')
- assert_shared(cli, share, project, 'compose-all.bst')
+ assert_shared(cli, share, project, "target.bst")
+ assert_shared(cli, share, project, "import-bin.bst")
+ assert_shared(cli, share, project, "import-dev.bst")
+ assert_shared(cli, share, project, "compose-all.bst")
+
# Tests that `bst artifact push --deps run $artifact_ref` fails
@pytest.mark.datafiles(DATA_DIR)
def test_push_artifacts_all_deps_fails(cli, tmpdir, datafiles):
project = str(datafiles)
- element = 'checkout-deps.bst'
+ element = "checkout-deps.bst"
# Configure a local cache
- local_cache = os.path.join(str(tmpdir), 'cache')
- cli.configure({'cachedir': local_cache})
+ local_cache = os.path.join(str(tmpdir), "cache")
+ cli.configure({"cachedir": local_cache})
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
# First build it without the artifact cache configured
- result = cli.run(project=project, args=['build', element])
+ result = cli.run(project=project, args=["build", element])
result.assert_success()
# Assert that the *artifact* is cached locally
cache_key = cli.get_element_key(project, element)
- artifact_ref = os.path.join('test', os.path.splitext(element)[0], cache_key)
- assert os.path.exists(os.path.join(local_cache, 'artifacts', 'refs', artifact_ref))
+ artifact_ref = os.path.join("test", os.path.splitext(element)[0], cache_key)
+ assert os.path.exists(os.path.join(local_cache, "artifacts", "refs", artifact_ref))
# Configure artifact share
- cli.configure({
- #
- # FIXME: This test hangs "sometimes" if we allow
- # concurrent push.
- #
- # It's not too bad to ignore since we're
- # using the local artifact cache functionality
- # only, but it should probably be fixed.
- #
- 'scheduler': {
- 'pushers': 1
- },
- 'artifacts': {
- 'url': share.repo,
- 'push': True,
+ cli.configure(
+ {
+ #
+ # FIXME: This test hangs "sometimes" if we allow
+ # concurrent push.
+ #
+ # It's not too bad to ignore since we're
+ # using the local artifact cache functionality
+ # only, but it should probably be fixed.
+ #
+ "scheduler": {"pushers": 1},
+ "artifacts": {"url": share.repo, "push": True,},
}
- })
+ )
# Now try bst artifact push all the deps
- result = cli.run(project=project, args=[
- 'artifact', 'push', '--deps', 'all', artifact_ref
- ])
+ result = cli.run(project=project, args=["artifact", "push", "--deps", "all", artifact_ref])
result.assert_main_error(ErrorDomain.STREAM, None)
assert "Error: '--deps all' is not supported for artifact refs" in result.stderr
@@ -342,47 +313,43 @@ def test_push_after_pull(cli, tmpdir, datafiles):
project = str(datafiles)
# Set up two artifact shares.
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare1')) as share1,\
- create_artifact_share(os.path.join(str(tmpdir), 'artifactshare2')) as share2:
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare1")) as share1, create_artifact_share(
+ os.path.join(str(tmpdir), "artifactshare2")
+ ) as share2:
# Set the scene: share1 has the artifact, share2 does not.
#
- cli.configure({
- 'artifacts': {'url': share1.repo, 'push': True},
- })
+ cli.configure(
+ {"artifacts": {"url": share1.repo, "push": True},}
+ )
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- cli.remove_artifact_from_cache(project, 'target.bst')
+ cli.remove_artifact_from_cache(project, "target.bst")
- assert_shared(cli, share1, project, 'target.bst')
- assert_not_shared(cli, share2, project, 'target.bst')
- assert cli.get_element_state(project, 'target.bst') != 'cached'
+ assert_shared(cli, share1, project, "target.bst")
+ assert_not_shared(cli, share2, project, "target.bst")
+ assert cli.get_element_state(project, "target.bst") != "cached"
# Now run the build again. Correct `bst build` behaviour is to download the
# artifact from share1 but not push it back again.
#
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- assert result.get_pulled_elements() == ['target.bst']
+ assert result.get_pulled_elements() == ["target.bst"]
assert result.get_pushed_elements() == []
# Delete the artifact locally again.
- cli.remove_artifact_from_cache(project, 'target.bst')
+ cli.remove_artifact_from_cache(project, "target.bst")
# Now we add share2 into the mix as a second push remote. This time,
# `bst build` should push to share2 after pulling from share1.
- cli.configure({
- 'artifacts': [
- {'url': share1.repo, 'push': True},
- {'url': share2.repo, 'push': True},
- ]
- })
- result = cli.run(project=project, args=['build', 'target.bst'])
+ cli.configure({"artifacts": [{"url": share1.repo, "push": True}, {"url": share2.repo, "push": True},]})
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- assert result.get_pulled_elements() == ['target.bst']
- assert result.get_pushed_elements() == ['target.bst']
+ assert result.get_pulled_elements() == ["target.bst"]
+ assert result.get_pushed_elements() == ["target.bst"]
# Ensure that when an artifact's size exceeds available disk space
@@ -391,52 +358,51 @@ def test_push_after_pull(cli, tmpdir, datafiles):
@pytest.mark.datafiles(DATA_DIR)
def test_artifact_expires(cli, datafiles, tmpdir):
project = str(datafiles)
- element_path = 'elements'
+ element_path = "elements"
# Create an artifact share (remote artifact cache) in the tmpdir/artifactshare
# Set a 22 MB quota
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare'),
- quota=int(22e6)) as share:
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare"), quota=int(22e6)) as share:
# Configure bst to push to the cache
- cli.configure({
- 'artifacts': {'url': share.repo, 'push': True},
- })
+ cli.configure(
+ {"artifacts": {"url": share.repo, "push": True},}
+ )
# Create and build an element of 15 MB
- create_element_size('element1.bst', project, element_path, [], int(15e6))
- result = cli.run(project=project, args=['build', 'element1.bst'])
+ create_element_size("element1.bst", project, element_path, [], int(15e6))
+ result = cli.run(project=project, args=["build", "element1.bst"])
result.assert_success()
# Create and build an element of 5 MB
- create_element_size('element2.bst', project, element_path, [], int(5e6))
- result = cli.run(project=project, args=['build', 'element2.bst'])
+ create_element_size("element2.bst", project, element_path, [], int(5e6))
+ result = cli.run(project=project, args=["build", "element2.bst"])
result.assert_success()
# check that element's 1 and 2 are cached both locally and remotely
- states = cli.get_element_states(project, ['element1.bst', 'element2.bst'])
+ states = cli.get_element_states(project, ["element1.bst", "element2.bst"])
assert states == {
"element1.bst": "cached",
"element2.bst": "cached",
}
- assert_shared(cli, share, project, 'element1.bst')
- assert_shared(cli, share, project, 'element2.bst')
+ assert_shared(cli, share, project, "element1.bst")
+ assert_shared(cli, share, project, "element2.bst")
# Create and build another element of 5 MB (This will exceed the free disk space available)
- create_element_size('element3.bst', project, element_path, [], int(5e6))
- result = cli.run(project=project, args=['build', 'element3.bst'])
+ create_element_size("element3.bst", project, element_path, [], int(5e6))
+ result = cli.run(project=project, args=["build", "element3.bst"])
result.assert_success()
# Ensure it is cached both locally and remotely
- assert cli.get_element_state(project, 'element3.bst') == 'cached'
- assert_shared(cli, share, project, 'element3.bst')
+ assert cli.get_element_state(project, "element3.bst") == "cached"
+ assert_shared(cli, share, project, "element3.bst")
# Ensure element1 has been removed from the share
- assert_not_shared(cli, share, project, 'element1.bst')
+ assert_not_shared(cli, share, project, "element1.bst")
# Ensure that elemen2 remains
- assert_shared(cli, share, project, 'element2.bst')
+ assert_shared(cli, share, project, "element2.bst")
# Test that a large artifact, whose size exceeds the quota, is not pushed
@@ -444,26 +410,25 @@ def test_artifact_expires(cli, datafiles, tmpdir):
@pytest.mark.datafiles(DATA_DIR)
def test_artifact_too_large(cli, datafiles, tmpdir):
project = str(datafiles)
- element_path = 'elements'
+ element_path = "elements"
# Create an artifact share (remote cache) in tmpdir/artifactshare
# Mock a file system with 5 MB total space
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare'),
- quota=int(5e6)) as share:
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare"), quota=int(5e6)) as share:
# Configure bst to push to the remote cache
- cli.configure({
- 'artifacts': {'url': share.repo, 'push': True},
- })
+ cli.configure(
+ {"artifacts": {"url": share.repo, "push": True},}
+ )
# Create and push a 3MB element
- create_element_size('small_element.bst', project, element_path, [], int(3e6))
- result = cli.run(project=project, args=['build', 'small_element.bst'])
+ create_element_size("small_element.bst", project, element_path, [], int(3e6))
+ result = cli.run(project=project, args=["build", "small_element.bst"])
result.assert_success()
# Create and try to push a 6MB element.
- create_element_size('large_element.bst', project, element_path, [], int(6e6))
- result = cli.run(project=project, args=['build', 'large_element.bst'])
+ create_element_size("large_element.bst", project, element_path, [], int(6e6))
+ result = cli.run(project=project, args=["build", "large_element.bst"])
# This should fail; the server will refuse to store the CAS
# blobs for the artifact, and then fail to find the files for
# the uploaded artifact proto.
@@ -476,100 +441,98 @@ def test_artifact_too_large(cli, datafiles, tmpdir):
result.assert_main_error(ErrorDomain.STREAM, None)
# Ensure that the small artifact is still in the share
- states = cli.get_element_states(project, ['small_element.bst', 'large_element.bst'])
- assert states['small_element.bst'] == 'cached'
- assert_shared(cli, share, project, 'small_element.bst')
+ states = cli.get_element_states(project, ["small_element.bst", "large_element.bst"])
+ assert states["small_element.bst"] == "cached"
+ assert_shared(cli, share, project, "small_element.bst")
# Ensure that the artifact is cached locally but NOT remotely
- assert states['large_element.bst'] == 'cached'
- assert_not_shared(cli, share, project, 'large_element.bst')
+ assert states["large_element.bst"] == "cached"
+ assert_not_shared(cli, share, project, "large_element.bst")
# Test that when an element is pulled recently, it is not considered the LRU element.
@pytest.mark.datafiles(DATA_DIR)
def test_recently_pulled_artifact_does_not_expire(cli, datafiles, tmpdir):
project = str(datafiles)
- element_path = 'elements'
+ element_path = "elements"
# Create an artifact share (remote cache) in tmpdir/artifactshare
# Set a 22 MB quota
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare'),
- quota=int(22e6)) as share:
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare"), quota=int(22e6)) as share:
# Configure bst to push to the cache
- cli.configure({
- 'artifacts': {'url': share.repo, 'push': True},
- })
+ cli.configure(
+ {"artifacts": {"url": share.repo, "push": True},}
+ )
# Create and build 2 elements, one 5 MB and one 15 MB.
- create_element_size('element1.bst', project, element_path, [], int(5e6))
- result = cli.run(project=project, args=['build', 'element1.bst'])
+ create_element_size("element1.bst", project, element_path, [], int(5e6))
+ result = cli.run(project=project, args=["build", "element1.bst"])
result.assert_success()
- create_element_size('element2.bst', project, element_path, [], int(15e6))
- result = cli.run(project=project, args=['build', 'element2.bst'])
+ create_element_size("element2.bst", project, element_path, [], int(15e6))
+ result = cli.run(project=project, args=["build", "element2.bst"])
result.assert_success()
# Ensure they are cached locally
- states = cli.get_element_states(project, ['element1.bst', 'element2.bst'])
+ states = cli.get_element_states(project, ["element1.bst", "element2.bst"])
assert states == {
"element1.bst": "cached",
"element2.bst": "cached",
}
# Ensure that they have been pushed to the cache
- assert_shared(cli, share, project, 'element1.bst')
- assert_shared(cli, share, project, 'element2.bst')
+ assert_shared(cli, share, project, "element1.bst")
+ assert_shared(cli, share, project, "element2.bst")
# Remove element1 from the local cache
- cli.remove_artifact_from_cache(project, 'element1.bst')
- assert cli.get_element_state(project, 'element1.bst') != 'cached'
+ cli.remove_artifact_from_cache(project, "element1.bst")
+ assert cli.get_element_state(project, "element1.bst") != "cached"
# Pull the element1 from the remote cache (this should update its mtime)
- result = cli.run(project=project, args=['artifact', 'pull', 'element1.bst', '--remote',
- share.repo])
+ result = cli.run(project=project, args=["artifact", "pull", "element1.bst", "--remote", share.repo])
result.assert_success()
# Ensure element1 is cached locally
- assert cli.get_element_state(project, 'element1.bst') == 'cached'
+ assert cli.get_element_state(project, "element1.bst") == "cached"
wait_for_cache_granularity()
# Create and build the element3 (of 5 MB)
- create_element_size('element3.bst', project, element_path, [], int(5e6))
- result = cli.run(project=project, args=['build', 'element3.bst'])
+ create_element_size("element3.bst", project, element_path, [], int(5e6))
+ result = cli.run(project=project, args=["build", "element3.bst"])
result.assert_success()
# Make sure it's cached locally and remotely
- assert cli.get_element_state(project, 'element3.bst') == 'cached'
- assert_shared(cli, share, project, 'element3.bst')
+ assert cli.get_element_state(project, "element3.bst") == "cached"
+ assert_shared(cli, share, project, "element3.bst")
# Ensure that element2 was deleted from the share and element1 remains
- assert_not_shared(cli, share, project, 'element2.bst')
- assert_shared(cli, share, project, 'element1.bst')
+ assert_not_shared(cli, share, project, "element2.bst")
+ assert_shared(cli, share, project, "element1.bst")
@pytest.mark.datafiles(DATA_DIR)
def test_push_cross_junction(cli, tmpdir, datafiles):
project = str(datafiles)
- subproject_path = os.path.join(project, 'files', 'sub-project')
- junction_path = os.path.join(project, 'elements', 'junction.bst')
+ subproject_path = os.path.join(project, "files", "sub-project")
+ junction_path = os.path.join(project, "elements", "junction.bst")
generate_junction(tmpdir, subproject_path, junction_path, store_ref=True)
- result = cli.run(project=project, args=['build', 'junction.bst:import-etc.bst'])
+ result = cli.run(project=project, args=["build", "junction.bst:import-etc.bst"])
result.assert_success()
- assert cli.get_element_state(project, 'junction.bst:import-etc.bst') == 'cached'
+ assert cli.get_element_state(project, "junction.bst:import-etc.bst") == "cached"
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
- cli.configure({
- 'artifacts': {'url': share.repo, 'push': True},
- })
- cli.run(project=project, args=['artifact', 'push', 'junction.bst:import-etc.bst'])
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
+ cli.configure(
+ {"artifacts": {"url": share.repo, "push": True},}
+ )
+ cli.run(project=project, args=["artifact", "push", "junction.bst:import-etc.bst"])
- cache_key = cli.get_element_key(project, 'junction.bst:import-etc.bst')
- assert share.get_artifact(cli.get_artifact_name(project, 'subtest', 'import-etc.bst', cache_key=cache_key))
+ cache_key = cli.get_element_key(project, "junction.bst:import-etc.bst")
+ assert share.get_artifact(cli.get_artifact_name(project, "subtest", "import-etc.bst", cache_key=cache_key))
@pytest.mark.datafiles(DATA_DIR)
@@ -577,17 +540,15 @@ def test_push_already_cached(caplog, cli, tmpdir, datafiles):
project = str(datafiles)
caplog.set_level(1)
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
- cli.configure({
- 'artifacts': {'url': share.repo, 'push': True}
- })
- result = cli.run(project=project, args=['build', 'target.bst'])
+ cli.configure({"artifacts": {"url": share.repo, "push": True}})
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
assert "SKIPPED Push" not in result.stderr
- result = cli.run(project=project, args=['artifact', 'push', 'target.bst'])
+ result = cli.run(project=project, args=["artifact", "push", "target.bst"])
result.assert_success()
assert not result.get_pushed_elements(), "No elements should have been pushed since the cache was populated"
@@ -600,24 +561,22 @@ def test_build_remote_option(caplog, cli, tmpdir, datafiles):
project = str(datafiles)
caplog.set_level(1)
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare1')) as shareuser,\
- create_artifact_share(os.path.join(str(tmpdir), 'artifactshare2')) as shareproject,\
- create_artifact_share(os.path.join(str(tmpdir), 'artifactshare3')) as sharecli:
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare1")) as shareuser, create_artifact_share(
+ os.path.join(str(tmpdir), "artifactshare2")
+ ) as shareproject, create_artifact_share(os.path.join(str(tmpdir), "artifactshare3")) as sharecli:
# Add shareproject repo url to project.conf
with open(os.path.join(project, "project.conf"), "a") as projconf:
projconf.write("artifacts:\n url: {}\n push: True".format(shareproject.repo))
# Configure shareuser remote in user conf
- cli.configure({
- 'artifacts': {'url': shareuser.repo, 'push': True}
- })
+ cli.configure({"artifacts": {"url": shareuser.repo, "push": True}})
- result = cli.run(project=project, args=['build', '--remote', sharecli.repo, 'target.bst'])
+ result = cli.run(project=project, args=["build", "--remote", sharecli.repo, "target.bst"])
# Artifacts should have only been pushed to sharecli, as that was provided via the cli
result.assert_success()
- all_elements = ['target.bst', 'import-bin.bst', 'compose-all.bst']
+ all_elements = ["target.bst", "import-bin.bst", "compose-all.bst"]
for element_name in all_elements:
assert element_name in result.get_pushed_elements()
assert_shared(cli, sharecli, project, element_name)
@@ -632,26 +591,16 @@ def test_build_remote_option(caplog, cli, tmpdir, datafiles):
# This is a regression test for issue #990
#
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("buildtrees", [('buildtrees'), ('normal')])
+@pytest.mark.parametrize("buildtrees", [("buildtrees"), ("normal")])
def test_push_no_strict(caplog, cli, tmpdir, datafiles, buildtrees):
project = os.path.join(datafiles.dirname, datafiles.basename)
caplog.set_level(1)
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
- cli.configure({
- 'artifacts': {
- 'url': share.repo,
- 'push': True
- },
- 'projects': {
- 'test': {
- 'strict': False
- }
- }
- })
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
+ cli.configure({"artifacts": {"url": share.repo, "push": True}, "projects": {"test": {"strict": False}}})
# First get us a build
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
# Now cause one of the dependenies to change their cache key
@@ -659,12 +608,12 @@ def test_push_no_strict(caplog, cli, tmpdir, datafiles, buildtrees):
# Here we just add a file, causing the strong cache key of the
# import-bin.bst element to change due to the local files it
# imports changing.
- path = os.path.join(project, 'files', 'bin-files', 'newfile')
- with open(path, 'w') as f:
+ path = os.path.join(project, "files", "bin-files", "newfile")
+ with open(path, "w") as f:
f.write("PONY !")
# Now build again after having changed the dependencies
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
# Now run `bst artifact push`.
@@ -673,8 +622,8 @@ def test_push_no_strict(caplog, cli, tmpdir, datafiles, buildtrees):
# a pull queue to be added to the `push` command, the behavior
# around this is different.
args = []
- if buildtrees == 'buildtrees':
- args += ['--pull-buildtrees']
- args += ['artifact', 'push', '--deps', 'all', 'target.bst']
+ if buildtrees == "buildtrees":
+ args += ["--pull-buildtrees"]
+ args += ["artifact", "push", "--deps", "all", "target.bst"]
result = cli.run(project=project, args=args)
result.assert_success()
diff --git a/tests/frontend/rebuild.py b/tests/frontend/rebuild.py
index 1cdb45d11..1aef8e423 100644
--- a/tests/frontend/rebuild.py
+++ b/tests/frontend/rebuild.py
@@ -6,15 +6,12 @@ import pytest
from buildstream.testing import cli # pylint: disable=unused-import
# Project directory
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project",
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",)
def strict_args(args, strict):
if strict != "strict":
- return ['--no-strict', *args]
+ return ["--no-strict", *args]
return args
@@ -24,15 +21,15 @@ def test_rebuild(datafiles, cli, strict):
project = str(datafiles)
# First build intermediate target.bst
- result = cli.run(project=project, args=strict_args(['build', 'target.bst'], strict))
+ result = cli.run(project=project, args=strict_args(["build", "target.bst"], strict))
result.assert_success()
# Modify base import
- with open(os.path.join(project, 'files', 'dev-files', 'usr', 'include', 'new.h'), "w") as f:
+ with open(os.path.join(project, "files", "dev-files", "usr", "include", "new.h"), "w") as f:
f.write("#define NEW")
# Rebuild base import and build top-level rebuild-target.bst
# In non-strict mode, this does not rebuild intermediate target.bst,
# which means that a weakly cached target.bst will be staged as dependency.
- result = cli.run(project=project, args=strict_args(['build', 'rebuild-target.bst'], strict))
+ result = cli.run(project=project, args=strict_args(["build", "rebuild-target.bst"], strict))
result.assert_success()
diff --git a/tests/frontend/remote-caches.py b/tests/frontend/remote-caches.py
index 6ee57df23..b112e0882 100644
--- a/tests/frontend/remote-caches.py
+++ b/tests/frontend/remote-caches.py
@@ -28,7 +28,7 @@ from buildstream import _yaml
from tests.testutils import create_artifact_share, create_element_size
-DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'project')
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
def message_handler(message, context):
@@ -37,54 +37,46 @@ def message_handler(message, context):
@pytest.mark.datafiles(DATA_DIR)
def test_source_artifact_caches(cli, tmpdir, datafiles):
- cachedir = os.path.join(str(tmpdir), 'cache')
+ cachedir = os.path.join(str(tmpdir), "cache")
project_dir = str(datafiles)
- element_path = os.path.join(project_dir, 'elements')
+ element_path = os.path.join(project_dir, "elements")
- with create_artifact_share(os.path.join(str(tmpdir), 'share')) as share:
- user_config_file = str(tmpdir.join('buildstream.conf'))
+ with create_artifact_share(os.path.join(str(tmpdir), "share")) as share:
+ user_config_file = str(tmpdir.join("buildstream.conf"))
user_config = {
- 'scheduler': {
- 'pushers': 1
- },
- 'source-caches': {
- 'url': share.repo,
- 'push': True,
- },
- 'artifacts': {
- 'url': share.repo,
- 'push': True,
- },
- 'cachedir': cachedir
+ "scheduler": {"pushers": 1},
+ "source-caches": {"url": share.repo, "push": True,},
+ "artifacts": {"url": share.repo, "push": True,},
+ "cachedir": cachedir,
}
_yaml.roundtrip_dump(user_config, file=user_config_file)
cli.configure(user_config)
- create_element_size('repo.bst', project_dir, element_path, [], 10000)
+ create_element_size("repo.bst", project_dir, element_path, [], 10000)
- res = cli.run(project=project_dir, args=['build', 'repo.bst'])
+ res = cli.run(project=project_dir, args=["build", "repo.bst"])
res.assert_success()
assert "Pushed source " in res.stderr
assert "Pushed artifact " in res.stderr
# delete local sources and artifacts and check it pulls them
- shutil.rmtree(os.path.join(cachedir, 'cas'))
- shutil.rmtree(os.path.join(cachedir, 'sources'))
+ shutil.rmtree(os.path.join(cachedir, "cas"))
+ shutil.rmtree(os.path.join(cachedir, "sources"))
# this should just fetch the artifacts
- res = cli.run(project=project_dir, args=['build', 'repo.bst'])
+ res = cli.run(project=project_dir, args=["build", "repo.bst"])
res.assert_success()
assert "Pulled artifact " in res.stderr
assert "Pulled source " not in res.stderr
# remove the artifact from the repo and check it pulls sources, builds
# and then pushes the artifacts
- shutil.rmtree(os.path.join(cachedir, 'cas'))
- shutil.rmtree(os.path.join(cachedir, 'artifacts'))
- print(os.listdir(os.path.join(share.repodir, 'artifacts', 'refs')))
- shutil.rmtree(os.path.join(share.repodir, 'artifacts', 'refs', 'test'))
+ shutil.rmtree(os.path.join(cachedir, "cas"))
+ shutil.rmtree(os.path.join(cachedir, "artifacts"))
+ print(os.listdir(os.path.join(share.repodir, "artifacts", "refs")))
+ shutil.rmtree(os.path.join(share.repodir, "artifacts", "refs", "test"))
- res = cli.run(project=project_dir, args=['build', 'repo.bst'])
+ res = cli.run(project=project_dir, args=["build", "repo.bst"])
res.assert_success()
assert "Remote ({}) does not have artifact ".format(share.repo) in res.stderr
assert "Pulled source" in res.stderr
diff --git a/tests/frontend/show.py b/tests/frontend/show.py
index bc51d2967..b08670a30 100644
--- a/tests/frontend/show.py
+++ b/tests/frontend/show.py
@@ -15,85 +15,77 @@ from tests.testutils import generate_junction
from . import configure_project
# Project directory
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)),)
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project'))
-@pytest.mark.parametrize("target,fmt,expected", [
- ('import-bin.bst', '%{name}', 'import-bin.bst'),
- ('import-bin.bst', '%{state}', 'buildable'),
- ('compose-all.bst', '%{state}', 'waiting')
-])
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "project"))
+@pytest.mark.parametrize(
+ "target,fmt,expected",
+ [
+ ("import-bin.bst", "%{name}", "import-bin.bst"),
+ ("import-bin.bst", "%{state}", "buildable"),
+ ("compose-all.bst", "%{state}", "waiting"),
+ ],
+)
def test_show(cli, datafiles, target, fmt, expected):
project = str(datafiles)
- result = cli.run(project=project, silent=True, args=[
- 'show',
- '--deps', 'none',
- '--format', fmt,
- target])
+ result = cli.run(project=project, silent=True, args=["show", "--deps", "none", "--format", fmt, target])
result.assert_success()
if result.output.strip() != expected:
- raise AssertionError("Expected output:\n{}\nInstead received output:\n{}"
- .format(expected, result.output))
+ raise AssertionError("Expected output:\n{}\nInstead received output:\n{}".format(expected, result.output))
-@pytest.mark.datafiles(os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "invalid_element_path",
-))
+@pytest.mark.datafiles(os.path.join(os.path.dirname(os.path.realpath(__file__)), "invalid_element_path",))
def test_show_invalid_element_path(cli, datafiles):
project = str(datafiles)
- cli.run(project=project, silent=True, args=['show', "foo.bst"])
+ cli.run(project=project, silent=True, args=["show", "foo.bst"])
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project_default'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "project_default"))
def test_show_default(cli, datafiles):
project = str(datafiles)
- result = cli.run(project=project, silent=True, args=[
- 'show'])
+ result = cli.run(project=project, silent=True, args=["show"])
result.assert_success()
# Get the result output of "[state sha element]" and turn into a list
results = result.output.strip().split(" ")
- expected = 'target2.bst'
+ expected = "target2.bst"
assert results[2] == expected
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project_fail'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "project_fail"))
def test_show_fail(cli, datafiles):
project = str(datafiles)
- result = cli.run(project=project, silent=True, args=[
- 'show'])
+ result = cli.run(project=project, silent=True, args=["show"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project'))
-@pytest.mark.parametrize("target,except_,expected", [
- ('target.bst', 'import-bin.bst', ['import-dev.bst', 'compose-all.bst', 'target.bst']),
- ('target.bst', 'import-dev.bst', ['import-bin.bst', 'compose-all.bst', 'target.bst']),
- ('target.bst', 'compose-all.bst', ['import-bin.bst', 'target.bst']),
- ('compose-all.bst', 'import-bin.bst', ['import-dev.bst', 'compose-all.bst'])
-])
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "project"))
+@pytest.mark.parametrize(
+ "target,except_,expected",
+ [
+ ("target.bst", "import-bin.bst", ["import-dev.bst", "compose-all.bst", "target.bst"]),
+ ("target.bst", "import-dev.bst", ["import-bin.bst", "compose-all.bst", "target.bst"]),
+ ("target.bst", "compose-all.bst", ["import-bin.bst", "target.bst"]),
+ ("compose-all.bst", "import-bin.bst", ["import-dev.bst", "compose-all.bst"]),
+ ],
+)
def test_show_except_simple(cli, datafiles, target, except_, expected):
project = str(datafiles)
- result = cli.run(project=project, silent=True, args=[
- 'show',
- '--deps', 'all',
- '--format', '%{name}',
- '--except', except_,
- target])
+ result = cli.run(
+ project=project,
+ silent=True,
+ args=["show", "--deps", "all", "--format", "%{name}", "--except", except_, target],
+ )
result.assert_success()
results = result.output.strip().splitlines()
if results != expected:
- raise AssertionError("Expected elements:\n{}\nInstead received elements:\n{}"
- .format(expected, results))
+ raise AssertionError("Expected elements:\n{}\nInstead received elements:\n{}".format(expected, results))
# This test checks various constructions of a pipeline
@@ -101,104 +93,116 @@ def test_show_except_simple(cli, datafiles, target, except_, expected):
# each data set provides the targets, exceptions and expected
# result list.
#
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'exceptions'))
-@pytest.mark.parametrize("targets,exceptions,expected", [
-
- # Test without exceptions, lets just see the whole list here
- (['build.bst'], None, [
- 'fourth-level-1.bst',
- 'third-level-1.bst',
- 'fourth-level-2.bst',
- 'third-level-2.bst',
- 'fourth-level-3.bst',
- 'third-level-3.bst',
- 'second-level-1.bst',
- 'first-level-1.bst',
- 'first-level-2.bst',
- 'build.bst',
- ]),
-
- # Test one target and excepting a part of the pipeline, this
- # removes forth-level-1 and third-level-1
- (['build.bst'], ['third-level-1.bst'], [
- 'fourth-level-2.bst',
- 'third-level-2.bst',
- 'fourth-level-3.bst',
- 'third-level-3.bst',
- 'second-level-1.bst',
- 'first-level-1.bst',
- 'first-level-2.bst',
- 'build.bst',
- ]),
-
- # Test one target and excepting a part of the pipeline, check that
- # excepted dependencies remain in the pipeline if depended on from
- # outside of the except element
- (['build.bst'], ['second-level-1.bst'], [
- 'fourth-level-2.bst',
- 'third-level-2.bst', # first-level-2 depends on this, so not excepted
- 'first-level-1.bst',
- 'first-level-2.bst',
- 'build.bst',
- ]),
-
- # The same as the above test, but excluding the toplevel build.bst,
- # instead only select the two toplevel dependencies as targets
- (['first-level-1.bst', 'first-level-2.bst'], ['second-level-1.bst'], [
- 'fourth-level-2.bst',
- 'third-level-2.bst', # first-level-2 depends on this, so not excepted
- 'first-level-1.bst',
- 'first-level-2.bst',
- ]),
-
- # Test one target and excepting an element outisde the pipeline
- (['build.bst'], ['unrelated-1.bst'], [
- 'fourth-level-2.bst',
- 'third-level-2.bst', # first-level-2 depends on this, so not excepted
- 'first-level-1.bst',
- 'first-level-2.bst',
- 'build.bst',
- ]),
-
- # Test one target and excepting two elements
- (['build.bst'], ['unrelated-1.bst', 'unrelated-2.bst'], [
- 'first-level-1.bst',
- 'build.bst',
- ]),
-])
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "exceptions"))
+@pytest.mark.parametrize(
+ "targets,exceptions,expected",
+ [
+ # Test without exceptions, lets just see the whole list here
+ (
+ ["build.bst"],
+ None,
+ [
+ "fourth-level-1.bst",
+ "third-level-1.bst",
+ "fourth-level-2.bst",
+ "third-level-2.bst",
+ "fourth-level-3.bst",
+ "third-level-3.bst",
+ "second-level-1.bst",
+ "first-level-1.bst",
+ "first-level-2.bst",
+ "build.bst",
+ ],
+ ),
+ # Test one target and excepting a part of the pipeline, this
+ # removes forth-level-1 and third-level-1
+ (
+ ["build.bst"],
+ ["third-level-1.bst"],
+ [
+ "fourth-level-2.bst",
+ "third-level-2.bst",
+ "fourth-level-3.bst",
+ "third-level-3.bst",
+ "second-level-1.bst",
+ "first-level-1.bst",
+ "first-level-2.bst",
+ "build.bst",
+ ],
+ ),
+ # Test one target and excepting a part of the pipeline, check that
+ # excepted dependencies remain in the pipeline if depended on from
+ # outside of the except element
+ (
+ ["build.bst"],
+ ["second-level-1.bst"],
+ [
+ "fourth-level-2.bst",
+ "third-level-2.bst", # first-level-2 depends on this, so not excepted
+ "first-level-1.bst",
+ "first-level-2.bst",
+ "build.bst",
+ ],
+ ),
+ # The same as the above test, but excluding the toplevel build.bst,
+ # instead only select the two toplevel dependencies as targets
+ (
+ ["first-level-1.bst", "first-level-2.bst"],
+ ["second-level-1.bst"],
+ [
+ "fourth-level-2.bst",
+ "third-level-2.bst", # first-level-2 depends on this, so not excepted
+ "first-level-1.bst",
+ "first-level-2.bst",
+ ],
+ ),
+ # Test one target and excepting an element outisde the pipeline
+ (
+ ["build.bst"],
+ ["unrelated-1.bst"],
+ [
+ "fourth-level-2.bst",
+ "third-level-2.bst", # first-level-2 depends on this, so not excepted
+ "first-level-1.bst",
+ "first-level-2.bst",
+ "build.bst",
+ ],
+ ),
+ # Test one target and excepting two elements
+ (["build.bst"], ["unrelated-1.bst", "unrelated-2.bst"], ["first-level-1.bst", "build.bst",]),
+ ],
+)
def test_show_except(cli, datafiles, targets, exceptions, expected):
basedir = str(datafiles)
- results = cli.get_pipeline(basedir, targets, except_=exceptions, scope='all')
+ results = cli.get_pipeline(basedir, targets, except_=exceptions, scope="all")
if results != expected:
- raise AssertionError("Expected elements:\n{}\nInstead received elements:\n{}"
- .format(expected, results))
+ raise AssertionError("Expected elements:\n{}\nInstead received elements:\n{}".format(expected, results))
###############################################################
# Testing multiple targets #
###############################################################
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "project"))
def test_parallel_order(cli, datafiles):
project = str(datafiles)
- elements = ['multiple_targets/order/0.bst',
- 'multiple_targets/order/1.bst']
+ elements = ["multiple_targets/order/0.bst", "multiple_targets/order/1.bst"]
- args = ['show', '-d', 'plan', '-f', '%{name}', *elements]
+ args = ["show", "-d", "plan", "-f", "%{name}", *elements]
result = cli.run(project=project, args=args)
result.assert_success()
# Get the planned order
names = result.output.splitlines()
- names = [name[len('multiple_targets/order/'):] for name in names]
+ names = [name[len("multiple_targets/order/") :] for name in names]
# Create all possible 'correct' topological orderings
orderings = itertools.product(
- [('5.bst', '6.bst')],
- itertools.permutations(['4.bst', '7.bst']),
- itertools.permutations(['3.bst', '8.bst']),
- itertools.permutations(['2.bst', '9.bst']),
- itertools.permutations(['0.bst', '1.bst', 'run.bst'])
+ [("5.bst", "6.bst")],
+ itertools.permutations(["4.bst", "7.bst"]),
+ itertools.permutations(["3.bst", "8.bst"]),
+ itertools.permutations(["2.bst", "9.bst"]),
+ itertools.permutations(["0.bst", "1.bst", "run.bst"]),
)
orderings = [list(itertools.chain.from_iterable(perm)) for perm in orderings]
@@ -206,126 +210,96 @@ def test_parallel_order(cli, datafiles):
assert names in orderings, "We got: {}".format(", ".join(names))
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "project"))
def test_target_is_dependency(cli, datafiles):
project = str(datafiles)
- elements = ['multiple_targets/dependency/zebry.bst',
- 'multiple_targets/dependency/horsey.bst']
+ elements = ["multiple_targets/dependency/zebry.bst", "multiple_targets/dependency/horsey.bst"]
- args = ['show', '-d', 'plan', '-f', '%{name}', *elements]
+ args = ["show", "-d", "plan", "-f", "%{name}", *elements]
result = cli.run(project=project, args=args)
result.assert_success()
# Get the planned order
names = result.output.splitlines()
- names = [name[len('multiple_targets/dependency/'):] for name in names]
+ names = [name[len("multiple_targets/dependency/") :] for name in names]
- assert names == ['pony.bst', 'horsey.bst', 'zebry.bst']
+ assert names == ["pony.bst", "horsey.bst", "zebry.bst"]
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project'))
-@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
-@pytest.mark.parametrize("element_name", ['junction-dep.bst', 'junction.bst:import-etc.bst'])
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "project"))
+@pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")])
+@pytest.mark.parametrize("element_name", ["junction-dep.bst", "junction.bst:import-etc.bst"])
@pytest.mark.parametrize("workspaced", [True, False], ids=["workspace", "no-workspace"])
def test_unfetched_junction(cli, tmpdir, datafiles, ref_storage, element_name, workspaced):
project = str(datafiles)
- subproject_path = os.path.join(project, 'files', 'sub-project')
- junction_path = os.path.join(project, 'elements', 'junction.bst')
- element_path = os.path.join(project, 'elements', 'junction-dep.bst')
+ subproject_path = os.path.join(project, "files", "sub-project")
+ junction_path = os.path.join(project, "elements", "junction.bst")
+ element_path = os.path.join(project, "elements", "junction-dep.bst")
- configure_project(project, {
- 'ref-storage': ref_storage
- })
+ configure_project(project, {"ref-storage": ref_storage})
# Create a repo to hold the subproject and generate a junction element for it
- ref = generate_junction(tmpdir, subproject_path, junction_path, store_ref=(ref_storage == 'inline'))
+ ref = generate_junction(tmpdir, subproject_path, junction_path, store_ref=(ref_storage == "inline"))
# Create a stack element to depend on a cross junction element
#
- element = {
- 'kind': 'stack',
- 'depends': [
- {
- 'junction': 'junction.bst',
- 'filename': 'import-etc.bst'
- }
- ]
- }
+ element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]}
_yaml.roundtrip_dump(element, element_path)
# Dump a project.refs if we're using project.refs storage
#
- if ref_storage == 'project.refs':
- project_refs = {
- 'projects': {
- 'test': {
- 'junction.bst': [
- {
- 'ref': ref
- }
- ]
- }
- }
- }
- _yaml.roundtrip_dump(project_refs, os.path.join(project, 'junction.refs'))
+ if ref_storage == "project.refs":
+ project_refs = {"projects": {"test": {"junction.bst": [{"ref": ref}]}}}
+ _yaml.roundtrip_dump(project_refs, os.path.join(project, "junction.refs"))
# Open a workspace if we're testing workspaced behavior
if workspaced:
- result = cli.run(project=project, silent=True, args=[
- 'workspace', 'open', '--no-checkout', '--directory', subproject_path, 'junction.bst'
- ])
+ result = cli.run(
+ project=project,
+ silent=True,
+ args=["workspace", "open", "--no-checkout", "--directory", subproject_path, "junction.bst"],
+ )
result.assert_success()
# Assert successful bst show (requires implicit subproject fetching)
- result = cli.run(project=project, silent=True, args=[
- 'show', element_name])
+ result = cli.run(project=project, silent=True, args=["show", element_name])
result.assert_success()
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project'))
-@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "project"))
+@pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")])
@pytest.mark.parametrize("workspaced", [True, False], ids=["workspace", "no-workspace"])
def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage, workspaced):
project = str(datafiles)
- subproject_path = os.path.join(project, 'files', 'sub-project')
- junction_path = os.path.join(project, 'elements', 'junction.bst')
- element_path = os.path.join(project, 'elements', 'junction-dep.bst')
+ subproject_path = os.path.join(project, "files", "sub-project")
+ junction_path = os.path.join(project, "elements", "junction.bst")
+ element_path = os.path.join(project, "elements", "junction-dep.bst")
- configure_project(project, {
- 'ref-storage': ref_storage
- })
+ configure_project(project, {"ref-storage": ref_storage})
# Create a repo to hold the subproject and generate a junction element for it
generate_junction(tmpdir, subproject_path, junction_path, store_ref=False)
# Create a stack element to depend on a cross junction element
#
- element = {
- 'kind': 'stack',
- 'depends': [
- {
- 'junction': 'junction.bst',
- 'filename': 'import-etc.bst'
- }
- ]
- }
+ element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]}
_yaml.roundtrip_dump(element, element_path)
# Open a workspace if we're testing workspaced behavior
if workspaced:
- result = cli.run(project=project, silent=True, args=[
- 'workspace', 'open', '--no-checkout', '--directory', subproject_path, 'junction.bst'
- ])
+ result = cli.run(
+ project=project,
+ silent=True,
+ args=["workspace", "open", "--no-checkout", "--directory", subproject_path, "junction.bst"],
+ )
result.assert_success()
# Assert the correct error when trying to show the pipeline
- dep_result = cli.run(project=project, silent=True, args=[
- 'show', 'junction-dep.bst'])
+ dep_result = cli.run(project=project, silent=True, args=["show", "junction-dep.bst"])
# Assert the correct error when trying to show the pipeline
- etc_result = cli.run(project=project, silent=True, args=[
- 'show', 'junction.bst:import-etc.bst'])
+ etc_result = cli.run(project=project, silent=True, args=["show", "junction.bst:import-etc.bst"])
# If a workspace is open, no ref is needed
if workspaced:
@@ -333,8 +307,8 @@ def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage, workspaced):
etc_result.assert_success()
else:
# Assert that we have the expected provenance encoded into the error
- element_node = _yaml.load(element_path, shortname='junction-dep.bst')
- ref_node = element_node.get_sequence('depends').mapping_at(0)
+ element_node = _yaml.load(element_path, shortname="junction-dep.bst")
+ ref_node = element_node.get_sequence("depends").mapping_at(0)
provenance = ref_node.get_provenance()
assert str(provenance) in dep_result.stderr
@@ -342,49 +316,41 @@ def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage, workspaced):
etc_result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT)
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project'))
-@pytest.mark.parametrize("element_name", ['junction-dep.bst', 'junction.bst:import-etc.bst'])
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "project"))
+@pytest.mark.parametrize("element_name", ["junction-dep.bst", "junction.bst:import-etc.bst"])
@pytest.mark.parametrize("workspaced", [True, False], ids=["workspace", "no-workspace"])
def test_fetched_junction(cli, tmpdir, datafiles, element_name, workspaced):
project = str(datafiles)
project = os.path.join(datafiles.dirname, datafiles.basename)
- subproject_path = os.path.join(project, 'files', 'sub-project')
- junction_path = os.path.join(project, 'elements', 'junction.bst')
- element_path = os.path.join(project, 'elements', 'junction-dep.bst')
+ subproject_path = os.path.join(project, "files", "sub-project")
+ junction_path = os.path.join(project, "elements", "junction.bst")
+ element_path = os.path.join(project, "elements", "junction-dep.bst")
# Create a repo to hold the subproject and generate a junction element for it
generate_junction(tmpdir, subproject_path, junction_path, store_ref=True)
# Create a stack element to depend on a cross junction element
#
- element = {
- 'kind': 'stack',
- 'depends': [
- {
- 'junction': 'junction.bst',
- 'filename': 'import-etc.bst'
- }
- ]
- }
+ element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]}
_yaml.roundtrip_dump(element, element_path)
- result = cli.run(project=project, silent=True, args=[
- 'source', 'fetch', 'junction.bst'])
+ result = cli.run(project=project, silent=True, args=["source", "fetch", "junction.bst"])
result.assert_success()
# Open a workspace if we're testing workspaced behavior
if workspaced:
- result = cli.run(project=project, silent=True, args=[
- 'workspace', 'open', '--no-checkout', '--directory', subproject_path, 'junction.bst'
- ])
+ result = cli.run(
+ project=project,
+ silent=True,
+ args=["workspace", "open", "--no-checkout", "--directory", subproject_path, "junction.bst"],
+ )
result.assert_success()
# Assert the correct error when trying to show the pipeline
- result = cli.run(project=project, silent=True, args=[
- 'show', '--format', '%{name}-%{state}', element_name])
+ result = cli.run(project=project, silent=True, args=["show", "--format", "%{name}-%{state}", element_name])
results = result.output.strip().splitlines()
- assert 'junction.bst:import-etc.bst-buildable' in results
+ assert "junction.bst:import-etc.bst-buildable" in results
###############################################################
@@ -404,8 +370,7 @@ def test_exceed_max_recursion_depth(cli, tmpdir, dependency_depth):
"""
os.mkdir(project_path)
- result = cli.run(silent=True,
- args=['init', '--project-name', project_name, project_path])
+ result = cli.run(silent=True, args=["init", "--project-name", project_name, project_path])
result.assert_success()
sourcefiles_path = os.path.join(project_path, "files")
@@ -414,22 +379,20 @@ def test_exceed_max_recursion_depth(cli, tmpdir, dependency_depth):
element_path = os.path.join(project_path, "elements")
for i in range(0, dependency_depth + 1):
element = {
- 'kind': 'import',
- 'sources': [{'kind': 'local',
- 'path': 'files/source{}'.format(str(i))}],
- 'depends': ['element{}.bst'.format(str(i - 1))]
+ "kind": "import",
+ "sources": [{"kind": "local", "path": "files/source{}".format(str(i))}],
+ "depends": ["element{}.bst".format(str(i - 1))],
}
if i == 0:
- del element['depends']
+ del element["depends"]
_yaml.roundtrip_dump(element, os.path.join(element_path, "element{}.bst".format(str(i))))
source = os.path.join(sourcefiles_path, "source{}".format(str(i)))
- open(source, 'x').close()
+ open(source, "x").close()
assert os.path.exists(source)
setup_test()
- result = cli.run(project=project_path, silent=True,
- args=['show', "element{}.bst".format(str(dependency_depth))])
+ result = cli.run(project=project_path, silent=True, args=["show", "element{}.bst".format(str(dependency_depth))])
recursion_limit = sys.getrecursionlimit()
if dependency_depth <= recursion_limit:
@@ -445,71 +408,62 @@ def test_exceed_max_recursion_depth(cli, tmpdir, dependency_depth):
###############################################################
# Testing format symbols #
###############################################################
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project'))
-@pytest.mark.parametrize("dep_kind, expected_deps", [
- ('%{deps}', '[import-dev.bst, import-bin.bst]'),
- ('%{build-deps}', '[import-dev.bst]'),
- ('%{runtime-deps}', '[import-bin.bst]')
-])
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "project"))
+@pytest.mark.parametrize(
+ "dep_kind, expected_deps",
+ [
+ ("%{deps}", "[import-dev.bst, import-bin.bst]"),
+ ("%{build-deps}", "[import-dev.bst]"),
+ ("%{runtime-deps}", "[import-bin.bst]"),
+ ],
+)
def test_format_deps(cli, datafiles, dep_kind, expected_deps):
project = str(datafiles)
- target = 'checkout-deps.bst'
- result = cli.run(project=project, silent=True, args=[
- 'show',
- '--deps', 'none',
- '--format', '%{name}: ' + dep_kind,
- target])
+ target = "checkout-deps.bst"
+ result = cli.run(
+ project=project, silent=True, args=["show", "--deps", "none", "--format", "%{name}: " + dep_kind, target]
+ )
result.assert_success()
- expected = '{name}: {deps}'.format(name=target, deps=expected_deps)
+ expected = "{name}: {deps}".format(name=target, deps=expected_deps)
if result.output.strip() != expected:
- raise AssertionError("Expected output:\n{}\nInstead received output:\n{}"
- .format(expected, result.output))
+ raise AssertionError("Expected output:\n{}\nInstead received output:\n{}".format(expected, result.output))
# This tests the resolved value of the 'max-jobs' variable,
# ensuring at least that the variables are resolved according
# to how the user has configured max-jobs
#
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project'))
-@pytest.mark.parametrize("cli_value, config_value", [
- (None, None),
- (None, '16'),
- ('16', None),
- ('5', '16'),
- ('0', '16'),
- ('16', '0'),
-])
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "project"))
+@pytest.mark.parametrize(
+ "cli_value, config_value", [(None, None), (None, "16"), ("16", None), ("5", "16"), ("0", "16"), ("16", "0"),]
+)
def test_max_jobs(cli, datafiles, cli_value, config_value):
project = str(datafiles)
- target = 'target.bst'
+ target = "target.bst"
# Specify `--max-jobs` if this test sets it
args = []
if cli_value is not None:
- args += ['--max-jobs', cli_value]
- args += ['show', '--deps', 'none', '--format', '%{vars}', target]
+ args += ["--max-jobs", cli_value]
+ args += ["show", "--deps", "none", "--format", "%{vars}", target]
# Specify `max-jobs` in user configuration if this test sets it
if config_value is not None:
- cli.configure({
- 'build': {
- 'max-jobs': config_value
- }
- })
+ cli.configure({"build": {"max-jobs": config_value}})
result = cli.run(project=project, silent=True, args=args)
result.assert_success()
loaded = _yaml.load_data(result.output)
- loaded_value = loaded.get_int('max-jobs')
+ loaded_value = loaded.get_int("max-jobs")
# We expect the value provided on the command line to take
# precedence over the configuration file value, if specified.
#
# If neither are specified then we expect the default
- expected_value = cli_value or config_value or '0'
+ expected_value = cli_value or config_value or "0"
- if expected_value == '0':
+ if expected_value == "0":
# If we are expecting the automatic behavior of using the maximum
# number of cores available, just check that it is a value > 0
assert loaded_value > 0, "Automatic setting of max-jobs didnt work"
@@ -534,39 +488,32 @@ def test_max_jobs(cli, datafiles, cli_value, config_value):
# depends on the changing import element, and one which
# depends on it regularly.
#
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'strict-depends'))
-@pytest.mark.parametrize("target, expected_state", [
- ("non-strict-depends.bst", "cached"),
- ("strict-depends.bst", "waiting"),
-])
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "strict-depends"))
+@pytest.mark.parametrize(
+ "target, expected_state", [("non-strict-depends.bst", "cached"), ("strict-depends.bst", "waiting"),]
+)
def test_strict_dependencies(cli, datafiles, target, expected_state):
project = str(datafiles)
# Configure non strict mode, this will have
# an effect on the build and the `bst show`
# commands run via cli.get_element_states()
- cli.configure({
- 'projects': {
- 'test': {
- 'strict': False
- }
- }
- })
+ cli.configure({"projects": {"test": {"strict": False}}})
- result = cli.run(project=project, silent=True, args=['build', target])
+ result = cli.run(project=project, silent=True, args=["build", target])
result.assert_success()
- states = cli.get_element_states(project, ['base.bst', target])
- assert states['base.bst'] == 'cached'
- assert states[target] == 'cached'
+ states = cli.get_element_states(project, ["base.bst", target])
+ assert states["base.bst"] == "cached"
+ assert states[target] == "cached"
# Now modify the file, effectively causing the common base.bst
# dependency to change it's cache key
- hello_path = os.path.join(project, 'files', 'hello.txt')
- with open(hello_path, 'w') as f:
+ hello_path = os.path.join(project, "files", "hello.txt")
+ with open(hello_path, "w") as f:
f.write("Goodbye")
# Now assert that we have the states we expect as a result
- states = cli.get_element_states(project, ['base.bst', target])
- assert states['base.bst'] == 'buildable'
+ states = cli.get_element_states(project, ["base.bst", target])
+ assert states["base.bst"] == "buildable"
assert states[target] == expected_state
diff --git a/tests/frontend/source_checkout.py b/tests/frontend/source_checkout.py
index 8d6bae83b..624e997cd 100644
--- a/tests/frontend/source_checkout.py
+++ b/tests/frontend/source_checkout.py
@@ -11,23 +11,20 @@ from buildstream.testing import cli # pylint: disable=unused-import
from buildstream import utils, _yaml
# Project directory
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- 'project',
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",)
def generate_remote_import_element(input_path, output_path):
return {
- 'kind': 'import',
- 'sources': [
+ "kind": "import",
+ "sources": [
{
- 'kind': 'remote',
- 'url': 'file://{}'.format(input_path),
- 'filename': output_path,
- 'ref': utils.sha256sum(input_path),
+ "kind": "remote",
+ "url": "file://{}".format(input_path),
+ "filename": output_path,
+ "ref": utils.sha256sum(input_path),
}
- ]
+ ],
}
@@ -35,65 +32,60 @@ def generate_remote_import_element(input_path, output_path):
@pytest.mark.parametrize(
"with_workspace,guess_element",
[(True, True), (True, False), (False, False)],
- ids=["workspace-guess", "workspace-no-guess", "no-workspace-no-guess"]
+ ids=["workspace-guess", "workspace-no-guess", "no-workspace-no-guess"],
)
def test_source_checkout(datafiles, cli, tmpdir_factory, with_workspace, guess_element):
tmpdir = tmpdir_factory.mktemp("")
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'source-checkout')
- target = 'checkout-deps.bst'
- workspace = os.path.join(str(tmpdir), 'workspace')
+ checkout = os.path.join(cli.directory, "source-checkout")
+ target = "checkout-deps.bst"
+ workspace = os.path.join(str(tmpdir), "workspace")
elm_cmd = [target] if not guess_element else []
if with_workspace:
- ws_cmd = ['-C', workspace]
+ ws_cmd = ["-C", workspace]
result = cli.run(project=project, args=["workspace", "open", "--directory", workspace, target])
result.assert_success()
else:
ws_cmd = []
- args = ws_cmd + ['source', 'checkout', '--deps', 'none', '--directory', checkout, *elm_cmd]
+ args = ws_cmd + ["source", "checkout", "--deps", "none", "--directory", checkout, *elm_cmd]
result = cli.run(project=project, args=args)
result.assert_success()
- assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
+ assert os.path.exists(os.path.join(checkout, "checkout-deps", "etc", "buildstream", "config"))
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize('force_flag', ['--force', '-f'])
+@pytest.mark.parametrize("force_flag", ["--force", "-f"])
def test_source_checkout_force(datafiles, cli, force_flag):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'source-checkout')
- target = 'checkout-deps.bst'
+ checkout = os.path.join(cli.directory, "source-checkout")
+ target = "checkout-deps.bst"
# Make the checkout directory with 'some-thing' inside it
- os.makedirs(os.path.join(checkout, 'some-thing'))
+ os.makedirs(os.path.join(checkout, "some-thing"))
- result = cli.run(project=project, args=['source', 'checkout',
- force_flag,
- '--deps', 'none',
- '--directory', checkout,
- target])
+ result = cli.run(
+ project=project, args=["source", "checkout", force_flag, "--deps", "none", "--directory", checkout, target]
+ )
result.assert_success()
- assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
+ assert os.path.exists(os.path.join(checkout, "checkout-deps", "etc", "buildstream", "config"))
@pytest.mark.datafiles(DATA_DIR)
def test_source_checkout_tar(datafiles, cli):
project = str(datafiles)
- tar = os.path.join(cli.directory, 'source-checkout.tar')
- target = 'checkout-deps.bst'
+ tar = os.path.join(cli.directory, "source-checkout.tar")
+ target = "checkout-deps.bst"
- result = cli.run(project=project, args=['source', 'checkout',
- '--tar', tar,
- '--deps', 'none',
- target])
+ result = cli.run(project=project, args=["source", "checkout", "--tar", tar, "--deps", "none", target])
result.assert_success()
assert os.path.exists(tar)
with tarfile.open(tar) as tf:
- expected_content = os.path.join(tar, 'checkout-deps', 'etc', 'buildstream', 'config')
+ expected_content = os.path.join(tar, "checkout-deps", "etc", "buildstream", "config")
tar_members = [f.name for f in tf]
for member in tar_members:
assert member in expected_content
@@ -105,111 +97,106 @@ def test_source_checkout_compressed_tar(datafiles, cli, compression):
project = str(datafiles)
tarfile_name = "source-checkout.tar" + compression
tar = os.path.join(cli.directory, tarfile_name)
- target = 'checkout-deps.bst'
+ target = "checkout-deps.bst"
- result = cli.run(project=project, args=['source', 'checkout',
- '--tar', tar,
- '--compression', compression,
- '--deps', 'none',
- target])
+ result = cli.run(
+ project=project,
+ args=["source", "checkout", "--tar", tar, "--compression", compression, "--deps", "none", target],
+ )
result.assert_success()
- tar = tarfile.open(name=tar, mode='r:' + compression)
- assert os.path.join('checkout-deps', 'etc', 'buildstream', 'config') in tar.getnames()
+ tar = tarfile.open(name=tar, mode="r:" + compression)
+ assert os.path.join("checkout-deps", "etc", "buildstream", "config") in tar.getnames()
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize('deps', [('build'), ('none'), ('run'), ('all')])
+@pytest.mark.parametrize("deps", [("build"), ("none"), ("run"), ("all")])
def test_source_checkout_deps(datafiles, cli, deps):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'source-checkout')
- target = 'checkout-deps.bst'
+ checkout = os.path.join(cli.directory, "source-checkout")
+ target = "checkout-deps.bst"
- result = cli.run(project=project, args=['source', 'checkout',
- '--directory', checkout,
- '--deps', deps,
- target])
+ result = cli.run(project=project, args=["source", "checkout", "--directory", checkout, "--deps", deps, target])
result.assert_success()
# Sources of the target
- if deps == 'build':
- assert not os.path.exists(os.path.join(checkout, 'checkout-deps'))
+ if deps == "build":
+ assert not os.path.exists(os.path.join(checkout, "checkout-deps"))
else:
- assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
+ assert os.path.exists(os.path.join(checkout, "checkout-deps", "etc", "buildstream", "config"))
# Sources of the target's build dependencies
- if deps in ('build', 'all'):
- assert os.path.exists(os.path.join(checkout, 'import-dev', 'usr', 'include', 'pony.h'))
+ if deps in ("build", "all"):
+ assert os.path.exists(os.path.join(checkout, "import-dev", "usr", "include", "pony.h"))
else:
- assert not os.path.exists(os.path.join(checkout, 'import-dev'))
+ assert not os.path.exists(os.path.join(checkout, "import-dev"))
# Sources of the target's runtime dependencies
- if deps in ('run', 'all'):
- assert os.path.exists(os.path.join(checkout, 'import-bin', 'usr', 'bin', 'hello'))
+ if deps in ("run", "all"):
+ assert os.path.exists(os.path.join(checkout, "import-bin", "usr", "bin", "hello"))
else:
- assert not os.path.exists(os.path.join(checkout, 'import-bin'))
+ assert not os.path.exists(os.path.join(checkout, "import-bin"))
@pytest.mark.datafiles(DATA_DIR)
def test_source_checkout_except(datafiles, cli):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'source-checkout')
- target = 'checkout-deps.bst'
-
- result = cli.run(project=project, args=['source', 'checkout',
- '--directory', checkout,
- '--deps', 'all',
- '--except', 'import-bin.bst',
- target])
+ checkout = os.path.join(cli.directory, "source-checkout")
+ target = "checkout-deps.bst"
+
+ result = cli.run(
+ project=project,
+ args=["source", "checkout", "--directory", checkout, "--deps", "all", "--except", "import-bin.bst", target],
+ )
result.assert_success()
# Sources for the target should be present
- assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
+ assert os.path.exists(os.path.join(checkout, "checkout-deps", "etc", "buildstream", "config"))
# Sources for import-bin.bst should not be present
- assert not os.path.exists(os.path.join(checkout, 'import-bin'))
+ assert not os.path.exists(os.path.join(checkout, "import-bin"))
# Sources for other dependencies should be present
- assert os.path.exists(os.path.join(checkout, 'import-dev', 'usr', 'include', 'pony.h'))
+ assert os.path.exists(os.path.join(checkout, "import-dev", "usr", "include", "pony.h"))
@pytest.mark.datafiles(DATA_DIR)
def test_source_checkout_fetch(datafiles, cli):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'source-checkout')
- target = 'remote-import-dev.bst'
- target_path = os.path.join(project, 'elements', target)
+ checkout = os.path.join(cli.directory, "source-checkout")
+ target = "remote-import-dev.bst"
+ target_path = os.path.join(project, "elements", target)
# Create an element with remote source
element = generate_remote_import_element(
- os.path.join(project, 'files', 'dev-files', 'usr', 'include', 'pony.h'),
- 'pony.h')
+ os.path.join(project, "files", "dev-files", "usr", "include", "pony.h"), "pony.h"
+ )
_yaml.roundtrip_dump(element, target_path)
# Testing implicit fetching requires that we do not have the sources
# cached already
- assert cli.get_element_state(project, target) == 'fetch needed'
+ assert cli.get_element_state(project, target) == "fetch needed"
- args = ['source', 'checkout']
+ args = ["source", "checkout"]
args += [target, checkout]
- result = cli.run(project=project, args=['source', 'checkout', '--directory', checkout, target])
+ result = cli.run(project=project, args=["source", "checkout", "--directory", checkout, target])
result.assert_success()
- assert os.path.exists(os.path.join(checkout, 'remote-import-dev', 'pony.h'))
+ assert os.path.exists(os.path.join(checkout, "remote-import-dev", "pony.h"))
@pytest.mark.datafiles(DATA_DIR)
def test_source_checkout_build_scripts(cli, tmpdir, datafiles):
project_path = str(datafiles)
- element_name = 'source-bundle/source-bundle-hello.bst'
- normal_name = 'source-bundle-source-bundle-hello'
- checkout = os.path.join(str(tmpdir), 'source-checkout')
+ element_name = "source-bundle/source-bundle-hello.bst"
+ normal_name = "source-bundle-source-bundle-hello"
+ checkout = os.path.join(str(tmpdir), "source-checkout")
- args = ['source', 'checkout', '--include-build-scripts', '--directory', checkout, element_name]
+ args = ["source", "checkout", "--include-build-scripts", "--directory", checkout, element_name]
result = cli.run(project=project_path, args=args)
result.assert_success()
# There sould be a script for each element (just one in this case) and a top level build script
- expected_scripts = ['build.sh', 'build-' + normal_name]
+ expected_scripts = ["build.sh", "build-" + normal_name]
for script in expected_scripts:
assert script in os.listdir(checkout)
@@ -217,17 +204,17 @@ def test_source_checkout_build_scripts(cli, tmpdir, datafiles):
@pytest.mark.datafiles(DATA_DIR)
def test_source_checkout_tar_buildscripts(cli, tmpdir, datafiles):
project_path = str(datafiles)
- element_name = 'source-bundle/source-bundle-hello.bst'
- normal_name = 'source-bundle-source-bundle-hello'
- tar_file = os.path.join(str(tmpdir), 'source-checkout.tar')
+ element_name = "source-bundle/source-bundle-hello.bst"
+ normal_name = "source-bundle-source-bundle-hello"
+ tar_file = os.path.join(str(tmpdir), "source-checkout.tar")
- args = ['source', 'checkout', '--include-build-scripts', '--tar', tar_file, element_name]
+ args = ["source", "checkout", "--include-build-scripts", "--tar", tar_file, element_name]
result = cli.run(project=project_path, args=args)
result.assert_success()
- expected_scripts = ['build.sh', 'build-' + normal_name]
+ expected_scripts = ["build.sh", "build-" + normal_name]
- with tarfile.open(tar_file, 'r') as tf:
+ with tarfile.open(tar_file, "r") as tf:
for script in expected_scripts:
assert script in tf.getnames()
@@ -236,14 +223,11 @@ def test_source_checkout_tar_buildscripts(cli, tmpdir, datafiles):
@pytest.mark.datafiles(DATA_DIR)
def test_source_checkout_options_tar_and_dir_conflict(cli, tmpdir, datafiles):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'source-checkout')
- tar_file = os.path.join(str(tmpdir), 'source-checkout.tar')
- target = 'checkout-deps.bst'
+ checkout = os.path.join(cli.directory, "source-checkout")
+ tar_file = os.path.join(str(tmpdir), "source-checkout.tar")
+ target = "checkout-deps.bst"
- result = cli.run(project=project, args=['source', 'checkout',
- '--directory', checkout,
- '--tar', tar_file,
- target])
+ result = cli.run(project=project, args=["source", "checkout", "--directory", checkout, "--tar", tar_file, target])
assert result.exit_code != 0
assert "ERROR: options --directory and --tar conflict" in result.stderr
@@ -253,13 +237,12 @@ def test_source_checkout_options_tar_and_dir_conflict(cli, tmpdir, datafiles):
@pytest.mark.datafiles(DATA_DIR)
def test_source_checkout_compression_without_tar(cli, tmpdir, datafiles):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'source-checkout')
- target = 'checkout-deps.bst'
+ checkout = os.path.join(cli.directory, "source-checkout")
+ target = "checkout-deps.bst"
- result = cli.run(project=project, args=['source', 'checkout',
- '--directory', checkout,
- '--compression', 'xz',
- target])
+ result = cli.run(
+ project=project, args=["source", "checkout", "--directory", checkout, "--compression", "xz", target]
+ )
assert result.exit_code != 0
assert "ERROR: --compression specified without --tar" in result.stderr
diff --git a/tests/frontend/track.py b/tests/frontend/track.py
index a628043d8..477c81556 100644
--- a/tests/frontend/track.py
+++ b/tests/frontend/track.py
@@ -14,18 +14,13 @@ from . import configure_project
# Project directory
TOP_DIR = os.path.dirname(os.path.realpath(__file__))
-DATA_DIR = os.path.join(TOP_DIR, 'project')
+DATA_DIR = os.path.join(TOP_DIR, "project")
def generate_element(repo, element_path, dep_name=None):
- element = {
- 'kind': 'import',
- 'sources': [
- repo.source_config()
- ]
- }
+ element = {"kind": "import", "sources": [repo.source_config()]}
if dep_name:
- element['depends'] = [dep_name]
+ element["depends"] = [dep_name]
_yaml.roundtrip_dump(element, element_path)
@@ -33,21 +28,20 @@ def generate_element(repo, element_path, dep_name=None):
@pytest.mark.datafiles(DATA_DIR)
def test_track_single(cli, tmpdir, datafiles):
project = str(datafiles)
- dev_files_path = os.path.join(project, 'files', 'dev-files')
- element_path = os.path.join(project, 'elements')
- element_dep_name = 'track-test-dep.bst'
- element_target_name = 'track-test-target.bst'
+ dev_files_path = os.path.join(project, "files", "dev-files")
+ element_path = os.path.join(project, "elements")
+ element_dep_name = "track-test-dep.bst"
+ element_target_name = "track-test-target.bst"
# Create our repo object of the given source type with
# the dev files, and then collect the initial ref.
#
- repo = create_repo('git', str(tmpdir))
+ repo = create_repo("git", str(tmpdir))
repo.create(dev_files_path)
# Write out our test targets
generate_element(repo, os.path.join(element_path, element_dep_name))
- generate_element(repo, os.path.join(element_path, element_target_name),
- dep_name=element_dep_name)
+ generate_element(repo, os.path.join(element_path, element_target_name), dep_name=element_dep_name)
# Assert that tracking is needed for both elements
states = cli.get_element_states(project, [element_target_name])
@@ -57,71 +51,69 @@ def test_track_single(cli, tmpdir, datafiles):
}
# Now first try to track only one element
- result = cli.run(project=project, args=[
- 'source', 'track', '--deps', 'none',
- element_target_name])
+ result = cli.run(project=project, args=["source", "track", "--deps", "none", element_target_name])
result.assert_success()
# And now fetch it
- result = cli.run(project=project, args=[
- 'source', 'fetch', '--deps', 'none',
- element_target_name])
+ result = cli.run(project=project, args=["source", "fetch", "--deps", "none", element_target_name])
result.assert_success()
# Assert that the dependency is waiting and the target has still never been tracked
states = cli.get_element_states(project, [element_target_name])
assert states == {
- element_dep_name: 'no reference',
- element_target_name: 'waiting',
+ element_dep_name: "no reference",
+ element_target_name: "waiting",
}
@pytest.mark.datafiles(os.path.join(TOP_DIR))
-@pytest.mark.parametrize("ref_storage", [('inline'), ('project-refs')])
+@pytest.mark.parametrize("ref_storage", [("inline"), ("project-refs")])
def test_track_optional(cli, tmpdir, datafiles, ref_storage):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'track-optional-' + ref_storage)
- dev_files_path = os.path.join(project, 'files')
- element_path = os.path.join(project, 'target.bst')
+ project = os.path.join(datafiles.dirname, datafiles.basename, "track-optional-" + ref_storage)
+ dev_files_path = os.path.join(project, "files")
+ element_path = os.path.join(project, "target.bst")
# Create our repo object of the given source type with
# the dev files, and then collect the initial ref.
#
- repo = create_repo('git', str(tmpdir))
+ repo = create_repo("git", str(tmpdir))
repo.create(dev_files_path)
# Now create an optional test branch and add a commit to that,
# so two branches with different heads now exist.
#
- repo.branch('test')
+ repo.branch("test")
repo.add_commit()
# Substitute the {repo} for the git repo we created
with open(element_path) as f:
target_bst = f.read()
target_bst = target_bst.format(repo=repo.repo)
- with open(element_path, 'w') as f:
+ with open(element_path, "w") as f:
f.write(target_bst)
# First track for both options
#
# We want to track and persist the ref separately in this test
#
- result = cli.run(project=project, args=['--option', 'test', 'False', 'source', 'track', 'target.bst'])
+ result = cli.run(project=project, args=["--option", "test", "False", "source", "track", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['--option', 'test', 'True', 'source', 'track', 'target.bst'])
+ result = cli.run(project=project, args=["--option", "test", "True", "source", "track", "target.bst"])
result.assert_success()
# Now fetch the key for both options
#
- result = cli.run(project=project, args=[
- '--option', 'test', 'False', 'show', '--deps', 'none', '--format', '%{key}', 'target.bst'
- ])
+ result = cli.run(
+ project=project,
+ args=["--option", "test", "False", "show", "--deps", "none", "--format", "%{key}", "target.bst"],
+ )
result.assert_success()
master_key = result.output
- result = cli.run(project=project, args=[
- '--option', 'test', 'True', 'show', '--deps', 'none', '--format', '%{key}', 'target.bst'
- ])
+ result = cli.run(
+ project=project,
+ args=["--option", "test", "True", "show", "--deps", "none", "--format", "%{key}", "target.bst"],
+ )
result.assert_success()
test_key = result.output
@@ -130,81 +122,72 @@ def test_track_optional(cli, tmpdir, datafiles, ref_storage):
assert test_key != master_key
-@pytest.mark.datafiles(os.path.join(TOP_DIR, 'track-cross-junction'))
-@pytest.mark.parametrize("cross_junction", [('cross'), ('nocross')])
-@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
+@pytest.mark.datafiles(os.path.join(TOP_DIR, "track-cross-junction"))
+@pytest.mark.parametrize("cross_junction", [("cross"), ("nocross")])
+@pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")])
def test_track_cross_junction(cli, tmpdir, datafiles, cross_junction, ref_storage):
project = str(datafiles)
- dev_files_path = os.path.join(project, 'files')
- target_path = os.path.join(project, 'target.bst')
- subtarget_path = os.path.join(project, 'subproject', 'subtarget.bst')
+ dev_files_path = os.path.join(project, "files")
+ target_path = os.path.join(project, "target.bst")
+ subtarget_path = os.path.join(project, "subproject", "subtarget.bst")
# Create our repo object of the given source type with
# the dev files, and then collect the initial ref.
#
- repo = create_repo('git', str(tmpdir))
+ repo = create_repo("git", str(tmpdir))
repo.create(dev_files_path)
# Generate two elements using the git source, one in
# the main project and one in the subproject.
- generate_element(repo, target_path, dep_name='subproject.bst')
+ generate_element(repo, target_path, dep_name="subproject.bst")
generate_element(repo, subtarget_path)
# Generate project.conf
#
- project_conf = {
- 'name': 'test',
- 'ref-storage': ref_storage
- }
- _yaml.roundtrip_dump(project_conf, os.path.join(project, 'project.conf'))
+ project_conf = {"name": "test", "ref-storage": ref_storage}
+ _yaml.roundtrip_dump(project_conf, os.path.join(project, "project.conf"))
#
# FIXME: This can be simplified when we have support
# for addressing of junctioned elements.
#
def get_subproject_element_state():
- result = cli.run(project=project, args=[
- 'show', '--deps', 'all',
- '--format', '%{name}|%{state}', 'target.bst'
- ])
+ result = cli.run(project=project, args=["show", "--deps", "all", "--format", "%{name}|%{state}", "target.bst"])
result.assert_success()
# Create two dimentional list of the result,
# first line should be the junctioned element
- lines = [
- line.split('|')
- for line in result.output.splitlines()
- ]
- assert lines[0][0] == 'subproject-junction.bst:subtarget.bst'
+ lines = [line.split("|") for line in result.output.splitlines()]
+ assert lines[0][0] == "subproject-junction.bst:subtarget.bst"
return lines[0][1]
#
# Assert that we have no reference yet for the cross junction element
#
- assert get_subproject_element_state() == 'no reference'
+ assert get_subproject_element_state() == "no reference"
# Track recursively across the junction
- args = ['source', 'track', '--deps', 'all']
- if cross_junction == 'cross':
- args += ['--cross-junctions']
- args += ['target.bst']
+ args = ["source", "track", "--deps", "all"]
+ if cross_junction == "cross":
+ args += ["--cross-junctions"]
+ args += ["target.bst"]
result = cli.run(project=project, args=args)
- if ref_storage == 'inline':
+ if ref_storage == "inline":
- if cross_junction == 'cross':
+ if cross_junction == "cross":
#
# Cross junction tracking is not allowed when the toplevel project
# is using inline ref storage.
#
- result.assert_main_error(ErrorDomain.PIPELINE, 'untrackable-sources')
+ result.assert_main_error(ErrorDomain.PIPELINE, "untrackable-sources")
else:
#
# No cross juction tracking was requested
#
result.assert_success()
- assert get_subproject_element_state() == 'no reference'
+ assert get_subproject_element_state() == "no reference"
else:
#
# Tracking is allowed with project.refs ref storage
@@ -214,117 +197,97 @@ def test_track_cross_junction(cli, tmpdir, datafiles, cross_junction, ref_storag
#
# If cross junction tracking was enabled, we should now be buildable
#
- if cross_junction == 'cross':
- assert get_subproject_element_state() == 'buildable'
+ if cross_junction == "cross":
+ assert get_subproject_element_state() == "buildable"
else:
- assert get_subproject_element_state() == 'no reference'
+ assert get_subproject_element_state() == "no reference"
-@pytest.mark.datafiles(os.path.join(TOP_DIR, 'consistencyerror'))
+@pytest.mark.datafiles(os.path.join(TOP_DIR, "consistencyerror"))
def test_track_consistency_error(cli, datafiles):
project = str(datafiles)
# Track the element causing a consistency error
- result = cli.run(project=project, args=['source', 'track', 'error.bst'])
+ result = cli.run(project=project, args=["source", "track", "error.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
- result.assert_task_error(ErrorDomain.SOURCE, 'the-consistency-error')
+ result.assert_task_error(ErrorDomain.SOURCE, "the-consistency-error")
-@pytest.mark.datafiles(os.path.join(TOP_DIR, 'consistencyerror'))
+@pytest.mark.datafiles(os.path.join(TOP_DIR, "consistencyerror"))
def test_track_consistency_bug(cli, datafiles):
project = str(datafiles)
# Track the element causing an unhandled exception
- result = cli.run(project=project, args=['source', 'track', 'bug.bst'])
+ result = cli.run(project=project, args=["source", "track", "bug.bst"])
# We expect BuildStream to fail gracefully, with no recorded exception.
result.assert_main_error(ErrorDomain.STREAM, None)
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
+@pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")])
def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage):
project = str(datafiles)
- subproject_path = os.path.join(project, 'files', 'sub-project')
- junction_path = os.path.join(project, 'elements', 'junction.bst')
- element_path = os.path.join(project, 'elements', 'junction-dep.bst')
+ subproject_path = os.path.join(project, "files", "sub-project")
+ junction_path = os.path.join(project, "elements", "junction.bst")
+ element_path = os.path.join(project, "elements", "junction-dep.bst")
- configure_project(project, {
- 'ref-storage': ref_storage
- })
+ configure_project(project, {"ref-storage": ref_storage})
# Create a repo to hold the subproject and generate a junction element for it
generate_junction(tmpdir, subproject_path, junction_path, store_ref=False)
# Create a stack element to depend on a cross junction element
#
- element = {
- 'kind': 'stack',
- 'depends': [
- {
- 'junction': 'junction.bst',
- 'filename': 'import-etc.bst'
- }
- ]
- }
+ element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]}
_yaml.roundtrip_dump(element, element_path)
# Now try to track it, this will bail with the appropriate error
# informing the user to track the junction first
- result = cli.run(project=project, args=['source', 'track', 'junction-dep.bst'])
+ result = cli.run(project=project, args=["source", "track", "junction-dep.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT)
# Assert that we have the expected provenance encoded into the error
- element_node = _yaml.load(element_path, shortname='junction-dep.bst')
- ref_node = element_node.get_sequence('depends').mapping_at(0)
+ element_node = _yaml.load(element_path, shortname="junction-dep.bst")
+ ref_node = element_node.get_sequence("depends").mapping_at(0)
provenance = ref_node.get_provenance()
assert str(provenance) in result.stderr
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
+@pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")])
def test_junction_element(cli, tmpdir, datafiles, ref_storage):
project = str(datafiles)
- subproject_path = os.path.join(project, 'files', 'sub-project')
- junction_path = os.path.join(project, 'elements', 'junction.bst')
- element_path = os.path.join(project, 'elements', 'junction-dep.bst')
+ subproject_path = os.path.join(project, "files", "sub-project")
+ junction_path = os.path.join(project, "elements", "junction.bst")
+ element_path = os.path.join(project, "elements", "junction-dep.bst")
- configure_project(project, {
- 'ref-storage': ref_storage
- })
+ configure_project(project, {"ref-storage": ref_storage})
# Create a repo to hold the subproject and generate a junction element for it
generate_junction(tmpdir, subproject_path, junction_path, store_ref=False)
# Create a stack element to depend on a cross junction element
#
- element = {
- 'kind': 'stack',
- 'depends': [
- {
- 'junction': 'junction.bst',
- 'filename': 'import-etc.bst'
- }
- ]
- }
+ element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]}
_yaml.roundtrip_dump(element, element_path)
# First demonstrate that showing the pipeline yields an error
- result = cli.run(project=project, args=['show', 'junction-dep.bst'])
+ result = cli.run(project=project, args=["show", "junction-dep.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT)
# Assert that we have the expected provenance encoded into the error
- element_node = _yaml.load(element_path, shortname='junction-dep.bst')
- ref_node = element_node.get_sequence('depends').mapping_at(0)
+ element_node = _yaml.load(element_path, shortname="junction-dep.bst")
+ ref_node = element_node.get_sequence("depends").mapping_at(0)
provenance = ref_node.get_provenance()
assert str(provenance) in result.stderr
# Now track the junction itself
- result = cli.run(project=project, args=['source', 'track', 'junction.bst'])
+ result = cli.run(project=project, args=["source", "track", "junction.bst"])
result.assert_success()
# Now assert element state (via bst show under the hood) of the dep again
- assert cli.get_element_state(project, 'junction-dep.bst') == 'waiting'
+ assert cli.get_element_state(project, "junction-dep.bst") == "waiting"
@pytest.mark.datafiles(DATA_DIR)
@@ -333,15 +296,13 @@ def test_track_error_cannot_write_file(cli, tmpdir, datafiles):
pytest.skip("This is not testable with root permissions")
project = str(datafiles)
- dev_files_path = os.path.join(project, 'files', 'dev-files')
- element_path = os.path.join(project, 'elements')
- element_name = 'track-test.bst'
+ dev_files_path = os.path.join(project, "files", "dev-files")
+ element_path = os.path.join(project, "elements")
+ element_name = "track-test.bst"
- configure_project(project, {
- 'ref-storage': 'inline'
- })
+ configure_project(project, {"ref-storage": "inline"})
- repo = create_repo('git', str(tmpdir))
+ repo = create_repo("git", str(tmpdir))
repo.create(dev_files_path)
element_full_path = os.path.join(element_path, element_name)
@@ -352,9 +313,9 @@ def test_track_error_cannot_write_file(cli, tmpdir, datafiles):
read_mask = stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH
os.chmod(element_path, stat.S_IMODE(st.st_mode) & ~read_mask)
- result = cli.run(project=project, args=['source', 'track', element_name])
+ result = cli.run(project=project, args=["source", "track", element_name])
result.assert_main_error(ErrorDomain.STREAM, None)
- result.assert_task_error(ErrorDomain.SOURCE, 'save-ref-error')
+ result.assert_task_error(ErrorDomain.SOURCE, "save-ref-error")
finally:
os.chmod(element_path, stat.S_IMODE(st.st_mode))
@@ -362,14 +323,14 @@ def test_track_error_cannot_write_file(cli, tmpdir, datafiles):
@pytest.mark.datafiles(DATA_DIR)
def test_no_needless_overwrite(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
- dev_files_path = os.path.join(project, 'files', 'dev-files')
- element_path = os.path.join(project, 'elements')
- target = 'track-test-target.bst'
+ dev_files_path = os.path.join(project, "files", "dev-files")
+ element_path = os.path.join(project, "elements")
+ target = "track-test-target.bst"
# Create our repo object of the given source type with
# the dev files, and then collect the initial ref.
#
- repo = create_repo('git', str(tmpdir))
+ repo = create_repo("git", str(tmpdir))
repo.create(dev_files_path)
# Write out our test target and assert it exists
@@ -380,10 +341,10 @@ def test_no_needless_overwrite(cli, tmpdir, datafiles):
# Assert tracking is needed
states = cli.get_element_states(project, [target])
- assert states[target] == 'no reference'
+ assert states[target] == "no reference"
# Perform the track
- result = cli.run(project=project, args=['source', 'track', target])
+ result = cli.run(project=project, args=["source", "track", target])
result.assert_success()
track1_mtime = os.path.getmtime(path_to_target)
@@ -391,7 +352,7 @@ def test_no_needless_overwrite(cli, tmpdir, datafiles):
assert creation_mtime != track1_mtime
# Now (needlessly) track again
- result = cli.run(project=project, args=['source', 'track', target])
+ result = cli.run(project=project, args=["source", "track", target])
result.assert_success()
track2_mtime = os.path.getmtime(path_to_target)
diff --git a/tests/frontend/version.py b/tests/frontend/version.py
index e7db19915..279a51747 100644
--- a/tests/frontend/version.py
+++ b/tests/frontend/version.py
@@ -12,13 +12,13 @@ def assert_version(cli_version_output):
major, minor = utils.get_bst_version()
expected_start = "{}.{}".format(major, minor)
if not cli_version_output.startswith(expected_start):
- raise AssertionError("Version output expected to begin with '{}',"
- .format(expected_start) +
- " output was: {}"
- .format(cli_version_output))
+ raise AssertionError(
+ "Version output expected to begin with '{}',".format(expected_start)
+ + " output was: {}".format(cli_version_output)
+ )
def test_version(cli):
- result = cli.run(args=['--version'])
+ result = cli.run(args=["--version"])
result.assert_success()
assert_version(result.output)
diff --git a/tests/frontend/workspace.py b/tests/frontend/workspace.py
index 0c7e63525..80db8cc98 100644
--- a/tests/frontend/workspace.py
+++ b/tests/frontend/workspace.py
@@ -45,13 +45,10 @@ from tests.testutils import create_artifact_share, create_element_size, wait_for
repo_kinds = [(kind) for kind in ALL_REPO_KINDS]
# Project directory
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project",
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",)
-class WorkspaceCreator():
+class WorkspaceCreator:
def __init__(self, cli, tmpdir, datafiles, project_path=None):
self.cli = cli
self.tmpdir = tmpdir
@@ -63,17 +60,16 @@ class WorkspaceCreator():
shutil.copytree(str(datafiles), project_path)
self.project_path = project_path
- self.bin_files_path = os.path.join(project_path, 'files', 'bin-files')
+ self.bin_files_path = os.path.join(project_path, "files", "bin-files")
- self.workspace_cmd = os.path.join(self.project_path, 'workspace_cmd')
+ self.workspace_cmd = os.path.join(self.project_path, "workspace_cmd")
- def create_workspace_element(self, kind, suffix='', workspace_dir=None,
- element_attrs=None):
- element_name = 'workspace-test-{}{}.bst'.format(kind, suffix)
- element_path = os.path.join(self.project_path, 'elements')
+ def create_workspace_element(self, kind, suffix="", workspace_dir=None, element_attrs=None):
+ element_name = "workspace-test-{}{}.bst".format(kind, suffix)
+ element_path = os.path.join(self.project_path, "elements")
if not workspace_dir:
workspace_dir = os.path.join(self.workspace_cmd, element_name)
- if workspace_dir[-4:] == '.bst':
+ if workspace_dir[-4:] == ".bst":
workspace_dir = workspace_dir[:-4]
# Create our repo object of the given source type with
@@ -82,59 +78,48 @@ class WorkspaceCreator():
ref = repo.create(self.bin_files_path)
# Write out our test target
- element = {
- 'kind': 'import',
- 'sources': [
- repo.source_config(ref=ref)
- ]
- }
+ element = {"kind": "import", "sources": [repo.source_config(ref=ref)]}
if element_attrs:
element = {**element, **element_attrs}
- _yaml.roundtrip_dump(element,
- os.path.join(element_path, element_name))
+ _yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
return element_name, element_path, workspace_dir
- def create_workspace_elements(self, kinds, suffixs=None, workspace_dir_usr=None,
- element_attrs=None):
+ def create_workspace_elements(self, kinds, suffixs=None, workspace_dir_usr=None, element_attrs=None):
element_tuples = []
if suffixs is None:
- suffixs = ['', ] * len(kinds)
+ suffixs = ["",] * len(kinds)
else:
if len(suffixs) != len(kinds):
raise "terable error"
for suffix, kind in zip(suffixs, kinds):
- element_name, _, workspace_dir = \
- self.create_workspace_element(kind, suffix, workspace_dir_usr,
- element_attrs)
+ element_name, _, workspace_dir = self.create_workspace_element(
+ kind, suffix, workspace_dir_usr, element_attrs
+ )
element_tuples.append((element_name, workspace_dir))
# Assert that there is a fetch is needed
- states = self.cli.get_element_states(self.project_path, [
- e for e, _ in element_tuples
- ])
- assert not any(states[e] != 'fetch needed' for e, _ in element_tuples)
+ states = self.cli.get_element_states(self.project_path, [e for e, _ in element_tuples])
+ assert not any(states[e] != "fetch needed" for e, _ in element_tuples)
return element_tuples
- def open_workspaces(self, kinds, suffixs=None, workspace_dir=None,
- element_attrs=None, no_checkout=False):
+ def open_workspaces(self, kinds, suffixs=None, workspace_dir=None, element_attrs=None, no_checkout=False):
- element_tuples = self.create_workspace_elements(kinds, suffixs, workspace_dir,
- element_attrs)
+ element_tuples = self.create_workspace_elements(kinds, suffixs, workspace_dir, element_attrs)
os.makedirs(self.workspace_cmd, exist_ok=True)
# Now open the workspace, this should have the effect of automatically
# tracking & fetching the source from the repo.
- args = ['workspace', 'open']
+ args = ["workspace", "open"]
if no_checkout:
- args.append('--no-checkout')
+ args.append("--no-checkout")
if workspace_dir is not None:
assert len(element_tuples) == 1, "test logic error"
_, workspace_dir = element_tuples[0]
- args.extend(['--directory', workspace_dir])
+ args.extend(["--directory", workspace_dir])
args.extend([element_name for element_name, workspace_dir_suffix in element_tuples])
result = self.cli.run(cwd=self.workspace_cmd, project=self.project_path, args=args)
@@ -143,24 +128,30 @@ class WorkspaceCreator():
if not no_checkout:
# Assert that we are now buildable because the source is now cached.
- states = self.cli.get_element_states(self.project_path, [
- e for e, _ in element_tuples
- ])
- assert not any(states[e] != 'buildable' for e, _ in element_tuples)
+ states = self.cli.get_element_states(self.project_path, [e for e, _ in element_tuples])
+ assert not any(states[e] != "buildable" for e, _ in element_tuples)
# Check that the executable hello file is found in each workspace
for _, workspace in element_tuples:
- filename = os.path.join(workspace, 'usr', 'bin', 'hello')
+ filename = os.path.join(workspace, "usr", "bin", "hello")
assert os.path.exists(filename)
return element_tuples
-def open_workspace(cli, tmpdir, datafiles, kind, suffix='', workspace_dir=None,
- project_path=None, element_attrs=None, no_checkout=False):
+def open_workspace(
+ cli,
+ tmpdir,
+ datafiles,
+ kind,
+ suffix="",
+ workspace_dir=None,
+ project_path=None,
+ element_attrs=None,
+ no_checkout=False,
+):
workspace_object = WorkspaceCreator(cli, tmpdir, datafiles, project_path)
- workspaces = workspace_object.open_workspaces((kind, ), (suffix, ), workspace_dir,
- element_attrs, no_checkout)
+ workspaces = workspace_object.open_workspaces((kind,), (suffix,), workspace_dir, element_attrs, no_checkout)
assert len(workspaces) == 1
element_name, workspace = workspaces[0]
return element_name, workspace_object.project_path, workspace
@@ -176,11 +167,10 @@ def test_open_bzr_customize(cli, tmpdir, datafiles):
# Check that the correct origin branch is set
element_config = _yaml.load(os.path.join(project, "elements", element_name))
- source_config = element_config.get_sequence('sources').mapping_at(0)
+ source_config = element_config.get_sequence("sources").mapping_at(0)
output = subprocess.check_output(["bzr", "info"], cwd=workspace)
- stripped_url = source_config.get_str('url').lstrip("file:///")
- expected_output_str = ("checkout of branch: /{}/{}"
- .format(stripped_url, source_config.get_str('track')))
+ stripped_url = source_config.get_str("url").lstrip("file:///")
+ expected_output_str = "checkout of branch: /{}/{}".format(stripped_url, source_config.get_str("track"))
assert expected_output_str in str(output)
@@ -193,13 +183,13 @@ def test_open_multi(cli, tmpdir, datafiles):
for (elname, workspace), kind in zip(workspaces, repo_kinds):
assert kind in elname
workspace_lsdir = os.listdir(workspace)
- if kind == 'git':
- assert '.git' in workspace_lsdir
- elif kind == 'bzr':
- assert '.bzr' in workspace_lsdir
+ if kind == "git":
+ assert ".git" in workspace_lsdir
+ elif kind == "bzr":
+ assert ".bzr" in workspace_lsdir
else:
- assert '.git' not in workspace_lsdir
- assert '.bzr' not in workspace_lsdir
+ assert ".git" not in workspace_lsdir
+ assert ".bzr" not in workspace_lsdir
@pytest.mark.skipif(os.geteuid() == 0, reason="root may have CAP_DAC_OVERRIDE and ignore permissions")
@@ -212,9 +202,9 @@ def test_open_multi_unwritable(cli, tmpdir, datafiles):
# Now open the workspace, this should have the effect of automatically
# tracking & fetching the source from the repo.
- args = ['workspace', 'open']
+ args = ["workspace", "open"]
args.extend([element_name for element_name, workspace_dir_suffix in element_tuples])
- cli.configure({'workspacedir': workspace_object.workspace_cmd})
+ cli.configure({"workspacedir": workspace_object.workspace_cmd})
cwdstat = os.stat(workspace_object.workspace_cmd)
try:
@@ -239,14 +229,15 @@ def test_open_multi_with_directory(cli, tmpdir, datafiles):
# Now open the workspace, this should have the effect of automatically
# tracking & fetching the source from the repo.
- args = ['workspace', 'open']
- args.extend(['--directory', 'any/dir/should/fail'])
+ args = ["workspace", "open"]
+ args.extend(["--directory", "any/dir/should/fail"])
args.extend([element_name for element_name, workspace_dir_suffix in element_tuples])
- result = workspace_object.cli.run(cwd=workspace_object.workspace_cmd, project=workspace_object.project_path,
- args=args)
+ result = workspace_object.cli.run(
+ cwd=workspace_object.workspace_cmd, project=workspace_object.project_path, args=args
+ )
- result.assert_main_error(ErrorDomain.STREAM, 'directory-with-multiple-elements')
+ result.assert_main_error(ErrorDomain.STREAM, "directory-with-multiple-elements")
@pytest.mark.datafiles(DATA_DIR)
@@ -254,31 +245,30 @@ def test_open_defaultlocation(cli, tmpdir, datafiles):
workspace_object = WorkspaceCreator(cli, tmpdir, datafiles)
# pylint: disable=unbalanced-tuple-unpacking
- ((element_name, workspace_dir), ) = workspace_object.create_workspace_elements(['git'], ['git'])
+ ((element_name, workspace_dir),) = workspace_object.create_workspace_elements(["git"], ["git"])
os.makedirs(workspace_object.workspace_cmd, exist_ok=True)
# Now open the workspace, this should have the effect of automatically
# tracking & fetching the source from the repo.
- args = ['workspace', 'open']
+ args = ["workspace", "open"]
args.append(element_name)
# In the other tests we set the cmd to workspace_object.workspace_cmd with the optional
# argument, cwd for the workspace_object.cli.run function. But hear we set the default
# workspace location to workspace_object.workspace_cmd and run the cli.run function with
# no cwd option so that it runs in the project directory.
- cli.configure({'workspacedir': workspace_object.workspace_cmd})
- result = workspace_object.cli.run(project=workspace_object.project_path,
- args=args)
+ cli.configure({"workspacedir": workspace_object.workspace_cmd})
+ result = workspace_object.cli.run(project=workspace_object.project_path, args=args)
result.assert_success()
- assert cli.get_element_state(workspace_object.project_path, element_name) == 'buildable'
+ assert cli.get_element_state(workspace_object.project_path, element_name) == "buildable"
# Check that the executable hello file is found in the workspace
# even though the cli.run function was not run with cwd = workspace_object.workspace_cmd
# the workspace should be created in there as we used the 'workspacedir' configuration
# option.
- filename = os.path.join(workspace_dir, 'usr', 'bin', 'hello')
+ filename = os.path.join(workspace_dir, "usr", "bin", "hello")
assert os.path.exists(filename)
@@ -287,117 +277,110 @@ def test_open_defaultlocation_exists(cli, tmpdir, datafiles):
workspace_object = WorkspaceCreator(cli, tmpdir, datafiles)
# pylint: disable=unbalanced-tuple-unpacking
- ((element_name, workspace_dir), ) = workspace_object.create_workspace_elements(['git'], ['git'])
+ ((element_name, workspace_dir),) = workspace_object.create_workspace_elements(["git"], ["git"])
os.makedirs(workspace_object.workspace_cmd, exist_ok=True)
- with open(workspace_dir, 'w') as fl:
- fl.write('foo')
+ with open(workspace_dir, "w") as fl:
+ fl.write("foo")
# Now open the workspace, this should have the effect of automatically
# tracking & fetching the source from the repo.
- args = ['workspace', 'open']
+ args = ["workspace", "open"]
args.append(element_name)
# In the other tests we set the cmd to workspace_object.workspace_cmd with the optional
# argument, cwd for the workspace_object.cli.run function. But hear we set the default
# workspace location to workspace_object.workspace_cmd and run the cli.run function with
# no cwd option so that it runs in the project directory.
- cli.configure({'workspacedir': workspace_object.workspace_cmd})
- result = workspace_object.cli.run(project=workspace_object.project_path,
- args=args)
+ cli.configure({"workspacedir": workspace_object.workspace_cmd})
+ result = workspace_object.cli.run(project=workspace_object.project_path, args=args)
- result.assert_main_error(ErrorDomain.STREAM, 'bad-directory')
+ result.assert_main_error(ErrorDomain.STREAM, "bad-directory")
@pytest.mark.datafiles(DATA_DIR)
def test_open_track(cli, tmpdir, datafiles):
- open_workspace(cli, tmpdir, datafiles, 'git')
+ open_workspace(cli, tmpdir, datafiles, "git")
@pytest.mark.datafiles(DATA_DIR)
def test_open_noclose_open(cli, tmpdir, datafiles):
# opening the same workspace twice without closing it should fail
- element_name, project, _ = open_workspace(cli, tmpdir, datafiles, 'git')
+ element_name, project, _ = open_workspace(cli, tmpdir, datafiles, "git")
- result = cli.run(project=project, args=['workspace', 'open', element_name])
+ result = cli.run(project=project, args=["workspace", "open", element_name])
result.assert_main_error(ErrorDomain.STREAM, None)
@pytest.mark.datafiles(DATA_DIR)
def test_open_force(cli, tmpdir, datafiles):
- element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git')
+ element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git")
# Close the workspace
- result = cli.run(project=project, args=[
- 'workspace', 'close', element_name
- ])
+ result = cli.run(project=project, args=["workspace", "close", element_name])
result.assert_success()
# Assert the workspace dir still exists
assert os.path.exists(workspace)
# Now open the workspace again with --force, this should happily succeed
- result = cli.run(project=project, args=[
- 'workspace', 'open', '--force', '--directory', workspace, element_name
- ])
+ result = cli.run(project=project, args=["workspace", "open", "--force", "--directory", workspace, element_name])
result.assert_success()
@pytest.mark.datafiles(DATA_DIR)
def test_open_force_open(cli, tmpdir, datafiles):
- element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git')
+ element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git")
- result = cli.run(project=project, args=['workspace', 'close', element_name])
+ result = cli.run(project=project, args=["workspace", "close", element_name])
result.assert_success()
# Assert the workspace dir exists
assert os.path.exists(workspace)
# Now open the workspace again with --force, this should happily succeed
- result = cli.run(project=project, args=[
- 'workspace', 'open', '--force', '--directory', workspace, element_name
- ])
+ result = cli.run(project=project, args=["workspace", "open", "--force", "--directory", workspace, element_name])
result.assert_success()
# Regression test for #1086.
@pytest.mark.datafiles(DATA_DIR)
def test_open_force_open_no_checkout(cli, tmpdir, datafiles):
- element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git')
- hello_path = os.path.join(workspace, 'hello.txt')
+ element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git")
+ hello_path = os.path.join(workspace, "hello.txt")
# Assert the workspace dir exists
assert os.path.exists(workspace)
# Create a new file in the workspace
- with open(hello_path, 'w') as f:
- f.write('hello')
+ with open(hello_path, "w") as f:
+ f.write("hello")
# Now open the workspace again with --force and --no-checkout
- result = cli.run(project=project, args=[
- 'workspace', 'open', '--force', '--no-checkout', '--directory', workspace, element_name
- ])
+ result = cli.run(
+ project=project, args=["workspace", "open", "--force", "--no-checkout", "--directory", workspace, element_name]
+ )
result.assert_success()
# Ensure that our files were not overwritten
assert os.path.exists(hello_path)
with open(hello_path) as f:
- assert f.read() == 'hello'
+ assert f.read() == "hello"
@pytest.mark.datafiles(DATA_DIR)
def test_open_force_different_workspace(cli, tmpdir, datafiles):
- _, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git', "-alpha")
+ _, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", "-alpha")
# Assert the workspace dir exists
assert os.path.exists(workspace)
- hello_path = os.path.join(workspace, 'usr', 'bin', 'hello')
- hello1_path = os.path.join(workspace, 'usr', 'bin', 'hello1')
+ hello_path = os.path.join(workspace, "usr", "bin", "hello")
+ hello1_path = os.path.join(workspace, "usr", "bin", "hello1")
tmpdir = os.path.join(str(tmpdir), "-beta")
shutil.move(hello_path, hello1_path)
- element_name2, _, workspace2 = open_workspace(cli, tmpdir, datafiles, 'git', "-beta")
+ element_name2, _, workspace2 = open_workspace(cli, tmpdir, datafiles, "git", "-beta")
# Assert the workspace dir exists
assert os.path.exists(workspace2)
@@ -406,15 +389,13 @@ def test_open_force_different_workspace(cli, tmpdir, datafiles):
assert os.path.exists(hello1_path)
# Assert that workspace 2 contains the unmodified file
- assert os.path.exists(os.path.join(workspace2, 'usr', 'bin', 'hello'))
+ assert os.path.exists(os.path.join(workspace2, "usr", "bin", "hello"))
- result = cli.run(project=project, args=['workspace', 'close', element_name2])
+ result = cli.run(project=project, args=["workspace", "close", element_name2])
result.assert_success()
# Now open the workspace again with --force, this should happily succeed
- result = cli.run(project=project, args=[
- 'workspace', 'open', '--force', '--directory', workspace, element_name2
- ])
+ result = cli.run(project=project, args=["workspace", "open", "--force", "--directory", workspace, element_name2])
result.assert_success()
# Assert that the file in workspace 1 has been replaced
@@ -425,12 +406,10 @@ def test_open_force_different_workspace(cli, tmpdir, datafiles):
@pytest.mark.datafiles(DATA_DIR)
def test_close(cli, tmpdir, datafiles):
- element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git')
+ element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git")
# Close the workspace
- result = cli.run(project=project, args=[
- 'workspace', 'close', '--remove-dir', element_name
- ])
+ result = cli.run(project=project, args=["workspace", "close", "--remove-dir", element_name])
result.assert_success()
# Assert the workspace dir has been deleted
@@ -440,17 +419,15 @@ def test_close(cli, tmpdir, datafiles):
@pytest.mark.datafiles(DATA_DIR)
def test_close_external_after_move_project(cli, tmpdir, datafiles):
workspace_dir = os.path.join(str(tmpdir), "workspace")
- project_path = os.path.join(str(tmpdir), 'initial_project')
- element_name, _, _ = open_workspace(cli, tmpdir, datafiles, 'git', "", workspace_dir, project_path)
+ project_path = os.path.join(str(tmpdir), "initial_project")
+ element_name, _, _ = open_workspace(cli, tmpdir, datafiles, "git", "", workspace_dir, project_path)
assert os.path.exists(workspace_dir)
- moved_dir = os.path.join(str(tmpdir), 'external_project')
+ moved_dir = os.path.join(str(tmpdir), "external_project")
shutil.move(project_path, moved_dir)
assert os.path.exists(moved_dir)
# Close the workspace
- result = cli.run(project=moved_dir, args=[
- 'workspace', 'close', '--remove-dir', element_name
- ])
+ result = cli.run(project=moved_dir, args=["workspace", "close", "--remove-dir", element_name])
result.assert_success()
# Assert the workspace dir has been deleted
@@ -459,36 +436,33 @@ def test_close_external_after_move_project(cli, tmpdir, datafiles):
@pytest.mark.datafiles(DATA_DIR)
def test_close_internal_after_move_project(cli, tmpdir, datafiles):
- initial_dir = os.path.join(str(tmpdir), 'initial_project')
- initial_workspace = os.path.join(initial_dir, 'workspace')
- element_name, _, _ = open_workspace(cli, tmpdir, datafiles, 'git',
- workspace_dir=initial_workspace, project_path=initial_dir)
- moved_dir = os.path.join(str(tmpdir), 'internal_project')
+ initial_dir = os.path.join(str(tmpdir), "initial_project")
+ initial_workspace = os.path.join(initial_dir, "workspace")
+ element_name, _, _ = open_workspace(
+ cli, tmpdir, datafiles, "git", workspace_dir=initial_workspace, project_path=initial_dir
+ )
+ moved_dir = os.path.join(str(tmpdir), "internal_project")
shutil.move(initial_dir, moved_dir)
assert os.path.exists(moved_dir)
# Close the workspace
- result = cli.run(project=moved_dir, args=[
- 'workspace', 'close', '--remove-dir', element_name
- ])
+ result = cli.run(project=moved_dir, args=["workspace", "close", "--remove-dir", element_name])
result.assert_success()
# Assert the workspace dir has been deleted
- workspace = os.path.join(moved_dir, 'workspace')
+ workspace = os.path.join(moved_dir, "workspace")
assert not os.path.exists(workspace)
@pytest.mark.datafiles(DATA_DIR)
def test_close_removed(cli, tmpdir, datafiles):
- element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git')
+ element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git")
# Remove it first, closing the workspace should work
shutil.rmtree(workspace)
# Close the workspace
- result = cli.run(project=project, args=[
- 'workspace', 'close', element_name
- ])
+ result = cli.run(project=project, args=["workspace", "close", element_name])
result.assert_success()
# Assert the workspace dir has been deleted
@@ -497,8 +471,8 @@ def test_close_removed(cli, tmpdir, datafiles):
@pytest.mark.datafiles(DATA_DIR)
def test_close_nonexistant_element(cli, tmpdir, datafiles):
- element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git')
- element_path = os.path.join(datafiles.dirname, datafiles.basename, 'elements', element_name)
+ element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git")
+ element_path = os.path.join(datafiles.dirname, datafiles.basename, "elements", element_name)
# First brutally remove the element.bst file, ensuring that
# the element does not exist anymore in the project where
@@ -506,9 +480,7 @@ def test_close_nonexistant_element(cli, tmpdir, datafiles):
os.remove(element_path)
# Close the workspace
- result = cli.run(project=project, args=[
- 'workspace', 'close', '--remove-dir', element_name
- ])
+ result = cli.run(project=project, args=["workspace", "close", "--remove-dir", element_name])
result.assert_success()
# Assert the workspace dir has been deleted
@@ -517,17 +489,13 @@ def test_close_nonexistant_element(cli, tmpdir, datafiles):
@pytest.mark.datafiles(DATA_DIR)
def test_close_multiple(cli, tmpdir, datafiles):
- tmpdir_alpha = os.path.join(str(tmpdir), 'alpha')
- tmpdir_beta = os.path.join(str(tmpdir), 'beta')
- alpha, project, workspace_alpha = open_workspace(
- cli, tmpdir_alpha, datafiles, 'git', suffix='-alpha')
- beta, project, workspace_beta = open_workspace(
- cli, tmpdir_beta, datafiles, 'git', suffix='-beta')
+ tmpdir_alpha = os.path.join(str(tmpdir), "alpha")
+ tmpdir_beta = os.path.join(str(tmpdir), "beta")
+ alpha, project, workspace_alpha = open_workspace(cli, tmpdir_alpha, datafiles, "git", suffix="-alpha")
+ beta, project, workspace_beta = open_workspace(cli, tmpdir_beta, datafiles, "git", suffix="-beta")
# Close the workspaces
- result = cli.run(project=project, args=[
- 'workspace', 'close', '--remove-dir', alpha, beta
- ])
+ result = cli.run(project=project, args=["workspace", "close", "--remove-dir", alpha, beta])
result.assert_success()
# Assert the workspace dirs have been deleted
@@ -537,17 +505,13 @@ def test_close_multiple(cli, tmpdir, datafiles):
@pytest.mark.datafiles(DATA_DIR)
def test_close_all(cli, tmpdir, datafiles):
- tmpdir_alpha = os.path.join(str(tmpdir), 'alpha')
- tmpdir_beta = os.path.join(str(tmpdir), 'beta')
- _, project, workspace_alpha = open_workspace(
- cli, tmpdir_alpha, datafiles, 'git', suffix='-alpha')
- _, project, workspace_beta = open_workspace(
- cli, tmpdir_beta, datafiles, 'git', suffix='-beta')
+ tmpdir_alpha = os.path.join(str(tmpdir), "alpha")
+ tmpdir_beta = os.path.join(str(tmpdir), "beta")
+ _, project, workspace_alpha = open_workspace(cli, tmpdir_alpha, datafiles, "git", suffix="-alpha")
+ _, project, workspace_beta = open_workspace(cli, tmpdir_beta, datafiles, "git", suffix="-beta")
# Close the workspaces
- result = cli.run(project=project, args=[
- 'workspace', 'close', '--remove-dir', '--all'
- ])
+ result = cli.run(project=project, args=["workspace", "close", "--remove-dir", "--all"])
result.assert_success()
# Assert the workspace dirs have been deleted
@@ -558,45 +522,43 @@ def test_close_all(cli, tmpdir, datafiles):
@pytest.mark.datafiles(DATA_DIR)
def test_reset(cli, tmpdir, datafiles):
# Open the workspace
- element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git')
+ element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git")
# Modify workspace
- shutil.rmtree(os.path.join(workspace, 'usr', 'bin'))
- os.makedirs(os.path.join(workspace, 'etc'))
- with open(os.path.join(workspace, 'etc', 'pony.conf'), 'w') as f:
+ shutil.rmtree(os.path.join(workspace, "usr", "bin"))
+ os.makedirs(os.path.join(workspace, "etc"))
+ with open(os.path.join(workspace, "etc", "pony.conf"), "w") as f:
f.write("PONY='pink'")
# Now reset the open workspace, this should have the
# effect of reverting our changes.
- result = cli.run(project=project, args=[
- 'workspace', 'reset', element_name
- ])
+ result = cli.run(project=project, args=["workspace", "reset", element_name])
result.assert_success()
- assert os.path.exists(os.path.join(workspace, 'usr', 'bin', 'hello'))
- assert not os.path.exists(os.path.join(workspace, 'etc', 'pony.conf'))
+ assert os.path.exists(os.path.join(workspace, "usr", "bin", "hello"))
+ assert not os.path.exists(os.path.join(workspace, "etc", "pony.conf"))
@pytest.mark.datafiles(DATA_DIR)
def test_reset_soft(cli, tmpdir, datafiles):
# Open the workspace
- element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git')
+ element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git")
- assert cli.get_element_state(project, element_name) == 'buildable'
+ assert cli.get_element_state(project, element_name) == "buildable"
- hello_path = os.path.join(workspace, 'usr', 'bin', 'hello')
- pony_path = os.path.join(workspace, 'etc', 'pony.conf')
+ hello_path = os.path.join(workspace, "usr", "bin", "hello")
+ pony_path = os.path.join(workspace, "etc", "pony.conf")
- assert os.path.exists(os.path.join(workspace, 'usr', 'bin'))
+ assert os.path.exists(os.path.join(workspace, "usr", "bin"))
assert os.path.exists(hello_path)
assert not os.path.exists(pony_path)
key_1 = cli.get_element_key(project, element_name)
- assert key_1 != "{:?<64}".format('')
- result = cli.run(project=project, args=['build', element_name])
+ assert key_1 != "{:?<64}".format("")
+ result = cli.run(project=project, args=["build", element_name])
result.assert_success()
- assert cli.get_element_state(project, element_name) == 'cached'
+ assert cli.get_element_state(project, element_name) == "cached"
key_2 = cli.get_element_key(project, element_name)
- assert key_2 != "{:?<64}".format('')
+ assert key_2 != "{:?<64}".format("")
# workspace keys are not recalculated
assert key_1 == key_2
@@ -604,99 +566,87 @@ def test_reset_soft(cli, tmpdir, datafiles):
wait_for_cache_granularity()
# Modify workspace
- shutil.rmtree(os.path.join(workspace, 'usr', 'bin'))
- os.makedirs(os.path.join(workspace, 'etc'))
- with open(os.path.join(workspace, 'etc', 'pony.conf'), 'w') as f:
+ shutil.rmtree(os.path.join(workspace, "usr", "bin"))
+ os.makedirs(os.path.join(workspace, "etc"))
+ with open(os.path.join(workspace, "etc", "pony.conf"), "w") as f:
f.write("PONY='pink'")
- assert not os.path.exists(os.path.join(workspace, 'usr', 'bin'))
+ assert not os.path.exists(os.path.join(workspace, "usr", "bin"))
assert os.path.exists(pony_path)
# Now soft-reset the open workspace, this should not revert the changes
- result = cli.run(project=project, args=[
- 'workspace', 'reset', '--soft', element_name
- ])
+ result = cli.run(project=project, args=["workspace", "reset", "--soft", element_name])
result.assert_success()
# we removed this dir
- assert not os.path.exists(os.path.join(workspace, 'usr', 'bin'))
+ assert not os.path.exists(os.path.join(workspace, "usr", "bin"))
# and added this one
- assert os.path.exists(os.path.join(workspace, 'etc', 'pony.conf'))
+ assert os.path.exists(os.path.join(workspace, "etc", "pony.conf"))
- assert cli.get_element_state(project, element_name) == 'buildable'
+ assert cli.get_element_state(project, element_name) == "buildable"
key_3 = cli.get_element_key(project, element_name)
- assert key_3 != "{:?<64}".format('')
+ assert key_3 != "{:?<64}".format("")
assert key_1 != key_3
@pytest.mark.datafiles(DATA_DIR)
def test_reset_multiple(cli, tmpdir, datafiles):
# Open the workspaces
- tmpdir_alpha = os.path.join(str(tmpdir), 'alpha')
- tmpdir_beta = os.path.join(str(tmpdir), 'beta')
- alpha, project, workspace_alpha = open_workspace(
- cli, tmpdir_alpha, datafiles, 'git', suffix='-alpha')
- beta, project, workspace_beta = open_workspace(
- cli, tmpdir_beta, datafiles, 'git', suffix='-beta')
+ tmpdir_alpha = os.path.join(str(tmpdir), "alpha")
+ tmpdir_beta = os.path.join(str(tmpdir), "beta")
+ alpha, project, workspace_alpha = open_workspace(cli, tmpdir_alpha, datafiles, "git", suffix="-alpha")
+ beta, project, workspace_beta = open_workspace(cli, tmpdir_beta, datafiles, "git", suffix="-beta")
# Modify workspaces
- shutil.rmtree(os.path.join(workspace_alpha, 'usr', 'bin'))
- os.makedirs(os.path.join(workspace_beta, 'etc'))
- with open(os.path.join(workspace_beta, 'etc', 'pony.conf'), 'w') as f:
+ shutil.rmtree(os.path.join(workspace_alpha, "usr", "bin"))
+ os.makedirs(os.path.join(workspace_beta, "etc"))
+ with open(os.path.join(workspace_beta, "etc", "pony.conf"), "w") as f:
f.write("PONY='pink'")
# Now reset the open workspaces, this should have the
# effect of reverting our changes.
- result = cli.run(project=project, args=[
- 'workspace', 'reset', alpha, beta,
- ])
+ result = cli.run(project=project, args=["workspace", "reset", alpha, beta,])
result.assert_success()
- assert os.path.exists(os.path.join(workspace_alpha, 'usr', 'bin', 'hello'))
- assert not os.path.exists(os.path.join(workspace_beta, 'etc', 'pony.conf'))
+ assert os.path.exists(os.path.join(workspace_alpha, "usr", "bin", "hello"))
+ assert not os.path.exists(os.path.join(workspace_beta, "etc", "pony.conf"))
@pytest.mark.datafiles(DATA_DIR)
def test_reset_all(cli, tmpdir, datafiles):
# Open the workspaces
- tmpdir_alpha = os.path.join(str(tmpdir), 'alpha')
- tmpdir_beta = os.path.join(str(tmpdir), 'beta')
- _, project, workspace_alpha = open_workspace(
- cli, tmpdir_alpha, datafiles, 'git', suffix='-alpha')
- _, project, workspace_beta = open_workspace(
- cli, tmpdir_beta, datafiles, 'git', suffix='-beta')
+ tmpdir_alpha = os.path.join(str(tmpdir), "alpha")
+ tmpdir_beta = os.path.join(str(tmpdir), "beta")
+ _, project, workspace_alpha = open_workspace(cli, tmpdir_alpha, datafiles, "git", suffix="-alpha")
+ _, project, workspace_beta = open_workspace(cli, tmpdir_beta, datafiles, "git", suffix="-beta")
# Modify workspaces
- shutil.rmtree(os.path.join(workspace_alpha, 'usr', 'bin'))
- os.makedirs(os.path.join(workspace_beta, 'etc'))
- with open(os.path.join(workspace_beta, 'etc', 'pony.conf'), 'w') as f:
+ shutil.rmtree(os.path.join(workspace_alpha, "usr", "bin"))
+ os.makedirs(os.path.join(workspace_beta, "etc"))
+ with open(os.path.join(workspace_beta, "etc", "pony.conf"), "w") as f:
f.write("PONY='pink'")
# Now reset the open workspace, this should have the
# effect of reverting our changes.
- result = cli.run(project=project, args=[
- 'workspace', 'reset', '--all'
- ])
+ result = cli.run(project=project, args=["workspace", "reset", "--all"])
result.assert_success()
- assert os.path.exists(os.path.join(workspace_alpha, 'usr', 'bin', 'hello'))
- assert not os.path.exists(os.path.join(workspace_beta, 'etc', 'pony.conf'))
+ assert os.path.exists(os.path.join(workspace_alpha, "usr", "bin", "hello"))
+ assert not os.path.exists(os.path.join(workspace_beta, "etc", "pony.conf"))
@pytest.mark.datafiles(DATA_DIR)
def test_list(cli, tmpdir, datafiles):
- element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git')
+ element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git")
# Now list the workspaces
- result = cli.run(project=project, args=[
- 'workspace', 'list'
- ])
+ result = cli.run(project=project, args=["workspace", "list"])
result.assert_success()
loaded = _yaml.load_data(result.output)
- workspaces = loaded.get_sequence('workspaces')
+ workspaces = loaded.get_sequence("workspaces")
assert len(workspaces) == 1
space = workspaces.mapping_at(0)
- assert space.get_str('element') == element_name
- assert space.get_str('directory') == workspace
+ assert space.get_str("element") == element_name
+ assert space.get_str("directory") == workspace
@pytest.mark.datafiles(DATA_DIR)
@@ -705,117 +655,99 @@ def test_list(cli, tmpdir, datafiles):
@pytest.mark.parametrize(
"from_workspace,guess_element",
[(False, False), (True, True), (True, False)],
- ids=["project-no-guess", "workspace-guess", "workspace-no-guess"])
+ ids=["project-no-guess", "workspace-guess", "workspace-no-guess"],
+)
def test_build(cli, tmpdir_factory, datafiles, kind, strict, from_workspace, guess_element):
- tmpdir = tmpdir_factory.mktemp('')
+ tmpdir = tmpdir_factory.mktemp("")
element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, kind, False)
- checkout = os.path.join(str(tmpdir), 'checkout')
- args_dir = ['-C', workspace] if from_workspace else []
+ checkout = os.path.join(str(tmpdir), "checkout")
+ args_dir = ["-C", workspace] if from_workspace else []
args_elm = [element_name] if not guess_element else []
# Modify workspace
- shutil.rmtree(os.path.join(workspace, 'usr', 'bin'))
- os.makedirs(os.path.join(workspace, 'etc'))
- with open(os.path.join(workspace, 'etc', 'pony.conf'), 'w') as f:
+ shutil.rmtree(os.path.join(workspace, "usr", "bin"))
+ os.makedirs(os.path.join(workspace, "etc"))
+ with open(os.path.join(workspace, "etc", "pony.conf"), "w") as f:
f.write("PONY='pink'")
# Configure strict mode
strict_mode = True
- if strict != 'strict':
+ if strict != "strict":
strict_mode = False
- cli.configure({
- 'projects': {
- 'test': {
- 'strict': strict_mode
- }
- }
- })
+ cli.configure({"projects": {"test": {"strict": strict_mode}}})
# Build modified workspace
- assert cli.get_element_state(project, element_name) == 'buildable'
+ assert cli.get_element_state(project, element_name) == "buildable"
key_1 = cli.get_element_key(project, element_name)
- assert key_1 != "{:?<64}".format('')
- result = cli.run(project=project, args=args_dir + ['build', *args_elm])
+ assert key_1 != "{:?<64}".format("")
+ result = cli.run(project=project, args=args_dir + ["build", *args_elm])
result.assert_success()
- assert cli.get_element_state(project, element_name) == 'cached'
+ assert cli.get_element_state(project, element_name) == "cached"
key_2 = cli.get_element_key(project, element_name)
- assert key_2 != "{:?<64}".format('')
+ assert key_2 != "{:?<64}".format("")
# workspace keys are not recalculated
assert key_1 == key_2
# Checkout the result
- result = cli.run(project=project,
- args=args_dir + ['artifact', 'checkout', '--directory', checkout, *args_elm])
+ result = cli.run(project=project, args=args_dir + ["artifact", "checkout", "--directory", checkout, *args_elm])
result.assert_success()
# Check that the pony.conf from the modified workspace exists
- filename = os.path.join(checkout, 'etc', 'pony.conf')
+ filename = os.path.join(checkout, "etc", "pony.conf")
assert os.path.exists(filename)
# Check that the original /usr/bin/hello is not in the checkout
- assert not os.path.exists(os.path.join(checkout, 'usr', 'bin', 'hello'))
+ assert not os.path.exists(os.path.join(checkout, "usr", "bin", "hello"))
@pytest.mark.datafiles(DATA_DIR)
def test_buildable_no_ref(cli, tmpdir, datafiles):
project = str(datafiles)
- element_name = 'workspace-test-no-ref.bst'
- element_path = os.path.join(project, 'elements')
+ element_name = "workspace-test-no-ref.bst"
+ element_path = os.path.join(project, "elements")
# Write out our test target without any source ref
- repo = create_repo('git', str(tmpdir))
- element = {
- 'kind': 'import',
- 'sources': [
- repo.source_config()
- ]
- }
- _yaml.roundtrip_dump(element,
- os.path.join(element_path, element_name))
+ repo = create_repo("git", str(tmpdir))
+ element = {"kind": "import", "sources": [repo.source_config()]}
+ _yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
# Assert that this target is not buildable when no workspace is associated.
- assert cli.get_element_state(project, element_name) == 'no reference'
+ assert cli.get_element_state(project, element_name) == "no reference"
# Now open the workspace. We don't need to checkout the source though.
- workspace = os.path.join(str(tmpdir), 'workspace-no-ref')
+ workspace = os.path.join(str(tmpdir), "workspace-no-ref")
os.makedirs(workspace)
- args = ['workspace', 'open', '--no-checkout', '--directory', workspace, element_name]
+ args = ["workspace", "open", "--no-checkout", "--directory", workspace, element_name]
result = cli.run(project=project, args=args)
result.assert_success()
# Assert that the target is now buildable.
- assert cli.get_element_state(project, element_name) == 'buildable'
+ assert cli.get_element_state(project, element_name) == "buildable"
@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.parametrize("modification", [("addfile"), ("removefile"), ("modifyfile")])
@pytest.mark.parametrize("strict", [("strict"), ("non-strict")])
def test_detect_modifications(cli, tmpdir, datafiles, modification, strict):
- element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git')
- checkout = os.path.join(str(tmpdir), 'checkout')
+ element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git")
+ checkout = os.path.join(str(tmpdir), "checkout")
# Configure strict mode
strict_mode = True
- if strict != 'strict':
+ if strict != "strict":
strict_mode = False
- cli.configure({
- 'projects': {
- 'test': {
- 'strict': strict_mode
- }
- }
- })
+ cli.configure({"projects": {"test": {"strict": strict_mode}}})
# Build clean workspace
- assert cli.get_element_state(project, element_name) == 'buildable'
+ assert cli.get_element_state(project, element_name) == "buildable"
key_1 = cli.get_element_key(project, element_name)
- assert key_1 != "{:?<64}".format('')
- result = cli.run(project=project, args=['build', element_name])
+ assert key_1 != "{:?<64}".format("")
+ result = cli.run(project=project, args=["build", element_name])
result.assert_success()
- assert cli.get_element_state(project, element_name) == 'cached'
+ assert cli.get_element_state(project, element_name) == "cached"
key_2 = cli.get_element_key(project, element_name)
- assert key_2 != "{:?<64}".format('')
+ assert key_2 != "{:?<64}".format("")
# workspace keys are not recalculated
assert key_1 == key_2
@@ -825,32 +757,32 @@ def test_detect_modifications(cli, tmpdir, datafiles, modification, strict):
# Modify the workspace in various different ways, ensuring we
# properly detect the changes.
#
- if modification == 'addfile':
- os.makedirs(os.path.join(workspace, 'etc'))
- with open(os.path.join(workspace, 'etc', 'pony.conf'), 'w') as f:
+ if modification == "addfile":
+ os.makedirs(os.path.join(workspace, "etc"))
+ with open(os.path.join(workspace, "etc", "pony.conf"), "w") as f:
f.write("PONY='pink'")
- elif modification == 'removefile':
- os.remove(os.path.join(workspace, 'usr', 'bin', 'hello'))
- elif modification == 'modifyfile':
- with open(os.path.join(workspace, 'usr', 'bin', 'hello'), 'w') as f:
- f.write('cookie')
+ elif modification == "removefile":
+ os.remove(os.path.join(workspace, "usr", "bin", "hello"))
+ elif modification == "modifyfile":
+ with open(os.path.join(workspace, "usr", "bin", "hello"), "w") as f:
+ f.write("cookie")
else:
# This cannot be reached
assert 0
# First assert that the state is properly detected
- assert cli.get_element_state(project, element_name) == 'buildable'
+ assert cli.get_element_state(project, element_name) == "buildable"
key_3 = cli.get_element_key(project, element_name)
- assert key_3 != "{:?<64}".format('')
+ assert key_3 != "{:?<64}".format("")
# Since there are different things going on at `bst build` time
# than `bst show` time, we also want to build / checkout again,
# and ensure that the result contains what we expect.
- result = cli.run(project=project, args=['build', element_name])
+ result = cli.run(project=project, args=["build", element_name])
result.assert_success()
- assert cli.get_element_state(project, element_name) == 'cached'
+ assert cli.get_element_state(project, element_name) == "cached"
key_4 = cli.get_element_key(project, element_name)
- assert key_4 != "{:?<64}".format('')
+ assert key_4 != "{:?<64}".format("")
# workspace keys are not recalculated
assert key_3 == key_4
@@ -858,22 +790,20 @@ def test_detect_modifications(cli, tmpdir, datafiles, modification, strict):
assert key_1 != key_3
# Checkout the result
- result = cli.run(project=project, args=[
- 'artifact', 'checkout', element_name, '--directory', checkout
- ])
+ result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
result.assert_success()
# Check the result for the changes we made
#
- if modification == 'addfile':
- filename = os.path.join(checkout, 'etc', 'pony.conf')
+ if modification == "addfile":
+ filename = os.path.join(checkout, "etc", "pony.conf")
assert os.path.exists(filename)
- elif modification == 'removefile':
- assert not os.path.exists(os.path.join(checkout, 'usr', 'bin', 'hello'))
- elif modification == 'modifyfile':
- with open(os.path.join(workspace, 'usr', 'bin', 'hello'), 'r') as f:
+ elif modification == "removefile":
+ assert not os.path.exists(os.path.join(checkout, "usr", "bin", "hello"))
+ elif modification == "modifyfile":
+ with open(os.path.join(workspace, "usr", "bin", "hello"), "r") as f:
data = f.read()
- assert data == 'cookie'
+ assert data == "cookie"
else:
# This cannot be reached
assert 0
@@ -882,143 +812,111 @@ def test_detect_modifications(cli, tmpdir, datafiles, modification, strict):
# Ensure that various versions that should not be accepted raise a
# LoadError.INVALID_DATA
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("workspace_cfg", [
- # Test loading a negative workspace version
- {"format-version": -1},
- # Test loading version 0 with two sources
- {
- "format-version": 0,
- "alpha.bst": {
- 0: "/workspaces/bravo",
- 1: "/workspaces/charlie",
- }
- },
- # Test loading a version with decimals
- {"format-version": 0.5},
- # Test loading a future version
- {"format-version": BST_WORKSPACE_FORMAT_VERSION + 1}
-])
+@pytest.mark.parametrize(
+ "workspace_cfg",
+ [
+ # Test loading a negative workspace version
+ {"format-version": -1},
+ # Test loading version 0 with two sources
+ {"format-version": 0, "alpha.bst": {0: "/workspaces/bravo", 1: "/workspaces/charlie",}},
+ # Test loading a version with decimals
+ {"format-version": 0.5},
+ # Test loading a future version
+ {"format-version": BST_WORKSPACE_FORMAT_VERSION + 1},
+ ],
+)
def test_list_unsupported_workspace(cli, datafiles, workspace_cfg):
project = str(datafiles)
- os.makedirs(os.path.join(project, '.bst'))
- workspace_config_path = os.path.join(project, '.bst', 'workspaces.yml')
+ os.makedirs(os.path.join(project, ".bst"))
+ workspace_config_path = os.path.join(project, ".bst", "workspaces.yml")
_yaml.roundtrip_dump(workspace_cfg, workspace_config_path)
- result = cli.run(project=project, args=['workspace', 'list'])
+ result = cli.run(project=project, args=["workspace", "list"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
# Ensure that various versions that should be accepted are parsed
# correctly.
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("workspace_cfg,expected", [
- # Test loading version 0 without a dict
- ({
- "alpha.bst": "/workspaces/bravo"
- }, {
- "format-version": BST_WORKSPACE_FORMAT_VERSION,
- "workspaces": {
- "alpha.bst": {
- "prepared": False,
- "path": "/workspaces/bravo",
- "running_files": {}
- }
- }
- }),
- # Test loading version 0 with only one source
- ({
- "alpha.bst": {
- 0: "/workspaces/bravo"
- }
- }, {
- "format-version": BST_WORKSPACE_FORMAT_VERSION,
- "workspaces": {
- "alpha.bst": {
- "prepared": False,
- "path": "/workspaces/bravo",
- "running_files": {}
- }
- }
- }),
- # Test loading version 1
- ({
- "format-version": 1,
- "workspaces": {
- "alpha.bst": {
- "path": "/workspaces/bravo"
- }
- }
- }, {
- "format-version": BST_WORKSPACE_FORMAT_VERSION,
- "workspaces": {
- "alpha.bst": {
- "prepared": False,
- "path": "/workspaces/bravo",
- "running_files": {}
- }
- }
- }),
- # Test loading version 2
- ({
- "format-version": 2,
- "workspaces": {
- "alpha.bst": {
- "path": "/workspaces/bravo",
- "last_successful": "some_key",
- "running_files": {
- "beta.bst": ["some_file"]
- }
- }
- }
- }, {
- "format-version": BST_WORKSPACE_FORMAT_VERSION,
- "workspaces": {
- "alpha.bst": {
- "prepared": False,
- "path": "/workspaces/bravo",
- "last_successful": "some_key",
- "running_files": {
- "beta.bst": ["some_file"]
- }
- }
- }
- }),
- # Test loading version 3
- ({
- "format-version": 3,
- "workspaces": {
- "alpha.bst": {
- "prepared": True,
- "path": "/workspaces/bravo",
- "running_files": {}
- }
- }
- }, {
- "format-version": BST_WORKSPACE_FORMAT_VERSION,
- "workspaces": {
- "alpha.bst": {
- "prepared": True,
- "path": "/workspaces/bravo",
- "running_files": {}
- }
- }
- })
-])
+@pytest.mark.parametrize(
+ "workspace_cfg,expected",
+ [
+ # Test loading version 0 without a dict
+ (
+ {"alpha.bst": "/workspaces/bravo"},
+ {
+ "format-version": BST_WORKSPACE_FORMAT_VERSION,
+ "workspaces": {"alpha.bst": {"prepared": False, "path": "/workspaces/bravo", "running_files": {}}},
+ },
+ ),
+ # Test loading version 0 with only one source
+ (
+ {"alpha.bst": {0: "/workspaces/bravo"}},
+ {
+ "format-version": BST_WORKSPACE_FORMAT_VERSION,
+ "workspaces": {"alpha.bst": {"prepared": False, "path": "/workspaces/bravo", "running_files": {}}},
+ },
+ ),
+ # Test loading version 1
+ (
+ {"format-version": 1, "workspaces": {"alpha.bst": {"path": "/workspaces/bravo"}}},
+ {
+ "format-version": BST_WORKSPACE_FORMAT_VERSION,
+ "workspaces": {"alpha.bst": {"prepared": False, "path": "/workspaces/bravo", "running_files": {}}},
+ },
+ ),
+ # Test loading version 2
+ (
+ {
+ "format-version": 2,
+ "workspaces": {
+ "alpha.bst": {
+ "path": "/workspaces/bravo",
+ "last_successful": "some_key",
+ "running_files": {"beta.bst": ["some_file"]},
+ }
+ },
+ },
+ {
+ "format-version": BST_WORKSPACE_FORMAT_VERSION,
+ "workspaces": {
+ "alpha.bst": {
+ "prepared": False,
+ "path": "/workspaces/bravo",
+ "last_successful": "some_key",
+ "running_files": {"beta.bst": ["some_file"]},
+ }
+ },
+ },
+ ),
+ # Test loading version 3
+ (
+ {
+ "format-version": 3,
+ "workspaces": {"alpha.bst": {"prepared": True, "path": "/workspaces/bravo", "running_files": {}}},
+ },
+ {
+ "format-version": BST_WORKSPACE_FORMAT_VERSION,
+ "workspaces": {"alpha.bst": {"prepared": True, "path": "/workspaces/bravo", "running_files": {}}},
+ },
+ ),
+ ],
+)
def test_list_supported_workspace(cli, tmpdir, datafiles, workspace_cfg, expected):
def parse_dict_as_yaml(node):
- tempfile = os.path.join(str(tmpdir), 'yaml_dump')
+ tempfile = os.path.join(str(tmpdir), "yaml_dump")
_yaml.roundtrip_dump(node, tempfile)
return _yaml.load(tempfile).strip_node_info()
project = str(datafiles)
- os.makedirs(os.path.join(project, '.bst'))
- workspace_config_path = os.path.join(project, '.bst', 'workspaces.yml')
+ os.makedirs(os.path.join(project, ".bst"))
+ workspace_config_path = os.path.join(project, ".bst", "workspaces.yml")
_yaml.roundtrip_dump(workspace_cfg, workspace_config_path)
# Check that we can still read workspace config that is in old format
- result = cli.run(project=project, args=['workspace', 'list'])
+ result = cli.run(project=project, args=["workspace", "list"])
result.assert_success()
loaded_config = _yaml.load(workspace_config_path).strip_node_info()
@@ -1028,31 +926,25 @@ def test_list_supported_workspace(cli, tmpdir, datafiles, workspace_cfg, expecte
assert loaded_config == parse_dict_as_yaml(workspace_cfg)
# Create a test bst file
- bin_files_path = os.path.join(project, 'files', 'bin-files')
- element_path = os.path.join(project, 'elements')
- element_name = 'workspace-test.bst'
- workspace = os.path.join(str(tmpdir), 'workspace')
+ bin_files_path = os.path.join(project, "files", "bin-files")
+ element_path = os.path.join(project, "elements")
+ element_name = "workspace-test.bst"
+ workspace = os.path.join(str(tmpdir), "workspace")
# Create our repo object of the given source type with
# the bin files, and then collect the initial ref.
#
- repo = create_repo('git', str(tmpdir))
+ repo = create_repo("git", str(tmpdir))
ref = repo.create(bin_files_path)
# Write out our test target
- element = {
- 'kind': 'import',
- 'sources': [
- repo.source_config(ref=ref)
- ]
- }
- _yaml.roundtrip_dump(element,
- os.path.join(element_path, element_name))
+ element = {"kind": "import", "sources": [repo.source_config(ref=ref)]}
+ _yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
# Make a change to the workspaces file
- result = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name])
+ result = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name])
result.assert_success()
- result = cli.run(project=project, args=['workspace', 'close', '--remove-dir', element_name])
+ result = cli.run(project=project, args=["workspace", "close", "--remove-dir", element_name])
result.assert_success()
# Check that workspace config is converted correctly if necessary
@@ -1062,73 +954,55 @@ def test_list_supported_workspace(cli, tmpdir, datafiles, workspace_cfg, expecte
@pytest.mark.datafiles(DATA_DIR)
def test_inconsitent_pipeline_message(cli, tmpdir, datafiles):
- element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git')
+ element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git")
shutil.rmtree(workspace)
- result = cli.run(project=project, args=[
- 'build', element_name
- ])
+ result = cli.run(project=project, args=["build", element_name])
result.assert_main_error(ErrorDomain.PIPELINE, "inconsistent-pipeline-workspaced")
@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.parametrize("strict", [("strict"), ("non-strict")])
def test_cache_key_workspace_in_dependencies(cli, tmpdir, datafiles, strict):
- checkout = os.path.join(str(tmpdir), 'checkout')
- element_name, project, workspace = open_workspace(cli, os.path.join(str(tmpdir), 'repo-a'),
- datafiles, 'git')
+ checkout = os.path.join(str(tmpdir), "checkout")
+ element_name, project, workspace = open_workspace(cli, os.path.join(str(tmpdir), "repo-a"), datafiles, "git")
- element_path = os.path.join(project, 'elements')
- back_dep_element_name = 'workspace-test-back-dep.bst'
+ element_path = os.path.join(project, "elements")
+ back_dep_element_name = "workspace-test-back-dep.bst"
# Write out our test target
- element = {
- 'kind': 'compose',
- 'depends': [
- {
- 'filename': element_name,
- 'type': 'build'
- }
- ]
- }
- _yaml.roundtrip_dump(element,
- os.path.join(element_path, back_dep_element_name))
+ element = {"kind": "compose", "depends": [{"filename": element_name, "type": "build"}]}
+ _yaml.roundtrip_dump(element, os.path.join(element_path, back_dep_element_name))
# Modify workspace
- shutil.rmtree(os.path.join(workspace, 'usr', 'bin'))
- os.makedirs(os.path.join(workspace, 'etc'))
- with open(os.path.join(workspace, 'etc', 'pony.conf'), 'w') as f:
+ shutil.rmtree(os.path.join(workspace, "usr", "bin"))
+ os.makedirs(os.path.join(workspace, "etc"))
+ with open(os.path.join(workspace, "etc", "pony.conf"), "w") as f:
f.write("PONY='pink'")
# Configure strict mode
strict_mode = True
- if strict != 'strict':
+ if strict != "strict":
strict_mode = False
- cli.configure({
- 'projects': {
- 'test': {
- 'strict': strict_mode
- }
- }
- })
+ cli.configure({"projects": {"test": {"strict": strict_mode}}})
# Build artifact with dependency's modified workspace
- assert cli.get_element_state(project, element_name) == 'buildable'
+ assert cli.get_element_state(project, element_name) == "buildable"
key_a1 = cli.get_element_key(project, element_name)
- assert key_a1 != "{:?<64}".format('')
- assert cli.get_element_state(project, back_dep_element_name) == 'waiting'
+ assert key_a1 != "{:?<64}".format("")
+ assert cli.get_element_state(project, back_dep_element_name) == "waiting"
key_b1 = cli.get_element_key(project, back_dep_element_name)
- assert key_b1 != "{:?<64}".format('')
- result = cli.run(project=project, args=['build', back_dep_element_name])
+ assert key_b1 != "{:?<64}".format("")
+ result = cli.run(project=project, args=["build", back_dep_element_name])
result.assert_success()
- assert cli.get_element_state(project, element_name) == 'cached'
+ assert cli.get_element_state(project, element_name) == "cached"
key_a2 = cli.get_element_key(project, element_name)
- assert key_a2 != "{:?<64}".format('')
- assert cli.get_element_state(project, back_dep_element_name) == 'cached'
+ assert key_a2 != "{:?<64}".format("")
+ assert cli.get_element_state(project, back_dep_element_name) == "cached"
key_b2 = cli.get_element_key(project, back_dep_element_name)
- assert key_b2 != "{:?<64}".format('')
- result = cli.run(project=project, args=['build', back_dep_element_name])
+ assert key_b2 != "{:?<64}".format("")
+ result = cli.run(project=project, args=["build", back_dep_element_name])
result.assert_success()
# workspace keys are not recalculated
@@ -1136,31 +1010,21 @@ def test_cache_key_workspace_in_dependencies(cli, tmpdir, datafiles, strict):
assert key_b1 == key_b2
# Checkout the result
- result = cli.run(project=project, args=[
- 'artifact', 'checkout', back_dep_element_name, '--directory', checkout
- ])
+ result = cli.run(project=project, args=["artifact", "checkout", back_dep_element_name, "--directory", checkout])
result.assert_success()
# Check that the pony.conf from the modified workspace exists
- filename = os.path.join(checkout, 'etc', 'pony.conf')
+ filename = os.path.join(checkout, "etc", "pony.conf")
assert os.path.exists(filename)
# Check that the original /usr/bin/hello is not in the checkout
- assert not os.path.exists(os.path.join(checkout, 'usr', 'bin', 'hello'))
+ assert not os.path.exists(os.path.join(checkout, "usr", "bin", "hello"))
@pytest.mark.datafiles(DATA_DIR)
def test_multiple_failed_builds(cli, tmpdir, datafiles):
- element_config = {
- "kind": "manual",
- "config": {
- "configure-commands": [
- "unknown_command_that_will_fail"
- ]
- }
- }
- element_name, project, _ = open_workspace(cli, tmpdir, datafiles,
- "git", element_attrs=element_config)
+ element_config = {"kind": "manual", "config": {"configure-commands": ["unknown_command_that_will_fail"]}}
+ element_name, project, _ = open_workspace(cli, tmpdir, datafiles, "git", element_attrs=element_config)
for _ in range(2):
result = cli.run(project=project, args=["build", element_name])
@@ -1169,60 +1033,57 @@ def test_multiple_failed_builds(cli, tmpdir, datafiles):
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize('subdir', [True, False], ids=["subdir", "no-subdir"])
+@pytest.mark.parametrize("subdir", [True, False], ids=["subdir", "no-subdir"])
@pytest.mark.parametrize("guess_element", [True, False], ids=["guess", "no-guess"])
def test_external_fetch(cli, datafiles, tmpdir_factory, subdir, guess_element):
# An element with an open workspace can't be fetched, but we still expect fetches
# to fetch any dependencies
- tmpdir = tmpdir_factory.mktemp('')
- depend_element = 'fetchable.bst'
+ tmpdir = tmpdir_factory.mktemp("")
+ depend_element = "fetchable.bst"
# Create an element to fetch (local sources do not need to fetch)
- create_element_size(depend_element, str(datafiles), 'elements', [], 1024)
+ create_element_size(depend_element, str(datafiles), "elements", [], 1024)
element_name, project, workspace = open_workspace(
- cli, tmpdir, datafiles, "git", no_checkout=True,
- element_attrs={'depends': [depend_element]}
+ cli, tmpdir, datafiles, "git", no_checkout=True, element_attrs={"depends": [depend_element]}
)
arg_elm = [element_name] if not guess_element else []
if subdir:
- call_dir = os.path.join(workspace, 'usr')
+ call_dir = os.path.join(workspace, "usr")
os.makedirs(call_dir, exist_ok=True)
else:
call_dir = workspace
# Assert that the depended element is not fetched yet
- assert cli.get_element_state(str(datafiles), depend_element) == 'fetch needed'
+ assert cli.get_element_state(str(datafiles), depend_element) == "fetch needed"
# Fetch the workspaced element
- result = cli.run(project=project, args=['-C', call_dir, 'source', 'fetch', *arg_elm])
+ result = cli.run(project=project, args=["-C", call_dir, "source", "fetch", *arg_elm])
result.assert_success()
# Assert that the depended element has now been fetched
- assert cli.get_element_state(str(datafiles), depend_element) == 'buildable'
+ assert cli.get_element_state(str(datafiles), depend_element) == "buildable"
@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.parametrize("guess_element", [True, False], ids=["guess", "no-guess"])
def test_external_push_pull(cli, datafiles, tmpdir_factory, guess_element):
# Pushing and pulling to/from an artifact cache works from an external workspace
- tmpdir = tmpdir_factory.mktemp('')
+ tmpdir = tmpdir_factory.mktemp("")
element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git")
arg_elm = [element_name] if not guess_element else []
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
- result = cli.run(project=project, args=['-C', workspace, 'build', element_name])
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
+ result = cli.run(project=project, args=["-C", workspace, "build", element_name])
result.assert_success()
- cli.configure({
- 'artifacts': {'url': share.repo, 'push': True}
- })
+ cli.configure({"artifacts": {"url": share.repo, "push": True}})
- result = cli.run(project=project, args=['-C', workspace, 'artifact', 'push', *arg_elm])
+ result = cli.run(project=project, args=["-C", workspace, "artifact", "push", *arg_elm])
result.assert_success()
- result = cli.run(project=project, args=['-C', workspace, 'artifact', 'pull', '--deps', 'all', *arg_elm])
+ result = cli.run(project=project, args=["-C", workspace, "artifact", "pull", "--deps", "all", *arg_elm])
result.assert_success()
@@ -1232,35 +1093,35 @@ def test_external_push_pull(cli, datafiles, tmpdir_factory, guess_element):
@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.parametrize("guess_element", [True, False], ids=["guess", "no-guess"])
def test_external_track(cli, datafiles, tmpdir_factory, guess_element):
- tmpdir = tmpdir_factory.mktemp('')
+ tmpdir = tmpdir_factory.mktemp("")
element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git")
- element_file = os.path.join(str(datafiles), 'elements', element_name)
+ element_file = os.path.join(str(datafiles), "elements", element_name)
arg_elm = [element_name] if not guess_element else []
# Delete the ref from the source so that we can detect if the
# element has been tracked after closing the workspace
element_contents = _yaml.load(element_file)
- ref1 = element_contents.get_sequence('sources').mapping_at(0).get_str('ref')
- del element_contents.get_sequence('sources').mapping_at(0)['ref']
+ ref1 = element_contents.get_sequence("sources").mapping_at(0).get_str("ref")
+ del element_contents.get_sequence("sources").mapping_at(0)["ref"]
_yaml.roundtrip_dump(element_contents, element_file)
- result = cli.run(project=project, args=['-C', workspace, 'source', 'track', *arg_elm])
+ result = cli.run(project=project, args=["-C", workspace, "source", "track", *arg_elm])
result.assert_success()
# Element is not tracked now
element_contents = _yaml.load(element_file)
- assert 'ref' not in element_contents.get_sequence('sources').mapping_at(0)
+ assert "ref" not in element_contents.get_sequence("sources").mapping_at(0)
# close the workspace
- result = cli.run(project=project, args=['-C', workspace, 'workspace', 'close', *arg_elm])
+ result = cli.run(project=project, args=["-C", workspace, "workspace", "close", *arg_elm])
result.assert_success()
# and retrack the element
- result = cli.run(project=project, args=['source', 'track', element_name])
+ result = cli.run(project=project, args=["source", "track", element_name])
result.assert_success()
element_contents = _yaml.load(element_file)
- ref2 = element_contents.get_sequence('sources').mapping_at(0).get_str('ref')
+ ref2 = element_contents.get_sequence("sources").mapping_at(0).get_str("ref")
# these values should be equivalent
assert ref1 == ref2
@@ -1268,62 +1129,63 @@ def test_external_track(cli, datafiles, tmpdir_factory, guess_element):
@pytest.mark.datafiles(DATA_DIR)
def test_external_open_other(cli, datafiles, tmpdir_factory):
# From inside an external workspace, open another workspace
- tmpdir1 = tmpdir_factory.mktemp('')
- tmpdir2 = tmpdir_factory.mktemp('')
+ tmpdir1 = tmpdir_factory.mktemp("")
+ tmpdir2 = tmpdir_factory.mktemp("")
# Making use of the assumption that it's the same project in both invocations of open_workspace
_, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", suffix="-alpha")
beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", suffix="-beta")
# Closing the other element first, because I'm too lazy to create an
# element without opening it
- result = cli.run(project=project, args=['workspace', 'close', beta_element])
+ result = cli.run(project=project, args=["workspace", "close", beta_element])
result.assert_success()
- result = cli.run(project=project, args=[
- '-C', alpha_workspace, 'workspace', 'open', '--force', '--directory', beta_workspace, beta_element
- ])
+ result = cli.run(
+ project=project,
+ args=["-C", alpha_workspace, "workspace", "open", "--force", "--directory", beta_workspace, beta_element],
+ )
result.assert_success()
@pytest.mark.datafiles(DATA_DIR)
def test_external_close_other(cli, datafiles, tmpdir_factory):
# From inside an external workspace, close the other workspace
- tmpdir1 = tmpdir_factory.mktemp('')
- tmpdir2 = tmpdir_factory.mktemp('')
+ tmpdir1 = tmpdir_factory.mktemp("")
+ tmpdir2 = tmpdir_factory.mktemp("")
# Making use of the assumption that it's the same project in both invocations of open_workspace
_, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", suffix="-alpha")
beta_element, _, _ = open_workspace(cli, tmpdir2, datafiles, "git", suffix="-beta")
- result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'close', beta_element])
+ result = cli.run(project=project, args=["-C", alpha_workspace, "workspace", "close", beta_element])
result.assert_success()
- assert 'you can no longer run BuildStream' not in result.stderr
+ assert "you can no longer run BuildStream" not in result.stderr
@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.parametrize("guess_element", [True, False], ids=["guess", "no-guess"])
def test_external_close_self(cli, datafiles, tmpdir_factory, guess_element):
# From inside an external workspace, close it
- tmpdir1 = tmpdir_factory.mktemp('')
- tmpdir2 = tmpdir_factory.mktemp('')
+ tmpdir1 = tmpdir_factory.mktemp("")
+ tmpdir2 = tmpdir_factory.mktemp("")
# Making use of the assumption that it's the same project in both invocations of open_workspace
alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", suffix="-alpha")
_, _, _ = open_workspace(cli, tmpdir2, datafiles, "git", suffix="-beta")
arg_elm = [alpha_element] if not guess_element else []
- result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'close', *arg_elm])
+ result = cli.run(project=project, args=["-C", alpha_workspace, "workspace", "close", *arg_elm])
result.assert_success()
- assert 'you can no longer run BuildStream' in result.stderr
+ assert "you can no longer run BuildStream" in result.stderr
@pytest.mark.datafiles(DATA_DIR)
def test_external_reset_other(cli, datafiles, tmpdir_factory):
- tmpdir1 = tmpdir_factory.mktemp('')
- tmpdir2 = tmpdir_factory.mktemp('')
+ tmpdir1 = tmpdir_factory.mktemp("")
+ tmpdir2 = tmpdir_factory.mktemp("")
# Making use of the assumption that it's the same project in both invocations of open_workspace
_, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", suffix="-alpha")
beta_element, _, _ = open_workspace(cli, tmpdir2, datafiles, "git", suffix="-beta")
- result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'reset', beta_element])
+ result = cli.run(project=project, args=["-C", alpha_workspace, "workspace", "reset", beta_element])
result.assert_success()
@@ -1334,21 +1196,21 @@ def test_external_reset_self(cli, datafiles, tmpdir, guess_element):
arg_elm = [element] if not guess_element else []
# Command succeeds
- result = cli.run(project=project, args=['-C', workspace, 'workspace', 'reset', *arg_elm])
+ result = cli.run(project=project, args=["-C", workspace, "workspace", "reset", *arg_elm])
result.assert_success()
# Successive commands still work (i.e. .bstproject.yaml hasn't been deleted)
- result = cli.run(project=project, args=['-C', workspace, 'workspace', 'list'])
+ result = cli.run(project=project, args=["-C", workspace, "workspace", "list"])
result.assert_success()
@pytest.mark.datafiles(DATA_DIR)
def test_external_list(cli, datafiles, tmpdir_factory):
- tmpdir = tmpdir_factory.mktemp('')
+ tmpdir = tmpdir_factory.mktemp("")
# Making use of the assumption that it's the same project in both invocations of open_workspace
_, project, workspace = open_workspace(cli, tmpdir, datafiles, "git")
- result = cli.run(project=project, args=['-C', workspace, 'workspace', 'list'])
+ result = cli.run(project=project, args=["-C", workspace, "workspace", "list"])
result.assert_success()
@@ -1359,26 +1221,18 @@ def test_multisource_workspace(cli, datafiles, tmpdir):
project = str(datafiles)
element_name = "multisource.bst"
element = {
- 'kind': 'import',
- 'sources': [{
- 'kind': 'local',
- 'path': 'files/bin-files'
- }, {
- 'kind': 'local',
- 'path': 'files/dev-files'
- }]
+ "kind": "import",
+ "sources": [{"kind": "local", "path": "files/bin-files"}, {"kind": "local", "path": "files/dev-files"}],
}
- element_path = os.path.join(project, 'elements', element_name)
+ element_path = os.path.join(project, "elements", element_name)
_yaml.roundtrip_dump(element, element_path)
- workspace_dir = os.path.join(str(tmpdir), 'multisource')
- res = cli.run(project=project,
- args=['workspace', 'open', 'multisource.bst',
- '--directory', workspace_dir])
+ workspace_dir = os.path.join(str(tmpdir), "multisource")
+ res = cli.run(project=project, args=["workspace", "open", "multisource.bst", "--directory", workspace_dir])
res.assert_success()
- directories = os.listdir(os.path.join(workspace_dir, 'usr'))
- assert 'bin' in directories and 'include' in directories
+ directories = os.listdir(os.path.join(workspace_dir, "usr"))
+ assert "bin" in directories and "include" in directories
# This strange test tests against a regression raised in issue #919,
@@ -1387,9 +1241,7 @@ def test_multisource_workspace(cli, datafiles, tmpdir):
# but just successfully builds the workspaced element and happily
# exits without completing the build.
#
-TEST_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__))
-)
+TEST_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)))
@pytest.mark.datafiles(TEST_DIR)
@@ -1397,73 +1249,61 @@ TEST_DIR = os.path.join(
["case", "non_workspaced_elements_state"],
[
("workspaced-build-dep", ["waiting", "waiting", "waiting", "waiting", "waiting"]),
- ("workspaced-runtime-dep", ["buildable", "buildable", "waiting", "waiting", "waiting"])
+ ("workspaced-runtime-dep", ["buildable", "buildable", "waiting", "waiting", "waiting"]),
],
)
@pytest.mark.parametrize("strict", [("strict"), ("non-strict")])
def test_build_all(cli, tmpdir, datafiles, case, strict, non_workspaced_elements_state):
project = os.path.join(str(datafiles), case)
- workspace = os.path.join(str(tmpdir), 'workspace')
+ workspace = os.path.join(str(tmpdir), "workspace")
non_leaf_elements = ["elem2.bst", "elem3.bst", "stack.bst", "elem4.bst", "elem5.bst"]
all_elements = ["elem1.bst", *non_leaf_elements]
# Configure strict mode
strict_mode = True
- if strict != 'strict':
+ if strict != "strict":
strict_mode = False
- cli.configure({
- 'projects': {
- 'test': {
- 'strict': strict_mode
- }
- }
- })
+ cli.configure({"projects": {"test": {"strict": strict_mode}}})
# First open the workspace
- result = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, 'elem1.bst'])
+ result = cli.run(project=project, args=["workspace", "open", "--directory", workspace, "elem1.bst"])
result.assert_success()
# Ensure all elements are waiting build the first
- assert cli.get_element_states(project, all_elements) == \
- dict(zip(all_elements, ['buildable', *non_workspaced_elements_state]))
+ assert cli.get_element_states(project, all_elements) == dict(
+ zip(all_elements, ["buildable", *non_workspaced_elements_state])
+ )
# Now build the targets elem4.bst and elem5.bst
- result = cli.run(project=project, args=['build', 'elem4.bst', 'elem5.bst'])
+ result = cli.run(project=project, args=["build", "elem4.bst", "elem5.bst"])
result.assert_success()
# Assert that the target is built
- assert cli.get_element_states(project, all_elements) == \
- {elem: "cached" for elem in all_elements}
+ assert cli.get_element_states(project, all_elements) == {elem: "cached" for elem in all_elements}
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize('strict', ['strict', 'non-strict'])
+@pytest.mark.parametrize("strict", ["strict", "non-strict"])
def test_show_workspace_logs(cli, tmpdir, datafiles, strict):
project = str(datafiles)
- workspace = os.path.join(str(tmpdir), 'workspace')
- target = 'manual.bst'
+ workspace = os.path.join(str(tmpdir), "workspace")
+ target = "manual.bst"
# Configure strict mode
strict_mode = True
- if strict != 'strict':
+ if strict != "strict":
strict_mode = False
- cli.configure({
- 'projects': {
- 'test': {
- 'strict': strict_mode
- }
- }
- })
+ cli.configure({"projects": {"test": {"strict": strict_mode}}})
# First open the workspace
- result = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, target])
+ result = cli.run(project=project, args=["workspace", "open", "--directory", workspace, target])
result.assert_success()
# Build the element
- result = cli.run(project=project, args=['build', target])
- result.assert_task_error(ErrorDomain.SANDBOX, 'missing-command')
+ result = cli.run(project=project, args=["build", target])
+ result.assert_task_error(ErrorDomain.SANDBOX, "missing-command")
- result = cli.run(project=project, args=['artifact', 'log', target])
+ result = cli.run(project=project, args=["artifact", "log", target])
result.assert_success()
# Assert that the log is not empty
diff --git a/tests/integration/artifact.py b/tests/integration/artifact.py
index 59b7bfaad..e21dd4296 100644
--- a/tests/integration/artifact.py
+++ b/tests/integration/artifact.py
@@ -36,10 +36,7 @@ pytestmark = pytest.mark.integration
# Project directory
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project",
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",)
# A test to capture the integration of the cachebuildtrees
@@ -48,70 +45,64 @@ DATA_DIR = os.path.join(
# Dse this really need a sandbox?
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_cache_buildtrees(cli, tmpdir, datafiles):
project = str(datafiles)
- element_name = 'autotools/amhello.bst'
+ element_name = "autotools/amhello.bst"
cwd = str(tmpdir)
# Create artifact shares for pull & push testing
- with create_artifact_share(os.path.join(str(tmpdir), 'share1')) as share1,\
- create_artifact_share(os.path.join(str(tmpdir), 'share2')) as share2,\
- create_artifact_share(os.path.join(str(tmpdir), 'share3')) as share3:
- cli.configure({
- 'artifacts': {'url': share1.repo, 'push': True},
- 'cachedir': str(tmpdir)
- })
+ with create_artifact_share(os.path.join(str(tmpdir), "share1")) as share1, create_artifact_share(
+ os.path.join(str(tmpdir), "share2")
+ ) as share2, create_artifact_share(os.path.join(str(tmpdir), "share3")) as share3:
+ cli.configure({"artifacts": {"url": share1.repo, "push": True}, "cachedir": str(tmpdir)})
# Build autotools element with the default behavior of caching buildtrees
# only when necessary. The artifact should be successfully pushed to the share1 remote
# and cached locally with an 'empty' buildtree digest, as it's not a
# dangling ref
- result = cli.run(project=project, args=['build', element_name])
+ result = cli.run(project=project, args=["build", element_name])
assert result.exit_code == 0
- assert cli.get_element_state(project, element_name) == 'cached'
- assert share1.get_artifact(cli.get_artifact_name(project, 'test', element_name))
+ assert cli.get_element_state(project, element_name) == "cached"
+ assert share1.get_artifact(cli.get_artifact_name(project, "test", element_name))
# The buildtree dir should not exist, as we set the config to not cache buildtrees.
- artifact_name = cli.get_artifact_name(project, 'test', element_name)
+ artifact_name = cli.get_artifact_name(project, "test", element_name)
assert share1.get_artifact(artifact_name)
with cli.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir:
assert not buildtreedir
# Delete the local cached artifacts, and assert the when pulled with --pull-buildtrees
# that is was cached in share1 as expected without a buildtree dir
- shutil.rmtree(os.path.join(str(tmpdir), 'cas'))
- shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
- assert cli.get_element_state(project, element_name) != 'cached'
- result = cli.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name])
+ shutil.rmtree(os.path.join(str(tmpdir), "cas"))
+ shutil.rmtree(os.path.join(str(tmpdir), "artifacts"))
+ assert cli.get_element_state(project, element_name) != "cached"
+ result = cli.run(project=project, args=["--pull-buildtrees", "artifact", "pull", element_name])
assert element_name in result.get_pulled_elements()
with cli.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir:
assert not buildtreedir
- shutil.rmtree(os.path.join(str(tmpdir), 'cas'))
- shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
+ shutil.rmtree(os.path.join(str(tmpdir), "cas"))
+ shutil.rmtree(os.path.join(str(tmpdir), "artifacts"))
# Assert that the default behaviour of pull to not include buildtrees on the artifact
# in share1 which was purposely cached with an empty one behaves as expected. As such the
# pulled artifact will have a dangling ref for the buildtree dir, regardless of content,
# leading to no buildtreedir being extracted
- result = cli.run(project=project, args=['artifact', 'pull', element_name])
+ result = cli.run(project=project, args=["artifact", "pull", element_name])
assert element_name in result.get_pulled_elements()
with cli.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir:
assert not buildtreedir
- shutil.rmtree(os.path.join(str(tmpdir), 'cas'))
- shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
+ shutil.rmtree(os.path.join(str(tmpdir), "cas"))
+ shutil.rmtree(os.path.join(str(tmpdir), "artifacts"))
# Repeat building the artifacts, this time with cache-buildtrees set to
# 'always' via the cli, as such the buildtree dir should not be empty
- cli.configure({
- 'artifacts': {'url': share2.repo, 'push': True},
- 'cachedir': str(tmpdir)
- })
- result = cli.run(project=project, args=['--cache-buildtrees', 'always', 'build', element_name])
+ cli.configure({"artifacts": {"url": share2.repo, "push": True}, "cachedir": str(tmpdir)})
+ result = cli.run(project=project, args=["--cache-buildtrees", "always", "build", element_name])
assert result.exit_code == 0
- assert cli.get_element_state(project, element_name) == 'cached'
- assert share2.get_artifact(cli.get_artifact_name(project, 'test', element_name))
+ assert cli.get_element_state(project, element_name) == "cached"
+ assert share2.get_artifact(cli.get_artifact_name(project, "test", element_name))
# Cache key will be the same however the digest hash will have changed as expected, so reconstruct paths
with cli.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir:
@@ -120,28 +111,30 @@ def test_cache_buildtrees(cli, tmpdir, datafiles):
# Delete the local cached artifacts, and assert that when pulled with --pull-buildtrees
# that it was cached in share2 as expected with a populated buildtree dir
- shutil.rmtree(os.path.join(str(tmpdir), 'cas'))
- shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
- assert cli.get_element_state(project, element_name) != 'cached'
- result = cli.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name])
+ shutil.rmtree(os.path.join(str(tmpdir), "cas"))
+ shutil.rmtree(os.path.join(str(tmpdir), "artifacts"))
+ assert cli.get_element_state(project, element_name) != "cached"
+ result = cli.run(project=project, args=["--pull-buildtrees", "artifact", "pull", element_name])
assert element_name in result.get_pulled_elements()
with cli.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir:
assert os.path.isdir(buildtreedir)
assert os.listdir(buildtreedir)
- shutil.rmtree(os.path.join(str(tmpdir), 'cas'))
- shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
+ shutil.rmtree(os.path.join(str(tmpdir), "cas"))
+ shutil.rmtree(os.path.join(str(tmpdir), "artifacts"))
# Clarify that the user config option for cache-buildtrees works as the cli
# main option does. Point to share3 which does not have the artifacts cached to force
# a build
- cli.configure({
- 'artifacts': {'url': share3.repo, 'push': True},
- 'cachedir': str(tmpdir),
- 'cache': {'cache-buildtrees': 'always'}
- })
- result = cli.run(project=project, args=['build', element_name])
+ cli.configure(
+ {
+ "artifacts": {"url": share3.repo, "push": True},
+ "cachedir": str(tmpdir),
+ "cache": {"cache-buildtrees": "always"},
+ }
+ )
+ result = cli.run(project=project, args=["build", element_name])
assert result.exit_code == 0
- assert cli.get_element_state(project, element_name) == 'cached'
+ assert cli.get_element_state(project, element_name) == "cached"
with cli.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir:
assert os.path.isdir(buildtreedir)
assert os.listdir(buildtreedir)
diff --git a/tests/integration/autotools.py b/tests/integration/autotools.py
index 16cb38af7..7a3b946b2 100644
--- a/tests/integration/autotools.py
+++ b/tests/integration/autotools.py
@@ -12,67 +12,80 @@ from buildstream.testing._utils.site import HAVE_SANDBOX
pytestmark = pytest.mark.integration
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project"
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
# Test that an autotools build 'works' - we use the autotools sample
# amhello project for this.
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_autotools_build(cli, datafiles):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout')
- element_name = 'autotools/amhello.bst'
+ checkout = os.path.join(cli.directory, "checkout")
+ element_name = "autotools/amhello.bst"
- result = cli.run(project=project, args=['build', element_name])
+ result = cli.run(project=project, args=["build", element_name])
assert result.exit_code == 0
- result = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout])
+ result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
assert result.exit_code == 0
- assert_contains(checkout, ['/usr', '/usr/lib', '/usr/bin',
- '/usr/share',
- '/usr/bin/hello', '/usr/share/doc',
- '/usr/share/doc/amhello',
- '/usr/share/doc/amhello/README'])
+ assert_contains(
+ checkout,
+ [
+ "/usr",
+ "/usr/lib",
+ "/usr/bin",
+ "/usr/share",
+ "/usr/bin/hello",
+ "/usr/share/doc",
+ "/usr/share/doc/amhello",
+ "/usr/share/doc/amhello/README",
+ ],
+ )
# Test that an autotools build 'works' - we use the autotools sample
# amhello project for this.
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
+@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox")
def test_autotools_confroot_build(cli, datafiles):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout')
- element_name = 'autotools/amhelloconfroot.bst'
+ checkout = os.path.join(cli.directory, "checkout")
+ element_name = "autotools/amhelloconfroot.bst"
- result = cli.run(project=project, args=['build', element_name])
+ result = cli.run(project=project, args=["build", element_name])
assert result.exit_code == 0
- result = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout])
+ result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
assert result.exit_code == 0
- assert_contains(checkout, ['/usr', '/usr/lib', '/usr/bin',
- '/usr/share',
- '/usr/bin/hello', '/usr/share/doc',
- '/usr/share/doc/amhello',
- '/usr/share/doc/amhello/README'])
+ assert_contains(
+ checkout,
+ [
+ "/usr",
+ "/usr/lib",
+ "/usr/bin",
+ "/usr/share",
+ "/usr/bin/hello",
+ "/usr/share/doc",
+ "/usr/share/doc/amhello",
+ "/usr/share/doc/amhello/README",
+ ],
+ )
# Test running an executable built with autotools
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_autotools_run(cli, datafiles):
project = str(datafiles)
- element_name = 'autotools/amhello.bst'
+ element_name = "autotools/amhello.bst"
- result = cli.run(project=project, args=['build', element_name])
+ result = cli.run(project=project, args=["build", element_name])
assert result.exit_code == 0
- result = cli.run(project=project, args=['shell', element_name, '/usr/bin/hello'])
+ result = cli.run(project=project, args=["shell", element_name, "/usr/bin/hello"])
assert result.exit_code == 0
- assert result.output == 'Hello World!\nThis is amhello 1.0.\n'
+ assert result.output == "Hello World!\nThis is amhello 1.0.\n"
diff --git a/tests/integration/build-uid.py b/tests/integration/build-uid.py
index 66f9b3fbc..2fc78d263 100644
--- a/tests/integration/build-uid.py
+++ b/tests/integration/build-uid.py
@@ -10,55 +10,38 @@ from buildstream.testing._utils.site import HAVE_SANDBOX, IS_LINUX
pytestmark = pytest.mark.integration
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project"
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
-@pytest.mark.skipif(not IS_LINUX or HAVE_SANDBOX != "bwrap", reason='Only available on linux with bubblewrap')
+@pytest.mark.skipif(not IS_LINUX or HAVE_SANDBOX != "bwrap", reason="Only available on linux with bubblewrap")
@pytest.mark.datafiles(DATA_DIR)
def test_build_uid_overridden(cli, datafiles):
project = str(datafiles)
- element_name = 'build-uid/build-uid.bst'
-
- project_config = {
- 'name': 'build-uid-test',
- 'sandbox': {
- 'build-uid': 800,
- 'build-gid': 900
- }
- }
-
- result = cli.run_project_config(
- project=project, project_config=project_config, args=['build', element_name])
+ element_name = "build-uid/build-uid.bst"
+
+ project_config = {"name": "build-uid-test", "sandbox": {"build-uid": 800, "build-gid": 900}}
+
+ result = cli.run_project_config(project=project, project_config=project_config, args=["build", element_name])
assert result.exit_code == 0
-@pytest.mark.skipif(not IS_LINUX or HAVE_SANDBOX != "bwrap", reason='Only available on linux with bubbelwrap')
+@pytest.mark.skipif(not IS_LINUX or HAVE_SANDBOX != "bwrap", reason="Only available on linux with bubbelwrap")
@pytest.mark.datafiles(DATA_DIR)
def test_build_uid_in_project(cli, datafiles):
project = str(datafiles)
- element_name = 'build-uid/build-uid-1023.bst'
-
- project_config = {
- 'name': 'build-uid-test',
- 'sandbox': {
- 'build-uid': 1023,
- 'build-gid': 3490
- }
- }
-
- result = cli.run_project_config(
- project=project, project_config=project_config, args=['build', element_name])
+ element_name = "build-uid/build-uid-1023.bst"
+
+ project_config = {"name": "build-uid-test", "sandbox": {"build-uid": 1023, "build-gid": 3490}}
+
+ result = cli.run_project_config(project=project, project_config=project_config, args=["build", element_name])
assert result.exit_code == 0
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(HAVE_SANDBOX != "bwrap", reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(HAVE_SANDBOX != "bwrap", reason="Only available with a functioning sandbox")
def test_build_uid_default(cli, datafiles):
project = str(datafiles)
- element_name = 'build-uid/build-uid-default.bst'
+ element_name = "build-uid/build-uid-default.bst"
- result = cli.run(project=project, args=['build', element_name])
+ result = cli.run(project=project, args=["build", element_name])
assert result.exit_code == 0
diff --git a/tests/integration/cachedfail.py b/tests/integration/cachedfail.py
index 63ad2d4d3..9d68635e8 100644
--- a/tests/integration/cachedfail.py
+++ b/tests/integration/cachedfail.py
@@ -31,158 +31,115 @@ from tests.testutils import create_artifact_share
pytestmark = pytest.mark.integration
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project"
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_build_checkout_cached_fail(cli, datafiles):
project = str(datafiles)
- element_path = os.path.join(project, 'elements', 'element.bst')
- checkout = os.path.join(cli.directory, 'checkout')
+ element_path = os.path.join(project, "elements", "element.bst")
+ checkout = os.path.join(cli.directory, "checkout")
# Write out our test target
element = {
- 'kind': 'script',
- 'depends': [
- {
- 'filename': 'base.bst',
- 'type': 'build',
- },
- ],
- 'config': {
- 'commands': [
- 'touch %{install-root}/foo',
- 'false',
- ],
- },
+ "kind": "script",
+ "depends": [{"filename": "base.bst", "type": "build",},],
+ "config": {"commands": ["touch %{install-root}/foo", "false",],},
}
_yaml.roundtrip_dump(element, element_path)
# Try to build it, this should result in a failure that contains the content
- result = cli.run(project=project, args=['build', 'element.bst'])
+ result = cli.run(project=project, args=["build", "element.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
# Assert that it's cached in a failed artifact
- assert cli.get_element_state(project, 'element.bst') == 'failed'
+ assert cli.get_element_state(project, "element.bst") == "failed"
# Now check it out
- result = cli.run(project=project, args=[
- 'artifact', 'checkout', 'element.bst', '--directory', checkout
- ])
+ result = cli.run(project=project, args=["artifact", "checkout", "element.bst", "--directory", checkout])
result.assert_success()
# Check that the checkout contains the file created before failure
- filename = os.path.join(checkout, 'foo')
+ filename = os.path.join(checkout, "foo")
assert os.path.exists(filename)
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_build_depend_on_cached_fail(cli, datafiles):
project = str(datafiles)
- dep_path = os.path.join(project, 'elements', 'dep.bst')
- target_path = os.path.join(project, 'elements', 'target.bst')
+ dep_path = os.path.join(project, "elements", "dep.bst")
+ target_path = os.path.join(project, "elements", "target.bst")
dep = {
- 'kind': 'script',
- 'depends': [
- {
- 'filename': 'base.bst',
- 'type': 'build',
- },
- ],
- 'config': {
- 'commands': [
- 'touch %{install-root}/foo',
- 'false',
- ],
- },
+ "kind": "script",
+ "depends": [{"filename": "base.bst", "type": "build",},],
+ "config": {"commands": ["touch %{install-root}/foo", "false",],},
}
_yaml.roundtrip_dump(dep, dep_path)
target = {
- 'kind': 'script',
- 'depends': [
- {
- 'filename': 'base.bst',
- 'type': 'build',
- },
- {
- 'filename': 'dep.bst',
- 'type': 'build',
- },
- ],
- 'config': {
- 'commands': [
- 'test -e /foo',
- ],
- },
+ "kind": "script",
+ "depends": [{"filename": "base.bst", "type": "build",}, {"filename": "dep.bst", "type": "build",},],
+ "config": {"commands": ["test -e /foo",],},
}
_yaml.roundtrip_dump(target, target_path)
# Try to build it, this should result in caching a failure to build dep
- result = cli.run(project=project, args=['build', 'dep.bst'])
+ result = cli.run(project=project, args=["build", "dep.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
# Assert that it's cached in a failed artifact
- assert cli.get_element_state(project, 'dep.bst') == 'failed'
+ assert cli.get_element_state(project, "dep.bst") == "failed"
# Now we should fail because we've a cached fail of dep
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
# Assert that it's not yet built, since one of its dependencies isn't ready.
- assert cli.get_element_state(project, 'target.bst') == 'waiting'
+ assert cli.get_element_state(project, "target.bst") == "waiting"
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.parametrize("on_error", ("continue", "quit"))
def test_push_cached_fail(cli, tmpdir, datafiles, on_error):
- if on_error == 'quit':
- pytest.xfail('https://gitlab.com/BuildStream/buildstream/issues/534')
+ if on_error == "quit":
+ pytest.xfail("https://gitlab.com/BuildStream/buildstream/issues/534")
project = str(datafiles)
- element_path = os.path.join(project, 'elements', 'element.bst')
+ element_path = os.path.join(project, "elements", "element.bst")
# Write out our test target
element = {
- 'kind': 'script',
- 'depends': [
- {
- 'filename': 'base.bst',
- 'type': 'build',
- },
- ],
- 'config': {
- 'commands': [
- 'false',
+ "kind": "script",
+ "depends": [{"filename": "base.bst", "type": "build",},],
+ "config": {
+ "commands": [
+ "false",
# Ensure unique cache key for different test variants
- 'TEST="{}"'.format(os.environ.get('PYTEST_CURRENT_TEST')),
+ 'TEST="{}"'.format(os.environ.get("PYTEST_CURRENT_TEST")),
],
},
}
_yaml.roundtrip_dump(element, element_path)
- with create_artifact_share(os.path.join(str(tmpdir), 'remote')) as share:
- cli.configure({
- 'artifacts': {'url': share.repo, 'push': True},
- })
+ with create_artifact_share(os.path.join(str(tmpdir), "remote")) as share:
+ cli.configure(
+ {"artifacts": {"url": share.repo, "push": True},}
+ )
# Build the element, continuing to finish active jobs on error.
- result = cli.run(project=project, args=['--on-error={}'.format(on_error), 'build', 'element.bst'])
+ result = cli.run(project=project, args=["--on-error={}".format(on_error), "build", "element.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
# This element should have failed
- assert cli.get_element_state(project, 'element.bst') == 'failed'
+ assert cli.get_element_state(project, "element.bst") == "failed"
# This element should have been pushed to the remote
- assert share.get_artifact(cli.get_artifact_name(project, 'test', 'element.bst'))
+ assert share.get_artifact(cli.get_artifact_name(project, "test", "element.bst"))
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.parametrize("on_error", ("continue", "quit"))
def test_push_failed_missing_shell(cli, tmpdir, datafiles, on_error):
@@ -191,78 +148,69 @@ def test_push_failed_missing_shell(cli, tmpdir, datafiles, on_error):
When we don't have a valid shell, the artifact will be empty, not even the root directory.
This ensures we handle the case of an entirely empty artifact correctly.
"""
- if on_error == 'quit':
- pytest.xfail('https://gitlab.com/BuildStream/buildstream/issues/534')
+ if on_error == "quit":
+ pytest.xfail("https://gitlab.com/BuildStream/buildstream/issues/534")
project = str(datafiles)
- element_path = os.path.join(project, 'elements', 'element.bst')
+ element_path = os.path.join(project, "elements", "element.bst")
# Write out our test target
element = {
- 'kind': 'script',
- 'config': {
- 'commands': [
- 'false',
+ "kind": "script",
+ "config": {
+ "commands": [
+ "false",
# Ensure unique cache key for different test variants
- 'TEST="{}"'.format(os.environ.get('PYTEST_CURRENT_TEST')),
+ 'TEST="{}"'.format(os.environ.get("PYTEST_CURRENT_TEST")),
],
},
}
_yaml.roundtrip_dump(element, element_path)
- with create_artifact_share(os.path.join(str(tmpdir), 'remote')) as share:
- cli.configure({
- 'artifacts': {'url': share.repo, 'push': True},
- })
+ with create_artifact_share(os.path.join(str(tmpdir), "remote")) as share:
+ cli.configure(
+ {"artifacts": {"url": share.repo, "push": True},}
+ )
# Build the element, continuing to finish active jobs on error.
- result = cli.run(project=project, args=['--on-error={}'.format(on_error), 'build', 'element.bst'])
+ result = cli.run(project=project, args=["--on-error={}".format(on_error), "build", "element.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
# This element should have failed
- assert cli.get_element_state(project, 'element.bst') == 'failed'
+ assert cli.get_element_state(project, "element.bst") == "failed"
# This element should have been pushed to the remote
- assert share.get_artifact(cli.get_artifact_name(project, 'test', 'element.bst'))
+ assert share.get_artifact(cli.get_artifact_name(project, "test", "element.bst"))
-@pytest.mark.skipif(HAVE_SANDBOX != 'bwrap', reason='Only available with bubblewrap on Linux')
+@pytest.mark.skipif(HAVE_SANDBOX != "bwrap", reason="Only available with bubblewrap on Linux")
@pytest.mark.datafiles(DATA_DIR)
def test_host_tools_errors_are_not_cached(cli, datafiles, tmp_path):
# Create symlink to buildbox-casd to work with custom PATH
- buildbox_casd = tmp_path.joinpath('bin/buildbox-casd')
+ buildbox_casd = tmp_path.joinpath("bin/buildbox-casd")
buildbox_casd.parent.mkdir()
- os.symlink(utils.get_host_tool('buildbox-casd'), str(buildbox_casd))
+ os.symlink(utils.get_host_tool("buildbox-casd"), str(buildbox_casd))
project = str(datafiles)
- element_path = os.path.join(project, 'elements', 'element.bst')
+ element_path = os.path.join(project, "elements", "element.bst")
# Write out our test target
element = {
- 'kind': 'script',
- 'depends': [
- {
- 'filename': 'base.bst',
- 'type': 'build',
- },
- ],
- 'config': {
- 'commands': [
- 'true',
- ],
- },
+ "kind": "script",
+ "depends": [{"filename": "base.bst", "type": "build",},],
+ "config": {"commands": ["true",],},
}
_yaml.roundtrip_dump(element, element_path)
# Build without access to host tools, this will fail
result1 = cli.run(
project=project,
- args=['build', 'element.bst'],
- env={'PATH': str(tmp_path.joinpath('bin')),
- 'BST_FORCE_SANDBOX': None})
- result1.assert_task_error(ErrorDomain.SANDBOX, 'unavailable-local-sandbox')
- assert cli.get_element_state(project, 'element.bst') == 'buildable'
+ args=["build", "element.bst"],
+ env={"PATH": str(tmp_path.joinpath("bin")), "BST_FORCE_SANDBOX": None},
+ )
+ result1.assert_task_error(ErrorDomain.SANDBOX, "unavailable-local-sandbox")
+ assert cli.get_element_state(project, "element.bst") == "buildable"
# When rebuilding, this should work
- result2 = cli.run(project=project, args=['build', 'element.bst'])
+ result2 = cli.run(project=project, args=["build", "element.bst"])
result2.assert_success()
- assert cli.get_element_state(project, 'element.bst') == 'cached'
+ assert cli.get_element_state(project, "element.bst") == "cached"
diff --git a/tests/integration/cmake.py b/tests/integration/cmake.py
index a0298c2c3..f5d2f6cc9 100644
--- a/tests/integration/cmake.py
+++ b/tests/integration/cmake.py
@@ -12,57 +12,57 @@ from buildstream.testing._utils.site import HAVE_SANDBOX
pytestmark = pytest.mark.integration
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project"
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_cmake_build(cli, datafiles):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout')
- element_name = 'cmake/cmakehello.bst'
+ checkout = os.path.join(cli.directory, "checkout")
+ element_name = "cmake/cmakehello.bst"
- result = cli.run(project=project, args=['build', element_name])
+ result = cli.run(project=project, args=["build", element_name])
assert result.exit_code == 0
- result = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout])
+ result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
assert result.exit_code == 0
- assert_contains(checkout, ['/usr', '/usr/bin', '/usr/bin/hello'])
+ assert_contains(checkout, ["/usr", "/usr/bin", "/usr/bin/hello"])
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
+@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox")
def test_cmake_confroot_build(cli, datafiles):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout')
- element_name = 'cmake/cmakeconfroothello.bst'
+ checkout = os.path.join(cli.directory, "checkout")
+ element_name = "cmake/cmakeconfroothello.bst"
- result = cli.run(project=project, args=['build', element_name])
+ result = cli.run(project=project, args=["build", element_name])
assert result.exit_code == 0
- result = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout])
+ result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
assert result.exit_code == 0
- assert_contains(checkout, ['/usr', '/usr/bin', '/usr/bin/hello'])
+ assert_contains(checkout, ["/usr", "/usr/bin", "/usr/bin/hello"])
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_cmake_run(cli, datafiles):
project = str(datafiles)
- element_name = 'cmake/cmakehello.bst'
+ element_name = "cmake/cmakehello.bst"
- result = cli.run(project=project, args=['build', element_name])
+ result = cli.run(project=project, args=["build", element_name])
assert result.exit_code == 0
- result = cli.run(project=project, args=['shell', element_name, '/usr/bin/hello'])
+ result = cli.run(project=project, args=["shell", element_name, "/usr/bin/hello"])
assert result.exit_code == 0
- assert result.output == """Hello World!
+ assert (
+ result.output
+ == """Hello World!
This is hello.
"""
+ )
diff --git a/tests/integration/compose-symlinks.py b/tests/integration/compose-symlinks.py
index 061d8f8e4..7b807d1d5 100644
--- a/tests/integration/compose-symlinks.py
+++ b/tests/integration/compose-symlinks.py
@@ -10,10 +10,7 @@ from buildstream.testing import cli_integration as cli # pylint: disable=unused
pytestmark = pytest.mark.integration
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project"
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
# Test that staging a file inside a directory symlink fails.
@@ -26,11 +23,11 @@ def test_compose_symlinks(cli, tmpdir, datafiles):
# Symlinks do not survive being placed in a source distribution
# ('setup.py sdist'), so we have to create the one we need here.
- project_files = os.path.join(project, 'files', 'compose-symlinks', 'base')
- symlink_file = os.path.join(project_files, 'sbin')
- os.symlink(os.path.join('usr', 'sbin'), symlink_file, target_is_directory=True)
+ project_files = os.path.join(project, "files", "compose-symlinks", "base")
+ symlink_file = os.path.join(project_files, "sbin")
+ os.symlink(os.path.join("usr", "sbin"), symlink_file, target_is_directory=True)
- result = cli.run(project=project, args=['build', 'compose-symlinks/compose.bst'])
+ result = cli.run(project=project, args=["build", "compose-symlinks/compose.bst"])
assert result.exit_code == -1
- assert 'Destination is a symlink, not a directory: /sbin' in result.stderr
+ assert "Destination is a symlink, not a directory: /sbin" in result.stderr
diff --git a/tests/integration/compose.py b/tests/integration/compose.py
index 3562ed94b..638dbc8dd 100644
--- a/tests/integration/compose.py
+++ b/tests/integration/compose.py
@@ -14,10 +14,7 @@ from buildstream.testing._utils.site import HAVE_SANDBOX
pytestmark = pytest.mark.integration
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project"
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
def create_compose_element(name, path, config=None):
@@ -25,83 +22,110 @@ def create_compose_element(name, path, config=None):
config = {}
element = {
- 'kind': 'compose',
- 'depends': [{
- 'filename': 'compose/amhello.bst',
- 'type': 'build'
- }, {
- 'filename': 'compose/test.bst',
- 'type': 'build'
- }],
- 'config': config
+ "kind": "compose",
+ "depends": [
+ {"filename": "compose/amhello.bst", "type": "build"},
+ {"filename": "compose/test.bst", "type": "build"},
+ ],
+ "config": config,
}
os.makedirs(os.path.dirname(os.path.join(path, name)), exist_ok=True)
_yaml.roundtrip_dump(element, os.path.join(path, name))
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("include_domains,exclude_domains,expected", [
- # Test flat inclusion
- ([], [], ['/usr', '/usr/bin',
- '/usr/share',
- '/usr/bin/hello',
- '/usr/share/doc', '/usr/share/doc/amhello',
- '/usr/share/doc/amhello/README',
- '/tests', '/tests/test']),
- # Test only runtime
- (['runtime'], [], ['/usr', '/usr/share',
- '/usr/bin', '/usr/bin/hello']),
- # Test with runtime and doc
- (['runtime', 'doc'], [], ['/usr', '/usr/share',
- '/usr/bin', '/usr/bin/hello',
- '/usr/share/doc', '/usr/share/doc/amhello',
- '/usr/share/doc/amhello/README']),
- # Test with only runtime excluded
- ([], ['runtime'], ['/usr', '/usr/share',
- '/usr/share/doc', '/usr/share/doc/amhello',
- '/usr/share/doc/amhello/README',
- '/tests', '/tests/test']),
- # Test with runtime and doc excluded
- ([], ['runtime', 'doc'], ['/usr', '/usr/share',
- '/tests', '/tests/test']),
- # Test with runtime simultaneously in- and excluded
- (['runtime'], ['runtime'], ['/usr', '/usr/share']),
- # Test with runtime included and doc excluded
- (['runtime'], ['doc'], ['/usr', '/usr/share',
- '/usr/bin', '/usr/bin/hello']),
- # Test including a custom 'test' domain
- (['test'], [], ['/usr', '/usr/share',
- '/tests', '/tests/test']),
- # Test excluding a custom 'test' domain
- ([], ['test'], ['/usr', '/usr/bin',
- '/usr/share',
- '/usr/bin/hello',
- '/usr/share/doc', '/usr/share/doc/amhello',
- '/usr/share/doc/amhello/README'])
-])
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-def test_compose_include(cli, datafiles, include_domains,
- exclude_domains, expected):
+@pytest.mark.parametrize(
+ "include_domains,exclude_domains,expected",
+ [
+ # Test flat inclusion
+ (
+ [],
+ [],
+ [
+ "/usr",
+ "/usr/bin",
+ "/usr/share",
+ "/usr/bin/hello",
+ "/usr/share/doc",
+ "/usr/share/doc/amhello",
+ "/usr/share/doc/amhello/README",
+ "/tests",
+ "/tests/test",
+ ],
+ ),
+ # Test only runtime
+ (["runtime"], [], ["/usr", "/usr/share", "/usr/bin", "/usr/bin/hello"]),
+ # Test with runtime and doc
+ (
+ ["runtime", "doc"],
+ [],
+ [
+ "/usr",
+ "/usr/share",
+ "/usr/bin",
+ "/usr/bin/hello",
+ "/usr/share/doc",
+ "/usr/share/doc/amhello",
+ "/usr/share/doc/amhello/README",
+ ],
+ ),
+ # Test with only runtime excluded
+ (
+ [],
+ ["runtime"],
+ [
+ "/usr",
+ "/usr/share",
+ "/usr/share/doc",
+ "/usr/share/doc/amhello",
+ "/usr/share/doc/amhello/README",
+ "/tests",
+ "/tests/test",
+ ],
+ ),
+ # Test with runtime and doc excluded
+ ([], ["runtime", "doc"], ["/usr", "/usr/share", "/tests", "/tests/test"]),
+ # Test with runtime simultaneously in- and excluded
+ (["runtime"], ["runtime"], ["/usr", "/usr/share"]),
+ # Test with runtime included and doc excluded
+ (["runtime"], ["doc"], ["/usr", "/usr/share", "/usr/bin", "/usr/bin/hello"]),
+ # Test including a custom 'test' domain
+ (["test"], [], ["/usr", "/usr/share", "/tests", "/tests/test"]),
+ # Test excluding a custom 'test' domain
+ (
+ [],
+ ["test"],
+ [
+ "/usr",
+ "/usr/bin",
+ "/usr/share",
+ "/usr/bin/hello",
+ "/usr/share/doc",
+ "/usr/share/doc/amhello",
+ "/usr/share/doc/amhello/README",
+ ],
+ ),
+ ],
+)
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
+def test_compose_include(cli, datafiles, include_domains, exclude_domains, expected):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout')
- element_path = os.path.join(project, 'elements')
- element_name = 'compose/compose-amhello.bst'
+ checkout = os.path.join(cli.directory, "checkout")
+ element_path = os.path.join(project, "elements")
+ element_name = "compose/compose-amhello.bst"
# Create a yaml configuration from the specified include and
# exclude domains
- config = {
- 'include': include_domains,
- 'exclude': exclude_domains
- }
+ config = {"include": include_domains, "exclude": exclude_domains}
create_compose_element(element_name, element_path, config=config)
- result = cli.run(project=project, args=['source', 'track', 'compose/amhello.bst'])
+ result = cli.run(project=project, args=["source", "track", "compose/amhello.bst"])
assert result.exit_code == 0
- result = cli.run(project=project, args=['build', element_name])
+ result = cli.run(project=project, args=["build", element_name])
assert result.exit_code == 0
- result = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout])
+ result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
assert result.exit_code == 0
assert set(walk_dir(checkout)) == set(expected)
diff --git a/tests/integration/filter.py b/tests/integration/filter.py
index 2a32d4010..f13521cc8 100644
--- a/tests/integration/filter.py
+++ b/tests/integration/filter.py
@@ -13,27 +13,26 @@ from buildstream.testing._utils.site import HAVE_SANDBOX
pytestmark = pytest.mark.integration
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- 'project'
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
@pytest.mark.datafiles(os.path.join(DATA_DIR))
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_filter_pass_integration(datafiles, cli):
project = str(datafiles)
# Passing integration commands should build nicely
- result = cli.run(project=project, args=['build', 'filter/filter.bst'])
+ result = cli.run(project=project, args=["build", "filter/filter.bst"])
result.assert_success()
# Checking out the element should work
- checkout_dir = os.path.join(project, 'filter')
- result = cli.run(project=project, args=['artifact', 'checkout', '--integrate', '--hardlinks',
- '--directory', checkout_dir, 'filter/filter.bst'])
+ checkout_dir = os.path.join(project, "filter")
+ result = cli.run(
+ project=project,
+ args=["artifact", "checkout", "--integrate", "--hardlinks", "--directory", checkout_dir, "filter/filter.bst"],
+ )
result.assert_success()
# Check that the integration command was run
- assert_contains(checkout_dir, ['/foo'])
+ assert_contains(checkout_dir, ["/foo"])
shutil.rmtree(checkout_dir)
diff --git a/tests/integration/import.py b/tests/integration/import.py
index bac92cadf..f7510e2e5 100644
--- a/tests/integration/import.py
+++ b/tests/integration/import.py
@@ -13,50 +13,46 @@ from buildstream.testing.integration import walk_dir
pytestmark = pytest.mark.integration
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project"
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
def create_import_element(name, path, source, target, source_path):
element = {
- 'kind': 'import',
- 'sources': [{
- 'kind': 'local',
- 'path': source_path
- }],
- 'config': {
- 'source': source,
- 'target': target
- }
+ "kind": "import",
+ "sources": [{"kind": "local", "path": source_path}],
+ "config": {"source": source, "target": target},
}
os.makedirs(os.path.dirname(os.path.join(path, name)), exist_ok=True)
_yaml.roundtrip_dump(element, os.path.join(path, name))
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("source,target,path,expected", [
- ('/', '/', 'files/import-source', ['/test.txt', '/subdir',
- '/subdir/test.txt']),
- ('/subdir', '/', 'files/import-source', ['/test.txt']),
- ('/', '/', 'files/import-source/subdir', ['/test.txt']),
- ('/', '/output', 'files/import-source', ['/output', '/output/test.txt',
- '/output/subdir',
- '/output/subdir/test.txt']),
-])
+@pytest.mark.parametrize(
+ "source,target,path,expected",
+ [
+ ("/", "/", "files/import-source", ["/test.txt", "/subdir", "/subdir/test.txt"]),
+ ("/subdir", "/", "files/import-source", ["/test.txt"]),
+ ("/", "/", "files/import-source/subdir", ["/test.txt"]),
+ (
+ "/",
+ "/output",
+ "files/import-source",
+ ["/output", "/output/test.txt", "/output/subdir", "/output/subdir/test.txt"],
+ ),
+ ],
+)
def test_import(cli, datafiles, source, target, path, expected):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout')
- element_path = os.path.join(project, 'elements')
- element_name = 'import/import.bst'
+ checkout = os.path.join(cli.directory, "checkout")
+ element_path = os.path.join(project, "elements")
+ element_name = "import/import.bst"
create_import_element(element_name, element_path, source, target, path)
- res = cli.run(project=project, args=['build', element_name])
+ res = cli.run(project=project, args=["build", element_name])
assert res.exit_code == 0
- cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout])
+ cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
assert res.exit_code == 0
assert set(walk_dir(checkout)) == set(expected)
diff --git a/tests/integration/make.py b/tests/integration/make.py
index 664e7ca7a..109777622 100644
--- a/tests/integration/make.py
+++ b/tests/integration/make.py
@@ -12,41 +12,37 @@ from buildstream.testing._utils.site import HAVE_SANDBOX
pytestmark = pytest.mark.integration
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project"
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
# Test that a make build 'works' - we use the make sample
# makehello project for this.
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_make_build(cli, datafiles):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout')
- element_name = 'make/makehello.bst'
+ checkout = os.path.join(cli.directory, "checkout")
+ element_name = "make/makehello.bst"
- result = cli.run(project=project, args=['build', element_name])
+ result = cli.run(project=project, args=["build", element_name])
assert result.exit_code == 0
- result = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout])
+ result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
assert result.exit_code == 0
- assert_contains(checkout, ['/usr', '/usr/bin',
- '/usr/bin/hello'])
+ assert_contains(checkout, ["/usr", "/usr/bin", "/usr/bin/hello"])
# Test running an executable built with make
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_make_run(cli, datafiles):
project = str(datafiles)
- element_name = 'make/makehello.bst'
+ element_name = "make/makehello.bst"
- result = cli.run(project=project, args=['build', element_name])
+ result = cli.run(project=project, args=["build", element_name])
assert result.exit_code == 0
- result = cli.run(project=project, args=['shell', element_name, '/usr/bin/hello'])
+ result = cli.run(project=project, args=["shell", element_name, "/usr/bin/hello"])
assert result.exit_code == 0
- assert result.output == 'Hello, world\n'
+ assert result.output == "Hello, world\n"
diff --git a/tests/integration/manual.py b/tests/integration/manual.py
index 2ac7f74d0..c6a84b062 100644
--- a/tests/integration/manual.py
+++ b/tests/integration/manual.py
@@ -13,141 +13,139 @@ from buildstream.testing._utils.site import HAVE_SANDBOX
pytestmark = pytest.mark.integration
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project"
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
def create_manual_element(name, path, config, variables, environment):
element = {
- 'kind': 'manual',
- 'depends': [{
- 'filename': 'base.bst',
- 'type': 'build'
- }],
- 'config': config,
- 'variables': variables,
- 'environment': environment
+ "kind": "manual",
+ "depends": [{"filename": "base.bst", "type": "build"}],
+ "config": config,
+ "variables": variables,
+ "environment": environment,
}
os.makedirs(os.path.dirname(os.path.join(path, name)), exist_ok=True)
_yaml.roundtrip_dump(element, os.path.join(path, name))
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_manual_element(cli, datafiles):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout')
- element_path = os.path.join(project, 'elements')
- element_name = 'import/import.bst'
-
- create_manual_element(element_name, element_path, {
- 'configure-commands': ["echo './configure' >> test"],
- 'build-commands': ["echo 'make' >> test"],
- 'install-commands': [
- "echo 'make install' >> test",
- "cp test %{install-root}"
- ],
- 'strip-commands': ["echo 'strip' >> %{install-root}/test"]
- }, {}, {})
-
- res = cli.run(project=project, args=['build', element_name])
+ checkout = os.path.join(cli.directory, "checkout")
+ element_path = os.path.join(project, "elements")
+ element_name = "import/import.bst"
+
+ create_manual_element(
+ element_name,
+ element_path,
+ {
+ "configure-commands": ["echo './configure' >> test"],
+ "build-commands": ["echo 'make' >> test"],
+ "install-commands": ["echo 'make install' >> test", "cp test %{install-root}"],
+ "strip-commands": ["echo 'strip' >> %{install-root}/test"],
+ },
+ {},
+ {},
+ )
+
+ res = cli.run(project=project, args=["build", element_name])
assert res.exit_code == 0
- cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout])
+ cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
assert res.exit_code == 0
- with open(os.path.join(checkout, 'test')) as f:
+ with open(os.path.join(checkout, "test")) as f:
text = f.read()
- assert text == """./configure
+ assert (
+ text
+ == """./configure
make
make install
strip
"""
+ )
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_manual_element_environment(cli, datafiles):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout')
- element_path = os.path.join(project, 'elements')
- element_name = 'import/import.bst'
-
- create_manual_element(element_name, element_path, {
- 'install-commands': [
- "echo $V >> test",
- "cp test %{install-root}"
- ]
- }, {
- }, {
- 'V': 2
- })
-
- res = cli.run(project=project, args=['build', element_name])
+ checkout = os.path.join(cli.directory, "checkout")
+ element_path = os.path.join(project, "elements")
+ element_name = "import/import.bst"
+
+ create_manual_element(
+ element_name, element_path, {"install-commands": ["echo $V >> test", "cp test %{install-root}"]}, {}, {"V": 2}
+ )
+
+ res = cli.run(project=project, args=["build", element_name])
assert res.exit_code == 0
- cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout])
+ cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
assert res.exit_code == 0
- with open(os.path.join(checkout, 'test')) as f:
+ with open(os.path.join(checkout, "test")) as f:
text = f.read()
assert text == "2\n"
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_manual_element_noparallel(cli, datafiles):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout')
- element_path = os.path.join(project, 'elements')
- element_name = 'import/import.bst'
-
- create_manual_element(element_name, element_path, {
- 'install-commands': [
- "echo $MAKEFLAGS >> test",
- "echo $V >> test",
- "cp test %{install-root}"
- ]
- }, {
- 'notparallel': True
- }, {
- 'MAKEFLAGS': '-j%{max-jobs} -Wall',
- 'V': 2
- })
-
- res = cli.run(project=project, args=['build', element_name])
+ checkout = os.path.join(cli.directory, "checkout")
+ element_path = os.path.join(project, "elements")
+ element_name = "import/import.bst"
+
+ create_manual_element(
+ element_name,
+ element_path,
+ {"install-commands": ["echo $MAKEFLAGS >> test", "echo $V >> test", "cp test %{install-root}"]},
+ {"notparallel": True},
+ {"MAKEFLAGS": "-j%{max-jobs} -Wall", "V": 2},
+ )
+
+ res = cli.run(project=project, args=["build", element_name])
assert res.exit_code == 0
- cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout])
+ cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
assert res.exit_code == 0
- with open(os.path.join(checkout, 'test')) as f:
+ with open(os.path.join(checkout, "test")) as f:
text = f.read()
- assert text == """-j1 -Wall
+ assert (
+ text
+ == """-j1 -Wall
2
"""
+ )
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_manual_element_logging(cli, datafiles):
project = str(datafiles)
- element_path = os.path.join(project, 'elements')
- element_name = 'import/import.bst'
-
- create_manual_element(element_name, element_path, {
- 'configure-commands': ["echo configure"],
- 'build-commands': ["echo build"],
- 'install-commands': ["echo install"],
- 'strip-commands': ["echo strip"]
- }, {}, {})
-
- res = cli.run(project=project, args=['build', element_name])
+ element_path = os.path.join(project, "elements")
+ element_name = "import/import.bst"
+
+ create_manual_element(
+ element_name,
+ element_path,
+ {
+ "configure-commands": ["echo configure"],
+ "build-commands": ["echo build"],
+ "install-commands": ["echo install"],
+ "strip-commands": ["echo strip"],
+ },
+ {},
+ {},
+ )
+
+ res = cli.run(project=project, args=["build", element_name])
assert res.exit_code == 0
# Verify that individual commands are logged
diff --git a/tests/integration/messages.py b/tests/integration/messages.py
index 42725fc5b..f35b778d6 100644
--- a/tests/integration/messages.py
+++ b/tests/integration/messages.py
@@ -33,29 +33,20 @@ pytestmark = pytest.mark.integration
# Project directory
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project",
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",)
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_disable_message_lines(cli, datafiles):
project = str(datafiles)
- element_path = os.path.join(project, 'elements')
- element_name = 'message.bst'
+ element_path = os.path.join(project, "elements")
+ element_name = "message.bst"
element = {
- 'kind': 'manual',
- 'depends': [{
- 'filename': 'base.bst'
- }],
- 'config': {
- 'build-commands':
- ['echo "Silly message"'],
- 'strip-commands': []
- }
+ "kind": "manual",
+ "depends": [{"filename": "base.bst"}],
+ "config": {"build-commands": ['echo "Silly message"'], "strip-commands": []},
}
os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True)
@@ -68,43 +59,34 @@ def test_disable_message_lines(cli, datafiles):
# Let's now build it again, but with --message-lines 0
cli.remove_artifact_from_cache(project, element_name)
- result = cli.run(project=project, args=["--message-lines", "0",
- "build", element_name])
+ result = cli.run(project=project, args=["--message-lines", "0", "build", element_name])
result.assert_success()
assert "Message contains " not in result.stderr
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_disable_error_lines(cli, datafiles):
project = str(datafiles)
- element_path = os.path.join(project, 'elements')
- element_name = 'message.bst'
+ element_path = os.path.join(project, "elements")
+ element_name = "message.bst"
element = {
- 'kind': 'manual',
- 'depends': [{
- 'filename': 'base.bst'
- }],
- 'config': {
- 'build-commands':
- ['This is a syntax error > >'],
- 'strip-commands': []
- }
+ "kind": "manual",
+ "depends": [{"filename": "base.bst"}],
+ "config": {"build-commands": ["This is a syntax error > >"], "strip-commands": []},
}
os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True)
_yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
# First we check that we get the syntax error
- result = cli.run(project=project, args=["--error-lines", "0",
- "build", element_name])
+ result = cli.run(project=project, args=["--error-lines", "0", "build", element_name])
result.assert_main_error(ErrorDomain.STREAM, None)
assert "This is a syntax error" in result.stderr
# Let's now build it again, but with --error-lines 0
cli.remove_artifact_from_cache(project, element_name)
- result = cli.run(project=project, args=["--error-lines", "0",
- "build", element_name])
+ result = cli.run(project=project, args=["--error-lines", "0", "build", element_name])
result.assert_main_error(ErrorDomain.STREAM, None)
assert "Printing the last" not in result.stderr
diff --git a/tests/integration/pip_element.py b/tests/integration/pip_element.py
index da0badcb3..bfc4f8098 100644
--- a/tests/integration/pip_element.py
+++ b/tests/integration/pip_element.py
@@ -17,119 +17,101 @@ from tests.testutils import setup_pypi_repo # pylint: disable=unused-import
pytestmark = pytest.mark.integration
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project"
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_pip_build(cli, datafiles):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout')
- element_path = os.path.join(project, 'elements')
- element_name = 'pip/hello.bst'
+ checkout = os.path.join(cli.directory, "checkout")
+ element_path = os.path.join(project, "elements")
+ element_name = "pip/hello.bst"
element = {
- 'kind': 'pip',
- 'variables': {
- 'pip': 'pip3'
- },
- 'depends': [{
- 'filename': 'base.bst'
- }],
- 'sources': [{
- 'kind': 'tar',
- 'url': 'file://{}/files/hello.tar.xz'.format(project),
- 'ref': 'ad96570b552498807abec33c06210bf68378d854ced6753b77916c5ed517610d'
-
- }]
+ "kind": "pip",
+ "variables": {"pip": "pip3"},
+ "depends": [{"filename": "base.bst"}],
+ "sources": [
+ {
+ "kind": "tar",
+ "url": "file://{}/files/hello.tar.xz".format(project),
+ "ref": "ad96570b552498807abec33c06210bf68378d854ced6753b77916c5ed517610d",
+ }
+ ],
}
os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True)
_yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
- result = cli.run(project=project, args=['build', element_name])
+ result = cli.run(project=project, args=["build", element_name])
assert result.exit_code == 0
- result = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout])
+ result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
assert result.exit_code == 0
- assert_contains(checkout, ['/usr', '/usr/lib', '/usr/bin',
- '/usr/bin/hello', '/usr/lib/python3.6'])
+ assert_contains(checkout, ["/usr", "/usr/lib", "/usr/bin", "/usr/bin/hello", "/usr/lib/python3.6"])
# Test running an executable built with pip
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_pip_run(cli, datafiles):
# Create and build our test element
test_pip_build(cli, datafiles)
project = str(datafiles)
- element_name = 'pip/hello.bst'
+ element_name = "pip/hello.bst"
- result = cli.run(project=project, args=['shell', element_name, '/usr/bin/hello'])
+ result = cli.run(project=project, args=["shell", element_name, "/usr/bin/hello"])
assert result.exit_code == 0
- assert result.output == 'Hello, world!\n'
+ assert result.output == "Hello, world!\n"
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_pip_element_should_install_pip_deps(cli, datafiles, setup_pypi_repo):
project = str(datafiles)
- elements_path = os.path.join(project, 'elements')
- element_name = 'pip/hello.bst'
+ elements_path = os.path.join(project, "elements")
+ element_name = "pip/hello.bst"
# check that exotically named packages are imported correctly
- myreqs_packages = 'alohalib'
- dependencies = ['app2', 'app.3', 'app-4', 'app_5', 'app.no.6', 'app-no-7', 'app_no_8']
- mock_packages = {
- myreqs_packages: {
- package: {} for package in dependencies
- }
- }
+ myreqs_packages = "alohalib"
+ dependencies = ["app2", "app.3", "app-4", "app_5", "app.no.6", "app-no-7", "app_no_8"]
+ mock_packages = {myreqs_packages: {package: {} for package in dependencies}}
# set up directories
- pypi_repo = os.path.join(project, 'files', 'pypi-repo')
+ pypi_repo = os.path.join(project, "files", "pypi-repo")
os.makedirs(pypi_repo, exist_ok=True)
os.makedirs(os.path.dirname(os.path.join(elements_path, element_name)), exist_ok=True)
setup_pypi_repo(mock_packages, pypi_repo)
# create pip element
element = {
- 'kind': 'pip',
- 'variables': {
- 'pip': 'pip3'
- },
- 'depends': [{
- 'filename': 'base.bst'
- }],
- 'sources': [
+ "kind": "pip",
+ "variables": {"pip": "pip3"},
+ "depends": [{"filename": "base.bst"}],
+ "sources": [
{
- 'kind': 'tar',
- 'url': 'file://{}/files/hello.tar.xz'.format(project),
+ "kind": "tar",
+ "url": "file://{}/files/hello.tar.xz".format(project),
# FIXME: remove hardcoded ref once issue #1010 is closed
- 'ref': 'ad96570b552498807abec33c06210bf68378d854ced6753b77916c5ed517610d'
+ "ref": "ad96570b552498807abec33c06210bf68378d854ced6753b77916c5ed517610d",
},
- {
- 'kind': 'pip',
- 'url': 'file://{}'.format(os.path.realpath(pypi_repo)),
- 'packages': [myreqs_packages],
- }
- ]
+ {"kind": "pip", "url": "file://{}".format(os.path.realpath(pypi_repo)), "packages": [myreqs_packages],},
+ ],
}
_yaml.roundtrip_dump(element, os.path.join(elements_path, element_name))
- result = cli.run(project=project, args=['source', 'track', element_name])
+ result = cli.run(project=project, args=["source", "track", element_name])
assert result.exit_code == 0
- result = cli.run(project=project, args=['build', element_name])
+ result = cli.run(project=project, args=["build", element_name])
assert result.exit_code == 0
# get installed packages in sandbox
installed_packages = set(
- cli.run(project=project, args=['shell', element_name, 'pip3', 'freeze']).output.split('\n'))
+ cli.run(project=project, args=["shell", element_name, "pip3", "freeze"]).output.split("\n")
+ )
# compare with packages that are expected to be installed
- pip_source_packages = {package.replace('_', "-") + '==0.1' for package in dependencies + [myreqs_packages]}
+ pip_source_packages = {package.replace("_", "-") + "==0.1" for package in dependencies + [myreqs_packages]}
assert pip_source_packages.issubset(installed_packages)
diff --git a/tests/integration/pip_source.py b/tests/integration/pip_source.py
index c221910a6..5d314974d 100644
--- a/tests/integration/pip_source.py
+++ b/tests/integration/pip_source.py
@@ -16,180 +16,162 @@ from tests.testutils.python_repo import setup_pypi_repo # pylint: disable=unuse
pytestmark = pytest.mark.integration
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project"
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
@pytest.mark.datafiles(DATA_DIR)
def test_pip_source_import_packages(cli, datafiles, setup_pypi_repo):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout')
- element_path = os.path.join(project, 'elements')
- element_name = 'pip/hello.bst'
+ checkout = os.path.join(cli.directory, "checkout")
+ element_path = os.path.join(project, "elements")
+ element_name = "pip/hello.bst"
# check that exotically named packages are imported correctly
- myreqs_packages = 'hellolib'
- dependencies = ['app2', 'app.3', 'app-4', 'app_5', 'app.no.6', 'app-no-7', 'app_no_8']
- mock_packages = {
- myreqs_packages: {
- package: {} for package in dependencies
- }
- }
+ myreqs_packages = "hellolib"
+ dependencies = ["app2", "app.3", "app-4", "app_5", "app.no.6", "app-no-7", "app_no_8"]
+ mock_packages = {myreqs_packages: {package: {} for package in dependencies}}
# create mock pypi repository
- pypi_repo = os.path.join(project, 'files', 'pypi-repo')
+ pypi_repo = os.path.join(project, "files", "pypi-repo")
os.makedirs(pypi_repo, exist_ok=True)
setup_pypi_repo(mock_packages, pypi_repo)
element = {
- 'kind': 'import',
- 'sources': [
- {
- 'kind': 'local',
- 'path': 'files/pip-source'
- },
- {
- 'kind': 'pip',
- 'url': 'file://{}'.format(os.path.realpath(pypi_repo)),
- 'packages': [myreqs_packages]
- }
- ]
+ "kind": "import",
+ "sources": [
+ {"kind": "local", "path": "files/pip-source"},
+ {"kind": "pip", "url": "file://{}".format(os.path.realpath(pypi_repo)), "packages": [myreqs_packages]},
+ ],
}
os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True)
_yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
- result = cli.run(project=project, args=['source', 'track', element_name])
+ result = cli.run(project=project, args=["source", "track", element_name])
assert result.exit_code == 0
- result = cli.run(project=project, args=['build', element_name])
+ result = cli.run(project=project, args=["build", element_name])
assert result.exit_code == 0
- result = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout])
+ result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
assert result.exit_code == 0
- assert_contains(checkout, ['/.bst_pip_downloads',
- '/.bst_pip_downloads/hellolib-0.1.tar.gz',
- '/.bst_pip_downloads/app2-0.1.tar.gz',
- '/.bst_pip_downloads/app.3-0.1.tar.gz',
- '/.bst_pip_downloads/app-4-0.1.tar.gz',
- '/.bst_pip_downloads/app_5-0.1.tar.gz',
- '/.bst_pip_downloads/app.no.6-0.1.tar.gz',
- '/.bst_pip_downloads/app-no-7-0.1.tar.gz',
- '/.bst_pip_downloads/app_no_8-0.1.tar.gz'])
+ assert_contains(
+ checkout,
+ [
+ "/.bst_pip_downloads",
+ "/.bst_pip_downloads/hellolib-0.1.tar.gz",
+ "/.bst_pip_downloads/app2-0.1.tar.gz",
+ "/.bst_pip_downloads/app.3-0.1.tar.gz",
+ "/.bst_pip_downloads/app-4-0.1.tar.gz",
+ "/.bst_pip_downloads/app_5-0.1.tar.gz",
+ "/.bst_pip_downloads/app.no.6-0.1.tar.gz",
+ "/.bst_pip_downloads/app-no-7-0.1.tar.gz",
+ "/.bst_pip_downloads/app_no_8-0.1.tar.gz",
+ ],
+ )
@pytest.mark.datafiles(DATA_DIR)
def test_pip_source_import_requirements_files(cli, datafiles, setup_pypi_repo):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout')
- element_path = os.path.join(project, 'elements')
- element_name = 'pip/hello.bst'
+ checkout = os.path.join(cli.directory, "checkout")
+ element_path = os.path.join(project, "elements")
+ element_name = "pip/hello.bst"
# check that exotically named packages are imported correctly
- myreqs_packages = 'hellolib'
- dependencies = ['app2', 'app.3', 'app-4', 'app_5', 'app.no.6', 'app-no-7', 'app_no_8']
- mock_packages = {
- myreqs_packages: {
- package: {} for package in dependencies
- }
- }
+ myreqs_packages = "hellolib"
+ dependencies = ["app2", "app.3", "app-4", "app_5", "app.no.6", "app-no-7", "app_no_8"]
+ mock_packages = {myreqs_packages: {package: {} for package in dependencies}}
# create mock pypi repository
- pypi_repo = os.path.join(project, 'files', 'pypi-repo')
+ pypi_repo = os.path.join(project, "files", "pypi-repo")
os.makedirs(pypi_repo, exist_ok=True)
setup_pypi_repo(mock_packages, pypi_repo)
element = {
- 'kind': 'import',
- 'sources': [
+ "kind": "import",
+ "sources": [
+ {"kind": "local", "path": "files/pip-source"},
{
- 'kind': 'local',
- 'path': 'files/pip-source'
+ "kind": "pip",
+ "url": "file://{}".format(os.path.realpath(pypi_repo)),
+ "requirements-files": ["myreqs.txt"],
},
- {
- 'kind': 'pip',
- 'url': 'file://{}'.format(os.path.realpath(pypi_repo)),
- 'requirements-files': ['myreqs.txt'],
- }
- ]
+ ],
}
os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True)
_yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
- result = cli.run(project=project, args=['source', 'track', element_name])
+ result = cli.run(project=project, args=["source", "track", element_name])
assert result.exit_code == 0
- result = cli.run(project=project, args=['build', element_name])
+ result = cli.run(project=project, args=["build", element_name])
assert result.exit_code == 0
- result = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout])
+ result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
assert result.exit_code == 0
- assert_contains(checkout, ['/.bst_pip_downloads',
- '/.bst_pip_downloads/hellolib-0.1.tar.gz',
- '/.bst_pip_downloads/app2-0.1.tar.gz',
- '/.bst_pip_downloads/app.3-0.1.tar.gz',
- '/.bst_pip_downloads/app-4-0.1.tar.gz',
- '/.bst_pip_downloads/app_5-0.1.tar.gz',
- '/.bst_pip_downloads/app.no.6-0.1.tar.gz',
- '/.bst_pip_downloads/app-no-7-0.1.tar.gz',
- '/.bst_pip_downloads/app_no_8-0.1.tar.gz'])
+ assert_contains(
+ checkout,
+ [
+ "/.bst_pip_downloads",
+ "/.bst_pip_downloads/hellolib-0.1.tar.gz",
+ "/.bst_pip_downloads/app2-0.1.tar.gz",
+ "/.bst_pip_downloads/app.3-0.1.tar.gz",
+ "/.bst_pip_downloads/app-4-0.1.tar.gz",
+ "/.bst_pip_downloads/app_5-0.1.tar.gz",
+ "/.bst_pip_downloads/app.no.6-0.1.tar.gz",
+ "/.bst_pip_downloads/app-no-7-0.1.tar.gz",
+ "/.bst_pip_downloads/app_no_8-0.1.tar.gz",
+ ],
+ )
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_pip_source_build(cli, datafiles, setup_pypi_repo):
project = str(datafiles)
- element_path = os.path.join(project, 'elements')
- element_name = 'pip/hello.bst'
+ element_path = os.path.join(project, "elements")
+ element_name = "pip/hello.bst"
# check that exotically named packages are imported correctly
- myreqs_packages = 'hellolib'
- dependencies = ['app2', 'app.3', 'app-4', 'app_5', 'app.no.6', 'app-no-7', 'app_no_8']
- mock_packages = {
- myreqs_packages: {
- package: {} for package in dependencies
- }
- }
+ myreqs_packages = "hellolib"
+ dependencies = ["app2", "app.3", "app-4", "app_5", "app.no.6", "app-no-7", "app_no_8"]
+ mock_packages = {myreqs_packages: {package: {} for package in dependencies}}
# create mock pypi repository
- pypi_repo = os.path.join(project, 'files', 'pypi-repo')
+ pypi_repo = os.path.join(project, "files", "pypi-repo")
os.makedirs(pypi_repo, exist_ok=True)
setup_pypi_repo(mock_packages, pypi_repo)
element = {
- 'kind': 'manual',
- 'depends': ['base.bst'],
- 'sources': [
+ "kind": "manual",
+ "depends": ["base.bst"],
+ "sources": [
+ {"kind": "local", "path": "files/pip-source"},
{
- 'kind': 'local',
- 'path': 'files/pip-source'
+ "kind": "pip",
+ "url": "file://{}".format(os.path.realpath(pypi_repo)),
+ "requirements-files": ["myreqs.txt"],
+ "packages": dependencies,
},
- {
- 'kind': 'pip',
- 'url': 'file://{}'.format(os.path.realpath(pypi_repo)),
- 'requirements-files': ['myreqs.txt'],
- 'packages': dependencies
- }
],
- 'config': {
- 'install-commands': [
- 'pip3 install --no-index --prefix %{install-root}/usr .bst_pip_downloads/*.tar.gz',
- 'install app1.py %{install-root}/usr/bin/'
+ "config": {
+ "install-commands": [
+ "pip3 install --no-index --prefix %{install-root}/usr .bst_pip_downloads/*.tar.gz",
+ "install app1.py %{install-root}/usr/bin/",
]
- }
+ },
}
os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True)
_yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
- result = cli.run(project=project, args=['source', 'track', element_name])
+ result = cli.run(project=project, args=["source", "track", element_name])
assert result.exit_code == 0
- result = cli.run(project=project, args=['build', element_name])
+ result = cli.run(project=project, args=["build", element_name])
assert result.exit_code == 0
- result = cli.run(project=project, args=['shell', element_name, '/usr/bin/app1.py'])
+ result = cli.run(project=project, args=["shell", element_name, "/usr/bin/app1.py"])
assert result.exit_code == 0
assert result.output == "Hello App1! This is hellolib\n"
diff --git a/tests/integration/project/files/pip-source/app1.py b/tests/integration/project/files/pip-source/app1.py
index ab1005ba4..b96d14b00 100644
--- a/tests/integration/project/files/pip-source/app1.py
+++ b/tests/integration/project/files/pip-source/app1.py
@@ -4,8 +4,8 @@ from hellolib import hello
def main():
- hello('App1')
+ hello("App1")
-if __name__ == '__main__':
+if __name__ == "__main__":
main()
diff --git a/tests/integration/pullbuildtrees.py b/tests/integration/pullbuildtrees.py
index a1b188e5a..b43a07121 100644
--- a/tests/integration/pullbuildtrees.py
+++ b/tests/integration/pullbuildtrees.py
@@ -16,22 +16,21 @@ from tests.testutils import create_artifact_share
pytestmark = pytest.mark.integration
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project"
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
# Remove artifact cache & set cli.config value of pull-buildtrees
# to false, which is the default user context. The cache has to be
# cleared as just forcefully removing the refpath leaves dangling objects.
def default_state(cli, tmpdir, share):
- shutil.rmtree(os.path.join(str(tmpdir), 'cas'))
- cli.configure({
- 'artifacts': {'url': share.repo, 'push': False},
- 'cachedir': str(tmpdir),
- 'cache': {'pull-buildtrees': False},
- })
+ shutil.rmtree(os.path.join(str(tmpdir), "cas"))
+ cli.configure(
+ {
+ "artifacts": {"url": share.repo, "push": False},
+ "cachedir": str(tmpdir),
+ "cache": {"pull-buildtrees": False},
+ }
+ )
# A test to capture the integration of the pullbuildtrees
@@ -39,50 +38,52 @@ def default_state(cli, tmpdir, share):
# directory of an element.
@pytest.mark.integration
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_pullbuildtrees(cli2, tmpdir, datafiles):
project = str(datafiles)
- element_name = 'autotools/amhello.bst'
+ element_name = "autotools/amhello.bst"
cwd = str(tmpdir)
# Create artifact shares for pull & push testing
- with create_artifact_share(os.path.join(str(tmpdir), 'share1')) as share1,\
- create_artifact_share(os.path.join(str(tmpdir), 'share2')) as share2,\
- create_artifact_share(os.path.join(str(tmpdir), 'share3')) as share3:
- cli2.configure({
- 'artifacts': {'url': share1.repo, 'push': True},
- 'cachedir': str(tmpdir),
- 'cache': {'cache-buildtrees': 'always'},
- })
+ with create_artifact_share(os.path.join(str(tmpdir), "share1")) as share1, create_artifact_share(
+ os.path.join(str(tmpdir), "share2")
+ ) as share2, create_artifact_share(os.path.join(str(tmpdir), "share3")) as share3:
+ cli2.configure(
+ {
+ "artifacts": {"url": share1.repo, "push": True},
+ "cachedir": str(tmpdir),
+ "cache": {"cache-buildtrees": "always"},
+ }
+ )
# Build autotools element, checked pushed, delete local
- result = cli2.run(project=project, args=['build', element_name])
+ result = cli2.run(project=project, args=["build", element_name])
assert result.exit_code == 0
- assert cli2.get_element_state(project, element_name) == 'cached'
- assert share1.get_artifact(cli2.get_artifact_name(project, 'test', element_name))
+ assert cli2.get_element_state(project, element_name) == "cached"
+ assert share1.get_artifact(cli2.get_artifact_name(project, "test", element_name))
default_state(cli2, tmpdir, share1)
# Pull artifact with default config, assert that pulling again
# doesn't create a pull job, then assert with buildtrees user
# config set creates a pull job.
- result = cli2.run(project=project, args=['artifact', 'pull', element_name])
+ result = cli2.run(project=project, args=["artifact", "pull", element_name])
assert element_name in result.get_pulled_elements()
- result = cli2.run(project=project, args=['artifact', 'pull', element_name])
+ result = cli2.run(project=project, args=["artifact", "pull", element_name])
assert element_name not in result.get_pulled_elements()
- cli2.configure({'cache': {'pull-buildtrees': True}})
- result = cli2.run(project=project, args=['artifact', 'pull', element_name])
+ cli2.configure({"cache": {"pull-buildtrees": True}})
+ result = cli2.run(project=project, args=["artifact", "pull", element_name])
assert element_name in result.get_pulled_elements()
default_state(cli2, tmpdir, share1)
# Pull artifact with default config, then assert that pulling
# with buildtrees cli flag set creates a pull job.
# Also assert that the buildtree is added to the local CAS.
- result = cli2.run(project=project, args=['artifact', 'pull', element_name])
+ result = cli2.run(project=project, args=["artifact", "pull", element_name])
assert element_name in result.get_pulled_elements()
- artifact_name = cli2.get_artifact_name(project, 'test', element_name)
+ artifact_name = cli2.get_artifact_name(project, "test", element_name)
with cli2.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir:
assert not buildtreedir
- result = cli2.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name])
+ result = cli2.run(project=project, args=["--pull-buildtrees", "artifact", "pull", element_name])
assert element_name in result.get_pulled_elements()
with cli2.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir:
assert os.path.isdir(buildtreedir)
@@ -91,22 +92,22 @@ def test_pullbuildtrees(cli2, tmpdir, datafiles):
# Pull artifact with pullbuildtrees set in user config, then assert
# that pulling with the same user config doesn't creates a pull job,
# or when buildtrees cli flag is set.
- cli2.configure({'cache': {'pull-buildtrees': True}})
- result = cli2.run(project=project, args=['artifact', 'pull', element_name])
+ cli2.configure({"cache": {"pull-buildtrees": True}})
+ result = cli2.run(project=project, args=["artifact", "pull", element_name])
assert element_name in result.get_pulled_elements()
- result = cli2.run(project=project, args=['artifact', 'pull', element_name])
+ result = cli2.run(project=project, args=["artifact", "pull", element_name])
assert element_name not in result.get_pulled_elements()
- result = cli2.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name])
+ result = cli2.run(project=project, args=["--pull-buildtrees", "artifact", "pull", element_name])
assert element_name not in result.get_pulled_elements()
default_state(cli2, tmpdir, share1)
# Pull artifact with default config and buildtrees cli flag set, then assert
# that pulling with pullbuildtrees set in user config doesn't create a pull
# job.
- result = cli2.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name])
+ result = cli2.run(project=project, args=["--pull-buildtrees", "artifact", "pull", element_name])
assert element_name in result.get_pulled_elements()
- cli2.configure({'cache': {'pull-buildtrees': True}})
- result = cli2.run(project=project, args=['artifact', 'pull', element_name])
+ cli2.configure({"cache": {"pull-buildtrees": True}})
+ result = cli2.run(project=project, args=["artifact", "pull", element_name])
assert element_name not in result.get_pulled_elements()
default_state(cli2, tmpdir, share1)
@@ -114,73 +115,64 @@ def test_pullbuildtrees(cli2, tmpdir, datafiles):
# can't be pushed to an artifact share, then assert that a complete build element
# can be. This will attempt a partial pull from share1 and then a partial push
# to share2
- result = cli2.run(project=project, args=['artifact', 'pull', element_name])
+ result = cli2.run(project=project, args=["artifact", "pull", element_name])
assert element_name in result.get_pulled_elements()
- cli2.configure({'artifacts': {'url': share2.repo, 'push': True}})
- result = cli2.run(project=project, args=['artifact', 'push', element_name])
+ cli2.configure({"artifacts": {"url": share2.repo, "push": True}})
+ result = cli2.run(project=project, args=["artifact", "push", element_name])
assert element_name not in result.get_pushed_elements()
- assert not share2.get_artifact(cli2.get_artifact_name(project, 'test', element_name))
+ assert not share2.get_artifact(cli2.get_artifact_name(project, "test", element_name))
# Assert that after pulling the missing buildtree the element artifact can be
# successfully pushed to the remote. This will attempt to pull the buildtree
# from share1 and then a 'complete' push to share2
- cli2.configure({'artifacts': {'url': share1.repo, 'push': False}})
- result = cli2.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name])
+ cli2.configure({"artifacts": {"url": share1.repo, "push": False}})
+ result = cli2.run(project=project, args=["--pull-buildtrees", "artifact", "pull", element_name])
assert element_name in result.get_pulled_elements()
- cli2.configure({'artifacts': {'url': share2.repo, 'push': True}})
- result = cli2.run(project=project, args=['artifact', 'push', element_name])
+ cli2.configure({"artifacts": {"url": share2.repo, "push": True}})
+ result = cli2.run(project=project, args=["artifact", "push", element_name])
assert element_name in result.get_pushed_elements()
- assert share2.get_artifact(cli2.get_artifact_name(project, 'test', element_name))
+ assert share2.get_artifact(cli2.get_artifact_name(project, "test", element_name))
default_state(cli2, tmpdir, share1)
# Assert that bst artifact push will automatically attempt to pull a missing buildtree
# if pull-buildtrees is set, however as share3 is the only defined remote and is empty,
# assert that no element artifact buildtrees are pulled (no available remote buildtree) and thus the
# artifact cannot be pushed.
- result = cli2.run(project=project, args=['artifact', 'pull', element_name])
+ result = cli2.run(project=project, args=["artifact", "pull", element_name])
assert element_name in result.get_pulled_elements()
- cli2.configure({'artifacts': {'url': share3.repo, 'push': True}})
- result = cli2.run(project=project, args=['--pull-buildtrees', 'artifact', 'push', element_name])
+ cli2.configure({"artifacts": {"url": share3.repo, "push": True}})
+ result = cli2.run(project=project, args=["--pull-buildtrees", "artifact", "push", element_name])
assert "Attempting to fetch missing artifact buildtrees" in result.stderr
assert element_name not in result.get_pulled_elements()
with cli2.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir:
assert not buildtreedir
assert element_name not in result.get_pushed_elements()
- assert not share3.get_artifact(cli2.get_artifact_name(project, 'test', element_name))
+ assert not share3.get_artifact(cli2.get_artifact_name(project, "test", element_name))
# Assert that if we add an extra remote that has the buildtree artfact cached, bst artifact push will
# automatically attempt to pull it and will be successful, leading to the full artifact being pushed
# to the empty share3. This gives the ability to attempt push currently partial artifacts to a remote,
# without exlipictly requiring a bst artifact pull.
- cli2.configure({'artifacts': [{'url': share1.repo, 'push': False}, {'url': share3.repo, 'push': True}]})
- result = cli2.run(project=project, args=['--pull-buildtrees', 'artifact', 'push', element_name])
+ cli2.configure({"artifacts": [{"url": share1.repo, "push": False}, {"url": share3.repo, "push": True}]})
+ result = cli2.run(project=project, args=["--pull-buildtrees", "artifact", "push", element_name])
assert "Attempting to fetch missing artifact buildtrees" in result.stderr
assert element_name in result.get_pulled_elements()
with cli2.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir:
assert os.path.isdir(buildtreedir)
assert element_name in result.get_pushed_elements()
- assert share3.get_artifact(cli2.get_artifact_name(project, 'test', element_name))
+ assert share3.get_artifact(cli2.get_artifact_name(project, "test", element_name))
# Ensure that only valid pull-buildtrees boolean options make it through the loading
# process.
-@pytest.mark.parametrize("value,success", [
- (True, True),
- (False, True),
- ("pony", False),
- ("1", False)
-])
+@pytest.mark.parametrize("value,success", [(True, True), (False, True), ("pony", False), ("1", False)])
@pytest.mark.datafiles(DATA_DIR)
def test_invalid_cache_pullbuildtrees(cli, datafiles, value, success):
project = str(datafiles)
- cli.configure({
- 'cache': {
- 'pull-buildtrees': value,
- }
- })
+ cli.configure({"cache": {"pull-buildtrees": value,}})
- res = cli.run(project=project, args=['workspace', 'list'])
+ res = cli.run(project=project, args=["workspace", "list"])
if success:
res.assert_success()
else:
diff --git a/tests/integration/sandbox-bwrap.py b/tests/integration/sandbox-bwrap.py
index 1f410ea92..2908752de 100644
--- a/tests/integration/sandbox-bwrap.py
+++ b/tests/integration/sandbox-bwrap.py
@@ -13,60 +13,55 @@ from buildstream.testing._utils.site import HAVE_SANDBOX, HAVE_BWRAP_JSON_STATUS
pytestmark = pytest.mark.integration
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project"
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
# Bubblewrap sandbox doesn't remove the dirs it created during its execution,
# so BuildStream tries to remove them to do good. BuildStream should be extra
# careful when those folders already exist and should not touch them, though.
-@pytest.mark.skipif(HAVE_SANDBOX != 'bwrap', reason='Only available with bubblewrap')
+@pytest.mark.skipif(HAVE_SANDBOX != "bwrap", reason="Only available with bubblewrap")
@pytest.mark.datafiles(DATA_DIR)
def test_sandbox_bwrap_cleanup_build(cli, datafiles):
project = str(datafiles)
# This element depends on a base image with non-empty `/tmp` folder.
- element_name = 'sandbox-bwrap/test-cleanup.bst'
+ element_name = "sandbox-bwrap/test-cleanup.bst"
# Here, BuildStream should not attempt any rmdir etc.
- result = cli.run(project=project, args=['build', element_name])
+ result = cli.run(project=project, args=["build", element_name])
assert result.exit_code == 0
-@pytest.mark.skipif(HAVE_SANDBOX != 'bwrap', reason='Only available with bubblewrap')
-@pytest.mark.skipif(not HAVE_BWRAP_JSON_STATUS, reason='Only available with bubblewrap supporting --json-status-fd')
+@pytest.mark.skipif(HAVE_SANDBOX != "bwrap", reason="Only available with bubblewrap")
+@pytest.mark.skipif(not HAVE_BWRAP_JSON_STATUS, reason="Only available with bubblewrap supporting --json-status-fd")
@pytest.mark.datafiles(DATA_DIR)
def test_sandbox_bwrap_distinguish_setup_error(cli, datafiles):
project = str(datafiles)
- element_name = 'sandbox-bwrap/non-executable-shell.bst'
+ element_name = "sandbox-bwrap/non-executable-shell.bst"
- result = cli.run(project=project, args=['build', element_name])
+ result = cli.run(project=project, args=["build", element_name])
result.assert_task_error(error_domain=ErrorDomain.SANDBOX, error_reason="bwrap-sandbox-fail")
-@pytest.mark.skipif(HAVE_SANDBOX != 'bwrap', reason='Only available with bubblewrap')
+@pytest.mark.skipif(HAVE_SANDBOX != "bwrap", reason="Only available with bubblewrap")
@pytest.mark.datafiles(DATA_DIR)
def test_sandbox_bwrap_return_subprocess(cli, datafiles):
project = str(datafiles)
- element_name = 'sandbox-bwrap/command-exit-42.bst'
+ element_name = "sandbox-bwrap/command-exit-42.bst"
- cli.configure({
- "logging": {
- "message-format": "%{element}|%{message}",
- },
- })
+ cli.configure(
+ {"logging": {"message-format": "%{element}|%{message}",},}
+ )
- result = cli.run(project=project, args=['build', element_name])
+ result = cli.run(project=project, args=["build", element_name])
result.assert_task_error(error_domain=ErrorDomain.SANDBOX, error_reason="command-failed")
assert "sandbox-bwrap/command-exit-42.bst|Command failed with exitcode 42" in result.stderr
-@pytest.mark.skipif(HAVE_SANDBOX != 'bwrap', reason='Only available with bubblewrap')
+@pytest.mark.skipif(HAVE_SANDBOX != "bwrap", reason="Only available with bubblewrap")
@pytest.mark.datafiles(DATA_DIR)
def test_sandbox_bwrap_dev_shm(cli, datafiles):
project = str(datafiles)
- element_name = 'sandbox-bwrap/test-dev-shm.bst'
+ element_name = "sandbox-bwrap/test-dev-shm.bst"
- result = cli.run(project=project, args=['build', element_name])
+ result = cli.run(project=project, args=["build", element_name])
assert result.exit_code == 0
diff --git a/tests/integration/script.py b/tests/integration/script.py
index 0e88ae53c..b27f1507e 100644
--- a/tests/integration/script.py
+++ b/tests/integration/script.py
@@ -12,10 +12,7 @@ from buildstream.testing._utils.site import HAVE_SANDBOX
pytestmark = pytest.mark.integration
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project"
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
def create_script_element(name, path, config=None, variables=None):
@@ -26,218 +23,211 @@ def create_script_element(name, path, config=None, variables=None):
variables = {}
element = {
- 'kind': 'script',
- 'depends': [{
- 'filename': 'base.bst',
- 'type': 'build'
- }],
- 'config': config,
- 'variables': variables
+ "kind": "script",
+ "depends": [{"filename": "base.bst", "type": "build"}],
+ "config": config,
+ "variables": variables,
}
os.makedirs(os.path.dirname(os.path.join(path, name)), exist_ok=True)
_yaml.roundtrip_dump(element, os.path.join(path, name))
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_script(cli, datafiles):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout')
- element_path = os.path.join(project, 'elements')
- element_name = 'script/script-layout.bst'
+ checkout = os.path.join(cli.directory, "checkout")
+ element_path = os.path.join(project, "elements")
+ element_name = "script/script-layout.bst"
- create_script_element(element_name, element_path,
- config={
- 'commands': [
- "mkdir -p %{install-root}",
- "echo 'Hi' > %{install-root}/test"
- ],
- })
+ create_script_element(
+ element_name,
+ element_path,
+ config={"commands": ["mkdir -p %{install-root}", "echo 'Hi' > %{install-root}/test"],},
+ )
- res = cli.run(project=project, args=['build', element_name])
+ res = cli.run(project=project, args=["build", element_name])
assert res.exit_code == 0
- res = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout])
+ res = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
assert res.exit_code == 0
- with open(os.path.join(checkout, 'test')) as f:
+ with open(os.path.join(checkout, "test")) as f:
text = f.read()
assert text == "Hi\n"
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_script_root(cli, datafiles):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout')
- element_path = os.path.join(project, 'elements')
- element_name = 'script/script-layout.bst'
+ checkout = os.path.join(cli.directory, "checkout")
+ element_path = os.path.join(project, "elements")
+ element_name = "script/script-layout.bst"
- create_script_element(element_name, element_path,
- config={
- # Root-read only is False by default, we
- # want to check the default here
- # 'root-read-only': False,
- 'commands': [
- "mkdir -p %{install-root}",
- "echo 'I can write to root' > /test",
- "cp /test %{install-root}"
- ],
- })
+ create_script_element(
+ element_name,
+ element_path,
+ config={
+ # Root-read only is False by default, we
+ # want to check the default here
+ # 'root-read-only': False,
+ "commands": ["mkdir -p %{install-root}", "echo 'I can write to root' > /test", "cp /test %{install-root}"],
+ },
+ )
- res = cli.run(project=project, args=['build', element_name])
+ res = cli.run(project=project, args=["build", element_name])
assert res.exit_code == 0
- res = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout])
+ res = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
assert res.exit_code == 0
- with open(os.path.join(checkout, 'test')) as f:
+ with open(os.path.join(checkout, "test")) as f:
text = f.read()
assert text == "I can write to root\n"
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
+@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox")
def test_script_no_root(cli, datafiles):
project = str(datafiles)
- element_path = os.path.join(project, 'elements')
- element_name = 'script/script-layout.bst'
-
- create_script_element(element_name, element_path,
- config={
- 'root-read-only': True,
- 'commands': [
- "mkdir -p %{install-root}",
- "echo 'I can not write to root' > /test",
- "cp /test %{install-root}"
- ],
- })
-
- res = cli.run(project=project, args=['build', element_name])
+ element_path = os.path.join(project, "elements")
+ element_name = "script/script-layout.bst"
+
+ create_script_element(
+ element_name,
+ element_path,
+ config={
+ "root-read-only": True,
+ "commands": [
+ "mkdir -p %{install-root}",
+ "echo 'I can not write to root' > /test",
+ "cp /test %{install-root}",
+ ],
+ },
+ )
+
+ res = cli.run(project=project, args=["build", element_name])
assert res.exit_code != 0
assert "/test: Read-only file system" in res.stderr
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
+@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox")
def test_script_cwd(cli, datafiles):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout')
- element_path = os.path.join(project, 'elements')
- element_name = 'script/script-layout.bst'
+ checkout = os.path.join(cli.directory, "checkout")
+ element_path = os.path.join(project, "elements")
+ element_name = "script/script-layout.bst"
- create_script_element(element_name, element_path,
- config={
- 'commands': [
- "echo 'test' > test",
- "cp /buildstream/test %{install-root}"
- ],
- },
- variables={
- 'cwd': '/buildstream'
- })
+ create_script_element(
+ element_name,
+ element_path,
+ config={"commands": ["echo 'test' > test", "cp /buildstream/test %{install-root}"],},
+ variables={"cwd": "/buildstream"},
+ )
- res = cli.run(project=project, args=['build', element_name])
+ res = cli.run(project=project, args=["build", element_name])
assert res.exit_code == 0
- res = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout])
+ res = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
assert res.exit_code == 0
- with open(os.path.join(checkout, 'test')) as f:
+ with open(os.path.join(checkout, "test")) as f:
text = f.read()
assert text == "test\n"
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_script_layout(cli, datafiles):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout')
- element_name = 'script/script-layout.bst'
+ checkout = os.path.join(cli.directory, "checkout")
+ element_name = "script/script-layout.bst"
- res = cli.run(project=project, args=['build', element_name])
+ res = cli.run(project=project, args=["build", element_name])
assert res.exit_code == 0
- cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout])
+ cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
assert res.exit_code == 0
- with open(os.path.join(checkout, 'test')) as f:
+ with open(os.path.join(checkout, "test")) as f:
text = f.read()
assert text == "Hi\n"
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_regression_cache_corruption(cli, datafiles):
project = str(datafiles)
- checkout_original = os.path.join(cli.directory, 'checkout-original')
- checkout_after = os.path.join(cli.directory, 'checkout-after')
- element_name = 'script/corruption.bst'
- canary_element_name = 'script/corruption-image.bst'
+ checkout_original = os.path.join(cli.directory, "checkout-original")
+ checkout_after = os.path.join(cli.directory, "checkout-after")
+ element_name = "script/corruption.bst"
+ canary_element_name = "script/corruption-image.bst"
- res = cli.run(project=project, args=['build', canary_element_name])
+ res = cli.run(project=project, args=["build", canary_element_name])
assert res.exit_code == 0
- res = cli.run(project=project, args=['artifact', 'checkout', canary_element_name,
- '--directory', checkout_original])
+ res = cli.run(
+ project=project, args=["artifact", "checkout", canary_element_name, "--directory", checkout_original]
+ )
assert res.exit_code == 0
- with open(os.path.join(checkout_original, 'canary')) as f:
- assert f.read() == 'alive\n'
+ with open(os.path.join(checkout_original, "canary")) as f:
+ assert f.read() == "alive\n"
- res = cli.run(project=project, args=['build', element_name])
+ res = cli.run(project=project, args=["build", element_name])
assert res.exit_code == 0
- res = cli.run(project=project, args=['artifact', 'checkout', canary_element_name,
- '--directory', checkout_after])
+ res = cli.run(project=project, args=["artifact", "checkout", canary_element_name, "--directory", checkout_after])
assert res.exit_code == 0
- with open(os.path.join(checkout_after, 'canary')) as f:
- assert f.read() == 'alive\n'
+ with open(os.path.join(checkout_after, "canary")) as f:
+ assert f.read() == "alive\n"
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_regression_tmpdir(cli, datafiles):
project = str(datafiles)
- element_name = 'script/tmpdir.bst'
+ element_name = "script/tmpdir.bst"
- res = cli.run(project=project, args=['build', element_name])
+ res = cli.run(project=project, args=["build", element_name])
assert res.exit_code == 0
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_regression_cache_corruption_2(cli, datafiles):
project = str(datafiles)
- checkout_original = os.path.join(cli.directory, 'checkout-original')
- checkout_after = os.path.join(cli.directory, 'checkout-after')
- element_name = 'script/corruption-2.bst'
- canary_element_name = 'script/corruption-image.bst'
+ checkout_original = os.path.join(cli.directory, "checkout-original")
+ checkout_after = os.path.join(cli.directory, "checkout-after")
+ element_name = "script/corruption-2.bst"
+ canary_element_name = "script/corruption-image.bst"
- res = cli.run(project=project, args=['build', canary_element_name])
+ res = cli.run(project=project, args=["build", canary_element_name])
assert res.exit_code == 0
- res = cli.run(project=project, args=['artifact', 'checkout', canary_element_name,
- '--directory', checkout_original])
+ res = cli.run(
+ project=project, args=["artifact", "checkout", canary_element_name, "--directory", checkout_original]
+ )
assert res.exit_code == 0
- with open(os.path.join(checkout_original, 'canary')) as f:
- assert f.read() == 'alive\n'
+ with open(os.path.join(checkout_original, "canary")) as f:
+ assert f.read() == "alive\n"
- res = cli.run(project=project, args=['build', element_name])
+ res = cli.run(project=project, args=["build", element_name])
assert res.exit_code == 0
- res = cli.run(project=project, args=['artifact', 'checkout', canary_element_name,
- '--directory', checkout_after])
+ res = cli.run(project=project, args=["artifact", "checkout", canary_element_name, "--directory", checkout_after])
assert res.exit_code == 0
- with open(os.path.join(checkout_after, 'canary')) as f:
- assert f.read() == 'alive\n'
+ with open(os.path.join(checkout_after, "canary")) as f:
+ assert f.read() == "alive\n"
diff --git a/tests/integration/shell.py b/tests/integration/shell.py
index 49e9a5950..e331a5fe0 100644
--- a/tests/integration/shell.py
+++ b/tests/integration/shell.py
@@ -16,10 +16,7 @@ from tests.testutils import create_artifact_share
pytestmark = pytest.mark.integration
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project"
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
# execute_shell()
@@ -35,28 +32,26 @@ DATA_DIR = os.path.join(
# element (str): The element to build and run a shell with
# isolate (bool): Whether to pass --isolate to `bst shell`
#
-def execute_shell(cli, project, command, *, config=None, mount=None, element='base.bst', isolate=False):
+def execute_shell(cli, project, command, *, config=None, mount=None, element="base.bst", isolate=False):
# Ensure the element is built
- result = cli.run_project_config(
- project=project, project_config=config, args=['build', element])
+ result = cli.run_project_config(project=project, project_config=config, args=["build", element])
assert result.exit_code == 0
- args = ['shell']
+ args = ["shell"]
if isolate:
- args += ['--isolate']
+ args += ["--isolate"]
if mount is not None:
host_path, target_path = mount
- args += ['--mount', host_path, target_path]
- args += [element, '--', *command]
+ args += ["--mount", host_path, target_path]
+ args += [element, "--", *command]
- return cli.run_project_config(
- project=project, project_config=config, args=args)
+ return cli.run_project_config(project=project, project_config=config, args=args)
# Test running something through a shell, allowing it to find the
# executable
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_shell(cli, datafiles):
project = str(datafiles)
@@ -67,7 +62,7 @@ def test_shell(cli, datafiles):
# Test running an executable directly
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_executable(cli, datafiles):
project = str(datafiles)
@@ -79,19 +74,15 @@ def test_executable(cli, datafiles):
# Test shell environment variable explicit assignments
@pytest.mark.parametrize("animal", [("Horse"), ("Pony")])
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
# This test seems to fail or pass depending on if this file is run or the hole test suite
def test_env_assign(cli, datafiles, animal):
project = str(datafiles)
- expected = animal + '\n'
+ expected = animal + "\n"
- result = execute_shell(cli, project, ['/bin/sh', '-c', 'echo ${ANIMAL}'], config={
- 'shell': {
- 'environment': {
- 'ANIMAL': animal
- }
- }
- })
+ result = execute_shell(
+ cli, project, ["/bin/sh", "-c", "echo ${ANIMAL}"], config={"shell": {"environment": {"ANIMAL": animal}}}
+ )
assert result.exit_code == 0
assert result.output == expected
@@ -100,21 +91,20 @@ def test_env_assign(cli, datafiles, animal):
# Test shell environment variable explicit assignments with host env var expansion
@pytest.mark.parametrize("animal", [("Horse"), ("Pony")])
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
# This test seems to fail or pass depending on if this file is run or the hole test suite
def test_env_assign_expand_host_environ(cli, datafiles, animal):
project = str(datafiles)
- expected = 'The animal is: {}\n'.format(animal)
+ expected = "The animal is: {}\n".format(animal)
- os.environ['BEAST'] = animal
+ os.environ["BEAST"] = animal
- result = execute_shell(cli, project, ['/bin/sh', '-c', 'echo ${ANIMAL}'], config={
- 'shell': {
- 'environment': {
- 'ANIMAL': 'The animal is: ${BEAST}'
- }
- }
- })
+ result = execute_shell(
+ cli,
+ project,
+ ["/bin/sh", "-c", "echo ${ANIMAL}"],
+ config={"shell": {"environment": {"ANIMAL": "The animal is: ${BEAST}"}}},
+ )
assert result.exit_code == 0
assert result.output == expected
@@ -124,51 +114,42 @@ def test_env_assign_expand_host_environ(cli, datafiles, animal):
# when running an isolated shell
@pytest.mark.parametrize("animal", [("Horse"), ("Pony")])
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
# This test seems to faili or pass depending on if this file is run or the hole test suite
def test_env_assign_isolated(cli, datafiles, animal):
project = str(datafiles)
- result = execute_shell(cli, project, ['/bin/sh', '-c', 'echo ${ANIMAL}'], isolate=True, config={
- 'shell': {
- 'environment': {
- 'ANIMAL': animal
- }
- }
- })
+ result = execute_shell(
+ cli,
+ project,
+ ["/bin/sh", "-c", "echo ${ANIMAL}"],
+ isolate=True,
+ config={"shell": {"environment": {"ANIMAL": animal}}},
+ )
assert result.exit_code == 0
- assert result.output == '\n'
+ assert result.output == "\n"
# Test running an executable in a runtime with no shell (i.e., no
# /bin/sh)
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_no_shell(cli, datafiles):
project = str(datafiles)
- element_path = os.path.join(project, 'elements')
- element_name = 'shell/no-shell.bst'
+ element_path = os.path.join(project, "elements")
+ element_name = "shell/no-shell.bst"
# Create an element that removes /bin/sh from the base runtime
element = {
- 'kind': 'script',
- 'depends': [{
- 'filename': 'base.bst',
- 'type': 'build'
- }],
- 'variables': {
- 'install-root': '/'
- },
- 'config': {
- 'commands': [
- 'rm /bin/sh'
- ]
- }
+ "kind": "script",
+ "depends": [{"filename": "base.bst", "type": "build"}],
+ "variables": {"install-root": "/"},
+ "config": {"commands": ["rm /bin/sh"]},
}
os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True)
_yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
- result = execute_shell(cli, project, ['/bin/echo', 'Pegasissies!'], element=element_name)
+ result = execute_shell(cli, project, ["/bin/echo", "Pegasissies!"], element=element_name)
assert result.exit_code == 0
assert result.output == "Pegasissies!\n"
@@ -176,99 +157,82 @@ def test_no_shell(cli, datafiles):
# Test that bind mounts defined in project.conf work
@pytest.mark.parametrize("path", [("/etc/pony.conf"), ("/usr/share/pony/pony.txt")])
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
+@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox")
def test_host_files(cli, datafiles, path):
project = str(datafiles)
- ponyfile = os.path.join(project, 'files', 'shell-mount', 'pony.txt')
- result = execute_shell(cli, project, ['cat', path], config={
- 'shell': {
- 'host-files': [
- {
- 'host_path': ponyfile,
- 'path': path
- }
- ]
- }
- })
+ ponyfile = os.path.join(project, "files", "shell-mount", "pony.txt")
+ result = execute_shell(
+ cli, project, ["cat", path], config={"shell": {"host-files": [{"host_path": ponyfile, "path": path}]}}
+ )
assert result.exit_code == 0
- assert result.output == 'pony\n'
+ assert result.output == "pony\n"
# Test that bind mounts defined in project.conf work
@pytest.mark.parametrize("path", [("/etc"), ("/usr/share/pony")])
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
+@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox")
def test_host_files_expand_environ(cli, datafiles, path):
project = str(datafiles)
- hostpath = os.path.join(project, 'files', 'shell-mount')
- fullpath = os.path.join(path, 'pony.txt')
-
- os.environ['BASE_PONY'] = path
- os.environ['HOST_PONY_PATH'] = hostpath
-
- result = execute_shell(cli, project, ['cat', fullpath], config={
- 'shell': {
- 'host-files': [
- {
- 'host_path': '${HOST_PONY_PATH}/pony.txt',
- 'path': '${BASE_PONY}/pony.txt'
- }
- ]
- }
- })
+ hostpath = os.path.join(project, "files", "shell-mount")
+ fullpath = os.path.join(path, "pony.txt")
+
+ os.environ["BASE_PONY"] = path
+ os.environ["HOST_PONY_PATH"] = hostpath
+
+ result = execute_shell(
+ cli,
+ project,
+ ["cat", fullpath],
+ config={
+ "shell": {"host-files": [{"host_path": "${HOST_PONY_PATH}/pony.txt", "path": "${BASE_PONY}/pony.txt"}]}
+ },
+ )
assert result.exit_code == 0
- assert result.output == 'pony\n'
+ assert result.output == "pony\n"
# Test that bind mounts defined in project.conf dont mount in isolation
@pytest.mark.parametrize("path", [("/etc/pony.conf"), ("/usr/share/pony/pony.txt")])
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_isolated_no_mount(cli, datafiles, path):
project = str(datafiles)
- ponyfile = os.path.join(project, 'files', 'shell-mount', 'pony.txt')
- result = execute_shell(cli, project, ['cat', path], isolate=True, config={
- 'shell': {
- 'host-files': [
- {
- 'host_path': ponyfile,
- 'path': path
- }
- ]
- }
- })
+ ponyfile = os.path.join(project, "files", "shell-mount", "pony.txt")
+ result = execute_shell(
+ cli,
+ project,
+ ["cat", path],
+ isolate=True,
+ config={"shell": {"host-files": [{"host_path": ponyfile, "path": path}]}},
+ )
assert result.exit_code != 0
assert path in result.stderr
- assert 'No such file or directory' in result.stderr
+ assert "No such file or directory" in result.stderr
# Test that we warn about non-existing files on the host if the mount is not
# declared as optional, and that there is no warning if it is optional
@pytest.mark.parametrize("optional", [("mandatory"), ("optional")])
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_host_files_missing(cli, datafiles, optional):
project = str(datafiles)
- ponyfile = os.path.join(project, 'files', 'shell-mount', 'horsy.txt')
+ ponyfile = os.path.join(project, "files", "shell-mount", "horsy.txt")
- option = (optional == "optional")
+ option = optional == "optional"
# Assert that we did successfully run something in the shell anyway
- result = execute_shell(cli, project, ['echo', 'Hello'], config={
- 'shell': {
- 'host-files': [
- {
- 'host_path': ponyfile,
- 'path': '/etc/pony.conf',
- 'optional': option
- }
- ]
- }
- })
+ result = execute_shell(
+ cli,
+ project,
+ ["echo", "Hello"],
+ config={"shell": {"host-files": [{"host_path": ponyfile, "path": "/etc/pony.conf", "optional": option}]}},
+ )
assert result.exit_code == 0
- assert result.output == 'Hello\n'
+ assert result.output == "Hello\n"
if option:
# Assert that there was no warning about the mount
@@ -281,92 +245,100 @@ def test_host_files_missing(cli, datafiles, optional):
# Test that bind mounts defined in project.conf work
@pytest.mark.parametrize("path", [("/etc/pony.conf"), ("/usr/share/pony/pony.txt")])
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
+@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox")
def test_cli_mount(cli, datafiles, path):
project = str(datafiles)
- ponyfile = os.path.join(project, 'files', 'shell-mount', 'pony.txt')
+ ponyfile = os.path.join(project, "files", "shell-mount", "pony.txt")
- result = execute_shell(cli, project, ['cat', path], mount=(ponyfile, path))
+ result = execute_shell(cli, project, ["cat", path], mount=(ponyfile, path))
assert result.exit_code == 0
- assert result.output == 'pony\n'
+ assert result.output == "pony\n"
# Test that we can see the workspace files in a shell
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_workspace_visible(cli, datafiles):
project = str(datafiles)
- workspace = os.path.join(cli.directory, 'workspace')
- element_name = 'workspace/workspace-mount-fail.bst'
+ workspace = os.path.join(cli.directory, "workspace")
+ element_name = "workspace/workspace-mount-fail.bst"
# Open a workspace on our build failing element
#
- res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name])
+ res = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name])
assert res.exit_code == 0
# Ensure the dependencies of our build failing element are built
- result = cli.run(project=project, args=['build', 'base.bst'])
+ result = cli.run(project=project, args=["build", "base.bst"])
assert result.exit_code == 0
# Obtain a copy of the hello.c content from the workspace
#
- workspace_hello_path = os.path.join(cli.directory, 'workspace', 'hello.c')
+ workspace_hello_path = os.path.join(cli.directory, "workspace", "hello.c")
assert os.path.exists(workspace_hello_path)
- with open(workspace_hello_path, 'r') as f:
+ with open(workspace_hello_path, "r") as f:
workspace_hello = f.read()
# Cat the hello.c file from a bst shell command, and assert
# that we got the same content here
#
- result = cli.run(project=project, args=[
- 'source', 'fetch', element_name])
+ result = cli.run(project=project, args=["source", "fetch", element_name])
assert result.exit_code == 0
- result = cli.run(project=project, args=[
- 'shell', '--build', element_name, '--', 'cat', 'hello.c'
- ])
+ result = cli.run(project=project, args=["shell", "--build", element_name, "--", "cat", "hello.c"])
assert result.exit_code == 0
assert result.output == workspace_hello
# Test that '--sysroot' works
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
+@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox")
def test_sysroot(cli, tmpdir, datafiles):
project = str(datafiles)
base_element = "base/base-alpine.bst"
# test element only needs to be something lightweight for this test
test_element = "script/script.bst"
- checkout_dir = os.path.join(str(tmpdir), 'alpine-sysroot')
- test_file = 'hello'
+ checkout_dir = os.path.join(str(tmpdir), "alpine-sysroot")
+ test_file = "hello"
# Build and check out a sysroot
- res = cli.run(project=project, args=['build', base_element])
+ res = cli.run(project=project, args=["build", base_element])
res.assert_success()
- res = cli.run(project=project, args=['artifact', 'checkout', base_element, '--directory', checkout_dir])
+ res = cli.run(project=project, args=["artifact", "checkout", base_element, "--directory", checkout_dir])
res.assert_success()
# Mutate the sysroot
test_path = os.path.join(checkout_dir, test_file)
- with open(test_path, 'w') as f:
- f.write('hello\n')
+ with open(test_path, "w") as f:
+ f.write("hello\n")
# Shell into the sysroot and check the test file exists
- res = cli.run(project=project, args=[
- 'shell', '--build', '--sysroot', checkout_dir, test_element, '--',
- 'grep', '-q', 'hello', '/' + test_file
- ])
+ res = cli.run(
+ project=project,
+ args=[
+ "shell",
+ "--build",
+ "--sysroot",
+ checkout_dir,
+ test_element,
+ "--",
+ "grep",
+ "-q",
+ "hello",
+ "/" + test_file,
+ ],
+ )
res.assert_success()
# Test system integration commands can access devices in /dev
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_integration_devices(cli, datafiles):
project = str(datafiles)
- element_name = 'integration.bst'
+ element_name = "integration.bst"
result = execute_shell(cli, project, ["true"], element=element_name)
assert result.exit_code == 0
@@ -376,79 +348,81 @@ def test_integration_devices(cli, datafiles):
@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.parametrize("build_shell", [("build"), ("nobuild")])
@pytest.mark.parametrize("guess_element", [True, False], ids=["guess", "no-guess"])
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_integration_external_workspace(cli, tmpdir_factory, datafiles, build_shell, guess_element):
tmpdir = tmpdir_factory.mktemp("")
project = str(datafiles)
- element_name = 'autotools/amhello.bst'
- workspace_dir = os.path.join(str(tmpdir), 'workspace')
+ element_name = "autotools/amhello.bst"
+ workspace_dir = os.path.join(str(tmpdir), "workspace")
if guess_element:
# Mutate the project.conf to use a default shell command
- project_file = os.path.join(project, 'project.conf')
- config_text = "shell:\n"\
- " command: ['true']\n"
- with open(project_file, 'a') as f:
+ project_file = os.path.join(project, "project.conf")
+ config_text = "shell:\n" " command: ['true']\n"
+ with open(project_file, "a") as f:
f.write(config_text)
- result = cli.run(project=project, args=[
- 'workspace', 'open', '--directory', workspace_dir, element_name
- ])
+ result = cli.run(project=project, args=["workspace", "open", "--directory", workspace_dir, element_name])
result.assert_success()
- result = cli.run(project=project, args=['-C', workspace_dir, 'build', element_name])
+ result = cli.run(project=project, args=["-C", workspace_dir, "build", element_name])
result.assert_success()
- command = ['-C', workspace_dir, 'shell']
- if build_shell == 'build':
- command.append('--build')
+ command = ["-C", workspace_dir, "shell"]
+ if build_shell == "build":
+ command.append("--build")
if not guess_element:
- command.extend([element_name, '--', 'true'])
+ command.extend([element_name, "--", "true"])
result = cli.run(project=project, cwd=workspace_dir, args=command)
result.assert_success()
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_integration_partial_artifact(cli, datafiles, tmpdir, integration_cache):
project = str(datafiles)
- element_name = 'autotools/amhello.bst'
+ element_name = "autotools/amhello.bst"
# push to an artifact server so we can pull from it later.
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
- cli.configure({'artifacts': {
- 'url': share.repo,
- 'push': True
- }})
- result = cli.run(project=project, args=['build', element_name])
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
+ cli.configure({"artifacts": {"url": share.repo, "push": True}})
+ result = cli.run(project=project, args=["build", element_name])
result.assert_success()
# If the build is cached then it might not push to the artifact cache
- result = cli.run(project=project, args=['artifact', 'push', element_name])
+ result = cli.run(project=project, args=["artifact", "push", element_name])
result.assert_success()
- result = cli.run(project=project, args=['shell', element_name])
+ result = cli.run(project=project, args=["shell", element_name])
result.assert_success()
# do a checkout and get the digest of the hello binary.
- result = cli.run(project=project, args=[
- 'artifact', 'checkout', '--deps', 'none',
- '--directory', os.path.join(str(tmpdir), 'tmp'),
- 'autotools/amhello.bst'])
+ result = cli.run(
+ project=project,
+ args=[
+ "artifact",
+ "checkout",
+ "--deps",
+ "none",
+ "--directory",
+ os.path.join(str(tmpdir), "tmp"),
+ "autotools/amhello.bst",
+ ],
+ )
result.assert_success()
- digest = utils.sha256sum(os.path.join(str(tmpdir), 'tmp', 'usr', 'bin', 'hello'))
+ digest = utils.sha256sum(os.path.join(str(tmpdir), "tmp", "usr", "bin", "hello"))
# Remove the binary from the CAS
- cachedir = cli.config['cachedir']
- objpath = os.path.join(cachedir, 'cas', 'objects', digest[:2], digest[2:])
+ cachedir = cli.config["cachedir"]
+ objpath = os.path.join(cachedir, "cas", "objects", digest[:2], digest[2:])
os.unlink(objpath)
# check shell doesn't work
- result = cli.run(project=project, args=['shell', element_name, '--', 'hello'])
+ result = cli.run(project=project, args=["shell", element_name, "--", "hello"])
result.assert_main_error(ErrorDomain.APP, None)
# check the artifact gets completed with '--pull' specified
- result = cli.run(project=project, args=['shell', '--pull', element_name, '--', 'hello'])
+ result = cli.run(project=project, args=["shell", "--pull", element_name, "--", "hello"])
result.assert_success()
- assert 'autotools/amhello.bst' in result.get_pulled_elements()
+ assert "autotools/amhello.bst" in result.get_pulled_elements()
diff --git a/tests/integration/shellbuildtrees.py b/tests/integration/shellbuildtrees.py
index 146bc6062..d3191870a 100644
--- a/tests/integration/shellbuildtrees.py
+++ b/tests/integration/shellbuildtrees.py
@@ -16,321 +16,336 @@ from tests.testutils import create_artifact_share
pytestmark = pytest.mark.integration
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project"
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_buildtree_staged(cli_integration, datafiles):
# We can only test the non interacitve case
# The non interactive case defaults to not using buildtrees
# for `bst shell --build`
project = str(datafiles)
- element_name = 'build-shell/buildtree.bst'
+ element_name = "build-shell/buildtree.bst"
- res = cli_integration.run(project=project, args=['--cache-buildtrees', 'always', 'build', element_name])
+ res = cli_integration.run(project=project, args=["--cache-buildtrees", "always", "build", element_name])
res.assert_success()
- res = cli_integration.run(project=project, args=[
- 'shell', '--build', element_name, '--', 'cat', 'test'
- ])
+ res = cli_integration.run(project=project, args=["shell", "--build", element_name, "--", "cat", "test"])
res.assert_shell_error()
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_buildtree_staged_forced_true(cli_integration, datafiles):
# Test that if we ask for a build tree it is there.
project = str(datafiles)
- element_name = 'build-shell/buildtree.bst'
+ element_name = "build-shell/buildtree.bst"
- res = cli_integration.run(project=project, args=['--cache-buildtrees', 'always', 'build', element_name])
+ res = cli_integration.run(project=project, args=["--cache-buildtrees", "always", "build", element_name])
res.assert_success()
- res = cli_integration.run(project=project, args=[
- 'shell', '--build', '--use-buildtree', 'always', element_name, '--', 'cat', 'test'
- ])
+ res = cli_integration.run(
+ project=project, args=["shell", "--build", "--use-buildtree", "always", element_name, "--", "cat", "test"]
+ )
res.assert_success()
- assert 'Hi' in res.output
+ assert "Hi" in res.output
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_buildtree_staged_warn_empty_cached(cli_integration, tmpdir, datafiles):
# Test that if we stage a cached and empty buildtree, we warn the user.
project = str(datafiles)
- element_name = 'build-shell/buildtree.bst'
+ element_name = "build-shell/buildtree.bst"
# Switch to a temp artifact cache dir to ensure the artifact is rebuilt,
# without caching a buildtree which is the default bst behaviour
- cli_integration.configure({
- 'cachedir': str(tmpdir)
- })
+ cli_integration.configure({"cachedir": str(tmpdir)})
- res = cli_integration.run(project=project, args=['build', element_name])
+ res = cli_integration.run(project=project, args=["build", element_name])
res.assert_success()
- res = cli_integration.run(project=project, args=[
- 'shell', '--build', '--use-buildtree', 'always', element_name, '--', 'cat', 'test'
- ])
+ res = cli_integration.run(
+ project=project, args=["shell", "--build", "--use-buildtree", "always", element_name, "--", "cat", "test"]
+ )
res.assert_main_error(ErrorDomain.APP, None)
assert "Artifact was created without buildtree, unable to launch shell with it" in res.stderr
# Now attempt the same with the try option, this should not attempt to find a buildtree
# and just launch the shell, however the cat should still fail.
- res = cli_integration.run(project=project, args=[
- 'shell', '--build', '--use-buildtree', 'try', element_name, '--', 'cat', 'test'
- ])
+ res = cli_integration.run(
+ project=project, args=["shell", "--build", "--use-buildtree", "try", element_name, "--", "cat", "test"]
+ )
assert "Artifact created without buildtree, shell will be loaded without it" in res.stderr
- assert 'Hi' not in res.output
+ assert "Hi" not in res.output
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_buildtree_staged_if_available(cli_integration, datafiles):
# Test that a build tree can be correctly detected.
project = str(datafiles)
- element_name = 'build-shell/buildtree.bst'
+ element_name = "build-shell/buildtree.bst"
- res = cli_integration.run(project=project, args=['--cache-buildtrees', 'always', 'build', element_name])
+ res = cli_integration.run(project=project, args=["--cache-buildtrees", "always", "build", element_name])
res.assert_success()
- res = cli_integration.run(project=project, args=[
- 'shell', '--build', '--use-buildtree', 'try', element_name, '--', 'cat', 'test'
- ])
+ res = cli_integration.run(
+ project=project, args=["shell", "--build", "--use-buildtree", "try", element_name, "--", "cat", "test"]
+ )
res.assert_success()
- assert 'Hi' in res.output
+ assert "Hi" in res.output
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_buildtree_staged_forced_false(cli_integration, datafiles):
# Test that if we ask not to have a build tree it is not there
project = str(datafiles)
- element_name = 'build-shell/buildtree.bst'
+ element_name = "build-shell/buildtree.bst"
- res = cli_integration.run(project=project, args=['--cache-buildtrees', 'always', 'build', element_name])
+ res = cli_integration.run(project=project, args=["--cache-buildtrees", "always", "build", element_name])
res.assert_success()
- res = cli_integration.run(project=project, args=[
- 'shell', '--build', '--use-buildtree', 'never', element_name, '--', 'cat', 'test'
- ])
+ res = cli_integration.run(
+ project=project, args=["shell", "--build", "--use-buildtree", "never", element_name, "--", "cat", "test"]
+ )
res.assert_shell_error()
- assert 'Hi' not in res.output
+ assert "Hi" not in res.output
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_buildtree_from_failure(cli_integration, datafiles):
# Test that we can use a build tree after a failure
project = str(datafiles)
- element_name = 'build-shell/buildtree-fail.bst'
+ element_name = "build-shell/buildtree-fail.bst"
- res = cli_integration.run(project=project, args=['build', element_name])
+ res = cli_integration.run(project=project, args=["build", element_name])
res.assert_main_error(ErrorDomain.STREAM, None)
# Assert that file has expected contents
- res = cli_integration.run(project=project, args=[
- 'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test'
- ])
+ res = cli_integration.run(
+ project=project, args=["shell", "--build", element_name, "--use-buildtree", "always", "--", "cat", "test"]
+ )
res.assert_success()
assert "WARNING: using a buildtree from a failed build" in res.stderr
- assert 'Hi' in res.output
+ assert "Hi" in res.output
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_buildtree_from_failure_option_never(cli_integration, tmpdir, datafiles):
project = str(datafiles)
- element_name = 'build-shell/buildtree-fail.bst'
+ element_name = "build-shell/buildtree-fail.bst"
# Switch to a temp artifact cache dir to ensure the artifact is rebuilt,
# without caching a buildtree explicitly
- cli_integration.configure({
- 'cachedir': str(tmpdir)
- })
+ cli_integration.configure({"cachedir": str(tmpdir)})
- res = cli_integration.run(project=project, args=['--cache-buildtrees', 'never', 'build', element_name])
+ res = cli_integration.run(project=project, args=["--cache-buildtrees", "never", "build", element_name])
res.assert_main_error(ErrorDomain.STREAM, None)
- res = cli_integration.run(project=project, args=[
- 'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test'
- ])
+ res = cli_integration.run(
+ project=project, args=["shell", "--build", element_name, "--use-buildtree", "always", "--", "cat", "test"]
+ )
res.assert_main_error(ErrorDomain.APP, None)
assert "Artifact was created without buildtree, unable to launch shell with it" in res.stderr
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_buildtree_from_failure_option_always(cli_integration, tmpdir, datafiles):
project = str(datafiles)
- element_name = 'build-shell/buildtree-fail.bst'
+ element_name = "build-shell/buildtree-fail.bst"
# build with --cache-buildtrees set to 'always', behaviour should match
# default behaviour (which is always) as the buildtree will explicitly have been
# cached with content.
- cli_integration.configure({
- 'cachedir': str(tmpdir)
- })
+ cli_integration.configure({"cachedir": str(tmpdir)})
- res = cli_integration.run(project=project, args=['--cache-buildtrees', 'always', 'build', element_name])
+ res = cli_integration.run(project=project, args=["--cache-buildtrees", "always", "build", element_name])
res.assert_main_error(ErrorDomain.STREAM, None)
- res = cli_integration.run(project=project, args=[
- 'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test'
- ])
+ res = cli_integration.run(
+ project=project, args=["shell", "--build", element_name, "--use-buildtree", "always", "--", "cat", "test"]
+ )
res.assert_success()
assert "WARNING: using a buildtree from a failed build" in res.stderr
- assert 'Hi' in res.output
+ assert "Hi" in res.output
# Check that build shells work when pulled from a remote cache
# This is to roughly simulate remote execution
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_buildtree_pulled(cli, tmpdir, datafiles):
project = str(datafiles)
- element_name = 'build-shell/buildtree.bst'
+ element_name = "build-shell/buildtree.bst"
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
# Build the element to push it to cache
- cli.configure({
- 'artifacts': {'url': share.repo, 'push': True}
- })
- result = cli.run(project=project, args=['--cache-buildtrees', 'always', 'build', element_name])
+ cli.configure({"artifacts": {"url": share.repo, "push": True}})
+ result = cli.run(project=project, args=["--cache-buildtrees", "always", "build", element_name])
result.assert_success()
- assert cli.get_element_state(project, element_name) == 'cached'
+ assert cli.get_element_state(project, element_name) == "cached"
# Discard the cache
- shutil.rmtree(str(os.path.join(str(tmpdir), 'cache', 'cas')))
- shutil.rmtree(str(os.path.join(str(tmpdir), 'cache', 'artifacts')))
- assert cli.get_element_state(project, element_name) != 'cached'
+ shutil.rmtree(str(os.path.join(str(tmpdir), "cache", "cas")))
+ shutil.rmtree(str(os.path.join(str(tmpdir), "cache", "artifacts")))
+ assert cli.get_element_state(project, element_name) != "cached"
# Pull from cache, ensuring cli options is set to pull the buildtree
- result = cli.run(project=project,
- args=['--pull-buildtrees', 'artifact', 'pull', '--deps', 'all', element_name])
+ result = cli.run(
+ project=project, args=["--pull-buildtrees", "artifact", "pull", "--deps", "all", element_name]
+ )
result.assert_success()
# Check it's using the cached build tree
- res = cli.run(project=project, args=[
- 'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test'
- ])
+ res = cli.run(
+ project=project, args=["shell", "--build", element_name, "--use-buildtree", "always", "--", "cat", "test"]
+ )
res.assert_success()
# This test checks for correct behaviour if a buildtree is not present in the local cache.
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_buildtree_options(cli, tmpdir, datafiles):
project = str(datafiles)
- element_name = 'build-shell/buildtree.bst'
+ element_name = "build-shell/buildtree.bst"
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
# Build the element to push it to cache
- cli.configure({
- 'artifacts': {'url': share.repo, 'push': True}
- })
- result = cli.run(project=project, args=['--cache-buildtrees', 'always', 'build', element_name])
+ cli.configure({"artifacts": {"url": share.repo, "push": True}})
+ result = cli.run(project=project, args=["--cache-buildtrees", "always", "build", element_name])
result.assert_success()
- assert cli.get_element_state(project, element_name) == 'cached'
- assert share.get_artifact(cli.get_artifact_name(project, 'test', element_name))
+ assert cli.get_element_state(project, element_name) == "cached"
+ assert share.get_artifact(cli.get_artifact_name(project, "test", element_name))
# Discard the cache
- shutil.rmtree(str(os.path.join(str(tmpdir), 'cache', 'cas')))
- shutil.rmtree(str(os.path.join(str(tmpdir), 'cache', 'artifacts')))
- assert cli.get_element_state(project, element_name) != 'cached'
+ shutil.rmtree(str(os.path.join(str(tmpdir), "cache", "cas")))
+ shutil.rmtree(str(os.path.join(str(tmpdir), "cache", "artifacts")))
+ assert cli.get_element_state(project, element_name) != "cached"
# Pull from cache, but do not include buildtrees.
- result = cli.run(project=project, args=['artifact', 'pull', '--deps', 'all', element_name])
+ result = cli.run(project=project, args=["artifact", "pull", "--deps", "all", element_name])
result.assert_success()
# Check it's not using the cached build tree
- res = cli.run(project=project, args=[
- 'shell', '--build', element_name, '--use-buildtree', 'never', '--', 'cat', 'test'
- ])
+ res = cli.run(
+ project=project, args=["shell", "--build", element_name, "--use-buildtree", "never", "--", "cat", "test"]
+ )
res.assert_shell_error()
- assert 'Hi' not in res.output
+ assert "Hi" not in res.output
# Check it's not using the cached build tree, default is to ask, and fall back to not
# for non interactive behavior
- res = cli.run(project=project, args=[
- 'shell', '--build', element_name, '--', 'cat', 'test'
- ])
+ res = cli.run(project=project, args=["shell", "--build", element_name, "--", "cat", "test"])
res.assert_shell_error()
- assert 'Hi' not in res.output
+ assert "Hi" not in res.output
# Check correctly handling the lack of buildtree, with 'try' not attempting to
# pull the buildtree as the user context is by default set to not pull them
- res = cli.run(project=project, args=[
- 'shell', '--build', element_name, '--use-buildtree', 'try', '--', 'cat', 'test'
- ])
- assert 'Hi' not in res.output
- assert 'Attempting to fetch missing artifact buildtrees' not in res.stderr
+ res = cli.run(
+ project=project, args=["shell", "--build", element_name, "--use-buildtree", "try", "--", "cat", "test"]
+ )
+ assert "Hi" not in res.output
+ assert "Attempting to fetch missing artifact buildtrees" not in res.stderr
# Check correctly handling the lack of buildtree, with 'try' attempting and succeeding
# to pull the buildtree as the user context allow the pulling of buildtrees and it is
# available in the remote
- res = cli.run(project=project, args=[
- '--pull-buildtrees', 'shell', '--build', element_name, '--use-buildtree', 'try', '--', 'cat', 'test'
- ])
- assert 'Attempting to fetch missing artifact buildtree' in res.stderr
- assert 'Hi' in res.output
- shutil.rmtree(os.path.join(os.path.join(str(tmpdir), 'cache', 'cas')))
- shutil.rmtree(os.path.join(os.path.join(str(tmpdir), 'cache', 'artifacts')))
- assert cli.get_element_state(project, element_name) != 'cached'
+ res = cli.run(
+ project=project,
+ args=[
+ "--pull-buildtrees",
+ "shell",
+ "--build",
+ element_name,
+ "--use-buildtree",
+ "try",
+ "--",
+ "cat",
+ "test",
+ ],
+ )
+ assert "Attempting to fetch missing artifact buildtree" in res.stderr
+ assert "Hi" in res.output
+ shutil.rmtree(os.path.join(os.path.join(str(tmpdir), "cache", "cas")))
+ shutil.rmtree(os.path.join(os.path.join(str(tmpdir), "cache", "artifacts")))
+ assert cli.get_element_state(project, element_name) != "cached"
# Check it's not loading the shell at all with always set for the buildtree, when the
# user context does not allow for buildtree pulling
- result = cli.run(project=project, args=['artifact', 'pull', '--deps', 'all', element_name])
+ result = cli.run(project=project, args=["artifact", "pull", "--deps", "all", element_name])
result.assert_success()
- res = cli.run(project=project, args=[
- 'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test'
- ])
+ res = cli.run(
+ project=project, args=["shell", "--build", element_name, "--use-buildtree", "always", "--", "cat", "test"]
+ )
res.assert_main_error(ErrorDomain.APP, None)
- assert 'Buildtree is not cached locally or in available remotes' in res.stderr
- assert 'Hi' not in res.output
- assert 'Attempting to fetch missing artifact buildtree' not in res.stderr
+ assert "Buildtree is not cached locally or in available remotes" in res.stderr
+ assert "Hi" not in res.output
+ assert "Attempting to fetch missing artifact buildtree" not in res.stderr
# Check that when user context is set to pull buildtrees and a remote has the buildtree,
# 'always' will attempt and succeed at pulling the missing buildtree.
- res = cli.run(project=project, args=[
- '--pull-buildtrees', 'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test'
- ])
- assert 'Hi' in res.output
+ res = cli.run(
+ project=project,
+ args=[
+ "--pull-buildtrees",
+ "shell",
+ "--build",
+ element_name,
+ "--use-buildtree",
+ "always",
+ "--",
+ "cat",
+ "test",
+ ],
+ )
+ assert "Hi" in res.output
assert "buildtree is not cached locally, will attempt to pull from available remotes" in res.stderr
- assert 'Attempting to fetch missing artifact buildtree' in res.stderr
+ assert "Attempting to fetch missing artifact buildtree" in res.stderr
# Tests running pull and pull-buildtree options at the same time.
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_pull_buildtree_pulled(cli, tmpdir, datafiles):
project = str(datafiles)
- element_name = 'build-shell/buildtree.bst'
+ element_name = "build-shell/buildtree.bst"
- with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
+ with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
# Build the element to push it to cache
- cli.configure({
- 'artifacts': {'url': share.repo, 'push': True}
- })
- result = cli.run(project=project, args=['--cache-buildtrees', 'always', 'build', element_name])
+ cli.configure({"artifacts": {"url": share.repo, "push": True}})
+ result = cli.run(project=project, args=["--cache-buildtrees", "always", "build", element_name])
result.assert_success()
- assert cli.get_element_state(project, element_name) == 'cached'
+ assert cli.get_element_state(project, element_name) == "cached"
# Discard the cache
- shutil.rmtree(str(os.path.join(str(tmpdir), 'cache', 'cas')))
- shutil.rmtree(str(os.path.join(str(tmpdir), 'cache', 'artifacts')))
- assert cli.get_element_state(project, element_name) != 'cached'
+ shutil.rmtree(str(os.path.join(str(tmpdir), "cache", "cas")))
+ shutil.rmtree(str(os.path.join(str(tmpdir), "cache", "artifacts")))
+ assert cli.get_element_state(project, element_name) != "cached"
# Check it's using the cached build tree
- res = cli.run(project=project, args=[
- '--pull-buildtrees', 'shell', '--build', element_name, '--pull',
- '--use-buildtree', 'always', '--', 'cat', 'test'
- ])
+ res = cli.run(
+ project=project,
+ args=[
+ "--pull-buildtrees",
+ "shell",
+ "--build",
+ element_name,
+ "--pull",
+ "--use-buildtree",
+ "always",
+ "--",
+ "cat",
+ "test",
+ ],
+ )
res.assert_success()
diff --git a/tests/integration/sockets.py b/tests/integration/sockets.py
index 763238baf..3fb656e95 100644
--- a/tests/integration/sockets.py
+++ b/tests/integration/sockets.py
@@ -10,27 +10,24 @@ from buildstream.testing._utils.site import HAVE_SANDBOX
pytestmark = pytest.mark.integration
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project"
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_builddir_socket_ignored(cli, datafiles):
project = str(datafiles)
- element_name = 'sockets/make-builddir-socket.bst'
+ element_name = "sockets/make-builddir-socket.bst"
- result = cli.run(project=project, args=['build', element_name])
+ result = cli.run(project=project, args=["build", element_name])
assert result.exit_code == 0
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_install_root_socket_ignored(cli, datafiles):
project = str(datafiles)
- element_name = 'sockets/make-install-root-socket.bst'
+ element_name = "sockets/make-install-root-socket.bst"
- result = cli.run(project=project, args=['build', element_name])
+ result = cli.run(project=project, args=["build", element_name])
assert result.exit_code == 0
diff --git a/tests/integration/source-determinism.py b/tests/integration/source-determinism.py
index 70c4b79de..a349f9eeb 100644
--- a/tests/integration/source-determinism.py
+++ b/tests/integration/source-determinism.py
@@ -9,79 +9,62 @@ from buildstream.testing import cli_integration as cli # pylint: disable=unused
from buildstream.testing._utils.site import HAVE_SANDBOX
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project"
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
-def create_test_file(*path, mode=0o644, content='content\n'):
+def create_test_file(*path, mode=0o644, content="content\n"):
path = os.path.join(*path)
os.makedirs(os.path.dirname(path), exist_ok=True)
- with open(path, 'w') as f:
+ with open(path, "w") as f:
f.write(content)
os.fchmod(f.fileno(), mode)
def create_test_directory(*path, mode=0o644):
- create_test_file(*path, '.keep', content='')
+ create_test_file(*path, ".keep", content="")
path = os.path.join(*path)
os.chmod(path, mode)
@pytest.mark.integration
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_deterministic_source_local(cli, tmpdir, datafiles):
"""Only user rights should be considered for local source.
"""
project = str(datafiles)
- element_name = 'test.bst'
- element_path = os.path.join(project, 'elements', element_name)
- sourcedir = os.path.join(project, 'source')
+ element_name = "test.bst"
+ element_path = os.path.join(project, "elements", element_name)
+ sourcedir = os.path.join(project, "source")
element = {
- 'kind': 'manual',
- 'depends': [
- {
- 'filename': 'base.bst',
- 'type': 'build'
- }
- ],
- 'sources': [
- {
- 'kind': 'local',
- 'path': 'source'
- }
- ],
- 'config': {
- 'install-commands': [
- 'ls -l >"%{install-root}/ls-l"'
- ]
- }
+ "kind": "manual",
+ "depends": [{"filename": "base.bst", "type": "build"}],
+ "sources": [{"kind": "local", "path": "source"}],
+ "config": {"install-commands": ['ls -l >"%{install-root}/ls-l"']},
}
_yaml.roundtrip_dump(element, element_path)
def get_value_for_mask(mask):
- checkoutdir = os.path.join(str(tmpdir), 'checkout-{}'.format(mask))
-
- create_test_file(sourcedir, 'a.txt', mode=0o644 & mask)
- create_test_file(sourcedir, 'b.txt', mode=0o755 & mask)
- create_test_file(sourcedir, 'c.txt', mode=0o4755 & mask)
- create_test_file(sourcedir, 'd.txt', mode=0o2755 & mask)
- create_test_file(sourcedir, 'e.txt', mode=0o1755 & mask)
- create_test_directory(sourcedir, 'dir-a', mode=0o0755 & mask)
- create_test_directory(sourcedir, 'dir-b', mode=0o4755 & mask)
- create_test_directory(sourcedir, 'dir-c', mode=0o2755 & mask)
- create_test_directory(sourcedir, 'dir-d', mode=0o1755 & mask)
+ checkoutdir = os.path.join(str(tmpdir), "checkout-{}".format(mask))
+
+ create_test_file(sourcedir, "a.txt", mode=0o644 & mask)
+ create_test_file(sourcedir, "b.txt", mode=0o755 & mask)
+ create_test_file(sourcedir, "c.txt", mode=0o4755 & mask)
+ create_test_file(sourcedir, "d.txt", mode=0o2755 & mask)
+ create_test_file(sourcedir, "e.txt", mode=0o1755 & mask)
+ create_test_directory(sourcedir, "dir-a", mode=0o0755 & mask)
+ create_test_directory(sourcedir, "dir-b", mode=0o4755 & mask)
+ create_test_directory(sourcedir, "dir-c", mode=0o2755 & mask)
+ create_test_directory(sourcedir, "dir-d", mode=0o1755 & mask)
try:
- result = cli.run(project=project, args=['build', element_name])
+ result = cli.run(project=project, args=["build", element_name])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkoutdir])
result.assert_success()
- with open(os.path.join(checkoutdir, 'ls-l'), 'r') as f:
+ with open(os.path.join(checkoutdir, "ls-l"), "r") as f:
return f.read()
finally:
cli.remove_artifact_from_cache(project, element_name)
diff --git a/tests/integration/stack.py b/tests/integration/stack.py
index 9d6b38345..bad807fd6 100644
--- a/tests/integration/stack.py
+++ b/tests/integration/stack.py
@@ -11,29 +11,26 @@ from buildstream.testing._utils.site import HAVE_SANDBOX
pytestmark = pytest.mark.integration
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project"
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_stack(cli, datafiles):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout')
- element_name = 'stack/stack.bst'
+ checkout = os.path.join(cli.directory, "checkout")
+ element_name = "stack/stack.bst"
- res = cli.run(project=project, args=['build', element_name])
+ res = cli.run(project=project, args=["build", element_name])
assert res.exit_code == 0
- cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout])
+ cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
assert res.exit_code == 0
- with open(os.path.join(checkout, 'hi')) as f:
+ with open(os.path.join(checkout, "hi")) as f:
hi = f.read()
- with open(os.path.join(checkout, 'another-hi')) as f:
+ with open(os.path.join(checkout, "another-hi")) as f:
another_hi = f.read()
assert hi == "Hi\n"
diff --git a/tests/integration/symlinks.py b/tests/integration/symlinks.py
index 85bbc53fd..f1d044f07 100644
--- a/tests/integration/symlinks.py
+++ b/tests/integration/symlinks.py
@@ -11,80 +11,77 @@ from buildstream.testing._utils.site import HAVE_SANDBOX
pytestmark = pytest.mark.integration
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project"
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_absolute_symlinks(cli, datafiles):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout')
- element_name = 'symlinks/dangling-symlink.bst'
+ checkout = os.path.join(cli.directory, "checkout")
+ element_name = "symlinks/dangling-symlink.bst"
- result = cli.run(project=project, args=['build', element_name])
+ result = cli.run(project=project, args=["build", element_name])
assert result.exit_code == 0
- result = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout])
+ result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
assert result.exit_code == 0
- symlink = os.path.join(checkout, 'opt', 'orgname')
+ symlink = os.path.join(checkout, "opt", "orgname")
assert os.path.islink(symlink)
# The symlink is created to point to /usr/orgs/orgname and BuildStream
# should not mangle symlinks.
- assert os.readlink(symlink) == '/usr/orgs/orgname'
+ assert os.readlink(symlink) == "/usr/orgs/orgname"
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_disallow_overlaps_inside_symlink_with_dangling_target(cli, datafiles):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout')
- element_name = 'symlinks/dangling-symlink-overlap.bst'
+ checkout = os.path.join(cli.directory, "checkout")
+ element_name = "symlinks/dangling-symlink-overlap.bst"
- result = cli.run(project=project, args=['build', element_name])
+ result = cli.run(project=project, args=["build", element_name])
assert result.exit_code == 0
- result = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout])
+ result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
assert result.exit_code == -1
- assert 'Destination is a symlink, not a directory: /opt/orgname' in result.stderr
+ assert "Destination is a symlink, not a directory: /opt/orgname" in result.stderr
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_detect_symlink_overlaps_pointing_outside_sandbox(cli, datafiles):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout')
- element_name = 'symlinks/symlink-to-outside-sandbox-overlap.bst'
+ checkout = os.path.join(cli.directory, "checkout")
+ element_name = "symlinks/symlink-to-outside-sandbox-overlap.bst"
# Building the two elements should succeed...
- result = cli.run(project=project, args=['build', element_name])
+ result = cli.run(project=project, args=["build", element_name])
assert result.exit_code == 0
# ...but when we compose them together, the overlaps create paths that
# point outside the sandbox which BuildStream needs to detect before it
# tries to actually write there.
- result = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout])
+ result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
assert result.exit_code == -1
- assert 'Destination is a symlink, not a directory: /opt/escape-hatch' in result.stderr
+ assert "Destination is a symlink, not a directory: /opt/escape-hatch" in result.stderr
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_symlink_in_sandbox_path(cli, datafiles):
project = str(datafiles)
- element_name = 'symlinks/link-on-path-use.bst'
- base_element_name = 'symlinks/link-on-path.bst'
+ element_name = "symlinks/link-on-path-use.bst"
+ base_element_name = "symlinks/link-on-path.bst"
# This test is inspired by how freedesktop-SDK has /bin -> /usr/bin
# Create a element that has sh in altbin and a link from bin to altbin
- result1 = cli.run(project=project, args=['build', base_element_name])
+ result1 = cli.run(project=project, args=["build", base_element_name])
result1.assert_success()
# Build a element that uses the element that has sh in altbin.
- result2 = cli.run(project=project, args=['build', element_name])
+ result2 = cli.run(project=project, args=["build", element_name])
result2.assert_success()
# When this element is built it demonstrates that the virtual sandbox
# can detect sh across links and that the sandbox can find sh accross
diff --git a/tests/integration/workspace.py b/tests/integration/workspace.py
index 0d8a88007..6c6503f96 100644
--- a/tests/integration/workspace.py
+++ b/tests/integration/workspace.py
@@ -13,113 +13,107 @@ from buildstream._exceptions import ErrorDomain
pytestmark = pytest.mark.integration
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project"
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_workspace_stages_once(cli, datafiles):
project = str(datafiles)
- workspace = os.path.join(cli.directory, 'workspace')
- element_name = 'workspace/workspace-mount.bst'
+ workspace = os.path.join(cli.directory, "workspace")
+ element_name = "workspace/workspace-mount.bst"
- res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name])
+ res = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name])
assert res.exit_code == 0
- assert cli.get_element_key(project, element_name) != "{:?<64}".format('')
- res = cli.run(project=project, args=['build', element_name])
+ assert cli.get_element_key(project, element_name) != "{:?<64}".format("")
+ res = cli.run(project=project, args=["build", element_name])
assert res.exit_code == 0
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_workspace_mount(cli, datafiles):
project = str(datafiles)
- workspace = os.path.join(cli.directory, 'workspace')
- element_name = 'workspace/workspace-mount.bst'
+ workspace = os.path.join(cli.directory, "workspace")
+ element_name = "workspace/workspace-mount.bst"
- res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name])
+ res = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name])
assert res.exit_code == 0
- res = cli.run(project=project, args=['build', element_name])
+ res = cli.run(project=project, args=["build", element_name])
assert res.exit_code == 0
- assert os.path.exists(os.path.join(cli.directory, 'workspace'))
+ assert os.path.exists(os.path.join(cli.directory, "workspace"))
@pytest.mark.datafiles(DATA_DIR)
def test_workspace_mount_on_read_only_directory(cli, datafiles):
project = str(datafiles)
- workspace = os.path.join(cli.directory, 'workspace')
+ workspace = os.path.join(cli.directory, "workspace")
os.makedirs(workspace)
- element_name = 'workspace/workspace-mount.bst'
+ element_name = "workspace/workspace-mount.bst"
# make directory RO
os.chmod(workspace, 0o555)
- res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name])
+ res = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name])
assert res.exit_code == 0
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
+@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox")
@pytest.mark.xfail(reason="Incremental builds are currently incompatible with workspace source plugin.")
def test_workspace_commanddir(cli, datafiles):
project = str(datafiles)
- workspace = os.path.join(cli.directory, 'workspace')
- element_name = 'workspace/workspace-commanddir.bst'
+ workspace = os.path.join(cli.directory, "workspace")
+ element_name = "workspace/workspace-commanddir.bst"
- res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name])
+ res = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name])
assert res.exit_code == 0
- res = cli.run(project=project, args=['build', element_name])
+ res = cli.run(project=project, args=["build", element_name])
assert res.exit_code == 0
- assert os.path.exists(os.path.join(cli.directory, 'workspace'))
- assert os.path.exists(os.path.join(cli.directory, 'workspace', 'build'))
+ assert os.path.exists(os.path.join(cli.directory, "workspace"))
+ assert os.path.exists(os.path.join(cli.directory, "workspace", "build"))
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_workspace_updated_dependency(cli, datafiles):
project = str(datafiles)
- workspace = os.path.join(cli.directory, 'workspace')
- element_path = os.path.join(project, 'elements')
- element_name = 'workspace/workspace-updated-dependency.bst'
- dep_name = 'workspace/dependency.bst'
+ workspace = os.path.join(cli.directory, "workspace")
+ element_path = os.path.join(project, "elements")
+ element_name = "workspace/workspace-updated-dependency.bst"
+ dep_name = "workspace/dependency.bst"
dependency = {
- 'kind': 'manual',
- 'depends': [{
- 'filename': 'base.bst',
- 'type': 'build'
- }],
- 'config': {
- 'build-commands': [
- 'mkdir -p %{install-root}/etc/test/',
- 'echo "Hello world!" > %{install-root}/etc/test/hello.txt'
+ "kind": "manual",
+ "depends": [{"filename": "base.bst", "type": "build"}],
+ "config": {
+ "build-commands": [
+ "mkdir -p %{install-root}/etc/test/",
+ 'echo "Hello world!" > %{install-root}/etc/test/hello.txt',
]
- }
+ },
}
os.makedirs(os.path.dirname(os.path.join(element_path, dep_name)), exist_ok=True)
_yaml.roundtrip_dump(dependency, os.path.join(element_path, dep_name))
# First open the workspace
- res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name])
+ res = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name])
assert res.exit_code == 0
# We build the workspaced element, so that we have an artifact
# with specific built dependencies
- res = cli.run(project=project, args=['build', element_name])
+ res = cli.run(project=project, args=["build", element_name])
assert res.exit_code == 0
# Now we update a dependency of our element.
- dependency['config']['build-commands'] = [
- 'mkdir -p %{install-root}/etc/test/',
- 'echo "Hello china!" > %{install-root}/etc/test/hello.txt'
+ dependency["config"]["build-commands"] = [
+ "mkdir -p %{install-root}/etc/test/",
+ 'echo "Hello china!" > %{install-root}/etc/test/hello.txt',
]
_yaml.roundtrip_dump(dependency, os.path.join(element_path, dep_name))
@@ -128,185 +122,169 @@ def test_workspace_updated_dependency(cli, datafiles):
# therefore ensure that we change the mtimes of any files touched
# since the last successful build of this element, otherwise this
# build will fail.
- res = cli.run(project=project, args=['build', element_name])
+ res = cli.run(project=project, args=["build", element_name])
assert res.exit_code == 0
- res = cli.run(project=project, args=['shell', element_name, '/usr/bin/test.sh'])
+ res = cli.run(project=project, args=["shell", element_name, "/usr/bin/test.sh"])
assert res.exit_code == 0
- assert res.output == 'Hello china!\n\n'
+ assert res.output == "Hello china!\n\n"
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_workspace_update_dependency_failed(cli, datafiles):
project = str(datafiles)
- workspace = os.path.join(cli.directory, 'workspace')
- element_path = os.path.join(project, 'elements')
- element_name = 'workspace/workspace-updated-dependency-failed.bst'
- dep_name = 'workspace/dependency.bst'
+ workspace = os.path.join(cli.directory, "workspace")
+ element_path = os.path.join(project, "elements")
+ element_name = "workspace/workspace-updated-dependency-failed.bst"
+ dep_name = "workspace/dependency.bst"
dependency = {
- 'kind': 'manual',
- 'depends': [{
- 'filename': 'base.bst',
- 'type': 'build'
- }],
- 'config': {
- 'build-commands': [
- 'mkdir -p %{install-root}/etc/test/',
+ "kind": "manual",
+ "depends": [{"filename": "base.bst", "type": "build"}],
+ "config": {
+ "build-commands": [
+ "mkdir -p %{install-root}/etc/test/",
'echo "Hello world!" > %{install-root}/etc/test/hello.txt',
- 'echo "Hello brazil!" > %{install-root}/etc/test/brazil.txt'
+ 'echo "Hello brazil!" > %{install-root}/etc/test/brazil.txt',
]
- }
+ },
}
os.makedirs(os.path.dirname(os.path.join(element_path, dep_name)), exist_ok=True)
_yaml.roundtrip_dump(dependency, os.path.join(element_path, dep_name))
# First open the workspace
- res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name])
+ res = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name])
assert res.exit_code == 0
# We build the workspaced element, so that we have an artifact
# with specific built dependencies
- res = cli.run(project=project, args=['build', element_name])
+ res = cli.run(project=project, args=["build", element_name])
assert res.exit_code == 0
# Now we update a dependency of our element.
- dependency['config']['build-commands'] = [
- 'mkdir -p %{install-root}/etc/test/',
+ dependency["config"]["build-commands"] = [
+ "mkdir -p %{install-root}/etc/test/",
'echo "Hello china!" > %{install-root}/etc/test/hello.txt',
- 'echo "Hello brazil!" > %{install-root}/etc/test/brazil.txt'
+ 'echo "Hello brazil!" > %{install-root}/etc/test/brazil.txt',
]
_yaml.roundtrip_dump(dependency, os.path.join(element_path, dep_name))
# And our build fails!
- with open(os.path.join(workspace, 'Makefile'), 'a') as f:
+ with open(os.path.join(workspace, "Makefile"), "a") as f:
f.write("\texit 1")
- res = cli.run(project=project, args=['build', element_name])
+ res = cli.run(project=project, args=["build", element_name])
assert res.exit_code != 0
# We update our dependency again...
- dependency['config']['build-commands'] = [
- 'mkdir -p %{install-root}/etc/test/',
+ dependency["config"]["build-commands"] = [
+ "mkdir -p %{install-root}/etc/test/",
'echo "Hello world!" > %{install-root}/etc/test/hello.txt',
- 'echo "Hello spain!" > %{install-root}/etc/test/brazil.txt'
+ 'echo "Hello spain!" > %{install-root}/etc/test/brazil.txt',
]
_yaml.roundtrip_dump(dependency, os.path.join(element_path, dep_name))
# And fix the source
- with open(os.path.join(workspace, 'Makefile'), 'r') as f:
+ with open(os.path.join(workspace, "Makefile"), "r") as f:
makefile = f.readlines()
- with open(os.path.join(workspace, 'Makefile'), 'w') as f:
+ with open(os.path.join(workspace, "Makefile"), "w") as f:
f.write("\n".join(makefile[:-1]))
# Since buildstream thinks hello.txt did not change, we could end
# up not rebuilding a file! We need to make sure that a case like
# this can't blind-side us.
- res = cli.run(project=project, args=['build', element_name])
+ res = cli.run(project=project, args=["build", element_name])
assert res.exit_code == 0
- res = cli.run(project=project, args=['shell', element_name, '/usr/bin/test.sh'])
+ res = cli.run(project=project, args=["shell", element_name, "/usr/bin/test.sh"])
assert res.exit_code == 0
- assert res.output == 'Hello world!\nHello spain!\n\n'
+ assert res.output == "Hello world!\nHello spain!\n\n"
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_updated_dependency_nested(cli, datafiles):
project = str(datafiles)
- workspace = os.path.join(cli.directory, 'workspace')
- element_path = os.path.join(project, 'elements')
- element_name = 'workspace/workspace-updated-dependency-nested.bst'
- dep_name = 'workspace/dependency.bst'
+ workspace = os.path.join(cli.directory, "workspace")
+ element_path = os.path.join(project, "elements")
+ element_name = "workspace/workspace-updated-dependency-nested.bst"
+ dep_name = "workspace/dependency.bst"
dependency = {
- 'kind': 'manual',
- 'depends': [{
- 'filename': 'base.bst',
- 'type': 'build'
- }],
- 'config': {
- 'build-commands': [
- 'mkdir -p %{install-root}/etc/test/tests/',
+ "kind": "manual",
+ "depends": [{"filename": "base.bst", "type": "build"}],
+ "config": {
+ "build-commands": [
+ "mkdir -p %{install-root}/etc/test/tests/",
'echo "Hello world!" > %{install-root}/etc/test/hello.txt',
- 'echo "Hello brazil!" > %{install-root}/etc/test/tests/brazil.txt'
+ 'echo "Hello brazil!" > %{install-root}/etc/test/tests/brazil.txt',
]
- }
+ },
}
os.makedirs(os.path.dirname(os.path.join(element_path, dep_name)), exist_ok=True)
_yaml.roundtrip_dump(dependency, os.path.join(element_path, dep_name))
# First open the workspace
- res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name])
+ res = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name])
assert res.exit_code == 0
# We build the workspaced element, so that we have an artifact
# with specific built dependencies
- res = cli.run(project=project, args=['build', element_name])
+ res = cli.run(project=project, args=["build", element_name])
assert res.exit_code == 0
# Now we update a dependency of our element.
- dependency['config']['build-commands'] = [
- 'mkdir -p %{install-root}/etc/test/tests/',
+ dependency["config"]["build-commands"] = [
+ "mkdir -p %{install-root}/etc/test/tests/",
'echo "Hello world!" > %{install-root}/etc/test/hello.txt',
- 'echo "Hello test!" > %{install-root}/etc/test/tests/tests.txt'
+ 'echo "Hello test!" > %{install-root}/etc/test/tests/tests.txt',
]
_yaml.roundtrip_dump(dependency, os.path.join(element_path, dep_name))
- res = cli.run(project=project, args=['build', element_name])
+ res = cli.run(project=project, args=["build", element_name])
assert res.exit_code == 0
# Buildstream should pick up the newly added element, and pick up
# the lack of the newly removed element
- res = cli.run(project=project, args=['shell', element_name, '/usr/bin/test.sh'])
+ res = cli.run(project=project, args=["shell", element_name, "/usr/bin/test.sh"])
assert res.exit_code == 0
- assert res.output == 'Hello world!\nHello test!\n\n'
+ assert res.output == "Hello world!\nHello test!\n\n"
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
+@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox")
@pytest.mark.xfail(reason="Incremental builds are currently incompatible with workspace source plugin.")
def test_incremental_configure_commands_run_only_once(cli, datafiles):
project = str(datafiles)
- workspace = os.path.join(cli.directory, 'workspace')
- element_path = os.path.join(project, 'elements')
- element_name = 'workspace/incremental.bst'
+ workspace = os.path.join(cli.directory, "workspace")
+ element_path = os.path.join(project, "elements")
+ element_name = "workspace/incremental.bst"
element = {
- 'kind': 'manual',
- 'depends': [{
- 'filename': 'base.bst',
- 'type': 'build'
- }],
- 'sources': [{
- 'kind': 'local',
- 'path': 'files/workspace-configure-only-once'
- }],
- 'config': {
- 'configure-commands': [
- '$SHELL configure'
- ]
- }
+ "kind": "manual",
+ "depends": [{"filename": "base.bst", "type": "build"}],
+ "sources": [{"kind": "local", "path": "files/workspace-configure-only-once"}],
+ "config": {"configure-commands": ["$SHELL configure"]},
}
_yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
# We open a workspace on the above element
- res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name])
+ res = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name])
res.assert_success()
# Then we build, and check whether the configure step succeeded
- res = cli.run(project=project, args=['build', element_name])
+ res = cli.run(project=project, args=["build", element_name])
res.assert_success()
- assert os.path.exists(os.path.join(workspace, 'prepared'))
+ assert os.path.exists(os.path.join(workspace, "prepared"))
# When we build again, the configure commands should not be
# called, and we should therefore exit cleanly (the configure
# commands are set to always fail after the first run)
- res = cli.run(project=project, args=['build', element_name])
+ res = cli.run(project=project, args=["build", element_name])
res.assert_success()
- assert not os.path.exists(os.path.join(workspace, 'prepared-again'))
+ assert not os.path.exists(os.path.join(workspace, "prepared-again"))
# Test that rebuilding an already built workspaced element does
@@ -319,48 +297,48 @@ def test_incremental_configure_commands_run_only_once(cli, datafiles):
# part of a cleanup job.
#
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
+@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox")
def test_workspace_missing_last_successful(cli, datafiles):
project = str(datafiles)
- workspace = os.path.join(cli.directory, 'workspace')
- element_name = 'workspace/workspace-commanddir.bst'
+ workspace = os.path.join(cli.directory, "workspace")
+ element_name = "workspace/workspace-commanddir.bst"
# Open workspace
- res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name])
+ res = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name])
assert res.exit_code == 0
# Build first, this will record the last successful build in local state
- res = cli.run(project=project, args=['build', element_name])
+ res = cli.run(project=project, args=["build", element_name])
assert res.exit_code == 0
# Remove the artifact from the cache, invalidating the last successful build
- res = cli.run(project=project, args=['artifact', 'delete', element_name])
+ res = cli.run(project=project, args=["artifact", "delete", element_name])
assert res.exit_code == 0
# Build again, ensure we dont crash just because the artifact went missing
- res = cli.run(project=project, args=['build', element_name])
+ res = cli.run(project=project, args=["build", element_name])
assert res.exit_code == 0
# Check that we can still read failed workspace logs
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_workspace_failed_logs(cli, datafiles):
project = str(datafiles)
- workspace = os.path.join(cli.directory, 'failing_amhello')
- element_name = 'autotools/amhello-failure.bst'
+ workspace = os.path.join(cli.directory, "failing_amhello")
+ element_name = "autotools/amhello-failure.bst"
# Open workspace
- res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name])
+ res = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name])
res.assert_success()
# Try to build and ensure the build fails
- res = cli.run(project=project, args=['build', element_name])
+ res = cli.run(project=project, args=["build", element_name])
res.assert_main_error(ErrorDomain.STREAM, None)
- assert cli.get_element_state(project, element_name) == 'failed'
+ assert cli.get_element_state(project, element_name) == "failed"
- res = cli.run(project=project, args=['artifact', 'log', element_name])
+ res = cli.run(project=project, args=["artifact", "log", element_name])
res.assert_success()
log = res.output
diff --git a/tests/internals/context.py b/tests/internals/context.py
index ddd558b6c..c2ee1efb5 100644
--- a/tests/internals/context.py
+++ b/tests/internals/context.py
@@ -7,32 +7,26 @@ import pytest
from buildstream._context import Context
from buildstream._exceptions import LoadError, LoadErrorReason
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- 'context',
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "context",)
# Simple fixture to create a Context object.
@pytest.fixture()
def context_fixture():
- if os.environ.get('XDG_CACHE_HOME'):
- cache_home = os.environ['XDG_CACHE_HOME']
+ if os.environ.get("XDG_CACHE_HOME"):
+ cache_home = os.environ["XDG_CACHE_HOME"]
else:
- cache_home = os.path.expanduser('~/.cache')
+ cache_home = os.path.expanduser("~/.cache")
with Context() as context:
- yield {
- 'xdg-cache': cache_home,
- 'context': context
- }
+ yield {"xdg-cache": cache_home, "context": context}
#######################################
# Test instantiation #
#######################################
def test_context_create(context_fixture):
- context = context_fixture['context']
+ context = context_fixture["context"]
assert isinstance(context, Context)
@@ -40,51 +34,49 @@ def test_context_create(context_fixture):
# Test configuration loading #
#######################################
def test_context_load(context_fixture):
- context = context_fixture['context']
- cache_home = os.path.normpath(context_fixture['xdg-cache'])
+ context = context_fixture["context"]
+ cache_home = os.path.normpath(context_fixture["xdg-cache"])
assert isinstance(context, Context)
context.load(config=os.devnull)
- assert context.sourcedir == os.path.join(cache_home, 'buildstream', 'sources')
- assert context.builddir == os.path.join(cache_home, 'buildstream', 'build')
- assert context.cachedir == os.path.join(cache_home, 'buildstream')
- assert context.logdir == os.path.join(cache_home, 'buildstream', 'logs')
+ assert context.sourcedir == os.path.join(cache_home, "buildstream", "sources")
+ assert context.builddir == os.path.join(cache_home, "buildstream", "build")
+ assert context.cachedir == os.path.join(cache_home, "buildstream")
+ assert context.logdir == os.path.join(cache_home, "buildstream", "logs")
# Assert that a changed XDG_CACHE_HOME doesn't cause issues
def test_context_load_envvar(context_fixture):
- os.environ['XDG_CACHE_HOME'] = '/some/path/'
+ os.environ["XDG_CACHE_HOME"] = "/some/path/"
- context = context_fixture['context']
+ context = context_fixture["context"]
assert isinstance(context, Context)
context.load(config=os.devnull)
- assert context.sourcedir == os.path.join('/', 'some', 'path', 'buildstream', 'sources')
- assert context.builddir == os.path.join('/', 'some', 'path', 'buildstream', 'build')
- assert context.cachedir == os.path.join('/', 'some', 'path', 'buildstream')
- assert context.logdir == os.path.join('/', 'some', 'path', 'buildstream', 'logs')
+ assert context.sourcedir == os.path.join("/", "some", "path", "buildstream", "sources")
+ assert context.builddir == os.path.join("/", "some", "path", "buildstream", "build")
+ assert context.cachedir == os.path.join("/", "some", "path", "buildstream")
+ assert context.logdir == os.path.join("/", "some", "path", "buildstream", "logs")
# Reset the environment variable
- del os.environ['XDG_CACHE_HOME']
+ del os.environ["XDG_CACHE_HOME"]
# Test that values in a user specified config file
# override the defaults
@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_context_load_user_config(context_fixture, datafiles):
- context = context_fixture['context']
- cache_home = context_fixture['xdg-cache']
+ context = context_fixture["context"]
+ cache_home = context_fixture["xdg-cache"]
assert isinstance(context, Context)
- conf_file = os.path.join(datafiles.dirname,
- datafiles.basename,
- 'userconf.yaml')
+ conf_file = os.path.join(datafiles.dirname, datafiles.basename, "userconf.yaml")
context.load(conf_file)
- assert context.sourcedir == os.path.expanduser('~/pony')
- assert context.builddir == os.path.join(cache_home, 'buildstream', 'build')
- assert context.cachedir == os.path.join(cache_home, 'buildstream')
- assert context.logdir == os.path.join(cache_home, 'buildstream', 'logs')
+ assert context.sourcedir == os.path.expanduser("~/pony")
+ assert context.builddir == os.path.join(cache_home, "buildstream", "build")
+ assert context.cachedir == os.path.join(cache_home, "buildstream")
+ assert context.logdir == os.path.join(cache_home, "buildstream", "logs")
#######################################
@@ -92,12 +84,10 @@ def test_context_load_user_config(context_fixture, datafiles):
#######################################
@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_context_load_missing_config(context_fixture, datafiles):
- context = context_fixture['context']
+ context = context_fixture["context"]
assert isinstance(context, Context)
- conf_file = os.path.join(datafiles.dirname,
- datafiles.basename,
- 'nonexistant.yaml')
+ conf_file = os.path.join(datafiles.dirname, datafiles.basename, "nonexistant.yaml")
with pytest.raises(LoadError) as exc:
context.load(conf_file)
@@ -107,12 +97,10 @@ def test_context_load_missing_config(context_fixture, datafiles):
@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_context_load_malformed_config(context_fixture, datafiles):
- context = context_fixture['context']
+ context = context_fixture["context"]
assert isinstance(context, Context)
- conf_file = os.path.join(datafiles.dirname,
- datafiles.basename,
- 'malformed.yaml')
+ conf_file = os.path.join(datafiles.dirname, datafiles.basename, "malformed.yaml")
with pytest.raises(LoadError) as exc:
context.load(conf_file)
@@ -122,12 +110,10 @@ def test_context_load_malformed_config(context_fixture, datafiles):
@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_context_load_notdict_config(context_fixture, datafiles):
- context = context_fixture['context']
+ context = context_fixture["context"]
assert isinstance(context, Context)
- conf_file = os.path.join(datafiles.dirname,
- datafiles.basename,
- 'notdict.yaml')
+ conf_file = os.path.join(datafiles.dirname, datafiles.basename, "notdict.yaml")
with pytest.raises(LoadError) as exc:
context.load(conf_file)
diff --git a/tests/internals/loader.py b/tests/internals/loader.py
index 39ef8ac99..781d144ae 100644
--- a/tests/internals/loader.py
+++ b/tests/internals/loader.py
@@ -10,10 +10,7 @@ from buildstream._loader.loader import _NO_PROGRESS
from tests.testutils import dummy_context
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- 'loader',
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "loader",)
@contextmanager
@@ -26,52 +23,52 @@ def make_loader(basedir):
##############################################################
# Basics: Test behavior loading the simplest of projects #
##############################################################
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'onefile'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "onefile"))
def test_one_file(datafiles):
basedir = str(datafiles)
with make_loader(basedir) as loader:
- element = loader.load(['elements/onefile.bst'], _NO_PROGRESS)[0]
+ element = loader.load(["elements/onefile.bst"], _NO_PROGRESS)[0]
assert isinstance(element, MetaElement)
- assert element.kind == 'pony'
+ assert element.kind == "pony"
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'onefile'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "onefile"))
def test_missing_file(datafiles):
basedir = str(datafiles)
with make_loader(basedir) as loader, pytest.raises(LoadError) as exc:
- loader.load(['elements/missing.bst'], _NO_PROGRESS)
+ loader.load(["elements/missing.bst"], _NO_PROGRESS)
assert exc.value.reason == LoadErrorReason.MISSING_FILE
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'onefile'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "onefile"))
def test_invalid_reference(datafiles):
basedir = str(datafiles)
with make_loader(basedir) as loader, pytest.raises(LoadError) as exc:
- loader.load(['elements/badreference.bst'], _NO_PROGRESS)
+ loader.load(["elements/badreference.bst"], _NO_PROGRESS)
assert exc.value.reason == LoadErrorReason.INVALID_YAML
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'onefile'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "onefile"))
def test_invalid_yaml(datafiles):
basedir = str(datafiles)
with make_loader(basedir) as loader, pytest.raises(LoadError) as exc:
- loader.load(['elements/badfile.bst'], _NO_PROGRESS)
+ loader.load(["elements/badfile.bst"], _NO_PROGRESS)
assert exc.value.reason == LoadErrorReason.INVALID_YAML
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'onefile'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "onefile"))
def test_fail_fullpath_target(datafiles):
basedir = str(datafiles)
- fullpath = os.path.join(basedir, 'elements', 'onefile.bst')
+ fullpath = os.path.join(basedir, "elements", "onefile.bst")
with make_loader(basedir) as loader, pytest.raises(LoadError) as exc:
loader.load([fullpath], _NO_PROGRESS)
@@ -79,21 +76,21 @@ def test_fail_fullpath_target(datafiles):
assert exc.value.reason == LoadErrorReason.INVALID_DATA
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'onefile'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "onefile"))
def test_invalid_key(datafiles):
basedir = str(datafiles)
with make_loader(basedir) as loader, pytest.raises(LoadError) as exc:
- loader.load(['elements/invalidkey.bst'], _NO_PROGRESS)
+ loader.load(["elements/invalidkey.bst"], _NO_PROGRESS)
assert exc.value.reason == LoadErrorReason.INVALID_DATA
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'onefile'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "onefile"))
def test_invalid_directory_load(datafiles):
basedir = str(datafiles)
with make_loader(basedir) as loader, pytest.raises(LoadError) as exc:
- loader.load(['elements/'], _NO_PROGRESS)
+ loader.load(["elements/"], _NO_PROGRESS)
assert exc.value.reason == LoadErrorReason.LOADING_DIRECTORY
diff --git a/tests/internals/pluginfactory.py b/tests/internals/pluginfactory.py
index b3f77c8b1..13b9d3aae 100644
--- a/tests/internals/pluginfactory.py
+++ b/tests/internals/pluginfactory.py
@@ -10,236 +10,219 @@ from buildstream._elementfactory import ElementFactory
from buildstream._sourcefactory import SourceFactory
from buildstream._exceptions import PluginError
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- 'pluginfactory',
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "pluginfactory",)
# Simple fixture to create a PluginBase object that
# we use for loading plugins.
@pytest.fixture()
def plugin_fixture():
- return {
- 'base': PluginBase(package='buildstream.plugins')
- }
+ return {"base": PluginBase(package="buildstream.plugins")}
##############################################################
# Basics: test the fixture, test we can create the factories #
##############################################################
def test_fixture(plugin_fixture):
- assert isinstance(plugin_fixture['base'], PluginBase)
+ assert isinstance(plugin_fixture["base"], PluginBase)
def test_source_factory(plugin_fixture):
- factory = SourceFactory(plugin_fixture['base'])
+ factory = SourceFactory(plugin_fixture["base"])
assert isinstance(factory, SourceFactory)
def test_element_factory(plugin_fixture):
- factory = ElementFactory(plugin_fixture['base'])
+ factory = ElementFactory(plugin_fixture["base"])
assert isinstance(factory, ElementFactory)
##############################################################
# Check that we can load custom sources & elements #
##############################################################
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'customsource'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "customsource"))
def test_custom_source(plugin_fixture, datafiles):
- plugins = [Node.from_dict({
- 'origin': 'local',
- 'path': os.path.join(datafiles.dirname,
- datafiles.basename),
- 'plugins': ['foo']
- })]
- factory = SourceFactory(plugin_fixture['base'], plugin_origins=plugins)
+ plugins = [
+ Node.from_dict(
+ {"origin": "local", "path": os.path.join(datafiles.dirname, datafiles.basename), "plugins": ["foo"]}
+ )
+ ]
+ factory = SourceFactory(plugin_fixture["base"], plugin_origins=plugins)
assert isinstance(factory, SourceFactory)
- foo_type, _ = factory.lookup('foo')
- assert foo_type.__name__ == 'FooSource'
+ foo_type, _ = factory.lookup("foo")
+ assert foo_type.__name__ == "FooSource"
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'customelement'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "customelement"))
def test_custom_element(plugin_fixture, datafiles):
- plugins = [Node.from_dict({
- 'origin': 'local',
- 'path': os.path.join(datafiles.dirname,
- datafiles.basename),
- 'plugins': ['foo']
- })]
- factory = ElementFactory(plugin_fixture['base'], plugin_origins=plugins)
+ plugins = [
+ Node.from_dict(
+ {"origin": "local", "path": os.path.join(datafiles.dirname, datafiles.basename), "plugins": ["foo"]}
+ )
+ ]
+ factory = ElementFactory(plugin_fixture["base"], plugin_origins=plugins)
assert isinstance(factory, ElementFactory)
- foo_type, _ = factory.lookup('foo')
- assert foo_type.__name__ == 'FooElement'
+ foo_type, _ = factory.lookup("foo")
+ assert foo_type.__name__ == "FooElement"
##############################################################
# Check plugin loading failure modes #
##############################################################
def test_missing_source(plugin_fixture):
- factory = SourceFactory(plugin_fixture['base'])
+ factory = SourceFactory(plugin_fixture["base"])
assert isinstance(factory, SourceFactory)
# Test fails if PluginError is not raised
with pytest.raises(PluginError):
- factory.lookup('foo')
+ factory.lookup("foo")
def test_missing_element(plugin_fixture):
- factory = ElementFactory(plugin_fixture['base'])
+ factory = ElementFactory(plugin_fixture["base"])
assert isinstance(factory, ElementFactory)
# Test fails if PluginError is not raised
with pytest.raises(PluginError):
- factory.lookup('foo')
+ factory.lookup("foo")
# Load a factory with a plugin that returns a value instead of Source subclass
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'notatype'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "notatype"))
def test_source_notatype(plugin_fixture, datafiles):
- plugins = [Node.from_dict({
- 'origin': 'local',
- 'path': os.path.join(datafiles.dirname,
- datafiles.basename),
- 'plugins': ['foo']
- })]
- factory = SourceFactory(plugin_fixture['base'], plugin_origins=plugins)
+ plugins = [
+ Node.from_dict(
+ {"origin": "local", "path": os.path.join(datafiles.dirname, datafiles.basename), "plugins": ["foo"]}
+ )
+ ]
+ factory = SourceFactory(plugin_fixture["base"], plugin_origins=plugins)
with pytest.raises(PluginError):
- factory.lookup('foo')
+ factory.lookup("foo")
# Load a factory with a plugin that returns a value instead of Element subclass
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'notatype'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "notatype"))
def test_element_notatype(plugin_fixture, datafiles):
- plugins = [Node.from_dict({
- 'origin': 'local',
- 'path': os.path.join(datafiles.dirname,
- datafiles.basename),
- 'plugins': ['foo']
- })]
- factory = ElementFactory(plugin_fixture['base'], plugin_origins=plugins)
+ plugins = [
+ Node.from_dict(
+ {"origin": "local", "path": os.path.join(datafiles.dirname, datafiles.basename), "plugins": ["foo"]}
+ )
+ ]
+ factory = ElementFactory(plugin_fixture["base"], plugin_origins=plugins)
with pytest.raises(PluginError):
- factory.lookup('foo')
+ factory.lookup("foo")
# Load a factory with a plugin that returns a type
# which is not a Source subclass
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'wrongtype'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "wrongtype"))
def test_source_wrongtype(plugin_fixture, datafiles):
- plugins = [Node.from_dict({
- 'origin': 'local',
- 'path': os.path.join(datafiles.dirname,
- datafiles.basename),
- 'plugins': ['foo']
- })]
- factory = SourceFactory(plugin_fixture['base'], plugin_origins=plugins)
+ plugins = [
+ Node.from_dict(
+ {"origin": "local", "path": os.path.join(datafiles.dirname, datafiles.basename), "plugins": ["foo"]}
+ )
+ ]
+ factory = SourceFactory(plugin_fixture["base"], plugin_origins=plugins)
with pytest.raises(PluginError):
- factory.lookup('foo')
+ factory.lookup("foo")
# Load a factory with a plugin that returns a type
# which is not a Element subclass
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'wrongtype'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "wrongtype"))
def test_element_wrongtype(plugin_fixture, datafiles):
- plugins = [Node.from_dict({
- 'origin': 'local',
- 'path': os.path.join(datafiles.dirname,
- datafiles.basename),
- 'plugins': ['foo']
- })]
- factory = ElementFactory(plugin_fixture['base'], plugin_origins=plugins)
+ plugins = [
+ Node.from_dict(
+ {"origin": "local", "path": os.path.join(datafiles.dirname, datafiles.basename), "plugins": ["foo"]}
+ )
+ ]
+ factory = ElementFactory(plugin_fixture["base"], plugin_origins=plugins)
with pytest.raises(PluginError):
- factory.lookup('foo')
+ factory.lookup("foo")
# Load a factory with a plugin which fails to provide a setup() function
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'nosetup'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "nosetup"))
def test_source_missing_setup(plugin_fixture, datafiles):
- plugins = [Node.from_dict({
- 'origin': 'local',
- 'path': os.path.join(datafiles.dirname,
- datafiles.basename),
- 'plugins': ['foo']
- })]
- factory = SourceFactory(plugin_fixture['base'], plugin_origins=plugins)
+ plugins = [
+ Node.from_dict(
+ {"origin": "local", "path": os.path.join(datafiles.dirname, datafiles.basename), "plugins": ["foo"]}
+ )
+ ]
+ factory = SourceFactory(plugin_fixture["base"], plugin_origins=plugins)
with pytest.raises(PluginError):
- factory.lookup('foo')
+ factory.lookup("foo")
# Load a factory with a plugin which fails to provide a setup() function
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'nosetup'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "nosetup"))
def test_element_missing_setup(plugin_fixture, datafiles):
- plugins = [Node.from_dict({
- 'origin': 'local',
- 'path': os.path.join(datafiles.dirname,
- datafiles.basename),
- 'plugins': ['foo']
- })]
- factory = ElementFactory(plugin_fixture['base'], plugin_origins=plugins)
+ plugins = [
+ Node.from_dict(
+ {"origin": "local", "path": os.path.join(datafiles.dirname, datafiles.basename), "plugins": ["foo"]}
+ )
+ ]
+ factory = ElementFactory(plugin_fixture["base"], plugin_origins=plugins)
with pytest.raises(PluginError):
- factory.lookup('foo')
+ factory.lookup("foo")
# Load a factory with a plugin which provides a setup symbol
# that is not a function
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'badsetup'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "badsetup"))
def test_source_bad_setup(plugin_fixture, datafiles):
- plugins = [Node.from_dict({
- 'origin': 'local',
- 'path': os.path.join(datafiles.dirname,
- datafiles.basename),
- 'plugins': ['foo']
- })]
- factory = SourceFactory(plugin_fixture['base'], plugin_origins=plugins)
+ plugins = [
+ Node.from_dict(
+ {"origin": "local", "path": os.path.join(datafiles.dirname, datafiles.basename), "plugins": ["foo"]}
+ )
+ ]
+ factory = SourceFactory(plugin_fixture["base"], plugin_origins=plugins)
with pytest.raises(PluginError):
- factory.lookup('foo')
+ factory.lookup("foo")
# Load a factory with a plugin which provides a setup symbol
# that is not a function
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'badsetup'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "badsetup"))
def test_element_bad_setup(plugin_fixture, datafiles):
- plugins = [Node.from_dict({
- 'origin': 'local',
- 'path': os.path.join(datafiles.dirname,
- datafiles.basename),
- 'plugins': ['foo']
- })]
- factory = ElementFactory(plugin_fixture['base'], plugin_origins=plugins)
+ plugins = [
+ Node.from_dict(
+ {"origin": "local", "path": os.path.join(datafiles.dirname, datafiles.basename), "plugins": ["foo"]}
+ )
+ ]
+ factory = ElementFactory(plugin_fixture["base"], plugin_origins=plugins)
with pytest.raises(PluginError):
- factory.lookup('foo')
+ factory.lookup("foo")
# Load a factory with a plugin which requires an absurdly
# high version of buildstream
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'badversionsource'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "badversionsource"))
def test_source_badversion(plugin_fixture, datafiles):
- plugins = [Node.from_dict({
- 'origin': 'local',
- 'path': os.path.join(datafiles.dirname,
- datafiles.basename),
- 'plugins': ['foo']
- })]
- factory = SourceFactory(plugin_fixture['base'], plugin_origins=plugins)
+ plugins = [
+ Node.from_dict(
+ {"origin": "local", "path": os.path.join(datafiles.dirname, datafiles.basename), "plugins": ["foo"]}
+ )
+ ]
+ factory = SourceFactory(plugin_fixture["base"], plugin_origins=plugins)
with pytest.raises(PluginError):
- factory.lookup('foo')
+ factory.lookup("foo")
# Load a factory with a plugin which requires an absurdly
# high version of buildstream
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'badversionelement'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "badversionelement"))
def test_element_badversion(plugin_fixture, datafiles):
- plugins = [Node.from_dict({
- 'origin': 'local',
- 'path': os.path.join(datafiles.dirname,
- datafiles.basename),
- 'plugins': ['foo']
- })]
- factory = ElementFactory(plugin_fixture['base'], plugin_origins=plugins)
+ plugins = [
+ Node.from_dict(
+ {"origin": "local", "path": os.path.join(datafiles.dirname, datafiles.basename), "plugins": ["foo"]}
+ )
+ ]
+ factory = ElementFactory(plugin_fixture["base"], plugin_origins=plugins)
with pytest.raises(PluginError):
- factory.lookup('foo')
+ factory.lookup("foo")
##############################################################
@@ -249,56 +232,56 @@ def test_element_badversion(plugin_fixture, datafiles):
# Load two factories, both of which define a different 'foo' plugin
@pytest.mark.datafiles(DATA_DIR)
def test_source_multicontext(plugin_fixture, datafiles):
- plugins1 = Node.from_dict({
- 'origin': 'local',
- 'path': os.path.join(datafiles.dirname,
- datafiles.basename,
- 'customsource'),
- 'plugins': ['foo']
- })
- plugins2 = Node.from_dict({
- 'origin': 'local',
- 'path': os.path.join(datafiles.dirname,
- datafiles.basename,
- 'anothersource'),
- 'plugins': ['foo']
- })
-
- factory1 = SourceFactory(plugin_fixture['base'], plugin_origins=[plugins1])
- factory2 = SourceFactory(plugin_fixture['base'], plugin_origins=[plugins2])
+ plugins1 = Node.from_dict(
+ {
+ "origin": "local",
+ "path": os.path.join(datafiles.dirname, datafiles.basename, "customsource"),
+ "plugins": ["foo"],
+ }
+ )
+ plugins2 = Node.from_dict(
+ {
+ "origin": "local",
+ "path": os.path.join(datafiles.dirname, datafiles.basename, "anothersource"),
+ "plugins": ["foo"],
+ }
+ )
+
+ factory1 = SourceFactory(plugin_fixture["base"], plugin_origins=[plugins1])
+ factory2 = SourceFactory(plugin_fixture["base"], plugin_origins=[plugins2])
assert isinstance(factory1, SourceFactory)
assert isinstance(factory2, SourceFactory)
- foo_type1, _ = factory1.lookup('foo')
- foo_type2, _ = factory2.lookup('foo')
- assert foo_type1.__name__ == 'FooSource'
- assert foo_type2.__name__ == 'AnotherFooSource'
+ foo_type1, _ = factory1.lookup("foo")
+ foo_type2, _ = factory2.lookup("foo")
+ assert foo_type1.__name__ == "FooSource"
+ assert foo_type2.__name__ == "AnotherFooSource"
# Load two factories, both of which define a different 'foo' plugin
@pytest.mark.datafiles(DATA_DIR)
def test_element_multicontext(plugin_fixture, datafiles):
- plugins1 = Node.from_dict({
- 'origin': 'local',
- 'path': os.path.join(datafiles.dirname,
- datafiles.basename,
- 'customelement'),
- 'plugins': ['foo']
- })
- plugins2 = Node.from_dict({
- 'origin': 'local',
- 'path': os.path.join(datafiles.dirname,
- datafiles.basename,
- 'anotherelement'),
- 'plugins': ['foo']
- })
-
- factory1 = ElementFactory(plugin_fixture['base'], plugin_origins=[plugins1])
- factory2 = ElementFactory(plugin_fixture['base'], plugin_origins=[plugins2])
+ plugins1 = Node.from_dict(
+ {
+ "origin": "local",
+ "path": os.path.join(datafiles.dirname, datafiles.basename, "customelement"),
+ "plugins": ["foo"],
+ }
+ )
+ plugins2 = Node.from_dict(
+ {
+ "origin": "local",
+ "path": os.path.join(datafiles.dirname, datafiles.basename, "anotherelement"),
+ "plugins": ["foo"],
+ }
+ )
+
+ factory1 = ElementFactory(plugin_fixture["base"], plugin_origins=[plugins1])
+ factory2 = ElementFactory(plugin_fixture["base"], plugin_origins=[plugins2])
assert isinstance(factory1, ElementFactory)
assert isinstance(factory2, ElementFactory)
- foo_type1, _ = factory1.lookup('foo')
- foo_type2, _ = factory2.lookup('foo')
- assert foo_type1.__name__ == 'FooElement'
- assert foo_type2.__name__ == 'AnotherFooElement'
+ foo_type1, _ = factory1.lookup("foo")
+ foo_type2, _ = factory2.lookup("foo")
+ assert foo_type1.__name__ == "FooElement"
+ assert foo_type2.__name__ == "AnotherFooElement"
diff --git a/tests/internals/pluginfactory/wrongtype/foo.py b/tests/internals/pluginfactory/wrongtype/foo.py
index 3fe9a1a62..37d9f6bfe 100644
--- a/tests/internals/pluginfactory/wrongtype/foo.py
+++ b/tests/internals/pluginfactory/wrongtype/foo.py
@@ -4,7 +4,7 @@
# This one fails the requirement
-class Foo():
+class Foo:
pass
diff --git a/tests/internals/pluginloading.py b/tests/internals/pluginloading.py
index 4b6baf229..83944bbd9 100644
--- a/tests/internals/pluginloading.py
+++ b/tests/internals/pluginloading.py
@@ -8,55 +8,52 @@ from buildstream._pipeline import Pipeline
from tests.testutils import dummy_context
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- 'pluginloading',
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "pluginloading",)
@contextmanager
def create_pipeline(tmpdir, basedir, target):
with dummy_context() as context:
- context.deploydir = os.path.join(str(tmpdir), 'deploy')
- context.casdir = os.path.join(str(tmpdir), 'cas')
+ context.deploydir = os.path.join(str(tmpdir), "deploy")
+ context.casdir = os.path.join(str(tmpdir), "cas")
project = Project(basedir, context)
pipeline = Pipeline(context, project, None)
- targets, = pipeline.load([(target,)])
+ (targets,) = pipeline.load([(target,)])
yield targets
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'customsource'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "customsource"))
def test_customsource(datafiles, tmpdir):
basedir = str(datafiles)
- with create_pipeline(tmpdir, basedir, 'simple.bst') as targets:
+ with create_pipeline(tmpdir, basedir, "simple.bst") as targets:
assert targets[0].get_kind() == "autotools"
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'customelement'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "customelement"))
def test_customelement(datafiles, tmpdir):
basedir = str(datafiles)
- with create_pipeline(tmpdir, basedir, 'simple.bst') as targets:
+ with create_pipeline(tmpdir, basedir, "simple.bst") as targets:
assert targets[0].get_kind() == "foo"
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'badversionsource'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "badversionsource"))
def test_badversionsource(datafiles, tmpdir):
basedir = str(datafiles)
- with pytest.raises(LoadError) as exc, create_pipeline(tmpdir, basedir, 'simple.bst'):
+ with pytest.raises(LoadError) as exc, create_pipeline(tmpdir, basedir, "simple.bst"):
pass
assert exc.value.reason == LoadErrorReason.UNSUPPORTED_PLUGIN
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'badversionelement'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "badversionelement"))
def test_badversionelement(datafiles, tmpdir):
basedir = str(datafiles)
- with pytest.raises(LoadError) as exc, create_pipeline(tmpdir, basedir, 'simple.bst'):
+ with pytest.raises(LoadError) as exc, create_pipeline(tmpdir, basedir, "simple.bst"):
pass
assert exc.value.reason == LoadErrorReason.UNSUPPORTED_PLUGIN
diff --git a/tests/internals/pluginloading/customelement/pluginelements/foo.py b/tests/internals/pluginloading/customelement/pluginelements/foo.py
index 823306ebc..c6a85a5b1 100644
--- a/tests/internals/pluginloading/customelement/pluginelements/foo.py
+++ b/tests/internals/pluginloading/customelement/pluginelements/foo.py
@@ -2,7 +2,6 @@ from buildstream import Element
class FooElement(Element):
-
def preflight(self):
pass
diff --git a/tests/internals/pluginloading/customsource/pluginsources/foo.py b/tests/internals/pluginloading/customsource/pluginsources/foo.py
index 8dd16801c..706c96f3b 100644
--- a/tests/internals/pluginloading/customsource/pluginsources/foo.py
+++ b/tests/internals/pluginloading/customsource/pluginsources/foo.py
@@ -2,7 +2,6 @@ from buildstream import Source, Consistency
class FooSource(Source):
-
def preflight(self):
pass
diff --git a/tests/internals/storage.py b/tests/internals/storage.py
index a26ca4858..8aa7f4a17 100644
--- a/tests/internals/storage.py
+++ b/tests/internals/storage.py
@@ -7,10 +7,7 @@ from buildstream._cas import CASCache
from buildstream.storage._casbaseddirectory import CasBasedDirectory
from buildstream.storage._filebaseddirectory import FileBasedDirectory
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "storage"
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "storage")
@contextmanager
@@ -25,8 +22,7 @@ def setup_backend(backend_class, tmpdir):
cas_cache.release_resources()
-@pytest.mark.parametrize("backend", [
- FileBasedDirectory, CasBasedDirectory])
+@pytest.mark.parametrize("backend", [FileBasedDirectory, CasBasedDirectory])
@pytest.mark.datafiles(DATA_DIR)
def test_import(tmpdir, datafiles, backend):
original = os.path.join(str(datafiles), "original")
@@ -38,8 +34,7 @@ def test_import(tmpdir, datafiles, backend):
assert "bin/hello" in c.list_relative_paths()
-@pytest.mark.parametrize("backend", [
- FileBasedDirectory, CasBasedDirectory])
+@pytest.mark.parametrize("backend", [FileBasedDirectory, CasBasedDirectory])
@pytest.mark.datafiles(DATA_DIR)
def test_modified_file_list(tmpdir, datafiles, backend):
original = os.path.join(str(datafiles), "original")
diff --git a/tests/internals/storage_vdir_import.py b/tests/internals/storage_vdir_import.py
index 808e1be9a..6b70f92b5 100644
--- a/tests/internals/storage_vdir_import.py
+++ b/tests/internals/storage_vdir_import.py
@@ -38,11 +38,11 @@ from buildstream.storage.directory import VirtualDirectoryError
# (directory) with content being the contents for a file or the
# destination for a symlink.
root_filesets = [
- [('a/b/c/textfile1', 'F', 'This is textfile 1\n')],
- [('a/b/c/textfile1', 'F', 'This is the replacement textfile 1\n')],
- [('a/b/f', 'S', '/a/b/c')],
- [('a/b/c', 'D', ''), ('a/b/f', 'S', '/a/b/c')],
- [('a/b/f', 'F', 'This is textfile 1\n')],
+ [("a/b/c/textfile1", "F", "This is textfile 1\n")],
+ [("a/b/c/textfile1", "F", "This is the replacement textfile 1\n")],
+ [("a/b/f", "S", "/a/b/c")],
+ [("a/b/c", "D", ""), ("a/b/f", "S", "/a/b/c")],
+ [("a/b/f", "F", "This is textfile 1\n")],
]
empty_hash_ref = sha256().hexdigest()
@@ -60,14 +60,14 @@ def generate_import_root(rootdir, filelist):
if os.path.exists(rootdir):
return
for (path, typesymbol, content) in filelist:
- if typesymbol == 'F':
+ if typesymbol == "F":
(dirnames, filename) = os.path.split(path)
os.makedirs(os.path.join(rootdir, dirnames), exist_ok=True)
with open(os.path.join(rootdir, dirnames, filename), "wt") as f:
f.write(content)
- elif typesymbol == 'D':
+ elif typesymbol == "D":
os.makedirs(os.path.join(rootdir, path), exist_ok=True)
- elif typesymbol == 'S':
+ elif typesymbol == "S":
(dirnames, filename) = os.path.split(path)
os.makedirs(os.path.join(rootdir, dirnames), exist_ok=True)
os.symlink(content, os.path.join(rootdir, path))
@@ -83,26 +83,26 @@ def generate_random_root(rootno, directory):
if os.path.exists(rootdir):
return
things = []
- locations = ['.']
+ locations = ["."]
os.makedirs(rootdir)
for i in range(0, 100):
location = random.choice(locations)
thingname = "node{}".format(i)
- thing = random.choice(['dir', 'link', 'file'])
- if thing == 'dir':
+ thing = random.choice(["dir", "link", "file"])
+ if thing == "dir":
thingname = "dir" + thingname
target = os.path.join(rootdir, location, thingname)
- if thing == 'dir':
+ if thing == "dir":
os.makedirs(target)
locations.append(os.path.join(location, thingname))
- elif thing == 'file':
+ elif thing == "file":
with open(target, "wt") as f:
f.write("This is node {}\n".format(i))
- elif thing == 'link':
- symlink_type = random.choice(['absolute', 'relative', 'broken'])
- if symlink_type == 'broken' or not things:
+ elif thing == "link":
+ symlink_type = random.choice(["absolute", "relative", "broken"])
+ if symlink_type == "broken" or not things:
os.symlink("/broken", target)
- elif symlink_type == 'absolute':
+ elif symlink_type == "absolute":
symlink_destination = random.choice(things)
os.symlink(symlink_destination, target)
else:
@@ -159,7 +159,7 @@ def resolve_symlinks(path, root):
if os.path.islink(location):
# Resolve the link, add on all the remaining components
target = os.path.join(os.readlink(location))
- tail = os.path.sep.join(components[i + 1:])
+ tail = os.path.sep.join(components[i + 1 :])
if target.startswith(os.path.sep):
# Absolute link - relative to root
@@ -202,22 +202,23 @@ def _import_test(tmpdir, original, overlay, generator_function, verify_contents=
for item in root_filesets[overlay - 1]:
(path, typename, content) = item
realpath = resolve_symlinks(path, export_dir)
- if typename == 'F':
+ if typename == "F":
if os.path.isdir(realpath) and directory_not_empty(realpath):
# The file should not have overwritten the directory in this case.
pass
else:
- assert os.path.isfile(realpath), \
- "{} did not exist in the combined virtual directory".format(path)
+ assert os.path.isfile(realpath), "{} did not exist in the combined virtual directory".format(
+ path
+ )
assert file_contents_are(realpath, content)
- elif typename == 'S':
+ elif typename == "S":
if os.path.isdir(realpath) and directory_not_empty(realpath):
# The symlink should not have overwritten the directory in this case.
pass
else:
assert os.path.islink(realpath)
assert os.readlink(realpath) == content
- elif typename == 'D':
+ elif typename == "D":
# We can't do any more tests than this because it
# depends on things present in the original. Blank
# directories here will be ignored and the original
@@ -274,22 +275,18 @@ def test_fixed_directory_listing(tmpdir, root):
# Check that the vdir is decending and readable
def test_descend(tmpdir):
- cas_dir = os.path.join(str(tmpdir), 'cas')
+ cas_dir = os.path.join(str(tmpdir), "cas")
cas_cache = CASCache(cas_dir)
try:
d = CasBasedDirectory(cas_cache)
- Content_to_check = 'You got me'
- test_dir = os.path.join(str(tmpdir), 'importfrom')
- filesys_discription = [
- ('a', 'D', ''),
- ('a/l', 'D', ''),
- ('a/l/g', 'F', Content_to_check)
- ]
+ Content_to_check = "You got me"
+ test_dir = os.path.join(str(tmpdir), "importfrom")
+ filesys_discription = [("a", "D", ""), ("a/l", "D", ""), ("a/l/g", "F", Content_to_check)]
generate_import_root(test_dir, filesys_discription)
d.import_files(test_dir)
- digest = d.descend('a', 'l').index['g'].get_digest()
+ digest = d.descend("a", "l").index["g"].get_digest()
assert Content_to_check == open(cas_cache.objpath(digest)).read()
finally:
@@ -300,31 +297,27 @@ def test_descend(tmpdir):
# Make sure the correct erros are raised when trying
# to decend in to files or links to files
def test_bad_symlinks(tmpdir):
- cas_dir = os.path.join(str(tmpdir), 'cas')
+ cas_dir = os.path.join(str(tmpdir), "cas")
cas_cache = CASCache(cas_dir)
try:
d = CasBasedDirectory(cas_cache)
- test_dir = os.path.join(str(tmpdir), 'importfrom')
- filesys_discription = [
- ('a', 'D', ''),
- ('a/l', 'S', '../target'),
- ('target', 'F', 'You got me')
- ]
+ test_dir = os.path.join(str(tmpdir), "importfrom")
+ filesys_discription = [("a", "D", ""), ("a/l", "S", "../target"), ("target", "F", "You got me")]
generate_import_root(test_dir, filesys_discription)
d.import_files(test_dir)
exp_reason = "not-a-directory"
with pytest.raises(VirtualDirectoryError) as error:
- d.descend('a', 'l', follow_symlinks=True)
+ d.descend("a", "l", follow_symlinks=True)
assert error.reason == exp_reason
with pytest.raises(VirtualDirectoryError) as error:
- d.descend('a', 'l')
+ d.descend("a", "l")
assert error.reason == exp_reason
with pytest.raises(VirtualDirectoryError) as error:
- d.descend('a', 'f')
+ d.descend("a", "f")
assert error.reason == exp_reason
finally:
cas_cache.release_resources()
@@ -333,23 +326,23 @@ def test_bad_symlinks(tmpdir):
# Check symlink logic for edgecases
# Check decend accross relitive link
def test_relative_symlink(tmpdir):
- cas_dir = os.path.join(str(tmpdir), 'cas')
+ cas_dir = os.path.join(str(tmpdir), "cas")
cas_cache = CASCache(cas_dir)
try:
d = CasBasedDirectory(cas_cache)
- Content_to_check = 'You got me'
- test_dir = os.path.join(str(tmpdir), 'importfrom')
+ Content_to_check = "You got me"
+ test_dir = os.path.join(str(tmpdir), "importfrom")
filesys_discription = [
- ('a', 'D', ''),
- ('a/l', 'S', '../target'),
- ('target', 'D', ''),
- ('target/file', 'F', Content_to_check)
+ ("a", "D", ""),
+ ("a/l", "S", "../target"),
+ ("target", "D", ""),
+ ("target/file", "F", Content_to_check),
]
generate_import_root(test_dir, filesys_discription)
d.import_files(test_dir)
- digest = d.descend('a', 'l', follow_symlinks=True).index['file'].get_digest()
+ digest = d.descend("a", "l", follow_symlinks=True).index["file"].get_digest()
assert Content_to_check == open(cas_cache.objpath(digest)).read()
finally:
cas_cache.release_resources()
@@ -358,23 +351,23 @@ def test_relative_symlink(tmpdir):
# Check symlink logic for edgecases
# Check deccend accross abs link
def test_abs_symlink(tmpdir):
- cas_dir = os.path.join(str(tmpdir), 'cas')
+ cas_dir = os.path.join(str(tmpdir), "cas")
cas_cache = CASCache(cas_dir)
try:
d = CasBasedDirectory(cas_cache)
- Content_to_check = 'two step file'
- test_dir = os.path.join(str(tmpdir), 'importfrom')
+ Content_to_check = "two step file"
+ test_dir = os.path.join(str(tmpdir), "importfrom")
filesys_discription = [
- ('a', 'D', ''),
- ('a/l', 'S', '/target'),
- ('target', 'D', ''),
- ('target/file', 'F', Content_to_check)
+ ("a", "D", ""),
+ ("a/l", "S", "/target"),
+ ("target", "D", ""),
+ ("target/file", "F", Content_to_check),
]
generate_import_root(test_dir, filesys_discription)
d.import_files(test_dir)
- digest = d.descend('a', 'l', follow_symlinks=True).index['file'].get_digest()
+ digest = d.descend("a", "l", follow_symlinks=True).index["file"].get_digest()
assert Content_to_check == open(cas_cache.objpath(digest)).read()
finally:
@@ -384,24 +377,24 @@ def test_abs_symlink(tmpdir):
# Check symlink logic for edgecases
# Check symlink can not escape root
def test_bad_sym_escape(tmpdir):
- cas_dir = os.path.join(str(tmpdir), 'cas')
+ cas_dir = os.path.join(str(tmpdir), "cas")
cas_cache = CASCache(cas_dir)
try:
d = CasBasedDirectory(cas_cache)
- test_dir = os.path.join(str(tmpdir), 'importfrom')
+ test_dir = os.path.join(str(tmpdir), "importfrom")
filesys_discription = [
- ('jail', 'D', ''),
- ('jail/a', 'D', ''),
- ('jail/a/l', 'S', '../../target'),
- ('target', 'D', ''),
- ('target/file', 'F', 'two step file')
+ ("jail", "D", ""),
+ ("jail/a", "D", ""),
+ ("jail/a/l", "S", "../../target"),
+ ("target", "D", ""),
+ ("target/file", "F", "two step file"),
]
generate_import_root(test_dir, filesys_discription)
- d.import_files(os.path.join(test_dir, 'jail'))
+ d.import_files(os.path.join(test_dir, "jail"))
with pytest.raises(VirtualDirectoryError) as error:
- d.descend('a', 'l', follow_symlinks=True)
+ d.descend("a", "l", follow_symlinks=True)
assert error.reason == "directory-not-found"
finally:
cas_cache.release_resources()
diff --git a/tests/internals/utils_save_atomic.py b/tests/internals/utils_save_atomic.py
index 0731f7bea..898286076 100644
--- a/tests/internals/utils_save_atomic.py
+++ b/tests/internals/utils_save_atomic.py
@@ -5,58 +5,58 @@ from buildstream.utils import save_file_atomic
def test_save_new_file(tmpdir):
- filename = os.path.join(str(tmpdir), 'savefile-success.test')
- with save_file_atomic(filename, 'w') as f:
- f.write('foo\n')
+ filename = os.path.join(str(tmpdir), "savefile-success.test")
+ with save_file_atomic(filename, "w") as f:
+ f.write("foo\n")
- assert os.listdir(str(tmpdir)) == ['savefile-success.test']
+ assert os.listdir(str(tmpdir)) == ["savefile-success.test"]
with open(filename) as f:
- assert f.read() == 'foo\n'
+ assert f.read() == "foo\n"
def test_save_over_existing_file(tmpdir):
- filename = os.path.join(str(tmpdir), 'savefile-overwrite.test')
+ filename = os.path.join(str(tmpdir), "savefile-overwrite.test")
- with open(filename, 'w') as f:
- f.write('existing contents\n')
+ with open(filename, "w") as f:
+ f.write("existing contents\n")
- with save_file_atomic(filename, 'w') as f:
- f.write('overwritten contents\n')
+ with save_file_atomic(filename, "w") as f:
+ f.write("overwritten contents\n")
- assert os.listdir(str(tmpdir)) == ['savefile-overwrite.test']
+ assert os.listdir(str(tmpdir)) == ["savefile-overwrite.test"]
with open(filename) as f:
- assert f.read() == 'overwritten contents\n'
+ assert f.read() == "overwritten contents\n"
def test_exception_new_file(tmpdir):
- filename = os.path.join(str(tmpdir), 'savefile-exception.test')
+ filename = os.path.join(str(tmpdir), "savefile-exception.test")
with pytest.raises(RuntimeError):
- with save_file_atomic(filename, 'w') as f:
- f.write('Some junk\n')
+ with save_file_atomic(filename, "w") as f:
+ f.write("Some junk\n")
raise RuntimeError("Something goes wrong")
assert os.listdir(str(tmpdir)) == []
def test_exception_existing_file(tmpdir):
- filename = os.path.join(str(tmpdir), 'savefile-existing.test')
+ filename = os.path.join(str(tmpdir), "savefile-existing.test")
- with open(filename, 'w') as f:
- f.write('existing contents\n')
+ with open(filename, "w") as f:
+ f.write("existing contents\n")
with pytest.raises(RuntimeError):
- with save_file_atomic(filename, 'w') as f:
- f.write('Some junk\n')
+ with save_file_atomic(filename, "w") as f:
+ f.write("Some junk\n")
raise RuntimeError("Something goes wrong")
- assert os.listdir(str(tmpdir)) == ['savefile-existing.test']
+ assert os.listdir(str(tmpdir)) == ["savefile-existing.test"]
with open(filename) as f:
- assert f.read() == 'existing contents\n'
+ assert f.read() == "existing contents\n"
def test_attributes(tmpdir):
- filename = os.path.join(str(tmpdir), 'savefile-attributes.test')
- with save_file_atomic(filename, 'w') as f:
+ filename = os.path.join(str(tmpdir), "savefile-attributes.test")
+ with save_file_atomic(filename, "w") as f:
assert f.real_filename == filename
assert f.name != filename
diff --git a/tests/internals/yaml.py b/tests/internals/yaml.py
index 93619dc4c..7b711575c 100644
--- a/tests/internals/yaml.py
+++ b/tests/internals/yaml.py
@@ -7,21 +7,16 @@ from buildstream import _yaml, Node, ProvenanceInformation, SequenceNode
from buildstream._exceptions import LoadError, LoadErrorReason
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- 'yaml',
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "yaml",)
@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_load_yaml(datafiles):
- filename = os.path.join(datafiles.dirname,
- datafiles.basename,
- 'basics.yaml')
+ filename = os.path.join(datafiles.dirname, datafiles.basename, "basics.yaml")
loaded = _yaml.load(filename)
- assert loaded.get_str('kind') == 'pony'
+ assert loaded.get_str("kind") == "pony"
def assert_provenance(filename, line, col, node):
@@ -37,12 +32,10 @@ def assert_provenance(filename, line, col, node):
@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_basic_provenance(datafiles):
- filename = os.path.join(datafiles.dirname,
- datafiles.basename,
- 'basics.yaml')
+ filename = os.path.join(datafiles.dirname, datafiles.basename, "basics.yaml")
loaded = _yaml.load(filename)
- assert loaded.get_str('kind') == 'pony'
+ assert loaded.get_str("kind") == "pony"
assert_provenance(filename, 1, 0, loaded)
@@ -50,45 +43,37 @@ def test_basic_provenance(datafiles):
@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_member_provenance(datafiles):
- filename = os.path.join(datafiles.dirname,
- datafiles.basename,
- 'basics.yaml')
+ filename = os.path.join(datafiles.dirname, datafiles.basename, "basics.yaml")
loaded = _yaml.load(filename)
- assert loaded.get_str('kind') == 'pony'
- assert_provenance(filename, 2, 13, loaded.get_scalar('description'))
+ assert loaded.get_str("kind") == "pony"
+ assert_provenance(filename, 2, 13, loaded.get_scalar("description"))
@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_element_provenance(datafiles):
- filename = os.path.join(datafiles.dirname,
- datafiles.basename,
- 'basics.yaml')
+ filename = os.path.join(datafiles.dirname, datafiles.basename, "basics.yaml")
loaded = _yaml.load(filename)
- assert loaded.get_str('kind') == 'pony'
- assert_provenance(filename, 5, 2, loaded.get_sequence('moods').scalar_at(1))
+ assert loaded.get_str("kind") == "pony"
+ assert_provenance(filename, 5, 2, loaded.get_sequence("moods").scalar_at(1))
@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_mapping_validate_keys(datafiles):
- valid = os.path.join(datafiles.dirname,
- datafiles.basename,
- 'basics.yaml')
- invalid = os.path.join(datafiles.dirname,
- datafiles.basename,
- 'invalid.yaml')
+ valid = os.path.join(datafiles.dirname, datafiles.basename, "basics.yaml")
+ invalid = os.path.join(datafiles.dirname, datafiles.basename, "invalid.yaml")
base = _yaml.load(valid)
- base.validate_keys(['kind', 'description', 'moods', 'children', 'extra'])
+ base.validate_keys(["kind", "description", "moods", "children", "extra"])
base = _yaml.load(invalid)
with pytest.raises(LoadError) as exc:
- base.validate_keys(['kind', 'description', 'moods', 'children', 'extra'])
+ base.validate_keys(["kind", "description", "moods", "children", "extra"])
assert exc.value.reason == LoadErrorReason.INVALID_DATA
@@ -96,23 +81,21 @@ def test_mapping_validate_keys(datafiles):
@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_node_get(datafiles):
- filename = os.path.join(datafiles.dirname,
- datafiles.basename,
- 'basics.yaml')
+ filename = os.path.join(datafiles.dirname, datafiles.basename, "basics.yaml")
base = _yaml.load(filename)
- assert base.get_str('kind') == 'pony'
+ assert base.get_str("kind") == "pony"
- children = base.get_sequence('children')
+ children = base.get_sequence("children")
assert isinstance(children, SequenceNode)
assert len(children) == 7
- child = base.get_sequence('children').mapping_at(6)
- assert_provenance(filename, 20, 8, child.get_scalar('mood'))
+ child = base.get_sequence("children").mapping_at(6)
+ assert_provenance(filename, 20, 8, child.get_scalar("mood"))
- extra = base.get_mapping('extra')
+ extra = base.get_mapping("extra")
with pytest.raises(LoadError) as exc:
- extra.get_mapping('old')
+ extra.get_mapping("old")
assert exc.value.reason == LoadErrorReason.INVALID_DATA
@@ -120,50 +103,44 @@ def test_node_get(datafiles):
@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_node_set(datafiles):
- filename = os.path.join(datafiles.dirname,
- datafiles.basename,
- 'basics.yaml')
+ filename = os.path.join(datafiles.dirname, datafiles.basename, "basics.yaml")
base = _yaml.load(filename)
- assert 'mother' not in base
- base['mother'] = 'snow white'
- assert base.get_str('mother') == 'snow white'
+ assert "mother" not in base
+ base["mother"] = "snow white"
+ assert base.get_str("mother") == "snow white"
@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_node_set_overwrite(datafiles):
- filename = os.path.join(datafiles.dirname,
- datafiles.basename,
- 'basics.yaml')
+ filename = os.path.join(datafiles.dirname, datafiles.basename, "basics.yaml")
base = _yaml.load(filename)
# Overwrite a string
- assert base.get_str('kind') == 'pony'
- base['kind'] = 'cow'
- assert base.get_str('kind') == 'cow'
+ assert base.get_str("kind") == "pony"
+ base["kind"] = "cow"
+ assert base.get_str("kind") == "cow"
# Overwrite a list as a string
- assert base.get_str_list('moods') == ['happy', 'sad']
- base['moods'] = 'unemotional'
- assert base.get_str('moods') == 'unemotional'
+ assert base.get_str_list("moods") == ["happy", "sad"]
+ base["moods"] = "unemotional"
+ assert base.get_str("moods") == "unemotional"
@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_node_set_list_element(datafiles):
- filename = os.path.join(datafiles.dirname,
- datafiles.basename,
- 'basics.yaml')
+ filename = os.path.join(datafiles.dirname, datafiles.basename, "basics.yaml")
base = _yaml.load(filename)
- assert base.get_str_list('moods') == ['happy', 'sad']
- base.get_sequence('moods')[0] = 'confused'
+ assert base.get_str_list("moods") == ["happy", "sad"]
+ base.get_sequence("moods")[0] = "confused"
- assert base.get_str_list('moods') == ['confused', 'sad']
+ assert base.get_str_list("moods") == ["confused", "sad"]
# Really this is testing _yaml.node_copy(), we want to
@@ -173,26 +150,22 @@ def test_node_set_list_element(datafiles):
@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_composite_preserve_originals(datafiles):
- filename = os.path.join(datafiles.dirname,
- datafiles.basename,
- 'basics.yaml')
- overlayfile = os.path.join(datafiles.dirname,
- datafiles.basename,
- 'composite.yaml')
+ filename = os.path.join(datafiles.dirname, datafiles.basename, "basics.yaml")
+ overlayfile = os.path.join(datafiles.dirname, datafiles.basename, "composite.yaml")
base = _yaml.load(filename)
overlay = _yaml.load(overlayfile)
base_copy = base.clone()
overlay._composite(base_copy)
- copy_extra = base_copy.get_mapping('extra')
- orig_extra = base.get_mapping('extra')
+ copy_extra = base_copy.get_mapping("extra")
+ orig_extra = base.get_mapping("extra")
# Test that the node copy has the overridden value...
- assert copy_extra.get_str('old') == 'override'
+ assert copy_extra.get_str("old") == "override"
# But the original node is not effected by the override.
- assert orig_extra.get_str('old') == 'new'
+ assert orig_extra.get_str("old") == "new"
# Tests for list composition
@@ -210,66 +183,62 @@ def test_composite_preserve_originals(datafiles):
# prov_col: The expected provenance column of "mood"
#
@pytest.mark.datafiles(os.path.join(DATA_DIR))
-@pytest.mark.parametrize("filename,index,length,mood,prov_file,prov_line,prov_col", [
-
- # Test results of compositing with the (<) prepend directive
- ('listprepend.yaml', 0, 9, 'prepended1', 'listprepend.yaml', 5, 10),
- ('listprepend.yaml', 1, 9, 'prepended2', 'listprepend.yaml', 7, 10),
- ('listprepend.yaml', 2, 9, 'silly', 'basics.yaml', 8, 8),
- ('listprepend.yaml', 8, 9, 'sleepy', 'basics.yaml', 20, 8),
-
- # Test results of compositing with the (>) append directive
- ('listappend.yaml', 7, 9, 'appended1', 'listappend.yaml', 5, 10),
- ('listappend.yaml', 8, 9, 'appended2', 'listappend.yaml', 7, 10),
- ('listappend.yaml', 0, 9, 'silly', 'basics.yaml', 8, 8),
- ('listappend.yaml', 6, 9, 'sleepy', 'basics.yaml', 20, 8),
-
- # Test results of compositing with both (<) and (>) directives
- ('listappendprepend.yaml', 0, 11, 'prepended1', 'listappendprepend.yaml', 5, 10),
- ('listappendprepend.yaml', 1, 11, 'prepended2', 'listappendprepend.yaml', 7, 10),
- ('listappendprepend.yaml', 2, 11, 'silly', 'basics.yaml', 8, 8),
- ('listappendprepend.yaml', 8, 11, 'sleepy', 'basics.yaml', 20, 8),
- ('listappendprepend.yaml', 9, 11, 'appended1', 'listappendprepend.yaml', 10, 10),
- ('listappendprepend.yaml', 10, 11, 'appended2', 'listappendprepend.yaml', 12, 10),
-
- # Test results of compositing with the (=) overwrite directive
- ('listoverwrite.yaml', 0, 2, 'overwrite1', 'listoverwrite.yaml', 5, 10),
- ('listoverwrite.yaml', 1, 2, 'overwrite2', 'listoverwrite.yaml', 7, 10),
-
- # Test results of compositing without any directive, implicitly overwriting
- ('implicitoverwrite.yaml', 0, 2, 'overwrite1', 'implicitoverwrite.yaml', 4, 8),
- ('implicitoverwrite.yaml', 1, 2, 'overwrite2', 'implicitoverwrite.yaml', 6, 8),
-])
-def test_list_composition(datafiles, filename, tmpdir,
- index, length, mood,
- prov_file, prov_line, prov_col):
- base_file = os.path.join(datafiles.dirname, datafiles.basename, 'basics.yaml')
+@pytest.mark.parametrize(
+ "filename,index,length,mood,prov_file,prov_line,prov_col",
+ [
+ # Test results of compositing with the (<) prepend directive
+ ("listprepend.yaml", 0, 9, "prepended1", "listprepend.yaml", 5, 10),
+ ("listprepend.yaml", 1, 9, "prepended2", "listprepend.yaml", 7, 10),
+ ("listprepend.yaml", 2, 9, "silly", "basics.yaml", 8, 8),
+ ("listprepend.yaml", 8, 9, "sleepy", "basics.yaml", 20, 8),
+ # Test results of compositing with the (>) append directive
+ ("listappend.yaml", 7, 9, "appended1", "listappend.yaml", 5, 10),
+ ("listappend.yaml", 8, 9, "appended2", "listappend.yaml", 7, 10),
+ ("listappend.yaml", 0, 9, "silly", "basics.yaml", 8, 8),
+ ("listappend.yaml", 6, 9, "sleepy", "basics.yaml", 20, 8),
+ # Test results of compositing with both (<) and (>) directives
+ ("listappendprepend.yaml", 0, 11, "prepended1", "listappendprepend.yaml", 5, 10),
+ ("listappendprepend.yaml", 1, 11, "prepended2", "listappendprepend.yaml", 7, 10),
+ ("listappendprepend.yaml", 2, 11, "silly", "basics.yaml", 8, 8),
+ ("listappendprepend.yaml", 8, 11, "sleepy", "basics.yaml", 20, 8),
+ ("listappendprepend.yaml", 9, 11, "appended1", "listappendprepend.yaml", 10, 10),
+ ("listappendprepend.yaml", 10, 11, "appended2", "listappendprepend.yaml", 12, 10),
+ # Test results of compositing with the (=) overwrite directive
+ ("listoverwrite.yaml", 0, 2, "overwrite1", "listoverwrite.yaml", 5, 10),
+ ("listoverwrite.yaml", 1, 2, "overwrite2", "listoverwrite.yaml", 7, 10),
+ # Test results of compositing without any directive, implicitly overwriting
+ ("implicitoverwrite.yaml", 0, 2, "overwrite1", "implicitoverwrite.yaml", 4, 8),
+ ("implicitoverwrite.yaml", 1, 2, "overwrite2", "implicitoverwrite.yaml", 6, 8),
+ ],
+)
+def test_list_composition(datafiles, filename, tmpdir, index, length, mood, prov_file, prov_line, prov_col):
+ base_file = os.path.join(datafiles.dirname, datafiles.basename, "basics.yaml")
overlay_file = os.path.join(datafiles.dirname, datafiles.basename, filename)
- base = _yaml.load(base_file, 'basics.yaml')
+ base = _yaml.load(base_file, "basics.yaml")
overlay = _yaml.load(overlay_file, shortname=filename)
overlay._composite(base)
- children = base.get_sequence('children')
+ children = base.get_sequence("children")
assert len(children) == length
child = children.mapping_at(index)
- assert child.get_str('mood') == mood
- assert_provenance(prov_file, prov_line, prov_col, child.get_node('mood'))
+ assert child.get_str("mood") == mood
+ assert_provenance(prov_file, prov_line, prov_col, child.get_node("mood"))
# Test that overwriting a list with an empty list works as expected.
@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_list_deletion(datafiles):
- base = os.path.join(datafiles.dirname, datafiles.basename, 'basics.yaml')
- overlay = os.path.join(datafiles.dirname, datafiles.basename, 'listoverwriteempty.yaml')
+ base = os.path.join(datafiles.dirname, datafiles.basename, "basics.yaml")
+ overlay = os.path.join(datafiles.dirname, datafiles.basename, "listoverwriteempty.yaml")
- base = _yaml.load(base, shortname='basics.yaml')
- overlay = _yaml.load(overlay, shortname='listoverwriteempty.yaml')
+ base = _yaml.load(base, shortname="basics.yaml")
+ overlay = _yaml.load(overlay, shortname="listoverwriteempty.yaml")
overlay._composite(base)
- children = base.get_sequence('children')
+ children = base.get_sequence("children")
assert not children
@@ -300,109 +269,103 @@ def test_list_deletion(datafiles):
# prov_col: The expected provenance column of "mood"
#
@pytest.mark.datafiles(os.path.join(DATA_DIR))
-@pytest.mark.parametrize("filename1,filename2,index,length,mood,prov_file,prov_line,prov_col", [
-
- # Test results of compositing literal list with (>) and then (<)
- ('listprepend.yaml', 'listappend.yaml', 0, 11, 'prepended1', 'listprepend.yaml', 5, 10),
- ('listprepend.yaml', 'listappend.yaml', 1, 11, 'prepended2', 'listprepend.yaml', 7, 10),
- ('listprepend.yaml', 'listappend.yaml', 2, 11, 'silly', 'basics.yaml', 8, 8),
- ('listprepend.yaml', 'listappend.yaml', 8, 11, 'sleepy', 'basics.yaml', 20, 8),
- ('listprepend.yaml', 'listappend.yaml', 9, 11, 'appended1', 'listappend.yaml', 5, 10),
- ('listprepend.yaml', 'listappend.yaml', 10, 11, 'appended2', 'listappend.yaml', 7, 10),
-
- # Test results of compositing literal list with (<) and then (>)
- ('listappend.yaml', 'listprepend.yaml', 0, 11, 'prepended1', 'listprepend.yaml', 5, 10),
- ('listappend.yaml', 'listprepend.yaml', 1, 11, 'prepended2', 'listprepend.yaml', 7, 10),
- ('listappend.yaml', 'listprepend.yaml', 2, 11, 'silly', 'basics.yaml', 8, 8),
- ('listappend.yaml', 'listprepend.yaml', 8, 11, 'sleepy', 'basics.yaml', 20, 8),
- ('listappend.yaml', 'listprepend.yaml', 9, 11, 'appended1', 'listappend.yaml', 5, 10),
- ('listappend.yaml', 'listprepend.yaml', 10, 11, 'appended2', 'listappend.yaml', 7, 10),
-
- # Test results of compositing literal list with (>) and then (>)
- ('listappend.yaml', 'secondappend.yaml', 0, 11, 'silly', 'basics.yaml', 8, 8),
- ('listappend.yaml', 'secondappend.yaml', 6, 11, 'sleepy', 'basics.yaml', 20, 8),
- ('listappend.yaml', 'secondappend.yaml', 7, 11, 'appended1', 'listappend.yaml', 5, 10),
- ('listappend.yaml', 'secondappend.yaml', 8, 11, 'appended2', 'listappend.yaml', 7, 10),
- ('listappend.yaml', 'secondappend.yaml', 9, 11, 'secondappend1', 'secondappend.yaml', 5, 10),
- ('listappend.yaml', 'secondappend.yaml', 10, 11, 'secondappend2', 'secondappend.yaml', 7, 10),
-
- # Test results of compositing literal list with (>) and then (>)
- ('listprepend.yaml', 'secondprepend.yaml', 0, 11, 'secondprepend1', 'secondprepend.yaml', 5, 10),
- ('listprepend.yaml', 'secondprepend.yaml', 1, 11, 'secondprepend2', 'secondprepend.yaml', 7, 10),
- ('listprepend.yaml', 'secondprepend.yaml', 2, 11, 'prepended1', 'listprepend.yaml', 5, 10),
- ('listprepend.yaml', 'secondprepend.yaml', 3, 11, 'prepended2', 'listprepend.yaml', 7, 10),
- ('listprepend.yaml', 'secondprepend.yaml', 4, 11, 'silly', 'basics.yaml', 8, 8),
- ('listprepend.yaml', 'secondprepend.yaml', 10, 11, 'sleepy', 'basics.yaml', 20, 8),
-
- # Test results of compositing literal list with (>) or (<) and then another literal list
- ('listappend.yaml', 'implicitoverwrite.yaml', 0, 2, 'overwrite1', 'implicitoverwrite.yaml', 4, 8),
- ('listappend.yaml', 'implicitoverwrite.yaml', 1, 2, 'overwrite2', 'implicitoverwrite.yaml', 6, 8),
- ('listprepend.yaml', 'implicitoverwrite.yaml', 0, 2, 'overwrite1', 'implicitoverwrite.yaml', 4, 8),
- ('listprepend.yaml', 'implicitoverwrite.yaml', 1, 2, 'overwrite2', 'implicitoverwrite.yaml', 6, 8),
-
- # Test results of compositing literal list with (>) or (<) and then an explicit (=) overwrite
- ('listappend.yaml', 'listoverwrite.yaml', 0, 2, 'overwrite1', 'listoverwrite.yaml', 5, 10),
- ('listappend.yaml', 'listoverwrite.yaml', 1, 2, 'overwrite2', 'listoverwrite.yaml', 7, 10),
- ('listprepend.yaml', 'listoverwrite.yaml', 0, 2, 'overwrite1', 'listoverwrite.yaml', 5, 10),
- ('listprepend.yaml', 'listoverwrite.yaml', 1, 2, 'overwrite2', 'listoverwrite.yaml', 7, 10),
-
- # Test results of compositing literal list an explicit overwrite (=) and then with (>) or (<)
- ('listoverwrite.yaml', 'listappend.yaml', 0, 4, 'overwrite1', 'listoverwrite.yaml', 5, 10),
- ('listoverwrite.yaml', 'listappend.yaml', 1, 4, 'overwrite2', 'listoverwrite.yaml', 7, 10),
- ('listoverwrite.yaml', 'listappend.yaml', 2, 4, 'appended1', 'listappend.yaml', 5, 10),
- ('listoverwrite.yaml', 'listappend.yaml', 3, 4, 'appended2', 'listappend.yaml', 7, 10),
- ('listoverwrite.yaml', 'listprepend.yaml', 0, 4, 'prepended1', 'listprepend.yaml', 5, 10),
- ('listoverwrite.yaml', 'listprepend.yaml', 1, 4, 'prepended2', 'listprepend.yaml', 7, 10),
- ('listoverwrite.yaml', 'listprepend.yaml', 2, 4, 'overwrite1', 'listoverwrite.yaml', 5, 10),
- ('listoverwrite.yaml', 'listprepend.yaml', 3, 4, 'overwrite2', 'listoverwrite.yaml', 7, 10),
-])
-def test_list_composition_twice(datafiles, tmpdir, filename1, filename2,
- index, length, mood,
- prov_file, prov_line, prov_col):
- file_base = os.path.join(datafiles.dirname, datafiles.basename, 'basics.yaml')
+@pytest.mark.parametrize(
+ "filename1,filename2,index,length,mood,prov_file,prov_line,prov_col",
+ [
+ # Test results of compositing literal list with (>) and then (<)
+ ("listprepend.yaml", "listappend.yaml", 0, 11, "prepended1", "listprepend.yaml", 5, 10),
+ ("listprepend.yaml", "listappend.yaml", 1, 11, "prepended2", "listprepend.yaml", 7, 10),
+ ("listprepend.yaml", "listappend.yaml", 2, 11, "silly", "basics.yaml", 8, 8),
+ ("listprepend.yaml", "listappend.yaml", 8, 11, "sleepy", "basics.yaml", 20, 8),
+ ("listprepend.yaml", "listappend.yaml", 9, 11, "appended1", "listappend.yaml", 5, 10),
+ ("listprepend.yaml", "listappend.yaml", 10, 11, "appended2", "listappend.yaml", 7, 10),
+ # Test results of compositing literal list with (<) and then (>)
+ ("listappend.yaml", "listprepend.yaml", 0, 11, "prepended1", "listprepend.yaml", 5, 10),
+ ("listappend.yaml", "listprepend.yaml", 1, 11, "prepended2", "listprepend.yaml", 7, 10),
+ ("listappend.yaml", "listprepend.yaml", 2, 11, "silly", "basics.yaml", 8, 8),
+ ("listappend.yaml", "listprepend.yaml", 8, 11, "sleepy", "basics.yaml", 20, 8),
+ ("listappend.yaml", "listprepend.yaml", 9, 11, "appended1", "listappend.yaml", 5, 10),
+ ("listappend.yaml", "listprepend.yaml", 10, 11, "appended2", "listappend.yaml", 7, 10),
+ # Test results of compositing literal list with (>) and then (>)
+ ("listappend.yaml", "secondappend.yaml", 0, 11, "silly", "basics.yaml", 8, 8),
+ ("listappend.yaml", "secondappend.yaml", 6, 11, "sleepy", "basics.yaml", 20, 8),
+ ("listappend.yaml", "secondappend.yaml", 7, 11, "appended1", "listappend.yaml", 5, 10),
+ ("listappend.yaml", "secondappend.yaml", 8, 11, "appended2", "listappend.yaml", 7, 10),
+ ("listappend.yaml", "secondappend.yaml", 9, 11, "secondappend1", "secondappend.yaml", 5, 10),
+ ("listappend.yaml", "secondappend.yaml", 10, 11, "secondappend2", "secondappend.yaml", 7, 10),
+ # Test results of compositing literal list with (>) and then (>)
+ ("listprepend.yaml", "secondprepend.yaml", 0, 11, "secondprepend1", "secondprepend.yaml", 5, 10),
+ ("listprepend.yaml", "secondprepend.yaml", 1, 11, "secondprepend2", "secondprepend.yaml", 7, 10),
+ ("listprepend.yaml", "secondprepend.yaml", 2, 11, "prepended1", "listprepend.yaml", 5, 10),
+ ("listprepend.yaml", "secondprepend.yaml", 3, 11, "prepended2", "listprepend.yaml", 7, 10),
+ ("listprepend.yaml", "secondprepend.yaml", 4, 11, "silly", "basics.yaml", 8, 8),
+ ("listprepend.yaml", "secondprepend.yaml", 10, 11, "sleepy", "basics.yaml", 20, 8),
+ # Test results of compositing literal list with (>) or (<) and then another literal list
+ ("listappend.yaml", "implicitoverwrite.yaml", 0, 2, "overwrite1", "implicitoverwrite.yaml", 4, 8),
+ ("listappend.yaml", "implicitoverwrite.yaml", 1, 2, "overwrite2", "implicitoverwrite.yaml", 6, 8),
+ ("listprepend.yaml", "implicitoverwrite.yaml", 0, 2, "overwrite1", "implicitoverwrite.yaml", 4, 8),
+ ("listprepend.yaml", "implicitoverwrite.yaml", 1, 2, "overwrite2", "implicitoverwrite.yaml", 6, 8),
+ # Test results of compositing literal list with (>) or (<) and then an explicit (=) overwrite
+ ("listappend.yaml", "listoverwrite.yaml", 0, 2, "overwrite1", "listoverwrite.yaml", 5, 10),
+ ("listappend.yaml", "listoverwrite.yaml", 1, 2, "overwrite2", "listoverwrite.yaml", 7, 10),
+ ("listprepend.yaml", "listoverwrite.yaml", 0, 2, "overwrite1", "listoverwrite.yaml", 5, 10),
+ ("listprepend.yaml", "listoverwrite.yaml", 1, 2, "overwrite2", "listoverwrite.yaml", 7, 10),
+ # Test results of compositing literal list an explicit overwrite (=) and then with (>) or (<)
+ ("listoverwrite.yaml", "listappend.yaml", 0, 4, "overwrite1", "listoverwrite.yaml", 5, 10),
+ ("listoverwrite.yaml", "listappend.yaml", 1, 4, "overwrite2", "listoverwrite.yaml", 7, 10),
+ ("listoverwrite.yaml", "listappend.yaml", 2, 4, "appended1", "listappend.yaml", 5, 10),
+ ("listoverwrite.yaml", "listappend.yaml", 3, 4, "appended2", "listappend.yaml", 7, 10),
+ ("listoverwrite.yaml", "listprepend.yaml", 0, 4, "prepended1", "listprepend.yaml", 5, 10),
+ ("listoverwrite.yaml", "listprepend.yaml", 1, 4, "prepended2", "listprepend.yaml", 7, 10),
+ ("listoverwrite.yaml", "listprepend.yaml", 2, 4, "overwrite1", "listoverwrite.yaml", 5, 10),
+ ("listoverwrite.yaml", "listprepend.yaml", 3, 4, "overwrite2", "listoverwrite.yaml", 7, 10),
+ ],
+)
+def test_list_composition_twice(
+ datafiles, tmpdir, filename1, filename2, index, length, mood, prov_file, prov_line, prov_col
+):
+ file_base = os.path.join(datafiles.dirname, datafiles.basename, "basics.yaml")
file1 = os.path.join(datafiles.dirname, datafiles.basename, filename1)
file2 = os.path.join(datafiles.dirname, datafiles.basename, filename2)
#####################
# Round 1 - Fight !
#####################
- base = _yaml.load(file_base, shortname='basics.yaml')
+ base = _yaml.load(file_base, shortname="basics.yaml")
overlay1 = _yaml.load(file1, shortname=filename1)
overlay2 = _yaml.load(file2, shortname=filename2)
overlay1._composite(base)
overlay2._composite(base)
- children = base.get_sequence('children')
+ children = base.get_sequence("children")
assert len(children) == length
child = children.mapping_at(index)
- assert child.get_str('mood') == mood
- assert_provenance(prov_file, prov_line, prov_col, child.get_node('mood'))
+ assert child.get_str("mood") == mood
+ assert_provenance(prov_file, prov_line, prov_col, child.get_node("mood"))
#####################
# Round 2 - Fight !
#####################
- base = _yaml.load(file_base, shortname='basics.yaml')
+ base = _yaml.load(file_base, shortname="basics.yaml")
overlay1 = _yaml.load(file1, shortname=filename1)
overlay2 = _yaml.load(file2, shortname=filename2)
overlay2._composite(overlay1)
overlay1._composite(base)
- children = base.get_sequence('children')
+ children = base.get_sequence("children")
assert len(children) == length
child = children.mapping_at(index)
- assert child.get_str('mood') == mood
- assert_provenance(prov_file, prov_line, prov_col, child.get_node('mood'))
+ assert child.get_str("mood") == mood
+ assert_provenance(prov_file, prov_line, prov_col, child.get_node("mood"))
@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_convert_value_to_string(datafiles):
- conf_file = os.path.join(datafiles.dirname,
- datafiles.basename,
- 'convert_value_to_str.yaml')
+ conf_file = os.path.join(datafiles.dirname, datafiles.basename, "convert_value_to_str.yaml")
# Run file through yaml to convert it
test_dict = _yaml.load(conf_file)
@@ -426,9 +389,7 @@ def test_convert_value_to_string(datafiles):
@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_value_doesnt_match_expected(datafiles):
- conf_file = os.path.join(datafiles.dirname,
- datafiles.basename,
- 'convert_value_to_str.yaml')
+ conf_file = os.path.join(datafiles.dirname, datafiles.basename, "convert_value_to_str.yaml")
# Run file through yaml to convert it
test_dict = _yaml.load(conf_file)
@@ -439,11 +400,9 @@ def test_value_doesnt_match_expected(datafiles):
@pytest.mark.datafiles(os.path.join(DATA_DIR))
-@pytest.mark.parametrize('fromdisk', [(True), (False)])
+@pytest.mark.parametrize("fromdisk", [(True), (False)])
def test_roundtrip_dump(datafiles, fromdisk):
- filename = os.path.join(datafiles.dirname,
- datafiles.basename,
- "roundtrip-test.yaml")
+ filename = os.path.join(datafiles.dirname, datafiles.basename, "roundtrip-test.yaml")
with open(filename, "r") as fh:
rt_raw = fh.read()
if fromdisk:
@@ -480,16 +439,9 @@ def test_roundtrip_dump(datafiles, fromdisk):
@pytest.mark.datafiles(os.path.join(DATA_DIR))
-@pytest.mark.parametrize('case', [
- ['a', 'b', 'c'],
- ['foo', 1],
- ['stuff', 0, 'colour'],
- ['bird', 0, 1],
-])
+@pytest.mark.parametrize("case", [["a", "b", "c"], ["foo", 1], ["stuff", 0, "colour"], ["bird", 0, 1],])
def test_node_find_target(datafiles, case):
- filename = os.path.join(datafiles.dirname,
- datafiles.basename,
- "traversal.yaml")
+ filename = os.path.join(datafiles.dirname, datafiles.basename, "traversal.yaml")
# We set copy_tree in order to ensure that the nodes in `loaded`
# are not the same nodes as in `prov.toplevel`
loaded = _yaml.load(filename, copy_tree=True)
@@ -523,9 +475,7 @@ def test_node_find_target(datafiles, case):
@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_node_find_target_fails(datafiles):
- filename = os.path.join(datafiles.dirname,
- datafiles.basename,
- "traversal.yaml")
+ filename = os.path.join(datafiles.dirname, datafiles.basename, "traversal.yaml")
loaded = _yaml.load(filename, copy_tree=True)
brand_new = Node.from_dict({})
diff --git a/tests/plugins/deprecationwarnings/deprecationwarnings.py b/tests/plugins/deprecationwarnings/deprecationwarnings.py
index 4d2d22c05..628faea68 100644
--- a/tests/plugins/deprecationwarnings/deprecationwarnings.py
+++ b/tests/plugins/deprecationwarnings/deprecationwarnings.py
@@ -8,10 +8,7 @@ import pytest
from buildstream.testing import cli # pylint: disable=unused-import
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project"
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
_DEPRECATION_MESSAGE = "Here is some detail."
_DEPRECATION_WARNING = "Using deprecated plugin deprecated_plugin: {}".format(_DEPRECATION_MESSAGE)
@@ -20,7 +17,7 @@ _DEPRECATION_WARNING = "Using deprecated plugin deprecated_plugin: {}".format(_D
@pytest.mark.datafiles(DATA_DIR)
def test_deprecation_warning_present(cli, datafiles):
project = str(datafiles)
- result = cli.run(project=project, args=['show', 'deprecated.bst'])
+ result = cli.run(project=project, args=["show", "deprecated.bst"])
result.assert_success()
assert _DEPRECATION_WARNING in result.stderr
@@ -28,16 +25,14 @@ def test_deprecation_warning_present(cli, datafiles):
@pytest.mark.datafiles(DATA_DIR)
def test_suppress_deprecation_warning(cli, datafiles):
project = str(datafiles)
- cli.run(project=project, args=['show', 'manual.bst'])
+ cli.run(project=project, args=["show", "manual.bst"])
- element_overrides = "elements:\n" \
- " deprecated_plugin:\n" \
- " suppress-deprecation-warnings : True\n"
+ element_overrides = "elements:\n" " deprecated_plugin:\n" " suppress-deprecation-warnings : True\n"
- project_conf = os.path.join(project, 'project.conf')
- with open(project_conf, 'a') as f:
+ project_conf = os.path.join(project, "project.conf")
+ with open(project_conf, "a") as f:
f.write(element_overrides)
- result = cli.run(project=project, args=['show', 'deprecated.bst'])
+ result = cli.run(project=project, args=["show", "deprecated.bst"])
result.assert_success()
assert _DEPRECATION_WARNING not in result.stderr
diff --git a/tests/remoteexecution/buildfail.py b/tests/remoteexecution/buildfail.py
index 0fb4cdb95..22d9c825f 100644
--- a/tests/remoteexecution/buildfail.py
+++ b/tests/remoteexecution/buildfail.py
@@ -28,48 +28,33 @@ from buildstream.testing import cli_remote_execution as cli # pylint: disable=u
pytestmark = pytest.mark.remoteexecution
# Project directory
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project",
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",)
@pytest.mark.datafiles(DATA_DIR)
def test_build_remote_failure(cli, datafiles):
project = str(datafiles)
- element_path = os.path.join(project, 'elements', 'element.bst')
- checkout_path = os.path.join(cli.directory, 'checkout')
+ element_path = os.path.join(project, "elements", "element.bst")
+ checkout_path = os.path.join(cli.directory, "checkout")
# Write out our test target
element = {
- 'kind': 'script',
- 'depends': [
- {
- 'filename': 'base.bst',
- 'type': 'build',
- },
- ],
- 'config': {
- 'commands': [
- 'touch %{install-root}/foo',
- 'false',
- ],
- },
+ "kind": "script",
+ "depends": [{"filename": "base.bst", "type": "build",},],
+ "config": {"commands": ["touch %{install-root}/foo", "false",],},
}
_yaml.roundtrip_dump(element, element_path)
services = cli.ensure_services()
- assert set(services) == set(['action-cache', 'execution', 'storage'])
+ assert set(services) == set(["action-cache", "execution", "storage"])
# Try to build it, this should result in a failure that contains the content
- result = cli.run(project=project, args=['build', 'element.bst'])
+ result = cli.run(project=project, args=["build", "element.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
- result = cli.run(project=project, args=[
- 'artifact', 'checkout', 'element.bst', '--directory', checkout_path
- ])
+ result = cli.run(project=project, args=["artifact", "checkout", "element.bst", "--directory", checkout_path])
result.assert_success()
# check that the file created before the failure exists
- filename = os.path.join(checkout_path, 'foo')
+ filename = os.path.join(checkout_path, "foo")
assert os.path.isfile(filename)
diff --git a/tests/remoteexecution/buildtree.py b/tests/remoteexecution/buildtree.py
index a64b8716c..57e25cd14 100644
--- a/tests/remoteexecution/buildtree.py
+++ b/tests/remoteexecution/buildtree.py
@@ -28,54 +28,43 @@ from tests.testutils import create_artifact_share
pytestmark = pytest.mark.remoteexecution
# Project directory
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project",
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",)
@pytest.mark.datafiles(DATA_DIR)
def test_buildtree_remote(cli, tmpdir, datafiles):
project = str(datafiles)
- element_name = 'build-shell/buildtree.bst'
- share_path = os.path.join(str(tmpdir), 'share')
+ element_name = "build-shell/buildtree.bst"
+ share_path = os.path.join(str(tmpdir), "share")
services = cli.ensure_services()
- assert set(services) == set(['action-cache', 'execution', 'storage'])
+ assert set(services) == set(["action-cache", "execution", "storage"])
with create_artifact_share(share_path) as share:
- cli.configure({
- 'artifacts': {'url': share.repo, 'push': True},
- 'cache': {'pull-buildtrees': False}
- })
+ cli.configure({"artifacts": {"url": share.repo, "push": True}, "cache": {"pull-buildtrees": False}})
- res = cli.run(project=project, args=[
- '--cache-buildtrees', 'always', 'build', element_name])
+ res = cli.run(project=project, args=["--cache-buildtrees", "always", "build", element_name])
res.assert_success()
# remove local cache
- shutil.rmtree(os.path.join(str(tmpdir), 'cache', 'cas'))
- shutil.rmtree(os.path.join(str(tmpdir), 'cache', 'artifacts'))
+ shutil.rmtree(os.path.join(str(tmpdir), "cache", "cas"))
+ shutil.rmtree(os.path.join(str(tmpdir), "cache", "artifacts"))
# pull without buildtree
- res = cli.run(project=project, args=[
- 'artifact', 'pull', '--deps', 'all', element_name])
+ res = cli.run(project=project, args=["artifact", "pull", "--deps", "all", element_name])
res.assert_success()
# check shell doesn't work
- res = cli.run(project=project, args=[
- 'shell', '--build', element_name, '--', 'cat', 'test'
- ])
+ res = cli.run(project=project, args=["shell", "--build", element_name, "--", "cat", "test"])
res.assert_shell_error()
# pull with buildtree
- res = cli.run(project=project, args=[
- '--pull-buildtrees', 'artifact', 'pull', '--deps', 'all', element_name])
+ res = cli.run(project=project, args=["--pull-buildtrees", "artifact", "pull", "--deps", "all", element_name])
res.assert_success()
# check it works this time
- res = cli.run(project=project, args=[
- 'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test'
- ])
+ res = cli.run(
+ project=project, args=["shell", "--build", element_name, "--use-buildtree", "always", "--", "cat", "test"]
+ )
res.assert_success()
assert "Hi" in res.output
diff --git a/tests/remoteexecution/junction.py b/tests/remoteexecution/junction.py
index db087fd90..2b0261612 100644
--- a/tests/remoteexecution/junction.py
+++ b/tests/remoteexecution/junction.py
@@ -28,92 +28,70 @@ from tests.testutils import generate_junction
pytestmark = pytest.mark.remoteexecution
# Project directory
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project",
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",)
def configure_project(path, config):
- config['name'] = 'test'
- config['element-path'] = 'elements'
- _yaml.roundtrip_dump(config, os.path.join(path, 'project.conf'))
+ config["name"] = "test"
+ config["element-path"] = "elements"
+ _yaml.roundtrip_dump(config, os.path.join(path, "project.conf"))
def create_element(repo, name, path, dependencies, ref=None):
- element = {
- 'kind': 'import',
- 'sources': [
- repo.source_config(ref=ref)
- ],
- 'depends': dependencies
- }
+ element = {"kind": "import", "sources": [repo.source_config(ref=ref)], "depends": dependencies}
_yaml.roundtrip_dump(element, os.path.join(path, name))
@pytest.mark.datafiles(DATA_DIR)
def test_junction_build_remote(cli, tmpdir, datafiles):
project = str(datafiles)
- subproject_path = os.path.join(project, 'files', 'sub-project')
- subproject_element_path = os.path.join(subproject_path, 'elements')
- amhello_files_path = os.path.join(subproject_path, 'files')
- element_path = os.path.join(project, 'elements')
- junction_path = os.path.join(element_path, 'junction.bst')
+ subproject_path = os.path.join(project, "files", "sub-project")
+ subproject_element_path = os.path.join(subproject_path, "elements")
+ amhello_files_path = os.path.join(subproject_path, "files")
+ element_path = os.path.join(project, "elements")
+ junction_path = os.path.join(element_path, "junction.bst")
# We need a repo for real trackable elements
- repo = create_repo('git', str(tmpdir))
+ repo = create_repo("git", str(tmpdir))
ref = repo.create(amhello_files_path)
# ensure that the correct project directory is also listed in the junction
- subproject_conf = os.path.join(subproject_path, 'project.conf')
+ subproject_conf = os.path.join(subproject_path, "project.conf")
with open(subproject_conf) as f:
config = f.read()
config = config.format(project_dir=subproject_path)
- with open(subproject_conf, 'w') as f:
+ with open(subproject_conf, "w") as f:
f.write(config)
# Create a trackable element to depend on the cross junction element,
# this one has it's ref resolved already
- create_element(repo, 'sub-target.bst', subproject_element_path, ['autotools/amhello.bst'], ref=ref)
+ create_element(repo, "sub-target.bst", subproject_element_path, ["autotools/amhello.bst"], ref=ref)
# Create a trackable element to depend on the cross junction element
- create_element(repo, 'target.bst', element_path, [
- {
- 'junction': 'junction.bst',
- 'filename': 'sub-target.bst'
- }
- ])
+ create_element(repo, "target.bst", element_path, [{"junction": "junction.bst", "filename": "sub-target.bst"}])
# Create a repo to hold the subproject and generate a junction element for it
generate_junction(tmpdir, subproject_path, junction_path, store_ref=False)
# Now create a compose element at the top level
- element = {
- 'kind': 'compose',
- 'depends': [
- {
- 'filename': 'target.bst',
- 'type': 'build'
- }
- ]
- }
- _yaml.roundtrip_dump(element, os.path.join(element_path, 'composed.bst'))
+ element = {"kind": "compose", "depends": [{"filename": "target.bst", "type": "build"}]}
+ _yaml.roundtrip_dump(element, os.path.join(element_path, "composed.bst"))
# We're doing remote execution so ensure services are available
services = cli.ensure_services()
- assert set(services) == set(['action-cache', 'execution', 'storage'])
+ assert set(services) == set(["action-cache", "execution", "storage"])
# track the junction first to ensure we have refs
- result = cli.run(project=project, args=['source', 'track', 'junction.bst'])
+ result = cli.run(project=project, args=["source", "track", "junction.bst"])
result.assert_success()
# track target to ensure we have refs
- result = cli.run(project=project, args=['source', 'track', '--deps', 'all', 'composed.bst'])
+ result = cli.run(project=project, args=["source", "track", "--deps", "all", "composed.bst"])
result.assert_success()
# build
- result = cli.run(project=project, silent=True, args=['build', 'composed.bst'])
+ result = cli.run(project=project, silent=True, args=["build", "composed.bst"])
result.assert_success()
# Assert that the main target is cached as a result
- assert cli.get_element_state(project, 'composed.bst') == 'cached'
+ assert cli.get_element_state(project, "composed.bst") == "cached"
diff --git a/tests/remoteexecution/partial.py b/tests/remoteexecution/partial.py
index a640f27d5..a452d6613 100644
--- a/tests/remoteexecution/partial.py
+++ b/tests/remoteexecution/partial.py
@@ -14,75 +14,68 @@ from tests.testutils.artifactshare import create_artifact_share
pytestmark = pytest.mark.remoteexecution
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project"
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
# Test that `bst build` does not download file blobs of a build-only dependency
# to the local cache.
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize('pull_artifact_files', [True, False])
-@pytest.mark.parametrize('build_all', [True, False])
+@pytest.mark.parametrize("pull_artifact_files", [True, False])
+@pytest.mark.parametrize("build_all", [True, False])
def test_build_dependency_partial_local_cas(cli, datafiles, pull_artifact_files, build_all):
project = str(datafiles)
- element_name = 'no-runtime-deps.bst'
- builddep_element_name = 'autotools/amhello.bst'
- checkout = os.path.join(cli.directory, 'checkout')
- builddep_checkout = os.path.join(cli.directory, 'builddep-checkout')
+ element_name = "no-runtime-deps.bst"
+ builddep_element_name = "autotools/amhello.bst"
+ checkout = os.path.join(cli.directory, "checkout")
+ builddep_checkout = os.path.join(cli.directory, "builddep-checkout")
services = cli.ensure_services()
- assert set(services) == set(['action-cache', 'execution', 'storage'])
+ assert set(services) == set(["action-cache", "execution", "storage"])
# configure pull blobs
if build_all:
- cli.configure({
- 'build': {
- 'dependencies': 'all'
- }
- })
- cli.config['remote-execution']['pull-artifact-files'] = pull_artifact_files
-
- result = cli.run(project=project, args=['build', element_name])
+ cli.configure({"build": {"dependencies": "all"}})
+ cli.config["remote-execution"]["pull-artifact-files"] = pull_artifact_files
+
+ result = cli.run(project=project, args=["build", element_name])
result.assert_success()
# Verify artifact is pulled bar files when ensure artifact files is set
- result = cli.run(project=project, args=['artifact', 'checkout', element_name,
- '--directory', checkout])
+ result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
if pull_artifact_files:
result.assert_success()
- assert_contains(checkout, ['/test'])
+ assert_contains(checkout, ["/test"])
else:
- result.assert_main_error(ErrorDomain.STREAM, 'uncached-checkout-attempt')
+ result.assert_main_error(ErrorDomain.STREAM, "uncached-checkout-attempt")
# Verify build dependencies is pulled for ALL and BUILD
- result = cli.run(project=project, args=['artifact', 'checkout', builddep_element_name,
- '--directory', builddep_checkout])
+ result = cli.run(
+ project=project, args=["artifact", "checkout", builddep_element_name, "--directory", builddep_checkout]
+ )
if build_all and pull_artifact_files:
result.assert_success()
else:
- result.assert_main_error(ErrorDomain.STREAM, 'uncached-checkout-attempt')
+ result.assert_main_error(ErrorDomain.STREAM, "uncached-checkout-attempt")
@pytest.mark.datafiles(DATA_DIR)
def test_build_partial_push(cli, tmpdir, datafiles):
project = str(datafiles)
share_dir = os.path.join(str(tmpdir), "artifactshare")
- element_name = 'no-runtime-deps.bst'
- builddep_element_name = 'autotools/amhello.bst'
+ element_name = "no-runtime-deps.bst"
+ builddep_element_name = "autotools/amhello.bst"
with create_artifact_share(share_dir) as share:
services = cli.ensure_services()
- assert set(services) == set(['action-cache', 'execution', 'storage'])
+ assert set(services) == set(["action-cache", "execution", "storage"])
- cli.config['artifacts'] = {
- 'url': share.repo,
- 'push': True,
+ cli.config["artifacts"] = {
+ "url": share.repo,
+ "push": True,
}
- res = cli.run(project=project, args=['build', element_name])
+ res = cli.run(project=project, args=["build", element_name])
res.assert_success()
assert builddep_element_name in res.get_pushed_elements()
diff --git a/tests/remoteexecution/simple.py b/tests/remoteexecution/simple.py
index 1b7f7818a..cb8f80930 100644
--- a/tests/remoteexecution/simple.py
+++ b/tests/remoteexecution/simple.py
@@ -11,49 +11,54 @@ from buildstream.testing.integration import assert_contains
pytestmark = pytest.mark.remoteexecution
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project"
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
# Test building an executable with remote-execution:
@pytest.mark.datafiles(DATA_DIR)
def test_remote_autotools_build(cli, datafiles):
project = str(datafiles)
- checkout = os.path.join(cli.directory, 'checkout')
- element_name = 'autotools/amhello.bst'
+ checkout = os.path.join(cli.directory, "checkout")
+ element_name = "autotools/amhello.bst"
services = cli.ensure_services()
- assert set(services) == set(['action-cache', 'execution', 'storage'])
+ assert set(services) == set(["action-cache", "execution", "storage"])
- result = cli.run(project=project, args=['build', element_name])
+ result = cli.run(project=project, args=["build", element_name])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout])
+ result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
result.assert_success()
- assert_contains(checkout, ['/usr', '/usr/lib', '/usr/bin',
- '/usr/share',
- '/usr/bin/hello', '/usr/share/doc',
- '/usr/share/doc/amhello',
- '/usr/share/doc/amhello/README'])
+ assert_contains(
+ checkout,
+ [
+ "/usr",
+ "/usr/lib",
+ "/usr/bin",
+ "/usr/share",
+ "/usr/bin/hello",
+ "/usr/share/doc",
+ "/usr/share/doc/amhello",
+ "/usr/share/doc/amhello/README",
+ ],
+ )
# Test running an executable built with remote-execution:
@pytest.mark.datafiles(DATA_DIR)
def test_remote_autotools_run(cli, datafiles):
project = str(datafiles)
- element_name = 'autotools/amhello.bst'
+ element_name = "autotools/amhello.bst"
services = cli.ensure_services()
- assert set(services) == set(['action-cache', 'execution', 'storage'])
+ assert set(services) == set(["action-cache", "execution", "storage"])
services = cli.ensure_services()
- result = cli.run(project=project, args=['build', element_name])
+ result = cli.run(project=project, args=["build", element_name])
result.assert_success()
- result = cli.run(project=project, args=['shell', element_name, '/usr/bin/hello'])
+ result = cli.run(project=project, args=["shell", element_name, "/usr/bin/hello"])
result.assert_success()
- assert result.output == 'Hello World!\nThis is amhello 1.0.\n'
+ assert result.output == "Hello World!\nThis is amhello 1.0.\n"
diff --git a/tests/sandboxes/fallback.py b/tests/sandboxes/fallback.py
index f2f585e70..948e3a6de 100644
--- a/tests/sandboxes/fallback.py
+++ b/tests/sandboxes/fallback.py
@@ -26,51 +26,43 @@ from buildstream.testing import cli # pylint: disable=unused-import
pytestmark = pytest.mark.integration
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project"
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
@pytest.mark.datafiles(DATA_DIR)
def test_fallback_platform_fails(cli, datafiles):
project = str(datafiles)
- element_path = os.path.join(project, 'elements', 'element.bst')
+ element_path = os.path.join(project, "elements", "element.bst")
# Write out our test target
element = {
- 'kind': 'script',
- 'depends': [
- {
- 'filename': 'base.bst',
- 'type': 'build',
- },
- ],
- 'config': {
- 'commands': [
- 'true',
- ],
- },
+ "kind": "script",
+ "depends": [{"filename": "base.bst", "type": "build",},],
+ "config": {"commands": ["true",],},
}
_yaml.roundtrip_dump(element, element_path)
- result = cli.run(project=project, args=['build', 'element.bst'],
- env={'BST_FORCE_BACKEND': 'fallback',
- 'BST_FORCE_SANDBOX': None})
+ result = cli.run(
+ project=project,
+ args=["build", "element.bst"],
+ env={"BST_FORCE_BACKEND": "fallback", "BST_FORCE_SANDBOX": None},
+ )
result.assert_main_error(ErrorDomain.STREAM, None)
assert "FallBack platform only implements dummy sandbox" in result.stderr
# The dummy sandbox can not build the element but it can get the element read
# There for the element should be `buildable` rather than `waiting`
- assert cli.get_element_state(project, 'element.bst') == 'buildable'
+ assert cli.get_element_state(project, "element.bst") == "buildable"
@pytest.mark.datafiles(DATA_DIR)
def test_fallback_platform_can_use_dummy(cli, datafiles):
project = str(datafiles)
- result = cli.run(project=project, args=['build', 'import-file1.bst'],
- env={'BST_FORCE_BACKEND': 'fallback',
- 'BST_FORCE_SANDBOX': None})
+ result = cli.run(
+ project=project,
+ args=["build", "import-file1.bst"],
+ env={"BST_FORCE_BACKEND": "fallback", "BST_FORCE_SANDBOX": None},
+ )
result.assert_success()
# The fallback platform can still provide a dummy sandbox that alows simple elemnts that do not need
# a full sandbox to still be built on new platforms.
diff --git a/tests/sandboxes/missing-command.py b/tests/sandboxes/missing-command.py
index 171e855f7..87e668966 100644
--- a/tests/sandboxes/missing-command.py
+++ b/tests/sandboxes/missing-command.py
@@ -9,15 +9,12 @@ from buildstream._exceptions import ErrorDomain
from buildstream.testing import cli # pylint: disable=unused-import
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "missing-command"
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "missing-command")
@pytest.mark.datafiles(DATA_DIR)
def test_missing_command(cli, datafiles):
project = str(datafiles)
- result = cli.run(project=project, args=['build', 'no-runtime.bst'])
- result.assert_task_error(ErrorDomain.SANDBOX, 'missing-command')
- assert cli.get_element_state(project, 'no-runtime.bst') == 'failed'
+ result = cli.run(project=project, args=["build", "no-runtime.bst"])
+ result.assert_task_error(ErrorDomain.SANDBOX, "missing-command")
+ assert cli.get_element_state(project, "no-runtime.bst") == "failed"
diff --git a/tests/sandboxes/missing_dependencies.py b/tests/sandboxes/missing_dependencies.py
index a5bf31e76..722cfc647 100644
--- a/tests/sandboxes/missing_dependencies.py
+++ b/tests/sandboxes/missing_dependencies.py
@@ -12,10 +12,7 @@ from buildstream.testing import cli # pylint: disable=unused-import
# Project directory
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "missing-dependencies",
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "missing-dependencies",)
def _symlink_host_tools_to_dir(host_tools, dir_):
@@ -25,85 +22,67 @@ def _symlink_host_tools_to_dir(host_tools, dir_):
os.symlink(utils.get_host_tool(tool), str(target_path))
-@pytest.mark.skipif(not IS_LINUX, reason='Only available on Linux')
+@pytest.mark.skipif(not IS_LINUX, reason="Only available on Linux")
@pytest.mark.datafiles(DATA_DIR)
def test_missing_bwrap_has_nice_error_message(cli, datafiles, tmp_path):
# Create symlink to buildbox-casd and git to work with custom PATH
bin_dir = tmp_path / "bin"
- _symlink_host_tools_to_dir(['buildbox-casd', 'git'], bin_dir)
+ _symlink_host_tools_to_dir(["buildbox-casd", "git"], bin_dir)
project = str(datafiles)
- element_path = os.path.join(project, 'elements', 'element.bst')
+ element_path = os.path.join(project, "elements", "element.bst")
# Write out our test target
element = {
- 'kind': 'script',
- 'depends': [
- {
- 'filename': 'base.bst',
- 'type': 'build',
- },
- ],
- 'config': {
- 'commands': [
- 'false',
- ],
- },
+ "kind": "script",
+ "depends": [{"filename": "base.bst", "type": "build",},],
+ "config": {"commands": ["false",],},
}
_yaml.roundtrip_dump(element, element_path)
# Build without access to host tools, this should fail with a nice error
result = cli.run(
- project=project,
- args=['build', 'element.bst'],
- env={'PATH': str(bin_dir),
- 'BST_FORCE_SANDBOX': None})
- result.assert_task_error(ErrorDomain.SANDBOX, 'unavailable-local-sandbox')
+ project=project, args=["build", "element.bst"], env={"PATH": str(bin_dir), "BST_FORCE_SANDBOX": None}
+ )
+ result.assert_task_error(ErrorDomain.SANDBOX, "unavailable-local-sandbox")
assert "not found" in result.stderr
-@pytest.mark.skipif(not IS_LINUX, reason='Only available on Linux')
+@pytest.mark.skipif(not IS_LINUX, reason="Only available on Linux")
@pytest.mark.datafiles(DATA_DIR)
def test_old_brwap_has_nice_error_message(cli, datafiles, tmp_path):
- bwrap = tmp_path.joinpath('bin/bwrap')
+ bwrap = tmp_path.joinpath("bin/bwrap")
bwrap.parent.mkdir()
- with bwrap.open('w') as fp:
- fp.write('''
+ with bwrap.open("w") as fp:
+ fp.write(
+ """
#!/bin/sh
echo bubblewrap 0.0.1
- '''.strip())
+ """.strip()
+ )
bwrap.chmod(0o755)
# Create symlink to buildbox-casd and git to work with custom PATH
bin_dir = tmp_path / "bin"
- _symlink_host_tools_to_dir(['buildbox-casd', 'git'], bin_dir)
+ _symlink_host_tools_to_dir(["buildbox-casd", "git"], bin_dir)
project = str(datafiles)
- element_path = os.path.join(project, 'elements', 'element3.bst')
+ element_path = os.path.join(project, "elements", "element3.bst")
# Write out our test target
element = {
- 'kind': 'script',
- 'depends': [
- {
- 'filename': 'base.bst',
- 'type': 'build',
- },
- ],
- 'config': {
- 'commands': [
- 'false',
- ],
- },
+ "kind": "script",
+ "depends": [{"filename": "base.bst", "type": "build",},],
+ "config": {"commands": ["false",],},
}
_yaml.roundtrip_dump(element, element_path)
# Build without access to host tools, this should fail with a nice error
result = cli.run(
project=project,
- args=['--debug', '--verbose', 'build', 'element3.bst'],
- env={'PATH': str(bin_dir),
- 'BST_FORCE_SANDBOX': None})
- result.assert_task_error(ErrorDomain.SANDBOX, 'unavailable-local-sandbox')
+ args=["--debug", "--verbose", "build", "element3.bst"],
+ env={"PATH": str(bin_dir), "BST_FORCE_SANDBOX": None},
+ )
+ result.assert_task_error(ErrorDomain.SANDBOX, "unavailable-local-sandbox")
assert "too old" in result.stderr
diff --git a/tests/sandboxes/mounting/mount_simple.py b/tests/sandboxes/mounting/mount_simple.py
index 65aaf209d..0e78a5603 100644
--- a/tests/sandboxes/mounting/mount_simple.py
+++ b/tests/sandboxes/mounting/mount_simple.py
@@ -13,35 +13,35 @@ def test_bind_mount():
src = stack.enter_context(tempfile.TemporaryDirectory())
target = stack.enter_context(tempfile.TemporaryDirectory())
- with open(os.path.join(src, 'test'), 'a') as test:
- test.write('Test')
+ with open(os.path.join(src, "test"), "a") as test:
+ test.write("Test")
with Mounter.bind_mount(target, src) as dest:
# Ensure we get the correct path back
assert dest == target
# Ensure we can access files from src from target
- with open(os.path.join(target, 'test'), 'r') as test:
- assert test.read() == 'Test'
+ with open(os.path.join(target, "test"), "r") as test:
+ assert test.read() == "Test"
# Ensure the files from src are gone from target
with pytest.raises(FileNotFoundError):
- with open(os.path.join(target, 'test'), 'r'):
+ with open(os.path.join(target, "test"), "r"):
# Actual contents don't matter
pass
# Ensure the files in src are still in src
- with open(os.path.join(src, 'test'), 'r') as test:
- assert test.read() == 'Test'
+ with open(os.path.join(src, "test"), "r") as test:
+ assert test.read() == "Test"
@pytest.mark.skipif(not os.geteuid() == 0, reason="requires root permissions")
def test_mount_proc():
with ExitStack() as stack:
- src = '/proc'
+ src = "/proc"
target = stack.enter_context(tempfile.TemporaryDirectory())
- with Mounter.mount(target, src, mount_type='proc', ro=True) as dest:
+ with Mounter.mount(target, src, mount_type="proc", ro=True) as dest:
# Ensure we get the correct path back
assert dest == target
diff --git a/tests/sandboxes/remote-exec-config.py b/tests/sandboxes/remote-exec-config.py
index a6aeeb7ab..7066ddeab 100644
--- a/tests/sandboxes/remote-exec-config.py
+++ b/tests/sandboxes/remote-exec-config.py
@@ -9,10 +9,7 @@ from buildstream import _yaml
from buildstream._exceptions import ErrorDomain, LoadErrorReason
from buildstream.testing.runcli import cli # pylint: disable=unused-import
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "remote-exec-config"
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "remote-exec-config")
# Tests that we get a useful error message when supplying invalid
# remote execution configurations.
@@ -22,28 +19,23 @@ DATA_DIR = os.path.join(
# are used at once, a LoadError results.
@pytest.mark.datafiles(DATA_DIR)
def test_old_and_new_configs(cli, datafiles):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'missing-certs')
+ project = os.path.join(datafiles.dirname, datafiles.basename, "missing-certs")
project_conf = {
- 'name': 'test',
-
- 'remote-execution': {
- 'url': 'https://cache.example.com:12345',
- 'execution-service': {
- 'url': 'http://localhost:8088'
- },
- 'storage-service': {
- 'url': 'http://charactron:11001',
- }
- }
+ "name": "test",
+ "remote-execution": {
+ "url": "https://cache.example.com:12345",
+ "execution-service": {"url": "http://localhost:8088"},
+ "storage-service": {"url": "http://charactron:11001",},
+ },
}
- project_conf_file = os.path.join(project, 'project.conf')
+ project_conf_file = os.path.join(project, "project.conf")
_yaml.roundtrip_dump(project_conf, project_conf_file)
# Use `pull` here to ensure we try to initialize the remotes, triggering the error
#
# This does not happen for a simple `bst show`.
- result = cli.run(project=project, args=['artifact', 'pull', 'element.bst'])
+ result = cli.run(project=project, args=["artifact", "pull", "element.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA, "specify one")
@@ -51,52 +43,38 @@ def test_old_and_new_configs(cli, datafiles):
# without specifying its counterpart, we get a comprehensive LoadError
# instead of an unhandled exception.
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize('config_key, config_value', [
- ('client-cert', 'client.crt'),
- ('client-key', 'client.key')
-])
+@pytest.mark.parametrize("config_key, config_value", [("client-cert", "client.crt"), ("client-key", "client.key")])
def test_missing_certs(cli, datafiles, config_key, config_value):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'missing-certs')
+ project = os.path.join(datafiles.dirname, datafiles.basename, "missing-certs")
project_conf = {
- 'name': 'test',
-
- 'remote-execution': {
- 'execution-service': {
- 'url': 'http://localhost:8088'
- },
- 'storage-service': {
- 'url': 'http://charactron:11001',
- config_key: config_value,
- }
- }
+ "name": "test",
+ "remote-execution": {
+ "execution-service": {"url": "http://localhost:8088"},
+ "storage-service": {"url": "http://charactron:11001", config_key: config_value,},
+ },
}
- project_conf_file = os.path.join(project, 'project.conf')
+ project_conf_file = os.path.join(project, "project.conf")
_yaml.roundtrip_dump(project_conf, project_conf_file)
# Use `pull` here to ensure we try to initialize the remotes, triggering the error
#
# This does not happen for a simple `bst show`.
- result = cli.run(project=project, args=['show', 'element.bst'])
+ result = cli.run(project=project, args=["show", "element.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA, "Your config is missing")
# Assert that if incomplete information is supplied we get a sensible error message.
@pytest.mark.datafiles(DATA_DIR)
def test_empty_config(cli, datafiles):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'missing-certs')
+ project = os.path.join(datafiles.dirname, datafiles.basename, "missing-certs")
- project_conf = {
- 'name': 'test',
-
- 'remote-execution': {
- }
- }
- project_conf_file = os.path.join(project, 'project.conf')
+ project_conf = {"name": "test", "remote-execution": {}}
+ project_conf_file = os.path.join(project, "project.conf")
_yaml.roundtrip_dump(project_conf, project_conf_file)
# Use `pull` here to ensure we try to initialize the remotes, triggering the error
#
# This does not happen for a simple `bst show`.
- result = cli.run(project=project, args=['artifact', 'pull', 'element.bst'])
+ result = cli.run(project=project, args=["artifact", "pull", "element.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA, "specify one")
diff --git a/tests/sandboxes/selection.py b/tests/sandboxes/selection.py
index b4bbb1b00..a338fe0bb 100644
--- a/tests/sandboxes/selection.py
+++ b/tests/sandboxes/selection.py
@@ -26,79 +26,58 @@ from buildstream.testing import cli # pylint: disable=unused-import
pytestmark = pytest.mark.integration
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project"
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
@pytest.mark.datafiles(DATA_DIR)
def test_force_sandbox(cli, datafiles):
project = str(datafiles)
- element_path = os.path.join(project, 'elements', 'element.bst')
+ element_path = os.path.join(project, "elements", "element.bst")
# Write out our test target
element = {
- 'kind': 'script',
- 'depends': [
- {
- 'filename': 'base.bst',
- 'type': 'build',
- },
- ],
- 'config': {
- 'commands': [
- 'true',
- ],
- },
+ "kind": "script",
+ "depends": [{"filename": "base.bst", "type": "build",},],
+ "config": {"commands": ["true",],},
}
_yaml.roundtrip_dump(element, element_path)
# Build without access to host tools, this will fail
- result = cli.run(project=project, args=['build', 'element.bst'], env={'PATH': '', 'BST_FORCE_SANDBOX': 'bwrap'})
+ result = cli.run(project=project, args=["build", "element.bst"], env={"PATH": "", "BST_FORCE_SANDBOX": "bwrap"})
result.assert_main_error(ErrorDomain.PLATFORM, None)
assert "Bubblewrap not found" in result.stderr
# we have asked for a spesific sand box, but it is not avalble so
# bst should fail early and the element should be waiting
- assert cli.get_element_state(project, 'element.bst') == 'waiting'
+ assert cli.get_element_state(project, "element.bst") == "waiting"
@pytest.mark.datafiles(DATA_DIR)
def test_dummy_sandbox_fallback(cli, datafiles, tmp_path):
# Create symlink to buildbox-casd to work with custom PATH
- buildbox_casd = tmp_path.joinpath('bin/buildbox-casd')
+ buildbox_casd = tmp_path.joinpath("bin/buildbox-casd")
buildbox_casd.parent.mkdir()
- os.symlink(utils.get_host_tool('buildbox-casd'), str(buildbox_casd))
+ os.symlink(utils.get_host_tool("buildbox-casd"), str(buildbox_casd))
project = str(datafiles)
- element_path = os.path.join(project, 'elements', 'element.bst')
+ element_path = os.path.join(project, "elements", "element.bst")
# Write out our test target
element = {
- 'kind': 'script',
- 'depends': [
- {
- 'filename': 'base.bst',
- 'type': 'build',
- },
- ],
- 'config': {
- 'commands': [
- 'true',
- ],
- },
+ "kind": "script",
+ "depends": [{"filename": "base.bst", "type": "build",},],
+ "config": {"commands": ["true",],},
}
_yaml.roundtrip_dump(element, element_path)
# Build without access to host tools, this will fail
result = cli.run(
project=project,
- args=['build', 'element.bst'],
- env={'PATH': str(tmp_path.joinpath('bin')),
- 'BST_FORCE_SANDBOX': None})
+ args=["build", "element.bst"],
+ env={"PATH": str(tmp_path.joinpath("bin")), "BST_FORCE_SANDBOX": None},
+ )
# But if we dont spesify a sandbox then we fall back to dummy, we still
# fail early but only once we know we need a facny sandbox and that
# dumy is not enough, there for element gets fetched and so is buildable
- result.assert_task_error(ErrorDomain.SANDBOX, 'unavailable-local-sandbox')
- assert cli.get_element_state(project, 'element.bst') == 'buildable'
+ result.assert_task_error(ErrorDomain.SANDBOX, "unavailable-local-sandbox")
+ assert cli.get_element_state(project, "element.bst") == "buildable"
diff --git a/tests/sourcecache/cache.py b/tests/sourcecache/cache.py
index 9aa2c67ac..bbc3d8329 100644
--- a/tests/sourcecache/cache.py
+++ b/tests/sourcecache/cache.py
@@ -39,10 +39,10 @@ def test_patch_sources_cached_1(cli, datafiles):
# as we have a local, patch, local config, the first local and patch should
# be cached together, and the last local on it's own
- source_protos = os.path.join(project_dir, 'cache', 'source_protos')
+ source_protos = os.path.join(project_dir, "cache", "source_protos")
- assert len(os.listdir(os.path.join(source_protos, 'patch'))) == 1
- assert len(os.listdir(os.path.join(source_protos, 'local'))) == 2
+ assert len(os.listdir(os.path.join(source_protos, "patch"))) == 1
+ assert len(os.listdir(os.path.join(source_protos, "local"))) == 2
@pytest.mark.datafiles(DATA_DIR)
@@ -53,9 +53,9 @@ def test_patch_sources_cached_2(cli, datafiles):
res.assert_success()
# As everything is before the patch it should all be cached together
- source_protos = os.path.join(project_dir, 'cache', 'source_protos')
+ source_protos = os.path.join(project_dir, "cache", "source_protos")
- assert len(os.listdir(os.path.join(source_protos, 'patch'))) == 1
+ assert len(os.listdir(os.path.join(source_protos, "patch"))) == 1
@pytest.mark.datafiles(DATA_DIR)
@@ -66,35 +66,34 @@ def test_sources_without_patch(cli, datafiles):
res.assert_success()
# No patches so everything should be cached seperately
- source_protos = os.path.join(project_dir, 'cache', 'source_protos')
+ source_protos = os.path.join(project_dir, "cache", "source_protos")
- assert len(os.listdir(os.path.join(source_protos, 'local'))) == 3
+ assert len(os.listdir(os.path.join(source_protos, "local"))) == 3
@pytest.mark.datafiles(DATA_DIR)
def test_source_cache_key(cli, datafiles):
project_dir = str(datafiles)
- file_path = os.path.join(project_dir, 'files')
- file_url = 'file://' + file_path
- element_path = os.path.join(project_dir, 'elements')
- element_name = 'key_check.bst'
+ file_path = os.path.join(project_dir, "files")
+ file_url = "file://" + file_path
+ element_path = os.path.join(project_dir, "elements")
+ element_name = "key_check.bst"
element = {
- 'kind': 'import',
- 'sources': [
+ "kind": "import",
+ "sources": [
{
- 'kind': 'remote',
- 'url': os.path.join(file_url, 'bin-files', 'usr', 'bin', 'hello'),
- 'directory': 'usr/bin'
- }, {
- 'kind': 'remote',
- 'url': os.path.join(file_url, 'dev-files', 'usr', 'include', 'pony.h'),
- 'directory': 'usr/include'
- }, {
- 'kind': 'patch',
- 'path': 'files/hello-patch.diff'
- }
- ]
+ "kind": "remote",
+ "url": os.path.join(file_url, "bin-files", "usr", "bin", "hello"),
+ "directory": "usr/bin",
+ },
+ {
+ "kind": "remote",
+ "url": os.path.join(file_url, "dev-files", "usr", "include", "pony.h"),
+ "directory": "usr/include",
+ },
+ {"kind": "patch", "path": "files/hello-patch.diff"},
+ ],
}
_yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
@@ -105,11 +104,11 @@ def test_source_cache_key(cli, datafiles):
res.assert_success()
# Should have one source ref
- patch_protos = os.path.join(project_dir, 'cache', 'source_protos', 'patch')
+ patch_protos = os.path.join(project_dir, "cache", "source_protos", "patch")
assert len(os.listdir(patch_protos)) == 1
# modify hello-patch file and check tracking updates refs
- with open(os.path.join(file_path, 'dev-files', 'usr', 'include', 'pony.h'), 'a') as f:
+ with open(os.path.join(file_path, "dev-files", "usr", "include", "pony.h"), "a") as f:
f.write("\nappending nonsense")
res = cli.run(project=project_dir, args=["source", "track", element_name])
diff --git a/tests/sourcecache/capabilities.py b/tests/sourcecache/capabilities.py
index 18dfddbab..9d41eba11 100644
--- a/tests/sourcecache/capabilities.py
+++ b/tests/sourcecache/capabilities.py
@@ -13,10 +13,7 @@ from tests.testutils import dummy_context
from tests.testutils.artifactshare import create_dummy_artifact_share
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- "project",
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",)
@pytest.mark.datafiles(DATA_DIR)
@@ -26,17 +23,9 @@ def test_artifact_cache_with_missing_capabilities_is_skipped(cli, tmpdir, datafi
# Set up an artifact cache.
with create_dummy_artifact_share() as share:
# Configure artifact share
- cache_dir = os.path.join(str(tmpdir), 'cache')
- user_config_file = str(tmpdir.join('buildstream.conf'))
- user_config = {
- 'scheduler': {
- 'pushers': 1
- },
- 'source-caches': {
- 'url': share.repo,
- },
- 'cachedir': cache_dir
- }
+ cache_dir = os.path.join(str(tmpdir), "cache")
+ user_config_file = str(tmpdir.join("buildstream.conf"))
+ user_config = {"scheduler": {"pushers": 1}, "source-caches": {"url": share.repo,}, "cachedir": cache_dir}
_yaml.roundtrip_dump(user_config, file=user_config_file)
with dummy_context(config=user_config_file) as context:
@@ -50,5 +39,6 @@ def test_artifact_cache_with_missing_capabilities_is_skipped(cli, tmpdir, datafi
# Manually setup the CAS remote
sourcecache.setup_remotes(use_config=True)
- assert not sourcecache.has_fetch_remotes(), \
- "System didn't realize the source cache didn't support BuildStream"
+ assert (
+ not sourcecache.has_fetch_remotes()
+ ), "System didn't realize the source cache didn't support BuildStream"
diff --git a/tests/sourcecache/config.py b/tests/sourcecache/config.py
index 2ab11e9f9..9233e9f44 100644
--- a/tests/sourcecache/config.py
+++ b/tests/sourcecache/config.py
@@ -36,27 +36,19 @@ DATA_DIR = os.path.dirname(os.path.realpath(__file__))
# without specifying its counterpart, we get a comprehensive LoadError
# instead of an unhandled exception.
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize('config_key, config_value', [
- ('client-cert', 'client.crt'),
- ('client-key', 'client.key')
-])
+@pytest.mark.parametrize("config_key, config_value", [("client-cert", "client.crt"), ("client-key", "client.key")])
def test_missing_certs(cli, datafiles, config_key, config_value):
- project = os.path.join(datafiles.dirname, datafiles.basename, 'missing-certs')
+ project = os.path.join(datafiles.dirname, datafiles.basename, "missing-certs")
project_conf = {
- 'name': 'test',
-
- 'source-caches': {
- 'url': 'https://cache.example.com:12345',
- 'push': 'true',
- config_key: config_value
- }
+ "name": "test",
+ "source-caches": {"url": "https://cache.example.com:12345", "push": "true", config_key: config_value},
}
- project_conf_file = os.path.join(project, 'project.conf')
+ project_conf_file = os.path.join(project, "project.conf")
_yaml.roundtrip_dump(project_conf, project_conf_file)
# Use `pull` here to ensure we try to initialize the remotes, triggering the error
#
# This does not happen for a simple `bst show`.
- result = cli.run(project=project, args=['source', 'fetch', 'element.bst'])
+ result = cli.run(project=project, args=["source", "fetch", "element.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
diff --git a/tests/sourcecache/fetch.py b/tests/sourcecache/fetch.py
index a5863b867..0c347ebbf 100644
--- a/tests/sourcecache/fetch.py
+++ b/tests/sourcecache/fetch.py
@@ -36,22 +36,19 @@ DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
def move_local_cas_to_remote_source_share(local, remote):
- shutil.rmtree(os.path.join(remote, 'repo', 'cas'))
- shutil.move(os.path.join(local, 'source_protos'), os.path.join(remote, 'repo'))
- shutil.move(os.path.join(local, 'cas'), os.path.join(remote, 'repo'))
- shutil.rmtree(os.path.join(local, 'sources'))
- shutil.rmtree(os.path.join(local, 'artifacts'))
+ shutil.rmtree(os.path.join(remote, "repo", "cas"))
+ shutil.move(os.path.join(local, "source_protos"), os.path.join(remote, "repo"))
+ shutil.move(os.path.join(local, "cas"), os.path.join(remote, "repo"))
+ shutil.rmtree(os.path.join(local, "sources"))
+ shutil.rmtree(os.path.join(local, "artifacts"))
def create_test_element(tmpdir, project_dir):
- repo = create_repo('git', str(tmpdir))
- ref = repo.create(os.path.join(project_dir, 'files'))
- element_path = os.path.join(project_dir, 'elements')
- element_name = 'fetch.bst'
- element = {
- 'kind': 'import',
- 'sources': [repo.source_config(ref=ref)]
- }
+ repo = create_repo("git", str(tmpdir))
+ ref = repo.create(os.path.join(project_dir, "files"))
+ element_path = os.path.join(project_dir, "elements")
+ element_name = "fetch.bst"
+ element = {"kind": "import", "sources": [repo.source_config(ref=ref)]}
_yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
return element_name, repo, ref
@@ -59,15 +56,11 @@ def create_test_element(tmpdir, project_dir):
@contextmanager
def context_with_source_cache(cli, cache, share, tmpdir):
- user_config_file = str(tmpdir.join('buildstream.conf'))
+ user_config_file = str(tmpdir.join("buildstream.conf"))
user_config = {
- 'scheduler': {
- 'pushers': 1
- },
- 'source-caches': {
- 'url': share.repo,
- },
- 'cachedir': cache,
+ "scheduler": {"pushers": 1},
+ "source-caches": {"url": share.repo,},
+ "cachedir": cache,
}
_yaml.roundtrip_dump(user_config, file=user_config_file)
cli.configure(user_config)
@@ -80,10 +73,10 @@ def context_with_source_cache(cli, cache, share, tmpdir):
def test_source_fetch(cli, tmpdir, datafiles):
project_dir = str(datafiles)
element_name, _repo, _ref = create_test_element(tmpdir, project_dir)
- cache_dir = os.path.join(str(tmpdir), 'cache')
+ cache_dir = os.path.join(str(tmpdir), "cache")
# use artifact cache for sources for now, they should work the same
- with create_artifact_share(os.path.join(str(tmpdir), 'sourceshare')) as share:
+ with create_artifact_share(os.path.join(str(tmpdir), "sourceshare")) as share:
with context_with_source_cache(cli, cache_dir, share, tmpdir) as context:
project = Project(project_dir, context)
project.ensure_fully_loaded()
@@ -96,10 +89,10 @@ def test_source_fetch(cli, tmpdir, datafiles):
assert not cas.contains(source._get_source_name())
# Just check that we sensibly fetch and build the element
- res = cli.run(project=project_dir, args=['build', element_name])
+ res = cli.run(project=project_dir, args=["build", element_name])
res.assert_success()
- assert os.listdir(os.path.join(str(tmpdir), 'cache', 'sources', 'git')) != []
+ assert os.listdir(os.path.join(str(tmpdir), "cache", "sources", "git")) != []
# get root digest of source
sourcecache = context.sourcecache
@@ -111,26 +104,26 @@ def test_source_fetch(cli, tmpdir, datafiles):
assert share.has_object(digest)
state = cli.get_element_state(project_dir, element_name)
- assert state == 'fetch needed'
+ assert state == "fetch needed"
# Now fetch the source and check
- res = cli.run(project=project_dir, args=['source', 'fetch', element_name])
+ res = cli.run(project=project_dir, args=["source", "fetch", element_name])
res.assert_success()
assert "Pulled source" in res.stderr
# check that we have the source in the cas now and it's not fetched
assert element._source_cached()
- assert os.listdir(os.path.join(str(tmpdir), 'cache', 'sources', 'git')) == []
+ assert os.listdir(os.path.join(str(tmpdir), "cache", "sources", "git")) == []
@pytest.mark.datafiles(DATA_DIR)
def test_fetch_fallback(cli, tmpdir, datafiles):
project_dir = str(datafiles)
element_name, repo, ref = create_test_element(tmpdir, project_dir)
- cache_dir = os.path.join(str(tmpdir), 'cache')
+ cache_dir = os.path.join(str(tmpdir), "cache")
# use artifact cache for sources for now, they should work the same
- with create_artifact_share(os.path.join(str(tmpdir), 'sourceshare')) as share:
+ with create_artifact_share(os.path.join(str(tmpdir), "sourceshare")) as share:
with context_with_source_cache(cli, cache_dir, share, tmpdir) as context:
project = Project(project_dir, context)
project.ensure_fully_loaded()
@@ -141,16 +134,16 @@ def test_fetch_fallback(cli, tmpdir, datafiles):
cas = context.get_cascache()
assert not cas.contains(source._get_source_name())
- assert not os.path.exists(os.path.join(cache_dir, 'sources'))
+ assert not os.path.exists(os.path.join(cache_dir, "sources"))
# Now check if it falls back to the source fetch method.
- res = cli.run(project=project_dir, args=['source', 'fetch', element_name])
+ res = cli.run(project=project_dir, args=["source", "fetch", element_name])
res.assert_success()
brief_key = source._get_brief_display_key()
- assert ("Remote source service ({}) does not have source {} cached"
- .format(share.repo, brief_key)) in res.stderr
- assert ("SUCCESS Fetching from {}"
- .format(repo.source_config(ref=ref)['url'])) in res.stderr
+ assert (
+ "Remote source service ({}) does not have source {} cached".format(share.repo, brief_key)
+ ) in res.stderr
+ assert ("SUCCESS Fetching from {}".format(repo.source_config(ref=ref)["url"])) in res.stderr
# Check that the source in both in the source dir and the local CAS
assert element._source_cached()
@@ -160,9 +153,9 @@ def test_fetch_fallback(cli, tmpdir, datafiles):
def test_pull_fail(cli, tmpdir, datafiles):
project_dir = str(datafiles)
element_name, repo, _ref = create_test_element(tmpdir, project_dir)
- cache_dir = os.path.join(str(tmpdir), 'cache')
+ cache_dir = os.path.join(str(tmpdir), "cache")
- with create_artifact_share(os.path.join(str(tmpdir), 'sourceshare')) as share:
+ with create_artifact_share(os.path.join(str(tmpdir), "sourceshare")) as share:
with context_with_source_cache(cli, cache_dir, share, tmpdir) as context:
project = Project(project_dir, context)
project.ensure_fully_loaded()
@@ -175,21 +168,25 @@ def test_pull_fail(cli, tmpdir, datafiles):
shutil.rmtree(repo.repo)
# Should fail in stream, with a plugin task causing the error
- res = cli.run(project=project_dir, args=['build', element_name])
+ res = cli.run(project=project_dir, args=["build", element_name])
res.assert_main_error(ErrorDomain.STREAM, None)
res.assert_task_error(ErrorDomain.PLUGIN, None)
- assert "Remote source service ({}) does not have source {} cached".format(
- share.repo, source._get_brief_display_key()) in res.stderr
+ assert (
+ "Remote source service ({}) does not have source {} cached".format(
+ share.repo, source._get_brief_display_key()
+ )
+ in res.stderr
+ )
@pytest.mark.datafiles(DATA_DIR)
def test_source_pull_partial_fallback_fetch(cli, tmpdir, datafiles):
project_dir = str(datafiles)
element_name, repo, ref = create_test_element(tmpdir, project_dir)
- cache_dir = os.path.join(str(tmpdir), 'cache')
+ cache_dir = os.path.join(str(tmpdir), "cache")
# use artifact cache for sources for now, they should work the same
- with create_artifact_share(os.path.join(str(tmpdir), 'sourceshare')) as share:
+ with create_artifact_share(os.path.join(str(tmpdir), "sourceshare")) as share:
with context_with_source_cache(cli, cache_dir, share, tmpdir) as context:
project = Project(project_dir, context)
project.ensure_fully_loaded()
@@ -202,10 +199,10 @@ def test_source_pull_partial_fallback_fetch(cli, tmpdir, datafiles):
assert not cas.contains(source._get_source_name())
# Just check that we sensibly fetch and build the element
- res = cli.run(project=project_dir, args=['build', element_name])
+ res = cli.run(project=project_dir, args=["build", element_name])
res.assert_success()
- assert os.listdir(os.path.join(str(tmpdir), 'cache', 'sources', 'git')) != []
+ assert os.listdir(os.path.join(str(tmpdir), "cache", "sources", "git")) != []
# get root digest of source
sourcecache = context.sourcecache
@@ -219,11 +216,10 @@ def test_source_pull_partial_fallback_fetch(cli, tmpdir, datafiles):
assert not share.has_object(digest)
state = cli.get_element_state(project_dir, element_name)
- assert state == 'fetch needed'
+ assert state == "fetch needed"
# Now fetch the source and check
- res = cli.run(project=project_dir, args=['source', 'fetch', element_name])
+ res = cli.run(project=project_dir, args=["source", "fetch", element_name])
res.assert_success()
- assert ("SUCCESS Fetching from {}"
- .format(repo.source_config(ref=ref)['url'])) in res.stderr
+ assert ("SUCCESS Fetching from {}".format(repo.source_config(ref=ref)["url"])) in res.stderr
diff --git a/tests/sourcecache/project/plugins/elements/always_fail.py b/tests/sourcecache/project/plugins/elements/always_fail.py
index 99ef0d7de..43dba5626 100644
--- a/tests/sourcecache/project/plugins/elements/always_fail.py
+++ b/tests/sourcecache/project/plugins/elements/always_fail.py
@@ -23,7 +23,6 @@ from buildstream.buildelement import BuildElement
class AlwaysFail(BuildElement):
-
def assemble(self, sandbox):
raise ElementError("Always fails")
diff --git a/tests/sourcecache/push.py b/tests/sourcecache/push.py
index 406aeba9f..719860425 100644
--- a/tests/sourcecache/push.py
+++ b/tests/sourcecache/push.py
@@ -47,6 +47,7 @@ def message_handler(message, is_silenced):
@contextmanager
def _configure_caches(tmpdir, *directories):
with ExitStack() as stack:
+
def create_share(directory):
return create_artifact_share(os.path.join(str(tmpdir), directory))
@@ -55,37 +56,27 @@ def _configure_caches(tmpdir, *directories):
@pytest.mark.datafiles(DATA_DIR)
def test_source_push_split(cli, tmpdir, datafiles):
- cache_dir = os.path.join(str(tmpdir), 'cache')
+ cache_dir = os.path.join(str(tmpdir), "cache")
project_dir = str(datafiles)
- with _configure_caches(tmpdir, 'indexshare', 'storageshare') as (index, storage):
- user_config_file = str(tmpdir.join('buildstream.conf'))
+ with _configure_caches(tmpdir, "indexshare", "storageshare") as (index, storage):
+ user_config_file = str(tmpdir.join("buildstream.conf"))
user_config = {
- 'scheduler': {
- 'pushers': 1
- },
- 'source-caches': [{
- 'url': index.repo,
- 'push': True,
- 'type': 'index'
- }, {
- 'url': storage.repo,
- 'push': True,
- 'type': 'storage'
- }],
- 'cachedir': cache_dir
+ "scheduler": {"pushers": 1},
+ "source-caches": [
+ {"url": index.repo, "push": True, "type": "index"},
+ {"url": storage.repo, "push": True, "type": "storage"},
+ ],
+ "cachedir": cache_dir,
}
_yaml.roundtrip_dump(user_config, file=user_config_file)
cli.configure(user_config)
- repo = create_repo('git', str(tmpdir))
- ref = repo.create(os.path.join(project_dir, 'files'))
- element_path = os.path.join(project_dir, 'elements')
- element_name = 'push.bst'
- element = {
- 'kind': 'import',
- 'sources': [repo.source_config(ref=ref)]
- }
+ repo = create_repo("git", str(tmpdir))
+ ref = repo.create(os.path.join(project_dir, "files"))
+ element_path = os.path.join(project_dir, "elements")
+ element_name = "push.bst"
+ element = {"kind": "import", "sources": [repo.source_config(ref=ref)]}
_yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
# get the source object
@@ -93,7 +84,7 @@ def test_source_push_split(cli, tmpdir, datafiles):
project = Project(project_dir, context)
project.ensure_fully_loaded()
- element = project.load_elements(['push.bst'])[0]
+ element = project.load_elements(["push.bst"])[0]
assert not element._source_cached()
source = list(element.sources())[0]
@@ -103,7 +94,7 @@ def test_source_push_split(cli, tmpdir, datafiles):
# build the element, this should fetch and then push the source to the
# remote
- res = cli.run(project=project_dir, args=['build', 'push.bst'])
+ res = cli.run(project=project_dir, args=["build", "push.bst"])
res.assert_success()
assert "Pushed source" in res.stderr
@@ -118,32 +109,24 @@ def test_source_push_split(cli, tmpdir, datafiles):
@pytest.mark.datafiles(DATA_DIR)
def test_source_push(cli, tmpdir, datafiles):
- cache_dir = os.path.join(str(tmpdir), 'cache')
+ cache_dir = os.path.join(str(tmpdir), "cache")
project_dir = str(datafiles)
- with create_artifact_share(os.path.join(str(tmpdir), 'sourceshare')) as share:
- user_config_file = str(tmpdir.join('buildstream.conf'))
+ with create_artifact_share(os.path.join(str(tmpdir), "sourceshare")) as share:
+ user_config_file = str(tmpdir.join("buildstream.conf"))
user_config = {
- 'scheduler': {
- 'pushers': 1
- },
- 'source-caches': {
- 'url': share.repo,
- 'push': True,
- },
- 'cachedir': cache_dir,
+ "scheduler": {"pushers": 1},
+ "source-caches": {"url": share.repo, "push": True,},
+ "cachedir": cache_dir,
}
_yaml.roundtrip_dump(user_config, file=user_config_file)
cli.configure(user_config)
- repo = create_repo('git', str(tmpdir))
- ref = repo.create(os.path.join(project_dir, 'files'))
- element_path = os.path.join(project_dir, 'elements')
- element_name = 'push.bst'
- element = {
- 'kind': 'import',
- 'sources': [repo.source_config(ref=ref)]
- }
+ repo = create_repo("git", str(tmpdir))
+ ref = repo.create(os.path.join(project_dir, "files"))
+ element_path = os.path.join(project_dir, "elements")
+ element_name = "push.bst"
+ element = {"kind": "import", "sources": [repo.source_config(ref=ref)]}
_yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
# get the source object
@@ -151,7 +134,7 @@ def test_source_push(cli, tmpdir, datafiles):
project = Project(project_dir, context)
project.ensure_fully_loaded()
- element = project.load_elements(['push.bst'])[0]
+ element = project.load_elements(["push.bst"])[0]
assert not element._source_cached()
source = list(element.sources())[0]
@@ -161,7 +144,7 @@ def test_source_push(cli, tmpdir, datafiles):
# build the element, this should fetch and then push the source to the
# remote
- res = cli.run(project=project_dir, args=['build', 'push.bst'])
+ res = cli.run(project=project_dir, args=["build", "push.bst"])
res.assert_success()
assert "Pushed source" in res.stderr
@@ -177,35 +160,27 @@ def test_source_push(cli, tmpdir, datafiles):
@pytest.mark.datafiles(DATA_DIR)
def test_push_pull(cli, datafiles, tmpdir):
project_dir = str(datafiles)
- cache_dir = os.path.join(str(tmpdir), 'cache')
+ cache_dir = os.path.join(str(tmpdir), "cache")
- with create_artifact_share(os.path.join(str(tmpdir), 'sourceshare')) as share:
- user_config_file = str(tmpdir.join('buildstream.conf'))
+ with create_artifact_share(os.path.join(str(tmpdir), "sourceshare")) as share:
+ user_config_file = str(tmpdir.join("buildstream.conf"))
user_config = {
- 'scheduler': {
- 'pushers': 1
- },
- 'source-caches': {
- 'url': share.repo,
- 'push': True,
- },
- 'cachedir': cache_dir,
+ "scheduler": {"pushers": 1},
+ "source-caches": {"url": share.repo, "push": True,},
+ "cachedir": cache_dir,
}
_yaml.roundtrip_dump(user_config, file=user_config_file)
cli.configure(user_config)
# create repo to pull from
- repo = create_repo('git', str(tmpdir))
- ref = repo.create(os.path.join(project_dir, 'files'))
- element_path = os.path.join(project_dir, 'elements')
- element_name = 'push.bst'
- element = {
- 'kind': 'import',
- 'sources': [repo.source_config(ref=ref)]
- }
+ repo = create_repo("git", str(tmpdir))
+ ref = repo.create(os.path.join(project_dir, "files"))
+ element_path = os.path.join(project_dir, "elements")
+ element_name = "push.bst"
+ element = {"kind": "import", "sources": [repo.source_config(ref=ref)]}
_yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
- res = cli.run(project=project_dir, args=['build', 'push.bst'])
+ res = cli.run(project=project_dir, args=["build", "push.bst"])
res.assert_success()
# remove local cache dir, and repo files and check it all works
@@ -214,45 +189,37 @@ def test_push_pull(cli, datafiles, tmpdir):
shutil.rmtree(repo.repo)
# check it's pulls from the share
- res = cli.run(project=project_dir, args=['build', 'push.bst'])
+ res = cli.run(project=project_dir, args=["build", "push.bst"])
res.assert_success()
@pytest.mark.datafiles(DATA_DIR)
def test_push_fail(cli, tmpdir, datafiles):
project_dir = str(datafiles)
- cache_dir = os.path.join(str(tmpdir), 'cache')
+ cache_dir = os.path.join(str(tmpdir), "cache")
# set up config with remote that we'll take down
- with create_artifact_share(os.path.join(str(tmpdir), 'sourceshare')) as share:
+ with create_artifact_share(os.path.join(str(tmpdir), "sourceshare")) as share:
remote = share.repo
- user_config_file = str(tmpdir.join('buildstream.conf'))
+ user_config_file = str(tmpdir.join("buildstream.conf"))
user_config = {
- 'scheduler': {
- 'pushers': 1
- },
- 'source-caches': {
- 'url': share.repo,
- 'push': True,
- },
- 'cachedir': cache_dir,
+ "scheduler": {"pushers": 1},
+ "source-caches": {"url": share.repo, "push": True,},
+ "cachedir": cache_dir,
}
_yaml.roundtrip_dump(user_config, file=user_config_file)
cli.configure(user_config)
# create repo to pull from
- repo = create_repo('git', str(tmpdir))
- ref = repo.create(os.path.join(project_dir, 'files'))
- element_path = os.path.join(project_dir, 'elements')
- element_name = 'push.bst'
- element = {
- 'kind': 'import',
- 'sources': [repo.source_config(ref=ref)]
- }
+ repo = create_repo("git", str(tmpdir))
+ ref = repo.create(os.path.join(project_dir, "files"))
+ element_path = os.path.join(project_dir, "elements")
+ element_name = "push.bst"
+ element = {"kind": "import", "sources": [repo.source_config(ref=ref)]}
_yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
# build and check that it fails to set up the remote
- res = cli.run(project=project_dir, args=['build', 'push.bst'])
+ res = cli.run(project=project_dir, args=["build", "push.bst"])
res.assert_success()
assert "Failed to initialize remote {}".format(remote) in res.stderr
@@ -260,37 +227,29 @@ def test_push_fail(cli, tmpdir, datafiles):
assert "Pushed" not in res.stderr
-@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
+@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox")
@pytest.mark.datafiles(DATA_DIR)
def test_source_push_build_fail(cli, tmpdir, datafiles):
project_dir = str(datafiles)
- cache_dir = os.path.join(str(tmpdir), 'cache')
+ cache_dir = os.path.join(str(tmpdir), "cache")
- with create_artifact_share(os.path.join(str(tmpdir), 'share')) as share:
+ with create_artifact_share(os.path.join(str(tmpdir), "share")) as share:
user_config = {
- 'scheduler': {
- 'pushers': 1
- },
- 'source-caches': {
- 'url': share.repo,
- 'push': True,
- },
- 'cachedir': cache_dir,
+ "scheduler": {"pushers": 1},
+ "source-caches": {"url": share.repo, "push": True,},
+ "cachedir": cache_dir,
}
cli.configure(user_config)
- repo = create_repo('git', str(tmpdir))
- ref = repo.create(os.path.join(project_dir, 'files'))
- element_path = os.path.join(project_dir, 'elements')
+ repo = create_repo("git", str(tmpdir))
+ ref = repo.create(os.path.join(project_dir, "files"))
+ element_path = os.path.join(project_dir, "elements")
- element_name = 'always-fail.bst'
- element = {
- 'kind': 'always_fail',
- 'sources': [repo.source_config(ref=ref)]
- }
+ element_name = "always-fail.bst"
+ element = {"kind": "always_fail", "sources": [repo.source_config(ref=ref)]}
_yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
- res = cli.run(project=project_dir, args=['build', 'always-fail.bst'])
+ res = cli.run(project=project_dir, args=["build", "always-fail.bst"])
res.assert_main_error(ErrorDomain.STREAM, None)
res.assert_task_error(ErrorDomain.ELEMENT, None)
diff --git a/tests/sourcecache/source-checkout.py b/tests/sourcecache/source-checkout.py
index 4e3391c12..f1e0706a7 100644
--- a/tests/sourcecache/source-checkout.py
+++ b/tests/sourcecache/source-checkout.py
@@ -36,20 +36,20 @@ DATA_DIR = os.path.dirname(os.path.realpath(__file__))
@pytest.mark.datafiles(DATA_DIR)
def test_source_checkout(tmpdir, datafiles, cli):
- project_dir = os.path.join(str(tmpdir), 'project')
- element_path = 'elements'
- cache_dir = os.path.join(str(tmpdir), 'cache')
- source_dir = os.path.join(cache_dir, 'sources')
+ project_dir = os.path.join(str(tmpdir), "project")
+ element_path = "elements"
+ cache_dir = os.path.join(str(tmpdir), "cache")
+ source_dir = os.path.join(cache_dir, "sources")
- cli.configure({
- 'cachedir': cache_dir,
- })
- target_dir = os.path.join(str(tmpdir), 'target')
+ cli.configure(
+ {"cachedir": cache_dir,}
+ )
+ target_dir = os.path.join(str(tmpdir), "target")
- repo = create_element_size('target.bst', project_dir, element_path, [], 100000)
+ repo = create_element_size("target.bst", project_dir, element_path, [], 100000)
# check implicit fetching
- res = cli.run(project=project_dir, args=['source', 'checkout', '--directory', target_dir, 'target.bst'])
+ res = cli.run(project=project_dir, args=["source", "checkout", "--directory", target_dir, "target.bst"])
res.assert_success()
assert "Fetching from" in res.stderr
@@ -59,15 +59,13 @@ def test_source_checkout(tmpdir, datafiles, cli):
shutil.rmtree(target_dir)
shutil.rmtree(source_dir)
- res = cli.run(project=project_dir,
- args=['source', 'checkout', '--directory', target_dir, 'target.bst'])
+ res = cli.run(project=project_dir, args=["source", "checkout", "--directory", target_dir, "target.bst"])
res.assert_success()
assert "Fetching from" not in res.stderr
# remove the CAS and check it doesn't work again
shutil.rmtree(target_dir)
- shutil.rmtree(os.path.join(cache_dir, 'cas'))
+ shutil.rmtree(os.path.join(cache_dir, "cas"))
- res = cli.run(project=project_dir,
- args=['source', 'checkout', '--directory', target_dir, 'target.bst'])
+ res = cli.run(project=project_dir, args=["source", "checkout", "--directory", target_dir, "target.bst"])
res.assert_task_error(ErrorDomain.PLUGIN, None)
diff --git a/tests/sourcecache/staging.py b/tests/sourcecache/staging.py
index 186a4bd9f..994adb32a 100644
--- a/tests/sourcecache/staging.py
+++ b/tests/sourcecache/staging.py
@@ -45,12 +45,10 @@ def relative_walk(rootdir):
@pytest.mark.datafiles(DATA_DIR)
def test_source_staged(tmpdir, cli, datafiles):
- project_dir = os.path.join(datafiles.dirname, datafiles.basename, 'project')
- cachedir = os.path.join(str(tmpdir), 'cache')
+ project_dir = os.path.join(datafiles.dirname, datafiles.basename, "project")
+ cachedir = os.path.join(str(tmpdir), "cache")
- cli.configure({
- 'cachedir': cachedir
- })
+ cli.configure({"cachedir": cachedir})
res = cli.run(project=project_dir, args=["build", "import-bin.bst"])
res.assert_success()
@@ -83,12 +81,10 @@ def test_source_staged(tmpdir, cli, datafiles):
# Check sources are staged during a fetch
@pytest.mark.datafiles(DATA_DIR)
def test_source_fetch(tmpdir, cli, datafiles):
- project_dir = os.path.join(datafiles.dirname, datafiles.basename, 'project')
- cachedir = os.path.join(str(tmpdir), 'cache')
+ project_dir = os.path.join(datafiles.dirname, datafiles.basename, "project")
+ cachedir = os.path.join(str(tmpdir), "cache")
- cli.configure({
- 'cachedir': cachedir
- })
+ cli.configure({"cachedir": cachedir})
res = cli.run(project=project_dir, args=["source", "fetch", "import-dev.bst"])
res.assert_success()
@@ -118,17 +114,15 @@ def test_source_fetch(tmpdir, cli, datafiles):
# Check that with sources only in the CAS build successfully completes
@pytest.mark.datafiles(DATA_DIR)
def test_staged_source_build(tmpdir, datafiles, cli):
- project_dir = os.path.join(datafiles.dirname, datafiles.basename, 'project')
- cachedir = os.path.join(str(tmpdir), 'cache')
- element_path = 'elements'
- source_protos = os.path.join(str(tmpdir), 'cache', 'source_protos')
- source_dir = os.path.join(str(tmpdir), 'cache', 'sources')
+ project_dir = os.path.join(datafiles.dirname, datafiles.basename, "project")
+ cachedir = os.path.join(str(tmpdir), "cache")
+ element_path = "elements"
+ source_protos = os.path.join(str(tmpdir), "cache", "source_protos")
+ source_dir = os.path.join(str(tmpdir), "cache", "sources")
- cli.configure({
- 'cachedir': cachedir
- })
+ cli.configure({"cachedir": cachedir})
- create_element_size('target.bst', project_dir, element_path, [], 10000)
+ create_element_size("target.bst", project_dir, element_path, [], 10000)
with dummy_context() as context:
context.cachedir = cachedir
@@ -140,23 +134,23 @@ def test_staged_source_build(tmpdir, datafiles, cli):
# check consistency of the source
assert not element._source_cached()
- res = cli.run(project=project_dir, args=['build', 'target.bst'])
+ res = cli.run(project=project_dir, args=["build", "target.bst"])
res.assert_success()
# delete artifacts check state is buildable
- cli.remove_artifact_from_cache(project_dir, 'target.bst')
- states = cli.get_element_states(project_dir, ['target.bst'])
- assert states['target.bst'] == 'buildable'
+ cli.remove_artifact_from_cache(project_dir, "target.bst")
+ states = cli.get_element_states(project_dir, ["target.bst"])
+ assert states["target.bst"] == "buildable"
# delete source dir and check that state is still buildable
shutil.rmtree(source_dir)
- states = cli.get_element_states(project_dir, ['target.bst'])
- assert states['target.bst'] == 'buildable'
+ states = cli.get_element_states(project_dir, ["target.bst"])
+ assert states["target.bst"] == "buildable"
# build and check that no fetching was done.
- res = cli.run(project=project_dir, args=['build', 'target.bst'])
+ res = cli.run(project=project_dir, args=["build", "target.bst"])
res.assert_success()
- assert 'Fetching from' not in res.stderr
+ assert "Fetching from" not in res.stderr
# assert the source directory is still empty (though there may be
# directories from staging etc.)
@@ -167,11 +161,11 @@ def test_staged_source_build(tmpdir, datafiles, cli):
# Now remove the source refs and check the state
shutil.rmtree(source_protos)
- cli.remove_artifact_from_cache(project_dir, 'target.bst')
- states = cli.get_element_states(project_dir, ['target.bst'])
- assert states['target.bst'] == 'fetch needed'
+ cli.remove_artifact_from_cache(project_dir, "target.bst")
+ states = cli.get_element_states(project_dir, ["target.bst"])
+ assert states["target.bst"] == "fetch needed"
# Check that it now fetches from when building the target
- res = cli.run(project=project_dir, args=['build', 'target.bst'])
+ res = cli.run(project=project_dir, args=["build", "target.bst"])
res.assert_success()
- assert 'Fetching from' in res.stderr
+ assert "Fetching from" in res.stderr
diff --git a/tests/sourcecache/workspace.py b/tests/sourcecache/workspace.py
index 22316c4fe..bb1ea505a 100644
--- a/tests/sourcecache/workspace.py
+++ b/tests/sourcecache/workspace.py
@@ -39,66 +39,55 @@ DATA_DIR = os.path.dirname(os.path.realpath(__file__))
# for opening a workspace
@pytest.mark.datafiles(DATA_DIR)
def test_workspace_source_fetch(tmpdir, datafiles, cli):
- project_dir = os.path.join(str(tmpdir), 'project')
- element_path = 'elements'
- source_dir = os.path.join(str(tmpdir), 'cache', 'sources')
- workspace = os.path.join(cli.directory, 'workspace')
+ project_dir = os.path.join(str(tmpdir), "project")
+ element_path = "elements"
+ source_dir = os.path.join(str(tmpdir), "cache", "sources")
+ workspace = os.path.join(cli.directory, "workspace")
- cli.configure({
- 'cachedir': os.path.join(str(tmpdir), 'cache')
- })
+ cli.configure({"cachedir": os.path.join(str(tmpdir), "cache")})
- create_element_size('target.bst', project_dir, element_path, [], 10000)
- res = cli.run(project=project_dir, args=['build', 'target.bst'])
+ create_element_size("target.bst", project_dir, element_path, [], 10000)
+ res = cli.run(project=project_dir, args=["build", "target.bst"])
res.assert_success()
- assert 'Fetching from' in res.stderr
+ assert "Fetching from" in res.stderr
# remove the original sources
shutil.rmtree(source_dir)
# Open a workspace and check that fetches the original sources
- res = cli.run(project=project_dir,
- args=['workspace', 'open', 'target.bst', '--directory', workspace])
+ res = cli.run(project=project_dir, args=["workspace", "open", "target.bst", "--directory", workspace])
res.assert_success()
- assert 'Fetching from' in res.stderr
+ assert "Fetching from" in res.stderr
assert os.listdir(workspace) != []
@pytest.mark.datafiles(DATA_DIR)
def test_workspace_open_no_source_push(tmpdir, datafiles, cli):
- project_dir = os.path.join(str(tmpdir), 'project')
- element_path = 'elements'
- cache_dir = os.path.join(str(tmpdir), 'cache')
- share_dir = os.path.join(str(tmpdir), 'share')
- workspace = os.path.join(cli.directory, 'workspace')
+ project_dir = os.path.join(str(tmpdir), "project")
+ element_path = "elements"
+ cache_dir = os.path.join(str(tmpdir), "cache")
+ share_dir = os.path.join(str(tmpdir), "share")
+ workspace = os.path.join(cli.directory, "workspace")
with create_artifact_share(share_dir) as share:
- cli.configure({
- 'cachedir': cache_dir,
- 'scheduler': {
- 'pushers': 1
- },
- 'source-caches': {
- 'url': share.repo,
- 'push': True,
- },
- })
+ cli.configure(
+ {"cachedir": cache_dir, "scheduler": {"pushers": 1}, "source-caches": {"url": share.repo, "push": True,},}
+ )
# Fetch as in previous test and check it pushes the source
- create_element_size('target.bst', project_dir, element_path, [], 10000)
- res = cli.run(project=project_dir, args=['build', 'target.bst'])
+ create_element_size("target.bst", project_dir, element_path, [], 10000)
+ res = cli.run(project=project_dir, args=["build", "target.bst"])
res.assert_success()
- assert 'Fetching from' in res.stderr
- assert 'Pushed source' in res.stderr
+ assert "Fetching from" in res.stderr
+ assert "Pushed source" in res.stderr
# clear the cas and open a workspace
- shutil.rmtree(os.path.join(cache_dir, 'cas'))
- res = cli.run(project=project_dir,
- args=['workspace', 'open', 'target.bst', '--directory', workspace])
+ shutil.rmtree(os.path.join(cache_dir, "cas"))
+ res = cli.run(project=project_dir, args=["workspace", "open", "target.bst", "--directory", workspace])
res.assert_success()
# Check that this time it does not push the sources
- res = cli.run(project=project_dir, args=['build', 'target.bst'])
+ res = cli.run(project=project_dir, args=["build", "target.bst"])
res.assert_success()
assert "Pushed source" not in res.stderr
diff --git a/tests/sources/bzr.py b/tests/sources/bzr.py
index c6e78f8c1..ca727c793 100644
--- a/tests/sources/bzr.py
+++ b/tests/sources/bzr.py
@@ -10,40 +10,32 @@ from buildstream.testing import cli # pylint: disable=unused-import
from buildstream.testing import create_repo
from buildstream.testing._utils.site import HAVE_BZR
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- 'bzr'
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "bzr")
@pytest.mark.skipif(HAVE_BZR is False, reason="bzr is not available")
@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_fetch_checkout(cli, tmpdir, datafiles):
project = str(datafiles)
- checkoutdir = os.path.join(str(tmpdir), 'checkout')
+ checkoutdir = os.path.join(str(tmpdir), "checkout")
- repo = create_repo('bzr', str(tmpdir))
- ref = repo.create(os.path.join(project, 'basic'))
+ repo = create_repo("bzr", str(tmpdir))
+ ref = repo.create(os.path.join(project, "basic"))
# Write out our test target
- element = {
- 'kind': 'import',
- 'sources': [
- repo.source_config(ref=ref)
- ]
- }
- _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst'))
+ element = {"kind": "import", "sources": [repo.source_config(ref=ref)]}
+ _yaml.roundtrip_dump(element, os.path.join(project, "target.bst"))
# Fetch, build, checkout
- result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
assert result.exit_code == 0
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
assert result.exit_code == 0
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
assert result.exit_code == 0
# Assert we checked out the file as it was commited
- with open(os.path.join(checkoutdir, 'test')) as f:
+ with open(os.path.join(checkoutdir, "test")) as f:
text = f.read()
- assert text == 'test\n'
+ assert text == "test\n"
diff --git a/tests/sources/deb.py b/tests/sources/deb.py
index e536e522a..96060a1a4 100644
--- a/tests/sources/deb.py
+++ b/tests/sources/deb.py
@@ -12,22 +12,14 @@ from buildstream.testing import cli # pylint: disable=unused-import
from buildstream.testing._utils.site import HAVE_ARPY
from . import list_dir_contents
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- 'deb',
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "deb",)
deb_name = "a_deb.deb"
def generate_project(project_dir, tmpdir):
project_file = os.path.join(project_dir, "project.conf")
- _yaml.roundtrip_dump({
- 'name': 'foo',
- 'aliases': {
- 'tmpdir': "file:///" + str(tmpdir)
- }
- }, project_file)
+ _yaml.roundtrip_dump({"name": "foo", "aliases": {"tmpdir": "file:///" + str(tmpdir)}}, project_file)
def _copy_deb(start_location, tmpdir):
@@ -38,31 +30,29 @@ def _copy_deb(start_location, tmpdir):
# Test that without ref, consistency is set appropriately.
@pytest.mark.skipif(HAVE_ARPY is False, reason="arpy is not available")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-ref'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "no-ref"))
def test_no_ref(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project, tmpdir)
- assert cli.get_element_state(project, 'target.bst') == 'no reference'
+ assert cli.get_element_state(project, "target.bst") == "no reference"
# Test that when I fetch a nonexistent URL, errors are handled gracefully and a retry is performed.
@pytest.mark.skipif(HAVE_ARPY is False, reason="arpy is not available")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch"))
def test_fetch_bad_url(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project, tmpdir)
# Try to fetch it
- result = cli.run(project=project, args=[
- 'source', 'fetch', 'target.bst'
- ])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
assert "FAILURE Try #" in result.stderr
result.assert_main_error(ErrorDomain.STREAM, None)
result.assert_task_error(ErrorDomain.SOURCE, None)
@pytest.mark.skipif(HAVE_ARPY is False, reason="arpy is not available")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch"))
def test_fetch_bad_ref(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project, tmpdir)
@@ -71,16 +61,14 @@ def test_fetch_bad_ref(cli, tmpdir, datafiles):
_copy_deb(DATA_DIR, tmpdir)
# Try to fetch it
- result = cli.run(project=project, args=[
- 'source', 'fetch', 'target.bst'
- ])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
result.assert_task_error(ErrorDomain.SOURCE, None)
# Test that when tracking with a ref set, there is a warning
@pytest.mark.skipif(HAVE_ARPY is False, reason="arpy is not available")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch"))
def test_track_warning(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project, tmpdir)
@@ -89,16 +77,14 @@ def test_track_warning(cli, tmpdir, datafiles):
_copy_deb(DATA_DIR, tmpdir)
# Track it
- result = cli.run(project=project, args=[
- 'source', 'track', 'target.bst'
- ])
+ result = cli.run(project=project, args=["source", "track", "target.bst"])
result.assert_success()
assert "Potential man-in-the-middle attack!" in result.stderr
# Test that a staged checkout matches what was tarred up, with the default first subdir
@pytest.mark.skipif(HAVE_ARPY is False, reason="arpy is not available")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch"))
def test_stage_default_basedir(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project, tmpdir)
@@ -108,13 +94,13 @@ def test_stage_default_basedir(cli, tmpdir, datafiles):
_copy_deb(DATA_DIR, tmpdir)
# Track, fetch, build, checkout
- result = cli.run(project=project, args=['source', 'track', 'target.bst'])
+ result = cli.run(project=project, args=["source", "track", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
# Check that the content of the first directory is checked out (base-dir: '')
@@ -126,7 +112,7 @@ def test_stage_default_basedir(cli, tmpdir, datafiles):
# Test that a staged checkout matches what was tarred up, with an empty base-dir
@pytest.mark.skipif(HAVE_ARPY is False, reason="arpy is not available")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-basedir'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "no-basedir"))
def test_stage_no_basedir(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project, tmpdir)
@@ -136,13 +122,13 @@ def test_stage_no_basedir(cli, tmpdir, datafiles):
_copy_deb(DATA_DIR, tmpdir)
# Track, fetch, build, checkout
- result = cli.run(project=project, args=['source', 'track', 'target.bst'])
+ result = cli.run(project=project, args=["source", "track", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
# Check that the full content of the tarball is checked out (base-dir: '')
@@ -154,7 +140,7 @@ def test_stage_no_basedir(cli, tmpdir, datafiles):
# Test that a staged checkout matches what was tarred up, with an explicit basedir
@pytest.mark.skipif(HAVE_ARPY is False, reason="arpy is not available")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'explicit-basedir'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "explicit-basedir"))
def test_stage_explicit_basedir(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project, tmpdir)
@@ -164,13 +150,13 @@ def test_stage_explicit_basedir(cli, tmpdir, datafiles):
_copy_deb(DATA_DIR, tmpdir)
# Track, fetch, build, checkout
- result = cli.run(project=project, args=['source', 'track', 'target.bst'])
+ result = cli.run(project=project, args=["source", "track", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
# Check that the content of the first directory is checked out (base-dir: '')
diff --git a/tests/sources/git.py b/tests/sources/git.py
index 245f90131..fbb1394ec 100644
--- a/tests/sources/git.py
+++ b/tests/sources/git.py
@@ -36,354 +36,304 @@ from buildstream.testing import cli # pylint: disable=unused-import
from buildstream.testing import create_repo
from buildstream.testing._utils.site import HAVE_GIT, HAVE_OLD_GIT
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- 'git',
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "git",)
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "template"))
def test_fetch_bad_ref(cli, tmpdir, datafiles):
project = str(datafiles)
# Create the repo from 'repofiles' subdir
- repo = create_repo('git', str(tmpdir))
- repo.create(os.path.join(project, 'repofiles'))
+ repo = create_repo("git", str(tmpdir))
+ repo.create(os.path.join(project, "repofiles"))
# Write out our test target with a bad ref
- element = {
- 'kind': 'import',
- 'sources': [
- repo.source_config(ref='5')
- ]
- }
- _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst'))
+ element = {"kind": "import", "sources": [repo.source_config(ref="5")]}
+ _yaml.roundtrip_dump(element, os.path.join(project, "target.bst"))
# Assert that fetch raises an error here
- result = cli.run(project=project, args=[
- 'source', 'fetch', 'target.bst'
- ])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
result.assert_task_error(ErrorDomain.SOURCE, None)
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "template"))
def test_submodule_fetch_checkout(cli, tmpdir, datafiles):
project = str(datafiles)
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Create the submodule first from the 'subrepofiles' subdir
- subrepo = create_repo('git', str(tmpdir), 'subrepo')
- subrepo.create(os.path.join(project, 'subrepofiles'))
+ subrepo = create_repo("git", str(tmpdir), "subrepo")
+ subrepo.create(os.path.join(project, "subrepofiles"))
# Create the repo from 'repofiles' subdir
- repo = create_repo('git', str(tmpdir))
- repo.create(os.path.join(project, 'repofiles'))
+ repo = create_repo("git", str(tmpdir))
+ repo.create(os.path.join(project, "repofiles"))
# Add a submodule pointing to the one we created
- ref = repo.add_submodule('subdir', 'file://' + subrepo.repo)
+ ref = repo.add_submodule("subdir", "file://" + subrepo.repo)
# Write out our test target
- element = {
- 'kind': 'import',
- 'sources': [
- repo.source_config(ref=ref)
- ]
- }
- _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst'))
+ element = {"kind": "import", "sources": [repo.source_config(ref=ref)]}
+ _yaml.roundtrip_dump(element, os.path.join(project, "target.bst"))
# Fetch, build, checkout
- result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
# Assert we checked out both files at their expected location
- assert os.path.exists(os.path.join(checkoutdir, 'file.txt'))
- assert os.path.exists(os.path.join(checkoutdir, 'subdir', 'ponyfile.txt'))
+ assert os.path.exists(os.path.join(checkoutdir, "file.txt"))
+ assert os.path.exists(os.path.join(checkoutdir, "subdir", "ponyfile.txt"))
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "template"))
def test_submodule_fetch_source_enable_explicit(cli, tmpdir, datafiles):
project = str(datafiles)
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Create the submodule first from the 'subrepofiles' subdir
- subrepo = create_repo('git', str(tmpdir), 'subrepo')
- subrepo.create(os.path.join(project, 'subrepofiles'))
+ subrepo = create_repo("git", str(tmpdir), "subrepo")
+ subrepo.create(os.path.join(project, "subrepofiles"))
# Create the repo from 'repofiles' subdir
- repo = create_repo('git', str(tmpdir))
- repo.create(os.path.join(project, 'repofiles'))
+ repo = create_repo("git", str(tmpdir))
+ repo.create(os.path.join(project, "repofiles"))
# Add a submodule pointing to the one we created
- ref = repo.add_submodule('subdir', 'file://' + subrepo.repo)
+ ref = repo.add_submodule("subdir", "file://" + subrepo.repo)
# Write out our test target
- element = {
- 'kind': 'import',
- 'sources': [
- repo.source_config_extra(ref=ref, checkout_submodules=True)
- ]
- }
- _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst'))
+ element = {"kind": "import", "sources": [repo.source_config_extra(ref=ref, checkout_submodules=True)]}
+ _yaml.roundtrip_dump(element, os.path.join(project, "target.bst"))
# Fetch, build, checkout
- result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
# Assert we checked out both files at their expected location
- assert os.path.exists(os.path.join(checkoutdir, 'file.txt'))
- assert os.path.exists(os.path.join(checkoutdir, 'subdir', 'ponyfile.txt'))
+ assert os.path.exists(os.path.join(checkoutdir, "file.txt"))
+ assert os.path.exists(os.path.join(checkoutdir, "subdir", "ponyfile.txt"))
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "template"))
def test_submodule_fetch_source_disable(cli, tmpdir, datafiles):
project = str(datafiles)
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Create the submodule first from the 'subrepofiles' subdir
- subrepo = create_repo('git', str(tmpdir), 'subrepo')
- subrepo.create(os.path.join(project, 'subrepofiles'))
+ subrepo = create_repo("git", str(tmpdir), "subrepo")
+ subrepo.create(os.path.join(project, "subrepofiles"))
# Create the repo from 'repofiles' subdir
- repo = create_repo('git', str(tmpdir))
- repo.create(os.path.join(project, 'repofiles'))
+ repo = create_repo("git", str(tmpdir))
+ repo.create(os.path.join(project, "repofiles"))
# Add a submodule pointing to the one we created
- ref = repo.add_submodule('subdir', 'file://' + subrepo.repo)
+ ref = repo.add_submodule("subdir", "file://" + subrepo.repo)
# Write out our test target
- element = {
- 'kind': 'import',
- 'sources': [
- repo.source_config_extra(ref=ref, checkout_submodules=False)
- ]
- }
- _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst'))
+ element = {"kind": "import", "sources": [repo.source_config_extra(ref=ref, checkout_submodules=False)]}
+ _yaml.roundtrip_dump(element, os.path.join(project, "target.bst"))
# Fetch, build, checkout
- result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
# Assert we checked out both files at their expected location
- assert os.path.exists(os.path.join(checkoutdir, 'file.txt'))
- assert not os.path.exists(os.path.join(checkoutdir, 'subdir', 'ponyfile.txt'))
+ assert os.path.exists(os.path.join(checkoutdir, "file.txt"))
+ assert not os.path.exists(os.path.join(checkoutdir, "subdir", "ponyfile.txt"))
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "template"))
def test_submodule_fetch_submodule_does_override(cli, tmpdir, datafiles):
project = str(datafiles)
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Create the submodule first from the 'subrepofiles' subdir
- subrepo = create_repo('git', str(tmpdir), 'subrepo')
- subrepo.create(os.path.join(project, 'subrepofiles'))
+ subrepo = create_repo("git", str(tmpdir), "subrepo")
+ subrepo.create(os.path.join(project, "subrepofiles"))
# Create the repo from 'repofiles' subdir
- repo = create_repo('git', str(tmpdir))
- repo.create(os.path.join(project, 'repofiles'))
+ repo = create_repo("git", str(tmpdir))
+ repo.create(os.path.join(project, "repofiles"))
# Add a submodule pointing to the one we created
- ref = repo.add_submodule('subdir', 'file://' + subrepo.repo, checkout=True)
+ ref = repo.add_submodule("subdir", "file://" + subrepo.repo, checkout=True)
# Write out our test target
- element = {
- 'kind': 'import',
- 'sources': [
- repo.source_config_extra(ref=ref, checkout_submodules=False)
- ]
- }
- _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst'))
+ element = {"kind": "import", "sources": [repo.source_config_extra(ref=ref, checkout_submodules=False)]}
+ _yaml.roundtrip_dump(element, os.path.join(project, "target.bst"))
# Fetch, build, checkout
- result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
# Assert we checked out both files at their expected location
- assert os.path.exists(os.path.join(checkoutdir, 'file.txt'))
- assert os.path.exists(os.path.join(checkoutdir, 'subdir', 'ponyfile.txt'))
+ assert os.path.exists(os.path.join(checkoutdir, "file.txt"))
+ assert os.path.exists(os.path.join(checkoutdir, "subdir", "ponyfile.txt"))
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "template"))
def test_submodule_fetch_submodule_individual_checkout(cli, tmpdir, datafiles):
project = str(datafiles)
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Create the submodule first from the 'subrepofiles' subdir
- subrepo = create_repo('git', str(tmpdir), 'subrepo')
- subrepo.create(os.path.join(project, 'subrepofiles'))
+ subrepo = create_repo("git", str(tmpdir), "subrepo")
+ subrepo.create(os.path.join(project, "subrepofiles"))
# Create another submodule from the 'othersubrepofiles' subdir
- other_subrepo = create_repo('git', str(tmpdir), 'othersubrepo')
- other_subrepo.create(os.path.join(project, 'othersubrepofiles'))
+ other_subrepo = create_repo("git", str(tmpdir), "othersubrepo")
+ other_subrepo.create(os.path.join(project, "othersubrepofiles"))
# Create the repo from 'repofiles' subdir
- repo = create_repo('git', str(tmpdir))
- repo.create(os.path.join(project, 'repofiles'))
+ repo = create_repo("git", str(tmpdir))
+ repo.create(os.path.join(project, "repofiles"))
# Add a submodule pointing to the one we created
- repo.add_submodule('subdir', 'file://' + subrepo.repo, checkout=False)
- ref = repo.add_submodule('othersubdir', 'file://' + other_subrepo.repo)
+ repo.add_submodule("subdir", "file://" + subrepo.repo, checkout=False)
+ ref = repo.add_submodule("othersubdir", "file://" + other_subrepo.repo)
# Write out our test target
- element = {
- 'kind': 'import',
- 'sources': [
- repo.source_config_extra(ref=ref, checkout_submodules=True)
- ]
- }
- _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst'))
+ element = {"kind": "import", "sources": [repo.source_config_extra(ref=ref, checkout_submodules=True)]}
+ _yaml.roundtrip_dump(element, os.path.join(project, "target.bst"))
# Fetch, build, checkout
- result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
# Assert we checked out files at their expected location
- assert os.path.exists(os.path.join(checkoutdir, 'file.txt'))
- assert not os.path.exists(os.path.join(checkoutdir, 'subdir', 'ponyfile.txt'))
- assert os.path.exists(os.path.join(checkoutdir, 'othersubdir', 'unicornfile.txt'))
+ assert os.path.exists(os.path.join(checkoutdir, "file.txt"))
+ assert not os.path.exists(os.path.join(checkoutdir, "subdir", "ponyfile.txt"))
+ assert os.path.exists(os.path.join(checkoutdir, "othersubdir", "unicornfile.txt"))
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "template"))
def test_submodule_fetch_submodule_individual_checkout_explicit(cli, tmpdir, datafiles):
project = str(datafiles)
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Create the submodule first from the 'subrepofiles' subdir
- subrepo = create_repo('git', str(tmpdir), 'subrepo')
- subrepo.create(os.path.join(project, 'subrepofiles'))
+ subrepo = create_repo("git", str(tmpdir), "subrepo")
+ subrepo.create(os.path.join(project, "subrepofiles"))
# Create another submodule from the 'othersubrepofiles' subdir
- other_subrepo = create_repo('git', str(tmpdir), 'othersubrepo')
- other_subrepo.create(os.path.join(project, 'othersubrepofiles'))
+ other_subrepo = create_repo("git", str(tmpdir), "othersubrepo")
+ other_subrepo.create(os.path.join(project, "othersubrepofiles"))
# Create the repo from 'repofiles' subdir
- repo = create_repo('git', str(tmpdir))
- repo.create(os.path.join(project, 'repofiles'))
+ repo = create_repo("git", str(tmpdir))
+ repo.create(os.path.join(project, "repofiles"))
# Add a submodule pointing to the one we created
- repo.add_submodule('subdir', 'file://' + subrepo.repo, checkout=False)
- ref = repo.add_submodule('othersubdir', 'file://' + other_subrepo.repo, checkout=True)
+ repo.add_submodule("subdir", "file://" + subrepo.repo, checkout=False)
+ ref = repo.add_submodule("othersubdir", "file://" + other_subrepo.repo, checkout=True)
# Write out our test target
- element = {
- 'kind': 'import',
- 'sources': [
- repo.source_config_extra(ref=ref, checkout_submodules=True)
- ]
- }
- _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst'))
+ element = {"kind": "import", "sources": [repo.source_config_extra(ref=ref, checkout_submodules=True)]}
+ _yaml.roundtrip_dump(element, os.path.join(project, "target.bst"))
# Fetch, build, checkout
- result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
# Assert we checked out files at their expected location
- assert os.path.exists(os.path.join(checkoutdir, 'file.txt'))
- assert not os.path.exists(os.path.join(checkoutdir, 'subdir', 'ponyfile.txt'))
- assert os.path.exists(os.path.join(checkoutdir, 'othersubdir', 'unicornfile.txt'))
+ assert os.path.exists(os.path.join(checkoutdir, "file.txt"))
+ assert not os.path.exists(os.path.join(checkoutdir, "subdir", "ponyfile.txt"))
+ assert os.path.exists(os.path.join(checkoutdir, "othersubdir", "unicornfile.txt"))
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project-override'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "project-override"))
def test_submodule_fetch_project_override(cli, tmpdir, datafiles):
project = str(datafiles)
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Create the submodule first from the 'subrepofiles' subdir
- subrepo = create_repo('git', str(tmpdir), 'subrepo')
- subrepo.create(os.path.join(project, 'subrepofiles'))
+ subrepo = create_repo("git", str(tmpdir), "subrepo")
+ subrepo.create(os.path.join(project, "subrepofiles"))
# Create the repo from 'repofiles' subdir
- repo = create_repo('git', str(tmpdir))
- repo.create(os.path.join(project, 'repofiles'))
+ repo = create_repo("git", str(tmpdir))
+ repo.create(os.path.join(project, "repofiles"))
# Add a submodule pointing to the one we created
- ref = repo.add_submodule('subdir', 'file://' + subrepo.repo)
+ ref = repo.add_submodule("subdir", "file://" + subrepo.repo)
# Write out our test target
- element = {
- 'kind': 'import',
- 'sources': [
- repo.source_config(ref=ref)
- ]
- }
- _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst'))
+ element = {"kind": "import", "sources": [repo.source_config(ref=ref)]}
+ _yaml.roundtrip_dump(element, os.path.join(project, "target.bst"))
# Fetch, build, checkout
- result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
# Assert we checked out both files at their expected location
- assert os.path.exists(os.path.join(checkoutdir, 'file.txt'))
- assert not os.path.exists(os.path.join(checkoutdir, 'subdir', 'ponyfile.txt'))
+ assert os.path.exists(os.path.join(checkoutdir, "file.txt"))
+ assert not os.path.exists(os.path.join(checkoutdir, "subdir", "ponyfile.txt"))
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "template"))
def test_submodule_track_ignore_inconsistent(cli, tmpdir, datafiles):
project = str(datafiles)
# Create the repo from 'repofiles' subdir
- repo = create_repo('git', str(tmpdir))
- ref = repo.create(os.path.join(project, 'repofiles'))
+ repo = create_repo("git", str(tmpdir))
+ ref = repo.create(os.path.join(project, "repofiles"))
# Write out our test target
- element = {
- 'kind': 'import',
- 'sources': [
- repo.source_config(ref=ref)
- ]
- }
- _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst'))
+ element = {"kind": "import", "sources": [repo.source_config(ref=ref)]}
+ _yaml.roundtrip_dump(element, os.path.join(project, "target.bst"))
# Now add a .gitmodules file with an inconsistent submodule,
# we are calling this inconsistent because the file was created
# but `git submodule add` was never called, so there is no reference
# associated to the submodule.
#
- repo.add_file(os.path.join(project, 'inconsistent-submodule', '.gitmodules'))
+ repo.add_file(os.path.join(project, "inconsistent-submodule", ".gitmodules"))
# Fetch should work, we're not yet at the offending ref
- result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_success()
# Track will encounter an inconsistent submodule without any ref
- result = cli.run(project=project, args=['source', 'track', 'target.bst'])
+ result = cli.run(project=project, args=["source", "track", "target.bst"])
result.assert_success()
# Assert that we are just fine without it, and emit a warning to the user.
@@ -391,68 +341,55 @@ def test_submodule_track_ignore_inconsistent(cli, tmpdir, datafiles):
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "template"))
def test_submodule_track_no_ref_or_track(cli, tmpdir, datafiles):
project = str(datafiles)
# Create the repo from 'repofiles' subdir
- repo = create_repo('git', str(tmpdir))
- repo.create(os.path.join(project, 'repofiles'))
+ repo = create_repo("git", str(tmpdir))
+ repo.create(os.path.join(project, "repofiles"))
# Write out our test target
gitsource = repo.source_config(ref=None)
- gitsource.pop('track')
- element = {
- 'kind': 'import',
- 'sources': [
- gitsource
- ]
- }
+ gitsource.pop("track")
+ element = {"kind": "import", "sources": [gitsource]}
- _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst'))
+ _yaml.roundtrip_dump(element, os.path.join(project, "target.bst"))
# Track will encounter an inconsistent submodule without any ref
- result = cli.run(project=project, args=['show', 'target.bst'])
+ result = cli.run(project=project, args=["show", "target.bst"])
result.assert_main_error(ErrorDomain.SOURCE, "missing-track-and-ref")
result.assert_task_error(None, None)
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
-@pytest.mark.parametrize("fail", ['warn', 'error'])
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "template"))
+@pytest.mark.parametrize("fail", ["warn", "error"])
def test_ref_not_in_track(cli, tmpdir, datafiles, fail):
project = str(datafiles)
# Make the warning an error if we're testing errors
- if fail == 'error':
- project_template = {
- "name": "foo",
- "fatal-warnings": [CoreWarnings.REF_NOT_IN_TRACK]
- }
- _yaml.roundtrip_dump(project_template, os.path.join(project, 'project.conf'))
+ if fail == "error":
+ project_template = {"name": "foo", "fatal-warnings": [CoreWarnings.REF_NOT_IN_TRACK]}
+ _yaml.roundtrip_dump(project_template, os.path.join(project, "project.conf"))
# Create the repo from 'repofiles', create a branch without latest commit
- repo = create_repo('git', str(tmpdir))
- ref = repo.create(os.path.join(project, 'repofiles'))
+ repo = create_repo("git", str(tmpdir))
+ ref = repo.create(os.path.join(project, "repofiles"))
gitsource = repo.source_config(ref=ref)
# Overwrite the track value to the added branch
- gitsource['track'] = 'foo'
+ gitsource["track"] = "foo"
# Write out our test target
- element = {
- 'kind': 'import',
- 'sources': [
- gitsource
- ]
- }
- _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst'))
+ element = {"kind": "import", "sources": [gitsource]}
+ _yaml.roundtrip_dump(element, os.path.join(project, "target.bst"))
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
# Assert a warning or an error depending on what we're checking
- if fail == 'error':
+ if fail == "error":
result.assert_main_error(ErrorDomain.STREAM, None)
result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.REF_NOT_IN_TRACK)
else:
@@ -461,29 +398,26 @@ def test_ref_not_in_track(cli, tmpdir, datafiles, fail):
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
-@pytest.mark.parametrize("fail", ['warn', 'error'])
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "template"))
+@pytest.mark.parametrize("fail", ["warn", "error"])
def test_unlisted_submodule(cli, tmpdir, datafiles, fail):
project = str(datafiles)
# Make the warning an error if we're testing errors
- if fail == 'error':
- project_template = {
- "name": "foo",
- "fatal-warnings": ['git:unlisted-submodule']
- }
- _yaml.roundtrip_dump(project_template, os.path.join(project, 'project.conf'))
+ if fail == "error":
+ project_template = {"name": "foo", "fatal-warnings": ["git:unlisted-submodule"]}
+ _yaml.roundtrip_dump(project_template, os.path.join(project, "project.conf"))
# Create the submodule first from the 'subrepofiles' subdir
- subrepo = create_repo('git', str(tmpdir), 'subrepo')
- subrepo.create(os.path.join(project, 'subrepofiles'))
+ subrepo = create_repo("git", str(tmpdir), "subrepo")
+ subrepo.create(os.path.join(project, "subrepofiles"))
# Create the repo from 'repofiles' subdir
- repo = create_repo('git', str(tmpdir))
- repo.create(os.path.join(project, 'repofiles'))
+ repo = create_repo("git", str(tmpdir))
+ repo.create(os.path.join(project, "repofiles"))
# Add a submodule pointing to the one we created
- ref = repo.add_submodule('subdir', 'file://' + subrepo.repo)
+ ref = repo.add_submodule("subdir", "file://" + subrepo.repo)
# Create the source, and delete the explicit configuration
# of the submodules.
@@ -492,127 +426,111 @@ def test_unlisted_submodule(cli, tmpdir, datafiles, fail):
# after the source has been fetched.
#
gitsource = repo.source_config(ref=ref)
- del gitsource['submodules']
+ del gitsource["submodules"]
# Write out our test target
- element = {
- 'kind': 'import',
- 'sources': [
- gitsource
- ]
- }
- _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst'))
+ element = {"kind": "import", "sources": [gitsource]}
+ _yaml.roundtrip_dump(element, os.path.join(project, "target.bst"))
# We will not see the warning or error before the first fetch, because
# we don't have the repository yet and so we have no knowledge of
# the unlisted submodule.
- result = cli.run(project=project, args=['show', 'target.bst'])
+ result = cli.run(project=project, args=["show", "target.bst"])
result.assert_success()
assert "git:unlisted-submodule" not in result.stderr
# We will notice this directly in fetch, as it will try to fetch
# the submodules it discovers as a result of fetching the primary repo.
- result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
# Assert a warning or an error depending on what we're checking
- if fail == 'error':
+ if fail == "error":
result.assert_main_error(ErrorDomain.STREAM, None)
- result.assert_task_error(ErrorDomain.PLUGIN, 'git:unlisted-submodule')
+ result.assert_task_error(ErrorDomain.PLUGIN, "git:unlisted-submodule")
else:
result.assert_success()
assert "git:unlisted-submodule" in result.stderr
# Now that we've fetched it, `bst show` will discover the unlisted submodule too
- result = cli.run(project=project, args=['show', 'target.bst'])
+ result = cli.run(project=project, args=["show", "target.bst"])
# Assert a warning or an error depending on what we're checking
- if fail == 'error':
- result.assert_main_error(ErrorDomain.PLUGIN, 'git:unlisted-submodule')
+ if fail == "error":
+ result.assert_main_error(ErrorDomain.PLUGIN, "git:unlisted-submodule")
else:
result.assert_success()
assert "git:unlisted-submodule" in result.stderr
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
-@pytest.mark.parametrize("fail", ['warn', 'error'])
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "template"))
+@pytest.mark.parametrize("fail", ["warn", "error"])
def test_track_unlisted_submodule(cli, tmpdir, datafiles, fail):
project = str(datafiles)
# Make the warning an error if we're testing errors
- if fail == 'error':
- project_template = {
- "name": "foo",
- "fatal-warnings": ['git:unlisted-submodule']
- }
- _yaml.roundtrip_dump(project_template, os.path.join(project, 'project.conf'))
+ if fail == "error":
+ project_template = {"name": "foo", "fatal-warnings": ["git:unlisted-submodule"]}
+ _yaml.roundtrip_dump(project_template, os.path.join(project, "project.conf"))
# Create the submodule first from the 'subrepofiles' subdir
- subrepo = create_repo('git', str(tmpdir), 'subrepo')
- subrepo.create(os.path.join(project, 'subrepofiles'))
+ subrepo = create_repo("git", str(tmpdir), "subrepo")
+ subrepo.create(os.path.join(project, "subrepofiles"))
# Create the repo from 'repofiles' subdir
- repo = create_repo('git', str(tmpdir))
- ref = repo.create(os.path.join(project, 'repofiles'))
+ repo = create_repo("git", str(tmpdir))
+ ref = repo.create(os.path.join(project, "repofiles"))
# Add a submodule pointing to the one we created, but use
# the original ref, let the submodules appear after tracking
- repo.add_submodule('subdir', 'file://' + subrepo.repo)
+ repo.add_submodule("subdir", "file://" + subrepo.repo)
# Create the source, and delete the explicit configuration
# of the submodules.
gitsource = repo.source_config(ref=ref)
- del gitsource['submodules']
+ del gitsource["submodules"]
# Write out our test target
- element = {
- 'kind': 'import',
- 'sources': [
- gitsource
- ]
- }
- _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst'))
+ element = {"kind": "import", "sources": [gitsource]}
+ _yaml.roundtrip_dump(element, os.path.join(project, "target.bst"))
# Fetch the repo, we will not see the warning because we
# are still pointing to a ref which predates the submodules
- result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_success()
assert "git:unlisted-submodule" not in result.stderr
# We won't get a warning/error when tracking either, the source
# has not become Consistency.CACHED so the opportunity to check
# for the warning has not yet arisen.
- result = cli.run(project=project, args=['source', 'track', 'target.bst'])
+ result = cli.run(project=project, args=["source", "track", "target.bst"])
result.assert_success()
assert "git:unlisted-submodule" not in result.stderr
# Fetching the repo at the new ref will finally reveal the warning
- result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
- if fail == 'error':
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
+ if fail == "error":
result.assert_main_error(ErrorDomain.STREAM, None)
- result.assert_task_error(ErrorDomain.PLUGIN, 'git:unlisted-submodule')
+ result.assert_task_error(ErrorDomain.PLUGIN, "git:unlisted-submodule")
else:
result.assert_success()
assert "git:unlisted-submodule" in result.stderr
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
-@pytest.mark.parametrize("fail", ['warn', 'error'])
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "template"))
+@pytest.mark.parametrize("fail", ["warn", "error"])
def test_invalid_submodule(cli, tmpdir, datafiles, fail):
project = str(datafiles)
# Make the warning an error if we're testing errors
- if fail == 'error':
- project_template = {
- "name": "foo",
- "fatal-warnings": ['git:invalid-submodule']
- }
- _yaml.roundtrip_dump(project_template, os.path.join(project, 'project.conf'))
+ if fail == "error":
+ project_template = {"name": "foo", "fatal-warnings": ["git:invalid-submodule"]}
+ _yaml.roundtrip_dump(project_template, os.path.join(project, "project.conf"))
# Create the repo from 'repofiles' subdir
- repo = create_repo('git', str(tmpdir))
- ref = repo.create(os.path.join(project, 'repofiles'))
+ repo = create_repo("git", str(tmpdir))
+ ref = repo.create(os.path.join(project, "repofiles"))
# Create the source without any submodules, and add
# an invalid submodule configuration to it.
@@ -622,46 +540,37 @@ def test_invalid_submodule(cli, tmpdir, datafiles, fail):
# the real submodules actually are.
#
gitsource = repo.source_config(ref=ref)
- gitsource['submodules'] = {
- 'subdir': {
- 'url': 'https://pony.org/repo.git'
- }
- }
+ gitsource["submodules"] = {"subdir": {"url": "https://pony.org/repo.git"}}
# Write out our test target
- element = {
- 'kind': 'import',
- 'sources': [
- gitsource
- ]
- }
- _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst'))
+ element = {"kind": "import", "sources": [gitsource]}
+ _yaml.roundtrip_dump(element, os.path.join(project, "target.bst"))
# We will not see the warning or error before the first fetch, because
# we don't have the repository yet and so we have no knowledge of
# the unlisted submodule.
- result = cli.run(project=project, args=['show', 'target.bst'])
+ result = cli.run(project=project, args=["show", "target.bst"])
result.assert_success()
assert "git:invalid-submodule" not in result.stderr
# We will notice this directly in fetch, as it will try to fetch
# the submodules it discovers as a result of fetching the primary repo.
- result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
# Assert a warning or an error depending on what we're checking
- if fail == 'error':
+ if fail == "error":
result.assert_main_error(ErrorDomain.STREAM, None)
- result.assert_task_error(ErrorDomain.PLUGIN, 'git:invalid-submodule')
+ result.assert_task_error(ErrorDomain.PLUGIN, "git:invalid-submodule")
else:
result.assert_success()
assert "git:invalid-submodule" in result.stderr
# Now that we've fetched it, `bst show` will discover the unlisted submodule too
- result = cli.run(project=project, args=['show', 'target.bst'])
+ result = cli.run(project=project, args=["show", "target.bst"])
# Assert a warning or an error depending on what we're checking
- if fail == 'error':
- result.assert_main_error(ErrorDomain.PLUGIN, 'git:invalid-submodule')
+ if fail == "error":
+ result.assert_main_error(ErrorDomain.PLUGIN, "git:invalid-submodule")
else:
result.assert_success()
assert "git:invalid-submodule" in result.stderr
@@ -669,49 +578,41 @@ def test_invalid_submodule(cli, tmpdir, datafiles, fail):
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
@pytest.mark.skipif(HAVE_OLD_GIT, reason="old git rm does not update .gitmodules")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
-@pytest.mark.parametrize("fail", ['warn', 'error'])
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "template"))
+@pytest.mark.parametrize("fail", ["warn", "error"])
def test_track_invalid_submodule(cli, tmpdir, datafiles, fail):
project = str(datafiles)
# Make the warning an error if we're testing errors
- if fail == 'error':
- project_template = {
- "name": "foo",
- "fatal-warnings": ['git:invalid-submodule']
- }
- _yaml.roundtrip_dump(project_template, os.path.join(project, 'project.conf'))
+ if fail == "error":
+ project_template = {"name": "foo", "fatal-warnings": ["git:invalid-submodule"]}
+ _yaml.roundtrip_dump(project_template, os.path.join(project, "project.conf"))
# Create the submodule first from the 'subrepofiles' subdir
- subrepo = create_repo('git', str(tmpdir), 'subrepo')
- subrepo.create(os.path.join(project, 'subrepofiles'))
+ subrepo = create_repo("git", str(tmpdir), "subrepo")
+ subrepo.create(os.path.join(project, "subrepofiles"))
# Create the repo from 'repofiles' subdir
- repo = create_repo('git', str(tmpdir))
- repo.create(os.path.join(project, 'repofiles'))
+ repo = create_repo("git", str(tmpdir))
+ repo.create(os.path.join(project, "repofiles"))
# Add a submodule pointing to the one we created
- ref = repo.add_submodule('subdir', 'file://' + subrepo.repo)
+ ref = repo.add_submodule("subdir", "file://" + subrepo.repo)
# Add a commit beyond the ref which *removes* the submodule we've added
- repo.remove_path('subdir')
+ repo.remove_path("subdir")
# Create the source, this will keep the submodules so initially
# the configuration is valid for the ref we're using
gitsource = repo.source_config(ref=ref)
# Write out our test target
- element = {
- 'kind': 'import',
- 'sources': [
- gitsource
- ]
- }
- _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst'))
+ element = {"kind": "import", "sources": [gitsource]}
+ _yaml.roundtrip_dump(element, os.path.join(project, "target.bst"))
# Fetch the repo, we will not see the warning because we
# are still pointing to a ref which predates the submodules
- result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_success()
assert "git:invalid-submodule" not in result.stderr
@@ -720,417 +621,392 @@ def test_track_invalid_submodule(cli, tmpdir, datafiles, fail):
# not locally cached, the Source will be CACHED directly after
# tracking and the validations will occur as a result.
#
- result = cli.run(project=project, args=['source', 'track', 'target.bst'])
- if fail == 'error':
+ result = cli.run(project=project, args=["source", "track", "target.bst"])
+ if fail == "error":
result.assert_main_error(ErrorDomain.STREAM, None)
- result.assert_task_error(ErrorDomain.PLUGIN, 'git:invalid-submodule')
+ result.assert_task_error(ErrorDomain.PLUGIN, "git:invalid-submodule")
else:
result.assert_success()
assert "git:invalid-submodule" in result.stderr
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
-@pytest.mark.parametrize("ref_format", ['sha1', 'git-describe'])
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "template"))
+@pytest.mark.parametrize("ref_format", ["sha1", "git-describe"])
@pytest.mark.parametrize("tag,extra_commit", [(False, False), (True, False), (True, True)])
def test_track_fetch(cli, tmpdir, datafiles, ref_format, tag, extra_commit):
project = str(datafiles)
# Create the repo from 'repofiles' subdir
- repo = create_repo('git', str(tmpdir))
- repo.create(os.path.join(project, 'repofiles'))
+ repo = create_repo("git", str(tmpdir))
+ repo.create(os.path.join(project, "repofiles"))
if tag:
- repo.add_tag('tag')
+ repo.add_tag("tag")
if extra_commit:
repo.add_commit()
# Write out our test target
- element = {
- 'kind': 'import',
- 'sources': [
- repo.source_config()
- ]
- }
- element['sources'][0]['ref-format'] = ref_format
- element_path = os.path.join(project, 'target.bst')
+ element = {"kind": "import", "sources": [repo.source_config()]}
+ element["sources"][0]["ref-format"] = ref_format
+ element_path = os.path.join(project, "target.bst")
_yaml.roundtrip_dump(element, element_path)
# Track it
- result = cli.run(project=project, args=['source', 'track', 'target.bst'])
+ result = cli.run(project=project, args=["source", "track", "target.bst"])
result.assert_success()
element = _yaml.load(element_path)
- new_ref = element.get_sequence('sources').mapping_at(0).get_str('ref')
+ new_ref = element.get_sequence("sources").mapping_at(0).get_str("ref")
- if ref_format == 'git-describe' and tag:
+ if ref_format == "git-describe" and tag:
# Check and strip prefix
- prefix = 'tag-{}-g'.format(0 if not extra_commit else 1)
+ prefix = "tag-{}-g".format(0 if not extra_commit else 1)
assert new_ref.startswith(prefix)
- new_ref = new_ref[len(prefix):]
+ new_ref = new_ref[len(prefix) :]
# 40 chars for SHA-1
assert len(new_ref) == 40
# Fetch it
- result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_success()
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
@pytest.mark.skipif(HAVE_OLD_GIT, reason="old git describe lacks --first-parent")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
-@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
-@pytest.mark.parametrize("tag_type", [('annotated'), ('lightweight')])
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "template"))
+@pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")])
+@pytest.mark.parametrize("tag_type", [("annotated"), ("lightweight")])
def test_git_describe(cli, tmpdir, datafiles, ref_storage, tag_type):
project = str(datafiles)
- project_config = _yaml.load(os.path.join(project, 'project.conf'))
- project_config['ref-storage'] = ref_storage
- _yaml.roundtrip_dump(project_config, os.path.join(project, 'project.conf'))
+ project_config = _yaml.load(os.path.join(project, "project.conf"))
+ project_config["ref-storage"] = ref_storage
+ _yaml.roundtrip_dump(project_config, os.path.join(project, "project.conf"))
- repofiles = os.path.join(str(tmpdir), 'repofiles')
+ repofiles = os.path.join(str(tmpdir), "repofiles")
os.makedirs(repofiles, exist_ok=True)
- file0 = os.path.join(repofiles, 'file0')
- with open(file0, 'w') as f:
- f.write('test\n')
+ file0 = os.path.join(repofiles, "file0")
+ with open(file0, "w") as f:
+ f.write("test\n")
- repo = create_repo('git', str(tmpdir))
+ repo = create_repo("git", str(tmpdir))
def tag(name):
- if tag_type == 'annotated':
+ if tag_type == "annotated":
repo.add_annotated_tag(name, name)
else:
repo.add_tag(name)
repo.create(repofiles)
- tag('uselesstag')
+ tag("uselesstag")
- file1 = os.path.join(str(tmpdir), 'file1')
- with open(file1, 'w') as f:
- f.write('test\n')
+ file1 = os.path.join(str(tmpdir), "file1")
+ with open(file1, "w") as f:
+ f.write("test\n")
repo.add_file(file1)
- tag('tag1')
+ tag("tag1")
- file2 = os.path.join(str(tmpdir), 'file2')
- with open(file2, 'w') as f:
- f.write('test\n')
- repo.branch('branch2')
+ file2 = os.path.join(str(tmpdir), "file2")
+ with open(file2, "w") as f:
+ f.write("test\n")
+ repo.branch("branch2")
repo.add_file(file2)
- tag('tag2')
+ tag("tag2")
- repo.checkout('master')
- file3 = os.path.join(str(tmpdir), 'file3')
- with open(file3, 'w') as f:
- f.write('test\n')
+ repo.checkout("master")
+ file3 = os.path.join(str(tmpdir), "file3")
+ with open(file3, "w") as f:
+ f.write("test\n")
repo.add_file(file3)
- repo.merge('branch2')
+ repo.merge("branch2")
config = repo.source_config()
- config['track'] = repo.latest_commit()
- config['track-tags'] = True
+ config["track"] = repo.latest_commit()
+ config["track-tags"] = True
# Write out our test target
element = {
- 'kind': 'import',
- 'sources': [
- config
- ],
+ "kind": "import",
+ "sources": [config],
}
- element_path = os.path.join(project, 'target.bst')
+ element_path = os.path.join(project, "target.bst")
_yaml.roundtrip_dump(element, element_path)
- if ref_storage == 'inline':
- result = cli.run(project=project, args=['source', 'track', 'target.bst'])
+ if ref_storage == "inline":
+ result = cli.run(project=project, args=["source", "track", "target.bst"])
result.assert_success()
else:
- result = cli.run(project=project, args=['source', 'track', 'target.bst', '--deps', 'all'])
+ result = cli.run(project=project, args=["source", "track", "target.bst", "--deps", "all"])
result.assert_success()
- if ref_storage == 'inline':
+ if ref_storage == "inline":
element = _yaml.load(element_path)
- tags = element.get_sequence('sources').mapping_at(0).get_sequence('tags')
+ tags = element.get_sequence("sources").mapping_at(0).get_sequence("tags")
assert len(tags) == 2
for tag in tags:
- assert 'tag' in tag
- assert 'commit' in tag
- assert 'annotated' in tag
- assert tag.get_bool('annotated') == (tag_type == 'annotated')
-
- assert {(tag.get_str('tag'),
- tag.get_str('commit'))
- for tag in tags} == {('tag1', repo.rev_parse('tag1^{commit}')),
- ('tag2', repo.rev_parse('tag2^{commit}'))}
+ assert "tag" in tag
+ assert "commit" in tag
+ assert "annotated" in tag
+ assert tag.get_bool("annotated") == (tag_type == "annotated")
+
+ assert {(tag.get_str("tag"), tag.get_str("commit")) for tag in tags} == {
+ ("tag1", repo.rev_parse("tag1^{commit}")),
+ ("tag2", repo.rev_parse("tag2^{commit}")),
+ }
- checkout = os.path.join(str(tmpdir), 'checkout')
+ checkout = os.path.join(str(tmpdir), "checkout")
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkout])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkout])
result.assert_success()
- if tag_type == 'annotated':
+ if tag_type == "annotated":
options = []
else:
- options = ['--tags']
- describe = subprocess.check_output(['git', 'describe', *options],
- cwd=checkout, universal_newlines=True)
- assert describe.startswith('tag2-2-')
+ options = ["--tags"]
+ describe = subprocess.check_output(["git", "describe", *options], cwd=checkout, universal_newlines=True)
+ assert describe.startswith("tag2-2-")
- describe_fp = subprocess.check_output(['git', 'describe', '--first-parent', *options],
- cwd=checkout, universal_newlines=True)
- assert describe_fp.startswith('tag1-2-')
+ describe_fp = subprocess.check_output(
+ ["git", "describe", "--first-parent", *options], cwd=checkout, universal_newlines=True
+ )
+ assert describe_fp.startswith("tag1-2-")
- tags = subprocess.check_output(['git', 'tag'],
- cwd=checkout, universal_newlines=True)
+ tags = subprocess.check_output(["git", "tag"], cwd=checkout, universal_newlines=True)
tags = set(tags.splitlines())
- assert tags == set(['tag1', 'tag2'])
+ assert tags == set(["tag1", "tag2"])
- p = subprocess.run(['git', 'log', repo.rev_parse('uselesstag')],
- cwd=checkout)
+ p = subprocess.run(["git", "log", repo.rev_parse("uselesstag")], cwd=checkout)
assert p.returncode != 0
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
-@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
-@pytest.mark.parametrize("tag_type", [('annotated'), ('lightweight')])
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "template"))
+@pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")])
+@pytest.mark.parametrize("tag_type", [("annotated"), ("lightweight")])
def test_git_describe_head_is_tagged(cli, tmpdir, datafiles, ref_storage, tag_type):
project = str(datafiles)
- project_config = _yaml.load(os.path.join(project, 'project.conf'))
- project_config['ref-storage'] = ref_storage
- _yaml.roundtrip_dump(project_config, os.path.join(project, 'project.conf'))
+ project_config = _yaml.load(os.path.join(project, "project.conf"))
+ project_config["ref-storage"] = ref_storage
+ _yaml.roundtrip_dump(project_config, os.path.join(project, "project.conf"))
- repofiles = os.path.join(str(tmpdir), 'repofiles')
+ repofiles = os.path.join(str(tmpdir), "repofiles")
os.makedirs(repofiles, exist_ok=True)
- file0 = os.path.join(repofiles, 'file0')
- with open(file0, 'w') as f:
- f.write('test\n')
+ file0 = os.path.join(repofiles, "file0")
+ with open(file0, "w") as f:
+ f.write("test\n")
- repo = create_repo('git', str(tmpdir))
+ repo = create_repo("git", str(tmpdir))
def tag(name):
- if tag_type == 'annotated':
+ if tag_type == "annotated":
repo.add_annotated_tag(name, name)
else:
repo.add_tag(name)
repo.create(repofiles)
- tag('uselesstag')
+ tag("uselesstag")
- file1 = os.path.join(str(tmpdir), 'file1')
- with open(file1, 'w') as f:
- f.write('test\n')
+ file1 = os.path.join(str(tmpdir), "file1")
+ with open(file1, "w") as f:
+ f.write("test\n")
repo.add_file(file1)
- file2 = os.path.join(str(tmpdir), 'file2')
- with open(file2, 'w') as f:
- f.write('test\n')
- repo.branch('branch2')
+ file2 = os.path.join(str(tmpdir), "file2")
+ with open(file2, "w") as f:
+ f.write("test\n")
+ repo.branch("branch2")
repo.add_file(file2)
- repo.checkout('master')
- file3 = os.path.join(str(tmpdir), 'file3')
- with open(file3, 'w') as f:
- f.write('test\n')
+ repo.checkout("master")
+ file3 = os.path.join(str(tmpdir), "file3")
+ with open(file3, "w") as f:
+ f.write("test\n")
repo.add_file(file3)
- tagged_ref = repo.merge('branch2')
- tag('tag')
+ tagged_ref = repo.merge("branch2")
+ tag("tag")
config = repo.source_config()
- config['track'] = repo.latest_commit()
- config['track-tags'] = True
+ config["track"] = repo.latest_commit()
+ config["track-tags"] = True
# Write out our test target
element = {
- 'kind': 'import',
- 'sources': [
- config
- ],
+ "kind": "import",
+ "sources": [config],
}
- element_path = os.path.join(project, 'target.bst')
+ element_path = os.path.join(project, "target.bst")
_yaml.roundtrip_dump(element, element_path)
- if ref_storage == 'inline':
- result = cli.run(project=project, args=['source', 'track', 'target.bst'])
+ if ref_storage == "inline":
+ result = cli.run(project=project, args=["source", "track", "target.bst"])
result.assert_success()
else:
- result = cli.run(project=project, args=['source', 'track', 'target.bst', '--deps', 'all'])
+ result = cli.run(project=project, args=["source", "track", "target.bst", "--deps", "all"])
result.assert_success()
- if ref_storage == 'inline':
+ if ref_storage == "inline":
element = _yaml.load(element_path)
- source = element.get_sequence('sources').mapping_at(0)
- tags = source.get_sequence('tags')
+ source = element.get_sequence("sources").mapping_at(0)
+ tags = source.get_sequence("tags")
assert len(tags) == 1
- tag = source.get_sequence('tags').mapping_at(0)
- assert 'tag' in tag
- assert 'commit' in tag
- assert 'annotated' in tag
- assert tag.get_bool('annotated') == (tag_type == 'annotated')
+ tag = source.get_sequence("tags").mapping_at(0)
+ assert "tag" in tag
+ assert "commit" in tag
+ assert "annotated" in tag
+ assert tag.get_bool("annotated") == (tag_type == "annotated")
- tag_name = tag.get_str('tag')
- commit = tag.get_str('commit')
- assert (tag_name, commit) == ('tag', repo.rev_parse('tag^{commit}'))
+ tag_name = tag.get_str("tag")
+ commit = tag.get_str("commit")
+ assert (tag_name, commit) == ("tag", repo.rev_parse("tag^{commit}"))
- checkout = os.path.join(str(tmpdir), 'checkout')
+ checkout = os.path.join(str(tmpdir), "checkout")
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkout])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkout])
result.assert_success()
- if tag_type == 'annotated':
+ if tag_type == "annotated":
options = []
else:
- options = ['--tags']
- describe = subprocess.check_output(['git', 'describe', *options],
- cwd=checkout, universal_newlines=True)
- assert describe.startswith('tag')
-
- tags = subprocess.check_output(['git', 'tag'],
- cwd=checkout,
- universal_newlines=True)
+ options = ["--tags"]
+ describe = subprocess.check_output(["git", "describe", *options], cwd=checkout, universal_newlines=True)
+ assert describe.startswith("tag")
+
+ tags = subprocess.check_output(["git", "tag"], cwd=checkout, universal_newlines=True)
tags = set(tags.splitlines())
- assert tags == set(['tag'])
+ assert tags == set(["tag"])
- rev_list = subprocess.check_output(['git', 'rev-list', '--all'],
- cwd=checkout,
- universal_newlines=True)
+ rev_list = subprocess.check_output(["git", "rev-list", "--all"], cwd=checkout, universal_newlines=True)
assert set(rev_list.splitlines()) == set([tagged_ref])
- p = subprocess.run(['git', 'log', repo.rev_parse('uselesstag')],
- cwd=checkout)
+ p = subprocess.run(["git", "log", repo.rev_parse("uselesstag")], cwd=checkout)
assert p.returncode != 0
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "template"))
def test_git_describe_relevant_history(cli, tmpdir, datafiles):
project = str(datafiles)
- project_config = _yaml.load(os.path.join(project, 'project.conf'))
- project_config['ref-storage'] = 'project.refs'
- _yaml.roundtrip_dump(project_config, os.path.join(project, 'project.conf'))
+ project_config = _yaml.load(os.path.join(project, "project.conf"))
+ project_config["ref-storage"] = "project.refs"
+ _yaml.roundtrip_dump(project_config, os.path.join(project, "project.conf"))
- repofiles = os.path.join(str(tmpdir), 'repofiles')
+ repofiles = os.path.join(str(tmpdir), "repofiles")
os.makedirs(repofiles, exist_ok=True)
- file0 = os.path.join(repofiles, 'file0')
- with open(file0, 'w') as f:
- f.write('test\n')
+ file0 = os.path.join(repofiles, "file0")
+ with open(file0, "w") as f:
+ f.write("test\n")
- repo = create_repo('git', str(tmpdir))
+ repo = create_repo("git", str(tmpdir))
repo.create(repofiles)
- file1 = os.path.join(str(tmpdir), 'file1')
- with open(file1, 'w') as f:
- f.write('test\n')
+ file1 = os.path.join(str(tmpdir), "file1")
+ with open(file1, "w") as f:
+ f.write("test\n")
repo.add_file(file1)
- repo.branch('branch')
- repo.checkout('master')
+ repo.branch("branch")
+ repo.checkout("master")
- file2 = os.path.join(str(tmpdir), 'file2')
- with open(file2, 'w') as f:
- f.write('test\n')
+ file2 = os.path.join(str(tmpdir), "file2")
+ with open(file2, "w") as f:
+ f.write("test\n")
repo.add_file(file2)
- file3 = os.path.join(str(tmpdir), 'file3')
- with open(file3, 'w') as f:
- f.write('test\n')
+ file3 = os.path.join(str(tmpdir), "file3")
+ with open(file3, "w") as f:
+ f.write("test\n")
branch_boundary = repo.add_file(file3)
- repo.checkout('branch')
- file4 = os.path.join(str(tmpdir), 'file4')
- with open(file4, 'w') as f:
- f.write('test\n')
+ repo.checkout("branch")
+ file4 = os.path.join(str(tmpdir), "file4")
+ with open(file4, "w") as f:
+ f.write("test\n")
tagged_ref = repo.add_file(file4)
- repo.add_annotated_tag('tag1', 'tag1')
+ repo.add_annotated_tag("tag1", "tag1")
- head = repo.merge('master')
+ head = repo.merge("master")
config = repo.source_config()
- config['track'] = head
- config['track-tags'] = True
+ config["track"] = head
+ config["track-tags"] = True
# Write out our test target
element = {
- 'kind': 'import',
- 'sources': [
- config
- ],
+ "kind": "import",
+ "sources": [config],
}
- element_path = os.path.join(project, 'target.bst')
+ element_path = os.path.join(project, "target.bst")
_yaml.roundtrip_dump(element, element_path)
- result = cli.run(project=project, args=['source', 'track', 'target.bst', '--deps', 'all'])
+ result = cli.run(project=project, args=["source", "track", "target.bst", "--deps", "all"])
result.assert_success()
- checkout = os.path.join(str(tmpdir), 'checkout')
+ checkout = os.path.join(str(tmpdir), "checkout")
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkout])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkout])
result.assert_success()
- describe = subprocess.check_output(['git', 'describe'],
- cwd=checkout,
- universal_newlines=True)
- assert describe.startswith('tag1-2-')
+ describe = subprocess.check_output(["git", "describe"], cwd=checkout, universal_newlines=True)
+ assert describe.startswith("tag1-2-")
- rev_list = subprocess.check_output(['git', 'rev-list', '--all'],
- cwd=checkout,
- universal_newlines=True)
+ rev_list = subprocess.check_output(["git", "rev-list", "--all"], cwd=checkout, universal_newlines=True)
assert set(rev_list.splitlines()) == set([head, tagged_ref, branch_boundary])
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "template"))
def test_default_do_not_track_tags(cli, tmpdir, datafiles):
project = str(datafiles)
- project_config = _yaml.load(os.path.join(project, 'project.conf'))
- project_config['ref-storage'] = 'inline'
- _yaml.roundtrip_dump(project_config, os.path.join(project, 'project.conf'))
+ project_config = _yaml.load(os.path.join(project, "project.conf"))
+ project_config["ref-storage"] = "inline"
+ _yaml.roundtrip_dump(project_config, os.path.join(project, "project.conf"))
- repofiles = os.path.join(str(tmpdir), 'repofiles')
+ repofiles = os.path.join(str(tmpdir), "repofiles")
os.makedirs(repofiles, exist_ok=True)
- file0 = os.path.join(repofiles, 'file0')
- with open(file0, 'w') as f:
- f.write('test\n')
+ file0 = os.path.join(repofiles, "file0")
+ with open(file0, "w") as f:
+ f.write("test\n")
- repo = create_repo('git', str(tmpdir))
+ repo = create_repo("git", str(tmpdir))
repo.create(repofiles)
- repo.add_tag('tag')
+ repo.add_tag("tag")
config = repo.source_config()
- config['track'] = repo.latest_commit()
+ config["track"] = repo.latest_commit()
# Write out our test target
element = {
- 'kind': 'import',
- 'sources': [
- config
- ],
+ "kind": "import",
+ "sources": [config],
}
- element_path = os.path.join(project, 'target.bst')
+ element_path = os.path.join(project, "target.bst")
_yaml.roundtrip_dump(element, element_path)
- result = cli.run(project=project, args=['source', 'track', 'target.bst'])
+ result = cli.run(project=project, args=["source", "track", "target.bst"])
result.assert_success()
element = _yaml.load(element_path)
- source = element.get_sequence('sources').mapping_at(0)
- assert 'tags' not in source
+ source = element.get_sequence("sources").mapping_at(0)
+ assert "tags" not in source
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "template"))
def test_overwrite_rogue_tag_multiple_remotes(cli, tmpdir, datafiles):
"""When using multiple remotes in cache (i.e. when using aliases), we
need to make sure we override tags. This is not allowed to fetch
@@ -1139,88 +1015,74 @@ def test_overwrite_rogue_tag_multiple_remotes(cli, tmpdir, datafiles):
project = str(datafiles)
- repofiles = os.path.join(str(tmpdir), 'repofiles')
+ repofiles = os.path.join(str(tmpdir), "repofiles")
os.makedirs(repofiles, exist_ok=True)
- file0 = os.path.join(repofiles, 'file0')
- with open(file0, 'w') as f:
- f.write('test\n')
+ file0 = os.path.join(repofiles, "file0")
+ with open(file0, "w") as f:
+ f.write("test\n")
- repo = create_repo('git', str(tmpdir))
+ repo = create_repo("git", str(tmpdir))
top_commit = repo.create(repofiles)
repodir, reponame = os.path.split(repo.repo)
- project_config = _yaml.load(os.path.join(project, 'project.conf'))
- project_config['aliases'] = Node.from_dict({
- 'repo': 'http://example.com/'
- })
- project_config['mirrors'] = [
- {
- 'name': 'middle-earth',
- 'aliases': {
- 'repo': ['file://{}/'.format(repodir)]
- }
- }
- ]
- _yaml.roundtrip_dump(project_config, os.path.join(project, 'project.conf'))
+ project_config = _yaml.load(os.path.join(project, "project.conf"))
+ project_config["aliases"] = Node.from_dict({"repo": "http://example.com/"})
+ project_config["mirrors"] = [{"name": "middle-earth", "aliases": {"repo": ["file://{}/".format(repodir)]}}]
+ _yaml.roundtrip_dump(project_config, os.path.join(project, "project.conf"))
- repo.add_annotated_tag('tag', 'tag')
+ repo.add_annotated_tag("tag", "tag")
- file1 = os.path.join(repofiles, 'file1')
- with open(file1, 'w') as f:
- f.write('test\n')
+ file1 = os.path.join(repofiles, "file1")
+ with open(file1, "w") as f:
+ f.write("test\n")
ref = repo.add_file(file1)
config = repo.source_config(ref=ref)
- del config['track']
- config['url'] = 'repo:{}'.format(reponame)
+ del config["track"]
+ config["url"] = "repo:{}".format(reponame)
# Write out our test target
element = {
- 'kind': 'import',
- 'sources': [
- config
- ],
+ "kind": "import",
+ "sources": [config],
}
- element_path = os.path.join(project, 'target.bst')
+ element_path = os.path.join(project, "target.bst")
_yaml.roundtrip_dump(element, element_path)
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
repo.checkout(top_commit)
- file2 = os.path.join(repofiles, 'file2')
- with open(file2, 'w') as f:
- f.write('test\n')
+ file2 = os.path.join(repofiles, "file2")
+ with open(file2, "w") as f:
+ f.write("test\n")
new_ref = repo.add_file(file2)
- repo.delete_tag('tag')
- repo.add_annotated_tag('tag', 'tag')
- repo.checkout('master')
+ repo.delete_tag("tag")
+ repo.add_annotated_tag("tag", "tag")
+ repo.checkout("master")
- otherpath = os.path.join(str(tmpdir), 'other_path')
- shutil.copytree(repo.repo,
- os.path.join(otherpath, 'repo'))
- create_repo('git', otherpath)
+ otherpath = os.path.join(str(tmpdir), "other_path")
+ shutil.copytree(repo.repo, os.path.join(otherpath, "repo"))
+ create_repo("git", otherpath)
repodir, reponame = os.path.split(repo.repo)
- _yaml.roundtrip_dump(project_config, os.path.join(project, 'project.conf'))
+ _yaml.roundtrip_dump(project_config, os.path.join(project, "project.conf"))
config = repo.source_config(ref=new_ref)
- del config['track']
- config['url'] = 'repo:{}'.format(reponame)
+ del config["track"]
+ config["url"] = "repo:{}".format(reponame)
element = {
- 'kind': 'import',
- 'sources': [
- config
- ],
+ "kind": "import",
+ "sources": [config],
}
_yaml.roundtrip_dump(element, element_path)
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
diff --git a/tests/sources/keytest.py b/tests/sources/keytest.py
index d3eab8d6b..46d0d07fe 100644
--- a/tests/sources/keytest.py
+++ b/tests/sources/keytest.py
@@ -27,8 +27,7 @@ import pytest
from buildstream._exceptions import ErrorDomain
from buildstream.testing import cli # pylint: disable=unused-import
-DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)),
- "project_key_test")
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project_key_test")
# using the key-test plugin to ensure get_unique_key is never called before
diff --git a/tests/sources/local.py b/tests/sources/local.py
index 4b72a4343..92313ca7f 100644
--- a/tests/sources/local.py
+++ b/tests/sources/local.py
@@ -10,103 +10,93 @@ from buildstream.testing import cli # pylint: disable=unused-import
from buildstream.testing._utils.site import HAVE_SANDBOX
from tests.testutils import filetypegenerator
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- 'local',
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "local",)
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_missing_path(cli, datafiles):
project = str(datafiles)
# Removing the local file causes preflight to fail
- localfile = os.path.join(project, 'file.txt')
+ localfile = os.path.join(project, "file.txt")
os.remove(localfile)
- result = cli.run(project=project, args=[
- 'show', 'target.bst'
- ])
+ result = cli.run(project=project, args=["show", "target.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE)
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_non_regular_file_or_directory(cli, datafiles):
project = str(datafiles)
- localfile = os.path.join(project, 'file.txt')
+ localfile = os.path.join(project, "file.txt")
for _file_type in filetypegenerator.generate_file_types(localfile):
- result = cli.run(project=project, args=[
- 'show', 'target.bst'
- ])
+ result = cli.run(project=project, args=["show", "target.bst"])
if os.path.isdir(localfile) and not os.path.islink(localfile):
result.assert_success()
elif os.path.isfile(localfile) and not os.path.islink(localfile):
result.assert_success()
else:
- result.assert_main_error(ErrorDomain.LOAD,
- LoadErrorReason.PROJ_PATH_INVALID_KIND)
+ result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.PROJ_PATH_INVALID_KIND)
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_invalid_absolute_path(cli, datafiles):
project = str(datafiles)
- with open(os.path.join(project, "target.bst"), 'r') as f:
+ with open(os.path.join(project, "target.bst"), "r") as f:
old_yaml = f.read()
new_yaml = old_yaml.replace("file.txt", os.path.join(project, "file.txt"))
assert old_yaml != new_yaml
- with open(os.path.join(project, "target.bst"), 'w') as f:
+ with open(os.path.join(project, "target.bst"), "w") as f:
f.write(new_yaml)
- result = cli.run(project=project, args=['show', 'target.bst'])
- result.assert_main_error(ErrorDomain.LOAD,
- LoadErrorReason.PROJ_PATH_INVALID)
+ result = cli.run(project=project, args=["show", "target.bst"])
+ result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.PROJ_PATH_INVALID)
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'invalid-relative-path'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "invalid-relative-path"))
def test_invalid_relative_path(cli, datafiles):
project = str(datafiles)
- result = cli.run(project=project, args=['show', 'target.bst'])
- result.assert_main_error(ErrorDomain.LOAD,
- LoadErrorReason.PROJ_PATH_INVALID)
+ result = cli.run(project=project, args=["show", "target.bst"])
+ result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.PROJ_PATH_INVALID)
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_stage_file(cli, tmpdir, datafiles):
project = str(datafiles)
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Build, checkout
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
# Check that the checkout contains the expected file
- assert os.path.exists(os.path.join(checkoutdir, 'file.txt'))
+ assert os.path.exists(os.path.join(checkoutdir, "file.txt"))
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'directory'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "directory"))
def test_stage_directory(cli, tmpdir, datafiles):
project = str(datafiles)
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Build, checkout
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
# Check that the checkout contains the expected file and directory and other file
- assert os.path.exists(os.path.join(checkoutdir, 'file.txt'))
- assert os.path.exists(os.path.join(checkoutdir, 'subdir', 'anotherfile.txt'))
+ assert os.path.exists(os.path.join(checkoutdir, "file.txt"))
+ assert os.path.exists(os.path.join(checkoutdir, "subdir", "anotherfile.txt"))
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'symlink'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "symlink"))
def test_stage_symlink(cli, tmpdir, datafiles):
project = str(datafiles)
@@ -117,104 +107,91 @@ def test_stage_symlink(cli, tmpdir, datafiles):
# https://github.com/omarkohl/pytest-datafiles/issues/1
#
# Create the symlink by hand.
- symlink = os.path.join(project, 'files', 'symlink-to-file.txt')
- os.symlink('file.txt', symlink)
+ symlink = os.path.join(project, "files", "symlink-to-file.txt")
+ os.symlink("file.txt", symlink)
# Build, checkout
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
# Check that the checkout contains the expected file and directory and other file
- assert os.path.exists(os.path.join(checkoutdir, 'file.txt'))
- assert os.path.exists(os.path.join(checkoutdir, 'symlink-to-file.txt'))
- assert os.path.islink(os.path.join(checkoutdir, 'symlink-to-file.txt'))
+ assert os.path.exists(os.path.join(checkoutdir, "file.txt"))
+ assert os.path.exists(os.path.join(checkoutdir, "symlink-to-file.txt"))
+ assert os.path.islink(os.path.join(checkoutdir, "symlink-to-file.txt"))
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'file-exists'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "file-exists"))
def test_stage_file_exists(cli, datafiles):
project = str(datafiles)
# Build, checkout
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
result.assert_task_error(ErrorDomain.ELEMENT, "import-source-files-fail")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'directory'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "directory"))
def test_stage_directory_symlink(cli, tmpdir, datafiles):
project = str(datafiles)
checkoutdir = os.path.join(str(tmpdir), "checkout")
- symlink = os.path.join(project, 'files', 'symlink-to-subdir')
- os.symlink('subdir', symlink)
+ symlink = os.path.join(project, "files", "symlink-to-subdir")
+ os.symlink("subdir", symlink)
# Build, checkout
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
# Check that the checkout contains the expected directory and directory symlink
- assert os.path.exists(os.path.join(checkoutdir, 'subdir', 'anotherfile.txt'))
- assert os.path.exists(os.path.join(checkoutdir, 'symlink-to-subdir', 'anotherfile.txt'))
- assert os.path.islink(os.path.join(checkoutdir, 'symlink-to-subdir'))
+ assert os.path.exists(os.path.join(checkoutdir, "subdir", "anotherfile.txt"))
+ assert os.path.exists(os.path.join(checkoutdir, "symlink-to-subdir", "anotherfile.txt"))
+ assert os.path.islink(os.path.join(checkoutdir, "symlink-to-subdir"))
@pytest.mark.integration
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'deterministic-umask'))
-@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "deterministic-umask"))
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_deterministic_source_umask(cli, tmpdir, datafiles):
-
- def create_test_file(*path, mode=0o644, content='content\n'):
+ def create_test_file(*path, mode=0o644, content="content\n"):
path = os.path.join(*path)
os.makedirs(os.path.dirname(path), exist_ok=True)
- with open(path, 'w') as f:
+ with open(path, "w") as f:
f.write(content)
os.fchmod(f.fileno(), mode)
def create_test_directory(*path, mode=0o644):
- create_test_file(*path, '.keep', content='')
+ create_test_file(*path, ".keep", content="")
path = os.path.join(*path)
os.chmod(path, mode)
project = str(datafiles)
- element_name = 'list.bst'
- element_path = os.path.join(project, 'elements', element_name)
- sourcedir = os.path.join(project, 'source')
-
- create_test_file(sourcedir, 'a.txt', mode=0o700)
- create_test_file(sourcedir, 'b.txt', mode=0o755)
- create_test_file(sourcedir, 'c.txt', mode=0o600)
- create_test_file(sourcedir, 'd.txt', mode=0o400)
- create_test_file(sourcedir, 'e.txt', mode=0o644)
- create_test_file(sourcedir, 'f.txt', mode=0o4755)
- create_test_file(sourcedir, 'g.txt', mode=0o2755)
- create_test_file(sourcedir, 'h.txt', mode=0o1755)
- create_test_directory(sourcedir, 'dir-a', mode=0o0700)
- create_test_directory(sourcedir, 'dir-c', mode=0o0755)
- create_test_directory(sourcedir, 'dir-d', mode=0o4755)
- create_test_directory(sourcedir, 'dir-e', mode=0o2755)
- create_test_directory(sourcedir, 'dir-f', mode=0o1755)
-
- source = {'kind': 'local',
- 'path': 'source'}
+ element_name = "list.bst"
+ element_path = os.path.join(project, "elements", element_name)
+ sourcedir = os.path.join(project, "source")
+
+ create_test_file(sourcedir, "a.txt", mode=0o700)
+ create_test_file(sourcedir, "b.txt", mode=0o755)
+ create_test_file(sourcedir, "c.txt", mode=0o600)
+ create_test_file(sourcedir, "d.txt", mode=0o400)
+ create_test_file(sourcedir, "e.txt", mode=0o644)
+ create_test_file(sourcedir, "f.txt", mode=0o4755)
+ create_test_file(sourcedir, "g.txt", mode=0o2755)
+ create_test_file(sourcedir, "h.txt", mode=0o1755)
+ create_test_directory(sourcedir, "dir-a", mode=0o0700)
+ create_test_directory(sourcedir, "dir-c", mode=0o0755)
+ create_test_directory(sourcedir, "dir-d", mode=0o4755)
+ create_test_directory(sourcedir, "dir-e", mode=0o2755)
+ create_test_directory(sourcedir, "dir-f", mode=0o1755)
+
+ source = {"kind": "local", "path": "source"}
element = {
- 'kind': 'manual',
- 'depends': [
- {
- 'filename': 'base.bst',
- 'type': 'build'
- }
- ],
- 'sources': [
- source
- ],
- 'config': {
- 'install-commands': [
- 'ls -l >"%{install-root}/ls-l"'
- ]
- }
+ "kind": "manual",
+ "depends": [{"filename": "base.bst", "type": "build"}],
+ "sources": [source],
+ "config": {"install-commands": ['ls -l >"%{install-root}/ls-l"']},
}
_yaml.roundtrip_dump(element, element_path)
diff --git a/tests/sources/no-fetch-cached/plugins/sources/always_cached.py b/tests/sources/no-fetch-cached/plugins/sources/always_cached.py
index fa143a020..623ab19ab 100644
--- a/tests/sources/no-fetch-cached/plugins/sources/always_cached.py
+++ b/tests/sources/no-fetch-cached/plugins/sources/always_cached.py
@@ -11,7 +11,6 @@ from buildstream import Consistency, Source
class AlwaysCachedSource(Source):
-
def configure(self, node):
pass
diff --git a/tests/sources/no_fetch_cached.py b/tests/sources/no_fetch_cached.py
index 81032881c..1ee3dd7bd 100644
--- a/tests/sources/no_fetch_cached.py
+++ b/tests/sources/no_fetch_cached.py
@@ -10,10 +10,7 @@ from buildstream.testing import cli # pylint: disable=unused-import
from buildstream.testing import create_repo
from buildstream.testing._utils.site import HAVE_GIT
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- 'no-fetch-cached'
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "no-fetch-cached")
##################################################################
@@ -26,23 +23,13 @@ def test_no_fetch_cached(cli, tmpdir, datafiles):
project = str(datafiles)
# Create the repo from 'files' subdir
- repo = create_repo('git', str(tmpdir))
- ref = repo.create(os.path.join(project, 'files'))
+ repo = create_repo("git", str(tmpdir))
+ ref = repo.create(os.path.join(project, "files"))
# Write out test target with a cached and a non-cached source
- element = {
- 'kind': 'import',
- 'sources': [
- repo.source_config(ref=ref),
- {
- 'kind': 'always_cached'
- }
- ]
- }
- _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst'))
+ element = {"kind": "import", "sources": [repo.source_config(ref=ref), {"kind": "always_cached"}]}
+ _yaml.roundtrip_dump(element, os.path.join(project, "target.bst"))
# Test fetch of target with a cached and a non-cached source
- result = cli.run(project=project, args=[
- 'source', 'fetch', 'target.bst'
- ])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_success()
diff --git a/tests/sources/patch.py b/tests/sources/patch.py
index 4f9db815b..2b80a7055 100644
--- a/tests/sources/patch.py
+++ b/tests/sources/patch.py
@@ -8,148 +8,137 @@ from buildstream._exceptions import ErrorDomain, LoadErrorReason
from buildstream.testing import cli # pylint: disable=unused-import
from tests.testutils import filetypegenerator
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- 'patch',
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "patch",)
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_missing_patch(cli, datafiles):
project = str(datafiles)
# Removing the local file causes preflight to fail
- localfile = os.path.join(project, 'file_1.patch')
+ localfile = os.path.join(project, "file_1.patch")
os.remove(localfile)
- result = cli.run(project=project, args=[
- 'show', 'target.bst'
- ])
+ result = cli.run(project=project, args=["show", "target.bst"])
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE)
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_non_regular_file_patch(cli, datafiles):
project = str(datafiles)
- patch_path = os.path.join(project, 'irregular_file.patch')
+ patch_path = os.path.join(project, "irregular_file.patch")
for _file_type in filetypegenerator.generate_file_types(patch_path):
- result = cli.run(project=project, args=[
- 'show', 'irregular.bst'
- ])
+ result = cli.run(project=project, args=["show", "irregular.bst"])
if os.path.isfile(patch_path) and not os.path.islink(patch_path):
result.assert_success()
else:
- result.assert_main_error(ErrorDomain.LOAD,
- LoadErrorReason.PROJ_PATH_INVALID_KIND)
+ result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.PROJ_PATH_INVALID_KIND)
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_invalid_absolute_path(cli, datafiles):
project = str(datafiles)
- with open(os.path.join(project, "target.bst"), 'r') as f:
+ with open(os.path.join(project, "target.bst"), "r") as f:
old_yaml = f.read()
- new_yaml = old_yaml.replace("file_1.patch",
- os.path.join(project, "file_1.patch"))
+ new_yaml = old_yaml.replace("file_1.patch", os.path.join(project, "file_1.patch"))
assert old_yaml != new_yaml
- with open(os.path.join(project, "target.bst"), 'w') as f:
+ with open(os.path.join(project, "target.bst"), "w") as f:
f.write(new_yaml)
- result = cli.run(project=project, args=['show', 'target.bst'])
- result.assert_main_error(ErrorDomain.LOAD,
- LoadErrorReason.PROJ_PATH_INVALID)
+ result = cli.run(project=project, args=["show", "target.bst"])
+ result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.PROJ_PATH_INVALID)
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'invalid-relative-path'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "invalid-relative-path"))
def test_invalid_relative_path(cli, datafiles):
project = str(datafiles)
- result = cli.run(project=project, args=['show', 'irregular.bst'])
- result.assert_main_error(ErrorDomain.LOAD,
- LoadErrorReason.PROJ_PATH_INVALID)
+ result = cli.run(project=project, args=["show", "irregular.bst"])
+ result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.PROJ_PATH_INVALID)
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_stage_and_patch(cli, tmpdir, datafiles):
project = str(datafiles)
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Build, checkout
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
# Test the file.txt was patched and changed
- with open(os.path.join(checkoutdir, 'file.txt')) as f:
- assert f.read() == 'This is text file with superpowers\n'
+ with open(os.path.join(checkoutdir, "file.txt")) as f:
+ assert f.read() == "This is text file with superpowers\n"
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_stage_file_nonexistent_dir(cli, datafiles):
project = str(datafiles)
# Fails at build time because it tries to patch into a non-existing directory
- result = cli.run(project=project, args=['build', 'failure-nonexistent-dir.bst'])
+ result = cli.run(project=project, args=["build", "failure-nonexistent-dir.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
result.assert_task_error(ErrorDomain.SOURCE, "patch-no-files")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_stage_file_empty_dir(cli, datafiles):
project = str(datafiles)
# Fails at build time because it tries to patch with nothing else staged
- result = cli.run(project=project, args=['build', 'failure-empty-dir.bst'])
+ result = cli.run(project=project, args=["build", "failure-empty-dir.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
result.assert_task_error(ErrorDomain.SOURCE, "patch-no-files")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'separate-patch-dir'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "separate-patch-dir"))
def test_stage_separate_patch_dir(cli, tmpdir, datafiles):
project = str(datafiles)
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Track, fetch, build, checkout
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
# Test the file.txt was patched and changed
- with open(os.path.join(checkoutdir, 'test-dir', 'file.txt')) as f:
- assert f.read() == 'This is text file in a directory with superpowers\n'
+ with open(os.path.join(checkoutdir, "test-dir", "file.txt")) as f:
+ assert f.read() == "This is text file in a directory with superpowers\n"
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'multiple-patches'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "multiple-patches"))
def test_stage_multiple_patches(cli, tmpdir, datafiles):
project = str(datafiles)
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Track, fetch, build, checkout
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
# Test the file.txt was patched and changed
- with open(os.path.join(checkoutdir, 'file.txt')) as f:
- assert f.read() == 'This is text file with more superpowers\n'
+ with open(os.path.join(checkoutdir, "file.txt")) as f:
+ assert f.read() == "This is text file with more superpowers\n"
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'different-strip-level'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "different-strip-level"))
def test_patch_strip_level(cli, tmpdir, datafiles):
project = str(datafiles)
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Track, fetch, build, checkout
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
# Test the file.txt was patched and changed
- with open(os.path.join(checkoutdir, 'file.txt')) as f:
- assert f.read() == 'This is text file with superpowers\n'
+ with open(os.path.join(checkoutdir, "file.txt")) as f:
+ assert f.read() == "This is text file with superpowers\n"
diff --git a/tests/sources/pip.py b/tests/sources/pip.py
index 7f91ba701..aafdfaf1c 100644
--- a/tests/sources/pip.py
+++ b/tests/sources/pip.py
@@ -9,62 +9,56 @@ from buildstream import _yaml
from buildstream.plugins.sources.pip import _match_package_name
from buildstream.testing import cli # pylint: disable=unused-import
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- 'pip',
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "pip",)
def generate_project(project_dir):
project_file = os.path.join(project_dir, "project.conf")
- _yaml.roundtrip_dump({'name': 'foo'}, project_file)
+ _yaml.roundtrip_dump({"name": "foo"}, project_file)
# Test that without ref, consistency is set appropriately.
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-ref'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "no-ref"))
def test_no_ref(cli, datafiles):
project = str(datafiles)
generate_project(project)
- assert cli.get_element_state(project, 'target.bst') == 'no reference'
+ assert cli.get_element_state(project, "target.bst") == "no reference"
# Test that pip is not allowed to be the first source
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'first-source-pip'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "first-source-pip"))
def test_first_source(cli, datafiles):
project = str(datafiles)
generate_project(project)
- result = cli.run(project=project, args=[
- 'show', 'target.bst'
- ])
+ result = cli.run(project=project, args=["show", "target.bst"])
result.assert_main_error(ErrorDomain.ELEMENT, None)
# Test that error is raised when neither packges nor requirements files
# have been specified
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-packages'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "no-packages"))
def test_no_packages(cli, datafiles):
project = str(datafiles)
generate_project(project)
- result = cli.run(project=project, args=[
- 'show', 'target.bst'
- ])
+ result = cli.run(project=project, args=["show", "target.bst"])
result.assert_main_error(ErrorDomain.SOURCE, None)
# Test that pip source parses tar ball names correctly for the ref
@pytest.mark.parametrize(
- 'tarball, expected_name, expected_version',
+ "tarball, expected_name, expected_version",
[
- ('dotted.package-0.9.8.tar.gz', 'dotted.package', '0.9.8'),
- ('hyphenated-package-2.6.0.tar.gz', 'hyphenated-package', '2.6.0'),
- ('underscore_pkg-3.1.0.tar.gz', 'underscore_pkg', '3.1.0'),
- ('numbers2and5-1.0.1.tar.gz', 'numbers2and5', '1.0.1'),
- ('multiple.dots.package-5.6.7.tar.gz', 'multiple.dots.package', '5.6.7'),
- ('multiple-hyphens-package-1.2.3.tar.gz', 'multiple-hyphens-package', '1.2.3'),
- ('multiple_underscore_pkg-3.4.5.tar.gz', 'multiple_underscore_pkg', '3.4.5'),
- ('shortversion-1.0.tar.gz', 'shortversion', '1.0'),
- ('longversion-1.2.3.4.tar.gz', 'longversion', '1.2.3.4')
- ])
+ ("dotted.package-0.9.8.tar.gz", "dotted.package", "0.9.8"),
+ ("hyphenated-package-2.6.0.tar.gz", "hyphenated-package", "2.6.0"),
+ ("underscore_pkg-3.1.0.tar.gz", "underscore_pkg", "3.1.0"),
+ ("numbers2and5-1.0.1.tar.gz", "numbers2and5", "1.0.1"),
+ ("multiple.dots.package-5.6.7.tar.gz", "multiple.dots.package", "5.6.7"),
+ ("multiple-hyphens-package-1.2.3.tar.gz", "multiple-hyphens-package", "1.2.3"),
+ ("multiple_underscore_pkg-3.4.5.tar.gz", "multiple_underscore_pkg", "3.4.5"),
+ ("shortversion-1.0.tar.gz", "shortversion", "1.0"),
+ ("longversion-1.2.3.4.tar.gz", "longversion", "1.2.3.4"),
+ ],
+)
def test_match_package_name(tarball, expected_name, expected_version):
name, version = _match_package_name(tarball)
assert (expected_name, expected_version) == (name, version)
diff --git a/tests/sources/previous_source_access.py b/tests/sources/previous_source_access.py
index 750b94381..fadf6710c 100644
--- a/tests/sources/previous_source_access.py
+++ b/tests/sources/previous_source_access.py
@@ -7,10 +7,7 @@ import pytest
from buildstream import _yaml
from buildstream.testing import cli # pylint: disable=unused-import
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- 'previous_source_access'
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "previous_source_access")
##################################################################
@@ -29,30 +26,22 @@ def test_custom_transform_source(cli, datafiles):
_yaml.roundtrip_dump(project_config, project_config_path)
# Ensure we can track
- result = cli.run(project=project, args=[
- 'source', 'track', 'target.bst'
- ])
+ result = cli.run(project=project, args=["source", "track", "target.bst"])
result.assert_success()
# Ensure we can fetch
- result = cli.run(project=project, args=[
- 'source', 'fetch', 'target.bst'
- ])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_success()
# Ensure we get correct output from foo_transform
- cli.run(project=project, args=[
- 'build', 'target.bst'
- ])
- destpath = os.path.join(cli.directory, 'checkout')
- result = cli.run(project=project, args=[
- 'artifact', 'checkout', 'target.bst', '--directory', destpath
- ])
+ cli.run(project=project, args=["build", "target.bst"])
+ destpath = os.path.join(cli.directory, "checkout")
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", destpath])
result.assert_success()
# Assert that files from both sources exist, and that they have
# the same content
- assert os.path.exists(os.path.join(destpath, 'file'))
- assert os.path.exists(os.path.join(destpath, 'filetransform'))
- with open(os.path.join(destpath, 'file')) as file1:
- with open(os.path.join(destpath, 'filetransform')) as file2:
+ assert os.path.exists(os.path.join(destpath, "file"))
+ assert os.path.exists(os.path.join(destpath, "filetransform"))
+ with open(os.path.join(destpath, "file")) as file1:
+ with open(os.path.join(destpath, "filetransform")) as file2:
assert file1.read() == file2.read()
diff --git a/tests/sources/previous_source_access/plugins/sources/foo_transform.py b/tests/sources/previous_source_access/plugins/sources/foo_transform.py
index 4b423a1b3..906a8f6be 100644
--- a/tests/sources/previous_source_access/plugins/sources/foo_transform.py
+++ b/tests/sources/previous_source_access/plugins/sources/foo_transform.py
@@ -25,14 +25,13 @@ class FooTransformSource(Source):
"""Directory where this source should stage its files
"""
- path = os.path.join(self.get_mirror_directory(), self.name,
- self.ref.strip())
+ path = os.path.join(self.get_mirror_directory(), self.name, self.ref.strip())
os.makedirs(path, exist_ok=True)
return path
def configure(self, node):
- node.validate_keys(['ref', *Source.COMMON_CONFIG_KEYS])
- self.ref = node.get_str('ref', None)
+ node.validate_keys(["ref", *Source.COMMON_CONFIG_KEYS])
+ self.ref = node.get_str("ref", None)
def preflight(self):
pass
@@ -45,9 +44,9 @@ class FooTransformSource(Source):
return Consistency.INCONSISTENT
# If we have a file called "filetransform", verify that its checksum
# matches our ref. Otherwise, it resolved but not cached.
- fpath = os.path.join(self.mirror, 'filetransform')
+ fpath = os.path.join(self.mirror, "filetransform")
try:
- with open(fpath, 'rb') as f:
+ with open(fpath, "rb") as f:
if hashlib.sha256(f.read()).hexdigest() == self.ref.strip():
return Consistency.CACHED
except Exception:
@@ -58,30 +57,29 @@ class FooTransformSource(Source):
return self.ref
def set_ref(self, ref, node):
- self.ref = node['ref'] = ref
+ self.ref = node["ref"] = ref
def track(self, previous_sources_dir):
# Store the checksum of the file from previous source as our ref
- fpath = os.path.join(previous_sources_dir, 'file')
- with open(fpath, 'rb') as f:
+ fpath = os.path.join(previous_sources_dir, "file")
+ with open(fpath, "rb") as f:
return hashlib.sha256(f.read()).hexdigest()
def fetch(self, previous_sources_dir):
- fpath = os.path.join(previous_sources_dir, 'file')
+ fpath = os.path.join(previous_sources_dir, "file")
# Verify that the checksum of the file from previous source matches
# our ref
- with open(fpath, 'rb') as f:
+ with open(fpath, "rb") as f:
if hashlib.sha256(f.read()).hexdigest() != self.ref.strip():
raise SourceError("Element references do not match")
# Copy "file" as "filetransform"
- newfpath = os.path.join(self.mirror, 'filetransform')
+ newfpath = os.path.join(self.mirror, "filetransform")
utils.safe_copy(fpath, newfpath)
def stage(self, directory):
# Simply stage the "filetransform" file
- utils.safe_copy(os.path.join(self.mirror, 'filetransform'),
- os.path.join(directory, 'filetransform'))
+ utils.safe_copy(os.path.join(self.mirror, "filetransform"), os.path.join(directory, "filetransform"))
def setup():
diff --git a/tests/sources/remote.py b/tests/sources/remote.py
index 5b818b960..76e469c60 100644
--- a/tests/sources/remote.py
+++ b/tests/sources/remote.py
@@ -10,94 +10,71 @@ from buildstream import _yaml
from buildstream.testing import cli # pylint: disable=unused-import
from tests.testutils.file_server import create_file_server
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- 'remote',
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "remote",)
def generate_project(project_dir, tmpdir):
project_file = os.path.join(project_dir, "project.conf")
- _yaml.roundtrip_dump({
- 'name': 'foo',
- 'aliases': {
- 'tmpdir': "file:///" + str(tmpdir)
- }
- }, project_file)
+ _yaml.roundtrip_dump({"name": "foo", "aliases": {"tmpdir": "file:///" + str(tmpdir)}}, project_file)
def generate_project_file_server(server, project_dir):
project_file = os.path.join(project_dir, "project.conf")
- _yaml.roundtrip_dump({
- 'name': 'foo',
- 'aliases': {
- 'tmpdir': server.base_url()
- }
- }, project_file)
+ _yaml.roundtrip_dump({"name": "foo", "aliases": {"tmpdir": server.base_url()}}, project_file)
# Test that without ref, consistency is set appropriately.
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-ref'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "no-ref"))
def test_no_ref(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project, tmpdir)
- assert cli.get_element_state(project, 'target.bst') == 'no reference'
+ assert cli.get_element_state(project, "target.bst") == "no reference"
# Here we are doing a fetch on a file that doesn't exist. target.bst
# refers to 'file' but that file is not present.
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'missing-file'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "missing-file"))
def test_missing_file(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project, tmpdir)
# Try to fetch it
- result = cli.run(project=project, args=[
- 'source', 'fetch', 'target.bst'
- ])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
result.assert_task_error(ErrorDomain.SOURCE, None)
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'path-in-filename'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "path-in-filename"))
def test_path_in_filename(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project, tmpdir)
# Try to fetch it
- result = cli.run(project=project, args=[
- 'source', 'fetch', 'target.bst'
- ])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
# The bst file has a / in the filename param
result.assert_main_error(ErrorDomain.SOURCE, "filename-contains-directory")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'single-file'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "single-file"))
def test_simple_file_build(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project, tmpdir)
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Try to fetch it
- result = cli.run(project=project, args=[
- 'source', 'fetch', 'target.bst'
- ])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=[
- 'build', 'target.bst'
- ])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=[
- 'artifact', 'checkout', 'target.bst', '--directory', checkoutdir
- ])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
# Note that the url of the file in target.bst is actually /dir/file
# but this tests confirms we take the basename
- checkout_file = os.path.join(checkoutdir, 'file')
+ checkout_file = os.path.join(checkoutdir, "file")
assert os.path.exists(checkout_file)
mode = os.stat(checkout_file).st_mode
@@ -107,113 +84,99 @@ def test_simple_file_build(cli, tmpdir, datafiles):
assert not mode & (stat.S_IWGRP | stat.S_IWOTH)
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'single-file-custom-name'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "single-file-custom-name"))
def test_simple_file_custom_name_build(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project, tmpdir)
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Try to fetch it
- result = cli.run(project=project, args=[
- 'source', 'fetch', 'target.bst'
- ])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=[
- 'build', 'target.bst'
- ])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=[
- 'artifact', 'checkout', 'target.bst', '--directory', checkoutdir
- ])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
- assert not os.path.exists(os.path.join(checkoutdir, 'file'))
- assert os.path.exists(os.path.join(checkoutdir, 'custom-file'))
+ assert not os.path.exists(os.path.join(checkoutdir, "file"))
+ assert os.path.exists(os.path.join(checkoutdir, "custom-file"))
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'unique-keys'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "unique-keys"))
def test_unique_key(cli, tmpdir, datafiles):
- '''This test confirms that the 'filename' parameter is honoured when it comes
+ """This test confirms that the 'filename' parameter is honoured when it comes
to generating a cache key for the source.
- '''
+ """
project = str(datafiles)
generate_project(project, tmpdir)
- states = cli.get_element_states(project, [
- 'target.bst', 'target-custom.bst', 'target-custom-executable.bst'
- ])
- assert states['target.bst'] == "fetch needed"
- assert states['target-custom.bst'] == "fetch needed"
- assert states['target-custom-executable.bst'] == "fetch needed"
+ states = cli.get_element_states(project, ["target.bst", "target-custom.bst", "target-custom-executable.bst"])
+ assert states["target.bst"] == "fetch needed"
+ assert states["target-custom.bst"] == "fetch needed"
+ assert states["target-custom-executable.bst"] == "fetch needed"
# Try to fetch it
- cli.run(project=project, args=[
- 'source', 'fetch', 'target.bst'
- ])
+ cli.run(project=project, args=["source", "fetch", "target.bst"])
# We should download the file only once
- states = cli.get_element_states(project, [
- 'target.bst', 'target-custom.bst', 'target-custom-executable.bst'
- ])
- assert states['target.bst'] == 'buildable'
- assert states['target-custom.bst'] == 'buildable'
- assert states['target-custom-executable.bst'] == 'buildable'
+ states = cli.get_element_states(project, ["target.bst", "target-custom.bst", "target-custom-executable.bst"])
+ assert states["target.bst"] == "buildable"
+ assert states["target-custom.bst"] == "buildable"
+ assert states["target-custom-executable.bst"] == "buildable"
# But the cache key is different because the 'filename' is different.
- assert cli.get_element_key(project, 'target.bst') != \
- cli.get_element_key(project, 'target-custom.bst') != \
- cli.get_element_key(project, 'target-custom-executable.bst')
+ assert (
+ cli.get_element_key(project, "target.bst")
+ != cli.get_element_key(project, "target-custom.bst")
+ != cli.get_element_key(project, "target-custom-executable.bst")
+ )
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'unique-keys'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "unique-keys"))
def test_executable(cli, tmpdir, datafiles):
- '''This test confirms that the 'ecxecutable' parameter is honoured.
- '''
+ """This test confirms that the 'ecxecutable' parameter is honoured.
+ """
project = str(datafiles)
generate_project(project, tmpdir)
checkoutdir = os.path.join(str(tmpdir), "checkout")
- assert cli.get_element_state(project, 'target-custom-executable.bst') == "fetch needed"
+ assert cli.get_element_state(project, "target-custom-executable.bst") == "fetch needed"
# Try to fetch it
- cli.run(project=project, args=[
- 'build', 'target-custom-executable.bst'
- ])
-
- cli.run(project=project, args=[
- 'artifact', 'checkout', 'target-custom-executable.bst', '--directory', checkoutdir
- ])
- mode = os.stat(os.path.join(checkoutdir, 'some-custom-file')).st_mode
+ cli.run(project=project, args=["build", "target-custom-executable.bst"])
+
+ cli.run(project=project, args=["artifact", "checkout", "target-custom-executable.bst", "--directory", checkoutdir])
+ mode = os.stat(os.path.join(checkoutdir, "some-custom-file")).st_mode
assert mode & stat.S_IEXEC
# Assert executable by anyone
assert mode & (stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH)
-@pytest.mark.parametrize('server_type', ('FTP', 'HTTP'))
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'single-file'))
+@pytest.mark.parametrize("server_type", ("FTP", "HTTP"))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "single-file"))
def test_use_netrc(cli, datafiles, server_type, tmpdir):
- fake_home = os.path.join(str(tmpdir), 'fake_home')
+ fake_home = os.path.join(str(tmpdir), "fake_home")
os.makedirs(fake_home, exist_ok=True)
project = str(datafiles)
- checkoutdir = os.path.join(str(tmpdir), 'checkout')
+ checkoutdir = os.path.join(str(tmpdir), "checkout")
- os.environ['HOME'] = fake_home
- with open(os.path.join(fake_home, '.netrc'), 'wb') as f:
+ os.environ["HOME"] = fake_home
+ with open(os.path.join(fake_home, ".netrc"), "wb") as f:
os.fchmod(f.fileno(), 0o700)
- f.write(b'machine 127.0.0.1\n')
- f.write(b'login testuser\n')
- f.write(b'password 12345\n')
+ f.write(b"machine 127.0.0.1\n")
+ f.write(b"login testuser\n")
+ f.write(b"password 12345\n")
with create_file_server(server_type) as server:
- server.add_user('testuser', '12345', project)
+ server.add_user("testuser", "12345", project)
generate_project_file_server(server, project)
server.start()
- result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
- checkout_file = os.path.join(checkoutdir, 'file')
+ checkout_file = os.path.join(checkoutdir, "file")
assert os.path.exists(checkout_file)
diff --git a/tests/sources/tar.py b/tests/sources/tar.py
index fac6f3f8b..f5f3dde2d 100644
--- a/tests/sources/tar.py
+++ b/tests/sources/tar.py
@@ -17,10 +17,7 @@ from buildstream.testing._utils.site import HAVE_LZIP
from tests.testutils.file_server import create_file_server
from . import list_dir_contents
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- 'tar',
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "tar",)
def _assemble_tar(workingdir, srcdir, dstfile):
@@ -38,58 +35,44 @@ def _assemble_tar_lz(workingdir, srcdir, dstfile):
with tarfile.open(fileobj=uncompressed, mode="w:") as tar:
tar.add(srcdir)
uncompressed.seek(0, 0)
- with open(dstfile, 'wb') as dst:
- subprocess.call(['lzip'],
- stdin=uncompressed,
- stdout=dst)
+ with open(dstfile, "wb") as dst:
+ subprocess.call(["lzip"], stdin=uncompressed, stdout=dst)
os.chdir(old_dir)
def generate_project(project_dir, tmpdir):
project_file = os.path.join(project_dir, "project.conf")
- _yaml.roundtrip_dump({
- 'name': 'foo',
- 'aliases': {
- 'tmpdir': "file:///" + str(tmpdir)
- }
- }, project_file)
+ _yaml.roundtrip_dump({"name": "foo", "aliases": {"tmpdir": "file:///" + str(tmpdir)}}, project_file)
def generate_project_file_server(base_url, project_dir):
project_file = os.path.join(project_dir, "project.conf")
- _yaml.roundtrip_dump({
- 'name': 'foo',
- 'aliases': {
- 'tmpdir': base_url
- }
- }, project_file)
+ _yaml.roundtrip_dump({"name": "foo", "aliases": {"tmpdir": base_url}}, project_file)
# Test that without ref, consistency is set appropriately.
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-ref'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "no-ref"))
def test_no_ref(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project, tmpdir)
- assert cli.get_element_state(project, 'target.bst') == 'no reference'
+ assert cli.get_element_state(project, "target.bst") == "no reference"
# Test that when I fetch a nonexistent URL, errors are handled gracefully and a retry is performed.
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch"))
def test_fetch_bad_url(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project, tmpdir)
# Try to fetch it
- result = cli.run(project=project, args=[
- 'source', 'fetch', 'target.bst'
- ])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
assert "FAILURE Try #" in result.stderr
result.assert_main_error(ErrorDomain.STREAM, None)
result.assert_task_error(ErrorDomain.SOURCE, None)
# Test that when I fetch with an invalid ref, it fails.
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch"))
def test_fetch_bad_ref(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project, tmpdir)
@@ -99,15 +82,13 @@ def test_fetch_bad_ref(cli, tmpdir, datafiles):
_assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar)
# Try to fetch it
- result = cli.run(project=project, args=[
- 'source', 'fetch', 'target.bst'
- ])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
result.assert_task_error(ErrorDomain.SOURCE, None)
# Test that when tracking with a ref set, there is a warning
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch"))
def test_track_warning(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project, tmpdir)
@@ -117,15 +98,13 @@ def test_track_warning(cli, tmpdir, datafiles):
_assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar)
# Track it
- result = cli.run(project=project, args=[
- 'source', 'track', 'target.bst'
- ])
+ result = cli.run(project=project, args=["source", "track", "target.bst"])
result.assert_success()
assert "Potential man-in-the-middle attack!" in result.stderr
# Test that a staged checkout matches what was tarred up, with the default first subdir
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch"))
@pytest.mark.parametrize("srcdir", ["a", "./a"])
def test_stage_default_basedir(cli, tmpdir, datafiles, srcdir):
project = str(datafiles)
@@ -137,13 +116,13 @@ def test_stage_default_basedir(cli, tmpdir, datafiles, srcdir):
_assemble_tar(os.path.join(str(datafiles), "content"), srcdir, src_tar)
# Track, fetch, build, checkout
- result = cli.run(project=project, args=['source', 'track', 'target.bst'])
+ result = cli.run(project=project, args=["source", "track", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
# Check that the content of the first directory is checked out (base-dir: '*')
@@ -154,7 +133,7 @@ def test_stage_default_basedir(cli, tmpdir, datafiles, srcdir):
# Test that a staged checkout matches what was tarred up, with an empty base-dir
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-basedir'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "no-basedir"))
@pytest.mark.parametrize("srcdir", ["a", "./a"])
def test_stage_no_basedir(cli, tmpdir, datafiles, srcdir):
project = str(datafiles)
@@ -166,13 +145,13 @@ def test_stage_no_basedir(cli, tmpdir, datafiles, srcdir):
_assemble_tar(os.path.join(str(datafiles), "content"), srcdir, src_tar)
# Track, fetch, build, checkout
- result = cli.run(project=project, args=['source', 'track', 'target.bst'])
+ result = cli.run(project=project, args=["source", "track", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
# Check that the full content of the tarball is checked out (base-dir: '')
@@ -183,7 +162,7 @@ def test_stage_no_basedir(cli, tmpdir, datafiles, srcdir):
# Test that a staged checkout matches what was tarred up, with an explicit basedir
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'explicit-basedir'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "explicit-basedir"))
@pytest.mark.parametrize("srcdir", ["a", "./a"])
def test_stage_explicit_basedir(cli, tmpdir, datafiles, srcdir):
project = str(datafiles)
@@ -195,13 +174,13 @@ def test_stage_explicit_basedir(cli, tmpdir, datafiles, srcdir):
_assemble_tar(os.path.join(str(datafiles), "content"), srcdir, src_tar)
# Track, fetch, build, checkout
- result = cli.run(project=project, args=['source', 'track', 'target.bst'])
+ result = cli.run(project=project, args=["source", "track", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
# Check that the content of the first directory is checked out (base-dir: '*')
@@ -213,7 +192,7 @@ def test_stage_explicit_basedir(cli, tmpdir, datafiles, srcdir):
# Test that we succeed to extract tarballs with hardlinks when stripping the
# leading paths
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'contains-links'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "contains-links"))
def test_stage_contains_links(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project, tmpdir)
@@ -231,13 +210,13 @@ def test_stage_contains_links(cli, tmpdir, datafiles):
_assemble_tar(os.path.join(str(datafiles), "content"), "base-directory", src_tar)
# Track, fetch, build, checkout
- result = cli.run(project=project, args=['source', 'track', 'target.bst'])
+ result = cli.run(project=project, args=["source", "track", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
# Check that the content of the first directory is checked out (base-dir: '*')
@@ -247,8 +226,8 @@ def test_stage_contains_links(cli, tmpdir, datafiles):
assert checkout_contents == original_contents
-@pytest.mark.skipif(not HAVE_LZIP, reason='lzip is not available')
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
+@pytest.mark.skipif(not HAVE_LZIP, reason="lzip is not available")
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch"))
@pytest.mark.parametrize("srcdir", ["a", "./a"])
def test_stage_default_basedir_lzip(cli, tmpdir, datafiles, srcdir):
project = str(datafiles)
@@ -260,13 +239,13 @@ def test_stage_default_basedir_lzip(cli, tmpdir, datafiles, srcdir):
_assemble_tar_lz(os.path.join(str(datafiles), "content"), srcdir, src_tar)
# Track, fetch, build, checkout
- result = cli.run(project=project, args=['source', 'track', 'target-lz.bst'])
+ result = cli.run(project=project, args=["source", "track", "target-lz.bst"])
result.assert_success()
- result = cli.run(project=project, args=['source', 'fetch', 'target-lz.bst'])
+ result = cli.run(project=project, args=["source", "fetch", "target-lz.bst"])
result.assert_success()
- result = cli.run(project=project, args=['build', 'target-lz.bst'])
+ result = cli.run(project=project, args=["build", "target-lz.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target-lz.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target-lz.bst", "--directory", checkoutdir])
result.assert_success()
# Check that the content of the first directory is checked out (base-dir: '*')
@@ -280,8 +259,8 @@ def test_stage_default_basedir_lzip(cli, tmpdir, datafiles, srcdir):
# a - contains read-only files in a writable directory
# b - root directory has read-only permission
# c - contains one file that has no read nor write permissions. Base-dir set to '' to extract root of tarball
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'read-only'))
-@pytest.mark.parametrize("tar_name, base_dir", [("a", "*"), ("b", '*'), ("c", '')])
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "read-only"))
+@pytest.mark.parametrize("tar_name, base_dir", [("a", "*"), ("b", "*"), ("c", "")])
def test_read_only_dir(cli, tmpdir, datafiles, tar_name, base_dir):
try:
project = str(datafiles)
@@ -290,25 +269,21 @@ def test_read_only_dir(cli, tmpdir, datafiles, tar_name, base_dir):
bst_path = os.path.join(project, "target.bst")
tar_file = "{}.tar.gz".format(tar_name)
- _yaml.roundtrip_dump({
- 'kind': 'import',
- 'sources': [
- {
- 'kind': 'tar',
- 'url': 'tmpdir:/{}'.format(tar_file),
- 'ref': 'foo',
- 'base-dir': base_dir
- }
- ]
- }, bst_path)
+ _yaml.roundtrip_dump(
+ {
+ "kind": "import",
+ "sources": [{"kind": "tar", "url": "tmpdir:/{}".format(tar_file), "ref": "foo", "base-dir": base_dir}],
+ },
+ bst_path,
+ )
# Get the tarball in tests/sources/tar/read-only/content
#
# NOTE that we need to do this because tarfile.open and tar.add()
# are packing the tar up with writeable files and dirs
- tarball = os.path.join(str(datafiles), 'content', tar_file)
+ tarball = os.path.join(str(datafiles), "content", tar_file)
if not os.path.exists(tarball):
- raise FileNotFoundError('{} does not exist'.format(tarball))
+ raise FileNotFoundError("{} does not exist".format(tarball))
copyfile(tarball, os.path.join(str(tmpdir), tar_file))
# Because this test can potentially leave directories behind
@@ -320,11 +295,11 @@ def test_read_only_dir(cli, tmpdir, datafiles, tar_name, base_dir):
env = {"TMP": tmpdir_str}
# Track, fetch, build, checkout
- result = cli.run(project=project, args=['source', 'track', 'target.bst'], env=env)
+ result = cli.run(project=project, args=["source", "track", "target.bst"], env=env)
result.assert_success()
- result = cli.run(project=project, args=['source', 'fetch', 'target.bst'], env=env)
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"], env=env)
result.assert_success()
- result = cli.run(project=project, args=['build', 'target.bst'], env=env)
+ result = cli.run(project=project, args=["build", "target.bst"], env=env)
result.assert_success()
finally:
@@ -336,85 +311,86 @@ def test_read_only_dir(cli, tmpdir, datafiles, tar_name, base_dir):
os.rmdir(path)
else:
os.remove(path)
+
rmtree(str(tmpdir), onerror=make_dir_writable)
-@pytest.mark.parametrize('server_type', ('FTP', 'HTTP'))
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
+@pytest.mark.parametrize("server_type", ("FTP", "HTTP"))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch"))
def test_use_netrc(cli, datafiles, server_type, tmpdir):
- file_server_files = os.path.join(str(tmpdir), 'file_server')
- fake_home = os.path.join(str(tmpdir), 'fake_home')
+ file_server_files = os.path.join(str(tmpdir), "file_server")
+ fake_home = os.path.join(str(tmpdir), "fake_home")
os.makedirs(file_server_files, exist_ok=True)
os.makedirs(fake_home, exist_ok=True)
project = str(datafiles)
- checkoutdir = os.path.join(str(tmpdir), 'checkout')
+ checkoutdir = os.path.join(str(tmpdir), "checkout")
- os.environ['HOME'] = fake_home
- with open(os.path.join(fake_home, '.netrc'), 'wb') as f:
+ os.environ["HOME"] = fake_home
+ with open(os.path.join(fake_home, ".netrc"), "wb") as f:
os.fchmod(f.fileno(), 0o700)
- f.write(b'machine 127.0.0.1\n')
- f.write(b'login testuser\n')
- f.write(b'password 12345\n')
+ f.write(b"machine 127.0.0.1\n")
+ f.write(b"login testuser\n")
+ f.write(b"password 12345\n")
with create_file_server(server_type) as server:
- server.add_user('testuser', '12345', file_server_files)
+ server.add_user("testuser", "12345", file_server_files)
generate_project_file_server(server.base_url(), project)
- src_tar = os.path.join(file_server_files, 'a.tar.gz')
- _assemble_tar(os.path.join(str(datafiles), 'content'), 'a', src_tar)
+ src_tar = os.path.join(file_server_files, "a.tar.gz")
+ _assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar)
server.start()
- result = cli.run(project=project, args=['source', 'track', 'target.bst'])
+ result = cli.run(project=project, args=["source", "track", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
- original_dir = os.path.join(str(datafiles), 'content', 'a')
+ original_dir = os.path.join(str(datafiles), "content", "a")
original_contents = list_dir_contents(original_dir)
checkout_contents = list_dir_contents(checkoutdir)
assert checkout_contents == original_contents
-@pytest.mark.parametrize('server_type', ('FTP', 'HTTP'))
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
+@pytest.mark.parametrize("server_type", ("FTP", "HTTP"))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch"))
def test_netrc_already_specified_user(cli, datafiles, server_type, tmpdir):
- file_server_files = os.path.join(str(tmpdir), 'file_server')
- fake_home = os.path.join(str(tmpdir), 'fake_home')
+ file_server_files = os.path.join(str(tmpdir), "file_server")
+ fake_home = os.path.join(str(tmpdir), "fake_home")
os.makedirs(file_server_files, exist_ok=True)
os.makedirs(fake_home, exist_ok=True)
project = str(datafiles)
- os.environ['HOME'] = fake_home
- with open(os.path.join(fake_home, '.netrc'), 'wb') as f:
+ os.environ["HOME"] = fake_home
+ with open(os.path.join(fake_home, ".netrc"), "wb") as f:
os.fchmod(f.fileno(), 0o700)
- f.write(b'machine 127.0.0.1\n')
- f.write(b'login testuser\n')
- f.write(b'password 12345\n')
+ f.write(b"machine 127.0.0.1\n")
+ f.write(b"login testuser\n")
+ f.write(b"password 12345\n")
with create_file_server(server_type) as server:
- server.add_user('otheruser', '12345', file_server_files)
+ server.add_user("otheruser", "12345", file_server_files)
parts = urllib.parse.urlsplit(server.base_url())
- base_url = urllib.parse.urlunsplit([parts[0], 'otheruser@{}'.format(parts[1]), *parts[2:]])
+ base_url = urllib.parse.urlunsplit([parts[0], "otheruser@{}".format(parts[1]), *parts[2:]])
generate_project_file_server(base_url, project)
- src_tar = os.path.join(file_server_files, 'a.tar.gz')
- _assemble_tar(os.path.join(str(datafiles), 'content'), 'a', src_tar)
+ src_tar = os.path.join(file_server_files, "a.tar.gz")
+ _assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar)
server.start()
- result = cli.run(project=project, args=['source', 'track', 'target.bst'])
+ result = cli.run(project=project, args=["source", "track", "target.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
result.assert_task_error(ErrorDomain.SOURCE, None)
# Test that BuildStream doesnt crash if HOME is unset while
# the netrc module is trying to find it's ~/.netrc file.
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch"))
def test_homeless_environment(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project, tmpdir)
@@ -424,11 +400,11 @@ def test_homeless_environment(cli, tmpdir, datafiles):
_assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar)
# Use a track, make sure the plugin tries to find a ~/.netrc
- result = cli.run(project=project, args=['source', 'track', 'target.bst'], env={'HOME': None})
+ result = cli.run(project=project, args=["source", "track", "target.bst"], env={"HOME": None})
result.assert_success()
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'out-of-basedir-hardlinks'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "out-of-basedir-hardlinks"))
def test_out_of_basedir_hardlinks(cli, tmpdir, datafiles):
def ensure_link(member):
# By default, python will simply duplicate files - we want
@@ -453,28 +429,28 @@ def test_out_of_basedir_hardlinks(cli, tmpdir, datafiles):
# Make sure our tarfile is actually created with the desired
# attributes set
with tarfile.open(src_tar, "r:gz") as tar:
- assert any(member.islnk() and
- member.path == "contents/to_extract/a" and
- member.linkname == "contents/elsewhere/a"
- for member in tar.getmembers())
+ assert any(
+ member.islnk() and member.path == "contents/to_extract/a" and member.linkname == "contents/elsewhere/a"
+ for member in tar.getmembers()
+ )
# Assert that we will actually create a singular copy of the file
- result = cli.run(project=project, args=['source', 'track', 'target.bst'])
+ result = cli.run(project=project, args=["source", "track", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
- original_dir = os.path.join(str(datafiles), 'contents', 'to_extract')
+ original_dir = os.path.join(str(datafiles), "contents", "to_extract")
original_contents = list_dir_contents(original_dir)
checkout_contents = list_dir_contents(checkoutdir)
assert checkout_contents == original_contents
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'out-of-basedir-hardlinks'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "out-of-basedir-hardlinks"))
def test_malicious_out_of_basedir_hardlinks(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project, tmpdir)
@@ -499,13 +475,15 @@ def test_malicious_out_of_basedir_hardlinks(cli, tmpdir, datafiles):
# Make sure our tarfile is actually created with the desired
# attributes set
with tarfile.open(src_tar, "r:gz") as tar:
- assert any(member.islnk() and
- member.path == "contents/elsewhere/malicious" and
- member.linkname == "../../../malicious_target.bst"
- for member in tar.getmembers())
+ assert any(
+ member.islnk()
+ and member.path == "contents/elsewhere/malicious"
+ and member.linkname == "../../../malicious_target.bst"
+ for member in tar.getmembers()
+ )
# Try to execute the exploit
- result = cli.run(project=project, args=['source', 'track', 'malicious_target.bst'])
+ result = cli.run(project=project, args=["source", "track", "malicious_target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['source', 'fetch', 'malicious_target.bst'])
+ result = cli.run(project=project, args=["source", "fetch", "malicious_target.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
diff --git a/tests/sources/zip.py b/tests/sources/zip.py
index 3fd43b4bb..dcb1e2637 100644
--- a/tests/sources/zip.py
+++ b/tests/sources/zip.py
@@ -12,17 +12,14 @@ from buildstream.testing import cli # pylint: disable=unused-import
from tests.testutils.file_server import create_file_server
from . import list_dir_contents
-DATA_DIR = os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- 'zip',
-)
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "zip",)
def _assemble_zip(workingdir, dstfile):
old_dir = os.getcwd()
os.chdir(workingdir)
with zipfile.ZipFile(dstfile, "w") as zipfp:
- for root, dirs, files in os.walk('.'):
+ for root, dirs, files in os.walk("."):
names = dirs + files
names = [os.path.join(root, name) for name in names]
for name in names:
@@ -32,49 +29,37 @@ def _assemble_zip(workingdir, dstfile):
def generate_project(project_dir, tmpdir):
project_file = os.path.join(project_dir, "project.conf")
- _yaml.roundtrip_dump({
- 'name': 'foo',
- 'aliases': {
- 'tmpdir': "file:///" + str(tmpdir)
- }
- }, project_file)
+ _yaml.roundtrip_dump({"name": "foo", "aliases": {"tmpdir": "file:///" + str(tmpdir)}}, project_file)
def generate_project_file_server(server, project_dir):
project_file = os.path.join(project_dir, "project.conf")
- _yaml.roundtrip_dump({
- 'name': 'foo',
- 'aliases': {
- 'tmpdir': server.base_url()
- }
- }, project_file)
+ _yaml.roundtrip_dump({"name": "foo", "aliases": {"tmpdir": server.base_url()}}, project_file)
# Test that without ref, consistency is set appropriately.
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-ref'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "no-ref"))
def test_no_ref(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project, tmpdir)
- assert cli.get_element_state(project, 'target.bst') == 'no reference'
+ assert cli.get_element_state(project, "target.bst") == "no reference"
# Test that when I fetch a nonexistent URL, errors are handled gracefully and a retry is performed.
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch"))
def test_fetch_bad_url(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project, tmpdir)
# Try to fetch it
- result = cli.run(project=project, args=[
- 'source', 'fetch', 'target.bst'
- ])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
assert "FAILURE Try #" in result.stderr
result.assert_main_error(ErrorDomain.STREAM, None)
result.assert_task_error(ErrorDomain.SOURCE, None)
# Test that when I fetch with an invalid ref, it fails.
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch"))
def test_fetch_bad_ref(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project, tmpdir)
@@ -84,15 +69,13 @@ def test_fetch_bad_ref(cli, tmpdir, datafiles):
_assemble_zip(os.path.join(str(datafiles), "content"), src_zip)
# Try to fetch it
- result = cli.run(project=project, args=[
- 'source', 'fetch', 'target.bst'
- ])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_main_error(ErrorDomain.STREAM, None)
result.assert_task_error(ErrorDomain.SOURCE, None)
# Test that when tracking with a ref set, there is a warning
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch"))
def test_track_warning(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project, tmpdir)
@@ -102,15 +85,13 @@ def test_track_warning(cli, tmpdir, datafiles):
_assemble_zip(os.path.join(str(datafiles), "content"), src_zip)
# Track it
- result = cli.run(project=project, args=[
- 'source', 'track', 'target.bst'
- ])
+ result = cli.run(project=project, args=["source", "track", "target.bst"])
result.assert_success()
assert "Potential man-in-the-middle attack!" in result.stderr
# Test that a staged checkout matches what was tarred up, with the default first subdir
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch"))
def test_stage_default_basedir(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project, tmpdir)
@@ -121,13 +102,13 @@ def test_stage_default_basedir(cli, tmpdir, datafiles):
_assemble_zip(os.path.join(str(datafiles), "content"), src_zip)
# Track, fetch, build, checkout
- result = cli.run(project=project, args=['source', 'track', 'target.bst'])
+ result = cli.run(project=project, args=["source", "track", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
# Check that the content of the first directory is checked out (base-dir: '*')
@@ -138,7 +119,7 @@ def test_stage_default_basedir(cli, tmpdir, datafiles):
# Test that a staged checkout matches what was tarred up, with an empty base-dir
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-basedir'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "no-basedir"))
def test_stage_no_basedir(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project, tmpdir)
@@ -149,13 +130,13 @@ def test_stage_no_basedir(cli, tmpdir, datafiles):
_assemble_zip(os.path.join(str(datafiles), "content"), src_zip)
# Track, fetch, build, checkout
- result = cli.run(project=project, args=['source', 'track', 'target.bst'])
+ result = cli.run(project=project, args=["source", "track", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
# Check that the full content of the tarball is checked out (base-dir: '')
@@ -166,7 +147,7 @@ def test_stage_no_basedir(cli, tmpdir, datafiles):
# Test that a staged checkout matches what was tarred up, with an explicit basedir
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'explicit-basedir'))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "explicit-basedir"))
def test_stage_explicit_basedir(cli, tmpdir, datafiles):
project = str(datafiles)
generate_project(project, tmpdir)
@@ -177,13 +158,13 @@ def test_stage_explicit_basedir(cli, tmpdir, datafiles):
_assemble_zip(os.path.join(str(datafiles), "content"), src_zip)
# Track, fetch, build, checkout
- result = cli.run(project=project, args=['source', 'track', 'target.bst'])
+ result = cli.run(project=project, args=["source", "track", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
# Check that the content of the first directory is checked out (base-dir: '*')
@@ -193,42 +174,42 @@ def test_stage_explicit_basedir(cli, tmpdir, datafiles):
assert checkout_contents == original_contents
-@pytest.mark.parametrize('server_type', ('FTP', 'HTTP'))
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
+@pytest.mark.parametrize("server_type", ("FTP", "HTTP"))
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch"))
def test_use_netrc(cli, datafiles, server_type, tmpdir):
- file_server_files = os.path.join(str(tmpdir), 'file_server')
- fake_home = os.path.join(str(tmpdir), 'fake_home')
+ file_server_files = os.path.join(str(tmpdir), "file_server")
+ fake_home = os.path.join(str(tmpdir), "fake_home")
os.makedirs(file_server_files, exist_ok=True)
os.makedirs(fake_home, exist_ok=True)
project = str(datafiles)
- checkoutdir = os.path.join(str(tmpdir), 'checkout')
+ checkoutdir = os.path.join(str(tmpdir), "checkout")
- os.environ['HOME'] = fake_home
- with open(os.path.join(fake_home, '.netrc'), 'wb') as f:
+ os.environ["HOME"] = fake_home
+ with open(os.path.join(fake_home, ".netrc"), "wb") as f:
os.fchmod(f.fileno(), 0o700)
- f.write(b'machine 127.0.0.1\n')
- f.write(b'login testuser\n')
- f.write(b'password 12345\n')
+ f.write(b"machine 127.0.0.1\n")
+ f.write(b"login testuser\n")
+ f.write(b"password 12345\n")
with create_file_server(server_type) as server:
- server.add_user('testuser', '12345', file_server_files)
+ server.add_user("testuser", "12345", file_server_files)
generate_project_file_server(server, project)
- src_zip = os.path.join(file_server_files, 'a.zip')
- _assemble_zip(os.path.join(str(datafiles), 'content'), src_zip)
+ src_zip = os.path.join(file_server_files, "a.zip")
+ _assemble_zip(os.path.join(str(datafiles), "content"), src_zip)
server.start()
- result = cli.run(project=project, args=['source', 'track', 'target.bst'])
+ result = cli.run(project=project, args=["source", "track", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
+ result = cli.run(project=project, args=["source", "fetch", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
- result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
result.assert_success()
- original_dir = os.path.join(str(datafiles), 'content', 'a')
+ original_dir = os.path.join(str(datafiles), "content", "a")
original_contents = list_dir_contents(original_dir)
checkout_contents = list_dir_contents(checkoutdir)
assert checkout_contents == original_contents
diff --git a/tests/testutils/artifactshare.py b/tests/testutils/artifactshare.py
index d96612686..8d0448f8a 100644
--- a/tests/testutils/artifactshare.py
+++ b/tests/testutils/artifactshare.py
@@ -29,7 +29,7 @@ class BaseArtifactShare:
if port is None:
raise Exception("Error occurred when starting artifact server.")
- self.repo = 'http://localhost:{}'.format(port)
+ self.repo = "http://localhost:{}".format(port)
# run():
#
@@ -51,7 +51,7 @@ class BaseArtifactShare:
cleanup_on_sigterm()
server = stack.enter_context(self._create_server())
- port = server.add_insecure_port('localhost:0')
+ port = server.add_insecure_port("localhost:0")
server.start()
except Exception:
q.put(None)
@@ -104,7 +104,6 @@ class DummyArtifactShare(BaseArtifactShare):
# enable_push (bool): Whether the share should allow pushes
#
class ArtifactShare(BaseArtifactShare):
-
def __init__(self, directory, *, quota=None, casd=False, index_only=False):
# The working directory for the artifact share (in case it
@@ -117,9 +116,9 @@ class ArtifactShare(BaseArtifactShare):
# Unless this gets more complicated, just use this directly
# in tests as a remote artifact push/pull configuration
#
- self.repodir = os.path.join(self.directory, 'repo')
+ self.repodir = os.path.join(self.directory, "repo")
os.makedirs(self.repodir)
- self.artifactdir = os.path.join(self.repodir, 'artifacts', 'refs')
+ self.artifactdir = os.path.join(self.repodir, "artifacts", "refs")
os.makedirs(self.artifactdir)
self.cas = CASCache(self.repodir, casd=casd)
@@ -130,12 +129,7 @@ class ArtifactShare(BaseArtifactShare):
super().__init__()
def _create_server(self):
- return create_server(
- self.repodir,
- quota=self.quota,
- enable_push=True,
- index_only=self.index_only,
- )
+ return create_server(self.repodir, quota=self.quota, enable_push=True, index_only=self.index_only,)
# has_object():
#
@@ -159,7 +153,7 @@ class ArtifactShare(BaseArtifactShare):
artifact_path = os.path.join(self.artifactdir, artifact_name)
try:
- with open(artifact_path, 'rb') as f:
+ with open(artifact_path, "rb") as f:
artifact_proto.ParseFromString(f.read())
except FileNotFoundError:
return None
@@ -171,8 +165,7 @@ class ArtifactShare(BaseArtifactShare):
reachable = set()
def reachable_dir(digest):
- self.cas._reachable_refs_dir(
- reachable, digest, update_mtime=False, check_exists=True)
+ self.cas._reachable_refs_dir(reachable, digest, update_mtime=False, check_exists=True)
try:
if str(artifact_proto.files):
@@ -262,20 +255,22 @@ def create_dummy_artifact_share():
share.close()
-statvfs_result = namedtuple('statvfs_result', 'f_blocks f_bfree f_bsize f_bavail')
+statvfs_result = namedtuple("statvfs_result", "f_blocks f_bfree f_bsize f_bavail")
# Assert that a given artifact is in the share
#
-def assert_shared(cli, share, project, element_name, *, project_name='test'):
+def assert_shared(cli, share, project, element_name, *, project_name="test"):
if not share.get_artifact(cli.get_artifact_name(project, project_name, element_name)):
- raise AssertionError("Artifact share at {} does not contain the expected element {}"
- .format(share.repo, element_name))
+ raise AssertionError(
+ "Artifact share at {} does not contain the expected element {}".format(share.repo, element_name)
+ )
# Assert that a given artifact is not in the share
#
-def assert_not_shared(cli, share, project, element_name, *, project_name='test'):
+def assert_not_shared(cli, share, project, element_name, *, project_name="test"):
if share.get_artifact(cli.get_artifact_name(project, project_name, element_name)):
- raise AssertionError("Artifact share at {} unexpectedly contains the element {}"
- .format(share.repo, element_name))
+ raise AssertionError(
+ "Artifact share at {} unexpectedly contains the element {}".format(share.repo, element_name)
+ )
diff --git a/tests/testutils/element_generators.py b/tests/testutils/element_generators.py
index 0fbca7f3e..6da465ab7 100644
--- a/tests/testutils/element_generators.py
+++ b/tests/testutils/element_generators.py
@@ -28,8 +28,8 @@ def create_element_size(name, project_dir, elements_path, dependencies, size):
os.makedirs(full_elements_path, exist_ok=True)
# Create a git repo
- repodir = os.path.join(project_dir, 'repos')
- repo = create_repo('git', repodir, subdir=name)
+ repodir = os.path.join(project_dir, "repos")
+ repo = create_repo("git", repodir, subdir=name)
with utils._tempdir(dir=project_dir) as tmp:
@@ -38,26 +38,24 @@ def create_element_size(name, project_dir, elements_path, dependencies, size):
# part; this ensures we never include a .git/ directory
# in the cached artifacts for these sized elements.
#
- datadir = os.path.join(tmp, 'data')
+ datadir = os.path.join(tmp, "data")
os.makedirs(datadir)
# Use /dev/urandom to create the sized file in the datadir
- with open(os.path.join(datadir, name), 'wb+') as f:
+ with open(os.path.join(datadir, name), "wb+") as f:
f.write(os.urandom(size))
# Create the git repo from the temp directory
ref = repo.create(tmp)
element = {
- 'kind': 'import',
- 'sources': [
- repo.source_config(ref=ref)
- ],
- 'config': {
+ "kind": "import",
+ "sources": [repo.source_config(ref=ref)],
+ "config": {
# Extract only the data directory
- 'source': 'data'
+ "source": "data"
},
- 'depends': dependencies
+ "depends": dependencies,
}
_yaml.roundtrip_dump(element, os.path.join(project_dir, elements_path, name))
@@ -91,9 +89,9 @@ def update_element_size(name, project_dir, repo, size):
new_file = os.path.join(tmp, name)
# Use /dev/urandom to create the sized file in the datadir
- with open(new_file, 'wb+') as f:
+ with open(new_file, "wb+") as f:
f.write(os.urandom(size))
# Modify the git repo with a new commit to the same path,
# replacing the original file with a new one.
- repo.modify_file(new_file, os.path.join('data', name))
+ repo.modify_file(new_file, os.path.join("data", name))
diff --git a/tests/testutils/file_server.py b/tests/testutils/file_server.py
index 05f896013..3e6e41954 100644
--- a/tests/testutils/file_server.py
+++ b/tests/testutils/file_server.py
@@ -6,9 +6,9 @@ from .http_server import SimpleHttpServer
@contextmanager
def create_file_server(file_server_type):
- if file_server_type == 'FTP':
+ if file_server_type == "FTP":
server = SimpleFtpServer()
- elif file_server_type == 'HTTP':
+ elif file_server_type == "HTTP":
server = SimpleHttpServer()
else:
assert False
diff --git a/tests/testutils/filetypegenerator.py b/tests/testutils/filetypegenerator.py
index 8b7d818d8..41c502be2 100644
--- a/tests/testutils/filetypegenerator.py
+++ b/tests/testutils/filetypegenerator.py
@@ -39,7 +39,7 @@ def generate_file_types(path):
clean()
- with open(path, 'w'):
+ with open(path, "w"):
pass
yield
clean()
diff --git a/tests/testutils/ftp_server.py b/tests/testutils/ftp_server.py
index 52c05f8ba..7eda90b70 100644
--- a/tests/testutils/ftp_server.py
+++ b/tests/testutils/ftp_server.py
@@ -11,7 +11,7 @@ class SimpleFtpServer(multiprocessing.Process):
self.authorizer = DummyAuthorizer()
handler = FTPHandler
handler.authorizer = self.authorizer
- self.server = FTPServer(('127.0.0.1', 0), handler)
+ self.server = FTPServer(("127.0.0.1", 0), handler)
def run(self):
self.server.serve_forever()
@@ -26,7 +26,7 @@ class SimpleFtpServer(multiprocessing.Process):
self.authorizer.add_anonymous(cwd)
def add_user(self, user, password, cwd):
- self.authorizer.add_user(user, password, cwd, perm='elradfmwMT')
+ self.authorizer.add_user(user, password, cwd, perm="elradfmwMT")
def base_url(self):
- return 'ftp://127.0.0.1:{}'.format(self.server.address[1])
+ return "ftp://127.0.0.1:{}".format(self.server.address[1])
diff --git a/tests/testutils/http_server.py b/tests/testutils/http_server.py
index b72e745c5..8591159f8 100644
--- a/tests/testutils/http_server.py
+++ b/tests/testutils/http_server.py
@@ -11,45 +11,44 @@ class Unauthorized(Exception):
class RequestHandler(SimpleHTTPRequestHandler):
-
def get_root_dir(self):
- authorization = self.headers.get('authorization')
+ authorization = self.headers.get("authorization")
if not authorization:
if not self.server.anonymous_dir:
- raise Unauthorized('unauthorized')
+ raise Unauthorized("unauthorized")
return self.server.anonymous_dir
else:
authorization = authorization.split()
- if len(authorization) != 2 or authorization[0].lower() != 'basic':
- raise Unauthorized('unauthorized')
+ if len(authorization) != 2 or authorization[0].lower() != "basic":
+ raise Unauthorized("unauthorized")
try:
- decoded = base64.decodebytes(authorization[1].encode('ascii'))
- user, password = decoded.decode('ascii').split(':')
+ decoded = base64.decodebytes(authorization[1].encode("ascii"))
+ user, password = decoded.decode("ascii").split(":")
expected_password, directory = self.server.users[user]
if password == expected_password:
return directory
- except: # noqa
- raise Unauthorized('unauthorized')
+ except: # noqa
+ raise Unauthorized("unauthorized")
return None
def unauthorized(self):
shortmsg, longmsg = self.responses[HTTPStatus.UNAUTHORIZED]
self.send_response(HTTPStatus.UNAUTHORIZED, shortmsg)
- self.send_header('Connection', 'close')
-
- content = (self.error_message_format % {
- 'code': HTTPStatus.UNAUTHORIZED,
- 'message': html.escape(longmsg, quote=False),
- 'explain': html.escape(longmsg, quote=False)
- })
- body = content.encode('UTF-8', 'replace')
- self.send_header('Content-Type', self.error_content_type)
- self.send_header('Content-Length', str(len(body)))
- self.send_header('WWW-Authenticate', 'Basic realm="{}"'.format(self.server.realm))
+ self.send_header("Connection", "close")
+
+ content = self.error_message_format % {
+ "code": HTTPStatus.UNAUTHORIZED,
+ "message": html.escape(longmsg, quote=False),
+ "explain": html.escape(longmsg, quote=False),
+ }
+ body = content.encode("UTF-8", "replace")
+ self.send_header("Content-Type", self.error_content_type)
+ self.send_header("Content-Length", str(len(body)))
+ self.send_header("WWW-Authenticate", 'Basic realm="{}"'.format(self.server.realm))
self.end_headers()
self.end_headers()
- if self.command != 'HEAD' and body:
+ if self.command != "HEAD" and body:
self.wfile.write(body)
def do_GET(self):
@@ -65,11 +64,11 @@ class RequestHandler(SimpleHTTPRequestHandler):
self.unauthorized()
def translate_path(self, path):
- path = path.split('?', 1)[0]
- path = path.split('#', 1)[0]
+ path = path.split("?", 1)[0]
+ path = path.split("#", 1)[0]
path = posixpath.normpath(path)
assert posixpath.isabs(path)
- path = posixpath.relpath(path, '/')
+ path = posixpath.relpath(path, "/")
return os.path.join(self.get_root_dir(), path)
@@ -77,14 +76,14 @@ class AuthHTTPServer(HTTPServer):
def __init__(self, *args, **kwargs):
self.users = {}
self.anonymous_dir = None
- self.realm = 'Realm'
+ self.realm = "Realm"
super().__init__(*args, **kwargs)
class SimpleHttpServer(multiprocessing.Process):
def __init__(self):
super().__init__()
- self.server = AuthHTTPServer(('127.0.0.1', 0), RequestHandler)
+ self.server = AuthHTTPServer(("127.0.0.1", 0), RequestHandler)
self.started = False
def start(self):
@@ -107,4 +106,4 @@ class SimpleHttpServer(multiprocessing.Process):
self.server.users[user] = (password, cwd)
def base_url(self):
- return 'http://127.0.0.1:{}'.format(self.server.server_port)
+ return "http://127.0.0.1:{}".format(self.server.server_port)
diff --git a/tests/testutils/junction.py b/tests/testutils/junction.py
index e867695c4..c086f6f17 100644
--- a/tests/testutils/junction.py
+++ b/tests/testutils/junction.py
@@ -19,17 +19,12 @@ def generate_junction(tmpdir, subproject_path, junction_path, *, store_ref=True)
# Create a repo to hold the subproject and generate
# a junction element for it
#
- repo = create_repo('git', str(tmpdir))
+ repo = create_repo("git", str(tmpdir))
source_ref = ref = repo.create(subproject_path)
if not store_ref:
source_ref = None
- element = {
- 'kind': 'junction',
- 'sources': [
- repo.source_config(ref=source_ref)
- ]
- }
+ element = {"kind": "junction", "sources": [repo.source_config(ref=source_ref)]}
_yaml.roundtrip_dump(element, junction_path)
return ref
diff --git a/tests/testutils/patch.py b/tests/testutils/patch.py
index df287f374..85b38def8 100644
--- a/tests/testutils/patch.py
+++ b/tests/testutils/patch.py
@@ -3,19 +3,17 @@ import subprocess
def apply(file, patch):
try:
- subprocess.check_output(['patch', file, patch])
+ subprocess.check_output(["patch", file, patch])
except subprocess.CalledProcessError as e:
- message = "Patch failed with exit code {}\n Output:\n {}".format(
- e.returncode, e.output)
+ message = "Patch failed with exit code {}\n Output:\n {}".format(e.returncode, e.output)
print(message)
raise
def remove(file, patch):
try:
- subprocess.check_output(['patch', '--reverse', file, patch])
+ subprocess.check_output(["patch", "--reverse", file, patch])
except subprocess.CalledProcessError as e:
- message = "patch --reverse failed with exit code {}\n Output:\n {}".format(
- e.returncode, e.output)
+ message = "patch --reverse failed with exit code {}\n Output:\n {}".format(e.returncode, e.output)
print(message)
raise
diff --git a/tests/testutils/python_repo.py b/tests/testutils/python_repo.py
index c8e5bf343..7d9ae4e47 100644
--- a/tests/testutils/python_repo.py
+++ b/tests/testutils/python_repo.py
@@ -7,7 +7,7 @@ import sys
import pytest
-SETUP_TEMPLATE = '''\
+SETUP_TEMPLATE = """\
from setuptools import setup
setup(
@@ -22,18 +22,18 @@ setup(
]
}}
)
-'''
+"""
# All packages generated via generate_pip_package will have the functions below
-INIT_TEMPLATE = '''\
+INIT_TEMPLATE = """\
def main():
print('This is {name}')
def hello(actor='world'):
print('Hello {{}}! This is {name}'.format(actor))
-'''
+"""
-HTML_TEMPLATE = '''\
+HTML_TEMPLATE = """\
<html>
<head>
<title>Links for {name}</title>
@@ -42,7 +42,7 @@ HTML_TEMPLATE = '''\
<a href='{name}-{version}.tar.gz'>{name}-{version}.tar.gz</a><br />
</body>
</html>
-'''
+"""
# Creates a simple python source distribution and copies this into a specified
@@ -57,11 +57,11 @@ HTML_TEMPLATE = '''\
# Returns:
# None
#
-def generate_pip_package(tmpdir, pypi, name, version='0.1', dependencies=None):
+def generate_pip_package(tmpdir, pypi, name, version="0.1", dependencies=None):
if dependencies is None:
dependencies = []
# check if package already exists in pypi
- pypi_package = os.path.join(pypi, re.sub('[^0-9a-zA-Z]+', '-', name))
+ pypi_package = os.path.join(pypi, re.sub("[^0-9a-zA-Z]+", "-", name))
if os.path.exists(pypi_package):
return
@@ -73,29 +73,22 @@ def generate_pip_package(tmpdir, pypi, name, version='0.1', dependencies=None):
# `-- package
# `-- __init__.py
#
- setup_file = os.path.join(tmpdir, 'setup.py')
- pkgdirname = re.sub('[^0-9a-zA-Z]+', '', name)
- with open(setup_file, 'w') as f:
- f.write(
- SETUP_TEMPLATE.format(
- name=name,
- version=version,
- pkgdirname=pkgdirname,
- pkgdeps=dependencies
- )
- )
+ setup_file = os.path.join(tmpdir, "setup.py")
+ pkgdirname = re.sub("[^0-9a-zA-Z]+", "", name)
+ with open(setup_file, "w") as f:
+ f.write(SETUP_TEMPLATE.format(name=name, version=version, pkgdirname=pkgdirname, pkgdeps=dependencies))
os.chmod(setup_file, 0o755)
package = os.path.join(tmpdir, pkgdirname)
os.makedirs(package)
- main_file = os.path.join(package, '__init__.py')
- with open(main_file, 'w') as f:
+ main_file = os.path.join(package, "__init__.py")
+ with open(main_file, "w") as f:
f.write(INIT_TEMPLATE.format(name=name))
os.chmod(main_file, 0o644)
# Run sdist with a fresh process
- p = subprocess.run([sys.executable, 'setup.py', 'sdist'], cwd=tmpdir)
+ p = subprocess.run([sys.executable, "setup.py", "sdist"], cwd=tmpdir)
assert p.returncode == 0
# create directory for this package in pypi resulting in a directory
@@ -109,12 +102,12 @@ def generate_pip_package(tmpdir, pypi, name, version='0.1', dependencies=None):
os.makedirs(pypi_package)
# add an index html page
- index_html = os.path.join(pypi_package, 'index.html')
- with open(index_html, 'w') as f:
+ index_html = os.path.join(pypi_package, "index.html")
+ with open(index_html, "w") as f:
f.write(HTML_TEMPLATE.format(name=name, version=version))
# copy generated tarfile to pypi package
- dist_dir = os.path.join(tmpdir, 'dist')
+ dist_dir = os.path.join(tmpdir, "dist")
for tar in os.listdir(dist_dir):
tarpath = os.path.join(dist_dir, tar)
shutil.copy(tarpath, pypi_package)
@@ -123,7 +116,7 @@ def generate_pip_package(tmpdir, pypi, name, version='0.1', dependencies=None):
@pytest.fixture
def setup_pypi_repo(tmpdir):
def create_pkgdir(package):
- pkgdirname = re.sub('[^0-9a-zA-Z]+', '', package)
+ pkgdirname = re.sub("[^0-9a-zA-Z]+", "", package)
pkgdir = os.path.join(str(tmpdir), pkgdirname)
os.makedirs(pkgdir)
return pkgdir
diff --git a/tests/testutils/repo/bzr.py b/tests/testutils/repo/bzr.py
index 074712af1..b6983416e 100644
--- a/tests/testutils/repo/bzr.py
+++ b/tests/testutils/repo/bzr.py
@@ -7,7 +7,6 @@ from buildstream.testing._utils.site import BZR, BZR_ENV, HAVE_BZR
class Bzr(Repo):
-
def __init__(self, directory, subdir):
if not HAVE_BZR:
pytest.skip("bzr is not available")
@@ -20,34 +19,29 @@ class Bzr(Repo):
def create(self, directory):
# Work around race condition in bzr's creation of ~/.bazaar in
# ensure_config_dir_exists() when running tests in parallel.
- bazaar_config_dir = os.path.expanduser('~/.bazaar')
+ bazaar_config_dir = os.path.expanduser("~/.bazaar")
os.makedirs(bazaar_config_dir, exist_ok=True)
- branch_dir = os.path.join(self.repo, 'trunk')
+ branch_dir = os.path.join(self.repo, "trunk")
- subprocess.call([self.bzr, 'init-repo', self.repo], env=self.env)
- subprocess.call([self.bzr, 'init', branch_dir], env=self.env)
+ subprocess.call([self.bzr, "init-repo", self.repo], env=self.env)
+ subprocess.call([self.bzr, "init", branch_dir], env=self.env)
self.copy_directory(directory, branch_dir)
- subprocess.call([self.bzr, 'add', '.'], env=self.env, cwd=branch_dir)
- subprocess.call([self.bzr, 'commit', '--message="Initial commit"'],
- env=self.env, cwd=branch_dir)
+ subprocess.call([self.bzr, "add", "."], env=self.env, cwd=branch_dir)
+ subprocess.call([self.bzr, "commit", '--message="Initial commit"'], env=self.env, cwd=branch_dir)
return self.latest_commit()
def source_config(self, ref=None):
- config = {
- 'kind': 'bzr',
- 'url': 'file://' + self.repo,
- 'track': 'trunk'
- }
+ config = {"kind": "bzr", "url": "file://" + self.repo, "track": "trunk"}
if ref is not None:
- config['ref'] = ref
+ config["ref"] = ref
return config
def latest_commit(self):
- return subprocess.check_output([
- self.bzr, 'version-info',
- '--custom', '--template={revno}',
- os.path.join(self.repo, 'trunk')
- ], env=self.env, universal_newlines=True).strip()
+ return subprocess.check_output(
+ [self.bzr, "version-info", "--custom", "--template={revno}", os.path.join(self.repo, "trunk")],
+ env=self.env,
+ universal_newlines=True,
+ ).strip()
diff --git a/tests/testutils/repo/git.py b/tests/testutils/repo/git.py
index 46694fcf2..b9360e9cd 100644
--- a/tests/testutils/repo/git.py
+++ b/tests/testutils/repo/git.py
@@ -9,7 +9,6 @@ from buildstream.testing._utils.site import GIT, GIT_ENV, HAVE_GIT
class Git(Repo):
-
def __init__(self, directory, subdir):
if not HAVE_GIT:
pytest.skip("git is not available")
@@ -24,99 +23,87 @@ class Git(Repo):
def _run_git(self, *args, **kwargs):
argv = [GIT]
argv.extend(args)
- if 'env' not in kwargs:
- kwargs['env'] = dict(self.env, PWD=self.repo)
- kwargs.setdefault('cwd', self.repo)
- kwargs.setdefault('check', True)
+ if "env" not in kwargs:
+ kwargs["env"] = dict(self.env, PWD=self.repo)
+ kwargs.setdefault("cwd", self.repo)
+ kwargs.setdefault("check", True)
return subprocess.run(argv, **kwargs)
def create(self, directory):
self.copy_directory(directory, self.repo)
- self._run_git('init', '.')
- self._run_git('add', '.')
- self._run_git('commit', '-m', 'Initial commit')
+ self._run_git("init", ".")
+ self._run_git("add", ".")
+ self._run_git("commit", "-m", "Initial commit")
return self.latest_commit()
def add_tag(self, tag):
- self._run_git('tag', tag)
+ self._run_git("tag", tag)
def add_annotated_tag(self, tag, message):
- self._run_git('tag', '-a', tag, '-m', message)
+ self._run_git("tag", "-a", tag, "-m", message)
def add_commit(self):
- self._run_git('commit', '--allow-empty', '-m', 'Additional commit')
+ self._run_git("commit", "--allow-empty", "-m", "Additional commit")
return self.latest_commit()
def add_file(self, filename):
shutil.copy(filename, self.repo)
- self._run_git('add', os.path.basename(filename))
- self._run_git('commit', '-m', 'Added {}'.format(os.path.basename(filename)))
+ self._run_git("add", os.path.basename(filename))
+ self._run_git("commit", "-m", "Added {}".format(os.path.basename(filename)))
return self.latest_commit()
def modify_file(self, new_file, path):
shutil.copy(new_file, os.path.join(self.repo, path))
- self._run_git('commit', path, '-m', 'Modified {}'.format(os.path.basename(path)))
+ self._run_git("commit", path, "-m", "Modified {}".format(os.path.basename(path)))
return self.latest_commit()
def add_submodule(self, subdir, url=None, checkout=None):
submodule = {}
if checkout is not None:
- submodule['checkout'] = checkout
+ submodule["checkout"] = checkout
if url is not None:
- submodule['url'] = url
+ submodule["url"] = url
self.submodules[subdir] = submodule
- self._run_git('submodule', 'add', url, subdir)
- self._run_git('commit', '-m', 'Added the submodule')
+ self._run_git("submodule", "add", url, subdir)
+ self._run_git("commit", "-m", "Added the submodule")
return self.latest_commit()
# This can also be used to a file or a submodule
def remove_path(self, path):
- self._run_git('rm', path)
- self._run_git('commit', '-m', 'Removing {}'.format(path))
+ self._run_git("rm", path)
+ self._run_git("commit", "-m", "Removing {}".format(path))
return self.latest_commit()
def source_config(self, ref=None):
return self.source_config_extra(ref)
def source_config_extra(self, ref=None, checkout_submodules=None):
- config = {
- 'kind': 'git',
- 'url': 'file://' + self.repo,
- 'track': 'master'
- }
+ config = {"kind": "git", "url": "file://" + self.repo, "track": "master"}
if ref is not None:
- config['ref'] = ref
+ config["ref"] = ref
if checkout_submodules is not None:
- config['checkout-submodules'] = checkout_submodules
+ config["checkout-submodules"] = checkout_submodules
if self.submodules:
- config['submodules'] = dict(self.submodules)
+ config["submodules"] = dict(self.submodules)
return config
def latest_commit(self):
- return self._run_git(
- 'rev-parse', 'HEAD',
- stdout=subprocess.PIPE,
- universal_newlines=True,
- ).stdout.strip()
+ return self._run_git("rev-parse", "HEAD", stdout=subprocess.PIPE, universal_newlines=True,).stdout.strip()
def branch(self, branch_name):
- self._run_git('checkout', '-b', branch_name)
+ self._run_git("checkout", "-b", branch_name)
def delete_tag(self, tag_name):
- self._run_git('tag', '-d', tag_name)
+ self._run_git("tag", "-d", tag_name)
def checkout(self, commit):
- self._run_git('checkout', commit)
+ self._run_git("checkout", commit)
def merge(self, commit):
- self._run_git('merge', '-m', 'Merge', commit)
+ self._run_git("merge", "-m", "Merge", commit)
return self.latest_commit()
def rev_parse(self, rev):
- return self._run_git(
- 'rev-parse', rev,
- stdout=subprocess.PIPE,
- universal_newlines=True,
- ).stdout.strip()
+ return self._run_git("rev-parse", rev, stdout=subprocess.PIPE, universal_newlines=True,).stdout.strip()
diff --git a/tests/testutils/repo/tar.py b/tests/testutils/repo/tar.py
index 63231fa4b..3eb9d896b 100644
--- a/tests/testutils/repo/tar.py
+++ b/tests/testutils/repo/tar.py
@@ -7,9 +7,8 @@ from buildstream.testing import Repo
class Tar(Repo):
-
def create(self, directory):
- tarball = os.path.join(self.repo, 'file.tar.gz')
+ tarball = os.path.join(self.repo, "file.tar.gz")
old_dir = os.getcwd()
os.chdir(directory)
@@ -20,14 +19,9 @@ class Tar(Repo):
return sha256sum(tarball)
def source_config(self, ref=None):
- tarball = os.path.join(self.repo, 'file.tar.gz')
- config = {
- 'kind': 'tar',
- 'url': 'file://' + tarball,
- 'directory': '',
- 'base-dir': ''
- }
+ tarball = os.path.join(self.repo, "file.tar.gz")
+ config = {"kind": "tar", "url": "file://" + tarball, "directory": "", "base-dir": ""}
if ref is not None:
- config['ref'] = ref
+ config["ref"] = ref
return config
diff --git a/tests/testutils/repo/zip.py b/tests/testutils/repo/zip.py
index df3f834a9..38419e642 100644
--- a/tests/testutils/repo/zip.py
+++ b/tests/testutils/repo/zip.py
@@ -7,14 +7,13 @@ from buildstream.testing import Repo
class Zip(Repo):
-
def create(self, directory):
- archive = os.path.join(self.repo, 'file.zip')
+ archive = os.path.join(self.repo, "file.zip")
old_dir = os.getcwd()
os.chdir(directory)
with zipfile.ZipFile(archive, "w") as zipfp:
- for root, dirs, files in os.walk('.'):
+ for root, dirs, files in os.walk("."):
names = dirs + files
names = [os.path.join(root, name) for name in names]
@@ -26,14 +25,9 @@ class Zip(Repo):
return sha256sum(archive)
def source_config(self, ref=None):
- archive = os.path.join(self.repo, 'file.zip')
- config = {
- 'kind': 'zip',
- 'url': 'file://' + archive,
- 'directory': '',
- 'base-dir': ''
- }
+ archive = os.path.join(self.repo, "file.zip")
+ config = {"kind": "zip", "url": "file://" + archive, "directory": "", "base-dir": ""}
if ref is not None:
- config['ref'] = ref
+ config["ref"] = ref
return config
diff --git a/tests/testutils/setuptools.py b/tests/testutils/setuptools.py
index cb61e1976..0f7f30f91 100644
--- a/tests/testutils/setuptools.py
+++ b/tests/testutils/setuptools.py
@@ -4,19 +4,17 @@ import pkg_resources
# A mock setuptools dist object.
-class MockDist():
+class MockDist:
def __init__(self, datafiles, module_name):
self.datafiles = datafiles
self.module_name = module_name
def get_resource_filename(self, *_args, **_kwargs):
- return os.path.join(self.datafiles.dirname,
- self.datafiles.basename,
- self.module_name)
+ return os.path.join(self.datafiles.dirname, self.datafiles.basename, self.module_name)
# A mock setuptools entry object.
-class MockEntry():
+class MockEntry:
def __init__(self, datafiles, module_name):
self.dist = MockDist(datafiles, module_name)
self.module_name = module_name
@@ -30,7 +28,7 @@ class MockEntry():
@pytest.fixture()
def entry_fixture(monkeypatch):
def patch(datafiles, entry_point, lookup_string):
- dist, package = lookup_string.split(':')
+ dist, package = lookup_string.split(":")
def mock_entry(pdist, pentry_point, ppackage):
assert pdist == dist
@@ -38,6 +36,7 @@ def entry_fixture(monkeypatch):
assert ppackage == package
return MockEntry(datafiles, package)
- monkeypatch.setattr(pkg_resources, 'get_entry_info', mock_entry)
+
+ monkeypatch.setattr(pkg_resources, "get_entry_info", mock_entry)
return patch
diff --git a/tox.ini b/tox.ini
index 4c18f2c14..30e398ceb 100644
--- a/tox.ini
+++ b/tox.ini
@@ -103,6 +103,26 @@ setenv =
COVERAGE_FILE = {toxinidir}/.coverage-reports/.coverage
#
+# Code formatters
+#
+[testenv:format]
+skip_install = True
+deps =
+ black==19.10b0
+commands =
+ black {posargs: src tests doc/source/conf.py}
+
+#
+# Code format checkers
+#
+[testenv:format-check]
+skip_install = True
+deps =
+ black==19.10b0
+commands =
+ black --check --diff {posargs: src tests doc/source/conf.py}
+
+#
# Running linters
#
[testenv:lint]
@@ -111,8 +131,7 @@ commands_pre =
{envpython} setup.py build_ext --inplace
commands =
- pycodestyle {posargs}
- pylint {posargs: src tests}
+ pylint {posargs: src tests doc/source/conf.py}
deps =
-rrequirements/requirements.txt
-rrequirements/dev-requirements.txt