diff options
author | Chandan Singh <chandan@chandansingh.net> | 2019-02-04 19:54:49 +0000 |
---|---|---|
committer | Chandan Singh <chandan@chandansingh.net> | 2019-02-04 19:54:49 +0000 |
commit | f4fa33c8583607d8f481259a81fd77bf1ef33945 (patch) | |
tree | a452dc3f7cea0f55e46f608c1ce6162dd6d0541e | |
parent | 2a50a3db153f004c44d803c249ed3039060aecb6 (diff) | |
parent | c70d2bb3ea078c50ed4119661d79a01e54bf73f2 (diff) | |
download | buildstream-f4fa33c8583607d8f481259a81fd77bf1ef33945.tar.gz |
Merge branch 'chandan/toxify-1.2' into 'bst-1.2'
Run tests using tox
See merge request BuildStream/buildstream!1111
-rw-r--r-- | .coveragerc | 6 | ||||
-rw-r--r-- | .gitignore | 3 | ||||
-rw-r--r-- | .gitlab-ci.yml | 145 | ||||
-rw-r--r-- | MANIFEST.in | 10 | ||||
-rw-r--r-- | Makefile | 21 | ||||
-rwxr-xr-x | conftest.py | 2 | ||||
-rwxr-xr-x | doc/bst2html.py | 2 | ||||
-rw-r--r-- | requirements/dev-requirements.in (renamed from dev-requirements.txt) | 0 | ||||
-rw-r--r-- | requirements/dev-requirements.txt | 27 | ||||
-rw-r--r-- | requirements/plugin-requirements.in | 2 | ||||
-rw-r--r-- | requirements/plugin-requirements.txt | 4 | ||||
-rw-r--r-- | requirements/requirements.in | 8 | ||||
-rw-r--r-- | requirements/requirements.txt | 11 | ||||
-rwxr-xr-x | setup.py | 16 | ||||
-rw-r--r-- | tox.ini | 70 |
15 files changed, 202 insertions, 125 deletions
diff --git a/.coveragerc b/.coveragerc index d81aec1a2..ca47c8a90 100644 --- a/.coveragerc +++ b/.coveragerc @@ -4,11 +4,15 @@ include = */buildstream/* omit = - # Omit profiling helper module + # Omit some internals */buildstream/_profile.py + */buildstream/__main__.py + */buildstream/_version.py # Omit generated code */buildstream/_protos/* */.eggs/* + # Omit .tox directory + */.tox/* [report] show_missing = True diff --git a/.gitignore b/.gitignore index fbcda7210..02c98fda8 100644 --- a/.gitignore +++ b/.gitignore @@ -13,9 +13,12 @@ tests/**/*.pyc integration-cache/ tmp .coverage +.coverage-reports/ .coverage.* .cache +.pytest_cache/ *.bst/ +.tox/ # Pycache, in case buildstream is ran directly from within the source # tree diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 186bcc6da..3a1fe532c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: buildstream/testsuite-debian:9-master-114-4cab18e3 +image: buildstream/testsuite-debian:9-5da27168-32c47d1c cache: key: "$CI_JOB_NAME-" @@ -6,49 +6,13 @@ cache: - cache/ stages: - - prepare - test - post -##################################################### -# Prepare stage # -##################################################### - -# Create a source distribution -# -source_dist: - stage: prepare - script: - - # Generate the source distribution tarball - # - - python3 setup.py sdist - - tar -ztf dist/* - - tarball=$(cd dist && echo $(ls *)) - - # Create an installer script - - | - cat > dist/install.sh << EOF - #!/bin/sh - tar -zxf ${tarball} - cd ${tarball%.tar.gz} - pip3 install --no-index . - EOF - - # unpack tarball as `dist/buildstream` directory - - | - cat > dist/unpack.sh << EOF - #!/bin/sh - tar -zxf ${tarball} - mv ${tarball%.tar.gz} buildstream - EOF - - # Make our helpers executable - - chmod +x dist/install.sh - - chmod +x dist/unpack.sh - artifacts: - paths: - - dist/ +variables: + INTEGRATION_CACHE: "${CI_PROJECT_DIR}/cache/integration-cache" + TEST_COMMAND: "tox -- --color=yes --integration" + COVERAGE_PREFIX: "${CI_JOB_NAME}." ##################################################### @@ -57,66 +21,53 @@ source_dist: # Run premerge commits # -.linux-tests-template: &linux-tests +.tests-template: &tests stage: test - variables: - PYTEST_ADDOPTS: "--color=yes" - script: + + before_script: # Diagnostics - mount - df -h + script: + - mkdir -p "${INTEGRATION_CACHE}" - useradd -Um buildstream - chown -R buildstream:buildstream . - - export INTEGRATION_CACHE="$(pwd)/cache/integration-cache" - - # Unpack and get into dist/buildstream - - cd dist && ./unpack.sh - - chown -R buildstream:buildstream buildstream - - cd buildstream - - # Run the tests from the source distribution, We run as a simple - # user to test for permission issues - - su buildstream -c 'python3 setup.py test --index-url invalid://uri --addopts --integration' + # Run the tests as a simple user to test for permission issues + - su buildstream -c "${TEST_COMMAND}" - # Go back to the toplevel and collect our reports - - cd ../.. - - mkdir -p coverage-linux/ - - cp dist/buildstream/.coverage.* coverage-linux/coverage."${CI_JOB_NAME}" except: - schedules artifacts: paths: - - coverage-linux/ + - .coverage-reports tests-debian-9: - image: buildstream/testsuite-debian:9-master-117-aa3a33b3 - <<: *linux-tests + image: buildstream/testsuite-debian:9-5da27168-32c47d1c + <<: *tests tests-fedora-27: - image: buildstream/testsuite-fedora:27-master-117-aa3a33b3 - <<: *linux-tests + image: buildstream/testsuite-fedora:27-5da27168-32c47d1c + <<: *tests tests-fedora-28: - image: buildstream/testsuite-fedora:28-master-117-aa3a33b3 - <<: *linux-tests + image: buildstream/testsuite-fedora:28-5da27168-32c47d1c + <<: *tests tests-ubuntu-18.04: - image: buildstream/testsuite-ubuntu:18.04-master-117-aa3a33b3 - <<: *linux-tests + image: buildstream/testsuite-ubuntu:18.04-5da27168-32c47d1c + <<: *tests tests-unix: # Use fedora here, to a) run a test on fedora and b) ensure that we # can get rid of ostree - this is not possible with debian-8 - image: buildstream/testsuite-fedora:27-master-117-aa3a33b3 - stage: test + image: buildstream/testsuite-fedora:27-5da27168-32c47d1c + <<: *tests variables: BST_FORCE_BACKEND: "unix" - PYTEST_ADDOPTS: "--color=yes" - script: - - export INTEGRATION_CACHE="$(pwd)/cache/integration-cache" + script: # We remove the Bubblewrap and OSTree packages here so that we catch any # codepaths that try to use them. Removing OSTree causes fuse-libs to @@ -124,39 +75,19 @@ tests-unix: - dnf mark install fuse-libs - dnf erase -y bubblewrap ostree - # Unpack and get into dist/buildstream - - cd dist && ./unpack.sh && cd buildstream - # Since the unix platform is required to run as root, no user change required - - python3 setup.py test --index-url invalid://uri --addopts --integration - - # Go back to the toplevel and collect our reports - - cd ../.. - - mkdir -p coverage-unix/ - - cp dist/buildstream/.coverage.* coverage-unix/coverage.unix - except: - - schedules - artifacts: - paths: - - coverage-unix/ - - logs-unix/ + - ${TEST_COMMAND} # Automatically build documentation for every commit, we want to know # if building documentation fails even if we're not deploying it. -# Note: We still do not enforce a consistent installation of python3-sphinx, -# as it will significantly grow the backing image. docs: stage: test + variables: + BST_FORCE_SESSION_REBUILD: 1 script: - - export BST_SOURCE_CACHE="$(pwd)/cache/integration-cache/sources" - - pip3 install sphinx - - pip3 install sphinx-click - - pip3 install sphinx_rtd_theme - - cd dist && ./unpack.sh && cd buildstream - - make BST_FORCE_SESSION_REBUILD=1 -C doc - - cd ../.. - - mv dist/buildstream/doc/build/html public + - env BST_SOURCE_CACHE="$(pwd)/cache/integration-cache/sources" tox -e docs + - mv doc/build/html public except: - schedules artifacts: @@ -170,7 +101,8 @@ docs: bst_ext_ref: 1d6ab71151b93c8cbc0a91a36ffe9270f3b835f1 # 0.5.1 fd_sdk_ref: 88d7c22c2281b987faa02edd57df80d430eecf1f # 18.08.12 before_script: - - (cd dist && ./unpack.sh && cd buildstream && pip3 install .) + - pip3 install -r requirements/requirements.txt -r requirements/plugin-requirements.txt + - pip3 install --no-index . - pip3 install --user -e ${bst_ext_url}@${bst_ext_ref}#egg=bst_ext - git clone https://gitlab.com/freedesktop-sdk/freedesktop-sdk.git - git -C freedesktop-sdk checkout ${fd_sdk_ref} @@ -253,21 +185,15 @@ coverage: stage: post coverage: '/TOTAL +\d+ +\d+ +(\d+\.\d+)%/' script: - - cd dist && ./unpack.sh && cd buildstream - - pip3 install --no-index . - - mkdir report - - cd report - - cp ../../../coverage-unix/coverage.unix . - - cp ../../../coverage-linux/coverage.* . - - ls coverage.* - - coverage combine --rcfile=../.coveragerc -a coverage.* - - coverage report --rcfile=../.coveragerc -m + - cp -a .coverage-reports/ ./coverage-sources + - tox -e coverage + - cp -a .coverage-reports/ ./coverage-report dependencies: - tests-debian-9 - tests-fedora-27 - tests-fedora-28 + - tests-ubuntu-18.04 - tests-unix - - source_dist except: - schedules @@ -276,7 +202,6 @@ coverage: pages: stage: post dependencies: - - source_dist - docs script: - find public/ diff --git a/MANIFEST.in b/MANIFEST.in index f9e506027..7d234d1d4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -23,4 +23,12 @@ recursive-include tests *.expected recursive-include buildstream/_protos *.proto # Requirements files -include dev-requirements.txt +include requirements/requirements.in +include requirements/requirements.txt +include requirements/dev-requirements.in +include requirements/dev-requirements.txt +include requirements/plugin-requirements.in +include requirements/plugin-requirements.txt + +# Versioneer +include versioneer.py diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..c49a2e2d0 --- /dev/null +++ b/Makefile @@ -0,0 +1,21 @@ +# Makefile for updating BuildStream's requirements files. +# + +REQUIREMENTS_IN := $(wildcard *.in) +REQUIREMENTS_TXT := $(REQUIREMENTS_IN:.in=.txt) +PYTHON := python3 +VENV := $(PYTHON) -m venv + +VENV_PIP = $(VENVDIR)/bin/pip + + +.PHONY: all + +all: $(REQUIREMENTS_TXT) + +%.txt: %.in + $(eval VENVDIR := $(shell mktemp -d $(CURDIR)/.bst-venv.XXXXXX)) + $(VENV) $(VENVDIR) + $(VENV_PIP) install -r $^ + $(VENV_PIP) freeze -r $^ > $@ + rm -rf $(VENVDIR) diff --git a/conftest.py b/conftest.py index 99a602db4..193bef02d 100755 --- a/conftest.py +++ b/conftest.py @@ -30,7 +30,7 @@ def pytest_addoption(parser): def pytest_runtest_setup(item): - if item.get_marker('integration') and not item.config.getvalue('integration'): + if item.get_closest_marker('integration') and not item.config.getvalue('integration'): pytest.skip('skipping integration test') diff --git a/doc/bst2html.py b/doc/bst2html.py index 47ea662b5..193e29a43 100755 --- a/doc/bst2html.py +++ b/doc/bst2html.py @@ -440,7 +440,7 @@ def run_session(description, tempdir, source_cache, palette, config_file, force) @click.option('--palette', '-p', default='tango', type=click.Choice(['solarized', 'solarized-xterm', 'tango', 'xterm', 'console']), help="Selects a palette for the output style") -@click.argument('description', click.Path(file_okay=True, dir_okay=False, readable=True)) +@click.argument('description', type=click.Path(file_okay=True, dir_okay=False, readable=True)) def run_bst(directory, force, source_cache, description, palette): """Run a bst command and capture stdout/stderr in html diff --git a/dev-requirements.txt b/requirements/dev-requirements.in index c88b4c723..c88b4c723 100644 --- a/dev-requirements.txt +++ b/requirements/dev-requirements.in diff --git a/requirements/dev-requirements.txt b/requirements/dev-requirements.txt new file mode 100644 index 000000000..6d0886d99 --- /dev/null +++ b/requirements/dev-requirements.txt @@ -0,0 +1,27 @@ +coverage==4.4 +pep8==1.7.1 +pylint==2.1.1 +pytest==4.1.1 +pytest-cov==2.6.1 +pytest-datafiles==2.0 +pytest-env==0.6.2 +pytest-pep8==1.0.6 +pytest-pylint==0.14.0 +pytest-xdist==1.26.0 +pytest-timeout==1.3.3 +## The following requirements were added by pip freeze: +apipkg==1.5 +astroid==2.1.0 +atomicwrites==1.2.1 +attrs==18.2.0 +execnet==1.5.0 +isort==4.3.4 +lazy-object-proxy==1.3.1 +mccabe==0.6.1 +more-itertools==5.0.0 +pluggy==0.8.1 +py==1.7.0 +pytest-cache==1.0 +pytest-forked==1.0.1 +six==1.12.0 +wrapt==1.11.0 diff --git a/requirements/plugin-requirements.in b/requirements/plugin-requirements.in new file mode 100644 index 000000000..e3527285d --- /dev/null +++ b/requirements/plugin-requirements.in @@ -0,0 +1,2 @@ +arpy +PyGObject diff --git a/requirements/plugin-requirements.txt b/requirements/plugin-requirements.txt new file mode 100644 index 000000000..6841612bd --- /dev/null +++ b/requirements/plugin-requirements.txt @@ -0,0 +1,4 @@ +arpy==1.1.1 +PyGObject==3.30.4 +## The following requirements were added by pip freeze: +pycairo==1.18.0 diff --git a/requirements/requirements.in b/requirements/requirements.in new file mode 100644 index 000000000..1743e1c3f --- /dev/null +++ b/requirements/requirements.in @@ -0,0 +1,8 @@ +Click +grpcio >= 1.10 +jinja2 >= 2.10 +pluginbase +protobuf >= 3.5 +psutil +ruamel.yaml < 0.15.52 +setuptools diff --git a/requirements/requirements.txt b/requirements/requirements.txt new file mode 100644 index 000000000..96645cb4b --- /dev/null +++ b/requirements/requirements.txt @@ -0,0 +1,11 @@ +Click==7.0 +grpcio==1.18.0 +pluginbase==0.7 +protobuf==3.6.1 +psutil==5.4.8 +ruamel.yaml==0.15.51 +setuptools==40.6.2 +## The following requirements were added by pip freeze: +Jinja2==2.10 +MarkupSafe==1.1.0 +six==1.12.0 @@ -228,9 +228,12 @@ def get_cmdclass(): ##################################################### # Gather requirements # ##################################################### -with open('dev-requirements.txt') as dev_reqs: +with open('requirements/dev-requirements.in') as dev_reqs: dev_requires = dev_reqs.read().splitlines() +with open('requirements/requirements.in') as install_reqs: + install_requires = install_reqs.read().splitlines() + ##################################################### # Prepare package description from README # ##################################################### @@ -291,16 +294,7 @@ setup(name='BuildStream', os.path.join('buildstream', 'data', 'bst') ]) ], - install_requires=[ - 'setuptools', - 'psutil', - 'ruamel.yaml < 0.15.52', - 'pluginbase', - 'Click', - 'jinja2 >= 2.10', - 'protobuf >= 3.5', - 'grpcio >= 1.10', - ], + install_requires=install_requires, entry_points=bst_install_entry_points, setup_requires=['pytest-runner'], tests_require=dev_requires, diff --git a/tox.ini b/tox.ini new file mode 100644 index 000000000..2a3654722 --- /dev/null +++ b/tox.ini @@ -0,0 +1,70 @@ +# +# Tox global configuration +# +[tox] +envlist = py35,py36,py37 +skip_missing_interpreters = true + +# +# Defaults for all environments +# +# Anything specified here is iherited by the sections +# +[testenv] +commands = + pytest --basetemp {envtmpdir} {posargs} + mkdir -p .coverage-reports + mv {envtmpdir}/.coverage {toxinidir}/.coverage-reports/.coverage.{env:COVERAGE_PREFIX:}{envname} +deps = + -rrequirements/requirements.txt + -rrequirements/dev-requirements.txt + -rrequirements/plugin-requirements.txt +passenv = + BST_FORCE_BACKEND + GI_TYPELIB_PATH + INTEGRATION_CACHE + +# +# These keys are not inherited by any other sections +# +setenv = + py{35,36,37}: COVERAGE_FILE = {envtmpdir}/.coverage +whitelist_externals = + py{35,36,37}: + mv + mkdir + +# +# Coverage reporting +# +[testenv:coverage] +commands = + - coverage combine --rcfile={toxinidir}/.coveragerc {toxinidir}/.coverage-reports/ + coverage report --rcfile={toxinidir}/.coveragerc -m +deps = + -rrequirements/requirements.txt + -rrequirements/dev-requirements.txt +setenv = + COVERAGE_FILE = {toxinidir}/.coverage-reports/.coverage + +# +# Building documentation +# +[testenv:docs] +commands = + make -C doc +# sphinx_rtd_theme < 0.4.2 breaks search functionality for Sphinx >= 1.8 +deps = + sphinx + sphinx-click + sphinx_rtd_theme >= 0.4.2 + -rrequirements/requirements.txt + -rrequirements/plugin-requirements.txt +passenv = + BST_FORCE_SESSION_REBUILD + BST_SOURCE_CACHE + HOME + LANG + LC_ALL +whitelist_externals = + make |