Add tox env for external standard plugin tests

Adds tooling to define a list of external plugin repos in
`requirements/external-requirements.yml` in a nice yaml format, which
seemed the easiest way to sort out the potential config issues. The main
thing is the addition of the `requirements/parse-external.py` script,
which generates a list of package names and two requirements.txt files,
one for fixed versions, one for master.

It is assumed that `master` more means "test against a branch" and
`fixed` means "test against a tag".

Also adds tox environments to run these tests.

6 files changed, 82 insertions, 21 deletions

diff --git a/requirements/external-fixed.txt b/requirements/external-fixed.txt
new file mode 100644
index 000000000..81607389d
--- /dev/null
+++ b/requirements/external-fixed.txt
@@ -0,0 +1 @@
+git+https://gitlab.com/buildstream/bst-plugins-experimental.git@0.12.0#egg=bst_plugins_experimental[ostree]
diff --git a/requirements/external-master.txt b/requirements/external-master.txt
new file mode 100644
index 000000000..2d84de27d
--- /dev/null
+++ b/requirements/external-master.txt
@@ -0,0 +1 @@
+git+https://gitlab.com/buildstream/bst-plugins-experimental.git@master#egg=bst_plugins_experimental[ostree]
diff --git a/requirements/external-requirements.yml b/requirements/external-requirements.yml
new file mode 100644
index 000000000..cf753577b
--- /dev/null
+++ b/requirements/external-requirements.yml
@@ -0,0 +1,6 @@
+- package_name: bst_plugins_experimental
+  url: https://gitlab.com/buildstream/bst-plugins-experimental.git
+  branch: master
+  tag: 0.12.0
+  extras:
+  - ostree
diff --git a/requirements/parse-external.py b/requirements/parse-external.py
new file mode 100755
index 000000000..9df95a225
--- /dev/null
+++ b/requirements/parse-external.py
@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+
+from enum import Enum
+from ruamel import yaml
+
+
+class PluginVersion(Enum):
+    MASTER = 0
+    FIXED = 1
+
+
+def load_file(filename):
+    with open(filename, 'r') as stream:
+        try:
+            return yaml.safe_load(stream)
+        except yaml.YAMLError as exc:
+            print(exc)
+
+
+def format_single_req(dep, version):
+    extras = ''
+    if 'extras' in dep.keys():
+        extras = ','.join(dep['extras'])
+
+    if version == PluginVersion.MASTER:
+        ref = 'master'
+        if 'branch' in dep.keys():
+            ref = dep['branch']
+    else:
+        ref = dep['tag']
+
+    return "git+{}@{}#egg={}[{}]\n".format(
+        dep['url'],
+        ref,
+        dep['package_name'],
+        extras)
+
+
+def write_requirements_file(filename, deps, version):
+    with open(filename, 'w') as f:
+        for dep in deps:
+            f.write(format_single_req(dep, version))
+
+
+if __name__ == '__main__':
+    deps = load_file('requirements/external-requirements.yml')
+
+    write_requirements_file('requirements/external-master.txt', deps, PluginVersion.MASTER)
+    write_requirements_file('requirements/external-fixed.txt', deps, PluginVersion.FIXED)
diff --git a/tests/conftest.py b/tests/conftest.py
index a1ca97fb7..079a76317 100755
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -20,6 +20,7 @@
 #        Tristan Maat <tristan.maat@codethink.co.uk>
 #
 import os
+import pkg_resources
 import pytest
 
 from buildstream.testing import register_repo_kind, sourcetests_collection_hook
@@ -51,8 +52,16 @@ def pytest_addoption(parser):
     parser.addoption('--remote-execution', action='store_true', default=False,
                      help='Run remote-execution tests only')
 
+    # This doesn't actually do anything at the moment, as we don't have standardised
+    # element tests
+    parser.addoption('--external-elements', action='store_true', default=False,
+                     help="Run standardised tests for external elements")
+
+    parser.addoption('--external-sources', action='store_true', default=False,
+                     help="Run standardised tests for external elements")
+
     parser.addoption('--external-plugins', action='store_true', default=False,
-                     help="Run standardised tests for external plugins")
+                     help="Run standardised tests for external elements and sources")
 
 
 def pytest_runtest_setup(item):
@@ -139,11 +148,14 @@ register_repo_kind('zip', Zip, None)
 # This hook enables pytest to collect the templated source tests from
 # buildstream.testing
 def pytest_sessionstart(session):
-    sourcetests_collection_hook(session)
+    # Register sources from external plugins, if option passed
+    if session.config.getvalue('external_plugins') or session.config.getvalue('external_sources'):
+        packages = {entrypoint.module_name.split('.')[0]
+                    for entrypoint in pkg_resources.iter_entry_points('buildstream.plugins')}
+        for package in packages:
+            __import__(package).testutils.register_sources()
 
-    if session.config.getvalue('external_plugins'):
-        import bst_plugins_experimental
-        bst_plugins_experimental.testutils.register_sources()
+    sourcetests_collection_hook(session)
 
 
 #################################################
diff --git a/tox.ini b/tox.ini
index ce36da90e..2acf118b6 100644
--- a/tox.ini
+++ b/tox.ini
@@ -15,30 +15,26 @@ isolated_build = true
 usedevelop =
     # This is required by Cython in order to get coverage for cython files.
     py{35,36,37}-!nocover: True
-    plugins: True
 
 commands =
+    py{35,36,37}-plugins: python3 {toxinidir}/requirements/parse-external.py
+    py{35,36,37}-plugins-master: pip3 install -r {toxinidir}/requirements/external-master.txt
+    py{35,36,37}-plugins-fixed: pip3 install -r {toxinidir}/requirements/external-fixed.txt
+    py{35,36,37}-plugins: pytest --basetemp {envtmpdir} --external-plugins {posargs}
+
     # Running with coverage reporting enabled
-    py{35,36,37}-!nocover: pytest --basetemp {envtmpdir} --cov=buildstream --cov-config .coveragerc {posargs}
-    py{35,36,37}-!nocover: mkdir -p .coverage-reports
-    py{35,36,37}-!nocover: mv {envtmpdir}/.coverage {toxinidir}/.coverage-reports/.coverage.{env:COVERAGE_PREFIX:}{envname}
+    py{35,36,37}-!nocover-!plugins: pytest --basetemp {envtmpdir} --cov=buildstream --cov-config .coveragerc {posargs}
+    py{35,36,37}-!nocover-!plugins: mkdir -p .coverage-reports
+    py{35,36,37}-!nocover-!plugins: mv {envtmpdir}/.coverage {toxinidir}/.coverage-reports/.coverage.{env:COVERAGE_PREFIX:}{envname}
     # Running with coverage reporting disabled
     py{35,36,37}-nocover: pytest --basetemp {envtmpdir} {posargs}
 
-    plugins: pytest --basetemp {envtmpdir} --external-plugins {posargs}
-
    
 deps =
     py{35,36,37}: -rrequirements/requirements.txt
     py{35,36,37}: -rrequirements/dev-requirements.txt
     py{35,36,37}: -rrequirements/plugin-requirements.txt
     
-    plugins: -rrequirements/requirements.txt
-    plugins: -rrequirements/dev-requirements.txt
-    plugins: -rrequirements/plugin-requirements.txt
-    plugins: git+https://gitlab.com/buildstream/bst-plugins-experimental@coldtom/setup-py-improvements
-
-
     # Only require coverage and pytest-cov when using it
     py{35,36,37}-!nocover: -rrequirements/cov-requirements.txt
 passenv =
@@ -73,10 +69,6 @@ whitelist_externals =
     py{35,36,37}:
         mv
         mkdir
-    plugins:
-        rm
-        mv
-        git
 #
 # Coverage reporting
 #