Add requirements files for install, test and plugin dependencies

Add `.in` and `.txt` requirements files for BuildStream's pure python
dependencies. For each pair, the `.in` file is supposed to capture the
loose version requirements, and the corresponding `.txt` file is
supposed to have frozen requirements. We have 3 such sets:

* `requirements`: BuildStream's runtime dependencies
* `dev-requirements`: Dependencies for running tests
* `plugin-requirements`: Dependencies for core plugins

Note that the frozen requirements files will only be used for testing
purposes, and `setup.py` will continue to read loose requirements.

This commit comes with a Makefile to update the requirements files as
well.

---

!1027 is related MR for the master branch.

9 files changed, 87 insertions, 12 deletions

diff --git a/MANIFEST.in b/MANIFEST.in
index f9e506027..7d234d1d4 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -23,4 +23,12 @@ recursive-include tests *.expected
 recursive-include buildstream/_protos *.proto
 
 # Requirements files
-include dev-requirements.txt
+include requirements/requirements.in
+include requirements/requirements.txt
+include requirements/dev-requirements.in
+include requirements/dev-requirements.txt
+include requirements/plugin-requirements.in
+include requirements/plugin-requirements.txt
+
+# Versioneer
+include versioneer.py
diff --git a/Makefile b/Makefile
new file mode 100644
index 000000000..c49a2e2d0
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,21 @@
+# Makefile for updating BuildStream's requirements files.
+#
+
+REQUIREMENTS_IN := $(wildcard *.in)
+REQUIREMENTS_TXT := $(REQUIREMENTS_IN:.in=.txt)
+PYTHON := python3
+VENV := $(PYTHON) -m venv
+
+VENV_PIP = $(VENVDIR)/bin/pip
+
+
+.PHONY: all
+
+all: $(REQUIREMENTS_TXT)
+
+%.txt: %.in
+	$(eval VENVDIR := $(shell mktemp -d $(CURDIR)/.bst-venv.XXXXXX))
+	$(VENV) $(VENVDIR)
+	$(VENV_PIP) install -r $^
+	$(VENV_PIP) freeze -r $^ > $@
+	rm -rf $(VENVDIR)
diff --git a/dev-requirements.txt b/requirements/dev-requirements.in
index c88b4c723..c88b4c723 100644
--- a/dev-requirements.txt
+++ b/requirements/dev-requirements.in
diff --git a/requirements/dev-requirements.txt b/requirements/dev-requirements.txt
new file mode 100644
index 000000000..6d0886d99
--- /dev/null
+++ b/requirements/dev-requirements.txt
@@ -0,0 +1,27 @@
+coverage==4.4
+pep8==1.7.1
+pylint==2.1.1
+pytest==4.1.1
+pytest-cov==2.6.1
+pytest-datafiles==2.0
+pytest-env==0.6.2
+pytest-pep8==1.0.6
+pytest-pylint==0.14.0
+pytest-xdist==1.26.0
+pytest-timeout==1.3.3
+## The following requirements were added by pip freeze:
+apipkg==1.5
+astroid==2.1.0
+atomicwrites==1.2.1
+attrs==18.2.0
+execnet==1.5.0
+isort==4.3.4
+lazy-object-proxy==1.3.1
+mccabe==0.6.1
+more-itertools==5.0.0
+pluggy==0.8.1
+py==1.7.0
+pytest-cache==1.0
+pytest-forked==1.0.1
+six==1.12.0
+wrapt==1.11.0
diff --git a/requirements/plugin-requirements.in b/requirements/plugin-requirements.in
new file mode 100644
index 000000000..e3527285d
--- /dev/null
+++ b/requirements/plugin-requirements.in
@@ -0,0 +1,2 @@
+arpy
+PyGObject
diff --git a/requirements/plugin-requirements.txt b/requirements/plugin-requirements.txt
new file mode 100644
index 000000000..6841612bd
--- /dev/null
+++ b/requirements/plugin-requirements.txt
@@ -0,0 +1,4 @@
+arpy==1.1.1
+PyGObject==3.30.4
+## The following requirements were added by pip freeze:
+pycairo==1.18.0
diff --git a/requirements/requirements.in b/requirements/requirements.in
new file mode 100644
index 000000000..1743e1c3f
--- /dev/null
+++ b/requirements/requirements.in
@@ -0,0 +1,8 @@
+Click
+grpcio >= 1.10
+jinja2 >= 2.10
+pluginbase
+protobuf >= 3.5
+psutil
+ruamel.yaml < 0.15.52
+setuptools
diff --git a/requirements/requirements.txt b/requirements/requirements.txt
new file mode 100644
index 000000000..96645cb4b
--- /dev/null
+++ b/requirements/requirements.txt
@@ -0,0 +1,11 @@
+Click==7.0
+grpcio==1.18.0
+pluginbase==0.7
+protobuf==3.6.1
+psutil==5.4.8
+ruamel.yaml==0.15.51
+setuptools==40.6.2
+## The following requirements were added by pip freeze:
+Jinja2==2.10
+MarkupSafe==1.1.0
+six==1.12.0
diff --git a/setup.py b/setup.py
index a0686ebea..407602115 100755
--- a/setup.py
+++ b/setup.py
@@ -228,9 +228,12 @@ def get_cmdclass():
 #####################################################
 #               Gather requirements                 #
 #####################################################
-with open('dev-requirements.txt') as dev_reqs:
+with open('requirements/dev-requirements.in') as dev_reqs:
     dev_requires = dev_reqs.read().splitlines()
 
+with open('requirements/requirements.in') as install_reqs:
+    install_requires = install_reqs.read().splitlines()
+
 #####################################################
 #     Prepare package description from README       #
 #####################################################
@@ -291,16 +294,7 @@ setup(name='BuildStream',
               os.path.join('buildstream', 'data', 'bst')
           ])
       ],
-      install_requires=[
-          'setuptools',
-          'psutil',
-          'ruamel.yaml < 0.15.52',
-          'pluginbase',
-          'Click',
-          'jinja2 >= 2.10',
-          'protobuf >= 3.5',
-          'grpcio >= 1.10',
-      ],
+      install_requires=install_requires,
       entry_points=bst_install_entry_points,
       setup_requires=['pytest-runner'],
       tests_require=dev_requires,