Merge branch 'chandan/update-requirements-one-liner' into 'master'

Add Makefile to update requirements files

See merge request BuildStream/buildstream!1035

12 files changed, 60 insertions, 21 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 67248bc26..b6be45a88 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -276,7 +276,7 @@ coverage:
   coverage: '/TOTAL +\d+ +\d+ +(\d+\.\d+)%/'
   script:
     - cd dist && ./unpack.sh && cd buildstream
-    - pip3 install -r tools/requirements.txt -r tools/dev-requirements.txt
+    - pip3 install -r requirements/requirements.txt -r requirements/dev-requirements.txt
     - pip3 install --no-index .
     - mkdir report
     - cd report
diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
index ddcc4130d..1e672adf1 100644
--- a/CONTRIBUTING.rst
+++ b/CONTRIBUTING.rst
@@ -1736,10 +1736,8 @@ obtain profiles::
       ForceCommand BST_PROFILE=artifact-receive cd /tmp && bst-artifact-receive --pull-url https://example.com/ /home/artifacts/artifacts
 
 
-The MANIFEST.in and setup.py
-----------------------------
-When adding a dependency to BuildStream, it's important to update the setup.py accordingly.
-
+Managing data files
+-------------------
 When adding data files which need to be discovered at runtime by BuildStream, update setup.py accordingly.
 
 When adding data files for the purpose of docs or tests, or anything that is not covered by
@@ -1749,3 +1747,23 @@ At any time, running the following command to create a source distribution shoul
 creating a tarball which contains everything we want it to include::
 
   ./setup.py sdist
+
+
+Updating BuildStream's Python dependencies
+------------------------------------------
+BuildStream's Python dependencies are listed in multiple
+`requirements files <https://pip.readthedocs.io/en/latest/reference/pip_install/#requirements-file-format>`
+present in the ``requirements`` directory.
+
+All ``.txt`` files in this directory are generated from the corresponding
+``.in`` file, and each ``.in`` file represents a set of dependencies. For
+example, ``requirements.in`` contains all runtime dependencies of BuildStream.
+``requirements.txt`` is generated from it, and contains pinned versions of all
+runtime dependencies (including transitive dependencies) of BuildStream.
+
+When adding a new dependency to BuildStream, or updating existing dependencies,
+it is important to update the appropriate requirements file accordingly. After
+changing the ``.in`` file, run the following to update the matching ``.txt``
+file::
+
+   make -C requirements
diff --git a/MANIFEST.in b/MANIFEST.in
index 7b86e7cb1..d23bb79cc 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -32,12 +32,12 @@ include .pylintrc
 recursive-include buildstream/_protos *.proto
 
 # Requirements files
-include tools/requirements.in
-include tools/requirements.txt
-include tools/dev-requirements.in
-include tools/dev-requirements.txt
-include tools/plugin-requirements.in
-include tools/plugin-requirements.txt
+include requirements/requirements.in
+include requirements/requirements.txt
+include requirements/dev-requirements.in
+include requirements/dev-requirements.txt
+include requirements/plugin-requirements.in
+include requirements/plugin-requirements.txt
 
 # Versioneer
 include versioneer.py
diff --git a/requirements/Makefile b/requirements/Makefile
new file mode 100644
index 000000000..c49a2e2d0
--- /dev/null
+++ b/requirements/Makefile
@@ -0,0 +1,21 @@
+# Makefile for updating BuildStream's requirements files.
+#
+
+REQUIREMENTS_IN := $(wildcard *.in)
+REQUIREMENTS_TXT := $(REQUIREMENTS_IN:.in=.txt)
+PYTHON := python3
+VENV := $(PYTHON) -m venv
+
+VENV_PIP = $(VENVDIR)/bin/pip
+
+
+.PHONY: all
+
+all: $(REQUIREMENTS_TXT)
+
+%.txt: %.in
+	$(eval VENVDIR := $(shell mktemp -d $(CURDIR)/.bst-venv.XXXXXX))
+	$(VENV) $(VENVDIR)
+	$(VENV_PIP) install -r $^
+	$(VENV_PIP) freeze -r $^ > $@
+	rm -rf $(VENVDIR)
diff --git a/tools/dev-requirements.in b/requirements/dev-requirements.in
index 9edb23423..9edb23423 100644
--- a/tools/dev-requirements.in
+++ b/requirements/dev-requirements.in
diff --git a/tools/dev-requirements.txt b/requirements/dev-requirements.txt
index d706bc9f5..d706bc9f5 100644
--- a/tools/dev-requirements.txt
+++ b/requirements/dev-requirements.txt
diff --git a/tools/plugin-requirements.in b/requirements/plugin-requirements.in
index e3527285d..e3527285d 100644
--- a/tools/plugin-requirements.in
+++ b/requirements/plugin-requirements.in
diff --git a/tools/plugin-requirements.txt b/requirements/plugin-requirements.txt
index 6841612bd..6841612bd 100644
--- a/tools/plugin-requirements.txt
+++ b/requirements/plugin-requirements.txt
diff --git a/tools/requirements.in b/requirements/requirements.in
index a3c62c79b..a3c62c79b 100644
--- a/tools/requirements.in
+++ b/requirements/requirements.in
diff --git a/tools/requirements.txt b/requirements/requirements.txt
index 7bf3205f7..7bf3205f7 100644
--- a/tools/requirements.txt
+++ b/requirements/requirements.txt
diff --git a/setup.py b/setup.py
index 7ac51692e..d2c21cd5a 100755
--- a/setup.py
+++ b/setup.py
@@ -270,10 +270,10 @@ def get_cmdclass():
 #####################################################
 #               Gather requirements                 #
 #####################################################
-with open('tools/dev-requirements.in') as dev_reqs:
+with open('requirements/dev-requirements.in') as dev_reqs:
     dev_requires = dev_reqs.read().splitlines()
 
-with open('tools/requirements.in') as install_reqs:
+with open('requirements/requirements.in') as install_reqs:
     install_requires = install_reqs.read().splitlines()
 
 #####################################################
diff --git a/tox.ini b/tox.ini
index 8122bf67a..3b6496d3f 100644
--- a/tox.ini
+++ b/tox.ini
@@ -5,9 +5,9 @@ skip_missing_interpreters = true
 [testenv]
 commands = pytest {posargs}
 deps =
-    -rtools/requirements.txt
-    -rtools/dev-requirements.txt
-    -rtools/plugin-requirements.txt
+    -rrequirements/requirements.txt
+    -rrequirements/dev-requirements.txt
+    -rrequirements/plugin-requirements.txt
 passenv =
     BST_FORCE_BACKEND
     GI_TYPELIB_PATH
@@ -18,9 +18,9 @@ commands =
     pycodestyle
     pylint buildstream
 deps =
-    -rtools/requirements.txt
-    -rtools/dev-requirements.txt
-    -rtools/plugin-requirements.txt
+    -rrequirements/requirements.txt
+    -rrequirements/dev-requirements.txt
+    -rrequirements/plugin-requirements.txt
 
 [testenv:docs]
 commands =
@@ -30,8 +30,8 @@ deps =
     sphinx==1.7.9
     sphinx-click
     sphinx_rtd_theme
-    -rtools/requirements.txt
-    -rtools/plugin-requirements.txt
+    -rrequirements/requirements.txt
+    -rrequirements/plugin-requirements.txt
 passenv =
     BST_FORCE_SESSION_REBUILD
     BST_SOURCE_CACHE