setup.py: Add grpcio dependency and support for code generation

This allows code generation with ./setup.py build_grpc

7 files changed, 69 insertions, 5 deletions

diff --git a/.coveragerc b/.coveragerc
index 6014b7fd0..d81aec1a2 100644
--- a/.coveragerc
+++ b/.coveragerc
@@ -6,6 +6,8 @@ include =
 omit =
   # Omit profiling helper module
   */buildstream/_profile.py
+  # Omit generated code
+  */buildstream/_protos/*
   */.eggs/*
 
 [report]
diff --git a/.pylintrc b/.pylintrc
index c38309372..93f9eeadf 100644
--- a/.pylintrc
+++ b/.pylintrc
@@ -11,7 +11,7 @@ ignore=CVS,tests,doc
 
 # Add files or directories matching the regex patterns to the blacklist. The
 # regex matches against base names, not paths.
-ignore-patterns=
+ignore-patterns=.*_pb2.py,.*_pb2_grpc.py
 
 # Python code to execute, usually for sys.path manipulation such as
 # pygtk.require().
@@ -190,7 +190,7 @@ ignored-classes=optparse.Values,thread._local,_thread._local,contextlib.closing,
 # (useful for modules/projects where namespaces are manipulated during runtime
 # and thus existing member attributes cannot be deduced by static analysis. It
 # supports qualified module names, as well as Unix pattern matching.
-ignored-modules=pkg_resources,gi.repository
+ignored-modules=pkg_resources,gi.repository,grpc,buildstream._protos.*
 
 # Show a hint with possible names when a member name was not found. The aspect
 # of finding the hint is based on edit distance.
diff --git a/MANIFEST.in b/MANIFEST.in
index 0ef33d078..3c8cc64b4 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -18,3 +18,6 @@ recursive-include tests *.bst
 recursive-include tests *.conf
 recursive-include tests *.sh
 recursive-include tests *.expected
+
+# Protocol Buffers
+recursive-include buildstream/_protos *.proto
diff --git a/buildstream/_protos/__init__.py b/buildstream/_protos/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/buildstream/_protos/__init__.py
diff --git a/doc/Makefile b/doc/Makefile
index eaef15a61..3557ac505 100644
--- a/doc/Makefile
+++ b/doc/Makefile
@@ -76,7 +76,7 @@ clean: templates-clean sessions-clean
 templates:
 	mkdir -p source/elements
 	mkdir -p source/sources
-	$(SPHINXAPIDOC) --force --separate --module-first --no-headings --no-toc -o source $(CURDIR)/../buildstream
+	$(SPHINXAPIDOC) --force --separate --module-first --no-headings --no-toc -o source $(CURDIR)/../buildstream *_pb2*.py
 	$(call plugin-doc-skeleton,$(CURDIR)/../buildstream/plugins/elements,elements)
 	$(call plugin-doc-skeleton,$(CURDIR)/../buildstream/plugins/sources,sources)
 
diff --git a/setup.cfg b/setup.cfg
index e0b3c9970..d37db7839 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -23,5 +23,7 @@ pep8ignore =
     */bin/* ALL
     buildstream/_fuse/fuse.py ALL
     .eggs/* ALL
+    *_pb2.py ALL
+    *_pb2_grpc.py ALL
 env =
     D:BST_TEST_SUITE=True
diff --git a/setup.py b/setup.py
index 1f9ff0081..983a0ac18 100755
--- a/setup.py
+++ b/setup.py
@@ -19,6 +19,7 @@
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 
 import os
+import re
 import shutil
 import subprocess
 import sys
@@ -29,7 +30,7 @@ if sys.version_info[0] != 3 or sys.version_info[1] < 4:
     sys.exit(1)
 
 try:
-    from setuptools import setup, find_packages
+    from setuptools import setup, find_packages, Command
     from setuptools.command.easy_install import ScriptWriter
 except ImportError:
     print("BuildStream requires setuptools in order to build. Install it using"
@@ -206,12 +207,66 @@ ScriptWriter.get_args = get_args
 
 
 #####################################################
+#         gRPC command for code generation          #
+#####################################################
+class BuildGRPC(Command):
+    """Command to generate project *_pb2.py modules from proto files."""
+
+    description = 'build gRPC protobuf modules'
+    user_options = []
+
+    def initialize_options(self):
+        pass
+
+    def finalize_options(self):
+        pass
+
+    def run(self):
+        try:
+            import grpc_tools.command
+        except ImportError:
+            print("BuildStream requires grpc_tools in order to build gRPC modules.\n"
+                  "Install it via pip (pip3 install grpcio-tools).")
+            exit(1)
+
+        protos_root = 'buildstream/_protos'
+
+        grpc_tools.command.build_package_protos(protos_root)
+
+        # Postprocess imports in generated code
+        for root, _, files in os.walk(protos_root):
+            for filename in files:
+                if filename.endswith('.py'):
+                    path = os.path.join(root, filename)
+                    with open(path, 'r') as f:
+                        code = f.read()
+
+                    # All protos are in buildstream._protos
+                    code = re.sub(r'^from ', r'from buildstream._protos.',
+                                  code, flags=re.MULTILINE)
+                    # Except for the core google.protobuf protos
+                    code = re.sub(r'^from buildstream._protos.google.protobuf', r'from google.protobuf',
+                                  code, flags=re.MULTILINE)
+
+                    with open(path, 'w') as f:
+                        f.write(code)
+
+
+def get_cmdclass():
+    cmdclass = {
+        'build_grpc': BuildGRPC,
+    }
+    cmdclass.update(versioneer.get_cmdclass())
+    return cmdclass
+
+
+#####################################################
 #             Main setup() Invocation               #
 #####################################################
 setup(name='BuildStream',
       # Use versioneer
       version=versioneer.get_version(),
-      cmdclass=versioneer.get_cmdclass(),
+      cmdclass=get_cmdclass(),
 
       description='A framework for modelling build pipelines in YAML',
       license='LGPL',
@@ -243,6 +298,8 @@ setup(name='BuildStream',
           'Click',
           'blessings',
           'jinja2 >= 2.10',
+          'protobuf >= 3.5',
+          'grpcio >= 1.10',
       ],
       entry_points=bst_install_entry_points,
       setup_requires=['pytest-runner'],