Add Progress reporting for element loading

This is very primitive and does not have a very good way to find the
total number of elements while loading.

4 files changed, 83 insertions, 3 deletions

diff --git a/src/buildstream/_context.py b/src/buildstream/_context.py
index 151ea636a..f01df7a4a 100644
--- a/src/buildstream/_context.py
+++ b/src/buildstream/_context.py
@@ -34,6 +34,7 @@ from ._profile import Topics, PROFILER
 from ._artifactcache import ArtifactCache
 from ._sourcecache import SourceCache
 from ._cas import CASCache, CASQuota, CASCacheUsage
+from ._progress import Progress
 from ._workspaces import Workspaces, WorkspaceProjectCache
 from .plugin import Plugin
 from .sandbox import SandboxRemote
@@ -554,6 +555,18 @@ class Context():
             self._pop_message_depth()
             self.message(message)
 
+    @contextmanager
+    def progress_activity(self, activity_name, *, total=None, unique_id=None,
+                          detail=None, silent_nested=False):
+        with self.timed_activity(activity_name, unique_id=unique_id,
+                                 detail=detail, silent_nested=silent_nested):
+            progress = Progress(self, activity_name, total=total, unique_id=unique_id)
+            yield progress
+
+    def report_progress(self, message_text, unique_id=None):
+        message = Message(unique_id, MessageType.INFO, message_text)
+        self.message(message)
+
     # recorded_messages()
     #
     # Records all messages in a log file while the context manager
diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py
index 5bd048707..d68c4b3ea 100644
--- a/src/buildstream/_loader/loader.py
+++ b/src/buildstream/_loader/loader.py
@@ -74,6 +74,10 @@ class Loader():
         self._elements = {}       # Dict of elements
         self._loaders = {}        # Dict of junction loaders
 
+        # Progress object stored in the loader during element loading because
+        # passing it through function args would be a huge mess
+        self._progress = None
+
         self._includes = Includes(self, copy_tree=True)
 
     # load():
@@ -86,11 +90,14 @@ class Loader():
     #    ticker (callable): An optional function for tracking load progress
     #    targets (list of str): Target, element-path relative bst filenames in the project
     #    fetch_subprojects (bool): Whether to fetch subprojects while loading
+    #    progress (Progress): The object to report progress to
     #
     # Raises: LoadError
     #
     # Returns: The toplevel LoadElement
-    def load(self, targets, rewritable=False, ticker=None, fetch_subprojects=False):
+    def load(self, targets, progress, rewritable=False, ticker=None, fetch_subprojects=False):
+
+        self._progress = progress
 
         for filename in targets:
             if os.path.isabs(filename):
@@ -143,6 +150,7 @@ class Loader():
             ret.append(loader._collect_element(element))
 
         self._clean_caches()
+        self._progress = None
 
         return ret
 
@@ -405,6 +413,9 @@ class Loader():
         if meta_element:
             return meta_element
 
+        if self._progress:
+            self._progress.add_total(1)
+
         node = element.node
         elt_provenance = _yaml.node_get_provenance(node)
         meta_sources = []
@@ -450,6 +461,9 @@ class Loader():
             if dep.dep_type != 'build':
                 meta_element.dependencies.append(meta_dep)
 
+        if self._progress:
+            self._progress.add_progress(1)
+
         return meta_element
 
     # _get_loader():
diff --git a/src/buildstream/_progress.py b/src/buildstream/_progress.py
new file mode 100644
index 000000000..2ad9e6a9d
--- /dev/null
+++ b/src/buildstream/_progress.py
@@ -0,0 +1,53 @@
+#  Copyright (C) 2019 Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Jonathan Maw <jonathan.maw@codethink.co.uk>
+
+import time
+
+class Progress():
+    def __init__(self, context, activity_name, *, total=None, unique_id=None):
+        self._context = context
+        self._activity_name = activity_name
+        self._last_reported = time.monotonic()
+        self._interval = 3.0 # seconds
+        self._count = 0
+        self._total = total
+        self._unique_id = unique_id
+
+    def add_total(self, count):
+        if self._total is None:
+            self._total = count
+        else:
+            self._total += count
+        self._check_report_progress()
+
+    def add_progress(self, count):
+        self._count += count
+        self._check_report_progress()
+
+    def _check_report_progress(self):
+        # It would be more efficient to have a time-based poll rather than
+        # a regular check whether enough time has passed
+        now = time.monotonic()
+        if now >= self._last_reported + self._interval:
+            self._last_reported = now
+            message_text = self._activity_name + ": " + str(self._count)
+            if self._total is not None:
+                message_text += "/" + str(self._total)
+            self._context.report_progress(message_text, self._unique_id)
+
+        
diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py
index c40321c66..3d5714f86 100644
--- a/src/buildstream/_project.py
+++ b/src/buildstream/_project.py
@@ -449,8 +449,8 @@ class Project():
     #
     def load_elements(self, targets, *,
                       rewritable=False, fetch_subprojects=False):
-        with self._context.timed_activity("Loading elements", silent_nested=True):
-            meta_elements = self.loader.load(targets, rewritable=rewritable,
+        with self._context.progress_activity("Loading elements", silent_nested=True) as progress:
+            meta_elements = self.loader.load(targets, progress, rewritable=rewritable,
                                              ticker=None,
                                              fetch_subprojects=fetch_subprojects)