Merge branch 'jennis/tasks' into 'master'

Improve long-running task reporting

See merge request BuildStream/buildstream!1573

4 files changed, 30 insertions, 10 deletions

diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py
index 89458d40c..b2dc7c1c3 100644
--- a/src/buildstream/_loader/loader.py
+++ b/src/buildstream/_loader/loader.py
@@ -62,6 +62,7 @@ class Loader():
         # Public members
         #
         self.project = project   # The associated Project
+        self.loaded = None       # The number of loaded Elements
 
         #
         # Private members
@@ -146,6 +147,11 @@ class Loader():
 
         self._clean_caches()
 
+        # Cache how many Elements have just been loaded
+        if task:
+            # Workaround for task potentially being None (because no State object)
+            self.loaded = task.current_progress
+
         return ret
 
     # get_state_for_child_job_pickling(self)
diff --git a/src/buildstream/_messenger.py b/src/buildstream/_messenger.py
index be5f12c0a..f5f570319 100644
--- a/src/buildstream/_messenger.py
+++ b/src/buildstream/_messenger.py
@@ -30,6 +30,13 @@ from ._message import Message, MessageType
 _RENDER_INTERVAL = datetime.timedelta(seconds=1)
 
 
+# Time in seconds for which we decide that we want to display subtask information
+_DISPLAY_LIMIT = datetime.timedelta(seconds=3)
+# If we're in the test suite, we need to ensure that we don't set a limit
+if "BST_TEST_SUITE" in os.environ:
+    _DISPLAY_LIMIT = datetime.timedelta(seconds=0)
+
+
 # TimeData class to contain times in an object that can be passed around
 # and updated from different places
 class _TimeData():
@@ -224,13 +231,13 @@ class Messenger():
                     self._next_render = None
 
             elapsed = datetime.datetime.now() - timedata.start_time
-            if task.current_progress is not None:
+            detail = None
+
+            if task.current_progress is not None and elapsed > _DISPLAY_LIMIT:
                 if task.maximum_progress is not None:
                     detail = "{} of {} subtasks processed".format(task.current_progress, task.maximum_progress)
                 else:
                     detail = "{} subtasks processed".format(task.current_progress)
-            else:
-                detail = None
             message = Message(MessageType.SUCCESS, activity_name, elapsed=elapsed, detail=detail,
                               element_name=element_name)
             self.message(message)
diff --git a/src/buildstream/_pipeline.py b/src/buildstream/_pipeline.py
index 4b0c6ad94..23fc52f63 100644
--- a/src/buildstream/_pipeline.py
+++ b/src/buildstream/_pipeline.py
@@ -138,7 +138,11 @@ class Pipeline():
     #    targets (list of Element): The list of toplevel element targets
     #
     def resolve_elements(self, targets):
-        with self._context.messenger.timed_activity("Resolving cached state", silent_nested=True):
+        with self._context.messenger.simple_task("Resolving cached state", silent_nested=True) as task:
+            # We need to go through the project to access the loader
+            if task:
+                task.set_maximum_progress(self._project.loader.loaded)
+
             # XXX: Now that Element._update_state() can trigger recursive update_state calls
             # it is possible that we could get a RecursionError. However, this is unlikely
             # to happen, even for large projects (tested with the Debian stack). Although,
@@ -154,6 +158,9 @@ class Pipeline():
                 # dependencies.
                 element._update_ready_for_runtime_and_cached()
 
+                if task:
+                    task.add_current_progress()
+
     # check_remotes()
     #
     # Check if the target artifact is cached in any of the available remotes
@@ -162,10 +169,14 @@ class Pipeline():
     #    targets (list [Element]): The list of element targets
     #
     def check_remotes(self, targets):
-        with self._context.messenger.timed_activity("Querying remotes for cached status", silent_nested=True):
+        with self._context.messenger.simple_task("Querying remotes for cached status", silent_nested=True) as task:
+            task.set_maximum_progress(len(targets))
+
             for element in targets:
                 element._cached_remotely()
 
+                task.add_current_progress()
+
     # dependencies()
     #
     # Generator function to iterate over elements and optionally
diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py
index 00beebfab..9cff40868 100644
--- a/src/buildstream/_project.py
+++ b/src/buildstream/_project.py
@@ -434,13 +434,9 @@ class Project():
         with self._context.messenger.simple_task("Loading elements", silent_nested=True) as task:
             meta_elements = self.loader.load(targets, rewritable=rewritable, ticker=None, task=task)
 
-            # workaround for task potentially being None (because no State object)
-            if task:
-                total_elements = task.current_progress
-
         with self._context.messenger.simple_task("Resolving elements") as task:
             if task:
-                task.set_maximum_progress(total_elements)
+                task.set_maximum_progress(self.loader.loaded)
             elements = [
                 Element._new_from_meta(meta, task)
                 for meta in meta_elements