frontend: Implement interactive 'retry' option

Not useful for builds, but interesting for network related tasks.

3 files changed, 17 insertions, 11 deletions

diff --git a/buildstream/_frontend/main.py b/buildstream/_frontend/main.py
index 02d31ac2f..79adf11d7 100644
--- a/buildstream/_frontend/main.py
+++ b/buildstream/_frontend/main.py
@@ -764,7 +764,7 @@ class App():
         self.status.add_job(element, action_name)
         self.maybe_render_status()
 
-    def job_completed(self, element, action_name, success):
+    def job_completed(self, element, queue, action_name, success):
         self.status.remove_job(element, action_name)
         self.maybe_render_status()
 
@@ -783,9 +783,9 @@ class App():
                            "unable to retrieve failure message for element {}\n\n\n\n\n"
                            .format(element))
             else:
-                self.handle_failure(element, failure)
+                self.handle_failure(element, queue, failure)
 
-    def handle_failure(self, element, failure):
+    def handle_failure(self, element, queue, failure):
 
         # Handle non interactive mode setting of what to do when a job fails.
         if not self.interactive_failures:
@@ -806,21 +806,22 @@ class App():
                        "Choose one of the following options:\n" +
                        "  continue  - Continue queueing jobs as much as possible\n" +
                        "  quit      - Exit after all ongoing jobs complete\n" +
-                       "  terminate - Terminate any ongoing jobs and exit\n")
+                       "  terminate - Terminate any ongoing jobs and exit\n" +
+                       "  retry     - Retry this job\n")
             if failure.logfile:
                 summary += "  log       - View the full log file\n"
             if failure.sandbox:
                 summary += "  shell     - Drop into a shell in the failed build sandbox\n"
             summary += "\nPressing ^C will terminate jobs and exit\n"
 
-            choices = ['continue', 'quit', 'terminate']
+            choices = ['continue', 'quit', 'terminate', 'retry']
             if failure.logfile:
                 choices += ['log']
             if failure.sandbox:
                 choices += ['shell']
 
             choice = ''
-            while choice not in ['continue', 'quit', 'terminate']:
+            while choice not in ['continue', 'quit', 'terminate', 'retry']:
                 click.echo(summary, err=True)
 
                 try:
@@ -850,6 +851,10 @@ class App():
                     self.scheduler.stop_queueing()
                 elif choice == 'continue':
                     click.echo("\nContinuing with other non failing elements\n", err=True)
+                elif choice == 'retry':
+                    click.echo("\nRetrying failed job\n", err=True)
+                    queue.failed_elements.remove(element)
+                    queue.enqueue([element])
 
     def tick(self, elapsed):
         self.maybe_render_status()
diff --git a/buildstream/_scheduler/queue.py b/buildstream/_scheduler/queue.py
index 97fec98d5..4642edaa7 100644
--- a/buildstream/_scheduler/queue.py
+++ b/buildstream/_scheduler/queue.py
@@ -210,10 +210,11 @@ class Queue():
         else:
             self.failed_elements.append(element)
 
-        # Notify frontend
-        self.scheduler.job_completed(job, returncode == 0)
-
         # Give the token for this job back to the scheduler
         # immediately before invoking another round of scheduling
         self.scheduler.put_job_token(self.queue_type)
+
+        # Notify frontend
+        self.scheduler.job_completed(self, job, returncode == 0)
+
         self.scheduler.sched()
diff --git a/buildstream/_scheduler/scheduler.py b/buildstream/_scheduler/scheduler.py
index 24b42c774..7ef1a83d5 100644
--- a/buildstream/_scheduler/scheduler.py
+++ b/buildstream/_scheduler/scheduler.py
@@ -361,6 +361,6 @@ class Scheduler():
             self.job_start_callback(job.element, job.action_name)
 
     # Called by the Queue when a Job completed
-    def job_completed(self, job, success):
+    def job_completed(self, queue, job, success):
         if self.job_complete_callback:
-            self.job_complete_callback(job.element, job.action_name, success)
+            self.job_complete_callback(job.element, queue, job.action_name, success)