From 79edb1ba6d587fcbc81a6d65cba37758bc48d4ad Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 27 Mar 2014 17:55:38 +0000 Subject: Add max-jobs concept to WEBAPP --- lorrycontroller/__init__.py | 1 + lorrycontroller/givemejob.py | 9 ++++++- lorrycontroller/maxjobs.py | 55 ++++++++++++++++++++++++++++++++++++++ lorrycontroller/statedb.py | 2 ++ lorrycontroller/status.py | 7 +++++ templates/status.tpl | 1 + yarns.webapp/040-running-jobs.yarn | 47 ++++++++++++++++++++++++++++++++ 7 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 lorrycontroller/maxjobs.py diff --git a/lorrycontroller/__init__.py b/lorrycontroller/__init__.py index 99e0d29..214024b 100644 --- a/lorrycontroller/__init__.py +++ b/lorrycontroller/__init__.py @@ -35,6 +35,7 @@ from showjob import ShowJob from removejob import RemoveJob from lstroves import LsTroves, ForceLsTrove from pretendtime import PretendTime +from maxjobs import GetMaxJobs, SetMaxJobs from static import StaticFile diff --git a/lorrycontroller/givemejob.py b/lorrycontroller/givemejob.py index 0510a26..e40de5f 100644 --- a/lorrycontroller/givemejob.py +++ b/lorrycontroller/givemejob.py @@ -41,7 +41,7 @@ class GiveMeJob(lorrycontroller.LorryControllerRoute): logging.info('%s %s called', self.http_method, self.path) readdb = self.open_statedb() - if readdb.get_running_queue(): + if readdb.get_running_queue() and not self.max_jobs_reached(readdb): statedb = self.open_statedb() with statedb: lorry_infos = statedb.get_all_lorries_info() @@ -63,6 +63,13 @@ class GiveMeJob(lorrycontroller.LorryControllerRoute): logging.info('No job to give MINION') return { 'job_id': None } + def max_jobs_reached(self, statedb): + max_jobs = statedb.get_max_jobs() + if max_jobs is None: + return False + running_jobs = statedb.get_running_jobs() + return len(running_jobs) >= max_jobs + def ready_to_run(self, lorry_info, now): due = lorry_info['last_run'] + lorry_info['interval'] return (lorry_info['running_job'] is None and due <= now) diff --git a/lorrycontroller/maxjobs.py b/lorrycontroller/maxjobs.py new file mode 100644 index 0000000..ce594c2 --- /dev/null +++ b/lorrycontroller/maxjobs.py @@ -0,0 +1,55 @@ +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import logging +import os +import time + +import bottle + +import lorrycontroller + + +class GetMaxJobs(lorrycontroller.LorryControllerRoute): + + http_method = 'GET' + path = '/1.0/get-max-jobs' + + def run(self, **kwargs): + logging.info('%s %s called', self.http_method, self.path) + + statedb = self.open_statedb() + return { + 'max_jobs': statedb.get_max_jobs(), + } + + +class SetMaxJobs(lorrycontroller.LorryControllerRoute): + + http_method = 'POST' + path = '/1.0/set-max-jobs' + + def run(self, **kwargs): + logging.info('%s %s called', self.http_method, self.path) + + statedb = self.open_statedb() + max_jobs = bottle.request.forms.max_jobs + + with statedb: + statedb.set_max_jobs(max_jobs) + return { + 'max_jobs': statedb.get_max_jobs(), + } diff --git a/lorrycontroller/statedb.py b/lorrycontroller/statedb.py index 7353b0f..db91782 100644 --- a/lorrycontroller/statedb.py +++ b/lorrycontroller/statedb.py @@ -532,7 +532,9 @@ class StateDB(object): c.execute('SELECT max_jobs FROM max_jobs') row = c.fetchone() if row: + logging.info('returning max_jobs as %r', row[0]) return row[0] + logging.info('returning max_jobs as None') return None def set_max_jobs(self, max_jobs): diff --git a/lorrycontroller/status.py b/lorrycontroller/status.py index b8ce1c4..bd32e6b 100644 --- a/lorrycontroller/status.py +++ b/lorrycontroller/status.py @@ -43,6 +43,7 @@ class StatusRenderer(object): 'run_queue': self.get_run_queue(statedb), 'troves': self.get_troves(statedb), 'warning_msg': '', + 'max_jobs': self.get_max_jobs(statedb), } status.update(self.get_free_disk_space(work_directory)) return status @@ -124,6 +125,12 @@ class StatusRenderer(object): troves.append(trove_info) return troves + def get_max_jobs(self, statedb): + max_jobs = statedb.get_max_jobs() + if max_jobs is None: + return 'unlimited' + return max_jobs + class Status(lorrycontroller.LorryControllerRoute): diff --git a/templates/status.tpl b/templates/status.tpl index 3d4e78b..e283a76 100644 --- a/templates/status.tpl +++ b/templates/status.tpl @@ -34,6 +34,7 @@

+

Maximum number of jobs: {{max_jobs}}.

Free disk space: {{disk_free_gib}} GiB.

diff --git a/yarns.webapp/040-running-jobs.yarn b/yarns.webapp/040-running-jobs.yarn index c6b8a72..e67cd4b 100644 --- a/yarns.webapp/040-running-jobs.yarn +++ b/yarns.webapp/040-running-jobs.yarn @@ -69,6 +69,53 @@ Cleanup. FINALLY WEBAPP terminates +Limit number of jobs running at the same time +--------------------------------------------- + +WEBAPP can be told to limit the number of jobs running at the same +time. + +Set things up. Note that we have two local Lorry files, so that we +could, in principle, run two jobs at the same time. + + SCENARIO limit concurrent jobs + GIVEN a new git repository in CONFGIT + AND an empty lorry-controller.conf in CONFGIT + AND lorry-controller.conf in CONFGIT adds lorries *.lorry using prefix upstream + AND Lorry file CONFGIT/foo.lorry with {"foo":{"type":"git","url":"git://foo"}} + AND Lorry file CONFGIT/bar.lorry with {"bar":{"type":"git","url":"git://bar"}} + AND WEBAPP uses CONFGIT as its configuration directory + AND a running WEBAPP + WHEN admin makes request POST /1.0/read-configuration with dummy=value + +Check the current set of the `max_jobs` setting. + + WHEN admin makes request GET /1.0/get-max-jobs + THEN response has max_jobs set to null + +Set the limit to 1. + + WHEN admin makes request POST /1.0/set-max-jobs with max_jobs=1 + THEN response has max_jobs set to 1 + WHEN admin makes request GET /1.0/get-max-jobs + THEN response has max_jobs set to 1 + +Get a job. This should succeed. + + WHEN MINION makes request POST /1.0/give-me-job with host=testhost&pid=1 + THEN response has job_id set to 1 + +Get a second job. This should not succeed. + + WHEN MINION makes request POST /1.0/give-me-job with host=testhost&pid=2 + THEN response has job_id set to null + +Finish the first job. Then get a new job. This should succeed. + + WHEN MINION makes request POST /1.0/job-update with job_id=1&exit=0 + AND MINION makes request POST /1.0/give-me-job with host=testhost&pid=2 + THEN response has job_id set to 2 + Stop job in the middle ---------------------- -- cgit v1.2.1