Merge branch 'ps-kbas-cull' into 'master'

Add culling functionality to KBAS

Closes #172

See merge request !386

4 files changed, 85 insertions, 12 deletions

diff --git a/README.md b/README.md
index 2e2e46d..30adea8 100644
--- a/README.md
+++ b/README.md
@@ -177,8 +177,13 @@ By default ybd is configured to check for artifacts from a kbas server at
 Config for kbas follows the same approach as ybd, defaulting to config in
 `kbas/config/kbas.conf`.
 
-NOTE: the default password is 'insecure' and the uploading is disabled unless
-you change it.
+NOTES:
+- the default password is 'insecure' and upload functionality is disabled
+unless you change it.
+- for testing kbas you can use cURL to upload files:
+
+    curl  -X POST  -i  -F 'filename=<cachid>' -F 'file=@path/to/file' \
+    -F 'password=<password>' <host>:<port>/upload
 
 ### Git LFS Support
 
diff --git a/kbas/__main__.py b/kbas/__main__.py
index 80de7ef..383a00a 100755
--- a/kbas/__main__.py
+++ b/kbas/__main__.py
@@ -26,6 +26,7 @@ from bottle import Bottle, request, response, template, static_file
 from subprocess import call, check_output
 
 from ybd import app, cache
+from ybd.utils import cull_directory, get_free, sorted_ls
 
 bottle = Bottle()
 
@@ -43,6 +44,7 @@ class KeyedBinaryArtifactServer(object):
         app.config['start-time'] = datetime.now()
         app.config['last-upload'] = datetime.now()
         app.config['downloads'] = 0
+        app.config['tmp'] = app.config.get('tmp', app.config['artifact-dir'])
 
         try:
             import cherrypy
@@ -91,6 +93,7 @@ class KeyedBinaryArtifactServer(object):
     @bottle.get('/get/<cache_id>')
     def get_artifact(cache_id):
         f = os.path.join(cache_id, cache_id)
+        call(['touch', f])
         app.config['downloads'] += 1
         return static_file(f, root=app.config['artifact-dir'], download=True,
                            mimetype='application/x-tar')
@@ -98,9 +101,8 @@ class KeyedBinaryArtifactServer(object):
     @bottle.get('/')
     @bottle.get('/status')
     def status():
-        stat = os.statvfs(app.config['artifact-dir'])
-        free = stat.f_frsize * stat.f_bavail / 1000000000
-        artifacts = len(os.listdir(app.config['artifact-dir']))
+        artifact_dir = app.config['artifact-dir']
+        artifacts = len(os.listdir(artifact_dir))
         started = app.config['start-time'].strftime('%y-%m-%d %H:%M:%S')
         downloads = app.config['downloads']
         last_upload = app.config['last-upload'].strftime('%y-%m-%d %H:%M:%S')
@@ -108,7 +110,8 @@ class KeyedBinaryArtifactServer(object):
         content += [['Last upload:', last_upload, None]]
         if app.config.get('last-reject'):
             content += [['Last reject:', app.config['last-reject'], None]]
-        content += [['Space:', str(free) + 'GB', None]]
+        content += [['Artifact directory:', artifact_dir, None]]
+        content += [['Space:', str(get_free(artifact_dir)) + 'GB', None]]
         content += [['Artifacts:', str(artifacts), None]]
         content += [['Downloads:', downloads, None]]
         return template('kbas',
@@ -116,6 +119,34 @@ class KeyedBinaryArtifactServer(object):
                         content=content,
                         css='static/style.css')
 
+    @bottle.get('/cull/')
+    @bottle.get('/cull/<space>')
+    def cull(space=""):
+        artifact_dir = app.config['artifact-dir']
+        try:
+            target = int(space)
+            if target in range(app.config.get('min-cull', 0),
+                               app.config.get('max-cull', 0) + 1):
+
+                deleted = cull_directory(artifact_dir, target)
+                free = str(get_free(artifact_dir))
+                app.log('CULL', 'Culled %s items, %sGB free space in'
+                        % (deleted, free), artifact_dir)
+
+                content = [['Culled files: ', deleted, None]]
+                return template('kbas',
+                                title='KBAS Cull Status',
+                                content=content,
+                                css='/static/style.css')
+        except ValueError:
+            pass
+
+        content = [['No cull with:', space, None]]
+        return template('kbas',
+                        title='KBAS Cull Status',
+                        content=content,
+                        css='/static/style.css')
+
     @bottle.post('/upload')
     def post_artifact():
         if app.config['password'] is 'insecure' or \
@@ -138,7 +169,8 @@ class KeyedBinaryArtifactServer(object):
             response.status = 405  # not allowed, this artifact exists
             return
 
-        tempfile.tempdir = app.config['artifact-dir']
+        artifact_dir = app.config['artifact-dir']
+        tempfile.tempdir = artifact_dir
         tmpdir = tempfile.mkdtemp()
         try:
             upload = request.files.get('file')
@@ -164,10 +196,18 @@ class KeyedBinaryArtifactServer(object):
             checksum = cache.md5(artifact)
             with open(artifact + '.md5', "a") as f:
                 f.write(checksum)
-            shutil.move(tmpdir, os.path.join(app.config['artifact-dir'],
-                                             cache_id))
+
+            shutil.move(tmpdir, os.path.join(artifact_dir, cache_id))
             response.status = 201  # success!
             app.config['last-upload'] = datetime.now()
+
+            if os.fork() == 0:
+                deleted = cull_directory(artifact_dir, app.config['min-cull'])
+                if deleted > 0:
+                    app.log('CULL', 'Culled %s items, %sGB free space in'
+                            % (deleted, get_free(artifact_dir)), artifact_dir)
+                os._exit(0)
+
             return
         except:
             # something went wrong, clean up
diff --git a/kbas/config/kbas.conf b/kbas/config/kbas.conf
index 2c7c5fd..5a219f2 100644
--- a/kbas/config/kbas.conf
+++ b/kbas/config/kbas.conf
@@ -14,9 +14,9 @@
 #
 # =*= License: GPL-2 =*=
 
-# ybd is designed to be run from the command line and/or as part of an
-# automated pipeline. all configuration is taken from conf files and/or
-# environment variables, in the following order of precedence:
+# kbas is designed to be run from the command line.
+# all configuration is taken from conf files and/or environment variables,
+# in the following order of precedence:
 #
 #    YBD_* environment variables            # if found
 #    ./kbas.conf                            # if found
@@ -45,6 +45,12 @@ artifact-dir: '/src/artifacts'
 # cases... '0.0.0.0 means all IPv4 addresses on the local machine'
 host: 0.0.0.0
 
+# max-cull sets the maximum space the cull algorithm will accept (in GB)
+# min-cull sets the minimum space the cull algorithm will accept (in GB)
+# by default the cull will try to ensure min-cull GB is available
+max-cull: 10
+min-cull: 5
+
 # port to serve on
 port: 8000
 
diff --git a/ybd/utils.py b/ybd/utils.py
index 9ee321e..2f878b3 100644
--- a/ybd/utils.py
+++ b/ybd/utils.py
@@ -435,6 +435,28 @@ def sorted_ls(path):
     return list(sorted(os.listdir(path), key=mtime))
 
 
+def cull_directory(artifact_dir, target_space):
+    tempfile.tempdir = artifact_dir
+    artifacts = sorted_ls(artifact_dir)
+    deleted = 0
+    for artifact in artifacts:
+        if get_free(artifact_dir) < target_space:
+            path = os.path.join(artifact_dir, artifact)
+            if os.path.exists(path):
+                tmpdir = tempfile.mkdtemp()
+                shutil.move(path, os.path.join(tmpdir, 'to-delete'))
+                app.remove_dir(tmpdir)
+                deleted += 1
+    return deleted
+
+
+def get_free(directory):
+    # calculate free space in GB
+    gigabytes = 1073741824
+    stat = os.statvfs(directory)
+    return stat.f_frsize * stat.f_bavail / gigabytes
+
+
 @contextlib.contextmanager
 def monkeypatch(obj, attr, new_value):
     '''Temporarily override the attribute of some object.