diff options
authorSam Thursfield <>2014-10-02 11:46:11 +0100
committerSam Thursfield <>2014-10-13 16:49:05 +0100
commite359b0ee79cb4edf6c59af0b4c411ec2d5f808c5 (patch)
parent109c02002baf3c2d8eac57027f29f6c74e5233b5 (diff)
Add automatic garbage collection service to Trovebaserock/sam/artifact-cache-garbage-collect-v3
We are now using Trove as an artifact server for the Mason continuous delivery system, which can fill up the disk pretty quickly. This adds a rudimetary system for cleaning up old artifacts from an artifact cache server. Note that it does not have any smarts about what to remove other than checking the 'mtime', If you have important release artifacts in the artifact cache, don't run `trove-gc` or enable the trove-garbage-collect service. It wraps the `morph gc` command, currently, so Morph is required on the system. Morph is included in Trove systems via the 'tools' stratum right now. It is hardcoded to remove artifacts until there is at least 15GB of free disk space, and to run once an hour. No cleanup of Git or Lorry data is done.
4 files changed, 70 insertions, 0 deletions
diff --git a/Makefile b/Makefile
index 134436b..8943895 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,6 @@
+ mkdir -p "${DESTDIR}/usr/bin"
+ install -m 755 bin/trove-gc "${DESTDIR}/usr/bin"
mkdir -p "${DESTDIR}/usr/lib/trove-setup/ansible"
cp -r ansible/* "${DESTDIR}/usr/lib/trove-setup/ansible"
mkdir -p "${DESTDIR}/usr/lib/systemd/system/"
diff --git a/bin/trove-gc b/bin/trove-gc
new file mode 100755
index 0000000..82545f5
--- /dev/null
+++ b/bin/trove-gc
@@ -0,0 +1,50 @@
+# Trove garbage collection script
+# Copyright (C) 2014 Codethink Limited
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# GNU General Public License for more details.
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+# This is a rudimentary garbage collector for Trove instances. It wraps the
+# `morph gc` command.
+# Artifacts older than 1 week will be deleted, and artifacts less than 1 day
+# old will not be. Age is calculated from mtime, which is set at the time the
+# artifact is uploaded to the cache and after that is not updated by any Trove
+# components.
+# Morph's default `cachedir-min-space` value is taken as the amount of space
+# to free. Pass --cachedir-min-space to this command to specify your own value.
+# `morph gc` is designed for running on build machines so it expects there to
+# be a tempdir. A harmless side-effect of this script is that you'll get some
+# pointless directories created in /tmp/trove-gc-fake-tempdir.
+# The 'remove all artifacts older than xx' rule is effectively disabled, to
+# prevent artifacts that are still needed but do not change often (such as
+# stage1 and stage2 of the bootstrap) from being automatically deleted. Those
+# artifacts will still be first to be removed in a low disk space condition,
+# which is an unfortunate consequence of the simple heuristic we are using
+# right now.
+set -eu
+/usr/bin/morph gc --no-default-config \
+ --cachedir=/home/cache \
+ --cachedir-artifact-delete-older-than=31557600000 \
+ --cachedir-artifact-keep-younger-than=86400 \
+ --tempdir=/tmp/trove-gc-fake-tempdir \
+ --tempdir-min-space=0M \
+ "$@"
diff --git a/units/trove-garbage-collect.service b/units/trove-garbage-collect.service
new file mode 100644
index 0000000..9f58873
--- /dev/null
+++ b/units/trove-garbage-collect.service
@@ -0,0 +1,9 @@
+Description=Remove old artifacts from the artifact cache until 15GB of space is free.
+ExecStart=/usr/bin/trove-gc --cachedir-min-space=15G --log=syslog --verbose
diff --git a/units/trove-garbage-collect.timer b/units/trove-garbage-collect.timer
new file mode 100644
index 0000000..77b0dfe
--- /dev/null
+++ b/units/trove-garbage-collect.timer
@@ -0,0 +1,9 @@
+Description=Remove old artifacts once every hour