From 766e1dd0c8ef498bd830252200be79915a177cb0 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Thu, 2 Oct 2014 11:46:11 +0100 Subject: Add automatic garbage collection service to Trove We are now using Trove as an artifact server for the Mason continuous delivery system, which can fill up the disk pretty quickly. This adds a rudimetary system for cleaning up old artifacts from an artifact cache server. Note that it does not have any smarts about what to remove other than checking the 'mtime', If you have important release artifacts in the artifact cache, don't run `trove-gc` or enable the trove-garbage-collect service. It wraps the `morph gc` command, currently, so Morph is required on the system. Morph is included in Trove systems via the 'tools' stratum right now. It is hardcoded to remove artifacts until there is at least 15GB of free disk space, and to run once an hour. No cleanup of Git or Lorry data is done. --- Makefile | 2 ++ bin/trove-gc | 41 +++++++++++++++++++++++++++++++++++++ units/trove-garbage-collect.service | 6 ++++++ units/trove-garbage-collect.timer | 9 ++++++++ 4 files changed, 58 insertions(+) create mode 100755 bin/trove-gc create mode 100644 units/trove-garbage-collect.service create mode 100644 units/trove-garbage-collect.timer diff --git a/Makefile b/Makefile index 134436b..d633dd3 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,6 @@ install: + mkdir -p "${DESTDIR}/usr/bin" + install -m 755 -d bin/trove-gc "${DESTDIR}/usr/bin" mkdir -p "${DESTDIR}/usr/lib/trove-setup/ansible" cp -r ansible/* "${DESTDIR}/usr/lib/trove-setup/ansible" mkdir -p "${DESTDIR}/usr/lib/systemd/system/multi-user.target.wants" diff --git a/bin/trove-gc b/bin/trove-gc new file mode 100755 index 0000000..058d5fb --- /dev/null +++ b/bin/trove-gc @@ -0,0 +1,41 @@ +#!/bin/sh +# +# Trove garbage collection script +# +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# This is a rudimentary garbage collector for Trove instances. It wraps the +# `morph gc` command. +# +# Artifacts older than 1 week will be deleted, and artifacts less than 1 day +# old will not be. Age is calculated from mtime, which is set at the time the +# artifact is uploaded to the cache and after that is not updated by any Trove +# components. +# +# Morph's default `cachedir-min-space` value is taken as the amount of space +# to free. Pass --cachedir-min-space to this command to specify your own value. +# +# `morph gc` is designed for running on build machines so it expects there to +# be a tempdir. A harmless side-effect of this script is that you'll get some +# pointless directories created in /tmp/trove-gc-fake-tempdir. + +set -eu + +/usr/bin/morph gc --no-default-config \ + --cachedir=/home/cache/artifacts \ + --tempdir=/tmp/trove-gc-fake-tempdir \ + --tempdir-min-space=0M \ + $@ diff --git a/units/trove-garbage-collect.service b/units/trove-garbage-collect.service new file mode 100644 index 0000000..12ac8cb --- /dev/null +++ b/units/trove-garbage-collect.service @@ -0,0 +1,6 @@ +[Unit] +Description=Remove old artifacts from the artifact cache until 15GB of space is free. + +[Service] +Type=idle +ExecStart=/usr/bin/trove-gc --cachedir-min-space=15G --log=syslog --verbose diff --git a/units/trove-garbage-collect.timer b/units/trove-garbage-collect.timer new file mode 100644 index 0000000..636acea --- /dev/null +++ b/units/trove-garbage-collect.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Remove old artifacts once every hour + +[Timer] +OnUnitActiveSec=1h + +[Install] +WantedBy=multi-user.target +Also=trove-garbage-collect.service -- cgit v1.2.1