e2scrub: add service (cron, systemd) support

Add the ability to run the e2scrub utilities as a periodically scheduled
system service.

Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
Signed-off-by: Theodore Ts'o <tytso@mit.edu>

11 files changed, 330 insertions, 11 deletions

diff --git a/scrub/Makefile.in b/scrub/Makefile.in
index 17449419..f58331de 100644
--- a/scrub/Makefile.in
+++ b/scrub/Makefile.in
@@ -22,7 +22,23 @@ INSTALL_TGT	+= install-udev
 UNINSTALL_TGT	+= uninstall-udev
 endif
 
-all:: $(PROGS) $(MANPAGES) $(CONFFILES) $(UDEV_RULES)
+ifeq ($(HAVE_CROND),yes)
+CRONTABS	= e2scrub_all.cron
+LIBPROGS	+= e2scrub_all_cron
+INSTALLDIRS_TGT	+= installdirs-crond installdirs-libprogs
+INSTALL_TGT	+= install-crond install-libprogs
+UNINSTALL_TGT	+= uninstall-crond uninstall-libprogs
+endif
+
+ifeq ($(HAVE_SYSTEMD),yes)
+SERVICE_FILES	= e2scrub@.service e2scrub_all.service e2scrub_all.timer e2scrub_fail@.service e2scrub_reap.service
+LIBPROGS	+= e2scrub_fail
+INSTALLDIRS_TGT	+= installdirs-systemd installdirs-libprogs
+INSTALL_TGT	+= install-systemd install-libprogs
+UNINSTALL_TGT	+= uninstall-systemd uninstall-libprogs
+endif
+
+all:: $(PROGS) $(MANPAGES) $(CONFFILES) $(UDEV_RULES) $(SERVICE_FILES) $(CRONTABS) $(LIBPROGS)
 
 e2scrub: $(DEP_SUBSTITUTE) e2scrub.in
 	$(E) "	SUBST $@"
@@ -34,6 +50,16 @@ e2scrub_all: e2scrub_all.in
 	$(Q) $(SUBSTITUTE_UPTIME) $(srcdir)/e2scrub_all.in $@
 	$(Q) chmod a+x $@
 
+e2scrub_fail: e2scrub_fail.in
+	$(E) "	SUBST $@"
+	$(Q) $(SUBSTITUTE_UPTIME) $(srcdir)/e2scrub_fail.in $@
+	$(Q) chmod a+x $@
+
+e2scrub_all_cron: e2scrub_all_cron.in
+	$(E) "	SUBST $@"
+	$(Q) $(SUBSTITUTE_UPTIME) $(srcdir)/e2scrub_all_cron.in $@
+	$(Q) chmod a+x $@
+
 %.8: %.8.in $(DEP_SUBSTITUTE)
 	$(E) "	SUBST $@"
 	$(Q) $(SUBSTITUTE_UPTIME) $< $@
@@ -46,10 +72,34 @@ e2scrub_all: e2scrub_all.in
 	$(E) "	SUBST $@"
 	$(Q) $(SUBSTITUTE_UPTIME) $< $@
 
+%.service: %.service.in $(DEP_SUBSTITUTE)
+	$(E) "	SUBST $@"
+	$(Q) $(SUBSTITUTE_UPTIME) $< $@
+
+%.cron: %.cron.in $(DEP_SUBSTITUTE)
+	$(E) "	SUBST $@"
+	$(Q) $(SUBSTITUTE_UPTIME) $< $@
+
+%.timer: %.timer.in $(DEP_SUBSTITUTE)
+	$(E) "	SUBST $@"
+	$(Q) $(SUBSTITUTE_UPTIME) $< $@
+
 installdirs-udev:
 	$(E) "	MKINSTALLDIRS $(UDEV_RULES_DIR)"
 	$(Q) $(MKINSTALLDIRS) $(DESTDIR)$(UDEV_RULES_DIR)
 
+installdirs-crond:
+	$(E) "	MKINSTALLDIRS $(CROND_DIR)"
+	$(Q) $(MKINSTALLDIRS) $(DESTDIR)$(CROND_DIR)
+
+installdirs-libprogs:
+	$(E) "	MKINSTALLDIRS $(pkglibdir)"
+	$(Q) $(MKINSTALLDIRS) $(DESTDIR)$(pkglibdir)
+
+installdirs-systemd:
+	$(E) "	MKINSTALLDIRS $(SYSTEMD_SYSTEM_UNIT_DIR)"
+	$(Q) $(MKINSTALLDIRS) $(DESTDIR)$(SYSTEMD_SYSTEM_UNIT_DIR)
+
 installdirs: $(INSTALLDIRS_TGT)
 	$(E) "	MKINSTALLDIRS $(root_sbindir) $(man8dir) $(root_sysconfdir)"
 	$(Q) $(MKINSTALLDIRS) $(DESTDIR)$(root_sbindir) \
@@ -61,6 +111,24 @@ install-udev:
 		$(INSTALL_PROGRAM) $$i $(DESTDIR)$(UDEV_RULES_DIR)/96-$$i; \
 	done
 
+install-crond:
+	$(Q) for i in $(CRONTABS); do \
+		$(ES) "	INSTALL $(CROND_DIR)/$$i"; \
+		$(INSTALL_PROGRAM) $$i $(DESTDIR)$(CROND_DIR)/$$i; \
+	done
+
+install-libprogs: $(LIBPROGS)
+	$(Q) for i in $(LIBPROGS); do \
+		$(ES) "	INSTALL $(pkglibdir)/$$i"; \
+		$(INSTALL_PROGRAM) $$i $(DESTDIR)$(pkglibdir)/$$i; \
+	done
+
+install-systemd: $(SERVICE_FILES)
+	$(Q) for i in $(SERVICE_FILES); do \
+		$(ES) "	INSTALL_DATA $(SYSTEMD_SYSTEM_UNIT_DIR)/$$i"; \
+		$(INSTALL_DATA) $$i $(DESTDIR)$(SYSTEMD_SYSTEM_UNIT_DIR)/$$i; \
+	done
+
 install: $(PROGS) $(MANPAGES) $(FMANPAGES) installdirs $(INSTALL_TGT)
 	$(Q) for i in $(PROGS); do \
 		$(ES) "	INSTALL $(root_sbindir)/$$i"; \
@@ -83,6 +151,21 @@ uninstall-udev:
 		$(RM) -f $(DESTDIR)$(UDEV_RULES_DIR)/96-$$i; \
 	done
 
+uninstall-crond:
+	for i in $(CRONTABS); do \
+		$(RM) -f $(DESTDIR)$(CROND_DIR)/$$i; \
+	done
+
+uninstall-libprogs:
+	for i in $(LIBPROGS); do \
+		$(RM) -f $(DESTDIR)$(pkglibdir)/$$i; \
+	done
+
+uninstall-systemd:
+	for i in $(SERVICE_FILES); do \
+		$(RM) -f $(DESTDIR)$(SYSTEMD_SYSTEM_UNIT_DIR)/$$i; \
+	done
+
 uninstall: $(UNINSTALL_TGT)
 	for i in $(PROGS); do \
 		$(RM) -f $(DESTDIR)$(root_sbindir)/$$i; \
diff --git a/scrub/e2scrub.in b/scrub/e2scrub.in
index 08b36002..e1965db4 100644
--- a/scrub/e2scrub.in
+++ b/scrub/e2scrub.in
@@ -44,12 +44,34 @@ print_version() {
 	echo "e2scrub @E2FSPROGS_VERSION@ (@E2FSPROGS_DATE@)"
 }
 
+exitcode() {
+	ret="$1"
+
+	# If we're being run as a service, the return code must fit the LSB
+	# init script action error guidelines, which is to say that we
+	# compress all errors to 1 ("generic or unspecified error", LSB 5.0
+	# section 22.2) and hope the admin will scan the log for what
+	# actually happened.
+
+	# We have to sleep 2 seconds here because journald uses the pid to
+	# connect our log messages to the systemd service.  This is critical
+	# for capturing all the log messages if the scrub fails, because the
+	# fail service uses the service name to gather log messages for the
+	# error report.
+	if [ -n "${SERVICE_MODE}" ]; then
+		test "${ret}" -ne 0 && ret=1
+		sleep 2
+	fi
+
+	exit "${ret}"
+}
+
 while getopts "rtV" opt; do
 	case "${opt}" in
 	"r") reap=1;;
 	"t") fstrim=1;;
-	"V") print_version; exit 0;;
-	*) print_help; exit 2;;
+	"V") print_version; exitcode 0;;
+	*) print_help; exitcode 2;;
 	esac
 done
 shift "$((OPTIND - 1))"
@@ -57,7 +79,7 @@ shift "$((OPTIND - 1))"
 arg="$1"
 if [ -z "${arg}" ]; then
 	print_help
-	exit 1
+	exitcode 1
 fi
 
 # Find the device for a given mountpoint
@@ -112,7 +134,7 @@ fi
 if [ ! -e "${dev}" ]; then
 	echo "${arg}: Not an ext[234] filesystem."
 	print_help
-	exit 16
+	exitcode 16
 fi
 
 # Make sure this is an LVM device we can snapshot
@@ -122,7 +144,7 @@ if [ -z "${LVM2_VG_NAME}" ] || [ -z "${LVM2_LV_NAME}" ] ||
    echo "${LVM2_LV_ROLE}" | grep -q "snapshot"; then
 	echo "${arg}: Not connnected to a LVM logical volume."
 	print_help
-	exit 16
+	exitcode 16
 fi
 start_time="$(date +'%Y%m%d%H%M%S')"
 snap="${LVM2_LV_NAME}.e2scrub"
@@ -185,7 +207,7 @@ if [ "${reap}" -gt 0 ]; then
 	exit 0
 fi
 if ! setup; then
-	exit 8
+	exitcode 8
 fi
 trap "teardown; exit 1" EXIT INT QUIT TERM
 
@@ -236,4 +258,4 @@ case "$?" in
 	;;
 esac
 
-exit "${ret}"
+exitcode "${ret}"
diff --git a/scrub/e2scrub@.service.in b/scrub/e2scrub@.service.in
new file mode 100644
index 00000000..496f8948
--- /dev/null
+++ b/scrub/e2scrub@.service.in
@@ -0,0 +1,20 @@
+[Unit]
+Description=Online ext4 Metadata Check for %I
+OnFailure=e2scrub_fail@%i.service
+Documentation=man:e2scrub(8)
+
+[Service]
+Type=oneshot
+WorkingDirectory=/
+PrivateNetwork=true
+ProtectSystem=true
+ProtectHome=read-only
+PrivateTmp=yes
+AmbientCapabilities=CAP_SYS_ADMIN CAP_SYS_RAWIO
+NoNewPrivileges=yes
+User=root
+IOSchedulingClass=idle
+CPUSchedulingPolicy=idle
+Environment=SERVICE_MODE=1
+ExecStart=@root_sbindir@/e2scrub -t %I
+SyslogIdentifier=%N
diff --git a/scrub/e2scrub_all.cron.in b/scrub/e2scrub_all.cron.in
new file mode 100644
index 00000000..7d42c3f2
--- /dev/null
+++ b/scrub/e2scrub_all.cron.in
@@ -0,0 +1,2 @@
+30 3 * * 0 root test -e /run/systemd/system || @pkglibdir@/e2scrub_all_cron
+10 3 * * * root test -e /run/systemd/system || @root_sbindir@/e2scrub_all -A -r
diff --git a/scrub/e2scrub_all.in b/scrub/e2scrub_all.in
index b9e5dea1..9581dc2c 100644
--- a/scrub/e2scrub_all.in
+++ b/scrub/e2scrub_all.in
@@ -36,12 +36,34 @@ print_version() {
 	echo "e2scrub_all @E2FSPROGS_VERSION@ (@E2FSPROGS_DATE@)"
 }
 
+exitcode() {
+	ret="$1"
+
+	# If we're being run as a service, the return code must fit the LSB
+	# init script action error guidelines, which is to say that we
+	# compress all errors to 1 ("generic or unspecified error", LSB 5.0
+	# section 22.2) and hope the admin will scan the log for what
+	# actually happened.
+
+	# We have to sleep 2 seconds here because journald uses the pid to
+	# connect our log messages to the systemd service.  This is critical
+	# for capturing all the log messages if the scrub fails, because the
+	# fail service uses the service name to gather log messages for the
+	# error report.
+	if [ -n "${SERVICE_MODE}" ]; then
+		test "${ret}" -ne 0 && ret=1
+		sleep 2
+	fi
+
+	exit "${ret}"
+}
+
 while getopts "ArV" opt; do
 	case "${opt}" in
 	"A") scrub_all=1;;
 	"r") scrub_args="${scrub_args} -r";;
-	"V") print_version; exit 0;;
-	*) print_help; exit 2;;
+	"V") print_version; exitcode 0;;
+	*) print_help; exitcode 2;;
 	esac
 done
 shift "$((OPTIND - 1))"
@@ -76,10 +98,35 @@ ls_scrub_targets() {
 	done | sort | uniq
 }
 
+# systemd doesn't know to do path escaping on the instance variable we pass
+# to the e2scrub service, which breaks things if there is a dash in the path
+# name.  Therefore, do the path escaping ourselves if needed.
+escape_path_for_systemd() {
+	local path="$1"
+
+	if echo "${path}" | grep -q -- "-"; then
+		echo "-$(systemd-escape --path "${path}")"
+	else
+		echo "${path}"
+	fi
+}
+
 # Scrub any mounted fs on lvm by creating a snapshot and fscking that.
 stdin="$(realpath /dev/stdin)"
 ls_scrub_targets | while read tgt; do
+	# If we're not reaping and systemd is present, try invoking the
+	# systemd service.
+	if [ -z "${scrub_args}" ] && type systemctl > /dev/null 2>&1; then
+		tgt_esc="$(escape_path_for_systemd "${tgt}")"
+		${DBG} systemctl start "e2scrub@${tgt_esc}" 2> /dev/null < "${stdin}"
+		res=$?
+		if [ "${res}" -eq 0 ] || [ "${res}" -eq 1 ]; then
+			continue;
+		fi
+	fi
+
+	# Otherwise use direct invocation
 	${DBG} "@root_sbindir@/e2scrub" ${scrub_args} "${tgt}" < "${stdin}"
 done
 
-exit 0
+exitcode 0
diff --git a/scrub/e2scrub_all.service.in b/scrub/e2scrub_all.service.in
new file mode 100644
index 00000000..bc05184b
--- /dev/null
+++ b/scrub/e2scrub_all.service.in
@@ -0,0 +1,10 @@
+[Unit]
+Description=Online ext4 Metadata Check for All Filesystems
+ConditionACPower=true
+Documentation=man:e2scrub_all(8)
+
+[Service]
+Type=oneshot
+Environment=SERVICE_MODE=1
+ExecStart=@root_sbindir@/e2scrub_all
+SyslogIdentifier=e2scrub_all
diff --git a/scrub/e2scrub_all.timer.in b/scrub/e2scrub_all.timer.in
new file mode 100644
index 00000000..3d558bbc
--- /dev/null
+++ b/scrub/e2scrub_all.timer.in
@@ -0,0 +1,11 @@
+[Unit]
+Description=Periodic ext4 Online Metadata Check for All Filesystems
+
+[Timer]
+# Run on Sunday at 3:10am, to avoid running afoul of DST changes
+OnCalendar=Sun *-*-* 03:10:00
+RandomizedDelaySec=60
+Persistent=true
+
+[Install]
+WantedBy=timers.target
diff --git a/scrub/e2scrub_all_cron.in b/scrub/e2scrub_all_cron.in
new file mode 100644
index 00000000..f9cff878
--- /dev/null
+++ b/scrub/e2scrub_all_cron.in
@@ -0,0 +1,68 @@
+#!/bin/bash
+
+#  Copyright (C) 2018 Oracle.  All Rights Reserved.
+#
+#  Author: Darrick J. Wong <darrick.wong@oracle.com>
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU General Public License
+#  as published by the Free Software Foundation; either version 2
+#  of the License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it would be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write the Free Software Foundation,
+#  Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301, USA.
+
+# Run e2scrub_all from a cronjob if we don't have systemd and we're not
+# running on AC power.
+
+on_ac_power() {
+	local any_known=no
+
+	# try sysfs power class first
+	if [ -d /sys/class/power_supply ]; then
+		for psu in /sys/class/power_supply/*; do
+			if [ -r "$psu/type" ]; then
+				type=$(cat "$psu/type")
+
+				# ignore batteries
+				[ "$type" = "Battery" ] && continue
+
+				online=$(cat "$psu/online")
+
+				[ "$online" = 1 ] && return 0
+				[ "$online" = 0 ] && any_known=yes
+			fi
+		done
+
+		[ "$any_known" = "yes" ] && return 1
+	fi
+
+	# else fall back to AC adapters in /proc
+	if [ -d /proc/acpi/ac_adapter ]; then
+		for ac in /proc/acpi/ac_adapter/*; do
+			if [ -r "$ac/state" ]; then
+				grep -q on-line "$ac/state" && return 0
+				grep -q off-line "$ac/state" && any_known=yes
+			elif [ -r "$ac/status" ]; then
+				grep -q on-line "$ac/status" && return 0
+				grep -q off-line "$ac/status" && any_known=yes
+			fi
+		done
+
+		[ "$any_known" = "yes" ] && return 1
+	fi
+
+	# Can't tell, just assume we're on AC.
+	return 0
+}
+
+test -e /run/systemd/system && exit 0
+on_ac_power || exit 0
+
+exec @root_sbindir@/e2scrub_all
diff --git a/scrub/e2scrub_fail.in b/scrub/e2scrub_fail.in
new file mode 100644
index 00000000..f27197a9
--- /dev/null
+++ b/scrub/e2scrub_fail.in
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Email logs of failed e2scrub unit runs when the systemd service fails.
+
+recipient="$1"
+test -z "${recipient}" && exit 0
+device="$2"
+test -z "${device}" && exit 0
+hostname="$(hostname -f 2>/dev/null)"
+test -z "${hostname}" && hostname="${HOSTNAME}"
+if ! type sendmail > /dev/null 2>&1; then
+	echo "$0: sendmail program not found."
+	exit 1
+fi
+
+(cat << ENDL
+To: $1
+From: <e2scrub@${hostname}>
+Subject: e2scrub failure on ${device}
+
+So sorry, the automatic e2scrub of ${device} on ${hostname} failed.
+
+A log of what happened follows:
+ENDL
+systemctl status --full --lines 4294967295 "e2scrub@${device}") | sendmail -t -i
diff --git a/scrub/e2scrub_fail@.service.in b/scrub/e2scrub_fail@.service.in
new file mode 100644
index 00000000..df879492
--- /dev/null
+++ b/scrub/e2scrub_fail@.service.in
@@ -0,0 +1,10 @@
+[Unit]
+Description=Online ext4 Metadata Check Failure Reporting for %I
+
+[Service]
+Type=oneshot
+Environment=EMAIL_ADDR=root
+ExecStart=@pkglibdir@/e2scrub_fail "${EMAIL_ADDR}" %I
+User=mail
+Group=mail
+SupplementaryGroups=systemd-journal
diff --git a/scrub/e2scrub_reap.service.in b/scrub/e2scrub_reap.service.in
new file mode 100644
index 00000000..8a320775
--- /dev/null
+++ b/scrub/e2scrub_reap.service.in
@@ -0,0 +1,21 @@
+[Unit]
+Description=Remove Stale Online ext4 Metadata Check Snapshots
+
+[Service]
+Type=oneshot
+WorkingDirectory=/
+PrivateNetwork=true
+ProtectSystem=true
+ProtectHome=read-only
+PrivateTmp=yes
+AmbientCapabilities=CAP_SYS_ADMIN CAP_SYS_RAWIO
+NoNewPrivileges=yes
+User=root
+IOSchedulingClass=idle
+CPUSchedulingPolicy=idle
+ExecStart=@root_sbindir@/e2scrub_all -A -r
+SyslogIdentifier=%N
+RemainAfterExit=no
+
+[Install]
+WantedBy=default.target