XenAPI: Fix race on rotate_xen_guest_logs

There is a race between where Xen creates the domain and starts logging
and when XAPI returns to say that the domain has started.

If the cronjob to rotate the guest logs runs in this period, it can
delete the console logs before we set the last_dom_id which is used to
preserve logs.

This change adds a fixed delay before deleting any log files, allowing
the VM start operation to return after the domain has been successfully
created.

Change-Id: I82ae3e008974f33b60256366f171abf1f0369e60
Closes-Bug: 1535616

1 files changed, 8 insertions, 2 deletions

diff --git a/tools/xenserver/rotate_xen_guest_logs.sh b/tools/xenserver/rotate_xen_guest_logs.sh
index 587ec194de..f01051c96c 100755
--- a/tools/xenserver/rotate_xen_guest_logs.sh
+++ b/tools/xenserver/rotate_xen_guest_logs.sh
@@ -22,6 +22,12 @@ syslog_tag='rotate_xen_guest_logs'
 
 log_file_base="${log_dir}/console."
 
+# Only delete log files older than this number of minutes
+# to avoid a race where Xen creates the domain and starts
+# logging before the XAPI VM start returns (and allows us
+# to preserve the log file using last_dom_id)
+min_logfile_age=10
+
 # Ensure logging is setup correctly for all domains
 xenstore-write /local/logconsole/@ "${log_file_base}%d"
 
@@ -39,9 +45,9 @@ done
 valid_last_dom_ids=$(xe vm-list params=other-config --minimal | tr ';,' '\n\n' | grep last_dom_id | sed -e 's/last_dom_id: //g' | xargs)
 echo "Valid dom IDs: $valid_last_dom_ids" | /usr/bin/logger -t $syslog_tag
 
-# Remove console files that do not correspond to valid last_dom_id's
+# Remove old console files that do not correspond to valid last_dom_id's
 allowed_consoles=".*console.\(${valid_last_dom_ids// /\\|}\)$"
-delete_logs=`find "$log_dir" -type f -not -regex "$allowed_consoles"`
+delete_logs=`find "$log_dir" -type f -mmin +${min_logfile_age} -not -regex "$allowed_consoles"`
 for log in $delete_logs; do
     if echo "$current_logs" | grep -q -w "$log"; then
         echo "Deleting: $log" | /usr/bin/logger -t $syslog_tag