add option to abandon chef run if blocked by another for too long

4 files changed, 61 insertions, 8 deletions

diff --git a/lib/chef/client.rb b/lib/chef/client.rb
index 2e5963e996..635cfddfb3 100644
--- a/lib/chef/client.rb
+++ b/lib/chef/client.rb
@@ -224,7 +224,12 @@ class Chef
         Chef::Log.debug "Forked instance successfully reaped (pid: #{pid})"
         true
       else
-        do_run
+        begin
+          do_run
+        rescue Exception => e
+          Chef::Log.error(e.to_s)
+          exit 1
+        end
       end
     end
 
@@ -496,7 +501,7 @@ class Chef
       if Chef::Config[:ssl_verify_mode] == :verify_none and !Chef::Config[:verify_api_cert]
         Chef::Log.warn(<<-WARN)
 
-* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 
+* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
 SSL validation of HTTPS requests is disabled. HTTPS connections are still
 encrypted, but chef is not able to detect forged replies or man in the middle
 attacks.
@@ -518,7 +523,7 @@ To check your SSL configuration, or troubleshoot errors, you can use the
   knife ssl check -c #{Chef::Config.config_file}
 ```
 
-* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 
+* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
 WARN
       end
     end
@@ -530,4 +535,3 @@ end
 require 'chef/cookbook_loader'
 require 'chef/cookbook_version'
 require 'chef/cookbook/synchronizer'
-
diff --git a/lib/chef/config.rb b/lib/chef/config.rb
index a836c62456..f9a3289b30 100644
--- a/lib/chef/config.rb
+++ b/lib/chef/config.rb
@@ -547,6 +547,11 @@ class Chef
     # the directory that files are going to reside.
     default :file_staging_uses_destdir, false
 
+    # Exit if another run is in progress and the chef-client is unable to
+    # get the lock before time expires. If nil, no timeout is enforced. (Exits
+    # immediately if 0.)
+    default :run_lock_timeout, nil
+
     # If installed via an omnibus installer, this gives the path to the
     # "embedded" directory which contains all of the software packaged with
     # omnibus. This is used to locate the cacert.pem file on windows.
diff --git a/lib/chef/exceptions.rb b/lib/chef/exceptions.rb
index bd99cb3ebd..782ecc3fd8 100644
--- a/lib/chef/exceptions.rb
+++ b/lib/chef/exceptions.rb
@@ -321,5 +321,13 @@ class Chef
         super "This functionality is not supported on platform #{platform}."
       end
     end
+
+    # Raised when Chef::Config[:run_lock_timeout] is set and some other client run fails
+    # to release the run lock becure Chef::Config[:run_lock_timeout] seconds pass.
+    class RunLockTimeout < RuntimeError
+      def initialize(duration, blocking_pid)
+        super "Unable to acquire lock. Waited #{duration} seconds for #{blocking_pid} to release."
+      end
+    end
   end
 end
diff --git a/lib/chef/run_lock.rb b/lib/chef/run_lock.rb
index 972db1a4e1..68f697bb2b 100644
--- a/lib/chef/run_lock.rb
+++ b/lib/chef/run_lock.rb
@@ -20,6 +20,9 @@ require 'fcntl'
 if Chef::Platform.windows?
   require 'chef/win32/mutex'
 end
+require 'chef/config'
+require 'chef/exceptions'
+require 'timeout'
 
 class Chef
 
@@ -46,7 +49,9 @@ class Chef
     end
 
     # Acquire the system-wide lock. Will block indefinitely if another process
-    # already has the lock.
+    # already has the lock and Chef::Config[:run_lock_timeout] is
+    # not set. Otherwise will block for Chef::Config[:run_lock_timeout]
+    # seconds and exit if the lock is not acquired.
     #
     # Each call to acquire should have a corresponding call to #release.
     #
@@ -55,7 +60,23 @@ class Chef
     # Either acquire() or test() methods should be called in order to
     # get the ownership of run_lock.
     def acquire
-      wait unless test
+      if timeout_given?
+        begin
+          Timeout::timeout(time_to_wait) do
+            unless test
+              if time_to_wait > 0.0
+                wait
+              else
+                exit_from_timeout
+              end
+            end
+          end
+        rescue Timeout::Error => e
+          exit_from_timeout
+        end
+      else
+        wait unless test
+      end
     end
 
     #
@@ -92,7 +113,6 @@ class Chef
     # Waits until acquiring the system-wide lock.
     #
     def wait
-      runpid = runlock.read.strip.chomp
       Chef::Log.warn("Chef client #{runpid} is running, will wait for it to finish and then run.")
       if Chef::Platform.windows?
         mutex.wait
@@ -144,6 +164,22 @@ class Chef
       @mutex = Chef::ReservedNames::Win32::Mutex.new("Global\\#{runlock_file.gsub(/[\\]/, "/").downcase}")
       mutex.test
     end
+
+    def runpid
+      runlock.read.strip
+    end
+
+    def timeout_given?
+      !time_to_wait.nil?
+    end
+
+    def time_to_wait
+      Chef::Config[:run_lock_timeout]
+    end
+
+    def exit_from_timeout
+      release # Just to be on the safe side...
+      raise Chef::Exceptions::RunLockTimeout.new(time_to_wait, runpid)
+    end
   end
 end
-