From 691e0195fdbea22c35f2b472c7c2330a432942ac Mon Sep 17 00:00:00 2001 From: Jesse Campbell Date: Tue, 26 Feb 2013 13:29:01 -0500 Subject: support etags and last modified times for remote_file --- lib/chef/provider/remote_file.rb | 91 ++++++++++++++++++++++++++++++++---- lib/chef/provider/remote_file/ftp.rb | 55 ++++++++++++++++------ lib/chef/resource/remote_file.rb | 27 +++++++++++ 3 files changed, 148 insertions(+), 25 deletions(-) (limited to 'lib/chef') diff --git a/lib/chef/provider/remote_file.rb b/lib/chef/provider/remote_file.rb index 4d1e696d0e..e979f5a4ba 100644 --- a/lib/chef/provider/remote_file.rb +++ b/lib/chef/provider/remote_file.rb @@ -30,6 +30,12 @@ class Chef def load_current_resource @current_resource = Chef::Resource::RemoteFile.new(@new_resource.name) + fileinfo = load_fileinfo + if fileinfo["src"] + @current_resource.etag fileinfo["etag"] + @current_resource.last_modified fileinfo["last_modified"] + @current_resource.source fileinfo["src"] + end super end @@ -40,10 +46,13 @@ class Chef Chef::Log.debug("#{@new_resource} checksum matches target checksum (#{@new_resource.checksum}) - not updating") else sources = @new_resource.source - raw_file, raw_file_source = try_multiple_sources(sources) - if matches_current_checksum?(raw_file) - Chef::Log.debug "#{@new_resource} target and source checksums are the same - not updating" + raw_file, raw_file_source, target_matched = try_multiple_sources(sources) + if target_matched + Chef::Log.info("#{@new_resource} matched #{raw_file_source}, not updating") + elsif matches_current_checksum?(raw_file) + Chef::Log.info("#{@new_resource} downloaded from #{raw_file_source}, checksums match, not updating") else + Chef::Log.info("#{@new_resource} downloaded from #{raw_file_source}") description = [] description << "copy file downloaded from #{raw_file_source} into #{@new_resource.path}" description << diff_current(raw_file.path) @@ -98,7 +107,7 @@ class Chef source = sources.shift begin uri = URI.parse(source) - raw_file = grab_file_from_uri(uri) + raw_file, target_matched = grab_file_from_uri(uri) rescue ArgumentError => e raise e rescue => e @@ -107,9 +116,9 @@ class Chef else error = e.to_s end - Chef::Log.debug("#{@new_resource} cannot be downloaded from #{source}: #{error}") + Chef::Log.info("#{@new_resource} cannot be downloaded from #{source}: #{error}") if source = sources.shift - Chef::Log.debug("#{@new_resource} trying to download from another mirror") + Chef::Log.info("#{@new_resource} trying to download from another mirror") retry else raise e @@ -118,29 +127,91 @@ class Chef if uri.userinfo uri.password = "********" end - return raw_file, uri.to_s + return raw_file, uri.to_s, target_matched end # Given a source uri, return a Tempfile, or a File that acts like a Tempfile (close! method) def grab_file_from_uri(uri) + if_modified_since = @new_resource.last_modified + if_none_match = @new_resource.etag + if uri == @current_resource.source[0] + if_modified_since ||= @current_resource.last_modified + if_none_match ||= @current_resource.etag + end + target_matched = false + raw_file = nil + last_modified = nil + etag = nil if URI::HTTP === uri #HTTP or HTTPS - raw_file = RestClient::Request.execute(:method => :get, :url => uri.to_s, :raw_response => true).file + begin + headers = Hash.new + if if_none_match + headers[:if_none_match] = "\"#{if_none_match}\"" + elsif if_modified_since + headers[:if_modified_since] = if_modified_since.strftime("%a, %d %b %Y %H:%M:%S %Z") + end + rest = RestClient::Request.execute(:method => :get, :url => uri.to_s, :headers => headers, :raw_response => true) + raw_file = rest.file + if rest.headers.include?(:last_modified) + last_modified = Time.parse(rest.headers[:last_modified]) + end + if rest.headers.include?(:etag) + etag = rest.headers[:etag] + end + rescue RestClient::Exception => e + if e.http_code == 304 + target_matched = true + else + raise e + end + end elsif URI::FTP === uri #FTP - raw_file = FTP::fetch(uri, @new_resource.ftp_active_mode) + raw_file, last_modified = FTP::fetch_if_modified(uri, @new_resource.ftp_active_mode, if_modified_since) + if last_modified && if_modified_since && last_modified <= if_modified_since + target_matched = true + end elsif uri.scheme == "file" #local/network file + last_modified = ::File.mtime(uri.path) raw_file = ::File.new(uri.path, "r") def raw_file.close! self.close end + if last_modified && if_modified_since && last_modified.to_i <= if_modified_since.to_i + target_matched = true + end else raise ArgumentError, "Invalid uri. Only http(s), ftp, and file are currently supported" end - raw_file + unless target_matched + @new_resource.etag etag unless @new_resource.etag + @new_resource.last_modified last_modified unless @new_resource.last_modified + save_fileinfo(uri) + end + return raw_file, target_matched end + def load_fileinfo + begin + Chef::JSONCompat.from_json(Chef::FileCache.load("remote_file/#{new_resource.name}")) + rescue Chef::Exceptions::FileNotFound + cache = Hash.new + cache["etag"] = nil + cache["last_modified"] = nil + cache["src"] = nil + cache + end + end + + def save_fileinfo(uri) + cache = Hash.new + cache["etag"] = @new_resource.etag + cache["last_modified"] = @new_resource.last_modified + cache["src"] = uri + Chef::FileCache.store("remote_file/#{new_resource.name}", cache.to_json) + end end end end diff --git a/lib/chef/provider/remote_file/ftp.rb b/lib/chef/provider/remote_file/ftp.rb index 3c5d3e0a91..b0e28a4de8 100644 --- a/lib/chef/provider/remote_file/ftp.rb +++ b/lib/chef/provider/remote_file/ftp.rb @@ -28,7 +28,24 @@ class Chef # Fetches the file at uri using Net::FTP, returning a Tempfile def self.fetch(uri, ftp_active_mode) - self.new(uri, ftp_active_mode).fetch() + ftp = self.new(uri, ftp_active_mode) + ftp.connect + tempfile = ftp.fetch + ftp.disconnect + tempfile + end + + def self.fetch_if_modified(uri, ftp_active_mode, last_modified) + ftp = self.new(uri, ftp_active_mode) + ftp.connect + mtime = ftp.mtime + if mtime && last_modified && mtime.to_i <= last_modified.to_i + tempfile = nil + else + tempfile = ftp.fetch + end + ftp.disconnect + return tempfile, mtime end # Parse the uri into instance variables @@ -42,6 +59,7 @@ class Chef @ftp_active_mode = ftp_active_mode @hostname = uri.hostname @port = uri.port + @ftp = Net::FTP.new if uri.userinfo @user = URI.unescape(uri.user) @pass = URI.unescape(uri.password) @@ -51,28 +69,35 @@ class Chef end end - # Fetches using Net::FTP, returns a Tempfile with the content - def fetch() - tempfile = Tempfile.new(@filename) - + def connect # The access sequence is defined by RFC 1738 - ftp = Net::FTP.new - ftp.connect(@hostname, @port) - ftp.passive = !@ftp_active_mode - ftp.login(@user, @pass) + @ftp.connect(@hostname, @port) + @ftp.passive = !@ftp_active_mode + @ftp.login(@user, @pass) @directories.each do |cwd| - ftp.voidcmd("CWD #{cwd}") + @ftp.voidcmd("CWD #{cwd}") end + end + + def disconnect + @ftp.close + end + + def mtime + @ftp.mtime(@filename) + end + + # Fetches using Net::FTP, returns a Tempfile with the content + def fetch + tempfile = Tempfile.new(@filename) if @typecode - ftp.voidcmd("TYPE #{@typecode.upcase}") + @ftp.voidcmd("TYPE #{@typecode.upcase}") end - ftp.getbinaryfile(@filename, tempfile.path) - ftp.close - + @ftp.getbinaryfile(@filename, tempfile.path) tempfile end - private + private def parse_path(path) path = path.sub(%r{\A/}, '%2F') # re-encode the beginning slash because uri library decodes it. diff --git a/lib/chef/resource/remote_file.rb b/lib/chef/resource/remote_file.rb index 524e00186b..df055ab6b4 100644 --- a/lib/chef/resource/remote_file.rb +++ b/lib/chef/resource/remote_file.rb @@ -33,6 +33,8 @@ class Chef @resource_name = :remote_file @action = "create" @source = [] + @etag = nil + @last_modified = nil @ftp_active_mode = false @provider = Chef::Provider::RemoteFile end @@ -55,6 +57,31 @@ class Chef ) end + def etag(args=nil) + # Only store the etag itself, skip the quotes (and leading W/ if it's there) + if args.is_a?(String) && args.include?('"') + args = args.split('"')[1] + end + set_or_return( + :etag, + args, + :kind_of => String + ) + end + + def last_modified(args=nil) + if args.is_a?(String) + args = Time.parse(args).gmtime + elsif args.is_a?(Time) + args.gmtime + end + set_or_return( + :last_modified, + args, + :kind_of => Time + ) + end + def ftp_active_mode(args=nil) set_or_return( :ftp_active_mode, -- cgit v1.2.1