summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorThom May <thom@may.lt>2016-03-15 21:05:49 +0000
committerThom May <thom@may.lt>2016-03-15 21:05:49 +0000
commitc1a389c2a8452e9b796aa1d34c4d9e51f4af30c7 (patch)
treeed2cc9b0c226a21ba3b9ab6b101fa76cb2db4891
parent47cd0cb9f2c14ced5a17ea0d1da34b9aeaaf36d8 (diff)
parentff539423f067ee83c07dcf73cbf688c6a07f64ae (diff)
downloadchef-c1a389c2a8452e9b796aa1d34c4d9e51f4af30c7.tar.gz
Merge pull request #4658 from chef/tm/remote_file_download_progress11.9
Remote file download progress
-rw-r--r--chef-config/lib/chef-config/config.rb6
-rw-r--r--chef-config/spec/unit/config_spec.rb8
-rw-r--r--lib/chef/event_dispatch/base.rb6
-rw-r--r--lib/chef/formatters/doc.rb16
-rw-r--r--lib/chef/http.rb32
-rw-r--r--lib/chef/provider/remote_file/http.rb20
-rw-r--r--lib/chef/resource/remote_file.rb9
-rw-r--r--spec/unit/formatters/doc_spec.rb14
-rw-r--r--spec/unit/provider/remote_file/http_spec.rb21
9 files changed, 130 insertions, 2 deletions
diff --git a/chef-config/lib/chef-config/config.rb b/chef-config/lib/chef-config/config.rb
index d7a8a1b997..ec41ccdf8c 100644
--- a/chef-config/lib/chef-config/config.rb
+++ b/chef-config/lib/chef-config/config.rb
@@ -299,6 +299,12 @@ module ChefConfig
# Using 'stream_execute_output' will have Chef always stream the execute output
default :stream_execute_output, false
+ # Using `show_download_progress` will display the overall progress
+ # of a remote file download
+ default :show_download_progress, false
+ # How often to update the progress meter, in percent
+ default :download_progress_interval, 10
+
default :http_retry_count, 5
default :http_retry_delay, 5
default :interval, nil
diff --git a/chef-config/spec/unit/config_spec.rb b/chef-config/spec/unit/config_spec.rb
index dbde3d3160..797fe3a233 100644
--- a/chef-config/spec/unit/config_spec.rb
+++ b/chef-config/spec/unit/config_spec.rb
@@ -274,6 +274,14 @@ RSpec.describe ChefConfig::Config do
expect(ChefConfig::Config[:stream_execute_output]).to eq(false)
end
+ it "ChefConfig::Config[:show_download_progress] defaults to false" do
+ expect(ChefConfig::Config[:show_download_progress]).to eq(false)
+ end
+
+ it "ChefConfig::Config[:download_progress_interval] defaults to every 10%" do
+ expect(ChefConfig::Config[:download_progress_interval]).to eq(10)
+ end
+
it "ChefConfig::Config[:file_backup_path] defaults to /var/chef/backup" do
allow(ChefConfig::Config).to receive(:cache_path).and_return(primary_cache_path)
backup_path = is_windows ? "#{primary_cache_path}\\backup" : "#{primary_cache_path}/backup"
diff --git a/lib/chef/event_dispatch/base.rb b/lib/chef/event_dispatch/base.rb
index b3271a139a..6c6b2fa3bb 100644
--- a/lib/chef/event_dispatch/base.rb
+++ b/lib/chef/event_dispatch/base.rb
@@ -344,6 +344,12 @@ class Chef
def resource_update_applied(resource, action, update)
end
+ # Called when a progress notification should be sent to the user to
+ # indicate the overall progress of a long running operation, such as
+ # a large file download.
+ def resource_update_progress(resource, current, total, interval)
+ end
+
# Called when a resource fails, but will retry.
def resource_failed_retriable(resource, action, retry_count, exception)
end
diff --git a/lib/chef/formatters/doc.rb b/lib/chef/formatters/doc.rb
index 3f832f1e92..d43f1993b2 100644
--- a/lib/chef/formatters/doc.rb
+++ b/lib/chef/formatters/doc.rb
@@ -23,6 +23,7 @@ class Chef
@start_time = Time.now
@end_time = @start_time
@skipped_resources = 0
+ @progress = {}
end
def elapsed_time
@@ -284,6 +285,21 @@ class Chef
indent
end
+ def resource_update_progress(resource, current, total, interval)
+ @progress[resource] ||= 0
+
+ percent_complete = (current.to_f / total.to_f * 100).to_i
+
+ if percent_complete > @progress[resource]
+
+ @progress[resource] = percent_complete
+
+ if percent_complete % interval == 0
+ start_line " - Progress: #{percent_complete}%", :green
+ end
+ end
+ end
+
# Called when a resource fails, but will retry.
def resource_failed_retriable(resource, action, retry_count, exception)
end
diff --git a/lib/chef/http.rb b/lib/chef/http.rb
index 15fb20d588..c6afa97d23 100644
--- a/lib/chef/http.rb
+++ b/lib/chef/http.rb
@@ -74,6 +74,7 @@ class Chef
attr_reader :redirect_limit
attr_reader :options
+
attr_reader :middlewares
# Create a HTTP client object. The supplied +url+ is used as the base for
@@ -153,6 +154,33 @@ class Chef
raise
end
+ def streaming_request_with_progress(path, headers = {}, &progress_block)
+ url = create_url(path)
+ response, rest_request, return_value = nil, nil, nil
+ tempfile = nil
+
+ method = :GET
+ method, url, headers, data = apply_request_middleware(method, url, headers, data)
+
+ response, rest_request, return_value = send_http_request(method, url, headers, data) do |http_response|
+ if http_response.kind_of?(Net::HTTPSuccess)
+ tempfile = stream_to_tempfile(url, http_response, &progress_block)
+ end
+ apply_stream_complete_middleware(http_response, rest_request, return_value)
+ end
+ return nil if response.kind_of?(Net::HTTPRedirection)
+ unless response.kind_of?(Net::HTTPSuccess)
+ response.error!
+ end
+ tempfile
+ rescue Exception => e
+ log_failed_request(response, return_value) unless response.nil?
+ if e.respond_to?(:chef_rest_request=)
+ e.chef_rest_request = rest_request
+ end
+ raise
+ end
+
# Makes a streaming download request, streaming the response body to a
# tempfile. If a block is given, the tempfile is passed to the block and
# the tempfile will automatically be unlinked after the block is executed.
@@ -392,7 +420,8 @@ class Chef
headers
end
- def stream_to_tempfile(url, response)
+ def stream_to_tempfile(url, response, &progress_block)
+ content_length = response["Content-Length"]
tf = Tempfile.open("chef-rest")
if Chef::Platform.windows?
tf.binmode # required for binary files on Windows platforms
@@ -405,6 +434,7 @@ class Chef
response.read_body do |chunk|
tf.write(stream_handler.handle_chunk(chunk))
+ yield tf.size, content_length if block_given?
end
tf.close
tf
diff --git a/lib/chef/provider/remote_file/http.rb b/lib/chef/provider/remote_file/http.rb
index 6a9738c02b..ad044f9e3c 100644
--- a/lib/chef/provider/remote_file/http.rb
+++ b/lib/chef/provider/remote_file/http.rb
@@ -39,6 +39,10 @@ class Chef
@current_resource = current_resource
end
+ def events
+ new_resource.events
+ end
+
def headers
conditional_get_headers.merge(new_resource.headers)
end
@@ -57,7 +61,13 @@ class Chef
def fetch
http = Chef::HTTP::Simple.new(uri, http_client_opts)
- tempfile = http.streaming_request(uri, headers)
+ if want_progress?
+ tempfile = http.streaming_request_with_progress(uri, headers) do |size, total|
+ events.resource_update_progress(new_resource, size, total, progress_interval)
+ end
+ else
+ tempfile = http.streaming_request(uri, headers)
+ end
if tempfile
update_cache_control_data(tempfile, http.last_response)
tempfile.close
@@ -78,6 +88,14 @@ class Chef
@cache_control_data ||= CacheControlData.load_and_validate(uri, current_resource.checksum)
end
+ def want_progress?
+ events.formatter? && (Chef::Config[:show_download_progress] || !!new_resource.show_progress)
+ end
+
+ def progress_interval
+ Chef::Config[:download_progress_interval]
+ end
+
def want_mtime_cache_control?
new_resource.use_last_modified
end
diff --git a/lib/chef/resource/remote_file.rb b/lib/chef/resource/remote_file.rb
index 595280a583..4a1d1c6cff 100644
--- a/lib/chef/resource/remote_file.rb
+++ b/lib/chef/resource/remote_file.rb
@@ -122,6 +122,15 @@ class Chef
)
end
+ def show_progress(args = nil)
+ set_or_return(
+ :show_progress,
+ args,
+ :default => false,
+ :kind_of => [ TrueClass, FalseClass ]
+ )
+ end
+
private
include Chef::Mixin::Uris
diff --git a/spec/unit/formatters/doc_spec.rb b/spec/unit/formatters/doc_spec.rb
index e9bff0759d..b8eccc1bc9 100644
--- a/spec/unit/formatters/doc_spec.rb
+++ b/spec/unit/formatters/doc_spec.rb
@@ -75,4 +75,18 @@ describe Chef::Formatters::Base do
expect(formatter.elapsed_time).to eql(36610.0)
expect(formatter.pretty_elapsed_time).to include("10 hours 10 minutes 10 seconds")
end
+
+ it "shows the percentage completion of an action" do
+ res = Chef::Resource::RemoteFile.new("canteloupe")
+ formatter.resource_update_progress(res, 35, 50, 10)
+ expect(out.string).to include(" - Progress: 70%")
+ end
+
+ it "updates the percentage completion of an action" do
+ res = Chef::Resource::RemoteFile.new("canteloupe")
+ formatter.resource_update_progress(res, 70, 100, 10)
+ expect(out.string).to include(" - Progress: 70%")
+ formatter.resource_update_progress(res, 80, 100, 10)
+ expect(out.string).to include(" - Progress: 80%")
+ end
end
diff --git a/spec/unit/provider/remote_file/http_spec.rb b/spec/unit/provider/remote_file/http_spec.rb
index 60ecd45dbb..f58a3d3c14 100644
--- a/spec/unit/provider/remote_file/http_spec.rb
+++ b/spec/unit/provider/remote_file/http_spec.rb
@@ -163,6 +163,12 @@ describe Chef::Provider::RemoteFile::HTTP do
let(:last_response) { {} }
+ let(:event_dispatcher) do
+ event_dispatcher = double(Chef::EventDispatch::Dispatcher)
+ allow(event_dispatcher).to receive(:formatter?).and_return(false)
+ event_dispatcher
+ end
+
let(:rest) do
rest = double(Chef::HTTP::Simple)
allow(rest).to receive(:streaming_request).and_return(tempfile)
@@ -173,6 +179,7 @@ describe Chef::Provider::RemoteFile::HTTP do
before do
new_resource.headers({})
new_resource.use_last_modified(false)
+ allow(new_resource).to receive(:events).and_return(event_dispatcher)
expect(Chef::Provider::RemoteFile::CacheControlData).to receive(:load_and_validate).with(uri, current_resource_checksum).and_return(cache_control_data)
expect(Chef::HTTP::Simple).to receive(:new).with(*expected_http_args).and_return(rest)
@@ -205,6 +212,20 @@ describe Chef::Provider::RemoteFile::HTTP do
expect(cache_control_data.checksum).to eq(fetched_content_checksum)
end
+ context "with progress reports" do
+ before do
+ Chef::Config[:show_download_progress] = true
+ end
+
+ it "should yield its progress" do
+ allow(rest).to receive(:streaming_request_with_progress).and_yield(50, 100).and_yield(70, 100).and_return(tempfile)
+ expect(event_dispatcher).to receive(:formatter?).and_return(true)
+ expect(event_dispatcher).to receive(:resource_update_progress).with(new_resource, 50, 100, 10).ordered
+ expect(event_dispatcher).to receive(:resource_update_progress).with(new_resource, 70, 100, 10).ordered
+ fetcher.fetch
+ end
+ end
+
context "and the response does not contain an etag" do
let(:last_response) { { "etag" => nil } }
it "does not include an etag in the result" do