summaryrefslogtreecommitdiff
path: root/lib/chef/http.rb
blob: c9df4e123586964191b1e91a65e4b23e8d0bd0fa (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
#--
# Author:: Adam Jacob (<adam@chef.io>)
# Author:: Thom May (<thom@clearairturbulence.org>)
# Author:: Nuo Yan (<nuo@chef.io>)
# Author:: Christopher Brown (<cb@chef.io>)
# Author:: Christopher Walters (<cw@chef.io>)
# Author:: Daniel DeLeo (<dan@chef.io>)
# Copyright:: Copyright 2009-2017, Chef Software Inc.
# License:: Apache License, Version 2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

require "tempfile"
require "net/https"
require "uri"
require "chef/http/basic_client"
require "chef/monkey_patches/net_http"
require "chef/config"
require "chef/platform/query_helpers"
require "chef/exceptions"

class Chef

  # == Chef::HTTP
  # Basic HTTP client, with support for adding features via middleware
  class HTTP

    # Class for applying middleware behaviors to streaming
    # responses. Collects stream handlers (if any) from each
    # middleware. When #handle_chunk is called, the chunk gets
    # passed to all handlers in turn for processing.
    class StreamHandler
      def initialize(middlewares, response)
        middlewares = middlewares.flatten
        @stream_handlers = []
        middlewares.each do |middleware|
          stream_handler = middleware.stream_response_handler(response)
          @stream_handlers << stream_handler unless stream_handler.nil?
        end
      end

      def handle_chunk(next_chunk)
        # stream handlers handle responses so must be applied in reverse order
        # (same as #apply_stream_complete_middleware or #apply_response_midddleware)
        @stream_handlers.reverse.inject(next_chunk) do |chunk, handler|
          Chef::Log.debug("Chef::HTTP::StreamHandler calling #{handler.class}#handle_chunk")
          handler.handle_chunk(chunk)
        end
      end

    end

    def self.middlewares
      @middlewares ||= []
    end

    def self.use(middleware_class)
      middlewares << middleware_class
    end

    attr_reader :url
    attr_reader :sign_on_redirect
    attr_reader :redirect_limit

    attr_reader :options

    attr_reader :middlewares

    # [Boolean] if we're doing keepalives or not
    attr_reader :keepalives

    # Create a HTTP client object. The supplied +url+ is used as the base for
    # all subsequent requests. For example, when initialized with a base url
    # http://localhost:4000, a call to +get+ with 'nodes' will make an
    # HTTP GET request to http://localhost:4000/nodes
    def initialize(url, options = {})
      @url = url
      @default_headers = options[:headers] || {}
      @sign_on_redirect = true
      @redirects_followed = 0
      @redirect_limit = 10
      @keepalives = options[:keepalives] || false
      @options = options

      @middlewares = []
      self.class.middlewares.each do |middleware_class|
        @middlewares << middleware_class.new(options)
      end
    end

    # Send an HTTP HEAD request to the path
    #
    # === Parameters
    # path:: path part of the request URL
    def head(path, headers = {})
      request(:HEAD, path, headers)
    end

    # Send an HTTP GET request to the path
    #
    # === Parameters
    # path:: The path to GET
    def get(path, headers = {})
      request(:GET, path, headers)
    end

    # Send an HTTP PUT request to the path
    #
    # === Parameters
    # path:: path part of the request URL
    def put(path, json, headers = {})
      request(:PUT, path, headers, json)
    end

    # Send an HTTP POST request to the path
    #
    # === Parameters
    # path:: path part of the request URL
    def post(path, json, headers = {})
      request(:POST, path, headers, json)
    end

    # Send an HTTP DELETE request to the path
    #
    # === Parameters
    # path:: path part of the request URL
    def delete(path, headers = {})
      request(:DELETE, path, headers)
    end

    # Makes an HTTP request to +path+ with the given +method+, +headers+, and
    # +data+ (if applicable).
    def request(method, path, headers = {}, data = false)
      http_attempts ||= 0
      url = create_url(path)
      processed_method, url, processed_headers, processed_data = apply_request_middleware(method, url, headers, data)

      response, rest_request, return_value = send_http_request(processed_method, url, processed_headers, processed_data)
      response, rest_request, return_value = apply_response_middleware(response, rest_request, return_value)

      response.error! unless success_response?(response)
      return_value

    rescue Net::HTTPServerException => e
      http_attempts += 1
      response = e.response
      if response.kind_of?(Net::HTTPNotAcceptable) && version_retries - http_attempts > 0
        Chef::Log.debug("Negotiating protocol version with #{url}, retry #{http_attempts}/#{version_retries}")
        retry
      else
        raise
      end
    rescue Exception => exception
      log_failed_request(response, return_value) unless response.nil?

      if exception.respond_to?(:chef_rest_request=)
        exception.chef_rest_request = rest_request
      end
      raise
    end

    def streaming_request_with_progress(path, headers = {}, tempfile = nil, &progress_block)
      http_attempts ||= 0
      url = create_url(path)
      response, rest_request, return_value = nil, nil, nil
      data = nil

      method = :GET
      method, url, processed_headers, data = apply_request_middleware(method, url, headers, data)

      response, rest_request, return_value = send_http_request(method, url, processed_headers, data) do |http_response|
        if http_response.kind_of?(Net::HTTPSuccess)
          tempfile = stream_to_tempfile(url, http_response, tempfile, &progress_block)
        end
        apply_stream_complete_middleware(http_response, rest_request, return_value)
      end
      return nil if response.kind_of?(Net::HTTPRedirection)
      unless response.kind_of?(Net::HTTPSuccess)
        response.error!
      end
      tempfile
    rescue Net::HTTPServerException => e
      http_attempts += 1
      response = e.response
      if response.kind_of?(Net::HTTPNotAcceptable) && version_retries - http_attempts > 0
        Chef::Log.debug("Negotiating protocol version with #{url}, retry #{http_attempts}/#{version_retries}")
        retry
      else
        raise
      end
    rescue Exception => e
      log_failed_request(response, return_value) unless response.nil?
      if e.respond_to?(:chef_rest_request=)
        e.chef_rest_request = rest_request
      end
      raise
    end

    # Makes a streaming download request, streaming the response body to a
    # tempfile. If a block is given, the tempfile is passed to the block and
    # the tempfile will automatically be unlinked after the block is executed.
    #
    # If no block is given, the tempfile is returned, which means it's up to
    # you to unlink the tempfile when you're done with it.
    #
    # @yield [tempfile] block to process the tempfile
    # @yieldparams [tempfile<Tempfile>] tempfile
    def streaming_request(path, headers = {}, tempfile = nil)
      http_attempts ||= 0
      url = create_url(path)
      response, rest_request, return_value = nil, nil, nil
      data = nil

      method = :GET
      method, url, processed_headers, data = apply_request_middleware(method, url, headers, data)

      response, rest_request, return_value = send_http_request(method, url, processed_headers, data) do |http_response|
        if http_response.kind_of?(Net::HTTPSuccess)
          tempfile = stream_to_tempfile(url, http_response, tempfile)
        end
        apply_stream_complete_middleware(http_response, rest_request, return_value)
      end

      return nil if response.kind_of?(Net::HTTPRedirection)
      unless response.kind_of?(Net::HTTPSuccess)
        response.error!
      end

      if block_given?
        begin
          yield tempfile
        ensure
          tempfile && tempfile.close!
        end
      end
      tempfile
    rescue Net::HTTPServerException => e
      http_attempts += 1
      response = e.response
      if response.kind_of?(Net::HTTPNotAcceptable) && version_retries - http_attempts > 0
        Chef::Log.debug("Negotiating protocol version with #{url}, retry #{http_attempts}/#{version_retries}")
        retry
      else
        raise
      end
    rescue Exception => e
      log_failed_request(response, return_value) unless response.nil?
      if e.respond_to?(:chef_rest_request=)
        e.chef_rest_request = rest_request
      end
      raise
    end

    def http_client(base_url = nil)
      base_url ||= url
      if keepalives && !base_url.nil?
        # only reuse the http_client if we want keepalives and have a base_url
        @http_client ||= {}
        # the per-host per-port cache here gets peristent connections correct when
        # redirecting to different servers
        if base_url.is_a?(String) # sigh, this kind of abuse can't happen with strongly typed languages
          @http_client[base_url] ||= build_http_client(base_url)
        else
          @http_client[base_url.host] ||= {}
          @http_client[base_url.host][base_url.port] ||= build_http_client(base_url)
        end
      else
        build_http_client(base_url)
      end
    end

    # DEPRECATED: This is only kept around to provide access to cache control data in
    # lib/chef/provider/remote_file/http.rb
    # FIXME: Find a better API.
    def last_response
      @last_response
    end

    private

    # @api private
    def build_http_client(base_url)
      if chef_zero_uri?(base_url)
        # PERFORMANCE CRITICAL: *MUST* lazy require here otherwise we load up webrick
        # via chef-zero and that hits DNS (at *require* time) which may timeout,
        # when for most knife/chef-client work we never need/want this loaded.

        unless defined?(SocketlessChefZeroClient)
          require "chef/http/socketless_chef_zero_client"
        end

        SocketlessChefZeroClient.new(base_url)
      else
        BasicClient.new(base_url, ssl_policy: Chef::HTTP::APISSLPolicy, keepalives: keepalives)
      end
    end

    # @api private
    def create_url(path)
      return path if path.is_a?(URI)
      if path =~ /^(http|https|chefzero):\/\//i
        URI.parse(path)
      elsif path.nil? || path.empty?
        URI.parse(@url)
      else
        # The regular expressions used here are to make sure '@url' does not have
        # any trailing slashes and 'path' does not have any leading slashes. This
        # way they are always joined correctly using just one slash.
        URI.parse(@url.gsub(%r{/+$}, "") + "/" + path.gsub(%r{^/+}, ""))
      end
    end

    # @api private
    def apply_request_middleware(method, url, headers, data)
      middlewares.inject([method, url, headers, data]) do |req_data, middleware|
        Chef::Log.debug("Chef::HTTP calling #{middleware.class}#handle_request")
        middleware.handle_request(*req_data)
      end
    end

    # @api private
    def apply_response_middleware(response, rest_request, return_value)
      middlewares.reverse.inject([response, rest_request, return_value]) do |res_data, middleware|
        Chef::Log.debug("Chef::HTTP calling #{middleware.class}#handle_response")
        middleware.handle_response(*res_data)
      end
    end

    # @api private
    def apply_stream_complete_middleware(response, rest_request, return_value)
      middlewares.reverse.inject([response, rest_request, return_value]) do |res_data, middleware|
        Chef::Log.debug("Chef::HTTP calling #{middleware.class}#handle_stream_complete")
        middleware.handle_stream_complete(*res_data)
      end
    end

    # @api private
    def log_failed_request(response, return_value)
      return_value ||= {}
      error_message = "HTTP Request Returned #{response.code} #{response.message}: "
      error_message << (return_value["error"].respond_to?(:join) ? return_value["error"].join(", ") : return_value["error"].to_s)
      Chef::Log.info(error_message)
    end

    # @api private
    def success_response?(response)
      response.kind_of?(Net::HTTPSuccess) || response.kind_of?(Net::HTTPRedirection)
    end

    # Runs a synchronous HTTP request, with no middleware applied (use #request
    # to have the middleware applied). The entire response will be loaded into memory.
    # @api private
    def send_http_request(method, url, base_headers, body, &response_handler)
      retrying_http_errors(url) do
        headers = build_headers(method, url, base_headers, body)
        client = http_client(url)
        return_value = nil
        if block_given?
          request, response = client.request(method, url, body, headers, &response_handler)
        else
          request, response = client.request(method, url, body, headers) { |r| r.read_body }
          return_value = response.read_body
        end
        @last_response = response

        if response.kind_of?(Net::HTTPSuccess)
          [response, request, return_value]
        elsif response.kind_of?(Net::HTTPNotModified) # Must be tested before Net::HTTPRedirection because it's subclass.
          [response, request, false]
        elsif redirect_location = redirected_to(response)
          if [:GET, :HEAD].include?(method)
            follow_redirect do
              send_http_request(method, url + redirect_location, headers, body, &response_handler)
            end
          else
            raise Exceptions::InvalidRedirect, "#{method} request was redirected from #{url} to #{redirect_location}. Only GET and HEAD support redirects."
          end
        else
          [response, request, nil]
        end
      end
    end

    # Wraps an HTTP request with retry logic.
    # === Arguments
    # url:: URL of the request, used for error messages
    # @api private
    def retrying_http_errors(url)
      http_attempts = 0
      begin
        loop do
          http_attempts += 1
          response, request, return_value = yield
          # handle HTTP 50X Error
          if response.kind_of?(Net::HTTPServerError) && !Chef::Config.local_mode
            if http_retry_count - http_attempts + 1 > 0
              sleep_time = 1 + (2**http_attempts) + rand(2**http_attempts)
              Chef::Log.error("Server returned error #{response.code} for #{url}, retrying #{http_attempts}/#{http_retry_count} in #{sleep_time}s")
              sleep(sleep_time)
              redo
            end
          end
          return [response, request, return_value]
        end
      rescue SocketError, Errno::ETIMEDOUT, Errno::ECONNRESET => e
        if http_retry_count - http_attempts + 1 > 0
          Chef::Log.error("Error connecting to #{url}, retry #{http_attempts}/#{http_retry_count}")
          sleep(http_retry_delay)
          retry
        end
        e.message.replace "Error connecting to #{url} - #{e.message}"
        raise e
      rescue Errno::ECONNREFUSED
        if http_retry_count - http_attempts + 1 > 0
          Chef::Log.error("Connection refused connecting to #{url}, retry #{http_attempts}/#{http_retry_count}")
          sleep(http_retry_delay)
          retry
        end
        raise Errno::ECONNREFUSED, "Connection refused connecting to #{url}, giving up"
      rescue Timeout::Error
        if http_retry_count - http_attempts + 1 > 0
          Chef::Log.error("Timeout connecting to #{url}, retry #{http_attempts}/#{http_retry_count}")
          sleep(http_retry_delay)
          retry
        end
        raise Timeout::Error, "Timeout connecting to #{url}, giving up"
      rescue OpenSSL::SSL::SSLError => e
        if (http_retry_count - http_attempts + 1 > 0) && !e.message.include?("certificate verify failed")
          Chef::Log.error("SSL Error connecting to #{url}, retry #{http_attempts}/#{http_retry_count}")
          sleep(http_retry_delay)
          retry
        end
        raise OpenSSL::SSL::SSLError, "SSL Error connecting to #{url} - #{e.message}"
      end
    end

    def version_retries
      @version_retries ||= options[:version_class].possible_requests
    end

    # @api private
    def http_retry_delay
      config[:http_retry_delay]
    end

    # @api private
    def http_retry_count
      config[:http_retry_count]
    end

    # @api private
    def config
      Chef::Config
    end

    # @api private
    def follow_redirect
      raise Chef::Exceptions::RedirectLimitExceeded if @redirects_followed >= redirect_limit
      @redirects_followed += 1
      Chef::Log.debug("Following redirect #{@redirects_followed}/#{redirect_limit}")

      yield
    ensure
      @redirects_followed = 0
    end

    # @api private
    def chef_zero_uri?(uri)
      uri = URI.parse(uri) unless uri.respond_to?(:scheme)
      uri.scheme == "chefzero"
    end

    # @api private
    def redirected_to(response)
      return nil  unless response.kind_of?(Net::HTTPRedirection)
      # Net::HTTPNotModified is undesired subclass of Net::HTTPRedirection so test for this
      return nil  if response.kind_of?(Net::HTTPNotModified)
      response["location"]
    end

    # @api private
    def build_headers(method, url, headers = {}, json_body = false)
      headers = @default_headers.merge(headers)
      headers["Content-Length"] = json_body.bytesize.to_s if json_body
      headers.merge!(Chef::Config[:custom_http_headers]) if Chef::Config[:custom_http_headers]
      headers
    end

    # @api private
    def stream_to_tempfile(url, response, tf = nil, &progress_block)
      content_length = response["Content-Length"]
      if tf.nil?
        tf = Tempfile.open("chef-rest")
        if Chef::Platform.windows?
          tf.binmode # required for binary files on Windows platforms
        end
      end
      Chef::Log.debug("Streaming download from #{url} to tempfile #{tf.path}")
      # Stolen from http://www.ruby-forum.com/topic/166423
      # Kudos to _why!

      stream_handler = StreamHandler.new(middlewares, response)

      response.read_body do |chunk|
        tf.write(stream_handler.handle_chunk(chunk))
        yield tf.size, content_length if block_given?
      end
      tf.close
      tf
    rescue Exception
      tf.close! if tf
      raise
    end

  end
end