Merge pull request #3320 from chef/lcg/node-utf8-sanitize

Lcg/node utf8 sanitize

6 files changed, 23 insertions, 4 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index aff96bb8bd..15d093da29 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,6 +15,7 @@
 * Convert bootstrap template to use sh #2877
 * [Issue #3316](https://github.com/chef/chef/issues/3316) Fix idempotency issues with the `windows_package` resource
 * [pr#3295](https://github.com/chef/chef/pull/3295): Stop mutating `new_resource.checksum` in file providers.  Fixes some ChecksumMismatch exceptions like [issue#3168](https://github.com/chef/chef/issues/3168)
+* [pr#3320] Sanitize non-UTF8 characters in the node data before doing node.save().  Works around many UTF8 exception issues reported on node.save().
 
 ## 12.3.0
 
diff --git a/chef.gemspec b/chef.gemspec
index 8bec26ea7c..ec600f1f89 100644
--- a/chef.gemspec
+++ b/chef.gemspec
@@ -22,7 +22,7 @@ Gem::Specification.new do |s|
   s.add_dependency "mixlib-shellout", ">= 2.0.0.rc.0", "< 3.0"
   s.add_dependency "ohai", "~> 8.0"
 
-  s.add_dependency "ffi-yajl", ">= 1.2", "< 3.0"
+  s.add_dependency "ffi-yajl", "~> 2.2"
   s.add_dependency "net-ssh", "~> 2.6"
   s.add_dependency "net-ssh-multi", "~> 1.1"
   # CHEF-3027: The knife-cloud plugins require newer features from highline, core chef should not.
diff --git a/lib/chef/http/json_input.rb b/lib/chef/http/json_input.rb
index 23ccc3a8a7..3296d8821f 100644
--- a/lib/chef/http/json_input.rb
+++ b/lib/chef/http/json_input.rb
@@ -25,14 +25,19 @@ class Chef
     # Middleware that takes json input and turns it into raw text
     class JSONInput
 
+      attr_accessor :opts
+
       def initialize(opts={})
+        @opts = opts
       end
 
       def handle_request(method, url, headers={}, data=false)
         if data && should_encode_as_json?(headers)
           headers.delete_if { |key, _value| key.downcase == 'content-type' }
           headers["Content-Type"] = 'application/json'
-          data = Chef::JSONCompat.to_json(data)
+          json_opts = {}
+          json_opts[:validate_utf8] = opts[:validate_utf8] if opts.has_key?(:validate_utf8)
+          data = Chef::JSONCompat.to_json(data, json_opts)
           # Force encoding to binary to fix SSL related EOFErrors
           # cf. http://tickets.opscode.com/browse/CHEF-2363
           # http://redmine.ruby-lang.org/issues/5233
diff --git a/lib/chef/node.rb b/lib/chef/node.rb
index 9823185ede..8c41d7e10e 100644
--- a/lib/chef/node.rb
+++ b/lib/chef/node.rb
@@ -83,7 +83,15 @@ class Chef
     end
 
     def chef_server_rest
-      @chef_server_rest ||= Chef::REST.new(Chef::Config[:chef_server_url])
+      # for saving node data we use validate_utf8: false which will not
+      # raise an exception on bad utf8 data, but will replace the bad
+      # characters and render valid JSON.
+      @chef_server_rest ||= Chef::REST.new(
+        Chef::Config[:chef_server_url],
+        Chef::Config[:node_name],
+        Chef::Config[:client_key],
+        validate_utf8: false,
+      )
     end
 
     # Set the name of this Node, or return the current name.
diff --git a/spec/integration/client/client_spec.rb b/spec/integration/client/client_spec.rb
index 69822b8a25..1a97acb561 100644
--- a/spec/integration/client/client_spec.rb
+++ b/spec/integration/client/client_spec.rb
@@ -87,6 +87,11 @@ EOM
       result.error!
     end
 
+    it "should be able to node.save with bad utf8 characters in the node data" do
+      file "cookbooks/x/attributes/default.rb", 'default["badutf8"] = "Elan Ruusam\xE4e"'
+      result = shell_out("#{chef_client} -z -r 'x::default' --disable-config", :cwd => path_to(''))
+      result.error!
+    end
 
     context 'and no config file' do
       it 'should complete with success when cwd is just above cookbooks and paths are not specified' do
diff --git a/spec/unit/client_spec.rb b/spec/unit/client_spec.rb
index affbd0544d..4cfac35cb4 100644
--- a/spec/unit/client_spec.rb
+++ b/spec/unit/client_spec.rb
@@ -279,7 +279,7 @@ describe Chef::Client do
         allow(node).to receive(:data_for_save).and_return(node.for_json)
 
         # --Client#save_updated_node
-        expect(Chef::REST).to receive(:new).with(Chef::Config[:chef_server_url]).and_return(http_node_save)
+        expect(Chef::REST).to receive(:new).with(Chef::Config[:chef_server_url], fqdn, Chef::Config[:client_key], validate_utf8: false).and_return(http_node_save)
         expect(http_node_save).to receive(:put_rest).with("nodes/#{fqdn}", node.for_json).and_return(true)
       end