Merge pull request #4529 from chef/jdm/knife-bootstrap-clientd

Implement knife bootstrap client.d RFC

17 files changed, 311 insertions, 9 deletions

diff --git a/chef-config/lib/chef-config/config.rb b/chef-config/lib/chef-config/config.rb
index 9d18f4f2be..8161cd9ea7 100644
--- a/chef-config/lib/chef-config/config.rb
+++ b/chef-config/lib/chef-config/config.rb
@@ -490,6 +490,16 @@ module ChefConfig
     # HTTP file servers.
     default(:trusted_certs_dir) { PathHelper.join(config_dir, "trusted_certs") }
 
+    # A directory that contains additional configuration scripts to load for chef-client
+    default(:client_d_dir) { PathHelper.join(config_dir, "client.d") }
+
+    # A directory that contains additional configuration scripts to load for solo
+    default(:solo_d_dir) { PathHelper.join(config_dir, "solo.d") }
+
+    # A directory that contains additional configuration scripts to load for
+    # the workstation config
+    default(:config_d_dir) { PathHelper.join(config_dir, "config.d") }
+
     # Where should chef-solo download recipes from?
     default :recipe_url, nil
 
diff --git a/chef-config/lib/chef-config/mixin/dot_d.rb b/chef-config/lib/chef-config/mixin/dot_d.rb
new file mode 100644
index 0000000000..778c25d7f9
--- /dev/null
+++ b/chef-config/lib/chef-config/mixin/dot_d.rb
@@ -0,0 +1,38 @@
+#
+# Copyright:: Copyright 2016, Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "chef-config/path_helper"
+
+module ChefConfig
+  module Mixin
+    module DotD
+      def load_dot_d(path)
+        dot_d_files =
+          begin
+            entries = Array.new
+            entries << Dir.glob(File.join(
+              ChefConfig::PathHelper.escape_glob_dir(path), "*.rb"))
+            entries.flatten.select do |entry|
+              File.file?(entry)
+            end
+          end
+        dot_d_files.sort.map do |conf|
+          apply_config(IO.read(conf), conf)
+        end
+      end
+    end
+  end
+end
diff --git a/chef-config/lib/chef-config/workstation_config_loader.rb b/chef-config/lib/chef-config/workstation_config_loader.rb
index 34ba6d6537..babb78aeb8 100644
--- a/chef-config/lib/chef-config/workstation_config_loader.rb
+++ b/chef-config/lib/chef-config/workstation_config_loader.rb
@@ -21,9 +21,11 @@ require "chef-config/exceptions"
 require "chef-config/logger"
 require "chef-config/path_helper"
 require "chef-config/windows"
+require "chef-config/mixin/dot_d"
 
 module ChefConfig
   class WorkstationConfigLoader
+    include ChefConfig::Mixin::DotD
 
     # Path to a config file requested by user, (e.g., via command line option). Can be nil
     attr_accessor :explicit_config_file
@@ -62,15 +64,17 @@ module ChefConfig
     def load
       # Ignore it if there's no explicit_config_file and can't find one at a
       # default path.
-      return false if config_location.nil?
+      if !config_location.nil?
+        if explicit_config_file && !path_exists?(config_location)
+          raise ChefConfig::ConfigurationError, "Specified config file #{config_location} does not exist"
+        end
 
-      if explicit_config_file && !path_exists?(config_location)
-        raise ChefConfig::ConfigurationError, "Specified config file #{config_location} does not exist"
+        # Have to set Config.config_file b/c other config is derived from it.
+        Config.config_file = config_location
+        apply_config(IO.read(config_location), config_location)
       end
 
-      # Have to set Config.config_file b/c other config is derived from it.
-      Config.config_file = config_location
-      read_config(IO.read(config_location), config_location)
+      load_dot_d(Config[:config_d_dir]) if Config[:config_d_dir]
     end
 
     # (Private API, public for test purposes)
@@ -134,7 +138,7 @@ module ChefConfig
       a
     end
 
-    def read_config(config_content, config_file_path)
+    def apply_config(config_content, config_file_path)
       Config.from_string(config_content, config_file_path)
     rescue SignalException
       raise
diff --git a/chef-config/spec/unit/workstation_config_loader_spec.rb b/chef-config/spec/unit/workstation_config_loader_spec.rb
index df53f87de9..087f249724 100644
--- a/chef-config/spec/unit/workstation_config_loader_spec.rb
+++ b/chef-config/spec/unit/workstation_config_loader_spec.rb
@@ -35,6 +35,12 @@ RSpec.describe ChefConfig::WorkstationConfigLoader do
     end
   end
 
+  before do
+    # We set this to nil so that a dev workstation will
+    # not interfere with the tests.
+    ChefConfig::Config[:config_d_dir] = nil
+  end
+
   # Test methods that do I/O or reference external state which are stubbed out
   # elsewhere.
   describe "external dependencies" do
@@ -215,7 +221,8 @@ RSpec.describe ChefConfig::WorkstationConfigLoader do
 
       it "skips loading" do
         expect(config_loader.config_location).to be(nil)
-        expect(config_loader.load).to be(false)
+        expect(config_loader).not_to receive(:apply_config)
+        config_loader.load
       end
 
     end
@@ -254,7 +261,8 @@ RSpec.describe ChefConfig::WorkstationConfigLoader do
         let(:config_content) { "config_file_evaluated(true)" }
 
         it "loads the config" do
-          expect(config_loader.load).to be(true)
+          expect(config_loader).to receive(:apply_config).and_call_original
+          config_loader.load
           expect(ChefConfig::Config.config_file_evaluated).to be(true)
         end
 
@@ -286,4 +294,73 @@ RSpec.describe ChefConfig::WorkstationConfigLoader do
 
   end
 
+  describe "when loading config.d" do
+    context "when the conf.d directory exists" do
+      let(:config_content) { "" }
+
+      let(:tempdir) { Dir.mktmpdir("chef-workstation-test") }
+
+      let!(:confd_file) do
+        Tempfile.new(["Chef-WorkstationConfigLoader-rspec-test", ".rb"], tempdir).tap do |t|
+          t.print(config_content)
+          t.close
+        end
+      end
+
+      before do
+        ChefConfig::Config[:config_d_dir] = tempdir
+        allow(config_loader).to receive(:path_exists?).with(
+          an_instance_of(String)).and_return(false)
+      end
+
+      after do
+        FileUtils.remove_entry_secure tempdir
+      end
+
+      context "and is valid" do
+        let(:config_content) { "config_d_file_evaluated(true)" }
+
+        it "loads the config" do
+          expect(config_loader).to receive(:apply_config).and_call_original
+          config_loader.load
+          expect(ChefConfig::Config.config_d_file_evaluated).to be(true)
+        end
+      end
+
+      context "and has a syntax error" do
+        let(:config_content) { "{{{{{:{{" }
+
+        it "raises a ConfigurationError" do
+          expect { config_loader.load }.to raise_error(ChefConfig::ConfigurationError)
+        end
+      end
+
+      context "has a non rb file" do
+        let(:sytax_error_content) { "{{{{{:{{" }
+        let(:config_content) { "config_d_file_evaluated(true)" }
+
+        let!(:not_confd_file) do
+          Tempfile.new(["Chef-WorkstationConfigLoader-rspec-test", ".foorb"], tempdir).tap do |t|
+            t.print(sytax_error_content)
+            t.close
+          end
+        end
+
+        it "does not load the non rb file" do
+          expect { config_loader.load }.not_to raise_error
+          expect(ChefConfig::Config.config_d_file_evaluated).to be(true)
+        end
+      end
+    end
+
+    context "when the conf.d directory does not exist" do
+      before do
+        ChefConfig::Config[:config_d_dir] = "/nope/nope/nope/nope/notdoingit"
+      end
+
+      it "does not load anything" do
+        expect(config_loader).not_to receive(:apply_config)
+      end
+    end
+  end
 end
diff --git a/lib/chef/application/client.rb b/lib/chef/application/client.rb
index 9ec553fb8a..8f30037ac7 100644
--- a/lib/chef/application/client.rb
+++ b/lib/chef/application/client.rb
@@ -26,9 +26,11 @@ require "chef/config_fetcher"
 require "chef/handler/error_report"
 require "chef/workstation_config_loader"
 require "chef/mixin/shell_out"
+require "chef-config/mixin/dot_d"
 
 class Chef::Application::Client < Chef::Application
   include Chef::Mixin::ShellOut
+  include ChefConfig::Mixin::DotD
 
   # Mimic self_pipe sleep from Unicorn to capture signals safely
   SELF_PIPE = []
@@ -372,7 +374,12 @@ class Chef::Application::Client < Chef::Application
         config[:config_file] = Chef::Config.platform_specific_path("/etc/chef/client.rb")
       end
     end
+
+    # Load the client.rb configuration
     super
+
+    # Load all config files in client.d
+    load_dot_d(Chef::Config[:client_d_dir]) if Chef::Config[:client_d_dir]
   end
 
   def configure_logging
diff --git a/lib/chef/application/solo.rb b/lib/chef/application/solo.rb
index 3917a080b9..a39546092e 100644
--- a/lib/chef/application/solo.rb
+++ b/lib/chef/application/solo.rb
@@ -27,9 +27,11 @@ require "chef/config_fetcher"
 require "fileutils"
 require "chef/mixin/shell_out"
 require "pathname"
+require "chef-config/mixin/dot_d"
 
 class Chef::Application::Solo < Chef::Application
   include Chef::Mixin::ShellOut
+  include ChefConfig::Mixin::DotD
 
   option :config_file,
     :short => "-c CONFIG",
@@ -207,6 +209,8 @@ class Chef::Application::Solo < Chef::Application
   def reconfigure
     super
 
+    load_dot_d(Chef::Config[:solo_d_dir]) if Chef::Config[:solo_d_dir]
+
     set_specific_recipes
 
     Chef::Config[:solo] = true
diff --git a/lib/chef/knife/bootstrap/templates/chef-full.erb b/lib/chef/knife/bootstrap/templates/chef-full.erb
index 020645c869..6007ff9859 100644
--- a/lib/chef/knife/bootstrap/templates/chef-full.erb
+++ b/lib/chef/knife/bootstrap/templates/chef-full.erb
@@ -226,6 +226,11 @@ cat > /etc/chef/first-boot.json <<EOP
 <%= Chef::JSONCompat.to_json(first_boot) %>
 EOP
 
+<% unless client_d.empty? -%>
+mkdir -p /etc/chef/client.d
+<%= client_d %>
+<% end -%>
+
 echo "Starting the first Chef Client run..."
 
 <%= start_chef %>'
diff --git a/lib/chef/knife/core/bootstrap_context.rb b/lib/chef/knife/core/bootstrap_context.rb
index b05cae688c..6f1c234796 100644
--- a/lib/chef/knife/core/bootstrap_context.rb
+++ b/lib/chef/knife/core/bootstrap_context.rb
@@ -18,6 +18,7 @@
 
 require "chef/run_list"
 require "chef/util/path_helper"
+require "pathname"
 
 class Chef
   class Knife
@@ -52,6 +53,10 @@ class Chef
           end
         end
 
+        def client_d
+          @cliend_d ||= client_d_content
+        end
+
         def encrypted_data_bag_secret
           @secret
         end
@@ -195,6 +200,26 @@ validation_client_name "#{@chef_config[:validation_client_name]}"
           content
         end
 
+        def client_d_content
+          content = ""
+          if @chef_config[:client_d_dir] && File.exist?(@chef_config[:client_d_dir])
+            root = Pathname(@chef_config[:client_d_dir])
+            root.find do |f|
+              relative = f.relative_path_from(root)
+              if f != root
+                file_on_node = "/etc/chef/client.d/#{relative}"
+                if f.directory?
+                  content << "mkdir #{file_on_node}\n"
+                else
+                  content << "cat > #{file_on_node} <<'EOP'\n" +
+                    f.read + "\nEOP\n"
+                end
+              end
+            end
+          end
+          content
+        end
+
       end
     end
   end
diff --git a/spec/data/client.d_00/00-foo.rb b/spec/data/client.d_00/00-foo.rb
new file mode 100644
index 0000000000..44a763aca1
--- /dev/null
+++ b/spec/data/client.d_00/00-foo.rb
@@ -0,0 +1,2 @@
+# 00-foo.rb
+# d6f9b976-289c-4149-baf7-81e6ffecf228
diff --git a/spec/data/client.d_00/01-bar.rb b/spec/data/client.d_00/01-bar.rb
new file mode 100644
index 0000000000..73f91386bc
--- /dev/null
+++ b/spec/data/client.d_00/01-bar.rb
@@ -0,0 +1 @@
+# 01-bar.rb
diff --git a/spec/data/client.d_00/bar b/spec/data/client.d_00/bar
new file mode 100644
index 0000000000..72dca4d5e4
--- /dev/null
+++ b/spec/data/client.d_00/bar
@@ -0,0 +1 @@
+1 / 0
diff --git a/spec/data/client.d_01/foo/bar.rb b/spec/data/client.d_01/foo/bar.rb
new file mode 100644
index 0000000000..72dca4d5e4
--- /dev/null
+++ b/spec/data/client.d_01/foo/bar.rb
@@ -0,0 +1 @@
+1 / 0
diff --git a/spec/data/client.d_02/foo.rb/foo.txt b/spec/data/client.d_02/foo.rb/foo.txt
new file mode 100644
index 0000000000..d724c93bef
--- /dev/null
+++ b/spec/data/client.d_02/foo.rb/foo.txt
@@ -0,0 +1 @@
+# foo.txt
diff --git a/spec/support/shared/unit/application_dot_d.rb b/spec/support/shared/unit/application_dot_d.rb
new file mode 100644
index 0000000000..a8769d6d03
--- /dev/null
+++ b/spec/support/shared/unit/application_dot_d.rb
@@ -0,0 +1,70 @@
+#
+# Copyright:: Copyright 2016, Chef Software, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+shared_examples_for "an application that loads a dot d" do
+  before do
+    Chef::Config[dot_d_config_name] = client_d_dir
+  end
+
+  context "when client_d_dir is set to nil" do
+    let(:client_d_dir) { nil }
+
+    it "does not raise an exception" do
+      expect { app.reconfigure }.not_to raise_error
+    end
+  end
+
+  context "when client_d_dir is set to a directory with configuration" do
+    # We're not going to mock out globbing the directory. We want to
+    # make sure that we are correctly globbing.
+    let(:client_d_dir) { Chef::Util::PathHelper.cleanpath(
+      File.join(File.dirname(__FILE__), "../../../data/client.d_00")) }
+
+    it "loads the configuration in order" do
+      expect(IO).to receive(:read).with(Pathname.new("#{client_d_dir}/00-foo.rb").cleanpath.to_s).and_return("foo 0")
+      expect(IO).to receive(:read).with(Pathname.new("#{client_d_dir}/01-bar.rb").cleanpath.to_s).and_return("bar 0")
+      allow(app).to receive(:apply_config).with(anything(), Chef::Config.platform_specific_path("/etc/chef/client.rb")).and_call_original.ordered
+      expect(app).to receive(:apply_config).with("foo 0", Pathname.new("#{client_d_dir}/00-foo.rb").cleanpath.to_s).and_call_original.ordered
+      expect(app).to receive(:apply_config).with("bar 0", Pathname.new("#{client_d_dir}/01-bar.rb").cleanpath.to_s).and_call_original.ordered
+      app.reconfigure
+    end
+  end
+
+  context "when client_d_dir is set to a directory without configuration" do
+    let(:client_d_dir) { Chef::Util::PathHelper.cleanpath(
+      File.join(File.dirname(__FILE__), "../../data/client.d_01")) }
+
+    # client.d_01 has a nested folder with a rb file that if
+    # executed, would raise an exception. If it is executed,
+    # it means we are loading configs that are deeply nested
+    # inside of client.d. For example, client.d/foo/bar.rb
+    # should not run, but client.d/foo.rb should.
+    it "does not raise an exception" do
+      expect { app.reconfigure }.not_to raise_error
+    end
+  end
+
+  context "when client_d_dir is set to a directory containing a directory named foo.rb" do
+    # foo.rb as a directory should be ignored
+    let(:client_d_dir) { Chef::Util::PathHelper.cleanpath(
+      File.join(File.dirname(__FILE__), "../../data/client.d_02")) }
+
+    it "does not raise an exception" do
+      expect { app.reconfigure }.not_to raise_error
+    end
+  end
+end
diff --git a/spec/unit/application/client_spec.rb b/spec/unit/application/client_spec.rb
index ff6f460c13..97a297ccb5 100644
--- a/spec/unit/application/client_spec.rb
+++ b/spec/unit/application/client_spec.rb
@@ -257,6 +257,10 @@ Enable chef-client interval runs by setting `:client_fork = true` in your config
       expect { app.reconfigure }.to raise_error(Chef::Exceptions::PIDFileLockfileMatch)
     end
   end
+
+  it_behaves_like "an application that loads a dot d" do
+    let(:dot_d_config_name) { :client_d_dir }
+  end
 end
 
 describe Chef::Application::Client, "setup_application" do
diff --git a/spec/unit/application/solo_spec.rb b/spec/unit/application/solo_spec.rb
index 4361a2cd33..85799d73db 100644
--- a/spec/unit/application/solo_spec.rb
+++ b/spec/unit/application/solo_spec.rb
@@ -164,4 +164,7 @@ Enable chef-client interval runs by setting `:client_fork = true` in your config
     end
   end
 
+  it_behaves_like "an application that loads a dot d" do
+    let(:dot_d_config_name) { :solo_d_dir }
+  end
 end
diff --git a/spec/unit/knife/bootstrap_spec.rb b/spec/unit/knife/bootstrap_spec.rb
index 3425b94c76..8ad5c338c3 100644
--- a/spec/unit/knife/bootstrap_spec.rb
+++ b/spec/unit/knife/bootstrap_spec.rb
@@ -458,6 +458,55 @@ describe Chef::Knife::Bootstrap do
     end
   end
 
+  describe "when transferring client.d" do
+
+    let(:rendered_template) do
+      knife.merge_configs
+      knife.render_template
+    end
+
+    before do
+      Chef::Config[:client_d_dir] = client_d_dir
+    end
+
+    context "when client_d_dir is nil" do
+      let(:client_d_dir) { nil }
+
+      it "does not create /etc/chef/client.d" do
+        expect(rendered_template).not_to match(%r{mkdir -p /etc/chef/client\.d})
+      end
+    end
+
+    context "when client_d_dir is set" do
+      let(:client_d_dir) { Chef::Util::PathHelper.cleanpath(
+        File.join(File.dirname(__FILE__), "../../data/client.d_00")) }
+
+      it "creates /etc/chef/client.d" do
+        expect(rendered_template).to match("mkdir -p /etc/chef/client\.d")
+      end
+
+      context "a flat directory structure" do
+        it "creates a file 00-foo.rb" do
+          expect(rendered_template).to match("cat > /etc/chef/client.d/00-foo.rb <<'EOP'")
+          expect(rendered_template).to match("d6f9b976-289c-4149-baf7-81e6ffecf228")
+        end
+        it "creates a file bar" do
+          expect(rendered_template).to match("cat > /etc/chef/client.d/bar <<'EOP'")
+          expect(rendered_template).to match("1 / 0")
+        end
+      end
+
+      context "a nested directory structure" do
+        let(:client_d_dir) { Chef::Util::PathHelper.cleanpath(
+          File.join(File.dirname(__FILE__), "../../data/client.d_01")) }
+        it "creates a file foo/bar.rb" do
+          expect(rendered_template).to match("cat > /etc/chef/client.d/foo/bar.rb <<'EOP'")
+          expect(rendered_template).to match("1 / 0")
+        end
+      end
+    end
+  end
+
   describe "handling policyfile options" do
 
     context "when only policy_name is given" do