Merge pull request #3307 from chef/ssd/rehash

Add knife-rehash command for subcommand location hashing

14 files changed, 887 insertions, 392 deletions

diff --git a/lib/chef/knife.rb b/lib/chef/knife.rb
index 4a93697a1b..46e968827e 100644
--- a/lib/chef/knife.rb
+++ b/lib/chef/knife.rb
@@ -87,6 +87,7 @@ class Chef
     def self.inherited(subclass)
       unless subclass.unnamed?
         subcommands[subclass.snake_case_name] = subclass
+        subcommand_files[subclass.snake_case_name] += [caller[0].split(/:\d+/).first]
       end
     end
 
@@ -121,17 +122,29 @@ class Chef
     end
 
     def self.subcommand_loader
-      @subcommand_loader ||= Knife::SubcommandLoader.new(chef_config_dir)
+      @subcommand_loader ||= Chef::Knife::SubcommandLoader.for_config(chef_config_dir)
     end
 
     def self.load_commands
       @commands_loaded ||= subcommand_loader.load_commands
     end
 
+    def self.guess_category(args)
+      subcommand_loader.guess_category(args)
+    end
+
+    def self.subcommand_class_from(args)
+      subcommand_loader.command_class_from(args) || subcommand_not_found!(args)
+    end
+
     def self.subcommands
       @@subcommands ||= {}
     end
 
+    def self.subcommand_files
+      @@subcommand_files ||= Hash.new([])
+    end
+
     def self.subcommands_by_category
       unless @subcommands_by_category
         @subcommands_by_category = Hash.new { |hash, key| hash[key] = [] }
@@ -142,30 +155,6 @@ class Chef
       @subcommands_by_category
     end
 
-    # Print the list of subcommands knife knows about. If +preferred_category+
-    # is given, only subcommands in that category are shown
-    def self.list_commands(preferred_category=nil)
-      load_commands
-
-      category_desc = preferred_category ? preferred_category + " " : ''
-      msg "Available #{category_desc}subcommands: (for details, knife SUB-COMMAND --help)\n\n"
-
-      if preferred_category && subcommands_by_category.key?(preferred_category)
-        commands_to_show = {preferred_category => subcommands_by_category[preferred_category]}
-      else
-        commands_to_show = subcommands_by_category
-      end
-
-      commands_to_show.sort.each do |category, commands|
-        next if category =~ /deprecated/i
-        msg "** #{category.upcase} COMMANDS **"
-        commands.sort.each do |command|
-          msg subcommands[command].banner if subcommands[command]
-        end
-        msg
-      end
-    end
-
     # Shared with subclasses
     @@chef_config_dir = nil
 
@@ -206,7 +195,6 @@ class Chef
         Chef::Log.level(:debug)
       end
 
-      load_commands
       subcommand_class = subcommand_class_from(args)
       subcommand_class.options = options.merge!(subcommand_class.options)
       subcommand_class.load_deps
@@ -215,34 +203,6 @@ class Chef
       instance.run_with_pretty_exceptions
     end
 
-    def self.guess_category(args)
-      category_words = args.select {|arg| arg =~ /^(([[:alnum:]])[[:alnum:]\_\-]+)$/ }
-      category_words.map! {|w| w.split('-')}.flatten!
-      matching_category = nil
-      while (!matching_category) && (!category_words.empty?)
-        candidate_category = category_words.join(' ')
-        matching_category = candidate_category if subcommands_by_category.key?(candidate_category)
-        matching_category || category_words.pop
-      end
-      matching_category
-    end
-
-    def self.subcommand_class_from(args)
-      command_words = args.select {|arg| arg =~ /^(([[:alnum:]])[[:alnum:]\_\-]+)$/ }
-
-      subcommand_class = nil
-
-      while ( !subcommand_class ) && ( !command_words.empty? )
-        snake_case_class_name = command_words.join("_")
-        unless subcommand_class = subcommands[snake_case_class_name]
-          command_words.pop
-        end
-      end
-      # see if we got the command as e.g., knife node-list
-      subcommand_class ||= subcommands[args.first.gsub('-', '_')]
-      subcommand_class || subcommand_not_found!(args)
-    end
-
     def self.dependency_loaders
       @dependency_loaders ||= []
     end
@@ -265,7 +225,13 @@ class Chef
     # Error out and print usage. probably because the arguments given by the
     # user could not be resolved to a subcommand.
     def self.subcommand_not_found!(args)
-      ui.fatal("Cannot find sub command for: '#{args.join(' ')}'")
+      ui.fatal("Cannot find subcommand for: '#{args.join(' ')}'")
+
+      # Mention rehash when the subcommands cache(plugin_manifest.json) is used
+      if subcommand_loader.is_a?(Chef::Knife::SubcommandLoader::HashedCommandLoader) ||
+         subcommand_loader.is_a?(Chef::Knife::SubcommandLoader::CustomManifestLoader)
+        ui.info("If this is a recently installed plugin, please run 'knife rehash' to update the subcommands cache.")
+      end
 
       if category_commands = guess_category(args)
         list_commands(category_commands)
@@ -280,6 +246,20 @@ class Chef
       exit 10
     end
 
+    def self.list_commands(preferred_category=nil)
+      category_desc = preferred_category ? preferred_category + " " : ''
+      msg "Available #{category_desc}subcommands: (for details, knife SUB-COMMAND --help)\n\n"
+      subcommand_loader.list_commands(preferred_category).sort.each do |category, commands|
+        next if category =~ /deprecated/i
+        msg "** #{category.upcase} COMMANDS **"
+        commands.sort.each do |command|
+          subcommand_loader.load_command(command)
+          msg subcommands[command].banner if subcommands[command]
+        end
+        msg
+      end
+    end
+
     def self.reset_config_path!
       @@chef_config_dir = nil
     end
diff --git a/lib/chef/knife/bootstrap/chef_vault_handler.rb b/lib/chef/knife/bootstrap/chef_vault_handler.rb
index 749f61e6da..f658957499 100644
--- a/lib/chef/knife/bootstrap/chef_vault_handler.rb
+++ b/lib/chef/knife/bootstrap/chef_vault_handler.rb
@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+require 'chef/knife/bootstrap'
 
 class Chef
   class Knife
diff --git a/lib/chef/knife/bootstrap/client_builder.rb b/lib/chef/knife/bootstrap/client_builder.rb
index b9c1d98bec..304b06b8b7 100644
--- a/lib/chef/knife/bootstrap/client_builder.rb
+++ b/lib/chef/knife/bootstrap/client_builder.rb
@@ -20,6 +20,7 @@ require 'chef/node'
 require 'chef/rest'
 require 'chef/api_client/registration'
 require 'chef/api_client'
+require 'chef/knife/bootstrap'
 require 'tmpdir'
 
 class Chef
diff --git a/lib/chef/knife/core/custom_manifest_loader.rb b/lib/chef/knife/core/custom_manifest_loader.rb
new file mode 100644
index 0000000000..c19e749f32
--- /dev/null
+++ b/lib/chef/knife/core/custom_manifest_loader.rb
@@ -0,0 +1,69 @@
+# Copyright:: Copyright (c) 2015 Chef Software, Inc
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/version'
+class Chef
+  class Knife
+    class SubcommandLoader
+
+      #
+      # Load a subcommand from a user-supplied
+      # manifest file
+      #
+      class CustomManifestLoader < Chef::Knife::SubcommandLoader
+        attr_accessor :manifest
+        def initialize(chef_config_dir, plugin_manifest)
+          super(chef_config_dir)
+          @manifest = plugin_manifest
+        end
+
+        # If the user has created a ~/.chef/plugin_manifest.json file, we'll use
+        # that instead of inspecting the on-system gems to find the plugins. The
+        # file format is expected to look like:
+        #
+        #   { "plugins": {
+        #       "knife-ec2": {
+        #         "paths": [
+        #           "/home/alice/.rubymanagerthing/gems/knife-ec2-x.y.z/lib/chef/knife/ec2_server_create.rb",
+        #           "/home/alice/.rubymanagerthing/gems/knife-ec2-x.y.z/lib/chef/knife/ec2_server_delete.rb"
+        #         ]
+        #       }
+        #     }
+        #   }
+        #
+        # Extraneous content in this file is ignored. This is intentional so that we
+        # can adapt the file format for potential behavior changes to knife in
+        # the future.
+        def find_subcommands_via_manifest
+          # Format of subcommand_files is "relative_path" (something you can
+          # Kernel.require()) => full_path. The relative path isn't used
+          # currently, so we just map full_path => full_path.
+          subcommand_files = {}
+          manifest["plugins"].each do |plugin_name, plugin_manifest|
+            plugin_manifest["paths"].each do |cmd_path|
+              subcommand_files[cmd_path] = cmd_path
+            end
+          end
+          subcommand_files.merge(find_subcommands_via_dirglob)
+        end
+
+        def subcommand_files
+          subcommand_files ||= (find_subcommands_via_manifest.values + site_subcommands).flatten.uniq
+        end
+      end
+    end
+  end
+end
diff --git a/lib/chef/knife/core/gem_glob_loader.rb b/lib/chef/knife/core/gem_glob_loader.rb
new file mode 100644
index 0000000000..d09131aacb
--- /dev/null
+++ b/lib/chef/knife/core/gem_glob_loader.rb
@@ -0,0 +1,138 @@
+# Author:: Christopher Brown (<cb@opscode.com>)
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Copyright:: Copyright (c) 2009-2015 Chef Software, Inc
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/version'
+require 'chef/util/path_helper'
+class Chef
+  class Knife
+    class SubcommandLoader
+      class GemGlobLoader < Chef::Knife::SubcommandLoader
+        MATCHES_CHEF_GEM = %r{/chef-[\d]+\.[\d]+\.[\d]+}.freeze
+        MATCHES_THIS_CHEF_GEM = %r{/chef-#{Chef::VERSION}(-\w+)?(-\w+)?/}.freeze
+
+        def subcommand_files
+          @subcommand_files ||= (gem_and_builtin_subcommands.values + site_subcommands).flatten.uniq
+        end
+
+        # Returns a Hash of paths to knife commands built-in to chef, or installed via gem.
+        # If rubygems is not installed, falls back to globbing the knife directory.
+        # The Hash is of the form {"relative/path" => "/absolute/path"}
+        #--
+        # Note: the "right" way to load the plugins is to require the relative path, i.e.,
+        #   require 'chef/knife/command'
+        # but we're getting frustrated by bugs at every turn, and it's slow besides. So
+        # subcommand loader has been modified to load the plugins by using Kernel.load
+        # with the absolute path.
+        def gem_and_builtin_subcommands
+          require 'rubygems'
+          find_subcommands_via_rubygems
+        rescue LoadError
+          find_subcommands_via_dirglob
+        end
+
+        def find_subcommands_via_dirglob
+          # The "require paths" of the core knife subcommands bundled with chef
+          files = Dir[File.join(Chef::Util::PathHelper.escape_glob(File.expand_path('../../../knife', __FILE__)), '*.rb')]
+          subcommand_files = {}
+          files.each do |knife_file|
+            rel_path = knife_file[/#{CHEF_ROOT}#{Regexp.escape(File::SEPARATOR)}(.*)\.rb/,1]
+            subcommand_files[rel_path] = knife_file
+          end
+          subcommand_files
+        end
+
+        def find_subcommands_via_rubygems
+          files = find_files_latest_gems 'chef/knife/*.rb'
+          subcommand_files = {}
+          files.each do |file|
+            rel_path = file[/(#{Regexp.escape File.join('chef', 'knife', '')}.*)\.rb/, 1]
+
+            # When not installed as a gem (ChefDK/appbundler in particular), AND
+            # a different version of Chef is installed via gems, `files` will
+            # include some files from the 'other' Chef install. If this contains
+            # a knife command that doesn't exist in this version of Chef, we will
+            # get a LoadError later when we try to require it.
+            next if from_different_chef_version?(file)
+
+            subcommand_files[rel_path] = file
+          end
+
+          subcommand_files.merge(find_subcommands_via_dirglob)
+        end
+
+        private
+
+        def find_files_latest_gems(glob, check_load_path=true)
+          files = []
+
+          if check_load_path
+            files = $LOAD_PATH.map { |load_path|
+              Dir["#{File.expand_path glob, Chef::Util::PathHelper.escape_glob(load_path)}#{Gem.suffix_pattern}"]
+            }.flatten.select { |file| File.file? file.untaint }
+          end
+
+          gem_files = latest_gem_specs.map do |spec|
+            # Gem::Specification#matches_for_glob wasn't added until RubyGems 1.8
+            if spec.respond_to? :matches_for_glob
+              spec.matches_for_glob("#{glob}#{Gem.suffix_pattern}")
+            else
+              check_spec_for_glob(spec, glob)
+            end
+          end.flatten
+
+          files.concat gem_files
+          files.uniq! if check_load_path
+
+          return files
+        end
+
+        def latest_gem_specs
+          @latest_gem_specs ||= if Gem::Specification.respond_to? :latest_specs
+                                  Gem::Specification.latest_specs(true)  # find prerelease gems
+                                else
+                                  Gem.source_index.latest_specs(true)
+                                end
+        end
+
+        def check_spec_for_glob(spec, glob)
+          dirs = if spec.require_paths.size > 1 then
+                   "{#{spec.require_paths.join(',')}}"
+                 else
+                   spec.require_paths.first
+                 end
+
+          glob = File.join(Chef::Util::PathHelper.escape_glob(spec.full_gem_path, dirs), glob)
+
+          Dir[glob].map { |f| f.untaint }
+        end
+
+        def from_different_chef_version?(path)
+          matches_any_chef_gem?(path) && !matches_this_chef_gem?(path)
+        end
+
+        def matches_any_chef_gem?(path)
+          path =~ MATCHES_CHEF_GEM
+        end
+
+        def matches_this_chef_gem?(path)
+          path =~ MATCHES_THIS_CHEF_GEM
+        end
+      end
+    end
+  end
+end
diff --git a/lib/chef/knife/core/hashed_command_loader.rb b/lib/chef/knife/core/hashed_command_loader.rb
new file mode 100644
index 0000000000..6eb3635726
--- /dev/null
+++ b/lib/chef/knife/core/hashed_command_loader.rb
@@ -0,0 +1,80 @@
+# Author:: Steven Danna (<steve@chef.io>)
+# Copyright:: Copyright (c) 2015 Chef Software, Inc
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/version'
+class Chef
+  class Knife
+    class SubcommandLoader
+      #
+      # Load a subcommand from a pre-computed path
+      # for the given command.
+      #
+      class HashedCommandLoader < Chef::Knife::SubcommandLoader
+        KEY = '_autogenerated_command_paths'
+
+        attr_accessor :manifest
+        def initialize(chef_config_dir, plugin_manifest)
+          super(chef_config_dir)
+          @manifest = plugin_manifest
+        end
+
+        def guess_category(args)
+          category_words = positional_arguments(args)
+          category_words.map! { |w| w.split('-') }.flatten!
+          find_longest_key(manifest[KEY]["plugins_by_category"], category_words, ' ')
+        end
+
+        def list_commands(pref_category=nil)
+          if pref_category || manifest[KEY]["plugins_by_category"].key?(pref_category)
+            { pref_category => manifest[KEY]["plugins_by_category"][pref_category] }
+          else
+            manifest[KEY]["plugins_by_category"]
+          end
+        end
+
+        def subcommand_files
+          manifest[KEY]["plugins_paths"].values.flatten
+        end
+
+        def load_command(args)
+          paths = manifest[KEY]["plugins_paths"][subcommand_for_args(args)]
+          if paths.nil? || paths.empty? || (! paths.is_a? Array)
+            false
+          else
+            paths.each do |sc|
+              if File.exists?(sc)
+                Kernel.load sc
+              else
+                Chef::Log.error "The file #{sc} is missing for subcommand '#{subcommand_for_args(args)}'. Please rehash to update the subcommands cache."
+                return false
+              end
+            end
+            true
+          end
+        end
+
+        def subcommand_for_args(args)
+          if manifest[KEY]["plugins_paths"].key?(args)
+            args
+          else
+            find_longest_key(manifest[KEY]["plugins_paths"], args, "_")
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/chef/knife/core/object_loader.rb b/lib/chef/knife/core/object_loader.rb
index 698b09ac84..97ca381471 100644
--- a/lib/chef/knife/core/object_loader.rb
+++ b/lib/chef/knife/core/object_loader.rb
@@ -18,6 +18,7 @@
 
 require 'ffi_yajl'
 require 'chef/util/path_helper'
+require 'chef/data_bag_item'
 
 class Chef
   class Knife
diff --git a/lib/chef/knife/core/subcommand_loader.rb b/lib/chef/knife/core/subcommand_loader.rb
index a8705c724f..1d359ffd53 100644
--- a/lib/chef/knife/core/subcommand_loader.rb
+++ b/lib/chef/knife/core/subcommand_loader.rb
@@ -18,19 +18,68 @@
 
 require 'chef/version'
 require 'chef/util/path_helper'
+require 'chef/knife/core/gem_glob_loader'
+require 'chef/knife/core/hashed_command_loader'
+require 'chef/knife/core/custom_manifest_loader'
+
 class Chef
   class Knife
+    #
+    # Public Methods of a Subcommand Loader
+    #
+    # load_commands            - loads all available subcommands
+    # load_command(args)       - loads subcommands for the given args
+    # list_commands(args)      - lists all available subcommands,
+    #                            optionally filtering by category
+    # subcommand_files         - returns an array of all subcommand files
+    #                            that could be loaded
+    # commnad_class_from(args) - returns the subcommand class for the
+    #                            user-requested command
+    #
     class SubcommandLoader
-
-      MATCHES_CHEF_GEM = %r{/chef-[\d]+\.[\d]+\.[\d]+}.freeze
-      MATCHES_THIS_CHEF_GEM = %r{/chef-#{Chef::VERSION}(-\w+)?(-\w+)?/}.freeze
-
       attr_reader :chef_config_dir
       attr_reader :env
 
-      def initialize(chef_config_dir, env=nil)
+      # A small factory method.  Eventually, this is the only place
+      # where SubcommandLoader should know about its subclasses, but
+      # to maintain backwards compatibility many of the instance
+      # methods in this base class contain default implementations
+      # of the functions sub classes should otherwise provide
+      # or directly instantiate the appropriate subclass
+      def self.for_config(chef_config_dir)
+        if autogenerated_manifest?
+          Chef::Log.debug("Using autogenerated hashed command manifest #{plugin_manifest_path}")
+          Knife::SubcommandLoader::HashedCommandLoader.new(chef_config_dir, plugin_manifest)
+        elsif custom_manifest?
+          Chef::Log.deprecation("Using custom manifest #{plugin_manifest_path} is deprecated.  Please use a `knife rehash` autogenerated manifest instead.")
+          Knife::SubcommandLoader::CustomManifestLoader.new(chef_config_dir, plugin_manifest)
+        else
+          Knife::SubcommandLoader::GemGlobLoader.new(chef_config_dir)
+        end
+      end
+
+      def self.plugin_manifest?
+        plugin_manifest_path && File.exist?(plugin_manifest_path)
+      end
+
+      def self.autogenerated_manifest?
+        plugin_manifest? && plugin_manifest.key?(HashedCommandLoader::KEY)
+      end
+
+      def self.custom_manifest?
+        plugin_manifest? && plugin_manifest.key?("plugins")
+      end
+
+      def self.plugin_manifest
+        Chef::JSONCompat.from_json(File.read(plugin_manifest_path))
+      end
+
+      def self.plugin_manifest_path
+        Chef::Util::PathHelper.home('.chef', 'plugin_manifest.json')
+      end
+
+      def initialize(chef_config_dir, env = nil)
         @chef_config_dir = chef_config_dir
-        @forced_activate = {}
 
         # Deprecated and un-used instance variable.
         @env = env
@@ -41,82 +90,48 @@ class Chef
 
       # Load all the sub-commands
       def load_commands
+        return true if @loaded
         subcommand_files.each { |subcommand| Kernel.load subcommand }
-        true
+        @loaded = true
       end
 
-      # Returns an Array of paths to knife commands located in chef_config_dir/plugins/knife/
-      # and ~/.chef/plugins/knife/
-      def site_subcommands
-        user_specific_files = []
-
-        if chef_config_dir
-          user_specific_files.concat Dir.glob(File.expand_path("plugins/knife/*.rb", Chef::Util::PathHelper.escape_glob(chef_config_dir)))
-        end
-
-        # finally search ~/.chef/plugins/knife/*.rb
-        Chef::Util::PathHelper.home('.chef', 'plugins', 'knife') do |p|
-          user_specific_files.concat Dir.glob(File.join(Chef::Util::PathHelper.escape_glob(p), '*.rb'))
-        end
+      def force_load
+        @loaded=false
+        load_commands
+      end
 
-        user_specific_files
+      def load_command(_command_args)
+        load_commands
       end
 
-      # Returns a Hash of paths to knife commands built-in to chef, or installed via gem.
-      # If rubygems is not installed, falls back to globbing the knife directory.
-      # The Hash is of the form {"relative/path" => "/absolute/path"}
-      #--
-      # Note: the "right" way to load the plugins is to require the relative path, i.e.,
-      #   require 'chef/knife/command'
-      # but we're getting frustrated by bugs at every turn, and it's slow besides. So
-      # subcommand loader has been modified to load the plugins by using Kernel.load
-      # with the absolute path.
-      def gem_and_builtin_subcommands
-        if have_plugin_manifest?
-          find_subcommands_via_manifest
+      def list_commands(pref_cat = nil)
+        load_commands
+        if pref_cat && Chef::Knife.subcommands_by_category.key?(pref_cat)
+          { pref_cat => Chef::Knife.subcommands_by_category[pref_cat] }
         else
-          # search all gems for chef/knife/*.rb
-          require 'rubygems'
-          find_subcommands_via_rubygems
+          Chef::Knife.subcommands_by_category
         end
-      rescue LoadError
-        find_subcommands_via_dirglob
       end
 
-      def subcommand_files
-        @subcommand_files ||= (gem_and_builtin_subcommands.values + site_subcommands).flatten.uniq
+      def command_class_from(args)
+        cmd_words = positional_arguments(args)
+        load_command(cmd_words)
+        result = Chef::Knife.subcommands[find_longest_key(Chef::Knife.subcommands,
+                                                          cmd_words, '_')]
+        result || Chef::Knife.subcommands[args.first.gsub('-', '_')]
       end
 
-      # If the user has created a ~/.chef/plugin_manifest.json file, we'll use
-      # that instead of inspecting the on-system gems to find the plugins. The
-      # file format is expected to look like:
-      #
-      #   { "plugins": {
-      #       "knife-ec2": {
-      #         "paths": [
-      #           "/home/alice/.rubymanagerthing/gems/knife-ec2-x.y.z/lib/chef/knife/ec2_server_create.rb",
-      #           "/home/alice/.rubymanagerthing/gems/knife-ec2-x.y.z/lib/chef/knife/ec2_server_delete.rb"
-      #         ]
-      #       }
-      #     }
-      #   }
-      #
-      # Extraneous content in this file is ignored. This intentional so that we
-      # can adapt the file format for potential behavior changes to knife in
-      # the future.
-      def find_subcommands_via_manifest
-        # Format of subcommand_files is "relative_path" (something you can
-        # Kernel.require()) => full_path. The relative path isn't used
-        # currently, so we just map full_path => full_path.
-        subcommand_files = {}
-        plugin_manifest["plugins"].each do |plugin_name, plugin_manifest|
-          plugin_manifest["paths"].each do |cmd_path|
-            subcommand_files[cmd_path] = cmd_path
-          end
-        end
-        subcommand_files.merge(find_subcommands_via_dirglob)
+      def guess_category(args)
+        category_words = positional_arguments(args)
+        category_words.map! { |w| w.split('-') }.flatten!
+        find_longest_key(Chef::Knife.subcommands_by_category,
+                         category_words, ' ')
       end
 
+
+      #
+      # This is shared between the custom_manifest_loader and the gem_glob_loader
+      #
       def find_subcommands_via_dirglob
         # The "require paths" of the core knife subcommands bundled with chef
         files = Dir[File.join(Chef::Util::PathHelper.escape_glob(File.expand_path('../../../knife', __FILE__)), '*.rb')]
@@ -128,95 +143,65 @@ class Chef
         subcommand_files
       end
 
-      def find_subcommands_via_rubygems
-        files = find_files_latest_gems 'chef/knife/*.rb'
-        subcommand_files = {}
-        files.each do |file|
-          rel_path = file[/(#{Regexp.escape File.join('chef', 'knife', '')}.*)\.rb/, 1]
-
-          # When not installed as a gem (ChefDK/appbundler in particular), AND
-          # a different version of Chef is installed via gems, `files` will
-          # include some files from the 'other' Chef install. If this contains
-          # a knife command that doesn't exist in this version of Chef, we will
-          # get a LoadError later when we try to require it.
-          next if from_different_chef_version?(file)
-
-          subcommand_files[rel_path] = file
-        end
-
-        subcommand_files.merge(find_subcommands_via_dirglob)
-      end
-
-      def have_plugin_manifest?
-        plugin_manifest_path && File.exist?(plugin_manifest_path)
-      end
-
-      def plugin_manifest
-        Chef::JSONCompat.from_json(File.read(plugin_manifest_path))
-      end
-
-      def plugin_manifest_path
-        Chef::Util::PathHelper.home('.chef', 'plugin_manifest.json')
+      #
+      # Subclassses should define this themselves.  Eventually, this will raise a
+      # NotImplemented error, but for now, we mimic the behavior the user was likely
+      # to get in the past.
+      #
+      def subcommand_files
+        Chef::Log.deprecation "Using Chef::Knife::SubcommandLoader directly is deprecated.
+Please use Chef::Knife::SubcommandLoader.for_config(chef_config_dir, env)"
+        @subcommand_files ||= if Chef::Knife::SubcommandLoader.plugin_manifest?
+                                Chef::Knife::SubcommandLoader::CustomManifestLoader.new(chef_config_dir, env).subcommand_files
+                              else
+                                Chef::Knife::SubcommandLoader::GemGlobLoader.new(chef_config_dir, env).subcommand_files
+                              end
       end
 
-      private
-
-      def find_files_latest_gems(glob, check_load_path=true)
-        files = []
-
-        if check_load_path
-          files = $LOAD_PATH.map { |load_path|
-            Dir["#{File.expand_path glob, Chef::Util::PathHelper.escape_glob(load_path)}#{Gem.suffix_pattern}"]
-          }.flatten.select { |file| File.file? file.untaint }
-        end
-
-        gem_files = latest_gem_specs.map do |spec|
-          # Gem::Specification#matches_for_glob wasn't added until RubyGems 1.8
-          if spec.respond_to? :matches_for_glob
-            spec.matches_for_glob("#{glob}#{Gem.suffix_pattern}")
+      #
+      # Utility function for finding an element in a hash given an array
+      # of words and a separator.  We find the the longest key in the
+      # hash composed of the given words joined by the separator.
+      #
+      def find_longest_key(hash, words, sep = '_')
+        match = nil
+        until match || words.empty?
+          candidate = words.join(sep)
+          if hash.key?(candidate)
+            match = candidate
           else
-            check_spec_for_glob(spec, glob)
+            words.pop
           end
-        end.flatten
-
-        files.concat gem_files
-        files.uniq! if check_load_path
-
-        return files
-      end
-
-      def latest_gem_specs
-        @latest_gem_specs ||= if Gem::Specification.respond_to? :latest_specs
-          Gem::Specification.latest_specs(true)  # find prerelease gems
-        else
-          Gem.source_index.latest_specs(true)
         end
+        match
       end
 
-      def check_spec_for_glob(spec, glob)
-        dirs = if spec.require_paths.size > 1 then
-          "{#{spec.require_paths.join(',')}}"
-        else
-          spec.require_paths.first
-        end
-
-        glob = File.join(Chef::Util::PathHelper.escape_glob(spec.full_gem_path, dirs), glob)
-
-        Dir[glob].map { |f| f.untaint }
+      #
+      # The positional arguments from the argument list provided by the
+      # users. Used to search for subcommands and categories.
+      #
+      # @return [Array<String>]
+      #
+      def positional_arguments(args)
+        args.select { |arg| arg =~ /^(([[:alnum:]])[[:alnum:]\_\-]+)$/ }
       end
 
-      def from_different_chef_version?(path)
-        matches_any_chef_gem?(path) && !matches_this_chef_gem?(path)
-      end
+      # Returns an Array of paths to knife commands located in
+      # chef_config_dir/plugins/knife/ and ~/.chef/plugins/knife/
+      def site_subcommands
+        user_specific_files = []
 
-      def matches_any_chef_gem?(path)
-        path =~ MATCHES_CHEF_GEM
-      end
+        if chef_config_dir
+          user_specific_files.concat Dir.glob(File.expand_path("plugins/knife/*.rb", Chef::Util::PathHelper.escape_glob(chef_config_dir)))
+        end
 
-      def matches_this_chef_gem?(path)
-        path =~ MATCHES_THIS_CHEF_GEM
-      end
+        # finally search ~/.chef/plugins/knife/*.rb
+        Chef::Util::PathHelper.home('.chef', 'plugins', 'knife') do |p|
+          user_specific_files.concat Dir.glob(File.join(Chef::Util::PathHelper.escape_glob(p), '*.rb'))
+        end
 
+        user_specific_files
+      end
     end
   end
 end
diff --git a/lib/chef/knife/null.rb b/lib/chef/knife/null.rb
new file mode 100644
index 0000000000..0b5058e8ea
--- /dev/null
+++ b/lib/chef/knife/null.rb
@@ -0,0 +1,10 @@
+class Chef
+  class Knife
+    class Null < Chef::Knife
+      banner "knife null"
+
+      def run
+      end
+    end
+  end
+end
diff --git a/lib/chef/knife/rehash.rb b/lib/chef/knife/rehash.rb
new file mode 100644
index 0000000000..6f1fd91911
--- /dev/null
+++ b/lib/chef/knife/rehash.rb
@@ -0,0 +1,62 @@
+#
+# Author:: Steven Danna <steve@chef.io>
+# Copyright:: Copyright (c) 2015 Chef Software, Inc
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/knife'
+require 'chef/knife/core/subcommand_loader'
+
+class Chef
+  class Knife
+    class Rehash <  Chef::Knife
+      banner "knife rehash"
+
+      def run
+        if ! Chef::Knife::SubcommandLoader.autogenerated_manifest?
+          ui.msg "Using knife-rehash will speed up knife's load time by caching the location of subcommands on disk."
+          ui.msg "However, you will need to update the cache by running `knife rehash` anytime you install a new knife plugin."
+        else
+          reload_plugins
+        end
+        write_hash(generate_hash)
+      end
+
+      def reload_plugins
+        Chef::Knife::SubcommandLoader::GemGlobLoader.new(@@chef_config_dir).load_commands
+      end
+
+      def generate_hash
+        output = if Chef::Knife::SubcommandLoader.plugin_manifest?
+                   Chef::Knife::SubcommandLoader.plugin_manifest
+                 else
+                   { Chef::Knife::SubcommandLoader::HashedCommandLoader::KEY => {}}
+                 end
+        output[Chef::Knife::SubcommandLoader::HashedCommandLoader::KEY]['plugins_paths'] = Chef::Knife.subcommand_files
+        output[Chef::Knife::SubcommandLoader::HashedCommandLoader::KEY]['plugins_by_category'] = Chef::Knife.subcommands_by_category
+        output
+      end
+
+      def write_hash(data)
+        plugin_manifest_dir = File.expand_path('..', Chef::Knife::SubcommandLoader.plugin_manifest_path)
+        FileUtils.mkdir_p(plugin_manifest_dir) unless File.directory?(plugin_manifest_dir)
+        File.open(Chef::Knife::SubcommandLoader.plugin_manifest_path, 'w') do |f|
+          f.write(Chef::JSONCompat.to_json_pretty(data))
+          ui.msg "Knife subcommands are cached in #{Chef::Knife::SubcommandLoader.plugin_manifest_path}. Delete this file to disable the caching."
+        end
+      end
+    end
+  end
+end
diff --git a/spec/unit/knife/core/custom_manifest_loader_spec.rb b/spec/unit/knife/core/custom_manifest_loader_spec.rb
new file mode 100644
index 0000000000..1edbedd3c8
--- /dev/null
+++ b/spec/unit/knife/core/custom_manifest_loader_spec.rb
@@ -0,0 +1,41 @@
+#
+# Copyright:: Copyright (c) 2015 Chef Software, Inc
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'spec_helper'
+
+describe Chef::Knife::SubcommandLoader::CustomManifestLoader do
+  let(:ec2_server_create_plugin) { "/usr/lib/ruby/gems/knife-ec2-0.5.12/lib/chef/knife/ec2_server_create.rb" }
+  let(:manifest_content) do
+    { "plugins" => {
+        "knife-ec2" => {
+          "paths" => [
+                      ec2_server_create_plugin
+                     ]
+        }
+      }
+    }
+  end
+  let(:loader) do
+    Chef::Knife::SubcommandLoader::CustomManifestLoader.new(File.join(CHEF_SPEC_DATA, 'knife-site-subcommands'),
+                                                            manifest_content)
+  end
+
+  it "uses paths from the manifest instead of searching gems" do
+    expect(Gem::Specification).not_to receive(:latest_specs).and_call_original
+    expect(loader.subcommand_files).to include(ec2_server_create_plugin)
+  end
+end
diff --git a/spec/unit/knife/core/gem_glob_loader_spec.rb b/spec/unit/knife/core/gem_glob_loader_spec.rb
new file mode 100644
index 0000000000..465eea2656
--- /dev/null
+++ b/spec/unit/knife/core/gem_glob_loader_spec.rb
@@ -0,0 +1,210 @@
+#
+# Copyright:: Copyright (c) 2015 Chef Software, Inc
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'spec_helper'
+
+describe Chef::Knife::SubcommandLoader::GemGlobLoader do
+   let(:loader) { Chef::Knife::SubcommandLoader::GemGlobLoader.new(File.join(CHEF_SPEC_DATA, 'knife-site-subcommands')) }
+   let(:home) { File.join(CHEF_SPEC_DATA, 'knife-home') }
+   let(:plugin_dir) { File.join(home, '.chef', 'plugins', 'knife') }
+
+   before do
+     allow(ChefConfig).to receive(:windows?) { false }
+     Chef::Util::PathHelper.class_variable_set(:@@home_dir, home)
+   end
+
+  after do
+    Chef::Util::PathHelper.class_variable_set(:@@home_dir, nil)
+  end
+
+  it "builds a list of the core subcommand file require paths" do
+    expect(loader.subcommand_files).not_to be_empty
+    loader.subcommand_files.each do |require_path|
+      expect(require_path).to match(/chef\/knife\/.*|plugins\/knife\/.*/)
+    end
+  end
+
+  it "finds files installed via rubygems" do
+    expect(loader.find_subcommands_via_rubygems).to include('chef/knife/node_create')
+    loader.find_subcommands_via_rubygems.each {|rel_path, abs_path| expect(abs_path).to match(%r[chef/knife/.+])}
+  end
+
+  it "finds files from latest version of installed gems" do
+    gems = [ double('knife-ec2-0.5.12') ]
+    gem_files = [
+      '/usr/lib/ruby/gems/knife-ec2-0.5.12/lib/chef/knife/ec2_base.rb',
+      '/usr/lib/ruby/gems/knife-ec2-0.5.12/lib/chef/knife/ec2_otherstuff.rb'
+    ]
+    expect($LOAD_PATH).to receive(:map).and_return([])
+    if Gem::Specification.respond_to? :latest_specs
+      expect(Gem::Specification).to receive(:latest_specs).with(true).and_return(gems)
+      expect(gems[0]).to receive(:matches_for_glob).with(/chef\/knife\/\*\.rb\{(.*),\.rb,(.*)\}/).and_return(gem_files)
+    else
+      expect(Gem.source_index).to receive(:latest_specs).with(true).and_return(gems)
+      expect(gems[0]).to receive(:require_paths).twice.and_return(['lib'])
+      expect(gems[0]).to receive(:full_gem_path).and_return('/usr/lib/ruby/gems/knife-ec2-0.5.12')
+      expect(Dir).to receive(:[]).with('/usr/lib/ruby/gems/knife-ec2-0.5.12/lib/chef/knife/*.rb').and_return(gem_files)
+    end
+    expect(loader).to receive(:find_subcommands_via_dirglob).and_return({})
+    expect(loader.subcommand_files.select { |file| file =~ /knife-ec2/ }.sort).to eq(gem_files)
+  end
+
+  it "finds files using a dirglob when rubygems is not available" do
+    expect(loader.find_subcommands_via_dirglob).to include('chef/knife/node_create')
+    loader.find_subcommands_via_dirglob.each {|rel_path, abs_path| expect(abs_path).to match(%r[chef/knife/.+])}
+  end
+
+  it "finds user-specific subcommands in the user's ~/.chef directory" do
+    expected_command = File.join(home, '.chef', 'plugins', 'knife', 'example_home_subcommand.rb')
+    expect(loader.site_subcommands).to include(expected_command)
+  end
+
+  it "finds repo specific subcommands by searching for a .chef directory" do
+    expected_command = File.join(CHEF_SPEC_DATA, 'knife-site-subcommands', 'plugins', 'knife', 'example_subcommand.rb')
+    expect(loader.site_subcommands).to include(expected_command)
+  end
+
+  # https://github.com/opscode/chef-dk/issues/227
+  #
+  # `knife` in ChefDK isn't from a gem install, it's directly run from a clone
+  # of the source, but there can be one or more versions of chef also installed
+  # as a gem. If the gem install contains a command that doesn't exist in the
+  # source tree of the "primary" chef install, it can be loaded and cause an
+  # error. We also want to ensure that we only load builtin commands from the
+  # "primary" chef install.
+  context "when a different version of chef is also installed as a gem" do
+
+    let(:all_found_commands) do
+      [
+        "/opt/chefdk/embedded/apps/chef/lib/chef/knife/bootstrap.rb",
+        "/opt/chefdk/embedded/apps/chef/lib/chef/knife/client_bulk_delete.rb",
+        "/opt/chefdk/embedded/apps/chef/lib/chef/knife/client_create.rb",
+
+        # We use the fake version 1.0.0 because that version doesn't exist,
+        # which ensures it won't ever equal "chef-#{Chef::VERSION}"
+        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-1.0.0/lib/chef/knife/bootstrap.rb",
+        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-1.0.0/lib/chef/knife/client_bulk_delete.rb",
+        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-1.0.0/lib/chef/knife/client_create.rb",
+
+        # Test that we don't accept a version number that is different only in
+        # trailing characters, e.g. we are running Chef 12.0.0 but there is a
+        # Chef 12.0.0.rc.0 gem also:
+        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-#{Chef::VERSION}.rc.0/lib/chef/knife/thing.rb",
+
+        # Test that we ignore the platform suffix when checking for different
+        # gem versions.
+        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-#{Chef::VERSION}-x86-mingw32/lib/chef/knife/valid.rb",
+        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-#{Chef::VERSION}-i386-mingw64/lib/chef/knife/valid-too.rb",
+        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-#{Chef::VERSION}-mswin32/lib/chef/knife/also-valid.rb",
+        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-#{Chef::VERSION}-universal-mingw32/lib/chef/knife/universal-is-valid.rb",
+        # ...but don't ignore the .rc / .dev parts in the case when we have
+        # platform suffixes
+        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-#{Chef::VERSION}.rc.0-x86-mingw32/lib/chef/knife/invalid.rb",
+        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-#{Chef::VERSION}.dev-mswin32/lib/chef/knife/invalid-too.rb",
+        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-#{Chef::VERSION}.dev.0-x86-mingw64/lib/chef/knife/still-invalid.rb",
+
+        # This command is "extra" compared to what's in the embedded/apps/chef install:
+        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-1.0.0/lib/chef/knife/data_bag_secret_options.rb",
+        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-vault-2.2.4/lib/chef/knife/decrypt.rb",
+        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/knife-spork-1.4.1/lib/chef/knife/spork-bump.rb",
+
+        # These are fake commands that have names designed to test that the
+        # regex is strict enough
+        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-foo-#{Chef::VERSION}/lib/chef/knife/chef-foo.rb",
+        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/foo-chef-#{Chef::VERSION}/lib/chef/knife/foo-chef.rb",
+
+        # In a real scenario, we'd use rubygems APIs to only select the most
+        # recent gem, but for this test we want to check that we're doing the
+        # right thing both when the plugin version matches and does not match
+        # the current chef version. Looking at
+        # `SubcommandLoader::MATCHES_THIS_CHEF_GEM` and
+        # `SubcommandLoader::MATCHES_CHEF_GEM` should make it clear why we want
+        # to test these two cases.
+        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-bar-1.0.0/lib/chef/knife/chef-bar.rb",
+        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/bar-chef-1.0.0/lib/chef/knife/bar-chef.rb"
+      ]
+    end
+
+    let(:expected_valid_commands) do
+      [
+        "/opt/chefdk/embedded/apps/chef/lib/chef/knife/bootstrap.rb",
+        "/opt/chefdk/embedded/apps/chef/lib/chef/knife/client_bulk_delete.rb",
+        "/opt/chefdk/embedded/apps/chef/lib/chef/knife/client_create.rb",
+        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-#{Chef::VERSION}-x86-mingw32/lib/chef/knife/valid.rb",
+        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-#{Chef::VERSION}-i386-mingw64/lib/chef/knife/valid-too.rb",
+        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-#{Chef::VERSION}-mswin32/lib/chef/knife/also-valid.rb",
+        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-#{Chef::VERSION}-universal-mingw32/lib/chef/knife/universal-is-valid.rb",
+        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-vault-2.2.4/lib/chef/knife/decrypt.rb",
+        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/knife-spork-1.4.1/lib/chef/knife/spork-bump.rb",
+        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-foo-#{Chef::VERSION}/lib/chef/knife/chef-foo.rb",
+        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/foo-chef-#{Chef::VERSION}/lib/chef/knife/foo-chef.rb",
+        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-bar-1.0.0/lib/chef/knife/chef-bar.rb",
+        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/bar-chef-1.0.0/lib/chef/knife/bar-chef.rb"
+      ]
+    end
+
+    before do
+      expect(loader).to receive(:find_files_latest_gems).with("chef/knife/*.rb").and_return(all_found_commands)
+      expect(loader).to receive(:find_subcommands_via_dirglob).and_return({})
+    end
+
+    it "ignores commands from the non-matching gem install" do
+      expect(loader.find_subcommands_via_rubygems.values).to eq(expected_valid_commands)
+    end
+
+  end
+
+  describe "finding 3rd party plugins" do
+    let(:env_home) { "/home/alice" }
+    let(:manifest_path) { env_home + "/.chef/plugin_manifest.json" }
+
+    before do
+      env_dup = ENV.to_hash
+      allow(ENV).to receive(:[]) { |key| env_dup[key] }
+      allow(ENV).to receive(:[]).with("HOME").and_return(env_home)
+    end
+
+
+    it "searches rubygems for plugins" do
+      if Gem::Specification.respond_to?(:latest_specs)
+        expect(Gem::Specification).to receive(:latest_specs).and_call_original
+      else
+        expect(Gem.source_index).to receive(:latest_specs).and_call_original
+      end
+      loader.subcommand_files.each do |require_path|
+        expect(require_path).to match(/chef\/knife\/.*|plugins\/knife\/.*/)
+      end
+    end
+
+    context "and HOME environment variable is not set" do
+      before do
+        allow(ENV).to receive(:[]).with("HOME").and_return(nil)
+      end
+
+      it "searches rubygems for plugins" do
+        if Gem::Specification.respond_to?(:latest_specs)
+          expect(Gem::Specification).to receive(:latest_specs).and_call_original
+        else
+          expect(Gem.source_index).to receive(:latest_specs).and_call_original
+        end
+        loader.subcommand_files.each do |require_path|
+          expect(require_path).to match(/chef\/knife\/.*|plugins\/knife\/.*/)
+        end
+      end
+    end
+  end
+end
diff --git a/spec/unit/knife/core/hashed_command_loader_spec.rb b/spec/unit/knife/core/hashed_command_loader_spec.rb
new file mode 100644
index 0000000000..00e7ba377b
--- /dev/null
+++ b/spec/unit/knife/core/hashed_command_loader_spec.rb
@@ -0,0 +1,93 @@
+#
+# Copyright:: Copyright (c) 2015 Chef Software, Inc
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'spec_helper'
+
+describe Chef::Knife::SubcommandLoader::HashedCommandLoader do
+  before do
+    allow(ChefConfig).to receive(:windows?) { false }
+  end
+
+  let(:plugin_manifest) {
+    {
+      "_autogenerated_command_paths" => {
+        "plugins_paths" => {
+          "cool_a" => ["/file/for/plugin/a"],
+          "cooler_b" => ["/file/for/plugin/b"]
+        },
+        "plugins_by_category" => {
+          "cool" => [
+            "cool_a"
+          ],
+          "cooler" => [
+            "cooler_b"
+          ]
+        }
+      }
+    }
+  }
+
+  let(:loader) { Chef::Knife::SubcommandLoader::HashedCommandLoader.new(
+    File.join(CHEF_SPEC_DATA, 'knife-site-subcommands'),
+    plugin_manifest)}
+
+  describe "#list_commands" do
+    it "lists all commands by category when no argument is given" do
+      expect(loader.list_commands).to eq({"cool" => ["cool_a"], "cooler" => ["cooler_b"]})
+    end
+
+    it "lists only commands in the given category when a category is given" do
+      expect(loader.list_commands("cool")).to eq({"cool" => ["cool_a"]})
+    end
+  end
+
+  describe "#subcommand_files" do
+    it "lists all the files" do
+      expect(loader.subcommand_files).to eq(["/file/for/plugin/a", "/file/for/plugin/b"])
+    end
+  end
+
+  describe "#load_commands" do
+    before do
+      allow(Kernel).to receive(:load).and_return(true)
+    end
+
+    it "returns false for non-existant commands" do
+      expect(loader.load_command(["nothere"])).to eq(false)
+    end
+
+    it "loads the correct file and returns true if the command exists" do
+      allow(File).to receive(:exists?).and_return(true)
+      expect(Kernel).to receive(:load).with("/file/for/plugin/a").and_return(true)
+      expect(loader.load_command(["cool_a"])).to eq(true)
+    end
+  end
+
+  describe "#subcommand_for_args" do
+    it "returns the subcommands for an exact match" do
+      expect(loader.subcommand_for_args(["cooler_b"])).to eq("cooler_b")
+    end
+
+    it "finds the right subcommand even when _'s are elided" do
+      expect(loader.subcommand_for_args(["cooler", "b"])).to eq("cooler_b")
+    end
+
+    it "returns nil if the the subcommand isn't in our manifest" do
+      expect(loader.subcommand_for_args(["cooler c"])).to eq(nil)
+    end
+  end
+end
diff --git a/spec/unit/knife/core/subcommand_loader_spec.rb b/spec/unit/knife/core/subcommand_loader_spec.rb
index 219a1f2906..2386465c75 100644
--- a/spec/unit/knife/core/subcommand_loader_spec.rb
+++ b/spec/unit/knife/core/subcommand_loader_spec.rb
@@ -1,6 +1,5 @@
 #
-# Author:: Daniel DeLeo (<dan@opscode.com>)
-# Copyright:: Copyright (c) 2011 Opscode, Inc.
+# Copyright:: Copyright (c) 2015 Chef Software, Inc
 # License:: Apache License, Version 2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -32,209 +31,34 @@ describe Chef::Knife::SubcommandLoader do
     Chef::Util::PathHelper.class_variable_set(:@@home_dir, nil)
   end
 
-  it "builds a list of the core subcommand file require paths" do
-    expect(loader.subcommand_files).not_to be_empty
-    loader.subcommand_files.each do |require_path|
-      expect(require_path).to match(/chef\/knife\/.*|plugins\/knife\/.*/)
-    end
-  end
-
-  it "finds files installed via rubygems" do
-    expect(loader.find_subcommands_via_rubygems).to include('chef/knife/node_create')
-    loader.find_subcommands_via_rubygems.each {|rel_path, abs_path| expect(abs_path).to match(%r[chef/knife/.+])}
-  end
-
-  it "finds files from latest version of installed gems" do
-    gems = [ double('knife-ec2-0.5.12') ]
-    gem_files = [
-      '/usr/lib/ruby/gems/knife-ec2-0.5.12/lib/chef/knife/ec2_base.rb',
-      '/usr/lib/ruby/gems/knife-ec2-0.5.12/lib/chef/knife/ec2_otherstuff.rb'
-    ]
-    expect($LOAD_PATH).to receive(:map).and_return([])
-    if Gem::Specification.respond_to? :latest_specs
-      expect(Gem::Specification).to receive(:latest_specs).with(true).and_return(gems)
-      expect(gems[0]).to receive(:matches_for_glob).with(/chef\/knife\/\*\.rb\{(.*),\.rb,(.*)\}/).and_return(gem_files)
-    else
-      expect(Gem.source_index).to receive(:latest_specs).with(true).and_return(gems)
-      expect(gems[0]).to receive(:require_paths).twice.and_return(['lib'])
-      expect(gems[0]).to receive(:full_gem_path).and_return('/usr/lib/ruby/gems/knife-ec2-0.5.12')
-      expect(Dir).to receive(:[]).with('/usr/lib/ruby/gems/knife-ec2-0.5.12/lib/chef/knife/*.rb').and_return(gem_files)
-    end
-    expect(loader).to receive(:find_subcommands_via_dirglob).and_return({})
-    expect(loader.find_subcommands_via_rubygems.values.select { |file| file =~ /knife-ec2/ }.sort).to eq(gem_files)
-  end
-
-  it "finds files using a dirglob when rubygems is not available" do
-    expect(loader.find_subcommands_via_dirglob).to include('chef/knife/node_create')
-    loader.find_subcommands_via_dirglob.each {|rel_path, abs_path| expect(abs_path).to match(%r[chef/knife/.+])}
-  end
-
-  it "finds user-specific subcommands in the user's ~/.chef directory" do
-    expected_command = File.join(home, '.chef', 'plugins', 'knife', 'example_home_subcommand.rb')
-    expect(loader.site_subcommands).to include(expected_command)
-  end
-
-  it "finds repo specific subcommands by searching for a .chef directory" do
-    expected_command = File.join(CHEF_SPEC_DATA, 'knife-site-subcommands', 'plugins', 'knife', 'example_subcommand.rb')
-    expect(loader.site_subcommands).to include(expected_command)
-  end
-
-  # https://github.com/opscode/chef-dk/issues/227
-  #
-  # `knife` in ChefDK isn't from a gem install, it's directly run from a clone
-  # of the source, but there can be one or more versions of chef also installed
-  # as a gem. If the gem install contains a command that doesn't exist in the
-  # source tree of the "primary" chef install, it can be loaded and cause an
-  # error. We also want to ensure that we only load builtin commands from the
-  # "primary" chef install.
-  context "when a different version of chef is also installed as a gem" do
-
-    let(:all_found_commands) do
-      [
-        "/opt/chefdk/embedded/apps/chef/lib/chef/knife/bootstrap.rb",
-        "/opt/chefdk/embedded/apps/chef/lib/chef/knife/client_bulk_delete.rb",
-        "/opt/chefdk/embedded/apps/chef/lib/chef/knife/client_create.rb",
-
-        # We use the fake version 1.0.0 because that version doesn't exist,
-        # which ensures it won't ever equal "chef-#{Chef::VERSION}"
-        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-1.0.0/lib/chef/knife/bootstrap.rb",
-        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-1.0.0/lib/chef/knife/client_bulk_delete.rb",
-        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-1.0.0/lib/chef/knife/client_create.rb",
-
-        # Test that we don't accept a version number that is different only in
-        # trailing characters, e.g. we are running Chef 12.0.0 but there is a
-        # Chef 12.0.0.rc.0 gem also:
-        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-#{Chef::VERSION}.rc.0/lib/chef/knife/thing.rb",
-
-        # Test that we ignore the platform suffix when checking for different
-        # gem versions.
-        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-#{Chef::VERSION}-x86-mingw32/lib/chef/knife/valid.rb",
-        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-#{Chef::VERSION}-i386-mingw64/lib/chef/knife/valid-too.rb",
-        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-#{Chef::VERSION}-mswin32/lib/chef/knife/also-valid.rb",
-        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-#{Chef::VERSION}-universal-mingw32/lib/chef/knife/universal-is-valid.rb",
-        # ...but don't ignore the .rc / .dev parts in the case when we have
-        # platform suffixes
-        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-#{Chef::VERSION}.rc.0-x86-mingw32/lib/chef/knife/invalid.rb",
-        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-#{Chef::VERSION}.dev-mswin32/lib/chef/knife/invalid-too.rb",
-        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-#{Chef::VERSION}.dev.0-x86-mingw64/lib/chef/knife/still-invalid.rb",
-
-        # This command is "extra" compared to what's in the embedded/apps/chef install:
-        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-1.0.0/lib/chef/knife/data_bag_secret_options.rb",
-        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-vault-2.2.4/lib/chef/knife/decrypt.rb",
-        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/knife-spork-1.4.1/lib/chef/knife/spork-bump.rb",
-
-        # These are fake commands that have names designed to test that the
-        # regex is strict enough
-        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-foo-#{Chef::VERSION}/lib/chef/knife/chef-foo.rb",
-        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/foo-chef-#{Chef::VERSION}/lib/chef/knife/foo-chef.rb",
+  let(:config_dir) { File.join(CHEF_SPEC_DATA, 'knife-site-subcommands') }
 
-        # In a real scenario, we'd use rubygems APIs to only select the most
-        # recent gem, but for this test we want to check that we're doing the
-        # right thing both when the plugin version matches and does not match
-        # the current chef version. Looking at
-        # `SubcommandLoader::MATCHES_THIS_CHEF_GEM` and
-        # `SubcommandLoader::MATCHES_CHEF_GEM` should make it clear why we want
-        # to test these two cases.
-        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-bar-1.0.0/lib/chef/knife/chef-bar.rb",
-        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/bar-chef-1.0.0/lib/chef/knife/bar-chef.rb"
-      ]
-    end
-
-    let(:expected_valid_commands) do
-      [
-        "/opt/chefdk/embedded/apps/chef/lib/chef/knife/bootstrap.rb",
-        "/opt/chefdk/embedded/apps/chef/lib/chef/knife/client_bulk_delete.rb",
-        "/opt/chefdk/embedded/apps/chef/lib/chef/knife/client_create.rb",
-        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-#{Chef::VERSION}-x86-mingw32/lib/chef/knife/valid.rb",
-        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-#{Chef::VERSION}-i386-mingw64/lib/chef/knife/valid-too.rb",
-        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-#{Chef::VERSION}-mswin32/lib/chef/knife/also-valid.rb",
-        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-#{Chef::VERSION}-universal-mingw32/lib/chef/knife/universal-is-valid.rb",
-        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-vault-2.2.4/lib/chef/knife/decrypt.rb",
-        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/knife-spork-1.4.1/lib/chef/knife/spork-bump.rb",
-        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-foo-#{Chef::VERSION}/lib/chef/knife/chef-foo.rb",
-        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/foo-chef-#{Chef::VERSION}/lib/chef/knife/foo-chef.rb",
-        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/chef-bar-1.0.0/lib/chef/knife/chef-bar.rb",
-        "/opt/chefdk/embedded/lib/ruby/gems/2.1.0/gems/bar-chef-1.0.0/lib/chef/knife/bar-chef.rb"
-      ]
-    end
-
-    before do
-      expect(loader).to receive(:find_files_latest_gems).with("chef/knife/*.rb").and_return(all_found_commands)
-      expect(loader).to receive(:find_subcommands_via_dirglob).and_return({})
-    end
-
-    it "ignores commands from the non-matching gem install" do
-      expect(loader.find_subcommands_via_rubygems.values).to eq(expected_valid_commands)
-    end
-
-  end
-
-  describe "finding 3rd party plugins" do
-    let(:home) { "/home/alice" }
-    let(:manifest_path) { home + "/.chef/plugin_manifest.json" }
-
-    context "when there is not a ~/.chef/plugin_manifest.json file" do
+  describe "#for_config" do
+    context "when ~/.chef/plugin_manifest.json exists" do
       before do
-        allow(File).to receive(:exist?).with(manifest_path).and_return(false)
+        allow(File).to receive(:exist?).with(File.join(home, '.chef', 'plugin_manifest.json')).and_return(true)
       end
 
-      it "searches rubygems for plugins" do
-        if Gem::Specification.respond_to?(:latest_specs)
-          expect(Gem::Specification).to receive(:latest_specs).and_call_original
-        else
-          expect(Gem.source_index).to receive(:latest_specs).and_call_original
-        end
-        loader.subcommand_files.each do |require_path|
-          expect(require_path).to match(/chef\/knife\/.*|plugins\/knife\/.*/)
-        end
+      it "creates a HashedCommandLoader with the manifest has _autogenerated_command_paths" do
+        allow(File).to receive(:read).with(File.join(home, '.chef', 'plugin_manifest.json')).and_return("{ \"_autogenerated_command_paths\": {}}")
+        expect(Chef::Knife::SubcommandLoader.for_config(config_dir)).to be_a Chef::Knife::SubcommandLoader::HashedCommandLoader
       end
 
-      context "and HOME environment variable is not set" do
-        before do
-          allow(Chef::Util::PathHelper).to receive(:home).and_return(nil)
-        end
-
-        it "searches rubygems for plugins" do
-          if Gem::Specification.respond_to?(:latest_specs)
-            expect(Gem::Specification).to receive(:latest_specs).and_call_original
-          else
-            expect(Gem.source_index).to receive(:latest_specs).and_call_original
-          end
-          loader.subcommand_files.each do |require_path|
-            expect(require_path).to match(/chef\/knife\/.*|plugins\/knife\/.*/)
-          end
-        end
+      it "creates a CustomManifestLoader with then manifest has a key other than _autogenerated_command_paths" do
+        Chef::Config[:treat_deprecation_warnings_as_errors] = false
+        allow(File).to receive(:read).with(File.join(home, '.chef', 'plugin_manifest.json')).and_return("{ \"plugins\": {}}")
+        expect(Chef::Knife::SubcommandLoader.for_config(config_dir)).to be_a Chef::Knife::SubcommandLoader::CustomManifestLoader
       end
-
     end
 
-    context "when there is a ~/.chef/plugin_manifest.json file" do
-      let(:ec2_server_create_plugin) { "/usr/lib/ruby/gems/knife-ec2-0.5.12/lib/chef/knife/ec2_server_create.rb" }
-
-      let(:manifest_content) do
-        { "plugins" => {
-            "knife-ec2" => {
-              "paths" => [
-                ec2_server_create_plugin
-              ]
-            }
-          }
-        }
-      end
-
-      let(:manifest_json) { Chef::JSONCompat.to_json(manifest_content) }
-
+    context "when ~/.chef/plugin_manifest.json does not exist" do
       before do
-        allow(File).to receive(:exist?).with(manifest_path).and_return(true)
-        allow(File).to receive(:read).with(manifest_path).and_return(manifest_json)
+        allow(File).to receive(:exist?).with(File.join(home, '.chef', 'plugin_manifest.json')).and_return(false)
       end
 
-      it "uses paths from the manifest instead of searching gems" do
-        expect(Gem::Specification).not_to receive(:latest_specs).and_call_original
-        expect(loader.subcommand_files).to include(ec2_server_create_plugin)
+      it "creates a GemGlobLoader" do
+        expect(Chef::Knife::SubcommandLoader.for_config(config_dir)).to be_a Chef::Knife::SubcommandLoader::GemGlobLoader
       end
-
     end
   end
-
 end