From 41d55bb75799021a3c42f547860586ee21cb2804 Mon Sep 17 00:00:00 2001 From: Steven Danna Date: Tue, 10 Feb 2015 10:10:30 +0000 Subject: Add knife-rehash command for subcommand location hashing `knife rehash` stores the paths to knife plugins in a specially formatted plugin_manifest entry. This lowers the overhead of subsequent knife invocations. --- lib/chef/knife/core/custom_manifest_loader.rb | 70 +++++++ lib/chef/knife/core/gem_glob_loader.rb | 138 +++++++++++++ lib/chef/knife/core/hashed_command_loader.rb | 75 +++++++ lib/chef/knife/core/subcommand_loader.rb | 273 ++++++++++++-------------- lib/chef/knife/null.rb | 8 + lib/chef/knife/rehash.rb | 59 ++++++ 6 files changed, 478 insertions(+), 145 deletions(-) create mode 100644 lib/chef/knife/core/custom_manifest_loader.rb create mode 100644 lib/chef/knife/core/gem_glob_loader.rb create mode 100644 lib/chef/knife/core/hashed_command_loader.rb create mode 100644 lib/chef/knife/null.rb create mode 100644 lib/chef/knife/rehash.rb (limited to 'lib/chef/knife') diff --git a/lib/chef/knife/core/custom_manifest_loader.rb b/lib/chef/knife/core/custom_manifest_loader.rb new file mode 100644 index 0000000000..ae8a8199b8 --- /dev/null +++ b/lib/chef/knife/core/custom_manifest_loader.rb @@ -0,0 +1,70 @@ +# Author:: Steven Danna () +# Copyright:: Copyright (c) 2015 Chef Software, Inc +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'chef/version' +class Chef + class Knife + class SubcommandLoader + + # + # Load a subcommand from a user-supplied + # manifest file + # + class CustomManifestLoader < Chef::Knife::SubcommandLoader + attr_accessor :manifest + def initialize(chef_config_dir, plugin_manifest) + super(chef_config_dir) + @manifest = plugin_manifest + end + + # If the user has created a ~/.chef/plugin_manifest.json file, we'll use + # that instead of inspecting the on-system gems to find the plugins. The + # file format is expected to look like: + # + # { "plugins": { + # "knife-ec2": { + # "paths": [ + # "/home/alice/.rubymanagerthing/gems/knife-ec2-x.y.z/lib/chef/knife/ec2_server_create.rb", + # "/home/alice/.rubymanagerthing/gems/knife-ec2-x.y.z/lib/chef/knife/ec2_server_delete.rb" + # ] + # } + # } + # } + # + # Extraneous content in this file is ignored. This intentional so that we + # can adapt the file format for potential behavior changes to knife in + # the future. + def find_subcommands_via_manifest + # Format of subcommand_files is "relative_path" (something you can + # Kernel.require()) => full_path. The relative path isn't used + # currently, so we just map full_path => full_path. + subcommand_files = {} + manifest["plugins"].each do |plugin_name, plugin_manifest| + plugin_manifest["paths"].each do |cmd_path| + subcommand_files[cmd_path] = cmd_path + end + end + subcommand_files.merge(find_subcommands_via_dirglob) + end + + def subcommand_files + subcommand_files ||= (find_subcommands_via_manifest.values + site_subcommands).flatten.uniq + end + end + end + end +end diff --git a/lib/chef/knife/core/gem_glob_loader.rb b/lib/chef/knife/core/gem_glob_loader.rb new file mode 100644 index 0000000000..fc21687c80 --- /dev/null +++ b/lib/chef/knife/core/gem_glob_loader.rb @@ -0,0 +1,138 @@ +# Author:: Christopher Brown () +# Author:: Daniel DeLeo () +# Copyright:: Copyright (c) 2009, 2011 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'chef/version' +require 'chef/util/path_helper' +class Chef + class Knife + class SubcommandLoader + class GemGlobLoader < Chef::Knife::SubcommandLoader + MATCHES_CHEF_GEM = %r{/chef-[\d]+\.[\d]+\.[\d]+}.freeze + MATCHES_THIS_CHEF_GEM = %r{/chef-#{Chef::VERSION}(-\w+)?(-\w+)?/}.freeze + + def subcommand_files + @subcommand_files ||= (gem_and_builtin_subcommands.values + site_subcommands).flatten.uniq + end + + # Returns a Hash of paths to knife commands built-in to chef, or installed via gem. + # If rubygems is not installed, falls back to globbing the knife directory. + # The Hash is of the form {"relative/path" => "/absolute/path"} + #-- + # Note: the "right" way to load the plugins is to require the relative path, i.e., + # require 'chef/knife/command' + # but we're getting frustrated by bugs at every turn, and it's slow besides. So + # subcommand loader has been modified to load the plugins by using Kernel.load + # with the absolute path. + def gem_and_builtin_subcommands + require 'rubygems' + find_subcommands_via_rubygems + rescue LoadError + find_subcommands_via_dirglob + end + + def find_subcommands_via_dirglob + # The "require paths" of the core knife subcommands bundled with chef + files = Dir[File.join(Chef::Util::PathHelper.escape_glob(File.expand_path('../../../knife', __FILE__)), '*.rb')] + subcommand_files = {} + files.each do |knife_file| + rel_path = knife_file[/#{CHEF_ROOT}#{Regexp.escape(File::SEPARATOR)}(.*)\.rb/,1] + subcommand_files[rel_path] = knife_file + end + subcommand_files + end + + def find_subcommands_via_rubygems + files = find_files_latest_gems 'chef/knife/*.rb' + subcommand_files = {} + files.each do |file| + rel_path = file[/(#{Regexp.escape File.join('chef', 'knife', '')}.*)\.rb/, 1] + + # When not installed as a gem (ChefDK/appbundler in particular), AND + # a different version of Chef is installed via gems, `files` will + # include some files from the 'other' Chef install. If this contains + # a knife command that doesn't exist in this version of Chef, we will + # get a LoadError later when we try to require it. + next if from_different_chef_version?(file) + + subcommand_files[rel_path] = file + end + + subcommand_files.merge(find_subcommands_via_dirglob) + end + + private + + def find_files_latest_gems(glob, check_load_path=true) + files = [] + + if check_load_path + files = $LOAD_PATH.map { |load_path| + Dir["#{File.expand_path glob, Chef::Util::PathHelper.escape_glob(load_path)}#{Gem.suffix_pattern}"] + }.flatten.select { |file| File.file? file.untaint } + end + + gem_files = latest_gem_specs.map do |spec| + # Gem::Specification#matches_for_glob wasn't added until RubyGems 1.8 + if spec.respond_to? :matches_for_glob + spec.matches_for_glob("#{glob}#{Gem.suffix_pattern}") + else + check_spec_for_glob(spec, glob) + end + end.flatten + + files.concat gem_files + files.uniq! if check_load_path + + return files + end + + def latest_gem_specs + @latest_gem_specs ||= if Gem::Specification.respond_to? :latest_specs + Gem::Specification.latest_specs(true) # find prerelease gems + else + Gem.source_index.latest_specs(true) + end + end + + def check_spec_for_glob(spec, glob) + dirs = if spec.require_paths.size > 1 then + "{#{spec.require_paths.join(',')}}" + else + spec.require_paths.first + end + + glob = File.join(Chef::Util::PathHelper.escape_glob(spec.full_gem_path, dirs), glob) + + Dir[glob].map { |f| f.untaint } + end + + def from_different_chef_version?(path) + matches_any_chef_gem?(path) && !matches_this_chef_gem?(path) + end + + def matches_any_chef_gem?(path) + path =~ MATCHES_CHEF_GEM + end + + def matches_this_chef_gem?(path) + path =~ MATCHES_THIS_CHEF_GEM + end + end + end + end +end diff --git a/lib/chef/knife/core/hashed_command_loader.rb b/lib/chef/knife/core/hashed_command_loader.rb new file mode 100644 index 0000000000..6729e59bb0 --- /dev/null +++ b/lib/chef/knife/core/hashed_command_loader.rb @@ -0,0 +1,75 @@ +# Author:: Steven Danna () +# Copyright:: Copyright (c) 2015 Chef Software, Inc +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'chef/version' +class Chef + class Knife + class SubcommandLoader + # + # Load a subcommand from a pre-computed path + # for the given command. + # + class HashedCommandLoader < Chef::Knife::SubcommandLoader + KEY = '_autogenerated_command_paths' + + attr_accessor :manifest + def initialize(chef_config_dir, plugin_manifest) + super(chef_config_dir) + @manifest = plugin_manifest + end + + def guess_category(args) + category_words = positional_arguments(args) + category_words.map! { |w| w.split('-') }.flatten! + find_longest_key(manifest[KEY]["plugins_by_category"], category_words, ' ') + end + + def list_commands(pref_category=nil) + if pref_category || manifest[KEY]["plugins_by_category"].key?(pref_category) + { pref_category => manifest[KEY]["plugins_by_category"][pref_category] } + else + manifest[KEY]["plugins_by_category"] + end + end + + def subcommand_files + manifest[KEY]["plugins_paths"].values.flatten + end + + def load_command(args) + paths = manifest[KEY]["plugins_paths"][subcommand_for_args(args)] + if paths.nil? || paths.empty? || (! paths.is_a? Array) + false + else + paths.each do |sc| + Kernel.load sc + end + true + end + end + + def subcommand_for_args(args) + if manifest[KEY]["plugins_paths"].key?(args) + args + else + find_longest_key(manifest[KEY]["plugins_paths"], args, "_") + end + end + end + end + end +end diff --git a/lib/chef/knife/core/subcommand_loader.rb b/lib/chef/knife/core/subcommand_loader.rb index a8705c724f..8aa2205679 100644 --- a/lib/chef/knife/core/subcommand_loader.rb +++ b/lib/chef/knife/core/subcommand_loader.rb @@ -18,19 +18,66 @@ require 'chef/version' require 'chef/util/path_helper' +require 'chef/knife/core/gem_glob_loader' +require 'chef/knife/core/hashed_command_loader' +require 'chef/knife/core/custom_manifest_loader' + class Chef class Knife + # + # Public Methods of a Subcommand Loader + # + # load_commands - loads all available subcommands + # load_command(args) - loads subcommands for the given args + # list_commands(args) - lists all available subcommands, + # optionally filtering by category + # subcommand_files - returns an array of all subcommand files + # that could be loaded + # commnad_class_from(args) - returns the subcommand class for the + # user-requested command + # class SubcommandLoader - - MATCHES_CHEF_GEM = %r{/chef-[\d]+\.[\d]+\.[\d]+}.freeze - MATCHES_THIS_CHEF_GEM = %r{/chef-#{Chef::VERSION}(-\w+)?(-\w+)?/}.freeze - attr_reader :chef_config_dir attr_reader :env - def initialize(chef_config_dir, env=nil) + # A small factory method. Eventually, this is the only place + # where SubcommandLoader should know about its subclasses, but + # to maintain backwards compatibility many of the instance + # methods in this base class contain default implementations + # of the functions sub classes should otherwise provide + # or directly instantiate the appropriate subclass + def self.for_config(chef_config_dir) + if autogenerated_manifest? + Knife::SubcommandLoader::HashedCommandLoader.new(chef_config_dir, plugin_manifest) + elsif custom_manifest? + Knife::SubcommandLoader::CustomManifestLoader.new(chef_config_dir, plugin_manifest) + else + Knife::SubcommandLoader::GemGlobLoader.new(chef_config_dir) + end + end + + def self.plugin_manifest? + plugin_manifest_path && File.exist?(plugin_manifest_path) + end + + def self.autogenerated_manifest? + plugin_manifest? && plugin_manifest.key?(HashedCommandLoader::KEY) + end + + def self.custom_manifest? + plugin_manifest? && plugin_manifest.key?("plugins") + end + + def self.plugin_manifest + Chef::JSONCompat.from_json(File.read(plugin_manifest_path)) + end + + def self.plugin_manifest_path + Chef::Util::PathHelper.home('.chef', 'plugin_manifest.json') + end + + def initialize(chef_config_dir, env = nil) @chef_config_dir = chef_config_dir - @forced_activate = {} # Deprecated and un-used instance variable. @env = env @@ -41,82 +88,48 @@ class Chef # Load all the sub-commands def load_commands + return true if @loaded subcommand_files.each { |subcommand| Kernel.load subcommand } - true + @loaded = true end - # Returns an Array of paths to knife commands located in chef_config_dir/plugins/knife/ - # and ~/.chef/plugins/knife/ - def site_subcommands - user_specific_files = [] - - if chef_config_dir - user_specific_files.concat Dir.glob(File.expand_path("plugins/knife/*.rb", Chef::Util::PathHelper.escape_glob(chef_config_dir))) - end - - # finally search ~/.chef/plugins/knife/*.rb - Chef::Util::PathHelper.home('.chef', 'plugins', 'knife') do |p| - user_specific_files.concat Dir.glob(File.join(Chef::Util::PathHelper.escape_glob(p), '*.rb')) - end + def force_load + @loaded=false + load_commands + end - user_specific_files + def load_command(_command_args) + load_commands end - # Returns a Hash of paths to knife commands built-in to chef, or installed via gem. - # If rubygems is not installed, falls back to globbing the knife directory. - # The Hash is of the form {"relative/path" => "/absolute/path"} - #-- - # Note: the "right" way to load the plugins is to require the relative path, i.e., - # require 'chef/knife/command' - # but we're getting frustrated by bugs at every turn, and it's slow besides. So - # subcommand loader has been modified to load the plugins by using Kernel.load - # with the absolute path. - def gem_and_builtin_subcommands - if have_plugin_manifest? - find_subcommands_via_manifest + def list_commands(pref_cat = nil) + load_commands + if pref_cat && Chef::Knife.subcommands_by_category.key?(pref_cat) + { pref_cat => Chef::Knife.subcommands_by_category[pref_cat] } else - # search all gems for chef/knife/*.rb - require 'rubygems' - find_subcommands_via_rubygems + Chef::Knife.subcommands_by_category end - rescue LoadError - find_subcommands_via_dirglob end - def subcommand_files - @subcommand_files ||= (gem_and_builtin_subcommands.values + site_subcommands).flatten.uniq + def command_class_from(args) + cmd_words = positional_arguments(args) + load_command(cmd_words) + result = Chef::Knife.subcommands[find_longest_key(Chef::Knife.subcommands, + cmd_words, '_')] + result || Chef::Knife.subcommands[args.first.gsub('-', '_')] end - # If the user has created a ~/.chef/plugin_manifest.json file, we'll use - # that instead of inspecting the on-system gems to find the plugins. The - # file format is expected to look like: - # - # { "plugins": { - # "knife-ec2": { - # "paths": [ - # "/home/alice/.rubymanagerthing/gems/knife-ec2-x.y.z/lib/chef/knife/ec2_server_create.rb", - # "/home/alice/.rubymanagerthing/gems/knife-ec2-x.y.z/lib/chef/knife/ec2_server_delete.rb" - # ] - # } - # } - # } - # - # Extraneous content in this file is ignored. This intentional so that we - # can adapt the file format for potential behavior changes to knife in - # the future. - def find_subcommands_via_manifest - # Format of subcommand_files is "relative_path" (something you can - # Kernel.require()) => full_path. The relative path isn't used - # currently, so we just map full_path => full_path. - subcommand_files = {} - plugin_manifest["plugins"].each do |plugin_name, plugin_manifest| - plugin_manifest["paths"].each do |cmd_path| - subcommand_files[cmd_path] = cmd_path - end - end - subcommand_files.merge(find_subcommands_via_dirglob) + def guess_category(args) + category_words = positional_arguments(args) + category_words.map! { |w| w.split('-') }.flatten! + find_longest_key(Chef::Knife.subcommands_by_category, + category_words, ' ') end + + # + # This is shared between the custom_manifest_loader and the gem_glob_loader + # def find_subcommands_via_dirglob # The "require paths" of the core knife subcommands bundled with chef files = Dir[File.join(Chef::Util::PathHelper.escape_glob(File.expand_path('../../../knife', __FILE__)), '*.rb')] @@ -128,95 +141,65 @@ class Chef subcommand_files end - def find_subcommands_via_rubygems - files = find_files_latest_gems 'chef/knife/*.rb' - subcommand_files = {} - files.each do |file| - rel_path = file[/(#{Regexp.escape File.join('chef', 'knife', '')}.*)\.rb/, 1] - - # When not installed as a gem (ChefDK/appbundler in particular), AND - # a different version of Chef is installed via gems, `files` will - # include some files from the 'other' Chef install. If this contains - # a knife command that doesn't exist in this version of Chef, we will - # get a LoadError later when we try to require it. - next if from_different_chef_version?(file) - - subcommand_files[rel_path] = file - end - - subcommand_files.merge(find_subcommands_via_dirglob) - end - - def have_plugin_manifest? - plugin_manifest_path && File.exist?(plugin_manifest_path) - end - - def plugin_manifest - Chef::JSONCompat.from_json(File.read(plugin_manifest_path)) - end - - def plugin_manifest_path - Chef::Util::PathHelper.home('.chef', 'plugin_manifest.json') + # + # Subclassses should define this themselves. Eventually, this will raise a + # NotImplemented error, but for now, we mimic the behavior the user was likely + # to get in the past. + # + def subcommand_files + Chef::Log.warn "DEPRECATED: Using Chef::Knife::SubcommandLoader directly is deprecated. +Please use Chef::Knife::SubcommandLoader.for_config(chef_config_dir, env)" + @subcommand_files ||= if Chef::Knife::SubcommandLoader.plugin_manifest? + Chef::Knife::SubcommandLoader::CustomManifestLoader.new(chef_config_dir, env).subcommand_files + else + Chef::Knife::SubcommandLoader::GemGlobLoader.new(chef_config_dir, env).subcommand_files + end end - private - - def find_files_latest_gems(glob, check_load_path=true) - files = [] - - if check_load_path - files = $LOAD_PATH.map { |load_path| - Dir["#{File.expand_path glob, Chef::Util::PathHelper.escape_glob(load_path)}#{Gem.suffix_pattern}"] - }.flatten.select { |file| File.file? file.untaint } - end - - gem_files = latest_gem_specs.map do |spec| - # Gem::Specification#matches_for_glob wasn't added until RubyGems 1.8 - if spec.respond_to? :matches_for_glob - spec.matches_for_glob("#{glob}#{Gem.suffix_pattern}") + # + # Utility function for finding an element in a hash given an array + # of words and a separator. We find the the longest key in the + # hash composed of the given words joined by the separator. + # + def find_longest_key(hash, words, sep = '_') + match = nil + until match || words.empty? + candidate = words.join(sep) + if hash.key?(candidate) + match = candidate else - check_spec_for_glob(spec, glob) + words.pop end - end.flatten - - files.concat gem_files - files.uniq! if check_load_path - - return files - end - - def latest_gem_specs - @latest_gem_specs ||= if Gem::Specification.respond_to? :latest_specs - Gem::Specification.latest_specs(true) # find prerelease gems - else - Gem.source_index.latest_specs(true) end + match end - def check_spec_for_glob(spec, glob) - dirs = if spec.require_paths.size > 1 then - "{#{spec.require_paths.join(',')}}" - else - spec.require_paths.first - end - - glob = File.join(Chef::Util::PathHelper.escape_glob(spec.full_gem_path, dirs), glob) - - Dir[glob].map { |f| f.untaint } + # + # The positional arguments from the argument list provided by the + # users. Used to search for subcommands and categories. + # + # @return [Array] + # + def positional_arguments(args) + args.select { |arg| arg =~ /^(([[:alnum:]])[[:alnum:]\_\-]+)$/ } end - def from_different_chef_version?(path) - matches_any_chef_gem?(path) && !matches_this_chef_gem?(path) - end + # Returns an Array of paths to knife commands located in + # chef_config_dir/plugins/knife/ and ~/.chef/plugins/knife/ + def site_subcommands + user_specific_files = [] - def matches_any_chef_gem?(path) - path =~ MATCHES_CHEF_GEM - end + if chef_config_dir + user_specific_files.concat Dir.glob(File.expand_path("plugins/knife/*.rb", Chef::Util::PathHelper.escape_glob(chef_config_dir))) + end - def matches_this_chef_gem?(path) - path =~ MATCHES_THIS_CHEF_GEM - end + # finally search ~/.chef/plugins/knife/*.rb + Chef::Util::PathHelper.home('.chef', 'plugins', 'knife') do |p| + user_specific_files.concat Dir.glob(File.join(Chef::Util::PathHelper.escape_glob(p), '*.rb')) + end + user_specific_files + end end end end diff --git a/lib/chef/knife/null.rb b/lib/chef/knife/null.rb new file mode 100644 index 0000000000..c8f04e5cd3 --- /dev/null +++ b/lib/chef/knife/null.rb @@ -0,0 +1,8 @@ +class Chef + class Knife + class Null < Chef::Knife + def run + end + end + end +end diff --git a/lib/chef/knife/rehash.rb b/lib/chef/knife/rehash.rb new file mode 100644 index 0000000000..ef58a687b8 --- /dev/null +++ b/lib/chef/knife/rehash.rb @@ -0,0 +1,59 @@ +# +# Author:: Steven Danna +# Copyright:: Copyright (c) 2015 Chef Software, Inc +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'chef/knife' +require 'chef/knife/core/subcommand_loader' + +class Chef + class Knife + class Rehash < Chef::Knife + + def run + if ! Chef::Knife::SubcommandLoader.autogenerated_manifest? + ui.msg "Using knife-rehash will speed up knife's load time by caching the location of subcommands on disk" + ui.msg "However, you will need to update the cache by running `knife rehash` anytime you install a new knife plugin." + ui.confirm "Would you like to continue?" + else + reload_plugins + end + write_hash(generate_hash) + end + + def reload_plugins + Chef::Knife::SubcommandLoader::GemGlobLoader.new(@@chef_config_dir).load_commands + end + + def generate_hash + output = if Chef::Knife::SubcommandLoader.plugin_manifest? + Chef::Knife::SubcommandLoader.plugin_manifest + else + { Chef::Knife::SubcommandLoader::HashedCommandLoader::KEY => {}} + end + output[Chef::Knife::SubcommandLoader::HashedCommandLoader::KEY]['plugins_paths'] = Chef::Knife.subcommand_files + output[Chef::Knife::SubcommandLoader::HashedCommandLoader::KEY]['plugins_by_category'] = Chef::Knife.subcommands_by_category + output + end + + def write_hash(data) + File.open(Chef::Knife::SubcommandLoader.plugin_manifest_path, 'w') do |f| + f.write(Chef::JSONCompat.to_json_pretty(data)) + end + end + end + end +end -- cgit v1.2.1