diff options
author | Daniel DeLeo <dan@opscode.com> | 2011-04-08 09:03:35 -0700 |
---|---|---|
committer | Daniel DeLeo <dan@opscode.com> | 2011-04-08 09:03:35 -0700 |
commit | f0f5f804876105385332551f72433ba0a561d5ad (patch) | |
tree | 750ca8c897fc72fa0e6f35127cc2e64648d799a8 | |
parent | 80f97bc6f1ba22b9d107f10d477899d499a28364 (diff) | |
download | chef-f0f5f804876105385332551f72433ba0a561d5ad.tar.gz |
[CHEF-2200] only load the data we actually need for cookbook solving
-rw-r--r-- | chef/lib/chef/cookbook/metadata.rb | 116 | ||||
-rw-r--r-- | chef/lib/chef/cookbook_version.rb | 88 | ||||
-rw-r--r-- | chef/lib/chef/cookbook_version_selector.rb | 9 | ||||
-rw-r--r-- | chef/lib/chef/environment.rb | 41 | ||||
-rw-r--r-- | chef/spec/unit/cookbook_version_spec.rb | 45 | ||||
-rw-r--r-- | features/api/nodes/cookbook_sync_api.feature | 2 |
6 files changed, 256 insertions, 45 deletions
diff --git a/chef/lib/chef/cookbook/metadata.rb b/chef/lib/chef/cookbook/metadata.rb index 9e5e3b4dc1..407a29ddaa 100644 --- a/chef/lib/chef/cookbook/metadata.rb +++ b/chef/lib/chef/cookbook/metadata.rb @@ -27,22 +27,41 @@ require 'chef/version_constraint' class Chef class Cookbook + # == Chef::Cookbook::Metadata # Chef::Cookbook::Metadata provides a convenient DSL for declaring metadata # about Chef Cookbooks. class Metadata + NAME = 'name'.freeze + DESCRIPTION = 'description'.freeze + LONG_DESCRIPTION = 'long_description'.freeze + MAINTAINER = 'maintainer'.freeze + MAINTAINER_EMAIL = 'maintainer_email'.freeze + LICENSE = 'license'.freeze + PLATFORMS = 'platforms'.freeze + DEPENDENCIES = 'dependencies'.freeze + RECOMMENDATIONS = 'recommendations'.freeze + SUGGESTIONS = 'suggestions'.freeze + CONFLICTING = 'conflicting'.freeze + PROVIDING = 'providing'.freeze + REPLACING = 'replacing'.freeze + ATTRIBUTES = 'attributes'.freeze + GROUPINGS = 'groupings'.freeze + RECIPES = 'recipes'.freeze + VERSION = 'version'.freeze + COMPARISON_FIELDS = [ :name, :description, :long_description, :maintainer, :maintainer_email, :license, :platforms, :dependencies, :recommendations, :suggestions, :conflicting, :providing, :replacing, :attributes, :groupings, :recipes, :version] - VERSION_CONSTRAINTS = {:depends => "dependencies", - :recommends => "recommendations", - :suggests => "suggestions", - :conflicts => "conflicting", - :provides => "providing", - :replaces => "replacing" } + VERSION_CONSTRAINTS = {:depends => DEPENDENCIES, + :recommends => RECOMMENDATIONS, + :suggests => SUGGESTIONS, + :conflicts => CONFLICTING, + :provides => PROVIDING, + :replaces => REPLACING } include Chef::Mixin::CheckHelper include Chef::Mixin::ParamsValidate @@ -402,23 +421,23 @@ class Chef def to_hash { - 'name' => self.name, - 'description' => self.description, - 'long_description' => self.long_description, - 'maintainer' => self.maintainer, - 'maintainer_email' => self.maintainer_email, - 'license' => self.license, - 'platforms' => self.platforms, - 'dependencies' => self.dependencies, - 'recommendations' => self.recommendations, - 'suggestions' => self.suggestions, - 'conflicting' => self.conflicting, - 'providing' => self.providing, - 'replacing' => self.replacing, - 'attributes' => self.attributes, - 'groupings' => self.groupings, - 'recipes' => self.recipes, - 'version' => self.version + NAME => self.name, + DESCRIPTION => self.description, + LONG_DESCRIPTION => self.long_description, + MAINTAINER => self.maintainer, + MAINTAINER_EMAIL => self.maintainer_email, + LICENSE => self.license, + PLATFORMS => self.platforms, + DEPENDENCIES => self.dependencies, + RECOMMENDATIONS => self.recommendations, + SUGGESTIONS => self.suggestions, + CONFLICTING => self.conflicting, + PROVIDING => self.providing, + REPLACING => self.replacing, + ATTRIBUTES => self.attributes, + GROUPINGS => self.groupings, + RECIPES => self.recipes, + VERSION => self.version } end @@ -433,23 +452,23 @@ class Chef end def from_hash(o) - @name = o['name'] if o.has_key?('name') - @description = o['description'] if o.has_key?('description') - @long_description = o['long_description'] if o.has_key?('long_description') - @maintainer = o['maintainer'] if o.has_key?('maintainer') - @maintainer_email = o['maintainer_email'] if o.has_key?('maintainer_email') - @license = o['license'] if o.has_key?('license') - @platforms = o['platforms'] if o.has_key?('platforms') - @dependencies = handle_deprecated_constraints(o['dependencies']) if o.has_key?('dependencies') - @recommendations = handle_deprecated_constraints(o['recommendations']) if o.has_key?('recommendations') - @suggestions = handle_deprecated_constraints(o['suggestions']) if o.has_key?('suggestions') - @conflicting = handle_deprecated_constraints(o['conflicting']) if o.has_key?('conflicting') - @providing = o['providing'] if o.has_key?('providing') - @replacing = handle_deprecated_constraints(o['replacing']) if o.has_key?('replacing') - @attributes = o['attributes'] if o.has_key?('attributes') - @groupings = o['groupings'] if o.has_key?('groupings') - @recipes = o['recipes'] if o.has_key?('recipes') - @version = o['version'] if o.has_key?('version') + @name = o[NAME] if o.has_key?(NAME) + @description = o[DESCRIPTION] if o.has_key?(DESCRIPTION) + @long_description = o[LONG_DESCRIPTION] if o.has_key?(LONG_DESCRIPTION) + @maintainer = o[MAINTAINER] if o.has_key?(MAINTAINER) + @maintainer_email = o[MAINTAINER_EMAIL] if o.has_key?(MAINTAINER_EMAIL) + @license = o[LICENSE] if o.has_key?(LICENSE) + @platforms = o[PLATFORMS] if o.has_key?(PLATFORMS) + @dependencies = handle_deprecated_constraints(o[DEPENDENCIES]) if o.has_key?(DEPENDENCIES) + @recommendations = handle_deprecated_constraints(o[RECOMMENDATIONS]) if o.has_key?(RECOMMENDATIONS) + @suggestions = handle_deprecated_constraints(o[SUGGESTIONS]) if o.has_key?(SUGGESTIONS) + @conflicting = handle_deprecated_constraints(o[CONFLICTING]) if o.has_key?(CONFLICTING) + @providing = o[PROVIDING] if o.has_key?(PROVIDING) + @replacing = handle_deprecated_constraints(o[REPLACING]) if o.has_key?(REPLACING) + @attributes = o[ATTRIBUTES] if o.has_key?(ATTRIBUTES) + @groupings = o[GROUPINGS] if o.has_key?(GROUPINGS) + @recipes = o[RECIPES] if o.has_key?(RECIPES) + @version = o[VERSION] if o.has_key?(VERSION) self end @@ -588,5 +607,22 @@ INVALID end end + + #== Chef::Cookbook::MinimalMetadata + # MinimalMetadata is a duck type of Cookbook::Metadata, used + # internally by Chef Server when determining the optimal set of + # cookbooks for a node. + # + # MinimalMetadata objects typically contain only enough information + # to solve the cookbook collection for a run list, but not enough to + # generate the proper response + class MinimalMetadata < Metadata + def initialize(name, params) + @name = name + from_hash(params) + end + end + + end end diff --git a/chef/lib/chef/cookbook_version.rb b/chef/lib/chef/cookbook_version.rb index a692c32a16..45ac21c6ef 100644 --- a/chef/lib/chef/cookbook_version.rb +++ b/chef/lib/chef/cookbook_version.rb @@ -3,7 +3,8 @@ # Author:: Christopher Walters (<cw@opscode.com>) # Author:: Tim Hinderliter (<tim@opscode.com>) # Author:: Seth Falcon (<seth@opscode.com>) -# Copyright:: Copyright 2008-2010 Opscode, Inc. +# Author:: Daniel DeLeo (<dan@opscode.com>) +# Copyright:: Copyright 2008-2011 Opscode, Inc. # License:: Apache License, Version 2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -29,6 +30,80 @@ require 'chef/cookbook/metadata' require 'chef/version_class' class Chef + + #== Chef::MinimalCookbookVersion + # MinimalCookbookVersion is a duck type of CookbookVersion, used + # internally by Chef Server as an optimization when determining the + # optimal cookbook set for a chef-client. + # + # MinimalCookbookVersion objects contain only enough information to + # solve the cookbook collection for a given run list. They *do not* + # contain enough information to generate the response. + # + # See also: Chef::CookbookVersionSelector + class MinimalCookbookVersion + + include Comparable + + ID = "id".freeze + NAME = 'name'.freeze + KEY = 'key'.freeze + VERSION = 'version'.freeze + VALUE = 'value'.freeze + DEPS = 'deps'.freeze + + DEPENDENCIES = 'dependencies'.freeze + + # Loads the full list of cookbooks, using a couchdb view to fetch + # only the id, name, version, and dependency constraints. This is + # enough information to solve for the cookbook collection for a + # given run list. After solving for the cookbook collection, you + # need to call +load_full_versions_of+ to convert + # MinimalCookbookVersion objects to their non-minimal counterparts + def self.load_all(couchdb) + # Example: + # {"id"=>"1a806f1c-b409-4d8e-abab-fa414ff5b96d", "key"=>"activemq", "value"=>{"version"=>"0.3.3", "deps"=>{"java"=>">= 0.0.0", "runit"=>">= 0.0.0"}}} + couchdb ||= Chef::CouchDB.new + couchdb.get_view("cookbooks", "all_with_version_and_deps")["rows"].map {|params| self.new(params) } + end + + # Loads the non-minimal CookbookVersion objects corresponding to + # +minimal_cookbook_versions+ from couchdb using a bulk GET. + def self.load_full_versions_of(minimal_cookbook_versions, couchdb) + database_ids = Array(minimal_cookbook_versions).map {|mcv| mcv.couchdb_id } + couchdb ||= Chef::CouchDB.new + couchdb.bulk_get(*database_ids) + end + + attr_reader :couchdb_id + attr_reader :name + attr_reader :version + attr_reader :deps + + def initialize(params) + @couchdb_id = params[ID] + @name = params[KEY] + @version = params[VALUE][VERSION] + @deps = params[VALUE][DEPS] + end + + # Returns the Cookbook::MinimalMetadata object for this cookbook + # version. + def metadata + @metadata ||= Cookbook::MinimalMetadata.new(@name, DEPENDENCIES => @deps) + end + + def legit_version + @legit_version ||= Chef::Version.new(@version) + end + + def <=>(o) + raise Chef::Exceptions::CookbookVersionNameMismatch if self.name != o.name + raise "Unexpected comparison to #{o}" unless o.respond_to?(:legit_version) + legit_version <=> o.legit_version + end + end + # == Chef::CookbookVersion # CookbookVersion is a model object encapsulating the data about a Chef # cookbook. Chef supports maintaining multiple versions of a cookbook on a @@ -44,7 +119,7 @@ class Chef COOKBOOK_SEGMENTS = [ :resources, :providers, :recipes, :definitions, :libraries, :attributes, :files, :templates, :root_files ] DESIGN_DOCUMENT = { - "version" => 7, + "version" => 8, "language" => "javascript", "views" => { "all" => { @@ -74,6 +149,15 @@ class Chef } EOJS }, + "all_with_version_and_deps" => { + "map" => <<-JS + function(doc) { + if (doc.chef_type == "cookbook_version") { + emit(doc.cookbook_name, {version: doc.version, deps: doc.metadata.dependencies}); + } + } + JS + }, "all_latest_version" => { "map" => %q@ function(doc) { diff --git a/chef/lib/chef/cookbook_version_selector.rb b/chef/lib/chef/cookbook_version_selector.rb index cd00e08a92..9e60f85639 100644 --- a/chef/lib/chef/cookbook_version_selector.rb +++ b/chef/lib/chef/cookbook_version_selector.rb @@ -156,8 +156,13 @@ class Chef # expand any roles in this run_list. expanded_run_list = run_list.expand(environment, 'couchdb', :couchdb => couchdb).recipes.with_version_constraints - cookbooks_for_environment = Chef::Environment.cdb_load_filtered_cookbook_versions(environment, couchdb) - constrain(cookbooks_for_environment, expanded_run_list) + cookbooks_for_environment = Chef::Environment.cdb_minimal_filtered_versions(environment, couchdb) + cookbook_collection = constrain(cookbooks_for_environment, expanded_run_list) + full_cookbooks = Chef::MinimalCookbookVersion.load_full_versions_of(cookbook_collection.values, couchdb) + full_cookbooks.inject({}) do |cb_map, cookbook_version| + cb_map[cookbook_version.name] = cookbook_version + cb_map + end end end end diff --git a/chef/lib/chef/environment.rb b/chef/lib/chef/environment.rb index 252b3ca56d..cfd0af4484 100644 --- a/chef/lib/chef/environment.rb +++ b/chef/lib/chef/environment.rb @@ -355,6 +355,47 @@ class Chef sorted_list end + # Like +cdb_load_filtered_cookbook_versions+, loads the set of + # cookbooks available in a given environment. The difference is that + # this method will load Chef::MinimalCookbookVersion objects that + # contain only the information necessary for solving a cookbook + # collection for a given run list. The user of this method must call + # Chef::MinimalCookbookVersion.load_full_versions_of() after solving + # the cookbook collection to get the full objects. + # === Returns + # Hash + # i.e. + # { + # "cookbook_name" => [ Chef::CookbookVersion ... ] ## the array of CookbookVersions is sorted highest to lowest + # } + # + # There will be a key for every cookbook. If no CookbookVersions + # are available for the specified environment the value will be an + # empty list. + def self.cdb_minimal_filtered_versions(name, couchdb=nil) + version_constraints = cdb_load(name, couchdb).cookbook_versions.inject({}) {|res, (k,v)| res[k] = Chef::VersionConstraint.new(v); res} + + # inject all cookbooks into the hash while filtering out restricted versions, then sort the individual arrays + cookbook_list = Chef::MinimalCookbookVersion.load_all(couchdb) + + filtered_list = cookbook_list.inject({}) do |res, cookbook| + # FIXME: should cookbook.version return a Chef::Version? + version = Chef::Version.new(cookbook.version) + requirement_satisfied = version_constraints.has_key?(cookbook.name) ? version_constraints[cookbook.name].include?(version) : true + # we want a key for every cookbook, even if no versions are available + res[cookbook.name] ||= [] + res[cookbook.name] << cookbook if requirement_satisfied + res + end + + sorted_list = filtered_list.inject({}) do |res, (cookbook_name, versions)| + res[cookbook_name] = versions.sort.reverse + res + end + + sorted_list + end + def self.cdb_load_filtered_recipe_list(name, couchdb=nil) cdb_load_filtered_cookbook_versions(name, couchdb).map do |cb_name, cb| cb.first.recipe_filenames_by_name.keys.map do |recipe| diff --git a/chef/spec/unit/cookbook_version_spec.rb b/chef/spec/unit/cookbook_version_spec.rb index 0f37c66566..ec6fa35f06 100644 --- a/chef/spec/unit/cookbook_version_spec.rb +++ b/chef/spec/unit/cookbook_version_spec.rb @@ -17,6 +17,51 @@ require File.expand_path(File.dirname(__FILE__) + '/../spec_helper') +describe Chef::MinimalCookbookVersion do + describe "when first created" do + before do + @params = { "id"=>"1a806f1c-b409-4d8e-abab-fa414ff5b96d", + "key"=>"activemq", + "value"=>{"version"=>"0.3.3", "deps"=>{"java"=>">= 0.0.0", "runit"=>">= 0.0.0"}}} + @minimal_cookbook_version = Chef::MinimalCookbookVersion.new(@params) + end + + it "has a name" do + @minimal_cookbook_version.name.should == 'activemq' + end + + it "has a version" do + @minimal_cookbook_version.version.should == '0.3.3' + end + + it "has a list of dependencies" do + @minimal_cookbook_version.deps.should == {"java" => ">= 0.0.0", "runit" => ">= 0.0.0"} + end + + it "has cookbook metadata" do + metadata = @minimal_cookbook_version.metadata + + metadata.name.should == 'activemq' + metadata.dependencies['java'].should == '>= 0.0.0' + metadata.dependencies['runit'].should == '>= 0.0.0' + end + end + + describe "when created from cookbooks with old style version contraints" do + before do + @params = { "id"=>"1a806f1c-b409-4d8e-abab-fa414ff5b96d", + "key"=>"activemq", + "value"=>{"version"=>"0.3.3", "deps"=>{"apt" => ">> 1.0.0"}}} + @minimal_cookbook_version = Chef::MinimalCookbookVersion.new(@params) + end + + it "translates the version constraints" do + metadata = @minimal_cookbook_version.metadata + metadata.dependencies['apt'].should == '> 1.0.0' + end + end +end + describe Chef::CookbookVersion do describe "when first created" do before do diff --git a/features/api/nodes/cookbook_sync_api.feature b/features/api/nodes/cookbook_sync_api.feature index db6ac7561c..1a833ae8b1 100644 --- a/features/api/nodes/cookbook_sync_api.feature +++ b/features/api/nodes/cookbook_sync_api.feature @@ -59,7 +59,7 @@ Feature: Synchronize cookbooks to the edge When I 'GET' the path '/nodes/sync/cookbooks' Then I should get a '403 "Forbidden"' exception - @cookbook_dependencies + @cookbook_dependencies @positive Scenario: Retrieve the list of cookbooks when dependencies are resolvable Given I am an administrator And I upload the set of 'dep_test_*' cookbooks |