[CHEF-2200] only load the data we actually need for cookbook solving

6 files changed, 256 insertions, 45 deletions

diff --git a/chef/lib/chef/cookbook/metadata.rb b/chef/lib/chef/cookbook/metadata.rb
index 9e5e3b4dc1..407a29ddaa 100644
--- a/chef/lib/chef/cookbook/metadata.rb
+++ b/chef/lib/chef/cookbook/metadata.rb
@@ -27,22 +27,41 @@ require 'chef/version_constraint'
 
 class Chef
   class Cookbook
+
     # == Chef::Cookbook::Metadata
     # Chef::Cookbook::Metadata provides a convenient DSL for declaring metadata
     # about Chef Cookbooks.
     class Metadata
 
+      NAME              = 'name'.freeze
+      DESCRIPTION       = 'description'.freeze
+      LONG_DESCRIPTION  = 'long_description'.freeze
+      MAINTAINER        = 'maintainer'.freeze
+      MAINTAINER_EMAIL  = 'maintainer_email'.freeze
+      LICENSE           = 'license'.freeze
+      PLATFORMS         = 'platforms'.freeze
+      DEPENDENCIES      = 'dependencies'.freeze
+      RECOMMENDATIONS   = 'recommendations'.freeze
+      SUGGESTIONS       = 'suggestions'.freeze
+      CONFLICTING       = 'conflicting'.freeze
+      PROVIDING         = 'providing'.freeze
+      REPLACING         = 'replacing'.freeze
+      ATTRIBUTES        = 'attributes'.freeze
+      GROUPINGS         = 'groupings'.freeze
+      RECIPES           = 'recipes'.freeze
+      VERSION           = 'version'.freeze
+
       COMPARISON_FIELDS = [ :name, :description, :long_description, :maintainer,
                             :maintainer_email, :license, :platforms, :dependencies,
                             :recommendations, :suggestions, :conflicting, :providing,
                             :replacing, :attributes, :groupings, :recipes, :version]
 
-      VERSION_CONSTRAINTS = {:depends     => "dependencies",
-                             :recommends  => "recommendations",
-                             :suggests    => "suggestions",
-                             :conflicts   => "conflicting",
-                             :provides    => "providing",
-                             :replaces    => "replacing" }
+      VERSION_CONSTRAINTS = {:depends     => DEPENDENCIES,
+                             :recommends  => RECOMMENDATIONS,
+                             :suggests    => SUGGESTIONS,
+                             :conflicts   => CONFLICTING,
+                             :provides    => PROVIDING,
+                             :replaces    => REPLACING }
 
       include Chef::Mixin::CheckHelper
       include Chef::Mixin::ParamsValidate
@@ -402,23 +421,23 @@ class Chef
 
       def to_hash
         {
-          'name'             => self.name,
-          'description'      => self.description,
-          'long_description' => self.long_description,
-          'maintainer'       => self.maintainer,
-          'maintainer_email' => self.maintainer_email,
-          'license'          => self.license,
-          'platforms'        => self.platforms,
-          'dependencies'     => self.dependencies,
-          'recommendations'  => self.recommendations,
-          'suggestions'      => self.suggestions,
-          'conflicting'      => self.conflicting,
-          'providing'        => self.providing,
-          'replacing'        => self.replacing,
-          'attributes'       => self.attributes,
-          'groupings'        => self.groupings,
-          'recipes'          => self.recipes,
-          'version'          => self.version
+          NAME             => self.name,
+          DESCRIPTION      => self.description,
+          LONG_DESCRIPTION => self.long_description,
+          MAINTAINER       => self.maintainer,
+          MAINTAINER_EMAIL => self.maintainer_email,
+          LICENSE          => self.license,
+          PLATFORMS        => self.platforms,
+          DEPENDENCIES     => self.dependencies,
+          RECOMMENDATIONS  => self.recommendations,
+          SUGGESTIONS      => self.suggestions,
+          CONFLICTING      => self.conflicting,
+          PROVIDING        => self.providing,
+          REPLACING        => self.replacing,
+          ATTRIBUTES       => self.attributes,
+          GROUPINGS        => self.groupings,
+          RECIPES          => self.recipes,
+          VERSION          => self.version
         }
       end
 
@@ -433,23 +452,23 @@ class Chef
       end
 
       def from_hash(o)
-        @name                         = o['name'] if o.has_key?('name')
-        @description                  = o['description'] if o.has_key?('description')
-        @long_description             = o['long_description'] if o.has_key?('long_description')
-        @maintainer                   = o['maintainer'] if o.has_key?('maintainer')
-        @maintainer_email             = o['maintainer_email'] if o.has_key?('maintainer_email')
-        @license                      = o['license'] if o.has_key?('license')
-        @platforms                    = o['platforms'] if o.has_key?('platforms')
-        @dependencies                 = handle_deprecated_constraints(o['dependencies']) if o.has_key?('dependencies')
-        @recommendations              = handle_deprecated_constraints(o['recommendations']) if o.has_key?('recommendations')
-        @suggestions                  = handle_deprecated_constraints(o['suggestions']) if o.has_key?('suggestions')
-        @conflicting                  = handle_deprecated_constraints(o['conflicting']) if o.has_key?('conflicting')
-        @providing                    = o['providing'] if o.has_key?('providing')
-        @replacing                    = handle_deprecated_constraints(o['replacing']) if o.has_key?('replacing')
-        @attributes                   = o['attributes'] if o.has_key?('attributes')
-        @groupings                    = o['groupings'] if o.has_key?('groupings')
-        @recipes                      = o['recipes'] if o.has_key?('recipes')
-        @version                      = o['version'] if o.has_key?('version')
+        @name                         = o[NAME] if o.has_key?(NAME)
+        @description                  = o[DESCRIPTION] if o.has_key?(DESCRIPTION)
+        @long_description             = o[LONG_DESCRIPTION] if o.has_key?(LONG_DESCRIPTION)
+        @maintainer                   = o[MAINTAINER] if o.has_key?(MAINTAINER)
+        @maintainer_email             = o[MAINTAINER_EMAIL] if o.has_key?(MAINTAINER_EMAIL)
+        @license                      = o[LICENSE] if o.has_key?(LICENSE)
+        @platforms                    = o[PLATFORMS] if o.has_key?(PLATFORMS)
+        @dependencies                 = handle_deprecated_constraints(o[DEPENDENCIES]) if o.has_key?(DEPENDENCIES)
+        @recommendations              = handle_deprecated_constraints(o[RECOMMENDATIONS]) if o.has_key?(RECOMMENDATIONS)
+        @suggestions                  = handle_deprecated_constraints(o[SUGGESTIONS]) if o.has_key?(SUGGESTIONS)
+        @conflicting                  = handle_deprecated_constraints(o[CONFLICTING]) if o.has_key?(CONFLICTING)
+        @providing                    = o[PROVIDING] if o.has_key?(PROVIDING)
+        @replacing                    = handle_deprecated_constraints(o[REPLACING]) if o.has_key?(REPLACING)
+        @attributes                   = o[ATTRIBUTES] if o.has_key?(ATTRIBUTES)
+        @groupings                    = o[GROUPINGS] if o.has_key?(GROUPINGS)
+        @recipes                      = o[RECIPES] if o.has_key?(RECIPES)
+        @version                      = o[VERSION] if o.has_key?(VERSION)
         self
       end
 
@@ -588,5 +607,22 @@ INVALID
       end
 
     end
+
+    #== Chef::Cookbook::MinimalMetadata
+    # MinimalMetadata is a duck type of Cookbook::Metadata, used
+    # internally by Chef Server when determining the optimal set of
+    # cookbooks for a node.
+    #
+    # MinimalMetadata objects typically contain only enough information
+    # to solve the cookbook collection for a run list, but not enough to
+    # generate the proper response
+    class MinimalMetadata < Metadata
+      def initialize(name, params)
+        @name = name
+        from_hash(params)
+      end
+    end
+
+
   end
 end
diff --git a/chef/lib/chef/cookbook_version.rb b/chef/lib/chef/cookbook_version.rb
index a692c32a16..45ac21c6ef 100644
--- a/chef/lib/chef/cookbook_version.rb
+++ b/chef/lib/chef/cookbook_version.rb
@@ -3,7 +3,8 @@
 # Author:: Christopher Walters (<cw@opscode.com>)
 # Author:: Tim Hinderliter (<tim@opscode.com>)
 # Author:: Seth Falcon (<seth@opscode.com>)
-# Copyright:: Copyright 2008-2010 Opscode, Inc.
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Copyright:: Copyright 2008-2011 Opscode, Inc.
 # License:: Apache License, Version 2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -29,6 +30,80 @@ require 'chef/cookbook/metadata'
 require 'chef/version_class'
 
 class Chef
+
+  #== Chef::MinimalCookbookVersion
+  # MinimalCookbookVersion is a duck type of CookbookVersion, used
+  # internally by Chef Server as an optimization when determining the
+  # optimal cookbook set for a chef-client.
+  #
+  # MinimalCookbookVersion objects contain only enough information to
+  # solve the cookbook collection for a given run list. They *do not*
+  # contain enough information to generate the response.
+  #
+  # See also: Chef::CookbookVersionSelector
+  class MinimalCookbookVersion
+
+    include Comparable
+
+    ID   = "id".freeze
+    NAME = 'name'.freeze
+    KEY  = 'key'.freeze
+    VERSION = 'version'.freeze
+    VALUE   = 'value'.freeze
+    DEPS    = 'deps'.freeze
+
+    DEPENDENCIES      = 'dependencies'.freeze
+
+    # Loads the full list of cookbooks, using a couchdb view to fetch
+    # only the id, name, version, and dependency constraints. This is
+    # enough information to solve for the cookbook collection for a
+    # given run list. After solving for the cookbook collection, you
+    # need to call +load_full_versions_of+ to convert
+    # MinimalCookbookVersion objects to their non-minimal counterparts
+    def self.load_all(couchdb)
+      # Example:
+      # {"id"=>"1a806f1c-b409-4d8e-abab-fa414ff5b96d", "key"=>"activemq", "value"=>{"version"=>"0.3.3", "deps"=>{"java"=>">= 0.0.0", "runit"=>">= 0.0.0"}}}
+      couchdb ||= Chef::CouchDB.new
+      couchdb.get_view("cookbooks", "all_with_version_and_deps")["rows"].map {|params| self.new(params) }
+    end
+
+    # Loads the non-minimal CookbookVersion objects corresponding to
+    # +minimal_cookbook_versions+ from couchdb using a bulk GET.
+    def self.load_full_versions_of(minimal_cookbook_versions, couchdb)
+      database_ids = Array(minimal_cookbook_versions).map {|mcv| mcv.couchdb_id }
+      couchdb ||= Chef::CouchDB.new
+      couchdb.bulk_get(*database_ids)
+    end
+
+    attr_reader :couchdb_id
+    attr_reader :name
+    attr_reader :version
+    attr_reader :deps
+
+    def initialize(params)
+      @couchdb_id = params[ID]
+      @name = params[KEY]
+      @version = params[VALUE][VERSION]
+      @deps    = params[VALUE][DEPS]
+    end
+
+    # Returns the Cookbook::MinimalMetadata object for this cookbook
+    # version.
+    def metadata
+      @metadata ||= Cookbook::MinimalMetadata.new(@name, DEPENDENCIES => @deps)
+    end
+
+    def legit_version
+      @legit_version ||= Chef::Version.new(@version)
+    end
+
+    def <=>(o)
+      raise Chef::Exceptions::CookbookVersionNameMismatch if self.name != o.name
+      raise "Unexpected comparison to #{o}" unless o.respond_to?(:legit_version)
+      legit_version <=> o.legit_version
+    end
+  end
+
   # == Chef::CookbookVersion
   # CookbookVersion is a model object encapsulating the data about a Chef
   # cookbook. Chef supports maintaining multiple versions of a cookbook on a
@@ -44,7 +119,7 @@ class Chef
     COOKBOOK_SEGMENTS = [ :resources, :providers, :recipes, :definitions, :libraries, :attributes, :files, :templates, :root_files ]
 
     DESIGN_DOCUMENT = {
-      "version" => 7,
+      "version" => 8,
       "language" => "javascript",
       "views" => {
         "all" => {
@@ -74,6 +149,15 @@ class Chef
           }
           EOJS
         },
+        "all_with_version_and_deps" => {
+          "map" => <<-JS
+          function(doc) {
+            if (doc.chef_type == "cookbook_version") {
+              emit(doc.cookbook_name, {version: doc.version, deps: doc.metadata.dependencies});
+            }
+          }
+          JS
+        },
         "all_latest_version" => {
           "map" => %q@
           function(doc) {
diff --git a/chef/lib/chef/cookbook_version_selector.rb b/chef/lib/chef/cookbook_version_selector.rb
index cd00e08a92..9e60f85639 100644
--- a/chef/lib/chef/cookbook_version_selector.rb
+++ b/chef/lib/chef/cookbook_version_selector.rb
@@ -156,8 +156,13 @@ class Chef
       # expand any roles in this run_list.
       expanded_run_list = run_list.expand(environment, 'couchdb', :couchdb => couchdb).recipes.with_version_constraints
 
-      cookbooks_for_environment = Chef::Environment.cdb_load_filtered_cookbook_versions(environment, couchdb)
-      constrain(cookbooks_for_environment, expanded_run_list)
+      cookbooks_for_environment = Chef::Environment.cdb_minimal_filtered_versions(environment, couchdb)
+      cookbook_collection = constrain(cookbooks_for_environment, expanded_run_list)
+      full_cookbooks = Chef::MinimalCookbookVersion.load_full_versions_of(cookbook_collection.values, couchdb)
+      full_cookbooks.inject({}) do |cb_map, cookbook_version|
+        cb_map[cookbook_version.name] = cookbook_version
+        cb_map
+      end
     end
   end
 end
diff --git a/chef/lib/chef/environment.rb b/chef/lib/chef/environment.rb
index 252b3ca56d..cfd0af4484 100644
--- a/chef/lib/chef/environment.rb
+++ b/chef/lib/chef/environment.rb
@@ -355,6 +355,47 @@ class Chef
       sorted_list
     end
 
+    # Like +cdb_load_filtered_cookbook_versions+, loads the set of
+    # cookbooks available in a given environment. The difference is that
+    # this method will load Chef::MinimalCookbookVersion objects that
+    # contain only the information necessary for solving a cookbook
+    # collection for a given run list. The user of this method must call
+    # Chef::MinimalCookbookVersion.load_full_versions_of() after solving
+    # the cookbook collection to get the full objects.
+    # === Returns
+    # Hash
+    # i.e.
+    # {
+    #   "cookbook_name" => [ Chef::CookbookVersion ... ] ## the array of CookbookVersions is sorted highest to lowest
+    # }
+    #
+    # There will be a key for every cookbook.  If no CookbookVersions
+    # are available for the specified environment the value will be an
+    # empty list.
+    def self.cdb_minimal_filtered_versions(name, couchdb=nil)
+      version_constraints = cdb_load(name, couchdb).cookbook_versions.inject({}) {|res, (k,v)| res[k] = Chef::VersionConstraint.new(v); res}
+
+      # inject all cookbooks into the hash while filtering out restricted versions, then sort the individual arrays
+      cookbook_list = Chef::MinimalCookbookVersion.load_all(couchdb)
+
+      filtered_list = cookbook_list.inject({}) do |res, cookbook|
+        # FIXME: should cookbook.version return a Chef::Version?
+        version               = Chef::Version.new(cookbook.version)
+        requirement_satisfied = version_constraints.has_key?(cookbook.name) ? version_constraints[cookbook.name].include?(version) : true
+        # we want a key for every cookbook, even if no versions are available
+        res[cookbook.name] ||= []
+        res[cookbook.name] << cookbook if requirement_satisfied
+        res
+      end
+
+      sorted_list = filtered_list.inject({}) do |res, (cookbook_name, versions)|
+        res[cookbook_name] = versions.sort.reverse
+        res
+      end
+
+      sorted_list
+    end
+
     def self.cdb_load_filtered_recipe_list(name, couchdb=nil)
       cdb_load_filtered_cookbook_versions(name, couchdb).map do |cb_name, cb|
         cb.first.recipe_filenames_by_name.keys.map do |recipe|
diff --git a/chef/spec/unit/cookbook_version_spec.rb b/chef/spec/unit/cookbook_version_spec.rb
index 0f37c66566..ec6fa35f06 100644
--- a/chef/spec/unit/cookbook_version_spec.rb
+++ b/chef/spec/unit/cookbook_version_spec.rb
@@ -17,6 +17,51 @@
 
 require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
 
+describe Chef::MinimalCookbookVersion do
+  describe "when first created" do
+    before do
+      @params = { "id"=>"1a806f1c-b409-4d8e-abab-fa414ff5b96d",
+                  "key"=>"activemq",
+                  "value"=>{"version"=>"0.3.3", "deps"=>{"java"=>">= 0.0.0", "runit"=>">= 0.0.0"}}}
+      @minimal_cookbook_version = Chef::MinimalCookbookVersion.new(@params)
+    end
+
+    it "has a name" do
+      @minimal_cookbook_version.name.should == 'activemq'
+    end
+
+    it "has a version" do
+      @minimal_cookbook_version.version.should == '0.3.3'
+    end
+
+    it "has a list of dependencies" do
+      @minimal_cookbook_version.deps.should == {"java" => ">= 0.0.0", "runit" => ">= 0.0.0"}
+    end
+
+    it "has cookbook metadata" do
+      metadata = @minimal_cookbook_version.metadata
+
+      metadata.name.should == 'activemq'
+      metadata.dependencies['java'].should == '>= 0.0.0'
+      metadata.dependencies['runit'].should == '>= 0.0.0'
+    end
+  end
+
+  describe "when created from cookbooks with old style version contraints" do
+    before do
+      @params = { "id"=>"1a806f1c-b409-4d8e-abab-fa414ff5b96d",
+                  "key"=>"activemq",
+                  "value"=>{"version"=>"0.3.3", "deps"=>{"apt" => ">> 1.0.0"}}}
+      @minimal_cookbook_version = Chef::MinimalCookbookVersion.new(@params)
+    end
+
+    it "translates the version constraints" do
+      metadata = @minimal_cookbook_version.metadata
+      metadata.dependencies['apt'].should == '> 1.0.0'
+    end
+  end
+end
+
 describe Chef::CookbookVersion do
   describe "when first created" do
     before do
diff --git a/features/api/nodes/cookbook_sync_api.feature b/features/api/nodes/cookbook_sync_api.feature
index db6ac7561c..1a833ae8b1 100644
--- a/features/api/nodes/cookbook_sync_api.feature
+++ b/features/api/nodes/cookbook_sync_api.feature
@@ -59,7 +59,7 @@ Feature: Synchronize cookbooks to the edge
      When I 'GET' the path '/nodes/sync/cookbooks'
      Then I should get a '403 "Forbidden"' exception
 
-  @cookbook_dependencies
+  @cookbook_dependencies @positive
   Scenario: Retrieve the list of cookbooks when dependencies are resolvable
     Given I am an administrator
       And I upload the set of 'dep_test_*' cookbooks