diff options
author | John Keiser <john@johnkeiser.com> | 2015-11-18 10:53:57 -0800 |
---|---|---|
committer | John Keiser <john@johnkeiser.com> | 2015-11-18 10:53:57 -0800 |
commit | 09cd994c9094054d5a9efe8c2f7efbdf22cd7d64 (patch) | |
tree | f0d2fd2f290dbb2f5638ed567786357e77b83fc4 | |
parent | d42285ad34b3991842b1866fbbb3511465f054bc (diff) | |
parent | c82552be06e7305cae6bf7c00ff6759888df66b3 (diff) | |
download | chef-09cd994c9094054d5a9efe8c2f7efbdf22cd7d64.tar.gz |
Merge branch 'invitations-and-members'
-rw-r--r-- | .travis.yml | 5 | ||||
-rw-r--r-- | Rakefile | 5 | ||||
-rw-r--r-- | lib/chef/chef_fs/chef_fs_data_store.rb | 183 | ||||
-rw-r--r-- | lib/chef/chef_fs/file_system/file_system_entry.rb | 16 | ||||
-rw-r--r-- | spec/support/pedant/Gemfile | 3 | ||||
-rw-r--r-- | spec/support/pedant/pedant_config.rb | 129 | ||||
-rw-r--r-- | spec/support/pedant/run_pedant.rb | 63 | ||||
-rw-r--r-- | spec/support/pedant/stickywicket.pem | 27 | ||||
-rw-r--r-- | tasks/external_tests.rb | 2 |
9 files changed, 196 insertions, 237 deletions
diff --git a/.travis.yml b/.travis.yml index 421f3fd764..44317aaeff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,8 +31,9 @@ matrix: - rvm: 2.1 - rvm: 2.2 - rvm: 2.2 - gemfile: pedant.gemfile - script: bundle exec rake pedant + cache: + env: "GEMFILE_MOD=\"gem 'chef-zero', github: 'chef/chef-zero'\"" + script: bundle exec rake chef_zero_spec - rvm: 2.2 cache: env: "GEMFILE_MOD=\"gem 'cheffish', github: 'chef/cheffish'\"" @@ -31,9 +31,7 @@ ChefConfig::PackageTask.new(File.expand_path('..', __FILE__), 'Chef') do |packag package.generate_version_class = true end -task :pedant do - require File.expand_path('spec/support/pedant/run_pedant') -end +task :pedant, :chef_zero_spec task :build_eventlog do Dir.chdir 'ext/win32-eventlog/' do @@ -62,4 +60,3 @@ begin rescue LoadError puts "yard is not available. (sudo) gem install yard to generate yard documentation." end - diff --git a/lib/chef/chef_fs/chef_fs_data_store.rb b/lib/chef/chef_fs/chef_fs_data_store.rb index 4084fb80d3..4d07135a0a 100644 --- a/lib/chef/chef_fs/chef_fs_data_store.rb +++ b/lib/chef/chef_fs/chef_fs_data_store.rb @@ -66,12 +66,65 @@ class Chef # - ChefFSDataStore lets cookbooks be uploaded into a temporary memory # storage, and when the cookbook is committed, copies the files onto the # disk in the correct place (/cookbooks/apache2/recipes/default.rb). + # # 3. Data bags: # - The Chef server expects data bags in /data/BAG/ITEM # - The repository stores data bags in /data_bags/BAG/ITEM # # 4. JSON filenames are generally NAME.json in the repository (e.g. /nodes/foo.json). # + # 5. Org membership: + # chef-zero stores user membership in an org as a series of empty files. + # If an org has jkeiser and cdoherty as members, chef-zero expects these + # files to exist: + # + # - `users/jkeiser` (content: '{}') + # - `users/cdoherty` (content: '{}') + # + # ChefFS, on the other hand, stores user membership in an org as a single + # file, `members.json`, with content: + # + # ```json + # [ + # { "user": { "username": "jkeiser" } }, + # { "user": { "username": "cdoherty" } } + # ] + # ``` + # + # To translate between the two, we need to intercept requests to `users` + # like so: + # + # - `list(users)` -> `get(/members.json)` + # - `get(users/NAME)` -> `get(/members.json)`, see if it's in there + # - `create(users/NAME)` -> `get(/members.json)`, add name, `set(/members.json)` + # - `delete(users/NAME)` -> `get(/members.json)`, remove name, `set(/members.json)` + # + # 6. Org invitations: + # chef-zero stores org membership invitations as a series of empty files. + # If an org has invited jkeiser and cdoherty (and they have not yet accepted + # the invite), chef-zero expects these files to exist: + # + # - `association_requests/jkeiser` (content: '{}') + # - `association_requests/cdoherty` (content: '{}') + # + # ChefFS, on the other hand, stores invitations as a single file, + # `invitations.json`, with content: + # + # ```json + # [ + # { "id" => "jkeiser-chef", 'username' => 'jkeiser' }, + # { "id" => "cdoherty-chef", 'username' => 'cdoherty' } + # ] + # ``` + # + # To translate between the two, we need to intercept requests to `users` + # like so: + # + # - `list(association_requests)` -> `get(/invitations.json)` + # - `get(association_requests/NAME)` -> `get(/invitations.json)`, see if it's in there + # - `create(association_requests/NAME)` -> `get(/invitations.json)`, add name, `set(/invitations.json)` + # - `delete(association_requests/NAME)` -> `get(/invitations.json)`, remove name, `set(/invitations.json)` + # class ChefFSDataStore # # Create a new ChefFSDataStore @@ -83,9 +136,10 @@ class Chef # Generally will be a +ChefFS::FileSystem::ChefRepositoryFileSystemRoot+ # object, created from +ChefFS::Config.local_fs+. # - def initialize(chef_fs) + def initialize(chef_fs, chef_config=Chef::Config) @chef_fs = chef_fs @memory_store = ChefZero::DataStore::MemoryStore.new + @repo_mode = chef_config[:repo_mode] end def publish_description @@ -93,6 +147,7 @@ class Chef end attr_reader :chef_fs + attr_reader :repo_mode def create_dir(path, name, *options) if use_memory_store?(path) @@ -108,6 +163,24 @@ class Chef end end + # + # If you want to get the contents of /data/x/y from the server, + # you say chef_fs.child('data').child('x').child('y').read. + # It will make exactly one network request: GET /data/x/y + # And that will return 404 if it doesn't exist. + # + # ChefFS objects do not go to the network until you ask them for data. + # This means you can construct a /data/x/y ChefFS entry early. + # + # Alternative: + # chef_fs.child('data') could have done a GET /data preemptively, + # allowing it to know whether child('x') was valid (GET /data gives you + # a list of data bags). Then child('x') could have done a GET /data/x, + # allowing it to know whether child('y') (the item) existed. Finally, + # we would do the GET /data/x/y to read the contents. Three network + # requests instead of 1. + # + def create(path, name, data, *options) if use_memory_store?(path) @memory_store.create(path, name, data, *options) @@ -115,6 +188,32 @@ class Chef elsif path[0] == 'cookbooks' && path.length == 2 # Do nothing. The entry gets created when the cookbook is created. + # create [/organizations/ORG]/users/NAME (with content '{}') + # Manipulate the `members.json` file that contains a list of all users + elsif is_org? && path == [ 'users' ] + update_json('members.json', []) do |members| + # Format of each entry: { "user": { "username": "jkeiser" } } + if members.any? { |member| member['user']['username'] == name } + raise ChefZero::DataStore::DataAlreadyExistsError.new(path, entry) + end + + # Actually add the user + members << { "user" => { "username" => name } } + end + + # create [/organizations/ORG]/association_requests/NAME (with content '{}') + # Manipulate the `invitations.json` file that contains a list of all users + elsif is_org? && path == [ 'association_requests' ] + update_json('invitations.json', []) do |invitations| + # Format of each entry: { "id" => "jkeiser-chef", 'username' => 'jkeiser' } + if invitations.any? { |member| member['username'] == name } + raise ChefZero::DataStore::DataAlreadyExistsError.new(path) + end + + # Actually add the user (TODO insert org name??) + invitations << { "username" => name } + end + else if !data.is_a?(String) raise "set only works with strings" @@ -142,6 +241,24 @@ class Chef raise ChefZero::DataStore::DataNotFoundError.new(to_zero_path(e.entry), e) end + # GET [/organizations/ORG]/users/NAME -> /users/NAME + # Manipulates members.json + elsif is_org? && path[0] == 'users' && path.length == 2 + if get_json('members.json', []).any? { |member| member['user']['username'] == path[1] } + '{}' + else + raise ChefZero::DataStore::DataNotFoundError.new(path) + end + + # GET [/organizations/ORG]/association_requests/NAME -> /users/NAME + # Manipulates invites.json + elsif is_org? && path[0] == 'association_requests' && path.length == 2 + if get_json('invites.json', []).any? { |member| member['user']['username'] == path[1] } + '{}' + else + raise ChefZero::DataStore::DataNotFoundError.new(path) + end + else with_entry(path) do |entry| if path[0] == 'cookbooks' && path.length == 3 @@ -209,6 +326,29 @@ class Chef def delete(path) if use_memory_store?(path) @memory_store.delete(path) + + # DELETE [/organizations/ORG]/users/NAME + # Manipulates members.json + elsif is_org? && path[0] == 'users' && path.length == 2 + update_json('members.json', []) do |members| + result = members.reject { |member| member['user']['username'] == path[1] } + if result.size == members.size + raise ChefZero::DataStore::DataNotFoundError.new(path) + end + result + end + + # DELETE [/organizations/ORG]/users/NAME + # Manipulates members.json + elsif is_org? && path[0] == 'association_requests' && path.length == 2 + update_json('invitations.json', []) do |invitations| + result = invitations.reject { |invitation| invitation['username'] == path[1] } + if result.size == invitations.size + raise ChefZero::DataStore::DataNotFoundError.new(path) + end + result + end + else with_entry(path) do |entry| begin @@ -394,9 +534,22 @@ class Chef end end end + + elsif path[0] == 'acls' + # /acls/containers|nodes|.../x.json + # /acls/organization.json + if path.length == 3 || path == [ 'acls', 'organization' ] + path = path.dup + path[-1] = "#{path[-1]}.json" + end + + # /acls/containers|nodes|... do NOT drop into the next elsif, and do + # not get .json appended + + # /nodes|clients|.../x.json elsif path.length == 2 path = path.dup - path[1] = "#{path[1]}.json" + path[-1] = "#{path[-1]}.json" end path end @@ -477,6 +630,32 @@ class Chef metadata = ChefZero::CookbookData.metadata_from(dir, path[1], nil, []) metadata[:version] || '0.0.0' end + + def update_json(path, default_value) + entry = Chef::ChefFS::FileSystem.resolve_path(chef_fs, path) + begin + input = Chef::JSONCompat.parse(entry.read) + output = yield input.dup + entry.write(Chef::JSONCompat.to_json_pretty(output)) if output != input + rescue Chef::ChefFS::FileSystem::NotFoundError + # Send the default value to the caller, and create the entry if the caller updates it + output = yield default_value + entry.parent.create_child(entry.name, Chef::JSONCompat.to_json_pretty(output)) if output != [] + end + end + + def get_json(path, default_value) + entry = Chef::ChefFS::FileSystem.resolve_path(chef_fs, path) + begin + Chef::JSONCompat.parse(entry.read) + rescue Chef::ChefFS::FileSystem::NotFoundError + default_value + end + end + + def is_org? + repo_mode == 'hosted_everything' + end end end end diff --git a/lib/chef/chef_fs/file_system/file_system_entry.rb b/lib/chef/chef_fs/file_system/file_system_entry.rb index 478631eac2..5ce8b3320a 100644 --- a/lib/chef/chef_fs/file_system/file_system_entry.rb +++ b/lib/chef/chef_fs/file_system/file_system_entry.rb @@ -72,13 +72,17 @@ class Chef end def delete(recurse) - if dir? - if !recurse - raise MustDeleteRecursivelyError.new(self, $!) + begin + if dir? + if !recurse + raise MustDeleteRecursivelyError.new(self, $!) + end + FileUtils.rm_r(file_path) + else + File.delete(file_path) end - FileUtils.rm_rf(file_path) - else - File.delete(file_path) + rescue Errno::ENOENT + raise Chef::ChefFS::FileSystem::NotFoundError.new(self, $!) end end diff --git a/spec/support/pedant/Gemfile b/spec/support/pedant/Gemfile deleted file mode 100644 index d4224cd439..0000000000 --- a/spec/support/pedant/Gemfile +++ /dev/null @@ -1,3 +0,0 @@ -source "https://rubygems.org" - -gem 'chef-pedant', :github => 'opscode/chef-pedant', :ref => "server-cli-option" diff --git a/spec/support/pedant/pedant_config.rb b/spec/support/pedant/pedant_config.rb deleted file mode 100644 index 3f8219fc59..0000000000 --- a/spec/support/pedant/pedant_config.rb +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright: Copyright (c) 2012 Opscode, Inc. -# License: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This annotated Pedant configuration file details the various -# configuration settings available to you. It is separate from the -# actual Pedant::Config class because not all settings have sane -# defaults, and not all settings are appropriate in all settings. - -################################################################################ -# You MUST specify the address of the server the API requests will be -# sent to. Only specify protocol, hostname, and port. -# NOTE this is assigned in run_pedant.rb, because it's possible 8889 will not be the port chosen. -#chef_server 'http://127.0.0.1:8889' - -# If you are doing development testing, you can specify the address of -# the Solr server. The presence of this parameter will enable tests -# to force commits to Solr, greatly decreasing the amount of time -# needed for testing the search endpoint. This is only an -# optimization for development! If you are testing a "live" Chef -# Server, or otherwise do not have access to the Solr server from your -# testing location, you should not specify a value for this parameter. -# The tests will still run, albeit slower, as they will now need to -# poll for a period to ensure they are querying committed results. -#search_server "http://localhost:8983" - -# Related to the 'search_server' parameter, this specifies the maximum -# amount of time (in seconds) that search endpoint requests should be -# retried before giving up. If not explicitly set, it will default to -# 65 seconds; only set it if you know that your Solr commit interval -# differs significantly from this. -maximum_search_time 0 - -# OSC sends erchef a host header with a port, so this option needs -# # to be enabled for Pedant tests to work correctly -explicit_port_url true - -# We're starting to break tests up into groups based on different -# criteria. The proper API tests (the results of which are viewable -# to OPC customers) should be the only ones run by Pedant embedded in -# OPC installs. There are other specs that help us keep track of API -# cruft that we want to come back and fix later; these shouldn't be -# viewable to customers, but we should be able to run them in -# development and CI environments. If this parameter is missing or -# explicitly `false` only the customer-friendly tests will be run. -# -# This is mainly here for documentation purposes, since the -# command-line `opscode-pedant` utility ultimately determines this -# value. -include_internal false - -# Test users. The five users specified below are required; their -# names (:user, :non_org_user, etc.) are indicative of their role -# within the tests. All users must have a ':name' key. If they have -# a ':create_me' key, Pedant will create these users for you. If you -# are using pre-existing users, you must supply a ':key_file' key, -# which should be the fully-qualified path /on the machine Pedant is -# running on/ to a private key for that user. -key = 'spec/support/pedant/stickywicket.pem' -superuser_name 'admin' -superuser_key key -webui_key key - -# When we updated Chef to RSpec 3 there were gem conflicts with chef-pedant. -# We removed chef as a chef-pedant gem dependency in pedant.gemfile, but this -# caused chef-pedant to fail because it could not query for the chef version -# on the box pedant is running on. X-Chef-Version isn't needed in server -# requests for these tests, so we've disabled it. -ingore_x_chef_version true - -# Set the platform_class -platform_class Pedant::OpenSourcePlatform - -requestors({ - :clients => { - # The the admin user, for the purposes of getting things rolling - :admin => { - :name => "pedant_admin_client", - :create_me => true, - :create_knife => true, - :admin => true - }, - :non_admin => { - :name => 'pedant_client', - :create_me => true, - :create_knife => true - }, - :bad => { - :name => 'bad_client', - :bogus => true - } - }, - :users => { - :admin => { - :name => "admin", - :key_file => key, - :create_me => false, - :create_knife => false, - :admin => true - }, - :non_admin => { - :name => "pedant_non_admin_user", - :create_me => true, - :create_knife => true, - :admin => false - }, - # A user for Knife tests. A knife.rb and key files will be set up - # for this user - :knife_user => { - :name => "knifey", - :create_me => true, - :create_knife => true - } - } -}) - -self[:tags] = [:validation, :authentication, :authorization] -verify_error_messages false diff --git a/spec/support/pedant/run_pedant.rb b/spec/support/pedant/run_pedant.rb deleted file mode 100644 index aac2c2df1a..0000000000 --- a/spec/support/pedant/run_pedant.rb +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env ruby -require 'bundler' -require 'bundler/setup' -require 'chef_zero/server' -require 'rspec/core' -require 'chef/chef_fs/chef_fs_data_store' -require 'chef/chef_fs/config' -require 'tmpdir' -require 'fileutils' -require 'chef/version' -require 'chef/mixin/shell_out' - -def start_server(chef_repo_path) - Dir.mkdir(chef_repo_path) if !File.exists?(chef_repo_path) - - # 11.6 and below had a bug where it couldn't create the repo children automatically - if Chef::VERSION.to_f < 11.8 - %w(clients cookbooks data_bags environments nodes roles users).each do |child| - Dir.mkdir("#{chef_repo_path}/#{child}") if !File.exists?("#{chef_repo_path}/#{child}") - end - end - - # Start the new server - Chef::Config.repo_mode = 'everything' - Chef::Config.chef_repo_path = chef_repo_path - Chef::Config.versioned_cookbooks = true - chef_fs = Chef::ChefFS::Config.new.local_fs - data_store = Chef::ChefFS::ChefFSDataStore.new(chef_fs) - server = ChefZero::Server.new(:port => 8889.upto(9999), :data_store => data_store)#, :log_level => :debug) - server.start_background - server -end - -tmpdir = Dir.mktmpdir -begin - # Create chef repository - chef_repo_path = "#{tmpdir}/repo" - - # Capture setup data into master_chef_repo_path - server = start_server(chef_repo_path) - so = nil - - include Chef::Mixin::ShellOut - - Bundler.with_clean_env do - - shell_out("bundle install --gemfile spec/support/pedant/Gemfile", :live_stream => STDOUT) - - pedant_cmd = "chef-pedant " + - " --config spec/support/pedant/pedant_config.rb" + - " --server '#{server.url}'" + - " --skip-knife --skip-validation --skip-authentication" + - " --skip-authorization --skip-omnibus" - so = shell_out("bundle exec #{pedant_cmd}", :live_stream => STDOUT, :env => {'BUNDLE_GEMFILE' => 'spec/support/pedant/Gemfile'}) - - end - -ensure - server.stop if server && server.running? - FileUtils.remove_entry_secure(tmpdir) if tmpdir -end - -exit(so.exitstatus) diff --git a/spec/support/pedant/stickywicket.pem b/spec/support/pedant/stickywicket.pem deleted file mode 100644 index ff09e73903..0000000000 --- a/spec/support/pedant/stickywicket.pem +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEpQIBAAKCAQEApNCkX2k+lFGDWRVhX4uClaVQrumG9XXvk6X7M2izrIg7RzMP -Dk4thhZkpx5gr22By7PZQdMEjWC/Zo8MBjtoJ0GV0jw8npefbU1MGKs2dtpYgo0N -Fq8fX8MdFPu4h2W3g0dMEdhT8icc2H4EjhZmdeUhUn3RIEt2duCgp3YDYnUUZx3j -N7MHcTIdzD58ikr6zQrZzHOv+OOI86Xk9EpyEEQizOLoQxkICNrhqN7ElQDuvXaX -BSBrYDRKH2umBMMcXzvsR/SvkmqxoEESSpIlW8zeKAWQ+znNjDC0tmTg7jZmgSP7 -siKrwo4t4ebjcmjpIoi/JKww/nGN3Uhz1ZOZuwIDAQABAoIBAQCaJQD2s0nyEeKU -uKhfYe155Cl3zbWJcQnmv4AXbr9MiAVY6+oS6Q8ur1bn7kNjDzoruENjiuZhC7E3 -TGZklb8tp+tluyy+7vQOmBKpp8fClSfewekR5CultqhGbb8B8yIVR+NfdUHd4rLZ -z9KWyWB+txPZQQ8L80gSmrfmpzs3IuT7oPvmtBU1Wq9QapC4n/rUohHUpUV1du4G -0wCIF4zQTg6cbYW2YXozwVQvw+P7P3RVEqZt+aZlbVcy0fNr6jNao0hi1KFC9OH2 -VjjU+PioreoA/NU3aZPIUzmJpWtsu31yuOZxXmytAkYooCZgiEQNEHnJlNPv0RmC -6BPMzVoBAoGBAM7yZoSNJpzdP/q1/4+H3zyy7o4I0VTW9u/GqUzhnbjm5poK30X9 -YXh/7WOVV0OoVqdO6ljRKygP3Oggf41ZEbi1C6bbsO57pksBWgx9bD9V35XscZ0J -F1ERe//kMHwVQy74R8/cIuRwm75haLSBj5/fwGbLeeVDglJkCVqPjtuBAoGBAMvh -qsAGG5k9u6voTcXlFwS+B5YjULhK4NSxdJ2BnOxzYzxQ3IYQZMlb2xt8yZYx/ZZK -wjkr9rcAPEQIQZ2A6NUbGq6qCD7sSmg6UAi0CgiqTokQ/Wtag0UDvFMzwerdg/On -37uxffpxpte8z1jYi/MxRaoTYueuc1UVnqofVIM7AoGBALZJzwPzUY/bVAADUJmd -lYZiFsAGBF42/E05MOgH1GaK/ZWy/fkouDLsfK67XaK7JZk6ajLSDLG9R1kxRym6 -y2FoGFtiKPfo8xIenrNhx3gCrG/jVjB9UYyXWiKNXifukr9M8/SkdBfFGWsZYqGd -fmXVMiVaFoVcce8hLxwWWEABAoGBAKcyhKX/HEj6YFqlIoqkydDAylXs1jicZ27l -rF2yum8KXZpMMdzbutuKsdAD8Ql0K6NB4a+jByuiTMn5/11cJxUEqkgM9sArZQW+ -tH2+r+/VQpyTS0/rpXVGj/2nl2K1kI2T4R36e/aTl6CanWweAf9JK/lC9rxKyxg+ -p6SaFuObAoGACP6TKCkp2oymXlKgdUUgPrnsaz2VAw8jD5QHtx10U4wty0C8gxsk -MLe00h09iLPyFmvJpD+MgbxV/r6RrZeVdsKdU/5LG52YgiVSTaizyy+ciEfW7xoQ -CL5EtZd8Cn5OKinBEzzFpELqunlqepIKCIDOcLKz/cjR+3a+E6Zx5Wo= ------END RSA PRIVATE KEY----- diff --git a/tasks/external_tests.rb b/tasks/external_tests.rb index 91fa249824..856d1c502f 100644 --- a/tasks/external_tests.rb +++ b/tasks/external_tests.rb @@ -40,7 +40,7 @@ def bundle_exec_with_chef(test_gem, commands) end EXTERNAL_PROJECTS = { - "chef-zero" => [ "rake spec", "rake pedant" ], + "chef-zero" => [ "rake spec", "rake cheffs" ], "cheffish" => "rake spec", "chef-provisioning" => "rake spec", "chef-provisioning-aws" => "rake spec", |