diff options
author | Lamont Granquist <lamont@scriptkiddie.org> | 2017-01-11 13:56:52 -0800 |
---|---|---|
committer | GitHub <noreply@github.com> | 2017-01-11 13:56:52 -0800 |
commit | 00e7bb54cd90e9080f24e19d3beca604fa070afc (patch) | |
tree | 1b0bf2046c655a730e01d0e5b35f2ad8a54ee71a /lib/chef/provider | |
parent | c79ed56d52ecff6452740e14ebb9546f3baf399e (diff) | |
parent | aebaee14c2d6c106914d1b3e3c9c3cbdf5bc29cb (diff) | |
download | chef-00e7bb54cd90e9080f24e19d3beca604fa070afc.tar.gz |
Merge pull request #4894 from chef/lcg/dnf-provider
DNF Provider PR #2
Diffstat (limited to 'lib/chef/provider')
-rw-r--r-- | lib/chef/provider/package.rb | 19 | ||||
-rw-r--r-- | lib/chef/provider/package/dnf.rb | 183 | ||||
-rw-r--r-- | lib/chef/provider/package/dnf/dnf_helper.py | 91 | ||||
-rw-r--r-- | lib/chef/provider/package/dnf/python_helper.rb | 120 | ||||
-rw-r--r-- | lib/chef/provider/package/dnf/version.rb | 56 |
5 files changed, 464 insertions, 5 deletions
diff --git a/lib/chef/provider/package.rb b/lib/chef/provider/package.rb index ecf3dbecb5..f52614672a 100644 --- a/lib/chef/provider/package.rb +++ b/lib/chef/provider/package.rb @@ -37,6 +37,9 @@ class Chef subclass_directive :use_multipackage_api # subclasses declare this if they want sources (filenames) pulled from their package names subclass_directive :use_package_name_for_source + # keeps package_names_for_targets and versions_for_targets indexed the same as package_name at + # the cost of having the subclass needing to deal with nils + subclass_directive :allow_nils # # Hook that subclasses use to populate the candidate_version(s) @@ -390,9 +393,12 @@ class Chef def package_names_for_targets package_names_for_targets = [] target_version_array.each_with_index do |target_version, i| - next if target_version.nil? - package_name = package_name_array[i] - package_names_for_targets.push(package_name) + if !target_version.nil? + package_name = package_name_array[i] + package_names_for_targets.push(package_name) + else + package_names_for_targets.push(nil) if allow_nils? + end end multipackage? ? package_names_for_targets : package_names_for_targets[0] end @@ -407,8 +413,11 @@ class Chef def versions_for_targets versions_for_targets = [] target_version_array.each_with_index do |target_version, i| - next if target_version.nil? - versions_for_targets.push(target_version) + if !target_version.nil? + versions_for_targets.push(target_version) + else + versions_for_targets.push(nil) if allow_nils? + end end multipackage? ? versions_for_targets : versions_for_targets[0] end diff --git a/lib/chef/provider/package/dnf.rb b/lib/chef/provider/package/dnf.rb new file mode 100644 index 0000000000..bf6aa2438f --- /dev/null +++ b/lib/chef/provider/package/dnf.rb @@ -0,0 +1,183 @@ +# +# Copyright:: Copyright 2016, Chef Software, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require "chef/provider/package" +require "chef/resource/dnf_package" +require "chef/mixin/which" +require "chef/mixin/get_source_from_package" +require "chef/provider/package/dnf/python_helper" +require "chef/provider/package/dnf/version" + +class Chef + class Provider + class Package + class Dnf < Chef::Provider::Package + extend Chef::Mixin::Which + include Chef::Mixin::GetSourceFromPackage + + allow_nils + use_multipackage_api + use_package_name_for_source + + provides :package, platform_family: %w{rhel fedora} do + which("dnf") + end + + provides :dnf_package, os: "linux" + + # + # Most of the magic in this class happens in the python helper script. The ruby side of this + # provider knows only enough to translate Chef-style new_resource name+package+version into + # a request to the python side. The python side is then responsible for knowing everything + # about RPMs and what is installed and what is available. The ruby side of this class should + # remain a lightweight translation layer to translate Chef requests into RPC requests to + # python. This class knows nothing about how to compare RPM versions, and does not maintain + # any cached state of installed/available versions and should be kept that way. + # + def python_helper + @python_helper ||= PythonHelper.instance + end + + def load_current_resource + flushcache if new_resource.flush_cache[:before] + + @current_resource = Chef::Resource::DnfPackage.new(new_resource.name) + current_resource.package_name(new_resource.package_name) + current_resource.version(get_current_versions) + + current_resource + end + + def define_resource_requirements + requirements.assert(:install, :upgrade, :remove, :purge) do |a| + a.assertion { !new_resource.source || ::File.exist?(new_resource.source) } + a.failure_message Chef::Exceptions::Package, "Package #{new_resource.package_name} not found: #{new_resource.source}" + a.whyrun "assuming #{new_resource.source} would have previously been created" + end + + super + end + + def candidate_version + package_name_array.each_with_index.map do |pkg, i| + available_version(i).version_with_arch + end + end + + def get_current_versions + package_name_array.each_with_index.map do |pkg, i| + installed_version(i).version_with_arch + end + end + + def install_package(names, versions) + if new_resource.source + dnf(new_resource.options, "-y install", new_resource.source) + else + resolved_names = names.each_with_index.map { |name, i| available_version(i).to_s unless name.nil? } + dnf(new_resource.options, "-y install", resolved_names) + end + flushcache + end + + # dnf upgrade does not work on uninstalled packaged, while install will upgrade + alias_method :upgrade_package, :install_package + + def remove_package(names, versions) + resolved_names = names.each_with_index.map { |name, i| installed_version(i).to_s unless name.nil? } + dnf(new_resource.options, "-y remove", resolved_names) + flushcache + end + + alias_method :purge_package, :remove_package + + action :flush_cache do + flushcache + end + + private + + def resolve_source_to_version_obj + shell_out_with_timeout!("rpm -qp --queryformat '%{NAME} %{EPOCH} %{VERSION} %{RELEASE} %{ARCH}\n' #{new_resource.source}").stdout.each_line do |line| + # this is another case of committing the sin of doing some lightweight mangling of RPM versions in ruby -- but the output of the rpm command + # does not match what the dnf library accepts. + case line + when /^(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)$/ + return Version.new($1, "#{$2 == "(none)" ? "0" : $2}:#{$3}-#{$4}", $5) + end + end + end + + # @returns Array<Version> + def available_version(index) + @available_version ||= [] + + if new_resource.source + @available_version[index] ||= resolve_source_to_version_obj + else + @available_version[index] ||= python_helper.query(:whatavailable, package_name_array[index], safe_version_array[index], safe_arch_array[index]) + end + + @available_version[index] + end + + # @returns Array<Version> + def installed_version(index) + @installed_version ||= [] + if new_resource.source + @installed_version[index] ||= python_helper.query(:whatinstalled, available_version(index).name, safe_version_array[index], safe_arch_array[index]) + else + @installed_version[index] ||= python_helper.query(:whatinstalled, package_name_array[index], safe_version_array[index], safe_arch_array[index]) + end + @installed_version[index] + end + + # cache flushing is accomplished by simply restarting the python helper. this produces a roughly + # 15% hit to the runtime of installing/removing/upgrading packages. correctly using multipackage + # array installs (and the multipackage cookbook) can produce 600% improvements in runtime. + def flushcache + python_helper.restart + end + + def dnf(*args) + shell_out_with_timeout!(a_to_s("dnf", *args)) + end + + def safe_version_array + if new_resource.version.is_a?(Array) + new_resource.version + elsif new_resource.version.nil? + package_name_array.map { nil } + else + [ new_resource.version ] + end + end + + def safe_arch_array + if new_resource.arch.is_a?(Array) + new_resource.arch + elsif new_resource.arch.nil? + package_name_array.map { nil } + else + [ new_resource.arch ] + end + end + + end + end + end +end diff --git a/lib/chef/provider/package/dnf/dnf_helper.py b/lib/chef/provider/package/dnf/dnf_helper.py new file mode 100644 index 0000000000..236b967710 --- /dev/null +++ b/lib/chef/provider/package/dnf/dnf_helper.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4 + +import sys +import dnf +import hawkey +import signal +import os +import json + +base = None + +def get_sack(): + global base + if base is None: + base = dnf.Base() + base.read_all_repos() + base.fill_sack() + return base.sack + +# FIXME: leaks memory and does not work +def flushcache(): + try: + os.remove('/var/cache/dnf/@System.solv') + except OSError: + pass + get_sack().load_system_repo(build_cache=True) + +def query(command): + sack = get_sack() + + subj = dnf.subject.Subject(command['provides']) + q = subj.get_best_query(sack, with_provides=True) + + if command['action'] == "whatinstalled": + q = q.installed() + + if command['action'] == "whatavailable": + q = q.available() + + if 'epoch' in command: + q = q.filterm(epoch=int(command['epoch'])) + if 'version' in command: + q = q.filterm(version__glob=command['version']) + if 'release' in command: + q = q.filterm(release__glob=command['release']) + + if 'arch' in command: + q = q.filterm(arch__glob=command['arch']) + + # only apply the default arch query filter if it returns something + archq = q.filter(arch=[ 'noarch', hawkey.detect_arch() ]) + if len(archq.run()) > 0: + q = archq + + pkgs = dnf.query.latest_limit_pkgs(q, 1) + + if not pkgs: + sys.stdout.write('{} nil nil\n'.format(command['provides'].split().pop(0))) + else: + # make sure we picked the package with the highest version + pkgs.sort + pkg = pkgs.pop() + sys.stdout.write('{} {}:{}-{} {}\n'.format(pkg.name, pkg.epoch, pkg.version, pkg.release, pkg.arch)) + +# the design of this helper is that it should try to be 'brittle' and fail hard and exit in order +# to keep process tables clean. additional error handling should probably be added to the retry loop +# on the ruby side. +def exit_handler(signal, frame): + sys.exit(0) + +signal.signal(signal.SIGINT, exit_handler) +signal.signal(signal.SIGHUP, exit_handler) +signal.signal(signal.SIGPIPE, exit_handler) +signal.signal(signal.SIGCHLD, exit_handler) + +while 1: + # kill self if we get orphaned (tragic) + ppid = os.getppid() + if ppid == 1: + sys.exit(0) + line = sys.stdin.readline() + command = json.loads(line) + if command['action'] == "whatinstalled": + query(command) + elif command['action'] == "whatavailable": + query(command) + elif command['action'] == "flushcache": + flushcache() + else: + raise RuntimeError("bad command") diff --git a/lib/chef/provider/package/dnf/python_helper.rb b/lib/chef/provider/package/dnf/python_helper.rb new file mode 100644 index 0000000000..466114b339 --- /dev/null +++ b/lib/chef/provider/package/dnf/python_helper.rb @@ -0,0 +1,120 @@ +# +# Copyright:: Copyright 2016, Chef Software, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require "chef/provider/package/dnf/version" +require "timeout" + +class Chef + class Provider + class Package + class Dnf < Chef::Provider::Package + class PythonHelper + include Singleton + extend Chef::Mixin::Which + + attr_accessor :stdin + attr_accessor :stdout + attr_accessor :stderr + attr_accessor :wait_thr + + DNF_HELPER = ::File.expand_path(::File.join(::File.dirname(__FILE__), "dnf_helper.py")).freeze + DNF_COMMAND = "#{which("python3")} #{DNF_HELPER}" + + def start + ENV["PYTHONUNBUFFERED"] = "1" + @stdin, @stdout, @stderr, @wait_thr = Open3.popen3(DNF_COMMAND) + end + + def reap + unless wait_thr.nil? + Process.kill("KILL", wait_thr.pid) rescue nil + stdin.close unless stdin.nil? + stdout.close unless stdout.nil? + stderr.close unless stderr.nil? + wait_thr.value # this calls waitpit() + end + end + + def check + start if stdin.nil? + end + + # i couldn't figure out how to decompose an evr on the python side, it seems reasonably + # painless to do it in ruby (generally massaging nevras in the ruby side is HIGHLY + # discouraged -- this is an "every rule has an exception" exception -- any additional + # functionality should probably trigger moving this regexp logic into python) + def add_version(hash, version) + epoch = nil + if version =~ /(\S+):(\S+)/ + epoch, version = $1, $2 + end + if version =~ /(\S+)-(\S+)/ + version, release = $1, $2 + end + hash["epoch"] = epoch unless epoch.nil? + hash["release"] = release unless release.nil? + hash["version"] = version + end + + def build_query(action, provides, version, arch) + hash = { "action" => action } + hash["provides"] = provides + add_version(hash, version) unless version.nil? + hash["arch" ] = arch unless arch.nil? + FFI_Yajl::Encoder.encode(hash) + end + + def parse_response(output) + array = output.split.map { |x| x == "nil" ? nil : x } + array.each_slice(3).map { |x| Version.new(*x) }.first + end + + # @returns Array<Version> + def query(action, provides, version = nil, arch = nil) + with_helper do + json = build_query(action, provides, version, arch) + Chef::Log.debug "sending '#{json}' to python helper" + stdin.syswrite json + "\n" + output = stdout.sysread(4096).chomp + Chef::Log.debug "got '#{output}' from python helper" + version = parse_response(output) + Chef::Log.debug "parsed #{version} from python helper" + version + end + end + + def restart + reap + start + end + + def with_helper + max_retries ||= 5 + Timeout.timeout(600) do + check + yield + end + rescue EOFError, Errno::EPIPE, Timeout::Error, Errno::ESRCH => e + raise e unless ( max_retries -= 1 ) > 0 + restart + retry + end + end + end + end + end +end diff --git a/lib/chef/provider/package/dnf/version.rb b/lib/chef/provider/package/dnf/version.rb new file mode 100644 index 0000000000..b326913c3a --- /dev/null +++ b/lib/chef/provider/package/dnf/version.rb @@ -0,0 +1,56 @@ +# +# Copyright:: Copyright 2016, Chef Software, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +class Chef + class Provider + class Package + class Dnf < Chef::Provider::Package + + # helper class to assist in passing around name/version/arch triples + class Version + attr_accessor :name + attr_accessor :version + attr_accessor :arch + + def initialize(name, version, arch) + @name = name + @version = version + @arch = arch + end + + def to_s + "#{name}-#{version}.#{arch}" + end + + def version_with_arch + "#{version}.#{arch}" unless version.nil? + end + + def matches_name_and_arch?(other) + other.version == version && other.arch == arch + end + + def ==(other) + name == other.name && version == other.version && arch == other.arch + end + + alias_method :eql?, :== + end + end + end + end +end |