Merge pull request #4894 from chef/lcg/dnf-provider

DNF Provider PR #2

11 files changed, 563 insertions, 28 deletions

diff --git a/lib/chef/deprecated.rb b/lib/chef/deprecated.rb
index 5b0bac552e..1cadacec98 100644
--- a/lib/chef/deprecated.rb
+++ b/lib/chef/deprecated.rb
@@ -196,6 +196,16 @@ class Chef
       end
     end
 
+    class DnfPackageAllowDowngrade < Base
+      def id
+        10
+      end
+
+      def target
+        "dnf_package_allow_downgrade.html"
+      end
+    end
+
     class Generic < Base
       def url
         "https://docs.chef.io/chef_deprecations_client.html"
diff --git a/lib/chef/mixin/which.rb b/lib/chef/mixin/which.rb
index 63c84883d5..4fa79eeccb 100644
--- a/lib/chef/mixin/which.rb
+++ b/lib/chef/mixin/which.rb
@@ -18,17 +18,13 @@
 class Chef
   module Mixin
     module Which
-      def which(cmd, opts = {})
-        extra_path =
-          if opts[:extra_path].nil?
-            [ "/bin", "/usr/bin", "/sbin", "/usr/sbin" ]
-          else
-            [ opts[:extra_path] ].flatten
-          end
+      def which(cmd, extra_path: nil)
+        # NOTE: unnecessarily duplicates function of path_sanity
+        extra_path ||= [ "/bin", "/usr/bin", "/sbin", "/usr/sbin" ]
         paths = ENV["PATH"].split(File::PATH_SEPARATOR) + extra_path
         paths.each do |path|
-          filename = File.join(path, cmd)
-          return filename if File.executable?(Chef.path_to(filename))
+          filename = Chef.path_to(File.join(path, cmd))
+          return filename if File.executable?(filename)
         end
         false
       end
diff --git a/lib/chef/provider/package.rb b/lib/chef/provider/package.rb
index ecf3dbecb5..f52614672a 100644
--- a/lib/chef/provider/package.rb
+++ b/lib/chef/provider/package.rb
@@ -37,6 +37,9 @@ class Chef
       subclass_directive :use_multipackage_api
       # subclasses declare this if they want sources (filenames) pulled from their package names
       subclass_directive :use_package_name_for_source
+      # keeps package_names_for_targets and versions_for_targets indexed the same as package_name at
+      # the cost of having the subclass needing to deal with nils
+      subclass_directive :allow_nils
 
       #
       # Hook that subclasses use to populate the candidate_version(s)
@@ -390,9 +393,12 @@ class Chef
       def package_names_for_targets
         package_names_for_targets = []
         target_version_array.each_with_index do |target_version, i|
-          next if target_version.nil?
-          package_name = package_name_array[i]
-          package_names_for_targets.push(package_name)
+          if !target_version.nil?
+            package_name = package_name_array[i]
+            package_names_for_targets.push(package_name)
+          else
+            package_names_for_targets.push(nil) if allow_nils?
+          end
         end
         multipackage? ? package_names_for_targets : package_names_for_targets[0]
       end
@@ -407,8 +413,11 @@ class Chef
       def versions_for_targets
         versions_for_targets = []
         target_version_array.each_with_index do |target_version, i|
-          next if target_version.nil?
-          versions_for_targets.push(target_version)
+          if !target_version.nil?
+            versions_for_targets.push(target_version)
+          else
+            versions_for_targets.push(nil) if allow_nils?
+          end
         end
         multipackage? ? versions_for_targets : versions_for_targets[0]
       end
diff --git a/lib/chef/provider/package/dnf.rb b/lib/chef/provider/package/dnf.rb
new file mode 100644
index 0000000000..bf6aa2438f
--- /dev/null
+++ b/lib/chef/provider/package/dnf.rb
@@ -0,0 +1,183 @@
+#
+# Copyright:: Copyright 2016, Chef Software, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "chef/provider/package"
+require "chef/resource/dnf_package"
+require "chef/mixin/which"
+require "chef/mixin/get_source_from_package"
+require "chef/provider/package/dnf/python_helper"
+require "chef/provider/package/dnf/version"
+
+class Chef
+  class Provider
+    class Package
+      class Dnf < Chef::Provider::Package
+        extend Chef::Mixin::Which
+        include Chef::Mixin::GetSourceFromPackage
+
+        allow_nils
+        use_multipackage_api
+        use_package_name_for_source
+
+        provides :package, platform_family: %w{rhel fedora} do
+          which("dnf")
+        end
+
+        provides :dnf_package, os: "linux"
+
+        #
+        # Most of the magic in this class happens in the python helper script.  The ruby side of this
+        # provider knows only enough to translate Chef-style new_resource name+package+version into
+        # a request to the python side.  The python side is then responsible for knowing everything
+        # about RPMs and what is installed and what is available.  The ruby side of this class should
+        # remain a lightweight translation layer to translate Chef requests into RPC requests to
+        # python.  This class knows nothing about how to compare RPM versions, and does not maintain
+        # any cached state of installed/available versions and should be kept that way.
+        #
+        def python_helper
+          @python_helper ||= PythonHelper.instance
+        end
+
+        def load_current_resource
+          flushcache if new_resource.flush_cache[:before]
+
+          @current_resource = Chef::Resource::DnfPackage.new(new_resource.name)
+          current_resource.package_name(new_resource.package_name)
+          current_resource.version(get_current_versions)
+
+          current_resource
+        end
+
+        def define_resource_requirements
+          requirements.assert(:install, :upgrade, :remove, :purge) do |a|
+            a.assertion { !new_resource.source || ::File.exist?(new_resource.source) }
+            a.failure_message Chef::Exceptions::Package, "Package #{new_resource.package_name} not found: #{new_resource.source}"
+            a.whyrun "assuming #{new_resource.source} would have previously been created"
+          end
+
+          super
+        end
+
+        def candidate_version
+          package_name_array.each_with_index.map do |pkg, i|
+            available_version(i).version_with_arch
+          end
+        end
+
+        def get_current_versions
+          package_name_array.each_with_index.map do |pkg, i|
+            installed_version(i).version_with_arch
+          end
+        end
+
+        def install_package(names, versions)
+          if new_resource.source
+            dnf(new_resource.options, "-y install", new_resource.source)
+          else
+            resolved_names = names.each_with_index.map { |name, i| available_version(i).to_s unless name.nil? }
+            dnf(new_resource.options, "-y install", resolved_names)
+          end
+          flushcache
+        end
+
+        # dnf upgrade does not work on uninstalled packaged, while install will upgrade
+        alias_method :upgrade_package, :install_package
+
+        def remove_package(names, versions)
+          resolved_names = names.each_with_index.map { |name, i| installed_version(i).to_s unless name.nil? }
+          dnf(new_resource.options, "-y remove", resolved_names)
+          flushcache
+        end
+
+        alias_method :purge_package, :remove_package
+
+        action :flush_cache do
+          flushcache
+        end
+
+        private
+
+        def resolve_source_to_version_obj
+          shell_out_with_timeout!("rpm -qp --queryformat '%{NAME} %{EPOCH} %{VERSION} %{RELEASE} %{ARCH}\n' #{new_resource.source}").stdout.each_line do |line|
+            # this is another case of committing the sin of doing some lightweight mangling of RPM versions in ruby -- but the output of the rpm command
+            # does not match what the dnf library accepts.
+            case line
+            when /^(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)$/
+              return Version.new($1, "#{$2 == "(none)" ? "0" : $2}:#{$3}-#{$4}", $5)
+            end
+          end
+        end
+
+        # @returns Array<Version>
+        def available_version(index)
+          @available_version ||= []
+
+          if new_resource.source
+            @available_version[index] ||= resolve_source_to_version_obj
+          else
+            @available_version[index] ||= python_helper.query(:whatavailable, package_name_array[index], safe_version_array[index], safe_arch_array[index])
+          end
+
+          @available_version[index]
+        end
+
+        # @returns Array<Version>
+        def installed_version(index)
+          @installed_version ||= []
+          if new_resource.source
+            @installed_version[index] ||= python_helper.query(:whatinstalled, available_version(index).name, safe_version_array[index], safe_arch_array[index])
+          else
+            @installed_version[index] ||= python_helper.query(:whatinstalled, package_name_array[index], safe_version_array[index], safe_arch_array[index])
+          end
+          @installed_version[index]
+        end
+
+        # cache flushing is accomplished by simply restarting the python helper.  this produces a roughly
+        # 15% hit to the runtime of installing/removing/upgrading packages.  correctly using multipackage
+        # array installs (and the multipackage cookbook) can produce 600% improvements in runtime.
+        def flushcache
+          python_helper.restart
+        end
+
+        def dnf(*args)
+          shell_out_with_timeout!(a_to_s("dnf", *args))
+        end
+
+        def safe_version_array
+          if new_resource.version.is_a?(Array)
+            new_resource.version
+          elsif new_resource.version.nil?
+            package_name_array.map { nil }
+          else
+            [ new_resource.version ]
+          end
+        end
+
+        def safe_arch_array
+          if new_resource.arch.is_a?(Array)
+            new_resource.arch
+          elsif new_resource.arch.nil?
+            package_name_array.map { nil }
+          else
+            [ new_resource.arch ]
+          end
+        end
+
+      end
+    end
+  end
+end
diff --git a/lib/chef/provider/package/dnf/dnf_helper.py b/lib/chef/provider/package/dnf/dnf_helper.py
new file mode 100644
index 0000000000..236b967710
--- /dev/null
+++ b/lib/chef/provider/package/dnf/dnf_helper.py
@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
+
+import sys
+import dnf
+import hawkey
+import signal
+import os
+import json
+
+base = None
+
+def get_sack():
+    global base
+    if base is None:
+        base = dnf.Base()
+        base.read_all_repos()
+        base.fill_sack()
+    return base.sack
+
+# FIXME: leaks memory and does not work
+def flushcache():
+    try:
+        os.remove('/var/cache/dnf/@System.solv')
+    except OSError:
+        pass
+    get_sack().load_system_repo(build_cache=True)
+
+def query(command):
+    sack = get_sack()
+
+    subj = dnf.subject.Subject(command['provides'])
+    q = subj.get_best_query(sack, with_provides=True)
+
+    if command['action'] == "whatinstalled":
+        q = q.installed()
+
+    if command['action'] == "whatavailable":
+        q = q.available()
+
+    if 'epoch' in command:
+        q = q.filterm(epoch=int(command['epoch']))
+    if 'version' in command:
+        q = q.filterm(version__glob=command['version'])
+    if 'release' in command:
+        q = q.filterm(release__glob=command['release'])
+
+    if 'arch' in command:
+        q = q.filterm(arch__glob=command['arch'])
+
+    # only apply the default arch query filter if it returns something
+    archq = q.filter(arch=[ 'noarch', hawkey.detect_arch() ])
+    if len(archq.run()) > 0:
+        q = archq
+
+    pkgs = dnf.query.latest_limit_pkgs(q, 1)
+
+    if not pkgs:
+        sys.stdout.write('{} nil nil\n'.format(command['provides'].split().pop(0)))
+    else:
+        # make sure we picked the package with the highest version
+        pkgs.sort
+        pkg = pkgs.pop()
+        sys.stdout.write('{} {}:{}-{} {}\n'.format(pkg.name, pkg.epoch, pkg.version, pkg.release, pkg.arch))
+
+# the design of this helper is that it should try to be 'brittle' and fail hard and exit in order
+# to keep process tables clean.  additional error handling should probably be added to the retry loop
+# on the ruby side.
+def exit_handler(signal, frame):
+    sys.exit(0)
+
+signal.signal(signal.SIGINT, exit_handler)
+signal.signal(signal.SIGHUP, exit_handler)
+signal.signal(signal.SIGPIPE, exit_handler)
+signal.signal(signal.SIGCHLD, exit_handler)
+
+while 1:
+    # kill self if we get orphaned (tragic)
+    ppid = os.getppid()
+    if ppid == 1:
+        sys.exit(0)
+    line = sys.stdin.readline()
+    command = json.loads(line)
+    if command['action'] == "whatinstalled":
+        query(command)
+    elif command['action'] == "whatavailable":
+        query(command)
+    elif command['action'] == "flushcache":
+        flushcache()
+    else:
+        raise RuntimeError("bad command")
diff --git a/lib/chef/provider/package/dnf/python_helper.rb b/lib/chef/provider/package/dnf/python_helper.rb
new file mode 100644
index 0000000000..466114b339
--- /dev/null
+++ b/lib/chef/provider/package/dnf/python_helper.rb
@@ -0,0 +1,120 @@
+#
+# Copyright:: Copyright 2016, Chef Software, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "chef/provider/package/dnf/version"
+require "timeout"
+
+class Chef
+  class Provider
+    class Package
+      class Dnf < Chef::Provider::Package
+        class PythonHelper
+          include Singleton
+          extend Chef::Mixin::Which
+
+          attr_accessor :stdin
+          attr_accessor :stdout
+          attr_accessor :stderr
+          attr_accessor :wait_thr
+
+          DNF_HELPER = ::File.expand_path(::File.join(::File.dirname(__FILE__), "dnf_helper.py")).freeze
+          DNF_COMMAND = "#{which("python3")} #{DNF_HELPER}"
+
+          def start
+            ENV["PYTHONUNBUFFERED"] = "1"
+            @stdin, @stdout, @stderr, @wait_thr = Open3.popen3(DNF_COMMAND)
+          end
+
+          def reap
+            unless wait_thr.nil?
+              Process.kill("KILL", wait_thr.pid) rescue nil
+              stdin.close unless stdin.nil?
+              stdout.close unless stdout.nil?
+              stderr.close unless stderr.nil?
+              wait_thr.value # this calls waitpit()
+            end
+          end
+
+          def check
+            start if stdin.nil?
+          end
+
+          # i couldn't figure out how to decompose an evr on the python side, it seems reasonably
+          # painless to do it in ruby (generally massaging nevras in the ruby side is HIGHLY
+          # discouraged -- this is an "every rule has an exception" exception -- any additional
+          # functionality should probably trigger moving this regexp logic into python)
+          def add_version(hash, version)
+            epoch = nil
+            if version =~ /(\S+):(\S+)/
+              epoch, version = $1, $2
+            end
+            if version =~ /(\S+)-(\S+)/
+              version, release = $1, $2
+            end
+            hash["epoch"] = epoch unless epoch.nil?
+            hash["release"] = release unless release.nil?
+            hash["version"] = version
+          end
+
+          def build_query(action, provides, version, arch)
+            hash = { "action" => action }
+            hash["provides"] = provides
+            add_version(hash, version) unless version.nil?
+            hash["arch" ] = arch unless arch.nil?
+            FFI_Yajl::Encoder.encode(hash)
+          end
+
+          def parse_response(output)
+            array = output.split.map { |x| x == "nil" ? nil : x }
+            array.each_slice(3).map { |x| Version.new(*x) }.first
+          end
+
+          # @returns Array<Version>
+          def query(action, provides, version = nil, arch = nil)
+            with_helper do
+              json = build_query(action, provides, version, arch)
+              Chef::Log.debug "sending '#{json}' to python helper"
+              stdin.syswrite json + "\n"
+              output = stdout.sysread(4096).chomp
+              Chef::Log.debug "got '#{output}' from python helper"
+              version = parse_response(output)
+              Chef::Log.debug "parsed #{version} from python helper"
+              version
+            end
+          end
+
+          def restart
+            reap
+            start
+          end
+
+          def with_helper
+            max_retries ||= 5
+            Timeout.timeout(600) do
+              check
+              yield
+            end
+          rescue EOFError, Errno::EPIPE, Timeout::Error, Errno::ESRCH => e
+            raise e unless ( max_retries -= 1 ) > 0
+            restart
+            retry
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/chef/provider/package/dnf/version.rb b/lib/chef/provider/package/dnf/version.rb
new file mode 100644
index 0000000000..b326913c3a
--- /dev/null
+++ b/lib/chef/provider/package/dnf/version.rb
@@ -0,0 +1,56 @@
+#
+# Copyright:: Copyright 2016, Chef Software, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+class Chef
+  class Provider
+    class Package
+      class Dnf < Chef::Provider::Package
+
+        # helper class to assist in passing around name/version/arch triples
+        class Version
+          attr_accessor :name
+          attr_accessor :version
+          attr_accessor :arch
+
+          def initialize(name, version, arch)
+            @name    = name
+            @version = version
+            @arch    = arch
+          end
+
+          def to_s
+            "#{name}-#{version}.#{arch}"
+          end
+
+          def version_with_arch
+            "#{version}.#{arch}" unless version.nil?
+          end
+
+          def matches_name_and_arch?(other)
+            other.version == version && other.arch == arch
+          end
+
+          def ==(other)
+            name == other.name && version == other.version && arch == other.arch
+          end
+
+          alias_method :eql?, :==
+        end
+      end
+    end
+  end
+end
diff --git a/lib/chef/providers.rb b/lib/chef/providers.rb
index ebb4c45ae8..35722840e6 100644
--- a/lib/chef/providers.rb
+++ b/lib/chef/providers.rb
@@ -65,6 +65,7 @@ require "chef/provider/env/windows"
 require "chef/provider/package/apt"
 require "chef/provider/package/chocolatey"
 require "chef/provider/package/dpkg"
+require "chef/provider/package/dnf"
 require "chef/provider/package/easy_install"
 require "chef/provider/package/freebsd/port"
 require "chef/provider/package/freebsd/pkg"
diff --git a/lib/chef/resource/dnf_package.rb b/lib/chef/resource/dnf_package.rb
new file mode 100644
index 0000000000..92f7532fc2
--- /dev/null
+++ b/lib/chef/resource/dnf_package.rb
@@ -0,0 +1,64 @@
+#
+# Copyright:: Copyright 2016, Chef Software, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "chef/resource/package"
+
+class Chef
+  class Resource
+    class DnfPackage < Chef::Resource::Package
+      extend Chef::Mixin::Which
+
+      resource_name :dnf_package
+
+      allowed_actions :install, :upgrade, :remove, :purge, :reconfig, :lock, :unlock, :flush_cache
+
+      provides :package, os: "linux", platform_family: %w{rhel fedora} do
+        which("dnf")
+      end
+
+      provides :dnf_package
+
+      # Install a specific arch
+      property :arch, [String, Array], coerce: proc { |x| [x].flatten }
+
+      # Flush the in-memory available/installed cache, this does not flush the dnf caches on disk
+      property :flush_cache,
+        Hash,
+        default: { before: false, after: false },
+        coerce: proc { |v|
+          if v.is_a?(Hash)
+            v
+          elsif v.is_a?(Array)
+            v.each_with_object({}) { |arg, obj| obj[arg] = true }
+          elsif v.is_a?(TrueClass) || v.is_a?(FalseClass)
+            { before: v, after: v }
+          elsif v == :before
+            { before: true, after: false }
+          elsif v == :after
+            { after: true, before: false }
+          end
+        }
+
+      def allow_downgrade(arg = nil)
+        if !arg.nil?
+          Chef.deprecated(:dnf_package_allow_downgrade, "the allow_downgrade property on the dnf_package provider is not used, DNF supports downgrades by default.")
+        end
+        false
+      end
+    end
+  end
+end
diff --git a/lib/chef/resource/yum_package.rb b/lib/chef/resource/yum_package.rb
index 9d69897f5f..1e0ad197ba 100644
--- a/lib/chef/resource/yum_package.rb
+++ b/lib/chef/resource/yum_package.rb
@@ -17,7 +17,6 @@
 #
 
 require "chef/resource/package"
-require "chef/provider/package/yum"
 
 class Chef
   class Resource
@@ -27,22 +26,27 @@ class Chef
 
       # Install a specific arch
       property :arch, [ String, Array ]
-      # the {} on the proc here is because rspec chokes if it's do...end
+
       property :flush_cache,
-               Hash,
-               default: { before: false, after: false },
-               coerce: proc { |v|
-                 if v.is_a?(Array)
-                   v.each_with_object({}) { |arg, obj| obj[arg] = true }
-                 elsif v.any?
-                   v
-                 else
-                   { before: v, after: v }
-                 end
-               }
+        Hash,
+        default: { before: false, after: false },
+        coerce: proc { |v|
+          if v.is_a?(Hash)
+            v
+          elsif v.is_a?(Array)
+            v.each_with_object({}) { |arg, obj| obj[arg] = true }
+          elsif v.is_a?(TrueClass) || v.is_a?(FalseClass)
+            { before: v, after: v }
+          elsif v == :before
+            { before: true, after: false }
+          elsif v == :after
+            { after: true, before: false }
+          end
+        }
+
       property :allow_downgrade, [ true, false ], default: false
-      property :yum_binary, String
 
+      property :yum_binary, String
     end
   end
 end
diff --git a/lib/chef/resources.rb b/lib/chef/resources.rb
index de421839e0..ab89ce66e0 100644
--- a/lib/chef/resources.rb
+++ b/lib/chef/resources.rb
@@ -31,6 +31,7 @@ require "chef/resource/deploy"
 require "chef/resource/deploy_revision"
 require "chef/resource/directory"
 require "chef/resource/dpkg_package"
+require "chef/resource/dnf_package"
 require "chef/resource/dsc_script"
 require "chef/resource/dsc_resource"
 require "chef/resource/easy_install_package"