path: root/omnibus/jenkins/release.rb

#!/usr/bin/env ruby

## release.rb #################################################################
#------------------------------------------------------------------------------
# This script runs from the root of a jenkins workspace where artifacts from
# the omnibus build matrix are collected.
#
# # Primary command line options:
# * `--project PROJECT`: Project to be released. This also controls where the
#   script looks for config JSON.
# * `--bucket S3_BUCKET`: Name of the S3 bucket where artifacts are released
#   to.
#
# Other options are available, run `release.rb --help`.
#
# # Config
# release.rb looks in the same directory where it's located for files named
# "$project.json" and "$project-platform-names.json".
#
# ## $project.json
# The project.json file controls the mapping of build platforms to release
# platforms so that a single build artifact can be reused on compatible
# platforms. See chef.json for an example.
#
# ## $project-platform-names.json
# The project-platform-names.json file maps short platform names to long ones.
# see chef-platform-names.json for an example.
#
# # Tests
# This file contains the script's tests. Tests are written in rspec. To run the
# tests, run rspec with this file as the argument, e.g.,
# `rspec -cfs release.rb`.

require 'optparse'
require 'digest'
require 'rubygems'
require 'json'
require 'mixlib/shellout'

# required for Artifactory publishing
require 'artifactory'
require 'omnibus'
require 'tempfile'

# Represnts the collection of artifacts on disk that we plan to upload. Handles
# finding the artifacts and dealing with the mapping between build platform and
# install platforms.
class ArtifactCollection

  class MissingArtifact < RuntimeError
  end

  attr_reader :project
  attr_reader :config

  def initialize(project, config)
    @project = project
    @config = config
  end

  def platform_map_json
    IO.read(File.expand_path("../#{project}.json", __FILE__))
  end

  def platform_map
    JSON.parse(platform_map_json)
  end

  def platform_name_map_path
    File.expand_path("../#{project}-platform-names.json", __FILE__)
  end

  def platform_name_map_json
    IO.read(platform_name_map_path)
  end

  def platform_name_map
    JSON.parse(platform_name_map_json)
  end

  def package_paths
    @package_paths ||= Dir['**/pkg/*'].
      sort.
      reject {|path| path.include?("BUILD_VERSION") }.
      reject {|path| path.include?("metadata.json") }
  end

  def artifacts
    artifacts = []
    missing_packages = []
    platform_map.each do |build_platform_spec, supported_platforms|
      if path = package_paths.find { |p| p.include?(build_platform_spec) }
        artifacts << Artifact.new(path, supported_platforms, config)
      else
        missing_packages << build_platform_spec
      end
    end
    error_on_missing_pkgs!(missing_packages)
    artifacts
  end

  def error_on_missing_pkgs!(missing_packages)
    unless missing_packages.empty?
      if config[:ignore_missing_packages]
        missing_packages.each do |pkg_config|
          # TODO: this should go to $stderr
          puts "WARN: Missing package for config: #{pkg_config}"
        end
      else
        raise MissingArtifact, "Missing packages for config(s): '#{missing_packages.join("' '")}'"
      end
    end
  end
end

# Represents an individual package which has one or more supported platforms.
class Artifact

  attr_reader :path
  attr_reader :platforms
  attr_reader :config

  def initialize(path, platforms, config)
    @path = path
    @platforms = platforms
    @config = config
  end

  # Adds the package to +release_manifest+, which is a Hash. The result is in this form:
  #   "el" => {
  #     "5" => { "x86_64" => { "11.4.0-1" => "/el/5/x86_64/demoproject-11.4.0-1.el5.x86_64.rpm" } }
  #   }
  # This method mutates the argument (hence the `!` at the end). The updated
  # release manifest is returned.
  def add_to_release_manifest!(release_manifest)
    platforms.each do |distro, version, arch|
      release_manifest[distro] ||= {}
      release_manifest[distro][version] ||= {}
      release_manifest[distro][version][arch] = { build_version => relpath }
      # TODO: when adding checksums, the desired format is like this:
      # build_support_json[platform][platform_version][machine_architecture][options[:version]]["relpath"] = build_location
    end
    release_manifest
  end

  # Adds the package to +release_manifest+, which is a Hash. The result is in this form:
  #   "el" => {
  #     "5" => {
  #       "x86_64" => {
  #         "11.4.0-1" => {
  #           "relpath" => "/el/5/x86_64/demoproject-11.4.0-1.el5.x86_64.rpm",
  #           "md5" => "123f00d...",
  #           "sha256" => 456beef..."
  #         }
  #       }
  #     }
  #   }
  # This method mutates the argument (hence the `!` at the end). The updated
  # release manifest is returned.
  def add_to_v2_release_manifest!(release_manifest)
    platforms.each do |distro, version, arch|
      pkg_info = {
        "relpath" => relpath,
        "md5" => md5,
        "sha256" => sha256
      }

      release_manifest[distro] ||= {}
      release_manifest[distro][version] ||= {}
      release_manifest[distro][version][arch] = { build_version => pkg_info  }
    end
    release_manifest
  end

  def build_platform
    platforms.first
  end

  def build_version
    config[:version]
  end

  def relpath
    # upload build to build platform directory
    "/#{build_platform.join('/')}/#{path.split('/').last}"
  end

  def md5
    @md5 ||= digest(Digest::MD5)
  end

  def sha256
    @sha256 ||= digest(Digest::SHA256)
  end

  private

  def digest(digest_class)
    digest = digest_class.new
    File.open(path) do |io|
      while chunk = io.read(1024 * 8)
        digest.update(chunk)
      end
    end
    digest.hexdigest
  end
end

class ShipIt
  attr_reader :argv
  attr_reader :options

  def initialize(argv=[])
    @argv = argv
    @options = {:package_s3_config_file => "~/.s3cfg"}
  end

  def release_it
    $stdout.sync = true
    parse_options
    artifact_collection = ArtifactCollection.new(options[:project], options)
    artifacts = artifact_collection.artifacts

    v2_metadata = {}

    artifacts.each do |artifact|
      artifact.add_to_v2_release_manifest!(v2_metadata)
      upload_package(artifact.path, artifact.relpath)

      # Optionally publish to the new Artifactory infrastructure
      if options[:publish_to_artifactory]
        Omnibus.load_configuration(options[:omnibus_config])
        publish_to_artifactory(artifact)
      end

    end
    upload_v2_platform_name_map(artifact_collection.platform_name_map_path)
    upload_v2_manifest(v2_metadata)
  end

  def option_parser
    @option_parser ||= OptionParser.new do |opts|
      opts.banner = "Usage: #{$0} [options]"

      opts.on("-p", "--project PROJECT", "the project to release") do |project|
        options[:project] = project
      end

      opts.on("-v", "--version VERSION", "the version of the installer to release") do |version|
        options[:version] = version
      end

      opts.on("-b", "--bucket S3_BUCKET_NAME", "the name of the s3 bucket to release to") do |bucket|
        options[:bucket] = bucket
      end

      opts.on("-c", "--package-s3-config S3_CMD_CONFIG_FILE", "path to the s3cmd config file for packages bucket") do |config|
        options[:package_s3_config_file] = config
      end

      opts.on("-M", "--metadata-bucket S3_BUCKET_NAME", "the name of the S3 bucket for v2 metadata") do |bucket|
        options[:metadata_bucket] = bucket
      end

      opts.on("-m", "--metadata-s3-config S3_CMD_CONFIG_FILE", "path to the s3cmd config file for the v2 metadata AWS account") do |config_path|
        options[:metadata_s3_config_file] = config_path
      end

      opts.on("--ignore-missing-packages",
              "indicates the release should continue if any build packages are missing") do |missing|
        options[:ignore_missing_packages] = missing
      end

      opts.on("--publish-to-artifactory",
              "indicates the release should be published to Artifactory") do |artifactory|
        options[:publish_to_artifactory] = artifactory
      end

      opts.on("--omnibus-config OMNIBUS_CONFIG_FILE",
              "path to the Omnibus config file which is required for Artifactory publishing") do |config_path|
        options[:omnibus_config] = config_path
      end
    end
  end

  def parse_options
    option_parser.parse(argv)

    # check for an optional BUILD_VERSION file which is generated by the build script
    if options[:version].nil?
      # this file should be the same across all platforms so grab the first one
      build_version_file = Dir['**/pkg/BUILD_VERSION'].first
      options[:version] = IO.read(build_version_file).chomp if build_version_file
    end

    required = [:project, :version, :bucket, :metadata_bucket, :metadata_s3_config_file]

    # If --publish-to-artifactory was provided then --omnibus-config is required
    required << :omnibus_config if options[:publish_to_artifactory]

    missing = required.select {|param| options[param].nil?}
    if !missing.empty?
      puts "Missing required options: #{missing.join(', ')}"
      puts option_parser
      exit 1
    end
  rescue OptionParser::InvalidOption, OptionParser::MissingArgument
    puts $!.to_s
    puts option_parser
    exit 1
  end

  def shellout_opts
    {:timeout => 1200, :live_stream => STDOUT}
  end

  def progress
    if STDOUT.tty?
      "--progress"
    else
      "--no-progress"
    end
  end

  def upload_package(local_path, s3_path)
    s3_cmd = ["s3cmd",
              "-c #{options[:package_s3_config_file]}",
              "put",
              progress,
              "--acl-public",
              local_path,
              "s3://#{options[:bucket]}#{s3_path}"].join(" ")
    shell = Mixlib::ShellOut.new(s3_cmd, shellout_opts)
    shell.run_command
    shell.error!
  end

  def upload_v2_manifest(manifest)
    File.open("v2-release-manifest.json", "w") {|f| f.puts JSON.pretty_generate(manifest)}

    s3_location = "s3://#{options[:metadata_bucket]}/#{options[:project]}-release-manifest/#{options[:version]}.json"
    puts "UPLOAD: v2-release-manifest.json -> #{s3_location}"
    s3_cmd = ["s3cmd",
              "-c #{options[:metadata_s3_config_file]}",
              "put",
              "--acl-public",
              "v2-release-manifest.json",
              s3_location].join(" ")
    shell = Mixlib::ShellOut.new(s3_cmd, shellout_opts)
    shell.run_command
    shell.error!
  end

  def upload_v2_platform_name_map(platform_names_file)
    s3_location = "s3://#{options[:metadata_bucket]}/#{options[:project]}-release-manifest/#{options[:project]}-platform-names.json"
    puts "UPLOAD: #{options[:project]}-platform-names.json -> #{s3_location}"
    s3_cmd = ["s3cmd",
              "-c #{options[:metadata_s3_config_file]}",
              "put",
              "--acl-public",
              platform_names_file,
              s3_location].join(" ")
    shell = Mixlib::ShellOut.new(s3_cmd, shellout_opts)
    shell.run_command
    shell.error!
  end

  def upload_v2_manifest?
    !options[:metadata_bucket].nil?
  end

  def publish_to_artifactory(artifact)

    metadata_json = "#{artifact.path}.metadata.json"

    if File.exist?(metadata_json)
      metadata = JSON.parse(IO.read(metadata_json), :symbolize_names => true)

      # Historically we name our Windows-specific project definitions
      # PROJECT_NAME-windows. We want to publish this project under
      # PROJECT_NAME though. Luckily we are moving to a place where the
      # Windows-specific project definition will just become part of the
      # regular *nix project definition. Until then we need to modify the
      # the artifact's associated `*.metadata.json` file as the Omnibus
      # publisher uses this metadata when publishing to Artifactory.
      if match = metadata[:name].match(/^(?<project_basename>.*)-windows$/)
        metadata[:name] = match[:project_basename]

        File.open(metadata_json, 'w+')  do |f|
          f.write(JSON.pretty_generate(metadata))
        end
      end
    end

    # Publish artifact under each tested platform, platform version and arch!
    artifact.platforms.each do |distro, version, arch|

      # We only really care about publishing for Windows 64-bit so ignore
      # the 32-bit mappings
      next if (distro == 'windows') && (arch == 'i686')

      tries = 3

      begin
        # Apply various 'fixes' to the metadata before uploading
        fix_metadata(artifact.path)

        publisher = Omnibus::ArtifactoryPublisher.new(
                    artifact.path,
                    repository: 'omnibus-current-local',
                    platform: distro,
                    platform_version: version,
                  )


        publisher.publish do |package|
          puts "Uploaded '#{package.name}'"
        end

      rescue Omnibus::NoPackageMetadataFile => e

        puts "Could not locate package metadata file '#{artifact.path}.metadata.json'...skipping publish."

      rescue Artifactory::Error::ArtifactoryError => e
        puts "\nError during publishing: #{e.message}"
        puts "Backtrace:\n\t#{e.backtrace.join("\n\t")}"

        if (tries -= 1) != 0
          puts "\nRetrying failed publish #{tries} more time(s)..."
          retry
        else
          raise
        end
      end
    end
  end

  # Attempts to fix an out-of-date `*.metadata.json` file produced by an
  # an older version of Omnibus. The data contained in this file is very
  # important in publishing activities.
  def fix_metadata(package_local_path)
    package = Omnibus::Package.new(package_local_path)

    old_metadata = JSON.parse(File.read(package.metadata.path), symbolize_names: true)
    new_metadata = old_metadata.dup

    # OLD Omnibus does not include a project name!!!! O_o
    unless new_metadata[:name]
      match = package.name.match(/^(?<project_name>[a-z].*)(_|-)\d\.\d\.\d/)
      new_metadata[:name] = match[:project_name]
    end

    # Historically we name our Windows-specific project definitions
    # PROJECT_NAME-windows. We want to publish this project under
    # PROJECT_NAME though. Luckily we are moving to a place where the
    # Windows-specific project definition will just become part of the
    # regular *nix project definition. Until then we need to modify the
    # the artifact's associated `*.metadata.json` file as the Omnibus
    # publisher uses this metadata when publishing to Artifactory.
    match = new_metadata[:name].match(/^(?<project_basename>.*)-windows$/)
    new_metadata[:name] = match[:project_basename] if match

    # In pre-4.0 Omnibus packages are signed outside of the Omnibus
    # build process which means Omnibus's generated checksums are wrong.
    # As we use the Omnibus-generated metadata when publishing to
    # Artifactory checksum mismatches are OK until Omnibus 4.0 is in
    # general use.
    if new_metadata[:platform] == 'el'
      new_metadata[:md5]    = package.md5
      new_metadata[:sha1]   = package.sha1
      new_metadata[:sha256] = package.sha256
      new_metadata[:sha512] = package.sha512
    end

    # OLD Omnibus does not generate all checksum types
    new_metadata[:sha256] = package.sha256 unless new_metadata[:sha256]
    new_metadata[:sha512] = package.sha512 unless new_metadata[:sha512]

    # If the metadata has changed write the file
    if new_metadata != old_metadata
      metadata = if Omnibus.const_defined?('Metadata')
        Omnibus::Metadata.new(package, new_metadata)
      else
        Omnibus::Package::Metadata.new(package, new_metadata)
      end
      puts "\nMetadata has changed...updating '#{metadata.path}'"
      metadata.save
    end
  end
end


if !$0.include?("rspec")
  ShipIt.new(ARGV).release_it
else
  describe ArtifactCollection do

    # project_json is the thing that maps a build to. It is stored in the same
    # directory with basename determined by project, e.g., "chef.json" for
    # chef-client, "chef-server.json" for chef-server. By convention, the first
    # entry is the platform that we actually do the build on.
    let(:platform_map_json) do
      <<-E
{
    "build_os=centos-5,machine_architecture=x64,role=oss-builder": [
        [
            "el",
            "5",
            "x86_64"
        ],
        [
            "sles",
            "11.2",
            "x86_64"
        ]
    ],
    "build_os=centos-5,machine_architecture=x86,role=oss-builder": [
        [
            "el",
            "5",
            "i686"
        ],
        [
            "sles",
            "11.2",
            "i686"
        ]
    ]
}
E
    end

    let(:platform_map) do
      JSON.parse(platform_map_json)
    end

    # mapping of short platform names to longer ones.
    # This file lives in this script's directory under $project-platform-names.json
    let(:platform_name_map_json) do
      <<-E
{
    "el" : "Enterprise Linux",
    "debian" : "Debian",
    "mac_os_x" : "OS X",
    "ubuntu" : "Ubuntu",
    "solaris2" : "Solaris",
    "sles" : "SUSE Enterprise",
    "suse" : "openSUSE",
    "windows" : "Windows"
}
E
    end

    let(:platform_name_map) do
      JSON.parse(platform_name_map_json)
    end

    let(:directory_contents) do
      %w[
        build_os=centos-5,machine_architecture=x64,role=oss-builder/pkg/demoproject-10.22.0-1.el5.x86_64.rpm.metadata.json
        build_os=centos-5,machine_architecture=x64,role=oss-builder/pkg/demoproject-10.22.0-1.el5.x86_64.rpm
        build_os=centos-5,machine_architecture=x64,role=oss-builder/pkg/BUILD_VERSION
        build_os=centos-5,machine_architecture=x86,role=oss-builder/pkg/demoproject-10.22.0-1.el5.i686.rpm.metadata.json
        build_os=centos-5,machine_architecture=x86,role=oss-builder/pkg/demoproject-10.22.0-1.el5.i686.rpm
        build_os=centos-5,machine_architecture=x86,role=oss-builder/pkg/BUILD_VERSION
      ]
    end

    let(:artifact_collection) do
      ArtifactCollection.new("demoproject", {})
    end

    it "has a project name" do
      artifact_collection.project.should == "demoproject"
    end

    it "has config" do
      artifact_collection.config.should == {}
    end

    it "loads the mapping of build platforms to install platforms from the local copy" do
      expected_path = File.expand_path("../demoproject.json", __FILE__)
      IO.should_receive(:read).with(expected_path).and_return(platform_map_json)
      artifact_collection.platform_map_json.should == platform_map_json
    end

    it "loads the mapping of platform short names to long names from the local copy" do
      expected_path = File.expand_path("../demoproject-platform-names.json", __FILE__)
      IO.should_receive(:read).with(expected_path).and_return(platform_name_map_json)
      artifact_collection.platform_name_map_json.should == platform_name_map_json
    end

    it "finds the package files among the artifacts" do
      Dir.should_receive(:[]).with("**/pkg/*").and_return(directory_contents)
      expected = %w[
        build_os=centos-5,machine_architecture=x64,role=oss-builder/pkg/demoproject-10.22.0-1.el5.x86_64.rpm
        build_os=centos-5,machine_architecture=x86,role=oss-builder/pkg/demoproject-10.22.0-1.el5.i686.rpm
      ]
      artifact_collection.package_paths.should == expected
    end

    context "after loading the build and platform mappings" do

      before do
        artifact_collection.should respond_to(:platform_map_json)
        artifact_collection.stub!(:platform_map_json).and_return(platform_map_json)
        artifact_collection.should respond_to(:platform_name_map_json)
        artifact_collection.stub!(:platform_name_map_json).and_return(platform_name_map_json)
      end

      it "parses the build platform mapping" do
        artifact_collection.platform_map.should == platform_map
      end

      it "parses the platform short name => long name mapping" do
        artifact_collection.platform_name_map.should == platform_name_map
      end

      it "returns a list of artifacts for each package" do
        Dir.should_receive(:[]).with("**/pkg/*").and_return(directory_contents)

        artifact_collection.should have(2).artifacts
        centos5_64bit_artifact = artifact_collection.artifacts.first

        path = "build_os=centos-5,machine_architecture=x64,role=oss-builder/pkg/demoproject-10.22.0-1.el5.x86_64.rpm"
        centos5_64bit_artifact.path.should == path

        platforms = [ [ "el", "5", "x86_64" ], [ "sles","11.2","x86_64" ] ]
        centos5_64bit_artifact.platforms.should == platforms
      end

      context "and some expected packages are missing" do
        let(:directory_contents) do
          %w[
            build_os=centos-5,machine_architecture=x86,role=oss-builder/pkg/demoproject-10.22.0-1.el5.i686.rpm
            build_os=centos-5,machine_architecture=x86,role=oss-builder/pkg/BUILD_VERSION
          ]
        end

        before do
          Dir.should_receive(:[]).with("**/pkg/*").and_return(directory_contents)
        end

        it "errors out verifying all packages are available" do
          err_msg = "Missing packages for config(s): 'build_os=centos-5,machine_architecture=x64,role=oss-builder'"
          lambda {artifact_collection.artifacts}.should raise_error(ArtifactCollection::MissingArtifact, err_msg)
        end

      end
    end

  end # describe ArtifactCollection

  describe Artifact do

    let(:path) { "build_os=centos-5,machine_architecture=x86,role=oss-builder/pkg/demoproject-11.4.0-1.el5.x86_64.rpm" }

    let(:content) { StringIO.new("this is the package content\n") }

    let(:md5) { "d41d8cd98f00b204e9800998ecf8427e" }

    let(:sha256) { "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }

    let(:platforms) { [ [ "el", "5", "x86_64" ], [ "sles","11.2","x86_64" ] ] }

    let(:artifact) { Artifact.new(path, platforms, { :version => "11.4.0-1" }) }

    it "has the path to the package" do
      artifact.path.should == path
    end

    it "has a list of platforms the package supports" do
      artifact.platforms.should == platforms
    end

    it "generates a MD5 of an artifact" do
      File.should_receive(:open).with(path).and_return(content)
      artifact.md5.should == md5
    end

    it "generates a SHA256 of an artifact" do
      File.should_receive(:open).with(path).and_return(content)
      artifact.sha256.should == sha256
    end

    it "adds the package to a release manifest" do
      expected = {
        "el" => {
          "5" => { "x86_64" => { "11.4.0-1" => "/el/5/x86_64/demoproject-11.4.0-1.el5.x86_64.rpm" } }
        },
        "sles" => {
          "11.2" => { "x86_64" => { "11.4.0-1" => "/el/5/x86_64/demoproject-11.4.0-1.el5.x86_64.rpm" } }
        }
      }

      manifest = artifact.add_to_release_manifest!({})
      manifest.should == expected
    end

    it "adds the package to a v2 release manifest" do
      File.should_receive(:open).with(path).twice.and_return(content)
      expected = {
        "el" => {
          "5" => { "x86_64" => { "11.4.0-1" => {
            "relpath" => "/el/5/x86_64/demoproject-11.4.0-1.el5.x86_64.rpm",
            "md5" => md5,
            "sha256" => sha256
              }
            }
          }
        },
        "sles" => {
          "11.2" => { "x86_64" => { "11.4.0-1" => {
            "relpath" => "/el/5/x86_64/demoproject-11.4.0-1.el5.x86_64.rpm",
            "md5" => md5,
            "sha256" => sha256
              }
            }
          }
        }
      }
      v2_manifest = artifact.add_to_v2_release_manifest!({})
      v2_manifest.should == expected
    end

  end
end