Merge pull request #823 from chef/GenPage-digitalocean_metadata

Add Digital Ocean metadata from their metadata API

5 files changed, 147 insertions, 172 deletions

diff --git a/lib/ohai/mixin/do_metadata.rb b/lib/ohai/mixin/do_metadata.rb
new file mode 100644
index 00000000..0a056d82
--- /dev/null
+++ b/lib/ohai/mixin/do_metadata.rb
@@ -0,0 +1,47 @@
+
+# Author:: Dylan Page (<dpage@digitalocean.com>)
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "net/http"
+
+module Ohai
+  module Mixin
+    module DOMetadata
+
+      DO_METADATA_ADDR = "169.254.169.254" unless defined?(DO_METADATA_ADDR)
+      DO_METADATA_URL = "/metadata/v1.json" unless defined?(DO_METADATA_URL)
+
+      def http_client
+        Net::HTTP.start(DO_METADATA_ADDR).tap { |h| h.read_timeout = 6 }
+      end
+
+      def fetch_metadata
+        uri = "#{DO_METADATA_URL}"
+        response = http_client.get(uri)
+        case response.code
+        when "200"
+          parser = FFI_Yajl::Parser.new
+          parser.parse(response.body)
+        when "404"
+          Ohai::Log.debug("Mixin DOMetadata: Encountered 404 response retreiving Digital Ocean metadata: #{uri} ; continuing.")
+          {}
+        else
+          raise "Mixin DOMetadata: Encountered error retrieving Digital Ocean metadata (#{uri} returned #{response.code} response)"
+        end
+      end
+
+    end
+  end
+end
diff --git a/lib/ohai/plugins/cloud.rb b/lib/ohai/plugins/cloud.rb
index c204f292..551016d5 100644
--- a/lib/ohai/plugins/cloud.rb
+++ b/lib/ohai/plugins/cloud.rb
@@ -233,17 +233,17 @@ Ohai.plugin(:Cloud) do
 
   # Fill cloud hash with linode values
   def get_digital_ocean_values
-    public_ipv4 = digital_ocean["networks"]["v4"].select { |address| address["type"] == "public" }
-    private_ipv4 = digital_ocean["networks"]["v4"].select { |address| address["type"] == "private" }
-    public_ipv6 = digital_ocean["networks"]["v6"].select { |address| address["type"] == "public" }
-    private_ipv6 = digital_ocean["networks"]["v6"].select { |address| address["type"] == "private" }
+    public_ipv4 = digital_ocean["interfaces"]["public"].map { |iface| iface["ipv4"]["ip_address"] }
+    private_ipv4 = digital_ocean["interfaces"]["private"] ? digital_ocean["interfaces"]["private"].map { |iface| iface["ipv4"]["ip_address"] } : []
+    public_ipv6 = digital_ocean["interfaces"]["public"].map { |iface| iface["ipv6"]["ip_address"] }
+    private_ipv6 = digital_ocean["interfaces"]["private"] ? digital_ocean["interfaces"]["private"].map { |iface| iface["ipv6"]["ip_address"] } : []
     cloud[:public_ips].concat public_ipv4 + public_ipv6
     cloud[:private_ips].concat private_ipv4 + private_ipv6
     cloud[:public_ipv4] = public_ipv4.first
     cloud[:public_ipv6] = public_ipv6.first
     cloud[:local_ipv4] = private_ipv4.first
     cloud[:local_ipv6] = private_ipv6.first
-    cloud[:public_hostname] = digital_ocean["name"]
+    cloud[:public_hostname] = digital_ocean["hostname"]
     cloud[:provider] = "digital_ocean"
   end
 
diff --git a/lib/ohai/plugins/digital_ocean.rb b/lib/ohai/plugins/digital_ocean.rb
index a06c3c67..d43f02e3 100644
--- a/lib/ohai/plugins/digital_ocean.rb
+++ b/lib/ohai/plugins/digital_ocean.rb
@@ -1,4 +1,5 @@
 #
+# Author:: Dylan Page (<dpage@digitalocean.com>)
 # Author:: Stafford Brunk (<stafford.brunk@gmail.com>)
 # License:: Apache License, Version 2.0
 #
@@ -14,65 +15,46 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-require "ohai/util/ip_helper"
+require "ohai/mixin/do_metadata"
+require "ohai/mixin/http_helper"
 
 Ohai.plugin(:DigitalOcean) do
-  include Ohai::Util::IpHelper
-
-  DIGITALOCEAN_FILE = "/etc/digitalocean" unless defined?(DIGITALOCEAN_FILE)
+  include Ohai::Mixin::DOMetadata
+  include Ohai::Mixin::HttpHelper
 
   provides "digital_ocean"
-  depends "network/interfaces"
 
-  def extract_droplet_ip_addresses
-    addresses = Mash.new({ "v4" => [], "v6" => [] })
-    network[:interfaces].each_value do |iface|
-      iface[:addresses].each do |address, details|
-        next if details[:family] == "lladdr" || loopback?(address)
+  depends "dmi"
 
-        ip = IPAddress(address)
-        type = digital_ocean_address_type(ip)
-        address_hash = build_address_hash(ip, details)
-        addresses[type] << address_hash
+  # look for digitalocean string in dmi bios data
+  def has_do_dmi?
+    begin
+      # detect a vendor of "DigitalOcean"
+      if dmi[:bios][:all_records][0][:Vendor] == "DigitalOcean"
+        Ohai::Log.debug("Plugin DigitalOcean: has_do_dmi? == true")
+        return true
       end
+    rescue NoMethodError
+      # dmi[:bios][:all_records][0][:Vendor] may not exist
     end
-    addresses
-  end
-
-  def build_address_hash(ip, details)
-    address_hash = Mash.new({
-      "ip_address" => ip.address,
-      "type" => private_address?(ip.address) ? "private" : "public",
-    })
-
-    if ip.ipv4?
-      address_hash["netmask"] = details[:netmask]
-    elsif ip.ipv6?
-      address_hash["cidr"] = ip.prefix
-    end
-    address_hash
-  end
-
-  def digital_ocean_address_type(ip)
-    ip.ipv4? ? "v4" : "v6"
+    Ohai::Log.debug("Plugin DigitalOcean: has_do_dmi? == false")
+    return false
   end
 
   def looks_like_digital_ocean?
-    hint?("digital_ocean") || File.exist?(DIGITALOCEAN_FILE)
+    return true if hint?("digital_ocean")
+    return true if has_do_dmi? && can_socket_connect?(Ohai::Mixin::DOMetadata::DO_METADATA_ADDR, 80)
+    return false
   end
 
   collect_data do
     if looks_like_digital_ocean?
       Ohai::Log.debug("Plugin Digitalocean: looks_like_digital_ocean? == true")
       digital_ocean Mash.new
-      hint = hint?("digital_ocean") || {}
-      hint.each { |k, v| digital_ocean[k] = v unless k == "ip_addresses" }
-
-      # Extract actual ip addresses
-      # The networks sub-hash is structured similarly to how
-      # Digital Ocean's v2 API structures things:
-      # https://developers.digitalocean.com/#droplets
-      digital_ocean[:networks] = extract_droplet_ip_addresses
+      fetch_metadata.each do |k, v|
+        next if k == "vendor_data" # this may have sensitive data we shouldn't store
+        digital_ocean[k] = v
+      end
     else
       Ohai::Log.debug("Plugin Digitalocean: No hints present for and doesn't look like digitalocean")
       false
diff --git a/spec/unit/plugins/cloud_spec.rb b/spec/unit/plugins/cloud_spec.rb
index bd2e9d02..83a59469 100644
--- a/spec/unit/plugins/cloud_spec.rb
+++ b/spec/unit/plugins/cloud_spec.rb
@@ -215,12 +215,31 @@ describe Ohai::System, "plugin cloud" do
   describe "with digital_ocean mash" do
     before do
       @plugin[:digital_ocean] = Mash.new
-      @plugin[:digital_ocean][:name] = "public.example.com"
-      @plugin[:digital_ocean][:networks] = Mash.new
-      @plugin[:digital_ocean][:networks][:v4] = [{ "ip_address" => "1.2.3.4", "type" => "public" },
-                                                 { "ip_address" => "5.6.7.8", "type" => "private" }]
-      @plugin[:digital_ocean][:networks][:v6] = [{ "ip_address" => "fe80::4240:95ff:fe47:6eee", "type" => "public" },
-                                                 { "ip_address" => "fdf8:f53b:82e4::53", "type" => "private" }]
+      @plugin[:digital_ocean][:hostname] = "public.example.com"
+      @plugin[:digital_ocean][:interfaces] = Mash.new
+      @plugin[:digital_ocean][:interfaces] = {
+        "public" => [
+          {
+            "ipv4" => {
+              "ip_address" => "159.203.92.161",
+              "netmask" => "255.255.240.0",
+              "gateway" => "159.203.80.1",
+            },
+            "ipv6" => {
+              "ip_address" => "2604:A880:0800:00A1:0000:0000:0201:0001",
+              "cidr" => 64,
+              "gateway" => "2604:A880:0800:00A1:0000:0000:0000:0001",
+            },
+            "anchor_ipv4" => {
+              "ip_address" => "10.17.0.5",
+              "netmask" => "255.255.0.0",
+              "gateway" => "10.17.0.1",
+            },
+            "mac" => "04:01:e5:14:03:01",
+            "type" => "public",
+          },
+        ],
+      }
     end
 
     before(:each) do
@@ -236,29 +255,27 @@ describe Ohai::System, "plugin cloud" do
     end
 
     it "populates cloud public ips" do
-      expect(@plugin[:cloud][:public_ips]).to eq(@plugin[:digital_ocean][:networks][:v4].select { |ip| ip["type"] == "public" } +
-                                             @plugin[:digital_ocean][:networks][:v6].select { |ip| ip["type"] == "public" })
+      expect(@plugin[:cloud][:public_ips]).to eq([ "159.203.92.161", "2604:A880:0800:00A1:0000:0000:0201:0001" ])
     end
 
     it "populates cloud private ips" do
-      expect(@plugin[:cloud][:private_ips]).to eq(@plugin[:digital_ocean][:networks][:v4].select { |ip| ip["type"] == "private" } +
-                                              @plugin[:digital_ocean][:networks][:v6].select { |ip| ip["type"] == "private" })
+      expect(@plugin[:cloud][:private_ips]).to be_empty
     end
 
     it "populates cloud public_ipv4" do
-      expect(@plugin[:cloud][:public_ipv4]).to eq(@plugin[:digital_ocean][:networks][:v4].find { |ip| ip["type"] == "public" })
+      expect(@plugin[:cloud][:public_ipv4]).to eq("159.203.92.161")
     end
 
     it "populates cloud local_ipv4" do
-      expect(@plugin[:cloud][:local_ipv4]).to eq(@plugin[:digital_ocean][:networks][:v4].find { |ip| ip["type"] == "private" })
+      expect(@plugin[:cloud][:local_ipv4]).to be_nil
     end
 
     it "populates cloud public_ipv6" do
-      expect(@plugin[:cloud][:public_ipv6]).to eq(@plugin[:digital_ocean][:networks][:v6].find { |ip| ip["type"] == "public" })
+      expect(@plugin[:cloud][:public_ipv6]).to eq("2604:A880:0800:00A1:0000:0000:0201:0001")
     end
 
     it "populates cloud local_ipv6" do
-      expect(@plugin[:cloud][:local_ipv6]).to eq(@plugin[:digital_ocean][:networks][:v6].find { |ip| ip["type"] == "private" })
+      expect(@plugin[:cloud][:local_ipv6]).to be_nil
     end
 
     it "populates cloud provider" do
diff --git a/spec/unit/plugins/digital_ocean_spec.rb b/spec/unit/plugins/digital_ocean_spec.rb
index a593451a..71500266 100644
--- a/spec/unit/plugins/digital_ocean_spec.rb
+++ b/spec/unit/plugins/digital_ocean_spec.rb
@@ -1,4 +1,5 @@
 #
+# Author:: Dylan Page (<dpage@digitalocean.com>)
 # Author:: Stafford Brunk (<stafford.brunk@gmail.com>)
 # License:: Apache License, Version 2.0
 #
@@ -15,12 +16,10 @@
 # limitations under the License.
 #
 
-require "ipaddress"
-require_relative "../../spec_helper.rb"
+require "spec_helper"
 
 describe Ohai::System, "plugin digital_ocean" do
   let(:plugin) { get_plugin("digital_ocean") }
-  let(:digitalocean_path) { "/etc/digitalocean" }
   let(:hint) do
     {
       "droplet_id" => 12345678,
@@ -35,141 +34,71 @@ describe Ohai::System, "plugin digital_ocean" do
     }
   end
 
-  before do
-    plugin[:network] = {
-      "interfaces" => {
-        "eth0" => {
-          "addresses" => {
-            "00:D3:AD:B3:3F:00" => {
-              "family" => "lladdr",
-            },
-            "1.2.3.4" => {
-              "netmask" => "255.255.255.0",
-            },
-            "2400:6180:0000:00d0:0000:0000:0009:7001" => {},
-          },
-        },
-      },
-    }
-
-    allow(plugin).to receive(:hint?).with("digital_ocean").and_return(hint)
+  before(:each) do
+    allow(plugin).to receive(:hint?).with("digital_ocean").and_return(false)
   end
 
   shared_examples_for "!digital_ocean" do
-    before(:each) do
-      plugin.run
-    end
-
-    it "does not create the digital_ocean mash" do
+    it "should NOT attempt to fetch the digital_ocean metadata" do
+      expect(plugin).not_to receive(:http_client)
       expect(plugin[:digital_ocean]).to be_nil
+      plugin.run
     end
   end
 
-  shared_examples_for "digital_ocean_networking" do
-    it "creates the networks attribute" do
-      expect(plugin[:digital_ocean][:networks]).not_to be_nil
+  shared_examples_for "digital_ocean" do
+    before(:each) do
+      @http_client = double("Net::HTTP client")
+      allow(plugin).to receive(:http_client).and_return(@http_client)
+      allow(IO).to receive(:select).and_return([[], [1], []])
+      t = double("connection")
+      allow(t).to receive(:connect_nonblock).and_raise(Errno::EINPROGRESS)
+      allow(Socket).to receive(:new).and_return(t)
+      allow(Socket).to receive(:pack_sockaddr_in).and_return(nil)
     end
 
-    it "pulls ip addresses from the network interfaces" do
-      expect(plugin[:digital_ocean][:networks][:v4]).to eq([{ "ip_address" => "1.2.3.4",
-                                                              "type" => "public",
-                                                              "netmask" => "255.255.255.0" }])
-      expect(plugin[:digital_ocean][:networks][:v6]).to eq([{ "ip_address" => "2400:6180:0000:00d0:0000:0000:0009:7001",
-                                                              "type" => "public",
-                                                              "cidr" => 128 }])
+    let(:body) do
+      '{"droplet_id":2756924,"hostname":"sample-droplet","vendor_data":"#cloud-config\ndisable_root: false\nmanage_etc_hosts: true\n\n# The modules that run in the \'init\' stage\ncloud_init_modules:\n - migrator\n - ubuntu-init-switch\n - seed_random\n - bootcmd\n - write-files\n - growpart\n - resizefs\n - set_hostname\n - update_hostname\n - [ update_etc_hosts, once-per-instance ]\n - ca-certs\n - rsyslog\n - users-groups\n - ssh\n\n# The modules that run in the \'config\' stage\ncloud_config_modules:\n - disk_setup\n - mounts\n - ssh-import-id\n - locale\n - set-passwords\n - grub-dpkg\n - apt-pipelining\n - apt-configure\n - package-update-upgrade-install\n - landscape\n - timezone\n - puppet\n - chef\n - salt-minion\n - mcollective\n - disable-ec2-metadata\n - runcmd\n - byobu\n\n# The modules that run in the \'final\' stage\ncloud_final_modules:\n - rightscale_userdata\n - scripts-vendor\n - scripts-per-once\n - scripts-per-boot\n - scripts-per-instance\n - scripts-user\n - ssh-authkey-fingerprints\n - keys-to-console\n - phone-home\n - final-message\n - power-state-change\n","public_keys":["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDAkMD3PYKHaH0KbDiXrRE6KCBo/OKcFqhM+fmnnb0+LUh4RalJWX4edeJmnT5bxLeqmLV/Yggjlpfq73R+Dy7JB4pbBLuM959mSM9ohBCSnByAGoT2iUPev4aZFZZ/ahUzTCylNxXrhZV/bopD399CvYREt7Q+FlauBv0O8MMuMGR8aC69Z3jNL+r+fGWNq98JVHGFO/UgoNL15wGCaidMhzfRqkt1u+m1nY77SFM5qWJz2R0CEC4fMlOiCg8mWBklnryV4yDEPgiXp2I8Rli1Eu2GHwuY1YX9elMeQS7n3Pzq7l6aIQmSgvcEWx6TgMD2V7nQUWpfcud/8dpp/t7z9UyfzLmNwnULHNmUeEp52sejcH5lYzISnkkWa1LzlKSeIrhF3y45m9AyxIfjEqyh/mlKQtUaW3NVXXLPwrNitxHtMIZPU5b16BODn0wb8bqPxpDNpUYrQd/BS7mWDxNpICP2ObLPhd9LW9KIYRNTzryE+uKwxm9NkMlhRku2fu415fH0G0+7aURsHviNN9SO4zct3Pj6QE5rnbVHqxt3biplUTOScdWxSk2Nv3V2dGdt/lBfu6iRPAV9IAS31s7Po3qK1t2jpEPCJwstaCBOM80kmoi3zAgotiAW50X8CelaWsHNrq5jBBgeHUZWgn/c8BkcI61pUE9l34Q6gsiEMQ== tsmith84@gmail.com"],"region":"nyc3","interfaces":{"public":[{"ipv4":{"ip_address":"159.203.92.161","netmask":"255.255.240.0","gateway":"159.203.80.1"},"ipv6":{"ip_address":"2604:A880:0800:00A1:0000:0000:0201:0001","cidr":64,"gateway":"2604:A880:0800:00A1:0000:0000:0000:0001"},"anchor_ipv4":{"ip_address":"10.17.0.5","netmask":"255.255.0.0","gateway":"10.17.0.1"},"mac":"04:01:e5:14:03:01","type":"public"}]},"floating_ip":{"ipv4":{"active":false}},"dns":{"nameservers":["2001:4860:4860::8844","2001:4860:4860::8888","8.8.8.8"]}}'
     end
-  end
 
-  shared_examples_for "digital_ocean" do
-    before(:each) do
+    it "should fetch and properly parse json metadata" do
+      expect(@http_client).to receive(:get).
+        with("/metadata/v1.json").
+        and_return(double("Net::HTTP Response", :body => body, :code => "200"))
       plugin.run
-    end
 
-    it "creates a digital_ocean mash" do
       expect(plugin[:digital_ocean]).not_to be_nil
+      expect(plugin[:digital_ocean]["droplet_id"]).to eq(2756924)
+      expect(plugin[:digital_ocean]["hostname"]).to eq("sample-droplet")
     end
 
-    it "has all hint attributes" do
-      expect(plugin[:digital_ocean][:droplet_id]).not_to be_nil
-      expect(plugin[:digital_ocean][:name]).not_to be_nil
-      expect(plugin[:digital_ocean][:image_id]).not_to be_nil
-      expect(plugin[:digital_ocean][:size_id]).not_to be_nil
-      expect(plugin[:digital_ocean][:region_id]).not_to be_nil
-    end
-
-    it "skips the ip_addresses hint attribute" do
-      expect(plugin[:digital_ocean][:ip_addresses]).to be_nil
-    end
+    it "should complete the run despite unavailable metadata" do
+      expect(@http_client).to receive(:get).
+        with("/metadata/v1.json").
+        and_return(double("Net::HTTP Response", :body => "", :code => "404"))
+      plugin.run
 
-    it "has correct values for all hint attributes" do
-      expect(plugin[:digital_ocean][:droplet_id]).to eq(12345678)
-      expect(plugin[:digital_ocean][:name]).to eq("example.com")
-      expect(plugin[:digital_ocean][:image_id]).to eq(3240036)
-      expect(plugin[:digital_ocean][:size_id]).to eq(66)
-      expect(plugin[:digital_ocean][:region_id]).to eq(4)
+      expect(plugin[:digitalocean]).to be_nil
     end
+  end
 
-    include_examples "digital_ocean_networking"
+  describe "without hint or dmi data" do
+    it_should_behave_like "!digital_ocean"
   end
 
   describe "with digital_ocean hint file" do
-    before do
-      allow(plugin).to receive(:hint?).with("digital_ocean").and_return(hint)
-    end
-
-    context "without private networking enabled" do
-      it_behaves_like "digital_ocean"
-    end
+    it_should_behave_like "digital_ocean"
 
-    context "with private networking enabled" do
-      before do
-        plugin[:network][:interfaces][:eth1] = {
-          "addresses" => {
-            "10.128.142.89" => {
-              "netmask" => "255.255.255.0",
-            },
-            "fdf8:f53b:82e4:0000:0000:0000:0000:0053" => {},
-          },
-        }
-
-        plugin.run
-      end
-
-      it "extracts the private networking ips" do
-        expect(plugin[:digital_ocean][:networks][:v4]).to eq([{ "ip_address" => "1.2.3.4",
-                                                                "type" => "public",
-                                                                "netmask" => "255.255.255.0" },
-                                                            { "ip_address" => "10.128.142.89",
-                                                              "type" => "private",
-                                                              "netmask" => "255.255.255.0" }])
-        expect(plugin[:digital_ocean][:networks][:v6]).to eq([{ "ip_address" => "2400:6180:0000:00d0:0000:0000:0009:7001",
-                                                                "type" => "public",
-                                                                "cidr" => 128 },
-                                                           { "ip_address" => "fdf8:f53b:82e4:0000:0000:0000:0000:0053",
-                                                             "type" => "private",
-                                                             "cidr" => 128 }])
-      end
+    before(:each) do
+      allow(plugin).to receive(:hint?).with("digital_ocean").and_return(true)
     end
   end
 
-  describe "without digital_ocean hint file" do
-    before do
-      allow(plugin).to receive(:hint?).with("digital_ocean").and_return(false)
-    end
+  describe "with digital_ocean DMI data" do
+    it_should_behave_like "digital_ocean"
 
-    describe "with the /etc/digitalocean file" do
-      before do
-        allow(File).to receive(:exist?).with(digitalocean_path).and_return(true)
-        plugin.run
-      end
-      it_behaves_like "digital_ocean_networking"
-    end
-
-    describe "without the /etc/digitalocean file" do
-      before do
-        allow(File).to receive(:exist?).with(digitalocean_path).and_return(false)
-      end
-      it_behaves_like "!digital_ocean"
+    before(:each) do
+      plugin[:dmi] = { :bios => { :all_records => [ { :Vendor => "DigitalOcean" } ] } }
     end
   end
 end