Merge pull request #1033 from chef/azure

Add Azure metadata endpoint support

8 files changed, 313 insertions, 20 deletions

diff --git a/lib/ohai/mixin/azure_metadata.rb b/lib/ohai/mixin/azure_metadata.rb
new file mode 100644
index 00000000..9a3765a9
--- /dev/null
+++ b/lib/ohai/mixin/azure_metadata.rb
@@ -0,0 +1,53 @@
+#
+# Author:: Tim Smith (<tsmith@chef.io>)
+# Copyright:: Copyright 2017 Chef Software, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "net/http"
+
+module Ohai
+  module Mixin
+    module AzureMetadata
+
+      AZURE_METADATA_ADDR = "169.254.169.254" unless defined?(AZURE_METADATA_ADDR)
+      AZURE_METADATA_URL = "/metadata/instance?api-version=2017-04-02" unless defined?(AZURE_METADATA_URL)
+
+      # fetch the meta content with a timeout and the required header
+      def http_get(uri)
+        conn = Net::HTTP.start(AZURE_METADATA_ADDR)
+        conn.read_timeout = 6
+        conn.get(uri, initheader = { "Metadata" => "true" })
+      end
+
+      def fetch_metadata
+        Ohai::Log.debug("Mixin AzureMetadata: Fetching metadata from host #{AZURE_METADATA_ADDR} at #{AZURE_METADATA_URL}")
+        response = http_get(AZURE_METADATA_URL)
+        if response.code == "200"
+          begin
+            data = StringIO.new(response.body)
+            parser = FFI_Yajl::Parser.new
+            parser.parse(data)
+          rescue FFI_Yajl::ParseError
+            Ohai::Log.warn("Mixin AzureMetadata: Metadata response is NOT valid JSON")
+            nil
+          end
+        else
+          Ohai::Log.warn("Mixin AzureMetadata: Received resonse code #{response.code} requesting metadata")
+          nil
+        end
+      end
+    end
+  end
+end
diff --git a/lib/ohai/plugins/azure.rb b/lib/ohai/plugins/azure.rb
index 7eae3955..cf173a59 100644
--- a/lib/ohai/plugins/azure.rb
+++ b/lib/ohai/plugins/azure.rb
@@ -1,4 +1,4 @@
-# Copyright:: Copyright (c) 2013-2016 Chef Software, Inc.
+# Copyright:: Copyright 2013-2017 Chef Software, Inc.
 # License:: Apache License, Version 2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,23 +15,31 @@
 #
 
 Ohai.plugin(:Azure) do
+  require "ohai/mixin/azure_metadata"
+  require "ohai/mixin/http_helper"
+
+  include Ohai::Mixin::AzureMetadata
+  include Ohai::Mixin::HttpHelper
+
   provides "azure"
 
   collect_data do
-    # The azure hints are populated by the knife plugin for Azure.
-    # The project is located at https://github.com/chef/knife-azure
+    # Before we had the metadata endpoint we relied exclusively on
+    # the knife-azure plugin populating data to the hint file.
     # Please see the lib/chef/knife/azure_server_create.rb file in that
     # project for details
     azure_metadata_from_hints = hint?("azure")
     if azure_metadata_from_hints
-      Ohai::Log.debug("Plugin Azure: azure_metadata_from_hints is present.")
+      Ohai::Log.debug("Plugin Azure: Azure hint is present. Parsing any hint data.")
       azure Mash.new
       azure_metadata_from_hints.each { |k, v| azure[k] = v }
+      azure["metadata"] = parse_metadata
     elsif has_waagent? || has_dhcp_option_245?
-      Ohai::Log.debug("Plugin Azure: No hints present, but system appears to be on azure.")
+      Ohai::Log.debug("Plugin Azure: No hints present, but system appears to be on Azure.")
       azure Mash.new
+      azure["metadata"] = parse_metadata
     else
-      Ohai::Log.debug("Plugin Azure: No hints present for azure and doesn't appear to be azure.")
+      Ohai::Log.debug("Plugin Azure: No hints present and doesn't appear to be on Azure.")
       false
     end
   end
@@ -40,7 +48,7 @@ Ohai.plugin(:Azure) do
   # http://blog.mszcool.com/index.php/2015/04/detecting-if-a-virtual-machine-runs-in-microsoft-azure-linux-windows-to-protect-your-software-when-distributed-via-the-azure-marketplace/
   def has_waagent?
     if File.exist?("/usr/sbin/waagent") || Dir.exist?('C:\WindowsAzure')
-      Ohai::Log.debug("Plugin Azure: Found waagent used by MS Azure.")
+      Ohai::Log.debug("Plugin Azure: Found waagent used by Azure.")
       true
     end
   end
@@ -50,7 +58,7 @@ Ohai.plugin(:Azure) do
     if File.exist?("/var/lib/dhcp/dhclient.eth0.leases")
       File.open("/var/lib/dhcp/dhclient.eth0.leases").each do |line|
         if line =~ /unknown-245/
-          Ohai::Log.debug("Plugin Azure: Found unknown-245 DHCP option used by MS Azure.")
+          Ohai::Log.debug("Plugin Azure: Found unknown-245 DHCP option used by Azure.")
           has_245 = true
           break
         end
@@ -59,4 +67,56 @@ Ohai.plugin(:Azure) do
     has_245
   end
 
+  # create the basic structure we'll store our data in
+  def initialize_metadata_mash
+    metadata = Mash.new
+    metadata["compute"] = Mash.new
+    metadata["network"] = Mash.new
+    metadata["network"]["interfaces"] = Mash.new
+    %w{public_ipv4 local_ipv4 public_ipv6 local_ipv6}.each do |type|
+      metadata["network"][type] = []
+    end
+    metadata
+  end
+
+  def fetch_ip_data(data, type, field)
+    ips = []
+
+    data[type]["ipAddress"].each do |val|
+      ips << val[field] unless val[field].empty?
+    end
+    ips
+  end
+
+  def parse_metadata
+    return nil unless can_socket_connect?(Ohai::Mixin::AzureMetadata::AZURE_METADATA_ADDR, 80)
+
+    endpoint_data = fetch_metadata
+    return nil if endpoint_data.nil?
+    metadata = initialize_metadata_mash
+
+    # blindly add everything in compute to our data structure
+    endpoint_data["compute"].each do |k, v|
+      metadata["compute"][k] = v
+    end
+
+    # parse out per interface interface IP data
+    endpoint_data["network"]["interface"].each do |int|
+      metadata["network"]["interfaces"][int["macAddress"]] = Mash.new
+      metadata["network"]["interfaces"][int["macAddress"]]["mac"] = int["macAddress"]
+      metadata["network"]["interfaces"][int["macAddress"]]["public_ipv6"] = fetch_ip_data(int, "ipv6", "publicIpAddress")
+      metadata["network"]["interfaces"][int["macAddress"]]["public_ipv4"] = fetch_ip_data(int, "ipv4", "publicIpAddress")
+      metadata["network"]["interfaces"][int["macAddress"]]["local_ipv6"] = fetch_ip_data(int, "ipv6", "privateIpAddress")
+      metadata["network"]["interfaces"][int["macAddress"]]["local_ipv4"] = fetch_ip_data(int, "ipv4", "privateIpAddress")
+    end
+
+    # aggregate the total IP data
+    %w{public_ipv4 local_ipv4 public_ipv6 local_ipv6}.each do |type|
+      metadata["network"]["interfaces"].each_value do |val|
+        metadata["network"][type].concat val[type] unless val[type].empty?
+      end
+    end
+
+    metadata
+  end
 end
diff --git a/lib/ohai/plugins/cloud.rb b/lib/ohai/plugins/cloud.rb
index 1c9c2832..58011d65 100644
--- a/lib/ohai/plugins/cloud.rb
+++ b/lib/ohai/plugins/cloud.rb
@@ -265,8 +265,10 @@ Ohai.plugin(:Cloud) do
 
   # Fill cloud hash with azure values
   def get_azure_values
-    @cloud_attr_obj.add_ipv4_addr(azure["public_ip"], :public)
-    @cloud_attr_obj.add_ipv4_addr(azure["private_ip"], :private)
+    azure["metadata"]["network"]["public_ipv4"].each { |ipaddr| @cloud_attr_obj.add_ipv4_addr(ipaddr, :public) }
+    azure["metadata"]["network"]["public_ipv6"].each { |ipaddr| @cloud_attr_obj.add_ipv6_addr(ipaddr, :public) }
+    azure["metadata"]["network"]["local_ipv4"].each { |ipaddr| @cloud_attr_obj.add_ipv4_addr(ipaddr, :private) }
+    azure["metadata"]["network"]["local_ipv6"].each { |ipaddr| @cloud_attr_obj.add_ipv6_addr(ipaddr, :private) }
     @cloud_attr_obj.public_hostname = azure["public_fqdn"]
     @cloud_attr_obj.provider = "azure"
   end
diff --git a/spec/unit/mixin/azure_metadata_spec.rb b/spec/unit/mixin/azure_metadata_spec.rb
new file mode 100644
index 00000000..e7414120
--- /dev/null
+++ b/spec/unit/mixin/azure_metadata_spec.rb
@@ -0,0 +1,67 @@
+#
+# Author:: Tim Smith <tsmith@chef.io>
+# Copyright:: 2017 Chef Software, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDIT"Net::HTTP Response"NS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require_relative "../../spec_helper.rb"
+require "ohai/mixin/azure_metadata"
+
+describe Ohai::Mixin::AzureMetadata do
+  let(:mixin) do
+    mixin = Object.new.extend(::Ohai::Mixin::AzureMetadata)
+    mixin
+  end
+
+  describe "#http_get" do
+    it "gets the passed URI" do
+      http_mock = double("http")
+      allow(http_mock).to receive(:read_timeout=)
+      allow(Net::HTTP).to receive(:start).with("169.254.169.254").and_return(http_mock)
+
+      expect(http_mock).to receive(:get).with("http://www.chef.io", initheader = { "Metadata" => "true" })
+      mixin.http_get("http://www.chef.io")
+    end
+  end
+
+  describe "#fetch_metadata" do
+    it "returns an empty hash given a non-200 response" do
+      http_mock = double("http", { :code => "500" })
+      allow(mixin).to receive(:http_get).and_return(http_mock)
+
+      expect(Ohai::Log).to receive(:warn)
+      vals = mixin.fetch_metadata
+      expect(vals).to eq(nil)
+    end
+
+    it "returns an empty hash given invalid JSON response" do
+      http_mock = double("http", { :code => "200", :body => '{ "foo" "bar"}' })
+      allow(mixin).to receive(:http_get).and_return(http_mock)
+
+      expect(Ohai::Log).to receive(:warn)
+      vals = mixin.fetch_metadata
+      expect(vals).to eq(nil)
+    end
+
+    it "returns a populated hash given valid JSON response" do
+      http_mock = double("http", { :code => "200", :body => '{ "foo": "bar"}' })
+      allow(mixin).to receive(:http_get).and_return(http_mock)
+
+      expect(Ohai::Log).not_to receive(:warn)
+      vals = mixin.fetch_metadata
+      expect(vals).to eq({ "foo" => "bar" })
+    end
+  end
+end
diff --git a/spec/unit/mixin/softlayer_metadata_spec.rb b/spec/unit/mixin/softlayer_metadata_spec.rb
index 6fe2ddee..ec20e12b 100644
--- a/spec/unit/mixin/softlayer_metadata_spec.rb
+++ b/spec/unit/mixin/softlayer_metadata_spec.rb
@@ -36,7 +36,7 @@ describe ::Ohai::Mixin::SoftlayerMetadata do
   end
 
   context "fetch_metadata" do
-    it "riases error if softlayer api query results with error" do
+    it "raises error if softlayer api query results with error" do
       http_mock = double("http", { :ssl_version= => true, :use_ssl= => true, :ca_file= => true })
       allow(http_mock).to receive(:get).and_raise(StandardError.new("API return fake error"))
       allow(::Net::HTTP).to receive(:new).with("api.service.softlayer.com", 443).and_return(http_mock)
diff --git a/spec/unit/plugins/azure_spec.rb b/spec/unit/plugins/azure_spec.rb
index 82277406..cf386657 100644
--- a/spec/unit/plugins/azure_spec.rb
+++ b/spec/unit/plugins/azure_spec.rb
@@ -31,6 +31,33 @@ describe Ohai::System, "plugin azure" do
     }
   end
 
+  let(:response_data) do
+    { "compute" => { "location" => "westus",
+                     "name" => "timtest",
+                     "offer" => "UbuntuServer",
+                     "osType" => "Linux",
+                     "platformFaultDomain" => "0",
+                     "platformUpdateDomain" => "0",
+                     "publisher" => "Canonical",
+                     "sku" => "16.04-LTS",
+                     "version" => "16.04.201707270",
+                     "vmId" => "f78151b3-da8b-4bd8-a592-d9ce8a57ea65",
+                     "vmSize" => "Standard_DS2_v2" },
+      "network" => { "interface" => [ { "ipv4" =>
+                                          {  "ipAddress" => [{ "privateIpAddress" => "10.0.1.6", "publicIpAddress" => "40.118.212.225" }],
+                                             "subnet" => [{ "address" => "10.0.1.0", "prefix" => "24" }] },
+                                        "ipv6" =>
+                                          { "ipAddress" => [] },
+                                        "macAddress" => "000D3A37F080" }] } }
+  end
+
+  before do
+    # skips all the metadata logic unless we want to test it
+    allow(plugin).to receive(:can_socket_connect?).
+      with(Ohai::Mixin::AzureMetadata::AZURE_METADATA_ADDR, 80).
+      and_return(false)
+  end
+
   shared_examples_for "!azure" do
     it "does not set the azure attribute" do
       plugin.run
@@ -42,6 +69,7 @@ describe Ohai::System, "plugin azure" do
     it "sets the azure attribute" do
       plugin.run
       expect(plugin[:azure]).to be_truthy
+      expect(plugin[:azure]).to have_key(:metadata)
     end
   end
 
@@ -159,4 +187,57 @@ describe Ohai::System, "plugin azure" do
     it_behaves_like "azure"
   end
 
+  describe "with non-responsive metadata endpoint" do
+    before(:each) do
+      allow(plugin).to receive(:hint?).with("azure").and_return({})
+    end
+
+    it "does not return metadata information" do
+      allow(plugin).to receive(:can_socket_connect?).
+        with(Ohai::Mixin::AzureMetadata::AZURE_METADATA_ADDR, 80).
+        and_return(true)
+      allow(plugin).to receive(:fetch_metadata).and_return(nil)
+
+      plugin.run
+      expect(plugin[:azure]).to have_key(:metadata)
+      expect(plugin[:azure][:metadata]).to be_nil
+    end
+  end
+
+  describe "with responsive metadata endpoint" do
+    before(:each) do
+      allow(plugin).to receive(:hint?).with("azure").and_return({})
+      allow(plugin).to receive(:can_socket_connect?).
+        with(Ohai::Mixin::AzureMetadata::AZURE_METADATA_ADDR, 80).
+        and_return(true)
+      allow(plugin).to receive(:fetch_metadata).and_return(response_data)
+      plugin.run
+    end
+
+    it "returns metadata compute information" do
+      expect(plugin[:azure][:metadata][:compute][:location]).to eq("westus")
+      expect(plugin[:azure][:metadata][:compute][:name]).to eq("timtest")
+      expect(plugin[:azure][:metadata][:compute][:offer]).to eq("UbuntuServer")
+      expect(plugin[:azure][:metadata][:compute][:osType]).to eq("Linux")
+      expect(plugin[:azure][:metadata][:compute][:platformFaultDomain]).to eq("0")
+      expect(plugin[:azure][:metadata][:compute][:platformUpdateDomain]).to eq("0")
+      expect(plugin[:azure][:metadata][:compute][:publisher]).to eq("Canonical")
+      expect(plugin[:azure][:metadata][:compute][:sku]).to eq("16.04-LTS")
+      expect(plugin[:azure][:metadata][:compute][:version]).to eq("16.04.201707270")
+      expect(plugin[:azure][:metadata][:compute][:vmId]).to eq("f78151b3-da8b-4bd8-a592-d9ce8a57ea65")
+      expect(plugin[:azure][:metadata][:compute][:vmSize]).to eq("Standard_DS2_v2")
+    end
+
+    it "returns metadata network information" do
+      expect(plugin[:azure][:metadata][:network][:interfaces]["000D3A37F080"][:mac]).to eq("000D3A37F080")
+      expect(plugin[:azure][:metadata][:network][:interfaces]["000D3A37F080"][:public_ipv6]).to eq([])
+      expect(plugin[:azure][:metadata][:network][:interfaces]["000D3A37F080"][:public_ipv4]).to eq(["40.118.212.225"])
+      expect(plugin[:azure][:metadata][:network][:interfaces]["000D3A37F080"][:local_ipv6]).to eq([])
+      expect(plugin[:azure][:metadata][:network][:interfaces]["000D3A37F080"][:local_ipv4]).to eq(["10.0.1.6"])
+      expect(plugin[:azure][:metadata][:network][:public_ipv6]).to eq([])
+      expect(plugin[:azure][:metadata][:network][:public_ipv4]).to eq(["40.118.212.225"])
+      expect(plugin[:azure][:metadata][:network][:local_ipv6]).to eq([])
+      expect(plugin[:azure][:metadata][:network][:local_ipv4]).to eq(["10.0.1.6"])
+    end
+  end
 end
diff --git a/spec/unit/plugins/cloud_spec.rb b/spec/unit/plugins/cloud_spec.rb
index 1efe7f40..ae9de7b3 100644
--- a/spec/unit/plugins/cloud_spec.rb
+++ b/spec/unit/plugins/cloud_spec.rb
@@ -288,24 +288,56 @@ describe Ohai::System, "plugin cloud" do
   describe "with Azure mash" do
     before do
       @plugin[:azure] = Mash.new
+      @plugin[:azure][:metadata] = {
+        "compute" =>
+          {
+            "location" => "westus",
+            "name" => "timtest",
+            "offer" => "UbuntuServer",
+            "osType" => "Linux",
+            "platformFaultDomain" => "0",
+            "platformUpdateDomain" => "0",
+            "publisher" => "Canonical",
+            "sku" => "16.04-LTS",
+            "version" => "16.04.201707270",
+            "vmId" => "f78151b3-da8b-4bd8-a592-d9ce8357e365",
+            "vmSize" => "Standard_DS2_v2",
+          },
+        "network" =>
+          {
+            "interfaces" =>
+              {
+                "000D3A37F080" =>
+                  {
+                    "mac" => "000D3A37F080",
+                    "public_ipv6" => [],
+                    "public_ipv4" => ["40.118.212.225"],
+                    "local_ipv6" => [],
+                    "local_ipv4" => ["10.0.1.6"],
+                  },
+              },
+            "public_ipv4" => ["40.118.212.225"],
+            "local_ipv4" => ["10.0.1.6"],
+            "public_ipv6" => [],
+            "local_ipv6" => [],
+            },
+          }
     end
 
     it "populates cloud public ip" do
-      @plugin[:azure]["public_ip"] = "174.129.150.8"
       @plugin.run
-      expect(@plugin[:cloud][:public_ipv4_addrs][0]).to eq(@plugin[:azure]["public_ip"])
+      expect(@plugin[:cloud][:public_ipv4_addrs][0]).to eq("40.118.212.225")
     end
 
     it "doesn't populates cloud vm_name" do
-      @plugin[:azure]["vm_name"] = "linux-vm"
       @plugin.run
-      expect(@plugin[:cloud][:vm_name]).not_to eq(@plugin[:azure]["vm_name"])
+      expect(@plugin[:cloud][:vm_name]).not_to eq("timtest")
     end
 
     it "populates cloud public_hostname" do
       @plugin[:azure]["public_fqdn"] = "linux-vm-svc.cloudapp.net"
       @plugin.run
-      expect(@plugin[:cloud][:public_hostname]).to eq(@plugin[:azure]["public_fqdn"])
+      expect(@plugin[:cloud][:public_hostname]).to eq("linux-vm-svc.cloudapp.net")
     end
 
     it "doesn't populate cloud public_ssh_port" do
diff --git a/spec/unit/plugins/ec2_spec.rb b/spec/unit/plugins/ec2_spec.rb
index 25c3eba7..bf64da2b 100644
--- a/spec/unit/plugins/ec2_spec.rb
+++ b/spec/unit/plugins/ec2_spec.rb
@@ -23,14 +23,14 @@ require "base64"
 
 describe Ohai::System, "plugin ec2" do
 
+  let(:plugin) { get_plugin("ec2") }
+
   before(:each) do
     allow(plugin).to receive(:hint?).with("ec2").and_return(false)
     allow(File).to receive(:exist?).with("/sys/hypervisor/uuid").and_return(false)
   end
 
   shared_examples_for "!ec2" do
-    let(:plugin) { get_plugin("ec2") }
-
     it "DOESN'T attempt to fetch the ec2 metadata or set ec2 attribute" do
       expect(plugin).not_to receive(:http_client)
       expect(plugin[:ec2]).to be_nil
@@ -39,8 +39,6 @@ describe Ohai::System, "plugin ec2" do
   end
 
   shared_examples_for "ec2" do
-    let(:plugin) { get_plugin("ec2") }
-
     before(:each) do
       @http_client = double("Net::HTTP client")
       allow(plugin).to receive(:http_client).and_return(@http_client)