openstack: read the dynamic metadata group vendor_data2.json (#777)

Add support for openstack's dynamic vendor data, which appears under openstack/latest/vendor_data2.json

This adds vendor_data2 to all pathways; it should be a no-op for non-OpenStack providers.

LP: #1841104

2 files changed, 62 insertions, 7 deletions

diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py
index fb2b55e8..8c968ae9 100644
--- a/tests/unittests/test_data.py
+++ b/tests/unittests/test_data.py
@@ -33,11 +33,12 @@ INSTANCE_ID = "i-testing"
 
 class FakeDataSource(sources.DataSource):
 
-    def __init__(self, userdata=None, vendordata=None):
+    def __init__(self, userdata=None, vendordata=None, vendordata2=None):
         sources.DataSource.__init__(self, {}, None, None)
         self.metadata = {'instance-id': INSTANCE_ID}
         self.userdata_raw = userdata
         self.vendordata_raw = vendordata
+        self.vendordata2_raw = vendordata2
 
 
 def count_messages(root):
@@ -105,13 +106,14 @@ class TestConsumeUserData(helpers.FilesystemMockingTestCase):
         self.assertEqual('qux', cc['baz'])
         self.assertEqual('qux2', cc['bar'])
 
-    def test_simple_jsonp_vendor_and_user(self):
+    def test_simple_jsonp_vendor_and_vendor2_and_user(self):
         # test that user-data wins over vendor
         user_blob = '''
 #cloud-config-jsonp
 [
      { "op": "add", "path": "/baz", "value": "qux" },
-     { "op": "add", "path": "/bar", "value": "qux2" }
+     { "op": "add", "path": "/bar", "value": "qux2" },
+     { "op": "add", "path": "/foobar", "value": "qux3" }
 ]
 '''
         vendor_blob = '''
@@ -119,12 +121,23 @@ class TestConsumeUserData(helpers.FilesystemMockingTestCase):
 [
      { "op": "add", "path": "/baz", "value": "quxA" },
      { "op": "add", "path": "/bar", "value": "quxB" },
-     { "op": "add", "path": "/foo", "value": "quxC" }
+     { "op": "add", "path": "/foo", "value": "quxC" },
+     { "op": "add", "path": "/corge", "value": "quxEE" }
+]
+'''
+        vendor2_blob = '''
+#cloud-config-jsonp
+[
+     { "op": "add", "path": "/corge", "value": "quxD" },
+     { "op": "add", "path": "/grault", "value": "quxFF" },
+     { "op": "add", "path": "/foobar", "value": "quxGG" }
 ]
 '''
         self.reRoot()
         initer = stages.Init()
-        initer.datasource = FakeDataSource(user_blob, vendordata=vendor_blob)
+        initer.datasource = FakeDataSource(user_blob,
+                                           vendordata=vendor_blob,
+                                           vendordata2=vendor2_blob)
         initer.read_cfg()
         initer.initialize()
         initer.fetch()
@@ -138,9 +151,15 @@ class TestConsumeUserData(helpers.FilesystemMockingTestCase):
         (_which_ran, _failures) = mods.run_section('cloud_init_modules')
         cfg = mods.cfg
         self.assertIn('vendor_data', cfg)
+        self.assertIn('vendor_data2', cfg)
+        # Confirm that vendordata2 overrides vendordata, and that
+        #  userdata overrides both
         self.assertEqual('qux', cfg['baz'])
         self.assertEqual('qux2', cfg['bar'])
+        self.assertEqual('qux3', cfg['foobar'])
         self.assertEqual('quxC', cfg['foo'])
+        self.assertEqual('quxD', cfg['corge'])
+        self.assertEqual('quxFF', cfg['grault'])
 
     def test_simple_jsonp_no_vendor_consumed(self):
         # make sure that vendor data is not consumed
@@ -294,6 +313,10 @@ run:
 #!/bin/bash
 echo "test"
 '''
+        vendor2_blob = '''
+#!/bin/bash
+echo "dynamic test"
+'''
 
         user_blob = '''
 #cloud-config
@@ -303,7 +326,9 @@ vendor_data:
 '''
         new_root = self.reRoot()
         initer = stages.Init()
-        initer.datasource = FakeDataSource(user_blob, vendordata=vendor_blob)
+        initer.datasource = FakeDataSource(user_blob,
+                                           vendordata=vendor_blob,
+                                           vendordata2=vendor2_blob)
         initer.read_cfg()
         initer.initialize()
         initer.fetch()
diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py
index 415755aa..478f3503 100644
--- a/tests/unittests/test_datasource/test_openstack.py
+++ b/tests/unittests/test_datasource/test_openstack.py
@@ -40,6 +40,9 @@ USER_DATA = b'#!/bin/sh\necho This is user data\n'
 VENDOR_DATA = {
     'magic': '',
 }
+VENDOR_DATA2 = {
+    'static': {}
+}
 OSTACK_META = {
     'availability_zone': 'nova',
     'files': [{'content_path': '/content/0000', 'path': '/etc/foo.cfg'},
@@ -60,6 +63,7 @@ OS_FILES = {
         {'links': [], 'networks': [], 'services': []}),
     'openstack/latest/user_data': USER_DATA,
     'openstack/latest/vendor_data.json': json.dumps(VENDOR_DATA),
+    'openstack/latest/vendor_data2.json': json.dumps(VENDOR_DATA2),
 }
 EC2_FILES = {
     'latest/user-data': USER_DATA,
@@ -142,6 +146,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
         _register_uris(self.VERSION, EC2_FILES, EC2_META, OS_FILES)
         f = _read_metadata_service()
         self.assertEqual(VENDOR_DATA, f.get('vendordata'))
+        self.assertEqual(VENDOR_DATA2, f.get('vendordata2'))
         self.assertEqual(CONTENT_0, f['files']['/etc/foo.cfg'])
         self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg'])
         self.assertEqual(2, len(f['files']))
@@ -163,6 +168,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
         _register_uris(self.VERSION, {}, {}, OS_FILES)
         f = _read_metadata_service()
         self.assertEqual(VENDOR_DATA, f.get('vendordata'))
+        self.assertEqual(VENDOR_DATA2, f.get('vendordata2'))
         self.assertEqual(CONTENT_0, f['files']['/etc/foo.cfg'])
         self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg'])
         self.assertEqual(USER_DATA, f.get('userdata'))
@@ -195,6 +201,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
         _register_uris(self.VERSION, {}, {}, os_files)
         f = _read_metadata_service()
         self.assertEqual(VENDOR_DATA, f.get('vendordata'))
+        self.assertEqual(VENDOR_DATA2, f.get('vendordata2'))
         self.assertEqual(CONTENT_0, f['files']['/etc/foo.cfg'])
         self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg'])
         self.assertFalse(f.get('userdata'))
@@ -210,6 +217,17 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
         self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg'])
         self.assertFalse(f.get('vendordata'))
 
+    def test_vendordata2_empty(self):
+        os_files = copy.deepcopy(OS_FILES)
+        for k in list(os_files.keys()):
+            if k.endswith('vendor_data2.json'):
+                os_files.pop(k, None)
+        _register_uris(self.VERSION, {}, {}, os_files)
+        f = _read_metadata_service()
+        self.assertEqual(CONTENT_0, f['files']['/etc/foo.cfg'])
+        self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg'])
+        self.assertFalse(f.get('vendordata2'))
+
     def test_vendordata_invalid(self):
         os_files = copy.deepcopy(OS_FILES)
         for k in list(os_files.keys()):
@@ -218,6 +236,14 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
         _register_uris(self.VERSION, {}, {}, os_files)
         self.assertRaises(BrokenMetadata, _read_metadata_service)
 
+    def test_vendordata2_invalid(self):
+        os_files = copy.deepcopy(OS_FILES)
+        for k in list(os_files.keys()):
+            if k.endswith('vendor_data2.json'):
+                os_files[k] = '{'  # some invalid json
+        _register_uris(self.VERSION, {}, {}, os_files)
+        self.assertRaises(BrokenMetadata, _read_metadata_service)
+
     def test_metadata_invalid(self):
         os_files = copy.deepcopy(OS_FILES)
         for k in list(os_files.keys()):
@@ -246,6 +272,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
         self.assertEqual(USER_DATA, ds_os.userdata_raw)
         self.assertEqual(2, len(ds_os.files))
         self.assertEqual(VENDOR_DATA, ds_os.vendordata_pure)
+        self.assertEqual(VENDOR_DATA2, ds_os.vendordata2_pure)
         self.assertIsNone(ds_os.vendordata_raw)
         m_dhcp.assert_not_called()
 
@@ -278,6 +305,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
         self.assertEqual(USER_DATA, ds_os_local.userdata_raw)
         self.assertEqual(2, len(ds_os_local.files))
         self.assertEqual(VENDOR_DATA, ds_os_local.vendordata_pure)
+        self.assertEqual(VENDOR_DATA2, ds_os_local.vendordata2_pure)
         self.assertIsNone(ds_os_local.vendordata_raw)
         m_dhcp.assert_called_with('eth9', None)
 
@@ -401,7 +429,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
         self.assertIsNone(ds_os.vendordata_raw)
         self.assertEqual(
             ['dsmode', 'ec2-metadata', 'files', 'metadata', 'networkdata',
-             'userdata', 'vendordata', 'version'],
+             'userdata', 'vendordata', 'vendordata2', 'version'],
             sorted(crawled_data.keys()))
         self.assertEqual('local', crawled_data['dsmode'])
         self.assertEqual(EC2_META, crawled_data['ec2-metadata'])
@@ -415,6 +443,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase):
             crawled_data['networkdata'])
         self.assertEqual(USER_DATA, crawled_data['userdata'])
         self.assertEqual(VENDOR_DATA, crawled_data['vendordata'])
+        self.assertEqual(VENDOR_DATA2, crawled_data['vendordata2'])
         self.assertEqual(2, crawled_data['version'])
 
 
@@ -681,6 +710,7 @@ class TestMetadataReader(test_helpers.HttprettyTestCase):
             'version': 2,
             'metadata': expected_md,
             'vendordata': vendor_data,
+            'vendordata2': vendor_data2,
             'networkdata': network_data,
             'ec2-metadata': mock_read_ec2.return_value,
             'files': {},