Merge "Add support for configdrive in anaconda interface"

5 files changed, 311 insertions, 0 deletions

diff --git a/ironic/common/kickstart_utils.py b/ironic/common/kickstart_utils.py
new file mode 100644
index 000000000..519cb5326
--- /dev/null
+++ b/ironic/common/kickstart_utils.py
@@ -0,0 +1,165 @@
+# Copyright 2021 Verizon Media
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+import base64
+import gzip
+import io
+import os
+import tempfile
+
+from ironic_lib import utils as ironic_utils
+from oslo_log import log as logging
+import pycdlib
+import requests
+
+from ironic.common import exception
+
+LOG = logging.getLogger(__name__)
+
+
+def _get_config_drive_dict_from_iso(
+        iso_reader, drive_dict,
+        target_path='/var/lib/cloud/seed/config_drive'):
+    """Traverse the config drive iso and extract content and filenames
+
+    :param iso_reader: pycdlib.PyCdlib object representing ISO files.
+    :param drive_dict: Mutable dictionary to store path and contents.
+    :param target_path: Path on the local disk in which the files in config
+                        drive files has to be written.
+    """
+    for path, dirlist, filelist in iso_reader.walk(iso_path='/'):
+        for f in filelist:
+            # In iso9660 file extensions are mangled. Example '/FOO/BAR;1'.
+            iso_file_path = os.path.join(path, f)
+            file_record = iso_reader.get_record(iso_path=iso_file_path)
+            # This converts /FOO/BAR;1 -> /foo/bar
+            posix_file_path = iso_reader.full_path_from_dirrecord(
+                file_record, rockridge=True
+            )
+            # Path to which the file in config drive to be written on the
+            # server.
+            posix_file_path = posix_file_path.lstrip('/')
+            target_file_path = os.path.join(target_path, posix_file_path)
+            b_buf = io.BytesIO()
+            iso_reader.get_file_from_iso_fp(
+                iso_path=iso_file_path, outfp=b_buf
+            )
+            b_buf.seek(0)
+            content = b"\n".join(b_buf.readlines()).decode('utf-8')
+            drive_dict[target_file_path] = content
+
+
+def read_iso9600_config_drive(config_drive):
+    """Read config drive and store it's contents in a dict
+
+    :param config_drive: Config drive in iso9600 format
+    :returns: A dict containing path as key and contents of the configdrive
+              file as value.
+    """
+    config_drive_dict = dict()
+    with tempfile.NamedTemporaryFile(suffix='.iso') as iso:
+        iso.write(config_drive)
+        iso.flush()
+        try:
+            iso_reader = pycdlib.PyCdlib()
+            iso_reader.open(iso.name)
+            _get_config_drive_dict_from_iso(iso_reader, config_drive_dict)
+            iso_reader.close()
+        except Exception as e:
+            msg = "Error reading the config drive iso: %s" % e
+            LOG.error(msg)
+    return config_drive_dict
+
+
+def decode_and_extract_config_drive_iso(config_drive_iso_gz):
+    try:
+        iso_gz_obj = io.BytesIO(base64.b64decode(config_drive_iso_gz))
+        iso_gz_obj.seek(0)
+    except Exception as exc:
+        if isinstance(config_drive_iso_gz, bytes):
+            LOG.debug('Config drive is not base64 encoded (%(error)s), '
+                      'assuming binary', {'error': exc})
+            iso_gz_obj = config_drive_iso_gz
+        else:
+            error_msg = ('Config drive is not base64 encoded or the content '
+                         'is malformed. %(cls)s: %(err)s.'
+                         % {'err': exc, 'cls': type(exc).__name__})
+            raise exception.InstanceDeployFailure(error_msg)
+
+    try:
+        with gzip.GzipFile(fileobj=iso_gz_obj, mode='rb') as f:
+            config_drive_iso = f.read()
+    except Exception as exc:
+        error_msg = "Decoding/Extraction of config drive failed: %s" % exc
+        raise exception.InstanceDeployFailure(error_msg)
+    return config_drive_iso
+
+
+def _fetch_config_drive_from_url(url):
+    try:
+        config_drive = requests.get(url).content
+    except requests.exceptions.RequestException as e:
+        raise exception.InstanceDeployFailure(
+            "Can't download the configdrive content from '%(url)s'. "
+            "Reason: %(reason)s" %
+            {'url': url, 'reason': e})
+    config_drive_iso = decode_and_extract_config_drive_iso(config_drive)
+    return read_iso9600_config_drive(config_drive_iso)
+
+
+def _write_config_drive_content(content, file_path):
+    """Generate post ks script to write each userdata content."""
+
+    content = base64.b64encode(str.encode(content))
+    kickstart_data = []
+    kickstart_data.append("\n")
+    kickstart_data.append("%post\n")
+    kickstart_data.append(("DIRPATH=`/usr/bin/dirname "
+                           "{file_path}`\n").format(
+        file_path=file_path))
+    kickstart_data.append("/bin/mkdir -p $DIRPATH\n")
+    kickstart_data.append("CONTENT='{content}'\n".format(
+        content=content))
+    kickstart_data.append("echo $CONTENT | "
+                          "/usr/bin/base64 --decode > "
+                          "{file_path}".format(file_path=file_path))
+    kickstart_data.append("\n")
+    kickstart_data.append(
+        "/bin/chmod 600 {file_path}\n".format(file_path=file_path)
+    )
+    kickstart_data.append("%end\n\n")
+
+    return "".join(kickstart_data)
+
+
+def prepare_config_drive(task,
+                         config_drive_path='/var/lib/cloud/seed/config_drive'):
+    """Prepare config_drive for writing to kickstart file"""
+    LOG.debug("Preparing config_drive to write to kickstart file")
+    node = task.node
+    config_drive = node.instance_info.get('configdrive')
+    ks_config_drive = ''
+    if not config_drive:
+        return ks_config_drive
+
+    if not isinstance(config_drive, dict) and \
+            ironic_utils.is_http_url(config_drive):
+        config_drive = _fetch_config_drive_from_url(config_drive)
+
+    for key in sorted(config_drive.keys()):
+        target_path = os.path.join(config_drive_path, key)
+        ks_config_drive += _write_config_drive_content(
+            config_drive[key], target_path
+        )
+
+    return ks_config_drive
diff --git a/ironic/common/pxe_utils.py b/ironic/common/pxe_utils.py
index d425cffe4..cd5a57084 100644
--- a/ironic/common/pxe_utils.py
+++ b/ironic/common/pxe_utils.py
@@ -31,6 +31,7 @@ from ironic.common.glance_service import service_utils
 from ironic.common.i18n import _
 from ironic.common import image_service as service
 from ironic.common import images
+from ironic.common import kickstart_utils as ks_utils
 from ironic.common import states
 from ironic.common import utils
 from ironic.conductor import utils as manager_utils
@@ -1195,6 +1196,9 @@ def prepare_instance_kickstart_config(task, image_info, anaconda_boot=False):
     ks_options = build_kickstart_config_options(task)
     kickstart_template = image_info['ks_template'][1]
     ks_cfg = utils.render_template(kickstart_template, ks_options)
+    ks_config_drive = ks_utils.prepare_config_drive(task)
+    if ks_config_drive:
+        ks_cfg = ks_cfg + ks_config_drive
     utils.write_to_file(image_info['ks_cfg'][1], ks_cfg)
 
 
diff --git a/ironic/tests/unit/common/test_kickstart_utils.py b/ironic/tests/unit/common/test_kickstart_utils.py
new file mode 100644
index 000000000..fffacf7d4
--- /dev/null
+++ b/ironic/tests/unit/common/test_kickstart_utils.py
@@ -0,0 +1,132 @@
+# Copyright 2021 Verizon Media
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import base64
+import os
+from unittest import mock
+
+from oslo_config import cfg
+
+from ironic.common import kickstart_utils as ks_utils
+from ironic.conductor import task_manager
+from ironic.drivers.modules import ipxe
+from ironic.tests.unit.db import base as db_base
+from ironic.tests.unit.db import utils as db_utils
+from ironic.tests.unit.objects import utils as object_utils
+
+CONF = cfg.CONF
+INST_INFO_DICT = db_utils.get_test_pxe_instance_info()
+DRV_INFO_DICT = db_utils.get_test_pxe_driver_info()
+DRV_INTERNAL_INFO_DICT = db_utils.get_test_pxe_driver_internal_info()
+CONFIG_DRIVE = ('H4sICDw0S2AC/3RtcGhYdnFvdADt3X1vFMcdAOBZkwbTIquiL6oiJ9kkkJBKN'
+                'mcTkTiVKl3Oa3uTe9PdOYK/0AmOvNqO4IJatZWav5pK/UztV8kXiPoR2tm98x'
+                's+fCQQMPA8i71zs7Mz4/VJvx0vMxcCAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'
+                'AAAAAAAAAAJDUViuVpSTU8+bm1fT+buxs3/rsk4Xl+x3fre8/h3bHtBv/FV9h'
+                'dja8P8p6/9f7h39bfHs9zI9ezYfZYjcb/nzqbDI//93M7vnpE7bTH36a7nw12'
+                'L4z7N/4Ih0O+lvp82Q9a+bdVt6ormdpTKQrV65ULm2sddO1vJ51r3V7WSOtdb'
+                'Jqr9VJL9beTpdWVi6n2eK11mZzfbVaz3Yz311YrlSupB8utrNqp9tqXvpwsVv'
+                'byOvxXblelikOF2XeTWurnY/yXtrLqo3H/uMuV5aXKpeXlitLy+8tv1epfHck'
+                'o3KPcKTEk3/T8mQJOpwYM+P4H+ohD82wGa6GdOJ2I+yE7XArfBY+CQth+cjxe'
+                '+L/hUvZA8f/5iir+bv9wy+N4v+42vR+8f8+fX18207oh2H4tEx9FQbxCt2Jr/'
+                'vxan0R84Yxpx+2nngvf7ptPWTx15eHbmjF741QLXPScU4aVsKVuFXC9bAR1mJ'
+                'eGr/n8b2WxfS1+NWLqUbMrYVOTFXj61ZMpeFizHk77pdiDSvhckxlYTGe0Yrv'
+                '0GZsYzWWrZctTd8eXSHxH/GfZ8j/duM/AAAA8MxKymfsxfj/THi5TO09zg6nw'
+                '6sxZybc2NkeDraH4cXwSvn6y/5wcGfo2gEAAMDTM/4Pxf+vT4rxf/RySA6O/6'
+                'NXw8z++D96JcwY/wMAAMDTNv5Px38FOBdeG6WOzGSbC2+E4rn/eA7gsDw6PBt'
+                'eH+V+Wc6BG5TlAQAAgBM5/g/F2idJMf6PXismABwd/0dvFBMBDo//Q7FEz4zx'
+                'PwAAAJx0305dY7/bPp38+7+h0/lZ8k376vlkq1qUq26dGp136t4ae2svJXPjS'
+                'g7vatl8cn5U6Pxu6e/Hu1vT+pE8gg6Ev5ZrHIRinsPEVs7sTX4oWvtnszF3YD'
+                '2Eg22/MKrmhR/QNgCcHLemRMTkaOD/EbHv8UT3P5XrFYVizuLEVk6PJzKOY/v'
+                'ZZHdlo4PtzoyqmPkB7d4t10UKxdzIie2+OJ4wOW73F8l4BaWHbBYAHiL+Hx+7'
+                'JsT/HxGqpt5lJI/iLuPbcGFU5sJuF/dDZdHKL7cGw/71m/1hf/HzOzvbf1jaj'
+                'ci/SkJxaGHvUNGR898UVXxzfvzZCMmDd+Tv4c1RkTfnRvu5w/04+/Wdwe1RP/'
+                'b7MJeEveyHaz78K7w1KvPW5Otw7u5g++bO7UlX4jdJuPfgQ3YGgBMa/48fMz9'
+                'N8X8YLo7KXJwd7WcPx73TxSeyxZA7jnVnklBkiG8APH+mf8bu1BLJO+XKAaGY'
+                'PTCxxLkJH44LADzJ+H987H6Q+F8p1wcKxRzBiSXmDk8cDIvlykFl4xPLnzWlE'
+                'AB+4vh/fCxOpt8hJH+c8tx9PmzFWF6M/BfCzTKy9+M9wOcxuhd3Be9MeVp+Ln'
+                'wdSw7C7XB97+wPpjzhTsPd8l7jZmzh4Hn7rQLA8x3/jx+7P0j8//2U5+6zoTL'
+                'eAICTIOt8n/y894+k08nb15dWVpaqvY0s7bRqH6WdfHU9S/NmL+vUNqrNmG53'
+                'Wr1WrVUvEh/nq1k37W62261OL11rddJ2q5tfTdfyepZ2r3V7WSPtZo1qs5fXu'
+                'u16Vu1maa3V7FVrvXQ179bS9uYH9by7kXXKk7vtrJav5bVqL281025rs1PLFt'
+                'NYQ3agYGwyVreWF8lm7ETeqHaupR+36puNLI3dqcUfotcaVbjbVt6MrxpltYt'
+                '+3QBQ+svfXAMAeN4U69CkexPPXQ8AMP4HAJ5F24PhgpE/AAAAAAAAAAAAAAAA'
+                'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'
+                'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'
+                'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'
+                'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'
+                'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'
+                'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'
+                'AAAAAAAAAAAAAAAAAn3f8BeXAIEgD4BQA=')
+
+
+@mock.patch.object(ipxe.iPXEBoot, '__init__', lambda self: None)
+class KSUtilsTestCase(db_base.DbTestCase):
+    def setUp(self):
+        super(KSUtilsTestCase, self).setUp()
+        n = {
+            'driver': 'fake-hardware',
+            'boot_interface': 'ipxe',
+            'instance_info': INST_INFO_DICT,
+            'driver_info': DRV_INFO_DICT,
+            'driver_internal_info': DRV_INTERNAL_INFO_DICT,
+        }
+        self.config(enabled_boot_interfaces=['ipxe'])
+        self.config_temp_dir('http_root', group='deploy')
+        self.node = object_utils.create_test_node(self.context, **n)
+        self.config_drive_dict = {
+            "openstack/content/0000": "net-data",
+            "openstack/latest/meta-data.json": "{}",
+            "openstack/latest/user_data": "test user_data",
+            "openstack/latest/vendor_data.json": "{}"
+        }
+
+    def _get_expected_ks_config_drive(self, config_drive_dict):
+        config_drive_ks_template = """\
+\n%post\nDIRPATH=`/usr/bin/dirname {file_path}`\n\
+/bin/mkdir -p $DIRPATH\n\
+CONTENT='{content}'\n\
+echo $CONTENT | /usr/bin/base64 --decode > {file_path}\n\
+/bin/chmod 600 {file_path}\n\
+%end\n\n"""
+
+        target_path = '/var/lib/cloud/seed/config_drive'
+        config_drive_ks = ''
+        for key in sorted(config_drive_dict.keys()):
+            config_drive_ks += config_drive_ks_template.format(
+                file_path=os.path.join(target_path, key),
+                content=base64.b64encode(str.encode(config_drive_dict[key]))
+            )
+        return config_drive_ks
+
+    def test_prepare_config_drive(self):
+
+        expected = self._get_expected_ks_config_drive(self.config_drive_dict)
+        with task_manager.acquire(self.context, self.node.uuid) as task:
+            i_info = task.node.instance_info
+            i_info['configdrive'] = self.config_drive_dict
+            task.node.instance_info = i_info
+            task.node.save()
+            self.assertEqual(expected, ks_utils.prepare_config_drive(task))
+
+    @mock.patch('requests.get', autospec=True)
+    def test_prepare_config_drive_in_swift(self, mock_get):
+        expected = self._get_expected_ks_config_drive(self.config_drive_dict)
+        mock_get.return_value = mock.MagicMock(content=CONFIG_DRIVE)
+        with task_manager.acquire(self.context, self.node.uuid) as task:
+            i_info = task.node.instance_info
+            i_info['configdrive'] = 'http://server/fake-configdrive-url'
+            task.node.instance_info = i_info
+            task.node.save()
+            self.assertEqual(expected, ks_utils.prepare_config_drive(task))
+            mock_get.assert_called_with('http://server/fake-configdrive-url')
diff --git a/releasenotes/notes/configdrive-support-in-anaconda-deploy-f2aad59b4ff809ec.yaml b/releasenotes/notes/configdrive-support-in-anaconda-deploy-f2aad59b4ff809ec.yaml
new file mode 100644
index 000000000..5b11a11f3
--- /dev/null
+++ b/releasenotes/notes/configdrive-support-in-anaconda-deploy-f2aad59b4ff809ec.yaml
@@ -0,0 +1,9 @@
+---
+features:
+  - |
+    The anaconda deploy interface now handles config drive. The config drive
+    contents are written to the disk at /var/lib/cloud/seed/config_drive
+    directory by the driver via kickstart files %post section. cloud-init
+    should be able to pick up the the config drive information and process
+    them. Because the config drive is extracted on to disk as plain text files
+    tools like glean will not work with this deploy interface.
diff --git a/requirements.txt b/requirements.txt
index 805993b42..27dcec63c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -29,6 +29,7 @@ oslo.utils>=4.5.0 # Apache-2.0
 osprofiler>=1.5.0 # Apache-2.0
 os-traits>=0.4.0 # Apache-2.0
 pecan!=1.0.2,!=1.0.3,!=1.0.4,!=1.2,>=1.0.0 # BSD
+pycdlib>=1.11.0 # LGPLv2
 requests>=2.14.2 # Apache-2.0
 rfc3986>=0.3.1 # Apache-2.0
 jsonpatch!=1.20,>=1.16 # BSD