Deploy templates: data model, DB API & objects

Adds deploy_templates and deploy_template_steps tables to the database,
provides a DB API for these tables, and a DeployTemplate versioned
object.

Change-Id: I5b8b59bbea1594b1220438050b80f1c603dbc346
Story: 1722275
Task: 28674

4 files changed, 398 insertions, 0 deletions

diff --git a/ironic/db/api.py b/ironic/db/api.py
index c21d7454d..135b762e4 100644
--- a/ironic/db/api.py
+++ b/ironic/db/api.py
@@ -1165,3 +1165,99 @@ class Connection(object):
         :param allocation_id: Allocation ID
         :raises: AllocationNotFound
         """
+
+    @abc.abstractmethod
+    def create_deploy_template(self, values, version):
+        """Create a deployment template.
+
+        :param values: A dict describing the deployment template. For example:
+
+                     ::
+
+                      {
+                       'uuid': uuidutils.generate_uuid(),
+                       'name': 'CUSTOM_DT1',
+                      }
+        :param version: the version of the object.DeployTemplate.
+        :raises: DeployTemplateDuplicateName if a deploy template with the same
+            name exists.
+        :raises: DeployTemplateAlreadyExists if a deploy template with the same
+            UUID exists.
+        :returns: A deploy template.
+        """
+
+    @abc.abstractmethod
+    def update_deploy_template(self, template_id, values):
+        """Update a deployment template.
+
+        :param template_id: ID of the deployment template to update.
+        :param values: A dict describing the deployment template. For example:
+
+                     ::
+
+                      {
+                       'uuid': uuidutils.generate_uuid(),
+                       'name': 'CUSTOM_DT1',
+                      }
+        :raises: DeployTemplateDuplicateName if a deploy template with the same
+            name exists.
+        :raises: DeployTemplateNotFound if the deploy template does not exist.
+        :returns: A deploy template.
+        """
+
+    @abc.abstractmethod
+    def destroy_deploy_template(self, template_id):
+        """Destroy a deployment template.
+
+        :param template_id: ID of the deployment template to destroy.
+        :raises: DeployTemplateNotFound if the deploy template does not exist.
+        """
+
+    @abc.abstractmethod
+    def get_deploy_template_by_id(self, template_id):
+        """Retrieve a deployment template by ID.
+
+        :param template_id: ID of the deployment template to retrieve.
+        :raises: DeployTemplateNotFound if the deploy template does not exist.
+        :returns: A deploy template.
+        """
+
+    @abc.abstractmethod
+    def get_deploy_template_by_uuid(self, template_uuid):
+        """Retrieve a deployment template by UUID.
+
+        :param template_uuid: UUID of the deployment template to retrieve.
+        :raises: DeployTemplateNotFound if the deploy template does not exist.
+        :returns: A deploy template.
+        """
+
+    @abc.abstractmethod
+    def get_deploy_template_by_name(self, template_name):
+        """Retrieve a deployment template by name.
+
+        :param template_name: name of the deployment template to retrieve.
+        :raises: DeployTemplateNotFound if the deploy template does not exist.
+        :returns: A deploy template.
+        """
+
+    @abc.abstractmethod
+    def get_deploy_template_list(self, limit=None, marker=None,
+                                 sort_key=None, sort_dir=None):
+        """Retrieve a list of deployment templates.
+
+        :param limit: Maximum number of deploy templates to return.
+        :param marker: The last item of the previous page; we return the next
+                       result set.
+        :param sort_key: Attribute by which results should be sorted.
+        :param sort_dir: Direction in which results should be sorted.
+                         (asc, desc)
+        :returns: A list of deploy templates.
+        """
+
+    @abc.abstractmethod
+    def get_deploy_template_list_by_names(self, names):
+        """Return a list of deployment templates with one of a list of names.
+
+        :param names: List of names to filter by.
+        :returns: A list of deploy templates.
+        """
diff --git a/ironic/db/sqlalchemy/alembic/versions/2aac7e0872f6_add_deploy_templates.py b/ironic/db/sqlalchemy/alembic/versions/2aac7e0872f6_add_deploy_templates.py
new file mode 100644
index 000000000..0b5e8ff10
--- /dev/null
+++ b/ironic/db/sqlalchemy/alembic/versions/2aac7e0872f6_add_deploy_templates.py
@@ -0,0 +1,67 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Create deploy_templates and deploy_template_steps tables.
+
+Revision ID: 2aac7e0872f6
+Revises: 28c44432c9c3
+Create Date: 2018-12-27 11:49:15.029650
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '2aac7e0872f6'
+down_revision = '28c44432c9c3'
+
+
+def upgrade():
+    op.create_table(
+        'deploy_templates',
+        sa.Column('version', sa.String(length=15), nullable=True),
+        sa.Column('created_at', sa.DateTime(), nullable=True),
+        sa.Column('updated_at', sa.DateTime(), nullable=True),
+        sa.Column('id', sa.Integer(), nullable=False,
+                  autoincrement=True),
+        sa.Column('uuid', sa.String(length=36)),
+        sa.Column('name', sa.String(length=255), nullable=False),
+        sa.PrimaryKeyConstraint('id'),
+        sa.UniqueConstraint('uuid', name='uniq_deploytemplates0uuid'),
+        sa.UniqueConstraint('name', name='uniq_deploytemplates0name'),
+        mysql_ENGINE='InnoDB',
+        mysql_DEFAULT_CHARSET='UTF8'
+    )
+
+    op.create_table(
+        'deploy_template_steps',
+        sa.Column('version', sa.String(length=15), nullable=True),
+        sa.Column('created_at', sa.DateTime(), nullable=True),
+        sa.Column('updated_at', sa.DateTime(), nullable=True),
+        sa.Column('id', sa.Integer(), nullable=False,
+                  autoincrement=True),
+        sa.Column('deploy_template_id', sa.Integer(), nullable=False,
+                  autoincrement=False),
+        sa.Column('interface', sa.String(length=255), nullable=False),
+        sa.Column('step', sa.String(length=255), nullable=False),
+        sa.Column('args', sa.Text, nullable=False),
+        sa.Column('priority', sa.Integer, nullable=False),
+        sa.PrimaryKeyConstraint('id'),
+        sa.ForeignKeyConstraint(['deploy_template_id'],
+                                ['deploy_templates.id']),
+        sa.Index('deploy_template_id', 'deploy_template_id'),
+        sa.Index('deploy_template_steps_interface_idx', 'interface'),
+        sa.Index('deploy_template_steps_step_idx', 'step'),
+        mysql_ENGINE='InnoDB',
+        mysql_DEFAULT_CHARSET='UTF8'
+    )
diff --git a/ironic/db/sqlalchemy/api.py b/ironic/db/sqlalchemy/api.py
index b134325f5..ec7e9dd67 100644
--- a/ironic/db/sqlalchemy/api.py
+++ b/ironic/db/sqlalchemy/api.py
@@ -85,6 +85,14 @@ def _get_node_query_with_all():
             .options(joinedload('traits')))
 
 
+def _get_deploy_template_query_with_steps():
+    """Return a query object for the DeployTemplate joined with steps.
+
+    :returns: a query object.
+    """
+    return model_query(models.DeployTemplate).options(joinedload('steps'))
+
+
 def model_query(model, *args, **kwargs):
     """Query helper for simpler session usage.
 
@@ -218,6 +226,42 @@ def _filter_active_conductors(query, interval=None):
     return query
 
 
+def _zip_matching(a, b, key):
+    """Zip two unsorted lists, yielding matching items or None.
+
+    Each zipped item is a tuple taking one of three forms:
+
+    (a[i], b[j]) if a[i] and b[j] are equal.
+    (a[i], None) if a[i] is less than b[j] or b is empty.
+    (None, b[j]) if a[i] is greater than b[j] or a is empty.
+
+    Note that the returned list may be longer than either of the two
+    lists.
+
+    Adapted from https://stackoverflow.com/a/11426702.
+
+    :param a: the first list.
+    :param b: the second list.
+    :param key: a function that generates a key used to compare items.
+    """
+    a = collections.deque(sorted(a, key=key))
+    b = collections.deque(sorted(b, key=key))
+    while a and b:
+        k_a = key(a[0])
+        k_b = key(b[0])
+        if k_a == k_b:
+            yield a.popleft(), b.popleft()
+        elif k_a < k_b:
+            yield a.popleft(), None
+        else:
+            yield None, b.popleft()
+    # Consume any remaining items in each deque.
+    for i in a:
+        yield i, None
+    for i in b:
+        yield None, i
+
+
 @profiler.trace_cls("db_api")
 class Connection(api.Connection):
     """SqlAlchemy connection."""
@@ -1710,3 +1754,155 @@ class Connection(api.Connection):
             node_query.update({'allocation_id': None, 'instance_uuid': None})
 
             query.delete()
+
+    @staticmethod
+    def _get_deploy_template_steps(steps, deploy_template_id=None):
+        results = []
+        for values in steps:
+            step = models.DeployTemplateStep()
+            step.update(values)
+            if deploy_template_id:
+                step['deploy_template_id'] = deploy_template_id
+            results.append(step)
+        return results
+
+    @oslo_db_api.retry_on_deadlock
+    def create_deploy_template(self, values, version):
+        steps = values.get('steps', [])
+        values['steps'] = self._get_deploy_template_steps(steps)
+
+        template = models.DeployTemplate()
+        template.update(values)
+        with _session_for_write() as session:
+            try:
+                session.add(template)
+                session.flush()
+            except db_exc.DBDuplicateEntry as e:
+                if 'name' in e.columns:
+                    raise exception.DeployTemplateDuplicateName(
+                        name=values['name'])
+                raise exception.DeployTemplateAlreadyExists(
+                    uuid=values['uuid'])
+        return template
+
+    def _update_deploy_template_steps(self, session, template_id, steps):
+        """Update the steps for a deploy template.
+
+        :param session: DB session object.
+        :param template_id: deploy template ID.
+        :param steps: list of steps that should exist for the deploy template.
+        """
+
+        def _step_key(step):
+            """Compare two deploy template steps."""
+            return step.interface, step.step, step.args, step.priority
+
+        # List all existing steps for the template.
+        query = (model_query(models.DeployTemplateStep)
+                 .filter_by(deploy_template_id=template_id))
+        current_steps = query.all()
+
+        # List the new steps for the template.
+        new_steps = self._get_deploy_template_steps(steps, template_id)
+
+        # The following is an efficient way to ensure that the steps in the
+        # database match those that have been requested. We compare the current
+        # and requested steps in a single pass using the _zip_matching
+        # function.
+        steps_to_create = []
+        step_ids_to_delete = []
+        for current_step, new_step in _zip_matching(current_steps, new_steps,
+                                                    _step_key):
+            if current_step is None:
+                # No matching current step found for this new step - create.
+                steps_to_create.append(new_step)
+            elif new_step is None:
+                # No matching new step found for this current step - delete.
+                step_ids_to_delete.append(current_step.id)
+            # else: steps match, no work required.
+
+        # Delete and create steps in bulk as necessary.
+        if step_ids_to_delete:
+            ((model_query(models.DeployTemplateStep)
+              .filter(models.DeployTemplateStep.id.in_(step_ids_to_delete)))
+             .delete(synchronize_session=False))
+        if steps_to_create:
+            session.bulk_save_objects(steps_to_create)
+
+    @oslo_db_api.retry_on_deadlock
+    def update_deploy_template(self, template_id, values):
+        if 'uuid' in values:
+            msg = _("Cannot overwrite UUID for an existing deploy template.")
+            raise exception.InvalidParameterValue(err=msg)
+
+        try:
+            with _session_for_write() as session:
+                # NOTE(mgoddard): Don't issue a joined query for the update as
+                # this does not work with PostgreSQL.
+                query = model_query(models.DeployTemplate)
+                query = add_identity_filter(query, template_id)
+                try:
+                    ref = query.with_lockmode('update').one()
+                except NoResultFound:
+                    raise exception.DeployTemplateNotFound(
+                        template=template_id)
+
+                # First, update non-step columns.
+                steps = None
+                if 'steps' in values:
+                    steps = values.pop('steps')
+
+                ref.update(values)
+
+                # If necessary, update steps.
+                if steps is not None:
+                    self._update_deploy_template_steps(session, ref.id, steps)
+
+                # Return the updated template joined with all relevant fields.
+                query = _get_deploy_template_query_with_steps()
+                query = add_identity_filter(query, template_id)
+                return query.one()
+        except db_exc.DBDuplicateEntry as e:
+            if 'name' in e.columns:
+                raise exception.DeployTemplateDuplicateName(
+                    name=values['name'])
+            raise
+
+    @oslo_db_api.retry_on_deadlock
+    def destroy_deploy_template(self, template_id):
+        with _session_for_write():
+            model_query(models.DeployTemplateStep).filter_by(
+                deploy_template_id=template_id).delete()
+            count = model_query(models.DeployTemplate).filter_by(
+                id=template_id).delete()
+            if count == 0:
+                raise exception.DeployTemplateNotFound(template=template_id)
+
+    def _get_deploy_template(self, field, value):
+        """Helper method for retrieving a deploy template."""
+        query = (_get_deploy_template_query_with_steps()
+                 .filter_by(**{field: value}))
+        try:
+            return query.one()
+        except NoResultFound:
+            raise exception.DeployTemplateNotFound(template=value)
+
+    def get_deploy_template_by_id(self, template_id):
+        return self._get_deploy_template('id', template_id)
+
+    def get_deploy_template_by_uuid(self, template_uuid):
+        return self._get_deploy_template('uuid', template_uuid)
+
+    def get_deploy_template_by_name(self, template_name):
+        return self._get_deploy_template('name', template_name)
+
+    def get_deploy_template_list(self, limit=None, marker=None,
+                                 sort_key=None, sort_dir=None):
+        query = _get_deploy_template_query_with_steps()
+        return _paginate_query(models.DeployTemplate, limit, marker,
+                               sort_key, sort_dir, query)
+
+    def get_deploy_template_list_by_names(self, names):
+        query = (_get_deploy_template_query_with_steps()
+                 .filter(models.DeployTemplate.name.in_(names)))
+        return query.all()
diff --git a/ironic/db/sqlalchemy/models.py b/ironic/db/sqlalchemy/models.py
index e70fefcc6..9d90d9aa3 100644
--- a/ironic/db/sqlalchemy/models.py
+++ b/ironic/db/sqlalchemy/models.py
@@ -350,6 +350,45 @@ class Allocation(Base):
                                 nullable=True)
 
 
+class DeployTemplate(Base):
+    """Represents a deployment template."""
+
+    __tablename__ = 'deploy_templates'
+    __table_args__ = (
+        schema.UniqueConstraint('uuid', name='uniq_deploytemplates0uuid'),
+        schema.UniqueConstraint('name', name='uniq_deploytemplates0name'),
+        table_args())
+    id = Column(Integer, primary_key=True)
+    uuid = Column(String(36))
+    name = Column(String(255), nullable=False)
+
+
+class DeployTemplateStep(Base):
+    """Represents a deployment step in a deployment template."""
+
+    __tablename__ = 'deploy_template_steps'
+    __table_args__ = (
+        Index('deploy_template_id', 'deploy_template_id'),
+        Index('deploy_template_steps_interface_idx', 'interface'),
+        Index('deploy_template_steps_step_idx', 'step'),
+        table_args())
+    id = Column(Integer, primary_key=True)
+    deploy_template_id = Column(Integer, ForeignKey('deploy_templates.id'),
+                                nullable=False)
+    interface = Column(String(255), nullable=False)
+    step = Column(String(255), nullable=False)
+    args = Column(db_types.JsonEncodedDict, nullable=False)
+    priority = Column(Integer, nullable=False)
+    deploy_template = orm.relationship(
+        "DeployTemplate",
+        backref='steps',
+        primaryjoin=(
+            'and_(DeployTemplateStep.deploy_template_id == '
+            'DeployTemplate.id)'),
+        foreign_keys=deploy_template_id
+    )
+
+
 def get_class(model_name):
     """Returns the model class with the specified name.