cli.py: Add --build-manifest to build command

This allows the user to opt in generating a build manifest containing
all of the elements and sources used for their build alongside a buildstream
version and build date/time.

_manifest.py: New module to provide functionality for producing a
manifest from a collection of elements.

source.py: Add _get_element_index() to allow access to
Source.__element_index.

3 files changed, 151 insertions, 1 deletions

diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py
index 20624e2ac..b883956ad 100644
--- a/buildstream/_frontend/cli.py
+++ b/buildstream/_frontend/cli.py
@@ -3,6 +3,7 @@ import sys
 
 import click
 from .. import _yaml
+from .. import _manifest
 from .._exceptions import BstError, LoadError, AppError
 from .._versions import BST_FORMAT_VERSION
 from .complete import main_bashcomplete, complete_path, CompleteUnhandled
@@ -289,6 +290,15 @@ def init(app, project_name, format_version, element_path, force):
 ##################################################################
 #                          Build Command                         #
 ##################################################################
+def _validate_manifest_path(ctx, param, value):
+    if not value:
+        return
+    if value.lower().endswith(".yaml") or value.lower().endswith(".yml"):
+        return os.path.abspath(value)
+    else:
+        raise click.BadParameter("Manifest files are outputted as YAML\n\t" +
+                                 "Please provide a path with a valid file extension (yml, yaml)")
+
 @cli.command(short_help="Build elements in a pipeline")
 @click.option('--all', 'all_', default=False, is_flag=True,
               help="Build elements that would not be needed for the current build plan")
@@ -305,10 +315,16 @@ def init(app, project_name, format_version, element_path, force):
               help="Allow tracking to cross junction boundaries")
 @click.option('--track-save', default=False, is_flag=True,
               help="Deprecated: This is ignored")
+@click.option('--build-manifest', default=False, is_flag=True,
+              help="Produces a build manifest containing elements and sources.")
+@click.option('--manifest-path', default=None, type=click.Path(readable=False),
+              help="Provides a path for a build manifest to be written to.",
+              callback=_validate_manifest_path)
 @click.argument('elements', nargs=-1,
                 type=click.Path(readable=False))
 @click.pass_obj
-def build(app, elements, all_, track_, track_save, track_all, track_except, track_cross_junctions):
+def build(app, elements, all_, track_, track_save, build_manifest,
+          manifest_path, track_all, track_except, track_cross_junctions):
     """Build elements in a pipeline"""
 
     if (track_except or track_cross_junctions) and not (track_ or track_all):
@@ -329,6 +345,13 @@ def build(app, elements, all_, track_, track_save, track_all, track_except, trac
                          track_cross_junctions=track_cross_junctions,
                          build_all=all_)
 
+        if build_manifest and not manifest_path:
+            manifest_path = os.path.join(app.project.directory,
+                                         "build_manifest.yaml")
+        if manifest_path:
+            _manifest.generate(app.context, app.stream.total_elements,
+                               app._session_start, manifest_path)
+
 
 ##################################################################
 #                          Fetch Command                         #
diff --git a/buildstream/_manifest.py b/buildstream/_manifest.py
new file mode 100644
index 000000000..a897866a6
--- /dev/null
+++ b/buildstream/_manifest.py
@@ -0,0 +1,124 @@
+#
+#  Copyright (C) 2018 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Josh Smith <josh.smith@codethink.co.uk>
+
+from ruamel import yaml
+from . import __version__ as bst_version
+from . import _yaml
+from ._message import MessageType, Message
+
+# generate():
+#
+# Generates a manfiest file from the collection of elements.
+#
+# Args:
+#    context (Context): Application context
+#    elements (list of Element): Collection of elements to build the manifest from
+#    build_start_datetime (datetime): The start of the build assosciated with this manifest
+#    manifest_path (str): Absolute path to write the manifest file to.
+#
+def generate(context, elements, build_start_datetime, manifest_path):
+    with context.timed_activity("Building Manifest"):
+        manifest = _build(elements, build_start_datetime)
+
+    _yaml.dump(manifest, manifest_path)
+    context.message(Message(None, MessageType.STATUS,
+                            "Manifest saved to {}".format(manifest_path)))
+
+# _build():
+#
+# Builds a manifest (dictionary) using the provided elements to be stored.
+#
+# Args:
+#    elements (list of Element): Collection of elements to build the manifest from
+#    build_start_datetime (datetime): The start of the build assosciated with this manifest
+#
+# Returns:
+#    (CommentedMap): A dictionary containing the entire
+#                    manifest produced from the provided elements.
+#
+def _build(elements, build_start_datetime):
+    manifest = yaml.comments.CommentedMap()
+
+    # Add BuildStream Version
+    manifest['BuildStream_Version'] = "{}".format(bst_version)
+    # Add Build Date
+    manifest['Build_Date'] = build_start_datetime.isoformat()
+    manifest['Elements'] = yaml.comments.CommentedMap()
+
+    # Sort elements
+    elements = sorted(elements, key=lambda e: e.normal_name)
+
+    # Add Elements
+    for elem in elements:
+        manifest['Elements'][elem.normal_name] = _build_element(elem)
+
+    return manifest
+
+# _build_element():
+#
+# Builds a manifest segment for an individual element.
+#
+# Args:
+#    element (Element): Element to extract information from
+#
+# Returns:
+#    (CommentedMap): A dictionary containing the information
+#                    extracted from the provided element
+#
+def _build_element(element):
+    element_dict = yaml.comments.CommentedMap()
+    sources = yaml.comments.CommentedMap()
+    # Add Cache Key
+    cache_key = element._get_cache_key()
+    if cache_key:
+        element_dict["Cache_Key"] = cache_key
+
+    # Add sources
+    for source in element.sources():
+        src = _build_source(source)
+        if src:
+            source_desc = "{}({})".format(source._get_element_index(), type(source).__name__)
+            sources[source_desc] = src
+    if sources:
+        element_dict['Sources'] = sources
+
+
+    return element_dict
+
+# _build_source():
+#
+# Builds a manifest segment for an individual source.
+#
+# Args:
+#    source (Source): Source to extract information from
+#
+# Returns:
+#    (CommentedMap): A dictionary containing the information
+#                    extracted from the provided source
+#
+def _build_source(source):
+    src = yaml.comments.CommentedMap()
+    if hasattr(source, "url") and source.url:
+        src["url"] = source.url
+    if hasattr(source, "ref") and source.ref:
+        src["ref"] = source.ref
+    if hasattr(source, "path") and source.path:
+        src["path"] = source.path
+
+    return src if src else None
diff --git a/buildstream/source.py b/buildstream/source.py
index 2f3f1c281..8f45ce0f8 100644
--- a/buildstream/source.py
+++ b/buildstream/source.py
@@ -786,6 +786,9 @@ class Source(Plugin):
         else:
             return None
 
+    def _get_element_index(self):
+        return self.__element_index
+
     #############################################################
     #                   Local Private Methods                   #
     #############################################################