Populate files with content from get_file function calls

This is the client-side implementation of the get_file HOT
intrinsic function. The data for each resource is recursively
crawled to look for {get_file: <path>} calls. The content for
the URL specified by get_file is fetched and the content stored
in the files field in the API request.

If a relative path is specified, that path is converted to an
absolute URL, and that URL is used as the key for the files
content entry. The template is also modified to replace the
relative path with the absolute URL.

This implements the client-side portion of
blueprint get-file

Change-Id: If1da98578bb2919e3c8325442224376d3fadb0bc

3 files changed, 194 insertions, 57 deletions

diff --git a/heatclient/common/template_utils.py b/heatclient/common/template_utils.py
index bd54c9d..cdc71cc 100644
--- a/heatclient/common/template_utils.py
+++ b/heatclient/common/template_utils.py
@@ -14,6 +14,7 @@
 #    under the License.
 
 import os
+import six
 import urllib
 
 from heatclient.common import environment_format
@@ -46,28 +47,60 @@ def get_template_contents(template_file=None, template_url=None,
                                % template_url)
 
     try:
-        return template_format.parse(tpl)
+        template = template_format.parse(tpl)
     except ValueError as e:
         raise exc.CommandError(
             'Error parsing template %s %s' % (template_url, e))
 
+    files = {}
+    tmpl_base_url = base_url_for_url(template_url)
+    resolve_template_get_files(template, files, tmpl_base_url)
+    return files, template
+
+
+def resolve_template_get_files(template, files, template_base_url):
+
+    def ignore_if(key, value):
+        if key != 'get_file':
+            return True
+        if not isinstance(value, six.string_types):
+            return True
 
-def get_file_contents(from_dict, files, base_url=None,
-                      ignore_if=None):
-    for key, value in iter(from_dict.items()):
-        if ignore_if and ignore_if(key, value):
-            continue
+    def recurse_if(value):
+        return isinstance(value, (dict, list))
 
-        if base_url and not base_url.endswith('/'):
-            base_url = base_url + '/'
+    get_file_contents(template.get('resources'), files, template_base_url,
+                      ignore_if, recurse_if)
 
-        str_url = urlutils.urljoin(base_url, value)
-        try:
-            files[str_url] = urlutils.urlopen(str_url).read()
-        except urlutils.URLError:
-            raise exc.CommandError('Could not fetch %s from the environment'
-                                   % str_url)
-        from_dict[key] = str_url
+
+def get_file_contents(from_data, files, base_url=None,
+                      ignore_if=None, recurse_if=None):
+
+    if recurse_if and recurse_if(from_data):
+        if isinstance(from_data, dict):
+            recurse_data = from_data.itervalues()
+        else:
+            recurse_data = from_data
+        for value in recurse_data:
+            get_file_contents(value, files, base_url, ignore_if, recurse_if)
+
+    if isinstance(from_data, dict):
+        for key, value in iter(from_data.items()):
+            if ignore_if and ignore_if(key, value):
+                continue
+
+            if base_url and not base_url.endswith('/'):
+                base_url = base_url + '/'
+
+            str_url = urlutils.urljoin(base_url, value)
+            try:
+                files[str_url] = urlutils.urlopen(str_url).read()
+            except urlutils.URLError:
+                raise exc.CommandError('Could not fetch contents for %s'
+                                       % str_url)
+
+            # replace the data value with the normalised absolute URL
+            from_data[key] = str_url
 
 
 def base_url_for_url(url):
@@ -83,22 +116,21 @@ def normalise_file_path_to_url(path):
     return urlutils.urljoin('file:', urllib.pathname2url(path))
 
 
-def process_environment_and_files(env_path=None, template_path=None):
+def process_environment_and_files(env_path=None, template=None,
+                                  template_url=None):
     files = {}
     env = {}
 
-    if not env_path:
-        return files, env
-
-    env_url = normalise_file_path_to_url(env_path)
-    env_base_url = base_url_for_url(env_url)
-    raw_env = urlutils.urlopen(env_url).read()
-    env = environment_format.parse(raw_env)
+    if env_path:
+        env_url = normalise_file_path_to_url(env_path)
+        env_base_url = base_url_for_url(env_url)
+        raw_env = urlutils.urlopen(env_url).read()
+        env = environment_format.parse(raw_env)
 
-    resolve_environment_urls(
-        env.get('resource_registry'),
-        files,
-        env_base_url)
+        resolve_environment_urls(
+            env.get('resource_registry'),
+            files,
+            env_base_url)
 
     return files, env
 
diff --git a/heatclient/tests/test_template_utils.py b/heatclient/tests/test_template_utils.py
index 2b9b2ae..2b2fce1 100644
--- a/heatclient/tests/test_template_utils.py
+++ b/heatclient/tests/test_template_utils.py
@@ -269,9 +269,10 @@ class TestGetTemplateContents(testtools.TestCase):
             tmpl_file.write(tmpl)
             tmpl_file.flush()
 
-            tmpl_parsed = template_utils.get_template_contents(
+            files, tmpl_parsed = template_utils.get_template_contents(
                 tmpl_file.name)
             self.assertEqual({"foo": "bar"}, tmpl_parsed)
+            self.assertEqual({}, files)
 
     def test_get_template_contents_file_empty(self):
         with tempfile.NamedTemporaryFile() as tmpl_file:
@@ -316,8 +317,10 @@ class TestGetTemplateContents(testtools.TestCase):
         urlutils.urlopen(url).AndReturn(six.StringIO(tmpl))
         self.m.ReplayAll()
 
-        tmpl_parsed = template_utils.get_template_contents(template_url=url)
+        files, tmpl_parsed = template_utils.get_template_contents(
+            template_url=url)
         self.assertEqual({"foo": "bar"}, tmpl_parsed)
+        self.assertEqual({}, files)
 
     def test_get_template_contents_object(self):
         tmpl = '{"foo": "bar"}'
@@ -332,14 +335,109 @@ class TestGetTemplateContents(testtools.TestCase):
             self.assertEqual('http://no.where/path/to/a.yaml', object_url)
             return tmpl
 
-        tmpl_parsed = template_utils.get_template_contents(
+        files, tmpl_parsed = template_utils.get_template_contents(
             template_object=url,
             object_request=object_request)
 
         self.assertEqual({"foo": "bar"}, tmpl_parsed)
+        self.assertEqual({}, files)
         self.assertTrue(self.object_requested)
 
 
+class TestTemplateGetFileFunctions(testtools.TestCase):
+
+    hot_template = '''heat_template_version: 2013-05-23
+resources:
+  resource1:
+    type: type1
+    properties:
+      foo: {get_file: foo.yaml}
+      bar:
+        get_file:
+          'http://localhost/bar.yaml'
+  resource2:
+    type: type1
+    properties:
+      baz:
+      - {get_file: baz/baz1.yaml}
+      - {get_file: baz/baz2.yaml}
+      - {get_file: baz/baz3.yaml}
+      ignored_list: {get_file: [ignore, me]}
+      ignored_dict: {get_file: {ignore: me}}
+      ignored_none: {get_file: }
+    '''
+
+    def setUp(self):
+        super(TestTemplateGetFileFunctions, self).setUp()
+        self.m = mox.Mox()
+
+        self.addCleanup(self.m.VerifyAll)
+        self.addCleanup(self.m.UnsetStubs)
+
+    def test_hot_template(self):
+        self.m.StubOutWithMock(urlutils, 'urlopen')
+
+        tmpl_file = '/home/my/dir/template.yaml'
+        url = 'file:///home/my/dir/template.yaml'
+        urlutils.urlopen(url).AndReturn(
+            six.StringIO(self.hot_template))
+        urlutils.urlopen(
+            'http://localhost/bar.yaml').InAnyOrder().AndReturn(
+                six.StringIO('bar contents'))
+        urlutils.urlopen(
+            'file:///home/my/dir/foo.yaml').InAnyOrder().AndReturn(
+                six.StringIO('foo contents'))
+        urlutils.urlopen(
+            'file:///home/my/dir/baz/baz1.yaml').InAnyOrder().AndReturn(
+                six.StringIO('baz1 contents'))
+        urlutils.urlopen(
+            'file:///home/my/dir/baz/baz2.yaml').InAnyOrder().AndReturn(
+                six.StringIO('baz2 contents'))
+        urlutils.urlopen(
+            'file:///home/my/dir/baz/baz3.yaml').InAnyOrder().AndReturn(
+                six.StringIO('baz3 contents'))
+
+        self.m.ReplayAll()
+
+        files, tmpl_parsed = template_utils.get_template_contents(
+            template_file=tmpl_file)
+
+        self.assertEqual({
+            'http://localhost/bar.yaml': 'bar contents',
+            'file:///home/my/dir/foo.yaml': 'foo contents',
+            'file:///home/my/dir/baz/baz1.yaml': 'baz1 contents',
+            'file:///home/my/dir/baz/baz2.yaml': 'baz2 contents',
+            'file:///home/my/dir/baz/baz3.yaml': 'baz3 contents',
+        }, files)
+        self.assertEqual({
+            'heat_template_version': '2013-05-23',
+            'resources': {
+                'resource1': {
+                    'type': 'type1',
+                    'properties': {
+                        'bar': {'get_file': 'http://localhost/bar.yaml'},
+                        'foo': {'get_file': 'file:///home/my/dir/foo.yaml'},
+                    },
+                },
+                'resource2': {
+                    'type': 'type1',
+                    'properties': {
+                        'baz': [
+                            {'get_file': 'file:///home/my/dir/baz/baz1.yaml'},
+                            {'get_file': 'file:///home/my/dir/baz/baz2.yaml'},
+                            {'get_file': 'file:///home/my/dir/baz/baz3.yaml'},
+                        ],
+                        'ignored_list': {'get_file': ['ignore', 'me']},
+                        'ignored_dict': {'get_file': {'ignore': 'me'}},
+                        'ignored_none': {'get_file': None},
+                    },
+                }
+            }
+        }, tmpl_parsed)
+
+        self.m.VerifyAll()
+
+
 class TestURLFunctions(testtools.TestCase):
 
     def setUp(self):
diff --git a/heatclient/v1/shell.py b/heatclient/v1/shell.py
index 91f1a97..11b22fa 100644
--- a/heatclient/v1/shell.py
+++ b/heatclient/v1/shell.py
@@ -24,7 +24,7 @@ import heatclient.exc as exc
 
 @utils.arg('-f', '--template-file', metavar='<FILE>',
            help='Path to the template.')
-@utils.arg('-e', '--environment-file', metavar='<FILE>',
+@utils.arg('-e', '--environment-file', metavar='<FILE or URL>',
            help='Path to the environment.')
 @utils.arg('-u', '--template-url', metavar='<URL>',
            help='URL of template.')
@@ -49,7 +49,7 @@ def do_create(hc, args):
 
 @utils.arg('-f', '--template-file', metavar='<FILE>',
            help='Path to the template.')
-@utils.arg('-e', '--environment-file', metavar='<FILE>',
+@utils.arg('-e', '--environment-file', metavar='<FILE or URL>',
            help='Path to the environment.')
 @utils.arg('-u', '--template-url', metavar='<URL>',
            help='URL of template.')
@@ -69,7 +69,12 @@ def do_create(hc, args):
            help='Name of the stack to create.')
 def do_stack_create(hc, args):
     '''Create the stack.'''
-    files, env = template_utils.process_environment_and_files(
+    tpl_files, template = template_utils.get_template_contents(
+        args.template_file,
+        args.template_url,
+        args.template_object,
+        hc.http_client.raw_request)
+    env_files, env = template_utils.process_environment_and_files(
         env_path=args.environment_file)
 
     fields = {
@@ -77,12 +82,8 @@ def do_stack_create(hc, args):
         'timeout_mins': args.create_timeout,
         'disable_rollback': not(args.enable_rollback),
         'parameters': utils.format_parameters(args.parameters),
-        'template': template_utils.get_template_contents(
-            args.template_file,
-            args.template_url,
-            args.template_object,
-            hc.http_client.raw_request),
-        'files': files,
+        'template': template,
+        'files': dict(tpl_files.items() + env_files.items()),
         'environment': env
     }
 
@@ -171,7 +172,7 @@ def do_stack_show(hc, args):
 
 @utils.arg('-f', '--template-file', metavar='<FILE>',
            help='Path to the template.')
-@utils.arg('-e', '--environment-file', metavar='<FILE>',
+@utils.arg('-e', '--environment-file', metavar='<FILE or URL>',
            help='Path to the environment.')
 @utils.arg('-u', '--template-url', metavar='<URL>',
            help='URL of template.')
@@ -191,7 +192,7 @@ def do_update(hc, args):
 
 @utils.arg('-f', '--template-file', metavar='<FILE>',
            help='Path to the template.')
-@utils.arg('-e', '--environment-file', metavar='<FILE>',
+@utils.arg('-e', '--environment-file', metavar='<FILE or URL>',
            help='Path to the environment.')
 @utils.arg('-u', '--template-url', metavar='<URL>',
            help='URL of template.')
@@ -206,18 +207,21 @@ def do_update(hc, args):
            help='Name or ID of stack to update.')
 def do_stack_update(hc, args):
     '''Update the stack.'''
-    files, env = template_utils.process_environment_and_files(
+
+    tpl_files, template = template_utils.get_template_contents(
+        args.template_file,
+        args.template_url,
+        args.template_object,
+        hc.http_client.raw_request)
+
+    env_files, env = template_utils.process_environment_and_files(
         env_path=args.environment_file)
 
     fields = {
         'stack_id': args.id,
         'parameters': utils.format_parameters(args.parameters),
-        'template': template_utils.get_template_contents(
-            args.template_file,
-            args.template_url,
-            args.template_object,
-            hc.http_client.raw_request),
-        'files': files,
+        'template': template,
+        'files': dict(tpl_files.items() + env_files.items()),
         'environment': env
     }
 
@@ -285,7 +289,7 @@ def do_template_show(hc, args):
            help='URL of template.')
 @utils.arg('-f', '--template-file', metavar='<FILE>',
            help='Path to the template.')
-@utils.arg('-e', '--environment-file', metavar='<FILE>',
+@utils.arg('-e', '--environment-file', metavar='<FILE or URL>',
            help='Path to the environment.')
 @utils.arg('-o', '--template-object', metavar='<URL>',
            help='URL to retrieve template object (e.g from swift)')
@@ -303,7 +307,7 @@ def do_validate(hc, args):
            help='URL of template.')
 @utils.arg('-f', '--template-file', metavar='<FILE>',
            help='Path to the template.')
-@utils.arg('-e', '--environment-file', metavar='<FILE>',
+@utils.arg('-e', '--environment-file', metavar='<FILE or URL>',
            help='Path to the environment.')
 @utils.arg('-o', '--template-object', metavar='<URL>',
            help='URL to retrieve template object (e.g from swift)')
@@ -314,16 +318,19 @@ def do_validate(hc, args):
            action='append')
 def do_template_validate(hc, args):
     '''Validate a template with parameters.'''
-    files, env = template_utils.process_environment_and_files(
+
+    tpl_files, template = template_utils.get_template_contents(
+        args.template_file,
+        args.template_url,
+        args.template_object,
+        hc.http_client.raw_request)
+
+    env_files, env = template_utils.process_environment_and_files(
         env_path=args.environment_file)
     fields = {
         'parameters': utils.format_parameters(args.parameters),
-        'template': template_utils.get_template_contents(
-            args.template_file,
-            args.template_url,
-            args.template_object,
-            hc.http_client.raw_request),
-        'files': files,
+        'template': template,
+        'files': dict(tpl_files.items() + env_files.items()),
         'environment': env
     }