diff options
| -rw-r--r-- | heatclient/common/http.py | 10 | ||||
| -rw-r--r-- | heatclient/common/utils.py | 6 | ||||
| -rw-r--r-- | heatclient/exc.py | 9 | ||||
| -rw-r--r-- | heatclient/openstack/common/importutils.py | 19 | ||||
| -rw-r--r-- | heatclient/openstack/common/jsonutils.py | 182 | ||||
| -rw-r--r-- | heatclient/openstack/common/timeutils.py | 204 | ||||
| -rw-r--r-- | heatclient/tests/fakes.py | 9 | ||||
| -rw-r--r-- | heatclient/tests/test_shell.py | 23 | ||||
| -rw-r--r-- | heatclient/v1/shell.py | 14 |
9 files changed, 429 insertions, 47 deletions
diff --git a/heatclient/common/http.py b/heatclient/common/http.py index 6e801bd..82ecd34 100644 --- a/heatclient/common/http.py +++ b/heatclient/common/http.py @@ -19,6 +19,7 @@ import os import posixpath import socket +from heatclient.openstack.common import jsonutils from heatclient.openstack.common.py3kcompat import urlutils from six.moves import http_client as httplib @@ -28,11 +29,6 @@ except ImportError: #TODO(bcwaldon): Handle this failure more gracefully pass -try: - import json -except ImportError: - import simplejson as json - from heatclient import exc @@ -205,14 +201,14 @@ class HTTPClient(object): kwargs['headers'].setdefault('Accept', 'application/json') if 'body' in kwargs: - kwargs['body'] = json.dumps(kwargs['body']) + kwargs['body'] = jsonutils.dumps(kwargs['body']) resp, body_str = self._http_request(url, method, **kwargs) if 'application/json' in resp.getheader('content-type', None): body = body_str try: - body = json.loads(body) + body = jsonutils.loads(body) except ValueError: LOG.error('Could not decode response body as JSON') else: diff --git a/heatclient/common/utils.py b/heatclient/common/utils.py index 532ad17..f9c6619 100644 --- a/heatclient/common/utils.py +++ b/heatclient/common/utils.py @@ -14,7 +14,6 @@ # under the License. from __future__ import print_function -import json import os import prettytable import sys @@ -24,9 +23,10 @@ import yaml from heatclient import exc from heatclient.openstack.common import importutils +from heatclient.openstack.common import jsonutils supported_formats = { - "json": lambda x: json.dumps(x, indent=2), + "json": lambda x: jsonutils.dumps(x, indent=2), "yaml": yaml.safe_dump } @@ -46,7 +46,7 @@ def link_formatter(links): def json_formatter(js): - return json.dumps(js, indent=2) + return jsonutils.dumps(js, indent=2) def text_wrap_formatter(d): diff --git a/heatclient/exc.py b/heatclient/exc.py index 60be0fd..c3d4117 100644 --- a/heatclient/exc.py +++ b/heatclient/exc.py @@ -12,12 +12,9 @@ import sys -verbose = 0 +from heatclient.openstack.common import jsonutils -try: - import json -except ImportError: - import simplejson as json +verbose = 0 class BaseException(Exception): @@ -48,7 +45,7 @@ class HTTPException(BaseException): def __init__(self, message=None): super(HTTPException, self).__init__(message) try: - self.error = json.loads(message) + self.error = jsonutils.loads(message) if 'error' not in self.error: raise KeyError('Key "error" not exists') except KeyError: diff --git a/heatclient/openstack/common/importutils.py b/heatclient/openstack/common/importutils.py index 2a28b45..4fd9ae2 100644 --- a/heatclient/openstack/common/importutils.py +++ b/heatclient/openstack/common/importutils.py @@ -1,6 +1,4 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 OpenStack LLC. +# Copyright 2011 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -24,7 +22,7 @@ import traceback def import_class(import_str): - """Returns a class from a string including module and class""" + """Returns a class from a string including module and class.""" mod_str, _sep, class_str = import_str.rpartition('.') try: __import__(mod_str) @@ -41,8 +39,9 @@ def import_object(import_str, *args, **kwargs): def import_object_ns(name_space, import_str, *args, **kwargs): - """ - Import a class and return an instance of it, first by trying + """Tries to import object from default namespace. + + Imports a class and return an instance of it, first by trying to find the class in a default namespace, then failing back to a full path if not found in the default namespace. """ @@ -57,3 +56,11 @@ def import_module(import_str): """Import a module.""" __import__(import_str) return sys.modules[import_str] + + +def try_import(import_str, default=None): + """Try to import a module and if it fails return default.""" + try: + return import_module(import_str) + except ImportError: + return default diff --git a/heatclient/openstack/common/jsonutils.py b/heatclient/openstack/common/jsonutils.py new file mode 100644 index 0000000..18915c1 --- /dev/null +++ b/heatclient/openstack/common/jsonutils.py @@ -0,0 +1,182 @@ +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2011 Justin Santa Barbara +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +''' +JSON related utilities. + +This module provides a few things: + + 1) A handy function for getting an object down to something that can be + JSON serialized. See to_primitive(). + + 2) Wrappers around loads() and dumps(). The dumps() wrapper will + automatically use to_primitive() for you if needed. + + 3) This sets up anyjson to use the loads() and dumps() wrappers if anyjson + is available. +''' + + +import datetime +import functools +import inspect +import itertools +import json +try: + import xmlrpclib +except ImportError: + # NOTE(jaypipes): xmlrpclib was renamed to xmlrpc.client in Python3 + # however the function and object call signatures + # remained the same. This whole try/except block should + # be removed and replaced with a call to six.moves once + # six 1.4.2 is released. See http://bit.ly/1bqrVzu + import xmlrpc.client as xmlrpclib + +import six + +from heatclient.openstack.common import gettextutils +from heatclient.openstack.common import importutils +from heatclient.openstack.common import timeutils + +netaddr = importutils.try_import("netaddr") + +_nasty_type_tests = [inspect.ismodule, inspect.isclass, inspect.ismethod, + inspect.isfunction, inspect.isgeneratorfunction, + inspect.isgenerator, inspect.istraceback, inspect.isframe, + inspect.iscode, inspect.isbuiltin, inspect.isroutine, + inspect.isabstract] + +_simple_types = (six.string_types + six.integer_types + + (type(None), bool, float)) + + +def to_primitive(value, convert_instances=False, convert_datetime=True, + level=0, max_depth=3): + """Convert a complex object into primitives. + + Handy for JSON serialization. We can optionally handle instances, + but since this is a recursive function, we could have cyclical + data structures. + + To handle cyclical data structures we could track the actual objects + visited in a set, but not all objects are hashable. Instead we just + track the depth of the object inspections and don't go too deep. + + Therefore, convert_instances=True is lossy ... be aware. + + """ + # handle obvious types first - order of basic types determined by running + # full tests on nova project, resulting in the following counts: + # 572754 <type 'NoneType'> + # 460353 <type 'int'> + # 379632 <type 'unicode'> + # 274610 <type 'str'> + # 199918 <type 'dict'> + # 114200 <type 'datetime.datetime'> + # 51817 <type 'bool'> + # 26164 <type 'list'> + # 6491 <type 'float'> + # 283 <type 'tuple'> + # 19 <type 'long'> + if isinstance(value, _simple_types): + return value + + if isinstance(value, datetime.datetime): + if convert_datetime: + return timeutils.strtime(value) + else: + return value + + # value of itertools.count doesn't get caught by nasty_type_tests + # and results in infinite loop when list(value) is called. + if type(value) == itertools.count: + return six.text_type(value) + + # FIXME(vish): Workaround for LP bug 852095. Without this workaround, + # tests that raise an exception in a mocked method that + # has a @wrap_exception with a notifier will fail. If + # we up the dependency to 0.5.4 (when it is released) we + # can remove this workaround. + if getattr(value, '__module__', None) == 'mox': + return 'mock' + + if level > max_depth: + return '?' + + # The try block may not be necessary after the class check above, + # but just in case ... + try: + recursive = functools.partial(to_primitive, + convert_instances=convert_instances, + convert_datetime=convert_datetime, + level=level, + max_depth=max_depth) + if isinstance(value, dict): + return dict((k, recursive(v)) for k, v in six.iteritems(value)) + elif isinstance(value, (list, tuple)): + return [recursive(lv) for lv in value] + + # It's not clear why xmlrpclib created their own DateTime type, but + # for our purposes, make it a datetime type which is explicitly + # handled + if isinstance(value, xmlrpclib.DateTime): + value = datetime.datetime(*tuple(value.timetuple())[:6]) + + if convert_datetime and isinstance(value, datetime.datetime): + return timeutils.strtime(value) + elif isinstance(value, gettextutils.Message): + return value.data + elif hasattr(value, 'iteritems'): + return recursive(dict(value.iteritems()), level=level + 1) + elif hasattr(value, '__iter__'): + return recursive(list(value)) + elif convert_instances and hasattr(value, '__dict__'): + # Likely an instance of something. Watch for cycles. + # Ignore class member vars. + return recursive(value.__dict__, level=level + 1) + elif netaddr and isinstance(value, netaddr.IPAddress): + return six.text_type(value) + else: + if any(test(value) for test in _nasty_type_tests): + return six.text_type(value) + return value + except TypeError: + # Class objects are tricky since they may define something like + # __iter__ defined but it isn't callable as list(). + return six.text_type(value) + + +def dumps(value, default=to_primitive, **kwargs): + return json.dumps(value, default=default, **kwargs) + + +def loads(s): + return json.loads(s) + + +def load(s): + return json.load(s) + + +try: + import anyjson +except ImportError: + pass +else: + anyjson._modules.append((__name__, 'dumps', TypeError, + 'loads', ValueError, 'load')) + anyjson.force_implementation(__name__) diff --git a/heatclient/openstack/common/timeutils.py b/heatclient/openstack/common/timeutils.py new file mode 100644 index 0000000..c8b0b15 --- /dev/null +++ b/heatclient/openstack/common/timeutils.py @@ -0,0 +1,204 @@ +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Time related utilities and helper functions. +""" + +import calendar +import datetime +import time + +import iso8601 +import six + + +# ISO 8601 extended time format with microseconds +_ISO8601_TIME_FORMAT_SUBSECOND = '%Y-%m-%dT%H:%M:%S.%f' +_ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S' +PERFECT_TIME_FORMAT = _ISO8601_TIME_FORMAT_SUBSECOND + + +def isotime(at=None, subsecond=False): + """Stringify time in ISO 8601 format.""" + if not at: + at = utcnow() + st = at.strftime(_ISO8601_TIME_FORMAT + if not subsecond + else _ISO8601_TIME_FORMAT_SUBSECOND) + tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC' + st += ('Z' if tz == 'UTC' else tz) + return st + + +def parse_isotime(timestr): + """Parse time from ISO 8601 format.""" + try: + return iso8601.parse_date(timestr) + except iso8601.ParseError as e: + raise ValueError(six.text_type(e)) + except TypeError as e: + raise ValueError(six.text_type(e)) + + +def strtime(at=None, fmt=PERFECT_TIME_FORMAT): + """Returns formatted utcnow.""" + if not at: + at = utcnow() + return at.strftime(fmt) + + +def parse_strtime(timestr, fmt=PERFECT_TIME_FORMAT): + """Turn a formatted time back into a datetime.""" + return datetime.datetime.strptime(timestr, fmt) + + +def normalize_time(timestamp): + """Normalize time in arbitrary timezone to UTC naive object.""" + offset = timestamp.utcoffset() + if offset is None: + return timestamp + return timestamp.replace(tzinfo=None) - offset + + +def is_older_than(before, seconds): + """Return True if before is older than seconds.""" + if isinstance(before, six.string_types): + before = parse_strtime(before).replace(tzinfo=None) + return utcnow() - before > datetime.timedelta(seconds=seconds) + + +def is_newer_than(after, seconds): + """Return True if after is newer than seconds.""" + if isinstance(after, six.string_types): + after = parse_strtime(after).replace(tzinfo=None) + return after - utcnow() > datetime.timedelta(seconds=seconds) + + +def utcnow_ts(): + """Timestamp version of our utcnow function.""" + if utcnow.override_time is None: + # NOTE(kgriffs): This is several times faster + # than going through calendar.timegm(...) + return int(time.time()) + + return calendar.timegm(utcnow().timetuple()) + + +def utcnow(): + """Overridable version of utils.utcnow.""" + if utcnow.override_time: + try: + return utcnow.override_time.pop(0) + except AttributeError: + return utcnow.override_time + return datetime.datetime.utcnow() + + +def iso8601_from_timestamp(timestamp): + """Returns a iso8601 formated date from timestamp.""" + return isotime(datetime.datetime.utcfromtimestamp(timestamp)) + + +utcnow.override_time = None + + +def set_time_override(override_time=None): + """Overrides utils.utcnow. + + Make it return a constant time or a list thereof, one at a time. + + :param override_time: datetime instance or list thereof. If not + given, defaults to the current UTC time. + """ + utcnow.override_time = override_time or datetime.datetime.utcnow() + + +def advance_time_delta(timedelta): + """Advance overridden time using a datetime.timedelta.""" + assert(not utcnow.override_time is None) + try: + for dt in utcnow.override_time: + dt += timedelta + except TypeError: + utcnow.override_time += timedelta + + +def advance_time_seconds(seconds): + """Advance overridden time by seconds.""" + advance_time_delta(datetime.timedelta(0, seconds)) + + +def clear_time_override(): + """Remove the overridden time.""" + utcnow.override_time = None + + +def marshall_now(now=None): + """Make an rpc-safe datetime with microseconds. + + Note: tzinfo is stripped, but not required for relative times. + """ + if not now: + now = utcnow() + return dict(day=now.day, month=now.month, year=now.year, hour=now.hour, + minute=now.minute, second=now.second, + microsecond=now.microsecond) + + +def unmarshall_time(tyme): + """Unmarshall a datetime dict.""" + return datetime.datetime(day=tyme['day'], + month=tyme['month'], + year=tyme['year'], + hour=tyme['hour'], + minute=tyme['minute'], + second=tyme['second'], + microsecond=tyme['microsecond']) + + +def delta_seconds(before, after): + """Return the difference between two timing objects. + + Compute the difference in seconds between two date, time, or + datetime objects (as a float, to microsecond resolution). + """ + delta = after - before + return total_seconds(delta) + + +def total_seconds(delta): + """Return the total seconds of datetime.timedelta object. + + Compute total seconds of datetime.timedelta, datetime.timedelta + doesn't have method total_seconds in Python2.6, calculate it manually. + """ + try: + return delta.total_seconds() + except AttributeError: + return ((delta.days * 24 * 3600) + delta.seconds + + float(delta.microseconds) / (10 ** 6)) + + +def is_soon(dt, window): + """Determines if time is going to happen in the next window seconds. + + :params dt: the time + :params window: minimum seconds to remain to consider the time not soon + + :return: True if expiration is within the given duration + """ + soon = (utcnow() + datetime.timedelta(seconds=window)) + return normalize_time(dt) <= soon diff --git a/heatclient/tests/fakes.py b/heatclient/tests/fakes.py index 077a76c..4b5fd11 100644 --- a/heatclient/tests/fakes.py +++ b/heatclient/tests/fakes.py @@ -11,10 +11,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json - from heatclient.common import http from heatclient import exc +from heatclient.openstack.common import jsonutils from keystoneclient.v2_0 import client as ksclient @@ -49,7 +48,7 @@ def script_heat_list(): resp = FakeHTTPResponse(200, 'success, you', {'content-type': 'application/json'}, - json.dumps(resp_dict)) + jsonutils.dumps(resp_dict)) http.HTTPClient.json_request('GET', '/stacks?').AndReturn( (resp, resp_dict)) @@ -68,9 +67,9 @@ def script_heat_normal_error(): resp = FakeHTTPResponse(400, 'The resource could not be found', {'content-type': 'application/json'}, - json.dumps(resp_dict)) + jsonutils.dumps(resp_dict)) http.HTTPClient.json_request('GET', '/stacks/bad').AndRaise( - exc.from_response(resp, json.dumps(resp_dict))) + exc.from_response(resp, jsonutils.dumps(resp_dict))) def script_heat_error(resp_string): diff --git a/heatclient/tests/test_shell.py b/heatclient/tests/test_shell.py index edfffc0..2401f0c 100644 --- a/heatclient/tests/test_shell.py +++ b/heatclient/tests/test_shell.py @@ -23,14 +23,11 @@ import tempfile import testscenarios import testtools +from heatclient.openstack.common import jsonutils from heatclient.openstack.common.py3kcompat import urlutils from heatclient.openstack.common import strutils from mox3 import mox -try: - import json -except ImportError: - import simplejson as json from keystoneclient.v2_0 import client as ksclient from heatclient.common import http @@ -356,7 +353,7 @@ class ShellTestUserPass(ShellBase): } self._script_keystone_client() - fakes.script_heat_error(json.dumps(resp_dict)) + fakes.script_heat_error(jsonutils.dumps(resp_dict)) self.m.ReplayAll() @@ -379,7 +376,7 @@ class ShellTestUserPass(ShellBase): } self._script_keystone_client() - fakes.script_heat_error(json.dumps(resp_dict)) + fakes.script_heat_error(jsonutils.dumps(resp_dict)) self.m.ReplayAll() @@ -414,7 +411,7 @@ class ShellTestUserPass(ShellBase): } self._script_keystone_client() - fakes.script_heat_error(json.dumps(missing_message)) + fakes.script_heat_error(jsonutils.dumps(missing_message)) self.m.ReplayAll() try: @@ -435,7 +432,7 @@ class ShellTestUserPass(ShellBase): } self._script_keystone_client() - fakes.script_heat_error(json.dumps(resp_dict)) + fakes.script_heat_error(jsonutils.dumps(resp_dict)) self.m.ReplayAll() exc.verbose = 1 @@ -458,7 +455,7 @@ class ShellTestUserPass(ShellBase): 200, 'OK', {'content-type': 'application/json'}, - json.dumps(resp_dict)) + jsonutils.dumps(resp_dict)) http.HTTPClient.json_request( 'GET', '/stacks/teststack/1').AndReturn((resp, resp_dict)) @@ -487,7 +484,7 @@ class ShellTestUserPass(ShellBase): 'OK', {'content-type': 'application/json'}, template_data) - resp_dict = json.loads(template_data) + resp_dict = jsonutils.loads(template_data) http.HTTPClient.json_request( 'GET', '/stacks/teststack/template').AndReturn((resp, resp_dict)) @@ -515,7 +512,7 @@ class ShellTestUserPass(ShellBase): 200, 'OK', {'content-type': 'application/json'}, - json.dumps(resp_dict)) + jsonutils.dumps(resp_dict)) http.HTTPClient.json_request( 'GET', '/stacks/teststack/template').AndReturn((resp, resp_dict)) @@ -755,7 +752,7 @@ class ShellTestEvents(ShellBase): 200, 'OK', {'content-type': 'application/json'}, - json.dumps(resp_dict)) + jsonutils.dumps(resp_dict)) stack_id = 'teststack/1' resource_name = 'testresource/1' http.HTTPClient.json_request( @@ -811,7 +808,7 @@ class ShellTestEvents(ShellBase): 200, 'OK', {'content-type': 'application/json'}, - json.dumps(resp_dict)) + jsonutils.dumps(resp_dict)) stack_id = 'teststack/1' resource_name = 'testresource/1' http.HTTPClient.json_request( diff --git a/heatclient/v1/shell.py b/heatclient/v1/shell.py index 5a76950..9111bea 100644 --- a/heatclient/v1/shell.py +++ b/heatclient/v1/shell.py @@ -13,12 +13,12 @@ # License for the specific language governing permissions and limitations # under the License. -import json import os import urllib import yaml from heatclient.common import utils +from heatclient.openstack.common import jsonutils from heatclient.openstack.common.py3kcompat import urlutils import heatclient.exc as exc @@ -28,7 +28,7 @@ def _set_template_fields(hc, args, fields): if args.template_file: tpl = open(args.template_file).read() if tpl.startswith('{'): - fields['template'] = json.loads(tpl) + fields['template'] = jsonutils.loads(tpl) else: fields['template'] = tpl elif args.template_url: @@ -36,7 +36,7 @@ def _set_template_fields(hc, args, fields): elif args.template_object: template_body = hc.http_client.raw_request('GET', args.template_object) if template_body: - fields['template'] = json.loads(template_body) + fields['template'] = jsonutils.loads(template_body) else: raise exc.CommandError('Could not fetch template from %s' % args.template_object) @@ -310,7 +310,7 @@ def do_resource_type_show(hc, args={}): raise exc.CommandError( 'Resource Type not found: %s' % args.resource_type) else: - print(json.dumps(resource_type, indent=2)) + print(jsonutils.dumps(resource_type, indent=2)) @utils.arg('id', metavar='<NAME or ID>', @@ -333,7 +333,7 @@ def do_template_show(hc, args): if 'heat_template_version' in template: print yaml.safe_dump(template, indent=2) else: - print json.dumps(template, indent=2) + print jsonutils.dumps(template, indent=2) @utils.arg('-u', '--template-url', metavar='<URL>', @@ -374,7 +374,7 @@ def do_template_validate(hc, args): _process_environment_and_files(args, fields) validation = hc.stacks.validate(**fields) - print json.dumps(validation, indent=2) + print jsonutils.dumps(validation, indent=2) @utils.arg('id', metavar='<NAME or ID>', @@ -459,7 +459,7 @@ def do_resource_metadata(hc, args): raise exc.CommandError('Stack or resource not found: %s %s' % (args.id, args.resource)) else: - print json.dumps(metadata, indent=2) + print jsonutils.dumps(metadata, indent=2) @utils.arg('id', metavar='<NAME or ID>', |