update oslo

bring in latest oslo code

Change-Id: Id05edcd9c23370fbba7610fb9de0ac7977bae717

4 files changed, 137 insertions, 40 deletions

diff --git a/pycadf/openstack/common/config/generator.py b/pycadf/openstack/common/config/generator.py
index 01f7608..83fc666 100644
--- a/pycadf/openstack/common/config/generator.py
+++ b/pycadf/openstack/common/config/generator.py
@@ -78,6 +78,13 @@ def generate(srcfiles):
     # The options list is a list of (module, options) tuples
     opts_by_group = {'DEFAULT': []}
 
+    for module_name in os.getenv(
+            "OSLO_CONFIG_GENERATOR_EXTRA_MODULES", "").split(','):
+        module = _import_module(module_name)
+        if module:
+            for group, opts in _list_opts(module):
+                opts_by_group.setdefault(group, []).append((module_name, opts))
+
     for pkg_name in pkg_names:
         mods = mods_by_pkg.get(pkg_name)
         mods.sort()
@@ -87,7 +94,7 @@ def generate(srcfiles):
 
             mod_obj = _import_module(mod_str)
             if not mod_obj:
-                continue
+                raise RuntimeError("Unable to import module %s" % mod_str)
 
             for group, opts in _list_opts(mod_obj):
                 opts_by_group.setdefault(group, []).append((mod_str, opts))
@@ -214,6 +221,14 @@ def _print_opt(opt):
         sys.exit(1)
     opt_help += ' (' + OPT_TYPES[opt_type] + ')'
     print('#', "\n# ".join(textwrap.wrap(opt_help, WORDWRAP_WIDTH)))
+    if opt.deprecated_opts:
+        for deprecated_opt in opt.deprecated_opts:
+            if deprecated_opt.name:
+                deprecated_group = (deprecated_opt.group if
+                                    deprecated_opt.group else "DEFAULT")
+                print('# Deprecated group/name - [%s]/%s' %
+                      (deprecated_group,
+                       deprecated_opt.name))
     try:
         if opt_default is None:
             print('#%s=<None>' % opt_name)
@@ -247,9 +262,6 @@ def _print_opt(opt):
 
 
 def main():
-    if len(sys.argv) < 2:
-        print("usage: %s [srcfile]...\n" % sys.argv[0])
-        sys.exit(0)
     generate(sys.argv[1:])
 
 if __name__ == '__main__':
diff --git a/pycadf/openstack/common/gettextutils.py b/pycadf/openstack/common/gettextutils.py
index 9f5524b..f964fc0 100644
--- a/pycadf/openstack/common/gettextutils.py
+++ b/pycadf/openstack/common/gettextutils.py
@@ -26,10 +26,13 @@ Usual usage in an openstack.common module:
 
 import copy
 import gettext
-import logging.handlers
+import logging
 import os
 import re
-import UserString
+try:
+    import UserString as _userString
+except ImportError:
+    import collections as _userString
 
 from babel import localedata
 import six
@@ -37,11 +40,29 @@ import six
 _localedir = os.environ.get('pycadf'.upper() + '_LOCALEDIR')
 _t = gettext.translation('pycadf', localedir=_localedir, fallback=True)
 
-_AVAILABLE_LANGUAGES = []
+_AVAILABLE_LANGUAGES = {}
+USE_LAZY = False
+
+
+def enable_lazy():
+    """Convenience function for configuring _() to use lazy gettext
+
+    Call this at the start of execution to enable the gettextutils._
+    function to use lazy gettext functionality. This is useful if
+    your project is importing _ directly instead of using the
+    gettextutils.install() way of importing the _ function.
+    """
+    global USE_LAZY
+    USE_LAZY = True
 
 
 def _(msg):
-    return _t.ugettext(msg)
+    if USE_LAZY:
+        return Message(msg, 'pycadf')
+    else:
+        if six.PY3:
+            return _t.gettext(msg)
+        return _t.ugettext(msg)
 
 
 def install(domain, lazy=False):
@@ -86,24 +107,28 @@ def install(domain, lazy=False):
             """
             return Message(msg, domain)
 
-        import __builtin__
-        __builtin__.__dict__['_'] = _lazy_gettext
+        from six import moves
+        moves.builtins.__dict__['_'] = _lazy_gettext
     else:
         localedir = '%s_LOCALEDIR' % domain.upper()
-        gettext.install(domain,
-                        localedir=os.environ.get(localedir),
-                        unicode=True)
+        if six.PY3:
+            gettext.install(domain,
+                            localedir=os.environ.get(localedir))
+        else:
+            gettext.install(domain,
+                            localedir=os.environ.get(localedir),
+                            unicode=True)
 
 
-class Message(UserString.UserString, object):
+class Message(_userString.UserString, object):
     """Class used to encapsulate translatable messages."""
     def __init__(self, msg, domain):
         # _msg is the gettext msgid and should never change
         self._msg = msg
         self._left_extra_msg = ''
         self._right_extra_msg = ''
+        self._locale = None
         self.params = None
-        self.locale = None
         self.domain = domain
 
     @property
@@ -123,8 +148,13 @@ class Message(UserString.UserString, object):
                                        localedir=localedir,
                                        fallback=True)
 
+        if six.PY3:
+            ugettext = lang.gettext
+        else:
+            ugettext = lang.ugettext
+
         full_msg = (self._left_extra_msg +
-                    lang.ugettext(self._msg) +
+                    ugettext(self._msg) +
                     self._right_extra_msg)
 
         if self.params is not None:
@@ -132,6 +162,33 @@ class Message(UserString.UserString, object):
 
         return six.text_type(full_msg)
 
+    @property
+    def locale(self):
+        return self._locale
+
+    @locale.setter
+    def locale(self, value):
+        self._locale = value
+        if not self.params:
+            return
+
+        # This Message object may have been constructed with one or more
+        # Message objects as substitution parameters, given as a single
+        # Message, or a tuple or Map containing some, so when setting the
+        # locale for this Message we need to set it for those Messages too.
+        if isinstance(self.params, Message):
+            self.params.locale = value
+            return
+        if isinstance(self.params, tuple):
+            for param in self.params:
+                if isinstance(param, Message):
+                    param.locale = value
+            return
+        if isinstance(self.params, dict):
+            for param in self.params.values():
+                if isinstance(param, Message):
+                    param.locale = value
+
     def _save_dictionary_parameter(self, dict_param):
         full_msg = self.data
         # look for %(blah) fields in string;
@@ -150,7 +207,7 @@ class Message(UserString.UserString, object):
                     params[key] = copy.deepcopy(dict_param[key])
                 except TypeError:
                     # cast uncopyable thing to unicode string
-                    params[key] = unicode(dict_param[key])
+                    params[key] = six.text_type(dict_param[key])
 
         return params
 
@@ -169,7 +226,7 @@ class Message(UserString.UserString, object):
             try:
                 self.params = copy.deepcopy(other)
             except TypeError:
-                self.params = unicode(other)
+                self.params = six.text_type(other)
 
         return self
 
@@ -178,11 +235,13 @@ class Message(UserString.UserString, object):
         return self.data
 
     def __str__(self):
+        if six.PY3:
+            return self.__unicode__()
         return self.data.encode('utf-8')
 
     def __getstate__(self):
         to_copy = ['_msg', '_right_extra_msg', '_left_extra_msg',
-                   'domain', 'params', 'locale']
+                   'domain', 'params', '_locale']
         new_dict = self.__dict__.fromkeys(to_copy)
         for attr in to_copy:
             new_dict[attr] = copy.deepcopy(self.__dict__[attr])
@@ -236,7 +295,7 @@ class Message(UserString.UserString, object):
         if name in ops:
             return getattr(self.data, name)
         else:
-            return UserString.UserString.__getattribute__(self, name)
+            return _userString.UserString.__getattribute__(self, name)
 
 
 def get_available_languages(domain):
@@ -244,8 +303,8 @@ def get_available_languages(domain):
 
     :param domain: the domain to get languages for
     """
-    if _AVAILABLE_LANGUAGES:
-        return _AVAILABLE_LANGUAGES
+    if domain in _AVAILABLE_LANGUAGES:
+        return copy.copy(_AVAILABLE_LANGUAGES[domain])
 
     localedir = '%s_LOCALEDIR' % domain.upper()
     find = lambda x: gettext.find(domain,
@@ -254,7 +313,7 @@ def get_available_languages(domain):
 
     # NOTE(mrodden): en_US should always be available (and first in case
     # order matters) since our in-line message strings are en_US
-    _AVAILABLE_LANGUAGES.append('en_US')
+    language_list = ['en_US']
     # NOTE(luisg): Babel <1.0 used a function called list(), which was
     # renamed to locale_identifiers() in >=1.0, the requirements master list
     # requires >=0.9.6, uncapped, so defensively work with both. We can remove
@@ -264,18 +323,27 @@ def get_available_languages(domain):
     locale_identifiers = list_identifiers()
     for i in locale_identifiers:
         if find(i) is not None:
-            _AVAILABLE_LANGUAGES.append(i)
-    return _AVAILABLE_LANGUAGES
+            language_list.append(i)
+    _AVAILABLE_LANGUAGES[domain] = language_list
+    return copy.copy(language_list)
 
 
 def get_localized_message(message, user_locale):
-    """Gets a localized version of the given message in the given locale."""
-    if (isinstance(message, Message)):
-        if user_locale:
-            message.locale = user_locale
-        return unicode(message)
-    else:
-        return message
+    """Gets a localized version of the given message in the given locale.
+
+    If the message is not a Message object the message is returned as-is.
+    If the locale is None the message is translated to the default locale.
+
+    :returns: the translated message in unicode, or the original message if
+              it could not be translated
+    """
+    translated = message
+    if isinstance(message, Message):
+        original_locale = message.locale
+        message.locale = user_locale
+        translated = six.text_type(message)
+        message.locale = original_locale
+    return translated
 
 
 class LocaleHandler(logging.Handler):
diff --git a/pycadf/openstack/common/jsonutils.py b/pycadf/openstack/common/jsonutils.py
index 370c95a..8ac4efe 100644
--- a/pycadf/openstack/common/jsonutils.py
+++ b/pycadf/openstack/common/jsonutils.py
@@ -38,14 +38,19 @@ import functools
 import inspect
 import itertools
 import json
-import types
-import xmlrpclib
+try:
+    import xmlrpclib
+except ImportError:
+    # NOTE(jd): xmlrpclib is not shipped with Python 3
+    xmlrpclib = None
 
-import netaddr
 import six
 
+from pycadf.openstack.common import gettextutils
+from pycadf.openstack.common import importutils
 from pycadf.openstack.common import timeutils
 
+netaddr = importutils.try_import("netaddr")
 
 _nasty_type_tests = [inspect.ismodule, inspect.isclass, inspect.ismethod,
                      inspect.isfunction, inspect.isgeneratorfunction,
@@ -53,7 +58,8 @@ _nasty_type_tests = [inspect.ismodule, inspect.isclass, inspect.ismethod,
                      inspect.iscode, inspect.isbuiltin, inspect.isroutine,
                      inspect.isabstract]
 
-_simple_types = (types.NoneType, int, basestring, bool, float, long)
+_simple_types = (six.string_types + six.integer_types
+                 + (type(None), bool, float))
 
 
 def to_primitive(value, convert_instances=False, convert_datetime=True,
@@ -125,11 +131,13 @@ def to_primitive(value, convert_instances=False, convert_datetime=True,
         # It's not clear why xmlrpclib created their own DateTime type, but
         # for our purposes, make it a datetime type which is explicitly
         # handled
-        if isinstance(value, xmlrpclib.DateTime):
+        if xmlrpclib and isinstance(value, xmlrpclib.DateTime):
             value = datetime.datetime(*tuple(value.timetuple())[:6])
 
         if convert_datetime and isinstance(value, datetime.datetime):
             return timeutils.strtime(value)
+        elif isinstance(value, gettextutils.Message):
+            return value.data
         elif hasattr(value, 'iteritems'):
             return recursive(dict(value.iteritems()), level=level + 1)
         elif hasattr(value, '__iter__'):
@@ -138,7 +146,7 @@ def to_primitive(value, convert_instances=False, convert_datetime=True,
             # Likely an instance of something. Watch for cycles.
             # Ignore class member vars.
             return recursive(value.__dict__, level=level + 1)
-        elif isinstance(value, netaddr.IPAddress):
+        elif netaddr and isinstance(value, netaddr.IPAddress):
             return six.text_type(value)
         else:
             if any(test(value) for test in _nasty_type_tests):
diff --git a/pycadf/openstack/common/timeutils.py b/pycadf/openstack/common/timeutils.py
index aa9f708..98d877d 100644
--- a/pycadf/openstack/common/timeutils.py
+++ b/pycadf/openstack/common/timeutils.py
@@ -21,6 +21,7 @@ Time related utilities and helper functions.
 
 import calendar
 import datetime
+import time
 
 import iso8601
 import six
@@ -90,6 +91,11 @@ def is_newer_than(after, seconds):
 
 def utcnow_ts():
     """Timestamp version of our utcnow function."""
+    if utcnow.override_time is None:
+        # NOTE(kgriffs): This is several times faster
+        # than going through calendar.timegm(...)
+        return int(time.time())
+
     return calendar.timegm(utcnow().timetuple())
 
 
@@ -111,12 +117,15 @@ def iso8601_from_timestamp(timestamp):
 utcnow.override_time = None
 
 
-def set_time_override(override_time=datetime.datetime.utcnow()):
+def set_time_override(override_time=None):
     """Overrides utils.utcnow.
 
     Make it return a constant time or a list thereof, one at a time.
+
+    :param override_time: datetime instance or list thereof. If not
+                          given, defaults to the current UTC time.
     """
-    utcnow.override_time = override_time
+    utcnow.override_time = override_time or datetime.datetime.utcnow()
 
 
 def advance_time_delta(timedelta):