7 files changed, 185 insertions, 49 deletions

diff --git a/lib/sqlalchemy/util/__init__.py b/lib/sqlalchemy/util/__init__.py
index dfed5b90a..d777d2e06 100644
--- a/lib/sqlalchemy/util/__init__.py
+++ b/lib/sqlalchemy/util/__init__.py
@@ -1,5 +1,5 @@
 # util/__init__.py
-# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors
+# Copyright (C) 2005-2015 the SQLAlchemy authors and contributors
 # <see AUTHORS file>
 #
 # This module is part of SQLAlchemy and is released under
@@ -36,7 +36,7 @@ from .langhelpers import iterate_attributes, class_hierarchy, \
     generic_repr, counter, PluginLoader, hybridproperty, hybridmethod, \
     safe_reraise,\
     get_callable_argspec, only_once, attrsetter, ellipses_string, \
-    warn_limited
+    warn_limited, map_bits, MemoizedSlots, EnsureKWArgType
 
 from .deprecations import warn_deprecated, warn_pending_deprecation, \
     deprecated, pending_deprecation, inject_docstring_text
diff --git a/lib/sqlalchemy/util/_collections.py b/lib/sqlalchemy/util/_collections.py
index a1fbc0fa0..4fb12d71b 100644
--- a/lib/sqlalchemy/util/_collections.py
+++ b/lib/sqlalchemy/util/_collections.py
@@ -1,5 +1,5 @@
 # util/_collections.py
-# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors
+# Copyright (C) 2005-2015 the SQLAlchemy authors and contributors
 # <see AUTHORS file>
 #
 # This module is part of SQLAlchemy and is released under
@@ -10,9 +10,10 @@
 from __future__ import absolute_import
 import weakref
 import operator
-from .compat import threading, itertools_filterfalse
+from .compat import threading, itertools_filterfalse, string_types
 from . import py2k
 import types
+import collections
 
 EMPTY_SET = frozenset()
 
@@ -126,20 +127,6 @@ class _LW(AbstractKeyedTuple):
         return d
 
 
-def lightweight_named_tuple(name, fields):
-
-    tp_cls = type(name, (_LW,), {})
-    for idx, field in enumerate(fields):
-        if field is None:
-            continue
-        setattr(tp_cls, field, property(operator.itemgetter(idx)))
-
-    tp_cls._real_fields = fields
-    tp_cls._fields = tuple([f for f in fields if f is not None])
-
-    return tp_cls
-
-
 class ImmutableContainer(object):
     def _immutable(self, *arg, **kw):
         raise TypeError("%s object is immutable" % self.__class__.__name__)
@@ -164,8 +151,13 @@ class immutabledict(ImmutableContainer, dict):
         return immutabledict, (dict(self), )
 
     def union(self, d):
-        if not self:
-            return immutabledict(d)
+        if not d:
+            return self
+        elif not self:
+            if isinstance(d, immutabledict):
+                return d
+            else:
+                return immutabledict(d)
         else:
             d2 = immutabledict(self)
             dict.update(d2, d)
@@ -178,8 +170,10 @@ class immutabledict(ImmutableContainer, dict):
 class Properties(object):
     """Provide a __getattr__/__setattr__ interface over a dict."""
 
+    __slots__ = '_data',
+
     def __init__(self, data):
-        self.__dict__['_data'] = data
+        object.__setattr__(self, '_data', data)
 
     def __len__(self):
         return len(self._data)
@@ -199,8 +193,8 @@ class Properties(object):
     def __delitem__(self, key):
         del self._data[key]
 
-    def __setattr__(self, key, object):
-        self._data[key] = object
+    def __setattr__(self, key, obj):
+        self._data[key] = obj
 
     def __getstate__(self):
         return {'_data': self.__dict__['_data']}
@@ -251,6 +245,8 @@ class OrderedProperties(Properties):
     """Provide a __getattr__/__setattr__ interface with an OrderedDict
     as backing store."""
 
+    __slots__ = ()
+
     def __init__(self):
         Properties.__init__(self, OrderedDict())
 
@@ -258,10 +254,17 @@ class OrderedProperties(Properties):
 class ImmutableProperties(ImmutableContainer, Properties):
     """Provide immutable dict/object attribute to an underlying dictionary."""
 
+    __slots__ = ()
+
 
 class OrderedDict(dict):
     """A dict that returns keys/values/items in the order they were added."""
 
+    __slots__ = '_list',
+
+    def __reduce__(self):
+        return OrderedDict, (self.items(),)
+
     def __init__(self, ____sequence=None, **kwargs):
         self._list = []
         if ____sequence is None:
@@ -355,7 +358,10 @@ class OrderedSet(set):
         set.__init__(self)
         self._list = []
         if d is not None:
-            self.update(d)
+            self._list = unique_list(d)
+            set.update(self, self._list)
+        else:
+            self._list = []
 
     def add(self, element):
         if element not in self:
@@ -730,6 +736,12 @@ ordered_column_set = OrderedSet
 populate_column_dict = PopulateDict
 
 
+_getters = PopulateDict(operator.itemgetter)
+
+_property_getters = PopulateDict(
+    lambda idx: property(operator.itemgetter(idx)))
+
+
 def unique_list(seq, hashfunc=None):
     seen = {}
     if not hashfunc:
@@ -779,10 +791,12 @@ def coerce_generator_arg(arg):
 def to_list(x, default=None):
     if x is None:
         return default
-    if not isinstance(x, (list, tuple)):
+    if not isinstance(x, collections.Iterable) or isinstance(x, string_types):
         return [x]
-    else:
+    elif isinstance(x, list):
         return x
+    else:
+        return list(x)
 
 
 def to_set(x):
@@ -830,17 +844,30 @@ class LRUCache(dict):
     """Dictionary with 'squishy' removal of least
     recently used items.
 
+    Note that either get() or [] should be used here, but
+    generally its not safe to do an "in" check first as the dictionary
+    can change subsequent to that call.
+
     """
 
     def __init__(self, capacity=100, threshold=.5):
         self.capacity = capacity
         self.threshold = threshold
         self._counter = 0
+        self._mutex = threading.Lock()
 
     def _inc_counter(self):
         self._counter += 1
         return self._counter
 
+    def get(self, key, default=None):
+        item = dict.get(self, key, default)
+        if item is not default:
+            item[2] = self._inc_counter()
+            return item[1]
+        else:
+            return default
+
     def __getitem__(self, key):
         item = dict.__getitem__(self, key)
         item[2] = self._inc_counter()
@@ -866,18 +893,45 @@ class LRUCache(dict):
         self._manage_size()
 
     def _manage_size(self):
-        while len(self) > self.capacity + self.capacity * self.threshold:
-            by_counter = sorted(dict.values(self),
-                                key=operator.itemgetter(2),
-                                reverse=True)
-            for item in by_counter[self.capacity:]:
-                try:
-                    del self[item[0]]
-                except KeyError:
-                    # if we couldn't find a key, most
-                    # likely some other thread broke in
-                    # on us. loop around and try again
-                    break
+        if not self._mutex.acquire(False):
+            return
+        try:
+            while len(self) > self.capacity + self.capacity * self.threshold:
+                by_counter = sorted(dict.values(self),
+                                    key=operator.itemgetter(2),
+                                    reverse=True)
+                for item in by_counter[self.capacity:]:
+                    try:
+                        del self[item[0]]
+                    except KeyError:
+                        # deleted elsewhere; skip
+                        continue
+        finally:
+            self._mutex.release()
+
+
+_lw_tuples = LRUCache(100)
+
+
+def lightweight_named_tuple(name, fields):
+    hash_ = (name, ) + tuple(fields)
+    tp_cls = _lw_tuples.get(hash_)
+    if tp_cls:
+        return tp_cls
+
+    tp_cls = type(
+        name, (_LW,),
+        dict([
+            (field, _property_getters[idx])
+            for idx, field in enumerate(fields) if field is not None
+        ] + [('__slots__', ())])
+    )
+
+    tp_cls._real_fields = fields
+    tp_cls._fields = tuple([f for f in fields if f is not None])
+
+    _lw_tuples[hash_] = tp_cls
+    return tp_cls
 
 
 class ScopedRegistry(object):
diff --git a/lib/sqlalchemy/util/compat.py b/lib/sqlalchemy/util/compat.py
index 972fda667..5b6f691f1 100644
--- a/lib/sqlalchemy/util/compat.py
+++ b/lib/sqlalchemy/util/compat.py
@@ -1,5 +1,5 @@
 # util/compat.py
-# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors
+# Copyright (C) 2005-2015 the SQLAlchemy authors and contributors
 # <see AUTHORS file>
 #
 # This module is part of SQLAlchemy and is released under
diff --git a/lib/sqlalchemy/util/deprecations.py b/lib/sqlalchemy/util/deprecations.py
index 124f304fc..4c7ea47e3 100644
--- a/lib/sqlalchemy/util/deprecations.py
+++ b/lib/sqlalchemy/util/deprecations.py
@@ -1,5 +1,5 @@
 # util/deprecations.py
-# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors
+# Copyright (C) 2005-2015 the SQLAlchemy authors and contributors
 # <see AUTHORS file>
 #
 # This module is part of SQLAlchemy and is released under
diff --git a/lib/sqlalchemy/util/langhelpers.py b/lib/sqlalchemy/util/langhelpers.py
index 5c17bea88..3d7bfad0a 100644
--- a/lib/sqlalchemy/util/langhelpers.py
+++ b/lib/sqlalchemy/util/langhelpers.py
@@ -1,5 +1,5 @@
 # util/langhelpers.py
-# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors
+# Copyright (C) 2005-2015 the SQLAlchemy authors and contributors
 # <see AUTHORS file>
 #
 # This module is part of SQLAlchemy and is released under
@@ -92,6 +92,15 @@ def _unique_symbols(used, *bases):
             raise NameError("exhausted namespace for symbol base %s" % base)
 
 
+def map_bits(fn, n):
+    """Call the given function given each nonzero bit from n."""
+
+    while n:
+        b = n & (~n + 1)
+        yield fn(b)
+        n ^= b
+
+
 def decorator(target):
     """A signature-matching decorator factory."""
 
@@ -513,6 +522,15 @@ class portable_instancemethod(object):
 
     """
 
+    __slots__ = 'target', 'name', '__weakref__'
+
+    def __getstate__(self):
+        return {'target': self.target, 'name': self.name}
+
+    def __setstate__(self, state):
+        self.target = state['target']
+        self.name = state['name']
+
     def __init__(self, meth):
         self.target = meth.__self__
         self.name = meth.__name__
@@ -791,6 +809,40 @@ class group_expirable_memoized_property(object):
         return memoized_instancemethod(fn)
 
 
+class MemoizedSlots(object):
+    """Apply memoized items to an object using a __getattr__ scheme.
+
+    This allows the functionality of memoized_property and
+    memoized_instancemethod to be available to a class using __slots__.
+
+    """
+
+    def _fallback_getattr(self, key):
+        raise AttributeError(key)
+
+    def __getattr__(self, key):
+        if key.startswith('_memoized'):
+            raise AttributeError(key)
+        elif hasattr(self, '_memoized_attr_%s' % key):
+            value = getattr(self, '_memoized_attr_%s' % key)()
+            setattr(self, key, value)
+            return value
+        elif hasattr(self, '_memoized_method_%s' % key):
+            fn = getattr(self, '_memoized_method_%s' % key)
+
+            def oneshot(*args, **kw):
+                result = fn(*args, **kw)
+                memo = lambda *a, **kw: result
+                memo.__name__ = fn.__name__
+                memo.__doc__ = fn.__doc__
+                setattr(self, key, memo)
+                return result
+            oneshot.__doc__ = fn.__doc__
+            return oneshot
+        else:
+            return self._fallback_getattr(key)
+
+
 def dependency_for(modulename):
     def decorate(obj):
         # TODO: would be nice to improve on this import silliness,
@@ -936,7 +988,7 @@ def asbool(obj):
 
 
 def bool_or_str(*text):
-    """Return a callable that will evaulate a string as
+    """Return a callable that will evaluate a string as
     boolean, or one of a set of "alternate" string values.
 
     """
@@ -969,7 +1021,7 @@ def coerce_kw_type(kw, key, type_, flexi_bool=True):
             kw[key] = type_(kw[key])
 
 
-def constructor_copy(obj, cls, **kw):
+def constructor_copy(obj, cls, *args, **kw):
     """Instantiate cls using the __dict__ of obj as constructor arguments.
 
     Uses inspect to match the named arguments of ``cls``.
@@ -978,7 +1030,7 @@ def constructor_copy(obj, cls, **kw):
 
     names = get_cls_kwargs(cls)
     kw.update((k, obj.__dict__[k]) for k in names if k in obj.__dict__)
-    return cls(**kw)
+    return cls(*args, **kw)
 
 
 def counter():
@@ -1205,9 +1257,12 @@ def warn_exception(func, *args, **kwargs):
 
 
 def ellipses_string(value, len_=25):
-    if len(value) > len_:
-        return "%s..." % value[0:len_]
-    else:
+    try:
+        if len(value) > len_:
+            return "%s..." % value[0:len_]
+        else:
+            return value
+    except TypeError:
         return value
 
 
@@ -1296,6 +1351,7 @@ def chop_traceback(tb, exclude_prefix=_UNITTEST_RE, exclude_suffix=_SQLA_RE):
 
 NoneType = type(None)
 
+
 def attrsetter(attrname):
     code = \
         "def set(obj, value):"\
@@ -1303,3 +1359,29 @@ def attrsetter(attrname):
     env = locals().copy()
     exec(code, env)
     return env['set']
+
+
+class EnsureKWArgType(type):
+    """Apply translation of functions to accept **kw arguments if they
+    don't already.
+
+    """
+    def __init__(cls, clsname, bases, clsdict):
+        fn_reg = cls.ensure_kwarg
+        if fn_reg:
+            for key in clsdict:
+                m = re.match(fn_reg, key)
+                if m:
+                    fn = clsdict[key]
+                    spec = inspect.getargspec(fn)
+                    if not spec.keywords:
+                        clsdict[key] = wrapped = cls._wrap_w_kw(fn)
+                        setattr(cls, key, wrapped)
+        super(EnsureKWArgType, cls).__init__(clsname, bases, clsdict)
+
+    def _wrap_w_kw(self, fn):
+
+        def wrap(*arg, **kw):
+            return fn(*arg)
+        return update_wrapper(wrap, fn)
+
diff --git a/lib/sqlalchemy/util/queue.py b/lib/sqlalchemy/util/queue.py
index 796c6a33e..29e00a434 100644
--- a/lib/sqlalchemy/util/queue.py
+++ b/lib/sqlalchemy/util/queue.py
@@ -1,5 +1,5 @@
 # util/queue.py
-# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors
+# Copyright (C) 2005-2015 the SQLAlchemy authors and contributors
 # <see AUTHORS file>
 #
 # This module is part of SQLAlchemy and is released under
diff --git a/lib/sqlalchemy/util/topological.py b/lib/sqlalchemy/util/topological.py
index 2bfcccc63..80735c4df 100644
--- a/lib/sqlalchemy/util/topological.py
+++ b/lib/sqlalchemy/util/topological.py
@@ -1,5 +1,5 @@
 # util/topological.py
-# Copyright (C) 2005-2014 the SQLAlchemy authors and contributors
+# Copyright (C) 2005-2015 the SQLAlchemy authors and contributors
 # <see AUTHORS file>
 #
 # This module is part of SQLAlchemy and is released under