diff options
author | Stuart Bishop <stuart@stuartbishop.net> | 2013-09-05 14:05:22 +0000 |
---|---|---|
committer | Stuart Bishop <stuart@stuartbishop.net> | 2013-09-05 14:05:22 +0000 |
commit | fbb465db8df63d8a5b5d15ad915be71eb405f2be (patch) | |
tree | ebb5dfabb543aebbf10deb27840161abd2fedc55 | |
parent | a9e92b0f979c8e42a269f13ae5ae5e9df1718148 (diff) | |
download | pytz-fbb465db8df63d8a5b5d15ad915be71eb405f2be.tar.gz |
Lazy load the all_timezones and common_timezones data structuresrelease_2013d
-rw-r--r-- | gen_tzinfo.py | 12 | ||||
-rw-r--r-- | src/pytz/__init__.py | 37 | ||||
-rw-r--r-- | src/pytz/lazy.py | 150 |
3 files changed, 159 insertions, 40 deletions
diff --git a/gen_tzinfo.py b/gen_tzinfo.py index 052cbe6..a1ae7e5 100644 --- a/gen_tzinfo.py +++ b/gen_tzinfo.py @@ -122,17 +122,17 @@ def add_allzones(filename): print >> outf, 'all_timezones = \\' pprint(sorted(allzones()), outf) - print >> outf, '''all_timezones = [ - tz for tz in all_timezones if resource_exists(tz)] + print >> outf, '''all_timezones = LazyList( + tz for tz in all_timezones if resource_exists(tz)) ''' - print >> outf, 'all_timezones_set = set(all_timezones)' + print >> outf, 'all_timezones_set = LazySet(all_timezones)' print >> outf, 'common_timezones = \\' pprint(cz, outf) - print >> outf, '''common_timezones = [ - tz for tz in common_timezones if tz in all_timezones] + print >> outf, '''common_timezones = LazyList( + tz for tz in common_timezones if tz in all_timezones) ''' - print >> outf, 'common_timezones_set = set(common_timezones)' + print >> outf, 'common_timezones_set = LazySet(common_timezones)' outf.close() diff --git a/src/pytz/__init__.py b/src/pytz/__init__.py index 59fc4a4..a6e5282 100644 --- a/src/pytz/__init__.py +++ b/src/pytz/__init__.py @@ -26,10 +26,6 @@ __all__ = [ ] import sys, datetime, os.path, gettext -try: - from UserDict import DictMixin -except ImportError: - from collections import Mapping as DictMixin try: from pkg_resources import resource_stream @@ -40,6 +36,7 @@ from pytz.exceptions import AmbiguousTimeError from pytz.exceptions import InvalidTimeError from pytz.exceptions import NonExistentTimeError from pytz.exceptions import UnknownTimeZoneError +from pytz.lazy import LazyDict, LazyList, LazySet from pytz.tzinfo import unpickler from pytz.tzfile import build_tzinfo, _byte_string @@ -292,36 +289,8 @@ def _p(*args): _p.__safe_for_unpickling__ = True -class _LazyDict(DictMixin): - """Dictionary populated on first use.""" - data = None - def __getitem__(self, key): - if self.data is None: - self._fill() - return self.data[key.upper()] - - def __contains__(self, key): - if self.data is None: - self._fill() - return key in self.data - - def __iter__(self): - if self.data is None: - self._fill() - return iter(self.data) - - def __len__(self): - if self.data is None: - self._fill() - return len(self.data) - - def keys(self): - if self.data is None: - self._fill() - return self.data.keys() - -class _CountryTimezoneDict(_LazyDict): +class _CountryTimezoneDict(LazyDict): """Map ISO 3166 country code to a list of timezone names commonly used in that country. @@ -379,7 +348,7 @@ class _CountryTimezoneDict(_LazyDict): country_timezones = _CountryTimezoneDict() -class _CountryNameDict(_LazyDict): +class _CountryNameDict(LazyDict): '''Dictionary proving ISO3166 code -> English name. >>> print(country_names['au']) diff --git a/src/pytz/lazy.py b/src/pytz/lazy.py new file mode 100644 index 0000000..aed7e10 --- /dev/null +++ b/src/pytz/lazy.py @@ -0,0 +1,150 @@ +from threading import RLock +try: + from UserDict import DictMixin +except ImportError: + from collections import Mapping as DictMixin + + +# With lazy loading, we might end up with multiple threads triggering +# it at the same time. We need a lock. +_fill_lock = RLock() + + +class LazyDict(DictMixin): + """Dictionary populated on first use.""" + data = None + def __getitem__(self, key): + if self.data is None: + _fill_lock.acquire() + try: + if self.data is None: + self._fill() + finally: + _fill_lock.release() + return self.data[key.upper()] + + def __contains__(self, key): + if self.data is None: + _fill_lock.acquire() + try: + if self.data is None: + self._fill() + finally: + _fill_lock_release() + return key in self.data + + def __iter__(self): + if self.data is None: + _fill_lock.acquire() + try: + if self.data is None: + self._fill() + finally: + _fill_lock.release() + return iter(self.data) + + def __len__(self): + if self.data is None: + _fill_lock.acquire() + try: + if self.data is None: + self._fill() + finally: + _fill_lock.release() + return len(self.data) + + def keys(self): + if self.data is None: + _fill_lock.acquire() + try: + if self.data is None: + self._fill() + finally: + _fill_lock.release() + return self.data.keys() + + +class LazyList(list): + """List populated on first use.""" + def __new__(cls, fill_iter): + + class LazyList(list): + _fill_iter = None + + _props = ( + '__str__', '__repr__', '__unicode__', + '__hash__', '__sizeof__', '__cmp__', '__nonzero__', + '__lt__', '__le__', '__eq__', '__ne__', '__gt__', '__ge__', + 'append', 'count', 'index', 'extend', 'insert', 'pop', 'remove', + 'reverse', 'sort', '__add__', '__radd__', '__iadd__', '__mul__', + '__rmul__', '__imul__', '__contains__', '__len__', '__nonzero__', + '__getitem__', '__setitem__', '__delitem__', '__iter__', + '__reversed__', '__getslice__', '__setslice__', '__delslice__') + + def lazy(name): + def _lazy(self, *args, **kw): + if self._fill_iter is not None: + _fill_lock.acquire() + try: + if self._fill_iter is not None: + list.extend(self, self._fill_iter) + self._fill_iter = None + finally: + _fill_lock.release() + real = getattr(list, name) + setattr(self.__class__, name, real) + return real(self, *args, **kw) + return _lazy + + for name in _props: + setattr(LazyList, name, lazy(name)) + + new_list = LazyList() + new_list._fill_iter = fill_iter + return new_list + + +class LazySet(set): + """Set populated on first use.""" + def __new__(cls, fill_iter): + + class LazySet(set): + _fill_iter = None + + _props = ( + '__str__', '__repr__', '__unicode__', + '__hash__', '__sizeof__', '__cmp__', '__nonzero__', + '__lt__', '__le__', '__eq__', '__ne__', '__gt__', '__ge__', + '__contains__', '__len__', '__nonzero__', + '__getitem__', '__setitem__', '__delitem__', '__iter__', + '__sub__', '__and__', '__xor__', '__or__', + '__rsub__', '__rand__', '__rxor__', '__ror__', + '__isub__', '__iand__', '__ixor__', '__ior__', + 'add', 'clear', 'copy', 'difference', 'difference_update', + 'discard', 'intersection', 'intersection_update', 'isdisjoint', + 'issubset', 'issuperset', 'pop', 'remove', + 'symmetric_difference', 'symmetric_difference_update', + 'union', 'update') + + def lazy(name): + def _lazy(self, *args, **kw): + if self._fill_iter is not None: + _fill_lock.acquire() + try: + if self._fill_iter is not None: + for i in self._fill_iter: + set.add(self, i) + self._fill_iter = None + finally: + _fill_lock.release() + real = getattr(set, name) + setattr(self.__class__, name, real) + return real(self, *args, **kw) + return _lazy + + for name in _props: + setattr(LazySet, name, lazy(name)) + + new_set = LazySet() + new_set._fill_iter = fill_iter + return new_set |