diff options
author | Sylvain Th?nault <sylvain.thenault@logilab.fr> | 2010-09-24 16:12:21 +0200 |
---|---|---|
committer | Sylvain Th?nault <sylvain.thenault@logilab.fr> | 2010-09-24 16:12:21 +0200 |
commit | c429f20605314f304551c6b40e0146415b7ae0b8 (patch) | |
tree | dc016208d4d325932a94d014220c4ed2525a1709 | |
parent | 1ca37b55f874d3df6f97a006c8690b985a86c00b (diff) | |
parent | 8e51e99c31e01f7db033f1bb19b8910059547089 (diff) | |
download | logilab-common-c429f20605314f304551c6b40e0146415b7ae0b8.tar.gz |
backport stable into default
-rw-r--r-- | __init__.py | 15 | ||||
-rw-r--r-- | cli.py | 6 | ||||
-rw-r--r-- | compat.py | 38 | ||||
-rw-r--r-- | configuration.py | 11 | ||||
-rw-r--r-- | fileutils.py | 72 | ||||
-rw-r--r-- | interface.py | 1 | ||||
-rw-r--r-- | modutils.py | 58 | ||||
-rw-r--r-- | optik_ext.py | 5 | ||||
-rw-r--r-- | pytest.py | 2 | ||||
-rw-r--r-- | shellutils.py | 35 | ||||
-rw-r--r-- | test/unittest_compat.py | 24 | ||||
-rw-r--r-- | test/unittest_modutils.py | 16 | ||||
-rw-r--r-- | test/unittest_shellutils.py | 10 | ||||
-rw-r--r-- | test/unittest_testlib.py | 1 | ||||
-rw-r--r-- | testlib.py | 28 | ||||
-rw-r--r-- | umessage.py | 10 |
16 files changed, 192 insertions, 140 deletions
diff --git a/__init__.py b/__init__.py index c31a274..558868e 100644 --- a/__init__.py +++ b/__init__.py @@ -138,3 +138,18 @@ def make_domains(lists): i += 1 domains.append(new_domain) return domains + + +# private stuff ################################################################ + +def _handle_blacklist(blacklist, dirnames, filenames): + """remove files/directories in the black list + + dirnames/filenames are usually from os.walk + """ + for norecurs in blacklist: + if norecurs in dirnames: + dirnames.remove(norecurs) + elif norecurs in filenames: + filenames.remove(norecurs) + @@ -44,9 +44,9 @@ Example:: __docformat__ = "restructuredtext en" -import __builtin__ -if not hasattr(__builtin__, '_'): - __builtin__._ = str +from logilab.common.compat import raw_input, builtins +if not hasattr(builtins, '_'): + builtins._ = str def init_readline(complete_method, histfile=None): @@ -28,7 +28,41 @@ import os import sys from warnings import warn -import __builtin__ +import __builtin__ as builtins # 2to3 will tranform '__builtin__' to 'builtins' + +if sys.version_info < (3, 0): + str_to_bytes = str + def str_encode(string, encoding): + if isinstance(string, unicode): + return string.encode(encoding) + return str(string) +else: + def str_to_bytes(string): + return str.encode(string) + # we have to ignore the encoding in py3k to be able to write a string into a + # TextIOWrapper or like object (which expect an unicode string) + def str_encode(string, encoding): + return str(string) + +try: + callable = callable +except NameError:# callable removed from py3k + import collections + def callable(something): + return isinstance(something, collections.Callable) + del collections + +if sys.version_info < (3, 0): + raw_input = raw_input +else: + raw_input = input + +if sys.version_info < (3, 0): + FileIO = file +else: + import io + FileIO = io.FileIO + del io try: set = set @@ -201,7 +235,7 @@ except TypeError: key= kargs.pop("key", None) #default implementation if key is None: - return __builtin__.max(*args,**kargs) + return builtins.max(*args,**kargs) for karg in kargs: raise TypeError("unexpected keyword argument %s for function max") % karg diff --git a/configuration.py b/configuration.py index 543f534..7e0333d 100644 --- a/configuration.py +++ b/configuration.py @@ -113,7 +113,9 @@ from ConfigParser import ConfigParser, NoOptionError, NoSectionError, \ DuplicateSectionError from warnings import warn -from logilab.common.compat import set, reversed +from logilab.common.compat import set, reversed, callable, raw_input +from logilab.common.compat import str_encode as _encode + from logilab.common.textutils import normalize_text, unquote from logilab.common import optik_ext as optparse @@ -130,11 +132,6 @@ def _get_encoding(encoding, stream): encoding = locale.getpreferredencoding() return encoding -def _encode(string, encoding): - if isinstance(string, unicode): - return string.encode(encoding) - return str(string) - # validation functions ######################################################## @@ -479,7 +476,7 @@ class OptionsManagerMixIn(object): args, optdict = self.optik_option(provider, opt, optdict) option = optikcontainer.add_option(*args, **optdict) self._all_options[opt] = provider - self._maxlevel = max(self._maxlevel, option.level) + self._maxlevel = max(self._maxlevel, option.level or 0) def optik_option(self, provider, opt, optdict): """get our personal option definition and return a suitable form for diff --git a/fileutils.py b/fileutils.py index 9ffd67c..807f2b5 100644 --- a/fileutils.py +++ b/fileutils.py @@ -28,14 +28,15 @@ __docformat__ = "restructuredtext en" import sys import shutil import mimetypes -from os.path import isabs, isdir, islink, split, exists, walk, normpath, join +from os.path import isabs, isdir, islink, split, exists, normpath, join from os.path import abspath -from os import sep, mkdir, remove, listdir, stat, chmod +from os import sep, mkdir, remove, listdir, stat, chmod, walk from stat import ST_MODE, S_IWRITE from cStringIO import StringIO from logilab.common import STD_BLACKLIST as BASE_BLACKLIST, IGNORED_EXTENSIONS from logilab.common.shellutils import find +from logilab.common.compat import FileIO def first_level_directory(path): """Return the first level directory of a path. @@ -123,9 +124,10 @@ def ensure_fs_mode(filepath, desired_mode=S_IWRITE): chmod(filepath, mode | desired_mode) -class ProtectedFile(file): - """A special file-object class that automatically that automatically - does a 'chmod +w' when needed. +# XXX (syt) unused? kill? +class ProtectedFile(FileIO): + """A special file-object class that automatically does a 'chmod +w' when + needed. XXX: for now, the way it is done allows 'normal file-objects' to be created during the ProtectedFile object lifetime. @@ -150,7 +152,7 @@ class ProtectedFile(file): if not self.original_mode & S_IWRITE: chmod(filepath, self.original_mode | S_IWRITE) self.mode_changed = True - file.__init__(self, filepath, mode) + FileIO.__init__(self, filepath, mode) def _restore_mode(self): """restores the original mode if needed""" @@ -162,7 +164,7 @@ class ProtectedFile(file): def close(self): """restore mode before closing""" self._restore_mode() - file.close(self) + FileIO.close(self) def __del__(self): if not self.closed: @@ -356,35 +358,34 @@ def export(from_dir, to_dir, flag indicating whether information about exported files should be printed to stderr, default to False """ - def make_mirror(_, directory, fnames): - """walk handler""" + try: + mkdir(to_dir) + except OSError: + pass # FIXME we should use "exists" if the point is about existing dir + # else (permission problems?) shouldn't return / raise ? + for directory, dirnames, filenames in walk(from_dir): for norecurs in blacklist: try: - fnames.remove(norecurs) + dirnames.remove(norecurs) except ValueError: continue - for filename in fnames: + for dirname in dirnames: + src = join(directory, dirname) + dest = to_dir + src[len(from_dir):] + if isdir(src): + if not exists(dest): + mkdir(dest) + for filename in filenames: # don't include binary files - for ext in ignore_ext: - if filename.endswith(ext): - break - else: - src = join(directory, filename) - dest = to_dir + src[len(from_dir):] - if verbose: - print >> sys.stderr, src, '->', dest - if isdir(src): - if not exists(dest): - mkdir(dest) - else: - if exists(dest): - remove(dest) - shutil.copy2(src, dest) - try: - mkdir(to_dir) - except OSError: - pass - walk(from_dir, make_mirror, None) + if filename.endswith(ignore_ext): + continue + src = join(directory, filename) + dest = to_dir + src[len(from_dir):] + if verbose: + print >> sys.stderr, src, '->', dest + if exists(dest): + remove(dest) + shutil.copy2(src, dest) def remove_dead_links(directory, verbose=0): @@ -398,12 +399,11 @@ def remove_dead_links(directory, verbose=0): flag indicating whether information about deleted links should be printed to stderr, default to False """ - def _remove_dead_link(_, directory, fnames): - """walk handler""" - for filename in fnames: - src = join(directory, filename) + for dirpath, dirname, filenames in walk(directory): + for filename in dirnames + filenames: + src = join(dirpath, filename) if islink(src) and not exists(src): if verbose: print 'remove dead link', src remove(src) - walk(directory, _remove_dead_link, None) + diff --git a/interface.py b/interface.py index 0d5835d..8641905 100644 --- a/interface.py +++ b/interface.py @@ -26,7 +26,6 @@ """ __docformat__ = "restructuredtext en" -from types import ListType, TupleType class Interface(object): """Base class for interfaces.""" diff --git a/modutils.py b/modutils.py index 1f66e26..543955f 100644 --- a/modutils.py +++ b/modutils.py @@ -31,7 +31,7 @@ __docformat__ = "restructuredtext en" import sys import os -from os.path import walk, splitext, join, abspath, isdir, dirname, exists +from os.path import splitext, join, abspath, isdir, dirname, exists, basename from imp import find_module, load_module, C_BUILTIN, PY_COMPILED, PKG_DIRECTORY try: @@ -41,7 +41,7 @@ except ImportError: ZIPFILE = object() -from logilab.common import STD_BLACKLIST +from logilab.common import STD_BLACKLIST, _handle_blacklist if sys.platform.startswith('win'): PY_SOURCE_EXTS = ('py', 'pyw') @@ -341,7 +341,6 @@ def get_module_part(dotted_name, context_file=None): return dotted_name - def get_modules(package, src_directory, blacklist=STD_BLACKLIST): """given a package directory return a list of all available python modules in the package and its subpackages @@ -363,31 +362,21 @@ def get_modules(package, src_directory, blacklist=STD_BLACKLIST): the list of all available python modules in the package and its subpackages """ - def func(modules, directory, fnames): - """walk handler""" - # remove files/directories in the black list - for norecurs in blacklist: - try: - fnames.remove(norecurs) - except ValueError: - continue + modules = [] + for directory, dirnames, filenames in os.walk(src_directory): + _handle_blacklist(blacklist, dirnames, filenames) # check for __init__.py - if not '__init__.py' in fnames: - while fnames: - fnames.pop() - elif directory != src_directory: - #src = join(directory, file) + if not '__init__.py' in filenames: + dirnames[:] = () + continue + if directory != src_directory: dir_package = directory[len(src_directory):].replace(os.sep, '.') modules.append(package + dir_package) - for filename in fnames: - src = join(directory, filename) - if isdir(src): - continue + for filename in filenames: if _is_python_file(filename) and filename != '__init__.py': + src = join(directory, filename) module = package + src[len(src_directory):-3] modules.append(module.replace(os.sep, '.')) - modules = [] - walk(src_directory, func, modules) return modules @@ -410,26 +399,17 @@ def get_module_files(src_directory, blacklist=STD_BLACKLIST): the list of all available python module's files in the package and its subpackages """ - def func(files, directory, fnames): - """walk handler""" - # remove files/directories in the black list - for norecurs in blacklist: - try: - fnames.remove(norecurs) - except ValueError: - continue + files = [] + for directory, dirnames, filenames in os.walk(src_directory): + _handle_blacklist(blacklist, dirnames, filenames) # check for __init__.py - if not '__init__.py' in fnames: - while fnames: - fnames.pop() - for filename in fnames: - src = join(directory, filename) - if isdir(src): - continue + if not '__init__.py' in filenames: + dirnames[:] = () + continue + for filename in filenames: if _is_python_file(filename): + src = join(directory, filename) files.append(src) - files = [] - walk(src_directory, func, files) return files diff --git a/optik_ext.py b/optik_ext.py index 3565877..58c84e3 100644 --- a/optik_ext.py +++ b/optik_ext.py @@ -274,12 +274,13 @@ OptionGroup.level = 0 def level_options(group, outputlevel): return [option for option in group.option_list - if getattr(option, 'level', 0) <= outputlevel + if (getattr(option, 'level', 0) or 0) <= outputlevel and not option.help is SUPPRESS_HELP] def format_option_help(self, formatter): result = [] - for option in level_options(self, getattr(formatter, 'output_level', 0)): + outputlevel = getattr(formatter, 'output_level', 0) or 0 + for option in level_options(self, outputlevel): result.append(formatter.format_option(option)) return "".join(result) OptionContainer.format_option_help = format_option_help @@ -123,8 +123,6 @@ import unittest import imp -import __builtin__ - try: import django diff --git a/shellutils.py b/shellutils.py index 8e3cc3e..deaafb5 100644 --- a/shellutils.py +++ b/shellutils.py @@ -29,9 +29,12 @@ import tempfile import time import fnmatch import errno -from os.path import exists, isdir, islink, basename, join, walk +from os.path import exists, isdir, islink, basename, join + +from logilab.common import STD_BLACKLIST, _handle_blacklist +from logilab.common.compat import raw_input +from logilab.common.compat import str_to_bytes -from logilab.common import STD_BLACKLIST try: from logilab.common.proc import ProcInfo, NoSuchProcess except ImportError: @@ -168,22 +171,14 @@ def find(directory, exts, exclude=False, blacklist=STD_BLACKLIST): if filename.endswith(ext): return True return False - def func(files, directory, fnames): - """walk handler""" - # remove files/directories in the black list - for norecurs in blacklist: - try: - fnames.remove(norecurs) - except ValueError: - continue - for filename in fnames: - src = join(directory, filename) - if isdir(src): - continue - if match(filename, exts): - files.append(src) files = [] - walk(directory, func, files) + for dirpath, dirnames, filenames in os.walk(directory): + _handle_blacklist(blacklist, dirnames, filenames) + # don't append files if the directory is blacklisted + dirname = basename(dirpath) + if dirname in blacklist: + continue + files.extend([join(dirpath, f) for f in filenames if match(f, exts)]) return files @@ -210,11 +205,9 @@ def globfind(directory, pattern, blacklist=STD_BLACKLIST): iterator over the list of all matching files """ for curdir, dirnames, filenames in os.walk(directory): + _handle_blacklist(blacklist, dirnames, filenames) for fname in fnmatch.filter(filenames, pattern): yield join(curdir, fname) - for skipped in blacklist: - if skipped in dirnames: - dirnames.remove(skipped) def unzip(archive, destdir): import zipfile @@ -255,7 +248,7 @@ def acquire_lock(lock_file, max_try=10, delay=10, max_delay=3600): while count: try: fd = os.open(lock_file, os.O_EXCL | os.O_RDWR | os.O_CREAT) - os.write(fd, str(os.getpid())) + os.write(fd, str_to_bytes(str(os.getpid())) ) os.close(fd) return True except OSError, e: diff --git a/test/unittest_compat.py b/test/unittest_compat.py index 899eece..ec566b0 100644 --- a/test/unittest_compat.py +++ b/test/unittest_compat.py @@ -20,7 +20,7 @@ from logilab.common.testlib import TestCase, unittest_main import sys import types -import __builtin__ +from logilab.common.compat import builtins class CompatTCMixIn: MODNAMES = {} @@ -38,8 +38,8 @@ class CompatTCMixIn: for modname in self.MODNAMES: del sys.modules[modname] for funcname, func in self.builtins_backup.items(): - setattr(__builtin__, funcname, func) - # delattr(__builtin__, 'builtin_%s' % funcname) + setattr(builtins, funcname, func) + # delattr(builtins, 'builtin_%s' % funcname) for modname, mod in self.modules_backup.items(): sys.modules[modname] = mod try: @@ -49,18 +49,18 @@ class CompatTCMixIn: def remove_builtins(self): for builtin in self.BUILTINS: - func = getattr(__builtin__, builtin, None) + func = getattr(builtins, builtin, None) if func is not None: self.builtins_backup[builtin] = func - delattr(__builtin__, builtin) - # setattr(__builtin__, 'builtin_%s' % builtin, func) + delattr(builtins, builtin) + # setattr(builtins, 'builtin_%s' % builtin, func) def alter_builtins(self): for builtin, func in self.ALTERED_BUILTINS.iteritems(): - old_func = getattr(__builtin__, builtin, None) + old_func = getattr(builtins, builtin, None) if func is not None: self.builtins_backup[builtin] = old_func - setattr(__builtin__, builtin, func) - # setattr(__builtin__, 'builtin_%s' % builtin, func) + setattr(builtins, builtin, func) + # setattr(builtins, 'builtin_%s' % builtin, func) def remove_modules(self): for modname in self.MODNAMES: @@ -90,6 +90,8 @@ class Py23CompatTC(CompatTCMixIn, TestCase): } def test_sum(self): + if sys.version_info >= (3, 0): + self.skip("don't test 2.3 compat 'sum' on >= 3.0") from logilab.common.compat import sum self.assertEqual(sum(range(5)), 10) self.assertRaises(TypeError, sum, 'abc') @@ -142,6 +144,8 @@ class Py24CompatTC(CompatTCMixIn, TestCase): BUILTINS = ('reversed', 'sorted', 'set', 'frozenset',) def test_sorted(self): + if sys.version_info >= (3, 0): + self.skip("don't test 2.4 compat 'sorted' on >= 3.0") from logilab.common.compat import sorted l = [3, 1, 2, 5, 4] s = sorted(l) @@ -163,6 +167,8 @@ class Py24CompatTC(CompatTCMixIn, TestCase): self.assertEqual(l, range(5)) def test_set(self): + if sys.version_info >= (3, 0): + self.skip("don't test 2.4 compat 'set' on >= 3.0") from logilab.common.compat import set s1 = set(range(5)) s2 = set(range(2, 6)) diff --git a/test/unittest_modutils.py b/test/unittest_modutils.py index 92b55de..92e3d04 100644 --- a/test/unittest_modutils.py +++ b/test/unittest_modutils.py @@ -164,7 +164,12 @@ class is_standard_module_tc(TestCase): """ def test_knownValues_is_standard_module_0(self): - self.assertEqual(modutils.is_standard_module('__builtin__'), True) + if sys.version_info < (3, 0): + self.assertEqual(modutils.is_standard_module('__builtin__'), True) + self.assertEqual(modutils.is_standard_module('builtins'), False) + else: + self.assertEqual(modutils.is_standard_module('__builtin__'), False) + self.assertEqual(modutils.is_standard_module('builtins'), True) def test_knownValues_is_standard_module_1(self): self.assertEqual(modutils.is_standard_module('sys'), True) @@ -176,7 +181,14 @@ class is_standard_module_tc(TestCase): self.assertEqual(modutils.is_standard_module('unknown'), False) def test_knownValues_is_standard_module_4(self): - self.assertEqual(modutils.is_standard_module('StringIO'), True) + if sys.version_info < (3, 0): + self.assertEqual(modutils.is_standard_module('StringIO'), True) + else: + self.assertEqual(modutils.is_standard_module('StringIO'), False) + if sys.version_info < (2, 6): + self.assertEqual(modutils.is_standard_module('io'), False) + else: + self.assertEqual(modutils.is_standard_module('io'), True) def test_knownValues_is_standard_module_5(self): self.assertEqual(modutils.is_standard_module('data.module', (DATADIR,)), True) diff --git a/test/unittest_shellutils.py b/test/unittest_shellutils.py index ff9e4fa..444e75e 100644 --- a/test/unittest_shellutils.py +++ b/test/unittest_shellutils.py @@ -20,15 +20,15 @@ import sys, os, tempfile, shutil from os.path import join import datetime, time +from StringIO import StringIO from logilab.common.testlib import TestCase, unittest_main from logilab.common.shellutils import (globfind, find, ProgressBar, acquire_lock, release_lock, RawInput) - +from logilab.common.compat import str_to_bytes from logilab.common.proc import NoSuchProcess -from StringIO import StringIO DATA_DIR = join('data','find_test') @@ -127,7 +127,7 @@ class ProgressBarTC(TestCase): """Test the progress bar for nbops > size""" def half(total): for counter in range(1,total+1): - yield counter / 2 + yield counter // 2 self._update_test(40, half(40)) def test_nbops_lt_size(self): @@ -165,14 +165,14 @@ class AcquireLockTC(TestCase): def test_wrong_process(self): fd = os.open(self.lock, os.O_EXCL | os.O_RDWR | os.O_CREAT) - os.write(fd, '1111111111') + os.write(fd, str_to_bytes('1111111111')) os.close(fd) self.assertTrue(os.path.exists(self.lock)) self.assertRaises(Exception, acquire_lock, self.lock, 1, 1) def test_wrong_process_and_continue(self): fd = os.open(self.lock, os.O_EXCL | os.O_RDWR | os.O_CREAT) - os.write(fd, '1111111111') + os.write(fd, str_to_bytes('1111111111')) os.close(fd) self.assertTrue(os.path.exists(self.lock)) self.assertTrue(acquire_lock(self.lock)) diff --git a/test/unittest_testlib.py b/test/unittest_testlib.py index 614dafd..7738808 100644 --- a/test/unittest_testlib.py +++ b/test/unittest_testlib.py @@ -17,7 +17,6 @@ # with logilab-common. If not, see <http://www.gnu.org/licenses/>. """unittest module for logilab.comon.testlib""" -import unittest import os import sys from os.path import join, dirname, isdir, isfile, abspath, exists @@ -55,7 +55,6 @@ import math from shutil import rmtree from operator import itemgetter import warnings -from compiler.consts import CO_GENERATOR from ConfigParser import ConfigParser from logilab.common.deprecation import deprecated from itertools import dropwhile @@ -85,7 +84,8 @@ except ImportError: test_support = TestSupport() # pylint: disable=W0622 -from logilab.common.compat import set, enumerate, any, sorted, InheritableSet +from logilab.common.compat import (set, enumerate, any, sorted, InheritableSet, + callable) # pylint: enable-msg=W0622 from logilab.common.modutils import load_module_from_name from logilab.common.debugger import Debugger, colorize_source @@ -102,6 +102,23 @@ ENABLE_DBC = False FILE_RESTART = ".pytest.restart" +if sys.version_info >= (2, 6): + # FIXME : this does not work as expected / breaks tests on testlib + # however testlib does not work on py3k for many reasons ... + from inspect import CO_GENERATOR +else: + from compiler.consts import CO_GENERATOR + +if sys.version_info >= (3, 0): + def is_generator(function): + flags = function.__code__.co_flags + return flags & CO_GENERATOR + +else: + def is_generator(function): + flags = function.func_code.co_flags + return flags & CO_GENERATOR + UNITTEST2 = getattr(unittest, "__package__", "") == 'unittest2' # used by unittest to count the number of relevant levels in the traceback @@ -490,7 +507,6 @@ class SkipAwareTextTestRunner(unittest.TextTestRunner): testname = '%s.%s' % (test.im_class.__name__, func.__name__) else: return True # Not sure when this happens - if is_generator(func) and skipgenerator: return self.does_match_tags(func) # Let inner tests decide at run time @@ -964,10 +980,6 @@ class InnerTestSkipped(SkipTest): """raised when a test is skipped""" pass -def is_generator(function): - flags = function.func_code.co_flags - return flags & CO_GENERATOR - def parse_generative_args(params): args = [] @@ -1038,7 +1050,7 @@ class TestCase(unittest.TestCase): super(TestCase, self).__init__(methodName) # internal API changed in python2.5 if sys.version_info >= (2, 5): - self.__exc_info = self._exc_info + self.__exc_info = sys.exc_info self.__testMethodName = self._testMethodName else: # let's give easier access to _testMethodName to every subclasses diff --git a/umessage.py b/umessage.py index 6662f1e..ef92ba4 100644 --- a/umessage.py +++ b/umessage.py @@ -25,8 +25,14 @@ __docformat__ = "restructuredtext en" import email from encodings import search_function -from email.Utils import parseaddr, parsedate -from email.Header import decode_header +import sys +if sys.version_info >= (2, 5): + from email.utils import parseaddr, parsedate + from email.header import decode_header +else: + from email.Utils import parseaddr, parsedate + from email.Header import decode_header + from datetime import datetime try: |