merge

9 files changed, 194 insertions, 166 deletions

diff --git a/ChangeLog b/ChangeLog
index 0433597..c7b9448 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -8,7 +8,9 @@ ChangeLog for logilab.common
        mx.DateTime is available
    * moved some stuff which was in common __init__ file into specific
      module. At this occasion new "decorators" and "deprecation" modules
-     has been add
+     has been added
+   * deprecated fileutils.[files_by_ext,include_files_by_ext,exclude_files_by_ext]
+     functions in favor of new function shellutils.find	
    * mark the following modules for deprecation, they will be removed in a
      near version:
      * astutils: moved to astng
@@ -23,6 +25,7 @@ ChangeLog for logilab.common
      * modutils.load_module_from_parts
      * textutils.searchall
      * tree.Node.leafs
+     * fileutils.get_by_ext, filetutils.get_mode, fileutils.ensure_mode
 
 2006-11-03  --  0.20.2
     * fileutils: new remove_dead_links function
diff --git a/__init__.py b/__init__.py
index f8f2e17..6c88683 100644
--- a/__init__.py
+++ b/__init__.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2004-2006 LOGILAB S.A. (Paris, FRANCE).
+# Copyright (c) 2003-2006 LOGILAB S.A. (Paris, FRANCE).
 # http://www.logilab.fr/ -- mailto:contact@logilab.fr
 #
 # This program is free software; you can redistribute it and/or modify it under
@@ -16,8 +16,24 @@
 """Logilab common libraries:
 
 a set of common functionnalities shared among logilab projects
+
+
+:type STD_BLACKLIST: tuple
+:var STD_BLACKLIST:
+  directories ignored by default by the functions in this package which have
+  to recurse into directories
+
+:type IGNORED_EXTENSIONS: tuple
+:var IGNORED_EXTENSIONS:
+  file extensions that may usually be ignored
 """
 
+STD_BLACKLIST = ('CVS', '.svn', '.hg', 'debian', 'dist', 'build')
+
+IGNORED_EXTENSIONS = ('.pyc', '.pyo', '.elc', '~')
+
+
+
 from logilab.common.deprecation import moved
 
 get_cycles = moved('logilab.common.graph', 'get_cycles')
@@ -110,6 +126,3 @@ def flatten(iterable, tr_func=None, results=None):
         else:
             results.append(tr_func(val))
     return results
-
-
-
diff --git a/deprecation.py b/deprecation.py
index eeb1c3b..d4555f6 100644
--- a/deprecation.py
+++ b/deprecation.py
@@ -16,6 +16,7 @@
 """Deprecation utilities"""
 
 from warnings import warn
+
 from logilab.common.modutils import load_module_from_name
 
 class deprecated(type):
diff --git a/fileutils.py b/fileutils.py
index 272d3a9..7566a11 100644
--- a/fileutils.py
+++ b/fileutils.py
@@ -1,3 +1,6 @@
+# Copyright (c) 2003-2006 LOGILAB S.A. (Paris, FRANCE).
+# http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
 # This program is free software; you can redistribute it and/or modify it under
 # the terms of the GNU General Public License as published by the Free Software
 # Foundation; either version 2 of the License, or (at your option) any later
@@ -12,31 +15,13 @@
 # 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 """Some file / file path manipulation utilities.
 
-:author:    Logilab
-:copyright: 2003-2006 LOGILAB S.A. (Paris, FRANCE)
-:contact:   http://www.logilab.fr/ -- mailto:python-projects@logilab.org
-
 :group path manipulation: first_level_directory, relative_path, is_binary,\
-files_by_ext, include_files_by_ext, exclude_files_by_ext, get_by_ext, \
-remove_dead_links
+get_by_ext, remove_dead_links
 :group file manipulation: norm_read, norm_open, lines, stream_lines, lines,\
 write_open_mode, ensure_fs_mode, export
 :sort: path manipulation, file manipulation
-
-
-
-:type BASE_BLACKLIST: tuple
-:var BASE_BLACKLIST:
-  list files or directories ignored by default by the `export` function
-
-:type IGNORED_EXTENSIONS: tuple
-:var IGNORED_EXTENSIONS:
-  list file extensions ignored by default by the `export` function
-
 """
 
-from __future__ import nested_scopes
-
 __docformat__ = "restructuredtext en"
 
 import sys
@@ -46,8 +31,9 @@ from os.path import isabs, isdir, islink, split, exists, walk, normpath, join
 from os import sep, mkdir, remove, listdir, stat, chmod
 from stat import ST_MODE, S_IWRITE
 from cStringIO import StringIO
-from warnings import warn
 
+from logilab.common import STD_BLACKLIST as BASE_BLACKLIST, IGNORED_EXTENSIONS
+from logilab.common.shellutils import find
 
 def first_level_directory(path):
     """return the first level directory of a path
@@ -106,12 +92,6 @@ def write_open_mode(filename):
         return 'wb'
     return 'w'
 
-# backward compat
-def get_mode(*args, **kwargs):
-    """deprecated, use `files_by_ext` instead"""
-    warn('logilab.common.fileutils.get_mode() is deprecated, use '
-         'write_open_mode() instead', DeprecationWarning)
-    return write_open_mode(*args, **kwargs)
 
 def ensure_fs_mode(filepath, desired_mode=S_IWRITE):
     """check that the given file has the given mode(s) set, else try to
@@ -129,7 +109,6 @@ def ensure_fs_mode(filepath, desired_mode=S_IWRITE):
     if not mode & desired_mode:
         chmod(filepath, mode | desired_mode)
         
-ensure_mode = ensure_fs_mode # backward compat
 
 class ProtectedFile(file):
     """a special file-object class that automatically that automatically
@@ -267,6 +246,7 @@ def norm_read(path):
         return open(path, 'U').read()
     return _LINE_RGX.sub('\n', open(path).read())
 
+
 def norm_open(path):
     """return a stream for a file with content with normalized line feeds
 
@@ -304,6 +284,7 @@ def lines(path, comments=None):
     stream.close()
     return result
 
+
 def stream_lines(stream, comments=None):
     """return a list of non empty lines in the given `stream`
 
@@ -334,13 +315,8 @@ def stream_lines(stream, comments=None):
     return result
 
 
-
-BASE_BLACKLIST = ('CVS', '.svn', 'debian', 'dist', 'build', '__buildlog', '.hg')
-IGNORED_EXTENSIONS = ('.pyc', '.pyo', '.elc', '~')
-
 def export(from_dir, to_dir,
-           blacklist=BASE_BLACKLIST,
-           ignore_ext=IGNORED_EXTENSIONS,
+           blacklist=BASE_BLACKLIST, ignore_ext=IGNORED_EXTENSIONS,
            verbose=0):
     """make a mirror of `from_dir` in `to_dir`, omitting directories and
     files listed in the black list or ending with one of the given
@@ -398,8 +374,32 @@ def export(from_dir, to_dir,
     walk(from_dir, make_mirror, None)
 
 
+def remove_dead_links(directory, verbose=0):
+    """recursivly traverse directory and remove all dead links
+
+    :type directory: str
+    :param directory: directory to cleanup
+
+    :type verbose: bool
+    :param verbose:
+      flag indicating wether information about deleted links should be
+      printed to stderr, default to False
+    """
+    def _remove_dead_link(_, directory, fnames):
+        """walk handler"""
+        for filename in fnames:
+            src = join(directory, filename)
+            if islink(src) and not exists(src):
+                if verbose:
+                    print 'remove dead link', src
+                remove(src)
+    walk(directory, _remove_dead_link, None)
+
+
+from warnings import warn
+
 def files_by_ext(directory, include_exts=None, exclude_exts=None,
-                 exclude_dirs=('CVS', '.svn')):
+                 exclude_dirs=BASE_BLACKLIST):
     """return a list of files in a directory matching (or not) some
     extensions: you should either give the `include_exts` argument (and
     only files ending with one of the listed extensions will be
@@ -423,22 +423,13 @@ def files_by_ext(directory, include_exts=None, exclude_exts=None,
     :return: the list of files matching input criteria
     """
     assert not (include_exts and exclude_exts)
-    if directory in exclude_dirs:
-        return []
-    if exclude_exts:
-        return exclude_files_by_ext(directory, exclude_exts, exclude_dirs)
-    return include_files_by_ext(directory, include_exts, exclude_dirs)
-
-# backward compat
-def get_by_ext(*args, **kwargs):
-    """deprecated, use `files_by_ext` instead"""
-    warn('logilab.common.fileutils.get_by_ext() is deprecated, use '
-         'files_by_ext() instead', DeprecationWarning)
-    return files_by_ext(*args, **kwargs)
-
-
-def include_files_by_ext(directory, include_exts,
-                         exclude_dirs=('CVS', '.svn')):
+    warn("files_by_ext is deprecated, use shellutils.find instead" ,
+         DeprecationWarning, stacklevel=2)
+    if include_exts:
+        return find(directory, include_exts, blacklist=exclude_dirs)
+    return find(directory, exclude_exts, exclude=True, blacklist=exclude_dirs)
+
+def include_files_by_ext(directory, include_exts, exclude_dirs=BASE_BLACKLIST):
     """return a list of files in a directory matching some extensions
 
     :type directory: str
@@ -451,25 +442,13 @@ def include_files_by_ext(directory, include_exts,
     :param exclude_dirs: list of directory where we should not recurse
 
     :rtype: list
-    :return: the list of files matching input criteria
+    :return: the list of files matching input criterias
     """
-    result = []
-    for fname in listdir(directory):
-        absfile = join(directory, fname)
-        for ext in include_exts:
-            if fname.endswith(ext):
-                result.append(join(directory, fname))
-                break
-        else:
-            if isdir(absfile):
-                if fname in exclude_dirs:
-                    continue
-                result += include_files_by_ext(join(directory, fname),
-                                               include_exts, exclude_dirs)
-    return result
+    warn("include_files_by_ext is deprecated, use shellutils.find instead" ,
+         DeprecationWarning, stacklevel=2)
+    return find(directory, include_exts, blacklist=exclude_dirs)
 
-def exclude_files_by_ext(directory, exclude_exts,
-                         exclude_dirs=('CVS', '.svn')):
+def exclude_files_by_ext(directory, exclude_exts, exclude_dirs=BASE_BLACKLIST):
     """return a list of files in a directory not matching some extensions
 
     :type directory: str
@@ -482,43 +461,8 @@ def exclude_files_by_ext(directory, exclude_exts,
     :param exclude_dirs: list of directory where we should not recurse
 
     :rtype: list
-    :return: the list of files matching input criteria
+    :return: the list of files matching input criterias
     """
-    result = []
-    for fname in listdir(directory):
-        absfile = join(directory, fname)
-        for ext in exclude_exts:
-            if fname.endswith(ext) or fname == 'makefile':
-                break
-        else:
-            if isdir(absfile):
-                if fname in exclude_dirs:
-                    continue
-                result += exclude_files_by_ext(absfile, exclude_exts,
-                                               exclude_dirs)
-            else:
-                result.append(join(directory, fname))
-    return result
-
-
-
-def remove_dead_links(directory, verbose=0):
-    """recursivly traverse directory and remove all dead links
-
-    :type directory: str
-    :param directory: directory to cleanup
-
-    :type verbose: bool
-    :param verbose:
-      flag indicating wether information about deleted links should be
-      printed to stderr, default to False
-    """
-    def _remove_dead_link(_, directory, fnames):
-        """walk handler"""
-        for filename in fnames:
-            src = join(directory, filename)
-            if islink(src) and not exists(src):
-                if verbose:
-                    print 'remove dead link', src
-                remove(src)
-    walk(directory, _remove_dead_link, None)
+    warn("exclude_files_by_ext is deprecated, use shellutils.find instead" ,
+         DeprecationWarning, stacklevel=2)
+    return find(directory, exclude_exts, exclude=True, blacklist=exclude_dirs)
diff --git a/modutils.py b/modutils.py
index fadba5b..4ab4656 100644
--- a/modutils.py
+++ b/modutils.py
@@ -39,6 +39,8 @@ import os
 from os.path import walk, splitext, join, abspath, isdir, dirname, exists
 from imp import find_module, load_module, C_BUILTIN, PY_COMPILED, PKG_DIRECTORY
 
+from logilab.common import STD_BLACKLIST
+
 if sys.platform.startswith('win'):
     PY_SOURCE_EXTS = ('py', 'pyw')
     PY_COMPILED_EXTS = ('dll', 'pyd')
@@ -259,7 +261,7 @@ def get_module_part(dotted_name, context_file=None):
 
 
     
-def get_modules(package, src_directory, blacklist=('CVS', '.svn', 'debian')):
+def get_modules(package, src_directory, blacklist=STD_BLACKLIST):
     """given a package directory return a list of all available python
     modules in the package and its subpackages
 
@@ -272,8 +274,8 @@ def get_modules(package, src_directory, blacklist=('CVS', '.svn', 'debian')):
 
     :type blacklist: list or tuple
     :param blacklist:
-      optional list of files or directory to ignore, default to 'CVS',
-      '.svn' and 'debian'
+      optional list of files or directory to ignore, default to
+      the value of `logilab.common.STD_BLACKLIST`
 
     :rtype: list
     :return:
@@ -309,7 +311,7 @@ def get_modules(package, src_directory, blacklist=('CVS', '.svn', 'debian')):
 
 
 
-def get_module_files(src_directory, blacklist=('CVS', '.svn', 'debian')):
+def get_module_files(src_directory, blacklist=STD_BLACKLIST):
     """given a package directory return a list of all available python
     module's files in the package and its subpackages
 
@@ -317,10 +319,10 @@ def get_module_files(src_directory, blacklist=('CVS', '.svn', 'debian')):
     :param src_directory:
       path of the directory corresponding to the package
 
-    :type blacklist: list(str) or tuple(str) 
+    :type blacklist: list or tuple
     :param blacklist:
-      optional list of files or directory to ignore, default to 'CVS',
-      '.svn' and 'debian'
+      optional list of files or directory to ignore, default to the value of
+      `logilab.common.STD_BLACKLIST`
 
     :rtype: list
     :return:
diff --git a/setup.py b/setup.py
index 8a1d555..54a0def 100644
--- a/setup.py
+++ b/setup.py
@@ -17,10 +17,6 @@
 # 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 """ Generic Setup script, takes package info from __pkginfo__.py file """
 
-from __future__ import nested_scopes
-
-__revision__ = '$Id: setup.py,v 1.24 2006-04-10 15:12:53 syt Exp $'
-
 import os
 import sys
 import shutil
@@ -58,6 +54,7 @@ try:
 except ImportError:
     ext_modules = None
 
+from __init__ import STD_BLACKLIST, IGNORED_EXTENSIONS
 BASE_BLACKLIST = ('CVS', 'debian', 'dist', 'build', '__buildlog', '.svn')
 IGNORED_EXTENSIONS = ('.pyc', '.pyo', '.elc')
     
@@ -92,7 +89,7 @@ def get_packages(directory, prefix):
     return result
 
 def export(from_dir, to_dir,
-           blacklist=BASE_BLACKLIST,
+           blacklist=STD_BLACKLIST,
            ignore_ext=IGNORED_EXTENSIONS):
     """make a mirror of from_dir in to_dir, omitting directories and files
     listed in the black list
diff --git a/shellutils.py b/shellutils.py
index da4f355..fefd05c 100644
--- a/shellutils.py
+++ b/shellutils.py
@@ -24,7 +24,9 @@ import shutil
 import sys
 import tempfile
 import time
-from os.path import exists, isdir, basename, join
+from os.path import exists, isdir, basename, join, walk
+
+from logilab.common import STD_BLACKLIST
 
 def mv(source, destination, _action=os.rename):
     """a shell like mv, supporting wildcards
@@ -61,7 +63,65 @@ def cp(source, destination):
     """a shell like cp, supporting wildcards
     """
     mv(source, destination, _action=shutil.copy)
-    
+
+
+def find(directory, exts, exclude=False, blacklist=STD_BLACKLIST):
+    """recursivly find files ending with the given extensions from the directory
+
+    :type directory: str
+    :param directory:
+      directory where the search should start
+
+    :type exts: basestring or list or tuple
+    :param exts:
+      extensions or lists or extensions to search
+
+    :type exclude: boolean
+    :param exts:
+      if this argument is True, returning files NOT ending with the given
+      extensions
+
+    :type blacklist: list or tuple
+    :param blacklist:
+      optional list of files or directory to ignore, default to the value of
+      `logilab.common.STD_BLACKLIST`
+
+    :rtype: list
+    :return:
+      the list of all matching files
+    """
+    if isinstance(exts, basestring):
+        exts = (exts,)
+    if exclude:
+        def match(filename, exts):
+            for ext in exts:
+                if filename.endswith(ext):
+                    return False
+            return True
+    else:
+        def match(filename, exts):
+            for ext in exts:
+                if filename.endswith(ext):
+                    return True
+            return False
+    def func(files, directory, fnames):
+        """walk handler"""
+        # remove files/directories in the black list
+        for norecurs in blacklist:
+            try:
+                fnames.remove(norecurs)
+            except ValueError:
+                continue
+        for filename in fnames:
+            src = join(directory, filename)
+            if isdir(src):
+                continue
+            if match(filename, exts):
+                files.append(src)
+    files = []
+    walk(directory, func, files)
+    return files
+
 
 class Execute:
     """This is a deadlock safe version of popen2 (no stdin), that returns
@@ -78,6 +138,7 @@ class Execute:
         os.remove(outfile)
         os.remove(errfile)
 
+
 def acquire_lock(lock_file, max_try=10, delay=10):
     """acquire a lock represented by a file on the file system"""
     count = 0
diff --git a/test/unittest_fileutils.py b/test/unittest_fileutils.py
index 34f24d7..f3c3136 100755..100644
--- a/test/unittest_fileutils.py
+++ b/test/unittest_fileutils.py
@@ -1,20 +1,14 @@
-"""unit tests for logilab.common.fileutils
+"""unit tests for logilab.common.fileutils"""
 
-Some file / path manipulation utilities
-"""
-__revision__ = "$Id: unittest_fileutils.py,v 1.22 2006-01-03 15:31:16 syt Exp $"
-
-import unittest
 import sys, os, tempfile, shutil
 from os.path import join
 
-from logilab.common.testlib import TestCase
+from logilab.common.testlib import TestCase, unittest_main
 
 from logilab.common.fileutils import *
 
 
-#import data
-DATA_DIR = 'data' #data.__path__[0]
+DATA_DIR = 'data'
 NEWLINES_TXT = join(DATA_DIR,'newlines.txt')
 
 class FirstleveldirectoryTC(TestCase):
@@ -54,37 +48,6 @@ class LinesTC(TestCase):
         self.assertEqual(lines(NEWLINES_TXT, comments='#'),
                          ['1', '2', '3'])
 
-class GetByExtTC(TestCase):
-    def test_include(self):
-        files = files_by_ext(DATA_DIR, include_exts=('.py',))
-        self.assertSetEqual(files,
-                            [join('data', f) for f in ['__init__.py', 'module.py',
-                                                       'module2.py', 'noendingnewline.py',
-                                                       'nonregr.py', join('sub', 'momo.py')]])
-        files = files_by_ext(DATA_DIR, include_exts=('.py',), exclude_dirs=('sub', 'CVS'))
-        self.assertSetEqual(files,
-                            [join('data', f) for f in ['__init__.py', 'module.py',
-                                                       'module2.py', 'noendingnewline.py',
-                                                       'nonregr.py']])
-        
-    def test_exclude(self):
-        files = files_by_ext(DATA_DIR, exclude_exts=('.py', '.pyc'))
-        self.assertSetEqual(files,
-                            [join('data', f) for f in ['foo.txt',
-                                                       'newlines.txt',
-                                                       'normal_file.txt',
-                                                       'test.ini',
-                                                       'test1.msg',
-                                                       'test2.msg',
-                                                       'spam.txt',
-                                                       join('sub', 'doc.txt'),
-                                                       'write_protected_file.txt',
-                                                       ]])
-        
-    def test_exclude_base_dir(self):
-        self.assertEquals(files_by_ext(DATA_DIR, include_exts=('.py',), exclude_dirs=(DATA_DIR,)),
-                          [])
-
 class ExportTC(TestCase):
     def setUp(self):
         self.tempdir = tempfile.mktemp()
@@ -147,4 +110,4 @@ class ModuleDocTest(DocTest):
 del DocTest # necessary if we don't want it to be executed (we don't...)
 
 if __name__ == '__main__':
-    unittest.main()
+    unittest_main()
diff --git a/test/unittest_shellutils.py b/test/unittest_shellutils.py
new file mode 100644
index 0000000..d26ddb2
--- /dev/null
+++ b/test/unittest_shellutils.py
@@ -0,0 +1,44 @@
+"""unit tests for logilab.common.shellutils"""
+
+import sys, os, tempfile, shutil
+from os.path import join
+
+from logilab.common.testlib import TestCase, unittest_main
+
+from logilab.common.fileutils import *
+
+DATA_DIR = 'data'
+
+class FindTC(TestCase):
+    def test_include(self):
+        files = find(DATA_DIR, '.py')
+        self.assertSetEqual(files,
+                            [join('data', f) for f in ['__init__.py', 'module.py',
+                                                       'module2.py', 'noendingnewline.py',
+                                                       'nonregr.py', join('sub', 'momo.py')]])
+        files = find(DATA_DIR, ('.py',), blacklist=('sub',))
+        self.assertSetEqual(files,
+                            [join('data', f) for f in ['__init__.py', 'module.py',
+                                                       'module2.py', 'noendingnewline.py',
+                                                       'nonregr.py']])
+        
+    def test_exclude(self):
+        files = find(DATA_DIR, ('.py', '.pyc'), exclude=True)
+        self.assertSetEqual(files,
+                            [join('data', f) for f in ['foo.txt',
+                                                       'newlines.txt',
+                                                       'normal_file.txt',
+                                                       'test.ini',
+                                                       'test1.msg',
+                                                       'test2.msg',
+                                                       'spam.txt',
+                                                       join('sub', 'doc.txt'),
+                                                       'write_protected_file.txt',
+                                                       ]])
+        
+#    def test_exclude_base_dir(self):
+#        self.assertEquals(files_by_ext(DATA_DIR, include_exts=('.py',), exclude_dirs=(DATA_DIR,)),
+#                          [])
+
+if __name__ == '__main__':
+    unittest_main()