[py3k] remove "os.path.walk" occurencies by using "os.path"

4 files changed, 73 insertions, 90 deletions

diff --git a/__init__.py b/__init__.py
index c31a274..558868e 100644
--- a/__init__.py
+++ b/__init__.py
@@ -138,3 +138,18 @@ def make_domains(lists):
                 i += 1
         domains.append(new_domain)
     return domains
+
+
+# private stuff ################################################################
+
+def _handle_blacklist(blacklist, dirnames, filenames):
+    """remove files/directories in the black list
+
+    dirnames/filenames are usually from os.walk
+    """
+    for norecurs in blacklist:
+        if norecurs in dirnames:
+            dirnames.remove(norecurs)
+        elif norecurs in filenames:
+            filenames.remove(norecurs)
+
diff --git a/fileutils.py b/fileutils.py
index 9ffd67c..86a83a4 100644
--- a/fileutils.py
+++ b/fileutils.py
@@ -28,9 +28,9 @@ __docformat__ = "restructuredtext en"
 import sys
 import shutil
 import mimetypes
-from os.path import isabs, isdir, islink, split, exists, walk, normpath, join
+from os.path import isabs, isdir, islink, split, exists, normpath, join
 from os.path import abspath
-from os import sep, mkdir, remove, listdir, stat, chmod
+from os import sep, mkdir, remove, listdir, stat, chmod, walk
 from stat import ST_MODE, S_IWRITE
 from cStringIO import StringIO
 
@@ -356,35 +356,34 @@ def export(from_dir, to_dir,
       flag indicating whether information about exported files should be
       printed to stderr, default to False
     """
-    def make_mirror(_, directory, fnames):
-        """walk handler"""
+    try:
+        mkdir(to_dir)
+    except OSError:
+        pass # FIXME we should use "exists" if the point is about existing dir
+             # else (permission problems?) shouldn't return / raise ?
+    for directory, dirnames, filenames in walk(from_dir):
         for norecurs in blacklist:
             try:
-                fnames.remove(norecurs)
+                dirnames.remove(norecurs)
             except ValueError:
                 continue
-        for filename in fnames:
+        for dirname in dirnames:
+            src = join(directory, dirname)
+            dest = to_dir + src[len(from_dir):]
+            if isdir(src):
+                if not exists(dest):
+                    mkdir(dest)
+        for filename in filenames:
             # don't include binary files
-            for ext in ignore_ext:
-                if filename.endswith(ext):
-                    break
-            else:
-                src = join(directory, filename)
-                dest = to_dir + src[len(from_dir):]
-                if verbose:
-                    print >> sys.stderr, src, '->', dest
-                if isdir(src):
-                    if not exists(dest):
-                        mkdir(dest)
-                else:
-                    if exists(dest):
-                        remove(dest)
-                    shutil.copy2(src, dest)
-    try:
-        mkdir(to_dir)
-    except OSError:
-        pass
-    walk(from_dir, make_mirror, None)
+            if filename.endswith(ignore_ext):
+                continue
+            src = join(directory, filename)
+            dest = to_dir + src[len(from_dir):]
+            if verbose:
+                print >> sys.stderr, src, '->', dest
+            if exists(dest):
+                remove(dest)
+            shutil.copy2(src, dest)
 
 
 def remove_dead_links(directory, verbose=0):
@@ -398,12 +397,11 @@ def remove_dead_links(directory, verbose=0):
       flag indicating whether information about deleted links should be
       printed to stderr, default to False
     """
-    def _remove_dead_link(_, directory, fnames):
-        """walk handler"""
-        for filename in fnames:
-            src = join(directory, filename)
+    for dirpath, dirname, filenames in walk(directory):
+        for filename in dirnames + filenames:
+            src = join(dirpath, filename)
             if islink(src) and not exists(src):
                 if verbose:
                     print 'remove dead link', src
                 remove(src)
-    walk(directory, _remove_dead_link, None)
+
diff --git a/modutils.py b/modutils.py
index 1f66e26..543955f 100644
--- a/modutils.py
+++ b/modutils.py
@@ -31,7 +31,7 @@ __docformat__ = "restructuredtext en"
 
 import sys
 import os
-from os.path import walk, splitext, join, abspath, isdir, dirname, exists
+from os.path import splitext, join, abspath, isdir, dirname, exists, basename
 from imp import find_module, load_module, C_BUILTIN, PY_COMPILED, PKG_DIRECTORY
 
 try:
@@ -41,7 +41,7 @@ except ImportError:
 
 ZIPFILE = object()
 
-from logilab.common import STD_BLACKLIST
+from logilab.common import STD_BLACKLIST, _handle_blacklist
 
 if sys.platform.startswith('win'):
     PY_SOURCE_EXTS = ('py', 'pyw')
@@ -341,7 +341,6 @@ def get_module_part(dotted_name, context_file=None):
     return dotted_name
 
 
-
 def get_modules(package, src_directory, blacklist=STD_BLACKLIST):
     """given a package directory return a list of all available python
     modules in the package and its subpackages
@@ -363,31 +362,21 @@ def get_modules(package, src_directory, blacklist=STD_BLACKLIST):
       the list of all available python modules in the package and its
       subpackages
     """
-    def func(modules, directory, fnames):
-        """walk handler"""
-        # remove files/directories in the black list
-        for norecurs in blacklist:
-            try:
-                fnames.remove(norecurs)
-            except ValueError:
-                continue
+    modules = []
+    for directory, dirnames, filenames in os.walk(src_directory):
+        _handle_blacklist(blacklist, dirnames, filenames)
         # check for __init__.py
-        if not '__init__.py' in fnames:
-            while fnames:
-                fnames.pop()
-        elif directory != src_directory:
-            #src = join(directory, file)
+        if not '__init__.py' in filenames:
+            dirnames[:] = ()
+            continue
+        if directory != src_directory:
             dir_package = directory[len(src_directory):].replace(os.sep, '.')
             modules.append(package + dir_package)
-        for filename in fnames:
-            src = join(directory, filename)
-            if isdir(src):
-                continue
+        for filename in filenames:
             if _is_python_file(filename) and filename != '__init__.py':
+                src = join(directory, filename)
                 module = package + src[len(src_directory):-3]
                 modules.append(module.replace(os.sep, '.'))
-    modules = []
-    walk(src_directory, func, modules)
     return modules
 
 
@@ -410,26 +399,17 @@ def get_module_files(src_directory, blacklist=STD_BLACKLIST):
       the list of all available python module's files in the package and
       its subpackages
     """
-    def func(files, directory, fnames):
-        """walk handler"""
-        # remove files/directories in the black list
-        for norecurs in blacklist:
-            try:
-                fnames.remove(norecurs)
-            except ValueError:
-                continue
+    files = []
+    for directory, dirnames, filenames in os.walk(src_directory):
+        _handle_blacklist(blacklist, dirnames, filenames)
         # check for __init__.py
-        if not '__init__.py' in fnames:
-            while fnames:
-                fnames.pop()
-        for filename in fnames:
-            src = join(directory, filename)
-            if isdir(src):
-                continue
+        if not '__init__.py' in filenames:
+            dirnames[:] = ()
+            continue
+        for filename in filenames:
             if _is_python_file(filename):
+                src = join(directory, filename)
                 files.append(src)
-    files = []
-    walk(src_directory, func, files)
     return files
 
 
diff --git a/shellutils.py b/shellutils.py
index 4c6e8a8..7180b5e 100644
--- a/shellutils.py
+++ b/shellutils.py
@@ -29,9 +29,9 @@ import tempfile
 import time
 import fnmatch
 import errno
-from os.path import exists, isdir, islink, basename, join, walk
+from os.path import exists, isdir, islink, basename, join
 
-from logilab.common import STD_BLACKLIST
+from logilab.common import STD_BLACKLIST, _handle_blacklist
 try:
     from logilab.common.proc import ProcInfo, NoSuchProcess
 except ImportError:
@@ -168,22 +168,14 @@ def find(directory, exts, exclude=False, blacklist=STD_BLACKLIST):
                 if filename.endswith(ext):
                     return True
             return False
-    def func(files, directory, fnames):
-        """walk handler"""
-        # remove files/directories in the black list
-        for norecurs in blacklist:
-            try:
-                fnames.remove(norecurs)
-            except ValueError:
-                continue
-        for filename in fnames:
-            src = join(directory, filename)
-            if isdir(src):
-                continue
-            if match(filename, exts):
-                files.append(src)
     files = []
-    walk(directory, func, files)
+    for dirpath, dirnames, filenames in os.walk(directory):
+        _handle_blacklist(blacklist, dirnames, filenames)
+        # don't append files if the directory is blacklisted
+        dirname = basename(dirpath)
+        if dirname in blacklist:
+            continue
+        files.extend([join(dirpath, f) for f in filenames if match(f, exts)])
     return files
 
 
@@ -210,11 +202,9 @@ def globfind(directory, pattern, blacklist=STD_BLACKLIST):
       iterator over the list of all matching files
     """
     for curdir, dirnames, filenames in os.walk(directory):
+        _handle_blacklist(blacklist, dirnames, filenames)
         for fname in fnmatch.filter(filenames, pattern):
             yield join(curdir, fname)
-        for skipped in blacklist:
-            if skipped in dirnames:
-                dirnames.remove(skipped)
 
 def unzip(archive, destdir):
     import zipfile