From b42ef986b9ce537638ea931422063f97a18f8981 Mon Sep 17 00:00:00 2001 From: Philip Thiem Date: Thu, 7 Nov 2013 21:11:18 -0600 Subject: removed fsencode and fsdecode replaced fsencode with decode_as_string - decoded utf8 or console input to decode_as_string should be called on encoded string before joining BUG_FIX - It would seem that there is a issue wiht the legacy file listing which this commit fits. I have no clue how any of the other stuff would had even worked. --- setuptools/svn_utils.py | 97 ++++++++++++++++++++++--------------------------- 1 file changed, 44 insertions(+), 53 deletions(-) (limited to 'setuptools/svn_utils.py') diff --git a/setuptools/svn_utils.py b/setuptools/svn_utils.py index 55dbb74d..6ac31a24 100644 --- a/setuptools/svn_utils.py +++ b/setuptools/svn_utils.py @@ -27,18 +27,18 @@ from subprocess import Popen as _Popen, PIPE as _PIPE # python-subprocess-popen-environment-path -def _run_command(args, stdout=_PIPE, stderr=_PIPE): +def _run_command(args, stdout=_PIPE, stderr=_PIPE, encoding=None, stream=0): #regarding the shell argument, see: http://bugs.python.org/issue8557 try: - args = [fsdecode(x) for x in args] proc = _Popen(args, stdout=stdout, stderr=stderr, shell=(sys.platform == 'win32')) - data = proc.communicate()[0] + data = proc.communicate()[stream] except OSError: return 1, '' - data = consoledecode(data) + #doubled checked and + data = decode_as_string(data, encoding) #communciate calls wait() return proc.returncode, data @@ -73,40 +73,28 @@ def joinpath(prefix, *suffix): return os.path.join(prefix, *suffix) -def fsencode(path): - "Path must be unicode or in file system encoding already" - encoding = sys.getfilesystemencoding() - - if isinstance(path, unicode): - path = path.encode() - elif not isinstance(path, bytes): - raise TypeError('%s is not a string or byte type' - % type(path).__name__) - - #getfilessystemencoding doesn't have the mac-roman issue - if encoding == 'utf-8' and sys.platform == 'darwin': - path = path.decode('utf-8') - path = unicodedata.normalize('NFD', path) - path = path.encode('utf-8') - - return path - - -def fsdecode(path): - "Path must be unicode or in file system encoding already" - encoding = sys.getfilesystemencoding() - if isinstance(path, bytes): - path = path.decode(encoding) - elif not isinstance(path, unicode): - raise TypeError('%s is not a byte type' - % type(path).__name__) - - return unicodedata.normalize('NFC', path) +def decode_as_string(text, encoding=None): + """ + Decode the console or file output explicitly using getpreferredencoding. + The text paraemeter should be a encoded string, if not no decode occurs + If no encoding is given, getpreferredencoding is used. If encoding is + specified, that is used instead. This would be needed for SVN --xml + output. Unicode is explicitly put in composed NFC form. + + --xml should be UTF-8 (SVN Issue 2938) the discussion on the Subversion + DEV List from 2007 seems to indicate the same. + """ + #text should be a byte string + if encoding is None: + encoding = locale.getpreferredencoding() -def consoledecode(text): - encoding = locale.getpreferredencoding() - return text.decode(encoding) + if not isinstance(text, unicode): + text = text.decode(encoding) + + text = unicodedata.normalize('NFC', text) + + return text def parse_dir_entries(decoded_str): @@ -141,6 +129,7 @@ def parse_externals_xml(decoded_str, prefix=''): path = path[len(prefix)+1:] data = _get_target_property(node) + #data should be decoded already for external in parse_external_prop(data): externals.append(joinpath(path, external)) @@ -177,6 +166,7 @@ def parse_external_prop(lines): else: external = line[-1] + external = decode_as_string(external, encoding="utf-8") externals.append(os.path.normpath(external)) return externals @@ -214,9 +204,9 @@ class SvnInfo(object): def get_svn_version(): code, data = _run_command(['svn', '--version', '--quiet']) if code == 0 and data: - return unicode(data).strip() + return data.strip() else: - return unicode('') + return '' #svnversion return values (previous implementations return max revision) # 4123:4168 mixed revision working copy @@ -314,7 +304,8 @@ class SvnInfo(object): class Svn13Info(SvnInfo): def get_entries(self): - code, data = _run_command(['svn', 'info', '-R', '--xml', self.path]) + code, data = _run_command(['svn', 'info', '-R', '--xml', self.path], + encoding="utf-8") if code: log.debug("svn info failed") @@ -328,10 +319,11 @@ class Svn13Info(SvnInfo): cmd = ['svn', 'propget', 'svn:externals'] result = [] for folder in self.iter_dirs(): - code, lines = _run_command(cmd + [folder]) + code, lines = _run_command(cmd + [folder], encoding="utf-8") if code != 0: log.warn("svn propget failed") return [] + #lines should a str for external in parse_external_prop(lines): if folder: external = os.path.join(folder, external) @@ -343,7 +335,7 @@ class Svn13Info(SvnInfo): class Svn15Info(Svn13Info): def get_externals(self): cmd = ['svn', 'propget', 'svn:externals', self.path, '-R', '--xml'] - code, lines = _run_command(cmd) + code, lines = _run_command(cmd, encoding="utf-8") if code: log.debug("svn propget failed") return [] @@ -363,6 +355,7 @@ class SvnFileInfo(SvnInfo): entries = SVNEntriesFile.load(base) yield (base, False, entries.parse_revision()) for path in entries.get_undeleted_records(): + path = decode_as_string(path) path = joinpath(base, path) if os.path.isfile(path): yield (path, True, None) @@ -371,18 +364,17 @@ class SvnFileInfo(SvnInfo): yield item def _build_entries(self): - dirs = list() - files = list() + entries = list() + rev = 0 for path, isfile, dir_rev in self._walk_svn(self.path): if isfile: - files.append(path) + entries.append((path, 'file')) else: - dirs.append(path) + entries.append((path, 'dir')) rev = max(rev, dir_rev) - self._directories = dirs - self._entries = files + self._entries = entries self._revision = rev def get_entries(self): @@ -396,14 +388,11 @@ class SvnFileInfo(SvnInfo): return self._revision def get_externals(self): - if self._directories is None: - self._build_entries() - prop_files = [['.svn', 'dir-prop-base'], ['.svn', 'dir-props']] externals = [] - for dirname in self._directories: + for dirname in self.iter_dirs(): prop_file = None for rel_parts in prop_files: filename = joinpath(dirname, *rel_parts) @@ -412,6 +401,8 @@ class SvnFileInfo(SvnInfo): if prop_file is not None: ext_prop = parse_prop_file(prop_file, 'svn:externals') + #ext_prop should be utf-8 coming from svn:externals + ext_prop = decode_as_string(ext_prop, encoding="utf-8") externals.extend(parse_external_prop(ext_prop)) return externals @@ -422,12 +413,12 @@ def svn_finder(dirname=''): #combined externals and entries due to lack of dir_props in 1.7 info = SvnInfo.load(dirname) for path in info.iter_files(): - yield fsencode(path) + yield path for path in info.iter_externals(): sub_info = SvnInfo.load(path) for sub_path in sub_info.iter_files(): - yield fsencode(sub_path) + yield sub_path class SVNEntriesFile(object): -- cgit v1.2.1