diff options
author | Jonathan Abrahams <jonathan@mongodb.com> | 2018-05-01 15:41:48 -0400 |
---|---|---|
committer | Jonathan Abrahams <jonathan@mongodb.com> | 2018-05-11 15:28:16 -0400 |
commit | d6837a12b3586b0738dcd4214951a1d6f6b1415e (patch) | |
tree | 764afde4849ded75bedf5f55ec5b864408daf8e8 | |
parent | ecf8ad987548705e773d23d5ddc3973cbc1ea7e5 (diff) | |
download | mongo-d6837a12b3586b0738dcd4214951a1d6f6b1415e.tar.gz |
SERVER-34374 Wrap shutil.rmtree() in resmoke to handle path names which may contain non-ASCII characters
-rw-r--r-- | buildscripts/resmokelib/testing/fixtures/standalone.py | 3 | ||||
-rw-r--r-- | buildscripts/resmokelib/testing/testcases/dbtest.py | 3 | ||||
-rw-r--r-- | buildscripts/resmokelib/testing/testcases/jstest.py | 3 | ||||
-rw-r--r-- | buildscripts/resmokelib/utils/__init__.py | 23 | ||||
-rw-r--r-- | buildscripts/tests/resmokelib/utils/__init__.py | 1 | ||||
-rw-r--r-- | buildscripts/tests/resmokelib/utils/test_rmtree.py | 126 | ||||
-rw-r--r-- | etc/evergreen.yml | 1 |
7 files changed, 154 insertions, 6 deletions
diff --git a/buildscripts/resmokelib/testing/fixtures/standalone.py b/buildscripts/resmokelib/testing/fixtures/standalone.py index 3cae2e7884f..7aaa34fa3de 100644 --- a/buildscripts/resmokelib/testing/fixtures/standalone.py +++ b/buildscripts/resmokelib/testing/fixtures/standalone.py @@ -4,7 +4,6 @@ from __future__ import absolute_import import os import os.path -import shutil import time import pymongo @@ -50,7 +49,7 @@ class MongoDFixture(interface.Fixture): def setup(self): """Set up the mongod.""" if not self.preserve_dbpath: - shutil.rmtree(self._dbpath, ignore_errors=True) + utils.rmtree(self._dbpath, ignore_errors=True) try: os.makedirs(self._dbpath) diff --git a/buildscripts/resmokelib/testing/testcases/dbtest.py b/buildscripts/resmokelib/testing/testcases/dbtest.py index 295114d60a5..4cfbb8c6385 100644 --- a/buildscripts/resmokelib/testing/testcases/dbtest.py +++ b/buildscripts/resmokelib/testing/testcases/dbtest.py @@ -4,7 +4,6 @@ from __future__ import absolute_import import os import os.path -import shutil from . import interface from ... import config @@ -50,7 +49,7 @@ class DBTestCase(interface.ProcessTestCase): self._clear_dbpath() def _clear_dbpath(self): - shutil.rmtree(self.dbtest_options["dbpath"], ignore_errors=True) + utils.rmtree(self.dbtest_options["dbpath"], ignore_errors=True) def _make_process(self): return core.programs.dbtest_program(self.logger, executable=self.dbtest_executable, diff --git a/buildscripts/resmokelib/testing/testcases/jstest.py b/buildscripts/resmokelib/testing/testcases/jstest.py index 9386ddf9306..ae12ac34ed6 100644 --- a/buildscripts/resmokelib/testing/testcases/jstest.py +++ b/buildscripts/resmokelib/testing/testcases/jstest.py @@ -4,7 +4,6 @@ from __future__ import absolute_import import os import os.path -import shutil import sys import threading @@ -73,7 +72,7 @@ class _SingleJSTestCase(interface.ProcessTestCase): global_vars["TestData"] = test_data self.shell_options["global_vars"] = global_vars - shutil.rmtree(data_dir, ignore_errors=True) + utils.rmtree(data_dir, ignore_errors=True) try: os.makedirs(data_dir) diff --git a/buildscripts/resmokelib/utils/__init__.py b/buildscripts/resmokelib/utils/__init__.py index 1db36de5cb2..9213302f8be 100644 --- a/buildscripts/resmokelib/utils/__init__.py +++ b/buildscripts/resmokelib/utils/__init__.py @@ -5,6 +5,7 @@ from __future__ import print_function import contextlib import os.path +import shutil import sys import yaml @@ -38,6 +39,28 @@ def default_if_none(value, default): return value if value is not None else default +def rmtree(path, **kwargs): + """Wrap shutil.rmtreee. + + Use a UTF-8 unicode path if Windows. + See https://bugs.python.org/issue24672, where shutil.rmtree can fail with UTF-8. + Use a bytes path to rmtree, otherwise. + See https://github.com/pypa/setuptools/issues/706. + """ + if is_windows(): + if not isinstance(path, unicode): + path = unicode(path, "utf-8") + else: + if isinstance(path, unicode): + path = path.encode("utf-8") + shutil.rmtree(path, **kwargs) + + +def is_windows(): + """Return True if Windows.""" + return sys.platform.startswith("win32") or sys.platform.startswith("cygwin") + + def is_string_list(lst): """Return true if 'lst' is a list of strings, and false otherwise.""" return isinstance(lst, list) and all(isinstance(x, basestring) for x in lst) diff --git a/buildscripts/tests/resmokelib/utils/__init__.py b/buildscripts/tests/resmokelib/utils/__init__.py new file mode 100644 index 00000000000..4b7a2bb941b --- /dev/null +++ b/buildscripts/tests/resmokelib/utils/__init__.py @@ -0,0 +1 @@ +"""Empty.""" diff --git a/buildscripts/tests/resmokelib/utils/test_rmtree.py b/buildscripts/tests/resmokelib/utils/test_rmtree.py new file mode 100644 index 00000000000..1908395766c --- /dev/null +++ b/buildscripts/tests/resmokelib/utils/test_rmtree.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- +""" Unit tests for utils.rmtree. """ + +from __future__ import absolute_import +from __future__ import print_function + +import os +import shutil +import sys +import tempfile +import unittest + +from buildscripts.resmokelib import utils + +# pylint: disable=missing-docstring,protected-access + + +def rmtree(dir_root): + """Invoke utils.rmtree(dir_root) and return True if removed.""" + utils.rmtree(dir_root) + return not os.path.exists(dir_root) + + +def create_file(path): + """Create file named 'path'.""" + with open(path, "w") as fh: + fh.write("") + + +def ascii_filesystemencoding(): + """Return True if the file system encoding is type ASCII. + + See https://www.iana.org/assignments/character-sets/character-sets.xhtml. + """ + encoding = sys.getfilesystemencoding() + return encoding.startswith("ANSI_X3.4") or encoding == "US-ASCII" + + +def string_for_ascii_filesystem_encoding(path): + """Return encoded string type for unicode on ASCII file system encoding. + + Some file system encodings are set to ASCII if LANG=C or LC_ALL=C is specified. + """ + if ascii_filesystemencoding() and isinstance(path, unicode): + return path.encode("utf-8") + return path + + +class RmtreeTestCase(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.temp_dir_root = tempfile.mkdtemp() + cls.cwd = os.getcwd() + + @classmethod + def tearDownClass(cls): + os.chdir(cls.cwd) + shutil.rmtree(cls.temp_dir_root, ignore_errors=True) + + def do_test(self, name): + pass + + def test_ascii(self): + # ASCII name + self.do_test("ascii") + + def test_unicode(self): + # Unicode name + self.do_test(u"unicode") + + def test_greek(self): + # Name with Greek + self.do_test(string_for_ascii_filesystem_encoding(u"ελληνικά")) + + def test_japanese(self): + # Name with Japanese + self.do_test(string_for_ascii_filesystem_encoding(u"会社案")) + + +class RmtreeFileTests(RmtreeTestCase): + def do_test(self, file_name): # pylint: disable=arguments-differ + """Execute file test.""" + temp_dir = tempfile.mkdtemp(dir=self.temp_dir_root) + os.chdir(temp_dir) + create_file(file_name) + os.chdir(self.temp_dir_root) + self.assertTrue(rmtree(temp_dir)) + + +class RmtreeDirectoryTests(RmtreeTestCase): + def do_test(self, dir_name): # pylint: disable=arguments-differ + """Execute directory test.""" + os.chdir(self.temp_dir_root) + os.mkdir(dir_name) + self.assertTrue(rmtree(dir_name)) + + +class RmtreeDirectoryWithNonAsciiTests(RmtreeTestCase): + def do_test(self, name): + """Execute directory with non-ASCII file test.""" + os.chdir(self.temp_dir_root) + os.mkdir(name) + os.chdir(name) + create_file(name) + os.chdir(self.temp_dir_root) + self.assertTrue(rmtree(name)) + + +class ShutilWindowsRmtreeFileTests(RmtreeFileTests): + def do_test(self, file_name): + """Execute file test that are known to fail in shutil.rmtree.""" + if not utils.is_windows(): + print("Skipping ShutilWindowsRmtreeFileTests on non-Windows platforms") + return + temp_dir = tempfile.mkdtemp(dir=self.temp_dir_root) + os.chdir(temp_dir) + create_file(file_name) + os.chdir(self.temp_dir_root) + with self.assertRaises(WindowsError): # pylint: disable=undefined-variable + shutil.rmtree(temp_dir) + + def test_ascii(self): + pass + + def test_unicode(self): + pass diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 5763c94e144..37e6690e7c2 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -10688,6 +10688,7 @@ buildvariants: - name: compile_all_run_unittests_TG distros: - ubuntu1604-build + - name: buildscripts_test ########################################### # SUSE buildvariants # |