SERVER-34374 Wrap shutil.rmtree() in resmoke to handle path names which may contain non-ASCII characters

(cherry picked from commit d6837a12b3586b0738dcd4214951a1d6f6b1415e)

5 files changed, 154 insertions, 5 deletions

diff --git a/buildscripts/resmokelib/testing/fixtures/standalone.py b/buildscripts/resmokelib/testing/fixtures/standalone.py
index 7e44aa2db72..98e18241df8 100644
--- a/buildscripts/resmokelib/testing/fixtures/standalone.py
+++ b/buildscripts/resmokelib/testing/fixtures/standalone.py
@@ -6,7 +6,6 @@ from __future__ import absolute_import
 
 import os
 import os.path
-import shutil
 import socket
 import time
 
@@ -94,7 +93,7 @@ class MongoDFixture(interface.Fixture):
 
     def setup(self):
         if not self.preserve_dbpath:
-            shutil.rmtree(self._dbpath, ignore_errors=True)
+            utils.rmtree(self._dbpath, ignore_errors=True)
 
         try:
             os.makedirs(self._dbpath)
diff --git a/buildscripts/resmokelib/testing/testcases.py b/buildscripts/resmokelib/testing/testcases.py
index 0f6dfb8b447..b2d32c1eb80 100644
--- a/buildscripts/resmokelib/testing/testcases.py
+++ b/buildscripts/resmokelib/testing/testcases.py
@@ -6,7 +6,6 @@ from __future__ import absolute_import
 
 import os
 import os.path
-import shutil
 import unittest
 
 from .. import config
@@ -222,7 +221,7 @@ class DBTestCase(TestCase):
         dbpath = os.path.join(dbpath_prefix, "job%d" % (self.fixture.job_num), "unittest")
         self.dbtest_options["dbpath"] = dbpath
 
-        shutil.rmtree(dbpath, ignore_errors=True)
+        utils.rmtree(dbpath, ignore_errors=True)
 
         try:
             os.makedirs(dbpath)
@@ -318,7 +317,7 @@ class JSTestCase(TestCase):
         global_vars["TestData"] = test_data
         self.shell_options["global_vars"] = global_vars
 
-        shutil.rmtree(data_dir, ignore_errors=True)
+        utils.rmtree(data_dir, ignore_errors=True)
 
         try:
             os.makedirs(data_dir)
diff --git a/buildscripts/resmokelib/utils/__init__.py b/buildscripts/resmokelib/utils/__init__.py
index df387cc3323..7d84bb4459d 100644
--- a/buildscripts/resmokelib/utils/__init__.py
+++ b/buildscripts/resmokelib/utils/__init__.py
@@ -5,6 +5,8 @@ Helper functions.
 from __future__ import absolute_import
 
 import os.path
+import shutil
+import sys
 
 import pymongo
 import yaml
@@ -14,6 +16,28 @@ def default_if_none(value, default):
     return value if value is not None else default
 
 
+def rmtree(path, **kwargs):
+    """Wrap shutil.rmtreee.
+
+    Use a UTF-8 unicode path if Windows.
+    See https://bugs.python.org/issue24672, where shutil.rmtree can fail with UTF-8.
+    Use a bytes path to rmtree, otherwise.
+    See https://github.com/pypa/setuptools/issues/706.
+    """
+    if is_windows():
+        if not isinstance(path, unicode):
+            path = unicode(path, "utf-8")
+    else:
+        if isinstance(path, unicode):
+            path = path.encode("utf-8")
+    shutil.rmtree(path, **kwargs)
+
+
+def is_windows():
+    """Return True if Windows."""
+    return sys.platform.startswith("win32") or sys.platform.startswith("cygwin")
+
+
 def is_string_list(lst):
     """
     Returns true if 'lst' is a list of strings, and false otherwise.
diff --git a/buildscripts/tests/resmokelib/utils/__init__.py b/buildscripts/tests/resmokelib/utils/__init__.py
new file mode 100644
index 00000000000..4b7a2bb941b
--- /dev/null
+++ b/buildscripts/tests/resmokelib/utils/__init__.py
@@ -0,0 +1 @@
+"""Empty."""
diff --git a/buildscripts/tests/resmokelib/utils/test_rmtree.py b/buildscripts/tests/resmokelib/utils/test_rmtree.py
new file mode 100644
index 00000000000..1908395766c
--- /dev/null
+++ b/buildscripts/tests/resmokelib/utils/test_rmtree.py
@@ -0,0 +1,126 @@
+# -*- coding: utf-8 -*-
+""" Unit tests for utils.rmtree. """
+
+from __future__ import absolute_import
+from __future__ import print_function
+
+import os
+import shutil
+import sys
+import tempfile
+import unittest
+
+from buildscripts.resmokelib import utils
+
+# pylint: disable=missing-docstring,protected-access
+
+
+def rmtree(dir_root):
+    """Invoke utils.rmtree(dir_root) and return True if removed."""
+    utils.rmtree(dir_root)
+    return not os.path.exists(dir_root)
+
+
+def create_file(path):
+    """Create file named 'path'."""
+    with open(path, "w") as fh:
+        fh.write("")
+
+
+def ascii_filesystemencoding():
+    """Return True if the file system encoding is type ASCII.
+
+    See https://www.iana.org/assignments/character-sets/character-sets.xhtml.
+    """
+    encoding = sys.getfilesystemencoding()
+    return encoding.startswith("ANSI_X3.4") or encoding == "US-ASCII"
+
+
+def string_for_ascii_filesystem_encoding(path):
+    """Return encoded string type for unicode on ASCII file system encoding.
+
+    Some file system encodings are set to ASCII if LANG=C or LC_ALL=C is specified.
+    """
+    if ascii_filesystemencoding() and isinstance(path, unicode):
+        return path.encode("utf-8")
+    return path
+
+
+class RmtreeTestCase(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.temp_dir_root = tempfile.mkdtemp()
+        cls.cwd = os.getcwd()
+
+    @classmethod
+    def tearDownClass(cls):
+        os.chdir(cls.cwd)
+        shutil.rmtree(cls.temp_dir_root, ignore_errors=True)
+
+    def do_test(self, name):
+        pass
+
+    def test_ascii(self):
+        # ASCII name
+        self.do_test("ascii")
+
+    def test_unicode(self):
+        # Unicode name
+        self.do_test(u"unicode")
+
+    def test_greek(self):
+        # Name with Greek
+        self.do_test(string_for_ascii_filesystem_encoding(u"ελληνικά"))
+
+    def test_japanese(self):
+        # Name with Japanese
+        self.do_test(string_for_ascii_filesystem_encoding(u"会社案"))
+
+
+class RmtreeFileTests(RmtreeTestCase):
+    def do_test(self, file_name):  # pylint: disable=arguments-differ
+        """Execute file test."""
+        temp_dir = tempfile.mkdtemp(dir=self.temp_dir_root)
+        os.chdir(temp_dir)
+        create_file(file_name)
+        os.chdir(self.temp_dir_root)
+        self.assertTrue(rmtree(temp_dir))
+
+
+class RmtreeDirectoryTests(RmtreeTestCase):
+    def do_test(self, dir_name):  # pylint: disable=arguments-differ
+        """Execute directory test."""
+        os.chdir(self.temp_dir_root)
+        os.mkdir(dir_name)
+        self.assertTrue(rmtree(dir_name))
+
+
+class RmtreeDirectoryWithNonAsciiTests(RmtreeTestCase):
+    def do_test(self, name):
+        """Execute directory with non-ASCII file test."""
+        os.chdir(self.temp_dir_root)
+        os.mkdir(name)
+        os.chdir(name)
+        create_file(name)
+        os.chdir(self.temp_dir_root)
+        self.assertTrue(rmtree(name))
+
+
+class ShutilWindowsRmtreeFileTests(RmtreeFileTests):
+    def do_test(self, file_name):
+        """Execute file test that are known to fail in shutil.rmtree."""
+        if not utils.is_windows():
+            print("Skipping ShutilWindowsRmtreeFileTests on non-Windows platforms")
+            return
+        temp_dir = tempfile.mkdtemp(dir=self.temp_dir_root)
+        os.chdir(temp_dir)
+        create_file(file_name)
+        os.chdir(self.temp_dir_root)
+        with self.assertRaises(WindowsError):  # pylint: disable=undefined-variable
+            shutil.rmtree(temp_dir)
+
+    def test_ascii(self):
+        pass
+
+    def test_unicode(self):
+        pass