Do a hot backup to test recovery, otherwise the copy can race with log archiving removing a log file. Switch a few tests to use the "standard" backup, use shutil.rmtree instead of our own version.

6 files changed, 30 insertions, 40 deletions

diff --git a/test/suite/test_backup01.py b/test/suite/test_backup01.py
index 656e981e337..334c7e6670f 100644
--- a/test/suite/test_backup01.py
+++ b/test/suite/test_backup01.py
@@ -111,7 +111,7 @@ class test_backup(wttest.WiredTigerTestCase, suite_subprocess):
     # Backup a set of chosen tables/files using the wt backup command.
     def backup_table(self, l):
         # Remove any previous backup directories.
-        shutil.rmtree(self.dir, True)
+        shutil.rmtree(self.dir, ignore_errors=True)
         os.mkdir(self.dir)
 
         # Build a command line of objects to back up and run wt.
diff --git a/test/suite/test_bulk02.py b/test/suite/test_bulk02.py
index 46c8ec17244..a1f90c88925 100644
--- a/test/suite/test_bulk02.py
+++ b/test/suite/test_bulk02.py
@@ -100,20 +100,9 @@ class test_bulkload_backup(wttest.WiredTigerTestCase, suite_subprocess):
 
     # Backup a set of chosen tables/files using the wt backup command.
     # The only files are bulk-load files, so they shouldn't be copied.
-    def backup(self, session):
-        # Create a backup directory.
+    def check_backup(self, session):
         backupdir = 'backup.dir'
-        os.mkdir(backupdir)
-
-        # Open up the backup cursor, and copy the files.
-        cursor = session.open_cursor('backup:', None, None)
-        while True:
-            ret = cursor.next()
-            if ret != 0:
-                break
-            shutil.copy(cursor.get_key(), backupdir)
-        self.assertEqual(ret, wiredtiger.WT_NOTFOUND)
-        cursor.close()
+        self.backup(backupdir, session)
 
         # Open the target directory, and confirm the object has no contents.
         conn = wiredtiger.wiredtiger_open(backupdir)
@@ -141,9 +130,9 @@ class test_bulkload_backup(wttest.WiredTigerTestCase, suite_subprocess):
         # Test with the same and different sessions than the bulk-get call,
         # test both the database handle and session handle caches.
         if self.session_type == 'same':
-            self.backup(self.session)
+            self.check_backup(self.session)
         else:
-            self.backup(self.conn.open_session())
+            self.check_backup(self.conn.open_session())
 
 
 if __name__ == '__main__':
diff --git a/test/suite/test_txn02.py b/test/suite/test_txn02.py
index 96c1d52ff65..75733fc07f7 100644
--- a/test/suite/test_txn02.py
+++ b/test/suite/test_txn02.py
@@ -125,8 +125,7 @@ class test_txn02(wttest.WiredTigerTestCase, suite_subprocess):
 
         # Opening a clone of the database home directory should run
         # recovery and see the committed results.
-        wttest.removeAll(self.backup_dir)
-        shutil.copytree(self.home, self.backup_dir)
+        self.backup(self.backup_dir)
         backup_conn_params = 'log=(enabled,file_max=%s)' % self.logmax
         backup_conn = wiredtiger_open(self.backup_dir, backup_conn_params)
         try:
@@ -135,8 +134,7 @@ class test_txn02(wttest.WiredTigerTestCase, suite_subprocess):
             backup_conn.close()
 
     def check_log(self, committed):
-        wttest.removeAll(self.backup_dir)
-        shutil.copytree(self.home, self.backup_dir)
+        self.backup(self.backup_dir)
         #
         # Open and close the backup connection a few times to force
         # repeated recovery and log archiving even if later recoveries
diff --git a/test/suite/test_txn04.py b/test/suite/test_txn04.py
index 74c14926f07..064e8579b1a 100644
--- a/test/suite/test_txn04.py
+++ b/test/suite/test_txn04.py
@@ -29,7 +29,7 @@
 #   Transactions: hot backup and recovery
 #
 
-import os
+import shutil, os
 from suite_subprocess import suite_subprocess
 from wiredtiger import wiredtiger_open
 from wtscenario import multiply_scenarios, number_scenarios
@@ -110,7 +110,7 @@ class test_txn04(wttest.WiredTigerTestCase, suite_subprocess):
         if backup_uri != None:
             cmd += '-t ' + backup_uri + ' '
         else:
-            wttest.removeAll(self.backup_dir)
+            shutil.rmtree(self.backup_dir, ignore_errors=True)
             os.mkdir(self.backup_dir)
 
         cmd += self.backup_dir
diff --git a/test/suite/test_txn05.py b/test/suite/test_txn05.py
index e017aab9cb0..a4b8dc3c3d6 100644
--- a/test/suite/test_txn05.py
+++ b/test/suite/test_txn05.py
@@ -105,8 +105,7 @@ class test_txn05(wttest.WiredTigerTestCase, suite_subprocess):
 
         # Opening a clone of the database home directory should run
         # recovery and see the committed results.
-        wttest.removeAll(self.backup_dir)
-        shutil.copytree(self.home, self.backup_dir)
+        self.backup(self.backup_dir)
         backup_conn_params = 'log=(enabled,file_max=%s)' % self.logmax
         backup_conn = wiredtiger_open(self.backup_dir, backup_conn_params)
         try:
@@ -115,8 +114,7 @@ class test_txn05(wttest.WiredTigerTestCase, suite_subprocess):
             backup_conn.close()
 
     def check_log(self, committed):
-        wttest.removeAll(self.backup_dir)
-        shutil.copytree(self.home, self.backup_dir)
+        self.backup(self.backup_dir)
         #
         # Open and close the backup connection a few times to force
         # repeated recovery and log archiving even if later recoveries
diff --git a/test/suite/wttest.py b/test/suite/wttest.py
index 22eb808403a..ec156e15f9a 100644
--- a/test/suite/wttest.py
+++ b/test/suite/wttest.py
@@ -35,19 +35,10 @@ try:
 except ImportError:
     import unittest
 
-import sys, time, traceback, os, re
-import wiredtiger
 from contextlib import contextmanager
+import os, re, shutil, sys, time, traceback
 
-def removeAll(top):
-    if not os.path.isdir(top):
-        return
-    for root, dirs, files in os.walk(top, topdown=False):
-        for name in files:
-            os.remove(os.path.join(root, name))
-        for name in dirs:
-            os.rmdir(os.path.join(root, name))
-    os.rmdir(top)
+import wiredtiger
 
 def shortenWithEllipsis(s, maxlen):
     if len(s) > maxlen:
@@ -155,7 +146,7 @@ class WiredTigerTestCase(unittest.TestCase):
             d = 'WT_TEST.' + time.strftime('%Y%m%d-%H%M%S', time.localtime())
         else:
             d = 'WT_TEST'
-        removeAll(d)
+        shutil.rmtree(d, ignore_errors=True)
         os.makedirs(d)
         WiredTigerTestCase._parentTestdir = d
         WiredTigerTestCase._resultfile = open(os.path.join(d, 'results.txt'), "w", 0)  # unbuffered
@@ -240,7 +231,7 @@ class WiredTigerTestCase(unittest.TestCase):
         if WiredTigerTestCase._verbose > 2:
             self.prhead('started in ' + self.testdir, True)
         self.origcwd = os.getcwd()
-        removeAll(self.testdir)
+        shutil.rmtree(self.testdir, ignore_errors=True)
         if os.path.exists(self.testdir):
             raise Exception(self.testdir + ": cannot remove directory")
         os.makedirs(self.testdir)
@@ -277,7 +268,7 @@ class WiredTigerTestCase(unittest.TestCase):
 
         # Clean up unless there's a failure
         if passed and not WiredTigerTestCase._preserveFiles:
-            removeAll(self.testdir)
+            shutil.rmtree(self.testdir, ignore_errors=True)
         else:
             self.pr('preserving directory ' + self.testdir)
 
@@ -289,6 +280,20 @@ class WiredTigerTestCase(unittest.TestCase):
         if WiredTigerTestCase._verbose > 2:
             self.prhead('TEST COMPLETED')
 
+    def backup(self, backup_dir, session=None):
+        if session is None:
+            session = self.session
+        shutil.rmtree(backup_dir, ignore_errors=True)
+        os.mkdir(backup_dir)
+        bkp_cursor = session.open_cursor('backup:', None, None)
+        while True:
+            ret = bkp_cursor.next()
+            if ret != 0:
+                break
+            shutil.copy(bkp_cursor.get_key(), backup_dir)
+        self.assertEqual(ret, wiredtiger.WT_NOTFOUND)
+        bkp_cursor.close()
+
     @contextmanager
     def expectedStdout(self, expect):
         self.captureout.check(self)