From 02184ed1daa077d6d30f5799c44eeff26d85b429 Mon Sep 17 00:00:00 2001 From: Jonathan Abrahams Date: Mon, 12 Feb 2018 12:03:55 -0500 Subject: SERVER-33144 Support archiving data files in Evergreen on test failure - Windows (cherry picked from commit 49c2e3abc780316a494a188203a0f827ebf89a61) --- buildscripts/resmoke.py | 1 - buildscripts/resmokelib/utils/archival.py | 73 +++++++++++++++++++++---------- 2 files changed, 51 insertions(+), 23 deletions(-) diff --git a/buildscripts/resmoke.py b/buildscripts/resmoke.py index 3fa24bd6553..cee9ce4e391 100755 --- a/buildscripts/resmoke.py +++ b/buildscripts/resmoke.py @@ -58,7 +58,6 @@ def _execute_suite(suite, logging_config): if resmokelib.config.ARCHIVE_FILE: archive = resmokelib.utils.archival.Archival( archival_json_file=resmokelib.config.ARCHIVE_FILE, - execution=resmokelib.config.EVERGREEN_EXECUTION, limit_size_mb=resmokelib.config.ARCHIVE_LIMIT_MB, limit_files=resmokelib.config.ARCHIVE_LIMIT_TESTS, logger=logger) diff --git a/buildscripts/resmokelib/utils/archival.py b/buildscripts/resmokelib/utils/archival.py index 9d31e053846..5d3545159d3 100644 --- a/buildscripts/resmokelib/utils/archival.py +++ b/buildscripts/resmokelib/utils/archival.py @@ -17,6 +17,9 @@ import time _IS_WINDOWS = sys.platform == "win32" or sys.platform == "cygwin" +if _IS_WINDOWS: + import ctypes + UploadArgs = collections.namedtuple( "UploadArgs", ["archival_file", @@ -65,8 +68,30 @@ def directory_size(directory): def free_space(path): """ Return file system free space (in bytes) for 'path'. """ - stat = os.statvfs(path) - return stat.f_bavail * stat.f_bsize + if _IS_WINDOWS: + dirname = os.path.dirname(path) + free_bytes = ctypes.c_ulonglong(0) + ctypes.windll.kernel32.GetDiskFreeSpaceExW( + ctypes.c_wchar_p(dirname), None, None, ctypes.pointer(free_bytes)) + return free_bytes.value + else: + stat = os.statvfs(path) + return stat.f_bavail * stat.f_bsize + + +def remove_file(file_name): + """ Attempts to remove file. Returns status and message. """ + try: + # File descriptors, on Windows, are inherited by all subprocesses and file removal may fail + # because the file is still open. + # See https://www.python.org/dev/peps/pep-0446/#issues-with-inheritable-file-descriptors + os.remove(file_name) + status = 0 + message = "Successfully deleted file {}".format(file_name) + except Exception as err: + status = 1 + message = "Error deleting file {}: {}".format(file_name, err) + return status, message class Archival(object): @@ -75,7 +100,6 @@ class Archival(object): def __init__(self, logger, archival_json_file="archive.json", - execution=0, limit_size_mb=0, limit_files=0, s3_client=None): @@ -96,7 +120,7 @@ class Archival(object): self._archive_file_queue = Queue.Queue() self._archive_file_worker = threading.Thread( target=self._update_archive_file_wkr, - args=(self._archive_file_queue, execution, logger), + args=(self._archive_file_queue, logger), name="archive_file_worker") self._archive_file_worker.setDaemon(True) self._archive_file_worker.start() @@ -131,10 +155,6 @@ class Archival(object): Returns status and message, where message contains information if status is non-0. """ - # TODO: Support archival on Windows (SERVER-33144). - if _IS_WINDOWS: - return 1, "Archival not supported on Windows" - start_time = time.time() with self._lock: if not input_files: @@ -153,31 +173,33 @@ class Archival(object): s3_bucket, s3_path) - self.num_files += 1 + if status == 0: + self.num_files += 1 self.size_mb += file_size_mb self.archive_time += time.time() - start_time return status, message @staticmethod - def _update_archive_file_wkr(queue, execution, logger): + def _update_archive_file_wkr(queue, logger): """ Worker thread: Update the archival JSON file from 'queue'. """ - archival_json = {"files": [], "execution": execution} + archival_json = [] while True: archive_args = queue.get() # Exit worker thread when sentinel is received. if archive_args is None: queue.task_done() break - logger.debug("Updating archive file %s", archive_args.archival_file) archival_record = { "name": archive_args.display_name, "link": archive_args.remote_file, "visibility": "private" } - archival_json["files"].append(archival_record) + logger.debug( + "Updating archive file %s with %s", archive_args.archival_file, archival_record) + archival_json.append(archival_record) with open(archive_args.archival_file, "w") as archival_fh: - json.dump([archival_json], archival_fh) + json.dump(archival_json, archival_fh) queue.task_done() @staticmethod @@ -191,7 +213,7 @@ class Archival(object): archive_file_queue.put(None) break extra_args = {"ContentType": upload_args.content_type, "ACL": "public-read"} - logger.debug("Uploading to S3 %s to %s %s", + logger.debug("Uploading to S3 %s to bucket %s path %s", upload_args.local_file, upload_args.s3_bucket, upload_args.s3_path) @@ -210,10 +232,9 @@ class Archival(object): logger.exception("Upload to S3 error %s", err) if upload_args.delete_file: - try: - os.remove(upload_args.local_file) - except Exception as err: - logger.exception("Upload to S3 file removal error %s", err) + status, message = remove_file(upload_args.local_file) + if status: + logger.error("Upload to S3 delete file error %s", message) remote_file = "https://s3.amazonaws.com/{}/{}".format( upload_args.s3_bucket, upload_args.s3_path) @@ -246,15 +267,23 @@ class Archival(object): # Check if there is sufficient space for the temporary tgz file. if file_list_size(input_files) > free_space(temp_file): - os.remove(temp_file) + status, message = remove_file(temp_file) + if status: + self.logger.warning("Removing tarfile due to insufficient space - %s", message) return 1, "Insufficient space for {}".format(message), 0 try: with tarfile.open(temp_file, "w:gz") as tar_handle: for input_file in input_files: - tar_handle.add(input_file) + try: + tar_handle.add(input_file) + except (IOError, tarfile.TarError) as err: + message = "{}; Unable to add {} to archive file: {}".format( + message, input_file, err) except (IOError, tarfile.TarError) as err: - os.remove(temp_file) + status, message = remove_file(temp_file) + if status: + self.logger.warning("Removing tarfile due to creation failure - %s", message) return 1, str(err), 0 # Round up the size of the archive. -- cgit v1.2.1