Applying InnoDB snapshot 5.1-ss6242, part 2. Fixes BUG#3139

1. BUG#3139 - Mysql crashes: "windows error 995" after several 
              selects on a large DB

Detailed revision comments:

r6154 | calvin | 2009-11-11 02:51:17 +0200 (Wed, 11 Nov 2009) | 17 lines
branches/5.1: fix bug#3139: Mysql crashes: 'windows error 995'
after several selects on a large DB

During stress environment, Windows AIO may fail with error code
ERROR_OPERATION_ABORTED. InnoDB does not handle the error, rather
crashes. The cause of the error is unknown, but likely due to
faulty hardware or driver.

This patch introduces a new error code OS_FILE_OPERATION_ABORTED,
which maps to Windows ERROR_OPERATION_ABORTED (995). When the error
is detected during AIO, the InnoDB will issue a synchronous retry
(read/write).

This patch has been extensively tested by MySQL support.

Approved by: Marko
rb://196

2 files changed, 55 insertions, 1 deletions

diff --git a/storage/innobase/include/os0file.h b/storage/innobase/include/os0file.h
index 9eb44d3f4a8..e09e1e00408 100644
--- a/storage/innobase/include/os0file.h
+++ b/storage/innobase/include/os0file.h
@@ -96,6 +96,8 @@ log. */
 						to become available again */
 #define	OS_FILE_SHARING_VIOLATION	76
 #define	OS_FILE_ERROR_NOT_SPECIFIED	77
+						/* 78 is used in the plugin */
+#define	OS_FILE_OPERATION_ABORTED	79
 
 /* Types for aio operations */
 #define OS_FILE_READ	10
diff --git a/storage/innobase/os/os0file.c b/storage/innobase/os/os0file.c
index 8fd959512c1..085f62daacc 100644
--- a/storage/innobase/os/os0file.c
+++ b/storage/innobase/os/os0file.c
@@ -257,6 +257,13 @@ os_file_get_last_error(
 				" software or another instance\n"
 				"InnoDB: of MySQL."
 				" Please close it to get rid of this error.\n");
+		} else if (err == ERROR_OPERATION_ABORTED) {
+			fprintf(stderr,
+				"InnoDB: The error means that the I/O"
+				" operation has been aborted\n"
+				"InnoDB: because of either a thread exit"
+				" or an application request.\n"
+				"InnoDB: Retry attempt is made.\n");
 		} else {
 			fprintf(stderr,
 				"InnoDB: Some operating system error numbers"
@@ -278,6 +285,8 @@ os_file_get_last_error(
 	} else if (err == ERROR_SHARING_VIOLATION
 		   || err == ERROR_LOCK_VIOLATION) {
 		return(OS_FILE_SHARING_VIOLATION);
+	} else if (err == ERROR_OPERATION_ABORTED) {
+		return(OS_FILE_OPERATION_ABORTED);
 	} else {
 		return(100 + err);
 	}
@@ -402,6 +411,10 @@ os_file_handle_error_cond_exit(
 
 		os_thread_sleep(10000000);  /* 10 sec */
 		return(TRUE);
+	} else if (err == OS_FILE_OPERATION_ABORTED) {
+
+		os_thread_sleep(100000);	/* 100 ms */
+		return(TRUE);
 	} else {
 		if (name) {
 			fprintf(stderr, "InnoDB: File name %s\n", name);
@@ -3692,6 +3705,7 @@ os_aio_windows_handle(
 	ibool		ret_val;
 	BOOL		ret;
 	DWORD		len;
+	BOOL		retry		= FALSE;
 
 	if (segment == ULINT_UNDEFINED) {
 		array = os_aio_sync_array;
@@ -3745,14 +3759,52 @@ os_aio_windows_handle(
 			ut_a(TRUE == os_file_flush(slot->file));
 		}
 # endif /* UNIV_DO_FLUSH */
+	} else if (os_file_handle_error(slot->name, "Windows aio")) {
+
+		retry = TRUE;
 	} else {
-		os_file_handle_error(slot->name, "Windows aio");
 
 		ret_val = FALSE;
 	}
 
 	os_mutex_exit(array->mutex);
 
+	if (retry) {
+		/* retry failed read/write operation synchronously.
+		No need to hold array->mutex. */
+
+		switch (slot->type) {
+		case OS_FILE_WRITE:
+			ret = WriteFile(slot->file, slot->buf,
+					slot->len, &len,
+					&(slot->control));
+
+			break;
+		case OS_FILE_READ:
+			ret = ReadFile(slot->file, slot->buf,
+				       slot->len, &len,
+				       &(slot->control));
+
+			break;
+		default:
+			ut_error;
+		}
+
+		if (!ret && GetLastError() == ERROR_IO_PENDING) {
+			/* aio was queued successfully!
+			We want a synchronous i/o operation on a
+			file where we also use async i/o: in Windows
+			we must use the same wait mechanism as for
+			async i/o */
+
+			ret = GetOverlappedResult(slot->file,
+						  &(slot->control),
+						  &len, TRUE);
+		}
+
+		ret_val = ret && len == slot->len;
+	}
+
 	os_aio_array_free_slot(array, slot);
 
 	return(ret_val);