Many files:

  Exit all threads created by innoDB at shutdown


innobase/os/os0file.c:
  Exit all threads created by innoDB at shutdown
innobase/os/os0sync.c:
  Exit all threads created by innoDB at shutdown
innobase/os/os0thread.c:
  Exit all threads created by innoDB at shutdown
innobase/include/os0file.h:
  Exit all threads created by innoDB at shutdown
innobase/include/os0sync.h:
  Exit all threads created by innoDB at shutdown
innobase/include/os0thread.h:
  Exit all threads created by innoDB at shutdown
innobase/log/log0log.c:
  Exit all threads created by innoDB at shutdown
innobase/srv/srv0srv.c:
  Exit all threads created by innoDB at shutdown
innobase/srv/srv0start.c:
  Exit all threads created by innoDB at shutdown

9 files changed, 205 insertions, 22 deletions

diff --git a/innobase/include/os0file.h b/innobase/include/os0file.h
index a7624a90d5e..86f27a2d3eb 100644
--- a/innobase/include/os0file.h
+++ b/innobase/include/os0file.h
@@ -301,6 +301,13 @@ os_aio(
 				are ignored */
 	void*		message2);
 /****************************************************************************
+Wakes up all async i/o threads so that they know to exit themselves in
+shutdown. */
+
+void
+os_aio_wake_all_threads_at_shutdown(void);
+/*=====================================*/
+/****************************************************************************
 Waits until there are no pending writes in os_aio_write_array. There can
 be other, synchronous, pending writes. */
 
diff --git a/innobase/include/os0sync.h b/innobase/include/os0sync.h
index b2d613c4619..d52444d02ec 100644
--- a/innobase/include/os0sync.h
+++ b/innobase/include/os0sync.h
@@ -38,6 +38,13 @@ typedef os_mutex_str_t*		os_mutex_t;
 
 #define OS_SYNC_TIME_EXCEEDED	1
 
+/* Mutex protecting the thread count */
+extern os_mutex_t			os_thread_count_mutex;
+
+/* This is incremented by 1 in os_thread_create and decremented by 1 in
+os_thread_exit */
+extern ulint				os_thread_count;
+
 /*************************************************************
 Creates an event semaphore, i.e., a semaphore which may
 just have two states: signaled and nonsignaled.
@@ -85,7 +92,10 @@ os_event_free(
 /*==========*/
 	os_event_t	event);	/* in: event to free */
 /**************************************************************
-Waits for an event object until it is in the signaled state. */
+Waits for an event object until it is in the signaled state. If
+srv_shutdown_state == SRV_SHUTDOWN_EXIT_THREADS this also exits the
+waiting thread when the event becomes signaled (or immediately if the
+event is already in the signaled state). */
 
 void
 os_event_wait(
diff --git a/innobase/include/os0thread.h b/innobase/include/os0thread.h
index 629cfef23a8..29154a9e7cf 100644
--- a/innobase/include/os0thread.h
+++ b/innobase/include/os0thread.h
@@ -11,6 +11,7 @@ Created 9/8/1995 Heikki Tuuri
 #define os0thread_h
 
 #include "univ.i"
+#include "os0sync.h"
 
 /* Maximum number of threads which can be created in the program;
 this is also the size of the wait slot array for MySQL threads which
@@ -41,7 +42,6 @@ typedef os_thread_t          	os_thread_id_t;	/* In Unix we use the thread
 						the thread */
 #endif
 
-
 /* Define a function pointer type to use in a typecast */
 typedef void* (*os_posix_f_t) (void*);
 
@@ -83,12 +83,13 @@ os_thread_create(
 	os_thread_id_t*		thread_id);	/* out: id of the created
 						thread */
 /*********************************************************************
-A thread calling this function ends its execution. */
+Exits the current thread. */
 
 void
 os_thread_exit(
 /*===========*/
-	ulint	code);	/* in: exit code */
+	void*	exit_value);	/* in: exit value; in Windows this void*
+				is cast as a DWORD */
 /*********************************************************************
 Returns the thread identifier of current thread. */
 
@@ -144,7 +145,6 @@ ulint
 os_thread_get_last_error(void);
 /*==========================*/
 
-
 #ifndef UNIV_NONINL
 #include "os0thread.ic"
 #endif
diff --git a/innobase/log/log0log.c b/innobase/log/log0log.c
index 25cc666e802..e15812e03af 100644
--- a/innobase/log/log0log.c
+++ b/innobase/log/log0log.c
@@ -375,7 +375,7 @@ log_pad_current_log_block(void)
 	log_close();
 	log_release();
 
-	ut_a((ut_dulint_get_low(lsn) % OS_FILE_LOG_BLOCK_SIZE)
+	ut_ad((ut_dulint_get_low(lsn) % OS_FILE_LOG_BLOCK_SIZE)
 						== LOG_BLOCK_HDR_SIZE);
 }
 
@@ -998,6 +998,8 @@ log_group_file_header_flush(
 {
 	byte*	buf;
 	ulint	dest_offset;
+	
+	UT_NOT_USED(type);
 
 	ut_ad(mutex_own(&(log_sys->mutex)));
 
@@ -1068,8 +1070,8 @@ log_group_write_buf(
 	ulint	i;
 	
 	ut_ad(mutex_own(&(log_sys->mutex)));
-	ut_a(len % OS_FILE_LOG_BLOCK_SIZE == 0);
-	ut_a(ut_dulint_get_low(start_lsn) % OS_FILE_LOG_BLOCK_SIZE == 0);
+	ut_ad(len % OS_FILE_LOG_BLOCK_SIZE == 0);
+	ut_ad(ut_dulint_get_low(start_lsn) % OS_FILE_LOG_BLOCK_SIZE == 0);
 
 	if (new_data_offset == 0) {
 		write_header = TRUE;
@@ -2901,10 +2903,9 @@ logs_empty_and_mark_files_at_shutdown(void)
 	dulint	lsn;
 	ulint	arch_log_no;
 
-	if (srv_print_verbose_log)
-	{
-	  ut_print_timestamp(stderr);
-	  fprintf(stderr, "  InnoDB: Starting shutdown...\n");
+	if (srv_print_verbose_log) {
+	        ut_print_timestamp(stderr);
+	        fprintf(stderr, "  InnoDB: Starting shutdown...\n");
 	}
 	/* Wait until the master thread and all other operations are idle: our
 	algorithm only works if the server is idle at shutdown */
@@ -3006,15 +3007,17 @@ loop:
 		goto loop;
 	}
 
+	/* Make some checks that the server really is quiet */
+	ut_a(buf_all_freed());
+	ut_a(0 == ut_dulint_cmp(lsn, log_sys->lsn));
+
 	fil_write_flushed_lsn_to_data_files(lsn, arch_log_no);	
 
 	fil_flush_file_spaces(FIL_TABLESPACE);
 
-	if (srv_print_verbose_log)
-	{
-	  ut_print_timestamp(stderr);
-	  fprintf(stderr, "  InnoDB: Shutdown completed\n");
-	}
+	/* Make some checks that the server really is quiet */
+	ut_a(buf_all_freed());
+	ut_a(0 == ut_dulint_cmp(lsn, log_sys->lsn));
 }
 
 /**********************************************************
diff --git a/innobase/os/os0file.c b/innobase/os/os0file.c
index c7f95d79104..640ffec122f 100644
--- a/innobase/os/os0file.c
+++ b/innobase/os/os0file.c
@@ -1295,7 +1295,6 @@ os_aio_array_create(
 #endif	
 	ut_a(n > 0);
 	ut_a(n_segments > 0);
-	ut_a(n % n_segments == 0);
 
 	array = ut_malloc(sizeof(os_aio_array_t));
 
@@ -1404,6 +1403,50 @@ os_aio_init(
 	pthread_sigmask(SIG_BLOCK, &sigset, NULL); */
 #endif
 }
+
+#ifdef WIN_ASYNC_IO
+/****************************************************************************
+Wakes up all async i/o threads in the array in Windows async i/o at
+shutdown. */
+static
+void
+os_aio_array_wake_win_aio_at_shutdown(
+/*==================================*/
+	os_aio_array_t*	array)	/* in: aio array */
+{
+	ulint	i;
+
+	for (i = 0; i < array->n_slots; i++) {
+
+	        os_event_set(*(array->events + i));
+	}
+}
+#endif
+
+/****************************************************************************
+Wakes up all async i/o threads so that they know to exit themselves in
+shutdown. */
+
+void
+os_aio_wake_all_threads_at_shutdown(void)
+/*=====================================*/
+{
+	ulint	i;
+
+#ifdef WIN_ASYNC_IO
+        /* This code wakes up all ai/o threads in Windows native aio */
+	os_aio_array_wake_win_aio_at_shutdown(os_aio_read_array);
+	os_aio_array_wake_win_aio_at_shutdown(os_aio_write_array);
+	os_aio_array_wake_win_aio_at_shutdown(os_aio_ibuf_array);
+	os_aio_array_wake_win_aio_at_shutdown(os_aio_log_array);
+#endif
+	/* This loop wakes up all simulated ai/o threads */
+
+	for (i = 0; i < os_aio_n_segments; i++) {
+	    	
+		os_event_set(os_aio_segment_wait_events[i]);
+	}	
+}
 				
 /****************************************************************************
 Waits until there are no pending writes in os_aio_write_array. There can
diff --git a/innobase/os/os0sync.c b/innobase/os/os0sync.c
index 407b280f805..abcfa254710 100644
--- a/innobase/os/os0sync.c
+++ b/innobase/os/os0sync.c
@@ -17,6 +17,7 @@ Created 9/6/1995 Heikki Tuuri
 #endif
 
 #include "ut0mem.h"
+#include "srv0start.h"
 
 /* Type definition for an operating system mutex struct */
 struct os_mutex_struct{ 
@@ -26,9 +27,16 @@ struct os_mutex_struct{
 				recursively lock the mutex: we
 				do not assume that the OS mutex
 				supports recursive locking, though
-				NT seems to do that */				
+				NT seems to do that */
 };
 
+/* Mutex protecting the thread count */
+os_mutex_t			os_thread_count_mutex;
+
+/* This is incremented by 1 in os_thread_create and decremented by 1 in
+os_thread_exit */
+ulint				os_thread_count		= 0;
+
 /*************************************************************
 Creates an event semaphore, i.e., a semaphore which may
 just have two states: signaled and nonsignaled.
@@ -190,7 +198,10 @@ os_event_free(
 }
 
 /**************************************************************
-Waits for an event object until it is in the signaled state. */
+Waits for an event object until it is in the signaled state. If
+srv_shutdown_state == SRV_SHUTDOWN_EXIT_THREADS this also exits the
+waiting thread when the event becomes signaled (or immediately if the
+event is already in the signaled state). */
 
 void
 os_event_wait(
@@ -206,12 +217,20 @@ os_event_wait(
 	err = WaitForSingleObject(event, INFINITE);
 
 	ut_a(err == WAIT_OBJECT_0);
+
+	if (srv_shutdown_state == SRV_SHUTDOWN_EXIT_THREADS) {
+	        os_thread_exit(NULL);
+	}
 #else
 	os_fast_mutex_lock(&(event->os_mutex));
 loop:
 	if (event->is_set == TRUE) {
 		os_fast_mutex_unlock(&(event->os_mutex));
 
+		if (srv_shutdown_state == SRV_SHUTDOWN_EXIT_THREADS) {
+
+		        os_thread_exit(NULL);
+		}
 		/* Ok, we may return */
 
 		return;
@@ -299,6 +318,10 @@ os_event_wait_multiple(
 	ut_a(index >= WAIT_OBJECT_0);
 	ut_a(index < WAIT_OBJECT_0 + n);
 
+	if (srv_shutdown_state == SRV_SHUTDOWN_EXIT_THREADS) {
+	        os_thread_exit(NULL);
+	}
+
 	return(index - WAIT_OBJECT_0);
 #else
 	ut_a(n == 0);
diff --git a/innobase/os/os0thread.c b/innobase/os/os0thread.c
index b0076921e43..a68f6a3b8bc 100644
--- a/innobase/os/os0thread.c
+++ b/innobase/os/os0thread.c
@@ -1,6 +1,5 @@
 /******************************************************
-The interface to the operating system
-process and thread control primitives
+The interface to the operating system thread control primitives
 
 (c) 1995 Innobase Oy
 
@@ -102,6 +101,10 @@ os_thread_create(
 	os_thread_t	thread;
 	ulint           win_thread_id;
 
+	os_mutex_enter(os_thread_count_mutex);
+	os_thread_count++;
+	os_mutex_exit(os_thread_count_mutex);
+
 	thread = CreateThread(NULL,	/* no security attributes */
 				0,	/* default size stack */
 				(LPTHREAD_START_ROUTINE)start_f,
@@ -144,6 +147,9 @@ os_thread_create(
 		 exit(1);
 	}
 #endif
+	os_mutex_enter(os_thread_count_mutex);
+	os_thread_count++;
+	os_mutex_exit(os_thread_count_mutex);
 
 #if defined(UNIV_HOTBACKUP) && defined(UNIV_HPUX10)
 	ret = pthread_create(&pthread, pthread_attr_default, start_f, arg);
@@ -171,6 +177,26 @@ os_thread_create(
 }
 
 /*********************************************************************
+Exits the current thread. */
+
+void
+os_thread_exit(
+/*===========*/
+	void*	exit_value)	/* in: exit value; in Windows this void*
+				is cast as a DWORD */
+{
+	os_mutex_enter(os_thread_count_mutex);
+	os_thread_count--;
+	os_mutex_exit(os_thread_count_mutex);
+
+#ifdef __WIN__
+        ExitThread((DWORD)exit_value);
+#else
+	pthread_exit(exit_value);
+#endif
+}
+
+/*********************************************************************
 Returns handle to the current thread. */
 
 os_thread_t
diff --git a/innobase/srv/srv0srv.c b/innobase/srv/srv0srv.c
index ef50a4ca261..a44bc0147e4 100644
--- a/innobase/srv/srv0srv.c
+++ b/innobase/srv/srv0srv.c
@@ -1702,6 +1702,8 @@ void
 srv_general_init(void)
 /*==================*/
 {
+	os_thread_count_mutex = os_mutex_create(NULL);
+
 	sync_init();
 	mem_init(srv_mem_pool_size);
 	thr_local_init();
@@ -1720,6 +1722,8 @@ srv_general_free(void)
 /*==================*/
 {
   sync_close();
+
+  os_mutex_free(os_thread_count_mutex);
 }
 #endif /* __NETWARE__ */
 
@@ -2700,6 +2704,8 @@ loop:
 
 	srv_error_monitor_active = FALSE;
 
+	os_thread_exit(NULL);
+
 #ifndef __WIN__
         return(NULL);
 #else
@@ -3139,6 +3145,13 @@ suspend_thread:
 
 	os_event_wait(event);
 
+	if (srv_shutdown_state == SRV_SHUTDOWN_EXIT_THREADS) {
+	        /* This is only extra safety, the thread should exit
+		already when the event wait ends */
+
+	        os_thread_exit(NULL);
+	}
+
 	/* When there is user activity, InnoDB will set the event and the main
 	thread goes back to loop: */
 
diff --git a/innobase/srv/srv0start.c b/innobase/srv/srv0start.c
index d47af68d663..3d98e116905 100644
--- a/innobase/srv/srv0start.c
+++ b/innobase/srv/srv0start.c
@@ -1470,6 +1470,8 @@ innobase_shutdown_for_mysql(void)
 /*=============================*/
 				/* out: DB_SUCCESS or error code */
 {
+	ulint   i;
+
         if (!srv_was_started) {
 	  	if (srv_is_being_started) {
 	    		ut_print_timestamp(stderr);
@@ -1494,6 +1496,58 @@ innobase_shutdown_for_mysql(void)
 		srv_conc_n_threads);
 	}
 
+	/* Now we will exit all threads InnoDB created */
+
+	srv_shutdown_state = SRV_SHUTDOWN_EXIT_THREADS;
+
+	/* All threads end up waiting for certain events. Put those events
+	to the signaled state. Then the threads will exit themselves in
+	os_thread_event_wait(). */
+
+	for (i = 0; i < 1000; i++) {
+	        /* NOTE: IF YOU CREATE THREADS IN INNODB, YOU MUST EXIT THEM
+	        HERE OR EARLIER */
+		
+		/* 1. Let the lock timeout thread exit */
+		os_event_set(srv_lock_timeout_thread_event);		
+
+		/* 2. srv error monitor thread exits automatically, no need
+		to do anything here */
+
+		/* 3. We wake the master thread so that it exits */
+		srv_wake_master_thread();
+
+		/* 4. Exit the i/o threads */
+
+		os_aio_wake_all_threads_at_shutdown();
+
+		os_mutex_enter(os_thread_count_mutex);
+
+		if (os_thread_count == 0) {
+		        /* All the threads have exited or are just exiting;
+			NOTE that the threads may not have completed their
+			exit yet. Should we use pthread_join() to make sure
+			they have exited? Now we just sleep 0.1 seconds and
+			hope that is enough! */
+
+			os_mutex_exit(os_thread_count_mutex);
+
+			os_thread_sleep(100000);
+
+			break;
+		}
+
+		os_mutex_exit(os_thread_count_mutex);
+
+		os_thread_sleep(100000);
+	}
+
+	if (i == 1000) {
+	        fprintf(stderr,
+"InnoDB: Warning: %lu threads created by InnoDB had not exited at shutdown!\n",
+		      os_thread_count);
+	}
+
 #if defined(__NETWARE__) || defined(SAFE_MUTEX_DETECT_DESTROY)
 	/*
 	  TODO: Fix this temporary solution
@@ -1518,6 +1572,10 @@ innobase_shutdown_for_mysql(void)
         /* NetWare requires this free */
         ut_free_all_mem();
 #endif 
+	if (srv_print_verbose_log) {
+	        ut_print_timestamp(stderr);
+	        fprintf(stderr, "  InnoDB: Shutdown completed\n");
+	}
 
 	return((int) DB_SUCCESS);
 }