Give better error from reconnect()

Fixed hang in start_slave_threads() when thread dies quickly.


Docs/manual.texi:
  Changelog
client/mysqltest.c:
  Indentation cleanup
  More DBUG info
libmysql/libmysql.c:
  More DBUG info
  Give better error from reconnect()
mysql-test/r/rpl_rotate_logs.result:
  Update results
mysql-test/t/rpl_log_pos.test:
  Fix for fast machines
mysql-test/t/rpl_rotate_logs.test:
  Updated test to be more portable
scripts/mysql_zap.sh:
  Update for MacOSX
sql/mini_client.cc:
  Better error messages from reconnect.
  Indentation cleanups
sql/slave.cc:
  Fixed hang in start_slave_threads() when thread dies quickly.
sql/slave.h:
  Fixed hang in start_slave_threads() when thread dies quickly.

3 files changed, 63 insertions, 46 deletions

diff --git a/sql/mini_client.cc b/sql/mini_client.cc
index 743d522e4bd..5bd88e9b09a 100644
--- a/sql/mini_client.cc
+++ b/sql/mini_client.cc
@@ -414,10 +414,8 @@ my_bool mc_mysql_reconnect(MYSQL *mysql)
 			mysql->db, mysql->port, mysql->unix_socket,
 			mysql->client_flag, mysql->net.read_timeout))
   {
-#ifdef NOT_USED
-    mysql->net.last_errno=CR_RECONNECT_FAILED;
-    strmov(mysql->net.last_error, ER(mysql->net.last_errno));
-#endif
+    mysql->net.last_errno= tmp_mysql.net.last_errno;
+    strmov(mysql->net.last_error, tmp_mysql.net.last_error);
     DBUG_RETURN(1);
   }
   tmp_mysql.free_me=mysql->free_me;
@@ -888,7 +886,6 @@ mc_mysql_close(MYSQL *mysql)
     /* Clear pointers for better safety */
     mysql->host_info=mysql->user=mysql->passwd=mysql->db=0;
     bzero((char*) &mysql->options,sizeof(mysql->options));
-    mysql->net.vio = 0;
 #ifdef HAVE_OPENSSL
     mysql_ssl_clear(mysql);
 #endif /* HAVE_OPENSSL */
@@ -976,13 +973,13 @@ mc_unpack_fields(MYSQL_DATA *data,MEM_ROOT *alloc,uint fields,
   DBUG_RETURN(result);
 }
 
-int 
-mc_mysql_send_query(MYSQL* mysql, const char* query, uint length)
+int mc_mysql_send_query(MYSQL* mysql, const char* query, uint length)
 {
   return mc_simple_command(mysql, COM_QUERY, query, length, 1);
 }
 
-int  mc_mysql_read_query_result(MYSQL *mysql)
+
+int mc_mysql_read_query_result(MYSQL *mysql)
 {
   uchar *pos;
   ulong field_count;
diff --git a/sql/slave.cc b/sql/slave.cc
index 93a5c6171df..27e9030c000 100644
--- a/sql/slave.cc
+++ b/sql/slave.cc
@@ -443,14 +443,18 @@ int terminate_slave_thread(THD* thd, pthread_mutex_t* term_lock,
 }
 
 
-int start_slave_thread(pthread_handler h_func, pthread_mutex_t* start_lock,
+int start_slave_thread(pthread_handler h_func, pthread_mutex_t *start_lock,
 		       pthread_mutex_t *cond_lock,
-		       pthread_cond_t* start_cond,
-		       volatile bool* slave_running,
+		       pthread_cond_t *start_cond,
+		       volatile bool *slave_running,
+		       volatile ulong *slave_run_id,
 		       MASTER_INFO* mi)
 {
   pthread_t th;
+  ulong start_id;
   DBUG_ASSERT(mi->inited);
+  DBUG_ENTER("start_slave_thread");
+
   if (start_lock)
     pthread_mutex_lock(start_lock);
   if (!server_id)
@@ -460,7 +464,7 @@ int start_slave_thread(pthread_handler h_func, pthread_mutex_t* start_lock,
     if (start_lock)
       pthread_mutex_unlock(start_lock);
     sql_print_error("Server id not set, will not start slave");
-    return ER_BAD_SLAVE;
+    DBUG_RETURN(ER_BAD_SLAVE);
   }
   
   if (*slave_running)
@@ -469,39 +473,36 @@ int start_slave_thread(pthread_handler h_func, pthread_mutex_t* start_lock,
       pthread_cond_broadcast(start_cond);
     if (start_lock)
       pthread_mutex_unlock(start_lock);
-    return ER_SLAVE_MUST_STOP;
+    DBUG_RETURN(ER_SLAVE_MUST_STOP);
   }
+  start_id= *slave_run_id;
+  DBUG_PRINT("info",("Creating new slave thread"));
   if (pthread_create(&th, &connection_attrib, h_func, (void*)mi))
   {
     if (start_lock)
       pthread_mutex_unlock(start_lock);
-    return ER_SLAVE_THREAD;
+    DBUG_RETURN(ER_SLAVE_THREAD);
   }
   if (start_cond && cond_lock)
   {
     THD* thd = current_thd;
-    while (!*slave_running)
+    while (start_id == *slave_run_id)
     {
+      DBUG_PRINT("sleep",("Waiting for slave thread to start"));
       const char* old_msg = thd->enter_cond(start_cond,cond_lock,
 					    "Waiting for slave thread to start");
       pthread_cond_wait(start_cond,cond_lock);
       thd->exit_cond(old_msg);
-      /*
-	TODO: in a very rare case of init_slave_thread failing, it is
-	possible that we can get stuck here since slave_running will not
-	be set. We need to change slave_running to int and have -1 as
-	error code.
-      */
       if (thd->killed)
       {
 	pthread_mutex_unlock(cond_lock);
-	return ER_SERVER_SHUTDOWN;
+	DBUG_RETURN(ER_SERVER_SHUTDOWN);
       }
     }
   }
   if (start_lock)
     pthread_mutex_unlock(start_lock);
-  return 0;
+  DBUG_RETURN(0);
 }
 
 
@@ -535,13 +536,15 @@ int start_slave_threads(bool need_slave_mutex, bool wait_for_start,
 
   if (thread_mask & SLAVE_IO)
     error=start_slave_thread(handle_slave_io,lock_io,lock_cond_io,
-			     cond_io,&mi->slave_running,
+			     cond_io,
+			     &mi->slave_running, &mi->slave_run_id,
 			     mi);
   if (!error && (thread_mask & SLAVE_SQL))
   {
     error=start_slave_thread(handle_slave_sql,lock_sql,lock_cond_sql,
 			     cond_sql,
-			     &mi->rli.slave_running,mi);
+			     &mi->rli.slave_running, &mi->rli.slave_run_id,
+			     mi);
     if (error)
       terminate_slave_threads(mi, thread_mask & SLAVE_IO, 0);
   }
@@ -1807,23 +1810,30 @@ This may also be a network problem, or just a bug in the master or slave code.\
 /* slave I/O thread */
 pthread_handler_decl(handle_slave_io,arg)
 {
+  THD *thd; // needs to be first for thread_stack
+  MYSQL *mysql;
+  MASTER_INFO *mi = (MASTER_INFO*)arg; 
+  char llbuff[22];
+  uint retry_count;
+  
+  // needs to call my_thread_init(), otherwise we get a coredump in DBUG_ stuff
+  my_thread_init();
+
 #ifndef DBUG_OFF
 slave_begin:  
 #endif  
-  THD *thd; // needs to be first for thread_stack
-  MYSQL *mysql = NULL ;
-  MASTER_INFO* mi = (MASTER_INFO*)arg; 
-  char llbuff[22];
-  uint retry_count= 0;
   DBUG_ASSERT(mi->inited);
-  
+  mysql= NULL ;
+  retry_count= 0;
+
   pthread_mutex_lock(&mi->run_lock);
+  /* Inform waiting threads that slave has started */
+  mi->slave_run_id++;
+
 #ifndef DBUG_OFF  
   mi->events_till_abort = abort_slave_event_count;
 #endif  
   
-  // needs to call my_thread_init(), otherwise we get a coredump in DBUG_ stuff
-  my_thread_init();
   thd= new THD; // note that contructor of THD uses DBUG_ !
   DBUG_ENTER("handle_slave_io");
   THD_CHECK_SENTRY(thd);
@@ -2071,26 +2081,32 @@ err:
 
 pthread_handler_decl(handle_slave_sql,arg)
 {
-#ifndef DBUG_OFF
-slave_begin:  
-#endif  
   THD *thd;			/* needs to be first for thread_stack */
   char llbuff[22],llbuff1[22];
   RELAY_LOG_INFO* rli = &((MASTER_INFO*)arg)->rli; 
-  const char* errmsg=0;
+  const char *errmsg;
+
+  // needs to call my_thread_init(), otherwise we get a coredump in DBUG_ stuff
+  my_thread_init();
+
+#ifndef DBUG_OFF
+slave_begin:  
+#endif  
+
   DBUG_ASSERT(rli->inited);
   pthread_mutex_lock(&rli->run_lock);
   DBUG_ASSERT(!rli->slave_running);
+  errmsg= 0;
 #ifndef DBUG_OFF  
   rli->events_till_abort = abort_slave_event_count;
 #endif  
-  
-  // needs to call my_thread_init(), otherwise we get a coredump in DBUG_ stuff
-  my_thread_init();
-  thd = new THD; // note that contructor of THD uses DBUG_ !
   DBUG_ENTER("handle_slave_sql");
 
+  thd = new THD; // note that contructor of THD uses DBUG_ !
   THD_CHECK_SENTRY(thd);
+  /* Inform waiting threads that slave has started */
+  rli->slave_run_id++;
+
   pthread_detach_this_thread();
   if (init_slave_thread(thd, SLAVE_THD_SQL))
   {
diff --git a/sql/slave.h b/sql/slave.h
index 4be01785177..b527aceb432 100644
--- a/sql/slave.h
+++ b/sql/slave.h
@@ -154,6 +154,7 @@ typedef struct st_relay_log_info
   */
   volatile uint32 slave_skip_counter;
   volatile ulong abort_pos_wait;	/* Incremented on change master */
+  volatile ulong slave_run_id;		/* Incremented on slave start */
   pthread_mutex_t log_space_lock;
   pthread_cond_t log_space_cond;
   THD * sql_thd;
@@ -171,8 +172,8 @@ typedef struct st_relay_log_info
   
   st_relay_log_info()
   :info_fd(-1),cur_log_fd(-1), cur_log_old_open_count(0), abort_pos_wait(0),
-   inited(0), abort_slave(0), slave_running(0), log_pos_current(0),
-   skip_log_purge(0)
+   slave_run_id(0), inited(0), abort_slave(0), slave_running(0),
+   log_pos_current(0), skip_log_purge(0)
   {
     relay_log_name[0] = master_log_name[0] = 0;
     bzero(&info_file,sizeof(info_file));
@@ -283,11 +284,13 @@ typedef struct st_master_info
   bool inited;
   bool old_format;			/* master binlog is in 3.23 format */
   volatile bool abort_slave, slave_running;
+  volatile ulong slave_run_id;
   bool ignore_stop_event;
   
   
-  st_master_info():fd(-1), io_thd(0), inited(0), old_format(0),abort_slave(0),
-		   slave_running(0)
+  st_master_info()
+    :fd(-1), io_thd(0), inited(0), old_format(0),abort_slave(0),
+     slave_running(0), slave_run_id(0)
   {
     host[0] = 0; user[0] = 0; password[0] = 0;
     bzero(&file, sizeof(file));
@@ -360,7 +363,8 @@ int start_slave_threads(bool need_slave_mutex, bool wait_for_start,
 int start_slave_thread(pthread_handler h_func, pthread_mutex_t* start_lock,
 		       pthread_mutex_t *cond_lock,
 		       pthread_cond_t* start_cond,
-		       volatile bool* slave_running,
+		       volatile bool *slave_running,
+		       volatile ulong *slave_run_id,
 		       MASTER_INFO* mi);
 
 /* If fd is -1, dump to NET */