Bugfix: New table creation/renaming block added if old encoded table present.

mysql-test/r/create.result:
  test of renaming
mysql-test/r/upgrade.result:
  Now such behaviour prohibited to avoid problems.
mysql-test/t/create.test:
  test of renaming
mysql-test/t/upgrade.test:
  Now such behaviour prohibited to avoid problems.
sql/mysql_priv.h:
  Function to test table name presence added.
sql/sql_rename.cc:
  Rename fixed.
sql/sql_table.cc:
  Function to test table name presence added.
  Create fixed.

7 files changed, 119 insertions, 20 deletions

diff --git a/mysql-test/r/create.result b/mysql-test/r/create.result
index e871d9526ca..720e7c1d24e 100644
--- a/mysql-test/r/create.result
+++ b/mysql-test/r/create.result
@@ -1890,5 +1890,24 @@ create table t3 (a int) row_format=page;
 drop table t1,t2,t3;
 
 # -- End of Bug#45829
+# new table creation/renaming blocked if old encoded table present
+create table `t-1` (a int) engine=myisam;
+insert into `t-1` values (1);
+show tables;
+Tables_in_test
+t-1
+flush tables;
+convert table files in mysql 5.0 file name encoding
+show tables;
+Tables_in_test
+#mysql50#t-1
+create table `t-1` (a int);
+ERROR 42S01: Table '#mysql50#t-1' already exists
+create table t1 (a int);
+alter table t1 rename `t-1`;
+ERROR 42S01: Table '#mysql50#t-1' already exists
+rename table t1 to `t-1`;
+ERROR 42S01: Table '#mysql50#t-1' already exists
+drop table `#mysql50#t-1`, t1;
 
 End of 5.1 tests
diff --git a/mysql-test/r/upgrade.result b/mysql-test/r/upgrade.result
index f2c8886c915..ec63cc0d98c 100644
--- a/mysql-test/r/upgrade.result
+++ b/mysql-test/r/upgrade.result
@@ -48,14 +48,10 @@ insert into `txu#p#p1` values (1);
 select * from `txu@0023p@0023p1`;
 ERROR 42S02: Table 'test.txu@0023p@0023p1' doesn't exist
 create table `txu@0023p@0023p1` (s1 int);
-insert into `txu@0023p@0023p1` values (2);
-select * from `txu@0023p@0023p1`;
-s1
-2
+ERROR 42S01: Table '#mysql50#txu@0023p@0023p1' already exists
 select * from `txu#p#p1`;
 s1
 1
-drop table `txu@0023p@0023p1`;
 drop table `txu#p#p1`;
 #
 # Bug#37631 Incorrect key file for table after upgrading from 5.0 to 5.1
diff --git a/mysql-test/t/create.test b/mysql-test/t/create.test
index f3cd48af543..fa991aab0d7 100644
--- a/mysql-test/t/create.test
+++ b/mysql-test/t/create.test
@@ -1534,5 +1534,30 @@ drop table t1,t2,t3;
 --echo
 --echo # -- End of Bug#45829
 
+#
+--echo # new table creation/renaming blocked if old encoded table present
+#
+let $MYSQLD_DATADIR= `select @@datadir`;
+create table `t-1` (a int) engine=myisam;
+insert into `t-1` values (1);
+show tables;
+flush tables;
+--echo convert table files in mysql 5.0 file name encoding
+--copy_file $MYSQLD_DATADIR/test/t@002d1.MYD $MYSQLD_DATADIR/test/t-1.MYD
+--copy_file $MYSQLD_DATADIR/test/t@002d1.MYI $MYSQLD_DATADIR/test/t-1.MYI
+--copy_file $MYSQLD_DATADIR/test/t@002d1.frm $MYSQLD_DATADIR/test/t-1.frm
+--remove_file $MYSQLD_DATADIR/test/t@002d1.MYD
+--remove_file $MYSQLD_DATADIR/test/t@002d1.MYI
+--remove_file $MYSQLD_DATADIR/test/t@002d1.frm
+show tables;
+--error ER_TABLE_EXISTS_ERROR
+create table `t-1` (a int);
+create table t1 (a int);
+--error ER_TABLE_EXISTS_ERROR
+alter table t1 rename `t-1`;
+--error ER_TABLE_EXISTS_ERROR
+rename table t1 to `t-1`;
+drop table `#mysql50#t-1`, t1;
+
 --echo
 --echo End of 5.1 tests
diff --git a/mysql-test/t/upgrade.test b/mysql-test/t/upgrade.test
index 400ce07b7fd..a8c875ef764 100644
--- a/mysql-test/t/upgrade.test
+++ b/mysql-test/t/upgrade.test
@@ -41,11 +41,9 @@ create table `txu#p#p1` (s1 int);
 insert into `txu#p#p1` values (1);
 --error 1146
 select * from `txu@0023p@0023p1`;
+--error ER_TABLE_EXISTS_ERROR
 create table `txu@0023p@0023p1` (s1 int);
-insert into `txu@0023p@0023p1` values (2);
-select * from `txu@0023p@0023p1`;
 select * from `txu#p#p1`;
-drop table `txu@0023p@0023p1`;
 drop table `txu#p#p1`;
 
 --echo #
diff --git a/sql/mysql_priv.h b/sql/mysql_priv.h
index 24c8ba0b257..51450950521 100644
--- a/sql/mysql_priv.h
+++ b/sql/mysql_priv.h
@@ -1206,6 +1206,11 @@ bool mysql_handle_derived(LEX *lex, bool (*processor)(THD *thd,
                                                       TABLE_LIST *table));
 bool mysql_derived_prepare(THD *thd, LEX *lex, TABLE_LIST *t);
 bool mysql_derived_filling(THD *thd, LEX *lex, TABLE_LIST *t);
+bool check_table_file_presence(char *old_path, char *path,
+                               const char *db,
+                               const char *table_name,
+                               const char *alias,
+                               bool issue_error);
 Field *create_tmp_field(THD *thd, TABLE *table,Item *item, Item::Type type,
 			Item ***copy_func, Field **from_field,
                         Field **def_field,
diff --git a/sql/sql_rename.cc b/sql/sql_rename.cc
index df7054c94d0..81c6e3ad4be 100644
--- a/sql/sql_rename.cc
+++ b/sql/sql_rename.cc
@@ -245,7 +245,7 @@ do_rename(THD *thd, TABLE_LIST *ren_table, char *new_db, char *new_table_name,
           char *new_table_alias, bool skip_error)
 {
   int rc= 1;
-  char name[FN_REFLEN + 1];
+  char new_name[FN_REFLEN + 1], old_name[FN_REFLEN + 1];
   const char *new_alias, *old_alias;
   frm_type_enum frm_type;
   enum legacy_db_type table_type;
@@ -264,17 +264,17 @@ do_rename(THD *thd, TABLE_LIST *ren_table, char *new_db, char *new_table_name,
   }
   DBUG_ASSERT(new_alias);
 
-  build_table_filename(name, sizeof(name) - 1,
+  build_table_filename(new_name, sizeof(new_name) - 1,
                        new_db, new_alias, reg_ext, 0);
-  if (!access(name,F_OK))
+  build_table_filename(old_name, sizeof(old_name) - 1,
+                       ren_table->db, old_alias, reg_ext, 0);
+  if (check_table_file_presence(old_name,
+                                new_name, new_db, new_alias, new_alias, TRUE))
   {
-    my_error(ER_TABLE_EXISTS_ERROR, MYF(0), new_alias);
     DBUG_RETURN(1);			// This can't be skipped
   }
-  build_table_filename(name, sizeof(name) - 1,
-                       ren_table->db, old_alias, reg_ext, 0);
 
-  frm_type= mysql_frm_type(thd, name, &table_type);
+  frm_type= mysql_frm_type(thd, old_name, &table_type);
   switch (frm_type)
   {
     case FRMTYPE_TABLE:
@@ -319,7 +319,7 @@ do_rename(THD *thd, TABLE_LIST *ren_table, char *new_db, char *new_table_name,
     default:
       DBUG_ASSERT(0); // should never happen
     case FRMTYPE_ERROR:
-      my_error(ER_FILE_NOT_FOUND, MYF(0), name, my_errno);
+      my_error(ER_FILE_NOT_FOUND, MYF(0), old_name, my_errno);
       break;
   }
   if (rc && !skip_error)
diff --git a/sql/sql_table.cc b/sql/sql_table.cc
index 939ca6302d9..d76f1a427be 100644
--- a/sql/sql_table.cc
+++ b/sql/sql_table.cc
@@ -3611,6 +3611,58 @@ static inline int write_create_table_bin_log(THD *thd,
 }
 
 
+/**
+  Check that there is no frm file for given table
+
+  @param old_path        path to the old frm file
+  @param path            path to the frm file in new encoding
+  @param db              database name
+  @param table_name      table name
+  @param alias           table name for error message (for new encoding)
+  @param issue_error     should we issue error messages
+
+  @retval FALSE there is no frm file
+  @retval TRUE  there is frm file
+*/
+
+bool check_table_file_presence(char *old_path,
+                               char *path,
+                               const char *db,
+                               const char *table_name,
+                               const char *alias,
+                               bool issue_error)
+{
+  if (!access(path,F_OK))
+  {
+    if (issue_error)
+      my_error(ER_TABLE_EXISTS_ERROR,MYF(0),alias);
+    return TRUE;
+  }
+  {
+    /*
+      Check if file of the table in 5.0 file name encoding exists.
+
+      Except case when it is the same table.
+    */
+    char tbl50[FN_REFLEN];
+    strxmov(tbl50, mysql_data_home, "/", db, "/", table_name, NullS);
+    if (!access(fn_format(tbl50, tbl50, "", reg_ext,
+                          MY_UNPACK_FILENAME), F_OK) &&
+        (old_path == NULL ||
+         strcmp(old_path, tbl50) != 0))
+    {
+      if (issue_error)
+      {
+        strxmov(tbl50, MYSQL50_TABLE_NAME_PREFIX, table_name, NullS);
+        my_error(ER_TABLE_EXISTS_ERROR, MYF(0), tbl50);
+      }
+      return TRUE;
+    }
+  }
+  return FALSE;
+}
+
+
 /*
   Create a table
 
@@ -3892,11 +3944,12 @@ bool mysql_create_table_no_lock(THD *thd,
   VOID(pthread_mutex_lock(&LOCK_open));
   if (!internal_tmp_table && !(create_info->options & HA_LEX_CREATE_TMP_TABLE))
   {
-    if (!access(path,F_OK))
+    if (check_table_file_presence(NULL, path, db, table_name, table_name,
+                                  !(create_info->options &
+                                    HA_LEX_CREATE_IF_NOT_EXISTS)))
     {
       if (create_info->options & HA_LEX_CREATE_IF_NOT_EXISTS)
         goto warn;
-      my_error(ER_TABLE_EXISTS_ERROR,MYF(0),table_name);
       goto unlock_and_end;
     }
     /*
@@ -6512,6 +6565,7 @@ bool mysql_alter_table(THD *thd,char *new_db, char *new_name,
   TABLE *table, *new_table= 0, *name_lock= 0;
   int error= 0;
   char tmp_name[80],old_name[32],new_name_buff[FN_REFLEN + 1];
+  char old_name_buff[FN_REFLEN + 1];
   char new_alias_buff[FN_REFLEN], *table_name, *db, *new_alias, *alias;
   char index_file[FN_REFLEN], data_file[FN_REFLEN];
   char path[FN_REFLEN + 1];
@@ -6741,10 +6795,12 @@ view_err:
 
         build_table_filename(new_name_buff, sizeof(new_name_buff) - 1,
                              new_db, new_name_buff, reg_ext, 0);
-        if (!access(new_name_buff, F_OK))
+        build_table_filename(old_name_buff, sizeof(old_name_buff) - 1,
+                             db, table_name, reg_ext, 0);
+        if (check_table_file_presence(old_name_buff, new_name_buff, new_db,
+                                      new_name, new_alias, TRUE))
 	{
 	  /* Table will be closed in do_command() */
-	  my_error(ER_TABLE_EXISTS_ERROR, MYF(0), new_alias);
           goto err;
 	}
       }