file-based table discovery for discovering storage engines

also:
* move few tests from archive/archive.test to archive/discover.test
* fix (unintentionally) bug#34104, updated result files

12 files changed, 483 insertions, 182 deletions

diff --git a/mysql-test/r/grant4.result b/mysql-test/r/grant4.result
index d0b6627cd0b..65ab92c1afd 100644
--- a/mysql-test/r/grant4.result
+++ b/mysql-test/r/grant4.result
@@ -86,9 +86,9 @@ GRANT SHOW VIEW, SELECT ON v3 to mysqltest_u1@localhost;
 use mysqltest_db1;
 ** Connect as restricted user mysqltest_u1.
 ** SELECT FROM INFORMATION_SCHEMA.STATISTICS will succeed because any privileges will do (authentication is enough).
+** but will return no rows
 SELECT * FROM INFORMATION_SCHEMA.STATISTICS WHERE table_name='t5';
 TABLE_CATALOG	TABLE_SCHEMA	TABLE_NAME	NON_UNIQUE	INDEX_SCHEMA	INDEX_NAME	SEQ_IN_INDEX	COLUMN_NAME	COLLATION	CARDINALITY	SUB_PART	PACKED	NULLABLE	INDEX_TYPE	COMMENT	INDEX_COMMENT
-def	mysqltest_db1	t5	1	mysqltest_db1	i	1	s1	A	NULL	NULL	NULL	YES	BTREE		
 ** SHOW INDEX FROM t5 will fail because we don't have any privileges on any column combination.
 SHOW INDEX FROM t5;
 ERROR 42000: SELECT command denied to user 'mysqltest_u1'@'localhost' for table 't5'
diff --git a/mysql-test/suite/archive/archive.result b/mysql-test/suite/archive/archive.result
index 98608a95f37..618433fe5e0 100644
--- a/mysql-test/suite/archive/archive.result
+++ b/mysql-test/suite/archive/archive.result
@@ -12785,38 +12785,6 @@ a
 2
 DROP TABLE t1;
 #
-# Bug#45377: ARCHIVE tables aren't discoverable after OPTIMIZE
-#
-DROP TABLE IF EXISTS t1;
-CREATE TABLE t1 (a int) ENGINE=ARCHIVE;
-SHOW CREATE TABLE t1;
-Table	Create Table
-t1	CREATE TABLE `t1` (
-  `a` int(11) DEFAULT NULL
-) ENGINE=ARCHIVE DEFAULT CHARSET=latin1
-INSERT INTO t1 VALUES (1);
-OPTIMIZE TABLE t1;
-Table	Op	Msg_type	Msg_text
-test.t1	optimize	status	OK
-FLUSH TABLES;
-INSERT INTO t1 VALUES (2);
-SELECT * FROM t1 ORDER BY a;
-a
-1
-2
-SHOW CREATE TABLE t1;
-Table	Create Table
-t1	CREATE TABLE `t1` (
-  `a` int(11) DEFAULT NULL
-) ENGINE=ARCHIVE DEFAULT CHARSET=latin1
-DROP TABLE t1;
-#
-# BUG#58205 - Valgrind failure in fn_format when called from
-#             archive_discover
-#
-CREATE TABLE `a/../`(a INT) ENGINE=ARCHIVE;
-DROP TABLE `a/../`;
-#
 # BUG#57162 - valgrind errors, random data when returning
 #             ordered data from archive tables
 #
diff --git a/mysql-test/suite/archive/archive.test b/mysql-test/suite/archive/archive.test
index 5bcefa3dcd5..92868158ed4 100644
--- a/mysql-test/suite/archive/archive.test
+++ b/mysql-test/suite/archive/archive.test
@@ -1712,34 +1712,6 @@ SELECT * FROM t1;
 DROP TABLE t1;
 
 --echo #
---echo # Bug#45377: ARCHIVE tables aren't discoverable after OPTIMIZE
---echo #
-
---disable_warnings
-DROP TABLE IF EXISTS t1;
---enable_warnings
-
-CREATE TABLE t1 (a int) ENGINE=ARCHIVE;
-SHOW CREATE TABLE t1;
-INSERT INTO t1 VALUES (1);
-OPTIMIZE TABLE t1;
-let $MYSQLD_DATADIR= `select @@datadir`;
-remove_file $MYSQLD_DATADIR/test/t1.frm;
-FLUSH TABLES;
-INSERT INTO t1 VALUES (2);
-SELECT * FROM t1 ORDER BY a;
-SHOW CREATE TABLE t1;
-DROP TABLE t1;
-
---echo #
---echo # BUG#58205 - Valgrind failure in fn_format when called from
---echo #             archive_discover
---echo #
-CREATE TABLE `a/../`(a INT) ENGINE=ARCHIVE;
-remove_file $MYSQLD_DATADIR/test/a@002f@002e@002e@002f.frm;
-DROP TABLE `a/../`;
-
---echo #
 --echo # BUG#57162 - valgrind errors, random data when returning
 --echo #             ordered data from archive tables
 --echo #
diff --git a/mysql-test/suite/archive/discover.result b/mysql-test/suite/archive/discover.result
new file mode 100644
index 00000000000..51f66ee701b
--- /dev/null
+++ b/mysql-test/suite/archive/discover.result
@@ -0,0 +1,70 @@
+create table t1 (a int) engine=archive;
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` int(11) DEFAULT NULL
+) ENGINE=ARCHIVE DEFAULT CHARSET=latin1
+insert t1 values (1);
+show tables;
+Tables_in_test
+t1
+#
+# simple discover on use
+#
+flush tables;
+insert t1 values (2);
+select * from t1;
+a
+1
+2
+#
+# list tables
+#
+create table t0 (a int) engine=archive;
+flush tables;
+show tables;
+Tables_in_test
+t0
+t1
+select * from t1;
+a
+1
+2
+#
+# discover on drop
+#
+flush tables;
+drop table t0, t1;
+show tables;
+Tables_in_test
+#
+# Bug#45377: ARCHIVE tables aren't discoverable after OPTIMIZE
+#
+create table t1 (a int) engine=archive;
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` int(11) DEFAULT NULL
+) ENGINE=ARCHIVE DEFAULT CHARSET=latin1
+insert into t1 values (1);
+optimize table t1;
+Table	Op	Msg_type	Msg_text
+test.t1	optimize	status	OK
+flush tables;
+insert into t1 values (2);
+select * from t1 order by a;
+a
+1
+2
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` int(11) DEFAULT NULL
+) ENGINE=ARCHIVE DEFAULT CHARSET=latin1
+drop table t1;
+#
+# BUG#58205 - Valgrind failure in fn_format when called from
+#             archive_discover
+#
+create table `a/../`(a int) engine=archive;
+drop table `a/../`;
diff --git a/mysql-test/suite/archive/discover.test b/mysql-test/suite/archive/discover.test
new file mode 100644
index 00000000000..843cb0caf63
--- /dev/null
+++ b/mysql-test/suite/archive/discover.test
@@ -0,0 +1,56 @@
+-- source include/have_archive.inc
+let $mysqld_datadir= `select @@datadir`;
+
+create table t1 (a int) engine=archive;
+show create table t1;
+insert t1 values (1);
+show tables;
+
+--echo #
+--echo # simple discover on use
+--echo #
+remove_file $mysqld_datadir/test/t1.frm;
+flush tables;
+insert t1 values (2);
+select * from t1;
+
+--echo #
+--echo # list tables
+--echo #
+create table t0 (a int) engine=archive;
+remove_file $mysqld_datadir/test/t1.frm;
+flush tables;
+show tables;
+select * from t1;
+
+--echo #
+--echo # discover on drop
+--echo #
+remove_file $mysqld_datadir/test/t1.frm;
+flush tables;
+drop table t0, t1;
+show tables;
+
+--echo #
+--echo # Bug#45377: ARCHIVE tables aren't discoverable after OPTIMIZE
+--echo #
+
+create table t1 (a int) engine=archive;
+show create table t1;
+insert into t1 values (1);
+optimize table t1;
+remove_file $mysqld_datadir/test/t1.frm;
+flush tables;
+insert into t1 values (2);
+select * from t1 order by a;
+show create table t1;
+drop table t1;
+
+--echo #
+--echo # BUG#58205 - Valgrind failure in fn_format when called from
+--echo #             archive_discover
+--echo #
+create table `a/../`(a int) engine=archive;
+remove_file $mysqld_datadir/test/a@002f@002e@002e@002f.frm;
+drop table `a/../`;
+
diff --git a/mysql-test/t/grant4.test b/mysql-test/t/grant4.test
index f3e551cd623..c30e0c28f31 100644
--- a/mysql-test/t/grant4.test
+++ b/mysql-test/t/grant4.test
@@ -104,9 +104,7 @@ connection con1;
 use mysqltest_db1;
 --echo ** Connect as restricted user mysqltest_u1.
 --echo ** SELECT FROM INFORMATION_SCHEMA.STATISTICS will succeed because any privileges will do (authentication is enough).
-#
-# this result is wrong. reported as bug#34104
-#
+--echo ** but will return no rows
 SELECT * FROM INFORMATION_SCHEMA.STATISTICS WHERE table_name='t5';
 #
 # Bug27145 EXTRA_ACL trouble
diff --git a/sql/discover.cc b/sql/discover.cc
index b9dba92a780..72b88c8780b 100644
--- a/sql/discover.cc
+++ b/sql/discover.cc
@@ -128,7 +128,131 @@ int writefrm(const char *name, const uchar *frmdata, size_t len)
   DBUG_RETURN(error);
 } /* writefrm */
 
+static inline void advance(FILEINFO* &from, FILEINFO* &to,
+                           FILEINFO* cur, bool &skip)
+{
+  if (skip)                   // if not copying
+    from= cur;                //   just advance the start pointer
+  else                        // if copying
+    if (to == from)           //   but to the same place (not shifting the data)
+      from= to= cur;          //     advance both pointers
+    else                      //   otherwise
+      while (from < cur)      //     have to copy [from...cur) to [to...)
+        *to++ = *from++;
+  skip= false;
+}
+
+/**
+  Go through the directory listing looking for files with a specified
+  extension and add them to the result list
+
+  @details
+  This function may be called many times on the same directory listing
+  but with different extensions. To avoid discovering the same table twice,
+  whenever a table file is discovered, all files with the same name
+  (independently from the extensions) are removed from the list.
+
+  Example: the list contained
+     { "db.opt", "t1.MYD", "t1.MYI", "t1.frm", "t2.ARZ", "t3.ARZ", "t3.frm" }
+  on discovering all ".frm" files, tables "t1" and "t3" will be found,
+  and list will become
+     { "db.opt", "t2.ARZ" }
+  and now ".ARZ" discovery can discover the table "t2"
+
+  @note
+  This function assumes that the directory listing is sorted alphabetically.
+
+  @note  Partitioning makes this more complicated. A partitioned table t1 might
+  have files, like t1.frm, t1#P#part1.ibd, t1#P#foo.ibd, etc.
+  That means we need to compare file names only up to the first '#' or '.'
+  whichever comes first.
+*/
+int extension_based_table_discovery(MY_DIR *dirp, const char *ext_meta,
+                                    handlerton::discovered_list *result)
+{
+  CHARSET_INFO *cs= character_set_filesystem;
+  size_t ext_meta_len= strlen(ext_meta);
+  FILEINFO *from, *to, *cur, *end;
+  bool skip= false;
+  
+  from= to= cur= dirp->dir_entry;
+  end= cur + dirp->number_of_files;
+  while (cur < end)
+  {
+    char *octothorp= strrchr(cur->name, '#');
+    char *ext= strchr(octothorp ? octothorp : cur->name, FN_EXTCHAR);
 
+    if (ext && octothorp != cur->name)
+    {
+      size_t len= (octothorp ? octothorp : ext) - cur->name;
+      if (from != cur &&
+          (my_strnncoll(cs, (uchar*)from->name, len, (uchar*)cur->name, len) ||
+           (from->name[len] != FN_EXTCHAR && from->name[len] != '#')))
+        advance(from, to, cur, skip);
 
+      if (my_strnncoll(cs, (uchar*)ext, strlen(ext),
+                           (uchar*)ext_meta, ext_meta_len) == 0)
+      {
+        *ext = 0;
+        if (result->add_file(cur->name))
+          return 1;
+        *ext = FN_EXTCHAR;
+        skip= true; // table discovered, skip all files with the same name
+      }
+    }
+    else
+    {
+      advance(from, to, cur, skip);
+      from++;
+    }
 
+    cur++;
+  }
+  advance(from, to, cur, skip);
+  dirp->number_of_files= to - dirp->dir_entry;
+  return 0;
+}
+
+/**
+  Simple, not reusable file-based table discovery
+
+  @details
+  simplified version of extension_based_table_discovery(), that does not
+  modify the list of files. It cannot be called many times for the same
+  directory listing, otherwise it'll produce duplicate results.
+
+  @note
+  For backward compatibility reasons, this will find tables with names,
+  starting from '#', as long as they don't start from '#sql-'.
+  These names are invalid since 5.0, and the compex discovery function
+  will ignore them. Anyone still having these files, should disable
+  discovering engines, and rename these invalid table files.
+*/
+int ext_table_discovery_simple(MY_DIR *dirp, const char *ext_meta,
+                               handlerton::discovered_list *result)
+{
+  CHARSET_INFO *cs= character_set_filesystem;
+  size_t ext_meta_len= strlen(ext_meta);
+  FILEINFO *cur, *end;
+  
+  cur= dirp->dir_entry;
+  end= cur + dirp->number_of_files;
+  while (cur < end)
+  {
+    char *ext= strrchr(cur->name, FN_EXTCHAR);
+
+    if (ext && !is_prefix(cur->name, tmp_file_prefix))
+    {
+      if (my_strnncoll(cs, (uchar*)ext, strlen(ext),
+                           (uchar*)ext_meta, ext_meta_len) == 0)
+      {
+        *ext = 0;
+        if (result->add_file(cur->name))
+          return 1;
+      }
+    }
+    cur++;
+  }
+  return 0;
+}
 
diff --git a/sql/discover.h b/sql/discover.h
index a663e44128d..d7f94240fa6 100644
--- a/sql/discover.h
+++ b/sql/discover.h
@@ -21,4 +21,12 @@
 int readfrm(const char *name, uchar **data, size_t *length);
 int writefrm(const char* name, const uchar* data, size_t len);
 
+int extension_based_table_discovery(MY_DIR *dirp, const char *ext,
+                                    handlerton::discovered_list *tl);
+
+#ifdef MYSQL_SERVER
+int ext_table_discovery_simple(MY_DIR *dirp, const char *ext_meta,
+                               handlerton::discovered_list *result);
+#endif
+
 #endif /* DISCOVER_INCLUDED */
diff --git a/sql/handler.cc b/sql/handler.cc
index 56e9ec35ad8..8dd0a5d6995 100644
--- a/sql/handler.cc
+++ b/sql/handler.cc
@@ -433,6 +433,9 @@ int ha_finalize_handlerton(st_plugin_int *plugin)
     hton2plugin[hton->slot]= NULL;
   }
 
+  if (hton->discover_table_names)
+    my_atomic_add32(&engines_with_discover_table_names, -1);
+
   my_free(hton);
 
  end:
@@ -440,6 +443,13 @@ int ha_finalize_handlerton(st_plugin_int *plugin)
 }
 
 
+static int hton_ext_based_table_discovery(handlerton *hton, LEX_STRING *db,
+                             MY_DIR *dir, handlerton::discovered_list *result)
+{
+  return extension_based_table_discovery(dir, hton->tablefile_extensions[0],
+                                         result);
+}
+
 int ha_initialize_handlerton(st_plugin_int *plugin)
 {
   handlerton *hton;
@@ -449,8 +459,6 @@ int ha_initialize_handlerton(st_plugin_int *plugin)
 
   hton= (handlerton *)my_malloc(sizeof(handlerton),
                                 MYF(MY_WME | MY_ZEROFILL));
-  hton->tablefile_extensions= no_exts;
-
   if (hton == NULL)
   {
     sql_print_error("Unable to allocate memory for plugin '%s' handlerton.",
@@ -468,6 +476,16 @@ int ha_initialize_handlerton(st_plugin_int *plugin)
     goto err;
   }
 
+  // default list file extensions: empty
+  if (!hton->tablefile_extensions)
+    hton->tablefile_extensions= no_exts;
+
+  // if the enfine can discover a single table and it is file-based
+  // then it can use a default file-based table names discovery
+  if (!hton->discover_table_names &&
+      hton->discover && hton->tablefile_extensions[0])
+    hton->discover_table_names= hton_ext_based_table_discovery;
+
   /*
     the switch below and hton->state should be removed when
     command-line options for plugins will be implemented
@@ -557,6 +575,9 @@ int ha_initialize_handlerton(st_plugin_int *plugin)
     break;
   };
 
+  if (hton->discover_table_names)
+    my_atomic_add32(&engines_with_discover_table_names, 1);
+
   DBUG_RETURN(0);
 
 err_deinit:
@@ -4373,6 +4394,50 @@ int ha_discover(THD *thd, const char *db, const char *name,
 }
 
 
+/**
+  Discover all table names in a given database
+*/
+volatile int32 engines_with_discover_table_names= 0;
+
+struct st_discover_names_args
+{
+  LEX_STRING *db;
+  MY_DIR *dirp;
+  handlerton::discovered_list *result;
+};
+
+static my_bool discover_names(THD *thd, plugin_ref plugin,
+                              void *arg)
+{
+  st_discover_names_args *args= (st_discover_names_args *)arg;
+  handlerton *ht= plugin_data(plugin, handlerton *);
+  if (ht->state == SHOW_OPTION_YES && ht->discover_table_names &&
+      ht->discover_table_names(ht, args->db, args->dirp, args->result))
+    return 1;
+
+  return 0;
+}
+
+int ha_discover_table_names(THD *thd, LEX_STRING *db, MY_DIR *dirp,
+                            handlerton::discovered_list *result)
+{
+  int error;
+  DBUG_ENTER("ha_discover_table_names");
+  st_discover_names_args args= {db, dirp, result};
+
+  if (engines_with_discover_table_names == 0)
+    DBUG_RETURN(ext_table_discovery_simple(dirp, reg_ext, result));
+
+  error= extension_based_table_discovery(dirp, reg_ext, result);
+
+  if (!error)
+    error= plugin_foreach(thd, discover_names, MYSQL_STORAGE_ENGINE_PLUGIN,
+                          &args);
+
+  DBUG_RETURN(error);
+}
+
+
 #ifdef HAVE_NDB_BINLOG
 /*
   TODO: change this into a dynamic struct
diff --git a/sql/handler.h b/sql/handler.h
index 43c5246c156..93edfb2296e 100644
--- a/sql/handler.h
+++ b/sql/handler.h
@@ -1086,11 +1086,60 @@ struct handlerton
    ha_create_table_option *field_options; // these are specified per field
    ha_create_table_option *index_options; // these are specified per index
 
-   const char **tablefile_extensions;
+   /**
+     The list of extensions of files created for a single table in the
+     database directory (datadir/db_name/).
+
+     Used by open_table_error(), by the default rename_table and delete_table
+     handler methods, and by the default discovery implementation.
+  
+     For engines that have more than one file name extentions (separate
+     metadata, index, and/or data files), the order of elements is relevant.
+     First element of engine file name extentions array should be metadata
+     file extention. This is implied by the open_table_error()
+     and the default discovery implementation.
+     
+     Second element - data file extention. This is implied
+     assumed by REPAIR TABLE ... USE_FRM implementation.
+   */
+   const char **tablefile_extensions; // by default - empty list
+
+   /*********************************************************************
+     Table discovery API.
+     It allows the server to "discover" tables that exist in the storage
+     engine, without user issuing an explicit CREATE TABLE statement.
+   **********************************************************************/
+
+   /*
+     The discover_table_names method tells the server
+     about all tables in the specified database that the engine
+     knows about. Tables (or file names of tables) are added to
+     the provided discovered_list collector object using
+     add_table() or add_file() methods.
+   */
+   class discovered_list
+   {
+     public:
+     virtual bool add_table(const char *tname, size_t tlen) = 0;
+     virtual bool add_file(const char *fname) = 0;
+     protected: virtual ~discovered_list() {}
+   };
+
+   /*
+     By default (if not implemented by the engine, but the discovery_table() is
+     implemented) it will perform a file-based discovery:
+
+     - if tablefile_extensions[0] is not null, this will discovers all tables
+       with the tablefile_extensions[0] extension.
+
+     Returns 0 on success and 1 on error.
+   */
+   int (*discover_table_names)(handlerton *hton, LEX_STRING *db, MY_DIR *dir,
+                               discovered_list *result);
 };
 
 
-inline LEX_STRING *hton_name(const handlerton *hton)
+static inline LEX_STRING *hton_name(const handlerton *hton)
 {
   return &(hton2plugin[hton->slot]->name);
 }
@@ -2406,21 +2455,7 @@ public:
   virtual void free_foreign_key_create_info(char* str) {}
   /** The following can be called without an open handler */
   const char *table_type() const { return hton_name(ht)->str; }
-  /**
-    If frm_error() is called then we will use this to find out what file
-    extentions exist for the storage engine. This is also used by the default
-    rename_table and delete_table method in handler.cc.
-
-    For engines that have two file name extentions (separate meta/index file
-    and data file), the order of elements is relevant. First element of engine
-    file name extentions array should be meta/index file extention. Second
-    element - data file extention. This order is assumed by
-    prepare_for_repair() when REPAIR TABLE ... USE_FRM is issued.
-  */
-  const char **bas_ext() const
-  {
-    return ht->tablefile_extensions;
-  }
+  const char **bas_ext() const { return ht->tablefile_extensions; }
 
   virtual int get_default_no_partitions(HA_CREATE_INFO *create_info)
   { return 1;}
@@ -3037,6 +3072,11 @@ bool ha_check_if_table_exists(THD* thd, const char *db, const char *name,
                              bool *exists);
 int ha_discover(THD* thd, const char* dbname, const char* name,
                 uchar** frmblob, size_t* frmlen);
+int ha_discover_table_names(THD *thd, LEX_STRING *db, MY_DIR *dirp,
+                            handlerton::discovered_list *result);
+#ifdef MYSQL_SERVER
+extern volatile int32 engines_with_discover_table_names;
+#endif
 
 /* key cache */
 extern "C" int ha_init_key_cache(const char *name, KEY_CACHE *key_cache, void *);
diff --git a/sql/sql_show.cc b/sql/sql_show.cc
index 92be469becc..44736cd375a 100644
--- a/sql/sql_show.cc
+++ b/sql/sql_show.cc
@@ -687,6 +687,50 @@ db_name_is_in_ignore_db_dirs_list(const char *directory)
   return my_hash_search(&ignore_db_dirs_hash, (uchar *) buff, buff_len)!=NULL;
 }
 
+class Discovered_table_list: public handlerton::discovered_list
+{
+  THD *thd;
+  const char *wild;
+  size_t wild_length;
+  Dynamic_array<LEX_STRING*> *tables;
+
+public:
+  Discovered_table_list(THD *thd_arg, Dynamic_array<LEX_STRING*> *tables_arg,
+                        const char *wild_arg)
+  {
+    thd= thd_arg;
+    tables= tables_arg;
+    if (wild_arg && wild_arg[0])
+    {
+      wild= wild_arg;
+      wild_length= strlen(wild_arg);
+    }
+    else
+      wild= 0;
+  }
+  ~Discovered_table_list() {}
+
+  bool add_table(const char *tname, size_t tlen)
+  {
+    if (wild && my_wildcmp(files_charset_info, tname, tname + tlen,
+                           wild, wild + wild_length,
+                           wild_prefix, wild_one, wild_many))
+        return 0;
+
+    LEX_STRING *name= thd->make_lex_string(tname, tlen);
+    if (!name || tables->append(name))
+      return 1;
+    return 0;
+  }
+
+  bool add_file(const char *fname)
+  {
+    char tname[SAFE_NAME_LEN + 1];
+    size_t tlen= filename_to_tablename(fname, tname, sizeof(tname));
+    return add_table(tname, tlen);
+  }
+};
+
 extern "C" {
 static int cmp_table_names(LEX_STRING * const *a, LEX_STRING * const *b)
 {
@@ -709,11 +753,10 @@ enum find_files_result {
     find_files()
     thd                 thread handler
     files               put found files in this list
-    db                  database name to set in TABLE_LIST structure
+    db                  database name to search tables in
+                        or NULL to search for databases
     path                path to database
     wild                filter for found files
-    dir                 read databases in path if TRUE, read .frm files in
-                        database otherwise
 
   RETURN
     FIND_FILES_OK       success
@@ -723,60 +766,46 @@ enum find_files_result {
 
 
 static find_files_result
-find_files(THD *thd, Dynamic_array<LEX_STRING*> *files, const char *db,
-           const char *path, const char *wild, bool dir)
+find_files(THD *thd, Dynamic_array<LEX_STRING*> *files, LEX_STRING *db,
+           const char *path, const char *wild)
 {
-  uint i;
-  char *ext;
   MY_DIR *dirp;
-  FILEINFO *file;
-  LEX_STRING *file_name= 0;
-  uint file_name_len;
-#ifndef NO_EMBEDDED_ACCESS_CHECKS
-  uint col_access=thd->col_access;
-#endif
-  uint wild_length= 0;
-  TABLE_LIST table_list;
+  myf my_dir_flags= MY_THREAD_SPECIFIC;
+  Discovered_table_list tl(thd, files, wild);
   DBUG_ENTER("find_files");
 
-  if (wild)
-  {
-    if (!wild[0])
-      wild= 0;
-    else
-      wild_length= strlen(wild);
-  }
+  if (!db)
+    my_dir_flags|= MY_WANT_STAT;
 
-  bzero((char*) &table_list,sizeof(table_list));
+  if (engines_with_discover_table_names)
+    my_dir_flags|= MY_WANT_SORT;
 
-  if (!(dirp = my_dir(path, MYF((dir ? MY_WANT_STAT : 0) |
-                               MY_THREAD_SPECIFIC))))
+  if (!(dirp = my_dir(path, my_dir_flags)))
   {
     if (my_errno == ENOENT)
-      my_error(ER_BAD_DB_ERROR, MYF(ME_BELL | ME_WAITTANG), db);
+      my_error(ER_BAD_DB_ERROR, MYF(ME_BELL | ME_WAITTANG), db->str);
     else
       my_error(ER_CANT_READ_DIR, MYF(ME_BELL | ME_WAITTANG), path, my_errno);
     DBUG_RETURN(FIND_FILES_DIR);
   }
 
-  for (i=0 ; i < (uint) dirp->number_of_files  ; i++)
+  if (!db)                                           /* Return databases */
   {
-    char uname[SAFE_NAME_LEN + 1];              /* Unencoded name */
-    file=dirp->dir_entry+i;
-    if (dir)
-    {                                           /* Return databases */
+    for (uint i=0; i < (uint) dirp->number_of_files; i++)
+    {
+      FILEINFO *file= dirp->dir_entry+i;
 #ifdef USE_SYMDIR
       char *ext;
       char buff[FN_REFLEN];
       if (my_use_symdir && !strcmp(ext=fn_ext(file->name), ".sym"))
       {
-	/* Only show the sym file if it points to a directory */
-	char *end;
+        /* Only show the sym file if it points to a directory */
+        char *end;
         *ext=0;                                 /* Remove extension */
-	unpack_dirname(buff, file->name);
-	end= strend(buff);
-	if (end != buff && end[-1] == FN_LIBCHAR)
-	  end[-1]= 0;				// Remove end FN_LIBCHAR
+        unpack_dirname(buff, file->name);
+        end= strend(buff);
+        if (end != buff && end[-1] == FN_LIBCHAR)
+          end[-1]= 0;				// Remove end FN_LIBCHAR
         if (!mysql_file_stat(key_file_misc, buff, file->mystat, MYF(0)))
                continue;
        }
@@ -787,69 +816,26 @@ find_files(THD *thd, Dynamic_array<LEX_STRING*> *files, const char *db,
       if (is_in_ignore_db_dirs_list(file->name))
         continue;
 
-      file_name_len= filename_to_tablename(file->name, uname, sizeof(uname));
-      if (wild)
-      {
-	if (lower_case_table_names)
-	{
-          if (my_wildcmp(files_charset_info,
-                         uname, uname + file_name_len,
-                         wild, wild + wild_length,
-                         wild_prefix, wild_one, wild_many))
-            continue;
-	}
-	else if (wild_compare(uname, wild, 0))
-	  continue;
-      }
-    }
-    else
-    {
-        // Return only .frm files which aren't temp files.
-      if (my_strcasecmp(system_charset_info, ext=fn_rext(file->name),reg_ext) ||
-          is_prefix(file->name, tmp_file_prefix))
-        continue;
-      *ext=0;
-      file_name_len= filename_to_tablename(file->name, uname, sizeof(uname));
-      if (wild)
-      {
-	if (lower_case_table_names)
-	{
-          if (my_wildcmp(files_charset_info,
-                         uname, uname + file_name_len,
-                         wild, wild + wild_length,
-                         wild_prefix, wild_one,wild_many))
-            continue;
-	}
-	else if (wild_compare(uname, wild, 0))
-	  continue;
-      }
-    }
-#ifndef NO_EMBEDDED_ACCESS_CHECKS
-    /* Don't show tables where we don't have any privileges */
-    if (db && !(col_access & TABLE_ACLS))
-    {
-      table_list.db= (char*) db;
-      table_list.db_length= strlen(db);
-      table_list.table_name= uname;
-      table_list.table_name_length= file_name_len;
-      table_list.grant.privilege=col_access;
-      if (check_grant(thd, TABLE_ACLS, &table_list, TRUE, 1, TRUE))
-        continue;
-    }
-#endif
-    if (!(file_name= thd->make_lex_string(uname, file_name_len)) ||
-        files->append(file_name))
-    {
-      my_dirend(dirp);
-      DBUG_RETURN(FIND_FILES_OOM);
+      if (tl.add_file(file->name))
+        goto err;
     }
   }
+  else
+  {
+    if (ha_discover_table_names(thd, db, dirp, &tl))
+      goto err;
+  }
+
   DBUG_PRINT("info",("found: %d files", files->elements()));
   my_dirend(dirp);
 
   files->sort(cmp_table_names);
 
   DBUG_RETURN(FIND_FILES_OK);
+
+err:
+  my_dirend(dirp);
+  DBUG_RETURN(FIND_FILES_OOM);
 }
 
 
@@ -3729,8 +3715,8 @@ int make_db_list(THD *thd, Dynamic_array<LEX_STRING*> *files,
       if (files->append_val(&INFORMATION_SCHEMA_NAME))
         return 1;
     }
-    return (find_files(thd, files, NullS, mysql_data_home,
-                       lookup_field_vals->db_value.str, 1) != FIND_FILES_OK);
+    return find_files(thd, files, 0, mysql_data_home,
+                      lookup_field_vals->db_value.str);
   }
 
 
@@ -3761,8 +3747,7 @@ int make_db_list(THD *thd, Dynamic_array<LEX_STRING*> *files,
   */
   if (files->append_val(&INFORMATION_SCHEMA_NAME))
     return 1;
-  return (find_files(thd, files, NullS,
-                     mysql_data_home, NullS, 1) != FIND_FILES_OK);
+  return find_files(thd, files, 0, mysql_data_home, NullS);
 }
 
 
@@ -3905,8 +3890,8 @@ make_table_name_list(THD *thd, Dynamic_array<LEX_STRING*> *table_names,
     return (schema_tables_add(thd, table_names,
                               lookup_field_vals->table_value.str));
 
-  find_files_result res= find_files(thd, table_names, db_name->str, path,
-                                    lookup_field_vals->table_value.str, 0);
+  find_files_result res= find_files(thd, table_names, db_name, path,
+                                    lookup_field_vals->table_value.str);
   if (res != FIND_FILES_OK)
   {
     /*
@@ -4545,6 +4530,7 @@ int get_all_tables(THD *thd, TABLE_LIST *tables, COND *cond)
 {
   LEX *lex= thd->lex;
   TABLE *table= tables->table;
+  TABLE_LIST table_acl_check;
   SELECT_LEX *lsel= tables->schema_select_lex;
   ST_SCHEMA_TABLE *schema_table= tables->schema_table;
   LOOKUP_FIELD_VALUES lookup_field_vals;
@@ -4651,6 +4637,8 @@ int get_all_tables(THD *thd, TABLE_LIST *tables, COND *cond)
     goto err;
   }
 
+  bzero((char*) &table_acl_check, sizeof(table_acl_check));
+
   if (make_db_list(thd, &db_names, &lookup_field_vals))
     goto err;
   for (int i=0; i < db_names.elements(); i++)
@@ -4675,6 +4663,19 @@ int get_all_tables(THD *thd, TABLE_LIST *tables, COND *cond)
       for (int i=0; i < table_names.elements(); i++)
       {
         LEX_STRING *table_name= table_names.at(i);
+
+#ifndef NO_EMBEDDED_ACCESS_CHECKS
+        if (!(thd->col_access & TABLE_ACLS))
+        {
+          table_acl_check.db= db_name->str;
+          table_acl_check.db_length= db_name->length;
+          table_acl_check.table_name= table_name->str;
+          table_acl_check.table_name_length= table_name->length;
+          table_acl_check.grant.privilege= thd->col_access;
+          if (check_grant(thd, TABLE_ACLS, &table_acl_check, TRUE, 1, TRUE))
+            continue;
+        }
+#endif
 	restore_record(table, s->default_values);
         table->field[schema_table->idx_field1]->
           store(db_name->str, db_name->length, system_charset_info);
diff --git a/storage/archive/ha_archive.cc b/storage/archive/ha_archive.cc
index 02813ff8dc7..57d6fc947f8 100644
--- a/storage/archive/ha_archive.cc
+++ b/storage/archive/ha_archive.cc
@@ -105,7 +105,6 @@ static HASH archive_open_tables;
 /* The file extension */
 #define ARZ ".ARZ"               // The data file
 #define ARN ".ARN"               // Files used during an optimize call
-#define ARM ".ARM"               // Meta file (deprecated)
 
 /*
   uchar + uchar