MDEV-11412 Ensure that table is truly dropped when using DROP TABLE

The used code is largely based on code from Tencent

The problem is that in some rare cases there may be a conflict between .frm
files and the files in the storage engine. In this case the DROP TABLE
was not able to properly drop the table.

Some MariaDB/MySQL forks has solved this by adding a FORCE option to
DROP TABLE. After some discussion among MariaDB developers, we concluded
that users expects that DROP TABLE should always work, even if the
table would not be consistent. There should not be a need to use a
separate keyword to ensure that the table is really deleted.

The used solution is:
- If a .frm table doesn't exists, try dropping the table from all storage
  engines.
- If the .frm table exists but the table does not exist in the engine
  try dropping the table from all storage engines.
- Update storage engines using many table files (.CVS, MyISAM, Aria) to
  succeed with the drop even if some of the files are missing.
- Add HTON_AUTOMATIC_DELETE_TABLE to handlerton's where delete_table()
  is not needed and always succeed. This is used by ha_delete_table_force()
  to know which handlers to ignore when trying to drop a table without
  a .frm file.

The disadvantage of this solution is that a DROP TABLE on a non existing
table will be a bit slower as we have to ask all active storage engines
if they know anything about the table.

Other things:
- Added a new flag MY_IGNORE_ENOENT to my_delete() to not give an error
  if the file doesn't exist. This simplifies some of the code.
- Don't clear thd->error in ha_delete_table() if there was an active
  error. This is a bug fix.
- handler::delete_table() will not abort if first file doesn't exists.
  This is bug fix to handle the case when a drop table was aborted in
  the middle.
- Cleaned up mysql_rm_table_no_locks() to ensure that if_exists uses
  same code path as when it's not used.
- Use non_existing_Table_error() to detect if table didn't exists.
  Old code used different errors tests in different position.
- Table_triggers_list::drop_all_triggers() now drops trigger file if
  it can't be parsed instead of leaving it hanging around (bug fix)
- InnoDB doesn't anymore print error about .frm file out of sync with
  InnoDB directory if .frm file does not exists. This change was required
  to be able to try to drop an InnoDB file when .frm doesn't exists.
- Fixed bug in mi_delete_table() where the .MYD file would not be dropped
  if the .MYI file didn't exists.
- Fixed memory leak in Mroonga when deleting non existing table
- Fixed memory leak in Connect when deleting non existing table

Bugs fixed introduced by the original version of this commit:
MDEV-22826 Presence of Spider prevents tables from being force-deleted from
           other engines

8 files changed, 421 insertions, 173 deletions

diff --git a/sql/ha_sequence.cc b/sql/ha_sequence.cc
index 5362dd2fc73..4eabd820071 100644
--- a/sql/ha_sequence.cc
+++ b/sql/ha_sequence.cc
@@ -19,13 +19,13 @@
 #include "mariadb.h"
 #include "sql_list.h"
 #include "table.h"
+#include "sql_table.h"
 #include "sql_sequence.h"
 #include "ha_sequence.h"
 #include "sql_plugin.h"
 #include "mysql/plugin.h"
 #include "sql_priv.h"
 #include "sql_parse.h"
-#include "sql_table.h"
 #include "sql_update.h"
 #include "sql_base.h"
 #include "log_event.h"
@@ -380,6 +380,13 @@ static handler *sequence_create_handler(handlerton *hton,
                                         MEM_ROOT *mem_root)
 {
   DBUG_ENTER("sequence_create_handler");
+  if (unlikely(!share))
+  {
+    /*
+      This can happen if we call get_new_handler with a non existing share
+    */
+    DBUG_RETURN(0);
+  }
   DBUG_RETURN(new (mem_root) ha_sequence(hton, share));
 }
 
@@ -427,7 +434,8 @@ static int sequence_initialize(void *p)
                                HTON_HIDDEN |
                                HTON_TEMPORARY_NOT_SUPPORTED |
                                HTON_ALTER_NOT_SUPPORTED |
-                               HTON_NO_PARTITION);
+                               HTON_NO_PARTITION |
+                               HTON_AUTOMATIC_DELETE_TABLE);
   DBUG_RETURN(0);
 }
 
diff --git a/sql/handler.cc b/sql/handler.cc
index 64d7fae1f1a..ed1ac7356cc 100644
--- a/sql/handler.cc
+++ b/sql/handler.cc
@@ -148,6 +148,44 @@ TYPELIB tx_isolation_typelib= {array_elements(tx_isolation_names)-1,"",
 static TYPELIB known_extensions= {0,"known_exts", NULL, NULL};
 uint known_extensions_id= 0;
 
+
+class Table_exists_error_handler : public Internal_error_handler
+{
+public:
+  Table_exists_error_handler()
+    : m_handled_errors(0), m_unhandled_errors(0)
+  {}
+
+  bool handle_condition(THD *thd,
+                        uint sql_errno,
+                        const char* sqlstate,
+                        Sql_condition::enum_warning_level *level,
+                        const char* msg,
+                        Sql_condition ** cond_hdl)
+  {
+    *cond_hdl= NULL;
+    if (non_existing_table_error(sql_errno))
+    {
+      m_handled_errors++;
+      return TRUE;
+    }
+
+    if (*level == Sql_condition::WARN_LEVEL_ERROR)
+      m_unhandled_errors++;
+    return FALSE;
+  }
+
+  bool safely_trapped_errors()
+  {
+    return ((m_handled_errors > 0) && (m_unhandled_errors == 0));
+  }
+
+private:
+  int m_handled_errors;
+  int m_unhandled_errors;
+};
+
+
 static int commit_one_phase_2(THD *thd, bool all, THD_TRANS *trans,
                               bool is_real_trans);
 
@@ -2674,26 +2712,34 @@ const char *get_canonical_filename(handler *file, const char *path,
 }
 
 
-/** delete a table in the engine
+/**
+   Delete a table in the engine
+
+   @return 0   Table was deleted
+   @return -1  Table didn't exists, no error given
+   @return #   Error from table handler
 
   @note
   ENOENT and HA_ERR_NO_SUCH_TABLE are not considered errors.
-  The .frm file will be deleted only if we return 0.
+  The .frm file should be deleted by the caller only if we return <= 0.
 */
+
 int ha_delete_table(THD *thd, handlerton *table_type, const char *path,
-                    const LEX_CSTRING *db, const LEX_CSTRING *alias, bool generate_warning)
+                    const LEX_CSTRING *db, const LEX_CSTRING *alias,
+                    bool generate_warning)
 {
   handler *file;
   char tmp_path[FN_REFLEN];
   int error;
   TABLE dummy_table;
   TABLE_SHARE dummy_share;
+  bool is_error= thd->is_error();
   DBUG_ENTER("ha_delete_table");
 
   /* table_type is NULL in ALTER TABLE when renaming only .frm files */
   if (table_type == NULL || table_type == view_pseudo_hton ||
       ! (file=get_new_handler((TABLE_SHARE*)0, thd->mem_root, table_type)))
-    DBUG_RETURN(0);
+    DBUG_RETURN(-1);
 
   bzero((char*) &dummy_table, sizeof(dummy_table));
   bzero((char*) &dummy_share, sizeof(dummy_share));
@@ -2703,11 +2749,11 @@ int ha_delete_table(THD *thd, handlerton *table_type, const char *path,
   if (unlikely((error= file->ha_delete_table(path))))
   {
     /*
-      it's not an error if the table doesn't exist in the engine.
+      It's not an error if the table doesn't exist in the engine.
       warn the user, but still report DROP being a success
     */
-    bool intercept= (error == ENOENT || error == HA_ERR_NO_SUCH_TABLE ||
-                     error == HA_ERR_UNSUPPORTED);
+    bool intercept= non_existing_table_error(error);
+    DBUG_ASSERT(error > 0);
 
     if ((!intercept || generate_warning) && ! thd->is_error())
     {
@@ -2723,8 +2769,10 @@ int ha_delete_table(THD *thd, handlerton *table_type, const char *path,
     }
     if (intercept)
     {
-      thd->clear_error();
-      error= 0;
+      /* Clear error if we got it in this function */
+      if (!is_error)
+        thd->clear_error();
+      error= -1;
     }
   }
   delete file;
@@ -4368,45 +4416,64 @@ uint handler::get_dup_key(int error)
 
   @note
     We assume that the handler may return more extensions than
-    was actually used for the file.
+    was actually used for the file. We also assume that the first
+    extension is the most important one (see the comment near
+    handlerton::tablefile_extensions). If this exist and we can't delete
+    that it, we will abort the delete.
+    If the first one doesn't exists, we have to try to delete all other
+    extension as there is chance that the server had crashed between
+    the delete of the first file and the next
 
   @retval
     0   If we successfully deleted at least one file from base_ext and
-    didn't get any other errors than ENOENT
+        didn't get any other errors than ENOENT
+
   @retval
     !0  Error
 */
+
 int handler::delete_table(const char *name)
 {
-  int saved_error= 0;
-  int error= 0;
-  int enoent_or_zero;
+  int saved_error= ENOENT;
+  bool abort_if_first_file_error= 1;
+  bool some_file_deleted= 0;
+  DBUG_ENTER("handler::delete_table");
 
+  // For discovery tables, it's ok if first file doesn't exists
   if (ht->discover_table)
-    enoent_or_zero= 0; // the table may not exist in the engine, it's ok
-  else
-    enoent_or_zero= ENOENT;  // the first file of bas_ext() *must* exist
+  {
+    abort_if_first_file_error= 0;
+    saved_error= 0;
+    if (!bas_ext())
+    {
+      DBUG_ASSERT(ht->flags & HTON_AUTOMATIC_DELETE_TABLE);
+      DBUG_RETURN(0);                           // Drop succeded
+    }
+  }
 
-  for (const char **ext=bas_ext(); *ext ; ext++)
+  for (const char **ext= bas_ext(); *ext ; ext++)
   {
-    if (mysql_file_delete_with_symlink(key_file_misc, name, *ext, 0))
+    int error;
+    if ((error= mysql_file_delete_with_symlink(key_file_misc, name, *ext,
+                                              MYF(0))))
     {
       if (my_errno != ENOENT)
       {
+        saved_error= my_errno;
         /*
-          If error on the first existing file, return the error.
+          If error other than file not found on the first existing file,
+          return the error.
           Otherwise delete as much as possible.
         */
-        if (enoent_or_zero)
-          return my_errno;
-	saved_error= my_errno;
+        if (abort_if_first_file_error)
+          DBUG_RETURN(saved_error);
       }
     }
     else
-      enoent_or_zero= 0;                        // No error for ENOENT
-    error= enoent_or_zero;
+      some_file_deleted= 1;
+    abort_if_first_file_error= 0;
   }
-  return saved_error ? saved_error : error;
+  DBUG_RETURN(some_file_deleted && saved_error == ENOENT ? 0 : saved_error);
 }
 
 
@@ -4442,6 +4509,21 @@ void handler::drop_table(const char *name)
 
 
 /**
+   Return true if the error from drop table means that the
+   table didn't exists
+*/
+
+bool non_existing_table_error(int error)
+{
+  return (error == ENOENT || error == HA_ERR_NO_SUCH_TABLE ||
+          error == HA_ERR_UNSUPPORTED ||
+          error == ER_NO_SUCH_TABLE ||
+          error == ER_NO_SUCH_TABLE_IN_ENGINE ||
+          error == ER_WRONG_OBJECT);
+}
+
+
+/**
   Performs checks upon the table.
 
   @param thd                thread doing CHECK TABLE operation
@@ -4456,6 +4538,7 @@ void handler::drop_table(const char *name)
   @retval
     HA_ADMIN_NOT_IMPLEMENTED
 */
+
 int handler::ha_check(THD *thd, HA_CHECK_OPT *check_opt)
 {
   int error;
@@ -4902,6 +4985,88 @@ handler::ha_drop_table(const char *name)
 
 
 /**
+   Structure used during force drop table.
+*/
+
+struct st_force_drop_table_params
+{
+  const char *path;
+  const LEX_CSTRING *db;
+  const LEX_CSTRING *alias;
+  int error;
+};
+
+
+/**
+   Try to delete table from a given plugin
+   Table types with discovery is ignored as these .frm files would have
+   been created during discovery and thus doesn't need to be found
+   for drop table force
+*/
+
+static my_bool delete_table_force(THD *thd, plugin_ref plugin, void *arg)
+{
+  handlerton *hton = plugin_hton(plugin);
+  st_force_drop_table_params *param = (st_force_drop_table_params *)arg;
+
+  /*
+    We have to ignore HEAP tables as these may not have been created yet
+    We also remove engines that is using discovery (as these will recrate
+    any missing .frm if needed) and tables marked with
+    HTON_AUTOMATIC_DELETE_TABLE as for these we can't check if the table
+    ever existed.
+  */
+  if (!hton->discover_table && hton->db_type != DB_TYPE_HEAP &&
+      !(hton->flags & HTON_AUTOMATIC_DELETE_TABLE))
+  {
+    int error;
+    error= ha_delete_table(thd, hton, param->path, param->db,
+                           param->alias, 0);
+    if (error > 0 && !non_existing_table_error(error))
+      param->error= error;
+    if (error == 0)
+    {
+      param->error= 0;
+      return TRUE;                                // Table was deleted
+    }
+  }
+  return FALSE;
+}
+
+/**
+   @brief
+   Traverse all plugins to delete table when .frm file is missing.
+
+   @return -1  Table was not found in any engine
+   @return 0  Table was found in some engine and delete succeded
+   @return #  Error from first engine that had a table but didn't succeed to
+              delete the table
+   @return HA_ERR_ROW_IS_REFERENCED if foreign key reference is encountered,
+
+*/
+
+int ha_delete_table_force(THD *thd, const char *path, const LEX_CSTRING *db,
+                          const LEX_CSTRING *alias)
+{
+  st_force_drop_table_params param;
+  Table_exists_error_handler no_such_table_handler;
+  DBUG_ENTER("ha_delete_table_force");
+
+  param.path=             path;
+  param.db=               db;
+  param.alias=            alias;
+  param.error=            -1;                   // Table not found
+
+  thd->push_internal_handler(&no_such_table_handler);
+  if (plugin_foreach(thd, delete_table_force, MYSQL_STORAGE_ENGINE_PLUGIN,
+                     &param))
+    param.error= 0;                            // Delete succeded
+  thd->pop_internal_handler();
+  DBUG_RETURN(param.error);
+}
+
+
+/**
   Create a table in the engine: public interface.
 
   @sa handler::create()
@@ -5615,43 +5780,6 @@ static my_bool discover_existence(THD *thd, plugin_ref plugin,
   return ht->discover_table_existence(ht, args->db, args->table_name);
 }
 
-class Table_exists_error_handler : public Internal_error_handler
-{
-public:
-  Table_exists_error_handler()
-    : m_handled_errors(0), m_unhandled_errors(0)
-  {}
-
-  bool handle_condition(THD *thd,
-                        uint sql_errno,
-                        const char* sqlstate,
-                        Sql_condition::enum_warning_level *level,
-                        const char* msg,
-                        Sql_condition ** cond_hdl)
-  {
-    *cond_hdl= NULL;
-    if (sql_errno == ER_NO_SUCH_TABLE ||
-        sql_errno == ER_NO_SUCH_TABLE_IN_ENGINE ||
-        sql_errno == ER_WRONG_OBJECT)
-    {
-      m_handled_errors++;
-      return TRUE;
-    }
-
-    if (*level == Sql_condition::WARN_LEVEL_ERROR)
-      m_unhandled_errors++;
-    return FALSE;
-  }
-
-  bool safely_trapped_errors()
-  {
-    return ((m_handled_errors > 0) && (m_unhandled_errors == 0));
-  }
-
-private:
-  int m_handled_errors;
-  int m_unhandled_errors;
-};
 
 /**
   Check if a given table exists, without doing a full discover, if possible
@@ -5722,7 +5850,7 @@ bool ha_table_exists(THD *thd, const LEX_CSTRING *db,
 
       if ((type= dd_frm_type(thd, path, &engine, is_sequence)) ==
           TABLE_TYPE_UNKNOWN)
-        DBUG_RETURN(0);
+        DBUG_RETURN(true);                      // Frm exists
       
       if (type != TABLE_TYPE_VIEW)
       {
@@ -5730,8 +5858,10 @@ bool ha_table_exists(THD *thd, const LEX_CSTRING *db,
                                            MYSQL_STORAGE_ENGINE_PLUGIN);
         *hton= p ? plugin_hton(p) : NULL;
         if (*hton)
+        {
           // verify that the table really exists
           exists= discover_existence(thd, p, &args);
+        }
       }
       else
         *hton= view_pseudo_hton;
diff --git a/sql/handler.h b/sql/handler.h
index f8482501b3f..11731ae3749 100644
--- a/sql/handler.h
+++ b/sql/handler.h
@@ -1788,6 +1788,13 @@ handlerton *ha_default_tmp_handlerton(THD *thd);
 */
 #define HTON_TRANSACTIONAL_AND_NON_TRANSACTIONAL (1 << 17)
 
+/*
+  The engine doesn't keep track of tables, delete_table() is not
+  needed and delete_table() always returns 0 (table deleted). This flag
+  mainly used to skip storage engines in case of ha_delete_table_force()
+*/
+#define HTON_AUTOMATIC_DELETE_TABLE (1 << 18)
+
 class Ha_trx_info;
 
 struct THD_TRANS
@@ -5106,7 +5113,11 @@ int ha_create_table(THD *thd, const char *path,
                     const char *db, const char *table_name,
                     HA_CREATE_INFO *create_info, LEX_CUSTRING *frm);
 int ha_delete_table(THD *thd, handlerton *db_type, const char *path,
-                    const LEX_CSTRING *db, const LEX_CSTRING *alias, bool generate_warning);
+                    const LEX_CSTRING *db, const LEX_CSTRING *alias,
+                    bool generate_warning);
+int ha_delete_table_force(THD *thd, const char *path, const LEX_CSTRING *db,
+                          const LEX_CSTRING *alias);
+
 void ha_prepare_for_backup();
 void ha_end_backup();
 void ha_pre_shutdown();
@@ -5295,4 +5306,5 @@ void print_keydup_error(TABLE *table, KEY *key, myf errflag);
 int del_global_index_stat(THD *thd, TABLE* table, KEY* key_info);
 int del_global_table_stat(THD *thd, const  LEX_CSTRING *db, const LEX_CSTRING *table);
 uint ha_count_rw_all(THD *thd, Ha_trx_info **ptr_ha_info);
+bool non_existing_table_error(int error);
 #endif /* HANDLER_INCLUDED */
diff --git a/sql/log.cc b/sql/log.cc
index 0182d41431e..d5fcb853dc6 100644
--- a/sql/log.cc
+++ b/sql/log.cc
@@ -1701,7 +1701,10 @@ int binlog_init(void *p)
     // recover needs to be set to make xa{commit,rollback}_handlerton effective
     binlog_hton->recover= binlog_xa_recover_dummy;
   }
-  binlog_hton->flags= HTON_NOT_USER_SELECTABLE | HTON_HIDDEN | HTON_NO_ROLLBACK;
+  binlog_hton->flags= (HTON_NOT_USER_SELECTABLE |
+                       HTON_HIDDEN |
+                       HTON_NO_ROLLBACK |
+                       HTON_AUTOMATIC_DELETE_TABLE);
   return 0;
 }
 
diff --git a/sql/sql_table.cc b/sql/sql_table.cc
index 05bef5f4bda..2569665fc59 100644
--- a/sql/sql_table.cc
+++ b/sql/sql_table.cc
@@ -1107,7 +1107,7 @@ static int execute_ddl_log_action(THD *thd, DDL_LOG_ENTRY *ddl_log_entry)
   LEX_CSTRING handler_name;
   handler *file= NULL;
   MEM_ROOT mem_root;
-  int error= TRUE;
+  int error= 1;
   char to_path[FN_REFLEN];
   char from_path[FN_REFLEN];
   handlerton *hton;
@@ -1157,28 +1157,27 @@ static int execute_ddl_log_action(THD *thd, DDL_LOG_ENTRY *ddl_log_entry)
         {
           strxmov(to_path, ddl_log_entry->name, reg_ext, NullS);
           if (unlikely((error= mysql_file_delete(key_file_frm, to_path,
-                                                 MYF(MY_WME)))))
-          {
-            if (my_errno != ENOENT)
-              break;
-          }
+                                                 MYF(MY_WME |
+                                                     MY_IGNORE_ENOENT)))))
+            break;
 #ifdef WITH_PARTITION_STORAGE_ENGINE
           strxmov(to_path, ddl_log_entry->name, PAR_EXT, NullS);
-          (void) mysql_file_delete(key_file_partition_ddl_log, to_path, MYF(MY_WME));
+          (void) mysql_file_delete(key_file_partition_ddl_log, to_path,
+                                   MYF(0));
 #endif
         }
         else
         {
           if (unlikely((error= file->ha_delete_table(ddl_log_entry->name))))
           {
-            if (error != ENOENT && error != HA_ERR_NO_SUCH_TABLE)
+            if (!non_existing_table_error(error))
               break;
           }
         }
         if ((deactivate_ddl_log_entry_no_lock(ddl_log_entry->entry_pos)))
           break;
         (void) sync_ddl_log_no_lock();
-        error= FALSE;
+        error= 0;
         if (ddl_log_entry->action_type == DDL_LOG_DELETE_ACTION)
           break;
       }
@@ -2308,11 +2307,14 @@ int mysql_rm_table_no_locks(THD *thd, TABLE_LIST *tables, bool if_exists,
 
   for (table= tables; table; table= table->next_local)
   {
-    bool is_trans= 0;
-    bool table_creation_was_logged= 0;
+    bool is_trans= 0, frm_was_deleted= 0, temporary_table_was_dropped= 0;
+    bool table_creation_was_logged= 0, trigger_drop_executed= 0;
+    bool local_non_tmp_error= 0, frm_exists= 0, wrong_drop_sequence= 0;
+    bool drop_table_not_done= 0;
     LEX_CSTRING db= table->db;
     handlerton *table_type= 0;
 
+    error= 0;
     DBUG_PRINT("table", ("table_l: '%s'.'%s'  table: %p  s: %p",
                          table->db.str, table->table_name.str,  table->table,
                          table->table ?  table->table->s : NULL));
@@ -2327,36 +2329,42 @@ int mysql_rm_table_no_locks(THD *thd, TABLE_LIST *tables, bool if_exists,
                   thd->find_temporary_table(table) &&
                   table->mdl_request.ticket != NULL));
 
-    if (table->open_type == OT_BASE_ONLY || !is_temporary_table(table) ||
-        (drop_sequence && table->table->s->table_type != TABLE_TYPE_SEQUENCE))
-      error= 1;
-    else
+    /* First try to delete temporary tables and temporary sequences */
+    if ((table->open_type != OT_BASE_ONLY && is_temporary_table(table)) &&
+        (!drop_sequence || table->table->s->table_type == TABLE_TYPE_SEQUENCE))
     {
       table_creation_was_logged= table->table->s->table_creation_was_logged;
       if (thd->drop_temporary_table(table->table, &is_trans, true))
       {
+        /*
+          This is a very unlikely scenaro as dropping a temporary table
+          should always work. Would be better if we tried to drop all
+          temporary tables before giving the error.
+        */
         error= 1;
         goto err;
       }
-      error= 0;
       table->table= 0;
+      temporary_table_was_dropped= 1;
     }
 
-    if ((drop_temporary && if_exists) || !error)
+    if ((drop_temporary && if_exists) || temporary_table_was_dropped)
     {
       /*
         This handles the case of temporary tables. We have the following cases:
 
-          . "DROP TEMPORARY" was executed and a temporary table was affected
-          (i.e. drop_temporary && !error) or the if_exists was specified (i.e.
-          drop_temporary && if_exists).
-
-          . "DROP" was executed but a temporary table was affected (.i.e
-          !error).
+          - "DROP TEMPORARY" was executed and table was dropped
+            temporary_table_was_dropped == 1
+          - "DROP TEMPORARY IF EXISTS" was specified but no temporary table
+            existed
+            temporary_table_was_dropped == 0
       */
       if (!dont_log_query && table_creation_was_logged)
       {
         /*
+          DROP TEMPORARY succeded. For the moment when we only come
+          here on success (error == 0)
+
           If there is an error, we don't know the type of the engine
           at this point. So, we keep it in the trx-cache.
         */
@@ -2387,7 +2395,8 @@ int mysql_rm_table_no_locks(THD *thd, TABLE_LIST *tables, bool if_exists,
         is no need to proceed with the code that tries to drop a regular
         table.
       */
-      if (!error) continue;
+      if (temporary_table_was_dropped)
+        continue;
     }
     else if (!drop_temporary)
     {
@@ -2402,48 +2411,33 @@ int mysql_rm_table_no_locks(THD *thd, TABLE_LIST *tables, bool if_exists,
       path_length= build_table_filename(path, sizeof(path) - 1, db.str,
                                         alias.str, reg_ext, 0);
     }
+
     DEBUG_SYNC(thd, "rm_table_no_locks_before_delete_table");
     error= 0;
-    if (drop_temporary ||
-        (ha_table_exists(thd, &db, &alias, &table_type, &is_sequence) == 0 &&
-         table_type == 0) ||
-        (!drop_view && (was_view= (table_type == view_pseudo_hton))) ||
-        (drop_sequence && !is_sequence))
+    if (drop_temporary)
+    {
+      /* "DROP TEMPORARY" but a temporary table was not found */
+      error= ENOENT;
+    }
+    else if (((frm_exists= ha_table_exists(thd, &db, &alias, &table_type,
+                                           &is_sequence)) == 0 &&
+              table_type == 0) ||
+             (!drop_view && (was_view= (table_type == view_pseudo_hton))) ||
+             (drop_sequence && !is_sequence))
     {
       /*
         One of the following cases happened:
-          . "DROP TEMPORARY" but a temporary table was not found.
           . "DROP" but table was not found
           . "DROP TABLE" statement, but it's a view.
           . "DROP SEQUENCE", but it's not a sequence
       */
-      was_table= drop_sequence && table_type;
-      if (if_exists)
-      {
-        char buff[FN_REFLEN];
-        int err= (drop_sequence ? ER_UNKNOWN_SEQUENCES :
-                  ER_BAD_TABLE_ERROR);
-        String tbl_name(buff, sizeof(buff), system_charset_info);
-        tbl_name.length(0);
-        tbl_name.append(&db);
-        tbl_name.append('.');
-        tbl_name.append(&table->table_name);
-        push_warning_printf(thd, Sql_condition::WARN_LEVEL_NOTE,
-                            err, ER_THD(thd, err),
-                            tbl_name.c_ptr_safe());
-
-        /*
-          Our job is done here. This statement was added to avoid executing
-          unnecessary code farther below which in some strange corner cases
-          caused the server to crash (see MDEV-17896).
-        */
-        goto log_query;
-      }
-      else
-      {
-        non_tmp_error = (drop_temporary ? non_tmp_error : TRUE);
-        error= 1;
-      }
+      wrong_drop_sequence= drop_sequence && table_type;
+      was_table|= wrong_drop_sequence;
+      local_non_tmp_error= 1;
+      error= -1;
+      if ((!frm_exists && !table_type) ||       // no .frm
+          if_exists)
+        error= ENOENT;
     }
     else
     {
@@ -2500,8 +2494,12 @@ int mysql_rm_table_no_locks(THD *thd, TABLE_LIST *tables, bool if_exists,
         log_if_exists= 1;
 
       thd->replication_flags= 0;
-      if ((error= ha_delete_table(thd, table_type, path, &db,
-                                  &table->table_name, !dont_log_query)))
+      error= ha_delete_table(thd, table_type, path, &db,
+                             &table->table_name, !dont_log_query);
+
+      if (error < 0)                            // Table didn't exists
+        error= 0;
+      if (error)
       {
         if (thd->is_killed())
         {
@@ -2509,49 +2507,137 @@ int mysql_rm_table_no_locks(THD *thd, TABLE_LIST *tables, bool if_exists,
           goto err;
         }
       }
-      else
+
+      /*
+        Delete the .frm file if we managed to delete the table from the
+        engine or the table didn't exists in the engine
+      */
+      if (likely(!error) || non_existing_table_error(error))
       {
         /* Delete the table definition file */
         strmov(end,reg_ext);
         if (table_type && table_type != view_pseudo_hton &&
-            table_type->discover_table)
+            (table_type->discover_table || error))
         {
           /*
-            Table type is using discovery and may not need a .frm file.
+            Table type is using discovery and may not need a .frm file
+            or the .frm file existed but no table in engine.
             Delete it silently if it exists
           */
-          (void) mysql_file_delete(key_file_frm, path, MYF(0));
+          if (mysql_file_delete(key_file_frm, path,
+                                MYF(MY_WME | MY_IGNORE_ENOENT)))
+            error= my_errno;
         }
         else if (unlikely(mysql_file_delete(key_file_frm, path,
-                                            MYF(MY_WME))))
+                                            !error ? MYF(MY_WME) :
+                                            MYF(MY_WME | MY_IGNORE_ENOENT))))
         {
           frm_delete_error= my_errno;
           DBUG_ASSERT(frm_delete_error);
         }
       }
+      frm_was_deleted= 1;
       if (thd->replication_flags & OPTION_IF_EXISTS)
         log_if_exists= 1;
 
-      if (likely(!error))
+      if (frm_delete_error)
       {
-        int trigger_drop_error= 0;
+        /*
+          Remember error if unexpected error from dropping the .frm file
+          or we got an error from ha_delete_table()
+        */
+        if (frm_delete_error != ENOENT)
+          error= frm_delete_error;
+        else if (if_exists && ! error)
+          thd->clear_error();
+      }
+      if (likely(!error) || !frm_delete_error)
+        non_tmp_table_deleted= TRUE;
+
+      if (likely(!error) || non_existing_table_error(error))
+      {
+        trigger_drop_executed= 1;
+
+        if (Table_triggers_list::drop_all_triggers(thd, &db,
+                                                   &table->table_name,
+                                                   MYF(MY_WME |
+                                                       MY_IGNORE_ENOENT)))
+          error= error ? error : -1;
+      }
+      local_non_tmp_error|= MY_TEST(error);
+    }
+
+    /*
+      If there was no .frm file and the table is not temporary,
+      scan all engines try to drop the table from there.
+      This is to ensure we don't have any partial table files left.
+
+      We check for trigger_drop_executed to ensure we don't again try
+      to drop triggers when it failed above (after sucecssfully dropping
+      the table).
+    */
+    if (non_existing_table_error(error) && !drop_temporary &&
+        table_type != view_pseudo_hton && !trigger_drop_executed &&
+        !wrong_drop_sequence)
+    {
+      char *end;
+      int ferror= 0;
+
+      /* Remove extension for delete */
+      *(end = path + path_length - reg_ext_length) = '\0';
+      ferror= ha_delete_table_force(thd, path, &db, &table->table_name);
+      if (!ferror)
+      {
+        /* Table existed and was deleted */
+        non_tmp_table_deleted= TRUE;
+        local_non_tmp_error= 0;
+        error= 0;
+      }
+      if (ferror <= 0)
+      {
+        ferror= 0;                              // Ignore table not found
 
-        if (likely(!frm_delete_error))
+        /* Delete the table definition file */
+        if (!frm_was_deleted)
         {
-          non_tmp_table_deleted= TRUE;
-          trigger_drop_error=
-            Table_triggers_list::drop_all_triggers(thd, &db,
-                                                   &table->table_name);
+          strmov(end, reg_ext);
+          if (mysql_file_delete(key_file_frm, path,
+                                MYF(MY_WME | MY_IGNORE_ENOENT)))
+            ferror= my_errno;
         }
-
-        if (unlikely(trigger_drop_error) ||
-            (frm_delete_error && frm_delete_error != ENOENT))
-          error= 1;
-        else if (frm_delete_error && if_exists)
-          thd->clear_error();
+        if (Table_triggers_list::drop_all_triggers(thd, &db,
+                                                   &table->table_name,
+                                                   MYF(MY_WME |
+                                                       MY_IGNORE_ENOENT)))
+          ferror= -1;
       }
-      non_tmp_error|= MY_TEST(error);
+      if (!error)
+        error= ferror;
+    }
+
+    /*
+      Don't give an error if we are using IF EXISTS for a table that
+      didn't exists
+    */
+
+    if (if_exists && non_existing_table_error(error))
+    {
+      char buff[FN_REFLEN];
+      int err= (drop_sequence ? ER_UNKNOWN_SEQUENCES :
+                ER_BAD_TABLE_ERROR);
+      String tbl_name(buff, sizeof(buff), system_charset_info);
+      tbl_name.length(0);
+      tbl_name.append(&db);
+      tbl_name.append('.');
+      tbl_name.append(&table->table_name);
+      push_warning_printf(thd, Sql_condition::WARN_LEVEL_NOTE,
+                          err, ER_THD(thd, err),
+                          tbl_name.c_ptr_safe());
+      error= 0;
+      local_non_tmp_error= 0;
+      drop_table_not_done= 1;
     }
+    non_tmp_error|= local_non_tmp_error;
 
     if (error)
     {
@@ -2562,14 +2648,14 @@ int mysql_rm_table_no_locks(THD *thd, TABLE_LIST *tables, bool if_exists,
       wrong_tables.append(&table->table_name);
       errors++;
     }
-    else
+    else if (!drop_table_not_done)
     {
-      PSI_CALL_drop_table_share(false, table->db.str, (uint)table->db.length,
+      PSI_CALL_drop_table_share(temporary_table_was_dropped,
+                                table->db.str, (uint)table->db.length,
                                 table->table_name.str, (uint)table->table_name.length);
       mysql_audit_drop_table(thd, table);
     }
 
-log_query:
     if (!dont_log_query && !drop_temporary)
     {
       non_tmp_table_deleted= (if_exists ? TRUE : non_tmp_table_deleted);
@@ -2727,9 +2813,10 @@ err:
   }
 
 end:
-  DBUG_RETURN(error);
+  DBUG_RETURN(error || thd->is_error());
 }
 
+
 /**
   Log the drop of a table.
 
@@ -2809,7 +2896,8 @@ bool quick_rm_table(THD *thd, handlerton *base, const LEX_CSTRING *db,
     delete file;
   }
   if (!(flags & (FRM_ONLY|NO_HA_TABLE)))
-    error|= ha_delete_table(thd, base, path, db, table_name, 0);
+    if (ha_delete_table(thd, base, path, db, table_name, 0) > 0)
+    error= 1;
 
   if (likely(error == 0))
   {
diff --git a/sql/sql_trigger.cc b/sql/sql_trigger.cc
index 0b2d2d8ca1c..779faa96f82 100644
--- a/sql/sql_trigger.cc
+++ b/sql/sql_trigger.cc
@@ -1029,10 +1029,10 @@ bool Trigger::add_to_file_list(void* param_arg)
 */
 
 static bool rm_trigger_file(char *path, const LEX_CSTRING *db,
-                            const LEX_CSTRING *table_name)
+                            const LEX_CSTRING *table_name, myf MyFlags)
 {
   build_table_filename(path, FN_REFLEN-1, db->str, table_name->str, TRG_EXT, 0);
-  return mysql_file_delete(key_file_trg, path, MYF(MY_WME));
+  return mysql_file_delete(key_file_trg, path, MyFlags);
 }
 
 
@@ -1051,10 +1051,11 @@ static bool rm_trigger_file(char *path, const LEX_CSTRING *db,
 */
 
 static bool rm_trigname_file(char *path, const LEX_CSTRING *db,
-                             const LEX_CSTRING *trigger_name)
+                       const LEX_CSTRING *trigger_name, myf MyFlags)
 {
-  build_table_filename(path, FN_REFLEN - 1, db->str, trigger_name->str, TRN_EXT, 0);
-  return mysql_file_delete(key_file_trn, path, MYF(MY_WME));
+  build_table_filename(path, FN_REFLEN - 1, db->str, trigger_name->str,
+                       TRN_EXT, 0);
+  return mysql_file_delete(key_file_trn, path, MyFlags);
 }
 
 
@@ -1172,7 +1173,7 @@ bool Table_triggers_list::drop_trigger(THD *thd, TABLE_LIST *tables,
       parse_file.cc functionality (because we will need it
       elsewhere).
     */
-    if (rm_trigger_file(path, &tables->db, &tables->table_name))
+    if (rm_trigger_file(path, &tables->db, &tables->table_name, MYF(MY_WME)))
       return 1;
   }
   else
@@ -1181,7 +1182,7 @@ bool Table_triggers_list::drop_trigger(THD *thd, TABLE_LIST *tables,
       return 1;
   }
 
-  if (rm_trigname_file(path, &tables->db, sp_name))
+  if (rm_trigname_file(path, &tables->db, sp_name, MYF(MY_WME)))
     return 1;
 
   delete trigger;
@@ -1322,9 +1323,9 @@ bool Table_triggers_list::prepare_record_accessors(TABLE *table)
     This could be avoided if there is no triggers for UPDATE and DELETE.
 
   @retval
-    False   success
+    False   no triggers or triggers where correctly loaded
   @retval
-    True    error
+    True    error (wrong trigger file)
 */
 
 bool Table_triggers_list::check_n_load(THD *thd, const LEX_CSTRING *db,
@@ -1638,7 +1639,7 @@ err_with_lex_cleanup:
   }
 
 error:
-    if (unlikely(!thd->is_error()))
+  if (unlikely(!thd->is_error()))
   {
     /*
       We don't care about this error message much because .TRG files will
@@ -1815,7 +1816,8 @@ bool add_table_for_trigger(THD *thd,
 */
 
 bool Table_triggers_list::drop_all_triggers(THD *thd, const LEX_CSTRING *db,
-                                            const LEX_CSTRING *name)
+                                            const LEX_CSTRING *name,
+                                            myf MyFlags)
 {
   TABLE table;
   char path[FN_REFLEN];
@@ -1824,11 +1826,13 @@ bool Table_triggers_list::drop_all_triggers(THD *thd, const LEX_CSTRING *db,
 
   table.reset();
   init_sql_alloc(key_memory_Table_trigger_dispatcher,
-                 &table.mem_root, 8192, 0, MYF(0));
+                 &table.mem_root, 8192, 0, MYF(MY_WME));
 
   if (Table_triggers_list::check_n_load(thd, db, name, &table, 1))
   {
     result= 1;
+    /* We couldn't parse trigger file, best to just remove it */
+    rm_trigger_file(path, db, name, MyFlags);
     goto end;
   }
   if (table.triggers)
@@ -1848,7 +1852,7 @@ bool Table_triggers_list::drop_all_triggers(THD *thd, const LEX_CSTRING *db,
             Such triggers have zero-length name and are skipped here.
           */
           if (trigger->name.length &&
-              rm_trigname_file(path, db, &trigger->name))
+              rm_trigname_file(path, db, &trigger->name, MyFlags))
           {
             /*
               Instead of immediately bailing out with error if we were unable
@@ -1862,7 +1866,7 @@ bool Table_triggers_list::drop_all_triggers(THD *thd, const LEX_CSTRING *db,
         }
       }
     }
-    if (rm_trigger_file(path, db, name))
+    if (rm_trigger_file(path, db, name, MyFlags))
       result= 1;
     delete table.triggers;
   }
@@ -1923,9 +1927,10 @@ change_table_name_in_triggers(THD *thd,
   if (save_trigger_file(thd, new_db_name, new_table_name))
     return TRUE;
 
-  if (rm_trigger_file(path_buff, old_db_name, old_table_name))
+  if (rm_trigger_file(path_buff, old_db_name, old_table_name, MYF(MY_WME)))
   {
-    (void) rm_trigger_file(path_buff, new_db_name, new_table_name);
+    (void) rm_trigger_file(path_buff, new_db_name, new_table_name,
+                           MYF(MY_WME));
     return TRUE;
   }
   return FALSE;
@@ -2034,9 +2039,11 @@ bool Trigger::change_on_table_name(void* param_arg)
   /* Remove stale .TRN file in case of database upgrade */
   if (param->old_db_name)
   {
-    if (rm_trigname_file(trigname_buff, param->old_db_name, &name))
+    if (rm_trigname_file(trigname_buff, param->old_db_name, &name,
+                         MYF(MY_WME)))
     {
-      (void) rm_trigname_file(trigname_buff, param->new_db_name, &name);
+      (void) rm_trigname_file(trigname_buff, param->new_db_name, &name,
+                              MYF(MY_WME));
       return 1;
     }
   }
diff --git a/sql/sql_trigger.h b/sql/sql_trigger.h
index ae3d1738b16..040d8eba989 100644
--- a/sql/sql_trigger.h
+++ b/sql/sql_trigger.h
@@ -231,7 +231,7 @@ public:
   static bool check_n_load(THD *thd, const LEX_CSTRING *db, const LEX_CSTRING *table_name,
                            TABLE *table, bool names_only);
   static bool drop_all_triggers(THD *thd, const LEX_CSTRING *db,
-                                const LEX_CSTRING *table_name);
+                                const LEX_CSTRING *table_name, myf MyFlags);
   static bool change_table_name(THD *thd, const LEX_CSTRING *db,
                                 const LEX_CSTRING *old_alias,
                                 const LEX_CSTRING *old_table,
diff --git a/sql/temporary_tables.cc b/sql/temporary_tables.cc
index 203571441fc..a8d8113945b 100644
--- a/sql/temporary_tables.cc
+++ b/sql/temporary_tables.cc
@@ -698,10 +698,10 @@ bool THD::rm_temporary_table(handlerton *base, const char *path)
   char frm_path[FN_REFLEN + 1];
 
   strxnmov(frm_path, sizeof(frm_path) - 1, path, reg_ext, NullS);
-  if (mysql_file_delete(key_file_frm, frm_path, MYF(0)))
-  {
+  if (mysql_file_delete(key_file_frm, frm_path,
+                        MYF(MY_WME | MY_IGNORE_ENOENT)))
     error= true;
-  }
+
   file= get_new_handler((TABLE_SHARE*) 0, current_thd->mem_root, base);
   if (file && file->ha_delete_table(path))
   {