WIP Append FILE_ records to ib_logfile0

file_op(): Replaces mtr_t::log_file_op().

File operations (or checkpoints) will be appended to ib_logfile0
without advancing the LSN, and these operations will never be atomic
with mini-transactions that write to the circular ib_logdata file.

7 files changed, 51 insertions, 124 deletions

diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc
index ba4cd1a551c..af2223ab0ac 100644
--- a/storage/innobase/fil/fil0crypt.cc
+++ b/storage/innobase/fil/fil0crypt.cc
@@ -1866,7 +1866,6 @@ fil_crypt_rotate_page(
 		ut_ad(!mtr.has_modifications());
 		ut_ad(!mtr.is_dirty());
 		ut_ad(mtr.get_memo()->size() == 0);
-		ut_ad(mtr.get_log()->size() == 0);
 		mtr.commit();
 	}
 
diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc
index aaecb34f1d1..07a6c1a713e 100644
--- a/storage/innobase/fil/fil0fil.cc
+++ b/storage/innobase/fil/fil0fil.cc
@@ -1402,8 +1402,8 @@ fil_space_t *fil_space_t::get(uint32_t id)
 @param first_page_no  first page number in the file
 @param path           file path
 @param new_path       new file path for type=FILE_RENAME */
-inline void mtr_t::log_file_op(mfile_type_t type, uint32_t space_id,
-			       const char *path, const char *new_path)
+static void file_op(mfile_type_t type, uint32_t space_id,
+                    const char *path, const char *new_path= nullptr)
 {
   ut_ad((new_path != nullptr) == (type == FILE_RENAME));
   ut_ad(!(byte(type) & 15));
@@ -1413,18 +1413,18 @@ inline void mtr_t::log_file_op(mfile_type_t type, uint32_t space_id,
   ut_ad(strchr(path, '/'));
   ut_ad(!strcmp(&path[strlen(path) - strlen(DOT_IBD)], DOT_IBD));
 
-  if (m_log_mode != MTR_LOG_ALL)
-    return;
-  m_last= nullptr;
-
   const size_t len= strlen(path);
   const size_t new_len= type == FILE_RENAME ? 1 + strlen(new_path) : 0;
   ut_ad(len > 0);
-  byte *const log_ptr= m_log.open(1 + 3/*length*/ + 5/*space_id*/ +
-                                  1/*page_no=0*/);
+  const size_t size= 1 + 3/*length*/ + 5/*space_id*/ + 4/*CRC-32C*/ +
+    len + new_len;
+
+  byte *log_ptr= static_cast<byte*>(ut_malloc_nokey(size));
+  if (UNIV_UNLIKELY(!log_ptr))
+    return;
+
   byte *end= log_ptr + 1;
   end= mlog_encode_varint(end, space_id);
-  *end++= 0;
   if (UNIV_LIKELY(end + len + new_len >= &log_ptr[16]))
   {
     *log_ptr= type;
@@ -1435,7 +1435,6 @@ inline void mtr_t::log_file_op(mfile_type_t type, uint32_t space_id,
       total_len++;
     end= mlog_encode_varint(log_ptr + 1, total_len);
     end= mlog_encode_varint(end, space_id);
-    *end++= 0;
   }
   else
   {
@@ -1443,42 +1442,22 @@ inline void mtr_t::log_file_op(mfile_type_t type, uint32_t space_id,
     ut_ad(*log_ptr & 15);
   }
 
-  m_log.close(end);
-
-  if (type == FILE_RENAME)
+  memcpy(end, path, len);
+  end+= len;
+  if (new_path)
   {
-    ut_ad(strchr(new_path, '/'));
-    m_log.push(reinterpret_cast<const byte*>(path), uint32_t(len + 1));
-    m_log.push(reinterpret_cast<const byte*>(new_path), uint32_t(new_len));
+    *end++= 0;
+    memcpy(end, path, new_len);
+    end+= new_len;
   }
-  else
-    m_log.push(reinterpret_cast<const byte*>(path), uint32_t(len));
-}
 
-/** Write redo log for renaming a file.
-@param[in]	space_id	tablespace id
-@param[in]	old_name	tablespace file name
-@param[in]	new_name	tablespace file name after renaming
-@param[in,out]	mtr		mini-transaction */
-static void fil_name_write_rename_low(uint32_t space_id, const char *old_name,
-                                      const char *new_name, mtr_t *mtr)
-{
-  ut_ad(!is_predefined_tablespace(space_id));
-  mtr->log_file_op(FILE_RENAME, space_id, old_name, new_name);
-}
-
-/** Write redo log for renaming a file.
-@param[in]	space_id	tablespace id
-@param[in]	old_name	tablespace file name
-@param[in]	new_name	tablespace file name after renaming */
-static void fil_name_write_rename(uint32_t space_id,
-				  const char *old_name, const char* new_name)
-{
-  mtr_t mtr;
-  mtr.start();
-  fil_name_write_rename_low(space_id, old_name, new_name, &mtr);
-  mtr.commit();
-  log_write_up_to(mtr.commit_lsn(), true);
+  mach_write_to_4(end, ut_crc32(log_ptr, end - log_ptr));
+  end+= 4;
+  ut_ad(end <= &log_ptr[size]);
+#if 0 /* FIXME: implement this! */
+  write(ib_logfile0, log_ptr, end - log_ptr);
+#endif
+  ut_free(log_ptr);
 }
 
 fil_space_t *fil_space_t::check_pending_operations(uint32_t id)
@@ -1593,11 +1572,7 @@ pfs_os_file_t fil_delete_tablespace(uint32_t id)
   if (fil_space_t *space= fil_space_t::check_pending_operations(id))
   {
     /* Before deleting the file(s), persistently write a log record. */
-    mtr_t mtr;
-    mtr.start();
-    mtr.log_file_op(FILE_DELETE, id, space->chain.start->name);
-    mtr.commit();
-    log_write_up_to(mtr.commit_lsn(), true);
+    file_op(FILE_DELETE, id, space->chain.start->name);
 
     /* Remove any additional files. */
     if (char *cfg_name= fil_make_filepath(space->chain.start->name,
@@ -1745,7 +1720,7 @@ char *fil_make_filepath(const char* path, const table_name_t name,
 dberr_t fil_space_t::rename(const char *path, bool log, bool replace)
 {
   ut_ad(UT_LIST_GET_LEN(chain) == 1);
-  ut_ad(!is_system_tablespace(id));
+  ut_ad(!is_predefined_tablespace(id));
 
   const char *old_path= chain.start->name;
 
@@ -1773,7 +1748,7 @@ dberr_t fil_space_t::rename(const char *path, bool log, bool replace)
       return DB_TABLESPACE_EXISTS;
     }
 
-    fil_name_write_rename(id, old_path, path);
+    file_op(FILE_RENAME, id, old_path, path);
   }
 
   return fil_rename_tablespace(id, old_path, path) ? DB_SUCCESS : DB_ERROR;
@@ -1791,7 +1766,8 @@ static bool fil_rename_tablespace(uint32_t id, const char *old_path,
 {
 	fil_space_t*	space;
 	fil_node_t*	node;
-	ut_a(id != 0);
+
+	ut_ad(!is_predefined_tablespace(id));
 
 	mysql_mutex_lock(&fil_system.mutex);
 
@@ -1883,7 +1859,6 @@ fil_ibd_create(
 {
 	pfs_os_file_t	file;
 	bool		success;
-	mtr_t		mtr;
 	bool		has_data_dir = FSP_FLAGS_HAS_DATA_DIR(flags) != 0;
 
 	ut_ad(!is_system_tablespace(space_id));
@@ -1899,10 +1874,7 @@ fil_ibd_create(
 		return NULL;
 	}
 
-	mtr.start();
-	mtr.log_file_op(FILE_CREATE, space_id, path);
-	mtr.commit();
-	log_write_up_to(mtr.commit_lsn(), true);
+	file_op(FILE_CREATE, space_id, path);
 
 	ulint type;
 	static_assert(((UNIV_ZIP_SIZE_MIN >> 1) << 3) == 4096,
@@ -1996,6 +1968,7 @@ err_exit:
 						     crypt_data, mode)) {
 		fil_node_t* node = space->add(path, file, size, false, true);
 		IF_WIN(node->find_metadata(), node->find_metadata(file, true));
+		mtr_t mtr;
 		mtr.start();
 		fsp_header_init(space, size, &mtr);
 		mtr.commit();
diff --git a/storage/innobase/include/mtr0log.h b/storage/innobase/include/mtr0log.h
index 066eca62bc3..a7d29649b37 100644
--- a/storage/innobase/include/mtr0log.h
+++ b/storage/innobase/include/mtr0log.h
@@ -384,8 +384,8 @@ template<byte type>
 inline byte *mtr_t::log_write(const page_id_t id, const buf_page_t *bpage,
                               size_t len, bool alloc, size_t offset)
 {
-  static_assert(!(type & 15) && type != RESERVED && type != OPTION &&
-                type <= FILE_CHECKPOINT, "invalid type");
+  static_assert(!(type & 0x8f) && type != RESERVED && type != OPTION,
+                "invalid type");
   ut_ad(!bpage || bpage->id() == id);
   constexpr bool have_len= type != INIT_PAGE && type != FREE_PAGE;
   constexpr bool have_offset= type == WRITE || type == MEMSET ||
diff --git a/storage/innobase/include/mtr0mtr.h b/storage/innobase/include/mtr0mtr.h
index 7b9dd477c3d..7f6a2dff98c 100644
--- a/storage/innobase/include/mtr0mtr.h
+++ b/storage/innobase/include/mtr0mtr.h
@@ -299,14 +299,6 @@ public:
 	/** @return true if a record was added to the mini-transaction */
 	bool is_dirty() const { return m_made_dirty; }
 
-	/** Get the buffered redo log of this mini-transaction.
-	@return	redo log */
-	const mtr_buf_t* get_log() const { return &m_log; }
-
-	/** Get the buffered redo log of this mini-transaction.
-	@return	redo log */
-	mtr_buf_t* get_log() { return &m_log; }
-
 	/** Push an object to an mtr memo stack.
 	@param object	object
 	@param type	object type: MTR_MEMO_S_LOCK, ... */
@@ -489,15 +481,6 @@ public:
   @param id       first page identifier that will not be in the file */
   inline void trim_pages(const page_id_t id);
 
-  /** Write a log record about a file operation.
-  @param type           file operation
-  @param space_id       tablespace identifier
-  @param path           file path
-  @param new_path       new file path for type=FILE_RENAME */
-  inline void log_file_op(mfile_type_t type, uint32_t space_id,
-                          const char *path,
-                          const char *new_path= nullptr);
-
   /** Add freed page numbers to freed_pages */
   void add_freed_offset(fil_space_t *space, uint32_t page)
   {
diff --git a/storage/innobase/include/mtr0types.h b/storage/innobase/include/mtr0types.h
index 019b8bace93..acf58140045 100644
--- a/storage/innobase/include/mtr0types.h
+++ b/storage/innobase/include/mtr0types.h
@@ -288,8 +288,10 @@ enum mrec_ext_t
 
 
 /** Redo log record types for file-level operations. These bit
-patterns will be written to redo log files, so the existing codes or
-their interpretation on crash recovery must not be changed. */
+patterns will be written to the append-only ib_logfile0 file,
+followed by CRC-32C of the record contents.
+Changing these codes or their interpretation on crash recovery
+may break compatibility! */
 enum mfile_type_t
 {
   /** Create a file. Followed by tablespace ID and the file name. */
@@ -299,20 +301,14 @@ enum mfile_type_t
   /** Rename a file. Followed by tablespace ID and the old file name,
   NUL, and the new file name.  */
   FILE_RENAME = 0xa0,
-  /** Modify a file. Followed by tablespace ID and the file name. */
-  FILE_MODIFY = 0xb0,
-#if 1 /* MDEV-14425 FIXME: Remove this! */
-  /** End-of-checkpoint marker. Followed by 2 dummy bytes of page identifier,
-  8 bytes of LSN, and padded with a NUL; @see SIZE_OF_CHECKPOINT. */
+  /** Identify a file that already existed when ib_logfile0 was created.
+  Followed by tablespace ID and the file name. */
+  FILE_ID = 0xb0,
+  /** Checkpoint identifier. Followed by 8 bytes of LSN and 6 bytes
+  of ib_logdata byte offset. */
   FILE_CHECKPOINT = 0xf0
-#endif
 };
 
-#if 1 /* MDEV-14425 FIXME: Remove this! */
-/** Size of a FILE_CHECKPOINT record, including the checksum. */
-constexpr byte SIZE_OF_CHECKPOINT= 3/*type,page_id*/ + 8/*LSN*/ + 4;
-#endif
-
 #ifndef UNIV_INNOCHECKSUM
 /** Types for the mlock objects to store in the mtr_t::m_memo */
 enum mtr_memo_type_t {
diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc
index 22df7fca821..5ba46694ae8 100644
--- a/storage/innobase/log/log0recv.cc
+++ b/storage/innobase/log/log0recv.cc
@@ -532,7 +532,7 @@ inline size_t log_phys_t::alloc_size(size_t len)
 
 /** Tablespace item during recovery */
 struct file_name_t {
-	/** Tablespace file name (FILE_MODIFY) */
+	/** Tablespace file name (FILE_ID) */
 	std::string	name;
 	/** Tablespace object (NULL if not valid or not found) */
 	fil_space_t*	space = nullptr;
@@ -1172,7 +1172,7 @@ fil_name_process(char* name, ulint len, uint32_t space_id,
 	      || srv_operation == SRV_OPERATION_RESTORE_EXPORT);
 
 	/* We will also insert space=NULL into the map, so that
-	further checks can ensure that a FILE_MODIFY record was
+	further checks can ensure that a FILE_ID record was
 	scanned before applying any page records for the space_id. */
 
 	const file_name_t fname(std::string(name, len), deleted);
@@ -1198,7 +1198,7 @@ fil_name_process(char* name, ulint len, uint32_t space_id,
 		}
 
 		ut_ad(f.space == NULL);
-	} else if (p.second // the first FILE_MODIFY or FILE_RENAME
+	} else if (p.second // the first FILE_ID or FILE_RENAME
 		   || f.name != fname.name) {
 		fil_space_t*	space;
 
@@ -2143,7 +2143,7 @@ static void store_freed_or_init_rec(page_id_t page_id, bool freed)
   switch (b & 0xf0) {
 # if 1 /* MDEV-14425 FIXME: Remove this! */
   case FILE_CHECKPOINT:
-    if (space_id == 0 && page_no == 0 && rlen == 8)
+    if (rlen == 8 + 6)
     {
       const lsn_t lsn= mach_read_from_8(l);
 
@@ -2180,9 +2180,11 @@ static void store_freed_or_init_rec(page_id_t page_id, bool freed)
     ib::warn() << "Ignoring malformed log record at LSN " << recovered_lsn;
     continue;
   case FILE_DELETE:
-  case FILE_MODIFY:
+  case FILE_ID:
   case FILE_RENAME:
-    if (UNIV_UNLIKELY(page_no != 0))
+  case FILE_CREATE:
+    if (UNIV_UNLIKELY(is_predefined_tablespace(space_id)))
+      goto file_rec_error;
     {
     file_rec_error:
       if (!srv_force_recovery)
@@ -2196,10 +2198,6 @@ static void store_freed_or_init_rec(page_id_t page_id, bool freed)
                  << recovered_lsn;
       continue;
     }
-    /* fall through */
-  case FILE_CREATE:
-    if (UNIV_UNLIKELY(space_id == 0))
-      goto file_rec_error;
     /* There is no terminating NUL character. Names must end in .ibd.
     For FILE_RENAME, there is a NUL between the two file names. */
     const char * const fn= reinterpret_cast<const char*>(l);
@@ -2220,26 +2218,6 @@ static void store_freed_or_init_rec(page_id_t page_id, bool freed)
         goto file_rec_error;
     }
 
-    if (page_no)
-    {
-      if (UNIV_UNLIKELY((b & 0xf0) != FILE_CREATE))
-        goto file_rec_error;
-      /* truncating an undo log tablespace */
-      ut_ad(fnend - fn >= 7);
-      ut_ad(!memcmp(fnend - 7, "undo", 4));
-      ut_d(char n[4]; char *end; memcpy(n, fnend - 3, 3); n[3]= 0);
-      ut_ad(strtoul(n, &end, 10) <= 127);
-      ut_ad(end == &n[3]);
-      ut_ad(page_no == SRV_UNDO_TABLESPACE_SIZE_IN_PAGES);
-      ut_ad(srv_is_undo_tablespace(space_id));
-      static_assert(UT_ARR_SIZE(truncated_undo_spaces) ==
-                    TRX_SYS_MAX_UNDO_SPACES, "compatibility");
-      truncated_undo_spaces[space_id - srv_undo_space_id_start]=
-        { recovered_lsn, page_no };
-      continue;
-    }
-    if (is_predefined_tablespace(space_id))
-      goto file_rec_error;
     if (fnend - fn < 4 || memcmp(fnend - 4, DOT_IBD, 4))
       goto file_rec_error;
 
@@ -2250,7 +2228,7 @@ static void store_freed_or_init_rec(page_id_t page_id, bool freed)
     if (fn2)
       fil_name_process(const_cast<char*>(fn2), fn2end - fn2, space_id,
                        false);
-    if ((b & 0xf0) < FILE_MODIFY && log_file_op)
+    if ((b & 0xf0) < FILE_ID && log_file_op)
       log_file_op(space_id, (b & 0xf0) == FILE_CREATE,
                   l, static_cast<ulint>(fnend - fn),
                   reinterpret_cast<const byte*>(fn2),
@@ -3868,9 +3846,7 @@ recv_init_crash_recovery_spaces(bool rescan, bool& missing_tablespace)
 					std::move(rs.second.freed_ranges));
 			}
 		} else if (rs.second.name == "") {
-			ib::error() << "Missing FILE_CREATE, FILE_DELETE"
-				" or FILE_MODIFY before FILE_CHECKPOINT"
-				" for tablespace " << rs.first;
+			ib::error() << "Unknown tablespace " << rs.first;
 			recv_sys.set_corrupt_log();
 			return(DB_CORRUPTION);
 		} else {
diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc
index aaf4d2512b6..09c8dd65f67 100644
--- a/storage/innobase/mtr/mtr0mtr.cc
+++ b/storage/innobase/mtr/mtr0mtr.cc
@@ -1123,7 +1123,7 @@ mtr_t::print() const
 {
 	ib::info() << "Mini-transaction handle: memo size "
 		<< m_memo.size() << " bytes log size "
-		<< get_log()->size() << " bytes";
+		<< m_log.size() << " bytes";
 }
 
 #endif /* UNIV_DEBUG */