MDEV-13564 Mariabackup does not work with TRUNCATE

This is a merge from 10.2, but the 10.2 version of this will not
be pushed into 10.2 yet, because the 10.2 version would include
backports of MDEV-14717 and MDEV-14585, which would introduce
a crash recovery regression: Tables could be lost on
table-rebuilding DDL operations, such as ALTER TABLE,
OPTIMIZE TABLE or this new backup-friendly TRUNCATE TABLE.
The test innodb.truncate_crash occasionally loses the table due to
the following bug:

MDEV-17158 log_write_up_to() sometimes fails

10 files changed, 71 insertions, 119 deletions

diff --git a/storage/innobase/include/buf0lru.h b/storage/innobase/include/buf0lru.h
index dd7129a86ac..d3e953ad9c7 100644
--- a/storage/innobase/include/buf0lru.h
+++ b/storage/innobase/include/buf0lru.h
@@ -65,8 +65,10 @@ bool buf_LRU_drop_page_hash_for_tablespace(dict_table_t* table)
 /** Empty the flush list for all pages belonging to a tablespace.
 @param[in]	id		tablespace identifier
 @param[in,out]	observer	flush observer,
-				or NULL if nothing is to be written */
-void buf_LRU_flush_or_remove_pages(ulint id, FlushObserver* observer);
+				or NULL if nothing is to be written
+@param[in]	first		first page to be flushed or evicted */
+void buf_LRU_flush_or_remove_pages(ulint id, FlushObserver* observer,
+				   ulint first = 0);
 
 #if defined UNIV_DEBUG || defined UNIV_BUF_DEBUG
 /********************************************************************//**
diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h
index eb00239ca4e..890684af67e 100644
--- a/storage/innobase/include/fil0fil.h
+++ b/storage/innobase/include/fil0fil.h
@@ -97,10 +97,8 @@ struct fil_space_t {
 				new write operations because we don't
 				check this flag when doing flush
 				batches. */
+	/** whether undo tablespace truncation is in progress */
 	bool		is_being_truncated;
-				/*!< this is set to true when we prepare to
-				truncate a single-table tablespace and its
-				.ibd file */
 #ifdef UNIV_DEBUG
 	ulint		redo_skipped_count;
 				/*!< reference count for operations who want
@@ -181,12 +179,8 @@ struct fil_space_t {
 
 	ulint		magic_n;/*!< FIL_SPACE_MAGIC_N */
 
-	/** @return whether the tablespace is about to be dropped or
-	truncated */
-	bool is_stopping() const
-	{
-		return stop_new_ops || is_being_truncated;
-	}
+	/** @return whether the tablespace is about to be dropped */
+	bool is_stopping() const { return stop_new_ops;	}
 
 	/** @return whether doublewrite buffering is needed */
 	bool use_doublewrite() const
@@ -881,7 +875,8 @@ fil_op_replay_rename(
 not (necessarily) protected by meta-data locks.
 (Rollback would generally be protected, but rollback of
 FOREIGN KEY CASCADE/SET NULL is not protected by meta-data locks
-but only by InnoDB table locks, which may be broken by TRUNCATE TABLE.)
+but only by InnoDB table locks, which may be broken by
+lock_remove_all_on_table().)
 @param[in]	table	persistent table
 checked @return whether the table is accessible */
 bool
@@ -899,22 +894,15 @@ fil_delete_tablespace(
 #endif /* BTR_CUR_HASH_ADAPT */
 	);
 
-/** Truncate the tablespace to needed size.
-@param[in,out]	space		tablespace truncate
-@param[in]	size_in_pages	truncate size.
-@return true if truncate was successful. */
-bool fil_truncate_tablespace(fil_space_t* space, ulint size_in_pages);
+/** Prepare to truncate an undo tablespace.
+@param[in]	space_id	undo tablespace id
+@return	the tablespace
+@retval	NULL if the tablespace does not exist */
+fil_space_t* fil_truncate_prepare(ulint space_id);
 
-/*******************************************************************//**
-Prepare for truncating a single-table tablespace. The tablespace
-must be cached in the memory cache.
-1) Check pending operations on a tablespace;
-2) Remove all insert buffer entries for the tablespace;
-@return DB_SUCCESS or error */
-dberr_t
-fil_prepare_for_truncate(
-/*=====================*/
-	ulint	id);			/*!< in: space id */
+/** Write log about an undo tablespace truncate operation. */
+void fil_truncate_log(fil_space_t* space, ulint size, mtr_t* mtr)
+	MY_ATTRIBUTE((nonnull));
 
 /*******************************************************************//**
 Closes a single-table tablespace. The tablespace must be cached in the
diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h
index fd8295cfe12..b215ba34a77 100644
--- a/storage/innobase/include/log0log.h
+++ b/storage/innobase/include/log0log.h
@@ -450,10 +450,12 @@ to this checkpoint, or 0 if the information has not been written */
 This used to be called LOG_GROUP_ID and always written as 0,
 because InnoDB never supported more than one copy of the redo log. */
 #define LOG_HEADER_FORMAT	0
-/** 4 unused (zero-initialized) bytes. In format version 0, the
+/** Redo log subformat (originally 0). In format version 0, the
 LOG_FILE_START_LSN started here, 4 bytes earlier than LOG_HEADER_START_LSN,
-which the LOG_FILE_START_LSN was renamed to. */
-#define LOG_HEADER_PAD1		4
+which the LOG_FILE_START_LSN was renamed to.
+Subformat 1 is for the fully redo-logged TRUNCATE
+(no MLOG_TRUNCATE records or extra log checkpoints or log files) */
+#define LOG_HEADER_SUBFORMAT	4
 /** LSN of the start of data in this log file (with format version 1;
 in format version 0, it was called LOG_FILE_START_LSN and at offset 4). */
 #define LOG_HEADER_START_LSN	8
@@ -474,11 +476,18 @@ or the MySQL version that created the redo log file. */
 #define LOG_HEADER_FORMAT_3_23		0
 /** The MySQL 5.7.9/MariaDB 10.2.2 log format */
 #define LOG_HEADER_FORMAT_10_2		1
-/** The MariaDB 10.3.2 log format */
+/** The MariaDB 10.3.2 log format.
+To prevent crash-downgrade to earlier 10.2 due to the inability to
+roll back a retroactively introduced TRX_UNDO_RENAME_TABLE undo log record,
+MariaDB 10.2.18 and later will use the 10.3 format, but LOG_HEADER_SUBFORMAT
+1 instead of 0. MariaDB 10.3 will use subformat 0 (5.7-style TRUNCATE) or 2
+(MDEV-13564 backup-friendly TRUNCATE). */
 #define LOG_HEADER_FORMAT_10_3		103
 /** The redo log format identifier corresponding to the current format version.
 Stored in LOG_HEADER_FORMAT. */
 #define LOG_HEADER_FORMAT_CURRENT	LOG_HEADER_FORMAT_10_3
+/** Future MariaDB 10.4 log format */
+#define LOG_HEADER_FORMAT_10_4		104
 /** Encrypted MariaDB redo log */
 #define LOG_HEADER_FORMAT_ENCRYPTED	(1U<<31)
 
@@ -549,7 +558,10 @@ struct log_t{
     /** number of files */
     ulint				n_files;
     /** format of the redo log: e.g., LOG_HEADER_FORMAT_CURRENT */
-    ulint				format;
+    uint32_t				format;
+    /** redo log subformat: 0 with separately logged TRUNCATE,
+    2 with fully redo-logged TRUNCATE (1 in MariaDB 10.2) */
+    uint32_t				subformat;
     /** individual log file size in bytes, including the header */
     lsn_t				file_size;
     /** lsn used to fix coordinates within the log group */
diff --git a/storage/innobase/include/log0recv.h b/storage/innobase/include/log0recv.h
index 99530a3799c..d15ec19d86b 100644
--- a/storage/innobase/include/log0recv.h
+++ b/storage/innobase/include/log0recv.h
@@ -267,6 +267,15 @@ struct recv_sys_t{
 	ulint		n_addrs;/*!< number of not processed hashed file
 				addresses in the hash table */
 
+	/** Undo tablespaces for which truncate has been logged
+	(indexed by id - srv_undo_space_id_start) */
+	struct trunc {
+		/** log sequence number of MLOG_FILE_CREATE2, or 0 if none */
+		lsn_t		lsn;
+		/** truncated size of the tablespace, or 0 if not truncated */
+		unsigned	pages;
+	} truncated_undo_spaces[127];
+
 	recv_dblwr_t	dblwr;
 
 	/** Lastly added LSN to the hash table of log records. */
diff --git a/storage/innobase/include/os0file.h b/storage/innobase/include/os0file.h
index 20cab0ca7e9..71da751ad25 100644
--- a/storage/innobase/include/os0file.h
+++ b/storage/innobase/include/os0file.h
@@ -1241,17 +1241,18 @@ bool
 os_file_set_eof(
 	FILE*		file);	/*!< in: file to be truncated */
 
-/** Truncates a file to a specified size in bytes. Do nothing if the size
-preserved is smaller or equal than current size of file.
+/** Truncate a file to a specified size in bytes.
 @param[in]	pathname	file path
 @param[in]	file		file to be truncated
 @param[in]	size		size preserved in bytes
+@param[in]	allow_shrink	whether to allow the file to become smaller
 @return true if success */
 bool
 os_file_truncate(
 	const char*	pathname,
 	os_file_t	file,
-	os_offset_t	size);
+	os_offset_t	size,
+	bool		allow_shrink = false);
 
 /** NOTE! Use the corresponding macro os_file_flush(), not directly this
 function!
diff --git a/storage/innobase/include/row0mysql.h b/storage/innobase/include/row0mysql.h
index 4d8b055e13f..c59248d88c4 100644
--- a/storage/innobase/include/row0mysql.h
+++ b/storage/innobase/include/row0mysql.h
@@ -29,6 +29,8 @@ Created 9/17/2000 Heikki Tuuri
 #define row0mysql_h
 
 #include "ha_prototypes.h"
+#include "sql_list.h"
+#include "sql_cmd.h"
 
 #include "data0data.h"
 #include "que0types.h"
@@ -451,32 +453,28 @@ row_mysql_lock_table(
 	const char*	op_info)	/*!< in: string for trx->op_info */
 	MY_ATTRIBUTE((nonnull, warn_unused_result));
 
-/*********************************************************************//**
-Truncates a table for MySQL.
-@return error code or DB_SUCCESS */
-dberr_t
-row_truncate_table_for_mysql(
-/*=========================*/
-	dict_table_t*	table,	/*!< in: table handle */
-	trx_t*		trx)	/*!< in: transaction handle */
-	MY_ATTRIBUTE((nonnull, warn_unused_result));
-/*********************************************************************//**
-Drops a table for MySQL.  If the data dictionary was not already locked
-by the transaction, the transaction will be committed.  Otherwise, the
-data dictionary will remain locked.
-@return error code or DB_SUCCESS */
+/** Drop a table.
+If the data dictionary was not already locked by the transaction,
+the transaction will be committed.  Otherwise, the data dictionary
+will remain locked.
+@param[in]	name		Table name
+@param[in,out]	trx		Transaction handle
+@param[in]	sqlcom		type of SQL operation
+@param[in]	create_failed	true=create table failed
+				because e.g. foreign key column
+@param[in]	nonatomic	Whether it is permitted to release
+				and reacquire dict_operation_lock
+@return error code */
 dberr_t
 row_drop_table_for_mysql(
-/*=====================*/
-	const char*	name,	/*!< in: table name */
-	trx_t*		trx,	/*!< in: dictionary transaction handle */
-	bool		drop_db,/*!< in: true=dropping whole database */
-	ibool		create_failed,/*!<in: TRUE=create table failed
-					because e.g. foreign key column
-					type mismatch. */
-	bool		nonatomic = true);
-				/*!< in: whether it is permitted
-				to release and reacquire dict_operation_lock */
+	const char*		name,
+	trx_t*			trx,
+	enum_sql_command	sqlcom,
+	bool			create_failed = false,
+	bool			nonatomic = true);
+
+/** Drop a table after failed CREATE TABLE. */
+dberr_t row_drop_table_after_create_fail(const char* name, trx_t* trx);
 
 /*********************************************************************//**
 Discards the tablespace of a table which stored in an .ibd file. Discarding
diff --git a/storage/innobase/include/row0trunc.h b/storage/innobase/include/row0trunc.h
index 94b6b7046b4..993dac295da 100644
--- a/storage/innobase/include/row0trunc.h
+++ b/storage/innobase/include/row0trunc.h
@@ -414,14 +414,4 @@ private:
 		const char*		log_file_name);
 };
 
-
-/**
-Truncates a table for MySQL.
-@param table		table being truncated
-@param trx		transaction covering the truncate
-@return	error code or DB_SUCCESS */
-dberr_t
-row_truncate_table_for_mysql(dict_table_t* table, trx_t* trx);
-
 #endif /* row0trunc_h */
-
diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h
index ee82381032c..422b8ef39e4 100644
--- a/storage/innobase/include/srv0srv.h
+++ b/storage/innobase/include/srv0srv.h
@@ -997,6 +997,8 @@ struct export_var_t{
 	ulint innodb_truncated_status_writes;	/*!< srv_truncated_status_writes */
 	ulint innodb_available_undo_logs;       /*!< srv_available_undo_logs
 						*/
+	/** Number of undo tablespace truncation operations */
+	ulong innodb_undo_truncations;
 	ulint innodb_defragment_compression_failures; /*!< Number of
 						defragment re-compression
 						failures */
diff --git a/storage/innobase/include/trx0purge.h b/storage/innobase/include/trx0purge.h
index 171f1d2ce86..27807321212 100644
--- a/storage/innobase/include/trx0purge.h
+++ b/storage/innobase/include/trx0purge.h
@@ -140,27 +140,6 @@ namespace undo {
 	typedef std::vector<ulint>		undo_spaces_t;
 	typedef	std::vector<trx_rseg_t*>	rseg_for_trunc_t;
 
-	/** Magic Number to indicate truncate action is complete. */
-	const ib_uint32_t			s_magic = 76845412;
-
-	/** Truncate Log file Prefix. */
-	const char* const			s_log_prefix = "undo_";
-
-	/** Truncate Log file Extension. */
-	const char* const			s_log_ext = "trunc.log";
-
-	/** Populate log file name based on space_id
-	@param[in]	space_id	id of the undo tablespace.
-	@return DB_SUCCESS or error code */
-	dberr_t populate_log_file_name(
-		ulint	space_id,
-		char*&	log_file_name);
-
-	/** Create the truncate log file.
-	@param[in]	space_id	id of the undo tablespace to truncate.
-	@return DB_SUCCESS or error code. */
-	dberr_t init(ulint space_id);
-
 	/** Mark completion of undo truncate action by writing magic number to
 	the log file and then removing it from the disk.
 	If we are going to remove it from disk then why write magic number ?
@@ -322,23 +301,6 @@ namespace undo {
 			return(m_purge_rseg_truncate_frequency);
 		}
 
-		/* Start writing log information to a special file.
-		On successfull completion, file is removed.
-		On crash, file is used to complete the truncate action.
-		@param	space_id	space id of undo tablespace
-		@return DB_SUCCESS or error code. */
-		dberr_t start_logging(ulint space_id)
-		{
-			return(init(space_id));
-		}
-
-		/* Mark completion of logging./
-		@param	space_id	space id of undo tablespace */
-		void done_logging(ulint space_id)
-		{
-			return(done(space_id));
-		}
-
 	private:
 		/** UNDO tablespace is mark for truncate. */
 		ulint			m_undo_for_trunc;
diff --git a/storage/innobase/include/trx0undo.h b/storage/innobase/include/trx0undo.h
index 7d4e632d3ce..16e2a384424 100644
--- a/storage/innobase/include/trx0undo.h
+++ b/storage/innobase/include/trx0undo.h
@@ -260,18 +260,6 @@ trx_undo_commit_cleanup(trx_undo_t* undo, bool is_temp);
 void
 trx_undo_free_at_shutdown(trx_t *trx);
 
-/* Forward declaration. */
-namespace undo {
-	class Truncate;
-};
-
-/** Truncate UNDO tablespace, reinitialize header and rseg.
-@param[in]	undo_trunc	UNDO tablespace handler
-@return true if success else false. */
-bool
-trx_undo_truncate_tablespace(
-	undo::Truncate*	undo_trunc);
-
 /** Parse MLOG_UNDO_INIT.
 @param[in]	ptr	log record
 @param[in]	end_ptr	end of log record buffer