Merge 10.5 into 10.6

6 files changed, 181 insertions, 154 deletions

diff --git a/extra/aws_sdk/CMakeLists.txt b/extra/aws_sdk/CMakeLists.txt
index 85a196dccce..7c4b8d8195c 100644
--- a/extra/aws_sdk/CMakeLists.txt
+++ b/extra/aws_sdk/CMakeLists.txt
@@ -34,7 +34,7 @@ ENDFOREACH()
 IF(CMAKE_VERSION LESS "3.0")
   SET(GIT_TAG "1.0.8")
 ELSE()
-  SET(GIT_TAG "1.2.11")
+  SET(GIT_TAG "1.8.29")
 ENDIF()
 
 IF(MSVC_CRT_TYPE MATCHES "/MD")
diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc
index 4ad14dc4740..c841725f552 100644
--- a/extra/mariabackup/backup_copy.cc
+++ b/extra/mariabackup/backup_copy.cc
@@ -961,7 +961,7 @@ run_data_threads(datadir_iter_t *it, os_thread_func_t func, uint n)
 		data_threads[i].n_thread = i + 1;
 		data_threads[i].count = &count;
 		data_threads[i].count_mutex = &count_mutex;
-		os_thread_create(func, data_threads + i, &data_threads[i].id);
+		data_threads[i].id = os_thread_create(func, data_threads + i);
 	}
 
 	/* Wait for threads to exit */
@@ -1382,7 +1382,7 @@ out:
 
 void backup_fix_ddl(void);
 
-static lsn_t get_current_lsn(MYSQL *connection)
+lsn_t get_current_lsn(MYSQL *connection)
 {
 	static const char lsn_prefix[] = "\nLog sequence number ";
 	lsn_t lsn = 0;
diff --git a/extra/mariabackup/backup_copy.h b/extra/mariabackup/backup_copy.h
index fbc09eaded3..7c886719f37 100644
--- a/extra/mariabackup/backup_copy.h
+++ b/extra/mariabackup/backup_copy.h
@@ -3,6 +3,7 @@
 #define XTRABACKUP_BACKUP_COPY_H
 
 #include <my_global.h>
+#include <mysql.h>
 #include "datasink.h"
 
 /* special files */
@@ -48,4 +49,7 @@ is_path_separator(char);
 bool
 directory_exists(const char *dir, bool create);
 
+lsn_t
+get_current_lsn(MYSQL *connection);
+
 #endif
diff --git a/extra/mariabackup/backup_mysql.cc b/extra/mariabackup/backup_mysql.cc
index 83e877ea9bb..28aed958c73 100644
--- a/extra/mariabackup/backup_mysql.cc
+++ b/extra/mariabackup/backup_mysql.cc
@@ -75,7 +75,6 @@ bool have_multi_threaded_slave = false;
 bool have_gtid_slave = false;
 
 /* Kill long selects */
-os_thread_id_t	kill_query_thread_id;
 os_event_t	kill_query_thread_started;
 os_event_t	kill_query_thread_stopped;
 os_event_t	kill_query_thread_stop;
@@ -856,7 +855,7 @@ start_query_killer()
 	kill_query_thread_started	= os_event_create(0);
 	kill_query_thread_stopped	= os_event_create(0);
 
-	os_thread_create(kill_query_thread, NULL, &kill_query_thread_id);
+	os_thread_create(kill_query_thread);
 
 	os_event_wait(kill_query_thread_started);
 }
diff --git a/extra/mariabackup/fil_cur.cc b/extra/mariabackup/fil_cur.cc
index 8f06005a9e4..4f9e493b347 100644
--- a/extra/mariabackup/fil_cur.cc
+++ b/extra/mariabackup/fil_cur.cc
@@ -93,8 +93,6 @@ xb_fil_node_close_file(
 	mutex_enter(&fil_system.mutex);
 
 	ut_ad(node);
-	ut_a(node->n_pending == 0);
-	ut_a(node->n_pending_flushes == 0);
 	ut_a(!node->being_extended);
 
 	if (!node->is_open()) {
@@ -108,20 +106,10 @@ xb_fil_node_close_file(
 	ut_a(ret);
 
 	node->handle = OS_FILE_CLOSED;
+	mutex_exit(&fil_system.mutex);
 
 	ut_a(fil_system.n_open > 0);
 	fil_system.n_open--;
-
-	if (node->space->purpose == FIL_TYPE_TABLESPACE &&
-	    fil_is_user_tablespace_id(node->space->id)) {
-
-		ut_a(UT_LIST_GET_LEN(fil_system.LRU) > 0);
-
-		/* The node is in the LRU list, remove it */
-		UT_LIST_REMOVE(fil_system.LRU, node);
-	}
-
-	mutex_exit(&fil_system.mutex);
 }
 
 /************************************************************************
@@ -180,18 +168,8 @@ xb_fil_cur_open(
 
 			return(XB_FIL_CUR_SKIP);
 		}
-		mutex_enter(&fil_system.mutex);
 
 		fil_system.n_open++;
-
-		if (node->space->purpose == FIL_TYPE_TABLESPACE &&
-		    fil_is_user_tablespace_id(node->space->id)) {
-
-			/* Put the node to the LRU list */
-			UT_LIST_ADD_FIRST(fil_system.LRU, node);
-		}
-
-		mutex_exit(&fil_system.mutex);
 	}
 
 	ut_ad(node->is_open());
@@ -427,7 +405,7 @@ xb_fil_cur_read(
 	retry_count = 10;
 	ret = XB_FIL_CUR_SUCCESS;
 
-	fil_space_t *space = fil_space_acquire_for_io(cursor->space_id);
+	fil_space_t *space = fil_space_t::get(cursor->space_id);
 
 	if (!space) {
 		return XB_FIL_CUR_ERROR;
@@ -476,7 +454,7 @@ read_retry:
 
 	posix_fadvise(cursor->file, offset, to_read, POSIX_FADV_DONTNEED);
 func_exit:
-	space->release_for_io();
+	space->release();
 	return(ret);
 }
 
diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc
index 6442139f17d..097f2caa262 100644
--- a/extra/mariabackup/xtrabackup.cc
+++ b/extra/mariabackup/xtrabackup.cc
@@ -503,7 +503,7 @@ static os_event_t dbug_start_query_thread(
 	par->expect_errno = expected_errno;
 	par->done_event = os_event_create(0);
 	par->con =  xb_mysql_connect();
-	os_thread_create(dbug_execute_in_new_connection, par, 0);
+	os_thread_create(dbug_execute_in_new_connection, par);
 
 	if (!wait_state)
 		return par->done_event;
@@ -1251,7 +1251,7 @@ struct my_option xb_server_options[] =
    "Data file autoextend increment in megabytes",
    (G_PTR*) &sys_tablespace_auto_extend_increment,
    (G_PTR*) &sys_tablespace_auto_extend_increment,
-   0, GET_ULONG, REQUIRED_ARG, 8L, 1L, 1000L, 0, 1L, 0},
+   0, GET_UINT, REQUIRED_ARG, 8, 1, 1000, 0, 1, 0},
   {"innodb_data_file_path", OPT_INNODB_DATA_FILE_PATH,
    "Path to individual files and their sizes.", &innobase_data_file_path,
    &innobase_data_file_path, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
@@ -2283,7 +2283,7 @@ check_if_skip_database(
 	if (databases_exclude_hash.array &&
 		find_filter_in_hashtable(name, &databases_exclude_hash,
 					 &database) &&
-		!database->has_tables) {
+		(!database->has_tables || !databases_include_hash.array)) {
 		/* Database is found and there are no tables specified,
 		   skip entire db. */
 		return DATABASE_SKIP;
@@ -2453,17 +2453,24 @@ xb_get_copy_action(const char *dflt)
 	return(action);
 }
 
-/* TODO: We may tune the behavior (e.g. by fil_aio)*/
 
-static
-my_bool
-xtrabackup_copy_datafile(fil_node_t* node, uint thread_n, const char *dest_name=0, ulonglong max_size=ULLONG_MAX)
+/** Copy innodb data file to the specified destination.
+
+@param[in] node	file node of a tablespace
+@param[in] thread_n	thread id, used in the text of diagnostic messages
+@param[in] dest_name	destination file name
+@param[in] write_filter	write filter to copy data, can be pass-through filter
+for full backup, pages filter for incremental backup, etc.
+
+@return FALSE on success and TRUE on error */
+static my_bool xtrabackup_copy_datafile(fil_node_t *node, uint thread_n,
+                                        const char *dest_name,
+                                        const xb_write_filt_t &write_filter)
 {
 	char			 dst_name[FN_REFLEN];
 	ds_file_t		*dstfile = NULL;
 	xb_fil_cur_t		 cursor;
 	xb_fil_cur_result_t	 res;
-	xb_write_filt_t		*write_filter = NULL;
 	xb_write_filt_ctxt_t	 write_filt_ctxt;
 	const char		*action;
 	xb_read_filt_t		*read_filter;
@@ -2487,6 +2494,8 @@ xtrabackup_copy_datafile(fil_node_t* node, uint thread_n, const char *dest_name=
 		return(FALSE);
 	}
 
+	memset(&write_filt_ctxt, 0, sizeof(xb_write_filt_ctxt_t));
+
 	bool was_dropped;
 	pthread_mutex_lock(&backup_mutex);
 	was_dropped = (ddl_tracker.drops.find(node->space->id) != ddl_tracker.drops.end());
@@ -2507,7 +2516,7 @@ xtrabackup_copy_datafile(fil_node_t* node, uint thread_n, const char *dest_name=
 		read_filter = &rf_bitmap;
 	}
 
-	res = xb_fil_cur_open(&cursor, read_filter, node, thread_n,max_size);
+	res = xb_fil_cur_open(&cursor, read_filter, node, thread_n, ULLONG_MAX);
 	if (res == XB_FIL_CUR_SKIP) {
 		goto skip;
 	} else if (res == XB_FIL_CUR_ERROR) {
@@ -2518,18 +2527,10 @@ xtrabackup_copy_datafile(fil_node_t* node, uint thread_n, const char *dest_name=
 		sizeof dst_name - 1);
 	dst_name[sizeof dst_name - 1] = '\0';
 
-	/* Setup the page write filter */
-	if (xtrabackup_incremental) {
-		write_filter = &wf_incremental;
-	} else {
-		write_filter = &wf_write_through;
-	}
-
-	memset(&write_filt_ctxt, 0, sizeof(xb_write_filt_ctxt_t));
-	ut_a(write_filter->process != NULL);
+	ut_a(write_filter.process != NULL);
 
-	if (write_filter->init != NULL &&
-	    !write_filter->init(&write_filt_ctxt, dst_name, &cursor)) {
+	if (write_filter.init != NULL &&
+	    !write_filter.init(&write_filt_ctxt, dst_name, &cursor)) {
 		msg (thread_n, "mariabackup: error: failed to initialize page write filter.");
 		goto error;
 	}
@@ -2550,7 +2551,7 @@ xtrabackup_copy_datafile(fil_node_t* node, uint thread_n, const char *dest_name=
 
 	/* The main copy loop */
 	while ((res = xb_fil_cur_read(&cursor)) == XB_FIL_CUR_SUCCESS) {
-		if (!write_filter->process(&write_filt_ctxt, dstfile)) {
+		if (!write_filter.process(&write_filt_ctxt, dstfile)) {
 			goto error;
 		}
 	}
@@ -2559,8 +2560,8 @@ xtrabackup_copy_datafile(fil_node_t* node, uint thread_n, const char *dest_name=
 		goto error;
 	}
 
-	if (write_filter->finalize
-	    && !write_filter->finalize(&write_filt_ctxt, dstfile)) {
+	if (write_filter.finalize
+	    && !write_filter.finalize(&write_filt_ctxt, dstfile)) {
 		goto error;
 	}
 
@@ -2574,8 +2575,8 @@ xtrabackup_copy_datafile(fil_node_t* node, uint thread_n, const char *dest_name=
 	if (ds_close(dstfile)) {
 		rc = TRUE;
 	}
-	if (write_filter && write_filter->deinit) {
-		write_filter->deinit(&write_filt_ctxt);
+	if (write_filter.deinit) {
+		write_filter.deinit(&write_filt_ctxt);
 	}
 	return(rc);
 
@@ -2584,8 +2585,8 @@ error:
 	if (dstfile != NULL) {
 		ds_close(dstfile);
 	}
-	if (write_filter && write_filter->deinit) {
-		write_filter->deinit(&write_filt_ctxt);;
+	if (write_filter.deinit) {
+		write_filter.deinit(&write_filt_ctxt);;
 	}
 	msg(thread_n, "mariabackup: xtrabackup_copy_datafile() failed.");
 	return(TRUE); /*ERROR*/
@@ -2595,8 +2596,8 @@ skip:
 	if (dstfile != NULL) {
 		ds_close(dstfile);
 	}
-	if (write_filter && write_filter->deinit) {
-		write_filter->deinit(&write_filt_ctxt);
+	if (write_filter.deinit) {
+		write_filter.deinit(&write_filt_ctxt);
 	}
 	msg(thread_n,"Warning: We assume the  table was dropped during xtrabackup execution and ignore the tablespace %s", node_name);
 	return(FALSE);
@@ -2642,7 +2643,7 @@ static lsn_t xtrabackup_copy_log(lsn_t start_lsn, lsn_t end_lsn, bool last)
 			/* We got a full log block. */
 			scanned_lsn += data_len;
 		} else if (data_len >= log_sys.trailer_offset()
-			   || data_len <= LOG_BLOCK_HDR_SIZE) {
+			   || data_len < LOG_BLOCK_HDR_SIZE) {
 			/* We got a garbage block (abrupt end of the log). */
 			msg(0,"garbage block: " LSN_PF ",%zu",scanned_lsn, data_len);
 			break;
@@ -2703,7 +2704,7 @@ static bool xtrabackup_copy_logfile(bool last = false)
 
 		xtrabackup_io_throttling();
 
-		log_mutex_enter();
+		mysql_mutex_lock(&log_sys.mutex);
 		lsn_t lsn= start_lsn;
 		for (int retries= 0; retries < 100; retries++) {
 			if (log_sys.log.read_log_seg(&lsn, end_lsn)
@@ -2727,7 +2728,7 @@ static bool xtrabackup_copy_logfile(bool last = false)
 			mutex_exit(&recv_sys.mutex);
 		}
 
-		log_mutex_exit();
+		mysql_mutex_unlock(&log_sys.mutex);
 
 		if (!start_lsn) {
 			const char *reason = recv_sys.found_corrupt_log
@@ -2787,10 +2788,10 @@ static os_thread_ret_t DECLARE_THREAD(log_copying_thread)(void*)
 			break;
 		}
 
-		log_mutex_enter();
+		mysql_mutex_lock(&log_sys.mutex);
 		bool completed = metadata_to_lsn
 			&& metadata_to_lsn <= log_copy_scanned_lsn;
-		log_mutex_exit();
+		mysql_mutex_unlock(&log_sys.mutex);
 		if (completed) {
 			break;
 		}
@@ -2859,8 +2860,14 @@ static void dbug_mariabackup_event(const char *event,const char *key)
 
 }
 #define DBUG_MARIABACKUP_EVENT(A, B) DBUG_EXECUTE_IF("mariabackup_events", dbug_mariabackup_event(A,B););
+#define DBUG_MB_INJECT_CODE(EVENT, KEY, CODE) \
+	DBUG_EXECUTE_IF("mariabackup_inject_code", {\
+		char *env = getenv(EVENT); \
+		if (env && !strcmp(env, KEY)) { CODE } \
+	})
 #else
 #define DBUG_MARIABACKUP_EVENT(A,B)
+#define DBUG_MB_INJECT_CODE(EVENT, KEY, CODE)
 #endif
 
 /**************************************************************************
@@ -2885,10 +2892,12 @@ DECLARE_THREAD(data_copy_thread_func)(
 
 	while ((node = datafiles_iter_next(ctxt->it)) != NULL) {
 		DBUG_MARIABACKUP_EVENT("before_copy", node->space->name);
+		DBUG_MB_INJECT_CODE("wait_innodb_redo_before_copy", node->space->name,
+			backup_wait_for_lsn(get_current_lsn(mysql_connection)););
 		/* copy the datafile */
-		if(xtrabackup_copy_datafile(node, num)) {
+		if (xtrabackup_copy_datafile(node, num, NULL,
+			xtrabackup_incremental ? wf_incremental : wf_write_through))
 			die("failed to copy datafile.");
-		}
 
 		DBUG_MARIABACKUP_EVENT("after_copy", node->space->name);
 
@@ -3003,6 +3012,8 @@ void
 xb_fil_io_init()
 {
 	fil_system.create(srv_file_per_table ? 50000 : 5000);
+	fil_system.freeze_space_list = 1;
+	fil_system.space_id_reuse_warned = true;
 }
 
 static
@@ -3078,24 +3089,16 @@ xb_load_single_table_tablespace(
 	bool is_empty_file = file->exists() && file->is_empty_file();
 
 	if (err == DB_SUCCESS && file->space_id() != SRV_TMP_SPACE_ID) {
-		os_offset_t	node_size = os_file_get_size(file->handle());
-		os_offset_t	n_pages;
-
-		ut_a(node_size != (os_offset_t) -1);
-
-		n_pages = node_size / fil_space_t::physical_size(file->flags());
-
-		space = fil_space_create(
+		space = fil_space_t::create(
 			name, file->space_id(), file->flags(),
 			FIL_TYPE_TABLESPACE, NULL/* TODO: crypt_data */);
 
 		ut_a(space != NULL);
 
-		space->add(file->filepath(), OS_FILE_CLOSED, ulint(n_pages),
-			   false, false);
-		/* by opening the tablespace we forcing node and space objects
-		in the cache to be populated with fields from space header */
-		space->open();
+		space->add(file->filepath(), file->detach(), 0, false, false);
+		mutex_enter(&fil_system.mutex);
+		space->read_page0();
+		mutex_exit(&fil_system.mutex);
 
 		if (srv_operation == SRV_OPERATION_RESTORE_DELTA
 		    || xb_close_files) {
@@ -3397,30 +3400,11 @@ xb_load_tablespaces()
 	return(DB_SUCCESS);
 }
 
-/************************************************************************
-Initialize the tablespace memory cache and populate it by scanning for and
-opening data files.
-@returns DB_SUCCESS or error code.*/
-static
-dberr_t
-xb_data_files_init()
+/** Destroy the tablespace memory cache. */
+static void xb_data_files_close()
 {
-	xb_fil_io_init();
-
-	return(xb_load_tablespaces());
-}
-
-/************************************************************************
-Destroy the tablespace memory cache. */
-static
-void
-xb_data_files_close()
-{
-	ut_ad(!os_thread_count);
-	fil_close_all_files();
-	if (buf_dblwr) {
-		buf_dblwr_free();
-	}
+  fil_space_t::close_all();
+  buf_dblwr.close();
 }
 
 /***********************************************************************
@@ -3865,7 +3849,7 @@ static bool xtrabackup_backup_low()
 	{
 		ulint	max_cp_field;
 
-		log_mutex_enter();
+		mysql_mutex_lock(&log_sys.mutex);
 
 		if (recv_find_max_checkpoint(&max_cp_field) == DB_SUCCESS
 		    && log_sys.log.format != 0) {
@@ -3882,7 +3866,7 @@ static bool xtrabackup_backup_low()
 		} else {
 			msg("Error: recv_find_max_checkpoint() failed.");
 		}
-		log_mutex_exit();
+		mysql_mutex_unlock(&log_sys.mutex);
 	}
 
 	stop_backup_threads();
@@ -4009,7 +3993,6 @@ fail:
         }
 	srv_thread_pool_init();
 	sync_check_init();
-	ut_d(sync_check_enable());
 	/* Reset the system variables in the recovery module. */
 	trx_pool_init();
 	recv_sys.create();
@@ -4052,26 +4035,23 @@ fail:
 	/* definition from recv_recovery_from_checkpoint_start() */
 	ulint		max_cp_field;
 
-	/* start back ground thread to copy newer log */
-	os_thread_id_t log_copying_thread_id;
-
 	/* get current checkpoint_lsn */
 	/* Look for the latest checkpoint from any of the log groups */
 
-	log_mutex_enter();
+	mysql_mutex_lock(&log_sys.mutex);
 
 reread_log_header:
 	dberr_t err = recv_find_max_checkpoint(&max_cp_field);
 
 	if (err != DB_SUCCESS) {
 		msg("Error: cannot read redo log header");
-		log_mutex_exit();
+		mysql_mutex_unlock(&log_sys.mutex);
 		goto fail;
 	}
 
 	if (log_sys.log.format == 0) {
 		msg("Error: cannot process redo log before MariaDB 10.2.2");
-		log_mutex_exit();
+		mysql_mutex_unlock(&log_sys.mutex);
 		goto fail;
 	}
 
@@ -4088,7 +4068,7 @@ reread_log_header:
 	    != mach_read_from_8(buf + LOG_CHECKPOINT_OFFSET))
 		goto reread_log_header;
 
-	log_mutex_exit();
+	mysql_mutex_unlock(&log_sys.mutex);
 
 	xtrabackup_init_datasinks();
 
@@ -4144,14 +4124,11 @@ reread_log_header:
 	log_copying_running = true;
 	/* start io throttle */
 	if(xtrabackup_throttle) {
-		os_thread_id_t io_watching_thread_id;
-
 		io_ticket = xtrabackup_throttle;
 		wait_throttle = os_event_create(0);
 		io_watching_thread_running = true;
 
-		os_thread_create(io_watching_thread, NULL,
-				 &io_watching_thread_id);
+		os_thread_create(io_watching_thread);
 	}
 
 	/* Populate fil_system with tablespaces to copy */
@@ -4174,7 +4151,7 @@ fail_before_log_copying_thread_start:
 	DBUG_MARIABACKUP_EVENT("before_innodb_log_copy_thread_started",0);
 
 	log_copying_stop = os_event_create(0);
-	os_thread_create(log_copying_thread, NULL, &log_copying_thread_id);
+	os_thread_create(log_copying_thread);
 
 	/* FLUSH CHANGED_PAGE_BITMAPS call */
 	if (!flush_changed_page_bitmaps()) {
@@ -4216,8 +4193,8 @@ fail_before_log_copying_thread_start:
 		data_threads[i].num = i+1;
 		data_threads[i].count = &count;
 		data_threads[i].count_mutex = &count_mutex;
-		os_thread_create(data_copy_thread_func, data_threads + i,
-				 &data_threads[i].id);
+		data_threads[i].id = os_thread_create(data_copy_thread_func,
+						      data_threads + i);
 	}
 
 	/* Wait for threads to exit */
@@ -4430,7 +4407,7 @@ void backup_fix_ddl(void)
 			continue;
 		std::string dest_name(node->space->name);
 		dest_name.append(".new");
-		xtrabackup_copy_datafile(node, 0, dest_name.c_str()/*, do_full_copy ? ULONGLONG_MAX:UNIV_PAGE_SIZE */);
+		xtrabackup_copy_datafile(node, 0, dest_name.c_str(), wf_write_through);
 	}
 
 	datafiles_iter_free(it);
@@ -4611,7 +4588,23 @@ xb_delta_open_matching_space(
 		return file;
 	}
 
-	log_mutex_enter();
+	if (!info.space_id && fil_system.sys_space) {
+		fil_node_t *node
+			= UT_LIST_GET_FIRST(fil_system.sys_space->chain);
+		for (; node; node = UT_LIST_GET_NEXT(chain, node)) {
+			if (!strcmp(node->name, real_name)) {
+				break;
+			}
+		}
+		if (node && node->handle != OS_FILE_CLOSED) {
+			*success = true;
+			return node->handle;
+		}
+		msg("mariabackup: Cannot find file %s\n", real_name);
+		return OS_FILE_CLOSED;
+	}
+
+	mysql_mutex_lock(&log_sys.mutex);
 	if (!fil_is_user_tablespace_id(info.space_id)) {
 found:
 		/* open the file and return its handle */
@@ -4624,7 +4617,7 @@ found:
 			msg("mariabackup: Cannot open file %s\n", real_name);
 		}
 exit:
-		log_mutex_exit();
+		mysql_mutex_unlock(&log_sys.mutex);
 		return file;
 	}
 
@@ -4708,8 +4701,8 @@ exit:
 	ut_ad(fil_space_t::zip_size(flags) == info.zip_size);
 	ut_ad(fil_space_t::physical_size(flags) == info.page_size);
 
-	if (fil_space_create(dest_space_name, info.space_id, flags,
-			      FIL_TYPE_TABLESPACE, 0)) {
+	if (fil_space_t::create(dest_space_name, info.space_id, flags,
+				FIL_TYPE_TABLESPACE, 0)) {
 		*success = xb_space_create_file(real_name, info.space_id,
 						flags, &file);
 	} else {
@@ -4902,7 +4895,7 @@ xtrabackup_apply_delta(
 						space->chain);
 					bool fail = !strcmp(n->name, dst_path)
 						&& !fil_space_extend(
-							space, (ulint)n_pages);
+							space, uint32_t(n_pages));
 					if (fail) goto error;
 				}
 			}
@@ -4929,7 +4922,7 @@ xtrabackup_apply_delta(
 		os_file_close(src_file);
 		os_file_delete(0,src_path);
 	}
-	if (dst_file != OS_FILE_CLOSED)
+	if (dst_file != OS_FILE_CLOSED && info.space_id)
 		os_file_close(dst_file);
 	return TRUE;
 
@@ -4937,7 +4930,7 @@ error:
 	aligned_free(incremental_buffer);
 	if (src_file != OS_FILE_CLOSED)
 		os_file_close(src_file);
-	if (dst_file != OS_FILE_CLOSED)
+	if (dst_file != OS_FILE_CLOSED && info.space_id)
 		os_file_close(dst_file);
 	msg("Error: xtrabackup_apply_delta(): "
 	    "failed to apply %s to %s.\n", src_path, dst_path);
@@ -4987,22 +4980,66 @@ static void rename_force(const char *from, const char *to) {
 	rename_file(from,to);
 }
 
-/* During prepare phase, rename ".new" files , that were created in backup_fix_ddl(),
-  to ".ibd".*/
-static ibool prepare_handle_new_files(
-	const char*	data_home_dir,		/*!<in: path to datadir */
-	const char*	db_name,		/*!<in: database name */
-	const char*	file_name,		/*!<in: file name with suffix */
-	void *)
-{
 
+/** During prepare phase, rename ".new" files, that were created in
+backup_fix_ddl() and backup_optimized_ddl_op(), to ".ibd". In the case of
+incremental backup, i.e. of arg argument is set, move ".new" files to
+destination directory and rename them to ".ibd", remove existing ".ibd.delta"
+and ".idb.meta" files in incremental directory to avoid applying delta to
+".ibd" file.
+
+@param[in] data_home_dir	path to datadir
+@param[in] db_name	database name
+@param[in] file_name	file name with suffix
+@param[in] arg	destination path, used in incremental backup to notify, that
+*.new file must be moved to destibation directory
+
+@return true */
+static ibool prepare_handle_new_files(const char *data_home_dir,
+                                      const char *db_name,
+                                      const char *file_name, void *arg)
+{
+	const char *dest_dir = static_cast<const char *>(arg);
 	std::string src_path = std::string(data_home_dir) + '/' + std::string(db_name) + '/' + file_name;
-	std::string dest_path = src_path;
+	/* Copy "*.new" files from incremental to base dir for incremental backup */
+	std::string dest_path=
+		dest_dir ? std::string(dest_dir) + '/' + std::string(db_name) +
+			'/' + file_name : src_path;
 
 	size_t index = dest_path.find(".new");
 	DBUG_ASSERT(index != std::string::npos);
-	dest_path.replace(index, 4, ".ibd");
+	dest_path.replace(index, strlen(".ibd"), ".ibd");
 	rename_force(src_path.c_str(),dest_path.c_str());
+
+	if (dest_dir) {
+		/* remove delta and meta files to avoid delta applying for new file */
+		index = src_path.find(".new");
+		DBUG_ASSERT(index != std::string::npos);
+		src_path.replace(index, std::string::npos, ".ibd.delta");
+		if (access(src_path.c_str(), R_OK) == 0) {
+			msg("Removing %s", src_path.c_str());
+			if (my_delete(src_path.c_str(), MYF(MY_WME)))
+				die("Can't remove %s, errno %d", src_path.c_str(), errno);
+		}
+		src_path.replace(index, std::string::npos, ".ibd.meta");
+		if (access(src_path.c_str(), R_OK) == 0) {
+			msg("Removing %s", src_path.c_str());
+			if (my_delete(src_path.c_str(), MYF(MY_WME)))
+				die("Can't remove %s, errno %d", src_path.c_str(), errno);
+		}
+
+		/* add table name to the container to avoid it's deletion at the end of
+		 prepare */
+		std::string table_name = std::string(db_name) + '/'
+			+ std::string(file_name, file_name + strlen(file_name) - strlen(".new"));
+		xb_filter_entry_t *table = static_cast<xb_filter_entry_t *>
+			(malloc(sizeof(xb_filter_entry_t) + table_name.size() + 1));
+		table->name = ((char*)table) + sizeof(xb_filter_entry_t);
+		strcpy(table->name, table_name.c_str());
+		HASH_INSERT(xb_filter_entry_t, name_hash, &inc_dir_tables_hash,
+				ut_fold_string(table->name), table);
+	}
+
 	return TRUE;
 }
 
@@ -5039,17 +5076,18 @@ rm_if_not_found(
 	return(TRUE);
 }
 
-/************************************************************************
-Function enumerates files in datadir (provided by path) which are matched
+/** Function enumerates files in datadir (provided by path) which are matched
 by provided suffix. For each entry callback is called.
+
+@param[in] path	datadir path
+@param[in] suffix	suffix to match against
+@param[in] func	callback
+@param[in] func_arg	arguments for the above callback
+
 @return FALSE if callback for some entry returned FALSE */
-static
-ibool
-xb_process_datadir(
-	const char*			path,	/*!<in: datadir path */
-	const char*			suffix,	/*!<in: suffix to match
-						against */
-	handle_datadir_entry_func_t	func)	/*!<in: callback */
+static ibool xb_process_datadir(const char *path, const char *suffix,
+                                handle_datadir_entry_func_t func,
+                                void *func_arg = NULL)
 {
 	ulint		ret;
 	char		dbpath[OS_FILE_MAX_PATH+2];
@@ -5084,7 +5122,7 @@ xb_process_datadir(
 					suffix)) {
 				if (!func(
 					    path, NULL,
-					    fileinfo.name, NULL))
+					    fileinfo.name, func_arg))
 				{
 					os_file_closedir(dbdir);
 					return(FALSE);
@@ -5148,7 +5186,7 @@ next_file_item_1:
 					if (!func(
 						    path,
 						    dbinfo.name,
-						    fileinfo.name, NULL))
+						    fileinfo.name, func_arg))
 					{
 						os_file_closedir(dbdir);
 						os_file_closedir(dir);
@@ -5308,6 +5346,10 @@ static bool xtrabackup_prepare_func(char** argv)
 
 	fil_path_to_mysql_datadir = ".";
 
+	ut_ad(xtrabackup_incremental == xtrabackup_incremental_dir);
+	if (xtrabackup_incremental)
+		inc_dir_tables_hash.create(1000);
+
 	/* Fix DDL for prepare. Process .del,.ren, and .new files.
 	The order in which files are processed, is important
 	(see MDEV-18185, MDEV-18201)
@@ -5319,6 +5361,8 @@ static bool xtrabackup_prepare_func(char** argv)
 	if (xtrabackup_incremental_dir) {
 		xb_process_datadir(xtrabackup_incremental_dir, ".new.meta", prepare_handle_new_files);
 		xb_process_datadir(xtrabackup_incremental_dir, ".new.delta", prepare_handle_new_files);
+		xb_process_datadir(xtrabackup_incremental_dir, ".new",
+				prepare_handle_new_files, (void *)".");
 	}
 	else {
 		xb_process_datadir(".", ".new", prepare_handle_new_files);
@@ -5383,7 +5427,6 @@ static bool xtrabackup_prepare_func(char** argv)
 		}
 
 		sync_check_init();
-		ut_d(sync_check_enable());
 		recv_sys.create();
 		log_sys.create();
 		recv_sys.recovery_on = true;
@@ -5392,16 +5435,15 @@ static bool xtrabackup_prepare_func(char** argv)
 		srv_allow_writes_event = os_event_create(0);
 		os_event_set(srv_allow_writes_event);
 #endif
-		dberr_t err = xb_data_files_init();
-		if (err != DB_SUCCESS) {
+		xb_fil_io_init();
+		if (dberr_t err = xb_load_tablespaces()) {
 			msg("mariabackup: error: xb_data_files_init() failed "
 			    "with error %s\n", ut_strerr(err));
 			goto error_cleanup;
 		}
 
-		inc_dir_tables_hash.create(1000);
-
-		ok = xtrabackup_apply_deltas();
+		ok = fil_system.sys_space->open(false)
+			&& xtrabackup_apply_deltas();
 
 		xb_data_files_close();
 
@@ -5431,6 +5473,8 @@ static bool xtrabackup_prepare_func(char** argv)
 		goto error_cleanup;
 	}
 
+	fil_system.freeze_space_list = 0;
+
 	/* increase IO threads */
 	if (srv_n_file_io_threads < 10) {
 		srv_n_read_io_threads = 4;
@@ -5452,6 +5496,8 @@ static bool xtrabackup_prepare_func(char** argv)
 		goto error_cleanup;
 	}
 
+	ut_ad(!fil_system.freeze_space_list);
+
 	if (ok) {
 		msg("Last binlog file %s, position %lld",
 		    trx_sys.recovered_binlog_filename,