MDEV-13023 mariabackup does not preserve holes for page compressed tables.

Changed "local" datasink logic to detect page compressed Innodb tables.

Whenever such table is detected, holes in the copied files are created by
skipping over binary zeros at the end of each compressed page.

11 files changed, 281 insertions, 166 deletions

diff --git a/extra/mariabackup/CMakeLists.txt b/extra/mariabackup/CMakeLists.txt
index ac15460660c..0e6336fe5e6 100644
--- a/extra/mariabackup/CMakeLists.txt
+++ b/extra/mariabackup/CMakeLists.txt
@@ -61,7 +61,7 @@ MYSQL_ADD_EXECUTABLE(mariabackup
   datasink.c
   ds_buffer.c
   ds_compress.c
-  ds_local.c
+  ds_local.cc
   ds_stdout.c
   ds_tmpfile.c
   ds_xbstream.c
@@ -98,7 +98,7 @@ ENDIF()
 ########################################################################
 MYSQL_ADD_EXECUTABLE(mbstream
   ds_buffer.c
-  ds_local.c
+  ds_local.cc
   ds_stdout.c
   datasink.c
   xbstream.c
@@ -112,6 +112,7 @@ TARGET_LINK_LIBRARIES(mbstream
   mysys
   crc
 )
+ADD_DEPENDENCIES(mbstream GenError)
 
 IF(MSVC)
   SET_TARGET_PROPERTIES(mbstream PROPERTIES LINK_FLAGS setargv.obj)
diff --git a/extra/mariabackup/datasink.c b/extra/mariabackup/datasink.c
index 460e0e8ca19..1459da2fb57 100644
--- a/extra/mariabackup/datasink.c
+++ b/extra/mariabackup/datasink.c
@@ -108,7 +108,7 @@ Write to a datasink file.
 int
 ds_write(ds_file_t *file, const void *buf, size_t len)
 {
-	return file->datasink->write(file, buf, len);
+	return file->datasink->write(file, (const uchar *)buf, len);
 }
 
 /************************************************************************
diff --git a/extra/mariabackup/datasink.h b/extra/mariabackup/datasink.h
index 8bf1321aad1..5962e9ba4b7 100644
--- a/extra/mariabackup/datasink.h
+++ b/extra/mariabackup/datasink.h
@@ -48,7 +48,7 @@ typedef struct {
 struct datasink_struct {
 	ds_ctxt_t *(*init)(const char *root);
 	ds_file_t *(*open)(ds_ctxt_t *ctxt, const char *path, MY_STAT *stat);
-	int (*write)(ds_file_t *file, const void *buf, size_t len);
+	int (*write)(ds_file_t *file, const unsigned char *buf, size_t len);
 	int (*close)(ds_file_t *file);
 	void (*deinit)(ds_ctxt_t *ctxt);
 };
diff --git a/extra/mariabackup/ds_buffer.c b/extra/mariabackup/ds_buffer.c
index 4bb314c0f50..13c05f38918 100644
--- a/extra/mariabackup/ds_buffer.c
+++ b/extra/mariabackup/ds_buffer.c
@@ -45,7 +45,7 @@ typedef struct {
 static ds_ctxt_t *buffer_init(const char *root);
 static ds_file_t *buffer_open(ds_ctxt_t *ctxt, const char *path,
 			      MY_STAT *mystat);
-static int buffer_write(ds_file_t *file, const void *buf, size_t len);
+static int buffer_write(ds_file_t *file, const uchar *buf, size_t len);
 static int buffer_close(ds_file_t *file);
 static void buffer_deinit(ds_ctxt_t *ctxt);
 
@@ -119,7 +119,7 @@ buffer_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat)
 }
 
 static int
-buffer_write(ds_file_t *file, const void *buf, size_t len)
+buffer_write(ds_file_t *file, const uchar *buf, size_t len)
 {
 	ds_buffer_file_t	*buffer_file;
 
diff --git a/extra/mariabackup/ds_compress.c b/extra/mariabackup/ds_compress.c
index 15801c8abd4..88f50857362 100644
--- a/extra/mariabackup/ds_compress.c
+++ b/extra/mariabackup/ds_compress.c
@@ -65,7 +65,7 @@ extern ulonglong	xtrabackup_compress_chunk_size;
 static ds_ctxt_t *compress_init(const char *root);
 static ds_file_t *compress_open(ds_ctxt_t *ctxt, const char *path,
 				MY_STAT *mystat);
-static int compress_write(ds_file_t *file, const void *buf, size_t len);
+static int compress_write(ds_file_t *file, const uchar *buf, size_t len);
 static int compress_close(ds_file_t *file);
 static void compress_deinit(ds_ctxt_t *ctxt);
 
@@ -178,7 +178,7 @@ err:
 
 static
 int
-compress_write(ds_file_t *file, const void *buf, size_t len)
+compress_write(ds_file_t *file, const uchar *buf, size_t len)
 {
 	ds_compress_file_t	*comp_file;
 	ds_compress_ctxt_t	*comp_ctxt;
diff --git a/extra/mariabackup/ds_local.c b/extra/mariabackup/ds_local.c
deleted file mode 100644
index 3e2b1e0129b..00000000000
--- a/extra/mariabackup/ds_local.c
+++ /dev/null
@@ -1,151 +0,0 @@
-/******************************************************
-Copyright (c) 2011-2013 Percona LLC and/or its affiliates.
-
-Local datasink implementation for XtraBackup.
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; version 2 of the License.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
-
-*******************************************************/
-
-#include <mysql_version.h>
-#include <my_base.h>
-#include <mysys_err.h>
-#include "common.h"
-#include "datasink.h"
-
-typedef struct {
-	File fd;
-} ds_local_file_t;
-
-static ds_ctxt_t *local_init(const char *root);
-static ds_file_t *local_open(ds_ctxt_t *ctxt, const char *path,
-			     MY_STAT *mystat);
-static int local_write(ds_file_t *file, const void *buf, size_t len);
-static int local_close(ds_file_t *file);
-static void local_deinit(ds_ctxt_t *ctxt);
-
-datasink_t datasink_local = {
-	&local_init,
-	&local_open,
-	&local_write,
-	&local_close,
-	&local_deinit
-};
-
-static
-ds_ctxt_t *
-local_init(const char *root)
-{
-	ds_ctxt_t *ctxt;
-
-	if (my_mkdir(root, 0777, MYF(0)) < 0
-	    && my_errno != EEXIST && my_errno != EISDIR)
-	{
-		char errbuf[MYSYS_STRERROR_SIZE];
-		my_strerror(errbuf, sizeof(errbuf),my_errno);
-		my_error(EE_CANT_MKDIR, MYF(ME_BELL | ME_WAITTANG),
-			 root, my_errno,errbuf, my_errno);
-		return NULL;
-	}
-
-	ctxt = my_malloc(sizeof(ds_ctxt_t), MYF(MY_FAE));
-
-	ctxt->root = my_strdup(root, MYF(MY_FAE));
-
-	return ctxt;
-}
-
-static
-ds_file_t *
-local_open(ds_ctxt_t *ctxt, const char *path,
-	   MY_STAT *mystat __attribute__((unused)))
-{
-	char 		fullpath[FN_REFLEN];
-	char		dirpath[FN_REFLEN];
-	size_t		dirpath_len;
-	size_t		path_len;
-	ds_local_file_t *local_file;
-	ds_file_t	*file;
-	File 		fd;
-
-	fn_format(fullpath, path, ctxt->root, "", MYF(MY_RELATIVE_PATH));
-
-	/* Create the directory if needed */
-	dirname_part(dirpath, fullpath, &dirpath_len);
-	if (my_mkdir(dirpath, 0777, MYF(0)) < 0 && my_errno != EEXIST) {
-		char errbuf[MYSYS_STRERROR_SIZE];
-		my_strerror(errbuf, sizeof(errbuf), my_errno);
-		my_error(EE_CANT_MKDIR, MYF(ME_BELL | ME_WAITTANG),
-			 dirpath, my_errno, errbuf);
-		return NULL;
-	}
-
-	fd = my_create(fullpath, 0, O_WRONLY | O_BINARY | O_EXCL | O_NOFOLLOW,
-		     MYF(MY_WME));
-	if (fd < 0) {
-		return NULL;
-	}
-
-	path_len = strlen(fullpath) + 1; /* terminating '\0' */
-
-	file = (ds_file_t *) my_malloc(sizeof(ds_file_t) +
-				       sizeof(ds_local_file_t) +
-				       path_len,
-				       MYF(MY_FAE));
-	local_file = (ds_local_file_t *) (file + 1);
-
-	local_file->fd = fd;
-
-	file->path = (char *) local_file + sizeof(ds_local_file_t);
-	memcpy(file->path, fullpath, path_len);
-
-	file->ptr = local_file;
-
-	return file;
-}
-
-static
-int
-local_write(ds_file_t *file, const void *buf, size_t len)
-{
-	File fd = ((ds_local_file_t *) file->ptr)->fd;
-
-	if (!my_write(fd, buf, len, MYF(MY_WME | MY_NABP))) {
-		posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED);
-		return 0;
-	}
-
-	return 1;
-}
-
-static
-int
-local_close(ds_file_t *file)
-{
-	File fd = ((ds_local_file_t *) file->ptr)->fd;
-
-	my_free(file);
-
-	my_sync(fd, MYF(MY_WME));
-
-	return my_close(fd, MYF(MY_WME));
-}
-
-static
-void
-local_deinit(ds_ctxt_t *ctxt)
-{
-	my_free(ctxt->root);
-	my_free(ctxt);
-}
diff --git a/extra/mariabackup/ds_local.cc b/extra/mariabackup/ds_local.cc
new file mode 100644
index 00000000000..f1068d251dc
--- /dev/null
+++ b/extra/mariabackup/ds_local.cc
@@ -0,0 +1,259 @@
+/******************************************************
+Copyright (c) 2011-2013 Percona LLC and/or its affiliates.
+
+Local datasink implementation for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+*******************************************************/
+
+#include <my_config.h>
+#include <mysql_version.h>
+#include <my_base.h>
+#include <mysys_err.h>
+#include "common.h"
+#include "datasink.h"
+#include "univ.i"
+#include "fsp0fsp.h"
+#ifdef _WIN32
+#include <winioctl.h>
+#endif
+
+typedef struct {
+	File fd;
+	my_bool init_ibd_done;
+	my_bool is_ibd;
+	my_bool compressed;
+	size_t pagesize;
+} ds_local_file_t;
+
+static ds_ctxt_t *local_init(const char *root);
+static ds_file_t *local_open(ds_ctxt_t *ctxt, const char *path,
+			     MY_STAT *mystat);
+static int local_write(ds_file_t *file, const uchar *buf, size_t len);
+static int local_close(ds_file_t *file);
+static void local_deinit(ds_ctxt_t *ctxt);
+
+extern "C" {
+datasink_t datasink_local = {
+	&local_init,
+	&local_open,
+	&local_write,
+	&local_close,
+	&local_deinit
+};
+}
+
+static
+ds_ctxt_t *
+local_init(const char *root)
+{
+	ds_ctxt_t *ctxt;
+
+	if (my_mkdir(root, 0777, MYF(0)) < 0
+	    && my_errno != EEXIST && my_errno != EISDIR)
+	{
+		char errbuf[MYSYS_STRERROR_SIZE];
+		my_strerror(errbuf, sizeof(errbuf),my_errno);
+		my_error(EE_CANT_MKDIR, MYF(ME_BELL | ME_WAITTANG),
+			 root, my_errno,errbuf, my_errno);
+		return NULL;
+	}
+
+	ctxt = (ds_ctxt_t *)my_malloc(sizeof(ds_ctxt_t), MYF(MY_FAE));
+
+	ctxt->root = my_strdup(root, MYF(MY_FAE));
+
+	return ctxt;
+}
+
+static
+ds_file_t *
+local_open(ds_ctxt_t *ctxt, const char *path,
+	   MY_STAT *mystat __attribute__((unused)))
+{
+	char 		fullpath[FN_REFLEN];
+	char		dirpath[FN_REFLEN];
+	size_t		dirpath_len;
+	size_t		path_len;
+	ds_local_file_t *local_file;
+	ds_file_t	*file;
+	File 		fd;
+
+	fn_format(fullpath, path, ctxt->root, "", MYF(MY_RELATIVE_PATH));
+
+	/* Create the directory if needed */
+	dirname_part(dirpath, fullpath, &dirpath_len);
+	if (my_mkdir(dirpath, 0777, MYF(0)) < 0 && my_errno != EEXIST) {
+		char errbuf[MYSYS_STRERROR_SIZE];
+		my_strerror(errbuf, sizeof(errbuf), my_errno);
+		my_error(EE_CANT_MKDIR, MYF(ME_BELL | ME_WAITTANG),
+			 dirpath, my_errno, errbuf);
+		return NULL;
+	}
+
+	fd = my_create(fullpath, 0, O_WRONLY | O_BINARY | O_EXCL | O_NOFOLLOW,
+		     MYF(MY_WME));
+	if (fd < 0) {
+		return NULL;
+	}
+
+	path_len = strlen(fullpath) + 1; /* terminating '\0' */
+
+	file = (ds_file_t *) my_malloc(sizeof(ds_file_t) +
+				       sizeof(ds_local_file_t) +
+				       path_len,
+				       MYF(MY_FAE));
+	local_file = (ds_local_file_t *) (file + 1);
+
+	local_file->fd = fd;
+	local_file->init_ibd_done = 0;
+	local_file->is_ibd = (path_len > 5) && !strcmp(fullpath + path_len - 5, ".ibd");
+	local_file->compressed = 0;
+	local_file->pagesize = 0;
+	file->path = (char *) local_file + sizeof(ds_local_file_t);
+	memcpy(file->path, fullpath, path_len);
+
+	file->ptr = local_file;
+
+	return file;
+}
+
+/* Calculate size of data without trailing zero bytes. */
+static size_t trim_binary_zeros(uchar *buf, size_t pagesize)
+{
+	size_t i;
+	for (i = pagesize; (i > 0) && (buf[i - 1] == 0); i--) {};
+	return i;
+}
+
+
+/*  Write data to the output file, and punch "holes" if needed. */
+static int write_compressed(File fd,  uchar *data, size_t len, size_t pagesize)
+{
+	uchar *ptr = data;
+	for (size_t written= 0; written < len;)
+	{
+		size_t n_bytes =  MY_MIN(pagesize, len - written);
+		size_t datasize= trim_binary_zeros(ptr,n_bytes);
+		if (datasize > 0) {
+			if (!my_write(fd, ptr, datasize, MYF(MY_WME | MY_NABP)))
+				posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED);
+			else
+				return 1;
+		}
+		if (datasize < n_bytes) {
+			/* This punches a "hole" in the file. */
+			size_t hole_bytes = n_bytes - datasize;
+			if (my_seek(fd, hole_bytes, MY_SEEK_CUR, MYF(MY_WME | MY_NABP))
+				== MY_FILEPOS_ERROR)
+			 return 1;
+		}
+		written += n_bytes;
+		ptr += n_bytes;
+	}
+	return 0;
+}
+
+
+/* Calculate Innodb tablespace specific data, when first page is written.
+   We're interested in page compression and page size.
+*/
+static void init_ibd_data(ds_local_file_t *local_file, const uchar *buf, size_t len)
+{
+	if (len < FIL_PAGE_DATA + FSP_SPACE_FLAGS) {
+		/* Weird, bail out.*/
+		return;
+	}
+
+	ulint flags = mach_read_from_4(&buf[FIL_PAGE_DATA + FSP_SPACE_FLAGS]);
+	ulint ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags);
+	local_file->pagesize= ssize == 0 ? UNIV_PAGE_SIZE_ORIG : ((UNIV_ZIP_SIZE_MIN >> 1) << ssize);
+	local_file->compressed =  (my_bool)FSP_FLAGS_HAS_PAGE_COMPRESSION(flags);
+
+#if defined(_WIN32) && (MYSQL_VERSION_ID > 100200)
+	/* Make compressed file sparse, on Windows.
+	In 10.1, we do not use sparse files. */
+	if (local_file->compressed) {
+		HANDLE handle= my_get_osfhandle(local_file->fd);
+		if (!DeviceIoControl(handle, FSCTL_SET_SPARSE, NULL, 0, NULL, 0, NULL, 0)) {
+			fprintf(stderr, "Warning: cannot make file sparse");
+			local_file->compressed = 0;
+		}
+	}
+#endif
+}
+
+
+static
+int
+local_write(ds_file_t *file, const uchar *buf, size_t len)
+{
+	uchar *b = (uchar*)buf;
+	ds_local_file_t *local_file= (ds_local_file_t *)file->ptr;
+	File fd = local_file->fd;
+
+	if (local_file->is_ibd && !local_file->init_ibd_done) {
+		init_ibd_data(local_file, b , len);
+		local_file->init_ibd_done= 1;
+	}
+
+	if (local_file->compressed) {
+		return write_compressed(fd, b, len, local_file->pagesize);
+	}
+
+	if (!my_write(fd, b , len, MYF(MY_WME | MY_NABP))) {
+		posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED);
+		return 0;
+	}
+	return 1;
+}
+
+/* Set EOF at file's current position.*/
+static int set_eof(File fd)
+{
+#ifdef _WIN32
+	return !SetEndOfFile(my_get_osfhandle(fd));
+#elif defined(HAVE_FTRUNCATE)
+	return ftruncate(fd, my_tell(fd, MYF(MY_WME)));
+#else
+#error no ftruncate
+#endif
+}
+
+
+static
+int
+local_close(ds_file_t *file)
+{
+	ds_local_file_t *local_file= (ds_local_file_t *)file->ptr;
+	File fd = local_file->fd;
+	int ret= 0;
+
+	if (local_file->compressed) {
+		ret = set_eof(fd);
+	}
+
+	my_close(fd, MYF(MY_WME));
+	my_free(file);
+	return ret;
+}
+
+static
+void
+local_deinit(ds_ctxt_t *ctxt)
+{
+	my_free(ctxt->root);
+	my_free(ctxt);
+}
diff --git a/extra/mariabackup/ds_local.h b/extra/mariabackup/ds_local.h
index b0f0f04030c..e30906b575d 100644
--- a/extra/mariabackup/ds_local.h
+++ b/extra/mariabackup/ds_local.h
@@ -23,6 +23,12 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
 
 #include "datasink.h"
 
-extern datasink_t datasink_local;
+#ifdef __cplusplus
+extern "C"
+#else
+extern
+#endif
+
+datasink_t datasink_local;
 
 #endif
diff --git a/extra/mariabackup/ds_stdout.c b/extra/mariabackup/ds_stdout.c
index 91a514ddf64..391a3455195 100644
--- a/extra/mariabackup/ds_stdout.c
+++ b/extra/mariabackup/ds_stdout.c
@@ -30,7 +30,7 @@ typedef struct {
 static ds_ctxt_t *stdout_init(const char *root);
 static ds_file_t *stdout_open(ds_ctxt_t *ctxt, const char *path,
 			     MY_STAT *mystat);
-static int stdout_write(ds_file_t *file, const void *buf, size_t len);
+static int stdout_write(ds_file_t *file, const uchar *buf, size_t len);
 static int stdout_close(ds_file_t *file);
 static void stdout_deinit(ds_ctxt_t *ctxt);
 
@@ -91,7 +91,7 @@ stdout_open(ds_ctxt_t *ctxt __attribute__((unused)),
 
 static
 int
-stdout_write(ds_file_t *file, const void *buf, size_t len)
+stdout_write(ds_file_t *file, const uchar *buf, size_t len)
 {
 	File fd = ((ds_stdout_file_t *) file->ptr)->fd;
 
diff --git a/extra/mariabackup/ds_tmpfile.c b/extra/mariabackup/ds_tmpfile.c
index b039d83ba03..27a8d9688f4 100644
--- a/extra/mariabackup/ds_tmpfile.c
+++ b/extra/mariabackup/ds_tmpfile.c
@@ -41,7 +41,7 @@ typedef struct {
 static ds_ctxt_t *tmpfile_init(const char *root);
 static ds_file_t *tmpfile_open(ds_ctxt_t *ctxt, const char *path,
 			       MY_STAT *mystat);
-static int tmpfile_write(ds_file_t *file, const void *buf, size_t len);
+static int tmpfile_write(ds_file_t *file, const uchar *buf, size_t len);
 static int tmpfile_close(ds_file_t *file);
 static void tmpfile_deinit(ds_ctxt_t *ctxt);
 
@@ -143,7 +143,7 @@ tmpfile_open(ds_ctxt_t *ctxt, const char *path,
 }
 
 static int
-tmpfile_write(ds_file_t *file, const void *buf, size_t len)
+tmpfile_write(ds_file_t *file, const uchar *buf, size_t len)
 {
 	File fd = ((ds_tmp_file_t *) file->ptr)->fd;
 
diff --git a/extra/mariabackup/ds_xbstream.c b/extra/mariabackup/ds_xbstream.c
index 42924a72d7f..544929fb24c 100644
--- a/extra/mariabackup/ds_xbstream.c
+++ b/extra/mariabackup/ds_xbstream.c
@@ -41,7 +41,7 @@ General streaming interface */
 static ds_ctxt_t *xbstream_init(const char *root);
 static ds_file_t *xbstream_open(ds_ctxt_t *ctxt, const char *path,
 			      MY_STAT *mystat);
-static int xbstream_write(ds_file_t *file, const void *buf, size_t len);
+static int xbstream_write(ds_file_t *file, const uchar *buf, size_t len);
 static int xbstream_close(ds_file_t *file);
 static void xbstream_deinit(ds_ctxt_t *ctxt);
 
@@ -166,7 +166,7 @@ err:
 
 static
 int
-xbstream_write(ds_file_t *file, const void *buf, size_t len)
+xbstream_write(ds_file_t *file, const uchar *buf, size_t len)
 {
 	ds_stream_file_t	*stream_file;
 	xb_wstream_file_t	*xbstream_file;