summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorVladislav Vaintroub <wlad@mariadb.com>2017-04-19 13:09:03 +0000
committerSergei Golubchik <serg@mariadb.org>2017-04-27 19:12:42 +0200
commitecb25df21b2cc9bcbb038dd9b82b8469acd18fd9 (patch)
tree08eba6fd38ecb81d73570371ba213a958299abc3
parentc8ac0244a843252d59985ad270cad2f074aebfb4 (diff)
downloadmariadb-git-ecb25df21b2cc9bcbb038dd9b82b8469acd18fd9.tar.gz
Xtrabackup 2.3.8
-rw-r--r--extra/mariabackup/CMakeLists.txt15
-rw-r--r--extra/mariabackup/backup_copy.cc7
-rw-r--r--extra/mariabackup/backup_mysql.cc5
-rw-r--r--extra/mariabackup/crc/CMakeLists.txt33
-rw-r--r--extra/mariabackup/crc/config.h.cmake21
-rw-r--r--extra/mariabackup/crc/crc-intel-pclmul.c511
-rw-r--r--extra/mariabackup/crc/crc-intel-pclmul.h25
-rw-r--r--extra/mariabackup/crc/crc_glue.c72
-rw-r--r--extra/mariabackup/crc/crc_glue.h31
-rw-r--r--extra/mariabackup/datasink.c2
-rw-r--r--extra/mariabackup/datasink.h1
-rw-r--r--extra/mariabackup/ds_decrypt.c665
-rw-r--r--extra/mariabackup/ds_decrypt.h30
-rw-r--r--extra/mariabackup/ds_encrypt.c220
-rw-r--r--extra/mariabackup/ds_encrypt.h5
-rw-r--r--extra/mariabackup/xbcrypt.c12
-rw-r--r--extra/mariabackup/xbcrypt.h7
-rw-r--r--extra/mariabackup/xbcrypt_common.c284
-rw-r--r--extra/mariabackup/xbcrypt_common.h64
-rw-r--r--extra/mariabackup/xbcrypt_read.c3
-rw-r--r--extra/mariabackup/xbcrypt_write.c3
-rw-r--r--extra/mariabackup/xbstream.c257
-rw-r--r--extra/mariabackup/xbstream.h6
-rw-r--r--extra/mariabackup/xbstream_read.c53
-rw-r--r--extra/mariabackup/xbstream_write.c6
-rw-r--r--extra/mariabackup/xtrabackup.cc423
-rw-r--r--extra/mariabackup/xtrabackup.h13
-rw-r--r--mysql-test/suite/mariabackup/partial_exclude.result12
-rw-r--r--mysql-test/suite/mariabackup/partial_exclude.test30
-rw-r--r--mysql-test/suite/mariabackup/xbstream.test2
-rw-r--r--storage/xtradb/fil/fil0fil.cc18
31 files changed, 2414 insertions, 422 deletions
diff --git a/extra/mariabackup/CMakeLists.txt b/extra/mariabackup/CMakeLists.txt
index 6f5a17d3614..693082b765a 100644
--- a/extra/mariabackup/CMakeLists.txt
+++ b/extra/mariabackup/CMakeLists.txt
@@ -1,14 +1,14 @@
-# Copyright (c) 2013 Percona LLC and/or its affiliates.
+# Copyright (c) 2013, 2017 Percona LLC and/or its affiliates.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
-
+#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
-
+#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
@@ -129,6 +129,7 @@ INCLUDE_DIRECTORIES(
${CMAKE_SOURCE_DIR}/sql
${CMAKE_CURRENT_SOURCE_DIR}/quicklz
${CMAKE_CURRENT_SOURCE_DIR}
+ ${CMAKE_CURRENT_SOURCE_DIR}/crc
)
IF(NOT HAVE_SYSTEM_REGEX)
@@ -156,7 +157,6 @@ MYSQL_ADD_EXECUTABLE(mariabackup
${DS_ARCHIVE_SOURCE}
ds_buffer.c
ds_compress.c
- ds_encrypt.c
ds_local.c
ds_stdout.c
ds_tmpfile.c
@@ -166,8 +166,6 @@ MYSQL_ADD_EXECUTABLE(mariabackup
read_filt.cc
write_filt.cc
wsrep.cc
- xbcrypt_common.c
- xbcrypt_write.c
xbstream_write.c
backup_mysql.cc
backup_copy.cc
@@ -181,9 +179,10 @@ MYSQL_ADD_EXECUTABLE(mariabackup
# Export all symbols on Unix, for better crash callstacks
SET_TARGET_PROPERTIES(mariabackup PROPERTIES ENABLE_EXPORTS TRUE)
+ADD_SUBDIRECTORY(crc)
-TARGET_LINK_LIBRARIES(mariabackup sql)
+TARGET_LINK_LIBRARIES(mariabackup sql crc)
IF(NOT HAVE_SYSTEM_REGEX)
TARGET_LINK_LIBRARIES(mariabackup pcreposix)
@@ -201,13 +200,13 @@ MYSQL_ADD_EXECUTABLE(mbstream
xbstream.c
xbstream_read.c
xbstream_write.c
-
COMPONENT backup
)
TARGET_LINK_LIBRARIES(mbstream
mysys
+ crc
)
IF(MSVC)
diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc
index 478a6a66b97..67c96b4b648 100644
--- a/extra/mariabackup/backup_copy.cc
+++ b/extra/mariabackup/backup_copy.cc
@@ -265,6 +265,11 @@ datadir_iter_next_database(datadir_iter_t *it)
return(true);
}
+ if (check_if_skip_database_by_path(it->dbpath)) {
+ msg("Skipping db: %s\n", it->dbpath);
+ continue;
+ }
+
/* We want wrong directory permissions to be a fatal error for
XtraBackup. */
it->dbdir = os_file_opendir(it->dbpath, TRUE);
@@ -1704,7 +1709,7 @@ copy_back()
for (i = 1; i <= srv_undo_tablespaces; i++) {
char filename[20];
- sprintf(filename, "undo%03lu", i);
+ sprintf(filename, "undo%03u", (uint)i);
if (!(ret = copy_or_move_file(filename, filename,
dst_dir, 1))) {
goto cleanup;
diff --git a/extra/mariabackup/backup_mysql.cc b/extra/mariabackup/backup_mysql.cc
index ed5446b4806..52aaf138fc8 100644
--- a/extra/mariabackup/backup_mysql.cc
+++ b/extra/mariabackup/backup_mysql.cc
@@ -1414,7 +1414,10 @@ write_xtrabackup_info(MYSQL *connection)
bool is_partial = (xtrabackup_tables
|| xtrabackup_tables_file
|| xtrabackup_databases
- || xtrabackup_databases_file);
+ || xtrabackup_databases_file
+ || xtrabackup_tables_exclude
+ || xtrabackup_databases_exclude
+ );
backup_file_printf(XTRABACKUP_INFO,
"uuid = %s\n"
diff --git a/extra/mariabackup/crc/CMakeLists.txt b/extra/mariabackup/crc/CMakeLists.txt
new file mode 100644
index 00000000000..577cab6080c
--- /dev/null
+++ b/extra/mariabackup/crc/CMakeLists.txt
@@ -0,0 +1,33 @@
+# Copyright (c) 2017 Percona LLC and/or its affiliates.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+PROJECT(crc C)
+
+IF(NOT CMAKE_CROSSCOMPILING AND NOT MSVC)
+ STRING(TOLOWER ${CMAKE_SYSTEM_PROCESSOR} processor)
+ IF(processor MATCHES "86" OR processor MATCHES "amd64" OR processor MATCHES "x64")
+ # Check for PCLMUL instruction
+ CHECK_C_SOURCE_RUNS("
+ int main()
+ {
+ asm volatile (\"pclmulqdq \\$0x00, %%xmm1, %%xmm0\":::\"cc\");
+ return 0;
+ }" HAVE_CLMUL_INSTRUCTION)
+ ENDIF()
+ENDIF()
+IF(HAVE_CLMUL_INSTRUCTION)
+ ADD_DEFINITIONS(-DHAVE_CLMUL_INSTRUCTION)
+ENDIF()
+ADD_LIBRARY(crc crc_glue.c crc-intel-pclmul.c)
diff --git a/extra/mariabackup/crc/config.h.cmake b/extra/mariabackup/crc/config.h.cmake
new file mode 100644
index 00000000000..fe81c1859ae
--- /dev/null
+++ b/extra/mariabackup/crc/config.h.cmake
@@ -0,0 +1,21 @@
+/******************************************************
+Copyright (c) 2017 Percona LLC and/or its affiliates.
+
+Zlib compatible CRC-32 implementation.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+#cmakedefine HAVE_CLMUL_INSTRUCTION 1
diff --git a/extra/mariabackup/crc/crc-intel-pclmul.c b/extra/mariabackup/crc/crc-intel-pclmul.c
new file mode 100644
index 00000000000..d470c2bee43
--- /dev/null
+++ b/extra/mariabackup/crc/crc-intel-pclmul.c
@@ -0,0 +1,511 @@
+/******************************************************
+Copyright (c) 2017 Percona LLC and/or its affiliates.
+
+CRC32 using Intel's PCLMUL instruction.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+/* crc-intel-pclmul.c - Intel PCLMUL accelerated CRC implementation
+ * Copyright (C) 2016 Jussi Kivilinna <jussi.kivilinna@iki.fi>
+ *
+ * This file is part of Libgcrypt.
+ *
+ * Libgcrypt is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * Libgcrypt is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA
+ *
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+
+# define U64_C(c) (c ## UL)
+
+typedef uint32_t u32;
+typedef uint16_t u16;
+typedef uint64_t u64;
+#ifndef byte
+typedef uint8_t byte;
+#endif
+
+# define _gcry_bswap32 __builtin_bswap32
+
+#if __GNUC__ >= 4 && defined(__x86_64__) && defined(HAVE_CLMUL_INSTRUCTION)
+
+#if _GCRY_GCC_VERSION >= 40400 /* 4.4 */
+/* Prevent compiler from issuing SSE instructions between asm blocks. */
+# pragma GCC target("no-sse")
+#endif
+
+
+#define ALIGNED_16 __attribute__ ((aligned (16)))
+
+
+struct u16_unaligned_s
+{
+ u16 a;
+} __attribute__((packed, aligned (1), may_alias));
+
+
+/* Constants structure for generic reflected/non-reflected CRC32 CLMUL
+ * functions. */
+struct crc32_consts_s
+{
+ /* k: { x^(32*17), x^(32*15), x^(32*5), x^(32*3), x^(32*2), 0 } mod P(x) */
+ u64 k[6];
+ /* my_p: { floor(x^64 / P(x)), P(x) } */
+ u64 my_p[2];
+};
+
+
+/* CLMUL constants for CRC32 and CRC32RFC1510. */
+static const struct crc32_consts_s crc32_consts ALIGNED_16 =
+{
+ { /* k[6] = reverse_33bits( x^(32*y) mod P(x) ) */
+ U64_C(0x154442bd4), U64_C(0x1c6e41596), /* y = { 17, 15 } */
+ U64_C(0x1751997d0), U64_C(0x0ccaa009e), /* y = { 5, 3 } */
+ U64_C(0x163cd6124), 0 /* y = 2 */
+ },
+ { /* my_p[2] = reverse_33bits ( { floor(x^64 / P(x)), P(x) } ) */
+ U64_C(0x1f7011641), U64_C(0x1db710641)
+ }
+};
+
+/* Common constants for CRC32 algorithms. */
+static const byte crc32_refl_shuf_shift[3 * 16] ALIGNED_16 =
+ {
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ };
+static const byte crc32_partial_fold_input_mask[16 + 16] ALIGNED_16 =
+ {
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ };
+static const u64 crc32_merge9to15_shuf[15 - 9 + 1][2] ALIGNED_16 =
+ {
+ { U64_C(0x0706050403020100), U64_C(0xffffffffffffff0f) }, /* 9 */
+ { U64_C(0x0706050403020100), U64_C(0xffffffffffff0f0e) },
+ { U64_C(0x0706050403020100), U64_C(0xffffffffff0f0e0d) },
+ { U64_C(0x0706050403020100), U64_C(0xffffffff0f0e0d0c) },
+ { U64_C(0x0706050403020100), U64_C(0xffffff0f0e0d0c0b) },
+ { U64_C(0x0706050403020100), U64_C(0xffff0f0e0d0c0b0a) },
+ { U64_C(0x0706050403020100), U64_C(0xff0f0e0d0c0b0a09) }, /* 15 */
+ };
+static const u64 crc32_merge5to7_shuf[7 - 5 + 1][2] ALIGNED_16 =
+ {
+ { U64_C(0xffffff0703020100), U64_C(0xffffffffffffffff) }, /* 5 */
+ { U64_C(0xffff070603020100), U64_C(0xffffffffffffffff) },
+ { U64_C(0xff07060503020100), U64_C(0xffffffffffffffff) }, /* 7 */
+ };
+
+/* PCLMUL functions for reflected CRC32. */
+static inline void
+crc32_reflected_bulk (u32 *pcrc, const byte *inbuf, size_t inlen,
+ const struct crc32_consts_s *consts)
+{
+ if (inlen >= 8 * 16)
+ {
+ asm volatile ("movd %[crc], %%xmm4\n\t"
+ "movdqu %[inbuf_0], %%xmm0\n\t"
+ "movdqu %[inbuf_1], %%xmm1\n\t"
+ "movdqu %[inbuf_2], %%xmm2\n\t"
+ "movdqu %[inbuf_3], %%xmm3\n\t"
+ "pxor %%xmm4, %%xmm0\n\t"
+ :
+ : [inbuf_0] "m" (inbuf[0 * 16]),
+ [inbuf_1] "m" (inbuf[1 * 16]),
+ [inbuf_2] "m" (inbuf[2 * 16]),
+ [inbuf_3] "m" (inbuf[3 * 16]),
+ [crc] "m" (*pcrc)
+ );
+
+ inbuf += 4 * 16;
+ inlen -= 4 * 16;
+
+ asm volatile ("movdqa %[k1k2], %%xmm4\n\t"
+ :
+ : [k1k2] "m" (consts->k[1 - 1])
+ );
+
+ /* Fold by 4. */
+ while (inlen >= 4 * 16)
+ {
+ asm volatile ("movdqu %[inbuf_0], %%xmm5\n\t"
+ "movdqa %%xmm0, %%xmm6\n\t"
+ "pclmulqdq $0x00, %%xmm4, %%xmm0\n\t"
+ "pclmulqdq $0x11, %%xmm4, %%xmm6\n\t"
+ "pxor %%xmm5, %%xmm0\n\t"
+ "pxor %%xmm6, %%xmm0\n\t"
+
+ "movdqu %[inbuf_1], %%xmm5\n\t"
+ "movdqa %%xmm1, %%xmm6\n\t"
+ "pclmulqdq $0x00, %%xmm4, %%xmm1\n\t"
+ "pclmulqdq $0x11, %%xmm4, %%xmm6\n\t"
+ "pxor %%xmm5, %%xmm1\n\t"
+ "pxor %%xmm6, %%xmm1\n\t"
+
+ "movdqu %[inbuf_2], %%xmm5\n\t"
+ "movdqa %%xmm2, %%xmm6\n\t"
+ "pclmulqdq $0x00, %%xmm4, %%xmm2\n\t"
+ "pclmulqdq $0x11, %%xmm4, %%xmm6\n\t"
+ "pxor %%xmm5, %%xmm2\n\t"
+ "pxor %%xmm6, %%xmm2\n\t"
+
+ "movdqu %[inbuf_3], %%xmm5\n\t"
+ "movdqa %%xmm3, %%xmm6\n\t"
+ "pclmulqdq $0x00, %%xmm4, %%xmm3\n\t"
+ "pclmulqdq $0x11, %%xmm4, %%xmm6\n\t"
+ "pxor %%xmm5, %%xmm3\n\t"
+ "pxor %%xmm6, %%xmm3\n\t"
+ :
+ : [inbuf_0] "m" (inbuf[0 * 16]),
+ [inbuf_1] "m" (inbuf[1 * 16]),
+ [inbuf_2] "m" (inbuf[2 * 16]),
+ [inbuf_3] "m" (inbuf[3 * 16])
+ );
+
+ inbuf += 4 * 16;
+ inlen -= 4 * 16;
+ }
+
+ asm volatile ("movdqa %[k3k4], %%xmm6\n\t"
+ "movdqa %[my_p], %%xmm5\n\t"
+ :
+ : [k3k4] "m" (consts->k[3 - 1]),
+ [my_p] "m" (consts->my_p[0])
+ );
+
+ /* Fold 4 to 1. */
+
+ asm volatile ("movdqa %%xmm0, %%xmm4\n\t"
+ "pclmulqdq $0x00, %%xmm6, %%xmm0\n\t"
+ "pclmulqdq $0x11, %%xmm6, %%xmm4\n\t"
+ "pxor %%xmm1, %%xmm0\n\t"
+ "pxor %%xmm4, %%xmm0\n\t"
+
+ "movdqa %%xmm0, %%xmm4\n\t"
+ "pclmulqdq $0x00, %%xmm6, %%xmm0\n\t"
+ "pclmulqdq $0x11, %%xmm6, %%xmm4\n\t"
+ "pxor %%xmm2, %%xmm0\n\t"
+ "pxor %%xmm4, %%xmm0\n\t"
+
+ "movdqa %%xmm0, %%xmm4\n\t"
+ "pclmulqdq $0x00, %%xmm6, %%xmm0\n\t"
+ "pclmulqdq $0x11, %%xmm6, %%xmm4\n\t"
+ "pxor %%xmm3, %%xmm0\n\t"
+ "pxor %%xmm4, %%xmm0\n\t"
+ :
+ :
+ );
+ }
+ else
+ {
+ asm volatile ("movd %[crc], %%xmm1\n\t"
+ "movdqu %[inbuf], %%xmm0\n\t"
+ "movdqa %[k3k4], %%xmm6\n\t"
+ "pxor %%xmm1, %%xmm0\n\t"
+ "movdqa %[my_p], %%xmm5\n\t"
+ :
+ : [inbuf] "m" (*inbuf),
+ [crc] "m" (*pcrc),
+ [k3k4] "m" (consts->k[3 - 1]),
+ [my_p] "m" (consts->my_p[0])
+ );
+
+ inbuf += 16;
+ inlen -= 16;
+ }
+
+ /* Fold by 1. */
+ if (inlen >= 16)
+ {
+ while (inlen >= 16)
+ {
+ /* Load next block to XMM2. Fold XMM0 to XMM0:XMM1. */
+ asm volatile ("movdqu %[inbuf], %%xmm2\n\t"
+ "movdqa %%xmm0, %%xmm1\n\t"
+ "pclmulqdq $0x00, %%xmm6, %%xmm0\n\t"
+ "pclmulqdq $0x11, %%xmm6, %%xmm1\n\t"
+ "pxor %%xmm2, %%xmm0\n\t"
+ "pxor %%xmm1, %%xmm0\n\t"
+ :
+ : [inbuf] "m" (*inbuf)
+ );
+
+ inbuf += 16;
+ inlen -= 16;
+ }
+ }
+
+ /* Partial fold. */
+ if (inlen)
+ {
+ /* Load last input and add padding zeros. */
+ asm volatile ("movdqu %[shr_shuf], %%xmm3\n\t"
+ "movdqu %[shl_shuf], %%xmm4\n\t"
+ "movdqu %[mask], %%xmm2\n\t"
+
+ "movdqa %%xmm0, %%xmm1\n\t"
+ "pshufb %%xmm4, %%xmm0\n\t"
+ "movdqu %[inbuf], %%xmm4\n\t"
+ "pshufb %%xmm3, %%xmm1\n\t"
+ "pand %%xmm4, %%xmm2\n\t"
+ "por %%xmm1, %%xmm2\n\t"
+
+ "movdqa %%xmm0, %%xmm1\n\t"
+ "pclmulqdq $0x00, %%xmm6, %%xmm0\n\t"
+ "pclmulqdq $0x11, %%xmm6, %%xmm1\n\t"
+ "pxor %%xmm2, %%xmm0\n\t"
+ "pxor %%xmm1, %%xmm0\n\t"
+ :
+ : [inbuf] "m" (*(inbuf - 16 + inlen)),
+ [mask] "m" (crc32_partial_fold_input_mask[inlen]),
+ [shl_shuf] "m" (crc32_refl_shuf_shift[inlen]),
+ [shr_shuf] "m" (crc32_refl_shuf_shift[inlen + 16])
+ );
+
+ inbuf += inlen;
+ inlen -= inlen;
+ }
+
+ /* Final fold. */
+ asm volatile (/* reduce 128-bits to 96-bits */
+ "movdqa %%xmm0, %%xmm1\n\t"
+ "pclmulqdq $0x10, %%xmm6, %%xmm0\n\t"
+ "psrldq $8, %%xmm1\n\t"
+ "pxor %%xmm1, %%xmm0\n\t"
+
+ /* reduce 96-bits to 64-bits */
+ "pshufd $0xfc, %%xmm0, %%xmm1\n\t" /* [00][00][00][x] */
+ "pshufd $0xf9, %%xmm0, %%xmm0\n\t" /* [00][00][x>>64][x>>32] */
+ "pclmulqdq $0x00, %[k5], %%xmm1\n\t" /* [00][00][xx][xx] */
+ "pxor %%xmm1, %%xmm0\n\t" /* top 64-bit are zero */
+
+ /* barrett reduction */
+ "pshufd $0xf3, %%xmm0, %%xmm1\n\t" /* [00][00][x>>32][00] */
+ "pslldq $4, %%xmm0\n\t" /* [??][x>>32][??][??] */
+ "pclmulqdq $0x00, %%xmm5, %%xmm1\n\t" /* [00][xx][xx][00] */
+ "pclmulqdq $0x10, %%xmm5, %%xmm1\n\t" /* [00][xx][xx][00] */
+ "pxor %%xmm1, %%xmm0\n\t"
+
+ /* store CRC */
+ "pextrd $2, %%xmm0, %[out]\n\t"
+ : [out] "=m" (*pcrc)
+ : [k5] "m" (consts->k[5 - 1])
+ );
+}
+
+static inline void
+crc32_reflected_less_than_16 (u32 *pcrc, const byte *inbuf, size_t inlen,
+ const struct crc32_consts_s *consts)
+{
+ if (inlen < 4)
+ {
+ u32 crc = *pcrc;
+ u32 data;
+
+ asm volatile ("movdqa %[my_p], %%xmm5\n\t"
+ :
+ : [my_p] "m" (consts->my_p[0])
+ );
+
+ if (inlen == 1)
+ {
+ data = inbuf[0];
+ data ^= crc;
+ data <<= 24;
+ crc >>= 8;
+ }
+ else if (inlen == 2)
+ {
+ data = ((const struct u16_unaligned_s *)inbuf)->a;
+ data ^= crc;
+ data <<= 16;
+ crc >>= 16;
+ }
+ else
+ {
+ data = ((const struct u16_unaligned_s *)inbuf)->a;
+ data |= inbuf[2] << 16;
+ data ^= crc;
+ data <<= 8;
+ crc >>= 24;
+ }
+
+ /* Barrett reduction */
+ asm volatile ("movd %[in], %%xmm0\n\t"
+ "movd %[crc], %%xmm1\n\t"
+
+ "pclmulqdq $0x00, %%xmm5, %%xmm0\n\t" /* [00][00][xx][xx] */
+ "psllq $32, %%xmm1\n\t"
+ "pshufd $0xfc, %%xmm0, %%xmm0\n\t" /* [00][00][00][x] */
+ "pclmulqdq $0x10, %%xmm5, %%xmm0\n\t" /* [00][00][xx][xx] */
+ "pxor %%xmm1, %%xmm0\n\t"
+
+ "pextrd $1, %%xmm0, %[out]\n\t"
+ : [out] "=m" (*pcrc)
+ : [in] "rm" (data),
+ [crc] "rm" (crc)
+ );
+ }
+ else if (inlen == 4)
+ {
+ /* Barrett reduction */
+ asm volatile ("movd %[crc], %%xmm1\n\t"
+ "movd %[in], %%xmm0\n\t"
+ "movdqa %[my_p], %%xmm5\n\t"
+ "pxor %%xmm1, %%xmm0\n\t"
+
+ "pclmulqdq $0x00, %%xmm5, %%xmm0\n\t" /* [00][00][xx][xx] */
+ "pshufd $0xfc, %%xmm0, %%xmm0\n\t" /* [00][00][00][x] */
+ "pclmulqdq $0x10, %%xmm5, %%xmm0\n\t" /* [00][00][xx][xx] */
+
+ "pextrd $1, %%xmm0, %[out]\n\t"
+ : [out] "=m" (*pcrc)
+ : [in] "m" (*inbuf),
+ [crc] "m" (*pcrc),
+ [my_p] "m" (consts->my_p[0])
+ );
+ }
+ else
+ {
+ asm volatile ("movdqu %[shuf], %%xmm4\n\t"
+ "movd %[crc], %%xmm1\n\t"
+ "movdqa %[my_p], %%xmm5\n\t"
+ "movdqa %[k3k4], %%xmm6\n\t"
+ :
+ : [shuf] "m" (crc32_refl_shuf_shift[inlen]),
+ [crc] "m" (*pcrc),
+ [my_p] "m" (consts->my_p[0]),
+ [k3k4] "m" (consts->k[3 - 1])
+ );
+
+ if (inlen >= 8)
+ {
+ asm volatile ("movq %[inbuf], %%xmm0\n\t"
+ :
+ : [inbuf] "m" (*inbuf)
+ );
+ if (inlen > 8)
+ {
+ asm volatile (/*"pinsrq $1, %[inbuf_tail], %%xmm0\n\t"*/
+ "movq %[inbuf_tail], %%xmm2\n\t"
+ "punpcklqdq %%xmm2, %%xmm0\n\t"
+ "pshufb %[merge_shuf], %%xmm0\n\t"
+ :
+ : [inbuf_tail] "m" (inbuf[inlen - 8]),
+ [merge_shuf] "m"
+ (*crc32_merge9to15_shuf[inlen - 9])
+ );
+ }
+ }
+ else
+ {
+ asm volatile ("movd %[inbuf], %%xmm0\n\t"
+ "pinsrd $1, %[inbuf_tail], %%xmm0\n\t"
+ "pshufb %[merge_shuf], %%xmm0\n\t"
+ :
+ : [inbuf] "m" (*inbuf),
+ [inbuf_tail] "m" (inbuf[inlen - 4]),
+ [merge_shuf] "m"
+ (*crc32_merge5to7_shuf[inlen - 5])
+ );
+ }
+
+ /* Final fold. */
+ asm volatile ("pxor %%xmm1, %%xmm0\n\t"
+ "pshufb %%xmm4, %%xmm0\n\t"
+
+ /* reduce 128-bits to 96-bits */
+ "movdqa %%xmm0, %%xmm1\n\t"
+ "pclmulqdq $0x10, %%xmm6, %%xmm0\n\t"
+ "psrldq $8, %%xmm1\n\t"
+ "pxor %%xmm1, %%xmm0\n\t" /* top 32-bit are zero */
+
+ /* reduce 96-bits to 64-bits */
+ "pshufd $0xfc, %%xmm0, %%xmm1\n\t" /* [00][00][00][x] */
+ "pshufd $0xf9, %%xmm0, %%xmm0\n\t" /* [00][00][x>>64][x>>32] */
+ "pclmulqdq $0x00, %[k5], %%xmm1\n\t" /* [00][00][xx][xx] */
+ "pxor %%xmm1, %%xmm0\n\t" /* top 64-bit are zero */
+
+ /* barrett reduction */
+ "pshufd $0xf3, %%xmm0, %%xmm1\n\t" /* [00][00][x>>32][00] */
+ "pslldq $4, %%xmm0\n\t" /* [??][x>>32][??][??] */
+ "pclmulqdq $0x00, %%xmm5, %%xmm1\n\t" /* [00][xx][xx][00] */
+ "pclmulqdq $0x10, %%xmm5, %%xmm1\n\t" /* [00][xx][xx][00] */
+ "pxor %%xmm1, %%xmm0\n\t"
+
+ /* store CRC */
+ "pextrd $2, %%xmm0, %[out]\n\t"
+ : [out] "=m" (*pcrc)
+ : [k5] "m" (consts->k[5 - 1])
+ );
+ }
+}
+
+void
+crc32_intel_pclmul (u32 *pcrc, const byte *inbuf, size_t inlen)
+{
+ const struct crc32_consts_s *consts = &crc32_consts;
+#if defined(__x86_64__) && defined(__WIN64__)
+ char win64tmp[2 * 16];
+
+ /* XMM6-XMM7 need to be restored after use. */
+ asm volatile ("movdqu %%xmm6, 0*16(%0)\n\t"
+ "movdqu %%xmm7, 1*16(%0)\n\t"
+ :
+ : "r" (win64tmp)
+ : "memory");
+#endif
+
+ if (!inlen)
+ return;
+
+ if (inlen >= 16)
+ crc32_reflected_bulk(pcrc, inbuf, inlen, consts);
+ else
+ crc32_reflected_less_than_16(pcrc, inbuf, inlen, consts);
+
+#if defined(__x86_64__) && defined(__WIN64__)
+ /* Restore used registers. */
+ asm volatile("movdqu 0*16(%0), %%xmm6\n\t"
+ "movdqu 1*16(%0), %%xmm7\n\t"
+ :
+ : "r" (win64tmp)
+ : "memory");
+#endif
+}
+
+#endif
diff --git a/extra/mariabackup/crc/crc-intel-pclmul.h b/extra/mariabackup/crc/crc-intel-pclmul.h
new file mode 100644
index 00000000000..120058165a0
--- /dev/null
+++ b/extra/mariabackup/crc/crc-intel-pclmul.h
@@ -0,0 +1,25 @@
+/******************************************************
+Copyright (c) 2017 Percona LLC and/or its affiliates.
+
+CRC32 using Intel's PCLMUL instruction.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+#include <stdint.h>
+#include <stddef.h>
+
+void
+crc32_intel_pclmul(uint32_t *pcrc, const uint8_t *inbuf, size_t inlen);
diff --git a/extra/mariabackup/crc/crc_glue.c b/extra/mariabackup/crc/crc_glue.c
new file mode 100644
index 00000000000..f42910d8344
--- /dev/null
+++ b/extra/mariabackup/crc/crc_glue.c
@@ -0,0 +1,72 @@
+/******************************************************
+Copyright (c) 2017 Percona LLC and/or its affiliates.
+
+Zlib compatible CRC-32 implementation.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+#include <zlib.h>
+#include <stdint.h>
+#include <string.h>
+#include "crc_glue.h"
+#include "crc-intel-pclmul.h"
+
+#if __GNUC__ >= 4 && defined(__x86_64__)
+static int pclmul_enabled = 0;
+#endif
+
+#if defined(__GNUC__) && defined(__x86_64__)
+static
+uint32_t
+cpuid(uint32_t* ecx, uint32_t* edx)
+{
+ uint32_t level;
+
+ asm("cpuid" : "=a" (level) : "a" (0) : "ebx", "ecx", "edx");
+
+ if (level < 1) {
+ return level;
+ }
+
+ asm("cpuid" : "=c" (*ecx), "=d" (*edx)
+ : "a" (1)
+ : "ebx");
+
+ return level;
+}
+#endif
+
+void crc_init() {
+#if defined(__GNUC__) && defined(__x86_64__)
+ uint32_t ecx, edx;
+
+ if (cpuid(&ecx, &edx) > 0) {
+ pclmul_enabled = ((ecx >> 19) & 1) && ((ecx >> 1) & 1);
+ }
+#endif
+}
+
+unsigned long crc32_iso3309(unsigned long crc, const unsigned char *buf, unsigned int len)
+{
+#if __GNUC__ >= 4 && defined(__x86_64__) && defined(HAVE_CLMUL_INSTRUCTION)
+ if (pclmul_enabled) {
+ uint32_t crc_accum = crc ^ 0xffffffffL;
+ crc32_intel_pclmul(&crc_accum, buf, len);
+ return crc_accum ^ 0xffffffffL;
+ }
+#endif
+ return crc32(crc, buf, len);
+}
diff --git a/extra/mariabackup/crc/crc_glue.h b/extra/mariabackup/crc/crc_glue.h
new file mode 100644
index 00000000000..e287fa4a7aa
--- /dev/null
+++ b/extra/mariabackup/crc/crc_glue.h
@@ -0,0 +1,31 @@
+/******************************************************
+Copyright (c) 2017 Percona LLC and/or its affiliates.
+
+Zlib compatible CRC-32 implementation.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void crc_init();
+unsigned long crc32_iso3309(unsigned long crc, const unsigned char *buf, unsigned int len);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/extra/mariabackup/datasink.c b/extra/mariabackup/datasink.c
index 199eb77ad55..460e0e8ca19 100644
--- a/extra/mariabackup/datasink.c
+++ b/extra/mariabackup/datasink.c
@@ -27,7 +27,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#include "ds_local.h"
#include "ds_stdout.h"
#include "ds_tmpfile.h"
-#include "ds_encrypt.h"
#include "ds_buffer.h"
/************************************************************************
@@ -60,6 +59,7 @@ ds_create(const char *root, ds_type_t type)
ds = &datasink_compress;
break;
case DS_TYPE_ENCRYPT:
+ case DS_TYPE_DECRYPT:
msg("Error : mariabackup does not support encrypted backups.");
exit(EXIT_FAILURE);
break;
diff --git a/extra/mariabackup/datasink.h b/extra/mariabackup/datasink.h
index e378a806441..8bf1321aad1 100644
--- a/extra/mariabackup/datasink.h
+++ b/extra/mariabackup/datasink.h
@@ -61,6 +61,7 @@ typedef enum {
DS_TYPE_XBSTREAM,
DS_TYPE_COMPRESS,
DS_TYPE_ENCRYPT,
+ DS_TYPE_DECRYPT,
DS_TYPE_TMPFILE,
DS_TYPE_BUFFER
} ds_type_t;
diff --git a/extra/mariabackup/ds_decrypt.c b/extra/mariabackup/ds_decrypt.c
new file mode 100644
index 00000000000..e897ca101e5
--- /dev/null
+++ b/extra/mariabackup/ds_decrypt.c
@@ -0,0 +1,665 @@
+/******************************************************
+Copyright (c) 2017 Percona LLC and/or its affiliates.
+
+Encryption datasink implementation for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+
+#include <my_base.h>
+#include "common.h"
+#include "datasink.h"
+#include "xbcrypt.h"
+#include "xbcrypt_common.h"
+#include "crc_glue.h"
+
+typedef struct {
+ pthread_t id;
+ uint num;
+ pthread_mutex_t ctrl_mutex;
+ pthread_cond_t ctrl_cond;
+ pthread_mutex_t data_mutex;
+ pthread_cond_t data_cond;
+ my_bool started;
+ my_bool data_avail;
+ my_bool cancelled;
+ my_bool failed;
+ const uchar *from;
+ size_t from_len;
+ uchar *to;
+ size_t to_len;
+ size_t to_size;
+ const uchar *iv;
+ size_t iv_len;
+ unsigned long long offset;
+ my_bool hash_appended;
+ gcry_cipher_hd_t cipher_handle;
+ xb_rcrypt_result_t parse_result;
+} crypt_thread_ctxt_t;
+
+typedef struct {
+ crypt_thread_ctxt_t *threads;
+ uint nthreads;
+ int encrypt_algo;
+ size_t chunk_size;
+ char *encrypt_key;
+ char *encrypt_key_file;
+} ds_decrypt_ctxt_t;
+
+typedef struct {
+ ds_decrypt_ctxt_t *crypt_ctxt;
+ size_t bytes_processed;
+ ds_file_t *dest_file;
+ uchar *buf;
+ size_t buf_len;
+ size_t buf_size;
+} ds_decrypt_file_t;
+
+int ds_decrypt_encrypt_threads = 1;
+
+static ds_ctxt_t *decrypt_init(const char *root);
+static ds_file_t *decrypt_open(ds_ctxt_t *ctxt, const char *path,
+ MY_STAT *mystat);
+static int decrypt_write(ds_file_t *file, const void *buf, size_t len);
+static int decrypt_close(ds_file_t *file);
+static void decrypt_deinit(ds_ctxt_t *ctxt);
+
+datasink_t datasink_decrypt = {
+ &decrypt_init,
+ &decrypt_open,
+ &decrypt_write,
+ &decrypt_close,
+ &decrypt_deinit
+};
+
+static crypt_thread_ctxt_t *create_worker_threads(uint n);
+static void destroy_worker_threads(crypt_thread_ctxt_t *threads, uint n);
+static void *decrypt_worker_thread_func(void *arg);
+
+static
+ds_ctxt_t *
+decrypt_init(const char *root)
+{
+ ds_ctxt_t *ctxt;
+ ds_decrypt_ctxt_t *decrypt_ctxt;
+ crypt_thread_ctxt_t *threads;
+
+ if (xb_crypt_init(NULL)) {
+ return NULL;
+ }
+
+ /* Create and initialize the worker threads */
+ threads = create_worker_threads(ds_decrypt_encrypt_threads);
+ if (threads == NULL) {
+ msg("decrypt: failed to create worker threads.\n");
+ return NULL;
+ }
+
+ ctxt = (ds_ctxt_t *) my_malloc(sizeof(ds_ctxt_t) +
+ sizeof(ds_decrypt_ctxt_t),
+ MYF(MY_FAE));
+
+ decrypt_ctxt = (ds_decrypt_ctxt_t *) (ctxt + 1);
+ decrypt_ctxt->threads = threads;
+ decrypt_ctxt->nthreads = ds_decrypt_encrypt_threads;
+
+ ctxt->ptr = decrypt_ctxt;
+ ctxt->root = my_strdup(root, MYF(MY_FAE));
+
+ return ctxt;
+}
+
+static
+ds_file_t *
+decrypt_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat)
+{
+ ds_ctxt_t *dest_ctxt;
+
+ ds_decrypt_ctxt_t *crypt_ctxt;
+ ds_decrypt_file_t *crypt_file;
+
+ char new_name[FN_REFLEN];
+ ds_file_t *file;
+
+ xb_ad(ctxt->pipe_ctxt != NULL);
+ dest_ctxt = ctxt->pipe_ctxt;
+
+ crypt_ctxt = (ds_decrypt_ctxt_t *) ctxt->ptr;
+
+
+ file = (ds_file_t *) my_malloc(sizeof(ds_file_t) +
+ sizeof(ds_decrypt_file_t),
+ MYF(MY_FAE|MY_ZEROFILL));
+
+ crypt_file = (ds_decrypt_file_t *) (file + 1);
+
+ /* Remove the .xbcrypt extension from the filename */
+ strncpy(new_name, path, FN_REFLEN);
+ new_name[strlen(new_name) - 8] = 0;
+ crypt_file->dest_file = ds_open(dest_ctxt, new_name, mystat);
+ if (crypt_file->dest_file == NULL) {
+ msg("decrypt: ds_open(\"%s\") failed.\n", new_name);
+ goto err;
+ }
+
+ crypt_file->crypt_ctxt = crypt_ctxt;
+ crypt_file->buf = NULL;
+ crypt_file->buf_size = 0;
+ crypt_file->buf_len = 0;
+
+ file->ptr = crypt_file;
+ file->path = crypt_file->dest_file->path;
+
+ return file;
+
+err:
+ if (crypt_file->dest_file) {
+ ds_close(crypt_file->dest_file);
+ }
+ my_free(file);
+ return NULL;
+}
+
+#define CHECK_BUF_SIZE(ptr, size, buf, len) \
+ if (ptr + size - buf > (ssize_t) len) { \
+ result = XB_CRYPT_READ_INCOMPLETE; \
+ goto exit; \
+ }
+
+static
+xb_rcrypt_result_t
+parse_xbcrypt_chunk(crypt_thread_ctxt_t *thd, const uchar *buf, size_t len,
+ size_t *bytes_processed)
+{
+ const uchar *ptr;
+ uint version;
+ ulong checksum, checksum_exp;
+ ulonglong tmp;
+ xb_rcrypt_result_t result = XB_CRYPT_READ_CHUNK;
+
+ *bytes_processed = 0;
+ ptr = buf;
+
+ CHECK_BUF_SIZE(ptr, XB_CRYPT_CHUNK_MAGIC_SIZE, buf, len);
+ if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC3,
+ XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) {
+ version = 3;
+ } else if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC2,
+ XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) {
+ version = 2;
+ } else if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC1,
+ XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) {
+ version = 1;
+ } else {
+ msg("%s:%s: wrong chunk magic at offset 0x%llx.\n",
+ my_progname, __FUNCTION__, thd->offset);
+ result = XB_CRYPT_READ_ERROR;
+ goto exit;
+ }
+
+ ptr += XB_CRYPT_CHUNK_MAGIC_SIZE;
+ thd->offset += XB_CRYPT_CHUNK_MAGIC_SIZE;
+
+ CHECK_BUF_SIZE(ptr, 8, buf, len);
+ tmp = uint8korr(ptr); /* reserved */
+ ptr += 8;
+ thd->offset += 8;
+
+ CHECK_BUF_SIZE(ptr, 8, buf, len);
+ tmp = uint8korr(ptr); /* original size */
+ ptr += 8;
+ if (tmp > INT_MAX) {
+ msg("%s:%s: invalid original size at offset 0x%llx.\n",
+ my_progname, __FUNCTION__, thd->offset);
+ result = XB_CRYPT_READ_ERROR;
+ goto exit;
+ }
+ thd->offset += 8;
+ thd->to_len = (size_t)tmp;
+
+ if (thd->to_size < thd->to_len + XB_CRYPT_HASH_LEN) {
+ thd->to = (uchar *) my_realloc(
+ thd->to,
+ thd->to_len + XB_CRYPT_HASH_LEN,
+ MYF(MY_FAE | MY_ALLOW_ZERO_PTR));
+ thd->to_size = thd->to_len;
+ }
+
+ CHECK_BUF_SIZE(ptr, 8, buf, len);
+ tmp = uint8korr(ptr); /* encrypted size */
+ ptr += 8;
+ if (tmp > INT_MAX) {
+ msg("%s:%s: invalid encrypted size at offset 0x%llx.\n",
+ my_progname, __FUNCTION__, thd->offset);
+ result = XB_CRYPT_READ_ERROR;
+ goto exit;
+ }
+ thd->offset += 8;
+ thd->from_len = (size_t)tmp;
+
+ xb_a(thd->from_len <= thd->to_len + XB_CRYPT_HASH_LEN);
+
+ CHECK_BUF_SIZE(ptr, 4, buf, len);
+ checksum_exp = uint4korr(ptr); /* checksum */
+ ptr += 4;
+ thd->offset += 4;
+
+ /* iv size */
+ if (version == 1) {
+ thd->iv_len = 0;
+ thd->iv = NULL;
+ } else {
+ CHECK_BUF_SIZE(ptr, 8, buf, len);
+
+ tmp = uint8korr(ptr);
+ if (tmp > INT_MAX) {
+ msg("%s:%s: invalid iv size at offset 0x%llx.\n",
+ my_progname, __FUNCTION__, thd->offset);
+ result = XB_CRYPT_READ_ERROR;
+ goto exit;
+ }
+ ptr += 8;
+ thd->offset += 8;
+ thd->iv_len = (size_t)tmp;
+ }
+
+ if (thd->iv_len > 0) {
+ CHECK_BUF_SIZE(ptr, thd->iv_len, buf, len);
+ thd->iv = ptr;
+ ptr += thd->iv_len;
+ }
+
+ /* for version euqals 2 we need to read in the iv data but do not init
+ CTR with it */
+ if (version == 2) {
+ thd->iv_len = 0;
+ thd->iv = 0;
+ }
+
+ if (thd->from_len > 0) {
+ CHECK_BUF_SIZE(ptr, thd->from_len, buf, len);
+ thd->from = ptr;
+ ptr += thd->from_len;
+ }
+
+ xb_ad(thd->from_len <= thd->to_len);
+
+ checksum = crc32_iso3309(0, thd->from, thd->from_len);
+ if (checksum != checksum_exp) {
+ msg("%s:%s invalid checksum at offset 0x%llx, "
+ "expected 0x%lx, actual 0x%lx.\n", my_progname,
+ __FUNCTION__, thd->offset, checksum_exp, checksum);
+ result = XB_CRYPT_READ_ERROR;
+ goto exit;
+ }
+
+ thd->offset += thd->from_len;
+
+ thd->hash_appended = version > 2;
+
+exit:
+
+ *bytes_processed = (size_t) (ptr - buf);
+
+ return result;
+}
+
+static
+int
+decrypt_write(ds_file_t *file, const void *buf, size_t len)
+{
+ ds_decrypt_file_t *crypt_file;
+ ds_decrypt_ctxt_t *crypt_ctxt;
+ crypt_thread_ctxt_t *threads;
+ crypt_thread_ctxt_t *thd;
+ uint nthreads;
+ uint i;
+ size_t bytes_processed;
+ xb_rcrypt_result_t parse_result = XB_CRYPT_READ_CHUNK;
+ my_bool err = FALSE;
+
+ crypt_file = (ds_decrypt_file_t *) file->ptr;
+ crypt_ctxt = crypt_file->crypt_ctxt;
+
+ threads = crypt_ctxt->threads;
+ nthreads = crypt_ctxt->nthreads;
+
+ if (crypt_file->buf_len > 0) {
+ thd = threads;
+
+ pthread_mutex_lock(&thd->ctrl_mutex);
+
+ do {
+ if (parse_result == XB_CRYPT_READ_INCOMPLETE) {
+ crypt_file->buf_size = crypt_file->buf_size * 2;
+ crypt_file->buf = (uchar *) my_realloc(
+ crypt_file->buf,
+ crypt_file->buf_size,
+ MYF(MY_FAE|MY_ALLOW_ZERO_PTR));
+ }
+
+ memcpy(crypt_file->buf + crypt_file->buf_len,
+ buf, MY_MIN(crypt_file->buf_size -
+ crypt_file->buf_len, len));
+
+ parse_result = parse_xbcrypt_chunk(
+ thd, crypt_file->buf,
+ crypt_file->buf_size, &bytes_processed);
+
+ if (parse_result == XB_CRYPT_READ_ERROR) {
+ pthread_mutex_unlock(&thd->ctrl_mutex);
+ return 1;
+ }
+
+ } while (parse_result == XB_CRYPT_READ_INCOMPLETE &&
+ crypt_file->buf_size < len);
+
+ if (parse_result != XB_CRYPT_READ_CHUNK) {
+ msg("decrypt: incomplete data.\n");
+ pthread_mutex_unlock(&thd->ctrl_mutex);
+ return 1;
+ }
+
+ pthread_mutex_lock(&thd->data_mutex);
+ thd->data_avail = TRUE;
+ pthread_cond_signal(&thd->data_cond);
+ pthread_mutex_unlock(&thd->data_mutex);
+
+ len -= bytes_processed - crypt_file->buf_len;
+ buf += bytes_processed - crypt_file->buf_len;
+
+ /* reap */
+
+ pthread_mutex_lock(&thd->data_mutex);
+ while (thd->data_avail == TRUE) {
+ pthread_cond_wait(&thd->data_cond,
+ &thd->data_mutex);
+ }
+
+ if (thd->failed) {
+ msg("decrypt: failed to decrypt chunk.\n");
+ err = TRUE;
+ }
+
+ xb_a(thd->to_len > 0);
+
+ if (!err &&
+ ds_write(crypt_file->dest_file, thd->to, thd->to_len)) {
+ msg("decrypt: write to destination failed.\n");
+ err = TRUE;
+ }
+
+ crypt_file->bytes_processed += thd->from_len;
+
+ pthread_mutex_unlock(&thd->data_mutex);
+ pthread_mutex_unlock(&thd->ctrl_mutex);
+
+ crypt_file->buf_len = 0;
+
+ if (err) {
+ return 1;
+ }
+ }
+
+ while (parse_result == XB_CRYPT_READ_CHUNK && len > 0) {
+ uint max_thread;
+
+ for (i = 0; i < nthreads; i++) {
+ thd = threads + i;
+
+ pthread_mutex_lock(&thd->ctrl_mutex);
+
+ parse_result = parse_xbcrypt_chunk(
+ thd, buf, len, &bytes_processed);
+
+ if (parse_result == XB_CRYPT_READ_ERROR) {
+ pthread_mutex_unlock(&thd->ctrl_mutex);
+ err = TRUE;
+ break;
+ }
+
+ thd->parse_result = parse_result;
+
+ if (parse_result != XB_CRYPT_READ_CHUNK) {
+ pthread_mutex_unlock(&thd->ctrl_mutex);
+ break;
+ }
+
+ pthread_mutex_lock(&thd->data_mutex);
+ thd->data_avail = TRUE;
+ pthread_cond_signal(&thd->data_cond);
+ pthread_mutex_unlock(&thd->data_mutex);
+
+ len -= bytes_processed;
+ buf += bytes_processed;
+ }
+
+ max_thread = (i < nthreads) ? i : nthreads - 1;
+
+ /* Reap and write decrypted data */
+ for (i = 0; i <= max_thread; i++) {
+ thd = threads + i;
+
+ if (thd->parse_result != XB_CRYPT_READ_CHUNK) {
+ break;
+ }
+
+ pthread_mutex_lock(&thd->data_mutex);
+ while (thd->data_avail == TRUE) {
+ pthread_cond_wait(&thd->data_cond,
+ &thd->data_mutex);
+ }
+
+ if (thd->failed) {
+ msg("decrypt: failed to decrypt chunk.\n");
+ err = TRUE;
+ }
+
+ xb_a(thd->to_len > 0);
+
+ if (!err && ds_write(crypt_file->dest_file, thd->to,
+ thd->to_len)) {
+ msg("decrypt: write to destination failed.\n");
+ err = TRUE;
+ }
+
+ crypt_file->bytes_processed += thd->from_len;
+
+ pthread_mutex_unlock(&thd->data_mutex);
+ pthread_mutex_unlock(&thd->ctrl_mutex);
+ }
+
+ if (err) {
+ return 1;
+ }
+ }
+
+ if (parse_result == XB_CRYPT_READ_INCOMPLETE && len > 0) {
+ crypt_file->buf_len = len;
+ if (crypt_file->buf_size < len) {
+ crypt_file->buf = (uchar *) my_realloc(
+ crypt_file->buf,
+ crypt_file->buf_len,
+ MYF(MY_FAE | MY_ALLOW_ZERO_PTR));
+ crypt_file->buf_size = len;
+ }
+ memcpy(crypt_file->buf, buf, len);
+ }
+
+ return 0;
+}
+
+static
+int
+decrypt_close(ds_file_t *file)
+{
+ ds_decrypt_file_t *crypt_file;
+ ds_file_t *dest_file;
+ int rc = 0;
+
+ crypt_file = (ds_decrypt_file_t *) file->ptr;
+ dest_file = crypt_file->dest_file;
+
+ if (ds_close(dest_file)) {
+ rc = 1;
+ }
+
+ my_free(crypt_file->buf);
+ my_free(file);
+
+ return rc;
+}
+
+static
+void
+decrypt_deinit(ds_ctxt_t *ctxt)
+{
+ ds_decrypt_ctxt_t *crypt_ctxt;
+
+ xb_ad(ctxt->pipe_ctxt != NULL);
+
+ crypt_ctxt = (ds_decrypt_ctxt_t *) ctxt->ptr;
+
+ destroy_worker_threads(crypt_ctxt->threads, crypt_ctxt->nthreads);
+
+ my_free(ctxt->root);
+ my_free(ctxt);
+}
+
+static
+crypt_thread_ctxt_t *
+create_worker_threads(uint n)
+{
+ crypt_thread_ctxt_t *threads;
+ uint i;
+
+ threads = (crypt_thread_ctxt_t *)
+ my_malloc(sizeof(crypt_thread_ctxt_t) * n,
+ MYF(MY_FAE | MY_ZEROFILL));
+
+ for (i = 0; i < n; i++) {
+ crypt_thread_ctxt_t *thd = threads + i;
+
+ thd->num = i + 1;
+
+ /* Initialize the control mutex and condition var */
+ if (pthread_mutex_init(&thd->ctrl_mutex, NULL) ||
+ pthread_cond_init(&thd->ctrl_cond, NULL)) {
+ goto err;
+ }
+
+ /* Initialize and data mutex and condition var */
+ if (pthread_mutex_init(&thd->data_mutex, NULL) ||
+ pthread_cond_init(&thd->data_cond, NULL)) {
+ goto err;
+ }
+
+ xb_crypt_cipher_open(&thd->cipher_handle);
+
+ pthread_mutex_lock(&thd->ctrl_mutex);
+
+ if (pthread_create(&thd->id, NULL, decrypt_worker_thread_func,
+ thd)) {
+ msg("decrypt: pthread_create() failed: "
+ "errno = %d\n", errno);
+ goto err;
+ }
+ }
+
+ /* Wait for the threads to start */
+ for (i = 0; i < n; i++) {
+ crypt_thread_ctxt_t *thd = threads + i;
+
+ while (thd->started == FALSE)
+ pthread_cond_wait(&thd->ctrl_cond, &thd->ctrl_mutex);
+ pthread_mutex_unlock(&thd->ctrl_mutex);
+ }
+
+ return threads;
+
+err:
+ return NULL;
+}
+
+static
+void
+destroy_worker_threads(crypt_thread_ctxt_t *threads, uint n)
+{
+ uint i;
+
+ for (i = 0; i < n; i++) {
+ crypt_thread_ctxt_t *thd = threads + i;
+
+ pthread_mutex_lock(&thd->data_mutex);
+ threads[i].cancelled = TRUE;
+ pthread_cond_signal(&thd->data_cond);
+ pthread_mutex_unlock(&thd->data_mutex);
+
+ pthread_join(thd->id, NULL);
+
+ pthread_cond_destroy(&thd->data_cond);
+ pthread_mutex_destroy(&thd->data_mutex);
+ pthread_cond_destroy(&thd->ctrl_cond);
+ pthread_mutex_destroy(&thd->ctrl_mutex);
+
+ xb_crypt_cipher_close(thd->cipher_handle);
+
+ my_free(thd->to);
+ }
+
+ my_free(threads);
+}
+
+static
+void *
+decrypt_worker_thread_func(void *arg)
+{
+ crypt_thread_ctxt_t *thd = (crypt_thread_ctxt_t *) arg;
+
+ pthread_mutex_lock(&thd->ctrl_mutex);
+
+ pthread_mutex_lock(&thd->data_mutex);
+
+ thd->started = TRUE;
+ pthread_cond_signal(&thd->ctrl_cond);
+
+ pthread_mutex_unlock(&thd->ctrl_mutex);
+
+ while (1) {
+ thd->data_avail = FALSE;
+ pthread_cond_signal(&thd->data_cond);
+
+ while (!thd->data_avail && !thd->cancelled) {
+ pthread_cond_wait(&thd->data_cond, &thd->data_mutex);
+ }
+
+ if (thd->cancelled)
+ break;
+
+ if (xb_crypt_decrypt(thd->cipher_handle, thd->from,
+ thd->from_len, thd->to, &thd->to_len,
+ thd->iv, thd->iv_len,
+ thd->hash_appended)) {
+ thd->failed = TRUE;
+ continue;
+ }
+
+ }
+
+ pthread_mutex_unlock(&thd->data_mutex);
+
+ return NULL;
+}
diff --git a/extra/mariabackup/ds_decrypt.h b/extra/mariabackup/ds_decrypt.h
new file mode 100644
index 00000000000..3bb4de55f54
--- /dev/null
+++ b/extra/mariabackup/ds_decrypt.h
@@ -0,0 +1,30 @@
+/******************************************************
+Copyright (c) 2017 Percona LLC and/or its affiliates.
+
+Encryption interface for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+#ifndef DS_DECRYPT_H
+#define DS_DECRYPT_H
+
+#include "datasink.h"
+
+extern datasink_t datasink_decrypt;
+
+extern int ds_decrypt_encrypt_threads;
+
+#endif
diff --git a/extra/mariabackup/ds_encrypt.c b/extra/mariabackup/ds_encrypt.c
index 19ced416687..576ea207eb1 100644
--- a/extra/mariabackup/ds_encrypt.c
+++ b/extra/mariabackup/ds_encrypt.c
@@ -22,25 +22,11 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#include <my_base.h>
#include "common.h"
#include "datasink.h"
+#include "xbcrypt_common.h"
#ifdef HAVE_GRYPT
-#if GCC_VERSION >= 4002
-/* Workaround to avoid "gcry_ac_* is deprecated" warnings in gcrypt.h */
-# pragma GCC diagnostic ignored "-Wdeprecated-declarations"
-#endif
-
-#include <gcrypt.h>
-
-#if GCC_VERSION >= 4002
-# pragma GCC diagnostic warning "-Wdeprecated-declarations"
-#endif
-
#include "xbcrypt.h"
-#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600)
-GCRY_THREAD_OPTION_PTHREAD_IMPL;
-#endif
-
-#define XB_CRYPT_CHUNK_SIZE ((size_t) (xtrabackup_encrypt_chunk_size))
+#define XB_CRYPT_CHUNK_SIZE ((size_t) (ds_encrypt_encrypt_chunk_size))
typedef struct {
pthread_t id;
@@ -52,10 +38,10 @@ typedef struct {
my_bool started;
my_bool data_avail;
my_bool cancelled;
- const char *from;
+ const uchar *from;
size_t from_len;
- char *to;
- char *iv;
+ uchar *to;
+ uchar *iv;
size_t to_len;
gcry_cipher_hd_t cipher_handle;
} crypt_thread_ctxt_t;
@@ -73,11 +59,8 @@ typedef struct {
} ds_encrypt_file_t;
/* Encryption options */
-extern ulong xtrabackup_encrypt_algo;
-extern char *xtrabackup_encrypt_key;
-extern char *xtrabackup_encrypt_key_file;
-extern uint xtrabackup_encrypt_threads;
-extern ulonglong xtrabackup_encrypt_chunk_size;
+uint ds_encrypt_encrypt_threads;
+ulonglong ds_encrypt_encrypt_chunk_size;
static ds_ctxt_t *encrypt_init(const char *root);
static ds_file_t *encrypt_open(ds_ctxt_t *ctxt, const char *path,
@@ -98,12 +81,7 @@ static crypt_thread_ctxt_t *create_worker_threads(uint n);
static void destroy_worker_threads(crypt_thread_ctxt_t *threads, uint n);
static void *encrypt_worker_thread_func(void *arg);
-static uint encrypt_algos[] = { GCRY_CIPHER_NONE, GCRY_CIPHER_AES128,
- GCRY_CIPHER_AES192, GCRY_CIPHER_AES256 };
-static uint encrypt_algo;
-static const uint encrypt_mode = GCRY_CIPHER_MODE_CTR;
-static uint encrypt_key_len = 0;
-static size_t encrypt_iv_len = 0;
+static uint encrypt_iv_len = 0;
static
ssize_t
@@ -129,87 +107,13 @@ encrypt_init(const char *root)
ds_ctxt_t *ctxt;
ds_encrypt_ctxt_t *encrypt_ctxt;
crypt_thread_ctxt_t *threads;
- gcry_error_t gcry_error;
-
- /* Acording to gcrypt docs (and my testing), setting up the threading
- callbacks must be done first, so, lets give it a shot */
-#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600)
- gcry_error = gcry_control(GCRYCTL_SET_THREAD_CBS, &gcry_threads_pthread);
- if (gcry_error) {
- msg("encrypt: unable to set libgcrypt thread cbs - "
- "%s : %s\n",
- gcry_strsource(gcry_error),
- gcry_strerror(gcry_error));
- return NULL;
- }
-#endif
-
- /* Version check should be the very next call because it
- makes sure that important subsystems are intialized. */
- if (!gcry_control(GCRYCTL_ANY_INITIALIZATION_P)) {
- const char *gcrypt_version;
- gcrypt_version = gcry_check_version(NULL);
- /* No other library has already initialized libgcrypt. */
- if (!gcrypt_version) {
- msg("encrypt: failed to initialize libgcrypt\n");
- return NULL;
- } else {
- msg("encrypt: using gcrypt %s\n", gcrypt_version);
- }
- }
- /* Disable the gcry secure memory, not dealing with this for now */
- gcry_error = gcry_control(GCRYCTL_DISABLE_SECMEM, 0);
- if (gcry_error) {
- msg("encrypt: unable to disable libgcrypt secmem - "
- "%s : %s\n",
- gcry_strsource(gcry_error),
- gcry_strerror(gcry_error));
- return NULL;
- }
-
- /* Finalize gcry initialization. */
- gcry_error = gcry_control(GCRYCTL_INITIALIZATION_FINISHED, 0);
- if (gcry_error) {
- msg("encrypt: unable to finish libgcrypt initialization - "
- "%s : %s\n",
- gcry_strsource(gcry_error),
- gcry_strerror(gcry_error));
- return NULL;
- }
-
- /* Determine the algorithm */
- encrypt_algo = encrypt_algos[xtrabackup_encrypt_algo];
-
- /* Set up the iv length */
- encrypt_iv_len = gcry_cipher_get_algo_blklen(encrypt_algo);
- xb_a(encrypt_iv_len > 0);
-
- /* Now set up the key */
- if (xtrabackup_encrypt_key == NULL &&
- xtrabackup_encrypt_key_file == NULL) {
- msg("encrypt: no encryption key or key file specified.\n");
- return NULL;
- } else if (xtrabackup_encrypt_key && xtrabackup_encrypt_key_file) {
- msg("encrypt: both encryption key and key file specified.\n");
- return NULL;
- } else if (xtrabackup_encrypt_key_file) {
- if (!xb_crypt_read_key_file(xtrabackup_encrypt_key_file,
- (void**)&xtrabackup_encrypt_key,
- &encrypt_key_len)) {
- msg("encrypt: unable to read encryption key file"
- " \"%s\".\n", xtrabackup_encrypt_key_file);
- return NULL;
- }
- } else if (xtrabackup_encrypt_key) {
- encrypt_key_len = strlen(xtrabackup_encrypt_key);
- } else {
- msg("encrypt: no encryption key or key file specified.\n");
+ if (xb_crypt_init(&encrypt_iv_len)) {
return NULL;
}
/* Create and initialize the worker threads */
- threads = create_worker_threads(xtrabackup_encrypt_threads);
+ threads = create_worker_threads(ds_encrypt_encrypt_threads);
if (threads == NULL) {
msg("encrypt: failed to create worker threads.\n");
return NULL;
@@ -221,7 +125,7 @@ encrypt_init(const char *root)
encrypt_ctxt = (ds_encrypt_ctxt_t *) (ctxt + 1);
encrypt_ctxt->threads = threads;
- encrypt_ctxt->nthreads = xtrabackup_encrypt_threads;
+ encrypt_ctxt->nthreads = ds_encrypt_encrypt_threads;
ctxt->ptr = encrypt_ctxt;
ctxt->root = my_strdup(root, MYF(MY_FAE));
@@ -294,7 +198,7 @@ encrypt_write(ds_file_t *file, const void *buf, size_t len)
crypt_thread_ctxt_t *thd;
uint nthreads;
uint i;
- const char *ptr;
+ const uchar *ptr;
crypt_file = (ds_encrypt_file_t *) file->ptr;
crypt_ctxt = crypt_file->crypt_ctxt;
@@ -302,7 +206,7 @@ encrypt_write(ds_file_t *file, const void *buf, size_t len)
threads = crypt_ctxt->threads;
nthreads = crypt_ctxt->nthreads;
- ptr = (const char *) buf;
+ ptr = (const uchar *) buf;
while (len > 0) {
uint max_thread;
@@ -403,10 +307,6 @@ encrypt_deinit(ds_ctxt_t *ctxt)
my_free(ctxt->root);
my_free(ctxt);
- if (xtrabackup_encrypt_key)
- my_free(xtrabackup_encrypt_key);
- if (xtrabackup_encrypt_key_file)
- my_free(xtrabackup_encrypt_key_file);
}
static
@@ -427,11 +327,10 @@ create_worker_threads(uint n)
thd->cancelled = FALSE;
thd->data_avail = FALSE;
- thd->to = (char *) my_malloc(XB_CRYPT_CHUNK_SIZE +
- XB_CRYPT_HASH_LEN, MYF(MY_FAE));
+ thd->to = (uchar *) my_malloc(XB_CRYPT_CHUNK_SIZE +
+ XB_CRYPT_HASH_LEN, MYF(MY_FAE));
- thd->iv = (char *) my_malloc(encrypt_iv_len,
- MYF(MY_FAE));
+ thd->iv = (uchar *) my_malloc(encrypt_iv_len, MYF(MY_FAE));
/* Initialize the control mutex and condition var */
if (pthread_mutex_init(&thd->ctrl_mutex, NULL) ||
@@ -445,32 +344,8 @@ create_worker_threads(uint n)
goto err;
}
- if (encrypt_algo != GCRY_CIPHER_NONE) {
- gcry_error_t gcry_error;
-
- gcry_error = gcry_cipher_open(&thd->cipher_handle,
- encrypt_algo,
- encrypt_mode, 0);
- if (gcry_error) {
- msg("encrypt: unable to open libgcrypt"
- " cipher - %s : %s\n",
- gcry_strsource(gcry_error),
- gcry_strerror(gcry_error));
- gcry_cipher_close(thd->cipher_handle);
- goto err;
- }
-
- gcry_error = gcry_cipher_setkey(thd->cipher_handle,
- xtrabackup_encrypt_key,
- encrypt_key_len);
- if (gcry_error) {
- msg("encrypt: unable to set libgcrypt"
- " cipher key - %s : %s\n",
- gcry_strsource(gcry_error),
- gcry_strerror(gcry_error));
- gcry_cipher_close(thd->cipher_handle);
- goto err;
- }
+ if (xb_crypt_cipher_open(&thd->cipher_handle)) {
+ goto err;
}
pthread_mutex_lock(&thd->ctrl_mutex);
@@ -519,8 +394,7 @@ destroy_worker_threads(crypt_thread_ctxt_t *threads, uint n)
pthread_cond_destroy(&thd->ctrl_cond);
pthread_mutex_destroy(&thd->ctrl_mutex);
- if (encrypt_algo != GCRY_CIPHER_NONE)
- gcry_cipher_close(thd->cipher_handle);
+ xb_crypt_cipher_close(thd->cipher_handle);
my_free(thd->to);
my_free(thd->iv);
@@ -555,60 +429,14 @@ encrypt_worker_thread_func(void *arg)
if (thd->cancelled)
break;
- /* ensure that XB_CRYPT_HASH_LEN is the correct length
- of XB_CRYPT_HASH hashing algorithm output */
- assert(gcry_md_get_algo_dlen(XB_CRYPT_HASH) ==
- XB_CRYPT_HASH_LEN);
-
- memcpy(thd->to, thd->from, thd->from_len);
- gcry_md_hash_buffer(XB_CRYPT_HASH, thd->to + thd->from_len,
- thd->from, thd->from_len);
thd->to_len = thd->from_len;
- if (encrypt_algo != GCRY_CIPHER_NONE) {
- gcry_error_t gcry_error;
-
- gcry_error = gcry_cipher_reset(thd->cipher_handle);
- if (gcry_error) {
- msg("encrypt: unable to reset cipher - "
- "%s : %s\n",
- gcry_strsource(gcry_error),
- gcry_strerror(gcry_error));
- thd->to_len = 0;
- continue;
- }
-
- xb_crypt_create_iv(thd->iv, encrypt_iv_len);
- gcry_error = gcry_cipher_setctr(thd->cipher_handle,
- thd->iv,
- encrypt_iv_len);
- if (gcry_error) {
- msg("encrypt: unable to set cipher ctr - "
- "%s : %s\n",
- gcry_strsource(gcry_error),
- gcry_strerror(gcry_error));
- thd->to_len = 0;
- continue;
- }
-
- gcry_error = gcry_cipher_encrypt(thd->cipher_handle,
- thd->to,
- thd->to_len +
- XB_CRYPT_HASH_LEN,
- thd->to,
- thd->from_len +
- XB_CRYPT_HASH_LEN);
- if (gcry_error) {
- msg("encrypt: unable to encrypt buffer - "
- "%s : %s\n", gcry_strsource(gcry_error),
- gcry_strerror(gcry_error));
- thd->to_len = 0;
- }
- } else {
- memcpy(thd->to, thd->from,
- thd->from_len + XB_CRYPT_HASH_LEN);
+ if (xb_crypt_encrypt(thd->cipher_handle, thd->from,
+ thd->from_len, thd->to, &thd->to_len,
+ thd->iv)) {
+ thd->to_len = 0;
+ continue;
}
- thd->to_len += XB_CRYPT_HASH_LEN;
}
pthread_mutex_unlock(&thd->data_mutex);
diff --git a/extra/mariabackup/ds_encrypt.h b/extra/mariabackup/ds_encrypt.h
index 3e9e40ad354..c4d8d7f8427 100644
--- a/extra/mariabackup/ds_encrypt.h
+++ b/extra/mariabackup/ds_encrypt.h
@@ -25,4 +25,9 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#ifdef HAVE_GCRYPT
extern datasink_t datasink_encrypt;
#endif
+/* Encryption options */
+extern uint ds_encrypt_encrypt_threads;
+extern ulonglong ds_encrypt_encrypt_chunk_size;
+
+
#endif
diff --git a/extra/mariabackup/xbcrypt.c b/extra/mariabackup/xbcrypt.c
index 255da875de4..3da70e171f7 100644
--- a/extra/mariabackup/xbcrypt.c
+++ b/extra/mariabackup/xbcrypt.c
@@ -22,7 +22,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#include <my_getopt.h>
#include "common.h"
#include "xbcrypt.h"
-#include <gcrypt.h>
+#include "xbcrypt_common.h"
+#include "crc_glue.h"
#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600)
GCRY_THREAD_OPTION_PTHREAD_IMPL;
@@ -138,6 +139,8 @@ main(int argc, char **argv)
MY_INIT(argv[0]);
+ crc_init();
+
if (get_options(&argc, &argv)) {
goto err;
}
@@ -402,8 +405,8 @@ mode_decrypt(File filein, File fileout)
/* ensure that XB_CRYPT_HASH_LEN is the correct length
of XB_CRYPT_HASH hashing algorithm output */
- assert(gcry_md_get_algo_dlen(XB_CRYPT_HASH) ==
- XB_CRYPT_HASH_LEN);
+ xb_a(gcry_md_get_algo_dlen(XB_CRYPT_HASH) ==
+ XB_CRYPT_HASH_LEN);
gcry_md_hash_buffer(XB_CRYPT_HASH, hash, decryptbuf,
originalsize);
if (memcmp(hash, (char *) decryptbuf + originalsize,
@@ -529,8 +532,7 @@ mode_encrypt(File filein, File fileout)
/* ensure that XB_CRYPT_HASH_LEN is the correct length
of XB_CRYPT_HASH hashing algorithm output */
- assert(XB_CRYPT_HASH_LEN ==
- gcry_md_get_algo_dlen(XB_CRYPT_HASH));
+ xb_a(XB_CRYPT_HASH_LEN == gcry_md_get_algo_dlen(XB_CRYPT_HASH));
gcry_md_hash_buffer(XB_CRYPT_HASH, chunkbuf + bytesread,
chunkbuf, bytesread);
diff --git a/extra/mariabackup/xbcrypt.h b/extra/mariabackup/xbcrypt.h
index cdabf56a21a..0e832266847 100644
--- a/extra/mariabackup/xbcrypt.h
+++ b/extra/mariabackup/xbcrypt.h
@@ -65,6 +65,7 @@ xb_rcrypt_t *xb_crypt_read_open(void *userdata,
typedef enum {
XB_CRYPT_READ_CHUNK,
+ XB_CRYPT_READ_INCOMPLETE,
XB_CRYPT_READ_EOF,
XB_CRYPT_READ_ERROR
} xb_rcrypt_result_t;
@@ -75,10 +76,4 @@ xb_rcrypt_result_t xb_crypt_read_chunk(xb_rcrypt_t *crypt, void **buf,
int xb_crypt_read_close(xb_rcrypt_t *crypt);
-/******************************************************************************
-Utility interface */
-my_bool xb_crypt_read_key_file(const char *filename,
- void** key, uint *keylength);
-
-void xb_crypt_create_iv(void* ivbuf, size_t ivlen);
#endif
diff --git a/extra/mariabackup/xbcrypt_common.c b/extra/mariabackup/xbcrypt_common.c
index 52fa2ce3589..0cdb54dc66d 100644
--- a/extra/mariabackup/xbcrypt_common.c
+++ b/extra/mariabackup/xbcrypt_common.c
@@ -1,5 +1,5 @@
/******************************************************
-Copyright (c) 2013 Percona LLC and/or its affiliates.
+Copyright (c) 2013, 2017 Percona LLC and/or its affiliates.
Encryption configuration file interface for XtraBackup.
@@ -21,19 +21,27 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#include <my_base.h>
#include "common.h"
#include "xbcrypt.h"
+#include "xbcrypt_common.h"
-#if GCC_VERSION >= 4002
-/* Workaround to avoid "gcry_ac_* is deprecated" warnings in gcrypt.h */
-# pragma GCC diagnostic ignored "-Wdeprecated-declarations"
-#endif
+/* Encryption options */
+char *ds_encrypt_key = NULL;
+char *ds_encrypt_key_file = NULL;
+ulong ds_encrypt_algo;
+
+static uint encrypt_key_len;
+static uint encrypt_iv_len;
-#ifdef HAVE_GRYPT
-#include <gcrypt.h>
+static const uint encrypt_mode = GCRY_CIPHER_MODE_CTR;
-#if GCC_VERSION >= 4002
-# pragma GCC diagnostic warning "-Wdeprecated-declarations"
+static uint encrypt_algos[] = { GCRY_CIPHER_NONE, GCRY_CIPHER_AES128,
+ GCRY_CIPHER_AES192, GCRY_CIPHER_AES256 };
+static uint encrypt_algo;
+
+#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600)
+GCRY_THREAD_OPTION_PTHREAD_IMPL;
#endif
+
my_bool
xb_crypt_read_key_file(const char *filename, void** key, uint *keylength)
{
@@ -59,4 +67,262 @@ xb_crypt_create_iv(void* ivbuf, size_t ivlen)
{
gcry_create_nonce(ivbuf, ivlen);
}
+
+gcry_error_t
+xb_crypt_init(uint *iv_len)
+{
+ gcry_error_t gcry_error;
+
+ /* Acording to gcrypt docs (and my testing), setting up the threading
+ callbacks must be done first, so, lets give it a shot */
+#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600)
+ gcry_error = gcry_control(GCRYCTL_SET_THREAD_CBS, &gcry_threads_pthread);
+ if (gcry_error) {
+ msg("encryption: unable to set libgcrypt thread cbs - "
+ "%s : %s\n",
+ gcry_strsource(gcry_error),
+ gcry_strerror(gcry_error));
+ return gcry_error;
+ }
+#endif
+
+ /* Version check should be the very next call because it
+ makes sure that important subsystems are intialized. */
+ if (!gcry_control(GCRYCTL_ANY_INITIALIZATION_P)) {
+ const char *gcrypt_version;
+ gcrypt_version = gcry_check_version(NULL);
+ /* No other library has already initialized libgcrypt. */
+ if (!gcrypt_version) {
+ msg("encryption: failed to initialize libgcrypt\n");
+ return 1;
+ } else {
+ msg("encryption: using gcrypt %s\n", gcrypt_version);
+ }
+ }
+
+ /* Disable the gcry secure memory, not dealing with this for now */
+ gcry_error = gcry_control(GCRYCTL_DISABLE_SECMEM, 0);
+ if (gcry_error) {
+ msg("encryption: unable to disable libgcrypt secmem - "
+ "%s : %s\n",
+ gcry_strsource(gcry_error),
+ gcry_strerror(gcry_error));
+ return gcry_error;
+ }
+
+ /* Finalize gcry initialization. */
+ gcry_error = gcry_control(GCRYCTL_INITIALIZATION_FINISHED, 0);
+ if (gcry_error) {
+ msg("encryption: unable to finish libgcrypt initialization - "
+ "%s : %s\n",
+ gcry_strsource(gcry_error),
+ gcry_strerror(gcry_error));
+ return gcry_error;
+ }
+
+ /* Determine the algorithm */
+ encrypt_algo = encrypt_algos[ds_encrypt_algo];
+
+ /* Set up the iv length */
+ encrypt_iv_len = gcry_cipher_get_algo_blklen(encrypt_algo);
+ xb_a(encrypt_iv_len > 0);
+ if (iv_len != NULL) {
+ *iv_len = encrypt_iv_len;
+ }
+
+ /* Now set up the key */
+ if (ds_encrypt_key == NULL &&
+ ds_encrypt_key_file == NULL) {
+ msg("encryption: no encryption key or key file specified.\n");
+ return gcry_error;
+ } else if (ds_encrypt_key && ds_encrypt_key_file) {
+ msg("encryption: both encryption key and key file specified.\n");
+ return gcry_error;
+ } else if (ds_encrypt_key_file) {
+ if (!xb_crypt_read_key_file(ds_encrypt_key_file,
+ (void**)&ds_encrypt_key,
+ &encrypt_key_len)) {
+ msg("encryption: unable to read encryption key file"
+ " \"%s\".\n", ds_encrypt_key_file);
+ return gcry_error;
+ }
+ } else if (ds_encrypt_key) {
+ encrypt_key_len = strlen(ds_encrypt_key);
+ } else {
+ msg("encryption: no encryption key or key file specified.\n");
+ return gcry_error;
+ }
+
+ return 0;
+}
+
+gcry_error_t
+xb_crypt_cipher_open(gcry_cipher_hd_t *cipher_handle)
+{
+ if (encrypt_algo != GCRY_CIPHER_NONE) {
+ gcry_error_t gcry_error;
+
+ gcry_error = gcry_cipher_open(cipher_handle,
+ encrypt_algo,
+ encrypt_mode, 0);
+ if (gcry_error) {
+ msg("encryption: unable to open libgcrypt"
+ " cipher - %s : %s\n",
+ gcry_strsource(gcry_error),
+ gcry_strerror(gcry_error));
+ gcry_cipher_close(*cipher_handle);
+ return gcry_error;
+ }
+
+ gcry_error = gcry_cipher_setkey(*cipher_handle,
+ ds_encrypt_key,
+ encrypt_key_len);
+ if (gcry_error) {
+ msg("encryption: unable to set libgcrypt"
+ " cipher key - %s : %s\n",
+ gcry_strsource(gcry_error),
+ gcry_strerror(gcry_error));
+ gcry_cipher_close(*cipher_handle);
+ return gcry_error;
+ }
+ return gcry_error;
+ }
+ return 0;
+}
+
+void
+xb_crypt_cipher_close(gcry_cipher_hd_t cipher_handle)
+{
+ if (encrypt_algo != GCRY_CIPHER_NONE)
+ gcry_cipher_close(cipher_handle);
+}
+
+gcry_error_t
+xb_crypt_decrypt(gcry_cipher_hd_t cipher_handle, const uchar *from,
+ size_t from_len, uchar *to, size_t *to_len,
+ const uchar *iv, size_t iv_len, my_bool hash_appended)
+{
+ *to_len = from_len;
+
+ if (encrypt_algo != GCRY_CIPHER_NONE) {
+
+ gcry_error_t gcry_error;
+
+ gcry_error = gcry_cipher_reset(cipher_handle);
+ if (gcry_error) {
+ msg("%s:encryption: unable to reset libgcrypt"
+ " cipher - %s : %s\n", my_progname,
+ gcry_strsource(gcry_error),
+ gcry_strerror(gcry_error));
+ return gcry_error;
+ }
+
+ if (iv_len > 0) {
+ gcry_error = gcry_cipher_setctr(cipher_handle,
+ iv, iv_len);
+ }
+ if (gcry_error) {
+ msg("%s:encryption: unable to set cipher iv - "
+ "%s : %s\n", my_progname,
+ gcry_strsource(gcry_error),
+ gcry_strerror(gcry_error));
+ return gcry_error;
+ }
+
+ /* Try to decrypt it */
+ gcry_error = gcry_cipher_decrypt(cipher_handle, to, *to_len,
+ from, from_len);
+ if (gcry_error) {
+ msg("%s:encryption: unable to decrypt chunk - "
+ "%s : %s\n", my_progname,
+ gcry_strsource(gcry_error),
+ gcry_strerror(gcry_error));
+ gcry_cipher_close(cipher_handle);
+ return gcry_error;
+ }
+
+ if (hash_appended) {
+ uchar hash[XB_CRYPT_HASH_LEN];
+
+ *to_len -= XB_CRYPT_HASH_LEN;
+
+ /* ensure that XB_CRYPT_HASH_LEN is the correct length
+ of XB_CRYPT_HASH hashing algorithm output */
+ xb_ad(gcry_md_get_algo_dlen(XB_CRYPT_HASH) ==
+ XB_CRYPT_HASH_LEN);
+ gcry_md_hash_buffer(XB_CRYPT_HASH, hash, to,
+ *to_len);
+ if (memcmp(hash, (char *) to + *to_len,
+ XB_CRYPT_HASH_LEN) != 0) {
+ msg("%s:%s invalid plaintext hash. "
+ "Wrong encrytion key specified?\n",
+ my_progname, __FUNCTION__);
+ return 1;
+ }
+ }
+
+ } else {
+ memcpy(to, from, *to_len);
+ }
+
+ return 0;
+}
+
+gcry_error_t
+xb_crypt_encrypt(gcry_cipher_hd_t cipher_handle, const uchar *from,
+ size_t from_len, uchar *to, size_t *to_len, uchar *iv)
+{
+ gcry_error_t gcry_error;
+
+ /* ensure that XB_CRYPT_HASH_LEN is the correct length
+ of XB_CRYPT_HASH hashing algorithm output */
+ xb_ad(gcry_md_get_algo_dlen(XB_CRYPT_HASH) ==
+ XB_CRYPT_HASH_LEN);
+
+ memcpy(to, from, from_len);
+ gcry_md_hash_buffer(XB_CRYPT_HASH, to + from_len,
+ from, from_len);
+
+ *to_len = from_len;
+
+ if (encrypt_algo != GCRY_CIPHER_NONE) {
+
+ gcry_error = gcry_cipher_reset(cipher_handle);
+ if (gcry_error) {
+ msg("encrypt: unable to reset cipher - "
+ "%s : %s\n",
+ gcry_strsource(gcry_error),
+ gcry_strerror(gcry_error));
+ return gcry_error;
+ }
+
+ xb_crypt_create_iv(iv, encrypt_iv_len);
+ gcry_error = gcry_cipher_setctr(cipher_handle, iv,
+ encrypt_iv_len);
+ if (gcry_error) {
+ msg("encrypt: unable to set cipher ctr - "
+ "%s : %s\n",
+ gcry_strsource(gcry_error),
+ gcry_strerror(gcry_error));
+ return gcry_error;
+ }
+
+ gcry_error = gcry_cipher_encrypt(cipher_handle, to,
+ *to_len + XB_CRYPT_HASH_LEN,
+ to,
+ from_len + XB_CRYPT_HASH_LEN);
+ if (gcry_error) {
+ msg("encrypt: unable to encrypt buffer - "
+ "%s : %s\n", gcry_strsource(gcry_error),
+ gcry_strerror(gcry_error));
+ return gcry_error;
+ }
+ } else {
+ memcpy(to, from, from_len + XB_CRYPT_HASH_LEN);
+ }
+
+ *to_len += XB_CRYPT_HASH_LEN;
+
+ return 0;
+}
#endif \ No newline at end of file
diff --git a/extra/mariabackup/xbcrypt_common.h b/extra/mariabackup/xbcrypt_common.h
new file mode 100644
index 00000000000..85d13c01fc4
--- /dev/null
+++ b/extra/mariabackup/xbcrypt_common.h
@@ -0,0 +1,64 @@
+/******************************************************
+Copyright (c) 2017 Percona LLC and/or its affiliates.
+
+Encryption datasink implementation for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+#include <my_base.h>
+#if HAVE_GCRYPT
+#if GCC_VERSION >= 4002
+/* Workaround to avoid "gcry_ac_* is deprecated" warnings in gcrypt.h */
+# pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+#endif
+
+#include <gcrypt.h>
+
+extern char *ds_encrypt_key;
+extern char *ds_encrypt_key_file;
+extern int ds_encrypt_threads;
+extern ulong ds_encrypt_algo;
+
+/******************************************************************************
+Utility interface */
+my_bool xb_crypt_read_key_file(const char *filename,
+ void** key, uint *keylength);
+
+void xb_crypt_create_iv(void* ivbuf, size_t ivlen);
+
+/* Initialize gcrypt and setup encryption key and IV lengths */
+gcry_error_t
+xb_crypt_init(uint *iv_len);
+
+/* Setup gcrypt cipher */
+gcry_error_t
+xb_crypt_cipher_open(gcry_cipher_hd_t *cipher_handle);
+
+/* Close gcrypt cipher */
+void
+xb_crypt_cipher_close(gcry_cipher_hd_t cipher_handle);
+
+/* Decrypt buffer */
+gcry_error_t
+xb_crypt_decrypt(gcry_cipher_hd_t cipher_handle, const uchar *from,
+ size_t from_len, uchar *to, size_t *to_len, const uchar *iv,
+ size_t iv_len, my_bool hash_appended);
+
+/* Encrypt buffer */
+gcry_error_t
+xb_crypt_encrypt(gcry_cipher_hd_t cipher_handle, const uchar *from,
+ size_t from_len, uchar *to, size_t *to_len, uchar *iv);
+#endif
diff --git a/extra/mariabackup/xbcrypt_read.c b/extra/mariabackup/xbcrypt_read.c
index 6522333f023..41790c7035d 100644
--- a/extra/mariabackup/xbcrypt_read.c
+++ b/extra/mariabackup/xbcrypt_read.c
@@ -19,6 +19,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#include "xbcrypt.h"
+#include "crc_glue.h"
struct xb_rcrypt_struct {
void *userdata;
@@ -212,7 +213,7 @@ xb_crypt_read_chunk(xb_rcrypt_t *crypt, void **buf, size_t *olen, size_t *elen,
}
}
- checksum = crc32(0, crypt->buffer, *elen);
+ checksum = crc32_iso3309(0, crypt->buffer, *elen);
if (checksum != checksum_exp) {
msg("%s:%s invalid checksum at offset 0x%llx, "
"expected 0x%lx, actual 0x%lx.\n", my_progname, __FUNCTION__,
diff --git a/extra/mariabackup/xbcrypt_write.c b/extra/mariabackup/xbcrypt_write.c
index 6cc30852215..91dbfc4eb29 100644
--- a/extra/mariabackup/xbcrypt_write.c
+++ b/extra/mariabackup/xbcrypt_write.c
@@ -19,6 +19,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#include "xbcrypt.h"
+#include "crc_glue.h"
struct xb_wcrypt_struct {
void *userdata;
@@ -73,7 +74,7 @@ int xb_crypt_write_chunk(xb_wcrypt_t *crypt, const void *buf, size_t olen,
int8store(ptr, (ulonglong)elen); /* encrypted (actual) size */
ptr += 8;
- checksum = crc32(0, buf, (uint)elen);
+ checksum = crc32_iso3309(0, buf, elen);
int4store(ptr, checksum); /* checksum */
ptr += 4;
diff --git a/extra/mariabackup/xbstream.c b/extra/mariabackup/xbstream.c
index 9990b00ea4b..2cc47ec7273 100644
--- a/extra/mariabackup/xbstream.c
+++ b/extra/mariabackup/xbstream.c
@@ -22,10 +22,13 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#include <my_base.h>
#include <my_getopt.h>
#include <hash.h>
+#include <my_pthread.h>
#include "common.h"
#include "xbstream.h"
-#include "ds_local.h"
-#include "ds_stdout.h"
+#include "xbcrypt_common.h"
+#include "datasink.h"
+#include "ds_decrypt.h"
+#include "crc_glue.h"
#define XBSTREAM_VERSION "1.0"
#define XBSTREAM_BUFFER_SIZE (10 * 1024 * 1024UL)
@@ -38,6 +41,12 @@ typedef enum {
RUN_MODE_EXTRACT
} run_mode_t;
+const char *xbstream_encrypt_algo_names[] =
+{ "NONE", "AES128", "AES192", "AES256", NullS};
+TYPELIB xbstream_encrypt_algo_typelib=
+{array_elements(xbstream_encrypt_algo_names)-1,"",
+ xbstream_encrypt_algo_names, NULL};
+
/* Need the following definitions to avoid linking with ds_*.o and their link
dependencies */
datasink_t datasink_archive;
@@ -47,9 +56,18 @@ datasink_t datasink_tmpfile;
datasink_t datasink_encrypt;
datasink_t datasink_buffer;
-static run_mode_t opt_mode;
+static run_mode_t opt_mode;
static char * opt_directory = NULL;
static my_bool opt_verbose = 0;
+static int opt_parallel = 1;
+static ulong opt_encrypt_algo;
+static char *opt_encrypt_key_file = NULL;
+static void *opt_encrypt_key = NULL;
+static int opt_encrypt_threads = 1;
+
+enum {
+ OPT_ENCRYPT_THREADS = 256
+};
static struct my_option my_long_options[] =
{
@@ -65,21 +83,46 @@ static struct my_option my_long_options[] =
GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"verbose", 'v', "Print verbose output.", &opt_verbose, &opt_verbose,
0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+ {"parallel", 'p', "Number of worker threads for reading / writing.",
+ &opt_parallel, &opt_parallel, 0, GET_INT, REQUIRED_ARG,
+ 1, 1, INT_MAX, 0, 0, 0},
+ {"decrypt", 'd', "Decrypt files ending with .xbcrypt.",
+ &opt_encrypt_algo, &opt_encrypt_algo, &xbstream_encrypt_algo_typelib,
+ GET_ENUM, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+ {"encrypt-key", 'k', "Encryption key.",
+ &opt_encrypt_key, &opt_encrypt_key, 0,
+ GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+ {"encrypt-key-file", 'f', "File which contains encryption key.",
+ &opt_encrypt_key_file, &opt_encrypt_key_file, 0,
+ GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+ {"encrypt-threads", OPT_ENCRYPT_THREADS,
+ "Number of threads for parallel data encryption. "
+ "The default value is 1.",
+ &opt_encrypt_threads, &opt_encrypt_threads,
+ 0, GET_INT, REQUIRED_ARG, 1, 1, INT_MAX, 0, 0, 0},
{0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
};
typedef struct {
+ HASH *filehash;
+ xb_rstream_t *stream;
+ ds_ctxt_t *ds_ctxt;
+ ds_ctxt_t *ds_decrypt_ctxt;
+ pthread_mutex_t *mutex;
+} extract_ctxt_t;
+
+typedef struct {
char *path;
uint pathlen;
my_off_t offset;
- ds_ctxt_t *ds_ctxt;
ds_file_t *file;
+ pthread_mutex_t mutex;
} file_entry_t;
static int get_options(int *argc, char ***argv);
static int mode_create(int argc, char **argv);
-static int mode_extract(int argc, char **argv);
+static int mode_extract(int n_threads, int argc, char **argv);
static my_bool get_one_option(int optid, const struct my_option *opt,
char *argument);
@@ -88,6 +131,8 @@ main(int argc, char **argv)
{
MY_INIT(argv[0]);
+ crc_init();
+
if (get_options(&argc, &argv)) {
goto err;
}
@@ -104,7 +149,8 @@ main(int argc, char **argv)
if (opt_mode == RUN_MODE_CREATE && mode_create(argc, argv)) {
goto err;
- } else if (opt_mode == RUN_MODE_EXTRACT && mode_extract(argc, argv)) {
+ } else if (opt_mode == RUN_MODE_EXTRACT &&
+ mode_extract(opt_parallel, argc, argv)) {
goto err;
}
@@ -302,9 +348,22 @@ err:
return 1;
}
+/************************************************************************
+Check if string ends with given suffix.
+@return true if string ends with given suffix. */
+static
+my_bool
+ends_with(const char *str, const char *suffix)
+{
+ size_t suffix_len = strlen(suffix);
+ size_t str_len = strlen(str);
+ return(str_len >= suffix_len
+ && strcmp(str + str_len - suffix_len, suffix) == 0);
+}
+
static
file_entry_t *
-file_entry_new(ds_ctxt_t *ds_ctxt, const char *path, uint pathlen)
+file_entry_new(extract_ctxt_t *ctxt, const char *path, uint pathlen)
{
file_entry_t *entry;
ds_file_t *file;
@@ -321,7 +380,11 @@ file_entry_new(ds_ctxt_t *ds_ctxt, const char *path, uint pathlen)
}
entry->pathlen = pathlen;
- file = ds_open(ds_ctxt, path, NULL);
+ if (ctxt->ds_decrypt_ctxt && ends_with(path, ".xbcrypt")) {
+ file = ds_open(ctxt->ds_decrypt_ctxt, path, NULL);
+ } else {
+ file = ds_open(ctxt->ds_ctxt, path, NULL);
+ }
if (file == NULL) {
msg("%s: failed to create file.\n", my_progname);
goto err;
@@ -332,7 +395,8 @@ file_entry_new(ds_ctxt_t *ds_ctxt, const char *path, uint pathlen)
}
entry->file = file;
- entry->ds_ctxt = ds_ctxt;
+
+ pthread_mutex_init(&entry->mutex, NULL);
return entry;
@@ -358,68 +422,77 @@ static
void
file_entry_free(file_entry_t *entry)
{
+ pthread_mutex_destroy(&entry->mutex);
ds_close(entry->file);
my_free(entry->path);
my_free(entry);
}
static
-int
-mode_extract(int argc __attribute__((unused)),
- char **argv __attribute__((unused)))
+void *
+extract_worker_thread_func(void *arg)
{
- xb_rstream_t *stream;
- xb_rstream_result_t res;
xb_rstream_chunk_t chunk;
- HASH filehash;
file_entry_t *entry;
- ds_ctxt_t *ds_ctxt;
+ xb_rstream_result_t res;
- stream = xb_stream_read_new();
- if (stream == NULL) {
- msg("%s: xb_stream_read_new() failed.\n", my_progname);
- return 1;
- }
+ extract_ctxt_t *ctxt = (extract_ctxt_t *) arg;
- /* If --directory is specified, it is already set as CWD by now. */
- ds_ctxt = ds_create(".", DS_TYPE_LOCAL);
+ my_thread_init();
- if (my_hash_init(&filehash, &my_charset_bin, START_FILE_HASH_SIZE,
- 0, 0, (my_hash_get_key) get_file_entry_key,
- (my_hash_free_key) file_entry_free, MYF(0))) {
- msg("%s: failed to initialize file hash.\n", my_progname);
- goto err;
- }
+ memset(&chunk, 0, sizeof(chunk));
+
+ while (1) {
+
+ pthread_mutex_lock(ctxt->mutex);
+ res = xb_stream_read_chunk(ctxt->stream, &chunk);
+
+ if (res != XB_STREAM_READ_CHUNK) {
+ pthread_mutex_unlock(ctxt->mutex);
+ break;
+ }
- while ((res = xb_stream_read_chunk(stream, &chunk)) ==
- XB_STREAM_READ_CHUNK) {
/* If unknown type and ignorable flag is set, skip this chunk */
if (chunk.type == XB_CHUNK_TYPE_UNKNOWN && \
!(chunk.flags & XB_STREAM_FLAG_IGNORABLE)) {
+ pthread_mutex_unlock(ctxt->mutex);
continue;
}
/* See if we already have this file open */
- entry = (file_entry_t *) my_hash_search(&filehash,
+ entry = (file_entry_t *) my_hash_search(ctxt->filehash,
(uchar *) chunk.path,
chunk.pathlen);
if (entry == NULL) {
- entry = file_entry_new(ds_ctxt, chunk.path,
+ entry = file_entry_new(ctxt,
+ chunk.path,
chunk.pathlen);
if (entry == NULL) {
- goto err;
+ pthread_mutex_unlock(ctxt->mutex);
+ break;
}
- if (my_hash_insert(&filehash, (uchar *) entry)) {
+ if (my_hash_insert(ctxt->filehash, (uchar *) entry)) {
msg("%s: my_hash_insert() failed.\n",
my_progname);
- goto err;
+ pthread_mutex_unlock(ctxt->mutex);
+ break;
}
}
- if (chunk.type == XB_CHUNK_TYPE_EOF) {
- my_hash_delete(&filehash, (uchar *) entry);
+ pthread_mutex_lock(&entry->mutex);
+
+ pthread_mutex_unlock(ctxt->mutex);
+ res = xb_stream_validate_checksum(&chunk);
+
+ if (res != XB_STREAM_READ_CHUNK) {
+ pthread_mutex_unlock(&entry->mutex);
+ break;
+ }
+
+ if (chunk.type == XB_CHUNK_TYPE_EOF) {
+ pthread_mutex_unlock(&entry->mutex);
continue;
}
@@ -427,30 +500,114 @@ mode_extract(int argc __attribute__((unused)),
msg("%s: out-of-order chunk: real offset = 0x%llx, "
"expected offset = 0x%llx\n", my_progname,
chunk.offset, entry->offset);
- goto err;
+ pthread_mutex_unlock(&entry->mutex);
+ res = XB_STREAM_READ_ERROR;
+ break;
}
if (ds_write(entry->file, chunk.data, chunk.length)) {
msg("%s: my_write() failed.\n", my_progname);
- goto err;
+ pthread_mutex_unlock(&entry->mutex);
+ res = XB_STREAM_READ_ERROR;
+ break;
}
entry->offset += chunk.length;
- };
- if (res == XB_STREAM_READ_ERROR) {
- goto err;
+ pthread_mutex_unlock(&entry->mutex);
}
- my_hash_free(&filehash);
- ds_destroy(ds_ctxt);
- xb_stream_read_done(stream);
+ if (chunk.data)
+ my_free(chunk.data);
+
+ my_thread_end();
+
+ return (void *)(res);
+}
+
+
+static
+int
+mode_extract(int n_threads, int argc __attribute__((unused)),
+ char **argv __attribute__((unused)))
+{
+ xb_rstream_t *stream = NULL;
+ HASH filehash;
+ ds_ctxt_t *ds_ctxt = NULL;
+ ds_ctxt_t *ds_decrypt_ctxt = NULL;
+ extract_ctxt_t ctxt;
+ int i;
+ pthread_t *tids = NULL;
+ void **retvals = NULL;
+ pthread_mutex_t mutex;
+ int ret = 0;
+
+ if (my_hash_init(&filehash, &my_charset_bin, START_FILE_HASH_SIZE,
+ 0, 0, (my_hash_get_key) get_file_entry_key,
+ (my_hash_free_key) file_entry_free, MYF(0))) {
+ msg("%s: failed to initialize file hash.\n", my_progname);
+ return 1;
+ }
+
+ if (pthread_mutex_init(&mutex, NULL)) {
+ msg("%s: failed to initialize mutex.\n", my_progname);
+ my_hash_free(&filehash);
+ return 1;
+ }
+
+ /* If --directory is specified, it is already set as CWD by now. */
+ ds_ctxt = ds_create(".", DS_TYPE_LOCAL);
+ if (ds_ctxt == NULL) {
+ ret = 1;
+ goto exit;
+ }
+
+
+ stream = xb_stream_read_new();
+ if (stream == NULL) {
+ msg("%s: xb_stream_read_new() failed.\n", my_progname);
+ pthread_mutex_destroy(&mutex);
+ ret = 1;
+ goto exit;
+ }
+
+ ctxt.stream = stream;
+ ctxt.filehash = &filehash;
+ ctxt.ds_ctxt = ds_ctxt;
+ ctxt.ds_decrypt_ctxt = ds_decrypt_ctxt;
+ ctxt.mutex = &mutex;
+
+ tids = malloc(sizeof(pthread_t) * n_threads);
+ retvals = malloc(sizeof(void*) * n_threads);
+
+ for (i = 0; i < n_threads; i++)
+ pthread_create(tids + i, NULL, extract_worker_thread_func,
+ &ctxt);
+
+ for (i = 0; i < n_threads; i++)
+ pthread_join(tids[i], retvals + i);
+
+ for (i = 0; i < n_threads; i++) {
+ if ((ulong)retvals[i] == XB_STREAM_READ_ERROR) {
+ ret = 1;
+ goto exit;
+ }
+ }
+
+exit:
+ pthread_mutex_destroy(&mutex);
+
+ free(tids);
+ free(retvals);
- return 0;
-err:
my_hash_free(&filehash);
- ds_destroy(ds_ctxt);
+ if (ds_ctxt != NULL) {
+ ds_destroy(ds_ctxt);
+ }
+ if (ds_decrypt_ctxt) {
+ ds_destroy(ds_decrypt_ctxt);
+ }
xb_stream_read_done(stream);
- return 1;
+ return ret;
}
diff --git a/extra/mariabackup/xbstream.h b/extra/mariabackup/xbstream.h
index e9f1468e58d..ac1bf05e321 100644
--- a/extra/mariabackup/xbstream.h
+++ b/extra/mariabackup/xbstream.h
@@ -1,5 +1,5 @@
/******************************************************
-Copyright (c) 2011-2013 Percona LLC and/or its affiliates.
+Copyright (c) 2011-2017 Percona LLC and/or its affiliates.
The xbstream format interface.
@@ -89,8 +89,10 @@ typedef struct {
char path[FN_REFLEN];
size_t length;
my_off_t offset;
+ my_off_t checksum_offset;
void *data;
ulong checksum;
+ size_t buflen;
} xb_rstream_chunk_t;
xb_rstream_t *xb_stream_read_new(void);
@@ -100,4 +102,6 @@ xb_rstream_result_t xb_stream_read_chunk(xb_rstream_t *stream,
int xb_stream_read_done(xb_rstream_t *stream);
+int xb_stream_validate_checksum(xb_rstream_chunk_t *chunk);
+
#endif
diff --git a/extra/mariabackup/xbstream_read.c b/extra/mariabackup/xbstream_read.c
index f8d88926ec2..8d19242301b 100644
--- a/extra/mariabackup/xbstream_read.c
+++ b/extra/mariabackup/xbstream_read.c
@@ -1,5 +1,5 @@
/******************************************************
-Copyright (c) 2011-2013 Percona LLC and/or its affiliates.
+Copyright (c) 2011-2017 Percona LLC and/or its affiliates.
The xbstream format reader implementation.
@@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#include <zlib.h>
#include "common.h"
#include "xbstream.h"
+#include "crc_glue.h"
/* Allocate 1 MB for the payload buffer initially */
#define INIT_BUFFER_LEN (1024 * 1024)
@@ -34,8 +35,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
struct xb_rstream_struct {
my_off_t offset;
File fd;
- void *buffer;
- size_t buflen;
};
xb_rstream_t *
@@ -45,9 +44,6 @@ xb_stream_read_new(void)
stream = (xb_rstream_t *) my_malloc(sizeof(xb_rstream_t), MYF(MY_FAE));
- stream->buffer = my_malloc(INIT_BUFFER_LEN, MYF(MY_FAE));
- stream->buflen = INIT_BUFFER_LEN;
-
#ifdef __WIN__
setmode(fileno(stdin), _O_BINARY);
#endif
@@ -71,6 +67,23 @@ validate_chunk_type(uchar code)
}
}
+int
+xb_stream_validate_checksum(xb_rstream_chunk_t *chunk)
+{
+ ulong checksum;
+
+ checksum = crc32_iso3309(0, chunk->data, (uint)chunk->length);
+ if (checksum != chunk->checksum) {
+ msg("xb_stream_read_chunk(): invalid checksum at offset "
+ "0x%llx: expected 0x%lx, read 0x%lx.\n",
+ (ulonglong) chunk->checksum_offset, chunk->checksum,
+ checksum);
+ return XB_STREAM_READ_ERROR;
+ }
+
+ return XB_STREAM_READ_CHUNK;
+}
+
#define F_READ(buf,len) \
do { \
if (xb_read_full(fd, buf, len) < len) { \
@@ -87,8 +100,6 @@ xb_stream_read_chunk(xb_rstream_t *stream, xb_rstream_chunk_t *chunk)
uint pathlen;
size_t tbytes;
ulonglong ullval;
- ulong checksum_exp;
- ulong checksum;
File fd = stream->fd;
xb_ad(sizeof(tmpbuf) >= CHUNK_HEADER_CONSTANT_LEN);
@@ -178,39 +189,30 @@ xb_stream_read_chunk(xb_rstream_t *stream, xb_rstream_chunk_t *chunk)
stream->offset += 8;
/* Reallocate the buffer if needed */
- if (chunk->length > stream->buflen) {
- stream->buffer = my_realloc(stream->buffer, chunk->length,
- MYF(MY_WME));
- if (stream->buffer == NULL) {
+ if (chunk->length > chunk->buflen) {
+ chunk->data = my_realloc(chunk->data, chunk->length,
+ MYF(MY_WME | MY_ALLOW_ZERO_PTR));
+ if (chunk->data == NULL) {
msg("xb_stream_read_chunk(): failed to increase buffer "
"to %lu bytes.\n", (ulong) chunk->length);
goto err;
}
- stream->buflen = chunk->length;
+ chunk->buflen = chunk->length;
}
/* Checksum */
F_READ(tmpbuf, 4);
- checksum_exp = uint4korr(tmpbuf);
+ chunk->checksum = uint4korr(tmpbuf);
+ chunk->checksum_offset = stream->offset;
/* Payload */
if (chunk->length > 0) {
- F_READ(stream->buffer, chunk->length);
+ F_READ(chunk->data, chunk->length);
stream->offset += chunk->length;
}
- checksum = crc32(0, stream->buffer, chunk->length);
- if (checksum != checksum_exp) {
- msg("xb_stream_read_chunk(): invalid checksum at offset "
- "0x%llx: expected 0x%lx, read 0x%lx.\n",
- (ulonglong) stream->offset, checksum_exp, checksum);
- goto err;
- }
stream->offset += 4;
- chunk->data = stream->buffer;
- chunk->checksum = checksum;
-
return XB_STREAM_READ_CHUNK;
err:
@@ -220,7 +222,6 @@ err:
int
xb_stream_read_done(xb_rstream_t *stream)
{
- my_free(stream->buffer);
my_free(stream);
return 0;
diff --git a/extra/mariabackup/xbstream_write.c b/extra/mariabackup/xbstream_write.c
index a11811dc375..978be71e7dd 100644
--- a/extra/mariabackup/xbstream_write.c
+++ b/extra/mariabackup/xbstream_write.c
@@ -1,5 +1,5 @@
/******************************************************
-Copyright (c) 2011-2013 Percona LLC and/or its affiliates.
+Copyright (c) 2011-2017 Percona LLC and/or its affiliates.
The xbstream format writer implementation.
@@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#include <zlib.h>
#include "common.h"
#include "xbstream.h"
+#include "crc_glue.h"
/* Group writes smaller than this into a single chunk */
#define XB_STREAM_MIN_CHUNK_SIZE (10 * 1024 * 1024)
@@ -215,12 +216,13 @@ xb_stream_write_chunk(xb_wstream_file_t *file, const void *buf, size_t len)
int8store(ptr, len); /* Payload length */
ptr += 8;
+ checksum = crc32_iso3309(0, buf, (uint)len); /* checksum */
+
pthread_mutex_lock(&stream->mutex);
int8store(ptr, file->offset); /* Payload offset */
ptr += 8;
- checksum = crc32(0, buf, (uint)len); /* checksum */
int4store(ptr, checksum);
ptr += 4;
diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc
index 3ca6efaafcf..d16687926aa 100644
--- a/extra/mariabackup/xtrabackup.cc
+++ b/extra/mariabackup/xtrabackup.cc
@@ -1,6 +1,6 @@
/******************************************************
XtraBackup: hot backup tool for InnoDB
-(c) 2009-2015 Percona LLC and/or its affiliates
+(c) 2009-2017 Percona LLC and/or its affiliates
Originally Created 3/3/2009 Yasufumi Kinoshita
Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
@@ -67,6 +67,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA
#include <srv0start.h>
#include <buf0dblwr.h>
+#include <list>
#include <sstream>
#include <set>
#include <mysql.h>
@@ -94,6 +95,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA
#include "encryption_plugin.h"
#include <sql_plugin.h>
#include <srv0srv.h>
+#include <crc_glue.h>
/* TODO: replace with appropriate macros used in InnoDB 5.6 */
#define PAGE_ZIP_MIN_SIZE_SHIFT 10
@@ -144,25 +146,24 @@ char xtrabackup_real_incremental_dir[FN_REFLEN];
lsn_t xtrabackup_archived_to_lsn = 0; /* for --archived-to-lsn */
-char *xtrabackup_tables = NULL;
-
char *xtrabackup_tmpdir;
-/* List of regular expressions for filtering */
-typedef struct xb_regex_list_node_struct xb_regex_list_node_t;
-struct xb_regex_list_node_struct {
- UT_LIST_NODE_T(xb_regex_list_node_t) regex_list;
- regex_t regex;
-};
-static UT_LIST_BASE_NODE_T(xb_regex_list_node_t) regex_list;
-static regmatch_t tables_regmatch[1];
-
+char *xtrabackup_tables = NULL;
char *xtrabackup_tables_file = NULL;
-static hash_table_t* tables_hash = NULL;
+char *xtrabackup_tables_exclude = NULL;
+
+typedef std::list<regex_t> regex_list_t;
+static regex_list_t regex_include_list;
+static regex_list_t regex_exclude_list;
+
+static hash_table_t* tables_include_hash = NULL;
+static hash_table_t* tables_exclude_hash = NULL;
char *xtrabackup_databases = NULL;
char *xtrabackup_databases_file = NULL;
-static hash_table_t* databases_hash = NULL;
+char *xtrabackup_databases_exclude = NULL;
+static hash_table_t* databases_include_hash = NULL;
+static hash_table_t* databases_exclude_hash = NULL;
static hash_table_t* inc_dir_tables_hash;
@@ -615,6 +616,8 @@ enum options_xtrabackup
OPT_SSL_VERIFY_SERVER_CERT,
OPT_SERVER_PUBLIC_KEY,
+ OPT_XTRA_TABLES_EXCLUDE,
+ OPT_XTRA_DATABASES_EXCLUDE,
};
struct my_option xb_client_options[] =
@@ -685,6 +688,16 @@ struct my_option xb_client_options[] =
"filtering by list of databases in the file.",
(G_PTR*) &xtrabackup_databases_file, (G_PTR*) &xtrabackup_databases_file,
0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+ {"tables-exclude", OPT_XTRA_TABLES_EXCLUDE, "filtering by regexp for table names. "
+ "Operates the same way as --tables, but matched names are excluded from backup. "
+ "Note that this option has a higher priority than --tables.",
+ (G_PTR*) &xtrabackup_tables_exclude, (G_PTR*) &xtrabackup_tables_exclude,
+ 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+ {"databases-exclude", OPT_XTRA_DATABASES_EXCLUDE, "Excluding databases based on name, "
+ "Operates the same way as --databases, but matched names are excluded from backup. "
+ "Note that this option has a higher priority than --databases.",
+ (G_PTR*) &xtrabackup_databases_exclude, (G_PTR*) &xtrabackup_databases_exclude,
+ 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"create-ib-logfile", OPT_XTRA_CREATE_IB_LOGFILE, "** not work for now** creates ib_logfile* also after '--prepare'. ### If you want create ib_logfile*, only re-execute this command in same options. ###",
(G_PTR*) &xtrabackup_create_ib_logfile, (G_PTR*) &xtrabackup_create_ib_logfile,
0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
@@ -1036,8 +1049,8 @@ struct my_option xb_server_options[] =
(G_PTR*) &opt_mysql_tmpdir,
(G_PTR*) &opt_mysql_tmpdir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"parallel", OPT_XTRA_PARALLEL,
- "Number of threads to use for parallel datafiles transfer. Does not have "
- "any effect in the stream mode. The default value is 1.",
+ "Number of threads to use for parallel datafiles transfer. "
+ "The default value is 1.",
(G_PTR*) &xtrabackup_parallel, (G_PTR*) &xtrabackup_parallel, 0, GET_INT,
REQUIRED_ARG, 1, 1, INT_MAX, 0, 0, 0},
@@ -2162,43 +2175,139 @@ xtrabackup_io_throttling(void)
}
}
+static
+my_bool regex_list_check_match(
+ const regex_list_t& list,
+ const char* name)
+{
+ regmatch_t tables_regmatch[1];
+ for (regex_list_t::const_iterator i = list.begin(), end = list.end();
+ i != end; ++i) {
+ const regex_t& regex = *i;
+ int regres = regexec(&regex, name, 1, tables_regmatch, 0);
+
+ if (regres != REG_NOMATCH) {
+ return(TRUE);
+ }
+ }
+ return(FALSE);
+}
+
+static
+my_bool
+find_filter_in_hashtable(
+ const char* name,
+ hash_table_t* table,
+ xb_filter_entry_t** result
+)
+{
+ xb_filter_entry_t* found = NULL;
+ HASH_SEARCH(name_hash, table, ut_fold_string(name),
+ xb_filter_entry_t*,
+ found, (void) 0,
+ !strcmp(found->name, name));
+
+ if (found && result) {
+ *result = found;
+ }
+ return (found != NULL);
+}
+
/************************************************************************
-Checks if a given table name matches any of specifications in the --tables or
---tables-file options.
+Checks if a given table name matches any of specifications given in
+regex_list or tables_hash.
-@return TRUE on match. */
+@return TRUE on match or both regex_list and tables_hash are empty.*/
static my_bool
-check_if_table_matches_filters(const char *name)
+check_if_table_matches_filters(const char *name,
+ const regex_list_t& regex_list,
+ hash_table_t* tables_hash)
{
- int regres;
- xb_filter_entry_t* table;
- xb_regex_list_node_t* node;
+ if (regex_list.empty() && !tables_hash) {
+ return(FALSE);
+ }
- if (UT_LIST_GET_LEN(regex_list)) {
- /* Check against regular expressions list */
- for (node = UT_LIST_GET_FIRST(regex_list); node;
- node = UT_LIST_GET_NEXT(regex_list, node)) {
- regres = regexec(&node->regex, name, 1,
- tables_regmatch, 0);
- if (regres != REG_NOMATCH) {
+ if (regex_list_check_match(regex_list, name)) {
+ return(TRUE);
+ }
- return(TRUE);
- }
- }
+ if (tables_hash && find_filter_in_hashtable(name, tables_hash, NULL)) {
+ return(TRUE);
}
- if (tables_hash) {
- HASH_SEARCH(name_hash, tables_hash, ut_fold_string(name),
- xb_filter_entry_t*,
- table, (void) 0,
- !strcmp(table->name, name));
- if (table) {
+ return FALSE;
+}
+
+enum skip_database_check_result {
+ DATABASE_SKIP,
+ DATABASE_SKIP_SOME_TABLES,
+ DATABASE_DONT_SKIP,
+ DATABASE_DONT_SKIP_UNLESS_EXPLICITLY_EXCLUDED,
+};
- return(TRUE);
+/************************************************************************
+Checks if a database specified by name should be skipped from backup based on
+the --databases, --databases_file or --databases_exclude options.
+
+@return TRUE if entire database should be skipped,
+ FALSE otherwise.
+*/
+static
+skip_database_check_result
+check_if_skip_database(
+ const char* name /*!< in: path to the database */
+)
+{
+ /* There are some filters for databases, check them */
+ xb_filter_entry_t* database = NULL;
+
+ if (databases_exclude_hash &&
+ find_filter_in_hashtable(name, databases_exclude_hash,
+ &database) &&
+ !database->has_tables) {
+ /* Database is found and there are no tables specified,
+ skip entire db. */
+ return DATABASE_SKIP;
+ }
+
+ if (databases_include_hash) {
+ if (!find_filter_in_hashtable(name, databases_include_hash,
+ &database)) {
+ /* Database isn't found, skip the database */
+ return DATABASE_SKIP;
+ } else if (database->has_tables) {
+ return DATABASE_SKIP_SOME_TABLES;
+ } else {
+ return DATABASE_DONT_SKIP_UNLESS_EXPLICITLY_EXCLUDED;
}
}
- return(FALSE);
+ return DATABASE_DONT_SKIP;
+}
+
+/************************************************************************
+Checks if a database specified by path should be skipped from backup based on
+the --databases, --databases_file or --databases_exclude options.
+
+@return TRUE if the table should be skipped. */
+my_bool
+check_if_skip_database_by_path(
+ const char* path /*!< in: path to the db directory. */
+)
+{
+ if (databases_include_hash == NULL &&
+ databases_exclude_hash == NULL) {
+ return(FALSE);
+ }
+
+ const char* db_name = strrchr(path, SRV_PATH_SEPARATOR);
+ if (db_name == NULL) {
+ db_name = path;
+ } else {
+ ++db_name;
+ }
+
+ return check_if_skip_database(db_name) == DATABASE_SKIP;
}
/************************************************************************
@@ -2217,9 +2326,12 @@ check_if_skip_table(
const char *ptr;
char *eptr;
- if (UT_LIST_GET_LEN(regex_list) == 0 &&
- tables_hash == NULL &&
- databases_hash == NULL) {
+ if (regex_exclude_list.empty() &&
+ regex_include_list.empty() &&
+ tables_include_hash == NULL &&
+ tables_exclude_hash == NULL &&
+ databases_include_hash == NULL &&
+ databases_exclude_hash == NULL) {
return(FALSE);
}
@@ -2237,23 +2349,10 @@ check_if_skip_table(
strncpy(buf, dbname, FN_REFLEN);
buf[tbname - 1 - dbname] = 0;
- if (databases_hash) {
- /* There are some filters for databases, check them */
- xb_filter_entry_t* database;
-
- HASH_SEARCH(name_hash, databases_hash, ut_fold_string(buf),
- xb_filter_entry_t*,
- database, (void) 0,
- !strcmp(database->name, buf));
- /* Table's database isn't found, skip the table */
- if (!database) {
- return(TRUE);
- }
- /* There aren't tables specified for the database,
- it should be backed up entirely */
- if (!database->has_tables) {
- return(FALSE);
- }
+ const skip_database_check_result skip_database =
+ check_if_skip_database(buf);
+ if (skip_database == DATABASE_SKIP) {
+ return (TRUE);
}
buf[FN_REFLEN - 1] = '\0';
@@ -2270,21 +2369,43 @@ check_if_skip_table(
/* For partitioned tables first try to match against the regexp
without truncating the #P#... suffix so we can backup individual
partitions with regexps like '^test[.]t#P#p5' */
- if (check_if_table_matches_filters(buf)) {
-
+ if (check_if_table_matches_filters(buf, regex_exclude_list,
+ tables_exclude_hash)) {
+ return(TRUE);
+ }
+ if (check_if_table_matches_filters(buf, regex_include_list,
+ tables_include_hash)) {
return(FALSE);
}
if ((eptr = strstr(buf, "#P#")) != NULL) {
-
*eptr = 0;
- if (check_if_table_matches_filters(buf)) {
-
+ if (check_if_table_matches_filters(buf, regex_exclude_list,
+ tables_exclude_hash)) {
+ return (TRUE);
+ }
+ if (check_if_table_matches_filters(buf, regex_include_list,
+ tables_include_hash)) {
return(FALSE);
}
}
- return(TRUE);
+ if (skip_database == DATABASE_DONT_SKIP_UNLESS_EXPLICITLY_EXCLUDED) {
+ /* Database is in include-list, and qualified name wasn't
+ found in any of exclusion filters.*/
+ return (FALSE);
+ }
+
+ if (skip_database == DATABASE_SKIP_SOME_TABLES ||
+ !regex_include_list.empty() ||
+ tables_include_hash) {
+
+ /* Include lists are present, but qualified name
+ failed to match any.*/
+ return(TRUE);
+ }
+
+ return(FALSE);
}
/***********************************************************************
@@ -3040,6 +3161,8 @@ xtrabackup_init_datasinks(void)
if (xtrabackup_encrypt) {
ds_ctxt_t *ds;
+
+
ds = ds_create(xtrabackup_target_dir, DS_TYPE_ENCRYPT);
xtrabackup_add_datasink(ds);
@@ -3360,7 +3483,10 @@ static
void
xb_register_filter_entry(
/*=====================*/
- const char* name) /*!< in: name */
+ const char* name, /*!< in: name */
+ hash_table_t** databases_hash,
+ hash_table_t** tables_hash
+ )
{
const char* p;
size_t namelen;
@@ -3376,25 +3502,45 @@ xb_register_filter_entry(
strncpy(dbname, name, p - name);
dbname[p - name] = 0;
- if (databases_hash) {
- HASH_SEARCH(name_hash, databases_hash,
+ if (*databases_hash) {
+ HASH_SEARCH(name_hash, (*databases_hash),
ut_fold_string(dbname),
xb_filter_entry_t*,
db_entry, (void) 0,
!strcmp(db_entry->name, dbname));
}
if (!db_entry) {
- db_entry = xb_add_filter(dbname, &databases_hash);
+ db_entry = xb_add_filter(dbname, databases_hash);
}
db_entry->has_tables = TRUE;
- xb_add_filter(name, &tables_hash);
+ xb_add_filter(name, tables_hash);
} else {
xb_validate_name(name, namelen);
- xb_add_filter(name, &databases_hash);
+ xb_add_filter(name, databases_hash);
}
}
+static
+void
+xb_register_include_filter_entry(
+ const char* name
+)
+{
+ xb_register_filter_entry(name, &databases_include_hash,
+ &tables_include_hash);
+}
+
+static
+void
+xb_register_exclude_filter_entry(
+ const char* name
+)
+{
+ xb_register_filter_entry(name, &databases_exclude_hash,
+ &tables_exclude_hash);
+}
+
/***********************************************************************
Register new table for the filter. */
static
@@ -3408,33 +3554,52 @@ xb_register_table(
exit(EXIT_FAILURE);
}
- xb_register_filter_entry(name);
+ xb_register_include_filter_entry(name);
}
-/***********************************************************************
-Register new regex for the filter. */
static
void
-xb_register_regex(
-/*==============*/
- const char* regex) /*!< in: regex */
+xb_add_regex_to_list(
+ const char* regex, /*!< in: regex */
+ const char* error_context, /*!< in: context to error message */
+ regex_list_t* list) /*! in: list to put new regex to */
{
- xb_regex_list_node_t* node;
char errbuf[100];
int ret;
- node = static_cast<xb_regex_list_node_t *>
- (ut_malloc(sizeof(xb_regex_list_node_t)));
+ regex_t compiled_regex;
+ ret = regcomp(&compiled_regex, regex, REG_EXTENDED);
- ret = regcomp(&node->regex, regex, REG_EXTENDED);
if (ret != 0) {
- xb_regerror(ret, &node->regex, errbuf, sizeof(errbuf));
- msg("xtrabackup: error: tables regcomp(%s): %s\n",
- regex, errbuf);
+ regerror(ret, &compiled_regex, errbuf, sizeof(errbuf));
+ msg("xtrabackup: error: %s regcomp(%s): %s\n",
+ error_context, regex, errbuf);
exit(EXIT_FAILURE);
}
- UT_LIST_ADD_LAST(regex_list, regex_list, node);
+ list->push_back(compiled_regex);
+}
+
+/***********************************************************************
+Register new regex for the include filter. */
+static
+void
+xb_register_include_regex(
+/*==============*/
+ const char* regex) /*!< in: regex */
+{
+ xb_add_regex_to_list(regex, "tables", &regex_include_list);
+}
+
+/***********************************************************************
+Register new regex for the exclude filter. */
+static
+void
+xb_register_exclude_regex(
+/*==============*/
+ const char* regex) /*!< in: regex */
+{
+ xb_add_regex_to_list(regex, "tables-exclude", &regex_exclude_list);
}
typedef void (*insert_entry_func_t)(const char*);
@@ -3500,26 +3665,34 @@ static
void
xb_filters_init()
{
- UT_LIST_INIT(regex_list);
-
if (xtrabackup_databases) {
xb_load_list_string(xtrabackup_databases, " \t",
- xb_register_filter_entry);
+ xb_register_include_filter_entry);
}
if (xtrabackup_databases_file) {
xb_load_list_file(xtrabackup_databases_file,
- xb_register_filter_entry);
+ xb_register_include_filter_entry);
+ }
+
+ if (xtrabackup_databases_exclude) {
+ xb_load_list_string(xtrabackup_databases_exclude, " \t",
+ xb_register_exclude_filter_entry);
}
if (xtrabackup_tables) {
xb_load_list_string(xtrabackup_tables, ",",
- xb_register_regex);
+ xb_register_include_regex);
}
if (xtrabackup_tables_file) {
xb_load_list_file(xtrabackup_tables_file, xb_register_table);
}
+
+ if (xtrabackup_tables_exclude) {
+ xb_load_list_string(xtrabackup_tables_exclude, ",",
+ xb_register_exclude_regex);
+ }
}
static
@@ -3551,25 +3724,37 @@ xb_filter_hash_free(hash_table_t* hash)
hash_table_free(hash);
}
+static void xb_regex_list_free(regex_list_t* list)
+{
+ while (list->size() > 0) {
+ xb_regfree(&list->front());
+ list->pop_front();
+ }
+}
+
/************************************************************************
Destroy table filters for partial backup. */
static
void
xb_filters_free()
{
- while (UT_LIST_GET_LEN(regex_list) > 0) {
- xb_regex_list_node_t* node = UT_LIST_GET_FIRST(regex_list);
- UT_LIST_REMOVE(regex_list, regex_list, node);
- regfree(&node->regex);
- ut_free(node);
+ xb_regex_list_free(&regex_include_list);
+ xb_regex_list_free(&regex_exclude_list);
+
+ if (tables_include_hash) {
+ xb_filter_hash_free(tables_include_hash);
}
- if (tables_hash) {
- xb_filter_hash_free(tables_hash);
+ if (tables_exclude_hash) {
+ xb_filter_hash_free(tables_exclude_hash);
}
- if (databases_hash) {
- xb_filter_hash_free(databases_hash);
+ if (databases_include_hash) {
+ xb_filter_hash_free(databases_include_hash);
+ }
+
+ if (databases_exclude_hash) {
+ xb_filter_hash_free(databases_exclude_hash);
}
}
@@ -3854,6 +4039,7 @@ xtrabackup_backup_func(void)
srv_general_init();
ut_crc32_init();
+ crc_init();
#ifdef WITH_INNODB_DISALLOW_WRITES
srv_allow_writes_event = os_event_create();
@@ -6014,7 +6200,7 @@ xb_export_cfg_write(
file = fopen(file_path, "w+b");
if (file == NULL) {
- msg("xtrabackup: Error: cannot close %s\n", node->name);
+ msg("xtrabackup: Error: cannot open %s\n", node->name);
success = false;
} else {
@@ -6473,20 +6659,23 @@ skip_check:
table_name);
goto next_node;
}
- index = dict_table_get_first_index(table);
- n_index = UT_LIST_GET_LEN(table->indexes);
- if (n_index > 31) {
- msg("xtrabackup: error: "
- "sorry, cannot export over "
- "31 indexes for now.\n");
- goto next_node;
- }
/* Write MySQL 5.6 .cfg file */
if (!xb_export_cfg_write(node, table)) {
goto next_node;
}
+ index = dict_table_get_first_index(table);
+ n_index = UT_LIST_GET_LEN(table->indexes);
+ if (n_index > 31) {
+ msg("xtrabackup: warning: table '%s' has more "
+ "than 31 indexes, .exp file was not "
+ "generated. Table will fail to import "
+ "on server version prior to 5.6.\n",
+ table->name);
+ goto next_node;
+ }
+
/* init exp file */
memset(page, 0, UNIV_PAGE_SIZE);
mach_write_to_4(page , 0x78706f72UL);
@@ -6659,6 +6848,12 @@ next_node:
if (!xtrabackup_apply_log_only) {
+ /* xtrabackup_incremental_dir is used to indicate that
+ we are going to apply incremental backup. Here we already
+ applied incremental backup and are about to do final prepare
+ of the full backup */
+ xtrabackup_incremental_dir = NULL;
+
if(innodb_init_param()) {
goto error;
}
@@ -7016,20 +7211,24 @@ handle_options(int argc, char **argv, char ***argv_client, char ***argv_server)
}
/* ================= main =================== */
+extern my_bool(*fil_check_if_skip_database_by_path)(const char* name);
int main(int argc, char **argv)
{
char **client_defaults, **server_defaults;
char cwd[FN_REFLEN];
-
+ static char INNOBACKUPEX_EXE[]= "innobackupex";
if (argc > 1 && (strcmp(argv[1], "--innobackupex") == 0))
{
argv++;
argc--;
- argv[0] = "innobackupex";
+ argv[0] = INNOBACKUPEX_EXE;
innobackupex_mode = true;
}
+ /* Setup skip fil_load_single_tablespaces callback.*/
+ fil_check_if_skip_database_by_path = check_if_skip_database_by_path;
+
init_signals();
MY_INIT(argv[0]);
diff --git a/extra/mariabackup/xtrabackup.h b/extra/mariabackup/xtrabackup.h
index de21da31741..51491ce1f00 100644
--- a/extra/mariabackup/xtrabackup.h
+++ b/extra/mariabackup/xtrabackup.h
@@ -77,6 +77,8 @@ extern char *xtrabackup_tables;
extern char *xtrabackup_tables_file;
extern char *xtrabackup_databases;
extern char *xtrabackup_databases_file;
+extern char *xtrabackup_tables_exclude;
+extern char *xtrabackup_databases_exclude;
extern ibool xtrabackup_compress;
extern ibool xtrabackup_encrypt;
@@ -205,6 +207,17 @@ check_if_skip_table(
/******************/
const char* name); /*!< in: path to the table */
+
+/************************************************************************
+Checks if a database specified by path should be skipped from backup based on
+the --databases, --databases_file or --databases_exclude options.
+
+@return TRUE if the table should be skipped. */
+my_bool
+check_if_skip_database_by_path(
+ const char* path /*!< in: path to the db directory. */
+);
+
/************************************************************************
Check if parameter is set in defaults file or via command line argument
@return true if parameter is set. */
diff --git a/mysql-test/suite/mariabackup/partial_exclude.result b/mysql-test/suite/mariabackup/partial_exclude.result
new file mode 100644
index 00000000000..0da9b547caa
--- /dev/null
+++ b/mysql-test/suite/mariabackup/partial_exclude.result
@@ -0,0 +1,12 @@
+CREATE TABLE t1(i INT) ENGINE INNODB;
+INSERT INTO t1 VALUES(1);
+CREATE TABLE t2(i int) ENGINE INNODB;
+CREATE DATABASE db2;
+USE db2;
+CREATE TABLE t1(i INT) ENGINE INNODB;
+USE test;
+# xtrabackup backup
+t1.ibd
+DROP TABLE t1;
+DROP TABLE t2;
+DROP DATABASE db2;
diff --git a/mysql-test/suite/mariabackup/partial_exclude.test b/mysql-test/suite/mariabackup/partial_exclude.test
new file mode 100644
index 00000000000..631f9d7ee71
--- /dev/null
+++ b/mysql-test/suite/mariabackup/partial_exclude.test
@@ -0,0 +1,30 @@
+# Test --databases-exclude and --tables-exclude feature of xtrabackup 2.3.8
+
+CREATE TABLE t1(i INT) ENGINE INNODB;
+INSERT INTO t1 VALUES(1);
+CREATE TABLE t2(i int) ENGINE INNODB;
+
+CREATE DATABASE db2;
+USE db2;
+CREATE TABLE t1(i INT) ENGINE INNODB;
+
+USE test;
+
+echo # xtrabackup backup;
+
+let $targetdir=$MYSQLTEST_VARDIR/tmp/backup;
+--disable_result_log
+exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup "--tables-exclude=test.*2" "--databases-exclude=db2" --target-dir=$targetdir;
+--enable_result_log
+
+# check that only t1 table is in backup (t2 is excluded)
+list_files $targetdir/test *.ibd;
+# check that db2 database is not in the backup (excluded)
+--error 1
+list_files $targetdir/db2 *.ibd;
+
+DROP TABLE t1;
+DROP TABLE t2;
+DROP DATABASE db2;
+rmdir $targetdir;
+
diff --git a/mysql-test/suite/mariabackup/xbstream.test b/mysql-test/suite/mariabackup/xbstream.test
index f2b4704a87e..06e5685276c 100644
--- a/mysql-test/suite/mariabackup/xbstream.test
+++ b/mysql-test/suite/mariabackup/xbstream.test
@@ -9,7 +9,7 @@ echo # xtrabackup backup to stream;
exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --stream=xbstream > $streamfile 2>$targetdir/backup_stream.log;
echo # xbstream extract;
--disable_result_log
-exec $XBSTREAM -x -C $targetdir < $streamfile;
+exec $XBSTREAM -x -C $targetdir --parallel=16 < $streamfile;
echo # xtrabackup prepare;
exec $XTRABACKUP --prepare --target-dir=$targetdir;
diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc
index 216a36e2e82..c95fd872c67 100644
--- a/storage/xtradb/fil/fil0fil.cc
+++ b/storage/xtradb/fil/fil0fil.cc
@@ -71,6 +71,7 @@ static ulint srv_data_read, srv_data_written;
MYSQL_PLUGIN_IMPORT extern my_bool lower_case_file_system;
+
/*
IMPLEMENTATION OF THE TABLESPACE MEMORY CACHE
=============================================
@@ -5379,6 +5380,9 @@ fil_file_readdir_next_file(
return(-1);
}
+
+my_bool(*fil_check_if_skip_database_by_path)(const char* name);
+
#define CHECK_TIME_EVERY_N_FILES 10
/********************************************************************//**
At the server startup, if we need crash recovery, scans the database
@@ -5449,7 +5453,19 @@ fil_load_single_table_tablespaces(ibool (*pred)(const char*, const char*))
"%s/%s", fil_path_to_mysql_datadir, dbinfo.name);
srv_normalize_path_for_win(dbpath);
- dbdir = os_file_opendir(dbpath, FALSE);
+ if (IS_XTRABACKUP()) {
+ ut_a(fil_check_if_skip_database_by_path);
+ if (fil_check_if_skip_database_by_path(dbpath)) {
+ fprintf(stderr, "Skipping db: %s\n", dbpath);
+ dbdir = NULL;
+ } else {
+ /* We want wrong directory permissions to be a fatal
+ error for XtraBackup. */
+ dbdir = os_file_opendir(dbpath, TRUE);
+ }
+ } else {
+ dbdir = os_file_opendir(dbpath, FALSE);
+ }
if (dbdir != NULL) {