diff options
217 files changed, 30598 insertions, 1664 deletions
diff --git a/.gitignore b/.gitignore index 469aa0d894b..30b1521ce92 100644 --- a/.gitignore +++ b/.gitignore @@ -49,6 +49,8 @@ extra/comp_err extra/innochecksum extra/jemalloc/build/ extra/jemalloc/tmp/ +extra/mariabackup/mariabackup +extra/mariabackup/mbstream extra/my_print_defaults extra/mysql_waitpid extra/mysqld_safe_helper @@ -124,6 +126,7 @@ scripts/mytop scripts/wsrep_sst_common scripts/wsrep_sst_mysqldump scripts/wsrep_sst_rsync +scripts/wsrep_sst_mariabackup scripts/wsrep_sst_xtrabackup scripts/wsrep_sst_xtrabackup-v2 scripts/maria_add_gis_sp.sql @@ -248,6 +251,10 @@ storage/mroonga/vendor/groonga/src/groonga-benchmark storage/mroonga/vendor/groonga/src/suggest/groonga-suggest-create-dataset storage/mroonga/mysql-test/mroonga/storage/r/information_schema_plugins.result storage/mroonga/mysql-test/mroonga/storage/r/variable_version.result +xxx/* +yyy/* +zzz/* + # C and C++ # Compiled Object files diff --git a/client/mysqltest.cc b/client/mysqltest.cc index d930369e303..6c77275569c 100644 --- a/client/mysqltest.cc +++ b/client/mysqltest.cc @@ -3370,6 +3370,12 @@ void do_exec(struct st_command *command) #endif #endif + if (disable_result_log) + { + /* Collect stderr output as well, for the case app. crashes or returns error.*/ + dynstr_append(&ds_cmd, " 2>&1"); + } + DBUG_PRINT("info", ("Executing '%s' as '%s'", command->first_argument, ds_cmd.str)); @@ -3405,16 +3411,7 @@ void do_exec(struct st_command *command) len--; } #endif - if (disable_result_log) - { - if (len) - buf[len-1] = 0; - DBUG_PRINT("exec_result",("%s", buf)); - } - else - { - replace_dynstr_append_mem(ds_result, buf, len); - } + replace_dynstr_append_mem(ds_result, buf, len); } error= pclose(res_file); @@ -3424,7 +3421,7 @@ void do_exec(struct st_command *command) dynstr_free(&ds_sorted); } - if (error > 0) + if (error) { uint status= WEXITSTATUS(error); int i; @@ -3470,6 +3467,12 @@ void do_exec(struct st_command *command) } dynstr_free(&ds_cmd); + + if (disable_result_log) + { + /* Disable output in case of successful exit.*/ + dynstr_set(&ds_res,""); + } DBUG_VOID_RETURN; } @@ -3607,6 +3610,37 @@ void do_system(struct st_command *command) } +/* returns TRUE if path is inside a sandbox */ +bool is_sub_path(const char *path, size_t plen, const char *sandbox) +{ + size_t len= strlen(sandbox); + if (!sandbox || !len || plen <= len || memcmp(path, sandbox, len - 1) + || path[len] != '/') + return false; + return true; +} + + +/* returns TRUE if path cannot be modified */ +bool bad_path(const char *path) +{ + size_t plen= strlen(path); + + const char *vardir= getenv("MYSQLTEST_VARDIR"); + if (is_sub_path(path, plen, vardir)) + return false; + + const char *tmpdir= getenv("MYSQL_TMP_DIR"); + if (is_sub_path(path, plen, tmpdir)) + return false; + + report_or_die("Path '%s' is not a subdirectory of MYSQLTEST_VARDIR '%s'" + "or MYSQL_TMP_DIR '%s'", + path, vardir, tmpdir); + return true; +} + + /* SYNOPSIS set_wild_chars @@ -3665,6 +3699,9 @@ void do_remove_file(struct st_command *command) rm_args, sizeof(rm_args)/sizeof(struct command_arg), ' '); + if (bad_path(ds_filename.str)) + DBUG_VOID_RETURN; + DBUG_PRINT("info", ("removing file: %s", ds_filename.str)); error= my_delete(ds_filename.str, MYF(disable_warnings ? 0 : MY_WME)) != 0; handle_command_error(command, error, my_errno); @@ -3708,6 +3745,9 @@ void do_remove_files_wildcard(struct st_command *command) ' '); fn_format(dirname, ds_directory.str, "", "", MY_UNPACK_FILENAME); + if (bad_path(ds_directory.str)) + DBUG_VOID_RETURN; + DBUG_PRINT("info", ("listing directory: %s", dirname)); if (!(dir_info= my_dir(dirname, MYF(MY_DONT_SORT | MY_WANT_STAT | MY_WME)))) { @@ -3782,6 +3822,9 @@ void do_copy_file(struct st_command *command) sizeof(copy_file_args)/sizeof(struct command_arg), ' '); + if (bad_path(ds_to_file.str)) + DBUG_VOID_RETURN; + DBUG_PRINT("info", ("Copy %s to %s", ds_from_file.str, ds_to_file.str)); /* MY_HOLD_ORIGINAL_MODES prevents attempts to chown the file */ error= (my_copy(ds_from_file.str, ds_to_file.str, @@ -3819,6 +3862,9 @@ void do_move_file(struct st_command *command) sizeof(move_file_args)/sizeof(struct command_arg), ' '); + if (bad_path(ds_to_file.str)) + DBUG_VOID_RETURN; + DBUG_PRINT("info", ("Move %s to %s", ds_from_file.str, ds_to_file.str)); error= (my_rename(ds_from_file.str, ds_to_file.str, MYF(disable_warnings ? 0 : MY_WME)) != 0); @@ -3857,6 +3903,9 @@ void do_chmod_file(struct st_command *command) sizeof(chmod_file_args)/sizeof(struct command_arg), ' '); + if (bad_path(ds_file.str)) + DBUG_VOID_RETURN; + /* Parse what mode to set */ if (ds_mode.length != 4 || str2int(ds_mode.str, 8, 0, INT_MAX, &mode) == NullS) @@ -3928,6 +3977,9 @@ void do_mkdir(struct st_command *command) mkdir_args, sizeof(mkdir_args)/sizeof(struct command_arg), ' '); + if (bad_path(ds_dirname.str)) + DBUG_VOID_RETURN; + DBUG_PRINT("info", ("creating directory: %s", ds_dirname.str)); error= my_mkdir(ds_dirname.str, 0777, MYF(MY_WME)) != 0; handle_command_error(command, error, my_errno); @@ -3935,6 +3987,47 @@ void do_mkdir(struct st_command *command) DBUG_VOID_RETURN; } + +/* + Remove directory recursively. +*/ +static int rmtree(const char *dir) +{ + char path[FN_REFLEN]; + char sep[]={ FN_LIBCHAR, 0 }; + int err=0; + + MY_DIR *dir_info= my_dir(dir, MYF(MY_DONT_SORT | MY_WANT_STAT)); + if (!dir_info) + return 1; + + for (uint i= 0; i < dir_info->number_of_files; i++) + { + FILEINFO *file= dir_info->dir_entry + i; + /* Skip "." and ".." */ + if (!strcmp(file->name, ".") || !strcmp(file->name, "..")) + continue; + + strxnmov(path, sizeof(path), dir, sep, file->name, NULL); + + if (!MY_S_ISDIR(file->mystat->st_mode)) + err= my_delete(path, 0); + else + err= rmtree(path); + + if(err) + break; + } + + my_dirend(dir_info); + + if (!err) + err= rmdir(dir); + + return err; +} + + /* SYNOPSIS do_rmdir @@ -3942,12 +4035,11 @@ void do_mkdir(struct st_command *command) DESCRIPTION rmdir <dir_name> - Remove the empty directory <dir_name> + Remove the directory tree */ void do_rmdir(struct st_command *command) { - int error; static DYNAMIC_STRING ds_dirname; const struct command_arg rmdir_args[] = { { "dirname", ARG_STRING, TRUE, &ds_dirname, "Directory to remove" } @@ -3958,9 +4050,13 @@ void do_rmdir(struct st_command *command) rmdir_args, sizeof(rmdir_args)/sizeof(struct command_arg), ' '); + if (bad_path(ds_dirname.str)) + DBUG_VOID_RETURN; + DBUG_PRINT("info", ("removing directory: %s", ds_dirname.str)); - error= rmdir(ds_dirname.str) != 0; - handle_command_error(command, error, errno); + if (rmtree(ds_dirname.str)) + handle_command_error(command, 1, errno); + dynstr_free(&ds_dirname); DBUG_VOID_RETURN; } @@ -4073,6 +4169,9 @@ static void do_list_files_write_file_command(struct st_command *command, list_files_args, sizeof(list_files_args)/sizeof(struct command_arg), ' '); + if (bad_path(ds_filename.str)) + DBUG_VOID_RETURN; + init_dynamic_string(&ds_content, "", 1024, 1024); error= get_list_files(&ds_content, &ds_dirname, &ds_wild); handle_command_error(command, error, my_errno); @@ -4124,7 +4223,8 @@ void read_until_delimiter(DYNAMIC_STRING *ds, while (1) { c= my_getc(cur_file->file); - + if (c == '\r') + c= my_getc(cur_file->file); if (c == '\n') { cur_file->lineno++; @@ -4175,6 +4275,9 @@ void do_write_file_command(struct st_command *command, my_bool append) sizeof(write_file_args)/sizeof(struct command_arg), ' '); + if (bad_path(ds_filename.str)) + DBUG_VOID_RETURN; + if (!append && access(ds_filename.str, F_OK) == 0) { /* The file should not be overwritten */ diff --git a/cmake/build_configurations/mysql_release.cmake b/cmake/build_configurations/mysql_release.cmake index 726a151a5b1..fcac8dde6e9 100644 --- a/cmake/build_configurations/mysql_release.cmake +++ b/cmake/build_configurations/mysql_release.cmake @@ -88,20 +88,24 @@ ENDIF() OPTION(ENABLED_LOCAL_INFILE "" ON) SET(WITH_INNODB_SNAPPY OFF CACHE STRING "") IF(WIN32) + SET(WITH_LIBARCHIVE STATIC CACHE STRING "") ELSEIF(RPM) SET(WITH_SSL system CACHE STRING "") SET(WITH_ZLIB system CACHE STRING "") SET(CHECKMODULE /usr/bin/checkmodule CACHE STRING "") SET(SEMODULE_PACKAGE /usr/bin/semodule_package CACHE STRING "") + SET(WITH_LIBARCHIVE ON CACHE STRING "") ELSEIF(DEB) SET(WITH_SSL system CACHE STRING "") SET(WITH_ZLIB system CACHE STRING "") SET(WITH_LIBWRAP ON) SET(HAVE_EMBEDDED_PRIVILEGE_CONTROL ON) + SET(WITH_LIBARCHIVE ON CACHE STRING "") ELSE() SET(WITH_SSL bundled CACHE STRING "") SET(WITH_ZLIB bundled CACHE STRING "") SET(WITH_JEMALLOC static CACHE STRING "") + SET(WITH_LIBARCHIVE STATIC CACHE STRING "") ENDIF() IF(NOT COMPILATION_COMMENT) diff --git a/cmake/cpack_rpm.cmake b/cmake/cpack_rpm.cmake index dced85f3a22..7a037d7ce43 100644 --- a/cmake/cpack_rpm.cmake +++ b/cmake/cpack_rpm.cmake @@ -23,10 +23,14 @@ SET(CPACK_COMPONENT_SHAREDLIBRARIES_GROUP "shared") SET(CPACK_COMPONENT_COMMON_GROUP "common") SET(CPACK_COMPONENT_CLIENTPLUGINS_GROUP "common") SET(CPACK_COMPONENT_COMPAT_GROUP "compat") +SET(CPACK_COMPONENT_BACKUP_GROUP "backup") + SET(CPACK_COMPONENTS_ALL Server ManPagesServer IniFiles Server_Scripts SupportFiles Development ManPagesDevelopment ManPagesTest Readme ManPagesClient Test - Common Client SharedLibraries ClientPlugins) + Common Client SharedLibraries ClientPlugins + backup +) SET(CPACK_RPM_PACKAGE_NAME ${CPACK_PACKAGE_NAME}) SET(CPACK_PACKAGE_FILE_NAME "${CPACK_RPM_PACKAGE_NAME}-${VERSION}-${RPM}-${CMAKE_SYSTEM_PROCESSOR}") @@ -112,6 +116,7 @@ SET(CPACK_RPM_client_USER_FILELIST ${ignored} "%config(noreplace) ${INSTALL_SYSC SET(CPACK_RPM_compat_USER_FILELIST ${ignored}) SET(CPACK_RPM_devel_USER_FILELIST ${ignored}) SET(CPACK_RPM_test_USER_FILELIST ${ignored}) +SET(CPACK_RPM_backup_USER_FILELIST ${ignored}) # "set/append array" - append a set of strings, separated by a space MACRO(SETA var) diff --git a/cmake/plugin.cmake b/cmake/plugin.cmake index 354ee53c7bb..e6bb64b19c6 100644 --- a/cmake/plugin.cmake +++ b/cmake/plugin.cmake @@ -201,13 +201,15 @@ MACRO(MYSQL_ADD_PLUGIN) # executable to the linker command line (it would result into link error). # Thus we skip TARGET_LINK_LIBRARIES on Linux, as it would only generate # an additional dependency. - IF(NOT ARG_CLIENT) + IF(ARG_RECOMPILE_FOR_EMBEDDED OR ARG_STORAGE_ENGINE) IF(MSVC) ADD_DEPENDENCIES(${target} gen_mysqld_lib) TARGET_LINK_LIBRARIES(${target} mysqld_import_lib) ELSEIF(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux") TARGET_LINK_LIBRARIES (${target} mysqld) ENDIF() + ELSEIF(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND NOT WITH_ASAN) + TARGET_LINK_LIBRARIES (${target} "-Wl,--no-undefined") ENDIF() ADD_DEPENDENCIES(${target} GenError ${ARG_DEPENDENCIES}) diff --git a/debian/control b/debian/control index c501078f678..96271e046f3 100644 --- a/debian/control +++ b/debian/control @@ -622,4 +622,4 @@ Description: MariaDB database regression test suite - data files language in the world. The main goals of MariaDB are speed, robustness and ease of use. . - This package has the architecture independent data files for the test suite. + This package has the architecture independent data files for the test suite.
\ No newline at end of file diff --git a/debian/mariadb-backup-10.1.files b/debian/mariadb-backup-10.1.files new file mode 100644 index 00000000000..734117c92e2 --- /dev/null +++ b/debian/mariadb-backup-10.1.files @@ -0,0 +1,2 @@ +usr/bin/mariabackup +usr/bin/mbstream diff --git a/debian/mariadb-server-10.2.install b/debian/mariadb-server-10.2.install index 3b5a475f486..81981fe1f60 100644 --- a/debian/mariadb-server-10.2.install +++ b/debian/mariadb-server-10.2.install @@ -38,6 +38,7 @@ usr/bin/wsrep_sst_mysqldump usr/bin/wsrep_sst_rsync usr/bin/wsrep_sst_xtrabackup usr/bin/wsrep_sst_xtrabackup-v2 +usr/bin/wsrep_sst_mariabackup usr/lib/mysql/plugin/auth_pam.so usr/lib/mysql/plugin/auth_socket.so usr/lib/mysql/plugin/file_key_management.so diff --git a/extra/mariabackup/CMakeLists.txt b/extra/mariabackup/CMakeLists.txt new file mode 100644 index 00000000000..693082b765a --- /dev/null +++ b/extra/mariabackup/CMakeLists.txt @@ -0,0 +1,214 @@ +# Copyright (c) 2013, 2017 Percona LLC and/or its affiliates. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + + +OPTION(WITH_MARIABACKUP "Include mariabackup" ON) +IF(NOT WITH_MARIABACKUP) + RETURN() +ENDIF() + + +IF(NOT WIN32) + CHECK_SYMBOL_EXISTS(regcomp regex.h HAVE_SYSTEM_REGEX) + IF(HAVE_SYSTEM_REGEX) + ADD_DEFINITIONS(-DHAVE_SYSTEM_REGEX) + ENDIF() +ENDIF() + +IF(WITH_LIBARCHIVE STREQUAL "STATIC") + SET(CMAKE_FIND_LIBRARY_SUFFIXES .a .lib) +ENDIF() + +FIND_PACKAGE(LibArchive) + +IF(NOT DEFINED WITH_LIBARCHIVE) + IF(LibArchive_FOUND) + SET(WITH_LIBARCHIVE_DEFAULT ON) + ELSE() + SET(WITH_LIBARCHIVE_DEFAULT OFF) + ENDIF() + SET(WITH_LIBARCHIVE ${WITH_LIBARCHIVE_DEFAULT} CACHE STRING "Use libarchive for streaming features (ON, OFF or STATIC)" ) +ENDIF() + +IF(NOT WITH_LIBARCHIVE MATCHES "^(ON|OFF|STATIC)$") + MESSAGE(FATAL_ERROR "Invalid value for WITH_LIBARCHIVE: '${WITH_LIBARCHIVE}'. Use one of ON, OFF or STATIC") +ENDIF() + +IF(UNIX) + SET(PIC_FLAG -fPIC) +ENDIF() + +IF((NOT WITH_LIBARCHIVE STREQUAL "OFF") AND (NOT LibArchive_FOUND)) + IF(CMAKE_VERSION VERSION_LESS "2.8.12") + MESSAGE("libarchive can't be built, old cmake") + ELSE() + # Build a local version + INCLUDE(ExternalProject) + SET(LIBARCHIVE_DIR ${CMAKE_CURRENT_BINARY_DIR}/libarchive) + SET(libarchive_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/libarchive) + SET(libarchive_CMAKE_ARGS + -DCMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR> + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DENABLE_ICONV=OFF + -DENABLE_TAR=ON + -DENABLE_OPENSSL=OFF + -DENABLE_TEST=OFF + "-DCMAKE_C_FLAGS_DEBUG=${CMAKE_C_FLAGS_DEBUG} ${PIC_FLAG}" + "-DCMAKE_C_FLAGS_RELWITHDEBINFO=${CMAKE_C_FLAGS_RELWITHDEBINFO} ${PIC_FLAG}" + "-DCMAKE_C_FLAGS_RELEASE=${CMAKE_C_FLAGS_RELEASE} ${PIC_FLAG}" + "-DCMAKE_C_FLAGS_MINSIZEREL=${CMAKE_C_FLAGS_MINSIZEREL} ${PIC_FLAG}" + ) + IF(WIN32) + SET(libarchive_CMAKE_ARGS ${libarchive_CMAKE_ARGS} -DWINDOWS_VERSION=WIN7 -DCMAKE_DEBUG_POSTFIX=d) + SET(LIBARCHIVE_RELEASE_LIB ${LIBARCHIVE_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}archive_static${CMAKE_STATIC_LIBRARY_SUFFIX}) + SET(LIBARCHIVE_DEBUG_LIB ${LIBARCHIVE_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}archive_staticd${CMAKE_STATIC_LIBRARY_SUFFIX}) + SET(byproducts ${LIBARCHIVE_RELEASE_LIB} ${LIBARCHIVE_DEBUG_LIB}) + ELSE() + SET(LIBARCHIVE_LIB ${LIBARCHIVE_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}archive${CMAKE_STATIC_LIBRARY_SUFFIX}) + SET(byproducts ${LIBARCHIVE_LIB}) + ENDIF() + + IF(CMAKE_VERSION VERSION_GREATER "3.1") + SET(byproducts BUILD_BYPRODUCTS ${byproducts}) + ENDIF() + + ExternalProject_Add(libarchive + PREFIX ${libarchive_PREFIX} + DOWNLOAD_DIR ${LIBARCHIVE_DIR} + URL http://www.libarchive.org/downloads/libarchive-3.2.2.tar.gz + INSTALL_DIR ${LIBARCHIVE_DIR} + CMAKE_ARGS ${libarchive_CMAKE_ARGS} + ${byproducts} + ) + ADD_LIBRARY(archive_static STATIC IMPORTED) + ADD_DEPENDENCIES(archive_static libarchive) + IF(WIN32) + SET_PROPERTY(TARGET archive_static PROPERTY IMPORTED_LOCATION_RELWITHDEBINFO ${LIBARCHIVE_RELEASE_LIB}) + SET_PROPERTY(TARGET archive_static PROPERTY IMPORTED_LOCATION_RELEASE ${LIBARCHIVE_RELEASE_LIB}) + SET_PROPERTY(TARGET archive_static PROPERTY IMPORTED_LOCATION_DEBUG ${LIBARCHIVE_DEBUG_LIB}) + SET_PROPERTY(TARGET archive_static PROPERTY IMPORTED_LOCATION_MINSIZEREL ${LIBARCHIVE_RELEASE_LIB}) + ELSE() + SET_PROPERTY(TARGET archive_static PROPERTY IMPORTED_LOCATION ${LIBARCHIVE_LIB}) + ENDIF() + + SET(LibArchive_FOUND ON ) + SET(LibArchive_INCLUDE_DIRS ${LIBARCHIVE_DIR}/include ) + SET(LibArchive_LIBRARIES archive_static) + IF(WIN32) + SET(LIBARCHIVE_STATIC 1) + ENDIF() + ENDIF() +ENDIF() + + +IF(WITH_LIBARCHIVE AND LibArchive_FOUND) + ADD_DEFINITIONS(-DHAVE_LIBARCHIVE) + IF(LIBARCHIVE_STATIC) + ADD_DEFINITIONS(-DLIBARCHIVE_STATIC) + ENDIF() + INCLUDE_DIRECTORIES(${LibArchive_INCLUDE_DIRS}) + LINK_LIBRARIES(${LibArchive_LIBRARIES}) + SET(DS_ARCHIVE_SOURCE ds_archive.c) +ENDIF() + +INCLUDE_DIRECTORIES( + ${CMAKE_SOURCE_DIR}/include + ${CMAKE_SOURCE_DIR}/storage/xtradb/include + ${CMAKE_SOURCE_DIR}/sql + ${CMAKE_CURRENT_SOURCE_DIR}/quicklz + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/crc + ) + +IF(NOT HAVE_SYSTEM_REGEX) + INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/pcre) +ENDIF() + +ADD_DEFINITIONS(-UMYSQL_SERVER) +######################################################################## +# xtrabackup binary +######################################################################## + +IF(WIN32) + SET(NT_SERVICE_SOURCE ${PROJECT_SOURCE_DIR}/sql/nt_servc.cc) +ELSE() + SET(NT_SERVICE_SOURCE) +ENDIF() + +ADD_DEFINITIONS(-DPCRE_STATIC=1) + +MYSQL_ADD_EXECUTABLE(mariabackup + xtrabackup.cc + innobackupex.cc + changed_page_bitmap.cc + datasink.c + ${DS_ARCHIVE_SOURCE} + ds_buffer.c + ds_compress.c + ds_local.c + ds_stdout.c + ds_tmpfile.c + ds_xbstream.c + fil_cur.cc + quicklz/quicklz.c + read_filt.cc + write_filt.cc + wsrep.cc + xbstream_write.c + backup_mysql.cc + backup_copy.cc + encryption_plugin.cc + ${PROJECT_SOURCE_DIR}/libmysql/libmysql.c + ${PROJECT_SOURCE_DIR}/sql/net_serv.cc + ${NT_SERVICE_SOURCE} + COMPONENT backup + ) + + +# Export all symbols on Unix, for better crash callstacks +SET_TARGET_PROPERTIES(mariabackup PROPERTIES ENABLE_EXPORTS TRUE) +ADD_SUBDIRECTORY(crc) + + +TARGET_LINK_LIBRARIES(mariabackup sql crc) + +IF(NOT HAVE_SYSTEM_REGEX) + TARGET_LINK_LIBRARIES(mariabackup pcreposix) +ENDIF() + + +######################################################################## +# xbstream binary +######################################################################## +MYSQL_ADD_EXECUTABLE(mbstream + ds_buffer.c + ds_local.c + ds_stdout.c + datasink.c + xbstream.c + xbstream_read.c + xbstream_write.c + COMPONENT backup + ) + + +TARGET_LINK_LIBRARIES(mbstream + mysys + crc +) + +IF(MSVC) + SET_TARGET_PROPERTIES(mbstream PROPERTIES LINK_FLAGS setargv.obj) +ENDIF() diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc new file mode 100644 index 00000000000..1565e20d732 --- /dev/null +++ b/extra/mariabackup/backup_copy.cc @@ -0,0 +1,2075 @@ +/****************************************************** +hot backup tool for InnoDB +(c) 2009-2015 Percona LLC and/or its affiliates +(c) 2017 MariaDB +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +******************************************************* + +This file incorporates work covered by the following copyright and +permission notice: + +Copyright (c) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA + +*******************************************************/ + +#include <my_global.h> +#include <os0file.h> +#include <my_dir.h> +#include <ut0mem.h> +#include <srv0start.h> +#include <fil0fil.h> +#include <set> +#include <string> +#include <mysqld.h> +#include <sstream> +#include "fil_cur.h" +#include "xtrabackup.h" +#include "common.h" +#include "backup_copy.h" +#include "backup_mysql.h" +#include <btr0btr.h> +#include "xb0xb.h" + + +/* list of files to sync for --rsync mode */ +static std::set<std::string> rsync_list; +/* locations of tablespaces read from .isl files */ +static std::map<std::string, std::string> tablespace_locations; + +/* Whether LOCK BINLOG FOR BACKUP has been issued during backup */ +bool binlog_locked; + +/************************************************************************ +Struct represents file or directory. */ +struct datadir_node_t { + ulint dbpath_len; + char *filepath; + ulint filepath_len; + char *filepath_rel; + ulint filepath_rel_len; + bool is_empty_dir; + bool is_file; +}; + +/************************************************************************ +Holds the state needed to enumerate files in MySQL data directory. */ +struct datadir_iter_t { + char *datadir_path; + char *dbpath; + ulint dbpath_len; + char *filepath; + ulint filepath_len; + char *filepath_rel; + ulint filepath_rel_len; + os_ib_mutex_t mutex; + os_file_dir_t dir; + os_file_dir_t dbdir; + os_file_stat_t dbinfo; + os_file_stat_t fileinfo; + dberr_t err; + bool is_empty_dir; + bool is_file; + bool skip_first_level; +}; + + +/************************************************************************ +Represents the context of the thread processing MySQL data directory. */ +struct datadir_thread_ctxt_t { + datadir_iter_t *it; + uint n_thread; + uint *count; + os_ib_mutex_t count_mutex; + os_thread_id_t id; + bool ret; +}; + +static bool backup_files_from_datadir(const char *dir_path); + +/************************************************************************ +Retirn true if character if file separator */ +bool +is_path_separator(char c) +{ + return(c == FN_LIBCHAR || c == FN_LIBCHAR2); +} + + +/************************************************************************ +Fill the node struct. Memory for node need to be allocated and freed by +the caller. It is caller responsibility to initialize node with +datadir_node_init and cleanup the memory with datadir_node_free. +Node can not be shared between threads. */ +static +void +datadir_node_fill(datadir_node_t *node, datadir_iter_t *it) +{ + if (node->filepath_len < it->filepath_len) { + free(node->filepath); + node->filepath = (char*)(ut_malloc(it->filepath_len)); + node->filepath_len = it->filepath_len; + } + if (node->filepath_rel_len < it->filepath_rel_len) { + free(node->filepath_rel); + node->filepath_rel = (char*)(ut_malloc(it->filepath_rel_len)); + node->filepath_rel_len = it->filepath_rel_len; + } + + strcpy(node->filepath, it->filepath); + strcpy(node->filepath_rel, it->filepath_rel); + node->is_empty_dir = it->is_empty_dir; + node->is_file = it->is_file; +} + +static +void +datadir_node_free(datadir_node_t *node) +{ + ut_free(node->filepath); + ut_free(node->filepath_rel); + memset(node, 0, sizeof(datadir_node_t)); +} + +static +void +datadir_node_init(datadir_node_t *node) +{ + memset(node, 0, sizeof(datadir_node_t)); +} + + +/************************************************************************ +Create the MySQL data directory iterator. Memory needs to be released +with datadir_iter_free. Position should be advanced with +datadir_iter_next_file. Iterator can be shared between multiple +threads. It is guaranteed that each thread receives unique file from +data directory into its local node struct. */ +static +datadir_iter_t * +datadir_iter_new(const char *path, bool skip_first_level = true) +{ + datadir_iter_t *it; + + it = static_cast<datadir_iter_t *>(ut_malloc(sizeof(datadir_iter_t))); + memset(it, 0, sizeof(datadir_iter_t)); + + it->mutex = os_mutex_create(); + it->datadir_path = strdup(path); + + it->dir = os_file_opendir(it->datadir_path, TRUE); + + if (it->dir == NULL) { + + goto error; + } + + it->err = DB_SUCCESS; + + it->dbpath_len = FN_REFLEN; + it->dbpath = static_cast<char*>(ut_malloc(it->dbpath_len)); + + it->filepath_len = FN_REFLEN; + it->filepath = static_cast<char*>(ut_malloc(it->filepath_len)); + + it->filepath_rel_len = FN_REFLEN; + it->filepath_rel = static_cast<char*>(ut_malloc(it->filepath_rel_len)); + + it->skip_first_level = skip_first_level; + + return(it); + +error: + ut_free(it); + + return(NULL); +} + +static +bool +datadir_iter_next_database(datadir_iter_t *it) +{ + if (it->dbdir != NULL) { + if (os_file_closedir(it->dbdir) != 0) { + + msg("Warning: could not" + " close database directory %s\n", it->dbpath); + + it->err = DB_ERROR; + + } + it->dbdir = NULL; + } + + while (os_file_readdir_next_file(it->datadir_path, + it->dir, &it->dbinfo) == 0) { + ulint len; + + if ((it->dbinfo.type == OS_FILE_TYPE_FILE + && it->skip_first_level) + || it->dbinfo.type == OS_FILE_TYPE_UNKNOWN) { + + continue; + } + + /* We found a symlink or a directory; try opening it to see + if a symlink is a directory */ + + len = strlen(it->datadir_path) + + strlen (it->dbinfo.name) + 2; + if (len > it->dbpath_len) { + it->dbpath_len = len; + + if (it->dbpath) { + + ut_free(it->dbpath); + } + + it->dbpath = static_cast<char*> + (ut_malloc(it->dbpath_len)); + } + ut_snprintf(it->dbpath, it->dbpath_len, + "%s/%s", it->datadir_path, + it->dbinfo.name); + srv_normalize_path_for_win(it->dbpath); + + if (it->dbinfo.type == OS_FILE_TYPE_FILE) { + it->is_file = true; + return(true); + } + + if (check_if_skip_database_by_path(it->dbpath)) { + msg("Skipping db: %s\n", it->dbpath); + continue; + } + + /* We want wrong directory permissions to be a fatal error for + XtraBackup. */ + it->dbdir = os_file_opendir(it->dbpath, TRUE); + + if (it->dbdir != NULL) { + + it->is_file = false; + return(true); + } + + } + + return(false); +} + +/************************************************************************ +Concatenate n parts into single path */ +static +void +make_path_n(int n, char **path, ulint *path_len, ...) +{ + ulint len_needed = n + 1; + char *p; + int i; + va_list vl; + + ut_ad(n > 0); + + va_start(vl, path_len); + for (i = 0; i < n; i++) { + p = va_arg(vl, char*); + len_needed += strlen(p); + } + va_end(vl); + + if (len_needed < *path_len) { + ut_free(*path); + *path = static_cast<char*>(ut_malloc(len_needed)); + } + + va_start(vl, path_len); + p = va_arg(vl, char*); + strcpy(*path, p); + for (i = 1; i < n; i++) { + size_t plen; + p = va_arg(vl, char*); + plen = strlen(*path); + if (!is_path_separator((*path)[plen - 1])) { + (*path)[plen] = FN_LIBCHAR; + (*path)[plen + 1] = 0; + } + strcat(*path + plen, p); + } + va_end(vl); +} + +static +bool +datadir_iter_next_file(datadir_iter_t *it) +{ + if (it->is_file && it->dbpath) { + make_path_n(2, &it->filepath, &it->filepath_len, + it->datadir_path, it->dbinfo.name); + + make_path_n(1, &it->filepath_rel, &it->filepath_rel_len, + it->dbinfo.name); + + it->is_empty_dir = false; + it->is_file = false; + + return(true); + } + + if (!it->dbpath || !it->dbdir) { + + return(false); + } + + while (os_file_readdir_next_file(it->dbpath, it->dbdir, + &it->fileinfo) == 0) { + + if (it->fileinfo.type == OS_FILE_TYPE_DIR) { + + continue; + } + + /* We found a symlink or a file */ + make_path_n(3, &it->filepath, &it->filepath_len, + it->datadir_path, it->dbinfo.name, + it->fileinfo.name); + + make_path_n(2, &it->filepath_rel, &it->filepath_rel_len, + it->dbinfo.name, it->fileinfo.name); + + it->is_empty_dir = false; + + return(true); + } + + return(false); +} + +static +bool +datadir_iter_next(datadir_iter_t *it, datadir_node_t *node) +{ + bool ret = true; + + os_mutex_enter(it->mutex); + + if (datadir_iter_next_file(it)) { + + datadir_node_fill(node, it); + + goto done; + } + + while (datadir_iter_next_database(it)) { + + if (datadir_iter_next_file(it)) { + + datadir_node_fill(node, it); + + goto done; + } + + make_path_n(2, &it->filepath, &it->filepath_len, + it->datadir_path, it->dbinfo.name); + + make_path_n(1, &it->filepath_rel, &it->filepath_rel_len, + it->dbinfo.name); + + it->is_empty_dir = true; + + datadir_node_fill(node, it); + + goto done; + } + + /* nothing found */ + ret = false; + +done: + os_mutex_exit(it->mutex); + + return(ret); +} + +/************************************************************************ +Interface to read MySQL data file sequentially. One should open file +with datafile_open to get cursor and close the cursor with +datafile_close. Cursor can not be shared between multiple +threads. */ +static +void +datadir_iter_free(datadir_iter_t *it) +{ + os_mutex_free(it->mutex); + + if (it->dbdir) { + + os_file_closedir(it->dbdir); + } + + if (it->dir) { + + os_file_closedir(it->dir); + } + + ut_free(it->dbpath); + ut_free(it->filepath); + ut_free(it->filepath_rel); + free(it->datadir_path); + ut_free(it); +} + + +/************************************************************************ +Holds the state needed to copy single data file. */ +struct datafile_cur_t { + os_file_t file; + char rel_path[FN_REFLEN]; + char abs_path[FN_REFLEN]; + MY_STAT statinfo; + uint thread_n; + byte* orig_buf; + byte* buf; + size_t buf_size; + size_t buf_read; + size_t buf_offset; +}; + +static +void +datafile_close(datafile_cur_t *cursor) +{ + if (cursor->file != 0) { + os_file_close(cursor->file); + } + ut_free(cursor->buf); +} + +static +bool +datafile_open(const char *file, datafile_cur_t *cursor, uint thread_n) +{ + ulint success; + + memset(cursor, 0, sizeof(datafile_cur_t)); + + strncpy(cursor->abs_path, file, sizeof(cursor->abs_path)); + + /* Get the relative path for the destination tablespace name, i.e. the + one that can be appended to the backup root directory. Non-system + tablespaces may have absolute paths for remote tablespaces in MySQL + 5.6+. We want to make "local" copies for the backup. */ + strncpy(cursor->rel_path, + xb_get_relative_path(cursor->abs_path, FALSE), + sizeof(cursor->rel_path)); + + cursor->file = os_file_create_simple_no_error_handling(0, + cursor->abs_path, + OS_FILE_OPEN, + OS_FILE_READ_ONLY, + &success, 0); + if (!success) { + /* The following call prints an error message */ + os_file_get_last_error(TRUE); + + msg("[%02u] error: cannot open " + "file %s\n", + thread_n, cursor->abs_path); + + return(false); + } + + if (!my_stat(cursor->abs_path, &cursor->statinfo, 0)) { + msg("[%02u] error: cannot stat %s\n", + thread_n, cursor->abs_path); + + datafile_close(cursor); + + return(false); + } + + posix_fadvise(cursor->file, 0, 0, POSIX_FADV_SEQUENTIAL); + + cursor->buf_size = 10 * 1024 * 1024; + cursor->buf = static_cast<byte *>(ut_malloc((ulint)cursor->buf_size)); + + return(true); +} + + +static +xb_fil_cur_result_t +datafile_read(datafile_cur_t *cursor) +{ + ulint success; + ulint to_read; + + xtrabackup_io_throttling(); + + to_read = (ulint)MY_MIN(cursor->statinfo.st_size - cursor->buf_offset, + cursor->buf_size); + + if (to_read == 0) { + return(XB_FIL_CUR_EOF); + } + + success = os_file_read(cursor->file, cursor->buf, cursor->buf_offset, + to_read); + if (!success) { + return(XB_FIL_CUR_ERROR); + } + + posix_fadvise(cursor->file, cursor->buf_offset, to_read, + POSIX_FADV_DONTNEED); + + cursor->buf_read = to_read; + cursor->buf_offset += to_read; + + return(XB_FIL_CUR_SUCCESS); +} + + + +/************************************************************************ +Check to see if a file exists. +Takes name of the file to check. +@return true if file exists. */ +static +bool +file_exists(const char *filename) +{ + MY_STAT stat_arg; + + if (!my_stat(filename, &stat_arg, MYF(0))) { + + return(false); + } + + return(true); +} + +/************************************************************************ +Trim leading slashes from absolute path so it becomes relative */ +static +const char * +trim_dotslash(const char *path) +{ + while (*path) { + if (is_path_separator(*path)) { + ++path; + continue; + } + if (*path == '.' && is_path_separator(path[1])) { + path += 2; + continue; + } + break; + } + + return(path); +} + + + +/************************************************************************ +Check if string ends with given suffix. +@return true if string ends with given suffix. */ +static +bool +ends_with(const char *str, const char *suffix) +{ + size_t suffix_len = strlen(suffix); + size_t str_len = strlen(str); + return(str_len >= suffix_len + && strcmp(str + str_len - suffix_len, suffix) == 0); +} + +static bool starts_with(const char *str, const char *prefix) +{ + return strncmp(str, prefix, strlen(prefix)) == 0; +} + +/************************************************************************ +Create directories recursively. +@return 0 if directories created successfully. */ +static +int +mkdirp(const char *pathname, int Flags, myf MyFlags) +{ + char parent[PATH_MAX], *p; + + /* make a parent directory path */ + strncpy(parent, pathname, sizeof(parent)); + parent[sizeof(parent) - 1] = 0; + + for (p = parent + strlen(parent); + !is_path_separator(*p) && p != parent; p--); + + *p = 0; + + /* try to make parent directory */ + if (p != parent && mkdirp(parent, Flags, MyFlags) != 0) { + return(-1); + } + + /* make this one if parent has been made */ + if (my_mkdir(pathname, Flags, MyFlags) == 0) { + return(0); + } + + /* if it already exists that is fine */ + if (errno == EEXIST) { + return(0); + } + + return(-1); +} + +/************************************************************************ +Return true if first and second arguments are the same path. */ +bool +equal_paths(const char *first, const char *second) +{ +#ifdef HAVE_REALPATH + char real_first[PATH_MAX]; + char real_second[PATH_MAX]; + + if (realpath(first, real_first) == NULL) { + return false; + } + if (realpath(second, real_second) == NULL) { + return false; + } + + return (strcmp(real_first, real_second) == 0); +#else + return strcmp(first, second) == 0; +#endif +} + +/************************************************************************ +Check if directory exists. Optionally create directory if doesn't +exist. +@return true if directory exists and if it was created successfully. */ +bool +directory_exists(const char *dir, bool create) +{ + os_file_dir_t os_dir; + MY_STAT stat_arg; + char errbuf[MYSYS_STRERROR_SIZE]; + + if (my_stat(dir, &stat_arg, MYF(0)) == NULL) { + + if (!create) { + return(false); + } + + if (mkdirp(dir, 0777, MYF(0)) < 0) { + my_strerror(errbuf, sizeof(errbuf), my_errno); + msg("Can not create directory %s: %s\n", dir, errbuf); + return(false); + + } + } + + /* could be symlink */ + os_dir = os_file_opendir(dir, FALSE); + + if (os_dir == NULL) { + my_strerror(errbuf, sizeof(errbuf), my_errno); + msg("Can not open directory %s: %s\n", dir, + errbuf); + + return(false); + } + + os_file_closedir(os_dir); + + return(true); +} + +/************************************************************************ +Check that directory exists and it is empty. */ +static +bool +directory_exists_and_empty(const char *dir, const char *comment) +{ + os_file_dir_t os_dir; + dberr_t err; + os_file_stat_t info; + bool empty; + + if (!directory_exists(dir, true)) { + return(false); + } + + os_dir = os_file_opendir(dir, FALSE); + + if (os_dir == NULL) { + msg("%s can not open directory %s\n", comment, dir); + return(false); + } + + empty = (fil_file_readdir_next_file(&err, dir, os_dir, &info) != 0); + + os_file_closedir(os_dir); + + if (!empty) { + msg("%s directory %s is not empty!\n", comment, dir); + } + + return(empty); +} + + +/************************************************************************ +Check if file name ends with given set of suffixes. +@return true if it does. */ +static +bool +filename_matches(const char *filename, const char **ext_list) +{ + const char **ext; + + for (ext = ext_list; *ext; ext++) { + if (ends_with(filename, *ext)) { + return(true); + } + } + + return(false); +} + + +/************************************************************************ +Copy data file for backup. Also check if it is allowed to copy by +comparing its name to the list of known data file types and checking +if passes the rules for partial backup. +@return true if file backed up or skipped successfully. */ +static +bool +datafile_copy_backup(const char *filepath, uint thread_n) +{ + const char *ext_list[] = {"frm", "isl", "MYD", "MYI", "MAD", "MAI", + "MRG", "TRG", "TRN", "ARM", "ARZ", "CSM", "CSV", "opt", "par", + NULL}; + + /* Get the name and the path for the tablespace. node->name always + contains the path (which may be absolute for remote tablespaces in + 5.6+). space->name contains the tablespace name in the form + "./database/table.ibd" (in 5.5-) or "database/table" (in 5.6+). For a + multi-node shared tablespace, space->name contains the name of the first + node, but that's irrelevant, since we only need node_name to match them + against filters, and the shared tablespace is always copied regardless + of the filters value. */ + + if (check_if_skip_table(filepath)) { + msg_ts("[%02u] Skipping %s.\n", thread_n, filepath); + return(true); + } + + if (filename_matches(filepath, ext_list)) { + return copy_file(ds_data, filepath, filepath, thread_n); + } + + return(true); +} + + +/************************************************************************ +Same as datafile_copy_backup, but put file name into the list for +rsync command. */ +static +bool +datafile_rsync_backup(const char *filepath, bool save_to_list, FILE *f) +{ + const char *ext_list[] = {"frm", "isl", "MYD", "MYI", "MAD", "MAI", + "MRG", "TRG", "TRN", "ARM", "ARZ", "CSM", "CSV", "opt", "par", + NULL}; + + /* Get the name and the path for the tablespace. node->name always + contains the path (which may be absolute for remote tablespaces in + 5.6+). space->name contains the tablespace name in the form + "./database/table.ibd" (in 5.5-) or "database/table" (in 5.6+). For a + multi-node shared tablespace, space->name contains the name of the first + node, but that's irrelevant, since we only need node_name to match them + against filters, and the shared tablespace is always copied regardless + of the filters value. */ + + if (check_if_skip_table(filepath)) { + return(true); + } + + if (filename_matches(filepath, ext_list)) { + fprintf(f, "%s\n", filepath); + if (save_to_list) { + rsync_list.insert(filepath); + } + } + + return(true); +} + + +static +bool +backup_file_vprintf(const char *filename, const char *fmt, va_list ap) +{ + ds_file_t *dstfile = NULL; + MY_STAT stat; /* unused for now */ + char *buf = 0; + int buf_len; + const char *action; + + memset(&stat, 0, sizeof(stat)); + + buf_len = vasprintf(&buf, fmt, ap); + + stat.st_size = buf_len; + stat.st_mtime = my_time(0); + + dstfile = ds_open(ds_data, filename, &stat); + if (dstfile == NULL) { + msg("[%02u] error: " + "cannot open the destination stream for %s\n", + 0, filename); + goto error; + } + + action = xb_get_copy_action("Writing"); + msg_ts("[%02u] %s %s\n", 0, action, filename); + + if (buf_len == -1) { + goto error; + } + + if (ds_write(dstfile, buf, buf_len)) { + goto error; + } + + /* close */ + msg_ts("[%02u] ...done\n", 0); + free(buf); + + if (ds_close(dstfile)) { + goto error_close; + } + + return(true); + +error: + free(buf); + if (dstfile != NULL) { + ds_close(dstfile); + } + +error_close: + msg("[%02u] Error: backup file failed.\n", 0); + return(false); /*ERROR*/ +} + + +bool +backup_file_printf(const char *filename, const char *fmt, ...) +{ + bool result; + va_list ap; + + va_start(ap, fmt); + + result = backup_file_vprintf(filename, fmt, ap); + + va_end(ap); + + return(result); +} + +static +bool +run_data_threads(datadir_iter_t *it, os_thread_func_t func, uint n) +{ + datadir_thread_ctxt_t *data_threads; + uint i, count; + os_ib_mutex_t count_mutex; + bool ret; + + data_threads = (datadir_thread_ctxt_t*) + (ut_malloc(sizeof(datadir_thread_ctxt_t) * n)); + + count_mutex = os_mutex_create(); + count = n; + + for (i = 0; i < n; i++) { + data_threads[i].it = it; + data_threads[i].n_thread = i + 1; + data_threads[i].count = &count; + data_threads[i].count_mutex = count_mutex; + os_thread_create(func, data_threads + i, &data_threads[i].id); + } + + /* Wait for threads to exit */ + while (1) { + os_thread_sleep(100000); + os_mutex_enter(count_mutex); + if (count == 0) { + os_mutex_exit(count_mutex); + break; + } + os_mutex_exit(count_mutex); + } + + os_mutex_free(count_mutex); + + ret = true; + for (i = 0; i < n; i++) { + ret = data_threads[i].ret && ret; + if (!data_threads[i].ret) { + msg("Error: thread %u failed.\n", i); + } + } + + ut_free(data_threads); + + return(ret); +} + + +/************************************************************************ +Copy file for backup/restore. +@return true in case of success. */ +bool +copy_file(ds_ctxt_t *datasink, + const char *src_file_path, + const char *dst_file_path, + uint thread_n) +{ + char dst_name[FN_REFLEN]; + ds_file_t *dstfile = NULL; + datafile_cur_t cursor; + xb_fil_cur_result_t res; + const char *action; + + if (!datafile_open(src_file_path, &cursor, thread_n)) { + goto error_close; + } + + strncpy(dst_name, cursor.rel_path, sizeof(dst_name)); + + dstfile = ds_open(datasink, trim_dotslash(dst_file_path), + &cursor.statinfo); + if (dstfile == NULL) { + msg("[%02u] error: " + "cannot open the destination stream for %s\n", + thread_n, dst_name); + goto error; + } + + action = xb_get_copy_action(); + msg_ts("[%02u] %s %s to %s\n", + thread_n, action, src_file_path, dstfile->path); + + /* The main copy loop */ + while ((res = datafile_read(&cursor)) == XB_FIL_CUR_SUCCESS) { + + if (ds_write(dstfile, cursor.buf, cursor.buf_read)) { + goto error; + } + } + + if (res == XB_FIL_CUR_ERROR) { + goto error; + } + + /* close */ + msg_ts("[%02u] ...done\n", thread_n); + datafile_close(&cursor); + if (ds_close(dstfile)) { + goto error_close; + } + return(true); + +error: + datafile_close(&cursor); + if (dstfile != NULL) { + ds_close(dstfile); + } + +error_close: + msg("[%02u] Error: copy_file() failed.\n", thread_n); + return(false); /*ERROR*/ +} + + +/************************************************************************ +Try to move file by renaming it. If source and destination are on +different devices fall back to copy and unlink. +@return true in case of success. */ +static +bool +move_file(ds_ctxt_t *datasink, + const char *src_file_path, + const char *dst_file_path, + const char *dst_dir, uint thread_n) +{ + char errbuf[MYSYS_STRERROR_SIZE]; + char dst_file_path_abs[FN_REFLEN]; + char dst_dir_abs[FN_REFLEN]; + size_t dirname_length; + + ut_snprintf(dst_file_path_abs, sizeof(dst_file_path_abs), + "%s/%s", dst_dir, dst_file_path); + + dirname_part(dst_dir_abs, dst_file_path_abs, &dirname_length); + + if (!directory_exists(dst_dir_abs, true)) { + return(false); + } + + if (file_exists(dst_file_path_abs)) { + msg("Error: Move file %s to %s failed: Destination " + "file exists\n", + src_file_path, dst_file_path_abs); + return(false); + } + + msg_ts("[%02u] Moving %s to %s\n", + thread_n, src_file_path, dst_file_path_abs); + + if (my_rename(src_file_path, dst_file_path_abs, MYF(0)) != 0) { + if (my_errno == EXDEV) { + bool ret; + ret = copy_file(datasink, src_file_path, + dst_file_path, thread_n); + msg_ts("[%02u] Removing %s\n", thread_n, src_file_path); + if (unlink(src_file_path) != 0) { + my_strerror(errbuf, sizeof(errbuf), errno); + msg("Error: unlink %s failed: %s\n", + src_file_path, + errbuf); + } + return(ret); + } + my_strerror(errbuf, sizeof(errbuf), my_errno); + msg("Can not move file %s to %s: %s\n", + src_file_path, dst_file_path_abs, + errbuf); + return(false); + } + + msg_ts("[%02u] ...done\n", thread_n); + + return(true); +} + + +/************************************************************************ +Read link from .isl file if any and store it in the global map associated +with given tablespace. */ +static +void +read_link_file(const char *ibd_filepath, const char *link_filepath) +{ + char *filepath= NULL; + + FILE *file = fopen(link_filepath, "r+b"); + if (file) { + filepath = static_cast<char*>(malloc(OS_FILE_MAX_PATH)); + + os_file_read_string(file, filepath, OS_FILE_MAX_PATH); + fclose(file); + + if (strlen(filepath)) { + /* Trim whitespace from end of filepath */ + ulint lastch = strlen(filepath) - 1; + while (lastch > 4 && filepath[lastch] <= 0x20) { + filepath[lastch--] = 0x00; + } + srv_normalize_path_for_win(filepath); + } + + tablespace_locations[ibd_filepath] = filepath; + } + free(filepath); +} + + +/************************************************************************ +Return the location of given .ibd if it was previously read +from .isl file. +@return NULL or destination .ibd file path. */ +static +const char * +tablespace_filepath(const char *ibd_filepath) +{ + std::map<std::string, std::string>::iterator it; + + it = tablespace_locations.find(ibd_filepath); + + if (it != tablespace_locations.end()) { + return it->second.c_str(); + } + + return NULL; +} + + +/************************************************************************ +Copy or move file depending on current mode. +@return true in case of success. */ +static +bool +copy_or_move_file(const char *src_file_path, + const char *dst_file_path, + const char *dst_dir, + uint thread_n) +{ + ds_ctxt_t *datasink = ds_data; /* copy to datadir by default */ + char filedir[FN_REFLEN]; + size_t filedir_len; + bool ret; + + /* read the link from .isl file */ + if (ends_with(src_file_path, ".isl")) { + char *ibd_filepath; + + ibd_filepath = strdup(src_file_path); + strcpy(ibd_filepath + strlen(ibd_filepath) - 3, "ibd"); + + read_link_file(ibd_filepath, src_file_path); + + free(ibd_filepath); + } + + /* check if there is .isl file */ + if (ends_with(src_file_path, ".ibd")) { + char *link_filepath; + const char *filepath; + + link_filepath = strdup(src_file_path); + strcpy(link_filepath + strlen(link_filepath) - 3, "isl"); + + read_link_file(src_file_path, link_filepath); + + filepath = tablespace_filepath(src_file_path); + + if (filepath != NULL) { + dirname_part(filedir, filepath, &filedir_len); + + dst_file_path = filepath + filedir_len; + dst_dir = filedir; + + if (!directory_exists(dst_dir, true)) { + ret = false; + goto cleanup; + } + + datasink = ds_create(dst_dir, DS_TYPE_LOCAL); + } + + free(link_filepath); + } + + ret = (xtrabackup_copy_back ? + copy_file(datasink, src_file_path, dst_file_path, thread_n) : + move_file(datasink, src_file_path, dst_file_path, + dst_dir, thread_n)); + +cleanup: + + if (datasink != ds_data) { + ds_destroy(datasink); + } + + return(ret); +} + + + + +bool +backup_files(const char *from, bool prep_mode) +{ + char rsync_tmpfile_name[FN_REFLEN]; + FILE *rsync_tmpfile = NULL; + datadir_iter_t *it; + datadir_node_t node; + bool ret = true; + + if (prep_mode && !opt_rsync) { + return(true); + } + + if (opt_rsync) { + snprintf(rsync_tmpfile_name, sizeof(rsync_tmpfile_name), + "%s/%s%d", opt_mysql_tmpdir, + "xtrabackup_rsyncfiles_pass", + prep_mode ? 1 : 2); + rsync_tmpfile = fopen(rsync_tmpfile_name, "w"); + if (rsync_tmpfile == NULL) { + msg("Error: can't create file %s\n", + rsync_tmpfile_name); + return(false); + } + } + + msg_ts("Starting %s non-InnoDB tables and files\n", + prep_mode ? "prep copy of" : "to backup"); + + datadir_node_init(&node); + it = datadir_iter_new(from); + + while (datadir_iter_next(it, &node)) { + + if (!node.is_empty_dir) { + if (opt_rsync) { + ret = datafile_rsync_backup(node.filepath, + !prep_mode, rsync_tmpfile); + } else { + ret = datafile_copy_backup(node.filepath, 1); + } + if (!ret) { + msg("Failed to copy file %s\n", node.filepath); + goto out; + } + } else if (!prep_mode) { + /* backup fake file into empty directory */ + char path[FN_REFLEN]; + ut_snprintf(path, sizeof(path), + "%s/db.opt", node.filepath); + if (!(ret = backup_file_printf( + trim_dotslash(path), "%s", ""))) { + msg("Failed to create file %s\n", path); + goto out; + } + } + } + + if (opt_rsync) { + std::stringstream cmd; + int err; + + if (buffer_pool_filename && file_exists(buffer_pool_filename)) { + fprintf(rsync_tmpfile, "%s\n", buffer_pool_filename); + rsync_list.insert(buffer_pool_filename); + } + if (file_exists("ib_lru_dump")) { + fprintf(rsync_tmpfile, "%s\n", "ib_lru_dump"); + rsync_list.insert("ib_lru_dump"); + } + + fclose(rsync_tmpfile); + rsync_tmpfile = NULL; + + cmd << "rsync -t . --files-from=" << rsync_tmpfile_name + << " " << xtrabackup_target_dir; + + msg_ts("Starting rsync as: %s\n", cmd.str().c_str()); + if ((err = system(cmd.str().c_str()) && !prep_mode) != 0) { + msg_ts("Error: rsync failed with error code %d\n", err); + ret = false; + goto out; + } + msg_ts("rsync finished successfully.\n"); + + if (!prep_mode && !opt_no_lock) { + char path[FN_REFLEN]; + char dst_path[FN_REFLEN]; + char *newline; + + /* Remove files that have been removed between first and + second passes. Cannot use "rsync --delete" because it + does not work with --files-from. */ + snprintf(rsync_tmpfile_name, sizeof(rsync_tmpfile_name), + "%s/%s", opt_mysql_tmpdir, + "xtrabackup_rsyncfiles_pass1"); + + rsync_tmpfile = fopen(rsync_tmpfile_name, "r"); + if (rsync_tmpfile == NULL) { + msg("Error: can't open file %s\n", + rsync_tmpfile_name); + return(false); + } + + while (fgets(path, sizeof(path), rsync_tmpfile)) { + + newline = strchr(path, '\n'); + if (newline) { + *newline = 0; + } + if (rsync_list.count(path) < 1) { + snprintf(dst_path, sizeof(dst_path), + "%s/%s", xtrabackup_target_dir, + path); + msg_ts("Removing %s\n", dst_path); + unlink(dst_path); + } + } + + fclose(rsync_tmpfile); + rsync_tmpfile = NULL; + } + } + + msg_ts("Finished %s non-InnoDB tables and files\n", + prep_mode ? "a prep copy of" : "backing up"); + +out: + datadir_iter_free(it); + datadir_node_free(&node); + + if (rsync_tmpfile != NULL) { + fclose(rsync_tmpfile); + } + + return(ret); +} + +bool +backup_start() +{ + if (!opt_no_lock) { + if (opt_safe_slave_backup) { + if (!wait_for_safe_slave(mysql_connection)) { + return(false); + } + } + + if (!backup_files(fil_path_to_mysql_datadir, true)) { + return(false); + } + + history_lock_time = time(NULL); + + if (!lock_tables(mysql_connection)) { + return(false); + } + } + + if (!backup_files(fil_path_to_mysql_datadir, false)) { + return(false); + } + + if (!backup_files_from_datadir(fil_path_to_mysql_datadir)) { + return false; + } + + // There is no need to stop slave thread before coping non-Innodb data when + // --no-lock option is used because --no-lock option requires that no DDL or + // DML to non-transaction tables can occur. + if (opt_no_lock) { + if (opt_safe_slave_backup) { + if (!wait_for_safe_slave(mysql_connection)) { + return(false); + } + } + } + + if (opt_slave_info) { + lock_binlog_maybe(mysql_connection); + + if (!write_slave_info(mysql_connection)) { + return(false); + } + } + + /* The only reason why Galera/binlog info is written before + wait_for_ibbackup_log_copy_finish() is that after that call the xtrabackup + binary will start streamig a temporary copy of REDO log to stdout and + thus, any streaming from innobackupex would interfere. The only way to + avoid that is to have a single process, i.e. merge innobackupex and + xtrabackup. */ + if (opt_galera_info) { + if (!write_galera_info(mysql_connection)) { + return(false); + } + write_current_binlog_file(mysql_connection); + } + + if (opt_binlog_info == BINLOG_INFO_ON) { + + lock_binlog_maybe(mysql_connection); + write_binlog_info(mysql_connection); + } + + if (have_flush_engine_logs) { + msg_ts("Executing FLUSH NO_WRITE_TO_BINLOG ENGINE LOGS...\n"); + xb_mysql_query(mysql_connection, + "FLUSH NO_WRITE_TO_BINLOG ENGINE LOGS", false); + } + + return(true); +} + + +bool +backup_finish() +{ + /* release all locks */ + if (!opt_no_lock) { + unlock_all(mysql_connection); + history_lock_time = 0; + } else { + history_lock_time = time(NULL) - history_lock_time; + } + + if (opt_safe_slave_backup && sql_thread_started) { + msg("Starting slave SQL thread\n"); + xb_mysql_query(mysql_connection, + "START SLAVE SQL_THREAD", false); + } + + /* Copy buffer pool dump or LRU dump */ + if (!opt_rsync) { + if (buffer_pool_filename && file_exists(buffer_pool_filename)) { + const char *dst_name; + + dst_name = trim_dotslash(buffer_pool_filename); + copy_file(ds_data, buffer_pool_filename, dst_name, 0); + } + if (file_exists("ib_lru_dump")) { + copy_file(ds_data, "ib_lru_dump", "ib_lru_dump", 0); + } + } + + msg_ts("Backup created in directory '%s'\n", xtrabackup_target_dir); + if (mysql_binlog_position != NULL) { + msg("MySQL binlog position: %s\n", mysql_binlog_position); + } + if (mysql_slave_position && opt_slave_info) { + msg("MySQL slave binlog position: %s\n", + mysql_slave_position); + } + + if (!write_backup_config_file()) { + return(false); + } + + if (!write_xtrabackup_info(mysql_connection)) { + return(false); + } + + + + return(true); +} + +bool +ibx_copy_incremental_over_full() +{ + const char *ext_list[] = {"frm", "isl", "MYD", "MYI", "MAD", "MAI", + "MRG", "TRG", "TRN", "ARM", "ARZ", "CSM", "CSV", "opt", "par", + NULL}; + const char *sup_files[] = {"xtrabackup_binlog_info", + "xtrabackup_galera_info", + "xtrabackup_slave_info", + "xtrabackup_info", + "ib_lru_dump", + NULL}; + datadir_iter_t *it = NULL; + datadir_node_t node; + bool ret = true; + char path[FN_REFLEN]; + int i; + + datadir_node_init(&node); + + /* If we were applying an incremental change set, we need to make + sure non-InnoDB files and xtrabackup_* metainfo files are copied + to the full backup directory. */ + + if (xtrabackup_incremental) { + + ds_data = ds_create(xtrabackup_target_dir, DS_TYPE_LOCAL); + + it = datadir_iter_new(xtrabackup_incremental_dir); + + while (datadir_iter_next(it, &node)) { + + /* copy only non-innodb files */ + + if (node.is_empty_dir + || !filename_matches(node.filepath, ext_list)) { + continue; + } + + if (file_exists(node.filepath_rel)) { + unlink(node.filepath_rel); + } + + if (!(ret = copy_file(ds_data, node.filepath, + node.filepath_rel, 1))) { + msg("Failed to copy file %s\n", + node.filepath); + goto cleanup; + } + } + + /* copy buffer pool dump */ + if (innobase_buffer_pool_filename) { + const char *src_name; + + src_name = trim_dotslash(innobase_buffer_pool_filename); + + snprintf(path, sizeof(path), "%s/%s", + xtrabackup_incremental_dir, + src_name); + + if (file_exists(path)) { + copy_file(ds_data, path, + innobase_buffer_pool_filename, 0); + } + } + + /* copy supplementary files */ + + for (i = 0; sup_files[i]; i++) { + snprintf(path, sizeof(path), "%s/%s", + xtrabackup_incremental_dir, + sup_files[i]); + + if (file_exists(path)) + { + if (file_exists(sup_files[i])) { + unlink(sup_files[i]); + } + copy_file(ds_data, path, sup_files[i], 0); + } + } + + } + +cleanup: + if (it != NULL) { + datadir_iter_free(it); + } + + if (ds_data != NULL) { + ds_destroy(ds_data); + } + + datadir_node_free(&node); + + return(ret); +} + +bool +ibx_cleanup_full_backup() +{ + const char *ext_list[] = {"delta", "meta", "ibd", NULL}; + datadir_iter_t *it = NULL; + datadir_node_t node; + bool ret = true; + + datadir_node_init(&node); + + /* If we are applying an incremental change set, we need to make + sure non-InnoDB files are cleaned up from full backup dir before + we copy files from incremental dir. */ + + it = datadir_iter_new(xtrabackup_target_dir); + + while (datadir_iter_next(it, &node)) { + + if (node.is_empty_dir) { +#ifdef _WIN32 + DeleteFile(node.filepath); +#else + rmdir(node.filepath); +#endif + } + + if (xtrabackup_incremental && !node.is_empty_dir + && !filename_matches(node.filepath, ext_list)) { + unlink(node.filepath); + } + } + + datadir_iter_free(it); + + datadir_node_free(&node); + + return(ret); +} + +bool +apply_log_finish() +{ + if (!ibx_cleanup_full_backup() + || !ibx_copy_incremental_over_full()) { + return(false); + } + + return(true); +} + +extern void +os_io_init_simple(void); + +bool +copy_back() +{ + char *innobase_data_file_path_copy; + ulint i; + bool ret; + datadir_iter_t *it = NULL; + datadir_node_t node; + char *dst_dir; + + memset(&node, 0, sizeof(node)); + + if (!opt_force_non_empty_dirs) { + if (!directory_exists_and_empty(mysql_data_home, + "Original data")) { + return(false); + } + } else { + if (!directory_exists(mysql_data_home, true)) { + return(false); + } + } + if (srv_undo_dir && *srv_undo_dir + && !directory_exists(srv_undo_dir, true)) { + return(false); + } + if (innobase_data_home_dir && *innobase_data_home_dir + && !directory_exists(innobase_data_home_dir, true)) { + return(false); + } + if (srv_log_group_home_dir && *srv_log_group_home_dir + && !directory_exists(srv_log_group_home_dir, true)) { + return(false); + } + + /* cd to backup directory */ + if (my_setwd(xtrabackup_target_dir, MYF(MY_WME))) + { + msg("cannot my_setwd %s\n", xtrabackup_target_dir); + return(false); + } + + /* parse data file path */ + + if (!innobase_data_file_path) { + innobase_data_file_path = (char*) "ibdata1:10M:autoextend"; + } + innobase_data_file_path_copy = strdup(innobase_data_file_path); + + if (!(ret = srv_parse_data_file_paths_and_sizes( + innobase_data_file_path_copy))) { + msg("syntax error in innodb_data_file_path\n"); + return(false); + } + + srv_max_n_threads = 1000; + //os_sync_mutex = NULL; + ut_mem_init(); + /* temporally dummy value to avoid crash */ + srv_page_size_shift = 14; + srv_page_size = (1 << srv_page_size_shift); + os_sync_init(); + sync_init(); + os_io_init_simple(); + mem_init(srv_mem_pool_size); + ut_crc32_init(); + + /* copy undo tablespaces */ + if (srv_undo_tablespaces > 0) { + + dst_dir = (srv_undo_dir && *srv_undo_dir) + ? srv_undo_dir : mysql_data_home; + + ds_data = ds_create(dst_dir, DS_TYPE_LOCAL); + + for (i = 1; i <= srv_undo_tablespaces; i++) { + char filename[20]; + sprintf(filename, "undo%03u", (uint)i); + if (!(ret = copy_or_move_file(filename, filename, + dst_dir, 1))) { + goto cleanup; + } + } + + ds_destroy(ds_data); + ds_data = NULL; + } + + /* copy redo logs */ + + dst_dir = (srv_log_group_home_dir && *srv_log_group_home_dir) + ? srv_log_group_home_dir : mysql_data_home; + + ds_data = ds_create(dst_dir, DS_TYPE_LOCAL); + + for (i = 0; i < (ulong)innobase_log_files_in_group; i++) { + char filename[20]; + sprintf(filename, "ib_logfile%lu", i); + + if (!file_exists(filename)) { + continue; + } + + if (!(ret = copy_or_move_file(filename, filename, + dst_dir, 1))) { + goto cleanup; + } + } + + ds_destroy(ds_data); + ds_data = NULL; + + /* copy innodb system tablespace(s) */ + + dst_dir = (innobase_data_home_dir && *innobase_data_home_dir) + ? innobase_data_home_dir : mysql_data_home; + + ds_data = ds_create(dst_dir, DS_TYPE_LOCAL); + + for (i = 0; i < srv_n_data_files; i++) { + const char *filename = base_name(srv_data_file_names[i]); + + if (!(ret = copy_or_move_file(filename, srv_data_file_names[i], + dst_dir, 1))) { + goto cleanup; + } + } + + ds_destroy(ds_data); + ds_data = NULL; + + /* copy the rest of tablespaces */ + ds_data = ds_create(mysql_data_home, DS_TYPE_LOCAL); + + it = datadir_iter_new(".", false); + + datadir_node_init(&node); + + while (datadir_iter_next(it, &node)) { + const char *ext_list[] = {"backup-my.cnf", "xtrabackup_logfile", + "xtrabackup_binary", "xtrabackup_binlog_info", + "xtrabackup_checkpoints", ".qp", ".pmap", ".tmp", + ".xbcrypt", NULL}; + const char *filename; + char c_tmp; + int i_tmp; + bool is_ibdata_file; + + /* create empty directories */ + if (node.is_empty_dir) { + char path[FN_REFLEN]; + + snprintf(path, sizeof(path), "%s/%s", + mysql_data_home, node.filepath_rel); + + msg_ts("[%02u] Creating directory %s\n", 1, path); + + if (mkdirp(path, 0777, MYF(0)) < 0) { + char errbuf[MYSYS_STRERROR_SIZE]; + my_strerror(errbuf, sizeof(errbuf), my_errno); + msg("Can not create directory %s: %s\n", + path, errbuf); + ret = false; + + goto cleanup; + + } + + msg_ts("[%02u] ...done.", 1); + + continue; + } + + filename = base_name(node.filepath); + + /* skip .qp and .xbcrypt files */ + if (filename_matches(filename, ext_list)) { + continue; + } + + /* skip undo tablespaces */ + if (sscanf(filename, "undo%d%c", &i_tmp, &c_tmp) == 1) { + continue; + } + + /* skip redo logs */ + if (sscanf(filename, "ib_logfile%d%c", &i_tmp, &c_tmp) == 1) { + continue; + } + + /* skip innodb data files */ + is_ibdata_file = false; + for (i = 0; i < srv_n_data_files; i++) { + const char *ibfile; + + ibfile = base_name(srv_data_file_names[i]); + + if (strcmp(ibfile, filename) == 0) { + is_ibdata_file = true; + continue; + } + } + if (is_ibdata_file) { + continue; + } + + if (!(ret = copy_or_move_file(node.filepath, node.filepath_rel, + mysql_data_home, 1))) { + goto cleanup; + } + } + + /* copy buufer pool dump */ + + if (innobase_buffer_pool_filename) { + const char *src_name; + char path[FN_REFLEN]; + + src_name = trim_dotslash(innobase_buffer_pool_filename); + + snprintf(path, sizeof(path), "%s/%s", + mysql_data_home, + src_name); + + /* could be already copied with other files + from data directory */ + if (file_exists(src_name) && + !file_exists(innobase_buffer_pool_filename)) { + copy_or_move_file(src_name, + innobase_buffer_pool_filename, + mysql_data_home, 0); + } + } + +cleanup: + if (it != NULL) { + datadir_iter_free(it); + } + + datadir_node_free(&node); + + free(innobase_data_file_path_copy); + + if (ds_data != NULL) { + ds_destroy(ds_data); + } + + ds_data = NULL; + + //os_sync_free(); + mem_close(); + //os_sync_mutex = NULL; + ut_free_all_mem(); + sync_close(); + sync_initialized = FALSE; + return(ret); +} + +bool +decrypt_decompress_file(const char *filepath, uint thread_n) +{ + std::stringstream cmd, message; + char *dest_filepath = strdup(filepath); + bool needs_action = false; + + cmd << IF_WIN("type ","cat ") << filepath; + + if (ends_with(filepath, ".xbcrypt") && opt_decrypt) { + cmd << " | xbcrypt --decrypt --encrypt-algo=" + << xtrabackup_encrypt_algo_names[opt_decrypt_algo]; + if (xtrabackup_encrypt_key) { + cmd << " --encrypt-key=" << xtrabackup_encrypt_key; + } else { + cmd << " --encrypt-key-file=" + << xtrabackup_encrypt_key_file; + } + dest_filepath[strlen(dest_filepath) - 8] = 0; + message << "decrypting"; + needs_action = true; + } + + if (opt_decompress + && (ends_with(filepath, ".qp") + || (ends_with(filepath, ".qp.xbcrypt") + && opt_decrypt))) { + cmd << " | qpress -dio "; + dest_filepath[strlen(dest_filepath) - 3] = 0; + if (needs_action) { + message << " and "; + } + message << "decompressing"; + needs_action = true; + } + + cmd << " > " << dest_filepath; + message << " " << filepath; + + free(dest_filepath); + + if (needs_action) { + + msg_ts("[%02u] %s\n", thread_n, message.str().c_str()); + + if (system(cmd.str().c_str()) != 0) { + return(false); + } + + if (opt_remove_original) { + msg_ts("[%02u] removing %s\n", thread_n, filepath); + if (my_delete(filepath, MYF(MY_WME)) != 0) { + return(false); + } + } + } + + return(true); +} + +static +os_thread_ret_t STDCALL +decrypt_decompress_thread_func(void *arg) +{ + bool ret = true; + datadir_node_t node; + datadir_thread_ctxt_t *ctxt = (datadir_thread_ctxt_t *)(arg); + + datadir_node_init(&node); + + while (datadir_iter_next(ctxt->it, &node)) { + + /* skip empty directories in backup */ + if (node.is_empty_dir) { + continue; + } + + if (!ends_with(node.filepath, ".qp") + && !ends_with(node.filepath, ".xbcrypt")) { + continue; + } + + if (!(ret = decrypt_decompress_file(node.filepath, + ctxt->n_thread))) { + goto cleanup; + } + } + +cleanup: + + datadir_node_free(&node); + + os_mutex_enter(ctxt->count_mutex); + --(*ctxt->count); + os_mutex_exit(ctxt->count_mutex); + + ctxt->ret = ret; + + os_thread_exit(NULL); + OS_THREAD_DUMMY_RETURN; +} + +bool +decrypt_decompress() +{ + bool ret; + datadir_iter_t *it = NULL; + + srv_max_n_threads = 1000; + //os_sync_mutex = NULL; + ut_mem_init(); + os_sync_init(); + sync_init(); + + /* cd to backup directory */ + if (my_setwd(xtrabackup_target_dir, MYF(MY_WME))) + { + msg("cannot my_setwd %s\n", xtrabackup_target_dir); + return(false); + } + + /* copy the rest of tablespaces */ + ds_data = ds_create(".", DS_TYPE_LOCAL); + + it = datadir_iter_new(".", false); + + ut_a(xtrabackup_parallel >= 0); + + ret = run_data_threads(it, decrypt_decompress_thread_func, + xtrabackup_parallel ? xtrabackup_parallel : 1); + + if (it != NULL) { + datadir_iter_free(it); + } + + if (ds_data != NULL) { + ds_destroy(ds_data); + } + + ds_data = NULL; + + sync_close(); + sync_initialized = FALSE; + //os_sync_free(); + //os_sync_mutex = NULL; + ut_free_all_mem(); + + return(ret); +} + +/* + Copy some files from top level datadir. + Do not copy the Innodb files (ibdata1, redo log files), + as this is done in a separate step. +*/ +static bool backup_files_from_datadir(const char *dir_path) +{ + os_file_dir_t dir = os_file_opendir(dir_path, TRUE); + os_file_stat_t info; + bool ret = true; + while (os_file_readdir_next_file(dir_path, dir, &info) == 0) { + + if (info.type != OS_FILE_TYPE_FILE) + continue; + + const char *pname = strrchr(info.name, IF_WIN('\\', '/')); + if (!pname) + pname = info.name; + + /* Copy aria log files, and aws keys for encryption plugins.*/ + const char *prefixes[] = { "aria_log", "aws-kms-key" }; + for (size_t i = 0; i < array_elements(prefixes); i++) { + if (starts_with(pname, prefixes[i])) { + ret = copy_file(ds_data, info.name, info.name, 1); + if (!ret) { + break; + } + } + } + } + os_file_closedir(dir); + return ret; +} diff --git a/extra/mariabackup/backup_copy.h b/extra/mariabackup/backup_copy.h new file mode 100644 index 00000000000..4b829982764 --- /dev/null +++ b/extra/mariabackup/backup_copy.h @@ -0,0 +1,49 @@ + +#ifndef XTRABACKUP_BACKUP_COPY_H +#define XTRABACKUP_BACKUP_COPY_H + +#include <my_global.h> +#include "datasink.h" + +/* special files */ +#define XTRABACKUP_SLAVE_INFO "xtrabackup_slave_info" +#define XTRABACKUP_GALERA_INFO "xtrabackup_galera_info" +#define XTRABACKUP_BINLOG_INFO "xtrabackup_binlog_info" +#define XTRABACKUP_INFO "xtrabackup_info" + +extern bool binlog_locked; + +bool +backup_file_printf(const char *filename, const char *fmt, ...) + ATTRIBUTE_FORMAT(printf, 2, 0); + +/************************************************************************ +Return true if first and second arguments are the same path. */ +bool +equal_paths(const char *first, const char *second); + +/************************************************************************ +Copy file for backup/restore. +@return true in case of success. */ +bool +copy_file(ds_ctxt_t *datasink, + const char *src_file_path, + const char *dst_file_path, + uint thread_n); + +bool +backup_start(); +bool +backup_finish(); +bool +apply_log_finish(); +bool +copy_back(); +bool +decrypt_decompress(); +bool +is_path_separator(char); +bool +directory_exists(const char *dir, bool create); + +#endif diff --git a/extra/mariabackup/backup_mysql.cc b/extra/mariabackup/backup_mysql.cc new file mode 100644 index 00000000000..6299afffc6e --- /dev/null +++ b/extra/mariabackup/backup_mysql.cc @@ -0,0 +1,1648 @@ +/****************************************************** +hot backup tool for InnoDB +(c) 2009-2015 Percona LLC and/or its affiliates +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +******************************************************* + +This file incorporates work covered by the following copyright and +permission notice: + +Copyright (c) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA + +*******************************************************/ +#define MYSQL_CLIENT + +#include <my_global.h> +#include <mysql.h> +#include <mysqld.h> +#include <my_sys.h> +#include <string.h> +#include <limits> +#include "common.h" +#include "xtrabackup.h" +#include "mysql_version.h" +#include "backup_copy.h" +#include "backup_mysql.h" +#include "mysqld.h" +#include "encryption_plugin.h" +#include <sstream> + + +char *tool_name; +char tool_args[2048]; + +/* mysql flavor and version */ +mysql_flavor_t server_flavor = FLAVOR_UNKNOWN; +unsigned long mysql_server_version = 0; + +/* server capabilities */ +bool have_changed_page_bitmaps = false; +bool have_backup_locks = false; +bool have_backup_safe_binlog_info = false; +bool have_lock_wait_timeout = false; +bool have_galera_enabled = false; +bool have_flush_engine_logs = false; +bool have_multi_threaded_slave = false; +bool have_gtid_slave = false; + +/* Kill long selects */ +os_thread_id_t kill_query_thread_id; +os_event_t kill_query_thread_started; +os_event_t kill_query_thread_stopped; +os_event_t kill_query_thread_stop; + +bool sql_thread_started = false; +char *mysql_slave_position = NULL; +char *mysql_binlog_position = NULL; +char *buffer_pool_filename = NULL; + +/* History on server */ +time_t history_start_time; +time_t history_end_time; +time_t history_lock_time; + +MYSQL *mysql_connection; + +my_bool opt_ssl_verify_server_cert; + +MYSQL * +xb_mysql_connect() +{ + MYSQL *connection = mysql_init(NULL); + char mysql_port_str[std::numeric_limits<int>::digits10 + 3]; + + sprintf(mysql_port_str, "%d", opt_port); + + if (connection == NULL) { + msg("Failed to init MySQL struct: %s.\n", + mysql_error(connection)); + return(NULL); + } + + if (!opt_secure_auth) { + mysql_options(connection, MYSQL_SECURE_AUTH, + (char *) &opt_secure_auth); + } + + msg_ts("Connecting to MySQL server host: %s, user: %s, password: %s, " + "port: %s, socket: %s\n", opt_host ? opt_host : "localhost", + opt_user ? opt_user : "not set", + opt_password ? "set" : "not set", + opt_port != 0 ? mysql_port_str : "not set", + opt_socket ? opt_socket : "not set"); + +#ifdef HAVE_OPENSSL + if (opt_use_ssl) + { + mysql_ssl_set(connection, opt_ssl_key, opt_ssl_cert, + opt_ssl_ca, opt_ssl_capath, + opt_ssl_cipher); + mysql_options(connection, MYSQL_OPT_SSL_CRL, opt_ssl_crl); + mysql_options(connection, MYSQL_OPT_SSL_CRLPATH, + opt_ssl_crlpath); + } + mysql_options(connection,MYSQL_OPT_SSL_VERIFY_SERVER_CERT, + (char*)&opt_ssl_verify_server_cert); +#endif + + if (!mysql_real_connect(connection, + opt_host ? opt_host : "localhost", + opt_user, + opt_password, + "" /*database*/, opt_port, + opt_socket, 0)) { + msg("Failed to connect to MySQL server: %s.\n", + mysql_error(connection)); + mysql_close(connection); + return(NULL); + } + + xb_mysql_query(connection, "SET SESSION wait_timeout=2147483", + false, true); + + return(connection); +} + +/*********************************************************************//** +Execute mysql query. */ +MYSQL_RES * +xb_mysql_query(MYSQL *connection, const char *query, bool use_result, + bool die_on_error) +{ + MYSQL_RES *mysql_result = NULL; + + if (mysql_query(connection, query)) { + msg("Error: failed to execute query %s: %s\n", query, + mysql_error(connection)); + if (die_on_error) { + exit(EXIT_FAILURE); + } + return(NULL); + } + + /* store result set on client if there is a result */ + if (mysql_field_count(connection) > 0) { + if ((mysql_result = mysql_store_result(connection)) == NULL) { + msg("Error: failed to fetch query result %s: %s\n", + query, mysql_error(connection)); + exit(EXIT_FAILURE); + } + + if (!use_result) { + mysql_free_result(mysql_result); + } + } + + return mysql_result; +} + + +struct mysql_variable { + const char *name; + char **value; +}; + + +static +void +read_mysql_variables(MYSQL *connection, const char *query, mysql_variable *vars, + bool vertical_result) +{ + MYSQL_RES *mysql_result; + MYSQL_ROW row; + mysql_variable *var; + + mysql_result = xb_mysql_query(connection, query, true); + + ut_ad(!vertical_result || mysql_num_fields(mysql_result) == 2); + + if (vertical_result) { + while ((row = mysql_fetch_row(mysql_result))) { + char *name = row[0]; + char *value = row[1]; + for (var = vars; var->name; var++) { + if (strcmp(var->name, name) == 0 + && value != NULL) { + *(var->value) = strdup(value); + } + } + } + } else { + MYSQL_FIELD *field; + + if ((row = mysql_fetch_row(mysql_result)) != NULL) { + int i = 0; + while ((field = mysql_fetch_field(mysql_result)) + != NULL) { + char *name = field->name; + char *value = row[i]; + for (var = vars; var->name; var++) { + if (strcmp(var->name, name) == 0 + && value != NULL) { + *(var->value) = strdup(value); + } + } + ++i; + } + } + } + + mysql_free_result(mysql_result); +} + + +static +void +free_mysql_variables(mysql_variable *vars) +{ + mysql_variable *var; + + for (var = vars; var->name; var++) { + free(*(var->value)); + } +} + + +static +char * +read_mysql_one_value(MYSQL *connection, const char *query) +{ + MYSQL_RES *mysql_result; + MYSQL_ROW row; + char *result = NULL; + + mysql_result = xb_mysql_query(connection, query, true); + + ut_ad(mysql_num_fields(mysql_result) == 1); + + if ((row = mysql_fetch_row(mysql_result))) { + result = strdup(row[0]); + } + + mysql_free_result(mysql_result); + + return(result); +} + +static +bool +check_server_version(unsigned long version_number, + const char *version_string, + const char *version_comment, + const char *innodb_version) +{ + bool version_supported = false; + bool mysql51 = false; + + mysql_server_version = version_number; + + server_flavor = FLAVOR_UNKNOWN; + if (strstr(version_comment, "Percona") != NULL) { + server_flavor = FLAVOR_PERCONA_SERVER; + } else if (strstr(version_comment, "MariaDB") != NULL || + strstr(version_string, "MariaDB") != NULL) { + server_flavor = FLAVOR_MARIADB; + } else if (strstr(version_comment, "MySQL") != NULL) { + server_flavor = FLAVOR_MYSQL; + } + + mysql51 = version_number > 50100 && version_number < 50500; + version_supported = version_supported + || (mysql51 && innodb_version != NULL); + version_supported = version_supported + || (version_number > 50500 && version_number < 50700); + version_supported = version_supported + || ((version_number > 100000) + && server_flavor == FLAVOR_MARIADB); + + if (mysql51 && innodb_version == NULL) { + msg("Error: Built-in InnoDB in MySQL 5.1 is not " + "supported in this release. You can either use " + "Percona XtraBackup 2.0, or upgrade to InnoDB " + "plugin.\n"); + } else if (!version_supported) { + msg("Error: Unsupported server version: '%s'. Please " + "report a bug at " + "https://bugs.launchpad.net/percona-xtrabackup\n", + version_string); + } + + return(version_supported); +} + +/*********************************************************************//** +Receive options important for XtraBackup from MySQL server. +@return true on success. */ +bool +get_mysql_vars(MYSQL *connection) +{ + char *gtid_mode_var = NULL; + char *version_var = NULL; + char *version_comment_var = NULL; + char *innodb_version_var = NULL; + char *have_backup_locks_var = NULL; + char *have_backup_safe_binlog_info_var = NULL; + char *log_bin_var = NULL; + char *lock_wait_timeout_var= NULL; + char *wsrep_on_var = NULL; + char *slave_parallel_workers_var = NULL; + char *gtid_slave_pos_var = NULL; + char *innodb_buffer_pool_filename_var = NULL; + char *datadir_var = NULL; + char *innodb_log_group_home_dir_var = NULL; + char *innodb_log_file_size_var = NULL; + char *innodb_log_files_in_group_var = NULL; + char *innodb_data_file_path_var = NULL; + char *innodb_data_home_dir_var = NULL; + char *innodb_undo_directory_var = NULL; + char *innodb_page_size_var = NULL; + + unsigned long server_version = mysql_get_server_version(connection); + + bool ret = true; + + mysql_variable mysql_vars[] = { + {"have_backup_locks", &have_backup_locks_var}, + {"have_backup_safe_binlog_info", + &have_backup_safe_binlog_info_var}, + {"log_bin", &log_bin_var}, + {"lock_wait_timeout", &lock_wait_timeout_var}, + {"gtid_mode", >id_mode_var}, + {"version", &version_var}, + {"version_comment", &version_comment_var}, + {"innodb_version", &innodb_version_var}, + {"wsrep_on", &wsrep_on_var}, + {"slave_parallel_workers", &slave_parallel_workers_var}, + {"gtid_slave_pos", >id_slave_pos_var}, + {"innodb_buffer_pool_filename", + &innodb_buffer_pool_filename_var}, + {"datadir", &datadir_var}, + {"innodb_log_group_home_dir", &innodb_log_group_home_dir_var}, + {"innodb_log_file_size", &innodb_log_file_size_var}, + {"innodb_log_files_in_group", &innodb_log_files_in_group_var}, + {"innodb_data_file_path", &innodb_data_file_path_var}, + {"innodb_data_home_dir", &innodb_data_home_dir_var}, + {"innodb_undo_directory", &innodb_undo_directory_var}, + {"innodb_page_size", &innodb_page_size_var}, + {NULL, NULL} + }; + + read_mysql_variables(connection, "SHOW VARIABLES", + mysql_vars, true); + + if (have_backup_locks_var != NULL && !opt_no_backup_locks) { + have_backup_locks = true; + } + + if (opt_binlog_info == BINLOG_INFO_AUTO) { + + if (have_backup_safe_binlog_info_var != NULL) + opt_binlog_info = BINLOG_INFO_LOCKLESS; + else if (log_bin_var != NULL && !strcmp(log_bin_var, "ON")) + opt_binlog_info = BINLOG_INFO_ON; + else + opt_binlog_info = BINLOG_INFO_OFF; + } + + if (have_backup_safe_binlog_info_var == NULL && + opt_binlog_info == BINLOG_INFO_LOCKLESS) { + + msg("Error: --binlog-info=LOCKLESS is not supported by the " + "server\n"); + return(false); + } + + if (lock_wait_timeout_var != NULL) { + have_lock_wait_timeout = true; + } + + if (wsrep_on_var != NULL) { + have_galera_enabled = true; + } + + /* Check server version compatibility and detect server flavor */ + + if (!(ret = check_server_version(server_version, version_var, + version_comment_var, + innodb_version_var))) { + goto out; + } + + if (server_version > 50500) { + have_flush_engine_logs = true; + } + + if (slave_parallel_workers_var != NULL + && atoi(slave_parallel_workers_var) > 0) { + have_multi_threaded_slave = true; + } + + if (innodb_buffer_pool_filename_var != NULL) { + buffer_pool_filename = strdup(innodb_buffer_pool_filename_var); + } + + if ((gtid_mode_var && strcmp(gtid_mode_var, "ON") == 0) || + (gtid_slave_pos_var && *gtid_slave_pos_var)) { + have_gtid_slave = true; + } + + msg("Using server version %s\n", version_var); + + if (!(ret = detect_mysql_capabilities_for_backup())) { + goto out; + } + + /* make sure datadir value is the same in configuration file */ + if (check_if_param_set("datadir")) { + if (!directory_exists(mysql_data_home, false)) { + msg("Warning: option 'datadir' points to " + "nonexistent directory '%s'\n", mysql_data_home); + } + if (!directory_exists(datadir_var, false)) { + msg("Warning: MySQL variable 'datadir' points to " + "nonexistent directory '%s'\n", datadir_var); + } + if (!equal_paths(mysql_data_home, datadir_var)) { + msg("Warning: option 'datadir' has different " + "values:\n" + " '%s' in defaults file\n" + " '%s' in SHOW VARIABLES\n", + mysql_data_home, datadir_var); + } + } + + /* get some default values is they are missing from my.cnf */ + if (!check_if_param_set("datadir") && datadir_var && *datadir_var) { + strmake(mysql_real_data_home, datadir_var, FN_REFLEN - 1); + mysql_data_home= mysql_real_data_home; + } + + if (!check_if_param_set("innodb_data_file_path") + && innodb_data_file_path_var && *innodb_data_file_path_var) { + innobase_data_file_path = my_strdup( + innodb_data_file_path_var, MYF(MY_FAE)); + } + + if (!check_if_param_set("innodb_data_home_dir") + && innodb_data_home_dir_var && *innodb_data_home_dir_var) { + innobase_data_home_dir = my_strdup( + innodb_data_home_dir_var, MYF(MY_FAE)); + } + + if (!check_if_param_set("innodb_log_group_home_dir") + && innodb_log_group_home_dir_var + && *innodb_log_group_home_dir_var) { + srv_log_group_home_dir = my_strdup( + innodb_log_group_home_dir_var, MYF(MY_FAE)); + } + + if (!check_if_param_set("innodb_undo_directory") + && innodb_undo_directory_var && *innodb_undo_directory_var) { + srv_undo_dir = my_strdup( + innodb_undo_directory_var, MYF(MY_FAE)); + } + + if (!check_if_param_set("innodb_log_files_in_group") + && innodb_log_files_in_group_var) { + char *endptr; + + innobase_log_files_in_group = strtol( + innodb_log_files_in_group_var, &endptr, 10); + ut_ad(*endptr == 0); + } + + if (!check_if_param_set("innodb_log_file_size") + && innodb_log_file_size_var) { + char *endptr; + + innobase_log_file_size = strtoll( + innodb_log_file_size_var, &endptr, 10); + ut_ad(*endptr == 0); + } + + if (!check_if_param_set("innodb_page_size") && innodb_page_size_var) { + char *endptr; + + innobase_page_size = strtoll( + innodb_page_size_var, &endptr, 10); + ut_ad(*endptr == 0); + } + +out: + free_mysql_variables(mysql_vars); + + return(ret); +} + +/*********************************************************************//** +Query the server to find out what backup capabilities it supports. +@return true on success. */ +bool +detect_mysql_capabilities_for_backup() +{ + const char *query = "SELECT 'INNODB_CHANGED_PAGES', COUNT(*) FROM " + "INFORMATION_SCHEMA.PLUGINS " + "WHERE PLUGIN_NAME LIKE 'INNODB_CHANGED_PAGES'"; + char *innodb_changed_pages = NULL; + mysql_variable vars[] = { + {"INNODB_CHANGED_PAGES", &innodb_changed_pages}, {NULL, NULL}}; + + if (xtrabackup_incremental) { + + read_mysql_variables(mysql_connection, query, vars, true); + + ut_ad(innodb_changed_pages != NULL); + + have_changed_page_bitmaps = (atoi(innodb_changed_pages) == 1); + + /* INNODB_CHANGED_PAGES are listed in + INFORMATION_SCHEMA.PLUGINS in MariaDB, but + FLUSH NO_WRITE_TO_BINLOG CHANGED_PAGE_BITMAPS + is not supported for versions below 10.1.6 + (see MDEV-7472) */ + if (server_flavor == FLAVOR_MARIADB && + mysql_server_version < 100106) { + have_changed_page_bitmaps = false; + } + + free_mysql_variables(vars); + } + + /* do some sanity checks */ + if (opt_galera_info && !have_galera_enabled) { + msg("--galera-info is specified on the command " + "line, but the server does not support Galera " + "replication. Ignoring the option.\n"); + opt_galera_info = false; + } + + if (opt_slave_info && have_multi_threaded_slave && + !have_gtid_slave) { + msg("The --slave-info option requires GTID enabled for a " + "multi-threaded slave.\n"); + return(false); + } + + return(true); +} + +static +bool +select_incremental_lsn_from_history(lsn_t *incremental_lsn) +{ + MYSQL_RES *mysql_result; + MYSQL_ROW row; + char query[1000]; + char buf[100]; + + if (opt_incremental_history_name) { + mysql_real_escape_string(mysql_connection, buf, + opt_incremental_history_name, + (unsigned long)strlen(opt_incremental_history_name)); + ut_snprintf(query, sizeof(query), + "SELECT innodb_to_lsn " + "FROM PERCONA_SCHEMA.xtrabackup_history " + "WHERE name = '%s' " + "AND innodb_to_lsn IS NOT NULL " + "ORDER BY innodb_to_lsn DESC LIMIT 1", + buf); + } + + if (opt_incremental_history_uuid) { + mysql_real_escape_string(mysql_connection, buf, + opt_incremental_history_uuid, + (unsigned long)strlen(opt_incremental_history_uuid)); + ut_snprintf(query, sizeof(query), + "SELECT innodb_to_lsn " + "FROM PERCONA_SCHEMA.xtrabackup_history " + "WHERE uuid = '%s' " + "AND innodb_to_lsn IS NOT NULL " + "ORDER BY innodb_to_lsn DESC LIMIT 1", + buf); + } + + mysql_result = xb_mysql_query(mysql_connection, query, true); + + ut_ad(mysql_num_fields(mysql_result) == 1); + if (!(row = mysql_fetch_row(mysql_result))) { + msg("Error while attempting to find history record " + "for %s %s\n", + opt_incremental_history_uuid ? "uuid" : "name", + opt_incremental_history_uuid ? + opt_incremental_history_uuid : + opt_incremental_history_name); + return(false); + } + + *incremental_lsn = strtoull(row[0], NULL, 10); + + mysql_free_result(mysql_result); + + msg("Found and using lsn: " LSN_PF " for %s %s\n", *incremental_lsn, + opt_incremental_history_uuid ? "uuid" : "name", + opt_incremental_history_uuid ? + opt_incremental_history_uuid : + opt_incremental_history_name); + + return(true); +} + +static +const char * +eat_sql_whitespace(const char *query) +{ + bool comment = false; + + while (*query) { + if (comment) { + if (query[0] == '*' && query[1] == '/') { + query += 2; + comment = false; + continue; + } + ++query; + continue; + } + if (query[0] == '/' && query[1] == '*') { + query += 2; + comment = true; + continue; + } + if (strchr("\t\n\r (", query[0])) { + ++query; + continue; + } + break; + } + + return(query); +} + +static +bool +is_query_from_list(const char *query, const char **list) +{ + const char **item; + + query = eat_sql_whitespace(query); + + item = list; + while (*item) { + if (strncasecmp(query, *item, strlen(*item)) == 0) { + return(true); + } + ++item; + } + + return(false); +} + +static +bool +is_query(const char *query) +{ + const char *query_list[] = {"insert", "update", "delete", "replace", + "alter", "load", "select", "do", "handler", "call", "execute", + "begin", NULL}; + + return is_query_from_list(query, query_list); +} + +static +bool +is_select_query(const char *query) +{ + const char *query_list[] = {"select", NULL}; + + return is_query_from_list(query, query_list); +} + +static +bool +is_update_query(const char *query) +{ + const char *query_list[] = {"insert", "update", "delete", "replace", + "alter", "load", NULL}; + + return is_query_from_list(query, query_list); +} + +static +bool +have_queries_to_wait_for(MYSQL *connection, uint threshold) +{ + MYSQL_RES *result; + MYSQL_ROW row; + bool all_queries; + + result = xb_mysql_query(connection, "SHOW FULL PROCESSLIST", true); + + all_queries = (opt_lock_wait_query_type == QUERY_TYPE_ALL); + while ((row = mysql_fetch_row(result)) != NULL) { + const char *info = row[7]; + int duration = atoi(row[5]); + char *id = row[0]; + + if (info != NULL + && duration >= (int)threshold + && ((all_queries && is_query(info)) + || is_update_query(info))) { + msg_ts("Waiting for query %s (duration %d sec): %s", + id, duration, info); + return(true); + } + } + + return(false); +} + +static +void +kill_long_queries(MYSQL *connection, time_t timeout) +{ + MYSQL_RES *result; + MYSQL_ROW row; + bool all_queries; + char kill_stmt[100]; + + result = xb_mysql_query(connection, "SHOW FULL PROCESSLIST", true); + + all_queries = (opt_kill_long_query_type == QUERY_TYPE_ALL); + while ((row = mysql_fetch_row(result)) != NULL) { + const char *info = row[7]; + long long duration = atoll(row[5]); + char *id = row[0]; + + if (info != NULL && + (time_t)duration >= timeout && + ((all_queries && is_query(info)) || + is_select_query(info))) { + msg_ts("Killing query %s (duration %d sec): %s\n", + id, (int)duration, info); + ut_snprintf(kill_stmt, sizeof(kill_stmt), + "KILL %s", id); + xb_mysql_query(connection, kill_stmt, false, false); + } + } +} + +static +bool +wait_for_no_updates(MYSQL *connection, uint timeout, uint threshold) +{ + time_t start_time; + + start_time = time(NULL); + + msg_ts("Waiting %u seconds for queries running longer than %u seconds " + "to finish\n", timeout, threshold); + + while (time(NULL) <= (time_t)(start_time + timeout)) { + if (!have_queries_to_wait_for(connection, threshold)) { + return(true); + } + os_thread_sleep(1000000); + } + + msg_ts("Unable to obtain lock. Please try again later."); + + return(false); +} + +static +os_thread_ret_t +kill_query_thread( +/*===============*/ + void *arg __attribute__((unused))) +{ + MYSQL *mysql; + time_t start_time; + + start_time = time(NULL); + + os_event_set(kill_query_thread_started); + + msg_ts("Kill query timeout %d seconds.\n", + opt_kill_long_queries_timeout); + + while (time(NULL) - start_time < + (time_t)opt_kill_long_queries_timeout) { + if (os_event_wait_time(kill_query_thread_stop, 1000) != + OS_SYNC_TIME_EXCEEDED) { + goto stop_thread; + } + } + + if ((mysql = xb_mysql_connect()) == NULL) { + msg("Error: kill query thread failed\n"); + goto stop_thread; + } + + while (true) { + kill_long_queries(mysql, time(NULL) - start_time); + if (os_event_wait_time(kill_query_thread_stop, 1000) != + OS_SYNC_TIME_EXCEEDED) { + break; + } + } + + mysql_close(mysql); + +stop_thread: + msg_ts("Kill query thread stopped\n"); + + os_event_set(kill_query_thread_stopped); + + os_thread_exit(NULL); + OS_THREAD_DUMMY_RETURN; +} + + +static +void +start_query_killer() +{ + kill_query_thread_stop = os_event_create(); + kill_query_thread_started = os_event_create(); + kill_query_thread_stopped = os_event_create(); + + os_thread_create(kill_query_thread, NULL, &kill_query_thread_id); + + os_event_wait(kill_query_thread_started); +} + +static +void +stop_query_killer() +{ + os_event_set(kill_query_thread_stop); + os_event_wait_time(kill_query_thread_stopped, 60000); +} + +/*********************************************************************//** +Function acquires either a backup tables lock, if supported +by the server, or a global read lock (FLUSH TABLES WITH READ LOCK) +otherwise. +@returns true if lock acquired */ +bool +lock_tables(MYSQL *connection) +{ + if (have_lock_wait_timeout) { + /* Set the maximum supported session value for + lock_wait_timeout to prevent unnecessary timeouts when the + global value is changed from the default */ + xb_mysql_query(connection, + "SET SESSION lock_wait_timeout=31536000", false); + } + + if (have_backup_locks) { + msg_ts("Executing LOCK TABLES FOR BACKUP...\n"); + xb_mysql_query(connection, "LOCK TABLES FOR BACKUP", false); + return(true); + } + + if (!opt_lock_wait_timeout && !opt_kill_long_queries_timeout) { + + /* We do first a FLUSH TABLES. If a long update is running, the + FLUSH TABLES will wait but will not stall the whole mysqld, and + when the long update is done the FLUSH TABLES WITH READ LOCK + will start and succeed quickly. So, FLUSH TABLES is to lower + the probability of a stage where both mysqldump and most client + connections are stalled. Of course, if a second long update + starts between the two FLUSHes, we have that bad stall. + + Option lock_wait_timeout serve the same purpose and is not + compatible with this trick. + */ + + msg_ts("Executing FLUSH NO_WRITE_TO_BINLOG TABLES...\n"); + + xb_mysql_query(connection, + "FLUSH NO_WRITE_TO_BINLOG TABLES", false); + } + + if (opt_lock_wait_timeout) { + if (!wait_for_no_updates(connection, opt_lock_wait_timeout, + opt_lock_wait_threshold)) { + return(false); + } + } + + msg_ts("Executing FLUSH TABLES WITH READ LOCK...\n"); + + if (opt_kill_long_queries_timeout) { + start_query_killer(); + } + + if (have_galera_enabled) { + xb_mysql_query(connection, + "SET SESSION wsrep_causal_reads=0", false); + } + + xb_mysql_query(connection, "FLUSH TABLES WITH READ LOCK", false); + + if (opt_kill_long_queries_timeout) { + stop_query_killer(); + } + + return(true); +} + + +/*********************************************************************//** +If backup locks are used, execute LOCK BINLOG FOR BACKUP provided that we are +not in the --no-lock mode and the lock has not been acquired already. +@returns true if lock acquired */ +bool +lock_binlog_maybe(MYSQL *connection) +{ + if (have_backup_locks && !opt_no_lock && !binlog_locked) { + msg_ts("Executing LOCK BINLOG FOR BACKUP...\n"); + xb_mysql_query(connection, "LOCK BINLOG FOR BACKUP", false); + binlog_locked = true; + + return(true); + } + + return(false); +} + + +/*********************************************************************//** +Releases either global read lock acquired with FTWRL and the binlog +lock acquired with LOCK BINLOG FOR BACKUP, depending on +the locking strategy being used */ +void +unlock_all(MYSQL *connection) +{ + if (opt_debug_sleep_before_unlock) { + msg_ts("Debug sleep for %u seconds\n", + opt_debug_sleep_before_unlock); + os_thread_sleep(opt_debug_sleep_before_unlock * 1000); + } + + if (binlog_locked) { + msg_ts("Executing UNLOCK BINLOG\n"); + xb_mysql_query(connection, "UNLOCK BINLOG", false); + } + + msg_ts("Executing UNLOCK TABLES\n"); + xb_mysql_query(connection, "UNLOCK TABLES", false); + + msg_ts("All tables unlocked\n"); +} + + +static +int +get_open_temp_tables(MYSQL *connection) +{ + char *slave_open_temp_tables = NULL; + mysql_variable status[] = { + {"Slave_open_temp_tables", &slave_open_temp_tables}, + {NULL, NULL} + }; + int result = false; + + read_mysql_variables(connection, + "SHOW STATUS LIKE 'slave_open_temp_tables'", status, true); + + result = slave_open_temp_tables ? atoi(slave_open_temp_tables) : 0; + + free_mysql_variables(status); + + return(result); +} + +/*********************************************************************//** +Wait until it's safe to backup a slave. Returns immediately if +the host isn't a slave. Currently there's only one check: +Slave_open_temp_tables has to be zero. Dies on timeout. */ +bool +wait_for_safe_slave(MYSQL *connection) +{ + char *read_master_log_pos = NULL; + char *slave_sql_running = NULL; + int n_attempts = 1; + const int sleep_time = 3; + int open_temp_tables = 0; + bool result = true; + + mysql_variable status[] = { + {"Read_Master_Log_Pos", &read_master_log_pos}, + {"Slave_SQL_Running", &slave_sql_running}, + {NULL, NULL} + }; + + sql_thread_started = false; + + read_mysql_variables(connection, "SHOW SLAVE STATUS", status, false); + + if (!(read_master_log_pos && slave_sql_running)) { + msg("Not checking slave open temp tables for " + "--safe-slave-backup because host is not a slave\n"); + goto cleanup; + } + + if (strcmp(slave_sql_running, "Yes") == 0) { + sql_thread_started = true; + xb_mysql_query(connection, "STOP SLAVE SQL_THREAD", false); + } + + if (opt_safe_slave_backup_timeout > 0) { + n_attempts = opt_safe_slave_backup_timeout / sleep_time; + } + + open_temp_tables = get_open_temp_tables(connection); + msg_ts("Slave open temp tables: %d\n", open_temp_tables); + + while (open_temp_tables && n_attempts--) { + msg_ts("Starting slave SQL thread, waiting %d seconds, then " + "checking Slave_open_temp_tables again (%d attempts " + "remaining)...\n", sleep_time, n_attempts); + + xb_mysql_query(connection, "START SLAVE SQL_THREAD", false); + os_thread_sleep(sleep_time * 1000000); + xb_mysql_query(connection, "STOP SLAVE SQL_THREAD", false); + + open_temp_tables = get_open_temp_tables(connection); + msg_ts("Slave open temp tables: %d\n", open_temp_tables); + } + + /* Restart the slave if it was running at start */ + if (open_temp_tables == 0) { + msg_ts("Slave is safe to backup\n"); + goto cleanup; + } + + result = false; + + if (sql_thread_started) { + msg_ts("Restarting slave SQL thread.\n"); + xb_mysql_query(connection, "START SLAVE SQL_THREAD", false); + } + + msg_ts("Slave_open_temp_tables did not become zero after " + "%d seconds\n", opt_safe_slave_backup_timeout); + +cleanup: + free_mysql_variables(status); + + return(result); +} + + +/*********************************************************************//** +Retrieves MySQL binlog position of the master server in a replication +setup and saves it in a file. It also saves it in mysql_slave_position +variable. */ +bool +write_slave_info(MYSQL *connection) +{ + char *master = NULL; + char *filename = NULL; + char *gtid_executed = NULL; + char *position = NULL; + char *gtid_slave_pos = NULL; + char *ptr; + bool result = false; + + mysql_variable status[] = { + {"Master_Host", &master}, + {"Relay_Master_Log_File", &filename}, + {"Exec_Master_Log_Pos", &position}, + {"Executed_Gtid_Set", >id_executed}, + {NULL, NULL} + }; + + mysql_variable variables[] = { + {"gtid_slave_pos", >id_slave_pos}, + {NULL, NULL} + }; + + read_mysql_variables(connection, "SHOW SLAVE STATUS", status, false); + read_mysql_variables(connection, "SHOW VARIABLES", variables, true); + + if (master == NULL || filename == NULL || position == NULL) { + msg("Failed to get master binlog coordinates " + "from SHOW SLAVE STATUS\n"); + msg("This means that the server is not a " + "replication slave. Ignoring the --slave-info " + "option\n"); + /* we still want to continue the backup */ + result = true; + goto cleanup; + } + + /* Print slave status to a file. + If GTID mode is used, construct a CHANGE MASTER statement with + MASTER_AUTO_POSITION and correct a gtid_purged value. */ + if (gtid_executed != NULL && *gtid_executed) { + /* MySQL >= 5.6 with GTID enabled */ + + for (ptr = strchr(gtid_executed, '\n'); + ptr; + ptr = strchr(ptr, '\n')) { + *ptr = ' '; + } + + result = backup_file_printf(XTRABACKUP_SLAVE_INFO, + "SET GLOBAL gtid_purged='%s';\n" + "CHANGE MASTER TO MASTER_AUTO_POSITION=1\n", + gtid_executed); + + ut_a(asprintf(&mysql_slave_position, + "master host '%s', purge list '%s'", + master, gtid_executed) != -1); + } else if (gtid_slave_pos && *gtid_slave_pos) { + /* MariaDB >= 10.0 with GTID enabled */ + result = backup_file_printf(XTRABACKUP_SLAVE_INFO, + "SET GLOBAL gtid_slave_pos = '%s';\n" + "CHANGE MASTER TO master_use_gtid = slave_pos\n", + gtid_slave_pos); + ut_a(asprintf(&mysql_slave_position, + "master host '%s', gtid_slave_pos %s", + master, gtid_slave_pos) != -1); + } else { + result = backup_file_printf(XTRABACKUP_SLAVE_INFO, + "CHANGE MASTER TO MASTER_LOG_FILE='%s', " + "MASTER_LOG_POS=%s\n", filename, position); + ut_a(asprintf(&mysql_slave_position, + "master host '%s', filename '%s', position '%s'", + master, filename, position) != -1); + } + +cleanup: + free_mysql_variables(status); + free_mysql_variables(variables); + + return(result); +} + + +/*********************************************************************//** +Retrieves MySQL Galera and +saves it in a file. It also prints it to stdout. */ +bool +write_galera_info(MYSQL *connection) +{ + char *state_uuid = NULL, *state_uuid55 = NULL; + char *last_committed = NULL, *last_committed55 = NULL; + bool result; + + mysql_variable status[] = { + {"Wsrep_local_state_uuid", &state_uuid}, + {"wsrep_local_state_uuid", &state_uuid55}, + {"Wsrep_last_committed", &last_committed}, + {"wsrep_last_committed", &last_committed55}, + {NULL, NULL} + }; + + /* When backup locks are supported by the server, we should skip + creating xtrabackup_galera_info file on the backup stage, because + wsrep_local_state_uuid and wsrep_last_committed will be inconsistent + without blocking commits. The state file will be created on the prepare + stage using the WSREP recovery procedure. */ + if (have_backup_locks) { + return(true); + } + + read_mysql_variables(connection, "SHOW STATUS", status, true); + + if ((state_uuid == NULL && state_uuid55 == NULL) + || (last_committed == NULL && last_committed55 == NULL)) { + msg("Failed to get master wsrep state from SHOW STATUS.\n"); + result = false; + goto cleanup; + } + + result = backup_file_printf(XTRABACKUP_GALERA_INFO, + "%s:%s\n", state_uuid ? state_uuid : state_uuid55, + last_committed ? last_committed : last_committed55); + +cleanup: + free_mysql_variables(status); + + return(result); +} + + +/*********************************************************************//** +Flush and copy the current binary log file into the backup, +if GTID is enabled */ +bool +write_current_binlog_file(MYSQL *connection) +{ + char *executed_gtid_set = NULL; + char *gtid_binlog_state = NULL; + char *log_bin_file = NULL; + char *log_bin_dir = NULL; + bool gtid_exists; + bool result = true; + char filepath[FN_REFLEN]; + + mysql_variable status[] = { + {"Executed_Gtid_Set", &executed_gtid_set}, + {NULL, NULL} + }; + + mysql_variable status_after_flush[] = { + {"File", &log_bin_file}, + {NULL, NULL} + }; + + mysql_variable vars[] = { + {"gtid_binlog_state", >id_binlog_state}, + {"log_bin_basename", &log_bin_dir}, + {NULL, NULL} + }; + + read_mysql_variables(connection, "SHOW MASTER STATUS", status, false); + read_mysql_variables(connection, "SHOW VARIABLES", vars, true); + + gtid_exists = (executed_gtid_set && *executed_gtid_set) + || (gtid_binlog_state && *gtid_binlog_state); + + if (gtid_exists) { + size_t log_bin_dir_length; + + lock_binlog_maybe(connection); + + xb_mysql_query(connection, "FLUSH BINARY LOGS", false); + + read_mysql_variables(connection, "SHOW MASTER STATUS", + status_after_flush, false); + + if (opt_log_bin != NULL && strchr(opt_log_bin, FN_LIBCHAR)) { + /* If log_bin is set, it has priority */ + if (log_bin_dir) { + free(log_bin_dir); + } + log_bin_dir = strdup(opt_log_bin); + } else if (log_bin_dir == NULL) { + /* Default location is MySQL datadir */ + log_bin_dir = strdup("./"); + } + + dirname_part(log_bin_dir, log_bin_dir, &log_bin_dir_length); + + /* strip final slash if it is not the only path component */ + if (log_bin_dir_length > 1 && + log_bin_dir[log_bin_dir_length - 1] == FN_LIBCHAR) { + log_bin_dir[log_bin_dir_length - 1] = 0; + } + + if (log_bin_dir == NULL || log_bin_file == NULL) { + msg("Failed to get master binlog coordinates from " + "SHOW MASTER STATUS"); + result = false; + goto cleanup; + } + + ut_snprintf(filepath, sizeof(filepath), "%s%c%s", + log_bin_dir, FN_LIBCHAR, log_bin_file); + result = copy_file(ds_data, filepath, log_bin_file, 0); + } + +cleanup: + free_mysql_variables(status_after_flush); + free_mysql_variables(status); + free_mysql_variables(vars); + + return(result); +} + + +/*********************************************************************//** +Retrieves MySQL binlog position and +saves it in a file. It also prints it to stdout. */ +bool +write_binlog_info(MYSQL *connection) +{ + char *filename = NULL; + char *position = NULL; + char *gtid_mode = NULL; + char *gtid_current_pos = NULL; + char *gtid_executed = NULL; + char *gtid = NULL; + bool result; + bool mysql_gtid; + bool mariadb_gtid; + + mysql_variable status[] = { + {"File", &filename}, + {"Position", &position}, + {"Executed_Gtid_Set", >id_executed}, + {NULL, NULL} + }; + + mysql_variable vars[] = { + {"gtid_mode", >id_mode}, + {"gtid_current_pos", >id_current_pos}, + {NULL, NULL} + }; + + read_mysql_variables(connection, "SHOW MASTER STATUS", status, false); + read_mysql_variables(connection, "SHOW VARIABLES", vars, true); + + if (filename == NULL || position == NULL) { + /* Do not create xtrabackup_binlog_info if binary + log is disabled */ + result = true; + goto cleanup; + } + + mysql_gtid = ((gtid_mode != NULL) && (strcmp(gtid_mode, "ON") == 0)); + mariadb_gtid = (gtid_current_pos != NULL); + + gtid = (gtid_executed != NULL ? gtid_executed : gtid_current_pos); + + if (mariadb_gtid || mysql_gtid) { + ut_a(asprintf(&mysql_binlog_position, + "filename '%s', position '%s', " + "GTID of the last change '%s'", + filename, position, gtid) != -1); + result = backup_file_printf(XTRABACKUP_BINLOG_INFO, + "%s\t%s\t%s\n", filename, position, + gtid); + } else { + ut_a(asprintf(&mysql_binlog_position, + "filename '%s', position '%s'", + filename, position) != -1); + result = backup_file_printf(XTRABACKUP_BINLOG_INFO, + "%s\t%s\n", filename, position); + } + +cleanup: + free_mysql_variables(status); + free_mysql_variables(vars); + + return(result); +} + +static string escape_and_quote(MYSQL *mysql,const char *str) +{ + if (!str) + return "NULL"; + size_t len = strlen(str); + char* escaped = (char *)alloca(2 * len + 3); + escaped[0] = '\''; + size_t new_len = mysql_real_escape_string(mysql, escaped+1, str, len); + escaped[new_len + 1] = '\''; + escaped[new_len + 2] = 0; + return string(escaped); +} + +/*********************************************************************//** +Writes xtrabackup_info file and if backup_history is enable creates +PERCONA_SCHEMA.xtrabackup_history and writes a new history record to the +table containing all the history info particular to the just completed +backup. */ +bool +write_xtrabackup_info(MYSQL *connection) +{ + + char *uuid = NULL; + char *server_version = NULL; + char buf_start_time[100]; + char buf_end_time[100]; + tm tm; + ostringstream oss; + const char *xb_stream_name[] = {"file", "tar", "xbstream"}; + + + ut_ad(xtrabackup_stream_fmt < 3); + + uuid = read_mysql_one_value(connection, "SELECT UUID()"); + server_version = read_mysql_one_value(connection, "SELECT VERSION()"); + localtime_r(&history_start_time, &tm); + strftime(buf_start_time, sizeof(buf_start_time), + "%Y-%m-%d %H:%M:%S", &tm); + history_end_time = time(NULL); + localtime_r(&history_end_time, &tm); + strftime(buf_end_time, sizeof(buf_end_time), + "%Y-%m-%d %H:%M:%S", &tm); + bool is_partial = (xtrabackup_tables + || xtrabackup_tables_file + || xtrabackup_databases + || xtrabackup_databases_file + || xtrabackup_tables_exclude + || xtrabackup_databases_exclude + ); + + backup_file_printf(XTRABACKUP_INFO, + "uuid = %s\n" + "name = %s\n" + "tool_name = %s\n" + "tool_command = %s\n" + "tool_version = %s\n" + "ibbackup_version = %s\n" + "server_version = %s\n" + "start_time = %s\n" + "end_time = %s\n" + "lock_time = %d\n" + "binlog_pos = %s\n" + "innodb_from_lsn = %llu\n" + "innodb_to_lsn = %llu\n" + "partial = %s\n" + "incremental = %s\n" + "format = %s\n" + "compact = %s\n" + "compressed = %s\n" + "encrypted = %s\n", + uuid, /* uuid */ + opt_history ? opt_history : "", /* name */ + tool_name, /* tool_name */ + tool_args, /* tool_command */ + MYSQL_SERVER_VERSION, /* tool_version */ + MYSQL_SERVER_VERSION, /* ibbackup_version */ + server_version, /* server_version */ + buf_start_time, /* start_time */ + buf_end_time, /* end_time */ + (int)history_lock_time, /* lock_time */ + mysql_binlog_position ? + mysql_binlog_position : "", /* binlog_pos */ + incremental_lsn, /* innodb_from_lsn */ + metadata_to_lsn, /* innodb_to_lsn */ + is_partial? "Y" : "N", + xtrabackup_incremental ? "Y" : "N", /* incremental */ + xb_stream_name[xtrabackup_stream_fmt], /* format */ + "N", /* compact */ + xtrabackup_compress ? "compressed" : "N", /* compressed */ + xtrabackup_encrypt ? "Y" : "N"); /* encrypted */ + + if (!opt_history) { + goto cleanup; + } + + xb_mysql_query(connection, + "CREATE DATABASE IF NOT EXISTS PERCONA_SCHEMA", false); + xb_mysql_query(connection, + "CREATE TABLE IF NOT EXISTS PERCONA_SCHEMA.xtrabackup_history(" + "uuid VARCHAR(40) NOT NULL PRIMARY KEY," + "name VARCHAR(255) DEFAULT NULL," + "tool_name VARCHAR(255) DEFAULT NULL," + "tool_command TEXT DEFAULT NULL," + "tool_version VARCHAR(255) DEFAULT NULL," + "ibbackup_version VARCHAR(255) DEFAULT NULL," + "server_version VARCHAR(255) DEFAULT NULL," + "start_time TIMESTAMP NULL DEFAULT NULL," + "end_time TIMESTAMP NULL DEFAULT NULL," + "lock_time BIGINT UNSIGNED DEFAULT NULL," + "binlog_pos VARCHAR(128) DEFAULT NULL," + "innodb_from_lsn BIGINT UNSIGNED DEFAULT NULL," + "innodb_to_lsn BIGINT UNSIGNED DEFAULT NULL," + "partial ENUM('Y', 'N') DEFAULT NULL," + "incremental ENUM('Y', 'N') DEFAULT NULL," + "format ENUM('file', 'tar', 'xbstream') DEFAULT NULL," + "compact ENUM('Y', 'N') DEFAULT NULL," + "compressed ENUM('Y', 'N') DEFAULT NULL," + "encrypted ENUM('Y', 'N') DEFAULT NULL" + ") CHARACTER SET utf8 ENGINE=innodb", false); + + +#define ESCAPE_BOOL(expr) ((expr)?"'Y'":"'N'") + + oss << "insert into PERCONA_SCHEMA.xtrabackup_history(" + << "uuid, name, tool_name, tool_command, tool_version," + << "ibbackup_version, server_version, start_time, end_time," + << "lock_time, binlog_pos, innodb_from_lsn, innodb_to_lsn," + << "partial, incremental, format, compact, compressed, " + << "encrypted) values(" + << escape_and_quote(connection, uuid) << "," + << escape_and_quote(connection, opt_history) << "," + << escape_and_quote(connection, tool_name) << "," + << escape_and_quote(connection, tool_args) << "," + << escape_and_quote(connection, MYSQL_SERVER_VERSION) << "," + << escape_and_quote(connection, MYSQL_SERVER_VERSION) << "," + << escape_and_quote(connection, server_version) << "," + << "from_unixtime(" << history_start_time << ")," + << "from_unixtime(" << history_end_time << ")," + << history_lock_time << "," + << escape_and_quote(connection, mysql_binlog_position) << "," + << incremental_lsn << "," + << metadata_to_lsn << "," + << ESCAPE_BOOL(is_partial) << "," + << ESCAPE_BOOL(xtrabackup_incremental)<< "," + << escape_and_quote(connection,xb_stream_name[xtrabackup_stream_fmt]) <<"," + << ESCAPE_BOOL(false) << "," + << ESCAPE_BOOL(xtrabackup_compress) << "," + << ESCAPE_BOOL(xtrabackup_encrypt) <<")"; + + xb_mysql_query(mysql_connection, oss.str().c_str(), false); + +cleanup: + + free(uuid); + free(server_version); + + return(true); +} + +extern const char *innodb_checksum_algorithm_names[]; + +bool write_backup_config_file() +{ + int rc= backup_file_printf("backup-my.cnf", + "# This MySQL options file was generated by innobackupex.\n\n" + "# The MySQL server\n" + "[mysqld]\n" + "innodb_checksum_algorithm=%s\n" + "innodb_log_checksum_algorithm=%s\n" + "innodb_data_file_path=%s\n" + "innodb_log_files_in_group=%lu\n" + "innodb_log_file_size=%lld\n" + "innodb_page_size=%lu\n" + "innodb_log_block_size=%lu\n" + "innodb_undo_directory=%s\n" + "innodb_undo_tablespaces=%lu\n" + "%s%s\n" + "%s%s\n" + "%s\n", + innodb_checksum_algorithm_names[srv_checksum_algorithm], + innodb_checksum_algorithm_names[srv_log_checksum_algorithm], + innobase_data_file_path, + srv_n_log_files, + innobase_log_file_size, + srv_page_size, + srv_log_block_size, + srv_undo_dir, + srv_undo_tablespaces, + innobase_doublewrite_file ? "innodb_doublewrite_file=" : "", + innobase_doublewrite_file ? innobase_doublewrite_file : "", + innobase_buffer_pool_filename ? + "innodb_buffer_pool_filename=" : "", + innobase_buffer_pool_filename ? + innobase_buffer_pool_filename : "", + encryption_plugin_get_config()); + return rc; +} + + +static +char *make_argv(char *buf, size_t len, int argc, char **argv) +{ + size_t left= len; + const char *arg; + + buf[0]= 0; + ++argv; --argc; + while (argc > 0 && left > 0) + { + arg = *argv; + if (strncmp(*argv, "--password", strlen("--password")) == 0) { + arg = "--password=..."; + } + if (strncmp(*argv, "--encrypt-key", + strlen("--encrypt-key")) == 0) { + arg = "--encrypt-key=..."; + } + if (strncmp(*argv, "--encrypt_key", + strlen("--encrypt_key")) == 0) { + arg = "--encrypt_key=..."; + } + left-= ut_snprintf(buf + len - left, left, + "%s%c", arg, argc > 1 ? ' ' : 0); + ++argv; --argc; + } + + return buf; +} + +void +capture_tool_command(int argc, char **argv) +{ + /* capture tool name tool args */ + tool_name = strrchr(argv[0], '/'); + tool_name = tool_name ? tool_name + 1 : argv[0]; + + make_argv(tool_args, sizeof(tool_args), argc, argv); +} + + +bool +select_history() +{ + if (opt_incremental_history_name || opt_incremental_history_uuid) { + if (!select_incremental_lsn_from_history( + &incremental_lsn)) { + return(false); + } + } + return(true); +} + +bool +flush_changed_page_bitmaps() +{ + if (xtrabackup_incremental && have_changed_page_bitmaps && + !xtrabackup_incremental_force_scan) { + xb_mysql_query(mysql_connection, + "FLUSH NO_WRITE_TO_BINLOG CHANGED_PAGE_BITMAPS", false); + } + return(true); +} + + +/*********************************************************************//** +Deallocate memory, disconnect from MySQL server, etc. +@return true on success. */ +void +backup_cleanup() +{ + free(mysql_slave_position); + free(mysql_binlog_position); + free(buffer_pool_filename); + + if (mysql_connection) { + mysql_close(mysql_connection); + } +} diff --git a/extra/mariabackup/backup_mysql.h b/extra/mariabackup/backup_mysql.h new file mode 100644 index 00000000000..3ccd7bdb613 --- /dev/null +++ b/extra/mariabackup/backup_mysql.h @@ -0,0 +1,92 @@ +#ifndef XTRABACKUP_BACKUP_MYSQL_H +#define XTRABACKUP_BACKUP_MYSQL_H + +#include <mysql.h> + +/* mysql flavor and version */ +enum mysql_flavor_t { FLAVOR_UNKNOWN, FLAVOR_MYSQL, + FLAVOR_PERCONA_SERVER, FLAVOR_MARIADB }; +extern mysql_flavor_t server_flavor; +extern unsigned long mysql_server_version; + +/* server capabilities */ +extern bool have_changed_page_bitmaps; +extern bool have_backup_locks; +extern bool have_lock_wait_timeout; +extern bool have_galera_enabled; +extern bool have_flush_engine_logs; +extern bool have_multi_threaded_slave; +extern bool have_gtid_slave; + + +/* History on server */ +extern time_t history_start_time; +extern time_t history_end_time; +extern time_t history_lock_time; + + +extern bool sql_thread_started; +extern char *mysql_slave_position; +extern char *mysql_binlog_position; +extern char *buffer_pool_filename; + +/** connection to mysql server */ +extern MYSQL *mysql_connection; + +void +capture_tool_command(int argc, char **argv); + +bool +select_history(); + +bool +flush_changed_page_bitmaps(); + +void +backup_cleanup(); + +bool +get_mysql_vars(MYSQL *connection); + +bool +detect_mysql_capabilities_for_backup(); + +MYSQL * +xb_mysql_connect(); + +MYSQL_RES * +xb_mysql_query(MYSQL *connection, const char *query, bool use_result, + bool die_on_error = true); + +void +unlock_all(MYSQL *connection); + +bool +write_current_binlog_file(MYSQL *connection); + +bool +write_binlog_info(MYSQL *connection); + +bool +write_xtrabackup_info(MYSQL *connection); + +bool +write_backup_config_file(); + +bool +lock_binlog_maybe(MYSQL *connection); + +bool +lock_tables(MYSQL *connection); + +bool +wait_for_safe_slave(MYSQL *connection); + +bool +write_galera_info(MYSQL *connection); + +bool +write_slave_info(MYSQL *connection); + + +#endif diff --git a/extra/mariabackup/changed_page_bitmap.cc b/extra/mariabackup/changed_page_bitmap.cc new file mode 100644 index 00000000000..435b7fb6172 --- /dev/null +++ b/extra/mariabackup/changed_page_bitmap.cc @@ -0,0 +1,1018 @@ +/****************************************************** +XtraBackup: hot backup tool for InnoDB +(c) 2009-2012 Percona Inc. +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +/* Changed page bitmap implementation */ + +#include "changed_page_bitmap.h" + +#include "common.h" +#include "xtrabackup.h" + +/* TODO: copy-pasted shared definitions from the XtraDB bitmap write code. +Remove these on the first opportunity, i.e. single-binary XtraBackup. */ + +/* log0online.h */ + +/** Single bitmap file information */ +struct log_online_bitmap_file_t { + char name[FN_REFLEN]; /*!< Name with full path */ + os_file_t file; /*!< Handle to opened file */ + ib_uint64_t size; /*!< Size of the file */ + ib_uint64_t offset; /*!< Offset of the next read, + or count of already-read bytes + */ +}; + +/** A set of bitmap files containing some LSN range */ +struct log_online_bitmap_file_range_t { + size_t count; /*!< Number of files */ + /*!< Dynamically-allocated array of info about individual files */ + struct files_t { + char name[FN_REFLEN];/*!< Name of a file */ + lsn_t start_lsn; /*!< Starting LSN of data in this + file */ + ulong seq_num; /*!< Sequence number of this file */ + } *files; +}; + +/* log0online.c */ + +/** File name stem for bitmap files. */ +static const char* bmp_file_name_stem = "ib_modified_log_"; + +/** The bitmap file block size in bytes. All writes will be multiples of this. + */ +enum { + MODIFIED_PAGE_BLOCK_SIZE = 4096 +}; + +/** Offsets in a file bitmap block */ +enum { + MODIFIED_PAGE_IS_LAST_BLOCK = 0,/* 1 if last block in the current + write, 0 otherwise. */ + MODIFIED_PAGE_START_LSN = 4, /* The starting tracked LSN of this and + other blocks in the same write */ + MODIFIED_PAGE_END_LSN = 12, /* The ending tracked LSN of this and + other blocks in the same write */ + MODIFIED_PAGE_SPACE_ID = 20, /* The space ID of tracked pages in + this block */ + MODIFIED_PAGE_1ST_PAGE_ID = 24, /* The page ID of the first tracked + page in this block */ + MODIFIED_PAGE_BLOCK_UNUSED_1 = 28,/* Unused in order to align the start + of bitmap at 8 byte boundary */ + MODIFIED_PAGE_BLOCK_BITMAP = 32,/* Start of the bitmap itself */ + MODIFIED_PAGE_BLOCK_UNUSED_2 = MODIFIED_PAGE_BLOCK_SIZE - 8, + /* Unused in order to align the end of + bitmap at 8 byte boundary */ + MODIFIED_PAGE_BLOCK_CHECKSUM = MODIFIED_PAGE_BLOCK_SIZE - 4 + /* The checksum of the current block */ +}; + +/** Length of the bitmap data in a block */ +enum { MODIFIED_PAGE_BLOCK_BITMAP_LEN + = MODIFIED_PAGE_BLOCK_UNUSED_2 - MODIFIED_PAGE_BLOCK_BITMAP }; + +/** Length of the bitmap data in a block in page ids */ +enum { MODIFIED_PAGE_BLOCK_ID_COUNT = MODIFIED_PAGE_BLOCK_BITMAP_LEN * 8 }; + +typedef ib_uint64_t bitmap_word_t; + +/****************************************************************//** +Calculate a bitmap block checksum. Algorithm borrowed from +log_block_calc_checksum. +@return checksum */ +UNIV_INLINE +ulint +log_online_calc_checksum( +/*=====================*/ + const byte* block); /*!<in: bitmap block */ + +/****************************************************************//** +Provide a comparisson function for the RB-tree tree (space, +block_start_page) pairs. Actual implementation does not matter as +long as the ordering is full. +@return -1 if p1 < p2, 0 if p1 == p2, 1 if p1 > p2 +*/ +static +int +log_online_compare_bmp_keys( +/*========================*/ + const void* p1, /*!<in: 1st key to compare */ + const void* p2) /*!<in: 2nd key to compare */ +{ + const byte *k1 = (const byte *)p1; + const byte *k2 = (const byte *)p2; + + ulint k1_space = mach_read_from_4(k1 + MODIFIED_PAGE_SPACE_ID); + ulint k2_space = mach_read_from_4(k2 + MODIFIED_PAGE_SPACE_ID); + if (k1_space == k2_space) { + + ulint k1_start_page + = mach_read_from_4(k1 + MODIFIED_PAGE_1ST_PAGE_ID); + ulint k2_start_page + = mach_read_from_4(k2 + MODIFIED_PAGE_1ST_PAGE_ID); + return k1_start_page < k2_start_page + ? -1 : k1_start_page > k2_start_page ? 1 : 0; + } + return k1_space < k2_space ? -1 : 1; +} + +/****************************************************************//** +Calculate a bitmap block checksum. Algorithm borrowed from +log_block_calc_checksum. +@return checksum */ +UNIV_INLINE +ulint +log_online_calc_checksum( +/*=====================*/ + const byte* block) /*!<in: bitmap block */ +{ + ulint sum; + ulint sh; + ulint i; + + sum = 1; + sh = 0; + + for (i = 0; i < MODIFIED_PAGE_BLOCK_CHECKSUM; i++) { + + ulint b = block[i]; + sum &= 0x7FFFFFFFUL; + sum += b; + sum += b << sh; + sh++; + if (sh > 24) { + + sh = 0; + } + } + + return sum; +} + +/****************************************************************//** +Read one bitmap data page and check it for corruption. + +@return TRUE if page read OK, FALSE if I/O error */ +static +ibool +log_online_read_bitmap_page( +/*========================*/ + log_online_bitmap_file_t *bitmap_file, /*!<in/out: bitmap + file */ + byte *page, /*!<out: read page. Must be at + least MODIFIED_PAGE_BLOCK_SIZE + bytes long */ + ibool *checksum_ok) /*!<out: TRUE if page + checksum OK */ +{ + ulint checksum; + ulint actual_checksum; + ibool success; + + ut_a(bitmap_file->size >= MODIFIED_PAGE_BLOCK_SIZE); + ut_a(bitmap_file->offset + <= bitmap_file->size - MODIFIED_PAGE_BLOCK_SIZE); + ut_a(bitmap_file->offset % MODIFIED_PAGE_BLOCK_SIZE == 0); + + success = os_file_read(bitmap_file->file, page, bitmap_file->offset, + MODIFIED_PAGE_BLOCK_SIZE); + + if (UNIV_UNLIKELY(!success)) { + + /* The following call prints an error message */ + os_file_get_last_error(TRUE); + msg("InnoDB: Warning: failed reading changed page bitmap " + "file \'%s\'\n", bitmap_file->name); + return FALSE; + } + + bitmap_file->offset += MODIFIED_PAGE_BLOCK_SIZE; + ut_ad(bitmap_file->offset <= bitmap_file->size); + + checksum = mach_read_from_4(page + MODIFIED_PAGE_BLOCK_CHECKSUM); + actual_checksum = log_online_calc_checksum(page); + *checksum_ok = (checksum == actual_checksum); + + return TRUE; +} + +/*********************************************************************//** +Check the name of a given file if it's a changed page bitmap file and +return file sequence and start LSN name components if it is. If is not, +the values of output parameters are undefined. + +@return TRUE if a given file is a changed page bitmap file. */ +static +ibool +log_online_is_bitmap_file( +/*======================*/ + const os_file_stat_t* file_info, /*!<in: file to + check */ + ulong* bitmap_file_seq_num, /*!<out: bitmap file + sequence number */ + lsn_t* bitmap_file_start_lsn) /*!<out: bitmap file + start LSN */ +{ + char stem[FN_REFLEN]; + + ut_ad (strlen(file_info->name) < OS_FILE_MAX_PATH); + + return ((file_info->type == OS_FILE_TYPE_FILE + || file_info->type == OS_FILE_TYPE_LINK) + && (sscanf(file_info->name, "%[a-z_]%lu_" LSN_PF ".xdb", stem, + bitmap_file_seq_num, bitmap_file_start_lsn) == 3) + && (!strcmp(stem, bmp_file_name_stem))); +} + +/*********************************************************************//** +List the bitmap files in srv_data_home and setup their range that contains the +specified LSN interval. This range, if non-empty, will start with a file that +has the greatest LSN equal to or less than the start LSN and will include all +the files up to the one with the greatest LSN less than the end LSN. Caller +must free bitmap_files->files when done if bitmap_files set to non-NULL and +this function returned TRUE. Field bitmap_files->count might be set to a +larger value than the actual count of the files, and space for the unused array +slots will be allocated but cleared to zeroes. + +@return TRUE if succeeded +*/ +static +ibool +log_online_setup_bitmap_file_range( +/*===============================*/ + log_online_bitmap_file_range_t *bitmap_files, /*!<in/out: bitmap file + range */ + lsn_t range_start, /*!<in: start LSN */ + lsn_t range_end) /*!<in: end LSN */ +{ + os_file_dir_t bitmap_dir; + os_file_stat_t bitmap_dir_file_info; + ulong first_file_seq_num = ULONG_MAX; + ulong last_file_seq_num = 0; + lsn_t first_file_start_lsn = LSN_MAX; + + xb_ad(range_end >= range_start); + + bitmap_files->count = 0; + bitmap_files->files = NULL; + + /* 1st pass: size the info array */ + + bitmap_dir = os_file_opendir(srv_data_home, FALSE); + if (UNIV_UNLIKELY(!bitmap_dir)) { + + msg("InnoDB: Error: failed to open bitmap directory \'%s\'\n", + srv_data_home); + return FALSE; + } + + while (!os_file_readdir_next_file(srv_data_home, bitmap_dir, + &bitmap_dir_file_info)) { + + ulong file_seq_num; + lsn_t file_start_lsn; + + if (!log_online_is_bitmap_file(&bitmap_dir_file_info, + &file_seq_num, + &file_start_lsn) + || file_start_lsn >= range_end) { + + continue; + } + + if (file_seq_num > last_file_seq_num) { + + last_file_seq_num = file_seq_num; + } + + if (file_start_lsn >= range_start + || file_start_lsn == first_file_start_lsn + || first_file_start_lsn > range_start) { + + /* A file that falls into the range */ + + if (file_start_lsn < first_file_start_lsn) { + + first_file_start_lsn = file_start_lsn; + } + if (file_seq_num < first_file_seq_num) { + + first_file_seq_num = file_seq_num; + } + } else if (file_start_lsn > first_file_start_lsn) { + + /* A file that has LSN closer to the range start + but smaller than it, replacing another such file */ + first_file_start_lsn = file_start_lsn; + first_file_seq_num = file_seq_num; + } + } + + if (UNIV_UNLIKELY(os_file_closedir(bitmap_dir))) { + + os_file_get_last_error(TRUE); + msg("InnoDB: Error: cannot close \'%s\'\n",srv_data_home); + return FALSE; + } + + if (first_file_seq_num == ULONG_MAX && last_file_seq_num == 0) { + + bitmap_files->count = 0; + return TRUE; + } + + bitmap_files->count = last_file_seq_num - first_file_seq_num + 1; + + /* 2nd pass: get the file names in the file_seq_num order */ + + bitmap_dir = os_file_opendir(srv_data_home, FALSE); + if (UNIV_UNLIKELY(!bitmap_dir)) { + + msg("InnoDB: Error: failed to open bitmap directory \'%s\'\n", + srv_data_home); + return FALSE; + } + + bitmap_files->files = + static_cast<log_online_bitmap_file_range_t::files_t *> + (ut_malloc(bitmap_files->count + * sizeof(bitmap_files->files[0]))); + memset(bitmap_files->files, 0, + bitmap_files->count * sizeof(bitmap_files->files[0])); + + while (!os_file_readdir_next_file(srv_data_home, bitmap_dir, + &bitmap_dir_file_info)) { + + ulong file_seq_num; + lsn_t file_start_lsn; + size_t array_pos; + + if (!log_online_is_bitmap_file(&bitmap_dir_file_info, + &file_seq_num, + &file_start_lsn) + || file_start_lsn >= range_end + || file_start_lsn < first_file_start_lsn) { + + continue; + } + + array_pos = file_seq_num - first_file_seq_num; + if (UNIV_UNLIKELY(array_pos >= bitmap_files->count)) { + + msg("InnoDB: Error: inconsistent bitmap file " + "directory\n"); + free(bitmap_files->files); + return FALSE; + } + + if (file_seq_num > bitmap_files->files[array_pos].seq_num) { + + bitmap_files->files[array_pos].seq_num = file_seq_num; + strncpy(bitmap_files->files[array_pos].name, + bitmap_dir_file_info.name, FN_REFLEN); + bitmap_files->files[array_pos].name[FN_REFLEN - 1] + = '\0'; + bitmap_files->files[array_pos].start_lsn + = file_start_lsn; + } + } + + if (UNIV_UNLIKELY(os_file_closedir(bitmap_dir))) { + + os_file_get_last_error(TRUE); + msg("InnoDB: Error: cannot close \'%s\'\n", srv_data_home); + free(bitmap_files->files); + return FALSE; + } + +#ifdef UNIV_DEBUG + ut_ad(bitmap_files->files[0].seq_num == first_file_seq_num); + + for (size_t i = 1; i < bitmap_files->count; i++) { + if (!bitmap_files->files[i].seq_num) { + + break; + } + ut_ad(bitmap_files->files[i].seq_num + > bitmap_files->files[i - 1].seq_num); + ut_ad(bitmap_files->files[i].start_lsn + >= bitmap_files->files[i - 1].start_lsn); + } +#endif + + return TRUE; +} + +/****************************************************************//** +Open a bitmap file for reading. + +@return TRUE if opened successfully */ +static +ibool +log_online_open_bitmap_file_read_only( +/*==================================*/ + const char* name, /*!<in: bitmap file + name without directory, + which is assumed to be + srv_data_home */ + log_online_bitmap_file_t* bitmap_file) /*!<out: opened bitmap + file */ +{ + ibool success = FALSE; + + xb_ad(name[0] != '\0'); + + ut_snprintf(bitmap_file->name, FN_REFLEN, "%s%s", srv_data_home, name); + bitmap_file->file + = os_file_create_simple_no_error_handling(0, bitmap_file->name, + OS_FILE_OPEN, + OS_FILE_READ_ONLY, + &success,0); + if (UNIV_UNLIKELY(!success)) { + + /* Here and below assume that bitmap file names do not + contain apostrophes, thus no need for ut_print_filename(). */ + msg("InnoDB: Warning: error opening the changed page " + "bitmap \'%s\'\n", bitmap_file->name); + return FALSE; + } + + bitmap_file->size = os_file_get_size(bitmap_file->file); + bitmap_file->offset = 0; + +#ifdef UNIV_LINUX + posix_fadvise(bitmap_file->file, 0, 0, POSIX_FADV_SEQUENTIAL); + posix_fadvise(bitmap_file->file, 0, 0, POSIX_FADV_NOREUSE); +#endif + + return TRUE; +} + +/****************************************************************//** +Diagnose one or both of the following situations if we read close to +the end of bitmap file: +1) Warn if the remainder of the file is less than one page. +2) Error if we cannot read any more full pages but the last read page +did not have the last-in-run flag set. + +@return FALSE for the error */ +static +ibool +log_online_diagnose_bitmap_eof( +/*===========================*/ + const log_online_bitmap_file_t* bitmap_file, /*!< in: bitmap file */ + ibool last_page_in_run)/*!< in: "last page in + run" flag value in the + last read page */ +{ + /* Check if we are too close to EOF to read a full page */ + if ((bitmap_file->size < MODIFIED_PAGE_BLOCK_SIZE) + || (bitmap_file->offset + > bitmap_file->size - MODIFIED_PAGE_BLOCK_SIZE)) { + + if (UNIV_UNLIKELY(bitmap_file->offset != bitmap_file->size)) { + + /* If we are not at EOF and we have less than one page + to read, it's junk. This error is not fatal in + itself. */ + + msg("InnoDB: Warning: junk at the end of changed " + "page bitmap file \'%s\'.\n", bitmap_file->name); + } + + if (UNIV_UNLIKELY(!last_page_in_run)) { + + /* We are at EOF but the last read page did not finish + a run */ + /* It's a "Warning" here because it's not a fatal error + for the whole server */ + msg("InnoDB: Warning: changed page bitmap " + "file \'%s\' does not contain a complete run " + "at the end.\n", bitmap_file->name); + return FALSE; + } + } + return TRUE; +} + +/* End of copy-pasted definitions */ + +/** Iterator structure over changed page bitmap */ +struct xb_page_bitmap_range_struct { + const xb_page_bitmap *bitmap; /* Bitmap with data */ + ulint space_id; /* Space id for this + iterator */ + ulint bit_i; /* Bit index of the iterator + position in the current page */ + const ib_rbt_node_t *bitmap_node; /* Current bitmap tree node */ + const byte *bitmap_page; /* Current bitmap page */ + ulint current_page_id;/* Current page id */ +}; + +/****************************************************************//** +Print a diagnostic message on missing bitmap data for an LSN range. */ +static +void +xb_msg_missing_lsn_data( +/*====================*/ + lsn_t missing_interval_start, /*!<in: interval start */ + lsn_t missing_interval_end) /*!<in: interval end */ +{ + msg("xtrabackup: warning: changed page data missing for LSNs between " + LSN_PF " and " LSN_PF "\n", missing_interval_start, + missing_interval_end); +} + +/****************************************************************//** +Scan a bitmap file until data for a desired LSN or EOF is found and check that +the page before the starting one is not corrupted to ensure that the found page +indeed contains the very start of the desired LSN data. The caller must check +the page LSN values to determine if the bitmap file was scanned until the data +was found or until EOF. Page must be at least MODIFIED_PAGE_BLOCK_SIZE big. + +@return TRUE if the scan successful without corruption detected +*/ +static +ibool +xb_find_lsn_in_bitmap_file( +/*=======================*/ + log_online_bitmap_file_t *bitmap_file, /*!<in/out: bitmap + file */ + byte *page, /*!<in/out: last read + bitmap page */ + lsn_t *page_end_lsn, /*!<out: end LSN of the + last read page */ + lsn_t lsn) /*!<in: LSN to find */ +{ + ibool last_page_ok = TRUE; + ibool next_to_last_page_ok = TRUE; + + xb_ad (bitmap_file->size >= MODIFIED_PAGE_BLOCK_SIZE); + + *page_end_lsn = 0; + + while ((*page_end_lsn <= lsn) + && (bitmap_file->offset + <= bitmap_file->size - MODIFIED_PAGE_BLOCK_SIZE)) { + + next_to_last_page_ok = last_page_ok; + if (!log_online_read_bitmap_page(bitmap_file, page, + &last_page_ok)) { + + return FALSE; + } + + *page_end_lsn = mach_read_from_8(page + MODIFIED_PAGE_END_LSN); + } + + /* We check two pages here because the last read page already contains + the required LSN data. If the next to the last one page is corrupted, + then we have no way of telling if that page contained the required LSN + range data too */ + return last_page_ok && next_to_last_page_ok; +} + +/****************************************************************//** +Read the disk bitmap and build the changed page bitmap tree for the +LSN interval incremental_lsn to checkpoint_lsn_start. + +@return the built bitmap tree or NULL if unable to read the full interval for +any reason. */ +xb_page_bitmap* +xb_page_bitmap_init(void) +/*=====================*/ +{ + log_online_bitmap_file_t bitmap_file; + lsn_t bmp_start_lsn = incremental_lsn; + lsn_t bmp_end_lsn = checkpoint_lsn_start; + byte page[MODIFIED_PAGE_BLOCK_SIZE]; + lsn_t current_page_end_lsn; + xb_page_bitmap *result; + ibool last_page_in_run= FALSE; + log_online_bitmap_file_range_t bitmap_files; + size_t bmp_i; + ibool last_page_ok = TRUE; + + if (UNIV_UNLIKELY(bmp_start_lsn > bmp_end_lsn)) { + + msg("xtrabackup: incremental backup LSN " LSN_PF + " is larger than than the last checkpoint LSN " LSN_PF + "\n", bmp_start_lsn, bmp_end_lsn); + return NULL; + } + + if (!log_online_setup_bitmap_file_range(&bitmap_files, bmp_start_lsn, + bmp_end_lsn)) { + + return NULL; + } + + /* Only accept no bitmap files returned if start LSN == end LSN */ + if (bitmap_files.count == 0 && bmp_end_lsn != bmp_start_lsn) { + + return NULL; + } + + result = rbt_create(MODIFIED_PAGE_BLOCK_SIZE, + log_online_compare_bmp_keys); + + if (bmp_start_lsn == bmp_end_lsn) { + + /* Empty range - empty bitmap */ + return result; + } + + bmp_i = 0; + + if (UNIV_UNLIKELY(bitmap_files.files[bmp_i].start_lsn + > bmp_start_lsn)) { + + /* The 1st file does not have the starting LSN data */ + xb_msg_missing_lsn_data(bmp_start_lsn, + bitmap_files.files[bmp_i].start_lsn); + rbt_free(result); + free(bitmap_files.files); + return NULL; + } + + /* Skip any zero-sized files at the start */ + while ((bmp_i < bitmap_files.count - 1) + && (bitmap_files.files[bmp_i].start_lsn + == bitmap_files.files[bmp_i + 1].start_lsn)) { + + bmp_i++; + } + + /* Is the 1st bitmap file missing? */ + if (UNIV_UNLIKELY(bitmap_files.files[bmp_i].name[0] == '\0')) { + + /* TODO: this is not the exact missing range */ + xb_msg_missing_lsn_data(bmp_start_lsn, bmp_end_lsn); + rbt_free(result); + free(bitmap_files.files); + return NULL; + } + + /* Open the 1st bitmap file */ + if (UNIV_UNLIKELY(!log_online_open_bitmap_file_read_only( + bitmap_files.files[bmp_i].name, + &bitmap_file))) { + + rbt_free(result); + free(bitmap_files.files); + return NULL; + } + + /* If the 1st file is truncated, no data. Not merged with the case + below because zero-length file indicates not a corruption but missing + subsequent files instead. */ + if (UNIV_UNLIKELY(bitmap_file.size < MODIFIED_PAGE_BLOCK_SIZE)) { + + xb_msg_missing_lsn_data(bmp_start_lsn, bmp_end_lsn); + rbt_free(result); + free(bitmap_files.files); + os_file_close(bitmap_file.file); + return NULL; + } + + /* Find the start of the required LSN range in the file */ + if (UNIV_UNLIKELY(!xb_find_lsn_in_bitmap_file(&bitmap_file, page, + ¤t_page_end_lsn, + bmp_start_lsn))) { + + msg("xtrabackup: Warning: changed page bitmap file " + "\'%s\' corrupted\n", bitmap_file.name); + rbt_free(result); + free(bitmap_files.files); + os_file_close(bitmap_file.file); + return NULL; + } + + last_page_in_run + = mach_read_from_4(page + MODIFIED_PAGE_IS_LAST_BLOCK); + + if (UNIV_UNLIKELY(!log_online_diagnose_bitmap_eof(&bitmap_file, + last_page_in_run))) { + + rbt_free(result); + free(bitmap_files.files); + os_file_close(bitmap_file.file); + return NULL; + } + + if (UNIV_UNLIKELY(current_page_end_lsn < bmp_start_lsn)) { + + xb_msg_missing_lsn_data(current_page_end_lsn, bmp_start_lsn); + rbt_free(result); + free(bitmap_files.files); + os_file_close(bitmap_file.file); + return NULL; + } + + /* 1st bitmap page found, add it to the tree. */ + rbt_insert(result, page, page); + + /* Read next pages/files until all required data is read */ + while (last_page_ok + && (current_page_end_lsn < bmp_end_lsn + || (current_page_end_lsn == bmp_end_lsn + && !last_page_in_run))) { + + ib_rbt_bound_t tree_search_pos; + + /* If EOF, advance the file skipping over any empty files */ + while (bitmap_file.size < MODIFIED_PAGE_BLOCK_SIZE + || (bitmap_file.offset + > bitmap_file.size - MODIFIED_PAGE_BLOCK_SIZE)) { + + os_file_close(bitmap_file.file); + + if (UNIV_UNLIKELY( + !log_online_diagnose_bitmap_eof( + &bitmap_file, last_page_in_run))) { + + rbt_free(result); + free(bitmap_files.files); + return NULL; + } + + bmp_i++; + + if (UNIV_UNLIKELY(bmp_i == bitmap_files.count + || (bitmap_files.files[bmp_i].seq_num + == 0))) { + + xb_msg_missing_lsn_data(current_page_end_lsn, + bmp_end_lsn); + rbt_free(result); + free(bitmap_files.files); + return NULL; + } + + /* Is the next file missing? */ + if (UNIV_UNLIKELY(bitmap_files.files[bmp_i].name[0] + == '\0')) { + + /* TODO: this is not the exact missing range */ + xb_msg_missing_lsn_data(bitmap_files.files + [bmp_i - 1].start_lsn, + bmp_end_lsn); + rbt_free(result); + free(bitmap_files.files); + return NULL; + } + + if (UNIV_UNLIKELY( + !log_online_open_bitmap_file_read_only( + bitmap_files.files[bmp_i].name, + &bitmap_file))) { + + rbt_free(result); + free(bitmap_files.files); + return NULL; + } + } + + if (UNIV_UNLIKELY( + !log_online_read_bitmap_page(&bitmap_file, page, + &last_page_ok))) { + + rbt_free(result); + free(bitmap_files.files); + os_file_close(bitmap_file.file); + return NULL; + } + + if (UNIV_UNLIKELY(!last_page_ok)) { + + msg("xtrabackup: warning: changed page bitmap file " + "\'%s\' corrupted.\n", bitmap_file.name); + rbt_free(result); + free(bitmap_files.files); + os_file_close(bitmap_file.file); + return NULL; + } + + /* Merge the current page with an existing page or insert a new + page into the tree */ + + if (!rbt_search(result, &tree_search_pos, page)) { + + /* Merge the bitmap pages */ + byte *existing_page + = rbt_value(byte, tree_search_pos.last); + bitmap_word_t *bmp_word_1 = (bitmap_word_t *) + (existing_page + MODIFIED_PAGE_BLOCK_BITMAP); + bitmap_word_t *bmp_end = (bitmap_word_t *) + (existing_page + MODIFIED_PAGE_BLOCK_UNUSED_2); + bitmap_word_t *bmp_word_2 = (bitmap_word_t *) + (page + MODIFIED_PAGE_BLOCK_BITMAP); + while (bmp_word_1 < bmp_end) { + + *bmp_word_1++ |= *bmp_word_2++; + } + xb_a (bmp_word_1 == bmp_end); + } else { + + /* Add a new page */ + rbt_add_node(result, &tree_search_pos, page); + } + + current_page_end_lsn + = mach_read_from_8(page + MODIFIED_PAGE_END_LSN); + last_page_in_run + = mach_read_from_4(page + MODIFIED_PAGE_IS_LAST_BLOCK); + } + + xb_a (current_page_end_lsn >= bmp_end_lsn); + + free(bitmap_files.files); + os_file_close(bitmap_file.file); + + return result; +} + +/****************************************************************//** +Free the bitmap tree. */ +void +xb_page_bitmap_deinit( +/*==================*/ + xb_page_bitmap* bitmap) /*!<in/out: bitmap tree */ +{ + if (bitmap) { + + rbt_free(bitmap); + } +} + +/****************************************************************//** +Advance to the next bitmap page or setup the first bitmap page for the +given bitmap range. Assumes that bitmap_range->bitmap_page has been +already found/bumped by rbt_search()/rbt_next(). + +@return FALSE if no more bitmap data for the range space ID */ +static +ibool +xb_page_bitmap_setup_next_page( +/*===========================*/ + xb_page_bitmap_range* bitmap_range) /*!<in/out: the bitmap range */ +{ + ulint new_space_id; + ulint new_1st_page_id; + + if (bitmap_range->bitmap_node == NULL) { + + bitmap_range->current_page_id = ULINT_UNDEFINED; + return FALSE; + } + + bitmap_range->bitmap_page = rbt_value(byte, bitmap_range->bitmap_node); + + new_space_id = mach_read_from_4(bitmap_range->bitmap_page + + MODIFIED_PAGE_SPACE_ID); + if (new_space_id != bitmap_range->space_id) { + + /* No more data for the current page id. */ + xb_a(new_space_id > bitmap_range->space_id); + bitmap_range->current_page_id = ULINT_UNDEFINED; + return FALSE; + } + + new_1st_page_id = mach_read_from_4(bitmap_range->bitmap_page + + MODIFIED_PAGE_1ST_PAGE_ID); + xb_a (new_1st_page_id >= bitmap_range->current_page_id + || bitmap_range->current_page_id == ULINT_UNDEFINED); + + bitmap_range->current_page_id = new_1st_page_id; + bitmap_range->bit_i = 0; + + return TRUE; +} + +/****************************************************************//** +Set up a new bitmap range iterator over a given space id changed +pages in a given bitmap. + +@return bitmap range iterator */ +xb_page_bitmap_range* +xb_page_bitmap_range_init( +/*======================*/ + xb_page_bitmap* bitmap, /*!< in: bitmap to iterate over */ + ulint space_id) /*!< in: space id */ +{ + byte search_page[MODIFIED_PAGE_BLOCK_SIZE]; + xb_page_bitmap_range *result + = static_cast<xb_page_bitmap_range *> + (ut_malloc(sizeof(*result))); + + memset(result, 0, sizeof(*result)); + result->bitmap = bitmap; + result->space_id = space_id; + result->current_page_id = ULINT_UNDEFINED; + + /* Search for the 1st page for the given space id */ + /* This also sets MODIFIED_PAGE_1ST_PAGE_ID to 0, which is what we + want. */ + memset(search_page, 0, MODIFIED_PAGE_BLOCK_SIZE); + mach_write_to_4(search_page + MODIFIED_PAGE_SPACE_ID, space_id); + + result->bitmap_node = rbt_lower_bound(result->bitmap, search_page); + + xb_page_bitmap_setup_next_page(result); + + return result; +} + +/****************************************************************//** +Get the value of the bitmap->range->bit_i bitmap bit + +@return the current bit value */ +static inline +ibool +is_bit_set( +/*=======*/ + const xb_page_bitmap_range* bitmap_range) /*!< in: bitmap + range */ +{ + return ((*(((bitmap_word_t *)(bitmap_range->bitmap_page + + MODIFIED_PAGE_BLOCK_BITMAP)) + + (bitmap_range->bit_i >> 6))) + & (1ULL << (bitmap_range->bit_i & 0x3F))) ? TRUE : FALSE; +} + +/****************************************************************//** +Get the next page id that has its bit set or cleared, i.e. equal to +bit_value. + +@return page id */ +ulint +xb_page_bitmap_range_get_next_bit( +/*==============================*/ + xb_page_bitmap_range* bitmap_range, /*!< in/out: bitmap range */ + ibool bit_value) /*!< in: bit value */ +{ + if (UNIV_UNLIKELY(bitmap_range->current_page_id + == ULINT_UNDEFINED)) { + + return ULINT_UNDEFINED; + } + + do { + while (bitmap_range->bit_i < MODIFIED_PAGE_BLOCK_ID_COUNT) { + + while (is_bit_set(bitmap_range) != bit_value + && (bitmap_range->bit_i + < MODIFIED_PAGE_BLOCK_ID_COUNT)) { + + bitmap_range->current_page_id++; + bitmap_range->bit_i++; + } + + if (bitmap_range->bit_i + < MODIFIED_PAGE_BLOCK_ID_COUNT) { + + ulint result = bitmap_range->current_page_id; + bitmap_range->current_page_id++; + bitmap_range->bit_i++; + return result; + } + } + + bitmap_range->bitmap_node + = rbt_next(bitmap_range->bitmap, + bitmap_range->bitmap_node); + + } while (xb_page_bitmap_setup_next_page(bitmap_range)); + + return ULINT_UNDEFINED; +} + +/****************************************************************//** +Free the bitmap range iterator. */ +void +xb_page_bitmap_range_deinit( +/*========================*/ + xb_page_bitmap_range* bitmap_range) /*! in/out: bitmap range */ +{ + ut_free(bitmap_range); +} diff --git a/extra/mariabackup/changed_page_bitmap.h b/extra/mariabackup/changed_page_bitmap.h new file mode 100644 index 00000000000..6f549f47400 --- /dev/null +++ b/extra/mariabackup/changed_page_bitmap.h @@ -0,0 +1,85 @@ +/****************************************************** +XtraBackup: hot backup tool for InnoDB +(c) 2009-2012 Percona Inc. +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +/* Changed page bitmap interface */ + +#ifndef XB_CHANGED_PAGE_BITMAP_H +#define XB_CHANGED_PAGE_BITMAP_H + +#include <ut0rbt.h> +#include <fil0fil.h> + +/* The changed page bitmap structure */ +typedef ib_rbt_t xb_page_bitmap; + +struct xb_page_bitmap_range_struct; + +/* The bitmap range iterator over one space id */ +typedef struct xb_page_bitmap_range_struct xb_page_bitmap_range; + +/****************************************************************//** +Read the disk bitmap and build the changed page bitmap tree for the +LSN interval incremental_lsn to checkpoint_lsn_start. + +@return the built bitmap tree */ +xb_page_bitmap* +xb_page_bitmap_init(void); +/*=====================*/ + +/****************************************************************//** +Free the bitmap tree. */ +void +xb_page_bitmap_deinit( +/*==================*/ + xb_page_bitmap* bitmap); /*!<in/out: bitmap tree */ + + +/****************************************************************//** +Set up a new bitmap range iterator over a given space id changed +pages in a given bitmap. + +@return bitmap range iterator */ +xb_page_bitmap_range* +xb_page_bitmap_range_init( +/*======================*/ + xb_page_bitmap* bitmap, /*!< in: bitmap to iterate over */ + ulint space_id); /*!< in: space id */ + +/****************************************************************//** +Get the next page id that has its bit set or cleared, i.e. equal to +bit_value. + +@return page id */ +ulint +xb_page_bitmap_range_get_next_bit( +/*==============================*/ + xb_page_bitmap_range* bitmap_range, /*!< in/out: bitmap range */ + ibool bit_value); /*!< in: bit value */ + +/****************************************************************//** +Free the bitmap range iterator. */ +void +xb_page_bitmap_range_deinit( +/*========================*/ + xb_page_bitmap_range* bitmap_range); /*! in/out: bitmap range */ + +#endif diff --git a/extra/mariabackup/common.h b/extra/mariabackup/common.h new file mode 100644 index 00000000000..7b1dfd7a0db --- /dev/null +++ b/extra/mariabackup/common.h @@ -0,0 +1,174 @@ +/****************************************************** +Copyright (c) 2011-2013 Percona LLC and/or its affiliates. + +Common declarations for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#ifndef XB_COMMON_H +#define XB_COMMON_H + +#include <my_global.h> +#include <mysql_version.h> +#include <fcntl.h> +#include <stdarg.h> + + +# define fil_is_user_tablespace_id(i) ((i) > srv_undo_tablespaces_open) + +#ifdef _MSC_VER +#define stat _stati64 +#define PATH_MAX MAX_PATH +#endif + +#ifndef HAVE_VASPRINTF +static inline int vasprintf(char **strp, const char *fmt, va_list args) +{ + int len; +#ifdef _MSC_VER + len = _vscprintf(fmt, args); +#else + len = vsnprintf(NULL, 0, fmt, args); +#endif + if (len < 0) + { + return -1; + } + *strp = (char *)malloc(len + 1); + if (!*strp) + { + return -1; + } + vsprintf(*strp, fmt, args); + return len; +} + +static inline int asprintf(char **strp, const char *fmt,...) +{ + va_list args; + va_start(args, fmt); + int len = vasprintf(strp, fmt, args); + va_end(args); + return len; +} +#endif + +#define xb_a(expr) \ + do { \ + if (!(expr)) { \ + msg("Assertion \"%s\" failed at %s:%lu\n", \ + #expr, __FILE__, (ulong) __LINE__); \ + abort(); \ + } \ + } while (0); + +#ifdef XB_DEBUG +#define xb_ad(expr) xb_a(expr) +#else +#define xb_ad(expr) +#endif + +#define XB_DELTA_INFO_SUFFIX ".meta" + +static inline int msg(const char *fmt, ...) ATTRIBUTE_FORMAT(printf, 1, 2); +static inline int msg(const char *fmt, ...) +{ + int result; + va_list args; + + va_start(args, fmt); + result = vfprintf(stderr, fmt, args); + va_end(args); + + return result; +} + +static inline int msg_ts(const char *fmt, ...) ATTRIBUTE_FORMAT(printf, 1, 2); +static inline int msg_ts(const char *fmt, ...) +{ + int result; + time_t t = time(NULL); + char date[100]; + char *line; + va_list args; + + strftime(date, sizeof(date), "%y%m%d %H:%M:%S", localtime(&t)); + + va_start(args, fmt); + result = vasprintf(&line, fmt, args); + va_end(args); + + if (result != -1) { + result = fprintf(stderr, "%s %s", date, line); + free(line); + } + + return result; +} + +/* Use POSIX_FADV_NORMAL when available */ + +#ifdef POSIX_FADV_NORMAL +# define USE_POSIX_FADVISE +#else +# define POSIX_FADV_NORMAL +# define POSIX_FADV_SEQUENTIAL +# define POSIX_FADV_DONTNEED +# define posix_fadvise(a,b,c,d) do {} while(0) +#endif + +/*********************************************************************** +Computes bit shift for a given value. If the argument is not a power +of 2, returns 0.*/ +static inline size_t +get_bit_shift(size_t value) +{ + size_t shift; + + if (value == 0) + return 0; + + for (shift = 0; !(value & 1); shift++) { + value >>= 1; + } + return (value >> 1) ? 0 : shift; +} + +/**************************************************************************** +Read 'len' bytes from 'fd'. It is identical to my_read(..., MYF(MY_FULL_IO)), +i.e. tries to combine partial reads into a single block of size 'len', except +that it bails out on EOF or error, and returns the number of successfully read +bytes instead. */ +static inline size_t +xb_read_full(File fd, uchar *buf, size_t len) +{ + size_t tlen = 0; + size_t tbytes; + + while (tlen < len) { + tbytes = my_read(fd, buf, len - tlen, MYF(MY_WME)); + if (tbytes == 0 || tbytes == MY_FILE_ERROR) { + break; + } + + buf += tbytes; + tlen += tbytes; + } + + return tlen; +} + +#endif diff --git a/extra/mariabackup/crc/CMakeLists.txt b/extra/mariabackup/crc/CMakeLists.txt new file mode 100644 index 00000000000..577cab6080c --- /dev/null +++ b/extra/mariabackup/crc/CMakeLists.txt @@ -0,0 +1,33 @@ +# Copyright (c) 2017 Percona LLC and/or its affiliates. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +PROJECT(crc C) + +IF(NOT CMAKE_CROSSCOMPILING AND NOT MSVC) + STRING(TOLOWER ${CMAKE_SYSTEM_PROCESSOR} processor) + IF(processor MATCHES "86" OR processor MATCHES "amd64" OR processor MATCHES "x64") + # Check for PCLMUL instruction + CHECK_C_SOURCE_RUNS(" + int main() + { + asm volatile (\"pclmulqdq \\$0x00, %%xmm1, %%xmm0\":::\"cc\"); + return 0; + }" HAVE_CLMUL_INSTRUCTION) + ENDIF() +ENDIF() +IF(HAVE_CLMUL_INSTRUCTION) + ADD_DEFINITIONS(-DHAVE_CLMUL_INSTRUCTION) +ENDIF() +ADD_LIBRARY(crc crc_glue.c crc-intel-pclmul.c) diff --git a/extra/mariabackup/crc/config.h.cmake b/extra/mariabackup/crc/config.h.cmake new file mode 100644 index 00000000000..fe81c1859ae --- /dev/null +++ b/extra/mariabackup/crc/config.h.cmake @@ -0,0 +1,21 @@ +/****************************************************** +Copyright (c) 2017 Percona LLC and/or its affiliates. + +Zlib compatible CRC-32 implementation. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#cmakedefine HAVE_CLMUL_INSTRUCTION 1 diff --git a/extra/mariabackup/crc/crc-intel-pclmul.c b/extra/mariabackup/crc/crc-intel-pclmul.c new file mode 100644 index 00000000000..d470c2bee43 --- /dev/null +++ b/extra/mariabackup/crc/crc-intel-pclmul.c @@ -0,0 +1,511 @@ +/****************************************************** +Copyright (c) 2017 Percona LLC and/or its affiliates. + +CRC32 using Intel's PCLMUL instruction. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +/* crc-intel-pclmul.c - Intel PCLMUL accelerated CRC implementation + * Copyright (C) 2016 Jussi Kivilinna <jussi.kivilinna@iki.fi> + * + * This file is part of Libgcrypt. + * + * Libgcrypt is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of + * the License, or (at your option) any later version. + * + * Libgcrypt is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA + * + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <stdint.h> + +# define U64_C(c) (c ## UL) + +typedef uint32_t u32; +typedef uint16_t u16; +typedef uint64_t u64; +#ifndef byte +typedef uint8_t byte; +#endif + +# define _gcry_bswap32 __builtin_bswap32 + +#if __GNUC__ >= 4 && defined(__x86_64__) && defined(HAVE_CLMUL_INSTRUCTION) + +#if _GCRY_GCC_VERSION >= 40400 /* 4.4 */ +/* Prevent compiler from issuing SSE instructions between asm blocks. */ +# pragma GCC target("no-sse") +#endif + + +#define ALIGNED_16 __attribute__ ((aligned (16))) + + +struct u16_unaligned_s +{ + u16 a; +} __attribute__((packed, aligned (1), may_alias)); + + +/* Constants structure for generic reflected/non-reflected CRC32 CLMUL + * functions. */ +struct crc32_consts_s +{ + /* k: { x^(32*17), x^(32*15), x^(32*5), x^(32*3), x^(32*2), 0 } mod P(x) */ + u64 k[6]; + /* my_p: { floor(x^64 / P(x)), P(x) } */ + u64 my_p[2]; +}; + + +/* CLMUL constants for CRC32 and CRC32RFC1510. */ +static const struct crc32_consts_s crc32_consts ALIGNED_16 = +{ + { /* k[6] = reverse_33bits( x^(32*y) mod P(x) ) */ + U64_C(0x154442bd4), U64_C(0x1c6e41596), /* y = { 17, 15 } */ + U64_C(0x1751997d0), U64_C(0x0ccaa009e), /* y = { 5, 3 } */ + U64_C(0x163cd6124), 0 /* y = 2 */ + }, + { /* my_p[2] = reverse_33bits ( { floor(x^64 / P(x)), P(x) } ) */ + U64_C(0x1f7011641), U64_C(0x1db710641) + } +}; + +/* Common constants for CRC32 algorithms. */ +static const byte crc32_refl_shuf_shift[3 * 16] ALIGNED_16 = + { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + }; +static const byte crc32_partial_fold_input_mask[16 + 16] ALIGNED_16 = + { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + }; +static const u64 crc32_merge9to15_shuf[15 - 9 + 1][2] ALIGNED_16 = + { + { U64_C(0x0706050403020100), U64_C(0xffffffffffffff0f) }, /* 9 */ + { U64_C(0x0706050403020100), U64_C(0xffffffffffff0f0e) }, + { U64_C(0x0706050403020100), U64_C(0xffffffffff0f0e0d) }, + { U64_C(0x0706050403020100), U64_C(0xffffffff0f0e0d0c) }, + { U64_C(0x0706050403020100), U64_C(0xffffff0f0e0d0c0b) }, + { U64_C(0x0706050403020100), U64_C(0xffff0f0e0d0c0b0a) }, + { U64_C(0x0706050403020100), U64_C(0xff0f0e0d0c0b0a09) }, /* 15 */ + }; +static const u64 crc32_merge5to7_shuf[7 - 5 + 1][2] ALIGNED_16 = + { + { U64_C(0xffffff0703020100), U64_C(0xffffffffffffffff) }, /* 5 */ + { U64_C(0xffff070603020100), U64_C(0xffffffffffffffff) }, + { U64_C(0xff07060503020100), U64_C(0xffffffffffffffff) }, /* 7 */ + }; + +/* PCLMUL functions for reflected CRC32. */ +static inline void +crc32_reflected_bulk (u32 *pcrc, const byte *inbuf, size_t inlen, + const struct crc32_consts_s *consts) +{ + if (inlen >= 8 * 16) + { + asm volatile ("movd %[crc], %%xmm4\n\t" + "movdqu %[inbuf_0], %%xmm0\n\t" + "movdqu %[inbuf_1], %%xmm1\n\t" + "movdqu %[inbuf_2], %%xmm2\n\t" + "movdqu %[inbuf_3], %%xmm3\n\t" + "pxor %%xmm4, %%xmm0\n\t" + : + : [inbuf_0] "m" (inbuf[0 * 16]), + [inbuf_1] "m" (inbuf[1 * 16]), + [inbuf_2] "m" (inbuf[2 * 16]), + [inbuf_3] "m" (inbuf[3 * 16]), + [crc] "m" (*pcrc) + ); + + inbuf += 4 * 16; + inlen -= 4 * 16; + + asm volatile ("movdqa %[k1k2], %%xmm4\n\t" + : + : [k1k2] "m" (consts->k[1 - 1]) + ); + + /* Fold by 4. */ + while (inlen >= 4 * 16) + { + asm volatile ("movdqu %[inbuf_0], %%xmm5\n\t" + "movdqa %%xmm0, %%xmm6\n\t" + "pclmulqdq $0x00, %%xmm4, %%xmm0\n\t" + "pclmulqdq $0x11, %%xmm4, %%xmm6\n\t" + "pxor %%xmm5, %%xmm0\n\t" + "pxor %%xmm6, %%xmm0\n\t" + + "movdqu %[inbuf_1], %%xmm5\n\t" + "movdqa %%xmm1, %%xmm6\n\t" + "pclmulqdq $0x00, %%xmm4, %%xmm1\n\t" + "pclmulqdq $0x11, %%xmm4, %%xmm6\n\t" + "pxor %%xmm5, %%xmm1\n\t" + "pxor %%xmm6, %%xmm1\n\t" + + "movdqu %[inbuf_2], %%xmm5\n\t" + "movdqa %%xmm2, %%xmm6\n\t" + "pclmulqdq $0x00, %%xmm4, %%xmm2\n\t" + "pclmulqdq $0x11, %%xmm4, %%xmm6\n\t" + "pxor %%xmm5, %%xmm2\n\t" + "pxor %%xmm6, %%xmm2\n\t" + + "movdqu %[inbuf_3], %%xmm5\n\t" + "movdqa %%xmm3, %%xmm6\n\t" + "pclmulqdq $0x00, %%xmm4, %%xmm3\n\t" + "pclmulqdq $0x11, %%xmm4, %%xmm6\n\t" + "pxor %%xmm5, %%xmm3\n\t" + "pxor %%xmm6, %%xmm3\n\t" + : + : [inbuf_0] "m" (inbuf[0 * 16]), + [inbuf_1] "m" (inbuf[1 * 16]), + [inbuf_2] "m" (inbuf[2 * 16]), + [inbuf_3] "m" (inbuf[3 * 16]) + ); + + inbuf += 4 * 16; + inlen -= 4 * 16; + } + + asm volatile ("movdqa %[k3k4], %%xmm6\n\t" + "movdqa %[my_p], %%xmm5\n\t" + : + : [k3k4] "m" (consts->k[3 - 1]), + [my_p] "m" (consts->my_p[0]) + ); + + /* Fold 4 to 1. */ + + asm volatile ("movdqa %%xmm0, %%xmm4\n\t" + "pclmulqdq $0x00, %%xmm6, %%xmm0\n\t" + "pclmulqdq $0x11, %%xmm6, %%xmm4\n\t" + "pxor %%xmm1, %%xmm0\n\t" + "pxor %%xmm4, %%xmm0\n\t" + + "movdqa %%xmm0, %%xmm4\n\t" + "pclmulqdq $0x00, %%xmm6, %%xmm0\n\t" + "pclmulqdq $0x11, %%xmm6, %%xmm4\n\t" + "pxor %%xmm2, %%xmm0\n\t" + "pxor %%xmm4, %%xmm0\n\t" + + "movdqa %%xmm0, %%xmm4\n\t" + "pclmulqdq $0x00, %%xmm6, %%xmm0\n\t" + "pclmulqdq $0x11, %%xmm6, %%xmm4\n\t" + "pxor %%xmm3, %%xmm0\n\t" + "pxor %%xmm4, %%xmm0\n\t" + : + : + ); + } + else + { + asm volatile ("movd %[crc], %%xmm1\n\t" + "movdqu %[inbuf], %%xmm0\n\t" + "movdqa %[k3k4], %%xmm6\n\t" + "pxor %%xmm1, %%xmm0\n\t" + "movdqa %[my_p], %%xmm5\n\t" + : + : [inbuf] "m" (*inbuf), + [crc] "m" (*pcrc), + [k3k4] "m" (consts->k[3 - 1]), + [my_p] "m" (consts->my_p[0]) + ); + + inbuf += 16; + inlen -= 16; + } + + /* Fold by 1. */ + if (inlen >= 16) + { + while (inlen >= 16) + { + /* Load next block to XMM2. Fold XMM0 to XMM0:XMM1. */ + asm volatile ("movdqu %[inbuf], %%xmm2\n\t" + "movdqa %%xmm0, %%xmm1\n\t" + "pclmulqdq $0x00, %%xmm6, %%xmm0\n\t" + "pclmulqdq $0x11, %%xmm6, %%xmm1\n\t" + "pxor %%xmm2, %%xmm0\n\t" + "pxor %%xmm1, %%xmm0\n\t" + : + : [inbuf] "m" (*inbuf) + ); + + inbuf += 16; + inlen -= 16; + } + } + + /* Partial fold. */ + if (inlen) + { + /* Load last input and add padding zeros. */ + asm volatile ("movdqu %[shr_shuf], %%xmm3\n\t" + "movdqu %[shl_shuf], %%xmm4\n\t" + "movdqu %[mask], %%xmm2\n\t" + + "movdqa %%xmm0, %%xmm1\n\t" + "pshufb %%xmm4, %%xmm0\n\t" + "movdqu %[inbuf], %%xmm4\n\t" + "pshufb %%xmm3, %%xmm1\n\t" + "pand %%xmm4, %%xmm2\n\t" + "por %%xmm1, %%xmm2\n\t" + + "movdqa %%xmm0, %%xmm1\n\t" + "pclmulqdq $0x00, %%xmm6, %%xmm0\n\t" + "pclmulqdq $0x11, %%xmm6, %%xmm1\n\t" + "pxor %%xmm2, %%xmm0\n\t" + "pxor %%xmm1, %%xmm0\n\t" + : + : [inbuf] "m" (*(inbuf - 16 + inlen)), + [mask] "m" (crc32_partial_fold_input_mask[inlen]), + [shl_shuf] "m" (crc32_refl_shuf_shift[inlen]), + [shr_shuf] "m" (crc32_refl_shuf_shift[inlen + 16]) + ); + + inbuf += inlen; + inlen -= inlen; + } + + /* Final fold. */ + asm volatile (/* reduce 128-bits to 96-bits */ + "movdqa %%xmm0, %%xmm1\n\t" + "pclmulqdq $0x10, %%xmm6, %%xmm0\n\t" + "psrldq $8, %%xmm1\n\t" + "pxor %%xmm1, %%xmm0\n\t" + + /* reduce 96-bits to 64-bits */ + "pshufd $0xfc, %%xmm0, %%xmm1\n\t" /* [00][00][00][x] */ + "pshufd $0xf9, %%xmm0, %%xmm0\n\t" /* [00][00][x>>64][x>>32] */ + "pclmulqdq $0x00, %[k5], %%xmm1\n\t" /* [00][00][xx][xx] */ + "pxor %%xmm1, %%xmm0\n\t" /* top 64-bit are zero */ + + /* barrett reduction */ + "pshufd $0xf3, %%xmm0, %%xmm1\n\t" /* [00][00][x>>32][00] */ + "pslldq $4, %%xmm0\n\t" /* [??][x>>32][??][??] */ + "pclmulqdq $0x00, %%xmm5, %%xmm1\n\t" /* [00][xx][xx][00] */ + "pclmulqdq $0x10, %%xmm5, %%xmm1\n\t" /* [00][xx][xx][00] */ + "pxor %%xmm1, %%xmm0\n\t" + + /* store CRC */ + "pextrd $2, %%xmm0, %[out]\n\t" + : [out] "=m" (*pcrc) + : [k5] "m" (consts->k[5 - 1]) + ); +} + +static inline void +crc32_reflected_less_than_16 (u32 *pcrc, const byte *inbuf, size_t inlen, + const struct crc32_consts_s *consts) +{ + if (inlen < 4) + { + u32 crc = *pcrc; + u32 data; + + asm volatile ("movdqa %[my_p], %%xmm5\n\t" + : + : [my_p] "m" (consts->my_p[0]) + ); + + if (inlen == 1) + { + data = inbuf[0]; + data ^= crc; + data <<= 24; + crc >>= 8; + } + else if (inlen == 2) + { + data = ((const struct u16_unaligned_s *)inbuf)->a; + data ^= crc; + data <<= 16; + crc >>= 16; + } + else + { + data = ((const struct u16_unaligned_s *)inbuf)->a; + data |= inbuf[2] << 16; + data ^= crc; + data <<= 8; + crc >>= 24; + } + + /* Barrett reduction */ + asm volatile ("movd %[in], %%xmm0\n\t" + "movd %[crc], %%xmm1\n\t" + + "pclmulqdq $0x00, %%xmm5, %%xmm0\n\t" /* [00][00][xx][xx] */ + "psllq $32, %%xmm1\n\t" + "pshufd $0xfc, %%xmm0, %%xmm0\n\t" /* [00][00][00][x] */ + "pclmulqdq $0x10, %%xmm5, %%xmm0\n\t" /* [00][00][xx][xx] */ + "pxor %%xmm1, %%xmm0\n\t" + + "pextrd $1, %%xmm0, %[out]\n\t" + : [out] "=m" (*pcrc) + : [in] "rm" (data), + [crc] "rm" (crc) + ); + } + else if (inlen == 4) + { + /* Barrett reduction */ + asm volatile ("movd %[crc], %%xmm1\n\t" + "movd %[in], %%xmm0\n\t" + "movdqa %[my_p], %%xmm5\n\t" + "pxor %%xmm1, %%xmm0\n\t" + + "pclmulqdq $0x00, %%xmm5, %%xmm0\n\t" /* [00][00][xx][xx] */ + "pshufd $0xfc, %%xmm0, %%xmm0\n\t" /* [00][00][00][x] */ + "pclmulqdq $0x10, %%xmm5, %%xmm0\n\t" /* [00][00][xx][xx] */ + + "pextrd $1, %%xmm0, %[out]\n\t" + : [out] "=m" (*pcrc) + : [in] "m" (*inbuf), + [crc] "m" (*pcrc), + [my_p] "m" (consts->my_p[0]) + ); + } + else + { + asm volatile ("movdqu %[shuf], %%xmm4\n\t" + "movd %[crc], %%xmm1\n\t" + "movdqa %[my_p], %%xmm5\n\t" + "movdqa %[k3k4], %%xmm6\n\t" + : + : [shuf] "m" (crc32_refl_shuf_shift[inlen]), + [crc] "m" (*pcrc), + [my_p] "m" (consts->my_p[0]), + [k3k4] "m" (consts->k[3 - 1]) + ); + + if (inlen >= 8) + { + asm volatile ("movq %[inbuf], %%xmm0\n\t" + : + : [inbuf] "m" (*inbuf) + ); + if (inlen > 8) + { + asm volatile (/*"pinsrq $1, %[inbuf_tail], %%xmm0\n\t"*/ + "movq %[inbuf_tail], %%xmm2\n\t" + "punpcklqdq %%xmm2, %%xmm0\n\t" + "pshufb %[merge_shuf], %%xmm0\n\t" + : + : [inbuf_tail] "m" (inbuf[inlen - 8]), + [merge_shuf] "m" + (*crc32_merge9to15_shuf[inlen - 9]) + ); + } + } + else + { + asm volatile ("movd %[inbuf], %%xmm0\n\t" + "pinsrd $1, %[inbuf_tail], %%xmm0\n\t" + "pshufb %[merge_shuf], %%xmm0\n\t" + : + : [inbuf] "m" (*inbuf), + [inbuf_tail] "m" (inbuf[inlen - 4]), + [merge_shuf] "m" + (*crc32_merge5to7_shuf[inlen - 5]) + ); + } + + /* Final fold. */ + asm volatile ("pxor %%xmm1, %%xmm0\n\t" + "pshufb %%xmm4, %%xmm0\n\t" + + /* reduce 128-bits to 96-bits */ + "movdqa %%xmm0, %%xmm1\n\t" + "pclmulqdq $0x10, %%xmm6, %%xmm0\n\t" + "psrldq $8, %%xmm1\n\t" + "pxor %%xmm1, %%xmm0\n\t" /* top 32-bit are zero */ + + /* reduce 96-bits to 64-bits */ + "pshufd $0xfc, %%xmm0, %%xmm1\n\t" /* [00][00][00][x] */ + "pshufd $0xf9, %%xmm0, %%xmm0\n\t" /* [00][00][x>>64][x>>32] */ + "pclmulqdq $0x00, %[k5], %%xmm1\n\t" /* [00][00][xx][xx] */ + "pxor %%xmm1, %%xmm0\n\t" /* top 64-bit are zero */ + + /* barrett reduction */ + "pshufd $0xf3, %%xmm0, %%xmm1\n\t" /* [00][00][x>>32][00] */ + "pslldq $4, %%xmm0\n\t" /* [??][x>>32][??][??] */ + "pclmulqdq $0x00, %%xmm5, %%xmm1\n\t" /* [00][xx][xx][00] */ + "pclmulqdq $0x10, %%xmm5, %%xmm1\n\t" /* [00][xx][xx][00] */ + "pxor %%xmm1, %%xmm0\n\t" + + /* store CRC */ + "pextrd $2, %%xmm0, %[out]\n\t" + : [out] "=m" (*pcrc) + : [k5] "m" (consts->k[5 - 1]) + ); + } +} + +void +crc32_intel_pclmul (u32 *pcrc, const byte *inbuf, size_t inlen) +{ + const struct crc32_consts_s *consts = &crc32_consts; +#if defined(__x86_64__) && defined(__WIN64__) + char win64tmp[2 * 16]; + + /* XMM6-XMM7 need to be restored after use. */ + asm volatile ("movdqu %%xmm6, 0*16(%0)\n\t" + "movdqu %%xmm7, 1*16(%0)\n\t" + : + : "r" (win64tmp) + : "memory"); +#endif + + if (!inlen) + return; + + if (inlen >= 16) + crc32_reflected_bulk(pcrc, inbuf, inlen, consts); + else + crc32_reflected_less_than_16(pcrc, inbuf, inlen, consts); + +#if defined(__x86_64__) && defined(__WIN64__) + /* Restore used registers. */ + asm volatile("movdqu 0*16(%0), %%xmm6\n\t" + "movdqu 1*16(%0), %%xmm7\n\t" + : + : "r" (win64tmp) + : "memory"); +#endif +} + +#endif diff --git a/extra/mariabackup/crc/crc-intel-pclmul.h b/extra/mariabackup/crc/crc-intel-pclmul.h new file mode 100644 index 00000000000..120058165a0 --- /dev/null +++ b/extra/mariabackup/crc/crc-intel-pclmul.h @@ -0,0 +1,25 @@ +/****************************************************** +Copyright (c) 2017 Percona LLC and/or its affiliates. + +CRC32 using Intel's PCLMUL instruction. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#include <stdint.h> +#include <stddef.h> + +void +crc32_intel_pclmul(uint32_t *pcrc, const uint8_t *inbuf, size_t inlen); diff --git a/extra/mariabackup/crc/crc_glue.c b/extra/mariabackup/crc/crc_glue.c new file mode 100644 index 00000000000..ae3fa91c1b0 --- /dev/null +++ b/extra/mariabackup/crc/crc_glue.c @@ -0,0 +1,72 @@ +/****************************************************** +Copyright (c) 2017 Percona LLC and/or its affiliates. + +Zlib compatible CRC-32 implementation. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#include "crc_glue.h" +#include "crc-intel-pclmul.h" +#include <stdint.h> +#include <string.h> +#include <zlib.h> + +#if __GNUC__ >= 4 && defined(__x86_64__) +static int pclmul_enabled = 0; +#endif + +#if defined(__GNUC__) && defined(__x86_64__) +static +uint32_t +cpuid(uint32_t* ecx, uint32_t* edx) +{ + uint32_t level; + + asm("cpuid" : "=a" (level) : "a" (0) : "ebx", "ecx", "edx"); + + if (level < 1) { + return level; + } + + asm("cpuid" : "=c" (*ecx), "=d" (*edx) + : "a" (1) + : "ebx"); + + return level; +} +#endif + +void crc_init() { +#if defined(__GNUC__) && defined(__x86_64__) + uint32_t ecx, edx; + + if (cpuid(&ecx, &edx) > 0) { + pclmul_enabled = ((ecx >> 19) & 1) && ((ecx >> 1) & 1); + } +#endif +} + +unsigned long crc32_iso3309(unsigned long crc, const unsigned char *buf, unsigned int len) +{ +#if __GNUC__ >= 4 && defined(__x86_64__) && defined(HAVE_CLMUL_INSTRUCTION) + if (pclmul_enabled) { + uint32_t crc_accum = crc ^ 0xffffffffL; + crc32_intel_pclmul(&crc_accum, buf, len); + return crc_accum ^ 0xffffffffL; + } +#endif + return crc32(crc, buf, len); +} diff --git a/extra/mariabackup/crc/crc_glue.h b/extra/mariabackup/crc/crc_glue.h new file mode 100644 index 00000000000..e287fa4a7aa --- /dev/null +++ b/extra/mariabackup/crc/crc_glue.h @@ -0,0 +1,31 @@ +/****************************************************** +Copyright (c) 2017 Percona LLC and/or its affiliates. + +Zlib compatible CRC-32 implementation. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + + +#ifdef __cplusplus +extern "C" { +#endif + +void crc_init(); +unsigned long crc32_iso3309(unsigned long crc, const unsigned char *buf, unsigned int len); + +#ifdef __cplusplus +} +#endif diff --git a/extra/mariabackup/datasink.c b/extra/mariabackup/datasink.c new file mode 100644 index 00000000000..460e0e8ca19 --- /dev/null +++ b/extra/mariabackup/datasink.c @@ -0,0 +1,137 @@ +/****************************************************** +Copyright (c) 2011-2013 Percona LLC and/or its affiliates. + +Data sink interface. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#include <my_base.h> +#include "common.h" +#include "datasink.h" +#include "ds_compress.h" +#include "ds_archive.h" +#include "ds_xbstream.h" +#include "ds_local.h" +#include "ds_stdout.h" +#include "ds_tmpfile.h" +#include "ds_buffer.h" + +/************************************************************************ +Create a datasink of the specified type */ +ds_ctxt_t * +ds_create(const char *root, ds_type_t type) +{ + datasink_t *ds; + ds_ctxt_t *ctxt; + + switch (type) { + case DS_TYPE_STDOUT: + ds = &datasink_stdout; + break; + case DS_TYPE_LOCAL: + ds = &datasink_local; + break; + case DS_TYPE_ARCHIVE: +#ifdef HAVE_LIBARCHIVE + ds = &datasink_archive; +#else + msg("Error : mariabackup was built without libarchive support"); + exit(EXIT_FAILURE); +#endif + break; + case DS_TYPE_XBSTREAM: + ds = &datasink_xbstream; + break; + case DS_TYPE_COMPRESS: + ds = &datasink_compress; + break; + case DS_TYPE_ENCRYPT: + case DS_TYPE_DECRYPT: + msg("Error : mariabackup does not support encrypted backups."); + exit(EXIT_FAILURE); + break; + + case DS_TYPE_TMPFILE: + ds = &datasink_tmpfile; + break; + case DS_TYPE_BUFFER: + ds = &datasink_buffer; + break; + default: + msg("Unknown datasink type: %d\n", type); + xb_ad(0); + return NULL; + } + + ctxt = ds->init(root); + if (ctxt != NULL) { + ctxt->datasink = ds; + } else { + msg("Error: failed to initialize datasink.\n"); + exit(EXIT_FAILURE); + } + + return ctxt; +} + +/************************************************************************ +Open a datasink file */ +ds_file_t * +ds_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *stat) +{ + ds_file_t *file; + + file = ctxt->datasink->open(ctxt, path, stat); + if (file != NULL) { + file->datasink = ctxt->datasink; + } + + return file; +} + +/************************************************************************ +Write to a datasink file. +@return 0 on success, 1 on error. */ +int +ds_write(ds_file_t *file, const void *buf, size_t len) +{ + return file->datasink->write(file, buf, len); +} + +/************************************************************************ +Close a datasink file. +@return 0 on success, 1, on error. */ +int +ds_close(ds_file_t *file) +{ + return file->datasink->close(file); +} + +/************************************************************************ +Destroy a datasink handle */ +void +ds_destroy(ds_ctxt_t *ctxt) +{ + ctxt->datasink->deinit(ctxt); +} + +/************************************************************************ +Set the destination pipe for a datasink (only makes sense for compress and +tmpfile). */ +void ds_set_pipe(ds_ctxt_t *ctxt, ds_ctxt_t *pipe_ctxt) +{ + ctxt->pipe_ctxt = pipe_ctxt; +} diff --git a/extra/mariabackup/datasink.h b/extra/mariabackup/datasink.h new file mode 100644 index 00000000000..8bf1321aad1 --- /dev/null +++ b/extra/mariabackup/datasink.h @@ -0,0 +1,100 @@ +/****************************************************** +Copyright (c) 2011-2013 Percona LLC and/or its affiliates. + +Data sink interface. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#ifndef XB_DATASINK_H +#define XB_DATASINK_H + +#include <my_global.h> +#include <my_dir.h> + +#ifdef __cplusplus +extern "C" { +#endif + +extern char *xtrabackup_tmpdir; +struct datasink_struct; +typedef struct datasink_struct datasink_t; + +typedef struct ds_ctxt { + datasink_t *datasink; + char *root; + void *ptr; + struct ds_ctxt *pipe_ctxt; +} ds_ctxt_t; + +typedef struct { + void *ptr; + char *path; + datasink_t *datasink; +} ds_file_t; + +struct datasink_struct { + ds_ctxt_t *(*init)(const char *root); + ds_file_t *(*open)(ds_ctxt_t *ctxt, const char *path, MY_STAT *stat); + int (*write)(ds_file_t *file, const void *buf, size_t len); + int (*close)(ds_file_t *file); + void (*deinit)(ds_ctxt_t *ctxt); +}; + +/* Supported datasink types */ +typedef enum { + DS_TYPE_STDOUT, + DS_TYPE_LOCAL, + DS_TYPE_ARCHIVE, + DS_TYPE_XBSTREAM, + DS_TYPE_COMPRESS, + DS_TYPE_ENCRYPT, + DS_TYPE_DECRYPT, + DS_TYPE_TMPFILE, + DS_TYPE_BUFFER +} ds_type_t; + +/************************************************************************ +Create a datasink of the specified type */ +ds_ctxt_t *ds_create(const char *root, ds_type_t type); + +/************************************************************************ +Open a datasink file */ +ds_file_t *ds_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *stat); + +/************************************************************************ +Write to a datasink file. +@return 0 on success, 1 on error. */ +int ds_write(ds_file_t *file, const void *buf, size_t len); + +/************************************************************************ +Close a datasink file. +@return 0 on success, 1, on error. */ +int ds_close(ds_file_t *file); + +/************************************************************************ +Destroy a datasink handle */ +void ds_destroy(ds_ctxt_t *ctxt); + +/************************************************************************ +Set the destination pipe for a datasink (only makes sense for compress and +tmpfile). */ +void ds_set_pipe(ds_ctxt_t *ctxt, ds_ctxt_t *pipe_ctxt); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* XB_DATASINK_H */ diff --git a/extra/mariabackup/ds_archive.c b/extra/mariabackup/ds_archive.c new file mode 100644 index 00000000000..50afcce4bc7 --- /dev/null +++ b/extra/mariabackup/ds_archive.c @@ -0,0 +1,280 @@ +/****************************************************** +Copyright (c) 2013 Percona LLC and/or its affiliates. + +Streaming implementation for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#include <my_base.h> +#include <archive.h> +#include <archive_entry.h> +#include "common.h" +#include "datasink.h" + +#if ARCHIVE_VERSION_NUMBER < 3000000 +#define archive_write_add_filter_none(X) archive_write_set_compression_none(X) +#define archive_write_free(X) archive_write_finish(X) +#endif + +typedef struct { + struct archive *archive; + ds_file_t *dest_file; + pthread_mutex_t mutex; +} ds_archive_ctxt_t; + +typedef struct { + struct archive_entry *entry; + ds_archive_ctxt_t *archive_ctxt; +} ds_archive_file_t; + + +/*********************************************************************** +General archive interface */ + +static ds_ctxt_t *archive_init(const char *root); +static ds_file_t *archive_open(ds_ctxt_t *ctxt, const char *path, + MY_STAT *mystat); +static int archive_write(ds_file_t *file, const void *buf, size_t len); +static int archive_close(ds_file_t *file); +static void archive_deinit(ds_ctxt_t *ctxt); + +datasink_t datasink_archive = { + &archive_init, + &archive_open, + &archive_write, + &archive_close, + &archive_deinit +}; + +static +int +my_archive_open_callback(struct archive *a __attribute__((unused)), + void *data __attribute__((unused))) +{ + return ARCHIVE_OK; +} + +static +ssize_t +my_archive_write_callback(struct archive *a __attribute__((unused)), + void *data, const void *buffer, size_t length) +{ + ds_archive_ctxt_t *archive_ctxt; + + archive_ctxt = (ds_archive_ctxt_t *) data; + + xb_ad(archive_ctxt != NULL); + xb_ad(archive_ctxt->dest_file != NULL); + + if (!ds_write(archive_ctxt->dest_file, buffer, length)) { + return length; + } + return -1; +} + +static +int +my_archive_close_callback(struct archive *a __attribute__((unused)), + void *data __attribute__((unused))) +{ + return ARCHIVE_OK; +} + +static +ds_ctxt_t * +archive_init(const char *root __attribute__((unused))) +{ + ds_ctxt_t *ctxt; + ds_archive_ctxt_t *archive_ctxt; + struct archive *a; + + ctxt = my_malloc(sizeof(ds_ctxt_t) + sizeof(ds_archive_ctxt_t), + MYF(MY_FAE)); + archive_ctxt = (ds_archive_ctxt_t *)(ctxt + 1); + + if (pthread_mutex_init(&archive_ctxt->mutex, NULL)) { + msg("archive_init: pthread_mutex_init() failed.\n"); + goto err; + } + + a = archive_write_new(); + if (a == NULL) { + msg("archive_write_new() failed.\n"); + goto err; + } + + archive_ctxt->archive = a; + archive_ctxt->dest_file = NULL; + + if(archive_write_add_filter_none(a) != ARCHIVE_OK || + archive_write_set_format_pax_restricted(a) != ARCHIVE_OK || + /* disable internal buffering so we don't have to flush the + output in xtrabackup */ + archive_write_set_bytes_per_block(a, 0) != ARCHIVE_OK) { + msg("failed to set libarchive archive options: %s\n", + archive_error_string(a)); + archive_write_free(a); + goto err; + } + + if (archive_write_open(a, archive_ctxt, my_archive_open_callback, + my_archive_write_callback, + my_archive_close_callback) != ARCHIVE_OK) { + msg("cannot open output archive.\n"); + return NULL; + } + + ctxt->ptr = archive_ctxt; + + return ctxt; + +err: + my_free(ctxt); + return NULL; +} + +static +ds_file_t * +archive_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat) +{ + ds_archive_ctxt_t *archive_ctxt; + ds_ctxt_t *dest_ctxt; + ds_file_t *file; + ds_archive_file_t *archive_file; + + struct archive *a; + struct archive_entry *entry; + + xb_ad(ctxt->pipe_ctxt != NULL); + dest_ctxt = ctxt->pipe_ctxt; + + archive_ctxt = (ds_archive_ctxt_t *) ctxt->ptr; + + pthread_mutex_lock(&archive_ctxt->mutex); + if (archive_ctxt->dest_file == NULL) { + archive_ctxt->dest_file = ds_open(dest_ctxt, path, mystat); + if (archive_ctxt->dest_file == NULL) { + return NULL; + } + } + pthread_mutex_unlock(&archive_ctxt->mutex); + + file = (ds_file_t *) my_malloc(sizeof(ds_file_t) + + sizeof(ds_archive_file_t), + MYF(MY_FAE)); + + archive_file = (ds_archive_file_t *) (file + 1); + + a = archive_ctxt->archive; + + entry = archive_entry_new(); + if (entry == NULL) { + msg("archive_entry_new() failed.\n"); + goto err; + } + + archive_entry_set_size(entry, mystat->st_size); + archive_entry_set_mode(entry, 0660); + archive_entry_set_filetype(entry, AE_IFREG); + archive_entry_set_pathname(entry, path); + archive_entry_set_mtime(entry, mystat->st_mtime, 0); + + archive_file->entry = entry; + archive_file->archive_ctxt = archive_ctxt; + + if (archive_write_header(a, entry) != ARCHIVE_OK) { + msg("archive_write_header() failed.\n"); + archive_entry_free(entry); + goto err; + } + + file->ptr = archive_file; + file->path = archive_ctxt->dest_file->path; + + return file; + +err: + if (archive_ctxt->dest_file) { + ds_close(archive_ctxt->dest_file); + archive_ctxt->dest_file = NULL; + } + my_free(file); + + return NULL; +} + +static +int +archive_write(ds_file_t *file, const void *buf, size_t len) +{ + ds_archive_file_t *archive_file; + struct archive *a; + + archive_file = (ds_archive_file_t *) file->ptr; + + a = archive_file->archive_ctxt->archive; + + xb_ad(archive_file->archive_ctxt->dest_file != NULL); + if (archive_write_data(a, buf, len) < 0) { + msg("archive_write_data() failed: %s (errno = %d)\n", + archive_error_string(a), archive_errno(a)); + return 1; + } + + return 0; +} + +static +int +archive_close(ds_file_t *file) +{ + ds_archive_file_t *archive_file; + int rc = 0; + + archive_file = (ds_archive_file_t *)file->ptr; + + archive_entry_free(archive_file->entry); + + my_free(file); + + return rc; +} + +static +void +archive_deinit(ds_ctxt_t *ctxt) +{ + struct archive *a; + ds_archive_ctxt_t *archive_ctxt; + + archive_ctxt = (ds_archive_ctxt_t *) ctxt->ptr; + + a = archive_ctxt->archive; + + if (archive_write_close(a) != ARCHIVE_OK) { + msg("archive_write_close() failed.\n"); + } + archive_write_free(a); + + if (archive_ctxt->dest_file) { + ds_close(archive_ctxt->dest_file); + archive_ctxt->dest_file = NULL; + } + + pthread_mutex_destroy(&archive_ctxt->mutex); + + my_free(ctxt); +} diff --git a/extra/mariabackup/ds_archive.h b/extra/mariabackup/ds_archive.h new file mode 100644 index 00000000000..3f4e4463c58 --- /dev/null +++ b/extra/mariabackup/ds_archive.h @@ -0,0 +1,28 @@ +/****************************************************** +Copyright (c) 2013 Percona LLC and/or its affiliates. + +Streaming interface for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#ifndef DS_ARCHIVE_H +#define DS_ARCHIVE_H + +#include "datasink.h" + +extern datasink_t datasink_archive; + +#endif diff --git a/extra/mariabackup/ds_buffer.c b/extra/mariabackup/ds_buffer.c new file mode 100644 index 00000000000..4bb314c0f50 --- /dev/null +++ b/extra/mariabackup/ds_buffer.c @@ -0,0 +1,189 @@ +/****************************************************** +Copyright (c) 2012-2013 Percona LLC and/or its affiliates. + +buffer datasink for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +/* Does buffered output to a destination datasink set with ds_set_pipe(). +Writes to the destination datasink are guaranteed to not be smaller than a +specified buffer size (DS_DEFAULT_BUFFER_SIZE by default), with the only +exception for the last write for a file. */ + +#include <mysql_version.h> +#include <my_base.h> +#include "ds_buffer.h" +#include "common.h" +#include "datasink.h" + +#define DS_DEFAULT_BUFFER_SIZE (64 * 1024) + +typedef struct { + ds_file_t *dst_file; + char *buf; + size_t pos; + size_t size; +} ds_buffer_file_t; + +typedef struct { + size_t buffer_size; +} ds_buffer_ctxt_t; + +static ds_ctxt_t *buffer_init(const char *root); +static ds_file_t *buffer_open(ds_ctxt_t *ctxt, const char *path, + MY_STAT *mystat); +static int buffer_write(ds_file_t *file, const void *buf, size_t len); +static int buffer_close(ds_file_t *file); +static void buffer_deinit(ds_ctxt_t *ctxt); + +datasink_t datasink_buffer = { + &buffer_init, + &buffer_open, + &buffer_write, + &buffer_close, + &buffer_deinit +}; + +/* Change the default buffer size */ +void ds_buffer_set_size(ds_ctxt_t *ctxt, size_t size) +{ + ds_buffer_ctxt_t *buffer_ctxt = (ds_buffer_ctxt_t *) ctxt->ptr; + + buffer_ctxt->buffer_size = size; +} + +static ds_ctxt_t * +buffer_init(const char *root) +{ + ds_ctxt_t *ctxt; + ds_buffer_ctxt_t *buffer_ctxt; + + ctxt = my_malloc(sizeof(ds_ctxt_t) + sizeof(ds_buffer_ctxt_t), + MYF(MY_FAE)); + buffer_ctxt = (ds_buffer_ctxt_t *) (ctxt + 1); + buffer_ctxt->buffer_size = DS_DEFAULT_BUFFER_SIZE; + + ctxt->ptr = buffer_ctxt; + ctxt->root = my_strdup(root, MYF(MY_FAE)); + + return ctxt; +} + +static ds_file_t * +buffer_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat) +{ + ds_buffer_ctxt_t *buffer_ctxt; + ds_ctxt_t *pipe_ctxt; + ds_file_t *dst_file; + ds_file_t *file; + ds_buffer_file_t *buffer_file; + + pipe_ctxt = ctxt->pipe_ctxt; + xb_a(pipe_ctxt != NULL); + + dst_file = ds_open(pipe_ctxt, path, mystat); + if (dst_file == NULL) { + exit(EXIT_FAILURE); + } + + buffer_ctxt = (ds_buffer_ctxt_t *) ctxt->ptr; + + file = (ds_file_t *) my_malloc(sizeof(ds_file_t) + + sizeof(ds_buffer_file_t) + + buffer_ctxt->buffer_size, + MYF(MY_FAE)); + + buffer_file = (ds_buffer_file_t *) (file + 1); + buffer_file->dst_file = dst_file; + buffer_file->buf = (char *) (buffer_file + 1); + buffer_file->size = buffer_ctxt->buffer_size; + buffer_file->pos = 0; + + file->path = dst_file->path; + file->ptr = buffer_file; + + return file; +} + +static int +buffer_write(ds_file_t *file, const void *buf, size_t len) +{ + ds_buffer_file_t *buffer_file; + + buffer_file = (ds_buffer_file_t *) file->ptr; + + while (len > 0) { + if (buffer_file->pos + len > buffer_file->size) { + if (buffer_file->pos > 0) { + size_t bytes; + + bytes = buffer_file->size - buffer_file->pos; + memcpy(buffer_file->buf + buffer_file->pos, buf, + bytes); + + if (ds_write(buffer_file->dst_file, + buffer_file->buf, + buffer_file->size)) { + return 1; + } + + buffer_file->pos = 0; + + buf = (const char *) buf + bytes; + len -= bytes; + } else { + /* We don't have any buffered bytes, just write + the entire source buffer */ + if (ds_write(buffer_file->dst_file, buf, len)) { + return 1; + } + break; + } + } else { + memcpy(buffer_file->buf + buffer_file->pos, buf, len); + buffer_file->pos += len; + break; + } + } + + return 0; +} + +static int +buffer_close(ds_file_t *file) +{ + ds_buffer_file_t *buffer_file; + int ret; + + buffer_file = (ds_buffer_file_t *) file->ptr; + if (buffer_file->pos > 0) { + ds_write(buffer_file->dst_file, buffer_file->buf, + buffer_file->pos); + } + + ret = ds_close(buffer_file->dst_file); + + my_free(file); + + return ret; +} + +static void +buffer_deinit(ds_ctxt_t *ctxt) +{ + my_free(ctxt->root); + my_free(ctxt); +} diff --git a/extra/mariabackup/ds_buffer.h b/extra/mariabackup/ds_buffer.h new file mode 100644 index 00000000000..f8d2d63267d --- /dev/null +++ b/extra/mariabackup/ds_buffer.h @@ -0,0 +1,39 @@ +/****************************************************** +Copyright (c) 2012-2013 Percona LLC and/or its affiliates. + +buffer datasink for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#ifndef DS_BUFFER_H +#define DS_BUFFER_H + +#include "datasink.h" + +#ifdef __cplusplus +extern "C" { +#endif + +extern datasink_t datasink_buffer; + +/* Change the default buffer size */ +void ds_buffer_set_size(ds_ctxt_t *ctxt, size_t size); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/extra/mariabackup/ds_compress.c b/extra/mariabackup/ds_compress.c new file mode 100644 index 00000000000..15801c8abd4 --- /dev/null +++ b/extra/mariabackup/ds_compress.c @@ -0,0 +1,462 @@ +/****************************************************** +Copyright (c) 2011-2013 Percona LLC and/or its affiliates. + +Compressing datasink implementation for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#include <mysql_version.h> +#include <my_base.h> +#include <quicklz.h> +#include <zlib.h> +#include "common.h" +#include "datasink.h" + +#define COMPRESS_CHUNK_SIZE ((size_t) (xtrabackup_compress_chunk_size)) +#define MY_QLZ_COMPRESS_OVERHEAD 400 + +typedef struct { + pthread_t id; + uint num; + pthread_mutex_t ctrl_mutex; + pthread_cond_t ctrl_cond; + pthread_mutex_t data_mutex; + pthread_cond_t data_cond; + my_bool started; + my_bool data_avail; + my_bool cancelled; + const char *from; + size_t from_len; + char *to; + size_t to_len; + qlz_state_compress state; + ulong adler; +} comp_thread_ctxt_t; + +typedef struct { + comp_thread_ctxt_t *threads; + uint nthreads; +} ds_compress_ctxt_t; + +typedef struct { + ds_file_t *dest_file; + ds_compress_ctxt_t *comp_ctxt; + size_t bytes_processed; +} ds_compress_file_t; + +/* Compression options */ +extern char *xtrabackup_compress_alg; +extern uint xtrabackup_compress_threads; +extern ulonglong xtrabackup_compress_chunk_size; + +static ds_ctxt_t *compress_init(const char *root); +static ds_file_t *compress_open(ds_ctxt_t *ctxt, const char *path, + MY_STAT *mystat); +static int compress_write(ds_file_t *file, const void *buf, size_t len); +static int compress_close(ds_file_t *file); +static void compress_deinit(ds_ctxt_t *ctxt); + +datasink_t datasink_compress = { + &compress_init, + &compress_open, + &compress_write, + &compress_close, + &compress_deinit +}; + +static inline int write_uint32_le(ds_file_t *file, ulong n); +static inline int write_uint64_le(ds_file_t *file, ulonglong n); + +static comp_thread_ctxt_t *create_worker_threads(uint n); +static void destroy_worker_threads(comp_thread_ctxt_t *threads, uint n); +static void *compress_worker_thread_func(void *arg); + +static +ds_ctxt_t * +compress_init(const char *root) +{ + ds_ctxt_t *ctxt; + ds_compress_ctxt_t *compress_ctxt; + comp_thread_ctxt_t *threads; + + /* Create and initialize the worker threads */ + threads = create_worker_threads(xtrabackup_compress_threads); + if (threads == NULL) { + msg("compress: failed to create worker threads.\n"); + return NULL; + } + + ctxt = (ds_ctxt_t *) my_malloc(sizeof(ds_ctxt_t) + + sizeof(ds_compress_ctxt_t), + MYF(MY_FAE)); + + compress_ctxt = (ds_compress_ctxt_t *) (ctxt + 1); + compress_ctxt->threads = threads; + compress_ctxt->nthreads = xtrabackup_compress_threads; + + ctxt->ptr = compress_ctxt; + ctxt->root = my_strdup(root, MYF(MY_FAE)); + + return ctxt; +} + +static +ds_file_t * +compress_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat) +{ + ds_compress_ctxt_t *comp_ctxt; + ds_ctxt_t *dest_ctxt; + ds_file_t *dest_file; + char new_name[FN_REFLEN]; + size_t name_len; + ds_file_t *file; + ds_compress_file_t *comp_file; + + xb_ad(ctxt->pipe_ctxt != NULL); + dest_ctxt = ctxt->pipe_ctxt; + + comp_ctxt = (ds_compress_ctxt_t *) ctxt->ptr; + + /* Append the .qp extension to the filename */ + fn_format(new_name, path, "", ".qp", MYF(MY_APPEND_EXT)); + + dest_file = ds_open(dest_ctxt, new_name, mystat); + if (dest_file == NULL) { + return NULL; + } + + /* Write the qpress archive header */ + if (ds_write(dest_file, "qpress10", 8) || + write_uint64_le(dest_file, COMPRESS_CHUNK_SIZE)) { + goto err; + } + + /* We are going to create a one-file "flat" (i.e. with no + subdirectories) archive. So strip the directory part from the path and + remove the '.qp' suffix. */ + fn_format(new_name, path, "", "", MYF(MY_REPLACE_DIR)); + + /* Write the qpress file header */ + name_len = strlen(new_name); + if (ds_write(dest_file, "F", 1) || + write_uint32_le(dest_file, (uint)name_len) || + /* we want to write the terminating \0 as well */ + ds_write(dest_file, new_name, name_len + 1)) { + goto err; + } + + file = (ds_file_t *) my_malloc(sizeof(ds_file_t) + + sizeof(ds_compress_file_t), + MYF(MY_FAE)); + comp_file = (ds_compress_file_t *) (file + 1); + comp_file->dest_file = dest_file; + comp_file->comp_ctxt = comp_ctxt; + comp_file->bytes_processed = 0; + + file->ptr = comp_file; + file->path = dest_file->path; + + return file; + +err: + ds_close(dest_file); + return NULL; +} + +static +int +compress_write(ds_file_t *file, const void *buf, size_t len) +{ + ds_compress_file_t *comp_file; + ds_compress_ctxt_t *comp_ctxt; + comp_thread_ctxt_t *threads; + comp_thread_ctxt_t *thd; + uint nthreads; + uint i; + const char *ptr; + ds_file_t *dest_file; + + comp_file = (ds_compress_file_t *) file->ptr; + comp_ctxt = comp_file->comp_ctxt; + dest_file = comp_file->dest_file; + + threads = comp_ctxt->threads; + nthreads = comp_ctxt->nthreads; + + ptr = (const char *) buf; + while (len > 0) { + uint max_thread; + + /* Send data to worker threads for compression */ + for (i = 0; i < nthreads; i++) { + size_t chunk_len; + + thd = threads + i; + + pthread_mutex_lock(&thd->ctrl_mutex); + + chunk_len = (len > COMPRESS_CHUNK_SIZE) ? + COMPRESS_CHUNK_SIZE : len; + thd->from = ptr; + thd->from_len = chunk_len; + + pthread_mutex_lock(&thd->data_mutex); + thd->data_avail = TRUE; + pthread_cond_signal(&thd->data_cond); + pthread_mutex_unlock(&thd->data_mutex); + + len -= chunk_len; + if (len == 0) { + break; + } + ptr += chunk_len; + } + + max_thread = (i < nthreads) ? i : nthreads - 1; + + /* Reap and stream the compressed data */ + for (i = 0; i <= max_thread; i++) { + thd = threads + i; + + pthread_mutex_lock(&thd->data_mutex); + while (thd->data_avail == TRUE) { + pthread_cond_wait(&thd->data_cond, + &thd->data_mutex); + } + + xb_a(threads[i].to_len > 0); + + if (ds_write(dest_file, "NEWBNEWB", 8) || + write_uint64_le(dest_file, + comp_file->bytes_processed)) { + msg("compress: write to the destination stream " + "failed.\n"); + return 1; + } + + comp_file->bytes_processed += threads[i].from_len; + + if (write_uint32_le(dest_file, threads[i].adler) || + ds_write(dest_file, threads[i].to, + threads[i].to_len)) { + msg("compress: write to the destination stream " + "failed.\n"); + return 1; + } + + pthread_mutex_unlock(&threads[i].data_mutex); + pthread_mutex_unlock(&threads[i].ctrl_mutex); + } + } + + return 0; +} + +static +int +compress_close(ds_file_t *file) +{ + ds_compress_file_t *comp_file; + ds_file_t *dest_file; + int rc; + + comp_file = (ds_compress_file_t *) file->ptr; + dest_file = comp_file->dest_file; + + /* Write the qpress file trailer */ + ds_write(dest_file, "ENDSENDS", 8); + + /* Supposedly the number of written bytes should be written as a + "recovery information" in the file trailer, but in reality qpress + always writes 8 zeros here. Let's do the same */ + + write_uint64_le(dest_file, 0); + + rc = ds_close(dest_file); + + my_free(file); + + return rc; +} + +static +void +compress_deinit(ds_ctxt_t *ctxt) +{ + ds_compress_ctxt_t *comp_ctxt; + + xb_ad(ctxt->pipe_ctxt != NULL); + + comp_ctxt = (ds_compress_ctxt_t *) ctxt->ptr;; + + destroy_worker_threads(comp_ctxt->threads, comp_ctxt->nthreads); + + my_free(ctxt->root); + my_free(ctxt); +} + +static inline +int +write_uint32_le(ds_file_t *file, ulong n) +{ + char tmp[4]; + + int4store(tmp, n); + return ds_write(file, tmp, sizeof(tmp)); +} + +static inline +int +write_uint64_le(ds_file_t *file, ulonglong n) +{ + char tmp[8]; + + int8store(tmp, n); + return ds_write(file, tmp, sizeof(tmp)); +} + +static +comp_thread_ctxt_t * +create_worker_threads(uint n) +{ + comp_thread_ctxt_t *threads; + uint i; + + threads = (comp_thread_ctxt_t *) + my_malloc(sizeof(comp_thread_ctxt_t) * n, MYF(MY_FAE)); + + for (i = 0; i < n; i++) { + comp_thread_ctxt_t *thd = threads + i; + + thd->num = i + 1; + thd->started = FALSE; + thd->cancelled = FALSE; + thd->data_avail = FALSE; + + thd->to = (char *) my_malloc(COMPRESS_CHUNK_SIZE + + MY_QLZ_COMPRESS_OVERHEAD, + MYF(MY_FAE)); + + /* Initialize the control mutex and condition var */ + if (pthread_mutex_init(&thd->ctrl_mutex, NULL) || + pthread_cond_init(&thd->ctrl_cond, NULL)) { + goto err; + } + + /* Initialize and data mutex and condition var */ + if (pthread_mutex_init(&thd->data_mutex, NULL) || + pthread_cond_init(&thd->data_cond, NULL)) { + goto err; + } + + pthread_mutex_lock(&thd->ctrl_mutex); + + if (pthread_create(&thd->id, NULL, compress_worker_thread_func, + thd)) { + msg("compress: pthread_create() failed: " + "errno = %d\n", errno); + goto err; + } + } + + /* Wait for the threads to start */ + for (i = 0; i < n; i++) { + comp_thread_ctxt_t *thd = threads + i; + + while (thd->started == FALSE) + pthread_cond_wait(&thd->ctrl_cond, &thd->ctrl_mutex); + pthread_mutex_unlock(&thd->ctrl_mutex); + } + + return threads; + +err: + return NULL; +} + +static +void +destroy_worker_threads(comp_thread_ctxt_t *threads, uint n) +{ + uint i; + + for (i = 0; i < n; i++) { + comp_thread_ctxt_t *thd = threads + i; + + pthread_mutex_lock(&thd->data_mutex); + threads[i].cancelled = TRUE; + pthread_cond_signal(&thd->data_cond); + pthread_mutex_unlock(&thd->data_mutex); + + pthread_join(thd->id, NULL); + + pthread_cond_destroy(&thd->data_cond); + pthread_mutex_destroy(&thd->data_mutex); + pthread_cond_destroy(&thd->ctrl_cond); + pthread_mutex_destroy(&thd->ctrl_mutex); + + my_free(thd->to); + } + + my_free(threads); +} + +static +void * +compress_worker_thread_func(void *arg) +{ + comp_thread_ctxt_t *thd = (comp_thread_ctxt_t *) arg; + + pthread_mutex_lock(&thd->ctrl_mutex); + + pthread_mutex_lock(&thd->data_mutex); + + thd->started = TRUE; + pthread_cond_signal(&thd->ctrl_cond); + + pthread_mutex_unlock(&thd->ctrl_mutex); + + while (1) { + thd->data_avail = FALSE; + pthread_cond_signal(&thd->data_cond); + + while (!thd->data_avail && !thd->cancelled) { + pthread_cond_wait(&thd->data_cond, &thd->data_mutex); + } + + if (thd->cancelled) + break; + + thd->to_len = qlz_compress(thd->from, thd->to, thd->from_len, + &thd->state); + + /* qpress uses 0x00010000 as the initial value, but its own + Adler-32 implementation treats the value differently: + 1. higher order bits are the sum of all bytes in the sequence + 2. lower order bits are the sum of resulting values at every + step. + So it's the other way around as compared to zlib's adler32(). + That's why 0x00000001 is being passed here to be compatible + with qpress implementation. */ + + thd->adler = adler32(0x00000001, (uchar *) thd->to, + (uInt)thd->to_len); + } + + pthread_mutex_unlock(&thd->data_mutex); + + return NULL; +} diff --git a/extra/mariabackup/ds_compress.h b/extra/mariabackup/ds_compress.h new file mode 100644 index 00000000000..8498c965e13 --- /dev/null +++ b/extra/mariabackup/ds_compress.h @@ -0,0 +1,28 @@ +/****************************************************** +Copyright (c) 2011-2013 Percona LLC and/or its affiliates. + +Compression interface for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#ifndef DS_COMPRESS_H +#define DS_COMPRESS_H + +#include "datasink.h" + +extern datasink_t datasink_compress; + +#endif diff --git a/extra/mariabackup/ds_decrypt.c b/extra/mariabackup/ds_decrypt.c new file mode 100644 index 00000000000..e897ca101e5 --- /dev/null +++ b/extra/mariabackup/ds_decrypt.c @@ -0,0 +1,665 @@ +/****************************************************** +Copyright (c) 2017 Percona LLC and/or its affiliates. + +Encryption datasink implementation for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + + +#include <my_base.h> +#include "common.h" +#include "datasink.h" +#include "xbcrypt.h" +#include "xbcrypt_common.h" +#include "crc_glue.h" + +typedef struct { + pthread_t id; + uint num; + pthread_mutex_t ctrl_mutex; + pthread_cond_t ctrl_cond; + pthread_mutex_t data_mutex; + pthread_cond_t data_cond; + my_bool started; + my_bool data_avail; + my_bool cancelled; + my_bool failed; + const uchar *from; + size_t from_len; + uchar *to; + size_t to_len; + size_t to_size; + const uchar *iv; + size_t iv_len; + unsigned long long offset; + my_bool hash_appended; + gcry_cipher_hd_t cipher_handle; + xb_rcrypt_result_t parse_result; +} crypt_thread_ctxt_t; + +typedef struct { + crypt_thread_ctxt_t *threads; + uint nthreads; + int encrypt_algo; + size_t chunk_size; + char *encrypt_key; + char *encrypt_key_file; +} ds_decrypt_ctxt_t; + +typedef struct { + ds_decrypt_ctxt_t *crypt_ctxt; + size_t bytes_processed; + ds_file_t *dest_file; + uchar *buf; + size_t buf_len; + size_t buf_size; +} ds_decrypt_file_t; + +int ds_decrypt_encrypt_threads = 1; + +static ds_ctxt_t *decrypt_init(const char *root); +static ds_file_t *decrypt_open(ds_ctxt_t *ctxt, const char *path, + MY_STAT *mystat); +static int decrypt_write(ds_file_t *file, const void *buf, size_t len); +static int decrypt_close(ds_file_t *file); +static void decrypt_deinit(ds_ctxt_t *ctxt); + +datasink_t datasink_decrypt = { + &decrypt_init, + &decrypt_open, + &decrypt_write, + &decrypt_close, + &decrypt_deinit +}; + +static crypt_thread_ctxt_t *create_worker_threads(uint n); +static void destroy_worker_threads(crypt_thread_ctxt_t *threads, uint n); +static void *decrypt_worker_thread_func(void *arg); + +static +ds_ctxt_t * +decrypt_init(const char *root) +{ + ds_ctxt_t *ctxt; + ds_decrypt_ctxt_t *decrypt_ctxt; + crypt_thread_ctxt_t *threads; + + if (xb_crypt_init(NULL)) { + return NULL; + } + + /* Create and initialize the worker threads */ + threads = create_worker_threads(ds_decrypt_encrypt_threads); + if (threads == NULL) { + msg("decrypt: failed to create worker threads.\n"); + return NULL; + } + + ctxt = (ds_ctxt_t *) my_malloc(sizeof(ds_ctxt_t) + + sizeof(ds_decrypt_ctxt_t), + MYF(MY_FAE)); + + decrypt_ctxt = (ds_decrypt_ctxt_t *) (ctxt + 1); + decrypt_ctxt->threads = threads; + decrypt_ctxt->nthreads = ds_decrypt_encrypt_threads; + + ctxt->ptr = decrypt_ctxt; + ctxt->root = my_strdup(root, MYF(MY_FAE)); + + return ctxt; +} + +static +ds_file_t * +decrypt_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat) +{ + ds_ctxt_t *dest_ctxt; + + ds_decrypt_ctxt_t *crypt_ctxt; + ds_decrypt_file_t *crypt_file; + + char new_name[FN_REFLEN]; + ds_file_t *file; + + xb_ad(ctxt->pipe_ctxt != NULL); + dest_ctxt = ctxt->pipe_ctxt; + + crypt_ctxt = (ds_decrypt_ctxt_t *) ctxt->ptr; + + + file = (ds_file_t *) my_malloc(sizeof(ds_file_t) + + sizeof(ds_decrypt_file_t), + MYF(MY_FAE|MY_ZEROFILL)); + + crypt_file = (ds_decrypt_file_t *) (file + 1); + + /* Remove the .xbcrypt extension from the filename */ + strncpy(new_name, path, FN_REFLEN); + new_name[strlen(new_name) - 8] = 0; + crypt_file->dest_file = ds_open(dest_ctxt, new_name, mystat); + if (crypt_file->dest_file == NULL) { + msg("decrypt: ds_open(\"%s\") failed.\n", new_name); + goto err; + } + + crypt_file->crypt_ctxt = crypt_ctxt; + crypt_file->buf = NULL; + crypt_file->buf_size = 0; + crypt_file->buf_len = 0; + + file->ptr = crypt_file; + file->path = crypt_file->dest_file->path; + + return file; + +err: + if (crypt_file->dest_file) { + ds_close(crypt_file->dest_file); + } + my_free(file); + return NULL; +} + +#define CHECK_BUF_SIZE(ptr, size, buf, len) \ + if (ptr + size - buf > (ssize_t) len) { \ + result = XB_CRYPT_READ_INCOMPLETE; \ + goto exit; \ + } + +static +xb_rcrypt_result_t +parse_xbcrypt_chunk(crypt_thread_ctxt_t *thd, const uchar *buf, size_t len, + size_t *bytes_processed) +{ + const uchar *ptr; + uint version; + ulong checksum, checksum_exp; + ulonglong tmp; + xb_rcrypt_result_t result = XB_CRYPT_READ_CHUNK; + + *bytes_processed = 0; + ptr = buf; + + CHECK_BUF_SIZE(ptr, XB_CRYPT_CHUNK_MAGIC_SIZE, buf, len); + if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC3, + XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) { + version = 3; + } else if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC2, + XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) { + version = 2; + } else if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC1, + XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) { + version = 1; + } else { + msg("%s:%s: wrong chunk magic at offset 0x%llx.\n", + my_progname, __FUNCTION__, thd->offset); + result = XB_CRYPT_READ_ERROR; + goto exit; + } + + ptr += XB_CRYPT_CHUNK_MAGIC_SIZE; + thd->offset += XB_CRYPT_CHUNK_MAGIC_SIZE; + + CHECK_BUF_SIZE(ptr, 8, buf, len); + tmp = uint8korr(ptr); /* reserved */ + ptr += 8; + thd->offset += 8; + + CHECK_BUF_SIZE(ptr, 8, buf, len); + tmp = uint8korr(ptr); /* original size */ + ptr += 8; + if (tmp > INT_MAX) { + msg("%s:%s: invalid original size at offset 0x%llx.\n", + my_progname, __FUNCTION__, thd->offset); + result = XB_CRYPT_READ_ERROR; + goto exit; + } + thd->offset += 8; + thd->to_len = (size_t)tmp; + + if (thd->to_size < thd->to_len + XB_CRYPT_HASH_LEN) { + thd->to = (uchar *) my_realloc( + thd->to, + thd->to_len + XB_CRYPT_HASH_LEN, + MYF(MY_FAE | MY_ALLOW_ZERO_PTR)); + thd->to_size = thd->to_len; + } + + CHECK_BUF_SIZE(ptr, 8, buf, len); + tmp = uint8korr(ptr); /* encrypted size */ + ptr += 8; + if (tmp > INT_MAX) { + msg("%s:%s: invalid encrypted size at offset 0x%llx.\n", + my_progname, __FUNCTION__, thd->offset); + result = XB_CRYPT_READ_ERROR; + goto exit; + } + thd->offset += 8; + thd->from_len = (size_t)tmp; + + xb_a(thd->from_len <= thd->to_len + XB_CRYPT_HASH_LEN); + + CHECK_BUF_SIZE(ptr, 4, buf, len); + checksum_exp = uint4korr(ptr); /* checksum */ + ptr += 4; + thd->offset += 4; + + /* iv size */ + if (version == 1) { + thd->iv_len = 0; + thd->iv = NULL; + } else { + CHECK_BUF_SIZE(ptr, 8, buf, len); + + tmp = uint8korr(ptr); + if (tmp > INT_MAX) { + msg("%s:%s: invalid iv size at offset 0x%llx.\n", + my_progname, __FUNCTION__, thd->offset); + result = XB_CRYPT_READ_ERROR; + goto exit; + } + ptr += 8; + thd->offset += 8; + thd->iv_len = (size_t)tmp; + } + + if (thd->iv_len > 0) { + CHECK_BUF_SIZE(ptr, thd->iv_len, buf, len); + thd->iv = ptr; + ptr += thd->iv_len; + } + + /* for version euqals 2 we need to read in the iv data but do not init + CTR with it */ + if (version == 2) { + thd->iv_len = 0; + thd->iv = 0; + } + + if (thd->from_len > 0) { + CHECK_BUF_SIZE(ptr, thd->from_len, buf, len); + thd->from = ptr; + ptr += thd->from_len; + } + + xb_ad(thd->from_len <= thd->to_len); + + checksum = crc32_iso3309(0, thd->from, thd->from_len); + if (checksum != checksum_exp) { + msg("%s:%s invalid checksum at offset 0x%llx, " + "expected 0x%lx, actual 0x%lx.\n", my_progname, + __FUNCTION__, thd->offset, checksum_exp, checksum); + result = XB_CRYPT_READ_ERROR; + goto exit; + } + + thd->offset += thd->from_len; + + thd->hash_appended = version > 2; + +exit: + + *bytes_processed = (size_t) (ptr - buf); + + return result; +} + +static +int +decrypt_write(ds_file_t *file, const void *buf, size_t len) +{ + ds_decrypt_file_t *crypt_file; + ds_decrypt_ctxt_t *crypt_ctxt; + crypt_thread_ctxt_t *threads; + crypt_thread_ctxt_t *thd; + uint nthreads; + uint i; + size_t bytes_processed; + xb_rcrypt_result_t parse_result = XB_CRYPT_READ_CHUNK; + my_bool err = FALSE; + + crypt_file = (ds_decrypt_file_t *) file->ptr; + crypt_ctxt = crypt_file->crypt_ctxt; + + threads = crypt_ctxt->threads; + nthreads = crypt_ctxt->nthreads; + + if (crypt_file->buf_len > 0) { + thd = threads; + + pthread_mutex_lock(&thd->ctrl_mutex); + + do { + if (parse_result == XB_CRYPT_READ_INCOMPLETE) { + crypt_file->buf_size = crypt_file->buf_size * 2; + crypt_file->buf = (uchar *) my_realloc( + crypt_file->buf, + crypt_file->buf_size, + MYF(MY_FAE|MY_ALLOW_ZERO_PTR)); + } + + memcpy(crypt_file->buf + crypt_file->buf_len, + buf, MY_MIN(crypt_file->buf_size - + crypt_file->buf_len, len)); + + parse_result = parse_xbcrypt_chunk( + thd, crypt_file->buf, + crypt_file->buf_size, &bytes_processed); + + if (parse_result == XB_CRYPT_READ_ERROR) { + pthread_mutex_unlock(&thd->ctrl_mutex); + return 1; + } + + } while (parse_result == XB_CRYPT_READ_INCOMPLETE && + crypt_file->buf_size < len); + + if (parse_result != XB_CRYPT_READ_CHUNK) { + msg("decrypt: incomplete data.\n"); + pthread_mutex_unlock(&thd->ctrl_mutex); + return 1; + } + + pthread_mutex_lock(&thd->data_mutex); + thd->data_avail = TRUE; + pthread_cond_signal(&thd->data_cond); + pthread_mutex_unlock(&thd->data_mutex); + + len -= bytes_processed - crypt_file->buf_len; + buf += bytes_processed - crypt_file->buf_len; + + /* reap */ + + pthread_mutex_lock(&thd->data_mutex); + while (thd->data_avail == TRUE) { + pthread_cond_wait(&thd->data_cond, + &thd->data_mutex); + } + + if (thd->failed) { + msg("decrypt: failed to decrypt chunk.\n"); + err = TRUE; + } + + xb_a(thd->to_len > 0); + + if (!err && + ds_write(crypt_file->dest_file, thd->to, thd->to_len)) { + msg("decrypt: write to destination failed.\n"); + err = TRUE; + } + + crypt_file->bytes_processed += thd->from_len; + + pthread_mutex_unlock(&thd->data_mutex); + pthread_mutex_unlock(&thd->ctrl_mutex); + + crypt_file->buf_len = 0; + + if (err) { + return 1; + } + } + + while (parse_result == XB_CRYPT_READ_CHUNK && len > 0) { + uint max_thread; + + for (i = 0; i < nthreads; i++) { + thd = threads + i; + + pthread_mutex_lock(&thd->ctrl_mutex); + + parse_result = parse_xbcrypt_chunk( + thd, buf, len, &bytes_processed); + + if (parse_result == XB_CRYPT_READ_ERROR) { + pthread_mutex_unlock(&thd->ctrl_mutex); + err = TRUE; + break; + } + + thd->parse_result = parse_result; + + if (parse_result != XB_CRYPT_READ_CHUNK) { + pthread_mutex_unlock(&thd->ctrl_mutex); + break; + } + + pthread_mutex_lock(&thd->data_mutex); + thd->data_avail = TRUE; + pthread_cond_signal(&thd->data_cond); + pthread_mutex_unlock(&thd->data_mutex); + + len -= bytes_processed; + buf += bytes_processed; + } + + max_thread = (i < nthreads) ? i : nthreads - 1; + + /* Reap and write decrypted data */ + for (i = 0; i <= max_thread; i++) { + thd = threads + i; + + if (thd->parse_result != XB_CRYPT_READ_CHUNK) { + break; + } + + pthread_mutex_lock(&thd->data_mutex); + while (thd->data_avail == TRUE) { + pthread_cond_wait(&thd->data_cond, + &thd->data_mutex); + } + + if (thd->failed) { + msg("decrypt: failed to decrypt chunk.\n"); + err = TRUE; + } + + xb_a(thd->to_len > 0); + + if (!err && ds_write(crypt_file->dest_file, thd->to, + thd->to_len)) { + msg("decrypt: write to destination failed.\n"); + err = TRUE; + } + + crypt_file->bytes_processed += thd->from_len; + + pthread_mutex_unlock(&thd->data_mutex); + pthread_mutex_unlock(&thd->ctrl_mutex); + } + + if (err) { + return 1; + } + } + + if (parse_result == XB_CRYPT_READ_INCOMPLETE && len > 0) { + crypt_file->buf_len = len; + if (crypt_file->buf_size < len) { + crypt_file->buf = (uchar *) my_realloc( + crypt_file->buf, + crypt_file->buf_len, + MYF(MY_FAE | MY_ALLOW_ZERO_PTR)); + crypt_file->buf_size = len; + } + memcpy(crypt_file->buf, buf, len); + } + + return 0; +} + +static +int +decrypt_close(ds_file_t *file) +{ + ds_decrypt_file_t *crypt_file; + ds_file_t *dest_file; + int rc = 0; + + crypt_file = (ds_decrypt_file_t *) file->ptr; + dest_file = crypt_file->dest_file; + + if (ds_close(dest_file)) { + rc = 1; + } + + my_free(crypt_file->buf); + my_free(file); + + return rc; +} + +static +void +decrypt_deinit(ds_ctxt_t *ctxt) +{ + ds_decrypt_ctxt_t *crypt_ctxt; + + xb_ad(ctxt->pipe_ctxt != NULL); + + crypt_ctxt = (ds_decrypt_ctxt_t *) ctxt->ptr; + + destroy_worker_threads(crypt_ctxt->threads, crypt_ctxt->nthreads); + + my_free(ctxt->root); + my_free(ctxt); +} + +static +crypt_thread_ctxt_t * +create_worker_threads(uint n) +{ + crypt_thread_ctxt_t *threads; + uint i; + + threads = (crypt_thread_ctxt_t *) + my_malloc(sizeof(crypt_thread_ctxt_t) * n, + MYF(MY_FAE | MY_ZEROFILL)); + + for (i = 0; i < n; i++) { + crypt_thread_ctxt_t *thd = threads + i; + + thd->num = i + 1; + + /* Initialize the control mutex and condition var */ + if (pthread_mutex_init(&thd->ctrl_mutex, NULL) || + pthread_cond_init(&thd->ctrl_cond, NULL)) { + goto err; + } + + /* Initialize and data mutex and condition var */ + if (pthread_mutex_init(&thd->data_mutex, NULL) || + pthread_cond_init(&thd->data_cond, NULL)) { + goto err; + } + + xb_crypt_cipher_open(&thd->cipher_handle); + + pthread_mutex_lock(&thd->ctrl_mutex); + + if (pthread_create(&thd->id, NULL, decrypt_worker_thread_func, + thd)) { + msg("decrypt: pthread_create() failed: " + "errno = %d\n", errno); + goto err; + } + } + + /* Wait for the threads to start */ + for (i = 0; i < n; i++) { + crypt_thread_ctxt_t *thd = threads + i; + + while (thd->started == FALSE) + pthread_cond_wait(&thd->ctrl_cond, &thd->ctrl_mutex); + pthread_mutex_unlock(&thd->ctrl_mutex); + } + + return threads; + +err: + return NULL; +} + +static +void +destroy_worker_threads(crypt_thread_ctxt_t *threads, uint n) +{ + uint i; + + for (i = 0; i < n; i++) { + crypt_thread_ctxt_t *thd = threads + i; + + pthread_mutex_lock(&thd->data_mutex); + threads[i].cancelled = TRUE; + pthread_cond_signal(&thd->data_cond); + pthread_mutex_unlock(&thd->data_mutex); + + pthread_join(thd->id, NULL); + + pthread_cond_destroy(&thd->data_cond); + pthread_mutex_destroy(&thd->data_mutex); + pthread_cond_destroy(&thd->ctrl_cond); + pthread_mutex_destroy(&thd->ctrl_mutex); + + xb_crypt_cipher_close(thd->cipher_handle); + + my_free(thd->to); + } + + my_free(threads); +} + +static +void * +decrypt_worker_thread_func(void *arg) +{ + crypt_thread_ctxt_t *thd = (crypt_thread_ctxt_t *) arg; + + pthread_mutex_lock(&thd->ctrl_mutex); + + pthread_mutex_lock(&thd->data_mutex); + + thd->started = TRUE; + pthread_cond_signal(&thd->ctrl_cond); + + pthread_mutex_unlock(&thd->ctrl_mutex); + + while (1) { + thd->data_avail = FALSE; + pthread_cond_signal(&thd->data_cond); + + while (!thd->data_avail && !thd->cancelled) { + pthread_cond_wait(&thd->data_cond, &thd->data_mutex); + } + + if (thd->cancelled) + break; + + if (xb_crypt_decrypt(thd->cipher_handle, thd->from, + thd->from_len, thd->to, &thd->to_len, + thd->iv, thd->iv_len, + thd->hash_appended)) { + thd->failed = TRUE; + continue; + } + + } + + pthread_mutex_unlock(&thd->data_mutex); + + return NULL; +} diff --git a/extra/mariabackup/ds_decrypt.h b/extra/mariabackup/ds_decrypt.h new file mode 100644 index 00000000000..3bb4de55f54 --- /dev/null +++ b/extra/mariabackup/ds_decrypt.h @@ -0,0 +1,30 @@ +/****************************************************** +Copyright (c) 2017 Percona LLC and/or its affiliates. + +Encryption interface for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#ifndef DS_DECRYPT_H +#define DS_DECRYPT_H + +#include "datasink.h" + +extern datasink_t datasink_decrypt; + +extern int ds_decrypt_encrypt_threads; + +#endif diff --git a/extra/mariabackup/ds_encrypt.c b/extra/mariabackup/ds_encrypt.c new file mode 100644 index 00000000000..576ea207eb1 --- /dev/null +++ b/extra/mariabackup/ds_encrypt.c @@ -0,0 +1,446 @@ +/****************************************************** +Copyright (c) 2013 Percona LLC and/or its affiliates. + +Encryption datasink implementation for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + + +#include <my_base.h> +#include "common.h" +#include "datasink.h" +#include "xbcrypt_common.h" +#ifdef HAVE_GRYPT +#include "xbcrypt.h" + +#define XB_CRYPT_CHUNK_SIZE ((size_t) (ds_encrypt_encrypt_chunk_size)) + +typedef struct { + pthread_t id; + uint num; + pthread_mutex_t ctrl_mutex; + pthread_cond_t ctrl_cond; + pthread_mutex_t data_mutex; + pthread_cond_t data_cond; + my_bool started; + my_bool data_avail; + my_bool cancelled; + const uchar *from; + size_t from_len; + uchar *to; + uchar *iv; + size_t to_len; + gcry_cipher_hd_t cipher_handle; +} crypt_thread_ctxt_t; + +typedef struct { + crypt_thread_ctxt_t *threads; + uint nthreads; +} ds_encrypt_ctxt_t; + +typedef struct { + xb_wcrypt_t *xbcrypt_file; + ds_encrypt_ctxt_t *crypt_ctxt; + size_t bytes_processed; + ds_file_t *dest_file; +} ds_encrypt_file_t; + +/* Encryption options */ +uint ds_encrypt_encrypt_threads; +ulonglong ds_encrypt_encrypt_chunk_size; + +static ds_ctxt_t *encrypt_init(const char *root); +static ds_file_t *encrypt_open(ds_ctxt_t *ctxt, const char *path, + MY_STAT *mystat); +static int encrypt_write(ds_file_t *file, const void *buf, size_t len); +static int encrypt_close(ds_file_t *file); +static void encrypt_deinit(ds_ctxt_t *ctxt); + +datasink_t datasink_encrypt = { + &encrypt_init, + &encrypt_open, + &encrypt_write, + &encrypt_close, + &encrypt_deinit +}; + +static crypt_thread_ctxt_t *create_worker_threads(uint n); +static void destroy_worker_threads(crypt_thread_ctxt_t *threads, uint n); +static void *encrypt_worker_thread_func(void *arg); + +static uint encrypt_iv_len = 0; + +static +ssize_t +my_xb_crypt_write_callback(void *userdata, const void *buf, size_t len) +{ + ds_encrypt_file_t *encrypt_file; + + encrypt_file = (ds_encrypt_file_t *) userdata; + + xb_ad(encrypt_file != NULL); + xb_ad(encrypt_file->dest_file != NULL); + + if (!ds_write(encrypt_file->dest_file, buf, len)) { + return len; + } + return -1; +} + +static +ds_ctxt_t * +encrypt_init(const char *root) +{ + ds_ctxt_t *ctxt; + ds_encrypt_ctxt_t *encrypt_ctxt; + crypt_thread_ctxt_t *threads; + + if (xb_crypt_init(&encrypt_iv_len)) { + return NULL; + } + + /* Create and initialize the worker threads */ + threads = create_worker_threads(ds_encrypt_encrypt_threads); + if (threads == NULL) { + msg("encrypt: failed to create worker threads.\n"); + return NULL; + } + + ctxt = (ds_ctxt_t *) my_malloc(sizeof(ds_ctxt_t) + + sizeof(ds_encrypt_ctxt_t), + MYF(MY_FAE)); + + encrypt_ctxt = (ds_encrypt_ctxt_t *) (ctxt + 1); + encrypt_ctxt->threads = threads; + encrypt_ctxt->nthreads = ds_encrypt_encrypt_threads; + + ctxt->ptr = encrypt_ctxt; + ctxt->root = my_strdup(root, MYF(MY_FAE)); + + return ctxt; +} + +static +ds_file_t * +encrypt_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat) +{ + ds_ctxt_t *dest_ctxt; + + ds_encrypt_ctxt_t *crypt_ctxt; + ds_encrypt_file_t *crypt_file; + + char new_name[FN_REFLEN]; + ds_file_t *file; + + xb_ad(ctxt->pipe_ctxt != NULL); + dest_ctxt = ctxt->pipe_ctxt; + + crypt_ctxt = (ds_encrypt_ctxt_t *) ctxt->ptr; + + + file = (ds_file_t *) my_malloc(sizeof(ds_file_t) + + sizeof(ds_encrypt_file_t), + MYF(MY_FAE|MY_ZEROFILL)); + + crypt_file = (ds_encrypt_file_t *) (file + 1); + + /* Append the .xbcrypt extension to the filename */ + fn_format(new_name, path, "", ".xbcrypt", MYF(MY_APPEND_EXT)); + crypt_file->dest_file = ds_open(dest_ctxt, new_name, mystat); + if (crypt_file->dest_file == NULL) { + msg("encrypt: ds_open(\"%s\") failed.\n", new_name); + goto err; + } + + crypt_file->crypt_ctxt = crypt_ctxt; + crypt_file->xbcrypt_file = xb_crypt_write_open(crypt_file, + my_xb_crypt_write_callback); + + if (crypt_file->xbcrypt_file == NULL) { + msg("encrypt: xb_crypt_write_open() failed.\n"); + goto err; + } + + + file->ptr = crypt_file; + file->path = crypt_file->dest_file->path; + + return file; + +err: + if (crypt_file->dest_file) { + ds_close(crypt_file->dest_file); + } + my_free(file); + return NULL; +} + +static +int +encrypt_write(ds_file_t *file, const void *buf, size_t len) +{ + ds_encrypt_file_t *crypt_file; + ds_encrypt_ctxt_t *crypt_ctxt; + crypt_thread_ctxt_t *threads; + crypt_thread_ctxt_t *thd; + uint nthreads; + uint i; + const uchar *ptr; + + crypt_file = (ds_encrypt_file_t *) file->ptr; + crypt_ctxt = crypt_file->crypt_ctxt; + + threads = crypt_ctxt->threads; + nthreads = crypt_ctxt->nthreads; + + ptr = (const uchar *) buf; + while (len > 0) { + uint max_thread; + + /* Send data to worker threads for encryption */ + for (i = 0; i < nthreads; i++) { + size_t chunk_len; + + thd = threads + i; + + pthread_mutex_lock(&thd->ctrl_mutex); + + chunk_len = (len > XB_CRYPT_CHUNK_SIZE) ? + XB_CRYPT_CHUNK_SIZE : len; + thd->from = ptr; + thd->from_len = chunk_len; + + pthread_mutex_lock(&thd->data_mutex); + thd->data_avail = TRUE; + pthread_cond_signal(&thd->data_cond); + pthread_mutex_unlock(&thd->data_mutex); + + len -= chunk_len; + if (len == 0) { + break; + } + ptr += chunk_len; + } + + max_thread = (i < nthreads) ? i : nthreads - 1; + + /* Reap and stream the encrypted data */ + for (i = 0; i <= max_thread; i++) { + thd = threads + i; + + pthread_mutex_lock(&thd->data_mutex); + while (thd->data_avail == TRUE) { + pthread_cond_wait(&thd->data_cond, + &thd->data_mutex); + } + + xb_a(threads[i].to_len > 0); + + if (xb_crypt_write_chunk(crypt_file->xbcrypt_file, + threads[i].to, + threads[i].from_len + + XB_CRYPT_HASH_LEN, + threads[i].to_len, + threads[i].iv, + encrypt_iv_len)) { + msg("encrypt: write to the destination file " + "failed.\n"); + return 1; + } + + crypt_file->bytes_processed += threads[i].from_len; + + pthread_mutex_unlock(&threads[i].data_mutex); + pthread_mutex_unlock(&threads[i].ctrl_mutex); + } + } + + return 0; +} + +static +int +encrypt_close(ds_file_t *file) +{ + ds_encrypt_file_t *crypt_file; + ds_file_t *dest_file; + int rc = 0; + + crypt_file = (ds_encrypt_file_t *) file->ptr; + dest_file = crypt_file->dest_file; + + rc = xb_crypt_write_close(crypt_file->xbcrypt_file); + + if (ds_close(dest_file)) { + rc = 1; + } + + my_free(file); + + return rc; +} + +static +void +encrypt_deinit(ds_ctxt_t *ctxt) +{ + ds_encrypt_ctxt_t *crypt_ctxt; + + xb_ad(ctxt->pipe_ctxt != NULL); + + crypt_ctxt = (ds_encrypt_ctxt_t *) ctxt->ptr; + + destroy_worker_threads(crypt_ctxt->threads, crypt_ctxt->nthreads); + + my_free(ctxt->root); + my_free(ctxt); +} + +static +crypt_thread_ctxt_t * +create_worker_threads(uint n) +{ + crypt_thread_ctxt_t *threads; + uint i; + + threads = (crypt_thread_ctxt_t *) + my_malloc(sizeof(crypt_thread_ctxt_t) * n, MYF(MY_FAE)); + + for (i = 0; i < n; i++) { + crypt_thread_ctxt_t *thd = threads + i; + + thd->num = i + 1; + thd->started = FALSE; + thd->cancelled = FALSE; + thd->data_avail = FALSE; + + thd->to = (uchar *) my_malloc(XB_CRYPT_CHUNK_SIZE + + XB_CRYPT_HASH_LEN, MYF(MY_FAE)); + + thd->iv = (uchar *) my_malloc(encrypt_iv_len, MYF(MY_FAE)); + + /* Initialize the control mutex and condition var */ + if (pthread_mutex_init(&thd->ctrl_mutex, NULL) || + pthread_cond_init(&thd->ctrl_cond, NULL)) { + goto err; + } + + /* Initialize and data mutex and condition var */ + if (pthread_mutex_init(&thd->data_mutex, NULL) || + pthread_cond_init(&thd->data_cond, NULL)) { + goto err; + } + + if (xb_crypt_cipher_open(&thd->cipher_handle)) { + goto err; + } + + pthread_mutex_lock(&thd->ctrl_mutex); + + if (pthread_create(&thd->id, NULL, encrypt_worker_thread_func, + thd)) { + msg("encrypt: pthread_create() failed: " + "errno = %d\n", errno); + goto err; + } + } + + /* Wait for the threads to start */ + for (i = 0; i < n; i++) { + crypt_thread_ctxt_t *thd = threads + i; + + while (thd->started == FALSE) + pthread_cond_wait(&thd->ctrl_cond, &thd->ctrl_mutex); + pthread_mutex_unlock(&thd->ctrl_mutex); + } + + return threads; + +err: + return NULL; +} + +static +void +destroy_worker_threads(crypt_thread_ctxt_t *threads, uint n) +{ + uint i; + + for (i = 0; i < n; i++) { + crypt_thread_ctxt_t *thd = threads + i; + + pthread_mutex_lock(&thd->data_mutex); + threads[i].cancelled = TRUE; + pthread_cond_signal(&thd->data_cond); + pthread_mutex_unlock(&thd->data_mutex); + + pthread_join(thd->id, NULL); + + pthread_cond_destroy(&thd->data_cond); + pthread_mutex_destroy(&thd->data_mutex); + pthread_cond_destroy(&thd->ctrl_cond); + pthread_mutex_destroy(&thd->ctrl_mutex); + + xb_crypt_cipher_close(thd->cipher_handle); + + my_free(thd->to); + my_free(thd->iv); + } + + my_free(threads); +} + +static +void * +encrypt_worker_thread_func(void *arg) +{ + crypt_thread_ctxt_t *thd = (crypt_thread_ctxt_t *) arg; + + pthread_mutex_lock(&thd->ctrl_mutex); + + pthread_mutex_lock(&thd->data_mutex); + + thd->started = TRUE; + pthread_cond_signal(&thd->ctrl_cond); + + pthread_mutex_unlock(&thd->ctrl_mutex); + + while (1) { + thd->data_avail = FALSE; + pthread_cond_signal(&thd->data_cond); + + while (!thd->data_avail && !thd->cancelled) { + pthread_cond_wait(&thd->data_cond, &thd->data_mutex); + } + + if (thd->cancelled) + break; + + thd->to_len = thd->from_len; + + if (xb_crypt_encrypt(thd->cipher_handle, thd->from, + thd->from_len, thd->to, &thd->to_len, + thd->iv)) { + thd->to_len = 0; + continue; + } + } + + pthread_mutex_unlock(&thd->data_mutex); + + return NULL; +} +#endif /* HAVE_GCRYPT*/ diff --git a/extra/mariabackup/ds_encrypt.h b/extra/mariabackup/ds_encrypt.h new file mode 100644 index 00000000000..c4d8d7f8427 --- /dev/null +++ b/extra/mariabackup/ds_encrypt.h @@ -0,0 +1,33 @@ +/****************************************************** +Copyright (c) 2013 Percona LLC and/or its affiliates. + +Encryption interface for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#ifndef DS_ENCRYPT_H +#define DS_ENCRYPT_H + +#include "datasink.h" +#ifdef HAVE_GCRYPT +extern datasink_t datasink_encrypt; +#endif +/* Encryption options */ +extern uint ds_encrypt_encrypt_threads; +extern ulonglong ds_encrypt_encrypt_chunk_size; + + +#endif diff --git a/extra/mariabackup/ds_local.c b/extra/mariabackup/ds_local.c new file mode 100644 index 00000000000..3e2b1e0129b --- /dev/null +++ b/extra/mariabackup/ds_local.c @@ -0,0 +1,151 @@ +/****************************************************** +Copyright (c) 2011-2013 Percona LLC and/or its affiliates. + +Local datasink implementation for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#include <mysql_version.h> +#include <my_base.h> +#include <mysys_err.h> +#include "common.h" +#include "datasink.h" + +typedef struct { + File fd; +} ds_local_file_t; + +static ds_ctxt_t *local_init(const char *root); +static ds_file_t *local_open(ds_ctxt_t *ctxt, const char *path, + MY_STAT *mystat); +static int local_write(ds_file_t *file, const void *buf, size_t len); +static int local_close(ds_file_t *file); +static void local_deinit(ds_ctxt_t *ctxt); + +datasink_t datasink_local = { + &local_init, + &local_open, + &local_write, + &local_close, + &local_deinit +}; + +static +ds_ctxt_t * +local_init(const char *root) +{ + ds_ctxt_t *ctxt; + + if (my_mkdir(root, 0777, MYF(0)) < 0 + && my_errno != EEXIST && my_errno != EISDIR) + { + char errbuf[MYSYS_STRERROR_SIZE]; + my_strerror(errbuf, sizeof(errbuf),my_errno); + my_error(EE_CANT_MKDIR, MYF(ME_BELL | ME_WAITTANG), + root, my_errno,errbuf, my_errno); + return NULL; + } + + ctxt = my_malloc(sizeof(ds_ctxt_t), MYF(MY_FAE)); + + ctxt->root = my_strdup(root, MYF(MY_FAE)); + + return ctxt; +} + +static +ds_file_t * +local_open(ds_ctxt_t *ctxt, const char *path, + MY_STAT *mystat __attribute__((unused))) +{ + char fullpath[FN_REFLEN]; + char dirpath[FN_REFLEN]; + size_t dirpath_len; + size_t path_len; + ds_local_file_t *local_file; + ds_file_t *file; + File fd; + + fn_format(fullpath, path, ctxt->root, "", MYF(MY_RELATIVE_PATH)); + + /* Create the directory if needed */ + dirname_part(dirpath, fullpath, &dirpath_len); + if (my_mkdir(dirpath, 0777, MYF(0)) < 0 && my_errno != EEXIST) { + char errbuf[MYSYS_STRERROR_SIZE]; + my_strerror(errbuf, sizeof(errbuf), my_errno); + my_error(EE_CANT_MKDIR, MYF(ME_BELL | ME_WAITTANG), + dirpath, my_errno, errbuf); + return NULL; + } + + fd = my_create(fullpath, 0, O_WRONLY | O_BINARY | O_EXCL | O_NOFOLLOW, + MYF(MY_WME)); + if (fd < 0) { + return NULL; + } + + path_len = strlen(fullpath) + 1; /* terminating '\0' */ + + file = (ds_file_t *) my_malloc(sizeof(ds_file_t) + + sizeof(ds_local_file_t) + + path_len, + MYF(MY_FAE)); + local_file = (ds_local_file_t *) (file + 1); + + local_file->fd = fd; + + file->path = (char *) local_file + sizeof(ds_local_file_t); + memcpy(file->path, fullpath, path_len); + + file->ptr = local_file; + + return file; +} + +static +int +local_write(ds_file_t *file, const void *buf, size_t len) +{ + File fd = ((ds_local_file_t *) file->ptr)->fd; + + if (!my_write(fd, buf, len, MYF(MY_WME | MY_NABP))) { + posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED); + return 0; + } + + return 1; +} + +static +int +local_close(ds_file_t *file) +{ + File fd = ((ds_local_file_t *) file->ptr)->fd; + + my_free(file); + + my_sync(fd, MYF(MY_WME)); + + return my_close(fd, MYF(MY_WME)); +} + +static +void +local_deinit(ds_ctxt_t *ctxt) +{ + my_free(ctxt->root); + my_free(ctxt); +} diff --git a/extra/mariabackup/ds_local.h b/extra/mariabackup/ds_local.h new file mode 100644 index 00000000000..b0f0f04030c --- /dev/null +++ b/extra/mariabackup/ds_local.h @@ -0,0 +1,28 @@ +/****************************************************** +Copyright (c) 2011-2013 Percona LLC and/or its affiliates. + +Local datasink interface for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#ifndef DS_LOCAL_H +#define DS_LOCAL_H + +#include "datasink.h" + +extern datasink_t datasink_local; + +#endif diff --git a/extra/mariabackup/ds_stdout.c b/extra/mariabackup/ds_stdout.c new file mode 100644 index 00000000000..91a514ddf64 --- /dev/null +++ b/extra/mariabackup/ds_stdout.c @@ -0,0 +1,121 @@ +/****************************************************** +Copyright (c) 2013 Percona LLC and/or its affiliates. + +Local datasink implementation for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#include <my_base.h> +#include <mysys_err.h> +#include "common.h" +#include "datasink.h" + +typedef struct { + File fd; +} ds_stdout_file_t; + +static ds_ctxt_t *stdout_init(const char *root); +static ds_file_t *stdout_open(ds_ctxt_t *ctxt, const char *path, + MY_STAT *mystat); +static int stdout_write(ds_file_t *file, const void *buf, size_t len); +static int stdout_close(ds_file_t *file); +static void stdout_deinit(ds_ctxt_t *ctxt); + +datasink_t datasink_stdout = { + &stdout_init, + &stdout_open, + &stdout_write, + &stdout_close, + &stdout_deinit +}; + +static +ds_ctxt_t * +stdout_init(const char *root) +{ + ds_ctxt_t *ctxt; + + ctxt = my_malloc(sizeof(ds_ctxt_t), MYF(MY_FAE)); + + ctxt->root = my_strdup(root, MYF(MY_FAE)); + + return ctxt; +} + +static +ds_file_t * +stdout_open(ds_ctxt_t *ctxt __attribute__((unused)), + const char *path __attribute__((unused)), + MY_STAT *mystat __attribute__((unused))) +{ + ds_stdout_file_t *stdout_file; + ds_file_t *file; + size_t pathlen; + const char *fullpath = "<STDOUT>"; + + pathlen = strlen(fullpath) + 1; + + file = (ds_file_t *) my_malloc(sizeof(ds_file_t) + + sizeof(ds_stdout_file_t) + + pathlen, + MYF(MY_FAE)); + stdout_file = (ds_stdout_file_t *) (file + 1); + + +#ifdef __WIN__ + setmode(fileno(stdout), _O_BINARY); +#endif + + stdout_file->fd = my_fileno(stdout); + + file->path = (char *) stdout_file + sizeof(ds_stdout_file_t); + memcpy(file->path, fullpath, pathlen); + + file->ptr = stdout_file; + + return file; +} + +static +int +stdout_write(ds_file_t *file, const void *buf, size_t len) +{ + File fd = ((ds_stdout_file_t *) file->ptr)->fd; + + if (!my_write(fd, buf, len, MYF(MY_WME | MY_NABP))) { + posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED); + return 0; + } + + return 1; +} + +static +int +stdout_close(ds_file_t *file) +{ + my_free(file); + + return 1; +} + +static +void +stdout_deinit(ds_ctxt_t *ctxt) +{ + my_free(ctxt->root); + my_free(ctxt); +} diff --git a/extra/mariabackup/ds_stdout.h b/extra/mariabackup/ds_stdout.h new file mode 100644 index 00000000000..58940264fef --- /dev/null +++ b/extra/mariabackup/ds_stdout.h @@ -0,0 +1,28 @@ +/****************************************************** +Copyright (c) 2013 Percona LLC and/or its affiliates. + +Local datasink interface for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#ifndef DS_STDOUT_H +#define DS_STDOUT_H + +#include "datasink.h" + +extern datasink_t datasink_stdout; + +#endif diff --git a/extra/mariabackup/ds_tmpfile.c b/extra/mariabackup/ds_tmpfile.c new file mode 100644 index 00000000000..b039d83ba03 --- /dev/null +++ b/extra/mariabackup/ds_tmpfile.c @@ -0,0 +1,247 @@ +/****************************************************** +Copyright (c) 2012 Percona LLC and/or its affiliates. + +tmpfile datasink for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +/* Do all writes to temporary files first, then pipe them to the specified +datasink in a serialized way in deinit(). */ + +#include <my_base.h> +#include "common.h" +#include "datasink.h" + +typedef struct { + pthread_mutex_t mutex; + LIST *file_list; +} ds_tmpfile_ctxt_t; + +typedef struct { + LIST list; + File fd; + char *orig_path; + MY_STAT mystat; + ds_file_t *file; +} ds_tmp_file_t; + +static ds_ctxt_t *tmpfile_init(const char *root); +static ds_file_t *tmpfile_open(ds_ctxt_t *ctxt, const char *path, + MY_STAT *mystat); +static int tmpfile_write(ds_file_t *file, const void *buf, size_t len); +static int tmpfile_close(ds_file_t *file); +static void tmpfile_deinit(ds_ctxt_t *ctxt); + +datasink_t datasink_tmpfile = { + &tmpfile_init, + &tmpfile_open, + &tmpfile_write, + &tmpfile_close, + &tmpfile_deinit +}; + + +static ds_ctxt_t * +tmpfile_init(const char *root) +{ + ds_ctxt_t *ctxt; + ds_tmpfile_ctxt_t *tmpfile_ctxt; + + ctxt = my_malloc(sizeof(ds_ctxt_t) + sizeof(ds_tmpfile_ctxt_t), + MYF(MY_FAE)); + tmpfile_ctxt = (ds_tmpfile_ctxt_t *) (ctxt + 1); + tmpfile_ctxt->file_list = NULL; + if (pthread_mutex_init(&tmpfile_ctxt->mutex, NULL)) { + + my_free(ctxt); + return NULL; + } + + ctxt->ptr = tmpfile_ctxt; + ctxt->root = my_strdup(root, MYF(MY_FAE)); + + return ctxt; +} + +static ds_file_t * +tmpfile_open(ds_ctxt_t *ctxt, const char *path, + MY_STAT *mystat) +{ + ds_tmpfile_ctxt_t *tmpfile_ctxt; + char tmp_path[FN_REFLEN]; + ds_tmp_file_t *tmp_file; + ds_file_t *file; + size_t path_len; + File fd; + + /* Create a temporary file in tmpdir. The file will be automatically + removed on close. Code copied from mysql_tmpfile(). */ + fd = create_temp_file(tmp_path,xtrabackup_tmpdir, + "xbtemp", +#ifdef __WIN__ + O_BINARY | O_TRUNC | O_SEQUENTIAL | + O_TEMPORARY | O_SHORT_LIVED | +#endif /* __WIN__ */ + O_CREAT | O_EXCL | O_RDWR, + MYF(MY_WME)); + +#ifndef __WIN__ + if (fd >= 0) { + /* On Windows, open files cannot be removed, but files can be + created with the O_TEMPORARY flag to the same effect + ("delete on close"). */ + unlink(tmp_path); + } +#endif /* !__WIN__ */ + + if (fd < 0) { + return NULL; + } + + path_len = strlen(path) + 1; /* terminating '\0' */ + + file = (ds_file_t *) my_malloc(sizeof(ds_file_t) + + sizeof(ds_tmp_file_t) + path_len, + MYF(MY_FAE)); + + tmp_file = (ds_tmp_file_t *) (file + 1); + tmp_file->file = file; + memcpy(&tmp_file->mystat, mystat, sizeof(MY_STAT)); + /* Save a copy of 'path', since it may not be accessible later */ + tmp_file->orig_path = (char *) tmp_file + sizeof(ds_tmp_file_t); + + tmp_file->fd = fd; + memcpy(tmp_file->orig_path, path, path_len); + + /* Store the real temporary file name in file->path */ + file->path = my_strdup(tmp_path, MYF(MY_FAE)); + file->ptr = tmp_file; + + /* Store the file object in the list to be piped later */ + tmpfile_ctxt = (ds_tmpfile_ctxt_t *) ctxt->ptr; + tmp_file->list.data = tmp_file; + + pthread_mutex_lock(&tmpfile_ctxt->mutex); + tmpfile_ctxt->file_list = list_add(tmpfile_ctxt->file_list, + &tmp_file->list); + pthread_mutex_unlock(&tmpfile_ctxt->mutex); + + return file; +} + +static int +tmpfile_write(ds_file_t *file, const void *buf, size_t len) +{ + File fd = ((ds_tmp_file_t *) file->ptr)->fd; + + if (!my_write(fd, buf, len, MYF(MY_WME | MY_NABP))) { + posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED); + return 0; + } + + return 1; +} + +static int +tmpfile_close(ds_file_t *file) +{ + /* Do nothing -- we will close (and thus remove) the file after piping + it to the destination datasink in tmpfile_deinit(). */ + + my_free(file->path); + + return 0; +} + +static void +tmpfile_deinit(ds_ctxt_t *ctxt) +{ + LIST *list; + ds_tmpfile_ctxt_t *tmpfile_ctxt; + MY_STAT mystat; + ds_tmp_file_t *tmp_file; + ds_file_t *dst_file; + ds_ctxt_t *pipe_ctxt; + void *buf = NULL; + const size_t buf_size = 10 * 1024 * 1024; + size_t bytes; + size_t offset; + + pipe_ctxt = ctxt->pipe_ctxt; + xb_a(pipe_ctxt != NULL); + + buf = my_malloc(buf_size, MYF(MY_FAE)); + + tmpfile_ctxt = (ds_tmpfile_ctxt_t *) ctxt->ptr; + list = tmpfile_ctxt->file_list; + + /* Walk the files in the order they have been added */ + list = list_reverse(list); + while (list != NULL) { + tmp_file = list->data; + /* Stat the file to replace size and mtime on the original + * mystat struct */ + if (my_fstat(tmp_file->fd, &mystat, MYF(0))) { + msg("error: my_fstat() failed.\n"); + exit(EXIT_FAILURE); + } + tmp_file->mystat.st_size = mystat.st_size; + tmp_file->mystat.st_mtime = mystat.st_mtime; + + dst_file = ds_open(pipe_ctxt, tmp_file->orig_path, + &tmp_file->mystat); + if (dst_file == NULL) { + msg("error: could not stream a temporary file to " + "'%s'\n", tmp_file->orig_path); + exit(EXIT_FAILURE); + } + + /* copy to the destination datasink */ + posix_fadvise(tmp_file->fd, 0, 0, POSIX_FADV_SEQUENTIAL); + if (my_seek(tmp_file->fd, 0, SEEK_SET, MYF(0)) == + MY_FILEPOS_ERROR) { + msg("error: my_seek() failed for '%s', errno = %d.\n", + tmp_file->file->path, my_errno); + exit(EXIT_FAILURE); + } + offset = 0; + while ((bytes = my_read(tmp_file->fd, buf, buf_size, + MYF(MY_WME))) > 0) { + posix_fadvise(tmp_file->fd, offset, buf_size, POSIX_FADV_DONTNEED); + offset += buf_size; + if (ds_write(dst_file, buf, bytes)) { + msg("error: cannot write to stream for '%s'.\n", + tmp_file->orig_path); + exit(EXIT_FAILURE); + } + } + if (bytes == (size_t) -1) { + exit(EXIT_FAILURE); + } + + my_close(tmp_file->fd, MYF(MY_WME)); + ds_close(dst_file); + + list = list_rest(list); + my_free(tmp_file->file); + } + + pthread_mutex_destroy(&tmpfile_ctxt->mutex); + + my_free(buf); + my_free(ctxt->root); + my_free(ctxt); +} diff --git a/extra/mariabackup/ds_tmpfile.h b/extra/mariabackup/ds_tmpfile.h new file mode 100644 index 00000000000..c21f1a3f0b5 --- /dev/null +++ b/extra/mariabackup/ds_tmpfile.h @@ -0,0 +1,30 @@ +/****************************************************** +Copyright (c) 2012 Percona LLC and/or its affiliates. + +tmpfile datasink for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#ifndef DS_TMPFILE_H +#define DS_TMPFILE_H + +#include "datasink.h" + +extern datasink_t datasink_tmpfile; + +extern MY_TMPDIR mysql_tmpdir_list; + +#endif diff --git a/extra/mariabackup/ds_xbstream.c b/extra/mariabackup/ds_xbstream.c new file mode 100644 index 00000000000..42924a72d7f --- /dev/null +++ b/extra/mariabackup/ds_xbstream.c @@ -0,0 +1,223 @@ +/****************************************************** +Copyright (c) 2011-2013 Percona LLC and/or its affiliates. + +Streaming implementation for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#include <mysql_version.h> +#include <my_base.h> +#include "common.h" +#include "datasink.h" +#include "xbstream.h" + +typedef struct { + xb_wstream_t *xbstream; + ds_file_t *dest_file; + pthread_mutex_t mutex; +} ds_stream_ctxt_t; + +typedef struct { + xb_wstream_file_t *xbstream_file; + ds_stream_ctxt_t *stream_ctxt; +} ds_stream_file_t; + +/*********************************************************************** +General streaming interface */ + +static ds_ctxt_t *xbstream_init(const char *root); +static ds_file_t *xbstream_open(ds_ctxt_t *ctxt, const char *path, + MY_STAT *mystat); +static int xbstream_write(ds_file_t *file, const void *buf, size_t len); +static int xbstream_close(ds_file_t *file); +static void xbstream_deinit(ds_ctxt_t *ctxt); + +datasink_t datasink_xbstream = { + &xbstream_init, + &xbstream_open, + &xbstream_write, + &xbstream_close, + &xbstream_deinit +}; + +static +ssize_t +my_xbstream_write_callback(xb_wstream_file_t *f __attribute__((unused)), + void *userdata, const void *buf, size_t len) +{ + ds_stream_ctxt_t *stream_ctxt; + + stream_ctxt = (ds_stream_ctxt_t *) userdata; + + xb_ad(stream_ctxt != NULL); + xb_ad(stream_ctxt->dest_file != NULL); + + if (!ds_write(stream_ctxt->dest_file, buf, len)) { + return len; + } + return -1; +} + +static +ds_ctxt_t * +xbstream_init(const char *root __attribute__((unused))) +{ + ds_ctxt_t *ctxt; + ds_stream_ctxt_t *stream_ctxt; + xb_wstream_t *xbstream; + + ctxt = my_malloc(sizeof(ds_ctxt_t) + sizeof(ds_stream_ctxt_t), + MYF(MY_FAE)); + stream_ctxt = (ds_stream_ctxt_t *)(ctxt + 1); + + if (pthread_mutex_init(&stream_ctxt->mutex, NULL)) { + msg("xbstream_init: pthread_mutex_init() failed.\n"); + goto err; + } + + xbstream = xb_stream_write_new(); + if (xbstream == NULL) { + msg("xb_stream_write_new() failed.\n"); + goto err; + } + stream_ctxt->xbstream = xbstream; + stream_ctxt->dest_file = NULL; + + ctxt->ptr = stream_ctxt; + + return ctxt; + +err: + my_free(ctxt); + return NULL; +} + +static +ds_file_t * +xbstream_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat) +{ + ds_file_t *file; + ds_stream_file_t *stream_file; + ds_stream_ctxt_t *stream_ctxt; + ds_ctxt_t *dest_ctxt; + xb_wstream_t *xbstream; + xb_wstream_file_t *xbstream_file; + + + xb_ad(ctxt->pipe_ctxt != NULL); + dest_ctxt = ctxt->pipe_ctxt; + + stream_ctxt = (ds_stream_ctxt_t *) ctxt->ptr; + + pthread_mutex_lock(&stream_ctxt->mutex); + if (stream_ctxt->dest_file == NULL) { + stream_ctxt->dest_file = ds_open(dest_ctxt, path, mystat); + if (stream_ctxt->dest_file == NULL) { + return NULL; + } + } + pthread_mutex_unlock(&stream_ctxt->mutex); + + file = (ds_file_t *) my_malloc(sizeof(ds_file_t) + + sizeof(ds_stream_file_t), + MYF(MY_FAE)); + stream_file = (ds_stream_file_t *) (file + 1); + + xbstream = stream_ctxt->xbstream; + + xbstream_file = xb_stream_write_open(xbstream, path, mystat, + stream_ctxt, + my_xbstream_write_callback); + + if (xbstream_file == NULL) { + msg("xb_stream_write_open() failed.\n"); + goto err; + } + + stream_file->xbstream_file = xbstream_file; + stream_file->stream_ctxt = stream_ctxt; + file->ptr = stream_file; + file->path = stream_ctxt->dest_file->path; + + return file; + +err: + if (stream_ctxt->dest_file) { + ds_close(stream_ctxt->dest_file); + stream_ctxt->dest_file = NULL; + } + my_free(file); + + return NULL; +} + +static +int +xbstream_write(ds_file_t *file, const void *buf, size_t len) +{ + ds_stream_file_t *stream_file; + xb_wstream_file_t *xbstream_file; + + + stream_file = (ds_stream_file_t *) file->ptr; + + xbstream_file = stream_file->xbstream_file; + + if (xb_stream_write_data(xbstream_file, buf, len)) { + msg("xb_stream_write_data() failed.\n"); + return 1; + } + + return 0; +} + +static +int +xbstream_close(ds_file_t *file) +{ + ds_stream_file_t *stream_file; + int rc = 0; + + stream_file = (ds_stream_file_t *)file->ptr; + + rc = xb_stream_write_close(stream_file->xbstream_file); + + my_free(file); + + return rc; +} + +static +void +xbstream_deinit(ds_ctxt_t *ctxt) +{ + ds_stream_ctxt_t *stream_ctxt; + + stream_ctxt = (ds_stream_ctxt_t *) ctxt->ptr; + + if (xb_stream_write_done(stream_ctxt->xbstream)) { + msg("xb_stream_done() failed.\n"); + } + + if (stream_ctxt->dest_file) { + ds_close(stream_ctxt->dest_file); + stream_ctxt->dest_file = NULL; + } + + pthread_mutex_destroy(&stream_ctxt->mutex); + + my_free(ctxt); +} diff --git a/extra/mariabackup/ds_xbstream.h b/extra/mariabackup/ds_xbstream.h new file mode 100644 index 00000000000..30f34ac8318 --- /dev/null +++ b/extra/mariabackup/ds_xbstream.h @@ -0,0 +1,28 @@ +/****************************************************** +Copyright (c) 2011-2013 Percona LLC and/or its affiliates. + +Streaming interface for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#ifndef DS_XBSTREAM_H +#define DS_XBSTREAM_H + +#include "datasink.h" + +extern datasink_t datasink_xbstream; + +#endif diff --git a/extra/mariabackup/encryption_plugin.cc b/extra/mariabackup/encryption_plugin.cc new file mode 100644 index 00000000000..9f2782d89a1 --- /dev/null +++ b/extra/mariabackup/encryption_plugin.cc @@ -0,0 +1,157 @@ +#include <mysqld.h> +#include <mysql.h> +#include <xtrabackup.h> +#include <encryption_plugin.h> +#include <backup_copy.h> +#include <sql_plugin.h> +#include <sstream> +#include <vector> +#include <common.h> +#include <backup_mysql.h> + + +extern struct st_maria_plugin *mysql_optional_plugins[]; +extern struct st_maria_plugin *mysql_mandatory_plugins[]; +static void encryption_plugin_init(int argc, char **argv); + +extern char *xb_plugin_load; +extern char *xb_plugin_dir; + +const int PLUGIN_MAX_ARGS = 1024; +vector<string> backup_plugins_args; + +const char *QUERY_PLUGIN = +"SELECT plugin_name, plugin_library, @@plugin_dir" +" FROM information_schema.plugins WHERE plugin_type='ENCRYPTION'" +" AND plugin_status='ACTIVE'"; + +string encryption_plugin_config; + +static void add_to_plugin_load_list(const char *plugin_def) +{ + opt_plugin_load_list_ptr->push_back(new i_string(plugin_def)); +} + +static char XTRABACKUP_EXE[] = "xtrabackup"; + +void encryption_plugin_backup_init(MYSQL *mysql) +{ + MYSQL_RES *result; + MYSQL_ROW row; + ostringstream oss; + char *argv[PLUGIN_MAX_ARGS]; + int argc; + + result = xb_mysql_query(mysql, QUERY_PLUGIN, true, true); + row = mysql_fetch_row(result); + if (!row) + { + mysql_free_result(result); + return; + } + + char *name= row[0]; + char *library= row[1]; + char *dir= row[2]; + +#ifdef _WIN32 + for (char *p = dir; *p; p++) + if (*p == '\\') *p = '/'; +#endif + + string plugin_load(name); + if (library) + plugin_load += string("=") + library; + + oss << "plugin_load=" << plugin_load << endl; + + /* Required to load the plugin later.*/ + add_to_plugin_load_list(plugin_load.c_str()); + strncpy(opt_plugin_dir, dir, FN_REFLEN); + + oss << "plugin_dir=" << '"' << dir << '"' << endl; + + + /* Read plugin variables. */ + char query[1024]; + snprintf(query, 1024, "SHOW variables like '%s_%%'", name); + mysql_free_result(result); + + result = xb_mysql_query(mysql, query, true, true); + while ((row = mysql_fetch_row(result))) + { + string arg("--"); + arg += row[0]; + arg += "="; + arg += row[1]; + backup_plugins_args.push_back(arg); + oss << row[0] << "=" << row[1] << endl; + } + + mysql_free_result(result); + + /* Check whether to encrypt logs. */ + result = xb_mysql_query(mysql, "select @@innodb_encrypt_log", true, true); + row = mysql_fetch_row(result); + srv_encrypt_log = (row != 0 && row[0][0] == '1'); + oss << "innodb_encrypt_log=" << row[0] << endl; + + mysql_free_result(result); + + encryption_plugin_config = oss.str(); + + argc = 0; + argv[argc++] = XTRABACKUP_EXE; + for(size_t i = 0; i < backup_plugins_args.size(); i++) + { + argv[argc++] = (char *)backup_plugins_args[i].c_str(); + if (argc == PLUGIN_MAX_ARGS - 2) + break; + } + argv[argc] = 0; + + encryption_plugin_init(argc, argv); +} + +const char *encryption_plugin_get_config() +{ + return encryption_plugin_config.c_str(); +} + +extern int finalize_encryption_plugin(st_plugin_int *plugin); + + +void encryption_plugin_prepare_init(int argc, char **argv) +{ + + if (!xb_plugin_load) + { + /* This prevents crashes e.g in --stats with wrong my.cnf*/ + finalize_encryption_plugin(0); + return; + } + + add_to_plugin_load_list(xb_plugin_load); + + if (xb_plugin_dir) + strncpy(opt_plugin_dir, xb_plugin_dir, FN_REFLEN); + + char **new_argv = new char *[argc + 1]; + new_argv[0] = XTRABACKUP_EXE; + memcpy(&new_argv[1], argv, argc*sizeof(char *)); + + encryption_plugin_init(argc+1, new_argv); + + delete[] new_argv; +} + +static void encryption_plugin_init(int argc, char **argv) +{ + /* Patch optional and mandatory plugins, we only need to load the one in xb_plugin_load. */ + mysql_optional_plugins[0] = mysql_mandatory_plugins[0] = 0; + msg("Loading encryption plugin\n"); + for (int i= 1; i < argc; i++) + msg("\t Encryption plugin parameter : '%s'\n", argv[i]); + plugin_init(&argc, argv, PLUGIN_INIT_SKIP_PLUGIN_TABLE); +} + diff --git a/extra/mariabackup/encryption_plugin.h b/extra/mariabackup/encryption_plugin.h new file mode 100644 index 00000000000..16d74790254 --- /dev/null +++ b/extra/mariabackup/encryption_plugin.h @@ -0,0 +1,7 @@ +#include <mysql.h> +#include <string> +extern void encryption_plugin_backup_init(MYSQL *mysql); +extern const char* encryption_plugin_get_config(); +extern void encryption_plugin_prepare_init(int argc, char **argv); + +//extern void encryption_plugin_init(int argc, char **argv); diff --git a/extra/mariabackup/fil_cur.cc b/extra/mariabackup/fil_cur.cc new file mode 100644 index 00000000000..820d8e10c29 --- /dev/null +++ b/extra/mariabackup/fil_cur.cc @@ -0,0 +1,409 @@ +/****************************************************** +XtraBackup: hot backup tool for InnoDB +(c) 2009-2013 Percona LLC and/or its affiliates. +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +/* Source file cursor implementation */ + +#include <my_base.h> + +#include <univ.i> +#include <fil0fil.h> +#include <srv0start.h> +#include <trx0sys.h> + +#include "fil_cur.h" +#include "common.h" +#include "read_filt.h" +#include "xtrabackup.h" +#include "xb0xb.h" + +/* Size of read buffer in pages (640 pages = 10M for 16K sized pages) */ +#define XB_FIL_CUR_PAGES 640 + +/*********************************************************************** +Extracts the relative path ("database/table.ibd") of a tablespace from a +specified possibly absolute path. + +For user tablespaces both "./database/table.ibd" and +"/remote/dir/database/table.ibd" result in "database/table.ibd". + +For system tablepsaces (i.e. When is_system is TRUE) both "/remote/dir/ibdata1" +and "./ibdata1" yield "ibdata1" in the output. */ +const char * +xb_get_relative_path( +/*=================*/ + const char* path, /*!< in: tablespace path (either + relative or absolute) */ + ibool is_system) /*!< in: TRUE for system tablespaces, + i.e. when only the filename must be + returned. */ +{ + const char *next; + const char *cur; + const char *prev; + + prev = NULL; + cur = path; + + while ((next = strchr(cur, SRV_PATH_SEPARATOR)) != NULL) { + + prev = cur; + cur = next + 1; + } + + if (is_system) { + + return(cur); + } else { + + return((prev == NULL) ? cur : prev); + } + +} + +/**********************************************************************//** +Closes a file. */ +static +void +xb_fil_node_close_file( +/*===================*/ + fil_node_t* node) /*!< in: file node */ +{ + ibool ret; + + mutex_enter(&fil_system->mutex); + + ut_ad(node); + ut_a(node->n_pending == 0); + ut_a(node->n_pending_flushes == 0); + ut_a(!node->being_extended); + + if (!node->open) { + + mutex_exit(&fil_system->mutex); + + return; + } + + ret = os_file_close(node->handle); + ut_a(ret); + + node->open = FALSE; + + ut_a(fil_system->n_open > 0); + fil_system->n_open--; + fil_n_file_opened--; + + if (node->space->purpose == FIL_TABLESPACE && + fil_is_user_tablespace_id(node->space->id)) { + + ut_a(UT_LIST_GET_LEN(fil_system->LRU) > 0); + + /* The node is in the LRU list, remove it */ + UT_LIST_REMOVE(LRU, fil_system->LRU, node); + } + + mutex_exit(&fil_system->mutex); +} + +/************************************************************************ +Open a source file cursor and initialize the associated read filter. + +@return XB_FIL_CUR_SUCCESS on success, XB_FIL_CUR_SKIP if the source file must +be skipped and XB_FIL_CUR_ERROR on error. */ +xb_fil_cur_result_t +xb_fil_cur_open( +/*============*/ + xb_fil_cur_t* cursor, /*!< out: source file cursor */ + xb_read_filt_t* read_filter, /*!< in/out: the read filter */ + fil_node_t* node, /*!< in: source tablespace node */ + uint thread_n) /*!< thread number for diagnostics */ +{ + ulint page_size; + ulint page_size_shift; + ulint zip_size; + ibool success; + + /* Initialize these first so xb_fil_cur_close() handles them correctly + in case of error */ + cursor->orig_buf = NULL; + cursor->node = NULL; + + cursor->space_id = node->space->id; + cursor->is_system = !fil_is_user_tablespace_id(node->space->id); + + strncpy(cursor->abs_path, node->name, sizeof(cursor->abs_path)); + + /* Get the relative path for the destination tablespace name, i.e. the + one that can be appended to the backup root directory. Non-system + tablespaces may have absolute paths for remote tablespaces in MySQL + 5.6+. We want to make "local" copies for the backup. */ + strncpy(cursor->rel_path, + xb_get_relative_path(cursor->abs_path, cursor->is_system), + sizeof(cursor->rel_path)); + + /* In the backup mode we should already have a tablespace handle created + by fil_load_single_table_tablespace() unless it is a system + tablespace. Otherwise we open the file here. */ + if (cursor->is_system || !srv_backup_mode || srv_close_files) { + node->handle = + os_file_create_simple_no_error_handling(0, node->name, + OS_FILE_OPEN, + OS_FILE_READ_ONLY, + &success,0); + if (!success) { + /* The following call prints an error message */ + os_file_get_last_error(TRUE); + + msg("[%02u] xtrabackup: error: cannot open " + "tablespace %s\n", + thread_n, cursor->abs_path); + + return(XB_FIL_CUR_ERROR); + } + mutex_enter(&fil_system->mutex); + + node->open = TRUE; + + fil_system->n_open++; + fil_n_file_opened++; + + if (node->space->purpose == FIL_TABLESPACE && + fil_is_user_tablespace_id(node->space->id)) { + + /* Put the node to the LRU list */ + UT_LIST_ADD_FIRST(LRU, fil_system->LRU, node); + } + + mutex_exit(&fil_system->mutex); + } + + ut_ad(node->open); + + cursor->node = node; + cursor->file = node->handle; + + if (stat(cursor->abs_path, &cursor->statinfo)) { + msg("[%02u] xtrabackup: error: cannot stat %s\n", + thread_n, cursor->abs_path); + + xb_fil_cur_close(cursor); + + return(XB_FIL_CUR_ERROR); + } + + if (srv_unix_file_flush_method == SRV_UNIX_O_DIRECT + || srv_unix_file_flush_method == SRV_UNIX_O_DIRECT_NO_FSYNC) { + + os_file_set_nocache(cursor->file, node->name, "OPEN"); + } + + posix_fadvise(cursor->file, 0, 0, POSIX_FADV_SEQUENTIAL); + + /* Determine the page size */ + zip_size = xb_get_zip_size(cursor->file); + if (zip_size == ULINT_UNDEFINED) { + xb_fil_cur_close(cursor); + return(XB_FIL_CUR_SKIP); + } else if (zip_size) { + page_size = zip_size; + page_size_shift = get_bit_shift(page_size); + msg("[%02u] %s is compressed with page size = " + "%lu bytes\n", thread_n, node->name, page_size); + if (page_size_shift < 10 || page_size_shift > 14) { + msg("[%02u] xtrabackup: Error: Invalid " + "page size: %lu.\n", thread_n, page_size); + ut_error; + } + } else { + page_size = UNIV_PAGE_SIZE; + page_size_shift = UNIV_PAGE_SIZE_SHIFT; + } + cursor->page_size = page_size; + cursor->page_size_shift = page_size_shift; + cursor->zip_size = zip_size; + + /* Allocate read buffer */ + cursor->buf_size = XB_FIL_CUR_PAGES * page_size; + cursor->orig_buf = static_cast<byte *> + (ut_malloc(cursor->buf_size + UNIV_PAGE_SIZE)); + cursor->buf = static_cast<byte *> + (ut_align(cursor->orig_buf, UNIV_PAGE_SIZE)); + + cursor->buf_read = 0; + cursor->buf_npages = 0; + cursor->buf_offset = 0; + cursor->buf_page_no = 0; + cursor->thread_n = thread_n; + + cursor->space_size = (ulint)(cursor->statinfo.st_size / page_size); + + cursor->read_filter = read_filter; + cursor->read_filter->init(&cursor->read_filter_ctxt, cursor, + node->space->id); + + return(XB_FIL_CUR_SUCCESS); +} + +/************************************************************************ +Reads and verifies the next block of pages from the source +file. Positions the cursor after the last read non-corrupted page. + +@return XB_FIL_CUR_SUCCESS if some have been read successfully, XB_FIL_CUR_EOF +if there are no more pages to read and XB_FIL_CUR_ERROR on error. */ +xb_fil_cur_result_t +xb_fil_cur_read( +/*============*/ + xb_fil_cur_t* cursor) /*!< in/out: source file cursor */ +{ + ibool success; + byte* page; + ulint i; + ulint npages; + ulint retry_count; + xb_fil_cur_result_t ret; + ib_int64_t offset; + ib_int64_t to_read; + + cursor->read_filter->get_next_batch(&cursor->read_filter_ctxt, + &offset, &to_read); + + if (to_read == 0LL) { + return(XB_FIL_CUR_EOF); + } + + if (to_read > (ib_int64_t) cursor->buf_size) { + to_read = (ib_int64_t) cursor->buf_size; + } + + xb_a(to_read > 0 && to_read <= 0xFFFFFFFFLL); + + if (to_read % cursor->page_size != 0 && + offset + to_read == cursor->statinfo.st_size) { + + if (to_read < (ib_int64_t) cursor->page_size) { + msg("[%02u] xtrabackup: Warning: junk at the end of " + "%s:\n", cursor->thread_n, cursor->abs_path); + msg("[%02u] xtrabackup: Warning: offset = %llu, " + "to_read = %llu\n", + cursor->thread_n, + (unsigned long long) offset, + (unsigned long long) to_read); + + return(XB_FIL_CUR_EOF); + } + + to_read = (ib_int64_t) (((ulint) to_read) & + ~(cursor->page_size - 1)); + } + + xb_a(to_read % cursor->page_size == 0); + + npages = (ulint) (to_read >> cursor->page_size_shift); + + retry_count = 10; + ret = XB_FIL_CUR_SUCCESS; + +read_retry: + xtrabackup_io_throttling(); + + cursor->buf_read = 0; + cursor->buf_npages = 0; + cursor->buf_offset = offset; + cursor->buf_page_no = (ulint)(offset >> cursor->page_size_shift); + + success = os_file_read(cursor->file, cursor->buf, offset, + (ulint)to_read); + if (!success) { + return(XB_FIL_CUR_ERROR); + } + + fil_system_enter(); + fil_space_t *space = fil_space_get_by_id(cursor->space_id); + fil_system_exit(); + + /* check pages for corruption and re-read if necessary. i.e. in case of + partially written pages */ + for (page = cursor->buf, i = 0; i < npages; + page += cursor->page_size, i++) { + ib_int64_t page_no = cursor->buf_page_no + i; + + bool checksum_ok = fil_space_verify_crypt_checksum(page, cursor->zip_size,space, (ulint)page_no); + + if (!checksum_ok && + buf_page_is_corrupted(true, page, cursor->zip_size,space)) { + + if (cursor->is_system && + page_no >= (ib_int64_t)FSP_EXTENT_SIZE && + page_no < (ib_int64_t) FSP_EXTENT_SIZE * 3) { + /* skip doublewrite buffer pages */ + xb_a(cursor->page_size == UNIV_PAGE_SIZE); + msg("[%02u] xtrabackup: " + "Page %lu is a doublewrite buffer page, " + "skipping.\n", cursor->thread_n, page_no); + } else { + retry_count--; + if (retry_count == 0) { + msg("[%02u] xtrabackup: " + "Error: failed to read page after " + "10 retries. File %s seems to be " + "corrupted.\n", cursor->thread_n, + cursor->abs_path); + ret = XB_FIL_CUR_ERROR; + break; + } + msg("[%02u] xtrabackup: " + "Database page corruption detected at page " + "%lu, retrying...\n", cursor->thread_n, + page_no); + + os_thread_sleep(100000); + + goto read_retry; + } + } + cursor->buf_read += cursor->page_size; + cursor->buf_npages++; + } + + posix_fadvise(cursor->file, offset, to_read, POSIX_FADV_DONTNEED); + + return(ret); +} + +/************************************************************************ +Close the source file cursor opened with xb_fil_cur_open() and its +associated read filter. */ +void +xb_fil_cur_close( +/*=============*/ + xb_fil_cur_t *cursor) /*!< in/out: source file cursor */ +{ + cursor->read_filter->deinit(&cursor->read_filter_ctxt); + + if (cursor->orig_buf != NULL) { + ut_free(cursor->orig_buf); + } + if (cursor->node != NULL) { + xb_fil_node_close_file(cursor->node); + cursor->file = XB_FILE_UNDEFINED; + } +} diff --git a/extra/mariabackup/fil_cur.h b/extra/mariabackup/fil_cur.h new file mode 100644 index 00000000000..88239efd2bb --- /dev/null +++ b/extra/mariabackup/fil_cur.h @@ -0,0 +1,123 @@ +/****************************************************** +XtraBackup: hot backup tool for InnoDB +(c) 2009-2013 Percona LLC and/or its affiliates. +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +/* Source file cursor interface */ + +#ifndef FIL_CUR_H +#define FIL_CUR_H + +#include <my_dir.h> +#include "read_filt.h" + +struct xb_fil_cur_t { + os_file_t file; /*!< source file handle */ + fil_node_t* node; /*!< source tablespace node */ + char rel_path[FN_REFLEN]; + /*!< normalized file path */ + char abs_path[FN_REFLEN]; + /*!< absolute file path */ + MY_STAT statinfo; /*!< information about the file */ + ulint zip_size; /*!< compressed page size in bytes or 0 + for uncompressed pages */ + ulint page_size; /*!< = zip_size for compressed pages or + UNIV_PAGE_SIZE for uncompressed ones */ + ulint page_size_shift;/*!< bit shift corresponding to + page_size */ + my_bool is_system; /*!< TRUE for system tablespace, FALSE + otherwise */ + xb_read_filt_t* read_filter; /*!< read filter */ + xb_read_filt_ctxt_t read_filter_ctxt; + /*!< read filter context */ + byte* orig_buf; /*!< read buffer */ + byte* buf; /*!< aligned pointer for orig_buf */ + size_t buf_size; /*!< buffer size in bytes */ + size_t buf_read; /*!< number of read bytes in buffer + after the last cursor read */ + size_t buf_npages; /*!< number of pages in buffer after the + last cursor read */ + ib_int64_t buf_offset; /*!< file offset of the first page in + buffer */ + ulint buf_page_no; /*!< number of the first page in + buffer */ + uint thread_n; /*!< thread number for diagnostics */ + ulint space_id; /*!< ID of tablespace */ + ulint space_size; /*!< space size in pages */ +}; + +typedef enum { + XB_FIL_CUR_SUCCESS, + XB_FIL_CUR_SKIP, + XB_FIL_CUR_ERROR, + XB_FIL_CUR_EOF +} xb_fil_cur_result_t; + +/************************************************************************ +Open a source file cursor and initialize the associated read filter. + +@return XB_FIL_CUR_SUCCESS on success, XB_FIL_CUR_SKIP if the source file must +be skipped and XB_FIL_CUR_ERROR on error. */ +xb_fil_cur_result_t +xb_fil_cur_open( +/*============*/ + xb_fil_cur_t* cursor, /*!< out: source file cursor */ + xb_read_filt_t* read_filter, /*!< in/out: the read filter */ + fil_node_t* node, /*!< in: source tablespace node */ + uint thread_n); /*!< thread number for diagnostics */ + +/************************************************************************ +Reads and verifies the next block of pages from the source +file. Positions the cursor after the last read non-corrupted page. + +@return XB_FIL_CUR_SUCCESS if some have been read successfully, XB_FIL_CUR_EOF +if there are no more pages to read and XB_FIL_CUR_ERROR on error. */ +xb_fil_cur_result_t +xb_fil_cur_read( +/*============*/ + xb_fil_cur_t* cursor); /*!< in/out: source file cursor */ + +/************************************************************************ +Close the source file cursor opened with xb_fil_cur_open() and its +associated read filter. */ +void +xb_fil_cur_close( +/*=============*/ + xb_fil_cur_t *cursor); /*!< in/out: source file cursor */ + +/*********************************************************************** +Extracts the relative path ("database/table.ibd") of a tablespace from a +specified possibly absolute path. + +For user tablespaces both "./database/table.ibd" and +"/remote/dir/database/table.ibd" result in "database/table.ibd". + +For system tablepsaces (i.e. When is_system is TRUE) both "/remote/dir/ibdata1" +and "./ibdata1" yield "ibdata1" in the output. */ +const char * +xb_get_relative_path( +/*=================*/ + const char* path, /*!< in: tablespace path (either + relative or absolute) */ + ibool is_system); /*!< in: TRUE for system tablespaces, + i.e. when only the filename must be + returned. */ + +#endif diff --git a/extra/mariabackup/innobackupex.cc b/extra/mariabackup/innobackupex.cc new file mode 100644 index 00000000000..59fb8fb5565 --- /dev/null +++ b/extra/mariabackup/innobackupex.cc @@ -0,0 +1,1132 @@ +/****************************************************** +hot backup tool for InnoDB +(c) 2009-2015 Percona LLC and/or its affiliates +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +******************************************************* + +This file incorporates work covered by the following copyright and +permission notice: + +Copyright (c) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA + +*******************************************************/ + +#include <my_global.h> +#include <stdio.h> +#include <string.h> +#include <mysql.h> +#include <my_dir.h> +#include <ut0mem.h> +#include <os0sync.h> +#include <os0file.h> +#include <srv0start.h> +#include <algorithm> +#include <mysqld.h> +#include <my_default.h> +#include <my_getopt.h> +#include <string> +#include <sstream> +#include <set> +#include "common.h" +#include "innobackupex.h" +#include "xtrabackup.h" +#include "xbstream.h" +#include "fil_cur.h" +#include "write_filt.h" +#include "backup_copy.h" + +using std::min; +using std::max; + +/* options */ +my_bool opt_ibx_version = FALSE; +my_bool opt_ibx_help = FALSE; +my_bool opt_ibx_apply_log = FALSE; +my_bool opt_ibx_redo_only = FALSE; +my_bool opt_ibx_incremental = FALSE; +my_bool opt_ibx_notimestamp = FALSE; + +my_bool opt_ibx_copy_back = FALSE; +my_bool opt_ibx_move_back = FALSE; +my_bool opt_ibx_galera_info = FALSE; +my_bool opt_ibx_slave_info = FALSE; +my_bool opt_ibx_no_lock = FALSE; +my_bool opt_ibx_safe_slave_backup = FALSE; +my_bool opt_ibx_rsync = FALSE; +my_bool opt_ibx_force_non_empty_dirs = FALSE; +my_bool opt_ibx_noversioncheck = FALSE; +my_bool opt_ibx_no_backup_locks = FALSE; +my_bool opt_ibx_decompress = FALSE; + +char *opt_ibx_incremental_history_name = NULL; +char *opt_ibx_incremental_history_uuid = NULL; + +char *opt_ibx_user = NULL; +char *opt_ibx_password = NULL; +char *opt_ibx_host = NULL; +char *opt_ibx_defaults_group = NULL; +char *opt_ibx_socket = NULL; +uint opt_ibx_port = 0; +char *opt_ibx_login_path = NULL; + + +ulong opt_ibx_lock_wait_query_type; +ulong opt_ibx_kill_long_query_type; + +ulong opt_ibx_decrypt_algo = 0; + +uint opt_ibx_kill_long_queries_timeout = 0; +uint opt_ibx_lock_wait_timeout = 0; +uint opt_ibx_lock_wait_threshold = 0; +uint opt_ibx_debug_sleep_before_unlock = 0; +uint opt_ibx_safe_slave_backup_timeout = 0; + +const char *opt_ibx_history = NULL; +bool opt_ibx_decrypt = false; + +char *opt_ibx_include = NULL; +char *opt_ibx_databases = NULL; +bool ibx_partial_backup = false; + +char *ibx_position_arg = NULL; +char *ibx_backup_directory = NULL; + +/* copy of proxied xtrabackup options */ +my_bool ibx_xb_close_files; +my_bool ibx_xtrabackup_compact; +const char *ibx_xtrabackup_compress_alg; +uint ibx_xtrabackup_compress_threads; +ulonglong ibx_xtrabackup_compress_chunk_size; +ulong ibx_xtrabackup_encrypt_algo; +char *ibx_xtrabackup_encrypt_key; +char *ibx_xtrabackup_encrypt_key_file; +uint ibx_xtrabackup_encrypt_threads; +ulonglong ibx_xtrabackup_encrypt_chunk_size; +my_bool ibx_xtrabackup_export; +char *ibx_xtrabackup_extra_lsndir; +char *ibx_xtrabackup_incremental_basedir; +char *ibx_xtrabackup_incremental_dir; +my_bool ibx_xtrabackup_incremental_force_scan; +ulint ibx_xtrabackup_log_copy_interval; +char *ibx_xtrabackup_incremental; +int ibx_xtrabackup_parallel; +my_bool ibx_xtrabackup_rebuild_indexes; +ulint ibx_xtrabackup_rebuild_threads; +char *ibx_xtrabackup_stream_str; +char *ibx_xtrabackup_tables_file; +long ibx_xtrabackup_throttle; +char *ibx_opt_mysql_tmpdir; +longlong ibx_xtrabackup_use_memory; + + +static inline int ibx_msg(const char *fmt, ...) ATTRIBUTE_FORMAT(printf, 1, 2); +static inline int ibx_msg(const char *fmt, ...) +{ + int result; + time_t t = time(NULL); + char date[100]; + char *line; + va_list args; + + strftime(date, sizeof(date), "%y%m%d %H:%M:%S", localtime(&t)); + + va_start(args, fmt); + + result = vasprintf(&line, fmt, args); + + va_end(args); + + if (result != -1) { + result = fprintf(stderr, "%s %s: %s", + date, INNOBACKUPEX_BIN_NAME, line); + free(line); + } + + return result; +} + +enum innobackupex_options +{ + OPT_APPLY_LOG = 256, + OPT_COPY_BACK, + OPT_MOVE_BACK, + OPT_REDO_ONLY, + OPT_GALERA_INFO, + OPT_SLAVE_INFO, + OPT_INCREMENTAL, + OPT_INCREMENTAL_HISTORY_NAME, + OPT_INCREMENTAL_HISTORY_UUID, + OPT_LOCK_WAIT_QUERY_TYPE, + OPT_KILL_LONG_QUERY_TYPE, + OPT_KILL_LONG_QUERIES_TIMEOUT, + OPT_LOCK_WAIT_TIMEOUT, + OPT_LOCK_WAIT_THRESHOLD, + OPT_DEBUG_SLEEP_BEFORE_UNLOCK, + OPT_NO_LOCK, + OPT_SAFE_SLAVE_BACKUP, + OPT_SAFE_SLAVE_BACKUP_TIMEOUT, + OPT_RSYNC, + OPT_HISTORY, + OPT_INCLUDE, + OPT_FORCE_NON_EMPTY_DIRS, + OPT_NO_TIMESTAMP, + OPT_NO_VERSION_CHECK, + OPT_NO_BACKUP_LOCKS, + OPT_DATABASES, + OPT_DECRYPT, + OPT_DECOMPRESS, + + /* options wich are passed directly to xtrabackup */ + OPT_CLOSE_FILES, + OPT_COMPACT, + OPT_COMPRESS, + OPT_COMPRESS_THREADS, + OPT_COMPRESS_CHUNK_SIZE, + OPT_ENCRYPT, + OPT_ENCRYPT_KEY, + OPT_ENCRYPT_KEY_FILE, + OPT_ENCRYPT_THREADS, + OPT_ENCRYPT_CHUNK_SIZE, + OPT_EXPORT, + OPT_EXTRA_LSNDIR, + OPT_INCREMENTAL_BASEDIR, + OPT_INCREMENTAL_DIR, + OPT_INCREMENTAL_FORCE_SCAN, + OPT_LOG_COPY_INTERVAL, + OPT_PARALLEL, + OPT_REBUILD_INDEXES, + OPT_REBUILD_THREADS, + OPT_STREAM, + OPT_TABLES_FILE, + OPT_THROTTLE, + OPT_USE_MEMORY +}; + +ibx_mode_t ibx_mode = IBX_MODE_BACKUP; + +static struct my_option ibx_long_options[] = +{ + {"version", 'v', "print xtrabackup version information", + (uchar *) &opt_ibx_version, (uchar *) &opt_ibx_version, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"help", '?', "This option displays a help screen and exits.", + (uchar *) &opt_ibx_help, (uchar *) &opt_ibx_help, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"apply-log", OPT_APPLY_LOG, "Prepare a backup in BACKUP-DIR by " + "applying the transaction log file named \"xtrabackup_logfile\" " + "located in the same directory. Also, create new transaction logs. " + "The InnoDB configuration is read from the file \"backup-my.cnf\".", + (uchar*) &opt_ibx_apply_log, (uchar*) &opt_ibx_apply_log, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"redo-only", OPT_REDO_ONLY, "This option should be used when " + "preparing the base full backup and when merging all incrementals " + "except the last one. This forces xtrabackup to skip the \"rollback\" " + "phase and do a \"redo\" only. This is necessary if the backup will " + "have incremental changes applied to it later. See the xtrabackup " + "documentation for details.", + (uchar *) &opt_ibx_redo_only, (uchar *) &opt_ibx_redo_only, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"copy-back", OPT_COPY_BACK, "Copy all the files in a previously made " + "backup from the backup directory to their original locations.", + (uchar *) &opt_ibx_copy_back, (uchar *) &opt_ibx_copy_back, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"move-back", OPT_MOVE_BACK, "Move all the files in a previously made " + "backup from the backup directory to the actual datadir location. " + "Use with caution, as it removes backup files.", + (uchar *) &opt_ibx_move_back, (uchar *) &opt_ibx_move_back, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"galera-info", OPT_GALERA_INFO, "This options creates the " + "xtrabackup_galera_info file which contains the local node state at " + "the time of the backup. Option should be used when performing the " + "backup of Percona-XtraDB-Cluster. Has no effect when backup locks " + "are used to create the backup.", + (uchar *) &opt_ibx_galera_info, (uchar *) &opt_ibx_galera_info, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"slave-info", OPT_SLAVE_INFO, "This option is useful when backing " + "up a replication slave server. It prints the binary log position " + "and name of the master server. It also writes this information to " + "the \"xtrabackup_slave_info\" file as a \"CHANGE MASTER\" command. " + "A new slave for this master can be set up by starting a slave server " + "on this backup and issuing a \"CHANGE MASTER\" command with the " + "binary log position saved in the \"xtrabackup_slave_info\" file.", + (uchar *) &opt_ibx_slave_info, (uchar *) &opt_ibx_slave_info, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"incremental", OPT_INCREMENTAL, "This option tells xtrabackup to " + "create an incremental backup, rather than a full one. It is passed " + "to the xtrabackup child process. When this option is specified, " + "either --incremental-lsn or --incremental-basedir can also be given. " + "If neither option is given, option --incremental-basedir is passed " + "to xtrabackup by default, set to the first timestamped backup " + "directory in the backup base directory.", + (uchar *) &opt_ibx_incremental, (uchar *) &opt_ibx_incremental, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"no-lock", OPT_NO_LOCK, "Use this option to disable table lock " + "with \"FLUSH TABLES WITH READ LOCK\". Use it only if ALL your " + "tables are InnoDB and you DO NOT CARE about the binary log " + "position of the backup. This option shouldn't be used if there " + "are any DDL statements being executed or if any updates are " + "happening on non-InnoDB tables (this includes the system MyISAM " + "tables in the mysql database), otherwise it could lead to an " + "inconsistent backup. If you are considering to use --no-lock " + "because your backups are failing to acquire the lock, this could " + "be because of incoming replication events preventing the lock " + "from succeeding. Please try using --safe-slave-backup to " + "momentarily stop the replication slave thread, this may help " + "the backup to succeed and you then don't need to resort to " + "using this option.", + (uchar *) &opt_ibx_no_lock, (uchar *) &opt_ibx_no_lock, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"safe-slave-backup", OPT_SAFE_SLAVE_BACKUP, "Stop slave SQL thread " + "and wait to start backup until Slave_open_temp_tables in " + "\"SHOW STATUS\" is zero. If there are no open temporary tables, " + "the backup will take place, otherwise the SQL thread will be " + "started and stopped until there are no open temporary tables. " + "The backup will fail if Slave_open_temp_tables does not become " + "zero after --safe-slave-backup-timeout seconds. The slave SQL " + "thread will be restarted when the backup finishes.", + (uchar *) &opt_ibx_safe_slave_backup, + (uchar *) &opt_ibx_safe_slave_backup, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"rsync", OPT_RSYNC, "Uses the rsync utility to optimize local file " + "transfers. When this option is specified, innobackupex uses rsync " + "to copy all non-InnoDB files instead of spawning a separate cp for " + "each file, which can be much faster for servers with a large number " + "of databases or tables. This option cannot be used together with " + "--stream.", + (uchar *) &opt_ibx_rsync, (uchar *) &opt_ibx_rsync, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"force-non-empty-directories", OPT_FORCE_NON_EMPTY_DIRS, "This " + "option, when specified, makes --copy-back or --move-back transfer " + "files to non-empty directories. Note that no existing files will be " + "overwritten. If --copy-back or --nove-back has to copy a file from " + "the backup directory which already exists in the destination " + "directory, it will still fail with an error.", + (uchar *) &opt_ibx_force_non_empty_dirs, + (uchar *) &opt_ibx_force_non_empty_dirs, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"no-timestamp", OPT_NO_TIMESTAMP, "This option prevents creation of a " + "time-stamped subdirectory of the BACKUP-ROOT-DIR given on the " + "command line. When it is specified, the backup is done in " + "BACKUP-ROOT-DIR instead.", + (uchar *) &opt_ibx_notimestamp, + (uchar *) &opt_ibx_notimestamp, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"no-version-check", OPT_NO_VERSION_CHECK, "This option disables the " + "version check which is enabled by the --version-check option.", + (uchar *) &opt_ibx_noversioncheck, + (uchar *) &opt_ibx_noversioncheck, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"no-backup-locks", OPT_NO_BACKUP_LOCKS, "This option controls if " + "backup locks should be used instead of FLUSH TABLES WITH READ LOCK " + "on the backup stage. The option has no effect when backup locks are " + "not supported by the server. This option is enabled by default, " + "disable with --no-backup-locks.", + (uchar *) &opt_ibx_no_backup_locks, + (uchar *) &opt_ibx_no_backup_locks, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"decompress", OPT_DECOMPRESS, "Decompresses all files with the .qp " + "extension in a backup previously made with the --compress option.", + (uchar *) &opt_ibx_decompress, + (uchar *) &opt_ibx_decompress, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"user", 'u', "This option specifies the MySQL username used " + "when connecting to the server, if that's not the current user. " + "The option accepts a string argument. See mysql --help for details.", + (uchar*) &opt_ibx_user, (uchar*) &opt_ibx_user, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"host", 'H', "This option specifies the host to use when " + "connecting to the database server with TCP/IP. The option accepts " + "a string argument. See mysql --help for details.", + (uchar*) &opt_ibx_host, (uchar*) &opt_ibx_host, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"port", 'P', "This option specifies the port to use when " + "connecting to the database server with TCP/IP. The option accepts " + "a string argument. See mysql --help for details.", + &opt_ibx_port, &opt_ibx_port, 0, GET_UINT, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"password", 'p', "This option specifies the password to use " + "when connecting to the database. It accepts a string argument. " + "See mysql --help for details.", + 0, 0, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"socket", 'S', "This option specifies the socket to use when " + "connecting to the local database server with a UNIX domain socket. " + "The option accepts a string argument. See mysql --help for details.", + (uchar*) &opt_ibx_socket, (uchar*) &opt_ibx_socket, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"incremental-history-name", OPT_INCREMENTAL_HISTORY_NAME, + "This option specifies the name of the backup series stored in the " + "PERCONA_SCHEMA.xtrabackup_history history record to base an " + "incremental backup on. Xtrabackup will search the history table " + "looking for the most recent (highest innodb_to_lsn), successful " + "backup in the series and take the to_lsn value to use as the " + "starting lsn for the incremental backup. This will be mutually " + "exclusive with --incremental-history-uuid, --incremental-basedir " + "and --incremental-lsn. If no valid lsn can be found (no series by " + "that name, no successful backups by that name) xtrabackup will " + "return with an error. It is used with the --incremental option.", + (uchar*) &opt_ibx_incremental_history_name, + (uchar*) &opt_ibx_incremental_history_name, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"incremental-history-uuid", OPT_INCREMENTAL_HISTORY_UUID, + "This option specifies the UUID of the specific history record " + "stored in the PERCONA_SCHEMA.xtrabackup_history to base an " + "incremental backup on. --incremental-history-name, " + "--incremental-basedir and --incremental-lsn. If no valid lsn can be " + "found (no success record with that uuid) xtrabackup will return " + "with an error. It is used with the --incremental option.", + (uchar*) &opt_ibx_incremental_history_uuid, + (uchar*) &opt_ibx_incremental_history_uuid, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"decrypt", OPT_DECRYPT, "Decrypts all files with the .xbcrypt " + "extension in a backup previously made with --encrypt option.", + &opt_ibx_decrypt_algo, &opt_ibx_decrypt_algo, + &xtrabackup_encrypt_algo_typelib, GET_ENUM, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"ftwrl-wait-query-type", OPT_LOCK_WAIT_QUERY_TYPE, + "This option specifies which types of queries are allowed to complete " + "before innobackupex will issue the global lock. Default is all.", + (uchar*) &opt_ibx_lock_wait_query_type, + (uchar*) &opt_ibx_lock_wait_query_type, &query_type_typelib, + GET_ENUM, REQUIRED_ARG, QUERY_TYPE_ALL, 0, 0, 0, 0, 0}, + + {"kill-long-query-type", OPT_KILL_LONG_QUERY_TYPE, + "This option specifies which types of queries should be killed to " + "unblock the global lock. Default is \"all\".", + (uchar*) &opt_ibx_kill_long_query_type, + (uchar*) &opt_ibx_kill_long_query_type, &query_type_typelib, + GET_ENUM, REQUIRED_ARG, QUERY_TYPE_SELECT, 0, 0, 0, 0, 0}, + + {"history", OPT_HISTORY, + "This option enables the tracking of backup history in the " + "PERCONA_SCHEMA.xtrabackup_history table. An optional history " + "series name may be specified that will be placed with the history " + "record for the current backup being taken.", + NULL, NULL, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, + + {"include", OPT_INCLUDE, + "This option is a regular expression to be matched against table " + "names in databasename.tablename format. It is passed directly to " + "xtrabackup's --tables option. See the xtrabackup documentation for " + "details.", + (uchar*) &opt_ibx_include, + (uchar*) &opt_ibx_include, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"databases", OPT_DATABASES, + "This option specifies the list of databases that innobackupex should " + "back up. The option accepts a string argument or path to file that " + "contains the list of databases to back up. The list is of the form " + "\"databasename1[.table_name1] databasename2[.table_name2] . . .\". " + "If this option is not specified, all databases containing MyISAM and " + "InnoDB tables will be backed up. Please make sure that --databases " + "contains all of the InnoDB databases and tables, so that all of the " + "innodb.frm files are also backed up. In case the list is very long, " + "this can be specified in a file, and the full path of the file can " + "be specified instead of the list. (See option --tables-file.)", + (uchar*) &opt_ibx_databases, + (uchar*) &opt_ibx_databases, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"kill-long-queries-timeout", OPT_KILL_LONG_QUERIES_TIMEOUT, + "This option specifies the number of seconds innobackupex waits " + "between starting FLUSH TABLES WITH READ LOCK and killing those " + "queries that block it. Default is 0 seconds, which means " + "innobackupex will not attempt to kill any queries.", + (uchar*) &opt_ibx_kill_long_queries_timeout, + (uchar*) &opt_ibx_kill_long_queries_timeout, 0, GET_UINT, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"ftwrl-wait-timeout", OPT_LOCK_WAIT_TIMEOUT, + "This option specifies time in seconds that innobackupex should wait " + "for queries that would block FTWRL before running it. If there are " + "still such queries when the timeout expires, innobackupex terminates " + "with an error. Default is 0, in which case innobackupex does not " + "wait for queries to complete and starts FTWRL immediately.", + (uchar*) &opt_ibx_lock_wait_timeout, + (uchar*) &opt_ibx_lock_wait_timeout, 0, GET_UINT, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"ftwrl-wait-threshold", OPT_LOCK_WAIT_THRESHOLD, + "This option specifies the query run time threshold which is used by " + "innobackupex to detect long-running queries with a non-zero value " + "of --ftwrl-wait-timeout. FTWRL is not started until such " + "long-running queries exist. This option has no effect if " + "--ftwrl-wait-timeout is 0. Default value is 60 seconds.", + (uchar*) &opt_ibx_lock_wait_threshold, + (uchar*) &opt_ibx_lock_wait_threshold, 0, GET_UINT, + REQUIRED_ARG, 60, 0, 0, 0, 0, 0}, + + {"debug-sleep-before-unlock", OPT_DEBUG_SLEEP_BEFORE_UNLOCK, + "This is a debug-only option used by the XtraBackup test suite.", + (uchar*) &opt_ibx_debug_sleep_before_unlock, + (uchar*) &opt_ibx_debug_sleep_before_unlock, 0, GET_UINT, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"safe-slave-backup-timeout", OPT_SAFE_SLAVE_BACKUP_TIMEOUT, + "How many seconds --safe-slave-backup should wait for " + "Slave_open_temp_tables to become zero. (default 300)", + (uchar*) &opt_ibx_safe_slave_backup_timeout, + (uchar*) &opt_ibx_safe_slave_backup_timeout, 0, GET_UINT, + REQUIRED_ARG, 300, 0, 0, 0, 0, 0}, + + + /* Following command-line options are actually handled by xtrabackup. + We put them here with only purpose for them to showup in + innobackupex --help output */ + + {"close_files", OPT_CLOSE_FILES, "Do not keep files opened. This " + "option is passed directly to xtrabackup. Use at your own risk.", + (uchar*) &ibx_xb_close_files, (uchar*) &ibx_xb_close_files, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"compact", OPT_COMPACT, "Create a compact backup with all secondary " + "index pages omitted. This option is passed directly to xtrabackup. " + "See xtrabackup documentation for details.", + (uchar*) &ibx_xtrabackup_compact, (uchar*) &ibx_xtrabackup_compact, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"compress", OPT_COMPRESS, "This option instructs xtrabackup to " + "compress backup copies of InnoDB data files. It is passed directly " + "to the xtrabackup child process. Try 'xtrabackup --help' for more " + "details.", (uchar*) &ibx_xtrabackup_compress_alg, + (uchar*) &ibx_xtrabackup_compress_alg, 0, + GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, + + {"compress-threads", OPT_COMPRESS_THREADS, + "This option specifies the number of worker threads that will be used " + "for parallel compression. It is passed directly to the xtrabackup " + "child process. Try 'xtrabackup --help' for more details.", + (uchar*) &ibx_xtrabackup_compress_threads, + (uchar*) &ibx_xtrabackup_compress_threads, + 0, GET_UINT, REQUIRED_ARG, 1, 1, UINT_MAX, 0, 0, 0}, + + {"compress-chunk-size", OPT_COMPRESS_CHUNK_SIZE, "Size of working " + "buffer(s) for compression threads in bytes. The default value " + "is 64K.", (uchar*) &ibx_xtrabackup_compress_chunk_size, + (uchar*) &ibx_xtrabackup_compress_chunk_size, + 0, GET_ULL, REQUIRED_ARG, (1 << 16), 1024, ULONGLONG_MAX, 0, 0, 0}, + + {"encrypt", OPT_ENCRYPT, "This option instructs xtrabackup to encrypt " + "backup copies of InnoDB data files using the algorithm specified in " + "the ENCRYPTION-ALGORITHM. It is passed directly to the xtrabackup " + "child process. Try 'xtrabackup --help' for more details.", + &ibx_xtrabackup_encrypt_algo, &ibx_xtrabackup_encrypt_algo, + &xtrabackup_encrypt_algo_typelib, GET_ENUM, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"encrypt-key", OPT_ENCRYPT_KEY, "This option instructs xtrabackup to " + "use the given ENCRYPTION-KEY when using the --encrypt or --decrypt " + "options. During backup it is passed directly to the xtrabackup child " + "process. Try 'xtrabackup --help' for more details.", + (uchar*) &ibx_xtrabackup_encrypt_key, + (uchar*) &ibx_xtrabackup_encrypt_key, 0, + GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"encrypt-key-file", OPT_ENCRYPT_KEY_FILE, "This option instructs " + "xtrabackup to use the encryption key stored in the given " + "ENCRYPTION-KEY-FILE when using the --encrypt or --decrypt options.", + (uchar*) &ibx_xtrabackup_encrypt_key_file, + (uchar*) &ibx_xtrabackup_encrypt_key_file, 0, + GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"encrypt-threads", OPT_ENCRYPT_THREADS, + "This option specifies the number of worker threads that will be used " + "for parallel encryption. It is passed directly to the xtrabackup " + "child process. Try 'xtrabackup --help' for more details.", + (uchar*) &ibx_xtrabackup_encrypt_threads, + (uchar*) &ibx_xtrabackup_encrypt_threads, + 0, GET_UINT, REQUIRED_ARG, 1, 1, UINT_MAX, 0, 0, 0}, + + {"encrypt-chunk-size", OPT_ENCRYPT_CHUNK_SIZE, + "This option specifies the size of the internal working buffer for " + "each encryption thread, measured in bytes. It is passed directly to " + "the xtrabackup child process. Try 'xtrabackup --help' for more " + "details.", + (uchar*) &ibx_xtrabackup_encrypt_chunk_size, + (uchar*) &ibx_xtrabackup_encrypt_chunk_size, + 0, GET_ULL, REQUIRED_ARG, (1 << 16), 1024, ULONGLONG_MAX, 0, 0, 0}, + + {"export", OPT_EXPORT, "This option is passed directly to xtrabackup's " + "--export option. It enables exporting individual tables for import " + "into another server. See the xtrabackup documentation for details.", + (uchar*) &ibx_xtrabackup_export, (uchar*) &ibx_xtrabackup_export, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"extra-lsndir", OPT_EXTRA_LSNDIR, "This option specifies the " + "directory in which to save an extra copy of the " + "\"xtrabackup_checkpoints\" file. The option accepts a string " + "argument. It is passed directly to xtrabackup's --extra-lsndir " + "option. See the xtrabackup documentation for details.", + (uchar*) &ibx_xtrabackup_extra_lsndir, + (uchar*) &ibx_xtrabackup_extra_lsndir, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"incremental-basedir", OPT_INCREMENTAL_BASEDIR, "This option " + "specifies the directory containing the full backup that is the base " + "dataset for the incremental backup. The option accepts a string " + "argument. It is used with the --incremental option.", + (uchar*) &ibx_xtrabackup_incremental_basedir, + (uchar*) &ibx_xtrabackup_incremental_basedir, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"incremental-dir", OPT_INCREMENTAL_DIR, "This option specifies the " + "directory where the incremental backup will be combined with the " + "full backup to make a new full backup. The option accepts a string " + "argument. It is used with the --incremental option.", + (uchar*) &ibx_xtrabackup_incremental_dir, + (uchar*) &ibx_xtrabackup_incremental_dir, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"incremental-force-scan", OPT_INCREMENTAL_FORCE_SCAN, + "This options tells xtrabackup to perform full scan of data files " + "for taking an incremental backup even if full changed page bitmap " + "data is available to enable the backup without the full scan.", + (uchar*)&ibx_xtrabackup_incremental_force_scan, + (uchar*)&ibx_xtrabackup_incremental_force_scan, 0, GET_BOOL, NO_ARG, + 0, 0, 0, 0, 0, 0}, + + {"log-copy-interval", OPT_LOG_COPY_INTERVAL, "This option specifies " + "time interval between checks done by log copying thread in " + "milliseconds.", (uchar*) &ibx_xtrabackup_log_copy_interval, + (uchar*) &ibx_xtrabackup_log_copy_interval, + 0, GET_LONG, REQUIRED_ARG, 1000, 0, LONG_MAX, 0, 1, 0}, + + {"incremental-lsn", OPT_INCREMENTAL, "This option specifies the log " + "sequence number (LSN) to use for the incremental backup. The option " + "accepts a string argument. It is used with the --incremental option. " + "It is used instead of specifying --incremental-basedir. For " + "databases created by MySQL and Percona Server 5.0-series versions, " + "specify the LSN as two 32-bit integers in high:low format. For " + "databases created in 5.1 and later, specify the LSN as a single " + "64-bit integer.", + (uchar*) &ibx_xtrabackup_incremental, + (uchar*) &ibx_xtrabackup_incremental, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"parallel", OPT_PARALLEL, "On backup, this option specifies the " + "number of threads the xtrabackup child process should use to back " + "up files concurrently. The option accepts an integer argument. It " + "is passed directly to xtrabackup's --parallel option. See the " + "xtrabackup documentation for details.", + (uchar*) &ibx_xtrabackup_parallel, (uchar*) &ibx_xtrabackup_parallel, + 0, GET_INT, REQUIRED_ARG, 1, 1, INT_MAX, 0, 0, 0}, + + + {"stream", OPT_STREAM, "This option specifies the format in which to " + "do the streamed backup. The option accepts a string argument. The " + "backup will be done to STDOUT in the specified format. Currently, " + "the only supported formats are tar and xbstream. This option is " + "passed directly to xtrabackup's --stream option.", + (uchar*) &ibx_xtrabackup_stream_str, + (uchar*) &ibx_xtrabackup_stream_str, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"tables-file", OPT_TABLES_FILE, "This option specifies the file in " + "which there are a list of names of the form database. The option " + "accepts a string argument.table, one per line. The option is passed " + "directly to xtrabackup's --tables-file option.", + (uchar*) &ibx_xtrabackup_tables_file, + (uchar*) &ibx_xtrabackup_tables_file, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"throttle", OPT_THROTTLE, "This option specifies a number of I/O " + "operations (pairs of read+write) per second. It accepts an integer " + "argument. It is passed directly to xtrabackup's --throttle option.", + (uchar*) &ibx_xtrabackup_throttle, (uchar*) &ibx_xtrabackup_throttle, + 0, GET_LONG, REQUIRED_ARG, 0, 0, LONG_MAX, 0, 1, 0}, + + {"tmpdir", 't', "This option specifies the location where a temporary " + "files will be stored. If the option is not specified, the default is " + "to use the value of tmpdir read from the server configuration.", + (uchar*) &ibx_opt_mysql_tmpdir, + (uchar*) &ibx_opt_mysql_tmpdir, 0, GET_STR, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"use-memory", OPT_USE_MEMORY, "This option accepts a string argument " + "that specifies the amount of memory in bytes for xtrabackup to use " + "for crash recovery while preparing a backup. Multiples are supported " + "providing the unit (e.g. 1MB, 1GB). It is used only with the option " + "--apply-log. It is passed directly to xtrabackup's --use-memory " + "option. See the xtrabackup documentation for details.", + (uchar*) &ibx_xtrabackup_use_memory, + (uchar*) &ibx_xtrabackup_use_memory, + 0, GET_LL, REQUIRED_ARG, 100*1024*1024L, 1024*1024L, LONGLONG_MAX, 0, + 1024*1024L, 0}, + + { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} +}; + + +static void usage(void) +{ + puts("Open source backup tool for InnoDB and XtraDB\n\ +\n\ +Copyright (C) 2009-2015 Percona LLC and/or its affiliates.\n\ +Portions Copyright (C) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved.\n\ +\n\ +This program is free software; you can redistribute it and/or\n\ +modify it under the terms of the GNU General Public License\n\ +as published by the Free Software Foundation version 2\n\ +of the License.\n\ +\n\ +This program is distributed in the hope that it will be useful,\n\ +but WITHOUT ANY WARRANTY; without even the implied warranty of\n\ +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n\ +GNU General Public License for more details.\n\ +\n\ +You can download full text of the license on http://www.gnu.org/licenses/gpl-2.0.txt\n\n"); + + puts("innobackupex - Non-blocking backup tool for InnoDB, XtraDB and HailDB databases\n\ +\n\ +SYNOPOSIS\n\ +\n\ +innobackupex [--compress] [--compress-threads=NUMBER-OF-THREADS] [--compress-chunk-size=CHUNK-SIZE]\n\ + [--encrypt=ENCRYPTION-ALGORITHM] [--encrypt-threads=NUMBER-OF-THREADS] [--encrypt-chunk-size=CHUNK-SIZE]\n\ + [--encrypt-key=LITERAL-ENCRYPTION-KEY] | [--encryption-key-file=MY.KEY]\n\ + [--include=REGEXP] [--user=NAME]\n\ + [--password=WORD] [--port=PORT] [--socket=SOCKET]\n\ + [--no-timestamp] [--ibbackup=IBBACKUP-BINARY]\n\ + [--slave-info] [--galera-info] [--stream=tar|xbstream]\n\ + [--defaults-file=MY.CNF] [--defaults-group=GROUP-NAME]\n\ + [--databases=LIST] [--no-lock] \n\ + [--tmpdir=DIRECTORY] [--tables-file=FILE]\n\ + [--history=NAME]\n\ + [--incremental] [--incremental-basedir]\n\ + [--incremental-dir] [--incremental-force-scan] [--incremental-lsn]\n\ + [--incremental-history-name=NAME] [--incremental-history-uuid=UUID]\n\ + [--close-files] [--compact] \n\ + BACKUP-ROOT-DIR\n\ +\n\ +innobackupex --apply-log [--use-memory=B]\n\ + [--defaults-file=MY.CNF]\n\ + [--export] [--redo-only] [--ibbackup=IBBACKUP-BINARY]\n\ + BACKUP-DIR\n\ +\n\ +innobackupex --copy-back [--defaults-file=MY.CNF] [--defaults-group=GROUP-NAME] BACKUP-DIR\n\ +\n\ +innobackupex --move-back [--defaults-file=MY.CNF] [--defaults-group=GROUP-NAME] BACKUP-DIR\n\ +\n\ +innobackupex [--decompress] [--decrypt=ENCRYPTION-ALGORITHM]\n\ + [--encrypt-key=LITERAL-ENCRYPTION-KEY] | [--encryption-key-file=MY.KEY]\n\ + [--parallel=NUMBER-OF-FORKS] BACKUP-DIR\n\ +\n\ +DESCRIPTION\n\ +\n\ +The first command line above makes a hot backup of a MySQL database.\n\ +By default it creates a backup directory (named by the current date\n\ + and time) in the given backup root directory. With the --no-timestamp\n\ +option it does not create a time-stamped backup directory, but it puts\n\ +the backup in the given directory (which must not exist). This\n\ +command makes a complete backup of all MyISAM and InnoDB tables and\n\ +indexes in all databases or in all of the databases specified with the\n\ +--databases option. The created backup contains .frm, .MRG, .MYD,\n\ +.MYI, .MAD, .MAI, .TRG, .TRN, .ARM, .ARZ, .CSM, CSV, .opt, .par, and\n\ +InnoDB data and log files. The MY.CNF options file defines the\n\ +location of the database. This command connects to the MySQL server\n\ +using the mysql client program, and runs xtrabackup as a child\n\ +process.\n\ +\n\ +The --apply-log command prepares a backup for starting a MySQL\n\ +server on the backup. This command recovers InnoDB data files as specified\n\ +in BACKUP-DIR/backup-my.cnf using BACKUP-DIR/xtrabackup_logfile,\n\ +and creates new InnoDB log files as specified in BACKUP-DIR/backup-my.cnf.\n\ +The BACKUP-DIR should be the path to a backup directory created by\n\ +xtrabackup. This command runs xtrabackup as a child process, but it does not \n\ +connect to the database server.\n\ +\n\ +The --copy-back command copies data, index, and log files\n\ +from the backup directory back to their original locations.\n\ +The MY.CNF options file defines the original location of the database.\n\ +The BACKUP-DIR is the path to a backup directory created by xtrabackup.\n\ +\n\ +The --move-back command is similar to --copy-back with the only difference that\n\ +it moves files to their original locations rather than copies them. As this\n\ +option removes backup files, it must be used with caution. It may be useful in\n\ +cases when there is not enough free disk space to copy files.\n\ +\n\ +The --decompress --decrypt command will decrypt and/or decompress a backup made\n\ +with the --compress and/or --encrypt options. When decrypting, the encryption\n\ +algorithm and key used when the backup was taken MUST be provided via the\n\ +specified options. --decrypt and --decompress may be used together at the same\n\ +time to completely normalize a previously compressed and encrypted backup. The\n\ +--parallel option will allow multiple files to be decrypted and/or decompressed\n\ +simultaneously. In order to decompress, the qpress utility MUST be installed\n\ +and accessable within the path. This process will remove the original\n\ +compressed/encrypted files and leave the results in the same location.\n\ +\n\ +On success the exit code innobackupex is 0. A non-zero exit code \n\ +indicates an error.\n"); + printf("Usage: [%s [--defaults-file=#] --backup | %s [--defaults-file=#] --prepare] [OPTIONS]\n", my_progname, my_progname); + my_print_help(ibx_long_options); +} + + +static +my_bool +ibx_get_one_option(int optid, + const struct my_option *opt __attribute__((unused)), + char *argument) +{ + switch(optid) { + case '?': + usage(); + exit(0); + break; + case 'v': + msg("innobackupex version %s %s (%s)\n", + MYSQL_SERVER_VERSION, + SYSTEM_TYPE, MACHINE_TYPE); + exit(0); + break; + case OPT_HISTORY: + if (argument) { + opt_ibx_history = argument; + } else { + opt_ibx_history = ""; + } + break; + case OPT_DECRYPT: + if (argument == NULL) { + ibx_msg("Missing --decrypt argument, must specify a " + "valid encryption algorithm.\n"); + return(1); + } + opt_ibx_decrypt = true; + break; + case OPT_STREAM: + if (!strcasecmp(argument, "tar")) + xtrabackup_stream_fmt = XB_STREAM_FMT_TAR; + else if (!strcasecmp(argument, "xbstream")) + xtrabackup_stream_fmt = XB_STREAM_FMT_XBSTREAM; + else { + ibx_msg("Invalid --stream argument: %s\n", argument); + return 1; + } + xtrabackup_stream = TRUE; + break; + case OPT_COMPRESS: + if (argument == NULL) + xtrabackup_compress_alg = "quicklz"; + else if (strcasecmp(argument, "quicklz")) + { + ibx_msg("Invalid --compress argument: %s\n", argument); + return 1; + } + xtrabackup_compress = TRUE; + break; + case OPT_ENCRYPT: + if (argument == NULL) + { + msg("Missing --encrypt argument, must specify a " + "valid encryption algorithm.\n"); + return 1; + } + xtrabackup_encrypt = TRUE; + break; + case 'p': + if (argument) + { + char *start = argument; + my_free(opt_ibx_password); + opt_ibx_password= my_strdup(argument, MYF(MY_FAE)); + /* Destroy argument */ + while (*argument) + *argument++= 'x'; + if (*start) + start[1]=0 ; + } + break; + } + return(0); +} + +bool +make_backup_dir() +{ + time_t t = time(NULL); + char buf[100]; + + if (!opt_ibx_notimestamp && !ibx_xtrabackup_stream_str) { + strftime(buf, sizeof(buf), "%Y-%m-%d_%H-%M-%S", localtime(&t)); + ut_a(asprintf(&ibx_backup_directory, "%s/%s", + ibx_position_arg, buf) != -1); + } else { + ibx_backup_directory = strdup(ibx_position_arg); + } + + if (!directory_exists(ibx_backup_directory, true)) { + return(false); + } + + return(true); +} + +bool +ibx_handle_options(int *argc, char ***argv) +{ + int i, n_arguments; + + if (handle_options(argc, argv, ibx_long_options, ibx_get_one_option)) { + return(false); + } + + if (opt_ibx_apply_log) { + ibx_mode = IBX_MODE_APPLY_LOG; + } else if (opt_ibx_copy_back) { + ibx_mode = IBX_MODE_COPY_BACK; + } else if (opt_ibx_move_back) { + ibx_mode = IBX_MODE_MOVE_BACK; + } else if (opt_ibx_decrypt || opt_ibx_decompress) { + ibx_mode = IBX_MODE_DECRYPT_DECOMPRESS; + } else { + ibx_mode = IBX_MODE_BACKUP; + } + + /* find and save position argument */ + i = 0; + n_arguments = 0; + while (i < *argc) { + char *opt = (*argv)[i]; + + if (strncmp(opt, "--", 2) != 0 + && !(strlen(opt) == 2 && opt[0] == '-')) { + if (ibx_position_arg != NULL + && ibx_position_arg != opt) { + ibx_msg("Error: extra argument found %s\n", + opt); + } + ibx_position_arg = opt; + ++n_arguments; + } + ++i; + } + + *argc -= n_arguments; + if (n_arguments > 1) { + return(false); + } + + if (ibx_position_arg == NULL) { + ibx_msg("Missing argument\n"); + return(false); + } + + /* set argv[0] to be the program name */ + --(*argv); + ++(*argc); + + return(true); +} + +/*********************************************************************//** +Parse command-line options, connect to MySQL server, +detect server capabilities, etc. +@return true on success. */ +bool +ibx_init() +{ + const char *run; + + /*=====================*/ + xtrabackup_copy_back = opt_ibx_copy_back; + xtrabackup_move_back = opt_ibx_move_back; + opt_galera_info = opt_ibx_galera_info; + opt_slave_info = opt_ibx_slave_info; + opt_no_lock = opt_ibx_no_lock; + opt_safe_slave_backup = opt_ibx_safe_slave_backup; + opt_rsync = opt_ibx_rsync; + opt_force_non_empty_dirs = opt_ibx_force_non_empty_dirs; + opt_noversioncheck = opt_ibx_noversioncheck; + opt_no_backup_locks = opt_ibx_no_backup_locks; + opt_decompress = opt_ibx_decompress; + + opt_incremental_history_name = opt_ibx_incremental_history_name; + opt_incremental_history_uuid = opt_ibx_incremental_history_uuid; + + opt_user = opt_ibx_user; + opt_password = opt_ibx_password; + opt_host = opt_ibx_host; + opt_defaults_group = opt_ibx_defaults_group; + opt_socket = opt_ibx_socket; + opt_port = opt_ibx_port; + opt_login_path = opt_ibx_login_path; + + opt_lock_wait_query_type = opt_ibx_lock_wait_query_type; + opt_kill_long_query_type = opt_ibx_kill_long_query_type; + + opt_decrypt_algo = opt_ibx_decrypt_algo; + + opt_kill_long_queries_timeout = opt_ibx_kill_long_queries_timeout; + opt_lock_wait_timeout = opt_ibx_lock_wait_timeout; + opt_lock_wait_threshold = opt_ibx_lock_wait_threshold; + opt_debug_sleep_before_unlock = opt_ibx_debug_sleep_before_unlock; + opt_safe_slave_backup_timeout = opt_ibx_safe_slave_backup_timeout; + + opt_history = opt_ibx_history; + opt_decrypt = opt_ibx_decrypt; + + /* setup xtrabackup options */ + xb_close_files = ibx_xb_close_files; + xtrabackup_compress_alg = ibx_xtrabackup_compress_alg; + xtrabackup_compress_threads = ibx_xtrabackup_compress_threads; + xtrabackup_compress_chunk_size = ibx_xtrabackup_compress_chunk_size; + xtrabackup_encrypt_algo = ibx_xtrabackup_encrypt_algo; + xtrabackup_encrypt_key = ibx_xtrabackup_encrypt_key; + xtrabackup_encrypt_key_file = ibx_xtrabackup_encrypt_key_file; + xtrabackup_encrypt_threads = ibx_xtrabackup_encrypt_threads; + xtrabackup_encrypt_chunk_size = ibx_xtrabackup_encrypt_chunk_size; + xtrabackup_export = ibx_xtrabackup_export; + xtrabackup_extra_lsndir = ibx_xtrabackup_extra_lsndir; + xtrabackup_incremental_basedir = ibx_xtrabackup_incremental_basedir; + xtrabackup_incremental_dir = ibx_xtrabackup_incremental_dir; + xtrabackup_incremental_force_scan = + ibx_xtrabackup_incremental_force_scan; + xtrabackup_log_copy_interval = ibx_xtrabackup_log_copy_interval; + xtrabackup_incremental = ibx_xtrabackup_incremental; + xtrabackup_parallel = ibx_xtrabackup_parallel; + xtrabackup_stream_str = ibx_xtrabackup_stream_str; + xtrabackup_tables_file = ibx_xtrabackup_tables_file; + xtrabackup_throttle = ibx_xtrabackup_throttle; + opt_mysql_tmpdir = ibx_opt_mysql_tmpdir; + xtrabackup_use_memory = ibx_xtrabackup_use_memory; + + if (!opt_ibx_incremental + && (xtrabackup_incremental + || xtrabackup_incremental_basedir + || opt_ibx_incremental_history_name + || opt_ibx_incremental_history_uuid)) { + ibx_msg("Error: --incremental-lsn, --incremental-basedir, " + "--incremental-history-name and " + "--incremental-history-uuid require the " + "--incremental option.\n"); + return(false); + } + + if (opt_ibx_databases != NULL) { + if (is_path_separator(*opt_ibx_databases)) { + xtrabackup_databases_file = opt_ibx_databases; + } else { + xtrabackup_databases = opt_ibx_databases; + } + } + + /* --tables and --tables-file options are xtrabackup only */ + ibx_partial_backup = (opt_ibx_include || opt_ibx_databases); + + if (ibx_mode == IBX_MODE_BACKUP) { + + if (!make_backup_dir()) { + return(false); + } + } + + /* --binlog-info is xtrabackup only, so force + --binlog-info=ON. i.e. behavior before the feature had been + implemented */ + opt_binlog_info = BINLOG_INFO_ON; + + switch (ibx_mode) { + case IBX_MODE_APPLY_LOG: + xtrabackup_prepare = TRUE; + if (opt_ibx_redo_only) { + xtrabackup_apply_log_only = TRUE; + } + xtrabackup_target_dir = ibx_position_arg; + run = "apply-log"; + break; + case IBX_MODE_BACKUP: + xtrabackup_backup = TRUE; + xtrabackup_target_dir = ibx_backup_directory; + if (opt_ibx_include != NULL) { + xtrabackup_tables = opt_ibx_include; + } + run = "backup"; + break; + case IBX_MODE_COPY_BACK: + xtrabackup_copy_back = TRUE; + xtrabackup_target_dir = ibx_position_arg; + run = "copy-back"; + break; + case IBX_MODE_MOVE_BACK: + xtrabackup_move_back = TRUE; + xtrabackup_target_dir = ibx_position_arg; + run = "move-back"; + break; + case IBX_MODE_DECRYPT_DECOMPRESS: + xtrabackup_decrypt_decompress = TRUE; + xtrabackup_target_dir = ibx_position_arg; + run = "decrypt and decompress"; + break; + default: + ut_error; + } + + ibx_msg("Starting the %s operation\n\n" + "IMPORTANT: Please check that the %s run completes " + "successfully.\n" + " At the end of a successful %s run innobackupex\n" + " prints \"completed OK!\".\n\n", run, run, run); + + + return(true); +} + +void +ibx_cleanup() +{ + free(ibx_backup_directory); +} diff --git a/extra/mariabackup/innobackupex.h b/extra/mariabackup/innobackupex.h new file mode 100644 index 00000000000..e2ad9bd2511 --- /dev/null +++ b/extra/mariabackup/innobackupex.h @@ -0,0 +1,45 @@ +/****************************************************** +Copyright (c) 2011-2014 Percona LLC and/or its affiliates. + +Declarations for innobackupex.cc + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#ifndef INNOBACKUPEX_H +#define INNOBACKUPEX_H + +#define INNOBACKUPEX_BIN_NAME "innobackupex" + +enum ibx_mode_t { + IBX_MODE_BACKUP, + IBX_MODE_APPLY_LOG, + IBX_MODE_COPY_BACK, + IBX_MODE_MOVE_BACK, + IBX_MODE_DECRYPT_DECOMPRESS +}; + +extern ibx_mode_t ibx_mode; + +bool +ibx_handle_options(int *argc, char ***argv); + +bool +ibx_init(); + +void +ibx_cleanup(); + +#endif diff --git a/extra/mariabackup/quicklz/quicklz.c b/extra/mariabackup/quicklz/quicklz.c new file mode 100644 index 00000000000..3742129023a --- /dev/null +++ b/extra/mariabackup/quicklz/quicklz.c @@ -0,0 +1,848 @@ +// Fast data compression library +// Copyright (C) 2006-2011 Lasse Mikkel Reinhold +// lar@quicklz.com +// +// QuickLZ can be used for free under the GPL 1, 2 or 3 license (where anything +// released into public must be open source) or under a commercial license if such +// has been acquired (see http://www.quicklz.com/order.html). The commercial license +// does not cover derived or ported versions created by third parties under GPL. + +// 1.5.0 final + +#include "quicklz.h" + +#if QLZ_VERSION_MAJOR != 1 || QLZ_VERSION_MINOR != 5 || QLZ_VERSION_REVISION != 0 + #error quicklz.c and quicklz.h have different versions +#endif + +#if (defined(__X86__) || defined(__i386__) || defined(i386) || defined(_M_IX86) || defined(__386__) || defined(__x86_64__) || defined(_M_X64)) + #define X86X64 +#endif + +#define MINOFFSET 2 +#define UNCONDITIONAL_MATCHLEN 6 +#define UNCOMPRESSED_END 4 +#define CWORD_LEN 4 + +#if QLZ_COMPRESSION_LEVEL == 1 && defined QLZ_PTR_64 && QLZ_STREAMING_BUFFER == 0 + #define OFFSET_BASE source + #define CAST (ui32)(size_t) +#else + #define OFFSET_BASE 0 + #define CAST +#endif + +int qlz_get_setting(int setting) +{ + switch (setting) + { + case 0: return QLZ_COMPRESSION_LEVEL; + case 1: return sizeof(qlz_state_compress); + case 2: return sizeof(qlz_state_decompress); + case 3: return QLZ_STREAMING_BUFFER; +#ifdef QLZ_MEMORY_SAFE + case 6: return 1; +#else + case 6: return 0; +#endif + case 7: return QLZ_VERSION_MAJOR; + case 8: return QLZ_VERSION_MINOR; + case 9: return QLZ_VERSION_REVISION; + } + return -1; +} + +#if QLZ_COMPRESSION_LEVEL == 1 +static int same(const unsigned char *src, size_t n) +{ + while(n > 0 && *(src + n) == *src) + n--; + return n == 0 ? 1 : 0; +} +#endif + +static void reset_table_compress(qlz_state_compress *state) +{ + int i; + for(i = 0; i < QLZ_HASH_VALUES; i++) + { +#if QLZ_COMPRESSION_LEVEL == 1 + state->hash[i].offset = 0; +#else + state->hash_counter[i] = 0; +#endif + } +} + +static void reset_table_decompress(qlz_state_decompress *state) +{ + int i; + (void)state; + (void)i; +#if QLZ_COMPRESSION_LEVEL == 2 + for(i = 0; i < QLZ_HASH_VALUES; i++) + { + state->hash_counter[i] = 0; + } +#endif +} + +static __inline ui32 hash_func(ui32 i) +{ +#if QLZ_COMPRESSION_LEVEL == 2 + return ((i >> 9) ^ (i >> 13) ^ i) & (QLZ_HASH_VALUES - 1); +#else + return ((i >> 12) ^ i) & (QLZ_HASH_VALUES - 1); +#endif +} + +static __inline ui32 fast_read(void const *src, ui32 bytes) +{ +#ifndef X86X64 + unsigned char *p = (unsigned char*)src; + switch (bytes) + { + case 4: + return(*p | *(p + 1) << 8 | *(p + 2) << 16 | *(p + 3) << 24); + case 3: + return(*p | *(p + 1) << 8 | *(p + 2) << 16); + case 2: + return(*p | *(p + 1) << 8); + case 1: + return(*p); + } + return 0; +#else + if (bytes >= 1 && bytes <= 4) + return *((ui32*)src); + else + return 0; +#endif +} + +static __inline ui32 hashat(const unsigned char *src) +{ + ui32 fetch, hash; + fetch = fast_read(src, 3); + hash = hash_func(fetch); + return hash; +} + +static __inline void fast_write(ui32 f, void *dst, size_t bytes) +{ +#ifndef X86X64 + unsigned char *p = (unsigned char*)dst; + + switch (bytes) + { + case 4: + *p = (unsigned char)f; + *(p + 1) = (unsigned char)(f >> 8); + *(p + 2) = (unsigned char)(f >> 16); + *(p + 3) = (unsigned char)(f >> 24); + return; + case 3: + *p = (unsigned char)f; + *(p + 1) = (unsigned char)(f >> 8); + *(p + 2) = (unsigned char)(f >> 16); + return; + case 2: + *p = (unsigned char)f; + *(p + 1) = (unsigned char)(f >> 8); + return; + case 1: + *p = (unsigned char)f; + return; + } +#else + switch (bytes) + { + case 4: + *((ui32*)dst) = f; + return; + case 3: + *((ui32*)dst) = f; + return; + case 2: + *((ui16 *)dst) = (ui16)f; + return; + case 1: + *((unsigned char*)dst) = (unsigned char)f; + return; + } +#endif +} + + +size_t qlz_size_decompressed(const char *source) +{ + ui32 n, r; + n = (((*source) & 2) == 2) ? 4 : 1; + r = fast_read(source + 1 + n, n); + r = r & (0xffffffff >> ((4 - n)*8)); + return r; +} + +size_t qlz_size_compressed(const char *source) +{ + ui32 n, r; + n = (((*source) & 2) == 2) ? 4 : 1; + r = fast_read(source + 1, n); + r = r & (0xffffffff >> ((4 - n)*8)); + return r; +} + +size_t qlz_size_header(const char *source) +{ + size_t n = 2*((((*source) & 2) == 2) ? 4 : 1) + 1; + return n; +} + + +static __inline void memcpy_up(unsigned char *dst, const unsigned char *src, ui32 n) +{ + // Caution if modifying memcpy_up! Overlap of dst and src must be special handled. +#ifndef X86X64 + unsigned char *end = dst + n; + while(dst < end) + { + *dst = *src; + dst++; + src++; + } +#else + ui32 f = 0; + do + { + *(ui32 *)(dst + f) = *(ui32 *)(src + f); + f += MINOFFSET + 1; + } + while (f < n); +#endif +} + +static __inline void update_hash(qlz_state_decompress *state, const unsigned char *s) +{ +#if QLZ_COMPRESSION_LEVEL == 1 + ui32 hash; + hash = hashat(s); + state->hash[hash].offset = s; + state->hash_counter[hash] = 1; +#elif QLZ_COMPRESSION_LEVEL == 2 + ui32 hash; + unsigned char c; + hash = hashat(s); + c = state->hash_counter[hash]; + state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = s; + c++; + state->hash_counter[hash] = c; +#endif + (void)state; + (void)s; +} + +#if QLZ_COMPRESSION_LEVEL <= 2 +static void update_hash_upto(qlz_state_decompress *state, unsigned char **lh, const unsigned char *max) +{ + while(*lh < max) + { + (*lh)++; + update_hash(state, *lh); + } +} +#endif + +static size_t qlz_compress_core(const unsigned char *source, unsigned char *destination, size_t size, qlz_state_compress *state) +{ + const unsigned char *last_byte = source + size - 1; + const unsigned char *src = source; + unsigned char *cword_ptr = destination; + unsigned char *dst = destination + CWORD_LEN; + ui32 cword_val = 1U << 31; + const unsigned char *last_matchstart = last_byte - UNCONDITIONAL_MATCHLEN - UNCOMPRESSED_END; + ui32 fetch = 0; + unsigned int lits = 0; + + (void) lits; + + if(src <= last_matchstart) + fetch = fast_read(src, 3); + + while(src <= last_matchstart) + { + if ((cword_val & 1) == 1) + { + // store uncompressed if compression ratio is too low + if (src > source + (size >> 1) && dst - destination > src - source - ((src - source) >> 5)) + return 0; + + fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN); + + cword_ptr = dst; + dst += CWORD_LEN; + cword_val = 1U << 31; + fetch = fast_read(src, 3); + } +#if QLZ_COMPRESSION_LEVEL == 1 + { + const unsigned char *o; + ui32 hash, cached; + + hash = hash_func(fetch); + cached = fetch ^ state->hash[hash].cache; + state->hash[hash].cache = fetch; + + o = state->hash[hash].offset + OFFSET_BASE; + state->hash[hash].offset = CAST(src - OFFSET_BASE); + +#ifdef X86X64 + if ((cached & 0xffffff) == 0 && o != OFFSET_BASE && (src - o > MINOFFSET || (src == o + 1 && lits >= 3 && src > source + 3 && same(src - 3, 6)))) + { + if(cached != 0) + { +#else + if (cached == 0 && o != OFFSET_BASE && (src - o > MINOFFSET || (src == o + 1 && lits >= 3 && src > source + 3 && same(src - 3, 6)))) + { + if (*(o + 3) != *(src + 3)) + { +#endif + hash <<= 4; + cword_val = (cword_val >> 1) | (1U << 31); + fast_write((3 - 2) | hash, dst, 2); + src += 3; + dst += 2; + } + else + { + const unsigned char *old_src = src; + size_t matchlen; + hash <<= 4; + + cword_val = (cword_val >> 1) | (1U << 31); + src += 4; + + if(*(o + (src - old_src)) == *src) + { + src++; + if(*(o + (src - old_src)) == *src) + { + size_t q = last_byte - UNCOMPRESSED_END - (src - 5) + 1; + size_t remaining = q > 255 ? 255 : q; + src++; + while(*(o + (src - old_src)) == *src && (size_t)(src - old_src) < remaining) + src++; + } + } + + matchlen = src - old_src; + if (matchlen < 18) + { + fast_write((ui32)(matchlen - 2) | hash, dst, 2); + dst += 2; + } + else + { + fast_write((ui32)(matchlen << 16) | hash, dst, 3); + dst += 3; + } + } + fetch = fast_read(src, 3); + lits = 0; + } + else + { + lits++; + *dst = *src; + src++; + dst++; + cword_val = (cword_val >> 1); +#ifdef X86X64 + fetch = fast_read(src, 3); +#else + fetch = (fetch >> 8 & 0xffff) | (*(src + 2) << 16); +#endif + } + } +#elif QLZ_COMPRESSION_LEVEL >= 2 + { + const unsigned char *o, *offset2; + ui32 hash, matchlen, k, m, best_k = 0; + unsigned char c; + size_t remaining = (last_byte - UNCOMPRESSED_END - src + 1) > 255 ? 255 : (last_byte - UNCOMPRESSED_END - src + 1); + (void)best_k; + + + //hash = hashat(src); + fetch = fast_read(src, 3); + hash = hash_func(fetch); + + c = state->hash_counter[hash]; + + offset2 = state->hash[hash].offset[0]; + if(offset2 < src - MINOFFSET && c > 0 && ((fast_read(offset2, 3) ^ fetch) & 0xffffff) == 0) + { + matchlen = 3; + if(*(offset2 + matchlen) == *(src + matchlen)) + { + matchlen = 4; + while(*(offset2 + matchlen) == *(src + matchlen) && matchlen < remaining) + matchlen++; + } + } + else + matchlen = 0; + for(k = 1; k < QLZ_POINTERS && c > k; k++) + { + o = state->hash[hash].offset[k]; +#if QLZ_COMPRESSION_LEVEL == 3 + if(((fast_read(o, 3) ^ fetch) & 0xffffff) == 0 && o < src - MINOFFSET) +#elif QLZ_COMPRESSION_LEVEL == 2 + if(*(src + matchlen) == *(o + matchlen) && ((fast_read(o, 3) ^ fetch) & 0xffffff) == 0 && o < src - MINOFFSET) +#endif + { + m = 3; + while(*(o + m) == *(src + m) && m < remaining) + m++; +#if QLZ_COMPRESSION_LEVEL == 3 + if ((m > matchlen) || (m == matchlen && o > offset2)) +#elif QLZ_COMPRESSION_LEVEL == 2 + if (m > matchlen) +#endif + { + offset2 = o; + matchlen = m; + best_k = k; + } + } + } + o = offset2; + state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src; + c++; + state->hash_counter[hash] = c; + +#if QLZ_COMPRESSION_LEVEL == 3 + if(matchlen > 2 && src - o < 131071) + { + ui32 u; + size_t offset = src - o; + + for(u = 1; u < matchlen; u++) + { + hash = hashat(src + u); + c = state->hash_counter[hash]++; + state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src + u; + } + + cword_val = (cword_val >> 1) | (1U << 31); + src += matchlen; + + if(matchlen == 3 && offset <= 63) + { + *dst = (unsigned char)(offset << 2); + dst++; + } + else if (matchlen == 3 && offset <= 16383) + { + ui32 f = (ui32)((offset << 2) | 1); + fast_write(f, dst, 2); + dst += 2; + } + else if (matchlen <= 18 && offset <= 1023) + { + ui32 f = ((matchlen - 3) << 2) | ((ui32)offset << 6) | 2; + fast_write(f, dst, 2); + dst += 2; + } + + else if(matchlen <= 33) + { + ui32 f = ((matchlen - 2) << 2) | ((ui32)offset << 7) | 3; + fast_write(f, dst, 3); + dst += 3; + } + else + { + ui32 f = ((matchlen - 3) << 7) | ((ui32)offset << 15) | 3; + fast_write(f, dst, 4); + dst += 4; + } + } + else + { + *dst = *src; + src++; + dst++; + cword_val = (cword_val >> 1); + } +#elif QLZ_COMPRESSION_LEVEL == 2 + + if(matchlen > 2) + { + cword_val = (cword_val >> 1) | (1U << 31); + src += matchlen; + + if (matchlen < 10) + { + ui32 f = best_k | ((matchlen - 2) << 2) | (hash << 5); + fast_write(f, dst, 2); + dst += 2; + } + else + { + ui32 f = best_k | (matchlen << 16) | (hash << 5); + fast_write(f, dst, 3); + dst += 3; + } + } + else + { + *dst = *src; + src++; + dst++; + cword_val = (cword_val >> 1); + } +#endif + } +#endif + } + while (src <= last_byte) + { + if ((cword_val & 1) == 1) + { + fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN); + cword_ptr = dst; + dst += CWORD_LEN; + cword_val = 1U << 31; + } +#if QLZ_COMPRESSION_LEVEL < 3 + if (src <= last_byte - 3) + { +#if QLZ_COMPRESSION_LEVEL == 1 + ui32 hash, fetch; + fetch = fast_read(src, 3); + hash = hash_func(fetch); + state->hash[hash].offset = CAST(src - OFFSET_BASE); + state->hash[hash].cache = fetch; +#elif QLZ_COMPRESSION_LEVEL == 2 + ui32 hash; + unsigned char c; + hash = hashat(src); + c = state->hash_counter[hash]; + state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src; + c++; + state->hash_counter[hash] = c; +#endif + } +#endif + *dst = *src; + src++; + dst++; + cword_val = (cword_val >> 1); + } + + while((cword_val & 1) != 1) + cword_val = (cword_val >> 1); + + fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN); + + // min. size must be 9 bytes so that the qlz_size functions can take 9 bytes as argument + return dst - destination < 9 ? 9 : dst - destination; +} + +static size_t qlz_decompress_core(const unsigned char *source, unsigned char *destination, size_t size, qlz_state_decompress *state, const unsigned char *history) +{ + const unsigned char *src = source + qlz_size_header((const char *)source); + unsigned char *dst = destination; + const unsigned char *last_destination_byte = destination + size - 1; + ui32 cword_val = 1; + const unsigned char *last_matchstart = last_destination_byte - UNCONDITIONAL_MATCHLEN - UNCOMPRESSED_END; + unsigned char *last_hashed = destination - 1; + const unsigned char *last_source_byte = source + qlz_size_compressed((const char *)source) - 1; + static const ui32 bitlut[16] = {4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0}; + + (void) last_source_byte; + (void) last_hashed; + (void) state; + (void) history; + + for(;;) + { + ui32 fetch; + + if (cword_val == 1) + { +#ifdef QLZ_MEMORY_SAFE + if(src + CWORD_LEN - 1 > last_source_byte) + return 0; +#endif + cword_val = fast_read(src, CWORD_LEN); + src += CWORD_LEN; + } + +#ifdef QLZ_MEMORY_SAFE + if(src + 4 - 1 > last_source_byte) + return 0; +#endif + + fetch = fast_read(src, 4); + + if ((cword_val & 1) == 1) + { + ui32 matchlen; + const unsigned char *offset2; + +#if QLZ_COMPRESSION_LEVEL == 1 + ui32 hash; + cword_val = cword_val >> 1; + hash = (fetch >> 4) & 0xfff; + offset2 = (const unsigned char *)(size_t)state->hash[hash].offset; + + if((fetch & 0xf) != 0) + { + matchlen = (fetch & 0xf) + 2; + src += 2; + } + else + { + matchlen = *(src + 2); + src += 3; + } + +#elif QLZ_COMPRESSION_LEVEL == 2 + ui32 hash; + unsigned char c; + cword_val = cword_val >> 1; + hash = (fetch >> 5) & 0x7ff; + c = (unsigned char)(fetch & 0x3); + offset2 = state->hash[hash].offset[c]; + + if((fetch & (28)) != 0) + { + matchlen = ((fetch >> 2) & 0x7) + 2; + src += 2; + } + else + { + matchlen = *(src + 2); + src += 3; + } + +#elif QLZ_COMPRESSION_LEVEL == 3 + ui32 offset; + cword_val = cword_val >> 1; + if ((fetch & 3) == 0) + { + offset = (fetch & 0xff) >> 2; + matchlen = 3; + src++; + } + else if ((fetch & 2) == 0) + { + offset = (fetch & 0xffff) >> 2; + matchlen = 3; + src += 2; + } + else if ((fetch & 1) == 0) + { + offset = (fetch & 0xffff) >> 6; + matchlen = ((fetch >> 2) & 15) + 3; + src += 2; + } + else if ((fetch & 127) != 3) + { + offset = (fetch >> 7) & 0x1ffff; + matchlen = ((fetch >> 2) & 0x1f) + 2; + src += 3; + } + else + { + offset = (fetch >> 15); + matchlen = ((fetch >> 7) & 255) + 3; + src += 4; + } + + offset2 = dst - offset; +#endif + +#ifdef QLZ_MEMORY_SAFE + if(offset2 < history || offset2 > dst - MINOFFSET - 1) + return 0; + + if(matchlen > (ui32)(last_destination_byte - dst - UNCOMPRESSED_END + 1)) + return 0; +#endif + + memcpy_up(dst, offset2, matchlen); + dst += matchlen; + +#if QLZ_COMPRESSION_LEVEL <= 2 + update_hash_upto(state, &last_hashed, dst - matchlen); + last_hashed = dst - 1; +#endif + } + else + { + if (dst < last_matchstart) + { + unsigned int n = bitlut[cword_val & 0xf]; +#ifdef X86X64 + *(ui32 *)dst = *(ui32 *)src; +#else + memcpy_up(dst, src, 4); +#endif + cword_val = cword_val >> n; + dst += n; + src += n; +#if QLZ_COMPRESSION_LEVEL <= 2 + update_hash_upto(state, &last_hashed, dst - 3); +#endif + } + else + { + while(dst <= last_destination_byte) + { + if (cword_val == 1) + { + src += CWORD_LEN; + cword_val = 1U << 31; + } +#ifdef QLZ_MEMORY_SAFE + if(src >= last_source_byte + 1) + return 0; +#endif + *dst = *src; + dst++; + src++; + cword_val = cword_val >> 1; + } + +#if QLZ_COMPRESSION_LEVEL <= 2 + update_hash_upto(state, &last_hashed, last_destination_byte - 3); // todo, use constant +#endif + return size; + } + + } + } +} + +size_t qlz_compress(const void *source, char *destination, size_t size, qlz_state_compress *state) +{ + size_t r; + ui32 compressed; + size_t base; + + if(size == 0 || size > 0xffffffff - 400) + return 0; + + if(size < 216) + base = 3; + else + base = 9; + +#if QLZ_STREAMING_BUFFER > 0 + if (state->stream_counter + size - 1 >= QLZ_STREAMING_BUFFER) +#endif + { + reset_table_compress(state); + r = base + qlz_compress_core((const unsigned char *)source, (unsigned char*)destination + base, size, state); +#if QLZ_STREAMING_BUFFER > 0 + reset_table_compress(state); +#endif + if(r == base) + { + memcpy(destination + base, source, size); + r = size + base; + compressed = 0; + } + else + { + compressed = 1; + } + state->stream_counter = 0; + } +#if QLZ_STREAMING_BUFFER > 0 + else + { + unsigned char *src = state->stream_buffer + state->stream_counter; + + memcpy(src, source, size); + r = base + qlz_compress_core(src, (unsigned char*)destination + base, size, state); + + if(r == base) + { + memcpy(destination + base, src, size); + r = size + base; + compressed = 0; + reset_table_compress(state); + } + else + { + compressed = 1; + } + state->stream_counter += size; + } +#endif + if(base == 3) + { + *destination = (unsigned char)(0 | compressed); + *(destination + 1) = (unsigned char)r; + *(destination + 2) = (unsigned char)size; + } + else + { + *destination = (unsigned char)(2 | compressed); + fast_write((ui32)r, destination + 1, 4); + fast_write((ui32)size, destination + 5, 4); + } + + *destination |= (QLZ_COMPRESSION_LEVEL << 2); + *destination |= (1 << 6); + *destination |= ((QLZ_STREAMING_BUFFER == 0 ? 0 : (QLZ_STREAMING_BUFFER == 100000 ? 1 : (QLZ_STREAMING_BUFFER == 1000000 ? 2 : 3))) << 4); + +// 76543210 +// 01SSLLHC + + return r; +} + +size_t qlz_decompress(const char *source, void *destination, qlz_state_decompress *state) +{ + size_t dsiz = qlz_size_decompressed(source); + +#if QLZ_STREAMING_BUFFER > 0 + if (state->stream_counter + qlz_size_decompressed(source) - 1 >= QLZ_STREAMING_BUFFER) +#endif + { + if((*source & 1) == 1) + { + reset_table_decompress(state); + dsiz = qlz_decompress_core((const unsigned char *)source, (unsigned char *)destination, dsiz, state, (const unsigned char *)destination); + } + else + { + memcpy(destination, source + qlz_size_header(source), dsiz); + } + state->stream_counter = 0; + reset_table_decompress(state); + } +#if QLZ_STREAMING_BUFFER > 0 + else + { + unsigned char *dst = state->stream_buffer + state->stream_counter; + if((*source & 1) == 1) + { + dsiz = qlz_decompress_core((const unsigned char *)source, dst, dsiz, state, (const unsigned char *)state->stream_buffer); + } + else + { + memcpy(dst, source + qlz_size_header(source), dsiz); + reset_table_decompress(state); + } + memcpy(destination, dst, dsiz); + state->stream_counter += dsiz; + } +#endif + return dsiz; +} + diff --git a/extra/mariabackup/quicklz/quicklz.h b/extra/mariabackup/quicklz/quicklz.h new file mode 100644 index 00000000000..6ffe00f3a91 --- /dev/null +++ b/extra/mariabackup/quicklz/quicklz.h @@ -0,0 +1,144 @@ +#ifndef QLZ_HEADER +#define QLZ_HEADER + +// Fast data compression library +// Copyright (C) 2006-2011 Lasse Mikkel Reinhold +// lar@quicklz.com +// +// QuickLZ can be used for free under the GPL 1, 2 or 3 license (where anything +// released into public must be open source) or under a commercial license if such +// has been acquired (see http://www.quicklz.com/order.html). The commercial license +// does not cover derived or ported versions created by third parties under GPL. + +// You can edit following user settings. Data must be decompressed with the same +// setting of QLZ_COMPRESSION_LEVEL and QLZ_STREAMING_BUFFER as it was compressed +// (see manual). If QLZ_STREAMING_BUFFER > 0, scratch buffers must be initially +// zeroed out (see manual). First #ifndef makes it possible to define settings from +// the outside like the compiler command line. + +// 1.5.0 final + +#ifndef QLZ_COMPRESSION_LEVEL + #define QLZ_COMPRESSION_LEVEL 1 + //#define QLZ_COMPRESSION_LEVEL 2 + //#define QLZ_COMPRESSION_LEVEL 3 + + #define QLZ_STREAMING_BUFFER 0 + //#define QLZ_STREAMING_BUFFER 100000 + //#define QLZ_STREAMING_BUFFER 1000000 + + //#define QLZ_MEMORY_SAFE +#endif + +#define QLZ_VERSION_MAJOR 1 +#define QLZ_VERSION_MINOR 5 +#define QLZ_VERSION_REVISION 0 + +// Using size_t, memset() and memcpy() +#include <string.h> + +// Verify compression level +#if QLZ_COMPRESSION_LEVEL != 1 && QLZ_COMPRESSION_LEVEL != 2 && QLZ_COMPRESSION_LEVEL != 3 +#error QLZ_COMPRESSION_LEVEL must be 1, 2 or 3 +#endif + +typedef unsigned int ui32; +typedef unsigned short int ui16; + +// Decrease QLZ_POINTERS for level 3 to increase compression speed. Do not touch any other values! +#if QLZ_COMPRESSION_LEVEL == 1 +#define QLZ_POINTERS 1 +#define QLZ_HASH_VALUES 4096 +#elif QLZ_COMPRESSION_LEVEL == 2 +#define QLZ_POINTERS 4 +#define QLZ_HASH_VALUES 2048 +#elif QLZ_COMPRESSION_LEVEL == 3 +#define QLZ_POINTERS 16 +#define QLZ_HASH_VALUES 4096 +#endif + +// Detect if pointer size is 64-bit. It's not fatal if some 64-bit target is not detected because this is only for adding an optional 64-bit optimization. +#if defined _LP64 || defined __LP64__ || defined __64BIT__ || _ADDR64 || defined _WIN64 || defined __arch64__ || __WORDSIZE == 64 || (defined __sparc && defined __sparcv9) || defined __x86_64 || defined __amd64 || defined __x86_64__ || defined _M_X64 || defined _M_IA64 || defined __ia64 || defined __IA64__ + #define QLZ_PTR_64 +#endif + +// hash entry +typedef struct +{ +#if QLZ_COMPRESSION_LEVEL == 1 + ui32 cache; +#if defined QLZ_PTR_64 && QLZ_STREAMING_BUFFER == 0 + unsigned int offset; +#else + const unsigned char *offset; +#endif +#else + const unsigned char *offset[QLZ_POINTERS]; +#endif + +} qlz_hash_compress; + +typedef struct +{ +#if QLZ_COMPRESSION_LEVEL == 1 + const unsigned char *offset; +#else + const unsigned char *offset[QLZ_POINTERS]; +#endif +} qlz_hash_decompress; + + +// states +typedef struct +{ + #if QLZ_STREAMING_BUFFER > 0 + unsigned char stream_buffer[QLZ_STREAMING_BUFFER]; + #endif + size_t stream_counter; + qlz_hash_compress hash[QLZ_HASH_VALUES]; + unsigned char hash_counter[QLZ_HASH_VALUES]; +} qlz_state_compress; + + +#if QLZ_COMPRESSION_LEVEL == 1 || QLZ_COMPRESSION_LEVEL == 2 + typedef struct + { +#if QLZ_STREAMING_BUFFER > 0 + unsigned char stream_buffer[QLZ_STREAMING_BUFFER]; +#endif + qlz_hash_decompress hash[QLZ_HASH_VALUES]; + unsigned char hash_counter[QLZ_HASH_VALUES]; + size_t stream_counter; + } qlz_state_decompress; +#elif QLZ_COMPRESSION_LEVEL == 3 + typedef struct + { +#if QLZ_STREAMING_BUFFER > 0 + unsigned char stream_buffer[QLZ_STREAMING_BUFFER]; +#endif +#if QLZ_COMPRESSION_LEVEL <= 2 + qlz_hash_decompress hash[QLZ_HASH_VALUES]; +#endif + size_t stream_counter; + } qlz_state_decompress; +#endif + + +#if defined (__cplusplus) +extern "C" { +#endif + +// Public functions of QuickLZ +size_t qlz_size_decompressed(const char *source); +size_t qlz_size_compressed(const char *source); +size_t qlz_compress(const void *source, char *destination, size_t size, qlz_state_compress *state); +size_t qlz_decompress(const char *source, void *destination, qlz_state_decompress *state); +int qlz_get_setting(int setting); +size_t qlz_size_header(const char *source); + +#if defined (__cplusplus) +} +#endif + +#endif + diff --git a/extra/mariabackup/read_filt.cc b/extra/mariabackup/read_filt.cc new file mode 100644 index 00000000000..05e6b7c86c7 --- /dev/null +++ b/extra/mariabackup/read_filt.cc @@ -0,0 +1,206 @@ +/****************************************************** +XtraBackup: hot backup tool for InnoDB +(c) 2009-2012 Percona Inc. +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +/* Data file read filter implementation */ + +#include "read_filt.h" +#include "common.h" +#include "fil_cur.h" +#include "xtrabackup.h" + +/****************************************************************//** +Perform read filter context initialization that is common to all read +filters. */ +static +void +common_init( +/*========*/ + xb_read_filt_ctxt_t* ctxt, /*!<in/out: read filter context */ + const xb_fil_cur_t* cursor) /*!<in: file cursor */ +{ + ctxt->offset = 0; + ctxt->data_file_size = cursor->statinfo.st_size; + ctxt->buffer_capacity = cursor->buf_size; + ctxt->page_size = cursor->page_size; +} + +/****************************************************************//** +Initialize the pass-through read filter. */ +static +void +rf_pass_through_init( +/*=================*/ + xb_read_filt_ctxt_t* ctxt, /*!<in/out: read filter context */ + const xb_fil_cur_t* cursor, /*!<in: file cursor */ + ulint space_id __attribute__((unused))) + /*!<in: space id we are reading */ +{ + common_init(ctxt, cursor); +} + +/****************************************************************//** +Get the next batch of pages for the pass-through read filter. */ +static +void +rf_pass_through_get_next_batch( +/*===========================*/ + xb_read_filt_ctxt_t* ctxt, /*!<in/out: read filter + context */ + ib_int64_t* read_batch_start, /*!<out: starting read + offset in bytes for the + next batch of pages */ + ib_int64_t* read_batch_len) /*!<out: length in + bytes of the next batch + of pages */ +{ + *read_batch_start = ctxt->offset; + *read_batch_len = ctxt->data_file_size - ctxt->offset; + + if (*read_batch_len > (ib_int64_t)ctxt->buffer_capacity) { + *read_batch_len = ctxt->buffer_capacity; + } + + ctxt->offset += *read_batch_len; +} + +/****************************************************************//** +Deinitialize the pass-through read filter. */ +static +void +rf_pass_through_deinit( +/*===================*/ + xb_read_filt_ctxt_t* ctxt __attribute__((unused))) + /*!<in: read filter context */ +{ +} + +/****************************************************************//** +Initialize the changed page bitmap-based read filter. Assumes that +the bitmap is already set up in changed_page_bitmap. */ +static +void +rf_bitmap_init( +/*===========*/ + xb_read_filt_ctxt_t* ctxt, /*!<in/out: read filter + context */ + const xb_fil_cur_t* cursor, /*!<in: read cursor */ + ulint space_id) /*!<in: space id */ +{ + common_init(ctxt, cursor); + ctxt->bitmap_range = xb_page_bitmap_range_init(changed_page_bitmap, + space_id); + ctxt->filter_batch_end = 0; +} + +/****************************************************************//** +Get the next batch of pages for the bitmap read filter. */ +static +void +rf_bitmap_get_next_batch( +/*=====================*/ + xb_read_filt_ctxt_t* ctxt, /*!<in/out: read filter + context */ + ib_int64_t* read_batch_start, /*!<out: starting read + offset in bytes for the + next batch of pages */ + ib_int64_t* read_batch_len) /*!<out: length in + bytes of the next batch + of pages */ +{ + ulint start_page_id; + + start_page_id = (ulint)(ctxt->offset / ctxt->page_size); + + xb_a (ctxt->offset % ctxt->page_size == 0); + + if (start_page_id == ctxt->filter_batch_end) { + + /* Used up all the previous bitmap range, get some more */ + ulint next_page_id; + + /* Find the next changed page using the bitmap */ + next_page_id = xb_page_bitmap_range_get_next_bit + (ctxt->bitmap_range, TRUE); + + if (next_page_id == ULINT_UNDEFINED) { + *read_batch_len = 0; + return; + } + + ctxt->offset = next_page_id * ctxt->page_size; + + /* Find the end of the current changed page block by searching + for the next cleared bitmap bit */ + ctxt->filter_batch_end + = xb_page_bitmap_range_get_next_bit(ctxt->bitmap_range, + FALSE); + xb_a(next_page_id < ctxt->filter_batch_end); + } + + *read_batch_start = ctxt->offset; + if (ctxt->filter_batch_end == ULINT_UNDEFINED) { + /* No more cleared bits in the bitmap, need to copy all the + remaining pages. */ + *read_batch_len = ctxt->data_file_size - ctxt->offset; + } else { + *read_batch_len = ctxt->filter_batch_end * ctxt->page_size + - ctxt->offset; + } + + /* If the page block is larger than the buffer capacity, limit it to + buffer capacity. The subsequent invocations will continue returning + the current block in buffer-sized pieces until ctxt->filter_batch_end + is reached, trigerring the next bitmap query. */ + if (*read_batch_len > (ib_int64_t)ctxt->buffer_capacity) { + *read_batch_len = ctxt->buffer_capacity; + } + + ctxt->offset += *read_batch_len; + xb_a (ctxt->offset % ctxt->page_size == 0); + xb_a (*read_batch_start % ctxt->page_size == 0); + xb_a (*read_batch_len % ctxt->page_size == 0); +} + +/****************************************************************//** +Deinitialize the changed page bitmap-based read filter. */ +static +void +rf_bitmap_deinit( +/*=============*/ + xb_read_filt_ctxt_t* ctxt) /*!<in/out: read filter context */ +{ + xb_page_bitmap_range_deinit(ctxt->bitmap_range); +} + +/* The pass-through read filter */ +xb_read_filt_t rf_pass_through = { + &rf_pass_through_init, + &rf_pass_through_get_next_batch, + &rf_pass_through_deinit +}; + +/* The changed page bitmap-based read filter */ +xb_read_filt_t rf_bitmap = { + &rf_bitmap_init, + &rf_bitmap_get_next_batch, + &rf_bitmap_deinit +}; diff --git a/extra/mariabackup/read_filt.h b/extra/mariabackup/read_filt.h new file mode 100644 index 00000000000..d16f4e1093d --- /dev/null +++ b/extra/mariabackup/read_filt.h @@ -0,0 +1,62 @@ +/****************************************************** +XtraBackup: hot backup tool for InnoDB +(c) 2009-2012 Percona Inc. +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +/* Data file read filter interface */ + +#ifndef XB_READ_FILT_H +#define XB_READ_FILT_H + +#include "changed_page_bitmap.h" + +struct xb_fil_cur_t; + +/* The read filter context */ +struct xb_read_filt_ctxt_t { + ib_int64_t offset; /*!< current file offset */ + ib_int64_t data_file_size; /*!< data file size */ + size_t buffer_capacity;/*!< read buffer capacity */ + ib_int64_t space_id; /*!< space id */ + /* The following fields used only in bitmap filter */ + /* Move these to union if any other filters are added in future */ + xb_page_bitmap_range *bitmap_range; /*!< changed page bitmap range + iterator for space_id */ + size_t page_size; /*!< page size */ + ulint filter_batch_end;/*!< the ending page id of the + current changed page block in + the bitmap */ +}; + +/* The read filter */ +struct xb_read_filt_t { + void (*init)(xb_read_filt_ctxt_t* ctxt, + const xb_fil_cur_t* cursor, + ulint space_id); + void (*get_next_batch)(xb_read_filt_ctxt_t* ctxt, + ib_int64_t* read_batch_start, + ib_int64_t* read_batch_len); + void (*deinit)(xb_read_filt_ctxt_t* ctxt); +}; + +extern xb_read_filt_t rf_pass_through; +extern xb_read_filt_t rf_bitmap; + +#endif diff --git a/extra/mariabackup/write_filt.cc b/extra/mariabackup/write_filt.cc new file mode 100644 index 00000000000..cf7753bf380 --- /dev/null +++ b/extra/mariabackup/write_filt.cc @@ -0,0 +1,219 @@ +/****************************************************** +XtraBackup: hot backup tool for InnoDB +(c) 2009-2013 Percona LLC and/or its affiliates. +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +/* Page write filters implementation */ + +#include <my_base.h> +#include "common.h" +#include "write_filt.h" +#include "fil_cur.h" +#include "xtrabackup.h" + +/************************************************************************ +Write-through page write filter. */ +static my_bool wf_wt_init(xb_write_filt_ctxt_t *ctxt, char *dst_name, + xb_fil_cur_t *cursor); +static my_bool wf_wt_process(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile); + +xb_write_filt_t wf_write_through = { + &wf_wt_init, + &wf_wt_process, + NULL, + NULL +}; + +/************************************************************************ +Incremental page write filter. */ +static my_bool wf_incremental_init(xb_write_filt_ctxt_t *ctxt, char *dst_name, + xb_fil_cur_t *cursor); +static my_bool wf_incremental_process(xb_write_filt_ctxt_t *ctxt, + ds_file_t *dstfile); +static my_bool wf_incremental_finalize(xb_write_filt_ctxt_t *ctxt, + ds_file_t *dstfile); +static void wf_incremental_deinit(xb_write_filt_ctxt_t *ctxt); + +xb_write_filt_t wf_incremental = { + &wf_incremental_init, + &wf_incremental_process, + &wf_incremental_finalize, + &wf_incremental_deinit +}; + +/************************************************************************ +Initialize incremental page write filter. + +@return TRUE on success, FALSE on error. */ +static my_bool +wf_incremental_init(xb_write_filt_ctxt_t *ctxt, char *dst_name, + xb_fil_cur_t *cursor) +{ + char meta_name[FN_REFLEN]; + xb_delta_info_t info; + ulint buf_size; + xb_wf_incremental_ctxt_t *cp = + &(ctxt->u.wf_incremental_ctxt); + + ctxt->cursor = cursor; + + /* allocate buffer for incremental backup (4096 pages) */ + buf_size = (cursor->page_size / 4 + 1) * cursor->page_size; + cp->delta_buf_base = static_cast<byte *>(ut_malloc(buf_size)); + memset(cp->delta_buf_base, 0, buf_size); + cp->delta_buf = static_cast<byte *> + (ut_align(cp->delta_buf_base, UNIV_PAGE_SIZE_MAX)); + + /* write delta meta info */ + snprintf(meta_name, sizeof(meta_name), "%s%s", dst_name, + XB_DELTA_INFO_SUFFIX); + info.page_size = cursor->page_size; + info.zip_size = cursor->zip_size; + info.space_id = cursor->space_id; + if (!xb_write_delta_metadata(meta_name, &info)) { + msg("[%02u] xtrabackup: Error: " + "failed to write meta info for %s\n", + cursor->thread_n, cursor->rel_path); + return(FALSE); + } + + /* change the target file name, since we are only going to write + delta pages */ + strcat(dst_name, ".delta"); + + mach_write_to_4(cp->delta_buf, 0x78747261UL); /*"xtra"*/ + cp->npages = 1; + + return(TRUE); +} + +/************************************************************************ +Run the next batch of pages through incremental page write filter. + +@return TRUE on success, FALSE on error. */ +static my_bool +wf_incremental_process(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile) +{ + ulint i; + xb_fil_cur_t *cursor = ctxt->cursor; + ulint page_size = cursor->page_size; + byte *page; + xb_wf_incremental_ctxt_t *cp = &(ctxt->u.wf_incremental_ctxt); + + for (i = 0, page = cursor->buf; i < cursor->buf_npages; + i++, page += page_size) { + + if (incremental_lsn >= mach_read_from_8(page + FIL_PAGE_LSN)) { + + continue; + } + + /* updated page */ + if (cp->npages == page_size / 4) { + /* flush buffer */ + if (ds_write(dstfile, cp->delta_buf, + cp->npages * page_size)) { + return(FALSE); + } + + /* clear buffer */ + memset(cp->delta_buf, 0, page_size / 4 * page_size); + /*"xtra"*/ + mach_write_to_4(cp->delta_buf, 0x78747261UL); + cp->npages = 1; + } + + mach_write_to_4(cp->delta_buf + cp->npages * 4, + cursor->buf_page_no + i); + memcpy(cp->delta_buf + cp->npages * page_size, page, + page_size); + + cp->npages++; + } + + return(TRUE); +} + +/************************************************************************ +Flush the incremental page write filter's buffer. + +@return TRUE on success, FALSE on error. */ +static my_bool +wf_incremental_finalize(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile) +{ + xb_fil_cur_t *cursor = ctxt->cursor; + ulint page_size = cursor->page_size; + xb_wf_incremental_ctxt_t *cp = &(ctxt->u.wf_incremental_ctxt); + + if (cp->npages != page_size / 4) { + mach_write_to_4(cp->delta_buf + cp->npages * 4, 0xFFFFFFFFUL); + } + + /* Mark the final block */ + mach_write_to_4(cp->delta_buf, 0x58545241UL); /*"XTRA"*/ + + /* flush buffer */ + if (ds_write(dstfile, cp->delta_buf, cp->npages * page_size)) { + return(FALSE); + } + + return(TRUE); +} + +/************************************************************************ +Free the incremental page write filter's buffer. */ +static void +wf_incremental_deinit(xb_write_filt_ctxt_t *ctxt) +{ + xb_wf_incremental_ctxt_t *cp = &(ctxt->u.wf_incremental_ctxt); + + if (cp->delta_buf_base != NULL) { + ut_free(cp->delta_buf_base); + } +} + +/************************************************************************ +Initialize the write-through page write filter. + +@return TRUE on success, FALSE on error. */ +static my_bool +wf_wt_init(xb_write_filt_ctxt_t *ctxt, char *dst_name __attribute__((unused)), + xb_fil_cur_t *cursor) +{ + ctxt->cursor = cursor; + + return(TRUE); +} + +/************************************************************************ +Write the next batch of pages to the destination datasink. + +@return TRUE on success, FALSE on error. */ +static my_bool +wf_wt_process(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile) +{ + xb_fil_cur_t *cursor = ctxt->cursor; + + if (ds_write(dstfile, cursor->buf, cursor->buf_read)) { + return(FALSE); + } + + return(TRUE); +} diff --git a/extra/mariabackup/write_filt.h b/extra/mariabackup/write_filt.h new file mode 100644 index 00000000000..bcab263f1dd --- /dev/null +++ b/extra/mariabackup/write_filt.h @@ -0,0 +1,58 @@ +/****************************************************** +XtraBackup: hot backup tool for InnoDB +(c) 2009-2013 Percona LLC and/or its affiliates. +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +/* Page write filter interface */ + +#ifndef XB_WRITE_FILT_H +#define XB_WRITE_FILT_H + +#include "fil_cur.h" +#include "datasink.h" + +/* Incremental page filter context */ +typedef struct { + byte *delta_buf_base; + byte *delta_buf; + ulint npages; +} xb_wf_incremental_ctxt_t; + +/* Page filter context used as an opaque structure by callers */ +typedef struct { + xb_fil_cur_t *cursor; + union { + xb_wf_incremental_ctxt_t wf_incremental_ctxt; + } u; +} xb_write_filt_ctxt_t; + + +typedef struct { + my_bool (*init)(xb_write_filt_ctxt_t *ctxt, char *dst_name, + xb_fil_cur_t *cursor); + my_bool (*process)(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile); + my_bool (*finalize)(xb_write_filt_ctxt_t *, ds_file_t *dstfile); + void (*deinit)(xb_write_filt_ctxt_t *); +} xb_write_filt_t; + +extern xb_write_filt_t wf_write_through; +extern xb_write_filt_t wf_incremental; + +#endif /* XB_WRITE_FILT_H */ diff --git a/extra/mariabackup/wsrep.cc b/extra/mariabackup/wsrep.cc new file mode 100644 index 00000000000..be11e058255 --- /dev/null +++ b/extra/mariabackup/wsrep.cc @@ -0,0 +1,220 @@ +/****************************************************** +Percona XtraBackup: hot backup tool for InnoDB +(c) 2009-2014 Percona LLC and/or its affiliates +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +******************************************************* + +This file incorporates work covered by the following copyright and +permission notice: + + Copyright 2010 Codership Oy <http://www.codership.com> + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#include <mysql_version.h> +#include <my_base.h> +#include <handler.h> +#include <trx0sys.h> + +#include "common.h" +#ifdef WITH_WSREP +#define WSREP_XID_PREFIX "WSREPXid" +#define WSREP_XID_PREFIX_LEN MYSQL_XID_PREFIX_LEN +#define WSREP_XID_UUID_OFFSET 8 +#define WSREP_XID_SEQNO_OFFSET (WSREP_XID_UUID_OFFSET + sizeof(wsrep_uuid_t)) +#define WSREP_XID_GTRID_LEN (WSREP_XID_SEQNO_OFFSET + sizeof(wsrep_seqno_t)) + +/*! undefined seqno */ +#define WSREP_SEQNO_UNDEFINED (-1) + +/*! Name of file where Galera info is stored on recovery */ +#define XB_GALERA_INFO_FILENAME "xtrabackup_galera_info" + +/* Galera UUID type - for all unique IDs */ +typedef struct wsrep_uuid { + unsigned char data[16]; +} wsrep_uuid_t; + +/* sequence number of a writeset, etc. */ +typedef long long wsrep_seqno_t; + +/* Undefined UUID */ +static const wsrep_uuid_t WSREP_UUID_UNDEFINED = {{0,}}; + +/***********************************************************************//** +Check if a given WSREP XID is valid. + +@return true if valid. +*/ +static +bool +wsrep_is_wsrep_xid( +/*===============*/ + const void* xid_ptr) +{ + const XID* xid = reinterpret_cast<const XID*>(xid_ptr); + + return((xid->formatID == 1 && + xid->gtrid_length == WSREP_XID_GTRID_LEN && + xid->bqual_length == 0 && + !memcmp(xid->data, WSREP_XID_PREFIX, WSREP_XID_PREFIX_LEN))); +} + +/***********************************************************************//** +Retrieve binary WSREP UUID from XID. + +@return binary WSREP UUID represenataion, if UUID is valid, or + WSREP_UUID_UNDEFINED otherwise. +*/ +static +const wsrep_uuid_t* +wsrep_xid_uuid( +/*===========*/ + const XID* xid) +{ + if (wsrep_is_wsrep_xid(xid)) { + return(reinterpret_cast<const wsrep_uuid_t*> + (xid->data + WSREP_XID_UUID_OFFSET)); + } else { + return(&WSREP_UUID_UNDEFINED); + } +} + +/***********************************************************************//** +Retrieve WSREP seqno from XID. + +@return WSREP seqno, if it is valid, or WSREP_SEQNO_UNDEFINED otherwise. +*/ +wsrep_seqno_t wsrep_xid_seqno( +/*==========================*/ + const XID* xid) +{ + if (wsrep_is_wsrep_xid(xid)) { + wsrep_seqno_t seqno; + memcpy(&seqno, xid->data + WSREP_XID_SEQNO_OFFSET, + sizeof(wsrep_seqno_t)); + + return(seqno); + } else { + return(WSREP_SEQNO_UNDEFINED); + } +} + +/***********************************************************************//** +Write UUID to string. + +@return length of UUID string representation or -EMSGSIZE if string is too +short. +*/ +static +int +wsrep_uuid_print( +/*=============*/ + const wsrep_uuid_t* uuid, + char* str, + size_t str_len) +{ + if (str_len > 36) { + const unsigned char* u = uuid->data; + return snprintf(str, str_len, + "%02x%02x%02x%02x-%02x%02x-%02x%02x-" + "%02x%02x-%02x%02x%02x%02x%02x%02x", + u[ 0], u[ 1], u[ 2], u[ 3], u[ 4], u[ 5], u[ 6], + u[ 7], u[ 8], u[ 9], u[10], u[11], u[12], u[13], + u[14], u[15]); + } + else { + return -EMSGSIZE; + } +} + +/*********************************************************************** +Store Galera checkpoint info in the 'xtrabackup_galera_info' file, if that +information is present in the trx system header. Otherwise, do nothing. */ +void +xb_write_galera_info(bool incremental_prepare) +/*==================*/ +{ + FILE* fp; + XID xid; + char uuid_str[40]; + wsrep_seqno_t seqno; + MY_STAT statinfo; + + /* Do not overwrite existing an existing file to be compatible with + servers with older server versions */ + if (!incremental_prepare && + my_stat(XB_GALERA_INFO_FILENAME, &statinfo, MYF(0)) != NULL) { + + return; + } + + memset(&xid, 0, sizeof(xid)); + xid.formatID = -1; + + if (!trx_sys_read_wsrep_checkpoint(&xid)) { + + return; + } + + if (wsrep_uuid_print(wsrep_xid_uuid(&xid), uuid_str, + sizeof(uuid_str)) < 0) { + return; + } + + fp = fopen(XB_GALERA_INFO_FILENAME, "w"); + if (fp == NULL) { + + msg("xtrabackup: error: " + "could not create " XB_GALERA_INFO_FILENAME + ", errno = %d\n", + errno); + exit(EXIT_FAILURE); + } + + seqno = wsrep_xid_seqno(&xid); + + msg("xtrabackup: Recovered WSREP position: %s:%lld\n", + uuid_str, (long long) seqno); + + if (fprintf(fp, "%s:%lld", uuid_str, (long long) seqno) < 0) { + + msg("xtrabackup: error: " + "could not write to " XB_GALERA_INFO_FILENAME + ", errno = %d\n", + errno); + exit(EXIT_FAILURE); + } + + fclose(fp); +} +#endif diff --git a/extra/mariabackup/wsrep.h b/extra/mariabackup/wsrep.h new file mode 100644 index 00000000000..7638d1f2b54 --- /dev/null +++ b/extra/mariabackup/wsrep.h @@ -0,0 +1,32 @@ +/****************************************************** +Percona XtraBackup: hot backup tool for InnoDB +(c) 2009-2014 Percona LLC and/or its affiliates +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA +*******************************************************/ + +#ifndef WSREP_H +#define WSREP_H + +/*********************************************************************** +Store Galera checkpoint info in the 'xtrabackup_galera_info' file, if that +information is present in the trx system header. Otherwise, do nothing. */ +void +xb_write_galera_info(bool incremental_prepare); +/*==================*/ + +#endif diff --git a/extra/mariabackup/xb0xb.h b/extra/mariabackup/xb0xb.h new file mode 100644 index 00000000000..659ab8ea5d0 --- /dev/null +++ b/extra/mariabackup/xb0xb.h @@ -0,0 +1,78 @@ +/****************************************************** +Copyright (c) 2012 Percona LLC and/or its affiliates. + +Declarations of XtraBackup functions called by InnoDB code. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#ifndef xb0xb_h +#define xb0xb_h + + +extern void os_io_init_simple(void); +extern os_file_t files[1000]; +extern const char *innodb_checksum_algorithm_names[]; +extern TYPELIB innodb_checksum_algorithm_typelib; +extern dberr_t open_or_create_data_files( + ibool* create_new_db, +#ifdef UNIV_LOG_ARCHIVE + lsn_t* min_arch_log_no, + lsn_t* max_arch_log_no, +#endif + lsn_t* min_flushed_lsn, + lsn_t* max_flushed_lsn, + ulint* sum_of_new_sizes) + ; +int +fil_file_readdir_next_file( +/*=======================*/ +dberr_t* err, /*!< out: this is set to DB_ERROR if an error + was encountered, otherwise not changed */ + const char* dirname,/*!< in: directory name or path */ + os_file_dir_t dir, /*!< in: directory stream */ + os_file_stat_t* info) /*!< in/out: buffer where the + info is returned */; +buf_block_t* btr_node_ptr_get_child( + const rec_t* node_ptr,/*!< in: node pointer */ + dict_index_t* index, /*!< in: index */ + const ulint* offsets,/*!< in: array returned by rec_get_offsets() */ + mtr_t* mtr) /*!< in: mtr */; + +buf_block_t* +btr_root_block_get( +/*===============*/ +const dict_index_t* index, /*!< in: index tree */ +ulint mode, /*!< in: either RW_S_LATCH + or RW_X_LATCH */ + mtr_t* mtr) /*!< in: mtr */; +fil_space_t* +fil_space_get_by_name(const char *); +ibool +recv_check_cp_is_consistent(const byte* buf); +void +innodb_log_checksum_func_update( +/*============================*/ +ulint algorithm) /*!< in: algorithm */; +dberr_t recv_find_max_checkpoint(log_group_t** max_group, ulint* max_field); +dberr_t +srv_undo_tablespaces_init( +/*======================*/ +ibool create_new_db, +ibool backup_mode, +const ulint n_conf_tablespaces, +ulint* n_opened); + +#endif diff --git a/extra/mariabackup/xb_regex.h b/extra/mariabackup/xb_regex.h new file mode 100644 index 00000000000..2e07e434e27 --- /dev/null +++ b/extra/mariabackup/xb_regex.h @@ -0,0 +1,48 @@ +/****************************************************** +Copyright (c) 2011-2013 Percona LLC and/or its affiliates. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +/* This file is required to abstract away regex(3) calls so that +my_regex is used on Windows and native calls are used on POSIX platforms. */ + +#ifndef XB_REGEX_H +#define XB_REGEX_H + +#ifdef HAVE_SYSTEM_REGEX +#include <regex.h> +#else +#include <pcreposix.h> +#endif + +typedef regex_t* xb_regex_t; + +#define xb_regex_init() + +#define xb_regexec(preg,string,nmatch,pmatch,eflags) \ + regexec(preg, string, nmatch, pmatch, eflags) + +#define xb_regerror(errcode,preg,errbuf,errbuf_size) \ + regerror(errcode, preg, errbuf, errbuf_size) + +#define xb_regcomp(preg,regex,cflags) \ + regcomp(preg, regex, cflags) + +#define xb_regfree(preg) regfree(preg) + +#define xb_regex_end() + +#endif /* XB_REGEX_H */ diff --git a/extra/mariabackup/xbcloud.cc b/extra/mariabackup/xbcloud.cc new file mode 100644 index 00000000000..56661b03dd0 --- /dev/null +++ b/extra/mariabackup/xbcloud.cc @@ -0,0 +1,2721 @@ +/****************************************************** +Copyright (c) 2014 Percona LLC and/or its affiliates. + +The xbstream utility: serialize/deserialize files in the XBSTREAM format. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#include <my_global.h> +#include <my_default.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <curl/curl.h> +#include <ev.h> +#include <unistd.h> +#include <errno.h> +#include <gcrypt.h> +#include <assert.h> +#include <my_sys.h> +#include <my_dir.h> +#include <my_getopt.h> +#include <algorithm> +#include <map> +#include <string> +#include <jsmn.h> +#include "xbstream.h" + +using std::min; +using std::max; +using std::map; +using std::string; + +#define XBCLOUD_VERSION "1.0" + +#define SWIFT_MAX_URL_SIZE 8192 +#define SWIFT_MAX_HDR_SIZE 8192 + +#define SWIFT_CHUNK_SIZE 11 * 1024 * 1024 + +#if ((LIBCURL_VERSION_MAJOR >= 7) && (LIBCURL_VERSION_MINOR >= 16)) +#define OLD_CURL_MULTI 0 +#else +#define OLD_CURL_MULTI 1 +#endif + +/*****************************************************************************/ + +typedef struct swift_auth_info_struct swift_auth_info; +typedef struct connection_info_struct connection_info; +typedef struct socket_info_struct socket_info; +typedef struct global_io_info_struct global_io_info; +typedef struct slo_chunk_struct slo_chunk; +typedef struct container_list_struct container_list; +typedef struct object_info_struct object_info; + +struct swift_auth_info_struct { + char url[SWIFT_MAX_URL_SIZE]; + char token[SWIFT_MAX_HDR_SIZE]; +}; + +struct global_io_info_struct { + struct ev_loop *loop; + struct ev_io input_event; + struct ev_timer timer_event; + CURLM *multi; + int still_running; + int eof; + curl_socket_t input_fd; + connection_info **connections; + long chunk_no; + connection_info *current_connection; + const char *url; + const char *container; + const char *token; + const char *backup_name; +}; + +struct socket_info_struct { + curl_socket_t sockfd; + CURL *easy; + int action; + long timeout; + struct ev_io ev; + int evset; + global_io_info *global; +}; + +struct connection_info_struct { + CURL *easy; + global_io_info *global; + char *buffer; + size_t buffer_size; + size_t filled_size; + size_t upload_size; + bool chunk_uploaded; + bool chunk_acked; + char error[CURL_ERROR_SIZE]; + struct curl_slist *slist; + char *name; + size_t name_len; + char hash[33]; + size_t chunk_no; + bool magic_verified; + size_t chunk_path_len; + xb_chunk_type_t chunk_type; + size_t payload_size; + size_t chunk_size; + int retry_count; + bool upload_started; + ulong global_idx; +}; + +struct slo_chunk_struct { + char name[SWIFT_MAX_URL_SIZE]; + char md5[33]; + int idx; + size_t size; +}; + +struct object_info_struct { + char hash[33]; + char name[SWIFT_MAX_URL_SIZE]; + size_t bytes; +}; + +struct container_list_struct { + size_t content_length; + size_t content_bufsize; + char *content_json; + size_t object_count; + size_t idx; + object_info *objects; + bool final; +}; + +enum {SWIFT, S3}; +const char *storage_names[] = +{ "SWIFT", "S3", NullS}; + +static my_bool opt_verbose = 0; +static ulong opt_storage = SWIFT; +static const char *opt_swift_user = NULL; +static const char *opt_swift_user_id = NULL; +static const char *opt_swift_password = NULL; +static const char *opt_swift_tenant = NULL; +static const char *opt_swift_tenant_id = NULL; +static const char *opt_swift_project = NULL; +static const char *opt_swift_project_id = NULL; +static const char *opt_swift_domain = NULL; +static const char *opt_swift_domain_id = NULL; +static const char *opt_swift_region = NULL; +static const char *opt_swift_container = NULL; +static const char *opt_swift_storage_url = NULL; +static const char *opt_swift_auth_url = NULL; +static const char *opt_swift_key = NULL; +static const char *opt_swift_auth_version = NULL; +static const char *opt_name = NULL; +static const char *opt_cacert = NULL; +static ulong opt_parallel = 1; +static my_bool opt_insecure = 0; +static enum {MODE_GET, MODE_PUT, MODE_DELETE} opt_mode; + +static char **file_list = NULL; +static int file_list_size = 0; + +TYPELIB storage_typelib = +{array_elements(storage_names)-1, "", storage_names, NULL}; + +enum { + OPT_STORAGE = 256, + OPT_SWIFT_CONTAINER, + OPT_SWIFT_AUTH_URL, + OPT_SWIFT_KEY, + OPT_SWIFT_USER, + OPT_SWIFT_USER_ID, + OPT_SWIFT_PASSWORD, + OPT_SWIFT_TENANT, + OPT_SWIFT_TENANT_ID, + OPT_SWIFT_PROJECT, + OPT_SWIFT_PROJECT_ID, + OPT_SWIFT_DOMAIN, + OPT_SWIFT_DOMAIN_ID, + OPT_SWIFT_REGION, + OPT_SWIFT_STORAGE_URL, + OPT_SWIFT_AUTH_VERSION, + OPT_PARALLEL, + OPT_CACERT, + OPT_INSECURE, + OPT_VERBOSE +}; + + +static struct my_option my_long_options[] = +{ + {"help", '?', "Display this help and exit.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"storage", OPT_STORAGE, "Specify storage type S3/SWIFT.", + &opt_storage, &opt_storage, &storage_typelib, + GET_ENUM, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"swift-auth-version", OPT_SWIFT_AUTH_VERSION, + "Swift authentication verison to use.", + &opt_swift_auth_version, &opt_swift_auth_version, 0, + GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-container", OPT_SWIFT_CONTAINER, + "Swift container to store backups into.", + &opt_swift_container, &opt_swift_container, 0, + GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-user", OPT_SWIFT_USER, + "Swift user name.", + &opt_swift_user, &opt_swift_user, 0, GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-user-id", OPT_SWIFT_USER_ID, + "Swift user ID.", + &opt_swift_user_id, &opt_swift_user_id, 0, GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-auth-url", OPT_SWIFT_AUTH_URL, + "Base URL of SWIFT authentication service.", + &opt_swift_auth_url, &opt_swift_auth_url, 0, + GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-storage-url", OPT_SWIFT_STORAGE_URL, + "URL of object-store endpoint. Usually received from authentication " + "service. Specify to override this value.", + &opt_swift_storage_url, &opt_swift_storage_url, 0, + GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-key", OPT_SWIFT_KEY, + "Swift key.", + &opt_swift_key, &opt_swift_key, 0, GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-tenant", OPT_SWIFT_TENANT, + "The tenant name. Both the --swift-tenant and --swift-tenant-id " + "options are optional, but should not be specified together.", + &opt_swift_tenant, &opt_swift_tenant, 0, GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-tenant-id", OPT_SWIFT_TENANT_ID, + "The tenant ID. Both the --swift-tenant and --swift-tenant-id " + "options are optional, but should not be specified together.", + &opt_swift_tenant_id, &opt_swift_tenant_id, 0, + GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-project", OPT_SWIFT_PROJECT, + "The project name.", + &opt_swift_project, &opt_swift_project, 0, GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-project-id", OPT_SWIFT_PROJECT_ID, + "The project ID.", + &opt_swift_project_id, &opt_swift_project_id, 0, + GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-domain", OPT_SWIFT_DOMAIN, + "The domain name.", + &opt_swift_domain, &opt_swift_domain, 0, GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-domain-id", OPT_SWIFT_DOMAIN_ID, + "The domain ID.", + &opt_swift_domain_id, &opt_swift_domain_id, 0, + GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-password", OPT_SWIFT_PASSWORD, + "The password of the user.", + &opt_swift_password, &opt_swift_password, 0, + GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"swift-region", OPT_SWIFT_REGION, + "The region object-store endpoint.", + &opt_swift_region, &opt_swift_region, 0, + GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"parallel", OPT_PARALLEL, + "Number of parallel chunk uploads.", + &opt_parallel, &opt_parallel, 0, GET_ULONG, REQUIRED_ARG, + 1, 0, 0, 0, 0, 0}, + + {"cacert", OPT_CACERT, + "CA certificate file.", + &opt_cacert, &opt_cacert, 0, GET_STR_ALLOC, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"insecure", OPT_INSECURE, + "Do not verify server SSL certificate.", + &opt_insecure, &opt_insecure, 0, GET_BOOL, NO_ARG, + 0, 0, 0, 0, 0, 0}, + + {"verbose", OPT_VERBOSE, + "Turn ON cURL tracing.", + &opt_verbose, &opt_verbose, 0, GET_BOOL, NO_ARG, + 0, 0, 0, 0, 0, 0}, + + {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} +}; + +/* The values of these arguments should be masked + on the command line */ +static const char * const masked_args[] = { + "--swift-password", + "--swift-key", + "--swift-auth-url", + "--swift-storage-url", + "--swift-container", + "--swift-user", + "--swift-tenant", + "--swift-user-id", + "--swift-tenant-id", + 0 +}; + +static map<string, ulonglong> file_chunk_count; + +static +void +print_version() +{ + printf("%s Ver %s for %s (%s)\n", my_progname, XBCLOUD_VERSION, + SYSTEM_TYPE, MACHINE_TYPE); +} + +static +void +usage() +{ + print_version(); + puts("Copyright (C) 2015 Percona LLC and/or its affiliates."); + puts("This software comes with ABSOLUTELY NO WARRANTY. " + "This is free software,\nand you are welcome to modify and " + "redistribute it under the GPL license.\n"); + + puts("Manage backups on Cloud services.\n"); + + puts("Usage: "); + printf(" %s -c put [OPTIONS...] <NAME> upload backup from STDIN into " + "the cloud service with given name.\n", my_progname); + printf(" %s -c get [OPTIONS...] <NAME> [FILES...] stream specified " + "backup or individual files from cloud service into STDOUT.\n", + my_progname); + + puts("\nOptions:"); + my_print_help(my_long_options); +} + +static +my_bool +get_one_option(int optid, const struct my_option *opt __attribute__((unused)), + char *argument __attribute__((unused))) +{ + switch (optid) { + case '?': + usage(); + exit(0); + } + + return(FALSE); +} + +static const char *load_default_groups[]= + { "xbcloud", 0 }; + +/*********************************************************************//** +mask sensitive values on the command line */ +static +void +mask_args(int argc, char **argv) +{ + int i; + for (i = 0; i < argc-1; i++) { + int j = 0; + if (argv[i]) while (masked_args[j]) { + char *p; + if ((p = strstr(argv[i], masked_args[j]))) { + p += strlen(masked_args[j]); + while (*p && *p != '=') { + p++; + } + if (*p == '=') { + p++; + while (*p) { + *p++ = 'x'; + } + } + } + j++; + } + } +} + +static +int parse_args(int argc, char **argv) +{ + const char *command; + + if (argc < 2) { + fprintf(stderr, "Command isn't specified. " + "Supported commands are put and get\n"); + usage(); + exit(EXIT_FAILURE); + } + + command = argv[1]; + argc--; argv++; + + if (strcasecmp(command, "put") == 0) { + opt_mode = MODE_PUT; + } else if (strcasecmp(command, "get") == 0) { + opt_mode = MODE_GET; + } else if (strcasecmp(command, "delete") == 0) { + opt_mode = MODE_DELETE; + } else { + fprintf(stderr, "Unknown command %s. " + "Supported commands are put and get\n", command); + usage(); + exit(EXIT_FAILURE); + } + + if (load_defaults("my", load_default_groups, &argc, &argv)) { + exit(EXIT_FAILURE); + } + + if (handle_options(&argc, &argv, my_long_options, get_one_option)) { + exit(EXIT_FAILURE); + } + + /* make sure name is specified */ + if (argc < 1) { + fprintf(stderr, "Backup name is required argument\n"); + exit(EXIT_FAILURE); + } + opt_name = argv[0]; + argc--; argv++; + + /* validate arguments */ + if (opt_storage == SWIFT) { + if (opt_swift_user == NULL) { + fprintf(stderr, "Swift user is not specified\n"); + exit(EXIT_FAILURE); + } + if (opt_swift_container == NULL) { + fprintf(stderr, + "Swift container is not specified\n"); + exit(EXIT_FAILURE); + } + if (opt_swift_auth_url == NULL) { + fprintf(stderr, "Swift auth URL is not specified\n"); + exit(EXIT_FAILURE); + } + } else { + fprintf(stderr, "Swift is only supported storage API\n"); + } + + if (argc > 0) { + file_list = argv; + file_list_size = argc; + } + + return(0); +} + +static char *hex_md5(const unsigned char *hash, char *out) +{ + enum { hash_len = 16 }; + char *p; + int i; + + for (i = 0, p = out; i < hash_len; i++, p+=2) { + sprintf(p, "%02x", hash[i]); + } + + return out; +} + +/* If header starts with prefix it's value will be copied into output buffer */ +static +int get_http_header(const char *prefix, const char *buffer, + char *out, size_t out_size) +{ + const char *beg, *end; + size_t len, prefix_len; + + prefix_len = strlen(prefix); + + if (strncasecmp(buffer, prefix, prefix_len) == 0) { + beg = buffer + prefix_len; + end = strchr(beg, '\r'); + + len = min<size_t>(end - beg, out_size - 1); + + strncpy(out, beg, len); + + out[len] = 0; + + return 1; + } + + return 0; +} + +static +size_t swift_auth_header_read_cb(char *ptr, size_t size, size_t nmemb, + void *data) +{ + swift_auth_info *info = (swift_auth_info*)(data); + + get_http_header("X-Storage-Url: ", ptr, + info->url, array_elements(info->url)); + get_http_header("X-Auth-Token: ", ptr, + info->token, array_elements(info->token)); + + return nmemb * size; +} + +/*********************************************************************//** +Authenticate against Swift TempAuth. Fills swift_auth_info struct. +Uses creadentials privided as global variables. +@returns true if access is granted and token received. */ +static +bool +swift_temp_auth(const char *auth_url, swift_auth_info *info) +{ + CURL *curl; + CURLcode res; + long http_code; + char *hdr_buf = NULL; + struct curl_slist *slist = NULL; + + if (opt_swift_user == NULL) { + fprintf(stderr, "Swift user must be specified for TempAuth.\n"); + return(false); + } + + if (opt_swift_key == NULL) { + fprintf(stderr, "Swift key must be specified for TempAuth.\n"); + return(false); + } + + curl = curl_easy_init(); + + if (curl != NULL) { + + hdr_buf = (char *)(calloc(14 + max(strlen(opt_swift_user), + strlen(opt_swift_key)), 1)); + + if (!hdr_buf) { + res = CURLE_FAILED_INIT; + goto cleanup; + } + + sprintf(hdr_buf, "X-Auth-User: %s", opt_swift_user); + slist = curl_slist_append(slist, hdr_buf); + + sprintf(hdr_buf, "X-Auth-Key: %s", opt_swift_key); + slist = curl_slist_append(slist, hdr_buf); + + curl_easy_setopt(curl, CURLOPT_VERBOSE, opt_verbose); + curl_easy_setopt(curl, CURLOPT_URL, auth_url); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, + swift_auth_header_read_cb); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, info); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist); + if (opt_cacert != NULL) + curl_easy_setopt(curl, CURLOPT_CAINFO, opt_cacert); + if (opt_insecure) + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, FALSE); + + res = curl_easy_perform(curl); + + if (res != CURLE_OK) { + fprintf(stderr, "error: authentication failed: " + "curl_easy_perform(): %s\n", + curl_easy_strerror(res)); + goto cleanup; + } + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); + if (http_code != 200 && + http_code != 204) { + fprintf(stderr, "error: authentication failed " + "with response code: %ld\n", http_code); + res = CURLE_LOGIN_DENIED; + goto cleanup; + } + } else { + res = CURLE_FAILED_INIT; + fprintf(stderr, "error: curl_easy_init() failed\n"); + goto cleanup; + } + +cleanup: + if (hdr_buf) { + free(hdr_buf); + } + if (slist) { + curl_slist_free_all(slist); + } + if (curl) { + curl_easy_cleanup(curl); + } + + if (res == CURLE_OK) { + /* check that we received token and storage URL */ + if (*info->url == 0) { + fprintf(stderr, "error: malformed response: " + "X-Storage-Url is missing\n"); + return(false); + } + if (*info->token == 0) { + fprintf(stderr, "error: malformed response: " + "X-Auth-Token is missing\n"); + return(false); + } + return(true); + } + + return(false); +} + +static +size_t +write_null_cb(char *buffer, size_t size, size_t nmemb, void *stream) +{ + return fwrite(buffer, size, nmemb, stderr); +} + + +static +size_t +read_null_cb(char *ptr, size_t size, size_t nmemb, void *data) +{ + return 0; +} + + +static +int +swift_create_container(swift_auth_info *info, const char *name) +{ + char url[SWIFT_MAX_URL_SIZE]; + char auth_token[SWIFT_MAX_HDR_SIZE]; + CURLcode res; + long http_code; + CURL *curl; + struct curl_slist *slist = NULL; + + snprintf(url, array_elements(url), "%s/%s", info->url, name); + snprintf(auth_token, array_elements(auth_token), "X-Auth-Token: %s", + info->token); + + curl = curl_easy_init(); + + if (curl != NULL) { + slist = curl_slist_append(slist, auth_token); + slist = curl_slist_append(slist, "Content-Length: 0"); + + curl_easy_setopt(curl, CURLOPT_VERBOSE, opt_verbose); + curl_easy_setopt(curl, CURLOPT_URL, url); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_null_cb); + curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_null_cb); + curl_easy_setopt(curl, CURLOPT_INFILESIZE, 0L); + curl_easy_setopt(curl, CURLOPT_PUT, 1L); + if (opt_cacert != NULL) + curl_easy_setopt(curl, CURLOPT_CAINFO, opt_cacert); + if (opt_insecure) + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, FALSE); + + res = curl_easy_perform(curl); + + if (res != CURLE_OK) { + fprintf(stderr, + "error: curl_easy_perform() failed: %s\n", + curl_easy_strerror(res)); + goto cleanup; + } + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); + if (http_code != 201 && /* created */ + http_code != 202 /* accepted (already exists) */) { + fprintf(stderr, "error: request failed " + "with response code: %ld\n", http_code); + res = CURLE_LOGIN_DENIED; + goto cleanup; + } + } else { + res = CURLE_FAILED_INIT; + fprintf(stderr, "error: curl_easy_init() failed\n"); + goto cleanup; + } + +cleanup: + if (slist) { + curl_slist_free_all(slist); + } + if (curl) { + curl_easy_cleanup(curl); + } + + return res; +} + + +/*********************************************************************//** +Delete object with given url. +@returns true if object deleted successfully. */ +static +bool +swift_delete_object(swift_auth_info *info, const char *url) +{ + char auth_token[SWIFT_MAX_HDR_SIZE]; + CURLcode res; + long http_code; + CURL *curl; + struct curl_slist *slist = NULL; + bool ret = false; + + snprintf(auth_token, array_elements(auth_token), "X-Auth-Token: %s", + info->token); + + curl = curl_easy_init(); + + if (curl != NULL) { + slist = curl_slist_append(slist, auth_token); + + curl_easy_setopt(curl, CURLOPT_VERBOSE, opt_verbose); + curl_easy_setopt(curl, CURLOPT_URL, url); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist); + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE"); + if (opt_cacert != NULL) + curl_easy_setopt(curl, CURLOPT_CAINFO, opt_cacert); + if (opt_insecure) + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, FALSE); + + res = curl_easy_perform(curl); + + if (res != CURLE_OK) { + fprintf(stderr, + "error: curl_easy_perform() failed: %s\n", + curl_easy_strerror(res)); + goto cleanup; + } + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); + if (http_code != 200 && /* OK */ + http_code != 204 /* no content */) { + fprintf(stderr, "error: request failed " + "with response code: %ld\n", http_code); + goto cleanup; + } + ret = true; + } else { + fprintf(stderr, "error: curl_easy_init() failed\n"); + goto cleanup; + } + +cleanup: + if (slist) { + curl_slist_free_all(slist); + } + if (curl) { + curl_easy_cleanup(curl); + } + + return ret; +} + +static int conn_upload_init(connection_info *conn); +static void conn_buffer_updated(connection_info *conn); +static connection_info *conn_new(global_io_info *global, ulong global_idx); +static void conn_cleanup(connection_info *conn); +static void conn_upload_retry(connection_info *conn); + +/* Check for completed transfers, and remove their easy handles */ +static void check_multi_info(global_io_info *g) +{ + char *eff_url; + CURLMsg *msg; + int msgs_left; + connection_info *conn; + CURL *easy; + + while ((msg = curl_multi_info_read(g->multi, &msgs_left))) { + if (msg->msg == CURLMSG_DONE) { + easy = msg->easy_handle; + curl_easy_getinfo(easy, CURLINFO_PRIVATE, &conn); + curl_easy_getinfo(easy, CURLINFO_EFFECTIVE_URL, + &eff_url); + curl_multi_remove_handle(g->multi, easy); + curl_easy_cleanup(easy); + conn->easy = NULL; + if (conn->chunk_acked) { + conn->chunk_uploaded = true; + fprintf(stderr, "%s is done\n", conn->hash); + } else { + fprintf(stderr, "error: chunk %zu '%s' %s " + "is not uploaded, but socket closed " + "(%zu bytes of %zu left to upload)\n", + conn->chunk_no, + conn->name, + conn->hash, + conn->chunk_size - conn->upload_size, + conn->chunk_size); + conn_upload_retry(conn); + } + } + } +} + +/* Die if we get a bad CURLMcode somewhere */ +static void mcode_or_die(const char *where, CURLMcode code) +{ + if (code != CURLM_OK) + { + const char *s; + switch (code) + { + case CURLM_BAD_HANDLE: + s = "CURLM_BAD_HANDLE"; + break; + case CURLM_BAD_EASY_HANDLE: + s = "CURLM_BAD_EASY_HANDLE"; + break; + case CURLM_OUT_OF_MEMORY: + s = "CURLM_OUT_OF_MEMORY"; + break; + case CURLM_INTERNAL_ERROR: + s = "CURLM_INTERNAL_ERROR"; + break; + case CURLM_UNKNOWN_OPTION: + s = "CURLM_UNKNOWN_OPTION"; + break; + case CURLM_LAST: + s = "CURLM_LAST"; + break; + default: + s = "CURLM_unknown"; + break; + case CURLM_BAD_SOCKET: + s = "CURLM_BAD_SOCKET"; + fprintf(stderr, "error: %s returns (%d) %s\n", + where, code, s); + /* ignore this error */ + return; + } + fprintf(stderr, "error: %s returns (%d) %s\n", + where, code, s); + assert(0); + } +} + +/* Called by libev when we get action on a multi socket */ +static void event_cb(EV_P_ struct ev_io *w, int revents) +{ + global_io_info *global = (global_io_info*)(w->data); + CURLMcode rc; + +#if !(OLD_CURL_MULTI) + int action = (revents & EV_READ ? CURL_POLL_IN : 0) | + (revents & EV_WRITE ? CURL_POLL_OUT : 0); + + do { + rc = curl_multi_socket_action(global->multi, w->fd, action, + &global->still_running); + } while (rc == CURLM_CALL_MULTI_PERFORM); +#else + do { + rc = curl_multi_socket(global->multi, w->fd, + &global->still_running); + } while (rc == CURLM_CALL_MULTI_PERFORM); +#endif + mcode_or_die("error: event_cb: curl_multi_socket_action", rc); + check_multi_info(global); + if (global->still_running <= 0) { + ev_timer_stop(global->loop, &global->timer_event); + } +} + +static void remsock(curl_socket_t s, socket_info *fdp, global_io_info *global) +{ + if (fdp) { + if (fdp->evset) { + ev_io_stop(global->loop, &fdp->ev); + } + free(fdp); + } +} + +static void setsock(socket_info *fdp, curl_socket_t s, CURL *easy, int action, + global_io_info *global) +{ + int kind = (action & CURL_POLL_IN ? (int)(EV_READ) : 0) | + (action & CURL_POLL_OUT ? (int)(EV_WRITE) : 0); + + fdp->sockfd = s; + fdp->action = action; + fdp->easy = easy; + if (fdp->evset) + ev_io_stop(global->loop, &fdp->ev); + ev_io_init(&fdp->ev, event_cb, fdp->sockfd, kind); + fdp->ev.data = global; + fdp->evset = 1; + ev_io_start(global->loop, &fdp->ev); +} + +static void addsock(curl_socket_t s, CURL *easy, int action, + global_io_info *global) +{ + socket_info *fdp = (socket_info *)(calloc(sizeof(socket_info), 1)); + + fdp->global = global; + setsock(fdp, s, easy, action, global); + curl_multi_assign(global->multi, s, fdp); +} + +static int sock_cb(CURL *easy, curl_socket_t s, int what, void *cbp, + void *sockp) +{ + global_io_info *global = (global_io_info*)(cbp); + socket_info *fdp = (socket_info*)(sockp); + + if (what == CURL_POLL_REMOVE) { + remsock(s, fdp, global); + } else { + if (!fdp) { + addsock(s, easy, what, global); + } else { + setsock(fdp, s, easy, what, global); + } + } + return 0; +} + +/* Called by libev when our timeout expires */ +static void timer_cb(EV_P_ struct ev_timer *w, int revents) +{ + global_io_info *io_global = (global_io_info*)(w->data); + CURLMcode rc; + +#if !(OLD_CURL_MULTI) + do { + rc = curl_multi_socket_action(io_global->multi, + CURL_SOCKET_TIMEOUT, 0, + &io_global->still_running); + } while (rc == CURLM_CALL_MULTI_PERFORM); +#else + do { + rc = curl_multi_socket_all(io_global->multi, + &io_global->still_running); + } while (rc == CURLM_CALL_MULTI_PERFORM); +#endif + mcode_or_die("timer_cb: curl_multi_socket_action", rc); + check_multi_info(io_global); +} + +static connection_info *get_current_connection(global_io_info *global) +{ + connection_info *conn = global->current_connection; + ulong i; + + if (conn && conn->filled_size < conn->chunk_size) + return conn; + + for (i = 0; i < opt_parallel; i++) { + conn = global->connections[i]; + if (conn->chunk_uploaded || conn->filled_size == 0) { + global->current_connection = conn; + conn_upload_init(conn); + return conn; + } + } + + return NULL; +} + +/* This gets called whenever data is received from the input */ +static void input_cb(EV_P_ struct ev_io *w, int revents) +{ + global_io_info *io_global = (global_io_info *)(w->data); + connection_info *conn = get_current_connection(io_global); + + if (conn == NULL) + return; + + if (conn->filled_size < conn->chunk_size) { + if (revents & EV_READ) { + ssize_t nbytes = read(io_global->input_fd, + conn->buffer + conn->filled_size, + conn->chunk_size - + conn->filled_size); + if (nbytes > 0) { + conn->filled_size += nbytes; + conn_buffer_updated(conn); + } else if (nbytes < 0) { + if (errno != EAGAIN && errno != EINTR) { + char error[200]; + my_strerror(error, sizeof(error), + errno); + fprintf(stderr, "error: failed to read " + "input stream (%s)\n", error); + /* failed to read input */ + exit(1); + } + } else { + io_global->eof = 1; + ev_io_stop(io_global->loop, w); + } + } + } + + assert(conn->filled_size <= conn->chunk_size); +} + +static int swift_upload_read_cb(char *ptr, size_t size, size_t nmemb, + void *data) +{ + size_t realsize; + + connection_info *conn = (connection_info*)(data); + + if (conn->filled_size == conn->upload_size && + conn->upload_size < conn->chunk_size && !conn->global->eof) { + ssize_t nbytes; + assert(conn->global->current_connection == conn); + do { + nbytes = read(conn->global->input_fd, + conn->buffer + conn->filled_size, + conn->chunk_size - conn->filled_size); + } while (nbytes == -1 && errno == EAGAIN); + if (nbytes > 0) { + conn->filled_size += nbytes; + conn_buffer_updated(conn); + } else { + conn->global->eof = 1; + } + } + + realsize = min(size * nmemb, conn->filled_size - conn->upload_size); + + memcpy(ptr, conn->buffer + conn->upload_size, realsize); + conn->upload_size += realsize; + + assert(conn->filled_size <= conn->chunk_size); + assert(conn->upload_size <= conn->filled_size); + + return realsize; +} + +static +size_t upload_header_read_cb(char *ptr, size_t size, size_t nmemb, + void *data) +{ + connection_info *conn = (connection_info *)(data); + char etag[33]; + + if (get_http_header("Etag: ", ptr, etag, array_elements(etag))) { + if (strcmp(conn->hash, etag) != 0) { + fprintf(stderr, "error: ETag mismatch\n"); + exit(EXIT_FAILURE); + } + fprintf(stderr, "acked chunk %s\n", etag); + conn->chunk_acked = true; + } + + return nmemb * size; +} + +static int conn_upload_init(connection_info *conn) +{ + conn->filled_size = 0; + conn->upload_size = 0; + conn->chunk_uploaded = false; + conn->chunk_acked = false; + conn->chunk_size = CHUNK_HEADER_CONSTANT_LEN; + conn->magic_verified = false; + conn->chunk_path_len = 0; + conn->chunk_type = XB_CHUNK_TYPE_UNKNOWN; + conn->payload_size = 0; + conn->upload_started = false; + conn->retry_count = 0; + if (conn->name != NULL) { + conn->name[0] = 0; + } + + if (conn->easy != NULL) { + conn->easy = 0; + } + + if (conn->slist != NULL) { + curl_slist_free_all(conn->slist); + conn->slist = NULL; + } + + return 0; +} + +static void conn_upload_prepare(connection_info *conn) +{ + gcry_md_hd_t md5; + + gcry_md_open(&md5, GCRY_MD_MD5, 0); + gcry_md_write(md5, conn->buffer, conn->chunk_size); + hex_md5(gcry_md_read(md5, GCRY_MD_MD5), conn->hash); + gcry_md_close(md5); +} + +static int conn_upload_start(connection_info *conn) +{ + char token_header[SWIFT_MAX_HDR_SIZE]; + char object_url[SWIFT_MAX_URL_SIZE]; + char content_len[200], etag[200]; + global_io_info *global; + CURLMcode rc; + + global = conn->global; + + fprintf(stderr, "uploading chunk %s/%s/%s.%020zu " + "(md5: %s, size: %zu)\n", + global->container, global->backup_name, conn->name, + conn->chunk_no, conn->hash, conn->chunk_size); + + snprintf(object_url, array_elements(object_url), "%s/%s/%s/%s.%020zu", + global->url, global->container, global->backup_name, + conn->name, conn->chunk_no); + + snprintf(content_len, sizeof(content_len), "Content-Length: %lu", + (ulong)(conn->chunk_size)); + + snprintf(etag, sizeof(etag), "ETag: %s", conn->hash); + + snprintf(token_header, array_elements(token_header), + "X-Auth-Token: %s", global->token); + + conn->slist = curl_slist_append(conn->slist, token_header); + conn->slist = curl_slist_append(conn->slist, + "Connection: keep-alive"); + conn->slist = curl_slist_append(conn->slist, + "Content-Type: " + "application/octet-stream"); + conn->slist = curl_slist_append(conn->slist, content_len); + conn->slist = curl_slist_append(conn->slist, etag); + + conn->easy = curl_easy_init(); + if (!conn->easy) { + fprintf(stderr, "error: curl_easy_init() failed\n"); + return 1; + } + curl_easy_setopt(conn->easy, CURLOPT_URL, object_url); + curl_easy_setopt(conn->easy, CURLOPT_READFUNCTION, + swift_upload_read_cb); + curl_easy_setopt(conn->easy, CURLOPT_READDATA, conn); + curl_easy_setopt(conn->easy, CURLOPT_VERBOSE, opt_verbose); + curl_easy_setopt(conn->easy, CURLOPT_ERRORBUFFER, conn->error); + curl_easy_setopt(conn->easy, CURLOPT_PRIVATE, conn); + curl_easy_setopt(conn->easy, CURLOPT_NOPROGRESS, 1L); + curl_easy_setopt(conn->easy, CURLOPT_LOW_SPEED_TIME, 5L); + curl_easy_setopt(conn->easy, CURLOPT_LOW_SPEED_LIMIT, 1024L); + curl_easy_setopt(conn->easy, CURLOPT_PUT, 1L); + curl_easy_setopt(conn->easy, CURLOPT_HTTPHEADER, conn->slist); + curl_easy_setopt(conn->easy, CURLOPT_HEADERFUNCTION, + upload_header_read_cb); + curl_easy_setopt(conn->easy, CURLOPT_HEADERDATA, conn); + curl_easy_setopt(conn->easy, CURLOPT_INFILESIZE, + (long) conn->chunk_size); + if (opt_cacert != NULL) + curl_easy_setopt(conn->easy, CURLOPT_CAINFO, opt_cacert); + if (opt_insecure) + curl_easy_setopt(conn->easy, CURLOPT_SSL_VERIFYPEER, FALSE); + + rc = curl_multi_add_handle(conn->global->multi, conn->easy); + mcode_or_die("conn_upload_init: curl_multi_add_handle", rc); + +#if (OLD_CURL_MULTI) + do { + rc = curl_multi_socket_all(global->multi, + &global->still_running); + } while(rc == CURLM_CALL_MULTI_PERFORM); +#endif + + conn->upload_started = true; + + return 0; +} + +static void conn_cleanup(connection_info *conn) +{ + if (conn) { + free(conn->name); + free(conn->buffer); + if (conn->slist) { + curl_slist_free_all(conn->slist); + conn->slist = NULL; + } + if (conn->easy) { + curl_easy_cleanup(conn->easy); + conn->easy = NULL; + } + } + free(conn); +} + +static void conn_upload_retry(connection_info *conn) +{ + /* already closed by cURL */ + conn->easy = NULL; + + if (conn->slist != NULL) { + curl_slist_free_all(conn->slist); + conn->slist = NULL; + } + + if (conn->retry_count++ > 3) { + fprintf(stderr, "error: retry count limit reached\n"); + exit(EXIT_FAILURE); + } + + fprintf(stderr, "warning: retrying to upload chunk %zu of '%s'\n", + conn->chunk_no, conn->name); + + conn->upload_size = 0; + + conn_upload_start(conn); +} + +static connection_info *conn_new(global_io_info *global, ulong global_idx) +{ + connection_info *conn; + + conn = (connection_info *)(calloc(1, sizeof(connection_info))); + if (conn == NULL) { + goto error; + } + + conn->global = global; + conn->global_idx = global_idx; + conn->buffer_size = SWIFT_CHUNK_SIZE; + if ((conn->buffer = (char *)(calloc(conn->buffer_size, 1))) == + NULL) { + goto error; + } + + return conn; + +error: + if (conn != NULL) { + conn_cleanup(conn); + } + + fprintf(stderr, "error: out of memory\n"); + exit(EXIT_FAILURE); + + return NULL; +} + +/*********************************************************************//** +Handle input buffer updates. Parse chunk header and set appropriate +buffer size. */ +static +void +conn_buffer_updated(connection_info *conn) +{ + bool ready_for_upload = false; + + /* chunk header */ + if (!conn->magic_verified && + conn->filled_size >= CHUNK_HEADER_CONSTANT_LEN) { + if (strncmp(XB_STREAM_CHUNK_MAGIC, conn->buffer, + sizeof(XB_STREAM_CHUNK_MAGIC) - 1) != 0) { + + fprintf(stderr, "Error: magic expected\n"); + exit(EXIT_FAILURE); + } + conn->magic_verified = true; + conn->chunk_path_len = uint4korr(conn->buffer + + PATH_LENGTH_OFFSET); + conn->chunk_type = (xb_chunk_type_t) + (conn->buffer[CHUNK_TYPE_OFFSET]); + conn->chunk_size = CHUNK_HEADER_CONSTANT_LEN + + conn->chunk_path_len; + if (conn->chunk_type != XB_CHUNK_TYPE_EOF) { + conn->chunk_size += 16; + } + } + + /* ordinary chunk */ + if (conn->magic_verified && + conn->payload_size == 0 && + conn->chunk_type != XB_CHUNK_TYPE_EOF && + conn->filled_size >= CHUNK_HEADER_CONSTANT_LEN + + conn->chunk_path_len + 16) { + + conn->payload_size = uint8korr(conn->buffer + + CHUNK_HEADER_CONSTANT_LEN + + conn->chunk_path_len); + + conn->chunk_size = conn->payload_size + 4 + 16 + + conn->chunk_path_len + + CHUNK_HEADER_CONSTANT_LEN; + + if (conn->name == NULL) { + conn->name = (char*)(malloc(conn->chunk_path_len + 1)); + } else if (conn->name_len < conn->chunk_path_len + 1) { + conn->name = (char*)(realloc(conn->name, + conn->chunk_path_len + 1)); + } + conn->name_len = conn->chunk_path_len + 1; + + memcpy(conn->name, conn->buffer + CHUNK_HEADER_CONSTANT_LEN, + conn->chunk_path_len); + conn->name[conn->chunk_path_len] = 0; + + if (conn->buffer_size < conn->chunk_size) { + conn->buffer = + (char *)(realloc(conn->buffer, conn->chunk_size)); + conn->buffer_size = conn->chunk_size; + } + } + + /* EOF chunk has no payload */ + if (conn->magic_verified && + conn->chunk_type == XB_CHUNK_TYPE_EOF && + conn->filled_size >= CHUNK_HEADER_CONSTANT_LEN + + conn->chunk_path_len) { + + if (conn->name == NULL) { + conn->name = (char*)(malloc(conn->chunk_path_len + 1)); + } else if (conn->name_len < conn->chunk_path_len + 1) { + conn->name = (char*)(realloc(conn->name, + conn->chunk_path_len + 1)); + } + conn->name_len = conn->chunk_path_len + 1; + + memcpy(conn->name, conn->buffer + CHUNK_HEADER_CONSTANT_LEN, + conn->chunk_path_len); + conn->name[conn->chunk_path_len] = 0; + } + + if (conn->filled_size > 0 && conn->filled_size == conn->chunk_size) { + ready_for_upload = true; + } + + /* start upload once recieved the size of the chunk */ + if (!conn->upload_started && ready_for_upload) { + conn->chunk_no = file_chunk_count[conn->name]++; + conn_upload_prepare(conn); + conn_upload_start(conn); + } +} + +static int init_input(global_io_info *io_global) +{ + ev_io_init(&io_global->input_event, input_cb, STDIN_FILENO, EV_READ); + io_global->input_event.data = io_global; + ev_io_start(io_global->loop, &io_global->input_event); + + return 0; +} + +/* Update the event timer after curl_multi library calls */ +static int multi_timer_cb(CURLM *multi, long timeout_ms, global_io_info *global) +{ + ev_timer_stop(global->loop, &global->timer_event); + if (timeout_ms > 0) { + double t = timeout_ms / 1000.0; + ev_timer_init(&global->timer_event, timer_cb, t, 0.); + ev_timer_start(global->loop, &global->timer_event); + } else { + timer_cb(global->loop, &global->timer_event, 0); + } + return 0; +} + +static +int swift_upload_parts(swift_auth_info *auth, const char *container, + const char *name) +{ + global_io_info io_global; + ulong i; +#if (OLD_CURL_MULTI) + long timeout; +#endif + CURLMcode rc; + int n_dirty_buffers; + + memset(&io_global, 0, sizeof(io_global)); + + io_global.loop = ev_default_loop(0); + init_input(&io_global); + io_global.multi = curl_multi_init(); + ev_timer_init(&io_global.timer_event, timer_cb, 0., 0.); + io_global.timer_event.data = &io_global; + io_global.connections = (connection_info **) + (calloc(opt_parallel, sizeof(connection_info))); + io_global.url = auth->url; + io_global.container = container; + io_global.backup_name = name; + io_global.token = auth->token; + for (i = 0; i < opt_parallel; i++) { + io_global.connections[i] = conn_new(&io_global, i); + } + + /* setup the generic multi interface options we want */ + curl_multi_setopt(io_global.multi, CURLMOPT_SOCKETFUNCTION, sock_cb); + curl_multi_setopt(io_global.multi, CURLMOPT_SOCKETDATA, &io_global); +#if !(OLD_CURL_MULTI) + curl_multi_setopt(io_global.multi, CURLMOPT_TIMERFUNCTION, multi_timer_cb); + curl_multi_setopt(io_global.multi, CURLMOPT_TIMERDATA, &io_global); + do { + rc = curl_multi_socket_action(io_global.multi, + CURL_SOCKET_TIMEOUT, 0, + &io_global.still_running); + } while (rc == CURLM_CALL_MULTI_PERFORM); +#else + curl_multi_timeout(io_global.multi, &timeout); + if (timeout >= 0) { + multi_timer_cb(io_global.multi, timeout, &io_global); + } + do { + rc = curl_multi_socket_all(io_global.multi, &io_global.still_running); + } while(rc == CURLM_CALL_MULTI_PERFORM); +#endif + + ev_loop(io_global.loop, 0); + check_multi_info(&io_global); + curl_multi_cleanup(io_global.multi); + + n_dirty_buffers = 0; + for (i = 0; i < opt_parallel; i++) { + connection_info *conn = io_global.connections[i]; + if (conn && conn->upload_size != conn->filled_size) { + fprintf(stderr, "error: upload failed: %lu bytes left " + "in the buffer %s (uploaded = %d)\n", + (ulong)(conn->filled_size - conn->upload_size), + conn->name, conn->chunk_uploaded); + ++n_dirty_buffers; + } + } + + for (i = 0; i < opt_parallel; i++) { + if (io_global.connections[i] != NULL) { + conn_cleanup(io_global.connections[i]); + } + } + free(io_global.connections); + + if (n_dirty_buffers > 0) { + return(EXIT_FAILURE); + } + + return 0; +} + +struct download_buffer_info { + off_t offset; + size_t size; + size_t result_len; + char *buf; + curl_read_callback custom_header_callback; + void *custom_header_callback_data; +}; + +/*********************************************************************//** +Callback to parse header of GET request on swift contaier. */ +static +size_t fetch_buffer_header_cb(char *ptr, size_t size, size_t nmemb, + void *data) +{ + download_buffer_info *buffer_info = (download_buffer_info*)(data); + size_t buf_size; + char content_length_str[100]; + char *endptr; + + if (get_http_header("Content-Length: ", ptr, + content_length_str, sizeof(content_length_str))) { + + buf_size = strtoull(content_length_str, &endptr, 10); + + if (buffer_info->buf == NULL) { + buffer_info->buf = (char*)(malloc(buf_size)); + buffer_info->size = buf_size; + } + + if (buf_size > buffer_info->size) { + buffer_info->buf = (char*) + (realloc(buffer_info->buf, buf_size)); + buffer_info->size = buf_size; + } + + buffer_info->result_len = buf_size; + } + + if (buffer_info->custom_header_callback) { + buffer_info->custom_header_callback(ptr, size, nmemb, + buffer_info->custom_header_callback_data); + } + + return nmemb * size; +} + +/*********************************************************************//** +Write contents into string buffer */ +static +size_t +fetch_buffer_cb(char *buffer, size_t size, size_t nmemb, void *out_buffer) +{ + download_buffer_info *buffer_info = (download_buffer_info*)(out_buffer); + + assert(buffer_info->size >= buffer_info->offset + size * nmemb); + + memcpy(buffer_info->buf + buffer_info->offset, buffer, size * nmemb); + buffer_info->offset += size * nmemb; + + return size * nmemb; +} + + +/*********************************************************************//** +Downloads contents of URL into buffer. Caller is responsible for +deallocating the buffer. +@return pointer to a buffer or NULL */ +static +char * +swift_fetch_into_buffer(swift_auth_info *auth, const char *url, + char **buf, size_t *buf_size, size_t *result_len, + curl_read_callback header_callback, + void *header_callback_data) +{ + char auth_token[SWIFT_MAX_HDR_SIZE]; + download_buffer_info buffer_info; + struct curl_slist *slist = NULL; + long http_code; + CURL *curl; + CURLcode res; + + memset(&buffer_info, 0, sizeof(buffer_info)); + buffer_info.buf = *buf; + buffer_info.size = *buf_size; + buffer_info.custom_header_callback = header_callback; + buffer_info.custom_header_callback_data = header_callback_data; + + snprintf(auth_token, array_elements(auth_token), "X-Auth-Token: %s", + auth->token); + + curl = curl_easy_init(); + + if (curl != NULL) { + slist = curl_slist_append(slist, auth_token); + + curl_easy_setopt(curl, CURLOPT_VERBOSE, opt_verbose); + curl_easy_setopt(curl, CURLOPT_URL, url); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fetch_buffer_cb); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buffer_info); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, + fetch_buffer_header_cb); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, + &buffer_info); + if (opt_cacert != NULL) + curl_easy_setopt(curl, CURLOPT_CAINFO, opt_cacert); + if (opt_insecure) + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, FALSE); + + res = curl_easy_perform(curl); + + if (res != CURLE_OK) { + fprintf(stderr, + "error: curl_easy_perform() failed: %s\n", + curl_easy_strerror(res)); + goto cleanup; + } + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); + if (http_code < 200 || http_code >= 300) { + fprintf(stderr, "error: request failed " + "with response code: %ld\n", http_code); + res = CURLE_LOGIN_DENIED; + goto cleanup; + } + } else { + res = CURLE_FAILED_INIT; + fprintf(stderr, "error: curl_easy_init() failed\n"); + goto cleanup; + } + +cleanup: + if (slist) { + curl_slist_free_all(slist); + } + if (curl) { + curl_easy_cleanup(curl); + } + + if (res == CURLE_OK) { + *buf = buffer_info.buf; + *buf_size = buffer_info.size; + *result_len = buffer_info.result_len; + return(buffer_info.buf); + } + + free(buffer_info.buf); + *buf = NULL; + *buf_size = 0; + *result_len = 0; + + return(NULL); +} + +static +container_list * +container_list_new() +{ + container_list *list = + (container_list *)(calloc(1, sizeof(container_list))); + + list->object_count = 1000; + list->objects = (object_info*) + (calloc(list->object_count, sizeof(object_info))); + + if (list->objects == NULL) { + fprintf(stderr, "error: out of memory\n"); + free(list); + return(NULL); + } + + return(list); +} + +static +void +container_list_free(container_list *list) +{ + free(list->content_json); + free(list->objects); + free(list); +} + +static +void +container_list_add_object(container_list *list, const char *name, + const char *hash, size_t bytes) +{ + const size_t object_count_step = 1000; + + if (list->idx >= list->object_count) { + list->objects = (object_info*) + realloc(list->objects, + (list->object_count + object_count_step) * + sizeof(object_info)); + memset(list->objects + list->object_count, 0, + object_count_step * sizeof(object_info)); + list->object_count += object_count_step; + } + assert(list->idx <= list->object_count); + strcpy(list->objects[list->idx].name, name); + strcpy(list->objects[list->idx].hash, hash); + list->objects[list->idx].bytes = bytes; + ++list->idx; +} + + +/*********************************************************************//** +Tokenize json string. Return array of tokens. Caller is responsoble for +deallocating the array. */ +jsmntok_t * +json_tokenise(char *json, size_t len, int initial_tokens) +{ + jsmn_parser parser; + jsmn_init(&parser); + + unsigned int n = initial_tokens; + jsmntok_t *tokens = (jsmntok_t *)(malloc(sizeof(jsmntok_t) * n)); + + int ret = jsmn_parse(&parser, json, len, tokens, n); + + while (ret == JSMN_ERROR_NOMEM) + { + n = n * 2 + 1; + tokens = (jsmntok_t*)(realloc(tokens, sizeof(jsmntok_t) * n)); + ret = jsmn_parse(&parser, json, len, tokens, n); + } + + if (ret == JSMN_ERROR_INVAL) { + fprintf(stderr, "error: invalid JSON string\n"); + + } + if (ret == JSMN_ERROR_PART) { + fprintf(stderr, "error: truncated JSON string\n"); + } + + return tokens; +} + +/*********************************************************************//** +Return true if token representation equal to given string. */ +static +bool +json_token_eq(const char *buf, jsmntok_t *t, const char *s) +{ + size_t len = strlen(s); + + assert(t->end > t->start); + + return((size_t)(t->end - t->start) == len && + (strncmp(buf + t->start, s, len) == 0)); +} + +/*********************************************************************//** +Copy given token as string. */ +static +bool +json_token_str(const char *buf, jsmntok_t *t, char *out, int out_size) +{ + size_t len = min(t->end - t->start, out_size - 1); + + memcpy(out, buf + t->start, len); + out[len] = 0; + + return(true); +} + +/*********************************************************************//** +Parse SWIFT container list response and fill output array with values +sorted by object name. */ +static +bool +swift_parse_container_list(container_list *list) +{ + enum {MAX_DEPTH=20}; + enum label_t {NONE, OBJECT}; + + char name[SWIFT_MAX_URL_SIZE]; + char hash[33]; + char bytes[30]; + char *response = list->content_json; + + struct stack_t { + jsmntok_t *t; + int n_items; + label_t label; + }; + + stack_t stack[MAX_DEPTH]; + jsmntok_t *tokens; + int level; + size_t count = 0; + + tokens = json_tokenise(list->content_json, list->content_length, 200); + + stack[0].t = &tokens[0]; + stack[0].label = NONE; + stack[0].n_items = 1; + level = 0; + + for (size_t i = 0, j = 1; j > 0; i++, j--) { + jsmntok_t *t = &tokens[i]; + + assert(t->start != -1 && t->end != -1); + assert(level >= 0); + + --stack[level].n_items; + + switch (t->type) { + case JSMN_ARRAY: + case JSMN_OBJECT: + if (level < MAX_DEPTH - 1) { + level++; + } + stack[level].t = t; + stack[level].label = NONE; + if (t->type == JSMN_ARRAY) { + stack[level].n_items = t->size; + j += t->size; + } else { + stack[level].n_items = t->size * 2; + j += t->size * 2; + } + break; + case JSMN_PRIMITIVE: + case JSMN_STRING: + if (stack[level].t->type == JSMN_OBJECT && + stack[level].n_items % 2 == 1) { + /* key */ + if (json_token_eq(response, t, "name")) { + json_token_str(response, &tokens[i + 1], + name, sizeof(name)); + } + if (json_token_eq(response, t, "hash")) { + json_token_str(response, &tokens[i + 1], + hash, sizeof(hash)); + } + if (json_token_eq(response, t, "bytes")) { + json_token_str(response, &tokens[i + 1], + bytes, sizeof(bytes)); + } + } + break; + } + + while (stack[level].n_items == 0 && level > 0) { + if (stack[level].t->type == JSMN_OBJECT + && level == 2) { + char *endptr; + container_list_add_object(list, name, hash, + strtoull(bytes, &endptr, 10)); + ++count; + } + --level; + } + } + + if (count == 0) { + list->final = true; + } + + free(tokens); + + return(true); +} + +/*********************************************************************//** +List swift container with given name. Return list of objects sorted by +object name. */ +static +container_list * +swift_list(swift_auth_info *auth, const char *container, const char *path) +{ + container_list *list; + char url[SWIFT_MAX_URL_SIZE]; + + list = container_list_new(); + + while (!list->final) { + + /* download the list in json format */ + snprintf(url, array_elements(url), + "%s/%s?format=json&limit=1000%s%s%s%s", + auth->url, container, path ? "&prefix=" : "", + path ? path : "", list->idx > 0 ? "&marker=" : "", + list->idx > 0 ? + list->objects[list->idx - 1].name : ""); + + list->content_json = swift_fetch_into_buffer(auth, url, + &list->content_json, &list->content_bufsize, + &list->content_length, NULL, NULL); + + if (list->content_json == NULL) { + container_list_free(list); + return(NULL); + } + + /* parse downloaded list */ + if (!swift_parse_container_list(list)) { + fprintf(stderr, "error: unable to parse " + "container list\n"); + container_list_free(list); + return(NULL); + } + } + + return(list); +} + + +/*********************************************************************//** +Return true if chunk is a part of backup with given name. */ +static +bool +chunk_belongs_to(const char *chunk_name, const char *backup_name) +{ + size_t backup_name_len = strlen(backup_name); + + return((strlen(chunk_name) > backup_name_len) + && (chunk_name[backup_name_len] == '/') + && strncmp(chunk_name, backup_name, backup_name_len) == 0); +} + +/*********************************************************************//** +Return true if chunk is in given list. */ +static +bool +chunk_in_list(const char *chunk_name, char **list, int list_size) +{ + size_t chunk_name_len; + + if (list_size == 0) { + return(true); + } + + chunk_name_len = strlen(chunk_name); + if (chunk_name_len < 20) { + return(false); + } + + for (int i = 0; i < list_size; i++) { + size_t item_len = strlen(list[i]); + + if ((strncmp(chunk_name - item_len + chunk_name_len - 21, + list[i], item_len) == 0) + && (chunk_name[chunk_name_len - 21] == '.') + && (chunk_name[chunk_name_len - item_len - 22] == '/')) { + return(true); + } + } + + return(false); +} + +static +int swift_download(swift_auth_info *auth, const char *container, + const char *name) +{ + container_list *list; + char *buf = NULL; + size_t buf_size = 0; + size_t result_len = 0; + + if ((list = swift_list(auth, container, name)) == NULL) { + return(CURLE_FAILED_INIT); + } + + for (size_t i = 0; i < list->idx; i++) { + const char *chunk_name = list->objects[i].name; + + if (chunk_belongs_to(chunk_name, name) + && chunk_in_list(chunk_name, file_list, file_list_size)) { + char url[SWIFT_MAX_URL_SIZE]; + + snprintf(url, sizeof(url), "%s/%s/%s", + auth->url, container, chunk_name); + + if ((buf = swift_fetch_into_buffer( + auth, url, &buf, &buf_size, &result_len, + NULL, NULL)) == NULL) { + fprintf(stderr, "error: failed to download " + "chunk %s\n", chunk_name); + container_list_free(list); + return(CURLE_FAILED_INIT); + } + + fwrite(buf, 1, result_len, stdout); + } + } + + free(buf); + + container_list_free(list); + + return(CURLE_OK); +} + + +/*********************************************************************//** +Delete backup with given name from given container. +@return true if backup deleted successfully */ +static +bool swift_delete(swift_auth_info *auth, const char *container, + const char *name) +{ + container_list *list; + + if ((list = swift_list(auth, container, name)) == NULL) { + return(CURLE_FAILED_INIT); + } + + for (size_t i = 0; i < list->object_count; i++) { + const char *chunk_name = list->objects[i].name; + + if (chunk_belongs_to(chunk_name, name)) { + char url[SWIFT_MAX_URL_SIZE]; + + snprintf(url, sizeof(url), "%s/%s/%s", + auth->url, container, chunk_name); + + fprintf(stderr, "delete %s\n", chunk_name); + if (!swift_delete_object(auth, url)) { + fprintf(stderr, "error: failed to delete " + "chunk %s\n", chunk_name); + container_list_free(list); + return(CURLE_FAILED_INIT); + } + } + } + + container_list_free(list); + + return(CURLE_OK); +} + +/*********************************************************************//** +Check if backup with given name exists. +@return true if backup exists */ +static +bool swift_backup_exists(swift_auth_info *auth, const char *container, + const char *backup_name) +{ + container_list *list; + + if ((list = swift_list(auth, container, backup_name)) == NULL) { + fprintf(stderr, "error: unable to list container %s\n", + container); + exit(EXIT_FAILURE); + } + + for (size_t i = 0; i < list->object_count; i++) { + if (chunk_belongs_to(list->objects[i].name, backup_name)) { + container_list_free(list); + return(true); + } + } + + container_list_free(list); + + return(false); +} + +/*********************************************************************//** +Fills auth_info with response from keystone response. +@return true is response parsed successfully */ +static +bool +swift_parse_keystone_response_v2(char *response, size_t response_length, + swift_auth_info *auth_info) +{ + enum {MAX_DEPTH=20}; + enum label_t {NONE, ACCESS, CATALOG, ENDPOINTS, TOKEN}; + + char filtered_url[SWIFT_MAX_URL_SIZE]; + char public_url[SWIFT_MAX_URL_SIZE]; + char region[SWIFT_MAX_URL_SIZE]; + char id[SWIFT_MAX_URL_SIZE]; + char token_id[SWIFT_MAX_URL_SIZE]; + char type[SWIFT_MAX_URL_SIZE]; + + struct stack_t { + jsmntok_t *t; + int n_items; + label_t label; + }; + + stack_t stack[MAX_DEPTH]; + jsmntok_t *tokens; + int level; + + tokens = json_tokenise(response, response_length, 200); + + stack[0].t = &tokens[0]; + stack[0].label = NONE; + stack[0].n_items = 1; + level = 0; + + for (size_t i = 0, j = 1; j > 0; i++, j--) { + jsmntok_t *t = &tokens[i]; + + assert(t->start != -1 && t->end != -1); + assert(level >= 0); + + --stack[level].n_items; + + switch (t->type) { + case JSMN_ARRAY: + case JSMN_OBJECT: + if (level < MAX_DEPTH - 1) { + level++; + } + stack[level].t = t; + stack[level].label = NONE; + if (t->type == JSMN_ARRAY) { + stack[level].n_items = t->size; + j += t->size; + } else { + stack[level].n_items = t->size * 2; + j += t->size * 2; + } + break; + case JSMN_PRIMITIVE: + case JSMN_STRING: + if (stack[level].t->type == JSMN_OBJECT && + stack[level].n_items % 2 == 1) { + /* key */ + if (json_token_eq(response, t, "access")) { + stack[level].label = ACCESS; + } + if (json_token_eq(response, t, + "serviceCatalog")) { + stack[level].label = CATALOG; + } + if (json_token_eq(response, t, "endpoints")) { + stack[level].label = ENDPOINTS; + } + if (json_token_eq(response, t, "token")) { + stack[level].label = TOKEN; + } + if (json_token_eq(response, t, "id")) { + json_token_str(response, &tokens[i + 1], + id, sizeof(id)); + } + if (json_token_eq(response, t, "id") + && stack[level - 1].label == TOKEN) { + json_token_str(response, &tokens[i + 1], + token_id, sizeof(token_id)); + } + if (json_token_eq(response, t, "region")) { + json_token_str(response, &tokens[i + 1], + region, sizeof(region)); + } + if (json_token_eq(response, t, "publicURL")) { + json_token_str(response, &tokens[i + 1], + public_url, sizeof(public_url)); + } + if (json_token_eq(response, t, "type")) { + json_token_str(response, &tokens[i + 1], + type, sizeof(type)); + } + } + break; + } + + while (stack[level].n_items == 0 && level > 0) { + if (stack[level].t->type == JSMN_OBJECT + && level == 6 + && stack[level - 1].t->type == JSMN_ARRAY + && stack[level - 2].label == ENDPOINTS) { + if (opt_swift_region == NULL + || strcmp(opt_swift_region, region) == 0) { + strncpy(filtered_url, public_url, + sizeof(filtered_url)); + } + } + if (stack[level].t->type == JSMN_OBJECT && + level == 4 && + stack[level - 1].t->type == JSMN_ARRAY && + stack[level - 2].label == CATALOG) { + if (strcmp(type, "object-store") == 0) { + strncpy(auth_info->url, filtered_url, + sizeof(auth_info->url)); + } + } + --level; + } + } + + free(tokens); + + strncpy(auth_info->token, token_id, sizeof(auth_info->token)); + + assert(level == 0); + + if (*auth_info->token == 0) { + fprintf(stderr, "error: can not receive token from response\n"); + return(false); + } + + if (*auth_info->url == 0) { + fprintf(stderr, "error: can not get URL from response\n"); + return(false); + } + + return(true); +} + +/*********************************************************************//** +Authenticate against Swift TempAuth. Fills swift_auth_info struct. +Uses creadentials privided as global variables. +@returns true if access is granted and token received. */ +static +bool +swift_keystone_auth_v2(const char *auth_url, swift_auth_info *info) +{ + char tenant_arg[SWIFT_MAX_URL_SIZE]; + char payload[SWIFT_MAX_URL_SIZE]; + struct curl_slist *slist = NULL; + download_buffer_info buf_info; + long http_code; + CURLcode res; + CURL *curl; + bool auth_res = false; + + memset(&buf_info, 0, sizeof(buf_info)); + + if (opt_swift_user == NULL) { + fprintf(stderr, "error: both --swift-user is required " + "for keystone authentication.\n"); + return(false); + } + + if (opt_swift_password == NULL) { + fprintf(stderr, "error: both --swift-password is required " + "for keystone authentication.\n"); + return(false); + } + + if (opt_swift_tenant != NULL && opt_swift_tenant_id != NULL) { + fprintf(stderr, "error: both --swift-tenant and " + "--swift-tenant-id specified for keystone " + "authentication.\n"); + return(false); + } + + if (opt_swift_tenant != NULL) { + snprintf(tenant_arg, sizeof(tenant_arg), ",\"%s\":\"%s\"", + "tenantName", opt_swift_tenant); + } else if (opt_swift_tenant_id != NULL) { + snprintf(tenant_arg, sizeof(tenant_arg), ",\"%s\":\"%s\"", + "tenantId", opt_swift_tenant_id); + } else { + *tenant_arg = 0; + } + + snprintf(payload, sizeof(payload), "{\"auth\": " + "{\"passwordCredentials\": {\"username\":\"%s\"," + "\"password\":\"%s\"}%s}}", + opt_swift_user, opt_swift_password, tenant_arg); + + curl = curl_easy_init(); + + if (curl != NULL) { + + slist = curl_slist_append(slist, + "Content-Type: application/json"); + slist = curl_slist_append(slist, + "Accept: application/json"); + + curl_easy_setopt(curl, CURLOPT_VERBOSE, opt_verbose); + curl_easy_setopt(curl, CURLOPT_POST, 1L); + curl_easy_setopt(curl, CURLOPT_URL, auth_url); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, payload); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fetch_buffer_cb); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buf_info); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, + fetch_buffer_header_cb); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, + &buf_info); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist); + + if (opt_cacert != NULL) + curl_easy_setopt(curl, CURLOPT_CAINFO, opt_cacert); + if (opt_insecure) + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, FALSE); + + res = curl_easy_perform(curl); + + if (res != CURLE_OK) { + fprintf(stderr, + "error: curl_easy_perform() failed: %s\n", + curl_easy_strerror(res)); + goto cleanup; + } + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); + if (http_code < 200 || http_code >= 300) { + fprintf(stderr, "error: request failed " + "with response code: %ld\n", http_code); + res = CURLE_LOGIN_DENIED; + goto cleanup; + } + } else { + res = CURLE_FAILED_INIT; + fprintf(stderr, "error: curl_easy_init() failed\n"); + goto cleanup; + } + + if (!swift_parse_keystone_response_v2(buf_info.buf, + buf_info.size, info)) { + goto cleanup; + } + + auth_res = true; + +cleanup: + if (slist) { + curl_slist_free_all(slist); + } + if (curl) { + curl_easy_cleanup(curl); + } + + free(buf_info.buf); + + return(auth_res); +} + + +/*********************************************************************//** +Fills auth_info with response from keystone response. +@return true is response parsed successfully */ +static +bool +swift_parse_keystone_response_v3(char *response, size_t response_length, + swift_auth_info *auth_info) +{ + enum {MAX_DEPTH=20}; + enum label_t {NONE, TOKEN, CATALOG, ENDPOINTS}; + + char url[SWIFT_MAX_URL_SIZE]; + char filtered_url[SWIFT_MAX_URL_SIZE]; + char region[SWIFT_MAX_URL_SIZE]; + char interface[SWIFT_MAX_URL_SIZE]; + char type[SWIFT_MAX_URL_SIZE]; + + struct stack_t { + jsmntok_t *t; + int n_items; + label_t label; + }; + + stack_t stack[MAX_DEPTH]; + jsmntok_t *tokens; + int level; + + tokens = json_tokenise(response, response_length, 200); + + stack[0].t = &tokens[0]; + stack[0].label = NONE; + stack[0].n_items = 1; + level = 0; + + for (size_t i = 0, j = 1; j > 0; i++, j--) { + jsmntok_t *t = &tokens[i]; + + assert(t->start != -1 && t->end != -1); + assert(level >= 0); + + --stack[level].n_items; + + switch (t->type) { + case JSMN_ARRAY: + case JSMN_OBJECT: + if (level < MAX_DEPTH - 1) { + level++; + } + stack[level].t = t; + stack[level].label = NONE; + if (t->type == JSMN_ARRAY) { + stack[level].n_items = t->size; + j += t->size; + } else { + stack[level].n_items = t->size * 2; + j += t->size * 2; + } + break; + case JSMN_PRIMITIVE: + case JSMN_STRING: + if (stack[level].t->type == JSMN_OBJECT && + stack[level].n_items % 2 == 1) { + /* key */ + if (json_token_eq(response, t, "token")) { + stack[level].label = TOKEN; + fprintf(stderr, "token\n"); + } + if (json_token_eq(response, t, + "catalog")) { + stack[level].label = CATALOG; + fprintf(stderr, "catalog\n"); + } + if (json_token_eq(response, t, "endpoints")) { + stack[level].label = ENDPOINTS; + } + if (json_token_eq(response, t, "region")) { + json_token_str(response, &tokens[i + 1], + region, sizeof(region)); + } + if (json_token_eq(response, t, "url")) { + json_token_str(response, &tokens[i + 1], + url, sizeof(url)); + } + if (json_token_eq(response, t, "interface")) { + json_token_str(response, &tokens[i + 1], + interface, sizeof(interface)); + } + if (json_token_eq(response, t, "type")) { + json_token_str(response, &tokens[i + 1], + type, sizeof(type)); + } + } + break; + } + + while (stack[level].n_items == 0 && level > 0) { + if (stack[level].t->type == JSMN_OBJECT + && level == 6 + && stack[level - 1].t->type == JSMN_ARRAY + && stack[level - 2].label == ENDPOINTS) { + if ((opt_swift_region == NULL + || strcmp(opt_swift_region, region) == 0) + && strcmp(interface, "public") == 0) { + strncpy(filtered_url, url, + sizeof(filtered_url)); + } + } + if (stack[level].t->type == JSMN_OBJECT && + level == 4 && + stack[level - 1].t->type == JSMN_ARRAY && + stack[level - 2].label == CATALOG) { + if (strcmp(type, "object-store") == 0) { + strncpy(auth_info->url, filtered_url, + sizeof(auth_info->url)); + } + } + --level; + } + } + + free(tokens); + + assert(level == 0); + + if (*auth_info->url == 0) { + fprintf(stderr, "error: can not get URL from response\n"); + return(false); + } + + return(true); +} + +/*********************************************************************//** +Captures X-Subject-Token header. */ +static +size_t keystone_v3_header_cb(char *ptr, size_t size, size_t nmemb, void *data) +{ + swift_auth_info *info = (swift_auth_info*)(data); + + get_http_header("X-Subject-Token: ", ptr, + info->token, array_elements(info->token)); + + return nmemb * size; +} + +/*********************************************************************//** +Authenticate against Swift TempAuth. Fills swift_auth_info struct. +Uses creadentials privided as global variables. +@returns true if access is granted and token received. */ +static +bool +swift_keystone_auth_v3(const char *auth_url, swift_auth_info *info) +{ + char scope[SWIFT_MAX_URL_SIZE]; + char domain[SWIFT_MAX_URL_SIZE]; + char payload[SWIFT_MAX_URL_SIZE]; + struct curl_slist *slist = NULL; + download_buffer_info buf_info; + long http_code; + CURLcode res; + CURL *curl; + bool auth_res = false; + + memset(&buf_info, 0, sizeof(buf_info)); + buf_info.custom_header_callback = keystone_v3_header_cb; + buf_info.custom_header_callback_data = info; + + if (opt_swift_user == NULL) { + fprintf(stderr, "error: both --swift-user is required " + "for keystone authentication.\n"); + return(false); + } + + if (opt_swift_password == NULL) { + fprintf(stderr, "error: both --swift-password is required " + "for keystone authentication.\n"); + return(false); + } + + if (opt_swift_project_id != NULL && opt_swift_project != NULL) { + fprintf(stderr, "error: both --swift-project and " + "--swift-project-id specified for keystone " + "authentication.\n"); + return(false); + } + + if (opt_swift_domain_id != NULL && opt_swift_domain != NULL) { + fprintf(stderr, "error: both --swift-domain and " + "--swift-domain-id specified for keystone " + "authentication.\n"); + return(false); + } + + if (opt_swift_project_id != NULL && opt_swift_domain != NULL) { + fprintf(stderr, "error: both --swift-project-id and " + "--swift-domain specified for keystone " + "authentication.\n"); + return(false); + } + + if (opt_swift_project_id != NULL && opt_swift_domain_id != NULL) { + fprintf(stderr, "error: both --swift-project-id and " + "--swift-domain-id specified for keystone " + "authentication.\n"); + return(false); + } + + scope[0] = 0; domain[0] = 0; + + if (opt_swift_domain != NULL) { + snprintf(domain, sizeof(domain), + ",{\"domain\":{\"name\":\"%s\"}}", + opt_swift_domain); + } else if (opt_swift_domain_id != NULL) { + snprintf(domain, sizeof(domain), + ",{\"domain\":{\"id\":\"%s\"}}", + opt_swift_domain_id); + } + + if (opt_swift_project_id != NULL) { + snprintf(scope, sizeof(scope), + ",\"scope\":{\"project\":{\"id\":\"%s\"}}", + opt_swift_project_id); + } else if (opt_swift_project != NULL) { + snprintf(scope, sizeof(scope), + ",\"scope\":{\"project\":{\"name\":\"%s\"%s}}", + opt_swift_project_id, domain); + } + + snprintf(payload, sizeof(payload), "{\"auth\":{\"identity\":" + "{\"methods\":[\"password\"],\"password\":{\"user\":" + "{\"name\":\"%s\",\"password\":\"%s\"%s}}}%s}}", + opt_swift_user, opt_swift_password, + *scope ? "" : ",\"domain\":{\"id\":\"default\"}", + scope); + + curl = curl_easy_init(); + + if (curl != NULL) { + + slist = curl_slist_append(slist, + "Content-Type: application/json"); + slist = curl_slist_append(slist, + "Accept: application/json"); + + curl_easy_setopt(curl, CURLOPT_VERBOSE, opt_verbose); + curl_easy_setopt(curl, CURLOPT_POST, 1L); + curl_easy_setopt(curl, CURLOPT_URL, auth_url); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, payload); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fetch_buffer_cb); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buf_info); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, + fetch_buffer_header_cb); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, + &buf_info); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist); + + if (opt_cacert != NULL) + curl_easy_setopt(curl, CURLOPT_CAINFO, opt_cacert); + if (opt_insecure) + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, FALSE); + + res = curl_easy_perform(curl); + + if (res != CURLE_OK) { + fprintf(stderr, + "error: curl_easy_perform() failed: %s\n", + curl_easy_strerror(res)); + goto cleanup; + } + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); + if (http_code < 200 || http_code >= 300) { + fprintf(stderr, "error: request failed " + "with response code: %ld\n", http_code); + res = CURLE_LOGIN_DENIED; + goto cleanup; + } + } else { + res = CURLE_FAILED_INIT; + fprintf(stderr, "error: curl_easy_init() failed\n"); + goto cleanup; + } + + if (!swift_parse_keystone_response_v3(buf_info.buf, + buf_info.size, info)) { + goto cleanup; + } + + auth_res = true; + +cleanup: + if (slist) { + curl_slist_free_all(slist); + } + if (curl) { + curl_easy_cleanup(curl); + } + + free(buf_info.buf); + + return(auth_res); +} + +int main(int argc, char **argv) +{ + swift_auth_info info; + char auth_url[SWIFT_MAX_URL_SIZE]; + + MY_INIT(argv[0]); + + /* handle_options in parse_args is destructive so + * we make a copy of our argument pointers so we can + * mask the sensitive values afterwards */ + char **mask_argv = (char **)malloc(sizeof(char *) * (argc - 1)); + memcpy(mask_argv, argv + 1, sizeof(char *) * (argc - 1)); + + if (parse_args(argc, argv)) { + return(EXIT_FAILURE); + } + + mask_args(argc, mask_argv); /* mask args on cmdline */ + + curl_global_init(CURL_GLOBAL_ALL); + + if (opt_swift_auth_version == NULL || *opt_swift_auth_version == '1') { + /* TempAuth */ + snprintf(auth_url, SWIFT_MAX_URL_SIZE, "%sauth/v%s/", + opt_swift_auth_url, opt_swift_auth_version ? + opt_swift_auth_version : "1.0"); + + if (!swift_temp_auth(auth_url, &info)) { + fprintf(stderr, "error: failed to authenticate\n"); + return(EXIT_FAILURE); + } + + } else if (*opt_swift_auth_version == '2') { + /* Keystone v2 */ + snprintf(auth_url, SWIFT_MAX_URL_SIZE, "%sv%s/tokens", + opt_swift_auth_url, opt_swift_auth_version); + + if (!swift_keystone_auth_v2(auth_url, &info)) { + fprintf(stderr, "error: failed to authenticate\n"); + return(EXIT_FAILURE); + } + + } else if (*opt_swift_auth_version == '3') { + /* Keystone v3 */ + snprintf(auth_url, SWIFT_MAX_URL_SIZE, "%sv%s/auth/tokens", + opt_swift_auth_url, opt_swift_auth_version); + + if (!swift_keystone_auth_v3(auth_url, &info)) { + fprintf(stderr, "error: failed to authenticate\n"); + exit(EXIT_FAILURE); + } + + } + + if (opt_swift_storage_url != NULL) { + snprintf(info.url, sizeof(info.url), "%s", + opt_swift_storage_url); + } + + fprintf(stderr, "Object store URL: %s\n", info.url); + + if (opt_mode == MODE_PUT) { + + if (swift_create_container(&info, opt_swift_container) != 0) { + fprintf(stderr, "error: failed to create " + "container %s\n", + opt_swift_container); + return(EXIT_FAILURE); + } + + if (swift_backup_exists(&info, opt_swift_container, opt_name)) { + fprintf(stderr, "error: backup named '%s' " + "already exists!\n", + opt_name); + return(EXIT_FAILURE); + } + + if (swift_upload_parts(&info, opt_swift_container, + opt_name) != 0) { + fprintf(stderr, "error: upload failed\n"); + return(EXIT_FAILURE); + } + + } else if (opt_mode == MODE_GET) { + + if (swift_download(&info, opt_swift_container, opt_name) + != CURLE_OK) { + fprintf(stderr, "error: download failed\n"); + return(EXIT_FAILURE); + } + + } else if (opt_mode == MODE_DELETE) { + + if (swift_delete(&info, opt_swift_container, opt_name) + != CURLE_OK) { + fprintf(stderr, "error: delete failed\n"); + return(EXIT_FAILURE); + } + + } else { + fprintf(stderr, "Unknown command supplied.\n"); + exit(EXIT_FAILURE); + } + + curl_global_cleanup(); + + return(EXIT_SUCCESS); +} diff --git a/extra/mariabackup/xbcrypt.c b/extra/mariabackup/xbcrypt.c new file mode 100644 index 00000000000..3da70e171f7 --- /dev/null +++ b/extra/mariabackup/xbcrypt.c @@ -0,0 +1,696 @@ +/****************************************************** +Copyright (c) 2013 Percona LLC and/or its affiliates. + +The xbcrypt utility: decrypt files in the XBCRYPT format. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#include <my_base.h> +#include <my_getopt.h> +#include "common.h" +#include "xbcrypt.h" +#include "xbcrypt_common.h" +#include "crc_glue.h" + +#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600) +GCRY_THREAD_OPTION_PTHREAD_IMPL; +#endif + +#define XBCRYPT_VERSION "1.1" + +typedef enum { + RUN_MODE_NONE, + RUN_MODE_ENCRYPT, + RUN_MODE_DECRYPT +} run_mode_t; + +const char *xbcrypt_encrypt_algo_names[] = +{ "NONE", "AES128", "AES192", "AES256", NullS}; +TYPELIB xbcrypt_encrypt_algo_typelib= +{array_elements(xbcrypt_encrypt_algo_names)-1,"", + xbcrypt_encrypt_algo_names, NULL}; + +static run_mode_t opt_run_mode = RUN_MODE_ENCRYPT; +static char *opt_input_file = NULL; +static char *opt_output_file = NULL; +static ulong opt_encrypt_algo; +static char *opt_encrypt_key_file = NULL; +static void *opt_encrypt_key = NULL; +static ulonglong opt_encrypt_chunk_size = 0; +static my_bool opt_verbose = FALSE; + +static uint encrypt_algos[] = { GCRY_CIPHER_NONE, + GCRY_CIPHER_AES128, + GCRY_CIPHER_AES192, + GCRY_CIPHER_AES256 }; +static int encrypt_algo = 0; +static int encrypt_mode = GCRY_CIPHER_MODE_CTR; +static uint encrypt_key_len = 0; +static size_t encrypt_iv_len = 0; + +static struct my_option my_long_options[] = +{ + {"help", '?', "Display this help and exit.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"decrypt", 'd', "Decrypt data input to output.", + 0, 0, 0, + GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"input", 'i', "Optional input file. If not specified, input" + " will be read from standard input.", + &opt_input_file, &opt_input_file, 0, + GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"output", 'o', "Optional output file. If not specified, output" + " will be written to standard output.", + &opt_output_file, &opt_output_file, 0, + GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"encrypt-algo", 'a', "Encryption algorithm.", + &opt_encrypt_algo, &opt_encrypt_algo, &xbcrypt_encrypt_algo_typelib, + GET_ENUM, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"encrypt-key", 'k', "Encryption key.", + &opt_encrypt_key, &opt_encrypt_key, 0, + GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"encrypt-key-file", 'f', "File which contains encryption key.", + &opt_encrypt_key_file, &opt_encrypt_key_file, 0, + GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"encrypt-chunk-size", 's', "Size of working buffer for encryption in" + " bytes. The default value is 64K.", + &opt_encrypt_chunk_size, &opt_encrypt_chunk_size, 0, + GET_ULL, REQUIRED_ARG, (1 << 16), 1024, ULONGLONG_MAX, 0, 0, 0}, + + {"verbose", 'v', "Display verbose status output.", + &opt_verbose, &opt_verbose, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} +}; + +static +int +get_options(int *argc, char ***argv); + +static +my_bool +get_one_option(int optid, const struct my_option *opt __attribute__((unused)), + char *argument __attribute__((unused))); + +static +void +print_version(void); + +static +void +usage(void); + +static +int +mode_decrypt(File filein, File fileout); + +static +int +mode_encrypt(File filein, File fileout); + +int +main(int argc, char **argv) +{ +#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600) + gcry_error_t gcry_error; +#endif + File filein = 0; + File fileout = 0; + + MY_INIT(argv[0]); + + crc_init(); + + if (get_options(&argc, &argv)) { + goto err; + } + + /* Acording to gcrypt docs (and my testing), setting up the threading + callbacks must be done first, so, lets give it a shot */ +#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600) + gcry_error = gcry_control(GCRYCTL_SET_THREAD_CBS, &gcry_threads_pthread); + if (gcry_error) { + msg("%s: unable to set libgcrypt thread cbs - " + "%s : %s\n", my_progname, + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + return 1; + } +#endif + + /* Version check should be the very first call because it + makes sure that important subsystems are intialized. */ + if (!gcry_control(GCRYCTL_ANY_INITIALIZATION_P)) { + const char *gcrypt_version; + gcrypt_version = gcry_check_version(NULL); + /* No other library has already initialized libgcrypt. */ + if (!gcrypt_version) { + msg("%s: failed to initialize libgcrypt\n", + my_progname); + return 1; + } else if (opt_verbose) { + msg("%s: using gcrypt %s\n", my_progname, + gcrypt_version); + } + } + gcry_control(GCRYCTL_DISABLE_SECMEM, 0); + gcry_control(GCRYCTL_INITIALIZATION_FINISHED, 0); + + /* Determine the algorithm */ + encrypt_algo = encrypt_algos[opt_encrypt_algo]; + + /* Set up the iv length */ + encrypt_iv_len = gcry_cipher_get_algo_blklen(encrypt_algo); + + /* Now set up the key */ + if (opt_encrypt_key == NULL && opt_encrypt_key_file == NULL) { + msg("%s: no encryption key or key file specified.\n", + my_progname); + return 1; + } else if (opt_encrypt_key && opt_encrypt_key_file) { + msg("%s: both encryption key and key file specified.\n", + my_progname); + return 1; + } else if (opt_encrypt_key_file) { + if (!xb_crypt_read_key_file(opt_encrypt_key_file, + &opt_encrypt_key, + &encrypt_key_len)) { + msg("%s: unable to read encryption key file \"%s\".\n", + opt_encrypt_key_file, my_progname); + return 1; + } + } else { + encrypt_key_len = strlen(opt_encrypt_key); + } + + if (opt_input_file) { + MY_STAT mystat; + + if (opt_verbose) + msg("%s: input file \"%s\".\n", my_progname, + opt_input_file); + + if (my_stat(opt_input_file, &mystat, MYF(MY_WME)) == NULL) { + goto err; + } + if (!MY_S_ISREG(mystat.st_mode)) { + msg("%s: \"%s\" is not a regular file, exiting.\n", + my_progname, opt_input_file); + goto err; + } + if ((filein = my_open(opt_input_file, O_RDONLY, MYF(MY_WME))) + < 0) { + msg("%s: failed to open \"%s\".\n", my_progname, + opt_input_file); + goto err; + } + } else { + if (opt_verbose) + msg("%s: input from standard input.\n", my_progname); + filein = fileno(stdin); + } + + if (opt_output_file) { + if (opt_verbose) + msg("%s: output file \"%s\".\n", my_progname, + opt_output_file); + + if ((fileout = my_create(opt_output_file, 0, + O_WRONLY|O_BINARY|O_EXCL|O_NOFOLLOW, + MYF(MY_WME))) < 0) { + msg("%s: failed to create output file \"%s\".\n", + my_progname, opt_output_file); + goto err; + } + } else { + if (opt_verbose) + msg("%s: output to standard output.\n", my_progname); + fileout = fileno(stdout); + } + + if (opt_run_mode == RUN_MODE_DECRYPT + && mode_decrypt(filein, fileout)) { + goto err; + } else if (opt_run_mode == RUN_MODE_ENCRYPT + && mode_encrypt(filein, fileout)) { + goto err; + } + + if (opt_input_file && filein) { + my_close(filein, MYF(MY_WME)); + } + if (opt_output_file && fileout) { + my_close(fileout, MYF(MY_WME)); + } + + my_cleanup_options(my_long_options); + + my_end(0); + + return EXIT_SUCCESS; +err: + if (opt_input_file && filein) { + my_close(filein, MYF(MY_WME)); + } + if (opt_output_file && fileout) { + my_close(fileout, MYF(MY_WME)); + } + + my_cleanup_options(my_long_options); + + my_end(0); + + exit(EXIT_FAILURE); + +} + + +static +size_t +my_xb_crypt_read_callback(void *userdata, void *buf, size_t len) +{ + File* file = (File *) userdata; + return xb_read_full(*file, buf, len); +} + +static +int +mode_decrypt(File filein, File fileout) +{ + xb_rcrypt_t *xbcrypt_file = NULL; + void *chunkbuf = NULL; + size_t chunksize; + size_t originalsize; + void *ivbuf = NULL; + size_t ivsize; + void *decryptbuf = NULL; + size_t decryptbufsize = 0; + ulonglong ttlchunksread = 0; + ulonglong ttlbytesread = 0; + xb_rcrypt_result_t result; + gcry_cipher_hd_t cipher_handle; + gcry_error_t gcry_error; + my_bool hash_appended; + + if (encrypt_algo != GCRY_CIPHER_NONE) { + gcry_error = gcry_cipher_open(&cipher_handle, + encrypt_algo, + encrypt_mode, 0); + if (gcry_error) { + msg("%s:decrypt: unable to open libgcrypt" + " cipher - %s : %s\n", my_progname, + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + return 1; + } + + gcry_error = gcry_cipher_setkey(cipher_handle, + opt_encrypt_key, + encrypt_key_len); + if (gcry_error) { + msg("%s:decrypt: unable to set libgcrypt cipher" + "key - %s : %s\n", my_progname, + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + goto err; + } + } + + /* Initialize the xb_crypt format reader */ + xbcrypt_file = xb_crypt_read_open(&filein, my_xb_crypt_read_callback); + if (xbcrypt_file == NULL) { + msg("%s:decrypt: xb_crypt_read_open() failed.\n", my_progname); + goto err; + } + + /* Walk the encrypted chunks, decrypting them and writing out */ + while ((result = xb_crypt_read_chunk(xbcrypt_file, &chunkbuf, + &originalsize, &chunksize, + &ivbuf, &ivsize, &hash_appended)) + == XB_CRYPT_READ_CHUNK) { + + if (encrypt_algo != GCRY_CIPHER_NONE) { + gcry_error = gcry_cipher_reset(cipher_handle); + if (gcry_error) { + msg("%s:decrypt: unable to reset libgcrypt" + " cipher - %s : %s\n", my_progname, + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + goto err; + } + + if (ivsize) { + gcry_error = gcry_cipher_setctr(cipher_handle, + ivbuf, + ivsize); + } + if (gcry_error) { + msg("%s:decrypt: unable to set cipher iv - " + "%s : %s\n", my_progname, + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + continue; + } + + if (decryptbufsize < originalsize) { + decryptbuf = my_realloc(decryptbuf, + originalsize, + MYF(MY_WME | MY_ALLOW_ZERO_PTR)); + decryptbufsize = originalsize; + } + + /* Try to decrypt it */ + gcry_error = gcry_cipher_decrypt(cipher_handle, + decryptbuf, + originalsize, + chunkbuf, + chunksize); + if (gcry_error) { + msg("%s:decrypt: unable to decrypt chunk - " + "%s : %s\n", my_progname, + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + gcry_cipher_close(cipher_handle); + goto err; + } + + } else { + decryptbuf = chunkbuf; + } + + if (hash_appended) { + uchar hash[XB_CRYPT_HASH_LEN]; + + originalsize -= XB_CRYPT_HASH_LEN; + + /* ensure that XB_CRYPT_HASH_LEN is the correct length + of XB_CRYPT_HASH hashing algorithm output */ + xb_a(gcry_md_get_algo_dlen(XB_CRYPT_HASH) == + XB_CRYPT_HASH_LEN); + gcry_md_hash_buffer(XB_CRYPT_HASH, hash, decryptbuf, + originalsize); + if (memcmp(hash, (char *) decryptbuf + originalsize, + XB_CRYPT_HASH_LEN) != 0) { + msg("%s:%s invalid plaintext hash. " + "Wrong encrytion key specified?\n", + my_progname, __FUNCTION__); + result = XB_CRYPT_READ_ERROR; + goto err; + } + } + + /* Write it out */ + if (my_write(fileout, (const uchar *) decryptbuf, originalsize, + MYF(MY_WME | MY_NABP))) { + msg("%s:decrypt: unable to write output chunk.\n", + my_progname); + goto err; + } + ttlchunksread++; + ttlbytesread += chunksize; + if (opt_verbose) + msg("%s:decrypt: %llu chunks read, %llu bytes read\n.", + my_progname, ttlchunksread, ttlbytesread); + } + + xb_crypt_read_close(xbcrypt_file); + + if (encrypt_algo != GCRY_CIPHER_NONE) + gcry_cipher_close(cipher_handle); + + if (decryptbuf && decryptbufsize) + my_free(decryptbuf); + + if (opt_verbose) + msg("\n%s:decrypt: done\n", my_progname); + + return 0; +err: + if (xbcrypt_file) + xb_crypt_read_close(xbcrypt_file); + + if (encrypt_algo != GCRY_CIPHER_NONE) + gcry_cipher_close(cipher_handle); + + if (decryptbuf && decryptbufsize) + my_free(decryptbuf); + + return 1; +} + +static +ssize_t +my_xb_crypt_write_callback(void *userdata, const void *buf, size_t len) +{ + File* file = (File *) userdata; + + ssize_t ret = my_write(*file, buf, len, MYF(MY_WME)); + posix_fadvise(*file, 0, 0, POSIX_FADV_DONTNEED); + return ret; +} + +static +int +mode_encrypt(File filein, File fileout) +{ + size_t bytesread; + size_t chunkbuflen; + uchar *chunkbuf = NULL; + void *ivbuf = NULL; + size_t encryptbuflen = 0; + size_t encryptedlen = 0; + void *encryptbuf = NULL; + ulonglong ttlchunkswritten = 0; + ulonglong ttlbyteswritten = 0; + xb_wcrypt_t *xbcrypt_file = NULL; + gcry_cipher_hd_t cipher_handle; + gcry_error_t gcry_error; + + if (encrypt_algo != GCRY_CIPHER_NONE) { + gcry_error = gcry_cipher_open(&cipher_handle, + encrypt_algo, + encrypt_mode, 0); + if (gcry_error) { + msg("%s:encrypt: unable to open libgcrypt cipher - " + "%s : %s\n", my_progname, + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + return 1; + } + + gcry_error = gcry_cipher_setkey(cipher_handle, + opt_encrypt_key, + encrypt_key_len); + if (gcry_error) { + msg("%s:encrypt: unable to set libgcrypt cipher key - " + "%s : %s\n", my_progname, + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + goto err; + } + } + + posix_fadvise(filein, 0, 0, POSIX_FADV_SEQUENTIAL); + + xbcrypt_file = xb_crypt_write_open(&fileout, + my_xb_crypt_write_callback); + if (xbcrypt_file == NULL) { + msg("%s:encrypt: xb_crypt_write_open() failed.\n", + my_progname); + goto err; + } + + ivbuf = my_malloc(encrypt_iv_len, MYF(MY_FAE)); + + /* now read in data in chunk size, encrypt and write out */ + chunkbuflen = opt_encrypt_chunk_size + XB_CRYPT_HASH_LEN; + chunkbuf = (uchar *) my_malloc(chunkbuflen, MYF(MY_FAE)); + while ((bytesread = my_read(filein, chunkbuf, opt_encrypt_chunk_size, + MYF(MY_WME))) > 0) { + + size_t origbuflen = bytesread + XB_CRYPT_HASH_LEN; + + /* ensure that XB_CRYPT_HASH_LEN is the correct length + of XB_CRYPT_HASH hashing algorithm output */ + xb_a(XB_CRYPT_HASH_LEN == gcry_md_get_algo_dlen(XB_CRYPT_HASH)); + gcry_md_hash_buffer(XB_CRYPT_HASH, chunkbuf + bytesread, + chunkbuf, bytesread); + + if (encrypt_algo != GCRY_CIPHER_NONE) { + gcry_error = gcry_cipher_reset(cipher_handle); + + if (gcry_error) { + msg("%s:encrypt: unable to reset cipher - " + "%s : %s\n", my_progname, + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + goto err; + } + + xb_crypt_create_iv(ivbuf, encrypt_iv_len); + gcry_error = gcry_cipher_setctr(cipher_handle, + ivbuf, + encrypt_iv_len); + + if (gcry_error) { + msg("%s:encrypt: unable to set cipher iv - " + "%s : %s\n", my_progname, + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + continue; + } + + if (encryptbuflen < origbuflen) { + encryptbuf = my_realloc(encryptbuf, origbuflen, + MYF(MY_WME | MY_ALLOW_ZERO_PTR)); + encryptbuflen = origbuflen; + } + + gcry_error = gcry_cipher_encrypt(cipher_handle, + encryptbuf, + encryptbuflen, + chunkbuf, + origbuflen); + + encryptedlen = origbuflen; + + if (gcry_error) { + msg("%s:encrypt: unable to encrypt chunk - " + "%s : %s\n", my_progname, + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + gcry_cipher_close(cipher_handle); + goto err; + } + } else { + encryptedlen = origbuflen; + encryptbuf = chunkbuf; + } + + if (xb_crypt_write_chunk(xbcrypt_file, encryptbuf, + bytesread + XB_CRYPT_HASH_LEN, + encryptedlen, ivbuf, encrypt_iv_len)) { + msg("%s:encrypt: abcrypt_write_chunk() failed.\n", + my_progname); + goto err; + } + + ttlchunkswritten++; + ttlbyteswritten += encryptedlen; + + if (opt_verbose) + msg("%s:encrypt: %llu chunks written, %llu bytes " + "written\n.", my_progname, ttlchunkswritten, + ttlbyteswritten); + } + + my_free(ivbuf); + my_free(chunkbuf); + + if (encryptbuf && encryptbuflen) + my_free(encryptbuf); + + xb_crypt_write_close(xbcrypt_file); + + if (encrypt_algo != GCRY_CIPHER_NONE) + gcry_cipher_close(cipher_handle); + + if (opt_verbose) + msg("\n%s:encrypt: done\n", my_progname); + + return 0; +err: + if (chunkbuf) + my_free(chunkbuf); + + if (encryptbuf && encryptbuflen) + my_free(encryptbuf); + + if (xbcrypt_file) + xb_crypt_write_close(xbcrypt_file); + + if (encrypt_algo != GCRY_CIPHER_NONE) + gcry_cipher_close(cipher_handle); + + return 1; +} + +static +int +get_options(int *argc, char ***argv) +{ + int ho_error; + + if ((ho_error= handle_options(argc, argv, my_long_options, + get_one_option))) { + exit(EXIT_FAILURE); + } + + return 0; +} + +static +my_bool +get_one_option(int optid, const struct my_option *opt __attribute__((unused)), + char *argument __attribute__((unused))) +{ + switch (optid) { + case 'd': + opt_run_mode = RUN_MODE_DECRYPT; + break; + case '?': + usage(); + exit(0); + } + + return FALSE; +} + +static +void +print_version(void) +{ + printf("%s Ver %s for %s (%s)\n", my_progname, XBCRYPT_VERSION, + SYSTEM_TYPE, MACHINE_TYPE); +} + +static +void +usage(void) +{ + print_version(); + puts("Copyright (C) 2011 Percona Inc."); + puts("This software comes with ABSOLUTELY NO WARRANTY. " + "This is free software,\nand you are welcome to modify and " + "redistribute it under the GPL license.\n"); + + puts("Encrypt or decrypt files in the XBCRYPT format.\n"); + + puts("Usage: "); + printf(" %s [OPTIONS...]" + " # read data from specified input, encrypting or decrypting " + " and writing the result to the specified output.\n", + my_progname); + puts("\nOptions:"); + my_print_help(my_long_options); +} diff --git a/extra/mariabackup/xbcrypt.h b/extra/mariabackup/xbcrypt.h new file mode 100644 index 00000000000..0e832266847 --- /dev/null +++ b/extra/mariabackup/xbcrypt.h @@ -0,0 +1,79 @@ +/****************************************************** +Copyright (c) 2011 Percona LLC and/or its affiliates. + +Encryption interface for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#ifndef XBCRYPT_H +#define XBCRYPT_H + +#include <my_base.h> +#include "common.h" + +#define XB_CRYPT_CHUNK_MAGIC1 "XBCRYP01" +#define XB_CRYPT_CHUNK_MAGIC2 "XBCRYP02" +#define XB_CRYPT_CHUNK_MAGIC3 "XBCRYP03" /* must be same size as ^^ */ +#define XB_CRYPT_CHUNK_MAGIC_CURRENT XB_CRYPT_CHUNK_MAGIC3 +#define XB_CRYPT_CHUNK_MAGIC_SIZE (sizeof(XB_CRYPT_CHUNK_MAGIC1)-1) + +#define XB_CRYPT_HASH GCRY_MD_SHA256 +#define XB_CRYPT_HASH_LEN 32 + +/****************************************************************************** +Write interface */ +typedef struct xb_wcrypt_struct xb_wcrypt_t; + +/* Callback on write for i/o, must return # of bytes written or -1 on error */ +typedef ssize_t xb_crypt_write_callback(void *userdata, + const void *buf, size_t len); + +xb_wcrypt_t *xb_crypt_write_open(void *userdata, + xb_crypt_write_callback *onwrite); + +/* Takes buffer, original length, encrypted length iv and iv length, formats + output buffer and calls write callback. + Returns 0 on success, 1 on error */ +int xb_crypt_write_chunk(xb_wcrypt_t *crypt, const void *buf, size_t olen, + size_t elen, const void *iv, size_t ivlen); + +/* Returns 0 on success, 1 on error */ +int xb_crypt_write_close(xb_wcrypt_t *crypt); + +/****************************************************************************** +Read interface */ +typedef struct xb_rcrypt_struct xb_rcrypt_t; + +/* Callback on read for i/o, must return # of bytes read or -1 on error */ +typedef size_t xb_crypt_read_callback(void *userdata, void *buf, size_t len); + +xb_rcrypt_t *xb_crypt_read_open(void *userdata, + xb_crypt_read_callback *onread); + +typedef enum { + XB_CRYPT_READ_CHUNK, + XB_CRYPT_READ_INCOMPLETE, + XB_CRYPT_READ_EOF, + XB_CRYPT_READ_ERROR +} xb_rcrypt_result_t; + +xb_rcrypt_result_t xb_crypt_read_chunk(xb_rcrypt_t *crypt, void **buf, + size_t *olen, size_t *elen, void **iv, + size_t *ivlen, my_bool *hash_appended); + +int xb_crypt_read_close(xb_rcrypt_t *crypt); + +#endif diff --git a/extra/mariabackup/xbcrypt_common.c b/extra/mariabackup/xbcrypt_common.c new file mode 100644 index 00000000000..0cdb54dc66d --- /dev/null +++ b/extra/mariabackup/xbcrypt_common.c @@ -0,0 +1,328 @@ +/****************************************************** +Copyright (c) 2013, 2017 Percona LLC and/or its affiliates. + +Encryption configuration file interface for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#include <my_base.h> +#include "common.h" +#include "xbcrypt.h" +#include "xbcrypt_common.h" + +/* Encryption options */ +char *ds_encrypt_key = NULL; +char *ds_encrypt_key_file = NULL; +ulong ds_encrypt_algo; + +static uint encrypt_key_len; +static uint encrypt_iv_len; + +static const uint encrypt_mode = GCRY_CIPHER_MODE_CTR; + +static uint encrypt_algos[] = { GCRY_CIPHER_NONE, GCRY_CIPHER_AES128, + GCRY_CIPHER_AES192, GCRY_CIPHER_AES256 }; +static uint encrypt_algo; + +#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600) +GCRY_THREAD_OPTION_PTHREAD_IMPL; +#endif + + +my_bool +xb_crypt_read_key_file(const char *filename, void** key, uint *keylength) +{ + FILE *fp; + + if (!(fp = my_fopen(filename, O_RDONLY, MYF(0)))) { + msg("%s:%s: unable to open config file \"%s\", errno(%d)\n", + my_progname, __FUNCTION__, filename, my_errno); + return FALSE; + } + + fseek(fp, 0 , SEEK_END); + *keylength = ftell(fp); + rewind(fp); + *key = my_malloc(*keylength, MYF(MY_FAE)); + *keylength = fread(*key, 1, *keylength, fp); + my_fclose(fp, MYF(0)); + return TRUE; +} + +void +xb_crypt_create_iv(void* ivbuf, size_t ivlen) +{ + gcry_create_nonce(ivbuf, ivlen); +} + +gcry_error_t +xb_crypt_init(uint *iv_len) +{ + gcry_error_t gcry_error; + + /* Acording to gcrypt docs (and my testing), setting up the threading + callbacks must be done first, so, lets give it a shot */ +#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600) + gcry_error = gcry_control(GCRYCTL_SET_THREAD_CBS, &gcry_threads_pthread); + if (gcry_error) { + msg("encryption: unable to set libgcrypt thread cbs - " + "%s : %s\n", + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + return gcry_error; + } +#endif + + /* Version check should be the very next call because it + makes sure that important subsystems are intialized. */ + if (!gcry_control(GCRYCTL_ANY_INITIALIZATION_P)) { + const char *gcrypt_version; + gcrypt_version = gcry_check_version(NULL); + /* No other library has already initialized libgcrypt. */ + if (!gcrypt_version) { + msg("encryption: failed to initialize libgcrypt\n"); + return 1; + } else { + msg("encryption: using gcrypt %s\n", gcrypt_version); + } + } + + /* Disable the gcry secure memory, not dealing with this for now */ + gcry_error = gcry_control(GCRYCTL_DISABLE_SECMEM, 0); + if (gcry_error) { + msg("encryption: unable to disable libgcrypt secmem - " + "%s : %s\n", + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + return gcry_error; + } + + /* Finalize gcry initialization. */ + gcry_error = gcry_control(GCRYCTL_INITIALIZATION_FINISHED, 0); + if (gcry_error) { + msg("encryption: unable to finish libgcrypt initialization - " + "%s : %s\n", + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + return gcry_error; + } + + /* Determine the algorithm */ + encrypt_algo = encrypt_algos[ds_encrypt_algo]; + + /* Set up the iv length */ + encrypt_iv_len = gcry_cipher_get_algo_blklen(encrypt_algo); + xb_a(encrypt_iv_len > 0); + if (iv_len != NULL) { + *iv_len = encrypt_iv_len; + } + + /* Now set up the key */ + if (ds_encrypt_key == NULL && + ds_encrypt_key_file == NULL) { + msg("encryption: no encryption key or key file specified.\n"); + return gcry_error; + } else if (ds_encrypt_key && ds_encrypt_key_file) { + msg("encryption: both encryption key and key file specified.\n"); + return gcry_error; + } else if (ds_encrypt_key_file) { + if (!xb_crypt_read_key_file(ds_encrypt_key_file, + (void**)&ds_encrypt_key, + &encrypt_key_len)) { + msg("encryption: unable to read encryption key file" + " \"%s\".\n", ds_encrypt_key_file); + return gcry_error; + } + } else if (ds_encrypt_key) { + encrypt_key_len = strlen(ds_encrypt_key); + } else { + msg("encryption: no encryption key or key file specified.\n"); + return gcry_error; + } + + return 0; +} + +gcry_error_t +xb_crypt_cipher_open(gcry_cipher_hd_t *cipher_handle) +{ + if (encrypt_algo != GCRY_CIPHER_NONE) { + gcry_error_t gcry_error; + + gcry_error = gcry_cipher_open(cipher_handle, + encrypt_algo, + encrypt_mode, 0); + if (gcry_error) { + msg("encryption: unable to open libgcrypt" + " cipher - %s : %s\n", + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + gcry_cipher_close(*cipher_handle); + return gcry_error; + } + + gcry_error = gcry_cipher_setkey(*cipher_handle, + ds_encrypt_key, + encrypt_key_len); + if (gcry_error) { + msg("encryption: unable to set libgcrypt" + " cipher key - %s : %s\n", + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + gcry_cipher_close(*cipher_handle); + return gcry_error; + } + return gcry_error; + } + return 0; +} + +void +xb_crypt_cipher_close(gcry_cipher_hd_t cipher_handle) +{ + if (encrypt_algo != GCRY_CIPHER_NONE) + gcry_cipher_close(cipher_handle); +} + +gcry_error_t +xb_crypt_decrypt(gcry_cipher_hd_t cipher_handle, const uchar *from, + size_t from_len, uchar *to, size_t *to_len, + const uchar *iv, size_t iv_len, my_bool hash_appended) +{ + *to_len = from_len; + + if (encrypt_algo != GCRY_CIPHER_NONE) { + + gcry_error_t gcry_error; + + gcry_error = gcry_cipher_reset(cipher_handle); + if (gcry_error) { + msg("%s:encryption: unable to reset libgcrypt" + " cipher - %s : %s\n", my_progname, + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + return gcry_error; + } + + if (iv_len > 0) { + gcry_error = gcry_cipher_setctr(cipher_handle, + iv, iv_len); + } + if (gcry_error) { + msg("%s:encryption: unable to set cipher iv - " + "%s : %s\n", my_progname, + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + return gcry_error; + } + + /* Try to decrypt it */ + gcry_error = gcry_cipher_decrypt(cipher_handle, to, *to_len, + from, from_len); + if (gcry_error) { + msg("%s:encryption: unable to decrypt chunk - " + "%s : %s\n", my_progname, + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + gcry_cipher_close(cipher_handle); + return gcry_error; + } + + if (hash_appended) { + uchar hash[XB_CRYPT_HASH_LEN]; + + *to_len -= XB_CRYPT_HASH_LEN; + + /* ensure that XB_CRYPT_HASH_LEN is the correct length + of XB_CRYPT_HASH hashing algorithm output */ + xb_ad(gcry_md_get_algo_dlen(XB_CRYPT_HASH) == + XB_CRYPT_HASH_LEN); + gcry_md_hash_buffer(XB_CRYPT_HASH, hash, to, + *to_len); + if (memcmp(hash, (char *) to + *to_len, + XB_CRYPT_HASH_LEN) != 0) { + msg("%s:%s invalid plaintext hash. " + "Wrong encrytion key specified?\n", + my_progname, __FUNCTION__); + return 1; + } + } + + } else { + memcpy(to, from, *to_len); + } + + return 0; +} + +gcry_error_t +xb_crypt_encrypt(gcry_cipher_hd_t cipher_handle, const uchar *from, + size_t from_len, uchar *to, size_t *to_len, uchar *iv) +{ + gcry_error_t gcry_error; + + /* ensure that XB_CRYPT_HASH_LEN is the correct length + of XB_CRYPT_HASH hashing algorithm output */ + xb_ad(gcry_md_get_algo_dlen(XB_CRYPT_HASH) == + XB_CRYPT_HASH_LEN); + + memcpy(to, from, from_len); + gcry_md_hash_buffer(XB_CRYPT_HASH, to + from_len, + from, from_len); + + *to_len = from_len; + + if (encrypt_algo != GCRY_CIPHER_NONE) { + + gcry_error = gcry_cipher_reset(cipher_handle); + if (gcry_error) { + msg("encrypt: unable to reset cipher - " + "%s : %s\n", + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + return gcry_error; + } + + xb_crypt_create_iv(iv, encrypt_iv_len); + gcry_error = gcry_cipher_setctr(cipher_handle, iv, + encrypt_iv_len); + if (gcry_error) { + msg("encrypt: unable to set cipher ctr - " + "%s : %s\n", + gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + return gcry_error; + } + + gcry_error = gcry_cipher_encrypt(cipher_handle, to, + *to_len + XB_CRYPT_HASH_LEN, + to, + from_len + XB_CRYPT_HASH_LEN); + if (gcry_error) { + msg("encrypt: unable to encrypt buffer - " + "%s : %s\n", gcry_strsource(gcry_error), + gcry_strerror(gcry_error)); + return gcry_error; + } + } else { + memcpy(to, from, from_len + XB_CRYPT_HASH_LEN); + } + + *to_len += XB_CRYPT_HASH_LEN; + + return 0; +} +#endif
\ No newline at end of file diff --git a/extra/mariabackup/xbcrypt_common.h b/extra/mariabackup/xbcrypt_common.h new file mode 100644 index 00000000000..85d13c01fc4 --- /dev/null +++ b/extra/mariabackup/xbcrypt_common.h @@ -0,0 +1,64 @@ +/****************************************************** +Copyright (c) 2017 Percona LLC and/or its affiliates. + +Encryption datasink implementation for XtraBackup. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#include <my_base.h> +#if HAVE_GCRYPT +#if GCC_VERSION >= 4002 +/* Workaround to avoid "gcry_ac_* is deprecated" warnings in gcrypt.h */ +# pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + +#include <gcrypt.h> + +extern char *ds_encrypt_key; +extern char *ds_encrypt_key_file; +extern int ds_encrypt_threads; +extern ulong ds_encrypt_algo; + +/****************************************************************************** +Utility interface */ +my_bool xb_crypt_read_key_file(const char *filename, + void** key, uint *keylength); + +void xb_crypt_create_iv(void* ivbuf, size_t ivlen); + +/* Initialize gcrypt and setup encryption key and IV lengths */ +gcry_error_t +xb_crypt_init(uint *iv_len); + +/* Setup gcrypt cipher */ +gcry_error_t +xb_crypt_cipher_open(gcry_cipher_hd_t *cipher_handle); + +/* Close gcrypt cipher */ +void +xb_crypt_cipher_close(gcry_cipher_hd_t cipher_handle); + +/* Decrypt buffer */ +gcry_error_t +xb_crypt_decrypt(gcry_cipher_hd_t cipher_handle, const uchar *from, + size_t from_len, uchar *to, size_t *to_len, const uchar *iv, + size_t iv_len, my_bool hash_appended); + +/* Encrypt buffer */ +gcry_error_t +xb_crypt_encrypt(gcry_cipher_hd_t cipher_handle, const uchar *from, + size_t from_len, uchar *to, size_t *to_len, uchar *iv); +#endif diff --git a/extra/mariabackup/xbcrypt_read.c b/extra/mariabackup/xbcrypt_read.c new file mode 100644 index 00000000000..41790c7035d --- /dev/null +++ b/extra/mariabackup/xbcrypt_read.c @@ -0,0 +1,252 @@ +/****************************************************** +Copyright (c) 2013 Percona LLC and/or its affiliates. + +The xbcrypt format reader implementation. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#include "xbcrypt.h" +#include "crc_glue.h" + +struct xb_rcrypt_struct { + void *userdata; + xb_crypt_read_callback *read; + void *buffer; + size_t bufsize; + void *ivbuffer; + size_t ivbufsize; + ulonglong offset; +}; + +xb_rcrypt_t * +xb_crypt_read_open(void *userdata, xb_crypt_read_callback *onread) +{ + xb_rcrypt_t *crypt; + + xb_ad(onread); + + crypt = (xb_rcrypt_t *) my_malloc(sizeof(xb_rcrypt_t), MYF(MY_FAE)); + + crypt->userdata = userdata; + crypt->read = onread; + crypt->buffer = NULL; + crypt->bufsize = 0; + crypt->offset = 0; + crypt->ivbuffer = NULL; + crypt->ivbufsize = 0; + return crypt; +} + +xb_rcrypt_result_t +xb_crypt_read_chunk(xb_rcrypt_t *crypt, void **buf, size_t *olen, size_t *elen, + void **iv, size_t *ivlen, my_bool *hash_appended) + +{ + uchar tmpbuf[XB_CRYPT_CHUNK_MAGIC_SIZE + 8 + 8 + 8 + 4]; + uchar *ptr; + ulonglong tmp; + ulong checksum, checksum_exp, version; + size_t bytesread; + xb_rcrypt_result_t result = XB_CRYPT_READ_CHUNK; + + if ((bytesread = crypt->read(crypt->userdata, tmpbuf, sizeof(tmpbuf))) + != sizeof(tmpbuf)) { + if (bytesread == 0) { + result = XB_CRYPT_READ_EOF; + goto err; + } else { + msg("%s:%s: unable to read chunk header data at " + "offset 0x%llx.\n", + my_progname, __FUNCTION__, crypt->offset); + result = XB_CRYPT_READ_ERROR; + goto err; + } + } + + ptr = tmpbuf; + + if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC3, + XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) { + version = 3; + } else if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC2, + XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) { + version = 2; + } else if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC1, + XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) { + version = 1; + } else { + msg("%s:%s: wrong chunk magic at offset 0x%llx.\n", + my_progname, __FUNCTION__, crypt->offset); + result = XB_CRYPT_READ_ERROR; + goto err; + } + + ptr += XB_CRYPT_CHUNK_MAGIC_SIZE; + crypt->offset += XB_CRYPT_CHUNK_MAGIC_SIZE; + + tmp = uint8korr(ptr); /* reserved */ + ptr += 8; + crypt->offset += 8; + + tmp = uint8korr(ptr); /* original size */ + ptr += 8; + if (tmp > INT_MAX) { + msg("%s:%s: invalid original size at offset 0x%llx.\n", + my_progname, __FUNCTION__, crypt->offset); + result = XB_CRYPT_READ_ERROR; + goto err; + } + crypt->offset += 8; + *olen = (size_t)tmp; + + tmp = uint8korr(ptr); /* encrypted size */ + ptr += 8; + if (tmp > INT_MAX) { + msg("%s:%s: invalid encrypted size at offset 0x%llx.\n", + my_progname, __FUNCTION__, crypt->offset); + result = XB_CRYPT_READ_ERROR; + goto err; + } + crypt->offset += 8; + *elen = (size_t)tmp; + + checksum_exp = uint4korr(ptr); /* checksum */ + ptr += 4; + crypt->offset += 4; + + /* iv size */ + if (version == 1) { + *ivlen = 0; + *iv = 0; + } else { + if ((bytesread = crypt->read(crypt->userdata, tmpbuf, 8)) + != 8) { + if (bytesread == 0) { + result = XB_CRYPT_READ_EOF; + goto err; + } else { + msg("%s:%s: unable to read chunk iv size at " + "offset 0x%llx.\n", + my_progname, __FUNCTION__, crypt->offset); + result = XB_CRYPT_READ_ERROR; + goto err; + } + } + + tmp = uint8korr(tmpbuf); + if (tmp > INT_MAX) { + msg("%s:%s: invalid iv size at offset 0x%llx.\n", + my_progname, __FUNCTION__, crypt->offset); + result = XB_CRYPT_READ_ERROR; + goto err; + } + crypt->offset += 8; + *ivlen = (size_t)tmp; + } + + if (*ivlen > crypt->ivbufsize) { + crypt->ivbuffer = my_realloc(crypt->ivbuffer, *ivlen, + MYF(MY_WME | MY_ALLOW_ZERO_PTR)); + if (crypt->ivbuffer == NULL) { + msg("%s:%s: failed to increase iv buffer to " + "%llu bytes.\n", my_progname, __FUNCTION__, + (ulonglong)*ivlen); + result = XB_CRYPT_READ_ERROR; + goto err; + } + crypt->ivbufsize = *ivlen; + } + + if (*ivlen > 0) { + if (crypt->read(crypt->userdata, crypt->ivbuffer, *ivlen) + != *ivlen) { + msg("%s:%s: failed to read %lld bytes for chunk iv " + "at offset 0x%llx.\n", my_progname, __FUNCTION__, + (ulonglong)*ivlen, crypt->offset); + result = XB_CRYPT_READ_ERROR; + goto err; + } + *iv = crypt->ivbuffer; + } + + /* for version euqals 2 we need to read in the iv data but do not init + CTR with it */ + if (version == 2) { + *ivlen = 0; + *iv = 0; + } + + if (*olen > crypt->bufsize) { + crypt->buffer = my_realloc(crypt->buffer, *olen, + MYF(MY_WME | MY_ALLOW_ZERO_PTR)); + if (crypt->buffer == NULL) { + msg("%s:%s: failed to increase buffer to " + "%llu bytes.\n", my_progname, __FUNCTION__, + (ulonglong)*olen); + result = XB_CRYPT_READ_ERROR; + goto err; + } + crypt->bufsize = *olen; + } + + if (*elen > 0) { + if (crypt->read(crypt->userdata, crypt->buffer, *elen) + != *elen) { + msg("%s:%s: failed to read %lld bytes for chunk payload " + "at offset 0x%llx.\n", my_progname, __FUNCTION__, + (ulonglong)*elen, crypt->offset); + result = XB_CRYPT_READ_ERROR; + goto err; + } + } + + checksum = crc32_iso3309(0, crypt->buffer, *elen); + if (checksum != checksum_exp) { + msg("%s:%s invalid checksum at offset 0x%llx, " + "expected 0x%lx, actual 0x%lx.\n", my_progname, __FUNCTION__, + crypt->offset, checksum_exp, checksum); + result = XB_CRYPT_READ_ERROR; + goto err; + } + + crypt->offset += *elen; + *buf = crypt->buffer; + + *hash_appended = version > 2; + + goto exit; + +err: + *buf = NULL; + *olen = 0; + *elen = 0; + *ivlen = 0; + *iv = 0; +exit: + return result; +} + +int xb_crypt_read_close(xb_rcrypt_t *crypt) +{ + if (crypt->buffer) + my_free(crypt->buffer); + if (crypt->ivbuffer) + my_free(crypt->ivbuffer); + my_free(crypt); + + return 0; +} + diff --git a/extra/mariabackup/xbcrypt_write.c b/extra/mariabackup/xbcrypt_write.c new file mode 100644 index 00000000000..91dbfc4eb29 --- /dev/null +++ b/extra/mariabackup/xbcrypt_write.c @@ -0,0 +1,105 @@ +/****************************************************** +Copyright (c) 2013 Percona LLC and/or its affiliates. + +The xbcrypt format writer implementation. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#include "xbcrypt.h" +#include "crc_glue.h" + +struct xb_wcrypt_struct { + void *userdata; + xb_crypt_write_callback *write; +}; + +xb_wcrypt_t * +xb_crypt_write_open(void *userdata, xb_crypt_write_callback *onwrite) +{ + xb_wcrypt_t *crypt; + + xb_ad(onwrite); + + crypt = (xb_wcrypt_t *) my_malloc(sizeof(xb_wcrypt_t), MYF(MY_FAE)); + + crypt->userdata = userdata; + crypt->write = onwrite; + + return crypt; +} + +int xb_crypt_write_chunk(xb_wcrypt_t *crypt, const void *buf, size_t olen, + size_t elen, const void *iv, size_t ivlen) +{ + uchar tmpbuf[XB_CRYPT_CHUNK_MAGIC_SIZE + 8 + 8 + 8 + 4 + 8]; + uchar *ptr; + ulong checksum; + + xb_ad(olen <= INT_MAX); + if (olen > INT_MAX) + return 0; + + xb_ad(elen <= INT_MAX); + if (elen > INT_MAX) + return 0; + + xb_ad(ivlen <= INT_MAX); + if (ivlen > INT_MAX) + return 0; + + ptr = tmpbuf; + + memcpy(ptr, XB_CRYPT_CHUNK_MAGIC_CURRENT, XB_CRYPT_CHUNK_MAGIC_SIZE); + ptr += XB_CRYPT_CHUNK_MAGIC_SIZE; + + int8store(ptr, (ulonglong)0); /* reserved */ + ptr += 8; + + int8store(ptr, (ulonglong)olen); /* original size */ + ptr += 8; + + int8store(ptr, (ulonglong)elen); /* encrypted (actual) size */ + ptr += 8; + + checksum = crc32_iso3309(0, buf, elen); + int4store(ptr, checksum); /* checksum */ + ptr += 4; + + int8store(ptr, (ulonglong)ivlen); /* iv size */ + ptr += 8; + + xb_ad(ptr <= tmpbuf + sizeof(tmpbuf)); + + if (crypt->write(crypt->userdata, tmpbuf, ptr-tmpbuf) == -1) + return 1; + + if (crypt->write(crypt->userdata, iv, ivlen) == -1) + return 1; + + if (crypt->write(crypt->userdata, buf, elen) == -1) + return 1; + + return 0; +} + +int xb_crypt_write_close(xb_wcrypt_t *crypt) +{ + my_free(crypt); + + return 0; +} + + diff --git a/extra/mariabackup/xbstream.c b/extra/mariabackup/xbstream.c new file mode 100644 index 00000000000..2cc47ec7273 --- /dev/null +++ b/extra/mariabackup/xbstream.c @@ -0,0 +1,613 @@ +/****************************************************** +Copyright (c) 2011-2013 Percona LLC and/or its affiliates. + +The xbstream utility: serialize/deserialize files in the XBSTREAM format. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#include <mysql_version.h> +#include <my_base.h> +#include <my_getopt.h> +#include <hash.h> +#include <my_pthread.h> +#include "common.h" +#include "xbstream.h" +#include "xbcrypt_common.h" +#include "datasink.h" +#include "ds_decrypt.h" +#include "crc_glue.h" + +#define XBSTREAM_VERSION "1.0" +#define XBSTREAM_BUFFER_SIZE (10 * 1024 * 1024UL) + +#define START_FILE_HASH_SIZE 16 + +typedef enum { + RUN_MODE_NONE, + RUN_MODE_CREATE, + RUN_MODE_EXTRACT +} run_mode_t; + +const char *xbstream_encrypt_algo_names[] = +{ "NONE", "AES128", "AES192", "AES256", NullS}; +TYPELIB xbstream_encrypt_algo_typelib= +{array_elements(xbstream_encrypt_algo_names)-1,"", + xbstream_encrypt_algo_names, NULL}; + +/* Need the following definitions to avoid linking with ds_*.o and their link +dependencies */ +datasink_t datasink_archive; +datasink_t datasink_xbstream; +datasink_t datasink_compress; +datasink_t datasink_tmpfile; +datasink_t datasink_encrypt; +datasink_t datasink_buffer; + +static run_mode_t opt_mode; +static char * opt_directory = NULL; +static my_bool opt_verbose = 0; +static int opt_parallel = 1; +static ulong opt_encrypt_algo; +static char *opt_encrypt_key_file = NULL; +static void *opt_encrypt_key = NULL; +static int opt_encrypt_threads = 1; + +enum { + OPT_ENCRYPT_THREADS = 256 +}; + +static struct my_option my_long_options[] = +{ + {"help", '?', "Display this help and exit.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"create", 'c', "Stream the specified files to the standard output.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"extract", 'x', "Extract to disk files from the stream on the " + "standard input.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"directory", 'C', "Change the current directory to the specified one " + "before streaming or extracting.", &opt_directory, &opt_directory, 0, + GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"verbose", 'v', "Print verbose output.", &opt_verbose, &opt_verbose, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"parallel", 'p', "Number of worker threads for reading / writing.", + &opt_parallel, &opt_parallel, 0, GET_INT, REQUIRED_ARG, + 1, 1, INT_MAX, 0, 0, 0}, + {"decrypt", 'd', "Decrypt files ending with .xbcrypt.", + &opt_encrypt_algo, &opt_encrypt_algo, &xbstream_encrypt_algo_typelib, + GET_ENUM, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"encrypt-key", 'k', "Encryption key.", + &opt_encrypt_key, &opt_encrypt_key, 0, + GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"encrypt-key-file", 'f', "File which contains encryption key.", + &opt_encrypt_key_file, &opt_encrypt_key_file, 0, + GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"encrypt-threads", OPT_ENCRYPT_THREADS, + "Number of threads for parallel data encryption. " + "The default value is 1.", + &opt_encrypt_threads, &opt_encrypt_threads, + 0, GET_INT, REQUIRED_ARG, 1, 1, INT_MAX, 0, 0, 0}, + + {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} +}; + +typedef struct { + HASH *filehash; + xb_rstream_t *stream; + ds_ctxt_t *ds_ctxt; + ds_ctxt_t *ds_decrypt_ctxt; + pthread_mutex_t *mutex; +} extract_ctxt_t; + +typedef struct { + char *path; + uint pathlen; + my_off_t offset; + ds_file_t *file; + pthread_mutex_t mutex; +} file_entry_t; + +static int get_options(int *argc, char ***argv); +static int mode_create(int argc, char **argv); +static int mode_extract(int n_threads, int argc, char **argv); +static my_bool get_one_option(int optid, const struct my_option *opt, + char *argument); + +int +main(int argc, char **argv) +{ + MY_INIT(argv[0]); + + crc_init(); + + if (get_options(&argc, &argv)) { + goto err; + } + + if (opt_mode == RUN_MODE_NONE) { + msg("%s: either -c or -x must be specified.\n", my_progname); + goto err; + } + + /* Change the current directory if -C is specified */ + if (opt_directory && my_setwd(opt_directory, MYF(MY_WME))) { + goto err; + } + + if (opt_mode == RUN_MODE_CREATE && mode_create(argc, argv)) { + goto err; + } else if (opt_mode == RUN_MODE_EXTRACT && + mode_extract(opt_parallel, argc, argv)) { + goto err; + } + + my_cleanup_options(my_long_options); + + my_end(0); + + return EXIT_SUCCESS; +err: + my_cleanup_options(my_long_options); + + my_end(0); + + exit(EXIT_FAILURE); +} + +static +int +get_options(int *argc, char ***argv) +{ + int ho_error; + + if ((ho_error= handle_options(argc, argv, my_long_options, + get_one_option))) { + exit(EXIT_FAILURE); + } + + return 0; +} + +static +void +print_version(void) +{ + printf("%s Ver %s for %s (%s)\n", my_progname, XBSTREAM_VERSION, + SYSTEM_TYPE, MACHINE_TYPE); +} + +static +void +usage(void) +{ + print_version(); + puts("Copyright (C) 2011-2013 Percona LLC and/or its affiliates."); + puts("This software comes with ABSOLUTELY NO WARRANTY. " + "This is free software,\nand you are welcome to modify and " + "redistribute it under the GPL license.\n"); + + puts("Serialize/deserialize files in the XBSTREAM format.\n"); + + puts("Usage: "); + printf(" %s -c [OPTIONS...] FILES... # stream specified files to " + "standard output.\n", my_progname); + printf(" %s -x [OPTIONS...] # extract files from the stream" + "on the standard input.\n", my_progname); + + puts("\nOptions:"); + my_print_help(my_long_options); +} + +static +int +set_run_mode(run_mode_t mode) +{ + if (opt_mode != RUN_MODE_NONE) { + msg("%s: can't set specify both -c and -x.\n", my_progname); + return 1; + } + + opt_mode = mode; + + return 0; +} + +static +my_bool +get_one_option(int optid, const struct my_option *opt __attribute__((unused)), + char *argument __attribute__((unused))) +{ + switch (optid) { + case 'c': + if (set_run_mode(RUN_MODE_CREATE)) { + return TRUE; + } + break; + case 'x': + if (set_run_mode(RUN_MODE_EXTRACT)) { + return TRUE; + } + break; + case '?': + usage(); + exit(0); + } + + return FALSE; +} + +static +int +stream_one_file(File file, xb_wstream_file_t *xbfile) +{ + uchar *buf; + ssize_t bytes; + my_off_t offset; + + posix_fadvise(file, 0, 0, POSIX_FADV_SEQUENTIAL); + offset = my_tell(file, MYF(MY_WME)); + + buf = (uchar*)(my_malloc(XBSTREAM_BUFFER_SIZE, MYF(MY_FAE))); + + while ((bytes = (ssize_t)my_read(file, buf, XBSTREAM_BUFFER_SIZE, + MYF(MY_WME))) > 0) { + if (xb_stream_write_data(xbfile, buf, bytes)) { + msg("%s: xb_stream_write_data() failed.\n", + my_progname); + my_free(buf); + return 1; + } + posix_fadvise(file, offset, XBSTREAM_BUFFER_SIZE, + POSIX_FADV_DONTNEED); + offset += XBSTREAM_BUFFER_SIZE; + + } + + my_free(buf); + + if (bytes < 0) { + return 1; + } + + return 0; +} + +static +int +mode_create(int argc, char **argv) +{ + int i; + MY_STAT mystat; + xb_wstream_t *stream; + + if (argc < 1) { + msg("%s: no files are specified.\n", my_progname); + return 1; + } + + stream = xb_stream_write_new(); + if (stream == NULL) { + msg("%s: xb_stream_write_new() failed.\n", my_progname); + return 1; + } + + for (i = 0; i < argc; i++) { + char *filepath = argv[i]; + File src_file; + xb_wstream_file_t *file; + + if (my_stat(filepath, &mystat, MYF(MY_WME)) == NULL) { + goto err; + } + if (!MY_S_ISREG(mystat.st_mode)) { + msg("%s: %s is not a regular file, exiting.\n", + my_progname, filepath); + goto err; + } + + if ((src_file = my_open(filepath, O_RDONLY, MYF(MY_WME))) < 0) { + msg("%s: failed to open %s.\n", my_progname, filepath); + goto err; + } + + file = xb_stream_write_open(stream, filepath, &mystat, NULL, NULL); + if (file == NULL) { + goto err; + } + + if (opt_verbose) { + msg("%s\n", filepath); + } + + if (stream_one_file(src_file, file) || + xb_stream_write_close(file) || + my_close(src_file, MYF(MY_WME))) { + goto err; + } + } + + xb_stream_write_done(stream); + + return 0; +err: + xb_stream_write_done(stream); + + return 1; +} + +/************************************************************************ +Check if string ends with given suffix. +@return true if string ends with given suffix. */ +static +my_bool +ends_with(const char *str, const char *suffix) +{ + size_t suffix_len = strlen(suffix); + size_t str_len = strlen(str); + return(str_len >= suffix_len + && strcmp(str + str_len - suffix_len, suffix) == 0); +} + +static +file_entry_t * +file_entry_new(extract_ctxt_t *ctxt, const char *path, uint pathlen) +{ + file_entry_t *entry; + ds_file_t *file; + + entry = (file_entry_t *) my_malloc(sizeof(file_entry_t), + MYF(MY_WME | MY_ZEROFILL)); + if (entry == NULL) { + return NULL; + } + + entry->path = my_strndup(path, pathlen, MYF(MY_WME)); + if (entry->path == NULL) { + goto err; + } + entry->pathlen = pathlen; + + if (ctxt->ds_decrypt_ctxt && ends_with(path, ".xbcrypt")) { + file = ds_open(ctxt->ds_decrypt_ctxt, path, NULL); + } else { + file = ds_open(ctxt->ds_ctxt, path, NULL); + } + if (file == NULL) { + msg("%s: failed to create file.\n", my_progname); + goto err; + } + + if (opt_verbose) { + msg("%s\n", entry->path); + } + + entry->file = file; + + pthread_mutex_init(&entry->mutex, NULL); + + return entry; + +err: + if (entry->path != NULL) { + my_free(entry->path); + } + my_free(entry); + + return NULL; +} + +static +uchar * +get_file_entry_key(file_entry_t *entry, size_t *length, + my_bool not_used __attribute__((unused))) +{ + *length = entry->pathlen; + return (uchar *) entry->path; +} + +static +void +file_entry_free(file_entry_t *entry) +{ + pthread_mutex_destroy(&entry->mutex); + ds_close(entry->file); + my_free(entry->path); + my_free(entry); +} + +static +void * +extract_worker_thread_func(void *arg) +{ + xb_rstream_chunk_t chunk; + file_entry_t *entry; + xb_rstream_result_t res; + + extract_ctxt_t *ctxt = (extract_ctxt_t *) arg; + + my_thread_init(); + + memset(&chunk, 0, sizeof(chunk)); + + while (1) { + + pthread_mutex_lock(ctxt->mutex); + res = xb_stream_read_chunk(ctxt->stream, &chunk); + + if (res != XB_STREAM_READ_CHUNK) { + pthread_mutex_unlock(ctxt->mutex); + break; + } + + /* If unknown type and ignorable flag is set, skip this chunk */ + if (chunk.type == XB_CHUNK_TYPE_UNKNOWN && \ + !(chunk.flags & XB_STREAM_FLAG_IGNORABLE)) { + pthread_mutex_unlock(ctxt->mutex); + continue; + } + + /* See if we already have this file open */ + entry = (file_entry_t *) my_hash_search(ctxt->filehash, + (uchar *) chunk.path, + chunk.pathlen); + + if (entry == NULL) { + entry = file_entry_new(ctxt, + chunk.path, + chunk.pathlen); + if (entry == NULL) { + pthread_mutex_unlock(ctxt->mutex); + break; + } + if (my_hash_insert(ctxt->filehash, (uchar *) entry)) { + msg("%s: my_hash_insert() failed.\n", + my_progname); + pthread_mutex_unlock(ctxt->mutex); + break; + } + } + + pthread_mutex_lock(&entry->mutex); + + pthread_mutex_unlock(ctxt->mutex); + + res = xb_stream_validate_checksum(&chunk); + + if (res != XB_STREAM_READ_CHUNK) { + pthread_mutex_unlock(&entry->mutex); + break; + } + + if (chunk.type == XB_CHUNK_TYPE_EOF) { + pthread_mutex_unlock(&entry->mutex); + continue; + } + + if (entry->offset != chunk.offset) { + msg("%s: out-of-order chunk: real offset = 0x%llx, " + "expected offset = 0x%llx\n", my_progname, + chunk.offset, entry->offset); + pthread_mutex_unlock(&entry->mutex); + res = XB_STREAM_READ_ERROR; + break; + } + + if (ds_write(entry->file, chunk.data, chunk.length)) { + msg("%s: my_write() failed.\n", my_progname); + pthread_mutex_unlock(&entry->mutex); + res = XB_STREAM_READ_ERROR; + break; + } + + entry->offset += chunk.length; + + pthread_mutex_unlock(&entry->mutex); + } + + if (chunk.data) + my_free(chunk.data); + + my_thread_end(); + + return (void *)(res); +} + + +static +int +mode_extract(int n_threads, int argc __attribute__((unused)), + char **argv __attribute__((unused))) +{ + xb_rstream_t *stream = NULL; + HASH filehash; + ds_ctxt_t *ds_ctxt = NULL; + ds_ctxt_t *ds_decrypt_ctxt = NULL; + extract_ctxt_t ctxt; + int i; + pthread_t *tids = NULL; + void **retvals = NULL; + pthread_mutex_t mutex; + int ret = 0; + + if (my_hash_init(&filehash, &my_charset_bin, START_FILE_HASH_SIZE, + 0, 0, (my_hash_get_key) get_file_entry_key, + (my_hash_free_key) file_entry_free, MYF(0))) { + msg("%s: failed to initialize file hash.\n", my_progname); + return 1; + } + + if (pthread_mutex_init(&mutex, NULL)) { + msg("%s: failed to initialize mutex.\n", my_progname); + my_hash_free(&filehash); + return 1; + } + + /* If --directory is specified, it is already set as CWD by now. */ + ds_ctxt = ds_create(".", DS_TYPE_LOCAL); + if (ds_ctxt == NULL) { + ret = 1; + goto exit; + } + + + stream = xb_stream_read_new(); + if (stream == NULL) { + msg("%s: xb_stream_read_new() failed.\n", my_progname); + pthread_mutex_destroy(&mutex); + ret = 1; + goto exit; + } + + ctxt.stream = stream; + ctxt.filehash = &filehash; + ctxt.ds_ctxt = ds_ctxt; + ctxt.ds_decrypt_ctxt = ds_decrypt_ctxt; + ctxt.mutex = &mutex; + + tids = malloc(sizeof(pthread_t) * n_threads); + retvals = malloc(sizeof(void*) * n_threads); + + for (i = 0; i < n_threads; i++) + pthread_create(tids + i, NULL, extract_worker_thread_func, + &ctxt); + + for (i = 0; i < n_threads; i++) + pthread_join(tids[i], retvals + i); + + for (i = 0; i < n_threads; i++) { + if ((ulong)retvals[i] == XB_STREAM_READ_ERROR) { + ret = 1; + goto exit; + } + } + +exit: + pthread_mutex_destroy(&mutex); + + free(tids); + free(retvals); + + my_hash_free(&filehash); + if (ds_ctxt != NULL) { + ds_destroy(ds_ctxt); + } + if (ds_decrypt_ctxt) { + ds_destroy(ds_decrypt_ctxt); + } + xb_stream_read_done(stream); + + return ret; +} diff --git a/extra/mariabackup/xbstream.h b/extra/mariabackup/xbstream.h new file mode 100644 index 00000000000..ac1bf05e321 --- /dev/null +++ b/extra/mariabackup/xbstream.h @@ -0,0 +1,107 @@ +/****************************************************** +Copyright (c) 2011-2017 Percona LLC and/or its affiliates. + +The xbstream format interface. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#ifndef XBSTREAM_H +#define XBSTREAM_H + +#include <my_base.h> + +/* Magic value in a chunk header */ +#define XB_STREAM_CHUNK_MAGIC "XBSTCK01" + +/* Chunk flags */ +/* Chunk can be ignored if unknown version/format */ +#define XB_STREAM_FLAG_IGNORABLE 0x01 + +/* Magic + flags + type + path len */ +#define CHUNK_HEADER_CONSTANT_LEN ((sizeof(XB_STREAM_CHUNK_MAGIC) - 1) + \ + 1 + 1 + 4) +#define CHUNK_TYPE_OFFSET (sizeof(XB_STREAM_CHUNK_MAGIC) - 1 + 1) +#define PATH_LENGTH_OFFSET (sizeof(XB_STREAM_CHUNK_MAGIC) - 1 + 1 + 1) + +typedef struct xb_wstream_struct xb_wstream_t; + +typedef struct xb_wstream_file_struct xb_wstream_file_t; + +typedef enum { + XB_STREAM_FMT_NONE, + XB_STREAM_FMT_TAR, + XB_STREAM_FMT_XBSTREAM +} xb_stream_fmt_t; + +/************************************************************************ +Write interface. */ + +typedef ssize_t xb_stream_write_callback(xb_wstream_file_t *file, + void *userdata, + const void *buf, size_t len); + +xb_wstream_t *xb_stream_write_new(void); + +xb_wstream_file_t *xb_stream_write_open(xb_wstream_t *stream, const char *path, + MY_STAT *mystat, void *userdata, + xb_stream_write_callback *onwrite); + +int xb_stream_write_data(xb_wstream_file_t *file, const void *buf, size_t len); + +int xb_stream_write_close(xb_wstream_file_t *file); + +int xb_stream_write_done(xb_wstream_t *stream); + +/************************************************************************ +Read interface. */ + +typedef enum { + XB_STREAM_READ_CHUNK, + XB_STREAM_READ_EOF, + XB_STREAM_READ_ERROR +} xb_rstream_result_t; + +typedef enum { + XB_CHUNK_TYPE_UNKNOWN = '\0', + XB_CHUNK_TYPE_PAYLOAD = 'P', + XB_CHUNK_TYPE_EOF = 'E' +} xb_chunk_type_t; + +typedef struct xb_rstream_struct xb_rstream_t; + +typedef struct { + uchar flags; + xb_chunk_type_t type; + uint pathlen; + char path[FN_REFLEN]; + size_t length; + my_off_t offset; + my_off_t checksum_offset; + void *data; + ulong checksum; + size_t buflen; +} xb_rstream_chunk_t; + +xb_rstream_t *xb_stream_read_new(void); + +xb_rstream_result_t xb_stream_read_chunk(xb_rstream_t *stream, + xb_rstream_chunk_t *chunk); + +int xb_stream_read_done(xb_rstream_t *stream); + +int xb_stream_validate_checksum(xb_rstream_chunk_t *chunk); + +#endif diff --git a/extra/mariabackup/xbstream_read.c b/extra/mariabackup/xbstream_read.c new file mode 100644 index 00000000000..8d19242301b --- /dev/null +++ b/extra/mariabackup/xbstream_read.c @@ -0,0 +1,228 @@ +/****************************************************** +Copyright (c) 2011-2017 Percona LLC and/or its affiliates. + +The xbstream format reader implementation. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#include <mysql_version.h> +#include <my_base.h> +#include <zlib.h> +#include "common.h" +#include "xbstream.h" +#include "crc_glue.h" + +/* Allocate 1 MB for the payload buffer initially */ +#define INIT_BUFFER_LEN (1024 * 1024) + +#ifndef MY_OFF_T_MAX +#define MY_OFF_T_MAX (~(my_off_t)0UL) +#endif + +struct xb_rstream_struct { + my_off_t offset; + File fd; +}; + +xb_rstream_t * +xb_stream_read_new(void) +{ + xb_rstream_t *stream; + + stream = (xb_rstream_t *) my_malloc(sizeof(xb_rstream_t), MYF(MY_FAE)); + +#ifdef __WIN__ + setmode(fileno(stdin), _O_BINARY); +#endif + + stream->fd = my_fileno(stdin); + stream->offset = 0; + + return stream; +} + +static inline +xb_chunk_type_t +validate_chunk_type(uchar code) +{ + switch ((xb_chunk_type_t) code) { + case XB_CHUNK_TYPE_PAYLOAD: + case XB_CHUNK_TYPE_EOF: + return (xb_chunk_type_t) code; + default: + return XB_CHUNK_TYPE_UNKNOWN; + } +} + +int +xb_stream_validate_checksum(xb_rstream_chunk_t *chunk) +{ + ulong checksum; + + checksum = crc32_iso3309(0, chunk->data, (uint)chunk->length); + if (checksum != chunk->checksum) { + msg("xb_stream_read_chunk(): invalid checksum at offset " + "0x%llx: expected 0x%lx, read 0x%lx.\n", + (ulonglong) chunk->checksum_offset, chunk->checksum, + checksum); + return XB_STREAM_READ_ERROR; + } + + return XB_STREAM_READ_CHUNK; +} + +#define F_READ(buf,len) \ + do { \ + if (xb_read_full(fd, buf, len) < len) { \ + msg("xb_stream_read_chunk(): my_read() failed.\n"); \ + goto err; \ + } \ + } while (0) + +xb_rstream_result_t +xb_stream_read_chunk(xb_rstream_t *stream, xb_rstream_chunk_t *chunk) +{ + uchar tmpbuf[16]; + uchar *ptr = tmpbuf; + uint pathlen; + size_t tbytes; + ulonglong ullval; + File fd = stream->fd; + + xb_ad(sizeof(tmpbuf) >= CHUNK_HEADER_CONSTANT_LEN); + + /* This is the only place where we expect EOF, so read with + xb_read_full() rather than F_READ() */ + tbytes = xb_read_full(fd, ptr, CHUNK_HEADER_CONSTANT_LEN); + if (tbytes == 0) { + return XB_STREAM_READ_EOF; + } else if (tbytes < CHUNK_HEADER_CONSTANT_LEN) { + msg("xb_stream_read_chunk(): unexpected end of stream at " + "offset 0x%llx.\n", stream->offset); + goto err; + } + + ptr = tmpbuf; + + /* Chunk magic value */ + if (memcmp(tmpbuf, XB_STREAM_CHUNK_MAGIC, 8)) { + msg("xb_stream_read_chunk(): wrong chunk magic at offset " + "0x%llx.\n", (ulonglong) stream->offset); + goto err; + } + ptr += 8; + stream->offset += 8; + + /* Chunk flags */ + chunk->flags = *ptr++; + stream->offset++; + + /* Chunk type, ignore unknown ones if ignorable flag is set */ + chunk->type = validate_chunk_type(*ptr); + if (chunk->type == XB_CHUNK_TYPE_UNKNOWN && + !(chunk->flags & XB_STREAM_FLAG_IGNORABLE)) { + msg("xb_stream_read_chunk(): unknown chunk type 0x%lu at " + "offset 0x%llx.\n", (ulong) *ptr, + (ulonglong) stream->offset); + goto err; + } + ptr++; + stream->offset++; + + /* Path length */ + pathlen = uint4korr(ptr); + if (pathlen >= FN_REFLEN) { + msg("xb_stream_read_chunk(): path length (%lu) is too large at " + "offset 0x%llx.\n", (ulong) pathlen, stream->offset); + goto err; + } + chunk->pathlen = pathlen; + stream->offset +=4; + + xb_ad((ptr + 4 - tmpbuf) == CHUNK_HEADER_CONSTANT_LEN); + + /* Path */ + if (chunk->pathlen > 0) { + F_READ((uchar *) chunk->path, pathlen); + stream->offset += pathlen; + } + chunk->path[pathlen] = '\0'; + + if (chunk->type == XB_CHUNK_TYPE_EOF) { + return XB_STREAM_READ_CHUNK; + } + + /* Payload length */ + F_READ(tmpbuf, 16); + ullval = uint8korr(tmpbuf); + if (ullval > (ulonglong) SIZE_T_MAX) { + msg("xb_stream_read_chunk(): chunk length is too large at " + "offset 0x%llx: 0x%llx.\n", (ulonglong) stream->offset, + ullval); + goto err; + } + chunk->length = (size_t) ullval; + stream->offset += 8; + + /* Payload offset */ + ullval = uint8korr(tmpbuf + 8); + if (ullval > (ulonglong) MY_OFF_T_MAX) { + msg("xb_stream_read_chunk(): chunk offset is too large at " + "offset 0x%llx: 0x%llx.\n", (ulonglong) stream->offset, + ullval); + goto err; + } + chunk->offset = (my_off_t) ullval; + stream->offset += 8; + + /* Reallocate the buffer if needed */ + if (chunk->length > chunk->buflen) { + chunk->data = my_realloc(chunk->data, chunk->length, + MYF(MY_WME | MY_ALLOW_ZERO_PTR)); + if (chunk->data == NULL) { + msg("xb_stream_read_chunk(): failed to increase buffer " + "to %lu bytes.\n", (ulong) chunk->length); + goto err; + } + chunk->buflen = chunk->length; + } + + /* Checksum */ + F_READ(tmpbuf, 4); + chunk->checksum = uint4korr(tmpbuf); + chunk->checksum_offset = stream->offset; + + /* Payload */ + if (chunk->length > 0) { + F_READ(chunk->data, chunk->length); + stream->offset += chunk->length; + } + + stream->offset += 4; + + return XB_STREAM_READ_CHUNK; + +err: + return XB_STREAM_READ_ERROR; +} + +int +xb_stream_read_done(xb_rstream_t *stream) +{ + my_free(stream); + + return 0; +} diff --git a/extra/mariabackup/xbstream_write.c b/extra/mariabackup/xbstream_write.c new file mode 100644 index 00000000000..978be71e7dd --- /dev/null +++ b/extra/mariabackup/xbstream_write.c @@ -0,0 +1,294 @@ +/****************************************************** +Copyright (c) 2011-2017 Percona LLC and/or its affiliates. + +The xbstream format writer implementation. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#include <mysql_version.h> +#include <my_base.h> +#include <zlib.h> +#include "common.h" +#include "xbstream.h" +#include "crc_glue.h" + +/* Group writes smaller than this into a single chunk */ +#define XB_STREAM_MIN_CHUNK_SIZE (10 * 1024 * 1024) + +struct xb_wstream_struct { + pthread_mutex_t mutex; +}; + +struct xb_wstream_file_struct { + xb_wstream_t *stream; + char *path; + size_t path_len; + char chunk[XB_STREAM_MIN_CHUNK_SIZE]; + char *chunk_ptr; + size_t chunk_free; + my_off_t offset; + void *userdata; + xb_stream_write_callback *write; +}; + +static int xb_stream_flush(xb_wstream_file_t *file); +static int xb_stream_write_chunk(xb_wstream_file_t *file, + const void *buf, size_t len); +static int xb_stream_write_eof(xb_wstream_file_t *file); + +static +ssize_t +xb_stream_default_write_callback(xb_wstream_file_t *file __attribute__((unused)), + void *userdata __attribute__((unused)), + const void *buf, size_t len) +{ + if (my_write(my_fileno(stdout), buf, len, MYF(MY_WME | MY_NABP))) + return -1; + return len; +} + +xb_wstream_t * +xb_stream_write_new(void) +{ + xb_wstream_t *stream; + + stream = (xb_wstream_t *) my_malloc(sizeof(xb_wstream_t), MYF(MY_FAE)); + pthread_mutex_init(&stream->mutex, NULL); + + return stream;; +} + +xb_wstream_file_t * +xb_stream_write_open(xb_wstream_t *stream, const char *path, + MY_STAT *mystat __attribute__((unused)), + void *userdata, + xb_stream_write_callback *onwrite) +{ + xb_wstream_file_t *file; + size_t path_len; + + path_len = strlen(path); + + if (path_len > FN_REFLEN) { + msg("xb_stream_write_open(): file path is too long.\n"); + return NULL; + } + + file = (xb_wstream_file_t *) my_malloc(sizeof(xb_wstream_file_t) + + path_len + 1, MYF(MY_FAE)); + + file->path = (char *) (file + 1); +#ifdef _WIN32 + /* Normalize path on Windows, so we can restore elsewhere.*/ + { + int i; + for (i = 0; ; i++) { + file->path[i] = (path[i] == '\\') ? '/' : path[i]; + if (!path[i]) + break; + } + } +#else + memcpy(file->path, path, path_len + 1); +#endif + file->path_len = path_len; + + file->stream = stream; + file->offset = 0; + file->chunk_ptr = file->chunk; + file->chunk_free = XB_STREAM_MIN_CHUNK_SIZE; + if (onwrite) { +#ifdef __WIN__ + setmode(fileno(stdout), _O_BINARY); +#endif + file->userdata = userdata; + file->write = onwrite; + } else { + file->userdata = NULL; + file->write = xb_stream_default_write_callback; + } + + return file; +} + +int +xb_stream_write_data(xb_wstream_file_t *file, const void *buf, size_t len) +{ + if (len < file->chunk_free) { + memcpy(file->chunk_ptr, buf, len); + file->chunk_ptr += len; + file->chunk_free -= len; + + return 0; + } + + if (xb_stream_flush(file)) + return 1; + + return xb_stream_write_chunk(file, buf, len); +} + +int +xb_stream_write_close(xb_wstream_file_t *file) +{ + if (xb_stream_flush(file) || + xb_stream_write_eof(file)) { + my_free(file); + return 1; + } + + my_free(file); + + return 0; +} + +int +xb_stream_write_done(xb_wstream_t *stream) +{ + pthread_mutex_destroy(&stream->mutex); + + my_free(stream); + + return 0; +} + +static +int +xb_stream_flush(xb_wstream_file_t *file) +{ + if (file->chunk_ptr == file->chunk) { + return 0; + } + + if (xb_stream_write_chunk(file, file->chunk, + file->chunk_ptr - file->chunk)) { + return 1; + } + + file->chunk_ptr = file->chunk; + file->chunk_free = XB_STREAM_MIN_CHUNK_SIZE; + + return 0; +} + +static +int +xb_stream_write_chunk(xb_wstream_file_t *file, const void *buf, size_t len) +{ + /* Chunk magic + flags + chunk type + path_len + path + len + offset + + checksum */ + uchar tmpbuf[sizeof(XB_STREAM_CHUNK_MAGIC) - 1 + 1 + 1 + 4 + + FN_REFLEN + 8 + 8 + 4]; + uchar *ptr; + xb_wstream_t *stream = file->stream; + ulong checksum; + + /* Write xbstream header */ + ptr = tmpbuf; + + /* Chunk magic */ + memcpy(ptr, XB_STREAM_CHUNK_MAGIC, sizeof(XB_STREAM_CHUNK_MAGIC) - 1); + ptr += sizeof(XB_STREAM_CHUNK_MAGIC) - 1; + + *ptr++ = 0; /* Chunk flags */ + + *ptr++ = (uchar) XB_CHUNK_TYPE_PAYLOAD; /* Chunk type */ + + int4store(ptr, file->path_len); /* Path length */ + ptr += 4; + + memcpy(ptr, file->path, file->path_len); /* Path */ + ptr += file->path_len; + + int8store(ptr, len); /* Payload length */ + ptr += 8; + + checksum = crc32_iso3309(0, buf, (uint)len); /* checksum */ + + pthread_mutex_lock(&stream->mutex); + + int8store(ptr, file->offset); /* Payload offset */ + ptr += 8; + + int4store(ptr, checksum); + ptr += 4; + + xb_ad(ptr <= tmpbuf + sizeof(tmpbuf)); + + if (file->write(file, file->userdata, tmpbuf, ptr-tmpbuf) == -1) + goto err; + + + if (file->write(file, file->userdata, buf, len) == -1) /* Payload */ + goto err; + + file->offset+= len; + + pthread_mutex_unlock(&stream->mutex); + + return 0; + +err: + + pthread_mutex_unlock(&stream->mutex); + + return 1; +} + +static +int +xb_stream_write_eof(xb_wstream_file_t *file) +{ + /* Chunk magic + flags + chunk type + path_len + path */ + uchar tmpbuf[sizeof(XB_STREAM_CHUNK_MAGIC) - 1 + 1 + 1 + 4 + + FN_REFLEN]; + uchar *ptr; + xb_wstream_t *stream = file->stream; + + pthread_mutex_lock(&stream->mutex); + + /* Write xbstream header */ + ptr = tmpbuf; + + /* Chunk magic */ + memcpy(ptr, XB_STREAM_CHUNK_MAGIC, sizeof(XB_STREAM_CHUNK_MAGIC) - 1); + ptr += sizeof(XB_STREAM_CHUNK_MAGIC) - 1; + + *ptr++ = 0; /* Chunk flags */ + + *ptr++ = (uchar) XB_CHUNK_TYPE_EOF; /* Chunk type */ + + int4store(ptr, file->path_len); /* Path length */ + ptr += 4; + + memcpy(ptr, file->path, file->path_len); /* Path */ + ptr += file->path_len; + + xb_ad(ptr <= tmpbuf + sizeof(tmpbuf)); + + if (file->write(file, file->userdata, tmpbuf, + (ulonglong) (ptr - tmpbuf)) == -1) + goto err; + + pthread_mutex_unlock(&stream->mutex); + + return 0; +err: + + pthread_mutex_unlock(&stream->mutex); + + return 1; +} diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc new file mode 100644 index 00000000000..c116d119cee --- /dev/null +++ b/extra/mariabackup/xtrabackup.cc @@ -0,0 +1,7498 @@ +/****************************************************** +XtraBackup: hot backup tool for InnoDB +(c) 2009-2017 Percona LLC and/or its affiliates +Originally Created 3/3/2009 Yasufumi Kinoshita +Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, +Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +******************************************************* + +This file incorporates work covered by the following copyright and +permission notice: + +Copyright (c) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place, Suite 330, Boston, MA 02111-1307 USA + +*******************************************************/ + +//#define XTRABACKUP_TARGET_IS_PLUGIN + +#include <mysql_version.h> +#include <my_base.h> +#include <my_getopt.h> +#include <mysql_com.h> +#include <my_default.h> +#include <mysqld.h> + +#include <fcntl.h> +#include <string.h> + +#ifdef __linux__ +# include <sys/prctl.h> +#include <sys/resource.h> +#endif + + +#include <btr0sea.h> +#include <dict0priv.h> +#include <dict0stats.h> +#include <lock0lock.h> +#include <log0recv.h> +#include <row0mysql.h> +#include <row0quiesce.h> +#include <srv0start.h> +#include <buf0dblwr.h> + +#include <list> +#include <sstream> +#include <set> +#include <mysql.h> + +#define G_PTR uchar* + +#include "common.h" +#include "datasink.h" + +#include "xb_regex.h" +#include "fil_cur.h" +#include "write_filt.h" +#include "xtrabackup.h" +#include "ds_buffer.h" +#include "ds_tmpfile.h" +#include "xbstream.h" +#include "changed_page_bitmap.h" +#include "read_filt.h" +#include "wsrep.h" +#include "innobackupex.h" +#include "backup_mysql.h" +#include "backup_copy.h" +#include "backup_mysql.h" +#include "xb0xb.h" +#include "encryption_plugin.h" +#include <sql_plugin.h> +#include <srv0srv.h> +#include <crc_glue.h> + +/* TODO: replace with appropriate macros used in InnoDB 5.6 */ +#define PAGE_ZIP_MIN_SIZE_SHIFT 10 +#define DICT_TF_ZSSIZE_SHIFT 1 +#define DICT_TF_FORMAT_ZIP 1 +#define DICT_TF_FORMAT_SHIFT 5 + +int sys_var_init(); + +my_bool innodb_inited= 0; + +/* === xtrabackup specific options === */ +char xtrabackup_real_target_dir[FN_REFLEN] = "./xtrabackup_backupfiles/"; +char *xtrabackup_target_dir= xtrabackup_real_target_dir; +my_bool xtrabackup_version = FALSE; +my_bool xtrabackup_backup = FALSE; +my_bool xtrabackup_stats = FALSE; +my_bool xtrabackup_prepare = FALSE; +my_bool xtrabackup_copy_back = FALSE; +my_bool xtrabackup_move_back = FALSE; +my_bool xtrabackup_decrypt_decompress = FALSE; +my_bool xtrabackup_print_param = FALSE; + +my_bool xtrabackup_export = FALSE; +my_bool xtrabackup_apply_log_only = FALSE; + +longlong xtrabackup_use_memory = 100*1024*1024L; +my_bool xtrabackup_create_ib_logfile = FALSE; + +long xtrabackup_throttle = 0; /* 0:unlimited */ +lint io_ticket; +os_event_t wait_throttle = NULL; +os_event_t log_copying_stop = NULL; + +char *xtrabackup_incremental = NULL; +lsn_t incremental_lsn; +lsn_t incremental_to_lsn; +lsn_t incremental_last_lsn; +xb_page_bitmap *changed_page_bitmap = NULL; + +char *xtrabackup_incremental_basedir = NULL; /* for --backup */ +char *xtrabackup_extra_lsndir = NULL; /* for --backup with --extra-lsndir */ +char *xtrabackup_incremental_dir = NULL; /* for --prepare */ + +char xtrabackup_real_incremental_basedir[FN_REFLEN]; +char xtrabackup_real_extra_lsndir[FN_REFLEN]; +char xtrabackup_real_incremental_dir[FN_REFLEN]; + +lsn_t xtrabackup_archived_to_lsn = 0; /* for --archived-to-lsn */ + +char *xtrabackup_tmpdir; + +char *xtrabackup_tables = NULL; +char *xtrabackup_tables_file = NULL; +char *xtrabackup_tables_exclude = NULL; + +typedef std::list<regex_t> regex_list_t; +static regex_list_t regex_include_list; +static regex_list_t regex_exclude_list; + +static hash_table_t* tables_include_hash = NULL; +static hash_table_t* tables_exclude_hash = NULL; + +char *xtrabackup_databases = NULL; +char *xtrabackup_databases_file = NULL; +char *xtrabackup_databases_exclude = NULL; +static hash_table_t* databases_include_hash = NULL; +static hash_table_t* databases_exclude_hash = NULL; + +static hash_table_t* inc_dir_tables_hash; + +struct xb_filter_entry_struct{ + char* name; + ibool has_tables; + hash_node_t name_hash; +}; +typedef struct xb_filter_entry_struct xb_filter_entry_t; + +static ulint thread_nr[SRV_MAX_N_IO_THREADS + 6]; +static os_thread_id_t thread_ids[SRV_MAX_N_IO_THREADS + 6]; + +lsn_t checkpoint_lsn_start; +lsn_t checkpoint_no_start; +lsn_t log_copy_scanned_lsn; +ibool log_copying = TRUE; +ibool log_copying_running = FALSE; +ibool io_watching_thread_running = FALSE; + +ibool xtrabackup_logfile_is_renamed = FALSE; + +int xtrabackup_parallel; + +char *xtrabackup_stream_str = NULL; +xb_stream_fmt_t xtrabackup_stream_fmt = XB_STREAM_FMT_NONE; +ibool xtrabackup_stream = FALSE; + +const char *xtrabackup_compress_alg = NULL; +ibool xtrabackup_compress = FALSE; +uint xtrabackup_compress_threads; +ulonglong xtrabackup_compress_chunk_size = 0; + +const char *xtrabackup_encrypt_algo_names[] = +{ "NONE", "AES128", "AES192", "AES256", NullS}; +TYPELIB xtrabackup_encrypt_algo_typelib= +{array_elements(xtrabackup_encrypt_algo_names)-1,"", + xtrabackup_encrypt_algo_names, NULL}; + +ibool xtrabackup_encrypt = FALSE; +ulong xtrabackup_encrypt_algo; +char *xtrabackup_encrypt_key = NULL; +char *xtrabackup_encrypt_key_file = NULL; +uint xtrabackup_encrypt_threads; +ulonglong xtrabackup_encrypt_chunk_size = 0; + +ulint xtrabackup_rebuild_threads = 1; + +/* sleep interval beetween log copy iterations in log copying thread +in milliseconds (default is 1 second) */ +ulint xtrabackup_log_copy_interval = 1000; +static ulong max_buf_pool_modified_pct; + +/* Ignored option (--log) for MySQL option compatibility */ +char* log_ignored_opt = NULL; + +/* === metadata of backup === */ +#define XTRABACKUP_METADATA_FILENAME "xtrabackup_checkpoints" +char metadata_type[30] = ""; /*[full-backuped|log-applied| + full-prepared|incremental]*/ +lsn_t metadata_from_lsn = 0; +lsn_t metadata_to_lsn = 0; +lsn_t metadata_last_lsn = 0; + +#define XB_LOG_FILENAME "xtrabackup_logfile" + +ds_file_t *dst_log_file = NULL; + +static char mysql_data_home_buff[2]; + +const char *defaults_group = "mysqld"; + +/* === static parameters in ha_innodb.cc */ + +#define HA_INNOBASE_ROWS_IN_TABLE 10000 /* to get optimization right */ +#define HA_INNOBASE_RANGE_COUNT 100 + +ulong innobase_large_page_size = 0; + +/* The default values for the following, type long or longlong, start-up +parameters are declared in mysqld.cc: */ + +long innobase_additional_mem_pool_size = 1*1024*1024L; +long innobase_buffer_pool_awe_mem_mb = 0; +long innobase_file_io_threads = 4; +long innobase_read_io_threads = 4; +long innobase_write_io_threads = 4; +long innobase_force_recovery = 0; +long innobase_log_buffer_size = 1024*1024L; +long innobase_log_files_in_group = 2; +long innobase_open_files = 300L; + +longlong innobase_page_size = (1LL << 14); /* 16KB */ +static ulong innobase_log_block_size = 512; +my_bool innobase_fast_checksum = FALSE; +char* innobase_doublewrite_file = NULL; +char* innobase_buffer_pool_filename = NULL; + +longlong innobase_buffer_pool_size = 8*1024*1024L; +longlong innobase_log_file_size = 48*1024*1024L; + +/* The default values for the following char* start-up parameters +are determined in innobase_init below: */ + +char* innobase_ignored_opt = NULL; +char* innobase_data_home_dir = NULL; +char* innobase_data_file_path = NULL; +char* innobase_log_arch_dir = NULL;/* unused */ +/* The following has a misleading name: starting from 4.0.5, this also +affects Windows: */ +char* innobase_unix_file_flush_method = NULL; + +/* Below we have boolean-valued start-up parameters, and their default +values */ + +ulong innobase_fast_shutdown = 1; +my_bool innobase_log_archive = FALSE;/* unused */ +my_bool innobase_use_doublewrite = TRUE; +my_bool innobase_use_checksums = TRUE; +my_bool innobase_use_large_pages = FALSE; +my_bool innobase_file_per_table = FALSE; +my_bool innobase_locks_unsafe_for_binlog = FALSE; +my_bool innobase_rollback_on_timeout = FALSE; +my_bool innobase_create_status_file = FALSE; +my_bool innobase_adaptive_hash_index = TRUE; + +static char *internal_innobase_data_file_path = NULL; + +/* The following counter is used to convey information to InnoDB +about server activity: in selects it is not sensible to call +srv_active_wake_master_thread after each fetch or search, we only do +it every INNOBASE_WAKE_INTERVAL'th step. */ + +#define INNOBASE_WAKE_INTERVAL 32 +ulong innobase_active_counter = 0; + + +static char *xtrabackup_debug_sync = NULL; + +my_bool xtrabackup_compact = FALSE; +my_bool xtrabackup_rebuild_indexes = FALSE; + +my_bool xtrabackup_incremental_force_scan = FALSE; + +/* The flushed lsn which is read from data files */ +lsn_t min_flushed_lsn= 0; +lsn_t max_flushed_lsn= 0; + +/* The size of archived log file */ +ib_int64_t xtrabackup_arch_file_size = 0ULL; +/* The minimal LSN of found archived log files */ +lsn_t xtrabackup_arch_first_file_lsn = 0ULL; +/* The maximum LSN of found archived log files */ +lsn_t xtrabackup_arch_last_file_lsn = 0ULL; + +ulong xb_open_files_limit= 0; +char *xb_plugin_dir; +char *xb_plugin_load; +my_bool xb_close_files= FALSE; + +/* Datasinks */ +ds_ctxt_t *ds_data = NULL; +ds_ctxt_t *ds_meta = NULL; +ds_ctxt_t *ds_redo = NULL; + +static bool innobackupex_mode = false; + +static long innobase_log_files_in_group_save; +static char *srv_log_group_home_dir_save; +static longlong innobase_log_file_size_save; + +/* String buffer used by --print-param to accumulate server options as they are +parsed from the defaults file */ +static std::ostringstream print_param_str; + +/* Set of specified parameters */ +std::set<std::string> param_set; + +static ulonglong global_max_value; + +extern "C" sig_handler handle_fatal_signal(int sig); + +my_bool opt_galera_info = FALSE; +my_bool opt_slave_info = FALSE; +my_bool opt_no_lock = FALSE; +my_bool opt_safe_slave_backup = FALSE; +my_bool opt_rsync = FALSE; +my_bool opt_force_non_empty_dirs = FALSE; +my_bool opt_noversioncheck = FALSE; +my_bool opt_no_backup_locks = FALSE; +my_bool opt_decompress = FALSE; +my_bool opt_remove_original = FALSE; + +static const char *binlog_info_values[] = {"off", "lockless", "on", "auto", + NullS}; +static TYPELIB binlog_info_typelib = {array_elements(binlog_info_values)-1, "", + binlog_info_values, NULL}; +ulong opt_binlog_info; + +char *opt_incremental_history_name = NULL; +char *opt_incremental_history_uuid = NULL; + +char *opt_user = NULL; +char *opt_password = NULL; +char *opt_host = NULL; +char *opt_defaults_group = NULL; +char *opt_socket = NULL; +uint opt_port = 0; +char *opt_login_path = NULL; +char *opt_log_bin = NULL; + +const char *query_type_names[] = { "ALL", "UPDATE", "SELECT", NullS}; + +TYPELIB query_type_typelib= {array_elements(query_type_names) - 1, "", + query_type_names, NULL}; + +ulong opt_lock_wait_query_type; +ulong opt_kill_long_query_type; + +ulong opt_decrypt_algo = 0; + +uint opt_kill_long_queries_timeout = 0; +uint opt_lock_wait_timeout = 0; +uint opt_lock_wait_threshold = 0; +uint opt_debug_sleep_before_unlock = 0; +uint opt_safe_slave_backup_timeout = 0; + +const char *opt_history = NULL; +my_bool opt_decrypt = FALSE; + +#if defined(HAVE_OPENSSL) +my_bool opt_ssl_verify_server_cert = FALSE; +#if !defined(HAVE_YASSL) +char *opt_server_public_key = NULL; +#endif +#endif + +/* Whether xtrabackup_binlog_info should be created on recovery */ +static bool recover_binlog_info; + +/* Simple datasink creation tracking...add datasinks in the reverse order you +want them destroyed. */ +#define XTRABACKUP_MAX_DATASINKS 10 +static ds_ctxt_t *datasinks[XTRABACKUP_MAX_DATASINKS]; +static uint actual_datasinks = 0; +static inline +void +xtrabackup_add_datasink(ds_ctxt_t *ds) +{ + xb_ad(actual_datasinks < XTRABACKUP_MAX_DATASINKS); + datasinks[actual_datasinks] = ds; actual_datasinks++; +} + +/* ======== Datafiles iterator ======== */ +datafiles_iter_t * +datafiles_iter_new(fil_system_t *f_system) +{ + datafiles_iter_t *it; + + it = static_cast<datafiles_iter_t *> + (ut_malloc(sizeof(datafiles_iter_t))); + it->mutex = os_mutex_create(); + + it->system = f_system; + it->space = NULL; + it->node = NULL; + it->started = FALSE; + + return it; +} + +fil_node_t * +datafiles_iter_next(datafiles_iter_t *it) +{ + fil_node_t *new_node; + + os_mutex_enter(it->mutex); + + if (it->node == NULL) { + if (it->started) + goto end; + it->started = TRUE; + } else { + it->node = UT_LIST_GET_NEXT(chain, it->node); + if (it->node != NULL) + goto end; + } + + it->space = (it->space == NULL) ? + UT_LIST_GET_FIRST(it->system->space_list) : + UT_LIST_GET_NEXT(space_list, it->space); + + while (it->space != NULL && + (it->space->purpose != FIL_TABLESPACE || + UT_LIST_GET_LEN(it->space->chain) == 0)) + it->space = UT_LIST_GET_NEXT(space_list, it->space); + if (it->space == NULL) + goto end; + + it->node = UT_LIST_GET_FIRST(it->space->chain); + +end: + new_node = it->node; + os_mutex_exit(it->mutex); + + return new_node; +} + +void +datafiles_iter_free(datafiles_iter_t *it) +{ + os_mutex_free(it->mutex); + ut_free(it); +} + +/* ======== Date copying thread context ======== */ + +typedef struct { + datafiles_iter_t *it; + uint num; + uint *count; + os_ib_mutex_t count_mutex; + os_thread_id_t id; +} data_thread_ctxt_t; + +/* ======== for option and variables ======== */ + +enum options_xtrabackup +{ + OPT_XTRA_TARGET_DIR = 1000, /* make sure it is larger + than OPT_MAX_CLIENT_OPTION */ + OPT_XTRA_BACKUP, + OPT_XTRA_STATS, + OPT_XTRA_PREPARE, + OPT_XTRA_EXPORT, + OPT_XTRA_APPLY_LOG_ONLY, + OPT_XTRA_PRINT_PARAM, + OPT_XTRA_USE_MEMORY, + OPT_XTRA_THROTTLE, + OPT_XTRA_LOG_COPY_INTERVAL, + OPT_XTRA_INCREMENTAL, + OPT_XTRA_INCREMENTAL_BASEDIR, + OPT_XTRA_EXTRA_LSNDIR, + OPT_XTRA_INCREMENTAL_DIR, + OPT_XTRA_ARCHIVED_TO_LSN, + OPT_XTRA_TABLES, + OPT_XTRA_TABLES_FILE, + OPT_XTRA_DATABASES, + OPT_XTRA_DATABASES_FILE, + OPT_XTRA_CREATE_IB_LOGFILE, + OPT_XTRA_PARALLEL, + OPT_XTRA_STREAM, + OPT_XTRA_COMPRESS, + OPT_XTRA_COMPRESS_THREADS, + OPT_XTRA_COMPRESS_CHUNK_SIZE, + OPT_XTRA_ENCRYPT, + OPT_XTRA_ENCRYPT_KEY, + OPT_XTRA_ENCRYPT_KEY_FILE, + OPT_XTRA_ENCRYPT_THREADS, + OPT_XTRA_ENCRYPT_CHUNK_SIZE, + OPT_LOG, + OPT_INNODB, + OPT_INNODB_CHECKSUMS, + OPT_INNODB_DATA_FILE_PATH, + OPT_INNODB_DATA_HOME_DIR, + OPT_INNODB_ADAPTIVE_HASH_INDEX, + OPT_INNODB_DOUBLEWRITE, + OPT_INNODB_FAST_SHUTDOWN, + OPT_INNODB_FILE_PER_TABLE, + OPT_INNODB_FLUSH_LOG_AT_TRX_COMMIT, + OPT_INNODB_FLUSH_METHOD, + OPT_INNODB_LOCKS_UNSAFE_FOR_BINLOG, + OPT_INNODB_LOG_ARCH_DIR, + OPT_INNODB_LOG_ARCHIVE, + OPT_INNODB_LOG_GROUP_HOME_DIR, + OPT_INNODB_MAX_DIRTY_PAGES_PCT, + OPT_INNODB_MAX_PURGE_LAG, + OPT_INNODB_ROLLBACK_ON_TIMEOUT, + OPT_INNODB_STATUS_FILE, + OPT_INNODB_ADDITIONAL_MEM_POOL_SIZE, + OPT_INNODB_AUTOEXTEND_INCREMENT, + OPT_INNODB_BUFFER_POOL_SIZE, + OPT_INNODB_COMMIT_CONCURRENCY, + OPT_INNODB_CONCURRENCY_TICKETS, + OPT_INNODB_FILE_IO_THREADS, + OPT_INNODB_IO_CAPACITY, + OPT_INNODB_READ_IO_THREADS, + OPT_INNODB_WRITE_IO_THREADS, + OPT_INNODB_USE_NATIVE_AIO, + OPT_INNODB_PAGE_SIZE, + OPT_INNODB_LOG_BLOCK_SIZE, + OPT_INNODB_FAST_CHECKSUM, + OPT_INNODB_EXTRA_UNDOSLOTS, + OPT_INNODB_DOUBLEWRITE_FILE, + OPT_INNODB_BUFFER_POOL_FILENAME, + OPT_INNODB_FORCE_RECOVERY, + OPT_INNODB_LOCK_WAIT_TIMEOUT, + OPT_INNODB_LOG_BUFFER_SIZE, + OPT_INNODB_LOG_FILE_SIZE, + OPT_INNODB_LOG_FILES_IN_GROUP, + OPT_INNODB_MIRRORED_LOG_GROUPS, + OPT_INNODB_OPEN_FILES, + OPT_INNODB_SYNC_SPIN_LOOPS, + OPT_INNODB_THREAD_CONCURRENCY, + OPT_INNODB_THREAD_SLEEP_DELAY, + OPT_XTRA_DEBUG_SYNC, + OPT_XTRA_COMPACT, + OPT_XTRA_REBUILD_INDEXES, + OPT_XTRA_REBUILD_THREADS, + OPT_INNODB_CHECKSUM_ALGORITHM, + OPT_INNODB_UNDO_DIRECTORY, + OPT_INNODB_UNDO_TABLESPACES, + OPT_INNODB_LOG_CHECKSUM_ALGORITHM, + OPT_XTRA_INCREMENTAL_FORCE_SCAN, + OPT_DEFAULTS_GROUP, + OPT_OPEN_FILES_LIMIT, + OPT_PLUGIN_DIR, + OPT_PLUGIN_LOAD, + OPT_INNODB_ENCRYPT_LOG, + OPT_CLOSE_FILES, + OPT_CORE_FILE, + + OPT_COPY_BACK, + OPT_MOVE_BACK, + OPT_GALERA_INFO, + OPT_SLAVE_INFO, + OPT_NO_LOCK, + OPT_SAFE_SLAVE_BACKUP, + OPT_RSYNC, + OPT_FORCE_NON_EMPTY_DIRS, + OPT_NO_VERSION_CHECK, + OPT_NO_BACKUP_LOCKS, + OPT_DECOMPRESS, + OPT_INCREMENTAL_HISTORY_NAME, + OPT_INCREMENTAL_HISTORY_UUID, + OPT_DECRYPT, + OPT_REMOVE_ORIGINAL, + OPT_LOCK_WAIT_QUERY_TYPE, + OPT_KILL_LONG_QUERY_TYPE, + OPT_HISTORY, + OPT_KILL_LONG_QUERIES_TIMEOUT, + OPT_LOCK_WAIT_TIMEOUT, + OPT_LOCK_WAIT_THRESHOLD, + OPT_DEBUG_SLEEP_BEFORE_UNLOCK, + OPT_SAFE_SLAVE_BACKUP_TIMEOUT, + OPT_BINLOG_INFO, + OPT_XB_SECURE_AUTH, + + OPT_SSL_SSL, + OPT_SSL_VERIFY_SERVER_CERT, + OPT_SERVER_PUBLIC_KEY, + + OPT_XTRA_TABLES_EXCLUDE, + OPT_XTRA_DATABASES_EXCLUDE, +}; + +struct my_option xb_client_options[] = +{ + {"version", 'v', "print xtrabackup version information", + (G_PTR *) &xtrabackup_version, (G_PTR *) &xtrabackup_version, 0, GET_BOOL, + NO_ARG, 0, 0, 0, 0, 0, 0}, + {"target-dir", OPT_XTRA_TARGET_DIR, "destination directory", (G_PTR*) &xtrabackup_target_dir, + (G_PTR*) &xtrabackup_target_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"backup", OPT_XTRA_BACKUP, "take backup to target-dir", + (G_PTR*) &xtrabackup_backup, (G_PTR*) &xtrabackup_backup, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"stats", OPT_XTRA_STATS, "calc statistic of datadir (offline mysqld is recommended)", + (G_PTR*) &xtrabackup_stats, (G_PTR*) &xtrabackup_stats, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"prepare", OPT_XTRA_PREPARE, "prepare a backup for starting mysql server on the backup.", + (G_PTR*) &xtrabackup_prepare, (G_PTR*) &xtrabackup_prepare, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"export", OPT_XTRA_EXPORT, "create files to import to another database when prepare.", + (G_PTR*) &xtrabackup_export, (G_PTR*) &xtrabackup_export, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"apply-log-only", OPT_XTRA_APPLY_LOG_ONLY, + "stop recovery process not to progress LSN after applying log when prepare.", + (G_PTR*) &xtrabackup_apply_log_only, (G_PTR*) &xtrabackup_apply_log_only, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"print-param", OPT_XTRA_PRINT_PARAM, "print parameter of mysqld needed for copyback.", + (G_PTR*) &xtrabackup_print_param, (G_PTR*) &xtrabackup_print_param, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"use-memory", OPT_XTRA_USE_MEMORY, "The value is used instead of buffer_pool_size", + (G_PTR*) &xtrabackup_use_memory, (G_PTR*) &xtrabackup_use_memory, + 0, GET_LL, REQUIRED_ARG, 100*1024*1024L, 1024*1024L, LONGLONG_MAX, 0, + 1024*1024L, 0}, + {"throttle", OPT_XTRA_THROTTLE, "limit count of IO operations (pairs of read&write) per second to IOS values (for '--backup')", + (G_PTR*) &xtrabackup_throttle, (G_PTR*) &xtrabackup_throttle, + 0, GET_LONG, REQUIRED_ARG, 0, 0, LONG_MAX, 0, 1, 0}, + {"log", OPT_LOG, "Ignored option for MySQL option compatibility", + (G_PTR*) &log_ignored_opt, (G_PTR*) &log_ignored_opt, 0, + GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, + {"log-copy-interval", OPT_XTRA_LOG_COPY_INTERVAL, "time interval between checks done by log copying thread in milliseconds (default is 1 second).", + (G_PTR*) &xtrabackup_log_copy_interval, (G_PTR*) &xtrabackup_log_copy_interval, + 0, GET_LONG, REQUIRED_ARG, 1000, 0, LONG_MAX, 0, 1, 0}, + {"extra-lsndir", OPT_XTRA_EXTRA_LSNDIR, "(for --backup): save an extra copy of the xtrabackup_checkpoints file in this directory.", + (G_PTR*) &xtrabackup_extra_lsndir, (G_PTR*) &xtrabackup_extra_lsndir, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"incremental-lsn", OPT_XTRA_INCREMENTAL, "(for --backup): copy only .ibd pages newer than specified LSN 'high:low'. ##ATTENTION##: If a wrong LSN value is specified, it is impossible to diagnose this, causing the backup to be unusable. Be careful!", + (G_PTR*) &xtrabackup_incremental, (G_PTR*) &xtrabackup_incremental, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"incremental-basedir", OPT_XTRA_INCREMENTAL_BASEDIR, "(for --backup): copy only .ibd pages newer than backup at specified directory.", + (G_PTR*) &xtrabackup_incremental_basedir, (G_PTR*) &xtrabackup_incremental_basedir, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"incremental-dir", OPT_XTRA_INCREMENTAL_DIR, "(for --prepare): apply .delta files and logfile in the specified directory.", + (G_PTR*) &xtrabackup_incremental_dir, (G_PTR*) &xtrabackup_incremental_dir, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"to-archived-lsn", OPT_XTRA_ARCHIVED_TO_LSN, + "Don't apply archived logs with bigger log sequence number.", + (G_PTR*) &xtrabackup_archived_to_lsn, (G_PTR*) &xtrabackup_archived_to_lsn, 0, + GET_LL, REQUIRED_ARG, 0, 0, LONGLONG_MAX, 0, 0, 0}, + {"tables", OPT_XTRA_TABLES, "filtering by regexp for table names.", + (G_PTR*) &xtrabackup_tables, (G_PTR*) &xtrabackup_tables, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"tables_file", OPT_XTRA_TABLES_FILE, "filtering by list of the exact database.table name in the file.", + (G_PTR*) &xtrabackup_tables_file, (G_PTR*) &xtrabackup_tables_file, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"databases", OPT_XTRA_DATABASES, "filtering by list of databases.", + (G_PTR*) &xtrabackup_databases, (G_PTR*) &xtrabackup_databases, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"databases_file", OPT_XTRA_TABLES_FILE, + "filtering by list of databases in the file.", + (G_PTR*) &xtrabackup_databases_file, (G_PTR*) &xtrabackup_databases_file, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"tables-exclude", OPT_XTRA_TABLES_EXCLUDE, "filtering by regexp for table names. " + "Operates the same way as --tables, but matched names are excluded from backup. " + "Note that this option has a higher priority than --tables.", + (G_PTR*) &xtrabackup_tables_exclude, (G_PTR*) &xtrabackup_tables_exclude, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"databases-exclude", OPT_XTRA_DATABASES_EXCLUDE, "Excluding databases based on name, " + "Operates the same way as --databases, but matched names are excluded from backup. " + "Note that this option has a higher priority than --databases.", + (G_PTR*) &xtrabackup_databases_exclude, (G_PTR*) &xtrabackup_databases_exclude, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"create-ib-logfile", OPT_XTRA_CREATE_IB_LOGFILE, "** not work for now** creates ib_logfile* also after '--prepare'. ### If you want create ib_logfile*, only re-execute this command in same options. ###", + (G_PTR*) &xtrabackup_create_ib_logfile, (G_PTR*) &xtrabackup_create_ib_logfile, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"stream", OPT_XTRA_STREAM, "Stream all backup files to the standard output " + "in the specified format." +#ifdef HAVE_LIBARCHIVE + "Supported formats are 'tar' and 'xbstream'." +#else + "Supported format is 'xbstream'." +#endif + , + (G_PTR*) &xtrabackup_stream_str, (G_PTR*) &xtrabackup_stream_str, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"compress", OPT_XTRA_COMPRESS, "Compress individual backup files using the " + "specified compression algorithm. Currently the only supported algorithm " + "is 'quicklz'. It is also the default algorithm, i.e. the one used when " + "--compress is used without an argument.", + (G_PTR*) &xtrabackup_compress_alg, (G_PTR*) &xtrabackup_compress_alg, 0, + GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, + + {"compress-threads", OPT_XTRA_COMPRESS_THREADS, + "Number of threads for parallel data compression. The default value is 1.", + (G_PTR*) &xtrabackup_compress_threads, (G_PTR*) &xtrabackup_compress_threads, + 0, GET_UINT, REQUIRED_ARG, 1, 1, UINT_MAX, 0, 0, 0}, + + {"compress-chunk-size", OPT_XTRA_COMPRESS_CHUNK_SIZE, + "Size of working buffer(s) for compression threads in bytes. The default value is 64K.", + (G_PTR*) &xtrabackup_compress_chunk_size, (G_PTR*) &xtrabackup_compress_chunk_size, + 0, GET_ULL, REQUIRED_ARG, (1 << 16), 1024, ULONGLONG_MAX, 0, 0, 0}, + + {"encrypt", OPT_XTRA_ENCRYPT, "Encrypt individual backup files using the " + "specified encryption algorithm.", + &xtrabackup_encrypt_algo, &xtrabackup_encrypt_algo, + &xtrabackup_encrypt_algo_typelib, GET_ENUM, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"encrypt-key", OPT_XTRA_ENCRYPT_KEY, "Encryption key to use.", + (G_PTR*) &xtrabackup_encrypt_key, (G_PTR*) &xtrabackup_encrypt_key, 0, + GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"encrypt-key-file", OPT_XTRA_ENCRYPT_KEY_FILE, "File which contains encryption key to use.", + (G_PTR*) &xtrabackup_encrypt_key_file, (G_PTR*) &xtrabackup_encrypt_key_file, 0, + GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"encrypt-threads", OPT_XTRA_ENCRYPT_THREADS, + "Number of threads for parallel data encryption. The default value is 1.", + (G_PTR*) &xtrabackup_encrypt_threads, (G_PTR*) &xtrabackup_encrypt_threads, + 0, GET_UINT, REQUIRED_ARG, 1, 1, UINT_MAX, 0, 0, 0}, + + {"encrypt-chunk-size", OPT_XTRA_ENCRYPT_CHUNK_SIZE, + "Size of working buffer(S) for encryption threads in bytes. The default value is 64K.", + (G_PTR*) &xtrabackup_encrypt_chunk_size, (G_PTR*) &xtrabackup_encrypt_chunk_size, + 0, GET_ULL, REQUIRED_ARG, (1 << 16), 1024, ULONGLONG_MAX, 0, 0, 0}, + + {"compact", OPT_XTRA_COMPACT, + "Create a compact backup by skipping secondary index pages.", + (G_PTR*) &xtrabackup_compact, (G_PTR*) &xtrabackup_compact, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"rebuild_indexes", OPT_XTRA_REBUILD_INDEXES, + "Rebuild secondary indexes in InnoDB tables after applying the log. " + "Only has effect with --prepare.", + (G_PTR*) &xtrabackup_rebuild_indexes, (G_PTR*) &xtrabackup_rebuild_indexes, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"rebuild_threads", OPT_XTRA_REBUILD_THREADS, + "Use this number of threads to rebuild indexes in a compact backup. " + "Only has effect with --prepare and --rebuild-indexes.", + (G_PTR*) &xtrabackup_rebuild_threads, (G_PTR*) &xtrabackup_rebuild_threads, + 0, GET_UINT, REQUIRED_ARG, 1, 1, UINT_MAX, 0, 0, 0}, + + {"incremental-force-scan", OPT_XTRA_INCREMENTAL_FORCE_SCAN, + "Perform a full-scan incremental backup even in the presence of changed " + "page bitmap data", + (G_PTR*)&xtrabackup_incremental_force_scan, + (G_PTR*)&xtrabackup_incremental_force_scan, 0, GET_BOOL, NO_ARG, + 0, 0, 0, 0, 0, 0}, + + + {"close_files", OPT_CLOSE_FILES, "do not keep files opened. Use at your own " + "risk.", (G_PTR*) &xb_close_files, (G_PTR*) &xb_close_files, 0, GET_BOOL, + NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"core-file", OPT_CORE_FILE, "Write core on fatal signals", 0, 0, 0, + GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + + + {"copy-back", OPT_COPY_BACK, "Copy all the files in a previously made " + "backup from the backup directory to their original locations.", + (uchar *) &xtrabackup_copy_back, (uchar *) &xtrabackup_copy_back, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"move-back", OPT_MOVE_BACK, "Move all the files in a previously made " + "backup from the backup directory to the actual datadir location. " + "Use with caution, as it removes backup files.", + (uchar *) &xtrabackup_move_back, (uchar *) &xtrabackup_move_back, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"galera-info", OPT_GALERA_INFO, "This options creates the " + "xtrabackup_galera_info file which contains the local node state at " + "the time of the backup. Option should be used when performing the " + "backup of Percona-XtraDB-Cluster. Has no effect when backup locks " + "are used to create the backup.", + (uchar *) &opt_galera_info, (uchar *) &opt_galera_info, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"slave-info", OPT_SLAVE_INFO, "This option is useful when backing " + "up a replication slave server. It prints the binary log position " + "and name of the master server. It also writes this information to " + "the \"xtrabackup_slave_info\" file as a \"CHANGE MASTER\" command. " + "A new slave for this master can be set up by starting a slave server " + "on this backup and issuing a \"CHANGE MASTER\" command with the " + "binary log position saved in the \"xtrabackup_slave_info\" file.", + (uchar *) &opt_slave_info, (uchar *) &opt_slave_info, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"no-lock", OPT_NO_LOCK, "Use this option to disable table lock " + "with \"FLUSH TABLES WITH READ LOCK\". Use it only if ALL your " + "tables are InnoDB and you DO NOT CARE about the binary log " + "position of the backup. This option shouldn't be used if there " + "are any DDL statements being executed or if any updates are " + "happening on non-InnoDB tables (this includes the system MyISAM " + "tables in the mysql database), otherwise it could lead to an " + "inconsistent backup. If you are considering to use --no-lock " + "because your backups are failing to acquire the lock, this could " + "be because of incoming replication events preventing the lock " + "from succeeding. Please try using --safe-slave-backup to " + "momentarily stop the replication slave thread, this may help " + "the backup to succeed and you then don't need to resort to " + "using this option.", + (uchar *) &opt_no_lock, (uchar *) &opt_no_lock, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"safe-slave-backup", OPT_SAFE_SLAVE_BACKUP, "Stop slave SQL thread " + "and wait to start backup until Slave_open_temp_tables in " + "\"SHOW STATUS\" is zero. If there are no open temporary tables, " + "the backup will take place, otherwise the SQL thread will be " + "started and stopped until there are no open temporary tables. " + "The backup will fail if Slave_open_temp_tables does not become " + "zero after --safe-slave-backup-timeout seconds. The slave SQL " + "thread will be restarted when the backup finishes.", + (uchar *) &opt_safe_slave_backup, + (uchar *) &opt_safe_slave_backup, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"rsync", OPT_RSYNC, "Uses the rsync utility to optimize local file " + "transfers. When this option is specified, innobackupex uses rsync " + "to copy all non-InnoDB files instead of spawning a separate cp for " + "each file, which can be much faster for servers with a large number " + "of databases or tables. This option cannot be used together with " + "--stream.", + (uchar *) &opt_rsync, (uchar *) &opt_rsync, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"force-non-empty-directories", OPT_FORCE_NON_EMPTY_DIRS, "This " + "option, when specified, makes --copy-back or --move-back transfer " + "files to non-empty directories. Note that no existing files will be " + "overwritten. If --copy-back or --nove-back has to copy a file from " + "the backup directory which already exists in the destination " + "directory, it will still fail with an error.", + (uchar *) &opt_force_non_empty_dirs, + (uchar *) &opt_force_non_empty_dirs, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"no-version-check", OPT_NO_VERSION_CHECK, "This option disables the " + "version check which is enabled by the --version-check option.", + (uchar *) &opt_noversioncheck, + (uchar *) &opt_noversioncheck, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"no-backup-locks", OPT_NO_BACKUP_LOCKS, "This option controls if " + "backup locks should be used instead of FLUSH TABLES WITH READ LOCK " + "on the backup stage. The option has no effect when backup locks are " + "not supported by the server. This option is enabled by default, " + "disable with --no-backup-locks.", + (uchar *) &opt_no_backup_locks, + (uchar *) &opt_no_backup_locks, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"decompress", OPT_DECOMPRESS, "Decompresses all files with the .qp " + "extension in a backup previously made with the --compress option.", + (uchar *) &opt_decompress, + (uchar *) &opt_decompress, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"user", 'u', "This option specifies the MySQL username used " + "when connecting to the server, if that's not the current user. " + "The option accepts a string argument. See mysql --help for details.", + (uchar*) &opt_user, (uchar*) &opt_user, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"host", 'H', "This option specifies the host to use when " + "connecting to the database server with TCP/IP. The option accepts " + "a string argument. See mysql --help for details.", + (uchar*) &opt_host, (uchar*) &opt_host, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"port", 'P', "This option specifies the port to use when " + "connecting to the database server with TCP/IP. The option accepts " + "a string argument. See mysql --help for details.", + &opt_port, &opt_port, 0, GET_UINT, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"password", 'p', "This option specifies the password to use " + "when connecting to the database. It accepts a string argument. " + "See mysql --help for details.", + 0, 0, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"socket", 'S', "This option specifies the socket to use when " + "connecting to the local database server with a UNIX domain socket. " + "The option accepts a string argument. See mysql --help for details.", + (uchar*) &opt_socket, (uchar*) &opt_socket, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"incremental-history-name", OPT_INCREMENTAL_HISTORY_NAME, + "This option specifies the name of the backup series stored in the " + "PERCONA_SCHEMA.xtrabackup_history history record to base an " + "incremental backup on. Xtrabackup will search the history table " + "looking for the most recent (highest innodb_to_lsn), successful " + "backup in the series and take the to_lsn value to use as the " + "starting lsn for the incremental backup. This will be mutually " + "exclusive with --incremental-history-uuid, --incremental-basedir " + "and --incremental-lsn. If no valid lsn can be found (no series by " + "that name, no successful backups by that name) xtrabackup will " + "return with an error. It is used with the --incremental option.", + (uchar*) &opt_incremental_history_name, + (uchar*) &opt_incremental_history_name, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"incremental-history-uuid", OPT_INCREMENTAL_HISTORY_UUID, + "This option specifies the UUID of the specific history record " + "stored in the PERCONA_SCHEMA.xtrabackup_history to base an " + "incremental backup on. --incremental-history-name, " + "--incremental-basedir and --incremental-lsn. If no valid lsn can be " + "found (no success record with that uuid) xtrabackup will return " + "with an error. It is used with the --incremental option.", + (uchar*) &opt_incremental_history_uuid, + (uchar*) &opt_incremental_history_uuid, 0, GET_STR, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"decrypt", OPT_DECRYPT, "Decrypts all files with the .xbcrypt " + "extension in a backup previously made with --encrypt option.", + &opt_decrypt_algo, &opt_decrypt_algo, + &xtrabackup_encrypt_algo_typelib, GET_ENUM, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, + + {"remove-original", OPT_REMOVE_ORIGINAL, "Remove .qp and .xbcrypt files " + "after decryption and decompression.", + (uchar *) &opt_remove_original, + (uchar *) &opt_remove_original, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + + {"ftwrl-wait-query-type", OPT_LOCK_WAIT_QUERY_TYPE, + "This option specifies which types of queries are allowed to complete " + "before innobackupex will issue the global lock. Default is all.", + (uchar*) &opt_lock_wait_query_type, + (uchar*) &opt_lock_wait_query_type, &query_type_typelib, + GET_ENUM, REQUIRED_ARG, QUERY_TYPE_ALL, 0, 0, 0, 0, 0}, + + {"kill-long-query-type", OPT_KILL_LONG_QUERY_TYPE, + "This option specifies which types of queries should be killed to " + "unblock the global lock. Default is \"all\".", + (uchar*) &opt_kill_long_query_type, + (uchar*) &opt_kill_long_query_type, &query_type_typelib, + GET_ENUM, REQUIRED_ARG, QUERY_TYPE_SELECT, 0, 0, 0, 0, 0}, + + {"history", OPT_HISTORY, + "This option enables the tracking of backup history in the " + "PERCONA_SCHEMA.xtrabackup_history table. An optional history " + "series name may be specified that will be placed with the history " + "record for the current backup being taken.", + NULL, NULL, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, + + {"kill-long-queries-timeout", OPT_KILL_LONG_QUERIES_TIMEOUT, + "This option specifies the number of seconds innobackupex waits " + "between starting FLUSH TABLES WITH READ LOCK and killing those " + "queries that block it. Default is 0 seconds, which means " + "innobackupex will not attempt to kill any queries.", + (uchar*) &opt_kill_long_queries_timeout, + (uchar*) &opt_kill_long_queries_timeout, 0, GET_UINT, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"ftwrl-wait-timeout", OPT_LOCK_WAIT_TIMEOUT, + "This option specifies time in seconds that innobackupex should wait " + "for queries that would block FTWRL before running it. If there are " + "still such queries when the timeout expires, innobackupex terminates " + "with an error. Default is 0, in which case innobackupex does not " + "wait for queries to complete and starts FTWRL immediately.", + (uchar*) &opt_lock_wait_timeout, + (uchar*) &opt_lock_wait_timeout, 0, GET_UINT, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"ftwrl-wait-threshold", OPT_LOCK_WAIT_THRESHOLD, + "This option specifies the query run time threshold which is used by " + "innobackupex to detect long-running queries with a non-zero value " + "of --ftwrl-wait-timeout. FTWRL is not started until such " + "long-running queries exist. This option has no effect if " + "--ftwrl-wait-timeout is 0. Default value is 60 seconds.", + (uchar*) &opt_lock_wait_threshold, + (uchar*) &opt_lock_wait_threshold, 0, GET_UINT, + REQUIRED_ARG, 60, 0, 0, 0, 0, 0}, + + {"debug-sleep-before-unlock", OPT_DEBUG_SLEEP_BEFORE_UNLOCK, + "This is a debug-only option used by the XtraBackup test suite.", + (uchar*) &opt_debug_sleep_before_unlock, + (uchar*) &opt_debug_sleep_before_unlock, 0, GET_UINT, + REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"safe-slave-backup-timeout", OPT_SAFE_SLAVE_BACKUP_TIMEOUT, + "How many seconds --safe-slave-backup should wait for " + "Slave_open_temp_tables to become zero. (default 300)", + (uchar*) &opt_safe_slave_backup_timeout, + (uchar*) &opt_safe_slave_backup_timeout, 0, GET_UINT, + REQUIRED_ARG, 300, 0, 0, 0, 0, 0}, + + {"binlog-info", OPT_BINLOG_INFO, + "This option controls how XtraBackup should retrieve server's binary log " + "coordinates corresponding to the backup. Possible values are OFF, ON, " + "LOCKLESS and AUTO. See the XtraBackup manual for more information", + &opt_binlog_info, &opt_binlog_info, + &binlog_info_typelib, GET_ENUM, OPT_ARG, BINLOG_INFO_AUTO, 0, 0, 0, 0, 0}, + + {"secure-auth", OPT_XB_SECURE_AUTH, "Refuse client connecting to server if it" + " uses old (pre-4.1.1) protocol.", &opt_secure_auth, + &opt_secure_auth, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0}, + +#include "sslopt-longopts.h" + + + { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} +}; + +uint xb_client_options_count = array_elements(xb_client_options); + +struct my_option xb_server_options[] = +{ + {"datadir", 'h', "Path to the database root.", (G_PTR*) &mysql_data_home, + (G_PTR*) &mysql_data_home, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"tmpdir", 't', + "Path for temporary files. Several paths may be specified, separated by a " +#if defined(__WIN__) || defined(OS2) || defined(__NETWARE__) + "semicolon (;)" +#else + "colon (:)" +#endif + ", in this case they are used in a round-robin fashion.", + (G_PTR*) &opt_mysql_tmpdir, + (G_PTR*) &opt_mysql_tmpdir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"parallel", OPT_XTRA_PARALLEL, + "Number of threads to use for parallel datafiles transfer. " + "The default value is 1.", + (G_PTR*) &xtrabackup_parallel, (G_PTR*) &xtrabackup_parallel, 0, GET_INT, + REQUIRED_ARG, 1, 1, INT_MAX, 0, 0, 0}, + + {"log", OPT_LOG, "Ignored option for MySQL option compatibility", + (G_PTR*) &log_ignored_opt, (G_PTR*) &log_ignored_opt, 0, + GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, + + {"log_bin", OPT_LOG, "Base name for the log sequence", + &opt_log_bin, &opt_log_bin, 0, GET_STR_ALLOC, OPT_ARG, 0, 0, 0, 0, 0, 0}, + + {"innodb", OPT_INNODB, "Ignored option for MySQL option compatibility", + (G_PTR*) &innobase_ignored_opt, (G_PTR*) &innobase_ignored_opt, 0, + GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, + + {"innodb_adaptive_hash_index", OPT_INNODB_ADAPTIVE_HASH_INDEX, + "Enable InnoDB adaptive hash index (enabled by default). " + "Disable with --skip-innodb-adaptive-hash-index.", + (G_PTR*) &innobase_adaptive_hash_index, + (G_PTR*) &innobase_adaptive_hash_index, + 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0}, + {"innodb_additional_mem_pool_size", OPT_INNODB_ADDITIONAL_MEM_POOL_SIZE, + "Size of a memory pool InnoDB uses to store data dictionary information and other internal data structures.", + (G_PTR*) &innobase_additional_mem_pool_size, + (G_PTR*) &innobase_additional_mem_pool_size, 0, GET_LONG, REQUIRED_ARG, + 1*1024*1024L, 512*1024L, LONG_MAX, 0, 1024, 0}, + {"innodb_autoextend_increment", OPT_INNODB_AUTOEXTEND_INCREMENT, + "Data file autoextend increment in megabytes", + (G_PTR*) &srv_auto_extend_increment, + (G_PTR*) &srv_auto_extend_increment, + 0, GET_ULONG, REQUIRED_ARG, 8L, 1L, 1000L, 0, 1L, 0}, + {"innodb_buffer_pool_size", OPT_INNODB_BUFFER_POOL_SIZE, + "The size of the memory buffer InnoDB uses to cache data and indexes of its tables.", + (G_PTR*) &innobase_buffer_pool_size, (G_PTR*) &innobase_buffer_pool_size, 0, + GET_LL, REQUIRED_ARG, 8*1024*1024L, 1024*1024L, LONGLONG_MAX, 0, + 1024*1024L, 0}, + {"innodb_checksums", OPT_INNODB_CHECKSUMS, "Enable InnoDB checksums validation (enabled by default). \ +Disable with --skip-innodb-checksums.", (G_PTR*) &innobase_use_checksums, + (G_PTR*) &innobase_use_checksums, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0}, + {"innodb_data_file_path", OPT_INNODB_DATA_FILE_PATH, + "Path to individual files and their sizes.", &innobase_data_file_path, + &innobase_data_file_path, 0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"innodb_data_home_dir", OPT_INNODB_DATA_HOME_DIR, + "The common part for InnoDB table spaces.", &innobase_data_home_dir, + &innobase_data_home_dir, 0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"innodb_doublewrite", OPT_INNODB_DOUBLEWRITE, "Enable InnoDB doublewrite buffer (enabled by default). \ +Disable with --skip-innodb-doublewrite.", (G_PTR*) &innobase_use_doublewrite, + (G_PTR*) &innobase_use_doublewrite, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0}, + {"innodb_io_capacity", OPT_INNODB_IO_CAPACITY, + "Number of IOPs the server can do. Tunes the background IO rate", + (G_PTR*) &srv_io_capacity, (G_PTR*) &srv_io_capacity, + 0, GET_ULONG, OPT_ARG, 200, 100, ~0UL, 0, 0, 0}, + {"innodb_file_io_threads", OPT_INNODB_FILE_IO_THREADS, + "Number of file I/O threads in InnoDB.", (G_PTR*) &innobase_file_io_threads, + (G_PTR*) &innobase_file_io_threads, 0, GET_LONG, REQUIRED_ARG, 4, 4, 64, 0, + 1, 0}, + {"innodb_read_io_threads", OPT_INNODB_READ_IO_THREADS, + "Number of background read I/O threads in InnoDB.", (G_PTR*) &innobase_read_io_threads, + (G_PTR*) &innobase_read_io_threads, 0, GET_LONG, REQUIRED_ARG, 4, 1, 64, 0, + 1, 0}, + {"innodb_write_io_threads", OPT_INNODB_WRITE_IO_THREADS, + "Number of background write I/O threads in InnoDB.", (G_PTR*) &innobase_write_io_threads, + (G_PTR*) &innobase_write_io_threads, 0, GET_LONG, REQUIRED_ARG, 4, 1, 64, 0, + 1, 0}, + {"innodb_file_per_table", OPT_INNODB_FILE_PER_TABLE, + "Stores each InnoDB table to an .ibd file in the database dir.", + (G_PTR*) &innobase_file_per_table, + (G_PTR*) &innobase_file_per_table, 0, GET_BOOL, NO_ARG, + FALSE, 0, 0, 0, 0, 0}, + + {"innodb_flush_method", OPT_INNODB_FLUSH_METHOD, + "With which method to flush data.", (G_PTR*) &innobase_unix_file_flush_method, + (G_PTR*) &innobase_unix_file_flush_method, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, + 0, 0, 0}, + +/* ####### Should we use this option? ####### */ + {"innodb_force_recovery", OPT_INNODB_FORCE_RECOVERY, + "Helps to save your data in case the disk image of the database becomes corrupt.", + (G_PTR*) &innobase_force_recovery, (G_PTR*) &innobase_force_recovery, 0, + GET_LONG, REQUIRED_ARG, 0, 0, 6, 0, 1, 0}, + + {"innodb_log_arch_dir", OPT_INNODB_LOG_ARCH_DIR, + "Where full logs should be archived.", (G_PTR*) &innobase_log_arch_dir, + (G_PTR*) &innobase_log_arch_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"innodb_log_buffer_size", OPT_INNODB_LOG_BUFFER_SIZE, + "The size of the buffer which InnoDB uses to write log to the log files on disk.", + (G_PTR*) &innobase_log_buffer_size, (G_PTR*) &innobase_log_buffer_size, 0, + GET_LONG, REQUIRED_ARG, 1024*1024L, 256*1024L, LONG_MAX, 0, 1024, 0}, + {"innodb_log_file_size", OPT_INNODB_LOG_FILE_SIZE, + "Size of each log file in a log group.", + (G_PTR*) &innobase_log_file_size, (G_PTR*) &innobase_log_file_size, 0, + GET_LL, REQUIRED_ARG, 48*1024*1024L, 1*1024*1024L, LONGLONG_MAX, 0, + 1024*1024L, 0}, + {"innodb_log_files_in_group", OPT_INNODB_LOG_FILES_IN_GROUP, + "Number of log files in the log group. InnoDB writes to the files in a " + "circular fashion. Value 3 is recommended here.", + &innobase_log_files_in_group, &innobase_log_files_in_group, + 0, GET_LONG, REQUIRED_ARG, 2, 2, 100, 0, 1, 0}, + {"innodb_log_group_home_dir", OPT_INNODB_LOG_GROUP_HOME_DIR, + "Path to InnoDB log files.", &srv_log_group_home_dir, + &srv_log_group_home_dir, 0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"innodb_max_dirty_pages_pct", OPT_INNODB_MAX_DIRTY_PAGES_PCT, + "Percentage of dirty pages allowed in bufferpool.", (G_PTR*) &srv_max_buf_pool_modified_pct, + (G_PTR*) &srv_max_buf_pool_modified_pct, 0, GET_ULONG, REQUIRED_ARG, 90, 0, 100, 0, 0, 0}, + {"innodb_open_files", OPT_INNODB_OPEN_FILES, + "How many files at the maximum InnoDB keeps open at the same time.", + (G_PTR*) &innobase_open_files, (G_PTR*) &innobase_open_files, 0, + GET_LONG, REQUIRED_ARG, 300L, 10L, LONG_MAX, 0, 1L, 0}, + {"innodb_use_native_aio", OPT_INNODB_USE_NATIVE_AIO, + "Use native AIO if supported on this platform.", + (G_PTR*) &srv_use_native_aio, + (G_PTR*) &srv_use_native_aio, 0, GET_BOOL, NO_ARG, + FALSE, 0, 0, 0, 0, 0}, + {"innodb_page_size", OPT_INNODB_PAGE_SIZE, + "The universal page size of the database.", + (G_PTR*) &innobase_page_size, (G_PTR*) &innobase_page_size, 0, + /* Use GET_LL to support numeric suffixes in 5.6 */ + GET_LL, REQUIRED_ARG, + (1LL << 14), (1LL << 12), (1LL << UNIV_PAGE_SIZE_SHIFT_MAX), 0, 1L, 0}, + {"innodb_log_block_size", OPT_INNODB_LOG_BLOCK_SIZE, + "The log block size of the transaction log file. " + "Changing for created log file is not supported. Use on your own risk!", + (G_PTR*) &innobase_log_block_size, (G_PTR*) &innobase_log_block_size, 0, + GET_ULONG, REQUIRED_ARG, 512, 512, 1 << UNIV_PAGE_SIZE_SHIFT_MAX, 0, 1L, 0}, + {"innodb_fast_checksum", OPT_INNODB_FAST_CHECKSUM, + "Change the algorithm of checksum for the whole of datapage to 4-bytes word based.", + (G_PTR*) &innobase_fast_checksum, + (G_PTR*) &innobase_fast_checksum, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"innodb_doublewrite_file", OPT_INNODB_DOUBLEWRITE_FILE, + "Path to special datafile for doublewrite buffer. (default is "": not used)", + (G_PTR*) &innobase_doublewrite_file, (G_PTR*) &innobase_doublewrite_file, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"innodb_buffer_pool_filename", OPT_INNODB_BUFFER_POOL_FILENAME, + "Filename to/from which to dump/load the InnoDB buffer pool", + (G_PTR*) &innobase_buffer_pool_filename, + (G_PTR*) &innobase_buffer_pool_filename, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + +#ifndef __WIN__ + {"debug-sync", OPT_XTRA_DEBUG_SYNC, + "Debug sync point. This is only used by the xtrabackup test suite", + (G_PTR*) &xtrabackup_debug_sync, + (G_PTR*) &xtrabackup_debug_sync, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, +#endif + + {"innodb_checksum_algorithm", OPT_INNODB_CHECKSUM_ALGORITHM, + "The algorithm InnoDB uses for page checksumming. [CRC32, STRICT_CRC32, " + "INNODB, STRICT_INNODB, NONE, STRICT_NONE]", &srv_checksum_algorithm, + &srv_checksum_algorithm, &innodb_checksum_algorithm_typelib, GET_ENUM, + REQUIRED_ARG, SRV_CHECKSUM_ALGORITHM_INNODB, 0, 0, 0, 0, 0}, + {"innodb_log_checksum_algorithm", OPT_INNODB_LOG_CHECKSUM_ALGORITHM, + "The algorithm InnoDB uses for log checksumming. [CRC32, STRICT_CRC32, " + "INNODB, STRICT_INNODB, NONE, STRICT_NONE]", &srv_log_checksum_algorithm, + &srv_log_checksum_algorithm, &innodb_checksum_algorithm_typelib, GET_ENUM, + REQUIRED_ARG, SRV_CHECKSUM_ALGORITHM_INNODB, 0, 0, 0, 0, 0}, + {"innodb_undo_directory", OPT_INNODB_UNDO_DIRECTORY, + "Directory where undo tablespace files live, this path can be absolute.", + &srv_undo_dir, &srv_undo_dir, 0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, + 0}, + + {"innodb_undo_tablespaces", OPT_INNODB_UNDO_TABLESPACES, + "Number of undo tablespaces to use.", + (G_PTR*)&srv_undo_tablespaces, (G_PTR*)&srv_undo_tablespaces, + 0, GET_ULONG, REQUIRED_ARG, 0, 0, 126, 0, 1, 0}, + + {"defaults_group", OPT_DEFAULTS_GROUP, "defaults group in config file (default \"mysqld\").", + (G_PTR*) &defaults_group, (G_PTR*) &defaults_group, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + + {"plugin-dir", OPT_PLUGIN_DIR, "Server plugin directory", + &xb_plugin_dir, &xb_plugin_dir, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 }, + + { "plugin-load", OPT_PLUGIN_LOAD, "encrypton plugin to load", + &xb_plugin_load, &xb_plugin_load, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 }, + + { "innodb-encrypt-log", OPT_INNODB_ENCRYPT_LOG, "encrypton plugin to load", + &srv_encrypt_log, &srv_encrypt_log, + 0, GET_BOOL, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 }, + + {"open_files_limit", OPT_OPEN_FILES_LIMIT, "the maximum number of file " + "descriptors to reserve with setrlimit().", + (G_PTR*) &xb_open_files_limit, (G_PTR*) &xb_open_files_limit, 0, GET_ULONG, + REQUIRED_ARG, 0, 0, UINT_MAX, 0, 1, 0}, + + { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} +}; + +uint xb_server_options_count = array_elements(xb_server_options); + +#ifndef __WIN__ +static int debug_sync_resumed; + +static void sigcont_handler(int sig); + +static void sigcont_handler(int sig __attribute__((unused))) +{ + debug_sync_resumed= 1; +} +#endif + +static inline +void +debug_sync_point(const char *name) +{ +#ifndef __WIN__ + FILE *fp; + pid_t pid; + char pid_path[FN_REFLEN]; + + if (xtrabackup_debug_sync == NULL) { + return; + } + + if (strcmp(xtrabackup_debug_sync, name)) { + return; + } + + pid = getpid(); + + snprintf(pid_path, sizeof(pid_path), "%s/xtrabackup_debug_sync", + xtrabackup_target_dir); + fp = fopen(pid_path, "w"); + if (fp == NULL) { + msg("xtrabackup: Error: cannot open %s\n", pid_path); + exit(EXIT_FAILURE); + } + fprintf(fp, "%u\n", (uint) pid); + fclose(fp); + + msg("xtrabackup: DEBUG: Suspending at debug sync point '%s'. " + "Resume with 'kill -SIGCONT %u'.\n", name, (uint) pid); + + debug_sync_resumed= 0; + kill(pid, SIGSTOP); + while (!debug_sync_resumed) { + sleep(1); + } + + /* On resume */ + msg("xtrabackup: DEBUG: removing the pid file.\n"); + my_delete(pid_path, MYF(MY_WME)); +#endif +} + +static const char *xb_client_default_groups[]= + { "xtrabackup", "client", 0, 0, 0 }; + +static const char *xb_server_default_groups[]= + { "xtrabackup", "mysqld", 0, 0, 0 }; + +static void print_version(void) +{ + msg("%s based on MariaDB server %s %s (%s) \n", + my_progname, MYSQL_SERVER_VERSION, SYSTEM_TYPE, MACHINE_TYPE); +} + +static void usage(void) +{ + puts("Open source backup tool for InnoDB and XtraDB\n\ +\n\ +Copyright (C) 2009-2015 Percona LLC and/or its affiliates.\n\ +Portions Copyright (C) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved.\n\ +\n\ +This program is free software; you can redistribute it and/or\n\ +modify it under the terms of the GNU General Public License\n\ +as published by the Free Software Foundation version 2\n\ +of the License.\n\ +\n\ +This program is distributed in the hope that it will be useful,\n\ +but WITHOUT ANY WARRANTY; without even the implied warranty of\n\ +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n\ +GNU General Public License for more details.\n\ +\n\ +You can download full text of the license on http://www.gnu.org/licenses/gpl-2.0.txt\n"); + + printf("Usage: [%s [--defaults-file=#] --backup | %s [--defaults-file=#] --prepare] [OPTIONS]\n",my_progname,my_progname); + print_defaults("my", xb_server_default_groups); + my_print_help(xb_client_options); + my_print_help(xb_server_options); + my_print_variables(xb_server_options); + my_print_variables(xb_client_options); +} + +#define ADD_PRINT_PARAM_OPT(value) \ + { \ + print_param_str << opt->name << "=" << value << "\n"; \ + param_set.insert(opt->name); \ + } + +/************************************************************************ +Check if parameter is set in defaults file or via command line argument +@return true if parameter is set. */ +bool +check_if_param_set(const char *param) +{ + return param_set.find(param) != param_set.end(); +} + +my_bool +xb_get_one_option(int optid, + const struct my_option *opt __attribute__((unused)), + char *argument) +{ + switch(optid) { + case 'h': + strmake(mysql_real_data_home,argument, FN_REFLEN - 1); + mysql_data_home= mysql_real_data_home; + + ADD_PRINT_PARAM_OPT(mysql_real_data_home); + break; + + case 't': + + ADD_PRINT_PARAM_OPT(opt_mysql_tmpdir); + break; + + case OPT_INNODB_DATA_HOME_DIR: + + ADD_PRINT_PARAM_OPT(innobase_data_home_dir); + break; + + case OPT_INNODB_DATA_FILE_PATH: + + ADD_PRINT_PARAM_OPT(innobase_data_file_path); + break; + + case OPT_INNODB_LOG_GROUP_HOME_DIR: + + ADD_PRINT_PARAM_OPT(srv_log_group_home_dir); + break; + + case OPT_INNODB_LOG_FILES_IN_GROUP: + + ADD_PRINT_PARAM_OPT(innobase_log_files_in_group); + break; + + case OPT_INNODB_LOG_FILE_SIZE: + + ADD_PRINT_PARAM_OPT(innobase_log_file_size); + break; + + case OPT_INNODB_FLUSH_METHOD: + + ADD_PRINT_PARAM_OPT(innobase_unix_file_flush_method); + break; + + case OPT_INNODB_PAGE_SIZE: + + ADD_PRINT_PARAM_OPT(innobase_page_size); + break; + + case OPT_INNODB_FAST_CHECKSUM: + + ADD_PRINT_PARAM_OPT(!!innobase_fast_checksum); + break; + + case OPT_INNODB_LOG_BLOCK_SIZE: + + ADD_PRINT_PARAM_OPT(innobase_log_block_size); + break; + + case OPT_INNODB_DOUBLEWRITE_FILE: + + ADD_PRINT_PARAM_OPT(innobase_doublewrite_file); + break; + + case OPT_INNODB_UNDO_DIRECTORY: + + ADD_PRINT_PARAM_OPT(srv_undo_dir); + break; + + case OPT_INNODB_UNDO_TABLESPACES: + + ADD_PRINT_PARAM_OPT(srv_undo_tablespaces); + break; + + case OPT_INNODB_CHECKSUM_ALGORITHM: + + ut_a(srv_checksum_algorithm <= SRV_CHECKSUM_ALGORITHM_STRICT_NONE); + + ADD_PRINT_PARAM_OPT(innodb_checksum_algorithm_names[srv_checksum_algorithm]); + break; + + case OPT_INNODB_LOG_CHECKSUM_ALGORITHM: + + ut_a(srv_log_checksum_algorithm <= SRV_CHECKSUM_ALGORITHM_STRICT_NONE); + + ADD_PRINT_PARAM_OPT(innodb_checksum_algorithm_names[srv_log_checksum_algorithm]); + break; + + case OPT_INNODB_BUFFER_POOL_FILENAME: + + ADD_PRINT_PARAM_OPT(innobase_buffer_pool_filename); + break; + + case OPT_XTRA_TARGET_DIR: + strmake(xtrabackup_real_target_dir,argument, sizeof(xtrabackup_real_target_dir)-1); + xtrabackup_target_dir= xtrabackup_real_target_dir; + break; + case OPT_XTRA_STREAM: + if (!strcasecmp(argument, "tar")) + xtrabackup_stream_fmt = XB_STREAM_FMT_TAR; + else if (!strcasecmp(argument, "xbstream")) + xtrabackup_stream_fmt = XB_STREAM_FMT_XBSTREAM; + else + { + msg("Invalid --stream argument: %s\n", argument); + return 1; + } + xtrabackup_stream = TRUE; + break; + case OPT_XTRA_COMPRESS: + if (argument == NULL) + xtrabackup_compress_alg = "quicklz"; + else if (strcasecmp(argument, "quicklz")) + { + msg("Invalid --compress argument: %s\n", argument); + return 1; + } + xtrabackup_compress = TRUE; + break; + case OPT_XTRA_ENCRYPT: + if (argument == NULL) + { + msg("Missing --encrypt argument, must specify a valid encryption " + " algorithm.\n"); + return 1; + } + xtrabackup_encrypt = TRUE; + break; + case OPT_DECRYPT: + if (argument == NULL) { + msg("Missing --decrypt argument, must specify a " + "valid encryption algorithm.\n"); + return(1); + } + opt_decrypt = TRUE; + xtrabackup_decrypt_decompress = true; + break; + case OPT_DECOMPRESS: + opt_decompress = TRUE; + xtrabackup_decrypt_decompress = true; + break; + case (int) OPT_CORE_FILE: + test_flags |= TEST_CORE_ON_SIGNAL; + break; + case OPT_HISTORY: + if (argument) { + opt_history = argument; + } else { + opt_history = ""; + } + break; + case 'p': + if (argument) + { + char *start= argument; + my_free(opt_password); + opt_password= my_strdup(argument, MYF(MY_FAE)); + while (*argument) *argument++= 'x'; // Destroy argument + if (*start) + start[1]=0 ; + } + break; + + +#include "sslopt-case.h" + + case '?': + usage(); + exit(EXIT_SUCCESS); + break; + case 'v': + print_version(); + exit(EXIT_SUCCESS); + break; + default: + break; + } + return 0; +} + +/*********************************************************************** +Initializes log_block_size */ +static +ibool +xb_init_log_block_size(void) +{ + srv_log_block_size = 0; + if (innobase_log_block_size != 512) { + uint n_shift = (uint)get_bit_shift(innobase_log_block_size);; + + if (n_shift > 0) { + srv_log_block_size = (ulint)(1LL << n_shift); + msg("InnoDB: The log block size is set to %lu.\n", + srv_log_block_size); + } + } else { + srv_log_block_size = 512; + } + if (!srv_log_block_size) { + msg("InnoDB: Error: %lu is not valid value for " + "innodb_log_block_size.\n", innobase_log_block_size); + return FALSE; + } + + return TRUE; +} + +static my_bool +innodb_init_param(void) +{ + /* innobase_init */ + static char current_dir[3]; /* Set if using current lib */ + my_bool ret; + char *default_path; + srv_is_being_started = TRUE; + /* === some variables from mysqld === */ + memset((G_PTR) &mysql_tmpdir_list, 0, sizeof(mysql_tmpdir_list)); + + if (init_tmpdir(&mysql_tmpdir_list, opt_mysql_tmpdir)) + exit(EXIT_FAILURE); + xtrabackup_tmpdir = my_tmpdir(&mysql_tmpdir_list); + /* dummy for initialize all_charsets[] */ + get_charset_name(0); + + srv_page_size = 0; + srv_page_size_shift = 0; + + if (innobase_page_size != (1LL << 14)) { + int n_shift = (int)get_bit_shift((ulint) innobase_page_size); + + if (n_shift >= 12 && n_shift <= UNIV_PAGE_SIZE_SHIFT_MAX) { + srv_page_size_shift = n_shift; + srv_page_size = 1 << n_shift; + msg("InnoDB: The universal page size of the " + "database is set to %lu.\n", srv_page_size); + } else { + msg("InnoDB: Error: invalid value of " + "innobase_page_size: %lld", innobase_page_size); + exit(EXIT_FAILURE); + } + } else { + srv_page_size_shift = 14; + srv_page_size = (1 << srv_page_size_shift); + } + + if (!xb_init_log_block_size()) { + goto error; + } + + /* Check that values don't overflow on 32-bit systems. */ + if (sizeof(ulint) == 4) { + if (xtrabackup_use_memory > UINT_MAX32) { + msg("xtrabackup: use-memory can't be over 4GB" + " on 32-bit systems\n"); + } + + if (innobase_buffer_pool_size > UINT_MAX32) { + msg("xtrabackup: innobase_buffer_pool_size can't be " + "over 4GB on 32-bit systems\n"); + + goto error; + } + + if (innobase_log_file_size > UINT_MAX32) { + msg("xtrabackup: innobase_log_file_size can't be " + "over 4GB on 32-bit systemsi\n"); + + goto error; + } + } + + os_innodb_umask = (ulint)0664; + + /* First calculate the default path for innodb_data_home_dir etc., + in case the user has not given any value. + + Note that when using the embedded server, the datadirectory is not + necessarily the current directory of this program. */ + + /* It's better to use current lib, to keep paths short */ + current_dir[0] = FN_CURLIB; + current_dir[1] = FN_LIBCHAR; + current_dir[2] = 0; + default_path = current_dir; + + ut_a(default_path); + + /* Set InnoDB initialization parameters according to the values + read from MySQL .cnf file */ + + if (xtrabackup_backup || xtrabackup_stats) { + msg("xtrabackup: using the following InnoDB configuration:\n"); + } else { + msg("xtrabackup: using the following InnoDB configuration " + "for recovery:\n"); + } + + /*--------------- Data files -------------------------*/ + + /* The default dir for data files is the datadir of MySQL */ + + srv_data_home = ((xtrabackup_backup || xtrabackup_stats) && innobase_data_home_dir + ? innobase_data_home_dir : default_path); + msg("xtrabackup: innodb_data_home_dir = %s\n", srv_data_home); + + /* Set default InnoDB data file size to 10 MB and let it be + auto-extending. Thus users can use InnoDB in >= 4.0 without having + to specify any startup options. */ + + if (!innobase_data_file_path) { + innobase_data_file_path = (char*) "ibdata1:10M:autoextend"; + } + msg("xtrabackup: innodb_data_file_path = %s\n", + innobase_data_file_path); + + /* Since InnoDB edits the argument in the next call, we make another + copy of it: */ + + internal_innobase_data_file_path = strdup(innobase_data_file_path); + + ret = (my_bool) srv_parse_data_file_paths_and_sizes( + internal_innobase_data_file_path); + if (ret == FALSE) { + msg("xtrabackup: syntax error in innodb_data_file_path\n"); +mem_free_and_error: + free(internal_innobase_data_file_path); + internal_innobase_data_file_path = NULL; + goto error; + } + + if (xtrabackup_prepare) { + /* "--prepare" needs filenames only */ + ulint i; + + for (i=0; i < srv_n_data_files; i++) { + char *p; + + p = srv_data_file_names[i]; + while ((p = strchr(p, SRV_PATH_SEPARATOR)) != NULL) + { + p++; + srv_data_file_names[i] = p; + } + } + } + + /* -------------- Log files ---------------------------*/ + + /* The default dir for log files is the datadir of MySQL */ + + if (!((xtrabackup_backup || xtrabackup_stats) && + srv_log_group_home_dir)) { + srv_log_group_home_dir = default_path; + } + if (xtrabackup_prepare && xtrabackup_incremental_dir) { + srv_log_group_home_dir = xtrabackup_incremental_dir; + } + msg("xtrabackup: innodb_log_group_home_dir = %s\n", + srv_log_group_home_dir); + + srv_normalize_path_for_win(srv_log_group_home_dir); + + if (strchr(srv_log_group_home_dir, ';')) { + + msg("syntax error in innodb_log_group_home_dir, "); + + goto mem_free_and_error; + } + + srv_adaptive_flushing = FALSE; + srv_use_sys_malloc = TRUE; + srv_file_format = 1; /* Barracuda */ + srv_max_file_format_at_startup = UNIV_FORMAT_MIN; /* on */ + /* --------------------------------------------------*/ + + srv_file_flush_method_str = innobase_unix_file_flush_method; + + srv_n_log_files = (ulint) innobase_log_files_in_group; + srv_log_file_size = (ulint) innobase_log_file_size; + msg("xtrabackup: innodb_log_files_in_group = %ld\n", + srv_n_log_files); + msg("xtrabackup: innodb_log_file_size = %lld\n", + (long long int) srv_log_file_size); + + srv_log_archive_on = (ulint) innobase_log_archive; + srv_log_buffer_size = (ulint) innobase_log_buffer_size; + + /* We set srv_pool_size here in units of 1 kB. InnoDB internally + changes the value so that it becomes the number of database pages. */ + + //srv_buf_pool_size = (ulint) innobase_buffer_pool_size; + srv_buf_pool_size = (ulint) xtrabackup_use_memory; + + srv_mem_pool_size = (ulint) innobase_additional_mem_pool_size; + + srv_n_file_io_threads = (ulint) innobase_file_io_threads; + srv_n_read_io_threads = (ulint) innobase_read_io_threads; + srv_n_write_io_threads = (ulint) innobase_write_io_threads; + + srv_force_recovery = (ulint) innobase_force_recovery; + + srv_use_doublewrite_buf = (ibool) innobase_use_doublewrite; + + if (!innobase_use_checksums) { + + srv_checksum_algorithm = SRV_CHECKSUM_ALGORITHM_NONE; + } + + btr_search_enabled = (char) innobase_adaptive_hash_index; + btr_search_index_num = 1; + + os_use_large_pages = (ibool) innobase_use_large_pages; + os_large_page_size = (ulint) innobase_large_page_size; + + if (!innobase_log_arch_dir) { + static char default_dir[3] = "./"; + srv_arch_dir = default_dir; + } + row_rollback_on_timeout = (ibool) innobase_rollback_on_timeout; + + srv_file_per_table = (my_bool) innobase_file_per_table; + + srv_locks_unsafe_for_binlog = (ibool) innobase_locks_unsafe_for_binlog; + + srv_max_n_open_files = (ulint) innobase_open_files; + srv_innodb_status = (ibool) innobase_create_status_file; + + srv_print_verbose_log = 1; + + /* Store the default charset-collation number of this MySQL + installation */ + + /* We cannot treat characterset here for now!! */ + data_mysql_default_charset_coll = (ulint)default_charset_info->number; + + ut_a(DATA_MYSQL_LATIN1_SWEDISH_CHARSET_COLL == + my_charset_latin1.number); + ut_a(DATA_MYSQL_BINARY_CHARSET_COLL == my_charset_bin.number); + + /* Store the latin1_swedish_ci character ordering table to InnoDB. For + non-latin1_swedish_ci charsets we use the MySQL comparison functions, + and consequently we do not need to know the ordering internally in + InnoDB. */ + + ut_a(0 == strcmp(my_charset_latin1.name, "latin1_swedish_ci")); + srv_latin1_ordering = my_charset_latin1.sort_order; + + //innobase_commit_concurrency_init_default(); + + /* Since we in this module access directly the fields of a trx + struct, and due to different headers and flags it might happen that + mutex_t has a different size in this module and in InnoDB + modules, we check at run time that the size is the same in + these compilation modules. */ + + /* On 5.5+ srv_use_native_aio is TRUE by default. It is later reset + if it is not supported by the platform in + innobase_start_or_create_for_mysql(). As we don't call it in xtrabackup, + we have to duplicate checks from that function here. */ + +#ifdef __WIN__ + switch (os_get_os_version()) { + case OS_WIN95: + case OS_WIN31: + case OS_WINNT: + /* On Win 95, 98, ME, Win32 subsystem for Windows 3.1, + and NT use simulated aio. In NT Windows provides async i/o, + but when run in conjunction with InnoDB Hot Backup, it seemed + to corrupt the data files. */ + + srv_use_native_aio = FALSE; + break; + + case OS_WIN2000: + case OS_WINXP: + /* On 2000 and XP, async IO is available. */ + srv_use_native_aio = TRUE; + break; + + default: + /* Vista and later have both async IO and condition variables */ + srv_use_native_aio = TRUE; + srv_use_native_conditions = TRUE; + break; + } + +#elif defined(LINUX_NATIVE_AIO) + + if (srv_use_native_aio) { + ut_print_timestamp(stderr); + msg(" InnoDB: Using Linux native AIO\n"); + } +#else + /* Currently native AIO is supported only on windows and linux + and that also when the support is compiled in. In all other + cases, we ignore the setting of innodb_use_native_aio. */ + srv_use_native_aio = FALSE; + +#endif + + /* Assign the default value to srv_undo_dir if it's not specified, as + my_getopt does not support default values for string options. We also + ignore the option and override innodb_undo_directory on --prepare, + because separate undo tablespaces are copied to the root backup + directory. */ + + if (!srv_undo_dir || !xtrabackup_backup) { + my_free(srv_undo_dir); + srv_undo_dir = my_strdup(".", MYF(MY_FAE)); + } + + innodb_log_checksum_func_update(srv_log_checksum_algorithm); + + return(FALSE); + +error: + msg("xtrabackup: innodb_init_param(): Error occured.\n"); + return(TRUE); +} + +static my_bool +innodb_init(void) +{ + int err; + srv_is_being_started = TRUE; + err = innobase_start_or_create_for_mysql(); + + if (err != DB_SUCCESS) { + free(internal_innobase_data_file_path); + internal_innobase_data_file_path = NULL; + goto error; + } + + /* They may not be needed for now */ +// (void) hash_init(&innobase_open_tables,system_charset_info, 32, 0, 0, +// (hash_get_key) innobase_get_key, 0, 0); +// pthread_mutex_init(&innobase_share_mutex, MY_MUTEX_INIT_FAST); +// pthread_mutex_init(&prepare_commit_mutex, MY_MUTEX_INIT_FAST); +// pthread_mutex_init(&commit_threads_m, MY_MUTEX_INIT_FAST); +// pthread_mutex_init(&commit_cond_m, MY_MUTEX_INIT_FAST); +// pthread_cond_init(&commit_cond, NULL); + + innodb_inited= 1; + + return(FALSE); + +error: + msg("xtrabackup: innodb_init(): Error occured.\n"); + return(TRUE); +} + +static my_bool +innodb_end(void) +{ + srv_fast_shutdown = (ulint) innobase_fast_shutdown; + innodb_inited = 0; + + msg("xtrabackup: starting shutdown with innodb_fast_shutdown = %lu\n", + srv_fast_shutdown); + + if (innobase_shutdown_for_mysql() != DB_SUCCESS) { + goto error; + } + free(internal_innobase_data_file_path); + internal_innobase_data_file_path = NULL; + + /* They may not be needed for now */ +// hash_free(&innobase_open_tables); +// pthread_mutex_destroy(&innobase_share_mutex); +// pthread_mutex_destroy(&prepare_commit_mutex); +// pthread_mutex_destroy(&commit_threads_m); +// pthread_mutex_destroy(&commit_cond_m); +// pthread_cond_destroy(&commit_cond); + + return(FALSE); + +error: + msg("xtrabackup: innodb_end(): Error occured.\n"); + return(TRUE); +} + +/* ================= common ================= */ + +/*********************************************************************** +Read backup meta info. +@return TRUE on success, FALSE on failure. */ +static +my_bool +xtrabackup_read_metadata(char *filename) +{ + FILE *fp; + my_bool r = TRUE; + int t; + + fp = fopen(filename,"r"); + if(!fp) { + msg("xtrabackup: Error: cannot open %s\n", filename); + return(FALSE); + } + + if (fscanf(fp, "backup_type = %29s\n", metadata_type) + != 1) { + r = FALSE; + goto end; + } + /* Use UINT64PF instead of LSN_PF here, as we have to maintain the file + format. */ + if (fscanf(fp, "from_lsn = " UINT64PF "\n", &metadata_from_lsn) + != 1) { + r = FALSE; + goto end; + } + if (fscanf(fp, "to_lsn = " UINT64PF "\n", &metadata_to_lsn) + != 1) { + r = FALSE; + goto end; + } + if (fscanf(fp, "last_lsn = " UINT64PF "\n", &metadata_last_lsn) + != 1) { + metadata_last_lsn = 0; + } + /* Optional fields */ + + if (fscanf(fp, "recover_binlog_info = %d\n", &t) == 1) { + recover_binlog_info = (t == 1); + } +end: + fclose(fp); + + return(r); +} + +/*********************************************************************** +Print backup meta info to a specified buffer. */ +static +void +xtrabackup_print_metadata(char *buf, size_t buf_len) +{ + /* Use UINT64PF instead of LSN_PF here, as we have to maintain the file + format. */ + snprintf(buf, buf_len, + "backup_type = %s\n" + "from_lsn = " UINT64PF "\n" + "to_lsn = " UINT64PF "\n" + "last_lsn = " UINT64PF "\n" + "compact = %d\n" + "recover_binlog_info = %d\n", + metadata_type, + metadata_from_lsn, + metadata_to_lsn, + metadata_last_lsn, + MY_TEST(false), + MY_TEST(opt_binlog_info == BINLOG_INFO_LOCKLESS)); +} + +/*********************************************************************** +Stream backup meta info to a specified datasink. +@return TRUE on success, FALSE on failure. */ +static +my_bool +xtrabackup_stream_metadata(ds_ctxt_t *ds_ctxt) +{ + char buf[1024]; + size_t len; + ds_file_t *stream; + MY_STAT mystat; + my_bool rc = TRUE; + + xtrabackup_print_metadata(buf, sizeof(buf)); + + len = strlen(buf); + + mystat.st_size = len; + mystat.st_mtime = my_time(0); + + stream = ds_open(ds_ctxt, XTRABACKUP_METADATA_FILENAME, &mystat); + if (stream == NULL) { + msg("xtrabackup: Error: cannot open output stream " + "for %s\n", XTRABACKUP_METADATA_FILENAME); + return(FALSE); + } + + if (ds_write(stream, buf, len)) { + rc = FALSE; + } + + if (ds_close(stream)) { + rc = FALSE; + } + + return(rc); +} + +/*********************************************************************** +Write backup meta info to a specified file. +@return TRUE on success, FALSE on failure. */ +static +my_bool +xtrabackup_write_metadata(const char *filepath) +{ + char buf[1024]; + size_t len; + FILE *fp; + + xtrabackup_print_metadata(buf, sizeof(buf)); + + len = strlen(buf); + + fp = fopen(filepath, "w"); + if(!fp) { + msg("xtrabackup: Error: cannot open %s\n", filepath); + return(FALSE); + } + if (fwrite(buf, len, 1, fp) < 1) { + fclose(fp); + return(FALSE); + } + + fclose(fp); + + return(TRUE); +} + +/*********************************************************************** +Read meta info for an incremental delta. +@return TRUE on success, FALSE on failure. */ +static my_bool +xb_read_delta_metadata(const char *filepath, xb_delta_info_t *info) +{ + FILE* fp; + char key[51]; + char value[51]; + my_bool r = TRUE; + + /* set defaults */ + info->page_size = ULINT_UNDEFINED; + info->zip_size = ULINT_UNDEFINED; + info->space_id = ULINT_UNDEFINED; + + fp = fopen(filepath, "r"); + if (!fp) { + /* Meta files for incremental deltas are optional */ + return(TRUE); + } + + while (!feof(fp)) { + if (fscanf(fp, "%50s = %50s\n", key, value) == 2) { + if (strcmp(key, "page_size") == 0) { + info->page_size = strtoul(value, NULL, 10); + } else if (strcmp(key, "zip_size") == 0) { + info->zip_size = strtoul(value, NULL, 10); + } else if (strcmp(key, "space_id") == 0) { + info->space_id = strtoul(value, NULL, 10); + } + } + } + + fclose(fp); + + if (info->page_size == ULINT_UNDEFINED) { + msg("xtrabackup: page_size is required in %s\n", filepath); + r = FALSE; + } + if (info->space_id == ULINT_UNDEFINED) { + msg("xtrabackup: Warning: This backup was taken with XtraBackup 2.0.1 " + "or earlier, some DDL operations between full and incremental " + "backups may be handled incorrectly\n"); + } + + return(r); +} + +/*********************************************************************** +Write meta info for an incremental delta. +@return TRUE on success, FALSE on failure. */ +my_bool +xb_write_delta_metadata(const char *filename, const xb_delta_info_t *info) +{ + ds_file_t *f; + char buf[64]; + my_bool ret; + size_t len; + MY_STAT mystat; + + snprintf(buf, sizeof(buf), + "page_size = %lu\n" + "zip_size = %lu\n" + "space_id = %lu\n", + info->page_size, info->zip_size, info->space_id); + len = strlen(buf); + + mystat.st_size = len; + mystat.st_mtime = my_time(0); + + f = ds_open(ds_meta, filename, &mystat); + if (f == NULL) { + msg("xtrabackup: Error: cannot open output stream for %s\n", + filename); + return(FALSE); + } + + ret = (ds_write(f, buf, len) == 0); + + if (ds_close(f)) { + ret = FALSE; + } + + return(ret); +} + +/* ================= backup ================= */ +void +xtrabackup_io_throttling(void) +{ + if (xtrabackup_throttle && (io_ticket--) < 0) { + os_event_reset(wait_throttle); + os_event_wait(wait_throttle); + } +} + +static +my_bool regex_list_check_match( + const regex_list_t& list, + const char* name) +{ + regmatch_t tables_regmatch[1]; + for (regex_list_t::const_iterator i = list.begin(), end = list.end(); + i != end; ++i) { + const regex_t& regex = *i; + int regres = regexec(®ex, name, 1, tables_regmatch, 0); + + if (regres != REG_NOMATCH) { + return(TRUE); + } + } + return(FALSE); +} + +static +my_bool +find_filter_in_hashtable( + const char* name, + hash_table_t* table, + xb_filter_entry_t** result +) +{ + xb_filter_entry_t* found = NULL; + HASH_SEARCH(name_hash, table, ut_fold_string(name), + xb_filter_entry_t*, + found, (void) 0, + !strcmp(found->name, name)); + + if (found && result) { + *result = found; + } + return (found != NULL); +} + +/************************************************************************ +Checks if a given table name matches any of specifications given in +regex_list or tables_hash. + +@return TRUE on match or both regex_list and tables_hash are empty.*/ +static my_bool +check_if_table_matches_filters(const char *name, + const regex_list_t& regex_list, + hash_table_t* tables_hash) +{ + if (regex_list.empty() && !tables_hash) { + return(FALSE); + } + + if (regex_list_check_match(regex_list, name)) { + return(TRUE); + } + + if (tables_hash && find_filter_in_hashtable(name, tables_hash, NULL)) { + return(TRUE); + } + + return FALSE; +} + +enum skip_database_check_result { + DATABASE_SKIP, + DATABASE_SKIP_SOME_TABLES, + DATABASE_DONT_SKIP, + DATABASE_DONT_SKIP_UNLESS_EXPLICITLY_EXCLUDED, +}; + +/************************************************************************ +Checks if a database specified by name should be skipped from backup based on +the --databases, --databases_file or --databases_exclude options. + +@return TRUE if entire database should be skipped, + FALSE otherwise. +*/ +static +skip_database_check_result +check_if_skip_database( + const char* name /*!< in: path to the database */ +) +{ + /* There are some filters for databases, check them */ + xb_filter_entry_t* database = NULL; + + if (databases_exclude_hash && + find_filter_in_hashtable(name, databases_exclude_hash, + &database) && + !database->has_tables) { + /* Database is found and there are no tables specified, + skip entire db. */ + return DATABASE_SKIP; + } + + if (databases_include_hash) { + if (!find_filter_in_hashtable(name, databases_include_hash, + &database)) { + /* Database isn't found, skip the database */ + return DATABASE_SKIP; + } else if (database->has_tables) { + return DATABASE_SKIP_SOME_TABLES; + } else { + return DATABASE_DONT_SKIP_UNLESS_EXPLICITLY_EXCLUDED; + } + } + + return DATABASE_DONT_SKIP; +} + +/************************************************************************ +Checks if a database specified by path should be skipped from backup based on +the --databases, --databases_file or --databases_exclude options. + +@return TRUE if the table should be skipped. */ +my_bool +check_if_skip_database_by_path( + const char* path /*!< in: path to the db directory. */ +) +{ + if (databases_include_hash == NULL && + databases_exclude_hash == NULL) { + return(FALSE); + } + + const char* db_name = strrchr(path, SRV_PATH_SEPARATOR); + if (db_name == NULL) { + db_name = path; + } else { + ++db_name; + } + + return check_if_skip_database(db_name) == DATABASE_SKIP; +} + +/************************************************************************ +Checks if a table specified as a name in the form "database/name" (InnoDB 5.6) +or "./database/name.ibd" (InnoDB 5.5-) should be skipped from backup based on +the --tables or --tables-file options. + +@return TRUE if the table should be skipped. */ +my_bool +check_if_skip_table( +/******************/ + const char* name) /*!< in: path to the table */ +{ + char buf[FN_REFLEN]; + const char *dbname, *tbname; + const char *ptr; + char *eptr; + + if (regex_exclude_list.empty() && + regex_include_list.empty() && + tables_include_hash == NULL && + tables_exclude_hash == NULL && + databases_include_hash == NULL && + databases_exclude_hash == NULL) { + return(FALSE); + } + + dbname = NULL; + tbname = name; + while ((ptr = strchr(tbname, '/')) != NULL) { + dbname = tbname; + tbname = ptr + 1; + } + + if (dbname == NULL) { + return(FALSE); + } + + strncpy(buf, dbname, FN_REFLEN); + buf[tbname - 1 - dbname] = 0; + + const skip_database_check_result skip_database = + check_if_skip_database(buf); + if (skip_database == DATABASE_SKIP) { + return (TRUE); + } + + buf[FN_REFLEN - 1] = '\0'; + buf[tbname - 1 - dbname] = '.'; + + /* Check if there's a suffix in the table name. If so, truncate it. We + rely on the fact that a dot cannot be a part of a table name (it is + encoded by the server with the @NNNN syntax). */ + if ((eptr = strchr(&buf[tbname - dbname], '.')) != NULL) { + + *eptr = '\0'; + } + + /* For partitioned tables first try to match against the regexp + without truncating the #P#... suffix so we can backup individual + partitions with regexps like '^test[.]t#P#p5' */ + if (check_if_table_matches_filters(buf, regex_exclude_list, + tables_exclude_hash)) { + return(TRUE); + } + if (check_if_table_matches_filters(buf, regex_include_list, + tables_include_hash)) { + return(FALSE); + } + if ((eptr = strstr(buf, "#P#")) != NULL) { + *eptr = 0; + + if (check_if_table_matches_filters(buf, regex_exclude_list, + tables_exclude_hash)) { + return (TRUE); + } + if (check_if_table_matches_filters(buf, regex_include_list, + tables_include_hash)) { + return(FALSE); + } + } + + if (skip_database == DATABASE_DONT_SKIP_UNLESS_EXPLICITLY_EXCLUDED) { + /* Database is in include-list, and qualified name wasn't + found in any of exclusion filters.*/ + return (FALSE); + } + + if (skip_database == DATABASE_SKIP_SOME_TABLES || + !regex_include_list.empty() || + tables_include_hash) { + + /* Include lists are present, but qualified name + failed to match any.*/ + return(TRUE); + } + + return(FALSE); +} + +/*********************************************************************** +Reads the space flags from a given data file and returns the compressed +page size, or 0 if the space is not compressed. */ +ulint +xb_get_zip_size(os_file_t file) +{ + byte *buf; + byte *page; + ulint zip_size = ULINT_UNDEFINED; + ibool success; + ulint space; + + buf = static_cast<byte *>(ut_malloc(2 * UNIV_PAGE_SIZE)); + page = static_cast<byte *>(ut_align(buf, UNIV_PAGE_SIZE)); + + success = os_file_read(file, page, 0, UNIV_PAGE_SIZE); + if (!success) { + goto end; + } + + space = mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); + zip_size = (space == 0 ) ? 0 : + dict_tf_get_zip_size(fsp_header_get_flags(page)); +end: + ut_free(buf); + + return(zip_size); +} + +const char* +xb_get_copy_action(const char *dflt) +{ + const char *action; + + if (xtrabackup_stream) { + if (xtrabackup_compress) { + if (xtrabackup_encrypt) { + action = "Compressing, encrypting and streaming"; + } else { + action = "Compressing and streaming"; + } + } else if (xtrabackup_encrypt) { + action = "Encrypting and streaming"; + } else { + action = "Streaming"; + } + } else { + if (xtrabackup_compress) { + if (xtrabackup_encrypt) { + action = "Compressing and encrypting"; + } else { + action = "Compressing"; + } + } else if (xtrabackup_encrypt) { + action = "Encrypting"; + } else { + action = dflt; + } + } + + return(action); +} + +/* TODO: We may tune the behavior (e.g. by fil_aio)*/ + +static +my_bool +xtrabackup_copy_datafile(fil_node_t* node, uint thread_n) +{ + char dst_name[FN_REFLEN]; + ds_file_t *dstfile = NULL; + xb_fil_cur_t cursor; + xb_fil_cur_result_t res; + xb_write_filt_t *write_filter = NULL; + xb_write_filt_ctxt_t write_filt_ctxt; + const char *action; + xb_read_filt_t *read_filter; + ibool is_system; + my_bool rc = FALSE; + + /* Get the name and the path for the tablespace. node->name always + contains the path (which may be absolute for remote tablespaces in + 5.6+). space->name contains the tablespace name in the form + "./database/table.ibd" (in 5.5-) or "database/table" (in 5.6+). For a + multi-node shared tablespace, space->name contains the name of the first + node, but that's irrelevant, since we only need node_name to match them + against filters, and the shared tablespace is always copied regardless + of the filters value. */ + + const char* const node_name = node->space->name; + const char* const node_path = node->name; + + is_system = !fil_is_user_tablespace_id(node->space->id); + + if (!is_system && check_if_skip_table(node_name)) { + msg("[%02u] Skipping %s.\n", thread_n, node_name); + return(FALSE); + } + + if (!changed_page_bitmap) { + read_filter = &rf_pass_through; + } + else { + read_filter = &rf_bitmap; + } + res = xb_fil_cur_open(&cursor, read_filter, node, thread_n); + if (res == XB_FIL_CUR_SKIP) { + goto skip; + } else if (res == XB_FIL_CUR_ERROR) { + goto error; + } + + strncpy(dst_name, cursor.rel_path, sizeof(dst_name)); + + /* Setup the page write filter */ + if (xtrabackup_incremental) { + write_filter = &wf_incremental; + } else { + write_filter = &wf_write_through; + } + + memset(&write_filt_ctxt, 0, sizeof(xb_write_filt_ctxt_t)); + ut_a(write_filter->process != NULL); + + if (write_filter->init != NULL && + !write_filter->init(&write_filt_ctxt, dst_name, &cursor)) { + msg("[%02u] xtrabackup: error: " + "failed to initialize page write filter.\n", thread_n); + goto error; + } + + dstfile = ds_open(ds_data, dst_name, &cursor.statinfo); + if (dstfile == NULL) { + msg("[%02u] xtrabackup: error: " + "cannot open the destination stream for %s\n", + thread_n, dst_name); + goto error; + } + + action = xb_get_copy_action(); + + if (xtrabackup_stream) { + msg_ts("[%02u] %s %s\n", thread_n, action, node_path); + } else { + msg_ts("[%02u] %s %s to %s\n", thread_n, action, + node_path, dstfile->path); + } + + /* The main copy loop */ + while ((res = xb_fil_cur_read(&cursor)) == XB_FIL_CUR_SUCCESS) { + if (!write_filter->process(&write_filt_ctxt, dstfile)) { + goto error; + } + } + + if (res == XB_FIL_CUR_ERROR) { + goto error; + } + + if (write_filter->finalize + && !write_filter->finalize(&write_filt_ctxt, dstfile)) { + goto error; + } + + /* close */ + msg_ts("[%02u] ...done\n", thread_n); + xb_fil_cur_close(&cursor); + if (ds_close(dstfile)) { + rc = TRUE; + } + if (write_filter && write_filter->deinit) { + write_filter->deinit(&write_filt_ctxt); + } + return(rc); + +error: + xb_fil_cur_close(&cursor); + if (dstfile != NULL) { + ds_close(dstfile); + } + if (write_filter && write_filter->deinit) { + write_filter->deinit(&write_filt_ctxt);; + } + msg("[%02u] xtrabackup: Error: " + "xtrabackup_copy_datafile() failed.\n", thread_n); + return(TRUE); /*ERROR*/ + +skip: + + if (dstfile != NULL) { + ds_close(dstfile); + } + if (write_filter && write_filter->deinit) { + write_filter->deinit(&write_filt_ctxt); + } + msg("[%02u] xtrabackup: Warning: We assume the " + "table was dropped during xtrabackup execution " + "and ignore the file.\n", thread_n); + msg("[%02u] xtrabackup: Warning: skipping tablespace %s.\n", + thread_n, node_name); + return(FALSE); +} + +static +void +xtrabackup_choose_lsn_offset(lsn_t start_lsn) +{ +#if SUPPORT_PERCONA_5_5 + ulint no, alt_no, expected_no; + ulint blocks_in_group; + lsn_t tmp_offset, end_lsn; + int lsn_chosen = 0; + log_group_t *group; + + start_lsn = ut_uint64_align_down(start_lsn, OS_FILE_LOG_BLOCK_SIZE); + end_lsn = start_lsn + RECV_SCAN_SIZE; + + group = UT_LIST_GET_FIRST(log_sys->log_groups); + + if (mysql_server_version < 50500 || mysql_server_version > 50600) { + /* only make sense for Percona Server 5.5 */ + return; + } + + if (server_flavor == FLAVOR_PERCONA_SERVER) { + /* it is Percona Server 5.5 */ + group->alt_offset_chosen = true; + group->lsn_offset = group->lsn_offset_alt; + return; + } + + if (group->lsn_offset_alt == group->lsn_offset || + group->lsn_offset_alt == (lsn_t) -1) { + /* we have only one option */ + return; + } + + no = alt_no = (ulint) -1; + lsn_chosen = 0; + + blocks_in_group = log_block_convert_lsn_to_no( + log_group_get_capacity(group)) - 1; + + /* read log block number from usual offset */ + if (group->lsn_offset < group->file_size * group->n_files && + (log_group_calc_lsn_offset(start_lsn, group) % + UNIV_PAGE_SIZE) % OS_MIN_LOG_BLOCK_SIZE == 0) { + log_group_read_log_seg(LOG_RECOVER, log_sys->buf, + group, start_lsn, end_lsn); + no = log_block_get_hdr_no(log_sys->buf); + } + + /* read log block number from Percona Server 5.5 offset */ + tmp_offset = group->lsn_offset; + group->lsn_offset = group->lsn_offset_alt; + + if (group->lsn_offset < group->file_size * group->n_files && + (log_group_calc_lsn_offset(start_lsn, group) % + UNIV_PAGE_SIZE) % OS_MIN_LOG_BLOCK_SIZE == 0) { + log_group_read_log_seg(LOG_RECOVER, log_sys->buf, + group, start_lsn, end_lsn); + alt_no = log_block_get_hdr_no(log_sys->buf); + } + + expected_no = log_block_convert_lsn_to_no(start_lsn); + + ut_a(!(no == expected_no && alt_no == expected_no)); + + group->lsn_offset = tmp_offset; + + if ((no <= expected_no && + ((expected_no - no) % blocks_in_group) == 0) || + ((expected_no | 0x40000000UL) - no) % blocks_in_group == 0) { + /* default offset looks ok */ + ++lsn_chosen; + } + + if ((alt_no <= expected_no && + ((expected_no - alt_no) % blocks_in_group) == 0) || + ((expected_no | 0x40000000UL) - alt_no) % blocks_in_group == 0) { + /* PS 5.5 style offset looks ok */ + ++lsn_chosen; + group->alt_offset_chosen = true; + group->lsn_offset = group->lsn_offset_alt; + } + + /* We are in trouble, because we can not make a + decision to choose one over the other. Die just + like a Buridan's ass */ + ut_a(lsn_chosen == 1); +#endif +} + +extern ibool log_block_checksum_is_ok_or_old_format(const byte* block); + +/*******************************************************//** +Scans log from a buffer and writes new log data to the outpud datasinc. +@return true if success */ +static +bool +xtrabackup_scan_log_recs( +/*===============*/ + log_group_t* group, /*!< in: log group */ + bool is_last, /*!< in: whether it is last segment + to copy */ + lsn_t start_lsn, /*!< in: buffer start lsn */ + lsn_t* contiguous_lsn, /*!< in/out: it is known that all log + groups contain contiguous log data up + to this lsn */ + lsn_t* group_scanned_lsn,/*!< out: scanning succeeded up to + this lsn */ + bool* finished) /*!< out: false if is not able to scan + any more in this log group */ +{ + lsn_t scanned_lsn; + ulint data_len; + ulint write_size; + const byte* log_block; + + ulint scanned_checkpoint_no = 0; + + *finished = false; + scanned_lsn = start_lsn; + log_block = log_sys->buf; + + while (log_block < log_sys->buf + RECV_SCAN_SIZE && !*finished) { + ulint no = log_block_get_hdr_no(log_block); + ulint scanned_no = log_block_convert_lsn_to_no(scanned_lsn); + ibool checksum_is_ok = + log_block_checksum_is_ok_or_old_format(log_block); + + if (no != scanned_no && checksum_is_ok) { + ulint blocks_in_group; + + blocks_in_group = log_block_convert_lsn_to_no( + log_group_get_capacity(group)) - 1; + + if ((no < scanned_no && + ((scanned_no - no) % blocks_in_group) == 0) || + no == 0 || + /* Log block numbers wrap around at 0x3FFFFFFF */ + ((scanned_no | 0x40000000UL) - no) % + blocks_in_group == 0) { + + /* old log block, do nothing */ + *finished = true; + break; + } + + msg("xtrabackup: error:" + " log block numbers mismatch:\n" + "xtrabackup: error: expected log block no. %lu," + " but got no. %lu from the log file.\n", + (ulong) scanned_no, (ulong) no); + + if ((no - scanned_no) % blocks_in_group == 0) { + msg("xtrabackup: error:" + " it looks like InnoDB log has wrapped" + " around before xtrabackup could" + " process all records due to either" + " log copying being too slow, or " + " log files being too small.\n"); + } + + return(false); + } else if (!checksum_is_ok) { + /* Garbage or an incompletely written log block */ + + msg("xtrabackup: warning: Log block checksum mismatch" + " (block no %lu at lsn " LSN_PF "): \n" + "expected %lu, calculated checksum %lu\n", + (ulong) no, + scanned_lsn, + (ulong) log_block_get_checksum(log_block), + (ulong) log_block_calc_checksum(log_block)); + msg("xtrabackup: warning: this is possible when the " + "log block has not been fully written by the " + "server, will retry later.\n"); + *finished = true; + break; + } + + if (log_block_get_flush_bit(log_block)) { + /* This block was a start of a log flush operation: + we know that the previous flush operation must have + been completed for all log groups before this block + can have been flushed to any of the groups. Therefore, + we know that log data is contiguous up to scanned_lsn + in all non-corrupt log groups. */ + + if (scanned_lsn > *contiguous_lsn) { + + *contiguous_lsn = scanned_lsn; + } + } + + data_len = log_block_get_data_len(log_block); + + if ( + (scanned_checkpoint_no > 0) + && (log_block_get_checkpoint_no(log_block) + < scanned_checkpoint_no) + && (scanned_checkpoint_no + - log_block_get_checkpoint_no(log_block) + > 0x80000000UL)) { + + /* Garbage from a log buffer flush which was made + before the most recent database recovery */ + + *finished = true; + break; + } + + scanned_lsn = scanned_lsn + data_len; + scanned_checkpoint_no = log_block_get_checkpoint_no(log_block); + + if (data_len < OS_FILE_LOG_BLOCK_SIZE) { + /* Log data for this group ends here */ + + *finished = true; + } else { + log_block += OS_FILE_LOG_BLOCK_SIZE; + } + } + + *group_scanned_lsn = scanned_lsn; + + /* ===== write log to 'xtrabackup_logfile' ====== */ + if (!*finished) { + write_size = RECV_SCAN_SIZE; + } else { + write_size = (ulint)(ut_uint64_align_up(scanned_lsn, + OS_FILE_LOG_BLOCK_SIZE) - start_lsn); + if (!is_last && scanned_lsn % OS_FILE_LOG_BLOCK_SIZE) { + write_size -= OS_FILE_LOG_BLOCK_SIZE; + } + } + + if (write_size == 0) { + return(true); + } + + if (srv_encrypt_log) { + log_encrypt_before_write(scanned_checkpoint_no, + log_sys->buf, write_size); + } + + if (ds_write(dst_log_file, log_sys->buf, write_size)) { + msg("xtrabackup: Error: " + "write to logfile failed\n"); + return(false); + } + + return(true); +} + +static my_bool +xtrabackup_copy_logfile(lsn_t from_lsn, my_bool is_last) +{ + /* definition from recv_recovery_from_checkpoint_start() */ + log_group_t* group; + lsn_t group_scanned_lsn; + lsn_t contiguous_lsn; + + ut_a(dst_log_file != NULL); + + /* read from checkpoint_lsn_start to current */ + contiguous_lsn = ut_uint64_align_down(from_lsn, OS_FILE_LOG_BLOCK_SIZE); + + /* TODO: We must check the contiguous_lsn still exists in log file.. */ + + group = UT_LIST_GET_FIRST(log_sys->log_groups); + + while (group) { + bool finished; + lsn_t start_lsn; + lsn_t end_lsn; + + /* reference recv_group_scan_log_recs() */ + finished = false; + + start_lsn = contiguous_lsn; + + while (!finished) { + + end_lsn = start_lsn + RECV_SCAN_SIZE; + + xtrabackup_io_throttling(); + + mutex_enter(&log_sys->mutex); + + log_group_read_log_seg(LOG_RECOVER, log_sys->buf, + group, start_lsn, end_lsn, false); + + if (!xtrabackup_scan_log_recs(group, is_last, + start_lsn, &contiguous_lsn, &group_scanned_lsn, + &finished)) { + goto error; + } + + mutex_exit(&log_sys->mutex); + + start_lsn = end_lsn; + + } + + group->scanned_lsn = group_scanned_lsn; + + msg_ts(">> log scanned up to (" LSN_PF ")\n", + group->scanned_lsn); + + group = UT_LIST_GET_NEXT(log_groups, group); + + /* update global variable*/ + log_copy_scanned_lsn = group_scanned_lsn; + + /* innodb_mirrored_log_groups must be 1, no other groups */ + ut_a(group == NULL); + + debug_sync_point("xtrabackup_copy_logfile_pause"); + + } + + + return(FALSE); + +error: + mutex_exit(&log_sys->mutex); + ds_close(dst_log_file); + msg("xtrabackup: Error: xtrabackup_copy_logfile() failed.\n"); + return(TRUE); +} + +static +#ifndef __WIN__ +void* +#else +ulint +#endif +log_copying_thread( + void* arg __attribute__((unused))) +{ + /* + Initialize mysys thread-specific memory so we can + use mysys functions in this thread. + */ + my_thread_init(); + + ut_a(dst_log_file != NULL); + + log_copying_running = TRUE; + + while(log_copying) { + os_event_reset(log_copying_stop); + os_event_wait_time_low(log_copying_stop, + xtrabackup_log_copy_interval * 1000ULL, + 0); + if (log_copying) { + if(xtrabackup_copy_logfile(log_copy_scanned_lsn, + FALSE)) { + + exit(EXIT_FAILURE); + } + } + } + + /* last copying */ + if(xtrabackup_copy_logfile(log_copy_scanned_lsn, TRUE)) { + + exit(EXIT_FAILURE); + } + + log_copying_running = FALSE; + my_thread_end(); + os_thread_exit(NULL); + + return(0); +} + +/* io throttle watching (rough) */ +static +#ifndef __WIN__ +void* +#else +ulint +#endif +io_watching_thread( + void* arg) +{ + (void)arg; + /* currently, for --backup only */ + ut_a(xtrabackup_backup); + + io_watching_thread_running = TRUE; + + while (log_copying) { + os_thread_sleep(1000000); /*1 sec*/ + io_ticket = xtrabackup_throttle; + os_event_set(wait_throttle); + } + + /* stop io throttle */ + xtrabackup_throttle = 0; + os_event_set(wait_throttle); + + io_watching_thread_running = FALSE; + + os_thread_exit(NULL); + + return(0); +} + +/************************************************************************ +I/o-handler thread function. */ +static + +#ifndef __WIN__ +void* +#else +ulint +#endif +io_handler_thread( +/*==============*/ + void* arg) +{ + ulint segment; + + + segment = *((ulint*)arg); + + while (srv_shutdown_state != SRV_SHUTDOWN_EXIT_THREADS) { + fil_aio_wait(segment); + } + + /* We count the number of threads in os_thread_exit(). A created + thread should always use that to exit and not use return() to exit. + The thread actually never comes here because it is exited in an + os_event_wait(). */ + + os_thread_exit(NULL); + +#ifndef __WIN__ + return(NULL); /* Not reached */ +#else + return(0); +#endif +} + +/************************************************************************** +Datafiles copying thread.*/ +static +os_thread_ret_t +data_copy_thread_func( +/*==================*/ + void *arg) /* thread context */ +{ + data_thread_ctxt_t *ctxt = (data_thread_ctxt_t *) arg; + uint num = ctxt->num; + fil_node_t* node; + + /* + Initialize mysys thread-specific memory so we can + use mysys functions in this thread. + */ + my_thread_init(); + + debug_sync_point("data_copy_thread_func"); + + while ((node = datafiles_iter_next(ctxt->it)) != NULL) { + + /* copy the datafile */ + if(xtrabackup_copy_datafile(node, num)) { + msg("[%02u] xtrabackup: Error: " + "failed to copy datafile.\n", num); + exit(EXIT_FAILURE); + } + } + + os_mutex_enter(ctxt->count_mutex); + (*ctxt->count)--; + os_mutex_exit(ctxt->count_mutex); + + my_thread_end(); + os_thread_exit(NULL); + OS_THREAD_DUMMY_RETURN; +} + +/************************************************************************ +Initialize the appropriate datasink(s). Both local backups and streaming in the +'xbstream' format allow parallel writes so we can write directly. + +Otherwise (i.e. when streaming in the 'tar' format) we need 2 separate datasinks +for the data stream (and don't allow parallel data copying) and for metainfo +files (including xtrabackup_logfile). The second datasink writes to temporary +files first, and then streams them in a serialized way when closed. */ +static void +xtrabackup_init_datasinks(void) +{ + if (xtrabackup_parallel > 1 && xtrabackup_stream && + xtrabackup_stream_fmt == XB_STREAM_FMT_TAR) { + msg("xtrabackup: warning: the --parallel option does not have " + "any effect when streaming in the 'tar' format. " + "You can use the 'xbstream' format instead.\n"); + xtrabackup_parallel = 1; + } + + /* Start building out the pipelines from the terminus back */ + if (xtrabackup_stream) { + /* All streaming goes to stdout */ + ds_data = ds_meta = ds_redo = ds_create(xtrabackup_target_dir, + DS_TYPE_STDOUT); + } else { + /* Local filesystem */ + ds_data = ds_meta = ds_redo = ds_create(xtrabackup_target_dir, + DS_TYPE_LOCAL); + } + + /* Track it for destruction */ + xtrabackup_add_datasink(ds_data); + + /* Stream formatting */ + if (xtrabackup_stream) { + ds_ctxt_t *ds; + if (xtrabackup_stream_fmt == XB_STREAM_FMT_TAR) { + ds = ds_create(xtrabackup_target_dir, DS_TYPE_ARCHIVE); + } else if (xtrabackup_stream_fmt == XB_STREAM_FMT_XBSTREAM) { + ds = ds_create(xtrabackup_target_dir, DS_TYPE_XBSTREAM); + } else { + /* bad juju... */ + ds = NULL; + } + + xtrabackup_add_datasink(ds); + + ds_set_pipe(ds, ds_data); + ds_data = ds; + + if (xtrabackup_stream_fmt != XB_STREAM_FMT_XBSTREAM) { + + /* 'tar' does not allow parallel streams */ + ds_redo = ds_meta = ds_create(xtrabackup_target_dir, + DS_TYPE_TMPFILE); + xtrabackup_add_datasink(ds_meta); + ds_set_pipe(ds_meta, ds); + } else { + ds_redo = ds_meta = ds_data; + } + } + + /* Encryption */ + if (xtrabackup_encrypt) { + ds_ctxt_t *ds; + + + + ds = ds_create(xtrabackup_target_dir, DS_TYPE_ENCRYPT); + xtrabackup_add_datasink(ds); + + ds_set_pipe(ds, ds_data); + if (ds_data != ds_meta) { + ds_data = ds; + ds = ds_create(xtrabackup_target_dir, DS_TYPE_ENCRYPT); + xtrabackup_add_datasink(ds); + + ds_set_pipe(ds, ds_meta); + ds_redo = ds_meta = ds; + } else { + ds_redo = ds_data = ds_meta = ds; + } + } + + /* Compression for ds_data and ds_redo */ + if (xtrabackup_compress) { + ds_ctxt_t *ds; + + /* Use a 1 MB buffer for compressed output stream */ + ds = ds_create(xtrabackup_target_dir, DS_TYPE_BUFFER); + ds_buffer_set_size(ds, 1024 * 1024); + xtrabackup_add_datasink(ds); + ds_set_pipe(ds, ds_data); + if (ds_data != ds_redo) { + ds_data = ds; + ds = ds_create(xtrabackup_target_dir, DS_TYPE_BUFFER); + ds_buffer_set_size(ds, 1024 * 1024); + xtrabackup_add_datasink(ds); + ds_set_pipe(ds, ds_redo); + ds_redo = ds; + } else { + ds_redo = ds_data = ds; + } + + ds = ds_create(xtrabackup_target_dir, DS_TYPE_COMPRESS); + xtrabackup_add_datasink(ds); + ds_set_pipe(ds, ds_data); + if (ds_data != ds_redo) { + ds_data = ds; + ds = ds_create(xtrabackup_target_dir, DS_TYPE_COMPRESS); + xtrabackup_add_datasink(ds); + ds_set_pipe(ds, ds_redo); + ds_redo = ds; + } else { + ds_redo = ds_data = ds; + } + } +} + +/************************************************************************ +Destroy datasinks. + +Destruction is done in the specific order to not violate their order in the +pipeline so that each datasink is able to flush data down the pipeline. */ +static void xtrabackup_destroy_datasinks(void) +{ + for (uint i = actual_datasinks; i > 0; i--) { + ds_destroy(datasinks[i-1]); + datasinks[i-1] = NULL; + } + ds_data = NULL; + ds_meta = NULL; + ds_redo = NULL; +} + +#define SRV_N_PENDING_IOS_PER_THREAD OS_AIO_N_PENDING_IOS_PER_THREAD +#define SRV_MAX_N_PENDING_SYNC_IOS 100 + +/************************************************************************ +@return TRUE if table should be opened. */ +static +ibool +xb_check_if_open_tablespace( + const char* db, + const char* table) +{ + char buf[FN_REFLEN]; + + snprintf(buf, sizeof(buf), "%s/%s", db, table); + + return !check_if_skip_table(buf); +} + +/************************************************************************ +Initializes the I/O and tablespace cache subsystems. */ +static +void +xb_fil_io_init(void) +/*================*/ +{ + srv_n_file_io_threads = srv_n_read_io_threads; + + os_aio_init(8 * SRV_N_PENDING_IOS_PER_THREAD, + srv_n_read_io_threads, + srv_n_write_io_threads, + SRV_MAX_N_PENDING_SYNC_IOS); + + fil_init(srv_file_per_table ? 50000 : 5000, LONG_MAX); + + fsp_init(); +} + +/**************************************************************************** +Populates the tablespace memory cache by scanning for and opening data files. +@returns DB_SUCCESS or error code.*/ +static +ulint +xb_load_tablespaces(void) +/*=====================*/ +{ + ulint i; + ibool create_new_db; + ulint err; + ulint sum_of_new_sizes; + lsn_t min_arch_logno, max_arch_logno; + + for (i = 0; i < srv_n_file_io_threads; i++) { + thread_nr[i] = i; + + os_thread_create(io_handler_thread, thread_nr + i, + thread_ids + i); + } + + os_thread_sleep(200000); /*0.2 sec*/ + + err = open_or_create_data_files(&create_new_db, + &min_arch_logno, &max_arch_logno, + &min_flushed_lsn, &max_flushed_lsn, + &sum_of_new_sizes); + if (err != DB_SUCCESS) { + msg("xtrabackup: Could not open or create data files.\n" + "xtrabackup: If you tried to add new data files, and it " + "failed here,\n" + "xtrabackup: you should now edit innodb_data_file_path in " + "my.cnf back\n" + "xtrabackup: to what it was, and remove the new ibdata " + "files InnoDB created\n" + "xtrabackup: in this failed attempt. InnoDB only wrote " + "those files full of\n" + "xtrabackup: zeros, but did not yet use them in any way. " + "But be careful: do not\n" + "xtrabackup: remove old data files which contain your " + "precious data!\n"); + return(err); + } + + /* create_new_db must not be TRUE.. */ + if (create_new_db) { + msg("xtrabackup: could not find data files at the " + "specified datadir\n"); + return(DB_ERROR); + } + + /* Add separate undo tablespaces to fil_system */ + + err = srv_undo_tablespaces_init(FALSE, + TRUE, + srv_undo_tablespaces, + &srv_undo_tablespaces_open); + if (err != DB_SUCCESS) { + return(err); + } + + /* It is important to call fil_load_single_table_tablespace() after + srv_undo_tablespaces_init(), because fil_is_user_tablespace_id() * + relies on srv_undo_tablespaces_open to be properly initialized */ + + msg("xtrabackup: Generating a list of tablespaces\n"); + + err = fil_load_single_table_tablespaces(xb_check_if_open_tablespace); + if (err != DB_SUCCESS) { + return(err); + } + + debug_sync_point("xtrabackup_load_tablespaces_pause"); + + return(DB_SUCCESS); +} + +/************************************************************************ +Initialize the tablespace memory cache and populate it by scanning for and +opening data files. +@returns DB_SUCCESS or error code.*/ +ulint +xb_data_files_init(void) +/*====================*/ +{ + xb_fil_io_init(); + + return(xb_load_tablespaces()); +} + +/************************************************************************ +Destroy the tablespace memory cache. */ +void +xb_data_files_close(void) +/*====================*/ +{ + ulint i; + + /* Shutdown the aio threads. This has been copied from + innobase_shutdown_for_mysql(). */ + + srv_shutdown_state = SRV_SHUTDOWN_EXIT_THREADS; + + for (i = 0; i < 1000; i++) { + os_aio_wake_all_threads_at_shutdown(); + + if (os_thread_count == 0) { + break; + } + os_thread_sleep(10000); + } + + if (i == 1000) { + msg("xtrabackup: Warning: %lu threads created by InnoDB" + " had not exited at shutdown!\n", + (ulong) os_thread_count); + } + + os_aio_free(); + + fil_close_all_files(); + + /* Free the double write data structures. */ + if (buf_dblwr) { + buf_dblwr_free(); + } + + /* Reset srv_file_io_threads to its default value to avoid confusing + warning on --prepare in innobase_start_or_create_for_mysql()*/ + srv_n_file_io_threads = 4; + + srv_shutdown_state = SRV_SHUTDOWN_NONE; +} + +/*********************************************************************** +Allocate and initialize the entry for databases and tables filtering +hash tables. If memory allocation is not successful, terminate program. +@return pointer to the created entry. */ +static +xb_filter_entry_t * +xb_new_filter_entry( +/*================*/ + const char* name) /*!< in: name of table/database */ +{ + xb_filter_entry_t *entry; + ulint namelen = strlen(name); + + ut_a(namelen <= NAME_LEN * 2 + 1); + + entry = static_cast<xb_filter_entry_t *> + (ut_malloc(sizeof(xb_filter_entry_t) + namelen + 1)); + memset(entry, '\0', sizeof(xb_filter_entry_t) + namelen + 1); + entry->name = ((char*)entry) + sizeof(xb_filter_entry_t); + strcpy(entry->name, name); + entry->has_tables = FALSE; + + return entry; +} + +/*********************************************************************** +Add entry to hash table. If hash table is NULL, allocate and initialize +new hash table */ +static +xb_filter_entry_t* +xb_add_filter( +/*========================*/ + const char* name, /*!< in: name of table/database */ + hash_table_t** hash) /*!< in/out: hash to insert into */ +{ + xb_filter_entry_t* entry; + + entry = xb_new_filter_entry(name); + + if (UNIV_UNLIKELY(*hash == NULL)) { + *hash = hash_create(1000); + } + HASH_INSERT(xb_filter_entry_t, + name_hash, *hash, + ut_fold_string(entry->name), + entry); + + return entry; +} + +/*********************************************************************** +Validate name of table or database. If name is invalid, program will +be finished with error code */ +static +void +xb_validate_name( +/*=============*/ + const char* name, /*!< in: name */ + size_t len) /*!< in: length of name */ +{ + const char* p; + + /* perform only basic validation. validate length and + path symbols */ + if (len > NAME_LEN) { + msg("xtrabackup: name `%s` is too long.\n", name); + exit(EXIT_FAILURE); + } + p = strpbrk(name, "/\\~"); + if (p && p - name < NAME_LEN) { + msg("xtrabackup: name `%s` is not valid.\n", name); + exit(EXIT_FAILURE); + } +} + +/*********************************************************************** +Register new filter entry which can be either database +or table name. */ +static +void +xb_register_filter_entry( +/*=====================*/ + const char* name, /*!< in: name */ + hash_table_t** databases_hash, + hash_table_t** tables_hash + ) +{ + const char* p; + size_t namelen; + xb_filter_entry_t* db_entry = NULL; + + namelen = strlen(name); + if ((p = strchr(name, '.')) != NULL) { + char dbname[NAME_LEN + 1]; + + xb_validate_name(name, p - name); + xb_validate_name(p + 1, namelen - (p - name)); + + strncpy(dbname, name, p - name); + dbname[p - name] = 0; + + if (*databases_hash) { + HASH_SEARCH(name_hash, (*databases_hash), + ut_fold_string(dbname), + xb_filter_entry_t*, + db_entry, (void) 0, + !strcmp(db_entry->name, dbname)); + } + if (!db_entry) { + db_entry = xb_add_filter(dbname, databases_hash); + } + db_entry->has_tables = TRUE; + xb_add_filter(name, tables_hash); + } else { + xb_validate_name(name, namelen); + + xb_add_filter(name, databases_hash); + } +} + +static +void +xb_register_include_filter_entry( + const char* name +) +{ + xb_register_filter_entry(name, &databases_include_hash, + &tables_include_hash); +} + +static +void +xb_register_exclude_filter_entry( + const char* name +) +{ + xb_register_filter_entry(name, &databases_exclude_hash, + &tables_exclude_hash); +} + +/*********************************************************************** +Register new table for the filter. */ +static +void +xb_register_table( +/*==============*/ + const char* name) /*!< in: name of table */ +{ + if (strchr(name, '.') == NULL) { + msg("xtrabackup: `%s` is not fully qualified name.\n", name); + exit(EXIT_FAILURE); + } + + xb_register_include_filter_entry(name); +} + +static +void +xb_add_regex_to_list( + const char* regex, /*!< in: regex */ + const char* error_context, /*!< in: context to error message */ + regex_list_t* list) /*! in: list to put new regex to */ +{ + char errbuf[100]; + int ret; + + regex_t compiled_regex; + ret = regcomp(&compiled_regex, regex, REG_EXTENDED); + + if (ret != 0) { + regerror(ret, &compiled_regex, errbuf, sizeof(errbuf)); + msg("xtrabackup: error: %s regcomp(%s): %s\n", + error_context, regex, errbuf); + exit(EXIT_FAILURE); + } + + list->push_back(compiled_regex); +} + +/*********************************************************************** +Register new regex for the include filter. */ +static +void +xb_register_include_regex( +/*==============*/ + const char* regex) /*!< in: regex */ +{ + xb_add_regex_to_list(regex, "tables", ®ex_include_list); +} + +/*********************************************************************** +Register new regex for the exclude filter. */ +static +void +xb_register_exclude_regex( +/*==============*/ + const char* regex) /*!< in: regex */ +{ + xb_add_regex_to_list(regex, "tables-exclude", ®ex_exclude_list); +} + +typedef void (*insert_entry_func_t)(const char*); + +/*********************************************************************** +Scan string and load filter entries from it. */ +static +void +xb_load_list_string( +/*================*/ + char* list, /*!< in: string representing a list */ + const char* delimiters, /*!< in: delimiters of entries */ + insert_entry_func_t ins) /*!< in: callback to add entry */ +{ + char* p; + char* saveptr; + + p = strtok_r(list, delimiters, &saveptr); + while (p) { + + ins(p); + + p = strtok_r(NULL, delimiters, &saveptr); + } +} + +/*********************************************************************** +Scan file and load filter entries from it. */ +static +void +xb_load_list_file( +/*==============*/ + const char* filename, /*!< in: name of file */ + insert_entry_func_t ins) /*!< in: callback to add entry */ +{ + char name_buf[NAME_LEN*2+2]; + FILE* fp; + + /* read and store the filenames */ + fp = fopen(filename, "r"); + if (!fp) { + msg("xtrabackup: cannot open %s\n", + filename); + exit(EXIT_FAILURE); + } + while (fgets(name_buf, sizeof(name_buf), fp) != NULL) { + char* p = strchr(name_buf, '\n'); + if (p) { + *p = '\0'; + } else { + msg("xtrabackup: `%s...` name is too long", name_buf); + exit(EXIT_FAILURE); + } + + ins(name_buf); + } + + fclose(fp); +} + + +static +void +xb_filters_init() +{ + if (xtrabackup_databases) { + xb_load_list_string(xtrabackup_databases, " \t", + xb_register_include_filter_entry); + } + + if (xtrabackup_databases_file) { + xb_load_list_file(xtrabackup_databases_file, + xb_register_include_filter_entry); + } + + if (xtrabackup_databases_exclude) { + xb_load_list_string(xtrabackup_databases_exclude, " \t", + xb_register_exclude_filter_entry); + } + + if (xtrabackup_tables) { + xb_load_list_string(xtrabackup_tables, ",", + xb_register_include_regex); + } + + if (xtrabackup_tables_file) { + xb_load_list_file(xtrabackup_tables_file, xb_register_table); + } + + if (xtrabackup_tables_exclude) { + xb_load_list_string(xtrabackup_tables_exclude, ",", + xb_register_exclude_regex); + } +} + +static +void +xb_filter_hash_free(hash_table_t* hash) +{ + ulint i; + + /* free the hash elements */ + for (i = 0; i < hash_get_n_cells(hash); i++) { + xb_filter_entry_t* table; + + table = static_cast<xb_filter_entry_t *> + (HASH_GET_FIRST(hash, i)); + + while (table) { + xb_filter_entry_t* prev_table = table; + + table = static_cast<xb_filter_entry_t *> + (HASH_GET_NEXT(name_hash, prev_table)); + + HASH_DELETE(xb_filter_entry_t, name_hash, hash, + ut_fold_string(prev_table->name), prev_table); + ut_free(prev_table); + } + } + + /* free hash */ + hash_table_free(hash); +} + +static void xb_regex_list_free(regex_list_t* list) +{ + while (list->size() > 0) { + xb_regfree(&list->front()); + list->pop_front(); + } +} + +/************************************************************************ +Destroy table filters for partial backup. */ +static +void +xb_filters_free() +{ + xb_regex_list_free(®ex_include_list); + xb_regex_list_free(®ex_exclude_list); + + if (tables_include_hash) { + xb_filter_hash_free(tables_include_hash); + } + + if (tables_exclude_hash) { + xb_filter_hash_free(tables_exclude_hash); + } + + if (databases_include_hash) { + xb_filter_hash_free(databases_include_hash); + } + + if (databases_exclude_hash) { + xb_filter_hash_free(databases_exclude_hash); + } +} + +/*********************************************************************//** +Creates or opens the log files and closes them. +@return DB_SUCCESS or error code */ +static +ulint +open_or_create_log_file( +/*====================*/ + ibool create_new_db, /*!< in: TRUE if we should create a + new database */ + ibool* log_file_created, /*!< out: TRUE if new log file + created */ + ibool log_file_has_been_opened,/*!< in: TRUE if a log file has been + opened before: then it is an error + to try to create another log file */ + ulint k, /*!< in: log group number */ + ulint i) /*!< in: log file number in group */ +{ + ibool ret; + os_offset_t size; + char name[10000]; + ulint dirnamelen; + + UT_NOT_USED(create_new_db); + UT_NOT_USED(log_file_has_been_opened); + UT_NOT_USED(k); + ut_ad(k == 0); + + *log_file_created = FALSE; + + srv_normalize_path_for_win(srv_log_group_home_dir); + + dirnamelen = strlen(srv_log_group_home_dir); + ut_a(dirnamelen < (sizeof name) - 10 - sizeof "ib_logfile"); + memcpy(name, srv_log_group_home_dir, dirnamelen); + + /* Add a path separator if needed. */ + if (dirnamelen && name[dirnamelen - 1] != SRV_PATH_SEPARATOR) { + name[dirnamelen++] = SRV_PATH_SEPARATOR; + } + + sprintf(name + dirnamelen, "%s%lu", "ib_logfile", (ulong) i); + + files[i] = os_file_create(innodb_file_log_key, name, + OS_FILE_OPEN, OS_FILE_NORMAL, + OS_LOG_FILE, &ret,0); + if (ret == FALSE) { + fprintf(stderr, "InnoDB: Error in opening %s\n", name); + + return(DB_ERROR); + } + + size = os_file_get_size(files[i]); + + if (size != srv_log_file_size * UNIV_PAGE_SIZE) { + + fprintf(stderr, + "InnoDB: Error: log file %s is" + " of different size " UINT64PF " bytes\n" + "InnoDB: than specified in the .cnf" + " file " UINT64PF " bytes!\n", + name, size, srv_log_file_size * UNIV_PAGE_SIZE); + + return(DB_ERROR); + } + + ret = os_file_close(files[i]); + ut_a(ret); + + if (i == 0) { + /* Create in memory the file space object + which is for this log group */ + + fil_space_create(name, + 2 * k + SRV_LOG_SPACE_FIRST_ID, 0, FIL_LOG, 0, 0); + } + + ut_a(fil_validate()); + + ut_a(fil_node_create(name, (ulint)srv_log_file_size, + 2 * k + SRV_LOG_SPACE_FIRST_ID, FALSE)); + if (i == 0) { + log_group_init(k, srv_n_log_files, + srv_log_file_size * UNIV_PAGE_SIZE, + 2 * k + SRV_LOG_SPACE_FIRST_ID, + SRV_LOG_SPACE_FIRST_ID + 1); /* dummy arch + space id */ + } + + return(DB_SUCCESS); +} + +/*********************************************************************//** +Normalizes init parameter values to use units we use inside InnoDB. +@return DB_SUCCESS or error code */ +static +void +xb_normalize_init_values(void) +/*==========================*/ +{ + ulint i; + + for (i = 0; i < srv_n_data_files; i++) { + srv_data_file_sizes[i] = srv_data_file_sizes[i] + * ((1024 * 1024) / UNIV_PAGE_SIZE); + } + + srv_last_file_size_max = srv_last_file_size_max + * ((1024 * 1024) / UNIV_PAGE_SIZE); + + srv_log_file_size = srv_log_file_size / UNIV_PAGE_SIZE; + + srv_log_buffer_size = srv_log_buffer_size / UNIV_PAGE_SIZE; + + srv_lock_table_size = 5 * (srv_buf_pool_size / UNIV_PAGE_SIZE); +} + +/*********************************************************************** +Set the open files limit. Based on set_max_open_files(). + +@return the resulting open files limit. May be less or more than the requested +value. */ +static uint +xb_set_max_open_files( +/*==================*/ + uint max_file_limit) /*!<in: open files limit */ +{ +#if defined(RLIMIT_NOFILE) + struct rlimit rlimit; + uint old_cur; + + if (getrlimit(RLIMIT_NOFILE, &rlimit)) { + + goto end; + } + + old_cur = (uint) rlimit.rlim_cur; + + if (rlimit.rlim_cur == RLIM_INFINITY) { + + rlimit.rlim_cur = max_file_limit; + } + + if (rlimit.rlim_cur >= max_file_limit) { + + max_file_limit = rlimit.rlim_cur; + goto end; + } + + rlimit.rlim_cur = rlimit.rlim_max = max_file_limit; + + if (setrlimit(RLIMIT_NOFILE, &rlimit)) { + + max_file_limit = old_cur; /* Use original value */ + } else { + + rlimit.rlim_cur = 0; /* Safety if next call fails */ + + (void) getrlimit(RLIMIT_NOFILE, &rlimit); + + if (rlimit.rlim_cur) { + + /* If call didn't fail */ + max_file_limit = (uint) rlimit.rlim_cur; + } + } + +end: + return(max_file_limit); +#else + return(0); +#endif +} + +void +xtrabackup_backup_func(void) +{ + MY_STAT stat_info; + lsn_t latest_cp; + uint i; + uint count; + os_ib_mutex_t count_mutex; + data_thread_ctxt_t *data_threads; + +#ifdef USE_POSIX_FADVISE + msg("xtrabackup: uses posix_fadvise().\n"); +#endif + + /* cd to datadir */ + + if (my_setwd(mysql_real_data_home,MYF(MY_WME))) + { + msg("xtrabackup: cannot my_setwd %s\n", mysql_real_data_home); + exit(EXIT_FAILURE); + } + msg("xtrabackup: cd to %s\n", mysql_real_data_home); + + msg("xtrabackup: open files limit requested %u, set to %u\n", + (uint) xb_open_files_limit, + xb_set_max_open_files(xb_open_files_limit)); + + mysql_data_home= mysql_data_home_buff; + mysql_data_home[0]=FN_CURLIB; // all paths are relative from here + mysql_data_home[1]=0; + + srv_read_only_mode = TRUE; + + srv_backup_mode = TRUE; + srv_close_files = (bool)xb_close_files; + + if (srv_close_files) + msg("xtrabackup: warning: close-files specified. Use it " + "at your own risk. If there are DDL operations like table DROP TABLE " + "or RENAME TABLE during the backup, inconsistent backup will be " + "produced.\n"); + + /* initialize components */ + if(innodb_init_param()) + exit(EXIT_FAILURE); + + xb_normalize_init_values(); + + + if (srv_file_flush_method_str == NULL) { + /* These are the default options */ + srv_unix_file_flush_method = SRV_UNIX_FSYNC; + } else if (0 == ut_strcmp(srv_file_flush_method_str, "fsync")) { + srv_unix_file_flush_method = SRV_UNIX_FSYNC; + } else if (0 == ut_strcmp(srv_file_flush_method_str, "O_DSYNC")) { + srv_unix_file_flush_method = SRV_UNIX_O_DSYNC; + + } else if (0 == ut_strcmp(srv_file_flush_method_str, "O_DIRECT")) { + srv_unix_file_flush_method = SRV_UNIX_O_DIRECT; + msg("xtrabackup: using O_DIRECT\n"); + } else if (0 == ut_strcmp(srv_file_flush_method_str, "littlesync")) { + srv_unix_file_flush_method = SRV_UNIX_LITTLESYNC; + + } else if (0 == ut_strcmp(srv_file_flush_method_str, "nosync")) { + srv_unix_file_flush_method = SRV_UNIX_NOSYNC; + } else if (0 == ut_strcmp(srv_file_flush_method_str, "ALL_O_DIRECT")) { + srv_unix_file_flush_method = SRV_UNIX_ALL_O_DIRECT; + msg("xtrabackup: using ALL_O_DIRECT\n"); + } else if (0 == ut_strcmp(srv_file_flush_method_str, + "O_DIRECT_NO_FSYNC")) { + srv_unix_file_flush_method = SRV_UNIX_O_DIRECT_NO_FSYNC; + msg("xtrabackup: using O_DIRECT_NO_FSYNC\n"); + } else { + msg("xtrabackup: Unrecognized value %s for " + "innodb_flush_method\n", srv_file_flush_method_str); + exit(EXIT_FAILURE); + } + + /* We can only use synchronous unbuffered IO on Windows for now */ + if (srv_file_flush_method_str != NULL) { + msg("xtrabackupp: Warning: " + "ignoring innodb_flush_method = %s on Windows.\n", srv_file_flush_method_str); + } + +#ifdef _WIN32 + srv_win_file_flush_method = SRV_WIN_IO_UNBUFFERED; + srv_use_native_aio = FALSE; +#endif + + if (srv_buf_pool_size >= 1000 * 1024 * 1024) { + /* Here we still have srv_pool_size counted + in kilobytes (in 4.0 this was in bytes) + srv_boot() converts the value to + pages; if buffer pool is less than 1000 MB, + assume fewer threads. */ + srv_max_n_threads = 50000; + + } else if (srv_buf_pool_size >= 8 * 1024 * 1024) { + + srv_max_n_threads = 10000; + } else { + srv_max_n_threads = 1000; /* saves several MB of memory, + especially in 64-bit + computers */ + } + + srv_general_init(); + ut_crc32_init(); + crc_init(); + +#ifdef WITH_INNODB_DISALLOW_WRITES + srv_allow_writes_event = os_event_create(); + os_event_set(srv_allow_writes_event); +#endif + + xb_filters_init(); + + { + ibool log_file_created; + ibool log_created = FALSE; + ibool log_opened = FALSE; + ulint err; + ulint i; + + xb_fil_io_init(); + + log_init(); + + lock_sys_create(srv_lock_table_size); + + for (i = 0; i < srv_n_log_files; i++) { + err = open_or_create_log_file(FALSE, &log_file_created, + log_opened, 0, i); + if (err != DB_SUCCESS) { + + //return((int) err); + exit(EXIT_FAILURE); + } + + if (log_file_created) { + log_created = TRUE; + } else { + log_opened = TRUE; + } + if ((log_opened && log_created)) { + msg( + "xtrabackup: Error: all log files must be created at the same time.\n" + "xtrabackup: All log files must be created also in database creation.\n" + "xtrabackup: If you want bigger or smaller log files, shut down the\n" + "xtrabackup: database and make sure there were no errors in shutdown.\n" + "xtrabackup: Then delete the existing log files. Edit the .cnf file\n" + "xtrabackup: and start the database again.\n"); + + //return(DB_ERROR); + exit(EXIT_FAILURE); + } + } + + /* log_file_created must not be TRUE, if online */ + if (log_file_created) { + msg("xtrabackup: Something wrong with source files...\n"); + exit(EXIT_FAILURE); + } + + } + + /* create extra LSN dir if it does not exist. */ + if (xtrabackup_extra_lsndir + &&!my_stat(xtrabackup_extra_lsndir,&stat_info,MYF(0)) + && (my_mkdir(xtrabackup_extra_lsndir,0777,MYF(0)) < 0)) { + msg("xtrabackup: Error: cannot mkdir %d: %s\n", + my_errno, xtrabackup_extra_lsndir); + exit(EXIT_FAILURE); + } + + /* create target dir if not exist */ + if (!xtrabackup_stream_str && !my_stat(xtrabackup_target_dir,&stat_info,MYF(0)) + && (my_mkdir(xtrabackup_target_dir,0777,MYF(0)) < 0)){ + msg("xtrabackup: Error: cannot mkdir %d: %s\n", + my_errno, xtrabackup_target_dir); + exit(EXIT_FAILURE); + } + + { + fil_system_t* f_system = fil_system; + + /* definition from recv_recovery_from_checkpoint_start() */ + log_group_t* max_cp_group; + ulint max_cp_field; + byte* buf; + byte* log_hdr_buf_; + byte* log_hdr_buf; + ulint err; + + /* start back ground thread to copy newer log */ + os_thread_id_t log_copying_thread_id; + datafiles_iter_t *it; + + log_hdr_buf_ = static_cast<byte *> + (ut_malloc(LOG_FILE_HDR_SIZE + UNIV_PAGE_SIZE_MAX)); + log_hdr_buf = static_cast<byte *> + (ut_align(log_hdr_buf_, UNIV_PAGE_SIZE_MAX)); + + /* get current checkpoint_lsn */ + /* Look for the latest checkpoint from any of the log groups */ + + mutex_enter(&log_sys->mutex); + + err = recv_find_max_checkpoint(&max_cp_group, &max_cp_field); + + if (err != DB_SUCCESS) { + + ut_free(log_hdr_buf_); + exit(EXIT_FAILURE); + } + + log_group_read_checkpoint_info(max_cp_group, max_cp_field); + buf = log_sys->checkpoint_buf; + + checkpoint_lsn_start = mach_read_from_8(buf + LOG_CHECKPOINT_LSN); + checkpoint_no_start = mach_read_from_8(buf + LOG_CHECKPOINT_NO); + + mutex_exit(&log_sys->mutex); + +reread_log_header: + fil_io(OS_FILE_READ | OS_FILE_LOG, true, max_cp_group->space_id, + 0, + 0, 0, LOG_FILE_HDR_SIZE, + log_hdr_buf, max_cp_group, NULL); + + /* check consistency of log file header to copy */ + mutex_enter(&log_sys->mutex); + + err = recv_find_max_checkpoint(&max_cp_group, &max_cp_field); + + if (err != DB_SUCCESS) { + + ut_free(log_hdr_buf_); + exit(EXIT_FAILURE); + } + + log_group_read_checkpoint_info(max_cp_group, max_cp_field); + buf = log_sys->checkpoint_buf; + + if(checkpoint_no_start != mach_read_from_8(buf + LOG_CHECKPOINT_NO)) { + + checkpoint_lsn_start = mach_read_from_8(buf + LOG_CHECKPOINT_LSN); + checkpoint_no_start = mach_read_from_8(buf + LOG_CHECKPOINT_NO); + mutex_exit(&log_sys->mutex); + goto reread_log_header; + } + + mutex_exit(&log_sys->mutex); + + xtrabackup_init_datasinks(); + + if (!select_history()) { + exit(EXIT_FAILURE); + } + + /* open the log file */ + memset(&stat_info, 0, sizeof(MY_STAT)); + dst_log_file = ds_open(ds_redo, XB_LOG_FILENAME, &stat_info); + if (dst_log_file == NULL) { + msg("xtrabackup: error: failed to open the target stream for " + "'%s'.\n", XB_LOG_FILENAME); + ut_free(log_hdr_buf_); + exit(EXIT_FAILURE); + } + + /* label it */ + strcpy((char*) log_hdr_buf + LOG_FILE_WAS_CREATED_BY_HOT_BACKUP, + "xtrabkup "); + ut_sprintf_timestamp( + (char*) log_hdr_buf + (LOG_FILE_WAS_CREATED_BY_HOT_BACKUP + + (sizeof "xtrabkup ") - 1)); + + if (ds_write(dst_log_file, log_hdr_buf, LOG_FILE_HDR_SIZE)) { + msg("xtrabackup: error: write to logfile failed\n"); + ut_free(log_hdr_buf_); + exit(EXIT_FAILURE); + } + + ut_free(log_hdr_buf_); + + /* start flag */ + log_copying = TRUE; + + /* start io throttle */ + if(xtrabackup_throttle) { + os_thread_id_t io_watching_thread_id; + + io_ticket = xtrabackup_throttle; + wait_throttle = os_event_create(); + + os_thread_create(io_watching_thread, NULL, + &io_watching_thread_id); + } + + mutex_enter(&log_sys->mutex); + xtrabackup_choose_lsn_offset(checkpoint_lsn_start); + mutex_exit(&log_sys->mutex); + + /* copy log file by current position */ + if(xtrabackup_copy_logfile(checkpoint_lsn_start, FALSE)) + exit(EXIT_FAILURE); + + + log_copying_stop = os_event_create(); + os_thread_create(log_copying_thread, NULL, &log_copying_thread_id); + + /* Populate fil_system with tablespaces to copy */ + err = xb_load_tablespaces(); + if (err != DB_SUCCESS) { + msg("xtrabackup: error: xb_load_tablespaces() failed with" + "error code %lu\n", err); + exit(EXIT_FAILURE); + } + + /* FLUSH CHANGED_PAGE_BITMAPS call */ + if (!flush_changed_page_bitmaps()) { + exit(EXIT_FAILURE); + } + debug_sync_point("xtrabackup_suspend_at_start"); + + if (xtrabackup_incremental) { + if (!xtrabackup_incremental_force_scan) { + changed_page_bitmap = xb_page_bitmap_init(); + } + if (!changed_page_bitmap) { + msg("xtrabackup: using the full scan for incremental " + "backup\n"); + } else if (incremental_lsn != checkpoint_lsn_start) { + /* Do not print that bitmaps are used when dummy bitmap + is build for an empty LSN range. */ + msg("xtrabackup: using the changed page bitmap\n"); + } + } + + ut_a(xtrabackup_parallel > 0); + + if (xtrabackup_parallel > 1) { + msg("xtrabackup: Starting %u threads for parallel data " + "files transfer\n", xtrabackup_parallel); + } + + it = datafiles_iter_new(f_system); + if (it == NULL) { + msg("xtrabackup: Error: datafiles_iter_new() failed.\n"); + exit(EXIT_FAILURE); + } + + /* Create data copying threads */ + data_threads = (data_thread_ctxt_t *) + ut_malloc(sizeof(data_thread_ctxt_t) * xtrabackup_parallel); + count = xtrabackup_parallel; + count_mutex = os_mutex_create(); + + for (i = 0; i < (uint) xtrabackup_parallel; i++) { + data_threads[i].it = it; + data_threads[i].num = i+1; + data_threads[i].count = &count; + data_threads[i].count_mutex = count_mutex; + os_thread_create(data_copy_thread_func, data_threads + i, + &data_threads[i].id); + } + + /* Wait for threads to exit */ + while (1) { + os_thread_sleep(1000000); + os_mutex_enter(count_mutex); + if (count == 0) { + os_mutex_exit(count_mutex); + break; + } + os_mutex_exit(count_mutex); + } + + os_mutex_free(count_mutex); + ut_free(data_threads); + datafiles_iter_free(it); + + if (changed_page_bitmap) { + xb_page_bitmap_deinit(changed_page_bitmap); + } + } + + if (!backup_start()) { + exit(EXIT_FAILURE); + } + + /* read the latest checkpoint lsn */ + latest_cp = 0; + { + log_group_t* max_cp_group; + ulint max_cp_field; + ulint err; + + mutex_enter(&log_sys->mutex); + + err = recv_find_max_checkpoint(&max_cp_group, &max_cp_field); + + if (err != DB_SUCCESS) { + msg("xtrabackup: Error: recv_find_max_checkpoint() failed.\n"); + mutex_exit(&log_sys->mutex); + goto skip_last_cp; + } + + log_group_read_checkpoint_info(max_cp_group, max_cp_field); + + xtrabackup_choose_lsn_offset(checkpoint_lsn_start); + + latest_cp = mach_read_from_8(log_sys->checkpoint_buf + + LOG_CHECKPOINT_LSN); + + mutex_exit(&log_sys->mutex); + + msg("xtrabackup: The latest check point (for incremental): " + "'" LSN_PF "'\n", latest_cp); + } +skip_last_cp: + /* stop log_copying_thread */ + log_copying = FALSE; + os_event_set(log_copying_stop); + msg("xtrabackup: Stopping log copying thread.\n"); + while (log_copying_running) { + msg("."); + os_thread_sleep(200000); /*0.2 sec*/ + } + msg("\n"); + + os_event_free(log_copying_stop); + if (ds_close(dst_log_file)) { + exit(EXIT_FAILURE); + } + + if(!xtrabackup_incremental) { + strcpy(metadata_type, "full-backuped"); + metadata_from_lsn = 0; + } else { + strcpy(metadata_type, "incremental"); + metadata_from_lsn = incremental_lsn; + } + metadata_to_lsn = latest_cp; + metadata_last_lsn = log_copy_scanned_lsn; + + if (!xtrabackup_stream_metadata(ds_meta)) { + msg("xtrabackup: Error: failed to stream metadata.\n"); + exit(EXIT_FAILURE); + } + if (xtrabackup_extra_lsndir) { + char filename[FN_REFLEN]; + + sprintf(filename, "%s/%s", xtrabackup_extra_lsndir, + XTRABACKUP_METADATA_FILENAME); + if (!xtrabackup_write_metadata(filename)) { + msg("xtrabackup: Error: failed to write metadata " + "to '%s'.\n", filename); + exit(EXIT_FAILURE); + } + + } + + if (!backup_finish()) { + exit(EXIT_FAILURE); + } + + xtrabackup_destroy_datasinks(); + + if (wait_throttle) { + /* wait for io_watching_thread completion */ + while (io_watching_thread_running) { + os_thread_sleep(1000000); + } + os_event_free(wait_throttle); + wait_throttle = NULL; + } + + msg("xtrabackup: Transaction log of lsn (" LSN_PF ") to (" LSN_PF + ") was copied.\n", checkpoint_lsn_start, log_copy_scanned_lsn); + xb_filters_free(); + + xb_data_files_close(); + + /* Make sure that the latest checkpoint made it to xtrabackup_logfile */ + if (latest_cp > log_copy_scanned_lsn) { + msg("xtrabackup: error: last checkpoint LSN (" LSN_PF + ") is larger than last copied LSN (" LSN_PF ").\n", + latest_cp, log_copy_scanned_lsn); + exit(EXIT_FAILURE); + } +} + +/* ================= stats ================= */ +static my_bool +xtrabackup_stats_level( + dict_index_t* index, + ulint level) +{ + ulint space; + page_t* page; + + rec_t* node_ptr; + + ulint right_page_no; + + page_cur_t cursor; + + mtr_t mtr; + mem_heap_t* heap = mem_heap_create(256); + + ulint* offsets = NULL; + + ulonglong n_pages, n_pages_extern; + ulonglong sum_data, sum_data_extern; + ulonglong n_recs; + ulint page_size; + buf_block_t* block; + ulint zip_size; + + n_pages = sum_data = n_recs = 0; + n_pages_extern = sum_data_extern = 0; + + + if (level == 0) + fprintf(stdout, " leaf pages: "); + else + fprintf(stdout, " level %lu pages: ", level); + + mtr_start(&mtr); + + mtr_x_lock(&(index->lock), &mtr); + block = btr_root_block_get(index, RW_X_LATCH, &mtr); + page = buf_block_get_frame(block); + + space = page_get_space_id(page); + zip_size = fil_space_get_zip_size(space); + + while (level != btr_page_get_level(page, &mtr)) { + + ut_a(space == buf_block_get_space(block)); + ut_a(space == page_get_space_id(page)); + ut_a(!page_is_leaf(page)); + + page_cur_set_before_first(block, &cursor); + page_cur_move_to_next(&cursor); + + node_ptr = page_cur_get_rec(&cursor); + offsets = rec_get_offsets(node_ptr, index, offsets, + ULINT_UNDEFINED, &heap); + block = btr_node_ptr_get_child(node_ptr, index, offsets, &mtr); + page = buf_block_get_frame(block); + } + +loop: + mem_heap_empty(heap); + offsets = NULL; + mtr_x_lock(&(index->lock), &mtr); + + right_page_no = btr_page_get_next(page, &mtr); + + + /*=================================*/ + //fprintf(stdout, "%lu ", (ulint) buf_frame_get_page_no(page)); + + n_pages++; + sum_data += page_get_data_size(page); + n_recs += page_get_n_recs(page); + + + if (level == 0) { + page_cur_t cur; + ulint n_fields; + ulint i; + mem_heap_t* local_heap = NULL; + ulint offsets_[REC_OFFS_NORMAL_SIZE]; + ulint* local_offsets = offsets_; + + *offsets_ = (sizeof offsets_) / sizeof *offsets_; + + page_cur_set_before_first(block, &cur); + page_cur_move_to_next(&cur); + + for (;;) { + if (page_cur_is_after_last(&cur)) { + break; + } + + local_offsets = rec_get_offsets(cur.rec, index, local_offsets, + ULINT_UNDEFINED, &local_heap); + n_fields = rec_offs_n_fields(local_offsets); + + for (i = 0; i < n_fields; i++) { + if (rec_offs_nth_extern(local_offsets, i)) { + page_t* local_page; + ulint space_id; + ulint page_no; + ulint offset; + byte* blob_header; + ulint part_len; + mtr_t local_mtr; + ulint local_len; + byte* data; + buf_block_t* local_block; + + data = rec_get_nth_field(cur.rec, local_offsets, i, &local_len); + + ut_a(local_len >= BTR_EXTERN_FIELD_REF_SIZE); + local_len -= BTR_EXTERN_FIELD_REF_SIZE; + + space_id = mach_read_from_4(data + local_len + BTR_EXTERN_SPACE_ID); + page_no = mach_read_from_4(data + local_len + BTR_EXTERN_PAGE_NO); + offset = mach_read_from_4(data + local_len + BTR_EXTERN_OFFSET); + + if (offset != FIL_PAGE_DATA) + msg("\nWarning: several record may share same external page.\n"); + + for (;;) { + mtr_start(&local_mtr); + + local_block = btr_block_get(space_id, zip_size, page_no, RW_S_LATCH, index, &local_mtr); + local_page = buf_block_get_frame(local_block); + blob_header = local_page + offset; +#define BTR_BLOB_HDR_PART_LEN 0 +#define BTR_BLOB_HDR_NEXT_PAGE_NO 4 + //part_len = btr_blob_get_part_len(blob_header); + part_len = mach_read_from_4(blob_header + BTR_BLOB_HDR_PART_LEN); + + //page_no = btr_blob_get_next_page_no(blob_header); + page_no = mach_read_from_4(blob_header + BTR_BLOB_HDR_NEXT_PAGE_NO); + + offset = FIL_PAGE_DATA; + + + + + /*=================================*/ + //fprintf(stdout, "[%lu] ", (ulint) buf_frame_get_page_no(page)); + + n_pages_extern++; + sum_data_extern += part_len; + + + mtr_commit(&local_mtr); + + if (page_no == FIL_NULL) + break; + } + } + } + + page_cur_move_to_next(&cur); + } + } + + + + + mtr_commit(&mtr); + if (right_page_no != FIL_NULL) { + mtr_start(&mtr); + block = btr_block_get(space, zip_size, right_page_no, + RW_X_LATCH, index, &mtr); + page = buf_block_get_frame(block); + goto loop; + } + mem_heap_free(heap); + + if (zip_size) { + page_size = zip_size; + } else { + page_size = UNIV_PAGE_SIZE; + } + + if (level == 0) + fprintf(stdout, "recs=%llu, ", n_recs); + + fprintf(stdout, "pages=%llu, data=%llu bytes, data/pages=%lld%%", + n_pages, sum_data, + ((sum_data * 100)/ page_size)/n_pages); + + + if (level == 0 && n_pages_extern) { + putc('\n', stdout); + /* also scan blob pages*/ + fprintf(stdout, " external pages: "); + + fprintf(stdout, "pages=%llu, data=%llu bytes, data/pages=%lld%%", + n_pages_extern, sum_data_extern, + ((sum_data_extern * 100)/ page_size)/n_pages_extern); + } + + putc('\n', stdout); + + if (level > 0) { + xtrabackup_stats_level(index, level - 1); + } + + return(TRUE); +} + +static void +xtrabackup_stats_func(int argc, char **argv) +{ + ulint n; + + /* cd to datadir */ + + if (my_setwd(mysql_real_data_home,MYF(MY_WME))) + { + msg("xtrabackup: cannot my_setwd %s\n", mysql_real_data_home); + exit(EXIT_FAILURE); + } + msg("xtrabackup: cd to %s\n", mysql_real_data_home); + encryption_plugin_prepare_init(argc, argv); + mysql_data_home= mysql_data_home_buff; + mysql_data_home[0]=FN_CURLIB; // all paths are relative from here + mysql_data_home[1]=0; + + /* set read only */ + srv_read_only_mode = TRUE; + + /* initialize components */ + if(innodb_init_param()) + exit(EXIT_FAILURE); + + /* Check if the log files have been created, otherwise innodb_init() + will crash when called with srv_read_only == TRUE */ + for (n = 0; n < srv_n_log_files; n++) { + char logname[FN_REFLEN]; + ibool exists; + os_file_type_t type; + + snprintf(logname, sizeof(logname), "%s%c%s%lu", + srv_log_group_home_dir, SRV_PATH_SEPARATOR, + "ib_logfile", (ulong) n); + srv_normalize_path_for_win(logname); + + if (!os_file_status(logname, &exists, &type) || !exists || + type != OS_FILE_TYPE_FILE) { + msg("xtrabackup: Error: " + "Cannot find log file %s.\n", logname); + msg("xtrabackup: Error: " + "to use the statistics feature, you need a " + "clean copy of the database including " + "correctly sized log files, so you need to " + "execute with --prepare twice to use this " + "functionality on a backup.\n"); + exit(EXIT_FAILURE); + } + } + + msg("xtrabackup: Starting 'read-only' InnoDB instance to gather " + "index statistics.\n" + "xtrabackup: Using %lld bytes for buffer pool (set by " + "--use-memory parameter)\n", xtrabackup_use_memory); + + if(innodb_init()) + exit(EXIT_FAILURE); + + xb_filters_init(); + + fprintf(stdout, "\n\n<INDEX STATISTICS>\n"); + + /* gather stats */ + + { + dict_table_t* sys_tables; + dict_index_t* sys_index; + dict_table_t* table; + btr_pcur_t pcur; + rec_t* rec; + byte* field; + ulint len; + mtr_t mtr; + + /* Enlarge the fatal semaphore wait timeout during the InnoDB table + monitor printout */ + + os_increment_counter_by_amount(server_mutex, + srv_fatal_semaphore_wait_threshold, + 72000); + + mutex_enter(&(dict_sys->mutex)); + + mtr_start(&mtr); + + sys_tables = dict_table_get_low("SYS_TABLES"); + sys_index = UT_LIST_GET_FIRST(sys_tables->indexes); + + btr_pcur_open_at_index_side(TRUE, sys_index, BTR_SEARCH_LEAF, &pcur, + TRUE, 0, &mtr); +loop: + btr_pcur_move_to_next_user_rec(&pcur, &mtr); + + rec = btr_pcur_get_rec(&pcur); + + if (!btr_pcur_is_on_user_rec(&pcur)) + { + /* end of index */ + + btr_pcur_close(&pcur); + mtr_commit(&mtr); + + mutex_exit(&(dict_sys->mutex)); + + /* Restore the fatal semaphore wait timeout */ + os_increment_counter_by_amount(server_mutex, + srv_fatal_semaphore_wait_threshold, + -72000); + + goto end; + } + + field = rec_get_nth_field_old(rec, 0, &len); + + if (!rec_get_deleted_flag(rec, 0)) { + + /* We found one */ + + char* table_name = mem_strdupl((char*) field, len); + + btr_pcur_store_position(&pcur, &mtr); + + mtr_commit(&mtr); + + table = dict_table_get_low(table_name); + mem_free(table_name); + + if (table && check_if_skip_table(table->name)) + goto skip; + + + if (table == NULL) { + fputs("InnoDB: Failed to load table ", stderr); + ut_print_namel(stderr, NULL, TRUE, (char*) field, len); + putc('\n', stderr); + } else { + dict_index_t* index; + + /* The table definition was corrupt if there + is no index */ + + if (dict_table_get_first_index(table)) { + dict_stats_update_transient(table); + } + + //dict_table_print_low(table); + + index = UT_LIST_GET_FIRST(table->indexes); + while (index != NULL) { +{ + ib_int64_t n_vals; + + if (index->n_user_defined_cols > 0) { + n_vals = index->stat_n_diff_key_vals[ + index->n_user_defined_cols]; + } else { + n_vals = index->stat_n_diff_key_vals[1]; + } + + fprintf(stdout, + " table: %s, index: %s, space id: %lu, root page: %lu" + ", zip size: %lu" + "\n estimated statistics in dictionary:\n" + " key vals: %lu, leaf pages: %lu, size pages: %lu\n" + " real statistics:\n", + table->name, index->name, + (ulong) index->space, + (ulong) index->page, + (ulong) fil_space_get_zip_size(index->space), + (ulong) n_vals, + (ulong) index->stat_n_leaf_pages, + (ulong) index->stat_index_size); + + { + mtr_t local_mtr; + page_t* root; + ulint page_level; + + mtr_start(&local_mtr); + + mtr_x_lock(&(index->lock), &local_mtr); + root = btr_root_get(index, &local_mtr); + page_level = btr_page_get_level(root, &local_mtr); + + xtrabackup_stats_level(index, page_level); + + mtr_commit(&local_mtr); + } + + putc('\n', stdout); +} + index = UT_LIST_GET_NEXT(indexes, index); + } + } + +skip: + mtr_start(&mtr); + + btr_pcur_restore_position(BTR_SEARCH_LEAF, &pcur, &mtr); + } + + goto loop; + } + +end: + putc('\n', stdout); + + fflush(stdout); + + xb_filters_free(); + + /* shutdown InnoDB */ + if(innodb_end()) + exit(EXIT_FAILURE); +} + +/* ================= prepare ================= */ + +static my_bool +xtrabackup_init_temp_log(void) +{ + os_file_t src_file = XB_FILE_UNDEFINED; + char src_path[FN_REFLEN]; + char dst_path[FN_REFLEN]; + ibool success; + + ulint field; + byte* log_buf= (byte *)malloc(UNIV_PAGE_SIZE_MAX * 128); /* 2 MB */ + + ib_int64_t file_size; + + lsn_t max_no; + lsn_t max_lsn; + lsn_t checkpoint_no; + + ulint fold; + + bool checkpoint_found; + + max_no = 0; + + if (!log_buf) { + goto error; + } + + if (!xb_init_log_block_size()) { + goto error; + } + + if(!xtrabackup_incremental_dir) { + sprintf(dst_path, "%s/ib_logfile0", xtrabackup_target_dir); + sprintf(src_path, "%s/%s", xtrabackup_target_dir, + XB_LOG_FILENAME); + } else { + sprintf(dst_path, "%s/ib_logfile0", xtrabackup_incremental_dir); + sprintf(src_path, "%s/%s", xtrabackup_incremental_dir, + XB_LOG_FILENAME); + } + + srv_normalize_path_for_win(dst_path); + srv_normalize_path_for_win(src_path); +retry: + src_file = os_file_create_simple_no_error_handling(0, src_path, + OS_FILE_OPEN, + OS_FILE_READ_WRITE, + &success,0); + if (!success) { + /* The following call prints an error message */ + os_file_get_last_error(TRUE); + + msg("xtrabackup: Warning: cannot open %s. will try to find.\n", + src_path); + + /* check if ib_logfile0 may be xtrabackup_logfile */ + src_file = os_file_create_simple_no_error_handling(0, dst_path, + OS_FILE_OPEN, + OS_FILE_READ_WRITE, + &success,0); + if (!success) { + os_file_get_last_error(TRUE); + msg(" xtrabackup: Fatal error: cannot find %s.\n", + src_path); + + goto error; + } + + success = os_file_read(src_file, log_buf, 0, + LOG_FILE_HDR_SIZE); + if (!success) { + goto error; + } + + if ( ut_memcmp(log_buf + LOG_FILE_WAS_CREATED_BY_HOT_BACKUP, + (byte*)"xtrabkup", (sizeof "xtrabkup") - 1) == 0) { + msg(" xtrabackup: 'ib_logfile0' seems to be " + "'xtrabackup_logfile'. will retry.\n"); + + os_file_close(src_file); + src_file = XB_FILE_UNDEFINED; + + /* rename and try again */ + success = os_file_rename(0, dst_path, src_path); + if (!success) { + goto error; + } + + goto retry; + } + + msg(" xtrabackup: Fatal error: cannot find %s.\n", + src_path); + + os_file_close(src_file); + src_file = XB_FILE_UNDEFINED; + + goto error; + } + + file_size = os_file_get_size(src_file); + + + /* TODO: We should skip the following modifies, if it is not the first time. */ + + /* read log file header */ + success = os_file_read(src_file, log_buf, 0, LOG_FILE_HDR_SIZE); + if (!success) { + goto error; + } + + if ( ut_memcmp(log_buf + LOG_FILE_WAS_CREATED_BY_HOT_BACKUP, + (byte*)"xtrabkup", (sizeof "xtrabkup") - 1) != 0 ) { + msg("xtrabackup: notice: xtrabackup_logfile was already used " + "to '--prepare'.\n"); + goto skip_modify; + } else { + /* clear it later */ + //memset(log_buf + LOG_FILE_WAS_CREATED_BY_HOT_BACKUP, + // ' ', 4); + } + + checkpoint_found = false; + + /* read last checkpoint lsn */ + for (field = LOG_CHECKPOINT_1; field <= LOG_CHECKPOINT_2; + field += LOG_CHECKPOINT_2 - LOG_CHECKPOINT_1) { + if (!recv_check_cp_is_consistent(const_cast<const byte *> + (log_buf + field))) + goto not_consistent; + + checkpoint_no = mach_read_from_8(log_buf + field + + LOG_CHECKPOINT_NO); + + if (checkpoint_no >= max_no) { + + max_no = checkpoint_no; + max_lsn = mach_read_from_8(log_buf + field + + LOG_CHECKPOINT_LSN); + checkpoint_found = true; + } +not_consistent: + ; + } + + if (!checkpoint_found) { + msg("xtrabackup: No valid checkpoint found.\n"); + goto error; + } + + + /* It seems to be needed to overwrite the both checkpoint area. */ + mach_write_to_8(log_buf + LOG_CHECKPOINT_1 + LOG_CHECKPOINT_LSN, + max_lsn); + mach_write_to_4(log_buf + LOG_CHECKPOINT_1 + + LOG_CHECKPOINT_OFFSET_LOW32, + LOG_FILE_HDR_SIZE + + (ulint)(max_lsn - + ut_uint64_align_down(max_lsn, + OS_FILE_LOG_BLOCK_SIZE))); + mach_write_to_4(log_buf + LOG_CHECKPOINT_1 + + LOG_CHECKPOINT_OFFSET_HIGH32, 0); + fold = ut_fold_binary(log_buf + LOG_CHECKPOINT_1, LOG_CHECKPOINT_CHECKSUM_1); + mach_write_to_4(log_buf + LOG_CHECKPOINT_1 + LOG_CHECKPOINT_CHECKSUM_1, fold); + + fold = ut_fold_binary(log_buf + LOG_CHECKPOINT_1 + LOG_CHECKPOINT_LSN, + LOG_CHECKPOINT_CHECKSUM_2 - LOG_CHECKPOINT_LSN); + mach_write_to_4(log_buf + LOG_CHECKPOINT_1 + LOG_CHECKPOINT_CHECKSUM_2, fold); + + mach_write_to_8(log_buf + LOG_CHECKPOINT_2 + LOG_CHECKPOINT_LSN, + max_lsn); + mach_write_to_4(log_buf + LOG_CHECKPOINT_2 + + LOG_CHECKPOINT_OFFSET_LOW32, + LOG_FILE_HDR_SIZE + + (ulint)(max_lsn - + ut_uint64_align_down(max_lsn, + OS_FILE_LOG_BLOCK_SIZE))); + mach_write_to_4(log_buf + LOG_CHECKPOINT_2 + + LOG_CHECKPOINT_OFFSET_HIGH32, 0); + fold = ut_fold_binary(log_buf + LOG_CHECKPOINT_2, LOG_CHECKPOINT_CHECKSUM_1); + mach_write_to_4(log_buf + LOG_CHECKPOINT_2 + LOG_CHECKPOINT_CHECKSUM_1, fold); + + fold = ut_fold_binary(log_buf + LOG_CHECKPOINT_2 + LOG_CHECKPOINT_LSN, + LOG_CHECKPOINT_CHECKSUM_2 - LOG_CHECKPOINT_LSN); + mach_write_to_4(log_buf + LOG_CHECKPOINT_2 + LOG_CHECKPOINT_CHECKSUM_2, fold); + + + success = os_file_write(src_path, src_file, log_buf, 0, + LOG_FILE_HDR_SIZE); + if (!success) { + goto error; + } + + /* expand file size (9/8) and align to UNIV_PAGE_SIZE_MAX */ + + if (file_size % UNIV_PAGE_SIZE_MAX) { + memset(log_buf, 0, UNIV_PAGE_SIZE_MAX); + success = os_file_write(src_path, src_file, log_buf, + file_size, + UNIV_PAGE_SIZE_MAX + - (ulint) (file_size + % UNIV_PAGE_SIZE_MAX)); + if (!success) { + goto error; + } + + file_size = os_file_get_size(src_file); + } + + /* TODO: We should judge whether the file is already expanded or not... */ + { + ulint expand; + + memset(log_buf, 0, UNIV_PAGE_SIZE_MAX * 128); + expand = (ulint) (file_size / UNIV_PAGE_SIZE_MAX / 8); + + for (; expand > 128; expand -= 128) { + success = os_file_write(src_path, src_file, log_buf, + file_size, + UNIV_PAGE_SIZE_MAX * 128); + if (!success) { + goto error; + } + file_size += UNIV_PAGE_SIZE_MAX * 128; + } + + if (expand) { + success = os_file_write(src_path, src_file, log_buf, + file_size, + expand * UNIV_PAGE_SIZE_MAX); + if (!success) { + goto error; + } + file_size += UNIV_PAGE_SIZE_MAX * expand; + } + } + + /* make larger than 2MB */ + if (file_size < 2*1024*1024L) { + memset(log_buf, 0, UNIV_PAGE_SIZE_MAX); + while (file_size < 2*1024*1024L) { + success = os_file_write(src_path, src_file, log_buf, + file_size, + UNIV_PAGE_SIZE_MAX); + if (!success) { + goto error; + } + file_size += UNIV_PAGE_SIZE_MAX; + } + file_size = os_file_get_size(src_file); + } + + msg("xtrabackup: xtrabackup_logfile detected: size=" INT64PF ", " + "start_lsn=(" LSN_PF ")\n", file_size, max_lsn); + + os_file_close(src_file); + src_file = XB_FILE_UNDEFINED; + + /* fake InnoDB */ + innobase_log_files_in_group_save = innobase_log_files_in_group; + srv_log_group_home_dir_save = srv_log_group_home_dir; + innobase_log_file_size_save = innobase_log_file_size; + + srv_log_group_home_dir = NULL; + innobase_log_file_size = file_size; + innobase_log_files_in_group = 1; + + srv_thread_concurrency = 0; + + /* rename 'xtrabackup_logfile' to 'ib_logfile0' */ + success = os_file_rename(0, src_path, dst_path); + if (!success) { + goto error; + } + xtrabackup_logfile_is_renamed = TRUE; + free(log_buf); + return(FALSE); + +skip_modify: + free(log_buf); + os_file_close(src_file); + src_file = XB_FILE_UNDEFINED; + return(FALSE); + +error: + free(log_buf); + if (src_file != XB_FILE_UNDEFINED) + os_file_close(src_file); + msg("xtrabackup: Error: xtrabackup_init_temp_log() failed.\n"); + return(TRUE); /*ERROR*/ +} + +/*********************************************************************** +Generates path to the meta file path from a given path to an incremental .delta +by replacing trailing ".delta" with ".meta", or returns error if 'delta_path' +does not end with the ".delta" character sequence. +@return TRUE on success, FALSE on error. */ +static +ibool +get_meta_path( + const char *delta_path, /* in: path to a .delta file */ + char *meta_path) /* out: path to the corresponding .meta + file */ +{ + size_t len = strlen(delta_path); + + if (len <= 6 || strcmp(delta_path + len - 6, ".delta")) { + return FALSE; + } + memcpy(meta_path, delta_path, len - 6); + strcpy(meta_path + len - 6, XB_DELTA_INFO_SUFFIX); + + return TRUE; +} + +/****************************************************************//** +Create a new tablespace on disk and return the handle to its opened +file. Code adopted from fil_create_new_single_table_tablespace with +the main difference that only disk file is created without updating +the InnoDB in-memory dictionary data structures. + +@return TRUE on success, FALSE on error. */ +static +ibool +xb_space_create_file( +/*==================*/ + const char* path, /*!<in: path to tablespace */ + ulint space_id, /*!<in: space id */ + ulint flags __attribute__((unused)),/*!<in: tablespace + flags */ + os_file_t* file) /*!<out: file handle */ +{ + ibool ret; + byte* buf; + byte* page; + + *file = os_file_create_simple_no_error_handling(0, path, OS_FILE_CREATE, + OS_FILE_READ_WRITE, + &ret,0); + if (!ret) { + msg("xtrabackup: cannot create file %s\n", path); + return ret; + } + + ret = os_file_set_size(path, *file, + FIL_IBD_FILE_INITIAL_SIZE * UNIV_PAGE_SIZE); + if (!ret) { + msg("xtrabackup: cannot set size for file %s\n", path); + os_file_close(*file); + os_file_delete(0, path); + return ret; + } + + buf = static_cast<byte *>(ut_malloc(3 * UNIV_PAGE_SIZE)); + /* Align the memory for file i/o if we might have O_DIRECT set */ + page = static_cast<byte *>(ut_align(buf, UNIV_PAGE_SIZE)); + + memset(page, '\0', UNIV_PAGE_SIZE); + + fsp_header_init_fields(page, space_id, flags); + mach_write_to_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, space_id); + + if (!fsp_flags_is_compressed(flags)) { + buf_flush_init_for_writing(page, NULL, 0); + + ret = os_file_write(path, *file, page, 0, UNIV_PAGE_SIZE); + } + else { + page_zip_des_t page_zip; + ulint zip_size; + + zip_size = fsp_flags_get_zip_size(flags); + page_zip_set_size(&page_zip, zip_size); + page_zip.data = page + UNIV_PAGE_SIZE; + fprintf(stderr, "zip_size = %lu\n", zip_size); + +#ifdef UNIV_DEBUG + page_zip.m_start = +#endif /* UNIV_DEBUG */ + page_zip.m_end = page_zip.m_nonempty = + page_zip.n_blobs = 0; + + buf_flush_init_for_writing(page, &page_zip, 0); + + ret = os_file_write(path, *file, page_zip.data, 0, + zip_size); + } + + ut_free(buf); + + if (!ret) { + msg("xtrabackup: could not write the first page to %s\n", + path); + os_file_close(*file); + os_file_delete(0, path); + return ret; + } + + return TRUE; +} + +/*********************************************************************** +Searches for matching tablespace file for given .delta file and space_id +in given directory. When matching tablespace found, renames it to match the +name of .delta file. If there was a tablespace with matching name and +mismatching ID, renames it to xtrabackup_tmp_#ID.ibd. If there was no +matching file, creates a new tablespace. +@return file handle of matched or created file */ +static +os_file_t +xb_delta_open_matching_space( + const char* dbname, /* in: path to destination database dir */ + const char* name, /* in: name of delta file (without .delta) */ + ulint space_id, /* in: space id of delta file */ + ulint zip_size, /* in: zip_size of tablespace */ + char* real_name, /* out: full path of destination file */ + size_t real_name_len, /* out: buffer size for real_name */ + ibool* success) /* out: indicates error. TRUE = success */ +{ + char dest_dir[FN_REFLEN]; + char dest_space_name[FN_REFLEN]; + ibool ok; + fil_space_t* fil_space; + os_file_t file = 0; + ulint tablespace_flags; + xb_filter_entry_t* table; + + ut_a(dbname != NULL || + !fil_is_user_tablespace_id(space_id) || + space_id == ULINT_UNDEFINED); + + *success = FALSE; + + if (dbname) { + snprintf(dest_dir, FN_REFLEN, "%s/%s", + xtrabackup_target_dir, dbname); + srv_normalize_path_for_win(dest_dir); + + snprintf(dest_space_name, FN_REFLEN, "%s/%s", dbname, name); + } else { + snprintf(dest_dir, FN_REFLEN, "%s", xtrabackup_target_dir); + srv_normalize_path_for_win(dest_dir); + + snprintf(dest_space_name, FN_REFLEN, "%s", name); + } + + snprintf(real_name, real_name_len, + "%s/%s", + xtrabackup_target_dir, dest_space_name); + srv_normalize_path_for_win(real_name); + /* Truncate ".ibd" */ + dest_space_name[strlen(dest_space_name) - 4] = '\0'; + + /* Create the database directory if it doesn't exist yet */ + if (!os_file_create_directory(dest_dir, FALSE)) { + msg("xtrabackup: error: cannot create dir %s\n", dest_dir); + return file; + } + + if (!fil_is_user_tablespace_id(space_id)) { + goto found; + } + + /* remember space name for further reference */ + table = static_cast<xb_filter_entry_t *> + (ut_malloc(sizeof(xb_filter_entry_t) + + strlen(dest_space_name) + 1)); + + table->name = ((char*)table) + sizeof(xb_filter_entry_t); + strcpy(table->name, dest_space_name); + HASH_INSERT(xb_filter_entry_t, name_hash, inc_dir_tables_hash, + ut_fold_string(table->name), table); + + mutex_enter(&fil_system->mutex); + fil_space = fil_space_get_by_name(dest_space_name); + mutex_exit(&fil_system->mutex); + + if (fil_space != NULL) { + if (fil_space->id == space_id || space_id == ULINT_UNDEFINED) { + /* we found matching space */ + goto found; + } else { + + char tmpname[FN_REFLEN]; + + snprintf(tmpname, FN_REFLEN, "%s/xtrabackup_tmp_#%lu", + dbname, fil_space->id); + + msg("xtrabackup: Renaming %s to %s.ibd\n", + fil_space->name, tmpname); + + if (!fil_rename_tablespace(NULL, fil_space->id, + tmpname, NULL)) + { + msg("xtrabackup: Cannot rename %s to %s\n", + fil_space->name, tmpname); + goto exit; + } + } + } + + if (space_id == ULINT_UNDEFINED) + { + msg("xtrabackup: Error: Cannot handle DDL operation on tablespace " + "%s\n", dest_space_name); + exit(EXIT_FAILURE); + } + mutex_enter(&fil_system->mutex); + fil_space = fil_space_get_by_id(space_id); + mutex_exit(&fil_system->mutex); + if (fil_space != NULL) { + char tmpname[FN_REFLEN]; + + strncpy(tmpname, dest_space_name, FN_REFLEN); + + msg("xtrabackup: Renaming %s to %s\n", + fil_space->name, dest_space_name); + + if (!fil_rename_tablespace(NULL, fil_space->id, tmpname, + NULL)) + { + msg("xtrabackup: Cannot rename %s to %s\n", + fil_space->name, dest_space_name); + goto exit; + } + + goto found; + } + + /* No matching space found. create the new one. */ + + if (!fil_space_create(dest_space_name, space_id, 0, + FIL_TABLESPACE, 0, false)) { + msg("xtrabackup: Cannot create tablespace %s\n", + dest_space_name); + goto exit; + } + + /* Calculate correct tablespace flags for compressed tablespaces. */ + if (!zip_size || zip_size == ULINT_UNDEFINED) { + tablespace_flags = 0; + } + else { + tablespace_flags + = (get_bit_shift(zip_size >> PAGE_ZIP_MIN_SIZE_SHIFT + << 1) + << DICT_TF_ZSSIZE_SHIFT) + | DICT_TF_COMPACT + | (DICT_TF_FORMAT_ZIP << DICT_TF_FORMAT_SHIFT); + ut_a(dict_tf_get_zip_size(tablespace_flags) + == zip_size); + } + *success = xb_space_create_file(real_name, space_id, tablespace_flags, + &file); + goto exit; + +found: + /* open the file and return it's handle */ + + file = os_file_create_simple_no_error_handling(0, real_name, + OS_FILE_OPEN, + OS_FILE_READ_WRITE, + &ok,0); + + if (ok) { + *success = TRUE; + } else { + msg("xtrabackup: Cannot open file %s\n", real_name); + } + +exit: + + return file; +} + +/************************************************************************ +Applies a given .delta file to the corresponding data file. +@return TRUE on success */ +static +ibool +xtrabackup_apply_delta( + const char* dirname, /* in: dir name of incremental */ + const char* dbname, /* in: database name (ibdata: NULL) */ + const char* filename, /* in: file name (not a path), + including the .delta extension */ + void* /*data*/) +{ + os_file_t src_file = XB_FILE_UNDEFINED; + os_file_t dst_file = XB_FILE_UNDEFINED; + char src_path[FN_REFLEN]; + char dst_path[FN_REFLEN]; + char meta_path[FN_REFLEN]; + char space_name[FN_REFLEN]; + ibool success; + + ibool last_buffer = FALSE; + ulint page_in_buffer; + ulint incremental_buffers = 0; + + xb_delta_info_t info; + ulint page_size; + ulint page_size_shift; + byte* incremental_buffer_base = NULL; + byte* incremental_buffer; + + size_t offset; + + ut_a(xtrabackup_incremental); + + if (dbname) { + snprintf(src_path, sizeof(src_path), "%s/%s/%s", + dirname, dbname, filename); + snprintf(dst_path, sizeof(dst_path), "%s/%s/%s", + xtrabackup_real_target_dir, dbname, filename); + } else { + snprintf(src_path, sizeof(src_path), "%s/%s", + dirname, filename); + snprintf(dst_path, sizeof(dst_path), "%s/%s", + xtrabackup_real_target_dir, filename); + } + dst_path[strlen(dst_path) - 6] = '\0'; + + strncpy(space_name, filename, FN_REFLEN); + space_name[strlen(space_name) - 6] = 0; + + if (!get_meta_path(src_path, meta_path)) { + goto error; + } + + srv_normalize_path_for_win(dst_path); + srv_normalize_path_for_win(src_path); + srv_normalize_path_for_win(meta_path); + + if (!xb_read_delta_metadata(meta_path, &info)) { + goto error; + } + + page_size = info.page_size; + page_size_shift = get_bit_shift(page_size); + msg("xtrabackup: page size for %s is %lu bytes\n", + src_path, page_size); + if (page_size_shift < 10 || + page_size_shift > UNIV_PAGE_SIZE_SHIFT_MAX) { + msg("xtrabackup: error: invalid value of page_size " + "(%lu bytes) read from %s\n", page_size, meta_path); + goto error; + } + + src_file = os_file_create_simple_no_error_handling(0, src_path, + OS_FILE_OPEN, + OS_FILE_READ_WRITE, + &success,0); + if (!success) { + os_file_get_last_error(TRUE); + msg("xtrabackup: error: cannot open %s\n", src_path); + goto error; + } + + posix_fadvise(src_file, 0, 0, POSIX_FADV_SEQUENTIAL); + + os_file_set_nocache(src_file, src_path, "OPEN"); + + dst_file = xb_delta_open_matching_space( + dbname, space_name, info.space_id, info.zip_size, + dst_path, sizeof(dst_path), &success); + if (!success) { + msg("xtrabackup: error: cannot open %s\n", dst_path); + goto error; + } + + posix_fadvise(dst_file, 0, 0, POSIX_FADV_DONTNEED); + + os_file_set_nocache(dst_file, dst_path, "OPEN"); + + /* allocate buffer for incremental backup (4096 pages) */ + incremental_buffer_base = static_cast<byte *> + (ut_malloc((page_size / 4 + 1) * + page_size)); + incremental_buffer = static_cast<byte *> + (ut_align(incremental_buffer_base, + page_size)); + + msg("Applying %s to %s...\n", src_path, dst_path); + + while (!last_buffer) { + ulint cluster_header; + + /* read to buffer */ + /* first block of block cluster */ + offset = ((incremental_buffers * (page_size / 4)) + << page_size_shift); + success = os_file_read(src_file, incremental_buffer, + offset, page_size); + if (!success) { + goto error; + } + + cluster_header = mach_read_from_4(incremental_buffer); + switch(cluster_header) { + case 0x78747261UL: /*"xtra"*/ + break; + case 0x58545241UL: /*"XTRA"*/ + last_buffer = TRUE; + break; + default: + msg("xtrabackup: error: %s seems not " + ".delta file.\n", src_path); + goto error; + } + + for (page_in_buffer = 1; page_in_buffer < page_size / 4; + page_in_buffer++) { + if (mach_read_from_4(incremental_buffer + page_in_buffer * 4) + == 0xFFFFFFFFUL) + break; + } + + ut_a(last_buffer || page_in_buffer == page_size / 4); + + /* read whole of the cluster */ + success = os_file_read(src_file, incremental_buffer, + offset, page_in_buffer * page_size); + if (!success) { + goto error; + } + + posix_fadvise(src_file, offset, page_in_buffer * page_size, + POSIX_FADV_DONTNEED); + + for (page_in_buffer = 1; page_in_buffer < page_size / 4; + page_in_buffer++) { + ulint offset_on_page; + + offset_on_page = mach_read_from_4(incremental_buffer + page_in_buffer * 4); + + if (offset_on_page == 0xFFFFFFFFUL) + break; + + success = os_file_write(dst_path, dst_file, + incremental_buffer + + page_in_buffer * page_size, + (offset_on_page << + page_size_shift), + page_size); + if (!success) { + goto error; + } + } + + incremental_buffers++; + } + + if (incremental_buffer_base) + ut_free(incremental_buffer_base); + if (src_file != XB_FILE_UNDEFINED) + os_file_close(src_file); + if (dst_file != XB_FILE_UNDEFINED) + os_file_close(dst_file); + return TRUE; + +error: + if (incremental_buffer_base) + ut_free(incremental_buffer_base); + if (src_file != XB_FILE_UNDEFINED) + os_file_close(src_file); + if (dst_file != XB_FILE_UNDEFINED) + os_file_close(dst_file); + msg("xtrabackup: Error: xtrabackup_apply_delta(): " + "failed to apply %s to %s.\n", src_path, dst_path); + return FALSE; +} + +/************************************************************************ +Callback to handle datadir entry. Function of this type will be called +for each entry which matches the mask by xb_process_datadir. +@return should return TRUE on success */ +typedef ibool (*handle_datadir_entry_func_t)( +/*=========================================*/ + const char* data_home_dir, /*!<in: path to datadir */ + const char* db_name, /*!<in: database name */ + const char* file_name, /*!<in: file name with suffix */ + void* arg); /*!<in: caller-provided data */ + +/************************************************************************ +Callback to handle datadir entry. Deletes entry if it has no matching +fil_space in fil_system directory. +@return FALSE if delete attempt was unsuccessful */ +static +ibool +rm_if_not_found( + const char* data_home_dir, /*!<in: path to datadir */ + const char* db_name, /*!<in: database name */ + const char* file_name, /*!<in: file name with suffix */ + void* arg __attribute__((unused))) +{ + char name[FN_REFLEN]; + xb_filter_entry_t* table; + + snprintf(name, FN_REFLEN, "%s/%s", db_name, file_name); + /* Truncate ".ibd" */ + name[strlen(name) - 4] = '\0'; + + HASH_SEARCH(name_hash, inc_dir_tables_hash, ut_fold_string(name), + xb_filter_entry_t*, + table, (void) 0, + !strcmp(table->name, name)); + + if (!table) { + snprintf(name, FN_REFLEN, "%s/%s/%s", data_home_dir, + db_name, file_name); + return os_file_delete(0, name); + } + + return(TRUE); +} + +/************************************************************************ +Function enumerates files in datadir (provided by path) which are matched +by provided suffix. For each entry callback is called. +@return FALSE if callback for some entry returned FALSE */ +static +ibool +xb_process_datadir( + const char* path, /*!<in: datadir path */ + const char* suffix, /*!<in: suffix to match + against */ + handle_datadir_entry_func_t func, /*!<in: callback */ + void* data) /*!<in: additional argument for + callback */ +{ + ulint ret; + char dbpath[FN_REFLEN]; + os_file_dir_t dir; + os_file_dir_t dbdir; + os_file_stat_t dbinfo; + os_file_stat_t fileinfo; + ulint suffix_len; + dberr_t err = DB_SUCCESS; + static char current_dir[2]; + + current_dir[0] = FN_CURLIB; + current_dir[1] = 0; + srv_data_home = current_dir; + + suffix_len = strlen(suffix); + + /* datafile */ + dbdir = os_file_opendir(path, FALSE); + + if (dbdir != NULL) { + ret = fil_file_readdir_next_file(&err, path, dbdir, + &fileinfo); + while (ret == 0) { + if (fileinfo.type == OS_FILE_TYPE_DIR) { + goto next_file_item_1; + } + + if (strlen(fileinfo.name) > suffix_len + && 0 == strcmp(fileinfo.name + + strlen(fileinfo.name) - suffix_len, + suffix)) { + if (!func( + path, NULL, + fileinfo.name, data)) + { + return(FALSE); + } + } +next_file_item_1: + ret = fil_file_readdir_next_file(&err, + path, dbdir, + &fileinfo); + } + + os_file_closedir(dbdir); + } else { + msg("xtrabackup: Cannot open dir %s\n", + path); + } + + /* single table tablespaces */ + dir = os_file_opendir(path, FALSE); + + if (dir == NULL) { + msg("xtrabackup: Cannot open dir %s\n", + path); + } + + ret = fil_file_readdir_next_file(&err, path, dir, + &dbinfo); + while (ret == 0) { + if (dbinfo.type == OS_FILE_TYPE_FILE + || dbinfo.type == OS_FILE_TYPE_UNKNOWN) { + + goto next_datadir_item; + } + + sprintf(dbpath, "%s/%s", path, + dbinfo.name); + srv_normalize_path_for_win(dbpath); + + dbdir = os_file_opendir(dbpath, FALSE); + + if (dbdir != NULL) { + + ret = fil_file_readdir_next_file(&err, dbpath, dbdir, + &fileinfo); + while (ret == 0) { + + if (fileinfo.type == OS_FILE_TYPE_DIR) { + + goto next_file_item_2; + } + + if (strlen(fileinfo.name) > suffix_len + && 0 == strcmp(fileinfo.name + + strlen(fileinfo.name) - + suffix_len, + suffix)) { + /* The name ends in suffix; process + the file */ + if (!func( + path, + dbinfo.name, + fileinfo.name, data)) + { + return(FALSE); + } + } +next_file_item_2: + ret = fil_file_readdir_next_file(&err, + dbpath, dbdir, + &fileinfo); + } + + os_file_closedir(dbdir); + } +next_datadir_item: + ret = fil_file_readdir_next_file(&err, + path, + dir, &dbinfo); + } + + os_file_closedir(dir); + + return(TRUE); +} + +/************************************************************************ +Applies all .delta files from incremental_dir to the full backup. +@return TRUE on success. */ +static +ibool +xtrabackup_apply_deltas() +{ + return xb_process_datadir(xtrabackup_incremental_dir, ".delta", + xtrabackup_apply_delta, NULL); +} + +static my_bool +xtrabackup_close_temp_log(my_bool clear_flag) +{ + os_file_t src_file = XB_FILE_UNDEFINED; + char src_path[FN_REFLEN]; + char dst_path[FN_REFLEN]; + ibool success; + byte log_buf[UNIV_PAGE_SIZE_MAX]; + + if (!xtrabackup_logfile_is_renamed) + return(FALSE); + + /* rename 'ib_logfile0' to 'xtrabackup_logfile' */ + if(!xtrabackup_incremental_dir) { + sprintf(dst_path, "%s/ib_logfile0", xtrabackup_target_dir); + sprintf(src_path, "%s/%s", xtrabackup_target_dir, + XB_LOG_FILENAME); + } else { + sprintf(dst_path, "%s/ib_logfile0", xtrabackup_incremental_dir); + sprintf(src_path, "%s/%s", xtrabackup_incremental_dir, + XB_LOG_FILENAME); + } + + srv_normalize_path_for_win(dst_path); + srv_normalize_path_for_win(src_path); + + success = os_file_rename(0, dst_path, src_path); + if (!success) { + goto error; + } + xtrabackup_logfile_is_renamed = FALSE; + + if (!clear_flag) + return(FALSE); + + /* clear LOG_FILE_WAS_CREATED_BY_HOT_BACKUP field */ + src_file = os_file_create_simple_no_error_handling(0, src_path, + OS_FILE_OPEN, + OS_FILE_READ_WRITE, + &success,0); + if (!success) { + goto error; + } + + success = os_file_read(src_file, log_buf, 0, LOG_FILE_HDR_SIZE); + if (!success) { + goto error; + } + + memset(log_buf + LOG_FILE_WAS_CREATED_BY_HOT_BACKUP, ' ', 4); + + success = os_file_write(src_path, src_file, log_buf, 0, + LOG_FILE_HDR_SIZE); + if (!success) { + goto error; + } + + os_file_close(src_file); + src_file = XB_FILE_UNDEFINED; + + innobase_log_files_in_group = innobase_log_files_in_group_save; + srv_log_group_home_dir = srv_log_group_home_dir_save; + innobase_log_file_size = innobase_log_file_size_save; + + return(FALSE); +error: + if (src_file != XB_FILE_UNDEFINED) + os_file_close(src_file); + msg("xtrabackup: Error: xtrabackup_close_temp_log() failed.\n"); + return(TRUE); /*ERROR*/ +} + + +/*********************************************************************//** +Write the meta data (index user fields) config file. +@return true in case of success otherwise false. */ +static +bool +xb_export_cfg_write_index_fields( +/*===========================*/ + const dict_index_t* index, /*!< in: write the meta data for + this index */ + FILE* file) /*!< in: file to write to */ +{ + byte row[sizeof(ib_uint32_t) * 2]; + + for (ulint i = 0; i < index->n_fields; ++i) { + byte* ptr = row; + const dict_field_t* field = &index->fields[i]; + + mach_write_to_4(ptr, field->prefix_len); + ptr += sizeof(ib_uint32_t); + + mach_write_to_4(ptr, field->fixed_len); + + if (fwrite(row, 1, sizeof(row), file) != sizeof(row)) { + + msg("xtrabackup: Error: writing index fields."); + + return(false); + } + + /* Include the NUL byte in the length. */ + ib_uint32_t len = (ib_uint32_t)strlen(field->name) + 1; + ut_a(len > 1); + + mach_write_to_4(row, len); + + if (fwrite(row, 1, sizeof(len), file) != sizeof(len) + || fwrite(field->name, 1, len, file) != len) { + + msg("xtrabackup: Error: writing index column."); + + return(false); + } + } + + return(true); +} + +/*********************************************************************//** +Write the meta data config file index information. +@return true in case of success otherwise false. */ +static __attribute__((nonnull, warn_unused_result)) +bool +xb_export_cfg_write_indexes( +/*======================*/ + const dict_table_t* table, /*!< in: write the meta data for + this table */ + FILE* file) /*!< in: file to write to */ +{ + { + byte row[sizeof(ib_uint32_t)]; + + /* Write the number of indexes in the table. */ + mach_write_to_4(row, UT_LIST_GET_LEN(table->indexes)); + + if (fwrite(row, 1, sizeof(row), file) != sizeof(row)) { + msg("xtrabackup: Error: writing index count."); + + return(false); + } + } + + bool ret = true; + + /* Write the index meta data. */ + for (const dict_index_t* index = UT_LIST_GET_FIRST(table->indexes); + index != 0 && ret; + index = UT_LIST_GET_NEXT(indexes, index)) { + + byte* ptr; + byte row[sizeof(ib_uint64_t) + + sizeof(ib_uint32_t) * 8]; + + ptr = row; + + ut_ad(sizeof(ib_uint64_t) == 8); + mach_write_to_8(ptr, index->id); + ptr += sizeof(ib_uint64_t); + + mach_write_to_4(ptr, index->space); + ptr += sizeof(ib_uint32_t); + + mach_write_to_4(ptr, index->page); + ptr += sizeof(ib_uint32_t); + + mach_write_to_4(ptr, index->type); + ptr += sizeof(ib_uint32_t); + + mach_write_to_4(ptr, index->trx_id_offset); + ptr += sizeof(ib_uint32_t); + + mach_write_to_4(ptr, index->n_user_defined_cols); + ptr += sizeof(ib_uint32_t); + + mach_write_to_4(ptr, index->n_uniq); + ptr += sizeof(ib_uint32_t); + + mach_write_to_4(ptr, index->n_nullable); + ptr += sizeof(ib_uint32_t); + + mach_write_to_4(ptr, index->n_fields); + + if (fwrite(row, 1, sizeof(row), file) != sizeof(row)) { + + msg("xtrabackup: Error: writing index meta-data."); + + return(false); + } + + /* Write the length of the index name. + NUL byte is included in the length. */ + ib_uint32_t len = (ib_uint32_t)strlen(index->name) + 1; + ut_a(len > 1); + + mach_write_to_4(row, len); + + if (fwrite(row, 1, sizeof(len), file) != sizeof(len) + || fwrite(index->name, 1, len, file) != len) { + + msg("xtrabackup: Error: writing index name."); + + return(false); + } + + ret = xb_export_cfg_write_index_fields(index, file); + } + + return(ret); +} + +/*********************************************************************//** +Write the meta data (table columns) config file. Serialise the contents of +dict_col_t structure, along with the column name. All fields are serialized +as ib_uint32_t. +@return true in case of success otherwise false. */ +static __attribute__((nonnull, warn_unused_result)) +bool +xb_export_cfg_write_table( +/*====================*/ + const dict_table_t* table, /*!< in: write the meta data for + this table */ + FILE* file) /*!< in: file to write to */ +{ + dict_col_t* col; + byte row[sizeof(ib_uint32_t) * 7]; + + col = table->cols; + + for (ulint i = 0; i < table->n_cols; ++i, ++col) { + byte* ptr = row; + + mach_write_to_4(ptr, col->prtype); + ptr += sizeof(ib_uint32_t); + + mach_write_to_4(ptr, col->mtype); + ptr += sizeof(ib_uint32_t); + + mach_write_to_4(ptr, col->len); + ptr += sizeof(ib_uint32_t); + + mach_write_to_4(ptr, col->mbminmaxlen); + ptr += sizeof(ib_uint32_t); + + mach_write_to_4(ptr, col->ind); + ptr += sizeof(ib_uint32_t); + + mach_write_to_4(ptr, col->ord_part); + ptr += sizeof(ib_uint32_t); + + mach_write_to_4(ptr, col->max_prefix); + + if (fwrite(row, 1, sizeof(row), file) != sizeof(row)) { + msg("xtrabackup: Error: writing table column data."); + + return(false); + } + + /* Write out the column name as [len, byte array]. The len + includes the NUL byte. */ + ib_uint32_t len; + const char* col_name; + + col_name = dict_table_get_col_name(table, dict_col_get_no(col)); + + /* Include the NUL byte in the length. */ + len = (ib_uint32_t)strlen(col_name) + 1; + ut_a(len > 1); + + mach_write_to_4(row, len); + + if (fwrite(row, 1, sizeof(len), file) != sizeof(len) + || fwrite(col_name, 1, len, file) != len) { + + msg("xtrabackup: Error: writing column name."); + + return(false); + } + } + + return(true); +} + +/*********************************************************************//** +Write the meta data config file header. +@return true in case of success otherwise false. */ +static __attribute__((nonnull, warn_unused_result)) +bool +xb_export_cfg_write_header( +/*=====================*/ + const dict_table_t* table, /*!< in: write the meta data for + this table */ + FILE* file) /*!< in: file to write to */ +{ + byte value[sizeof(ib_uint32_t)]; + + /* Write the meta-data version number. */ + mach_write_to_4(value, IB_EXPORT_CFG_VERSION_V1); + + if (fwrite(&value, 1, sizeof(value), file) != sizeof(value)) { + msg("xtrabackup: Error: writing meta-data version number."); + + return(false); + } + + /* Write the server hostname. */ + ib_uint32_t len; + const char* hostname = "Hostname unknown"; + + /* The server hostname includes the NUL byte. */ + len = (ib_uint32_t)strlen(hostname) + 1; + mach_write_to_4(value, len); + + if (fwrite(&value, 1, sizeof(value), file) != sizeof(value) + || fwrite(hostname, 1, len, file) != len) { + + msg("xtrabackup: Error: writing hostname."); + + return(false); + } + + /* The table name includes the NUL byte. */ + ut_a(table->name != 0); + len = (ib_uint32_t)strlen(table->name) + 1; + + /* Write the table name. */ + mach_write_to_4(value, len); + + if (fwrite(&value, 1, sizeof(value), file) != sizeof(value) + || fwrite(table->name, 1, len, file) != len) { + + msg("xtrabackup: Error: writing table name."); + + return(false); + } + + byte row[sizeof(ib_uint32_t) * 3]; + + /* Write the next autoinc value. */ + mach_write_to_8(row, table->autoinc); + + if (fwrite(row, 1, sizeof(ib_uint64_t), file) != sizeof(ib_uint64_t)) { + msg("xtrabackup: Error: writing table autoinc value."); + + return(false); + } + + byte* ptr = row; + + /* Write the system page size. */ + mach_write_to_4(ptr, UNIV_PAGE_SIZE); + ptr += sizeof(ib_uint32_t); + + /* Write the table->flags. */ + mach_write_to_4(ptr, table->flags); + ptr += sizeof(ib_uint32_t); + + /* Write the number of columns in the table. */ + mach_write_to_4(ptr, table->n_cols); + + if (fwrite(row, 1, sizeof(row), file) != sizeof(row)) { + msg("xtrabackup: Error: writing table meta-data."); + + return(false); + } + + return(true); +} + +/*********************************************************************//** +Write MySQL 5.6-style meta data config file. +@return true in case of success otherwise false. */ +static +bool +xb_export_cfg_write( + const fil_node_t* node, + const dict_table_t* table) /*!< in: write the meta data for + this table */ +{ + char file_path[FN_REFLEN]; + FILE* file; + bool success; + + strcpy(file_path, node->name); + strcpy(file_path + strlen(file_path) - 4, ".cfg"); + + file = fopen(file_path, "w+b"); + + if (file == NULL) { + msg("xtrabackup: Error: cannot open %s\n", node->name); + + success = false; + } else { + + success = xb_export_cfg_write_header(table, file); + + if (success) { + success = xb_export_cfg_write_table(table, file); + } + + if (success) { + success = xb_export_cfg_write_indexes(table, file); + } + + if (fclose(file) != 0) { + msg("xtrabackup: Error: cannot close %s\n", node->name); + success = false; + } + + } + + return(success); + +} + +/********************************************************************//** +Searches archived log files in archived log directory. The min and max +LSN's of found files as well as archived log file size are stored in +xtrabackup_arch_first_file_lsn, xtrabackup_arch_last_file_lsn and +xtrabackup_arch_file_size respectively. +@return true on success +*/ +static +bool +xtrabackup_arch_search_files( +/*=========================*/ + ib_uint64_t start_lsn) /*!< in: filter out log files + witch does not contain data + with lsn < start_lsn */ +{ + os_file_dir_t dir; + os_file_stat_t fileinfo; + ut_ad(innobase_log_arch_dir); + + dir = os_file_opendir(innobase_log_arch_dir, FALSE); + if (!dir) { + msg("xtrabackup: error: cannot open archived log directory %s\n", + innobase_log_arch_dir); + return false; + } + + while(!os_file_readdir_next_file(innobase_log_arch_dir, + dir, + &fileinfo) ) { + lsn_t log_file_lsn; + char* log_str_end_lsn_ptr; + + if (strncmp(fileinfo.name, + IB_ARCHIVED_LOGS_PREFIX, + sizeof(IB_ARCHIVED_LOGS_PREFIX) - 1)) { + continue; + } + + log_file_lsn = strtoll(fileinfo.name + + sizeof(IB_ARCHIVED_LOGS_PREFIX) - 1, + &log_str_end_lsn_ptr, 10); + + if (*log_str_end_lsn_ptr) { + continue; + } + + if (log_file_lsn + (fileinfo.size - LOG_FILE_HDR_SIZE) < start_lsn) { + continue; + } + + if (!xtrabackup_arch_first_file_lsn || + log_file_lsn < xtrabackup_arch_first_file_lsn) { + xtrabackup_arch_first_file_lsn = log_file_lsn; + } + if (log_file_lsn > xtrabackup_arch_last_file_lsn) { + xtrabackup_arch_last_file_lsn = log_file_lsn; + } + + //TODO: find the more suitable way to extract archived log file + //size + if (fileinfo.size > (ib_int64_t)xtrabackup_arch_file_size) { + xtrabackup_arch_file_size = fileinfo.size; + } + } + + return xtrabackup_arch_first_file_lsn != 0; +} + +static +void +innodb_free_param() +{ + srv_free_paths_and_sizes(); + free(internal_innobase_data_file_path); + internal_innobase_data_file_path = NULL; + free_tmpdir(&mysql_tmpdir_list); +} + + +/************************************************************************** +Store the current binary log coordinates in a specified file. +@return 'false' on error. */ +static bool +store_binlog_info( +/*==============*/ + const char *filename) /*!< in: output file name */ +{ + FILE *fp; + + if (trx_sys_mysql_bin_log_name[0] == '\0') { + return(true); + } + + fp = fopen(filename, "w"); + + if (!fp) { + msg("xtrabackup: failed to open '%s'\n", filename); + return(false); + } + + fprintf(fp, "%s\t" UINT64PF "\n", + trx_sys_mysql_bin_log_name, trx_sys_mysql_bin_log_pos); + fclose(fp); + + return(true); +} + +static void +xtrabackup_prepare_func(int argc, char ** argv) +{ + ulint err; + datafiles_iter_t *it; + fil_node_t *node; + fil_space_t *space; + char metadata_path[FN_REFLEN]; + + /* cd to target-dir */ + + if (my_setwd(xtrabackup_real_target_dir,MYF(MY_WME))) + { + msg("xtrabackup: cannot my_setwd %s\n", + xtrabackup_real_target_dir); + exit(EXIT_FAILURE); + } + msg("xtrabackup: cd to %s\n", xtrabackup_real_target_dir); + + encryption_plugin_prepare_init(argc, argv); + + xtrabackup_target_dir= mysql_data_home_buff; + xtrabackup_target_dir[0]=FN_CURLIB; // all paths are relative from here + xtrabackup_target_dir[1]=0; + + /* + read metadata of target, we don't need metadata reading in the case + archived logs applying + */ + sprintf(metadata_path, "%s/%s", xtrabackup_target_dir, + XTRABACKUP_METADATA_FILENAME); + + if (!xtrabackup_read_metadata(metadata_path)) { + msg("xtrabackup: Error: failed to read metadata from '%s'\n", + metadata_path); + exit(EXIT_FAILURE); + } + + if (!innobase_log_arch_dir) + { + if (!strcmp(metadata_type, "full-backuped")) { + msg("xtrabackup: This target seems to be not prepared " + "yet.\n"); + } else if (!strcmp(metadata_type, "log-applied")) { + msg("xtrabackup: This target seems to be already " + "prepared with --apply-log-only.\n"); + goto skip_check; + } else if (!strcmp(metadata_type, "full-prepared")) { + msg("xtrabackup: This target seems to be already " + "prepared.\n"); + } else { + msg("xtrabackup: This target seems not to have correct " + "metadata...\n"); + exit(EXIT_FAILURE); + } + + if (xtrabackup_incremental) { + msg("xtrabackup: error: applying incremental backup " + "needs target prepared with --apply-log-only.\n"); + exit(EXIT_FAILURE); + } +skip_check: + if (xtrabackup_incremental + && metadata_to_lsn != incremental_lsn) { + msg("xtrabackup: error: This incremental backup seems " + "not to be proper for the target.\n" + "xtrabackup: Check 'to_lsn' of the target and " + "'from_lsn' of the incremental.\n"); + exit(EXIT_FAILURE); + } + } + + /* Create logfiles for recovery from 'xtrabackup_logfile', before start InnoDB */ + srv_max_n_threads = 1000; + ut_mem_init(); + /* temporally dummy value to avoid crash */ + srv_page_size_shift = 14; + srv_page_size = (1 << srv_page_size_shift); + os_sync_init(); + sync_init(); + os_io_init_simple(); + mem_init(srv_mem_pool_size); + ut_crc32_init(); + +#ifdef WITH_INNODB_DISALLOW_WRITES + srv_allow_writes_event = os_event_create(); + os_event_set(srv_allow_writes_event); +#endif + + xb_filters_init(); + + if(!innobase_log_arch_dir && xtrabackup_init_temp_log()) + goto error_cleanup; + + if(innodb_init_param()) { + goto error_cleanup; + } + + xb_normalize_init_values(); + + if (xtrabackup_incremental || innobase_log_arch_dir) { + err = xb_data_files_init(); + if (err != DB_SUCCESS) { + msg("xtrabackup: error: xb_data_files_init() failed " + "with error code %lu\n", err); + goto error_cleanup; + } + } + if (xtrabackup_incremental) { + inc_dir_tables_hash = hash_create(1000); + + if(!xtrabackup_apply_deltas()) { + xb_data_files_close(); + xb_filter_hash_free(inc_dir_tables_hash); + goto error_cleanup; + } + } + if (xtrabackup_incremental || innobase_log_arch_dir) { + xb_data_files_close(); + } + if (xtrabackup_incremental) { + /* Cleanup datadir from tablespaces deleted between full and + incremental backups */ + + xb_process_datadir("./", ".ibd", rm_if_not_found, NULL); + + xb_filter_hash_free(inc_dir_tables_hash); + } + if (fil_system) { + fil_close(); + } + + mem_close(); + ut_free_all_mem(); + + innodb_free_param(); + sync_close(); + sync_initialized = FALSE; + + /* Reset the configuration as it might have been changed by + xb_data_files_init(). */ + if(innodb_init_param()) { + goto error_cleanup; + } + + srv_apply_log_only = (bool) xtrabackup_apply_log_only; + + /* increase IO threads */ + if(srv_n_file_io_threads < 10) { + srv_n_read_io_threads = 4; + srv_n_write_io_threads = 4; + } + + if (innobase_log_arch_dir) { + srv_arch_dir = innobase_log_arch_dir; + srv_archive_recovery = true; + if (xtrabackup_archived_to_lsn) { + if (xtrabackup_archived_to_lsn < metadata_last_lsn) { + msg("xtrabackup: warning: logs applying lsn " + "limit " UINT64PF " is " + "less than metadata last-lsn " UINT64PF + " and will be set to metadata last-lsn value\n", + xtrabackup_archived_to_lsn, + metadata_last_lsn); + xtrabackup_archived_to_lsn = metadata_last_lsn; + } + if (xtrabackup_archived_to_lsn < min_flushed_lsn) { + msg("xtrabackup: error: logs applying " + "lsn limit " UINT64PF " is less than " + "min_flushed_lsn " UINT64PF + ", there is nothing to do\n", + xtrabackup_archived_to_lsn, + min_flushed_lsn); + goto error_cleanup; + } + } + srv_archive_recovery_limit_lsn= xtrabackup_archived_to_lsn; + /* + Unfinished transactions are not rolled back during log applying + as they can be finished at the firther files applyings. + */ + xtrabackup_apply_log_only = srv_apply_log_only = true; + + if (!xtrabackup_arch_search_files(min_flushed_lsn)) { + goto error_cleanup; + } + + /* + Check if last log file last lsn is big enough to overlap + last scanned lsn read from metadata. + */ + if (xtrabackup_arch_last_file_lsn + + xtrabackup_arch_file_size - + LOG_FILE_HDR_SIZE < metadata_last_lsn) { + msg("xtrabackup: error: there are no enough archived logs " + "to apply\n"); + goto error_cleanup; + } + } + + msg("xtrabackup: Starting InnoDB instance for recovery.\n" + "xtrabackup: Using %lld bytes for buffer pool " + "(set by --use-memory parameter)\n", xtrabackup_use_memory); + + srv_max_buf_pool_modified_pct = (double)max_buf_pool_modified_pct; + + if (srv_max_dirty_pages_pct_lwm > srv_max_buf_pool_modified_pct) { + srv_max_dirty_pages_pct_lwm = srv_max_buf_pool_modified_pct; + } + + if(innodb_init()) + goto error_cleanup; + + if (xtrabackup_incremental) { + + it = datafiles_iter_new(fil_system); + if (it == NULL) { + msg("xtrabackup: Error: datafiles_iter_new() failed.\n"); + exit(EXIT_FAILURE); + } + + while ((node = datafiles_iter_next(it)) != NULL) { + byte *header; + ulint size; + ulint actual_size; + mtr_t mtr; + buf_block_t *block; + ulint flags; + + space = node->space; + + /* Align space sizes along with fsp header. We want to process + each space once, so skip all nodes except the first one in a + multi-node space. */ + if (UT_LIST_GET_PREV(chain, node) != NULL) { + continue; + } + + mtr_start(&mtr); + + mtr_s_lock(fil_space_get_latch(space->id, &flags), &mtr); + + block = buf_page_get(space->id, + dict_tf_get_zip_size(flags), + 0, RW_S_LATCH, &mtr); + header = FSP_HEADER_OFFSET + buf_block_get_frame(block); + + size = mtr_read_ulint(header + FSP_SIZE, MLOG_4BYTES, + &mtr); + + mtr_commit(&mtr); + + fil_extend_space_to_desired_size(&actual_size, space->id, size); + } + + datafiles_iter_free(it); + + } /* if (xtrabackup_incremental) */ + + if (xtrabackup_export) { + msg("xtrabackup: export option is specified.\n"); + os_file_t info_file = XB_FILE_UNDEFINED; + char info_file_path[FN_REFLEN]; + ibool success; + char table_name[FN_REFLEN]; + + byte* page; + byte* buf = NULL; + + buf = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE * 2)); + page = static_cast<byte *>(ut_align(buf, UNIV_PAGE_SIZE)); + + /* flush insert buffer at shutdwon */ + innobase_fast_shutdown = 0; + + it = datafiles_iter_new(fil_system); + if (it == NULL) { + msg("xtrabackup: Error: datafiles_iter_new() " + "failed.\n"); + exit(EXIT_FAILURE); + } + while ((node = datafiles_iter_next(it)) != NULL) { + int len; + char *next, *prev, *p; + dict_table_t* table; + dict_index_t* index; + ulint n_index; + + space = node->space; + + /* treat file_per_table only */ + if (!fil_is_user_tablespace_id(space->id)) { + continue; + } + + /* node exist == file exist, here */ + strcpy(info_file_path, node->name); +#ifdef _WIN32 + for (int i = 0; info_file_path[i]; i++) + if (info_file_path[i] == '\\') + info_file_path[i]= '/'; +#endif + strcpy(info_file_path + + strlen(info_file_path) - + 4, ".exp"); + + len =(ib_uint32_t)strlen(info_file_path); + + p = info_file_path; + prev = NULL; + while ((next = strchr(p, '/')) != NULL) + { + prev = p; + p = next + 1; + } + info_file_path[len - 4] = 0; + strncpy(table_name, prev, FN_REFLEN); + + info_file_path[len - 4] = '.'; + + mutex_enter(&(dict_sys->mutex)); + + table = dict_table_get_low(table_name); + if (!table) { + msg("xtrabackup: error: " + "cannot find dictionary " + "record of table %s\n", + table_name); + goto next_node; + } + + /* Write MySQL 5.6 .cfg file */ + if (!xb_export_cfg_write(node, table)) { + goto next_node; + } + + index = dict_table_get_first_index(table); + n_index = UT_LIST_GET_LEN(table->indexes); + if (n_index > 31) { + msg("xtrabackup: warning: table '%s' has more " + "than 31 indexes, .exp file was not " + "generated. Table will fail to import " + "on server version prior to 5.6.\n", + table->name); + goto next_node; + } + + /* init exp file */ + memset(page, 0, UNIV_PAGE_SIZE); + mach_write_to_4(page , 0x78706f72UL); + mach_write_to_4(page + 4, 0x74696e66UL);/*"xportinf"*/ + mach_write_to_4(page + 8, n_index); + strncpy((char *) page + 12, + table_name, 500); + + msg("xtrabackup: export metadata of " + "table '%s' to file `%s` " + "(%lu indexes)\n", + table_name, info_file_path, + n_index); + + n_index = 1; + while (index) { + mach_write_to_8(page + n_index * 512, index->id); + mach_write_to_4(page + n_index * 512 + 8, + index->page); + strncpy((char *) page + n_index * 512 + + 12, index->name, 500); + + msg("xtrabackup: name=%s, " + "id.low=%lu, page=%lu\n", + index->name, + (ulint)(index->id & + 0xFFFFFFFFUL), + (ulint) index->page); + index = dict_table_get_next_index(index); + n_index++; + } + + srv_normalize_path_for_win(info_file_path); + info_file = os_file_create( + 0, + info_file_path, + OS_FILE_OVERWRITE, + OS_FILE_NORMAL, OS_DATA_FILE, + &success,0); + if (!success) { + os_file_get_last_error(TRUE); + goto next_node; + } + success = os_file_write(info_file_path, + info_file, page, + 0, UNIV_PAGE_SIZE); + if (!success) { + os_file_get_last_error(TRUE); + goto next_node; + } + success = os_file_flush(info_file); + if (!success) { + os_file_get_last_error(TRUE); + goto next_node; + } +next_node: + if (info_file != XB_FILE_UNDEFINED) { + os_file_close(info_file); + info_file = XB_FILE_UNDEFINED; + } + mutex_exit(&(dict_sys->mutex)); + } + + ut_free(buf); + } + + /* print the binary log position */ + trx_sys_print_mysql_binlog_offset(); + msg("\n"); + + /* output to xtrabackup_binlog_pos_innodb and (if + backup_safe_binlog_info was available on the server) to + xtrabackup_binlog_info. In the latter case xtrabackup_binlog_pos_innodb + becomes redundant and is created only for compatibility. */ + if (!store_binlog_info("xtrabackup_binlog_pos_innodb") || + (recover_binlog_info && + !store_binlog_info(XTRABACKUP_BINLOG_INFO))) { + + exit(EXIT_FAILURE); + } + + if (innobase_log_arch_dir) + srv_start_lsn = log_sys->lsn = recv_sys->recovered_lsn; + + /* Check whether the log is applied enough or not. */ + if ((xtrabackup_incremental + && srv_start_lsn < incremental_to_lsn) + ||(!xtrabackup_incremental + && srv_start_lsn < metadata_to_lsn)) { + msg("xtrabackup: error: " + "The transaction log file is corrupted.\n" + "xtrabackup: error: " + "The log was not applied to the intended LSN!\n"); + msg("xtrabackup: Log applied to lsn " LSN_PF "\n", + srv_start_lsn); + if (xtrabackup_incremental) { + msg("xtrabackup: The intended lsn is " LSN_PF "\n", + incremental_to_lsn); + } else { + msg("xtrabackup: The intended lsn is " LSN_PF "\n", + metadata_to_lsn); + } + exit(EXIT_FAILURE); + } +#ifdef WITH_WSREP + xb_write_galera_info(xtrabackup_incremental); +#endif + + if(innodb_end()) + goto error_cleanup; + + innodb_free_param(); + + sync_initialized = FALSE; + + /* re-init necessary components */ + ut_mem_init(); + os_sync_init(); + sync_init(); + os_io_init_simple(); + + if(xtrabackup_close_temp_log(TRUE)) + exit(EXIT_FAILURE); + + /* output to metadata file */ + { + char filename[FN_REFLEN]; + + strcpy(metadata_type, srv_apply_log_only ? + "log-applied" : "full-prepared"); + + if(xtrabackup_incremental + && metadata_to_lsn < incremental_to_lsn) + { + metadata_to_lsn = incremental_to_lsn; + metadata_last_lsn = incremental_last_lsn; + } + + sprintf(filename, "%s/%s", xtrabackup_target_dir, XTRABACKUP_METADATA_FILENAME); + if (!xtrabackup_write_metadata(filename)) { + + msg("xtrabackup: Error: failed to write metadata " + "to '%s'\n", filename); + exit(EXIT_FAILURE); + } + + if(xtrabackup_extra_lsndir) { + sprintf(filename, "%s/%s", xtrabackup_extra_lsndir, XTRABACKUP_METADATA_FILENAME); + if (!xtrabackup_write_metadata(filename)) { + msg("xtrabackup: Error: failed to write " + "metadata to '%s'\n", filename); + exit(EXIT_FAILURE); + } + } + } + + if (!apply_log_finish()) { + exit(EXIT_FAILURE); + } + + sync_close(); + sync_initialized = FALSE; + if (fil_system) { + fil_close(); + } + + ut_free_all_mem(); + + /* start InnoDB once again to create log files */ + + if (!xtrabackup_apply_log_only) { + + /* xtrabackup_incremental_dir is used to indicate that + we are going to apply incremental backup. Here we already + applied incremental backup and are about to do final prepare + of the full backup */ + xtrabackup_incremental_dir = NULL; + + if(innodb_init_param()) { + goto error; + } + + srv_apply_log_only = false; + + /* increase IO threads */ + if(srv_n_file_io_threads < 10) { + srv_n_read_io_threads = 4; + srv_n_write_io_threads = 4; + } + + srv_shutdown_state = SRV_SHUTDOWN_NONE; + + if(innodb_init()) + goto error; + + if(innodb_end()) + goto error; + + innodb_free_param(); + + } + + xb_filters_free(); + + return; + +error_cleanup: + xtrabackup_close_temp_log(FALSE); + xb_filters_free(); + +error: + exit(EXIT_FAILURE); +} + +/************************************************************************** +Append group name to xb_load_default_groups list. */ +static +void +append_defaults_group(const char *group, const char *default_groups[], + size_t default_groups_size) +{ + uint i; + bool appended = false; + for (i = 0; i < default_groups_size - 1; i++) { + if (default_groups[i] == NULL) { + default_groups[i] = group; + appended = true; + break; + } + } + ut_a(appended); +} + +bool +xb_init() +{ + const char *mixed_options[4] = {NULL, NULL, NULL, NULL}; + int n_mixed_options; + + /* sanity checks */ + + if (opt_slave_info + && opt_no_lock + && !opt_safe_slave_backup) { + msg("Error: --slave-info is used with --no-lock but " + "without --safe-slave-backup. The binlog position " + "cannot be consistent with the backup data.\n"); + return(false); + } + + if (opt_rsync && xtrabackup_stream_fmt) { + msg("Error: --rsync doesn't work with --stream\n"); + return(false); + } + + n_mixed_options = 0; + + if (opt_decompress) { + mixed_options[n_mixed_options++] = "--decompress"; + } else if (opt_decrypt) { + mixed_options[n_mixed_options++] = "--decrypt"; + } + + if (xtrabackup_copy_back) { + mixed_options[n_mixed_options++] = "--copy-back"; + } + + if (xtrabackup_move_back) { + mixed_options[n_mixed_options++] = "--move-back"; + } + + if (xtrabackup_prepare) { + mixed_options[n_mixed_options++] = "--apply-log"; + } + + if (n_mixed_options > 1) { + msg("Error: %s and %s are mutually exclusive\n", + mixed_options[0], mixed_options[1]); + return(false); + } + + if (xtrabackup_backup) { + if ((mysql_connection = xb_mysql_connect()) == NULL) { + return(false); + } + + if (!get_mysql_vars(mysql_connection)) { + return(false); + } + + encryption_plugin_backup_init(mysql_connection); + history_start_time = time(NULL); + + } + + return(true); +} + + +extern void init_signals(void); + +#include <sql_locale.h> + +/* Messages . Avoid loading errmsg.sys file */ +void setup_error_messages() +{ + static const char *all_msgs[ER_ERROR_LAST - ER_ERROR_FIRST +1]; + my_default_lc_messages = &my_locale_en_US; + my_default_lc_messages->errmsgs->errmsgs = all_msgs; + + /* Populate the necessary error messages */ + struct { + int id; + const char *fmt; + } + xb_msgs[] = + { + { ER_DATABASE_NAME,"Database" }, + { ER_TABLE_NAME,"Table"}, + { ER_PARTITION_NAME, "Partition" }, + { ER_SUBPARTITION_NAME, "Subpartition" }, + { ER_TEMPORARY_NAME, "Temporary"}, + { ER_RENAMED_NAME, "Renamed"}, + { ER_CANT_FIND_DL_ENTRY, "Can't find symbol '%-.128s' in library"}, + { ER_CANT_OPEN_LIBRARY, "Can't open shared library '%-.192s' (errno: %d, %-.128s)" }, + { ER_OUTOFMEMORY, "Out of memory; restart server and try again (needed %d bytes)" }, + { ER_CANT_OPEN_LIBRARY, "Can't open shared library '%-.192s' (errno: %d, %-.128s)" }, + { ER_UDF_NO_PATHS, "No paths allowed for shared library" }, + { ER_CANT_INITIALIZE_UDF,"Can't initialize function '%-.192s'; %-.80s"}, + { ER_PLUGIN_IS_NOT_LOADED,"Plugin '%-.192s' is not loaded" } + }; + + for (int i = 0; i < (int)array_elements(all_msgs); i++) + all_msgs[i] = "Unknown error"; + + for (int i = 0; i < (int)array_elements(xb_msgs); i++) + all_msgs[xb_msgs[i].id - ER_ERROR_FIRST] = xb_msgs[i].fmt; +} + +extern my_bool(*dict_check_if_skip_table)(const char* name) ; + +void +handle_options(int argc, char **argv, char ***argv_client, char ***argv_server) +{ + /* Setup some variables for Innodb.*/ + + srv_xtrabackup = true; + + + files_charset_info = &my_charset_utf8_general_ci; + dict_check_if_skip_table = check_if_skip_table; + + setup_error_messages(); + sys_var_init(); + plugin_mutex_init(); + mysql_rwlock_init(key_rwlock_LOCK_system_variables_hash, &LOCK_system_variables_hash); + opt_stack_trace = 1; + test_flags |= TEST_SIGINT; + init_signals(); +#ifndef _WIN32 + /* Exit process on SIGINT. */ + my_sigset(SIGINT, SIG_DFL); +#endif + + sf_leaking_memory = 0; /* don't report memory leaks on early exist */ + + int i; + int ho_error; + + char* target_dir = NULL; + bool prepare = false; + + char conf_file[FN_REFLEN]; + int argc_client = argc; + int argc_server = argc; + + /* scan options for group and config file to load defaults from */ + for (i = 1; i < argc; i++) { + + char *optend = strcend(argv[i], '='); + + if (strncmp(argv[i], "--defaults-group", + optend - argv[i]) == 0) { + defaults_group = optend + 1; + append_defaults_group(defaults_group, + xb_server_default_groups, + array_elements(xb_server_default_groups)); + } + + if (strncmp(argv[i], "--login-path", + optend - argv[i]) == 0) { + append_defaults_group(optend + 1, + xb_client_default_groups, + array_elements(xb_client_default_groups)); + } + + if (!strncmp(argv[i], "--prepare", + optend - argv[i])) { + prepare = true; + } + + if (!strncmp(argv[i], "--apply-log", + optend - argv[i])) { + prepare = true; + } + + if (!strncmp(argv[i], "--target-dir", + optend - argv[i]) && *optend) { + target_dir = optend + 1; + } + + if (!*optend && argv[i][0] != '-') { + target_dir = argv[i]; + } + } + + snprintf(conf_file, sizeof(conf_file), "my"); + + if (prepare && target_dir) { + snprintf(conf_file, sizeof(conf_file), + "%s/backup-my.cnf", target_dir); + if (!strncmp(argv[1], "--defaults-file=", 16)) { + /* Remove defaults-file*/ + for (int i = 2; ; i++) { + if ((argv[i-1]= argv[i]) == 0) + break; + } + argc--; + } + } + + *argv_client = argv; + *argv_server = argv; + if (load_defaults(conf_file, xb_server_default_groups, + &argc_server, argv_server)) { + exit(EXIT_FAILURE); + } + + int n; + for (n = 0; (*argv_server)[n]; n++) {}; + argc_server = n; + + print_param_str << + "# This MySQL options file was generated by XtraBackup.\n" + "[" << defaults_group << "]\n"; + + /* We want xtrabackup to ignore unknown options, because it only + recognizes a small subset of server variables */ + my_getopt_skip_unknown = TRUE; + + /* Reset u_max_value for all options, as we don't want the + --maximum-... modifier to set the actual option values */ + for (my_option *optp= xb_server_options; optp->name; optp++) { + optp->u_max_value = (G_PTR *) &global_max_value; + } + + + /* Throw a descriptive error if --defaults-file or --defaults-extra-file + is not the first command line argument */ + for (int i = 2 ; i < argc ; i++) { + char *optend = strcend((argv)[i], '='); + + if (optend - argv[i] == 15 && + !strncmp(argv[i], "--defaults-file", optend - argv[i])) { + + msg("xtrabackup: Error: --defaults-file " + "must be specified first on the command " + "line\n"); + exit(EXIT_FAILURE); + } + if (optend - argv[i] == 21 && + !strncmp(argv[i], "--defaults-extra-file", + optend - argv[i])) { + + msg("xtrabackup: Error: --defaults-extra-file " + "must be specified first on the command " + "line\n"); + exit(EXIT_FAILURE); + } + } + + if (argc_server > 0 + && (ho_error=handle_options(&argc_server, argv_server, + xb_server_options, xb_get_one_option))) + exit(ho_error); + + if (load_defaults(conf_file, xb_client_default_groups, + &argc_client, argv_client)) { + exit(EXIT_FAILURE); + } + + for (n = 0; (*argv_client)[n]; n++) {}; + argc_client = n; + + if (strcmp(base_name(my_progname), INNOBACKUPEX_BIN_NAME) == 0 && + argc_client > 0) { + /* emulate innobackupex script */ + innobackupex_mode = true; + if (!ibx_handle_options(&argc_client, argv_client)) { + exit(EXIT_FAILURE); + } + } + + if (argc_client > 0 + && (ho_error=handle_options(&argc_client, argv_client, + xb_client_options, xb_get_one_option))) + exit(ho_error); + + /* Reject command line arguments that don't look like options, i.e. are + not of the form '-X' (single-character options) or '--option' (long + options) */ + for (int i = 0 ; i < argc_client ; i++) { + const char * const opt = (*argv_client)[i]; + + if (strncmp(opt, "--", 2) && + !(strlen(opt) == 2 && opt[0] == '-')) { + bool server_option = true; + + for (int j = 0; j < argc_server; j++) { + if (opt == (*argv_server)[j]) { + server_option = false; + break; + } + } + + if (!server_option) { + msg("xtrabackup: Error:" + " unknown argument: '%s'\n", opt); + exit(EXIT_FAILURE); + } + } + } +} + +/* ================= main =================== */ +extern my_bool(*fil_check_if_skip_database_by_path)(const char* name); + +int main(int argc, char **argv) +{ + char **client_defaults, **server_defaults; + char cwd[FN_REFLEN]; + static char INNOBACKUPEX_EXE[]= "innobackupex"; + if (argc > 1 && (strcmp(argv[1], "--innobackupex") == 0)) + { + argv++; + argc--; + argv[0] = INNOBACKUPEX_EXE; + innobackupex_mode = true; + } + + /* Setup skip fil_load_single_tablespaces callback.*/ + fil_check_if_skip_database_by_path = check_if_skip_database_by_path; + + init_signals(); + MY_INIT(argv[0]); + + pthread_key_create(&THR_THD, NULL); + my_pthread_setspecific_ptr(THR_THD, NULL); + + xb_regex_init(); + + capture_tool_command(argc, argv); + + if (mysql_server_init(-1, NULL, NULL)) + { + exit(EXIT_FAILURE); + } + + system_charset_info = &my_charset_utf8_general_ci; + key_map_full.set_all(); + + handle_options(argc, argv, &client_defaults, &server_defaults); + + int argc_server; + for (argc_server = 0; server_defaults[argc_server]; argc_server++) {} + + int argc_client; + for (argc_client = 0; client_defaults[argc_client]; argc_client++) {} + + + if (innobackupex_mode) { + if (!ibx_init()) { + exit(EXIT_FAILURE); + } + } + + if ((!xtrabackup_print_param) && (!xtrabackup_prepare) && (strcmp(mysql_data_home, "./") == 0)) { + if (!xtrabackup_print_param) + usage(); + msg("\nxtrabackup: Error: Please set parameter 'datadir'\n"); + exit(EXIT_FAILURE); + } + + /* Expand target-dir, incremental-basedir, etc. */ + + my_getwd(cwd, sizeof(cwd), MYF(0)); + + my_load_path(xtrabackup_real_target_dir, + xtrabackup_target_dir, cwd); + unpack_dirname(xtrabackup_real_target_dir, + xtrabackup_real_target_dir); + xtrabackup_target_dir= xtrabackup_real_target_dir; + + if (xtrabackup_incremental_basedir) { + my_load_path(xtrabackup_real_incremental_basedir, + xtrabackup_incremental_basedir, cwd); + unpack_dirname(xtrabackup_real_incremental_basedir, + xtrabackup_real_incremental_basedir); + xtrabackup_incremental_basedir = + xtrabackup_real_incremental_basedir; + } + + if (xtrabackup_incremental_dir) { + my_load_path(xtrabackup_real_incremental_dir, + xtrabackup_incremental_dir, cwd); + unpack_dirname(xtrabackup_real_incremental_dir, + xtrabackup_real_incremental_dir); + xtrabackup_incremental_dir = xtrabackup_real_incremental_dir; + } + + if (xtrabackup_extra_lsndir) { + my_load_path(xtrabackup_real_extra_lsndir, + xtrabackup_extra_lsndir, cwd); + unpack_dirname(xtrabackup_real_extra_lsndir, + xtrabackup_real_extra_lsndir); + xtrabackup_extra_lsndir = xtrabackup_real_extra_lsndir; + } + + /* get default temporary directory */ + if (!opt_mysql_tmpdir || !opt_mysql_tmpdir[0]) { + opt_mysql_tmpdir = getenv("TMPDIR"); +#if defined(__WIN__) + if (!opt_mysql_tmpdir) { + opt_mysql_tmpdir = getenv("TEMP"); + } + if (!opt_mysql_tmpdir) { + opt_mysql_tmpdir = getenv("TMP"); + } +#endif + if (!opt_mysql_tmpdir || !opt_mysql_tmpdir[0]) { + opt_mysql_tmpdir = const_cast<char*>(DEFAULT_TMPDIR); + } + } + + /* temporary setting of enough size */ + srv_page_size_shift = UNIV_PAGE_SIZE_SHIFT_MAX; + srv_page_size = UNIV_PAGE_SIZE_MAX; + if (xtrabackup_backup && xtrabackup_incremental) { + /* direct specification is only for --backup */ + /* and the lsn is prior to the other option */ + + char* endchar; + int error = 0; + incremental_lsn = strtoll(xtrabackup_incremental, &endchar, 10); + if (*endchar != '\0') + error = 1; + + if (error) { + msg("xtrabackup: value '%s' may be wrong format for " + "incremental option.\n", xtrabackup_incremental); + exit(EXIT_FAILURE); + } + } else if (xtrabackup_backup && xtrabackup_incremental_basedir) { + char filename[FN_REFLEN]; + + sprintf(filename, "%s/%s", xtrabackup_incremental_basedir, XTRABACKUP_METADATA_FILENAME); + + if (!xtrabackup_read_metadata(filename)) { + msg("xtrabackup: error: failed to read metadata from " + "%s\n", filename); + exit(EXIT_FAILURE); + } + + incremental_lsn = metadata_to_lsn; + xtrabackup_incremental = xtrabackup_incremental_basedir; //dummy + } else if (xtrabackup_prepare && xtrabackup_incremental_dir) { + char filename[FN_REFLEN]; + + sprintf(filename, "%s/%s", xtrabackup_incremental_dir, XTRABACKUP_METADATA_FILENAME); + + if (!xtrabackup_read_metadata(filename)) { + msg("xtrabackup: error: failed to read metadata from " + "%s\n", filename); + exit(EXIT_FAILURE); + } + + incremental_lsn = metadata_from_lsn; + incremental_to_lsn = metadata_to_lsn; + incremental_last_lsn = metadata_last_lsn; + xtrabackup_incremental = xtrabackup_incremental_dir; //dummy + + } else if (opt_incremental_history_name) { + xtrabackup_incremental = opt_incremental_history_name; + } else if (opt_incremental_history_uuid) { + xtrabackup_incremental = opt_incremental_history_uuid; + } else { + xtrabackup_incremental = NULL; + } + + if (!xb_init()) { + exit(EXIT_FAILURE); + } + + /* --print-param */ + if (xtrabackup_print_param) { + + printf("%s", print_param_str.str().c_str()); + + exit(EXIT_SUCCESS); + } + + print_version(); + if (xtrabackup_incremental) { + msg("incremental backup from " LSN_PF " is enabled.\n", + incremental_lsn); + } + + if (xtrabackup_export && innobase_file_per_table == FALSE) { + msg("xtrabackup: auto-enabling --innodb-file-per-table due to " + "the --export option\n"); + innobase_file_per_table = TRUE; + } + + if (xtrabackup_incremental && xtrabackup_stream && + xtrabackup_stream_fmt == XB_STREAM_FMT_TAR) { + msg("xtrabackup: error: " + "streaming incremental backups are incompatible with the \n" + "'tar' streaming format. Use --stream=xbstream instead.\n"); + exit(EXIT_FAILURE); + } + + if ((xtrabackup_compress || xtrabackup_encrypt) && xtrabackup_stream && + xtrabackup_stream_fmt == XB_STREAM_FMT_TAR) { + msg("xtrabackup: error: " + "compressed and encrypted backups are incompatible with the \n" + "'tar' streaming format. Use --stream=xbstream instead.\n"); + exit(EXIT_FAILURE); + } + + if (!xtrabackup_prepare && + (innobase_log_arch_dir || xtrabackup_archived_to_lsn)) { + + /* Default my.cnf can contain innobase_log_arch_dir option set + for server, reset it to allow backup. */ + innobase_log_arch_dir= NULL; + xtrabackup_archived_to_lsn= 0; + msg("xtrabackup: warning: " + "as --innodb-log-arch-dir and --to-archived-lsn can be used " + "only with --prepare they will be reset\n"); + } + + /* cannot execute both for now */ + { + int num = 0; + + if (xtrabackup_backup) num++; + if (xtrabackup_stats) num++; + if (xtrabackup_prepare) num++; + if (xtrabackup_copy_back) num++; + if (xtrabackup_move_back) num++; + if (xtrabackup_decrypt_decompress) num++; + if (num != 1) { /* !XOR (for now) */ + usage(); + exit(EXIT_FAILURE); + } + } + +#ifndef __WIN__ + if (xtrabackup_debug_sync) { + signal(SIGCONT, sigcont_handler); + } +#endif + + /* --backup */ + if (xtrabackup_backup) + xtrabackup_backup_func(); + + /* --stats */ + if (xtrabackup_stats) + xtrabackup_stats_func(argc_server,server_defaults); + + /* --prepare */ + if (xtrabackup_prepare) { + xtrabackup_prepare_func(argc_server, server_defaults); + } + + if (xtrabackup_copy_back || xtrabackup_move_back) { + if (!check_if_param_set("datadir")) { + msg("Error: datadir must be specified.\n"); + exit(EXIT_FAILURE); + } + if (!copy_back()) + exit(EXIT_FAILURE); + } + + if (xtrabackup_decrypt_decompress && !decrypt_decompress()) { + exit(EXIT_FAILURE); + } + + backup_cleanup(); + + if (innobackupex_mode) { + ibx_cleanup(); + } + + + free_defaults(client_defaults); + free_defaults(server_defaults); + + if (THR_THD) + (void) pthread_key_delete(THR_THD); + + msg_ts("completed OK!\n"); + + exit(EXIT_SUCCESS); +} diff --git a/extra/mariabackup/xtrabackup.h b/extra/mariabackup/xtrabackup.h new file mode 100644 index 00000000000..51491ce1f00 --- /dev/null +++ b/extra/mariabackup/xtrabackup.h @@ -0,0 +1,247 @@ +/****************************************************** +Copyright (c) 2011-2015 Percona LLC and/or its affiliates. + +Declarations for xtrabackup.cc + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#ifndef XB_XTRABACKUP_H +#define XB_XTRABACKUP_H + +#include <my_getopt.h> +#include "datasink.h" +#include "xbstream.h" +#include "changed_page_bitmap.h" + +#ifdef __WIN__ +#define XB_FILE_UNDEFINED NULL +#else +#define XB_FILE_UNDEFINED (-1) +#endif + +typedef struct { + ulint page_size; + ulint zip_size; + ulint space_id; +} xb_delta_info_t; + +/* ======== Datafiles iterator ======== */ +typedef struct { + fil_system_t *system; + fil_space_t *space; + fil_node_t *node; + ibool started; + os_ib_mutex_t mutex; +} datafiles_iter_t; + +/* value of the --incremental option */ +extern lsn_t incremental_lsn; + +extern char *xtrabackup_target_dir; +extern char *xtrabackup_incremental_dir; +extern char *xtrabackup_incremental_basedir; +extern char *innobase_data_home_dir; +extern char *innobase_buffer_pool_filename; +extern ds_ctxt_t *ds_meta; +extern ds_ctxt_t *ds_data; + +/* The last checkpoint LSN at the backup startup time */ +extern lsn_t checkpoint_lsn_start; + +extern xb_page_bitmap *changed_page_bitmap; + +extern char *xtrabackup_incremental; +extern my_bool xtrabackup_incremental_force_scan; + +extern lsn_t metadata_from_lsn; +extern lsn_t metadata_to_lsn; +extern lsn_t metadata_last_lsn; + +extern xb_stream_fmt_t xtrabackup_stream_fmt; +extern ibool xtrabackup_stream; + +extern char *xtrabackup_tables; +extern char *xtrabackup_tables_file; +extern char *xtrabackup_databases; +extern char *xtrabackup_databases_file; +extern char *xtrabackup_tables_exclude; +extern char *xtrabackup_databases_exclude; + +extern ibool xtrabackup_compress; +extern ibool xtrabackup_encrypt; + +extern my_bool xtrabackup_backup; +extern my_bool xtrabackup_prepare; +extern my_bool xtrabackup_apply_log_only; +extern my_bool xtrabackup_copy_back; +extern my_bool xtrabackup_move_back; +extern my_bool xtrabackup_decrypt_decompress; + +extern char *innobase_data_file_path; +extern char *innobase_doublewrite_file; +extern char *xtrabackup_encrypt_key; +extern char *xtrabackup_encrypt_key_file; +extern longlong innobase_log_file_size; +extern long innobase_log_files_in_group; +extern longlong innobase_page_size; + +extern const char *xtrabackup_encrypt_algo_names[]; +extern TYPELIB xtrabackup_encrypt_algo_typelib; + +extern int xtrabackup_parallel; + +extern my_bool xb_close_files; +extern const char *xtrabackup_compress_alg; +#ifdef __cplusplus +extern "C"{ +#endif + extern uint xtrabackup_compress_threads; + extern ulonglong xtrabackup_compress_chunk_size; +#ifdef __cplusplus +} +#endif +extern ulong xtrabackup_encrypt_algo; +extern uint xtrabackup_encrypt_threads; +extern ulonglong xtrabackup_encrypt_chunk_size; +extern my_bool xtrabackup_export; +extern char *xtrabackup_incremental_basedir; +extern char *xtrabackup_extra_lsndir; +extern char *xtrabackup_incremental_dir; +extern ulint xtrabackup_log_copy_interval; +extern char *xtrabackup_stream_str; +extern long xtrabackup_throttle; +extern longlong xtrabackup_use_memory; + +extern my_bool opt_galera_info; +extern my_bool opt_slave_info; +extern my_bool opt_no_lock; +extern my_bool opt_safe_slave_backup; +extern my_bool opt_rsync; +extern my_bool opt_force_non_empty_dirs; +extern my_bool opt_noversioncheck; +extern my_bool opt_no_backup_locks; +extern my_bool opt_decompress; +extern my_bool opt_remove_original; + +extern char *opt_incremental_history_name; +extern char *opt_incremental_history_uuid; + +extern char *opt_user; +extern char *opt_password; +extern char *opt_host; +extern char *opt_defaults_group; +extern char *opt_socket; +extern uint opt_port; +extern char *opt_login_path; +extern char *opt_log_bin; + +extern const char *query_type_names[]; + +enum query_type_t {QUERY_TYPE_ALL, QUERY_TYPE_UPDATE, + QUERY_TYPE_SELECT}; + +extern TYPELIB query_type_typelib; + +extern ulong opt_lock_wait_query_type; +extern ulong opt_kill_long_query_type; + +extern ulong opt_decrypt_algo; + +extern uint opt_kill_long_queries_timeout; +extern uint opt_lock_wait_timeout; +extern uint opt_lock_wait_threshold; +extern uint opt_debug_sleep_before_unlock; +extern uint opt_safe_slave_backup_timeout; + +extern const char *opt_history; +extern my_bool opt_decrypt; + +enum binlog_info_enum { BINLOG_INFO_OFF, BINLOG_INFO_LOCKLESS, BINLOG_INFO_ON, + BINLOG_INFO_AUTO}; + +extern ulong opt_binlog_info; + +void xtrabackup_io_throttling(void); +my_bool xb_write_delta_metadata(const char *filename, + const xb_delta_info_t *info); + +datafiles_iter_t *datafiles_iter_new(fil_system_t *f_system); +fil_node_t *datafiles_iter_next(datafiles_iter_t *it); +void datafiles_iter_free(datafiles_iter_t *it); + +/************************************************************************ +Initialize the tablespace memory cache and populate it by scanning for and +opening data files */ +ulint xb_data_files_init(void); + +/************************************************************************ +Destroy the tablespace memory cache. */ +void xb_data_files_close(void); + +/*********************************************************************** +Reads the space flags from a given data file and returns the compressed +page size, or 0 if the space is not compressed. */ +ulint xb_get_zip_size(os_file_t file); + +/************************************************************************ +Checks if a table specified as a name in the form "database/name" (InnoDB 5.6) +or "./database/name.ibd" (InnoDB 5.5-) should be skipped from backup based on +the --tables or --tables-file options. + +@return TRUE if the table should be skipped. */ +my_bool +check_if_skip_table( +/******************/ + const char* name); /*!< in: path to the table */ + + +/************************************************************************ +Checks if a database specified by path should be skipped from backup based on +the --databases, --databases_file or --databases_exclude options. + +@return TRUE if the table should be skipped. */ +my_bool +check_if_skip_database_by_path( + const char* path /*!< in: path to the db directory. */ +); + +/************************************************************************ +Check if parameter is set in defaults file or via command line argument +@return true if parameter is set. */ +bool +check_if_param_set(const char *param); + +#if defined(HAVE_OPENSSL) +extern my_bool opt_use_ssl; +extern my_bool opt_ssl_verify_server_cert; +#if !defined(HAVE_YASSL) +extern char *opt_server_public_key; +#endif +#endif + + +void +xtrabackup_backup_func(void); + +my_bool +xb_get_one_option(int optid, + const struct my_option *opt __attribute__((unused)), + char *argument); + +const char* +xb_get_copy_action(const char *dflt = "Copying"); + +#endif /* XB_XTRABACKUP_H */ diff --git a/include/my_crypt.h b/include/my_crypt.h index e1e94c9bd9d..719e349bfb9 100644 --- a/include/my_crypt.h +++ b/include/my_crypt.h @@ -18,74 +18,7 @@ #ifndef MY_CRYPT_INCLUDED #define MY_CRYPT_INCLUDED -#include <my_global.h> - -#ifdef __cplusplus -extern "C" { -#endif - -/* return values from my_aes_encrypt/my_aes_decrypt functions */ -#define MY_AES_OK 0 -#define MY_AES_BAD_DATA -100 -#define MY_AES_OPENSSL_ERROR -101 -#define MY_AES_BAD_KEYSIZE -102 - -/* The block size for all supported algorithms */ -#define MY_AES_BLOCK_SIZE 16 - -/* The max key length of all supported algorithms */ -#define MY_AES_MAX_KEY_LENGTH 32 - -#define MY_AES_CTX_SIZE 512 - -enum my_aes_mode { - MY_AES_ECB, MY_AES_CBC -#ifdef HAVE_EncryptAes128Ctr - , MY_AES_CTR -#endif -#ifdef HAVE_EncryptAes128Gcm - , MY_AES_GCM -#endif -}; - -int my_aes_crypt_init(void *ctx, enum my_aes_mode mode, int flags, - const unsigned char* key, unsigned int klen, - const unsigned char* iv, unsigned int ivlen); -int my_aes_crypt_update(void *ctx, const uchar *src, uint slen, - uchar *dst, uint *dlen); -int my_aes_crypt_finish(void *ctx, uchar *dst, uint *dlen); -int my_aes_crypt(enum my_aes_mode mode, int flags, - const uchar *src, uint slen, uchar *dst, uint *dlen, - const uchar *key, uint klen, const uchar *iv, uint ivlen); - -/* - calculate the length of the cyphertext from the length of the plaintext - for different AES encryption modes with padding enabled. - Without padding (ENCRYPTION_FLAG_NOPAD) cyphertext has the same length - as the plaintext -*/ -static inline uint my_aes_get_size(enum my_aes_mode mode __attribute__((unused)), uint source_length) -{ -#ifdef HAVE_EncryptAes128Ctr - if (mode == MY_AES_CTR) - return source_length; -#ifdef HAVE_EncryptAes128Gcm - if (mode == MY_AES_GCM) - return source_length + MY_AES_BLOCK_SIZE; -#endif -#endif - return (source_length / MY_AES_BLOCK_SIZE + 1) * MY_AES_BLOCK_SIZE; -} - -static inline uint my_aes_ctx_size(enum my_aes_mode mode __attribute__((unused))) -{ - return MY_AES_CTX_SIZE; -} - -int my_random_bytes(uchar* buf, int num); - -#ifdef __cplusplus -} -#endif +#include <my_config.h> /* HAVE_EncryptAes128{Ctr,Gcm} */ +#include <mysql/service_my_crypt.h> #endif /* MY_CRYPT_INCLUDED */ diff --git a/include/my_pthread.h b/include/my_pthread.h index a69e0c49f55..672bca5ab39 100644 --- a/include/my_pthread.h +++ b/include/my_pthread.h @@ -346,6 +346,26 @@ int my_pthread_mutex_trylock(pthread_mutex_t *mutex); } while(0) #endif /* !set_timespec_time_nsec */ +#ifdef MYSQL_CLIENT +#define _current_thd() NULL +#elif defined(_WIN32) +#ifdef __cplusplus +extern "C" +#endif +MYSQL_THD _current_thd_noinline(); +#define _current_thd() _current_thd_noinline() +#else +/* + THR_THD is a key which will be used to set/get THD* for a thread, + using my_pthread_setspecific_ptr()/my_thread_getspecific_ptr(). +*/ +extern pthread_key(MYSQL_THD, THR_THD); +static inline MYSQL_THD _current_thd(void) +{ + return my_pthread_getspecific_ptr(MYSQL_THD,THR_THD); +} +#endif + /* safe_mutex adds checking to mutex for easier debugging */ struct st_hash; typedef struct st_safe_mutex_t diff --git a/include/my_sys.h b/include/my_sys.h index 9cc069d1fd3..dfabda42022 100644 --- a/include/my_sys.h +++ b/include/my_sys.h @@ -42,6 +42,7 @@ typedef struct my_aio_result { #include <malloc.h> /*for alloca*/ #endif #include <mysql/plugin.h> +#include <mysql/service_my_print_error.h> #define MY_INIT(name) { my_progname= name; my_init(); } @@ -104,18 +105,10 @@ typedef struct my_aio_result { #define MY_GIVE_INFO 2U /* Give time info about process*/ #define MY_DONT_FREE_DBUG 4U /* Do not call DBUG_END() in my_end() */ -#define ME_HIGHBYTE 8U /* Shift for colours */ -#define ME_NOCUR 1U /* Don't use curses message */ -#define ME_OLDWIN 2U /* Use old window */ -#define ME_BELL 4U /* Ring bell then printing message */ -#define ME_HOLDTANG 8U /* Don't delete last keys */ -#define ME_WAITTOT 16U /* Wait for errtime secs of for a action */ -#define ME_WAITTANG 32U /* Wait for a user action */ -#define ME_NOREFRESH 64U /* Write the error message to error log */ -#define ME_NOINPUT 128U /* Dont use the input libary */ -#define ME_COLOUR1 ((1U << ME_HIGHBYTE)) /* Possibly error-colours */ -#define ME_COLOUR2 ((2U << ME_HIGHBYTE)) -#define ME_COLOUR3 ((3U << ME_HIGHBYTE)) +#define ME_BELL 4U /* Ring bell then printing message */ +#define ME_WAITTANG 0 /* Wait for a user action */ +#define ME_NOREFRESH 64U /* Write the error message to error log */ +#define ME_NOINPUT 0 /* Dont use the input libary */ #define ME_JUST_INFO 1024U /**< not error but just info */ #define ME_JUST_WARNING 2048U /**< not error but just warning */ #define ME_FATALERROR 4096U /* Fatal statement error */ @@ -725,12 +718,6 @@ extern int my_sync(File fd, myf my_flags); extern int my_sync_dir(const char *dir_name, myf my_flags); extern int my_sync_dir_by_file(const char *file_name, myf my_flags); extern const char *my_get_err_msg(uint nr); -extern void my_error(uint nr,myf MyFlags, ...); -extern void my_printf_error(uint my_err, const char *format, - myf MyFlags, ...) - ATTRIBUTE_FORMAT(printf, 2, 4); -extern void my_printv_error(uint error, const char *format, myf MyFlags, - va_list ap); extern int my_error_register(const char** (*get_errmsgs) (int nr), uint first, uint last); extern my_bool my_error_unregister(uint first, uint last); diff --git a/include/mysql/plugin.h b/include/mysql/plugin.h index 2f077d8440e..ad5a792173a 100644 --- a/include/mysql/plugin.h +++ b/include/mysql/plugin.h @@ -75,7 +75,7 @@ typedef struct st_mysql_xid MYSQL_XID; #define MYSQL_PLUGIN_INTERFACE_VERSION 0x0104 /* MariaDB plugin interface version */ -#define MARIA_PLUGIN_INTERFACE_VERSION 0x010c +#define MARIA_PLUGIN_INTERFACE_VERSION 0x010d /* The allowable types of plugins diff --git a/include/mysql/plugin_audit.h.pp b/include/mysql/plugin_audit.h.pp index 2adde48dc22..03f6f0caba0 100644 --- a/include/mysql/plugin_audit.h.pp +++ b/include/mysql/plugin_audit.h.pp @@ -137,6 +137,43 @@ size_t my_md5_context_size(); void my_md5_init(void *context); void my_md5_input(void *context, const unsigned char *buf, size_t len); void my_md5_result(void *context, unsigned char *digest); +enum my_aes_mode { + MY_AES_ECB, MY_AES_CBC +}; +extern struct my_crypt_service_st { + int (*my_aes_crypt_init)(void *ctx, enum my_aes_mode mode, int flags, + const unsigned char* key, unsigned int klen, + const unsigned char* iv, unsigned int ivlen); + int (*my_aes_crypt_update)(void *ctx, const unsigned char *src, unsigned int slen, + unsigned char *dst, unsigned int *dlen); + int (*my_aes_crypt_finish)(void *ctx, unsigned char *dst, unsigned int *dlen); + int (*my_aes_crypt)(enum my_aes_mode mode, int flags, + const unsigned char *src, unsigned int slen, unsigned char *dst, unsigned int *dlen, + const unsigned char *key, unsigned int klen, const unsigned char *iv, unsigned int ivlen); + unsigned int (*my_aes_get_size)(enum my_aes_mode mode, unsigned int source_length); + unsigned int (*my_aes_ctx_size)(enum my_aes_mode mode); + int (*my_random_bytes)(unsigned char* buf, int num); +} *my_crypt_service; +int my_aes_crypt_init(void *ctx, enum my_aes_mode mode, int flags, + const unsigned char* key, unsigned int klen, + const unsigned char* iv, unsigned int ivlen); +int my_aes_crypt_update(void *ctx, const unsigned char *src, unsigned int slen, + unsigned char *dst, unsigned int *dlen); +int my_aes_crypt_finish(void *ctx, unsigned char *dst, unsigned int *dlen); +int my_aes_crypt(enum my_aes_mode mode, int flags, + const unsigned char *src, unsigned int slen, unsigned char *dst, unsigned int *dlen, + const unsigned char *key, unsigned int klen, const unsigned char *iv, unsigned int ivlen); +int my_random_bytes(unsigned char* buf, int num); +unsigned int my_aes_get_size(enum my_aes_mode mode, unsigned int source_length); +unsigned int my_aes_ctx_size(enum my_aes_mode mode); +extern struct my_print_error_service_st { + void(*my_error_func)(unsigned int nr, unsigned long MyFlags, ...); + void(*my_printf_error_func)(unsigned int nr, const char *fmt, unsigned long MyFlags,...); + void(*my_printv_error_func)(unsigned int error, const char *format, unsigned long MyFlags, va_list ap); +} *my_print_error_service; +extern void my_error(unsigned int nr, unsigned long MyFlags, ...); +extern void my_printf_error(unsigned int my_err, const char *format, unsigned long MyFlags, ...); +extern void my_printv_error(unsigned int error, const char *format, unsigned long MyFlags,va_list ap); extern struct my_snprintf_service_st { size_t (*my_snprintf_type)(char*, size_t, const char*, ...); size_t (*my_vsnprintf_type)(char *, size_t, const char*, va_list); diff --git a/include/mysql/plugin_auth.h.pp b/include/mysql/plugin_auth.h.pp index a9cb8044c4f..07cdb1b5f52 100644 --- a/include/mysql/plugin_auth.h.pp +++ b/include/mysql/plugin_auth.h.pp @@ -137,6 +137,43 @@ size_t my_md5_context_size(); void my_md5_init(void *context); void my_md5_input(void *context, const unsigned char *buf, size_t len); void my_md5_result(void *context, unsigned char *digest); +enum my_aes_mode { + MY_AES_ECB, MY_AES_CBC +}; +extern struct my_crypt_service_st { + int (*my_aes_crypt_init)(void *ctx, enum my_aes_mode mode, int flags, + const unsigned char* key, unsigned int klen, + const unsigned char* iv, unsigned int ivlen); + int (*my_aes_crypt_update)(void *ctx, const unsigned char *src, unsigned int slen, + unsigned char *dst, unsigned int *dlen); + int (*my_aes_crypt_finish)(void *ctx, unsigned char *dst, unsigned int *dlen); + int (*my_aes_crypt)(enum my_aes_mode mode, int flags, + const unsigned char *src, unsigned int slen, unsigned char *dst, unsigned int *dlen, + const unsigned char *key, unsigned int klen, const unsigned char *iv, unsigned int ivlen); + unsigned int (*my_aes_get_size)(enum my_aes_mode mode, unsigned int source_length); + unsigned int (*my_aes_ctx_size)(enum my_aes_mode mode); + int (*my_random_bytes)(unsigned char* buf, int num); +} *my_crypt_service; +int my_aes_crypt_init(void *ctx, enum my_aes_mode mode, int flags, + const unsigned char* key, unsigned int klen, + const unsigned char* iv, unsigned int ivlen); +int my_aes_crypt_update(void *ctx, const unsigned char *src, unsigned int slen, + unsigned char *dst, unsigned int *dlen); +int my_aes_crypt_finish(void *ctx, unsigned char *dst, unsigned int *dlen); +int my_aes_crypt(enum my_aes_mode mode, int flags, + const unsigned char *src, unsigned int slen, unsigned char *dst, unsigned int *dlen, + const unsigned char *key, unsigned int klen, const unsigned char *iv, unsigned int ivlen); +int my_random_bytes(unsigned char* buf, int num); +unsigned int my_aes_get_size(enum my_aes_mode mode, unsigned int source_length); +unsigned int my_aes_ctx_size(enum my_aes_mode mode); +extern struct my_print_error_service_st { + void(*my_error_func)(unsigned int nr, unsigned long MyFlags, ...); + void(*my_printf_error_func)(unsigned int nr, const char *fmt, unsigned long MyFlags,...); + void(*my_printv_error_func)(unsigned int error, const char *format, unsigned long MyFlags, va_list ap); +} *my_print_error_service; +extern void my_error(unsigned int nr, unsigned long MyFlags, ...); +extern void my_printf_error(unsigned int my_err, const char *format, unsigned long MyFlags, ...); +extern void my_printv_error(unsigned int error, const char *format, unsigned long MyFlags,va_list ap); extern struct my_snprintf_service_st { size_t (*my_snprintf_type)(char*, size_t, const char*, ...); size_t (*my_vsnprintf_type)(char *, size_t, const char*, va_list); diff --git a/include/mysql/plugin_encryption.h.pp b/include/mysql/plugin_encryption.h.pp index 4675f0cf6ec..7275f0a982e 100644 --- a/include/mysql/plugin_encryption.h.pp +++ b/include/mysql/plugin_encryption.h.pp @@ -137,6 +137,43 @@ size_t my_md5_context_size(); void my_md5_init(void *context); void my_md5_input(void *context, const unsigned char *buf, size_t len); void my_md5_result(void *context, unsigned char *digest); +enum my_aes_mode { + MY_AES_ECB, MY_AES_CBC +}; +extern struct my_crypt_service_st { + int (*my_aes_crypt_init)(void *ctx, enum my_aes_mode mode, int flags, + const unsigned char* key, unsigned int klen, + const unsigned char* iv, unsigned int ivlen); + int (*my_aes_crypt_update)(void *ctx, const unsigned char *src, unsigned int slen, + unsigned char *dst, unsigned int *dlen); + int (*my_aes_crypt_finish)(void *ctx, unsigned char *dst, unsigned int *dlen); + int (*my_aes_crypt)(enum my_aes_mode mode, int flags, + const unsigned char *src, unsigned int slen, unsigned char *dst, unsigned int *dlen, + const unsigned char *key, unsigned int klen, const unsigned char *iv, unsigned int ivlen); + unsigned int (*my_aes_get_size)(enum my_aes_mode mode, unsigned int source_length); + unsigned int (*my_aes_ctx_size)(enum my_aes_mode mode); + int (*my_random_bytes)(unsigned char* buf, int num); +} *my_crypt_service; +int my_aes_crypt_init(void *ctx, enum my_aes_mode mode, int flags, + const unsigned char* key, unsigned int klen, + const unsigned char* iv, unsigned int ivlen); +int my_aes_crypt_update(void *ctx, const unsigned char *src, unsigned int slen, + unsigned char *dst, unsigned int *dlen); +int my_aes_crypt_finish(void *ctx, unsigned char *dst, unsigned int *dlen); +int my_aes_crypt(enum my_aes_mode mode, int flags, + const unsigned char *src, unsigned int slen, unsigned char *dst, unsigned int *dlen, + const unsigned char *key, unsigned int klen, const unsigned char *iv, unsigned int ivlen); +int my_random_bytes(unsigned char* buf, int num); +unsigned int my_aes_get_size(enum my_aes_mode mode, unsigned int source_length); +unsigned int my_aes_ctx_size(enum my_aes_mode mode); +extern struct my_print_error_service_st { + void(*my_error_func)(unsigned int nr, unsigned long MyFlags, ...); + void(*my_printf_error_func)(unsigned int nr, const char *fmt, unsigned long MyFlags,...); + void(*my_printv_error_func)(unsigned int error, const char *format, unsigned long MyFlags, va_list ap); +} *my_print_error_service; +extern void my_error(unsigned int nr, unsigned long MyFlags, ...); +extern void my_printf_error(unsigned int my_err, const char *format, unsigned long MyFlags, ...); +extern void my_printv_error(unsigned int error, const char *format, unsigned long MyFlags,va_list ap); extern struct my_snprintf_service_st { size_t (*my_snprintf_type)(char*, size_t, const char*, ...); size_t (*my_vsnprintf_type)(char *, size_t, const char*, va_list); diff --git a/include/mysql/plugin_ftparser.h.pp b/include/mysql/plugin_ftparser.h.pp index 34d968b60ab..eee69281afe 100644 --- a/include/mysql/plugin_ftparser.h.pp +++ b/include/mysql/plugin_ftparser.h.pp @@ -137,6 +137,43 @@ size_t my_md5_context_size(); void my_md5_init(void *context); void my_md5_input(void *context, const unsigned char *buf, size_t len); void my_md5_result(void *context, unsigned char *digest); +enum my_aes_mode { + MY_AES_ECB, MY_AES_CBC +}; +extern struct my_crypt_service_st { + int (*my_aes_crypt_init)(void *ctx, enum my_aes_mode mode, int flags, + const unsigned char* key, unsigned int klen, + const unsigned char* iv, unsigned int ivlen); + int (*my_aes_crypt_update)(void *ctx, const unsigned char *src, unsigned int slen, + unsigned char *dst, unsigned int *dlen); + int (*my_aes_crypt_finish)(void *ctx, unsigned char *dst, unsigned int *dlen); + int (*my_aes_crypt)(enum my_aes_mode mode, int flags, + const unsigned char *src, unsigned int slen, unsigned char *dst, unsigned int *dlen, + const unsigned char *key, unsigned int klen, const unsigned char *iv, unsigned int ivlen); + unsigned int (*my_aes_get_size)(enum my_aes_mode mode, unsigned int source_length); + unsigned int (*my_aes_ctx_size)(enum my_aes_mode mode); + int (*my_random_bytes)(unsigned char* buf, int num); +} *my_crypt_service; +int my_aes_crypt_init(void *ctx, enum my_aes_mode mode, int flags, + const unsigned char* key, unsigned int klen, + const unsigned char* iv, unsigned int ivlen); +int my_aes_crypt_update(void *ctx, const unsigned char *src, unsigned int slen, + unsigned char *dst, unsigned int *dlen); +int my_aes_crypt_finish(void *ctx, unsigned char *dst, unsigned int *dlen); +int my_aes_crypt(enum my_aes_mode mode, int flags, + const unsigned char *src, unsigned int slen, unsigned char *dst, unsigned int *dlen, + const unsigned char *key, unsigned int klen, const unsigned char *iv, unsigned int ivlen); +int my_random_bytes(unsigned char* buf, int num); +unsigned int my_aes_get_size(enum my_aes_mode mode, unsigned int source_length); +unsigned int my_aes_ctx_size(enum my_aes_mode mode); +extern struct my_print_error_service_st { + void(*my_error_func)(unsigned int nr, unsigned long MyFlags, ...); + void(*my_printf_error_func)(unsigned int nr, const char *fmt, unsigned long MyFlags,...); + void(*my_printv_error_func)(unsigned int error, const char *format, unsigned long MyFlags, va_list ap); +} *my_print_error_service; +extern void my_error(unsigned int nr, unsigned long MyFlags, ...); +extern void my_printf_error(unsigned int my_err, const char *format, unsigned long MyFlags, ...); +extern void my_printv_error(unsigned int error, const char *format, unsigned long MyFlags,va_list ap); extern struct my_snprintf_service_st { size_t (*my_snprintf_type)(char*, size_t, const char*, ...); size_t (*my_vsnprintf_type)(char *, size_t, const char*, va_list); diff --git a/include/mysql/plugin_password_validation.h.pp b/include/mysql/plugin_password_validation.h.pp index 5a642a55d08..adb0afc734d 100644 --- a/include/mysql/plugin_password_validation.h.pp +++ b/include/mysql/plugin_password_validation.h.pp @@ -137,6 +137,43 @@ size_t my_md5_context_size(); void my_md5_init(void *context); void my_md5_input(void *context, const unsigned char *buf, size_t len); void my_md5_result(void *context, unsigned char *digest); +enum my_aes_mode { + MY_AES_ECB, MY_AES_CBC +}; +extern struct my_crypt_service_st { + int (*my_aes_crypt_init)(void *ctx, enum my_aes_mode mode, int flags, + const unsigned char* key, unsigned int klen, + const unsigned char* iv, unsigned int ivlen); + int (*my_aes_crypt_update)(void *ctx, const unsigned char *src, unsigned int slen, + unsigned char *dst, unsigned int *dlen); + int (*my_aes_crypt_finish)(void *ctx, unsigned char *dst, unsigned int *dlen); + int (*my_aes_crypt)(enum my_aes_mode mode, int flags, + const unsigned char *src, unsigned int slen, unsigned char *dst, unsigned int *dlen, + const unsigned char *key, unsigned int klen, const unsigned char *iv, unsigned int ivlen); + unsigned int (*my_aes_get_size)(enum my_aes_mode mode, unsigned int source_length); + unsigned int (*my_aes_ctx_size)(enum my_aes_mode mode); + int (*my_random_bytes)(unsigned char* buf, int num); +} *my_crypt_service; +int my_aes_crypt_init(void *ctx, enum my_aes_mode mode, int flags, + const unsigned char* key, unsigned int klen, + const unsigned char* iv, unsigned int ivlen); +int my_aes_crypt_update(void *ctx, const unsigned char *src, unsigned int slen, + unsigned char *dst, unsigned int *dlen); +int my_aes_crypt_finish(void *ctx, unsigned char *dst, unsigned int *dlen); +int my_aes_crypt(enum my_aes_mode mode, int flags, + const unsigned char *src, unsigned int slen, unsigned char *dst, unsigned int *dlen, + const unsigned char *key, unsigned int klen, const unsigned char *iv, unsigned int ivlen); +int my_random_bytes(unsigned char* buf, int num); +unsigned int my_aes_get_size(enum my_aes_mode mode, unsigned int source_length); +unsigned int my_aes_ctx_size(enum my_aes_mode mode); +extern struct my_print_error_service_st { + void(*my_error_func)(unsigned int nr, unsigned long MyFlags, ...); + void(*my_printf_error_func)(unsigned int nr, const char *fmt, unsigned long MyFlags,...); + void(*my_printv_error_func)(unsigned int error, const char *format, unsigned long MyFlags, va_list ap); +} *my_print_error_service; +extern void my_error(unsigned int nr, unsigned long MyFlags, ...); +extern void my_printf_error(unsigned int my_err, const char *format, unsigned long MyFlags, ...); +extern void my_printv_error(unsigned int error, const char *format, unsigned long MyFlags,va_list ap); extern struct my_snprintf_service_st { size_t (*my_snprintf_type)(char*, size_t, const char*, ...); size_t (*my_vsnprintf_type)(char *, size_t, const char*, va_list); diff --git a/include/mysql/service_my_crypt.h b/include/mysql/service_my_crypt.h new file mode 100644 index 00000000000..83de0378e4a --- /dev/null +++ b/include/mysql/service_my_crypt.h @@ -0,0 +1,120 @@ +#ifndef MYSQL_SERVICE_MY_CRYPT_INCLUDED +#define MYSQL_SERVICE_MY_CRYPT_INCLUDED + +/* + Copyright (c) 2014 Google Inc. + Copyright (c) 2014, 2015 MariaDB Corporation + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +/** + @file + my crypt service + + AES encryption functions, and a function to generate random bytes. + + Include my_config.h before this file to use CTR and GCM modes + (they only work if server was compiled with openssl). +*/ + + +#ifdef __cplusplus +extern "C" { +#endif + +/* return values from my_aes_encrypt/my_aes_decrypt functions */ +#define MY_AES_OK 0 +#define MY_AES_BAD_DATA -100 +#define MY_AES_OPENSSL_ERROR -101 +#define MY_AES_BAD_KEYSIZE -102 + +/* The block size for all supported algorithms */ +#define MY_AES_BLOCK_SIZE 16 + +/* The max key length of all supported algorithms */ +#define MY_AES_MAX_KEY_LENGTH 32 + +#define MY_AES_CTX_SIZE 512 + +enum my_aes_mode { + MY_AES_ECB, MY_AES_CBC +#ifdef HAVE_EncryptAes128Ctr + , MY_AES_CTR +#endif +#ifdef HAVE_EncryptAes128Gcm + , MY_AES_GCM +#endif +}; + +extern struct my_crypt_service_st { + int (*my_aes_crypt_init)(void *ctx, enum my_aes_mode mode, int flags, + const unsigned char* key, unsigned int klen, + const unsigned char* iv, unsigned int ivlen); + int (*my_aes_crypt_update)(void *ctx, const unsigned char *src, unsigned int slen, + unsigned char *dst, unsigned int *dlen); + int (*my_aes_crypt_finish)(void *ctx, unsigned char *dst, unsigned int *dlen); + int (*my_aes_crypt)(enum my_aes_mode mode, int flags, + const unsigned char *src, unsigned int slen, unsigned char *dst, unsigned int *dlen, + const unsigned char *key, unsigned int klen, const unsigned char *iv, unsigned int ivlen); + unsigned int (*my_aes_get_size)(enum my_aes_mode mode, unsigned int source_length); + unsigned int (*my_aes_ctx_size)(enum my_aes_mode mode); + int (*my_random_bytes)(unsigned char* buf, int num); +} *my_crypt_service; + +#ifdef MYSQL_DYNAMIC_PLUGIN + +#define my_aes_crypt_init(A,B,C,D,E,F,G) \ + my_crypt_service->my_aes_crypt_init(A,B,C,D,E,F,G) + +#define my_aes_crypt_update(A,B,C,D,E) \ + my_crypt_service->my_aes_crypt_update(A,B,C,D,E) + +#define my_aes_crypt_finish(A,B,C) \ + my_crypt_service->my_aes_crypt_finish(A,B,C) + +#define my_aes_crypt(A,B,C,D,E,F,G,H,I,J) \ + my_crypt_service->my_aes_crypt(A,B,C,D,E,F,G,H,I,J) + +#define my_aes_get_size(A,B)\ + my_crypt_service->my_aes_get_size(A,B) + +#define my_aes_ctx_size(A)\ + my_crypt_service->my_aes_ctx_size(A) + +#define my_random_bytes(A,B)\ + my_crypt_service->my_random_bytes(A,B) + +#else + +int my_aes_crypt_init(void *ctx, enum my_aes_mode mode, int flags, + const unsigned char* key, unsigned int klen, + const unsigned char* iv, unsigned int ivlen); +int my_aes_crypt_update(void *ctx, const unsigned char *src, unsigned int slen, + unsigned char *dst, unsigned int *dlen); +int my_aes_crypt_finish(void *ctx, unsigned char *dst, unsigned int *dlen); +int my_aes_crypt(enum my_aes_mode mode, int flags, + const unsigned char *src, unsigned int slen, unsigned char *dst, unsigned int *dlen, + const unsigned char *key, unsigned int klen, const unsigned char *iv, unsigned int ivlen); + +int my_random_bytes(unsigned char* buf, int num); +unsigned int my_aes_get_size(enum my_aes_mode mode, unsigned int source_length); +unsigned int my_aes_ctx_size(enum my_aes_mode mode); +#endif + + +#ifdef __cplusplus +} +#endif + +#endif /* MYSQL_SERVICE_MY_CRYPT_INCLUDED */ diff --git a/include/mysql/service_my_print_error.h b/include/mysql/service_my_print_error.h new file mode 100644 index 00000000000..636151655e5 --- /dev/null +++ b/include/mysql/service_my_print_error.h @@ -0,0 +1,64 @@ +/* Copyright (c) 2016, MariaDB + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#ifndef MYSQL_SERVICE_MY_PRINT_ERROR_INCLUDED +#define MYSQL_SERVICE_MY_PRINT_ERROR_INCLUDED + +/** + @file include/mysql/service_my_print_error.h + + This service provides functions for plugins to report + errors to client (without client, the errors are written to the error log). + +*/ +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef MYSQL_ABI_CHECK +#include <stdarg.h> +#include <stdlib.h> +#endif + +#define ME_ERROR_LOG 64 /* Write the message to the error log */ +#define ME_NOTE 1024 /* Not an error, just a note */ +#define ME_WARNING 2048 /* Not an error, just a warning */ +#define ME_FATAL 4096 /* Fatal statement error */ + +extern struct my_print_error_service_st { + void (*my_error_func)(unsigned int nr, unsigned long MyFlags, ...); + void (*my_printf_error_func)(unsigned int nr, const char *fmt, unsigned long MyFlags,...); + void (*my_printv_error_func)(unsigned int error, const char *format, unsigned long MyFlags, va_list ap); +} *my_print_error_service; + +#ifdef MYSQL_DYNAMIC_PLUGIN + +#define my_error my_print_error_service->my_error_func +#define my_printf_error my_print_error_service->my_printf_error_func +#define my_printv_error(A,B,C,D) my_print_error_service->my_printv_error_func(A,B,C,D) + +#else + +extern void my_error(unsigned int nr, unsigned long MyFlags, ...); +extern void my_printf_error(unsigned int my_err, const char *format, unsigned long MyFlags, ...); +extern void my_printv_error(unsigned int error, const char *format, unsigned long MyFlags,va_list ap); +#endif + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/include/mysql/services.h b/include/mysql/services.h index 420f2430a36..6168c5ed8dc 100644 --- a/include/mysql/services.h +++ b/include/mysql/services.h @@ -26,6 +26,8 @@ extern "C" { #include <mysql/service_kill_statement.h> #include <mysql/service_logger.h> #include <mysql/service_md5.h> +#include <mysql/service_my_crypt.h> +#include <mysql/service_my_print_error.h> #include <mysql/service_my_snprintf.h> #include <mysql/service_progress_report.h> #include <mysql/service_sha1.h> diff --git a/include/service_versions.h b/include/service_versions.h index d79474f1d36..ddc780a44b9 100644 --- a/include/service_versions.h +++ b/include/service_versions.h @@ -27,7 +27,9 @@ #define VERSION_encryption 0x0300 #define VERSION_encryption_scheme 0x0100 #define VERSION_logger 0x0100 +#define VERSION_my_crypt 0x0100 #define VERSION_my_md5 0x0100 +#define VERSION_my_print_error 0x0100 #define VERSION_my_sha1 0x0101 #define VERSION_my_sha2 0x0100 #define VERSION_my_snprintf 0x0100 diff --git a/libservices/CMakeLists.txt b/libservices/CMakeLists.txt index 0b68a156077..e20be6d7a7c 100644 --- a/libservices/CMakeLists.txt +++ b/libservices/CMakeLists.txt @@ -22,7 +22,9 @@ SET(MYSQLSERVICES_SOURCES encryption_service.c kill_statement_service.c logger_service.c + my_crypt_service.c my_md5_service.c + my_print_error_service.c my_sha1_service.c my_sha2_service.c my_snprintf_service.c @@ -35,7 +37,7 @@ SET(MYSQLSERVICES_SOURCES thd_timezone_service.c thd_wait_service.c wsrep_service.c -) + ) ADD_CONVENIENCE_LIBRARY(mysqlservices ${MYSQLSERVICES_SOURCES}) INSTALL(TARGETS mysqlservices DESTINATION ${INSTALL_LIBDIR} COMPONENT Development) diff --git a/libservices/my_crypt_service.c b/libservices/my_crypt_service.c new file mode 100644 index 00000000000..e6b9e273094 --- /dev/null +++ b/libservices/my_crypt_service.c @@ -0,0 +1,2 @@ +#include <service_versions.h> +SERVICE_VERSION my_crypt_service= (void*)VERSION_my_crypt; diff --git a/libservices/my_print_error_service.c b/libservices/my_print_error_service.c new file mode 100644 index 00000000000..7642668d470 --- /dev/null +++ b/libservices/my_print_error_service.c @@ -0,0 +1,17 @@ +/* Copyright (c) 2016 MariaDB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#include <service_versions.h> +SERVICE_VERSION my_print_error_service= (void*)VERSION_my_print_error;
\ No newline at end of file diff --git a/mysql-test/include/default_client.cnf b/mysql-test/include/default_client.cnf index bcd6a4849d2..e326b308f5f 100644 --- a/mysql-test/include/default_client.cnf +++ b/mysql-test/include/default_client.cnf @@ -20,3 +20,9 @@ default-character-set=latin1 [mysql_upgrade] default-character-set=latin1 + +[mysqltest] +loose-ssl-ca=@ENV.MYSQL_TEST_DIR/std_data/cacert.pem +loose-ssl-cert=@ENV.MYSQL_TEST_DIR/std_data/client-cert.pem +loose-ssl-key=@ENV.MYSQL_TEST_DIR/std_data/client-key.pem +loose-skip-ssl diff --git a/mysql-test/include/default_mysqld.cnf b/mysql-test/include/default_mysqld.cnf index b5b16461781..44a7fd12d27 100644 --- a/mysql-test/include/default_mysqld.cnf +++ b/mysql-test/include/default_mysqld.cnf @@ -108,6 +108,10 @@ binlog-direct-non-transactional-updates default-storage-engine=myisam +loose-ssl-ca=@ENV.MYSQL_TEST_DIR/std_data/cacert.pem +loose-ssl-cert=@ENV.MYSQL_TEST_DIR/std_data/server-cert.pem +loose-ssl-key=@ENV.MYSQL_TEST_DIR/std_data/server-key.pem + # here, at the end of [mysqld] group mtr will automatically disable # all optional plugins. diff --git a/mysql-test/include/write_var_to_file.inc b/mysql-test/include/write_var_to_file.inc index 08de195ccbb..7982c6fab31 100644 --- a/mysql-test/include/write_var_to_file.inc +++ b/mysql-test/include/write_var_to_file.inc @@ -43,9 +43,8 @@ if ($write_to_file == 'GENERATE') if (`SELECT LENGTH(@@secure_file_priv) > 0`) { - --let $_wvtf_secure_file_priv= `SELECT @@secure_file_priv` --let $_wvtf_suffix= `SELECT UUID()` - --let $_wvtf_tmp_file= $_wvtf_secure_file_priv/_wvtf_$_wvtf_suffix + --let $_wvtf_tmp_file= $MYSQLTEST_VARDIR/_wvtf_$_wvtf_suffix --eval SELECT '$write_var' INTO DUMPFILE '$_wvtf_tmp_file' --copy_file $_wvtf_tmp_file $write_to_file diff --git a/mysql-test/lib/My/ConfigFactory.pm b/mysql-test/lib/My/ConfigFactory.pm index 12c0095e80e..830b49d431f 100644 --- a/mysql-test/lib/My/ConfigFactory.pm +++ b/mysql-test/lib/My/ConfigFactory.pm @@ -182,55 +182,6 @@ sub fix_log_slow_queries { return "$dir/mysqld-slow.log"; } -sub fix_std_data { - my ($self, $config, $group_name, $group)= @_; - my $testdir= $self->get_testdir($group); - return "$testdir/std_data"; -} - -sub ssl_supported { - my ($self)= @_; - return $self->{ARGS}->{ssl}; -} - -sub fix_skip_ssl { - return if !ssl_supported(@_); - # Add skip-ssl if ssl is supported to avoid - # that mysqltest connects with SSL by default - return 1; -} - -sub fix_ssl_ca { - return if !ssl_supported(@_); - my $std_data= fix_std_data(@_); - return "$std_data/cacert.pem" -} - -sub fix_ssl_server_cert { - return if !ssl_supported(@_); - my $std_data= fix_std_data(@_); - return "$std_data/server-cert.pem" -} - -sub fix_ssl_client_cert { - return if !ssl_supported(@_); - my $std_data= fix_std_data(@_); - return "$std_data/client-cert.pem" -} - -sub fix_ssl_server_key { - return if !ssl_supported(@_); - my $std_data= fix_std_data(@_); - return "$std_data/server-key.pem" -} - -sub fix_ssl_client_key { - return if !ssl_supported(@_); - my $std_data= fix_std_data(@_); - return "$std_data/client-key.pem" -} - - # # Rules to run for each mysqld in the config # - will be run in order listed here @@ -255,9 +206,6 @@ my @mysqld_rules= { '#user' => sub { return shift->{ARGS}->{user} || ""; } }, { '#password' => sub { return shift->{ARGS}->{password} || ""; } }, { 'server-id' => \&fix_server_id, }, - { 'ssl-ca' => \&fix_ssl_ca }, - { 'ssl-cert' => \&fix_ssl_server_cert }, - { 'ssl-key' => \&fix_ssl_server_key }, { 'bind-address' => \&fix_bind_address }, ); @@ -284,10 +232,6 @@ my @client_rules= # my @mysqltest_rules= ( - { 'ssl-ca' => \&fix_ssl_ca }, - { 'ssl-cert' => \&fix_ssl_client_cert }, - { 'ssl-key' => \&fix_ssl_client_key }, - { 'skip-ssl' => \&fix_skip_ssl }, ); diff --git a/mysql-test/lib/generate-ssl-certs.sh b/mysql-test/lib/generate-ssl-certs.sh index cc919dfe32e..8f15ba9d521 100755 --- a/mysql-test/lib/generate-ssl-certs.sh +++ b/mysql-test/lib/generate-ssl-certs.sh @@ -1,30 +1,39 @@ -#!/bin/sh -xe +#!/bin/sh + +set -xe # simply run me from mysql-test/ cd std_data/ # boilerplace for "openssl ca" and /etc/ssl/openssl.cnf rm -rf demoCA -mkdir demoCA demoCA/private demoCA/newcerts +mkdir demoCA demoCA/newcerts touch demoCA/index.txt echo 01 > demoCA/serial # CA certificate, self-signed -openssl req -x509 -newkey rsa:2048 -keyout demoCA/private/cakey.pem -out cacert.pem -days 7300 -nodes -subj '/CN=cacert/C=FI/ST=Helsinki/L=Helsinki/O=MariaDB' -text +openssl req -x509 -newkey rsa:2048 -keyout cakey.pem -out cacert.pem -days 7300 -nodes -subj '/CN=cacert/C=FI/ST=Helsinki/L=Helsinki/O=MariaDB' -text # server certificate signing request and private key. Note the very long subject (for MDEV-7859) openssl req -newkey rsa:1024 -keyout server-key.pem -out demoCA/server-req.pem -days 7300 -nodes -subj '/CN=localhost/C=FI/ST=state or province within country, in other certificates in this file it is the same as L/L=location, usually an address but often ambiguously used/OU=organizational unit name, a division name within an organization/O=organization name, typically a company name' # convert the key to yassl compatible format openssl rsa -in server-key.pem -out server-key.pem # sign the server certificate with CA certificate -openssl ca -days 7300 -batch -cert cacert.pem -policy policy_anything -out server-cert.pem -infiles demoCA/server-req.pem +openssl ca -keyfile cakey.pem -days 7300 -batch -cert cacert.pem -policy policy_anything -out server-cert.pem -infiles demoCA/server-req.pem openssl req -newkey rsa:8192 -keyout server8k-key.pem -out demoCA/server8k-req.pem -days 7300 -nodes -subj '/CN=server8k/C=FI/ST=Helsinki/L=Helsinki/O=MariaDB' openssl rsa -in server8k-key.pem -out server8k-key.pem -openssl ca -days 7300 -batch -cert cacert.pem -policy policy_anything -out server8k-cert.pem -infiles demoCA/server8k-req.pem +openssl ca -keyfile cakey.pem -days 7300 -batch -cert cacert.pem -policy policy_anything -out server8k-cert.pem -infiles demoCA/server8k-req.pem openssl req -newkey rsa:1024 -keyout client-key.pem -out demoCA/client-req.pem -days 7300 -nodes -subj '/CN=client/C=FI/ST=Helsinki/L=Helsinki/O=MariaDB' openssl rsa -in client-key.pem -out client-key.pem -openssl ca -days 7300 -batch -cert cacert.pem -policy policy_anything -out client-cert.pem -infiles demoCA/client-req.pem +openssl ca -keyfile cakey.pem -days 7300 -batch -cert cacert.pem -policy policy_anything -out client-cert.pem -infiles demoCA/client-req.pem + +# with SubjectAltName, only for OpenSSL 1.0.2+ +cat > demoCA/sanext.conf <<EOF +subjectAltName=DNS:localhost +EOF +openssl req -newkey rsa:1024 -keyout serversan-key.pem -out demoCA/serversan-req.pem -days 7300 -nodes -subj '/CN=server/C=FI/ST=Helsinki/L=Helsinki/O=MariaDB' +openssl ca -keyfile cakey.pem -extfile demoCA/sanext.conf -days 7300 -batch -cert cacert.pem -policy policy_anything -out serversan-cert.pem -infiles demoCA/serversan-req.pem rm -rf demoCA diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index 2c6db5ba42b..42fd0ff7397 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -184,6 +184,7 @@ my @DEFAULT_SUITES= qw( innodb_zip- json- maria- + mariabackup- multi_source- optimizer_unfixed_bugs- parts- diff --git a/mysql-test/r/handlersocket.result b/mysql-test/r/handlersocket.result index 26c77813b26..1b3fc573548 100644 --- a/mysql-test/r/handlersocket.result +++ b/mysql-test/r/handlersocket.result @@ -5,7 +5,7 @@ plugin_version 1.0 plugin_status ACTIVE plugin_type DAEMON plugin_library handlersocket.so -plugin_library_version 1.12 +plugin_library_version 1.13 plugin_author higuchi dot akira at dena dot jp plugin_description Direct access into InnoDB plugin_license BSD diff --git a/mysql-test/r/mysql_plugin.result b/mysql-test/r/mysql_plugin.result deleted file mode 100644 index 0bcb47e4a10..00000000000 --- a/mysql-test/r/mysql_plugin.result +++ /dev/null @@ -1,132 +0,0 @@ -# -# Ensure the plugin isn't loaded. -# -SELECT * FROM mysql.plugin WHERE dl like 'libdaemon%' ORDER BY name; -name dl -# -# Enable the plugin... -# -# -# Simulate loading a plugin libary with multiple entry points. -# This will test the DISABLE to ensure all rows are removed. -# -INSERT INTO mysql.plugin VALUES ('wicky', 'libdaemon_example.so'); -INSERT INTO mysql.plugin VALUES ('wacky', 'libdaemon_example.so'); -INSERT INTO mysql.plugin VALUES ('wonky', 'libdaemon_example.so'); -# -# Ensure the plugin is now loaded. -# -SELECT * FROM mysql.plugin WHERE dl like 'libdaemon%' ORDER BY name; -name dl -daemon_example libdaemon_example.so -wacky libdaemon_example.so -wicky libdaemon_example.so -wonky libdaemon_example.so -# -# Ensure the plugin is loaded. -# -SELECT * FROM mysql.plugin WHERE dl like '%libdaemon%' ORDER BY name; -name dl -daemon_example libdaemon_example.so -# -# Ensure the plugin is replaced. -# -SELECT * FROM mysql.plugin WHERE dl like '%libdaemon%' ORDER BY name; -name dl -daemon_example liblibdaemon_example.so -# -# Disable the plugin... -# -# -# Ensure the plugin isn't loaded. -# -SELECT * FROM mysql.plugin WHERE dl like '%libdaemon%' ORDER BY name; -name dl -# -# Attempt to load non-existant plugin -# -ERROR: Cannot read plugin config file NOT_THERE_AT_ALL. File does not exist. -# -# Attempt to use non-existant plugin.ini file -# -ERROR: Cannot read plugin config file daemon_example. File does not exist. -# -# Attempt to omit the plugin -# -ERROR: No plugin specified. -# -# Attempt to omit DISABLE|ENABLE -# -ERROR: missing operation. Please specify either '<plugin> ENABLE' or '<plugin> DISABLE'. -# -# Attempt to use bad paths - datadir -# -ERROR: Cannot access datadir at '/data_not_there/'. -# -# Attempt to use bad paths - basedir -# -ERROR: Cannot access basedir at '/basedir_not_there/'. -# -# Attempt to use bad paths - plugin_dir -# -ERROR: Cannot read plugin config file daemon_example. File does not exist. -# -# Attempt to use bad paths - mysqld -# -ERROR: Cannot access mysqld path '/mysqld_not_there/'. -# -# Attempt to use bad paths - my_print_defaults -# -ERROR: Cannot access my-print-defaults path '/my_print_defaults_not_there/'. -# -# Missing library -# -ERROR: The plugin library is missing or in a different location. -# -# Bad format for config file -# -ERROR: Cannot read plugin config file daemon_example. Bad format in plugin configuration file. -# -# Missing base_dir option -# -ERROR: Missing --basedir option. -# -# Missing data_dir option -# -ERROR: Missing --datadir option. -# -# Missing plugin_dir option -# -ERROR: Missing --plugin_dir option. -# -# Show the help. -# -mysql_plugin Ver V.V.VV Distrib XX.XX.XX -Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved. - -Enable or disable plugins. - -Usage: mysql_plugin [options] <plugin> ENABLE|DISABLE - -Options: - -?, --help Display this help and exit. - -b, --basedir=name The basedir for the server. - -d, --datadir=name The datadir for the server. - -p, --plugin-dir=name - The plugin dir for the server. - -i, --plugin-ini=name - Read plugin information from configuration file specified - instead of from <plugin-dir>/<plugin_name>.ini. - -n, --no-defaults Do not read values from configuration file. - -P, --print-defaults - Show default values from configuration file. - -m, --mysqld=name Path to mysqld executable. Example: /sbin/temp1/mysql/bin - -f, --my-print-defaults=name - Path to my_print_defaults executable. Example: - /source/temp11/extra - -v, --verbose More verbose output; you can use this multiple times to - get even more verbose output. - -V, --version Output version information and exit. - - -mysql_plugin Ver V.V.VV Distrib XX.XX.XX diff --git a/mysql-test/r/plugin.result b/mysql-test/r/plugin.result index f278724cc9a..3a141a25b5c 100644 --- a/mysql-test/r/plugin.result +++ b/mysql-test/r/plugin.result @@ -12,7 +12,7 @@ PLUGIN_STATUS ACTIVE PLUGIN_TYPE STORAGE ENGINE PLUGIN_TYPE_VERSION # PLUGIN_LIBRARY ha_example.so -PLUGIN_LIBRARY_VERSION 1.12 +PLUGIN_LIBRARY_VERSION 1.13 PLUGIN_AUTHOR Brian Aker, MySQL AB PLUGIN_DESCRIPTION Example storage engine PLUGIN_LICENSE GPL @@ -25,7 +25,7 @@ PLUGIN_STATUS ACTIVE PLUGIN_TYPE DAEMON PLUGIN_TYPE_VERSION # PLUGIN_LIBRARY ha_example.so -PLUGIN_LIBRARY_VERSION 1.12 +PLUGIN_LIBRARY_VERSION 1.13 PLUGIN_AUTHOR Sergei Golubchik PLUGIN_DESCRIPTION Unusable Daemon PLUGIN_LICENSE GPL @@ -64,7 +64,7 @@ PLUGIN_STATUS DELETED PLUGIN_TYPE STORAGE ENGINE PLUGIN_TYPE_VERSION # PLUGIN_LIBRARY ha_example.so -PLUGIN_LIBRARY_VERSION 1.12 +PLUGIN_LIBRARY_VERSION 1.13 PLUGIN_AUTHOR Brian Aker, MySQL AB PLUGIN_DESCRIPTION Example storage engine PLUGIN_LICENSE GPL diff --git a/mysql-test/r/ssl.result b/mysql-test/r/ssl.result index ac18da81b93..5de9e5174d8 100644 --- a/mysql-test/r/ssl.result +++ b/mysql-test/r/ssl.result @@ -4,10 +4,10 @@ have_ssl 1 SHOW STATUS LIKE 'Ssl_server_not_before'; Variable_name Value -Ssl_server_not_before Apr 25 14:55:05 2015 GMT +Ssl_server_not_before Apr 25 20:52:21 2017 GMT SHOW STATUS LIKE 'Ssl_server_not_after'; Variable_name Value -Ssl_server_not_after Apr 20 14:55:05 2035 GMT +Ssl_server_not_after Apr 20 20:52:21 2037 GMT drop table if exists t1,t2,t3,t4; CREATE TABLE t1 ( Period smallint(4) unsigned zerofill DEFAULT '0000' NOT NULL, diff --git a/mysql-test/std_data/cacert.pem b/mysql-test/std_data/cacert.pem index e934823eea3..cc5d9100e30 100644 --- a/mysql-test/std_data/cacert.pem +++ b/mysql-test/std_data/cacert.pem @@ -1,78 +1,79 @@ Certificate: Data: Version: 3 (0x2) - Serial Number: 11580370790696127632 (0xa0b5bde0f2c08c90) - Signature Algorithm: sha1WithRSAEncryption + Serial Number: + e5:b1:e3:71:e9:6f:a9:e1 + Signature Algorithm: sha256WithRSAEncryption Issuer: CN=cacert, C=FI, ST=Helsinki, L=Helsinki, O=MariaDB Validity - Not Before: Apr 25 14:55:05 2015 GMT - Not After : Apr 20 14:55:05 2035 GMT + Not Before: Apr 25 20:52:21 2017 GMT + Not After : Apr 20 20:52:21 2037 GMT Subject: CN=cacert, C=FI, ST=Helsinki, L=Helsinki, O=MariaDB Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: - 00:c0:1f:90:7c:2b:c2:ea:01:93:ce:e0:c5:72:e8: - 1c:06:bd:63:4e:b6:d2:c6:00:32:13:27:42:9e:c9: - 3c:91:33:4d:15:90:67:7d:9d:d8:be:9b:12:e2:f6: - 1b:46:81:4a:8b:10:c5:b8:14:53:ab:6a:2c:c3:7f: - 66:87:6c:0e:18:51:4e:9c:93:7a:6d:a1:d4:06:47: - 58:61:a6:04:21:2c:bd:74:7a:e4:68:45:fe:91:fe: - fb:a6:29:47:ec:c5:c3:88:c8:c9:e7:d7:c6:1a:0d: - b8:f5:c5:02:57:25:01:cc:d5:8c:37:46:58:c6:71: - 30:ee:63:38:99:84:5e:9e:3c:af:40:d4:f0:f2:12: - 44:6e:2f:4d:cd:f9:da:4d:0e:1f:a6:fe:35:c3:9d: - 40:08:82:5e:6f:7d:4d:09:16:7d:a1:78:d6:9f:9f: - 44:d6:b1:ad:e7:50:25:1a:f3:4e:16:92:4a:17:5e: - 0b:e1:c8:9f:62:22:c4:e2:01:96:63:ed:37:a2:e5: - 70:b9:dc:c8:8e:c4:fe:00:21:f5:b9:48:c0:43:55: - 4a:d8:0c:9d:ce:d6:60:30:bb:81:31:c8:e9:0e:aa: - 1c:18:3d:e4:10:47:42:17:c0:4d:fb:f5:d9:c2:e4: - 07:33:f7:15:94:63:6d:11:ad:4f:d4:1d:11:41:c1: - e2:dd + 00:a0:ad:d5:b1:ec:45:6f:d6:33:fc:5a:03:29:14: + f1:8e:78:d5:27:53:79:e0:92:7c:10:3b:79:a0:d7: + b6:9d:a8:5c:4d:fa:68:11:b3:03:9e:ee:5e:20:79: + 23:d8:9c:49:34:9c:1d:c4:6e:53:1f:9a:92:1f:08: + c1:15:e2:ad:cf:59:cd:1e:55:84:79:f9:09:ca:36: + 8a:50:83:c6:38:48:c6:d3:fa:f6:f2:2a:4f:bd:5d: + 60:9d:eb:21:c4:8c:f2:dd:2d:49:10:63:46:47:de: + 2d:59:a0:4a:e0:58:e6:c0:ae:d8:d4:5e:9a:f8:f5: + 68:1d:ea:80:8a:d6:01:b0:d5:5f:30:4d:88:5a:c5: + 1f:81:92:c1:40:54:c8:bb:a6:a1:43:de:81:3c:4b: + 79:95:82:bb:52:da:a3:a4:a0:69:ff:7e:00:8c:86: + 85:ec:af:03:68:a8:83:48:a0:e4:1d:31:a9:5c:47: + 99:9d:3a:3f:b5:3e:12:7c:4d:47:15:72:f1:11:5c: + 4a:ef:08:1c:7b:8f:e6:03:06:07:4f:94:21:b0:5e: + 27:fa:93:8c:b4:cc:56:34:3b:6d:c4:4a:14:57:b2: + 21:1a:3e:2f:c5:9e:47:1a:59:05:22:0e:56:b1:a7: + e8:80:9b:82:c3:54:57:12:05:94:79:a2:03:d9:64: + 3c:63 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: - C7:2C:01:95:1A:F5:3E:CD:04:A6:24:35:35:04:D9:A7:16:01:2A:79 + 1C:C7:2B:AA:1B:B1:BB:2E:9A:F4:0F:B1:86:60:57:38:C2:41:05:12 X509v3 Authority Key Identifier: - keyid:C7:2C:01:95:1A:F5:3E:CD:04:A6:24:35:35:04:D9:A7:16:01:2A:79 + keyid:1C:C7:2B:AA:1B:B1:BB:2E:9A:F4:0F:B1:86:60:57:38:C2:41:05:12 X509v3 Basic Constraints: CA:TRUE - Signature Algorithm: sha1WithRSAEncryption - 40:6f:6a:54:f3:29:30:48:46:bd:da:46:71:64:52:14:a7:c2: - 34:b7:5e:1e:42:3d:e7:47:92:cd:87:e7:9d:5d:1a:82:77:82: - 62:32:d4:9d:b6:44:11:dc:88:78:38:a5:d3:1f:1e:be:c2:d6: - 14:b0:58:35:cd:66:22:43:97:ba:bb:e3:44:4f:9d:75:14:9f: - 6f:37:d3:50:07:09:36:bc:58:92:e8:fe:c0:a8:ba:29:55:65: - e2:6f:8f:ab:a5:1d:4f:56:37:de:c7:b4:39:20:4c:a8:4c:db: - 56:51:12:7e:e7:7f:83:9d:c4:c7:72:8f:6f:83:f0:af:e3:37: - 1c:40:fe:5e:38:26:2f:05:46:a7:0c:a5:81:79:d6:9c:9c:d7: - 56:eb:96:fe:c7:ae:8e:4f:5e:4a:6c:3a:fa:68:be:65:60:a2: - d3:3f:07:76:45:b3:95:3e:11:ef:3a:0e:6f:73:47:4c:90:dd: - 0b:36:b4:22:df:62:8d:58:d2:a6:34:5b:f0:06:5d:cd:bf:52: - fa:ee:9b:4f:e8:79:18:6e:1c:6e:5f:96:10:6d:2f:02:1b:dd: - bf:14:c9:32:3c:83:a5:6e:56:56:78:9d:ce:84:50:a4:df:cc: - b5:a9:b1:ec:09:07:74:02:27:7a:9d:d2:96:a9:80:95:9a:f2: - 8c:e9:ef:99 + Signature Algorithm: sha256WithRSAEncryption + 0d:4b:21:52:fa:49:34:56:14:db:83:ae:1c:3d:a7:4d:3e:ea: + 55:7e:1a:37:7a:65:89:ee:19:05:94:9d:3a:ad:59:c4:38:16: + b2:bd:02:ee:5a:a6:7e:e2:b1:21:a3:ad:af:8c:ae:c3:30:71: + ad:d7:d2:24:0f:c4:d9:47:80:c5:95:05:1d:7c:8a:49:0a:7d: + 8b:61:ca:b5:68:3d:3e:4e:f1:c7:45:62:c8:cc:a9:2f:f3:12: + f1:3f:92:34:7f:07:ab:d3:ac:ab:af:2d:c9:69:63:8a:b2:e5: + 35:ea:7d:b8:17:38:72:82:5f:96:3d:dc:8d:e5:11:bb:ae:f3: + 02:2d:20:77:5c:64:59:18:a6:e7:fa:c7:89:e8:30:12:14:04: + 40:5b:e9:b1:8f:86:81:b9:0d:6c:b6:fc:98:f9:b7:52:ab:8f: + 7e:53:c8:a0:05:e4:cd:0d:6b:d2:74:9f:17:7a:a1:c3:76:5e: + f3:29:1c:c6:be:56:ab:02:f7:5d:e1:c9:21:27:6d:66:7a:41: + 29:49:a3:f8:f5:2a:e7:03:2a:7c:52:4b:f5:46:58:45:be:a4: + 4c:a0:65:37:1d:d8:ac:f8:1f:81:ca:9c:79:f0:ff:22:8c:1d: + ce:2b:d0:1e:ce:99:f2:db:fa:66:84:e6:86:6f:19:3b:10:f1: + 92:ac:57:b2 -----BEGIN CERTIFICATE----- -MIIDfzCCAmegAwIBAgIJAKC1veDywIyQMA0GCSqGSIb3DQEBBQUAMFYxDzANBgNV +MIIDfzCCAmegAwIBAgIJAOWx43Hpb6nhMA0GCSqGSIb3DQEBCwUAMFYxDzANBgNV BAMMBmNhY2VydDELMAkGA1UEBhMCRkkxETAPBgNVBAgMCEhlbHNpbmtpMREwDwYD -VQQHDAhIZWxzaW5raTEQMA4GA1UECgwHTWFyaWFEQjAeFw0xNTA0MjUxNDU1MDVa -Fw0zNTA0MjAxNDU1MDVaMFYxDzANBgNVBAMMBmNhY2VydDELMAkGA1UEBhMCRkkx +VQQHDAhIZWxzaW5raTEQMA4GA1UECgwHTWFyaWFEQjAeFw0xNzA0MjUyMDUyMjFa +Fw0zNzA0MjAyMDUyMjFaMFYxDzANBgNVBAMMBmNhY2VydDELMAkGA1UEBhMCRkkx ETAPBgNVBAgMCEhlbHNpbmtpMREwDwYDVQQHDAhIZWxzaW5raTEQMA4GA1UECgwH -TWFyaWFEQjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMAfkHwrwuoB -k87gxXLoHAa9Y0620sYAMhMnQp7JPJEzTRWQZ32d2L6bEuL2G0aBSosQxbgUU6tq -LMN/ZodsDhhRTpyTem2h1AZHWGGmBCEsvXR65GhF/pH++6YpR+zFw4jIyefXxhoN -uPXFAlclAczVjDdGWMZxMO5jOJmEXp48r0DU8PISRG4vTc352k0OH6b+NcOdQAiC -Xm99TQkWfaF41p+fRNaxredQJRrzThaSShdeC+HIn2IixOIBlmPtN6LlcLncyI7E -/gAh9blIwENVStgMnc7WYDC7gTHI6Q6qHBg95BBHQhfATfv12cLkBzP3FZRjbRGt -T9QdEUHB4t0CAwEAAaNQME4wHQYDVR0OBBYEFMcsAZUa9T7NBKYkNTUE2acWASp5 -MB8GA1UdIwQYMBaAFMcsAZUa9T7NBKYkNTUE2acWASp5MAwGA1UdEwQFMAMBAf8w -DQYJKoZIhvcNAQEFBQADggEBAEBvalTzKTBIRr3aRnFkUhSnwjS3Xh5CPedHks2H -551dGoJ3gmIy1J22RBHciHg4pdMfHr7C1hSwWDXNZiJDl7q740RPnXUUn28301AH -CTa8WJLo/sCouilVZeJvj6ulHU9WN97HtDkgTKhM21ZREn7nf4OdxMdyj2+D8K/j -NxxA/l44Ji8FRqcMpYF51pyc11brlv7Hro5PXkpsOvpovmVgotM/B3ZFs5U+Ee86 -Dm9zR0yQ3Qs2tCLfYo1Y0qY0W/AGXc2/Uvrum0/oeRhuHG5flhBtLwIb3b8UyTI8 -g6VuVlZ4nc6EUKTfzLWpsewJB3QCJ3qd0papgJWa8ozp75k= +TWFyaWFEQjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKCt1bHsRW/W +M/xaAykU8Y541SdTeeCSfBA7eaDXtp2oXE36aBGzA57uXiB5I9icSTScHcRuUx+a +kh8IwRXirc9ZzR5VhHn5Cco2ilCDxjhIxtP69vIqT71dYJ3rIcSM8t0tSRBjRkfe +LVmgSuBY5sCu2NRemvj1aB3qgIrWAbDVXzBNiFrFH4GSwUBUyLumoUPegTxLeZWC +u1Lao6Sgaf9+AIyGheyvA2iog0ig5B0xqVxHmZ06P7U+EnxNRxVy8RFcSu8IHHuP +5gMGB0+UIbBeJ/qTjLTMVjQ7bcRKFFeyIRo+L8WeRxpZBSIOVrGn6ICbgsNUVxIF +lHmiA9lkPGMCAwEAAaNQME4wHQYDVR0OBBYEFBzHK6obsbsumvQPsYZgVzjCQQUS +MB8GA1UdIwQYMBaAFBzHK6obsbsumvQPsYZgVzjCQQUSMAwGA1UdEwQFMAMBAf8w +DQYJKoZIhvcNAQELBQADggEBAA1LIVL6STRWFNuDrhw9p00+6lV+Gjd6ZYnuGQWU +nTqtWcQ4FrK9Au5apn7isSGjra+MrsMwca3X0iQPxNlHgMWVBR18ikkKfYthyrVo +PT5O8cdFYsjMqS/zEvE/kjR/B6vTrKuvLclpY4qy5TXqfbgXOHKCX5Y93I3lEbuu +8wItIHdcZFkYpuf6x4noMBIUBEBb6bGPhoG5DWy2/Jj5t1Krj35TyKAF5M0Na9J0 +nxd6ocN2XvMpHMa+VqsC913hySEnbWZ6QSlJo/j1KucDKnxSS/VGWEW+pEygZTcd +2Kz4H4HKnHnw/yKMHc4r0B7OmfLb+maE5oZvGTsQ8ZKsV7I= -----END CERTIFICATE----- diff --git a/mysql-test/std_data/cakey.pem b/mysql-test/std_data/cakey.pem new file mode 100644 index 00000000000..88e251f00e2 --- /dev/null +++ b/mysql-test/std_data/cakey.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCgrdWx7EVv1jP8 +WgMpFPGOeNUnU3ngknwQO3mg17adqFxN+mgRswOe7l4geSPYnEk0nB3EblMfmpIf +CMEV4q3PWc0eVYR5+QnKNopQg8Y4SMbT+vbyKk+9XWCd6yHEjPLdLUkQY0ZH3i1Z +oErgWObArtjUXpr49Wgd6oCK1gGw1V8wTYhaxR+BksFAVMi7pqFD3oE8S3mVgrtS +2qOkoGn/fgCMhoXsrwNoqINIoOQdMalcR5mdOj+1PhJ8TUcVcvERXErvCBx7j+YD +BgdPlCGwXif6k4y0zFY0O23EShRXsiEaPi/FnkcaWQUiDlaxp+iAm4LDVFcSBZR5 +ogPZZDxjAgMBAAECggEAWmy6AGFpSmEP7IpzkOEaeAWEX5dY1YtaioAOGPiM6vje +yXuMqblG5mBbVIcYJ0T85cCd9/fmi7ifVxvEHh7tle2Bw/p4jXQbkFNVT655FR/P +1Wg9JVeufHFaeETlQgnYe6SKo9BaswNUHkZZHRyq7/D2Ub3UFRt2tq9MG9YIKY1m +rP9s7E+EDuH9UhYmaWdQfNm8muIXWK8WjicI5+PX0CQ1NtUy6vS7qBzcBzvT0chC +Jtja29S6Nvg12A96nHsRmQyUaQjRlqosSwiagpc5mZmNeCEUoY+3deIdYIUMSQnf +judZOKVPq0GOW5Y1U068LGODWaifPkinGBj+04VH0QKBgQDOp/jVCOUdEeqFJ/8m +wEsfsRIrXvtGJHgbDXcVJ69FwlX+yaKGEuC+4f21uyxPn6GoFw+NKAyTmGKH7VAX +OFQLrMQ/DMlNbZrCAAFcXMqrnLaVwqMeIIoVNfKAa8u15K40qc+B0it61Nlay5wq +wvXoSZrdqXSgsI29pav20+8pTQKBgQDHC3l1+gMZ1rCar+5KdVBN1Wq4Xh7cwZw6 +FxEvyrDCJePEU2L7FpH1pFuB4WpXdBu3CPo70ZgwfqBXn4qLOOI3gTtDHActyiUm ++WRG62O+5Ye7aLB4xy0MfnKNA2g/yHj1ozwM8kA5JRptAzDnzWfVE0k47/pVAVzt +E2bZuSykbwKBgQCL6SkMgjMr1T9j20phn/q8gBN/DZUtTe+K0Tj4N5/wqLuz/its +fkdutG4ipZBAcCDwPnym4qBxJNBAmqiIr/gm11ceILgBFd2azoodUC1etoDfL6Fj ++j/CUH3X+CM5CJPwz67Pg80wIf7t+7/FK611ELAqtllhmWa9KPcd6yqWWQKBgHh5 +Xnvk5kmWY3BNOgrBNOjXWu/asA1n9lpGqfVmVlQ8wL6MxiU5xQCMCYL0X/ws37WK +boMUWmxHyF8gxqd7t5hm1OrKpSG274PGgUZXpRjfLqdlNyLzUzXztvvY6xloCqaK +tYcUfYDZD0SaINi8v7L9KF2ZCsi2uXsZOjBf30BrAoGAXPPotkw/CkcPQBS13cha +ZWeeH5NDKBADWXfLfcRUs108c9xw4BYr5yGilSPscN2ZP0/iWONKp/c6/STS54t5 +lkOKKUbkAFbQu8UKa1J7zrnHZv+Mr4I/iBBy6VkN8Spp2vBI3Ng6jhPIJg3Gum9p +943wWtAnIhe/UqCRT3a/GZg= +-----END PRIVATE KEY----- diff --git a/mysql-test/std_data/client-cert.pem b/mysql-test/std_data/client-cert.pem index cbe8bc2c677..9f6f0cfde7c 100644 --- a/mysql-test/std_data/client-cert.pem +++ b/mysql-test/std_data/client-cert.pem @@ -2,25 +2,25 @@ Certificate: Data: Version: 3 (0x2) Serial Number: 3 (0x3) - Signature Algorithm: sha1WithRSAEncryption + Signature Algorithm: sha256WithRSAEncryption Issuer: CN=cacert, C=FI, ST=Helsinki, L=Helsinki, O=MariaDB Validity - Not Before: Apr 25 14:55:16 2015 GMT - Not After : Apr 20 14:55:16 2035 GMT + Not Before: Apr 25 20:52:33 2017 GMT + Not After : Apr 20 20:52:33 2037 GMT Subject: C=FI, ST=Helsinki, L=Helsinki, O=MariaDB, CN=client Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (1024 bit) Modulus: - 00:ce:a0:3d:3c:a4:bb:4f:a1:4f:91:0d:05:ac:5b: - 8a:15:7f:d7:aa:0c:a3:a7:9f:b2:c7:26:9d:65:28: - b1:84:d3:a0:ef:9e:b1:45:0f:33:df:98:6e:71:ff: - 2b:66:9c:9c:c1:25:13:27:42:b6:20:46:e7:e7:47: - a1:88:47:c2:9e:e2:45:25:99:9f:f9:28:1a:9a:13: - 67:5d:3e:b3:b8:fe:40:25:ac:26:49:46:2c:03:43: - 83:67:d8:0f:41:ae:2e:f4:d8:71:60:3c:8e:e7:91: - d0:bb:2c:ca:12:da:71:1a:7b:e3:fa:8c:8f:c3:bb: - 62:55:89:b3:bf:85:45:01:61 + 00:a1:10:ea:cc:8e:2c:73:6b:33:1a:5e:26:19:b6: + 4b:4c:bc:04:b8:c2:e2:33:eb:67:a2:7a:27:af:3f: + f7:ef:49:5f:c1:d2:b9:d9:71:fe:17:a0:93:da:dc: + f1:47:de:fa:1f:c3:c1:d1:a5:2a:06:cb:b3:e8:9a: + c1:bd:78:77:68:45:c1:55:cd:b1:c1:d3:df:8c:12: + 4f:c2:3a:0d:b7:58:dc:ca:13:08:b9:fb:12:24:90: + aa:b7:4e:04:eb:43:0d:45:be:1c:17:d6:a8:b1:af: + 10:3c:39:d6:08:45:ed:a9:7e:3a:69:ae:70:22:86: + 7e:71:1f:f1:0e:d0:0d:32:c3 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Basic Constraints: @@ -28,42 +28,42 @@ Certificate: Netscape Comment: OpenSSL Generated Certificate X509v3 Subject Key Identifier: - 5A:73:74:8E:14:29:C3:FB:B4:19:0F:97:8F:AA:6F:E1:E1:A8:F7:5B + 0C:20:76:A1:80:9C:2F:30:3D:F7:AB:8D:31:19:AD:E2:F7:E2:8D:12 X509v3 Authority Key Identifier: - keyid:C7:2C:01:95:1A:F5:3E:CD:04:A6:24:35:35:04:D9:A7:16:01:2A:79 + keyid:1C:C7:2B:AA:1B:B1:BB:2E:9A:F4:0F:B1:86:60:57:38:C2:41:05:12 - Signature Algorithm: sha1WithRSAEncryption - 32:42:4b:36:44:a5:6c:fb:70:d8:08:2b:cb:16:34:15:db:39: - 60:7b:7e:b4:4a:bc:fb:e5:16:04:97:0d:eb:f5:68:95:da:2f: - 23:57:4c:c9:29:2b:d1:1b:1b:9f:bd:f4:79:75:df:62:7f:63: - b4:84:7a:95:5c:c4:ee:f3:77:16:e4:0b:8a:5e:c9:64:bd:7c: - 04:50:ac:ff:9a:41:6b:b1:6a:9f:cd:45:10:72:83:10:8a:26: - 1d:7f:6c:84:34:5a:41:79:72:91:ee:87:5d:1d:3a:55:ff:91: - 7e:52:85:ff:42:41:eb:76:56:23:e5:bc:bc:79:b1:aa:4e:4c: - bf:7b:df:63:8b:1a:3c:4b:01:72:89:35:bb:0d:92:97:16:6e: - ae:50:cb:89:ee:c6:7a:d0:d3:32:22:0f:19:33:1e:ee:ff:41: - a5:a1:25:c5:4c:ce:8f:98:4c:b5:2c:1f:ec:cc:f1:21:e2:3a: - ff:7d:6a:87:fe:89:fd:2c:20:3e:fb:9b:b8:c0:f9:09:99:ce: - 45:63:82:09:1c:bb:79:d8:a8:40:21:46:c7:ae:3e:dd:89:9d: - 56:46:4a:f4:ed:7d:5b:a6:1e:a6:1b:26:f9:ec:26:b4:51:3a: - 87:b6:50:13:84:33:22:1a:8a:20:c5:44:64:b8:bb:de:32:ec: - 6b:58:db:17 + Signature Algorithm: sha256WithRSAEncryption + 39:c0:90:13:19:85:47:9d:c6:ab:8c:c6:c9:0f:33:11:19:f7: + 01:2c:1b:08:f6:81:98:11:ab:48:05:d9:b2:29:56:32:9c:ba: + e5:40:df:85:5e:6d:fd:6e:36:9a:14:eb:90:50:57:de:2f:ed: + 2d:89:a6:8a:40:1c:41:84:9b:da:e1:6d:e6:7c:46:b2:e0:90: + 93:02:1c:52:2e:af:b4:d4:a1:d8:9d:19:cf:0a:67:bf:c3:3e: + 2e:02:f4:3e:bc:2e:59:57:30:85:8a:32:ab:22:88:72:37:6e: + ee:ed:f8:53:72:c9:28:87:50:47:81:1b:80:4c:f8:80:ce:2f: + 47:ca:78:ce:38:51:70:ec:df:ee:fc:ea:5a:40:1e:4d:1c:fd: + 4e:f6:74:d0:22:a4:7e:57:df:16:1a:a0:8d:be:fe:ee:f2:07: + 2e:39:a1:97:40:19:f9:3b:b8:e7:c4:98:6e:1d:1a:27:d3:19: + 4c:5c:c9:c3:31:98:c1:3c:27:0e:6a:de:cf:88:72:cf:e4:65: + c9:0d:33:32:f1:ea:f7:dd:5b:9d:42:6d:ee:c7:a8:b7:85:d2: + 41:e0:84:38:ce:86:81:ba:6e:7d:d5:ad:7a:00:58:d7:c5:83: + 9e:5c:1d:38:32:72:49:f5:42:4b:e7:c6:5c:12:6d:e1:5d:51: + 2c:f5:52:f0 -----BEGIN CERTIFICATE----- -MIIDHjCCAgagAwIBAgIBAzANBgkqhkiG9w0BAQUFADBWMQ8wDQYDVQQDDAZjYWNl +MIIDHjCCAgagAwIBAgIBAzANBgkqhkiG9w0BAQsFADBWMQ8wDQYDVQQDDAZjYWNl cnQxCzAJBgNVBAYTAkZJMREwDwYDVQQIDAhIZWxzaW5raTERMA8GA1UEBwwISGVs -c2lua2kxEDAOBgNVBAoMB01hcmlhREIwHhcNMTUwNDI1MTQ1NTE2WhcNMzUwNDIw -MTQ1NTE2WjBWMQswCQYDVQQGEwJGSTERMA8GA1UECAwISGVsc2lua2kxETAPBgNV +c2lua2kxEDAOBgNVBAoMB01hcmlhREIwHhcNMTcwNDI1MjA1MjMzWhcNMzcwNDIw +MjA1MjMzWjBWMQswCQYDVQQGEwJGSTERMA8GA1UECAwISGVsc2lua2kxETAPBgNV BAcMCEhlbHNpbmtpMRAwDgYDVQQKDAdNYXJpYURCMQ8wDQYDVQQDDAZjbGllbnQw -gZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAM6gPTyku0+hT5ENBaxbihV/16oM -o6efsscmnWUosYTToO+esUUPM9+YbnH/K2acnMElEydCtiBG5+dHoYhHwp7iRSWZ -n/koGpoTZ10+s7j+QCWsJklGLANDg2fYD0GuLvTYcWA8jueR0LssyhLacRp74/qM -j8O7YlWJs7+FRQFhAgMBAAGjezB5MAkGA1UdEwQCMAAwLAYJYIZIAYb4QgENBB8W -HU9wZW5TU0wgR2VuZXJhdGVkIENlcnRpZmljYXRlMB0GA1UdDgQWBBRac3SOFCnD -+7QZD5ePqm/h4aj3WzAfBgNVHSMEGDAWgBTHLAGVGvU+zQSmJDU1BNmnFgEqeTAN -BgkqhkiG9w0BAQUFAAOCAQEAMkJLNkSlbPtw2AgryxY0Fds5YHt+tEq8++UWBJcN -6/VoldovI1dMySkr0Rsbn730eXXfYn9jtIR6lVzE7vN3FuQLil7JZL18BFCs/5pB -a7Fqn81FEHKDEIomHX9shDRaQXlyke6HXR06Vf+RflKF/0JB63ZWI+W8vHmxqk5M -v3vfY4saPEsBcok1uw2SlxZurlDLie7GetDTMiIPGTMe7v9BpaElxUzOj5hMtSwf -7MzxIeI6/31qh/6J/SwgPvubuMD5CZnORWOCCRy7edioQCFGx64+3YmdVkZK9O19 -W6Yephsm+ewmtFE6h7ZQE4QzIhqKIMVEZLi73jLsa1jbFw== +gZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAKEQ6syOLHNrMxpeJhm2S0y8BLjC +4jPrZ6J6J68/9+9JX8HSudlx/hegk9rc8Ufe+h/DwdGlKgbLs+iawb14d2hFwVXN +scHT34wST8I6DbdY3MoTCLn7EiSQqrdOBOtDDUW+HBfWqLGvEDw51ghF7al+Ommu +cCKGfnEf8Q7QDTLDAgMBAAGjezB5MAkGA1UdEwQCMAAwLAYJYIZIAYb4QgENBB8W +HU9wZW5TU0wgR2VuZXJhdGVkIENlcnRpZmljYXRlMB0GA1UdDgQWBBQMIHahgJwv +MD33q40xGa3i9+KNEjAfBgNVHSMEGDAWgBQcxyuqG7G7Lpr0D7GGYFc4wkEFEjAN +BgkqhkiG9w0BAQsFAAOCAQEAOcCQExmFR53Gq4zGyQ8zERn3ASwbCPaBmBGrSAXZ +silWMpy65UDfhV5t/W42mhTrkFBX3i/tLYmmikAcQYSb2uFt5nxGsuCQkwIcUi6v +tNSh2J0Zzwpnv8M+LgL0PrwuWVcwhYoyqyKIcjdu7u34U3LJKIdQR4EbgEz4gM4v +R8p4zjhRcOzf7vzqWkAeTRz9TvZ00CKkflffFhqgjb7+7vIHLjmhl0AZ+Tu458SY +bh0aJ9MZTFzJwzGYwTwnDmrez4hyz+RlyQ0zMvHq991bnUJt7seot4XSQeCEOM6G +gbpufdWtegBY18WDnlwdODJySfVCS+fGXBJt4V1RLPVS8A== -----END CERTIFICATE----- diff --git a/mysql-test/std_data/client-key.pem b/mysql-test/std_data/client-key.pem index 5037c6e2728..b6b5507cd15 100644 --- a/mysql-test/std_data/client-key.pem +++ b/mysql-test/std_data/client-key.pem @@ -1,15 +1,15 @@ -----BEGIN RSA PRIVATE KEY----- -MIICXAIBAAKBgQDOoD08pLtPoU+RDQWsW4oVf9eqDKOnn7LHJp1lKLGE06DvnrFF -DzPfmG5x/ytmnJzBJRMnQrYgRufnR6GIR8Ke4kUlmZ/5KBqaE2ddPrO4/kAlrCZJ -RiwDQ4Nn2A9Bri702HFgPI7nkdC7LMoS2nEae+P6jI/Du2JVibO/hUUBYQIDAQAB -AoGAa/FgLFcul3oA9BjmdtVXfMXNp8N0l3QhVFLC9P7eRjK8p5GysA4yHkQmpp0U -UkXMykYRDHiYZqJEMhnEtEowzBmodi7go+gpwAR2eUKwESmJoBhPvqDJAbS/fL5D -H2Wk6FGsdKoPhEpigWefu6ZqlX5GCGa601eMYLMR9i+6bbUCQQDspD4j2q8oihTU -RQt/XpF1l+5ZRHjQOokwRekuHdq0powtNxZ+X3V8Qy8JbDRNCM2YtfKMX4gXAfZp -JWs7HoPvAkEA34doY3AKxZSpXD84m4dnJ0/Ubfk3+tcC1EPYyDJ1DHpfz7fy6aoX -z8TWCQXtSBGaEa9Dgbz+EFXuctLbUR8/rwJACDjIo+xEK69oe9uOQ7WgbiqCMH3N -iMaP36p+KIkHAUHMGwIP+QIODewzpSsqQgbtRcIElFX5X3tE+XBAYoRz5wJAKH3/ -CwRg7ynfBDbvqjz9EsIDWWisG2SXvpwLyThau8fvU1GfT3Tgm2Ks4zWPpl6J6mo1 -cGssGwl2CJbp4+glQQJBAJAwvKufpB+M6OjvKh89GGsCEaV1ENJ41FPcQwJ2pjed -Fcq28ZP59v7bfBH2IkNu3pfEzmvQnmRlTEtXGjNn+i8= +MIICXQIBAAKBgQChEOrMjixzazMaXiYZtktMvAS4wuIz62eieievP/fvSV/B0rnZ +cf4XoJPa3PFH3vofw8HRpSoGy7PomsG9eHdoRcFVzbHB09+MEk/COg23WNzKEwi5 ++xIkkKq3TgTrQw1FvhwX1qixrxA8OdYIRe2pfjpprnAihn5xH/EO0A0ywwIDAQAB +AoGAaJMqT+vTcpDcmjcct78DPuwoiKmH4rvqCrUTRRbcbJGQSbD/F+6KUl7hAM5J +Hifo8GzST8LT+ZuS0OiB9/naVGJjcLzpV+us+keMctB/cjmsPAAsRdeA8Xk0jTWv +v+5a5ZWSSbRXycuUtywtoESY1RLEyB0k3Dcxvk6SD3LnNMkCQQDTyDJfP+eirhgC +mKiNvHJjLtlRT11IMLMVTELEntsyKqzVgficZCWGkFRoF9iEO1OyCJaug7RZsuxs +Z3lk/gq1AkEAwrHwzVPU+dTqsZ8tYHlq+d0xy+6eTtYy7e/5qH3AUz10us6BG/LY +XVTwRFAOKQOciKZ/zPQa4oYeAc0oozSalwJBAIuMbyS0Rz262bdcQDSk5/rS8//P ++/eFn3t5NMW6p1T3KcvGSLtEgjWZBQVSMSlwrkWxwxhbUIuKip8jz6nse8ECQDgm +g5FkLRdEfc9uXfLl8aFQVu0+y29nPVb8D9+1LMOVBNZfekLqPdZlCcpZ4EuZ3ApZ +IqCkgiB0l7DjT2YKZM8CQQCp8z1JvoNRwhIuojh7JajY5QBkXtvHootXr+vDUHNf +SgR5BRmqZb0F5/BK4/7JbumiHI11QUULxhOnNtlS7DDt -----END RSA PRIVATE KEY----- diff --git a/mysql-test/std_data/server-cert.pem b/mysql-test/std_data/server-cert.pem index 1cc1519ada9..b874f129ce3 100644 --- a/mysql-test/std_data/server-cert.pem +++ b/mysql-test/std_data/server-cert.pem @@ -2,25 +2,25 @@ Certificate: Data: Version: 3 (0x2) Serial Number: 1 (0x1) - Signature Algorithm: sha1WithRSAEncryption + Signature Algorithm: sha256WithRSAEncryption Issuer: CN=cacert, C=FI, ST=Helsinki, L=Helsinki, O=MariaDB Validity - Not Before: Apr 25 14:55:05 2015 GMT - Not After : Apr 20 14:55:05 2035 GMT + Not Before: Apr 25 20:52:21 2017 GMT + Not After : Apr 20 20:52:21 2037 GMT Subject: C=FI, ST=state or province within country, in other certificates in this file it is the same as L, L=location, usually an address but often ambiguously used, O=organization name, typically a company name, OU=organizational unit name, a division name within an organization, CN=localhost Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (1024 bit) Modulus: - 00:aa:e6:54:bd:dd:52:1e:16:f7:24:52:37:58:2b: - a7:af:49:e1:cd:75:2a:18:52:e1:48:f0:59:82:c0: - 7a:d9:66:b3:97:04:b3:77:f4:39:fd:d1:c0:1a:c5: - a6:ab:44:84:d2:17:39:53:25:63:9b:c3:24:78:51: - 5c:77:6b:df:b4:82:1d:e4:43:f4:67:0a:5d:89:a2: - fe:b0:ea:64:3a:1d:9d:49:78:c8:7f:79:a5:cd:45: - 4b:0c:ad:ae:4f:e2:d4:5d:ec:e8:73:06:ed:98:92: - 85:49:b2:9c:31:3b:44:38:5f:bb:5a:f1:68:84:a9: - c3:5b:31:39:d4:47:98:38:55 + 00:c9:fe:83:3f:0b:38:89:1d:43:15:93:5c:26:b9: + 80:65:41:bd:2f:63:66:5e:db:fa:33:20:d9:c7:e1: + 35:f3:14:3c:c8:b4:f7:09:d8:f5:b5:44:8f:6b:7e: + a0:a4:3b:45:5f:e6:f4:0f:08:67:f8:5a:4c:49:e4: + e5:39:31:69:8c:cf:25:78:93:a6:7f:58:e9:90:9c: + 61:cb:2e:14:b1:57:b2:15:9c:ea:8d:6f:96:20:fe: + 29:ed:2c:71:b8:4f:1f:e0:05:6c:04:b1:7e:e0:bc: + 42:8e:bf:95:5e:a9:5e:c9:c9:a4:64:c2:1f:59:94: + 14:c2:06:44:79:bc:d2:65:2d Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Basic Constraints: @@ -28,47 +28,47 @@ Certificate: Netscape Comment: OpenSSL Generated Certificate X509v3 Subject Key Identifier: - E5:72:8F:57:72:D6:75:63:28:7F:E2:BF:00:B7:1D:B8:AA:FE:94:59 + CB:22:3B:E6:DA:B8:3D:7E:39:61:18:38:50:C8:4D:B4:C8:9A:3E:2B X509v3 Authority Key Identifier: - keyid:C7:2C:01:95:1A:F5:3E:CD:04:A6:24:35:35:04:D9:A7:16:01:2A:79 + keyid:1C:C7:2B:AA:1B:B1:BB:2E:9A:F4:0F:B1:86:60:57:38:C2:41:05:12 - Signature Algorithm: sha1WithRSAEncryption - 88:44:46:fa:7d:16:ae:9d:16:5b:95:26:03:3c:71:f4:29:3d: - df:cb:f4:14:20:9f:87:24:b4:29:17:2d:7a:12:48:76:ac:00: - 44:26:ba:93:83:ad:58:7e:b7:77:e4:b0:32:0d:e5:dd:fb:cc: - 0e:9b:88:e0:24:82:e4:41:43:47:5a:4e:d3:b4:5b:47:4b:57: - eb:67:02:63:bb:dd:05:12:f5:95:01:0b:89:81:ca:c2:91:14: - 21:9a:9e:c9:84:91:46:35:0e:26:44:1e:91:88:74:4f:fe:d3: - 19:9e:65:fa:46:e2:46:04:ad:91:79:4c:70:1b:68:b2:49:e9: - 6c:f4:58:44:3b:43:15:85:56:64:1b:84:74:49:95:9f:cd:93: - 9d:8e:69:ab:ca:46:97:b6:74:e9:2a:83:85:62:cd:e5:be:c3: - 52:bd:cf:90:cc:60:27:76:ee:1b:3c:da:69:73:e2:11:68:14: - dc:7d:9f:b8:6f:20:a2:0c:b7:8e:33:40:89:d1:a3:89:e2:60: - 6a:ec:b5:9f:e8:c5:55:10:40:b2:95:5e:54:8a:10:8e:d5:90: - d9:98:86:d8:f9:b6:01:41:8c:d7:0d:0e:86:0e:50:6d:a2:64: - 00:2a:91:5e:35:64:15:e3:86:34:3a:39:eb:0f:4f:56:c7:15: - 4c:74:2e:91 + Signature Algorithm: sha256WithRSAEncryption + 8e:b7:3d:cc:2b:e5:27:49:49:5a:d4:3c:83:9b:2f:7d:11:de: + 6b:0f:b4:51:02:e4:37:d0:c4:b5:7b:4b:e3:42:93:75:32:d1: + eb:41:a2:27:fe:4d:91:ae:2b:a0:8b:3b:7f:e9:1b:47:85:73: + 9f:b7:05:74:34:eb:62:12:d8:f2:24:6a:b4:24:58:7a:6a:55: + 3e:ba:54:f9:51:de:54:0e:19:06:f6:01:23:32:0a:6c:81:e2: + 57:8a:9e:71:c5:ba:ca:c3:0f:18:d4:ef:54:4b:e1:ee:7c:3a: + f3:4d:c3:a3:44:21:9f:c8:ef:85:01:ab:fe:a9:7b:36:05:10: + 25:5e:61:b8:1d:7c:40:8a:5d:5b:c1:bb:7c:79:45:6e:84:63: + b5:3f:51:e9:9c:57:01:de:2d:1e:85:cb:83:f0:16:6d:78:bb: + 12:01:a5:e5:a2:a7:80:fa:54:8f:9a:5c:de:1c:52:a2:bd:00: + 49:e4:04:65:30:f9:b9:fc:4f:94:e2:d8:39:89:b6:a5:a5:2d: + db:25:a3:0d:5b:f9:e1:2a:13:19:5e:d7:1a:33:89:5b:ac:bd: + 29:26:da:1a:90:7e:08:19:dd:59:4b:50:fd:46:c3:0b:91:33: + 8b:c6:70:d1:22:18:37:a3:8b:d3:8f:f8:68:cf:38:8e:e1:ef: + ac:17:88:ca -----BEGIN CERTIFICATE----- -MIIEETCCAvmgAwIBAgIBATANBgkqhkiG9w0BAQUFADBWMQ8wDQYDVQQDDAZjYWNl +MIIEETCCAvmgAwIBAgIBATANBgkqhkiG9w0BAQsFADBWMQ8wDQYDVQQDDAZjYWNl cnQxCzAJBgNVBAYTAkZJMREwDwYDVQQIDAhIZWxzaW5raTERMA8GA1UEBwwISGVs -c2lua2kxEDAOBgNVBAoMB01hcmlhREIwHhcNMTUwNDI1MTQ1NTA1WhcNMzUwNDIw -MTQ1NTA1WjCCAUcxCzAJBgNVBAYTAkZJMWEwXwYDVQQIDFhzdGF0ZSBvciBwcm92 +c2lua2kxEDAOBgNVBAoMB01hcmlhREIwHhcNMTcwNDI1MjA1MjIxWhcNMzcwNDIw +MjA1MjIxWjCCAUcxCzAJBgNVBAYTAkZJMWEwXwYDVQQIDFhzdGF0ZSBvciBwcm92 aW5jZSB3aXRoaW4gY291bnRyeSwgaW4gb3RoZXIgY2VydGlmaWNhdGVzIGluIHRo aXMgZmlsZSBpdCBpcyB0aGUgc2FtZSBhcyBMMUAwPgYDVQQHDDdsb2NhdGlvbiwg dXN1YWxseSBhbiBhZGRyZXNzIGJ1dCBvZnRlbiBhbWJpZ3VvdXNseSB1c2VkMTQw MgYDVQQKDCtvcmdhbml6YXRpb24gbmFtZSwgdHlwaWNhbGx5IGEgY29tcGFueSBu YW1lMUkwRwYDVQQLDEBvcmdhbml6YXRpb25hbCB1bml0IG5hbWUsIGEgZGl2aXNp b24gbmFtZSB3aXRoaW4gYW4gb3JnYW5pemF0aW9uMRIwEAYDVQQDDAlsb2NhbGhv -c3QwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAKrmVL3dUh4W9yRSN1grp69J -4c11KhhS4UjwWYLAetlms5cEs3f0Of3RwBrFpqtEhNIXOVMlY5vDJHhRXHdr37SC -HeRD9GcKXYmi/rDqZDodnUl4yH95pc1FSwytrk/i1F3s6HMG7ZiShUmynDE7RDhf -u1rxaISpw1sxOdRHmDhVAgMBAAGjezB5MAkGA1UdEwQCMAAwLAYJYIZIAYb4QgEN -BB8WHU9wZW5TU0wgR2VuZXJhdGVkIENlcnRpZmljYXRlMB0GA1UdDgQWBBTlco9X -ctZ1Yyh/4r8Atx24qv6UWTAfBgNVHSMEGDAWgBTHLAGVGvU+zQSmJDU1BNmnFgEq -eTANBgkqhkiG9w0BAQUFAAOCAQEAiERG+n0Wrp0WW5UmAzxx9Ck938v0FCCfhyS0 -KRctehJIdqwARCa6k4OtWH63d+SwMg3l3fvMDpuI4CSC5EFDR1pO07RbR0tX62cC -Y7vdBRL1lQELiYHKwpEUIZqeyYSRRjUOJkQekYh0T/7TGZ5l+kbiRgStkXlMcBto -sknpbPRYRDtDFYVWZBuEdEmVn82TnY5pq8pGl7Z06SqDhWLN5b7DUr3PkMxgJ3bu -GzzaaXPiEWgU3H2fuG8gogy3jjNAidGjieJgauy1n+jFVRBAspVeVIoQjtWQ2ZiG -2Pm2AUGM1w0Ohg5QbaJkACqRXjVkFeOGNDo56w9PVscVTHQukQ== +c3QwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMn+gz8LOIkdQxWTXCa5gGVB +vS9jZl7b+jMg2cfhNfMUPMi09wnY9bVEj2t+oKQ7RV/m9A8IZ/haTEnk5TkxaYzP +JXiTpn9Y6ZCcYcsuFLFXshWc6o1vliD+Ke0scbhPH+AFbASxfuC8Qo6/lV6pXsnJ +pGTCH1mUFMIGRHm80mUtAgMBAAGjezB5MAkGA1UdEwQCMAAwLAYJYIZIAYb4QgEN +BB8WHU9wZW5TU0wgR2VuZXJhdGVkIENlcnRpZmljYXRlMB0GA1UdDgQWBBTLIjvm +2rg9fjlhGDhQyE20yJo+KzAfBgNVHSMEGDAWgBQcxyuqG7G7Lpr0D7GGYFc4wkEF +EjANBgkqhkiG9w0BAQsFAAOCAQEAjrc9zCvlJ0lJWtQ8g5svfRHeaw+0UQLkN9DE +tXtL40KTdTLR60GiJ/5Nka4roIs7f+kbR4Vzn7cFdDTrYhLY8iRqtCRYempVPrpU ++VHeVA4ZBvYBIzIKbIHiV4qeccW6ysMPGNTvVEvh7nw6803Do0Qhn8jvhQGr/ql7 +NgUQJV5huB18QIpdW8G7fHlFboRjtT9R6ZxXAd4tHoXLg/AWbXi7EgGl5aKngPpU +j5pc3hxSor0ASeQEZTD5ufxPlOLYOYm2paUt2yWjDVv54SoTGV7XGjOJW6y9KSba +GpB+CBndWUtQ/UbDC5Ezi8Zw0SIYN6OL04/4aM84juHvrBeIyg== -----END CERTIFICATE----- diff --git a/mysql-test/std_data/server-key.pem b/mysql-test/std_data/server-key.pem index 3125ae88a8a..feb7fbaf535 100644 --- a/mysql-test/std_data/server-key.pem +++ b/mysql-test/std_data/server-key.pem @@ -1,15 +1,15 @@ -----BEGIN RSA PRIVATE KEY----- -MIICXAIBAAKBgQCq5lS93VIeFvckUjdYK6evSeHNdSoYUuFI8FmCwHrZZrOXBLN3 -9Dn90cAaxaarRITSFzlTJWObwyR4UVx3a9+0gh3kQ/RnCl2Jov6w6mQ6HZ1JeMh/ -eaXNRUsMra5P4tRd7OhzBu2YkoVJspwxO0Q4X7ta8WiEqcNbMTnUR5g4VQIDAQAB -AoGAblQWXyBzdBN1Z5BgRF6ieYpj6OT70QoogJMR5lRmutUPma4iQo17pr3znBT/ -nU+1w3/UtTXNEXCwqbA01q/gkbP2PaW/sbHLVow1B7u/o42WW6I3Btnl3ClnCNjD -Mo7/Gj027hhp7mC61r81JeJVh8fJUgxdNqoH7AkDnA+FJAECQQDjIl3k6W2P+bHb -bp+8eyY7ITQbppZh+3hFJKRL7DZKFYL5J6gejiBURnG9DKnhoSP2nqzqdrRhWZhB -ZHr+ciEBAkEAwJ5rMpFoIwRzgPD4Q4iSqHcBbFcJE7dK1XLq6MYUVNQGfDU8pBvI -EocXphpsJ8CbR35dGDY19rmO2LjG3RBDVQJAetRN9Inrjw2YCjNzvKjYTuew1zcq -YghszO94zfoKjdu+PWEdwJBZmVmTDoo3oGXVHfxHRHA3MeISvWJKRSmRAQJAHL9H -9msXJKrEZkkQdFvMr5HbR4UR2LxxUbvt7UGqxSJDuYPkggWXbZR15hdpbuFjC1+D -m1pz4Ve+RwAExfdoZQJBANfmuWtlLU+SMpDG4zOyC7u4dz+TtnEOfDUECFNZtqvU -MWz98MIXAjiBDYU1Z0BrA7b0/FVsPR3t6JZFQWWI2y8= +MIICXQIBAAKBgQDJ/oM/CziJHUMVk1wmuYBlQb0vY2Ze2/ozINnH4TXzFDzItPcJ +2PW1RI9rfqCkO0Vf5vQPCGf4WkxJ5OU5MWmMzyV4k6Z/WOmQnGHLLhSxV7IVnOqN +b5Yg/intLHG4Tx/gBWwEsX7gvEKOv5VeqV7JyaRkwh9ZlBTCBkR5vNJlLQIDAQAB +AoGAZzoeMJW3arr9kYUhTtj1+MlDuXf+1PNuRbrAERnSzErHVXrF1M5/owdKlBC+ +X6+6oGwSxavyFkVvP0QusK/D1DxSs60Mk1p3Ax5nVpNYiBdeBULcYiXCSUbKItYL +vzyECIc02t+auWf/wp3Wy3g6sU6FoT2QCPSsy0qRC48VBzECQQD79P9w0XjJGk8y +3zPc4JOr3a1UUu4VLjFdFDR2eZZRE02NcXfLHE0+Idk3TYnNDRLoWDJjdfZ0thZZ +KuJ58wIDAkEAzTxGYmT/aieDC3G8sHMqLUvjN30TfTocv7mWlpxWo7zbRbQm7jsB +S5weRKtu3yVmdb6rW+5IZSCazc/j4T5tDwJAUgDRSpTCrSFE+Zevt4nYRi2mBjXf +i0E3i8XRtWWpSMXxjcGKba7ObRRzMA6qdPR2XOynqbtw9Vd2Ops8jthKpQJBAJJm +8tZxsXlqIiLhyXYdUPLq9XS5tlNYRvXFT9S0RWGb8NbyQesjEDN9dGIL4JUxurs3 +fkmf2ui4r3UtXSNqQqUCQQDjatAhBdibh5UawO5VpI87OJMzATCkY/mX3R+TnbOm +MkdydjF022P0M4N24DiM+2nBTDp+F45LwtQLa4vmB2No -----END RSA PRIVATE KEY----- diff --git a/mysql-test/std_data/server8k-cert.pem b/mysql-test/std_data/server8k-cert.pem index 7e41195cea4..df27c77e8e0 100644 --- a/mysql-test/std_data/server8k-cert.pem +++ b/mysql-test/std_data/server8k-cert.pem @@ -2,85 +2,85 @@ Certificate: Data: Version: 3 (0x2) Serial Number: 2 (0x2) - Signature Algorithm: sha1WithRSAEncryption + Signature Algorithm: sha256WithRSAEncryption Issuer: CN=cacert, C=FI, ST=Helsinki, L=Helsinki, O=MariaDB Validity - Not Before: Apr 25 14:55:16 2015 GMT - Not After : Apr 20 14:55:16 2035 GMT + Not Before: Apr 25 20:52:33 2017 GMT + Not After : Apr 20 20:52:33 2037 GMT Subject: C=FI, ST=Helsinki, L=Helsinki, O=MariaDB, CN=server8k Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (8192 bit) Modulus: - 00:e6:24:79:3d:3a:58:6a:12:1c:13:6a:43:d9:c1: - 65:ec:55:c1:4f:7d:fc:f2:a6:56:a5:ab:c5:48:2c: - ce:9c:9d:47:3d:94:93:d5:3b:a3:d0:09:a8:8f:e0: - 4b:36:a0:95:ae:2d:ad:7f:a2:a3:c5:f6:87:80:4c: - 6a:26:15:47:73:20:47:e1:f8:5f:49:b7:13:20:f7: - 32:9f:db:7d:62:41:1b:60:26:7b:41:26:16:0c:92: - 22:ef:b4:2e:b3:db:90:e9:09:ca:fe:1a:1b:e4:f3: - 78:69:7d:ca:6e:c5:39:4b:46:72:09:51:08:40:78: - 8e:04:2a:23:cb:d5:50:cf:96:dd:56:43:10:1b:d2: - 71:28:8f:10:a9:e4:44:1c:39:8a:06:a7:fa:37:48: - bd:5a:dd:37:7f:7a:00:cf:84:8a:48:a3:75:a5:67: - fb:7b:47:2d:26:00:2e:65:ed:63:4e:b7:94:18:3f: - 5a:08:74:54:e5:af:dd:86:1b:34:9b:4c:de:fe:d8: - 69:3d:72:90:c7:5c:83:82:78:ea:ab:06:d4:f1:06: - 20:ad:fd:24:bb:e9:94:e0:c6:32:f5:df:af:b4:14: - 4d:9c:ce:0d:62:3a:3a:2b:11:f9:9e:d7:8e:63:2b: - 57:35:10:7e:d5:44:64:8c:61:9b:4e:f8:e8:a5:fb: - 47:bb:85:33:ea:c6:e0:3d:e2:2c:e8:41:e1:15:e2: - a7:45:23:df:d0:f9:93:01:97:89:95:77:4d:d5:d4: - c1:db:61:ca:e2:84:36:9c:01:9b:33:ec:53:83:2f: - dd:d7:d8:20:c0:6b:4a:73:92:d2:6c:22:a6:a2:68: - 46:8b:aa:3f:aa:fe:47:b7:98:70:fd:ba:59:88:af: - 9f:0d:d7:cb:a0:42:44:f9:f0:54:39:c1:cd:fb:b4: - e4:c6:d6:7e:1d:f5:ed:b9:1a:0f:d7:e6:a2:ab:a2: - 25:1f:6c:f9:ab:9c:d8:a3:b9:da:32:72:51:6d:61: - f1:3b:7d:06:2b:3d:43:d5:52:1f:cb:62:14:53:69: - 4d:91:12:22:f0:55:f9:fc:4f:de:ee:e3:fc:fd:40: - 57:50:eb:0e:7a:45:cc:52:0f:24:6f:45:02:72:6b: - 6d:90:94:1c:d4:fb:34:f6:4b:4d:25:17:6f:df:4a: - 64:f8:ad:1e:6e:df:ad:6c:b3:1d:1a:e6:0e:59:7b: - f8:a7:13:77:78:85:bf:3f:7c:12:d4:8e:34:ff:01: - 90:03:42:85:60:e4:99:d6:19:32:46:41:e8:50:ca: - 2a:03:61:cd:c5:68:f4:92:0f:6e:48:89:41:9e:53: - bc:41:62:ed:4a:92:64:b5:cb:3d:55:6c:d9:87:87: - 9a:ab:fc:22:50:66:92:2b:b9:d7:9e:3b:ed:80:e3: - fa:19:69:38:87:b6:25:3a:db:b5:d0:f2:80:4d:af: - 6d:7d:92:90:de:aa:df:be:80:26:1c:69:ee:7a:e3: - 45:c9:a4:4e:6a:e0:56:5f:1f:61:44:3f:62:34:c9: - 1e:21:5f:f6:7f:68:c0:6e:bf:d2:35:1e:53:99:e4: - e1:bd:64:a4:49:3c:c3:ce:b6:e8:a9:3d:27:54:ea: - f1:3e:a1:fc:7b:bf:8d:71:60:90:c5:66:24:85:de: - 7d:47:1f:62:83:e2:63:8e:10:5c:14:cd:d0:7e:86: - 44:4d:df:05:10:43:b8:3c:87:64:69:ec:ea:fb:49: - 9a:c6:76:c1:8f:ea:49:98:0a:d3:97:af:64:ef:da: - 5f:a9:57:03:e3:a0:15:d8:68:c5:40:d8:7b:0f:26: - 0d:5f:f0:be:5a:4c:fd:af:9e:bf:2d:31:40:71:25: - de:d0:73:19:2d:ae:a6:cb:7c:f0:b8:a4:a9:5c:50: - 80:41:4e:dc:f7:20:a4:a6:66:65:fb:92:d1:43:2d: - bf:30:b2:0d:db:9b:a3:ac:28:08:c4:81:99:0c:0d: - 45:e9:a9:e5:6c:da:bf:10:bb:a7:3e:5a:5f:b7:93: - 4a:20:15:29:69:74:78:d1:eb:53:a8:88:49:cb:de: - 0c:e2:9b:31:e2:2f:56:95:cf:55:92:a3:8e:a9:ef: - 68:cb:00:11:d4:71:06:4b:e5:89:0c:b6:e7:2b:2f: - 98:65:21:8e:2a:a3:86:73:bb:1b:76:e1:94:02:d8: - a1:51:97:15:60:a2:39:d5:fd:dc:a8:be:30:12:44: - b1:49:0b:94:82:cf:5f:93:61:1c:3c:eb:05:5b:a4: - 17:ee:30:cd:7c:db:3f:ee:79:02:da:14:20:98:fd: - 9a:0a:f1:39:c8:59:5b:4e:a3:ad:f8:04:e6:0b:b6: - 81:7e:41:00:af:f7:37:ec:6e:bc:28:a3:3d:76:b6: - fd:e9:88:c7:1e:78:79:27:62:a2:83:34:15:61:b8: - e4:c3:ac:f5:7f:3e:4e:5f:5f:31:5b:e8:91:1b:80: - 5e:cb:74:b2:e5:a3:8a:d5:5a:89:fa:63:f1:ff:67: - bd:59:d0:70:77:b7:75:b5:34:74:3d:2e:99:46:0b: - 4b:c4:64:2f:93:48:fe:47:92:6a:0b:42:5e:ef:c9: - 06:64:84:60:89:2b:84:1f:31:0d:36:15:4e:6b:cd: - 14:f7:a0:d1:b2:b8:ff:53:f5:aa:b9:ed:63:50:7a: - 6f:62:e7:c7:7f:bc:f5:e2:0c:f8:28:a4:0d:ba:75: - d0:b8:c7:9b:e3:94:62:66:1c:d8:6c:02:2e:a5:a2: - 62:50:fe:cf:31 + 00:b3:a7:81:cf:a3:9b:3c:57:2a:8e:de:13:08:a6: + 15:6a:68:08:ce:a1:a0:ef:fc:32:95:5a:9a:b5:cc: + 84:bb:a3:15:a0:fc:29:bb:71:0e:f3:40:20:97:35: + 92:92:89:36:6a:5f:be:a5:24:5a:c8:0e:f7:53:a8: + e0:74:05:90:38:21:a5:25:72:2b:56:70:74:aa:d6: + 30:25:e0:95:dd:3d:4a:48:4b:25:a7:a8:c4:eb:e5: + d6:10:4f:95:42:91:b0:cb:68:2f:bf:96:0f:0a:9b: + 2b:01:0d:03:ae:3f:b7:7e:f6:1e:c1:14:42:04:7f: + 09:21:e8:3e:87:c0:b3:d7:dd:4f:7e:95:b6:83:33: + 8f:cc:f4:bb:11:8e:66:5f:b4:32:22:8b:8e:34:93: + 82:0f:02:d1:6f:85:b2:a7:7e:36:0c:f3:0d:91:46: + f9:a5:91:dc:60:4a:0d:eb:e5:37:e3:76:d8:13:bb: + 55:33:23:4c:b3:90:df:6a:b6:80:3d:f6:bd:9d:bd: + ef:d3:8d:7a:9a:61:ac:02:3d:10:b5:c2:53:d2:7e: + ee:1c:90:a5:b0:e8:db:71:8f:d3:53:e4:80:b0:b6: + cc:b0:f2:eb:46:c3:d9:3c:48:01:1d:3a:5c:d0:f2: + 17:9c:9a:8e:fc:2a:36:e7:1c:59:41:66:ad:e1:d3: + 82:d8:95:ee:ae:dc:ab:72:94:cb:0b:8a:df:ef:70: + f7:f9:e2:77:76:64:67:2d:02:dc:1a:db:02:e9:5e: + 63:3f:3e:07:e9:65:e4:02:78:62:55:e1:52:18:5a: + a3:9c:56:dd:cd:d3:f7:f2:55:45:62:20:ce:fd:41: + 93:b9:c2:20:90:23:a4:41:4f:30:5c:5e:a0:97:6f: + a4:1f:87:6a:97:87:70:f9:d5:56:b7:6a:cc:f1:28: + d2:26:34:57:65:9c:2d:41:43:8b:6c:bd:15:6f:ec: + ce:3e:9f:4d:2f:a6:21:d8:f7:4d:d1:63:4f:f0:9e: + 04:c3:82:33:58:34:20:4b:81:0d:42:0d:44:62:48: + 1b:4c:cf:e0:69:e4:c8:f8:9c:98:1f:74:37:3a:3f: + 23:40:3a:88:80:c2:9c:9b:ba:6a:b9:2c:3a:67:a6: + b6:5c:83:fd:93:3f:6e:53:19:48:e3:3d:fa:f5:8e: + 03:40:8f:e6:af:c5:de:aa:f8:70:77:36:2b:6e:3a: + df:bc:d7:f6:c7:4f:02:7f:a1:ad:2f:3c:e9:08:45: + 2f:57:1a:f1:5b:9c:27:07:3a:44:43:3e:05:b1:4a: + 1b:42:a4:90:ab:ce:4b:f7:c1:9b:d7:dd:6f:05:11: + f9:e8:a3:a1:99:6c:2e:27:3d:b1:54:6f:eb:b2:e8: + f8:e6:12:6f:8a:69:05:cc:c6:f6:c4:2d:b0:06:b6: + 21:50:28:50:dc:8c:b2:8c:65:ce:d9:2b:51:b8:62: + 56:70:b4:46:4d:e4:4c:a1:86:31:3d:e3:07:7f:73: + e3:09:6e:ee:a7:29:2a:80:23:8a:b6:0c:22:9e:43: + 1c:a9:8c:47:2c:02:6d:13:ef:88:a7:d7:d1:60:d9: + 6b:28:80:97:8a:b7:ff:62:6b:28:54:0d:ab:63:3a: + ac:57:9e:2f:95:2b:f4:a0:3d:a1:60:a0:57:1a:09: + 23:ee:fc:4d:9b:45:83:f2:0a:1f:b3:77:0e:98:20: + b3:16:fc:97:03:30:fc:ab:3e:7e:fc:3e:76:64:cb: + 4b:4d:c9:6c:8f:aa:70:ec:03:c5:81:31:43:11:9a: + fc:2d:ba:25:94:0b:73:7e:c1:39:01:ca:68:70:1a: + ef:a4:ba:dc:13:d6:37:17:09:0d:ca:aa:21:94:bb: + 3e:43:fd:8c:80:e6:58:fd:75:23:5a:ca:1f:0b:2e: + 35:c2:cd:77:0c:73:b3:2f:9a:e4:03:68:58:c8:3f: + f3:76:8d:05:53:d2:34:db:09:af:70:ed:67:0b:ff: + 2f:64:35:2d:71:22:7a:f2:f2:2b:7d:6d:be:80:bd: + 18:38:37:01:67:a4:5f:48:29:52:78:9f:b5:b8:80: + 70:ca:10:6a:e1:27:3b:c0:61:4e:96:f8:e8:0f:42: + b1:fa:bc:05:ef:31:91:38:ab:0f:a7:ef:2c:86:c1: + ab:85:36:7c:10:9d:c7:0a:6b:e6:2b:11:dd:fa:45: + e7:d2:c3:3a:c1:cd:60:27:68:26:d7:87:49:5f:e3: + 7f:26:20:1f:99:a8:f9:00:a6:0d:d5:13:81:6e:fc: + 27:8a:03:58:6b:0e:3a:c7:20:a1:5c:24:d1:08:a6: + 5a:1e:e7:81:3d:21:05:2e:4b:5d:18:14:f1:4f:ab: + 5c:e5:81:da:c1:2c:f6:12:ff:25:09:97:1e:9d:51: + 6d:1f:63:2e:9c:ed:7d:86:45:b0:35:73:f6:9f:82: + 65:bc:7f:46:88:e1:a1:60:a1:37:50:26:9c:d7:76: + f3:dc:94:89:6d:d7:be:a6:17:a1:2b:71:14:37:17: + 5f:ed:5a:39:99:fc:69:af:bb:63:55:c6:60:7b:64: + d8:bb:b7:4a:9f:b9:aa:89:4b:30:1f:9c:ef:23:7a: + af:7e:31:dd:fb:dd:0a:d5:04:0a:8c:57:6f:64:80: + eb:f3:ee:6c:33:d2:72:c2:24:68:a2:4e:99:88:1c: + f7:3b:5d:10:cc:4e:ea:a5:cb:00:40:8a:a9:63:2e: + 18:92:eb:b1:28:05:fe:19:ea:7b:32:fb:63:56:2c: + 0b:20:01:92:a3 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Basic Constraints: @@ -88,61 +88,61 @@ Certificate: Netscape Comment: OpenSSL Generated Certificate X509v3 Subject Key Identifier: - B3:A9:00:04:C7:9C:2B:CD:C0:5F:D1:28:5C:5C:C6:1F:26:F7:17:5A + 43:3B:26:15:A5:60:62:3F:07:2A:8C:8C:DA:F8:74:BE:36:B2:33:E4 X509v3 Authority Key Identifier: - keyid:C7:2C:01:95:1A:F5:3E:CD:04:A6:24:35:35:04:D9:A7:16:01:2A:79 + keyid:1C:C7:2B:AA:1B:B1:BB:2E:9A:F4:0F:B1:86:60:57:38:C2:41:05:12 - Signature Algorithm: sha1WithRSAEncryption - 01:43:2c:d5:1d:b5:36:05:31:ca:b2:f3:9e:71:ce:62:fe:b2: - e5:e3:2d:30:23:99:51:3d:50:ee:d6:ce:76:e8:43:8d:5d:9d: - 28:9b:43:90:0c:d4:24:54:3c:53:3c:18:92:ec:93:86:87:7a: - d6:7f:5d:3b:56:cd:9d:96:7a:06:c8:16:19:8b:ed:c8:21:cf: - 15:b9:fb:06:7d:cb:5c:46:9c:c5:2e:8e:de:77:21:e5:9d:35: - 95:00:c7:ad:d2:57:36:65:1a:43:6c:ee:75:ad:a2:d8:c0:60: - d5:07:d7:5c:5d:8e:ae:af:4a:e6:fa:6a:13:78:98:b8:11:c7: - dc:d2:a7:dd:29:b5:76:fa:ef:13:62:5f:9a:0d:e2:87:6a:04: - 3c:5f:72:44:d0:d0:7a:70:c6:09:2b:bf:92:91:6d:f4:2e:53: - f3:b4:c5:23:61:d6:87:c1:30:ef:fe:92:6f:97:78:f2:ea:3c: - ff:e8:54:3a:9f:49:ac:a9:2b:46:c3:76:b1:f9:eb:31:a3:4f: - 40:58:16:90:77:b6:3d:6f:85:95:12:a9:ca:70:0a:9a:cd:61: - 46:27:84:ce:9e:33:54:8f:9a:41:6d:4d:11:bd:14:7f:ff:32: - e9:06:bc:36:38:11:5c:0f:e9:a5:5a:0f:5c:7c:fa:f5:73:5b: - 4c:47:f2:f2 + Signature Algorithm: sha256WithRSAEncryption + 5c:c7:4e:56:eb:21:1b:f3:5d:c4:56:fa:bf:6d:e1:0d:fa:cc: + b0:7c:13:58:b9:d4:47:b3:de:b4:ae:f2:45:02:88:65:43:89: + cf:ea:9c:d0:ea:45:42:35:d2:ec:bc:b4:73:03:db:85:7b:c7: + f0:68:d7:dc:2b:70:71:63:ed:33:68:9f:29:ff:67:0a:69:3b: + c4:65:7c:25:00:cf:7f:ee:aa:fe:44:19:70:f2:74:db:da:9f: + 52:92:1a:03:e6:0a:49:85:2e:2a:02:c1:81:d4:6d:cf:98:d1: + e9:6a:2f:bb:fa:a6:d5:4b:55:38:c2:aa:8c:f5:d3:f9:e6:74: + db:00:d6:29:f9:d1:9c:7a:0e:98:c5:e5:8c:16:49:d8:cf:ee: + a0:cb:c2:2c:31:cf:62:2c:b1:7f:6d:60:b0:ce:d5:07:45:e8: + 44:17:7c:c4:12:fb:00:57:24:57:e8:17:78:3b:1e:0d:91:aa: + 67:98:d0:e5:9a:86:4b:88:4a:2f:55:5d:2e:13:ec:f5:4a:7d: + e7:13:a7:57:03:41:2f:f6:6c:08:8e:58:ef:b0:7a:79:32:e3: + bb:4b:eb:4d:42:cd:42:96:2d:67:f6:4c:c1:f6:62:fb:c0:3e: + 09:69:8f:36:7e:fa:c8:cd:ff:72:3e:df:92:f7:8f:44:cf:77: + fe:6e:74:de -----BEGIN CERTIFICATE----- -MIIGpDCCBYygAwIBAgIBAjANBgkqhkiG9w0BAQUFADBWMQ8wDQYDVQQDDAZjYWNl +MIIGpDCCBYygAwIBAgIBAjANBgkqhkiG9w0BAQsFADBWMQ8wDQYDVQQDDAZjYWNl cnQxCzAJBgNVBAYTAkZJMREwDwYDVQQIDAhIZWxzaW5raTERMA8GA1UEBwwISGVs -c2lua2kxEDAOBgNVBAoMB01hcmlhREIwHhcNMTUwNDI1MTQ1NTE2WhcNMzUwNDIw -MTQ1NTE2WjBYMQswCQYDVQQGEwJGSTERMA8GA1UECAwISGVsc2lua2kxETAPBgNV +c2lua2kxEDAOBgNVBAoMB01hcmlhREIwHhcNMTcwNDI1MjA1MjMzWhcNMzcwNDIw +MjA1MjMzWjBYMQswCQYDVQQGEwJGSTERMA8GA1UECAwISGVsc2lua2kxETAPBgNV BAcMCEhlbHNpbmtpMRAwDgYDVQQKDAdNYXJpYURCMREwDwYDVQQDDAhzZXJ2ZXI4 -azCCBCIwDQYJKoZIhvcNAQEBBQADggQPADCCBAoCggQBAOYkeT06WGoSHBNqQ9nB -ZexVwU99/PKmVqWrxUgszpydRz2Uk9U7o9AJqI/gSzagla4trX+io8X2h4BMaiYV -R3MgR+H4X0m3EyD3Mp/bfWJBG2Ame0EmFgySIu+0LrPbkOkJyv4aG+TzeGl9ym7F -OUtGcglRCEB4jgQqI8vVUM+W3VZDEBvScSiPEKnkRBw5igan+jdIvVrdN396AM+E -ikijdaVn+3tHLSYALmXtY063lBg/Wgh0VOWv3YYbNJtM3v7YaT1ykMdcg4J46qsG -1PEGIK39JLvplODGMvXfr7QUTZzODWI6OisR+Z7XjmMrVzUQftVEZIxhm0746KX7 -R7uFM+rG4D3iLOhB4RXip0Uj39D5kwGXiZV3TdXUwdthyuKENpwBmzPsU4Mv3dfY -IMBrSnOS0mwipqJoRouqP6r+R7eYcP26WYivnw3Xy6BCRPnwVDnBzfu05MbWfh31 -7bkaD9fmoquiJR9s+auc2KO52jJyUW1h8Tt9Bis9Q9VSH8tiFFNpTZESIvBV+fxP -3u7j/P1AV1DrDnpFzFIPJG9FAnJrbZCUHNT7NPZLTSUXb99KZPitHm7frWyzHRrm -Dll7+KcTd3iFvz98EtSONP8BkANChWDkmdYZMkZB6FDKKgNhzcVo9JIPbkiJQZ5T -vEFi7UqSZLXLPVVs2YeHmqv8IlBmkiu515477YDj+hlpOIe2JTrbtdDygE2vbX2S -kN6q376AJhxp7nrjRcmkTmrgVl8fYUQ/YjTJHiFf9n9owG6/0jUeU5nk4b1kpEk8 -w8626Kk9J1Tq8T6h/Hu/jXFgkMVmJIXefUcfYoPiY44QXBTN0H6GRE3fBRBDuDyH -ZGns6vtJmsZ2wY/qSZgK05evZO/aX6lXA+OgFdhoxUDYew8mDV/wvlpM/a+evy0x -QHEl3tBzGS2upst88LikqVxQgEFO3PcgpKZmZfuS0UMtvzCyDdubo6woCMSBmQwN -Remp5WzavxC7pz5aX7eTSiAVKWl0eNHrU6iIScveDOKbMeIvVpXPVZKjjqnvaMsA -EdRxBkvliQy25ysvmGUhjiqjhnO7G3bhlALYoVGXFWCiOdX93Ki+MBJEsUkLlILP -X5NhHDzrBVukF+4wzXzbP+55AtoUIJj9mgrxOchZW06jrfgE5gu2gX5BAK/3N+xu -vCijPXa2/emIxx54eSdiooM0FWG45MOs9X8+Tl9fMVvokRuAXst0suWjitVaifpj -8f9nvVnQcHe3dbU0dD0umUYLS8RkL5NI/keSagtCXu/JBmSEYIkrhB8xDTYVTmvN -FPeg0bK4/1P1qrntY1B6b2Lnx3+89eIM+CikDbp10LjHm+OUYmYc2GwCLqWiYlD+ -zzECAwEAAaN7MHkwCQYDVR0TBAIwADAsBglghkgBhvhCAQ0EHxYdT3BlblNTTCBH -ZW5lcmF0ZWQgQ2VydGlmaWNhdGUwHQYDVR0OBBYEFLOpAATHnCvNwF/RKFxcxh8m -9xdaMB8GA1UdIwQYMBaAFMcsAZUa9T7NBKYkNTUE2acWASp5MA0GCSqGSIb3DQEB -BQUAA4IBAQABQyzVHbU2BTHKsvOecc5i/rLl4y0wI5lRPVDu1s526EONXZ0om0OQ -DNQkVDxTPBiS7JOGh3rWf107Vs2dlnoGyBYZi+3IIc8VufsGfctcRpzFLo7edyHl -nTWVAMet0lc2ZRpDbO51raLYwGDVB9dcXY6ur0rm+moTeJi4Ecfc0qfdKbV2+u8T -Yl+aDeKHagQ8X3JE0NB6cMYJK7+SkW30LlPztMUjYdaHwTDv/pJvl3jy6jz/6FQ6 -n0msqStGw3ax+esxo09AWBaQd7Y9b4WVEqnKcAqazWFGJ4TOnjNUj5pBbU0RvRR/ -/zLpBrw2OBFcD+mlWg9cfPr1c1tMR/Ly +azCCBCIwDQYJKoZIhvcNAQEBBQADggQPADCCBAoCggQBALOngc+jmzxXKo7eEwim +FWpoCM6hoO/8MpVamrXMhLujFaD8KbtxDvNAIJc1kpKJNmpfvqUkWsgO91Oo4HQF +kDghpSVyK1ZwdKrWMCXgld09SkhLJaeoxOvl1hBPlUKRsMtoL7+WDwqbKwENA64/ +t372HsEUQgR/CSHoPofAs9fdT36VtoMzj8z0uxGOZl+0MiKLjjSTgg8C0W+Fsqd+ +NgzzDZFG+aWR3GBKDevlN+N22BO7VTMjTLOQ32q2gD32vZ2979ONepphrAI9ELXC +U9J+7hyQpbDo23GP01PkgLC2zLDy60bD2TxIAR06XNDyF5yajvwqNuccWUFmreHT +gtiV7q7cq3KUywuK3+9w9/nid3ZkZy0C3BrbAuleYz8+B+ll5AJ4YlXhUhhao5xW +3c3T9/JVRWIgzv1Bk7nCIJAjpEFPMFxeoJdvpB+HapeHcPnVVrdqzPEo0iY0V2Wc +LUFDi2y9FW/szj6fTS+mIdj3TdFjT/CeBMOCM1g0IEuBDUINRGJIG0zP4GnkyPic +mB90Nzo/I0A6iIDCnJu6arksOmemtlyD/ZM/blMZSOM9+vWOA0CP5q/F3qr4cHc2 +K24637zX9sdPAn+hrS886QhFL1ca8VucJwc6REM+BbFKG0KkkKvOS/fBm9fdbwUR ++eijoZlsLic9sVRv67Lo+OYSb4ppBczG9sQtsAa2IVAoUNyMsoxlztkrUbhiVnC0 +Rk3kTKGGMT3jB39z4wlu7qcpKoAjirYMIp5DHKmMRywCbRPviKfX0WDZayiAl4q3 +/2JrKFQNq2M6rFeeL5Ur9KA9oWCgVxoJI+78TZtFg/IKH7N3Dpggsxb8lwMw/Ks+ +fvw+dmTLS03JbI+qcOwDxYExQxGa/C26JZQLc37BOQHKaHAa76S63BPWNxcJDcqq +IZS7PkP9jIDmWP11I1rKHwsuNcLNdwxzsy+a5ANoWMg/83aNBVPSNNsJr3DtZwv/ +L2Q1LXEievLyK31tvoC9GDg3AWekX0gpUniftbiAcMoQauEnO8BhTpb46A9Csfq8 +Be8xkTirD6fvLIbBq4U2fBCdxwpr5isR3fpF59LDOsHNYCdoJteHSV/jfyYgH5mo ++QCmDdUTgW78J4oDWGsOOscgoVwk0QimWh7ngT0hBS5LXRgU8U+rXOWB2sEs9hL/ +JQmXHp1RbR9jLpztfYZFsDVz9p+CZbx/RojhoWChN1AmnNd289yUiW3XvqYXoStx +FDcXX+1aOZn8aa+7Y1XGYHtk2Lu3Sp+5qolLMB+c7yN6r34x3fvdCtUECoxXb2SA +6/PubDPScsIkaKJOmYgc9ztdEMxO6qXLAECKqWMuGJLrsSgF/hnqezL7Y1YsCyAB +kqMCAwEAAaN7MHkwCQYDVR0TBAIwADAsBglghkgBhvhCAQ0EHxYdT3BlblNTTCBH +ZW5lcmF0ZWQgQ2VydGlmaWNhdGUwHQYDVR0OBBYEFEM7JhWlYGI/ByqMjNr4dL42 +sjPkMB8GA1UdIwQYMBaAFBzHK6obsbsumvQPsYZgVzjCQQUSMA0GCSqGSIb3DQEB +CwUAA4IBAQBcx05W6yEb813EVvq/beEN+sywfBNYudRHs960rvJFAohlQ4nP6pzQ +6kVCNdLsvLRzA9uFe8fwaNfcK3BxY+0zaJ8p/2cKaTvEZXwlAM9/7qr+RBlw8nTb +2p9SkhoD5gpJhS4qAsGB1G3PmNHpai+7+qbVS1U4wqqM9dP55nTbANYp+dGceg6Y +xeWMFknYz+6gy8IsMc9iLLF/bWCwztUHRehEF3zEEvsAVyRX6Bd4Ox4NkapnmNDl +moZLiEovVV0uE+z1Sn3nE6dXA0Ev9mwIjljvsHp5MuO7S+tNQs1Cli1n9kzB9mL7 +wD4JaY82fvrIzf9yPt+S949Ez3f+bnTe -----END CERTIFICATE----- diff --git a/mysql-test/std_data/server8k-key.pem b/mysql-test/std_data/server8k-key.pem index 72d2756477c..a383b359275 100644 --- a/mysql-test/std_data/server8k-key.pem +++ b/mysql-test/std_data/server8k-key.pem @@ -1,99 +1,99 @@ -----BEGIN RSA PRIVATE KEY----- -MIISKgIBAAKCBAEA5iR5PTpYahIcE2pD2cFl7FXBT3388qZWpavFSCzOnJ1HPZST -1Tuj0Amoj+BLNqCVri2tf6KjxfaHgExqJhVHcyBH4fhfSbcTIPcyn9t9YkEbYCZ7 -QSYWDJIi77Qus9uQ6QnK/hob5PN4aX3KbsU5S0ZyCVEIQHiOBCojy9VQz5bdVkMQ -G9JxKI8QqeREHDmKBqf6N0i9Wt03f3oAz4SKSKN1pWf7e0ctJgAuZe1jTreUGD9a -CHRU5a/dhhs0m0ze/thpPXKQx1yDgnjqqwbU8QYgrf0ku+mU4MYy9d+vtBRNnM4N -Yjo6KxH5nteOYytXNRB+1URkjGGbTvjopftHu4Uz6sbgPeIs6EHhFeKnRSPf0PmT -AZeJlXdN1dTB22HK4oQ2nAGbM+xTgy/d19ggwGtKc5LSbCKmomhGi6o/qv5Ht5hw -/bpZiK+fDdfLoEJE+fBUOcHN+7TkxtZ+HfXtuRoP1+aiq6IlH2z5q5zYo7naMnJR -bWHxO30GKz1D1VIfy2IUU2lNkRIi8FX5/E/e7uP8/UBXUOsOekXMUg8kb0UCcmtt -kJQc1Ps09ktNJRdv30pk+K0ebt+tbLMdGuYOWXv4pxN3eIW/P3wS1I40/wGQA0KF -YOSZ1hkyRkHoUMoqA2HNxWj0kg9uSIlBnlO8QWLtSpJktcs9VWzZh4eaq/wiUGaS -K7nXnjvtgOP6GWk4h7YlOtu10PKATa9tfZKQ3qrfvoAmHGnueuNFyaROauBWXx9h -RD9iNMkeIV/2f2jAbr/SNR5TmeThvWSkSTzDzrboqT0nVOrxPqH8e7+NcWCQxWYk -hd59Rx9ig+JjjhBcFM3QfoZETd8FEEO4PIdkaezq+0maxnbBj+pJmArTl69k79pf -qVcD46AV2GjFQNh7DyYNX/C+Wkz9r56/LTFAcSXe0HMZLa6my3zwuKSpXFCAQU7c -9yCkpmZl+5LRQy2/MLIN25ujrCgIxIGZDA1F6anlbNq/ELunPlpft5NKIBUpaXR4 -0etTqIhJy94M4psx4i9Wlc9VkqOOqe9oywAR1HEGS+WJDLbnKy+YZSGOKqOGc7sb -duGUAtihUZcVYKI51f3cqL4wEkSxSQuUgs9fk2EcPOsFW6QX7jDNfNs/7nkC2hQg -mP2aCvE5yFlbTqOt+ATmC7aBfkEAr/c37G68KKM9drb96YjHHnh5J2KigzQVYbjk -w6z1fz5OX18xW+iRG4Bey3Sy5aOK1VqJ+mPx/2e9WdBwd7d1tTR0PS6ZRgtLxGQv -k0j+R5JqC0Je78kGZIRgiSuEHzENNhVOa80U96DRsrj/U/Wque1jUHpvYufHf7z1 -4gz4KKQNunXQuMeb45RiZhzYbAIupaJiUP7PMQIDAQABAoIEAQDdg63OaSJAtj2f -0mCMb8ISwFfYk4Osar5rp/Gzjq0vwZKYizHfxA/gZeuA0HqUkeyAQicE+x53pNq3 -etWQ4lprTV7i+ZV99mDLEiQACdudft1Cpsdr5aTDZMWLwvpQ072fEHX6Llc6/72e -jB0UkXCcK6oHnZ87rs3C5Gyf/SpTJPrV1KbkoKGaUFnRrIyCPj/EOFp2a+UWWGba -pCuzkfcoA21xT6yW8+NY+EOwh9VWJwy8af3WtWIh0ix+sCDqegsJcHObWXJQ8ZMD -Oi9lfqXnd+ZskYOR+zn5P8w9LJiJ1CEAFLR9H15tpleFtCSw/z5pLP9ndvTwyeIb -GSmU2VAqBgP6roGYDuL6iq6Dyi0GN4luM0pz9c/PtY2Ni8MrzeziKjAF6OXiDE41 -rxTwdG7RxnNa2q7+tjt9hrCgLbYqd6W/6VKYZY1YKvIE+PhdwGtzxwbKsOf6phqt -6DZr0BKIX7VwMeXRHbhtjw6hR86diH2koLQPfH4crDIL3GQ2J13C/RxjTLAEOli2 -y4paeOYzOe6ANDXXBOhieFw5f6mqD9pTF2PWDTnTJhfMGJ6gXPapDYoycS8huNnM -DnqtMVIOf2zOI8aSkumJyXq2bEpGSNK80IJgyKfpk0b4qOdbG7tklfbhVCxTida0 -qqgmJKLGrNmJNW8KLB1U4/hcowEVFz8MF0OZAnByDHi2axwSRSGD5aG2yhCfs3Sy -C269M25v0WuTeOs02RnHUvLPIhvbOEVTu6AjvAtYIfjEq+xqrjbZqUqJeh/xo2tF -SohWe5aQs5DntsMCkZWlqlXc4CCaCqVACegh5Rn+8JbpZvpzy4HUtThfVbB45a5P -6uRAsoxiIHbPLyIaKHFqqfog98hzD4TZYKpfb4RCgqKgEAHGKh4/KPM9O4iw67Oj -EnT2WduegazemzOxQIuZ1UcNu/2d7uy8v/zPGLEUfqhFLS8PMStEuLdQQZfU2DFM -rHJZsH3UtgSreZuQnlTHYy073UhB44mhcYbwa03jMkLHb1bt9q9l8ZAZ5FjstKJD -vN5hMqT5LBQtvArCB+aIVDQxEyZiqFxBlW658uAIhCkLXNB5BVfEUEB8w5vZXOPn -UpNyM8v0dYOkiSOH6+Kk0i7eC8CSrSQAaLnuFZxepGCGMFksywMv7mDg6yX1sAkE -RMakKaAaZ55QpUv1/heaBVK0xmwpjJBefAoMCyFh/I+o7BkFdWGsGF2Tc5ZxZB3F -DF75An4WV2qFMNEIb2SGBBUHGoG75C7tRC9ll39SqX7X2pXMaHvJvEx76CmhC9ub -TzkRDV0Zi/GqiBQXz5kn4AP3GCuJUQJIvK2l0HPqbqNULrhahwQWBOpIek/SLBFV -3fRQc5ABAoICAQDzqaS5/JEKUGMzVikWYnxl6fXgMp/l3Hq2DyifRqgYVoSr/wOJ -PTOOJrtuvwHK6kbOv4YRfKwgVWlJgoAP9Tz0GGpE7toSk3biHY4tTF4KR3xIIOnI -LQLNDR0LDrJSjQhAH5z8U/E+dpOCBHLfXodZ0/ZuXx9YS/pNDQBW/+HOmDtCfqpb -kLmxjZcvZnAZyrlSjGrHJoPjiFrVA/1xvY68TNWHWu8EXphKO/aBi5/e4wWS7I8q -0lDjjPB3GIm0rhumowoGYDPtchcTIvip9aGSPsw56elV+PiwbZaPHZXwHeI0Q25j -UAdkmkwO4VaYGeR4JSUTjoOwWdXXediRq9SKRwhOoat3GJdDwYByYgZb8MKdQCZV -H9hD0rEF4JOrUnOsCteXMNEczQocfSxduLD7XLie+oXoEggNE/hD9EJ397sOvyUg -rgmJLBPCbIPdnfCwm9rdLMsWUjvd4JFq1dyQ9QM1LkdOGQdSoUixtN8X75BdkWhL -/LUqVV/L5/1DStYniO6k4ry67u4POisAZa4czPAJrBEOakVxkCIoaH+mZ/J3GxWc -DvnV6myvcNXUEENhfMt2mikP5zcPthq1O1nm7RFqiow4s/f0jc6Epr/LToXwhm8j -5ch0TkcM+ioguCyYXWEzRJUt3GBWpHoSvxqKLxaWbo1mtxHpGxHDIIFAoQKCAgEA -8cuU0H0OE3JJaG+9Hp6q5mJfdrAzeIPTCT1jxVDptZaplMVo8X0+LIKYpzWqbGWa -Z3cTmReOZBukuKPKXuZEufYqzwIh7lEv1ls2SJ8Gedf3Dq3C2fCITHzXlsg58mrf -tD6cKE85EmI/GhXtaBc51Qa60w5MyXfzK3EPi8lD1v5Il5K1MG6sv+7+P1LgAQVj -c9mPVT1WmgBFVG2/syJ9L94smwhrtM2NhxMkMtEBrOiv0Wy0bEb1wIo5ndBGBoeF -KXBfz87HDbUCSxItDFZ5mDk2ARnzl/13+wr81aKcxW2LyQ0eiiN+HczmM1xpavU6 -KI/QnIvUNqW9XRSNpz/w+DO9+gs9BPIODMww4KpLVtgFfy5KYre1/MpgZHVlHDCI -r72WaDFHdMs75JN1rPNnYx81LtKp0JnE2bIDda3qlCKTBtwaFEUg1F0v9NK9yiZF -JytwUUIfURayT34hfDAJDaSemWT3b4U2kYBr7Fp+g6R5iAosx/TnCYxtuGhdSLA7 -JkL4gy6Ll6hQkIyxRxC6R38wzwUFg4pbV6qrhjP1qyuU1BsZrBSV1RSbcHghSAZj -UmT/sCKOhnliptnnz3e6ceP8I9Y2/TOeJXPdH6zvfaTpnVLiZC48FBKnXFCM4SaS -v5qejQChBBOyd9ybO40sxC/Db8WsPa1LCMnHzm9YtJECggIBAOitGIYZ7o7A0NnW -7wlXJp9TWN2suhAQCIYN16axHBeRlsN/k2rcLH0T82v/rcCNEuye63GLcfq2VQJD -QfL7JieZoPEOfOO8OYwW2QvcnviF5uIhu9zRTplsak2CzToTPsus5IX3yqjxJk7D -3MX8SJe41tvyrcXtV6l4PtjVJFXZ/My0lA04GHq6UKKM2YNnmKyBNf3Sg6MlNEex -tbWFY+XgIQ9OAGxVwGsy82QlZpa6hzoT92rWxzNkBMRVhDorAHCRxCLufufBSNOX -U4lI6+7xVKgA/DaCt0gSffFQOTu/7aAmvCTN6SFiTjrvZ3d7UjaAkfUo5Avk3VUU -H4CLazxYNFCSBWCG9PtYrkWL4bKuJ8lt1qz26ddqPGz2VU0GQTsKZ3ESxr0P9xXZ -WiJtGWQO2KXHaVOA1HtYlHyfpVFWGSQt11BroOOCWaQRZ6KbDZo4WjlWauB3yn9H -0NUKP8OUlGmWNbUYJWv8Y6R+qYL72wnero/RU1aHjCD0+V3m+rVqBykg80Q2oNGd -pC6O6kLKWHWEAA2Z4ZJeJqAbmbnYDSs7i9koW96wKvUldKs6lH1lZA1vjMKI8oVo -SfzCHklsHHN2tgv3n1HCkHEPopqL808JsNvAsziM/0AfK1dvR/Z/RTBlz2apyL0z -9vYr0zYdXR5tl018h/fnR4Dja0nBAoICAG7z5Ui6adAgnHDQ/W5kjrWDJJ05A5Vk -oF1YD0VG+Md76Ds9TwepWfNgxh3McXSmNvENYi/Y8F0dljun5UAg2B6hSEmPh/Rs -ys/JMSKn3nfg0xyoPYZ2aoT2sJMfM//6bMawCEYy/FRkQuJ8d2FRbxdCrE4W/MNi -SHKBwgl6BKhOLQ9oob4kux6j04VJSUMOrvFFPxF1QR0PzMCOT+4qgFmL0NofRhVE -UZJPBUhQQj2O1Mb749rfwPtmMkQpg0iRvVgjmcOo9gxjKDc8kvFbaRaiAhcFSpNP -G1RWWWjK4rdxqjhKzd3DpaaVFHkzCair+NEOFUMQowOkkD17Jfk+TRwH2kA4EfsH -+8yBklg1rLagD/2pRzb/aAIk95CBkjEaUE6cZ71OB6UfcU9VHyWK18FTW7strgDr -3rmqhw3kDYWGTANU0p7EjekFDfGBFtgBMbBkrXiMKAS0roPts/3hqPXauXBYNXa2 -ITEpRdzCNdXAjogPazciOTZXey2ork2hmLLY/mJr5GBmOvnppZufoGsYgVQt0Lbo -sTuMNIMzqqKuL/AYs3IUTCDoAN/eMB911vPDJtUzNkWcw9kemtHH9gU3AceMD9Wm -bwZuoWRcndUA7LZlhz9DPAxV9vhUeiEC60oC8IhxZnGRfE6mK5FrGcS4yiyq/3uZ -3pmcxhQDYigRAoICAQCzK0SUG9tSzwpyuEb7hbS7isT0XOnVvLDEXL2jTwVqbN8a -pC9jebbzJgxk5EA+np/eZlRU7h4aZjRhtDwi4HsarXzgIni6z1vswAiOSZRCfC17 -8/tjfER9w3pJMRTJ5MFeN7JvNUhWnXY5fX6SBl/Eocwwy8S/eIWUY53aI2k8UZMR -eLUb9I6eL21qZAPp/XD8/3pF5DH0ECOS96C8mDKTGnARUyfkKrlaASSTfe5XToNb -IpDEYtLwpHN0missq+ObqPc5lwot/nBy2fdllksb3mCCK8AxuKEDsVtWbb2maoa9 -lRGxIxr++ZV8n0dAwGCV18CbhBYh/B2YnrexI2ZfYOQbzPjHiuUgSLGBJvIUX+S+ -O8ij0vQBqECfwCq63SrdW340Huob52d/cod9pNolO5edftPNJVJP+44sZtEJoDBW -WgzdpVV9fA8TYsO6B3Dci960fbpWnKZ01uLFWVTkou8IwqcaxJIyMzS3Gmqo4UEF -oxBSrg9rTAMJSnm2YWBHK1Z8DDlLzw3SkNZVKm6Ai4yEcLs1PzGZejsJvsu3zGgq -nM4liRHK2QqY1SJf6PsJGxAI62tlyTRfjxtC3Jq9oJbmJs+jl0gtNKEbkfQc83IL -8wCHOm9G7KK7XJjMCb50Dh80YIDFj5GTQLU1tKdpDBfq6YTKUHqjPA8GRnz8vA== +MIISKAIBAAKCBAEAs6eBz6ObPFcqjt4TCKYVamgIzqGg7/wylVqatcyEu6MVoPwp +u3EO80AglzWSkok2al++pSRayA73U6jgdAWQOCGlJXIrVnB0qtYwJeCV3T1KSEsl +p6jE6+XWEE+VQpGwy2gvv5YPCpsrAQ0Drj+3fvYewRRCBH8JIeg+h8Cz191PfpW2 +gzOPzPS7EY5mX7QyIouONJOCDwLRb4Wyp342DPMNkUb5pZHcYEoN6+U343bYE7tV +MyNMs5DfaraAPfa9nb3v0416mmGsAj0QtcJT0n7uHJClsOjbcY/TU+SAsLbMsPLr +RsPZPEgBHTpc0PIXnJqO/Co25xxZQWat4dOC2JXurtyrcpTLC4rf73D3+eJ3dmRn +LQLcGtsC6V5jPz4H6WXkAnhiVeFSGFqjnFbdzdP38lVFYiDO/UGTucIgkCOkQU8w +XF6gl2+kH4dql4dw+dVWt2rM8SjSJjRXZZwtQUOLbL0Vb+zOPp9NL6Yh2PdN0WNP +8J4Ew4IzWDQgS4ENQg1EYkgbTM/gaeTI+JyYH3Q3Oj8jQDqIgMKcm7pquSw6Z6a2 +XIP9kz9uUxlI4z369Y4DQI/mr8XeqvhwdzYrbjrfvNf2x08Cf6GtLzzpCEUvVxrx +W5wnBzpEQz4FsUobQqSQq85L98Gb191vBRH56KOhmWwuJz2xVG/rsuj45hJvimkF +zMb2xC2wBrYhUChQ3IyyjGXO2StRuGJWcLRGTeRMoYYxPeMHf3PjCW7upykqgCOK +tgwinkMcqYxHLAJtE++Ip9fRYNlrKICXirf/YmsoVA2rYzqsV54vlSv0oD2hYKBX +Ggkj7vxNm0WD8gofs3cOmCCzFvyXAzD8qz5+/D52ZMtLTclsj6pw7APFgTFDEZr8 +LbollAtzfsE5AcpocBrvpLrcE9Y3FwkNyqohlLs+Q/2MgOZY/XUjWsofCy41ws13 +DHOzL5rkA2hYyD/zdo0FU9I02wmvcO1nC/8vZDUtcSJ68vIrfW2+gL0YODcBZ6Rf +SClSeJ+1uIBwyhBq4Sc7wGFOlvjoD0Kx+rwF7zGROKsPp+8shsGrhTZ8EJ3HCmvm +KxHd+kXn0sM6wc1gJ2gm14dJX+N/JiAfmaj5AKYN1ROBbvwnigNYaw46xyChXCTR +CKZaHueBPSEFLktdGBTxT6tc5YHawSz2Ev8lCZcenVFtH2MunO19hkWwNXP2n4Jl +vH9GiOGhYKE3UCac13bz3JSJbde+phehK3EUNxdf7Vo5mfxpr7tjVcZge2TYu7dK +n7mqiUswH5zvI3qvfjHd+90K1QQKjFdvZIDr8+5sM9JywiRook6ZiBz3O10QzE7q +pcsAQIqpYy4YkuuxKAX+Gep7MvtjViwLIAGSowIDAQABAoIEAG18dDwatQx9As62 +wFrQ/NQwOs3S6sXWqO0knoyU639+0Duf8b7uE/Ji7nm4iG8NeEAzwXfbRAgQNuKh +VMjgaxgKSR8dCRRQzIkgp48t46dFJvQP+A7JZ9lr5J1kIs7DAz+zawYTaW9JSVgG +mmujIS0ayXtZ4THkSaiEZTdOVUwE7+FxVB15WvXJYAg/BFzm8HblHfEkxGppiUGx +9ULmRAJ4TZhv7Mzeq7Ny7jEJLJpPXzgHPtE/fgshC0d5mYYpzT1I99OxsTpMtAtZ +M8nReEwcVvjJnnDGOJZo1J2X8GXBwdVE9eXWjZmTODqbytz77S/fcaxz/amzOSIG +dYne4SsllKS0jzfxPhz2saq26ik7vxpD7i0cRQ+I0alvgISWRTQUiHSI8UTtYJ6+ +/l7VAa/isoTZOsOmwv7M7D6U9EtjrUKwBAUHFk+0Fg2w9WELhep7/bEiQWvnOMtX +sruWBm5zv5CxW1MuujKn2t5jHNWhQaeOIrW9V6Hew3PJsLTTHUVX2Ikc8KZLObTP +XTtGE993cADrYYxPH8l40QCHB2t6bM2maBTRy8PlMFMsYaM7rqnftlOuXyRYLm97 +oO7F8InpD1mDND4bBKbL3UOgl1QW9dbb8KOn8pERDXD9Z3K4ewyORiyuONXxBAz2 +QiaQ4gDJWD6sEySPEHcpDjemi6mKJFtDw935JxS2brso/NHoJSVdcNxisd2irA8P +U8jyFTrTm7cO0nKFx9vv9r5dsbkqyQnJ9V0q4/DoWvnkPElKuvTeJMw6CvTjfvc1 +xIf8m3MQJTN8pqPpIOSj8DIhQIJBEVxtcajg+vlMuefR8hTCdLzQV2QEdLGjNGph +OhFuHrHw1b3I4nCyi9MOLzpdUDgZkaw4hPg38u71RJ6bUF6pE89Yj04/svB7SZgA +sE8+/pZcASZl1Y0T29OM3qFJvukBWwsNij5P5SLmTXDev+DXbzUvmVLCk8nJkJ5m +n1peGst0ApHnEzxVhU+9d5FWBdIgN7fp7V2AlFt1r0TRWUYndBn9/B6xiyfd5iyK +LpcbRjLpQ/9b3SYQZL9t65Hc40rP35rY0luQTzVd6E/D041E0NWdpAZai4GrpMKp +NUBXBbmwnCZEzExLi78We6aF5tG9Vh+uxyYUXlrABR98jONgYIvnWaATaw6oLgam +B84RhWUa1f+K70wrTorUO8QNvYO2n4jpwJikn4H9pFvvzzoOB7RaA+EF4HjaVmQr +Jm6S18c7sksYFGIJYsPrXMtgOMQVYiQ3s8LDgsWKoEQYgvgxE24AoXGlXXbALrpo +4WvRN7qLAm3n2OQYFUK0SGoELOADQFQhbNL91i0k4wzQ56ofQCb8aTmm3q76TnbK +4gmIUIECggIBAOm3PhMqIKb3ZEZOWYMKByb7v8seuVh6mwXmgITcZIiA2f8rTfyH +OZanYK72FsuGZEbf1ELwNUX0JobwklA1Uq9NajaJUdfe4wDpghf282APHK1nD3m3 +FX7Rsoof/dW9+bV/IIbQDwy+1ctB20m6dejt3kWyHSXly2y2L3bk1flee8oi+VCO +ROp4u23KJcglIYlL+cL1z76lX41v8JW3deCTL/JStmK6zuvFhY5fTlOGxsLhAY3X +VTZ0MNUpezlBaXTixi7ilgbh7rJSpCmeno5my1hdl/5Q5U/u5F5E7ZxKxlhuVWdA +cf9qbtolSsgJ2w8axPD3dLnQ6wsNnl4yL77TdNr8OlbwIADXHH9VzZbfEgOhP4r9 +Cwz4hIUbylJ9QGZVsEVZhvqR3M6OgqlYOmC72nLPXVYbL/Rr7cTx8t+WJ4wJRP3l +pi6AgDZXKbMKyMIhM1IscSR2swtAFyl6g6rcAjFyG+p1IKMOQ3YOqdVVUf5UYSpo +NBTpJ+ho3CyQJK25yT2Maex5Q91M6Rk9vmXaIs+K40t/zu8cnCIH9j3In0BrpSi7 +WUyLgzXW+7ZrW7cugC93VVgjGNbgBiDimZ5duViI+xY+3OlrsUxzSHN+YY+8H2G2 +TfzT+Mgo8lJXH9tseFBLvRKhoPbbCyCwnOJ6k2k8/iqZ2eE7vQodyW/zAoICAQDE +yK5Mlu0k3oTVasgYRNDYKJ9uyarVLaRcN6UjzNUSta2hABzQe/LG895EucK9yVa0 +4jT7EXILYQpHnQkaYtyHtN/deYWLRObT6RMws7te2MhwzQkaNavgi449dHP38TPS +qy3Vh6v4nMhRjkhLqNiiJoDWThtkB216t+71GxdF2l9trrcvlC6q0Z65PR4LK3B/ +3m8KmlyGN940Ovb/4kEh2EZmNwbMq+3SaXiS4R7ZvKoboyNmwvCh4t5ne+NXz3Vm +7srej0cnOM54dTuNVdXm6pajYjELNVJGsK0vo6CAZiAOyasFkwMFMr5w/ehBaZIo +9ewWjvgDdvQ0+JBMVZJot7e5D4tUgVTaIMpQryzAp/u3r5KC1bkcC1Zie+44CaPr +V2FspEyYQDPJHC2tRj33lbaCWDdfdhYhv6uzlcM9ExPj+RQA0xlCuxh1pxK43H4t +PdWvcIeIKtFRWq9gsH1/wDBaIsB5Kykq4xyvMOj3dyV+dyIO1D3GG2+ZoBt8gklk +ieDIzCMKsXbomgm64/Ewlx9tf1fbTouSWP1+Gh0Wne3sCEtCY72ebYgjtaA0w8Xr +HGTsEdw/rSer46uULYWXYC6VYZwlyTbPh4NSD7ATrQo6HnaKugtuya9NPfYD9S9w +PoXPXpNKvwSq9SA/XsodHet9uU+IlKnYI41hVGaukQKCAgEAu7dAFmh5vWFqmsWH +Q6akQ/1j+KL+v0AKj1yp0Qyreq/zZ+3CP5fk+0oIb76cZG/Wzs4sUG9aeos0/qDv +A5kwhjipnJACul38+diTcugYZgj9QZFLbrlMfIW03xf+6tQhYlNvLpih3dzRHuYQ +WVF4LtQO+O+sVaoSD4js6pO2AQqQZrRchwUd+S1rbS0112FrZDkvrBV+/GMbMiC8 +naOy++N8WSdx8i6Uz+3f7ZBC+vd/YTsT4ncXrBr6tdsRa9VL/GPARhwb3/9LltKl +pAAQ3dNWc+0IjW0wIVmA9u5p4mR0lsZXtgyfA0TbMD9PpwezeJhJojk++ZgBkOkp +UAMDMkPo0ZJQ0U8ghUxBSU0DSbB1aZDz1pCTdaGN3tOJV2Pee9NLNwhHT64kKYtx +Cs8gZF29gQVotCY2CB0/5jVmm4qgzOsNDNiM90CiugKcLX/162Z7L/8eCOmFuP2L +HPeYFX2MDWbXYSMeSZjFOmdgpUZYCux+9m6nljwGn7bJUMmdjCNyrQrAcPydM3v4 +pRtgd7ISz5uAID3RWWAjT3oRn2Ip9rX++44ulTpg8rdviP8FtUWWE4nHdHkMFFnu +0i7ur7Ibki7DbQSkrp6e0watJIZXmfjBQ9wHG5CteybYKqZrofOnVubeP76t6Ffa +wjDYcVd+WsJwaosT4sH2fc+a4V0CggIAX703N+ISHGb6f47FLTGF3jTdZJxPNgpP +teIIwUMtwz2K7RLN8gzr0Cnj0NID0iw7uCN6HVCeiLS8uJzJSgIhQxEX/XvbIify +mdUC1iYLmhikZGUM8ah9J/Ed0f5vUYkTaoKH+SRZ4cn9l9g+ijjqtPFAq4Myxlbq +x9LnOm7kwsJ8vTNMwqQ4lDkKHRuEG3EV+dAxfBofu9lAjW7GrzXX9Gxxy9cnwro/ +KajQ1b0bMloc9PdJCPz87/YduIFI2XcAqoSxRNi2iHmV0ntO7vN87yzg/zqNm+Nk +VSouCEh2lSGapTcusraUuJm4l+agcfq39p+3Lj5+ocjlBUSNh+X2JmaDx1Ctu/50 +Vu+vHtQrVMwlpneQM4Fh3ygQ7jdXbgIasyw/JQlGjqhlyYmhG3VaPrOjKQVcY2Bj +YiYLloLdi0BV5AtR1RjfaFz4iH0xmakE5mP4K7P27KPQVOFnBAa5+EZ7/856kDkU +fV4jnYSRQ5y/LCkv88EFEPZWNIr19Hy60mRWbbsLrW8r5/zRkRDHTpmn27EiAHvV +Ye1T3187Gb7ae4SJgRJftc2ad60s9Z7uEvYsj8Icwwoui+1nk98V6NENuPXVb3jW +5t8KYAlr5BbDLAWYLORZHZyYYl8r6z8FxnlUxtA/nxUSY06BUvN+J+5fnTpkXJHf +lF2vslPK5VECggIANpTlrc5Zp9c0jPOiBQT/92LdtMR9mB76WB8fX9xn1Tm7V/cI +MatRrFzaTvi5HqHxPWJJ5cgMYKQw8kHHk1q9U7jeexZGCICFOUjk3N2Z0aXIWrx+ +hiS1i5DzRaZNI682s436cdaS4GpZIhqjIOR4/JS15Nfu+DVqyHOevWnd0D/tGoJB +JpiePYl2k7b8sirM3bHnRDtBiMHrHyHxzpdTLImDLHXr7C5DTeQAzzo+YMu45mNm +utKq3SRlO2wizj0qpONgJ6cHI7pypAPalliA+hPRRJqaHmTr7LpkP3+g73hf26wK +PoF6CLkdaN945F4Z48/3zEOP3XRIQocMA46ayA+8TsOpx9rm8L1LrF1pgmJi5Ml9 +3yV93ZZ/w4Jy0pqZ28nn3gcX6JP7qtfgioLLeyuuBphpOQXl7Ys3w1d0EEErpATU +tRG+BS16o+FhM1iXAN8c8HrqOtyJzCpYnuoWiNc+G8BIn6ox3xmihwK4hiR6s/OM +Mxaa6T4FF/IWIzuCezYPGgybWMXj3nRHsD7eNP170irBLwS3GdD1HWM2BpmLdJa0 +m/7QQkONhCnNFtjIalfPWMX51z/8GOKRKWH5A8PY6CxVHRg1ZfHT4TlfbdRPIG8d +6JrWt2s4eLv9VRdM+oMifHR4qvLDgg7R8pBr2FSsVXM9U2+j+drIeAhkjHI= -----END RSA PRIVATE KEY----- diff --git a/mysql-test/std_data/serversan-cert.pem b/mysql-test/std_data/serversan-cert.pem new file mode 100644 index 00000000000..e47779f420d --- /dev/null +++ b/mysql-test/std_data/serversan-cert.pem @@ -0,0 +1,60 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 4 (0x4) + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=cacert, C=FI, ST=Helsinki, L=Helsinki, O=MariaDB + Validity + Not Before: Apr 25 20:52:33 2017 GMT + Not After : Apr 20 20:52:33 2037 GMT + Subject: C=FI, ST=Helsinki, L=Helsinki, O=MariaDB, CN=server + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (1024 bit) + Modulus: + 00:a7:74:d4:2b:80:cb:96:08:2a:b9:c2:87:18:0d: + 69:2b:da:cf:ef:21:cb:05:d4:80:2c:f3:85:bc:78: + b2:42:d9:9f:f1:dc:47:68:c5:af:5a:c9:01:f0:dd: + 91:cb:3a:b9:38:b2:36:6b:a3:66:ef:cd:44:0f:8f: + 39:57:60:ad:3b:44:33:51:c2:7f:cb:5c:8d:55:b8: + 1e:e8:80:e0:ed:9d:8d:10:7a:42:68:73:06:63:83: + ce:db:05:5b:e1:7b:f9:0e:87:20:38:b8:11:6a:b7: + 59:3d:4a:ca:cb:60:e6:e1:73:d9:a2:24:4a:70:93: + 5e:cf:d5:04:d5:ad:ac:96:a5 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Alternative Name: + DNS:localhost + Signature Algorithm: sha256WithRSAEncryption + 4b:78:d9:09:4c:25:cc:fb:17:8f:31:13:ac:d7:36:2d:5f:d4: + ce:94:84:d2:a7:fa:e2:1e:ae:b6:72:1f:01:56:0f:89:80:c0: + 01:ba:ad:d7:cb:24:c5:25:ec:f8:35:ac:52:1b:4f:af:7c:26: + 8d:d4:d4:91:05:21:b7:ba:3f:6b:1b:8d:1d:a5:6b:7e:7d:be: + 2f:6a:09:83:c2:c3:6c:2f:8a:31:fa:7b:36:3f:6d:e1:62:ca: + a0:3c:43:b8:53:5a:4a:b3:4d:7a:cb:9c:6e:db:a4:ce:a1:95: + 5e:26:d8:22:39:8c:34:0e:92:bd:87:a2:b1:7a:68:25:57:17: + b2:d8:43:3b:98:e4:80:6b:7d:3e:ab:32:82:6d:b8:80:45:83: + d6:55:f8:cd:31:74:17:8c:42:75:09:71:66:b9:e0:94:16:ca: + 1d:db:1e:89:12:a1:9f:00:cb:83:99:5d:5d:28:7a:df:2a:87: + b5:8d:f1:9c:b9:89:2a:0d:6c:af:61:00:41:cb:03:df:99:4a: + fe:93:81:88:ff:47:4e:2a:b5:2b:bf:85:0f:9a:21:7b:20:58: + 7a:1c:67:b5:8b:da:db:03:69:25:db:76:0e:f9:23:57:8d:8a: + 47:dc:15:16:7c:2d:66:8f:6a:10:f3:b2:ea:2e:31:c6:d4:2c: + 90:15:56:f4 +-----BEGIN CERTIFICATE----- +MIICuzCCAaOgAwIBAgIBBDANBgkqhkiG9w0BAQsFADBWMQ8wDQYDVQQDDAZjYWNl +cnQxCzAJBgNVBAYTAkZJMREwDwYDVQQIDAhIZWxzaW5raTERMA8GA1UEBwwISGVs +c2lua2kxEDAOBgNVBAoMB01hcmlhREIwHhcNMTcwNDI1MjA1MjMzWhcNMzcwNDIw +MjA1MjMzWjBWMQswCQYDVQQGEwJGSTERMA8GA1UECAwISGVsc2lua2kxETAPBgNV +BAcMCEhlbHNpbmtpMRAwDgYDVQQKDAdNYXJpYURCMQ8wDQYDVQQDDAZzZXJ2ZXIw +gZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAKd01CuAy5YIKrnChxgNaSvaz+8h +ywXUgCzzhbx4skLZn/HcR2jFr1rJAfDdkcs6uTiyNmujZu/NRA+POVdgrTtEM1HC +f8tcjVW4HuiA4O2djRB6QmhzBmODztsFW+F7+Q6HIDi4EWq3WT1Kystg5uFz2aIk +SnCTXs/VBNWtrJalAgMBAAGjGDAWMBQGA1UdEQQNMAuCCWxvY2FsaG9zdDANBgkq +hkiG9w0BAQsFAAOCAQEAS3jZCUwlzPsXjzETrNc2LV/UzpSE0qf64h6utnIfAVYP +iYDAAbqt18skxSXs+DWsUhtPr3wmjdTUkQUht7o/axuNHaVrfn2+L2oJg8LDbC+K +Mfp7Nj9t4WLKoDxDuFNaSrNNesucbtukzqGVXibYIjmMNA6SvYeisXpoJVcXsthD +O5jkgGt9Pqsygm24gEWD1lX4zTF0F4xCdQlxZrnglBbKHdseiRKhnwDLg5ldXSh6 +3yqHtY3xnLmJKg1sr2EAQcsD35lK/pOBiP9HTiq1K7+FD5oheyBYehxntYva2wNp +Jdt2DvkjV42KR9wVFnwtZo9qEPOy6i4xxtQskBVW9A== +-----END CERTIFICATE----- diff --git a/mysql-test/std_data/serversan-key.pem b/mysql-test/std_data/serversan-key.pem new file mode 100644 index 00000000000..393c0bc9c1a --- /dev/null +++ b/mysql-test/std_data/serversan-key.pem @@ -0,0 +1,16 @@ +-----BEGIN PRIVATE KEY----- +MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAKd01CuAy5YIKrnC +hxgNaSvaz+8hywXUgCzzhbx4skLZn/HcR2jFr1rJAfDdkcs6uTiyNmujZu/NRA+P +OVdgrTtEM1HCf8tcjVW4HuiA4O2djRB6QmhzBmODztsFW+F7+Q6HIDi4EWq3WT1K +ystg5uFz2aIkSnCTXs/VBNWtrJalAgMBAAECgYBReSgZmmpzLroK8zhjXXMEIUv1 +3w02YvOR61HwJxEkMVn+hNxBf50XoKDPHh5nMMUZbqvHpxLYLZilsVuGxcTCPVzw +YxTooPcJY8x61oUclI2Ls5czu/OfzoJhA9ESaFn6e4xReUFmNi8ygTMuPReZZ90T +ZvDikonKtCCk99MSaQJBANrmlPtfY57KJ18f1TqLvqy73I1vQjffSOrK3deYbvvB +jUJ79G9Wzj8Hje2y+XkkK+OIPcND1DnoTCTuqVazn+cCQQDD1jy8zrVg/JEPhQkS +BM7nvm4PIb0cgTPrOhsHDIF4hbaAZnA0N4ZEJ2q7YitXfOeR98x+aH/WJOrzzhmE +VXOTAkBQ4lK6b4zH57qUk5aeg3R5LxFX0XyOWJsA5uUB/PlFXUdtAZBYc6LR92Ci +LDeyY4M0F+t6c12/5+3615UKzGSRAkA+SGV6utcOqGTOJcZTt7nCFFtWbqmBZkoH +1qv/2udWWFhJj8rBoKMQC+UzAS69nVjcoI2l6kA17/nVXkfZQYAHAkEAmOHCZCVQ +9CCYTJICvoZR2euUYdnatLN8d2/ARWjzcRDTdS82P2oscATwAsvJxsphDmbOmVWP +Hfy1t8OOCHKYAQ== +-----END PRIVATE KEY----- diff --git a/mysql-test/suite.pm b/mysql-test/suite.pm index f501e610e53..a662a600afe 100644 --- a/mysql-test/suite.pm +++ b/mysql-test/suite.pm @@ -68,6 +68,10 @@ sub skip_combinations { unless $::mysqld_variables{'version-ssl-library'} =~ /OpenSSL (\S+)/ and $1 ge "1.0.1d"; + $skip{'t/ssl_7937.combinations'} = [ 'x509v3' ] + unless $::mysqld_variables{'version-ssl-library'} =~ /OpenSSL (\S+)/ + and $1 ge "1.0.2"; + %skip; } diff --git a/mysql-test/suite/encryption/r/filekeys_syntax.result b/mysql-test/suite/encryption/r/filekeys_syntax.result index eb8119bc4f5..a64d21eedbe 100644 --- a/mysql-test/suite/encryption/r/filekeys_syntax.result +++ b/mysql-test/suite/encryption/r/filekeys_syntax.result @@ -28,7 +28,7 @@ select plugin_status from information_schema.plugins where plugin_name = 'file_key_management'; plugin_status install soname 'file_key_management'; -ERROR HY000: Invalid key id at MYSQL_TMP_DIR/keys.txt line 2, column 11 +ERROR HY000: Invalid key id at MYSQL_TMP_DIR/keys.txt line 2, column 10 call mtr.add_suppression("Invalid key id"); call mtr.add_suppression("Plugin 'file_key_management' init function returned error"); call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); diff --git a/mysql-test/suite/mariabackup/bug1509812-master.opt b/mysql-test/suite/mariabackup/bug1509812-master.opt new file mode 100644 index 00000000000..de29ed41f59 --- /dev/null +++ b/mysql-test/suite/mariabackup/bug1509812-master.opt @@ -0,0 +1 @@ +--loose-skip-log-bin
\ No newline at end of file diff --git a/mysql-test/suite/mariabackup/filekeys-data.enc b/mysql-test/suite/mariabackup/filekeys-data.enc Binary files differnew file mode 100644 index 00000000000..a8adb2f939c --- /dev/null +++ b/mysql-test/suite/mariabackup/filekeys-data.enc diff --git a/mysql-test/suite/mariabackup/filekeys-data.key b/mysql-test/suite/mariabackup/filekeys-data.key new file mode 100644 index 00000000000..85fcd1fbb81 --- /dev/null +++ b/mysql-test/suite/mariabackup/filekeys-data.key @@ -0,0 +1,2 @@ +secret + diff --git a/mysql-test/suite/mariabackup/full_backup.result b/mysql-test/suite/mariabackup/full_backup.result new file mode 100644 index 00000000000..c387f5328a7 --- /dev/null +++ b/mysql-test/suite/mariabackup/full_backup.result @@ -0,0 +1,13 @@ +CREATE TABLE t(i INT) ENGINE INNODB; +INSERT INTO t VALUES(1); +# xtrabackup backup +INSERT INTO t VALUES(2); +# xtrabackup prepare +# shutdown server +# remove datadir +# xtrabackup move back +# restart server +SELECT * FROM t; +i +1 +DROP TABLE t; diff --git a/mysql-test/suite/mariabackup/full_backup.test b/mysql-test/suite/mariabackup/full_backup.test new file mode 100644 index 00000000000..a79f54c67e4 --- /dev/null +++ b/mysql-test/suite/mariabackup/full_backup.test @@ -0,0 +1,23 @@ +CREATE TABLE t(i INT) ENGINE INNODB; +INSERT INTO t VALUES(1); +echo # xtrabackup backup; +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; + +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir; +--enable_result_log + +INSERT INTO t VALUES(2); + + +echo # xtrabackup prepare; +--disable_result_log +exec $XTRABACKUP --prepare --target-dir=$targetdir; +exec $XTRABACKUP --defaults-file=$targetdir/backup-my.cnf --stats --datadir=$targetdir; +-- source include/restart_and_restore.inc +--enable_result_log + +SELECT * FROM t; +DROP TABLE t; +rmdir $targetdir; + diff --git a/mysql-test/suite/mariabackup/include/have_file_key_management.inc b/mysql-test/suite/mariabackup/include/have_file_key_management.inc new file mode 100644 index 00000000000..06fbb510d6b --- /dev/null +++ b/mysql-test/suite/mariabackup/include/have_file_key_management.inc @@ -0,0 +1,4 @@ +if (`SELECT COUNT(*)=0 FROM INFORMATION_SCHEMA.PLUGINS WHERE PLUGIN_NAME = 'file_key_management' AND PLUGIN_STATUS='ACTIVE'`) +{ + --skip Test requires file_key_management plugin +} diff --git a/mysql-test/suite/mariabackup/include/restart_and_restore.inc b/mysql-test/suite/mariabackup/include/restart_and_restore.inc new file mode 100644 index 00000000000..39616cc6f15 --- /dev/null +++ b/mysql-test/suite/mariabackup/include/restart_and_restore.inc @@ -0,0 +1,15 @@ +let $_server_id= `SELECT @@server_id`; +let $_datadir= `SELECT @@datadir`; +let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.$_server_id.expect; +exec echo "wait" > $_expect_file_name; +echo # shutdown server; +shutdown_server; +echo # remove datadir; +rmdir $_datadir; +echo # xtrabackup move back; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --copy-back --datadir=$_datadir --target-dir=$targetdir --parallel=2; +echo # restart server; +exec echo "restart" > $_expect_file_name; +enable_reconnect; +source include/wait_until_connected_again.inc; +disable_reconnect; diff --git a/mysql-test/suite/mariabackup/incremental_backup.result b/mysql-test/suite/mariabackup/incremental_backup.result new file mode 100644 index 00000000000..eeedc751d83 --- /dev/null +++ b/mysql-test/suite/mariabackup/incremental_backup.result @@ -0,0 +1,20 @@ +call mtr.add_suppression("InnoDB: New log files created"); +CREATE TABLE t(i INT) ENGINE INNODB; +INSERT INTO t VALUES(1); +# Create full backup , modify table, then create incremental/differential backup +INSERT INTO t VALUES(2); +SELECT * FROM t; +i +1 +2 +# Prepare full backup, apply incremental one +# Restore and check results +# shutdown server +# remove datadir +# xtrabackup move back +# restart server +SELECT * FROM t; +i +1 +2 +DROP TABLE t; diff --git a/mysql-test/suite/mariabackup/incremental_backup.test b/mysql-test/suite/mariabackup/incremental_backup.test new file mode 100644 index 00000000000..b92d7b323ac --- /dev/null +++ b/mysql-test/suite/mariabackup/incremental_backup.test @@ -0,0 +1,35 @@ +call mtr.add_suppression("InnoDB: New log files created"); + + +let $basedir=$MYSQLTEST_VARDIR/tmp/backup; +let $incremental_dir=$MYSQLTEST_VARDIR/tmp/backup_inc1; + + +CREATE TABLE t(i INT) ENGINE INNODB; +INSERT INTO t VALUES(1); + +echo # Create full backup , modify table, then create incremental/differential backup; +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$basedir; +--enable_result_log +INSERT INTO t VALUES(2); +SELECT * FROM t; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$incremental_dir --incremental-basedir=$basedir; + +--disable_result_log +echo # Prepare full backup, apply incremental one; +exec $XTRABACKUP --prepare --apply-log-only --target-dir=$basedir; +exec $XTRABACKUP --prepare --target-dir=$basedir --incremental-dir=$incremental_dir ; + +echo # Restore and check results; +let $targetdir=$basedir; +-- source include/restart_and_restore.inc +--enable_result_log + +SELECT * FROM t; +DROP TABLE t; + +# Cleanup +rmdir $basedir; +rmdir $incremental_dir; + diff --git a/mysql-test/suite/mariabackup/incremental_encrypted.opt b/mysql-test/suite/mariabackup/incremental_encrypted.opt new file mode 100644 index 00000000000..ea644cef6ce --- /dev/null +++ b/mysql-test/suite/mariabackup/incremental_encrypted.opt @@ -0,0 +1,3 @@ +--innodb-tablespaces-encryption +--plugin-load-add=$DEBUG_KEY_MANAGEMENT_SO +--loose-debug_key_management_version=2 diff --git a/mysql-test/suite/mariabackup/incremental_encrypted.result b/mysql-test/suite/mariabackup/incremental_encrypted.result new file mode 100644 index 00000000000..e8f81e9fa49 --- /dev/null +++ b/mysql-test/suite/mariabackup/incremental_encrypted.result @@ -0,0 +1,20 @@ +call mtr.add_suppression("InnoDB: New log files created"); +CREATE TABLE t(i INT) ENGINE INNODB ENCRYPTED=YES; +INSERT INTO t VALUES(1); +# Create full backup , modify table, then create incremental/differential backup +INSERT INTO t VALUES(2); +SELECT * FROM t; +i +1 +2 +# Prepare full backup, apply incremental one +# Restore and check results +# shutdown server +# remove datadir +# xtrabackup move back +# restart server +SELECT * FROM t; +i +1 +2 +DROP TABLE t; diff --git a/mysql-test/suite/mariabackup/incremental_encrypted.test b/mysql-test/suite/mariabackup/incremental_encrypted.test new file mode 100644 index 00000000000..c379b3c8165 --- /dev/null +++ b/mysql-test/suite/mariabackup/incremental_encrypted.test @@ -0,0 +1,45 @@ +if (!$EXAMPLE_KEY_MANAGEMENT_SO) +{ + --skip needs example_key_management plugin +} +call mtr.add_suppression("InnoDB: New log files created"); + + +let $basedir=$MYSQLTEST_VARDIR/tmp/backup; +let $incremental_dir=$MYSQLTEST_VARDIR/tmp/backup_inc1; + + +CREATE TABLE t(i INT) ENGINE INNODB ENCRYPTED=YES; +INSERT INTO t VALUES(1); + + +echo # Create full backup , modify table, then create incremental/differential backup; +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$basedir; +--enable_result_log + +INSERT INTO t VALUES(2); +SELECT * FROM t; + +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$incremental_dir --incremental-basedir=$basedir; +echo # Prepare full backup, apply incremental one; +exec $XTRABACKUP --prepare --apply-log-only --target-dir=$basedir; +exec $XTRABACKUP --prepare --target-dir=$basedir --incremental-dir=$incremental_dir; + +# stats also can support encryption, but needs plugin-load and plugin variables, they are stored in backup-my.cnf +# We need to prepare again to create log files though. +exec $XTRABACKUP --prepare --target-dir=$basedir; +exec $XTRABACKUP --defaults-file=$basedir/backup-my.cnf --stats --datadir=$basedir; + +echo # Restore and check results; +let $targetdir=$basedir; +-- source include/restart_and_restore.inc +--enable_result_log + +SELECT * FROM t; +DROP TABLE t; + +# Cleanup +rmdir $basedir; +rmdir $incremental_dir; diff --git a/mysql-test/suite/mariabackup/partial.result b/mysql-test/suite/mariabackup/partial.result new file mode 100644 index 00000000000..98c59be91bb --- /dev/null +++ b/mysql-test/suite/mariabackup/partial.result @@ -0,0 +1,13 @@ +CREATE TABLE t1(i INT) ENGINE INNODB; +INSERT INTO t1 VALUES(1); +CREATE TABLE t2(i int) ENGINE INNODB; +# xtrabackup backup +t1.ibd +# xtrabackup prepare +ALTER TABLE t1 DISCARD TABLESPACE; +ALTER TABLE t1 IMPORT TABLESPACE; +SELECT * FROM t1; +i +1 +DROP TABLE t1; +DROP TABLE t2; diff --git a/mysql-test/suite/mariabackup/partial.test b/mysql-test/suite/mariabackup/partial.test new file mode 100644 index 00000000000..3b1de8ebc25 --- /dev/null +++ b/mysql-test/suite/mariabackup/partial.test @@ -0,0 +1,31 @@ +# Export single table from backup +# (xtrabackup with --prepare --export) + +CREATE TABLE t1(i INT) ENGINE INNODB; +INSERT INTO t1 VALUES(1); +CREATE TABLE t2(i int) ENGINE INNODB; + +echo # xtrabackup backup; + +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup "--tables=test.*1" --target-dir=$targetdir; +--enable_result_log +list_files $targetdir/test *.ibd; + +echo # xtrabackup prepare; +--disable_result_log +exec $XTRABACKUP --prepare --export --target-dir=$targetdir; +--enable_result_log + +ALTER TABLE t1 DISCARD TABLESPACE; +let $MYSQLD_DATADIR= `select @@datadir`; +copy_file $targetdir/test/t1.ibd $MYSQLD_DATADIR/test/t1.ibd; +copy_file $targetdir/test/t1.cfg $MYSQLD_DATADIR/test/t1.cfg; +ALTER TABLE t1 IMPORT TABLESPACE; + +SELECT * FROM t1; +DROP TABLE t1; +DROP TABLE t2; +rmdir $targetdir; + diff --git a/mysql-test/suite/mariabackup/partial_exclude.result b/mysql-test/suite/mariabackup/partial_exclude.result new file mode 100644 index 00000000000..0da9b547caa --- /dev/null +++ b/mysql-test/suite/mariabackup/partial_exclude.result @@ -0,0 +1,12 @@ +CREATE TABLE t1(i INT) ENGINE INNODB; +INSERT INTO t1 VALUES(1); +CREATE TABLE t2(i int) ENGINE INNODB; +CREATE DATABASE db2; +USE db2; +CREATE TABLE t1(i INT) ENGINE INNODB; +USE test; +# xtrabackup backup +t1.ibd +DROP TABLE t1; +DROP TABLE t2; +DROP DATABASE db2; diff --git a/mysql-test/suite/mariabackup/partial_exclude.test b/mysql-test/suite/mariabackup/partial_exclude.test new file mode 100644 index 00000000000..631f9d7ee71 --- /dev/null +++ b/mysql-test/suite/mariabackup/partial_exclude.test @@ -0,0 +1,30 @@ +# Test --databases-exclude and --tables-exclude feature of xtrabackup 2.3.8 + +CREATE TABLE t1(i INT) ENGINE INNODB; +INSERT INTO t1 VALUES(1); +CREATE TABLE t2(i int) ENGINE INNODB; + +CREATE DATABASE db2; +USE db2; +CREATE TABLE t1(i INT) ENGINE INNODB; + +USE test; + +echo # xtrabackup backup; + +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup "--tables-exclude=test.*2" "--databases-exclude=db2" --target-dir=$targetdir; +--enable_result_log + +# check that only t1 table is in backup (t2 is excluded) +list_files $targetdir/test *.ibd; +# check that db2 database is not in the backup (excluded) +--error 1 +list_files $targetdir/db2 *.ibd; + +DROP TABLE t1; +DROP TABLE t2; +DROP DATABASE db2; +rmdir $targetdir; + diff --git a/mysql-test/suite/mariabackup/small_ibd.result b/mysql-test/suite/mariabackup/small_ibd.result new file mode 100644 index 00000000000..1c840a7a1b6 --- /dev/null +++ b/mysql-test/suite/mariabackup/small_ibd.result @@ -0,0 +1 @@ +#backup diff --git a/mysql-test/suite/mariabackup/small_ibd.test b/mysql-test/suite/mariabackup/small_ibd.test new file mode 100644 index 00000000000..23d6b08c047 --- /dev/null +++ b/mysql-test/suite/mariabackup/small_ibd.test @@ -0,0 +1,18 @@ +# Check if ibd smaller than page size are skipped +# It is possible, due to race conditions that new file +# is created by server while xtrabackup is running +# The first page in this file does not yet exist. +# xtrabackup should skip such file. + +let $_datadir= `SELECT @@datadir`; +write_file $_datadir/test/small.ibd; +EOF +echo #backup; + +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir; +--enable_result_log +remove_file $_datadir/test/small.ibd; +rmdir $targetdir; + diff --git a/mysql-test/suite/mariabackup/suite.opt b/mysql-test/suite/mariabackup/suite.opt new file mode 100644 index 00000000000..ec00a407620 --- /dev/null +++ b/mysql-test/suite/mariabackup/suite.opt @@ -0,0 +1 @@ +--innodb --changed_page_bitmaps --innodb-file-format=Barracuda
\ No newline at end of file diff --git a/mysql-test/suite/mariabackup/suite.pm b/mysql-test/suite/mariabackup/suite.pm new file mode 100644 index 00000000000..8eecd4e8018 --- /dev/null +++ b/mysql-test/suite/mariabackup/suite.pm @@ -0,0 +1,38 @@ +package My::Suite::MariaBackup; + +@ISA = qw(My::Suite); +use My::Find; +use File::Basename; +use strict; + +return "Not run for embedded server" if $::opt_embedded_server; + +my $mariabackup_exe= +::mtr_exe_maybe_exists( + "$::bindir/extra/mariabackup$::opt_vs_config/mariabackup", + "$::path_client_bindir/mariabackup"); + +return "No mariabackup" if !$mariabackup_exe; + + +$ENV{XTRABACKUP}= $mariabackup_exe; + +$ENV{XBSTREAM}= ::mtr_exe_maybe_exists( + "$::bindir/extra/mariabackup/$::opt_vs_config/mbstream", + "$::path_client_bindir/mbstream"); + +my $tar_version = `tar --version 2>&1`; +$ENV{HAVE_TAR} = $! ? 0: 1; +my $mariabackup_help=`$mariabackup_exe --help 2>&1`; +$ENV{HAVE_XTRABACKUP_TAR_SUPPORT} = (index($mariabackup_help,"'tar'") == -1) ? 0 : 1; + +$ENV{INNOBACKUPEX}= "$mariabackup_exe --innobackupex"; + +sub skip_combinations { + my %skip; + $skip{'include/have_file_key_management.inc'} = 'needs file_key_management plugin' unless $ENV{FILE_KEY_MANAGEMENT_SO}; + %skip; +} + +bless { }; + diff --git a/mysql-test/suite/mariabackup/tar.result b/mysql-test/suite/mariabackup/tar.result new file mode 100644 index 00000000000..bbb546d7add --- /dev/null +++ b/mysql-test/suite/mariabackup/tar.result @@ -0,0 +1,12 @@ +CREATE TABLE t(i INT) ENGINE INNODB; +INSERT INTO t VALUES(1); +# xtrabackup backup +# xtrabackup prepare +# shutdown server +# remove datadir +# xtrabackup move back +# restart server +SELECT * FROM t; +i +1 +DROP TABLE t; diff --git a/mysql-test/suite/mariabackup/tar.test b/mysql-test/suite/mariabackup/tar.test new file mode 100644 index 00000000000..3938d597e05 --- /dev/null +++ b/mysql-test/suite/mariabackup/tar.test @@ -0,0 +1,30 @@ +if (`select $HAVE_TAR = 0`) +{ + --skip No tar +} +if (`select $HAVE_XTRABACKUP_TAR_SUPPORT = 0`) +{ + --skip Compiled without libarchive +} + + +CREATE TABLE t(i INT) ENGINE INNODB; +INSERT INTO t VALUES(1); + +echo # xtrabackup backup; +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; +let $streamfile=$MYSQLTEST_VARDIR/tmp/backup.tar; +mkdir $targetdir; + + +exec $XTRABACKUP "--defaults-file=$MYSQLTEST_VARDIR/my.cnf" --backup --stream=tar > $streamfile 2>$targetdir/backup_stream.log; +--disable_result_log +exec tar -C $targetdir -x < $streamfile; +echo # xtrabackup prepare; +exec $XTRABACKUP --prepare --target-dir=$targetdir; + +-- source include/restart_and_restore.inc +--enable_result_log +SELECT * FROM t; +DROP TABLE t; +rmdir $targetdir; diff --git a/mysql-test/suite/mariabackup/xb_aws_key_management.opt b/mysql-test/suite/mariabackup/xb_aws_key_management.opt new file mode 100644 index 00000000000..62d4f864073 --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_aws_key_management.opt @@ -0,0 +1,3 @@ +--plugin-load-add=$AWS_KEY_MANAGEMENT_SO +--loose-aws-key-management +--loose-aws-key-management-master-key-id=$AWS_KEY_MANAGEMENT_MASTER_KEY_ID diff --git a/mysql-test/suite/mariabackup/xb_aws_key_management.result b/mysql-test/suite/mariabackup/xb_aws_key_management.result new file mode 100644 index 00000000000..ccad423f631 --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_aws_key_management.result @@ -0,0 +1,11 @@ +CREATE TABLE t(c VARCHAR(10)) ENGINE INNODB encrypted=yes; +INSERT INTO t VALUES('foobar1'); +# xtrabackup backup +# shutdown server +# remove datadir +# xtrabackup move back +# restart server +SELECT * from t; +c +foobar1 +DROP TABLE t; diff --git a/mysql-test/suite/mariabackup/xb_aws_key_management.test b/mysql-test/suite/mariabackup/xb_aws_key_management.test new file mode 100644 index 00000000000..ca01be607c7 --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_aws_key_management.test @@ -0,0 +1,22 @@ +if (`SELECT COUNT(*)=0 FROM INFORMATION_SCHEMA.PLUGINS WHERE PLUGIN_NAME = 'aws_key_management' AND PLUGIN_STATUS='ACTIVE'`) +{ + --skip needs aws_key_management plugin plugin +} + +if (`SELECT @@aws_key_management_master_key_id=''`) +{ + --skip Test requires AWS_KEY_MANAGEMEMENT_MASTER_KEY_ID env. variable +} + +CREATE TABLE t(c VARCHAR(10)) ENGINE INNODB encrypted=yes; +INSERT INTO t VALUES('foobar1'); +echo # xtrabackup backup; +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir; +exec $XTRABACKUP --prepare --target-dir=$targetdir; +-- source include/restart_and_restore.inc +--enable_result_log +SELECT * from t; +DROP TABLE t; +rmdir $targetdir; diff --git a/mysql-test/suite/mariabackup/xb_compressed_encrypted.opt b/mysql-test/suite/mariabackup/xb_compressed_encrypted.opt new file mode 100644 index 00000000000..e6cbe00bb7c --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_compressed_encrypted.opt @@ -0,0 +1,9 @@ +--innodb-encryption-rotate-key-age=2 +--innodb-encryption-threads=4 +--innodb-tablespaces-encryption +--plugin-load-add=$FILE_KEY_MANAGEMENT_SO +--loose-file-key-management +--loose-file-key-management-filename=$MYSQL_TEST_DIR/std_data/logkey.txt +--innodb_strict_mode +--innodb_file_per_table +--innodb_file_format=Barracuda diff --git a/mysql-test/suite/mariabackup/xb_compressed_encrypted.result b/mysql-test/suite/mariabackup/xb_compressed_encrypted.result new file mode 100644 index 00000000000..c0eb0e70631 --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_compressed_encrypted.result @@ -0,0 +1,25 @@ +CREATE TABLE t1(c1 INT, b VARCHAR(2400), index(b(100),c1)) ENGINE=INNODB ROW_FORMAT=compressed ENCRYPTED=YES; +CREATE PROCEDURE innodb_insert_proc (REPEAT_COUNT INT) +BEGIN +DECLARE CURRENT_NUM INT; +SET CURRENT_NUM = 0; +WHILE CURRENT_NUM < REPEAT_COUNT DO +INSERT INTO t1 VALUES(CURRENT_NUM, concat(uuid(), CURRENT_NUM, repeat('ab', floor(rand()*100) ), uuid())); +SET CURRENT_NUM = CURRENT_NUM + 1; +END WHILE; +END// +COMMIT; +SET AUTOCOMMIT=0; +CALL innodb_insert_proc(50000); +COMMIT; +# xtrabackup backup +drop table t1; +# shutdown server +# remove datadir +# xtrabackup move back +# restart server +select sum(c1) from t1; +sum(c1) +1249975000 +DROP TABLE t1; +drop procedure innodb_insert_proc; diff --git a/mysql-test/suite/mariabackup/xb_compressed_encrypted.test b/mysql-test/suite/mariabackup/xb_compressed_encrypted.test new file mode 100644 index 00000000000..11f63eb0330 --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_compressed_encrypted.test @@ -0,0 +1,35 @@ +source include/have_file_key_management.inc; + +CREATE TABLE t1(c1 INT, b VARCHAR(2400), index(b(100),c1)) ENGINE=INNODB ROW_FORMAT=compressed ENCRYPTED=YES; + +DELIMITER //; +CREATE PROCEDURE innodb_insert_proc (REPEAT_COUNT INT) +BEGIN + DECLARE CURRENT_NUM INT; + SET CURRENT_NUM = 0; + WHILE CURRENT_NUM < REPEAT_COUNT DO + INSERT INTO t1 VALUES(CURRENT_NUM, concat(uuid(), CURRENT_NUM, repeat('ab', floor(rand()*100) ), uuid())); + SET CURRENT_NUM = CURRENT_NUM + 1; + END WHILE; +END// +DELIMITER ;// +COMMIT; + +SET AUTOCOMMIT=0; +CALL innodb_insert_proc(50000); +COMMIT; + + +echo # xtrabackup backup; +--disable_result_log +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; +exec $INNOBACKUPEX --defaults-file=$MYSQLTEST_VARDIR/my.cnf --no-timestamp $targetdir; +drop table t1; +exec $INNOBACKUPEX --apply-log $targetdir; + +-- source include/restart_and_restore.inc +--enable_result_log +select sum(c1) from t1; +DROP TABLE t1; +drop procedure innodb_insert_proc; +rmdir $targetdir; diff --git a/mysql-test/suite/mariabackup/xb_file_key_management.opt b/mysql-test/suite/mariabackup/xb_file_key_management.opt new file mode 100644 index 00000000000..74a6450a1ef --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_file_key_management.opt @@ -0,0 +1,6 @@ +--innodb-encrypt-log=ON +--plugin-load-add=$FILE_KEY_MANAGEMENT_SO +--loose-file-key-management +--loose-file-key-management-filekey=FILE:$MTR_SUITE_DIR/filekeys-data.key +--loose-file-key-management-filename=$MTR_SUITE_DIR/filekeys-data.enc +--loose-file-key-management-encryption-algorithm=aes_cbc diff --git a/mysql-test/suite/mariabackup/xb_file_key_management.result b/mysql-test/suite/mariabackup/xb_file_key_management.result new file mode 100644 index 00000000000..30aa530698b --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_file_key_management.result @@ -0,0 +1,17 @@ +CREATE TABLE t(c VARCHAR(10)) ENGINE INNODB encrypted=yes; +INSERT INTO t VALUES('foobar1'); +# xtrabackup backup +NOT FOUND /foobar1/ in xtrabackup_logfile +# expect NOT FOUND +INSERT INTO t VALUES('foobar2'); +# xtrabackup prepare +# shutdown server +# remove datadir +# xtrabackup move back +# restart server +NOT FOUND /foobar1/ in xtrabackup_logfile +# expect NOT FOUND +SELECT * FROM t; +c +foobar1 +DROP TABLE t; diff --git a/mysql-test/suite/mariabackup/xb_file_key_management.test b/mysql-test/suite/mariabackup/xb_file_key_management.test new file mode 100644 index 00000000000..bc975a7cdc4 --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_file_key_management.test @@ -0,0 +1,40 @@ +source include/have_file_key_management.inc; + +CREATE TABLE t(c VARCHAR(10)) ENGINE INNODB encrypted=yes; +INSERT INTO t VALUES('foobar1'); +echo # xtrabackup backup; +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir; +--enable_result_log + +--let SEARCH_RANGE = 10000000 +--let SEARCH_PATTERN=foobar1 +--let SEARCH_FILE=$targetdir/xtrabackup_logfile +--source include/search_pattern_in_file.inc +--echo # expect NOT FOUND + +INSERT INTO t VALUES('foobar2'); +echo # xtrabackup prepare; + +--disable_result_log +exec $XTRABACKUP --prepare --target-dir=$targetdir; +exec $XTRABACKUP --defaults-file=$targetdir/backup-my.cnf --stats --datadir=$targetdir ; +-- source include/restart_and_restore.inc +--enable_result_log + +# +# Recheck that plain text data ( +# in not in the log, after prepare +# (MDEV-11538) + +--let SEARCH_RANGE = 10000000 +--let SEARCH_PATTERN=foobar1 +--let SEARCH_FILE=$targetdir/xtrabackup_logfile +--source include/search_pattern_in_file.inc +--echo # expect NOT FOUND + +SELECT * FROM t; +DROP TABLE t; +rmdir $targetdir; + diff --git a/mysql-test/suite/mariabackup/xb_fulltext_encrypted.opt b/mysql-test/suite/mariabackup/xb_fulltext_encrypted.opt new file mode 100644 index 00000000000..b3ef366361a --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_fulltext_encrypted.opt @@ -0,0 +1,8 @@ +--plugin-load-add=$FILE_KEY_MANAGEMENT_SO +--innodb_strict_mode +--innodb_file_per_table +--innodb-encryption-rotate-key-age=2 +--innodb-encryption-threads=4 +--innodb-tablespaces-encryption +--loose-file-key-management +--loose-file-key-management-filename=$MYSQL_TEST_DIR/std_data/logkey.txt
\ No newline at end of file diff --git a/mysql-test/suite/mariabackup/xb_fulltext_encrypted.result b/mysql-test/suite/mariabackup/xb_fulltext_encrypted.result new file mode 100644 index 00000000000..01a99e59200 --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_fulltext_encrypted.result @@ -0,0 +1,14 @@ +CREATE TABLE film_text ( +film_id SMALLINT NOT NULL, +title VARCHAR(255) NOT NULL, +description TEXT, +PRIMARY KEY (film_id), +FULLTEXT KEY idx_title_description (title,description), +FULLTEXT KEY (description), +FULLTEXT KEY (title) +)ENGINE=InnoDB DEFAULT CHARSET=utf8 ENCRYPTED=YES; +# shutdown server +# remove datadir +# xtrabackup move back +# restart server +drop table film_text; diff --git a/mysql-test/suite/mariabackup/xb_fulltext_encrypted.test b/mysql-test/suite/mariabackup/xb_fulltext_encrypted.test new file mode 100644 index 00000000000..a98d7802d76 --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_fulltext_encrypted.test @@ -0,0 +1,23 @@ +source include/have_file_key_management.inc; + +CREATE TABLE film_text ( +film_id SMALLINT NOT NULL, +title VARCHAR(255) NOT NULL, +description TEXT, +PRIMARY KEY (film_id), +FULLTEXT KEY idx_title_description (title,description), +FULLTEXT KEY (description), +FULLTEXT KEY (title) +)ENGINE=InnoDB DEFAULT CHARSET=utf8 ENCRYPTED=YES; + +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; + +--disable_result_log + +exec $INNOBACKUPEX --defaults-file=$MYSQLTEST_VARDIR/my.cnf --no-timestamp $targetdir; +exec $INNOBACKUPEX --apply-log --rebuild-indexes --rebuild-threads=2 $targetdir; +--source include/restart_and_restore.inc + +--enable_result_log + +drop table film_text; diff --git a/mysql-test/suite/mariabackup/xb_history.result b/mysql-test/suite/mariabackup/xb_history.result new file mode 100644 index 00000000000..b6a1ac21147 --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_history.result @@ -0,0 +1,5 @@ +SELECT COUNT(*) FROM PERCONA_SCHEMA.xtrabackup_history; +COUNT(*) +1 +DROP TABLE PERCONA_SCHEMA.xtrabackup_history; +DROP DATABASE PERCONA_SCHEMA; diff --git a/mysql-test/suite/mariabackup/xb_history.test b/mysql-test/suite/mariabackup/xb_history.test new file mode 100644 index 00000000000..28de50127c6 --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_history.test @@ -0,0 +1,8 @@ +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --history=foo --backup --target-dir=$targetdir; +--enable_result_log +SELECT COUNT(*) FROM PERCONA_SCHEMA.xtrabackup_history; +DROP TABLE PERCONA_SCHEMA.xtrabackup_history; +DROP DATABASE PERCONA_SCHEMA; +rmdir $targetdir; diff --git a/mysql-test/suite/mariabackup/xb_page_compress.result b/mysql-test/suite/mariabackup/xb_page_compress.result new file mode 100644 index 00000000000..7380856f394 --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_page_compress.result @@ -0,0 +1,28 @@ +CREATE TABLE t1(c1 INT, b CHAR(20)) ENGINE=INNODB PAGE_COMPRESSED=1; +CREATE PROCEDURE innodb_insert_proc (REPEAT_COUNT INT) +BEGIN +DECLARE CURRENT_NUM INT; +SET CURRENT_NUM = 0; +WHILE CURRENT_NUM < REPEAT_COUNT DO +INSERT INTO t1 VALUES(CURRENT_NUM,'TESTING..'); +SET CURRENT_NUM = CURRENT_NUM + 1; +END WHILE; +END// +COMMIT; +SET AUTOCOMMIT=0; +CALL innodb_insert_proc(5000); +COMMIT; +SELECT (VARIABLE_VALUE >= 0) AS HAVE_COMPRESSED_PAGES +FROM INFORMATION_SCHEMA.GLOBAL_STATUS +WHERE VARIABLE_NAME = 'INNODB_NUM_PAGES_PAGE_COMPRESSED'; +HAVE_COMPRESSED_PAGES +1 +# xtrabackup backup +# xtrabackup prepare +ALTER TABLE t1 DISCARD TABLESPACE; +ALTER TABLE t1 IMPORT TABLESPACE; +SELECT COUNT(*) FROM t1; +COUNT(*) +5000 +DROP PROCEDURE innodb_insert_proc; +DROP TABLE t1; diff --git a/mysql-test/suite/mariabackup/xb_page_compress.test b/mysql-test/suite/mariabackup/xb_page_compress.test new file mode 100644 index 00000000000..876aa1a2791 --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_page_compress.test @@ -0,0 +1,44 @@ +CREATE TABLE t1(c1 INT, b CHAR(20)) ENGINE=INNODB PAGE_COMPRESSED=1; + +DELIMITER //; +CREATE PROCEDURE innodb_insert_proc (REPEAT_COUNT INT) +BEGIN + DECLARE CURRENT_NUM INT; + SET CURRENT_NUM = 0; + WHILE CURRENT_NUM < REPEAT_COUNT DO + INSERT INTO t1 VALUES(CURRENT_NUM,'TESTING..'); + SET CURRENT_NUM = CURRENT_NUM + 1; + END WHILE; +END// +DELIMITER ;// +COMMIT; + +SET AUTOCOMMIT=0; +CALL innodb_insert_proc(5000); +COMMIT; + +SELECT (VARIABLE_VALUE >= 0) AS HAVE_COMPRESSED_PAGES + FROM INFORMATION_SCHEMA.GLOBAL_STATUS + WHERE VARIABLE_NAME = 'INNODB_NUM_PAGES_PAGE_COMPRESSED'; + +echo # xtrabackup backup; +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; + +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup "--tables=test.*1" --target-dir=$targetdir; +echo # xtrabackup prepare; +exec $XTRABACKUP --prepare --export --target-dir=$targetdir; +--enable_result_log + +ALTER TABLE t1 DISCARD TABLESPACE; +let $MYSQLD_DATADIR= `select @@datadir`; +copy_file $targetdir/test/t1.ibd $MYSQLD_DATADIR/test/t1.ibd; +copy_file $targetdir/test/t1.cfg $MYSQLD_DATADIR/test/t1.cfg; +ALTER TABLE t1 IMPORT TABLESPACE; + +SELECT COUNT(*) FROM t1; + +DROP PROCEDURE innodb_insert_proc; +DROP TABLE t1; +rmdir $targetdir; + diff --git a/mysql-test/suite/mariabackup/xb_partition.result b/mysql-test/suite/mariabackup/xb_partition.result new file mode 100644 index 00000000000..f5b6ae0b24d --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_partition.result @@ -0,0 +1,64 @@ +CREATE TABLE t1(a INT) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1), (2), (3); +CREATE TABLE t2(a INT) ENGINE=InnoDB; +INSERT INTO t2 VALUES (4), (5), (6); +CREATE TABLE p ( +a int +) ENGINE=InnoDB +PARTITION BY RANGE (a) +(PARTITION p0 VALUES LESS THAN (100), +PARTITION p1 VALUES LESS THAN (200), +PARTITION p2 VALUES LESS THAN (300), +PARTITION p3 VALUES LESS THAN (400)); +INSERT INTO p VALUES (1), (101), (201), (301); +CREATE TABLE isam_t1(a INT) ENGINE=MyISAM; +INSERT INTO isam_t1 VALUES (1), (2), (3); +CREATE TABLE isam_t2(a INT) ENGINE=MyISAM; +INSERT INTO isam_t2 VALUES (4), (5), (6); +CREATE TABLE isam_p ( +a int +) ENGINE=MyISAM +PARTITION BY RANGE (a) +(PARTITION p0 VALUES LESS THAN (100), +PARTITION p1 VALUES LESS THAN (200), +PARTITION p2 VALUES LESS THAN (300), +PARTITION p3 VALUES LESS THAN (400)); +INSERT INTO isam_p VALUES (1), (101), (201), (301); +DROP TABLE t1; +DROP TABLE t2; +CREATE TABLE t2(a INT) ENGINE=InnoDB; +INSERT INTO t2 VALUES (40), (50), (60); +ALTER TABLE p DROP PARTITION p0; +ALTER TABLE p DROP PARTITION p1; +ALTER TABLE p ADD PARTITION (PARTITION p4 VALUES LESS THAN (500)); +ALTER TABLE p ADD PARTITION (PARTITION p5 VALUES LESS THAN (600)); +INSERT INTO p VALUES (401), (501); +DROP TABLE isam_t1; +DROP TABLE isam_t2; +CREATE TABLE isam_t2(a INT) ENGINE=MyISAM; +INSERT INTO isam_t2 VALUES (40), (50), (60); +ALTER TABLE isam_p DROP PARTITION p0; +ALTER TABLE isam_p DROP PARTITION p1; +ALTER TABLE isam_p ADD PARTITION (PARTITION p4 VALUES LESS THAN (500)); +ALTER TABLE isam_p ADD PARTITION (PARTITION p5 VALUES LESS THAN (600)); +INSERT INTO isam_p VALUES (401), (501); +# shutdown server +# remove datadir +# xtrabackup move back +# restart server +SELECT * from p; +a +201 +301 +401 +501 +SELECT * from isam_p; +a +201 +301 +401 +501 +DROP TABLE isam_p; +DROP TABLE isam_t2; +DROP TABLE p; +DROP TABLE t2; diff --git a/mysql-test/suite/mariabackup/xb_partition.test b/mysql-test/suite/mariabackup/xb_partition.test new file mode 100644 index 00000000000..f051a52edbc --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_partition.test @@ -0,0 +1,87 @@ +--source include/have_partition.inc + +CREATE TABLE t1(a INT) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1), (2), (3); + +CREATE TABLE t2(a INT) ENGINE=InnoDB; +INSERT INTO t2 VALUES (4), (5), (6); + +CREATE TABLE p ( + a int +) ENGINE=InnoDB +PARTITION BY RANGE (a) +(PARTITION p0 VALUES LESS THAN (100), + PARTITION p1 VALUES LESS THAN (200), + PARTITION p2 VALUES LESS THAN (300), + PARTITION p3 VALUES LESS THAN (400)); + +INSERT INTO p VALUES (1), (101), (201), (301); + +CREATE TABLE isam_t1(a INT) ENGINE=MyISAM; +INSERT INTO isam_t1 VALUES (1), (2), (3); + +CREATE TABLE isam_t2(a INT) ENGINE=MyISAM; +INSERT INTO isam_t2 VALUES (4), (5), (6); + +CREATE TABLE isam_p ( + a int +) ENGINE=MyISAM +PARTITION BY RANGE (a) +(PARTITION p0 VALUES LESS THAN (100), + PARTITION p1 VALUES LESS THAN (200), + PARTITION p2 VALUES LESS THAN (300), + PARTITION p3 VALUES LESS THAN (400)); + +INSERT INTO isam_p VALUES (1), (101), (201), (301); + +let $targetdir=$MYSQLTEST_VARDIR/tmp; + +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --no-timestamp --backup --target-dir=$targetdir/full; +--enable_result_log + +DROP TABLE t1; +DROP TABLE t2; +CREATE TABLE t2(a INT) ENGINE=InnoDB; +INSERT INTO t2 VALUES (40), (50), (60); + +ALTER TABLE p DROP PARTITION p0; +ALTER TABLE p DROP PARTITION p1; +ALTER TABLE p ADD PARTITION (PARTITION p4 VALUES LESS THAN (500)); +ALTER TABLE p ADD PARTITION (PARTITION p5 VALUES LESS THAN (600)); + +INSERT INTO p VALUES (401), (501); + + +DROP TABLE isam_t1; +DROP TABLE isam_t2; +CREATE TABLE isam_t2(a INT) ENGINE=MyISAM; + +INSERT INTO isam_t2 VALUES (40), (50), (60); + +ALTER TABLE isam_p DROP PARTITION p0; +ALTER TABLE isam_p DROP PARTITION p1; +ALTER TABLE isam_p ADD PARTITION (PARTITION p4 VALUES LESS THAN (500)); +ALTER TABLE isam_p ADD PARTITION (PARTITION p5 VALUES LESS THAN (600)); + +INSERT INTO isam_p VALUES (401), (501); + +--disable_result_log +exec $INNOBACKUPEX --defaults-file=$MYSQLTEST_VARDIR/my.cnf --incremental --no-timestamp --incremental-basedir=$targetdir/full $targetdir/inc; +exec $INNOBACKUPEX --defaults-file=$MYSQLTEST_VARDIR/my.cnf --apply-log --redo-only $targetdir/full; +exec $INNOBACKUPEX --defaults-file=$MYSQLTEST_VARDIR/my.cnf --apply-log --redo-only --incremental-dir=$targetdir/inc $targetdir/full; +exec $INNOBACKUPEX --defaults-file=$MYSQLTEST_VARDIR/my.cnf --apply-log $targetdir/full; + +let $targetdir=$targetdir/full; +-- source include/restart_and_restore.inc +--enable_result_log + +SELECT * from p; +SELECT * from isam_p; + +DROP TABLE isam_p; +DROP TABLE isam_t2; +DROP TABLE p; +DROP TABLE t2; +rmdir $MYSQLTEST_VARDIR/tmp/full; +rmdir $MYSQLTEST_VARDIR/tmp/inc; diff --git a/mysql-test/suite/mariabackup/xbstream.result b/mysql-test/suite/mariabackup/xbstream.result new file mode 100644 index 00000000000..f340fedb861 --- /dev/null +++ b/mysql-test/suite/mariabackup/xbstream.result @@ -0,0 +1,13 @@ +CREATE TABLE t(i INT) ENGINE INNODB; +INSERT INTO t VALUES(1); +# xtrabackup backup to stream +# xbstream extract +# xtrabackup prepare +# shutdown server +# remove datadir +# xtrabackup move back +# restart server +SELECT * FROM t; +i +1 +DROP TABLE t; diff --git a/mysql-test/suite/mariabackup/xbstream.test b/mysql-test/suite/mariabackup/xbstream.test new file mode 100644 index 00000000000..06e5685276c --- /dev/null +++ b/mysql-test/suite/mariabackup/xbstream.test @@ -0,0 +1,22 @@ +CREATE TABLE t(i INT) ENGINE INNODB; +INSERT INTO t VALUES(1); + +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; +mkdir $targetdir; +let $streamfile=$MYSQLTEST_VARDIR/tmp/backup.xb; + +echo # xtrabackup backup to stream; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --stream=xbstream > $streamfile 2>$targetdir/backup_stream.log; +echo # xbstream extract; +--disable_result_log +exec $XBSTREAM -x -C $targetdir --parallel=16 < $streamfile; + +echo # xtrabackup prepare; +exec $XTRABACKUP --prepare --target-dir=$targetdir; + +-- source include/restart_and_restore.inc +--enable_result_log +SELECT * FROM t; +DROP TABLE t; +rmdir $targetdir; + diff --git a/mysql-test/suite/plugins/r/auth_ed25519.result b/mysql-test/suite/plugins/r/auth_ed25519.result index 5c5581d37b0..1baec60da40 100644 --- a/mysql-test/suite/plugins/r/auth_ed25519.result +++ b/mysql-test/suite/plugins/r/auth_ed25519.result @@ -27,7 +27,7 @@ PLUGIN_STATUS ACTIVE PLUGIN_TYPE AUTHENTICATION PLUGIN_TYPE_VERSION 2.1 PLUGIN_LIBRARY auth_ed25519.so -PLUGIN_LIBRARY_VERSION 1.12 +PLUGIN_LIBRARY_VERSION 1.13 PLUGIN_AUTHOR Sergei Golubchik PLUGIN_DESCRIPTION Elliptic curve ED25519 based authentication PLUGIN_LICENSE GPL diff --git a/mysql-test/suite/plugins/r/cracklib_password_check.result b/mysql-test/suite/plugins/r/cracklib_password_check.result index 479b4b00698..6b4e30b3d81 100644 --- a/mysql-test/suite/plugins/r/cracklib_password_check.result +++ b/mysql-test/suite/plugins/r/cracklib_password_check.result @@ -6,7 +6,7 @@ PLUGIN_STATUS ACTIVE PLUGIN_TYPE PASSWORD VALIDATION PLUGIN_TYPE_VERSION 1.0 PLUGIN_LIBRARY cracklib_password_check.so -PLUGIN_LIBRARY_VERSION 1.12 +PLUGIN_LIBRARY_VERSION 1.13 PLUGIN_AUTHOR Sergei Golubchik PLUGIN_DESCRIPTION Password validation via CrackLib PLUGIN_LICENSE GPL diff --git a/mysql-test/suite/plugins/r/show_all_plugins.result b/mysql-test/suite/plugins/r/show_all_plugins.result index 7ed26b8aef6..4471011b660 100644 --- a/mysql-test/suite/plugins/r/show_all_plugins.result +++ b/mysql-test/suite/plugins/r/show_all_plugins.result @@ -4,8 +4,8 @@ Variable_name Value Opened_plugin_libraries 0 select * from information_schema.all_plugins where plugin_library='ha_example.so'; PLUGIN_NAME PLUGIN_VERSION PLUGIN_STATUS PLUGIN_TYPE PLUGIN_TYPE_VERSION PLUGIN_LIBRARY PLUGIN_LIBRARY_VERSION PLUGIN_AUTHOR PLUGIN_DESCRIPTION PLUGIN_LICENSE LOAD_OPTION PLUGIN_MATURITY PLUGIN_AUTH_VERSION -EXAMPLE 0.1 NOT INSTALLED STORAGE ENGINE MYSQL_VERSION_ID ha_example.so 1.12 Brian Aker, MySQL AB Example storage engine GPL OFF Experimental 0.1 -UNUSABLE 3.14 NOT INSTALLED DAEMON MYSQL_VERSION_ID ha_example.so 1.12 Sergei Golubchik Unusable Daemon GPL OFF Experimental 3.14.15.926 +EXAMPLE 0.1 NOT INSTALLED STORAGE ENGINE MYSQL_VERSION_ID ha_example.so 1.13 Brian Aker, MySQL AB Example storage engine GPL OFF Experimental 0.1 +UNUSABLE 3.14 NOT INSTALLED DAEMON MYSQL_VERSION_ID ha_example.so 1.13 Sergei Golubchik Unusable Daemon GPL OFF Experimental 3.14.15.926 show status like '%libraries%'; Variable_name Value Opened_plugin_libraries 1 diff --git a/mysql-test/suite/plugins/r/simple_password_check.result b/mysql-test/suite/plugins/r/simple_password_check.result index 11385bd6b01..672d0107492 100644 --- a/mysql-test/suite/plugins/r/simple_password_check.result +++ b/mysql-test/suite/plugins/r/simple_password_check.result @@ -6,7 +6,7 @@ PLUGIN_STATUS ACTIVE PLUGIN_TYPE PASSWORD VALIDATION PLUGIN_TYPE_VERSION 1.0 PLUGIN_LIBRARY simple_password_check.so -PLUGIN_LIBRARY_VERSION 1.12 +PLUGIN_LIBRARY_VERSION 1.13 PLUGIN_AUTHOR Sergei Golubchik PLUGIN_DESCRIPTION Simple password strength checks PLUGIN_LICENSE GPL diff --git a/mysql-test/t/mysql_plugin.test b/mysql-test/t/mysql_plugin.test deleted file mode 100644 index 10bc03e0f06..00000000000 --- a/mysql-test/t/mysql_plugin.test +++ /dev/null @@ -1,368 +0,0 @@ -# -# Test mysql_plugin tool -# -# This test contains test cases for testing the mysql_plugin client with -# the daemon_example plugin. Test cases include tests for: -# -# - successful enable/disable -# - incorrect paths -# - missing paths/options -# -# Implementation Notes -# -# The mysql_plugin tool now accepts --mysqld the path to mysqld server. The -# mysqld path is extracted from MYSQLD_BOOTSTRAP_CMD line. We also extract -# the path of MYSQLD_BASEDIR (where mysql exists) and use it for the errmsg -# file. The directories differ between Windows and Unix but the Perl script -# included below will pick as per platform. -# -# The test is also designed to issue the --skip directive if the location of -# the mysqld, my_print_defaults, or daemon_example.ini files cannot be found. -# - ---source include/not_embedded.inc - -# Add the datadir, basedir, plugin_dir to the bootstrap command -let $MYSQLD_DATADIR= `select @@datadir`; -let $MYSQL_BASEDIR= `select @@basedir`; -let $MYSQL_ERRMSG_BASEDIR=`select @@lc_messages_dir`; -let $PLUGIN_DIR=`select @@plugin_dir`; - ---disable_abort_on_error - -# Perl script to extract the location of the basedir from environment -# variables. This is needed to ensure the test will run on the PB machines -# designed to test release as well as debug builds. It also checks for the -# location of the my_print_defaults and daemon_example.ini files. - -perl; -use File::Basename; - my ($mysqld)= split " ", $ENV{MYSQLD_BOOTSTRAP_CMD}; - my $mysqld_basedir=dirname($mysqld); - my $my_print_defaults= $ENV{MYSQL_MY_PRINT_DEFAULTS}; - my $my_print_defaults_basedir=dirname($my_print_defaults); - my $daemonexample_ini= "$ENV{DAEMONEXAMPLE_DIR}/daemon_example.ini"; - my $plugindir_ini= "$ENV{PLUGIN_DIR}/daemon_example.ini"; - my $notfound= ""; - open(FILE, ">", "$ENV{MYSQL_TMP_DIR}/mysqld.inc") or die; - print FILE "let \$MYSQLD_BASEDIR= $mysqld_basedir;\n"; - print FILE "let \$MYSQL_MY_PRINT_DEFAULTS_BASEDIR= $my_print_defaults_basedir;\n"; - if ((!-e $daemonexample_ini) || (!-r $daemonexample_ini)) - { - print FILE "let \$DAEMONEXAMPLE_DIR= $not_found;\n"; - } - if ((!-e $plugindir_ini) || (!-r $plugindir_ini)) - { - print FILE "let \$PLUGIN_DIR= $not_found;\n"; - } - close FILE; -EOF - - -source $MYSQL_TMP_DIR/mysqld.inc; -remove_file $MYSQL_TMP_DIR/mysqld.inc; - -# The mysql_plugin tool expects a directory structure like in the installed -# mysql version, so errmsg.sys will be copied to "basedir/share", we create -# and remove this structure. - ---mkdir $MYSQLD_BASEDIR/share ---mkdir $MYSQLD_BASEDIR/share/mysql ---copy_file $MYSQL_ERRMSG_BASEDIR/english/errmsg.sys $MYSQLD_BASEDIR/share/errmsg.sys ---copy_file $MYSQL_ERRMSG_BASEDIR/english/errmsg.sys $MYSQLD_BASEDIR/share/mysql/errmsg.sys - -# The mysql_plugin tool now accepts --my-print-defaults which points to the -# executable my_print_defaults.exe we can get this path from the variable -# $MYSQL_MY_PRINT_DEFAULTS. - -# Check for my_print_defaults location. Skip if not found. -if ($MYSQL_MY_PRINT_DEFAULTS_BASEDIR == '') -{ - --skip Test requires known location of my_print_defaults executable. -} - -# Check for mysqld location. Skip if not found. -if ($MYSQLD == '') -{ - --skip Test requires known location of mysqld executable. -} - -# Check for daemon_example.ini location. Skip if not found in either -# the plugin_dir path or the daemon_example_dir path. -if ($PLUGIN_DIR == '') -{ - if ($DAEMONEXAMPLE_DIR == '') - { - --skip Test requires known location of daemon_example.ini file. - } - let $PLUGIN_DIR = $DAEMONEXAMPLE_DIR; -} - -# Build client command for reuse. - -let $MYSQL_PLUGIN_CMD= $MYSQL_PLUGIN --datadir=$MYSQLD_DATADIR --basedir=$MYSQLD_BASEDIR --plugin-dir=$PLUGIN_DIR --mysqld=$MYSQLD_BASEDIR --my-print-defaults=$MYSQL_MY_PRINT_DEFAULTS_BASEDIR; - ---echo # ---echo # Ensure the plugin isn't loaded. ---echo # -SELECT * FROM mysql.plugin WHERE dl like 'libdaemon%' ORDER BY name; - ---echo # ---echo # Enable the plugin... ---echo # - ---exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect ---shutdown_server 10 ---source include/wait_until_disconnected.inc - -# -# Enable the plugin -# ---exec $MYSQL_PLUGIN_CMD ENABLE daemon_example - -# -# Ensure enabling an enabled plugin doesn't fail ---exec $MYSQL_PLUGIN_CMD ENABLE daemon_example - -# -# Restart the server -# - ---exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect ---enable_reconnect ---source include/wait_until_connected_again.inc - ---echo # ---echo # Simulate loading a plugin libary with multiple entry points. ---echo # This will test the DISABLE to ensure all rows are removed. ---echo # ---replace_regex /\.dll/.so/ -eval INSERT INTO mysql.plugin VALUES ('wicky', '$DAEMONEXAMPLE'); ---replace_regex /\.dll/.so/ -eval INSERT INTO mysql.plugin VALUES ('wacky', '$DAEMONEXAMPLE'); ---replace_regex /\.dll/.so/ -eval INSERT INTO mysql.plugin VALUES ('wonky', '$DAEMONEXAMPLE'); - ---echo # ---echo # Ensure the plugin is now loaded. ---echo # ---replace_regex /\.dll/.so/ -SELECT * FROM mysql.plugin WHERE dl like 'libdaemon%' ORDER BY name; - ---exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect ---shutdown_server 10 ---source include/wait_until_disconnected.inc - -# -# Disable the plugin - to remove winky, wonky entries -# ---exec $MYSQL_PLUGIN_CMD DISABLE daemon_example - -# -# Enable the plugin again -# ---exec $MYSQL_PLUGIN_CMD ENABLE daemon_example - -# -# Restart the server -# ---exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect ---enable_reconnect ---source include/wait_until_connected_again.inc - ---echo # ---echo # Ensure the plugin is loaded. ---echo # ---replace_regex /\.dll/.so/ -SELECT * FROM mysql.plugin WHERE dl like '%libdaemon%' ORDER BY name; - ---exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect ---shutdown_server 10 ---source include/wait_until_disconnected.inc - -# To test the case where the same plugin is reloaded with a different soname, -# we must copy the example daemon to a new location renaming it. - -let $DAEMON_RELOAD = lib$DAEMONEXAMPLE; ---copy_file $PLUGIN_DIR/$DAEMONEXAMPLE $PLUGIN_DIR/$DAEMON_RELOAD ---copy_file include/libdaemon_example.ini $PLUGIN_DIR/libdaemon_example.ini - -# Now reload it and see that it is a different name. ---exec $MYSQL_PLUGIN_CMD ENABLE libdaemon_example - -# -# Restart the server -# ---exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect ---enable_reconnect ---source include/wait_until_connected_again.inc - ---echo # ---echo # Ensure the plugin is replaced. ---echo # ---replace_regex /\.dll/.so/ -SELECT * FROM mysql.plugin WHERE dl like '%libdaemon%' ORDER BY name; - ---echo # ---echo # Disable the plugin... ---echo # - ---exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect ---shutdown_server 10 ---source include/wait_until_disconnected.inc - -# -# Disable the plugin -# ---exec $MYSQL_PLUGIN_CMD DISABLE libdaemon_example - -# Remove files for last test case. - ---remove_file $PLUGIN_DIR/$DAEMON_RELOAD ---remove_file $DAEMONEXAMPLE_DIR/libdaemon_example.ini - -# -# Restart the server -# ---exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect ---enable_reconnect ---source include/wait_until_connected_again.inc - ---echo # ---echo # Ensure the plugin isn't loaded. ---echo # -SELECT * FROM mysql.plugin WHERE dl like '%libdaemon%' ORDER BY name; - -# -# Stop the server for error conditions -# - ---exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect ---shutdown_server 10 ---source include/wait_until_disconnected.inc - ---echo # ---echo # Attempt to load non-existant plugin ---echo # ---error 1,2,256 ---exec $MYSQL_PLUGIN_CMD DISABLE NOT_THERE_AT_ALL 2>&1 - ---echo # ---echo # Attempt to use non-existant plugin.ini file ---echo # ---error 1,2,7,256 ---exec $MYSQL_PLUGIN_CMD DISABLE daemon_example --plugin-ini=/NOT/THERE/pi.ini 2>&1 - ---echo # ---echo # Attempt to omit the plugin ---echo # ---error 1,2,256 ---exec $MYSQL_PLUGIN_CMD DISABLE 2>&1 - ---echo # ---echo # Attempt to omit DISABLE|ENABLE ---echo # ---error 1,2,256 ---exec $MYSQL_PLUGIN_CMD daemon_example 2>&1 - ---echo # ---echo # Attempt to use bad paths - datadir ---echo # -let $MYSQL_PLUGIN_CMD= $MYSQL_PLUGIN -n --datadir=/data_not_there/ --basedir=$MYSQL_BASEDIR --plugin-dir=$PLUGIN_DIR --mysqld=$MYSQLD_BASEDIR --my-print-defaults=$MYSQL_MY_PRINT_DEFAULTS_BASEDIR; ---error 1,2,256 ---exec $MYSQL_PLUGIN_CMD DISABLE daemon_example 2>&1 - ---echo # ---echo # Attempt to use bad paths - basedir ---echo # -let $MYSQL_PLUGIN_CMD= $MYSQL_PLUGIN -n --datadir=$MYSQLD_DATADIR --basedir=/basedir_not_there/ --plugin-dir=$PLUGIN_DIR --mysqld=$MYSQLD_BASEDIR --my-print-defaults=$MYSQL_MY_PRINT_DEFAULTS_BASEDIR; -replace_result "/basedir_not_there//" "/basedir_not_there/"; ---error 1,2,256 ---exec $MYSQL_PLUGIN_CMD DISABLE daemon_example 2>&1 - ---echo # ---echo # Attempt to use bad paths - plugin_dir ---echo # -let $MYSQL_PLUGIN_CMD= $MYSQL_PLUGIN -n --datadir=$MYSQLD_DATADIR --basedir=$MYSQL_BASEDIR --plugin-dir=/plugin_not_there/ --mysqld=$MYSQLD_BASEDIR --my-print-defaults=$MYSQL_MY_PRINT_DEFAULTS_BASEDIR; ---error 1,2,256 ---exec $MYSQL_PLUGIN_CMD DISABLE daemon_example 2>&1 - ---echo # ---echo # Attempt to use bad paths - mysqld ---echo # -let $MYSQL_PLUGIN_CMD= $MYSQL_PLUGIN -n --datadir=$MYSQLD_DATADIR --basedir=$MYSQL_BASEDIR --plugin-dir=$PLUGIN_DIR --mysqld=/mysqld_not_there/ --my-print-defaults=$MYSQL_MY_PRINT_DEFAULTS_BASEDIR; ---error 1,2,256 ---exec $MYSQL_PLUGIN_CMD DISABLE daemon_example 2>&1 - ---echo # ---echo # Attempt to use bad paths - my_print_defaults ---echo # -let $MYSQL_PLUGIN_CMD= $MYSQL_PLUGIN -n --datadir=$MYSQLD_DATADIR --basedir=$MYSQL_BASEDIR --plugin-dir=$PLUGIN_DIR --mysqld=$MYSQLD_BASEDIR --my-print-defaults=/my_print_defaults_not_there/; ---error 1,2,256 ---exec $MYSQL_PLUGIN_CMD DISABLE daemon_example 2>&1 - - ---echo # ---echo # Missing library ---echo # -let $MYSQL_PLUGIN_CMD= $MYSQL_PLUGIN -n --datadir=$MYSQLD_DATADIR --basedir=$MYSQL_BASEDIR --plugin-dir=$PLUGIN_DIR --plugin-ini=$MYSQL_TEST_DIR/include/daemon_example_bad_soname.ini --mysqld=$MYSQLD_BASEDIR --my-print-defaults=$MYSQL_MY_PRINT_DEFAULTS_BASEDIR; ---error 1,2,256 ---exec $MYSQL_PLUGIN_CMD DISABLE daemon_example 2>&1 - ---echo # ---echo # Bad format for config file ---echo # -let $MYSQL_PLUGIN_CMD= $MYSQL_PLUGIN -n --datadir=$MYSQLD_DATADIR --basedir=$MYSQL_BASEDIR --plugin-dir=$PLUGIN_DIR --plugin-ini=$MYSQL_TEST_DIR/include/daemon_example_bad_format.ini --mysqld=$MYSQLD_BASEDIR --my-print-defaults=$MYSQL_MY_PRINT_DEFAULTS_BASEDIR; ---error 1,2,256 ---exec $MYSQL_PLUGIN_CMD DISABLE daemon_example 2>&1 - ---echo # ---echo # Missing base_dir option ---echo # -let $MYSQL_PLUGIN_CMD= $MYSQL_PLUGIN -n --datadir=$MYSQLD_DATADIR --plugin-dir=$PLUGIN_DIR --mysqld=$MYSQLD_BASEDIR --my-print-defaults=$MYSQL_MY_PRINT_DEFAULTS_BASEDIR; ---error 1,2,139,256 ---exec $MYSQL_PLUGIN_CMD DISABLE daemon_example 2>&1 - ---echo # ---echo # Missing data_dir option ---echo # -let $MYSQL_PLUGIN_CMD= $MYSQL_PLUGIN -n --basedir=$MYSQL_BASEDIR --plugin-dir=$PLUGIN_DIR --mysqld=$MYSQLD_BASEDIR --my-print-defaults=$MYSQL_MY_PRINT_DEFAULTS_BASEDIR; ---error 1,2,139,256 ---exec $MYSQL_PLUGIN_CMD DISABLE daemon_example 2>&1 - ---echo # ---echo # Missing plugin_dir option ---echo # -let $MYSQL_PLUGIN_CMD= $MYSQL_PLUGIN -n --datadir=$MYSQL_DATADIR --basedir=$MYSQL_BASEDIR --mysqld=$MYSQLD_BASEDIR --my-print-defaults=$MYSQL_MY_PRINT_DEFAULTS_BASEDIR; ---error 1,2,139,256 ---exec $MYSQL_PLUGIN_CMD DISABLE daemon_example 2>&1 - ---echo # ---echo # Show the help. ---echo # -replace_result $MYSQL_PLUGIN mysql_plugin; ---replace_regex /Ver [0-9.]+ Distrib [0-9.]+/Ver V.V.VV Distrib XX.XX.XX/ /XX-m[0-9]+/XX/ /XX[a-z]/XX/ ---exec $MYSQL_PLUGIN --help - -replace_result $MYSQL_PLUGIN mysql_plugin; ---replace_regex /Ver [0-9.]+ Distrib [0-9.]+/Ver V.V.VV Distrib XX.XX.XX/ /XX-m[0-9]+/XX/ /XX[a-z]/XX/ ---exec $MYSQL_PLUGIN --version - -# -# Restart the server -# ---exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect ---enable_reconnect ---source include/wait_until_connected_again.inc - -# -# Cleanup - ---remove_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect - -# Cleanup the share folder in the binary path. ---remove_file $MYSQLD_BASEDIR/share/errmsg.sys ---rmdir $MYSQLD_BASEDIR/share/mysql ---rmdir $MYSQLD_BASEDIR/share - ---enable_abort_on_error diff --git a/mysql-test/t/mysqltest.test b/mysql-test/t/mysqltest.test index b2706a8459f..4cc480b473e 100644 --- a/mysql-test/t/mysqltest.test +++ b/mysql-test/t/mysqltest.test @@ -406,7 +406,7 @@ select 3 from t1 ; --disable_abort_on_error ONCE garbage; --disable_abort_on_error ONCE ---remove_file DoesNotExist +--remove_file $MYSQLTEST_VARDIR/DoesNotExist --disable_result_log select 2; @@ -1939,8 +1939,6 @@ remove_file $MYSQLTEST_VARDIR/tmp/zero_length_file.result; remove_file $MYSQLTEST_VARDIR/tmp/zero_length_file.reject; --error 0,1 remove_file $MYSQLTEST_VARDIR/tmp/zero_length_file.log; ---error 0,1 -remove_file $MYSQL_TEST_DIR/r/zero_length_file.reject; --enable_warnings # @@ -2193,7 +2191,7 @@ drop table t1; --exec echo "remove_file ;" | $MYSQL_TEST 2>&1 --error 1 -remove_file non_existing_file; +remove_file $MYSQLTEST_VARDIR/non_existing_file; --enable_warnings # ---------------------------------------------------------------------------- @@ -2204,10 +2202,10 @@ remove_file non_existing_file; --exec echo "remove_files_wildcard ;" | $MYSQL_TEST 2>&1 --error 1 -remove_files_wildcard non_existing_dir; +remove_files_wildcard $MYSQLTEST_VARDIR/non_existing_dir; --error 1 -remove_files_wildcard non_existing_dir non_existing_file; +remove_files_wildcard $MYSQLTEST_VARDIR/non_existing_dir non_existing_file; # ---------------------------------------------------------------------------- # test for write_file @@ -2216,7 +2214,7 @@ remove_files_wildcard non_existing_dir non_existing_file; --exec echo "write_file ;" | $MYSQL_TEST 2>&1 --error 1 ---exec echo "write_file filename ;" | $MYSQL_TEST 2>&1 +--exec echo "write_file $MYSQLTEST_VARDIR/filename ;" | $MYSQL_TEST 2>&1 # Comment out this test as it confuses cmd.exe with unmatched " #--error 1 @@ -2462,19 +2460,19 @@ remove_file $MYSQLTEST_VARDIR/tmp/file1.tmp; --exec echo "chmod ;" | $MYSQL_TEST 2>&1 --error 1 ---exec echo "chmod 0 from_file;" | $MYSQL_TEST 2>&1 +--exec echo "chmod 0 $MYSQLTEST_VARDIR/from_file;" | $MYSQL_TEST 2>&1 --error 1 ---exec echo "chmod 08 from_file;" | $MYSQL_TEST 2>&1 +--exec echo "chmod 08 $MYSQLTEST_VARDIR/from_file;" | $MYSQL_TEST 2>&1 --error 1 ---exec echo "chmod from_file;" | $MYSQL_TEST 2>&1 +--exec echo "chmod $MYSQLTEST_VARDIR/from_file;" | $MYSQL_TEST 2>&1 --error 1 ---exec echo "chmod ABZD from_file;" | $MYSQL_TEST 2>&1 +--exec echo "chmod ABZD $MYSQLTEST_VARDIR/from_file;" | $MYSQL_TEST 2>&1 --error 1 ---exec echo "chmod 06789 from_file;" | $MYSQL_TEST 2>&1 +--exec echo "chmod 06789 $MYSQLTEST_VARDIR/from_file;" | $MYSQL_TEST 2>&1 # ---------------------------------------------------------------------------- @@ -2875,8 +2873,6 @@ list_files_append_file $MYSQLTEST_VARDIR/tmp/testdir/file2.txt $MYSQLTEST_VARDIR list_files_write_file $MYSQLTEST_VARDIR/tmp/testdir/file2.txt $MYSQLTEST_VARDIR/tmp/testdir file?.txt; list_files_append_file $MYSQLTEST_VARDIR/tmp/testdir/file3.txt $MYSQLTEST_VARDIR/tmp/testdir file*.txt; diff_files $MYSQLTEST_VARDIR/tmp/testdir/file2.txt $MYSQLTEST_VARDIR/tmp/testdir/file3.txt; ---error 1 -rmdir $MYSQLTEST_VARDIR/tmp/testdir; cat_file $MYSQLTEST_VARDIR/tmp/testdir/file3.txt; diff --git a/mysql-test/t/ssl_7937.combinations b/mysql-test/t/ssl_7937.combinations index 46a45686a9b..71b134e229a 100644 --- a/mysql-test/t/ssl_7937.combinations +++ b/mysql-test/t/ssl_7937.combinations @@ -1,3 +1,8 @@ +[x509v3] +--loose-enable-ssl +--loose-ssl-cert=$MYSQL_TEST_DIR/std_data/serversan-cert.pem +--loose-ssl-key=$MYSQL_TEST_DIR/std_data/serversan-key.pem + [ssl] --loose-enable-ssl diff --git a/mysys_ssl/my_crypt.cc b/mysys_ssl/my_crypt.cc index 5411a908bf8..2ab38711d0f 100644 --- a/mysys_ssl/my_crypt.cc +++ b/mysys_ssl/my_crypt.cc @@ -269,6 +269,32 @@ int my_aes_crypt(enum my_aes_mode mode, int flags, return res1 ? res1 : res2; } + +/* + calculate the length of the cyphertext from the length of the plaintext + for different AES encryption modes with padding enabled. + Without padding (ENCRYPTION_FLAG_NOPAD) cyphertext has the same length + as the plaintext +*/ +unsigned int my_aes_get_size(enum my_aes_mode mode __attribute__((unused)), unsigned int source_length) +{ +#ifdef HAVE_EncryptAes128Ctr + if (mode == MY_AES_CTR) + return source_length; +#ifdef HAVE_EncryptAes128Gcm + if (mode == MY_AES_GCM) + return source_length + MY_AES_BLOCK_SIZE; +#endif +#endif + return (source_length / MY_AES_BLOCK_SIZE + 1) * MY_AES_BLOCK_SIZE; +} + + +unsigned int my_aes_ctx_size(enum my_aes_mode) +{ + return MY_AES_CTX_SIZE; +} + #ifdef HAVE_YASSL #include <random.hpp> int my_random_bytes(uchar* buf, int num) diff --git a/plugin/auth_gssapi/gssapi_server.cc b/plugin/auth_gssapi/gssapi_server.cc index ac75a4f1593..50c34ecc573 100644 --- a/plugin/auth_gssapi/gssapi_server.cc +++ b/plugin/auth_gssapi/gssapi_server.cc @@ -44,26 +44,30 @@ static char* get_default_principal_name() if(krb5_init_context(&context)) { - sql_print_warning("GSSAPI plugin : krb5_init_context failed"); + my_printf_error(0, "GSSAPI plugin : krb5_init_context failed", + ME_ERROR_LOG | ME_WARNING); goto cleanup; } if (krb5_sname_to_principal(context, NULL, "mariadb", KRB5_NT_SRV_HST, &principal)) { - sql_print_warning("GSSAPI plugin : krb5_sname_to_principal failed"); + my_printf_error(0, "GSSAPI plugin : krb5_sname_to_principal failed", + ME_ERROR_LOG | ME_WARNING); goto cleanup; } if (krb5_unparse_name(context, principal, &unparsed_name)) { - sql_print_warning("GSSAPI plugin : krb5_unparse_name failed"); + my_printf_error(0, "GSSAPI plugin : krb5_unparse_name failed", + ME_ERROR_LOG | ME_WARNING); goto cleanup; } /* Check for entry in keytab */ if (krb5_kt_read_service_key(context, NULL, principal, 0, (krb5_enctype)0, &key)) { - sql_print_warning("GSSAPI plugin : default principal '%s' not found in keytab", unparsed_name); + my_printf_error(0, "GSSAPI plugin : default principal '%s' not found in keytab", + ME_ERROR_LOG | ME_WARNING, unparsed_name); goto cleanup; } @@ -100,7 +104,8 @@ int plugin_init() /* import service principal from plain text */ if(srv_principal_name && srv_principal_name[0]) { - sql_print_information("GSSAPI plugin : using principal name '%s'", srv_principal_name); + my_printf_error(0, "GSSAPI plugin : using principal name '%s'", + ME_ERROR_LOG | ME_NOTE, srv_principal_name); principal_name_buf.length= strlen(srv_principal_name); principal_name_buf.value= srv_principal_name; major= gss_import_name(&minor, &principal_name_buf, GSS_C_NT_USER_NAME, &service_name); @@ -115,8 +120,6 @@ int plugin_init() service_name= GSS_C_NO_NAME; } - - /* Check if SPN configuration is OK */ major= gss_acquire_cred(&minor, service_name, GSS_C_INDEFINITE, GSS_C_NO_OID_SET, GSS_C_ACCEPT, &cred, NULL, diff --git a/plugin/auth_gssapi/sspi_server.cc b/plugin/auth_gssapi/sspi_server.cc index 1dfd2986aaa..d2c2ae7e4b9 100644 --- a/plugin/auth_gssapi/sspi_server.cc +++ b/plugin/auth_gssapi/sspi_server.cc @@ -284,8 +284,8 @@ int plugin_init() { srv_principal_name= get_default_principal_name(); } - sql_print_information("SSPI: using principal name '%s', mech '%s'", - srv_principal_name, srv_mech_name); + my_printf_error(0, "SSPI: using principal name '%s', mech '%s'", + ME_ERROR_LOG | ME_NOTE, srv_principal_name, srv_mech_name); ret = AcquireCredentialsHandle( srv_principal_name, diff --git a/plugin/aws_key_management/CMakeLists.txt b/plugin/aws_key_management/CMakeLists.txt index 2202efe9e41..06e0565040a 100644 --- a/plugin/aws_key_management/CMakeLists.txt +++ b/plugin/aws_key_management/CMakeLists.txt @@ -86,6 +86,10 @@ ELSE() IF(NOT UUID_LIBRARIES) SKIP_AWS_PLUGIN("AWS C++ SDK requires uuid development package") ENDIF() + FIND_PACKAGE(OpenSSL) + IF(NOT OPENSSL_FOUND) + SKIP_AWS_PLUGIN("AWS C++ SDK requires openssl development package") + ENDIF() ENDIF() ENDIF() IF(MSVC) @@ -156,7 +160,7 @@ ENDIF() IF(WIN32) SET(AWS_CPP_SDK_DEPENDENCIES bcrypt winhttp wininet userenv version) ELSE() - SET(AWS_CPP_SDK_DEPENDENCIES ${SSL_LIBRARIES} ${CURL_LIBRARIES} ${UUID_LIBRARIES}) + SET(AWS_CPP_SDK_DEPENDENCIES ${OPENSSL_LIBRARIES} ${CURL_LIBRARIES} ${UUID_LIBRARIES}) ENDIF() MYSQL_ADD_PLUGIN(aws_key_management aws_key_management_plugin.cc LINK_LIBRARIES ${AWS_SDK_LIBS} ${AWS_CPP_SDK_DEPENDENCIES} diff --git a/plugin/aws_key_management/aws_key_management_plugin.cc b/plugin/aws_key_management/aws_key_management_plugin.cc index 83966b97c17..d7a948369f5 100644 --- a/plugin/aws_key_management/aws_key_management_plugin.cc +++ b/plugin/aws_key_management/aws_key_management_plugin.cc @@ -16,15 +16,14 @@ #include <my_global.h> -#include <my_pthread.h> -#include <my_sys.h> -#include <my_dir.h> +#include <typelib.h> #include <mysql/plugin_encryption.h> #include <my_crypt.h> #include <string.h> #include <stdio.h> #include <stdlib.h> #include <mysqld_error.h> +#include <my_sys.h> #include <map> #include <algorithm> #include <string> @@ -33,6 +32,10 @@ #include <sstream> #include <fstream> +#ifndef _WIN32 +#include <dirent.h> +#endif + #include <aws/core/Aws.h> #include <aws/core/client/AWSError.h> #include <aws/core/utils/logging/AWSLogging.h> @@ -48,9 +51,6 @@ using namespace std; using namespace Aws::KMS; using namespace Aws::KMS::Model; using namespace Aws::Utils::Logging; -extern void sql_print_error(const char *format, ...); -extern void sql_print_warning(const char *format, ...); -extern void sql_print_information(const char *format, ...); /* Plaintext key info struct */ @@ -90,14 +90,8 @@ static int extract_id_and_version(const char *name, uint *id, uint *ver); static unsigned int get_latest_key_version(unsigned int key_id); static unsigned int get_latest_key_version_nolock(unsigned int key_id); static int load_key(KEY_INFO *info); +static std::mutex mtx; -/* Mutex to serialize access to caches */ -static mysql_mutex_t mtx; - -#ifdef HAVE_PSI_INTERFACE -static uint mtx_key; -static PSI_mutex_info mtx_info = {&mtx_key, "mtx", 0}; -#endif static Aws::KMS::KMSClient *client; @@ -140,6 +134,33 @@ protected: } }; +/* Get list of files in current directory */ +static vector<string> traverse_current_directory() +{ + vector<string> v; +#ifdef _WIN32 + WIN32_FIND_DATA find_data; + HANDLE h= FindFirstFile("*.*", &find_data); + if (h == INVALID_HANDLE_VALUE) + return v; + do + { + v.push_back(find_data.cFileName); + } + while (FindNextFile(h, &find_data)); + FindClose(h); +#else + DIR *dir = opendir("."); + if (!dir) + return v; + struct dirent *e; + while ((e= readdir(dir))) + v.push_back(e->d_name); + closedir(dir); +#endif + return v; +} + Aws::SDKOptions sdkOptions; /* @@ -150,7 +171,6 @@ Aws::SDKOptions sdkOptions; */ static int plugin_init(void *p) { - DBUG_ENTER("plugin_init"); #ifdef HAVE_YASSL sdkOptions.cryptoOptions.initAndCleanupOpenSSL = true; @@ -175,47 +195,34 @@ static int plugin_init(void *p) client = new KMSClient(clientConfiguration); if (!client) { - sql_print_error("Can not initialize KMS client"); - DBUG_RETURN(-1); + my_printf_error(ER_UNKNOWN_ERROR, "Can not initialize KMS client", ME_ERROR_LOG | ME_WARNING); + return -1; } -#ifdef HAVE_PSI_INTERFACE - mysql_mutex_register("aws_key_management", &mtx_info, 1); -#endif - mysql_mutex_init(mtx_key, &mtx, NULL); - - MY_DIR *dirp = my_dir(".", MYF(0)); - if (!dirp) - { - sql_print_error("Can't scan current directory"); - DBUG_RETURN(-1); - } - for (unsigned int i=0; i < dirp->number_of_files; i++) + vector<string> files= traverse_current_directory(); + for (size_t i=0; i < files.size(); i++) { KEY_INFO info; - if (extract_id_and_version(dirp->dir_entry[i].name, &info.key_id, &info.key_version) == 0) + if (extract_id_and_version(files[i].c_str(), &info.key_id, &info.key_version) == 0) { key_info_cache[KEY_ID_AND_VERSION(info.key_id, info.key_version)]= info; latest_version_cache[info.key_id]= max(info.key_version, latest_version_cache[info.key_id]); } } - my_dirend(dirp); - DBUG_RETURN(0); + return 0; } static int plugin_deinit(void *p) { - DBUG_ENTER("plugin_deinit"); latest_version_cache.clear(); key_info_cache.clear(); - mysql_mutex_destroy(&mtx); delete client; ShutdownAWSLogging(); Aws::ShutdownAPI(sdkOptions); - DBUG_RETURN(0); + return 0; } /* Generate filename to store the ciphered key */ @@ -242,8 +249,7 @@ static int load_key(KEY_INFO *info) { int ret; char path[256]; - DBUG_ENTER("load_key"); - DBUG_PRINT("enter", ("id=%u,ver=%u", info->key_id, info->key_version)); + format_keyfile_name(path, sizeof(path), info->key_id, info->key_version); ret= aws_decrypt_key(path, info); if (ret) @@ -254,15 +260,15 @@ static int load_key(KEY_INFO *info) if (!ret) { - sql_print_information("AWS KMS plugin: loaded key %u, version %u, key length %u bit", + my_printf_error(ER_UNKNOWN_ERROR, "AWS KMS plugin: loaded key %u, version %u, key length %u bit", ME_ERROR_LOG | ME_NOTE, info->key_id, info->key_version,(uint)info->length*8); } else { - sql_print_warning("AWS KMS plugin: key %u, version %u could not be decrypted", + my_printf_error(ER_UNKNOWN_ERROR, "AWS KMS plugin: key %u, version %u could not be decrypted", ME_ERROR_LOG | ME_WARNING, info->key_id, info->key_version); } - DBUG_RETURN(ret); + return ret; } @@ -281,19 +287,17 @@ static int load_key(KEY_INFO *info) static unsigned int get_latest_key_version(unsigned int key_id) { unsigned int ret; - DBUG_ENTER("get_latest_key_version"); - mysql_mutex_lock(&mtx); + mtx.lock(); ret= get_latest_key_version_nolock(key_id); - mysql_mutex_unlock(&mtx); - DBUG_PRINT("info", ("key=%u,ret=%u", key_id, ret)); - DBUG_RETURN(ret); + mtx.unlock(); + return ret; } static unsigned int get_latest_key_version_nolock(unsigned int key_id) { KEY_INFO info; uint ver; - DBUG_ENTER("get_latest_key_version_nolock"); + ver= latest_version_cache[key_id]; if (ver > 0) { @@ -302,13 +306,13 @@ static unsigned int get_latest_key_version_nolock(unsigned int key_id) if (info.load_failed) { /* Decryption failed previously, don't retry */ - DBUG_RETURN(ENCRYPTION_KEY_VERSION_INVALID); + return(ENCRYPTION_KEY_VERSION_INVALID); } else if (ver > 0) { /* Key exists already, return it*/ if (info.length > 0) - DBUG_RETURN(ver); + return(ver); } else // (ver == 0) { @@ -318,18 +322,18 @@ static unsigned int get_latest_key_version_nolock(unsigned int key_id) my_printf_error(ER_UNKNOWN_ERROR, "Can't generate encryption key %u, because 'aws_key_management_master_key_id' parameter is not set", MYF(0), key_id); - DBUG_RETURN(ENCRYPTION_KEY_VERSION_INVALID); + return(ENCRYPTION_KEY_VERSION_INVALID); } if (aws_generate_datakey(key_id, 1) != 0) - DBUG_RETURN(ENCRYPTION_KEY_VERSION_INVALID); + return(ENCRYPTION_KEY_VERSION_INVALID); info.key_id= key_id; info.key_version= 1; info.length= 0; } if (load_key(&info)) - DBUG_RETURN(ENCRYPTION_KEY_VERSION_INVALID); - DBUG_RETURN(info.key_version); + return(ENCRYPTION_KEY_VERSION_INVALID); + return(info.key_version); } @@ -338,20 +342,19 @@ static unsigned int get_latest_key_version_nolock(unsigned int key_id) */ static int aws_decrypt_key(const char *path, KEY_INFO *info) { - DBUG_ENTER("aws_decrypt_key"); /* Read file content into memory */ ifstream ifs(path, ios::binary | ios::ate); if (!ifs.good()) { - sql_print_error("can't open file %s", path); - DBUG_RETURN(-1); + my_printf_error(ER_UNKNOWN_ERROR, "can't open file %s", ME_ERROR_LOG, path); + return(-1); } size_t pos = (size_t)ifs.tellg(); if (!pos || pos == SIZE_T_MAX) { - sql_print_error("invalid key file %s", path); - DBUG_RETURN(-1); + my_printf_error(ER_UNKNOWN_ERROR, "invalid key file %s", ME_ERROR_LOG, path); + return(-1); } std::vector<char> contents(pos); ifs.seekg(0, ios::beg); @@ -364,29 +367,27 @@ static int aws_decrypt_key(const char *path, KEY_INFO *info) DecryptOutcome outcome = client->Decrypt(request); if (!outcome.IsSuccess()) { - sql_print_error("AWS KMS plugin: Decrypt failed for %s : %s", path, + my_printf_error(ER_UNKNOWN_ERROR, "AWS KMS plugin: Decrypt failed for %s : %s", ME_ERROR_LOG, path, outcome.GetError().GetMessage().c_str()); - DBUG_RETURN(-1); + return(-1); } Aws::Utils::ByteBuffer plaintext = outcome.GetResult().GetPlaintext(); size_t len = plaintext.GetLength(); if (len > (int)sizeof(info->data)) { - sql_print_error("AWS KMS plugin: encoding key too large for %s", path); - DBUG_RETURN(ENCRYPTION_KEY_BUFFER_TOO_SMALL); + my_printf_error(ER_UNKNOWN_ERROR, "AWS KMS plugin: encoding key too large for %s", ME_ERROR_LOG, path); + return(ENCRYPTION_KEY_BUFFER_TOO_SMALL); } memcpy(info->data, plaintext.GetUnderlyingData(), len); info->length= len; - DBUG_RETURN(0); + return(0); } /* Generate a new datakey and store it a file */ static int aws_generate_datakey(uint keyid, uint version) { - - DBUG_ENTER("aws_generate_datakey"); GenerateDataKeyWithoutPlaintextRequest request; request.SetKeyId(master_key_id); request.SetKeySpec(DataKeySpecMapper::GetDataKeySpecForName(key_spec_names[key_spec])); @@ -395,10 +396,10 @@ static int aws_generate_datakey(uint keyid, uint version) outcome= client->GenerateDataKeyWithoutPlaintext(request); if (!outcome.IsSuccess()) { - sql_print_error("AWS KMS plugin : GenerateDataKeyWithoutPlaintext failed : %s - %s", + my_printf_error(ER_UNKNOWN_ERROR, "AWS KMS plugin : GenerateDataKeyWithoutPlaintext failed : %s - %s", ME_ERROR_LOG, outcome.GetError().GetExceptionName().c_str(), outcome.GetError().GetMessage().c_str()); - DBUG_RETURN(-1); + return(-1); } string out; @@ -406,24 +407,24 @@ static int aws_generate_datakey(uint keyid, uint version) Aws::Utils::ByteBuffer byteBuffer = outcome.GetResult().GetCiphertextBlob(); format_keyfile_name(filename, sizeof(filename), keyid, version); - int fd= my_open(filename, O_RDWR | O_CREAT, 0); + int fd= open(filename, O_WRONLY |O_CREAT|O_BINARY, IF_WIN(_S_IREAD, S_IRUSR| S_IRGRP| S_IROTH)); if (fd < 0) { - sql_print_error("AWS KMS plugin: Can't create file %s", filename); - DBUG_RETURN(-1); + my_printf_error(ER_UNKNOWN_ERROR, "AWS KMS plugin: Can't create file %s", ME_ERROR_LOG, filename); + return(-1); } size_t len= byteBuffer.GetLength(); - if (my_write(fd, byteBuffer.GetUnderlyingData(), len, 0) != len) + if (write(fd, byteBuffer.GetUnderlyingData(), len) != len) { - sql_print_error("AWS KMS plugin: can't write to %s", filename); - my_close(fd, 0); - my_delete(filename, 0); - DBUG_RETURN(-1); + my_printf_error(ER_UNKNOWN_ERROR, "AWS KMS plugin: can't write to %s", ME_ERROR_LOG, filename); + close(fd); + unlink(filename); + return(-1); } - my_close(fd, 0); - sql_print_information("AWS KMS plugin: generated encrypted datakey for key id=%u, version=%u", + close(fd); + my_printf_error(ER_UNKNOWN_ERROR, "AWS KMS plugin: generated encrypted datakey for key id=%u, version=%u", ME_ERROR_LOG | ME_NOTE, keyid, version); - DBUG_RETURN(0); + return(0); } /* Key rotation for a single key */ @@ -479,7 +480,7 @@ static void update_rotate(MYSQL_THD, struct st_mysql_sys_var *, void *, const vo "aws_key_management_master_key_id must be set to generate new data keys", MYF(ME_JUST_WARNING)); return; } - mysql_mutex_lock(&mtx); + mtx.lock(); rotate_key= *(int *)val; switch (rotate_key) { @@ -493,7 +494,7 @@ static void update_rotate(MYSQL_THD, struct st_mysql_sys_var *, void *, const vo break; } rotate_key= 0; - mysql_mutex_unlock(&mtx); + mtx.unlock(); } static unsigned int get_key( @@ -504,8 +505,7 @@ static unsigned int get_key( { KEY_INFO info; - DBUG_ENTER("get_key"); - mysql_mutex_lock(&mtx); + mtx.lock(); info= key_info_cache[KEY_ID_AND_VERSION(key_id, version)]; if (info.length == 0 && !info.load_failed) { @@ -513,17 +513,17 @@ static unsigned int get_key( info.key_version= version; load_key(&info); } - mysql_mutex_unlock(&mtx); + mtx.unlock(); if (info.load_failed) - DBUG_RETURN(ENCRYPTION_KEY_VERSION_INVALID); + return(ENCRYPTION_KEY_VERSION_INVALID); if (*buflen < info.length) { *buflen= info.length; - DBUG_RETURN(ENCRYPTION_KEY_BUFFER_TOO_SMALL); + return(ENCRYPTION_KEY_BUFFER_TOO_SMALL); } *buflen= info.length; memcpy(dstbuf, info.data, info.length); - DBUG_RETURN(0); + return(0); } diff --git a/plugin/daemon_example/CMakeLists.txt b/plugin/daemon_example/CMakeLists.txt index 3d674c4ef3e..28fbff78302 100644 --- a/plugin/daemon_example/CMakeLists.txt +++ b/plugin/daemon_example/CMakeLists.txt @@ -13,7 +13,8 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -MYSQL_ADD_PLUGIN(daemon_example daemon_example.cc +MYSQL_ADD_PLUGIN(daemon_example daemon_example.cc RECOMPILE_FOR_EMBEDDED MODULE_ONLY MODULE_OUTPUT_NAME "libdaemon_example" COMPONENT Test) -INSTALL(FILES daemon_example.ini DESTINATION ${INSTALL_PLUGINDIR} COMPONENT Test) +INSTALL(FILES daemon_example.ini DESTINATION ${INSTALL_PLUGINDIR} + COMPONENT Test) diff --git a/plugin/example_key_management/example_key_management_plugin.cc b/plugin/example_key_management/example_key_management_plugin.cc index 2b417866406..2b43729e3f7 100644 --- a/plugin/example_key_management/example_key_management_plugin.cc +++ b/plugin/example_key_management/example_key_management_plugin.cc @@ -28,18 +28,24 @@ #include <my_global.h> #include <my_pthread.h> #include <mysql/plugin_encryption.h> -#include <my_rnd.h> #include <my_crypt.h> /* rotate key randomly between 45 and 90 seconds */ #define KEY_ROTATION_MIN 45 #define KEY_ROTATION_MAX 90 -static struct my_rnd_struct seed; static time_t key_version = 0; static time_t next_key_version = 0; static pthread_mutex_t mutex; + +/* Random double value in 0..1 range */ +static double double_rnd() +{ + return ((double)rand()) / RAND_MAX; +} + + static unsigned int get_latest_key_version(unsigned int key_id) { @@ -50,7 +56,7 @@ get_latest_key_version(unsigned int key_id) key_version = now; unsigned int interval = KEY_ROTATION_MAX - KEY_ROTATION_MIN; next_key_version = (time_t) (now + KEY_ROTATION_MIN + - my_rnd(&seed) * interval); + double_rnd() * interval); } pthread_mutex_unlock(&mutex); @@ -101,7 +107,6 @@ static unsigned int get_length(unsigned int slen, unsigned int key_id, static int example_key_management_plugin_init(void *p) { /* init */ - my_rnd_init(&seed, time(0), 0); pthread_mutex_init(&mutex, NULL); get_latest_key_version(1); @@ -114,14 +119,32 @@ static int example_key_management_plugin_deinit(void *p) return 0; } + +static int ctx_update(void *ctx, const unsigned char *src, unsigned int slen, + unsigned char *dst, unsigned int *dlen) +{ + return my_aes_crypt_update(ctx, src, slen, dst, dlen); +} + + +int ctx_finish(void *ctx, unsigned char *dst, unsigned int *dlen) +{ + return my_aes_crypt_finish(ctx, dst, dlen); +} + +static uint ctx_size(unsigned int , unsigned int key_version) +{ + return my_aes_ctx_size(mode(key_version)); +} + struct st_mariadb_encryption example_key_management_plugin= { MariaDB_ENCRYPTION_INTERFACE_VERSION, get_latest_key_version, get_key, - (uint (*)(unsigned int, unsigned int))my_aes_ctx_size, + ctx_size, ctx_init, - my_aes_crypt_update, - my_aes_crypt_finish, + ctx_update, + ctx_finish, get_length }; diff --git a/plugin/file_key_management/file_key_management_plugin.cc b/plugin/file_key_management/file_key_management_plugin.cc index a1f1ed1fad4..141599c53de 100644 --- a/plugin/file_key_management/file_key_management_plugin.cc +++ b/plugin/file_key_management/file_key_management_plugin.cc @@ -13,7 +13,8 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ - +#include <my_global.h> +#include <typelib.h> #include "parser.h" #include <mysql/plugin_encryption.h> #include <string.h> @@ -65,22 +66,14 @@ static struct st_mysql_sys_var* settings[] = { NULL }; -Dynamic_array<keyentry> keys(static_cast<uint>(0)); +std::map<unsigned int,keyentry> keys; static keyentry *get_key(unsigned int key_id) { - keyentry *a= keys.front(), *b= keys.back() + 1, *c; - while (b - a > 1) - { - c= a + (b - a)/2; - if (c->id == key_id) - return c; - else if (c->id < key_id) - a= c; - else - b= c; - } - return a->id == key_id ? a : 0; + keyentry &key= keys[key_id]; + if (key.id == 0) + return 0; + return &key; } /* the version is always the same, no automatic key rotation */ @@ -146,20 +139,37 @@ static int ctx_init(void *ctx, const unsigned char* key, unsigned int klen, return my_aes_crypt_init(ctx, mode(flags), flags, key, klen, iv, ivlen); } +static int ctx_update(void *ctx, const unsigned char *src, unsigned int slen, + unsigned char *dst, unsigned int *dlen) +{ + return my_aes_crypt_update(ctx, src, slen, dst, dlen); +} + + +static int ctx_finish(void *ctx, unsigned char *dst, unsigned int *dlen) +{ + return my_aes_crypt_finish(ctx, dst, dlen); +} + static unsigned int get_length(unsigned int slen, unsigned int key_id, unsigned int key_version) { return my_aes_get_size(mode(0), slen); } +static uint ctx_size(uint, uint) +{ + return my_aes_ctx_size(mode(0)); +} + struct st_mariadb_encryption file_key_management_plugin= { MariaDB_ENCRYPTION_INTERFACE_VERSION, get_latest_version, get_key_from_key_file, - (uint (*)(unsigned int, unsigned int))my_aes_ctx_size, + ctx_size, ctx_init, - my_aes_crypt_update, - my_aes_crypt_finish, + ctx_update, + ctx_finish, get_length }; @@ -171,7 +181,7 @@ static int file_key_management_plugin_init(void *p) static int file_key_management_plugin_deinit(void *p) { - keys.free_memory(); + keys.clear(); return 0; } diff --git a/plugin/file_key_management/parser.cc b/plugin/file_key_management/parser.cc index 047e9153ec2..ac78186a488 100644 --- a/plugin/file_key_management/parser.cc +++ b/plugin/file_key_management/parser.cc @@ -143,13 +143,13 @@ void Parser::bytes_to_key(const unsigned char *salt, const char *input, } -bool Parser::parse(Dynamic_array<keyentry> *keys) +bool Parser::parse(std::map<uint,keyentry> *keys) { const char *secret= filekey; char buf[MAX_SECRET_SIZE + 1]; //If secret starts with FILE: interpret the secret as a filename. - if (is_prefix(filekey, FILE_PREFIX)) + if (strncmp(filekey, FILE_PREFIX,sizeof(FILE_PREFIX) -1) == 0) { if (read_filekey(filekey + sizeof(FILE_PREFIX) - 1, buf)) return 1; @@ -166,22 +166,26 @@ bool Parser::parse(Dynamic_array<keyentry> *keys) bool Parser::read_filekey(const char *filekey, char *secret) { - int f= my_open(filekey, O_RDONLY, MYF(MY_WME)); + int f= open(filekey, O_RDONLY|O_BINARY); if (f == -1) + { + my_error(EE_FILENOTFOUND,ME_ERROR_LOG, filekey, errno); return 1; - int len= my_read(f, (uchar*)secret, MAX_SECRET_SIZE, MYF(MY_WME)); - my_close(f, MYF(MY_WME)); + } + + int len= read(f, secret, MAX_SECRET_SIZE); if (len <= 0) + { + my_error(EE_READ,ME_ERROR_LOG, filekey, errno); + close(f); return 1; + } + close(f); while (secret[len - 1] == '\r' || secret[len - 1] == '\n') len--; secret[len]= '\0'; return 0; } -static int sort_keys(const keyentry *k1, const keyentry *k2) -{ - return k1->id < k2->id ? -1 : k1->id > k2->id; -} /** Get the keys from the key file <filename> and decrypt it with the @@ -191,7 +195,7 @@ static int sort_keys(const keyentry *k1, const keyentry *k2) @return 0 when ok, 1 for an error */ -bool Parser::parse_file(Dynamic_array<keyentry> *keys, const char *secret) +bool Parser::parse_file(std::map<uint,keyentry> *keys, const char *secret) { char *buffer= read_and_decrypt_file(secret); @@ -208,19 +212,16 @@ bool Parser::parse_file(Dynamic_array<keyentry> *keys, const char *secret) case 1: // comment break; case -1: // error - my_free(buffer); + free(buffer); return 1; case 0: - if (keys->push(key)) - return 1; + (*keys)[key.id] = key; break; } } - keys->sort(sort_keys); - my_free(buffer); - - if (keys->elements() == 0 || keys->at(0).id != 1) + free(buffer); + if (keys->size() == 0 || (*keys)[1].id == 0) { report_error("System key id 1 is missing", 0); return 1; @@ -232,7 +233,7 @@ bool Parser::parse_file(Dynamic_array<keyentry> *keys, const char *secret) void Parser::report_error(const char *reason, uint position) { my_printf_error(EE_READ, "%s at %s line %u, column %u", - MYF(ME_NOREFRESH), reason, filename, line_number, position + 1); + ME_ERROR_LOG, reason, filename, line_number, position + 1); } /* @@ -247,16 +248,25 @@ int Parser::parse_line(char **line_ptr, keyentry *key) while (isspace(*p) && *p != '\n') p++; if (*p != '#' && *p != '\n') { - int error; - p+= 100; // the number will surely end here (on a non-digit or with an overflow) - longlong id= my_strtoll10(p - 100, &p, &error); - if (error) + if (!isdigit(*p)) { report_error("Syntax error", p - *line_ptr); return -1; } - if (id < 1 || id > UINT_MAX32) + longlong id = 0; + while (isdigit(*p)) + { + id = id * 10 + *p - '0'; + if (id > UINT_MAX32) + { + report_error("Invalid key id", p - *line_ptr); + return -1; + } + p++; + } + + if (id < 1) { report_error("Invalid key id", p - *line_ptr); return -1; @@ -269,7 +279,7 @@ int Parser::parse_line(char **line_ptr, keyentry *key) } p++; - key->id= id; + key->id= (unsigned int)id; key->length=0; while (isxdigit(p[0]) && isxdigit(p[1]) && key->length < sizeof(key->key)) { @@ -295,26 +305,35 @@ int Parser::parse_line(char **line_ptr, keyentry *key) 'secret'. Store the content of the decrypted file in 'buffer'. The buffer has to be freed in the calling function. */ +#ifdef _WIN32 +#define lseek _lseeki64 +#endif char* Parser::read_and_decrypt_file(const char *secret) { + int f; if (!filename || !filename[0]) { - my_printf_error(EE_CANT_OPEN_STREAM, - "file-key-management-filename is not set", - MYF(ME_NOREFRESH)); + my_printf_error(EE_CANT_OPEN_STREAM, "file-key-management-filename is not set", + ME_ERROR_LOG); goto err0; } - int f; - if ((f= my_open(filename, O_RDONLY, MYF(MY_WME))) < 0) + f= open(filename, O_RDONLY|O_BINARY, 0); + if (f < 0) + { + my_error(EE_FILENOTFOUND, ME_ERROR_LOG, filename, errno); goto err0; + } my_off_t file_size; - file_size= my_seek(f, 0, SEEK_END, MYF(MY_WME)); + file_size= lseek(f, 0, SEEK_END); - if (file_size == MY_FILEPOS_ERROR) + if (file_size == MY_FILEPOS_ERROR || (my_off_t)lseek(f, 0, SEEK_SET) == MY_FILEPOS_ERROR) + { + my_error(EE_CANT_SEEK, MYF(0), filename, errno); goto err1; + } if (file_size > MAX_KEY_FILE_SIZE) { @@ -324,57 +343,67 @@ char* Parser::read_and_decrypt_file(const char *secret) //Read file into buffer uchar *buffer; - buffer= (uchar*)my_malloc(file_size + 1, MYF(MY_WME)); + buffer= (uchar*)malloc((size_t)file_size + 1); if (!buffer) + { + my_error(EE_OUTOFMEMORY, ME_ERROR_LOG| ME_FATAL, file_size); goto err1; + } - if (my_pread(f, buffer, file_size, 0, MYF(MY_WME)) != file_size) + if (read(f, buffer, (int)file_size) != (int)file_size) + { + my_printf_error(EE_READ, + "read from %s failed, errno %d", + MYF(ME_ERROR_LOG|ME_FATAL), filename, errno); goto err2; + } // Check for file encryption uchar *decrypted; - if (file_size > OpenSSL_prefix_len && is_prefix((char*)buffer, OpenSSL_prefix)) + if (file_size > OpenSSL_prefix_len && strncmp((char*)buffer, OpenSSL_prefix, OpenSSL_prefix_len) == 0) { uchar key[OpenSSL_key_len]; uchar iv[OpenSSL_iv_len]; - decrypted= (uchar*)my_malloc(file_size, MYF(MY_WME)); + decrypted= (uchar*)malloc((size_t)file_size); if (!decrypted) + { + my_error(EE_OUTOFMEMORY, ME_ERROR_LOG | ME_FATAL, file_size); goto err2; - + } bytes_to_key(buffer + OpenSSL_prefix_len, secret, key, iv); uint32 d_size; if (my_aes_crypt(MY_AES_CBC, ENCRYPTION_FLAG_DECRYPT, buffer + OpenSSL_prefix_len + OpenSSL_salt_len, - file_size - OpenSSL_prefix_len - OpenSSL_salt_len, + (unsigned int)file_size - OpenSSL_prefix_len - OpenSSL_salt_len, decrypted, &d_size, key, OpenSSL_key_len, iv, OpenSSL_iv_len)) { - my_printf_error(EE_READ, "Cannot decrypt %s. Wrong key?", MYF(ME_NOREFRESH), filename); + my_printf_error(EE_READ, "Cannot decrypt %s. Wrong key?", ME_ERROR_LOG, filename); goto err3; } - my_free(buffer); + free(buffer); buffer= decrypted; file_size= d_size; } else if (*secret) { - my_printf_error(EE_READ, "Cannot decrypt %s. Not encrypted", MYF(ME_NOREFRESH), filename); + my_printf_error(EE_READ, "Cannot decrypt %s. Not encrypted", ME_ERROR_LOG, filename); goto err2; } buffer[file_size]= '\0'; - my_close(f, MYF(MY_WME)); + close(f); return (char*) buffer; err3: - my_free(decrypted); + free(decrypted); err2: - my_free(buffer); + free(buffer); err1: - my_close(f, MYF(MY_WME)); + close(f); err0: return NULL; } diff --git a/plugin/file_key_management/parser.h b/plugin/file_key_management/parser.h index c8349db70a0..627b7fd84a6 100644 --- a/plugin/file_key_management/parser.h +++ b/plugin/file_key_management/parser.h @@ -22,7 +22,7 @@ Created 09/15/2014 #include <my_crypt.h> #include <ctype.h> -#include <sql_array.h> +#include <map> struct keyentry { unsigned int id; @@ -42,7 +42,7 @@ class Parser void bytes_to_key(const unsigned char *salt, const char *secret, unsigned char *key, unsigned char *iv); bool read_filekey(const char *filekey, char *secret); - bool parse_file(Dynamic_array<keyentry> *keys, const char *secret); + bool parse_file(std::map<unsigned int ,keyentry> *keys, const char *secret); void report_error(const char *reason, unsigned int position); int parse_line(char **line_ptr, keyentry *key); char* read_and_decrypt_file(const char *secret); @@ -50,5 +50,5 @@ class Parser public: Parser(const char* fn, const char *fk) : filename(fn), filekey(fk), line_number(0) { } - bool parse(Dynamic_array<keyentry> *keys); + bool parse(std::map<unsigned int ,keyentry> *keys); }; diff --git a/plugin/handler_socket/CMakeLists.txt b/plugin/handler_socket/CMakeLists.txt index 2e7caa80897..a10743210e9 100644 --- a/plugin/handler_socket/CMakeLists.txt +++ b/plugin/handler_socket/CMakeLists.txt @@ -34,6 +34,6 @@ SET(HANDLERSOCKET_SOURCES MYSQL_ADD_PLUGIN(handlersocket ${HANDLERSOCKET_SOURCES} MODULE_ONLY COMPONENT Server - LINK_LIBRARIES hsclient + LINK_LIBRARIES hsclient RECOMPILE_FOR_EMBEDDED ) diff --git a/plugin/locale_info/CMakeLists.txt b/plugin/locale_info/CMakeLists.txt index 1ace6619041..8f1dfa0d715 100644 --- a/plugin/locale_info/CMakeLists.txt +++ b/plugin/locale_info/CMakeLists.txt @@ -1,5 +1,5 @@ INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql ${CMAKE_SOURCE_DIR}/regex ${CMAKE_SOURCE_DIR}/extra/yassl/include) -MYSQL_ADD_PLUGIN(LOCALES locale_info.cc) +MYSQL_ADD_PLUGIN(LOCALES locale_info.cc RECOMPILE_FOR_EMBEDDED) diff --git a/plugin/metadata_lock_info/CMakeLists.txt b/plugin/metadata_lock_info/CMakeLists.txt index 44393c09eb6..6b1f5108bf1 100644 --- a/plugin/metadata_lock_info/CMakeLists.txt +++ b/plugin/metadata_lock_info/CMakeLists.txt @@ -1,2 +1,3 @@ SET(METADATA_LOCK_INFO_SOURCES metadata_lock_info.cc) -MYSQL_ADD_PLUGIN(metadata_lock_info ${METADATA_LOCK_INFO_SOURCES} MODULE_OUTPUT_NAME "metadata_lock_info") +MYSQL_ADD_PLUGIN(metadata_lock_info ${METADATA_LOCK_INFO_SOURCES} + RECOMPILE_FOR_EMBEDDED) diff --git a/plugin/qc_info/CMakeLists.txt b/plugin/qc_info/CMakeLists.txt index d10f4547227..821ffb79225 100644 --- a/plugin/qc_info/CMakeLists.txt +++ b/plugin/qc_info/CMakeLists.txt @@ -2,4 +2,4 @@ INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql ${PCRE_INCLUDES} ${CMAKE_SOURCE_DIR}/extra/yassl/include) -MYSQL_ADD_PLUGIN(QUERY_CACHE_INFO qc_info.cc) +MYSQL_ADD_PLUGIN(QUERY_CACHE_INFO qc_info.cc RECOMPILE_FOR_EMBEDDED) diff --git a/plugin/query_response_time/CMakeLists.txt b/plugin/query_response_time/CMakeLists.txt index f008d0256fc..112d72e429a 100644 --- a/plugin/query_response_time/CMakeLists.txt +++ b/plugin/query_response_time/CMakeLists.txt @@ -1,2 +1,3 @@ ADD_DEFINITIONS(-DHAVE_RESPONSE_TIME_DISTRIBUTION) -MYSQL_ADD_PLUGIN(QUERY_RESPONSE_TIME query_response_time.cc plugin.cc) +MYSQL_ADD_PLUGIN(QUERY_RESPONSE_TIME query_response_time.cc plugin.cc + RECOMPILE_FOR_EMBEDDED) diff --git a/plugin/semisync/CMakeLists.txt b/plugin/semisync/CMakeLists.txt index 33c0895e5e1..88998fb3093 100644 --- a/plugin/semisync/CMakeLists.txt +++ b/plugin/semisync/CMakeLists.txt @@ -17,10 +17,12 @@ SET(SEMISYNC_MASTER_SOURCES semisync.cc semisync_master.cc semisync_master_plugin.cc semisync.h semisync_master.h) -MYSQL_ADD_PLUGIN(semisync_master ${SEMISYNC_MASTER_SOURCES}) +MYSQL_ADD_PLUGIN(semisync_master ${SEMISYNC_MASTER_SOURCES} + RECOMPILE_FOR_EMBEDDED) SET(SEMISYNC_SLAVE_SOURCES semisync.cc semisync_slave.cc semisync_slave_plugin.cc semisync.h semisync_slave.h ) -MYSQL_ADD_PLUGIN(semisync_slave ${SEMISYNC_SLAVE_SOURCES}) +MYSQL_ADD_PLUGIN(semisync_slave ${SEMISYNC_SLAVE_SOURCES} + RECOMPILE_FOR_EMBEDDED) diff --git a/plugin/server_audit/CMakeLists.txt b/plugin/server_audit/CMakeLists.txt index 2c9964543bf..056a11f3753 100644 --- a/plugin/server_audit/CMakeLists.txt +++ b/plugin/server_audit/CMakeLists.txt @@ -13,7 +13,6 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA -SET(SERVER_AUDIT_SOURCES - server_audit.c test_audit_v4.c plugin_audit_v4.h) +SET(SOURCES server_audit.c test_audit_v4.c plugin_audit_v4.h) - MYSQL_ADD_PLUGIN(server_audit ${SERVER_AUDIT_SOURCES} MODULE_ONLY) +MYSQL_ADD_PLUGIN(server_audit ${SOURCES} MODULE_ONLY RECOMPILE_FOR_EMBEDDED) diff --git a/plugin/wsrep_info/CMakeLists.txt b/plugin/wsrep_info/CMakeLists.txt index 4dee10c34c1..34aee9fba2c 100644 --- a/plugin/wsrep_info/CMakeLists.txt +++ b/plugin/wsrep_info/CMakeLists.txt @@ -1,5 +1,5 @@ IF (WITH_WSREP) INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql ${CMAKE_SOURCE_DIR}/wsrep) - MYSQL_ADD_PLUGIN(WSREP_INFO plugin.cc MODULE_ONLY) + MYSQL_ADD_PLUGIN(WSREP_INFO plugin.cc MODULE_ONLY RECOMPILE_FOR_EMBEDDED) ENDIF() diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index e7f20e78a63..c789ee2ae63 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -259,9 +259,8 @@ ELSE() wsrep_sst_rsync wsrep_sst_xtrabackup wsrep_sst_xtrabackup-v2 + wsrep_sst_mariabackup ) - INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/wsrep_sst_common - DESTINATION ${INSTALL_BINDIR} COMPONENT Server) ENDIF() IF (NOT WITHOUT_SERVER) SET(SERVER_SCRIPTS diff --git a/scripts/wsrep_sst_common b/scripts/wsrep_sst_common.sh index 466bb46b382..466bb46b382 100644 --- a/scripts/wsrep_sst_common +++ b/scripts/wsrep_sst_common.sh diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh new file mode 100644 index 00000000000..9e3fc54290d --- /dev/null +++ b/scripts/wsrep_sst_mariabackup.sh @@ -0,0 +1,1040 @@ +#!/bin/bash -ue +# Copyright (C) 2013 Percona Inc +# Copyright (C) 2017 MariaDB +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; see the file COPYING. If not, write to the +# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston +# MA 02110-1301 USA. + +# Documentation: +# http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html +# Make sure to read that before proceeding! + +. $(dirname $0)/wsrep_sst_common + +ealgo="" +ekey="" +ekeyfile="" +encrypt=0 +nproc=1 +ecode=0 +ssyslog="" +ssystag="" +XTRABACKUP_PID="" +SST_PORT="" +REMOTEIP="" +REMOTEHOST="" +tcert="" +tpem="" +tkey="" +sockopt="" +progress="" +ttime=0 +totime=0 +lsn="" +ecmd="" +rlimit="" +# Initially +stagemsg="${WSREP_SST_OPT_ROLE}" +cpat="" +speciald=1 +ib_home_dir="" +ib_log_dir="" +ib_undo_dir="" + +sfmt="tar" +strmcmd="" +tfmt="" +tcmd="" +rebuild=0 +rebuildcmd="" +payload=0 +pvformat="-F '%N => Rate:%r Avg:%a Elapsed:%t %e Bytes: %b %p' " +pvopts="-f -i 10 -N $WSREP_SST_OPT_ROLE " +STATDIR="" +uextra=0 +disver="" + +tmpopts="" +itmpdir="" +xtmpdir="" + +scomp="" +sdecomp="" + +# Required for backup locks +# For backup locks it is 1 sent by joiner +# 5.6.21 PXC and later can't donate to an older joiner +sst_ver=1 + +if which pv &>/dev/null && pv --help | grep -q FORMAT;then + pvopts+=$pvformat +fi +pcmd="pv $pvopts" +declare -a RC + +INNOBACKUPEX_BIN=mariabackup +XBSTREAM_BIN=mbstream +XBCRYPT_BIN=xbcrypt # Not available in MariaBackup + +DATA="${WSREP_SST_OPT_DATA}" +INFO_FILE="xtrabackup_galera_info" +IST_FILE="xtrabackup_ist" +MAGIC_FILE="${DATA}/${INFO_FILE}" + +# Setting the path for ss and ip +export PATH="/usr/sbin:/sbin:$PATH" + +timeit(){ + local stage=$1 + shift + local cmd="$@" + local x1 x2 took extcode + + if [[ $ttime -eq 1 ]];then + x1=$(date +%s) + wsrep_log_info "Evaluating $cmd" + eval "$cmd" + extcode=$? + x2=$(date +%s) + took=$(( x2-x1 )) + wsrep_log_info "NOTE: $stage took $took seconds" + totime=$(( totime+took )) + else + wsrep_log_info "Evaluating $cmd" + eval "$cmd" + extcode=$? + fi + return $extcode +} + +get_keys() +{ + # $encrypt -eq 1 is for internal purposes only + if [[ $encrypt -ge 2 || $encrypt -eq -1 ]];then + return + fi + + if [[ $encrypt -eq 0 ]];then + if $MY_PRINT_DEFAULTS xtrabackup | grep -q encrypt;then + wsrep_log_error "Unexpected option combination. SST may fail. Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html " + fi + return + fi + + if [[ $sfmt == 'tar' ]];then + wsrep_log_info "NOTE: Xtrabackup-based encryption - encrypt=1 - cannot be enabled with tar format" + encrypt=-1 + return + fi + + wsrep_log_info "Xtrabackup based encryption enabled in my.cnf - Supported only from Xtrabackup 2.1.4" + + if [[ -z $ealgo ]];then + wsrep_log_error "FATAL: Encryption algorithm empty from my.cnf, bailing out" + exit 3 + fi + + if [[ -z $ekey && ! -r $ekeyfile ]];then + wsrep_log_error "FATAL: Either key or keyfile must be readable" + exit 3 + fi + + if [[ -z $ekey ]];then + ecmd="${XBCRYPT_BIN} --encrypt-algo=$ealgo --encrypt-key-file=$ekeyfile" + else + ecmd="${XBCRYPT_BIN} --encrypt-algo=$ealgo --encrypt-key=$ekey" + fi + + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + ecmd+=" -d" + fi + + stagemsg+="-XB-Encrypted" +} + +get_transfer() +{ + if [[ -z $SST_PORT ]];then + TSST_PORT=4444 + else + TSST_PORT=$SST_PORT + fi + + if [[ $tfmt == 'nc' ]];then + if [[ ! -x `which nc` ]];then + wsrep_log_error "nc(netcat) not found in path: $PATH" + exit 2 + fi + wsrep_log_info "Using netcat as streamer" + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + if nc -h 2>&1 | grep -q ncat;then + tcmd="nc -l ${TSST_PORT}" + else + tcmd="nc -dl ${TSST_PORT}" + fi + else + tcmd="nc ${REMOTEIP} ${TSST_PORT}" + fi + else + tfmt='socat' + wsrep_log_info "Using socat as streamer" + if [[ ! -x `which socat` ]];then + wsrep_log_error "socat not found in path: $PATH" + exit 2 + fi + + if [[ $encrypt -eq 2 || $encrypt -eq 3 ]] && ! socat -V | grep -q "WITH_OPENSSL 1";then + wsrep_log_error "Encryption requested, but socat is not OpenSSL enabled (encrypt=$encrypt)" + exit 2 + fi + + if [[ $encrypt -eq 2 ]];then + wsrep_log_info "Using openssl based encryption with socat: with crt and pem" + if [[ -z $tpem || -z $tcert ]];then + wsrep_log_error "Both PEM and CRT files required" + exit 22 + fi + stagemsg+="-OpenSSL-Encrypted-2" + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + wsrep_log_info "Decrypting with cert=${tpem}, cafile=${tcert}" + tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=${tpem},cafile=${tcert}${sockopt} stdio" + else + wsrep_log_info "Encrypting with cert=${tpem}, cafile=${tcert}" + tcmd="socat -u stdio openssl-connect:${REMOTEHOST}:${TSST_PORT},cert=${tpem},cafile=${tcert}${sockopt}" + fi + elif [[ $encrypt -eq 3 ]];then + wsrep_log_info "Using openssl based encryption with socat: with key and crt" + if [[ -z $tpem || -z $tkey ]];then + wsrep_log_error "Both certificate and key files required" + exit 22 + fi + stagemsg+="-OpenSSL-Encrypted-3" + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + if [[ -z $tcert ]];then + wsrep_log_info "Decrypting with cert=${tpem}, key=${tkey}, verify=0" + tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=${tpem},key=${tkey},verify=0${sockopt} stdio" + else + wsrep_log_info "Decrypting with cert=${tpem}, key=${tkey}, cafile=${tcert}" + tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=${tpem},key=${tkey},cafile=${tcert}${sockopt} stdio" + fi + else + if [[ -z $tcert ]];then + wsrep_log_info "Encrypting with cert=${tpem}, key=${tkey}, verify=0" + tcmd="socat -u stdio openssl-connect:${REMOTEIP}:${TSST_PORT},cert=${tpem},key=${tkey},verify=0${sockopt}" + else + wsrep_log_info "Encrypting with cert=${tpem}, key=${tkey}, cafile=${tcert}" + tcmd="socat -u stdio openssl-connect:${REMOTEHOST}:${TSST_PORT},cert=${tpem},key=${tkey},cafile=${tcert}${sockopt}" + fi + fi + + else + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + tcmd="socat -u TCP-LISTEN:${TSST_PORT},reuseaddr${sockopt} stdio" + else + tcmd="socat -u stdio TCP:${REMOTEIP}:${TSST_PORT}${sockopt}" + fi + fi + fi + +} + +parse_cnf() +{ + local group=$1 + local var=$2 + # print the default settings for given group using my_print_default. + # normalize the variable names specified in cnf file (user can use _ or - for example log-bin or log_bin) + # then grep for needed variable + # finally get the variable value (if variables has been specified multiple time use the last value only) + reval=$($MY_PRINT_DEFAULTS $group | awk -F= '{if ($1 ~ /_/) { gsub(/_/,"-",$1); print $1"="$2 } else { print $0 }}' | grep -- "--$var=" | cut -d= -f2- | tail -1) + if [[ -z $reval ]];then + [[ -n $3 ]] && reval=$3 + fi + echo $reval +} + +get_footprint() +{ + pushd $WSREP_SST_OPT_DATA 1>/dev/null + payload=$(find . -regex '.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' -type f -print0 | du --files0-from=- --block-size=1 -c | awk 'END { print $1 }') + if $MY_PRINT_DEFAULTS xtrabackup | grep -q -- "--compress";then + # QuickLZ has around 50% compression ratio + # When compression/compaction used, the progress is only an approximate. + payload=$(( payload*1/2 )) + fi + popd 1>/dev/null + pcmd+=" -s $payload" + adjust_progress +} + +adjust_progress() +{ + + if [[ ! -x `which pv` ]];then + wsrep_log_error "pv not found in path: $PATH" + wsrep_log_error "Disabling all progress/rate-limiting" + pcmd="" + rlimit="" + progress="" + return + fi + + if [[ -n $progress && $progress != '1' ]];then + if [[ -e $progress ]];then + pcmd+=" 2>>$progress" + else + pcmd+=" 2>$progress" + fi + elif [[ -z $progress && -n $rlimit ]];then + # When rlimit is non-zero + pcmd="pv -q" + fi + + if [[ -n $rlimit && "$WSREP_SST_OPT_ROLE" == "donor" ]];then + wsrep_log_info "Rate-limiting SST to $rlimit" + pcmd+=" -L \$rlimit" + fi +} + +read_cnf() +{ + sfmt=$(parse_cnf sst streamfmt "xbstream") + tfmt=$(parse_cnf sst transferfmt "socat") + tcert=$(parse_cnf sst tca "") + tpem=$(parse_cnf sst tcert "") + tkey=$(parse_cnf sst tkey "") + encrypt=$(parse_cnf sst encrypt 0) + sockopt=$(parse_cnf sst sockopt "") + progress=$(parse_cnf sst progress "") + rebuild=$(parse_cnf sst rebuild 0) + ttime=$(parse_cnf sst time 0) + cpat=$(parse_cnf sst cpat '.*galera\.cache$\|.*sst_in_progress$\|.*\.sst$\|.*gvwstate\.dat$\|.*grastate\.dat$\|.*\.err$\|.*\.log$\|.*RPM_UPGRADE_MARKER$\|.*RPM_UPGRADE_HISTORY$') + ealgo=$(parse_cnf xtrabackup encrypt "") + ekey=$(parse_cnf xtrabackup encrypt-key "") + ekeyfile=$(parse_cnf xtrabackup encrypt-key-file "") + scomp=$(parse_cnf sst compressor "") + sdecomp=$(parse_cnf sst decompressor "") + + # Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html + if [[ -z $ealgo ]];then + ealgo=$(parse_cnf sst encrypt-algo "") + ekey=$(parse_cnf sst encrypt-key "") + ekeyfile=$(parse_cnf sst encrypt-key-file "") + fi + + rlimit=$(parse_cnf sst rlimit "") + uextra=$(parse_cnf sst use-extra 0) + speciald=$(parse_cnf sst sst-special-dirs 1) + iopts=$(parse_cnf sst inno-backup-opts "") + iapts=$(parse_cnf sst inno-apply-opts "") + impts=$(parse_cnf sst inno-move-opts "") + stimeout=$(parse_cnf sst sst-initial-timeout 100) + ssyslog=$(parse_cnf sst sst-syslog 0) + ssystag=$(parse_cnf mysqld_safe syslog-tag "${SST_SYSLOG_TAG:-}") + ssystag+="-" + + if [[ $speciald -eq 0 ]];then + wsrep_log_error "sst-special-dirs equal to 0 is not supported, falling back to 1" + speciald=1 + fi + + if [[ $ssyslog -ne -1 ]];then + if $MY_PRINT_DEFAULTS mysqld_safe | tr '_' '-' | grep -q -- "--syslog";then + ssyslog=1 + fi + fi + + if [[ $encrypt -eq 1 ]]; then + wsrep_log_error "Xtrabackup-based encryption is currently not" \ + "supported with MariaBackup" + exit 2 + fi +} + +get_stream() +{ + if [[ $sfmt == 'xbstream' ]];then + wsrep_log_info "Streaming with xbstream" + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + strmcmd="${XBSTREAM_BIN} -x" + else + strmcmd="${XBSTREAM_BIN} -c \${INFO_FILE}" + fi + else + sfmt="tar" + wsrep_log_info "Streaming with tar" + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + strmcmd="tar xfi - " + else + strmcmd="tar cf - \${INFO_FILE} " + fi + + fi +} + +get_proc() +{ + set +e + nproc=$(grep -c processor /proc/cpuinfo) + [[ -z $nproc || $nproc -eq 0 ]] && nproc=1 + set -e +} + +sig_joiner_cleanup() +{ + wsrep_log_error "Removing $MAGIC_FILE file due to signal" + rm -f "$MAGIC_FILE" +} + +cleanup_joiner() +{ + # Since this is invoked just after exit NNN + local estatus=$? + if [[ $estatus -ne 0 ]];then + wsrep_log_error "Cleanup after exit with status:$estatus" + elif [ "${WSREP_SST_OPT_ROLE}" = "joiner" ];then + wsrep_log_info "Removing the sst_in_progress file" + wsrep_cleanup_progress_file + fi + if [[ -n $progress && -p $progress ]];then + wsrep_log_info "Cleaning up fifo file $progress" + rm $progress + fi + if [[ -n ${STATDIR:-} ]];then + [[ -d $STATDIR ]] && rm -rf $STATDIR + fi + + # Final cleanup + pgid=$(ps -o pgid= $$ | grep -o '[0-9]*') + + # This means no setsid done in mysqld. + # We don't want to kill mysqld here otherwise. + if [[ $$ -eq $pgid ]];then + + # This means a signal was delivered to the process. + # So, more cleanup. + if [[ $estatus -ge 128 ]];then + kill -KILL -$$ || true + fi + + fi + + exit $estatus +} + +check_pid() +{ + local pid_file="$1" + [ -r "$pid_file" ] && ps -p $(cat "$pid_file") >/dev/null 2>&1 +} + +cleanup_donor() +{ + # Since this is invoked just after exit NNN + local estatus=$? + if [[ $estatus -ne 0 ]];then + wsrep_log_error "Cleanup after exit with status:$estatus" + fi + + if [[ -n ${XTRABACKUP_PID:-} ]];then + if check_pid $XTRABACKUP_PID + then + wsrep_log_error "xtrabackup process is still running. Killing... " + kill_xtrabackup + fi + + fi + rm -f ${DATA}/${IST_FILE} || true + + if [[ -n $progress && -p $progress ]];then + wsrep_log_info "Cleaning up fifo file $progress" + rm -f $progress || true + fi + + wsrep_log_info "Cleaning up temporary directories" + + if [[ -n $xtmpdir ]];then + [[ -d $xtmpdir ]] && rm -rf $xtmpdir || true + fi + + if [[ -n $itmpdir ]];then + [[ -d $itmpdir ]] && rm -rf $itmpdir || true + fi + + # Final cleanup + pgid=$(ps -o pgid= $$ | grep -o '[0-9]*') + + # This means no setsid done in mysqld. + # We don't want to kill mysqld here otherwise. + if [[ $$ -eq $pgid ]];then + + # This means a signal was delivered to the process. + # So, more cleanup. + if [[ $estatus -ge 128 ]];then + kill -KILL -$$ || true + fi + + fi + + exit $estatus + +} + +kill_xtrabackup() +{ + local PID=$(cat $XTRABACKUP_PID) + [ -n "$PID" -a "0" != "$PID" ] && kill $PID && (kill $PID && kill -9 $PID) || : + wsrep_log_info "Removing xtrabackup pid file $XTRABACKUP_PID" + rm -f "$XTRABACKUP_PID" || true +} + +setup_ports() +{ + if [[ "$WSREP_SST_OPT_ROLE" == "donor" ]];then + SST_PORT=$(echo $WSREP_SST_OPT_ADDR | awk -F '[:/]' '{ print $2 }') + REMOTEIP=$(echo $WSREP_SST_OPT_ADDR | awk -F ':' '{ print $1 }') + REMOTEHOST=$(getent hosts $REMOTEIP | awk '{ print $2 }') + if [[ -z $REMOTEHOST ]];then + REMOTEHOST=$REMOTEIP + fi + lsn=$(echo $WSREP_SST_OPT_ADDR | awk -F '[:/]' '{ print $4 }') + sst_ver=$(echo $WSREP_SST_OPT_ADDR | awk -F '[:/]' '{ print $5 }') + else + SST_PORT=$(echo ${WSREP_SST_OPT_ADDR} | awk -F ':' '{ print $2 }') + fi +} + +# waits ~10 seconds for nc to open the port and then reports ready +# (regardless of timeout) +wait_for_listen() +{ + local PORT=$1 + local ADDR=$2 + local MODULE=$3 + for i in {1..50} + do + ss -p state listening "( sport = :$PORT )" | grep -qE 'socat|nc' && break + sleep 0.2 + done + echo "ready ${ADDR}/${MODULE}//$sst_ver" +} + +check_extra() +{ + local use_socket=1 + if [[ $uextra -eq 1 ]];then + if $MY_PRINT_DEFAULTS --mysqld | tr '_' '-' | grep -- "--thread-handling=" | grep -q 'pool-of-threads';then + local eport=$($MY_PRINT_DEFAULTS --mysqld | tr '_' '-' | grep -- "--extra-port=" | cut -d= -f2) + if [[ -n $eport ]];then + # Xtrabackup works only locally. + # Hence, setting host to 127.0.0.1 unconditionally. + wsrep_log_info "SST through extra_port $eport" + INNOEXTRA+=" --host=127.0.0.1 --port=$eport " + use_socket=0 + else + wsrep_log_error "Extra port $eport null, failing" + exit 1 + fi + else + wsrep_log_info "Thread pool not set, ignore the option use_extra" + fi + fi + if [[ $use_socket -eq 1 ]] && [[ -n "${WSREP_SST_OPT_SOCKET}" ]];then + INNOEXTRA+=" --socket=${WSREP_SST_OPT_SOCKET}" + fi +} + +recv_joiner() +{ + local dir=$1 + local msg=$2 + local tmt=$3 + local checkf=$4 + local ltcmd + + if [[ ! -d ${dir} ]];then + # This indicates that IST is in progress + return + fi + + pushd ${dir} 1>/dev/null + set +e + + if [[ $tmt -gt 0 && -x `which timeout` ]];then + if timeout --help | grep -q -- '-k';then + ltcmd="timeout -k $(( tmt+10 )) $tmt $tcmd" + else + ltcmd="timeout -s9 $tmt $tcmd" + fi + timeit "$msg" "$ltcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )" + else + timeit "$msg" "$tcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )" + fi + + set -e + popd 1>/dev/null + + if [[ ${RC[0]} -eq 124 ]];then + wsrep_log_error "Possible timeout in receving first data from donor in gtid stage" + exit 32 + fi + + for ecode in "${RC[@]}";do + if [[ $ecode -ne 0 ]];then + wsrep_log_error "Error while getting data from donor node: " \ + "exit codes: ${RC[@]}" + exit 32 + fi + done + + if [[ $checkf -eq 1 && ! -r "${MAGIC_FILE}" ]];then + # this message should cause joiner to abort + wsrep_log_error "xtrabackup process ended without creating '${MAGIC_FILE}'" + wsrep_log_info "Contents of datadir" + wsrep_log_info "$(ls -l ${dir}/*)" + exit 32 + fi +} + + +send_donor() +{ + local dir=$1 + local msg=$2 + + pushd ${dir} 1>/dev/null + set +e + timeit "$msg" "$strmcmd | $tcmd; RC=( "\${PIPESTATUS[@]}" )" + set -e + popd 1>/dev/null + + + for ecode in "${RC[@]}";do + if [[ $ecode -ne 0 ]];then + wsrep_log_error "Error while getting data from donor node: " \ + "exit codes: ${RC[@]}" + exit 32 + fi + done + +} + +if [[ ! -x `which $INNOBACKUPEX_BIN` ]];then + wsrep_log_error "${INNOBACKUPEX_BIN} not in path: $PATH" + exit 2 +fi + +rm -f "${MAGIC_FILE}" + +if [[ ! ${WSREP_SST_OPT_ROLE} == 'joiner' && ! ${WSREP_SST_OPT_ROLE} == 'donor' ]];then + wsrep_log_error "Invalid role ${WSREP_SST_OPT_ROLE}" + exit 22 +fi + +read_cnf +setup_ports + +if ${INNOBACKUPEX_BIN} /tmp --help 2>/dev/null | grep -q -- '--version-check'; then + disver="--no-version-check" +fi + +if [[ ${FORCE_FTWRL:-0} -eq 1 ]];then + wsrep_log_info "Forcing FTWRL due to environment variable FORCE_FTWRL equal to $FORCE_FTWRL" + iopts+=" --no-backup-locks " +fi + + +INNOEXTRA="" + +if [[ $ssyslog -eq 1 ]];then + + if [[ ! -x `which logger` ]];then + wsrep_log_error "logger not in path: $PATH. Ignoring" + else + + wsrep_log_info "Logging all stderr of SST/Innobackupex to syslog" + + exec 2> >(logger -p daemon.err -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE) + + wsrep_log_error() + { + logger -p daemon.err -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE "$@" + } + + wsrep_log_info() + { + logger -p daemon.info -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE "$@" + } + + INNOAPPLY="${INNOBACKUPEX_BIN} --innobackupex $disver $iapts --apply-log \$rebuildcmd \${DATA} 2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-apply " + INNOMOVE="${INNOBACKUPEX_BIN} --innobackupex ${WSREP_SST_OPT_CONF} $disver $impts --move-back --force-non-empty-directories \${DATA} 2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-move " + INNOBACKUP="${INNOBACKUPEX_BIN} --innobackupex ${WSREP_SST_OPT_CONF} $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt \$itmpdir 2> >(logger -p daemon.err -t ${ssystag}innobackupex-backup)" + fi + +else + INNOAPPLY="${INNOBACKUPEX_BIN} --innobackupex $disver $iapts --apply-log \$rebuildcmd \${DATA} &>\${DATA}/innobackup.prepare.log" + INNOMOVE="${INNOBACKUPEX_BIN} --innobackupex ${WSREP_SST_OPT_CONF} $disver $impts --move-back --force-non-empty-directories \${DATA} &>\${DATA}/innobackup.move.log" + INNOBACKUP="${INNOBACKUPEX_BIN} --innobackupex ${WSREP_SST_OPT_CONF} $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt \$itmpdir 2>\${DATA}/innobackup.backup.log" +fi + +get_stream +get_transfer + +if [ "$WSREP_SST_OPT_ROLE" = "donor" ] +then + trap cleanup_donor EXIT + + if [ $WSREP_SST_OPT_BYPASS -eq 0 ] + then + usrst=0 + if [[ -z $sst_ver ]];then + wsrep_log_error "Upgrade joiner to 5.6.21 or higher for backup locks support" + wsrep_log_error "The joiner is not supported for this version of donor" + exit 93 + fi + + if [[ -z $(parse_cnf mysqld tmpdir "") && -z $(parse_cnf xtrabackup tmpdir "") ]];then + xtmpdir=$(mktemp -d) + tmpopts=" --tmpdir=$xtmpdir " + wsrep_log_info "Using $xtmpdir as xtrabackup temporary directory" + fi + + itmpdir=$(mktemp -d) + wsrep_log_info "Using $itmpdir as innobackupex temporary directory" + + if [[ -n "${WSREP_SST_OPT_USER:-}" && "$WSREP_SST_OPT_USER" != "(null)" ]]; then + INNOEXTRA+=" --user=$WSREP_SST_OPT_USER" + usrst=1 + fi + + if [ -n "${WSREP_SST_OPT_PSWD:-}" ]; then + INNOEXTRA+=" --password=$WSREP_SST_OPT_PSWD" + elif [[ $usrst -eq 1 ]];then + # Empty password, used for testing, debugging etc. + INNOEXTRA+=" --password=" + fi + + get_keys + if [[ $encrypt -eq 1 ]];then + if [[ -n $ekey ]];then + INNOEXTRA+=" --encrypt=$ealgo --encrypt-key=$ekey " + else + INNOEXTRA+=" --encrypt=$ealgo --encrypt-key-file=$ekeyfile " + fi + fi + + + check_extra + + wsrep_log_info "Streaming GTID file before SST" + + # Store donor's wsrep GTID (state ID) and wsrep_gtid_domain_id + # (separated by a space). + echo "${WSREP_SST_OPT_GTID} ${WSREP_SST_OPT_GTID_DOMAIN_ID}" > "${MAGIC_FILE}" + + ttcmd="$tcmd" + + if [[ $encrypt -eq 1 ]];then + if [[ -n $scomp ]];then + tcmd=" $ecmd | $scomp | $tcmd " + else + tcmd=" $ecmd | $tcmd " + fi + elif [[ -n $scomp ]];then + tcmd=" $scomp | $tcmd " + fi + + + send_donor $DATA "${stagemsg}-gtid" + + tcmd="$ttcmd" + if [[ -n $progress ]];then + get_footprint + tcmd="$pcmd | $tcmd" + elif [[ -n $rlimit ]];then + adjust_progress + tcmd="$pcmd | $tcmd" + fi + + wsrep_log_info "Sleeping before data transfer for SST" + sleep 10 + + wsrep_log_info "Streaming the backup to joiner at ${REMOTEIP} ${SST_PORT:-4444}" + + if [[ -n $scomp ]];then + tcmd="$scomp | $tcmd" + fi + + set +e + timeit "${stagemsg}-SST" "$INNOBACKUP | $tcmd; RC=( "\${PIPESTATUS[@]}" )" + set -e + + if [ ${RC[0]} -ne 0 ]; then + wsrep_log_error "${INNOBACKUPEX_BIN} finished with error: ${RC[0]}. " \ + "Check ${DATA}/innobackup.backup.log" + exit 22 + elif [[ ${RC[$(( ${#RC[@]}-1 ))]} -eq 1 ]];then + wsrep_log_error "$tcmd finished with error: ${RC[1]}" + exit 22 + fi + + # innobackupex implicitly writes PID to fixed location in $xtmpdir + XTRABACKUP_PID="$xtmpdir/xtrabackup_pid" + + + else # BYPASS FOR IST + + wsrep_log_info "Bypassing the SST for IST" + echo "continue" # now server can resume updating data + + # Store donor's wsrep GTID (state ID) and wsrep_gtid_domain_id + # (separated by a space). + echo "${WSREP_SST_OPT_GTID} ${WSREP_SST_OPT_GTID_DOMAIN_ID}" > "${MAGIC_FILE}" + echo "1" > "${DATA}/${IST_FILE}" + get_keys + if [[ $encrypt -eq 1 ]];then + if [[ -n $scomp ]];then + tcmd=" $ecmd | $scomp | $tcmd " + else + tcmd=" $ecmd | $tcmd " + fi + elif [[ -n $scomp ]];then + tcmd=" $scomp | $tcmd " + fi + strmcmd+=" \${IST_FILE}" + + send_donor $DATA "${stagemsg}-IST" + + fi + + echo "done ${WSREP_SST_OPT_GTID}" + wsrep_log_info "Total time on donor: $totime seconds" + +elif [ "${WSREP_SST_OPT_ROLE}" = "joiner" ] +then + [[ -e $SST_PROGRESS_FILE ]] && wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE" + [[ -n $SST_PROGRESS_FILE ]] && touch $SST_PROGRESS_FILE + + ib_home_dir=$(parse_cnf mysqld innodb-data-home-dir "") + ib_log_dir=$(parse_cnf mysqld innodb-log-group-home-dir "") + ib_undo_dir=$(parse_cnf mysqld innodb-undo-directory "") + + stagemsg="Joiner-Recv" + + + sencrypted=1 + nthreads=1 + + MODULE="xtrabackup_sst" + + rm -f "${DATA}/${IST_FILE}" + + # May need xtrabackup_checkpoints later on + rm -f ${DATA}/xtrabackup_binary ${DATA}/xtrabackup_galera_info ${DATA}/xtrabackup_logfile + + ADDR=${WSREP_SST_OPT_ADDR} + if [ -z "${SST_PORT}" ] + then + SST_PORT=4444 + ADDR="$(echo ${WSREP_SST_OPT_ADDR} | awk -F ':' '{ print $1 }'):${SST_PORT}" + fi + + wait_for_listen ${SST_PORT} ${ADDR} ${MODULE} & + + trap sig_joiner_cleanup HUP PIPE INT TERM + trap cleanup_joiner EXIT + + if [[ -n $progress ]];then + adjust_progress + tcmd+=" | $pcmd" + fi + + get_keys + if [[ $encrypt -eq 1 && $sencrypted -eq 1 ]];then + if [[ -n $sdecomp ]];then + strmcmd=" $sdecomp | $ecmd | $strmcmd" + else + strmcmd=" $ecmd | $strmcmd" + fi + elif [[ -n $sdecomp ]];then + strmcmd=" $sdecomp | $strmcmd" + fi + + STATDIR=$(mktemp -d) + MAGIC_FILE="${STATDIR}/${INFO_FILE}" + recv_joiner $STATDIR "${stagemsg}-gtid" $stimeout 1 + + + if ! ps -p ${WSREP_SST_OPT_PARENT} &>/dev/null + then + wsrep_log_error "Parent mysqld process (PID:${WSREP_SST_OPT_PARENT}) terminated unexpectedly." + exit 32 + fi + + if [ ! -r "${STATDIR}/${IST_FILE}" ] + then + + if [[ -d ${DATA}/.sst ]];then + wsrep_log_info "WARNING: Stale temporary SST directory: ${DATA}/.sst from previous state transfer. Removing" + rm -rf ${DATA}/.sst + fi + mkdir -p ${DATA}/.sst + (recv_joiner $DATA/.sst "${stagemsg}-SST" 0 0) & + jpid=$! + wsrep_log_info "Proceeding with SST" + + + wsrep_log_info "Cleaning the existing datadir and innodb-data/log directories" + find $ib_home_dir $ib_log_dir $ib_undo_dir $DATA -mindepth 1 -regex $cpat -prune -o -exec rm -rfv {} 1>&2 \+ + + tempdir=$(parse_cnf mysqld log-bin "") + if [[ -n ${tempdir:-} ]];then + binlog_dir=$(dirname $tempdir) + binlog_file=$(basename $tempdir) + if [[ -n ${binlog_dir:-} && $binlog_dir != '.' && $binlog_dir != $DATA ]];then + pattern="$binlog_dir/$binlog_file\.[0-9]+$" + wsrep_log_info "Cleaning the binlog directory $binlog_dir as well" + find $binlog_dir -maxdepth 1 -type f -regex $pattern -exec rm -fv {} 1>&2 \+ || true + rm $binlog_dir/*.index || true + fi + fi + + + + TDATA=${DATA} + DATA="${DATA}/.sst" + + + MAGIC_FILE="${DATA}/${INFO_FILE}" + wsrep_log_info "Waiting for SST streaming to complete!" + wait $jpid + + get_proc + + if [[ ! -s ${DATA}/xtrabackup_checkpoints ]];then + wsrep_log_error "xtrabackup_checkpoints missing, failed innobackupex/SST on donor" + exit 2 + fi + + # Rebuild indexes for compact backups + if grep -q 'compact = 1' ${DATA}/xtrabackup_checkpoints;then + wsrep_log_info "Index compaction detected" + rebuild=1 + fi + + if [[ $rebuild -eq 1 ]];then + nthreads=$(parse_cnf xtrabackup rebuild-threads $nproc) + wsrep_log_info "Rebuilding during prepare with $nthreads threads" + rebuildcmd="--rebuild-indexes --rebuild-threads=$nthreads" + fi + + if test -n "$(find ${DATA} -maxdepth 1 -type f -name '*.qp' -print -quit)";then + + wsrep_log_info "Compressed qpress files found" + + if [[ ! -x `which qpress` ]];then + wsrep_log_error "qpress not found in path: $PATH" + exit 22 + fi + + if [[ -n $progress ]] && pv --help | grep -q 'line-mode';then + count=$(find ${DATA} -type f -name '*.qp' | wc -l) + count=$(( count*2 )) + if pv --help | grep -q FORMAT;then + pvopts="-f -s $count -l -N Decompression -F '%N => Rate:%r Elapsed:%t %e Progress: [%b/$count]'" + else + pvopts="-f -s $count -l -N Decompression" + fi + pcmd="pv $pvopts" + adjust_progress + dcmd="$pcmd | xargs -n 2 qpress -T${nproc}d" + else + dcmd="xargs -n 2 qpress -T${nproc}d" + fi + + + # Decompress the qpress files + wsrep_log_info "Decompression with $nproc threads" + timeit "Joiner-Decompression" "find ${DATA} -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd" + extcode=$? + + if [[ $extcode -eq 0 ]];then + wsrep_log_info "Removing qpress files after decompression" + find ${DATA} -type f -name '*.qp' -delete + if [[ $? -ne 0 ]];then + wsrep_log_error "Something went wrong with deletion of qpress files. Investigate" + fi + else + wsrep_log_error "Decompression failed. Exit code: $extcode" + exit 22 + fi + fi + + + if [[ ! -z $WSREP_SST_OPT_BINLOG ]];then + + BINLOG_DIRNAME=$(dirname $WSREP_SST_OPT_BINLOG) + BINLOG_FILENAME=$(basename $WSREP_SST_OPT_BINLOG) + + # To avoid comparing data directory and BINLOG_DIRNAME + mv $DATA/${BINLOG_FILENAME}.* $BINLOG_DIRNAME/ 2>/dev/null || true + + pushd $BINLOG_DIRNAME &>/dev/null + for bfiles in $(ls -1 ${BINLOG_FILENAME}.[0-9]*);do + echo ${BINLOG_DIRNAME}/${bfiles} >> ${BINLOG_FILENAME}.index + done + popd &> /dev/null + + fi + + wsrep_log_info "Preparing the backup at ${DATA}" + timeit "Xtrabackup prepare stage" "$INNOAPPLY" + + if [ $? -ne 0 ]; + then + wsrep_log_error "${INNOBACKUPEX_BIN} apply finished with errors. Check ${DATA}/innobackup.prepare.log" + exit 22 + fi + + MAGIC_FILE="${TDATA}/${INFO_FILE}" + set +e + rm $TDATA/innobackup.prepare.log $TDATA/innobackup.move.log + set -e + wsrep_log_info "Moving the backup to ${TDATA}" + timeit "Xtrabackup move stage" "$INNOMOVE" + if [[ $? -eq 0 ]];then + wsrep_log_info "Move successful, removing ${DATA}" + rm -rf $DATA + DATA=${TDATA} + else + wsrep_log_error "Move failed, keeping ${DATA} for further diagnosis" + wsrep_log_error "Check ${DATA}/innobackup.move.log for details" + exit 22 + fi + + + else + wsrep_log_info "${IST_FILE} received from donor: Running IST" + fi + + if [[ ! -r ${MAGIC_FILE} ]];then + wsrep_log_error "SST magic file ${MAGIC_FILE} not found/readable" + exit 2 + fi + wsrep_log_info "Galera co-ords from recovery: $(cat ${MAGIC_FILE})" + cat "${MAGIC_FILE}" # Output : UUID:seqno wsrep_gtid_domain_id + wsrep_log_info "Total time on joiner: $totime seconds" +fi + +exit 0 diff --git a/sql-common/client.c b/sql-common/client.c index 24e6bcf92e9..47a68651e05 100644 --- a/sql-common/client.c +++ b/sql-common/client.c @@ -1769,15 +1769,22 @@ mysql_get_ssl_cipher(MYSQL *mysql __attribute__((unused))) #if defined(HAVE_OPENSSL) +#if OPENSSL_VERSION_NUMBER >= 0x10002000L && !defined(HAVE_YASSL) +#include <openssl/x509v3.h> +#define HAVE_X509_check_host +#endif + static int ssl_verify_server_cert(Vio *vio, const char* server_hostname, const char **errptr) { SSL *ssl; X509 *server_cert= NULL; +#ifndef HAVE_X509_check_host char *cn= NULL; int cn_loc= -1; ASN1_STRING *cn_asn1= NULL; X509_NAME_ENTRY *cn_entry= NULL; X509_NAME *subject= NULL; +#endif int ret_validation= 1; DBUG_ENTER("ssl_verify_server_cert"); @@ -1812,14 +1819,9 @@ static int ssl_verify_server_cert(Vio *vio, const char* server_hostname, const c are what we expect. */ - /* - Some notes for future development - We should check host name in alternative name first and then if needed check in common name. - Currently yssl doesn't support alternative name. - openssl 1.0.2 support X509_check_host method for host name validation, we may need to start using - X509_check_host in the future. - */ - +#ifdef HAVE_X509_check_host + ret_validation= X509_check_host(server_cert, server_hostname, 0, 0, 0) != 1; +#else subject= X509_get_subject_name(server_cert); cn_loc= X509_NAME_get_index_by_NID(subject, NID_commonName, -1); if (cn_loc < 0) @@ -1827,7 +1829,6 @@ static int ssl_verify_server_cert(Vio *vio, const char* server_hostname, const c *errptr= "Failed to get CN location in the certificate subject"; goto error; } - cn_entry= X509_NAME_get_entry(subject, cn_loc); if (cn_entry == NULL) { @@ -1856,7 +1857,7 @@ static int ssl_verify_server_cert(Vio *vio, const char* server_hostname, const c /* Success */ ret_validation= 0; } - +#endif *errptr= "SSL certificate validation failure"; error: @@ -3408,7 +3409,7 @@ CLI_MYSQL_REAL_CONNECT(MYSQL *mysql,const char *host, const char *user, if (mysql->options.extension && mysql->options.extension->async_context) net->vio->async_context= mysql->options.extension->async_context; - if (my_net_init(net, net->vio, 0, MYF(0))) + if (my_net_init(net, net->vio, _current_thd(), MYF(0))) { vio_delete(net->vio); net->vio = 0; diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index f46d8e4ee83..c0a123a0c74 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -82,39 +82,39 @@ SET (SQL_SOURCE filesort_utils.cc filesort.cc gstream.cc signal_handler.cc - handler.cc hash_filo.h + handler.cc hostname.cc init.cc item.cc item_buff.cc item_cmpfunc.cc item_create.cc item_func.cc item_geofunc.cc item_row.cc item_strfunc.cc item_subselect.cc item_sum.cc item_timefunc.cc key.cc log.cc lock.cc log_event.cc rpl_record.cc rpl_reporting.cc log_event_old.cc rpl_record_old.cc - message.h mf_iocache.cc my_decimal.cc + mf_iocache.cc my_decimal.cc mysqld.cc net_serv.cc keycaches.cc ../sql-common/client_plugin.c - opt_range.cc opt_range.h opt_sum.cc + opt_range.cc opt_sum.cc ../sql-common/pack.c parse_file.cc password.c procedure.cc protocol.cc records.cc repl_failsafe.cc rpl_filter.cc session_tracker.cc set_var.cc slave.cc sp.cc sp_cache.cc sp_head.cc sp_pcontext.cc sp_rcontext.cc spatial.cc sql_acl.cc sql_analyse.cc sql_base.cc - sql_cache.cc sql_class.cc sql_client.cc sql_crypt.cc sql_crypt.h + sql_cache.cc sql_class.cc sql_client.cc sql_crypt.cc sql_cursor.cc sql_db.cc sql_delete.cc sql_derived.cc sql_digest.cc sql_do.cc sql_error.cc sql_handler.cc sql_get_diagnostics.cc sql_help.cc sql_insert.cc sql_lex.cc sql_list.cc sql_load.cc sql_manager.cc - sql_parse.cc sql_bootstrap.cc sql_bootstrap.h + sql_parse.cc sql_bootstrap.cc sql_partition.cc sql_plugin.cc sql_prepare.cc sql_rename.cc - debug_sync.cc debug_sync.h + debug_sync.cc sql_repl.cc sql_select.cc sql_show.cc sql_state.c group_by_handler.cc sql_statistics.cc sql_string.cc sql_table.cc sql_test.cc sql_trigger.cc sql_udf.cc sql_union.cc sql_update.cc sql_view.cc strfunc.cc table.cc thr_malloc.cc sql_time.cc tztime.cc unireg.cc item_xmlfunc.cc - uniques.cc uniques.h + uniques.cc rpl_tblmap.cc sql_binlog.cc event_scheduler.cc event_data_objects.cc event_queue.cc event_db_repository.cc sql_tablespace.cc events.cc ../sql-common/my_user.c @@ -124,23 +124,23 @@ SET (SQL_SOURCE sql_profile.cc event_parse_data.cc sql_alter.cc sql_signal.cc rpl_handler.cc mdl.cc sql_admin.cc transaction.cc sys_vars.cc sql_truncate.cc datadict.cc - sql_reload.cc sql_cmd.h item_inetfunc.cc + sql_reload.cc item_inetfunc.cc # added in MariaDB: - sql_explain.h sql_explain.cc - sql_analyze_stmt.h sql_analyze_stmt.cc - sql_lifo_buffer.h sql_join_cache.h sql_join_cache.cc + sql_explain.cc + sql_analyze_stmt.cc + sql_join_cache.cc create_options.cc multi_range_read.cc opt_index_cond_pushdown.cc opt_subselect.cc opt_table_elimination.cc sql_expression_cache.cc gcalc_slicescan.cc gcalc_tools.cc threadpool_common.cc ../sql-common/mysql_async.c - my_apc.cc my_apc.h mf_iocache_encr.cc item_jsonfunc.cc - my_json_writer.cc my_json_writer.h + my_apc.cc mf_iocache_encr.cc item_jsonfunc.cc + my_json_writer.cc rpl_gtid.cc rpl_parallel.cc - sql_type.cc sql_type.h + sql_type.cc item_windowfunc.cc sql_window.cc - sql_cte.cc sql_cte.h + sql_cte.cc ${WSREP_SOURCES} table_cache.cc encryption.cc temporary_tables.cc ${CMAKE_CURRENT_BINARY_DIR}/sql_builtin.cc @@ -176,7 +176,7 @@ TARGET_LINK_LIBRARIES(sql ${MYSQLD_STATIC_PLUGIN_LIBS} ${LIBSYSTEMD}) IF(WIN32) - SET(MYSQLD_SOURCE main.cc nt_servc.cc nt_servc.h message.rc) + SET(MYSQLD_SOURCE main.cc nt_servc.cc message.rc) TARGET_LINK_LIBRARIES(sql psapi) ELSE() SET(MYSQLD_SOURCE main.cc ${DTRACE_PROBES_ALL}) diff --git a/sql/innodb_priv.h b/sql/innodb_priv.h index ec85aa352f8..27aa9ac8645 100644 --- a/sql/innodb_priv.h +++ b/sql/innodb_priv.h @@ -28,6 +28,7 @@ void localtime_to_TIME(MYSQL_TIME *to, struct tm *from); uint strconvert(CHARSET_INFO *from_cs, const char *from, uint from_length, CHARSET_INFO *to_cs, char *to, uint to_length, uint *errors); + void sql_print_error(const char *format, ...); #define thd_binlog_pos(X, Y, Z) mysql_bin_log_commit_pos(X, Z, Y) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 68c8f6aa781..2bbdff6ac04 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -3188,7 +3188,7 @@ LONG WINAPI my_unhandler_exception_filter(EXCEPTION_POINTERS *ex_pointers) } -static void init_signals(void) +void init_signals(void) { if(opt_console) SetConsoleCtrlHandler(console_event_handler,TRUE); @@ -3319,7 +3319,7 @@ static size_t my_setstacksize(pthread_attr_t *attr, size_t stacksize) #ifndef EMBEDDED_LIBRARY -static void init_signals(void) +void init_signals(void) { sigset_t set; struct sigaction sa; @@ -6296,7 +6296,7 @@ static void bootstrap(MYSQL_FILE *file) thd->variables.wsrep_on= 0; #endif thd->bootstrap=1; - my_net_init(&thd->net,(st_vio*) 0, (void*) 0, MYF(0)); + my_net_init(&thd->net,(st_vio*) 0, thd, MYF(0)); thd->max_client_packet_length= thd->net.max_packet; thd->security_ctx->master_access= ~(ulong)0; in_bootstrap= TRUE; diff --git a/sql/mysqld.h b/sql/mysqld.h index 613b57b133d..4c864e90f1b 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -778,20 +778,6 @@ inline void dec_thread_running() extern void set_server_version(char *buf, size_t size); -#if defined(MYSQL_DYNAMIC_PLUGIN) && defined(_WIN32) -extern "C" THD *_current_thd_noinline(); -#define _current_thd() _current_thd_noinline() -#else -/* - THR_THD is a key which will be used to set/get THD* for a thread, - using my_pthread_setspecific_ptr()/my_thread_getspecific_ptr(). -*/ -extern pthread_key(THD*, THR_THD); -inline THD *_current_thd(void) -{ - return my_pthread_getspecific_ptr(THD*,THR_THD); -} -#endif #define current_thd _current_thd() inline int set_current_thd(THD *thd) { diff --git a/sql/sql_class.cc b/sql/sql_class.cc index f8ba23298e0..43419472026 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -4006,16 +4006,12 @@ my_bool thd_net_is_killed() void thd_increment_bytes_received(void *thd, ulong length) { - if (unlikely(!thd)) // Called from federatedx - thd= current_thd; ((THD*) thd)->status_var.bytes_received+= length; } void thd_increment_net_big_packet_count(void *thd, ulong length) { - if (unlikely(!thd)) // Called from federatedx - thd= current_thd; ((THD*) thd)->status_var.net_big_packet_count+= length; } diff --git a/sql/sql_plugin.cc b/sql/sql_plugin.cc index 101ea3fd3c7..d352d715aa5 100644 --- a/sql/sql_plugin.cc +++ b/sql/sql_plugin.cc @@ -1617,22 +1617,28 @@ int plugin_init(int *argc, char **argv, int flags) } } - /* First, we initialize only MyISAM - that should always succeed */ + /* + First, we initialize only MyISAM - that should almost always succeed + (almost always, because plugins can be loaded outside of the server, too). + */ plugin_ptr= plugin_find_internal(&MyISAM, MYSQL_STORAGE_ENGINE_PLUGIN); - DBUG_ASSERT(plugin_ptr); - DBUG_ASSERT(plugin_ptr->load_option == PLUGIN_FORCE); + DBUG_ASSERT(plugin_ptr || !mysql_mandatory_plugins[0]); + if (plugin_ptr) + { + DBUG_ASSERT(plugin_ptr->load_option == PLUGIN_FORCE); - if (plugin_initialize(&tmp_root, plugin_ptr, argc, argv, false)) - goto err_unlock; + if (plugin_initialize(&tmp_root, plugin_ptr, argc, argv, false)) + goto err_unlock; - /* - set the global default storage engine variable so that it will - not be null in any child thread. - */ - global_system_variables.table_plugin= - intern_plugin_lock(NULL, plugin_int_to_ref(plugin_ptr)); - DBUG_ASSERT(plugin_ptr->ref_count == 1); + /* + set the global default storage engine variable so that it will + not be null in any child thread. + */ + global_system_variables.table_plugin = + intern_plugin_lock(NULL, plugin_int_to_ref(plugin_ptr)); + DBUG_ASSERT(plugin_ptr->ref_count == 1); + } mysql_mutex_unlock(&LOCK_plugin); /* Register (not initialize!) all dynamic plugins */ diff --git a/sql/sql_plugin_services.ic b/sql/sql_plugin_services.ic index 2e71fac50be..ea6aefa5993 100644 --- a/sql/sql_plugin_services.ic +++ b/sql/sql_plugin_services.ic @@ -133,7 +133,7 @@ static struct base64_service_st base64_handler= { my_base64_decode }; -static struct thd_error_context_service_st thd_error_conext_handler= { +static struct thd_error_context_service_st thd_error_context_handler= { thd_get_error_message, thd_get_error_number, thd_get_error_row, @@ -196,6 +196,24 @@ static struct encryption_scheme_service_st encryption_scheme_handler= encryption_scheme_decrypt }; +static struct my_crypt_service_st crypt_handler= +{ + my_aes_crypt_init, + my_aes_crypt_update, + my_aes_crypt_finish, + my_aes_crypt, + my_aes_get_size, + my_aes_ctx_size, + my_random_bytes +}; + +static struct my_print_error_service_st my_print_error_handler= +{ + my_error, + my_printf_error, + my_printv_error +}; + static struct st_service_ref list_of_services[]= { { "base64_service", VERSION_base64, &base64_handler }, @@ -203,19 +221,21 @@ static struct st_service_ref list_of_services[]= { "encryption_scheme_service", VERSION_encryption_scheme, &encryption_scheme_handler }, { "encryption_service", VERSION_encryption, &encryption_handler }, { "logger_service", VERSION_logger, &logger_service_handler }, + { "my_crypt_service", VERSION_my_crypt, &crypt_handler}, { "my_md5_service", VERSION_my_md5, &my_md5_handler}, + { "my_print_error_service", VERSION_my_print_error, &my_print_error_handler}, { "my_sha1_service", VERSION_my_sha1, &my_sha1_handler}, { "my_sha2_service", VERSION_my_sha2, &my_sha2_handler}, { "my_snprintf_service", VERSION_my_snprintf, &my_snprintf_handler }, { "progress_report_service", VERSION_progress_report, &progress_report_handler }, { "thd_alloc_service", VERSION_thd_alloc, &thd_alloc_handler }, { "thd_autoinc_service", VERSION_thd_autoinc, &thd_autoinc_handler }, - { "thd_error_context_service", VERSION_thd_error_context, &thd_error_conext_handler }, + { "thd_error_context_service", VERSION_thd_error_context, &thd_error_context_handler }, { "thd_kill_statement_service", VERSION_kill_statement, &thd_kill_statement_handler }, { "thd_rnd_service", VERSION_thd_rnd, &thd_rnd_handler }, { "thd_specifics_service", VERSION_thd_specifics, &thd_specifics_handler }, { "thd_timezone_service", VERSION_thd_timezone, &thd_timezone_handler }, { "thd_wait_service", VERSION_thd_wait, &thd_wait_handler }, - { "wsrep_service", VERSION_wsrep, &wsrep_handler }, + { "wsrep_service", VERSION_wsrep, &wsrep_handler } }; diff --git a/sql/unireg.h b/sql/unireg.h index b1cab841092..b0cfb3841ef 100644 --- a/sql/unireg.h +++ b/sql/unireg.h @@ -54,8 +54,8 @@ #define ER(X) ER_THD(current_thd, (X)) #define ER_THD_OR_DEFAULT(thd,X) ((thd) ? ER_THD(thd, (X)) : ER_DEFAULT(X)) -#define ME_INFO (ME_HOLDTANG+ME_OLDWIN+ME_NOREFRESH) -#define ME_ERROR (ME_BELL+ME_OLDWIN+ME_NOREFRESH) +#define ME_INFO (ME_HOLDTANG | ME_NOREFRESH) +#define ME_ERROR (ME_BELL | ME_NOREFRESH) #define MYF_RW MYF(MY_WME+MY_NABP) /* Vid my_read & my_write */ #define SPECIAL_USE_LOCKS 1 /* Lock used databases */ diff --git a/storage/connect/mysql-test/connect/t/secure_file_priv.test b/storage/connect/mysql-test/connect/t/secure_file_priv.test index 46633502034..f7792536892 100644 --- a/storage/connect/mysql-test/connect/t/secure_file_priv.test +++ b/storage/connect/mysql-test/connect/t/secure_file_priv.test @@ -10,4 +10,4 @@ let $SECUREDIR= `select @@secure_file_priv`; INSERT INTO t1 VALUES (10); SELECT * FROM t1; DROP TABLE t1; ---remove_file $SECUREDIR/t1.dbf +--remove_file $MYSQL_TMP_DIR/t1.dbf diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index 1ecf80bb92d..22aaa8a6f5d 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -164,6 +164,7 @@ MYSQL_ADD_PLUGIN(innobase ${INNOBASE_SOURCES} STORAGE_ENGINE ${ZLIB_LIBRARY} ${CRC32_VPMSUM_LIBRARY} ${NUMA_LIBRARY} + ${LIBSYSTEMD} ${LINKER_SCRIPT}) IF(WITH_INNOBASE_STORAGE_ENGINE) diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index a5e9c9e8a36..e63c234b2f6 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -772,20 +772,18 @@ buf_page_is_checksum_valid_none( && checksum_field1 == BUF_NO_CHECKSUM_MAGIC); } -/** Checks if a page is corrupt. -@param[in] check_lsn true if we need to check and complain about -the LSN +/** Check if a page is corrupt. +@param[in] check_lsn whether the LSN should be checked @param[in] read_buf database page @param[in] page_size page size @param[in] space tablespace -@return TRUE if corrupted */ +@return whether the page is corrupted */ bool buf_page_is_corrupted( bool check_lsn, const byte* read_buf, const page_size_t& page_size, - const fil_space_t* space -) + const fil_space_t* space) { ulint checksum_field1; ulint checksum_field2; @@ -5174,7 +5172,7 @@ buf_page_init( } } -/** Inits a page for read to the buffer buf_pool. If the page is +/** Initialize a page for read to the buffer buf_pool. If the page is (1) already in buf_pool, or (2) if we specify to read only ibuf pages and the page is not an ibuf page, or (3) if the space is deleted or being deleted, @@ -5185,15 +5183,17 @@ and the lock released later. @param[out] err DB_SUCCESS or DB_TABLESPACE_DELETED @param[in] mode BUF_READ_IBUF_PAGES_ONLY, ... @param[in] page_id page id -@param[in] unzip TRUE=request uncompressed page -@return pointer to the block or NULL */ +@param[in] unzip whether the uncompressed page is + requested (for ROW_FORMAT=COMPRESSED) +@return pointer to the block +@retval NULL in case of an error */ buf_page_t* buf_page_init_for_read( dberr_t* err, ulint mode, const page_id_t& page_id, const page_size_t& page_size, - ibool unzip) + bool unzip) { buf_block_t* block; buf_page_t* bpage = NULL; @@ -5720,12 +5720,14 @@ buf_mark_space_corrupt(buf_page_t* bpage) /** Check if page is maybe compressed, encrypted or both when we encounter corrupted page. Note that we can't be 100% sure if page is corrupted or decrypt/decompress just failed. -@param[in,out] bpage Page -@param[in,out] space tablespace -@retval DB_SUCCESS if page has been read and is not corrupted, -@retval DB_PAGE_CORRUPTED if page based on checksum check is corrupted, -@retval DB_DECRYPTION_FAILED if page post encryption checksum matches but -after decryption normal page checksum does not match. */ +@param[in,out] bpage page +@param[in,out] space tablespace from fil_space_acquire_for_io() +@return whether the operation succeeded +@retval DB_SUCCESS if page has been read and is not corrupted +@retval DB_PAGE_CORRUPTED if page based on checksum check is corrupted +@retval DB_DECRYPTION_FAILED if page post encryption checksum matches but +after decryption normal page checksum does not match. +@retval DB_TABLESPACE_DELETED if accessed tablespace is not found */ static dberr_t buf_page_check_corrupt(buf_page_t* bpage, fil_space_t* space) @@ -5737,7 +5739,6 @@ buf_page_check_corrupt(buf_page_t* bpage, fil_space_t* space) bool still_encrypted = false; dberr_t err = DB_SUCCESS; bool corrupted = false; - fil_space_crypt_t* crypt_data = space->crypt_data; /* In buf_decrypt_after_read we have either decrypted the page if @@ -7410,7 +7411,7 @@ buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space) bpage->id.space(), bpage->id.page_no())) { if (space->crypt_data->type != CRYPT_SCHEME_UNENCRYPTED) { - bpage->encrypted=true; + bpage->encrypted = true; } return (false); } @@ -7421,12 +7422,8 @@ buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space) ut_d(fil_page_type_validate(dst_frame)); /* decrypt using crypt_buf to dst_frame */ - byte* res = fil_space_decrypt(space, - slot->crypt_buf, - dst_frame, - &bpage->encrypted); - - if (!res) { + if (!fil_space_decrypt(space, slot->crypt_buf, + dst_frame, &bpage->encrypted)) { success = false; } diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index ea5cd5a3f4c..928521e789e 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -812,9 +812,8 @@ buf_page_is_zeroes( const byte* read_buf, const page_size_t& page_size); -/** Checks if a page is corrupt. -@param[in] check_lsn true if we need to check and complain about -the LSN +/** Check if a page is corrupt. +@param[in] check_lsn whether the LSN should be checked @param[in] read_buf database page @param[in] page_size page size @param[in] space tablespace @@ -824,8 +823,8 @@ buf_page_is_corrupted( bool check_lsn, const byte* read_buf, const page_size_t& page_size, - const fil_space_t* space = NULL -) MY_ATTRIBUTE((warn_unused_result)); + const fil_space_t* space = NULL) + MY_ATTRIBUTE((warn_unused_result)); #ifndef UNIV_INNOCHECKSUM /**********************************************************************//** Gets the space id, page offset, and byte offset within page of a @@ -1249,7 +1248,7 @@ buf_pointer_is_block_field( #define buf_pool_is_block_lock(l) \ buf_pointer_is_block_field((const void*)(l)) -/** Inits a page for read to the buffer buf_pool. If the page is +/** Initialize a page for read to the buffer buf_pool. If the page is (1) already in buf_pool, or (2) if we specify to read only ibuf pages and the page is not an ibuf page, or (3) if the space is deleted or being deleted, @@ -1260,15 +1259,17 @@ and the lock released later. @param[out] err DB_SUCCESS or DB_TABLESPACE_DELETED @param[in] mode BUF_READ_IBUF_PAGES_ONLY, ... @param[in] page_id page id -@param[in] unzip TRUE=request uncompressed page -@return pointer to the block or NULL */ +@param[in] unzip whether the uncompressed page is + requested (for ROW_FORMAT=COMPRESSED) +@return pointer to the block +@retval NULL in case of an error */ buf_page_t* buf_page_init_for_read( dberr_t* err, ulint mode, const page_id_t& page_id, const page_size_t& page_size, - ibool unzip); + bool unzip); /** Complete a read or write request of a file page to or from the buffer pool. @param[in,out] bpage Page to complete diff --git a/storage/innobase/log/log0crypt.cc b/storage/innobase/log/log0crypt.cc index dbfda8ab7c4..79301254a0a 100644 --- a/storage/innobase/log/log0crypt.cc +++ b/storage/innobase/log/log0crypt.cc @@ -26,7 +26,7 @@ MDEV-11782: Rewritten for MariaDB 10.2 by Marko Mäkelä, MariaDB Corporation. *******************************************************/ #include "m_string.h" #include "log0crypt.h" -#include "my_crypt.h" +#include <mysql/service_my_crypt.h> #include "log0crypt.h" #include "srv0start.h" // for srv_start_lsn diff --git a/storage/xtradb/CMakeLists.txt b/storage/xtradb/CMakeLists.txt index ba0797dd422..4f9d2bd2cbb 100644 --- a/storage/xtradb/CMakeLists.txt +++ b/storage/xtradb/CMakeLists.txt @@ -510,3 +510,6 @@ MYSQL_ADD_PLUGIN(xtradb ${INNOBASE_SOURCES} STORAGE_ENGINE IF(TARGET xtradb AND NOT XTRADB_OK) MESSAGE(FATAL_ERROR "Percona XtraDB is not supported on this platform") ENDIF() + +ADD_SUBDIRECTORY(${CMAKE_SOURCE_DIR}/extra/mariabackup ${CMAKE_BINARY_DIR}/extra/mariabackup) + diff --git a/storage/xtradb/btr/btr0btr.cc b/storage/xtradb/btr/btr0btr.cc index c94b539c2c7..d84c93f8b3e 100644 --- a/storage/xtradb/btr/btr0btr.cc +++ b/storage/xtradb/btr/btr0btr.cc @@ -722,7 +722,6 @@ btr_root_fseg_validate( /**************************************************************//** Gets the root node of a tree and x- or s-latches it. @return root page, x- or s-latched */ -static buf_block_t* btr_root_block_get( /*===============*/ @@ -1531,7 +1530,6 @@ btr_node_ptr_set_child_page_no( /************************************************************//** Returns the child page of a node pointer and x-latches it. @return child page, x-latched */ -static buf_block_t* btr_node_ptr_get_child( /*===================*/ diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index ebf6bb10caa..c57dab79ef7 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -65,6 +65,15 @@ Created 11/5/1995 Heikki Tuuri #include "fil0pagecompress.h" #include "ha_prototypes.h" +/** Decrypt a page. +@param[in,out] bpage Page control block +@param[in,out] space tablespace +@return whether the operation was successful */ +static +bool +buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space) + MY_ATTRIBUTE((nonnull)); + /* prototypes for new functions added to ha_innodb.cc */ trx_t* innobase_get_trx(); @@ -548,16 +557,13 @@ buf_block_alloc( } #endif /* !UNIV_HOTBACKUP */ -/********************************************************************//** -Checks if a page is all zeroes. -@return TRUE if the page is all zeroes */ +/** Check if a page is all zeroes. +@param[in] read_buf database page +@param[in] zip_size ROW_FORMAT=COMPRESSED page size, or 0 +@return whether the page is all zeroes */ UNIV_INTERN bool -buf_page_is_zeroes( -/*===============*/ - const byte* read_buf, /*!< in: a database page */ - const ulint zip_size) /*!< in: size of compressed page; - 0 for uncompressed pages */ +buf_page_is_zeroes(const byte* read_buf, ulint zip_size) { const ulint page_size = zip_size ? zip_size : UNIV_PAGE_SIZE; @@ -673,8 +679,7 @@ buf_page_is_checksum_valid_none( && checksum_field1 == BUF_NO_CHECKSUM_MAGIC); } -/********************************************************************//** -Checks if a page is corrupt. +/** Check if a page is corrupt. @param[in] check_lsn true if LSN should be checked @param[in] read_buf Page to be checked @param[in] zip_size compressed size or 0 @@ -4529,34 +4534,30 @@ buf_mark_space_corrupt( mutex_exit(&buf_pool->LRU_list_mutex); } -/********************************************************************//** -Check if page is maybe compressed, encrypted or both when we encounter +/** Check if page is maybe compressed, encrypted or both when we encounter corrupted page. Note that we can't be 100% sure if page is corrupted or decrypt/decompress just failed. -@param[in,out] bpage Page -@return DB_SUCCESS if page has been read and is not corrupted, -@retval DB_PAGE_CORRUPTED if page based on checksum check is corrupted, -@retval DB_DECRYPTION_FAILED if page post encryption checksum matches but +@param[in,out] bpage page +@param[in,out] space tablespace from fil_space_acquire_for_io() +@return whether the operation succeeded +@retval DB_SUCCESS if page has been read and is not corrupted +@retval DB_PAGE_CORRUPTED if page based on checksum check is corrupted +@retval DB_DECRYPTION_FAILED if page post encryption checksum matches but after decryption normal page checksum does not match. -@retval DB_TABLESPACE_DELETED if accessed tablespace is not found */ +@retval DB_TABLESPACE_DELETED if accessed tablespace is not found */ static dberr_t -buf_page_check_corrupt(buf_page_t* bpage) +buf_page_check_corrupt(buf_page_t* bpage, fil_space_t* space) { + ut_ad(space->n_pending_ios > 0); + ulint zip_size = buf_page_get_zip_size(bpage); byte* dst_frame = (zip_size) ? bpage->zip.data : ((buf_block_t*) bpage)->frame; - FilSpace space(bpage->space, true); bool still_encrypted = false; dberr_t err = DB_SUCCESS; bool corrupted = false; - fil_space_crypt_t* crypt_data = NULL; - - if (!space()) { - return(DB_TABLESPACE_DELETED); - } - - crypt_data = space()->crypt_data; + fil_space_crypt_t* crypt_data = space->crypt_data; /* In buf_decrypt_after_read we have either decrypted the page if page post encryption checksum matches and used key_id is found @@ -4568,12 +4569,12 @@ buf_page_check_corrupt(buf_page_t* bpage) crypt_data->type != CRYPT_SCHEME_UNENCRYPTED && !bpage->encrypted && fil_space_verify_crypt_checksum(dst_frame, zip_size, - space(), bpage->offset)); - + space, bpage->offset)); if (!still_encrypted) { /* If traditional checksums match, we assume that page is not anymore encrypted. */ - corrupted = buf_page_is_corrupted(true, dst_frame, zip_size, space()); + corrupted = buf_page_is_corrupted(true, dst_frame, zip_size, + space); if (!corrupted) { bpage->encrypted = false; @@ -4596,7 +4597,7 @@ buf_page_check_corrupt(buf_page_t* bpage) ", page number=%u]" " in file %s cannot be decrypted.", bpage->space, bpage->offset, - space()->name); + space->name); ib_logf(IB_LOG_LEVEL_INFO, "However key management plugin or used key_version " ULINTPF @@ -4614,26 +4615,23 @@ buf_page_check_corrupt(buf_page_t* bpage) return (err); } -/********************************************************************//** -Completes an asynchronous read or write request of a file page to or from -the buffer pool. +/** Complete a read or write request of a file page to or from the buffer pool. @param[in,out] bpage Page to complete -@return DB_SUCCESS if page has been read and is not corrupted, -DB_PAGE_CORRUPTED if page based on checksum check is corrupted, -DB_DECRYPTION_FAILED if page post encryption checksum matches but -after decryption normal page checksum does not match. -in write only DB_SUCCESS is possible. */ +@return whether the operation succeeded +@retval DB_SUCCESS always when writing, or if a read page was OK +@retval DB_PAGE_CORRUPTED if the checksum fails on a page read +@retval DB_DECRYPTION_FAILED if page post encryption checksum matches but + after decryption normal page checksum does + not match */ UNIV_INTERN dberr_t -buf_page_io_complete( - buf_page_t* bpage) +buf_page_io_complete(buf_page_t* bpage) { enum buf_io_fix io_type; buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); const ibool uncompressed = (buf_page_get_state(bpage) == BUF_BLOCK_FILE_PAGE); bool have_LRU_mutex = false; - fil_space_t* space = NULL; byte* frame = NULL; dberr_t err = DB_SUCCESS; @@ -4653,7 +4651,13 @@ buf_page_io_complete( ulint read_space_id = 0; uint key_version = 0; - buf_page_decrypt_after_read(bpage); + ut_ad(bpage->zip.data || ((buf_block_t*)bpage)->frame); + fil_space_t* space = fil_space_acquire_for_io(bpage->space); + if (!space) { + return(DB_TABLESPACE_DELETED); + } + + buf_page_decrypt_after_read(bpage, space); if (buf_page_get_zip_size(bpage)) { frame = bpage->zip.data; @@ -4727,7 +4731,7 @@ buf_page_io_complete( if (UNIV_LIKELY(!bpage->is_corrupt || !srv_pass_corrupt_table)) { - err = buf_page_check_corrupt(bpage); + err = buf_page_check_corrupt(bpage, space); } database_corrupted: @@ -4740,6 +4744,7 @@ database_corrupted: buf_mark_space_corrupt(bpage); ib_logf(IB_LOG_LEVEL_INFO, "Simulated page corruption"); + fil_space_release_for_io(space); return(err); } err = DB_SUCCESS; @@ -4747,9 +4752,6 @@ database_corrupted: ); if (err == DB_PAGE_CORRUPTED) { - fil_system_enter(); - space = fil_space_get_by_id(bpage->space); - ib_logf(IB_LOG_LEVEL_ERROR, "Database page corruption on disk" " or a failed file read of tablespace %s" @@ -4760,8 +4762,6 @@ database_corrupted: space->name, bpage->space, bpage->offset); - fil_system_exit(); - buf_page_print(frame, buf_page_get_zip_size(bpage), BUF_PAGE_PRINT_NO_CRASH); @@ -4798,6 +4798,7 @@ database_corrupted: table as corrupted instead of crashing server */ if (bpage->space > TRX_SYS_SPACE) { buf_mark_space_corrupt(bpage); + fil_space_release_for_io(space); return(err); } else { ib_logf(IB_LOG_LEVEL_FATAL, @@ -4836,6 +4837,8 @@ database_corrupted: } } + + fil_space_release_for_io(space); } else { /* io_type == BUF_IO_WRITE */ if (bpage->slot) { @@ -6306,16 +6309,17 @@ buf_page_encrypt_before_write( return dst_frame; } -/********************************************************************//** -Decrypt page after it has been read from disk -@param[in,out] bpage Page control block -@return true if successfull, false if something went wrong -*/ -UNIV_INTERN +/** Decrypt a page. +@param[in,out] bpage Page control block +@param[in,out] space tablespace +@return whether the operation was successful */ +static bool -buf_page_decrypt_after_read( - buf_page_t* bpage) +buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space) { + ut_ad(space->n_pending_ios > 0); + ut_ad(space->id == bpage->space); + ulint zip_size = buf_page_get_zip_size(bpage); ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; @@ -6333,12 +6337,10 @@ buf_page_decrypt_after_read( return (true); } - FilSpace space(bpage->space, false, true); - /* Page is encrypted if encryption information is found from tablespace and page contains used key_version. This is true also for pages first compressed and then encrypted. */ - if (!space() || !space()->crypt_data) { + if (!space->crypt_data) { key_version = 0; } @@ -6375,8 +6377,8 @@ buf_page_decrypt_after_read( /* Mark page encrypted in case it should be. */ - if (key_version && space()->crypt_data && - space()->crypt_data->type != CRYPT_SCHEME_UNENCRYPTED) { + if (space->crypt_data->type + != CRYPT_SCHEME_UNENCRYPTED) { bpage->encrypted = true; } @@ -6391,12 +6393,8 @@ buf_page_decrypt_after_read( #endif /* decrypt using crypt_buf to dst_frame */ - byte* res = fil_space_decrypt(space(), - slot->crypt_buf, - dst_frame, - &bpage->encrypted); - - if (!res) { + if (!fil_space_decrypt(space, slot->crypt_buf, + dst_frame, &bpage->encrypted)) { success = false; } @@ -6427,5 +6425,6 @@ buf_page_decrypt_after_read( } } + ut_ad(space->n_pending_ios > 0); return (success); } diff --git a/storage/xtradb/buf/buf0flu.cc b/storage/xtradb/buf/buf0flu.cc index 62a2468ba66..1f5c3993be7 100644 --- a/storage/xtradb/buf/buf0flu.cc +++ b/storage/xtradb/buf/buf0flu.cc @@ -873,7 +873,7 @@ buf_flush_write_block_low( buf_flush_t flush_type, /*!< in: type of flush */ bool sync) /*!< in: true if sync IO request */ { - fil_space_t* space = fil_space_acquire(bpage->space, true); + fil_space_t* space = fil_space_acquire_for_io(bpage->space); if (!space) { return; } @@ -995,6 +995,13 @@ buf_flush_write_block_low( ut_ad(flush_type == BUF_FLUSH_SINGLE_PAGE); fil_flush(space); + /* The tablespace could already have been dropped, + because fil_io(request, sync) would already have + decremented the node->n_pending. However, + buf_page_io_complete() only needs to look up the + tablespace during read requests, not during writes. */ + ut_ad(buf_page_get_io_fix_unlocked(bpage) == BUF_IO_WRITE); + #ifdef UNIV_DEBUG dberr_t err = #endif @@ -1003,7 +1010,7 @@ buf_flush_write_block_low( ut_ad(err == DB_SUCCESS); } - fil_space_release(space); + fil_space_release_for_io(space); /* Increment the counter of I/O operations used for selecting LRU policy. */ @@ -2864,7 +2871,7 @@ DECLARE_THREAD(buf_flush_page_cleaner_thread)( success = buf_flush_list(PCT_IO(100), LSN_MAX, &n_flushed); buf_flush_wait_batch_end(NULL, BUF_FLUSH_LIST); - } while (!success || n_flushed > 0); + } while (!success || n_flushed > 0 || (IS_XTRABACKUP() && buf_get_n_pending_read_ios() > 0)); /* Some sanity checks */ ut_a(srv_get_active_thread_type() == SRV_NONE); diff --git a/storage/xtradb/buf/buf0rea.cc b/storage/xtradb/buf/buf0rea.cc index e275eead4cc..85b04d37a08 100644 --- a/storage/xtradb/buf/buf0rea.cc +++ b/storage/xtradb/buf/buf0rea.cc @@ -955,11 +955,8 @@ buf_read_ibuf_merge_pages( tablespace_deleted: /* We have deleted or are deleting the single-table - tablespace: remove the entries for that page */ - - ibuf_merge_or_delete_for_page(NULL, space_ids[i], - page_nos[i], - zip_size, FALSE); + tablespace: remove the entries for tablespace. */ + ibuf_delete_for_discarded_space(space_ids[i]); break; case DB_DECRYPTION_FAILED: ib_logf(IB_LOG_LEVEL_ERROR, diff --git a/storage/xtradb/dict/dict0load.cc b/storage/xtradb/dict/dict0load.cc index 2dbde465369..4991c4f3fcc 100644 --- a/storage/xtradb/dict/dict0load.cc +++ b/storage/xtradb/dict/dict0load.cc @@ -945,6 +945,10 @@ dict_insert_tablespace_and_filepath( return(err); } +/* Set by Xtrabackup */ +my_bool (*dict_check_if_skip_table)(const char* name) = 0; + + /********************************************************************//** This function looks at each table defined in SYS_TABLES. It checks the tablespace for any table with a space_id > 0. It looks up the tablespace @@ -1064,6 +1068,9 @@ loop: bool is_temp = false; bool discarded = false; + bool print_error_if_does_not_exist; + bool remove_from_data_dict_if_does_not_exist; + ib_uint32_t flags2 = static_cast<ib_uint32_t>( mach_read_from_4(field)); @@ -1089,6 +1096,19 @@ loop: goto loop; } + + ut_a(!IS_XTRABACKUP() || dict_check_if_skip_table); + + if (is_temp || discarded || + (IS_XTRABACKUP() && dict_check_if_skip_table(name))) { + print_error_if_does_not_exist = false; + } + else { + print_error_if_does_not_exist = true; + } + + remove_from_data_dict_if_does_not_exist = IS_XTRABACKUP() && !(is_temp || discarded); + mtr_commit(&mtr); switch (dict_check) { @@ -1096,8 +1116,8 @@ loop: /* All tablespaces should have been found in fil_load_single_table_tablespaces(). */ if (fil_space_for_table_exists_in_mem( - space_id, name, !(is_temp || discarded), - false, NULL, 0, flags) + space_id, name, print_error_if_does_not_exist, + remove_from_data_dict_if_does_not_exist , false, NULL, 0, flags) && !(is_temp || discarded)) { /* If user changes the path of .ibd files in *.isl files before doing crash recovery , @@ -1130,7 +1150,7 @@ loop: trx_resurrect_table_locks(). */ if (fil_space_for_table_exists_in_mem( space_id, name, false, - false, NULL, 0, flags)) { + false, false, NULL, 0, flags)) { break; } /* fall through */ @@ -2383,7 +2403,7 @@ err_exit: table->file_unreadable = true; } else if (!fil_space_for_table_exists_in_mem( - table->space, name, false, true, heap, + table->space, name, false, IS_XTRABACKUP(), true, heap, table->id, table->flags)) { if (DICT_TF2_FLAG_IS_SET(table, DICT_TF2_TEMPORARY)) { diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index c1ec1c7b1fd..a7373f451bc 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -683,7 +683,7 @@ fil_space_encrypt( } fil_space_crypt_t* crypt_data = space->crypt_data; - ut_ad(space->n_pending_ops); + ut_ad(space->n_pending_ios > 0); ulint zip_size = fsp_flags_get_zip_size(space->flags); byte* tmp = fil_encrypt_buf(crypt_data, space->id, offset, lsn, src_frame, zip_size, dst_frame); @@ -860,7 +860,7 @@ fil_space_decrypt( *decrypted = false; ut_ad(space->crypt_data != NULL && space->crypt_data->is_encrypted()); - ut_ad(space->n_pending_ops > 0); + ut_ad(space->n_pending_ios > 0); bool encrypted = fil_space_decrypt( space->crypt_data, diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 7ac08cc0e97..e39be46840c 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2014, 2017, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2014, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -67,9 +67,11 @@ static ulint srv_data_read, srv_data_written; #include <fcntl.h> #endif #include "row0mysql.h" +#include "trx0purge.h" MYSQL_PLUGIN_IMPORT extern my_bool lower_case_file_system; + /* IMPLEMENTATION OF THE TABLESPACE MEMORY CACHE ============================================= @@ -247,18 +249,16 @@ fil_node_complete_io( ulint type); /*!< in: OS_FILE_WRITE or OS_FILE_READ; marks the node as modified if type == OS_FILE_WRITE */ -/*******************************************************************//** -Frees a space object from the tablespace memory cache. Closes the files in -the chain but does not delete them. There must not be any pending i/o's or +/** Free a space object from the tablespace memory cache. Close the files in +the chain but do not delete them. There must not be any pending i/o's or flushes on the files. -@return TRUE on success */ +The fil_system->mutex will be released. +@param[in] id tablespace ID +@param[in] x_latched whether the caller holds exclusive space->latch +@return whether the tablespace existed */ static -ibool -fil_space_free( -/*===========*/ - ulint id, /* in: space id */ - ibool x_latched); /* in: TRUE if caller has space->latch - in X mode */ +bool +fil_space_free_and_mutex_exit(ulint id, bool x_latched); /********************************************************************//** Reads data from a space to a buffer. Remember that the possible incomplete blocks at the end of file are ignored: they are not taken into account when @@ -369,7 +369,6 @@ fil_node_get_space_id( /*******************************************************************//** Returns the table space by a given name, NULL if not found. */ -UNIV_INLINE fil_space_t* fil_space_get_by_name( /*==================*/ @@ -1368,18 +1367,14 @@ retry: } } -/*******************************************************************//** -Frees a file node object from a tablespace memory cache. */ +/** Prepare a data file object for freeing. +@param[in,out] space tablespace +@param[in,out] node data file */ static void -fil_node_free( -/*==========*/ - fil_node_t* node, /*!< in, own: file node */ - fil_system_t* system, /*!< in: tablespace memory cache */ - fil_space_t* space) /*!< in: space where the file node is chained */ +fil_node_free_part1(fil_space_t* space, fil_node_t* node) { - ut_ad(node && system && space); - ut_ad(mutex_own(&(system->mutex))); + ut_ad(mutex_own(&fil_system->mutex)); ut_a(node->magic_n == FIL_NODE_MAGIC_N); ut_a(node->n_pending == 0); ut_a(!node->being_extended); @@ -1402,12 +1397,22 @@ fil_node_free( space->is_in_unflushed_spaces = false; UT_LIST_REMOVE(unflushed_spaces, - system->unflushed_spaces, + fil_system->unflushed_spaces, space); } - fil_node_close_file(node, system); + fil_node_close_file(node, fil_system); } +} + +/** Free a data file object. +@param[in,out] space tablespace +@param[in] node data file */ +static +void +fil_node_free_part2(fil_space_t* space, fil_node_t* node) +{ + ut_ad(!node->open); space->size -= node->size; @@ -1447,7 +1452,8 @@ fil_space_truncate_start( trunc_len -= node->size * UNIV_PAGE_SIZE; - fil_node_free(node, fil_system, space); + fil_node_free_part1(space, node); + fil_node_free_part2(space, node); } mutex_exit(&fil_system->mutex); @@ -1539,10 +1545,9 @@ fil_space_create( "from the cache with id %lu", name, (ulong) id); - ibool success = fil_space_free(space->id, FALSE); + bool success = fil_space_free_and_mutex_exit( + space->id, false); ut_a(success); - - mutex_exit(&fil_system->mutex); } } while (space != 0); @@ -1574,12 +1579,13 @@ fil_space_create( if (!fil_system->space_id_reuse_warned) { fil_system->space_id_reuse_warned = TRUE; - - ib_logf(IB_LOG_LEVEL_WARN, - "Allocated tablespace %lu, old maximum " - "was %lu", - (ulong) id, - (ulong) fil_system->max_assigned_id); + if (!IS_XTRABACKUP()) { + ib_logf(IB_LOG_LEVEL_WARN, + "Allocated tablespace %lu, old maximum " + "was %lu", + (ulong)id, + (ulong)fil_system->max_assigned_id); + } } fil_system->max_assigned_id = id; @@ -1696,19 +1702,16 @@ fil_assign_new_space_id( return(success); } -/*******************************************************************//** -Frees a space object from the tablespace memory cache. Closes the files in -the chain but does not delete them. There must not be any pending i/o's or +/** Free a space object from the tablespace memory cache. Close the files in +the chain but do not delete them. There must not be any pending i/o's or flushes on the files. -@return TRUE if success */ +The fil_system->mutex will be released. +@param[in] id tablespace ID +@param[in] x_latched whether the caller holds exclusive space->latch +@return whether the tablespace existed */ static -ibool -fil_space_free( -/*===========*/ - /* out: TRUE if success */ - ulint id, /* in: space id */ - ibool x_latched) /* in: TRUE if caller has space->latch - in X mode */ +bool +fil_space_free_and_mutex_exit(ulint id, bool x_latched) { fil_space_t* space; fil_space_t* fnamespace; @@ -1718,13 +1721,11 @@ fil_space_free( space = fil_space_get_by_id(id); if (!space) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Error: trying to remove tablespace %lu" - " from the cache but\n" - "InnoDB: it is not there.\n", (ulong) id); - - return(FALSE); + ib_logf(IB_LOG_LEVEL_ERROR, + "trying to remove non-existing tablespace " ULINTPF, + id); + mutex_exit(&fil_system->mutex); + return(false); } HASH_DELETE(fil_space_t, hash, fil_system->spaces, id, space); @@ -1756,11 +1757,25 @@ fil_space_free( ut_a(space->magic_n == FIL_SPACE_MAGIC_N); ut_a(0 == space->n_pending_flushes); + for (fil_node_t* node = UT_LIST_GET_FIRST(space->chain); + node != NULL; + node = UT_LIST_GET_NEXT(chain, node)) { + fil_node_free_part1(space, node); + } + + mutex_exit(&fil_system->mutex); + + /* Wait for fil_space_release_for_io(); after + fil_space_detach(), the tablespace cannot be found, so + fil_space_acquire_for_io() would return NULL */ + while (space->n_pending_ios) { + os_thread_sleep(100); + } + for (fil_node_t* fil_node = UT_LIST_GET_FIRST(space->chain); fil_node != NULL; fil_node = UT_LIST_GET_FIRST(space->chain)) { - - fil_node_free(fil_node, fil_system, space); + fil_node_free_part2(space, fil_node); } ut_a(0 == UT_LIST_GET_LEN(space->chain)); @@ -2160,7 +2175,11 @@ fil_close_all_files(void) space = UT_LIST_GET_NEXT(space_list, space); - fil_space_free(prev_space->id, FALSE); + /* This is executed during shutdown. No other thread + can create or remove tablespaces while we are not + holding fil_system->mutex. */ + fil_space_free_and_mutex_exit(prev_space->id, false); + mutex_enter(&fil_system->mutex); } mutex_exit(&fil_system->mutex); @@ -2208,7 +2227,11 @@ fil_close_log_files( space = UT_LIST_GET_NEXT(space_list, space); if (free) { - fil_space_free(prev_space->id, FALSE); + /* This is executed during startup. No other thread + can create or remove tablespaces while we are not + holding fil_system->mutex. */ + fil_space_free_and_mutex_exit(prev_space->id, false); + mutex_enter(&fil_system->mutex); } } @@ -2413,6 +2436,19 @@ fil_read_first_page( const char* check_msg = NULL; fil_space_crypt_t* cdata; + if (IS_XTRABACKUP() && srv_backup_mode) { + /* Files smaller than page size may occur + in xtrabackup, when server creates new file + but has not yet written into it, or wrote only + partially. Checks size here, to avoid exit in os_file_read. + This file will be skipped by xtrabackup if it is too small. + */ + os_offset_t file_size; + file_size = os_file_get_size(data_file); + if (file_size < FIL_IBD_FILE_INITIAL_SIZE*UNIV_PAGE_SIZE) { + return "File size is less than minimum"; + } + } buf = static_cast<byte*>(ut_malloc(2 * UNIV_PAGE_SIZE)); /* Align the memory for a possible read from a raw device */ @@ -2443,7 +2479,9 @@ fil_read_first_page( } } - check_msg = fil_check_first_page(page, *space_id, *flags); + if (!(IS_XTRABACKUP() && srv_backup_mode)) { + check_msg = fil_check_first_page(page, *space_id, *flags); + } } flushed_lsn = mach_read_from_8(page + @@ -3003,15 +3041,13 @@ fil_close_tablespace( /* If the free is successful, the X lock will be released before the space memory data structure is freed. */ - if (!fil_space_free(id, TRUE)) { + if (!fil_space_free_and_mutex_exit(id, TRUE)) { rw_lock_x_unlock(&space->latch); err = DB_TABLESPACE_NOT_FOUND; } else { err = DB_SUCCESS; } - mutex_exit(&fil_system->mutex); - /* If it is a delete then also delete any generated files, otherwise when we drop the database the remove directory will fail. */ @@ -3120,12 +3156,10 @@ fil_delete_tablespace( ut_a(node->n_pending == 0); } - if (!fil_space_free(id, TRUE)) { + if (!fil_space_free_and_mutex_exit(id, true)) { err = DB_TABLESPACE_NOT_FOUND; } - mutex_exit(&fil_system->mutex); - if (err != DB_SUCCESS) { rw_lock_x_unlock(&space->latch); } else if (!os_file_delete(innodb_file_data_key, path) @@ -3137,7 +3171,7 @@ fil_delete_tablespace( err = DB_IO_ERROR; } - if (err == DB_SUCCESS) { + if (err == DB_SUCCESS && !IS_XTRABACKUP()) { #ifndef UNIV_HOTBACKUP /* Write a log record about the deletion of the .ibd file, so that mysqlbackup can replay it in the @@ -3536,7 +3570,7 @@ skip_second_rename: mutex_exit(&fil_system->mutex); #ifndef UNIV_HOTBACKUP - if (success && !recv_recovery_on) { + if (success && !recv_recovery_on && !IS_XTRABACKUP()) { mtr_t mtr; mtr_start(&mtr); @@ -3782,7 +3816,18 @@ fil_create_new_single_table_tablespace( ibool success; /* TRUE if a table is created with CREATE TEMPORARY TABLE */ bool is_temp = !!(flags2 & DICT_TF2_TEMPORARY); - bool has_data_dir = FSP_FLAGS_HAS_DATA_DIR(flags) != 0; + + + /* For XtraBackup recovery we force remote tablespaces to be local, + i.e. never execute the code path corresponding to has_data_dir == true. + We don't create .isl files either, because we rely on innobackupex to + copy them under a global lock, and use them to copy remote tablespaces + to their proper locations on --copy-back. + + See also MySQL bug #72022: dir_path is always NULL for remote + tablespaces when a MLOG_FILE_CREATE* log record is replayed (the remote + directory is not available from MLOG_FILE_CREATE*). */ + bool has_data_dir = FSP_FLAGS_HAS_DATA_DIR(flags) != 0 && !IS_XTRABACKUP(); ulint atomic_writes = FSP_FLAGS_GET_ATOMIC_WRITES(flags); fil_space_crypt_t *crypt_data = NULL; @@ -3964,6 +4009,7 @@ fil_create_new_single_table_tablespace( } #ifndef UNIV_HOTBACKUP + if (!IS_XTRABACKUP()) { mtr_t mtr; ulint mlog_file_flag = 0; @@ -4004,6 +4050,138 @@ error_exit_3: return(err); } +#include "pars0pars.h" +#include "que0que.h" +#include "dict0priv.h" +static +void +fil_remove_invalid_table_from_data_dict(const char *name) +{ + trx_t* trx; + pars_info_t* info = NULL; + + trx = trx_allocate_for_mysql(); + trx_start_for_ddl(trx, TRX_DICT_OP_TABLE); + + ut_ad(mutex_own(&dict_sys->mutex)); + + trx->op_info = "removing invalid table from data dictionary"; + + info = pars_info_create(); + + pars_info_add_str_literal(info, "table_name", name); + + que_eval_sql(info, + "PROCEDURE DROP_TABLE_PROC () IS\n" + "sys_foreign_id CHAR;\n" + "table_id CHAR;\n" + "index_id CHAR;\n" + "foreign_id CHAR;\n" + "found INT;\n" + + "DECLARE CURSOR cur_fk IS\n" + "SELECT ID FROM SYS_FOREIGN\n" + "WHERE FOR_NAME = :table_name\n" + "AND TO_BINARY(FOR_NAME)\n" + " = TO_BINARY(:table_name)\n" + "LOCK IN SHARE MODE;\n" + + "DECLARE CURSOR cur_idx IS\n" + "SELECT ID FROM SYS_INDEXES\n" + "WHERE TABLE_ID = table_id\n" + "LOCK IN SHARE MODE;\n" + + "BEGIN\n" + "SELECT ID INTO table_id\n" + "FROM SYS_TABLES\n" + "WHERE NAME = :table_name\n" + "LOCK IN SHARE MODE;\n" + "IF (SQL % NOTFOUND) THEN\n" + " RETURN;\n" + "END IF;\n" + "found := 1;\n" + "SELECT ID INTO sys_foreign_id\n" + "FROM SYS_TABLES\n" + "WHERE NAME = 'SYS_FOREIGN'\n" + "LOCK IN SHARE MODE;\n" + "IF (SQL % NOTFOUND) THEN\n" + " found := 0;\n" + "END IF;\n" + "IF (:table_name = 'SYS_FOREIGN') THEN\n" + " found := 0;\n" + "END IF;\n" + "IF (:table_name = 'SYS_FOREIGN_COLS') THEN\n" + " found := 0;\n" + "END IF;\n" + "OPEN cur_fk;\n" + "WHILE found = 1 LOOP\n" + " FETCH cur_fk INTO foreign_id;\n" + " IF (SQL % NOTFOUND) THEN\n" + " found := 0;\n" + " ELSE\n" + " DELETE FROM SYS_FOREIGN_COLS\n" + " WHERE ID = foreign_id;\n" + " DELETE FROM SYS_FOREIGN\n" + " WHERE ID = foreign_id;\n" + " END IF;\n" + "END LOOP;\n" + "CLOSE cur_fk;\n" + "found := 1;\n" + "OPEN cur_idx;\n" + "WHILE found = 1 LOOP\n" + " FETCH cur_idx INTO index_id;\n" + " IF (SQL % NOTFOUND) THEN\n" + " found := 0;\n" + " ELSE\n" + " DELETE FROM SYS_FIELDS\n" + " WHERE INDEX_ID = index_id;\n" + " DELETE FROM SYS_INDEXES\n" + " WHERE ID = index_id\n" + " AND TABLE_ID = table_id;\n" + " END IF;\n" + "END LOOP;\n" + "CLOSE cur_idx;\n" + "DELETE FROM SYS_COLUMNS\n" + "WHERE TABLE_ID = table_id;\n" + "DELETE FROM SYS_TABLES\n" + "WHERE NAME = :table_name;\n" + "END;\n" + , FALSE, trx); + + /* SYS_DATAFILES and SYS_TABLESPACES do not necessarily exist + on XtraBackup recovery. See comments around + dict_create_or_check_foreign_constraint_tables() in + innobase_start_or_create_for_mysql(). */ + if (dict_table_get_low("SYS_DATAFILES") != NULL) { + info = pars_info_create(); + + pars_info_add_str_literal(info, "table_name", name); + + que_eval_sql(info, + "PROCEDURE DROP_TABLE_PROC () IS\n" + "space_id INT;\n" + + "BEGIN\n" + "SELECT SPACE INTO space_id\n" + "FROM SYS_TABLES\n" + "WHERE NAME = :table_name;\n" + "IF (SQL % NOTFOUND) THEN\n" + " RETURN;\n" + "END IF;\n" + "DELETE FROM SYS_TABLESPACES\n" + "WHERE SPACE = space_id;\n" + "DELETE FROM SYS_DATAFILES\n" + "WHERE SPACE = space_id;\n" + "END;\n" + , FALSE, trx); + } + + trx_commit_for_mysql(trx); + + trx_free_for_mysql(trx); +} + + #ifndef UNIV_HOTBACKUP /********************************************************************//** Report information about a bad tablespace. */ @@ -4144,8 +4322,10 @@ fil_open_single_table_tablespace( in the default location. If it is remote, it should not be here. */ def.filepath = fil_make_ibd_name(tablename, false); - /* The path_in was read from SYS_DATAFILES. */ - if (path_in) { + /* The path_in was read from SYS_DATAFILES. + We skip SYS_DATAFILES validation and remote tablespaces discovery for + XtraBackup, as all tablespaces are local for XtraBackup recovery. */ + if (path_in && !IS_XTRABACKUP()) { if (strcmp(def.filepath, path_in)) { dict.filepath = mem_strdup(path_in); /* possibility of multiple files. */ @@ -4287,12 +4467,19 @@ fil_open_single_table_tablespace( /* The following call prints an error message */ os_file_get_last_error(true); - ib_logf(IB_LOG_LEVEL_ERROR, + ib_logf(IS_XTRABACKUP() ? IB_LOG_LEVEL_WARN : IB_LOG_LEVEL_ERROR, "Could not find a valid tablespace file for '%s'. " "See " REFMAN "innodb-troubleshooting-datadict.html " "for how to resolve the issue.", tablename); + if (IS_XTRABACKUP() && fix_dict) { + ib_logf(IB_LOG_LEVEL_WARN, + "It will be removed from the data dictionary."); + if (purge_sys) { + fil_remove_invalid_table_from_data_dict(tablename); + } + } err = DB_CORRUPTION; goto cleanup_and_exit; @@ -4717,6 +4904,11 @@ check_first_page: } if (!fsp->success) { + if (IS_XTRABACKUP()) { + /* Do not attempt restore from doublewrite buffer + in Xtrabackup, this does not work.*/ + return; + } if (!restore_attempted) { if (!fil_user_tablespace_find_space_id(fsp)) { return; @@ -4784,6 +4976,10 @@ fil_load_single_table_tablespace( os_offset_t size; fil_space_t* space; + fsp_open_info* fsp; + ulong minimum_size; + ibool file_space_create_success; + memset(&def, 0, sizeof(def)); memset(&remote, 0, sizeof(remote)); @@ -4839,6 +5035,7 @@ fil_load_single_table_tablespace( # endif /* !UNIV_HOTBACKUP */ #endif + /* Check for a link file which locates a remote tablespace. */ remote.success = fil_open_linked_file( tablename, &remote.filepath, &remote.file, FALSE); @@ -4849,6 +5046,17 @@ fil_load_single_table_tablespace( if (!remote.success) { os_file_close(remote.file); mem_free(remote.filepath); + + if (srv_backup_mode && (remote.id == ULINT_UNDEFINED + || remote.id == 0)) { + + /* Ignore files that have uninitialized space + IDs on the backup stage. This means that a + tablespace has just been created and we will + replay the corresponding log records on + prepare. */ + goto func_exit_after_close; + } } } @@ -4863,6 +5071,18 @@ fil_load_single_table_tablespace( fil_validate_single_table_tablespace(tablename, &def); if (!def.success) { os_file_close(def.file); + + if (IS_XTRABACKUP() && srv_backup_mode && (def.id == ULINT_UNDEFINED + || def.id == 0)) { + + /* Ignore files that have uninitialized space + IDs on the backup stage. This means that a + tablespace has just been created and we will + replay the corresponding log records on + prepare. */ + + goto func_exit_after_close; + } } } @@ -4948,7 +5168,7 @@ will_not_choose: /* At this point, only one tablespace is open */ ut_a(def.success == !remote.success); - fsp_open_info* fsp = def.success ? &def : &remote; + fsp = def.success ? &def : &remote; /* Get and test the file size. */ size = os_file_get_size(fsp->file); @@ -4967,19 +5187,14 @@ will_not_choose: /* Every .ibd file is created >= 4 pages in size. Smaller files cannot be ok. */ - ulong minimum_size = FIL_IBD_FILE_INITIAL_SIZE * UNIV_PAGE_SIZE; + minimum_size = FIL_IBD_FILE_INITIAL_SIZE * UNIV_PAGE_SIZE; if (size < minimum_size) { -#ifndef UNIV_HOTBACKUP ib_logf(IB_LOG_LEVEL_ERROR, "The size of single-table tablespace file %s " "is only " UINT64PF ", should be at least %lu!", fsp->filepath, size, minimum_size); os_file_close(fsp->file); goto no_good_file; -#else - fsp->id = ULINT_UNDEFINED; - fsp->flags = 0; -#endif /* !UNIV_HOTBACKUP */ } #ifdef UNIV_HOTBACKUP @@ -5050,6 +5265,7 @@ will_not_choose: } mutex_exit(&fil_system->mutex); #endif /* UNIV_HOTBACKUP */ + /* Adjust the memory-based flags that would normally be set by dict_tf_to_fsp_flags(). In recovery, we have no data dictionary. */ if (FSP_FLAGS_HAS_PAGE_COMPRESSION(fsp->flags)) { @@ -5060,7 +5276,7 @@ will_not_choose: /* We will leave atomic_writes at ATOMIC_WRITES_DEFAULT. That will be adjusted in fil_space_for_table_exists_in_mem(). */ - ibool file_space_create_success = fil_space_create( + file_space_create_success = fil_space_create( tablename, fsp->id, fsp->flags, FIL_TABLESPACE, fsp->crypt_data, false); @@ -5088,13 +5304,56 @@ will_not_choose: } func_exit: - os_file_close(fsp->file); + /* We reuse file handles on the backup stage in XtraBackup to avoid + inconsistencies between the file name and the actual tablespace contents + if a DDL occurs between a fil_load_single_table_tablespaces() call and + the actual copy operation. */ + if (IS_XTRABACKUP() && srv_backup_mode && !srv_close_files) { + + fil_node_t* node; + fil_space_t* space; + + mutex_enter(&fil_system->mutex); + + space = fil_space_get_by_id(fsp->id); + + if (space) { + node = UT_LIST_GET_LAST(space->chain); + + /* The handle will be closed by xtrabackup in + xtrabackup_copy_datafile(). We set node->open to TRUE to + make sure no one calls fil_node_open_file() + (i.e. attempts to reopen the tablespace by name) during + the backup stage. */ + + node->open = TRUE; + node->handle = fsp->file; + + /* The following is copied from fil_node_open_file() to + pass fil_system validaty checks. We cannot use + fil_node_open_file() directly, as that would re-open the + file by name and create another file handle. */ + + fil_system->n_open++; + fil_n_file_opened++; + + if (fil_space_belongs_in_lru(space)) { + + /* Put the node to the LRU list */ + UT_LIST_ADD_FIRST(LRU, fil_system->LRU, node); + } + } + + mutex_exit(&fil_system->mutex); + } + else { + os_file_close(fsp->file); + } + -#ifdef UNIV_HOTBACKUP func_exit_after_close: -#else ut_ad(!mutex_own(&fil_system->mutex)); -#endif + mem_free(tablename); if (remote.success) { mem_free(remote.filepath); @@ -5108,7 +5367,7 @@ directory. We retry 100 times if os_file_readdir_next_file() returns -1. The idea is to read as much good data as we can and jump over bad data. @return 0 if ok, -1 if error even after the retries, 1 if at the end of the directory */ -static +UNIV_INTERN int fil_file_readdir_next_file( /*=======================*/ @@ -5138,6 +5397,9 @@ fil_file_readdir_next_file( return(-1); } + +my_bool(*fil_check_if_skip_database_by_path)(const char* name); + #define CHECK_TIME_EVERY_N_FILES 10 /********************************************************************//** At the server startup, if we need crash recovery, scans the database @@ -5149,7 +5411,7 @@ space id is != 0. @return DB_SUCCESS or error number */ UNIV_INTERN dberr_t -fil_load_single_table_tablespaces(void) +fil_load_single_table_tablespaces(ibool (*pred)(const char*, const char*)) /*===================================*/ { int ret; @@ -5208,7 +5470,19 @@ fil_load_single_table_tablespaces(void) "%s/%s", fil_path_to_mysql_datadir, dbinfo.name); srv_normalize_path_for_win(dbpath); - dbdir = os_file_opendir(dbpath, FALSE); + if (IS_XTRABACKUP()) { + ut_a(fil_check_if_skip_database_by_path); + if (fil_check_if_skip_database_by_path(dbpath)) { + fprintf(stderr, "Skipping db: %s\n", dbpath); + dbdir = NULL; + } else { + /* We want wrong directory permissions to be a fatal + error for XtraBackup. */ + dbdir = os_file_opendir(dbpath, TRUE); + } + } else { + dbdir = os_file_opendir(dbpath, FALSE); + } if (dbdir != NULL) { @@ -5224,14 +5498,20 @@ fil_load_single_table_tablespaces(void) goto next_file_item; } - /* We found a symlink or a file */ + /* We found a symlink or a file + + Ignore .isl files on XtraBackup + recovery, all tablespaces must be local. */ if (strlen(fileinfo.name) > 4 && (0 == strcmp(fileinfo.name + strlen(fileinfo.name) - 4, ".ibd") - || 0 == strcmp(fileinfo.name - + strlen(fileinfo.name) - 4, - ".isl"))) { + || ((!IS_XTRABACKUP() || srv_backup_mode) + && 0 == strcmp(fileinfo.name + + strlen(fileinfo.name) - 4, + ".isl"))) + && (!pred || + pred(dbinfo.name, fileinfo.name))) { /* The name ends in .ibd or .isl; try opening the file */ fil_load_single_table_tablespace( @@ -5387,6 +5667,9 @@ fil_space_for_table_exists_in_mem( information to the .err log if a matching tablespace is not found from memory */ + bool remove_from_data_dict_if_does_not_exist, + /*!< in: remove from the data dictionary + if tablespace does not exist */ bool adjust_space, /*!< in: whether to adjust space id when find table space mismatch */ mem_heap_t* heap, /*!< in: heap memory */ @@ -5457,6 +5740,11 @@ fil_space_for_table_exists_in_mem( if (fnamespace == NULL) { if (print_error_if_does_not_exist) { fil_report_missing_tablespace(name, id); + if (IS_XTRABACKUP() && remove_from_data_dict_if_does_not_exist) { + ib_logf(IB_LOG_LEVEL_WARN, + "It will be removed from " + "the data dictionary."); + } } } else { ut_print_timestamp(stderr); @@ -5918,7 +6206,7 @@ UNIV_INTERN ulint fil_space_get_block_size(const fil_space_t* space, unsigned offset) { - ut_ad(space->n_pending_ops > 0); + ut_ad(space->n_pending_ios > 0); ulint block_size = 512; @@ -6119,7 +6407,7 @@ _fil_io( /* Check that at least the start offset is within the bounds of a single-table tablespace, including rollback tablespaces. */ if (UNIV_UNLIKELY(node->size <= block_offset) - && space->id != 0 && space->purpose == FIL_TABLESPACE) { + && space->id != 0 && space->purpose == FIL_TABLESPACE) { fil_report_invalid_page_access( block_offset, space_id, space->name, byte_offset, @@ -6355,7 +6643,7 @@ UNIV_INTERN void fil_flush(fil_space_t* space) { - ut_ad(space->n_pending_ops > 0); + ut_ad(space->n_pending_ios > 0); if (!space->is_stopping()) { mutex_enter(&fil_system->mutex); @@ -7284,13 +7572,11 @@ Used by background threads that do not necessarily hold proper locks for concurrency control. @param[in] id tablespace ID @param[in] silent whether to silently ignore missing tablespaces -@param[in] for_io whether to look up the tablespace while performing I/O - (possibly executing TRUNCATE) @return the tablespace @retval NULL if missing or being deleted or truncated */ UNIV_INTERN fil_space_t* -fil_space_acquire_low(ulint id, bool silent, bool for_io) +fil_space_acquire_low(ulint id, bool silent) { fil_space_t* space; @@ -7303,7 +7589,7 @@ fil_space_acquire_low(ulint id, bool silent, bool for_io) ib_logf(IB_LOG_LEVEL_WARN, "Trying to access missing" " tablespace " ULINTPF ".", id); } - } else if (!for_io && space->is_stopping()) { + } else if (space->is_stopping()) { space = NULL; } else { space->n_pending_ops++; @@ -7314,8 +7600,44 @@ fil_space_acquire_low(ulint id, bool silent, bool for_io) return(space); } +/** Acquire a tablespace for reading or writing a block, +when it could be dropped concurrently. +@param[in] id tablespace ID +@return the tablespace +@retval NULL if missing */ +UNIV_INTERN +fil_space_t* +fil_space_acquire_for_io(ulint id) +{ + mutex_enter(&fil_system->mutex); + + fil_space_t* space = fil_space_get_by_id(id); + + if (space) { + space->n_pending_ios++; + } + + mutex_exit(&fil_system->mutex); + + return(space); +} + +/** Release a tablespace acquired with fil_space_acquire_for_io(). +@param[in,out] space tablespace to release */ +UNIV_INTERN +void +fil_space_release_for_io(fil_space_t* space) +{ + mutex_enter(&fil_system->mutex); + ut_ad(space->magic_n == FIL_SPACE_MAGIC_N); + ut_ad(space->n_pending_ios > 0); + space->n_pending_ios--; + mutex_exit(&fil_system->mutex); +} + /** Release a tablespace acquired with fil_space_acquire(). @param[in,out] space tablespace to release */ +UNIV_INTERN void fil_space_release(fil_space_t* space) { @@ -7334,6 +7656,7 @@ blocks a concurrent operation from dropping the tablespace. If NULL, use the first fil_space_t on fil_system->space_list. @return pointer to the next fil_space_t. @retval NULL if this was the last*/ +UNIV_INTERN fil_space_t* fil_space_next(fil_space_t* prev_space) { @@ -7401,6 +7724,7 @@ blocks a concurrent operation from dropping the tablespace. If NULL, use the first fil_space_t on fil_system->space_list. @return pointer to the next fil_space_t. @retval NULL if this was the last*/ +UNIV_INTERN fil_space_t* fil_space_keyrotate_next( fil_space_t* prev_space) diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 91c9fc7824b..4353bc1d288 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -289,7 +289,8 @@ static TYPELIB innodb_stats_method_typelib = { /** Possible values for system variables "innodb_checksum_algorithm" and "innodb_log_checksum_algorithm". */ -static const char* innodb_checksum_algorithm_names[] = { +UNIV_INTERN +const char* innodb_checksum_algorithm_names[] = { "CRC32", "STRICT_CRC32", "INNODB", @@ -301,7 +302,8 @@ static const char* innodb_checksum_algorithm_names[] = { /** Used to define an enumerate type of the system variables innodb_checksum_algorithm and innodb_log_checksum_algorithm. */ -static TYPELIB innodb_checksum_algorithm_typelib = { +UNIV_INTERN +TYPELIB innodb_checksum_algorithm_typelib = { array_elements(innodb_checksum_algorithm_names) - 1, "innodb_checksum_algorithm_typelib", innodb_checksum_algorithm_names, @@ -2018,7 +2020,9 @@ thd_supports_xa( THD* thd) /*!< in: thread handle, or NULL to query the global innodb_supports_xa */ { - return(THDVAR(thd, support_xa)); + /* THDVAR cannot be used in xtrabackup, + plugin variables for innodb are not loaded. */ + return (thd || !IS_XTRABACKUP())? THDVAR(thd, support_xa): FALSE; } /** Get the value of innodb_tmpdir. @@ -2051,7 +2055,9 @@ thd_fake_changes( THD* thd) /*!< in: thread handle, or NULL to query the global innodb_supports_xa */ { - return(THDVAR((THD*) thd, fake_changes)); + /* THDVAR cannot be used in xtrabackup, + plugin variables for innodb are not loaded */ + return (thd || !IS_XTRABACKUP())? THDVAR((THD*) thd, fake_changes) : FALSE ; } /******************************************************************//** @@ -2091,7 +2097,10 @@ thd_flush_log_at_trx_commit( /*================================*/ void* thd) { - return(THDVAR((THD*) thd, flush_log_at_trx_commit)); + /* THDVAR cannot be used in xtrabackup, + plugin variables for innodb are not loaded, + this makes xtrabackup crash when trying to use them. */ + return (thd || !IS_XTRABACKUP())? THDVAR((THD*)thd, flush_log_at_trx_commit) : FALSE; } /********************************************************************//** @@ -3058,7 +3067,7 @@ trx_is_started( /****************************************************************//** Update log_checksum_algorithm_ptr with a pointer to the function corresponding to a given checksum algorithm. */ -static + void innodb_log_checksum_func_update( /*============================*/ @@ -17397,11 +17406,11 @@ innodb_log_archive_update( if (in_val) { /* turn archiving on */ - srv_log_archive_on = innobase_log_archive = 1; + innobase_log_archive = srv_log_archive_on = 1; log_archive_archivelog(); } else { /* turn archivng off */ - srv_log_archive_on = innobase_log_archive = 0; + innobase_log_archive = srv_log_archive_on = 0; log_archive_noarchivelog(); } } @@ -21956,22 +21965,27 @@ ib_logf( str = static_cast<char*>(malloc(BUFSIZ)); my_vsnprintf(str, BUFSIZ, format, args); #endif /* __WIN__ */ - - switch(level) { - case IB_LOG_LEVEL_INFO: - sql_print_information("InnoDB: %s", str); - break; - case IB_LOG_LEVEL_WARN: - sql_print_warning("InnoDB: %s", str); - break; - case IB_LOG_LEVEL_ERROR: - sql_print_error("InnoDB: %s", str); - sd_notifyf(0, "STATUS=InnoDB: Error: %s", str); - break; - case IB_LOG_LEVEL_FATAL: - sql_print_error("InnoDB: %s", str); - sd_notifyf(0, "STATUS=InnoDB: Fatal: %s", str); - break; + if (!IS_XTRABACKUP()) { + switch (level) { + case IB_LOG_LEVEL_INFO: + sql_print_information("InnoDB: %s", str); + break; + case IB_LOG_LEVEL_WARN: + sql_print_warning("InnoDB: %s", str); + break; + case IB_LOG_LEVEL_ERROR: + sql_print_error("InnoDB: %s", str); + sd_notifyf(0, "STATUS=InnoDB: Error: %s", str); + break; + case IB_LOG_LEVEL_FATAL: + sql_print_error("InnoDB: %s", str); + sd_notifyf(0, "STATUS=InnoDB: Fatal: %s", str); + break; + } + } + else { + /* Don't use server logger for XtraBackup, just print to stderr. */ + fprintf(stderr, "InnoDB: %s\n", str); } va_end(args); diff --git a/storage/xtradb/include/buf0buf.h b/storage/xtradb/include/buf0buf.h index 08c3a765a8b..9b4276efaa8 100644 --- a/storage/xtradb/include/buf0buf.h +++ b/storage/xtradb/include/buf0buf.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2017, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -675,13 +675,13 @@ buf_page_is_checksum_valid_none( ulint checksum_field2) MY_ATTRIBUTE((warn_unused_result)); -/********************************************************************//** -Checks if a page is corrupt. +/** Check if a page is corrupt. @param[in] check_lsn true if LSN should be checked @param[in] read_buf Page to be checked @param[in] zip_size compressed size or 0 @param[in] space Pointer to tablespace @return true if corrupted, false if not */ +UNIV_INTERN bool buf_page_is_corrupted( bool check_lsn, @@ -689,15 +689,13 @@ buf_page_is_corrupted( ulint zip_size, const fil_space_t* space) MY_ATTRIBUTE((warn_unused_result)); -/********************************************************************//** -Checks if a page is all zeroes. -@return TRUE if the page is all zeroes */ +/** Check if a page is all zeroes. +@param[in] read_buf database page +@param[in] zip_size ROW_FORMAT=COMPRESSED page size, or 0 +@return whether the page is all zeroes */ +UNIV_INTERN bool -buf_page_is_zeroes( -/*===============*/ - const byte* read_buf, /*!< in: a database page */ - const ulint zip_size); /*!< in: size of compressed page; - 0 for uncompressed pages */ +buf_page_is_zeroes(const byte* read_buf, ulint zip_size); #ifndef UNIV_HOTBACKUP /**********************************************************************//** Gets the space id, page offset, and byte offset within page of a @@ -1259,18 +1257,18 @@ buf_page_init_for_read( version of the tablespace in case we have done DISCARD + IMPORT */ ulint offset);/*!< in: page number */ -/********************************************************************//** -Completes an asynchronous read or write request of a file page to or from -the buffer pool. -@param[in,out] bpage pointer to the block in question -@return DB_SUCCESS if page has been read and is not corrupted, -DB_PAGE_CORRUPTED if page based on checksum check is corrupted, -DB_DECRYPTION_FAILED if page post encryption checksum matches but -after decryption normal page checksum does not match.*/ +/** Complete a read or write request of a file page to or from the buffer pool. +@param[in,out] bpage Page to complete +@return whether the operation succeeded +@retval DB_SUCCESS always when writing, or if a read page was OK +@retval DB_PAGE_CORRUPTED if the checksum fails on a page read +@retval DB_DECRYPTION_FAILED if page post encryption checksum matches but + after decryption normal page checksum does + not match */ UNIV_INTERN dberr_t -buf_page_io_complete( - buf_page_t* bpage); +buf_page_io_complete(buf_page_t* bpage) + MY_ATTRIBUTE((nonnull)); /********************************************************************//** Calculates a folded value of a file page address to use in the page hash table. diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index 0f7526a8e5e..2f03d2aa0f5 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -136,6 +136,7 @@ extern fil_addr_t fil_addr_null; #define FIL_PAGE_DATA 38 /*!< start of the data on the page */ /* Following are used when page compression is used */ + #define FIL_PAGE_COMPRESSED_SIZE 2 /*!< Number of bytes used to store actual payload data size on compressed pages. */ @@ -313,13 +314,21 @@ struct fil_space_t { ulint n_pending_flushes; /*!< this is positive when flushing the tablespace to disk; dropping of the tablespace is forbidden if this is positive */ - ulint n_pending_ops;/*!< this is positive when we - have pending operations against this - tablespace. The pending operations can - be ibuf merges or lock validation code - trying to read a block. - Dropping of the tablespace is forbidden - if this is positive */ + /** Number of pending buffer pool operations accessing the tablespace + without holding a table lock or dict_operation_lock S-latch + that would prevent the table (and tablespace) from being + dropped. An example is change buffer merge. + The tablespace cannot be dropped while this is nonzero, + or while fil_node_t::n_pending is nonzero. + Protected by fil_system->mutex. */ + ulint n_pending_ops; + /** Number of pending block read or write operations + (when a write is imminent or a read has recently completed). + The tablespace object cannot be freed while this is nonzero, + but it can be detached from fil_system. + Note that fil_node_t::n_pending tracks actual pending I/O requests. + Protected by fil_system->mutex. */ + ulint n_pending_ios; hash_node_t hash; /*!< hash chain node */ hash_node_t name_hash;/*!< hash chain the name_hash table */ #ifndef UNIV_HOTBACKUP @@ -651,13 +660,11 @@ Used by background threads that do not necessarily hold proper locks for concurrency control. @param[in] id tablespace ID @param[in] silent whether to silently ignore missing tablespaces -@param[in] for_io whether to look up the tablespace while performing I/O - (possibly executing TRUNCATE) @return the tablespace @retval NULL if missing or being deleted or truncated */ UNIV_INTERN fil_space_t* -fil_space_acquire_low(ulint id, bool silent, bool for_io = false) +fil_space_acquire_low(ulint id, bool silent) MY_ATTRIBUTE((warn_unused_result)); /** Acquire a tablespace when it could be dropped concurrently. @@ -670,31 +677,45 @@ for concurrency control. @retval NULL if missing or being deleted or truncated */ inline fil_space_t* -fil_space_acquire(ulint id, bool for_io = false) +fil_space_acquire(ulint id) { - return (fil_space_acquire_low(id, false, for_io)); + return(fil_space_acquire_low(id, false)); } /** Acquire a tablespace that may not exist. Used by background threads that do not necessarily hold proper locks for concurrency control. @param[in] id tablespace ID -@param[in] for_io whether to look up the tablespace while performing I/O - (possibly executing TRUNCATE) @return the tablespace @retval NULL if missing or being deleted */ inline fil_space_t* -fil_space_acquire_silent(ulint id, bool for_io = false) +fil_space_acquire_silent(ulint id) { - return (fil_space_acquire_low(id, true, for_io)); + return(fil_space_acquire_low(id, true)); } /** Release a tablespace acquired with fil_space_acquire(). @param[in,out] space tablespace to release */ +UNIV_INTERN void fil_space_release(fil_space_t* space); +/** Acquire a tablespace for reading or writing a block, +when it could be dropped concurrently. +@param[in] id tablespace ID +@return the tablespace +@retval NULL if missing */ +UNIV_INTERN +fil_space_t* +fil_space_acquire_for_io(ulint id); + +/** Release a tablespace acquired with fil_space_acquire_for_io(). +@param[in,out] space tablespace to release */ +UNIV_INTERN +void +fil_space_release_for_io(fil_space_t* space); + /** Return the next fil_space_t. Once started, the caller must keep calling this until it returns NULL. fil_space_acquire() and fil_space_release() are invoked here which @@ -703,6 +724,7 @@ blocks a concurrent operation from dropping the tablespace. If NULL, use the first fil_space_t on fil_system->space_list. @return pointer to the next fil_space_t. @retval NULL if this was the last */ +UNIV_INTERN fil_space_t* fil_space_next( fil_space_t* prev_space) @@ -716,6 +738,7 @@ blocks a concurrent operation from dropping the tablespace. If NULL, use the first fil_space_t on fil_system->space_list. @return pointer to the next fil_space_t. @retval NULL if this was the last*/ +UNIV_INTERN fil_space_t* fil_space_keyrotate_next( fil_space_t* prev_space) @@ -732,12 +755,9 @@ public: /** Constructor: Look up the tablespace and increment the reference count if found. @param[in] space_id tablespace ID - @param[in] silent whether not print any errors - @param[in] for_io whether to look up the tablespace - while performing I/O - (possibly executing TRUNCATE) */ - explicit FilSpace(ulint space_id, bool silent = false, bool for_io = false) - : m_space(fil_space_acquire_low(space_id, silent, for_io)) {} + @param[in] silent whether not to print any errors */ + explicit FilSpace(ulint space_id, bool silent = false) + : m_space(fil_space_acquire_low(space_id, silent)) {} /** Assignment operator: This assumes that fil_space_acquire() has already been done for the fil_space_t. The caller must @@ -1042,7 +1062,7 @@ space id is != 0. @return DB_SUCCESS or error number */ UNIV_INTERN dberr_t -fil_load_single_table_tablespaces(void); +fil_load_single_table_tablespaces(ibool (*pred)(const char*, const char*)=0); /*===================================*/ /*******************************************************************//** Returns TRUE if a single-table tablespace does not exist in the memory cache, @@ -1081,6 +1101,9 @@ fil_space_for_table_exists_in_mem( information to the .err log if a matching tablespace is not found from memory */ + bool remove_from_data_dict_if_does_not_exist, + /*!< in: remove from the data dictionary + if tablespace does not exist */ bool adjust_space, /*!< in: whether to adjust space id when find table space mismatch */ mem_heap_t* heap, /*!< in: heap memory */ diff --git a/storage/xtradb/include/srv0srv.h b/storage/xtradb/include/srv0srv.h index c18bc7c1fc3..74c118b7660 100644 --- a/storage/xtradb/include/srv0srv.h +++ b/storage/xtradb/include/srv0srv.h @@ -496,7 +496,9 @@ as enum type because the configure option takes unsigned integer type. */ extern ulong srv_innodb_stats_method; #ifdef UNIV_LOG_ARCHIVE -extern ibool srv_log_archive_on; +extern bool srv_log_archive_on; +extern bool srv_archive_recovery; +extern ib_uint64_t srv_archive_recovery_limit_lsn; #endif /* UNIV_LOG_ARCHIVE */ extern char* srv_file_flush_method_str; @@ -547,6 +549,14 @@ extern ulong srv_pass_corrupt_table; extern ulong srv_log_checksum_algorithm; +extern bool srv_apply_log_only; + +extern bool srv_backup_mode; +extern bool srv_close_files; +extern bool srv_xtrabackup; + +#define IS_XTRABACKUP() (srv_xtrabackup) + extern my_bool srv_force_primary_key; /* Helper macro to support srv_pass_corrupt_table checks. If 'cond' is FALSE, diff --git a/storage/xtradb/include/trx0sys.h b/storage/xtradb/include/trx0sys.h index 0c18b657fd7..9bfffd09532 100644 --- a/storage/xtradb/include/trx0sys.h +++ b/storage/xtradb/include/trx0sys.h @@ -336,8 +336,9 @@ trx_sys_update_wsrep_checkpoint( trx_sysf_t* sys_header, /*!< in: sys_header */ mtr_t* mtr); /*!< in: mtr */ -void -/** Read WSREP checkpoint XID from sys header. */ +/** Read WSREP checkpoint XID from sys header. +@return true on success, false on error. */ +bool trx_sys_read_wsrep_checkpoint( XID* xid); /*!< out: WSREP XID */ #endif /* WITH_WSREP */ diff --git a/storage/xtradb/include/univ.i b/storage/xtradb/include/univ.i index e698f08f15b..310053b9145 100644 --- a/storage/xtradb/include/univ.i +++ b/storage/xtradb/include/univ.i @@ -635,7 +635,7 @@ functions. */ #ifdef __WIN__ #define usleep(a) Sleep((a)/1000) -typedef ulint os_thread_ret_t; +typedef DWORD os_thread_ret_t; #define OS_THREAD_DUMMY_RETURN return(0) #else typedef void* os_thread_ret_t; diff --git a/storage/xtradb/log/log0crypt.cc b/storage/xtradb/log/log0crypt.cc index e6b5c845757..f6c1416d81a 100644 --- a/storage/xtradb/log/log0crypt.cc +++ b/storage/xtradb/log/log0crypt.cc @@ -25,8 +25,7 @@ Modified Jan Lindström jan.lindstrom@mariadb.com *******************************************************/ #include "m_string.h" #include "log0crypt.h" -#include <my_crypt.h> -#include <my_crypt.h> +#include <mysql/service_my_crypt.h> #include "log0log.h" #include "srv0start.h" // for srv_start_lsn @@ -34,8 +33,6 @@ Modified Jan Lindström jan.lindstrom@mariadb.com #include "ha_prototypes.h" // IB_LOG_ -#include "my_crypt.h" - /* Used for debugging */ // #define DEBUG_CRYPT 1 #define UNENCRYPTED_KEY_VER 0 diff --git a/storage/xtradb/log/log0log.cc b/storage/xtradb/log/log0log.cc index 25c8ed06981..309de7daaf8 100644 --- a/storage/xtradb/log/log0log.cc +++ b/storage/xtradb/log/log0log.cc @@ -2629,7 +2629,7 @@ loop: start_lsn += len; buf += len; - if (recv_sys->report(ut_time())) { + if (recv_sys && recv_sys->report(ut_time())) { ib_logf(IB_LOG_LEVEL_INFO, "Read redo log up to LSN=" LSN_PF, start_lsn); sd_notifyf(0, "STATUS=Read redo log up to LSN=" LSN_PF, diff --git a/storage/xtradb/log/log0recv.cc b/storage/xtradb/log/log0recv.cc index d6d31b0c572..978e6051711 100644 --- a/storage/xtradb/log/log0recv.cc +++ b/storage/xtradb/log/log0recv.cc @@ -692,7 +692,6 @@ recv_synchronize_groups( /***********************************************************************//** Checks the consistency of the checkpoint info @return TRUE if ok */ -static ibool recv_check_cp_is_consistent( /*========================*/ @@ -722,7 +721,7 @@ recv_check_cp_is_consistent( /********************************************************//** Looks for the maximum consistent checkpoint from the log groups. @return error code or DB_SUCCESS */ -static MY_ATTRIBUTE((nonnull, warn_unused_result)) +MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t recv_find_max_checkpoint( /*=====================*/ @@ -3474,6 +3473,7 @@ recv_recovery_from_checkpoint_finish(void) #ifdef __WIN__ if (recv_writer_thread_handle) { CloseHandle(recv_writer_thread_handle); + recv_writer_thread_handle = 0; } #endif /* __WIN__ */ @@ -3700,6 +3700,102 @@ recv_reset_log_files_for_backup( } #endif /* UNIV_HOTBACKUP */ +/******************************************************//** +Checks the 4-byte checksum to the trailer checksum field of a log +block. We also accept a log block in the old format before +InnoDB-3.23.52 where the checksum field contains the log block number. +@return TRUE if ok, or if the log block may be in the format of InnoDB +version predating 3.23.52 */ +UNIV_INTERN +ibool +log_block_checksum_is_ok_or_old_format( +/*===================================*/ + const byte* block) /*!< in: pointer to a log block */ +{ +#ifdef UNIV_LOG_DEBUG + return(TRUE); +#endif /* UNIV_LOG_DEBUG */ + + ulint block_checksum = log_block_get_checksum(block); + + if (UNIV_LIKELY(srv_log_checksum_algorithm == + SRV_CHECKSUM_ALGORITHM_NONE || + log_block_calc_checksum(block) == block_checksum)) { + + return(TRUE); + } + + if (srv_log_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_CRC32 || + srv_log_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB || + srv_log_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_NONE) { + + const char* algo = NULL; + + ib_logf(IB_LOG_LEVEL_ERROR, + "log block checksum mismatch: expected " ULINTPF ", " + "calculated checksum " ULINTPF, + block_checksum, + log_block_calc_checksum(block)); + + if (block_checksum == LOG_NO_CHECKSUM_MAGIC) { + + algo = "none"; + } else if (block_checksum == + log_block_calc_checksum_crc32(block)) { + + algo = "crc32"; + } else if (block_checksum == + log_block_calc_checksum_innodb(block)) { + + algo = "innodb"; + } + + if (algo) { + + const char* current_algo; + + current_algo = buf_checksum_algorithm_name( + (srv_checksum_algorithm_t) + srv_log_checksum_algorithm); + + ib_logf(IB_LOG_LEVEL_ERROR, + "current InnoDB log checksum type: %s, " + "detected log checksum type: %s", + current_algo, + algo); + } + + ib_logf(IB_LOG_LEVEL_FATAL, + "STRICT method was specified for innodb_log_checksum, " + "so we intentionally assert here."); + } + + ut_ad(srv_log_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_CRC32 || + srv_log_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_INNODB); + + if (block_checksum == LOG_NO_CHECKSUM_MAGIC || + block_checksum == log_block_calc_checksum_crc32(block) || + block_checksum == log_block_calc_checksum_innodb(block)) { + + return(TRUE); + } + + if (log_block_get_hdr_no(block) == block_checksum) { + + /* We assume the log block is in the format of + InnoDB version < 3.23.52 and the block is ok */ +#if 0 + fprintf(stderr, + "InnoDB: Scanned old format < InnoDB-3.23.52" + " log block number %lu\n", + log_block_get_hdr_no(block)); +#endif + return(TRUE); + } + + return(FALSE); +} + void recv_dblwr_t::add(byte* page) { pages.push_back(page); diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index 03200fee80b..4f219b18428 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -130,7 +130,7 @@ UNIV_INTERN os_ib_mutex_t os_file_seek_mutexes[OS_FILE_N_SEEK_MUTEXES]; #define OS_AIO_MERGE_N_CONSECUTIVE 64 #ifdef WITH_INNODB_DISALLOW_WRITES -#define WAIT_ALLOW_WRITES() os_event_wait(srv_allow_writes_event) +#define WAIT_ALLOW_WRITES() if (!IS_XTRABACKUP()) os_event_wait(srv_allow_writes_event) #else #define WAIT_ALLOW_WRITES() do { } while (0) #endif /* WITH_INNODB_DISALLOW_WRITES */ @@ -1001,7 +1001,6 @@ os_file_lock( #ifndef UNIV_HOTBACKUP /****************************************************************//** Creates the seek mutexes used in positioned reads and writes. */ -static void os_io_init_simple(void) /*===================*/ @@ -1640,6 +1639,10 @@ os_file_create_simple_no_error_handling_func( return((os_file_t) -1); } + if (IS_XTRABACKUP()) { + share_mode |= FILE_SHARE_DELETE | FILE_SHARE_WRITE; + } + file = CreateFile((LPCTSTR) name, access, share_mode, @@ -1921,7 +1924,10 @@ os_file_create_func( create_mode &= ~OS_FILE_ON_ERROR_NO_EXIT; create_mode &= ~OS_FILE_ON_ERROR_SILENT; - + if (srv_backup_mode){ + /* Permit others to write, while I'm reading. */ + share_mode |= FILE_SHARE_WRITE; + } if (create_mode == OS_FILE_OPEN_RAW) { ut_a(!srv_read_only_mode); @@ -3525,7 +3531,7 @@ os_file_get_status( fh = CreateFile( (LPCTSTR) path, // File to open access, - 0, // No sharing + FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, // Default security OPEN_EXISTING, // Existing file only FILE_ATTRIBUTE_NORMAL, // Normal file diff --git a/storage/xtradb/row/row0mysql.cc b/storage/xtradb/row/row0mysql.cc index a4141c36eda..b4d359d5009 100644 --- a/storage/xtradb/row/row0mysql.cc +++ b/storage/xtradb/row/row0mysql.cc @@ -4483,8 +4483,9 @@ row_drop_table_for_mysql( if (!is_temp && !fil_space_for_table_exists_in_mem( space_id, tablename, - print_msg, false, NULL, 0, + print_msg, IS_XTRABACKUP() && print_msg, false, NULL, 0, table_flags)) { + /* This might happen if we are dropping a discarded tablespace */ err = DB_SUCCESS; diff --git a/storage/xtradb/srv/srv0srv.cc b/storage/xtradb/srv/srv0srv.cc index 1c40baee04b..bf4b9124da7 100644 --- a/storage/xtradb/srv/srv0srv.cc +++ b/storage/xtradb/srv/srv0srv.cc @@ -203,7 +203,7 @@ performance killer causing calling thread to context switch. Besides, Innodb is preallocating large number (often millions) of os_events. With kernel event objects it takes a big chunk out of non-paged pool, which is better suited for tasks like IO than for storing idle event objects. */ -UNIV_INTERN ibool srv_use_native_conditions = FALSE; +UNIV_INTERN ibool srv_use_native_conditions = TRUE; #endif /* __WIN__ */ UNIV_INTERN ulint srv_n_data_files = 0; @@ -366,7 +366,9 @@ readahead request. */ UNIV_INTERN ulong srv_read_ahead_threshold = 56; #ifdef UNIV_LOG_ARCHIVE -UNIV_INTERN ibool srv_log_archive_on = FALSE; +UNIV_INTERN bool srv_log_archive_on; +UNIV_INTERN bool srv_archive_recovery; +UNIV_INTERN ib_uint64_t srv_archive_recovery_limit_lsn; #endif /* UNIV_LOG_ARCHIVE */ /* This parameter is used to throttle the number of insert buffers that are @@ -522,6 +524,12 @@ UNIV_INTERN ulong srv_doublewrite_batch_size = 120; UNIV_INTERN ulong srv_replication_delay = 0; +UNIV_INTERN bool srv_apply_log_only; + +UNIV_INTERN bool srv_backup_mode; +UNIV_INTERN bool srv_close_files; +UNIV_INTERN bool srv_xtrabackup; + UNIV_INTERN ulong srv_pass_corrupt_table = 0; /* 0:disable 1:enable */ UNIV_INTERN ulong srv_log_checksum_algorithm = diff --git a/storage/xtradb/srv/srv0start.cc b/storage/xtradb/srv/srv0start.cc index c5c594c82a7..aa51012816d 100644 --- a/storage/xtradb/srv/srv0start.cc +++ b/storage/xtradb/srv/srv0start.cc @@ -140,7 +140,7 @@ SRV_SHUTDOWN_CLEANUP and then to SRV_SHUTDOWN_LAST_PHASE, and so on */ UNIV_INTERN enum srv_shutdown_state srv_shutdown_state = SRV_SHUTDOWN_NONE; /** Files comprising the system tablespace */ -static os_file_t files[1000]; +os_file_t files[1000]; /** io_handler_thread parameters for thread identification */ static ulint n[SRV_MAX_N_IO_THREADS]; @@ -826,7 +826,7 @@ open_log_file( /*********************************************************************//** Creates or opens database data files and closes them. @return DB_SUCCESS or error code */ -static MY_ATTRIBUTE((nonnull, warn_unused_result)) +MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t open_or_create_data_files( /*======================*/ @@ -1080,8 +1080,10 @@ skip_size_check: /* This is the earliest location where we can load the double write buffer. */ if (i == 0) { + /* XtraBackup never loads corrupted pages from + the doublewrite buffer */ buf_dblwr_init_or_load_pages( - files[i], srv_data_file_names[i], true); + files[i], srv_data_file_names[i], !IS_XTRABACKUP()); } bool retry = true; @@ -1365,12 +1367,15 @@ srv_undo_tablespace_open( /******************************************************************** Opens the configured number of undo tablespaces. @return DB_SUCCESS or error code */ -static dberr_t srv_undo_tablespaces_init( /*======================*/ ibool create_new_db, /*!< in: TRUE if new db being created */ + ibool backup_mode, /*!< in: TRUE disables reading + the system tablespace (used in + XtraBackup), FALSE is passed on + recovery. */ const ulint n_conf_tablespaces, /*!< in: configured undo tablespaces */ ulint* n_opened) /*!< out: number of UNDO @@ -1424,7 +1429,7 @@ srv_undo_tablespaces_init( we build the undo_tablespace_ids ourselves since they don't already exist. */ - if (!create_new_db) { + if (!create_new_db && !backup_mode) { n_undo_tablespaces = trx_rseg_get_n_undo_tablespaces( undo_tablespace_ids); } else { @@ -2287,11 +2292,11 @@ innobase_start_or_create_for_mysql(void) max_flushed_lsn = min_flushed_lsn = log_get_lsn(); goto files_checked; - } else if (i < 2) { - /* must have at least 2 log files */ - ib_logf(IB_LOG_LEVEL_ERROR, - "Only one log file found."); - return(err); + } else if (i < 2 && !IS_XTRABACKUP()) { + /* must have at least 2 log files */ + ib_logf(IB_LOG_LEVEL_ERROR, + "Only one log file found."); + return(err); } /* opened all files */ @@ -2385,6 +2390,7 @@ files_checked: err = srv_undo_tablespaces_init( create_new_db, + FALSE, srv_undo_tablespaces, &srv_undo_tablespaces_open); @@ -2658,6 +2664,17 @@ files_checked: dict_check_tablespaces_and_store_max_id(dict_check); } + if (IS_XTRABACKUP() + && !srv_backup_mode + && srv_read_only_mode + && srv_log_file_size_requested != srv_log_file_size) { + + ib_logf(IB_LOG_LEVEL_WARN, + "Log files size mismatch, ignored in readonly mode"); + srv_log_file_size_requested = srv_log_file_size; + } + + if (!srv_force_recovery && !recv_sys->found_corrupt_log && (srv_log_file_size_requested != srv_log_file_size @@ -3292,7 +3309,8 @@ innobase_shutdown_for_mysql(void) srv_was_started = FALSE; srv_start_has_been_called = FALSE; - + /* reset io_tid_i, in case current process does second innodb start (xtrabackup might do that).*/ + io_tid_i = 0; return(DB_SUCCESS); } #endif /* !UNIV_HOTBACKUP */ diff --git a/storage/xtradb/trx/trx0sys.cc b/storage/xtradb/trx/trx0sys.cc index 1c4fb19430e..558fe8a2c49 100644 --- a/storage/xtradb/trx/trx0sys.cc +++ b/storage/xtradb/trx/trx0sys.cc @@ -404,7 +404,7 @@ trx_sys_update_wsrep_checkpoint( } -void +bool trx_sys_read_wsrep_checkpoint(XID* xid) /*===================================*/ { @@ -427,7 +427,7 @@ trx_sys_read_wsrep_checkpoint(XID* xid) xid->formatID = -1; trx_sys_update_wsrep_checkpoint(xid, sys_header, &mtr); mtr_commit(&mtr); - return; + return false; } xid->formatID = (int)mach_read_from_4( @@ -444,6 +444,7 @@ trx_sys_read_wsrep_checkpoint(XID* xid) XIDDATASIZE); mtr_commit(&mtr); + return true; } #endif /* WITH_WSREP */ @@ -1339,14 +1340,17 @@ trx_sys_close(void) trx_purge_sys_close(); /* Free the double write data structures. */ - buf_dblwr_free(); + if (buf_dblwr) { + buf_dblwr_free(); + } - ut_a(UT_LIST_GET_LEN(trx_sys->ro_trx_list) == 0); /* Only prepared transactions may be left in the system. Free them. */ ut_a(UT_LIST_GET_LEN(trx_sys->rw_trx_list) == trx_sys->n_prepared_trx || srv_read_only_mode - || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO); + || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO + || (IS_XTRABACKUP() && srv_apply_log_only)); + while ((trx = UT_LIST_GET_FIRST(trx_sys->rw_trx_list)) != NULL) { trx_free_prepared(trx); @@ -1377,10 +1381,12 @@ trx_sys_close(void) UT_LIST_REMOVE(view_list, trx_sys->view_list, prev_view); } - ut_a(UT_LIST_GET_LEN(trx_sys->view_list) == 0); - ut_a(UT_LIST_GET_LEN(trx_sys->ro_trx_list) == 0); - ut_a(UT_LIST_GET_LEN(trx_sys->rw_trx_list) == 0); - ut_a(UT_LIST_GET_LEN(trx_sys->mysql_trx_list) == 0); + if (!IS_XTRABACKUP() || !srv_apply_log_only) { + ut_a(UT_LIST_GET_LEN(trx_sys->view_list) == 0); + ut_a(UT_LIST_GET_LEN(trx_sys->ro_trx_list) == 0); + ut_a(UT_LIST_GET_LEN(trx_sys->rw_trx_list) == 0); + ut_a(UT_LIST_GET_LEN(trx_sys->mysql_trx_list) == 0); + } mutex_free(&trx_sys->mutex); @@ -1427,6 +1433,9 @@ ulint trx_sys_any_active_transactions(void) /*=================================*/ { + if (IS_XTRABACKUP() && srv_apply_log_only) { + return(0); + } mutex_enter(&trx_sys->mutex); ulint total_trx = UT_LIST_GET_LEN(trx_sys->mysql_trx_list); diff --git a/storage/xtradb/trx/trx0trx.cc b/storage/xtradb/trx/trx0trx.cc index ed80bafa6c6..d0cb4a883cc 100644 --- a/storage/xtradb/trx/trx0trx.cc +++ b/storage/xtradb/trx/trx0trx.cc @@ -721,9 +721,16 @@ trx_resurrect_insert( if (srv_force_recovery == 0) { - trx->state = TRX_STATE_PREPARED; - trx_sys->n_prepared_trx++; - trx_sys->n_prepared_recovered_trx++; + /* XtraBackup should rollback prepared XA + transactions */ + if (IS_XTRABACKUP()) { + trx->state = TRX_STATE_ACTIVE; + } + else { + trx->state = TRX_STATE_PREPARED; + trx_sys->n_prepared_trx++; + trx_sys->n_prepared_recovered_trx++; + } } else { fprintf(stderr, "InnoDB: Since innodb_force_recovery" @@ -790,13 +797,16 @@ trx_resurrect_update_in_prepared_state( if (srv_force_recovery == 0) { if (trx_state_eq(trx, TRX_STATE_NOT_STARTED)) { - trx_sys->n_prepared_trx++; - trx_sys->n_prepared_recovered_trx++; + if (!IS_XTRABACKUP()) { + trx_sys->n_prepared_trx++; + trx_sys->n_prepared_recovered_trx++; + } } else { ut_ad(trx_state_eq(trx, TRX_STATE_PREPARED)); } - - trx->state = TRX_STATE_PREPARED; + /* XtraBackup should rollback prepared XA + transactions */ + trx->state = IS_XTRABACKUP()?TRX_STATE_ACTIVE: TRX_STATE_PREPARED; } else { fprintf(stderr, "InnoDB: Since innodb_force_recovery" diff --git a/win/packaging/CPackWixConfig.cmake b/win/packaging/CPackWixConfig.cmake index e954110ef19..581f4925d41 100644 --- a/win/packaging/CPackWixConfig.cmake +++ b/win/packaging/CPackWixConfig.cmake @@ -9,7 +9,7 @@ IF(ESSENTIALS) ENDIF() ELSE() SET(CPACK_COMPONENTS_USED - "Server;Client;Development;SharedLibraries;Documentation;Readme;Debuginfo;Common;VCCRT;connect-engine;ClientPlugins;gssapi-server;gssapi-client;aws-key-management;rocksdb-engine") + "Server;Client;Development;SharedLibraries;Documentation;Readme;Debuginfo;Common;VCCRT;connect-engine;ClientPlugins;gssapi-server;gssapi-client;aws-key-management;rocksdb-engine;backup") ENDIF() SET( WIX_FEATURE_MySQLServer_EXTRA_FEATURES "DBInstance;SharedClientServerComponents") @@ -56,6 +56,11 @@ SET(CPACK_COMPONENT_GROUP_MYSQLSERVER_DESCRIPTION "Install server") "Debug/trace versions of executables and libraries" ) #SET(CPACK_COMPONENT_DEBUGBINARIES_WIX_LEVEL 2) + # Subfeature "Backup" + SET(CPACK_COMPONENT_BACKUP_GROUP "MySQLServer") + SET(CPACK_COMPONENT_BACKUP_DISPLAY_NAME "Backup utilities") + SET(CPACK_COMPONENT_BACKUP_DESCRIPTION "Installs backup utilities(mariabackup and mbstream)") + #Miscellaneous (hidden) components, part of server / or client programs FOREACH(comp connect-engine ClientPlugins gssapi-server gssapi-client aws-key-management rocksdb-engine) |