diff options
author | Vladislav Vaintroub <wlad@mariadb.com> | 2017-04-18 19:05:57 +0000 |
---|---|---|
committer | Sergei Golubchik <serg@mariadb.org> | 2017-04-27 19:12:40 +0200 |
commit | ce4c56db0c5806ccbea8ad78585f95f6c335b0ea (patch) | |
tree | 06c75c921b6947320e86487fc7a41dbb55307d95 | |
parent | d7714308e014a7335e1c0f60dc0afbf5eba5ff99 (diff) | |
download | mariadb-git-ce4c56db0c5806ccbea8ad78585f95f6c335b0ea.tar.gz |
MDEV-9566 Port Percona Xtrabackup to MariaDB as mariabackup
- Modify backup code to work with MariaDB's 10.1 xtradb
- Remove crypt encryption, version_check.pl, "compact backup"
- Add encryption plugin
38 files changed, 949 insertions, 3163 deletions
diff --git a/extra/mariabackup/CMakeLists.txt b/extra/mariabackup/CMakeLists.txt index a84ac304486..f3369b49cee 100644 --- a/extra/mariabackup/CMakeLists.txt +++ b/extra/mariabackup/CMakeLists.txt @@ -13,65 +13,138 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA -INCLUDE(gcrypt) -INCLUDE(curl) -INCLUDE(libev) -ADD_SUBDIRECTORY(libarchive) -ADD_SUBDIRECTORY(jsmn) - -FIND_GCRYPT() -FIND_CURL() -FIND_EV() - -# xxd is needed to embed version_check script -FIND_PROGRAM(XXD_PATH xxd) - -IF(NOT XXD_PATH) - MESSAGE(FATAL_ERROR "xxd not found. Try to install vim-common.") -ENDIF(NOT XXD_PATH) +OPTION(WITH_MARIABACKUP "Include mariabackup" ON) +IF(NOT WITH_MARIABACKUP) + RETURN() +ENDIF() + + +IF(NOT WIN32) + CHECK_SYMBOL_EXISTS(regcomp regex.h HAVE_SYSTEM_REGEX) + IF(HAVE_SYSTEM_REGEX) + ADD_DEFINITIONS(-DHAVE_SYSTEM_REGEX) + ENDIF() +ENDIF() + +IF(WITH_LIBARCHIVE STREQUAL "STATIC") + SET(CMAKE_FIND_LIBRARY_SUFFIXES .a .lib) +ENDIF() + +FIND_PACKAGE(LibArchive) + +IF(NOT DEFINED WITH_LIBARCHIVE) + IF(LibArchive_FOUND) + SET(WITH_LIBARCHIVE_DEFAULT ON) + ELSE() + SET(WITH_LIBARCHIVE_DEFAULT OFF) + ENDIF() + SET(WITH_LIBARCHIVE ${WITH_LIBARCHIVE_DEFAULT} CACHE STRING "Use libarchive for streaming features (ON, OFF or STATIC)" ) +ENDIF() + +IF(NOT WITH_LIBARCHIVE MATCHES "^(ON|OFF|STATIC)$") + MESSAGE(FATAL_ERROR "Invalid value for WITH_LIBARCHIVE: '${WITH_LIBARCHIVE}'. Use one of ON, OFF or STATIC") +ENDIF() + +IF(UNIX) + SET(PIC_FLAG -fPIC) +ENDIF() + +IF((NOT WITH_LIBARCHIVE STREQUAL "OFF") AND (NOT LibArchive_FOUND)) + IF(CMAKE_VERSION VERSION_LESS "2.8.12") + MESSAGE("libarchive can't be built, old cmake") + ELSE() + # Build a local version + INCLUDE(ExternalProject) + SET(libarchive_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/libarchive) + SET(libarchive_CMAKE_ARGS + -DCMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR> + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DENABLE_ICONV=OFF + -DENABLE_TAR=ON + -DENABLE_OPENSSL=OFF + -DENABLE_TEST=OFF + "-DCMAKE_C_FLAGS_DEBUG=${CMAKE_C_FLAGS_DEBUG} ${PIC_FLAG}" + "-DCMAKE_C_FLAGS_RELWITHDEBINFO=${CMAKE_C_FLAGS_RELWITHDEBINFO} ${PIC_FLAG}" + "-DCMAKE_C_FLAGS_RELEASE=${CMAKE_C_FLAGS_RELEASE} ${PIC_FLAG}" + "-DCMAKE_C_FLAGS_MINSIZEREL=${CMAKE_C_FLAGS_MINSIZEREL} ${PIC_FLAG}" + ) + IF(WIN32) + SET(libarchive_CMAKE_ARGS ${libarchive_CMAKE_ARGS} -DWINDOWS_VERSION=WIN7 -DCMAKE_DEBUG_POSTFIX=d) + ENDIF() + + SET(LIBARCHIVE_DIR ${CMAKE_CURRENT_BINARY_DIR}/libarchive) + ExternalProject_Add(libarchive + PREFIX ${libarchive_PREFIX} + DOWNLOAD_DIR ${LIBARCHIVE_DIR} + URL http://www.libarchive.org/downloads/libarchive-3.2.2.tar.gz + INSTALL_DIR ${LIBARCHIVE_DIR} + CMAKE_ARGS ${libarchive_CMAKE_ARGS} + ) + ADD_LIBRARY(archive_static STATIC IMPORTED) + ADD_DEPENDENCIES(archive_static libarchive) + IF(WIN32) + SET(LIBARCHIVE_RELEASE_LIB ${LIBARCHIVE_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}archive_static${CMAKE_STATIC_LIBRARY_SUFFIX}) + SET(LIBARCHIVE_DEBUG_LIB ${LIBARCHIVE_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}archive_staticd${CMAKE_STATIC_LIBRARY_SUFFIX}) + SET_PROPERTY(TARGET archive_static PROPERTY IMPORTED_LOCATION_RELWITHDEBINFO ${LIBARCHIVE_RELEASE_LIB}) + SET_PROPERTY(TARGET archive_static PROPERTY IMPORTED_LOCATION_RELEASE ${LIBARCHIVE_RELEASE_LIB}) + SET_PROPERTY(TARGET archive_static PROPERTY IMPORTED_LOCATION_DEBUG ${LIBARCHIVE_DEBUG_LIB}) + SET_PROPERTY(TARGET archive_static PROPERTY IMPORTED_LOCATION_MINSIZEREL ${LIBARCHIVE_RELEASE_LIB}) + ELSE() + SET_PROPERTY(TARGET archive_static PROPERTY IMPORTED_LOCATION ${LIBARCHIVE_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}archive${CMAKE_STATIC_LIBRARY_SUFFIX}) + ENDIF() + + SET(LibArchive_FOUND ON ) + SET(LibArchive_INCLUDE_DIRS ${LIBARCHIVE_DIR}/include ) + SET(LibArchive_LIBRARIES archive_static) + IF(WIN32) + SET(LIBARCHIVE_STATIC 1) + ENDIF() + ENDIF() +ENDIF() + + +IF(WITH_LIBARCHIVE AND LibArchive_FOUND) + ADD_DEFINITIONS(-DHAVE_LIBARCHIVE) + IF(LIBARCHIVE_STATIC) + ADD_DEFINITIONS(-DLIBARCHIVE_STATIC) + ENDIF() + INCLUDE_DIRECTORIES(${LibArchive_INCLUDE_DIRS}) + LINK_LIBRARIES(${LibArchive_LIBRARIES}) + SET(DS_ARCHIVE_SOURCE ds_archive.c) +ENDIF() INCLUDE_DIRECTORIES( ${CMAKE_SOURCE_DIR}/include - ${CMAKE_SOURCE_DIR}/storage/innobase/include + ${CMAKE_SOURCE_DIR}/storage/xtradb/include ${CMAKE_SOURCE_DIR}/sql - ${CMAKE_SOURCE_DIR}/storage/innobase/xtrabackup/src/libarchive/libarchive - ${CMAKE_SOURCE_DIR}/storage/innobase/xtrabackup/src/quicklz - ${CMAKE_SOURCE_DIR}/storage/innobase/xtrabackup/src/jsmn - ${GCRYPT_INCLUDE_DIR} - ${CURL_INCLUDE_DIRS} - ${LIBEV_INCLUDE_DIRS} - ${CMAKE_CURRENT_BINARY_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/quicklz + ${CMAKE_CURRENT_SOURCE_DIR} ) -ADD_DEFINITIONS(${SSL_DEFINES}) +IF(NOT HAVE_SYSTEM_REGEX) + INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/pcre) +ENDIF() +ADD_DEFINITIONS(-UMYSQL_SERVER) ######################################################################## # xtrabackup binary ######################################################################## -CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/xtrabackup_version.h.in - ${CMAKE_CURRENT_BINARY_DIR}/xtrabackup_version.h ) -ADD_CUSTOM_COMMAND(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/version_check_pl.h - COMMAND ${XXD_PATH} --include version_check.pl - ${CMAKE_CURRENT_BINARY_DIR}/version_check_pl.h - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) +IF(WIN32) + SET(NT_SERVICE_SOURCE ${PROJECT_SOURCE_DIR}/sql/nt_servc.cc) +ELSE() + SET(NT_SERVICE_SOURCE) +ENDIF() -ADD_CUSTOM_TARGET(GenVersionCheck - DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/version_check_pl.h) +ADD_DEFINITIONS(-DPCRE_STATIC=1) -SET_SOURCE_FILES_PROPERTIES( - xtrabackup.cc - backup_mysql.cc - PROPERTIES COMPILE_FLAGS -DMYSQL_CLIENT) - -MYSQL_ADD_EXECUTABLE(xtrabackup +MYSQL_ADD_EXECUTABLE(mariabackup xtrabackup.cc innobackupex.cc changed_page_bitmap.cc - compact.cc datasink.c - ds_archive.c + ${DS_ARCHIVE_SOURCE} ds_buffer.c ds_compress.c ds_encrypt.c @@ -89,31 +162,29 @@ MYSQL_ADD_EXECUTABLE(xtrabackup xbstream_write.c backup_mysql.cc backup_copy.cc - ../../../../sql-common/client_authentication.cc + encryption_plugin.cc + ${PROJECT_SOURCE_DIR}/libmysql/libmysql.c + ${PROJECT_SOURCE_DIR}/sql/net_serv.cc + ${NT_SERVICE_SOURCE} + COMPONENT backup ) -SET_TARGET_PROPERTIES(xtrabackup PROPERTIES ENABLE_EXPORTS TRUE) -TARGET_LINK_LIBRARIES(xtrabackup - mysqlserver - ${GCRYPT_LIBS} - archive_static - ) +# Export all symbols on Unix, for better crash callstacks +SET_TARGET_PROPERTIES(mariabackup PROPERTIES ENABLE_EXPORTS TRUE) -ADD_DEPENDENCIES(xtrabackup GenVersionCheck) -######################################################################## -# innobackupex symlink -######################################################################## -ADD_CUSTOM_COMMAND(TARGET xtrabackup - COMMAND ${CMAKE_COMMAND} ARGS -E create_symlink - xtrabackup innobackupex) -INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/innobackupex DESTINATION bin) +TARGET_LINK_LIBRARIES(mariabackup sql) + +IF(NOT HAVE_SYSTEM_REGEX) + TARGET_LINK_LIBRARIES(mariabackup pcreposix) +ENDIF() + ######################################################################## # xbstream binary ######################################################################## -MYSQL_ADD_EXECUTABLE(xbstream +MYSQL_ADD_EXECUTABLE(mbstream ds_buffer.c ds_local.c ds_stdout.c @@ -121,53 +192,15 @@ MYSQL_ADD_EXECUTABLE(xbstream xbstream.c xbstream_read.c xbstream_write.c - ) - -SET_TARGET_PROPERTIES(xbstream - PROPERTIES LINKER_LANGUAGE CXX - ) - -TARGET_LINK_LIBRARIES(xbstream - mysys - mysys_ssl - ) -######################################################################## -# xbcrypt binary -######################################################################## -MYSQL_ADD_EXECUTABLE(xbcrypt - xbcrypt.c - xbcrypt_common.c - xbcrypt_read.c - xbcrypt_write.c + COMPONENT backup ) -SET_TARGET_PROPERTIES(xbcrypt - PROPERTIES LINKER_LANGUAGE CXX - ) -TARGET_LINK_LIBRARIES(xbcrypt - ${GCRYPT_LIBS} +TARGET_LINK_LIBRARIES(mbstream mysys - mysys_ssl - ) +) -######################################################################## -# xbcloud binary -######################################################################## -MYSQL_ADD_EXECUTABLE(xbcloud - xbcloud.cc - ) - -SET_TARGET_PROPERTIES(xbcloud - PROPERTIES LINKER_LANGUAGE CXX - ) - -TARGET_LINK_LIBRARIES(xbcloud - ${GCRYPT_LIBS} - ${LIBEV_LIBRARIES} - ${CURL_LIBRARIES} - mysys - mysys_ssl - jsmn - ) +IF(MSVC) + SET_TARGET_PROPERTIES(mbstream PROPERTIES LINK_FLAGS setargv.obj) +ENDIF() diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc index a051c64ee3b..478a6a66b97 100644 --- a/extra/mariabackup/backup_copy.cc +++ b/extra/mariabackup/backup_copy.cc @@ -1,6 +1,7 @@ /****************************************************** hot backup tool for InnoDB (c) 2009-2015 Percona LLC and/or its affiliates +(c) 2017 MariaDB Originally Created 3/3/2009 Yasufumi Kinoshita Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. @@ -48,13 +49,14 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include <set> #include <string> #include <mysqld.h> -#include <version_check_pl.h> #include <sstream> #include "fil_cur.h" #include "xtrabackup.h" #include "common.h" #include "backup_copy.h" #include "backup_mysql.h" +#include <btr0btr.h> +#include "xb0xb.h" /* list of files to sync for --rsync mode */ @@ -110,6 +112,7 @@ struct datadir_thread_ctxt_t { bool ret; }; +static bool backup_files_from_datadir(const char *dir_path); /************************************************************************ Retirn true if character if file separator */ @@ -225,7 +228,7 @@ datadir_iter_next_database(datadir_iter_t *it) it->dbdir = NULL; } - while (fil_file_readdir_next_file(&it->err, it->datadir_path, + while (os_file_readdir_next_file(it->datadir_path, it->dir, &it->dbinfo) == 0) { ulint len; @@ -340,7 +343,7 @@ datadir_iter_next_file(datadir_iter_t *it) return(false); } - while (fil_file_readdir_next_file(&it->err, it->dbpath, it->dbdir, + while (os_file_readdir_next_file(it->dbpath, it->dbdir, &it->fileinfo) == 0) { if (it->fileinfo.type == OS_FILE_TYPE_DIR) { @@ -449,9 +452,9 @@ struct datafile_cur_t { uint thread_n; byte* orig_buf; byte* buf; - ib_int64_t buf_size; - ib_int64_t buf_read; - ib_int64_t buf_offset; + size_t buf_size; + size_t buf_read; + size_t buf_offset; }; static @@ -486,7 +489,7 @@ datafile_open(const char *file, datafile_cur_t *cursor, uint thread_n) cursor->abs_path, OS_FILE_OPEN, OS_FILE_READ_ONLY, - &success); + &success, 0); if (!success) { /* The following call prints an error message */ os_file_get_last_error(TRUE); @@ -498,7 +501,7 @@ datafile_open(const char *file, datafile_cur_t *cursor, uint thread_n) return(false); } - if (my_fstat(cursor->file, &cursor->statinfo, MYF(MY_WME))) { + if (!my_stat(cursor->abs_path, &cursor->statinfo, 0)) { msg("[%02u] error: cannot stat %s\n", thread_n, cursor->abs_path); @@ -510,7 +513,7 @@ datafile_open(const char *file, datafile_cur_t *cursor, uint thread_n) posix_fadvise(cursor->file, 0, 0, POSIX_FADV_SEQUENTIAL); cursor->buf_size = 10 * 1024 * 1024; - cursor->buf = static_cast<byte *>(ut_malloc(cursor->buf_size)); + cursor->buf = static_cast<byte *>(ut_malloc((ulint)cursor->buf_size)); return(true); } @@ -525,7 +528,7 @@ datafile_read(datafile_cur_t *cursor) xtrabackup_io_throttling(); - to_read = min(cursor->statinfo.st_size - cursor->buf_offset, + to_read = (ulint)MY_MIN(cursor->statinfo.st_size - cursor->buf_offset, cursor->buf_size); if (to_read == 0) { @@ -603,6 +606,11 @@ ends_with(const char *str, const char *suffix) && strcmp(str + str_len - suffix_len, suffix) == 0); } +static bool starts_with(const char *str, const char *prefix) +{ + return strncmp(str, prefix, strlen(prefix)) == 0; +} + /************************************************************************ Create directories recursively. @return 0 if directories created successfully. */ @@ -644,6 +652,7 @@ Return true if first and second arguments are the same path. */ bool equal_paths(const char *first, const char *second) { +#ifdef HAVE_REALPATH char real_first[PATH_MAX]; char real_second[PATH_MAX]; @@ -655,6 +664,9 @@ equal_paths(const char *first, const char *second) } return (strcmp(real_first, real_second) == 0); +#else + return strcmp(first, second) == 0; +#endif } /************************************************************************ @@ -675,10 +687,8 @@ directory_exists(const char *dir, bool create) } if (mkdirp(dir, 0777, MYF(0)) < 0) { - - msg("Can not create directory %s: %s\n", dir, - my_strerror(errbuf, sizeof(errbuf), my_errno)); - + my_strerror(errbuf, sizeof(errbuf), my_errno); + msg("Can not create directory %s: %s\n", dir, errbuf); return(false); } @@ -688,9 +698,9 @@ directory_exists(const char *dir, bool create) os_dir = os_file_opendir(dir, FALSE); if (os_dir == NULL) { - + my_strerror(errbuf, sizeof(errbuf), my_errno); msg("Can not open directory %s: %s\n", dir, - my_strerror(errbuf, sizeof(errbuf), my_errno)); + errbuf); return(false); } @@ -1054,16 +1064,17 @@ move_file(ds_ctxt_t *datasink, dst_file_path, thread_n); msg_ts("[%02u] Removing %s\n", thread_n, src_file_path); if (unlink(src_file_path) != 0) { + my_strerror(errbuf, sizeof(errbuf), errno); msg("Error: unlink %s failed: %s\n", src_file_path, - my_strerror(errbuf, - sizeof(errbuf), errno)); + errbuf); } return(ret); } + my_strerror(errbuf, sizeof(errbuf), my_errno); msg("Can not move file %s to %s: %s\n", src_file_path, dst_file_path_abs, - my_strerror(errbuf, sizeof(errbuf), my_errno)); + errbuf); return(false); } @@ -1360,6 +1371,10 @@ backup_start() return(false); } + if (!backup_files_from_datadir(fil_path_to_mysql_datadir)) { + return false; + } + // There is no need to stop slave thread before coping non-Innodb data when // --no-lock option is used because --no-lock option requires that no DDL or // DML to non-transaction tables can occur. @@ -1578,7 +1593,11 @@ ibx_cleanup_full_backup() while (datadir_iter_next(it, &node)) { if (node.is_empty_dir) { +#ifdef _WIN32 + DeleteFile(node.filepath); +#else rmdir(node.filepath); +#endif } if (xtrabackup_incremental && !node.is_empty_dir @@ -1605,6 +1624,9 @@ apply_log_finish() return(true); } +extern void +os_io_init_simple(void); + bool copy_back() { @@ -1661,7 +1683,7 @@ copy_back() } srv_max_n_threads = 1000; - os_sync_mutex = NULL; + //os_sync_mutex = NULL; ut_mem_init(); /* temporally dummy value to avoid crash */ srv_page_size_shift = 14; @@ -1764,10 +1786,9 @@ copy_back() if (mkdirp(path, 0777, MYF(0)) < 0) { char errbuf[MYSYS_STRERROR_SIZE]; - + my_strerror(errbuf, sizeof(errbuf), my_errno); msg("Can not create directory %s: %s\n", - path, my_strerror(errbuf, - sizeof(errbuf), my_errno)); + path, errbuf); ret = false; goto cleanup; @@ -1855,13 +1876,12 @@ cleanup: ds_data = NULL; - sync_close(); - sync_initialized = FALSE; - os_sync_free(); + //os_sync_free(); mem_close(); - os_sync_mutex = NULL; + //os_sync_mutex = NULL; ut_free_all_mem(); - + sync_close(); + sync_initialized = FALSE; return(ret); } @@ -1872,7 +1892,7 @@ decrypt_decompress_file(const char *filepath, uint thread_n) char *dest_filepath = strdup(filepath); bool needs_action = false; - cmd << "cat " << filepath; + cmd << IF_WIN("type ","cat ") << filepath; if (ends_with(filepath, ".xbcrypt") && opt_decrypt) { cmd << " | xbcrypt --decrypt --encrypt-algo=" @@ -1926,7 +1946,7 @@ decrypt_decompress_file(const char *filepath, uint thread_n) } static -os_thread_ret_t +os_thread_ret_t STDCALL decrypt_decompress_thread_func(void *arg) { bool ret = true; @@ -1974,7 +1994,7 @@ decrypt_decompress() datadir_iter_t *it = NULL; srv_max_n_threads = 1000; - os_sync_mutex = NULL; + //os_sync_mutex = NULL; ut_mem_init(); os_sync_init(); sync_init(); @@ -2008,40 +2028,43 @@ decrypt_decompress() sync_close(); sync_initialized = FALSE; - os_sync_free(); - os_sync_mutex = NULL; + //os_sync_free(); + //os_sync_mutex = NULL; ut_free_all_mem(); return(ret); } -void -version_check() +/* + Copy some files from top level datadir. + Do not copy the Innodb files (ibdata1, redo log files), + as this is done in a separate step. +*/ +static bool backup_files_from_datadir(const char *dir_path) { - if (opt_password != NULL) { - setenv("option_mysql_password", opt_password, 1); - } - if (opt_user != NULL) { - setenv("option_mysql_user", opt_user, 1); - } - if (opt_host != NULL) { - setenv("option_mysql_host", opt_host, 1); - } - if (opt_socket != NULL) { - setenv("option_mysql_socket", opt_socket, 1); - } - if (opt_port != 0) { - char port[20]; - snprintf(port, sizeof(port), "%u", opt_port); - setenv("option_mysql_port", port, 1); - } - - FILE *pipe = popen("perl", "w"); - if (pipe == NULL) { - return; - } + os_file_dir_t dir = os_file_opendir(dir_path, TRUE); + os_file_stat_t info; + bool ret = true; + while (os_file_readdir_next_file(dir_path, dir, &info) == 0) { - fputs((const char *)version_check_pl, pipe); + if (info.type != OS_FILE_TYPE_FILE) + continue; - pclose(pipe); + const char *pname = strrchr(info.name, IF_WIN('\\', '/')); + if (!pname) + pname = info.name; + + /* Copy aria log files, and aws keys for encryption plugins.*/ + const char *prefixes[] = { "aria_log", "aws-kms-key" }; + for (size_t i = 0; i < array_elements(prefixes); i++) { + if (starts_with(pname, prefixes[i])) { + ret = copy_file(ds_data, info.name, info.name, 1); + if (!ret) { + break; + } + } + } + } + os_file_closedir(dir); + return ret; } diff --git a/extra/mariabackup/backup_copy.h b/extra/mariabackup/backup_copy.h index c8fc5fc8ba9..4b829982764 100644 --- a/extra/mariabackup/backup_copy.h +++ b/extra/mariabackup/backup_copy.h @@ -41,8 +41,6 @@ bool copy_back(); bool decrypt_decompress(); -void -version_check(); bool is_path_separator(char); bool diff --git a/extra/mariabackup/backup_mysql.cc b/extra/mariabackup/backup_mysql.cc index 1ae8d10053e..ed5446b4806 100644 --- a/extra/mariabackup/backup_mysql.cc +++ b/extra/mariabackup/backup_mysql.cc @@ -38,6 +38,7 @@ this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *******************************************************/ +#define MYSQL_CLIENT #include <my_global.h> #include <mysql.h> @@ -47,10 +48,12 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include <limits> #include "common.h" #include "xtrabackup.h" -#include "xtrabackup_version.h" +#include "mysql_version.h" #include "backup_copy.h" #include "backup_mysql.h" #include "mysqld.h" +#include "encryption_plugin.h" +#include <sstream> char *tool_name; @@ -88,15 +91,7 @@ time_t history_lock_time; MYSQL *mysql_connection; -extern "C" { -MYSQL * STDCALL -cli_mysql_real_connect(MYSQL *mysql,const char *host, const char *user, - const char *passwd, const char *db, - uint port, const char *unix_socket,ulong client_flag); -} - -#define mysql_real_connect cli_mysql_real_connect - +my_bool opt_ssl_verify_server_cert; MYSQL * xb_mysql_connect() @@ -136,11 +131,6 @@ xb_mysql_connect() } mysql_options(connection,MYSQL_OPT_SSL_VERIFY_SERVER_CERT, (char*)&opt_ssl_verify_server_cert); -#if !defined(HAVE_YASSL) - if (opt_server_public_key && *opt_server_public_key) - mysql_options(connection, MYSQL_SERVER_PUBLIC_KEY, - opt_server_public_key); -#endif #endif if (!mysql_real_connect(connection, @@ -310,7 +300,7 @@ check_server_version(unsigned long version_number, version_supported = version_supported || (version_number > 50500 && version_number < 50700); version_supported = version_supported - || ((version_number > 100000 && version_number < 100300) + || ((version_number > 100000) && server_flavor == FLAVOR_MARIADB); if (mysql51 && innodb_version == NULL) { @@ -596,7 +586,7 @@ select_incremental_lsn_from_history(lsn_t *incremental_lsn) if (opt_incremental_history_name) { mysql_real_escape_string(mysql_connection, buf, opt_incremental_history_name, - strlen(opt_incremental_history_name)); + (unsigned long)strlen(opt_incremental_history_name)); ut_snprintf(query, sizeof(query), "SELECT innodb_to_lsn " "FROM PERCONA_SCHEMA.xtrabackup_history " @@ -609,7 +599,7 @@ select_incremental_lsn_from_history(lsn_t *incremental_lsn) if (opt_incremental_history_uuid) { mysql_real_escape_string(mysql_connection, buf, opt_incremental_history_uuid, - strlen(opt_incremental_history_uuid)); + (unsigned long)strlen(opt_incremental_history_uuid)); ut_snprintf(query, sizeof(query), "SELECT innodb_to_lsn " "FROM PERCONA_SCHEMA.xtrabackup_history " @@ -756,7 +746,7 @@ have_queries_to_wait_for(MYSQL *connection, uint threshold) static void -kill_long_queries(MYSQL *connection, uint timeout) +kill_long_queries(MYSQL *connection, time_t timeout) { MYSQL_RES *result; MYSQL_ROW row; @@ -768,15 +758,15 @@ kill_long_queries(MYSQL *connection, uint timeout) all_queries = (opt_kill_long_query_type == QUERY_TYPE_ALL); while ((row = mysql_fetch_row(result)) != NULL) { const char *info = row[7]; - int duration = atoi(row[5]); + long long duration = atoll(row[5]); char *id = row[0]; if (info != NULL && - duration >= (int)timeout && + (time_t)duration >= timeout && ((all_queries && is_query(info)) || is_select_query(info))) { msg_ts("Killing query %s (duration %d sec): %s\n", - id, duration, info); + id, (int)duration, info); ut_snprintf(kill_stmt, sizeof(kill_stmt), "KILL %s", id); xb_mysql_query(connection, kill_stmt, false, false); @@ -1378,7 +1368,18 @@ cleanup: return(result); } - +static string escape_and_quote(MYSQL *mysql,const char *str) +{ + if (!str) + return "NULL"; + size_t len = strlen(str); + char* escaped = (char *)alloca(2 * len + 3); + escaped[0] = '\''; + size_t new_len = mysql_real_escape_string(mysql, escaped+1, str, len); + escaped[new_len + 1] = '\''; + escaped[new_len + 2] = 0; + return string(escaped); +} /*********************************************************************//** Writes xtrabackup_info file and if backup_history is enable creates @@ -1388,25 +1389,16 @@ backup. */ bool write_xtrabackup_info(MYSQL *connection) { - MYSQL_STMT *stmt; - MYSQL_BIND bind[19]; + char *uuid = NULL; char *server_version = NULL; char buf_start_time[100]; char buf_end_time[100]; - int idx; tm tm; my_bool null = TRUE; - + ostringstream oss; const char *xb_stream_name[] = {"file", "tar", "xbstream"}; - const char *ins_query = "insert into PERCONA_SCHEMA.xtrabackup_history(" - "uuid, name, tool_name, tool_command, tool_version, " - "ibbackup_version, server_version, start_time, end_time, " - "lock_time, binlog_pos, innodb_from_lsn, innodb_to_lsn, " - "partial, incremental, format, compact, compressed, " - "encrypted) " - "values(?,?,?,?,?,?,?,from_unixtime(?),from_unixtime(?)," - "?,?,?,?,?,?,?,?,?,?)"; + ut_ad(xtrabackup_stream_fmt < 3); @@ -1419,6 +1411,11 @@ write_xtrabackup_info(MYSQL *connection) localtime_r(&history_end_time, &tm); strftime(buf_end_time, sizeof(buf_end_time), "%Y-%m-%d %H:%M:%S", &tm); + bool is_partial = (xtrabackup_tables + || xtrabackup_tables_file + || xtrabackup_databases + || xtrabackup_databases_file); + backup_file_printf(XTRABACKUP_INFO, "uuid = %s\n" "name = %s\n" @@ -1443,23 +1440,20 @@ write_xtrabackup_info(MYSQL *connection) opt_history ? opt_history : "", /* name */ tool_name, /* tool_name */ tool_args, /* tool_command */ - XTRABACKUP_VERSION, /* tool_version */ - XTRABACKUP_VERSION, /* ibbackup_version */ + MYSQL_SERVER_VERSION, /* tool_version */ + MYSQL_SERVER_VERSION, /* ibbackup_version */ server_version, /* server_version */ buf_start_time, /* start_time */ buf_end_time, /* end_time */ - history_lock_time, /* lock_time */ + (int)history_lock_time, /* lock_time */ mysql_binlog_position ? mysql_binlog_position : "", /* binlog_pos */ incremental_lsn, /* innodb_from_lsn */ metadata_to_lsn, /* innodb_to_lsn */ - (xtrabackup_tables /* partial */ - || xtrabackup_tables_file - || xtrabackup_databases - || xtrabackup_databases_file) ? "Y" : "N", + is_partial? "Y" : "N", xtrabackup_incremental ? "Y" : "N", /* incremental */ xb_stream_name[xtrabackup_stream_fmt], /* format */ - xtrabackup_compact ? "Y" : "N", /* compact */ + "N", /* compact */ xtrabackup_compress ? "compressed" : "N", /* compressed */ xtrabackup_encrypt ? "Y" : "N"); /* encrypted */ @@ -1492,142 +1486,36 @@ write_xtrabackup_info(MYSQL *connection) "encrypted ENUM('Y', 'N') DEFAULT NULL" ") CHARACTER SET utf8 ENGINE=innodb", false); - stmt = mysql_stmt_init(connection); - - mysql_stmt_prepare(stmt, ins_query, strlen(ins_query)); - - memset(bind, 0, sizeof(bind)); - idx = 0; - - /* uuid */ - bind[idx].buffer_type = MYSQL_TYPE_STRING; - bind[idx].buffer = uuid; - bind[idx].buffer_length = strlen(uuid); - ++idx; - - /* name */ - bind[idx].buffer_type = MYSQL_TYPE_STRING; - bind[idx].buffer = (char*)(opt_history); - bind[idx].buffer_length = strlen(opt_history); - if (!(opt_history && *opt_history)) { - bind[idx].is_null = &null; - } - ++idx; - - /* tool_name */ - bind[idx].buffer_type = MYSQL_TYPE_STRING; - bind[idx].buffer = tool_name; - bind[idx].buffer_length = strlen(tool_name); - ++idx; - - /* tool_command */ - bind[idx].buffer_type = MYSQL_TYPE_STRING; - bind[idx].buffer = tool_args; - bind[idx].buffer_length = strlen(tool_args); - ++idx; - - /* tool_version */ - bind[idx].buffer_type = MYSQL_TYPE_STRING; - bind[idx].buffer = (char*)(XTRABACKUP_VERSION); - bind[idx].buffer_length = strlen(XTRABACKUP_VERSION); - ++idx; - - /* ibbackup_version */ - bind[idx].buffer_type = MYSQL_TYPE_STRING; - bind[idx].buffer = (char*)(XTRABACKUP_VERSION); - bind[idx].buffer_length = strlen(XTRABACKUP_VERSION); - ++idx; - - /* server_version */ - bind[idx].buffer_type = MYSQL_TYPE_STRING; - bind[idx].buffer = server_version; - bind[idx].buffer_length = strlen(server_version); - ++idx; - - /* start_time */ - bind[idx].buffer_type = MYSQL_TYPE_LONG; - bind[idx].buffer = &history_start_time; - ++idx; - - /* end_time */ - bind[idx].buffer_type = MYSQL_TYPE_LONG; - bind[idx].buffer = &history_end_time; - ++idx; - - /* lock_time */ - bind[idx].buffer_type = MYSQL_TYPE_LONG; - bind[idx].buffer = &history_lock_time; - ++idx; - - /* binlog_pos */ - bind[idx].buffer_type = MYSQL_TYPE_STRING; - bind[idx].buffer = mysql_binlog_position; - if (mysql_binlog_position != NULL) { - bind[idx].buffer_length = strlen(mysql_binlog_position); - } else { - bind[idx].is_null = &null; - } - ++idx; - - /* innodb_from_lsn */ - bind[idx].buffer_type = MYSQL_TYPE_LONGLONG; - bind[idx].buffer = (char*)(&incremental_lsn); - ++idx; - - /* innodb_to_lsn */ - bind[idx].buffer_type = MYSQL_TYPE_LONGLONG; - bind[idx].buffer = (char*)(&metadata_to_lsn); - ++idx; - - /* partial (Y | N) */ - bind[idx].buffer_type = MYSQL_TYPE_STRING; - bind[idx].buffer = (char*)((xtrabackup_tables - || xtrabackup_tables_file - || xtrabackup_databases - || xtrabackup_databases_file) ? "Y" : "N"); - bind[idx].buffer_length = 1; - ++idx; - - /* incremental (Y | N) */ - bind[idx].buffer_type = MYSQL_TYPE_STRING; - bind[idx].buffer = (char*)( - (xtrabackup_incremental - || xtrabackup_incremental_basedir - || opt_incremental_history_name - || opt_incremental_history_uuid) ? "Y" : "N"); - bind[idx].buffer_length = 1; - ++idx; - - /* format (file | tar | xbstream) */ - bind[idx].buffer_type = MYSQL_TYPE_STRING; - bind[idx].buffer = (char*)(xb_stream_name[xtrabackup_stream_fmt]); - bind[idx].buffer_length = strlen(xb_stream_name[xtrabackup_stream_fmt]); - ++idx; - - /* compact (Y | N) */ - bind[idx].buffer_type = MYSQL_TYPE_STRING; - bind[idx].buffer = (char*)(xtrabackup_compact ? "Y" : "N"); - bind[idx].buffer_length = 1; - ++idx; - - /* compressed (Y | N) */ - bind[idx].buffer_type = MYSQL_TYPE_STRING; - bind[idx].buffer = (char*)(xtrabackup_compress ? "Y" : "N"); - bind[idx].buffer_length = 1; - ++idx; - - /* encrypted (Y | N) */ - bind[idx].buffer_type = MYSQL_TYPE_STRING; - bind[idx].buffer = (char*)(xtrabackup_encrypt ? "Y" : "N"); - bind[idx].buffer_length = 1; - ++idx; - - ut_ad(idx == 19); - - mysql_stmt_bind_param(stmt, bind); - - mysql_stmt_execute(stmt); - mysql_stmt_close(stmt); + +#define ESCAPE_BOOL(expr) ((expr)?"'Y'":"'N'") + + oss << "insert into PERCONA_SCHEMA.xtrabackup_history(" + << "uuid, name, tool_name, tool_command, tool_version," + << "ibbackup_version, server_version, start_time, end_time," + << "lock_time, binlog_pos, innodb_from_lsn, innodb_to_lsn," + << "partial, incremental, format, compact, compressed, " + << "encrypted) values(" + << escape_and_quote(connection, uuid) << "," + << escape_and_quote(connection, opt_history) << "," + << escape_and_quote(connection, tool_name) << "," + << escape_and_quote(connection, tool_args) << "," + << escape_and_quote(connection, MYSQL_SERVER_VERSION) << "," + << escape_and_quote(connection, MYSQL_SERVER_VERSION) << "," + << escape_and_quote(connection, server_version) << "," + << "from_unixtime(" << history_start_time << ")," + << "from_unixtime(" << history_end_time << ")," + << history_lock_time << "," + << escape_and_quote(connection, mysql_binlog_position) << "," + << incremental_lsn << "," + << metadata_to_lsn << "," + << ESCAPE_BOOL(is_partial) << "," + << ESCAPE_BOOL(xtrabackup_incremental)<< "," + << escape_and_quote(connection,xb_stream_name[xtrabackup_stream_fmt]) <<"," + << ESCAPE_BOOL(false) << "," + << ESCAPE_BOOL(xtrabackup_compress) << "," + << ESCAPE_BOOL(xtrabackup_encrypt) <<")"; + + xb_mysql_query(mysql_connection, oss.str().c_str(), false); cleanup: @@ -1637,10 +1525,11 @@ cleanup: return(true); } -bool -write_backup_config_file() +extern const char *innodb_checksum_algorithm_names[]; + +bool write_backup_config_file() { - return backup_file_printf("backup-my.cnf", + int rc= backup_file_printf("backup-my.cnf", "# This MySQL options file was generated by innobackupex.\n\n" "# The MySQL server\n" "[mysqld]\n" @@ -1649,19 +1538,18 @@ write_backup_config_file() "innodb_data_file_path=%s\n" "innodb_log_files_in_group=%lu\n" "innodb_log_file_size=%lld\n" - "innodb_fast_checksum=%s\n" "innodb_page_size=%lu\n" "innodb_log_block_size=%lu\n" "innodb_undo_directory=%s\n" "innodb_undo_tablespaces=%lu\n" "%s%s\n" - "%s%s\n", + "%s%s\n" + "%s\n", innodb_checksum_algorithm_names[srv_checksum_algorithm], innodb_checksum_algorithm_names[srv_log_checksum_algorithm], innobase_data_file_path, srv_n_log_files, innobase_log_file_size, - srv_fast_checksum ? "true" : "false", srv_page_size, srv_log_block_size, srv_undo_dir, @@ -1671,7 +1559,9 @@ write_backup_config_file() innobase_buffer_pool_filename ? "innodb_buffer_pool_filename=" : "", innobase_buffer_pool_filename ? - innobase_buffer_pool_filename : ""); + innobase_buffer_pool_filename : "", + encryption_plugin_get_config()); + return rc; } diff --git a/extra/mariabackup/changed_page_bitmap.cc b/extra/mariabackup/changed_page_bitmap.cc index e385474c7aa..435b7fb6172 100644 --- a/extra/mariabackup/changed_page_bitmap.cc +++ b/extra/mariabackup/changed_page_bitmap.cc @@ -447,7 +447,7 @@ log_online_open_bitmap_file_read_only( = os_file_create_simple_no_error_handling(0, bitmap_file->name, OS_FILE_OPEN, OS_FILE_READ_ONLY, - &success); + &success,0); if (UNIV_UNLIKELY(!success)) { /* Here and below assume that bitmap file names do not diff --git a/extra/mariabackup/common.h b/extra/mariabackup/common.h index abbc4b41a85..7b1dfd7a0db 100644 --- a/extra/mariabackup/common.h +++ b/extra/mariabackup/common.h @@ -26,6 +26,46 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA #include <fcntl.h> #include <stdarg.h> + +# define fil_is_user_tablespace_id(i) ((i) > srv_undo_tablespaces_open) + +#ifdef _MSC_VER +#define stat _stati64 +#define PATH_MAX MAX_PATH +#endif + +#ifndef HAVE_VASPRINTF +static inline int vasprintf(char **strp, const char *fmt, va_list args) +{ + int len; +#ifdef _MSC_VER + len = _vscprintf(fmt, args); +#else + len = vsnprintf(NULL, 0, fmt, args); +#endif + if (len < 0) + { + return -1; + } + *strp = (char *)malloc(len + 1); + if (!*strp) + { + return -1; + } + vsprintf(*strp, fmt, args); + return len; +} + +static inline int asprintf(char **strp, const char *fmt,...) +{ + va_list args; + va_start(args, fmt); + int len = vasprintf(strp, fmt, args); + va_end(args); + return len; +} +#endif + #define xb_a(expr) \ do { \ if (!(expr)) { \ @@ -93,15 +133,15 @@ static inline int msg_ts(const char *fmt, ...) /*********************************************************************** Computes bit shift for a given value. If the argument is not a power of 2, returns 0.*/ -static inline ulong -get_bit_shift(ulong value) +static inline size_t +get_bit_shift(size_t value) { - ulong shift; + size_t shift; if (value == 0) return 0; - for (shift = 0; !(value & 1UL); shift++) { + for (shift = 0; !(value & 1); shift++) { value >>= 1; } return (value >> 1) ? 0 : shift; diff --git a/extra/mariabackup/compact.cc b/extra/mariabackup/compact.cc deleted file mode 100644 index 5d08a6e02b2..00000000000 --- a/extra/mariabackup/compact.cc +++ /dev/null @@ -1,1059 +0,0 @@ -/****************************************************** -XtraBackup: hot backup tool for InnoDB -(c) 2009-2014 Percona LLC and/or its affiliates. -Originally Created 3/3/2009 Yasufumi Kinoshita -Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, -Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -*******************************************************/ - -/* Compact backups implementation */ - -#include <my_base.h> -#include <table.h> - -#include <univ.i> -#include <dict0mem.h> -#include <dict0priv.h> -#include <fsp0fsp.h> -#include <handler0alter.h> -#include <ibuf0ibuf.h> -#include <page0page.h> -#include <row0merge.h> -#include "common.h" -#include "write_filt.h" -#include "fil_cur.h" -#include "xtrabackup.h" -#include "ds_buffer.h" -#include "xb0xb.h" - -/* Number of the first primary key page in an .ibd file */ -#define XB_FIRST_CLUSTERED_INDEX_PAGE_NO 3 - -/* Suffix for page map files */ -#define XB_PAGE_MAP_SUFFIX ".pmap" -#define XB_TMPFILE_SUFFIX ".tmp" - -/* Page range */ -struct page_range_t { - ulint from; /*!< range start */ - ulint to; /*!< range end */ -}; - -/* Cursor in a page map file */ -struct page_map_cursor_t { - File fd; /*!< file descriptor */ - IO_CACHE cache; /*!< IO_CACHE associated with fd */ -}; - -/* Table descriptor for the index rebuild operation */ -struct index_rebuild_table_t { - char* name; /* table name */ - ulint space_id; /* space ID */ - UT_LIST_NODE_T(index_rebuild_table_t) list; /* list node */ -}; - -/* Thread descriptor for the index rebuild operation */ -struct index_rebuild_thread_t { - ulint num; /* thread number */ - pthread_t id; /* thread ID */ -}; - -/* Empty page use to replace skipped pages in the data files */ -static byte empty_page[UNIV_PAGE_SIZE_MAX]; -static const char compacted_page_magic[] = "COMPACTP"; -static const size_t compacted_page_magic_size = - sizeof(compacted_page_magic) - 1; -static const ulint compacted_page_magic_offset = FIL_PAGE_DATA; - -/* Mutex protecting table_list */ -static pthread_mutex_t table_list_mutex; -/* List of tablespaces to process by the index rebuild operation */ -static UT_LIST_BASE_NODE_T(index_rebuild_table_t) table_list; - - -/************************************************************************ -Compact page filter. */ -static my_bool wf_compact_init(xb_write_filt_ctxt_t *ctxt, char *dst_name, - xb_fil_cur_t *cursor); -static my_bool wf_compact_process(xb_write_filt_ctxt_t *ctxt, - ds_file_t *dstfile); -static my_bool wf_compact_finalize(xb_write_filt_ctxt_t *ctxt, - ds_file_t *dstfile); -xb_write_filt_t wf_compact = { - &wf_compact_init, - &wf_compact_process, - &wf_compact_finalize, - NULL -}; - -/************************************************************************ -Initialize the compact page filter. - -@return TRUE on success, FALSE on error. */ -static my_bool -wf_compact_init(xb_write_filt_ctxt_t *ctxt, - char *dst_name __attribute__((unused)), xb_fil_cur_t *cursor) -{ - xb_wf_compact_ctxt_t *cp = &(ctxt->u.wf_compact_ctxt); - char page_map_name[FN_REFLEN]; - MY_STAT mystat; - - ctxt->cursor = cursor; - cp->clustered_index_found = FALSE; - cp->inside_skipped_range = FALSE; - cp->free_limit = 0; - - /* Don't compact the system table space */ - cp->skip = cursor->is_system; - if (cp->skip) { - return(TRUE); - } - - snprintf(page_map_name, sizeof(page_map_name), "%s%s", dst_name, - XB_PAGE_MAP_SUFFIX); - - cp->ds_buffer = ds_create(xtrabackup_target_dir, DS_TYPE_BUFFER); - if (cp->ds_buffer == NULL) { - return(FALSE); - } - - ds_set_pipe(cp->ds_buffer, ds_meta); - - memset(&mystat, 0, sizeof(mystat)); - mystat.st_mtime = my_time(0); - cp->buffer = ds_open(cp->ds_buffer, page_map_name, &mystat); - if (cp->buffer == NULL) { - msg("xtrabackup: Error: cannot open output stream for %s\n", - page_map_name); - return(FALSE); - } - - return(TRUE); -} - -/************************************************************************ -Check if the specified page should be skipped. We currently skip all -non-clustered index pages for compact backups. - -@return TRUE if the page should be skipped. */ -static my_bool -check_if_skip_page(xb_wf_compact_ctxt_t *cp, xb_fil_cur_t *cursor, ulint offset) -{ - byte *page; - ulint page_no; - ulint page_type; - index_id_t index_id; - - - xb_ad(cursor->is_system == FALSE); - - page = cursor->buf + cursor->page_size * offset; - page_no = cursor->buf_page_no + offset; - page_type = fil_page_get_type(page); - - if (UNIV_UNLIKELY(page_no == 0)) { - - cp->free_limit = mach_read_from_4(page + FSP_HEADER_OFFSET + - FSP_FREE_LIMIT); - } else if (UNIV_UNLIKELY(page_no == XB_FIRST_CLUSTERED_INDEX_PAGE_NO)) { - - xb_ad(cp->clustered_index_found == FALSE); - - if (page_type != FIL_PAGE_INDEX) { - - /* Uninitialized clustered index root page, there's - nothing we can do to compact the space.*/ - - msg("[%02u] Uninitialized page type value (%lu) in the " - "clustered index root page of tablespace %s. " - "Will not be compacted.\n", - cursor->thread_n, - page_type, cursor->rel_path); - - cp->skip = TRUE; - - return(FALSE); - } - - cp->clustered_index = - mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID); - cp->clustered_index_found = TRUE; - } else if (UNIV_UNLIKELY(page_no >= cp->free_limit)) { - - /* Skip unused pages above free limit, if that value is set in - the FSP header.*/ - - return(cp->free_limit > 0); - } else if (cp->clustered_index_found && page_type == FIL_PAGE_INDEX) { - - index_id = mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID); - if (index_id != cp->clustered_index) { - - ulint fseg_hdr_space = - mach_read_from_4(page + PAGE_HEADER + - PAGE_BTR_SEG_TOP); - ulint fseg_hdr_page_no = - mach_read_from_4(page + PAGE_HEADER + - PAGE_BTR_SEG_TOP + 4); - ulint fseg_hdr_offset = - mach_read_from_2(page + PAGE_HEADER + - PAGE_BTR_SEG_TOP + 8); - - /* Don't skip root index pages, i.e. the ones where the - above fields are defined. We need root index pages to be - able to correctly drop the indexes later, as they - contain fseg inode pointers. */ - - return(fseg_hdr_space == 0 && - fseg_hdr_page_no == 0 && - fseg_hdr_offset == 0); - } - } - - return(FALSE); -} - -/************************************************************************ -Run the next batch of pages through the compact page filter. - -@return TRUE on success, FALSE on error. */ -static my_bool -wf_compact_process(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile) -{ - xb_fil_cur_t *cursor = ctxt->cursor; - ulint page_size = cursor->page_size; - byte *page; - byte *buf_end; - byte *write_from; - xb_wf_compact_ctxt_t *cp = &(ctxt->u.wf_compact_ctxt); - ulint i; - ulint page_no; - byte tmp[4]; - - if (cp->skip) { - return(!ds_write(dstfile, cursor->buf, cursor->buf_read)); - } - - write_from = NULL; - buf_end = cursor->buf + cursor->buf_read; - for (i = 0, page = cursor->buf; page < buf_end; - i++, page += page_size) { - - page_no = cursor->buf_page_no + i; - - if (!check_if_skip_page(cp, cursor, i)) { - - if (write_from == NULL) { - write_from = page; - } - - if (cp->inside_skipped_range) { - cp->inside_skipped_range = FALSE; - - /* Write the last range endpoint to the - skipped pages map */ - - xb_ad(page_no > 0); - mach_write_to_4(tmp, page_no - 1); - if (ds_write(cp->buffer, tmp, sizeof(tmp))) { - return(FALSE); - } - } - continue; - } - - if (write_from != NULL) { - - /* The first skipped page in this block, write the - non-skipped ones to the data file */ - - if (ds_write(dstfile, write_from, page - write_from)) { - return(FALSE); - } - - write_from = NULL; - } - - if (!cp->inside_skipped_range) { - - /* The first skipped page in range, write the first - range endpoint to the skipped pages map */ - - cp->inside_skipped_range = TRUE; - - mach_write_to_4(tmp, page_no); - if (ds_write(cp->buffer, tmp, sizeof(tmp))) { - return(FALSE); - } - } - } - - /* Write the remaining pages in the buffer, if any */ - if (write_from != NULL && - ds_write(dstfile, write_from, buf_end - write_from)) { - return(FALSE); - } - - return(TRUE); -} - -/************************************************************************ -Close the compact filter's page map stream. - -@return TRUE on success, FALSE on error. */ -static my_bool -wf_compact_finalize(xb_write_filt_ctxt_t *ctxt, - ds_file_t *dstfile __attribute__((unused))) -{ - xb_fil_cur_t *cursor = ctxt->cursor; - xb_wf_compact_ctxt_t *cp = &(ctxt->u.wf_compact_ctxt); - my_bool rc = TRUE; - - /* Write the last endpoint of the current range, if the last pages of - the space have been skipped. */ - if (cp->inside_skipped_range) { - byte tmp[4]; - - mach_write_to_4(tmp, cursor->space_size - 1); - if (ds_write(cp->buffer, tmp, sizeof(tmp))) { - return(FALSE); - } - - cp->inside_skipped_range = FALSE; - } - - if (cp->buffer) { - if (ds_close(cp->buffer)) { - rc = FALSE; - } - } - if (cp->ds_buffer) { - ds_destroy(cp->ds_buffer); - } - - return(rc); -} - -/************************************************************************ -Open a page map file and return a cursor. - -@return page map cursor, or NULL if the file doesn't exist. */ -static page_map_cursor_t * -page_map_file_open(const char *path) -{ - MY_STAT statinfo; - page_map_cursor_t *pmap_cur; - int rc; - - if (my_stat(path, &statinfo, MYF(0)) == NULL) { - - return(NULL); - } - - /* The maximum possible page map file corresponds to a 64 TB tablespace - and the worst case when every other page was skipped. That is, 2^32/2 - page ranges = 16 GB. */ - xb_a(statinfo.st_size < (off_t) 16 * 1024 * 1024 * 1024); - - /* Must be a series of 8-byte tuples */ - xb_a(statinfo.st_size % 8 == 0); - - pmap_cur = (page_map_cursor_t *) my_malloc(sizeof(page_map_cursor_t), - MYF(MY_FAE)); - - pmap_cur->fd = my_open(path, O_RDONLY, MYF(MY_WME)); - xb_a(pmap_cur->fd != 0); - - rc = init_io_cache(&pmap_cur->cache, pmap_cur->fd, 0, READ_CACHE, - 0, 0, MYF(MY_WME)); - xb_a(rc == 0); - - return(pmap_cur); -} - -/************************************************************************ -Read the next range from a page map file and update the cursor. - -@return TRUE on success, FALSE on end-of-file. */ -static ibool -page_map_file_next(page_map_cursor_t *pmap_cur, page_range_t *range) -{ - byte buf[8]; - - xb_ad(pmap_cur != NULL); - - if (my_b_read(&pmap_cur->cache, buf, sizeof(buf))) { - return(FALSE); - } - - range->from = mach_read_from_4(buf); - range->to = mach_read_from_4(buf + 4); - - return(TRUE); -} - -/************************************************************************ -Close the page map cursor.*/ -static void -page_map_file_close(page_map_cursor_t *pmap_cur) -{ - int rc; - - xb_ad(pmap_cur != NULL); - - rc = end_io_cache(&pmap_cur->cache); - xb_a(rc == 0); - - posix_fadvise(pmap_cur->fd, 0, 0, POSIX_FADV_DONTNEED); - - rc = my_close(pmap_cur->fd, MY_WME); - xb_a(rc == 0); - - my_free(pmap_cur); -} - -/**************************************************************************** -Expand a single data file according to the skipped pages maps created by ---compact. - -@return TRUE on success, FALSE on failure. */ -static my_bool -xb_expand_file(fil_node_t *node) -{ - char pmapfile_path[FN_REFLEN]; - char tmpfile_path[FN_REFLEN]; - xb_fil_cur_t cursor; - xb_fil_cur_result_t res; - ds_ctxt_t *ds_local; - ds_ctxt_t *ds_buffer; - ds_file_t *tmpfile; - my_bool success = FALSE; - ulint i; - byte *page; - ulint page_expected_no; - page_map_cursor_t *pmap_cur; - ibool have_next_range; - page_range_t pmap_range; - - xb_ad(trx_sys_sys_space(node->space->id) == FALSE); - - snprintf(pmapfile_path, sizeof(pmapfile_path), "%s%s", - node->name, XB_PAGE_MAP_SUFFIX); - - /* Skip files that don't have a corresponding page map file */ - - if (!(pmap_cur = page_map_file_open(pmapfile_path))) { - - msg("Not expanding %s\n", node->name); - - return(FALSE); - } - - msg("Expanding %s\n", node->name); - - ds_local = ds_create(".", DS_TYPE_LOCAL); - ds_buffer = ds_create(".", DS_TYPE_BUFFER); - - xb_a(ds_local != NULL && ds_buffer != NULL); - - ds_buffer_set_size(ds_buffer, FSP_EXTENT_SIZE * UNIV_PAGE_SIZE_MAX); - - ds_set_pipe(ds_buffer, ds_local); - - res = xb_fil_cur_open(&cursor, &rf_pass_through, node, 1); - xb_a(res == XB_FIL_CUR_SUCCESS); - - snprintf(tmpfile_path, sizeof(tmpfile_path), "%s%s", - node->name, XB_TMPFILE_SUFFIX); - - tmpfile = ds_open(ds_buffer, tmpfile_path, &cursor.statinfo); - if (tmpfile == NULL) { - - msg("Could not open temporary file '%s'\n", tmpfile_path); - goto error; - } - - have_next_range = page_map_file_next(pmap_cur, &pmap_range); - - page_expected_no = 0; - - /* Initialize and mark the empty page which is used to replace - skipped pages. */ - memset(empty_page, 0, cursor.page_size); - memcpy(empty_page + compacted_page_magic_offset, - compacted_page_magic, compacted_page_magic_size); - mach_write_to_4(empty_page + FIL_PAGE_SPACE_OR_CHKSUM, - BUF_NO_CHECKSUM_MAGIC); - mach_write_to_4(empty_page + cursor.page_size - - FIL_PAGE_END_LSN_OLD_CHKSUM, - BUF_NO_CHECKSUM_MAGIC); - - - /* Main copy loop */ - - while ((res = xb_fil_cur_read(&cursor)) == XB_FIL_CUR_SUCCESS) { - - for (i = 0, page = cursor.buf; i < cursor.buf_npages; - i++, page += cursor.page_size) { - - ulint page_read_no; - - page_read_no = mach_read_from_4(page + FIL_PAGE_OFFSET); - xb_a(!page_read_no || page_expected_no <= page_read_no); - - if (have_next_range && - page_expected_no == pmap_range.from) { - - xb_a(pmap_range.from <= pmap_range.to); - - /* Write empty pages instead of skipped ones, if - necessary. */ - - while (page_expected_no <= pmap_range.to) { - - if (ds_write(tmpfile, empty_page, - cursor.page_size)) { - - goto write_error; - } - - page_expected_no++; - } - - have_next_range = - page_map_file_next(pmap_cur, - &pmap_range); - } - - /* Write the current page */ - - if (ds_write(tmpfile, page, cursor.page_size)) { - - goto write_error; - } - - page_expected_no++; - } - } - - if (res != XB_FIL_CUR_EOF) { - - goto error; - } - - /* Write empty pages instead of trailing skipped ones, if any */ - - if (have_next_range) { - - xb_a(page_expected_no == pmap_range.from); - xb_a(pmap_range.from <= pmap_range.to); - - while (page_expected_no <= pmap_range.to) { - - if (ds_write(tmpfile, empty_page, - cursor.page_size)) { - - goto write_error; - } - - page_expected_no++; - } - - xb_a(!page_map_file_next(pmap_cur, &pmap_range)); - } - - /* Replace the original .ibd file with the expanded file */ - if (my_rename(tmpfile_path, node->name, MYF(MY_WME))) { - - msg("Failed to rename '%s' to '%s'\n", - tmpfile_path, node->name); - goto error; - } - - my_delete(pmapfile_path, MYF(MY_WME)); - - if (!ds_close(tmpfile)) { - success = TRUE; - } - tmpfile = NULL; - - goto end; - -write_error: - msg("Write to '%s' failed\n", tmpfile_path); - -error: - if (tmpfile != NULL) { - - ds_close(tmpfile); - my_delete(tmpfile_path, MYF(MY_WME)); - } - -end: - ds_destroy(ds_buffer); - ds_destroy(ds_local); - - xb_fil_cur_close(&cursor); - - page_map_file_close(pmap_cur); - - return(success); -} - -/****************************************************************************** -Expand the data files according to the skipped pages maps created by --compact. -@return TRUE on success, FALSE on failure. */ -my_bool -xb_expand_datafiles(void) -/*=====================*/ -{ - ulint nfiles; - datafiles_iter_t *it = NULL; - fil_node_t *node; - fil_space_t *space; - - msg("Starting to expand compacted .ibd files.\n"); - - /* Initialize the tablespace cache */ - if (xb_data_files_init() != DB_SUCCESS) { - return(FALSE); - } - - nfiles = UT_LIST_GET_LEN(fil_system->space_list); - xb_a(nfiles > 0); - - it = datafiles_iter_new(fil_system); - if (it == NULL) { - msg("xtrabackup: error: datafiles_iter_new() failed.\n"); - goto error; - } - - while ((node = datafiles_iter_next(it)) != NULL) { - - space = node->space; - - /* System tablespace cannot be compacted */ - if (!fil_is_user_tablespace_id(space->id)) { - - continue; - } - - if (!xb_expand_file(node)) { - - goto error; - } - } - - datafiles_iter_free(it); - xb_data_files_close(); - - return(TRUE); - -error: - if (it != NULL) { - datafiles_iter_free(it); - } - - xb_data_files_close(); - - return(FALSE); -} - -/****************************************************************************** -Callback used in buf_page_io_complete() to detect compacted pages. -@return TRUE if the page is marked as compacted, FALSE otherwise. */ -ibool -buf_page_is_compacted( -/*==================*/ - const byte* page) /*!< in: a database page */ -{ - return !memcmp(page + compacted_page_magic_offset, - compacted_page_magic, compacted_page_magic_size); -} - -/***************************************************************************** -Builds an index definition corresponding to an index object. It is roughly -similar to innobase_create_index_def() / innobase_create_index_field_def() and -the opposite to dict_mem_index_create() / dict_mem_index_add_field(). */ -static -void -xb_build_index_def( -/*=======================*/ - mem_heap_t* heap, /*!< in: heap */ - const dict_index_t* index, /*!< in: index */ - index_def_t* index_def) /*!< out: index definition */ -{ - index_field_t* fields; - ulint n_fields; - ulint i; - - ut_a(index->n_fields); - ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); - - /* Use n_user_defined_cols instead of n_fields, as the index will - contain a part of the primary key after n_user_defined_cols, and those - columns will be created automatically in - dict_index_build_internal_clust(). */ - n_fields = index->n_user_defined_cols; - - memset(index_def, 0, sizeof(*index_def)); - - index_def->name = mem_heap_strdup(heap, index->name); - index_def->ind_type = index->type; - - fields = static_cast<index_field_t *> - (mem_heap_alloc(heap, n_fields * sizeof(*fields))); - - for (i = 0; i < n_fields; i++) { - dict_field_t* field; - - field = dict_index_get_nth_field(index, i); - fields[i].col_no = dict_col_get_no(field->col); - fields[i].prefix_len = field->prefix_len; - } - - index_def->fields = fields; - index_def->n_fields = n_fields; -} - -/* A dummy autoc_inc sequence for row_merge_build_indexes(). */ -static ib_sequence_t null_seq(NULL, 0, 0); -/* A dummy table share and table for row_merge_build_indexes() error reporting. -Assumes that no errors are going to be reported. */ -static struct TABLE_SHARE dummy_table_share; -static struct TABLE dummy_table; - -/********************************************************************//** -Rebuild secondary indexes for a given table. */ -static -void -xb_rebuild_indexes_for_table( -/*=========================*/ - dict_table_t* table, /*!< in: table */ - trx_t* trx, /*!< in: transaction handle */ - ulint thread_n) /*!< in: thread number */ -{ - dict_index_t* index; - dict_index_t** indexes; - ulint n_indexes; - index_def_t* index_defs; - ulint i; - mem_heap_t* heap; - ulint error; - ulint* add_key_nums; - - ut_ad(!mutex_own(&(dict_sys->mutex))); - ut_ad(table); - - ut_a(UT_LIST_GET_LEN(table->indexes) > 0); - - n_indexes = UT_LIST_GET_LEN(table->indexes) - 1; - if (!n_indexes) { - /* Only the primary key, nothing to do. */ - return; - } - - heap = mem_heap_create(1024); - - indexes = (dict_index_t**) mem_heap_alloc(heap, - n_indexes * sizeof(*indexes)); - index_defs = (index_def_t*) mem_heap_alloc(heap, n_indexes * - sizeof(*index_defs)); - add_key_nums = static_cast<ulint *> - (mem_heap_alloc(heap, n_indexes * sizeof(*add_key_nums))); - - /* Skip the primary key. */ - index = dict_table_get_first_index(table); - ut_a(dict_index_is_clust(index)); - - row_mysql_lock_data_dictionary(trx); - - for (i = 0; (index = dict_table_get_next_index(index)); i++) { - - msg("[%02lu] Found index %s\n", thread_n, index->name); - - /* Pretend that it's the current trx that created this index. - Required to avoid 5.6+ debug assertions. */ - index->trx_id = trx->id; - - xb_build_index_def(heap, index, &index_defs[i]); - - /* In 5.6+, row_merge_drop_indexes() drops all the indexes on - the table that have the temp index prefix. It does not accept - an array of indexes to drop as in 5.5-. */ - row_merge_rename_index_to_drop(trx, table->id, index->id); - } - - ut_ad(i == n_indexes); - - row_merge_drop_indexes(trx, table, TRUE); - - index = dict_table_get_first_index(table); - ut_a(dict_index_is_clust(index)); - index = dict_table_get_next_index(index); - while (index) { - - /* In 5.6+, row_merge_drop_indexes() does not remove the - indexes from the dictionary cache nor from any foreign key - list. This may cause invalid dereferences as we try to access - the dropped indexes from other tables as FKs. */ - - dict_index_t* next_index = dict_table_get_next_index(index); - index->to_be_dropped = 1; - - /* Patch up any FK referencing this index with NULL */ - dict_foreign_replace_index(table, NULL, index); - - dict_index_remove_from_cache(table, index); - - index = next_index; - } - - msg("[%02lu] Rebuilding %lu index(es).\n", thread_n, n_indexes); - - error = row_merge_lock_table(trx, table, LOCK_X); - xb_a(error == DB_SUCCESS); - - for (i = 0; i < n_indexes; i++) { - indexes[i] = row_merge_create_index(trx, table, - &index_defs[i]); - add_key_nums[i] = index_defs[i].key_number; - } - - /* Commit trx to release latches on system tables */ - trx_commit_for_mysql(trx); - trx_start_for_ddl(trx, TRX_DICT_OP_INDEX); - - row_mysql_unlock_data_dictionary(trx); - - /* Reacquire table lock for row_merge_build_indexes() */ - error = row_merge_lock_table(trx, table, LOCK_X); - xb_a(error == DB_SUCCESS); - - error = row_merge_build_indexes(trx, table, table, FALSE, indexes, - add_key_nums, n_indexes, &dummy_table, - NULL, NULL, ULINT_UNDEFINED, null_seq); - ut_a(error == DB_SUCCESS); - - mem_heap_free(heap); - - trx_commit_for_mysql(trx); - - trx_start_for_ddl(trx, TRX_DICT_OP_INDEX); -} - -/************************************************************************** -Worker thread function for index rebuild. */ -static -void * -xb_rebuild_indexes_thread_func( -/*===========================*/ - void* arg) /* thread context */ -{ - dict_table_t* table; - index_rebuild_table_t* rebuild_table; - index_rebuild_thread_t* thread; - trx_t* trx; - - thread = (index_rebuild_thread_t *) arg; - - trx = trx_allocate_for_mysql(); - - /* Suppress foreign key checks, as we are going to drop and recreate all - secondary keys. */ - trx->check_foreigns = FALSE; - trx_start_for_ddl(trx, TRX_DICT_OP_INDEX); - - /* Loop until there are no more tables in tables list */ - for (;;) { - pthread_mutex_lock(&table_list_mutex); - - rebuild_table = UT_LIST_GET_FIRST(table_list); - - if (rebuild_table == NULL) { - - pthread_mutex_unlock(&table_list_mutex); - break; - } - - UT_LIST_REMOVE(list, table_list, rebuild_table); - - pthread_mutex_unlock(&table_list_mutex); - - ut_ad(rebuild_table->name); - ut_ad(fil_is_user_tablespace_id(rebuild_table->space_id)); - - row_mysql_lock_data_dictionary(trx); - - table = dict_table_get_low(rebuild_table->name); - - ut_d(table->n_ref_count++); - - row_mysql_unlock_data_dictionary(trx); - - ut_a(table != NULL); - ut_a(table->space == rebuild_table->space_id); - - /* Discard change buffer entries for this space */ - ibuf_delete_for_discarded_space(rebuild_table->space_id); - - msg("[%02lu] Checking if there are indexes to rebuild in table " - "%s (space id: %lu)\n", - thread->num, - rebuild_table->name, rebuild_table->space_id); - - xb_rebuild_indexes_for_table(table, trx, thread->num); - - ut_d(table->n_ref_count--); - - mem_free(rebuild_table->name); - mem_free(rebuild_table); - } - - trx_commit_for_mysql(trx); - - trx_free_for_mysql(trx); - - return(NULL); -} - -/****************************************************************************** -Rebuild all secondary indexes in all tables in separate spaces. Called from -innobase_start_or_create_for_mysql(). */ -void -xb_compact_rebuild_indexes(void) -/*=============================*/ -{ - dict_table_t* sys_tables; - dict_index_t* sys_index; - btr_pcur_t pcur; - const rec_t* rec; - mtr_t mtr; - const byte* field; - ulint len; - ulint space_id; - trx_t* trx; - index_rebuild_table_t* rebuild_table; - index_rebuild_thread_t* threads; - ulint i; - - /* Set up the dummy table for the index rebuild error reporting */ - dummy_table_share.fields = 0; - dummy_table.s = &dummy_table_share; - - /* Iterate all tables that are not in the system tablespace and add them - to the list of tables to be rebuilt later. */ - - trx = trx_allocate_for_mysql(); - trx_start_for_ddl(trx, TRX_DICT_OP_INDEX); - - row_mysql_lock_data_dictionary(trx); - - /* Enlarge the fatal lock wait timeout during index rebuild - operation. */ - os_increment_counter_by_amount(server_mutex, - srv_fatal_semaphore_wait_threshold, - 7200); - - mtr_start(&mtr); - - sys_tables = dict_table_get_low("SYS_TABLES"); - sys_index = UT_LIST_GET_FIRST(sys_tables->indexes); - ut_a(!dict_table_is_comp(sys_tables)); - - pthread_mutex_init(&table_list_mutex, NULL); - UT_LIST_INIT(table_list); - - btr_pcur_open_at_index_side(TRUE, sys_index, BTR_SEARCH_LEAF, &pcur, - TRUE, 0, &mtr); - for (;;) { - btr_pcur_move_to_next_user_rec(&pcur, &mtr); - - rec = btr_pcur_get_rec(&pcur); - - if (!btr_pcur_is_on_user_rec(&pcur)) { - /* end of index */ - - break; - } - - if (rec_get_deleted_flag(rec, 0)) { - continue; - } - - field = rec_get_nth_field_old(rec, 9, &len); - ut_a(len == 4); - - space_id = mach_read_from_4(field); - - /* Don't touch tables in the system tablespace */ - if (!fil_is_user_tablespace_id(space_id)) { - - continue; - } - - field = rec_get_nth_field_old(rec, 0, &len); - - rebuild_table = static_cast<index_rebuild_table_t *> - (mem_alloc(sizeof(*rebuild_table))); - rebuild_table->name = mem_strdupl((char*) field, len); - rebuild_table->space_id = space_id; - - UT_LIST_ADD_LAST(list, table_list, rebuild_table); - } - - btr_pcur_close(&pcur); - mtr_commit(&mtr); - - row_mysql_unlock_data_dictionary(trx); - - trx_commit_for_mysql(trx); - - trx_free_for_mysql(trx); - - /* Start worker threads for the index rebuild operation */ - ut_ad(xtrabackup_rebuild_threads > 0); - - if (xtrabackup_rebuild_threads > 1) { - msg("Starting %lu threads to rebuild indexes.\n", - xtrabackup_rebuild_threads); - } - - threads = (index_rebuild_thread_t *) - mem_alloc(sizeof(*threads) * - xtrabackup_rebuild_threads); - - for (i = 0; i < xtrabackup_rebuild_threads; i++) { - - threads[i].num = i+1; - if (pthread_create(&threads[i].id, NULL, - xb_rebuild_indexes_thread_func, - &threads[i])) { - - msg("error: pthread_create() failed: errno = %d\n", - errno); - ut_a(0); - } - } - - /* Wait for worker threads to finish */ - for (i = 0; i < xtrabackup_rebuild_threads; i++) { - pthread_join(threads[i].id, NULL); - } - - mem_free(threads); -} diff --git a/extra/mariabackup/compact.h b/extra/mariabackup/compact.h deleted file mode 100644 index d0d9840f66d..00000000000 --- a/extra/mariabackup/compact.h +++ /dev/null @@ -1,44 +0,0 @@ -/****************************************************** -XtraBackup: hot backup tool for InnoDB -(c) 2009-2013 Percona LLC and/or its affiliates. -Originally Created 3/3/2009 Yasufumi Kinoshita -Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, -Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -*******************************************************/ - -#ifndef XB_COMPACT_H -#define XB_COMPACT_H - -#include "write_filt.h" - -/* Compact page filter context */ -typedef struct { - my_bool skip; - ds_ctxt_t *ds_buffer; - ds_file_t *buffer; - index_id_t clustered_index; - my_bool clustered_index_found; - my_bool inside_skipped_range; - ulint free_limit; -} xb_wf_compact_ctxt_t; - -/****************************************************************************** -Expand the data files according to the skipped pages maps created by --compact. -@return TRUE on success, FALSE on failure. */ -my_bool xb_expand_datafiles(void); - -#endif diff --git a/extra/mariabackup/datasink.c b/extra/mariabackup/datasink.c index 2f4233ddc98..199eb77ad55 100644 --- a/extra/mariabackup/datasink.c +++ b/extra/mariabackup/datasink.c @@ -46,7 +46,12 @@ ds_create(const char *root, ds_type_t type) ds = &datasink_local; break; case DS_TYPE_ARCHIVE: +#ifdef HAVE_LIBARCHIVE ds = &datasink_archive; +#else + msg("Error : mariabackup was built without libarchive support"); + exit(EXIT_FAILURE); +#endif break; case DS_TYPE_XBSTREAM: ds = &datasink_xbstream; @@ -55,8 +60,10 @@ ds_create(const char *root, ds_type_t type) ds = &datasink_compress; break; case DS_TYPE_ENCRYPT: - ds = &datasink_encrypt; + msg("Error : mariabackup does not support encrypted backups."); + exit(EXIT_FAILURE); break; + case DS_TYPE_TMPFILE: ds = &datasink_tmpfile; break; diff --git a/extra/mariabackup/datasink.h b/extra/mariabackup/datasink.h index 36a3d564a57..e378a806441 100644 --- a/extra/mariabackup/datasink.h +++ b/extra/mariabackup/datasink.h @@ -28,6 +28,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA extern "C" { #endif +extern char *xtrabackup_tmpdir; struct datasink_struct; typedef struct datasink_struct datasink_t; diff --git a/extra/mariabackup/ds_compress.c b/extra/mariabackup/ds_compress.c index c49fceaf7fa..15801c8abd4 100644 --- a/extra/mariabackup/ds_compress.c +++ b/extra/mariabackup/ds_compress.c @@ -152,7 +152,7 @@ compress_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat) /* Write the qpress file header */ name_len = strlen(new_name); if (ds_write(dest_file, "F", 1) || - write_uint32_le(dest_file, name_len) || + write_uint32_le(dest_file, (uint)name_len) || /* we want to write the terminating \0 as well */ ds_write(dest_file, new_name, name_len + 1)) { goto err; @@ -453,7 +453,7 @@ compress_worker_thread_func(void *arg) with qpress implementation. */ thd->adler = adler32(0x00000001, (uchar *) thd->to, - thd->to_len); + (uInt)thd->to_len); } pthread_mutex_unlock(&thd->data_mutex); diff --git a/extra/mariabackup/ds_encrypt.c b/extra/mariabackup/ds_encrypt.c index f8d62a03e13..19ced416687 100644 --- a/extra/mariabackup/ds_encrypt.c +++ b/extra/mariabackup/ds_encrypt.c @@ -22,7 +22,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA #include <my_base.h> #include "common.h" #include "datasink.h" - +#ifdef HAVE_GRYPT #if GCC_VERSION >= 4002 /* Workaround to avoid "gcry_ac_* is deprecated" warnings in gcrypt.h */ # pragma GCC diagnostic ignored "-Wdeprecated-declarations" @@ -615,3 +615,4 @@ encrypt_worker_thread_func(void *arg) return NULL; } +#endif /* HAVE_GCRYPT*/ diff --git a/extra/mariabackup/ds_encrypt.h b/extra/mariabackup/ds_encrypt.h index ed869747d79..3e9e40ad354 100644 --- a/extra/mariabackup/ds_encrypt.h +++ b/extra/mariabackup/ds_encrypt.h @@ -22,7 +22,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA #define DS_ENCRYPT_H #include "datasink.h" - +#ifdef HAVE_GCRYPT extern datasink_t datasink_encrypt; - +#endif #endif diff --git a/extra/mariabackup/ds_local.c b/extra/mariabackup/ds_local.c index dc13ed7595e..3e2b1e0129b 100644 --- a/extra/mariabackup/ds_local.c +++ b/extra/mariabackup/ds_local.c @@ -53,9 +53,9 @@ local_init(const char *root) && my_errno != EEXIST && my_errno != EISDIR) { char errbuf[MYSYS_STRERROR_SIZE]; + my_strerror(errbuf, sizeof(errbuf),my_errno); my_error(EE_CANT_MKDIR, MYF(ME_BELL | ME_WAITTANG), - root, my_errno, my_strerror(errbuf, sizeof(errbuf), - my_errno)); + root, my_errno,errbuf, my_errno); return NULL; } @@ -85,9 +85,9 @@ local_open(ds_ctxt_t *ctxt, const char *path, dirname_part(dirpath, fullpath, &dirpath_len); if (my_mkdir(dirpath, 0777, MYF(0)) < 0 && my_errno != EEXIST) { char errbuf[MYSYS_STRERROR_SIZE]; + my_strerror(errbuf, sizeof(errbuf), my_errno); my_error(EE_CANT_MKDIR, MYF(ME_BELL | ME_WAITTANG), - dirpath, my_errno, my_strerror(errbuf, sizeof(errbuf), - my_errno)); + dirpath, my_errno, errbuf); return NULL; } diff --git a/extra/mariabackup/ds_stdout.c b/extra/mariabackup/ds_stdout.c index 616bcbd831e..91a514ddf64 100644 --- a/extra/mariabackup/ds_stdout.c +++ b/extra/mariabackup/ds_stdout.c @@ -79,7 +79,7 @@ stdout_open(ds_ctxt_t *ctxt __attribute__((unused)), setmode(fileno(stdout), _O_BINARY); #endif - stdout_file->fd = fileno(stdout); + stdout_file->fd = my_fileno(stdout); file->path = (char *) stdout_file + sizeof(ds_stdout_file_t); memcpy(file->path, fullpath, pathlen); diff --git a/extra/mariabackup/ds_tmpfile.c b/extra/mariabackup/ds_tmpfile.c index 915191dcdae..b039d83ba03 100644 --- a/extra/mariabackup/ds_tmpfile.c +++ b/extra/mariabackup/ds_tmpfile.c @@ -53,7 +53,6 @@ datasink_t datasink_tmpfile = { &tmpfile_deinit }; -MY_TMPDIR mysql_tmpdir_list; static ds_ctxt_t * tmpfile_init(const char *root) @@ -90,7 +89,7 @@ tmpfile_open(ds_ctxt_t *ctxt, const char *path, /* Create a temporary file in tmpdir. The file will be automatically removed on close. Code copied from mysql_tmpfile(). */ - fd = create_temp_file(tmp_path, my_tmpdir(&mysql_tmpdir_list), + fd = create_temp_file(tmp_path,xtrabackup_tmpdir, "xbtemp", #ifdef __WIN__ O_BINARY | O_TRUNC | O_SEQUENTIAL | diff --git a/extra/mariabackup/encryption_plugin.cc b/extra/mariabackup/encryption_plugin.cc new file mode 100644 index 00000000000..9f2782d89a1 --- /dev/null +++ b/extra/mariabackup/encryption_plugin.cc @@ -0,0 +1,157 @@ +#include <mysqld.h> +#include <mysql.h> +#include <xtrabackup.h> +#include <encryption_plugin.h> +#include <backup_copy.h> +#include <sql_plugin.h> +#include <sstream> +#include <vector> +#include <common.h> +#include <backup_mysql.h> + + +extern struct st_maria_plugin *mysql_optional_plugins[]; +extern struct st_maria_plugin *mysql_mandatory_plugins[]; +static void encryption_plugin_init(int argc, char **argv); + +extern char *xb_plugin_load; +extern char *xb_plugin_dir; + +const int PLUGIN_MAX_ARGS = 1024; +vector<string> backup_plugins_args; + +const char *QUERY_PLUGIN = +"SELECT plugin_name, plugin_library, @@plugin_dir" +" FROM information_schema.plugins WHERE plugin_type='ENCRYPTION'" +" AND plugin_status='ACTIVE'"; + +string encryption_plugin_config; + +static void add_to_plugin_load_list(const char *plugin_def) +{ + opt_plugin_load_list_ptr->push_back(new i_string(plugin_def)); +} + +static char XTRABACKUP_EXE[] = "xtrabackup"; + +void encryption_plugin_backup_init(MYSQL *mysql) +{ + MYSQL_RES *result; + MYSQL_ROW row; + ostringstream oss; + char *argv[PLUGIN_MAX_ARGS]; + int argc; + + result = xb_mysql_query(mysql, QUERY_PLUGIN, true, true); + row = mysql_fetch_row(result); + if (!row) + { + mysql_free_result(result); + return; + } + + char *name= row[0]; + char *library= row[1]; + char *dir= row[2]; + +#ifdef _WIN32 + for (char *p = dir; *p; p++) + if (*p == '\\') *p = '/'; +#endif + + string plugin_load(name); + if (library) + plugin_load += string("=") + library; + + oss << "plugin_load=" << plugin_load << endl; + + /* Required to load the plugin later.*/ + add_to_plugin_load_list(plugin_load.c_str()); + strncpy(opt_plugin_dir, dir, FN_REFLEN); + + oss << "plugin_dir=" << '"' << dir << '"' << endl; + + + /* Read plugin variables. */ + char query[1024]; + snprintf(query, 1024, "SHOW variables like '%s_%%'", name); + mysql_free_result(result); + + result = xb_mysql_query(mysql, query, true, true); + while ((row = mysql_fetch_row(result))) + { + string arg("--"); + arg += row[0]; + arg += "="; + arg += row[1]; + backup_plugins_args.push_back(arg); + oss << row[0] << "=" << row[1] << endl; + } + + mysql_free_result(result); + + /* Check whether to encrypt logs. */ + result = xb_mysql_query(mysql, "select @@innodb_encrypt_log", true, true); + row = mysql_fetch_row(result); + srv_encrypt_log = (row != 0 && row[0][0] == '1'); + oss << "innodb_encrypt_log=" << row[0] << endl; + + mysql_free_result(result); + + encryption_plugin_config = oss.str(); + + argc = 0; + argv[argc++] = XTRABACKUP_EXE; + for(size_t i = 0; i < backup_plugins_args.size(); i++) + { + argv[argc++] = (char *)backup_plugins_args[i].c_str(); + if (argc == PLUGIN_MAX_ARGS - 2) + break; + } + argv[argc] = 0; + + encryption_plugin_init(argc, argv); +} + +const char *encryption_plugin_get_config() +{ + return encryption_plugin_config.c_str(); +} + +extern int finalize_encryption_plugin(st_plugin_int *plugin); + + +void encryption_plugin_prepare_init(int argc, char **argv) +{ + + if (!xb_plugin_load) + { + /* This prevents crashes e.g in --stats with wrong my.cnf*/ + finalize_encryption_plugin(0); + return; + } + + add_to_plugin_load_list(xb_plugin_load); + + if (xb_plugin_dir) + strncpy(opt_plugin_dir, xb_plugin_dir, FN_REFLEN); + + char **new_argv = new char *[argc + 1]; + new_argv[0] = XTRABACKUP_EXE; + memcpy(&new_argv[1], argv, argc*sizeof(char *)); + + encryption_plugin_init(argc+1, new_argv); + + delete[] new_argv; +} + +static void encryption_plugin_init(int argc, char **argv) +{ + /* Patch optional and mandatory plugins, we only need to load the one in xb_plugin_load. */ + mysql_optional_plugins[0] = mysql_mandatory_plugins[0] = 0; + msg("Loading encryption plugin\n"); + for (int i= 1; i < argc; i++) + msg("\t Encryption plugin parameter : '%s'\n", argv[i]); + plugin_init(&argc, argv, PLUGIN_INIT_SKIP_PLUGIN_TABLE); +} + diff --git a/extra/mariabackup/encryption_plugin.h b/extra/mariabackup/encryption_plugin.h new file mode 100644 index 00000000000..16d74790254 --- /dev/null +++ b/extra/mariabackup/encryption_plugin.h @@ -0,0 +1,7 @@ +#include <mysql.h> +#include <string> +extern void encryption_plugin_backup_init(MYSQL *mysql); +extern const char* encryption_plugin_get_config(); +extern void encryption_plugin_prepare_init(int argc, char **argv); + +//extern void encryption_plugin_init(int argc, char **argv); diff --git a/extra/mariabackup/fil_cur.cc b/extra/mariabackup/fil_cur.cc index 22ebdb90215..820d8e10c29 100644 --- a/extra/mariabackup/fil_cur.cc +++ b/extra/mariabackup/fil_cur.cc @@ -33,6 +33,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA #include "common.h" #include "read_filt.h" #include "xtrabackup.h" +#include "xb0xb.h" /* Size of read buffer in pages (640 pages = 10M for 16K sized pages) */ #define XB_FIL_CUR_PAGES 640 @@ -167,7 +168,7 @@ xb_fil_cur_open( os_file_create_simple_no_error_handling(0, node->name, OS_FILE_OPEN, OS_FILE_READ_ONLY, - &success); + &success,0); if (!success) { /* The following call prints an error message */ os_file_get_last_error(TRUE); @@ -200,7 +201,7 @@ xb_fil_cur_open( cursor->node = node; cursor->file = node->handle; - if (my_fstat(cursor->file, &cursor->statinfo, MYF(MY_WME))) { + if (stat(cursor->abs_path, &cursor->statinfo)) { msg("[%02u] xtrabackup: error: cannot stat %s\n", thread_n, cursor->abs_path); @@ -253,7 +254,7 @@ xb_fil_cur_open( cursor->buf_page_no = 0; cursor->thread_n = thread_n; - cursor->space_size = cursor->statinfo.st_size / page_size; + cursor->space_size = (ulint)(cursor->statinfo.st_size / page_size); cursor->read_filter = read_filter; cursor->read_filter->init(&cursor->read_filter_ctxt, cursor, @@ -327,26 +328,32 @@ read_retry: cursor->buf_read = 0; cursor->buf_npages = 0; cursor->buf_offset = offset; - cursor->buf_page_no = (ulint) (offset >> cursor->page_size_shift); + cursor->buf_page_no = (ulint)(offset >> cursor->page_size_shift); success = os_file_read(cursor->file, cursor->buf, offset, - to_read); + (ulint)to_read); if (!success) { return(XB_FIL_CUR_ERROR); } + fil_system_enter(); + fil_space_t *space = fil_space_get_by_id(cursor->space_id); + fil_system_exit(); + /* check pages for corruption and re-read if necessary. i.e. in case of partially written pages */ for (page = cursor->buf, i = 0; i < npages; page += cursor->page_size, i++) { + ib_int64_t page_no = cursor->buf_page_no + i; - if (buf_page_is_corrupted(TRUE, page, cursor->zip_size)) { + bool checksum_ok = fil_space_verify_crypt_checksum(page, cursor->zip_size,space, (ulint)page_no); - ulint page_no = cursor->buf_page_no + i; + if (!checksum_ok && + buf_page_is_corrupted(true, page, cursor->zip_size,space)) { if (cursor->is_system && - page_no >= FSP_EXTENT_SIZE && - page_no < FSP_EXTENT_SIZE * 3) { + page_no >= (ib_int64_t)FSP_EXTENT_SIZE && + page_no < (ib_int64_t) FSP_EXTENT_SIZE * 3) { /* skip doublewrite buffer pages */ xb_a(cursor->page_size == UNIV_PAGE_SIZE); msg("[%02u] xtrabackup: " diff --git a/extra/mariabackup/fil_cur.h b/extra/mariabackup/fil_cur.h index 2057765dab5..88239efd2bb 100644 --- a/extra/mariabackup/fil_cur.h +++ b/extra/mariabackup/fil_cur.h @@ -49,10 +49,10 @@ struct xb_fil_cur_t { /*!< read filter context */ byte* orig_buf; /*!< read buffer */ byte* buf; /*!< aligned pointer for orig_buf */ - ulint buf_size; /*!< buffer size in bytes */ - ulint buf_read; /*!< number of read bytes in buffer + size_t buf_size; /*!< buffer size in bytes */ + size_t buf_read; /*!< number of read bytes in buffer after the last cursor read */ - ulint buf_npages; /*!< number of pages in buffer after the + size_t buf_npages; /*!< number of pages in buffer after the last cursor read */ ib_int64_t buf_offset; /*!< file offset of the first page in buffer */ diff --git a/extra/mariabackup/innobackupex.cc b/extra/mariabackup/innobackupex.cc index ec697bb56ff..59fb8fb5565 100644 --- a/extra/mariabackup/innobackupex.cc +++ b/extra/mariabackup/innobackupex.cc @@ -52,14 +52,12 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include <mysqld.h> #include <my_default.h> #include <my_getopt.h> -#include <strings.h> #include <string> #include <sstream> #include <set> #include "common.h" #include "innobackupex.h" #include "xtrabackup.h" -#include "xtrabackup_version.h" #include "xbstream.h" #include "fil_cur.h" #include "write_filt.h" @@ -667,22 +665,6 @@ static struct my_option ibx_long_options[] = (uchar*) &ibx_xtrabackup_parallel, (uchar*) &ibx_xtrabackup_parallel, 0, GET_INT, REQUIRED_ARG, 1, 1, INT_MAX, 0, 0, 0}, - {"rebuild-indexes", OPT_REBUILD_INDEXES, - "This option only has effect when used together with the --apply-log " - "option and is passed directly to xtrabackup. When used, makes " - "xtrabackup rebuild all secondary indexes after applying the log. " - "This option is normally used to prepare compact backups. See the " - "XtraBackup manual for more information.", - (uchar*) &ibx_xtrabackup_rebuild_indexes, - (uchar*) &ibx_xtrabackup_rebuild_indexes, - 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - - {"rebuild-threads", OPT_REBUILD_THREADS, - "Use this number of threads to rebuild indexes in a compact backup. " - "Only has effect with --prepare and --rebuild-indexes.", - (uchar*) &ibx_xtrabackup_rebuild_threads, - (uchar*) &ibx_xtrabackup_rebuild_threads, - 0, GET_UINT, REQUIRED_ARG, 1, 1, UINT_MAX, 0, 0, 0}, {"stream", OPT_STREAM, "This option specifies the format in which to " "do the streamed backup. The option accepts a string argument. The " @@ -845,9 +827,9 @@ ibx_get_one_option(int optid, exit(0); break; case 'v': - msg("innobackupex version %s %s (%s) (revision id: %s)\n", - XTRABACKUP_VERSION, - SYSTEM_TYPE, MACHINE_TYPE, XTRABACKUP_REVISION); + msg("innobackupex version %s %s (%s)\n", + MYSQL_SERVER_VERSION, + SYSTEM_TYPE, MACHINE_TYPE); exit(0); break; case OPT_HISTORY: @@ -1039,7 +1021,6 @@ ibx_init() /* setup xtrabackup options */ xb_close_files = ibx_xb_close_files; - xtrabackup_compact = ibx_xtrabackup_compact; xtrabackup_compress_alg = ibx_xtrabackup_compress_alg; xtrabackup_compress_threads = ibx_xtrabackup_compress_threads; xtrabackup_compress_chunk_size = ibx_xtrabackup_compress_chunk_size; @@ -1057,8 +1038,6 @@ ibx_init() xtrabackup_log_copy_interval = ibx_xtrabackup_log_copy_interval; xtrabackup_incremental = ibx_xtrabackup_incremental; xtrabackup_parallel = ibx_xtrabackup_parallel; - xtrabackup_rebuild_indexes = ibx_xtrabackup_rebuild_indexes; - xtrabackup_rebuild_threads = ibx_xtrabackup_rebuild_threads; xtrabackup_stream_str = ibx_xtrabackup_stream_str; xtrabackup_tables_file = ibx_xtrabackup_tables_file; xtrabackup_throttle = ibx_xtrabackup_throttle; diff --git a/extra/mariabackup/read_filt.cc b/extra/mariabackup/read_filt.cc index 8ebc735e99e..05e6b7c86c7 100644 --- a/extra/mariabackup/read_filt.cc +++ b/extra/mariabackup/read_filt.cc @@ -75,7 +75,7 @@ rf_pass_through_get_next_batch( *read_batch_start = ctxt->offset; *read_batch_len = ctxt->data_file_size - ctxt->offset; - if (*read_batch_len > ctxt->buffer_capacity) { + if (*read_batch_len > (ib_int64_t)ctxt->buffer_capacity) { *read_batch_len = ctxt->buffer_capacity; } @@ -128,7 +128,7 @@ rf_bitmap_get_next_batch( { ulint start_page_id; - start_page_id = ctxt->offset / ctxt->page_size; + start_page_id = (ulint)(ctxt->offset / ctxt->page_size); xb_a (ctxt->offset % ctxt->page_size == 0); @@ -170,7 +170,7 @@ rf_bitmap_get_next_batch( buffer capacity. The subsequent invocations will continue returning the current block in buffer-sized pieces until ctxt->filter_batch_end is reached, trigerring the next bitmap query. */ - if (*read_batch_len > ctxt->buffer_capacity) { + if (*read_batch_len > (ib_int64_t)ctxt->buffer_capacity) { *read_batch_len = ctxt->buffer_capacity; } diff --git a/extra/mariabackup/read_filt.h b/extra/mariabackup/read_filt.h index 73fef06a288..d16f4e1093d 100644 --- a/extra/mariabackup/read_filt.h +++ b/extra/mariabackup/read_filt.h @@ -33,14 +33,14 @@ struct xb_fil_cur_t; struct xb_read_filt_ctxt_t { ib_int64_t offset; /*!< current file offset */ ib_int64_t data_file_size; /*!< data file size */ - ib_int64_t buffer_capacity;/*!< read buffer capacity */ - ulint space_id; /*!< space id */ + size_t buffer_capacity;/*!< read buffer capacity */ + ib_int64_t space_id; /*!< space id */ /* The following fields used only in bitmap filter */ /* Move these to union if any other filters are added in future */ xb_page_bitmap_range *bitmap_range; /*!< changed page bitmap range iterator for space_id */ - ulint page_size; /*!< page size */ - ulint filter_batch_end;/*!< the ending page id of the + size_t page_size; /*!< page size */ + ulint filter_batch_end;/*!< the ending page id of the current changed page block in the bitmap */ }; diff --git a/extra/mariabackup/version_check.pl b/extra/mariabackup/version_check.pl deleted file mode 100644 index 865e2eacb4a..00000000000 --- a/extra/mariabackup/version_check.pl +++ /dev/null @@ -1,1373 +0,0 @@ -use warnings FATAL => 'all'; -use strict; -use English qw(-no_match_vars); -use POSIX "strftime"; - -my @required_perl_version = (5, 0, 5); -my $required_perl_version_old_style = 5.005; - -# check existence of DBD::mysql module -eval { - require DBD::mysql; -}; -my $dbd_mysql_installed = $EVAL_ERROR ? 0 : 1; - -my $now; -my %mysql; -my $prefix = "version_check"; - - -# ########################################################################### -# HTTPMicro package -# This package is a copy without comments from the original. The original -# with comments and its test file can be found in the Bazaar repository at, -# lib/HTTPMicro.pm -# t/lib/HTTPMicro.t -# See https://launchpad.net/percona-toolkit for more information. -# ########################################################################### -{ - -package HTTPMicro; -BEGIN { - $HTTPMicro::VERSION = '0.001'; -} -use strict; -use warnings; - -use Carp (); - - -my @attributes; -BEGIN { - @attributes = qw(agent timeout); - no strict 'refs'; - for my $accessor ( @attributes ) { - *{$accessor} = sub { - @_ > 1 ? $_[0]->{$accessor} = $_[1] : $_[0]->{$accessor}; - }; - } -} - -sub new { - my($class, %args) = @_; - (my $agent = $class) =~ s{::}{-}g; - my $self = { - agent => $agent . "/" . ($class->VERSION || 0), - timeout => 60, - }; - for my $key ( @attributes ) { - $self->{$key} = $args{$key} if exists $args{$key} - } - return bless $self, $class; -} - -my %DefaultPort = ( - http => 80, - https => 443, -); - -sub request { - my ($self, $method, $url, $args) = @_; - @_ == 3 || (@_ == 4 && ref $args eq 'HASH') - or Carp::croak(q/Usage: $http->request(METHOD, URL, [HASHREF])/); - $args ||= {}; # we keep some state in this during _request - - my $response; - for ( 0 .. 1 ) { - $response = eval { $self->_request($method, $url, $args) }; - last unless $@ && $method eq 'GET' - && $@ =~ m{^(?:Socket closed|Unexpected end)}; - } - - if (my $e = "$@") { - $response = { - success => q{}, - status => 599, - reason => 'Internal Exception', - content => $e, - headers => { - 'content-type' => 'text/plain', - 'content-length' => length $e, - } - }; - } - return $response; -} - -sub _request { - my ($self, $method, $url, $args) = @_; - - my ($scheme, $host, $port, $path_query) = $self->_split_url($url); - - my $request = { - method => $method, - scheme => $scheme, - host_port => ($port == $DefaultPort{$scheme} ? $host : "$host:$port"), - uri => $path_query, - headers => {}, - }; - - my $handle = HTTPMicro::Handle->new(timeout => $self->{timeout}); - - $handle->connect($scheme, $host, $port); - - $self->_prepare_headers_and_cb($request, $args); - $handle->write_request_header(@{$request}{qw/method uri headers/}); - $handle->write_content_body($request) if $request->{content}; - - my $response; - do { $response = $handle->read_response_header } - until (substr($response->{status},0,1) ne '1'); - - if (!($method eq 'HEAD' || $response->{status} =~ /^[23]04/)) { - $response->{content} = ''; - $handle->read_content_body(sub { $_[1]->{content} .= $_[0] }, $response); - } - - $handle->close; - $response->{success} = substr($response->{status},0,1) eq '2'; - return $response; -} - -sub _prepare_headers_and_cb { - my ($self, $request, $args) = @_; - - for ($args->{headers}) { - next unless defined; - while (my ($k, $v) = each %$_) { - $request->{headers}{lc $k} = $v; - } - } - $request->{headers}{'host'} = $request->{host_port}; - $request->{headers}{'connection'} = "close"; - $request->{headers}{'user-agent'} ||= $self->{agent}; - - if (defined $args->{content}) { - $request->{headers}{'content-type'} ||= "application/octet-stream"; - utf8::downgrade($args->{content}, 1) - or Carp::croak(q/Wide character in request message body/); - $request->{headers}{'content-length'} = length $args->{content}; - $request->{content} = $args->{content}; - } - return; -} - -sub _split_url { - my $url = pop; - - my ($scheme, $authority, $path_query) = $url =~ m<\A([^:/?#]+)://([^/?#]*)([^#]*)> - or Carp::croak(qq/Cannot parse URL: '$url'/); - - $scheme = lc $scheme; - $path_query = "/$path_query" unless $path_query =~ m<\A/>; - - my $host = (length($authority)) ? lc $authority : 'localhost'; - $host =~ s/\A[^@]*@//; # userinfo - my $port = do { - $host =~ s/:([0-9]*)\z// && length $1 - ? $1 - : $DefaultPort{$scheme} - }; - - return ($scheme, $host, $port, $path_query); -} - -package - HTTPMicro::Handle; # hide from PAUSE/indexers -use strict; -use warnings; - -use Carp qw[croak]; -use Errno qw[EINTR EPIPE]; -use IO::Socket qw[SOCK_STREAM]; - -sub BUFSIZE () { 32768 } - -my $Printable = sub { - local $_ = shift; - s/\r/\\r/g; - s/\n/\\n/g; - s/\t/\\t/g; - s/([^\x20-\x7E])/sprintf('\\x%.2X', ord($1))/ge; - $_; -}; - -sub new { - my ($class, %args) = @_; - return bless { - rbuf => '', - timeout => 60, - max_line_size => 16384, - %args - }, $class; -} - -my $ssl_verify_args = { - check_cn => "when_only", - wildcards_in_alt => "anywhere", - wildcards_in_cn => "anywhere" -}; - -sub connect { - @_ == 4 || croak(q/Usage: $handle->connect(scheme, host, port)/); - my ($self, $scheme, $host, $port) = @_; - - if ( $scheme eq 'https' ) { - eval "require IO::Socket::SSL" - unless exists $INC{'IO/Socket/SSL.pm'}; - croak(qq/IO::Socket::SSL must be installed for https support\n/) - unless $INC{'IO/Socket/SSL.pm'}; - } - elsif ( $scheme ne 'http' ) { - croak(qq/Unsupported URL scheme '$scheme'\n/); - } - - $self->{fh} = 'IO::Socket::INET'->new( - PeerHost => $host, - PeerPort => $port, - Proto => 'tcp', - Type => SOCK_STREAM, - Timeout => $self->{timeout} - ) or croak(qq/Could not connect to '$host:$port': $@/); - - binmode($self->{fh}) - or croak(qq/Could not binmode() socket: '$!'/); - - if ( $scheme eq 'https') { - IO::Socket::SSL->start_SSL($self->{fh}); - ref($self->{fh}) eq 'IO::Socket::SSL' - or die(qq/SSL connection failed for $host\n/); - if ( $self->{fh}->can("verify_hostname") ) { - $self->{fh}->verify_hostname( $host, $ssl_verify_args ) - or die(qq/SSL certificate not valid for $host\n/); - } - else { - my $fh = $self->{fh}; - _verify_hostname_of_cert($host, _peer_certificate($fh), $ssl_verify_args) - or die(qq/SSL certificate not valid for $host\n/); - } - } - - $self->{host} = $host; - $self->{port} = $port; - - return $self; -} - -sub close { - @_ == 1 || croak(q/Usage: $handle->close()/); - my ($self) = @_; - CORE::close($self->{fh}) - or croak(qq/Could not close socket: '$!'/); -} - -sub write { - @_ == 2 || croak(q/Usage: $handle->write(buf)/); - my ($self, $buf) = @_; - - my $len = length $buf; - my $off = 0; - - local $SIG{PIPE} = 'IGNORE'; - - while () { - $self->can_write - or croak(q/Timed out while waiting for socket to become ready for writing/); - my $r = syswrite($self->{fh}, $buf, $len, $off); - if (defined $r) { - $len -= $r; - $off += $r; - last unless $len > 0; - } - elsif ($! == EPIPE) { - croak(qq/Socket closed by remote server: $!/); - } - elsif ($! != EINTR) { - croak(qq/Could not write to socket: '$!'/); - } - } - return $off; -} - -sub read { - @_ == 2 || @_ == 3 || croak(q/Usage: $handle->read(len)/); - my ($self, $len) = @_; - - my $buf = ''; - my $got = length $self->{rbuf}; - - if ($got) { - my $take = ($got < $len) ? $got : $len; - $buf = substr($self->{rbuf}, 0, $take, ''); - $len -= $take; - } - - while ($len > 0) { - $self->can_read - or croak(q/Timed out while waiting for socket to become ready for reading/); - my $r = sysread($self->{fh}, $buf, $len, length $buf); - if (defined $r) { - last unless $r; - $len -= $r; - } - elsif ($! != EINTR) { - croak(qq/Could not read from socket: '$!'/); - } - } - if ($len) { - croak(q/Unexpected end of stream/); - } - return $buf; -} - -sub readline { - @_ == 1 || croak(q/Usage: $handle->readline()/); - my ($self) = @_; - - while () { - if ($self->{rbuf} =~ s/\A ([^\x0D\x0A]* \x0D?\x0A)//x) { - return $1; - } - $self->can_read - or croak(q/Timed out while waiting for socket to become ready for reading/); - my $r = sysread($self->{fh}, $self->{rbuf}, BUFSIZE, length $self->{rbuf}); - if (defined $r) { - last unless $r; - } - elsif ($! != EINTR) { - croak(qq/Could not read from socket: '$!'/); - } - } - croak(q/Unexpected end of stream while looking for line/); -} - -sub read_header_lines { - @_ == 1 || @_ == 2 || croak(q/Usage: $handle->read_header_lines([headers])/); - my ($self, $headers) = @_; - $headers ||= {}; - my $lines = 0; - my $val; - - while () { - my $line = $self->readline; - - if ($line =~ /\A ([^\x00-\x1F\x7F:]+) : [\x09\x20]* ([^\x0D\x0A]*)/x) { - my ($field_name) = lc $1; - $val = \($headers->{$field_name} = $2); - } - elsif ($line =~ /\A [\x09\x20]+ ([^\x0D\x0A]*)/x) { - $val - or croak(q/Unexpected header continuation line/); - next unless length $1; - $$val .= ' ' if length $$val; - $$val .= $1; - } - elsif ($line =~ /\A \x0D?\x0A \z/x) { - last; - } - else { - croak(q/Malformed header line: / . $Printable->($line)); - } - } - return $headers; -} - -sub write_header_lines { - (@_ == 2 && ref $_[1] eq 'HASH') || croak(q/Usage: $handle->write_header_lines(headers)/); - my($self, $headers) = @_; - - my $buf = ''; - while (my ($k, $v) = each %$headers) { - my $field_name = lc $k; - $field_name =~ /\A [\x21\x23-\x27\x2A\x2B\x2D\x2E\x30-\x39\x41-\x5A\x5E-\x7A\x7C\x7E]+ \z/x - or croak(q/Invalid HTTP header field name: / . $Printable->($field_name)); - $field_name =~ s/\b(\w)/\u$1/g; - $buf .= "$field_name: $v\x0D\x0A"; - } - $buf .= "\x0D\x0A"; - return $self->write($buf); -} - -sub read_content_body { - @_ == 3 || @_ == 4 || croak(q/Usage: $handle->read_content_body(callback, response, [read_length])/); - my ($self, $cb, $response, $len) = @_; - $len ||= $response->{headers}{'content-length'}; - - croak("No content-length in the returned response, and this " - . "UA doesn't implement chunking") unless defined $len; - - while ($len > 0) { - my $read = ($len > BUFSIZE) ? BUFSIZE : $len; - $cb->($self->read($read), $response); - $len -= $read; - } - - return; -} - -sub write_content_body { - @_ == 2 || croak(q/Usage: $handle->write_content_body(request)/); - my ($self, $request) = @_; - my ($len, $content_length) = (0, $request->{headers}{'content-length'}); - - $len += $self->write($request->{content}); - - $len == $content_length - or croak(qq/Content-Length missmatch (got: $len expected: $content_length)/); - - return $len; -} - -sub read_response_header { - @_ == 1 || croak(q/Usage: $handle->read_response_header()/); - my ($self) = @_; - - my $line = $self->readline; - - $line =~ /\A (HTTP\/(0*\d+\.0*\d+)) [\x09\x20]+ ([0-9]{3}) [\x09\x20]+ ([^\x0D\x0A]*) \x0D?\x0A/x - or croak(q/Malformed Status-Line: / . $Printable->($line)); - - my ($protocol, $version, $status, $reason) = ($1, $2, $3, $4); - - return { - status => $status, - reason => $reason, - headers => $self->read_header_lines, - protocol => $protocol, - }; -} - -sub write_request_header { - @_ == 4 || croak(q/Usage: $handle->write_request_header(method, request_uri, headers)/); - my ($self, $method, $request_uri, $headers) = @_; - - return $self->write("$method $request_uri HTTP/1.1\x0D\x0A") - + $self->write_header_lines($headers); -} - -sub _do_timeout { - my ($self, $type, $timeout) = @_; - $timeout = $self->{timeout} - unless defined $timeout && $timeout >= 0; - - my $fd = fileno $self->{fh}; - defined $fd && $fd >= 0 - or croak(q/select(2): 'Bad file descriptor'/); - - my $initial = time; - my $pending = $timeout; - my $nfound; - - vec(my $fdset = '', $fd, 1) = 1; - - while () { - $nfound = ($type eq 'read') - ? select($fdset, undef, undef, $pending) - : select(undef, $fdset, undef, $pending) ; - if ($nfound == -1) { - $! == EINTR - or croak(qq/select(2): '$!'/); - redo if !$timeout || ($pending = $timeout - (time - $initial)) > 0; - $nfound = 0; - } - last; - } - $! = 0; - return $nfound; -} - -sub can_read { - @_ == 1 || @_ == 2 || croak(q/Usage: $handle->can_read([timeout])/); - my $self = shift; - return $self->_do_timeout('read', @_) -} - -sub can_write { - @_ == 1 || @_ == 2 || croak(q/Usage: $handle->can_write([timeout])/); - my $self = shift; - return $self->_do_timeout('write', @_) -} - -my $prog = <<'EOP'; -BEGIN { - if ( defined &IO::Socket::SSL::CAN_IPV6 ) { - *CAN_IPV6 = \*IO::Socket::SSL::CAN_IPV6; - } - else { - constant->import( CAN_IPV6 => '' ); - } - my %const = ( - NID_CommonName => 13, - GEN_DNS => 2, - GEN_IPADD => 7, - ); - while ( my ($name,$value) = each %const ) { - no strict 'refs'; - *{$name} = UNIVERSAL::can( 'Net::SSLeay', $name ) || sub { $value }; - } -} -{ - my %dispatcher = ( - issuer => sub { Net::SSLeay::X509_NAME_oneline( Net::SSLeay::X509_get_issuer_name( shift )) }, - subject => sub { Net::SSLeay::X509_NAME_oneline( Net::SSLeay::X509_get_subject_name( shift )) }, - ); - if ( $Net::SSLeay::VERSION >= 1.30 ) { - $dispatcher{commonName} = sub { - my $cn = Net::SSLeay::X509_NAME_get_text_by_NID( - Net::SSLeay::X509_get_subject_name( shift ), NID_CommonName); - $cn =~s{\0$}{}; # work around Bug in Net::SSLeay <1.33 - $cn; - } - } else { - $dispatcher{commonName} = sub { - croak "you need at least Net::SSLeay version 1.30 for getting commonName" - } - } - - if ( $Net::SSLeay::VERSION >= 1.33 ) { - $dispatcher{subjectAltNames} = sub { Net::SSLeay::X509_get_subjectAltNames( shift ) }; - } else { - $dispatcher{subjectAltNames} = sub { - return; - }; - } - - $dispatcher{authority} = $dispatcher{issuer}; - $dispatcher{owner} = $dispatcher{subject}; - $dispatcher{cn} = $dispatcher{commonName}; - - sub _peer_certificate { - my ($self, $field) = @_; - my $ssl = $self->_get_ssl_object or return; - - my $cert = ${*$self}{_SSL_certificate} - ||= Net::SSLeay::get_peer_certificate($ssl) - or return $self->error("Could not retrieve peer certificate"); - - if ($field) { - my $sub = $dispatcher{$field} or croak - "invalid argument for peer_certificate, valid are: ".join( " ",keys %dispatcher ). - "\nMaybe you need to upgrade your Net::SSLeay"; - return $sub->($cert); - } else { - return $cert - } - } - - - my %scheme = ( - ldap => { - wildcards_in_cn => 0, - wildcards_in_alt => 'leftmost', - check_cn => 'always', - }, - http => { - wildcards_in_cn => 'anywhere', - wildcards_in_alt => 'anywhere', - check_cn => 'when_only', - }, - smtp => { - wildcards_in_cn => 0, - wildcards_in_alt => 0, - check_cn => 'always' - }, - none => {}, # do not check - ); - - $scheme{www} = $scheme{http}; # alias - $scheme{xmpp} = $scheme{http}; # rfc 3920 - $scheme{pop3} = $scheme{ldap}; # rfc 2595 - $scheme{imap} = $scheme{ldap}; # rfc 2595 - $scheme{acap} = $scheme{ldap}; # rfc 2595 - $scheme{nntp} = $scheme{ldap}; # rfc 4642 - $scheme{ftp} = $scheme{http}; # rfc 4217 - - - sub _verify_hostname_of_cert { - my $identity = shift; - my $cert = shift; - my $scheme = shift || 'none'; - if ( ! ref($scheme) ) { - $scheme = $scheme{$scheme} or croak "scheme $scheme not defined"; - } - - return 1 if ! %$scheme; # 'none' - - my $commonName = $dispatcher{cn}->($cert); - my @altNames = $dispatcher{subjectAltNames}->($cert); - - if ( my $sub = $scheme->{callback} ) { - return $sub->($identity,$commonName,@altNames); - } - - - my $ipn; - if ( CAN_IPV6 and $identity =~m{:} ) { - $ipn = IO::Socket::SSL::inet_pton(IO::Socket::SSL::AF_INET6,$identity) - or croak "'$identity' is not IPv6, but neither IPv4 nor hostname"; - } elsif ( $identity =~m{^\d+\.\d+\.\d+\.\d+$} ) { - $ipn = IO::Socket::SSL::inet_aton( $identity ) or croak "'$identity' is not IPv4, but neither IPv6 nor hostname"; - } else { - if ( $identity =~m{[^a-zA-Z0-9_.\-]} ) { - $identity =~m{\0} and croak("name '$identity' has \\0 byte"); - $identity = IO::Socket::SSL::idn_to_ascii($identity) or - croak "Warning: Given name '$identity' could not be converted to IDNA!"; - } - } - - my $check_name = sub { - my ($name,$identity,$wtyp) = @_; - $wtyp ||= ''; - my $pattern; - if ( $wtyp eq 'anywhere' and $name =~m{^([a-zA-Z0-9_\-]*)\*(.+)} ) { - $pattern = qr{^\Q$1\E[a-zA-Z0-9_\-]*\Q$2\E$}i; - } elsif ( $wtyp eq 'leftmost' and $name =~m{^\*(\..+)$} ) { - $pattern = qr{^[a-zA-Z0-9_\-]*\Q$1\E$}i; - } else { - $pattern = qr{^\Q$name\E$}i; - } - return $identity =~ $pattern; - }; - - my $alt_dnsNames = 0; - while (@altNames) { - my ($type, $name) = splice (@altNames, 0, 2); - if ( $ipn and $type == GEN_IPADD ) { - return 1 if $ipn eq $name; - - } elsif ( ! $ipn and $type == GEN_DNS ) { - $name =~s/\s+$//; $name =~s/^\s+//; - $alt_dnsNames++; - $check_name->($name,$identity,$scheme->{wildcards_in_alt}) - and return 1; - } - } - - if ( ! $ipn and ( - $scheme->{check_cn} eq 'always' or - $scheme->{check_cn} eq 'when_only' and !$alt_dnsNames)) { - $check_name->($commonName,$identity,$scheme->{wildcards_in_cn}) - and return 1; - } - - return 0; # no match - } -} -EOP - -eval { require IO::Socket::SSL }; -if ( $INC{"IO/Socket/SSL.pm"} ) { - eval $prog; - die $@ if $@; -} - -1; -} -# ########################################################################### -# End HTTPMicro package -# ########################################################################### - -# ########################################################################### -# VersionCheck package -# This package is a copy without comments from the original. The original -# with comments and its test file can be found in the Bazaar repository at, -# lib/VersionCheck.pm -# t/lib/VersionCheck.t -# See https://launchpad.net/percona-toolkit for more information. -# ########################################################################### -{ -package VersionCheck; - - -use strict; -use warnings FATAL => 'all'; -use English qw(-no_match_vars); - -use constant PTDEBUG => $ENV{PTDEBUG} || 0; - -use Data::Dumper; -local $Data::Dumper::Indent = 1; -local $Data::Dumper::Sortkeys = 1; -local $Data::Dumper::Quotekeys = 0; - -use Digest::MD5 qw(md5_hex); -use Sys::Hostname qw(hostname); -use File::Basename qw(); -use File::Spec; -use FindBin qw(); - -eval { - require Percona::Toolkit; - require HTTPMicro; -}; - -{ - my $file = 'percona-version-check'; - my $home = $ENV{HOME} || $ENV{HOMEPATH} || $ENV{USERPROFILE} || '.'; - my @vc_dirs = ( - '/etc/percona', - '/etc/percona-toolkit', - '/tmp', - "$home", - ); - - if ($ENV{PTDEBUG_VERSION_CHECK_HOME}) { - @vc_dirs = ( $ENV{PTDEBUG_VERSION_CHECK_HOME} ); - } - - sub version_check_file { - foreach my $dir ( @vc_dirs ) { - if ( -d $dir && -w $dir ) { - PTDEBUG && _d('Version check file', $file, 'in', $dir); - return $dir . '/' . $file; - } - } - PTDEBUG && _d('Version check file', $file, 'in', $ENV{PWD}); - return $file; # in the CWD - } -} - -sub version_check_time_limit { - return 60 * 60 * 24; # one day -} - - -sub version_check { - my (%args) = @_; - - my $instances = $args{instances} || []; - my $instances_to_check; - - PTDEBUG && _d('FindBin::Bin:', $FindBin::Bin); - if ( !$args{force} ) { - if ( $FindBin::Bin - && (-d "$FindBin::Bin/../.bzr" || -d "$FindBin::Bin/../../.bzr") ) { - PTDEBUG && _d("$FindBin::Bin/../.bzr disables --version-check"); - return; - } - } - - eval { - foreach my $instance ( @$instances ) { - my ($name, $id) = get_instance_id($instance); - $instance->{name} = $name; - $instance->{id} = $id; - } - - push @$instances, { name => 'system', id => 0 }; - - $instances_to_check = get_instances_to_check( - instances => $instances, - vc_file => $args{vc_file}, # testing - now => $args{now}, # testing - ); - PTDEBUG && _d(scalar @$instances_to_check, 'instances to check'); - return unless @$instances_to_check; - - my $protocol = 'https'; - eval { require IO::Socket::SSL; }; - if ( $EVAL_ERROR ) { - PTDEBUG && _d($EVAL_ERROR); - PTDEBUG && _d("SSL not available, won't run version_check"); - return; - } - PTDEBUG && _d('Using', $protocol); - - my $advice = pingback( - instances => $instances_to_check, - protocol => $protocol, - url => $args{url} # testing - || $ENV{PERCONA_VERSION_CHECK_URL} # testing - || "$protocol://v.percona.com", - ); - if ( $advice ) { - PTDEBUG && _d('Advice:', Dumper($advice)); - if ( scalar @$advice > 1) { - print "\n# " . scalar @$advice . " software updates are " - . "available:\n"; - } - else { - print "\n# A software update is available:\n"; - } - print join("\n", map { "# * $_" } @$advice), "\n\n"; - } - }; - if ( $EVAL_ERROR ) { - PTDEBUG && _d('Version check failed:', $EVAL_ERROR); - } - - if ( @$instances_to_check ) { - eval { - update_check_times( - instances => $instances_to_check, - vc_file => $args{vc_file}, # testing - now => $args{now}, # testing - ); - }; - if ( $EVAL_ERROR ) { - PTDEBUG && _d('Error updating version check file:', $EVAL_ERROR); - } - } - - if ( $ENV{PTDEBUG_VERSION_CHECK} ) { - warn "Exiting because the PTDEBUG_VERSION_CHECK " - . "environment variable is defined.\n"; - exit 255; - } - - return; -} - -sub get_instances_to_check { - my (%args) = @_; - - my $instances = $args{instances}; - my $now = $args{now} || int(time); - my $vc_file = $args{vc_file} || version_check_file(); - - if ( !-f $vc_file ) { - PTDEBUG && _d('Version check file', $vc_file, 'does not exist;', - 'version checking all instances'); - return $instances; - } - - open my $fh, '<', $vc_file or die "Cannot open $vc_file: $OS_ERROR"; - chomp(my $file_contents = do { local $/ = undef; <$fh> }); - PTDEBUG && _d('Version check file', $vc_file, 'contents:', $file_contents); - close $fh; - my %last_check_time_for = $file_contents =~ /^([^,]+),(.+)$/mg; - - my $check_time_limit = version_check_time_limit(); - my @instances_to_check; - foreach my $instance ( @$instances ) { - my $last_check_time = $last_check_time_for{ $instance->{id} }; - PTDEBUG && _d('Intsance', $instance->{id}, 'last checked', - $last_check_time, 'now', $now, 'diff', $now - ($last_check_time || 0), - 'hours until next check', - sprintf '%.2f', - ($check_time_limit - ($now - ($last_check_time || 0))) / 3600); - if ( !defined $last_check_time - || ($now - $last_check_time) >= $check_time_limit ) { - PTDEBUG && _d('Time to check', Dumper($instance)); - push @instances_to_check, $instance; - } - } - - return \@instances_to_check; -} - -sub update_check_times { - my (%args) = @_; - - my $instances = $args{instances}; - my $now = $args{now} || int(time); - my $vc_file = $args{vc_file} || version_check_file(); - PTDEBUG && _d('Updating last check time:', $now); - - my %all_instances = map { - $_->{id} => { name => $_->{name}, ts => $now } - } @$instances; - - if ( -f $vc_file ) { - open my $fh, '<', $vc_file or die "Cannot read $vc_file: $OS_ERROR"; - my $contents = do { local $/ = undef; <$fh> }; - close $fh; - - foreach my $line ( split("\n", ($contents || '')) ) { - my ($id, $ts) = split(',', $line); - if ( !exists $all_instances{$id} ) { - $all_instances{$id} = { ts => $ts }; # original ts, not updated - } - } - } - - open my $fh, '>', $vc_file or die "Cannot write to $vc_file: $OS_ERROR"; - foreach my $id ( sort keys %all_instances ) { - PTDEBUG && _d('Updated:', $id, Dumper($all_instances{$id})); - print { $fh } $id . ',' . $all_instances{$id}->{ts} . "\n"; - } - close $fh; - - return; -} - -sub get_instance_id { - my ($instance) = @_; - - my $dbh = $instance->{dbh}; - my $dsn = $instance->{dsn}; - - my $sql = q{SELECT CONCAT(@@hostname, @@port)}; - PTDEBUG && _d($sql); - my ($name) = eval { $dbh->selectrow_array($sql) }; - if ( $EVAL_ERROR ) { - PTDEBUG && _d($EVAL_ERROR); - $sql = q{SELECT @@hostname}; - PTDEBUG && _d($sql); - ($name) = eval { $dbh->selectrow_array($sql) }; - if ( $EVAL_ERROR ) { - PTDEBUG && _d($EVAL_ERROR); - $name = ($dsn->{h} || 'localhost') . ($dsn->{P} || 3306); - } - else { - $sql = q{SHOW VARIABLES LIKE 'port'}; - PTDEBUG && _d($sql); - my (undef, $port) = eval { $dbh->selectrow_array($sql) }; - PTDEBUG && _d('port:', $port); - $name .= $port || ''; - } - } - my $id = md5_hex($name); - - PTDEBUG && _d('MySQL instance:', $id, $name, Dumper($dsn)); - - return $name, $id; -} - - -sub pingback { - my (%args) = @_; - my @required_args = qw(url instances); - foreach my $arg ( @required_args ) { - die "I need a $arg arugment" unless $args{$arg}; - } - my $url = $args{url}; - my $instances = $args{instances}; - - my $ua = $args{ua} || HTTPMicro->new( timeout => 3 ); - - my $response = $ua->request('GET', $url); - PTDEBUG && _d('Server response:', Dumper($response)); - die "No response from GET $url" - if !$response; - die("GET on $url returned HTTP status $response->{status}; expected 200\n", - ($response->{content} || '')) if $response->{status} != 200; - die("GET on $url did not return any programs to check") - if !$response->{content}; - - my $items = parse_server_response( - response => $response->{content} - ); - die "Failed to parse server requested programs: $response->{content}" - if !scalar keys %$items; - - my $versions = get_versions( - items => $items, - instances => $instances, - ); - die "Failed to get any program versions; should have at least gotten Perl" - if !scalar keys %$versions; - - my $client_content = encode_client_response( - items => $items, - versions => $versions, - general_id => md5_hex( hostname() ), - ); - - my $client_response = { - headers => { "X-Percona-Toolkit-Tool" => File::Basename::basename($0) }, - content => $client_content, - }; - PTDEBUG && _d('Client response:', Dumper($client_response)); - - $response = $ua->request('POST', $url, $client_response); - PTDEBUG && _d('Server suggestions:', Dumper($response)); - die "No response from POST $url $client_response" - if !$response; - die "POST $url returned HTTP status $response->{status}; expected 200" - if $response->{status} != 200; - - return unless $response->{content}; - - $items = parse_server_response( - response => $response->{content}, - split_vars => 0, - ); - die "Failed to parse server suggestions: $response->{content}" - if !scalar keys %$items; - my @suggestions = map { $_->{vars} } - sort { $a->{item} cmp $b->{item} } - values %$items; - - return \@suggestions; -} - -sub encode_client_response { - my (%args) = @_; - my @required_args = qw(items versions general_id); - foreach my $arg ( @required_args ) { - die "I need a $arg arugment" unless $args{$arg}; - } - my ($items, $versions, $general_id) = @args{@required_args}; - - my @lines; - foreach my $item ( sort keys %$items ) { - next unless exists $versions->{$item}; - if ( ref($versions->{$item}) eq 'HASH' ) { - my $mysql_versions = $versions->{$item}; - for my $id ( sort keys %$mysql_versions ) { - push @lines, join(';', $id, $item, $mysql_versions->{$id}); - } - } - else { - push @lines, join(';', $general_id, $item, $versions->{$item}); - } - } - - my $client_response = join("\n", @lines) . "\n"; - return $client_response; -} - -sub parse_server_response { - my (%args) = @_; - my @required_args = qw(response); - foreach my $arg ( @required_args ) { - die "I need a $arg arugment" unless $args{$arg}; - } - my ($response) = @args{@required_args}; - - my %items = map { - my ($item, $type, $vars) = split(";", $_); - if ( !defined $args{split_vars} || $args{split_vars} ) { - $vars = [ split(",", ($vars || '')) ]; - } - $item => { - item => $item, - type => $type, - vars => $vars, - }; - } split("\n", $response); - - PTDEBUG && _d('Items:', Dumper(\%items)); - - return \%items; -} - -my %sub_for_type = ( - os_version => \&get_os_version, - perl_version => \&get_perl_version, - perl_module_version => \&get_perl_module_version, - mysql_variable => \&get_mysql_variable, -); - -sub valid_item { - my ($item) = @_; - return unless $item; - if ( !exists $sub_for_type{ $item->{type} } ) { - PTDEBUG && _d('Invalid type:', $item->{type}); - return 0; - } - return 1; -} - -sub get_versions { - my (%args) = @_; - my @required_args = qw(items); - foreach my $arg ( @required_args ) { - die "I need a $arg arugment" unless $args{$arg}; - } - my ($items) = @args{@required_args}; - - my %versions; - foreach my $item ( values %$items ) { - next unless valid_item($item); - eval { - my $version = $sub_for_type{ $item->{type} }->( - item => $item, - instances => $args{instances}, - ); - if ( $version ) { - chomp $version unless ref($version); - $versions{$item->{item}} = $version; - } - }; - if ( $EVAL_ERROR ) { - PTDEBUG && _d('Error getting version for', Dumper($item), $EVAL_ERROR); - } - } - - return \%versions; -} - - -sub get_os_version { - if ( $OSNAME eq 'MSWin32' ) { - require Win32; - return Win32::GetOSDisplayName(); - } - - chomp(my $platform = `uname -s`); - PTDEBUG && _d('platform:', $platform); - return $OSNAME unless $platform; - - chomp(my $lsb_release - = `which lsb_release 2>/dev/null | awk '{print \$1}'` || ''); - PTDEBUG && _d('lsb_release:', $lsb_release); - - my $release = ""; - - if ( $platform eq 'Linux' ) { - if ( -f "/etc/fedora-release" ) { - $release = `cat /etc/fedora-release`; - } - elsif ( -f "/etc/redhat-release" ) { - $release = `cat /etc/redhat-release`; - } - elsif ( -f "/etc/system-release" ) { - $release = `cat /etc/system-release`; - } - elsif ( $lsb_release ) { - $release = `$lsb_release -ds`; - } - elsif ( -f "/etc/lsb-release" ) { - $release = `grep DISTRIB_DESCRIPTION /etc/lsb-release`; - $release =~ s/^\w+="([^"]+)".+/$1/; - } - elsif ( -f "/etc/debian_version" ) { - chomp(my $rel = `cat /etc/debian_version`); - $release = "Debian $rel"; - if ( -f "/etc/apt/sources.list" ) { - chomp(my $code_name = `awk '/^deb/ {print \$3}' /etc/apt/sources.list | awk -F/ '{print \$1}'| awk 'BEGIN {FS="|"} {print \$1}' | sort | uniq -c | sort -rn | head -n1 | awk '{print \$2}'`); - $release .= " ($code_name)" if $code_name; - } - } - elsif ( -f "/etc/os-release" ) { # openSUSE - chomp($release = `grep PRETTY_NAME /etc/os-release`); - $release =~ s/^PRETTY_NAME="(.+)"$/$1/; - } - elsif ( `ls /etc/*release 2>/dev/null` ) { - if ( `grep DISTRIB_DESCRIPTION /etc/*release 2>/dev/null` ) { - $release = `grep DISTRIB_DESCRIPTION /etc/*release | head -n1`; - } - else { - $release = `cat /etc/*release | head -n1`; - } - } - } - elsif ( $platform =~ m/(?:BSD|^Darwin)$/ ) { - my $rel = `uname -r`; - $release = "$platform $rel"; - } - elsif ( $platform eq "SunOS" ) { - my $rel = `head -n1 /etc/release` || `uname -r`; - $release = "$platform $rel"; - } - - if ( !$release ) { - PTDEBUG && _d('Failed to get the release, using platform'); - $release = $platform; - } - chomp($release); - - $release =~ s/^"|"$//g; - - PTDEBUG && _d('OS version =', $release); - return $release; -} - -sub get_perl_version { - my (%args) = @_; - my $item = $args{item}; - return unless $item; - - my $version = sprintf '%vd', $PERL_VERSION; - PTDEBUG && _d('Perl version', $version); - return $version; -} - -sub get_perl_module_version { - my (%args) = @_; - my $item = $args{item}; - return unless $item; - - my $var = '$' . $item->{item} . '::VERSION'; - my $version = eval "use $item->{item}; $var;"; - PTDEBUG && _d('Perl version for', $var, '=', $version); - return $version; -} - -sub get_mysql_variable { - return get_from_mysql( - show => 'VARIABLES', - @_, - ); -} - -sub get_from_mysql { - my (%args) = @_; - my $show = $args{show}; - my $item = $args{item}; - my $instances = $args{instances}; - return unless $show && $item; - - if ( !$instances || !@$instances ) { - PTDEBUG && _d('Cannot check', $item, - 'because there are no MySQL instances'); - return; - } - - if ($item->{item} eq 'MySQL' && $item->{type} eq 'mysql_variable') { - $item->{vars} = ['version_comment', 'version']; - } - - my @versions; - my %version_for; - foreach my $instance ( @$instances ) { - next unless $instance->{id}; # special system instance has id=0 - my $dbh = $instance->{dbh}; - local $dbh->{FetchHashKeyName} = 'NAME_lc'; - my $sql = qq/SHOW $show/; - PTDEBUG && _d($sql); - my $rows = $dbh->selectall_hashref($sql, 'variable_name'); - - my @versions; - foreach my $var ( @{$item->{vars}} ) { - $var = lc($var); - my $version = $rows->{$var}->{value}; - PTDEBUG && _d('MySQL version for', $item->{item}, '=', $version, - 'on', $instance->{name}); - push @versions, $version; - } - $version_for{ $instance->{id} } = join(' ', @versions); - } - - return \%version_for; -} - -sub _d { - my ($package, undef, $line) = caller 0; - @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } - map { defined $_ ? $_ : 'undef' } - @_; - print STDERR "# $package:$line $PID ", join(' ', @_), "\n"; -} - -1; -} -# ########################################################################### -# End VersionCheck package -# ########################################################################### - -# -# parse_connection_options() subroutine parses connection-related command line -# options -# -sub parse_connection_options { - my $con = shift; - - $con->{dsn} = 'dbi:mysql:'; - - # this option has to be first - if ($ENV{option_defaults_file}) { - $con->{dsn} .= ";mysql_read_default_file=$ENV{option_defaults_file}"; - } - - if ($ENV{option_defaults_extra_file}) { - $con->{dsn} .= ";mysql_read_default_file=$ENV{option_defaults_extra_file}"; - } - - $con->{dsn} .= ";mysql_read_default_group=xtrabackup"; - - if ($ENV{option_mysql_password}) { - $con->{dsn_password} = "$ENV{option_mysql_password}"; - } - if ($ENV{option_mysql_user}) { - $con->{dsn_user} = "$ENV{option_mysql_user}"; - } - if ($ENV{option_mysql_host}) { - $con->{dsn} .= ";host=$ENV{option_mysql_host}"; - } - if ($ENV{option_mysql_port}) { - $con->{dsn} .= ";port=$ENV{option_mysql_port}"; - } - if ($ENV{option_mysql_socket}) { - $con->{dsn} .= ";mysql_socket=$ENV{option_mysql_socket}"; - } -} - -# -# mysql_connect subroutine connects to MySQL server -# -sub mysql_connect { - my %con; - my %args = ( - # Defaults - abort_on_error => 1, - @_ - ); - - $con{abort_on_error} = $args{abort_on_error}; - - parse_connection_options(\%con); - - $now = current_time(); - print STDERR "$now $prefix Connecting to MySQL server with DSN '$con{dsn}'" . - (defined($con{dsn_user}) ? " as '$con{dsn_user}' " : "") . - " (using password: "; - if (defined($con{dsn_password})) { - print STDERR "YES).\n"; - } else { - print STDERR "NO).\n"; - } - - eval { - $con{dbh}=DBI->connect($con{dsn}, $con{dsn_user}, - $con{dsn_password}, { RaiseError => 1 }); - }; - - if ($EVAL_ERROR) { - $con{connect_error}=$EVAL_ERROR; - } else { - $now = current_time(); - print STDERR "$now $prefix Connected to MySQL server\n"; - } - - if ($args{abort_on_error}) { - if (!$dbd_mysql_installed) { - die "Failed to connect to MySQL server as " . - "DBD::mysql module is not installed"; - } else { - if (!$con{dbh}) { - die "Failed to connect to MySQL server: " . - $con{connect_error}; - } - } - } - - if ($con{dbh}) { - $con{dbh}->do("SET SESSION wait_timeout=2147483"); - } - - return %con; -} - -# -# return current local time as string in form "070816 12:23:15" -# -sub current_time { - return strftime("%y%m%d %H:%M:%S", localtime()); -} - - -%mysql = mysql_connect(abort_on_error => 1); - -$now = current_time(); -print STDERR - "$now $prefix Executing a version check against the server...\n"; - -# Redirect STDOUT to STDERR, as VersionCheck prints alerts to STDOUT -select STDERR; - -VersionCheck::version_check( - force => 1, - instances => [ { - dbh => $mysql{dbh}, - dsn => $mysql{dsn} - } - ] - ); -# Restore STDOUT as the default filehandle -select STDOUT; - -$now = current_time(); -print STDERR "$now $prefix Done.\n"; diff --git a/extra/mariabackup/write_filt.cc b/extra/mariabackup/write_filt.cc index 129302d7fa0..cf7753bf380 100644 --- a/extra/mariabackup/write_filt.cc +++ b/extra/mariabackup/write_filt.cc @@ -75,7 +75,7 @@ wf_incremental_init(xb_write_filt_ctxt_t *ctxt, char *dst_name, ctxt->cursor = cursor; /* allocate buffer for incremental backup (4096 pages) */ - buf_size = (UNIV_PAGE_SIZE_MAX / 4 + 1) * UNIV_PAGE_SIZE_MAX; + buf_size = (cursor->page_size / 4 + 1) * cursor->page_size; cp->delta_buf_base = static_cast<byte *>(ut_malloc(buf_size)); memset(cp->delta_buf_base, 0, buf_size); cp->delta_buf = static_cast<byte *> diff --git a/extra/mariabackup/write_filt.h b/extra/mariabackup/write_filt.h index 20213b6f523..bcab263f1dd 100644 --- a/extra/mariabackup/write_filt.h +++ b/extra/mariabackup/write_filt.h @@ -27,7 +27,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA #include "fil_cur.h" #include "datasink.h" -#include "compact.h" /* Incremental page filter context */ typedef struct { @@ -41,7 +40,6 @@ typedef struct { xb_fil_cur_t *cursor; union { xb_wf_incremental_ctxt_t wf_incremental_ctxt; - xb_wf_compact_ctxt_t wf_compact_ctxt; } u; } xb_write_filt_ctxt_t; @@ -56,6 +54,5 @@ typedef struct { extern xb_write_filt_t wf_write_through; extern xb_write_filt_t wf_incremental; -extern xb_write_filt_t wf_compact; #endif /* XB_WRITE_FILT_H */ diff --git a/extra/mariabackup/wsrep.cc b/extra/mariabackup/wsrep.cc index 420ada75f36..be11e058255 100644 --- a/extra/mariabackup/wsrep.cc +++ b/extra/mariabackup/wsrep.cc @@ -46,7 +46,7 @@ permission notice: #include <trx0sys.h> #include "common.h" - +#ifdef WITH_WSREP #define WSREP_XID_PREFIX "WSREPXid" #define WSREP_XID_PREFIX_LEN MYSQL_XID_PREFIX_LEN #define WSREP_XID_UUID_OFFSET 8 @@ -61,11 +61,11 @@ permission notice: /* Galera UUID type - for all unique IDs */ typedef struct wsrep_uuid { - uint8_t data[16]; + unsigned char data[16]; } wsrep_uuid_t; /* sequence number of a writeset, etc. */ -typedef int64_t wsrep_seqno_t; +typedef long long wsrep_seqno_t; /* Undefined UUID */ static const wsrep_uuid_t WSREP_UUID_UNDEFINED = {{0,}}; @@ -217,3 +217,4 @@ xb_write_galera_info(bool incremental_prepare) fclose(fp); } +#endif diff --git a/extra/mariabackup/xb0xb.h b/extra/mariabackup/xb0xb.h new file mode 100644 index 00000000000..659ab8ea5d0 --- /dev/null +++ b/extra/mariabackup/xb0xb.h @@ -0,0 +1,78 @@ +/****************************************************** +Copyright (c) 2012 Percona LLC and/or its affiliates. + +Declarations of XtraBackup functions called by InnoDB code. + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +*******************************************************/ + +#ifndef xb0xb_h +#define xb0xb_h + + +extern void os_io_init_simple(void); +extern os_file_t files[1000]; +extern const char *innodb_checksum_algorithm_names[]; +extern TYPELIB innodb_checksum_algorithm_typelib; +extern dberr_t open_or_create_data_files( + ibool* create_new_db, +#ifdef UNIV_LOG_ARCHIVE + lsn_t* min_arch_log_no, + lsn_t* max_arch_log_no, +#endif + lsn_t* min_flushed_lsn, + lsn_t* max_flushed_lsn, + ulint* sum_of_new_sizes) + ; +int +fil_file_readdir_next_file( +/*=======================*/ +dberr_t* err, /*!< out: this is set to DB_ERROR if an error + was encountered, otherwise not changed */ + const char* dirname,/*!< in: directory name or path */ + os_file_dir_t dir, /*!< in: directory stream */ + os_file_stat_t* info) /*!< in/out: buffer where the + info is returned */; +buf_block_t* btr_node_ptr_get_child( + const rec_t* node_ptr,/*!< in: node pointer */ + dict_index_t* index, /*!< in: index */ + const ulint* offsets,/*!< in: array returned by rec_get_offsets() */ + mtr_t* mtr) /*!< in: mtr */; + +buf_block_t* +btr_root_block_get( +/*===============*/ +const dict_index_t* index, /*!< in: index tree */ +ulint mode, /*!< in: either RW_S_LATCH + or RW_X_LATCH */ + mtr_t* mtr) /*!< in: mtr */; +fil_space_t* +fil_space_get_by_name(const char *); +ibool +recv_check_cp_is_consistent(const byte* buf); +void +innodb_log_checksum_func_update( +/*============================*/ +ulint algorithm) /*!< in: algorithm */; +dberr_t recv_find_max_checkpoint(log_group_t** max_group, ulint* max_field); +dberr_t +srv_undo_tablespaces_init( +/*======================*/ +ibool create_new_db, +ibool backup_mode, +const ulint n_conf_tablespaces, +ulint* n_opened); + +#endif diff --git a/extra/mariabackup/xb_regex.h b/extra/mariabackup/xb_regex.h index 94c5e2a5fa7..2e07e434e27 100644 --- a/extra/mariabackup/xb_regex.h +++ b/extra/mariabackup/xb_regex.h @@ -22,50 +22,27 @@ my_regex is used on Windows and native calls are used on POSIX platforms. */ #ifndef XB_REGEX_H #define XB_REGEX_H -#ifdef _WIN32 - -#include <my_regex.h> - -typedef my_regex_t xb_regex_t; -typedef my_regmatch_t xb_regmatch_t; - -#define xb_regex_init() my_regex_init(&my_charset_latin1) - -#define xb_regexec(preg,string,nmatch,pmatch,eflags) \ - my_regexec(preg, string, nmatch, pmatch, eflags) - -#define xb_regerror(errcode,preg,errbuf,errbuf_size) \ - my_regerror(errcode, preg, errbuf, errbuf_size) - -#define xb_regcomp(preg,regex,cflags) \ - my_regcomp(preg, regex, cflags, &my_charset_latin1) - -#define xb_regfree(preg) my_regfree(preg) - -#define xb_regex_end() my_regex_end() - -#else /* ! _WIN32 */ - +#ifdef HAVE_SYSTEM_REGEX #include <regex.h> +#else +#include <pcreposix.h> +#endif -typedef regex_t xb_regex_t; -typedef regmatch_t xb_regmatch_t; +typedef regex_t* xb_regex_t; -#define xb_regex_init() do { } while(0) +#define xb_regex_init() -#define xb_regexec(preg,string,nmatch,pmatch,eflags) \ +#define xb_regexec(preg,string,nmatch,pmatch,eflags) \ regexec(preg, string, nmatch, pmatch, eflags) -#define xb_regerror(errcode,preg,errbuf,errbuf_size) \ +#define xb_regerror(errcode,preg,errbuf,errbuf_size) \ regerror(errcode, preg, errbuf, errbuf_size) -#define xb_regcomp(preg,regex,cflags) \ +#define xb_regcomp(preg,regex,cflags) \ regcomp(preg, regex, cflags) #define xb_regfree(preg) regfree(preg) -#define xb_regex_end() do { } while (0) - -#endif /* _WIN32 */ +#define xb_regex_end() #endif /* XB_REGEX_H */ diff --git a/extra/mariabackup/xbcrypt_common.c b/extra/mariabackup/xbcrypt_common.c index fe34fcb3bb0..52fa2ce3589 100644 --- a/extra/mariabackup/xbcrypt_common.c +++ b/extra/mariabackup/xbcrypt_common.c @@ -27,6 +27,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA # pragma GCC diagnostic ignored "-Wdeprecated-declarations" #endif +#ifdef HAVE_GRYPT #include <gcrypt.h> #if GCC_VERSION >= 4002 @@ -58,3 +59,4 @@ xb_crypt_create_iv(void* ivbuf, size_t ivlen) { gcry_create_nonce(ivbuf, ivlen); } +#endif
\ No newline at end of file diff --git a/extra/mariabackup/xbcrypt_write.c b/extra/mariabackup/xbcrypt_write.c index 5cbeb67f227..6cc30852215 100644 --- a/extra/mariabackup/xbcrypt_write.c +++ b/extra/mariabackup/xbcrypt_write.c @@ -73,7 +73,7 @@ int xb_crypt_write_chunk(xb_wcrypt_t *crypt, const void *buf, size_t olen, int8store(ptr, (ulonglong)elen); /* encrypted (actual) size */ ptr += 8; - checksum = crc32(0, buf, elen); + checksum = crc32(0, buf, (uint)elen); int4store(ptr, checksum); /* checksum */ ptr += 4; diff --git a/extra/mariabackup/xbstream.c b/extra/mariabackup/xbstream.c index ba3412a359b..9990b00ea4b 100644 --- a/extra/mariabackup/xbstream.c +++ b/extra/mariabackup/xbstream.c @@ -208,15 +208,15 @@ int stream_one_file(File file, xb_wstream_file_t *xbfile) { uchar *buf; - size_t bytes; - size_t offset; + ssize_t bytes; + my_off_t offset; posix_fadvise(file, 0, 0, POSIX_FADV_SEQUENTIAL); offset = my_tell(file, MYF(MY_WME)); buf = (uchar*)(my_malloc(XBSTREAM_BUFFER_SIZE, MYF(MY_FAE))); - while ((bytes = my_read(file, buf, XBSTREAM_BUFFER_SIZE, + while ((bytes = (ssize_t)my_read(file, buf, XBSTREAM_BUFFER_SIZE, MYF(MY_WME))) > 0) { if (xb_stream_write_data(xbfile, buf, bytes)) { msg("%s: xb_stream_write_data() failed.\n", @@ -232,7 +232,7 @@ stream_one_file(File file, xb_wstream_file_t *xbfile) my_free(buf); - if (bytes == (size_t) -1) { + if (bytes < 0) { return 1; } diff --git a/extra/mariabackup/xbstream_read.c b/extra/mariabackup/xbstream_read.c index 0ffcabd9270..f8d88926ec2 100644 --- a/extra/mariabackup/xbstream_read.c +++ b/extra/mariabackup/xbstream_read.c @@ -48,13 +48,13 @@ xb_stream_read_new(void) stream->buffer = my_malloc(INIT_BUFFER_LEN, MYF(MY_FAE)); stream->buflen = INIT_BUFFER_LEN; - stream->fd = fileno(stdin); - stream->offset = 0; - #ifdef __WIN__ - setmode(stream->fd, _O_BINARY); + setmode(fileno(stdin), _O_BINARY); #endif + stream->fd = my_fileno(stdin); + stream->offset = 0; + return stream; } diff --git a/extra/mariabackup/xbstream_write.c b/extra/mariabackup/xbstream_write.c index 7b042eea49c..a11811dc375 100644 --- a/extra/mariabackup/xbstream_write.c +++ b/extra/mariabackup/xbstream_write.c @@ -34,7 +34,7 @@ struct xb_wstream_struct { struct xb_wstream_file_struct { xb_wstream_t *stream; char *path; - ulong path_len; + size_t path_len; char chunk[XB_STREAM_MIN_CHUNK_SIZE]; char *chunk_ptr; size_t chunk_free; @@ -54,7 +54,7 @@ xb_stream_default_write_callback(xb_wstream_file_t *file __attribute__((unused)) void *userdata __attribute__((unused)), const void *buf, size_t len) { - if (my_write(fileno(stdout), buf, len, MYF(MY_WME | MY_NABP))) + if (my_write(my_fileno(stdout), buf, len, MYF(MY_WME | MY_NABP))) return -1; return len; } @@ -77,7 +77,7 @@ xb_stream_write_open(xb_wstream_t *stream, const char *path, xb_stream_write_callback *onwrite) { xb_wstream_file_t *file; - ulong path_len; + size_t path_len; path_len = strlen(path); @@ -90,7 +90,19 @@ xb_stream_write_open(xb_wstream_t *stream, const char *path, path_len + 1, MYF(MY_FAE)); file->path = (char *) (file + 1); +#ifdef _WIN32 + /* Normalize path on Windows, so we can restore elsewhere.*/ + { + int i; + for (i = 0; ; i++) { + file->path[i] = (path[i] == '\\') ? '/' : path[i]; + if (!path[i]) + break; + } + } +#else memcpy(file->path, path, path_len + 1); +#endif file->path_len = path_len; file->stream = stream; @@ -208,7 +220,7 @@ xb_stream_write_chunk(xb_wstream_file_t *file, const void *buf, size_t len) int8store(ptr, file->offset); /* Payload offset */ ptr += 8; - checksum = crc32(0, buf, len); /* checksum */ + checksum = crc32(0, buf, (uint)len); /* checksum */ int4store(ptr, checksum); ptr += 4; diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index d24c915bb48..3ca6efaafcf 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -53,9 +53,9 @@ Place, Suite 330, Boston, MA 02111-1307 USA #ifdef __linux__ # include <sys/prctl.h> +#include <sys/resource.h> #endif -#include <sys/resource.h> #include <btr0sea.h> #include <dict0priv.h> @@ -74,7 +74,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA #define G_PTR uchar* #include "common.h" -#include "xtrabackup_version.h" #include "datasink.h" #include "xb_regex.h" @@ -91,6 +90,10 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include "backup_mysql.h" #include "backup_copy.h" #include "backup_mysql.h" +#include "xb0xb.h" +#include "encryption_plugin.h" +#include <sql_plugin.h> +#include <srv0srv.h> /* TODO: replace with appropriate macros used in InnoDB 5.6 */ #define PAGE_ZIP_MIN_SIZE_SHIFT 10 @@ -143,15 +146,16 @@ lsn_t xtrabackup_archived_to_lsn = 0; /* for --archived-to-lsn */ char *xtrabackup_tables = NULL; +char *xtrabackup_tmpdir; + /* List of regular expressions for filtering */ typedef struct xb_regex_list_node_struct xb_regex_list_node_t; struct xb_regex_list_node_struct { UT_LIST_NODE_T(xb_regex_list_node_t) regex_list; - xb_regex_t regex; + regex_t regex; }; static UT_LIST_BASE_NODE_T(xb_regex_list_node_t) regex_list; - -static xb_regmatch_t tables_regmatch[1]; +static regmatch_t tables_regmatch[1]; char *xtrabackup_tables_file = NULL; static hash_table_t* tables_hash = NULL; @@ -210,6 +214,7 @@ ulint xtrabackup_rebuild_threads = 1; /* sleep interval beetween log copy iterations in log copying thread in milliseconds (default is 1 second) */ ulint xtrabackup_log_copy_interval = 1000; +static ulong max_buf_pool_modified_pct; /* Ignored option (--log) for MySQL option compatibility */ char* log_ignored_opt = NULL; @@ -294,8 +299,6 @@ it every INNOBASE_WAKE_INTERVAL'th step. */ #define INNOBASE_WAKE_INTERVAL 32 ulong innobase_active_counter = 0; -ibool srv_compact_backup = FALSE; -ibool srv_rebuild_indexes = FALSE; static char *xtrabackup_debug_sync = NULL; @@ -309,13 +312,15 @@ lsn_t min_flushed_lsn= 0; lsn_t max_flushed_lsn= 0; /* The size of archived log file */ -size_t xtrabackup_arch_file_size = 0ULL; +ib_int64_t xtrabackup_arch_file_size = 0ULL; /* The minimal LSN of found archived log files */ lsn_t xtrabackup_arch_first_file_lsn = 0ULL; /* The maximum LSN of found archived log files */ lsn_t xtrabackup_arch_last_file_lsn = 0ULL; ulong xb_open_files_limit= 0; +char *xb_plugin_dir; +char *xb_plugin_load; my_bool xb_close_files= FALSE; /* Datasinks */ @@ -574,6 +579,9 @@ enum options_xtrabackup OPT_XTRA_INCREMENTAL_FORCE_SCAN, OPT_DEFAULTS_GROUP, OPT_OPEN_FILES_LIMIT, + OPT_PLUGIN_DIR, + OPT_PLUGIN_LOAD, + OPT_INNODB_ENCRYPT_LOG, OPT_CLOSE_FILES, OPT_CORE_FILE, @@ -682,7 +690,13 @@ struct my_option xb_client_options[] = 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"stream", OPT_XTRA_STREAM, "Stream all backup files to the standard output " - "in the specified format. Currently the only supported format is 'tar'.", + "in the specified format." +#ifdef HAVE_LIBARCHIVE + "Supported formats are 'tar' and 'xbstream'." +#else + "Supported format is 'xbstream'." +#endif + , (G_PTR*) &xtrabackup_stream_str, (G_PTR*) &xtrabackup_stream_str, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, @@ -1001,12 +1015,6 @@ struct my_option xb_client_options[] = #include "sslopt-longopts.h" -#if !defined(HAVE_YASSL) - {"server-public-key-path", OPT_SERVER_PUBLIC_KEY, - "File path to the server public RSA key in PEM format.", - &opt_server_public_key, &opt_server_public_key, 0, - GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, -#endif { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} }; @@ -1098,11 +1106,7 @@ Disable with --skip-innodb-doublewrite.", (G_PTR*) &innobase_use_doublewrite, (G_PTR*) &innobase_file_per_table, (G_PTR*) &innobase_file_per_table, 0, GET_BOOL, NO_ARG, FALSE, 0, 0, 0, 0, 0}, - {"innodb_flush_log_at_trx_commit", OPT_INNODB_FLUSH_LOG_AT_TRX_COMMIT, - "Set to 0 (write and flush once per second), 1 (write and flush at each commit) or 2 (write at commit, flush once per second).", - (G_PTR*) &srv_flush_log_at_trx_commit, - (G_PTR*) &srv_flush_log_at_trx_commit, - 0, GET_ULONG, OPT_ARG, 1, 0, 2, 0, 0, 0}, + {"innodb_flush_method", OPT_INNODB_FLUSH_METHOD, "With which method to flush data.", (G_PTR*) &innobase_unix_file_flush_method, (G_PTR*) &innobase_unix_file_flush_method, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, @@ -1203,6 +1207,18 @@ Disable with --skip-innodb-doublewrite.", (G_PTR*) &innobase_use_doublewrite, (G_PTR*) &defaults_group, (G_PTR*) &defaults_group, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"plugin-dir", OPT_PLUGIN_DIR, "Server plugin directory", + &xb_plugin_dir, &xb_plugin_dir, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 }, + + { "plugin-load", OPT_PLUGIN_LOAD, "encrypton plugin to load", + &xb_plugin_load, &xb_plugin_load, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 }, + + { "innodb-encrypt-log", OPT_INNODB_ENCRYPT_LOG, "encrypton plugin to load", + &srv_encrypt_log, &srv_encrypt_log, + 0, GET_BOOL, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 }, + {"open_files_limit", OPT_OPEN_FILES_LIMIT, "the maximum number of file " "descriptors to reserve with setrlimit().", (G_PTR*) &xb_open_files_limit, (G_PTR*) &xb_open_files_limit, 0, GET_ULONG, @@ -1276,9 +1292,8 @@ static const char *xb_server_default_groups[]= static void print_version(void) { - msg("%s version %s based on MySQL server %s %s (%s) (revision id: %s)\n", - my_progname, XTRABACKUP_VERSION, MYSQL_SERVER_VERSION, SYSTEM_TYPE, - MACHINE_TYPE, XTRABACKUP_REVISION); + msg("%s based on MariaDB server %s %s (%s) \n", + my_progname, MYSQL_SERVER_VERSION, SYSTEM_TYPE, MACHINE_TYPE); } static void usage(void) @@ -1515,10 +1530,10 @@ xb_init_log_block_size(void) { srv_log_block_size = 0; if (innobase_log_block_size != 512) { - uint n_shift = get_bit_shift(innobase_log_block_size);; + uint n_shift = (uint)get_bit_shift(innobase_log_block_size);; if (n_shift > 0) { - srv_log_block_size = (1 << n_shift); + srv_log_block_size = (ulint)(1LL << n_shift); msg("InnoDB: The log block size is set to %lu.\n", srv_log_block_size); } @@ -1541,13 +1556,13 @@ innodb_init_param(void) static char current_dir[3]; /* Set if using current lib */ my_bool ret; char *default_path; - + srv_is_being_started = TRUE; /* === some variables from mysqld === */ memset((G_PTR) &mysql_tmpdir_list, 0, sizeof(mysql_tmpdir_list)); if (init_tmpdir(&mysql_tmpdir_list, opt_mysql_tmpdir)) exit(EXIT_FAILURE); - + xtrabackup_tmpdir = my_tmpdir(&mysql_tmpdir_list); /* dummy for initialize all_charsets[] */ get_charset_name(0); @@ -1555,7 +1570,7 @@ innodb_init_param(void) srv_page_size_shift = 0; if (innobase_page_size != (1LL << 14)) { - int n_shift = get_bit_shift((ulint) innobase_page_size); + int n_shift = (int)get_bit_shift((ulint) innobase_page_size); if (n_shift >= 12 && n_shift <= UNIV_PAGE_SIZE_SHIFT_MAX) { srv_page_size_shift = n_shift; @@ -1576,8 +1591,6 @@ innodb_init_param(void) goto error; } - srv_fast_checksum = (ibool) innobase_fast_checksum; - /* Check that values don't overflow on 32-bit systems. */ if (sizeof(ulint) == 4) { if (xtrabackup_use_memory > UINT_MAX32) { @@ -1738,10 +1751,15 @@ mem_free_and_error: } btr_search_enabled = (char) innobase_adaptive_hash_index; + btr_search_index_num = 1; os_use_large_pages = (ibool) innobase_use_large_pages; os_large_page_size = (ulint) innobase_large_page_size; + if (!innobase_log_arch_dir) { + static char default_dir[3] = "./"; + srv_arch_dir = default_dir; + } row_rollback_on_timeout = (ibool) innobase_rollback_on_timeout; srv_file_per_table = (my_bool) innobase_file_per_table; @@ -1848,7 +1866,7 @@ static my_bool innodb_init(void) { int err; - + srv_is_being_started = TRUE; err = innobase_start_or_create_for_mysql(); if (err != DB_SUCCESS) { @@ -1947,12 +1965,6 @@ xtrabackup_read_metadata(char *filename) } /* Optional fields */ - if (fscanf(fp, "compact = %d\n", &t) == 1) { - xtrabackup_compact = (t == 1); - } else { - xtrabackup_compact = 0; - } - if (fscanf(fp, "recover_binlog_info = %d\n", &t) == 1) { recover_binlog_info = (t == 1); } @@ -1981,7 +1993,7 @@ xtrabackup_print_metadata(char *buf, size_t buf_len) metadata_from_lsn, metadata_to_lsn, metadata_last_lsn, - MY_TEST(xtrabackup_compact == TRUE), + MY_TEST(false), MY_TEST(opt_binlog_info == BINLOG_INFO_LOCKLESS)); } @@ -2166,7 +2178,7 @@ check_if_table_matches_filters(const char *name) /* Check against regular expressions list */ for (node = UT_LIST_GET_FIRST(regex_list); node; node = UT_LIST_GET_NEXT(regex_list, node)) { - regres = xb_regexec(&node->regex, name, 1, + regres = regexec(&node->regex, name, 1, tables_regmatch, 0); if (regres != REG_NOMATCH) { @@ -2213,7 +2225,7 @@ check_if_skip_table( dbname = NULL; tbname = name; - while ((ptr = strchr(tbname, SRV_PATH_SEPARATOR)) != NULL) { + while ((ptr = strchr(tbname, '/')) != NULL) { dbname = tbname; tbname = ptr + 1; } @@ -2287,10 +2299,10 @@ xb_get_zip_size(os_file_t file) ibool success; ulint space; - buf = static_cast<byte *>(ut_malloc(2 * UNIV_PAGE_SIZE_MAX)); - page = static_cast<byte *>(ut_align(buf, UNIV_PAGE_SIZE_MAX)); + buf = static_cast<byte *>(ut_malloc(2 * UNIV_PAGE_SIZE)); + page = static_cast<byte *>(ut_align(buf, UNIV_PAGE_SIZE)); - success = os_file_read(file, page, 0, UNIV_PAGE_SIZE_MAX); + success = os_file_read(file, page, 0, UNIV_PAGE_SIZE); if (!success) { goto end; } @@ -2392,8 +2404,6 @@ xtrabackup_copy_datafile(fil_node_t* node, uint thread_n) /* Setup the page write filter */ if (xtrabackup_incremental) { write_filter = &wf_incremental; - } else if (xtrabackup_compact) { - write_filter = &wf_compact; } else { write_filter = &wf_write_through; } @@ -2484,6 +2494,7 @@ static void xtrabackup_choose_lsn_offset(lsn_t start_lsn) { +#if SUPPORT_PERCONA_5_5 ulint no, alt_no, expected_no; ulint blocks_in_group; lsn_t tmp_offset, end_lsn; @@ -2566,8 +2577,11 @@ xtrabackup_choose_lsn_offset(lsn_t start_lsn) decision to choose one over the other. Die just like a Buridan's ass */ ut_a(lsn_chosen == 1); +#endif } +extern ibool log_block_checksum_is_ok_or_old_format(const byte* block); + /*******************************************************//** Scans log from a buffer and writes new log data to the outpud datasinc. @return true if success */ @@ -2704,13 +2718,22 @@ xtrabackup_scan_log_recs( if (!*finished) { write_size = RECV_SCAN_SIZE; } else { - write_size = ut_uint64_align_up(scanned_lsn, - OS_FILE_LOG_BLOCK_SIZE) - start_lsn; + write_size = (ulint)(ut_uint64_align_up(scanned_lsn, + OS_FILE_LOG_BLOCK_SIZE) - start_lsn); if (!is_last && scanned_lsn % OS_FILE_LOG_BLOCK_SIZE) { write_size -= OS_FILE_LOG_BLOCK_SIZE; } } + if (write_size == 0) { + return(true); + } + + if (srv_encrypt_log) { + log_encrypt_before_write(scanned_checkpoint_no, + log_sys->buf, write_size); + } + if (ds_write(dst_log_file, log_sys->buf, write_size)) { msg("xtrabackup: Error: " "write to logfile failed\n"); @@ -2756,7 +2779,7 @@ xtrabackup_copy_logfile(lsn_t from_lsn, my_bool is_last) mutex_enter(&log_sys->mutex); log_group_read_log_seg(LOG_RECOVER, log_sys->buf, - group, start_lsn, end_lsn); + group, start_lsn, end_lsn, false); if (!xtrabackup_scan_log_recs(group, is_last, start_lsn, &contiguous_lsn, &group_scanned_lsn, @@ -3133,6 +3156,7 @@ xb_load_tablespaces(void) ibool create_new_db; ulint err; ulint sum_of_new_sizes; + lsn_t min_arch_logno, max_arch_logno; for (i = 0; i < srv_n_file_io_threads; i++) { thread_nr[i] = i; @@ -3144,6 +3168,7 @@ xb_load_tablespaces(void) os_thread_sleep(200000); /*0.2 sec*/ err = open_or_create_data_files(&create_new_db, + &min_arch_logno, &max_arch_logno, &min_flushed_lsn, &max_flushed_lsn, &sum_of_new_sizes); if (err != DB_SUCCESS) { @@ -3225,19 +3250,9 @@ xb_data_files_close(void) for (i = 0; i < 1000; i++) { os_aio_wake_all_threads_at_shutdown(); - os_mutex_enter(os_sync_mutex); - if (os_thread_count == 0) { - - os_mutex_exit(os_sync_mutex); - - os_thread_sleep(10000); - break; } - - os_mutex_exit(os_sync_mutex); - os_thread_sleep(10000); } @@ -3411,7 +3426,7 @@ xb_register_regex( node = static_cast<xb_regex_list_node_t *> (ut_malloc(sizeof(xb_regex_list_node_t))); - ret = xb_regcomp(&node->regex, regex, REG_EXTENDED); + ret = regcomp(&node->regex, regex, REG_EXTENDED); if (ret != 0) { xb_regerror(ret, &node->regex, errbuf, sizeof(errbuf)); msg("xtrabackup: error: tables regcomp(%s): %s\n", @@ -3545,7 +3560,7 @@ xb_filters_free() while (UT_LIST_GET_LEN(regex_list) > 0) { xb_regex_list_node_t* node = UT_LIST_GET_FIRST(regex_list); UT_LIST_REMOVE(regex_list, regex_list, node); - xb_regfree(&node->regex); + regfree(&node->regex); ut_free(node); } @@ -3602,7 +3617,7 @@ open_or_create_log_file( files[i] = os_file_create(innodb_file_log_key, name, OS_FILE_OPEN, OS_FILE_NORMAL, - OS_LOG_FILE, &ret); + OS_LOG_FILE, &ret,0); if (ret == FALSE) { fprintf(stderr, "InnoDB: Error in opening %s\n", name); @@ -3631,12 +3646,12 @@ open_or_create_log_file( which is for this log group */ fil_space_create(name, - 2 * k + SRV_LOG_SPACE_FIRST_ID, 0, FIL_LOG); + 2 * k + SRV_LOG_SPACE_FIRST_ID, 0, FIL_LOG, 0, 0); } ut_a(fil_validate()); - ut_a(fil_node_create(name, srv_log_file_size, + ut_a(fil_node_create(name, (ulint)srv_log_file_size, 2 * k + SRV_LOG_SPACE_FIRST_ID, FALSE)); if (i == 0) { log_group_init(k, srv_n_log_files, @@ -3765,7 +3780,7 @@ xtrabackup_backup_func(void) srv_read_only_mode = TRUE; srv_backup_mode = TRUE; - srv_close_files = xb_close_files; + srv_close_files = (bool)xb_close_files; if (srv_close_files) msg("xtrabackup: warning: close-files specified. Use it " @@ -3779,7 +3794,7 @@ xtrabackup_backup_func(void) xb_normalize_init_values(); -#ifndef __WIN__ + if (srv_file_flush_method_str == NULL) { /* These are the default options */ srv_unix_file_flush_method = SRV_UNIX_FSYNC; @@ -3808,13 +3823,14 @@ xtrabackup_backup_func(void) "innodb_flush_method\n", srv_file_flush_method_str); exit(EXIT_FAILURE); } -#else /* __WIN__ */ + /* We can only use synchronous unbuffered IO on Windows for now */ if (srv_file_flush_method_str != NULL) { msg("xtrabackupp: Warning: " - "ignoring innodb_flush_method = %s on Windows.\n"); + "ignoring innodb_flush_method = %s on Windows.\n", srv_file_flush_method_str); } +#ifdef _WIN32 srv_win_file_flush_method = SRV_WIN_IO_UNBUFFERED; srv_use_native_aio = FALSE; #endif @@ -3836,10 +3852,14 @@ xtrabackup_backup_func(void) computers */ } - os_sync_mutex = NULL; srv_general_init(); ut_crc32_init(); +#ifdef WITH_INNODB_DISALLOW_WRITES + srv_allow_writes_event = os_event_create(); + os_event_set(srv_allow_writes_event); +#endif + xb_filters_init(); { @@ -3901,7 +3921,7 @@ xtrabackup_backup_func(void) } /* create target dir if not exist */ - if (!my_stat(xtrabackup_target_dir,&stat_info,MYF(0)) + if (!xtrabackup_stream_str && !my_stat(xtrabackup_target_dir,&stat_info,MYF(0)) && (my_mkdir(xtrabackup_target_dir,0777,MYF(0)) < 0)){ msg("xtrabackup: Error: cannot mkdir %d: %s\n", my_errno, xtrabackup_target_dir); @@ -3950,10 +3970,10 @@ xtrabackup_backup_func(void) mutex_exit(&log_sys->mutex); reread_log_header: - fil_io(OS_FILE_READ | OS_FILE_LOG, TRUE, max_cp_group->space_id, + fil_io(OS_FILE_READ | OS_FILE_LOG, true, max_cp_group->space_id, 0, 0, 0, LOG_FILE_HDR_SIZE, - log_hdr_buf, max_cp_group); + log_hdr_buf, max_cp_group, NULL); /* check consistency of log file header to copy */ mutex_enter(&log_sys->mutex); @@ -4427,7 +4447,7 @@ loop: } static void -xtrabackup_stats_func(void) +xtrabackup_stats_func(int argc, char **argv) { ulint n; @@ -4439,7 +4459,7 @@ xtrabackup_stats_func(void) exit(EXIT_FAILURE); } msg("xtrabackup: cd to %s\n", mysql_real_data_home); - + encryption_plugin_prepare_init(argc, argv); mysql_data_home= mysql_data_home_buff; mysql_data_home[0]=FN_CURLIB; // all paths are relative from here mysql_data_home[1]=0; @@ -4654,7 +4674,7 @@ xtrabackup_init_temp_log(void) ibool success; ulint field; - byte log_buf[UNIV_PAGE_SIZE_MAX * 128]; /* 2 MB */ + byte* log_buf= (byte *)malloc(UNIV_PAGE_SIZE_MAX * 128); /* 2 MB */ ib_int64_t file_size; @@ -4668,6 +4688,10 @@ xtrabackup_init_temp_log(void) max_no = 0; + if (!log_buf) { + goto error; + } + if (!xb_init_log_block_size()) { goto error; } @@ -4688,7 +4712,7 @@ retry: src_file = os_file_create_simple_no_error_handling(0, src_path, OS_FILE_OPEN, OS_FILE_READ_WRITE, - &success); + &success,0); if (!success) { /* The following call prints an error message */ os_file_get_last_error(TRUE); @@ -4700,7 +4724,7 @@ retry: src_file = os_file_create_simple_no_error_handling(0, dst_path, OS_FILE_OPEN, OS_FILE_READ_WRITE, - &success); + &success,0); if (!success) { os_file_get_last_error(TRUE); msg(" xtrabackup: Fatal error: cannot find %s.\n", @@ -4798,7 +4822,7 @@ not_consistent: mach_write_to_4(log_buf + LOG_CHECKPOINT_1 + LOG_CHECKPOINT_OFFSET_LOW32, LOG_FILE_HDR_SIZE + - (max_lsn - + (ulint)(max_lsn - ut_uint64_align_down(max_lsn, OS_FILE_LOG_BLOCK_SIZE))); mach_write_to_4(log_buf + LOG_CHECKPOINT_1 @@ -4815,7 +4839,7 @@ not_consistent: mach_write_to_4(log_buf + LOG_CHECKPOINT_2 + LOG_CHECKPOINT_OFFSET_LOW32, LOG_FILE_HDR_SIZE + - (max_lsn - + (ulint)(max_lsn - ut_uint64_align_down(max_lsn, OS_FILE_LOG_BLOCK_SIZE))); mach_write_to_4(log_buf + LOG_CHECKPOINT_2 @@ -4916,15 +4940,17 @@ not_consistent: goto error; } xtrabackup_logfile_is_renamed = TRUE; - + free(log_buf); return(FALSE); skip_modify: + free(log_buf); os_file_close(src_file); src_file = XB_FILE_UNDEFINED; return(FALSE); error: + free(log_buf); if (src_file != XB_FILE_UNDEFINED) os_file_close(src_file); msg("xtrabackup: Error: xtrabackup_init_temp_log() failed.\n"); @@ -4977,7 +5003,7 @@ xb_space_create_file( *file = os_file_create_simple_no_error_handling(0, path, OS_FILE_CREATE, OS_FILE_READ_WRITE, - &ret); + &ret,0); if (!ret) { msg("xtrabackup: cannot create file %s\n", path); return ret; @@ -5171,7 +5197,7 @@ xb_delta_open_matching_space( /* No matching space found. create the new one. */ if (!fil_space_create(dest_space_name, space_id, 0, - FIL_TABLESPACE)) { + FIL_TABLESPACE, 0, false)) { msg("xtrabackup: Cannot create tablespace %s\n", dest_space_name); goto exit; @@ -5201,7 +5227,7 @@ found: file = os_file_create_simple_no_error_handling(0, real_name, OS_FILE_OPEN, OS_FILE_READ_WRITE, - &ok); + &ok,0); if (ok) { *success = TRUE; @@ -5290,7 +5316,7 @@ xtrabackup_apply_delta( src_file = os_file_create_simple_no_error_handling(0, src_path, OS_FILE_OPEN, OS_FILE_READ_WRITE, - &success); + &success,0); if (!success) { os_file_get_last_error(TRUE); msg("xtrabackup: error: cannot open %s\n", src_path); @@ -5315,11 +5341,11 @@ xtrabackup_apply_delta( /* allocate buffer for incremental backup (4096 pages) */ incremental_buffer_base = static_cast<byte *> - (ut_malloc((UNIV_PAGE_SIZE_MAX / 4 + 1) * - UNIV_PAGE_SIZE_MAX)); + (ut_malloc((page_size / 4 + 1) * + page_size)); incremental_buffer = static_cast<byte *> (ut_align(incremental_buffer_base, - UNIV_PAGE_SIZE_MAX)); + page_size)); msg("Applying %s to %s...\n", src_path, dst_path); @@ -5637,7 +5663,7 @@ xtrabackup_close_temp_log(my_bool clear_flag) src_file = os_file_create_simple_no_error_handling(0, src_path, OS_FILE_OPEN, OS_FILE_READ_WRITE, - &success); + &success,0); if (!success) { goto error; } @@ -5701,7 +5727,7 @@ xb_export_cfg_write_index_fields( } /* Include the NUL byte in the length. */ - ib_uint32_t len = strlen(field->name) + 1; + ib_uint32_t len = (ib_uint32_t)strlen(field->name) + 1; ut_a(len > 1); mach_write_to_4(row, len); @@ -5791,7 +5817,7 @@ xb_export_cfg_write_indexes( /* Write the length of the index name. NUL byte is included in the length. */ - ib_uint32_t len = strlen(index->name) + 1; + ib_uint32_t len = (ib_uint32_t)strlen(index->name) + 1; ut_a(len > 1); mach_write_to_4(row, len); @@ -5865,7 +5891,7 @@ xb_export_cfg_write_table( col_name = dict_table_get_col_name(table, dict_col_get_no(col)); /* Include the NUL byte in the length. */ - len = strlen(col_name) + 1; + len = (ib_uint32_t)strlen(col_name) + 1; ut_a(len > 1); mach_write_to_4(row, len); @@ -5909,7 +5935,7 @@ xb_export_cfg_write_header( const char* hostname = "Hostname unknown"; /* The server hostname includes the NUL byte. */ - len = strlen(hostname) + 1; + len = (ib_uint32_t)strlen(hostname) + 1; mach_write_to_4(value, len); if (fwrite(&value, 1, sizeof(value), file) != sizeof(value) @@ -5922,7 +5948,7 @@ xb_export_cfg_write_header( /* The table name includes the NUL byte. */ ut_a(table->name != 0); - len = strlen(table->name) + 1; + len = (ib_uint32_t)strlen(table->name) + 1; /* Write the table name. */ mach_write_to_4(value, len); @@ -6122,7 +6148,7 @@ store_binlog_info( } static void -xtrabackup_prepare_func(void) +xtrabackup_prepare_func(int argc, char ** argv) { ulint err; datafiles_iter_t *it; @@ -6140,6 +6166,8 @@ xtrabackup_prepare_func(void) } msg("xtrabackup: cd to %s\n", xtrabackup_real_target_dir); + encryption_plugin_prepare_init(argc, argv); + xtrabackup_target_dir= mysql_data_home_buff; xtrabackup_target_dir[0]=FN_CURLIB; // all paths are relative from here xtrabackup_target_dir[1]=0; @@ -6193,7 +6221,6 @@ skip_check: /* Create logfiles for recovery from 'xtrabackup_logfile', before start InnoDB */ srv_max_n_threads = 1000; - os_sync_mutex = NULL; ut_mem_init(); /* temporally dummy value to avoid crash */ srv_page_size_shift = 14; @@ -6204,6 +6231,11 @@ skip_check: mem_init(srv_mem_pool_size); ut_crc32_init(); +#ifdef WITH_INNODB_DISALLOW_WRITES + srv_allow_writes_event = os_event_create(); + os_event_set(srv_allow_writes_event); +#endif + xb_filters_init(); if(!innobase_log_arch_dir && xtrabackup_init_temp_log()) @@ -6213,25 +6245,6 @@ skip_check: goto error_cleanup; } - /* Expand compacted datafiles */ - - if (xtrabackup_compact) { - srv_compact_backup = TRUE; - - if (!xb_expand_datafiles()) { - goto error_cleanup; - } - - /* Reset the 'compact' flag in xtrabackup_checkpoints so we - don't expand on subsequent invocations. */ - xtrabackup_compact = FALSE; - if (!xtrabackup_write_metadata(metadata_path)) { - msg("xtrabackup: error: xtrabackup_write_metadata() " - "failed\n"); - goto error_cleanup; - } - } - xb_normalize_init_values(); if (xtrabackup_incremental || innobase_log_arch_dir) { @@ -6262,17 +6275,16 @@ skip_check: xb_filter_hash_free(inc_dir_tables_hash); } - sync_close(); - sync_initialized = FALSE; if (fil_system) { fil_close(); } - os_sync_free(); + mem_close(); - os_sync_mutex = NULL; ut_free_all_mem(); innodb_free_param(); + sync_close(); + sync_initialized = FALSE; /* Reset the configuration as it might have been changed by xb_data_files_init(). */ @@ -6280,8 +6292,7 @@ skip_check: goto error_cleanup; } - srv_apply_log_only = (ibool) xtrabackup_apply_log_only; - srv_rebuild_indexes = (ibool) xtrabackup_rebuild_indexes; + srv_apply_log_only = (bool) xtrabackup_apply_log_only; /* increase IO threads */ if(srv_n_file_io_threads < 10) { @@ -6291,7 +6302,7 @@ skip_check: if (innobase_log_arch_dir) { srv_arch_dir = innobase_log_arch_dir; - srv_archive_recovery = TRUE; + srv_archive_recovery = true; if (xtrabackup_archived_to_lsn) { if (xtrabackup_archived_to_lsn < metadata_last_lsn) { msg("xtrabackup: warning: logs applying lsn " @@ -6317,7 +6328,7 @@ skip_check: Unfinished transactions are not rolled back during log applying as they can be finished at the firther files applyings. */ - srv_apply_log_only = xtrabackup_apply_log_only = TRUE; + srv_apply_log_only = xtrabackup_apply_log_only = true; if (!xtrabackup_arch_search_files(min_flushed_lsn)) { goto error_cleanup; @@ -6340,6 +6351,12 @@ skip_check: "xtrabackup: Using %lld bytes for buffer pool " "(set by --use-memory parameter)\n", xtrabackup_use_memory); + srv_max_buf_pool_modified_pct = (double)max_buf_pool_modified_pct; + + if (srv_max_dirty_pages_pct_lwm > srv_max_buf_pool_modified_pct) { + srv_max_dirty_pages_pct_lwm = srv_max_buf_pool_modified_pct; + } + if(innodb_init()) goto error_cleanup; @@ -6423,15 +6440,20 @@ skip_check: /* node exist == file exist, here */ strcpy(info_file_path, node->name); +#ifdef _WIN32 + for (int i = 0; info_file_path[i]; i++) + if (info_file_path[i] == '\\') + info_file_path[i]= '/'; +#endif strcpy(info_file_path + strlen(info_file_path) - 4, ".exp"); - len = strlen(info_file_path); + len =(ib_uint32_t)strlen(info_file_path); p = info_file_path; prev = NULL; - while ((next = strchr(p, SRV_PATH_SEPARATOR)) != NULL) + while ((next = strchr(p, '/')) != NULL) { prev = p; p = next + 1; @@ -6503,7 +6525,7 @@ skip_check: info_file_path, OS_FILE_OVERWRITE, OS_FILE_NORMAL, OS_DATA_FILE, - &success); + &success,0); if (!success) { os_file_get_last_error(TRUE); goto next_node; @@ -6569,8 +6591,9 @@ next_node: } exit(EXIT_FAILURE); } - +#ifdef WITH_WSREP xb_write_galera_info(xtrabackup_incremental); +#endif if(innodb_end()) goto error_cleanup; @@ -6578,7 +6601,6 @@ next_node: innodb_free_param(); sync_initialized = FALSE; - os_sync_mutex = NULL; /* re-init necessary components */ ut_mem_init(); @@ -6630,9 +6652,7 @@ next_node: if (fil_system) { fil_close(); } - os_sync_free(); - // mem_close(); - os_sync_mutex = NULL; + ut_free_all_mem(); /* start InnoDB once again to create log files */ @@ -6643,8 +6663,7 @@ next_node: goto error; } - srv_apply_log_only = FALSE; - srv_rebuild_indexes = FALSE; + srv_apply_log_only = false; /* increase IO threads */ if(srv_n_file_io_threads < 10) { @@ -6677,40 +6696,6 @@ error: } /************************************************************************** -Signals-related setup. */ -static -void -setup_signals() -/*===========*/ -{ - struct sigaction sa; - - /* Print a stacktrace on some signals */ - sa.sa_flags = SA_RESETHAND | SA_NODEFER; - sigemptyset(&sa.sa_mask); - sigprocmask(SIG_SETMASK,&sa.sa_mask,NULL); -#ifdef HAVE_STACKTRACE - my_init_stacktrace(); -#endif - sa.sa_handler = handle_fatal_signal; - sigaction(SIGSEGV, &sa, NULL); - sigaction(SIGABRT, &sa, NULL); - sigaction(SIGBUS, &sa, NULL); - sigaction(SIGILL, &sa, NULL); - sigaction(SIGFPE, &sa, NULL); - -#ifdef __linux__ - /* Ensure xtrabackup process is killed when the parent one - (innobackupex) is terminated with an unhandled signal */ - - if (prctl(PR_SET_PDEATHSIG, SIGKILL)) { - msg("prctl() failed with errno = %d\n", errno); - exit(EXIT_FAILURE); - } -#endif -} - -/************************************************************************** Append group name to xb_load_default_groups list. */ static void @@ -6778,11 +6763,6 @@ xb_init() } if (xtrabackup_backup) { - - if (!opt_noversioncheck) { - version_check(); - } - if ((mysql_connection = xb_mysql_connect()) == NULL) { return(false); } @@ -6791,6 +6771,7 @@ xb_init() return(false); } + encryption_plugin_backup_init(mysql_connection); history_start_time = time(NULL); } @@ -6798,9 +6779,74 @@ xb_init() return(true); } + +extern void init_signals(void); + +#include <sql_locale.h> + +/* Messages . Avoid loading errmsg.sys file */ +void setup_error_messages() +{ + static const char *all_msgs[ER_ERROR_LAST - ER_ERROR_FIRST +1]; + my_default_lc_messages = &my_locale_en_US; + my_default_lc_messages->errmsgs->errmsgs = all_msgs; + + /* Populate the necessary error messages */ + struct { + int id; + const char *fmt; + } + xb_msgs[] = + { + { ER_DATABASE_NAME,"Database" }, + { ER_TABLE_NAME,"Table"}, + { ER_PARTITION_NAME, "Partition" }, + { ER_SUBPARTITION_NAME, "Subpartition" }, + { ER_TEMPORARY_NAME, "Temporary"}, + { ER_RENAMED_NAME, "Renamed"}, + { ER_CANT_FIND_DL_ENTRY, "Can't find symbol '%-.128s' in library"}, + { ER_CANT_OPEN_LIBRARY, "Can't open shared library '%-.192s' (errno: %d, %-.128s)" }, + { ER_OUTOFMEMORY, "Out of memory; restart server and try again (needed %d bytes)" }, + { ER_CANT_OPEN_LIBRARY, "Can't open shared library '%-.192s' (errno: %d, %-.128s)" }, + { ER_UDF_NO_PATHS, "No paths allowed for shared library" }, + { ER_CANT_INITIALIZE_UDF,"Can't initialize function '%-.192s'; %-.80s"}, + { ER_PLUGIN_IS_NOT_LOADED,"Plugin '%-.192s' is not loaded" } + }; + + for (int i = 0; i < (int)array_elements(all_msgs); i++) + all_msgs[i] = "Unknown error"; + + for (int i = 0; i < (int)array_elements(xb_msgs); i++) + all_msgs[xb_msgs[i].id - ER_ERROR_FIRST] = xb_msgs[i].fmt; +} + +extern my_bool(*dict_check_if_skip_table)(const char* name) ; + void handle_options(int argc, char **argv, char ***argv_client, char ***argv_server) { + /* Setup some variables for Innodb.*/ + + srv_xtrabackup = true; + + + files_charset_info = &my_charset_utf8_general_ci; + dict_check_if_skip_table = check_if_skip_table; + + setup_error_messages(); + sys_var_init(); + plugin_mutex_init(); + mysql_rwlock_init(key_rwlock_LOCK_system_variables_hash, &LOCK_system_variables_hash); + opt_stack_trace = 1; + test_flags |= TEST_SIGINT; + init_signals(); +#ifndef _WIN32 + /* Exit process on SIGINT. */ + my_sigset(SIGINT, SIG_DFL); +#endif + + sf_leaking_memory = 0; /* don't report memory leaks on early exist */ + int i; int ho_error; @@ -6811,9 +6857,6 @@ handle_options(int argc, char **argv, char ***argv_client, char ***argv_server) int argc_client = argc; int argc_server = argc; - *argv_client = argv; - *argv_server = argv; - /* scan options for group and config file to load defaults from */ for (i = 1; i < argc; i++) { @@ -6859,12 +6902,27 @@ handle_options(int argc, char **argv, char ***argv_client, char ***argv_server) if (prepare && target_dir) { snprintf(conf_file, sizeof(conf_file), "%s/backup-my.cnf", target_dir); + if (!strncmp(argv[1], "--defaults-file=", 16)) { + /* Remove defaults-file*/ + for (int i = 2; ; i++) { + if ((argv[i-1]= argv[i]) == 0) + break; + } + argc--; + } } + + *argv_client = argv; + *argv_server = argv; if (load_defaults(conf_file, xb_server_default_groups, &argc_server, argv_server)) { exit(EXIT_FAILURE); } + int n; + for (n = 0; (*argv_server)[n]; n++) {}; + argc_server = n; + print_param_str << "# This MySQL options file was generated by XtraBackup.\n" "[" << defaults_group << "]\n"; @@ -6879,6 +6937,7 @@ handle_options(int argc, char **argv, char ***argv_client, char ***argv_server) optp->u_max_value = (G_PTR *) &global_max_value; } + /* Throw a descriptive error if --defaults-file or --defaults-extra-file is not the first command line argument */ for (int i = 2 ; i < argc ; i++) { @@ -6913,6 +6972,9 @@ handle_options(int argc, char **argv, char ***argv_client, char ***argv_server) exit(EXIT_FAILURE); } + for (n = 0; (*argv_client)[n]; n++) {}; + argc_client = n; + if (strcmp(base_name(my_progname), INNOBACKUPEX_BIN_NAME) == 0 && argc_client > 0) { /* emulate innobackupex script */ @@ -6959,10 +7021,16 @@ int main(int argc, char **argv) { char **client_defaults, **server_defaults; char cwd[FN_REFLEN]; - my_bool is_symdir; - setup_signals(); + if (argc > 1 && (strcmp(argv[1], "--innobackupex") == 0)) + { + argv++; + argc--; + argv[0] = "innobackupex"; + innobackupex_mode = true; + } + init_signals(); MY_INIT(argv[0]); pthread_key_create(&THR_THD, NULL); @@ -6977,11 +7045,18 @@ int main(int argc, char **argv) exit(EXIT_FAILURE); } - system_charset_info= &my_charset_utf8_general_ci; + system_charset_info = &my_charset_utf8_general_ci; key_map_full.set_all(); handle_options(argc, argv, &client_defaults, &server_defaults); + int argc_server; + for (argc_server = 0; server_defaults[argc_server]; argc_server++) {} + + int argc_client; + for (argc_client = 0; client_defaults[argc_client]; argc_client++) {} + + if (innobackupex_mode) { if (!ibx_init()) { exit(EXIT_FAILURE); @@ -7002,14 +7077,14 @@ int main(int argc, char **argv) my_load_path(xtrabackup_real_target_dir, xtrabackup_target_dir, cwd); unpack_dirname(xtrabackup_real_target_dir, - xtrabackup_real_target_dir, &is_symdir); + xtrabackup_real_target_dir); xtrabackup_target_dir= xtrabackup_real_target_dir; if (xtrabackup_incremental_basedir) { my_load_path(xtrabackup_real_incremental_basedir, xtrabackup_incremental_basedir, cwd); unpack_dirname(xtrabackup_real_incremental_basedir, - xtrabackup_real_incremental_basedir, &is_symdir); + xtrabackup_real_incremental_basedir); xtrabackup_incremental_basedir = xtrabackup_real_incremental_basedir; } @@ -7018,7 +7093,7 @@ int main(int argc, char **argv) my_load_path(xtrabackup_real_incremental_dir, xtrabackup_incremental_dir, cwd); unpack_dirname(xtrabackup_real_incremental_dir, - xtrabackup_real_incremental_dir, &is_symdir); + xtrabackup_real_incremental_dir); xtrabackup_incremental_dir = xtrabackup_real_incremental_dir; } @@ -7026,7 +7101,7 @@ int main(int argc, char **argv) my_load_path(xtrabackup_real_extra_lsndir, xtrabackup_extra_lsndir, cwd); unpack_dirname(xtrabackup_real_extra_lsndir, - xtrabackup_real_extra_lsndir, &is_symdir); + xtrabackup_real_extra_lsndir); xtrabackup_extra_lsndir = xtrabackup_real_extra_lsndir; } @@ -7181,11 +7256,12 @@ int main(int argc, char **argv) /* --stats */ if (xtrabackup_stats) - xtrabackup_stats_func(); + xtrabackup_stats_func(argc_server,server_defaults); /* --prepare */ - if (xtrabackup_prepare) - xtrabackup_prepare_func(); + if (xtrabackup_prepare) { + xtrabackup_prepare_func(argc_server, server_defaults); + } if (xtrabackup_copy_back || xtrabackup_move_back) { if (!check_if_param_set("datadir")) { @@ -7206,7 +7282,6 @@ int main(int argc, char **argv) ibx_cleanup(); } - xb_regex_end(); free_defaults(client_defaults); free_defaults(server_defaults); diff --git a/extra/mariabackup/xtrabackup.h b/extra/mariabackup/xtrabackup.h index b31028175c9..de21da31741 100644 --- a/extra/mariabackup/xtrabackup.h +++ b/extra/mariabackup/xtrabackup.h @@ -63,8 +63,6 @@ extern lsn_t checkpoint_lsn_start; extern xb_page_bitmap *changed_page_bitmap; -extern ulint xtrabackup_rebuild_threads; - extern char *xtrabackup_incremental; extern my_bool xtrabackup_incremental_force_scan; @@ -80,7 +78,6 @@ extern char *xtrabackup_tables_file; extern char *xtrabackup_databases; extern char *xtrabackup_databases_file; -extern my_bool xtrabackup_compact; extern ibool xtrabackup_compress; extern ibool xtrabackup_encrypt; @@ -106,8 +103,14 @@ extern int xtrabackup_parallel; extern my_bool xb_close_files; extern const char *xtrabackup_compress_alg; -extern uint xtrabackup_compress_threads; -extern ulonglong xtrabackup_compress_chunk_size; +#ifdef __cplusplus +extern "C"{ +#endif + extern uint xtrabackup_compress_threads; + extern ulonglong xtrabackup_compress_chunk_size; +#ifdef __cplusplus +} +#endif extern ulong xtrabackup_encrypt_algo; extern uint xtrabackup_encrypt_threads; extern ulonglong xtrabackup_encrypt_chunk_size; @@ -116,7 +119,6 @@ extern char *xtrabackup_incremental_basedir; extern char *xtrabackup_extra_lsndir; extern char *xtrabackup_incremental_dir; extern ulint xtrabackup_log_copy_interval; -extern my_bool xtrabackup_rebuild_indexes; extern char *xtrabackup_stream_str; extern long xtrabackup_throttle; extern longlong xtrabackup_use_memory; @@ -165,14 +167,6 @@ extern uint opt_safe_slave_backup_timeout; extern const char *opt_history; extern my_bool opt_decrypt; -#if defined(HAVE_OPENSSL) -extern my_bool opt_use_ssl; -extern my_bool opt_ssl_verify_server_cert; -#if !defined(HAVE_YASSL) -extern char *opt_server_public_key; -#endif -#endif - enum binlog_info_enum { BINLOG_INFO_OFF, BINLOG_INFO_LOCKLESS, BINLOG_INFO_ON, BINLOG_INFO_AUTO}; @@ -217,6 +211,14 @@ Check if parameter is set in defaults file or via command line argument bool check_if_param_set(const char *param); +#if defined(HAVE_OPENSSL) +extern my_bool opt_use_ssl; +extern my_bool opt_ssl_verify_server_cert; +#if !defined(HAVE_YASSL) +extern char *opt_server_public_key; +#endif +#endif + void xtrabackup_backup_func(void); diff --git a/extra/mariabackup/xtrabackup_version.h.in b/extra/mariabackup/xtrabackup_version.h.in deleted file mode 100644 index dc4c7992f8f..00000000000 --- a/extra/mariabackup/xtrabackup_version.h.in +++ /dev/null @@ -1,27 +0,0 @@ -/****************************************************** -Copyright (c) 2013 Percona LLC and/or its affiliates. - -Version numbers definitions. - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -*******************************************************/ - -#ifndef XB_VERSION_H -#define XB_VERSION_H - -#define XTRABACKUP_VERSION "@XB_VERSION@" -#define XTRABACKUP_REVISION "@XB_REVISION@" - -#endif /* XB_VERSION_H */ diff --git a/storage/xtradb/CMakeLists.txt b/storage/xtradb/CMakeLists.txt index 8d1bd543d0c..77d8ca7fdbc 100644 --- a/storage/xtradb/CMakeLists.txt +++ b/storage/xtradb/CMakeLists.txt @@ -506,3 +506,6 @@ MYSQL_ADD_PLUGIN(xtradb ${INNOBASE_SOURCES} STORAGE_ENGINE IF(TARGET xtradb AND NOT XTRADB_OK) MESSAGE(FATAL_ERROR "Percona XtraDB is not supported on this platform") ENDIF() + +ADD_SUBDIRECTORY(${CMAKE_SOURCE_DIR}/extra/mariabackup ${CMAKE_BINARY_DIR}/extra/mariabackup) + |