diff options
344 files changed, 13090 insertions, 5710 deletions
diff --git a/.gitignore b/.gitignore index 433d14db7e9..191dbe23fad 100644 --- a/.gitignore +++ b/.gitignore @@ -47,6 +47,7 @@ extra/jemalloc/build/ extra/jemalloc/tmp/ extra/my_print_defaults extra/mysql_waitpid +extra/mysqld_safe_helper extra/perror extra/replace extra/resolve_stack_dump @@ -213,6 +214,7 @@ support-files/mysql.spec support-files/mysqld_multi.server support-files/wsrep.cnf support-files/wsrep_notify +support-files/SELinux/centos6-mariadb.pp tags tests/async_queries tests/bug25714 diff --git a/CMakeLists.txt b/CMakeLists.txt index 2b67eb46c54..9d71539f5cd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -311,7 +311,7 @@ ENDIF() # Set commonly used variables IF(WIN32) - SET(DEFAULT_MYSQL_HOME "C:/MariaDB${MYSQL_BASE_VERSION}") + SET(DEFAULT_MYSQL_HOME "C:/Program Files/MariaDB ${MYSQL_BASE_VERSION}") SET(SHAREDIR share) ELSE() SET(DEFAULT_MYSQL_HOME ${CMAKE_INSTALL_PREFIX}) diff --git a/client/mysqladmin.cc b/client/mysqladmin.cc index 56d2784a1f5..7f32658f1b2 100644 --- a/client/mysqladmin.cc +++ b/client/mysqladmin.cc @@ -1567,8 +1567,10 @@ static my_bool get_pidfile(MYSQL *mysql, char *pidfile) if (mysql_query(mysql, "SHOW VARIABLES LIKE 'pid_file'")) { - my_printf_error(0, "query failed; error: '%s'", error_flags, - mysql_error(mysql)); + my_printf_error(mysql_errno(mysql), + "The query to get the server's pid file failed," + " error: '%s'. Continuing.", error_flags, + mysql_error(mysql)); } result = mysql_store_result(mysql); if (result) diff --git a/client/mysqltest.cc b/client/mysqltest.cc index dede6527d11..15db5759ac9 100644 --- a/client/mysqltest.cc +++ b/client/mysqltest.cc @@ -1729,11 +1729,11 @@ int cat_file(DYNAMIC_STRING* ds, const char* filename) while((len= my_read(fd, (uchar*)&buff, sizeof(buff)-1, MYF(0))) > 0) { - char *p= buff, *start= buff; - while (p < buff+len) + char *p= buff, *start= buff,*end=buff+len; + while (p < end) { /* Convert cr/lf to lf */ - if (*p == '\r' && *(p+1) && *(p+1)== '\n') + if (*p == '\r' && p+1 < end && *(p+1)== '\n') { /* Add fake newline instead of cr and output the line */ *p= '\n'; @@ -3391,16 +3391,32 @@ void do_exec(struct st_command *command) ds_result= &ds_sorted; } +#ifdef _WIN32 + /* Workaround for CRT bug, MDEV-9409 */ + _setmode(fileno(res_file), O_BINARY); +#endif + while (fgets(buf, sizeof(buf), res_file)) { + int len = (int)strlen(buf); +#ifdef _WIN32 + /* Strip '\r' off newlines. */ + if (len > 1 && buf[len-2] == '\r' && buf[len-1] == '\n') + { + buf[len-2] = '\n'; + buf[len-1] = 0; + len--; + } +#endif if (disable_result_log) { - buf[strlen(buf)-1]=0; + if (len) + buf[len-1] = 0; DBUG_PRINT("exec_result",("%s", buf)); } else { - replace_dynstr_append(ds_result, buf); + replace_dynstr_append_mem(ds_result, buf, len); } } error= pclose(res_file); @@ -5929,6 +5945,7 @@ void do_connect(struct st_command *command) my_bool con_shm __attribute__ ((unused))= 0; int read_timeout= 0; int write_timeout= 0; + int connect_timeout= 0; struct st_connection* con_slot; static DYNAMIC_STRING ds_connection_name; @@ -6035,6 +6052,11 @@ void do_connect(struct st_command *command) { write_timeout= atoi(con_options + sizeof("write_timeout=")-1); } + else if (strncasecmp(con_options, "connect_timeout=", + sizeof("connect_timeout=")-1) == 0) + { + connect_timeout= atoi(con_options + sizeof("connect_timeout=")-1); + } else die("Illegal option to connect: %.*s", (int) (end - con_options), con_options); @@ -6119,6 +6141,12 @@ void do_connect(struct st_command *command) (char*)&write_timeout); } + if (connect_timeout) + { + mysql_options(con_slot->mysql, MYSQL_OPT_CONNECT_TIMEOUT, + (char*)&connect_timeout); + } + #ifdef HAVE_SMEM if (con_shm) { diff --git a/cmake/install_macros.cmake b/cmake/install_macros.cmake index e16467619c7..24659781b26 100644 --- a/cmake/install_macros.cmake +++ b/cmake/install_macros.cmake @@ -233,6 +233,7 @@ IF(WIN32) FIND_PROGRAM(SIGNTOOL_EXECUTABLE signtool PATHS "$ENV{ProgramFiles}/Microsoft SDKs/Windows/v7.0A/bin" "$ENV{ProgramFiles}/Windows Kits/8.0/bin/x86" + "$ENV{ProgramFiles}/Windows Kits/8.1/bin/x86" ) IF(NOT SIGNTOOL_EXECUTABLE) MESSAGE(FATAL_ERROR diff --git a/cmake/os/Windows.cmake b/cmake/os/Windows.cmake index 1ad7e5c288d..84276115723 100644 --- a/cmake/os/Windows.cmake +++ b/cmake/os/Windows.cmake @@ -116,7 +116,7 @@ IF(MSVC) #TODO: update the code and remove the disabled warnings SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4800 /wd4805 /wd4996") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4800 /wd4805 /wd4996 /wd4291 /we4099") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4800 /wd4805 /wd4996 /wd4291 /wd4577 /we4099") IF(CMAKE_SIZEOF_VOID_P MATCHES 8) # _WIN64 is defined by the compiler itself. diff --git a/cmake/readline.cmake b/cmake/readline.cmake index c72f8d0da5c..64e04ef8663 100644 --- a/cmake/readline.cmake +++ b/cmake/readline.cmake @@ -230,5 +230,6 @@ MACRO (MYSQL_CHECK_READLINE) SET(CMAKE_REQUIRED_LIBRARIES) SET(CMAKE_REQUIRED_INCLUDES) ENDIF(NOT WIN32) + CHECK_INCLUDE_FILES ("curses.h;term.h" HAVE_TERM_H) ENDMACRO() diff --git a/configure.cmake b/configure.cmake index 6f0ee92f774..63b227248b2 100644 --- a/configure.cmake +++ b/configure.cmake @@ -228,7 +228,6 @@ CHECK_INCLUDE_FILES (sys/socket.h HAVE_SYS_SOCKET_H) CHECK_INCLUDE_FILES (sys/stat.h HAVE_SYS_STAT_H) CHECK_INCLUDE_FILES (sys/stream.h HAVE_SYS_STREAM_H) CHECK_INCLUDE_FILES (sys/termcap.h HAVE_SYS_TERMCAP_H) -CHECK_INCLUDE_FILES ("curses.h;term.h" HAVE_TERM_H) CHECK_INCLUDE_FILES (asm/termbits.h HAVE_ASM_TERMBITS_H) CHECK_INCLUDE_FILES (termbits.h HAVE_TERMBITS_H) CHECK_INCLUDE_FILES (termios.h HAVE_TERMIOS_H) diff --git a/debian/control b/debian/control index 4b252f95eb0..f24aabaaf94 100644 --- a/debian/control +++ b/debian/control @@ -9,7 +9,6 @@ Build-Depends: bison, debhelper, dpatch, gawk, - hardening-wrapper, libjemalloc-dev (>= 3.0.0), libncurses5-dev (>= 5.0-6), libpam0g-dev, diff --git a/debian/dist/Debian/mariadb-galera-server-10.0.files.in b/debian/dist/Debian/mariadb-galera-server-10.0.files.in index bf1b04b5787..433febd0f86 100644 --- a/debian/dist/Debian/mariadb-galera-server-10.0.files.in +++ b/debian/dist/Debian/mariadb-galera-server-10.0.files.in @@ -39,6 +39,7 @@ usr/bin/mysql_zap usr/bin/mysqlbinlog usr/bin/mysqld_multi usr/bin/mysqld_safe +usr/bin/mysqld_safe_helper usr/bin/mysqlhotcopy usr/bin/perror usr/bin/replace diff --git a/debian/dist/Debian/rules b/debian/dist/Debian/rules index a0923c83b54..e050fd7cc95 100755 --- a/debian/dist/Debian/rules +++ b/debian/dist/Debian/rules @@ -1,7 +1,6 @@ #!/usr/bin/make -f export DH_VERBOSE=1 -export DEB_BUILD_HARDENING=1 PACKAGE=mariadb-10.0 diff --git a/debian/dist/Ubuntu/mariadb-galera-server-10.0.files.in b/debian/dist/Ubuntu/mariadb-galera-server-10.0.files.in index 368400c68b8..7fa0dd05cfb 100644 --- a/debian/dist/Ubuntu/mariadb-galera-server-10.0.files.in +++ b/debian/dist/Ubuntu/mariadb-galera-server-10.0.files.in @@ -41,6 +41,7 @@ usr/bin/mysql_zap usr/bin/mysqlbinlog usr/bin/mysqld_multi usr/bin/mysqld_safe +usr/bin/mysqld_safe_helper usr/bin/mysqlhotcopy usr/bin/perror usr/bin/replace diff --git a/debian/dist/Ubuntu/rules b/debian/dist/Ubuntu/rules index ae89db84f48..76b9dc93fb6 100755 --- a/debian/dist/Ubuntu/rules +++ b/debian/dist/Ubuntu/rules @@ -1,7 +1,6 @@ #!/usr/bin/make -f export DH_VERBOSE=1 -export DEB_BUILD_HARDENING=1 PACKAGE=mariadb-10.0 diff --git a/extra/CMakeLists.txt b/extra/CMakeLists.txt index 585b5aef6f6..f2738752c56 100644 --- a/extra/CMakeLists.txt +++ b/extra/CMakeLists.txt @@ -80,6 +80,9 @@ IF(UNIX) MYSQL_ADD_EXECUTABLE(mysql_waitpid mysql_waitpid.c COMPONENT Client) TARGET_LINK_LIBRARIES(mysql_waitpid mysys) + + MYSQL_ADD_EXECUTABLE(mysqld_safe_helper mysqld_safe_helper.c COMPONENT Server) + TARGET_LINK_LIBRARIES(mysqld_safe_helper mysys) ENDIF() diff --git a/extra/mysqld_safe_helper.c b/extra/mysqld_safe_helper.c new file mode 100644 index 00000000000..09e507c6e1c --- /dev/null +++ b/extra/mysqld_safe_helper.c @@ -0,0 +1,77 @@ +#include <my_global.h> +#include <m_string.h> +#include <my_sys.h> +#include <my_pthread.h> +#ifdef HAVE_PWD_H +#include <pwd.h> +#endif +#include <stdlib.h> +#include <stdio.h> + +void my_exit(int c) +{ + my_end(0); + exit(c); +} + +void do_usage() +{ + printf("Usage:\n" + " %s <user> log <filename>\n" + " %s <user> exec <command> <args>\n", + my_progname, my_progname); + my_exit(1); +} + +void do_log(const char *logfile) +{ + FILE *f; + uchar buf[4096]; + int size; + + if (!logfile) + do_usage(); + + f= my_fopen(logfile, O_WRONLY|O_APPEND|O_CREAT, MYF(MY_WME)); + if (!f) + my_exit(1); + + while ((size= my_fread(stdin, buf, sizeof(buf), MYF(MY_WME))) > 0) + if ((int)my_fwrite(f, buf, size, MYF(MY_WME)) != size) + my_exit(1); + + my_fclose(f, MYF(0)); + my_exit(0); +} + +void do_exec(char *args[]) +{ + if (!args[0]) + do_usage(); + + my_end(0); + execvp(args[0], args); +} + +int main(int argc, char *argv[]) +{ + struct passwd *user_info; + MY_INIT(argv[0]); + + if (argc < 3) + do_usage(argv[0]); + + user_info= my_check_user(argv[1], MYF(0)); + if (user_info ? my_set_user(argv[1], user_info, MYF(MY_WME)) + : my_errno == EINVAL) + my_exit(1); + + if (strcmp(argv[2], "log") == 0) + do_log(argv[3]); + + if (strcmp(argv[2], "exec") == 0) + do_exec(argv+3); + + my_end(0); + return 1; +} diff --git a/include/m_ctype.h b/include/m_ctype.h index 5994816cbfc..8d9838fdde2 100644 --- a/include/m_ctype.h +++ b/include/m_ctype.h @@ -180,6 +180,9 @@ extern MY_UNI_CTYPE my_uni_ctype[256]; /* A helper macros for "need at least n bytes" */ #define MY_CS_TOOSMALLN(n) (-100-(n)) +#define MY_CS_IS_TOOSMALL(rc) ((rc) >= MY_CS_TOOSMALL6 && (rc) <= MY_CS_TOOSMALL) + + #define MY_SEQ_INTTAIL 1 #define MY_SEQ_SPACES 2 diff --git a/include/my_sys.h b/include/my_sys.h index f6bf57e50a4..5e1f6baac77 100644 --- a/include/my_sys.h +++ b/include/my_sys.h @@ -620,8 +620,12 @@ extern void *my_memmem(const void *haystack, size_t haystacklen, #ifdef _WIN32 extern int my_access(const char *path, int amode); +#define my_check_user(A,B) (NULL) +#define my_set_user(A,B,C) (0) #else #define my_access access +struct passwd *my_check_user(const char *user, myf MyFlags); +int my_set_user(const char *user, struct passwd *user_info, myf MyFlags); #endif extern int check_if_legal_filename(const char *path); diff --git a/man/mysql_secure_installation.1 b/man/mysql_secure_installation.1 index 20a77bc3f1e..2a04e5d0b98 100644 --- a/man/mysql_secure_installation.1 +++ b/man/mysql_secure_installation.1 @@ -1,6 +1,6 @@ '\" t .\" -.TH "\FBMYSQL_SECURE_INST" "1" "04/08/2015" "MariaDB 10\&.0" "MariaDB Database System" +.TH "\FBMYSQL_SECURE_INST" "1" "3 January 2017" "MariaDB 10\&.0" "MariaDB Database System" .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- @@ -71,9 +71,8 @@ test database, which by default can be accessed by anonymous users\&. .RE .PP -Invoke \fBmysql_secure_installation\fR -without arguments: +can be invoked without arguments: .sp .if n \{\ .RS 4 @@ -86,10 +85,75 @@ shell> \fBmysql_secure_installation\fR .\} .PP The script will prompt you to determine which actions to perform\&. +.PP +\fBmysql_secure_installation\fR +accepts some options: +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +.\" mysql_secure_installation: basedir option +.\" basedir option: mysql_secure_installation +\fB\-\-basedir=\fR\fB\fIdir_name\fR\fR +.sp +Base directory\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +.\" mysql_secure_installation: defaults-extra-file option +.\" defaults-extra-file option: mysql_secure_installation +\fB\-\-defaults\-extra\-file=\fR\fB\fIfile_name\fR\fR +.sp +Additional option file\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +.\" mysql_secure_installation: defaults-file option +.\" defaults-file option: mysql_secure_installation +\fB\-\-defaults\-file=\fR\fB\fIfile_name\fR\fR +.sp +Option file\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +.\" mysql_secure_installation: no-defaults option +.\" no-defaults option: mysql_secure_installation +\fB\-\-no\-defaults\fR +.sp +Don't read any defaults file\&. +.RE +.sp +Other unrecognized options will be passed on to the server\&. .SH "COPYRIGHT" .br .PP -Copyright 2007-2008 MySQL AB, 2008-2010 Sun Microsystems, Inc., 2010-2015 MariaDB Foundation +Copyright 2007-2008 MySQL AB, 2008-2010 Sun Microsystems, Inc., 2010-2017 MariaDB Foundation .PP This documentation is free software; you can redistribute it and/or modify it only under the terms of the GNU General Public License as published by the Free Software Foundation; version 2 of the License. .PP diff --git a/man/mysqldump.1 b/man/mysqldump.1 index be5b855e3fa..6495619f4e0 100644 --- a/man/mysqldump.1 +++ b/man/mysqldump.1 @@ -753,6 +753,29 @@ suppresses date printing\& .sp -1 .IP \(bu 2.3 .\} +.\" mysqldump: dump-slave option +.\" dump-slave option: mysqldump +\fB\-\-dump\-slave[=\fR\fB\fIvalue\fR\fR\fB]\fR +.sp +Used for producing a dump file from a replication slave server that can be used to set up another slave server +with the same master\&. Causes the binary log position and filename of the master to be appended to the dumped +data output\&. Setting the value to 1 (the default) will print it as a CHANGE MASTER command in the dumped data +output; if set to 2, that command will be prefixed with a comment symbol\&. This option will turn +\-\-lock\-all\-tables on, unless \-\-single-transaction is specified too (in which case a global read lock is only +taken a short time at the beginning of the dump \- don't forget to read about \-\-single-transaction below)\&. In +all cases any action on logs will happen at the exact moment of the dump\&. Option automatically turns +\-\-lock\-tables off\&. Using this option causes mysqldump to stop the slave SQL thread before beginning the dump, +and restart it again after completion\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} .\" mysqldump: events option .\" events option: mysqldump \fB\-\-events\fR, diff --git a/mysql-test/extra/table_index_statistics.inc b/mysql-test/extra/table_index_statistics.inc new file mode 100644 index 00000000000..ba585320fc9 --- /dev/null +++ b/mysql-test/extra/table_index_statistics.inc @@ -0,0 +1,59 @@ +# include file to test index and table statstics for specific storage engine +# requires includer set the default strorage engine for the session + +# Bug 602047 (wrong rows_read value) + +FLUSH INDEX_STATISTICS; +FLUSH TABLE_STATISTICS; + +SET @userstat_old= @@userstat; +SET GLOBAL userstat=ON; + +CREATE TABLE t1 (id int(10), PRIMARY KEY (id)); +INSERT INTO t1 VALUES (1),(2),(3),(4),(5),(6),(7),(8),(9),(10); +SELECT COUNT(*) FROM t1; +SELECT ROWS_READ FROM INFORMATION_SCHEMA.TABLE_STATISTICS WHERE TABLE_NAME='t1'; +SELECT ROWS_READ FROM INFORMATION_SCHEMA.INDEX_STATISTICS WHERE TABLE_NAME='t1'; + +# Test that FLUSH clears one table but not another + +FLUSH TABLE_STATISTICS; + +SELECT ROWS_READ FROM INFORMATION_SCHEMA.TABLE_STATISTICS WHERE TABLE_NAME='t1'; +SELECT ROWS_READ FROM INFORMATION_SCHEMA.INDEX_STATISTICS WHERE TABLE_NAME='t1'; + +# Test that FLUSH clears both tables now + +FLUSH INDEX_STATISTICS; + +SELECT ROWS_READ FROM INFORMATION_SCHEMA.INDEX_STATISTICS WHERE TABLE_NAME='t1'; + +# Test that stats are collected after the FLUSH again + +SELECT COUNT(*) FROM t1; +SELECT ROWS_READ FROM INFORMATION_SCHEMA.TABLE_STATISTICS WHERE TABLE_NAME='t1'; +SELECT ROWS_READ FROM INFORMATION_SCHEMA.INDEX_STATISTICS WHERE TABLE_NAME='t1'; + +DROP TABLE t1; + +# Bug 1183625 (handler::update_global_table_stats crash). + +CREATE TABLE t2 (c1 INT UNSIGNED); + +ALTER TABLE t2 MODIFY c1 FLOAT; + +SELECT * FROM INFORMATION_SCHEMA.TABLE_STATISTICS WHERE TABLE_NAME='t2'; + +DROP TABLE t2; + +# Bug 1183625 (handler::update_global_table_stats crash). + +CREATE TABLE t2 (c1 INT UNSIGNED); + +ALTER TABLE t2 MODIFY c1 FLOAT; + +SELECT * FROM INFORMATION_SCHEMA.TABLE_STATISTICS WHERE TABLE_NAME='t2'; + +DROP TABLE t2; + +SET GLOBAL userstat= @userstat_old; diff --git a/mysql-test/include/sync_slave_sql_with_io.inc b/mysql-test/include/sync_slave_sql_with_io.inc index 8048f7a177c..9efede9a61f 100644 --- a/mysql-test/include/sync_slave_sql_with_io.inc +++ b/mysql-test/include/sync_slave_sql_with_io.inc @@ -26,6 +26,10 @@ let $_slave_timeout= $slave_timeout; if (!$_slave_timeout) { let $_slave_timeout= 300; + if ($VALGRIND_TEST) + { + let $_slave_timeout= 1500; + } } --let $_master_log_file= query_get_value(SHOW SLAVE STATUS, Master_Log_File, 1) diff --git a/mysql-test/include/sync_with_master_gtid.inc b/mysql-test/include/sync_with_master_gtid.inc index 97ada8eea29..777711b979c 100644 --- a/mysql-test/include/sync_with_master_gtid.inc +++ b/mysql-test/include/sync_with_master_gtid.inc @@ -34,6 +34,10 @@ let $_slave_timeout= $slave_timeout; if (!$_slave_timeout) { let $_slave_timeout= 120; + if ($VALGRIND_TEST) + { + let $_slave_timeout= 1200; + } } --let $_result= `SELECT master_gtid_wait('$master_pos', $_slave_timeout)` diff --git a/mysql-test/include/wait_for_slave_param.inc b/mysql-test/include/wait_for_slave_param.inc index d3f7ec56614..25020d46ed9 100644 --- a/mysql-test/include/wait_for_slave_param.inc +++ b/mysql-test/include/wait_for_slave_param.inc @@ -50,6 +50,10 @@ let $_slave_timeout= $slave_timeout; if (!$_slave_timeout) { let $_slave_timeout= 300; + if ($VALGRIND_TEST) + { + let $_slave_timeout= 1500; + } } if ($slave_error_param == '') diff --git a/mysql-test/r/alter_table.result b/mysql-test/r/alter_table.result index 2e371ac6ae6..72e81895816 100644 --- a/mysql-test/r/alter_table.result +++ b/mysql-test/r/alter_table.result @@ -1535,17 +1535,17 @@ ALTER TABLE t1 ADD INDEX i2(b), ALGORITHM= DEFAULT; affected rows: 0 info: Records: 0 Duplicates: 0 Warnings: 1 Warnings: -Note 1831 Duplicate index 'i2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i2`. This is deprecated and will be disallowed in a future release. ALTER TABLE t1 ADD INDEX i3(b), ALGORITHM= COPY; affected rows: 2 info: Records: 2 Duplicates: 0 Warnings: 1 Warnings: -Note 1831 Duplicate index 'i3' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i3`. This is deprecated and will be disallowed in a future release. ALTER TABLE t1 ADD INDEX i4(b), ALGORITHM= INPLACE; affected rows: 0 info: Records: 0 Duplicates: 0 Warnings: 1 Warnings: -Note 1831 Duplicate index 'i4' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i4`. This is deprecated and will be disallowed in a future release. ALTER TABLE t1 ADD INDEX i5(b), ALGORITHM= INVALID; ERROR HY000: Unknown ALGORITHM 'INVALID' ALTER TABLE m1 ENABLE KEYS; @@ -1570,17 +1570,17 @@ ALTER TABLE t1 ADD INDEX i2(b), ALGORITHM= DEFAULT; affected rows: 2 info: Records: 2 Duplicates: 0 Warnings: 1 Warnings: -Note 1831 Duplicate index 'i2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i2`. This is deprecated and will be disallowed in a future release. ALTER TABLE t1 ADD INDEX i3(b), ALGORITHM= COPY; affected rows: 2 info: Records: 2 Duplicates: 0 Warnings: 1 Warnings: -Note 1831 Duplicate index 'i3' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i3`. This is deprecated and will be disallowed in a future release. ALTER TABLE t1 ADD INDEX i4(b), ALGORITHM= INPLACE; affected rows: 0 info: Records: 0 Duplicates: 0 Warnings: 1 Warnings: -Note 1831 Duplicate index 'i4' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i4`. This is deprecated and will be disallowed in a future release. SET SESSION old_alter_table= 0; affected rows: 0 ALTER TABLE t1 DROP INDEX i1, DROP INDEX i2, DROP INDEX i3, DROP INDEX i4; @@ -1602,17 +1602,17 @@ ALTER TABLE t1 ADD INDEX i2(b), LOCK= NONE; affected rows: 0 info: Records: 0 Duplicates: 0 Warnings: 1 Warnings: -Note 1831 Duplicate index 'i2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i2`. This is deprecated and will be disallowed in a future release. ALTER TABLE t1 ADD INDEX i3(b), LOCK= SHARED; affected rows: 0 info: Records: 0 Duplicates: 0 Warnings: 1 Warnings: -Note 1831 Duplicate index 'i3' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i3`. This is deprecated and will be disallowed in a future release. ALTER TABLE t1 ADD INDEX i4(b), LOCK= EXCLUSIVE; affected rows: 0 info: Records: 0 Duplicates: 0 Warnings: 1 Warnings: -Note 1831 Duplicate index 'i4' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i4`. This is deprecated and will be disallowed in a future release. ALTER TABLE t1 ADD INDEX i5(b), LOCK= INVALID; ERROR HY000: Unknown LOCK type 'INVALID' ALTER TABLE m1 ENABLE KEYS, LOCK= DEFAULT; @@ -1632,24 +1632,24 @@ ALTER TABLE t1 ADD INDEX i2(b), ALGORITHM= INPLACE, LOCK= SHARED; affected rows: 0 info: Records: 0 Duplicates: 0 Warnings: 1 Warnings: -Note 1831 Duplicate index 'i2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i2`. This is deprecated and will be disallowed in a future release. ALTER TABLE t1 ADD INDEX i3(b), ALGORITHM= INPLACE, LOCK= EXCLUSIVE; affected rows: 0 info: Records: 0 Duplicates: 0 Warnings: 1 Warnings: -Note 1831 Duplicate index 'i3' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i3`. This is deprecated and will be disallowed in a future release. ALTER TABLE t1 ADD INDEX i4(b), ALGORITHM= COPY, LOCK= NONE; ERROR 0A000: LOCK=NONE is not supported. Reason: COPY algorithm requires a lock. Try LOCK=SHARED. ALTER TABLE t1 ADD INDEX i5(b), ALGORITHM= COPY, LOCK= SHARED; affected rows: 2 info: Records: 2 Duplicates: 0 Warnings: 1 Warnings: -Note 1831 Duplicate index 'i5' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i5`. This is deprecated and will be disallowed in a future release. ALTER TABLE t1 ADD INDEX i6(b), ALGORITHM= COPY, LOCK= EXCLUSIVE; affected rows: 2 info: Records: 2 Duplicates: 0 Warnings: 1 Warnings: -Note 1831 Duplicate index 'i6' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i6`. This is deprecated and will be disallowed in a future release. ALTER TABLE m1 ENABLE KEYS, ALGORITHM= INPLACE, LOCK= NONE; ERROR 0A000: LOCK=NONE/SHARED is not supported for this operation. Try LOCK=EXCLUSIVE. ALTER TABLE m1 ENABLE KEYS, ALGORITHM= INPLACE, LOCK= SHARED; @@ -2076,3 +2076,18 @@ tab1 CREATE TABLE `tab1` ( PRIMARY KEY (`id`) ) ENGINE=MyISAM DEFAULT CHARSET=latin1 DROP TABLE `tab1`; +# +# MDEV-11548 Reproducible server crash after the 2nd ALTER TABLE ADD FOREIGN KEY IF NOT EXISTS +# +CREATE TABLE t1 (id INT UNSIGNED NOT NULL PRIMARY KEY); +CREATE TABLE t2 (id1 INT UNSIGNED NOT NULL); +ALTER TABLE t2 +ADD FOREIGN KEY IF NOT EXISTS (id1) +REFERENCES t1 (id); +ALTER TABLE t2 +ADD FOREIGN KEY IF NOT EXISTS (id1) +REFERENCES t1 (id); +Warnings: +Note 1061 Duplicate key name 'id1' +DROP TABLE t2; +DROP TABLE t1; diff --git a/mysql-test/r/check.result b/mysql-test/r/check.result index 0ac42272c4f..715fe032b53 100644 --- a/mysql-test/r/check.result +++ b/mysql-test/r/check.result @@ -2,9 +2,9 @@ drop table if exists t1,t2; drop view if exists v1; create table t1(n int not null, key(n), key(n), key(n), key(n)); Warnings: -Note 1831 Duplicate index 'n_2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'n_3' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'n_4' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `n_2`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `n_3`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `n_4`. This is deprecated and will be disallowed in a future release. check table t1 extended; insert into t1 values (200000); Table Op Msg_type Msg_text diff --git a/mysql-test/r/constraints.result b/mysql-test/r/constraints.result index 3bec2c3e16d..891f12915cb 100644 --- a/mysql-test/r/constraints.result +++ b/mysql-test/r/constraints.result @@ -18,10 +18,10 @@ create table t1 (a int null); alter table t1 add constraint constraint_1 unique (a); alter table t1 add constraint unique key_1(a); Warnings: -Note 1831 Duplicate index 'key_1' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `key_1`. This is deprecated and will be disallowed in a future release. alter table t1 add constraint constraint_2 unique key_2(a); Warnings: -Note 1831 Duplicate index 'key_2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `key_2`. This is deprecated and will be disallowed in a future release. show create table t1; Table Create Table t1 CREATE TABLE `t1` ( diff --git a/mysql-test/r/create.result b/mysql-test/r/create.result index 20120bf7663..a664801a310 100644 --- a/mysql-test/r/create.result +++ b/mysql-test/r/create.result @@ -181,36 +181,36 @@ Warnings: Note 1051 Unknown table 'test.t2' create table t1 (a int not null, b int, primary key(a), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b), key (b)); Warnings: -Note 1831 Duplicate index 'b_2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_3' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_4' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_5' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_6' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_7' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_8' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_9' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_10' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_11' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_12' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_13' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_14' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_15' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_16' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_17' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_18' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_19' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_20' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_21' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_22' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_23' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_24' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_25' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_26' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_27' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_28' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_29' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_30' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_31' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_2`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_3`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_4`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_5`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_6`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_7`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_8`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_9`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_10`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_11`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_12`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_13`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_14`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_15`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_16`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_17`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_18`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_19`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_20`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_21`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_22`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_23`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_24`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_25`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_26`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_27`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_28`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_29`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_30`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_31`. This is deprecated and will be disallowed in a future release. show create table t1; Table Create Table t1 CREATE TABLE `t1` ( @@ -1045,69 +1045,69 @@ key a064_long_123456789_123456789_123456789_123456789_123456789_1234 ( c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16) ); Warnings: -Note 1831 Duplicate index 'a002_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a003_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a004_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a005_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a006_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a007_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a008_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a009_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a010_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a011_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a012_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a013_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a014_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a015_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a016_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a017_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a018_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a019_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a020_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a021_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a022_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a023_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a024_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a025_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a026_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a027_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a028_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a029_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a030_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a031_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a032_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a033_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a034_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a035_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a036_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a037_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a038_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a039_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a040_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a041_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a042_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a043_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a044_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a045_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a046_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a047_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a048_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a049_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a050_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a051_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a052_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a053_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a054_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a055_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a056_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a057_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a058_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a059_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a060_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a061_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a062_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a063_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a064_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a002_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a003_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a004_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a005_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a006_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a007_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a008_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a009_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a010_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a011_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a012_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a013_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a014_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a015_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a016_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a017_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a018_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a019_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a020_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a021_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a022_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a023_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a024_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a025_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a026_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a027_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a028_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a029_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a030_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a031_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a032_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a033_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a034_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a035_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a036_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a037_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a038_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a039_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a040_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a041_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a042_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a043_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a044_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a045_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a046_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a047_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a048_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a049_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a050_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a051_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a052_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a053_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a054_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a055_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a056_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a057_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a058_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a059_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a060_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a061_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a062_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a063_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a064_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. show create table t1; Table Create Table t1 CREATE TABLE `t1` ( @@ -1410,69 +1410,69 @@ c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16), add key a064_long_123456789_123456789_123456789_123456789_123456789_1234 ( c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16); Warnings: -Note 1831 Duplicate index 'a002_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a003_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a004_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a005_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a006_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a007_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a008_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a009_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a010_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a011_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a012_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a013_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a014_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a015_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a016_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a017_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a018_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a019_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a020_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a021_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a022_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a023_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a024_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a025_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a026_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a027_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a028_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a029_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a030_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a031_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a032_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a033_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a034_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a035_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a036_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a037_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a038_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a039_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a040_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a041_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a042_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a043_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a044_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a045_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a046_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a047_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a048_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a049_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a050_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a051_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a052_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a053_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a054_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a055_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a056_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a057_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a058_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a059_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a060_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a061_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a062_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a063_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a064_long_123456789_123456789_123456789_123456789_123456789_1234' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a002_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a003_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a004_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a005_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a006_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a007_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a008_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a009_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a010_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a011_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a012_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a013_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a014_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a015_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a016_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a017_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a018_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a019_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a020_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a021_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a022_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a023_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a024_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a025_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a026_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a027_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a028_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a029_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a030_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a031_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a032_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a033_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a034_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a035_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a036_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a037_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a038_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a039_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a040_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a041_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a042_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a043_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a044_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a045_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a046_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a047_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a048_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a049_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a050_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a051_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a052_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a053_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a054_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a055_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a056_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a057_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a058_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a059_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a060_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a061_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a062_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a063_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a064_long_123456789_123456789_123456789_123456789_123456789_1234`. This is deprecated and will be disallowed in a future release. show create table t1; Table Create Table t1 CREATE TABLE `t1` ( @@ -2678,6 +2678,17 @@ end| create table t1 as select f1(); ERROR 42S02: Table 'test.t1' doesn't exist drop function f1; +# +# MDEV-10274 Bundling insert with create statement +# for table with unsigned Decimal primary key issues warning 1194 +# +create table t1(ID decimal(2,1) unsigned NOT NULL, PRIMARY KEY (ID))engine=memory +select 2.1 ID; +drop table t1; End of 5.5 tests create table t1; ERROR 42000: A table must have at least 1 column +create table t1 (i int, j int, key(i), key(i)) as select 1 as i, 2 as j; +Warnings: +Note 1831 Duplicate index `i_2`. This is deprecated and will be disallowed in a future release. +drop table t1; diff --git a/mysql-test/r/ctype_ucs.result b/mysql-test/r/ctype_ucs.result index 6b3d87b79e1..843b49c1f75 100644 --- a/mysql-test/r/ctype_ucs.result +++ b/mysql-test/r/ctype_ucs.result @@ -4550,6 +4550,16 @@ SELECT CHAR_LENGTH(TRIM(BOTH 0x00 FROM _ucs2 0x0061)); CHAR_LENGTH(TRIM(BOTH 0x00 FROM _ucs2 0x0061)) 1 # +# MDEV-11685: sql_mode can't be set with non-ascii connection charset +# +SET character_set_connection=ucs2; +SET sql_mode='NO_ENGINE_SUBSTITUTION'; +SELECT @@sql_mode; +@@sql_mode +NO_ENGINE_SUBSTITUTION +SET sql_mode=DEFAULT; +SET NAMES utf8; +# # End of 5.5 tests # # diff --git a/mysql-test/r/ctype_ucs2_def.result b/mysql-test/r/ctype_ucs2_def.result index 5ca7d1689d2..9d8f5181103 100644 --- a/mysql-test/r/ctype_ucs2_def.result +++ b/mysql-test/r/ctype_ucs2_def.result @@ -1,4 +1,4 @@ -call mtr.add_suppression("Cannot use ucs2 as character_set_client"); +call mtr.add_suppression("'ucs2' can not be used as client character set"); show variables like 'collation_server'; Variable_name Value collation_server ucs2_unicode_ci diff --git a/mysql-test/r/ctype_ucs2_query_cache.result b/mysql-test/r/ctype_ucs2_query_cache.result index 9a7580324c1..eba7f2fb0fc 100644 --- a/mysql-test/r/ctype_ucs2_query_cache.result +++ b/mysql-test/r/ctype_ucs2_query_cache.result @@ -1,4 +1,4 @@ -call mtr.add_suppression("Cannot use ucs2 as character_set_client"); +call mtr.add_suppression("'ucs2' can not be used as client character set"); # # Start of 5.5 tests # diff --git a/mysql-test/r/ctype_utf16.result b/mysql-test/r/ctype_utf16.result index 2d91ce3dd6f..958ade902df 100644 --- a/mysql-test/r/ctype_utf16.result +++ b/mysql-test/r/ctype_utf16.result @@ -1580,6 +1580,16 @@ ERROR HY000: Invalid utf16 character string: 'DE9899' DO LPAD(_utf16 0x0061 COLLATE utf16_unicode_ci, 10000, 0x0061DE989999); ERROR HY000: Invalid utf16 character string: 'DE9899' # +# MDEV-11685: sql_mode can't be set with non-ascii connection charset +# +SET character_set_connection=utf16; +SET sql_mode='NO_ENGINE_SUBSTITUTION'; +SELECT @@sql_mode; +@@sql_mode +NO_ENGINE_SUBSTITUTION +SET sql_mode=DEFAULT; +SET NAMES utf8; +# # End of 5.5 tests # # diff --git a/mysql-test/r/ctype_utf16_def.result b/mysql-test/r/ctype_utf16_def.result index 9d3d110fc99..98b6f7d913d 100644 --- a/mysql-test/r/ctype_utf16_def.result +++ b/mysql-test/r/ctype_utf16_def.result @@ -1,4 +1,4 @@ -call mtr.add_suppression("Cannot use utf16 as character_set_client"); +call mtr.add_suppression("'utf16' can not be used as client character set"); SHOW VARIABLES LIKE 'collation_server'; Variable_name Value collation_server utf16_general_ci diff --git a/mysql-test/r/ctype_utf32.result b/mysql-test/r/ctype_utf32.result index 3269e6c900e..076cabf16f7 100644 --- a/mysql-test/r/ctype_utf32.result +++ b/mysql-test/r/ctype_utf32.result @@ -1662,6 +1662,16 @@ select hex(lower(cast(0xffff0000 as char character set utf32))) as c; c FFFF0000 # +# MDEV-11685: sql_mode can't be set with non-ascii connection charset +# +SET character_set_connection=utf32; +SET sql_mode='NO_ENGINE_SUBSTITUTION'; +SELECT @@sql_mode; +@@sql_mode +NO_ENGINE_SUBSTITUTION +SET sql_mode=DEFAULT; +SET NAMES utf8; +# # End of 5.5 tests # # diff --git a/mysql-test/r/ctype_utf8.result b/mysql-test/r/ctype_utf8.result index e5d6caf397b..b98f412e3f1 100644 --- a/mysql-test/r/ctype_utf8.result +++ b/mysql-test/r/ctype_utf8.result @@ -6147,6 +6147,32 @@ SELECT length(data) AS len FROM (SELECT REPEAT('ä', 65537) AS data) AS sub; len 131074 # +# MDEV-10717 Assertion `!null_value' failed in virtual bool Item::send(Protocol*, String*) +# +CREATE TABLE t1 (i INT, KEY(i)); +INSERT INTO t1 VALUES (20081205),(20050327); +SELECT HEX(i), HEX(CHAR(i USING utf8)) FROM t1; +HEX(i) HEX(CHAR(i USING utf8)) +131F197 0131 +1326A35 01326A35 +Warnings: +Warning 1300 Invalid utf8 character string: 'F197' +SET sql_mode='STRICT_ALL_TABLES'; +SELECT HEX(i), HEX(CHAR(i USING utf8)) FROM t1; +HEX(i) HEX(CHAR(i USING utf8)) +131F197 NULL +1326A35 01326A35 +Warnings: +Warning 1300 Invalid utf8 character string: 'F197' +SELECT CHAR(i USING utf8) FROM t1; +CHAR(i USING utf8) +### +### +Warnings: +### 1300 Invalid utf8 character string: 'F197' +SET sql_mode=DEFAULT; +DROP TABLE t1; +# # End of 5.5 tests # # diff --git a/mysql-test/r/ctype_utf8mb4.result b/mysql-test/r/ctype_utf8mb4.result index 38814bc36d6..50382b5d5ca 100644 --- a/mysql-test/r/ctype_utf8mb4.result +++ b/mysql-test/r/ctype_utf8mb4.result @@ -3356,5 +3356,28 @@ DFFFFFDFFFFF9CFFFF9DFFFF9EFFFF # End of 5.6 tests # # +# Start of 10.0 tests +# +# +# MDEV-11343 LOAD DATA INFILE fails to load data with an escape character followed by a multi-byte character +# +CREATE TABLE t1 (a TEXT CHARACTER SET utf8mb4); +LOAD DATA INFILE '../../std_data/loaddata/mdev-11343.txt' INTO TABLE t1 CHARACTER SET utf8mb4; +SELECT HEX(a) FROM t1; +HEX(a) +C3A4 +C3A478 +78C3A4 +78C3A478 +EA99A0 +EA99A078 +78EA99A0 +78EA99A078 +F09F988E +F09F988E78 +78F09F988E +78F09F988E78 +DROP TABLE t1; +# # End of tests # diff --git a/mysql-test/r/default.result b/mysql-test/r/default.result index 9afffe4c3bc..ad3569e7b3d 100644 --- a/mysql-test/r/default.result +++ b/mysql-test/r/default.result @@ -220,3 +220,44 @@ NULL 10 drop table t1, t2; End of 5.0 tests. +# +# Start of 10.0 tests +# +# +# MDEV-11265 Access defied when CREATE VIIEW v1 AS SELECT DEFAULT(column) FROM t1 +# +CREATE TABLE t1 (a INT DEFAULT 10); +INSERT INTO t1 VALUES (11); +CREATE VIEW v1 AS SELECT a AS a FROM t1; +CREATE VIEW v2 AS SELECT DEFAULT(a) AS a FROM t1; +CREATE VIEW v3 AS SELECT VALUES(a) AS a FROM t1; +SELECT * FROM v1; +a +11 +SELECT * FROM v2; +a +10 +SELECT * FROM v3; +a +NULL +UPDATE v2 SET a=123; +ERROR HY000: Column 'a' is not updatable +UPDATE v3 SET a=123; +ERROR HY000: Column 'a' is not updatable +DROP VIEW v3; +DROP VIEW v2; +DROP VIEW v1; +DROP TABLE t1; +# +# MDEV-10780 Server crashes in in create_tmp_table +# +CREATE TABLE t1 (pk INT AUTO_INCREMENT PRIMARY KEY) ENGINE=MyISAM; +INSERT INTO t1 VALUES (); +INSERT INTO t1 VALUES (); +SELECT DISTINCT DEFAULT (pk) FROM t1 GROUP BY RAND() WITH ROLLUP; +DEFAULT (pk) +0 +DROP TABLE t1; +# +# End of 10.0 tests +# diff --git a/mysql-test/r/derived.result b/mysql-test/r/derived.result index 97ba35ed9cd..0b392305bbf 100644 --- a/mysql-test/r/derived.result +++ b/mysql-test/r/derived.result @@ -950,3 +950,68 @@ id select_type table type possible_keys key key_len ref rows Extra 2 DERIVED NULL NULL NULL NULL NULL NULL NULL no matching row in const table DROP TABLES t1,t2; set optimizer_switch=@save_derived_optimizer_switch; +# +# MDEV-10663: Use of Inline table columns in HAVING clause +# throws 1463 Error +# +set @save_sql_mode = @@sql_mode; +set sql_mode='ONLY_FULL_GROUP_BY,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION'; +CREATE TABLE `example1463` ( +`Customer` varchar(255) NOT NULL, +`DeliveryStatus` varchar(255) NOT NULL, +`OrderSize` int(11) NOT NULL +); +INSERT INTO example1463 VALUES ('Charlie', 'Success', 100); +INSERT INTO example1463 VALUES ('David', 'Success', 110); +INSERT INTO example1463 VALUES ('Charlie', 'Failed', 200); +INSERT INTO example1463 VALUES ('David', 'Success', 100); +INSERT INTO example1463 VALUES ('David', 'Unknown', 100); +INSERT INTO example1463 VALUES ('Edward', 'Success', 150); +INSERT INTO example1463 VALUES ('Edward', 'Pending', 150); +SELECT Customer, Success, SUM(OrderSize) +FROM (SELECT Customer, +CASE WHEN DeliveryStatus='Success' THEN 'Yes' ELSE 'No' END AS Success, +OrderSize +FROM example1463) as subQ +GROUP BY Success, Customer +WITH ROLLUP; +Customer Success SUM(OrderSize) +Charlie No 200 +David No 100 +Edward No 150 +NULL No 450 +Charlie Yes 100 +David Yes 210 +Edward Yes 150 +NULL Yes 460 +NULL NULL 910 +SELECT Customer, Success, SUM(OrderSize) +FROM (SELECT Customer, +CASE WHEN DeliveryStatus='Success' THEN 'Yes' ELSE 'No' END AS Success, +OrderSize +FROM example1463) as subQ +GROUP BY Success, Customer; +Customer Success SUM(OrderSize) +Charlie No 200 +David No 100 +Edward No 150 +Charlie Yes 100 +David Yes 210 +Edward Yes 150 +SELECT Customer, Success, SUM(OrderSize) +FROM (SELECT Customer, +CASE WHEN DeliveryStatus='Success' THEN 'Yes' ELSE 'No' END AS Success, +OrderSize +FROM example1463) as subQ +GROUP BY Success, Customer +HAVING Success IS NOT NULL; +Customer Success SUM(OrderSize) +Charlie No 200 +David No 100 +Edward No 150 +Charlie Yes 100 +David Yes 210 +Edward Yes 150 +DROP TABLE example1463; +set sql_mode= @save_sql_mode; +# end of 5.5 diff --git a/mysql-test/r/derived_view.result b/mysql-test/r/derived_view.result index a3b1e37a403..4ee1a3849f5 100644 --- a/mysql-test/r/derived_view.result +++ b/mysql-test/r/derived_view.result @@ -2506,5 +2506,90 @@ DROP TABLE t1,t2; # # end of 5.3 tests # +# +# Bug mdev-11161: The second execution of prepared statement +# does not use generated key for materialized +# derived table / view +# (actually this is a 5.3 bug.) +# +create table t1 ( +mat_id MEDIUMINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, +matintnum CHAR(6) NOT NULL, +test MEDIUMINT UNSIGNED NULL +); +create table t2 ( +mat_id MEDIUMINT UNSIGNED NOT NULL, +pla_id MEDIUMINT UNSIGNED NOT NULL +); +insert into t1 values +(NULL, 'a', 1), (NULL, 'b', 2), (NULL, 'c', 3), (NULL, 'd', 4), +(NULL, 'e', 5), (NULL, 'f', 6), (NULL, 'g', 7), (NULL, 'h', 8), +(NULL, 'i', 9); +insert into t2 values +(1, 100), (1, 101), (1, 102), (2, 100), (2, 103), (2, 104), +(3, 101), (3, 102), (3, 105); +explain +SELECT STRAIGHT_JOIN d.pla_id, m2.mat_id +FROM t1 m2 +INNER JOIN +(SELECT mp.pla_id, MIN(m1.matintnum) AS matintnum +FROM t2 mp INNER JOIN t1 m1 ON mp.mat_id=m1.mat_id +GROUP BY mp.pla_id) d +ON d.matintnum=m2.matintnum; +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY m2 ALL NULL NULL NULL NULL 9 +1 PRIMARY <derived2> ref key0 key0 7 test.m2.matintnum 2 +2 DERIVED mp ALL NULL NULL NULL NULL 9 Using temporary; Using filesort +2 DERIVED m1 eq_ref PRIMARY PRIMARY 3 test.mp.mat_id 1 +prepare stmt1 from +"SELECT STRAIGHT_JOIN d.pla_id, m2.mat_id + FROM t1 m2 + INNER JOIN + (SELECT mp.pla_id, MIN(m1.matintnum) AS matintnum + FROM t2 mp INNER JOIN t1 m1 ON mp.mat_id=m1.mat_id + GROUP BY mp.pla_id) d + ON d.matintnum=m2.matintnum"; +flush status; +execute stmt1; +pla_id mat_id +102 1 +101 1 +100 1 +104 2 +103 2 +105 3 +show status like '%Handler_read%'; +Variable_name Value +Handler_read_first 0 +Handler_read_key 21 +Handler_read_last 0 +Handler_read_next 6 +Handler_read_prev 0 +Handler_read_retry 0 +Handler_read_rnd 6 +Handler_read_rnd_deleted 0 +Handler_read_rnd_next 27 +flush status; +execute stmt1; +pla_id mat_id +102 1 +101 1 +100 1 +104 2 +103 2 +105 3 +show status like '%Handler_read%'; +Variable_name Value +Handler_read_first 0 +Handler_read_key 21 +Handler_read_last 0 +Handler_read_next 6 +Handler_read_prev 0 +Handler_read_retry 0 +Handler_read_rnd 6 +Handler_read_rnd_deleted 0 +Handler_read_rnd_next 27 +deallocate prepare stmt1; +drop table t1,t2; set optimizer_switch=@exit_optimizer_switch; set join_cache_level=@exit_join_cache_level; diff --git a/mysql-test/r/events_slowlog.result b/mysql-test/r/events_slowlog.result new file mode 100644 index 00000000000..7de5925bc0f --- /dev/null +++ b/mysql-test/r/events_slowlog.result @@ -0,0 +1,13 @@ +set @event_scheduler_save= @@global.event_scheduler; +set @slow_query_log_save= @@global.slow_query_log; +set global event_scheduler= on; +set global slow_query_log= on; +set global long_query_time=0.2; +create table t1 (i int); +insert into t1 values (0); +create event ev on schedule at CURRENT_TIMESTAMP + INTERVAL 1 second do update t1 set i=1+sleep(0.5); +FOUND /update t1 set i=1/ in mysqld-slow.log +drop table t1; +set global event_scheduler= @event_scheduler_save; +set global slow_query_log= @slow_query_log_save; +set global long_query_time= @@session.long_query_time; diff --git a/mysql-test/r/explain.result b/mysql-test/r/explain.result index 5678f0f3968..b3c33e3559a 100644 --- a/mysql-test/r/explain.result +++ b/mysql-test/r/explain.result @@ -99,45 +99,45 @@ KEY(b),KEY(b),KEY(b),KEY(b),KEY(b), KEY(b),KEY(b),KEY(b),KEY(b),KEY(b), KEY(b),KEY(b),KEY(b),KEY(b),KEY(b)); Warnings: -Note 1831 Duplicate index 'b_2' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_3' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_4' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_5' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_6' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_7' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_8' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_9' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_10' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_11' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_12' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_13' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_14' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_15' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_16' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_17' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_18' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_19' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_20' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_21' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_22' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_23' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_24' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_25' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_26' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_27' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_28' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_29' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_30' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_31' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_32' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_33' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_34' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_35' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_36' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_37' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_38' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_39' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'b_40' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_2`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_3`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_4`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_5`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_6`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_7`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_8`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_9`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_10`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_11`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_12`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_13`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_14`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_15`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_16`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_17`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_18`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_19`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_20`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_21`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_22`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_23`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_24`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_25`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_26`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_27`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_28`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_29`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_30`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_31`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_32`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_33`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_34`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_35`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_36`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_37`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_38`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_39`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_40`. This is deprecated and will be disallowed in a future release. INSERT INTO t2 VALUES (),(),(); EXPLAIN SELECT 1 FROM (SELECT 1 FROM t2,t1 WHERE b < c GROUP BY 1 LIMIT 1) AS d2; diff --git a/mysql-test/r/fulltext_charsets.result b/mysql-test/r/fulltext_charsets.result new file mode 100644 index 00000000000..39ce02a3fce --- /dev/null +++ b/mysql-test/r/fulltext_charsets.result @@ -0,0 +1,7 @@ +set names utf8mb4; +create table t1 (a int, b text, fulltext (b)) charset=utf8mb4 collate=utf8mb4_unicode_ci; +insert t1 values (1000, 'C͓̙̯͔̩ͅͅi̩̘̜̲a̯̲̬̳̜̖̤o͕͓̜͓̺̖̗,Ì Ì¬Íš ̺T͇̲h͈̱e ̬̜DÌ–o̦̖͔̗͖̩̘c̣̼tÌ͉̫̮̗o͉̫Ìr̙͎̗.͓̪̥'); +select a from t1 where match(b) against ('ciao' in boolean mode); +a +1000 +drop table t1; diff --git a/mysql-test/r/func_time.result b/mysql-test/r/func_time.result index 85ba5d73f36..33aaaf21edb 100644 --- a/mysql-test/r/func_time.result +++ b/mysql-test/r/func_time.result @@ -2739,6 +2739,12 @@ id date1 date2 DATE_ADD(a.date1,INTERVAL -10 DAY) TO_DAYS(a.date1)-10 18 2010-10-13 2010-10-03 2010-10-03 734413 DROP TABLE t1; # +# MDEV-10524 Assertion `arg1_int >= 0' failed in Item_func_additive_op::result_precision() +# +SELECT 1 MOD ADDTIME( '13:58:57', '00:00:01' ) + 2; +1 MOD ADDTIME( '13:58:57', '00:00:01' ) + 2 +3 +# # Start of 10.0 tests # # @@ -2778,5 +2784,73 @@ Warning 1292 Truncated incorrect time value: '-1441:00:00' Warning 1292 Truncated incorrect time value: '-1441:00:00' Warning 1292 Truncated incorrect time value: '-1441:00:00' # +# MDEV-10787 Assertion `ltime->neg == 0' failed in void date_to_datetime(MYSQL_TIME*) +# +CREATE TABLE t1 (d DATE); +INSERT INTO t1 VALUES ('2005-07-20'),('2012-12-21'); +SELECT REPLACE( ADDDATE( d, INTERVAL 0.6732771076944444 HOUR_SECOND ), '2', 'x' ) FROM t1; +REPLACE( ADDDATE( d, INTERVAL 0.6732771076944444 HOUR_SECOND ), '2', 'x' ) +NULL +NULL +Warnings: +Warning 1441 Datetime function: datetime field overflow +Warning 1441 Datetime function: datetime field overflow +SELECT REPLACE( ADDDATE( d, INTERVAL '0.6732771076944444' HOUR_SECOND ), '2', 'x' ) FROM t1; +REPLACE( ADDDATE( d, INTERVAL '0.6732771076944444' HOUR_SECOND ), '2', 'x' ) +NULL +NULL +Warnings: +Warning 1441 Datetime function: datetime field overflow +Warning 1441 Datetime function: datetime field overflow +SELECT CAST(ADDDATE( d, INTERVAL 6732771076944444 SECOND) AS CHAR) FROM t1; +CAST(ADDDATE( d, INTERVAL 6732771076944444 SECOND) AS CHAR) +NULL +NULL +Warnings: +Warning 1441 Datetime function: datetime field overflow +Warning 1441 Datetime function: datetime field overflow +SELECT CAST(ADDDATE( d, INTERVAL '67327710769444:44' HOUR_SECOND) AS CHAR) FROM t1; +CAST(ADDDATE( d, INTERVAL '67327710769444:44' HOUR_SECOND) AS CHAR) +NULL +NULL +Warnings: +Warning 1441 Datetime function: datetime field overflow +Warning 1441 Datetime function: datetime field overflow +SELECT CAST(ADDDATE( d, INTERVAL '673277107694:44:44' HOUR_SECOND) AS CHAR) FROM t1; +CAST(ADDDATE( d, INTERVAL '673277107694:44:44' HOUR_SECOND) AS CHAR) +NULL +NULL +Warnings: +Warning 1441 Datetime function: datetime field overflow +Warning 1441 Datetime function: datetime field overflow +DROP TABLE t1; +SELECT ADDDATE(DATE'0000-01-01', INTERVAL '3652423:23:59:59' DAY_SECOND); +ADDDATE(DATE'0000-01-01', INTERVAL '3652423:23:59:59' DAY_SECOND) +9999-12-31 23:59:59 +SELECT ADDDATE(DATE'0000-01-01', INTERVAL '0:87658175:59:59' DAY_SECOND); +ADDDATE(DATE'0000-01-01', INTERVAL '0:87658175:59:59' DAY_SECOND) +9999-12-31 23:59:59 +SELECT ADDDATE(DATE'0000-01-01', INTERVAL '0:0:5259490559:59' DAY_SECOND); +ADDDATE(DATE'0000-01-01', INTERVAL '0:0:5259490559:59' DAY_SECOND) +9999-12-31 23:59:59 +SELECT ADDDATE(DATE'0000-01-01', INTERVAL '0:0:0:315569433599' DAY_SECOND); +ADDDATE(DATE'0000-01-01', INTERVAL '0:0:0:315569433599' DAY_SECOND) +9999-12-31 23:59:59 +SELECT ADDDATE(DATE'0000-01-01', INTERVAL '3652423:0:0:315569433559' DAY_SECOND); +ADDDATE(DATE'0000-01-01', INTERVAL '3652423:0:0:315569433559' DAY_SECOND) +NULL +Warnings: +Warning 1441 Datetime function: datetime field overflow +SELECT ADDDATE(DATE'0000-01-01', INTERVAL '0:87658175:0:315569433559' DAY_SECOND); +ADDDATE(DATE'0000-01-01', INTERVAL '0:87658175:0:315569433559' DAY_SECOND) +NULL +Warnings: +Warning 1441 Datetime function: datetime field overflow +SELECT ADDDATE(DATE'0000-01-01', INTERVAL '0:0:5259490559:315569433599' DAY_SECOND); +ADDDATE(DATE'0000-01-01', INTERVAL '0:0:5259490559:315569433599' DAY_SECOND) +NULL +Warnings: +Warning 1441 Datetime function: datetime field overflow +# # End of 10.0 tests # diff --git a/mysql-test/r/group_by.result b/mysql-test/r/group_by.result index 38abbfef261..7a105fa6172 100644 --- a/mysql-test/r/group_by.result +++ b/mysql-test/r/group_by.result @@ -2518,6 +2518,20 @@ MAX(i) c 7 foo drop table t1,t2; # +# ONLY_FULL_GROUP_BY references +# +set @save_sql_mode = @@sql_mode; +set sql_mode='ONLY_FULL_GROUP_BY'; +create table t1 (a int, b int); +select a+b as x from t1 group by x having x > 1; +x +select a as x from t1 group by x having x > 1; +x +select a from t1 group by a having a > 1; +a +drop table t1; +set sql_mode= @save_sql_mode; +# # Bug #58782 # Missing rows with SELECT .. WHERE .. IN subquery # with full GROUP BY and no aggr diff --git a/mysql-test/r/group_by_innodb.result b/mysql-test/r/group_by_innodb.result index 381e0d7493c..af68640c0b1 100644 --- a/mysql-test/r/group_by_innodb.result +++ b/mysql-test/r/group_by_innodb.result @@ -123,4 +123,14 @@ id xtext optionen 2 number 22,25 1 select Kabel mit Stecker 5-polig,Kabel ohne Stecker DROP TABLE t1, t2; +# +# MDEV-11162: Assertion `num_records == m_idx_array.size()' failed in Filesort_buffer::alloc_sort_buffer(uint, uint) +# +CREATE TABLE t1 (i INT) ENGINE=InnoDB; +SELECT ( SELECT DISTINCT GROUP_CONCAT(SLEEP(0)) FROM t1 GROUP BY i ); +( SELECT DISTINCT GROUP_CONCAT(SLEEP(0)) FROM t1 GROUP BY i ) +NULL +SELECT i FROM t1 order by i LIMIT 1; +i +DROP TABLE t1; # End of tests diff --git a/mysql-test/r/group_min_max.result b/mysql-test/r/group_min_max.result index 3fd87425e23..25cd4a25279 100644 --- a/mysql-test/r/group_min_max.result +++ b/mysql-test/r/group_min_max.result @@ -2562,7 +2562,7 @@ a MIN(b) MAX(b) AVG(b) DROP TABLE t1; create table t1 (a int, b int, key (a,b), key `index` (a,b)) engine=MyISAM; Warnings: -Note 1831 Duplicate index 'index' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `index`. This is deprecated and will be disallowed in a future release. insert into t1 (a,b) values (0,0),(0,1),(0,2),(0,3),(0,4),(0,5),(0,6), (0,7),(0,8),(0,9),(0,10),(0,11),(0,12),(0,13), diff --git a/mysql-test/r/index_merge_innodb.result b/mysql-test/r/index_merge_innodb.result index 5bf56e213ab..b3007408368 100644 --- a/mysql-test/r/index_merge_innodb.result +++ b/mysql-test/r/index_merge_innodb.result @@ -799,3 +799,32 @@ a b c 9 d d DROP TABLE t1; set optimizer_switch= @optimizer_switch_save; +# +# MDEV-10927: Crash When Using sort_union Optimization +# +set @tmp_optimizer_switch=@@optimizer_switch; +SET optimizer_switch='index_merge_sort_intersection=on'; +SET SESSION sort_buffer_size = 1024; +create table t1 ( +pk int(11) NOT NULL AUTO_INCREMENT, +col1 int(11) NOT NULL, +col2 int(11) NOT NULL, +col3 int(11) NOT NULL, +key2 int(11) NOT NULL, +col4 int(11) NOT NULL, +key1 int(11) NOT NULL, +PRIMARY KEY (pk), +KEY key1 (key1), +KEY key2 (key2) +) ENGINE=InnoDB AUTO_INCREMENT=12860259 DEFAULT CHARSET=utf8 ROW_FORMAT=COMPACT; +create table t2(a int); +insert into t2 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +create table t3(a int); +insert into t3 select A.a + B.a* 10 + C.a * 100 + D.a*1000 from t2 A, t2 B, t2 C, t2 D; +insert into t1 (key1, key2, col1,col2,col3,col4) +select a,a, a,a,a,a from t3; +SELECT sum(col1) FROM t1 FORCE INDEX (key1,key2) WHERE (key1 between 10 and 8191+10) or (key2= 5); +sum(col1) +33632261 +drop table t1,t2,t3; +set optimizer_switch=@tmp_optimizer_switch; diff --git a/mysql-test/r/index_merge_myisam.result b/mysql-test/r/index_merge_myisam.result index c63ed132662..b3beff5a967 100644 --- a/mysql-test/r/index_merge_myisam.result +++ b/mysql-test/r/index_merge_myisam.result @@ -224,7 +224,7 @@ index i2_1(key2, key2_1), index i2_2(key2, key2_1) ); Warnings: -Note 1831 Duplicate index 'i2_2' defined on the table 'test.t4'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i2_2`. This is deprecated and will be disallowed in a future release. insert into t4 select key1,key1,key1 div 10, key1 % 10, key1 % 10, key1 from t0; select * from t4 where key1a = 3 or key1b = 4; key1a key1b key2 key2_1 key2_2 key3 diff --git a/mysql-test/r/information_schema.result b/mysql-test/r/information_schema.result index 852b520d1fc..10d5ea6b8b4 100644 --- a/mysql-test/r/information_schema.result +++ b/mysql-test/r/information_schema.result @@ -478,10 +478,10 @@ create table t1 (a int null, primary key(a)); alter table t1 add constraint constraint_1 unique (a); alter table t1 add constraint unique key_1(a); Warnings: -Note 1831 Duplicate index 'key_1' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `key_1`. This is deprecated and will be disallowed in a future release. alter table t1 add constraint constraint_2 unique key_2(a); Warnings: -Note 1831 Duplicate index 'key_2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `key_2`. This is deprecated and will be disallowed in a future release. show create table t1; Table Create Table t1 CREATE TABLE `t1` ( diff --git a/mysql-test/r/information_schema_part.result b/mysql-test/r/information_schema_part.result index b34183ebdee..77959de256e 100644 --- a/mysql-test/r/information_schema_part.result +++ b/mysql-test/r/information_schema_part.result @@ -151,3 +151,11 @@ select create_options from information_schema.tables where table_schema="test"; create_options partitioned drop table t1; +# +# MDEV-11353 - Identical logical conditions +# +CREATE TABLE t1(a INT) CHECKSUM=1 SELECT 1; +SELECT CHECKSUM FROM INFORMATION_SCHEMA.PARTITIONS WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +CHECKSUM +3036305396 +DROP TABLE t1; diff --git a/mysql-test/r/innodb_icp.result b/mysql-test/r/innodb_icp.result index fb467494525..33a8027adf1 100644 --- a/mysql-test/r/innodb_icp.result +++ b/mysql-test/r/innodb_icp.result @@ -712,8 +712,8 @@ b INT, c INT, d DATE NOT NULL, e VARCHAR(1), KEY (c), KEY (d), KEY k2(b), KEY k3(b), KEY k4(b) ); Warnings: -Note 1831 Duplicate index 'k3' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'k4' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `k3`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `k4`. This is deprecated and will be disallowed in a future release. INSERT INTO t1 (b,c,d,e) VALUES (6,5,'2006-05-25','y'),(1,5,'2008-01-23','t'), (6,5,'2007-06-18','d'),(4,5,'1900-01-01','r'), diff --git a/mysql-test/r/innodb_mysql_sync.result b/mysql-test/r/innodb_mysql_sync.result index 478b0109816..cfe855fbb76 100644 --- a/mysql-test/r/innodb_mysql_sync.result +++ b/mysql-test/r/innodb_mysql_sync.result @@ -308,7 +308,7 @@ SET DEBUG_SYNC= 'now SIGNAL continue3'; # Connection default # Reaping ALTER TABLE ... Warnings: -Note 1831 Duplicate index 'i2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i2`. This is deprecated and will be disallowed in a future release. SET DEBUG_SYNC= 'RESET'; DELETE FROM t1 WHERE a= 3; # @@ -358,7 +358,7 @@ SET DEBUG_SYNC= 'now SIGNAL continue4'; # Connection default # Reaping ALTER TABLE ... Warnings: -Note 1831 Duplicate index 'i4' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i4`. This is deprecated and will be disallowed in a future release. SET DEBUG_SYNC= 'RESET'; DROP TABLE t1; SET DEBUG_SYNC= 'RESET'; diff --git a/mysql-test/r/join_cache.result b/mysql-test/r/join_cache.result index c1dbef08c04..5c4875546e6 100644 --- a/mysql-test/r/join_cache.result +++ b/mysql-test/r/join_cache.result @@ -5701,11 +5701,13 @@ LEFT JOIN t2 c1 ON c1.parent_id = t.id AND c1.col2 = "val" LEFT JOIN t2 c23 ON c23.parent_id = t.id AND c23.col2 = "val" LEFT JOIN t2 c24 ON c24.parent_id = t.id AND c24.col2 = "val" LEFT JOIN t2 c25 ON c25.parent_id = t.id AND c25.col2 = "val" + LEFT JOIN t2 c26 ON c26.parent_id = t.id AND c26.col2 = "val" + LEFT JOIN t2 c27 ON c27.parent_id = t.id AND c27.col2 = "val" ORDER BY col1; id col1 -select timestampdiff(second, @init_time, now()) <= 1; -timestampdiff(second, @init_time, now()) <= 1 +select timestampdiff(second, @init_time, now()) <= 5; +timestampdiff(second, @init_time, now()) <= 5 1 set join_cache_level=2; set @init_time:=now(); @@ -5737,11 +5739,13 @@ LEFT JOIN t2 c1 ON c1.parent_id = t.id AND c1.col2 = "val" LEFT JOIN t2 c23 ON c23.parent_id = t.id AND c23.col2 = "val" LEFT JOIN t2 c24 ON c24.parent_id = t.id AND c24.col2 = "val" LEFT JOIN t2 c25 ON c25.parent_id = t.id AND c25.col2 = "val" + LEFT JOIN t2 c26 ON c26.parent_id = t.id AND c26.col2 = "val" + LEFT JOIN t2 c27 ON c27.parent_id = t.id AND c27.col2 = "val" ORDER BY col1; id col1 -select timestampdiff(second, @init_time, now()) <= 1; -timestampdiff(second, @init_time, now()) <= 1 +select timestampdiff(second, @init_time, now()) <= 5; +timestampdiff(second, @init_time, now()) <= 5 1 EXPLAIN SELECT t.* @@ -5772,6 +5776,8 @@ LEFT JOIN t2 c1 ON c1.parent_id = t.id AND c1.col2 = "val" LEFT JOIN t2 c23 ON c23.parent_id = t.id AND c23.col2 = "val" LEFT JOIN t2 c24 ON c24.parent_id = t.id AND c24.col2 = "val" LEFT JOIN t2 c25 ON c25.parent_id = t.id AND c25.col2 = "val" + LEFT JOIN t2 c26 ON c26.parent_id = t.id AND c26.col2 = "val" + LEFT JOIN t2 c27 ON c27.parent_id = t.id AND c27.col2 = "val" ORDER BY col1; id select_type table type possible_keys key key_len ref rows Extra @@ -5801,6 +5807,8 @@ id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE c23 ALL NULL NULL NULL NULL 1 Using where; Using join buffer (incremental, BNL join) 1 SIMPLE c24 ALL NULL NULL NULL NULL 1 Using where; Using join buffer (incremental, BNL join) 1 SIMPLE c25 ALL NULL NULL NULL NULL 1 Using where; Using join buffer (incremental, BNL join) +1 SIMPLE c26 ALL NULL NULL NULL NULL 1 Using where; Using join buffer (incremental, BNL join) +1 SIMPLE c27 ALL NULL NULL NULL NULL 1 Using where; Using join buffer (incremental, BNL join) set join_buffer_size=default; set join_cache_level = default; DROP TABLE t1,t2; diff --git a/mysql-test/r/key.result b/mysql-test/r/key.result index a89a79dadbb..0616a1b0d0a 100644 --- a/mysql-test/r/key.result +++ b/mysql-test/r/key.result @@ -428,7 +428,7 @@ index i5 (c1, c2, c3, c4), primary key (c2, c3), index (c2, c4)); Warnings: -Note 1831 Duplicate index 'i1' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i1`. This is deprecated and will be disallowed in a future release. show create table t1; Table Create Table t1 CREATE TABLE `t1` ( @@ -448,17 +448,17 @@ t1 CREATE TABLE `t1` ( alter table t1 drop index c1; alter table t1 add index (c1); Warnings: -Note 1831 Duplicate index 'c1' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `c1`. This is deprecated and will be disallowed in a future release. alter table t1 add index (c1); Warnings: -Note 1831 Duplicate index 'c1_2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `c1_2`. This is deprecated and will be disallowed in a future release. alter table t1 drop index i3; alter table t1 add index i3 (c3); alter table t1 drop index i2, drop index i4; alter table t1 add index i2 (c2), add index i4 (c4); alter table t1 drop index i2, drop index i4, add index i6 (c2, c4); Warnings: -Note 1831 Duplicate index 'i6' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i6`. This is deprecated and will be disallowed in a future release. alter table t1 add index i2 (c2), add index i4 (c4), drop index i6; alter table t1 drop index i2, drop index i4, add unique i4 (c4); alter table t1 add index i2 (c2), drop index i4, add index i4 (c4); diff --git a/mysql-test/r/loaddata.result b/mysql-test/r/loaddata.result index 2f2a3579eec..9f90184a3f9 100644 --- a/mysql-test/r/loaddata.result +++ b/mysql-test/r/loaddata.result @@ -507,7 +507,7 @@ DROP TABLE t1; # Bug#11765139 58069: LOAD DATA INFILE: VALGRIND REPORTS INVALID MEMORY READS AND WRITES WITH U # CREATE TABLE t1(f1 INT); -SELECT 0xE1C330 INTO OUTFILE 't1.dat'; +SELECT 0xE1BB30 INTO OUTFILE 't1.dat'; LOAD DATA INFILE 't1.dat' IGNORE INTO TABLE t1 CHARACTER SET utf8; DROP TABLE t1; # @@ -532,27 +532,21 @@ FIELDS TERMINATED BY 't' LINES TERMINATED BY ''; Got one of the listed errors SET @@sql_mode= @old_mode; DROP TABLE t1; - # -# Bug#23080148 - Backport of Bug#20683959. -# Bug#20683959 LOAD DATA INFILE IGNORES A SPECIFIC ROW SILENTLY -# UNDER DB CHARSET IS UTF8. +# MDEV-11079 Regression: LOAD DATA INFILE lost BLOB support using utf8 load files # -CREATE DATABASE d1 CHARSET latin1; -USE d1; -CREATE TABLE t1 (val TEXT); -LOAD DATA INFILE '../../std_data/bug20683959loaddata.txt' INTO TABLE t1; -SELECT COUNT(*) FROM t1; -COUNT(*) -1 -SELECT HEX(val) FROM t1; -HEX(val) -C38322525420406E696F757A656368756E3A20E98198E2889AF58081AEE7B99DE4B88AE383A3E7B99DE69690F58087B3E7B9A7EFBDA8E7B99DEFBDB3E7B99DE78999E880B3E7B8BAEFBDAAE7B9A7E89699E296A1E7B8BAE4BBA3EFBD8CE7B8BAEFBDA9E7B8B2E2889AE38184E7B99DEFBDB3E7B99DE4B88AE383A3E7B99DE69690F58087B3E7B9A7EFBDA8E7B99DEFBDB3E7B99DE5B3A8EFBD84E8ABA0EFBDA8E89C89F580948EE599AAE7B8BAEFBDAAE7B8BAE9A198EFBDA9EFBDB1E7B9A7E581B5E289A0E7B8BAEFBDBEE7B9A7E9A194EFBDA9E882B4EFBDA5EFBDB5E980A7F5808B96E28693E99EABE38287E58F99E7B8BAE58AB1E28691E7B8BAF5808B9AE7828AE98095EFBDB1E7B8BAEFBDAFE7B8B2E288ABE6A89FE89EB3E6BA98F58081ADE88EA0EFBDBAE98095E6BA98F58081AEE89D93EFBDBAE8AD9BEFBDACE980A7F5808B96E28693E7B8BAF580918EE288AAE7B8BAE4B88AEFBC9EE7B8BAE4B99DE28691E7B8BAF5808B96EFBCA0E88DB3E6A68AEFBDB9EFBDB3E981B2E5B3A8E296A1E7B8BAE7A4BCE7828AE88DB3E6A68AEFBDB0EFBDBDE7B8BAA0E7B8BAE88B93EFBDBEE5B899EFBC9E -CREATE DATABASE d2 CHARSET utf8; -USE d2; -CREATE TABLE t1 (val TEXT); -LOAD DATA INFILE '../../std_data/bug20683959loaddata.txt' INTO TABLE t1; -ERROR HY000: Invalid utf8 character string: 'Ã"RT @niouzechun: \9058\221A' -DROP TABLE d1.t1, d2.t1; -DROP DATABASE d1; -DROP DATABASE d2; +CREATE TABLE t1 (a mediumblob NOT NULL) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_general_ci; +LOAD DATA INFILE '../../std_data/loaddata/mdev-11079.txt' INTO TABLE t1 CHARSET utf8 FIELDS TERMINATED BY ';' ENCLOSED BY '"' ESCAPED BY '\\' LINES TERMINATED BY '\n'; +SELECT HEX(a) FROM t1; +HEX(a) +25AAABAC +DROP TABLE t1; +# +# MDEV-11631 LOAD DATA INFILE fails to load data with an escape character followed by a multi-byte character +# +CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET utf8); +LOAD DATA INFILE '../../std_data/loaddata/mdev-11631.txt' INTO TABLE t1 CHARACTER SET utf8; +SELECT HEX(a) FROM t1; +HEX(a) +C3A4 +DROP TABLE t1; diff --git a/mysql-test/r/log_slow.result b/mysql-test/r/log_slow.result index ca8b9e7c439..4f5eb9359f6 100644 --- a/mysql-test/r/log_slow.result +++ b/mysql-test/r/log_slow.result @@ -67,9 +67,9 @@ sleep(0.5) select count(*) FROM mysql.slow_log; count(*) 1 -truncate mysql.slow_log; set @@long_query_time=default; set global slow_query_log= @org_slow_query_log; set @@log_slow_filter=default; set @@log_slow_verbosity=default; set global log_output= default; +truncate mysql.slow_log; diff --git a/mysql-test/r/lowercase_table2.result b/mysql-test/r/lowercase_table2.result index 7491334907e..1130e782291 100644 --- a/mysql-test/r/lowercase_table2.result +++ b/mysql-test/r/lowercase_table2.result @@ -138,7 +138,7 @@ Tables_in_test (T1%) T1 alter table t1 add index (A); Warnings: -Note 1831 Duplicate index 'A_2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `A_2`. This is deprecated and will be disallowed in a future release. show tables like 't1%'; Tables_in_test (t1%) t1 diff --git a/mysql-test/r/mix2_myisam.result b/mysql-test/r/mix2_myisam.result index e824db8928a..8aacbf060bd 100644 --- a/mysql-test/r/mix2_myisam.result +++ b/mysql-test/r/mix2_myisam.result @@ -257,7 +257,7 @@ drop table t1; CREATE TABLE t1 (a int not null, b int not null,c int not null, key(a),primary key(a,b), unique(c),key(a),unique(b)) ENGINE = MyISAM; Warnings: -Note 1831 Duplicate index 'a_2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a_2`. This is deprecated and will be disallowed in a future release. show index from t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment t1 0 PRIMARY 1 a A # NULL NULL BTREE @@ -1550,7 +1550,7 @@ alter table t1 add unique(v); ERROR 23000: Duplicate entry '{ ' for key 'v_2' alter table t1 add key(v); Warnings: -Note 1831 Duplicate index 'v_2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `v_2`. This is deprecated and will be disallowed in a future release. select concat('*',v,'*',c,'*',t,'*') as qq from t1 where v='a'; qq *a*a*a* diff --git a/mysql-test/r/mrr_icp_extra.result b/mysql-test/r/mrr_icp_extra.result index 855ac4beab5..eaabd3d8119 100644 --- a/mysql-test/r/mrr_icp_extra.result +++ b/mysql-test/r/mrr_icp_extra.result @@ -351,7 +351,7 @@ alter table t1 add unique(v); ERROR 23000: Duplicate entry '{ ' for key 'v_2' alter table t1 add key(v); Warnings: -Note 1831 Duplicate index 'v_2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `v_2`. This is deprecated and will be disallowed in a future release. select concat('*',v,'*',c,'*',t,'*') as qq from t1 where v='a'; qq *a*a*a* diff --git a/mysql-test/r/myisam.result b/mysql-test/r/myisam.result index 5506e4e419c..2a52b25b97f 100644 --- a/mysql-test/r/myisam.result +++ b/mysql-test/r/myisam.result @@ -1243,7 +1243,7 @@ alter table t1 add unique(v); ERROR 23000: Duplicate entry '{ ' for key 'v_2' alter table t1 add key(v); Warnings: -Note 1831 Duplicate index 'v_2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `v_2`. This is deprecated and will be disallowed in a future release. select concat('*',v,'*',c,'*',t,'*') as qq from t1 where v='a'; qq *a*a*a* @@ -2442,8 +2442,8 @@ SET myisam_repair_threads=2; SET myisam_sort_buffer_size=4096; CREATE TABLE t1(a CHAR(255), KEY(a), KEY(a), KEY(a)); Warnings: -Note 1831 Duplicate index 'a_2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a_3' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a_2`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a_3`. This is deprecated and will be disallowed in a future release. INSERT INTO t1 VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9),(0),(1),(2),(3); REPAIR TABLE t1; Table Op Msg_type Msg_text diff --git a/mysql-test/r/myisam_icp.result b/mysql-test/r/myisam_icp.result index 49cba60320d..95befeb5a42 100644 --- a/mysql-test/r/myisam_icp.result +++ b/mysql-test/r/myisam_icp.result @@ -716,8 +716,8 @@ b INT, c INT, d DATE NOT NULL, e VARCHAR(1), KEY (c), KEY (d), KEY k2(b), KEY k3(b), KEY k4(b) ); Warnings: -Note 1831 Duplicate index 'k3' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'k4' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `k3`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `k4`. This is deprecated and will be disallowed in a future release. INSERT INTO t1 (b,c,d,e) VALUES (6,5,'2006-05-25','y'),(1,5,'2008-01-23','t'), (6,5,'2007-06-18','d'),(4,5,'1900-01-01','r'), diff --git a/mysql-test/r/order_by.result b/mysql-test/r/order_by.result index 9c63570561d..07f278a6563 100644 --- a/mysql-test/r/order_by.result +++ b/mysql-test/r/order_by.result @@ -2937,6 +2937,17 @@ t2 A, t2 B where A.b = B.b order by A.col2, B.col2 limit 10, 1000000; drop table t1,t2,t3; +# +# mdev-10705 : long order by list that can be skipped +# +SELECT 1 +UNION +( SELECT 2 +ORDER BY NULL, @a0 := 3, @a1 := 3, @a2 := 3, @a3 := 3, @a4 := 3, +@a5 := 3, @a6 := 3, @a7 := 3, @a8 := 3, @a9 := 3, @a10 := 3 ); +1 +1 +2 End of 5.5 tests # # MDEV-5884: EXPLAIN UPDATE ... ORDER BY LIMIT shows wrong #rows diff --git a/mysql-test/r/parser.result b/mysql-test/r/parser.result index 7e2593eb9c9..cca754da295 100644 --- a/mysql-test/r/parser.result +++ b/mysql-test/r/parser.result @@ -663,3 +663,12 @@ select 1<<!0, 1 << !0; select 0<!0, 0 < ! 0; 0<!0 0 < ! 0 1 1 +# +# MDEV-11171 Assertion `m_cpp_buf <= ptr && ptr <= m_cpp_buf + m_buf_length' failed in Lex_input_stream::body_utf8_append(const char*, const char*) +# +CREATE TABLE t1 (id INT); +CREATE TRIGGER tr AFTER DELETE ON t1 FOR EACH ROW SET @a = 1\; +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near '\' at line 1 +PREPARE stmt FROM 'CREATE TRIGGER tr AFTER DELETE ON t1 FOR EACH ROW SET @a = 1\\'; +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near '\' at line 1 +DROP TABLE t1; diff --git a/mysql-test/r/partition.result b/mysql-test/r/partition.result index 09b22761bcb..d4b719f3a75 100644 --- a/mysql-test/r/partition.result +++ b/mysql-test/r/partition.result @@ -300,7 +300,7 @@ create index i on t1 (a); ERROR 42000: Duplicate key name 'i' create index i2 on t1 (a); Warnings: -Note 1831 Duplicate index 'i2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `i2`. This is deprecated and will be disallowed in a future release. drop table t1; CREATE TABLE t1 (a INT, FOREIGN KEY (a) REFERENCES t0 (a)) ENGINE=MyISAM diff --git a/mysql-test/r/partition_innodb.result b/mysql-test/r/partition_innodb.result index f1c3795791a..e1bc075f14a 100644 --- a/mysql-test/r/partition_innodb.result +++ b/mysql-test/r/partition_innodb.result @@ -609,7 +609,7 @@ c INT, PRIMARY KEY (c,a), KEY (a),KEY (a) ) ENGINE=INNODB PARTITION BY KEY () PARTITIONS 2; Warnings: -Note 1831 Duplicate index 'a_2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a_2`. This is deprecated and will be disallowed in a future release. INSERT INTO t1 VALUES (1,5,1),(2,4,1),(3,3,1),(4,2,1),(5,1,1); UPDATE t1 SET b = 0, c=1 WHERE a <=>0; SELECT * FROM t1; diff --git a/mysql-test/r/pool_of_threads.result b/mysql-test/r/pool_of_threads.result index 9611d7ff43b..ddc451516e2 100644 --- a/mysql-test/r/pool_of_threads.result +++ b/mysql-test/r/pool_of_threads.result @@ -2157,23 +2157,22 @@ Warnings: Warning 1052 Column 'kundentyp' in group statement is ambiguous drop table t1; SET optimizer_switch=@save_optimizer_switch; -SELECT sleep(5.5); -SELECT sleep(5); +SELECT sleep(50); +SELECT sleep(50); # -- Success: more than --thread_pool_max_threads normal connections not possible -sleep(5.5) -0 -sleep(5) -0 -SELECT sleep(5); -SELECT sleep(5); SELECT 'Connection on extra port ok'; Connection on extra port ok Connection on extra port ok +SELECT sleep(5.5); SELECT 'Connection on extra port 2 ok'; Connection on extra port 2 ok Connection on extra port 2 ok # -- Success: more than --extra-max-connections + 1 normal connections not possible -sleep(5) -0 -sleep(5) +KILL QUERY <default_connection_ID>; +KILL QUERY <con2_connection_ID>; +sleep(50) +1 +sleep(50) +1 +sleep(5.5) 0 diff --git a/mysql-test/r/selectivity.result b/mysql-test/r/selectivity.result index 8fb5cd17c51..61a77d135e7 100644 --- a/mysql-test/r/selectivity.result +++ b/mysql-test/r/selectivity.result @@ -1517,3 +1517,24 @@ Note 1003 select `test`.`t2`.`col1` AS `col1` from `test`.`t2` where 0 drop table t1,t2; set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; set use_stat_tables=@save_use_stat_tables; +# +# Bug mdev-11364: IS NULL over not nullable datetime column +# in mergeable derived +# +set use_stat_tables='preferably'; +set optimizer_use_condition_selectivity=4; +set HISTOGRAM_SIZE = 255; +CREATE TABLE t1 (t TIME, d DATE NOT NULL); +INSERT INTO t1 VALUES ('10:00:00', '0000-00-00'),('11:00:00','0000-00-00'); +ANALYZE TABLE t1; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +SELECT * FROM (SELECT t FROM t1 WHERE d IS NULL) sq; +t +10:00:00 +11:00:00 +DROP TABLE t1; +set histogram_size=@save_histogram_size; +set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; +set use_stat_tables=@save_use_stat_tables; diff --git a/mysql-test/r/selectivity_innodb.result b/mysql-test/r/selectivity_innodb.result index 3d15131dbb5..a026c2e6d92 100644 --- a/mysql-test/r/selectivity_innodb.result +++ b/mysql-test/r/selectivity_innodb.result @@ -1521,6 +1521,27 @@ Note 1003 select `test`.`t2`.`col1` AS `col1` from `test`.`t2` where 0 drop table t1,t2; set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; set use_stat_tables=@save_use_stat_tables; +# +# Bug mdev-11364: IS NULL over not nullable datetime column +# in mergeable derived +# +set use_stat_tables='preferably'; +set optimizer_use_condition_selectivity=4; +set HISTOGRAM_SIZE = 255; +CREATE TABLE t1 (t TIME, d DATE NOT NULL); +INSERT INTO t1 VALUES ('10:00:00', '0000-00-00'),('11:00:00','0000-00-00'); +ANALYZE TABLE t1; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +SELECT * FROM (SELECT t FROM t1 WHERE d IS NULL) sq; +t +10:00:00 +11:00:00 +DROP TABLE t1; +set histogram_size=@save_histogram_size; +set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; +set use_stat_tables=@save_use_stat_tables; set optimizer_switch=@save_optimizer_switch_for_selectivity_test; set @tmp_ust= @@use_stat_tables; set @tmp_oucs= @@optimizer_use_condition_selectivity; diff --git a/mysql-test/r/sp-prelocking.result b/mysql-test/r/sp-prelocking.result index 5b594e9ea55..eb47cc21f41 100644 --- a/mysql-test/r/sp-prelocking.result +++ b/mysql-test/r/sp-prelocking.result @@ -340,3 +340,26 @@ f1() DROP FUNCTION f1; DROP VIEW v1; DROP TABLE t1,t2; +# +# Bug #16672723 "CAN'T FIND TEMPORARY TABLE". +# +CREATE FUNCTION f1() RETURNS INT RETURN 1; +CREATE TEMPORARY TABLE tmp1(a INT); +PREPARE stmt1 FROM "CREATE TEMPORARY TABLE tmp2 AS SELECT b FROM (SELECT f1() AS b FROM tmp1) AS t"; +# The below statement failed before the fix. +EXECUTE stmt1; +DROP TEMPORARY TABLES tmp1, tmp2; +DEALLOCATE PREPARE stmt1; +DROP FUNCTION f1; +create procedure sp1() +begin +drop table if exists t1, t2; +create temporary table t1 select 1 v; +create table t2 (col varchar(45)) select distinct col from (select sf1() as col from t1) t; +end$$ +create function sf1() returns text return 'blah'; +call test.sp1(); +call test.sp1(); +drop procedure sp1; +drop function sf1; +drop table t2; diff --git a/mysql-test/r/sp.result b/mysql-test/r/sp.result index 45cd071e732..c059428be16 100644 --- a/mysql-test/r/sp.result +++ b/mysql-test/r/sp.result @@ -7859,6 +7859,45 @@ v1 6 DROP PROCEDURE p1; DROP TABLE t1; +# +# MDEV-10713: signal 11 error on multi-table update - crash in +# handler::increment_statistics or in make_select or assertion +# failure pfs_thread == ((PFS_thread*) pthread_getspecific((THR_PFS))) +# +CREATE TABLE `t1` ( +`CLOSE_YN` varchar(10) COLLATE utf8_bin DEFAULT NULL +) DEFAULT CHARSET=utf8 COLLATE=utf8_bin ; +CREATE TABLE `t2` ( +`ap_close_to` varchar(8) COLLATE utf8_bin DEFAULT NULL +) DEFAULT CHARSET=utf8 COLLATE=utf8_bin ; +insert t1 values (1); +CREATE FUNCTION `f1`(`P_DC_CD` VARBINARY(50), `P_SYS_DATE` DATETIME) RETURNS datetime +DETERMINISTIC +SQL SECURITY INVOKER +BEGIN +DECLARE V_SYS_DATE DATETIME; +SELECT now() AS LOC_DATE INTO V_SYS_DATE ; +RETURN v_sys_date ; +END $$ +update t1 S +JOIN +( +SELECT CASE +WHEN DATE_FORMAT( f1('F01', NOW()) , '%Y%m%d') <= CLOSE_YMD +THEN '99991231' + ELSE '' END ACCOUNT_APPLY_YYYYMMDD +FROM ( +select case +when 'AP'='AP' + then ap_close_to +end AS CLOSE_YMD +from t2 +) A +) X +SET S.CLOSE_YN = '' +where 1=1; +drop function if exists f1; +drop table t1,t2; # End of 5.5 test # # MDEV-7040: Crash in field_conv, memcpy_field_possible, part#2 diff --git a/mysql-test/r/subselect.result b/mysql-test/r/subselect.result index a58d35708fe..623c28ff7ca 100644 --- a/mysql-test/r/subselect.result +++ b/mysql-test/r/subselect.result @@ -7102,6 +7102,17 @@ a DROP TABLE t1; SET SESSION big_tables=0; # +# MDEV-10776: Server crash on query +# +create table t1 (field1 int); +insert into t1 values (1); +select round((select 1 from t1 limit 1)) +from t1 +group by round((select 1 from t1 limit 1)); +round((select 1 from t1 limit 1)) +1 +drop table t1; +# # MDEV-7930: Assertion `table_share->tmp_table != NO_TMP_TABLE || # m_lock_type != 2' failed in handler::ha_index_read_map # @@ -7125,3 +7136,14 @@ sq NULL drop view v2; drop table t1,t2; +# +# MDEV-10386 Assertion `fixed == 1' failed in virtual String* Item_func_conv_charset::val_str(String*) +# +CREATE TABLE t1 (f1 CHAR(3) CHARACTER SET utf8 NULL, f2 CHAR(3) CHARACTER SET latin1 NULL); +INSERT INTO t1 VALUES ('foo','bar'); +SELECT * FROM t1 WHERE f2 >= SOME ( SELECT f1 FROM t1 ); +f1 f2 +SELECT * FROM t1 WHERE f2 <= SOME ( SELECT f1 FROM t1 ); +f1 f2 +foo bar +DROP TABLE t1; diff --git a/mysql-test/r/subselect2.result b/mysql-test/r/subselect2.result index 034cba58c8f..401d333ccfb 100644 --- a/mysql-test/r/subselect2.result +++ b/mysql-test/r/subselect2.result @@ -348,4 +348,49 @@ where t1.a = t2.a and ( t1.a = ( select min(a) from t1 ) or 0 ); a a a FRA FRA FRA drop table t1,t2,t3; +# +# MDEV-10148: Database crashes in the query to the View +# +CREATE TABLE t1 ( +key_code INT(11) NOT NULL, +value_string VARCHAR(50) NULL DEFAULT NULL, +PRIMARY KEY (key_code) +) COLLATE='utf8_general_ci' ENGINE=InnoDB ; +CREATE TABLE t2 ( +key_code INT(11) NOT NULL, +target_date DATE NULL DEFAULT NULL, +PRIMARY KEY (key_code) +) COLLATE='utf8_general_ci' ENGINE=InnoDB ; +CREATE TABLE t3 ( +now_date DATE NOT NULL, +PRIMARY KEY (now_date) +) COLLATE='utf8_general_ci' ENGINE=InnoDB ; +CREATE VIEW v1 +AS +SELECT +B.key_code, +B.target_date +FROM +t2 B INNER JOIN t3 C ON +B.target_date = C.now_date +; +SET @s = 'SELECT A.* FROM t1 A WHERE A.key_code IN (SELECT key_code FROM v1)'; +PREPARE stmt FROM @s; +EXECUTE stmt; +key_code value_string +EXECUTE stmt; +key_code value_string +DEALLOCATE PREPARE stmt; +DROP VIEW v1; +DROP TABLE t1,t2,t3; set optimizer_switch=@subselect2_test_tmp; +create table t1 (a int); +create table t2 (a int); +create table t3(a int); +insert into t1 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +insert into t2 select a from t1; +insert into t3 select a from t1; +select null in (select a from t1 where a < out3.a union select a from t2 where +(select a from t3) +1 < out3.a+1) from t3 out3; +ERROR 21000: Subquery returns more than 1 row +drop table t1, t2, t3; diff --git a/mysql-test/r/subselect4.result b/mysql-test/r/subselect4.result index a909230121d..6e4e8abfeb5 100644 --- a/mysql-test/r/subselect4.result +++ b/mysql-test/r/subselect4.result @@ -2397,5 +2397,44 @@ SELECT x FROM t1 WHERE id > (SELECT MAX(id) - 1000 FROM t1) ORDER BY x LIMIT 1; x 0 drop table t1; +# +# MDEV-7691: Assertion `outer_context || !*from_field || *from_field == not_found_field' ... +# +set optimizer_switch=default; +CREATE TABLE t1 (a INT) ENGINE=MyISAM; +INSERT INTO t1 VALUES (4),(6); +CREATE TABLE t2 (b INT) ENGINE=MyISAM; +INSERT INTO t2 VALUES (1),(8); +PREPARE stmt FROM " +SELECT * FROM t2 +HAVING 0 IN ( + SELECT a FROM t1 + WHERE a IN ( + SELECT a FROM t1 + WHERE b = a + ) +) +"; +EXECUTE stmt; +b +EXECUTE stmt; +b +# Alternative test case, without HAVING +CREATE TABLE t3 (i INT) ENGINE=MyISAM; +INSERT INTO t3 VALUES (4),(6); +PREPARE stmt FROM " +SELECT * FROM t3 AS t10 +WHERE EXISTS ( + SELECT * FROM t3 AS t20 WHERE t10.i IN ( + SELECT i FROM t3 + ) +)"; +EXECUTE stmt; +i +6 +EXECUTE stmt; +i +6 +drop table t1, t2, t3; SET optimizer_switch= @@global.optimizer_switch; set @@tmp_table_size= @@global.tmp_table_size; diff --git a/mysql-test/r/subselect_no_exists_to_in.result b/mysql-test/r/subselect_no_exists_to_in.result index 8b2d12c50f6..8c0e6041a90 100644 --- a/mysql-test/r/subselect_no_exists_to_in.result +++ b/mysql-test/r/subselect_no_exists_to_in.result @@ -7102,6 +7102,17 @@ a DROP TABLE t1; SET SESSION big_tables=0; # +# MDEV-10776: Server crash on query +# +create table t1 (field1 int); +insert into t1 values (1); +select round((select 1 from t1 limit 1)) +from t1 +group by round((select 1 from t1 limit 1)); +round((select 1 from t1 limit 1)) +1 +drop table t1; +# # MDEV-7930: Assertion `table_share->tmp_table != NO_TMP_TABLE || # m_lock_type != 2' failed in handler::ha_index_read_map # @@ -7125,6 +7136,17 @@ sq NULL drop view v2; drop table t1,t2; +# +# MDEV-10386 Assertion `fixed == 1' failed in virtual String* Item_func_conv_charset::val_str(String*) +# +CREATE TABLE t1 (f1 CHAR(3) CHARACTER SET utf8 NULL, f2 CHAR(3) CHARACTER SET latin1 NULL); +INSERT INTO t1 VALUES ('foo','bar'); +SELECT * FROM t1 WHERE f2 >= SOME ( SELECT f1 FROM t1 ); +f1 f2 +SELECT * FROM t1 WHERE f2 <= SOME ( SELECT f1 FROM t1 ); +f1 f2 +foo bar +DROP TABLE t1; set optimizer_switch=default; select @@optimizer_switch like '%exists_to_in=off%'; @@optimizer_switch like '%exists_to_in=off%' diff --git a/mysql-test/r/subselect_no_mat.result b/mysql-test/r/subselect_no_mat.result index 6051a58c3be..5e2b4f2d374 100644 --- a/mysql-test/r/subselect_no_mat.result +++ b/mysql-test/r/subselect_no_mat.result @@ -7095,6 +7095,17 @@ a DROP TABLE t1; SET SESSION big_tables=0; # +# MDEV-10776: Server crash on query +# +create table t1 (field1 int); +insert into t1 values (1); +select round((select 1 from t1 limit 1)) +from t1 +group by round((select 1 from t1 limit 1)); +round((select 1 from t1 limit 1)) +1 +drop table t1; +# # MDEV-7930: Assertion `table_share->tmp_table != NO_TMP_TABLE || # m_lock_type != 2' failed in handler::ha_index_read_map # @@ -7118,6 +7129,17 @@ sq NULL drop view v2; drop table t1,t2; +# +# MDEV-10386 Assertion `fixed == 1' failed in virtual String* Item_func_conv_charset::val_str(String*) +# +CREATE TABLE t1 (f1 CHAR(3) CHARACTER SET utf8 NULL, f2 CHAR(3) CHARACTER SET latin1 NULL); +INSERT INTO t1 VALUES ('foo','bar'); +SELECT * FROM t1 WHERE f2 >= SOME ( SELECT f1 FROM t1 ); +f1 f2 +SELECT * FROM t1 WHERE f2 <= SOME ( SELECT f1 FROM t1 ); +f1 f2 +foo bar +DROP TABLE t1; set optimizer_switch=default; select @@optimizer_switch like '%materialization=on%'; @@optimizer_switch like '%materialization=on%' diff --git a/mysql-test/r/subselect_no_opts.result b/mysql-test/r/subselect_no_opts.result index 718cf8b6099..afe79f80015 100644 --- a/mysql-test/r/subselect_no_opts.result +++ b/mysql-test/r/subselect_no_opts.result @@ -7093,6 +7093,17 @@ a DROP TABLE t1; SET SESSION big_tables=0; # +# MDEV-10776: Server crash on query +# +create table t1 (field1 int); +insert into t1 values (1); +select round((select 1 from t1 limit 1)) +from t1 +group by round((select 1 from t1 limit 1)); +round((select 1 from t1 limit 1)) +1 +drop table t1; +# # MDEV-7930: Assertion `table_share->tmp_table != NO_TMP_TABLE || # m_lock_type != 2' failed in handler::ha_index_read_map # @@ -7116,4 +7127,15 @@ sq NULL drop view v2; drop table t1,t2; +# +# MDEV-10386 Assertion `fixed == 1' failed in virtual String* Item_func_conv_charset::val_str(String*) +# +CREATE TABLE t1 (f1 CHAR(3) CHARACTER SET utf8 NULL, f2 CHAR(3) CHARACTER SET latin1 NULL); +INSERT INTO t1 VALUES ('foo','bar'); +SELECT * FROM t1 WHERE f2 >= SOME ( SELECT f1 FROM t1 ); +f1 f2 +SELECT * FROM t1 WHERE f2 <= SOME ( SELECT f1 FROM t1 ); +f1 f2 +foo bar +DROP TABLE t1; set @optimizer_switch_for_subselect_test=null; diff --git a/mysql-test/r/subselect_no_scache.result b/mysql-test/r/subselect_no_scache.result index 2b84f569fc5..15f7b977c24 100644 --- a/mysql-test/r/subselect_no_scache.result +++ b/mysql-test/r/subselect_no_scache.result @@ -7108,6 +7108,17 @@ a DROP TABLE t1; SET SESSION big_tables=0; # +# MDEV-10776: Server crash on query +# +create table t1 (field1 int); +insert into t1 values (1); +select round((select 1 from t1 limit 1)) +from t1 +group by round((select 1 from t1 limit 1)); +round((select 1 from t1 limit 1)) +1 +drop table t1; +# # MDEV-7930: Assertion `table_share->tmp_table != NO_TMP_TABLE || # m_lock_type != 2' failed in handler::ha_index_read_map # @@ -7131,6 +7142,17 @@ sq NULL drop view v2; drop table t1,t2; +# +# MDEV-10386 Assertion `fixed == 1' failed in virtual String* Item_func_conv_charset::val_str(String*) +# +CREATE TABLE t1 (f1 CHAR(3) CHARACTER SET utf8 NULL, f2 CHAR(3) CHARACTER SET latin1 NULL); +INSERT INTO t1 VALUES ('foo','bar'); +SELECT * FROM t1 WHERE f2 >= SOME ( SELECT f1 FROM t1 ); +f1 f2 +SELECT * FROM t1 WHERE f2 <= SOME ( SELECT f1 FROM t1 ); +f1 f2 +foo bar +DROP TABLE t1; set optimizer_switch=default; select @@optimizer_switch like '%subquery_cache=on%'; @@optimizer_switch like '%subquery_cache=on%' diff --git a/mysql-test/r/subselect_no_semijoin.result b/mysql-test/r/subselect_no_semijoin.result index a2841793f15..1c5578b3575 100644 --- a/mysql-test/r/subselect_no_semijoin.result +++ b/mysql-test/r/subselect_no_semijoin.result @@ -7093,6 +7093,17 @@ a DROP TABLE t1; SET SESSION big_tables=0; # +# MDEV-10776: Server crash on query +# +create table t1 (field1 int); +insert into t1 values (1); +select round((select 1 from t1 limit 1)) +from t1 +group by round((select 1 from t1 limit 1)); +round((select 1 from t1 limit 1)) +1 +drop table t1; +# # MDEV-7930: Assertion `table_share->tmp_table != NO_TMP_TABLE || # m_lock_type != 2' failed in handler::ha_index_read_map # @@ -7116,5 +7127,16 @@ sq NULL drop view v2; drop table t1,t2; +# +# MDEV-10386 Assertion `fixed == 1' failed in virtual String* Item_func_conv_charset::val_str(String*) +# +CREATE TABLE t1 (f1 CHAR(3) CHARACTER SET utf8 NULL, f2 CHAR(3) CHARACTER SET latin1 NULL); +INSERT INTO t1 VALUES ('foo','bar'); +SELECT * FROM t1 WHERE f2 >= SOME ( SELECT f1 FROM t1 ); +f1 f2 +SELECT * FROM t1 WHERE f2 <= SOME ( SELECT f1 FROM t1 ); +f1 f2 +foo bar +DROP TABLE t1; set @optimizer_switch_for_subselect_test=null; set @join_cache_level_for_subselect_test=NULL; diff --git a/mysql-test/r/type_decimal.result b/mysql-test/r/type_decimal.result index cde8816dee4..7ff74c8270a 100644 --- a/mysql-test/r/type_decimal.result +++ b/mysql-test/r/type_decimal.result @@ -221,7 +221,6 @@ drop table t1; create table t1 (a decimal(10,2) unsigned); insert into t1 values ("0.0"),("-0.0"),("+0.0"),("01.0"),("+01.0"),("-01.0"); Warnings: -Warning 1264 Out of range value for column 'a' at row 2 Warning 1264 Out of range value for column 'a' at row 6 insert into t1 values ("-.1"),("+.1"),(".1"); Warnings: @@ -280,7 +279,6 @@ drop table t1; create table t1 (a decimal(10,2) zerofill); insert into t1 values ("0.0"),("-0.0"),("+0.0"),("01.0"),("+01.0"),("-01.0"); Warnings: -Warning 1264 Out of range value for column 'a' at row 2 Warning 1264 Out of range value for column 'a' at row 6 insert into t1 values ("-.1"),("+.1"),(".1"); Warnings: @@ -1012,6 +1010,9 @@ SELECT COLUMN_NAME, DATA_TYPE, COLUMN_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE COLUMN_NAME DATA_TYPE COLUMN_TYPE a decimal decimal(10,2)/*old*/ DROP TABLE t1dec102; +select cast('-0.0' as decimal(5,1)) < 0; +cast('-0.0' as decimal(5,1)) < 0 +0 # # End of 5.5 tests # diff --git a/mysql-test/r/type_ranges.result b/mysql-test/r/type_ranges.result index 53fc3104666..ad52ab5da7d 100644 --- a/mysql-test/r/type_ranges.result +++ b/mysql-test/r/type_ranges.result @@ -86,10 +86,10 @@ t1 1 options 2 flags A NULL NULL NULL BTREE CREATE UNIQUE INDEX test on t1 ( auto ) ; CREATE INDEX test2 on t1 ( ulonglong,ulong) ; Warnings: -Note 1831 Duplicate index 'test2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `test2`. This is deprecated and will be disallowed in a future release. CREATE INDEX test3 on t1 ( medium ) ; Warnings: -Note 1831 Duplicate index 'test3' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `test3`. This is deprecated and will be disallowed in a future release. DROP INDEX test ON t1; insert into t1 values (10, 1,1,1,1,1,1,1,1,1,1,1,1,1,NULL,0,0,0,1,1,1,1,'one','one'); insert into t1 values (NULL,2,2,2,2,2,2,2,2,2,2,2,2,2,NULL,NULL,NULL,NULL,NULL,NULL,2,2,'two','two,one'); @@ -308,7 +308,7 @@ const int(1) NULL NO 0 # drop table t1,t2,t3; create table t1 ( myfield INT NOT NULL, UNIQUE INDEX (myfield), unique (myfield), index(myfield)); Warnings: -Note 1831 Duplicate index 'myfield_2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `myfield_2`. This is deprecated and will be disallowed in a future release. drop table t1; create table t1 ( id integer unsigned not null primary key ); create table t2 ( id integer unsigned not null primary key ); diff --git a/mysql-test/r/union.result b/mysql-test/r/union.result index 40f5a77e3d0..5a6cd8907e9 100644 --- a/mysql-test/r/union.result +++ b/mysql-test/r/union.result @@ -362,7 +362,7 @@ a 2 select found_rows(); found_rows() -6 +5 SELECT SQL_CALC_FOUND_ROWS * FROM t1 UNION SELECT * FROM t2 LIMIT 100; a 1 @@ -1169,12 +1169,9 @@ a b select * from ((select * from t1 limit 1) union (select * from t1 limit 1)) a; a b 1 a -2 b select * from ((select * from t1 limit 1) union (select * from t1 limit 1) union (select * from t1 limit 1)) a; a b 1 a -2 b -3 c select * from ((((select * from t1))) union (select * from t1) union (select * from t1)) a; a b 1 a @@ -1553,7 +1550,6 @@ NULL UNION RESULT <union1,2> ALL NULL NULL NULL NULL NULL NULL Using filesort Warnings: Note 1003 select NULL AS `a` from `test`.`t1` union select NULL AS `a` from `test`.`t1` order by `a` DROP TABLE t1; -End of 5.0 tests # # Bug#32858: Error: "Incorrect usage of UNION and INTO" does not take # subselects into account @@ -1659,6 +1655,14 @@ a 4 5 6 +(select a from t1 where false) UNION (select a from t1) limit 8; +a +10 +2 +3 +4 +5 +6 7 8 drop table t1; @@ -1955,3 +1959,22 @@ cccc bbbb dddd drop table t1; +# +# MDEV-10172: UNION query returns incorrect rows outside +# conditional evaluation +# +create table t1 (d datetime not null primary key); +insert into t1(d) values ('2016-06-01'),('2016-06-02'),('2016-06-03'),('2016-06-04'); +select * from +( +select * from t1 where d between '2016-06-02' and '2016-06-05' + union +(select * from t1 where d < '2016-06-05' order by d desc limit 1) +) onlyJun2toJun4 +order by d; +d +2016-06-02 00:00:00 +2016-06-03 00:00:00 +2016-06-04 00:00:00 +drop table t1; +End of 5.0 tests diff --git a/mysql-test/r/view.result b/mysql-test/r/view.result index 924b3a11fef..6530fd10a6a 100644 --- a/mysql-test/r/view.result +++ b/mysql-test/r/view.result @@ -5799,6 +5799,31 @@ f1 f2 drop table t1, t2; SELECT 1 FROM (SELECT 1 as a) AS b HAVING (SELECT `SOME_GARBAGE`.b.a)=1; ERROR 42S22: Unknown column 'SOME_GARBAGE.b.a' in 'field list' +# +# MDEV-10035: DBUG_ASSERT on CREATE VIEW v1 AS SELECT * FROM t1 +# FOR UPDATE +# +CREATE TABLE t1 (a INT); +insert into t1 values (1),(2); +CREATE VIEW v1 AS SELECT * FROM t1 FOR UPDATE; +SHOW CREATE VIEW v1; +View Create View character_set_client collation_connection +v1 CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v1` AS select `t1`.`a` AS `a` from `t1` for update latin1 latin1_swedish_ci +select * from v1; +a +1 +2 +DROP VIEW v1; +CREATE VIEW v1 AS SELECT * FROM t1 LOCK IN SHARE MODE; +SHOW CREATE VIEW v1; +View Create View character_set_client collation_connection +v1 CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v1` AS select `t1`.`a` AS `a` from `t1` lock in share mode latin1 latin1_swedish_ci +select * from v1; +a +1 +2 +DROP VIEW v1; +DROP TABLE t1; # ----------------------------------------------------------------- # -- End of 10.0 tests. # ----------------------------------------------------------------- diff --git a/mysql-test/std_data/bug20683959loaddata.txt b/mysql-test/std_data/bug20683959loaddata.txt deleted file mode 100644 index 1878cc78879..00000000000 --- a/mysql-test/std_data/bug20683959loaddata.txt +++ /dev/null @@ -1 +0,0 @@ -Ã"RT @niouzechun: é˜âˆšõ€®ç¹ä¸Šãƒ£ç¹æ–õ€‡³ç¹§ï½¨ç¹ï½³ç¹ç‰™è€³ç¸ºï½ªç¹§è–™â–¡ç¸ºä»£ï½Œç¸ºï½©ç¸²âˆšã„ç¹ï½³ç¹ä¸Šãƒ£ç¹æ–õ€‡³ç¹§ï½¨ç¹ï½³ç¹å³¨ï½„è« ï½¨èœ‰õ€”Žå™ªç¸ºï½ªç¸ºé¡˜ï½©ï½±ç¹§åµâ‰ 縺ｾ繧顔ｩ肴･ｵ逧õ€‹–↓鞫ょå™ç¸ºåŠ±â†‘縺õ€‹šç‚Šé€•ï½±ç¸ºï½¯ç¸²âˆ«æ¨Ÿèž³æº˜õ€èŽ ｺ逕溘õ€®è“ｺè›ï½¬é€§õ€‹–↓縺õ€‘Žâˆªç¸ºä¸Šï¼žç¸ºä¹â†‘縺õ€‹–ï¼ è³æ¦Šï½¹ï½³é²å³¨â–¡ç¸ºç¤¼ç‚Šè³æ¦Šï½°ï½½ç¸º 縺苓ｾ帙＞ diff --git a/mysql-test/std_data/loaddata/mdev-11079.txt b/mysql-test/std_data/loaddata/mdev-11079.txt new file mode 100644 index 00000000000..a792f984d5f --- /dev/null +++ b/mysql-test/std_data/loaddata/mdev-11079.txt @@ -0,0 +1 @@ +"%ª«¬" diff --git a/mysql-test/std_data/loaddata/mdev-11343.txt b/mysql-test/std_data/loaddata/mdev-11343.txt new file mode 100644 index 00000000000..dded1215ffa --- /dev/null +++ b/mysql-test/std_data/loaddata/mdev-11343.txt @@ -0,0 +1,12 @@ +\ä +\äx +x\ä +x\äx +\ê™ +\ê™ x +x\ê™ +x\ê™ x +\😎 +\😎x +x\😎 +x\😎x diff --git a/mysql-test/std_data/loaddata/mdev-11631.txt b/mysql-test/std_data/loaddata/mdev-11631.txt new file mode 100644 index 00000000000..87b824b71ae --- /dev/null +++ b/mysql-test/std_data/loaddata/mdev-11631.txt @@ -0,0 +1 @@ +\ä diff --git a/mysql-test/suite/archive/discover.result b/mysql-test/suite/archive/discover.result index e1ca9cb6a65..0619ca2051a 100644 --- a/mysql-test/suite/archive/discover.result +++ b/mysql-test/suite/archive/discover.result @@ -139,3 +139,10 @@ flush tables; create table t1 (a int) engine=archive; ERROR 42S01: Table 't1' already exists drop table t1; +CREATE OR REPLACE TABLE t1 ( pk INT AUTO_INCREMENT PRIMARY KEY ) ENGINE = ARCHIVE; +CREATE OR REPLACE TABLE t1 ( pk INT AUTO_INCREMENT PRIMARY KEY ) ENGINE = ARCHIVE; +DROP TABLE t1; +CREATE OR REPLACE TABLE t1 ( pk INT AUTO_INCREMENT PRIMARY KEY ) ENGINE = ARCHIVE; +SELECT * FROM t1; +pk +DROP TABLE t1; diff --git a/mysql-test/suite/archive/discover.test b/mysql-test/suite/archive/discover.test index 20cb69efa00..4ab35cf1115 100644 --- a/mysql-test/suite/archive/discover.test +++ b/mysql-test/suite/archive/discover.test @@ -132,3 +132,13 @@ flush tables; create table t1 (a int) engine=archive; drop table t1; +# +# MDEV-11317: Error in deleting non existing .frm for tables with disocvery +# + +CREATE OR REPLACE TABLE t1 ( pk INT AUTO_INCREMENT PRIMARY KEY ) ENGINE = ARCHIVE; +CREATE OR REPLACE TABLE t1 ( pk INT AUTO_INCREMENT PRIMARY KEY ) ENGINE = ARCHIVE; +DROP TABLE t1; +CREATE OR REPLACE TABLE t1 ( pk INT AUTO_INCREMENT PRIMARY KEY ) ENGINE = ARCHIVE; +SELECT * FROM t1; +DROP TABLE t1; diff --git a/mysql-test/suite/binlog/r/binlog_index.result b/mysql-test/suite/binlog/r/binlog_index.result index 8cdca861737..bb5d9ff74f1 100644 --- a/mysql-test/suite/binlog/r/binlog_index.result +++ b/mysql-test/suite/binlog/r/binlog_index.result @@ -1,9 +1,9 @@ call mtr.add_suppression('Attempting backtrace'); -call mtr.add_suppression('MSYQL_BIN_LOG::purge_logs failed to process registered files that would be purged.'); -call mtr.add_suppression('MSYQL_BIN_LOG::open failed to sync the index file'); +call mtr.add_suppression('MYSQL_BIN_LOG::purge_logs failed to process registered files that would be purged.'); +call mtr.add_suppression('MYSQL_BIN_LOG::open failed to sync the index file'); call mtr.add_suppression('Turning logging off for the whole duration of the MySQL server process.'); call mtr.add_suppression('Could not open .*'); -call mtr.add_suppression('MSYQL_BIN_LOG::purge_logs failed to clean registers before purging logs.'); +call mtr.add_suppression('MYSQL_BIN_LOG::purge_logs failed to clean registers before purging logs.'); flush tables; RESET MASTER; flush logs; diff --git a/mysql-test/suite/binlog/t/binlog_index.test b/mysql-test/suite/binlog/t/binlog_index.test index a264549ce17..09e817c0469 100644 --- a/mysql-test/suite/binlog/t/binlog_index.test +++ b/mysql-test/suite/binlog/t/binlog_index.test @@ -9,11 +9,11 @@ source include/have_debug.inc; # Avoid CrashReporter popup on Mac --source include/not_crashrep.inc call mtr.add_suppression('Attempting backtrace'); -call mtr.add_suppression('MSYQL_BIN_LOG::purge_logs failed to process registered files that would be purged.'); -call mtr.add_suppression('MSYQL_BIN_LOG::open failed to sync the index file'); +call mtr.add_suppression('MYSQL_BIN_LOG::purge_logs failed to process registered files that would be purged.'); +call mtr.add_suppression('MYSQL_BIN_LOG::open failed to sync the index file'); call mtr.add_suppression('Turning logging off for the whole duration of the MySQL server process.'); call mtr.add_suppression('Could not open .*'); -call mtr.add_suppression('MSYQL_BIN_LOG::purge_logs failed to clean registers before purging logs.'); +call mtr.add_suppression('MYSQL_BIN_LOG::purge_logs failed to clean registers before purging logs.'); flush tables; let $old=`select @@debug`; diff --git a/mysql-test/suite/engines/iuds/r/insert_decimal.result b/mysql-test/suite/engines/iuds/r/insert_decimal.result index 50fde80d81d..eab8592c4ee 100644 --- a/mysql-test/suite/engines/iuds/r/insert_decimal.result +++ b/mysql-test/suite/engines/iuds/r/insert_decimal.result @@ -48,7 +48,6 @@ Warning 1264 Out of range value for column 'c2' at row 1 Note 1265 Data truncated for column 'c3' at row 1 insert into t2 values ("0.0","0.0","0.0",7),("-0.0","-0.0","-0.0",8),("+0.0","+0.0","+0.0",9),("01.0","01.0","01.0",10),("+01.0","+01.0","+01.0",11),("-01.0","-01.0","-01.0",12); Warnings: -Warning 1264 Out of range value for column 'c2' at row 2 Warning 1264 Out of range value for column 'c2' at row 6 insert into t2 values ("-.1","-.1","-.1",13),("+.1","+.1","+.1",14),(".1",".1",".1",15); Warnings: diff --git a/mysql-test/suite/funcs_2/t/innodb_charset.test b/mysql-test/suite/funcs_2/t/innodb_charset.test index b77bacfc01c..da4dea44ad7 100644 --- a/mysql-test/suite/funcs_2/t/innodb_charset.test +++ b/mysql-test/suite/funcs_2/t/innodb_charset.test @@ -6,6 +6,7 @@ # Checking of other prerequisites is in charset_master.test # ################################################################################ +--source include/no_valgrind_without_big.inc --source include/have_innodb.inc let $engine_type= InnoDB; diff --git a/mysql-test/suite/innodb/r/innodb-index.result b/mysql-test/suite/innodb/r/innodb-index.result index 996bc45f7e2..3d5a0f840c1 100644 --- a/mysql-test/suite/innodb/r/innodb-index.result +++ b/mysql-test/suite/innodb/r/innodb-index.result @@ -64,7 +64,7 @@ alter table t1 add unique index (c), add index (d); affected rows: 0 info: Records: 0 Duplicates: 0 Warnings: 1 Warnings: -Note 1831 Duplicate index 'd' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `d`. This is deprecated and will be disallowed in a future release. show create table t1; Table Create Table t1 CREATE TABLE `t1` ( diff --git a/mysql-test/suite/innodb/r/innodb-wl5522-debug-zip.result b/mysql-test/suite/innodb/r/innodb-wl5522-debug-zip.result index 0e863f5849e..3ce55ef0409 100644 --- a/mysql-test/suite/innodb/r/innodb-wl5522-debug-zip.result +++ b/mysql-test/suite/innodb/r/innodb-wl5522-debug-zip.result @@ -1,3 +1,6 @@ +call mtr.add_suppression("InnoDB: Page for tablespace .* "); +call mtr.add_suppression("InnoSB: Warning: database page corruption or a failed .*"); +FLUSH TABLES; SET GLOBAL innodb_file_per_table = 1; SELECT @@innodb_file_per_table; @@innodb_file_per_table diff --git a/mysql-test/suite/innodb/r/innodb.result b/mysql-test/suite/innodb/r/innodb.result index f8020e080e6..cded1f98fb3 100644 --- a/mysql-test/suite/innodb/r/innodb.result +++ b/mysql-test/suite/innodb/r/innodb.result @@ -405,7 +405,7 @@ drop table t1; CREATE TABLE t1 (a int not null, b int not null,c int not null, key(a),primary key(a,b), unique(c),key(a),unique(b)); Warnings: -Note 1831 Duplicate index 'a_2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a_2`. This is deprecated and will be disallowed in a future release. show index from t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment t1 0 PRIMARY 1 a A # NULL NULL BTREE @@ -1445,7 +1445,7 @@ t2 CREATE TABLE `t2` ( ) ENGINE=InnoDB DEFAULT CHARSET=latin1 create index id2 on t2 (id); Warnings: -Note 1831 Duplicate index 'id2' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `id2`. This is deprecated and will be disallowed in a future release. show create table t2; Table Create Table t2 CREATE TABLE `t2` ( @@ -1851,7 +1851,7 @@ alter table t1 add unique(v); ERROR 23000: Duplicate entry '{ ' for key 'v_2' alter table t1 add key(v); Warnings: -Note 1831 Duplicate index 'v_2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `v_2`. This is deprecated and will be disallowed in a future release. select concat('*',v,'*',c,'*',t,'*') as qq from t1 where v='a'; qq *a*a*a* diff --git a/mysql-test/suite/innodb/r/innodb_bug51378.result b/mysql-test/suite/innodb/r/innodb_bug51378.result index a2a9bcad564..08b6b656085 100644 --- a/mysql-test/suite/innodb/r/innodb_bug51378.result +++ b/mysql-test/suite/innodb/r/innodb_bug51378.result @@ -5,7 +5,7 @@ col3 time not null) engine = innodb; create unique index idx on bug51378(col1, col2(31)); alter table bug51378 add unique index idx2(col1, col2(31)); Warnings: -Note 1831 Duplicate index 'idx2' defined on the table 'test.bug51378'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `idx2`. This is deprecated and will be disallowed in a future release. create unique index idx3 on bug51378(col1, col3); SHOW CREATE TABLE bug51378; Table Create Table diff --git a/mysql-test/suite/innodb/r/table_index_statistics.result b/mysql-test/suite/innodb/r/table_index_statistics.result new file mode 100644 index 00000000000..286c5f9325f --- /dev/null +++ b/mysql-test/suite/innodb/r/table_index_statistics.result @@ -0,0 +1,48 @@ +SET @default_storage_engine_old = @@session.default_storage_engine; +SET SESSION default_storage_engine = INNODB; +FLUSH INDEX_STATISTICS; +FLUSH TABLE_STATISTICS; +SET @userstat_old= @@userstat; +SET GLOBAL userstat=ON; +CREATE TABLE t1 (id int(10), PRIMARY KEY (id)); +INSERT INTO t1 VALUES (1),(2),(3),(4),(5),(6),(7),(8),(9),(10); +SELECT COUNT(*) FROM t1; +COUNT(*) +10 +SELECT ROWS_READ FROM INFORMATION_SCHEMA.TABLE_STATISTICS WHERE TABLE_NAME='t1'; +ROWS_READ +10 +SELECT ROWS_READ FROM INFORMATION_SCHEMA.INDEX_STATISTICS WHERE TABLE_NAME='t1'; +ROWS_READ +10 +FLUSH TABLE_STATISTICS; +SELECT ROWS_READ FROM INFORMATION_SCHEMA.TABLE_STATISTICS WHERE TABLE_NAME='t1'; +ROWS_READ +SELECT ROWS_READ FROM INFORMATION_SCHEMA.INDEX_STATISTICS WHERE TABLE_NAME='t1'; +ROWS_READ +10 +FLUSH INDEX_STATISTICS; +SELECT ROWS_READ FROM INFORMATION_SCHEMA.INDEX_STATISTICS WHERE TABLE_NAME='t1'; +ROWS_READ +SELECT COUNT(*) FROM t1; +COUNT(*) +10 +SELECT ROWS_READ FROM INFORMATION_SCHEMA.TABLE_STATISTICS WHERE TABLE_NAME='t1'; +ROWS_READ +10 +SELECT ROWS_READ FROM INFORMATION_SCHEMA.INDEX_STATISTICS WHERE TABLE_NAME='t1'; +ROWS_READ +10 +DROP TABLE t1; +CREATE TABLE t2 (c1 INT UNSIGNED); +ALTER TABLE t2 MODIFY c1 FLOAT; +SELECT * FROM INFORMATION_SCHEMA.TABLE_STATISTICS WHERE TABLE_NAME='t2'; +TABLE_SCHEMA TABLE_NAME ROWS_READ ROWS_CHANGED ROWS_CHANGED_X_INDEXES +DROP TABLE t2; +CREATE TABLE t2 (c1 INT UNSIGNED); +ALTER TABLE t2 MODIFY c1 FLOAT; +SELECT * FROM INFORMATION_SCHEMA.TABLE_STATISTICS WHERE TABLE_NAME='t2'; +TABLE_SCHEMA TABLE_NAME ROWS_READ ROWS_CHANGED ROWS_CHANGED_X_INDEXES +DROP TABLE t2; +SET GLOBAL userstat= @userstat_old; +SET SESSION default_storage_engine = @default_storage_engine_old; diff --git a/mysql-test/suite/innodb/t/innodb-wl5522-debug-zip.test b/mysql-test/suite/innodb/t/innodb-wl5522-debug-zip.test index 4b03ac008d2..37b630ad47b 100644 --- a/mysql-test/suite/innodb/t/innodb-wl5522-debug-zip.test +++ b/mysql-test/suite/innodb/t/innodb-wl5522-debug-zip.test @@ -17,6 +17,10 @@ # allow test to run only when innodb-page-size=16 --source include/have_innodb_16k.inc +# Test intentionally crashes the server, corrupted pages possible +call mtr.add_suppression("InnoDB: Page for tablespace .* "); +call mtr.add_suppression("InnoSB: Warning: database page corruption or a failed .*"); +FLUSH TABLES; let MYSQLD_DATADIR =`SELECT @@datadir`; let $innodb_file_per_table = `SELECT @@innodb_file_per_table`; diff --git a/mysql-test/suite/innodb/t/table_index_statistics.test b/mysql-test/suite/innodb/t/table_index_statistics.test new file mode 100644 index 00000000000..af6f1946486 --- /dev/null +++ b/mysql-test/suite/innodb/t/table_index_statistics.test @@ -0,0 +1,8 @@ +--source include/have_innodb.inc + +SET @default_storage_engine_old = @@session.default_storage_engine; +SET SESSION default_storage_engine = INNODB; + +--source extra/table_index_statistics.inc + +SET SESSION default_storage_engine = @default_storage_engine_old; diff --git a/mysql-test/suite/innodb_fts/r/innodb_fts_misc.result b/mysql-test/suite/innodb_fts/r/innodb_fts_misc.result index f6be36a24ef..628fe8519ed 100644 --- a/mysql-test/suite/innodb_fts/r/innodb_fts_misc.result +++ b/mysql-test/suite/innodb_fts/r/innodb_fts_misc.result @@ -1018,7 +1018,7 @@ CREATE TABLE `t21` (`a` text, `b` int not null, fulltext key (`a`), fulltext key (`a`) ) ENGINE=INNODB DEFAULT CHARSET=LATIN1; Warnings: -Note 1831 Duplicate index 'a_2' defined on the table 'test.t21'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a_2`. This is deprecated and will be disallowed in a future release. ALTER TABLE `t21` ADD UNIQUE INDEX (`b`), ALGORITHM=INPLACE; ERROR 0A000: ALGORITHM=INPLACE is not supported. Reason: InnoDB presently supports one FULLTEXT index creation at a time. Try ALGORITHM=COPY. ALTER TABLE `t21` ADD UNIQUE INDEX (`b`); diff --git a/mysql-test/suite/innodb_zip/r/innodb_index_large_prefix.result b/mysql-test/suite/innodb_zip/r/innodb_index_large_prefix.result index b610d1ed7b8..a02a4acb740 100644 --- a/mysql-test/suite/innodb_zip/r/innodb_index_large_prefix.result +++ b/mysql-test/suite/innodb_zip/r/innodb_index_large_prefix.result @@ -193,10 +193,10 @@ Level Code Message Warning 1071 Specified key was too long; max key length is 3072 bytes create index idx3 on worklog5743_8(a2(3072)); Warnings: -Note 1831 Duplicate index 'idx3' defined on the table 'test.worklog5743_8'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `idx3`. This is deprecated and will be disallowed in a future release. show warnings; Level Code Message -Note 1831 Duplicate index 'idx3' defined on the table 'test.worklog5743_8'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `idx3`. This is deprecated and will be disallowed in a future release. create index idx4 on worklog5743_8(a1, a2(3069)); ERROR 42000: Specified key was too long; max key length is 3072 bytes show warnings; @@ -229,10 +229,10 @@ Level Code Message Warning 1071 Specified key was too long; max key length is 3072 bytes create index idx3 on worklog5743_16(a2(3072)); Warnings: -Note 1831 Duplicate index 'idx3' defined on the table 'test.worklog5743_16'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `idx3`. This is deprecated and will be disallowed in a future release. show warnings; Level Code Message -Note 1831 Duplicate index 'idx3' defined on the table 'test.worklog5743_16'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `idx3`. This is deprecated and will be disallowed in a future release. create index idx4 on worklog5743_16(a1, a2(3069)); ERROR 42000: Specified key was too long; max key length is 3072 bytes show warnings; @@ -462,7 +462,7 @@ Warnings: Warning 1071 Specified key was too long; max key length is 3072 bytes create index idx2 on worklog5743(a(3072)); Warnings: -Note 1831 Duplicate index 'idx2' defined on the table 'test.worklog5743'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `idx2`. This is deprecated and will be disallowed in a future release. show create table worklog5743; Table Create Table worklog5743 CREATE TABLE `worklog5743` ( diff --git a/mysql-test/suite/maria/icp.result b/mysql-test/suite/maria/icp.result index 85a21ada47d..1c11072cbde 100644 --- a/mysql-test/suite/maria/icp.result +++ b/mysql-test/suite/maria/icp.result @@ -718,8 +718,8 @@ b INT, c INT, d DATE NOT NULL, e VARCHAR(1), KEY (c), KEY (d), KEY k2(b), KEY k3(b), KEY k4(b) ); Warnings: -Note 1831 Duplicate index 'k3' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'k4' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `k3`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `k4`. This is deprecated and will be disallowed in a future release. INSERT INTO t1 (b,c,d,e) VALUES (6,5,'2006-05-25','y'),(1,5,'2008-01-23','t'), (6,5,'2007-06-18','d'),(4,5,'1900-01-01','r'), diff --git a/mysql-test/suite/maria/maria.result b/mysql-test/suite/maria/maria.result index 952cdc09c90..d410944ddb2 100644 --- a/mysql-test/suite/maria/maria.result +++ b/mysql-test/suite/maria/maria.result @@ -1139,7 +1139,7 @@ alter table t1 add unique(v); ERROR 23000: Duplicate entry '{ ' for key 'v_2' alter table t1 add key(v); Warnings: -Note 1831 Duplicate index 'v_2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `v_2`. This is deprecated and will be disallowed in a future release. select concat('*',v,'*',c,'*',t,'*') as qq from t1 where v='a'; qq *a*a*a* @@ -2696,8 +2696,8 @@ SET aria_repair_threads=2; SET aria_sort_buffer_size=8192; CREATE TABLE t1(a CHAR(255), KEY(a), KEY(a), KEY(a)); Warnings: -Note 1831 Duplicate index 'a_2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. -Note 1831 Duplicate index 'a_3' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a_2`. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a_3`. This is deprecated and will be disallowed in a future release. INSERT INTO t1 VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9),(0),(1),(2),(3); REPAIR TABLE t1; Table Op Msg_type Msg_text diff --git a/mysql-test/suite/optimizer_unfixed_bugs/r/bug45221.result b/mysql-test/suite/optimizer_unfixed_bugs/r/bug45221.result index 470d06c341d..92918959cda 100644 --- a/mysql-test/suite/optimizer_unfixed_bugs/r/bug45221.result +++ b/mysql-test/suite/optimizer_unfixed_bugs/r/bug45221.result @@ -8,7 +8,7 @@ KEY `int_key` (`int_key`), KEY `varchar_key` (`int_key`) ) ENGINE=MyISAM AUTO_INCREMENT=30 DEFAULT CHARSET=latin1; Warnings: -Note 1831 Duplicate index 'varchar_key' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `varchar_key`. This is deprecated and will be disallowed in a future release. INSERT INTO `t2` VALUES (10,8,NULL,'2002-02-26 06:14:37'),(11,9,'2006-06-14','1900-01-01 00:00:00'),(12,9,'2002-09-12','2006-12-03 09:37:26'),(13,186,'2005-02-15','2008-05-26 12:27:10'),(14,NULL,NULL,'2004-12-14 16:37:30'),(15,2,'2008-11-04','2003-02-11 21:19:41'),(16,3,'2004-09-04','2009-10-18 02:27:49'),(17,0,'2006-06-05','2000-09-26 07:45:57'),(18,133,'1900-01-01',NULL),(19,1,'1900-01-01','2005-11-10 12:40:29'),(20,8,'1900-01-01','2009-04-25 00:00:00'),(21,5,'2005-01-13','2002-11-27 00:00:00'),(22,5,'2006-05-21','2004-01-26 20:32:32'),(23,8,'2003-09-08','2007-10-26 11:41:40'),(24,6,'2006-12-23','2005-10-07 00:00:00'),(25,51,'2006-10-15','2000-07-15 05:00:34'),(26,4,'2005-04-06','2000-04-03 16:33:32'),(27,7,'2008-04-07',NULL),(28,6,'2006-10-10','2001-04-25 01:26:12'),(29,4,'1900-01-01','2000-12-27 00:00:00'); CREATE TABLE t1 ( `pk` int(11) NOT NULL AUTO_INCREMENT, @@ -20,7 +20,7 @@ KEY `int_key` (`int_key`), KEY `varchar_key` (`int_key`) ) ENGINE=MyISAM AUTO_INCREMENT=21 DEFAULT CHARSET=latin1; Warnings: -Note 1831 Duplicate index 'varchar_key' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `varchar_key`. This is deprecated and will be disallowed in a future release. INSERT INTO t1 VALUES (1,2,NULL,'2004-10-11 18:13:16'),(2,9,'2001-09-19',NULL),(3,3,'2004-09-12','1900-01-01 00:00:00'),(4,9,NULL,'2009-07-25 00:00:00'),(5,NULL,'2002-07-19',NULL),(6,9,'2002-12-16','2008-07-27 00:00:00'),(7,3,'2006-02-08','2002-11-13 16:37:31'),(8,8,'2006-08-28','1900-01-01 00:00:00'),(9,8,'2001-04-14','2003-12-10 00:00:00'),(10,53,'2000-01-05','2001-12-21 22:38:22'),(11,0,'2003-12-06','2008-12-13 23:16:44'),(12,5,'1900-01-01','2005-08-15 12:39:41'),(13,166,'2002-11-27',NULL),(14,3,NULL,'2006-09-11 12:06:14'),(15,0,'2003-05-27','2007-12-15 12:39:34'),(16,1,'2005-05-03','2005-08-09 00:00:00'),(17,9,'2001-04-18','2001-09-02 22:50:02'),(18,5,'2005-12-27','2005-12-16 22:58:11'),(19,6,'2004-08-20','2007-04-19 00:19:53'),(20,2,'1900-01-01','1900-01-01 00:00:00'); SELECT `pk` FROM t1 OUTR @@ -77,7 +77,7 @@ KEY `datetime_key` (`datetime_key`), KEY `varchar_key` (`int_key`) ) ENGINE=MyISAM AUTO_INCREMENT=30 DEFAULT CHARSET=latin1; Warnings: -Note 1831 Duplicate index 'varchar_key' defined on the table 'test.t2'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `varchar_key`. This is deprecated and will be disallowed in a future release. INSERT INTO `t2` VALUES (10,7,8,NULL,'2002-02-26 06:14:37','2002-02-26 06:14:37'),(11,1,9,'2006-06-14','1900-01-01 00:00:00','1900-01-01 00:00:00'),(12,5,9,'2002-09-12','2006-12-03 09:37:26','2006-12-03 09:37:26'),(13,3,186,'2005-02-15','2008-05-26 12:27:10','2008-05-26 12:27:10'),(14,6,NULL,NULL,'2004-12-14 16:37:30','2004-12-14 16:37:30'),(15,92,2,'2008-11-04','2003-02-11 21:19:41','2003-02-11 21:19:41'),(16,7,3,'2004-09-04','2009-10-18 02:27:49','2009-10-18 02:27:49'),(17,NULL,0,'2006-06-05','2000-09-26 07:45:57','2000-09-26 07:45:57'),(18,3,133,'1900-01-01',NULL,NULL),(19,5,1,'1900-01-01','2005-11-10 12:40:29','2005-11-10 12:40:29'),(20,1,8,'1900-01-01','2009-04-25 00:00:00','2009-04-25 00:00:00'),(21,2,5,'2005-01-13','2002-11-27 00:00:00','2002-11-27 00:00:00'),(22,NULL,5,'2006-05-21','2004-01-26 20:32:32','2004-01-26 20:32:32'),(23,1,8,'2003-09-08','2007-10-26 11:41:40','2007-10-26 11:41:40'),(24,0,6,'2006-12-23','2005-10-07 00:00:00','2005-10-07 00:00:00'),(25,210,51,'2006-10-15','2000-07-15 05:00:34','2000-07-15 05:00:34'),(26,8,4,'2005-04-06','2000-04-03 16:33:32','2000-04-03 16:33:32'),(27,7,7,'2008-04-07',NULL,NULL),(28,5,6,'2006-10-10','2001-04-25 01:26:12','2001-04-25 01:26:12'),(29,NULL,4,'1900-01-01','2000-12-27 00:00:00','2000-12-27 00:00:00'); CREATE TABLE t1 ( `pk` int(11) NOT NULL AUTO_INCREMENT, @@ -92,7 +92,7 @@ KEY `datetime_key` (`datetime_key`), KEY `varchar_key` (`int_key`) ) ENGINE=MyISAM AUTO_INCREMENT=21 DEFAULT CHARSET=latin1; Warnings: -Note 1831 Duplicate index 'varchar_key' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `varchar_key`. This is deprecated and will be disallowed in a future release. INSERT INTO t1 VALUES (1,NULL,2,NULL,'2004-10-11 18:13:16','2004-10-11 18:13:16'),(2,7,9,'2001-09-19',NULL,NULL),(3,9,3,'2004-09-12','1900-01-01 00:00:00','1900-01-01 00:00:00'),(4,7,9,NULL,'2009-07-25 00:00:00','2009-07-25 00:00:00'),(5,4,NULL,'2002-07-19',NULL,NULL),(6,2,9,'2002-12-16','2008-07-27 00:00:00','2008-07-27 00:00:00'),(7,6,3,'2006-02-08','2002-11-13 16:37:31','2002-11-13 16:37:31'),(8,8,8,'2006-08-28','1900-01-01 00:00:00','1900-01-01 00:00:00'),(9,NULL,8,'2001-04-14','2003-12-10 00:00:00','2003-12-10 00:00:00'),(10,5,53,'2000-01-05','2001-12-21 22:38:22','2001-12-21 22:38:22'),(11,NULL,0,'2003-12-06','2008-12-13 23:16:44','2008-12-13 23:16:44'),(12,6,5,'1900-01-01','2005-08-15 12:39:41','2005-08-15 12:39:41'),(13,188,166,'2002-11-27',NULL,NULL),(14,2,3,NULL,'2006-09-11 12:06:14','2006-09-11 12:06:14'),(15,1,0,'2003-05-27','2007-12-15 12:39:34','2007-12-15 12:39:34'),(16,1,1,'2005-05-03','2005-08-09 00:00:00','2005-08-09 00:00:00'),(17,0,9,'2001-04-18','2001-09-02 22:50:02','2001-09-02 22:50:02'),(18,9,5,'2005-12-27','2005-12-16 22:58:11','2005-12-16 22:58:11'),(19,NULL,6,'2004-08-20','2007-04-19 00:19:53','2007-04-19 00:19:53'),(20,4,2,'1900-01-01','1900-01-01 00:00:00','1900-01-01 00:00:00'); SELECT OUTR . `pk` AS X FROM t1 AS OUTR diff --git a/mysql-test/suite/roles/create_and_drop_role.result b/mysql-test/suite/roles/create_and_drop_role.result index d565b888c5f..13631f935bf 100644 --- a/mysql-test/suite/roles/create_and_drop_role.result +++ b/mysql-test/suite/roles/create_and_drop_role.result @@ -68,3 +68,19 @@ GRANT USAGE ON *.* TO 'r1'@'localhost' DROP USER u1; DROP ROLE r2; DROP USER r1@localhost; +create role 'foo '; +select concat(user, '__'), is_role from mysql.user where user like 'foo%'; +concat(user, '__') is_role +foo__ Y +select * from mysql.roles_mapping; +Host User Role Admin_option +localhost root foo Y +drop role foo; +select concat(user, '__'), is_role from mysql.user where user like 'foo%'; +concat(user, '__') is_role +select * from mysql.roles_mapping; +Host User Role Admin_option +show grants; +Grants for root@localhost +GRANT ALL PRIVILEGES ON *.* TO 'root'@'localhost' WITH GRANT OPTION +GRANT PROXY ON ''@'%' TO 'root'@'localhost' WITH GRANT OPTION diff --git a/mysql-test/suite/roles/create_and_drop_role.test b/mysql-test/suite/roles/create_and_drop_role.test index 71d6de7053f..b6e5bd2b297 100644 --- a/mysql-test/suite/roles/create_and_drop_role.test +++ b/mysql-test/suite/roles/create_and_drop_role.test @@ -95,3 +95,15 @@ SHOW GRANTS FOR r1@localhost; # Related to MDEV-7774, also caused a crash, by DROP USER u1; DROP ROLE r2; DROP USER r1@localhost; + +# +# MDEV-11533: Roles with trailing white spaces are not cleared correctly +# +create role 'foo '; +select concat(user, '__'), is_role from mysql.user where user like 'foo%'; +select * from mysql.roles_mapping; +drop role foo; +select concat(user, '__'), is_role from mysql.user where user like 'foo%'; +select * from mysql.roles_mapping; +--sorted_result +show grants; diff --git a/mysql-test/suite/roles/role_case_sensitive-10744.result b/mysql-test/suite/roles/role_case_sensitive-10744.result new file mode 100644 index 00000000000..e9b498eff26 --- /dev/null +++ b/mysql-test/suite/roles/role_case_sensitive-10744.result @@ -0,0 +1,58 @@ +# +# MDEV-10744 Roles are not fully case-sensitive +# +# +# Test creating two case-different roles. +# +create user test_user@'%'; +create role test_ROLE; +create role test_role; +# +# Test if mysql.user has the roles created. +# +select user, host from mysql.user where is_role='y' and user like 'test%'; +user host +test_ROLE +test_role +create database secret_db; +create table secret_db.t1 (secret varchar(100)); +insert into secret_db.t1 values ("Some Secret P4ssw0rd"); +grant select on secret_db.* to test_role; +grant test_role to test_user; +show grants for test_user; +Grants for test_user@% +GRANT test_role TO 'test_user'@'%' +GRANT USAGE ON *.* TO 'test_user'@'%' +# +# Now test the UPPER case role. +# +grant test_ROLE to test_user; +grant insert on secret_db.t1 to test_ROLE; +show grants for test_user; +Grants for test_user@% +GRANT test_role TO 'test_user'@'%' +GRANT test_ROLE TO 'test_user'@'%' +GRANT USAGE ON *.* TO 'test_user'@'%' +# +# Test users privileges when interacting with those roles; +# +show tables from secret_db; +ERROR 42000: Access denied for user 'test_user'@'%' to database 'secret_db' +set role test_ROLE; +show tables from secret_db; +Tables_in_secret_db +t1 +select * from secret_db.t1; +ERROR 42000: SELECT command denied to user 'test_user'@'localhost' for table 't1' +insert into secret_db.t1 values ("|-|4><"); +set role test_role; +select * from secret_db.t1 order by secret; +secret +Some Secret P4ssw0rd +|-|4>< +insert into secret_db.t1 values ("|_33T|-|4><"); +ERROR 42000: INSERT command denied to user 'test_user'@'localhost' for table 't1' +drop role test_ROLE; +drop role test_role; +drop user test_user; +drop database secret_db; diff --git a/mysql-test/suite/roles/role_case_sensitive-10744.test b/mysql-test/suite/roles/role_case_sensitive-10744.test new file mode 100644 index 00000000000..281d61bce00 --- /dev/null +++ b/mysql-test/suite/roles/role_case_sensitive-10744.test @@ -0,0 +1,54 @@ +--source include/not_embedded.inc +--echo # +--echo # MDEV-10744 Roles are not fully case-sensitive +--echo # + +--echo # +--echo # Test creating two case-different roles. +--echo # +create user test_user@'%'; +create role test_ROLE; +create role test_role; +--echo # +--echo # Test if mysql.user has the roles created. +--echo # +--sorted_result +select user, host from mysql.user where is_role='y' and user like 'test%'; + +create database secret_db; +create table secret_db.t1 (secret varchar(100)); +insert into secret_db.t1 values ("Some Secret P4ssw0rd"); + +grant select on secret_db.* to test_role; +grant test_role to test_user; +show grants for test_user; +--echo # +--echo # Now test the UPPER case role. +--echo # +grant test_ROLE to test_user; +grant insert on secret_db.t1 to test_ROLE; +show grants for test_user; +connect (test_user,localhost,test_user); + +--echo # +--echo # Test users privileges when interacting with those roles; +--echo # +--error ER_DBACCESS_DENIED_ERROR +show tables from secret_db; +set role test_ROLE; +show tables from secret_db; +--error ER_TABLEACCESS_DENIED_ERROR +select * from secret_db.t1; +insert into secret_db.t1 values ("|-|4><"); +set role test_role; +select * from secret_db.t1 order by secret; +--error ER_TABLEACCESS_DENIED_ERROR +insert into secret_db.t1 values ("|_33T|-|4><"); + +connection default; + + +drop role test_ROLE; +drop role test_role; +drop user test_user; +drop database secret_db; diff --git a/mysql-test/suite/rpl/r/rpl_binlog_errors.result b/mysql-test/suite/rpl/r/rpl_binlog_errors.result index 6234e953cdd..db642f21950 100644 --- a/mysql-test/suite/rpl/r/rpl_binlog_errors.result +++ b/mysql-test/suite/rpl/r/rpl_binlog_errors.result @@ -169,7 +169,7 @@ count(*) SET SQL_LOG_BIN=1; SET GLOBAL debug_dbug="-d,error_unique_log_filename"; ###################### TEST #10 -call mtr.add_suppression("MSYQL_BIN_LOG::open failed to sync the index file."); +call mtr.add_suppression("MYSQL_BIN_LOG::open failed to sync the index file."); call mtr.add_suppression("Could not open .*"); RESET MASTER; SHOW WARNINGS; @@ -226,7 +226,7 @@ include/rpl_reset.inc call mtr.add_suppression("Slave I/O: Relay log write failure: could not queue event from master.*"); call mtr.add_suppression("Error writing file .*"); call mtr.add_suppression("Could not open .*"); -call mtr.add_suppression("MSYQL_BIN_LOG::open failed to sync the index file."); +call mtr.add_suppression("MYSQL_BIN_LOG::open failed to sync the index file."); call mtr.add_suppression("Can't generate a unique log-filename .*"); ###################### TEST #13 SET @old_debug=@@global.debug; diff --git a/mysql-test/suite/rpl/r/rpl_mdev10863.result b/mysql-test/suite/rpl/r/rpl_mdev10863.result new file mode 100644 index 00000000000..475057d4c29 --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_mdev10863.result @@ -0,0 +1,39 @@ +include/rpl_init.inc [topology=1->2] +SET @old_parallel_threads=@@GLOBAL.slave_parallel_threads; +include/stop_slave.inc +SET GLOBAL slave_parallel_threads=10; +SET @old_max_relay= @@GLOBAL.max_relay_log_size; +SET GLOBAL max_relay_log_size = 4096; +CHANGE MASTER TO master_use_gtid=slave_pos; +include/start_slave.inc +ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; +CREATE TABLE t1 (a int PRIMARY KEY, b VARCHAR(100)) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1, "a"); +*** Create a long transaction that will span a relay log file. *** +SET @old_domain= @@gtid_domain_id; +SET gtid_domain_id=10; +INSERT INTO t1 VALUES (10000, "domain 10"); +SET gtid_domain_id=20; +INSERT INTO t1 VALUES (20000, "domain 20"); +SET gtid_domain_id=@old_domain; +BEGIN; +[lots of inserts omitted] +COMMIT; +BEGIN; +[lots of inserts omitted] +COMMIT; +include/stop_slave_sql.inc +START SLAVE SQL_THREAD; +include/wait_for_slave_to_start.inc +INSERT INTO t1 VALUES (100000, "More stuffs."); +INSERT INTO t1 VALUES (100001, "And even more"); +SELECT * FROM t1 WHERE a >= 100000 ORDER BY a; +a b +100000 More stuffs. +100001 And even more +include/stop_slave.inc +SET GLOBAL slave_parallel_threads=@old_parallel_threads; +SET GLOBAL max_relay_log_size= @old_max_relay; +include/start_slave.inc +DROP TABLE t1; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/r/rpl_row_mysqlbinlog.result b/mysql-test/suite/rpl/r/rpl_row_mysqlbinlog.result index 1489af830cc..19611ac52b5 100644 --- a/mysql-test/suite/rpl/r/rpl_row_mysqlbinlog.result +++ b/mysql-test/suite/rpl/r/rpl_row_mysqlbinlog.result @@ -1,14 +1,21 @@ include/master-slave.inc [connection master] - ---Setup Section -- set timestamp=1000000000; -DROP TABLE IF EXISTS t1,t2,t3; CREATE TABLE t1(word VARCHAR(20)); CREATE TABLE t2(id INT AUTO_INCREMENT NOT NULL PRIMARY KEY); CREATE TABLE t3(c1 INT NOT NULL PRIMARY KEY, c2 LONGBLOB, c3 TIMESTAMP, c4 TEXT, c5 FLOAT); - ----Test1 check table load -- +INSERT INTO t1 VALUES ("abirvalg"); +LOAD DATA INFILE '../../std_data/words.dat' INTO TABLE t1; +LOAD DATA INFILE '../../std_data/words.dat' INTO TABLE t1; +LOAD DATA INFILE '../../std_data/words.dat' INTO TABLE t1; +LOAD DATA INFILE '../../std_data/words.dat' INTO TABLE t1; +LOAD DATA INFILE '../../std_data/words.dat' INTO TABLE t1; +set @d1 = 'dd1'; +set @d1 = concat(@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1); +set @d1 = concat(@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1); +set @d1 = concat(@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1); +---Test 1 check table load -- SELECT COUNT(*) from t1; COUNT(*) 351 @@ -71,9 +78,7 @@ c1 c3 c4 c5 5 2006-02-22 00:00:00 Tested in Texas 11 insert into t1 values ("Alas"); flush logs; - --- Test 1 Dump binlog to file -- - --- Test 1 delete tables, clean master and slave -- DROP TABLE t1; DROP TABLE t2; @@ -84,9 +89,7 @@ reset master; reset slave; start slave; include/wait_for_slave_to_start.inc - --- Test 1 Load from Dump binlog file -- - --- Test 1 Check Load Results -- SELECT COUNT(*) from t1; COUNT(*) @@ -148,7 +151,6 @@ c1 c3 c4 c5 3 2006-02-22 00:00:00 Tested in Texas 6.6 4 2006-02-22 00:00:00 Tested in Texas 8.8 5 2006-02-22 00:00:00 Tested in Texas 11 - --- Test 2 position test -- /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/; /*!40019 SET @@session.max_insert_delayed_threads=0*/; @@ -172,7 +174,6 @@ DELIMITER ; ROLLBACK /* added by mysqlbinlog */; /*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/; /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/; - --- Test 3 First Remote test -- /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/; /*!40019 SET @@session.max_insert_delayed_threads=0*/; @@ -189,9 +190,6 @@ SET @@session.auto_increment_increment=1, @@session.auto_increment_offset=1/*!*/ SET @@session.character_set_client=8,@@session.collation_connection=8,@@session.collation_server=8/*!*/; SET @@session.lc_time_names=0/*!*/; SET @@session.collation_database=DEFAULT/*!*/; -DROP TABLE IF EXISTS `t1`,`t2`,`t3` /* generated by server */ -/*!*/; -SET TIMESTAMP=1000000000/*!*/; CREATE TABLE t1(word VARCHAR(20)) /*!*/; SET TIMESTAMP=1000000000/*!*/; @@ -205,7 +203,6 @@ DELIMITER ; ROLLBACK /* added by mysqlbinlog */; /*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/; /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/; - --- Test 4 Second Remote test -- DROP TABLE t1; DROP TABLE t2; @@ -276,7 +273,6 @@ c1 c3 c4 c5 3 2006-02-22 00:00:00 Tested in Texas 6.6 4 2006-02-22 00:00:00 Tested in Texas 8.8 5 2006-02-22 00:00:00 Tested in Texas 11 - --- Test 5 LOAD DATA -- /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/; /*!40019 SET @@session.max_insert_delayed_threads=0*/; @@ -287,7 +283,6 @@ DELIMITER ; ROLLBACK /* added by mysqlbinlog */; /*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/; /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/; - --- Test 6 reading stdin -- /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/; /*!40019 SET @@session.max_insert_delayed_threads=0*/; @@ -304,9 +299,6 @@ SET @@session.auto_increment_increment=1, @@session.auto_increment_offset=1/*!*/ SET @@session.character_set_client=8,@@session.collation_connection=8,@@session.collation_server=8/*!*/; SET @@session.lc_time_names=0/*!*/; SET @@session.collation_database=DEFAULT/*!*/; -DROP TABLE IF EXISTS `t1`,`t2`,`t3` /* generated by server */ -/*!*/; -SET TIMESTAMP=1000000000/*!*/; CREATE TABLE t1(word VARCHAR(20)) /*!*/; SET TIMESTAMP=1000000000/*!*/; @@ -320,7 +312,6 @@ DELIMITER ; ROLLBACK /* added by mysqlbinlog */; /*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/; /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/; - --- Test 7 reading stdin w/position -- /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/; /*!40019 SET @@session.max_insert_delayed_threads=0*/; @@ -344,7 +335,6 @@ DELIMITER ; ROLLBACK /* added by mysqlbinlog */; /*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/; /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/; - --- Test 8 switch internal charset -- stop slave; include/wait_for_slave_to_stop.inc @@ -380,14 +370,13 @@ HEX(f) select HEX(f) from t5; HEX(f) 835C - --- Test cleanup -- -DROP TABLE IF EXISTS t1; +DROP TABLE t1, t2, t3, t04, t05, t4, t5; CREATE TABLE t1 (a INT NOT NULL KEY, b INT); INSERT INTO t1 VALUES(1,1); SELECT * FROM t1; a b 1 1 FLUSH LOGS; -DROP TABLE IF EXISTS t1, t2, t3, t04, t05, t4, t5; +DROP TABLE t1; include/rpl_end.inc diff --git a/mysql-test/suite/rpl/r/rpl_sp.result b/mysql-test/suite/rpl/r/rpl_sp.result index 8ca72672073..b7b90cd1594 100644 --- a/mysql-test/suite/rpl/r/rpl_sp.result +++ b/mysql-test/suite/rpl/r/rpl_sp.result @@ -253,7 +253,7 @@ end delete from t2; alter table t2 add unique (a); Warnings: -Note 1831 Duplicate index 'a_2' defined on the table 'mysqltest1.t2'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `a_2`. This is deprecated and will be disallowed in a future release. drop function fn1; create function fn1(x int) returns int diff --git a/mysql-test/suite/rpl/r/rpl_special_charset.result b/mysql-test/suite/rpl/r/rpl_special_charset.result index 99b8d3a21d1..6efe81bf6cb 100644 --- a/mysql-test/suite/rpl/r/rpl_special_charset.result +++ b/mysql-test/suite/rpl/r/rpl_special_charset.result @@ -1,6 +1,6 @@ include/master-slave.inc [connection master] -call mtr.add_suppression("Cannot use utf16 as character_set_client"); +call mtr.add_suppression("'utf16' can not be used as client character set"); CREATE TABLE t1(i VARCHAR(20)); INSERT INTO t1 VALUES (0xFFFF); include/diff_tables.inc [master:t1, slave:t1] diff --git a/mysql-test/suite/rpl/r/sec_behind_master-5114.result b/mysql-test/suite/rpl/r/sec_behind_master-5114.result index 5554b17347d..17f50f0b612 100644 --- a/mysql-test/suite/rpl/r/sec_behind_master-5114.result +++ b/mysql-test/suite/rpl/r/sec_behind_master-5114.result @@ -3,7 +3,8 @@ include/master-slave.inc call mtr.add_suppression("Unsafe statement written to the binary log"); CREATE TABLE t1 (a int); INSERT INTO t1 VALUES(SLEEP(2)); -Seconds_Behind_Master: 1 +Seconds_Behind_Master_is_less_than_100 +1 Warnings: Note 1592 Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT. Statement is unsafe because it uses a system function that may return a different value on the slave. drop table t1; diff --git a/mysql-test/suite/rpl/t/rpl_binlog_errors.test b/mysql-test/suite/rpl/t/rpl_binlog_errors.test index 9e4a106a5b5..fb3b39fac9d 100644 --- a/mysql-test/suite/rpl/t/rpl_binlog_errors.test +++ b/mysql-test/suite/rpl/t/rpl_binlog_errors.test @@ -256,7 +256,7 @@ SET GLOBAL debug_dbug="-d,error_unique_log_filename"; ### while registering the index file and the binary log ### file or failure to write the rotate event. -call mtr.add_suppression("MSYQL_BIN_LOG::open failed to sync the index file."); +call mtr.add_suppression("MYSQL_BIN_LOG::open failed to sync the index file."); call mtr.add_suppression("Could not open .*"); RESET MASTER; @@ -362,7 +362,7 @@ RESET MASTER; call mtr.add_suppression("Slave I/O: Relay log write failure: could not queue event from master.*"); call mtr.add_suppression("Error writing file .*"); call mtr.add_suppression("Could not open .*"); -call mtr.add_suppression("MSYQL_BIN_LOG::open failed to sync the index file."); +call mtr.add_suppression("MYSQL_BIN_LOG::open failed to sync the index file."); call mtr.add_suppression("Can't generate a unique log-filename .*"); -- echo ###################### TEST #13 diff --git a/mysql-test/suite/rpl/t/rpl_mdev10863.test b/mysql-test/suite/rpl/t/rpl_mdev10863.test new file mode 100644 index 00000000000..796e770672d --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_mdev10863.test @@ -0,0 +1,104 @@ +--source include/have_innodb.inc +--let $rpl_topology=1->2 +--source include/rpl_init.inc + +# Test various aspects of parallel replication. + +--connection server_2 +SET @old_parallel_threads=@@GLOBAL.slave_parallel_threads; +--source include/stop_slave.inc +SET GLOBAL slave_parallel_threads=10; +SET @old_max_relay= @@GLOBAL.max_relay_log_size; +SET GLOBAL max_relay_log_size = 4096; +CHANGE MASTER TO master_use_gtid=slave_pos; +--source include/start_slave.inc + +--connection server_1 +ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; +CREATE TABLE t1 (a int PRIMARY KEY, b VARCHAR(100)) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1, "a"); +--save_master_pos + +--connection server_2 +--sync_with_master + +--echo *** Create a long transaction that will span a relay log file. *** +--connection server_1 + +# Add some transactions in separate domains, that will cause the need to +# have a multi-valued restart position in the relay log for the SQL thread. +SET @old_domain= @@gtid_domain_id; +SET gtid_domain_id=10; +INSERT INTO t1 VALUES (10000, "domain 10"); +SET gtid_domain_id=20; +INSERT INTO t1 VALUES (20000, "domain 20"); +SET gtid_domain_id=@old_domain; + +BEGIN; +--echo [lots of inserts omitted] +--disable_query_log +--let $count = 500 +while ($count) { + eval INSERT INTO t1 VALUES (1000+$count, REPEAT("hulubulu??!?", 8)); + dec $count; +} +--enable_query_log +COMMIT; + +--save_master_pos + +--connection server_2 +--sync_with_master + +--connection server_1 +# Now do another one, to make the inuse_relaylog proceed to somewhere inside +# the first large transaction. + +BEGIN; +--echo [lots of inserts omitted] +--disable_query_log +--let $count = 500 +while ($count) { + eval INSERT INTO t1 VALUES (2000+$count, REPEAT("hulubulu??!?", 8)); + dec $count; +} +--enable_query_log +COMMIT; + +--save_master_pos + +--connection server_2 +--sync_with_master + + +# Stop and restart the SQL thread only. +# The bug was that the SQL thread would restart at the start +# of a relay log file, which could be in the middle of an event group. +# This way, part of that event group could be wrongly re-applied. + +--source include/stop_slave_sql.inc +START SLAVE SQL_THREAD; +--source include/wait_for_slave_to_start.inc + + +--connection server_1 +INSERT INTO t1 VALUES (100000, "More stuffs."); +INSERT INTO t1 VALUES (100001, "And even more"); +--save_master_pos + +--connection server_2 +--sync_with_master +SELECT * FROM t1 WHERE a >= 100000 ORDER BY a; + + +# Clean up. +--connection server_2 +--source include/stop_slave.inc +SET GLOBAL slave_parallel_threads=@old_parallel_threads; +SET GLOBAL max_relay_log_size= @old_max_relay; +--source include/start_slave.inc + +--connection server_1 +DROP TABLE t1; + +--source include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_row_mysqlbinlog.test b/mysql-test/suite/rpl/t/rpl_row_mysqlbinlog.test index ed0f31b75be..678679f0cf1 100644 --- a/mysql-test/suite/rpl/t/rpl_row_mysqlbinlog.test +++ b/mysql-test/suite/rpl/t/rpl_row_mysqlbinlog.test @@ -4,43 +4,27 @@ # Purpose: To test changes to mysqlbinlog for row based bin logs # # We are using .opt file since we need small binlog size # ################################################################## -# Include Section -# Make sure that we have row based bin log -- source include/have_binlog_format_row.inc -# Embedded server doesn't support binlogging -- source include/not_embedded.inc -# This test requires the cp932 charset compiled in -- source include/have_cp932.inc -# Slow test, don't run during staging part --- source include/not_staging.inc - -- source include/master-slave.inc -# Setup Section -# we need this for getting fixed timestamps inside of this test - ---disable_query_log -select "---Setup Section --" as ""; ---enable_query_log +--echo ---Setup Section -- +# we need this for getting fixed timestamps inside of this test set timestamp=1000000000; ---disable_warnings -DROP TABLE IF EXISTS t1,t2,t3; ---enable_warnings - -connection master; CREATE TABLE t1(word VARCHAR(20)); CREATE TABLE t2(id INT AUTO_INCREMENT NOT NULL PRIMARY KEY); ---let $position= query_get_value(SHOW MASTER STATUS, Position, 1) +--let position= query_get_value(SHOW MASTER STATUS, Position, 1) CREATE TABLE t3(c1 INT NOT NULL PRIMARY KEY, c2 LONGBLOB, c3 TIMESTAMP, c4 TEXT, c5 FLOAT); ---let $stop_position=query_get_value(SHOW MASTER STATUS, Position, 1) ---let $stop_position1=`select $stop_position - 1` ---let $binlog_start_pos=query_get_value(SHOW BINLOG EVENTS LIMIT 1, End_log_pos, 1) +--let stop_position=query_get_value(SHOW MASTER STATUS, Position, 1) +--let stop_position1=`select $stop_position - 1` +--let binlog_start_pos=query_get_value(SHOW BINLOG EVENTS LIMIT 1, End_log_pos, 1) + # Test Section # Lets start by putting some data into the tables. ---disable_query_log INSERT INTO t1 VALUES ("abirvalg"); LOAD DATA INFILE '../../std_data/words.dat' INTO TABLE t1; LOAD DATA INFILE '../../std_data/words.dat' INTO TABLE t1; @@ -54,7 +38,8 @@ set @d1 = concat(@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1); set @d1 = concat(@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1); set @d1 = concat(@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1); -let $count=500; +--disable_query_log +let count=500; while ($count) { INSERT INTO t2 VALUES (NULL); @@ -63,10 +48,7 @@ while ($count) } --enable_query_log - ---disable_query_log -select "---Test1 check table load --" as ""; ---enable_query_log +--echo ---Test 1 check table load -- # Lets Check the tables on the Master SELECT COUNT(*) from t1; @@ -95,34 +77,26 @@ insert into t1 values ("Alas"); flush logs; # delimiters are for easier debugging in future ---disable_query_log -select "--- Test 1 Dump binlog to file --" as ""; ---enable_query_log +--echo --- Test 1 Dump binlog to file -- # # Prepare local temporary file to recreate what we have currently. -let $MYSQLD_DATADIR= `select @@datadir;`; +let MYSQLD_DATADIR= `select @@datadir;`; --exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 > $MYSQLTEST_VARDIR/tmp/master.sql --exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000002 >> $MYSQLTEST_VARDIR/tmp/master.sql # Now that we have our file, lets get rid of the current database. # Cleanup the master and the slave and try to recreate. ---disable_query_log -select "--- Test 1 delete tables, clean master and slave --" as ""; ---enable_query_log +--echo --- Test 1 delete tables, clean master and slave -- DROP TABLE t1; DROP TABLE t2; DROP TABLE t3; sync_slave_with_master; -#we expect STOP SLAVE to produce a warning as the slave is stopped -#(the server was started with skip-slave-start) ---disable_warnings stop slave; --source include/wait_for_slave_to_stop.inc ---enable_warnings connection master; reset master; connection slave; @@ -132,15 +106,11 @@ start slave; connection master; # We should be clean at this point, now we will run in the file from above. ---disable_query_log -select "--- Test 1 Load from Dump binlog file --" as ""; ---enable_query_log +--echo --- Test 1 Load from Dump binlog file -- --exec $MYSQL -e "source $MYSQLTEST_VARDIR/tmp/master.sql" ---disable_query_log -select "--- Test 1 Check Load Results --" as ""; ---enable_query_log +--echo --- Test 1 Check Load Results -- # Lets Check the tables on the Master SELECT COUNT(*) from t1; @@ -168,28 +138,20 @@ remove_file $MYSQLTEST_VARDIR/tmp/master.sql; # this test for start-position option # By setting this position to 416, we should only get the create of t3 ---disable_query_log -select "--- Test 2 position test --" as ""; ---enable_query_log -let $MYSQLD_DATADIR= `select @@datadir;`; +--echo --- Test 2 position test -- --exec $MYSQL_BINLOG --short-form --local-load=$MYSQLTEST_VARDIR/tmp/ --start-position=$position --stop-position=$stop_position $MYSQLD_DATADIR/master-bin.000001 # These are tests for remote binlog. # They should return the same as previous test. ---disable_query_log -select "--- Test 3 First Remote test --" as ""; ---enable_query_log +--echo --- Test 3 First Remote test -- # This is broken now --replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR --exec $MYSQL_BINLOG --short-form --local-load=$MYSQLTEST_VARDIR/tmp/ --stop-position=$stop_position --read-from-remote-server --user=root --host=127.0.0.1 --port=$MASTER_MYPORT master-bin.000001 ---disable_query_log -select "--- Test 4 Second Remote test --" as ""; ---enable_query_log +--echo --- Test 4 Second Remote test -- --exec $MYSQL_BINLOG --read-from-remote-server --user=root --host=127.0.0.1 --port=$MASTER_MYPORT master-bin.000001 > $MYSQLTEST_VARDIR/tmp/remote.sql - --exec $MYSQL_BINLOG --read-from-remote-server --user=root --host=127.0.0.1 --port=$MASTER_MYPORT master-bin.000002 >> $MYSQLTEST_VARDIR/tmp/remote.sql # Now that we have our file, lets get rid of the current database. @@ -201,13 +163,8 @@ DROP TABLE t3; sync_slave_with_master; -#we expect STOP SLAVE to produce a warning as the slave is stopped -#(the server was started with skip-slave-start) - ---disable_warnings stop slave; --source include/wait_for_slave_to_stop.inc ---enable_warnings connection master; reset master; connection slave; @@ -251,40 +208,26 @@ connection master; # transactions. /Matz # LOAD DATA ---disable_query_log -select "--- Test 5 LOAD DATA --" as ""; ---enable_query_log +--echo --- Test 5 LOAD DATA -- --replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR --exec $MYSQL_BINLOG --short-form --local-load=$MYSQLTEST_VARDIR/tmp/ --stop-position=$binlog_start_pos --read-from-remote-server --user=root --host=127.0.0.1 --port=$MASTER_MYPORT master-bin.000002 # Bug#7853 (mysqlbinlog does not accept input from stdin) ---disable_query_log -select "--- Test 6 reading stdin --" as ""; ---enable_query_log -let $MYSQLD_DATADIR= `select @@datadir;`; +--echo --- Test 6 reading stdin -- --replace_result $MYSQL_TEST_DIR MYSQL_TEST_DIR --exec $MYSQL_BINLOG --short-form --stop-position=$stop_position1 - < $MYSQLD_DATADIR/master-bin.000001 ---disable_query_log -select "--- Test 7 reading stdin w/position --" as ""; ---enable_query_log +--echo --- Test 7 reading stdin w/position -- --replace_result $MYSQL_TEST_DIR MYSQL_TEST_DIR --exec $MYSQL_BINLOG --short-form --start-position=$position --stop-position=$stop_position - < $MYSQLD_DATADIR/master-bin.000001 # Bug#16217 (mysql client did not know how not switch its internal charset) ---disable_query_log -select "--- Test 8 switch internal charset --" as ""; ---enable_query_log +--echo --- Test 8 switch internal charset -- sync_slave_with_master; -#we expect STOP SLAVE to produce a warning as the slave is stopped -#(the server was started with skip-slave-start) - ---disable_warnings stop slave; --source include/wait_for_slave_to_stop.inc ---enable_warnings connection master; reset master; connection slave; @@ -297,7 +240,6 @@ create table t4 (f text character set utf8); create table t5 (f text character set cp932); --exec $MYSQL --default-character-set=utf8 test -e "insert into t4 values(_utf8'ソ')" --exec $MYSQL --default-character-set=cp932 test -e "insert into t5 values(_cp932'ƒ\');" -let $MYSQLD_DATADIR= `select @@datadir;`; flush logs; rename table t4 to t04, t5 to t05; --exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 | $MYSQL --default-character-set=utf8 @@ -314,42 +256,30 @@ select HEX(f) from t4; select HEX(f) from t05; select HEX(f) from t5; ---disable_query_log -select "--- Test cleanup --" as ""; ---enable_query_log +--echo --- Test cleanup -- # clean up connection master; sync_slave_with_master; connection master; +DROP TABLE t1, t2, t3, t04, t05, t4, t5; # BUG#17654 also test mysqlbinlog to ensure it can read the binlog from a remote server # and ensure that the results are the same as if read from a file (the same file). ---disable_warnings -DROP TABLE IF EXISTS t1; ---enable_warnings - CREATE TABLE t1 (a INT NOT NULL KEY, b INT); - INSERT INTO t1 VALUES(1,1); - SELECT * FROM t1; - -let $MYSQLD_DATADIR= `select @@datadir;`; - FLUSH LOGS; --exec $MYSQL_BINLOG --read-from-remote-server --user=root --host=127.0.0.1 --port=$MASTER_MYPORT master-bin.000001 > $MYSQLTEST_VARDIR/tmp/remote.sql --exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 > $MYSQLTEST_VARDIR/tmp/local.sql --diff_files $MYSQLTEST_VARDIR/tmp/local.sql $MYSQLTEST_VARDIR/tmp/remote.sql - --remove_file $MYSQLTEST_VARDIR/tmp/remote.sql - --remove_file $MYSQLTEST_VARDIR/tmp/local.sql +DROP TABLE t1; -DROP TABLE IF EXISTS t1, t2, t3, t04, t05, t4, t5; sync_slave_with_master; # End of 4.1 tests diff --git a/mysql-test/suite/rpl/t/rpl_special_charset.test b/mysql-test/suite/rpl/t/rpl_special_charset.test index 8ccb1b4183f..3a947d413bb 100644 --- a/mysql-test/suite/rpl/t/rpl_special_charset.test +++ b/mysql-test/suite/rpl/t/rpl_special_charset.test @@ -14,7 +14,7 @@ # then set default's client character set(latin1) as client's character set. ############################################################################### --source include/master-slave.inc -call mtr.add_suppression("Cannot use utf16 as character_set_client"); +call mtr.add_suppression("'utf16' can not be used as client character set"); CREATE TABLE t1(i VARCHAR(20)); INSERT INTO t1 VALUES (0xFFFF); --sync_slave_with_master diff --git a/mysql-test/suite/rpl/t/sec_behind_master-5114.test b/mysql-test/suite/rpl/t/sec_behind_master-5114.test index 8c70da4f0a8..ff8cab54c4f 100644 --- a/mysql-test/suite/rpl/t/sec_behind_master-5114.test +++ b/mysql-test/suite/rpl/t/sec_behind_master-5114.test @@ -3,24 +3,60 @@ source include/have_binlog_format_statement.inc; call mtr.add_suppression("Unsafe statement written to the binary log"); + +# Make sure that the start time of the first event is certainly different +# from the next event +--let $timestamp= `SELECT @@timestamp` +--disable_query_log +eval SET TIMESTAMP= $timestamp-100; +--enable_query_log CREATE TABLE t1 (a int); + +# Make sure that the slave is done with the first event, and all checks +# that we'll perform later will be really against the second event +sync_slave_with_master; + +connection master; + +# Restore the timestamp now. It doesn't matter that it's not precise, +# it just needs to be very different from the earlier event +--disable_query_log +eval SET TIMESTAMP= $timestamp; +--enable_query_log + send INSERT INTO t1 VALUES(SLEEP(2)); connection slave; -let $run = 10; + +# When the slave starts executing the event, Seconds_Behind_Master +# should start growing steadilly. The bugfix ensures that they are +# calculated based on the start time of the current event, rather +# than the start time of the previous event. To check it, we only need +# the first non-zero value + +let $run = 20; while ($run) { dec $run; let $sbm=query_get_value(SHOW SLAVE STATUS, Seconds_Behind_Master, 1); # for debugging uncomment echo and remove the if() - #echo Seconds_Behind_Master: $sbm; +# echo Seconds_Behind_Master: $sbm; if ($sbm) { let $run = 0; } sleep 0.5; } -echo Seconds_Behind_Master: $sbm; + +# Normally the first non-zero value should be 1. However, due to race +# conditions on slow servers, sometimes the check might miss the value 1, +# and only catch a higher one. It does not matter, we just need to make +# sure it didn't start with 100+, as it would have with bug MDEV-5114 + +--disable_query_log +eval SELECT $sbm > 0 and $sbm < 99 AS Seconds_Behind_Master_is_less_than_100; +--enable_query_log + connection master; reap; drop table t1; diff --git a/mysql-test/suite/sys_vars/r/replicate_do_db_basic.result b/mysql-test/suite/sys_vars/r/replicate_do_db_basic.result index b964d3d14a1..a05b85a9bfd 100644 --- a/mysql-test/suite/sys_vars/r/replicate_do_db_basic.result +++ b/mysql-test/suite/sys_vars/r/replicate_do_db_basic.result @@ -33,5 +33,9 @@ SET @@GLOBAL.replicate_do_db=""; SELECT @@GLOBAL.replicate_do_db; @@GLOBAL.replicate_do_db +SET @@GLOBAL.replicate_do_db=null; +SELECT @@GLOBAL.replicate_do_db; +@@GLOBAL.replicate_do_db + # Cleanup. SET @@GLOBAL.replicate_do_db = @save_replicate_do_db; diff --git a/mysql-test/suite/sys_vars/r/replicate_do_table_basic.result b/mysql-test/suite/sys_vars/r/replicate_do_table_basic.result index fac237228ac..e67b1eeca01 100644 --- a/mysql-test/suite/sys_vars/r/replicate_do_table_basic.result +++ b/mysql-test/suite/sys_vars/r/replicate_do_table_basic.result @@ -40,5 +40,9 @@ SET @@GLOBAL.replicate_do_table=""; SELECT @@GLOBAL.replicate_do_table; @@GLOBAL.replicate_do_table +SET @@GLOBAL.replicate_do_table=null; +SELECT @@GLOBAL.replicate_do_table; +@@GLOBAL.replicate_do_table + # Cleanup. SET @@GLOBAL.replicate_do_table = @save_replicate_do_table; diff --git a/mysql-test/suite/sys_vars/r/replicate_ignore_db_basic.result b/mysql-test/suite/sys_vars/r/replicate_ignore_db_basic.result index c4d7a37321e..c7ff697b34f 100644 --- a/mysql-test/suite/sys_vars/r/replicate_ignore_db_basic.result +++ b/mysql-test/suite/sys_vars/r/replicate_ignore_db_basic.result @@ -33,5 +33,9 @@ SET @@GLOBAL.replicate_ignore_db=""; SELECT @@GLOBAL.replicate_ignore_db; @@GLOBAL.replicate_ignore_db +SET @@GLOBAL.replicate_ignore_db=null; +SELECT @@GLOBAL.replicate_ignore_db; +@@GLOBAL.replicate_ignore_db + # Cleanup. SET @@GLOBAL.replicate_ignore_db = @save_replicate_ignore_db; diff --git a/mysql-test/suite/sys_vars/r/replicate_ignore_table_basic.result b/mysql-test/suite/sys_vars/r/replicate_ignore_table_basic.result index bc463d07319..db97ce14c93 100644 --- a/mysql-test/suite/sys_vars/r/replicate_ignore_table_basic.result +++ b/mysql-test/suite/sys_vars/r/replicate_ignore_table_basic.result @@ -40,5 +40,9 @@ SET @@GLOBAL.replicate_ignore_table=""; SELECT @@GLOBAL.replicate_ignore_table; @@GLOBAL.replicate_ignore_table +SET @@GLOBAL.replicate_ignore_table=null; +SELECT @@GLOBAL.replicate_ignore_table; +@@GLOBAL.replicate_ignore_table + # Cleanup. SET @@GLOBAL.replicate_ignore_table = @save_replicate_ignore_table; diff --git a/mysql-test/suite/sys_vars/r/replicate_wild_do_table_basic.result b/mysql-test/suite/sys_vars/r/replicate_wild_do_table_basic.result index 5647cc964fb..8c55103080f 100644 --- a/mysql-test/suite/sys_vars/r/replicate_wild_do_table_basic.result +++ b/mysql-test/suite/sys_vars/r/replicate_wild_do_table_basic.result @@ -40,5 +40,9 @@ SET @@GLOBAL.replicate_wild_do_table=""; SELECT @@GLOBAL.replicate_wild_do_table; @@GLOBAL.replicate_wild_do_table +SET @@GLOBAL.replicate_wild_do_table=null; +SELECT @@GLOBAL.replicate_wild_do_table; +@@GLOBAL.replicate_wild_do_table + # Cleanup. SET @@GLOBAL.replicate_wild_do_table = @save_replicate_wild_do_table; diff --git a/mysql-test/suite/sys_vars/r/replicate_wild_ignore_table_basic.result b/mysql-test/suite/sys_vars/r/replicate_wild_ignore_table_basic.result index c6829b792a4..0f46ce38805 100644 --- a/mysql-test/suite/sys_vars/r/replicate_wild_ignore_table_basic.result +++ b/mysql-test/suite/sys_vars/r/replicate_wild_ignore_table_basic.result @@ -40,5 +40,9 @@ SET @@GLOBAL.replicate_wild_ignore_table=""; SELECT @@GLOBAL.replicate_wild_ignore_table; @@GLOBAL.replicate_wild_ignore_table +SET @@GLOBAL.replicate_wild_ignore_table=null; +SELECT @@GLOBAL.replicate_wild_ignore_table; +@@GLOBAL.replicate_wild_ignore_table + # Cleanup. SET @@GLOBAL.replicate_wild_ignore_table = @save_replicate_wild_ignore_table; diff --git a/mysql-test/suite/sys_vars/t/replicate_do_db_basic.test b/mysql-test/suite/sys_vars/t/replicate_do_db_basic.test index ccf50b1d6ab..59d0176add2 100644 --- a/mysql-test/suite/sys_vars/t/replicate_do_db_basic.test +++ b/mysql-test/suite/sys_vars/t/replicate_do_db_basic.test @@ -35,5 +35,8 @@ SELECT @@GLOBAL.replicate_do_db; SET @@GLOBAL.replicate_do_db=""; SELECT @@GLOBAL.replicate_do_db; +SET @@GLOBAL.replicate_do_db=null; +SELECT @@GLOBAL.replicate_do_db; + --echo # Cleanup. SET @@GLOBAL.replicate_do_db = @save_replicate_do_db; diff --git a/mysql-test/suite/sys_vars/t/replicate_do_table_basic.test b/mysql-test/suite/sys_vars/t/replicate_do_table_basic.test index f3b1585613e..346bdf3b038 100644 --- a/mysql-test/suite/sys_vars/t/replicate_do_table_basic.test +++ b/mysql-test/suite/sys_vars/t/replicate_do_table_basic.test @@ -44,5 +44,8 @@ SELECT @@GLOBAL.replicate_do_table; SET @@GLOBAL.replicate_do_table=""; SELECT @@GLOBAL.replicate_do_table; +SET @@GLOBAL.replicate_do_table=null; +SELECT @@GLOBAL.replicate_do_table; + --echo # Cleanup. SET @@GLOBAL.replicate_do_table = @save_replicate_do_table; diff --git a/mysql-test/suite/sys_vars/t/replicate_ignore_db_basic.test b/mysql-test/suite/sys_vars/t/replicate_ignore_db_basic.test index 3a0bc88109a..376397d1635 100644 --- a/mysql-test/suite/sys_vars/t/replicate_ignore_db_basic.test +++ b/mysql-test/suite/sys_vars/t/replicate_ignore_db_basic.test @@ -35,5 +35,8 @@ SELECT @@GLOBAL.replicate_ignore_db; SET @@GLOBAL.replicate_ignore_db=""; SELECT @@GLOBAL.replicate_ignore_db; +SET @@GLOBAL.replicate_ignore_db=null; +SELECT @@GLOBAL.replicate_ignore_db; + --echo # Cleanup. SET @@GLOBAL.replicate_ignore_db = @save_replicate_ignore_db; diff --git a/mysql-test/suite/sys_vars/t/replicate_ignore_table_basic.test b/mysql-test/suite/sys_vars/t/replicate_ignore_table_basic.test index aebe90732d2..56cf7f17c7f 100644 --- a/mysql-test/suite/sys_vars/t/replicate_ignore_table_basic.test +++ b/mysql-test/suite/sys_vars/t/replicate_ignore_table_basic.test @@ -44,5 +44,8 @@ SELECT @@GLOBAL.replicate_ignore_table; SET @@GLOBAL.replicate_ignore_table=""; SELECT @@GLOBAL.replicate_ignore_table; +SET @@GLOBAL.replicate_ignore_table=null; +SELECT @@GLOBAL.replicate_ignore_table; + --echo # Cleanup. SET @@GLOBAL.replicate_ignore_table = @save_replicate_ignore_table; diff --git a/mysql-test/suite/sys_vars/t/replicate_wild_do_table_basic.test b/mysql-test/suite/sys_vars/t/replicate_wild_do_table_basic.test index b96a62f8dd1..832d3397f89 100644 --- a/mysql-test/suite/sys_vars/t/replicate_wild_do_table_basic.test +++ b/mysql-test/suite/sys_vars/t/replicate_wild_do_table_basic.test @@ -44,5 +44,8 @@ SELECT @@GLOBAL.replicate_wild_do_table; SET @@GLOBAL.replicate_wild_do_table=""; SELECT @@GLOBAL.replicate_wild_do_table; +SET @@GLOBAL.replicate_wild_do_table=null; +SELECT @@GLOBAL.replicate_wild_do_table; + --echo # Cleanup. SET @@GLOBAL.replicate_wild_do_table = @save_replicate_wild_do_table; diff --git a/mysql-test/suite/sys_vars/t/replicate_wild_ignore_table_basic.test b/mysql-test/suite/sys_vars/t/replicate_wild_ignore_table_basic.test index 2900deab4d1..5cb1ff6c820 100644 --- a/mysql-test/suite/sys_vars/t/replicate_wild_ignore_table_basic.test +++ b/mysql-test/suite/sys_vars/t/replicate_wild_ignore_table_basic.test @@ -44,5 +44,8 @@ SELECT @@GLOBAL.replicate_wild_ignore_table; SET @@GLOBAL.replicate_wild_ignore_table=""; SELECT @@GLOBAL.replicate_wild_ignore_table; +SET @@GLOBAL.replicate_wild_ignore_table=null; +SELECT @@GLOBAL.replicate_wild_ignore_table; + --echo # Cleanup. SET @@GLOBAL.replicate_wild_ignore_table = @save_replicate_wild_ignore_table; diff --git a/mysql-test/t/alter_table.test b/mysql-test/t/alter_table.test index d2b8a6082a6..6fe0f4d30ee 100644 --- a/mysql-test/t/alter_table.test +++ b/mysql-test/t/alter_table.test @@ -1737,3 +1737,22 @@ SHOW CREATE TABLE `tab1`; ALTER TABLE `tab1` CHANGE COLUMN v_col `v_col` varchar(128) AS (IF(field11='option1',CONCAT_WS(":","field1",field2,field3,field4,field5,field6,field7,field8,field9,field10), CONCAT_WS(":","field1",field11,field2,field3,field4,field5,field6,field7,field8,field9,field10))) PERSISTENT; SHOW CREATE TABLE `tab1`; DROP TABLE `tab1`; + +--echo # +--echo # MDEV-11548 Reproducible server crash after the 2nd ALTER TABLE ADD FOREIGN KEY IF NOT EXISTS +--echo # + +CREATE TABLE t1 (id INT UNSIGNED NOT NULL PRIMARY KEY); +CREATE TABLE t2 (id1 INT UNSIGNED NOT NULL); + +ALTER TABLE t2 +ADD FOREIGN KEY IF NOT EXISTS (id1) + REFERENCES t1 (id); + +ALTER TABLE t2 +ADD FOREIGN KEY IF NOT EXISTS (id1) +REFERENCES t1 (id); + +DROP TABLE t2; +DROP TABLE t1; + diff --git a/mysql-test/t/create.test b/mysql-test/t/create.test index ded9096900c..a51fd0b8a8f 100644 --- a/mysql-test/t/create.test +++ b/mysql-test/t/create.test @@ -2081,6 +2081,15 @@ DELIMITER ;| create table t1 as select f1(); drop function f1; +--echo # +--echo # MDEV-10274 Bundling insert with create statement +--echo # for table with unsigned Decimal primary key issues warning 1194 +--echo # + +create table t1(ID decimal(2,1) unsigned NOT NULL, PRIMARY KEY (ID))engine=memory + select 2.1 ID; +drop table t1; + --echo End of 5.5 tests # @@ -2088,3 +2097,9 @@ drop function f1; # --error ER_TABLE_MUST_HAVE_COLUMNS create table t1; + +# +# MDEV-11231 Server crashes in check_duplicate_key on CREATE TABLE ... SELECT +# +create table t1 (i int, j int, key(i), key(i)) as select 1 as i, 2 as j; +drop table t1; diff --git a/mysql-test/t/ctype_ucs.test b/mysql-test/t/ctype_ucs.test index 33d41e9bc4c..165d945944e 100644 --- a/mysql-test/t/ctype_ucs.test +++ b/mysql-test/t/ctype_ucs.test @@ -811,6 +811,14 @@ SELECT CHAR_LENGTH(TRIM(BOTH 0x0001 FROM _ucs2 0x0061)); SELECT CHAR_LENGTH(TRIM(BOTH 0x61 FROM _ucs2 0x0061)); SELECT CHAR_LENGTH(TRIM(BOTH 0x00 FROM _ucs2 0x0061)); +--echo # +--echo # MDEV-11685: sql_mode can't be set with non-ascii connection charset +--echo # +SET character_set_connection=ucs2; +SET sql_mode='NO_ENGINE_SUBSTITUTION'; +SELECT @@sql_mode; +SET sql_mode=DEFAULT; +SET NAMES utf8; --echo # --echo # End of 5.5 tests diff --git a/mysql-test/t/ctype_ucs2_def.test b/mysql-test/t/ctype_ucs2_def.test index be8e044f2e4..e297fa5ccf1 100644 --- a/mysql-test/t/ctype_ucs2_def.test +++ b/mysql-test/t/ctype_ucs2_def.test @@ -1,6 +1,6 @@ -- source include/have_ucs2.inc -call mtr.add_suppression("Cannot use ucs2 as character_set_client"); +call mtr.add_suppression("'ucs2' can not be used as client character set"); # # MySQL Bug#15276: MySQL ignores collation-server diff --git a/mysql-test/t/ctype_ucs2_query_cache.test b/mysql-test/t/ctype_ucs2_query_cache.test index acb39419751..ace826aec44 100644 --- a/mysql-test/t/ctype_ucs2_query_cache.test +++ b/mysql-test/t/ctype_ucs2_query_cache.test @@ -1,7 +1,7 @@ -- source include/have_query_cache.inc -- source include/have_ucs2.inc -call mtr.add_suppression("Cannot use ucs2 as character_set_client"); +call mtr.add_suppression("'ucs2' can not be used as client character set"); --echo # --echo # Start of 5.5 tests diff --git a/mysql-test/t/ctype_utf16.test b/mysql-test/t/ctype_utf16.test index e4305ed9879..cda85239512 100644 --- a/mysql-test/t/ctype_utf16.test +++ b/mysql-test/t/ctype_utf16.test @@ -796,6 +796,15 @@ DO RPAD(_utf16 0x0061 COLLATE utf16_unicode_ci, 10000, 0x0061DE989999); DO LPAD(_utf16 0x0061 COLLATE utf16_unicode_ci, 10000, 0x0061DE989999); --echo # +--echo # MDEV-11685: sql_mode can't be set with non-ascii connection charset +--echo # +SET character_set_connection=utf16; +SET sql_mode='NO_ENGINE_SUBSTITUTION'; +SELECT @@sql_mode; +SET sql_mode=DEFAULT; +SET NAMES utf8; + +--echo # --echo # End of 5.5 tests --echo # diff --git a/mysql-test/t/ctype_utf16_def.test b/mysql-test/t/ctype_utf16_def.test index fad61b057c3..0829cd53285 100644 --- a/mysql-test/t/ctype_utf16_def.test +++ b/mysql-test/t/ctype_utf16_def.test @@ -1,5 +1,5 @@ --source include/have_utf16.inc -call mtr.add_suppression("Cannot use utf16 as character_set_client"); +call mtr.add_suppression("'utf16' can not be used as client character set"); # # Bug #32391 Character sets: crash with --character-set-server diff --git a/mysql-test/t/ctype_utf32.test b/mysql-test/t/ctype_utf32.test index 600880d6be5..2f5952045c5 100644 --- a/mysql-test/t/ctype_utf32.test +++ b/mysql-test/t/ctype_utf32.test @@ -895,6 +895,15 @@ SELECT CHAR_LENGTH(TRIM(BOTH 0x00 FROM _utf32 0x00000061)); select hex(lower(cast(0xffff0000 as char character set utf32))) as c; --echo # +--echo # MDEV-11685: sql_mode can't be set with non-ascii connection charset +--echo # +SET character_set_connection=utf32; +SET sql_mode='NO_ENGINE_SUBSTITUTION'; +SELECT @@sql_mode; +SET sql_mode=DEFAULT; +SET NAMES utf8; + +--echo # --echo # End of 5.5 tests --echo # diff --git a/mysql-test/t/ctype_utf8.test b/mysql-test/t/ctype_utf8.test index 7c0723ae82e..e56212cd4e8 100644 --- a/mysql-test/t/ctype_utf8.test +++ b/mysql-test/t/ctype_utf8.test @@ -1701,6 +1701,20 @@ SELECT length(data) AS len FROM (SELECT REPEAT('ä', 65536) AS data) AS sub; SELECT length(data) AS len FROM (SELECT REPEAT('ä', 65537) AS data) AS sub; --echo # +--echo # MDEV-10717 Assertion `!null_value' failed in virtual bool Item::send(Protocol*, String*) +--echo # +CREATE TABLE t1 (i INT, KEY(i)); +INSERT INTO t1 VALUES (20081205),(20050327); +SELECT HEX(i), HEX(CHAR(i USING utf8)) FROM t1; +SET sql_mode='STRICT_ALL_TABLES'; +SELECT HEX(i), HEX(CHAR(i USING utf8)) FROM t1; +# Avoid garbage in the output +--replace_column 1 ### +SELECT CHAR(i USING utf8) FROM t1; +SET sql_mode=DEFAULT; +DROP TABLE t1; + +--echo # --echo # End of 5.5 tests --echo # diff --git a/mysql-test/t/ctype_utf8mb4.test b/mysql-test/t/ctype_utf8mb4.test index 6b876cc5eba..cf1c103137e 100644 --- a/mysql-test/t/ctype_utf8mb4.test +++ b/mysql-test/t/ctype_utf8mb4.test @@ -1864,6 +1864,17 @@ set @@collation_connection=utf8mb4_bin; --echo # End of 5.6 tests --echo # +--echo # +--echo # Start of 10.0 tests +--echo # + +--echo # +--echo # MDEV-11343 LOAD DATA INFILE fails to load data with an escape character followed by a multi-byte character +--echo # +CREATE TABLE t1 (a TEXT CHARACTER SET utf8mb4); +LOAD DATA INFILE '../../std_data/loaddata/mdev-11343.txt' INTO TABLE t1 CHARACTER SET utf8mb4; +SELECT HEX(a) FROM t1; +DROP TABLE t1; --echo # --echo # End of tests diff --git a/mysql-test/t/default.test b/mysql-test/t/default.test index b719cb83448..432df9d4452 100644 --- a/mysql-test/t/default.test +++ b/mysql-test/t/default.test @@ -166,3 +166,46 @@ drop table t1, t2; --echo End of 5.0 tests. +--echo # +--echo # Start of 10.0 tests +--echo # + +--echo # +--echo # MDEV-11265 Access defied when CREATE VIIEW v1 AS SELECT DEFAULT(column) FROM t1 +--echo # + +CREATE TABLE t1 (a INT DEFAULT 10); +INSERT INTO t1 VALUES (11); +CREATE VIEW v1 AS SELECT a AS a FROM t1; +CREATE VIEW v2 AS SELECT DEFAULT(a) AS a FROM t1; +CREATE VIEW v3 AS SELECT VALUES(a) AS a FROM t1; +SELECT * FROM v1; +SELECT * FROM v2; +SELECT * FROM v3; +--error ER_NONUPDATEABLE_COLUMN +UPDATE v2 SET a=123; +--error ER_NONUPDATEABLE_COLUMN +UPDATE v3 SET a=123; +DROP VIEW v3; +DROP VIEW v2; +DROP VIEW v1; +DROP TABLE t1; + +--echo # +--echo # MDEV-10780 Server crashes in in create_tmp_table +--echo # + +# Note, the problem was not repeatable with a non-fresh connection. +--connect (con1,localhost,root,,test) +CREATE TABLE t1 (pk INT AUTO_INCREMENT PRIMARY KEY) ENGINE=MyISAM; +INSERT INTO t1 VALUES (); +INSERT INTO t1 VALUES (); +SELECT DISTINCT DEFAULT (pk) FROM t1 GROUP BY RAND() WITH ROLLUP; +--disconnect con1 +--connection default +DROP TABLE t1; + +--echo # +--echo # End of 10.0 tests +--echo # + diff --git a/mysql-test/t/derived.test b/mysql-test/t/derived.test index a7739dcf8a7..b517dfd29d1 100644 --- a/mysql-test/t/derived.test +++ b/mysql-test/t/derived.test @@ -829,3 +829,50 @@ ORDER BY TOTAL DESC; DROP TABLES t1,t2; set optimizer_switch=@save_derived_optimizer_switch; + +--echo # +--echo # MDEV-10663: Use of Inline table columns in HAVING clause +--echo # throws 1463 Error +--echo # + +set @save_sql_mode = @@sql_mode; +set sql_mode='ONLY_FULL_GROUP_BY,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION'; + +CREATE TABLE `example1463` ( + `Customer` varchar(255) NOT NULL, + `DeliveryStatus` varchar(255) NOT NULL, + `OrderSize` int(11) NOT NULL +); +INSERT INTO example1463 VALUES ('Charlie', 'Success', 100); +INSERT INTO example1463 VALUES ('David', 'Success', 110); +INSERT INTO example1463 VALUES ('Charlie', 'Failed', 200); +INSERT INTO example1463 VALUES ('David', 'Success', 100); +INSERT INTO example1463 VALUES ('David', 'Unknown', 100); +INSERT INTO example1463 VALUES ('Edward', 'Success', 150); +INSERT INTO example1463 VALUES ('Edward', 'Pending', 150); + +SELECT Customer, Success, SUM(OrderSize) + FROM (SELECT Customer, + CASE WHEN DeliveryStatus='Success' THEN 'Yes' ELSE 'No' END AS Success, + OrderSize + FROM example1463) as subQ + GROUP BY Success, Customer + WITH ROLLUP; +SELECT Customer, Success, SUM(OrderSize) + FROM (SELECT Customer, + CASE WHEN DeliveryStatus='Success' THEN 'Yes' ELSE 'No' END AS Success, + OrderSize + FROM example1463) as subQ + GROUP BY Success, Customer; +SELECT Customer, Success, SUM(OrderSize) + FROM (SELECT Customer, + CASE WHEN DeliveryStatus='Success' THEN 'Yes' ELSE 'No' END AS Success, + OrderSize + FROM example1463) as subQ + GROUP BY Success, Customer + HAVING Success IS NOT NULL; + +DROP TABLE example1463; +set sql_mode= @save_sql_mode; + +--echo # end of 5.5 diff --git a/mysql-test/t/derived_view.test b/mysql-test/t/derived_view.test index 67899837bb2..d017f847af9 100644 --- a/mysql-test/t/derived_view.test +++ b/mysql-test/t/derived_view.test @@ -1827,6 +1827,60 @@ DROP TABLE t1,t2; --echo # end of 5.3 tests --echo # +--echo # +--echo # Bug mdev-11161: The second execution of prepared statement +--echo # does not use generated key for materialized +--echo # derived table / view +--echo # (actually this is a 5.3 bug.) +--echo # + +create table t1 ( + mat_id MEDIUMINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + matintnum CHAR(6) NOT NULL, + test MEDIUMINT UNSIGNED NULL +); +create table t2 ( + mat_id MEDIUMINT UNSIGNED NOT NULL, + pla_id MEDIUMINT UNSIGNED NOT NULL +); +insert into t1 values + (NULL, 'a', 1), (NULL, 'b', 2), (NULL, 'c', 3), (NULL, 'd', 4), + (NULL, 'e', 5), (NULL, 'f', 6), (NULL, 'g', 7), (NULL, 'h', 8), + (NULL, 'i', 9); +insert into t2 values + (1, 100), (1, 101), (1, 102), (2, 100), (2, 103), (2, 104), + (3, 101), (3, 102), (3, 105); + +explain + SELECT STRAIGHT_JOIN d.pla_id, m2.mat_id + FROM t1 m2 + INNER JOIN + (SELECT mp.pla_id, MIN(m1.matintnum) AS matintnum + FROM t2 mp INNER JOIN t1 m1 ON mp.mat_id=m1.mat_id + GROUP BY mp.pla_id) d + ON d.matintnum=m2.matintnum; + +prepare stmt1 from +"SELECT STRAIGHT_JOIN d.pla_id, m2.mat_id + FROM t1 m2 + INNER JOIN + (SELECT mp.pla_id, MIN(m1.matintnum) AS matintnum + FROM t2 mp INNER JOIN t1 m1 ON mp.mat_id=m1.mat_id + GROUP BY mp.pla_id) d + ON d.matintnum=m2.matintnum"; + +flush status; +execute stmt1; +show status like '%Handler_read%'; + +flush status; +execute stmt1; +show status like '%Handler_read%'; + +deallocate prepare stmt1; + +drop table t1,t2; + # The following command must be the last one the file set optimizer_switch=@exit_optimizer_switch; set join_cache_level=@exit_join_cache_level; diff --git a/mysql-test/t/events_slowlog.test b/mysql-test/t/events_slowlog.test new file mode 100644 index 00000000000..9679714dba3 --- /dev/null +++ b/mysql-test/t/events_slowlog.test @@ -0,0 +1,28 @@ +--source include/not_embedded.inc +# +# MDEV-11552 Queries executed by event scheduler are written to slow log incorrectly or not written at all +# +set @event_scheduler_save= @@global.event_scheduler; +set @slow_query_log_save= @@global.slow_query_log; + +set global event_scheduler= on; +set global slow_query_log= on; +set global long_query_time=0.2; + +create table t1 (i int); +insert into t1 values (0); +create event ev on schedule at CURRENT_TIMESTAMP + INTERVAL 1 second do update t1 set i=1+sleep(0.5); + +--let wait_condition= select i from t1 where i > 0 +--source include/wait_condition.inc + +--let SEARCH_FILE = `SELECT @@slow_query_log_file` +--let SEARCH_PATTERN= update t1 set i=1 +--let SEARCH_RANGE= -1000 +--source include/search_pattern_in_file.inc + +drop table t1; + +set global event_scheduler= @event_scheduler_save; +set global slow_query_log= @slow_query_log_save; +set global long_query_time= @@session.long_query_time; diff --git a/mysql-test/t/fulltext_charsets.test b/mysql-test/t/fulltext_charsets.test new file mode 100644 index 00000000000..3ac9791bd1a --- /dev/null +++ b/mysql-test/t/fulltext_charsets.test @@ -0,0 +1,10 @@ +# +# MDEV-11241 Certain combining marks cause MariaDB to crash when doing Full-Text searches +# +set names utf8mb4; + +create table t1 (a int, b text, fulltext (b)) charset=utf8mb4 collate=utf8mb4_unicode_ci; +insert t1 values (1000, 'C͓̙̯͔̩ͅͅi̩̘̜̲a̯̲̬̳̜̖̤o͕͓̜͓̺̖̗,Ì Ì¬Íš ̺T͇̲h͈̱e ̬̜DÌ–o̦̖͔̗͖̩̘c̣̼tÌ͉̫̮̗o͉̫Ìr̙͎̗.͓̪̥'); +select a from t1 where match(b) against ('ciao' in boolean mode); +drop table t1; + diff --git a/mysql-test/t/func_time.test b/mysql-test/t/func_time.test index f2c53bd5ece..e807132ecc7 100644 --- a/mysql-test/t/func_time.test +++ b/mysql-test/t/func_time.test @@ -1668,6 +1668,11 @@ INSERT INTO t1 VALUES (18, '2010-10-13'); SELECT a.id,a.date1,FROM_DAYS(TO_DAYS(a.date1)-10) as date2, DATE_ADD(a.date1,INTERVAL -10 DAY),TO_DAYS(a.date1)-10 FROM t1 a ORDER BY a.id; DROP TABLE t1; +--echo # +--echo # MDEV-10524 Assertion `arg1_int >= 0' failed in Item_func_additive_op::result_precision() +--echo # +SELECT 1 MOD ADDTIME( '13:58:57', '00:00:01' ) + 2; + --echo # --echo # Start of 10.0 tests @@ -1705,6 +1710,31 @@ SELECT TIMEDIFF(20140101000000.000 , 20140302010000.000 ) AS dec_dec, TIMEDIFF(20140101000000.000 , '2014-03-02 01:00:00' ) AS dec_str; + +--echo # +--echo # MDEV-10787 Assertion `ltime->neg == 0' failed in void date_to_datetime(MYSQL_TIME*) +--echo # +CREATE TABLE t1 (d DATE); +INSERT INTO t1 VALUES ('2005-07-20'),('2012-12-21'); +SELECT REPLACE( ADDDATE( d, INTERVAL 0.6732771076944444 HOUR_SECOND ), '2', 'x' ) FROM t1; +SELECT REPLACE( ADDDATE( d, INTERVAL '0.6732771076944444' HOUR_SECOND ), '2', 'x' ) FROM t1; +SELECT CAST(ADDDATE( d, INTERVAL 6732771076944444 SECOND) AS CHAR) FROM t1; +SELECT CAST(ADDDATE( d, INTERVAL '67327710769444:44' HOUR_SECOND) AS CHAR) FROM t1; +SELECT CAST(ADDDATE( d, INTERVAL '673277107694:44:44' HOUR_SECOND) AS CHAR) FROM t1; +DROP TABLE t1; + +# Maximum possible DAY_SECOND values in various formats +SELECT ADDDATE(DATE'0000-01-01', INTERVAL '3652423:23:59:59' DAY_SECOND); +SELECT ADDDATE(DATE'0000-01-01', INTERVAL '0:87658175:59:59' DAY_SECOND); +SELECT ADDDATE(DATE'0000-01-01', INTERVAL '0:0:5259490559:59' DAY_SECOND); +SELECT ADDDATE(DATE'0000-01-01', INTERVAL '0:0:0:315569433599' DAY_SECOND); + +# Out-of-range INTERVAL DAY_SECOND values +SELECT ADDDATE(DATE'0000-01-01', INTERVAL '3652423:0:0:315569433559' DAY_SECOND); +SELECT ADDDATE(DATE'0000-01-01', INTERVAL '0:87658175:0:315569433559' DAY_SECOND); +SELECT ADDDATE(DATE'0000-01-01', INTERVAL '0:0:5259490559:315569433599' DAY_SECOND); + + --echo # --echo # End of 10.0 tests --echo # diff --git a/mysql-test/t/group_by.test b/mysql-test/t/group_by.test index 4b992faa306..8ea2f923eb1 100644 --- a/mysql-test/t/group_by.test +++ b/mysql-test/t/group_by.test @@ -1690,6 +1690,18 @@ SELECT MAX(i), c FROM t1 WHERE c != 'qux' AND ( SELECT SUM(j) FROM t1, t2 ) IS NOT NULL GROUP BY c; drop table t1,t2; +--echo # +--echo # ONLY_FULL_GROUP_BY references +--echo # + +set @save_sql_mode = @@sql_mode; +set sql_mode='ONLY_FULL_GROUP_BY'; +create table t1 (a int, b int); +select a+b as x from t1 group by x having x > 1; +select a as x from t1 group by x having x > 1; +select a from t1 group by a having a > 1; +drop table t1; +set sql_mode= @save_sql_mode; # # End of MariaDB 5.5 tests # diff --git a/mysql-test/t/group_by_innodb.test b/mysql-test/t/group_by_innodb.test index e072a94fada..074df222294 100644 --- a/mysql-test/t/group_by_innodb.test +++ b/mysql-test/t/group_by_innodb.test @@ -125,4 +125,14 @@ ORDER BY id DESC; DROP TABLE t1, t2; +--echo # +--echo # MDEV-11162: Assertion `num_records == m_idx_array.size()' failed in Filesort_buffer::alloc_sort_buffer(uint, uint) +--echo # + +CREATE TABLE t1 (i INT) ENGINE=InnoDB; +SELECT ( SELECT DISTINCT GROUP_CONCAT(SLEEP(0)) FROM t1 GROUP BY i ); +SELECT i FROM t1 order by i LIMIT 1; +DROP TABLE t1; + --echo # End of tests + diff --git a/mysql-test/t/index_merge_innodb.test b/mysql-test/t/index_merge_innodb.test index 6a1cb53dc40..fb56e44b5ae 100644 --- a/mysql-test/t/index_merge_innodb.test +++ b/mysql-test/t/index_merge_innodb.test @@ -171,6 +171,37 @@ WHERE ( tb.b != ta.b OR tb.a = ta.a ) AND ( tb.b = ta.c OR tb.b = ta.b ); DROP TABLE t1; - set optimizer_switch= @optimizer_switch_save; +--echo # +--echo # MDEV-10927: Crash When Using sort_union Optimization +--echo # + +set @tmp_optimizer_switch=@@optimizer_switch; +SET optimizer_switch='index_merge_sort_intersection=on'; +SET SESSION sort_buffer_size = 1024; + +create table t1 ( +pk int(11) NOT NULL AUTO_INCREMENT, +col1 int(11) NOT NULL, +col2 int(11) NOT NULL, +col3 int(11) NOT NULL, +key2 int(11) NOT NULL, +col4 int(11) NOT NULL, +key1 int(11) NOT NULL, +PRIMARY KEY (pk), +KEY key1 (key1), +KEY key2 (key2) +) ENGINE=InnoDB AUTO_INCREMENT=12860259 DEFAULT CHARSET=utf8 ROW_FORMAT=COMPACT; + +create table t2(a int); +insert into t2 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); + +create table t3(a int); +insert into t3 select A.a + B.a* 10 + C.a * 100 + D.a*1000 from t2 A, t2 B, t2 C, t2 D; + +insert into t1 (key1, key2, col1,col2,col3,col4) +select a,a, a,a,a,a from t3; +SELECT sum(col1) FROM t1 FORCE INDEX (key1,key2) WHERE (key1 between 10 and 8191+10) or (key2= 5); +drop table t1,t2,t3; +set optimizer_switch=@tmp_optimizer_switch; diff --git a/mysql-test/t/information_schema_part.test b/mysql-test/t/information_schema_part.test index f1415d12f79..ea88f364c07 100644 --- a/mysql-test/t/information_schema_part.test +++ b/mysql-test/t/information_schema_part.test @@ -131,3 +131,10 @@ drop table if exists t1; create table t1 (f1 int key) partition by key(f1) partitions 2; select create_options from information_schema.tables where table_schema="test"; drop table t1; + +--echo # +--echo # MDEV-11353 - Identical logical conditions +--echo # +CREATE TABLE t1(a INT) CHECKSUM=1 SELECT 1; +SELECT CHECKSUM FROM INFORMATION_SCHEMA.PARTITIONS WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DROP TABLE t1; diff --git a/mysql-test/t/join_cache.test b/mysql-test/t/join_cache.test index 7d873c555f6..aecc934aa2b 100644 --- a/mysql-test/t/join_cache.test +++ b/mysql-test/t/join_cache.test @@ -3708,9 +3708,11 @@ FROM LEFT JOIN t2 c23 ON c23.parent_id = t.id AND c23.col2 = "val" LEFT JOIN t2 c24 ON c24.parent_id = t.id AND c24.col2 = "val" LEFT JOIN t2 c25 ON c25.parent_id = t.id AND c25.col2 = "val" + LEFT JOIN t2 c26 ON c26.parent_id = t.id AND c26.col2 = "val" + LEFT JOIN t2 c27 ON c27.parent_id = t.id AND c27.col2 = "val" ORDER BY col1; -select timestampdiff(second, @init_time, now()) <= 1; +select timestampdiff(second, @init_time, now()) <= 5; set join_cache_level=2; @@ -3743,9 +3745,11 @@ FROM LEFT JOIN t2 c23 ON c23.parent_id = t.id AND c23.col2 = "val" LEFT JOIN t2 c24 ON c24.parent_id = t.id AND c24.col2 = "val" LEFT JOIN t2 c25 ON c25.parent_id = t.id AND c25.col2 = "val" + LEFT JOIN t2 c26 ON c26.parent_id = t.id AND c26.col2 = "val" + LEFT JOIN t2 c27 ON c27.parent_id = t.id AND c27.col2 = "val" ORDER BY col1; -select timestampdiff(second, @init_time, now()) <= 1; +select timestampdiff(second, @init_time, now()) <= 5; EXPLAIN SELECT t.* @@ -3776,6 +3780,8 @@ FROM LEFT JOIN t2 c23 ON c23.parent_id = t.id AND c23.col2 = "val" LEFT JOIN t2 c24 ON c24.parent_id = t.id AND c24.col2 = "val" LEFT JOIN t2 c25 ON c25.parent_id = t.id AND c25.col2 = "val" + LEFT JOIN t2 c26 ON c26.parent_id = t.id AND c26.col2 = "val" + LEFT JOIN t2 c27 ON c27.parent_id = t.id AND c27.col2 = "val" ORDER BY col1; diff --git a/mysql-test/t/loaddata.test b/mysql-test/t/loaddata.test index 7d0f3852b66..1bc7eb139b9 100644 --- a/mysql-test/t/loaddata.test +++ b/mysql-test/t/loaddata.test @@ -612,7 +612,7 @@ disconnect con1; --echo # CREATE TABLE t1(f1 INT); -EVAL SELECT 0xE1C330 INTO OUTFILE 't1.dat'; +EVAL SELECT 0xE1BB30 INTO OUTFILE 't1.dat'; --disable_warnings LOAD DATA INFILE 't1.dat' IGNORE INTO TABLE t1 CHARACTER SET utf8; --enable_warnings @@ -658,26 +658,21 @@ SET @@sql_mode= @old_mode; --remove_file $MYSQLTEST_VARDIR/mysql DROP TABLE t1; ---echo + +--echo # +--echo # MDEV-11079 Regression: LOAD DATA INFILE lost BLOB support using utf8 load files +--echo # + +CREATE TABLE t1 (a mediumblob NOT NULL) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_general_ci; +LOAD DATA INFILE '../../std_data/loaddata/mdev-11079.txt' INTO TABLE t1 CHARSET utf8 FIELDS TERMINATED BY ';' ENCLOSED BY '"' ESCAPED BY '\\' LINES TERMINATED BY '\n'; +SELECT HEX(a) FROM t1; +DROP TABLE t1; + --echo # ---echo # Bug#23080148 - Backport of Bug#20683959. ---echo # Bug#20683959 LOAD DATA INFILE IGNORES A SPECIFIC ROW SILENTLY ---echo # UNDER DB CHARSET IS UTF8. +--echo # MDEV-11631 LOAD DATA INFILE fails to load data with an escape character followed by a multi-byte character --echo # -CREATE DATABASE d1 CHARSET latin1; -USE d1; -CREATE TABLE t1 (val TEXT); -LOAD DATA INFILE '../../std_data/bug20683959loaddata.txt' INTO TABLE t1; -SELECT COUNT(*) FROM t1; -SELECT HEX(val) FROM t1; - -CREATE DATABASE d2 CHARSET utf8; -USE d2; -CREATE TABLE t1 (val TEXT); ---error ER_INVALID_CHARACTER_STRING -LOAD DATA INFILE '../../std_data/bug20683959loaddata.txt' INTO TABLE t1; - -DROP TABLE d1.t1, d2.t1; -DROP DATABASE d1; -DROP DATABASE d2; +CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET utf8); +LOAD DATA INFILE '../../std_data/loaddata/mdev-11631.txt' INTO TABLE t1 CHARACTER SET utf8; +SELECT HEX(a) FROM t1; +DROP TABLE t1; diff --git a/mysql-test/t/log_slow.test b/mysql-test/t/log_slow.test index 8d5a09d7a94..56e35bd5a20 100644 --- a/mysql-test/t/log_slow.test +++ b/mysql-test/t/log_slow.test @@ -50,7 +50,6 @@ set global slow_query_log=1; set global log_output='TABLE'; select sleep(0.5); select count(*) FROM mysql.slow_log; -truncate mysql.slow_log; # Reset used variables set @@long_query_time=default; @@ -58,3 +57,4 @@ set global slow_query_log= @org_slow_query_log; set @@log_slow_filter=default; set @@log_slow_verbosity=default; set global log_output= default; +truncate mysql.slow_log; diff --git a/mysql-test/t/mdev-504.test b/mysql-test/t/mdev-504.test index fb5c7666d33..b96c8779c68 100644 --- a/mysql-test/t/mdev-504.test +++ b/mysql-test/t/mdev-504.test @@ -1,3 +1,4 @@ +--source include/not_valgrind.inc --disable_ps_protocol SET GLOBAL net_write_timeout = 900; diff --git a/mysql-test/t/order_by.test b/mysql-test/t/order_by.test index cf6a4d473c3..a86405889b1 100644 --- a/mysql-test/t/order_by.test +++ b/mysql-test/t/order_by.test @@ -1945,6 +1945,16 @@ order by A.col2, B.col2 limit 10, 1000000; drop table t1,t2,t3; +--echo # +--echo # mdev-10705 : long order by list that can be skipped +--echo # + +SELECT 1 +UNION +( SELECT 2 + ORDER BY NULL, @a0 := 3, @a1 := 3, @a2 := 3, @a3 := 3, @a4 := 3, + @a5 := 3, @a6 := 3, @a7 := 3, @a8 := 3, @a9 := 3, @a10 := 3 ); + --echo End of 5.5 tests --echo # diff --git a/mysql-test/t/parser.test b/mysql-test/t/parser.test index 496a6829577..3058a70eefc 100644 --- a/mysql-test/t/parser.test +++ b/mysql-test/t/parser.test @@ -770,3 +770,13 @@ select 2>!0, 2 > ! 0; select 0<=!0, 0 <= !0; select 1<<!0, 1 << !0; select 0<!0, 0 < ! 0; + +--echo # +--echo # MDEV-11171 Assertion `m_cpp_buf <= ptr && ptr <= m_cpp_buf + m_buf_length' failed in Lex_input_stream::body_utf8_append(const char*, const char*) +--echo # +CREATE TABLE t1 (id INT); +--error ER_PARSE_ERROR +CREATE TRIGGER tr AFTER DELETE ON t1 FOR EACH ROW SET @a = 1\; +--error ER_PARSE_ERROR +PREPARE stmt FROM 'CREATE TRIGGER tr AFTER DELETE ON t1 FOR EACH ROW SET @a = 1\\'; +DROP TABLE t1; diff --git a/mysql-test/t/pool_of_threads.cnf b/mysql-test/t/pool_of_threads.cnf index c03e1da6450..f6651c878de 100644 --- a/mysql-test/t/pool_of_threads.cnf +++ b/mysql-test/t/pool_of_threads.cnf @@ -7,8 +7,5 @@ loose-thread_pool_max_threads= 2 extra-port= @ENV.MASTER_EXTRA_PORT extra-max-connections=1 -[client] -connect-timeout= 2 - [ENV] MASTER_EXTRA_PORT= @OPT.port diff --git a/mysql-test/t/pool_of_threads.test b/mysql-test/t/pool_of_threads.test index 24e0218db62..f13a096985c 100644 --- a/mysql-test/t/pool_of_threads.test +++ b/mysql-test/t/pool_of_threads.test @@ -15,23 +15,26 @@ SET optimizer_switch=@save_optimizer_switch; # connections on the extra port. # First set two connections running, and check that extra connection -# on normal port fails due to--thread-pool-max_threads=2 +# on normal port fails due to --thread-pool-max-threads=2. +# We can afford using a really long sleep, because we won't wait +# till it ends, we'll interrupt it as soon as we don't need it anymore + connection default; +--let $con1_id= `SELECT CONNECTION_ID()` -# Sleep for slightly longer than 5 sec to trigger MDEV-4566 -# (abort in interruptible wait connection check) -send SELECT sleep(5.5); +send SELECT sleep(50); --sleep 1 connect(con2,localhost,root,,); -connection con2; -send SELECT sleep(5); +--let $con2_id= `SELECT CONNECTION_ID()` + +send SELECT sleep(50); --sleep 0.5 --disable_abort_on_error --disable_result_log --disable_query_log -connect(con3,localhost,root,,); +connect(con3,localhost,root,,,,,connect_timeout=2); --enable_query_log --enable_result_log --enable_abort_on_error @@ -45,24 +48,15 @@ if ($error) --echo # -- Success: more than --thread_pool_max_threads normal connections not possible } -connection default; ---reap -connection con2; ---reap - -# Now try again, but this time use the extra port to successfully connect. - -connection default; -send SELECT sleep(5); - -connection con2; -send SELECT sleep(5); ---sleep 1 - connect(extracon,127.0.0.1,root,,test,$MASTER_EXTRA_PORT,); connection extracon; SELECT 'Connection on extra port ok'; +# Here, sleep just for slightly longer than 5 sec to trigger MDEV-4566 +# (abort in interruptible wait connection check). +send SELECT sleep(5.5); + + connect(extracon2,127.0.0.1,root,,test,$MASTER_EXTRA_PORT,); connection extracon2; SELECT 'Connection on extra port 2 ok'; @@ -70,7 +64,7 @@ SELECT 'Connection on extra port 2 ok'; --disable_abort_on_error --disable_result_log --disable_query_log -connect(extracon3,127.0.0.1,root,,test,$MASTER_EXTRA_PORT,); +connect(extracon3,127.0.0.1,root,,test,$MASTER_EXTRA_PORT,,connect_timeout=2); --enable_query_log --enable_result_log --enable_abort_on_error @@ -84,7 +78,16 @@ if ($error) --echo # -- Success: more than --extra-max-connections + 1 normal connections not possible } +connection extracon2; +--replace_result $con1_id <default_connection_ID> +eval KILL QUERY $con1_id; +--replace_result $con2_id <con2_connection_ID> +eval KILL QUERY $con2_id; + connection default; --reap connection con2; --reap + +connection extracon; +--reap diff --git a/mysql-test/t/selectivity.test b/mysql-test/t/selectivity.test index 8efc5216ba0..548ef295fb2 100644 --- a/mysql-test/t/selectivity.test +++ b/mysql-test/t/selectivity.test @@ -1025,3 +1025,24 @@ drop table t1,t2; set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; set use_stat_tables=@save_use_stat_tables; +--echo # +--echo # Bug mdev-11364: IS NULL over not nullable datetime column +--echo # in mergeable derived +--echo # + +set use_stat_tables='preferably'; +set optimizer_use_condition_selectivity=4; +set HISTOGRAM_SIZE = 255; + +CREATE TABLE t1 (t TIME, d DATE NOT NULL); +INSERT INTO t1 VALUES ('10:00:00', '0000-00-00'),('11:00:00','0000-00-00'); + +ANALYZE TABLE t1; + +SELECT * FROM (SELECT t FROM t1 WHERE d IS NULL) sq; + +DROP TABLE t1; + +set histogram_size=@save_histogram_size; +set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; +set use_stat_tables=@save_use_stat_tables; diff --git a/mysql-test/t/sp-prelocking.test b/mysql-test/t/sp-prelocking.test index c1378d59196..38cbd5aa110 100644 --- a/mysql-test/t/sp-prelocking.test +++ b/mysql-test/t/sp-prelocking.test @@ -414,3 +414,33 @@ SELECT f1(); DROP FUNCTION f1; DROP VIEW v1; DROP TABLE t1,t2; + +--echo # +--echo # Bug #16672723 "CAN'T FIND TEMPORARY TABLE". +--echo # +CREATE FUNCTION f1() RETURNS INT RETURN 1; +CREATE TEMPORARY TABLE tmp1(a INT); +PREPARE stmt1 FROM "CREATE TEMPORARY TABLE tmp2 AS SELECT b FROM (SELECT f1() AS b FROM tmp1) AS t"; +--echo # The below statement failed before the fix. +EXECUTE stmt1; +DROP TEMPORARY TABLES tmp1, tmp2; +DEALLOCATE PREPARE stmt1; +DROP FUNCTION f1; + +# +# MDEV-9084 Calling a stored function from a nested select from temporary table causes unpredictable behavior +# +delimiter $$; +create procedure sp1() +begin + drop table if exists t1, t2; + create temporary table t1 select 1 v; + create table t2 (col varchar(45)) select distinct col from (select sf1() as col from t1) t; +end$$ +delimiter ;$$ +create function sf1() returns text return 'blah'; +call test.sp1(); +call test.sp1(); +drop procedure sp1; +drop function sf1; +drop table t2; diff --git a/mysql-test/t/sp.test b/mysql-test/t/sp.test index c62257049ee..10c95f9a919 100644 --- a/mysql-test/t/sp.test +++ b/mysql-test/t/sp.test @@ -9302,6 +9302,57 @@ CALL p1; DROP PROCEDURE p1; DROP TABLE t1; +--echo # +--echo # MDEV-10713: signal 11 error on multi-table update - crash in +--echo # handler::increment_statistics or in make_select or assertion +--echo # failure pfs_thread == ((PFS_thread*) pthread_getspecific((THR_PFS))) +--echo # + +CREATE TABLE `t1` ( + `CLOSE_YN` varchar(10) COLLATE utf8_bin DEFAULT NULL +) DEFAULT CHARSET=utf8 COLLATE=utf8_bin ; + + +CREATE TABLE `t2` ( + `ap_close_to` varchar(8) COLLATE utf8_bin DEFAULT NULL +) DEFAULT CHARSET=utf8 COLLATE=utf8_bin ; +insert t1 values (1); + + +--delimiter $$ + +CREATE FUNCTION `f1`(`P_DC_CD` VARBINARY(50), `P_SYS_DATE` DATETIME) RETURNS datetime + DETERMINISTIC + SQL SECURITY INVOKER +BEGIN + DECLARE V_SYS_DATE DATETIME; + SELECT now() AS LOC_DATE INTO V_SYS_DATE ; + RETURN v_sys_date ; +END $$ + +--delimiter ; + +update t1 S +JOIN +( + SELECT CASE + WHEN DATE_FORMAT( f1('F01', NOW()) , '%Y%m%d') <= CLOSE_YMD + THEN '99991231' + ELSE '' END ACCOUNT_APPLY_YYYYMMDD + FROM ( + select case + when 'AP'='AP' + then ap_close_to + end AS CLOSE_YMD + from t2 + ) A +) X +SET S.CLOSE_YN = '' +where 1=1; + +drop function if exists f1; +drop table t1,t2; + --echo # End of 5.5 test --echo # diff --git a/mysql-test/t/subselect.test b/mysql-test/t/subselect.test index 45c4fe920b7..ab864803864 100644 --- a/mysql-test/t/subselect.test +++ b/mysql-test/t/subselect.test @@ -5977,6 +5977,19 @@ DROP TABLE t1; SET SESSION big_tables=0; --echo # +--echo # MDEV-10776: Server crash on query +--echo # +create table t1 (field1 int); + +insert into t1 values (1); + +select round((select 1 from t1 limit 1)) +from t1 +group by round((select 1 from t1 limit 1)); + +drop table t1; + +--echo # --echo # MDEV-7930: Assertion `table_share->tmp_table != NO_TMP_TABLE || --echo # m_lock_type != 2' failed in handler::ha_index_read_map --echo # @@ -5998,3 +6011,13 @@ SELECT ( SELECT MIN(v2.f2) FROM t1 ) AS sq FROM v2 GROUP BY sq; drop view v2; drop table t1,t2; + +--echo # +--echo # MDEV-10386 Assertion `fixed == 1' failed in virtual String* Item_func_conv_charset::val_str(String*) +--echo # + +CREATE TABLE t1 (f1 CHAR(3) CHARACTER SET utf8 NULL, f2 CHAR(3) CHARACTER SET latin1 NULL); +INSERT INTO t1 VALUES ('foo','bar'); +SELECT * FROM t1 WHERE f2 >= SOME ( SELECT f1 FROM t1 ); +SELECT * FROM t1 WHERE f2 <= SOME ( SELECT f1 FROM t1 ); +DROP TABLE t1; diff --git a/mysql-test/t/subselect2.test b/mysql-test/t/subselect2.test index b3c1322184d..ae210b865a2 100644 --- a/mysql-test/t/subselect2.test +++ b/mysql-test/t/subselect2.test @@ -359,5 +359,55 @@ where t1.a = t2.a and ( t1.a = ( select min(a) from t1 ) or 0 ); drop table t1,t2,t3; +--echo # +--echo # MDEV-10148: Database crashes in the query to the View +--echo # +CREATE TABLE t1 ( + key_code INT(11) NOT NULL, + value_string VARCHAR(50) NULL DEFAULT NULL, + PRIMARY KEY (key_code) +) COLLATE='utf8_general_ci' ENGINE=InnoDB ; + +CREATE TABLE t2 ( + key_code INT(11) NOT NULL, + target_date DATE NULL DEFAULT NULL, + PRIMARY KEY (key_code) +) COLLATE='utf8_general_ci' ENGINE=InnoDB ; + +CREATE TABLE t3 ( + now_date DATE NOT NULL, + PRIMARY KEY (now_date) +) COLLATE='utf8_general_ci' ENGINE=InnoDB ; + +CREATE VIEW v1 +AS +SELECT + B.key_code, + B.target_date +FROM + t2 B INNER JOIN t3 C ON + B.target_date = C.now_date +; +SET @s = 'SELECT A.* FROM t1 A WHERE A.key_code IN (SELECT key_code FROM v1)'; +PREPARE stmt FROM @s; +EXECUTE stmt; #1st time -> success +EXECUTE stmt; #2nd time -> crash +DEALLOCATE PREPARE stmt; +DROP VIEW v1; +DROP TABLE t1,t2,t3; + set optimizer_switch=@subselect2_test_tmp; +# +# Bug #23303485 : HANDLE_FATAL_SIGNAL (SIG=11) IN SUBSELECT_UNION_ENGINE::NO_ROWS +# +create table t1 (a int); +create table t2 (a int); +create table t3(a int); +insert into t1 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +insert into t2 select a from t1; +insert into t3 select a from t1; +--error ER_SUBQUERY_NO_1_ROW +select null in (select a from t1 where a < out3.a union select a from t2 where + (select a from t3) +1 < out3.a+1) from t3 out3; +drop table t1, t2, t3; diff --git a/mysql-test/t/subselect4.test b/mysql-test/t/subselect4.test index 4eb9701ee71..cb102f8319e 100644 --- a/mysql-test/t/subselect4.test +++ b/mysql-test/t/subselect4.test @@ -1956,5 +1956,46 @@ SELECT x FROM t1 WHERE id > (SELECT MAX(id) - 1000 FROM t1) ORDER BY x LIMIT 1; drop table t1; +--echo # +--echo # MDEV-7691: Assertion `outer_context || !*from_field || *from_field == not_found_field' ... +--echo # +set optimizer_switch=default; +CREATE TABLE t1 (a INT) ENGINE=MyISAM; +INSERT INTO t1 VALUES (4),(6); + +CREATE TABLE t2 (b INT) ENGINE=MyISAM; +INSERT INTO t2 VALUES (1),(8); + +PREPARE stmt FROM " +SELECT * FROM t2 +HAVING 0 IN ( + SELECT a FROM t1 + WHERE a IN ( + SELECT a FROM t1 + WHERE b = a + ) +) +"; + +EXECUTE stmt; +EXECUTE stmt; + +--echo # Alternative test case, without HAVING +CREATE TABLE t3 (i INT) ENGINE=MyISAM; +INSERT INTO t3 VALUES (4),(6); + +PREPARE stmt FROM " +SELECT * FROM t3 AS t10 +WHERE EXISTS ( + SELECT * FROM t3 AS t20 WHERE t10.i IN ( + SELECT i FROM t3 + ) +)"; + +EXECUTE stmt; +EXECUTE stmt; + +drop table t1, t2, t3; + SET optimizer_switch= @@global.optimizer_switch; set @@tmp_table_size= @@global.tmp_table_size; diff --git a/mysql-test/t/type_decimal.test b/mysql-test/t/type_decimal.test index 659e75270ca..834fd0c5327 100644 --- a/mysql-test/t/type_decimal.test +++ b/mysql-test/t/type_decimal.test @@ -604,6 +604,11 @@ SHOW COLUMNS FROM t1dec102; SELECT COLUMN_NAME, DATA_TYPE, COLUMN_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME='t1dec102'; DROP TABLE t1dec102; +# +# MDEV-10552 equality operation on cast of the value "-0.0" to decimal not working +# +select cast('-0.0' as decimal(5,1)) < 0; + --echo # --echo # End of 5.5 tests --echo # diff --git a/mysql-test/t/union.test b/mysql-test/t/union.test index 9204ddd22e5..f4dc6a5d449 100644 --- a/mysql-test/t/union.test +++ b/mysql-test/t/union.test @@ -1022,7 +1022,6 @@ ORDER BY a; DROP TABLE t1; ---echo End of 5.0 tests -- echo # -- echo # Bug#32858: Error: "Incorrect usage of UNION and INTO" does not take -- echo # subselects into account @@ -1126,6 +1125,8 @@ create table t1 (a int); insert into t1 values (10),(10),(10),(2),(3),(4),(5),(6),(7),(8),(9),(1),(10); --sorted_result select a from t1 where false UNION select a from t1 limit 8; +--sorted_result +(select a from t1 where false) UNION (select a from t1) limit 8; drop table t1; --echo # @@ -1350,3 +1351,22 @@ UNION ; drop table t1; + + +--echo # +--echo # MDEV-10172: UNION query returns incorrect rows outside +--echo # conditional evaluation +--echo # + +create table t1 (d datetime not null primary key); +insert into t1(d) values ('2016-06-01'),('2016-06-02'),('2016-06-03'),('2016-06-04'); +select * from +( + select * from t1 where d between '2016-06-02' and '2016-06-05' + union + (select * from t1 where d < '2016-06-05' order by d desc limit 1) +) onlyJun2toJun4 +order by d; +drop table t1; + +--echo End of 5.0 tests diff --git a/mysql-test/t/view.test b/mysql-test/t/view.test index 241b3b414f6..5ea97f6465e 100644 --- a/mysql-test/t/view.test +++ b/mysql-test/t/view.test @@ -5694,6 +5694,28 @@ drop table t1, t2; --error ER_BAD_FIELD_ERROR SELECT 1 FROM (SELECT 1 as a) AS b HAVING (SELECT `SOME_GARBAGE`.b.a)=1; + +--echo # +--echo # MDEV-10035: DBUG_ASSERT on CREATE VIEW v1 AS SELECT * FROM t1 +--echo # FOR UPDATE +--echo # + +CREATE TABLE t1 (a INT); +insert into t1 values (1),(2); + +CREATE VIEW v1 AS SELECT * FROM t1 FOR UPDATE; +SHOW CREATE VIEW v1; +select * from v1; +DROP VIEW v1; + +CREATE VIEW v1 AS SELECT * FROM t1 LOCK IN SHARE MODE; +SHOW CREATE VIEW v1; +select * from v1; +DROP VIEW v1; + +DROP TABLE t1; + + --echo # ----------------------------------------------------------------- --echo # -- End of 10.0 tests. --echo # ----------------------------------------------------------------- diff --git a/mysql-test/unstable-tests b/mysql-test/unstable-tests index 3e25115599f..e4c57d9f9d7 100644 --- a/mysql-test/unstable-tests +++ b/mysql-test/unstable-tests @@ -23,47 +23,65 @@ # ############################################################################## +main.count_distinct2 : MDEV-11768 - timeout +main.create : Modified in 10.0.29 main.create_delayed : MDEV-10605 - failed with timeout -main.ctype_utf32 : Modified on 2016-09-27 (merge) -main.func_group : Modified on 2016-08-08 (MDEV-10468) -main.func_math : Modified on 2016-08-10 (merge) -main.func_misc : Modified on 2016-08-10 (merge) -main.group_min_max_innodb : Modified on 2016-08-25 (MDEV-10595) +main.ctype_ucs2_def : Modified in 10.0.29 +main.ctype_ucs2_query_cache : Modified in 10.0.29 +main.ctype_utf16_def : Modified in 10.0.29 +main.ctype_utf8 : Modified in 10.0.29 +main.ctype_utf8mb4 : Modified in 10.0.29 +main.ctype_utf8mb4 : Modified in 10.0.29 (load file changed) +main.debug_sync : MDEV-10607 - internal error +main.default : Modified in 10.0.29 +main.derived : Modified in 10.0.29 +main.derived_opt : MDEV-11768 - timeout +main.derived_view : Modified in 10.0.29 +main.events_restart : MDEV-11221 - Assertion failure +main.events_slowlog : Added in 10.0.29 +main.fulltext_charsets : Added in 10.0.29 +main.func_time : Modified in 10.0.29 +main.group_by : Modified in 10.0.29 +main.group_by_innodb : Modified in 10.0.29 main.host_cache_size_functionality : MDEV-10606 - sporadic failure on shutdown main.index_intersect_innodb : MDEV-10643 - failed with timeout -main.index_merge_myisam : Modified on 2016-09-05 (include file changed) -main.index_merge_innodb : Modified on 2016-09-05 (MDEV-7142) -main.information_schema_stats : Modified on 2016-07-25 (MDEV-10428) +main.index_merge_innodb : MDEV-7142 - wrong result +main.information_schema_part : Modified in 10.0.29 main.innodb_mysql_lock : MDEV-7861 - sporadic lock detection failure -main.loaddata : Modified on 2016-08-10 (merge) +main.join_cache : Modified in 10.0.29 +main.loaddata : Modified in 10.0.29 +main.log_slow : Modified in 10.0.29 main.mdev-504 : MDEV-10607 - sporadic "can't connect" main.mdev375 : MDEV-10607 - sporadic "can't connect" main.merge : MDEV-10607 - sporadic "can't connect" -main.myisam_enable_keys-10506 : New test, added on 2016-08-10 (MDEV-10506) -main.mysqlcheck : Modified on 2016-08-10 (merge) -main.mysqldump : MDEV-10512 - sporadic assertion failure main.mysqlhotcopy_myisam : MDEV-10995 - test hangs on debug build main.mysqltest : MDEV-9269 - fails on Alpha -main.named_pipe : Modified on 2016-08-02 (MDEV-10383) -main.pool_of_threads : MDEV-10100 - sporadic error on detecting max connections +main.order_by : Modified in 10.0.29 +main.parser : Modified in 10.0.29 +main.pool_of_threads : Modified in 10.0.29 main.ps : MDEV-11017 - sporadic wrong Prepared_stmt_count -main.range : Modified on 2016-08-10 (merge) -main.range_mrr_icp : Modified on 2016-08-10 (merge) -main.query_cache : MDEV-10611 - sporadic mutex problem -main.shutdown : MDEV-10563 - sporadic crashes -main.sp-prelocking : Modified on 2016-08-10 (merge) +main.selectivity : Modified in 10.0.29 +main.signal_demo3 : MDEV-11720 - Thread stack overrun on Solaris +main.sp : Modified in 10.0.29 +main.sp_notembedded : MDEV-10607 - internal error +main.sp-prelocking : Modified in 10.0.29 main.sp-security : MDEV-10607 - sporadic "can't connect" -main.ssl_compress : MDEV-11110 - valgrind failures main.stat_tables_par_innodb : MDEV-10515 - sporadic wrong results +main.subselect : Modified in 10.0.29 +main.subselect2 : Modified in 10.0.29 +main.subselect4 : Modified in 10.0.29 main.subselect_innodb : MDEV-10614 - sporadic wrong results -main.type_date : Modified on 2016-08-10 (merge) -main.type_uint : Modified on 2016-09-27 (merge) -main.view : Modified on 2016-08-10 (merge) -main.xtradb_mrr : Modified on 2016-08-04 (MDEV-9946) +main.subselect_sj2_jcl6 : MDEV-11766 - unexpected warnings +main.type_bit_innodb : MDEV-11766 - unexpected warnings +main.type_decimal : Modified in 10.0.29 +main.union : Modified in 10.0.29 +main.view : Modified in 10.0.29 +main.xa : MDEV-11769 - lock wait timeout #---------------------------------------------------------------- archive.archive-big : MDEV-10615 - table is marked as crashed +archive.archive_bitfield : MDEV-11771 - table is marked as crashed archive.discover : MDEV-10510 - table is marked as crashed archive.mysqlhotcopy_archive : MDEV-10995 - test hangs on debug build @@ -74,6 +92,8 @@ binlog.binlog_xa_recover : MDEV-8517 - Extra checkpoint #---------------------------------------------------------------- +connect.jdbc : re-enabled in 10.0.29 +connect.jdbc_new : re-enabled in 10.0.29 connect.tbl : MDEV-9844, MDEV-10179 - sporadic crashes, valgrind warnings, wrong results #---------------------------------------------------------------- @@ -82,10 +102,6 @@ engines/rr_trx.* : MDEV-10998 - tests not maintained #---------------------------------------------------------------- -extra/binlog_tests.database : Modified on 2016-10-21 (Upstream MIPS test fixes) - -#---------------------------------------------------------------- - federated.federatedx : MDEV-10617 - Wrong checksum, timeouts federated.federated_innodb : MDEV-10617, MDEV-10417 - Wrong checksum, timeouts, fails on Mips federated.federated_partition : MDEV-10417 - Fails on Mips @@ -93,19 +109,23 @@ federated.federated_transactions : MDEV-10617, MDEV-10417 - Wrong checksum, time #---------------------------------------------------------------- -funcs_2/charset.* : MDEV-10999 - test not maintained +funcs_1.memory_views : MDEV-11773 - timeout +funcs_2/charset.* : MDEV-10999 - test not maintained #---------------------------------------------------------------- innodb.binlog_consistent : MDEV-10618 - Server fails to start +innodb.group_commit_crash_no_optimize_thread : MDEV-11770 - checksum mismatch innodb.innodb-alter-table : MDEV-10619 - Testcase timeout -innodb.innodb-alter-tempfile : Modified on 2016-08-09 (MDEV-10469) innodb.innodb_bug30423 : MDEV-7311 - Wrong number of rows in the plan -innodb.innodb_bug54044 : Modified on 2016-09-27 (merge) +innodb.innodb_bug53290 : MDEV-11767 - timeout +innodb.innodb_bug56143 : MDEV-11766 - unexpected warnings innodb.innodb_monitor : MDEV-10939 - Testcase timeout -innodb.innodb-wl5522 : rdiff file modified on 2016-08-10 (merge) -innodb.innodb-wl5522-debug-zip : MDEV-10427 - Warning: database page corruption -innodb.system_tables : Added on 2016-09-23 (MDEV-10775) +innodb.innodb-wl5522-debug-zip : Modified in 10.0.29 +innodb.table_index_statistics : Modified in 10.0.29 + +innodb_fts.innodb_fts_misc : MDEV-11767 - timeout +innodb_fts.innodb_fts_plugin : MDEV-11766 - unexpected warnings #---------------------------------------------------------------- @@ -125,29 +145,41 @@ multi_source.status_vars : MDEV-4632 - failed while waiting for Slave_received_h #---------------------------------------------------------------- +parts.partition_exch_qa_10 : MDEV-11765 - wrong result parts.partition_float_myisam : MDEV-10621 - Testcase timeout parts.partition_int_myisam : MDEV-10621 - Testcase timeout #---------------------------------------------------------------- +percona.percona_xtradb_bug317074 : MDEV-11767 - timeout + +#---------------------------------------------------------------- + perfschema.func_file_io : MDEV-5708 - fails for s390x perfschema.func_mutex : MDEV-5708 - fails for s390x perfschema.hostcache_ipv6_ssl : MDEV-10696 - crash on shutdown +perfschema.table_name : MDEV-11764 - wrong result perfschema.socket_summary_by_event_name_func : MDEV-10622 - Socket summary tables do not match perfschema_stress.* : MDEV-10996 - tests not maintained #---------------------------------------------------------------- -plugins.feedback_plugin_send : MDEV-7932 - ssl failed for url, MDEV-11112 - valgrind warnings -plugins.pam : Modified on 2016-08-03 (MDEV-7329) -plugins.pam_cleartext : Modified on 2016-08-03 +plugins.feedback_plugin_send : MDEV-7932 - ssl failed for url, MDEV-11118 - wrong result plugins.server_audit : MDEV-9562 - crashes on sol10-sparc plugins.thread_pool_server_audit : MDEV-9562 - crashes on sol10-sparc + +#---------------------------------------------------------------- + +roles.create_and_drop_role : Modified in 10.0.29 +roles.create_and_grant_role : MDEV-11772 - wrong result +roles.role_case_sensitive-10744 : Added in 10.0.29 + #---------------------------------------------------------------- rpl.last_insert_id : MDEV-10625 - warnings in error log +rpl.rpl_alter_extra_persistent : Added in 10.0.29 rpl.rpl_auto_increment : MDEV-10417 - Fails on Mips rpl.rpl_auto_increment_bug45679 : MDEV-10417 - Fails on Mips rpl.rpl_auto_increment_update_failure : MDEV-10625 - warnings in error log @@ -155,29 +187,31 @@ rpl.rpl_binlog_index : MDEV-9501 - Warning: failed registering rpl.rpl_checksum_cache : MDEV-10626 - Testcase timeout rpl.rpl_circular_for_4_hosts : MDEV-10627 - Testcase timeout rpl.rpl_ddl : MDEV-10417 - Fails on Mips -rpl.rpl_drop_db : Modified on 2016-10-21 (Upstream MIPS test fixes) rpl.rpl_gtid_crash : MDEV-9501 - Warning: failed registering on master rpl.rpl_gtid_master_promote : MDEV-10628 - Timeout in sync_with_master rpl.rpl_gtid_stop_start : MDEV-10629 - Crash on shutdown rpl.rpl_gtid_until : MDEV-10625 - warnings in error log +rpl.rpl_heartbeat_basic : MDEV-11668 - Wrong result rpl.rpl_innodb_bug30888 : MDEV-10417 - Fails on Mips rpl.rpl_insert : MDEV-9329 - Fails on Ubuntu/s390x rpl.rpl_insert_delayed : MDEV-9329 - Fails on Ubuntu/s390x rpl.rpl_invoked_features : MDEV-10417 - Fails on Mips +rpl.rpl_mdev10863 : Added in 10.0.29 rpl.rpl_mdev6020 : MDEV-10630, MDEV-10417 - Timeouts, fails on Mips rpl.rpl_mdev6386 : MDEV-10631 - Wrong result on slave rpl.rpl_parallel : MDEV-10632, MDEV-10653 - Failures to sync, timeouts rpl.rpl_parallel_temptable : MDEV-10356 - Crash in close_thread_tables rpl.rpl_partition_innodb : MDEV-10417 - Fails on Mips rpl.rpl_row_drop_create_temp_table : MDEV-10626 - Testcase timeout +rpl.rpl_row_mysqlbinlog : Modified in 10.0.29 rpl.rpl_row_sp001 : MDEV-9329 - Fails on Ubuntu/s390x rpl.rpl_semi_sync_uninstall_plugin : MDEV-7140 - Wrong plugin status rpl.rpl_slave_grp_exec : MDEV-10514 - Unexpected deadlock -rpl.rpl_switch_stm_row_mixed : MDEV-10611 - Wrong usage of mutex +rpl.rpl_special_charset : Modified in 10.0.29 rpl.rpl_sync : MDEV-10633 - Database page corruption rpl.rpl_temporary_error2 : MDEV-10634 - Wrong number of retries -rpl.sec_behind_master-5114 : MDEV-8518 - Wrong value of Seconds_Behind_Master rpl.rpl_skip_replication : MDEV-9268 - Fails with timeout in sync_slave_with_master on Alpha +rpl.sec_behind_master-5114 : Modified in 10.0.29 rpl/extra/rpl_tests.* : MDEV-10994 - tests not maintained @@ -187,6 +221,8 @@ spider.* : MDEV-9329 - tests are too memory-consuming spider/bg.direct_aggregate : MDEV-7098 - Trying to unlock mutex that wasn't locked spider/bg.direct_aggregate_part : MDEV-7098 - Trying to unlock mutex that wasn't locked +spider/bg.spider3_fixes : MDEV-7098 - Trying to unlock mutex that wasn't locked +spider/bg.spider_fixes_part : MDEV-7098 - Trying to unlock mutex that wasn't locked spider/bg.ha : MDEV-7914, MDEV-9329 - Crash, failures on s390x spider/bg.ha_part : MDEV-9329 - Fails on Ubuntu/s390x spider/bg.spider_fixes : MDEV-7098, MDEV-9329 - Mutex problem, failures on s390x @@ -194,7 +230,7 @@ spider/bg.vp_fixes : MDEV-9329 - Fails on Ubuntu/s390x #---------------------------------------------------------------- -sphinx.* : MDEV-10747 - tests are not run in buildbot, they can't be stable +sphinx.* : MDEV-10986 - sphinx tests fail in buildbot and outside #---------------------------------------------------------------- @@ -203,34 +239,33 @@ stress.ddl_innodb : MDEV-10635 - Testcase timeout #---------------------------------------------------------------- sys_vars.autocommit_func2 : MDEV-9329 - Fails on Ubuntu/s390x -sys_vars.general_log_file_basic : Modified on 2016-08-09 (MDEV-10465) -sys_vars.slow_query_log_file_basic : Modified on 2016-08-09 (MDEV-10465) sys_vars.innodb_buffer_pool_dump_pct_basic : MDEV-10651 - sporadic failure on file_exists +sys_vars.replicate_do_db_basic : Modified in 10.0.29 +sys_vars.replicate_do_table_basic : Modified in 10.0.29 +sys_vars.replicate_ignore_db_basic : Modified in 10.0.29 +sys_vars.replicate_ignore_table_basic : Modified in 10.0.29 +sys_vars.replicate_wild_do_table_basic : Modified in 10.0.29 +sys_vars.replicate_wild_ignore_table_basic : Modified in 10.0.29 +sys_vars.thread_cache_size_func : MDEV-11775 - wrong result #---------------------------------------------------------------- -tokudb.background_job_manager : MDEV-10327 - Assertion failure on server shutdown tokudb.cluster_filter_unpack_varchar : MDEV-10636 - Wrong execution plan -tokudb.* : MDEV-9891 - massive crashes on shutdown -tokudb_alter_table.* : MDEV-9891 - massive crashes on shutdown +tokudb.dir_per_db : MDEV-11537 - wrong result +tokudb.locks-select-update-3 : MDEV-11774 - lock wait timeout +tokudb.table_index_statistics : Added in 10.0.29 + tokudb_backup.* : MDEV-11001 - tests don't work tokudb_bugs.checkpoint_lock : MDEV-10637 - Wrong processlist output tokudb_bugs.checkpoint_lock_3 : MDEV-10637 - Wrong processlist output -tokudb_bugs.* : MDEV-9891 - massive crashes on shutdown -tokudb_parts.* : MDEV-9891 - massive crashes on shutdown -tokudb_rpl_suites.* : MDEV-11001 - tests don't work -tokudb_sys_vars.* : MDEV-11001 - tests don't work -rpl-tokudb.* : MDEV-9891 - massive crashes on shutdown -tokudb/tokudb_add_index.* : MDEV-9891 - massive crashes on shutdown -tokudb/tokudb_backup.* : MDEV-9891 - massive crashes on shutdown -tokudb/tokudb_mariadb.* : MDEV-9891 - massive crashes on shutdown -tokudb/tokudb_sys_vars.* : MDEV-9891 - massive crashes on shutdown -tokudb/tokudb_rpl.* : MDEV-9891 - massive crashes on shutdown +tokudb_rpl.* : MDEV-11001 - tests don't work +tokudb_sys_vars.* : MDEV-11001 - tests don't work #---------------------------------------------------------------- unit.ma_test_loghandler : MDEV-10638 - record read not ok +unit.pfs : MySQL:84457 - unittest pft-t failing #---------------------------------------------------------------- diff --git a/mysql-test/valgrind.supp b/mysql-test/valgrind.supp index 5f1af1a0905..bbef8a0fa04 100644 --- a/mysql-test/valgrind.supp +++ b/mysql-test/valgrind.supp @@ -353,10 +353,23 @@ Memcheck:Leak fun:memalign ... - fun:call_init + fun:call_init* + fun:_dl_init +} + +# This one is on OpenSuse 10.3 with gcc 5.4 +{ + memory "loss" from _dl_init 2 + Memcheck:Leak + fun:malloc + fun:pool + ... + fun:call_init* fun:_dl_init } + + # # dlclose can allocate memory for error message, the memory will be # freed by dlerror or other dl* function. @@ -954,6 +967,14 @@ fun:backtrace } +{ + memory leak in mysqld_exit + Memcheck:Leak + fun:malloc + fun:_dl_close_worker + fun:_dl_close +} + # # Bug in Glibc 2.9: http://sourceware.org/bugzilla/show_bug.cgi?id=10391 # Fixed in latest Glibc, but suppressed here for running tests on hosts diff --git a/mysys/CMakeLists.txt b/mysys/CMakeLists.txt index 4756bbccf2f..a2429e01eaa 100644 --- a/mysys/CMakeLists.txt +++ b/mysys/CMakeLists.txt @@ -46,7 +46,7 @@ IF (WIN32) ENDIF() IF(UNIX) - SET (MYSYS_SOURCES ${MYSYS_SOURCES} my_addr_resolve.c) + SET (MYSYS_SOURCES ${MYSYS_SOURCES} my_addr_resolve.c my_setuser.c) ENDIF() IF(HAVE_ALARM) diff --git a/mysys/my_default.c b/mysys/my_default.c index 1ad2fa2090d..d63090f95e1 100644 --- a/mysys/my_default.c +++ b/mysys/my_default.c @@ -99,7 +99,7 @@ char wsrep_defaults_group_suffix[FN_EXTLEN]= {0,}; /* Which directories are searched for options (and in which order) */ -#define MAX_DEFAULT_DIRS 6 +#define MAX_DEFAULT_DIRS 7 #define DEFAULT_DIRS_SIZE (MAX_DEFAULT_DIRS + 1) /* Terminate with NULL */ static const char **default_directories = NULL; @@ -1244,7 +1244,12 @@ static const char **init_default_directories(MEM_ROOT *alloc) errors += add_directory(alloc, "C:/", dirs); if (my_get_module_parent(fname_buffer, sizeof(fname_buffer)) != NULL) + { + errors += add_directory(alloc, fname_buffer, dirs); + + strncat(fname_buffer, "/data", sizeof(fname_buffer)); errors += add_directory(alloc, fname_buffer, dirs); + } } #else diff --git a/mysys/my_setuser.c b/mysys/my_setuser.c new file mode 100644 index 00000000000..14ab04dd10f --- /dev/null +++ b/mysys/my_setuser.c @@ -0,0 +1,82 @@ +#include <my_global.h> +#include <m_string.h> +#include <my_sys.h> +#include <my_pthread.h> +#ifdef HAVE_PWD_H +#include <pwd.h> +#endif +#ifdef HAVE_GRP_H +#include <grp.h> +#endif + +struct passwd *my_check_user(const char *user, myf MyFlags) +{ + struct passwd *user_info; + uid_t user_id= geteuid(); + DBUG_ENTER("my_check_user"); + + // Don't bother if we aren't superuser + if (user_id) + { + if (user) + { + /* Don't give a warning, if real user is same as given with --user */ + user_info= getpwnam(user); + if (!user_info || user_id != user_info->pw_uid) + { + my_errno= EPERM; + if (MyFlags & MY_WME) + my_printf_error(my_errno, "One can only use the --user switch if " + "running as root", MYF(ME_JUST_WARNING|ME_NOREFRESH)); + } + } + DBUG_RETURN(NULL); + } + if (!user) + { + if (MyFlags & MY_FAE) + { + my_errno= EINVAL; + my_printf_error(my_errno, "Please consult the Knowledge Base to find " + "out how to run mysqld as root!", MYF(ME_NOREFRESH)); + } + DBUG_RETURN(NULL); + } + if (!strcmp(user,"root")) + DBUG_RETURN(NULL); + + if (!(user_info= getpwnam(user))) + { + // Allow a numeric uid to be used + int err= 0; + user_id= my_strtoll10(user, NULL, &err); + if (err || !(user_info= getpwuid(user_id))) + { + my_errno= EINVAL; + my_printf_error(my_errno, "Can't change to run as user '%s'. Please " + "check that the user exists!", MYF(ME_NOREFRESH), user); + DBUG_RETURN(NULL); + } + } + DBUG_ASSERT(user_info); + DBUG_RETURN(user_info); +} + +int my_set_user(const char *user, struct passwd *user_info, myf MyFlags) +{ + DBUG_ENTER("my_set_user"); + + DBUG_ASSERT(user_info != 0); +#ifdef HAVE_INITGROUPS + initgroups(user, user_info->pw_gid); +#endif + if (setgid(user_info->pw_gid) == -1 || setuid(user_info->pw_uid) == -1) + { + my_errno= errno; + if (MyFlags & MY_WME) + my_printf_error(errno, "Cannot change uid/gid (errno: %d)", MYF(ME_NOREFRESH), + errno); + DBUG_RETURN(my_errno); + } + DBUG_RETURN(0); +} diff --git a/plugin/server_audit/server_audit.c b/plugin/server_audit/server_audit.c index d48b6c37728..4c5e6992a32 100644 --- a/plugin/server_audit/server_audit.c +++ b/plugin/server_audit/server_audit.c @@ -2843,6 +2843,7 @@ void __attribute__ ((constructor)) audit_plugin_so_init(void) _mysql_plugin_declarations_[0].info= mysql_v4_descriptor; use_event_data_for_disconnect= 1; } + MYSQL_SYSVAR_NAME(loc_info).flags= PLUGIN_VAR_READONLY | PLUGIN_VAR_MEMALLOC; } memset(locinfo_ini_value, 'O', sizeof(locinfo_ini_value)-1); diff --git a/scripts/mysql_install_db.sh b/scripts/mysql_install_db.sh index bdce0857d1e..c0a6675be58 100644 --- a/scripts/mysql_install_db.sh +++ b/scripts/mysql_install_db.sh @@ -513,10 +513,8 @@ then echo "The latest information about MariaDB is available at http://mariadb.org/." echo "You can find additional information about the MySQL part at:" echo "http://dev.mysql.com" - echo "Support MariaDB development by buying support/new features from MariaDB" - echo "Corporation Ab. You can contact us about this at sales@mariadb.com." - echo "Alternatively consider joining our community based development effort:" - echo "http://mariadb.com/kb/en/contributing-to-the-mariadb-project/" + echo "Consider joining MariaDB's strong and vibrant community:" + echo "https://mariadb.org/get-involved/" echo fi diff --git a/scripts/mysqld_safe.sh b/scripts/mysqld_safe.sh index 9d8bbce4860..fadf98c1ba4 100644 --- a/scripts/mysqld_safe.sh +++ b/scripts/mysqld_safe.sh @@ -20,6 +20,7 @@ mysqld_ld_preload= mysqld_ld_library_path= flush_caches=0 numa_interleave=0 +unsafe_my_cnf=0 # Initial logging status: error log is not open, and not using syslog logging=init @@ -129,6 +130,18 @@ my_which () return $ret # Success } +find_in_bin() { + if test -x "$MY_BASEDIR_VERSION/bin/$1" + then + echo "$MY_BASEDIR_VERSION/bin/$1" + elif test -x "@bindir@/$1" + then + echo "@bindir@/$1" + else + echo "$1" + fi +} + log_generic () { priority="$1" shift @@ -137,7 +150,7 @@ log_generic () { echo "$msg" case $logging in init) ;; # Just echo the message, don't save it anywhere - file) echo "$msg" >> "$err_log" ;; + file) echo "$msg" | "$helper" "$user" log "$err_log" ;; syslog) logger -t "$syslog_tag_mysqld_safe" -p "$priority" "$*" ;; *) echo "Internal program error (non-fatal):" \ @@ -157,7 +170,7 @@ log_notice () { eval_log_error () { local cmd="$1" case $logging in - file) cmd="$cmd >> "`shell_quote_string "$err_log"`" 2>&1" ;; + file) cmd="$cmd 2>&1 | "`shell_quote_string "$helper"`" $user log "`shell_quote_string "$err_log"` ;; syslog) # mysqld often prefixes its messages with a timestamp, which is # redundant when logging to syslog (which adds its own timestamp) @@ -274,11 +287,17 @@ wsrep_recover_position() { return $ret } +check_executable_location() { + if test "$unsafe_my_cnf" = 1 -a "$unrecognized_handling" != collect; then + log_error "Cannot accept $1 from a config file, when my.cnf is in the datadir" + exit 1 + fi +} + parse_arguments() { for arg do val=`echo "$arg" | sed -e "s;--[^=]*=;;"` case "$arg" in - --crash[-_]script=*) CRASH_SCRIPT="$val" ;; # these get passed explicitly to mysqld --basedir=*) MY_BASEDIR_VERSION="$val" ;; --datadir=*|--data=*) DATADIR="$val" ;; @@ -305,12 +324,14 @@ parse_arguments() { # mysqld_safe-specific options - must be set in my.cnf ([mysqld_safe])! --core[-_]file[-_]size=*) core_file_size="$val" ;; - --ledir=*) ledir="$val" ;; - --malloc[-_]lib=*) set_malloc_lib "$val" ;; - --mysqld=*) MYSQLD="$val" ;; + --ledir=*) check_executable_location "$arg" ; ledir="$val" ;; + --malloc[-_]lib=*) check_executable_location "$arg"; set_malloc_lib "$val" ;; + --crash[-_]script=*) check_executable_location "$arg"; crash_script="$val" ;; + --mysqld=*) check_executable_location "$arg"; MYSQLD="$val" ;; --mysqld[-_]version=*) if test -n "$val" then + check_executable_location "$arg" MYSQLD="mysqld-$val" PLUGIN_VARIANT="/$val" else @@ -477,15 +498,8 @@ set_malloc_lib() { # First, try to find BASEDIR and ledir (where mysqld is) # -if echo '@pkgdatadir@' | grep '^@prefix@' > /dev/null -then - relpkgdata=`echo '@pkgdatadir@' | sed -e 's,^@prefix@,,' -e 's,^/,,' -e 's,^,./,'` -else - # pkgdatadir is not relative to prefix - relpkgdata='@pkgdatadir@' -fi - -MY_PWD=`pwd` +MY_PWD=`dirname $0` +MY_PWD=`cd "$MY_PWD"/.. && pwd` # Check for the directories we would expect from a binary release install if test -n "$MY_BASEDIR_VERSION" -a -d "$MY_BASEDIR_VERSION" then @@ -501,16 +515,16 @@ then else ledir="$MY_BASEDIR_VERSION/bin" fi -elif test -f "$relpkgdata"/english/errmsg.sys -a -x "$MY_PWD/bin/mysqld" +elif test -x "$MY_PWD/bin/mysqld" then MY_BASEDIR_VERSION="$MY_PWD" # Where bin, share and data are ledir="$MY_PWD/bin" # Where mysqld is # Check for the directories we would expect from a source install -elif test -f "$relpkgdata"/english/errmsg.sys -a -x "$MY_PWD/libexec/mysqld" +elif test -x "$MY_PWD/libexec/mysqld" then MY_BASEDIR_VERSION="$MY_PWD" # Where libexec, share and var are ledir="$MY_PWD/libexec" # Where mysqld is -elif test -f "$relpkgdata"/english/errmsg.sys -a -x "$MY_PWD/sbin/mysqld" +elif test -x "$MY_PWD/sbin/mysqld" then MY_BASEDIR_VERSION="$MY_PWD" # Where sbin, share and var are ledir="$MY_PWD/sbin" # Where mysqld is @@ -520,6 +534,8 @@ else ledir='@libexecdir@' fi +helper=`find_in_bin mysqld_safe_helper` +print_defaults=`find_in_bin my_print_defaults` # # Second, try to find the data directory @@ -557,6 +573,7 @@ IGNORING $DATADIR/my.cnf" log_error "WARNING: Found $DATADIR/my.cnf The data directory is a deprecated location for my.cnf, please move it to $MY_BASEDIR_VERSION/my.cnf" + unsafe_my_cnf=1 MYSQL_HOME=$DATADIR else MYSQL_HOME=$MY_BASEDIR_VERSION @@ -564,34 +581,15 @@ $MY_BASEDIR_VERSION/my.cnf" fi export MYSQL_HOME - -# Get first arguments from the my.cnf file, groups [mysqld] and [mysqld_safe] -# and then merge with the command line arguments -if test -x "$MY_BASEDIR_VERSION/bin/my_print_defaults" -then - print_defaults="$MY_BASEDIR_VERSION/bin/my_print_defaults" -elif test -x `dirname $0`/my_print_defaults -then - print_defaults="`dirname $0`/my_print_defaults" -elif test -x ./bin/my_print_defaults -then - print_defaults="./bin/my_print_defaults" -elif test -x @bindir@/my_print_defaults -then - print_defaults="@bindir@/my_print_defaults" -elif test -x @bindir@/mysql_print_defaults -then - print_defaults="@bindir@/mysql_print_defaults" -else - print_defaults="my_print_defaults" -fi - append_arg_to_args () { args="$args "`shell_quote_string "$1"` } args= +# Get first arguments from the my.cnf file, groups [mysqld] and [mysqld_safe] +# and then merge with the command line arguments + SET_USER=2 parse_arguments `$print_defaults $defaults --loose-verbose --mysqld` if test $SET_USER -eq 2 @@ -700,11 +698,6 @@ then log_notice "Logging to '$err_log'." logging=file - if [ ! -f "$err_log" ]; then # if error log already exists, - touch "$err_log" # we just append. otherwise, - chmod "$fmode" "$err_log" # fix the permissions here! - fi - else if [ -n "$syslog_tag" ] then @@ -717,10 +710,6 @@ else logging=syslog fi -# close stdout and stderr, everything goes to $logging now -exec 1>&- -exec 2>&- - USER_OPTION="" if test -w / -o "$USER" = "root" then @@ -728,11 +717,6 @@ then then USER_OPTION="--user=$user" fi - # Change the err log to the right user, if it is in use - if [ $want_syslog -eq 0 ]; then - touch "$err_log" - chown $user "$err_log" - fi if test -n "$open_files" then ulimit -n $open_files @@ -977,6 +961,15 @@ max_fast_restarts=5 # flag whether a usable sleep command exists have_sleep=1 +# close stdout and stderr, everything goes to $logging now +if expr "${-}" : '.*x' > /dev/null +then + : +else + exec 1>&- + exec 2>&- +fi + # maximum number of wsrep restarts max_wsrep_restarts=0 @@ -1000,12 +993,6 @@ do eval_log_error "$cmd $wsrep_start_position_opt --wsrep_cluster_address=$url $nohup_redir" fi - if [ $want_syslog -eq 0 -a ! -f "$err_log" ]; then - touch "$err_log" # hypothetical: log was renamed but not - chown $user "$err_log" # flushed yet. we'd recreate it with - chmod "$fmode" "$err_log" # wrong owner next time we log, so set - fi # it up correctly while we can! - end_time=`date +%M%S` if test ! -f "$pid_file" # This is removed if normal shutdown @@ -1083,9 +1070,9 @@ do fi log_notice "mysqld restarted" - if test -n "$CRASH_SCRIPT" + if test -n "$crash_script" then - crash_script_output=`$CRASH_SCRIPT 2>&1` + crash_script_output=`$crash_script 2>&1` log_error "$crash_script_output" fi done diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index de46f4b64c1..e917c257a07 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -410,6 +410,7 @@ IF(WIN32) ${CMAKE_CURRENT_BINARY_DIR}/mysql_bootstrap_sql.c COMPONENT Server ) + SET_TARGET_PROPERTIES(mysql_install_db PROPERTIES COMPILE_FLAGS -DINSTALL_PLUGINDIR=${INSTALL_PLUGINDIR}) TARGET_LINK_LIBRARIES(mysql_install_db mysys) ADD_LIBRARY(winservice STATIC winservice.c) diff --git a/sql/event_db_repository.cc b/sql/event_db_repository.cc index 6785b46d0da..66606b18c49 100644 --- a/sql/event_db_repository.cc +++ b/sql/event_db_repository.cc @@ -18,6 +18,7 @@ #include "sql_priv.h" #include "unireg.h" #include "sql_base.h" // close_thread_tables +#include "sql_parse.h" #include "event_db_repository.h" #include "key.h" // key_copy #include "sql_db.h" // get_default_db_collation @@ -704,19 +705,17 @@ Event_db_repository::create_event(THD *thd, Event_parse_data *parse_data, restore_record(table, s->default_values); // Get default values for fields - if (system_charset_info->cset-> - numchars(system_charset_info, parse_data->dbname.str, - parse_data->dbname.str + parse_data->dbname.length) > - table->field[ET_FIELD_DB]->char_length()) + if (check_string_char_length(&parse_data->dbname, 0, + table->field[ET_FIELD_DB]->char_length(), + system_charset_info, 1)) { my_error(ER_TOO_LONG_IDENT, MYF(0), parse_data->dbname.str); goto end; } - if (system_charset_info->cset-> - numchars(system_charset_info, parse_data->name.str, - parse_data->name.str + parse_data->name.length) > - table->field[ET_FIELD_NAME]->char_length()) + if (check_string_char_length(&parse_data->name, 0, + table->field[ET_FIELD_NAME]->char_length(), + system_charset_info, 1)) { my_error(ER_TOO_LONG_IDENT, MYF(0), parse_data->name.str); goto end; diff --git a/sql/field.cc b/sql/field.cc index 49989a4a4ed..dec89e16674 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -1269,12 +1269,15 @@ void Field_num::prepend_zeros(String *value) int diff; if ((diff= (int) (field_length - value->length())) > 0) { - bmove_upp((uchar*) value->ptr()+field_length, - (uchar*) value->ptr()+value->length(), - value->length()); - bfill((uchar*) value->ptr(),diff,'0'); - value->length(field_length); - (void) value->c_ptr_quick(); // Avoid warnings in purify + const bool error= value->realloc(field_length); + if (!error) + { + bmove_upp((uchar*) value->ptr()+field_length, + (uchar*) value->ptr()+value->length(), + value->length()); + bfill((uchar*) value->ptr(),diff,'0'); + value->length(field_length); + } } } diff --git a/sql/filesort.cc b/sql/filesort.cc index 6ad7bee48c6..73a6c89e53f 100644 --- a/sql/filesort.cc +++ b/sql/filesort.cc @@ -1629,7 +1629,7 @@ int merge_buffers(Sort_param *param, IO_CACHE *from_file, if (!(error= (int) read_to_buffer(from_file, buffpek, rec_length))) { - queue_remove(&queue,0); + (void) queue_remove_top(&queue); reuse_freed_buff(&queue, buffpek, rec_length); } else if (error == -1) diff --git a/sql/handler.cc b/sql/handler.cc index a142bb4e0de..6749dd0257e 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -4594,7 +4594,7 @@ void handler::get_dynamic_partition_info(PARTITION_STATS *stat_info, stat_info->update_time= stats.update_time; stat_info->check_time= stats.check_time; stat_info->check_sum= 0; - if (table_flags() & (HA_HAS_OLD_CHECKSUM | HA_HAS_OLD_CHECKSUM)) + if (table_flags() & (HA_HAS_OLD_CHECKSUM | HA_HAS_NEW_CHECKSUM)) stat_info->check_sum= checksum(); return; } diff --git a/sql/item.cc b/sql/item.cc index 21c8b3f701e..86f1795a4da 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -1206,7 +1206,8 @@ Item *Item_cache::safe_charset_converter(CHARSET_INFO *tocs) if (conv == example) return this; Item_cache *cache; - if (!conv || !(cache= new Item_cache_str(conv))) + if (!conv || conv->fix_fields(current_thd, (Item **) NULL) || + !(cache= new Item_cache_str(conv))) return NULL; // Safe conversion is not possible, or OEM cache->setup(conv); cache->fixed= false; // Make Item::fix_fields() happy @@ -2742,6 +2743,44 @@ void Item_field::fix_after_pullout(st_select_lex *new_parent, Item **ref) depended_from= NULL; if (context) { + bool need_change= false; + /* + Suppose there are nested selects: + + select_id=1 + select_id=2 + select_id=3 <----+ + select_id=4 -+ + select_id=5 --+ + + Suppose, pullout operation has moved anything that had select_id=4 or 5 + in to select_id=3. + + If this Item_field had a name resolution context pointing into select_lex + with id=4 or id=5, it needs a new name resolution context. + + However, it could also be that this object is a part of outer reference: + Item_ref(Item_field(field in select with select_id=1))). + - The Item_ref object has a context with select_id=5, and so needs a new + name resolution context. + - The Item_field object has a context with select_id=1, and doesn't need + a new name resolution context. + + So, the following loop walks from Item_field's current context upwards. + If we find that the select we've been pulled out to is up there, we + create the new name resolution context. Otherwise, we don't. + */ + for (Name_resolution_context *ct= context; ct; ct= ct->outer_context) + { + if (new_parent == ct->select_lex) + { + need_change= true; + break; + } + } + if (!need_change) + return; + Name_resolution_context *ctx= new Name_resolution_context(); if (context->select_lex == new_parent) { @@ -4498,8 +4537,6 @@ static Item** find_field_in_group_list(Item *find_item, ORDER *group_list) const char *field_name; ORDER *found_group= NULL; int found_match_degree= 0; - Item_ident *cur_field; - int cur_match_degree= 0; char name_buff[SAFE_NAME_LEN+1]; if (find_item->type() == Item::FIELD_ITEM || @@ -4524,54 +4561,70 @@ static Item** find_field_in_group_list(Item *find_item, ORDER *group_list) for (ORDER *cur_group= group_list ; cur_group ; cur_group= cur_group->next) { - if ((*(cur_group->item))->real_item()->type() == Item::FIELD_ITEM) + int cur_match_degree= 0; + + /* SELECT list element with explicit alias */ + if ((*(cur_group->item))->name && + !(*(cur_group->item))->is_autogenerated_name && + !my_strcasecmp(system_charset_info, + (*(cur_group->item))->name, field_name)) + { + ++cur_match_degree; + } + /* Reference on the field or view/derived field. */ + else if ((*(cur_group->item))->type() == Item::FIELD_ITEM || + (*(cur_group->item))->type() == Item::REF_ITEM ) { - cur_field= (Item_ident*) *cur_group->item; - cur_match_degree= 0; - - DBUG_ASSERT(cur_field->field_name != 0); + Item_ident *cur_field= (Item_ident*) *cur_group->item; + const char *l_db_name= cur_field->db_name; + const char *l_table_name= cur_field->table_name; + const char *l_field_name= cur_field->field_name; + + DBUG_ASSERT(l_field_name != 0); if (!my_strcasecmp(system_charset_info, - cur_field->field_name, field_name)) + l_field_name, field_name)) ++cur_match_degree; else continue; - if (cur_field->table_name && table_name) + if (l_table_name && table_name) { /* If field_name is qualified by a table name. */ - if (my_strcasecmp(table_alias_charset, cur_field->table_name, table_name)) + if (my_strcasecmp(table_alias_charset, l_table_name, table_name)) /* Same field names, different tables. */ return NULL; ++cur_match_degree; - if (cur_field->db_name && db_name) + if (l_db_name && db_name) { /* If field_name is also qualified by a database name. */ - if (strcmp(cur_field->db_name, db_name)) + if (strcmp(l_db_name, db_name)) /* Same field names, different databases. */ return NULL; ++cur_match_degree; } } + } + else + continue; - if (cur_match_degree > found_match_degree) - { - found_match_degree= cur_match_degree; - found_group= cur_group; - } - else if (found_group && (cur_match_degree == found_match_degree) && - ! (*(found_group->item))->eq(cur_field, 0)) - { - /* - If the current resolve candidate matches equally well as the current - best match, they must reference the same column, otherwise the field - is ambiguous. - */ - my_error(ER_NON_UNIQ_ERROR, MYF(0), - find_item->full_name(), current_thd->where); - return NULL; - } + if (cur_match_degree > found_match_degree) + { + found_match_degree= cur_match_degree; + found_group= cur_group; + } + else if (found_group && (cur_match_degree == found_match_degree) && + !(*(found_group->item))->eq((*(cur_group->item)), 0)) + { + /* + If the current resolve candidate matches equally well as the current + best match, they must reference the same column, otherwise the field + is ambiguous. + */ + my_error(ER_NON_UNIQ_ERROR, MYF(0), + find_item->full_name(), current_thd->where); + return NULL; } } @@ -5675,6 +5728,7 @@ String *Item::check_well_formed_result(String *str, bool send_error) /* Check whether we got a well-formed string */ CHARSET_INFO *cs= str->charset(); uint wlen= str->well_formed_length(); + null_value= false; if (wlen < str->length()) { THD *thd= current_thd; @@ -9932,4 +9986,3 @@ const char *dbug_print_item(Item *item) } #endif /*DBUG_OFF*/ - diff --git a/sql/item.h b/sql/item.h index e402112b725..2f8607d8fad 100644 --- a/sql/item.h +++ b/sql/item.h @@ -3713,7 +3713,7 @@ public: if (result_type() == ROW_RESULT) orig_item->bring_value(); } - virtual bool is_expensive() { return orig_item->is_expensive(); } + bool is_expensive() { return orig_item->is_expensive(); } bool is_expensive_processor(uchar *arg) { return orig_item->is_expensive_processor(arg); } bool check_vcol_func_processor(uchar *arg) @@ -4326,6 +4326,10 @@ public: int save_in_field(Field *field_arg, bool no_conversions); table_map used_tables() const { return (table_map)0L; } + Field *get_tmp_table_field() { return 0; } + Item *get_tmp_table_item(THD *thd) { return this; } + Item_field *field_for_view_update() { return 0; } + bool walk(Item_processor processor, bool walk_subquery, uchar *args) { return (arg && arg->walk(processor, walk_subquery, args)) || @@ -4367,6 +4371,8 @@ public: */ table_map used_tables() const { return RAND_TABLE_BIT; } + Item_field *field_for_view_update() { return 0; } + bool walk(Item_processor processor, bool walk_subquery, uchar *args) { return arg->walk(processor, walk_subquery, args) || diff --git a/sql/item_cmpfunc.h b/sql/item_cmpfunc.h index dbd96a89a24..9e83b732dc7 100644 --- a/sql/item_cmpfunc.h +++ b/sql/item_cmpfunc.h @@ -1387,10 +1387,26 @@ public: update_used_tables(); } const char *func_name() const { return "isnull"; } + + bool arg_is_datetime_notnull_field() + { + Item **args= arguments(); + if (args[0]->type() == Item::FIELD_ITEM) + { + Field *field=((Item_field*) args[0])->field; + + if (((field->type() == MYSQL_TYPE_DATE) || + (field->type() == MYSQL_TYPE_DATETIME)) && + (field->flags & NOT_NULL_FLAG)) + return true; + } + return false; + } + /* Optimize case of not_null_column IS NULL */ virtual void update_used_tables() { - if (!args[0]->maybe_null) + if (!args[0]->maybe_null && !arg_is_datetime_notnull_field()) { used_tables_cache= 0; /* is always false */ const_item_cache= 1; diff --git a/sql/item_func.cc b/sql/item_func.cc index 3fafc67e39a..07acaacc862 100644 --- a/sql/item_func.cc +++ b/sql/item_func.cc @@ -6665,7 +6665,8 @@ Item_func_sp::init_result_field(THD *thd) bool Item_func_sp::is_expensive() { - return !(m_sp->m_chistics->detistic); + return !m_sp->m_chistics->detistic || + current_thd->locked_tables_mode < LTM_LOCK_TABLES; } diff --git a/sql/item_subselect.cc b/sql/item_subselect.cc index e70922bb5d3..f3a98785a0b 100644 --- a/sql/item_subselect.cc +++ b/sql/item_subselect.cc @@ -1,5 +1,5 @@ -/* Copyright (c) 2002, 2015, Oracle and/or its affiliates. - Copyright (c) 2010, 2015, MariaDB +/* Copyright (c) 2002, 2016, Oracle and/or its affiliates. + Copyright (c) 2010, 2016, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -222,6 +222,7 @@ bool Item_subselect::select_transformer(JOIN *join) { DBUG_ENTER("Item_subselect::select_transformer"); + DBUG_ASSERT(thd == join->thd); DBUG_RETURN(false); } @@ -592,7 +593,7 @@ bool Item_subselect::is_expensive() examined_rows+= cur_join->get_examined_rows(); } - + // here we are sure that subquery is optimized so thd is set return (examined_rows > thd->variables.expensive_subquery_limit); } @@ -656,6 +657,7 @@ bool Item_subselect::exec() subselect_engine *org_engine= engine; DBUG_ENTER("Item_subselect::exec"); + DBUG_ASSERT(fixed); /* Do not execute subselect in case of a fatal error @@ -704,6 +706,7 @@ int Item_in_subselect::optimize(double *out_rows, double *cost) { int res; DBUG_ENTER("Item_in_subselect::optimize"); + DBUG_ASSERT(fixed); SELECT_LEX *save_select= thd->lex->current_select; JOIN *join= unit->first_select()->join; @@ -818,6 +821,7 @@ bool Item_in_subselect::expr_cache_is_needed(THD *thd) bool Item_in_subselect::exec() { DBUG_ENTER("Item_in_subselect::exec"); + DBUG_ASSERT(fixed); /* Initialize the cache of the left predicate operand. This has to be done as late as now, because Cached_item directly contains a resolved field (not @@ -872,6 +876,7 @@ table_map Item_subselect::used_tables() const bool Item_subselect::const_item() const { + DBUG_ASSERT(thd); return (thd->lex->context_analysis_only ? FALSE : forced_const || const_item_cache); @@ -1065,10 +1070,11 @@ Item_singlerow_subselect::select_transformer(JOIN *join) DBUG_ENTER("Item_singlerow_subselect::select_transformer"); if (changed) DBUG_RETURN(false); + DBUG_ASSERT(join->thd == thd); SELECT_LEX *select_lex= join->select_lex; Query_arena *arena= thd->stmt_arena; - + if (!select_lex->master_unit()->is_union() && !select_lex->table_list.elements && select_lex->item_list.elements == 1 && @@ -1731,6 +1737,7 @@ Item_in_subselect::single_value_transformer(JOIN *join) { SELECT_LEX *select_lex= join->select_lex; DBUG_ENTER("Item_in_subselect::single_value_transformer"); + DBUG_ASSERT(thd == join->thd); /* Check that the right part of the subselect contains no more than one @@ -1842,9 +1849,9 @@ bool Item_allany_subselect::transform_into_max_min(JOIN *join) if (!test_strategy(SUBS_MAXMIN_INJECTED | SUBS_MAXMIN_ENGINE)) DBUG_RETURN(false); Item **place= optimizer->arguments() + 1; - THD *thd= join->thd; SELECT_LEX *select_lex= join->select_lex; Item *subs; + DBUG_ASSERT(thd == join->thd); /* */ @@ -1951,6 +1958,7 @@ bool Item_allany_subselect::transform_into_max_min(JOIN *join) bool Item_in_subselect::fix_having(Item *having, SELECT_LEX *select_lex) { bool fix_res= 0; + DBUG_ASSERT(thd); if (!having->fixed) { select_lex->having_fix_field= 1; @@ -2013,6 +2021,7 @@ Item_in_subselect::create_single_in_to_exists_cond(JOIN *join, Item **having_item) { SELECT_LEX *select_lex= join->select_lex; + DBUG_ASSERT(thd == join->thd); /* The non-transformed HAVING clause of 'join' may be stored in two ways during JOIN::optimize: this->tmp_having= this->having; this->having= 0; @@ -2149,6 +2158,7 @@ Item_in_subselect::row_value_transformer(JOIN *join) uint cols_num= left_expr->cols(); DBUG_ENTER("Item_in_subselect::row_value_transformer"); + DBUG_ASSERT(thd == join->thd); // psergey: duplicated_subselect_card_check if (select_lex->item_list.elements != cols_num) @@ -2260,6 +2270,7 @@ Item_in_subselect::create_row_in_to_exists_cond(JOIN * join, !select_lex->table_list.elements); DBUG_ENTER("Item_in_subselect::create_row_in_to_exists_cond"); + DBUG_ASSERT(thd == join->thd); *where_item= NULL; *having_item= NULL; @@ -2492,6 +2503,7 @@ bool Item_in_subselect::inject_in_to_exists_cond(JOIN *join_arg) Item *having_item= join_arg->in_to_exists_having; DBUG_ENTER("Item_in_subselect::inject_in_to_exists_cond"); + DBUG_ASSERT(thd == join_arg->thd); if (where_item) { @@ -3007,6 +3019,7 @@ Item_in_subselect::select_in_like_transformer(JOIN *join) bool result; DBUG_ENTER("Item_in_subselect::select_in_like_transformer"); + DBUG_ASSERT(thd == join->thd); /* IN/SOME/ALL/ANY subqueries aren't support LIMIT clause. Without it @@ -3217,6 +3230,7 @@ bool Item_in_subselect::setup_mat_engine() subselect_single_select_engine *select_engine; DBUG_ENTER("Item_in_subselect::setup_mat_engine"); + DBUG_ASSERT(thd); /* The select_engine (that executes transformed IN=>EXISTS subselects) is @@ -3255,6 +3269,7 @@ bool Item_in_subselect::setup_mat_engine() bool Item_in_subselect::init_left_expr_cache() { JOIN *outer_join; + DBUG_ASSERT(thd); outer_join= unit->outer_select()->join; /* @@ -3281,6 +3296,7 @@ bool Item_in_subselect::init_left_expr_cache() bool Item_in_subselect::init_cond_guards() { + DBUG_ASSERT(thd); uint cols_num= left_expr->cols(); if (!abort_on_null && left_expr->maybe_null && !pushed_cond_guards) { @@ -3408,8 +3424,12 @@ bool subselect_union_engine::is_executed() const bool subselect_union_engine::no_rows() { + bool rows_present= false; + /* Check if we got any rows when reading UNION result from temp. table: */ - return MY_TEST(!unit->fake_select_lex->join->send_records); + if (unit->fake_select_lex->join) + rows_present= MY_TEST(!unit->fake_select_lex->join->send_records); + return rows_present; } @@ -4792,9 +4812,9 @@ bool subselect_hash_sj_engine::init(List<Item> *tmp_columns, uint subquery_id) result= result_sink; /* - If the subquery has blobs, or the total key lenght is bigger than + If the subquery has blobs, or the total key length is bigger than some length, or the total number of key parts is more than the - allowed maximum (currently MAX_REF_PARTS == 16), then the created + allowed maximum (currently MAX_REF_PARTS == 32), then the created index cannot be used for lookups and we can't use hash semi join. If this is the case, delete the temporary table since it will not be used, and tell the caller we failed to initialize the @@ -6548,4 +6568,3 @@ end: void subselect_table_scan_engine::cleanup() { } - diff --git a/sql/item_timefunc.cc b/sql/item_timefunc.cc index 866dbe65ecc..fa1c3d0da90 100644 --- a/sql/item_timefunc.cc +++ b/sql/item_timefunc.cc @@ -426,7 +426,7 @@ static bool extract_date_time(DATE_TIME_FORMAT *format, { if (!my_isspace(&my_charset_latin1,*val)) { - make_truncated_value_warning(current_thd, Sql_condition::WARN_LEVEL_WARN, + make_truncated_value_warning(current_thd, Sql_condition::WARN_LEVEL_WARN, val_begin, length, cached_timestamp_type, NullS); break; @@ -708,7 +708,7 @@ static bool get_interval_info(const char *str,uint length,CHARSET_INFO *cs, { longlong value; const char *start= str; - for (value=0; str != end && my_isdigit(cs, *str) ; str++) + for (value= 0; str != end && my_isdigit(cs, *str); str++) value= value*10 + *str - '0'; msec_length= 6 - (str - start); values[i]= value; @@ -1460,6 +1460,7 @@ void Item_temporal_func::fix_length_and_dec() time can get us to return NULL. */ maybe_null= 1; + if (decimals) { if (decimals == NOT_FIXED_DEC) diff --git a/sql/log.cc b/sql/log.cc index 7fd185aa5d3..02e6ef98a65 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -292,6 +292,9 @@ public: { compute_statistics(); truncate(0); + if(cache_log.file != -1) + my_chsize(cache_log.file, 0, 0, MYF(MY_WME)); + changes_to_non_trans_temp_table_flag= FALSE; incident= FALSE; before_stmt_pos= MY_OFF_T_UNDEF; @@ -3408,7 +3411,7 @@ bool MYSQL_BIN_LOG::open(const char *log_name, if (init_and_set_log_file_name(log_name, new_name, log_type_arg, io_cache_type_arg)) { - sql_print_error("MSYQL_BIN_LOG::open failed to generate new file name."); + sql_print_error("MYSQL_BIN_LOG::open failed to generate new file name."); DBUG_RETURN(1); } @@ -3435,7 +3438,7 @@ bool MYSQL_BIN_LOG::open(const char *log_name, } }); - sql_print_error("MSYQL_BIN_LOG::open failed to sync the index file."); + sql_print_error("MYSQL_BIN_LOG::open failed to sync the index file."); DBUG_RETURN(1); } DBUG_EXECUTE_IF("crash_create_non_critical_before_update_index", DBUG_SUICIDE();); @@ -4481,14 +4484,14 @@ int MYSQL_BIN_LOG::purge_logs(const char *to_log, if ((error= sync_purge_index_file())) { - sql_print_error("MSYQL_BIN_LOG::purge_logs failed to flush register file."); + sql_print_error("MYSQL_BIN_LOG::purge_logs failed to flush register file."); goto err; } /* We know how many files to delete. Update index file. */ if ((error=update_log_index(&log_info, need_update_threads))) { - sql_print_error("MSYQL_BIN_LOG::purge_logs failed to update the index file"); + sql_print_error("MYSQL_BIN_LOG::purge_logs failed to update the index file"); goto err; } @@ -4498,7 +4501,7 @@ err: /* Read each entry from purge_index_file and delete the file. */ if (is_inited_purge_index_file() && (error= purge_index_entry(thd, reclaimed_space, FALSE))) - sql_print_error("MSYQL_BIN_LOG::purge_logs failed to process registered files" + sql_print_error("MYSQL_BIN_LOG::purge_logs failed to process registered files" " that would be purged."); close_purge_index_file(); @@ -4615,7 +4618,7 @@ int MYSQL_BIN_LOG::purge_index_entry(THD *thd, ulonglong *reclaimed_space, if ((error=reinit_io_cache(&purge_index_file, READ_CACHE, 0, 0, 0))) { - sql_print_error("MSYQL_BIN_LOG::purge_index_entry failed to reinit register file " + sql_print_error("MYSQL_BIN_LOG::purge_index_entry failed to reinit register file " "for read"); goto err; } @@ -4630,7 +4633,7 @@ int MYSQL_BIN_LOG::purge_index_entry(THD *thd, ulonglong *reclaimed_space, if (purge_index_file.error) { error= purge_index_file.error; - sql_print_error("MSYQL_BIN_LOG::purge_index_entry error %d reading from " + sql_print_error("MYSQL_BIN_LOG::purge_index_entry error %d reading from " "register file.", error); goto err; } diff --git a/sql/log_event.cc b/sql/log_event.cc index 1cc58cd76b4..ba1981d3318 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -13044,7 +13044,7 @@ bool rpl_get_position_info(const char **log_file_name, ulonglong *log_pos, return FALSE; #else const Relay_log_info *rli= &(active_mi->rli); - if (opt_slave_parallel_threads == 0) + if (!rli->mi->using_parallel()) { *log_file_name= rli->group_master_log_name; *log_pos= rli->group_master_log_pos + diff --git a/sql/mysql_install_db.cc b/sql/mysql_install_db.cc index c39789f7c97..c23a20ebac9 100644 --- a/sql/mysql_install_db.cc +++ b/sql/mysql_install_db.cc @@ -233,6 +233,20 @@ static void get_basedir(char *basedir, int size, const char *mysqld_path) } } +#define STR(s) _STR(s) +#define _STR(s) #s + +static char *get_plugindir() +{ + static char plugin_dir[2*MAX_PATH]; + get_basedir(plugin_dir, sizeof(plugin_dir), mysqld_path); + strcat(plugin_dir, "/" STR(INSTALL_PLUGINDIR)); + + if (access(plugin_dir, 0) == 0) + return plugin_dir; + + return NULL; +} /** Allocate and initialize command line for mysqld --bootstrap. @@ -313,6 +327,10 @@ static int create_myini() fprintf(myini,"protocol=pipe\n"); else if (opt_port) fprintf(myini,"port=%d\n",opt_port); + + char *plugin_dir = get_plugindir(); + if (plugin_dir) + fprintf(myini, "plugin-dir=%s\n", plugin_dir); fclose(myini); return 0; } diff --git a/sql/mysqld.cc b/sql/mysqld.cc index d43f14c176b..f252d19f50b 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -130,10 +130,7 @@ extern "C" { // Because of SCO 3.2V4.2 #include <sysent.h> #endif #ifdef HAVE_PWD_H -#include <pwd.h> // For getpwent -#endif -#ifdef HAVE_GRP_H -#include <grp.h> +#include <pwd.h> // For struct passwd #endif #include <my_net.h> @@ -481,12 +478,10 @@ my_bool opt_master_verify_checksum= 0; my_bool opt_slave_sql_verify_checksum= 1; const char *binlog_format_names[]= {"MIXED", "STATEMENT", "ROW", NullS}; #ifdef WITH_WSREP -const char *wsrep_binlog_format_names[]= +const char *wsrep_binlog_format_names[]= {"MIXED", "STATEMENT", "ROW", "NONE", NullS}; -#endif /*WITH_WSREP */ -#ifdef HAVE_INITGROUPS +#endif /* WITH_WSREP */ volatile sig_atomic_t calling_initgroups= 0; /**< Used in SIGSEGV handler. */ -#endif uint mysqld_port, test_flags, select_errors, dropping_tables, ha_open_options; uint mysqld_extra_port; uint mysqld_port_timeout; @@ -2365,59 +2360,18 @@ static void set_ports() static struct passwd *check_user(const char *user) { -#if !defined(__WIN__) - struct passwd *tmp_user_info; - uid_t user_id= geteuid(); + myf flags= 0; + if (global_system_variables.log_warnings) + flags|= MY_WME; + if (!opt_bootstrap && !opt_help) + flags|= MY_FAE; - // Don't bother if we aren't superuser - if (user_id) - { - if (user) - { - /* Don't give a warning, if real user is same as given with --user */ - /* purecov: begin tested */ - tmp_user_info= getpwnam(user); - if ((!tmp_user_info || user_id != tmp_user_info->pw_uid) && - global_system_variables.log_warnings) - sql_print_warning( - "One can only use the --user switch if running as root\n"); - /* purecov: end */ - } - return NULL; - } - if (!user) - { - if (!opt_bootstrap && !opt_help) - { - sql_print_error("Fatal error: Please consult the Knowledge Base " - "to find out how to run mysqld as root!\n"); - unireg_abort(1); - } - return NULL; - } - /* purecov: begin tested */ - if (!strcmp(user,"root")) - return NULL; // Avoid problem with dynamic libraries + struct passwd *tmp_user_info= my_check_user(user, MYF(flags)); - if (!(tmp_user_info= getpwnam(user))) - { - // Allow a numeric uid to be used - const char *pos; - for (pos= user; my_isdigit(mysqld_charset,*pos); pos++) ; - if (*pos) // Not numeric id - goto err; - if (!(tmp_user_info= getpwuid(atoi(user)))) - goto err; - } + if (!tmp_user_info && my_errno==EINVAL && (flags & MY_FAE)) + unireg_abort(1); return tmp_user_info; - /* purecov: end */ - -err: - sql_print_error("Fatal error: Can't change to run as user '%s' ; Please check that the user exists!\n",user); - unireg_abort(1); -#endif - return NULL; } static inline void allow_coredumps() @@ -2434,10 +2388,6 @@ static inline void allow_coredumps() static void set_user(const char *user, struct passwd *user_info_arg) { - /* purecov: begin tested */ -#if !defined(__WIN__) - DBUG_ASSERT(user_info_arg != 0); -#ifdef HAVE_INITGROUPS /* We can get a SIGSEGV when calling initgroups() on some systems when NSS is configured to use LDAP and the server is statically linked. We set @@ -2445,22 +2395,11 @@ static void set_user(const char *user, struct passwd *user_info_arg) output a specific message to help the user resolve this problem. */ calling_initgroups= 1; - initgroups((char*) user, user_info_arg->pw_gid); + int res= my_set_user(user, user_info_arg, MYF(MY_WME)); calling_initgroups= 0; -#endif - if (setgid(user_info_arg->pw_gid) == -1) - { - sql_perror("setgid"); + if (res) unireg_abort(1); - } - if (setuid(user_info_arg->pw_uid) == -1) - { - sql_perror("setuid"); - unireg_abort(1); - } allow_coredumps(); -#endif - /* purecov: end */ } @@ -4436,19 +4375,24 @@ static int init_common_variables() default_charset_info= default_collation; } /* Set collactions that depends on the default collation */ - global_system_variables.collation_server= default_charset_info; - global_system_variables.collation_database= default_charset_info; - global_system_variables.collation_connection= default_charset_info; - global_system_variables.character_set_results= default_charset_info; - if (default_charset_info->mbminlen > 1) - { - global_system_variables.character_set_client= &my_charset_latin1; - sql_print_warning("Cannot use %s as character_set_client, %s will be used instead", - default_charset_info->csname, - global_system_variables.character_set_client->csname); + global_system_variables.collation_server= default_charset_info; + global_system_variables.collation_database= default_charset_info; + if (is_supported_parser_charset(default_charset_info)) + { + global_system_variables.collation_connection= default_charset_info; + global_system_variables.character_set_results= default_charset_info; + global_system_variables.character_set_client= default_charset_info; } else - global_system_variables.character_set_client= default_charset_info; + { + sql_print_warning("'%s' can not be used as client character set. " + "'%s' will be used as default client character set.", + default_charset_info->csname, + my_charset_latin1.csname); + global_system_variables.collation_connection= &my_charset_latin1; + global_system_variables.character_set_results= &my_charset_latin1; + global_system_variables.character_set_client= &my_charset_latin1; + } if (!(character_set_filesystem= get_charset_by_csname(character_set_filesystem_name, @@ -6200,6 +6144,12 @@ int mysqld_main(int argc, char **argv) setbuf(stderr, NULL); FreeConsole(); // Remove window } + + if (fileno(stdin) >= 0) + { + /* Disable CRLF translation (MDEV-9409). */ + _setmode(fileno(stdin), O_BINARY); + } #endif #ifdef WITH_WSREP /* WSREP AFTER SE */ diff --git a/sql/rpl_filter.cc b/sql/rpl_filter.cc index 28859c2eb85..b992a71e52e 100644 --- a/sql/rpl_filter.cc +++ b/sql/rpl_filter.cc @@ -280,6 +280,9 @@ Rpl_filter::parse_filter_rule(const char* spec, Add_filter add) int status= 0; char *arg, *ptr, *pstr; + if (!spec) + return false; + if (! (ptr= my_strdup(spec, MYF(MY_WME)))) return true; diff --git a/sql/rpl_mi.h b/sql/rpl_mi.h index a27672e4c90..e58df0150f5 100644 --- a/sql/rpl_mi.h +++ b/sql/rpl_mi.h @@ -75,6 +75,10 @@ class Master_info : public Slave_reporting_capability return connection_name.str == 0; } static const char *using_gtid_astext(enum enum_using_gtid arg); + bool using_parallel() + { + return opt_slave_parallel_threads > 0; + } /* the variables below are needed because we can change masters on the fly */ char master_log_name[FN_REFLEN+6]; /* Room for multi-*/ diff --git a/sql/rpl_parallel.cc b/sql/rpl_parallel.cc index 98712f18c1b..23f61a82a90 100644 --- a/sql/rpl_parallel.cc +++ b/sql/rpl_parallel.cc @@ -2614,7 +2614,6 @@ rpl_parallel::do_event(rpl_group_info *serial_rgi, Log_event *ev, /* Queue the event for processing. */ - rli->event_relay_log_pos= rli->future_event_relay_log_pos; qev->ir= rli->last_inuse_relaylog; ++qev->ir->queued_count; cur_thread->enqueue(qev); diff --git a/sql/rpl_rli.cc b/sql/rpl_rli.cc index 4aed8cdcd94..c570105cdf6 100644 --- a/sql/rpl_rli.cc +++ b/sql/rpl_rli.cc @@ -1231,7 +1231,7 @@ bool Relay_log_info::is_until_satisfied(THD *thd, Log_event *ev) if (ev && ev->server_id == (uint32) global_system_variables.server_id && !replicate_same_server_id) DBUG_RETURN(FALSE); - log_name= (opt_slave_parallel_threads > 0 ? + log_name= (mi->using_parallel() ? future_event_master_log_name : group_master_log_name); log_pos= ((!ev)? group_master_log_pos : (get_flag(IN_TRANSACTION) || !ev->log_pos) ? diff --git a/sql/share/errmsg-utf8.txt b/sql/share/errmsg-utf8.txt index 627f54631a3..d8c64a9d534 100644 --- a/sql/share/errmsg-utf8.txt +++ b/sql/share/errmsg-utf8.txt @@ -6783,7 +6783,7 @@ ER_FK_COLUMN_NOT_NULL ger "Spalte '%-.192s' kann nicht NOT NULL sein: wird für eine Fremdschlüsselbeschränkung '%-.192s' SET NULL benötigt" ER_DUP_INDEX - eng "Duplicate index '%-.64s' defined on the table '%-.64s.%-.64s'. This is deprecated and will be disallowed in a future release." + eng "Duplicate index %`s. This is deprecated and will be disallowed in a future release." ER_FK_COLUMN_CANNOT_CHANGE eng "Cannot change column '%-.192s': used in a foreign key constraint '%-.192s'" diff --git a/sql/signal_handler.cc b/sql/signal_handler.cc index c3f25848e8a..2029c52d17d 100644 --- a/sql/signal_handler.cc +++ b/sql/signal_handler.cc @@ -149,7 +149,7 @@ extern "C" sig_handler handle_fatal_signal(int sig) if (opt_stack_trace) { - my_safe_printf_stderr("Thread pointer: 0x%p\n", thd); + my_safe_printf_stderr("Thread pointer: %p\n", thd); my_safe_printf_stderr("%s", "Attempting backtrace. You can use the following " "information to find out\n" diff --git a/sql/slave.cc b/sql/slave.cc index 359f4f8af9c..ae335a24811 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -623,8 +623,7 @@ int terminate_slave_threads(Master_info* mi,int thread_mask,bool skip_lock) if (thread_mask & (SLAVE_SQL|SLAVE_FORCE_ALL)) { DBUG_PRINT("info",("Terminating SQL thread")); - if (opt_slave_parallel_threads > 0 && - mi->rli.abort_slave && mi->rli.stop_for_until) + if (mi->using_parallel() && mi->rli.abort_slave && mi->rli.stop_for_until) { mi->rli.stop_for_until= false; mi->rli.parallel.stop_during_until(); @@ -2732,8 +2731,7 @@ static bool send_show_master_info_data(THD *thd, Master_info *mi, bool full, else { idle= mi->rli.sql_thread_caught_up; - if (opt_slave_parallel_threads > 0 && idle && - !mi->rli.parallel.workers_idle()) + if (mi->using_parallel() && idle && !mi->rli.parallel.workers_idle()) idle= false; } if (idle) @@ -3534,7 +3532,7 @@ static int exec_relay_log_event(THD* thd, Relay_log_info* rli, the user might be surprised to see a claim that the slave is up to date long before those queued events are actually executed. */ - if (opt_slave_parallel_threads == 0 && + if (!rli->mi->using_parallel() && !(ev->is_artificial_event() || ev->is_relay_log_event() || (ev->when == 0))) { rli->last_master_timestamp= ev->when + (time_t) ev->exec_time; @@ -3585,9 +3583,16 @@ static int exec_relay_log_event(THD* thd, Relay_log_info* rli, update_state_of_relay_log(rli, ev); - if (opt_slave_parallel_threads > 0) + if (rli->mi->using_parallel()) { int res= rli->parallel.do_event(serial_rgi, ev, event_size); + /* + In parallel replication, we need to update the relay log position + immediately so that it will be the correct position from which to + read the next event. + */ + if (res == 0) + rli->event_relay_log_pos= rli->future_event_relay_log_pos; if (res >= 0) DBUG_RETURN(res); /* @@ -4585,7 +4590,21 @@ pthread_handler_t handle_slave_sql(void *arg) serial_rgi->gtid_sub_id= 0; serial_rgi->gtid_pending= false; - rli->gtid_skip_flag = GTID_SKIP_NOT; + if (mi->using_gtid != Master_info::USE_GTID_NO && mi->using_parallel() && + rli->restart_gtid_pos.count() > 0) + { + /* + With parallel replication in GTID mode, if we have a multi-domain GTID + position, we need to start some way back in the relay log and skip any + GTID that was already applied before. Since event groups can be split + across multiple relay logs, this earlier starting point may be in the + middle of an already applied event group, so we also need to skip any + remaining part of such group. + */ + rli->gtid_skip_flag = GTID_SKIP_TRANSACTION; + } + else + rli->gtid_skip_flag = GTID_SKIP_NOT; if (init_relay_log_pos(rli, rli->group_relay_log_name, rli->group_relay_log_pos, @@ -4594,11 +4613,11 @@ pthread_handler_t handle_slave_sql(void *arg) { rli->report(ERROR_LEVEL, ER_SLAVE_FATAL_ERROR, NULL, "Error initializing relay log position: %s", errmsg); - goto err; + goto err_before_start; } rli->reset_inuse_relaylog(); if (rli->alloc_inuse_relaylog(rli->group_relay_log_name)) - goto err; + goto err_before_start; strcpy(rli->future_event_master_log_name, rli->group_master_log_name); THD_CHECK_SENTRY(thd); @@ -4780,7 +4799,8 @@ log '%s' at position %s, relay log '%s' position: %s%s", RPL_LOG_NAME, } } - if (opt_slave_parallel_threads > 0) + err: + if (mi->using_parallel()) rli->parallel.wait_for_done(thd, rli); /* Thread stopped. Print the current replication position to the log */ @@ -4799,15 +4819,7 @@ log '%s' at position %s, relay log '%s' position: %s%s", RPL_LOG_NAME, tmp.c_ptr_safe()); } - err: - - /* - Once again, in case we aborted with an error and skipped the first one. - (We want the first one to be before the printout of stop position to - get the correct position printed.) - */ - if (opt_slave_parallel_threads > 0) - rli->parallel.wait_for_done(thd, rli); + err_before_start: /* Some events set some playgrounds, which won't be cleared because thread @@ -4830,7 +4842,7 @@ log '%s' at position %s, relay log '%s' position: %s%s", RPL_LOG_NAME, ulong domain_count; flush_relay_log_info(rli); - if (opt_slave_parallel_threads > 0) + if (mi->using_parallel()) { /* In parallel replication GTID mode, we may stop with different domains @@ -6548,7 +6560,7 @@ static Log_event* next_event(rpl_group_info *rgi, ulonglong *event_size) llstr(my_b_tell(cur_log),llbuf1), llstr(rli->event_relay_log_pos,llbuf2))); DBUG_ASSERT(my_b_tell(cur_log) >= BIN_LOG_HEADER_SIZE); - DBUG_ASSERT(opt_slave_parallel_threads > 0 || + DBUG_ASSERT(rli->mi->using_parallel() || my_b_tell(cur_log) == rli->event_relay_log_pos); } #endif diff --git a/sql/sp_head.cc b/sql/sp_head.cc index b90108ed944..fb0f3132816 100644 --- a/sql/sp_head.cc +++ b/sql/sp_head.cc @@ -522,12 +522,8 @@ check_routine_name(LEX_STRING *ident) my_error(ER_SP_WRONG_NAME, MYF(0), ident->str); return TRUE; } - if (check_string_char_length(ident, "", NAME_CHAR_LEN, - system_charset_info, 1)) - { - my_error(ER_TOO_LONG_IDENT, MYF(0), ident->str); + if (check_ident_length(ident)) return TRUE; - } return FALSE; } @@ -3120,23 +3116,23 @@ sp_instr_stmt::execute(THD *thd, uint *nextp) thd->query_length()) <= 0) { res= m_lex_keeper.reset_lex_and_exec_core(thd, nextp, FALSE, this); + bool log_slow= !res && thd->enable_slow_log; - if (thd->get_stmt_da()->is_eof()) - { - /* Finalize server status flags after executing a statement. */ + /* Finalize server status flags after executing a statement. */ + if (log_slow || thd->get_stmt_da()->is_eof()) thd->update_server_status(); + if (thd->get_stmt_da()->is_eof()) thd->protocol->end_statement(); - } query_cache_end_of_result(thd); mysql_audit_general(thd, MYSQL_AUDIT_GENERAL_STATUS, - thd->get_stmt_da()->is_error() ? - thd->get_stmt_da()->sql_errno() : 0, - command_name[COM_QUERY].str); + thd->get_stmt_da()->is_error() ? + thd->get_stmt_da()->sql_errno() : 0, + command_name[COM_QUERY].str); - if (!res && unlikely(thd->enable_slow_log)) + if (log_slow) log_slow_statement(thd); } else diff --git a/sql/sql_acl.cc b/sql/sql_acl.cc index 6882ef2ca1f..be8b0c5a022 100644 --- a/sql/sql_acl.cc +++ b/sql/sql_acl.cc @@ -1600,7 +1600,7 @@ my_bool acl_reload(THD *thd) my_hash_init2(&acl_roles,50, &my_charset_utf8_bin, 0, 0, 0, (my_hash_get_key) acl_role_get_key, 0, (void (*)(void *))free_acl_role, 0); - my_hash_init2(&acl_roles_mappings, 50, system_charset_info, 0, 0, 0, + my_hash_init2(&acl_roles_mappings, 50, &my_charset_utf8_bin, 0, 0, 0, (my_hash_get_key) acl_role_map_get_key, 0, 0, 0); old_mem= acl_memroot; delete_dynamic(&acl_wild_hosts); diff --git a/sql/sql_connect.cc b/sql/sql_connect.cc index 3a67ea40eeb..695d1095125 100644 --- a/sql/sql_connect.cc +++ b/sql/sql_connect.cc @@ -1,6 +1,6 @@ /* Copyright (c) 2007, 2013, Oracle and/or its affiliates. - Copyright (c) 2008, 2014, SkySQL Ab. + Copyright (c) 2008, 2016, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -834,6 +834,7 @@ void update_global_user_stats(THD *thd, bool create_user, time_t now) bool thd_init_client_charset(THD *thd, uint cs_number) { + SV *gv=&global_system_variables; CHARSET_INFO *cs; /* Use server character set and collation if @@ -844,12 +845,10 @@ bool thd_init_client_charset(THD *thd, uint cs_number) if (!opt_character_set_client_handshake || !(cs= get_charset(cs_number, MYF(0)))) { - thd->variables.character_set_client= - global_system_variables.character_set_client; - thd->variables.collation_connection= - global_system_variables.collation_connection; - thd->variables.character_set_results= - global_system_variables.character_set_results; + DBUG_ASSERT(is_supported_parser_charset(gv->character_set_client)); + thd->variables.character_set_client= gv->character_set_client; + thd->variables.collation_connection= gv->collation_connection; + thd->variables.character_set_results= gv->character_set_results; } else { diff --git a/sql/sql_derived.cc b/sql/sql_derived.cc index fdc615d0fae..f2674cb8dab 100644 --- a/sql/sql_derived.cc +++ b/sql/sql_derived.cc @@ -663,6 +663,8 @@ bool mysql_derived_prepare(THD *thd, LEX *lex, TABLE_LIST *derived) unit->derived= derived; + derived->fill_me= FALSE; + if (!(derived->derived_result= new select_union)) DBUG_RETURN(TRUE); // out of memory diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc index a2cd337a11c..a98514acbb8 100644 --- a/sql/sql_lex.cc +++ b/sql/sql_lex.cc @@ -1101,7 +1101,7 @@ static int lex_one_token(YYSTYPE *yylval, THD *thd) state= (enum my_lex_states) state_map[c]; break; case MY_LEX_ESCAPE: - if (lip->yyGet() == 'N') + if (!lip->eof() && lip->yyGet() == 'N') { // Allow \N as shortcut for NULL yylval->lex_str.str=(char*) "\\N"; yylval->lex_str.length=2; @@ -1954,6 +1954,7 @@ void st_select_lex::init_select() m_agg_func_used= false; name_visibility_map= 0; join= 0; + lock_type= TL_READ_DEFAULT; } /* @@ -3168,6 +3169,9 @@ void LEX::first_lists_tables_same() if (query_tables_last == &first_table->next_global) query_tables_last= first_table->prev_global; + if (query_tables_own_last == &first_table->next_global) + query_tables_own_last= first_table->prev_global; + if ((next= *first_table->prev_global= first_table->next_global)) next->prev_global= first_table->prev_global; /* include in new place */ @@ -3493,12 +3497,28 @@ bool st_select_lex::add_index_hint (THD *thd, char *str, uint length) bool st_select_lex::optimize_unflattened_subqueries(bool const_only) { - for (SELECT_LEX_UNIT *un= first_inner_unit(); un; un= un->next_unit()) + SELECT_LEX_UNIT *next_unit= NULL; + for (SELECT_LEX_UNIT *un= first_inner_unit(); + un; + un= next_unit ? next_unit : un->next_unit()) { Item_subselect *subquery_predicate= un->item; - + next_unit= NULL; + if (subquery_predicate) { + if (!subquery_predicate->fixed) + { + /* + This subquery was excluded as part of some expression so it is + invisible from all prepared expression. + */ + next_unit= un->next_unit(); + un->exclude_level(); + if (next_unit) + continue; + break; + } if (subquery_predicate->substype() == Item_subselect::IN_SUBS) { Item_in_subselect *in_subs= (Item_in_subselect*) subquery_predicate; @@ -4265,6 +4285,12 @@ bool st_select_lex::is_merged_child_of(st_select_lex *ancestor) { continue; } + + if (sl->master_unit()->derived && + sl->master_unit()->derived->is_merged_derived()) + { + continue; + } all_merged= FALSE; break; } diff --git a/sql/sql_lex.h b/sql/sql_lex.h index 019231e2004..c4fd109009f 100644 --- a/sql/sql_lex.h +++ b/sql/sql_lex.h @@ -846,6 +846,9 @@ public: /* namp of nesting SELECT visibility (for aggregate functions check) */ nesting_map name_visibility_map; + /* it is for correct printing SELECT options */ + thr_lock_type lock_type; + void init_query(); void init_select(); st_select_lex_unit* master_unit(); diff --git a/sql/sql_load.cc b/sql/sql_load.cc index a21fc692ecc..6d803961621 100644 --- a/sql/sql_load.cc +++ b/sql/sql_load.cc @@ -79,6 +79,81 @@ class READ_INFO { NET *io_net; int level; /* for load xml */ + +#if MYSQL_VERSION_ID >= 100200 +#error This 10.0 and 10.1 specific fix should be removed in 10.2. +#error Fix read_mbtail() to use my_charlen() instead of my_charlen_tmp() +#else + int my_charlen_tmp(CHARSET_INFO *cs, const char *str, const char *end) + { + my_wc_t wc; + return cs->cset->mb_wc(cs, &wc, (const uchar *) str, (const uchar *) end); + } + + /** + Read a tail of a multi-byte character. + The first byte of the character is assumed to be already + read from the file and appended to "str". + + @returns true - if EOF happened unexpectedly + @returns false - no EOF happened: found a good multi-byte character, + or a bad byte sequence + + Note: + The return value depends only on EOF: + - read_mbtail() returns "false" is a good character was read, but also + - read_mbtail() returns "false" if an incomplete byte sequence was found + and no EOF happened. + + For example, suppose we have an ujis file with bytes 0x8FA10A, where: + - 0x8FA1 is an incomplete prefix of a 3-byte character + (it should be [8F][A1-FE][A1-FE] to make a full 3-byte character) + - 0x0A is a line demiliter + This file has some broken data, the trailing [A1-FE] is missing. + + In this example it works as follows: + - 0x8F is read from the file and put into "data" before the call + for read_mbtail() + - 0xA1 is read from the file and put into "data" by read_mbtail() + - 0x0A is kept in the read queue, so the next read iteration after + the current read_mbtail() call will normally find it and recognize as + a line delimiter + - the current call for read_mbtail() returns "false", + because no EOF happened + */ + bool read_mbtail(String *str) + { + int chlen; + if ((chlen= my_charlen_tmp(read_charset, str->end() - 1, str->end())) == 1) + return false; // Single byte character found + for (uint32 length0= str->length() - 1 ; MY_CS_IS_TOOSMALL(chlen); ) + { + int chr= GET; + if (chr == my_b_EOF) + { + DBUG_PRINT("info", ("read_mbtail: chlen=%d; unexpected EOF", chlen)); + return true; // EOF + } + str->append(chr); + chlen= my_charlen_tmp(read_charset, str->ptr() + length0, str->end()); + if (chlen == MY_CS_ILSEQ) + { + /** + It has been an incomplete (but a valid) sequence so far, + but the last byte turned it into a bad byte sequence. + Unget the very last byte. + */ + str->length(str->length() - 1); + PUSH(chr); + DBUG_PRINT("info", ("read_mbtail: ILSEQ")); + return false; // Bad byte sequence + } + } + DBUG_PRINT("info", ("read_mbtail: chlen=%d", chlen)); + return false; // Good multi-byte character + } +#endif + public: bool error,line_cuted,found_null,enclosed; uchar *row_start, /* Found row starts here */ @@ -521,7 +596,8 @@ int mysql_load(THD *thd,sql_exchange *ex,TABLE_LIST *table_list, *enclosed, skip_lines, ignore); thd_proc_info(thd, "End bulk insert"); - thd_progress_next_stage(thd); + if (!error) + thd_progress_next_stage(thd); if (thd->locked_tables_mode <= LTM_LOCK_TABLES && table->file->ha_end_bulk_insert() && !error) { @@ -1406,7 +1482,7 @@ READ_INFO::READ_INFO(File file_par, uint tot_length, CHARSET_INFO *cs, stack=stack_pos=(int*) sql_alloc(sizeof(int)*length); if (!(buffer=(uchar*) my_malloc(buff_length+1,MYF(MY_WME | MY_THREAD_SPECIFIC)))) - error=1; /* purecov: inspected */ + error= 1; /* purecov: inspected */ else { end_of_buff=buffer+buff_length; @@ -1473,6 +1549,54 @@ inline int READ_INFO::terminator(const uchar *ptr,uint length) } +/** + Read a field. + + The data in the loaded file was presumably escaped using + - either select_export::send_data() OUTFILE + - or mysql_real_escape_string() + using the same character set with the one specified in the current + "LOAD DATA INFILE ... CHARACTER SET ..." (or the default LOAD character set). + + Note, non-escaped multi-byte characters are scanned as a single entity. + This is needed to correctly distinguish between: + - 0x5C as an escape character versus + - 0x5C as the second byte in a multi-byte sequence (big5, cp932, gbk, sjis) + + Parts of escaped multi-byte characters are scanned on different loop + iterations. See the comment about 0x5C handling in select_export::send_data() + in sql_class.cc. + + READ_INFO::read_field() does not check wellformedness. + Raising wellformedness errors or warnings in READ_INFO::read_field() + would be wrong, as the data after unescaping can go into a BLOB field, + or into a TEXT/VARCHAR field of a different character set. + The loop below only makes sure to revert escaping made by + select_export::send_data() or mysql_real_escape_string(). + Wellformedness is checked later, during Field::store(str,length,cs) time. + + Note, in some cases users can supply data which did not go through + escaping properly. For example, utf8 "\<C3><A4>" + (backslash followed by LATIN SMALL LETTER A WITH DIAERESIS) + is improperly escaped data that could not be generated by + select_export::send_data() / mysql_real_escape_string(): + - either there should be two backslashes: "\\<C3><A4>" + - or there should be no backslashes at all: "<C3><A4>" + "\<C3>" and "<A4> are scanned on two different loop iterations and + store "<C3><A4>" into the field. + + Note, adding useless escapes before multi-byte characters like in the + example above is safe in case of utf8, but is not safe in case of + character sets that have escape_with_backslash_is_dangerous==TRUE, + such as big5, cp932, gbk, sjis. This can lead to mis-interpretation of the + data. Suppose we have a big5 character "<EE><5C>" followed by <30> (digit 0). + If we add an extra escape before this sequence, then we'll get + <5C><EE><5C><30>. The first loop iteration will turn <5C><EE> into <EE>. + The second loop iteration will turn <5C><30> into <30>. + So the program that generates a dump file for further use with LOAD DATA + must make sure to use escapes properly. +*/ + int READ_INFO::read_field() { int chr,found_enclosed_char; @@ -1509,7 +1633,8 @@ int READ_INFO::read_field() for (;;) { - while ( to < end_of_buff) + // Make sure we have enough space for the longest multi-byte character. + while ( to + read_charset->mbmaxlen < end_of_buff) { chr = GET; if (chr == my_b_EOF) @@ -1597,58 +1722,33 @@ int READ_INFO::read_field() } } #ifdef USE_MB - uint ml= my_mbcharlen(read_charset, chr); - if (ml == 0) - { - *to= '\0'; - my_error(ER_INVALID_CHARACTER_STRING, MYF(0), - read_charset->csname, buffer); - error= true; - return 1; - } - - if (ml > 1 && - to + ml <= end_of_buff) - { - uchar* p= to; - *to++ = chr; - - for (uint i= 1; i < ml; i++) - { - chr= GET; - if (chr == my_b_EOF) - { - /* - Need to back up the bytes already ready from illformed - multi-byte char - */ - to-= i; - goto found_eof; - } - *to++ = chr; - } - if (my_ismbchar(read_charset, - (const char *)p, - (const char *)to)) - continue; - for (uint i= 0; i < ml; i++) - PUSH(*--to); - chr= GET; - } - else if (ml > 1) - { - // Buffer is too small, exit while loop, and reallocate. - PUSH(chr); - break; - } #endif *to++ = (uchar) chr; +#if MYSQL_VERSION_ID >= 100200 +#error This 10.0 and 10.1 specific fix should be removed in 10.2 +#else + if (my_mbcharlen(read_charset, (uchar) chr) > 1) + { + /* + A known MBHEAD found. Try to scan the full multi-byte character. + Otherwise, a possible following second byte 0x5C would be + mis-interpreted as an escape on the next iteration. + (Important for big5, gbk, sjis, cp932). + */ + String tmp((char *) to - 1, read_charset->mbmaxlen, read_charset); + tmp.length(1); + bool eof= read_mbtail(&tmp); + to+= tmp.length() - 1; + if (eof) + goto found_eof; + } +#endif } /* ** We come here if buffer is too small. Enlarge it and continue */ if (!(new_buffer=(uchar*) my_realloc((char*) buffer,buff_length+1+IO_SIZE, - MYF(MY_WME | MY_THREAD_SPECIFIC)))) + MYF(MY_WME | MY_THREAD_SPECIFIC)))) return (error=1); to=new_buffer + (to-buffer); buffer=new_buffer; @@ -1884,15 +1984,7 @@ int READ_INFO::read_value(int delim, String *val) for (chr= GET; my_tospace(chr) != delim && chr != my_b_EOF;) { #ifdef USE_MB - uint ml= my_mbcharlen(read_charset, chr); - if (ml == 0) - { - chr= my_b_EOF; - val->length(0); - return chr; - } - - if (ml > 1) + if (my_mbcharlen(read_charset, chr) > 1) { DBUG_PRINT("read_xml",("multi byte")); int i, ml= my_mbcharlen(read_charset, chr); diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index d92ef810498..c906a941867 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -5409,11 +5409,8 @@ create_sp_error: } case SQLCOM_SHOW_CREATE_TRIGGER: { - if (lex->spname->m_name.length > NAME_LEN) - { - my_error(ER_TOO_LONG_IDENT, MYF(0), lex->spname->m_name.str); + if (check_ident_length(&lex->spname->m_name)) goto error; - } #ifdef WITH_WSREP if (WSREP_CLIENT(thd) && wsrep_sync_wait(thd)) goto error; @@ -7322,12 +7319,9 @@ bool add_field_to_list(THD *thd, LEX_STRING *field_name, enum_field_types type, uint8 datetime_precision= length ? atoi(length) : 0; DBUG_ENTER("add_field_to_list"); - if (check_string_char_length(field_name, "", NAME_CHAR_LEN, - system_charset_info, 1)) - { - my_error(ER_TOO_LONG_IDENT, MYF(0), field_name->str); /* purecov: inspected */ + if (check_ident_length(field_name)) DBUG_RETURN(1); /* purecov: inspected */ - } + if (type_modifier & PRI_KEY_FLAG) { Key *key; @@ -9107,6 +9101,18 @@ bool check_string_char_length(LEX_STRING *str, const char *err_msg, return TRUE; } + +bool check_ident_length(LEX_STRING *ident) +{ + if (check_string_char_length(ident, 0, NAME_CHAR_LEN, system_charset_info, 1)) + { + my_error(ER_TOO_LONG_IDENT, MYF(0), ident->str); + return 1; + } + return 0; +} + + C_MODE_START /* diff --git a/sql/sql_parse.h b/sql/sql_parse.h index ff22ec55922..0e60a5cc884 100644 --- a/sql/sql_parse.h +++ b/sql/sql_parse.h @@ -76,6 +76,7 @@ bool check_string_byte_length(LEX_STRING *str, const char *err_msg, bool check_string_char_length(LEX_STRING *str, const char *err_msg, uint max_char_length, CHARSET_INFO *cs, bool no_error); +bool check_ident_length(LEX_STRING *ident); CHARSET_INFO* merge_charset_and_collation(CHARSET_INFO *cs, CHARSET_INFO *cl); bool check_host_name(LEX_STRING *str); bool check_identifier_name(LEX_STRING *str, uint max_char_length, diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 0cfb964307d..7850c7a0dcf 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -777,10 +777,15 @@ JOIN::prepare(Item ***rref_pointer_array, if (mixed_implicit_grouping && tbl->table) tbl->table->maybe_null= 1; } + + uint real_og_num= og_num; + if (skip_order_by && + select_lex != select_lex->master_unit()->global_parameters) + real_og_num+= select_lex->order_list.elements; if ((wild_num && setup_wild(thd, tables_list, fields_list, &all_fields, wild_num)) || - select_lex->setup_ref_array(thd, og_num) || + select_lex->setup_ref_array(thd, real_og_num) || setup_fields(thd, (*rref_pointer_array), fields_list, MARK_COLUMNS_READ, &all_fields, 1) || setup_without_group(thd, (*rref_pointer_array), tables_list, @@ -3082,7 +3087,7 @@ void JOIN::exec_inner() *curr_fields_list), Protocol::SEND_NUM_ROWS | Protocol::SEND_EOF); error= do_select(curr_join, curr_fields_list, NULL, procedure); - thd->limit_found_rows= curr_join->send_records; + thd->limit_found_rows= curr_join->send_records - curr_join->duplicate_rows; if (curr_join->order && curr_join->sortorder && curr_join->filesort_found_rows) { @@ -3144,6 +3149,7 @@ JOIN::destroy() */ tmp_table_param.cleanup(); tmp_join->tmp_table_param.copy_field= 0; + cleanup(1); DBUG_RETURN(tmp_join->destroy()); } cond_equal= 0; @@ -12764,6 +12770,9 @@ static COND *build_equal_items_for_cond(THD *thd, COND *cond, COND_EQUAL cond_equal; cond_equal.upper_levels= inherited; + if (check_stack_overrun(thd, STACK_MIN_SIZE, NULL)) + return cond; // Fatal error flag is set! + if (cond->type() == Item::COND_ITEM) { List<Item> eq_list; @@ -14766,20 +14775,9 @@ bool cond_is_datetime_is_null(Item *cond) if (cond->type() == Item::FUNC_ITEM && ((Item_func*) cond)->functype() == Item_func::ISNULL_FUNC) { - Item **args= ((Item_func_isnull*) cond)->arguments(); - if (args[0]->type() == Item::FIELD_ITEM) - { - Field *field=((Item_field*) args[0])->field; - - if (((field->type() == MYSQL_TYPE_DATE) || - (field->type() == MYSQL_TYPE_DATETIME)) && - (field->flags & NOT_NULL_FLAG)) - { - return TRUE; - } - } + return ((Item_func_isnull*) cond)->arg_is_datetime_notnull_field(); } - return FALSE; + return false; } @@ -17463,7 +17461,7 @@ do_select(JOIN *join,List<Item> *fields,TABLE *table,Procedure *procedure) join->join_tab[join->top_join_tab_count - 1].next_select= end_select; join_tab=join->join_tab+join->const_tables; } - join->send_records=0; + join->duplicate_rows= join->send_records=0; if (join->table_count == join->const_tables) { /* @@ -18988,7 +18986,12 @@ end_send(JOIN *join, JOIN_TAB *join_tab __attribute__((unused)), int error; /* result < 0 if row was not accepted and should not be counted */ if ((error= join->result->send_data(*join->fields))) - DBUG_RETURN(error < 0 ? NESTED_LOOP_OK : NESTED_LOOP_ERROR); + { + if (error > 0) + DBUG_RETURN(NESTED_LOOP_ERROR); + // error < 0 => duplicate row + join->duplicate_rows++; + } } ++join->send_records; @@ -19122,7 +19125,7 @@ end_send_group(JOIN *join, JOIN_TAB *join_tab __attribute__((unused)), if (error < 0) { /* Duplicate row, don't count */ - join->send_records--; + join->duplicate_rows++; error= 0; } } @@ -24602,6 +24605,12 @@ void st_select_lex::print(THD *thd, String *str, enum_query_type query_type) // limit print_limit(thd, str, query_type); + // lock type + if (lock_type == TL_READ_WITH_SHARED_LOCKS) + str->append(" lock in share mode"); + else if (lock_type == TL_WRITE) + str->append(" for update"); + // PROCEDURE unsupported here } diff --git a/sql/sql_select.h b/sql/sql_select.h index cbad287c5e5..d51bca785a5 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -1015,7 +1015,8 @@ public: table_map outer_join; /* Bitmap of tables used in the select list items */ table_map select_list_used_tables; - ha_rows send_records,found_records,examined_rows,row_limit, select_limit; + ha_rows send_records, found_records, examined_rows, + row_limit, select_limit, duplicate_rows; /** Used to fetch no more than given amount of rows per one fetch operation of server side cursor. @@ -1281,7 +1282,7 @@ public: sort_and_group= 0; first_record= 0; do_send_rows= 1; - send_records= 0; + duplicate_rows= send_records= 0; found_records= 0; fetch_limit= HA_POS_ERROR; examined_rows= 0; diff --git a/sql/sql_string.h b/sql/sql_string.h index c287f051d98..557d14a79f8 100644 --- a/sql/sql_string.h +++ b/sql/sql_string.h @@ -136,6 +136,7 @@ public: inline bool is_empty() const { return (str_length == 0); } inline void mark_as_const() { Alloced_length= 0;} inline const char *ptr() const { return Ptr; } + inline const char *end() const { return Ptr + str_length; } inline char *c_ptr() { DBUG_ASSERT(!alloced || !Ptr || !Alloced_length || diff --git a/sql/sql_table.cc b/sql/sql_table.cc index a2e51ca8e93..aec81e17bd3 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -666,7 +666,7 @@ static bool read_ddl_log_file_entry(uint entry_no) bool error= FALSE; File file_id= global_ddl_log.file_id; uchar *file_entry_buf= (uchar*)global_ddl_log.file_entry_buf; - uint io_size= global_ddl_log.io_size; + size_t io_size= global_ddl_log.io_size; DBUG_ENTER("read_ddl_log_file_entry"); mysql_mutex_assert_owner(&LOCK_gdl); @@ -2497,7 +2497,19 @@ int mysql_rm_table_no_locks(THD *thd, TABLE_LIST *tables, bool if_exists, int frm_delete_error, trigger_drop_error= 0; /* Delete the table definition file */ strmov(end,reg_ext); - frm_delete_error= mysql_file_delete(key_file_frm, path, MYF(MY_WME)); + if (table_type && table_type != view_pseudo_hton && + table_type->discover_table) + { + /* + Table type is using discovery and may not need a .frm file. + Delete it silently if it exists + */ + (void) mysql_file_delete(key_file_frm, path, MYF(0)); + frm_delete_error= 0; + } + else + frm_delete_error= mysql_file_delete(key_file_frm, path, + MYF(MY_WME)); if (frm_delete_error) frm_delete_error= my_errno; else @@ -2513,7 +2525,7 @@ int mysql_rm_table_no_locks(THD *thd, TABLE_LIST *tables, bool if_exists, else if (frm_delete_error && if_exists) thd->clear_error(); } - non_tmp_error= error ? TRUE : non_tmp_error; + non_tmp_error|= MY_TEST(error); } if (error) { @@ -3120,8 +3132,7 @@ void promote_first_timestamp_column(List<Create_field> *column_definitions) @param key_info Key meta-data info. @param key_list List of existing keys. */ -static void check_duplicate_key(THD *thd, - Key *key, KEY *key_info, +static void check_duplicate_key(THD *thd, Key *key, KEY *key_info, List<Key> *key_list) { /* @@ -3183,14 +3194,11 @@ static void check_duplicate_key(THD *thd, // Report a warning if we have two identical keys. - DBUG_ASSERT(thd->lex->query_tables->alias); if (all_columns_are_identical) { push_warning_printf(thd, Sql_condition::WARN_LEVEL_NOTE, ER_DUP_INDEX, ER(ER_DUP_INDEX), - key_info->name, - thd->lex->query_tables->db, - thd->lex->query_tables->alias); + key_info->name); break; } } @@ -3508,7 +3516,6 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, sql_field->pack_length= dup_field->pack_length; sql_field->key_length= dup_field->key_length; sql_field->decimals= dup_field->decimals; - sql_field->create_length_to_internal_length(); sql_field->unireg_check= dup_field->unireg_check; /* We're making one field from two, the result field will have @@ -3518,6 +3525,7 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, if (!(sql_field->flags & NOT_NULL_FLAG)) null_fields--; sql_field->flags= dup_field->flags; + sql_field->create_length_to_internal_length(); sql_field->interval= dup_field->interval; sql_field->vcol_info= dup_field->vcol_info; sql_field->stored_in_db= dup_field->stored_in_db; @@ -3644,12 +3652,8 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, my_error(ER_TOO_MANY_KEY_PARTS,MYF(0),tmp); DBUG_RETURN(TRUE); } - if (check_string_char_length(&key->name, "", NAME_CHAR_LEN, - system_charset_info, 1)) - { - my_error(ER_TOO_LONG_IDENT, MYF(0), key->name.str); + if (check_ident_length(&key->name)) DBUG_RETURN(TRUE); - } key_iterator2.rewind (); if (key->type != Key::FOREIGN_KEY) { @@ -5998,7 +6002,7 @@ drop_create_field: while ((f_key= fk_key_it++)) { if (my_strcasecmp(system_charset_info, f_key->foreign_id->str, - key->name.str) == 0) + keyname) == 0) goto remove_key; } } diff --git a/sql/sql_time.cc b/sql/sql_time.cc index b55b1d76b99..1bd68e89ecb 100644 --- a/sql/sql_time.cc +++ b/sql/sql_time.cc @@ -930,7 +930,10 @@ bool date_add_interval(MYSQL_TIME *ltime, interval_type int_type, my_bool neg= 0; enum enum_mysql_timestamp_type time_type= ltime->time_type; - if ((ulong) interval.day > MAX_DAY_NUMBER) + if (((ulonglong) interval.day + + (ulonglong) interval.hour / 24 + + (ulonglong) interval.minute / 24 / 60 + + (ulonglong) interval.second / 24 / 60 / 60) > MAX_DAY_NUMBER) goto invalid_date; if (time_type != MYSQL_TIMESTAMP_TIME) diff --git a/sql/sql_udf.cc b/sql/sql_udf.cc index 74d2f6bc252..456a6bd4058 100644 --- a/sql/sql_udf.cc +++ b/sql/sql_udf.cc @@ -453,12 +453,8 @@ int mysql_create_function(THD *thd,udf_func *udf) my_message(ER_UDF_NO_PATHS, ER(ER_UDF_NO_PATHS), MYF(0)); DBUG_RETURN(1); } - if (check_string_char_length(&udf->name, "", NAME_CHAR_LEN, - system_charset_info, 1)) - { - my_error(ER_TOO_LONG_IDENT, MYF(0), udf->name.str); + if (check_ident_length(&udf->name)) DBUG_RETURN(1); - } tables.init_one_table(STRING_WITH_LEN("mysql"), STRING_WITH_LEN("func"), "func", TL_WRITE); diff --git a/sql/sql_view.cc b/sql/sql_view.cc index f3717e3ded2..9fe4dd4849d 100644 --- a/sql/sql_view.cc +++ b/sql/sql_view.cc @@ -430,6 +430,15 @@ bool mysql_create_view(THD *thd, TABLE_LIST *views, lex->link_first_table_back(view, link_to_local); view->open_type= OT_BASE_ONLY; + /* + ignore lock specs for CREATE statement + */ + if (lex->current_select->lock_type != TL_READ_DEFAULT) + { + lex->current_select->set_lock_for_tables(TL_READ_DEFAULT); + view->mdl_request.set_type(MDL_EXCLUSIVE); + } + if (open_temporary_tables(thd, lex->query_tables) || open_and_lock_tables(thd, lex->query_tables, TRUE, 0)) { diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index a5e5dffc62d..a7408f7bf50 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -5117,6 +5117,8 @@ part_name: { partition_info *part_info= Lex->part_info; partition_element *p_elem= part_info->curr_part_elem; + if (check_ident_length(&$1)) + MYSQL_YYABORT; p_elem->partition_name= $1.str; } ; @@ -5411,7 +5413,11 @@ sub_part_definition: sub_name: ident_or_text - { Lex->part_info->curr_part_elem->partition_name= $1.str; } + { + if (check_ident_length(&$1)) + MYSQL_YYABORT; + Lex->part_info->curr_part_elem->partition_name= $1.str; + } ; opt_part_options: @@ -8497,12 +8503,14 @@ select_lock_type: | FOR_SYM UPDATE_SYM { LEX *lex=Lex; + lex->current_select->lock_type= TL_WRITE; lex->current_select->set_lock_for_tables(TL_WRITE); lex->safe_to_cache_query=0; } | LOCK_SYM IN_SYM SHARE_SYM MODE_SYM { LEX *lex=Lex; + lex->current_select->lock_type= TL_READ_WITH_SHARED_LOCKS; lex->current_select-> set_lock_for_tables(TL_READ_WITH_SHARED_LOCKS); lex->safe_to_cache_query=0; @@ -15184,6 +15192,7 @@ grant_role: CHARSET_INFO *cs= system_charset_info; /* trim end spaces (as they'll be lost in mysql.user anyway) */ $1.length= cs->cset->lengthsp(cs, $1.str, $1.length); + $1.str[$1.length] = '\0'; if ($1.length == 0) { my_error(ER_INVALID_ROLE, MYF(0), ""); diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 93a863d6b88..379294fa7b2 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -34,6 +34,7 @@ #include "sql_plugin.h" // Includes my_global.h #include "sql_priv.h" #include "sql_class.h" // set_var.h: THD +#include "sql_parse.h" #include "sys_vars.h" #include "events.h" @@ -634,7 +635,7 @@ static bool check_cs_client(sys_var *self, THD *thd, set_var *var) return true; // Currently, UCS-2 cannot be used as a client character set - if (((CHARSET_INFO *)(var->save_result.ptr))->mbminlen > 1) + if (!is_supported_parser_charset((CHARSET_INFO *)(var->save_result.ptr))) return true; return false; @@ -4376,8 +4377,7 @@ static bool update_slave_skip_counter(sys_var *self, THD *thd, Master_info *mi) mi->connection_name.str); return true; } - if (mi->using_gtid != Master_info::USE_GTID_NO && - opt_slave_parallel_threads > 0) + if (mi->using_gtid != Master_info::USE_GTID_NO && mi->using_parallel()) { ulong domain_count; mysql_mutex_lock(&rpl_global_gtid_slave_state->LOCK_slave_state); diff --git a/sql/sys_vars.h b/sql/sys_vars.h index 79b67d3b45c..957c38749ba 100644 --- a/sql/sys_vars.h +++ b/sql/sys_vars.h @@ -1235,7 +1235,7 @@ public: if (var->value->result_type() == STRING_RESULT) { - if (!(res=var->value->val_str(&str))) + if (!(res=var->value->val_str_ascii(&str))) return true; else { diff --git a/storage/connect/ApacheInterface.java b/storage/connect/ApacheInterface.java index b4c8a4e9885..47b46dc0506 100644 --- a/storage/connect/ApacheInterface.java +++ b/storage/connect/ApacheInterface.java @@ -35,7 +35,10 @@ public class ApacheInterface extends JdbcInterface { ds.setPassword(parms[3]); pool.put(url, ds); } // endif ds - + + // if (parms.length > 4 && parms[4] != null) + // ds.setConnectionProperties(parms[4]); + // Get a connection from the data source conn = ds.getConnection(); diff --git a/storage/connect/CMakeLists.txt b/storage/connect/CMakeLists.txt index 95d88538119..ce6de424421 100644 --- a/storage/connect/CMakeLists.txt +++ b/storage/connect/CMakeLists.txt @@ -20,25 +20,25 @@ SET(CONNECT_SOURCES ha_connect.cc connect.cc user_connect.cc mycat.cc fmdlex.c osutil.c plugutil.c rcmsg.c rcmsg.h array.cpp blkfil.cpp colblk.cpp csort.cpp -filamap.cpp filamdbf.cpp filamfix.cpp filamtxt.cpp filamvct.cpp filamzip.cpp +filamap.cpp filamdbf.cpp filamfix.cpp filamgz.cpp filamtxt.cpp filter.cpp json.cpp jsonudf.cpp maputil.cpp myconn.cpp myutil.cpp plgdbutl.cpp reldef.cpp tabcol.cpp tabdos.cpp tabfix.cpp tabfmt.cpp tabjson.cpp table.cpp tabmul.cpp tabmysql.cpp taboccur.cpp tabpivot.cpp tabsys.cpp tabtbl.cpp tabutil.cpp -tabvct.cpp tabvir.cpp tabxcl.cpp valblk.cpp value.cpp xindex.cpp xobject.cpp +tabvir.cpp tabxcl.cpp valblk.cpp value.cpp xindex.cpp xobject.cpp array.h blkfil.h block.h catalog.h checklvl.h colblk.h connect.h csort.h -engmsg.h filamap.h filamdbf.h filamfix.h filamtxt.h filamvct.h filamzip.h +engmsg.h filamap.h filamdbf.h filamfix.h filamgz.h filamtxt.h filter.h global.h ha_connect.h inihandl.h json.h jsonudf.h maputil.h msgid.h mycat.h myconn.h myutil.h os.h osutil.h plgcnx.h plgdbsem.h preparse.h reldef.h resource.h tabcol.h tabdos.h tabfix.h tabfmt.h tabjson.h tabmul.h tabmysql.h -taboccur.h tabpivot.h tabsys.h tabtbl.h tabutil.h tabvct.h tabvir.h tabxcl.h +taboccur.h tabpivot.h tabsys.h tabtbl.h tabutil.h tabvir.h tabxcl.h user_connect.h valblk.h value.h xindex.h xobject.h xtable.h) # # Definitions that are shared for all OSes # add_definitions( -DMARIADB -DFORCE_INIT_OF_VARS -Dconnect_EXPORTS) -add_definitions( -DHUGE_SUPPORT -DZIP_SUPPORT -DPIVOT_SUPPORT ) +add_definitions( -DHUGE_SUPPORT -DGZ_SUPPORT -DPIVOT_SUPPORT ) # @@ -90,6 +90,18 @@ ENDIF(UNIX) # +# VCT: the VEC format might be not supported in future versions +# + +OPTION(CONNECT_WITH_VCT "Compile CONNECT storage engine with VCT support" ON) + +IF(CONNECT_WITH_VCT) + SET(CONNECT_SOURCES ${CONNECT_SOURCES} filamvct.cpp tabvct.cpp filamvct.h tabvct.h) + add_definitions(-DVCT_SUPPORT) +ENDIF(CONNECT_WITH_VCT) + + +# # XML # @@ -236,9 +248,9 @@ ENDIF(CONNECT_WITH_ODBC) # JDBC # IF(APPLE) - OPTION(CONNECT_WITH_JDBC "some comment" OFF) + OPTION(CONNECT_WITH_JDBC "Compile CONNECT storage engine without JDBC support" OFF) ELSE() - OPTION(CONNECT_WITH_JDBC "some comment" ON) + OPTION(CONNECT_WITH_JDBC "Compile CONNECT storage engine with JDBC support" ON) ENDIF() IF(CONNECT_WITH_JDBC) @@ -252,18 +264,33 @@ IF(CONNECT_WITH_JDBC) SET(CONNECT_SOURCES ${CONNECT_SOURCES} jdbconn.cpp tabjdbc.cpp jdbconn.h tabjdbc.h jdbccat.h JdbcInterface.java ApacheInterface.java MariadbInterface.java - MysqlInterface.java OracleInterface.java PostgresqlInterface.java) + MysqlInterface.java OracleInterface.java PostgresqlInterface.java + JavaWrappers.jar) # TODO: Find how to compile and install the java wrapper classes # Find required libraries and include directories SET (JAVA_SOURCES JdbcInterface.java) add_jar(JdbcInterface ${JAVA_SOURCES}) - install_jar(JdbcInterface DESTINATION ${INSTALL_PLUGINDIR} COMPONENT connect-engine) + install_jar(JdbcInterface DESTINATION ${INSTALL_PLUGINDIR} COMPONENT connect-engine) + INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/JavaWrappers.jar + DESTINATION ${INSTALL_PLUGINDIR} COMPONENT connect-engine) add_definitions(-DJDBC_SUPPORT) ELSE() SET(JDBC_LIBRARY "") ENDIF() ENDIF(CONNECT_WITH_JDBC) +# +# ZIP +# + +OPTION(CONNECT_WITH_ZIP "Compile CONNECT storage engine with ZIP support" ON) + +IF(CONNECT_WITH_ZIP) + SET(CONNECT_SOURCES ${CONNECT_SOURCES} filamzip.cpp tabzip.cpp unzip.c ioapi.c zip.c + filamzip.h tabzip.h ioapi.h unzip.h zip.h) + add_definitions(-DZIP_SUPPORT -DNOCRYPT) +ENDIF(CONNECT_WITH_ZIP) + # # XMAP diff --git a/storage/connect/JavaWrappers.jar b/storage/connect/JavaWrappers.jar Binary files differnew file mode 100644 index 00000000000..8c01c364a3f --- /dev/null +++ b/storage/connect/JavaWrappers.jar diff --git a/storage/connect/JdbcInterface.java b/storage/connect/JdbcInterface.java index 34af8c4e013..a1b1360e6ea 100644 --- a/storage/connect/JdbcInterface.java +++ b/storage/connect/JdbcInterface.java @@ -82,6 +82,9 @@ public class JdbcInterface { System.out.println("URL=" + parms[1]); CheckURL(parms[1], null); + + // This is required for drivers using context class loaders + Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); if (parms[2] != null && !parms[2].isEmpty()) { if (DEBUG) @@ -220,6 +223,19 @@ public class JdbcInterface { } // end of SetTimestampParm + public int SetNullParm(int i, int typ) { + int rc = 0; + + try { + pstmt.setNull(i, typ); + } catch (Exception e) { + SetErrmsg(e); + rc = -1; + } // end try/catch + + return rc; + } // end of SetNullParm + public int ExecutePrep() { int n = -3; diff --git a/storage/connect/domdoc.cpp b/storage/connect/domdoc.cpp index 64a0a172956..eb9660b439d 100644 --- a/storage/connect/domdoc.cpp +++ b/storage/connect/domdoc.cpp @@ -89,30 +89,43 @@ DOMDOC::DOMDOC(char *nsl, char *nsdf, char *enc, PFBLOCK fp) /******************************************************************/ /* Initialize XML parser and check library compatibility. */ /******************************************************************/ -bool DOMDOC::Initialize(PGLOBAL g) - { - if (TestHr(g, CoInitialize(NULL))) +bool DOMDOC::Initialize(PGLOBAL g, char *entry, bool zipped) +{ + if (zipped && InitZip(g, entry)) + return true; + + if (TestHr(g, CoInitialize(NULL))) return true; if (TestHr(g, Docp.CreateInstance("msxml2.domdocument"))) return true; return MakeNSlist(g); - } // end of Initialize +} // end of Initialize /******************************************************************/ /* Parse the XML file and construct node tree in memory. */ /******************************************************************/ -bool DOMDOC::ParseFile(char *fn) - { - // Load the document +bool DOMDOC::ParseFile(PGLOBAL g, char *fn) +{ + bool b; + Docp->async = false; - if (!(bool)Docp->load((_bstr_t)fn)) + if (zip) { + // Parse an in memory document + char *xdoc = GetMemDoc(g, fn); + + b = (xdoc) ? (bool)Docp->loadXML((_bstr_t)xdoc) : false; + } else + // Load the document + b = (bool)Docp->load((_bstr_t)fn); + + if (!b) return true; return false; - } // end of ParseFile +} // end of ParseFile /******************************************************************/ /* Create or reuse an Xblock for this document. */ @@ -239,6 +252,7 @@ int DOMDOC::DumpDoc(PGLOBAL g, char *ofn) void DOMDOC::CloseDoc(PGLOBAL g, PFBLOCK xp) { CloseXMLFile(g, xp, false); + CloseZip(); } // end of Close /* ----------------------- class DOMNODE ------------------------ */ @@ -616,13 +630,13 @@ PXNODE DOMNODELIST::GetItem(PGLOBAL g, int n, PXNODE np) /* Reset the pointer on the deleted item. */ /******************************************************************/ bool DOMNODELIST::DropItem(PGLOBAL g, int n) - { - if (Listp == NULL || Listp->length <= n) - return true; +{ + if (Listp == NULL || Listp->length < n) + return true; //Listp->item[n] = NULL; La propriété n'a pas de méthode 'set' return false; - } // end of DeleteItem +} // end of DeleteItem /* ----------------------- class DOMATTR ------------------------ */ diff --git a/storage/connect/domdoc.h b/storage/connect/domdoc.h index 2cffec499e2..cfec98a9422 100644 --- a/storage/connect/domdoc.h +++ b/storage/connect/domdoc.h @@ -37,8 +37,8 @@ class DOMDOC : public XMLDOCUMENT { virtual void SetNofree(bool b) {} // Only libxml2 // Methods - virtual bool Initialize(PGLOBAL g); - virtual bool ParseFile(char *fn); + virtual bool Initialize(PGLOBAL g, char *entry, bool zipped); + virtual bool ParseFile(PGLOBAL g, char *fn); virtual bool NewDoc(PGLOBAL g, char *ver); virtual void AddComment(PGLOBAL g, char *com); virtual PXNODE GetRoot(PGLOBAL g); diff --git a/storage/connect/filamap.cpp b/storage/connect/filamap.cpp index 5cf9a4d945c..94c562a9981 100644 --- a/storage/connect/filamap.cpp +++ b/storage/connect/filamap.cpp @@ -87,7 +87,7 @@ int MAPFAM::GetFileLength(PGLOBAL g) { int len; - len = (To_Fb) ? To_Fb->Length : TXTFAM::GetFileLength(g); + len = (To_Fb && To_Fb->Count) ? To_Fb->Length : TXTFAM::GetFileLength(g); if (trace) htrc("Mapped file length=%d\n", len); @@ -319,11 +319,13 @@ int MAPFAM::SkipRecord(PGLOBAL g, bool header) /***********************************************************************/ int MAPFAM::ReadBuffer(PGLOBAL g) { - int len; + int rc, len; // Are we at the end of the memory - if (Mempos >= Top) - return RC_EF; + if (Mempos >= Top) + if ((rc = GetNext(g)) != RC_OK) + return rc; + if (!Placed) { /*******************************************************************/ @@ -341,8 +343,10 @@ int MAPFAM::ReadBuffer(PGLOBAL g) /*******************************************************************/ switch (Tdbp->TestBlock(g)) { case RC_EF: - return RC_EF; - case RC_NF: + if ((rc = GetNext(g)) != RC_OK) + return rc; + + case RC_NF: // Skip this record if ((rc = SkipRecord(g, false)) != RC_OK) return rc; @@ -413,7 +417,7 @@ int MAPFAM::DeleteRecords(PGLOBAL g, int irc) if (Tpos == Spos) { /*******************************************************************/ - /* First line to delete. Move of eventual preceeding lines is */ + /* First line to delete. Move of eventual preceding lines is */ /* not required here, just setting of future Spos and Tpos. */ /*******************************************************************/ Tpos = Spos = Fpos; @@ -498,7 +502,7 @@ int MAPFAM::DeleteRecords(PGLOBAL g, int irc) void MAPFAM::CloseTableFile(PGLOBAL g, bool) { PlugCloseFile(g, To_Fb); - To_Fb = NULL; // To get correct file size in Cardinality +//To_Fb = NULL; // To get correct file size in Cardinality if (trace) htrc("MAP Close: closing %s count=%d\n", @@ -569,7 +573,7 @@ int MBKFAM::GetRowID(void) /***********************************************************************/ int MBKFAM::ReadBuffer(PGLOBAL g) { - int len; + int rc, len; /*********************************************************************/ /* Sequential block reading when Placed is not true. */ @@ -577,8 +581,10 @@ int MBKFAM::ReadBuffer(PGLOBAL g) if (Placed) { Placed = false; } else if (Mempos >= Top) { // Are we at the end of the memory - return RC_EF; - } else if (++CurNum < Nrec) { + if ((rc = GetNext(g)) != RC_OK) + return rc; + + } else if (++CurNum < Nrec) { Fpos = Mempos; } else { /*******************************************************************/ @@ -588,7 +594,8 @@ int MBKFAM::ReadBuffer(PGLOBAL g) next: if (++CurBlk >= Block) - return RC_EF; + if ((rc = GetNext(g)) != RC_OK) + return rc; /*******************************************************************/ /* Before reading a new block, check whether block optimization */ @@ -596,8 +603,11 @@ int MBKFAM::ReadBuffer(PGLOBAL g) /*******************************************************************/ switch (Tdbp->TestBlock(g)) { case RC_EF: - return RC_EF; - case RC_NF: + if ((rc = GetNext(g)) != RC_OK) + return rc; + + break; + case RC_NF: goto next; } // endswitch rc @@ -697,14 +707,18 @@ int MPXFAM::InitDelete(PGLOBAL, int fpos, int) /***********************************************************************/ int MPXFAM::ReadBuffer(PGLOBAL g) { + int rc; + /*********************************************************************/ /* Sequential block reading when Placed is not true. */ /*********************************************************************/ if (Placed) { Placed = false; } else if (Mempos >= Top) { // Are we at the end of the memory - return RC_EF; - } else if (++CurNum < Nrec) { + if ((rc = GetNext(g)) != RC_OK) + return rc; + + } else if (++CurNum < Nrec) { Fpos = Mempos; } else { /*******************************************************************/ @@ -714,7 +728,7 @@ int MPXFAM::ReadBuffer(PGLOBAL g) next: if (++CurBlk >= Block) - return RC_EF; + return GetNext(g); /*******************************************************************/ /* Before reading a new block, check whether block optimization */ @@ -722,8 +736,11 @@ int MPXFAM::ReadBuffer(PGLOBAL g) /*******************************************************************/ switch (Tdbp->TestBlock(g)) { case RC_EF: - return RC_EF; - case RC_NF: + if ((rc = GetNext(g)) != RC_OK) + return rc; + + break; + case RC_NF: goto next; } // endswitch rc diff --git a/storage/connect/filamap.h b/storage/connect/filamap.h index b9c8ad965fd..774eb8b91b3 100644 --- a/storage/connect/filamap.h +++ b/storage/connect/filamap.h @@ -41,7 +41,8 @@ class DllExport MAPFAM : public TXTFAM { virtual int SkipRecord(PGLOBAL g, bool header); virtual bool OpenTableFile(PGLOBAL g); virtual bool DeferReading(void) {return false;} - virtual int ReadBuffer(PGLOBAL g); + virtual int GetNext(PGLOBAL g) {return RC_EF;} + virtual int ReadBuffer(PGLOBAL g); virtual int WriteBuffer(PGLOBAL g); virtual int DeleteRecords(PGLOBAL g, int irc); virtual void CloseTableFile(PGLOBAL g, bool abort); diff --git a/storage/connect/filamgz.cpp b/storage/connect/filamgz.cpp new file mode 100644 index 00000000000..07242ea633c --- /dev/null +++ b/storage/connect/filamgz.cpp @@ -0,0 +1,1426 @@ +/************ File AM GZ C++ Program Source Code File (.CPP) ***********/ +/* PROGRAM NAME: FILAMGZ */ +/* ------------- */ +/* Version 1.5 */ +/* */ +/* COPYRIGHT: */ +/* ---------- */ +/* (C) Copyright to the author Olivier BERTRAND 2005-2016 */ +/* */ +/* WHAT THIS PROGRAM DOES: */ +/* ----------------------- */ +/* This program are the ZLIB compressed files classes. */ +/* */ +/***********************************************************************/ + +/***********************************************************************/ +/* Include relevant MariaDB header file. */ +/***********************************************************************/ +#include "my_global.h" +#if defined(__WIN__) +#include <io.h> +#include <fcntl.h> +#if defined(__BORLANDC__) +#define __MFC_COMPAT__ // To define min/max as macro +#endif +//#include <windows.h> +#else // !__WIN__ +#if defined(UNIX) +#include <errno.h> +#else // !UNIX +#include <io.h> +#endif +#include <fcntl.h> +#endif // !__WIN__ + +/***********************************************************************/ +/* Include application header files: */ +/* global.h is header containing all global declarations. */ +/* plgdbsem.h is header containing the DB application declarations. */ +/* tabdos.h is header containing the TABDOS class declarations. */ +/***********************************************************************/ +#include "global.h" +#include "plgdbsem.h" +//#include "catalog.h" +//#include "reldef.h" +//#include "xobject.h" +//#include "kindex.h" +#include "filamtxt.h" +#include "tabdos.h" +#if defined(UNIX) +#include "osutil.h" +#endif + +/***********************************************************************/ +/* This define prepares ZLIB function declarations. */ +/***********************************************************************/ +//#define ZLIB_DLL + +#include "filamgz.h" + +/***********************************************************************/ +/* DB static variables. */ +/***********************************************************************/ +extern int num_read, num_there, num_eq[]; // Statistics + +/* ------------------------------------------------------------------- */ + +/***********************************************************************/ +/* Implementation of the GZFAM class. */ +/***********************************************************************/ +GZFAM::GZFAM(PGZFAM txfp) : TXTFAM(txfp) + { + Zfile = txfp->Zfile; + Zpos = txfp->Zpos; + } // end of GZFAM copy constructor + +/***********************************************************************/ +/* Zerror: Error function for gz calls. */ +/* gzerror returns the error message for the last error which occurred*/ +/* on the given compressed file. errnum is set to zlib error number. */ +/* If an error occurred in the file system and not in the compression */ +/* library, errnum is set to Z_ERRNO and the application may consult */ +/* errno to get the exact error code. */ +/***********************************************************************/ +int GZFAM::Zerror(PGLOBAL g) + { + int errnum; + + strcpy(g->Message, gzerror(Zfile, &errnum)); + + if (errnum == Z_ERRNO) +#if defined(__WIN__) + sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(NULL)); +#else // !__WIN__ + sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(errno)); +#endif // !__WIN__ + + return (errnum == Z_STREAM_END) ? RC_EF : RC_FX; + } // end of Zerror + +/***********************************************************************/ +/* Reset: reset position values at the beginning of file. */ +/***********************************************************************/ +void GZFAM::Reset(void) + { + TXTFAM::Reset(); +//gzrewind(Zfile); // Useful ????? + Zpos = 0; + } // end of Reset + +/***********************************************************************/ +/* GZ GetFileLength: returns an estimate of what would be the */ +/* uncompressed file size in number of bytes. */ +/***********************************************************************/ +int GZFAM::GetFileLength(PGLOBAL g) + { + int len = TXTFAM::GetFileLength(g); + + if (len > 0) + // Estimate size reduction to a max of 6 + len *= 6; + + return len; + } // end of GetFileLength + +/***********************************************************************/ +/* GZ Access Method opening routine. */ +/***********************************************************************/ +bool GZFAM::OpenTableFile(PGLOBAL g) + { + char opmode[4], filename[_MAX_PATH]; + MODE mode = Tdbp->GetMode(); + + switch (mode) { + case MODE_READ: + strcpy(opmode, "r"); + break; + case MODE_UPDATE: + /*****************************************************************/ + /* Updating GZ files not implemented yet. */ + /*****************************************************************/ + strcpy(g->Message, MSG(UPD_ZIP_NOT_IMP)); + return true; + case MODE_DELETE: + if (!Tdbp->GetNext()) { + // Store the number of deleted lines + DelRows = Cardinality(g); + + // This will erase the entire file + strcpy(opmode, "w"); +// Block = 0; // For ZBKFAM +// Last = Nrec; // For ZBKFAM + Tdbp->ResetSize(); + } else { + sprintf(g->Message, MSG(NO_PART_DEL), "GZ"); + return true; + } // endif filter + + break; + case MODE_INSERT: + strcpy(opmode, "a+"); + break; + default: + sprintf(g->Message, MSG(BAD_OPEN_MODE), mode); + return true; + } // endswitch Mode + + /*********************************************************************/ + /* Open according to logical input/output mode required. */ + /* Use specific zlib functions. */ + /* Treat files as binary. */ + /*********************************************************************/ + strcat(opmode, "b"); + Zfile = gzopen(PlugSetPath(filename, To_File, Tdbp->GetPath()), opmode); + + if (Zfile == NULL) { + sprintf(g->Message, MSG(GZOPEN_ERROR), + opmode, (int)errno, filename); + strcat(strcat(g->Message, ": "), strerror(errno)); + return (mode == MODE_READ && errno == ENOENT) + ? PushWarning(g, Tdbp) : true; + } // endif Zfile + + /*********************************************************************/ + /* Something to be done here. >>>>>>>> NOT DONE <<<<<<<< */ + /*********************************************************************/ +//To_Fb = dbuserp->Openlist; // Keep track of File block + + /*********************************************************************/ + /* Allocate the line buffer. */ + /*********************************************************************/ + return AllocateBuffer(g); + } // end of OpenTableFile + +/***********************************************************************/ +/* Allocate the line buffer. For mode Delete a bigger buffer has to */ +/* be allocated because is it also used to move lines into the file. */ +/***********************************************************************/ +bool GZFAM::AllocateBuffer(PGLOBAL g) + { + MODE mode = Tdbp->GetMode(); + + Buflen = Lrecl + 2; // Lrecl does not include CRLF +//Buflen *= ((Mode == MODE_DELETE) ? DOS_BUFF_LEN : 1); NIY + + if (trace) + htrc("SubAllocating a buffer of %d bytes\n", Buflen); + + To_Buf = (char*)PlugSubAlloc(g, NULL, Buflen); + + if (mode == MODE_INSERT) { + /*******************************************************************/ + /* For Insert buffer must be prepared. */ + /*******************************************************************/ + memset(To_Buf, ' ', Buflen); + To_Buf[Buflen - 2] = '\n'; + To_Buf[Buflen - 1] = '\0'; + } // endif Insert + + return false; + } // end of AllocateBuffer + +/***********************************************************************/ +/* GetRowID: return the RowID of last read record. */ +/***********************************************************************/ +int GZFAM::GetRowID(void) + { + return Rows; + } // end of GetRowID + +/***********************************************************************/ +/* GetPos: return the position of last read record. */ +/***********************************************************************/ +int GZFAM::GetPos(void) + { + return (int)Zpos; + } // end of GetPos + +/***********************************************************************/ +/* GetNextPos: return the position of next record. */ +/***********************************************************************/ +int GZFAM::GetNextPos(void) + { + return gztell(Zfile); + } // end of GetNextPos + +/***********************************************************************/ +/* SetPos: Replace the table at the specified position. */ +/***********************************************************************/ +bool GZFAM::SetPos(PGLOBAL g, int pos __attribute__((unused))) + { + sprintf(g->Message, MSG(NO_SETPOS_YET), "GZ"); + return true; +#if 0 + Fpos = pos; + + if (fseek(Stream, Fpos, SEEK_SET)) { + sprintf(g->Message, MSG(FSETPOS_ERROR), Fpos); + return true; + } // endif + + Placed = true; + return false; +#endif // 0 + } // end of SetPos + +/***********************************************************************/ +/* Record file position in case of UPDATE or DELETE. */ +/***********************************************************************/ +bool GZFAM::RecordPos(PGLOBAL) + { + Zpos = gztell(Zfile); + return false; + } // end of RecordPos + +/***********************************************************************/ +/* Skip one record in file. */ +/***********************************************************************/ +int GZFAM::SkipRecord(PGLOBAL g, bool header) + { + // Skip this record + if (gzeof(Zfile)) + return RC_EF; + else if (gzgets(Zfile, To_Buf, Buflen) == Z_NULL) + return Zerror(g); + + if (header) + RecordPos(g); + + return RC_OK; + } // end of SkipRecord + +/***********************************************************************/ +/* ReadBuffer: Read one line from a compressed text file. */ +/***********************************************************************/ +int GZFAM::ReadBuffer(PGLOBAL g) + { + char *p; + int rc; + + if (!Zfile) + return RC_EF; + + if (!Placed) { + /*******************************************************************/ + /* Record file position in case of UPDATE or DELETE. */ + /*******************************************************************/ + next: + if (RecordPos(g)) + return RC_FX; + + CurBlk = Rows++; // Update RowID + + /*******************************************************************/ + /* Check whether optimization on ROWID */ + /* can be done, as well as for join as for local filtering. */ + /*******************************************************************/ + switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + // Skip this record + if ((rc = SkipRecord(g, FALSE)) != RC_OK) + return rc; + + goto next; + } // endswitch rc + + } else + Placed = false; + + if (gzeof(Zfile)) { + rc = RC_EF; + } else if (gzgets(Zfile, To_Buf, Buflen) != Z_NULL) { + p = To_Buf + strlen(To_Buf) - 1; + + if (*p == '\n') + *p = '\0'; // Eliminate ending new-line character + + if (*(--p) == '\r') + *p = '\0'; // Eliminate eventuel carriage return + + strcpy(Tdbp->GetLine(), To_Buf); + IsRead = true; + rc = RC_OK; + num_read++; + } else + rc = Zerror(g); + + if (trace > 1) + htrc(" Read: '%s' rc=%d\n", To_Buf, rc); + + return rc; + } // end of ReadBuffer + +/***********************************************************************/ +/* WriteDB: Data Base write routine for ZDOS access method. */ +/* Update is not possible without using a temporary file (NIY). */ +/***********************************************************************/ +int GZFAM::WriteBuffer(PGLOBAL g) + { + /*********************************************************************/ + /* Prepare the write buffer. */ + /*********************************************************************/ + strcat(strcpy(To_Buf, Tdbp->GetLine()), CrLf); + + /*********************************************************************/ + /* Now start the writing process. */ + /*********************************************************************/ + if (gzputs(Zfile, To_Buf) < 0) + return Zerror(g); + + return RC_OK; + } // end of WriteBuffer + +/***********************************************************************/ +/* Data Base delete line routine for ZDOS access method. (NIY) */ +/***********************************************************************/ +int GZFAM::DeleteRecords(PGLOBAL g, int) + { + strcpy(g->Message, MSG(NO_ZIP_DELETE)); + return RC_FX; + } // end of DeleteRecords + +/***********************************************************************/ +/* Data Base close routine for DOS access method. */ +/***********************************************************************/ +void GZFAM::CloseTableFile(PGLOBAL, bool) + { + int rc = gzclose(Zfile); + + if (trace) + htrc("GZ CloseDB: closing %s rc=%d\n", To_File, rc); + + Zfile = NULL; // So we can know whether table is open +//To_Fb->Count = 0; // Avoid double closing by PlugCloseAll + } // end of CloseTableFile + +/***********************************************************************/ +/* Rewind routine for GZ access method. */ +/***********************************************************************/ +void GZFAM::Rewind(void) + { + gzrewind(Zfile); + } // end of Rewind + +/* ------------------------------------------------------------------- */ + +/***********************************************************************/ +/* Constructors. */ +/***********************************************************************/ +ZBKFAM::ZBKFAM(PDOSDEF tdp) : GZFAM(tdp) + { + Blocked = true; + Block = tdp->GetBlock(); + Last = tdp->GetLast(); + Nrec = tdp->GetElemt(); + CurLine = NULL; + NxtLine = NULL; + Closing = false; + BlkPos = tdp->GetTo_Pos(); + } // end of ZBKFAM standard constructor + +ZBKFAM::ZBKFAM(PZBKFAM txfp) : GZFAM(txfp) + { + CurLine = txfp->CurLine; + NxtLine = txfp->NxtLine; + Closing = txfp->Closing; + } // end of ZBKFAM copy constructor + +/***********************************************************************/ +/* Use BlockTest to reduce the table estimated size. */ +/***********************************************************************/ +int ZBKFAM::MaxBlkSize(PGLOBAL g, int) + { + int rc = RC_OK, savcur = CurBlk; + int size; + + // Roughly estimate the table size as the sum of blocks + // that can contain good rows + for (size = 0, CurBlk = 0; CurBlk < Block; CurBlk++) + if ((rc = Tdbp->TestBlock(g)) == RC_OK) + size += (CurBlk == Block - 1) ? Last : Nrec; + else if (rc == RC_EF) + break; + + CurBlk = savcur; + return size; + } // end of MaxBlkSize + +/***********************************************************************/ +/* ZBK Cardinality: returns table cardinality in number of rows. */ +/* This function can be called with a null argument to test the */ +/* availability of Cardinality implementation (1 yes, 0 no). */ +/***********************************************************************/ +int ZBKFAM::Cardinality(PGLOBAL g) + { + return (g) ? (int)((Block - 1) * Nrec + Last) : 1; + } // end of Cardinality + +/***********************************************************************/ +/* Allocate the line buffer. For mode Delete a bigger buffer has to */ +/* be allocated because is it also used to move lines into the file. */ +/***********************************************************************/ +bool ZBKFAM::AllocateBuffer(PGLOBAL g) + { + Buflen = Nrec * (Lrecl + 2); + CurLine = To_Buf = (char*)PlugSubAlloc(g, NULL, Buflen); + + if (Tdbp->GetMode() == MODE_INSERT) { + // Set values so Block and Last can be recalculated + if (Last == Nrec) { + CurBlk = Block; + Rbuf = Nrec; // To be used by WriteDB + } else { + // The last block must be completed + CurBlk = Block - 1; + Rbuf = Nrec - Last; // To be used by WriteDB + } // endif Last + + } // endif Insert + + return false; + } // end of AllocateBuffer + +/***********************************************************************/ +/* GetRowID: return the RowID of last read record. */ +/***********************************************************************/ +int ZBKFAM::GetRowID(void) + { + return CurNum + Nrec * CurBlk + 1; + } // end of GetRowID + +/***********************************************************************/ +/* GetPos: return the position of last read record. */ +/***********************************************************************/ +int ZBKFAM::GetPos(void) + { + return CurNum + Nrec * CurBlk; // Computed file index + } // end of GetPos + +/***********************************************************************/ +/* Record file position in case of UPDATE or DELETE. */ +/* Not used yet for fixed tables. */ +/***********************************************************************/ +bool ZBKFAM::RecordPos(PGLOBAL /*g*/) + { +//strcpy(g->Message, "RecordPos not implemented for gz blocked tables"); +//return true; + return RC_OK; + } // end of RecordPos + +/***********************************************************************/ +/* Skip one record in file. */ +/***********************************************************************/ +int ZBKFAM::SkipRecord(PGLOBAL /*g*/, bool) + { +//strcpy(g->Message, "SkipRecord not implemented for gz blocked tables"); +//return RC_FX; + return RC_OK; + } // end of SkipRecord + +/***********************************************************************/ +/* ReadBuffer: Read one line from a compressed text file. */ +/***********************************************************************/ +int ZBKFAM::ReadBuffer(PGLOBAL g) + { + int n, skip, rc = RC_OK; + + /*********************************************************************/ + /* Sequential reading when Placed is not true. */ + /*********************************************************************/ + if (++CurNum < Rbuf) { + CurLine = NxtLine; + + // Get the position of the next line in the buffer + while (*NxtLine++ != '\n') ; + + // Set caller line buffer + n = NxtLine - CurLine - Ending; + memcpy(Tdbp->GetLine(), CurLine, n); + Tdbp->GetLine()[n] = '\0'; + return RC_OK; + } else if (Rbuf < Nrec && CurBlk != -1) + return RC_EF; + + /*********************************************************************/ + /* New block. */ + /*********************************************************************/ + CurNum = 0; + skip = 0; + + next: + if (++CurBlk >= Block) + return RC_EF; + + /*********************************************************************/ + /* Before using the new block, check whether block optimization */ + /* can be done, as well as for join as for local filtering. */ + /*********************************************************************/ + switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + skip++; + goto next; + } // endswitch rc + + if (skip) + // Skip blocks rejected by block optimization + for (int i = CurBlk - skip; i < CurBlk; i++) { + BlkLen = BlkPos[i + 1] - BlkPos[i]; + + if (gzseek(Zfile, (z_off_t)BlkLen, SEEK_CUR) < 0) + return Zerror(g); + + } // endfor i + + BlkLen = BlkPos[CurBlk + 1] - BlkPos[CurBlk]; + + if (!(n = gzread(Zfile, To_Buf, BlkLen))) { + rc = RC_EF; + } else if (n > 0) { + // Get the position of the current line + CurLine = To_Buf; + + // Now get the position of the next line + for (NxtLine = CurLine; *NxtLine++ != '\n';) ; + + // Set caller line buffer + n = NxtLine - CurLine - Ending; + memcpy(Tdbp->GetLine(), CurLine, n); + Tdbp->GetLine()[n] = '\0'; + Rbuf = (CurBlk == Block - 1) ? Last : Nrec; + IsRead = true; + rc = RC_OK; + num_read++; + } else + rc = Zerror(g); + + return rc; + } // end of ReadBuffer + +/***********************************************************************/ +/* WriteDB: Data Base write routine for ZDOS access method. */ +/* Update is not possible without using a temporary file (NIY). */ +/***********************************************************************/ +int ZBKFAM::WriteBuffer(PGLOBAL g) + { + /*********************************************************************/ + /* Prepare the write buffer. */ + /*********************************************************************/ + if (!Closing) + strcat(strcpy(CurLine, Tdbp->GetLine()), CrLf); + + /*********************************************************************/ + /* In Insert mode, blocs are added sequentialy to the file end. */ + /* Note: Update mode is not handled for gz files. */ + /*********************************************************************/ + if (++CurNum == Rbuf) { + /*******************************************************************/ + /* New block, start the writing process. */ + /*******************************************************************/ + BlkLen = CurLine + strlen(CurLine) - To_Buf; + + if (gzwrite(Zfile, To_Buf, BlkLen) != BlkLen || + gzflush(Zfile, Z_FULL_FLUSH)) { + Closing = true; + return Zerror(g); + } // endif gzwrite + + Rbuf = Nrec; + CurBlk++; + CurNum = 0; + CurLine = To_Buf; + } else + CurLine += strlen(CurLine); + + return RC_OK; + } // end of WriteBuffer + +/***********************************************************************/ +/* Data Base delete line routine for ZBK access method. */ +/* Implemented only for total deletion of the table, which is done */ +/* by opening the file in mode "wb". */ +/***********************************************************************/ +int ZBKFAM::DeleteRecords(PGLOBAL g, int irc) + { + if (irc == RC_EF) { + LPCSTR name = Tdbp->GetName(); + PDOSDEF defp = (PDOSDEF)Tdbp->GetDef(); + + defp->SetBlock(0); + defp->SetLast(Nrec); + + if (!defp->SetIntCatInfo("Blocks", 0) || + !defp->SetIntCatInfo("Last", 0)) { + sprintf(g->Message, MSG(UPDATE_ERROR), "Header"); + return RC_FX; + } else + return RC_OK; + + } else + return irc; + + } // end of DeleteRecords + +/***********************************************************************/ +/* Data Base close routine for ZBK access method. */ +/***********************************************************************/ +void ZBKFAM::CloseTableFile(PGLOBAL g, bool) + { + int rc = RC_OK; + + if (Tdbp->GetMode() == MODE_INSERT) { + LPCSTR name = Tdbp->GetName(); + PDOSDEF defp = (PDOSDEF)Tdbp->GetDef(); + + if (CurNum && !Closing) { + // Some more inserted lines remain to be written + Last = (Nrec - Rbuf) + CurNum; + Block = CurBlk + 1; + Rbuf = CurNum--; + Closing = true; + rc = WriteBuffer(g); + } else if (Rbuf == Nrec) { + Last = Nrec; + Block = CurBlk; + } // endif CurNum + + if (rc != RC_FX) { + defp->SetBlock(Block); + defp->SetLast(Last); + defp->SetIntCatInfo("Blocks", Block); + defp->SetIntCatInfo("Last", Last); + } // endif + + gzclose(Zfile); + } else if (Tdbp->GetMode() == MODE_DELETE) { + rc = DeleteRecords(g, RC_EF); + gzclose(Zfile); + } else + rc = gzclose(Zfile); + + if (trace) + htrc("GZ CloseDB: closing %s rc=%d\n", To_File, rc); + + Zfile = NULL; // So we can know whether table is open +//To_Fb->Count = 0; // Avoid double closing by PlugCloseAll + } // end of CloseTableFile + +/***********************************************************************/ +/* Rewind routine for ZBK access method. */ +/***********************************************************************/ +void ZBKFAM::Rewind(void) + { + gzrewind(Zfile); + CurBlk = -1; + CurNum = Rbuf; + } // end of Rewind + +/* ------------------------------------------------------------------- */ + +/***********************************************************************/ +/* Constructors. */ +/***********************************************************************/ +ZIXFAM::ZIXFAM(PDOSDEF tdp) : ZBKFAM(tdp) + { +//Block = tdp->GetBlock(); +//Last = tdp->GetLast(); + Nrec = (tdp->GetElemt()) ? tdp->GetElemt() : DOS_BUFF_LEN; + Blksize = Nrec * Lrecl; + } // end of ZIXFAM standard constructor + +/***********************************************************************/ +/* ZIX Cardinality: returns table cardinality in number of rows. */ +/* This function can be called with a null argument to test the */ +/* availability of Cardinality implementation (1 yes, 0 no). */ +/***********************************************************************/ +int ZIXFAM::Cardinality(PGLOBAL g) + { + if (Last) + return (g) ? (int)((Block - 1) * Nrec + Last) : 1; + else // Last and Block not defined, cannot do it yet + return 0; + + } // end of Cardinality + +/***********************************************************************/ +/* Allocate the line buffer. For mode Delete a bigger buffer has to */ +/* be allocated because is it also used to move lines into the file. */ +/***********************************************************************/ +bool ZIXFAM::AllocateBuffer(PGLOBAL g) + { + Buflen = Blksize; + To_Buf = (char*)PlugSubAlloc(g, NULL, Buflen); + + if (Tdbp->GetMode() == MODE_INSERT) { + /*******************************************************************/ + /* For Insert the buffer must be prepared. */ + /*******************************************************************/ + memset(To_Buf, ' ', Buflen); + + if (Tdbp->GetFtype() < 2) + // if not binary, the file is physically a text file + for (int len = Lrecl; len <= Buflen; len += Lrecl) { +#if defined(__WIN__) + To_Buf[len - 2] = '\r'; +#endif // __WIN__ + To_Buf[len - 1] = '\n'; + } // endfor len + + // Set values so Block and Last can be recalculated + if (Last == Nrec) { + CurBlk = Block; + Rbuf = Nrec; // To be used by WriteDB + } else { + // The last block must be completed + CurBlk = Block - 1; + Rbuf = Nrec - Last; // To be used by WriteDB + } // endif Last + + } // endif Insert + + return false; + } // end of AllocateBuffer + +/***********************************************************************/ +/* ReadBuffer: Read one line from a compressed text file. */ +/***********************************************************************/ +int ZIXFAM::ReadBuffer(PGLOBAL g) + { + int n, rc = RC_OK; + + /*********************************************************************/ + /* Sequential reading when Placed is not true. */ + /*********************************************************************/ + if (++CurNum < Rbuf) { + Tdbp->IncLine(Lrecl); // Used by DOSCOL functions + return RC_OK; + } else if (Rbuf < Nrec && CurBlk != -1) + return RC_EF; + + /*********************************************************************/ + /* New block. */ + /*********************************************************************/ + CurNum = 0; + Tdbp->SetLine(To_Buf); + + int skip = 0; + + next: + if (++CurBlk >= Block) + return RC_EF; + + /*********************************************************************/ + /* Before using the new block, check whether block optimization */ + /* can be done, as well as for join as for local filtering. */ + /*********************************************************************/ + switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + skip++; + goto next; + } // endswitch rc + + if (skip) + // Skip blocks rejected by block optimization + for (int i = 0; i < skip; i++) { + if (gzseek(Zfile, (z_off_t)Buflen, SEEK_CUR) < 0) + return Zerror(g); + + } // endfor i + + if (!(n = gzread(Zfile, To_Buf, Buflen))) { + rc = RC_EF; + } else if (n > 0) { + Rbuf = n / Lrecl; + IsRead = true; + rc = RC_OK; + num_read++; + } else + rc = Zerror(g); + + return rc; + } // end of ReadBuffer + +/***********************************************************************/ +/* WriteDB: Data Base write routine for ZDOS access method. */ +/* Update is not possible without using a temporary file (NIY). */ +/***********************************************************************/ +int ZIXFAM::WriteBuffer(PGLOBAL g) + { + /*********************************************************************/ + /* In Insert mode, blocs are added sequentialy to the file end. */ + /* Note: Update mode is not handled for gz files. */ + /*********************************************************************/ + if (++CurNum == Rbuf) { + /*******************************************************************/ + /* New block, start the writing process. */ + /*******************************************************************/ + BlkLen = Rbuf * Lrecl; + + if (gzwrite(Zfile, To_Buf, BlkLen) != BlkLen || + gzflush(Zfile, Z_FULL_FLUSH)) { + Closing = true; + return Zerror(g); + } // endif gzwrite + + Rbuf = Nrec; + CurBlk++; + CurNum = 0; + Tdbp->SetLine(To_Buf); + } else + Tdbp->IncLine(Lrecl); // Used by FIXCOL functions + + return RC_OK; + } // end of WriteBuffer + +/* --------------------------- Class ZLBFAM -------------------------- */ + +/***********************************************************************/ +/* Constructors. */ +/***********************************************************************/ +ZLBFAM::ZLBFAM(PDOSDEF tdp) : BLKFAM(tdp) + { + Zstream = NULL; + Zbuffer = NULL; + Zlenp = NULL; + Optimized = tdp->IsOptimized(); + } // end of ZLBFAM standard constructor + +ZLBFAM::ZLBFAM(PZLBFAM txfp) : BLKFAM(txfp) + { + Zstream = txfp->Zstream; + Zbuffer = txfp->Zbuffer; + Zlenp = txfp->Zlenp; + Optimized = txfp->Optimized; + } // end of ZLBFAM (dummy?) copy constructor + +/***********************************************************************/ +/* ZLB GetFileLength: returns an estimate of what would be the */ +/* uncompressed file size in number of bytes. */ +/***********************************************************************/ +int ZLBFAM::GetFileLength(PGLOBAL g) + { + int len = (Optimized) ? BlkPos[Block] : BLKFAM::GetFileLength(g); + + if (len > 0) + // Estimate size reduction to a max of 5 + len *= 5; + + return len; + } // end of GetFileLength + +/***********************************************************************/ +/* Allocate the line buffer. For mode Delete a bigger buffer has to */ +/* be allocated because is it also used to move lines into the file. */ +/***********************************************************************/ +bool ZLBFAM::AllocateBuffer(PGLOBAL g) + { + char *msg; + int n, zrc; + +#if 0 + if (!Optimized && Tdbp->NeedIndexing(g)) { + strcpy(g->Message, MSG(NOP_ZLIB_INDEX)); + return TRUE; + } // endif indexing +#endif // 0 + +#if defined(NOLIB) + if (!zlib && LoadZlib()) { + sprintf(g->Message, MSG(DLL_LOAD_ERROR), GetLastError(), "zlib.dll"); + return TRUE; + } // endif zlib +#endif + + BLKFAM::AllocateBuffer(g); +//Buflen = Nrec * (Lrecl + 2); +//Rbuf = Nrec; + + // Allocate the compressed buffer + n = Buflen + 16; // ????????????????????????????????? + Zlenp = (int*)PlugSubAlloc(g, NULL, n); + Zbuffer = (Byte*)(Zlenp + 1); + + // Allocate and initialize the Z stream + Zstream = (z_streamp)PlugSubAlloc(g, NULL, sizeof(z_stream)); + Zstream->zalloc = (alloc_func)0; + Zstream->zfree = (free_func)0; + Zstream->opaque = (voidpf)0; + Zstream->next_in = NULL; + Zstream->avail_in = 0; + + if (Tdbp->GetMode() == MODE_READ) { + msg = "inflateInit"; + zrc = inflateInit(Zstream); + } else { + msg = "deflateInit"; + zrc = deflateInit(Zstream, Z_DEFAULT_COMPRESSION); + } // endif Mode + + if (zrc != Z_OK) { + if (Zstream->msg) + sprintf(g->Message, "%s error: %s", msg, Zstream->msg); + else + sprintf(g->Message, "%s error: %d", msg, zrc); + + return TRUE; + } // endif zrc + + if (Tdbp->GetMode() == MODE_INSERT) { + // Write the file header block + if (Last == Nrec) { + CurBlk = Block; + CurNum = 0; + + if (!GetFileLength(g)) { + // Write the zlib header as an extra block + strcpy(To_Buf, "PlugDB"); + BlkLen = strlen("PlugDB") + 1; + + if (WriteCompressedBuffer(g)) + return TRUE; + + } // endif void file + + } else { + // In mode insert, if Last != Nrec, last block must be updated + CurBlk = Block - 1; + CurNum = Last; + + strcpy(g->Message, MSG(NO_PAR_BLK_INS)); + return TRUE; + } // endif Last + + } else { // MODE_READ + // First thing to do is to read the header block + void *rdbuf; + + if (Optimized) { + BlkLen = BlkPos[0]; + rdbuf = Zlenp; + } else { + // Get the stored length from the file itself + if (fread(Zlenp, sizeof(int), 1, Stream) != 1) + return FALSE; // Empty file + + BlkLen = *Zlenp; + rdbuf = Zbuffer; + } // endif Optimized + + switch (ReadCompressedBuffer(g, rdbuf)) { + case RC_EF: + return FALSE; + case RC_FX: +#if defined(UNIX) + sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(errno)); +#else + sprintf(g->Message, MSG(READ_ERROR), To_File, _strerror(NULL)); +#endif + case RC_NF: + return TRUE; + } // endswitch + + // Some old tables can have PlugDB in their header + if (strcmp(To_Buf, "PlugDB")) { + sprintf(g->Message, MSG(BAD_HEADER), Tdbp->GetFile(g)); + return TRUE; + } // endif strcmp + + } // endif Mode + + return FALSE; + } // end of AllocateBuffer + +/***********************************************************************/ +/* GetPos: return the position of last read record. */ +/***********************************************************************/ +int ZLBFAM::GetPos(void) + { + return (Optimized) ? (CurNum + Nrec * CurBlk) : Fpos; + } // end of GetPos + +/***********************************************************************/ +/* GetNextPos: should not be called for this class. */ +/***********************************************************************/ +int ZLBFAM::GetNextPos(void) + { + if (Optimized) { + assert(FALSE); + return 0; + } else + return ftell(Stream); + + } // end of GetNextPos + +/***********************************************************************/ +/* SetPos: Replace the table at the specified position. */ +/***********************************************************************/ +bool ZLBFAM::SetPos(PGLOBAL g, int pos __attribute__((unused))) + { + sprintf(g->Message, MSG(NO_SETPOS_YET), "GZ"); + return true; +#if 0 // All this must be checked + if (pos < 0) { + strcpy(g->Message, MSG(INV_REC_POS)); + return true; + } // endif recpos + + CurBlk = pos / Nrec; + CurNum = pos % Nrec; +#if defined(_DEBUG) + num_eq[(CurBlk == OldBlk) ? 1 : 0]++; +#endif + + // Indicate the table position was externally set + Placed = true; + return false; +#endif // 0 + } // end of SetPos + +/***********************************************************************/ +/* ReadBuffer: Read one line for a text file. */ +/***********************************************************************/ +int ZLBFAM::ReadBuffer(PGLOBAL g) + { + int n; + void *rdbuf; + + /*********************************************************************/ + /* Sequential reading when Placed is not true. */ + /*********************************************************************/ + if (Placed) { + Placed = FALSE; + } else if (++CurNum < Rbuf) { + CurLine = NxtLine; + + // Get the position of the next line in the buffer + if (Tdbp->GetFtype() == RECFM_VAR) + while (*NxtLine++ != '\n') ; + else + NxtLine += Lrecl; + + // Set caller line buffer + n = NxtLine - CurLine - ((Tdbp->GetFtype() == RECFM_BIN) ? 0 : Ending); + memcpy(Tdbp->GetLine(), CurLine, n); + Tdbp->GetLine()[n] = '\0'; + return RC_OK; + } else if (Rbuf < Nrec && CurBlk != -1) { + CurNum--; // To have a correct Last value when optimizing + return RC_EF; + } else { + /*******************************************************************/ + /* New block. */ + /*******************************************************************/ + CurNum = 0; + + next: + if (++CurBlk >= Block) + return RC_EF; + + /*******************************************************************/ + /* Before reading a new block, check whether block optimization */ + /* can be done, as well as for join as for local filtering. */ + /*******************************************************************/ + if (Optimized) switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + goto next; + } // endswitch rc + + } // endif's + + if (OldBlk == CurBlk) + goto ok; // Block is already there + + if (Optimized) { + // Store the position of next block + Fpos = BlkPos[CurBlk]; + + // fseek is required only in non sequential reading + if (CurBlk != OldBlk + 1) + if (fseek(Stream, Fpos, SEEK_SET)) { + sprintf(g->Message, MSG(FSETPOS_ERROR), Fpos); + return RC_FX; + } // endif fseek + + // Calculate the length of block to read + BlkLen = BlkPos[CurBlk + 1] - Fpos; + rdbuf = Zlenp; + } else { // !Optimized + if (CurBlk != OldBlk + 1) { + strcpy(g->Message, MSG(INV_RAND_ACC)); + return RC_FX; + } else + Fpos = ftell(Stream); // Used when optimizing + + // Get the stored length from the file itself + if (fread(Zlenp, sizeof(int), 1, Stream) != 1) { + if (feof(Stream)) + return RC_EF; + + goto err; + } // endif fread + + BlkLen = *Zlenp; + rdbuf = Zbuffer; + } // endif Optimized + + // Read the next block + switch (ReadCompressedBuffer(g, rdbuf)) { + case RC_FX: goto err; + case RC_NF: return RC_FX; + case RC_EF: return RC_EF; + default: Rbuf = (CurBlk == Block - 1) ? Last : Nrec; + } // endswitch ReadCompressedBuffer + + ok: + if (Tdbp->GetFtype() == RECFM_VAR) { + int i; + + // Get the position of the current line + for (i = 0, CurLine = To_Buf; i < CurNum; i++) + while (*CurLine++ != '\n') ; // What about Unix ??? + + // Now get the position of the next line + for (NxtLine = CurLine; *NxtLine++ != '\n';) ; + + // Set caller line buffer + n = NxtLine - CurLine - Ending; + } else { + CurLine = To_Buf + CurNum * Lrecl; + NxtLine = CurLine + Lrecl; + n = Lrecl - ((Tdbp->GetFtype() == RECFM_BIN) ? 0 : Ending); + } // endif Ftype + + memcpy(Tdbp->GetLine(), CurLine, n); + Tdbp->GetLine()[n] = '\0'; + + OldBlk = CurBlk; // Last block actually read + IsRead = TRUE; // Is read indeed + return RC_OK; + + err: +#if defined(UNIX) + sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(errno)); +#else + sprintf(g->Message, MSG(READ_ERROR), To_File, _strerror(NULL)); +#endif + return RC_FX; + } // end of ReadBuffer + +/***********************************************************************/ +/* Read and decompress a block from the stream. */ +/***********************************************************************/ +int ZLBFAM::ReadCompressedBuffer(PGLOBAL g, void *rdbuf) + { + if (fread(rdbuf, 1, (size_t)BlkLen, Stream) == (unsigned)BlkLen) { + int zrc; + + num_read++; + + if (Optimized && BlkLen != signed(*Zlenp + sizeof(int))) { + sprintf(g->Message, MSG(BAD_BLK_SIZE), CurBlk + 1); + return RC_NF; + } // endif BlkLen + + // HERE WE MUST INFLATE THE BLOCK + Zstream->next_in = Zbuffer; + Zstream->avail_in = (uInt)(*Zlenp); + Zstream->next_out = (Byte*)To_Buf; + Zstream->avail_out = Buflen; + zrc = inflate(Zstream, Z_SYNC_FLUSH); + + if (zrc != Z_OK) { + if (Zstream->msg) + sprintf(g->Message, MSG(FUNC_ERR_S), "inflate", Zstream->msg); + else + sprintf(g->Message, MSG(FUNCTION_ERROR), "inflate", (int)zrc); + + return RC_NF; + } // endif zrc + + } else if (feof(Stream)) { + return RC_EF; + } else + return RC_FX; + + return RC_OK; + } // end of ReadCompressedBuffer + +/***********************************************************************/ +/* WriteBuffer: File write routine for DOS access method. */ +/* Update is directly written back into the file, */ +/* with this (fast) method, record size cannot change. */ +/***********************************************************************/ +int ZLBFAM::WriteBuffer(PGLOBAL g) + { + assert (Tdbp->GetMode() == MODE_INSERT); + + /*********************************************************************/ + /* Prepare the write buffer. */ + /*********************************************************************/ + if (!Closing) { + if (Tdbp->GetFtype() == RECFM_BIN) + memcpy(CurLine, Tdbp->GetLine(), Lrecl); + else + strcat(strcpy(CurLine, Tdbp->GetLine()), CrLf); + +#if defined(_DEBUG) + if (Tdbp->GetFtype() == RECFM_FIX && + (signed)strlen(CurLine) != Lrecl + (signed)strlen(CrLf)) { + strcpy(g->Message, MSG(BAD_LINE_LEN)); + Closing = TRUE; + return RC_FX; + } // endif Lrecl +#endif // _DEBUG + } // endif Closing + + /*********************************************************************/ + /* In Insert mode, blocs are added sequentialy to the file end. */ + /*********************************************************************/ + if (++CurNum != Rbuf) { + if (Tdbp->GetFtype() == RECFM_VAR) + CurLine += strlen(CurLine); + else + CurLine += Lrecl; + + return RC_OK; // We write only full blocks + } // endif CurNum + + // HERE WE MUST DEFLATE THE BLOCK + if (Tdbp->GetFtype() == RECFM_VAR) + NxtLine = CurLine + strlen(CurLine); + else + NxtLine = CurLine + Lrecl; + + BlkLen = NxtLine - To_Buf; + + if (WriteCompressedBuffer(g)) { + Closing = TRUE; // To tell CloseDB about a Write error + return RC_FX; + } // endif WriteCompressedBuffer + + CurBlk++; + CurNum = 0; + CurLine = To_Buf; + return RC_OK; + } // end of WriteBuffer + +/***********************************************************************/ +/* Compress the buffer and write the deflated output to stream. */ +/***********************************************************************/ +bool ZLBFAM::WriteCompressedBuffer(PGLOBAL g) + { + int zrc; + + Zstream->next_in = (Byte*)To_Buf; + Zstream->avail_in = (uInt)BlkLen; + Zstream->next_out = Zbuffer; + Zstream->avail_out = Buflen + 16; + Zstream->total_out = 0; + zrc = deflate(Zstream, Z_FULL_FLUSH); + + if (zrc != Z_OK) { + if (Zstream->msg) + sprintf(g->Message, MSG(FUNC_ERR_S), "deflate", Zstream->msg); + else + sprintf(g->Message, MSG(FUNCTION_ERROR), "deflate", (int)zrc); + + return TRUE; + } else + *Zlenp = Zstream->total_out; + + // Now start the writing process. + BlkLen = *Zlenp + sizeof(int); + + if (fwrite(Zlenp, 1, BlkLen, Stream) != (size_t)BlkLen) { + sprintf(g->Message, MSG(FWRITE_ERROR), strerror(errno)); + return TRUE; + } // endif size + + return FALSE; + } // end of WriteCompressedBuffer + +/***********************************************************************/ +/* Table file close routine for DOS access method. */ +/***********************************************************************/ +void ZLBFAM::CloseTableFile(PGLOBAL g, bool) + { + int rc = RC_OK; + + if (Tdbp->GetMode() == MODE_INSERT) { + LPCSTR name = Tdbp->GetName(); + PDOSDEF defp = (PDOSDEF)Tdbp->GetDef(); + + // Closing is True if last Write was in error + if (CurNum && !Closing) { + // Some more inserted lines remain to be written + Last = (Nrec - Rbuf) + CurNum; + Block = CurBlk + 1; + Rbuf = CurNum--; + Closing = TRUE; + rc = WriteBuffer(g); + } else if (Rbuf == Nrec) { + Last = Nrec; + Block = CurBlk; + } // endif CurNum + + if (rc != RC_FX) { + defp->SetBlock(Block); + defp->SetLast(Last); + defp->SetIntCatInfo("Blocks", Block); + defp->SetIntCatInfo("Last", Last); + } // endif + + fclose(Stream); + } else + rc = fclose(Stream); + + if (trace) + htrc("ZLB CloseTableFile: closing %s mode=%d rc=%d\n", + To_File, Tdbp->GetMode(), rc); + + Stream = NULL; // So we can know whether table is open + To_Fb->Count = 0; // Avoid double closing by PlugCloseAll + + if (Tdbp->GetMode() == MODE_READ) + rc = inflateEnd(Zstream); + else + rc = deflateEnd(Zstream); + + } // end of CloseTableFile + +/***********************************************************************/ +/* Rewind routine for ZLIB access method. */ +/***********************************************************************/ +void ZLBFAM::Rewind(void) + { + // We must be positioned after the header block + if (CurBlk >= 0) { // Nothing to do if no block read yet + if (!Optimized) { // If optimized, fseek will be done in ReadBuffer + size_t st; + + rewind(Stream); + + if (!(st = fread(Zlenp, sizeof(int), 1, Stream)) && trace) + htrc("fread error %d in Rewind", errno); + + fseek(Stream, *Zlenp + sizeof(int), SEEK_SET); + OldBlk = -1; + } // endif Optimized + + CurBlk = -1; + CurNum = Rbuf; + } // endif CurBlk + +//OldBlk = -1; +//Rbuf = 0; commented out in case we reuse last read block + } // end of Rewind + +/* ------------------------ End of GzFam ---------------------------- */ diff --git a/storage/connect/filamgz.h b/storage/connect/filamgz.h new file mode 100644 index 00000000000..d667fdddcc2 --- /dev/null +++ b/storage/connect/filamgz.h @@ -0,0 +1,170 @@ +/*************** FilAmGz H Declares Source Code File (.H) **************/ +/* Name: FILAMGZ.H Version 1.3 */ +/* */ +/* (C) Copyright to the author Olivier BERTRAND 2005-2016 */ +/* */ +/* This file contains the GZIP access method classes declares. */ +/***********************************************************************/ +#ifndef __FILAMGZ_H +#define __FILAMGZ_H + +#include "zlib.h" + +typedef class GZFAM *PGZFAM; +typedef class ZBKFAM *PZBKFAM; +typedef class ZIXFAM *PZIXFAM; +typedef class ZLBFAM *PZLBFAM; + +/***********************************************************************/ +/* This is the access method class declaration for not optimized */ +/* variable record length files compressed using the gzip library */ +/* functions. File is accessed record by record (row). */ +/***********************************************************************/ +class DllExport GZFAM : public TXTFAM { +// friend class DOSCOL; + public: + // Constructor + GZFAM(PDOSDEF tdp) : TXTFAM(tdp) {Zfile = NULL; Zpos = 0;} + GZFAM(PGZFAM txfp); + + // Implementation + virtual AMT GetAmType(void) {return TYPE_AM_GZ;} + virtual int GetPos(void); + virtual int GetNextPos(void); + virtual PTXF Duplicate(PGLOBAL g) + {return (PTXF)new(g) GZFAM(this);} + + // Methods + virtual void Reset(void); + virtual int GetFileLength(PGLOBAL g); + virtual int Cardinality(PGLOBAL g) {return (g) ? -1 : 0;} + virtual int MaxBlkSize(PGLOBAL g, int s) {return s;} + virtual bool AllocateBuffer(PGLOBAL g); + virtual int GetRowID(void); + virtual bool RecordPos(PGLOBAL g); + virtual bool SetPos(PGLOBAL g, int recpos); + virtual int SkipRecord(PGLOBAL g, bool header); + virtual bool OpenTableFile(PGLOBAL g); + virtual int ReadBuffer(PGLOBAL g); + virtual int WriteBuffer(PGLOBAL g); + virtual int DeleteRecords(PGLOBAL g, int irc); + virtual void CloseTableFile(PGLOBAL g, bool abort); + virtual void Rewind(void); + + protected: + int Zerror(PGLOBAL g); // GZ error function + + // Members + gzFile Zfile; // Points to GZ file structure + z_off_t Zpos; // Uncompressed file position + }; // end of class GZFAM + +/***********************************************************************/ +/* This is the access method class declaration for optimized variable */ +/* record length files compressed using the gzip library functions. */ +/* The File is accessed by block (requires an opt file). */ +/***********************************************************************/ +class DllExport ZBKFAM : public GZFAM { + public: + // Constructor + ZBKFAM(PDOSDEF tdp); + ZBKFAM(PZBKFAM txfp); + + // Implementation + virtual int GetPos(void); + virtual int GetNextPos(void) {return 0;} + virtual PTXF Duplicate(PGLOBAL g) + {return (PTXF)new(g) ZBKFAM(this);} + + // Methods + virtual int Cardinality(PGLOBAL g); + virtual int MaxBlkSize(PGLOBAL g, int s); + virtual bool AllocateBuffer(PGLOBAL g); + virtual int GetRowID(void); + virtual bool RecordPos(PGLOBAL g); + virtual int SkipRecord(PGLOBAL g, bool header); + virtual int ReadBuffer(PGLOBAL g); + virtual int WriteBuffer(PGLOBAL g); + virtual int DeleteRecords(PGLOBAL g, int irc); + virtual void CloseTableFile(PGLOBAL g, bool abort); + virtual void Rewind(void); + + protected: + // Members + char *CurLine; // Position of current line in buffer + char *NxtLine; // Position of Next line in buffer + bool Closing; // True when closing on Insert + }; // end of class ZBKFAM + +/***********************************************************************/ +/* This is the access method class declaration for fixed record */ +/* length files compressed using the gzip library functions. */ +/* The file is always accessed by block. */ +/***********************************************************************/ +class DllExport ZIXFAM : public ZBKFAM { + public: + // Constructor + ZIXFAM(PDOSDEF tdp); + ZIXFAM(PZIXFAM txfp) : ZBKFAM(txfp) {} + + // Implementation + virtual int GetNextPos(void) {return 0;} + virtual PTXF Duplicate(PGLOBAL g) + {return (PTXF)new(g) ZIXFAM(this);} + + // Methods + virtual int Cardinality(PGLOBAL g); + virtual bool AllocateBuffer(PGLOBAL g); + virtual int ReadBuffer(PGLOBAL g); + virtual int WriteBuffer(PGLOBAL g); + + protected: + // No additional Members + }; // end of class ZIXFAM + +/***********************************************************************/ +/* This is the DOS/UNIX Access Method class declaration for PlugDB */ +/* fixed/variable files compressed using the zlib library functions. */ +/* Physically these are written and read using the same technique */ +/* than blocked variable files, only the contain of each block is */ +/* compressed using the deflate zlib function. The purpose of this */ +/* specific format is to have a fast mechanism for direct access of */ +/* records so blocked optimization is fast and direct access (joins) */ +/* is allowed. Note that the block length is written ahead of each */ +/* block to enable reading when optimization file is not available. */ +/***********************************************************************/ +class DllExport ZLBFAM : public BLKFAM { + public: + // Constructor + ZLBFAM(PDOSDEF tdp); + ZLBFAM(PZLBFAM txfp); + + // Implementation + virtual AMT GetAmType(void) {return TYPE_AM_ZLIB;} + virtual int GetPos(void); + virtual int GetNextPos(void); + virtual PTXF Duplicate(PGLOBAL g) + {return (PTXF)new(g) ZLBFAM(this);} + inline void SetOptimized(bool b) {Optimized = b;} + + // Methods + virtual int GetFileLength(PGLOBAL g); + virtual bool SetPos(PGLOBAL g, int recpos); + virtual bool AllocateBuffer(PGLOBAL g); + virtual int ReadBuffer(PGLOBAL g); + virtual int WriteBuffer(PGLOBAL g); + virtual void CloseTableFile(PGLOBAL g, bool abort); + virtual void Rewind(void); + + protected: + bool WriteCompressedBuffer(PGLOBAL g); + int ReadCompressedBuffer(PGLOBAL g, void *rdbuf); + + // Members + z_streamp Zstream; // Compression/decompression stream + Byte *Zbuffer; // Compressed block buffer + int *Zlenp; // Pointer to block length + bool Optimized; // true when opt file is available + }; // end of class ZLBFAM + +#endif // __FILAMGZ_H diff --git a/storage/connect/filamzip.cpp b/storage/connect/filamzip.cpp index d9834e56dcd..65013e170e4 100644 --- a/storage/connect/filamzip.cpp +++ b/storage/connect/filamzip.cpp @@ -1,35 +1,29 @@ /*********** File AM Zip C++ Program Source Code File (.CPP) ***********/ /* PROGRAM NAME: FILAMZIP */ /* ------------- */ -/* Version 1.5 */ +/* Version 1.0 */ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 2005-2015 */ +/* (C) Copyright to the author Olivier BERTRAND 2016 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ -/* This program are the ZLIB compressed files classes. */ +/* This program are the ZIP file access method classes. */ /* */ /***********************************************************************/ /***********************************************************************/ -/* Include relevant MariaDB header file. */ +/* Include relevant sections of the System header files. */ /***********************************************************************/ #include "my_global.h" -#if defined(__WIN__) -#include <io.h> -#include <fcntl.h> -#if defined(__BORLANDC__) -#define __MFC_COMPAT__ // To define min/max as macro -#endif -//#include <windows.h> -#else // !__WIN__ +#if !defined(__WIN__) #if defined(UNIX) #include <errno.h> -#else // !UNIX +#include <unistd.h> +#else // !UNIX #include <io.h> -#endif +#endif // !UNIX #include <fcntl.h> #endif // !__WIN__ @@ -37,1390 +31,592 @@ /* Include application header files: */ /* global.h is header containing all global declarations. */ /* plgdbsem.h is header containing the DB application declarations. */ -/* tabdos.h is header containing the TABDOS class declarations. */ /***********************************************************************/ #include "global.h" #include "plgdbsem.h" -//#include "catalog.h" -//#include "reldef.h" -//#include "xobject.h" -//#include "kindex.h" -#include "filamtxt.h" -#include "tabdos.h" -#if defined(UNIX) #include "osutil.h" -#endif - -/***********************************************************************/ -/* This define prepares ZLIB function declarations. */ -/***********************************************************************/ -//#define ZLIB_DLL - +#include "filamtxt.h" +#include "tabfmt.h" +//#include "tabzip.h" #include "filamzip.h" -/***********************************************************************/ -/* DB static variables. */ -/***********************************************************************/ -extern int num_read, num_there, num_eq[]; // Statistics - -/* ------------------------------------------------------------------- */ +/* -------------------------- class ZIPUTIL -------------------------- */ /***********************************************************************/ -/* Implementation of the ZIPFAM class. */ +/* Constructors. */ /***********************************************************************/ -ZIPFAM::ZIPFAM(PZIPFAM txfp) : TXTFAM(txfp) - { - Zfile = txfp->Zfile; - Zpos = txfp->Zpos; - } // end of ZIPFAM copy constructor +ZIPUTIL::ZIPUTIL(PSZ tgt, bool mul) +{ + zipfile = NULL; + target = tgt; + fp = NULL; + memory = NULL; + size = 0; + entryopen = false; + multiple = mul; + memset(fn, 0, sizeof(fn)); + + // Init the case mapping table. +#if defined(__WIN__) + for (int i = 0; i < 256; ++i) mapCaseTable[i] = toupper(i); +#else + for (int i = 0; i < 256; ++i) mapCaseTable[i] = i; +#endif +} // end of ZIPUTIL standard constructor + +#if 0 +ZIPUTIL::ZIPUTIL(PZIPUTIL zutp) +{ + zipfile = zutp->zipfile; + target = zutp->target; + fp = zutp->fp; + finfo = zutp->finfo; + entryopen = zutp->entryopen; + multiple = zutp->multiple; + for (int i = 0; i < 256; ++i) mapCaseTable[i] = zutp->mapCaseTable[i]; +} // end of ZIPUTIL copy constructor +#endif // 0 /***********************************************************************/ -/* Zerror: Error function for gz calls. */ -/* gzerror returns the error message for the last error which occurred*/ -/* on the given compressed file. errnum is set to zlib error number. */ -/* If an error occurred in the file system and not in the compression */ -/* library, errnum is set to Z_ERRNO and the application may consult */ -/* errno to get the exact error code. */ +/* This code is the copyright property of Alessandro Felice Cantatore. */ +/* http://xoomer.virgilio.it/acantato/dev/wildcard/wildmatch.html */ /***********************************************************************/ -int ZIPFAM::Zerror(PGLOBAL g) - { - int errnum; +bool ZIPUTIL::WildMatch(PSZ pat, PSZ str) { + PSZ s, p; + bool star = FALSE; - strcpy(g->Message, gzerror(Zfile, &errnum)); +loopStart: + for (s = str, p = pat; *s; ++s, ++p) { + switch (*p) { + case '?': + if (*s == '.') goto starCheck; + break; + case '*': + star = TRUE; + str = s, pat = p; + if (!*++pat) return TRUE; + goto loopStart; + default: + if (mapCaseTable[*s] != mapCaseTable[*p]) + goto starCheck; + break; + } /* endswitch */ + } /* endfor */ + if (*p == '*') ++p; + return (!*p); - if (errnum == Z_ERRNO) -#if defined(__WIN__) - sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(NULL)); -#else // !__WIN__ - sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(errno)); -#endif // !__WIN__ - - return (errnum == Z_STREAM_END) ? RC_EF : RC_FX; - } // end of Zerror +starCheck: + if (!star) return FALSE; + str++; + goto loopStart; +} // end of WildMatch /***********************************************************************/ -/* Reset: reset position values at the beginning of file. */ +/* open a zip file. */ +/* param: filename path and the filename of the zip file to open. */ +/* return: true if open, false otherwise. */ /***********************************************************************/ -void ZIPFAM::Reset(void) - { - TXTFAM::Reset(); -//gzrewind(Zfile); // Useful ????? - Zpos = 0; - } // end of Reset +bool ZIPUTIL::open(PGLOBAL g, char *filename) +{ + if (!zipfile && !(zipfile = unzOpen64(filename))) + sprintf(g->Message, "Zipfile open error on %s", filename); + + return (zipfile == NULL); +} // end of open /***********************************************************************/ -/* ZIP GetFileLength: returns an estimate of what would be the */ -/* uncompressed file size in number of bytes. */ +/* Close the zip file. */ /***********************************************************************/ -int ZIPFAM::GetFileLength(PGLOBAL g) - { - int len = TXTFAM::GetFileLength(g); +void ZIPUTIL::close() +{ + if (zipfile) { + closeEntry(); + unzClose(zipfile); + zipfile = NULL; + } // endif zipfile - if (len > 0) - // Estimate size reduction to a max of 6 - len *= 6; - - return len; - } // end of GetFileLength +} // end of close /***********************************************************************/ -/* ZIP Access Method opening routine. */ +/* Find next entry matching target pattern. */ /***********************************************************************/ -bool ZIPFAM::OpenTableFile(PGLOBAL g) - { - char opmode[4], filename[_MAX_PATH]; - MODE mode = Tdbp->GetMode(); - - switch (mode) { - case MODE_READ: - strcpy(opmode, "r"); - break; - case MODE_UPDATE: - /*****************************************************************/ - /* Updating ZIP files not implemented yet. */ - /*****************************************************************/ - strcpy(g->Message, MSG(UPD_ZIP_NOT_IMP)); - return true; - case MODE_DELETE: - if (!Tdbp->GetNext()) { - // Store the number of deleted lines - DelRows = Cardinality(g); - - // This will erase the entire file - strcpy(opmode, "w"); -// Block = 0; // For ZBKFAM -// Last = Nrec; // For ZBKFAM - Tdbp->ResetSize(); - } else { - sprintf(g->Message, MSG(NO_PART_DEL), "ZIP"); - return true; - } // endif filter - - break; - case MODE_INSERT: - strcpy(opmode, "a+"); - break; - default: - sprintf(g->Message, MSG(BAD_OPEN_MODE), mode); - return true; - } // endswitch Mode - - /*********************************************************************/ - /* Open according to logical input/output mode required. */ - /* Use specific zlib functions. */ - /* Treat files as binary. */ - /*********************************************************************/ - strcat(opmode, "b"); - Zfile = gzopen(PlugSetPath(filename, To_File, Tdbp->GetPath()), opmode); - - if (Zfile == NULL) { - sprintf(g->Message, MSG(GZOPEN_ERROR), - opmode, (int)errno, filename); - strcat(strcat(g->Message, ": "), strerror(errno)); - return (mode == MODE_READ && errno == ENOENT) - ? PushWarning(g, Tdbp) : true; - } // endif Zfile - - /*********************************************************************/ - /* Something to be done here. >>>>>>>> NOT DONE <<<<<<<< */ - /*********************************************************************/ -//To_Fb = dbuserp->Openlist; // Keep track of File block - - /*********************************************************************/ - /* Allocate the line buffer. */ - /*********************************************************************/ - return AllocateBuffer(g); - } // end of OpenTableFile +int ZIPUTIL::findEntry(PGLOBAL g, bool next) +{ + int rc; -/***********************************************************************/ -/* Allocate the line buffer. For mode Delete a bigger buffer has to */ -/* be allocated because is it also used to move lines into the file. */ -/***********************************************************************/ -bool ZIPFAM::AllocateBuffer(PGLOBAL g) - { - MODE mode = Tdbp->GetMode(); + do { + if (next) { + rc = unzGoToNextFile(zipfile); - Buflen = Lrecl + 2; // Lrecl does not include CRLF -//Buflen *= ((Mode == MODE_DELETE) ? DOS_BUFF_LEN : 1); NIY + if (rc == UNZ_END_OF_LIST_OF_FILE) + return RC_EF; + else if (rc != UNZ_OK) { + sprintf(g->Message, "unzGoToNextFile rc = ", rc); + return RC_FX; + } // endif rc - if (trace) - htrc("SubAllocating a buffer of %d bytes\n", Buflen); + } // endif next - To_Buf = (char*)PlugSubAlloc(g, NULL, Buflen); + if (target && *target) { + rc = unzGetCurrentFileInfo(zipfile, NULL, fn, sizeof(fn), + NULL, 0, NULL, 0); + if (rc == UNZ_OK) { + if (WildMatch(target, fn)) + return RC_OK; - if (mode == MODE_INSERT) { - /*******************************************************************/ - /* For Insert buffer must be prepared. */ - /*******************************************************************/ - memset(To_Buf, ' ', Buflen); - To_Buf[Buflen - 2] = '\n'; - To_Buf[Buflen - 1] = '\0'; - } // endif Insert + } else { + sprintf(g->Message, "GetCurrentFileInfo rc = %d", rc); + return RC_FX; + } // endif rc - return false; - } // end of AllocateBuffer + } else + return RC_OK; -/***********************************************************************/ -/* GetRowID: return the RowID of last read record. */ -/***********************************************************************/ -int ZIPFAM::GetRowID(void) - { - return Rows; - } // end of GetRowID + next = true; + } while (true); -/***********************************************************************/ -/* GetPos: return the position of last read record. */ -/***********************************************************************/ -int ZIPFAM::GetPos(void) - { - return (int)Zpos; - } // end of GetPos + strcpy(g->Message, "FindNext logical error"); + return RC_FX; +} // end of FindEntry -/***********************************************************************/ -/* GetNextPos: return the position of next record. */ -/***********************************************************************/ -int ZIPFAM::GetNextPos(void) - { - return gztell(Zfile); - } // end of GetNextPos /***********************************************************************/ -/* SetPos: Replace the table at the specified position. */ +/* Get the next used entry. */ /***********************************************************************/ -bool ZIPFAM::SetPos(PGLOBAL g, int pos __attribute__((unused))) - { - sprintf(g->Message, MSG(NO_SETPOS_YET), "ZIP"); - return true; -#if 0 - Fpos = pos; +int ZIPUTIL::nextEntry(PGLOBAL g) +{ + if (multiple) { + int rc; - if (fseek(Stream, Fpos, SEEK_SET)) { - sprintf(g->Message, MSG(FSETPOS_ERROR), Fpos); - return true; - } // endif + closeEntry(); - Placed = true; - return false; -#endif // 0 - } // end of SetPos + if ((rc = findEntry(g, true)) != RC_OK) + return rc; -/***********************************************************************/ -/* Record file position in case of UPDATE or DELETE. */ -/***********************************************************************/ -bool ZIPFAM::RecordPos(PGLOBAL) - { - Zpos = gztell(Zfile); - return false; - } // end of RecordPos + if (openEntry(g)) + return RC_FX; -/***********************************************************************/ -/* Skip one record in file. */ -/***********************************************************************/ -int ZIPFAM::SkipRecord(PGLOBAL g, bool header) - { - // Skip this record - if (gzeof(Zfile)) - return RC_EF; - else if (gzgets(Zfile, To_Buf, Buflen) == Z_NULL) - return Zerror(g); + return RC_OK; + } else + return RC_EF; - if (header) - RecordPos(g); +} // end of nextEntry - return RC_OK; - } // end of SkipRecord /***********************************************************************/ -/* ReadBuffer: Read one line from a compressed text file. */ +/* OpenTableFile: Open a DOS/UNIX table file from a ZIP file. */ /***********************************************************************/ -int ZIPFAM::ReadBuffer(PGLOBAL g) - { - char *p; - int rc; - - if (!Zfile) - return RC_EF; - - if (!Placed) { - /*******************************************************************/ - /* Record file position in case of UPDATE or DELETE. */ - /*******************************************************************/ - next: - if (RecordPos(g)) - return RC_FX; - - CurBlk = Rows++; // Update RowID - - /*******************************************************************/ - /* Check whether optimization on ROWID */ - /* can be done, as well as for join as for local filtering. */ - /*******************************************************************/ - switch (Tdbp->TestBlock(g)) { - case RC_EF: - return RC_EF; - case RC_NF: - // Skip this record - if ((rc = SkipRecord(g, FALSE)) != RC_OK) - return rc; - - goto next; - } // endswitch rc - - } else - Placed = false; - - if (gzeof(Zfile)) { - rc = RC_EF; - } else if (gzgets(Zfile, To_Buf, Buflen) != Z_NULL) { - p = To_Buf + strlen(To_Buf) - 1; - - if (*p == '\n') - *p = '\0'; // Eliminate ending new-line character - - if (*(--p) == '\r') - *p = '\0'; // Eliminate eventuel carriage return - - strcpy(Tdbp->GetLine(), To_Buf); - IsRead = true; - rc = RC_OK; - num_read++; - } else - rc = Zerror(g); - - if (trace > 1) - htrc(" Read: '%s' rc=%d\n", To_Buf, rc); - - return rc; - } // end of ReadBuffer +bool ZIPUTIL::OpenTable(PGLOBAL g, MODE mode, char *fn) +{ + /*********************************************************************/ + /* The file will be decompressed into virtual memory. */ + /*********************************************************************/ + if (mode == MODE_READ || mode == MODE_ANY) { + bool b = open(g, fn); -/***********************************************************************/ -/* WriteDB: Data Base write routine for ZDOS access method. */ -/* Update is not possible without using a temporary file (NIY). */ -/***********************************************************************/ -int ZIPFAM::WriteBuffer(PGLOBAL g) - { - /*********************************************************************/ - /* Prepare the write buffer. */ - /*********************************************************************/ - strcat(strcpy(To_Buf, Tdbp->GetLine()), CrLf); - - /*********************************************************************/ - /* Now start the writing process. */ - /*********************************************************************/ - if (gzputs(Zfile, To_Buf) < 0) - return Zerror(g); - - return RC_OK; - } // end of WriteBuffer + if (!b) { + int rc; -/***********************************************************************/ -/* Data Base delete line routine for ZDOS access method. (NIY) */ -/***********************************************************************/ -int ZIPFAM::DeleteRecords(PGLOBAL g, int) - { - strcpy(g->Message, MSG(NO_ZIP_DELETE)); - return RC_FX; - } // end of DeleteRecords + if (target && *target) { + if (!multiple) { + rc = unzLocateFile(zipfile, target, 0); -/***********************************************************************/ -/* Data Base close routine for DOS access method. */ -/***********************************************************************/ -void ZIPFAM::CloseTableFile(PGLOBAL, bool) - { - int rc = gzclose(Zfile); + if (rc == UNZ_END_OF_LIST_OF_FILE) { + sprintf(g->Message, "Target file %s not in %s", target, fn); + return true; + } else if (rc != UNZ_OK) { + sprintf(g->Message, "unzLocateFile rc=%d", rc); + return true; + } // endif's rc - if (trace) - htrc("ZIP CloseDB: closing %s rc=%d\n", To_File, rc); + } else { + if ((rc = findEntry(g, false)) == RC_FX) + return true; + else if (rc == RC_EF) { + sprintf(g->Message, "No match of %s in %s", target, fn); + return true; + } // endif rc - Zfile = NULL; // So we can know whether table is open -//To_Fb->Count = 0; // Avoid double closing by PlugCloseAll - } // end of CloseTableFile + } // endif multiple -/***********************************************************************/ -/* Rewind routine for ZIP access method. */ -/***********************************************************************/ -void ZIPFAM::Rewind(void) - { - gzrewind(Zfile); - } // end of Rewind + } // endif target -/* ------------------------------------------------------------------- */ + if (openEntry(g)) + return true; -/***********************************************************************/ -/* Constructors. */ -/***********************************************************************/ -ZBKFAM::ZBKFAM(PDOSDEF tdp) : ZIPFAM(tdp) - { - Blocked = true; - Block = tdp->GetBlock(); - Last = tdp->GetLast(); - Nrec = tdp->GetElemt(); - CurLine = NULL; - NxtLine = NULL; - Closing = false; - BlkPos = tdp->GetTo_Pos(); - } // end of ZBKFAM standard constructor - -ZBKFAM::ZBKFAM(PZBKFAM txfp) : ZIPFAM(txfp) - { - CurLine = txfp->CurLine; - NxtLine = txfp->NxtLine; - Closing = txfp->Closing; - } // end of ZBKFAM copy constructor + if (size > 0) { + /*******************************************************************/ + /* Link a Fblock. This make possible to automatically close it */ + /* in case of error g->jump. */ + /*******************************************************************/ + PDBUSER dbuserp = (PDBUSER)g->Activityp->Aptr; -/***********************************************************************/ -/* Use BlockTest to reduce the table estimated size. */ -/***********************************************************************/ -int ZBKFAM::MaxBlkSize(PGLOBAL g, int) - { - int rc = RC_OK, savcur = CurBlk; - int size; - - // Roughly estimate the table size as the sum of blocks - // that can contain good rows - for (size = 0, CurBlk = 0; CurBlk < Block; CurBlk++) - if ((rc = Tdbp->TestBlock(g)) == RC_OK) - size += (CurBlk == Block - 1) ? Last : Nrec; - else if (rc == RC_EF) - break; - - CurBlk = savcur; - return size; - } // end of MaxBlkSize + fp = (PFBLOCK)PlugSubAlloc(g, NULL, sizeof(FBLOCK)); + fp->Type = TYPE_FB_ZIP; + fp->Fname = PlugDup(g, fn); + fp->Next = dbuserp->Openlist; + dbuserp->Openlist = fp; + fp->Count = 1; + fp->Length = size; + fp->Memory = memory; + fp->Mode = mode; + fp->File = this; + fp->Handle = NULL; + } // endif fp -/***********************************************************************/ -/* ZBK Cardinality: returns table cardinality in number of rows. */ -/* This function can be called with a null argument to test the */ -/* availability of Cardinality implementation (1 yes, 0 no). */ -/***********************************************************************/ -int ZBKFAM::Cardinality(PGLOBAL g) - { - return (g) ? (int)((Block - 1) * Nrec + Last) : 1; - } // end of Cardinality + } else + return true; -/***********************************************************************/ -/* Allocate the line buffer. For mode Delete a bigger buffer has to */ -/* be allocated because is it also used to move lines into the file. */ -/***********************************************************************/ -bool ZBKFAM::AllocateBuffer(PGLOBAL g) - { - Buflen = Nrec * (Lrecl + 2); - CurLine = To_Buf = (char*)PlugSubAlloc(g, NULL, Buflen); - - if (Tdbp->GetMode() == MODE_INSERT) { - // Set values so Block and Last can be recalculated - if (Last == Nrec) { - CurBlk = Block; - Rbuf = Nrec; // To be used by WriteDB - } else { - // The last block must be completed - CurBlk = Block - 1; - Rbuf = Nrec - Last; // To be used by WriteDB - } // endif Last - - } // endif Insert - - return false; - } // end of AllocateBuffer + } else { + strcpy(g->Message, "Only READ mode supported for ZIP files"); + return true; + } // endif mode -/***********************************************************************/ -/* GetRowID: return the RowID of last read record. */ -/***********************************************************************/ -int ZBKFAM::GetRowID(void) - { - return CurNum + Nrec * CurBlk + 1; - } // end of GetRowID + return false; +} // end of OpenTableFile /***********************************************************************/ -/* GetPos: return the position of last read record. */ +/* Open target in zip file. */ /***********************************************************************/ -int ZBKFAM::GetPos(void) - { - return CurNum + Nrec * CurBlk; // Computed file index - } // end of GetPos +bool ZIPUTIL::openEntry(PGLOBAL g) +{ + int rc; -/***********************************************************************/ -/* Record file position in case of UPDATE or DELETE. */ -/* Not used yet for fixed tables. */ -/***********************************************************************/ -bool ZBKFAM::RecordPos(PGLOBAL /*g*/) - { -//strcpy(g->Message, "RecordPos not implemented for zip blocked tables"); -//return true; - return RC_OK; - } // end of RecordPos + rc = unzGetCurrentFileInfo(zipfile, &finfo, fn, sizeof(fn), + NULL, 0, NULL, 0); -/***********************************************************************/ -/* Skip one record in file. */ -/***********************************************************************/ -int ZBKFAM::SkipRecord(PGLOBAL /*g*/, bool) - { -//strcpy(g->Message, "SkipRecord not implemented for zip blocked tables"); -//return RC_FX; - return RC_OK; - } // end of SkipRecord + if (rc != UNZ_OK) { + sprintf(g->Message, "unzGetCurrentFileInfo64 rc=%d", rc); + return true; + } else if ((rc = unzOpenCurrentFile(zipfile)) != UNZ_OK) { + sprintf(g->Message, "unzOpen fn=%s rc=%d", fn, rc); + return true; + } // endif rc -/***********************************************************************/ -/* ReadBuffer: Read one line from a compressed text file. */ -/***********************************************************************/ -int ZBKFAM::ReadBuffer(PGLOBAL g) - { - int n, skip, rc = RC_OK; - - /*********************************************************************/ - /* Sequential reading when Placed is not true. */ - /*********************************************************************/ - if (++CurNum < Rbuf) { - CurLine = NxtLine; - - // Get the position of the next line in the buffer - while (*NxtLine++ != '\n') ; - - // Set caller line buffer - n = NxtLine - CurLine - Ending; - memcpy(Tdbp->GetLine(), CurLine, n); - Tdbp->GetLine()[n] = '\0'; - return RC_OK; - } else if (Rbuf < Nrec && CurBlk != -1) - return RC_EF; - - /*********************************************************************/ - /* New block. */ - /*********************************************************************/ - CurNum = 0; - skip = 0; - - next: - if (++CurBlk >= Block) - return RC_EF; - - /*********************************************************************/ - /* Before using the new block, check whether block optimization */ - /* can be done, as well as for join as for local filtering. */ - /*********************************************************************/ - switch (Tdbp->TestBlock(g)) { - case RC_EF: - return RC_EF; - case RC_NF: - skip++; - goto next; - } // endswitch rc - - if (skip) - // Skip blocks rejected by block optimization - for (int i = CurBlk - skip; i < CurBlk; i++) { - BlkLen = BlkPos[i + 1] - BlkPos[i]; - - if (gzseek(Zfile, (z_off_t)BlkLen, SEEK_CUR) < 0) - return Zerror(g); - - } // endfor i - - BlkLen = BlkPos[CurBlk + 1] - BlkPos[CurBlk]; - - if (!(n = gzread(Zfile, To_Buf, BlkLen))) { - rc = RC_EF; - } else if (n > 0) { - // Get the position of the current line - CurLine = To_Buf; - - // Now get the position of the next line - for (NxtLine = CurLine; *NxtLine++ != '\n';) ; - - // Set caller line buffer - n = NxtLine - CurLine - Ending; - memcpy(Tdbp->GetLine(), CurLine, n); - Tdbp->GetLine()[n] = '\0'; - Rbuf = (CurBlk == Block - 1) ? Last : Nrec; - IsRead = true; - rc = RC_OK; - num_read++; - } else - rc = Zerror(g); - - return rc; - } // end of ReadBuffer + size = finfo.uncompressed_size; + memory = new char[size + 1]; -/***********************************************************************/ -/* WriteDB: Data Base write routine for ZDOS access method. */ -/* Update is not possible without using a temporary file (NIY). */ -/***********************************************************************/ -int ZBKFAM::WriteBuffer(PGLOBAL g) - { - /*********************************************************************/ - /* Prepare the write buffer. */ - /*********************************************************************/ - if (!Closing) - strcat(strcpy(CurLine, Tdbp->GetLine()), CrLf); - - /*********************************************************************/ - /* In Insert mode, blocs are added sequentialy to the file end. */ - /* Note: Update mode is not handled for zip files. */ - /*********************************************************************/ - if (++CurNum == Rbuf) { - /*******************************************************************/ - /* New block, start the writing process. */ - /*******************************************************************/ - BlkLen = CurLine + strlen(CurLine) - To_Buf; - - if (gzwrite(Zfile, To_Buf, BlkLen) != BlkLen || - gzflush(Zfile, Z_FULL_FLUSH)) { - Closing = true; - return Zerror(g); - } // endif gzwrite - - Rbuf = Nrec; - CurBlk++; - CurNum = 0; - CurLine = To_Buf; - } else - CurLine += strlen(CurLine); - - return RC_OK; - } // end of WriteBuffer + if ((rc = unzReadCurrentFile(zipfile, memory, size)) < 0) { + sprintf(g->Message, "unzReadCurrentFile rc = ", rc); + unzCloseCurrentFile(zipfile); + free(memory); + memory = NULL; + entryopen = false; + } else { + memory[size] = 0; // Required by some table types (XML) + entryopen = true; + } // endif rc + + if (trace) + htrc("Openning entry%s %s\n", fn, (entryopen) ? "oked" : "failed"); + + return !entryopen; +} // end of openEntry + +/***********************************************************************/ +/* Close the zip file. */ +/***********************************************************************/ +void ZIPUTIL::closeEntry() +{ + if (entryopen) { + unzCloseCurrentFile(zipfile); + entryopen = false; + } // endif entryopen + + if (memory) { + free(memory); + memory = NULL; + } // endif memory + +} // end of closeEntry + +/* -------------------------- class ZIPFAM --------------------------- */ /***********************************************************************/ -/* Data Base delete line routine for ZBK access method. */ -/* Implemented only for total deletion of the table, which is done */ -/* by opening the file in mode "wb". */ +/* Constructors. */ /***********************************************************************/ -int ZBKFAM::DeleteRecords(PGLOBAL g, int irc) - { - if (irc == RC_EF) { - LPCSTR name = Tdbp->GetName(); - PDOSDEF defp = (PDOSDEF)Tdbp->GetDef(); - - defp->SetBlock(0); - defp->SetLast(Nrec); +ZIPFAM::ZIPFAM(PDOSDEF tdp) : MAPFAM(tdp) +{ + zutp = NULL; + target = tdp->GetEntry(); + mul = tdp->GetMul(); +} // end of ZIPFAM standard constructor - if (!defp->SetIntCatInfo("Blocks", 0) || - !defp->SetIntCatInfo("Last", 0)) { - sprintf(g->Message, MSG(UPDATE_ERROR), "Header"); - return RC_FX; - } else - return RC_OK; +ZIPFAM::ZIPFAM(PZIPFAM txfp) : MAPFAM(txfp) +{ + zutp = txfp->zutp; + target = txfp->target; + mul = txfp->mul; +} // end of ZIPFAM copy constructor - } else - return irc; - - } // end of DeleteRecords +ZIPFAM::ZIPFAM(PDOSDEF tdp, PZPXFAM txfp) : MAPFAM(tdp) +{ + zutp = txfp->zutp; + target = txfp->target; + mul = txfp->mul; +} // end of ZIPFAM constructor used in ResetTableOpt /***********************************************************************/ -/* Data Base close routine for ZBK access method. */ +/* ZIP GetFileLength: returns file size in number of bytes. */ /***********************************************************************/ -void ZBKFAM::CloseTableFile(PGLOBAL g, bool) - { - int rc = RC_OK; - - if (Tdbp->GetMode() == MODE_INSERT) { - LPCSTR name = Tdbp->GetName(); - PDOSDEF defp = (PDOSDEF)Tdbp->GetDef(); - - if (CurNum && !Closing) { - // Some more inserted lines remain to be written - Last = (Nrec - Rbuf) + CurNum; - Block = CurBlk + 1; - Rbuf = CurNum--; - Closing = true; - rc = WriteBuffer(g); - } else if (Rbuf == Nrec) { - Last = Nrec; - Block = CurBlk; - } // endif CurNum - - if (rc != RC_FX) { - defp->SetBlock(Block); - defp->SetLast(Last); - defp->SetIntCatInfo("Blocks", Block); - defp->SetIntCatInfo("Last", Last); - } // endif - - gzclose(Zfile); - } else if (Tdbp->GetMode() == MODE_DELETE) { - rc = DeleteRecords(g, RC_EF); - gzclose(Zfile); - } else - rc = gzclose(Zfile); - - if (trace) - htrc("ZIP CloseDB: closing %s rc=%d\n", To_File, rc); - - Zfile = NULL; // So we can know whether table is open -//To_Fb->Count = 0; // Avoid double closing by PlugCloseAll - } // end of CloseTableFile +int ZIPFAM::GetFileLength(PGLOBAL g) +{ + int len = (zutp && zutp->entryopen) ? Top - Memory + : TXTFAM::GetFileLength(g) * 3; -/***********************************************************************/ -/* Rewind routine for ZBK access method. */ -/***********************************************************************/ -void ZBKFAM::Rewind(void) - { - gzrewind(Zfile); - CurBlk = -1; - CurNum = Rbuf; - } // end of Rewind + if (trace) + htrc("Zipped file length=%d\n", len); -/* ------------------------------------------------------------------- */ + return len; +} // end of GetFileLength /***********************************************************************/ -/* Constructors. */ +/* ZIP Cardinality: return the number of rows if possible. */ /***********************************************************************/ -ZIXFAM::ZIXFAM(PDOSDEF tdp) : ZBKFAM(tdp) - { -//Block = tdp->GetBlock(); -//Last = tdp->GetLast(); - Nrec = (tdp->GetElemt()) ? tdp->GetElemt() : DOS_BUFF_LEN; - Blksize = Nrec * Lrecl; - } // end of ZIXFAM standard constructor +int ZIPFAM::Cardinality(PGLOBAL g) +{ + if (!g) + return 1; -/***********************************************************************/ -/* ZIX Cardinality: returns table cardinality in number of rows. */ -/* This function can be called with a null argument to test the */ -/* availability of Cardinality implementation (1 yes, 0 no). */ -/***********************************************************************/ -int ZIXFAM::Cardinality(PGLOBAL g) - { - if (Last) - return (g) ? (int)((Block - 1) * Nrec + Last) : 1; - else // Last and Block not defined, cannot do it yet - return 0; + int card = -1; + int len = GetFileLength(g); - } // end of Cardinality + card = (len / (int)Lrecl) * 2; // Estimated ??? + return card; +} // end of Cardinality /***********************************************************************/ -/* Allocate the line buffer. For mode Delete a bigger buffer has to */ -/* be allocated because is it also used to move lines into the file. */ +/* OpenTableFile: Open a DOS/UNIX table file from a ZIP file. */ /***********************************************************************/ -bool ZIXFAM::AllocateBuffer(PGLOBAL g) - { - Buflen = Blksize; - To_Buf = (char*)PlugSubAlloc(g, NULL, Buflen); - - if (Tdbp->GetMode() == MODE_INSERT) { - /*******************************************************************/ - /* For Insert the buffer must be prepared. */ - /*******************************************************************/ - memset(To_Buf, ' ', Buflen); - - if (Tdbp->GetFtype() < 2) - // if not binary, the file is physically a text file - for (int len = Lrecl; len <= Buflen; len += Lrecl) { -#if defined(__WIN__) - To_Buf[len - 2] = '\r'; -#endif // __WIN__ - To_Buf[len - 1] = '\n'; - } // endfor len - - // Set values so Block and Last can be recalculated - if (Last == Nrec) { - CurBlk = Block; - Rbuf = Nrec; // To be used by WriteDB - } else { - // The last block must be completed - CurBlk = Block - 1; - Rbuf = Nrec - Last; // To be used by WriteDB - } // endif Last - - } // endif Insert - - return false; - } // end of AllocateBuffer +bool ZIPFAM::OpenTableFile(PGLOBAL g) +{ + char filename[_MAX_PATH]; + MODE mode = Tdbp->GetMode(); -/***********************************************************************/ -/* ReadBuffer: Read one line from a compressed text file. */ -/***********************************************************************/ -int ZIXFAM::ReadBuffer(PGLOBAL g) - { - int n, rc = RC_OK; - - /*********************************************************************/ - /* Sequential reading when Placed is not true. */ - /*********************************************************************/ - if (++CurNum < Rbuf) { - Tdbp->IncLine(Lrecl); // Used by DOSCOL functions - return RC_OK; - } else if (Rbuf < Nrec && CurBlk != -1) - return RC_EF; - - /*********************************************************************/ - /* New block. */ - /*********************************************************************/ - CurNum = 0; - Tdbp->SetLine(To_Buf); - - int skip = 0; - - next: - if (++CurBlk >= Block) - return RC_EF; - - /*********************************************************************/ - /* Before using the new block, check whether block optimization */ - /* can be done, as well as for join as for local filtering. */ - /*********************************************************************/ - switch (Tdbp->TestBlock(g)) { - case RC_EF: - return RC_EF; - case RC_NF: - skip++; - goto next; - } // endswitch rc - - if (skip) - // Skip blocks rejected by block optimization - for (int i = 0; i < skip; i++) { - if (gzseek(Zfile, (z_off_t)Buflen, SEEK_CUR) < 0) - return Zerror(g); - - } // endfor i - - if (!(n = gzread(Zfile, To_Buf, Buflen))) { - rc = RC_EF; - } else if (n > 0) { - Rbuf = n / Lrecl; - IsRead = true; - rc = RC_OK; - num_read++; - } else - rc = Zerror(g); - - return rc; - } // end of ReadBuffer + /*********************************************************************/ + /* Allocate the ZIP utility class. */ + /*********************************************************************/ + zutp = new(g) ZIPUTIL(target, mul); -/***********************************************************************/ -/* WriteDB: Data Base write routine for ZDOS access method. */ -/* Update is not possible without using a temporary file (NIY). */ -/***********************************************************************/ -int ZIXFAM::WriteBuffer(PGLOBAL g) - { - /*********************************************************************/ - /* In Insert mode, blocs are added sequentialy to the file end. */ - /* Note: Update mode is not handled for zip files. */ - /*********************************************************************/ - if (++CurNum == Rbuf) { - /*******************************************************************/ - /* New block, start the writing process. */ - /*******************************************************************/ - BlkLen = Rbuf * Lrecl; - - if (gzwrite(Zfile, To_Buf, BlkLen) != BlkLen || - gzflush(Zfile, Z_FULL_FLUSH)) { - Closing = true; - return Zerror(g); - } // endif gzwrite - - Rbuf = Nrec; - CurBlk++; - CurNum = 0; - Tdbp->SetLine(To_Buf); - } else - Tdbp->IncLine(Lrecl); // Used by FIXCOL functions - - return RC_OK; - } // end of WriteBuffer - -/* --------------------------- Class ZLBFAM -------------------------- */ + // We used the file name relative to recorded datapath + PlugSetPath(filename, To_File, Tdbp->GetPath()); -/***********************************************************************/ -/* Constructors. */ -/***********************************************************************/ -ZLBFAM::ZLBFAM(PDOSDEF tdp) : BLKFAM(tdp) - { - Zstream = NULL; - Zbuffer = NULL; - Zlenp = NULL; - Optimized = tdp->IsOptimized(); - } // end of ZLBFAM standard constructor - -ZLBFAM::ZLBFAM(PZLBFAM txfp) : BLKFAM(txfp) - { - Zstream = txfp->Zstream; - Zbuffer = txfp->Zbuffer; - Zlenp = txfp->Zlenp; - Optimized = txfp->Optimized; - } // end of ZLBFAM (dummy?) copy constructor + if (!zutp->OpenTable(g, mode, filename)) { + // The pseudo "buffer" is here the entire real buffer + Fpos = Mempos = Memory = zutp->memory; + Top = Memory + zutp->size; + To_Fb = zutp->fp; // Useful when closing + } else + return true; + + return false; + } // end of OpenTableFile /***********************************************************************/ -/* ZLB GetFileLength: returns an estimate of what would be the */ -/* uncompressed file size in number of bytes. */ +/* GetNext: go to next entry. */ /***********************************************************************/ -int ZLBFAM::GetFileLength(PGLOBAL g) - { - int len = (Optimized) ? BlkPos[Block] : BLKFAM::GetFileLength(g); +int ZIPFAM::GetNext(PGLOBAL g) +{ + int rc = zutp->nextEntry(g); - if (len > 0) - // Estimate size reduction to a max of 5 - len *= 5; + if (rc != RC_OK) + return rc; - return len; - } // end of GetFileLength + Mempos = Memory = zutp->memory; + Top = Memory + zutp->size; + return RC_OK; +} // end of GetNext +#if 0 /***********************************************************************/ -/* Allocate the line buffer. For mode Delete a bigger buffer has to */ -/* be allocated because is it also used to move lines into the file. */ +/* ReadBuffer: Read one line for a ZIP file. */ /***********************************************************************/ -bool ZLBFAM::AllocateBuffer(PGLOBAL g) - { - char *msg; - int n, zrc; +int ZIPFAM::ReadBuffer(PGLOBAL g) +{ + int rc, len; -#if 0 - if (!Optimized && Tdbp->NeedIndexing(g)) { - strcpy(g->Message, MSG(NOP_ZLIB_INDEX)); - return TRUE; - } // endif indexing -#endif // 0 + // Are we at the end of the memory + if (Mempos >= Top) { + if ((rc = zutp->nextEntry(g)) != RC_OK) + return rc; -#if defined(NOLIB) - if (!zlib && LoadZlib()) { - sprintf(g->Message, MSG(DLL_LOAD_ERROR), GetLastError(), "zlib.dll"); - return TRUE; - } // endif zlib -#endif + Mempos = Memory = zutp->memory; + Top = Memory + zutp->size; + } // endif Mempos - BLKFAM::AllocateBuffer(g); -//Buflen = Nrec * (Lrecl + 2); -//Rbuf = Nrec; - - // Allocate the compressed buffer - n = Buflen + 16; // ????????????????????????????????? - Zlenp = (int*)PlugSubAlloc(g, NULL, n); - Zbuffer = (Byte*)(Zlenp + 1); - - // Allocate and initialize the Z stream - Zstream = (z_streamp)PlugSubAlloc(g, NULL, sizeof(z_stream)); - Zstream->zalloc = (alloc_func)0; - Zstream->zfree = (free_func)0; - Zstream->opaque = (voidpf)0; - Zstream->next_in = NULL; - Zstream->avail_in = 0; - - if (Tdbp->GetMode() == MODE_READ) { - msg = "inflateInit"; - zrc = inflateInit(Zstream); - } else { - msg = "deflateInit"; - zrc = deflateInit(Zstream, Z_DEFAULT_COMPRESSION); - } // endif Mode - - if (zrc != Z_OK) { - if (Zstream->msg) - sprintf(g->Message, "%s error: %s", msg, Zstream->msg); - else - sprintf(g->Message, "%s error: %d", msg, zrc); - - return TRUE; - } // endif zrc - - if (Tdbp->GetMode() == MODE_INSERT) { - // Write the file header block - if (Last == Nrec) { - CurBlk = Block; - CurNum = 0; - - if (!GetFileLength(g)) { - // Write the zlib header as an extra block - strcpy(To_Buf, "PlugDB"); - BlkLen = strlen("PlugDB") + 1; - - if (WriteCompressedBuffer(g)) - return TRUE; - - } // endif void file - - } else { - // In mode insert, if Last != Nrec, last block must be updated - CurBlk = Block - 1; - CurNum = Last; - - strcpy(g->Message, MSG(NO_PAR_BLK_INS)); - return TRUE; - } // endif Last - - } else { // MODE_READ - // First thing to do is to read the header block - void *rdbuf; - - if (Optimized) { - BlkLen = BlkPos[0]; - rdbuf = Zlenp; - } else { - // Get the stored length from the file itself - if (fread(Zlenp, sizeof(int), 1, Stream) != 1) - return FALSE; // Empty file - - BlkLen = *Zlenp; - rdbuf = Zbuffer; - } // endif Optimized - - switch (ReadCompressedBuffer(g, rdbuf)) { - case RC_EF: - return FALSE; - case RC_FX: -#if defined(UNIX) - sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(errno)); +#if 0 + if (!Placed) { + /*******************************************************************/ + /* Record file position in case of UPDATE or DELETE. */ + /*******************************************************************/ + int rc; + + next: + Fpos = Mempos; + CurBlk = (int)Rows++; + + /*******************************************************************/ + /* Check whether optimization on ROWID */ + /* can be done, as well as for join as for local filtering. */ + /*******************************************************************/ + switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + // Skip this record + if ((rc = SkipRecord(g, false)) != RC_OK) + return rc; + + goto next; + } // endswitch rc + + } else + Placed = false; #else - sprintf(g->Message, MSG(READ_ERROR), To_File, _strerror(NULL)); + // Perhaps unuseful + Fpos = Mempos; + CurBlk = (int)Rows++; + Placed = false; #endif - case RC_NF: - return TRUE; - } // endswitch - // Some old tables can have PlugDB in their header - if (strcmp(To_Buf, "PlugDB")) { - sprintf(g->Message, MSG(BAD_HEADER), Tdbp->GetFile(g)); - return TRUE; - } // endif strcmp + // Immediately calculate next position (Used by DeleteDB) + while (*Mempos++ != '\n'); // What about Unix ??? - } // endif Mode + // Set caller line buffer + len = (Mempos - Fpos) - 1; - return FALSE; - } // end of AllocateBuffer + // Don't rely on ENDING setting + if (len > 0 && *(Mempos - 2) == '\r') + len--; // Line ends by CRLF -/***********************************************************************/ -/* GetPos: return the position of last read record. */ -/***********************************************************************/ -int ZLBFAM::GetPos(void) - { - return (Optimized) ? (CurNum + Nrec * CurBlk) : Fpos; - } // end of GetPos + memcpy(Tdbp->GetLine(), Fpos, len); + Tdbp->GetLine()[len] = '\0'; + return RC_OK; +} // end of ReadBuffer /***********************************************************************/ -/* GetNextPos: should not be called for this class. */ +/* Table file close routine for MAP access method. */ /***********************************************************************/ -int ZLBFAM::GetNextPos(void) - { - if (Optimized) { - assert(FALSE); - return 0; - } else - return ftell(Stream); +void ZIPFAM::CloseTableFile(PGLOBAL g, bool) +{ + close(); +} // end of CloseTableFile +#endif // 0 - } // end of GetNextPos +/* -------------------------- class ZPXFAM --------------------------- */ /***********************************************************************/ -/* SetPos: Replace the table at the specified position. */ +/* Constructors. */ /***********************************************************************/ -bool ZLBFAM::SetPos(PGLOBAL g, int pos __attribute__((unused))) - { - sprintf(g->Message, MSG(NO_SETPOS_YET), "ZIP"); - return true; -#if 0 // All this must be checked - if (pos < 0) { - strcpy(g->Message, MSG(INV_REC_POS)); - return true; - } // endif recpos - - CurBlk = pos / Nrec; - CurNum = pos % Nrec; -#if defined(_DEBUG) - num_eq[(CurBlk == OldBlk) ? 1 : 0]++; -#endif +ZPXFAM::ZPXFAM(PDOSDEF tdp) : MPXFAM(tdp) +{ + zutp = NULL; + target = tdp->GetEntry(); + mul = tdp->GetMul(); + //Lrecl = tdp->GetLrecl(); +} // end of ZPXFAM standard constructor - // Indicate the table position was externally set - Placed = true; - return false; -#endif // 0 - } // end of SetPos +ZPXFAM::ZPXFAM(PZPXFAM txfp) : MPXFAM(txfp) +{ + zutp = txfp->zutp; + target = txfp->target; + mul = txfp->mul; +//Lrecl = txfp->Lrecl; +} // end of ZPXFAM copy constructor /***********************************************************************/ -/* ReadBuffer: Read one line for a text file. */ +/* ZIP GetFileLength: returns file size in number of bytes. */ /***********************************************************************/ -int ZLBFAM::ReadBuffer(PGLOBAL g) - { - int n; - void *rdbuf; - - /*********************************************************************/ - /* Sequential reading when Placed is not true. */ - /*********************************************************************/ - if (Placed) { - Placed = FALSE; - } else if (++CurNum < Rbuf) { - CurLine = NxtLine; - - // Get the position of the next line in the buffer - if (Tdbp->GetFtype() == RECFM_VAR) - while (*NxtLine++ != '\n') ; - else - NxtLine += Lrecl; - - // Set caller line buffer - n = NxtLine - CurLine - ((Tdbp->GetFtype() == RECFM_BIN) ? 0 : Ending); - memcpy(Tdbp->GetLine(), CurLine, n); - Tdbp->GetLine()[n] = '\0'; - return RC_OK; - } else if (Rbuf < Nrec && CurBlk != -1) { - CurNum--; // To have a correct Last value when optimizing - return RC_EF; - } else { - /*******************************************************************/ - /* New block. */ - /*******************************************************************/ - CurNum = 0; - - next: - if (++CurBlk >= Block) - return RC_EF; - - /*******************************************************************/ - /* Before reading a new block, check whether block optimization */ - /* can be done, as well as for join as for local filtering. */ - /*******************************************************************/ - if (Optimized) switch (Tdbp->TestBlock(g)) { - case RC_EF: - return RC_EF; - case RC_NF: - goto next; - } // endswitch rc - - } // endif's - - if (OldBlk == CurBlk) - goto ok; // Block is already there - - if (Optimized) { - // Store the position of next block - Fpos = BlkPos[CurBlk]; - - // fseek is required only in non sequential reading - if (CurBlk != OldBlk + 1) - if (fseek(Stream, Fpos, SEEK_SET)) { - sprintf(g->Message, MSG(FSETPOS_ERROR), Fpos); - return RC_FX; - } // endif fseek - - // Calculate the length of block to read - BlkLen = BlkPos[CurBlk + 1] - Fpos; - rdbuf = Zlenp; - } else { // !Optimized - if (CurBlk != OldBlk + 1) { - strcpy(g->Message, MSG(INV_RAND_ACC)); - return RC_FX; - } else - Fpos = ftell(Stream); // Used when optimizing - - // Get the stored length from the file itself - if (fread(Zlenp, sizeof(int), 1, Stream) != 1) { - if (feof(Stream)) - return RC_EF; - - goto err; - } // endif fread - - BlkLen = *Zlenp; - rdbuf = Zbuffer; - } // endif Optimized - - // Read the next block - switch (ReadCompressedBuffer(g, rdbuf)) { - case RC_FX: goto err; - case RC_NF: return RC_FX; - case RC_EF: return RC_EF; - default: Rbuf = (CurBlk == Block - 1) ? Last : Nrec; - } // endswitch ReadCompressedBuffer - - ok: - if (Tdbp->GetFtype() == RECFM_VAR) { - int i; - - // Get the position of the current line - for (i = 0, CurLine = To_Buf; i < CurNum; i++) - while (*CurLine++ != '\n') ; // What about Unix ??? - - // Now get the position of the next line - for (NxtLine = CurLine; *NxtLine++ != '\n';) ; - - // Set caller line buffer - n = NxtLine - CurLine - Ending; - } else { - CurLine = To_Buf + CurNum * Lrecl; - NxtLine = CurLine + Lrecl; - n = Lrecl - ((Tdbp->GetFtype() == RECFM_BIN) ? 0 : Ending); - } // endif Ftype - - memcpy(Tdbp->GetLine(), CurLine, n); - Tdbp->GetLine()[n] = '\0'; - - OldBlk = CurBlk; // Last block actually read - IsRead = TRUE; // Is read indeed - return RC_OK; - - err: -#if defined(UNIX) - sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(errno)); -#else - sprintf(g->Message, MSG(READ_ERROR), To_File, _strerror(NULL)); -#endif - return RC_FX; - } // end of ReadBuffer +int ZPXFAM::GetFileLength(PGLOBAL g) +{ + int len; -/***********************************************************************/ -/* Read and decompress a block from the stream. */ -/***********************************************************************/ -int ZLBFAM::ReadCompressedBuffer(PGLOBAL g, void *rdbuf) - { - if (fread(rdbuf, 1, (size_t)BlkLen, Stream) == (unsigned)BlkLen) { - int zrc; - - num_read++; - - if (Optimized && BlkLen != signed(*Zlenp + sizeof(int))) { - sprintf(g->Message, MSG(BAD_BLK_SIZE), CurBlk + 1); - return RC_NF; - } // endif BlkLen - - // HERE WE MUST INFLATE THE BLOCK - Zstream->next_in = Zbuffer; - Zstream->avail_in = (uInt)(*Zlenp); - Zstream->next_out = (Byte*)To_Buf; - Zstream->avail_out = Buflen; - zrc = inflate(Zstream, Z_SYNC_FLUSH); - - if (zrc != Z_OK) { - if (Zstream->msg) - sprintf(g->Message, MSG(FUNC_ERR_S), "inflate", Zstream->msg); - else - sprintf(g->Message, MSG(FUNCTION_ERROR), "inflate", (int)zrc); - - return RC_NF; - } // endif zrc - - } else if (feof(Stream)) { - return RC_EF; - } else - return RC_FX; - - return RC_OK; - } // end of ReadCompressedBuffer + if (!zutp && OpenTableFile(g)) + return 0; -/***********************************************************************/ -/* WriteBuffer: File write routine for DOS access method. */ -/* Update is directly written back into the file, */ -/* with this (fast) method, record size cannot change. */ -/***********************************************************************/ -int ZLBFAM::WriteBuffer(PGLOBAL g) - { - assert (Tdbp->GetMode() == MODE_INSERT); - - /*********************************************************************/ - /* Prepare the write buffer. */ - /*********************************************************************/ - if (!Closing) { - if (Tdbp->GetFtype() == RECFM_BIN) - memcpy(CurLine, Tdbp->GetLine(), Lrecl); - else - strcat(strcpy(CurLine, Tdbp->GetLine()), CrLf); - -#if defined(_DEBUG) - if (Tdbp->GetFtype() == RECFM_FIX && - (signed)strlen(CurLine) != Lrecl + (signed)strlen(CrLf)) { - strcpy(g->Message, MSG(BAD_LINE_LEN)); - Closing = TRUE; - return RC_FX; - } // endif Lrecl -#endif // _DEBUG - } // endif Closing - - /*********************************************************************/ - /* In Insert mode, blocs are added sequentialy to the file end. */ - /*********************************************************************/ - if (++CurNum != Rbuf) { - if (Tdbp->GetFtype() == RECFM_VAR) - CurLine += strlen(CurLine); - else - CurLine += Lrecl; - - return RC_OK; // We write only full blocks - } // endif CurNum - - // HERE WE MUST DEFLATE THE BLOCK - if (Tdbp->GetFtype() == RECFM_VAR) - NxtLine = CurLine + strlen(CurLine); - else - NxtLine = CurLine + Lrecl; - - BlkLen = NxtLine - To_Buf; - - if (WriteCompressedBuffer(g)) { - Closing = TRUE; // To tell CloseDB about a Write error - return RC_FX; - } // endif WriteCompressedBuffer - - CurBlk++; - CurNum = 0; - CurLine = To_Buf; - return RC_OK; - } // end of WriteBuffer + if (zutp->entryopen) + len = zutp->size; + else + len = 0; + + return len; +} // end of GetFileLength /***********************************************************************/ -/* Compress the buffer and write the deflated output to stream. */ +/* ZIP Cardinality: return the number of rows if possible. */ /***********************************************************************/ -bool ZLBFAM::WriteCompressedBuffer(PGLOBAL g) - { - int zrc; - - Zstream->next_in = (Byte*)To_Buf; - Zstream->avail_in = (uInt)BlkLen; - Zstream->next_out = Zbuffer; - Zstream->avail_out = Buflen + 16; - Zstream->total_out = 0; - zrc = deflate(Zstream, Z_FULL_FLUSH); - - if (zrc != Z_OK) { - if (Zstream->msg) - sprintf(g->Message, MSG(FUNC_ERR_S), "deflate", Zstream->msg); - else - sprintf(g->Message, MSG(FUNCTION_ERROR), "deflate", (int)zrc); - - return TRUE; - } else - *Zlenp = Zstream->total_out; - - // Now start the writing process. - BlkLen = *Zlenp + sizeof(int); - - if (fwrite(Zlenp, 1, BlkLen, Stream) != (size_t)BlkLen) { - sprintf(g->Message, MSG(FWRITE_ERROR), strerror(errno)); - return TRUE; - } // endif size - - return FALSE; - } // end of WriteCompressedBuffer +int ZPXFAM::Cardinality(PGLOBAL g) +{ + if (!g) + return 1; + + int card = -1; + int len = GetFileLength(g); + + if (!(len % Lrecl)) + card = len / (int)Lrecl; // Fixed length file + else + sprintf(g->Message, MSG(NOT_FIXED_LEN), zutp->fn, len, Lrecl); + + // Set number of blocks for later use + Block = (card > 0) ? (card + Nrec - 1) / Nrec : 0; + return card; +} // end of Cardinality /***********************************************************************/ -/* Table file close routine for DOS access method. */ +/* OpenTableFile: Open a DOS/UNIX table file from a ZIP file. */ /***********************************************************************/ -void ZLBFAM::CloseTableFile(PGLOBAL g, bool) - { - int rc = RC_OK; - - if (Tdbp->GetMode() == MODE_INSERT) { - LPCSTR name = Tdbp->GetName(); - PDOSDEF defp = (PDOSDEF)Tdbp->GetDef(); - - // Closing is True if last Write was in error - if (CurNum && !Closing) { - // Some more inserted lines remain to be written - Last = (Nrec - Rbuf) + CurNum; - Block = CurBlk + 1; - Rbuf = CurNum--; - Closing = TRUE; - rc = WriteBuffer(g); - } else if (Rbuf == Nrec) { - Last = Nrec; - Block = CurBlk; - } // endif CurNum - - if (rc != RC_FX) { - defp->SetBlock(Block); - defp->SetLast(Last); - defp->SetIntCatInfo("Blocks", Block); - defp->SetIntCatInfo("Last", Last); - } // endif - - fclose(Stream); - } else - rc = fclose(Stream); - - if (trace) - htrc("ZLB CloseTableFile: closing %s mode=%d rc=%d\n", - To_File, Tdbp->GetMode(), rc); - - Stream = NULL; // So we can know whether table is open - To_Fb->Count = 0; // Avoid double closing by PlugCloseAll - - if (Tdbp->GetMode() == MODE_READ) - rc = inflateEnd(Zstream); - else - rc = deflateEnd(Zstream); - - } // end of CloseTableFile +bool ZPXFAM::OpenTableFile(PGLOBAL g) +{ + // May have been already opened in GetFileLength + if (!zutp || !zutp->zipfile) { + char filename[_MAX_PATH]; + MODE mode = Tdbp->GetMode(); + + /*********************************************************************/ + /* Allocate the ZIP utility class. */ + /*********************************************************************/ + if (!zutp) + zutp = new(g)ZIPUTIL(target, mul); + + // We used the file name relative to recorded datapath + PlugSetPath(filename, To_File, Tdbp->GetPath()); + + if (!zutp->OpenTable(g, mode, filename)) { + // The pseudo "buffer" is here the entire real buffer + Memory = zutp->memory; + Fpos = Mempos = Memory + Headlen; + Top = Memory + zutp->size; + To_Fb = zutp->fp; // Useful when closing + } else + return true; + + } else + Reset(); + + return false; +} // end of OpenTableFile /***********************************************************************/ -/* Rewind routine for ZLIB access method. */ +/* GetNext: go to next entry. */ /***********************************************************************/ -void ZLBFAM::Rewind(void) - { - // We must be positioned after the header block - if (CurBlk >= 0) { // Nothing to do if no block read yet - if (!Optimized) { // If optimized, fseek will be done in ReadBuffer - size_t st; - - rewind(Stream); +int ZPXFAM::GetNext(PGLOBAL g) +{ + int rc = zutp->nextEntry(g); - if (!(st = fread(Zlenp, sizeof(int), 1, Stream)) && trace) - htrc("fread error %d in Rewind", errno); + if (rc != RC_OK) + return rc; - fseek(Stream, *Zlenp + sizeof(int), SEEK_SET); - OldBlk = -1; - } // endif Optimized + int len = zutp->size; - CurBlk = -1; - CurNum = Rbuf; - } // endif CurBlk + if (len % Lrecl) { + sprintf(g->Message, MSG(NOT_FIXED_LEN), zutp->fn, len, Lrecl); + return RC_FX; + } // endif size -//OldBlk = -1; -//Rbuf = 0; commented out in case we reuse last read block - } // end of Rewind + Memory = zutp->memory; + Top = Memory + len; + Rewind(); + return RC_OK; +} // end of GetNext -/* ------------------------ End of ZipFam ---------------------------- */ diff --git a/storage/connect/filamzip.h b/storage/connect/filamzip.h index edb8b5db323..9312fb2f70e 100644 --- a/storage/connect/filamzip.h +++ b/storage/connect/filamzip.h @@ -1,170 +1,117 @@ -/************** FilAmZip H Declares Source Code File (.H) **************/ -/* Name: FILAMZIP.H Version 1.2 */ +/************** filamzip H Declares Source Code File (.H) **************/ +/* Name: filamzip.h Version 1.0 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 2005-2014 */ +/* (C) Copyright to the author Olivier BERTRAND 2016 */ /* */ -/* This file contains the GZIP access method classes declares. */ +/* This file contains the ZIP file access method classes declares. */ /***********************************************************************/ #ifndef __FILAMZIP_H #define __FILAMZIP_H -#include "zlib.h" +#include "block.h" +#include "filamap.h" +#include "unzip.h" -typedef class ZIPFAM *PZIPFAM; -typedef class ZBKFAM *PZBKFAM; -typedef class ZIXFAM *PZIXFAM; -typedef class ZLBFAM *PZLBFAM; +#define DLLEXPORT extern "C" -/***********************************************************************/ -/* This is the access method class declaration for not optimized */ -/* variable record length files compressed using the gzip library */ -/* functions. File is accessed record by record (row). */ -/***********************************************************************/ -class DllExport ZIPFAM : public TXTFAM { -// friend class DOSCOL; - public: - // Constructor - ZIPFAM(PDOSDEF tdp) : TXTFAM(tdp) {Zfile = NULL; Zpos = 0;} - ZIPFAM(PZIPFAM txfp); - - // Implementation - virtual AMT GetAmType(void) {return TYPE_AM_ZIP;} - virtual int GetPos(void); - virtual int GetNextPos(void); - virtual PTXF Duplicate(PGLOBAL g) - {return (PTXF)new(g) ZIPFAM(this);} - - // Methods - virtual void Reset(void); - virtual int GetFileLength(PGLOBAL g); - virtual int Cardinality(PGLOBAL g) {return (g) ? -1 : 0;} - virtual int MaxBlkSize(PGLOBAL g, int s) {return s;} - virtual bool AllocateBuffer(PGLOBAL g); - virtual int GetRowID(void); - virtual bool RecordPos(PGLOBAL g); - virtual bool SetPos(PGLOBAL g, int recpos); - virtual int SkipRecord(PGLOBAL g, bool header); - virtual bool OpenTableFile(PGLOBAL g); - virtual int ReadBuffer(PGLOBAL g); - virtual int WriteBuffer(PGLOBAL g); - virtual int DeleteRecords(PGLOBAL g, int irc); - virtual void CloseTableFile(PGLOBAL g, bool abort); - virtual void Rewind(void); - - protected: - int Zerror(PGLOBAL g); // GZ error function - - // Members - gzFile Zfile; // Points to GZ file structure - z_off_t Zpos; // Uncompressed file position - }; // end of class ZIPFAM +typedef class ZIPFAM *PZIPFAM; +typedef class ZPXFAM *PZPXFAM; /***********************************************************************/ -/* This is the access method class declaration for optimized variable */ -/* record length files compressed using the gzip library functions. */ -/* The File is accessed by block (requires an opt file). */ +/* This is the ZIP utility fonctions class. */ /***********************************************************************/ -class DllExport ZBKFAM : public ZIPFAM { - public: - // Constructor - ZBKFAM(PDOSDEF tdp); - ZBKFAM(PZBKFAM txfp); - - // Implementation - virtual int GetPos(void); - virtual int GetNextPos(void) {return 0;} - virtual PTXF Duplicate(PGLOBAL g) - {return (PTXF)new(g) ZBKFAM(this);} - - // Methods - virtual int Cardinality(PGLOBAL g); - virtual int MaxBlkSize(PGLOBAL g, int s); - virtual bool AllocateBuffer(PGLOBAL g); - virtual int GetRowID(void); - virtual bool RecordPos(PGLOBAL g); - virtual int SkipRecord(PGLOBAL g, bool header); - virtual int ReadBuffer(PGLOBAL g); - virtual int WriteBuffer(PGLOBAL g); - virtual int DeleteRecords(PGLOBAL g, int irc); - virtual void CloseTableFile(PGLOBAL g, bool abort); - virtual void Rewind(void); - - protected: - // Members - char *CurLine; // Position of current line in buffer - char *NxtLine; // Position of Next line in buffer - bool Closing; // True when closing on Insert - }; // end of class ZBKFAM +class DllExport ZIPUTIL : public BLOCK { +public: + // Constructor + ZIPUTIL(PSZ tgt, bool mul); +//ZIPUTIL(ZIPUTIL *zutp); + + // Implementation +//PTXF Duplicate(PGLOBAL g) { return (PTXF) new(g)ZIPFAM(this); } + + // Methods + virtual bool OpenTable(PGLOBAL g, MODE mode, char *fn); + bool open(PGLOBAL g, char *fn); + bool openEntry(PGLOBAL g); + void close(void); + void closeEntry(void); + bool WildMatch(PSZ pat, PSZ str); + int findEntry(PGLOBAL g, bool next); + int nextEntry(PGLOBAL g); + + // Members + unzFile zipfile; // The ZIP container file + PSZ target; // The target file name + unz_file_info finfo; // The current file info + PFBLOCK fp; + char *memory; + uint size; + int multiple; // Multiple targets + bool entryopen; // True when open current entry + char fn[FILENAME_MAX]; // The current entry file name + char mapCaseTable[256]; +}; // end of ZIPFAM /***********************************************************************/ -/* This is the access method class declaration for fixed record */ -/* length files compressed using the gzip library functions. */ -/* The file is always accessed by block. */ +/* This is the ZIP file access method. */ /***********************************************************************/ -class DllExport ZIXFAM : public ZBKFAM { - public: - // Constructor - ZIXFAM(PDOSDEF tdp); - ZIXFAM(PZIXFAM txfp) : ZBKFAM(txfp) {} - - // Implementation - virtual int GetNextPos(void) {return 0;} - virtual PTXF Duplicate(PGLOBAL g) - {return (PTXF)new(g) ZIXFAM(this);} - - // Methods - virtual int Cardinality(PGLOBAL g); - virtual bool AllocateBuffer(PGLOBAL g); - virtual int ReadBuffer(PGLOBAL g); - virtual int WriteBuffer(PGLOBAL g); - - protected: - // No additional Members - }; // end of class ZIXFAM +class DllExport ZIPFAM : public MAPFAM { + friend class ZPXFAM; +public: + // Constructors + ZIPFAM(PDOSDEF tdp); + ZIPFAM(PZIPFAM txfp); + ZIPFAM(PDOSDEF tdp, PZPXFAM txfp); + + // Implementation + virtual AMT GetAmType(void) { return TYPE_AM_ZIP; } + virtual PTXF Duplicate(PGLOBAL g) { return (PTXF) new(g)ZIPFAM(this); } + + // Methods + virtual int Cardinality(PGLOBAL g); + virtual int GetFileLength(PGLOBAL g); +//virtual int MaxBlkSize(PGLOBAL g, int s) {return s;} + virtual bool OpenTableFile(PGLOBAL g); + virtual bool DeferReading(void) { return false; } + virtual int GetNext(PGLOBAL g); +//virtual int ReadBuffer(PGLOBAL g); +//virtual int WriteBuffer(PGLOBAL g); +//virtual int DeleteRecords(PGLOBAL g, int irc); +//virtual void CloseTableFile(PGLOBAL g, bool abort); + +protected: + // Members + ZIPUTIL *zutp; + PSZ target; + bool mul; +}; // end of ZIPFAM /***********************************************************************/ -/* This is the DOS/UNIX Access Method class declaration for PlugDB */ -/* fixed/variable files compressed using the zlib library functions. */ -/* Physically these are written and read using the same technique */ -/* than blocked variable files, only the contain of each block is */ -/* compressed using the deflate zlib function. The purpose of this */ -/* specific format is to have a fast mechanism for direct access of */ -/* records so blocked optimization is fast and direct access (joins) */ -/* is allowed. Note that the block length is written ahead of each */ -/* block to enable reading when optimization file is not available. */ +/* This is the fixed ZIP file access method. */ /***********************************************************************/ -class DllExport ZLBFAM : public BLKFAM { - public: - // Constructor - ZLBFAM(PDOSDEF tdp); - ZLBFAM(PZLBFAM txfp); - - // Implementation - virtual AMT GetAmType(void) {return TYPE_AM_ZLIB;} - virtual int GetPos(void); - virtual int GetNextPos(void); - virtual PTXF Duplicate(PGLOBAL g) - {return (PTXF)new(g) ZLBFAM(this);} - inline void SetOptimized(bool b) {Optimized = b;} - - // Methods - virtual int GetFileLength(PGLOBAL g); - virtual bool SetPos(PGLOBAL g, int recpos); - virtual bool AllocateBuffer(PGLOBAL g); - virtual int ReadBuffer(PGLOBAL g); - virtual int WriteBuffer(PGLOBAL g); - virtual void CloseTableFile(PGLOBAL g, bool abort); - virtual void Rewind(void); - - protected: - bool WriteCompressedBuffer(PGLOBAL g); - int ReadCompressedBuffer(PGLOBAL g, void *rdbuf); - - // Members - z_streamp Zstream; // Compression/decompression stream - Byte *Zbuffer; // Compressed block buffer - int *Zlenp; // Pointer to block length - bool Optimized; // true when opt file is available - }; // end of class ZLBFAM +class DllExport ZPXFAM : public MPXFAM { + friend class ZIPFAM; +public: + // Constructors + ZPXFAM(PDOSDEF tdp); + ZPXFAM(PZPXFAM txfp); + + // Implementation + virtual AMT GetAmType(void) { return TYPE_AM_ZIP; } + virtual PTXF Duplicate(PGLOBAL g) { return (PTXF) new(g)ZPXFAM(this); } + + // Methods + virtual int GetFileLength(PGLOBAL g); + virtual int Cardinality(PGLOBAL g); + virtual bool OpenTableFile(PGLOBAL g); + virtual int GetNext(PGLOBAL g); +//virtual int ReadBuffer(PGLOBAL g); + +protected: + // Members + ZIPUTIL *zutp; + PSZ target; + bool mul; +}; // end of ZPXFAM #endif // __FILAMZIP_H diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index cf945a73f46..33f02d8338c 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -171,9 +171,9 @@ #define JSONMAX 10 // JSON Default max grp size extern "C" { - char version[]= "Version 1.04.0008 August 10, 2016"; + char version[]= "Version 1.05.0001 December 13, 2016"; #if defined(__WIN__) - char compver[]= "Version 1.04.0008 " __DATE__ " " __TIME__; + char compver[]= "Version 1.05.0001 " __DATE__ " " __TIME__; char slash= '\\'; #else // !__WIN__ char slash= '/'; @@ -512,13 +512,13 @@ ha_create_table_option connect_table_option_list[]= HA_TOPTION_NUMBER("QUOTED", quoted, (ulonglong) -1, 0, 3, 1), HA_TOPTION_NUMBER("ENDING", ending, (ulonglong) -1, 0, INT_MAX32, 1), HA_TOPTION_NUMBER("COMPRESS", compressed, 0, 0, 2, 1), -//HA_TOPTION_BOOL("COMPRESS", compressed, 0), HA_TOPTION_BOOL("MAPPED", mapped, 0), HA_TOPTION_BOOL("HUGE", huge, 0), HA_TOPTION_BOOL("SPLIT", split, 0), HA_TOPTION_BOOL("READONLY", readonly, 0), HA_TOPTION_BOOL("SEPINDEX", sepindex, 0), - HA_TOPTION_END + HA_TOPTION_BOOL("ZIPPED", zipped, 0), + HA_TOPTION_END }; @@ -532,7 +532,6 @@ ha_create_table_option connect_field_option_list[]= { HA_FOPTION_NUMBER("FLAG", offset, (ulonglong) -1, 0, INT_MAX32, 1), HA_FOPTION_NUMBER("MAX_DIST", freq, 0, 0, INT_MAX32, 1), // BLK_INDX -//HA_FOPTION_NUMBER("DISTRIB", opt, 0, 0, 2, 1), // used for BLK_INDX HA_FOPTION_NUMBER("FIELD_LENGTH", fldlen, 0, 0, INT_MAX32, 1), HA_FOPTION_STRING("DATE_FORMAT", dateformat), HA_FOPTION_STRING("FIELD_FORMAT", fieldformat), @@ -678,7 +677,6 @@ static int connect_init_func(void *p) connect_hton= (handlerton *)p; connect_hton->state= SHOW_OPTION_YES; connect_hton->create= connect_create_handler; -//connect_hton->flags= HTON_TEMPORARY_NOT_SUPPORTED | HTON_NO_PARTITION; connect_hton->flags= HTON_TEMPORARY_NOT_SUPPORTED; connect_hton->table_options= connect_table_option_list; connect_hton->field_options= connect_field_option_list; @@ -1135,7 +1133,9 @@ bool GetBooleanTableOption(PGLOBAL g, PTOS options, char *opname, bool bdef) opval= options->sepindex; else if (!stricmp(opname, "Header")) opval= (options->header != 0); // Is Boolean for some table types - else if (options->oplist) + else if (!stricmp(opname, "Zipped")) + opval = options->zipped; + else if (options->oplist) if ((pv= GetListOption(g, opname, options->oplist))) opval= (!*pv || *pv == 'y' || *pv == 'Y' || atoi(pv) != 0); @@ -1242,8 +1242,10 @@ char *ha_connect::GetStringOption(char *opname, char *sdef) if (opval && (!stricmp(opname, "connect") || !stricmp(opname, "tabname") - || !stricmp(opname, "filename"))) - opval = GetRealString(opval); + || !stricmp(opname, "filename") + || !stricmp(opname, "optname") + || !stricmp(opname, "entry"))) + opval = GetRealString(opval); if (!opval) { if (sdef && !strcmp(sdef, "*")) { @@ -4165,7 +4167,8 @@ bool ha_connect::check_privileges(THD *thd, PTOS options, char *dbn, bool quick) case TAB_DIR: case TAB_MAC: case TAB_WMI: - case TAB_OEM: + case TAB_ZIP: + case TAB_OEM: #ifdef NO_EMBEDDED_ACCESS_CHECKS return false; #endif @@ -5173,13 +5176,13 @@ static int connect_assisted_discovery(handlerton *, THD* thd, char v=0, spc= ',', qch= 0; const char *fncn= "?"; const char *user, *fn, *db, *host, *pwd, *sep, *tbl, *src; - const char *col, *ocl, *rnk, *pic, *fcl, *skc; + const char *col, *ocl, *rnk, *pic, *fcl, *skc, *zfn; char *tab, *dsn, *shm, *dpath; #if defined(__WIN__) char *nsp= NULL, *cls= NULL; #endif // __WIN__ - int port= 0, hdr= 0, mxr= 0, mxe= 0, rc= 0; - int cop __attribute__((unused))= 0, lrecl= 0; +//int hdr, mxe; + int port = 0, mxr = 0, rc = 0, mul = 0, lrecl = 0; #if defined(ODBC_SUPPORT) POPARM sop= NULL; char *ucnc= NULL; @@ -5190,7 +5193,8 @@ static int connect_assisted_discovery(handlerton *, THD* thd, PJPARM sjp= NULL; char *driver= NULL; char *url= NULL; - char *tabtyp = NULL; +//char *prop= NULL; + char *tabtyp= NULL; #endif // JDBC_SUPPORT uint tm, fnc= FNC_NO, supfnc= (FNC_NO | FNC_COL); bool bif, ok= false, dbf= false; @@ -5210,7 +5214,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, if (!g) return HA_ERR_INTERNAL_ERROR; - user= host= pwd= tbl= src= col= ocl= pic= fcl= skc= rnk= dsn= NULL; + user= host= pwd= tbl= src= col= ocl= pic= fcl= skc= rnk= zfn= dsn= NULL; // Get the useful create options ttp= GetTypeID(topt->type); @@ -5223,7 +5227,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, sep= topt->separator; spc= (!sep) ? ',' : *sep; qch= topt->qchar ? *topt->qchar : (signed)topt->quoted >= 0 ? '"' : 0; - hdr= (int)topt->header; + mul = (int)topt->multiple; tbl= topt->tablist; col= topt->colist; @@ -5256,13 +5260,16 @@ static int connect_assisted_discovery(handlerton *, THD* thd, #if defined(JDBC_SUPPORT) driver= GetListOption(g, "Driver", topt->oplist, NULL); // url= GetListOption(g, "URL", topt->oplist, NULL); +// prop = GetListOption(g, "Properties", topt->oplist, NULL); tabtyp = GetListOption(g, "Tabtype", topt->oplist, NULL); #endif // JDBC_SUPPORT - mxe= atoi(GetListOption(g,"maxerr", topt->oplist, "0")); #if defined(PROMPT_OK) cop= atoi(GetListOption(g, "checkdsn", topt->oplist, "0")); #endif // PROMPT_OK - } else { +#if defined(ZIP_SUPPORT) + zfn = GetListOption(g, "Zipfile", topt->oplist, NULL); +#endif // ZIP_SUPPORT + } else { host= "localhost"; user= (ttp == TAB_ODBC ? NULL : "root"); } // endif option_list @@ -5366,6 +5373,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, jdef->SetName(create_info->alias); sjp= (PJPARM)PlugSubAlloc(g, NULL, sizeof(JDBCPARM)); sjp->Driver= driver; +// sjp->Properties = prop; sjp->Fsize= 0; sjp->Scrollable= false; @@ -5468,7 +5476,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, case TAB_XML: #endif // LIBXML2_SUPPORT || DOMDOC_SUPPORT case TAB_JSON: - if (!fn) + if (!fn && !zfn && !mul) sprintf(g->Message, "Missing %s file name", topt->type); else ok= true; @@ -5582,7 +5590,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, NULL, port, fnc == FNC_COL); break; case TAB_CSV: - qrp= CSVColumns(g, dpath, fn, spc, qch, hdr, mxe, fnc == FNC_COL); + qrp = CSVColumns(g, dpath, topt, fnc == FNC_COL); break; #if defined(__WIN__) case TAB_WMI: @@ -6947,7 +6955,7 @@ maria_declare_plugin(connect) 0x0104, /* version number (1.04) */ NULL, /* status variables */ connect_system_variables, /* system variables */ - "1.04.0008", /* string version */ + "1.05.0001", /* string version */ MariaDB_PLUGIN_MATURITY_GAMMA /* maturity */ } maria_declare_plugin_end; diff --git a/storage/connect/ha_connect.h b/storage/connect/ha_connect.h index 60194ac0e3c..3d9ff967618 100644 --- a/storage/connect/ha_connect.h +++ b/storage/connect/ha_connect.h @@ -83,42 +83,9 @@ extern handlerton *connect_hton; These can be specified in the CREATE TABLE: CREATE TABLE ( ... ) {...here...} -*/ -#if 0 // moved to mycat.h -typedef struct ha_table_option_struct TOS, *PTOS; - -struct ha_table_option_struct { - const char *type; - const char *filename; - const char *optname; - const char *tabname; - const char *tablist; - const char *dbname; - const char *separator; -//const char *connect; - const char *qchar; - const char *module; - const char *subtype; - const char *catfunc; - const char *srcdef; - const char *colist; - const char *oplist; - const char *data_charset; - ulonglong lrecl; - ulonglong elements; -//ulonglong estimate; - ulonglong multiple; - ulonglong header; - ulonglong quoted; - ulonglong ending; - ulonglong compressed; - bool mapped; - bool huge; - bool split; - bool readonly; - bool sepindex; - }; -#endif // 0 + + ------ Was moved to mycat.h ------ + */ /** structure for CREATE TABLE options (field options) diff --git a/storage/connect/ioapi.c b/storage/connect/ioapi.c new file mode 100644 index 00000000000..7f5c191b2af --- /dev/null +++ b/storage/connect/ioapi.c @@ -0,0 +1,247 @@ +/* ioapi.h -- IO base function header for compress/uncompress .zip + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications for Zip64 support + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + +*/ + +#if defined(_WIN32) && (!(defined(_CRT_SECURE_NO_WARNINGS))) + #define _CRT_SECURE_NO_WARNINGS +#endif + +#if defined(__APPLE__) || defined(IOAPI_NO_64) +// In darwin and perhaps other BSD variants off_t is a 64 bit value, hence no need for specific 64 bit functions +#define FOPEN_FUNC(filename, mode) fopen(filename, mode) +#define FTELLO_FUNC(stream) ftello(stream) +#define FSEEKO_FUNC(stream, offset, origin) fseeko(stream, offset, origin) +#else +#define FOPEN_FUNC(filename, mode) fopen64(filename, mode) +#define FTELLO_FUNC(stream) ftello64(stream) +#define FSEEKO_FUNC(stream, offset, origin) fseeko64(stream, offset, origin) +#endif + + +#include "ioapi.h" + +voidpf call_zopen64 (const zlib_filefunc64_32_def* pfilefunc,const void*filename,int mode) +{ + if (pfilefunc->zfile_func64.zopen64_file != NULL) + return (*(pfilefunc->zfile_func64.zopen64_file)) (pfilefunc->zfile_func64.opaque,filename,mode); + else + { + return (*(pfilefunc->zopen32_file))(pfilefunc->zfile_func64.opaque,(const char*)filename,mode); + } +} + +long call_zseek64 (const zlib_filefunc64_32_def* pfilefunc,voidpf filestream, ZPOS64_T offset, int origin) +{ + if (pfilefunc->zfile_func64.zseek64_file != NULL) + return (*(pfilefunc->zfile_func64.zseek64_file)) (pfilefunc->zfile_func64.opaque,filestream,offset,origin); + else + { + uLong offsetTruncated = (uLong)offset; + if (offsetTruncated != offset) + return -1; + else + return (*(pfilefunc->zseek32_file))(pfilefunc->zfile_func64.opaque,filestream,offsetTruncated,origin); + } +} + +ZPOS64_T call_ztell64 (const zlib_filefunc64_32_def* pfilefunc,voidpf filestream) +{ + if (pfilefunc->zfile_func64.zseek64_file != NULL) + return (*(pfilefunc->zfile_func64.ztell64_file)) (pfilefunc->zfile_func64.opaque,filestream); + else + { + uLong tell_uLong = (*(pfilefunc->ztell32_file))(pfilefunc->zfile_func64.opaque,filestream); + if ((tell_uLong) == MAXU32) + return (ZPOS64_T)-1; + else + return tell_uLong; + } +} + +void fill_zlib_filefunc64_32_def_from_filefunc32(zlib_filefunc64_32_def* p_filefunc64_32,const zlib_filefunc_def* p_filefunc32) +{ + p_filefunc64_32->zfile_func64.zopen64_file = NULL; + p_filefunc64_32->zopen32_file = p_filefunc32->zopen_file; + p_filefunc64_32->zfile_func64.zerror_file = p_filefunc32->zerror_file; + p_filefunc64_32->zfile_func64.zread_file = p_filefunc32->zread_file; + p_filefunc64_32->zfile_func64.zwrite_file = p_filefunc32->zwrite_file; + p_filefunc64_32->zfile_func64.ztell64_file = NULL; + p_filefunc64_32->zfile_func64.zseek64_file = NULL; + p_filefunc64_32->zfile_func64.zclose_file = p_filefunc32->zclose_file; + p_filefunc64_32->zfile_func64.zerror_file = p_filefunc32->zerror_file; + p_filefunc64_32->zfile_func64.opaque = p_filefunc32->opaque; + p_filefunc64_32->zseek32_file = p_filefunc32->zseek_file; + p_filefunc64_32->ztell32_file = p_filefunc32->ztell_file; +} + + + +static voidpf ZCALLBACK fopen_file_func OF((voidpf opaque, const char* filename, int mode)); +static uLong ZCALLBACK fread_file_func OF((voidpf opaque, voidpf stream, void* buf, uLong size)); +static uLong ZCALLBACK fwrite_file_func OF((voidpf opaque, voidpf stream, const void* buf,uLong size)); +static ZPOS64_T ZCALLBACK ftell64_file_func OF((voidpf opaque, voidpf stream)); +static long ZCALLBACK fseek64_file_func OF((voidpf opaque, voidpf stream, ZPOS64_T offset, int origin)); +static int ZCALLBACK fclose_file_func OF((voidpf opaque, voidpf stream)); +static int ZCALLBACK ferror_file_func OF((voidpf opaque, voidpf stream)); + +static voidpf ZCALLBACK fopen_file_func (voidpf opaque, const char* filename, int mode) +{ + FILE* file = NULL; + const char* mode_fopen = NULL; + if ((mode & ZLIB_FILEFUNC_MODE_READWRITEFILTER)==ZLIB_FILEFUNC_MODE_READ) + mode_fopen = "rb"; + else + if (mode & ZLIB_FILEFUNC_MODE_EXISTING) + mode_fopen = "r+b"; + else + if (mode & ZLIB_FILEFUNC_MODE_CREATE) + mode_fopen = "wb"; + + if ((filename!=NULL) && (mode_fopen != NULL)) + file = fopen(filename, mode_fopen); + return file; +} + +static voidpf ZCALLBACK fopen64_file_func (voidpf opaque, const void* filename, int mode) +{ + FILE* file = NULL; + const char* mode_fopen = NULL; + if ((mode & ZLIB_FILEFUNC_MODE_READWRITEFILTER)==ZLIB_FILEFUNC_MODE_READ) + mode_fopen = "rb"; + else + if (mode & ZLIB_FILEFUNC_MODE_EXISTING) + mode_fopen = "r+b"; + else + if (mode & ZLIB_FILEFUNC_MODE_CREATE) + mode_fopen = "wb"; + + if ((filename!=NULL) && (mode_fopen != NULL)) + file = FOPEN_FUNC((const char*)filename, mode_fopen); + return file; +} + + +static uLong ZCALLBACK fread_file_func (voidpf opaque, voidpf stream, void* buf, uLong size) +{ + uLong ret; + ret = (uLong)fread(buf, 1, (size_t)size, (FILE *)stream); + return ret; +} + +static uLong ZCALLBACK fwrite_file_func (voidpf opaque, voidpf stream, const void* buf, uLong size) +{ + uLong ret; + ret = (uLong)fwrite(buf, 1, (size_t)size, (FILE *)stream); + return ret; +} + +static long ZCALLBACK ftell_file_func (voidpf opaque, voidpf stream) +{ + long ret; + ret = ftell((FILE *)stream); + return ret; +} + + +static ZPOS64_T ZCALLBACK ftell64_file_func (voidpf opaque, voidpf stream) +{ + ZPOS64_T ret; + ret = FTELLO_FUNC((FILE *)stream); + return ret; +} + +static long ZCALLBACK fseek_file_func (voidpf opaque, voidpf stream, uLong offset, int origin) +{ + int fseek_origin=0; + long ret; + switch (origin) + { + case ZLIB_FILEFUNC_SEEK_CUR : + fseek_origin = SEEK_CUR; + break; + case ZLIB_FILEFUNC_SEEK_END : + fseek_origin = SEEK_END; + break; + case ZLIB_FILEFUNC_SEEK_SET : + fseek_origin = SEEK_SET; + break; + default: return -1; + } + ret = 0; + if (fseek((FILE *)stream, offset, fseek_origin) != 0) + ret = -1; + return ret; +} + +static long ZCALLBACK fseek64_file_func (voidpf opaque, voidpf stream, ZPOS64_T offset, int origin) +{ + int fseek_origin=0; + long ret; + switch (origin) + { + case ZLIB_FILEFUNC_SEEK_CUR : + fseek_origin = SEEK_CUR; + break; + case ZLIB_FILEFUNC_SEEK_END : + fseek_origin = SEEK_END; + break; + case ZLIB_FILEFUNC_SEEK_SET : + fseek_origin = SEEK_SET; + break; + default: return -1; + } + ret = 0; + + if(FSEEKO_FUNC((FILE *)stream, offset, fseek_origin) != 0) + ret = -1; + + return ret; +} + + +static int ZCALLBACK fclose_file_func (voidpf opaque, voidpf stream) +{ + int ret; + ret = fclose((FILE *)stream); + return ret; +} + +static int ZCALLBACK ferror_file_func (voidpf opaque, voidpf stream) +{ + int ret; + ret = ferror((FILE *)stream); + return ret; +} + +void fill_fopen_filefunc (pzlib_filefunc_def) + zlib_filefunc_def* pzlib_filefunc_def; +{ + pzlib_filefunc_def->zopen_file = fopen_file_func; + pzlib_filefunc_def->zread_file = fread_file_func; + pzlib_filefunc_def->zwrite_file = fwrite_file_func; + pzlib_filefunc_def->ztell_file = ftell_file_func; + pzlib_filefunc_def->zseek_file = fseek_file_func; + pzlib_filefunc_def->zclose_file = fclose_file_func; + pzlib_filefunc_def->zerror_file = ferror_file_func; + pzlib_filefunc_def->opaque = NULL; +} + +void fill_fopen64_filefunc (zlib_filefunc64_def* pzlib_filefunc_def) +{ + pzlib_filefunc_def->zopen64_file = fopen64_file_func; + pzlib_filefunc_def->zread_file = fread_file_func; + pzlib_filefunc_def->zwrite_file = fwrite_file_func; + pzlib_filefunc_def->ztell64_file = ftell64_file_func; + pzlib_filefunc_def->zseek64_file = fseek64_file_func; + pzlib_filefunc_def->zclose_file = fclose_file_func; + pzlib_filefunc_def->zerror_file = ferror_file_func; + pzlib_filefunc_def->opaque = NULL; +} diff --git a/storage/connect/ioapi.h b/storage/connect/ioapi.h new file mode 100644 index 00000000000..8dcbdb06e35 --- /dev/null +++ b/storage/connect/ioapi.h @@ -0,0 +1,208 @@ +/* ioapi.h -- IO base function header for compress/uncompress .zip + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications for Zip64 support + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + + Changes + + Oct-2009 - Defined ZPOS64_T to fpos_t on windows and u_int64_t on linux. (might need to find a better why for this) + Oct-2009 - Change to fseeko64, ftello64 and fopen64 so large files would work on linux. + More if/def section may be needed to support other platforms + Oct-2009 - Defined fxxxx64 calls to normal fopen/ftell/fseek so they would compile on windows. + (but you should use iowin32.c for windows instead) + +*/ + +#ifndef _ZLIBIOAPI64_H +#define _ZLIBIOAPI64_H + +#if (!defined(_WIN32)) && (!defined(WIN32)) && (!defined(__APPLE__)) + + // Linux needs this to support file operation on files larger then 4+GB + // But might need better if/def to select just the platforms that needs them. + + #ifndef __USE_FILE_OFFSET64 + #define __USE_FILE_OFFSET64 + #endif + #ifndef __USE_LARGEFILE64 + #define __USE_LARGEFILE64 + #endif + #ifndef _LARGEFILE64_SOURCE + #define _LARGEFILE64_SOURCE + #endif + #ifndef _FILE_OFFSET_BIT + #define _FILE_OFFSET_BIT 64 + #endif + +#endif + +#include <stdio.h> +#include <stdlib.h> +#include "zlib.h" + +#if defined(USE_FILE32API) +#define fopen64 fopen +#define ftello64 ftell +#define fseeko64 fseek +#else +#ifdef __FreeBSD__ +#define fopen64 fopen +#define ftello64 ftello +#define fseeko64 fseeko +#endif +#ifdef _MSC_VER + #define fopen64 fopen + #if (_MSC_VER >= 1400) && (!(defined(NO_MSCVER_FILE64_FUNC))) + #define ftello64 _ftelli64 + #define fseeko64 _fseeki64 + #else // old MSC + #define ftello64 ftell + #define fseeko64 fseek + #endif +#endif +#endif + +/* +#ifndef ZPOS64_T + #ifdef _WIN32 + #define ZPOS64_T fpos_t + #else + #include <stdint.h> + #define ZPOS64_T uint64_t + #endif +#endif +*/ + +#ifdef HAVE_MINIZIP64_CONF_H +#include "mz64conf.h" +#endif + +/* a type choosen by DEFINE */ +#ifdef HAVE_64BIT_INT_CUSTOM +typedef 64BIT_INT_CUSTOM_TYPE ZPOS64_T; +#else +#ifdef HAS_STDINT_H +#include "stdint.h" +typedef uint64_t ZPOS64_T; +#else + +/* Maximum unsigned 32-bit value used as placeholder for zip64 */ +#define MAXU32 0xffffffff + +#if defined(_MSC_VER) || defined(__BORLANDC__) +typedef unsigned __int64 ZPOS64_T; +#else +typedef unsigned long long int ZPOS64_T; +#endif +#endif +#endif + + + +#ifdef __cplusplus +extern "C" { +#endif + + +#define ZLIB_FILEFUNC_SEEK_CUR (1) +#define ZLIB_FILEFUNC_SEEK_END (2) +#define ZLIB_FILEFUNC_SEEK_SET (0) + +#define ZLIB_FILEFUNC_MODE_READ (1) +#define ZLIB_FILEFUNC_MODE_WRITE (2) +#define ZLIB_FILEFUNC_MODE_READWRITEFILTER (3) + +#define ZLIB_FILEFUNC_MODE_EXISTING (4) +#define ZLIB_FILEFUNC_MODE_CREATE (8) + + +#ifndef ZCALLBACK + #if (defined(WIN32) || defined(_WIN32) || defined (WINDOWS) || defined (_WINDOWS)) && defined(CALLBACK) && defined (USEWINDOWS_CALLBACK) + #define ZCALLBACK CALLBACK + #else + #define ZCALLBACK + #endif +#endif + + + + +typedef voidpf (ZCALLBACK *open_file_func) OF((voidpf opaque, const char* filename, int mode)); +typedef uLong (ZCALLBACK *read_file_func) OF((voidpf opaque, voidpf stream, void* buf, uLong size)); +typedef uLong (ZCALLBACK *write_file_func) OF((voidpf opaque, voidpf stream, const void* buf, uLong size)); +typedef int (ZCALLBACK *close_file_func) OF((voidpf opaque, voidpf stream)); +typedef int (ZCALLBACK *testerror_file_func) OF((voidpf opaque, voidpf stream)); + +typedef long (ZCALLBACK *tell_file_func) OF((voidpf opaque, voidpf stream)); +typedef long (ZCALLBACK *seek_file_func) OF((voidpf opaque, voidpf stream, uLong offset, int origin)); + + +/* here is the "old" 32 bits structure structure */ +typedef struct zlib_filefunc_def_s +{ + open_file_func zopen_file; + read_file_func zread_file; + write_file_func zwrite_file; + tell_file_func ztell_file; + seek_file_func zseek_file; + close_file_func zclose_file; + testerror_file_func zerror_file; + voidpf opaque; +} zlib_filefunc_def; + +typedef ZPOS64_T (ZCALLBACK *tell64_file_func) OF((voidpf opaque, voidpf stream)); +typedef long (ZCALLBACK *seek64_file_func) OF((voidpf opaque, voidpf stream, ZPOS64_T offset, int origin)); +typedef voidpf (ZCALLBACK *open64_file_func) OF((voidpf opaque, const void* filename, int mode)); + +typedef struct zlib_filefunc64_def_s +{ + open64_file_func zopen64_file; + read_file_func zread_file; + write_file_func zwrite_file; + tell64_file_func ztell64_file; + seek64_file_func zseek64_file; + close_file_func zclose_file; + testerror_file_func zerror_file; + voidpf opaque; +} zlib_filefunc64_def; + +void fill_fopen64_filefunc OF((zlib_filefunc64_def* pzlib_filefunc_def)); +void fill_fopen_filefunc OF((zlib_filefunc_def* pzlib_filefunc_def)); + +/* now internal definition, only for zip.c and unzip.h */ +typedef struct zlib_filefunc64_32_def_s +{ + zlib_filefunc64_def zfile_func64; + open_file_func zopen32_file; + tell_file_func ztell32_file; + seek_file_func zseek32_file; +} zlib_filefunc64_32_def; + + +#define ZREAD64(filefunc,filestream,buf,size) ((*((filefunc).zfile_func64.zread_file)) ((filefunc).zfile_func64.opaque,filestream,buf,size)) +#define ZWRITE64(filefunc,filestream,buf,size) ((*((filefunc).zfile_func64.zwrite_file)) ((filefunc).zfile_func64.opaque,filestream,buf,size)) +//#define ZTELL64(filefunc,filestream) ((*((filefunc).ztell64_file)) ((filefunc).opaque,filestream)) +//#define ZSEEK64(filefunc,filestream,pos,mode) ((*((filefunc).zseek64_file)) ((filefunc).opaque,filestream,pos,mode)) +#define ZCLOSE64(filefunc,filestream) ((*((filefunc).zfile_func64.zclose_file)) ((filefunc).zfile_func64.opaque,filestream)) +#define ZERROR64(filefunc,filestream) ((*((filefunc).zfile_func64.zerror_file)) ((filefunc).zfile_func64.opaque,filestream)) + +voidpf call_zopen64 OF((const zlib_filefunc64_32_def* pfilefunc,const void*filename,int mode)); +long call_zseek64 OF((const zlib_filefunc64_32_def* pfilefunc,voidpf filestream, ZPOS64_T offset, int origin)); +ZPOS64_T call_ztell64 OF((const zlib_filefunc64_32_def* pfilefunc,voidpf filestream)); + +void fill_zlib_filefunc64_32_def_from_filefunc32(zlib_filefunc64_32_def* p_filefunc64_32,const zlib_filefunc_def* p_filefunc32); + +#define ZOPEN64(filefunc,filename,mode) (call_zopen64((&(filefunc)),(filename),(mode))) +#define ZTELL64(filefunc,filestream) (call_ztell64((&(filefunc)),(filestream))) +#define ZSEEK64(filefunc,filestream,pos,mode) (call_zseek64((&(filefunc)),(filestream),(pos),(mode))) + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/storage/connect/jdbccat.h b/storage/connect/jdbccat.h index 37f33d7063d..7108aa376ce 100644 --- a/storage/connect/jdbccat.h +++ b/storage/connect/jdbccat.h @@ -8,6 +8,7 @@ typedef struct jdbc_parms { char *Url; // Driver URL char *User; // User connect info char *Pwd; // Password connect info +//char *Properties; // Connection property list //int Cto; // Connect timeout //int Qto; // Query timeout int Fsize; // Fetch size diff --git a/storage/connect/jdbconn.cpp b/storage/connect/jdbconn.cpp index dca9bd0eac4..a69f84a94a1 100644 --- a/storage/connect/jdbconn.cpp +++ b/storage/connect/jdbconn.cpp @@ -55,9 +55,8 @@ #if defined(__WIN__) extern "C" HINSTANCE s_hModule; // Saved module handle -#else // !__WIN__ +#endif // __WIN__ #define nullptr 0 -#endif // !__WIN__ TYPCONV GetTypeConv(); int GetConvSize(); @@ -525,10 +524,10 @@ JDBConn::JDBConn(PGLOBAL g, TDBJDBC *tdbp) m_Wrap = strcat(strcpy(wn, "wrappers/"), m_Wrap); } // endif m_Wrap - m_Driver = NULL; - m_Url = NULL; - m_User = NULL; - m_Pwd = NULL; +//m_Driver = NULL; +//m_Url = NULL; +//m_User = NULL; +//m_Pwd = NULL; m_Ncol = 0; m_Aff = 0; m_Rows = 0; @@ -772,7 +771,7 @@ bool JDBConn::GetJVM(PGLOBAL g) /***********************************************************************/ int JDBConn::Open(PJPARM sop) { - + int irc = RC_FX; bool err = false; jboolean jt = (trace > 0); PGLOBAL& g = m_G; @@ -865,30 +864,37 @@ int JDBConn::Open(PJPARM sop) switch (rc) { case JNI_OK: strcpy(g->Message, "VM successfully created"); + irc = RC_OK; break; case JNI_ERR: strcpy(g->Message, "Initialising JVM failed: unknown error"); - return RC_FX; + break; case JNI_EDETACHED: strcpy(g->Message, "Thread detached from the VM"); - return RC_FX; + break; case JNI_EVERSION: strcpy(g->Message, "JNI version error"); - return RC_FX; + break; case JNI_ENOMEM: strcpy(g->Message, "Not enough memory"); - return RC_FX; + break; case JNI_EEXIST: strcpy(g->Message, "VM already created"); - return RC_FX; + break; case JNI_EINVAL: strcpy(g->Message, "Invalid arguments"); - return RC_FX; + break; default: - sprintf(g->Message, "Unknown return code %d", rc); - return RC_FX; + sprintf(g->Message, "Unknown return code %d", (int)rc); + break; } // endswitch rc + if (trace) + htrc("%s\n", g->Message); + + if (irc != RC_OK) + return irc; + //=============== Display JVM version =============== jint ver = env->GetVersion(); printf("JVM Version %d.%d\n", ((ver>>16)&0x0f), (ver&0x0f)); @@ -978,10 +984,10 @@ int JDBConn::Open(PJPARM sop) jobjectArray parms = env->NewObjectArray(4, // constructs java array of 4 env->FindClass("java/lang/String"), NULL); // Strings - m_Driver = sop->Driver; - m_Url = sop->Url; - m_User = sop->User; - m_Pwd = sop->Pwd; +//m_Driver = sop->Driver; +//m_Url = sop->Url; +//m_User = sop->User; +//m_Pwd = sop->Pwd; m_Scrollable = sop->Scrollable; m_RowsetSize = sop->Fsize; //m_LoginTimeout = sop->Cto; @@ -989,17 +995,20 @@ int JDBConn::Open(PJPARM sop) //m_UseCnc = sop->UseCnc; // change some elements - if (m_Driver) - env->SetObjectArrayElement(parms, 0, env->NewStringUTF(m_Driver)); + if (sop->Driver) + env->SetObjectArrayElement(parms, 0, env->NewStringUTF(sop->Driver)); - if (m_Url) - env->SetObjectArrayElement(parms, 1, env->NewStringUTF(m_Url)); + if (sop->Url) + env->SetObjectArrayElement(parms, 1, env->NewStringUTF(sop->Url)); - if (m_User) - env->SetObjectArrayElement(parms, 2, env->NewStringUTF(m_User)); + if (sop->User) + env->SetObjectArrayElement(parms, 2, env->NewStringUTF(sop->User)); - if (m_Pwd) - env->SetObjectArrayElement(parms, 3, env->NewStringUTF(m_Pwd)); + if (sop->Pwd) + env->SetObjectArrayElement(parms, 3, env->NewStringUTF(sop->Pwd)); + +//if (sop->Properties) +// env->SetObjectArrayElement(parms, 4, env->NewStringUTF(sop->Properties)); // call method rc = env->CallIntMethod(job, cid, parms, m_RowsetSize, m_Scrollable); @@ -1432,7 +1441,7 @@ bool JDBConn::SetParam(JDBCCOL *colp) PGLOBAL& g = m_G; bool rc = false; PVAL val = colp->GetValue(); - jint n, i = (jint)colp->GetRank(); + jint n, jrc = 0, i = (jint)colp->GetRank(); jshort s; jlong lg; //jfloat f; @@ -1442,69 +1451,74 @@ bool JDBConn::SetParam(JDBCCOL *colp) jstring jst = nullptr; jmethodID dtc, setid = nullptr; - switch (val->GetType()) { - case TYPE_STRING: - if (gmID(g, setid, "SetStringParm", "(ILjava/lang/String;)V")) - return true; - - jst = env->NewStringUTF(val->GetCharValue()); - env->CallVoidMethod(job, setid, i, jst); - break; - case TYPE_INT: - if (gmID(g, setid, "SetIntParm", "(II)V")) + if (val->GetNullable() && val->IsNull()) { + if (gmID(g, setid, "SetNullParm", "(II)I")) return true; - n = (jint)val->GetIntValue(); - env->CallVoidMethod(job, setid, i, n); - break; - case TYPE_TINY: - case TYPE_SHORT: - if (gmID(g, setid, "SetShortParm", "(IS)V")) - return true; + jrc = env->CallIntMethod(job, setid, i, (jint)GetJDBCType(val->GetType())); + } else switch (val->GetType()) { + case TYPE_STRING: + if (gmID(g, setid, "SetStringParm", "(ILjava/lang/String;)V")) + return true; - s = (jshort)val->GetShortValue(); - env->CallVoidMethod(job, setid, i, s); - break; - case TYPE_BIGINT: - if (gmID(g, setid, "SetBigintParm", "(IJ)V")) - return true; + jst = env->NewStringUTF(val->GetCharValue()); + env->CallVoidMethod(job, setid, i, jst); + break; + case TYPE_INT: + if (gmID(g, setid, "SetIntParm", "(II)V")) + return true; - lg = (jlong)val->GetBigintValue(); - env->CallVoidMethod(job, setid, i, lg); - break; - case TYPE_DOUBLE: - case TYPE_DECIM: - if (gmID(g, setid, "SetDoubleParm", "(ID)V")) - return true; + n = (jint)val->GetIntValue(); + env->CallVoidMethod(job, setid, i, n); + break; + case TYPE_TINY: + case TYPE_SHORT: + if (gmID(g, setid, "SetShortParm", "(IS)V")) + return true; - d = (jdouble)val->GetFloatValue(); - env->CallVoidMethod(job, setid, i, d); - break; - case TYPE_DATE: - if ((dat = env->FindClass("java/sql/Timestamp")) == nullptr) { - strcpy(g->Message, "Cannot find Timestamp class"); - return true; - } else if (!(dtc = env->GetMethodID(dat, "<init>", "(J)V"))) { - strcpy(g->Message, "Cannot find Timestamp class constructor"); - return true; - } // endif's + s = (jshort)val->GetShortValue(); + env->CallVoidMethod(job, setid, i, s); + break; + case TYPE_BIGINT: + if (gmID(g, setid, "SetBigintParm", "(IJ)V")) + return true; - lg = (jlong)val->GetBigintValue() * 1000; + lg = (jlong)val->GetBigintValue(); + env->CallVoidMethod(job, setid, i, lg); + break; + case TYPE_DOUBLE: + case TYPE_DECIM: + if (gmID(g, setid, "SetDoubleParm", "(ID)V")) + return true; - if ((datobj = env->NewObject(dat, dtc, lg)) == nullptr) { - strcpy(g->Message, "Cannot make Timestamp object"); - return true; - } else if (gmID(g, setid, "SetTimestampParm", "(ILjava/sql/Timestamp;)V")) + d = (jdouble)val->GetFloatValue(); + env->CallVoidMethod(job, setid, i, d); + break; + case TYPE_DATE: + if ((dat = env->FindClass("java/sql/Timestamp")) == nullptr) { + strcpy(g->Message, "Cannot find Timestamp class"); + return true; + } else if (!(dtc = env->GetMethodID(dat, "<init>", "(J)V"))) { + strcpy(g->Message, "Cannot find Timestamp class constructor"); + return true; + } // endif's + + lg = (jlong)val->GetBigintValue() * 1000; + + if ((datobj = env->NewObject(dat, dtc, lg)) == nullptr) { + strcpy(g->Message, "Cannot make Timestamp object"); + return true; + } else if (gmID(g, setid, "SetTimestampParm", "(ILjava/sql/Timestamp;)V")) + return true; + + env->CallVoidMethod(job, setid, i, datobj); + break; + default: + sprintf(g->Message, "Parm type %d not supported", val->GetType()); return true; + } // endswitch Type - env->CallVoidMethod(job, setid, i, datobj); - break; - default: - sprintf(g->Message, "Parm type %d not supported", val->GetType()); - return true; - } // endswitch Type - - if (Check()) { + if (Check(jrc)) { sprintf(g->Message, "SetParam: col=%s msg=%s", colp->GetName(), Msg); rc = true; } // endif msg diff --git a/storage/connect/jdbconn.h b/storage/connect/jdbconn.h index 0a1c52d4576..9d428142839 100644 --- a/storage/connect/jdbconn.h +++ b/storage/connect/jdbconn.h @@ -180,9 +180,9 @@ protected: char *Msg; char *m_Wrap; char m_IDQuoteChar[2]; - PSZ m_Driver; - PSZ m_Url; - PSZ m_User; +//PSZ m_Driver; +//PSZ m_Url; +//PSZ m_User; PSZ m_Pwd; int m_Ncol; int m_Aff; diff --git a/storage/connect/jsonudf.cpp b/storage/connect/jsonudf.cpp index 8bddc68e2ae..f9034f25739 100644 --- a/storage/connect/jsonudf.cpp +++ b/storage/connect/jsonudf.cpp @@ -5264,3 +5264,50 @@ char *envar(UDF_INIT *initid, UDF_ARGS *args, char *result, return str; } // end of envar +/*********************************************************************************/ +/* Returns the distinct number of B occurences in A. */ +/*********************************************************************************/ +my_bool countin_init(UDF_INIT *initid, UDF_ARGS *args, char *message) +{ + if (args->arg_count != 2) { + strcpy(message, "This function must have 2 arguments"); + return true; + } else if (args->arg_type[0] != STRING_RESULT) { + strcpy(message, "First argument must be string"); + return true; + } else if (args->arg_type[1] != STRING_RESULT) { + strcpy(message, "Second argument is not a string"); + return true; + } // endif args + + return false; +} // end of countin_init + +long long countin(UDF_INIT *initid, UDF_ARGS *args, char *result, + unsigned long *res_length, char *is_null, char *) +{ + PSZ str1, str2; + char *s; + long long n = 0; + size_t lg; + + lg = (size_t)args->lengths[0]; + s = str1 = (PSZ)malloc(lg + 1); + memcpy(str1, args->args[0], lg); + str1[lg] = 0; + + lg = (size_t)args->lengths[1]; + str2 = (PSZ)malloc(lg + 1); + memcpy(str2, args->args[1], lg); + str2[lg] = 0; + + while (s = strstr(s, str2)) { + n++; + s += lg; + } // endwhile + + free(str1); + free(str2); + return n; +} // end of countin + diff --git a/storage/connect/jsonudf.h b/storage/connect/jsonudf.h index 1406d9f2f2e..d2890421c62 100644 --- a/storage/connect/jsonudf.h +++ b/storage/connect/jsonudf.h @@ -221,6 +221,9 @@ extern "C" { DllExport my_bool envar_init(UDF_INIT*, UDF_ARGS*, char*); DllExport char *envar(UDF_EXEC_ARGS); + + DllExport my_bool countin_init(UDF_INIT*, UDF_ARGS*, char*); + DllExport long long countin(UDF_EXEC_ARGS); } // extern "C" diff --git a/storage/connect/libdoc.cpp b/storage/connect/libdoc.cpp index c2882fc0d7c..2470d37c353 100644 --- a/storage/connect/libdoc.cpp +++ b/storage/connect/libdoc.cpp @@ -1,6 +1,6 @@ /******************************************************************/ /* Implementation of XML document processing using libxml2 */ -/* Author: Olivier Bertrand 2007-2015 */ +/* Author: Olivier Bertrand 2007-2016 */ /******************************************************************/ #include "my_global.h" #include <string.h> @@ -68,8 +68,8 @@ class LIBXMLDOC : public XMLDOCUMENT { virtual void SetNofree(bool b) {Nofreelist = b;} // Methods - virtual bool Initialize(PGLOBAL g); - virtual bool ParseFile(char *fn); + virtual bool Initialize(PGLOBAL g, char *entry, bool zipped); + virtual bool ParseFile(PGLOBAL g, char *fn); virtual bool NewDoc(PGLOBAL g, char *ver); virtual void AddComment(PGLOBAL g, char *com); virtual PXNODE GetRoot(PGLOBAL g); @@ -373,22 +373,33 @@ LIBXMLDOC::LIBXMLDOC(char *nsl, char *nsdf, char *enc, PFBLOCK fp) /******************************************************************/ /* Initialize XML parser and check library compatibility. */ /******************************************************************/ -bool LIBXMLDOC::Initialize(PGLOBAL g) - { +bool LIBXMLDOC::Initialize(PGLOBAL g, char *entry, bool zipped) +{ + if (zipped && InitZip(g, entry)) + return true; + int n = xmlKeepBlanksDefault(1); return MakeNSlist(g); - } // end of Initialize +} // end of Initialize /******************************************************************/ /* Parse the XML file and construct node tree in memory. */ /******************************************************************/ -bool LIBXMLDOC::ParseFile(char *fn) +bool LIBXMLDOC::ParseFile(PGLOBAL g, char *fn) { if (trace) htrc("ParseFile\n"); - if ((Docp = xmlParseFile(fn))) { - if (Docp->encoding) + if (zip) { + // Parse an in memory document + char *xdoc = GetMemDoc(g, fn); + + Docp = (xdoc) ? xmlParseDoc((const xmlChar *)xdoc) : NULL; + } else + Docp = xmlParseFile(fn); + + if (Docp) { + if (Docp->encoding) Encoding = (char*)Docp->encoding; return false; @@ -609,6 +620,7 @@ void LIBXMLDOC::CloseDoc(PGLOBAL g, PFBLOCK xp) } // endif xp CloseXML2File(g, xp, false); + CloseZip(); } // end of Close /******************************************************************/ diff --git a/storage/connect/mycat.cc b/storage/connect/mycat.cc index b4b03e6ba4a..497fe5e1aa8 100644 --- a/storage/connect/mycat.cc +++ b/storage/connect/mycat.cc @@ -16,7 +16,7 @@ /*************** Mycat CC Program Source Code File (.CC) ***************/ /* PROGRAM NAME: MYCAT */ /* ------------- */ -/* Version 1.4 */ +/* Version 1.5 */ /* */ /* Author: Olivier Bertrand 2012 - 2016 */ /* */ @@ -64,7 +64,9 @@ #include "filamtxt.h" #include "tabdos.h" #include "tabfmt.h" +#if defined(VCT_SUPPORT) #include "tabvct.h" +#endif // VCT_SUPPORT #include "tabsys.h" #if defined(__WIN__) #include "tabmac.h" @@ -93,6 +95,9 @@ #if defined(XML_SUPPORT) #include "tabxml.h" #endif // XML_SUPPORT +#if defined(ZIP_SUPPORT) +#include "tabzip.h" +#endif // ZIP_SUPPORT #include "mycat.h" /***********************************************************************/ @@ -152,7 +157,10 @@ TABTYPE GetTypeID(const char *type) #endif : (!stricmp(type, "VIR")) ? TAB_VIR : (!stricmp(type, "JSON")) ? TAB_JSON - : (!stricmp(type, "OEM")) ? TAB_OEM : TAB_NIY; +#ifdef ZIP_SUPPORT + : (!stricmp(type, "ZIP")) ? TAB_ZIP +#endif + : (!stricmp(type, "OEM")) ? TAB_OEM : TAB_NIY; } // end of GetTypeID /***********************************************************************/ @@ -173,6 +181,7 @@ bool IsFileType(TABTYPE type) case TAB_INI: case TAB_VEC: case TAB_JSON: +// case TAB_ZIP: isfile= true; break; default: @@ -549,7 +558,9 @@ PRELDEF MYCAT::MakeTableDesc(PGLOBAL g, PTABLE tablep, LPCSTR am) #if defined(XML_SUPPORT) case TAB_XML: tdp= new(g) XMLDEF; break; #endif // XML_SUPPORT - case TAB_VEC: tdp= new(g) VCTDEF; break; +#if defined(VCT_SUPPORT) + case TAB_VEC: tdp = new(g)VCTDEF; break; +#endif // VCT_SUPPORT #if defined(ODBC_SUPPORT) case TAB_ODBC: tdp= new(g) ODBCDEF; break; #endif // ODBC_SUPPORT @@ -571,7 +582,10 @@ PRELDEF MYCAT::MakeTableDesc(PGLOBAL g, PTABLE tablep, LPCSTR am) #endif // PIVOT_SUPPORT case TAB_VIR: tdp= new(g) VIRDEF; break; case TAB_JSON: tdp= new(g) JSONDEF; break; - default: +#if defined(ZIP_SUPPORT) + case TAB_ZIP: tdp= new(g) ZIPDEF; break; +#endif // ZIP_SUPPORT + default: sprintf(g->Message, MSG(BAD_TABLE_TYPE), am, name); } // endswitch diff --git a/storage/connect/mycat.h b/storage/connect/mycat.h index 05163f08f1b..663b68fd4b9 100644 --- a/storage/connect/mycat.h +++ b/storage/connect/mycat.h @@ -62,6 +62,7 @@ struct ha_table_option_struct { bool split; bool readonly; bool sepindex; + bool zipped; }; // Possible value for catalog functions diff --git a/storage/connect/mysql-test/connect/disabled.def b/storage/connect/mysql-test/connect/disabled.def index 9b4570915c7..5e15e0806ba 100644 --- a/storage/connect/mysql-test/connect/disabled.def +++ b/storage/connect/mysql-test/connect/disabled.def @@ -9,8 +9,7 @@ # Do not use any TAB characters for whitespace. # ############################################################################## -#json_udf_bin : broken upstream in --ps (fixed) -jdbc : Variable settings depend on machine configuration -jdbc_new : Variable settings depend on machine configuration +#jdbc : Variable settings depend on machine configuration +#jdbc_new : Variable settings depend on machine configuration jdbc_oracle : Variable settings depend on machine configuration jdbc_postgresql : Variable settings depend on machine configuration diff --git a/storage/connect/mysql-test/connect/r/jdbc_new.result b/storage/connect/mysql-test/connect/r/jdbc_new.result index 14381b0b11f..5cc4826213d 100644 --- a/storage/connect/mysql-test/connect/r/jdbc_new.result +++ b/storage/connect/mysql-test/connect/r/jdbc_new.result @@ -56,7 +56,7 @@ t1 CREATE TABLE `t1` ( ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='jdbc:mysql://127.0.0.1:SLAVE_PORT/test?user=root' `TABLE_TYPE`='JDBC' SELECT * FROM t1; a b -0 NULL +NULL NULL 0 test00 1 test01 2 test02 @@ -72,7 +72,7 @@ t1 CREATE TABLE `t1` ( ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='jdbc:mysql://127.0.0.1:SLAVE_PORT/test?user=root' `TABLE_TYPE`=JDBC `TABNAME`='t1' SELECT * FROM t1; a b -0 NULL +NULL NULL 0 test00 1 test01 2 test02 @@ -104,7 +104,7 @@ t1 CREATE TABLE `t1` ( ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='jdbc:mysql://127.0.0.1:SLAVE_PORT/test?user=root' `TABLE_TYPE`=JDBC SELECT * FROM t1; a b -0 NULL +NULL NULL 0 0 1 0 2 0 diff --git a/storage/connect/mysql-test/connect/std_data/JdbcMariaDB.jar b/storage/connect/mysql-test/connect/std_data/JdbcMariaDB.jar Binary files differindex 81f91e4465a..8525da74018 100644 --- a/storage/connect/mysql-test/connect/std_data/JdbcMariaDB.jar +++ b/storage/connect/mysql-test/connect/std_data/JdbcMariaDB.jar diff --git a/storage/connect/mysql-test/connect/t/jdbc.test b/storage/connect/mysql-test/connect/t/jdbc.test index 41fd298776b..58a527a3e6b 100644 --- a/storage/connect/mysql-test/connect/t/jdbc.test +++ b/storage/connect/mysql-test/connect/t/jdbc.test @@ -1,3 +1,4 @@ +-- source windows.inc -- source jdbconn.inc SET GLOBAL time_zone='+1:00'; diff --git a/storage/connect/mysql-test/connect/t/jdbc_new.test b/storage/connect/mysql-test/connect/t/jdbc_new.test index d1ad5117b72..5586cf8c027 100644 --- a/storage/connect/mysql-test/connect/t/jdbc_new.test +++ b/storage/connect/mysql-test/connect/t/jdbc_new.test @@ -5,6 +5,7 @@ connect (master,127.0.0.1,root,,test,$MASTER_MYPORT,); connect (slave,127.0.0.1,root,,test,$SLAVE_MYPORT,); connection master; +-- source windows.inc -- source jdbconn.inc connection slave; diff --git a/storage/connect/mysql-test/connect/t/windows.inc b/storage/connect/mysql-test/connect/t/windows.inc new file mode 100644 index 00000000000..88553d8aa59 --- /dev/null +++ b/storage/connect/mysql-test/connect/t/windows.inc @@ -0,0 +1,5 @@ +if (`select convert(@@version_compile_os using latin1) IN ("Win32","Win64","Windows") = 0`) +{ + skip Need windows; +} + diff --git a/storage/connect/plgdbsem.h b/storage/connect/plgdbsem.h index 910ce97f48a..cb408494319 100644 --- a/storage/connect/plgdbsem.h +++ b/storage/connect/plgdbsem.h @@ -1,9 +1,9 @@ /************** PlgDBSem H Declares Source Code File (.H) **************/ -/* Name: PLGDBSEM.H Version 3.6 */ +/* Name: PLGDBSEM.H Version 3.7 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 1998-2015 */ +/* (C) Copyright to the author Olivier BERTRAND 1998-2016 */ /* */ -/* This file contains the PlugDB++ application type definitions. */ +/* This file contains the CONNECT storage engine definitions. */ /***********************************************************************/ /***********************************************************************/ @@ -49,7 +49,8 @@ enum BLKTYP {TYPE_TABLE = 50, /* Table Name/Srcdef/... Block */ TYPE_FB_MAP = 23, /* Mapped file block (storage) */ TYPE_FB_HANDLE = 24, /* File block (handle) */ TYPE_FB_XML = 21, /* DOM XML file block */ - TYPE_FB_XML2 = 27}; /* libxml2 XML file block */ + TYPE_FB_XML2 = 27, /* libxml2 XML file block */ + TYPE_FB_ZIP = 28}; /* ZIP file block */ enum TABTYPE {TAB_UNDEF = 0, /* Table of undefined type */ TAB_DOS = 1, /* Fixed column offset, variable LRECL */ @@ -78,7 +79,8 @@ enum TABTYPE {TAB_UNDEF = 0, /* Table of undefined type */ TAB_JCT = 24, /* Junction tables NIY */ TAB_DMY = 25, /* DMY Dummy tables NIY */ TAB_JDBC = 26, /* Table accessed via JDBC */ - TAB_NIY = 27}; /* Table not implemented yet */ + TAB_ZIP = 27, /* ZIP file info table */ + TAB_NIY = 28}; /* Table not implemented yet */ enum AMT {TYPE_AM_ERROR = 0, /* Type not defined */ TYPE_AM_ROWID = 1, /* ROWID type (special column) */ @@ -123,7 +125,7 @@ enum AMT {TYPE_AM_ERROR = 0, /* Type not defined */ TYPE_AM_PRX = 129, /* PROXY access method type no */ TYPE_AM_XTB = 130, /* SYS table access method type */ TYPE_AM_BLK = 131, /* BLK access method type no */ - TYPE_AM_ZIP = 132, /* ZIP access method type no */ + TYPE_AM_GZ = 132, /* GZ access method type no */ TYPE_AM_ZLIB = 133, /* ZLIB access method type no */ TYPE_AM_JSON = 134, /* JSON access method type no */ TYPE_AM_JSN = 135, /* JSN access method type no */ @@ -140,7 +142,8 @@ enum AMT {TYPE_AM_ERROR = 0, /* Type not defined */ TYPE_AM_MYSQL = 192, /* MYSQL access method type no */ TYPE_AM_MYX = 193, /* MYSQL EXEC access method type */ TYPE_AM_CAT = 195, /* Catalog access method type no */ - TYPE_AM_OUT = 200}; /* Output relations (storage) */ + TYPE_AM_ZIP = 198, /* ZIP access method type no */ + TYPE_AM_OUT = 200}; /* Output relations (storage) */ enum RECFM {RECFM_NAF = -2, /* Not a file */ RECFM_OEM = -1, /* OEM file access method */ diff --git a/storage/connect/plgdbutl.cpp b/storage/connect/plgdbutl.cpp index 13c0dfd1e18..83975c6d8fa 100644 --- a/storage/connect/plgdbutl.cpp +++ b/storage/connect/plgdbutl.cpp @@ -68,6 +68,9 @@ #include "tabcol.h" // header of XTAB and COLUMN classes #include "valblk.h" #include "rcmsg.h" +#ifdef ZIP_SUPPORT +#include "filamzip.h" +#endif // ZIP_SUPPORT /***********************************************************************/ /* DB static variables. */ @@ -934,7 +937,16 @@ int PlugCloseFile(PGLOBAL g __attribute__((unused)), PFBLOCK fp, bool all) CloseXML2File(g, fp, all); break; #endif // LIBXML2_SUPPORT - default: +#ifdef ZIP_SUPPORT + case TYPE_FB_ZIP: + ((ZIPUTIL*)fp->File)->close(); + fp->Memory = NULL; + fp->Mode = MODE_ANY; + fp->Count = 0; + fp->File = NULL; + break; +#endif // ZIP_SUPPORT + default: rc = RC_FX; } // endswitch Type diff --git a/storage/connect/plgxml.cpp b/storage/connect/plgxml.cpp index 3061a6d697e..71b72621b06 100644 --- a/storage/connect/plgxml.cpp +++ b/storage/connect/plgxml.cpp @@ -30,19 +30,51 @@ PXDOC GetLibxmlDoc(PGLOBAL g, char *nsl, char *nsdf, /* XMLDOCUMENT constructor. */ /******************************************************************/ XMLDOCUMENT::XMLDOCUMENT(char *nsl, char *nsdf, char *enc) - { - Namespaces = NULL; +{ +#if defined(ZIP_SUPPORT) + zip = NULL; +#else // !ZIP_SUPPORT + zip = false; +#endif // !ZIP_SUPPORT + Namespaces = NULL; Encoding = enc; Nslist = nsl; DefNs = nsdf; - } // end of XMLDOCUMENT constructor +} // end of XMLDOCUMENT constructor + +/******************************************************************/ +/* Initialize zipped file processing. */ +/******************************************************************/ +bool XMLDOCUMENT::InitZip(PGLOBAL g, char *entry) +{ +#if defined(ZIP_SUPPORT) + bool mul = (entry) ? strchr(entry, '*') || strchr(entry, '?') : false; + zip = new(g) ZIPUTIL(entry, mul); + return zip == NULL; +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return true; +#endif // !ZIP_SUPPORT +} // end of InitZip + +/******************************************************************/ +/* Make the namespace structure list. */ +/******************************************************************/ +char* XMLDOCUMENT::GetMemDoc(PGLOBAL g, char *fn) +{ +#if defined(ZIP_SUPPORT) + return (zip->OpenTable(g, MODE_ANY, fn)) ? NULL : zip->memory; +#else // !ZIP_SUPPORT + return NULL; +#endif // !ZIP_SUPPORT +} // end of GetMemDoc /******************************************************************/ /* Make the namespace structure list. */ /******************************************************************/ bool XMLDOCUMENT::MakeNSlist(PGLOBAL g) - { - char *prefix, *href, *next = Nslist; +{ + char *prefix, *href, *next = Nslist; PNS nsp, *ppns = &Namespaces; while (next) { @@ -84,6 +116,19 @@ bool XMLDOCUMENT::MakeNSlist(PGLOBAL g) } // end of MakeNSlist /******************************************************************/ +/* Close ZIP file. */ +/******************************************************************/ +void XMLDOCUMENT::CloseZip(void) +{ +#if defined(ZIP_SUPPORT) + if (zip) { + zip->close(); + zip = NULL; + } // endif zip +#endif // ZIP_SUPPORT +} // end of CloseZip + +/******************************************************************/ /* XMLNODE constructor. */ /******************************************************************/ XMLNODE::XMLNODE(PXDOC dp) diff --git a/storage/connect/plgxml.h b/storage/connect/plgxml.h index b8e914e0bf1..db7dfa6bda5 100644 --- a/storage/connect/plgxml.h +++ b/storage/connect/plgxml.h @@ -1,3 +1,7 @@ +#if defined(ZIP_SUPPORT) +#include "filamzip.h" +#endif // ZIP_SUPPORT + /******************************************************************/ /* Dual XML implementation base classes defines. */ /******************************************************************/ @@ -72,8 +76,8 @@ class XMLDOCUMENT : public BLOCK { virtual void SetNofree(bool b) = 0; // Methods - virtual bool Initialize(PGLOBAL) = 0; - virtual bool ParseFile(char *) = 0; + virtual bool Initialize(PGLOBAL, char *, bool) = 0; + virtual bool ParseFile(PGLOBAL, char *) = 0; virtual bool NewDoc(PGLOBAL, char *) = 0; virtual void AddComment(PGLOBAL, char *) = 0; virtual PXNODE GetRoot(PGLOBAL) = 0; @@ -91,8 +95,16 @@ class XMLDOCUMENT : public BLOCK { // Utility bool MakeNSlist(PGLOBAL g); + bool InitZip(PGLOBAL g, char *entry); + char *GetMemDoc(PGLOBAL g, char *fn); + void CloseZip(void); // Members +#if defined(ZIP_SUPPORT) + ZIPUTIL *zip; /* Used for zipped file */ +#else // !ZIP_SUPPORT + bool zip; /* Always false */ +#endif // !ZIP_SUPPORT PNS Namespaces; /* To the namespaces */ char *Encoding; /* The document encoding */ char *Nslist; /* Namespace list */ diff --git a/storage/connect/reldef.cpp b/storage/connect/reldef.cpp index ac2327212e0..30e4d49d249 100644 --- a/storage/connect/reldef.cpp +++ b/storage/connect/reldef.cpp @@ -1,7 +1,7 @@ /************* RelDef CPP Program Source Code File (.CPP) **************/ /* PROGRAM NAME: RELDEF */ /* ------------- */ -/* Version 1.5 */ +/* Version 1.6 */ /* */ /* COPYRIGHT: */ /* ---------- */ @@ -40,10 +40,12 @@ #include "tabcol.h" #include "filamap.h" #include "filamfix.h" +#if defined(VCT_SUPPORT) #include "filamvct.h" -#if defined(ZIP_SUPPORT) -#include "filamzip.h" -#endif // ZIP_SUPPORT +#endif // VCT_SUPPORT +#if defined(GZ_SUPPORT) +#include "filamgz.h" +#endif // GZ_SUPPORT #include "tabdos.h" #include "valblk.h" #include "tabmul.h" @@ -663,15 +665,15 @@ PTDB OEMDEF::GetTable(PGLOBAL g, MODE mode) /*********************************************************************/ if (!((PTDBDOS)tdbp)->GetTxfp()) { if (cmpr) { -#if defined(ZIP_SUPPORT) +#if defined(GZ_SUPPORT) if (cmpr == 1) - txfp = new(g) ZIPFAM(defp); + txfp = new(g) GZFAM(defp); else txfp = new(g) ZLBFAM(defp); -#else // !ZIP_SUPPORT +#else // !GZ_SUPPORT strcpy(g->Message, "Compress not supported"); return NULL; -#endif // !ZIP_SUPPORT +#endif // !GZ_SUPPORT } else if (rfm == RECFM_VAR) { if (map) txfp = new(g) MAPFAM(defp); @@ -683,16 +685,19 @@ PTDB OEMDEF::GetTable(PGLOBAL g, MODE mode) txfp = new(g) MPXFAM(defp); else txfp = new(g) FIXFAM(defp); - } else if (rfm == RECFM_VCT) { - assert (Pxdef->GetDefType() == TYPE_AM_VCT); +#if defined(VCT_SUPPORT) + assert(Pxdef->GetDefType() == TYPE_AM_VCT); if (map) txfp = new(g) VCMFAM((PVCTDEF)defp); else txfp = new(g) VCTFAM((PVCTDEF)defp); - - } // endif's +#else // !VCT_SUPPORT + strcpy(g->Message, "VCT no more supported"); + return NULL; +#endif // !VCT_SUPPORT + } // endif's ((PTDBDOS)tdbp)->SetTxfp(txfp); } // endif Txfp diff --git a/storage/connect/tabdos.cpp b/storage/connect/tabdos.cpp index 98633f49d23..16cc6c33b44 100644 --- a/storage/connect/tabdos.cpp +++ b/storage/connect/tabdos.cpp @@ -1,11 +1,11 @@ /************* TabDos C++ Program Source Code File (.CPP) **************/ /* PROGRAM NAME: TABDOS */ /* ------------- */ -/* Version 4.9 */ +/* Version 4.9.2 */ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 1998-2015 */ +/* (C) Copyright to the author Olivier BERTRAND 1998-2016 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -51,6 +51,9 @@ #include "filamap.h" #include "filamfix.h" #include "filamdbf.h" +#if defined(GZ_SUPPORT) +#include "filamgz.h" +#endif // GZ_SUPPORT #if defined(ZIP_SUPPORT) #include "filamzip.h" #endif // ZIP_SUPPORT @@ -93,10 +96,13 @@ DOSDEF::DOSDEF(void) Pseudo = 3; Fn = NULL; Ofn = NULL; + Entry = NULL; To_Indx = NULL; Recfm = RECFM_VAR; Mapped = false; - Padded = false; + Zipped = false; + Mulentries = false; + Padded = false; Huge = false; Accept = false; Eof = false; @@ -126,6 +132,11 @@ bool DOSDEF::DefineAM(PGLOBAL g, LPCSTR am, int) : (am && (*am == 'B' || *am == 'b')) ? "B" : (am && !stricmp(am, "DBF")) ? "D" : "V"; + if ((Zipped = GetBoolCatInfo("Zipped", false))) + Mulentries = ((Entry = GetStringCatInfo(g, "Entry", NULL))) + ? strchr(Entry, '*') || strchr(Entry, '?') + : GetBoolCatInfo("Mulentries", false); + Desc = Fn = GetStringCatInfo(g, "Filename", NULL); Ofn = GetStringCatInfo(g, "Optname", Fn); GetCharCatInfo("Recfm", (PSZ)dfm, buf, sizeof(buf)); @@ -333,7 +344,21 @@ PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode) /* Allocate table and file processing class of the proper type. */ /* Column blocks will be allocated only when needed. */ /*********************************************************************/ - if (Recfm == RECFM_DBF) { + if (Zipped) { +#if defined(ZIP_SUPPORT) + if (Recfm == RECFM_VAR) { + txfp = new(g)ZIPFAM(this); + tdbp = new(g)TDBDOS(this, txfp); + } else { + txfp = new(g)ZPXFAM(this); + tdbp = new(g)TDBFIX(this, txfp); + } // endif Recfm + +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif // !ZIP_SUPPORT + } else if (Recfm == RECFM_DBF) { if (Catfunc == FNC_NO) { if (map) txfp = new(g) DBMFAM(this); @@ -350,28 +375,28 @@ PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode) else if (map) txfp = new(g) MPXFAM(this); else if (Compressed) { -#if defined(ZIP_SUPPORT) +#if defined(GZ_SUPPORT) txfp = new(g) ZIXFAM(this); -#else // !ZIP_SUPPORT - sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); +#else // !GZ_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "GZ"); return NULL; -#endif // !ZIP_SUPPORT +#endif // !GZ_SUPPORT } else txfp = new(g) FIXFAM(this); tdbp = new(g) TDBFIX(this, txfp); } else { if (Compressed) { -#if defined(ZIP_SUPPORT) +#if defined(GZ_SUPPORT) if (Compressed == 1) - txfp = new(g) ZIPFAM(this); + txfp = new(g) GZFAM(this); else txfp = new(g) ZLBFAM(this); -#else // !ZIP_SUPPORT - sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); +#else // !GZ_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "GZ"); return NULL; -#endif // !ZIP_SUPPORT +#endif // !GZ_SUPPORT } else if (map) txfp = new(g) MAPFAM(this); else @@ -396,7 +421,7 @@ PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode) if (map) { txfp = new(g) MBKFAM(this); } else if (Compressed) { -#if defined(ZIP_SUPPORT) +#if defined(GZ_SUPPORT) if (Compressed == 1) txfp = new(g) ZBKFAM(this); else { @@ -404,7 +429,7 @@ PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode) ((PZLBFAM)txfp)->SetOptimized(To_Pos != NULL); } // endelse #else - sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "GZ"); return NULL; #endif } else @@ -531,14 +556,14 @@ int TDBDOS::ResetTableOpt(PGLOBAL g, bool dop, bool dox) // except for ZLIB access method. if (Txfp->GetAmType() == TYPE_AM_MAP) { Txfp = new(g) MAPFAM((PDOSDEF)To_Def); -#if defined(ZIP_SUPPORT) - } else if (Txfp->GetAmType() == TYPE_AM_ZIP) { - Txfp = new(g) ZIPFAM((PDOSDEF)To_Def); +#if defined(GZ_SUPPORT) + } else if (Txfp->GetAmType() == TYPE_AM_GZ) { + Txfp = new(g) GZFAM((PDOSDEF)To_Def); } else if (Txfp->GetAmType() == TYPE_AM_ZLIB) { Txfp->Reset(); ((PZLBFAM)Txfp)->SetOptimized(false); -#endif // ZIP_SUPPORT - } else if (Txfp->GetAmType() == TYPE_AM_BLK) +#endif // GZ_SUPPORT + } else if (Txfp->GetAmType() == TYPE_AM_BLK) Txfp = new(g) DOSFAM((PDOSDEF)To_Def); Txfp->SetTdbp(this); @@ -609,7 +634,12 @@ int TDBDOS::MakeBlockValues(PGLOBAL g) defp->SetOptimized(0); // Estimate the number of needed blocks - block = (int)((MaxSize + (int)nrec - 1) / (int)nrec); + if ((block = (int)((MaxSize + (int)nrec - 1) / (int)nrec)) < 2) { + // This may be wrong to do in some cases + defp->RemoveOptValues(g); + strcpy(g->Message, MSG(TABLE_NOT_OPT)); + return RC_INFO; // Not to be optimized + } // endif block // We have to use local variables because Txfp->CurBlk is set // to Rows+1 by unblocked variable length table access methods. @@ -952,13 +982,14 @@ bool TDBDOS::GetBlockValues(PGLOBAL g) PCOLDEF cdp; PDOSDEF defp = (PDOSDEF)To_Def; PCATLG cat = defp->GetCat(); + PDBUSER dup = PlgGetUser(g); #if 0 if (Mode == MODE_INSERT && Txfp->GetAmType() == TYPE_AM_DOS) return false; #endif // __WIN__ - if (defp->Optimized) + if (defp->Optimized || !(dup->Check & CHK_OPT)) return false; // Already done or to be redone if (Ftype == RECFM_VAR || defp->Compressed == 2) { @@ -2079,10 +2110,10 @@ bool TDBDOS::OpenDB(PGLOBAL g) /*******************************************************************/ if (Txfp->GetAmType() == TYPE_AM_MAP && Mode == MODE_DELETE) Txfp = new(g) MAPFAM((PDOSDEF)To_Def); -#if defined(ZIP_SUPPORT) - else if (Txfp->GetAmType() == TYPE_AM_ZIP) - Txfp = new(g) ZIPFAM((PDOSDEF)To_Def); -#endif // ZIP_SUPPORT +#if defined(GZ_SUPPORT) + else if (Txfp->GetAmType() == TYPE_AM_GZ) + Txfp = new(g) GZFAM((PDOSDEF)To_Def); +#endif // GZ_SUPPORT else // if (Txfp->GetAmType() != TYPE_AM_DOS) ??? Txfp = new(g) DOSFAM((PDOSDEF)To_Def); diff --git a/storage/connect/tabdos.h b/storage/connect/tabdos.h index c098886f14b..4c8eb438a26 100644 --- a/storage/connect/tabdos.h +++ b/storage/connect/tabdos.h @@ -28,6 +28,7 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */ friend class TDBFIX; friend class TXTFAM; friend class DBFBASE; + friend class ZIPUTIL; public: // Constructor DOSDEF(void); @@ -40,7 +41,9 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */ virtual bool IsHuge(void) {return Huge;} PSZ GetFn(void) {return Fn;} PSZ GetOfn(void) {return Ofn;} - void SetBlock(int block) {Block = block;} + PSZ GetEntry(void) {return Entry;} + bool GetMul(void) {return Mulentries;} + void SetBlock(int block) {Block = block;} int GetBlock(void) {return Block;} int GetLast(void) {return Last;} void SetLast(int last) {Last = last;} @@ -57,9 +60,9 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */ int *GetTo_Pos(void) {return To_Pos;} // Methods - virtual int Indexable(void) - {return (!Multiple && Compressed != 1) ? 1 : 0;} - virtual bool DeleteIndexFile(PGLOBAL g, PIXDEF pxdf); + virtual int Indexable(void) + {return (!Multiple && !Mulentries && Compressed != 1) ? 1 : 0;} + virtual bool DeleteIndexFile(PGLOBAL g, PIXDEF pxdf); virtual bool DefineAM(PGLOBAL g, LPCSTR am, int poff); virtual PTDB GetTable(PGLOBAL g, MODE mode); bool InvalidateIndex(PGLOBAL g); @@ -72,10 +75,13 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */ // Members PSZ Fn; /* Path/Name of corresponding file */ PSZ Ofn; /* Base Path/Name of matching index files*/ - PIXDEF To_Indx; /* To index definitions blocks */ + PSZ Entry; /* Zip entry name or pattern */ + PIXDEF To_Indx; /* To index definitions blocks */ RECFM Recfm; /* 0:VAR, 1:FIX, 2:BIN, 3:VCT, 6:DBF */ bool Mapped; /* 0: disk file, 1: memory mapped file */ - bool Padded; /* true for padded table file */ + bool Zipped; /* true for zipped table file */ + bool Mulentries; /* true for multiple entries */ + bool Padded; /* true for padded table file */ bool Huge; /* true for files larger than 2GB */ bool Accept; /* true if wrong lines are accepted */ bool Eof; /* true if an EOF (0xA) character exists */ @@ -91,7 +97,7 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */ int Maxerr; /* Maximum number of bad records (DBF) */ int ReadMode; /* Specific to DBF */ int Ending; /* Length of end of lines */ - int Teds; /* Binary table default endian setting */ + char Teds; /* Binary table default endian setting */ }; // end of DOSDEF /***********************************************************************/ diff --git a/storage/connect/tabfix.cpp b/storage/connect/tabfix.cpp index 55c254f41ea..d99f7800f26 100644 --- a/storage/connect/tabfix.cpp +++ b/storage/connect/tabfix.cpp @@ -1,11 +1,11 @@ /************* TabFix C++ Program Source Code File (.CPP) **************/ /* PROGRAM NAME: TABFIX */ /* ------------- */ -/* Version 4.9 */ +/* Version 4.9.1 */ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 1998-2015 */ +/* (C) Copyright to the author Olivier BERTRAND 1998-2016 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -589,9 +589,10 @@ void BINCOL::WriteColumn(PGLOBAL g) switch (Fmt) { case 'X': // Standard not converted values - if (Eds && IsTypeChar(Buf_Type)) - *(longlong *)p = Value->GetBigintValue(); - else if (Value->GetBinValue(p, Long, Status)) { + if (Eds && IsTypeChar(Buf_Type)) { + if (Status) + Value->GetValueNonAligned<longlong>(p, Value->GetBigintValue()); + } else if (Value->GetBinValue(p, Long, Status)) { sprintf(g->Message, MSG(BIN_F_TOO_LONG), Name, Value->GetSize(), Long); longjmp(g->jumper[g->jump_level], 31); @@ -605,7 +606,7 @@ void BINCOL::WriteColumn(PGLOBAL g) sprintf(g->Message, MSG(VALUE_TOO_BIG), n, Name); longjmp(g->jumper[g->jump_level], 31); } else if (Status) - *(short *)p = (short)n; + Value->GetValueNonAligned<short>(p, (short)n); break; case 'T': // Tiny integer @@ -625,7 +626,7 @@ void BINCOL::WriteColumn(PGLOBAL g) sprintf(g->Message, MSG(VALUE_TOO_BIG), n, Name); longjmp(g->jumper[g->jump_level], 31); } else if (Status) - *(int *)p = Value->GetIntValue(); + Value->GetValueNonAligned<int>(p, (int)n); break; case 'G': // Large (great) integer @@ -636,12 +637,12 @@ void BINCOL::WriteColumn(PGLOBAL g) case 'F': // Float case 'R': // Real if (Status) - *(float *)p = (float)Value->GetFloatValue(); + Value->GetValueNonAligned<float>(p, (float)Value->GetFloatValue()); break; case 'D': // Double if (Status) - *(double *)p = Value->GetFloatValue(); + Value->GetValueNonAligned<double>(p, Value->GetFloatValue()); break; case 'C': // Characters diff --git a/storage/connect/tabfmt.cpp b/storage/connect/tabfmt.cpp index 015f8d93b15..b24375443f6 100644 --- a/storage/connect/tabfmt.cpp +++ b/storage/connect/tabfmt.cpp @@ -1,11 +1,11 @@ /************* TabFmt C++ Program Source Code File (.CPP) **************/ /* PROGRAM NAME: TABFMT */ /* ------------- */ -/* Version 3.9 */ +/* Version 3.9.2 */ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 2001 - 2015 */ +/* (C) Copyright to the author Olivier BERTRAND 2001 - 2016 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -51,6 +51,9 @@ #include "plgdbsem.h" #include "mycat.h" #include "filamap.h" +#if defined(GZ_SUPPORT) +#include "filamgz.h" +#endif // GZ_SUPPORT #if defined(ZIP_SUPPORT) #include "filamzip.h" #endif // ZIP_SUPPORT @@ -78,20 +81,24 @@ USETEMP UseTemp(void); /* of types (TYPE_STRING < TYPE_DOUBLE < TYPE_INT) (1 < 2 < 7). */ /* If these values are changed, this will have to be revisited. */ /***********************************************************************/ -PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, - char q, int hdr, int mxr, bool info) +PQRYRES CSVColumns(PGLOBAL g, char *dp, PTOS topt, bool info) { static int buftyp[] = {TYPE_STRING, TYPE_SHORT, TYPE_STRING, TYPE_INT, TYPE_INT, TYPE_SHORT}; static XFLD fldtyp[] = {FLD_NAME, FLD_TYPE, FLD_TYPENAME, FLD_PREC, FLD_LENGTH, FLD_SCALE}; static unsigned int length[] = {6, 6, 8, 10, 10, 6}; - char *p, *colname[MAXCOL], dechar, filename[_MAX_PATH], buf[4096]; + const char *fn; + char sep, q; + int rc, mxr; + bool hdr; + char *p, *colname[MAXCOL], dechar, buf[8]; int i, imax, hmax, n, nerr, phase, blank, digit, dec, type; int ncol = sizeof(buftyp) / sizeof(int); int num_read = 0, num_max = 10000000; // Statistics int len[MAXCOL], typ[MAXCOL], prc[MAXCOL]; - FILE *infile; + PCSVDEF tdp; + PTDBCSV tdbp; PQRYRES qrp; PCOLRES crp; @@ -101,27 +108,13 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, goto skipit; } // endif info -// num_max = atoi(p+1); // Max num of record to test -#if defined(__WIN__) - if (sep == ',' || strnicmp(setlocale(LC_NUMERIC, NULL), "French", 6)) - dechar = '.'; - else - dechar = ','; -#else // !__WIN__ - dechar = '.'; -#endif // !__WIN__ - - if (trace) - htrc("File %s sep=%c q=%c hdr=%d mxr=%d\n", - SVP(fn), sep, q, hdr, mxr); - - if (!fn) { - strcpy(g->Message, MSG(MISSING_FNAME)); - return NULL; - } // endif fn + if (GetIntegerTableOption(g, topt, "Multiple", 0)) { + strcpy(g->Message, "Cannot find column definition for multiple table"); + return NULL; + } // endif Multiple +// num_max = atoi(p+1); // Max num of record to test imax = hmax = nerr = 0; - mxr = MY_MAX(0, mxr); for (i = 0; i < MAXCOL; i++) { colname[i] = NULL; @@ -131,12 +124,76 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, } // endfor i /*********************************************************************/ - /* Open the input file. */ + /* Get the CSV table description block. */ /*********************************************************************/ - PlugSetPath(filename, fn, dp); + tdp = new(g) CSVDEF; +#if defined(ZIP_SUPPORT) + tdp->Entry = GetStringTableOption(g, topt, "Entry", NULL); + tdp->Zipped = GetBooleanTableOption(g, topt, "Zipped", false); +#endif // ZIP_SUPPORT + fn = tdp->Fn = GetStringTableOption(g, topt, "Filename", NULL); + + if (!tdp->Fn) { + strcpy(g->Message, MSG(MISSING_FNAME)); + return NULL; + } // endif Fn + + if (!(tdp->Lrecl = GetIntegerTableOption(g, topt, "Lrecl", 0))) + tdp->Lrecl = 4096; + + p = GetStringTableOption(g, topt, "Separator", ","); + tdp->Sep = (strlen(p) == 2 && p[0] == '\\' && p[1] == 't') ? '\t' : *p; + +#if defined(__WIN__) + if (tdp->Sep == ',' || strnicmp(setlocale(LC_NUMERIC, NULL), "French", 6)) + dechar = '.'; + else + dechar = ','; +#else // !__WIN__ + dechar = '.'; +#endif // !__WIN__ + + sep = tdp->Sep; + tdp->Quoted = GetIntegerTableOption(g, topt, "Quoted", -1); + p = GetStringTableOption(g, topt, "Qchar", ""); + tdp->Qot = *p; + + if (tdp->Qot && tdp->Quoted < 0) + tdp->Quoted = 0; + else if (!tdp->Qot && tdp->Quoted >= 0) + tdp->Qot = '"'; - if (!(infile= global_fopen(g, MSGID_CANNOT_OPEN, filename, "r"))) - return NULL; + q = tdp->Qot; + hdr = GetBooleanTableOption(g, topt, "Header", false); + tdp->Maxerr = GetIntegerTableOption(g, topt, "Maxerr", 0); + tdp->Accept = GetBooleanTableOption(g, topt, "Accept", false); + + if (tdp->Accept && tdp->Maxerr == 0) + tdp->Maxerr = INT_MAX32; // Accept all bad lines + + mxr = MY_MAX(0, tdp->Maxerr); + + if (trace) + htrc("File %s Sep=%c Qot=%c Header=%d maxerr=%d\n", + SVP(tdp->Fn), tdp->Sep, tdp->Qot, tdp->Header, tdp->Maxerr); + + if (tdp->Zipped) { +#if defined(ZIP_SUPPORT) + tdbp = new(g)TDBCSV(tdp, new(g)ZIPFAM(tdp)); +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif // !ZIP_SUPPORT + } else + tdbp = new(g) TDBCSV(tdp, new(g) DOSFAM(tdp)); + + tdbp->SetMode(MODE_READ); + + /*********************************************************************/ + /* Open the CSV file. */ + /*********************************************************************/ + if (tdbp->OpenDB(g)) + return NULL; if (hdr) { /*******************************************************************/ @@ -144,16 +201,8 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, /*******************************************************************/ phase = 0; - if (fgets(buf, sizeof(buf), infile)) { - n = strlen(buf) + 1; - buf[n - 2] = '\0'; -#if !defined(__WIN__) - // The file can be imported from Windows - if (buf[n - 3] == '\r') - buf[n - 3] = 0; -#endif // UNIX - p = (char*)PlugSubAlloc(g, NULL, n); - memcpy(p, buf, n); + if ((rc = tdbp->ReadDB(g)) == RC_OK) { + p = PlgDBDup(g, tdbp->To_Line); //skip leading blanks for (; *p == ' '; p++) ; @@ -165,10 +214,11 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, } // endif q colname[0] = p; - } else { + } else if (rc == RC_EF) { sprintf(g->Message, MSG(FILE_IS_EMPTY), fn); goto err; - } // endif's + } else + goto err; for (i = 1; *p; p++) if (phase == 1 && *p == q) { @@ -201,15 +251,8 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, /*******************************************************************/ /* Now start the reading process. Read one line. */ /*******************************************************************/ - if (fgets(buf, sizeof(buf), infile)) { - n = strlen(buf); - buf[n - 1] = '\0'; -#if !defined(__WIN__) - // The file can be imported from Windows - if (buf[n - 2] == '\r') - buf[n - 2] = 0; -#endif // UNIX - } else if (feof(infile)) { + if ((rc = tdbp->ReadDB(g)) == RC_OK) { + } else if (rc == RC_EF) { sprintf(g->Message, MSG(EOF_AFTER_LINE), num_read -1); break; } else { @@ -222,7 +265,7 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, /*******************************************************************/ i = n = phase = blank = digit = dec = 0; - for (p = buf; *p; p++) + for (p = tdbp->To_Line; *p; p++) if (*p == sep) { if (phase != 1) { if (i == MAXCOL - 1) { @@ -331,7 +374,7 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, htrc("\n"); } // endif trace - fclose(infile); + tdbp->CloseDB(g); skipit: if (trace) @@ -381,7 +424,7 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, return qrp; err: - fclose(infile); + tdbp->CloseDB(g); return NULL; } // end of CSVCColumns @@ -458,20 +501,27 @@ PTDB CSVDEF::GetTable(PGLOBAL g, MODE mode) /*******************************************************************/ /* Allocate a file processing class of the proper type. */ /*******************************************************************/ - if (map) { + if (Zipped) { +#if defined(ZIP_SUPPORT) + txfp = new(g) ZIPFAM(this); +#else // !ZIP_SUPPORT + strcpy(g->Message, "ZIP not supported"); + return NULL; +#endif // !ZIP_SUPPORT + } else if (map) { // Should be now compatible with UNIX txfp = new(g) MAPFAM(this); } else if (Compressed) { -#if defined(ZIP_SUPPORT) +#if defined(GZ_SUPPORT) if (Compressed == 1) - txfp = new(g) ZIPFAM(this); + txfp = new(g) GZFAM(this); else txfp = new(g) ZLBFAM(this); -#else // !ZIP_SUPPORT +#else // !GZ_SUPPORT strcpy(g->Message, "Compress not supported"); return NULL; -#endif // !ZIP_SUPPORT +#endif // !GZ_SUPPORT } else txfp = new(g) DOSFAM(this); @@ -498,7 +548,7 @@ PTDB CSVDEF::GetTable(PGLOBAL g, MODE mode) if (map) { txfp = new(g) MBKFAM(this); } else if (Compressed) { -#if defined(ZIP_SUPPORT) +#if defined(GZ_SUPPORT) if (Compressed == 1) txfp = new(g) ZBKFAM(this); else { @@ -506,7 +556,7 @@ PTDB CSVDEF::GetTable(PGLOBAL g, MODE mode) ((PZLBFAM)txfp)->SetOptimized(To_Pos != NULL); } // endelse #else - sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "GZ"); return NULL; #endif } else @@ -830,8 +880,9 @@ bool TDBCSV::SkipHeader(PGLOBAL g) /***********************************************************************/ int TDBCSV::ReadBuffer(PGLOBAL g) { - char *p1, *p2, *p = NULL; - int i, n, len, rc = Txfp->ReadBuffer(g); + //char *p1, *p2, *p = NULL; + char *p2, *p = NULL; + int i, n, len, rc = Txfp->ReadBuffer(g); bool bad = false; if (trace > 1) @@ -846,14 +897,23 @@ int TDBCSV::ReadBuffer(PGLOBAL g) for (i = 0; i < Fields; i++) { if (!bad) { if (Qot && *p2 == Qot) { // Quoted field - for (n = 0, p1 = ++p2; (p = strchr(p1, Qot)); p1 = p + 2) - if (*(p + 1) == Qot) - n++; // Doubled internal quotes - else - break; // Final quote + //for (n = 0, p1 = ++p2; (p = strchr(p1, Qot)); p1 = p + 2) + // if (*(p + 1) == Qot) + // n++; // Doubled internal quotes + // else + // break; // Final quote + + for (n = 0, p = ++p2; p; p++) + if (*p == Qot || *p == '\\') { + if (*(++p) == Qot) + n++; // Escaped internal quotes + else if (*(p - 1) == Qot) + break; // Final quote + } // endif *p if (p) { - len = p++ - p2; + //len = p++ - p2; + len = p - p2 - 1;; // if (Sep != ' ') // for (; *p == ' '; p++) ; // Skip blanks @@ -873,10 +933,12 @@ int TDBCSV::ReadBuffer(PGLOBAL g) if (n) { int j, k; - // Suppress the double of internal quotes + // Suppress the escape of internal quotes for (j = k = 0; j < len; j++, k++) { - if (p2[j] == Qot) - j++; // skip first one + if (p2[j] == Qot || (p2[j] == '\\' && p2[j + 1] == Qot)) + j++; // skip escape char + else if (p2[j] == '\\') + p2[k++] = p2[j++]; // avoid \\Qot p2[k] = p2[j]; } // endfor i, j @@ -1464,21 +1526,16 @@ void CSVCOL::WriteColumn(PGLOBAL g) /* TDBCCL class constructor. */ /***********************************************************************/ TDBCCL::TDBCCL(PCSVDEF tdp) : TDBCAT(tdp) - { - Fn = tdp->GetFn(); - Hdr = tdp->Header; - Mxr = tdp->Maxerr; - Qtd = tdp->Quoted; - Sep = tdp->Sep; - } // end of TDBCCL constructor +{ + Topt = tdp->GetTopt(); +} // end of TDBCCL constructor /***********************************************************************/ /* GetResult: Get the list the CSV file columns. */ /***********************************************************************/ PQRYRES TDBCCL::GetResult(PGLOBAL g) { - return CSVColumns(g, ((PTABDEF)To_Def)->GetPath(), - Fn, Sep, Qtd, Hdr, Mxr, false); + return CSVColumns(g, ((PTABDEF)To_Def)->GetPath(), Topt, false); } // end of GetResult /* ------------------------ End of TabFmt ---------------------------- */ diff --git a/storage/connect/tabfmt.h b/storage/connect/tabfmt.h index ce80a276cdc..5ce8d399a64 100644 --- a/storage/connect/tabfmt.h +++ b/storage/connect/tabfmt.h @@ -1,7 +1,7 @@ /*************** TabFmt H Declares Source Code File (.H) ***************/ -/* Name: TABFMT.H Version 2.4 */ +/* Name: TABFMT.H Version 2.5 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 2001-2014 */ +/* (C) Copyright to the author Olivier BERTRAND 2001-2016 */ /* */ /* This file contains the CSV and FMT classes declares. */ /***********************************************************************/ @@ -13,8 +13,7 @@ typedef class TDBFMT *PTDBFMT; /***********************************************************************/ /* Functions used externally. */ /***********************************************************************/ -PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, - char q, int hdr, int mxr, bool info); +PQRYRES CSVColumns(PGLOBAL g, char *dp, PTOS topt, bool info); /***********************************************************************/ /* CSV table. */ @@ -22,7 +21,8 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, class DllExport CSVDEF : public DOSDEF { /* Logical table description */ friend class TDBCSV; friend class TDBCCL; - public: + friend PQRYRES CSVColumns(PGLOBAL, char *, PTOS, bool); +public: // Constructor CSVDEF(void); @@ -50,9 +50,10 @@ class DllExport CSVDEF : public DOSDEF { /* Logical table description */ /* This is the DOS/UNIX Access Method class declaration for files */ /* that are CSV files with columns separated by the Sep character. */ /***********************************************************************/ -class TDBCSV : public TDBDOS { +class DllExport TDBCSV : public TDBDOS { friend class CSVCOL; - public: + friend PQRYRES CSVColumns(PGLOBAL, char *, PTOS, bool); +public: // Constructor TDBCSV(PCSVDEF tdp, PTXF txfp); TDBCSV(PGLOBAL g, PTDBCSV tdbp); @@ -101,7 +102,7 @@ class TDBCSV : public TDBDOS { /* Class CSVCOL: CSV access method column descriptor. */ /* This A.M. is used for Comma Separated V(?) files. */ /***********************************************************************/ -class CSVCOL : public DOSCOL { +class DllExport CSVCOL : public DOSCOL { friend class TDBCSV; friend class TDBFMT; public: @@ -129,7 +130,7 @@ class CSVCOL : public DOSCOL { /* This is the DOS/UNIX Access Method class declaration for files */ /* whose record format is described by a Format keyword. */ /***********************************************************************/ -class TDBFMT : public TDBCSV { +class DllExport TDBFMT : public TDBCSV { friend class CSVCOL; //friend class FMTCOL; public: @@ -173,7 +174,7 @@ class TDBFMT : public TDBCSV { /***********************************************************************/ /* This is the class declaration for the CSV catalog table. */ /***********************************************************************/ -class TDBCCL : public TDBCAT { +class DllExport TDBCCL : public TDBCAT { public: // Constructor TDBCCL(PCSVDEF tdp); @@ -183,11 +184,7 @@ class TDBCCL : public TDBCAT { virtual PQRYRES GetResult(PGLOBAL g); // Members - char *Fn; // The CSV file (path) name - bool Hdr; // true if first line contains headers - int Mxr; // Maximum number of bad records - int Qtd; // Quoting level for quoted fields - char Sep; // Separator for standard CSV files - }; // end of class TDBCCL + PTOS Topt; +}; // end of class TDBCCL /* ------------------------- End of TabFmt.H ------------------------- */ diff --git a/storage/connect/tabjdbc.cpp b/storage/connect/tabjdbc.cpp index e398523892f..912e6c7d530 100644 --- a/storage/connect/tabjdbc.cpp +++ b/storage/connect/tabjdbc.cpp @@ -110,6 +110,7 @@ bool JDBCDEF::SetParms(PJPARM sjp) sjp->Url= Url; sjp->User= Username; sjp->Pwd= Password; +//sjp->Properties = Prop; return true; } // end of SetParms @@ -234,6 +235,7 @@ bool JDBCDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) Read_Only = true; Wrapname = GetStringCatInfo(g, "Wrapper", NULL); +//Prop = GetStringCatInfo(g, "Properties", NULL); Tabcat = GetStringCatInfo(g, "Qualifier", NULL); Tabcat = GetStringCatInfo(g, "Catalog", Tabcat); Tabschema = GetStringCatInfo(g, "Dbname", NULL); @@ -337,6 +339,7 @@ TDBJDBC::TDBJDBC(PJDBCDEF tdp) : TDBASE(tdp) Schema = tdp->Tabschema; Ops.User = tdp->Username; Ops.Pwd = tdp->Password; +// Ops.Properties = tdp->Prop; Catalog = tdp->Tabcat; Srcdef = tdp->Srcdef; Qrystr = tdp->Qrystr; @@ -356,6 +359,7 @@ TDBJDBC::TDBJDBC(PJDBCDEF tdp) : TDBASE(tdp) Ops.Url = NULL; Ops.User = NULL; Ops.Pwd = NULL; +// Ops.Properties = NULL; Catalog = NULL; Srcdef = NULL; Qrystr = NULL; diff --git a/storage/connect/tabjdbc.h b/storage/connect/tabjdbc.h index 7244ebd3832..fee8223abaf 100644 --- a/storage/connect/tabjdbc.h +++ b/storage/connect/tabjdbc.h @@ -58,6 +58,7 @@ protected: PSZ Tabschema; /* External table schema */ PSZ Username; /* User connect name */ PSZ Password; /* Password connect info */ +//PSZ Prop; /* Connection Properties */ PSZ Tabcat; /* External table catalog */ PSZ Tabtype; /* External table type */ PSZ Colpat; /* Catalog column pattern */ diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp index 17260836371..1b9ce8b64c9 100644 --- a/storage/connect/tabjson.cpp +++ b/storage/connect/tabjson.cpp @@ -1,6 +1,6 @@ /************* tabjson C++ Program Source Code File (.CPP) *************/ -/* PROGRAM NAME: tabjson Version 1.1 */ -/* (C) Copyright to the author Olivier BERTRAND 2014 - 2015 */ +/* PROGRAM NAME: tabjson Version 1.3 */ +/* (C) Copyright to the author Olivier BERTRAND 2014 - 2016 */ /* This program are the JSON class DB execution routines. */ /***********************************************************************/ @@ -25,6 +25,9 @@ //#include "resource.h" // for IDS_COLUMNS #include "tabjson.h" #include "filamap.h" +#if defined(GZ_SUPPORT) +#include "filamgz.h" +#endif // GZ_SUPPORT #if defined(ZIP_SUPPORT) #include "filamzip.h" #endif // ZIP_SUPPORT @@ -67,7 +70,7 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info) static XFLD fldtyp[] = {FLD_NAME, FLD_TYPE, FLD_TYPENAME, FLD_PREC, FLD_LENGTH, FLD_SCALE, FLD_NULL, FLD_FORMAT}; static unsigned int length[] = {0, 6, 8, 10, 10, 6, 6, 0}; - char *fn, colname[65], fmt[129]; + char colname[65], fmt[129]; int i, j, lvl, n = 0; int ncol = sizeof(buftyp) / sizeof(int); PVAL valp; @@ -91,19 +94,29 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info) goto skipit; } // endif info - /*********************************************************************/ + if (GetIntegerTableOption(g, topt, "Multiple", 0)) { + strcpy(g->Message, "Cannot find column definition for multiple table"); + return NULL; + } // endif Multiple + + /*********************************************************************/ /* Open the input file. */ /*********************************************************************/ - if (!(fn = GetStringTableOption(g, topt, "Filename", NULL))) { - strcpy(g->Message, MSG(MISSING_FNAME)); - return NULL; - } else { - lvl = GetIntegerTableOption(g, topt, "Level", 0); - lvl = (lvl < 0) ? 0 : (lvl > 16) ? 16 : lvl; - } // endif fn + lvl = GetIntegerTableOption(g, topt, "Level", 0); + lvl = (lvl < 0) ? 0 : (lvl > 16) ? 16 : lvl; tdp = new(g) JSONDEF; - tdp->Fn = fn; +#if defined(ZIP_SUPPORT) + tdp->Entry = GetStringTableOption(g, topt, "Entry", NULL); + tdp->Zipped = GetBooleanTableOption(g, topt, "Zipped", false); +#endif // ZIP_SUPPORT + tdp->Fn = GetStringTableOption(g, topt, "Filename", NULL); + + if (!tdp->Fn) { + strcpy(g->Message, MSG(MISSING_FNAME)); + return NULL; + } // endif Fn + tdp->Database = SetPath(g, db); tdp->Objname = GetStringTableOption(g, topt, "Object", NULL); tdp->Base = GetIntegerTableOption(g, topt, "Base", 0) ? 1 : 0; @@ -114,7 +127,15 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info) tdp->Fn, tdp->Objname, tdp->Pretty, lvl); if (tdp->Pretty == 2) { - tjsp = new(g) TDBJSON(tdp, new(g) MAPFAM(tdp)); + if (tdp->Zipped) { +#if defined(ZIP_SUPPORT) + tjsp = new(g) TDBJSON(tdp, new(g) ZIPFAM(tdp)); +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif // !ZIP_SUPPORT + } else + tjsp = new(g) TDBJSON(tdp, new(g) MAPFAM(tdp)); if (tjsp->MakeDocument(g)) return NULL; @@ -127,10 +148,33 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info) } // endif lrecl tdp->Ending = GetIntegerTableOption(g, topt, "Ending", CRLF); - tjnp = new(g) TDBJSN(tdp, new(g) DOSFAM(tdp)); + + if (tdp->Zipped) { +#if defined(ZIP_SUPPORT) + tjnp = new(g)TDBJSN(tdp, new(g)ZIPFAM(tdp)); +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif // !ZIP_SUPPORT + } else + tjnp = new(g) TDBJSN(tdp, new(g) DOSFAM(tdp)); + tjnp->SetMode(MODE_READ); - if (tjnp->OpenDB(g)) +#if USE_G + // Allocate the parse work memory + PGLOBAL G = (PGLOBAL)PlugSubAlloc(g, NULL, sizeof(GLOBAL)); + memset(G, 0, sizeof(GLOBAL)); + G->Sarea_Size = tdp->Lrecl * 10; + G->Sarea = PlugSubAlloc(g, NULL, G->Sarea_Size); + PlugSubSet(G, G->Sarea, G->Sarea_Size); + G->jump_level = -1; + tjnp->SetG(G); +#else + tjnp->SetG(g); +#endif + + if (tjnp->OpenDB(g)) return NULL; switch (tjnp->ReadDB(g)) { @@ -395,16 +439,23 @@ PTDB JSONDEF::GetTable(PGLOBAL g, MODE m) !(tmp == TMP_FORCE && (m == MODE_UPDATE || m == MODE_DELETE)); - if (Compressed) { + if (Zipped) { #if defined(ZIP_SUPPORT) + txfp = new(g) ZIPFAM(this); +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif // !ZIP_SUPPORT + } else if (Compressed) { +#if defined(GZ_SUPPORT) if (Compressed == 1) - txfp = new(g) ZIPFAM(this); + txfp = new(g) GZFAM(this); else txfp = new(g) ZLBFAM(this); -#else // !ZIP_SUPPORT - sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); +#else // !GZ_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "GZ"); return NULL; -#endif // !ZIP_SUPPORT +#endif // !GZ_SUPPORT } else if (map) txfp = new(g) MAPFAM(this); else @@ -426,7 +477,16 @@ PTDB JSONDEF::GetTable(PGLOBAL g, MODE m) ((TDBJSN*)tdbp)->G = g; #endif } else { - txfp = new(g) MAPFAM(this); + if (Zipped) { +#if defined(ZIP_SUPPORT) + txfp = new(g)ZIPFAM(this); +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif // !ZIP_SUPPORT + } else + txfp = new(g) MAPFAM(this); + tdbp = new(g) TDBJSON(this, txfp); ((TDBJSON*)tdbp)->G = g; } // endif Pretty diff --git a/storage/connect/tabjson.h b/storage/connect/tabjson.h index f7cb74c3c4d..c9d30d48f2a 100644 --- a/storage/connect/tabjson.h +++ b/storage/connect/tabjson.h @@ -78,7 +78,8 @@ public: virtual AMT GetAmType(void) {return TYPE_AM_JSN;} virtual bool SkipHeader(PGLOBAL g); virtual PTDB Duplicate(PGLOBAL g) {return (PTDB)new(g) TDBJSN(this);} - PJSON GetRow(void) {return Row;} + PJSON GetRow(void) {return Row;} + void SetG(PGLOBAL g) {G = g;} // Methods virtual PTDB CopyOne(PTABS t); diff --git a/storage/connect/tabxml.cpp b/storage/connect/tabxml.cpp index 57d204a4286..3b8229fcf51 100644 --- a/storage/connect/tabxml.cpp +++ b/storage/connect/tabxml.cpp @@ -1,9 +1,9 @@ /************* Tabxml C++ Program Source Code File (.CPP) **************/ /* PROGRAM NAME: TABXML */ /* ------------- */ -/* Version 2.8 */ +/* Version 2.9 */ /* */ -/* Author Olivier BERTRAND 2007 - 2015 */ +/* Author Olivier BERTRAND 2007 - 2016 */ /* */ /* This program are the XML tables classes using MS-DOM or libxml2. */ /***********************************************************************/ @@ -136,7 +136,12 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info) goto skipit; } // endif info - /*********************************************************************/ + if (GetIntegerTableOption(g, topt, "Multiple", 0)) { + strcpy(g->Message, "Cannot find column definition for multiple table"); + return NULL; + } // endif Multiple + + /*********************************************************************/ /* Open the input file. */ /*********************************************************************/ if (!(fn = GetStringTableOption(g, topt, "Filename", NULL))) { @@ -154,6 +159,8 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info) tdp->Fn = fn; tdp->Database = SetPath(g, db); tdp->Tabname = tab; + tdp->Zipped = GetBooleanTableOption(g, topt, "Zipped", false); + tdp->Entry = GetStringTableOption(g, topt, "Entry", NULL); if (!(op = GetStringTableOption(g, topt, "Xmlsup", NULL))) #if defined(__WIN__) @@ -204,7 +211,8 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info) while (true) { if (!vp->atp && - !(node = (vp->nl) ? vp->nl->GetItem(g, vp->k++, node) : NULL)) + !(node = (vp->nl) ? vp->nl->GetItem(g, vp->k++, tdp->Usedom ? node : NULL) + : NULL)) if (j) { vp = lvlp[--j]; @@ -254,7 +262,8 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info) if (j < lvl && ok) { vp = lvlp[j+1]; vp->k = 0; - vp->atp = node->GetAttribute(g, NULL); + vp->pn = node; + vp->atp = node->GetAttribute(g, NULL); vp->nl = node->GetChildElements(g); if (tdp->Usedom && vp->nl->GetLength() == 1) { @@ -265,7 +274,7 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info) if (vp->atp || vp->b) { if (!vp->atp) - node = vp->nl->GetItem(g, vp->k++, node); + node = vp->nl->GetItem(g, vp->k++, tdp->Usedom ? node : NULL); strncat(fmt, colname, XLEN(fmt)); strncat(fmt, "/", XLEN(fmt)); @@ -424,11 +433,14 @@ XMLDEF::XMLDEF(void) DefNs = NULL; Attrib = NULL; Hdattr = NULL; + Entry = NULL; Coltype = 1; Limit = 0; Header = 0; Xpand = false; Usedom = false; + Zipped = false; + Mulentries = false; } // end of XMLDEF constructor /***********************************************************************/ @@ -507,7 +519,14 @@ bool XMLDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) // Get eventual table node attribute Attrib = GetStringCatInfo(g, "Attribute", NULL); Hdattr = GetStringCatInfo(g, "HeadAttr", NULL); - return false; + + // Specific for zipped files + if ((Zipped = GetBoolCatInfo("Zipped", false))) + Mulentries = ((Entry = GetStringCatInfo(g, "Entry", NULL))) + ? strchr(Entry, '*') || strchr(Entry, '?') + : GetBoolCatInfo("Mulentries", false); + + return false; } // end of DefineAM /***********************************************************************/ @@ -547,6 +566,7 @@ TDBXML::TDBXML(PXMLDEF tdp) : TDBASE(tdp) Xfile = tdp->Fn; Enc = tdp->Encoding; Tabname = tdp->Tabname; +#if 0 // why all these? Rowname = (tdp->Rowname) ? tdp->Rowname : NULL; Colname = (tdp->Colname) ? tdp->Colname : NULL; Mulnode = (tdp->Mulnode) ? tdp->Mulnode : NULL; @@ -555,10 +575,22 @@ TDBXML::TDBXML(PXMLDEF tdp) : TDBASE(tdp) DefNs = (tdp->DefNs) ? tdp->DefNs : NULL; Attrib = (tdp->Attrib) ? tdp->Attrib : NULL; Hdattr = (tdp->Hdattr) ? tdp->Hdattr : NULL; - Coltype = tdp->Coltype; +#endif // 0 + Rowname = tdp->Rowname; + Colname = tdp->Colname; + Mulnode = tdp->Mulnode; + XmlDB = tdp->XmlDB; + Nslist = tdp->Nslist; + DefNs = tdp->DefNs; + Attrib = tdp->Attrib; + Hdattr = tdp->Hdattr; + Entry = tdp->Entry; + Coltype = tdp->Coltype; Limit = tdp->Limit; Xpand = tdp->Xpand; - Changed = false; + Zipped = tdp->Zipped; + Mulentries = tdp->Mulentries; + Changed = false; Checked = false; NextSame = false; NewRow = false; @@ -600,10 +632,13 @@ TDBXML::TDBXML(PTDBXML tdbp) : TDBASE(tdbp) DefNs = tdbp->DefNs; Attrib = tdbp->Attrib; Hdattr = tdbp->Hdattr; - Coltype = tdbp->Coltype; + Entry = tdbp->Entry; + Coltype = tdbp->Coltype; Limit = tdbp->Limit; Xpand = tdbp->Xpand; - Changed = tdbp->Changed; + Zipped = tdbp->Zipped; + Mulentries = tdbp->Mulentries; + Changed = tdbp->Changed; Checked = tdbp->Checked; NextSame = tdbp->NextSame; NewRow = tdbp->NewRow; @@ -681,7 +716,7 @@ int TDBXML::LoadTableFile(PGLOBAL g, char *filename) /*********************************************************************/ /* Firstly we check whether this file have been already loaded. */ /*********************************************************************/ - if (Mode == MODE_READ || Mode == MODE_ANY) + if ((Mode == MODE_READ || Mode == MODE_ANY) && !Zipped) for (fp = dup->Openlist; fp; fp = fp->Next) if (fp->Type == type && fp->Length && fp->Count) if (!stricmp(fp->Fname, filename)) @@ -703,7 +738,7 @@ int TDBXML::LoadTableFile(PGLOBAL g, char *filename) return RC_FX; // Initialize the implementation - if (Docp->Initialize(g)) { + if (Docp->Initialize(g, Entry, Zipped)) { sprintf(g->Message, MSG(INIT_FAILED), (Usedom) ? "DOM" : "libxml2"); return RC_FX; } // endif init @@ -712,7 +747,7 @@ int TDBXML::LoadTableFile(PGLOBAL g, char *filename) htrc("TDBXML: parsing %s rc=%d\n", filename, rc); // Parse the XML file - if (Docp->ParseFile(filename)) { + if (Docp->ParseFile(g, filename)) { // Does the file exist? int h= global_open(g, MSGID_NONE, filename, _O_RDONLY); diff --git a/storage/connect/tabxml.h b/storage/connect/tabxml.h index 7ba3166881d..6c586d79dec 100644 --- a/storage/connect/tabxml.h +++ b/storage/connect/tabxml.h @@ -1,7 +1,7 @@ /*************** Tabxml H Declares Source Code File (.H) ***************/ -/* Name: TABXML.H Version 1.6 */ +/* Name: TABXML.H Version 1.7 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 2007-2015 */ +/* (C) Copyright to the author Olivier BERTRAND 2007-2016 */ /* */ /* This file contains the XML table classes declares. */ /***********************************************************************/ @@ -42,12 +42,15 @@ class DllExport XMLDEF : public TABDEF { /* Logical table description */ char *DefNs; /* Dummy name of default namespace */ char *Attrib; /* Table node attributes */ char *Hdattr; /* Header node attributes */ - int Coltype; /* Default column type */ + char *Entry; /* Zip entry name or pattern */ + int Coltype; /* Default column type */ int Limit; /* Limit of multiple values */ int Header; /* n first rows are header rows */ bool Xpand; /* Put multiple tags in several rows */ bool Usedom; /* True: DOM, False: libxml2 */ - }; // end of XMLDEF + bool Zipped; /* True: Zipped XML file(s) */ + bool Mulentries; /* True: multiple entries in zip file*/ +}; // end of XMLDEF #if defined(INCLUDE_TDBXML) /***********************************************************************/ @@ -122,7 +125,9 @@ class DllExport TDBXML : public TDBASE { bool Bufdone; // True when column buffers allocated bool Nodedone; // True when column nodes allocated bool Void; // True if the file does not exist - char *Xfile; // The XML file + bool Zipped; // True if Zipped XML file(s) + bool Mulentries; // True if multiple entries in zip file + char *Xfile; // The XML file char *Enc; // New XML table file encoding char *Tabname; // Name of Table node char *Rowname; // Name of first level nodes @@ -133,7 +138,8 @@ class DllExport TDBXML : public TDBASE { char *DefNs; // Dummy name of default namespace char *Attrib; // Table node attribut(s) char *Hdattr; // Header node attribut(s) - int Coltype; // Default column type + char *Entry; // Zip entry name or pattern + int Coltype; // Default column type int Limit; // Limit of multiple values int Header; // n first rows are header rows int Multiple; // If multiple files diff --git a/storage/connect/tabzip.cpp b/storage/connect/tabzip.cpp new file mode 100644 index 00000000000..11f414ee154 --- /dev/null +++ b/storage/connect/tabzip.cpp @@ -0,0 +1,230 @@ +/************* TabZip C++ Program Source Code File (.CPP) **************/ +/* PROGRAM NAME: TABZIP Version 1.0 */ +/* (C) Copyright to the author Olivier BERTRAND 2016 */ +/* This program are the TABZIP class DB execution routines. */ +/***********************************************************************/ + +/***********************************************************************/ +/* Include relevant sections of the MariaDB header file. */ +/***********************************************************************/ +#include <my_global.h> + +/***********************************************************************/ +/* Include application header files: */ +/* global.h is header containing all global declarations. */ +/* plgdbsem.h is header containing the DB application declarations. */ +/* (x)table.h is header containing the TDBASE declarations. */ +/* tabzip.h is header containing the TABZIP classes declarations. */ +/***********************************************************************/ +#include "global.h" +#include "plgdbsem.h" +#include "xtable.h" +#include "filamtxt.h" +#include "filamzip.h" +#include "resource.h" // for IDS_COLUMNS +#include "tabdos.h" +#include "tabzip.h" + +/* -------------------------- Class ZIPDEF --------------------------- */ + +/************************************************************************/ +/* DefineAM: define specific AM block values. */ +/************************************************************************/ +bool ZIPDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) +{ +//target = GetStringCatInfo(g, "Target", NULL); + return DOSDEF::DefineAM(g, "ZIP", poff); +} // end of DefineAM + +/***********************************************************************/ +/* GetTable: makes a new Table Description Block. */ +/***********************************************************************/ +PTDB ZIPDEF::GetTable(PGLOBAL g, MODE m) +{ + return new(g) TDBZIP(this); +} // end of GetTable + +/* ------------------------------------------------------------------- */ + +/***********************************************************************/ +/* Implementation of the TDBZIP class. */ +/***********************************************************************/ +TDBZIP::TDBZIP(PZIPDEF tdp) : TDBASE(tdp) +{ + zipfile = NULL; + zfn = tdp->Fn; +//target = tdp->target; + nexterr = UNZ_OK; +} // end of TDBZIP standard constructor + +/***********************************************************************/ +/* Allocate ZIP column description block. */ +/***********************************************************************/ +PCOL TDBZIP::MakeCol(PGLOBAL g, PCOLDEF cdp, PCOL cprec, int n) +{ + return new(g) ZIPCOL(cdp, this, cprec, n); +} // end of MakeCol + +/***********************************************************************/ +/* open a zip file. */ +/* param: filename path and the filename of the zip file to open. */ +/* return: true if open, false otherwise. */ +/***********************************************************************/ +bool TDBZIP::open(PGLOBAL g, const char *filename) +{ + if (!zipfile && !(zipfile = unzOpen64(filename))) + sprintf(g->Message, "Zipfile open error"); + + return (zipfile == NULL); +} // end of open + +/***********************************************************************/ +/* Close the zip file. */ +/***********************************************************************/ +void TDBZIP::close() +{ + if (zipfile) { + unzClose(zipfile); + zipfile = NULL; + } // endif zipfile + +} // end of close + +/***********************************************************************/ +/* ZIP Cardinality: returns table size in number of rows. */ +/***********************************************************************/ +int TDBZIP::Cardinality(PGLOBAL g) +{ + if (!g) + return 1; + else if (Cardinal < 0) { + if (!open(g, zfn)) { + unz_global_info64 ginfo; + int err = unzGetGlobalInfo64(zipfile, &ginfo); + + Cardinal = (err == UNZ_OK) ? ginfo.number_entry : 0; + } else + Cardinal = 0; + + } // endif Cardinal + + return Cardinal; +} // end of Cardinality + +/***********************************************************************/ +/* ZIP GetMaxSize: returns file size estimate in number of lines. */ +/***********************************************************************/ +int TDBZIP::GetMaxSize(PGLOBAL g) +{ + if (MaxSize < 0) + MaxSize = Cardinality(g); + + return MaxSize; +} // end of GetMaxSize + +/***********************************************************************/ +/* ZIP Access Method opening routine. */ +/***********************************************************************/ +bool TDBZIP::OpenDB(PGLOBAL g) +{ + if (Use == USE_OPEN) + // Table already open + return false; + + Use = USE_OPEN; // To be clean + return open(g, zfn); +} // end of OpenDB + +/***********************************************************************/ +/* ReadDB: Data Base read routine for ZIP access method. */ +/***********************************************************************/ +int TDBZIP::ReadDB(PGLOBAL g) +{ + if (nexterr == UNZ_END_OF_LIST_OF_FILE) + return RC_EF; + else if (nexterr != UNZ_OK) { + sprintf(g->Message, "unzGoToNextFile error %d", nexterr); + return RC_FX; + } // endif nexterr + + int err = unzGetCurrentFileInfo64(zipfile, &finfo, fn, + sizeof(fn), NULL, 0, NULL, 0); + + if (err != UNZ_OK) { + sprintf(g->Message, "unzGetCurrentFileInfo64 error %d", err); + return RC_FX; + } // endif err + + nexterr = unzGoToNextFile(zipfile); + return RC_OK; +} // end of ReadDB + +/***********************************************************************/ +/* WriteDB: Data Base write routine for ZIP access method. */ +/***********************************************************************/ +int TDBZIP::WriteDB(PGLOBAL g) +{ + strcpy(g->Message, "ZIP tables are read only"); + return RC_FX; +} // end of WriteDB + +/***********************************************************************/ +/* Data Base delete line routine for ZIP access method. */ +/***********************************************************************/ +int TDBZIP::DeleteDB(PGLOBAL g, int irc) +{ + strcpy(g->Message, "Delete not enabled for ZIP tables"); + return RC_FX; +} // end of DeleteDB + +/***********************************************************************/ +/* Data Base close routine for ZIP access method. */ +/***********************************************************************/ +void TDBZIP::CloseDB(PGLOBAL g) +{ + close(); + Use = USE_READY; // Just to be clean +} // end of CloseDB + +/* ---------------------------- ZIPCOL ------------------------------- */ + +/***********************************************************************/ +/* ZIPCOL public constructor. */ +/***********************************************************************/ +ZIPCOL::ZIPCOL(PCOLDEF cdp, PTDB tdbp, PCOL cprec, int i, PSZ am) + : COLBLK(cdp, tdbp, i) +{ + if (cprec) { + Next = cprec->GetNext(); + cprec->SetNext(this); + } else { + Next = tdbp->GetColumns(); + tdbp->SetColumns(this); + } // endif cprec + + Tdbz = (TDBZIP*)tdbp; + flag = cdp->GetOffset(); +} // end of ZIPCOL constructor + +/***********************************************************************/ +/* ReadColumn: */ +/***********************************************************************/ +void ZIPCOL::ReadColumn(PGLOBAL g) +{ + switch (flag) { + case 1: + Value->SetValue(Tdbz->finfo.compressed_size); + break; + case 2: + Value->SetValue(Tdbz->finfo.uncompressed_size); + break; + case 3: + Value->SetValue((int)Tdbz->finfo.compression_method); + break; + default: + Value->SetValue_psz((PSZ)Tdbz->fn); + } // endswitch flag + +} // end of ReadColumn + +/* -------------------------- End of tabzip -------------------------- */ diff --git a/storage/connect/tabzip.h b/storage/connect/tabzip.h new file mode 100644 index 00000000000..6f1735258e7 --- /dev/null +++ b/storage/connect/tabzip.h @@ -0,0 +1,100 @@ +/*************** tabzip H Declares Source Code File (.H) ***************/ +/* Name: tabzip.h Version 1.0 */ +/* */ +/* (C) Copyright to the author Olivier BERTRAND 2016 */ +/* */ +/* This file contains the ZIP classe declares. */ +/***********************************************************************/ +#include "osutil.h" +#include "block.h" +#include "colblk.h" +#include "xtable.h" +#include "unzip.h" + +typedef class ZIPDEF *PZIPDEF; +typedef class TDBZIP *PTDBZIP; +typedef class ZIPCOL *PZIPCOL; + +/***********************************************************************/ +/* ZIP table: display info about a ZIP file. */ +/***********************************************************************/ +class DllExport ZIPDEF : public DOSDEF { /* Table description */ + friend class TDBZIP; + friend class ZIPFAM; +public: + // Constructor + ZIPDEF(void) {} + + // Implementation + virtual const char *GetType(void) {return "ZIP";} + + // Methods + virtual bool DefineAM(PGLOBAL g, LPCSTR am, int poff); + virtual PTDB GetTable(PGLOBAL g, MODE m); + +protected: + // Members + PSZ target; // The inside file to query +}; // end of ZIPDEF + +/***********************************************************************/ +/* This is the ZIP Access Method class declaration. */ +/***********************************************************************/ +class DllExport TDBZIP : public TDBASE { + friend class ZIPCOL; +public: + // Constructor + TDBZIP(PZIPDEF tdp); + + // Implementation + virtual AMT GetAmType(void) {return TYPE_AM_ZIP;} + + // Methods + virtual PCOL MakeCol(PGLOBAL g, PCOLDEF cdp, PCOL cprec, int n); + virtual int Cardinality(PGLOBAL g); + virtual int GetMaxSize(PGLOBAL g); + virtual int GetRecpos(void) {return 0;} + + // Database routines + virtual bool OpenDB(PGLOBAL g); + virtual int ReadDB(PGLOBAL g); + virtual int WriteDB(PGLOBAL g); + virtual int DeleteDB(PGLOBAL g, int irc); + virtual void CloseDB(PGLOBAL g); + +protected: + bool open(PGLOBAL g, const char *filename); + void close(void); + + // Members + unzFile zipfile; // The ZIP container file + PSZ zfn; // The ZIP file name +//PSZ target; + unz_file_info64 finfo; // The current file info + char fn[FILENAME_MAX]; // The current file name + int nexterr; // Next file error +}; // end of class TDBZIP + +/***********************************************************************/ +/* Class ZIPCOL: ZIP access method column descriptor. */ +/***********************************************************************/ +class DllExport ZIPCOL : public COLBLK { + friend class TDBZIP; +public: + // Constructors + ZIPCOL(PCOLDEF cdp, PTDB tdbp, PCOL cprec, int i, PSZ am = "ZIP"); + + // Implementation + virtual int GetAmType(void) { return TYPE_AM_ZIP; } + + // Methods + virtual void ReadColumn(PGLOBAL g); + +protected: + // Default constructor not to be used + ZIPCOL(void) {} + + // Members + TDBZIP *Tdbz; + int flag; +}; // end of class ZIPCOL diff --git a/storage/connect/unzip.c b/storage/connect/unzip.c new file mode 100644 index 00000000000..909350435a5 --- /dev/null +++ b/storage/connect/unzip.c @@ -0,0 +1,2125 @@ +/* unzip.c -- IO for uncompress .zip files using zlib + Version 1.1, February 14h, 2010 + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications of Unzip for Zip64 + Copyright (C) 2007-2008 Even Rouault + + Modifications for Zip64 support on both zip and unzip + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + + + ------------------------------------------------------------------------------------ + Decryption code comes from crypt.c by Info-ZIP but has been greatly reduced in terms of + compatibility with older software. The following is from the original crypt.c. + Code woven in by Terry Thorsen 1/2003. + + Copyright (c) 1990-2000 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2000-Apr-09 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, all these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html + + crypt.c (full version) by Info-ZIP. Last revised: [see crypt.h] + + The encryption/decryption parts of this source code (as opposed to the + non-echoing password parts) were originally written in Europe. The + whole source package can be freely distributed, including from the USA. + (Prior to January 2000, re-export from the US was a violation of US law.) + + This encryption code is a direct transcription of the algorithm from + Roger Schlafly, described by Phil Katz in the file appnote.txt. This + file (appnote.txt) is distributed with the PKZIP program (even in the + version without encryption capabilities). + + ------------------------------------------------------------------------------------ + + Changes in unzip.c + + 2007-2008 - Even Rouault - Addition of cpl_unzGetCurrentFileZStreamPos + 2007-2008 - Even Rouault - Decoration of symbol names unz* -> cpl_unz* + 2007-2008 - Even Rouault - Remove old C style function prototypes + 2007-2008 - Even Rouault - Add unzip support for ZIP64 + + Copyright (C) 2007-2008 Even Rouault + + + Oct-2009 - Mathias Svensson - Removed cpl_* from symbol names (Even Rouault added them but since this is now moved to a new project (minizip64) I renamed them again). + Oct-2009 - Mathias Svensson - Fixed problem if uncompressed size was > 4G and compressed size was <4G + should only read the compressed/uncompressed size from the Zip64 format if + the size from normal header was 0xFFFFFFFF + Oct-2009 - Mathias Svensson - Applied some bug fixes from paches recived from Gilles Vollant + Oct-2009 - Mathias Svensson - Applied support to unzip files with compression mathod BZIP2 (bzip2 lib is required) + Patch created by Daniel Borca + + Jan-2010 - back to unzip and minizip 1.0 name scheme, with compatibility layer + + Copyright (C) 1998 - 2010 Gilles Vollant, Even Rouault, Mathias Svensson + +*/ + + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#ifndef NOUNCRYPT + #define NOUNCRYPT +#endif + +#include "zlib.h" +#include "unzip.h" + +#ifdef STDC +# include <stddef.h> +# include <string.h> +# include <stdlib.h> +#endif +#ifdef NO_ERRNO_H + extern int errno; +#else +# include <errno.h> +#endif + + +#ifndef local +# define local static +#endif +/* compile with -Dlocal if your debugger can't find static symbols */ + + +#ifndef CASESENSITIVITYDEFAULT_NO +# if !defined(unix) && !defined(CASESENSITIVITYDEFAULT_YES) +# define CASESENSITIVITYDEFAULT_NO +# endif +#endif + + +#ifndef UNZ_BUFSIZE +#define UNZ_BUFSIZE (16384) +#endif + +#ifndef UNZ_MAXFILENAMEINZIP +#define UNZ_MAXFILENAMEINZIP (256) +#endif + +#ifndef ALLOC +# define ALLOC(size) (malloc(size)) +#endif +#ifndef TRYFREE +# define TRYFREE(p) {if (p) free(p);} +#endif + +#define SIZECENTRALDIRITEM (0x2e) +#define SIZEZIPLOCALHEADER (0x1e) + + +const char unz_copyright[] = + " unzip 1.01 Copyright 1998-2004 Gilles Vollant - http://www.winimage.com/zLibDll"; + +/* unz_file_info_interntal contain internal info about a file in zipfile*/ +typedef struct unz_file_info64_internal_s +{ + ZPOS64_T offset_curfile;/* relative offset of local header 8 bytes */ +} unz_file_info64_internal; + + +/* file_in_zip_read_info_s contain internal information about a file in zipfile, + when reading and decompress it */ +typedef struct +{ + char *read_buffer; /* internal buffer for compressed data */ + z_stream stream; /* zLib stream structure for inflate */ + +#ifdef HAVE_BZIP2 + bz_stream bstream; /* bzLib stream structure for bziped */ +#endif + + ZPOS64_T pos_in_zipfile; /* position in byte on the zipfile, for fseek*/ + uLong stream_initialised; /* flag set if stream structure is initialised*/ + + ZPOS64_T offset_local_extrafield;/* offset of the local extra field */ + uInt size_local_extrafield;/* size of the local extra field */ + ZPOS64_T pos_local_extrafield; /* position in the local extra field in read*/ + ZPOS64_T total_out_64; + + uLong crc32; /* crc32 of all data uncompressed */ + uLong crc32_wait; /* crc32 we must obtain after decompress all */ + ZPOS64_T rest_read_compressed; /* number of byte to be decompressed */ + ZPOS64_T rest_read_uncompressed;/*number of byte to be obtained after decomp*/ + zlib_filefunc64_32_def z_filefunc; + voidpf filestream; /* io structore of the zipfile */ + uLong compression_method; /* compression method (0==store) */ + ZPOS64_T byte_before_the_zipfile;/* byte before the zipfile, (>0 for sfx)*/ + int raw; +} file_in_zip64_read_info_s; + + +/* unz64_s contain internal information about the zipfile +*/ +typedef struct +{ + zlib_filefunc64_32_def z_filefunc; + int is64bitOpenFunction; + voidpf filestream; /* io structore of the zipfile */ + unz_global_info64 gi; /* public global information */ + ZPOS64_T byte_before_the_zipfile;/* byte before the zipfile, (>0 for sfx)*/ + ZPOS64_T num_file; /* number of the current file in the zipfile*/ + ZPOS64_T pos_in_central_dir; /* pos of the current file in the central dir*/ + ZPOS64_T current_file_ok; /* flag about the usability of the current file*/ + ZPOS64_T central_pos; /* position of the beginning of the central dir*/ + + ZPOS64_T size_central_dir; /* size of the central directory */ + ZPOS64_T offset_central_dir; /* offset of start of central directory with + respect to the starting disk number */ + + unz_file_info64 cur_file_info; /* public info about the current file in zip*/ + unz_file_info64_internal cur_file_info_internal; /* private info about it*/ + file_in_zip64_read_info_s* pfile_in_zip_read; /* structure about the current + file if we are decompressing it */ + int encrypted; + + int isZip64; + +# ifndef NOUNCRYPT + unsigned long keys[3]; /* keys defining the pseudo-random sequence */ + const z_crc_t* pcrc_32_tab; +# endif +} unz64_s; + + +#ifndef NOUNCRYPT +#include "crypt.h" +#endif + +/* =========================================================================== + Read a byte from a gz_stream; update next_in and avail_in. Return EOF + for end of file. + IN assertion: the stream s has been sucessfully opened for reading. +*/ + + +local int unz64local_getByte OF(( + const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + int *pi)); + +local int unz64local_getByte(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, int *pi) +{ + unsigned char c; + int err = (int)ZREAD64(*pzlib_filefunc_def,filestream,&c,1); + if (err==1) + { + *pi = (int)c; + return UNZ_OK; + } + else + { + if (ZERROR64(*pzlib_filefunc_def,filestream)) + return UNZ_ERRNO; + else + return UNZ_EOF; + } +} + + +/* =========================================================================== + Reads a long in LSB order from the given gz_stream. Sets +*/ +local int unz64local_getShort OF(( + const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + uLong *pX)); + +local int unz64local_getShort (const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + uLong *pX) +{ + uLong x ; + int i = 0; + int err; + + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x = (uLong)i; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((uLong)i)<<8; + + if (err==UNZ_OK) + *pX = x; + else + *pX = 0; + return err; +} + +local int unz64local_getLong OF(( + const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + uLong *pX)); + +local int unz64local_getLong (const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + uLong *pX) +{ + uLong x ; + int i = 0; + int err; + + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x = (uLong)i; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((uLong)i)<<8; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((uLong)i)<<16; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((uLong)i)<<24; + + if (err==UNZ_OK) + *pX = x; + else + *pX = 0; + return err; +} + +local int unz64local_getLong64 OF(( + const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + ZPOS64_T *pX)); + + +local int unz64local_getLong64 (const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + ZPOS64_T *pX) +{ + ZPOS64_T x ; + int i = 0; + int err; + + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x = (ZPOS64_T)i; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<8; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<16; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<24; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<32; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<40; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<48; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<56; + + if (err==UNZ_OK) + *pX = x; + else + *pX = 0; + return err; +} + +/* My own strcmpi / strcasecmp */ +local int strcmpcasenosensitive_internal (const char* fileName1, const char* fileName2) +{ + for (;;) + { + char c1=*(fileName1++); + char c2=*(fileName2++); + if ((c1>='a') && (c1<='z')) + c1 -= 0x20; + if ((c2>='a') && (c2<='z')) + c2 -= 0x20; + if (c1=='\0') + return ((c2=='\0') ? 0 : -1); + if (c2=='\0') + return 1; + if (c1<c2) + return -1; + if (c1>c2) + return 1; + } +} + + +#ifdef CASESENSITIVITYDEFAULT_NO +#define CASESENSITIVITYDEFAULTVALUE 2 +#else +#define CASESENSITIVITYDEFAULTVALUE 1 +#endif + +#ifndef STRCMPCASENOSENTIVEFUNCTION +#define STRCMPCASENOSENTIVEFUNCTION strcmpcasenosensitive_internal +#endif + +/* + Compare two filename (fileName1,fileName2). + If iCaseSenisivity = 1, comparision is case sensitivity (like strcmp) + If iCaseSenisivity = 2, comparision is not case sensitivity (like strcmpi + or strcasecmp) + If iCaseSenisivity = 0, case sensitivity is defaut of your operating system + (like 1 on Unix, 2 on Windows) + +*/ +extern int ZEXPORT unzStringFileNameCompare (const char* fileName1, + const char* fileName2, + int iCaseSensitivity) + +{ + if (iCaseSensitivity==0) + iCaseSensitivity=CASESENSITIVITYDEFAULTVALUE; + + if (iCaseSensitivity==1) + return strcmp(fileName1,fileName2); + + return STRCMPCASENOSENTIVEFUNCTION(fileName1,fileName2); +} + +#ifndef BUFREADCOMMENT +#define BUFREADCOMMENT (0x400) +#endif + +/* + Locate the Central directory of a zipfile (at the end, just before + the global comment) +*/ +local ZPOS64_T unz64local_SearchCentralDir OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream)); +local ZPOS64_T unz64local_SearchCentralDir(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream) +{ + unsigned char* buf; + ZPOS64_T uSizeFile; + ZPOS64_T uBackRead; + ZPOS64_T uMaxBack=0xffff; /* maximum size of global comment */ + ZPOS64_T uPosFound=0; + + if (ZSEEK64(*pzlib_filefunc_def,filestream,0,ZLIB_FILEFUNC_SEEK_END) != 0) + return 0; + + + uSizeFile = ZTELL64(*pzlib_filefunc_def,filestream); + + if (uMaxBack>uSizeFile) + uMaxBack = uSizeFile; + + buf = (unsigned char*)ALLOC(BUFREADCOMMENT+4); + if (buf==NULL) + return 0; + + uBackRead = 4; + while (uBackRead<uMaxBack) + { + uLong uReadSize; + ZPOS64_T uReadPos ; + int i; + if (uBackRead+BUFREADCOMMENT>uMaxBack) + uBackRead = uMaxBack; + else + uBackRead+=BUFREADCOMMENT; + uReadPos = uSizeFile-uBackRead ; + + uReadSize = ((BUFREADCOMMENT+4) < (uSizeFile-uReadPos)) ? + (BUFREADCOMMENT+4) : (uLong)(uSizeFile-uReadPos); + if (ZSEEK64(*pzlib_filefunc_def,filestream,uReadPos,ZLIB_FILEFUNC_SEEK_SET)!=0) + break; + + if (ZREAD64(*pzlib_filefunc_def,filestream,buf,uReadSize)!=uReadSize) + break; + + for (i=(int)uReadSize-3; (i--)>0;) + if (((*(buf+i))==0x50) && ((*(buf+i+1))==0x4b) && + ((*(buf+i+2))==0x05) && ((*(buf+i+3))==0x06)) + { + uPosFound = uReadPos+i; + break; + } + + if (uPosFound!=0) + break; + } + TRYFREE(buf); + return uPosFound; +} + + +/* + Locate the Central directory 64 of a zipfile (at the end, just before + the global comment) +*/ +local ZPOS64_T unz64local_SearchCentralDir64 OF(( + const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream)); + +local ZPOS64_T unz64local_SearchCentralDir64(const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream) +{ + unsigned char* buf; + ZPOS64_T uSizeFile; + ZPOS64_T uBackRead; + ZPOS64_T uMaxBack=0xffff; /* maximum size of global comment */ + ZPOS64_T uPosFound=0; + uLong uL; + ZPOS64_T relativeOffset; + + if (ZSEEK64(*pzlib_filefunc_def,filestream,0,ZLIB_FILEFUNC_SEEK_END) != 0) + return 0; + + + uSizeFile = ZTELL64(*pzlib_filefunc_def,filestream); + + if (uMaxBack>uSizeFile) + uMaxBack = uSizeFile; + + buf = (unsigned char*)ALLOC(BUFREADCOMMENT+4); + if (buf==NULL) + return 0; + + uBackRead = 4; + while (uBackRead<uMaxBack) + { + uLong uReadSize; + ZPOS64_T uReadPos; + int i; + if (uBackRead+BUFREADCOMMENT>uMaxBack) + uBackRead = uMaxBack; + else + uBackRead+=BUFREADCOMMENT; + uReadPos = uSizeFile-uBackRead ; + + uReadSize = ((BUFREADCOMMENT+4) < (uSizeFile-uReadPos)) ? + (BUFREADCOMMENT+4) : (uLong)(uSizeFile-uReadPos); + if (ZSEEK64(*pzlib_filefunc_def,filestream,uReadPos,ZLIB_FILEFUNC_SEEK_SET)!=0) + break; + + if (ZREAD64(*pzlib_filefunc_def,filestream,buf,uReadSize)!=uReadSize) + break; + + for (i=(int)uReadSize-3; (i--)>0;) + if (((*(buf+i))==0x50) && ((*(buf+i+1))==0x4b) && + ((*(buf+i+2))==0x06) && ((*(buf+i+3))==0x07)) + { + uPosFound = uReadPos+i; + break; + } + + if (uPosFound!=0) + break; + } + TRYFREE(buf); + if (uPosFound == 0) + return 0; + + /* Zip64 end of central directory locator */ + if (ZSEEK64(*pzlib_filefunc_def,filestream, uPosFound,ZLIB_FILEFUNC_SEEK_SET)!=0) + return 0; + + /* the signature, already checked */ + if (unz64local_getLong(pzlib_filefunc_def,filestream,&uL)!=UNZ_OK) + return 0; + + /* number of the disk with the start of the zip64 end of central directory */ + if (unz64local_getLong(pzlib_filefunc_def,filestream,&uL)!=UNZ_OK) + return 0; + if (uL != 0) + return 0; + + /* relative offset of the zip64 end of central directory record */ + if (unz64local_getLong64(pzlib_filefunc_def,filestream,&relativeOffset)!=UNZ_OK) + return 0; + + /* total number of disks */ + if (unz64local_getLong(pzlib_filefunc_def,filestream,&uL)!=UNZ_OK) + return 0; + if (uL != 1) + return 0; + + /* Goto end of central directory record */ + if (ZSEEK64(*pzlib_filefunc_def,filestream, relativeOffset,ZLIB_FILEFUNC_SEEK_SET)!=0) + return 0; + + /* the signature */ + if (unz64local_getLong(pzlib_filefunc_def,filestream,&uL)!=UNZ_OK) + return 0; + + if (uL != 0x06064b50) + return 0; + + return relativeOffset; +} + +/* + Open a Zip file. path contain the full pathname (by example, + on a Windows NT computer "c:\\test\\zlib114.zip" or on an Unix computer + "zlib/zlib114.zip". + If the zipfile cannot be opened (file doesn't exist or in not valid), the + return value is NULL. + Else, the return value is a unzFile Handle, usable with other function + of this unzip package. +*/ +local unzFile unzOpenInternal (const void *path, + zlib_filefunc64_32_def* pzlib_filefunc64_32_def, + int is64bitOpenFunction) +{ + unz64_s us; + unz64_s *s; + ZPOS64_T central_pos; + uLong uL; + + uLong number_disk; /* number of the current dist, used for + spaning ZIP, unsupported, always 0*/ + uLong number_disk_with_CD; /* number the the disk with central dir, used + for spaning ZIP, unsupported, always 0*/ + ZPOS64_T number_entry_CD; /* total number of entries in + the central dir + (same than number_entry on nospan) */ + + int err=UNZ_OK; + + if (unz_copyright[0]!=' ') + return NULL; + + us.z_filefunc.zseek32_file = NULL; + us.z_filefunc.ztell32_file = NULL; + if (pzlib_filefunc64_32_def==NULL) + fill_fopen64_filefunc(&us.z_filefunc.zfile_func64); + else + us.z_filefunc = *pzlib_filefunc64_32_def; + us.is64bitOpenFunction = is64bitOpenFunction; + + + + us.filestream = ZOPEN64(us.z_filefunc, + path, + ZLIB_FILEFUNC_MODE_READ | + ZLIB_FILEFUNC_MODE_EXISTING); + if (us.filestream==NULL) + return NULL; + + central_pos = unz64local_SearchCentralDir64(&us.z_filefunc,us.filestream); + if (central_pos) + { + uLong uS; + ZPOS64_T uL64; + + us.isZip64 = 1; + + if (ZSEEK64(us.z_filefunc, us.filestream, + central_pos,ZLIB_FILEFUNC_SEEK_SET)!=0) + err=UNZ_ERRNO; + + /* the signature, already checked */ + if (unz64local_getLong(&us.z_filefunc, us.filestream,&uL)!=UNZ_OK) + err=UNZ_ERRNO; + + /* size of zip64 end of central directory record */ + if (unz64local_getLong64(&us.z_filefunc, us.filestream,&uL64)!=UNZ_OK) + err=UNZ_ERRNO; + + /* version made by */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&uS)!=UNZ_OK) + err=UNZ_ERRNO; + + /* version needed to extract */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&uS)!=UNZ_OK) + err=UNZ_ERRNO; + + /* number of this disk */ + if (unz64local_getLong(&us.z_filefunc, us.filestream,&number_disk)!=UNZ_OK) + err=UNZ_ERRNO; + + /* number of the disk with the start of the central directory */ + if (unz64local_getLong(&us.z_filefunc, us.filestream,&number_disk_with_CD)!=UNZ_OK) + err=UNZ_ERRNO; + + /* total number of entries in the central directory on this disk */ + if (unz64local_getLong64(&us.z_filefunc, us.filestream,&us.gi.number_entry)!=UNZ_OK) + err=UNZ_ERRNO; + + /* total number of entries in the central directory */ + if (unz64local_getLong64(&us.z_filefunc, us.filestream,&number_entry_CD)!=UNZ_OK) + err=UNZ_ERRNO; + + if ((number_entry_CD!=us.gi.number_entry) || + (number_disk_with_CD!=0) || + (number_disk!=0)) + err=UNZ_BADZIPFILE; + + /* size of the central directory */ + if (unz64local_getLong64(&us.z_filefunc, us.filestream,&us.size_central_dir)!=UNZ_OK) + err=UNZ_ERRNO; + + /* offset of start of central directory with respect to the + starting disk number */ + if (unz64local_getLong64(&us.z_filefunc, us.filestream,&us.offset_central_dir)!=UNZ_OK) + err=UNZ_ERRNO; + + us.gi.size_comment = 0; + } + else + { + central_pos = unz64local_SearchCentralDir(&us.z_filefunc,us.filestream); + if (central_pos==0) + err=UNZ_ERRNO; + + us.isZip64 = 0; + + if (ZSEEK64(us.z_filefunc, us.filestream, + central_pos,ZLIB_FILEFUNC_SEEK_SET)!=0) + err=UNZ_ERRNO; + + /* the signature, already checked */ + if (unz64local_getLong(&us.z_filefunc, us.filestream,&uL)!=UNZ_OK) + err=UNZ_ERRNO; + + /* number of this disk */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&number_disk)!=UNZ_OK) + err=UNZ_ERRNO; + + /* number of the disk with the start of the central directory */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&number_disk_with_CD)!=UNZ_OK) + err=UNZ_ERRNO; + + /* total number of entries in the central dir on this disk */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&uL)!=UNZ_OK) + err=UNZ_ERRNO; + us.gi.number_entry = uL; + + /* total number of entries in the central dir */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&uL)!=UNZ_OK) + err=UNZ_ERRNO; + number_entry_CD = uL; + + if ((number_entry_CD!=us.gi.number_entry) || + (number_disk_with_CD!=0) || + (number_disk!=0)) + err=UNZ_BADZIPFILE; + + /* size of the central directory */ + if (unz64local_getLong(&us.z_filefunc, us.filestream,&uL)!=UNZ_OK) + err=UNZ_ERRNO; + us.size_central_dir = uL; + + /* offset of start of central directory with respect to the + starting disk number */ + if (unz64local_getLong(&us.z_filefunc, us.filestream,&uL)!=UNZ_OK) + err=UNZ_ERRNO; + us.offset_central_dir = uL; + + /* zipfile comment length */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&us.gi.size_comment)!=UNZ_OK) + err=UNZ_ERRNO; + } + + if ((central_pos<us.offset_central_dir+us.size_central_dir) && + (err==UNZ_OK)) + err=UNZ_BADZIPFILE; + + if (err!=UNZ_OK) + { + ZCLOSE64(us.z_filefunc, us.filestream); + return NULL; + } + + us.byte_before_the_zipfile = central_pos - + (us.offset_central_dir+us.size_central_dir); + us.central_pos = central_pos; + us.pfile_in_zip_read = NULL; + us.encrypted = 0; + + + s=(unz64_s*)ALLOC(sizeof(unz64_s)); + if( s != NULL) + { + *s=us; + unzGoToFirstFile((unzFile)s); + } + return (unzFile)s; +} + + +extern unzFile ZEXPORT unzOpen2 (const char *path, + zlib_filefunc_def* pzlib_filefunc32_def) +{ + if (pzlib_filefunc32_def != NULL) + { + zlib_filefunc64_32_def zlib_filefunc64_32_def_fill; + fill_zlib_filefunc64_32_def_from_filefunc32(&zlib_filefunc64_32_def_fill,pzlib_filefunc32_def); + return unzOpenInternal(path, &zlib_filefunc64_32_def_fill, 0); + } + else + return unzOpenInternal(path, NULL, 0); +} + +extern unzFile ZEXPORT unzOpen2_64 (const void *path, + zlib_filefunc64_def* pzlib_filefunc_def) +{ + if (pzlib_filefunc_def != NULL) + { + zlib_filefunc64_32_def zlib_filefunc64_32_def_fill; + zlib_filefunc64_32_def_fill.zfile_func64 = *pzlib_filefunc_def; + zlib_filefunc64_32_def_fill.ztell32_file = NULL; + zlib_filefunc64_32_def_fill.zseek32_file = NULL; + return unzOpenInternal(path, &zlib_filefunc64_32_def_fill, 1); + } + else + return unzOpenInternal(path, NULL, 1); +} + +extern unzFile ZEXPORT unzOpen (const char *path) +{ + return unzOpenInternal(path, NULL, 0); +} + +extern unzFile ZEXPORT unzOpen64 (const void *path) +{ + return unzOpenInternal(path, NULL, 1); +} + +/* + Close a ZipFile opened with unzOpen. + If there is files inside the .Zip opened with unzOpenCurrentFile (see later), + these files MUST be closed with unzCloseCurrentFile before call unzClose. + return UNZ_OK if there is no problem. */ +extern int ZEXPORT unzClose (unzFile file) +{ + unz64_s* s; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + + if (s->pfile_in_zip_read!=NULL) + unzCloseCurrentFile(file); + + ZCLOSE64(s->z_filefunc, s->filestream); + TRYFREE(s); + return UNZ_OK; +} + + +/* + Write info about the ZipFile in the *pglobal_info structure. + No preparation of the structure is needed + return UNZ_OK if there is no problem. */ +extern int ZEXPORT unzGetGlobalInfo64 (unzFile file, unz_global_info64* pglobal_info) +{ + unz64_s* s; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + *pglobal_info=s->gi; + return UNZ_OK; +} + +extern int ZEXPORT unzGetGlobalInfo (unzFile file, unz_global_info* pglobal_info32) +{ + unz64_s* s; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + /* to do : check if number_entry is not truncated */ + pglobal_info32->number_entry = (uLong)s->gi.number_entry; + pglobal_info32->size_comment = s->gi.size_comment; + return UNZ_OK; +} +/* + Translate date/time from Dos format to tm_unz (readable more easilty) +*/ +local void unz64local_DosDateToTmuDate (ZPOS64_T ulDosDate, tm_unz* ptm) +{ + ZPOS64_T uDate; + uDate = (ZPOS64_T)(ulDosDate>>16); + ptm->tm_mday = (uInt)(uDate&0x1f) ; + ptm->tm_mon = (uInt)((((uDate)&0x1E0)/0x20)-1) ; + ptm->tm_year = (uInt)(((uDate&0x0FE00)/0x0200)+1980) ; + + ptm->tm_hour = (uInt) ((ulDosDate &0xF800)/0x800); + ptm->tm_min = (uInt) ((ulDosDate&0x7E0)/0x20) ; + ptm->tm_sec = (uInt) (2*(ulDosDate&0x1f)) ; +} + +/* + Get Info about the current file in the zipfile, with internal only info +*/ +local int unz64local_GetCurrentFileInfoInternal OF((unzFile file, + unz_file_info64 *pfile_info, + unz_file_info64_internal + *pfile_info_internal, + char *szFileName, + uLong fileNameBufferSize, + void *extraField, + uLong extraFieldBufferSize, + char *szComment, + uLong commentBufferSize)); + +local int unz64local_GetCurrentFileInfoInternal (unzFile file, + unz_file_info64 *pfile_info, + unz_file_info64_internal + *pfile_info_internal, + char *szFileName, + uLong fileNameBufferSize, + void *extraField, + uLong extraFieldBufferSize, + char *szComment, + uLong commentBufferSize) +{ + unz64_s* s; + unz_file_info64 file_info; + unz_file_info64_internal file_info_internal; + int err=UNZ_OK; + uLong uMagic; + long lSeek=0; + uLong uL; + + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + if (ZSEEK64(s->z_filefunc, s->filestream, + s->pos_in_central_dir+s->byte_before_the_zipfile, + ZLIB_FILEFUNC_SEEK_SET)!=0) + err=UNZ_ERRNO; + + + /* we check the magic */ + if (err==UNZ_OK) + { + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uMagic) != UNZ_OK) + err=UNZ_ERRNO; + else if (uMagic!=0x02014b50) + err=UNZ_BADZIPFILE; + } + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.version) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.version_needed) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.flag) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.compression_method) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&file_info.dosDate) != UNZ_OK) + err=UNZ_ERRNO; + + unz64local_DosDateToTmuDate(file_info.dosDate,&file_info.tmu_date); + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&file_info.crc) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uL) != UNZ_OK) + err=UNZ_ERRNO; + file_info.compressed_size = uL; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uL) != UNZ_OK) + err=UNZ_ERRNO; + file_info.uncompressed_size = uL; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.size_filename) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.size_file_extra) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.size_file_comment) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.disk_num_start) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.internal_fa) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&file_info.external_fa) != UNZ_OK) + err=UNZ_ERRNO; + + // relative offset of local header + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uL) != UNZ_OK) + err=UNZ_ERRNO; + file_info_internal.offset_curfile = uL; + + lSeek+=file_info.size_filename; + if ((err==UNZ_OK) && (szFileName!=NULL)) + { + uLong uSizeRead ; + if (file_info.size_filename<fileNameBufferSize) + { + *(szFileName+file_info.size_filename)='\0'; + uSizeRead = file_info.size_filename; + } + else + uSizeRead = fileNameBufferSize; + + if ((file_info.size_filename>0) && (fileNameBufferSize>0)) + if (ZREAD64(s->z_filefunc, s->filestream,szFileName,uSizeRead)!=uSizeRead) + err=UNZ_ERRNO; + lSeek -= uSizeRead; + } + + // Read extrafield + if ((err==UNZ_OK) && (extraField!=NULL)) + { + ZPOS64_T uSizeRead ; + if (file_info.size_file_extra<extraFieldBufferSize) + uSizeRead = file_info.size_file_extra; + else + uSizeRead = extraFieldBufferSize; + + if (lSeek!=0) + { + if (ZSEEK64(s->z_filefunc, s->filestream,lSeek,ZLIB_FILEFUNC_SEEK_CUR)==0) + lSeek=0; + else + err=UNZ_ERRNO; + } + + if ((file_info.size_file_extra>0) && (extraFieldBufferSize>0)) + if (ZREAD64(s->z_filefunc, s->filestream,extraField,(uLong)uSizeRead)!=uSizeRead) + err=UNZ_ERRNO; + + lSeek += file_info.size_file_extra - (uLong)uSizeRead; + } + else + lSeek += file_info.size_file_extra; + + + if ((err==UNZ_OK) && (file_info.size_file_extra != 0)) + { + uLong acc = 0; + + // since lSeek now points to after the extra field we need to move back + lSeek -= file_info.size_file_extra; + + if (lSeek!=0) + { + if (ZSEEK64(s->z_filefunc, s->filestream,lSeek,ZLIB_FILEFUNC_SEEK_CUR)==0) + lSeek=0; + else + err=UNZ_ERRNO; + } + + while(acc < file_info.size_file_extra) + { + uLong headerId; + uLong dataSize; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&headerId) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&dataSize) != UNZ_OK) + err=UNZ_ERRNO; + + /* ZIP64 extra fields */ + if (headerId == 0x0001) + { + uLong uL; + + if(file_info.uncompressed_size == MAXU32) + { + if (unz64local_getLong64(&s->z_filefunc, s->filestream,&file_info.uncompressed_size) != UNZ_OK) + err=UNZ_ERRNO; + } + + if(file_info.compressed_size == MAXU32) + { + if (unz64local_getLong64(&s->z_filefunc, s->filestream,&file_info.compressed_size) != UNZ_OK) + err=UNZ_ERRNO; + } + + if(file_info_internal.offset_curfile == MAXU32) + { + /* Relative Header offset */ + if (unz64local_getLong64(&s->z_filefunc, s->filestream,&file_info_internal.offset_curfile) != UNZ_OK) + err=UNZ_ERRNO; + } + + if(file_info.disk_num_start == MAXU32) + { + /* Disk Start Number */ + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uL) != UNZ_OK) + err=UNZ_ERRNO; + } + + } + else + { + if (ZSEEK64(s->z_filefunc, s->filestream,dataSize,ZLIB_FILEFUNC_SEEK_CUR)!=0) + err=UNZ_ERRNO; + } + + acc += 2 + 2 + dataSize; + } + } + + if ((err==UNZ_OK) && (szComment!=NULL)) + { + uLong uSizeRead ; + if (file_info.size_file_comment<commentBufferSize) + { + *(szComment+file_info.size_file_comment)='\0'; + uSizeRead = file_info.size_file_comment; + } + else + uSizeRead = commentBufferSize; + + if (lSeek!=0) + { + if (ZSEEK64(s->z_filefunc, s->filestream,lSeek,ZLIB_FILEFUNC_SEEK_CUR)==0) + lSeek=0; + else + err=UNZ_ERRNO; + } + + if ((file_info.size_file_comment>0) && (commentBufferSize>0)) + if (ZREAD64(s->z_filefunc, s->filestream,szComment,uSizeRead)!=uSizeRead) + err=UNZ_ERRNO; + lSeek+=file_info.size_file_comment - uSizeRead; + } + else + lSeek+=file_info.size_file_comment; + + + if ((err==UNZ_OK) && (pfile_info!=NULL)) + *pfile_info=file_info; + + if ((err==UNZ_OK) && (pfile_info_internal!=NULL)) + *pfile_info_internal=file_info_internal; + + return err; +} + + + +/* + Write info about the ZipFile in the *pglobal_info structure. + No preparation of the structure is needed + return UNZ_OK if there is no problem. +*/ +extern int ZEXPORT unzGetCurrentFileInfo64 (unzFile file, + unz_file_info64 * pfile_info, + char * szFileName, uLong fileNameBufferSize, + void *extraField, uLong extraFieldBufferSize, + char* szComment, uLong commentBufferSize) +{ + return unz64local_GetCurrentFileInfoInternal(file,pfile_info,NULL, + szFileName,fileNameBufferSize, + extraField,extraFieldBufferSize, + szComment,commentBufferSize); +} + +extern int ZEXPORT unzGetCurrentFileInfo (unzFile file, + unz_file_info * pfile_info, + char * szFileName, uLong fileNameBufferSize, + void *extraField, uLong extraFieldBufferSize, + char* szComment, uLong commentBufferSize) +{ + int err; + unz_file_info64 file_info64; + err = unz64local_GetCurrentFileInfoInternal(file,&file_info64,NULL, + szFileName,fileNameBufferSize, + extraField,extraFieldBufferSize, + szComment,commentBufferSize); + if ((err==UNZ_OK) && (pfile_info != NULL)) + { + pfile_info->version = file_info64.version; + pfile_info->version_needed = file_info64.version_needed; + pfile_info->flag = file_info64.flag; + pfile_info->compression_method = file_info64.compression_method; + pfile_info->dosDate = file_info64.dosDate; + pfile_info->crc = file_info64.crc; + + pfile_info->size_filename = file_info64.size_filename; + pfile_info->size_file_extra = file_info64.size_file_extra; + pfile_info->size_file_comment = file_info64.size_file_comment; + + pfile_info->disk_num_start = file_info64.disk_num_start; + pfile_info->internal_fa = file_info64.internal_fa; + pfile_info->external_fa = file_info64.external_fa; + + pfile_info->tmu_date = file_info64.tmu_date, + + + pfile_info->compressed_size = (uLong)file_info64.compressed_size; + pfile_info->uncompressed_size = (uLong)file_info64.uncompressed_size; + + } + return err; +} +/* + Set the current file of the zipfile to the first file. + return UNZ_OK if there is no problem +*/ +extern int ZEXPORT unzGoToFirstFile (unzFile file) +{ + int err=UNZ_OK; + unz64_s* s; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + s->pos_in_central_dir=s->offset_central_dir; + s->num_file=0; + err=unz64local_GetCurrentFileInfoInternal(file,&s->cur_file_info, + &s->cur_file_info_internal, + NULL,0,NULL,0,NULL,0); + s->current_file_ok = (err == UNZ_OK); + return err; +} + +/* + Set the current file of the zipfile to the next file. + return UNZ_OK if there is no problem + return UNZ_END_OF_LIST_OF_FILE if the actual file was the latest. +*/ +extern int ZEXPORT unzGoToNextFile (unzFile file) +{ + unz64_s* s; + int err; + + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + if (!s->current_file_ok) + return UNZ_END_OF_LIST_OF_FILE; + if (s->gi.number_entry != 0xffff) /* 2^16 files overflow hack */ + if (s->num_file+1==s->gi.number_entry) + return UNZ_END_OF_LIST_OF_FILE; + + s->pos_in_central_dir += SIZECENTRALDIRITEM + s->cur_file_info.size_filename + + s->cur_file_info.size_file_extra + s->cur_file_info.size_file_comment ; + s->num_file++; + err = unz64local_GetCurrentFileInfoInternal(file,&s->cur_file_info, + &s->cur_file_info_internal, + NULL,0,NULL,0,NULL,0); + s->current_file_ok = (err == UNZ_OK); + return err; +} + + +/* + Try locate the file szFileName in the zipfile. + For the iCaseSensitivity signification, see unzStringFileNameCompare + + return value : + UNZ_OK if the file is found. It becomes the current file. + UNZ_END_OF_LIST_OF_FILE if the file is not found +*/ +extern int ZEXPORT unzLocateFile (unzFile file, const char *szFileName, int iCaseSensitivity) +{ + unz64_s* s; + int err; + + /* We remember the 'current' position in the file so that we can jump + * back there if we fail. + */ + unz_file_info64 cur_file_infoSaved; + unz_file_info64_internal cur_file_info_internalSaved; + ZPOS64_T num_fileSaved; + ZPOS64_T pos_in_central_dirSaved; + + + if (file==NULL) + return UNZ_PARAMERROR; + + if (strlen(szFileName)>=UNZ_MAXFILENAMEINZIP) + return UNZ_PARAMERROR; + + s=(unz64_s*)file; + if (!s->current_file_ok) + return UNZ_END_OF_LIST_OF_FILE; + + /* Save the current state */ + num_fileSaved = s->num_file; + pos_in_central_dirSaved = s->pos_in_central_dir; + cur_file_infoSaved = s->cur_file_info; + cur_file_info_internalSaved = s->cur_file_info_internal; + + err = unzGoToFirstFile(file); + + while (err == UNZ_OK) + { + char szCurrentFileName[UNZ_MAXFILENAMEINZIP+1]; + err = unzGetCurrentFileInfo64(file,NULL, + szCurrentFileName,sizeof(szCurrentFileName)-1, + NULL,0,NULL,0); + if (err == UNZ_OK) + { + if (unzStringFileNameCompare(szCurrentFileName, + szFileName,iCaseSensitivity)==0) + return UNZ_OK; + err = unzGoToNextFile(file); + } + } + + /* We failed, so restore the state of the 'current file' to where we + * were. + */ + s->num_file = num_fileSaved ; + s->pos_in_central_dir = pos_in_central_dirSaved ; + s->cur_file_info = cur_file_infoSaved; + s->cur_file_info_internal = cur_file_info_internalSaved; + return err; +} + + +/* +/////////////////////////////////////////// +// Contributed by Ryan Haksi (mailto://cryogen@infoserve.net) +// I need random access +// +// Further optimization could be realized by adding an ability +// to cache the directory in memory. The goal being a single +// comprehensive file read to put the file I need in a memory. +*/ + +/* +typedef struct unz_file_pos_s +{ + ZPOS64_T pos_in_zip_directory; // offset in file + ZPOS64_T num_of_file; // # of file +} unz_file_pos; +*/ + +extern int ZEXPORT unzGetFilePos64(unzFile file, unz64_file_pos* file_pos) +{ + unz64_s* s; + + if (file==NULL || file_pos==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + if (!s->current_file_ok) + return UNZ_END_OF_LIST_OF_FILE; + + file_pos->pos_in_zip_directory = s->pos_in_central_dir; + file_pos->num_of_file = s->num_file; + + return UNZ_OK; +} + +extern int ZEXPORT unzGetFilePos( + unzFile file, + unz_file_pos* file_pos) +{ + unz64_file_pos file_pos64; + int err = unzGetFilePos64(file,&file_pos64); + if (err==UNZ_OK) + { + file_pos->pos_in_zip_directory = (uLong)file_pos64.pos_in_zip_directory; + file_pos->num_of_file = (uLong)file_pos64.num_of_file; + } + return err; +} + +extern int ZEXPORT unzGoToFilePos64(unzFile file, const unz64_file_pos* file_pos) +{ + unz64_s* s; + int err; + + if (file==NULL || file_pos==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + + /* jump to the right spot */ + s->pos_in_central_dir = file_pos->pos_in_zip_directory; + s->num_file = file_pos->num_of_file; + + /* set the current file */ + err = unz64local_GetCurrentFileInfoInternal(file,&s->cur_file_info, + &s->cur_file_info_internal, + NULL,0,NULL,0,NULL,0); + /* return results */ + s->current_file_ok = (err == UNZ_OK); + return err; +} + +extern int ZEXPORT unzGoToFilePos( + unzFile file, + unz_file_pos* file_pos) +{ + unz64_file_pos file_pos64; + if (file_pos == NULL) + return UNZ_PARAMERROR; + + file_pos64.pos_in_zip_directory = file_pos->pos_in_zip_directory; + file_pos64.num_of_file = file_pos->num_of_file; + return unzGoToFilePos64(file,&file_pos64); +} + +/* +// Unzip Helper Functions - should be here? +/////////////////////////////////////////// +*/ + +/* + Read the local header of the current zipfile + Check the coherency of the local header and info in the end of central + directory about this file + store in *piSizeVar the size of extra info in local header + (filename and size of extra field data) +*/ +local int unz64local_CheckCurrentFileCoherencyHeader (unz64_s* s, uInt* piSizeVar, + ZPOS64_T * poffset_local_extrafield, + uInt * psize_local_extrafield) +{ + uLong uMagic,uData,uFlags; + uLong size_filename; + uLong size_extra_field; + int err=UNZ_OK; + + *piSizeVar = 0; + *poffset_local_extrafield = 0; + *psize_local_extrafield = 0; + + if (ZSEEK64(s->z_filefunc, s->filestream,s->cur_file_info_internal.offset_curfile + + s->byte_before_the_zipfile,ZLIB_FILEFUNC_SEEK_SET)!=0) + return UNZ_ERRNO; + + + if (err==UNZ_OK) + { + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uMagic) != UNZ_OK) + err=UNZ_ERRNO; + else if (uMagic!=0x04034b50) + err=UNZ_BADZIPFILE; + } + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) + err=UNZ_ERRNO; +/* + else if ((err==UNZ_OK) && (uData!=s->cur_file_info.wVersion)) + err=UNZ_BADZIPFILE; +*/ + if (unz64local_getShort(&s->z_filefunc, s->filestream,&uFlags) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) + err=UNZ_ERRNO; + else if ((err==UNZ_OK) && (uData!=s->cur_file_info.compression_method)) + err=UNZ_BADZIPFILE; + + if ((err==UNZ_OK) && (s->cur_file_info.compression_method!=0) && +/* #ifdef HAVE_BZIP2 */ + (s->cur_file_info.compression_method!=Z_BZIP2ED) && +/* #endif */ + (s->cur_file_info.compression_method!=Z_DEFLATED)) + err=UNZ_BADZIPFILE; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) /* date/time */ + err=UNZ_ERRNO; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) /* crc */ + err=UNZ_ERRNO; + else if ((err==UNZ_OK) && (uData!=s->cur_file_info.crc) && ((uFlags & 8)==0)) + err=UNZ_BADZIPFILE; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) /* size compr */ + err=UNZ_ERRNO; + else if (uData != 0xFFFFFFFF && (err==UNZ_OK) && (uData!=s->cur_file_info.compressed_size) && ((uFlags & 8)==0)) + err=UNZ_BADZIPFILE; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) /* size uncompr */ + err=UNZ_ERRNO; + else if (uData != 0xFFFFFFFF && (err==UNZ_OK) && (uData!=s->cur_file_info.uncompressed_size) && ((uFlags & 8)==0)) + err=UNZ_BADZIPFILE; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&size_filename) != UNZ_OK) + err=UNZ_ERRNO; + else if ((err==UNZ_OK) && (size_filename!=s->cur_file_info.size_filename)) + err=UNZ_BADZIPFILE; + + *piSizeVar += (uInt)size_filename; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&size_extra_field) != UNZ_OK) + err=UNZ_ERRNO; + *poffset_local_extrafield= s->cur_file_info_internal.offset_curfile + + SIZEZIPLOCALHEADER + size_filename; + *psize_local_extrafield = (uInt)size_extra_field; + + *piSizeVar += (uInt)size_extra_field; + + return err; +} + +/* + Open for reading data the current file in the zipfile. + If there is no error and the file is opened, the return value is UNZ_OK. +*/ +extern int ZEXPORT unzOpenCurrentFile3 (unzFile file, int* method, + int* level, int raw, const char* password) +{ + int err=UNZ_OK; + uInt iSizeVar; + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + ZPOS64_T offset_local_extrafield; /* offset of the local extra field */ + uInt size_local_extrafield; /* size of the local extra field */ +# ifndef NOUNCRYPT + char source[12]; +# else + if (password != NULL) + return UNZ_PARAMERROR; +# endif + + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + if (!s->current_file_ok) + return UNZ_PARAMERROR; + + if (s->pfile_in_zip_read != NULL) + unzCloseCurrentFile(file); + + if (unz64local_CheckCurrentFileCoherencyHeader(s,&iSizeVar, &offset_local_extrafield,&size_local_extrafield)!=UNZ_OK) + return UNZ_BADZIPFILE; + + pfile_in_zip_read_info = (file_in_zip64_read_info_s*)ALLOC(sizeof(file_in_zip64_read_info_s)); + if (pfile_in_zip_read_info==NULL) + return UNZ_INTERNALERROR; + + pfile_in_zip_read_info->read_buffer=(char*)ALLOC(UNZ_BUFSIZE); + pfile_in_zip_read_info->offset_local_extrafield = offset_local_extrafield; + pfile_in_zip_read_info->size_local_extrafield = size_local_extrafield; + pfile_in_zip_read_info->pos_local_extrafield=0; + pfile_in_zip_read_info->raw=raw; + + if (pfile_in_zip_read_info->read_buffer==NULL) + { + TRYFREE(pfile_in_zip_read_info); + return UNZ_INTERNALERROR; + } + + pfile_in_zip_read_info->stream_initialised=0; + + if (method!=NULL) + *method = (int)s->cur_file_info.compression_method; + + if (level!=NULL) + { + *level = 6; + switch (s->cur_file_info.flag & 0x06) + { + case 6 : *level = 1; break; + case 4 : *level = 2; break; + case 2 : *level = 9; break; + } + } + + if ((s->cur_file_info.compression_method!=0) && +/* #ifdef HAVE_BZIP2 */ + (s->cur_file_info.compression_method!=Z_BZIP2ED) && +/* #endif */ + (s->cur_file_info.compression_method!=Z_DEFLATED)) + + err=UNZ_BADZIPFILE; + + pfile_in_zip_read_info->crc32_wait=s->cur_file_info.crc; + pfile_in_zip_read_info->crc32=0; + pfile_in_zip_read_info->total_out_64=0; + pfile_in_zip_read_info->compression_method = s->cur_file_info.compression_method; + pfile_in_zip_read_info->filestream=s->filestream; + pfile_in_zip_read_info->z_filefunc=s->z_filefunc; + pfile_in_zip_read_info->byte_before_the_zipfile=s->byte_before_the_zipfile; + + pfile_in_zip_read_info->stream.total_out = 0; + + if ((s->cur_file_info.compression_method==Z_BZIP2ED) && (!raw)) + { +#ifdef HAVE_BZIP2 + pfile_in_zip_read_info->bstream.bzalloc = (void *(*) (void *, int, int))0; + pfile_in_zip_read_info->bstream.bzfree = (free_func)0; + pfile_in_zip_read_info->bstream.opaque = (voidpf)0; + pfile_in_zip_read_info->bstream.state = (voidpf)0; + + pfile_in_zip_read_info->stream.zalloc = (alloc_func)0; + pfile_in_zip_read_info->stream.zfree = (free_func)0; + pfile_in_zip_read_info->stream.opaque = (voidpf)0; + pfile_in_zip_read_info->stream.next_in = (voidpf)0; + pfile_in_zip_read_info->stream.avail_in = 0; + + err=BZ2_bzDecompressInit(&pfile_in_zip_read_info->bstream, 0, 0); + if (err == Z_OK) + pfile_in_zip_read_info->stream_initialised=Z_BZIP2ED; + else + { + TRYFREE(pfile_in_zip_read_info); + return err; + } +#else + pfile_in_zip_read_info->raw=1; +#endif + } + else if ((s->cur_file_info.compression_method==Z_DEFLATED) && (!raw)) + { + pfile_in_zip_read_info->stream.zalloc = (alloc_func)0; + pfile_in_zip_read_info->stream.zfree = (free_func)0; + pfile_in_zip_read_info->stream.opaque = (voidpf)0; + pfile_in_zip_read_info->stream.next_in = 0; + pfile_in_zip_read_info->stream.avail_in = 0; + + err=inflateInit2(&pfile_in_zip_read_info->stream, -MAX_WBITS); + if (err == Z_OK) + pfile_in_zip_read_info->stream_initialised=Z_DEFLATED; + else + { + TRYFREE(pfile_in_zip_read_info); + return err; + } + /* windowBits is passed < 0 to tell that there is no zlib header. + * Note that in this case inflate *requires* an extra "dummy" byte + * after the compressed stream in order to complete decompression and + * return Z_STREAM_END. + * In unzip, i don't wait absolutely Z_STREAM_END because I known the + * size of both compressed and uncompressed data + */ + } + pfile_in_zip_read_info->rest_read_compressed = + s->cur_file_info.compressed_size ; + pfile_in_zip_read_info->rest_read_uncompressed = + s->cur_file_info.uncompressed_size ; + + + pfile_in_zip_read_info->pos_in_zipfile = + s->cur_file_info_internal.offset_curfile + SIZEZIPLOCALHEADER + + iSizeVar; + + pfile_in_zip_read_info->stream.avail_in = (uInt)0; + + s->pfile_in_zip_read = pfile_in_zip_read_info; + s->encrypted = 0; + +# ifndef NOUNCRYPT + if (password != NULL) + { + int i; + s->pcrc_32_tab = get_crc_table(); + init_keys(password,s->keys,s->pcrc_32_tab); + if (ZSEEK64(s->z_filefunc, s->filestream, + s->pfile_in_zip_read->pos_in_zipfile + + s->pfile_in_zip_read->byte_before_the_zipfile, + SEEK_SET)!=0) + return UNZ_INTERNALERROR; + if(ZREAD64(s->z_filefunc, s->filestream,source, 12)<12) + return UNZ_INTERNALERROR; + + for (i = 0; i<12; i++) + zdecode(s->keys,s->pcrc_32_tab,source[i]); + + s->pfile_in_zip_read->pos_in_zipfile+=12; + s->encrypted=1; + } +# endif + + + return UNZ_OK; +} + +extern int ZEXPORT unzOpenCurrentFile (unzFile file) +{ + return unzOpenCurrentFile3(file, NULL, NULL, 0, NULL); +} + +extern int ZEXPORT unzOpenCurrentFilePassword (unzFile file, const char* password) +{ + return unzOpenCurrentFile3(file, NULL, NULL, 0, password); +} + +extern int ZEXPORT unzOpenCurrentFile2 (unzFile file, int* method, int* level, int raw) +{ + return unzOpenCurrentFile3(file, method, level, raw, NULL); +} + +/** Addition for GDAL : START */ + +extern ZPOS64_T ZEXPORT unzGetCurrentFileZStreamPos64( unzFile file) +{ + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + s=(unz64_s*)file; + if (file==NULL) + return 0; //UNZ_PARAMERROR; + pfile_in_zip_read_info=s->pfile_in_zip_read; + if (pfile_in_zip_read_info==NULL) + return 0; //UNZ_PARAMERROR; + return pfile_in_zip_read_info->pos_in_zipfile + + pfile_in_zip_read_info->byte_before_the_zipfile; +} + +/** Addition for GDAL : END */ + +/* + Read bytes from the current file. + buf contain buffer where data must be copied + len the size of buf. + + return the number of byte copied if somes bytes are copied + return 0 if the end of file was reached + return <0 with error code if there is an error + (UNZ_ERRNO for IO error, or zLib error for uncompress error) +*/ +extern int ZEXPORT unzReadCurrentFile (unzFile file, voidp buf, unsigned len) +{ + int err=UNZ_OK; + uInt iRead = 0; + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + pfile_in_zip_read_info=s->pfile_in_zip_read; + + if (pfile_in_zip_read_info==NULL) + return UNZ_PARAMERROR; + + + if (pfile_in_zip_read_info->read_buffer == NULL) + return UNZ_END_OF_LIST_OF_FILE; + if (len==0) + return 0; + + pfile_in_zip_read_info->stream.next_out = (Bytef*)buf; + + pfile_in_zip_read_info->stream.avail_out = (uInt)len; + + if ((len>pfile_in_zip_read_info->rest_read_uncompressed) && + (!(pfile_in_zip_read_info->raw))) + pfile_in_zip_read_info->stream.avail_out = + (uInt)pfile_in_zip_read_info->rest_read_uncompressed; + + if ((len>pfile_in_zip_read_info->rest_read_compressed+ + pfile_in_zip_read_info->stream.avail_in) && + (pfile_in_zip_read_info->raw)) + pfile_in_zip_read_info->stream.avail_out = + (uInt)pfile_in_zip_read_info->rest_read_compressed+ + pfile_in_zip_read_info->stream.avail_in; + + while (pfile_in_zip_read_info->stream.avail_out>0) + { + if ((pfile_in_zip_read_info->stream.avail_in==0) && + (pfile_in_zip_read_info->rest_read_compressed>0)) + { + uInt uReadThis = UNZ_BUFSIZE; + if (pfile_in_zip_read_info->rest_read_compressed<uReadThis) + uReadThis = (uInt)pfile_in_zip_read_info->rest_read_compressed; + if (uReadThis == 0) + return UNZ_EOF; + if (ZSEEK64(pfile_in_zip_read_info->z_filefunc, + pfile_in_zip_read_info->filestream, + pfile_in_zip_read_info->pos_in_zipfile + + pfile_in_zip_read_info->byte_before_the_zipfile, + ZLIB_FILEFUNC_SEEK_SET)!=0) + return UNZ_ERRNO; + if (ZREAD64(pfile_in_zip_read_info->z_filefunc, + pfile_in_zip_read_info->filestream, + pfile_in_zip_read_info->read_buffer, + uReadThis)!=uReadThis) + return UNZ_ERRNO; + + +# ifndef NOUNCRYPT + if(s->encrypted) + { + uInt i; + for(i=0;i<uReadThis;i++) + pfile_in_zip_read_info->read_buffer[i] = + zdecode(s->keys,s->pcrc_32_tab, + pfile_in_zip_read_info->read_buffer[i]); + } +# endif + + + pfile_in_zip_read_info->pos_in_zipfile += uReadThis; + + pfile_in_zip_read_info->rest_read_compressed-=uReadThis; + + pfile_in_zip_read_info->stream.next_in = + (Bytef*)pfile_in_zip_read_info->read_buffer; + pfile_in_zip_read_info->stream.avail_in = (uInt)uReadThis; + } + + if ((pfile_in_zip_read_info->compression_method==0) || (pfile_in_zip_read_info->raw)) + { + uInt uDoCopy,i ; + + if ((pfile_in_zip_read_info->stream.avail_in == 0) && + (pfile_in_zip_read_info->rest_read_compressed == 0)) + return (iRead==0) ? UNZ_EOF : iRead; + + if (pfile_in_zip_read_info->stream.avail_out < + pfile_in_zip_read_info->stream.avail_in) + uDoCopy = pfile_in_zip_read_info->stream.avail_out ; + else + uDoCopy = pfile_in_zip_read_info->stream.avail_in ; + + for (i=0;i<uDoCopy;i++) + *(pfile_in_zip_read_info->stream.next_out+i) = + *(pfile_in_zip_read_info->stream.next_in+i); + + pfile_in_zip_read_info->total_out_64 = pfile_in_zip_read_info->total_out_64 + uDoCopy; + + pfile_in_zip_read_info->crc32 = crc32(pfile_in_zip_read_info->crc32, + pfile_in_zip_read_info->stream.next_out, + uDoCopy); + pfile_in_zip_read_info->rest_read_uncompressed-=uDoCopy; + pfile_in_zip_read_info->stream.avail_in -= uDoCopy; + pfile_in_zip_read_info->stream.avail_out -= uDoCopy; + pfile_in_zip_read_info->stream.next_out += uDoCopy; + pfile_in_zip_read_info->stream.next_in += uDoCopy; + pfile_in_zip_read_info->stream.total_out += uDoCopy; + iRead += uDoCopy; + } + else if (pfile_in_zip_read_info->compression_method==Z_BZIP2ED) + { +#ifdef HAVE_BZIP2 + uLong uTotalOutBefore,uTotalOutAfter; + const Bytef *bufBefore; + uLong uOutThis; + + pfile_in_zip_read_info->bstream.next_in = (char*)pfile_in_zip_read_info->stream.next_in; + pfile_in_zip_read_info->bstream.avail_in = pfile_in_zip_read_info->stream.avail_in; + pfile_in_zip_read_info->bstream.total_in_lo32 = pfile_in_zip_read_info->stream.total_in; + pfile_in_zip_read_info->bstream.total_in_hi32 = 0; + pfile_in_zip_read_info->bstream.next_out = (char*)pfile_in_zip_read_info->stream.next_out; + pfile_in_zip_read_info->bstream.avail_out = pfile_in_zip_read_info->stream.avail_out; + pfile_in_zip_read_info->bstream.total_out_lo32 = pfile_in_zip_read_info->stream.total_out; + pfile_in_zip_read_info->bstream.total_out_hi32 = 0; + + uTotalOutBefore = pfile_in_zip_read_info->bstream.total_out_lo32; + bufBefore = (const Bytef *)pfile_in_zip_read_info->bstream.next_out; + + err=BZ2_bzDecompress(&pfile_in_zip_read_info->bstream); + + uTotalOutAfter = pfile_in_zip_read_info->bstream.total_out_lo32; + uOutThis = uTotalOutAfter-uTotalOutBefore; + + pfile_in_zip_read_info->total_out_64 = pfile_in_zip_read_info->total_out_64 + uOutThis; + + pfile_in_zip_read_info->crc32 = crc32(pfile_in_zip_read_info->crc32,bufBefore, (uInt)(uOutThis)); + pfile_in_zip_read_info->rest_read_uncompressed -= uOutThis; + iRead += (uInt)(uTotalOutAfter - uTotalOutBefore); + + pfile_in_zip_read_info->stream.next_in = (Bytef*)pfile_in_zip_read_info->bstream.next_in; + pfile_in_zip_read_info->stream.avail_in = pfile_in_zip_read_info->bstream.avail_in; + pfile_in_zip_read_info->stream.total_in = pfile_in_zip_read_info->bstream.total_in_lo32; + pfile_in_zip_read_info->stream.next_out = (Bytef*)pfile_in_zip_read_info->bstream.next_out; + pfile_in_zip_read_info->stream.avail_out = pfile_in_zip_read_info->bstream.avail_out; + pfile_in_zip_read_info->stream.total_out = pfile_in_zip_read_info->bstream.total_out_lo32; + + if (err==BZ_STREAM_END) + return (iRead==0) ? UNZ_EOF : iRead; + if (err!=BZ_OK) + break; +#endif + } // end Z_BZIP2ED + else + { + ZPOS64_T uTotalOutBefore,uTotalOutAfter; + const Bytef *bufBefore; + ZPOS64_T uOutThis; + int flush=Z_SYNC_FLUSH; + + uTotalOutBefore = pfile_in_zip_read_info->stream.total_out; + bufBefore = pfile_in_zip_read_info->stream.next_out; + + /* + if ((pfile_in_zip_read_info->rest_read_uncompressed == + pfile_in_zip_read_info->stream.avail_out) && + (pfile_in_zip_read_info->rest_read_compressed == 0)) + flush = Z_FINISH; + */ + err=inflate(&pfile_in_zip_read_info->stream,flush); + + if ((err>=0) && (pfile_in_zip_read_info->stream.msg!=NULL)) + err = Z_DATA_ERROR; + + uTotalOutAfter = pfile_in_zip_read_info->stream.total_out; + uOutThis = uTotalOutAfter-uTotalOutBefore; + + pfile_in_zip_read_info->total_out_64 = pfile_in_zip_read_info->total_out_64 + uOutThis; + + pfile_in_zip_read_info->crc32 = + crc32(pfile_in_zip_read_info->crc32,bufBefore, + (uInt)(uOutThis)); + + pfile_in_zip_read_info->rest_read_uncompressed -= + uOutThis; + + iRead += (uInt)(uTotalOutAfter - uTotalOutBefore); + + if (err==Z_STREAM_END) + return (iRead==0) ? UNZ_EOF : iRead; + if (err!=Z_OK) + break; + } + } + + if (err==Z_OK) + return iRead; + return err; +} + + +/* + Give the current position in uncompressed data +*/ +extern z_off_t ZEXPORT unztell (unzFile file) +{ + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + pfile_in_zip_read_info=s->pfile_in_zip_read; + + if (pfile_in_zip_read_info==NULL) + return UNZ_PARAMERROR; + + return (z_off_t)pfile_in_zip_read_info->stream.total_out; +} + +extern ZPOS64_T ZEXPORT unztell64 (unzFile file) +{ + + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + if (file==NULL) + return (ZPOS64_T)-1; + s=(unz64_s*)file; + pfile_in_zip_read_info=s->pfile_in_zip_read; + + if (pfile_in_zip_read_info==NULL) + return (ZPOS64_T)-1; + + return pfile_in_zip_read_info->total_out_64; +} + + +/* + return 1 if the end of file was reached, 0 elsewhere +*/ +extern int ZEXPORT unzeof (unzFile file) +{ + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + pfile_in_zip_read_info=s->pfile_in_zip_read; + + if (pfile_in_zip_read_info==NULL) + return UNZ_PARAMERROR; + + if (pfile_in_zip_read_info->rest_read_uncompressed == 0) + return 1; + else + return 0; +} + + + +/* +Read extra field from the current file (opened by unzOpenCurrentFile) +This is the local-header version of the extra field (sometimes, there is +more info in the local-header version than in the central-header) + + if buf==NULL, it return the size of the local extra field that can be read + + if buf!=NULL, len is the size of the buffer, the extra header is copied in + buf. + the return value is the number of bytes copied in buf, or (if <0) + the error code +*/ +extern int ZEXPORT unzGetLocalExtrafield (unzFile file, voidp buf, unsigned len) +{ + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + uInt read_now; + ZPOS64_T size_to_read; + + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + pfile_in_zip_read_info=s->pfile_in_zip_read; + + if (pfile_in_zip_read_info==NULL) + return UNZ_PARAMERROR; + + size_to_read = (pfile_in_zip_read_info->size_local_extrafield - + pfile_in_zip_read_info->pos_local_extrafield); + + if (buf==NULL) + return (int)size_to_read; + + if (len>size_to_read) + read_now = (uInt)size_to_read; + else + read_now = (uInt)len ; + + if (read_now==0) + return 0; + + if (ZSEEK64(pfile_in_zip_read_info->z_filefunc, + pfile_in_zip_read_info->filestream, + pfile_in_zip_read_info->offset_local_extrafield + + pfile_in_zip_read_info->pos_local_extrafield, + ZLIB_FILEFUNC_SEEK_SET)!=0) + return UNZ_ERRNO; + + if (ZREAD64(pfile_in_zip_read_info->z_filefunc, + pfile_in_zip_read_info->filestream, + buf,read_now)!=read_now) + return UNZ_ERRNO; + + return (int)read_now; +} + +/* + Close the file in zip opened with unzOpenCurrentFile + Return UNZ_CRCERROR if all the file was read but the CRC is not good +*/ +extern int ZEXPORT unzCloseCurrentFile (unzFile file) +{ + int err=UNZ_OK; + + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + pfile_in_zip_read_info=s->pfile_in_zip_read; + + if (pfile_in_zip_read_info==NULL) + return UNZ_PARAMERROR; + + + if ((pfile_in_zip_read_info->rest_read_uncompressed == 0) && + (!pfile_in_zip_read_info->raw)) + { + if (pfile_in_zip_read_info->crc32 != pfile_in_zip_read_info->crc32_wait) + err=UNZ_CRCERROR; + } + + + TRYFREE(pfile_in_zip_read_info->read_buffer); + pfile_in_zip_read_info->read_buffer = NULL; + if (pfile_in_zip_read_info->stream_initialised == Z_DEFLATED) + inflateEnd(&pfile_in_zip_read_info->stream); +#ifdef HAVE_BZIP2 + else if (pfile_in_zip_read_info->stream_initialised == Z_BZIP2ED) + BZ2_bzDecompressEnd(&pfile_in_zip_read_info->bstream); +#endif + + + pfile_in_zip_read_info->stream_initialised = 0; + TRYFREE(pfile_in_zip_read_info); + + s->pfile_in_zip_read=NULL; + + return err; +} + + +/* + Get the global comment string of the ZipFile, in the szComment buffer. + uSizeBuf is the size of the szComment buffer. + return the number of byte copied or an error code <0 +*/ +extern int ZEXPORT unzGetGlobalComment (unzFile file, char * szComment, uLong uSizeBuf) +{ + unz64_s* s; + uLong uReadThis ; + if (file==NULL) + return (int)UNZ_PARAMERROR; + s=(unz64_s*)file; + + uReadThis = uSizeBuf; + if (uReadThis>s->gi.size_comment) + uReadThis = s->gi.size_comment; + + if (ZSEEK64(s->z_filefunc,s->filestream,s->central_pos+22,ZLIB_FILEFUNC_SEEK_SET)!=0) + return UNZ_ERRNO; + + if (uReadThis>0) + { + *szComment='\0'; + if (ZREAD64(s->z_filefunc,s->filestream,szComment,uReadThis)!=uReadThis) + return UNZ_ERRNO; + } + + if ((szComment != NULL) && (uSizeBuf > s->gi.size_comment)) + *(szComment+s->gi.size_comment)='\0'; + return (int)uReadThis; +} + +/* Additions by RX '2004 */ +extern ZPOS64_T ZEXPORT unzGetOffset64(unzFile file) +{ + unz64_s* s; + + if (file==NULL) + return 0; //UNZ_PARAMERROR; + s=(unz64_s*)file; + if (!s->current_file_ok) + return 0; + if (s->gi.number_entry != 0 && s->gi.number_entry != 0xffff) + if (s->num_file==s->gi.number_entry) + return 0; + return s->pos_in_central_dir; +} + +extern uLong ZEXPORT unzGetOffset (unzFile file) +{ + ZPOS64_T offset64; + + if (file==NULL) + return 0; //UNZ_PARAMERROR; + offset64 = unzGetOffset64(file); + return (uLong)offset64; +} + +extern int ZEXPORT unzSetOffset64(unzFile file, ZPOS64_T pos) +{ + unz64_s* s; + int err; + + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + + s->pos_in_central_dir = pos; + s->num_file = s->gi.number_entry; /* hack */ + err = unz64local_GetCurrentFileInfoInternal(file,&s->cur_file_info, + &s->cur_file_info_internal, + NULL,0,NULL,0,NULL,0); + s->current_file_ok = (err == UNZ_OK); + return err; +} + +extern int ZEXPORT unzSetOffset (unzFile file, uLong pos) +{ + return unzSetOffset64(file,pos); +} diff --git a/storage/connect/unzip.h b/storage/connect/unzip.h new file mode 100644 index 00000000000..2104e391507 --- /dev/null +++ b/storage/connect/unzip.h @@ -0,0 +1,437 @@ +/* unzip.h -- IO for uncompress .zip files using zlib + Version 1.1, February 14h, 2010 + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications of Unzip for Zip64 + Copyright (C) 2007-2008 Even Rouault + + Modifications for Zip64 support on both zip and unzip + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + + --------------------------------------------------------------------------------- + + Condition of use and distribution are the same than zlib : + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + --------------------------------------------------------------------------------- + + Changes + + See header of unzip64.c + +*/ + +#ifndef _unz64_H +#define _unz64_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef _ZLIB_H +#include "zlib.h" +#endif + +#ifndef _ZLIBIOAPI_H +#include "ioapi.h" +#endif + +#ifdef HAVE_BZIP2 +#include "bzlib.h" +#endif + +#define Z_BZIP2ED 12 + +#if defined(STRICTUNZIP) || defined(STRICTZIPUNZIP) +/* like the STRICT of WIN32, we define a pointer that cannot be converted + from (void*) without cast */ +typedef struct TagunzFile__ { int unused; } unzFile__; +typedef unzFile__ *unzFile; +#else +typedef voidp unzFile; +#endif + + +#define UNZ_OK (0) +#define UNZ_END_OF_LIST_OF_FILE (-100) +#define UNZ_ERRNO (Z_ERRNO) +#define UNZ_EOF (0) +#define UNZ_PARAMERROR (-102) +#define UNZ_BADZIPFILE (-103) +#define UNZ_INTERNALERROR (-104) +#define UNZ_CRCERROR (-105) + +/* tm_unz contain date/time info */ +typedef struct tm_unz_s +{ + uInt tm_sec; /* seconds after the minute - [0,59] */ + uInt tm_min; /* minutes after the hour - [0,59] */ + uInt tm_hour; /* hours since midnight - [0,23] */ + uInt tm_mday; /* day of the month - [1,31] */ + uInt tm_mon; /* months since January - [0,11] */ + uInt tm_year; /* years - [1980..2044] */ +} tm_unz; + +/* unz_global_info structure contain global data about the ZIPfile + These data comes from the end of central dir */ +typedef struct unz_global_info64_s +{ + ZPOS64_T number_entry; /* total number of entries in + the central dir on this disk */ + uLong size_comment; /* size of the global comment of the zipfile */ +} unz_global_info64; + +typedef struct unz_global_info_s +{ + uLong number_entry; /* total number of entries in + the central dir on this disk */ + uLong size_comment; /* size of the global comment of the zipfile */ +} unz_global_info; + +/* unz_file_info contain information about a file in the zipfile */ +typedef struct unz_file_info64_s +{ + uLong version; /* version made by 2 bytes */ + uLong version_needed; /* version needed to extract 2 bytes */ + uLong flag; /* general purpose bit flag 2 bytes */ + uLong compression_method; /* compression method 2 bytes */ + uLong dosDate; /* last mod file date in Dos fmt 4 bytes */ + uLong crc; /* crc-32 4 bytes */ + ZPOS64_T compressed_size; /* compressed size 8 bytes */ + ZPOS64_T uncompressed_size; /* uncompressed size 8 bytes */ + uLong size_filename; /* filename length 2 bytes */ + uLong size_file_extra; /* extra field length 2 bytes */ + uLong size_file_comment; /* file comment length 2 bytes */ + + uLong disk_num_start; /* disk number start 2 bytes */ + uLong internal_fa; /* internal file attributes 2 bytes */ + uLong external_fa; /* external file attributes 4 bytes */ + + tm_unz tmu_date; +} unz_file_info64; + +typedef struct unz_file_info_s +{ + uLong version; /* version made by 2 bytes */ + uLong version_needed; /* version needed to extract 2 bytes */ + uLong flag; /* general purpose bit flag 2 bytes */ + uLong compression_method; /* compression method 2 bytes */ + uLong dosDate; /* last mod file date in Dos fmt 4 bytes */ + uLong crc; /* crc-32 4 bytes */ + uLong compressed_size; /* compressed size 4 bytes */ + uLong uncompressed_size; /* uncompressed size 4 bytes */ + uLong size_filename; /* filename length 2 bytes */ + uLong size_file_extra; /* extra field length 2 bytes */ + uLong size_file_comment; /* file comment length 2 bytes */ + + uLong disk_num_start; /* disk number start 2 bytes */ + uLong internal_fa; /* internal file attributes 2 bytes */ + uLong external_fa; /* external file attributes 4 bytes */ + + tm_unz tmu_date; +} unz_file_info; + +extern int ZEXPORT unzStringFileNameCompare OF ((const char* fileName1, + const char* fileName2, + int iCaseSensitivity)); +/* + Compare two filename (fileName1,fileName2). + If iCaseSenisivity = 1, comparision is case sensitivity (like strcmp) + If iCaseSenisivity = 2, comparision is not case sensitivity (like strcmpi + or strcasecmp) + If iCaseSenisivity = 0, case sensitivity is defaut of your operating system + (like 1 on Unix, 2 on Windows) +*/ + + +extern unzFile ZEXPORT unzOpen OF((const char *path)); +extern unzFile ZEXPORT unzOpen64 OF((const void *path)); +/* + Open a Zip file. path contain the full pathname (by example, + on a Windows XP computer "c:\\zlib\\zlib113.zip" or on an Unix computer + "zlib/zlib113.zip". + If the zipfile cannot be opened (file don't exist or in not valid), the + return value is NULL. + Else, the return value is a unzFile Handle, usable with other function + of this unzip package. + the "64" function take a const void* pointer, because the path is just the + value passed to the open64_file_func callback. + Under Windows, if UNICODE is defined, using fill_fopen64_filefunc, the path + is a pointer to a wide unicode string (LPCTSTR is LPCWSTR), so const char* + does not describe the reality +*/ + + +extern unzFile ZEXPORT unzOpen2 OF((const char *path, + zlib_filefunc_def* pzlib_filefunc_def)); +/* + Open a Zip file, like unzOpen, but provide a set of file low level API + for read/write the zip file (see ioapi.h) +*/ + +extern unzFile ZEXPORT unzOpen2_64 OF((const void *path, + zlib_filefunc64_def* pzlib_filefunc_def)); +/* + Open a Zip file, like unz64Open, but provide a set of file low level API + for read/write the zip file (see ioapi.h) +*/ + +extern int ZEXPORT unzClose OF((unzFile file)); +/* + Close a ZipFile opened with unzOpen. + If there is files inside the .Zip opened with unzOpenCurrentFile (see later), + these files MUST be closed with unzCloseCurrentFile before call unzClose. + return UNZ_OK if there is no problem. */ + +extern int ZEXPORT unzGetGlobalInfo OF((unzFile file, + unz_global_info *pglobal_info)); + +extern int ZEXPORT unzGetGlobalInfo64 OF((unzFile file, + unz_global_info64 *pglobal_info)); +/* + Write info about the ZipFile in the *pglobal_info structure. + No preparation of the structure is needed + return UNZ_OK if there is no problem. */ + + +extern int ZEXPORT unzGetGlobalComment OF((unzFile file, + char *szComment, + uLong uSizeBuf)); +/* + Get the global comment string of the ZipFile, in the szComment buffer. + uSizeBuf is the size of the szComment buffer. + return the number of byte copied or an error code <0 +*/ + + +/***************************************************************************/ +/* Unzip package allow you browse the directory of the zipfile */ + +extern int ZEXPORT unzGoToFirstFile OF((unzFile file)); +/* + Set the current file of the zipfile to the first file. + return UNZ_OK if there is no problem +*/ + +extern int ZEXPORT unzGoToNextFile OF((unzFile file)); +/* + Set the current file of the zipfile to the next file. + return UNZ_OK if there is no problem + return UNZ_END_OF_LIST_OF_FILE if the actual file was the latest. +*/ + +extern int ZEXPORT unzLocateFile OF((unzFile file, + const char *szFileName, + int iCaseSensitivity)); +/* + Try locate the file szFileName in the zipfile. + For the iCaseSensitivity signification, see unzStringFileNameCompare + + return value : + UNZ_OK if the file is found. It becomes the current file. + UNZ_END_OF_LIST_OF_FILE if the file is not found +*/ + + +/* ****************************************** */ +/* Ryan supplied functions */ +/* unz_file_info contain information about a file in the zipfile */ +typedef struct unz_file_pos_s +{ + uLong pos_in_zip_directory; /* offset in zip file directory */ + uLong num_of_file; /* # of file */ +} unz_file_pos; + +extern int ZEXPORT unzGetFilePos( + unzFile file, + unz_file_pos* file_pos); + +extern int ZEXPORT unzGoToFilePos( + unzFile file, + unz_file_pos* file_pos); + +typedef struct unz64_file_pos_s +{ + ZPOS64_T pos_in_zip_directory; /* offset in zip file directory */ + ZPOS64_T num_of_file; /* # of file */ +} unz64_file_pos; + +extern int ZEXPORT unzGetFilePos64( + unzFile file, + unz64_file_pos* file_pos); + +extern int ZEXPORT unzGoToFilePos64( + unzFile file, + const unz64_file_pos* file_pos); + +/* ****************************************** */ + +extern int ZEXPORT unzGetCurrentFileInfo64 OF((unzFile file, + unz_file_info64 *pfile_info, + char *szFileName, + uLong fileNameBufferSize, + void *extraField, + uLong extraFieldBufferSize, + char *szComment, + uLong commentBufferSize)); + +extern int ZEXPORT unzGetCurrentFileInfo OF((unzFile file, + unz_file_info *pfile_info, + char *szFileName, + uLong fileNameBufferSize, + void *extraField, + uLong extraFieldBufferSize, + char *szComment, + uLong commentBufferSize)); +/* + Get Info about the current file + if pfile_info!=NULL, the *pfile_info structure will contain somes info about + the current file + if szFileName!=NULL, the filemane string will be copied in szFileName + (fileNameBufferSize is the size of the buffer) + if extraField!=NULL, the extra field information will be copied in extraField + (extraFieldBufferSize is the size of the buffer). + This is the Central-header version of the extra field + if szComment!=NULL, the comment string of the file will be copied in szComment + (commentBufferSize is the size of the buffer) +*/ + + +/** Addition for GDAL : START */ + +extern ZPOS64_T ZEXPORT unzGetCurrentFileZStreamPos64 OF((unzFile file)); + +/** Addition for GDAL : END */ + + +/***************************************************************************/ +/* for reading the content of the current zipfile, you can open it, read data + from it, and close it (you can close it before reading all the file) + */ + +extern int ZEXPORT unzOpenCurrentFile OF((unzFile file)); +/* + Open for reading data the current file in the zipfile. + If there is no error, the return value is UNZ_OK. +*/ + +extern int ZEXPORT unzOpenCurrentFilePassword OF((unzFile file, + const char* password)); +/* + Open for reading data the current file in the zipfile. + password is a crypting password + If there is no error, the return value is UNZ_OK. +*/ + +extern int ZEXPORT unzOpenCurrentFile2 OF((unzFile file, + int* method, + int* level, + int raw)); +/* + Same than unzOpenCurrentFile, but open for read raw the file (not uncompress) + if raw==1 + *method will receive method of compression, *level will receive level of + compression + note : you can set level parameter as NULL (if you did not want known level, + but you CANNOT set method parameter as NULL +*/ + +extern int ZEXPORT unzOpenCurrentFile3 OF((unzFile file, + int* method, + int* level, + int raw, + const char* password)); +/* + Same than unzOpenCurrentFile, but open for read raw the file (not uncompress) + if raw==1 + *method will receive method of compression, *level will receive level of + compression + note : you can set level parameter as NULL (if you did not want known level, + but you CANNOT set method parameter as NULL +*/ + + +extern int ZEXPORT unzCloseCurrentFile OF((unzFile file)); +/* + Close the file in zip opened with unzOpenCurrentFile + Return UNZ_CRCERROR if all the file was read but the CRC is not good +*/ + +extern int ZEXPORT unzReadCurrentFile OF((unzFile file, + voidp buf, + unsigned len)); +/* + Read bytes from the current file (opened by unzOpenCurrentFile) + buf contain buffer where data must be copied + len the size of buf. + + return the number of byte copied if somes bytes are copied + return 0 if the end of file was reached + return <0 with error code if there is an error + (UNZ_ERRNO for IO error, or zLib error for uncompress error) +*/ + +extern z_off_t ZEXPORT unztell OF((unzFile file)); + +extern ZPOS64_T ZEXPORT unztell64 OF((unzFile file)); +/* + Give the current position in uncompressed data +*/ + +extern int ZEXPORT unzeof OF((unzFile file)); +/* + return 1 if the end of file was reached, 0 elsewhere +*/ + +extern int ZEXPORT unzGetLocalExtrafield OF((unzFile file, + voidp buf, + unsigned len)); +/* + Read extra field from the current file (opened by unzOpenCurrentFile) + This is the local-header version of the extra field (sometimes, there is + more info in the local-header version than in the central-header) + + if buf==NULL, it return the size of the local extra field + + if buf!=NULL, len is the size of the buffer, the extra header is copied in + buf. + the return value is the number of bytes copied in buf, or (if <0) + the error code +*/ + +/***************************************************************************/ + +/* Get the current file offset */ +extern ZPOS64_T ZEXPORT unzGetOffset64 (unzFile file); +extern uLong ZEXPORT unzGetOffset (unzFile file); + +/* Set the current file offset */ +extern int ZEXPORT unzSetOffset64 (unzFile file, ZPOS64_T pos); +extern int ZEXPORT unzSetOffset (unzFile file, uLong pos); + + + +#ifdef __cplusplus +} +#endif + +#endif /* _unz64_H */ diff --git a/storage/connect/value.cpp b/storage/connect/value.cpp index 64d0e13e8c4..ced690e77c0 100644 --- a/storage/connect/value.cpp +++ b/storage/connect/value.cpp @@ -1,7 +1,7 @@ /************* Value C++ Functions Source Code File (.CPP) *************/ -/* Name: VALUE.CPP Version 2.5 */ +/* Name: VALUE.CPP Version 2.6 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 2001-2015 */ +/* (C) Copyright to the author Olivier BERTRAND 2001-2016 */ /* */ /* This file contains the VALUE and derived classes family functions. */ /* These classes contain values of different types. They are used so */ @@ -792,19 +792,29 @@ uchar TYPVAL<uchar>::GetTypedValue(PVBLK blk, int n) /***********************************************************************/ /* TYPVAL SetBinValue: with bytes extracted from a line. */ +/* Currently only used reading column of binary files. */ /***********************************************************************/ template <class TYPE> void TYPVAL<TYPE>::SetBinValue(void *p) - { - Tval = *(TYPE *)p; - Null = false; - } // end of SetBinValue +{ +#if defined(UNALIGNED_OK) + // x86 can cast non-aligned memory directly + Tval = *(TYPE *)p; +#else + // Prevent unaligned memory access on MIPS and ArmHF platforms. + // Make use of memcpy instead of straight pointer dereferencing. + // Currently only used by WriteColumn of binary files. + // From original author: Vicentiu Ciorbaru <vicentiu@mariadb.org> + memcpy(&Tval, p, sizeof(TYPE)); +#endif + Null = false; +} // end of SetBinValue /***********************************************************************/ /* GetBinValue: fill a buffer with the internal binary value. */ /* This function checks whether the buffer length is enough and */ /* returns true if not. Actual filling occurs only if go is true. */ -/* Currently used by WriteColumn of binary files. */ +/* Currently only used writing column of binary files. */ /***********************************************************************/ template <class TYPE> bool TYPVAL<TYPE>::GetBinValue(void *buf, int buflen, bool go) @@ -819,7 +829,16 @@ bool TYPVAL<TYPE>::GetBinValue(void *buf, int buflen, bool go) //#endif if (go) - *(TYPE *)buf = Tval; +#if defined(UNALIGNED_OK) + // x86 can cast non-aligned memory directly + *(TYPE *)buf = Tval; +#else + // Prevent unaligned memory access on MIPS and ArmHF platforms. + // Make use of memcpy instead of straight pointer dereferencing. + // Currently only used by WriteColumn of binary files. + // From original author: Vicentiu Ciorbaru <vicentiu@mariadb.org> + memcpy(buf, &Tval, sizeof(TYPE)); +#endif Null = false; return false; diff --git a/storage/connect/value.h b/storage/connect/value.h index c5a381e89da..a670ade4c28 100644 --- a/storage/connect/value.h +++ b/storage/connect/value.h @@ -1,7 +1,7 @@ /**************** Value H Declares Source Code File (.H) ***************/ -/* Name: VALUE.H Version 2.1 */ +/* Name: VALUE.H Version 2.2 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 2001-2014 */ +/* (C) Copyright to the author Olivier BERTRAND 2001-2016 */ /* */ /* This file contains the VALUE and derived classes declares. */ /***********************************************************************/ @@ -17,6 +17,13 @@ #include "block.h" /***********************************************************************/ +/* This should list the processors accepting unaligned numeral values.*/ +/***********************************************************************/ +#if defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) || defined(_M_X64) || defined(_M_AMD64) || defined(_M_IA64) +#define UNALIGNED_OK +#endif + +/***********************************************************************/ /* Types used in some class definitions. */ /***********************************************************************/ enum CONV {CNV_ANY = 0, /* Convert to any type */ @@ -116,27 +123,46 @@ class DllExport VALUE : public BLOCK { virtual bool Compute(PGLOBAL g, PVAL *vp, int np, OPVAL op); virtual bool FormatValue(PVAL vp, char *fmt) = 0; - /** - Set value from a non-aligned in-memory value in the machine byte order. - TYPE can be either of: - - int, short, longlong - - uint, ushort, ulonglong - - float, double - @param - a pointer to a non-aligned value of type TYPE. - */ - template<typename TYPE> - void SetValueNonAligned(const char *p) - { -#if defined(__i386__) || defined(__x86_64__) - SetValue(*((TYPE*) p)); // x86 can cast non-aligned memory directly + /** + Set value from a non-aligned in-memory value in the machine byte order. + TYPE can be either of: + - int, short, longlong + - uint, ushort, ulonglong + - float, double + @param - a pointer to a non-aligned value of type TYPE. + */ + template<typename TYPE> + void SetValueNonAligned(const char *p) + { +#if defined(UNALIGNED_OK) + SetValue(*((TYPE*)p)); // x86 can cast non-aligned memory directly #else - TYPE tmp; // a slower version for non-x86 platforms - memcpy(&tmp, p, sizeof(tmp)); - SetValue(tmp); + TYPE tmp; // a slower version for non-x86 platforms + memcpy(&tmp, p, sizeof(tmp)); + SetValue(tmp); #endif - } + } // end of SetValueNonAligned + + /** + Get value from a non-aligned in-memory value in the machine byte order. + TYPE can be either of: + - int, short, longlong + - uint, ushort, ulonglong + - float, double + @params - a pointer to a non-aligned value of type TYPE, the TYPE value. + */ + template<typename TYPE> + void GetValueNonAligned(char *p, TYPE n) + { +#if defined(UNALIGNED_OK) + *(TYPE *)p = n; // x86 can cast non-aligned memory directly +#else + TYPE tmp = n; // a slower version for non-x86 platforms + memcpy(p, &tmp, sizeof(tmp)); +#endif + } // end of SetValueNonAligned - protected: +protected: virtual bool SetConstFormat(PGLOBAL, FORMAT&) = 0; const char *GetXfmt(void); diff --git a/storage/connect/xindex.cpp b/storage/connect/xindex.cpp index 56312630278..a2cf4e77b80 100755 --- a/storage/connect/xindex.cpp +++ b/storage/connect/xindex.cpp @@ -45,7 +45,9 @@ //nclude "array.h" #include "filamtxt.h" #include "tabdos.h" +#if defined(VCT_SUPPORT) #include "tabvct.h" +#endif // VCT_SUPPORT /***********************************************************************/ /* Macro or external routine definition */ @@ -293,9 +295,11 @@ bool XINDEX::AddColumns(void) return false; // Not applying to static index else if (IsMul()) return false; // Not done yet for multiple index - else if (Tbxp->GetAmType() == TYPE_AM_VCT && ((PTDBVCT)Tbxp)->IsSplit()) +#if defined(VCT_SUPPORT) + else if (Tbxp->GetAmType() == TYPE_AM_VCT && ((PTDBVCT)Tbxp)->IsSplit()) return false; // This would require to read additional files - else +#endif // VCT_SUPPORT + else return true; } // end of AddColumns diff --git a/storage/connect/zip.c b/storage/connect/zip.c new file mode 100644 index 00000000000..ea54853e858 --- /dev/null +++ b/storage/connect/zip.c @@ -0,0 +1,2007 @@ +/* zip.c -- IO on .zip files using zlib + Version 1.1, February 14h, 2010 + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications for Zip64 support + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + + Changes + Oct-2009 - Mathias Svensson - Remove old C style function prototypes + Oct-2009 - Mathias Svensson - Added Zip64 Support when creating new file archives + Oct-2009 - Mathias Svensson - Did some code cleanup and refactoring to get better overview of some functions. + Oct-2009 - Mathias Svensson - Added zipRemoveExtraInfoBlock to strip extra field data from its ZIP64 data + It is used when recreting zip archive with RAW when deleting items from a zip. + ZIP64 data is automaticly added to items that needs it, and existing ZIP64 data need to be removed. + Oct-2009 - Mathias Svensson - Added support for BZIP2 as compression mode (bzip2 lib is required) + Jan-2010 - back to unzip and minizip 1.0 name scheme, with compatibility layer + +*/ + + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include "zlib.h" +#include "zip.h" + +#ifdef STDC +# include <stddef.h> +# include <string.h> +# include <stdlib.h> +#endif +#ifdef NO_ERRNO_H + extern int errno; +#else +# include <errno.h> +#endif + + +#ifndef local +# define local static +#endif +/* compile with -Dlocal if your debugger can't find static symbols */ + +#ifndef VERSIONMADEBY +# define VERSIONMADEBY (0x0) /* platform depedent */ +#endif + +#ifndef Z_BUFSIZE +#define Z_BUFSIZE (64*1024) //(16384) +#endif + +#ifndef Z_MAXFILENAMEINZIP +#define Z_MAXFILENAMEINZIP (256) +#endif + +#ifndef ALLOC +# define ALLOC(size) (malloc(size)) +#endif +#ifndef TRYFREE +# define TRYFREE(p) {if (p) free(p);} +#endif + +/* +#define SIZECENTRALDIRITEM (0x2e) +#define SIZEZIPLOCALHEADER (0x1e) +*/ + +/* I've found an old Unix (a SunOS 4.1.3_U1) without all SEEK_* defined.... */ + + +// NOT sure that this work on ALL platform +#define MAKEULONG64(a, b) ((ZPOS64_T)(((unsigned long)(a)) | ((ZPOS64_T)((unsigned long)(b))) << 32)) + +#ifndef SEEK_CUR +#define SEEK_CUR 1 +#endif + +#ifndef SEEK_END +#define SEEK_END 2 +#endif + +#ifndef SEEK_SET +#define SEEK_SET 0 +#endif + +#ifndef DEF_MEM_LEVEL +#if MAX_MEM_LEVEL >= 8 +# define DEF_MEM_LEVEL 8 +#else +# define DEF_MEM_LEVEL MAX_MEM_LEVEL +#endif +#endif +const char zip_copyright[] =" zip 1.01 Copyright 1998-2004 Gilles Vollant - http://www.winimage.com/zLibDll"; + + +#define SIZEDATA_INDATABLOCK (4096-(4*4)) + +#define LOCALHEADERMAGIC (0x04034b50) +#define CENTRALHEADERMAGIC (0x02014b50) +#define ENDHEADERMAGIC (0x06054b50) +#define ZIP64ENDHEADERMAGIC (0x6064b50) +#define ZIP64ENDLOCHEADERMAGIC (0x7064b50) + +#define FLAG_LOCALHEADER_OFFSET (0x06) +#define CRC_LOCALHEADER_OFFSET (0x0e) + +#define SIZECENTRALHEADER (0x2e) /* 46 */ + +typedef struct linkedlist_datablock_internal_s +{ + struct linkedlist_datablock_internal_s* next_datablock; + uLong avail_in_this_block; + uLong filled_in_this_block; + uLong unused; /* for future use and alignement */ + unsigned char data[SIZEDATA_INDATABLOCK]; +} linkedlist_datablock_internal; + +typedef struct linkedlist_data_s +{ + linkedlist_datablock_internal* first_block; + linkedlist_datablock_internal* last_block; +} linkedlist_data; + + +typedef struct +{ + z_stream stream; /* zLib stream structure for inflate */ +#ifdef HAVE_BZIP2 + bz_stream bstream; /* bzLib stream structure for bziped */ +#endif + + int stream_initialised; /* 1 is stream is initialised */ + uInt pos_in_buffered_data; /* last written byte in buffered_data */ + + ZPOS64_T pos_local_header; /* offset of the local header of the file + currenty writing */ + char* central_header; /* central header data for the current file */ + uLong size_centralExtra; + uLong size_centralheader; /* size of the central header for cur file */ + uLong size_centralExtraFree; /* Extra bytes allocated to the centralheader but that are not used */ + uLong flag; /* flag of the file currently writing */ + + int method; /* compression method of file currenty wr.*/ + int raw; /* 1 for directly writing raw data */ + Byte buffered_data[Z_BUFSIZE];/* buffer contain compressed data to be writ*/ + uLong dosDate; + uLong crc32; + int encrypt; + int zip64; /* Add ZIP64 extened information in the extra field */ + ZPOS64_T pos_zip64extrainfo; + ZPOS64_T totalCompressedData; + ZPOS64_T totalUncompressedData; +#ifndef NOCRYPT + unsigned long keys[3]; /* keys defining the pseudo-random sequence */ + const z_crc_t* pcrc_32_tab; + int crypt_header_size; +#endif +} curfile64_info; + +typedef struct +{ + zlib_filefunc64_32_def z_filefunc; + voidpf filestream; /* io structore of the zipfile */ + linkedlist_data central_dir;/* datablock with central dir in construction*/ + int in_opened_file_inzip; /* 1 if a file in the zip is currently writ.*/ + curfile64_info ci; /* info on the file curretly writing */ + + ZPOS64_T begin_pos; /* position of the beginning of the zipfile */ + ZPOS64_T add_position_when_writting_offset; + ZPOS64_T number_entry; + +#ifndef NO_ADDFILEINEXISTINGZIP + char *globalcomment; +#endif + +} zip64_internal; + + +#ifndef NOCRYPT +#define INCLUDECRYPTINGCODE_IFCRYPTALLOWED +#include "crypt.h" +#endif + +local linkedlist_datablock_internal* allocate_new_datablock() +{ + linkedlist_datablock_internal* ldi; + ldi = (linkedlist_datablock_internal*) + ALLOC(sizeof(linkedlist_datablock_internal)); + if (ldi!=NULL) + { + ldi->next_datablock = NULL ; + ldi->filled_in_this_block = 0 ; + ldi->avail_in_this_block = SIZEDATA_INDATABLOCK ; + } + return ldi; +} + +local void free_datablock(linkedlist_datablock_internal* ldi) +{ + while (ldi!=NULL) + { + linkedlist_datablock_internal* ldinext = ldi->next_datablock; + TRYFREE(ldi); + ldi = ldinext; + } +} + +local void init_linkedlist(linkedlist_data* ll) +{ + ll->first_block = ll->last_block = NULL; +} + +local void free_linkedlist(linkedlist_data* ll) +{ + free_datablock(ll->first_block); + ll->first_block = ll->last_block = NULL; +} + + +local int add_data_in_datablock(linkedlist_data* ll, const void* buf, uLong len) +{ + linkedlist_datablock_internal* ldi; + const unsigned char* from_copy; + + if (ll==NULL) + return ZIP_INTERNALERROR; + + if (ll->last_block == NULL) + { + ll->first_block = ll->last_block = allocate_new_datablock(); + if (ll->first_block == NULL) + return ZIP_INTERNALERROR; + } + + ldi = ll->last_block; + from_copy = (unsigned char*)buf; + + while (len>0) + { + uInt copy_this; + uInt i; + unsigned char* to_copy; + + if (ldi->avail_in_this_block==0) + { + ldi->next_datablock = allocate_new_datablock(); + if (ldi->next_datablock == NULL) + return ZIP_INTERNALERROR; + ldi = ldi->next_datablock ; + ll->last_block = ldi; + } + + if (ldi->avail_in_this_block < len) + copy_this = (uInt)ldi->avail_in_this_block; + else + copy_this = (uInt)len; + + to_copy = &(ldi->data[ldi->filled_in_this_block]); + + for (i=0;i<copy_this;i++) + *(to_copy+i)=*(from_copy+i); + + ldi->filled_in_this_block += copy_this; + ldi->avail_in_this_block -= copy_this; + from_copy += copy_this ; + len -= copy_this; + } + return ZIP_OK; +} + + + +/****************************************************************************/ + +#ifndef NO_ADDFILEINEXISTINGZIP +/* =========================================================================== + Inputs a long in LSB order to the given file + nbByte == 1, 2 ,4 or 8 (byte, short or long, ZPOS64_T) +*/ + +local int zip64local_putValue OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, ZPOS64_T x, int nbByte)); +local int zip64local_putValue (const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, ZPOS64_T x, int nbByte) +{ + unsigned char buf[8]; + int n; + for (n = 0; n < nbByte; n++) + { + buf[n] = (unsigned char)(x & 0xff); + x >>= 8; + } + if (x != 0) + { /* data overflow - hack for ZIP64 (X Roche) */ + for (n = 0; n < nbByte; n++) + { + buf[n] = 0xff; + } + } + + if (ZWRITE64(*pzlib_filefunc_def,filestream,buf,nbByte)!=(uLong)nbByte) + return ZIP_ERRNO; + else + return ZIP_OK; +} + +local void zip64local_putValue_inmemory OF((void* dest, ZPOS64_T x, int nbByte)); +local void zip64local_putValue_inmemory (void* dest, ZPOS64_T x, int nbByte) +{ + unsigned char* buf=(unsigned char*)dest; + int n; + for (n = 0; n < nbByte; n++) { + buf[n] = (unsigned char)(x & 0xff); + x >>= 8; + } + + if (x != 0) + { /* data overflow - hack for ZIP64 */ + for (n = 0; n < nbByte; n++) + { + buf[n] = 0xff; + } + } +} + +/****************************************************************************/ + + +local uLong zip64local_TmzDateToDosDate(const tm_zip* ptm) +{ + uLong year = (uLong)ptm->tm_year; + if (year>=1980) + year-=1980; + else if (year>=80) + year-=80; + return + (uLong) (((ptm->tm_mday) + (32 * (ptm->tm_mon+1)) + (512 * year)) << 16) | + ((ptm->tm_sec/2) + (32* ptm->tm_min) + (2048 * (uLong)ptm->tm_hour)); +} + + +/****************************************************************************/ + +local int zip64local_getByte OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, int *pi)); + +local int zip64local_getByte(const zlib_filefunc64_32_def* pzlib_filefunc_def,voidpf filestream,int* pi) +{ + unsigned char c; + int err = (int)ZREAD64(*pzlib_filefunc_def,filestream,&c,1); + if (err==1) + { + *pi = (int)c; + return ZIP_OK; + } + else + { + if (ZERROR64(*pzlib_filefunc_def,filestream)) + return ZIP_ERRNO; + else + return ZIP_EOF; + } +} + + +/* =========================================================================== + Reads a long in LSB order from the given gz_stream. Sets +*/ +local int zip64local_getShort OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, uLong *pX)); + +local int zip64local_getShort (const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, uLong* pX) +{ + uLong x ; + int i = 0; + int err; + + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x = (uLong)i; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((uLong)i)<<8; + + if (err==ZIP_OK) + *pX = x; + else + *pX = 0; + return err; +} + +local int zip64local_getLong OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, uLong *pX)); + +local int zip64local_getLong (const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, uLong* pX) +{ + uLong x ; + int i = 0; + int err; + + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x = (uLong)i; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((uLong)i)<<8; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((uLong)i)<<16; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((uLong)i)<<24; + + if (err==ZIP_OK) + *pX = x; + else + *pX = 0; + return err; +} + +local int zip64local_getLong64 OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, ZPOS64_T *pX)); + + +local int zip64local_getLong64 (const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, ZPOS64_T *pX) +{ + ZPOS64_T x; + int i = 0; + int err; + + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x = (ZPOS64_T)i; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<8; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<16; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<24; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<32; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<40; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<48; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<56; + + if (err==ZIP_OK) + *pX = x; + else + *pX = 0; + + return err; +} + +#ifndef BUFREADCOMMENT +#define BUFREADCOMMENT (0x400) +#endif +/* + Locate the Central directory of a zipfile (at the end, just before + the global comment) +*/ +local ZPOS64_T zip64local_SearchCentralDir OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream)); + +local ZPOS64_T zip64local_SearchCentralDir(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream) +{ + unsigned char* buf; + ZPOS64_T uSizeFile; + ZPOS64_T uBackRead; + ZPOS64_T uMaxBack=0xffff; /* maximum size of global comment */ + ZPOS64_T uPosFound=0; + + if (ZSEEK64(*pzlib_filefunc_def,filestream,0,ZLIB_FILEFUNC_SEEK_END) != 0) + return 0; + + + uSizeFile = ZTELL64(*pzlib_filefunc_def,filestream); + + if (uMaxBack>uSizeFile) + uMaxBack = uSizeFile; + + buf = (unsigned char*)ALLOC(BUFREADCOMMENT+4); + if (buf==NULL) + return 0; + + uBackRead = 4; + while (uBackRead<uMaxBack) + { + uLong uReadSize; + ZPOS64_T uReadPos ; + int i; + if (uBackRead+BUFREADCOMMENT>uMaxBack) + uBackRead = uMaxBack; + else + uBackRead+=BUFREADCOMMENT; + uReadPos = uSizeFile-uBackRead ; + + uReadSize = ((BUFREADCOMMENT+4) < (uSizeFile-uReadPos)) ? + (BUFREADCOMMENT+4) : (uLong)(uSizeFile-uReadPos); + if (ZSEEK64(*pzlib_filefunc_def,filestream,uReadPos,ZLIB_FILEFUNC_SEEK_SET)!=0) + break; + + if (ZREAD64(*pzlib_filefunc_def,filestream,buf,uReadSize)!=uReadSize) + break; + + for (i=(int)uReadSize-3; (i--)>0;) + if (((*(buf+i))==0x50) && ((*(buf+i+1))==0x4b) && + ((*(buf+i+2))==0x05) && ((*(buf+i+3))==0x06)) + { + uPosFound = uReadPos+i; + break; + } + + if (uPosFound!=0) + break; + } + TRYFREE(buf); + return uPosFound; +} + +/* +Locate the End of Zip64 Central directory locator and from there find the CD of a zipfile (at the end, just before +the global comment) +*/ +local ZPOS64_T zip64local_SearchCentralDir64 OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream)); + +local ZPOS64_T zip64local_SearchCentralDir64(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream) +{ + unsigned char* buf; + ZPOS64_T uSizeFile; + ZPOS64_T uBackRead; + ZPOS64_T uMaxBack=0xffff; /* maximum size of global comment */ + ZPOS64_T uPosFound=0; + uLong uL; + ZPOS64_T relativeOffset; + + if (ZSEEK64(*pzlib_filefunc_def,filestream,0,ZLIB_FILEFUNC_SEEK_END) != 0) + return 0; + + uSizeFile = ZTELL64(*pzlib_filefunc_def,filestream); + + if (uMaxBack>uSizeFile) + uMaxBack = uSizeFile; + + buf = (unsigned char*)ALLOC(BUFREADCOMMENT+4); + if (buf==NULL) + return 0; + + uBackRead = 4; + while (uBackRead<uMaxBack) + { + uLong uReadSize; + ZPOS64_T uReadPos; + int i; + if (uBackRead+BUFREADCOMMENT>uMaxBack) + uBackRead = uMaxBack; + else + uBackRead+=BUFREADCOMMENT; + uReadPos = uSizeFile-uBackRead ; + + uReadSize = ((BUFREADCOMMENT+4) < (uSizeFile-uReadPos)) ? + (BUFREADCOMMENT+4) : (uLong)(uSizeFile-uReadPos); + if (ZSEEK64(*pzlib_filefunc_def,filestream,uReadPos,ZLIB_FILEFUNC_SEEK_SET)!=0) + break; + + if (ZREAD64(*pzlib_filefunc_def,filestream,buf,uReadSize)!=uReadSize) + break; + + for (i=(int)uReadSize-3; (i--)>0;) + { + // Signature "0x07064b50" Zip64 end of central directory locater + if (((*(buf+i))==0x50) && ((*(buf+i+1))==0x4b) && ((*(buf+i+2))==0x06) && ((*(buf+i+3))==0x07)) + { + uPosFound = uReadPos+i; + break; + } + } + + if (uPosFound!=0) + break; + } + + TRYFREE(buf); + if (uPosFound == 0) + return 0; + + /* Zip64 end of central directory locator */ + if (ZSEEK64(*pzlib_filefunc_def,filestream, uPosFound,ZLIB_FILEFUNC_SEEK_SET)!=0) + return 0; + + /* the signature, already checked */ + if (zip64local_getLong(pzlib_filefunc_def,filestream,&uL)!=ZIP_OK) + return 0; + + /* number of the disk with the start of the zip64 end of central directory */ + if (zip64local_getLong(pzlib_filefunc_def,filestream,&uL)!=ZIP_OK) + return 0; + if (uL != 0) + return 0; + + /* relative offset of the zip64 end of central directory record */ + if (zip64local_getLong64(pzlib_filefunc_def,filestream,&relativeOffset)!=ZIP_OK) + return 0; + + /* total number of disks */ + if (zip64local_getLong(pzlib_filefunc_def,filestream,&uL)!=ZIP_OK) + return 0; + if (uL != 1) + return 0; + + /* Goto Zip64 end of central directory record */ + if (ZSEEK64(*pzlib_filefunc_def,filestream, relativeOffset,ZLIB_FILEFUNC_SEEK_SET)!=0) + return 0; + + /* the signature */ + if (zip64local_getLong(pzlib_filefunc_def,filestream,&uL)!=ZIP_OK) + return 0; + + if (uL != 0x06064b50) // signature of 'Zip64 end of central directory' + return 0; + + return relativeOffset; +} + +int LoadCentralDirectoryRecord(zip64_internal* pziinit) +{ + int err=ZIP_OK; + ZPOS64_T byte_before_the_zipfile;/* byte before the zipfile, (>0 for sfx)*/ + + ZPOS64_T size_central_dir; /* size of the central directory */ + ZPOS64_T offset_central_dir; /* offset of start of central directory */ + ZPOS64_T central_pos; + uLong uL; + + uLong number_disk; /* number of the current dist, used for + spaning ZIP, unsupported, always 0*/ + uLong number_disk_with_CD; /* number the the disk with central dir, used + for spaning ZIP, unsupported, always 0*/ + ZPOS64_T number_entry; + ZPOS64_T number_entry_CD; /* total number of entries in + the central dir + (same than number_entry on nospan) */ + uLong VersionMadeBy; + uLong VersionNeeded; + uLong size_comment; + + int hasZIP64Record = 0; + + // check first if we find a ZIP64 record + central_pos = zip64local_SearchCentralDir64(&pziinit->z_filefunc,pziinit->filestream); + if(central_pos > 0) + { + hasZIP64Record = 1; + } + else if(central_pos == 0) + { + central_pos = zip64local_SearchCentralDir(&pziinit->z_filefunc,pziinit->filestream); + } + +/* disable to allow appending to empty ZIP archive + if (central_pos==0) + err=ZIP_ERRNO; +*/ + + if(hasZIP64Record) + { + ZPOS64_T sizeEndOfCentralDirectory; + if (ZSEEK64(pziinit->z_filefunc, pziinit->filestream, central_pos, ZLIB_FILEFUNC_SEEK_SET) != 0) + err=ZIP_ERRNO; + + /* the signature, already checked */ + if (zip64local_getLong(&pziinit->z_filefunc, pziinit->filestream,&uL)!=ZIP_OK) + err=ZIP_ERRNO; + + /* size of zip64 end of central directory record */ + if (zip64local_getLong64(&pziinit->z_filefunc, pziinit->filestream, &sizeEndOfCentralDirectory)!=ZIP_OK) + err=ZIP_ERRNO; + + /* version made by */ + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream, &VersionMadeBy)!=ZIP_OK) + err=ZIP_ERRNO; + + /* version needed to extract */ + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream, &VersionNeeded)!=ZIP_OK) + err=ZIP_ERRNO; + + /* number of this disk */ + if (zip64local_getLong(&pziinit->z_filefunc, pziinit->filestream,&number_disk)!=ZIP_OK) + err=ZIP_ERRNO; + + /* number of the disk with the start of the central directory */ + if (zip64local_getLong(&pziinit->z_filefunc, pziinit->filestream,&number_disk_with_CD)!=ZIP_OK) + err=ZIP_ERRNO; + + /* total number of entries in the central directory on this disk */ + if (zip64local_getLong64(&pziinit->z_filefunc, pziinit->filestream, &number_entry)!=ZIP_OK) + err=ZIP_ERRNO; + + /* total number of entries in the central directory */ + if (zip64local_getLong64(&pziinit->z_filefunc, pziinit->filestream,&number_entry_CD)!=ZIP_OK) + err=ZIP_ERRNO; + + if ((number_entry_CD!=number_entry) || (number_disk_with_CD!=0) || (number_disk!=0)) + err=ZIP_BADZIPFILE; + + /* size of the central directory */ + if (zip64local_getLong64(&pziinit->z_filefunc, pziinit->filestream,&size_central_dir)!=ZIP_OK) + err=ZIP_ERRNO; + + /* offset of start of central directory with respect to the + starting disk number */ + if (zip64local_getLong64(&pziinit->z_filefunc, pziinit->filestream,&offset_central_dir)!=ZIP_OK) + err=ZIP_ERRNO; + + // TODO.. + // read the comment from the standard central header. + size_comment = 0; + } + else + { + // Read End of central Directory info + if (ZSEEK64(pziinit->z_filefunc, pziinit->filestream, central_pos,ZLIB_FILEFUNC_SEEK_SET)!=0) + err=ZIP_ERRNO; + + /* the signature, already checked */ + if (zip64local_getLong(&pziinit->z_filefunc, pziinit->filestream,&uL)!=ZIP_OK) + err=ZIP_ERRNO; + + /* number of this disk */ + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream,&number_disk)!=ZIP_OK) + err=ZIP_ERRNO; + + /* number of the disk with the start of the central directory */ + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream,&number_disk_with_CD)!=ZIP_OK) + err=ZIP_ERRNO; + + /* total number of entries in the central dir on this disk */ + number_entry = 0; + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream, &uL)!=ZIP_OK) + err=ZIP_ERRNO; + else + number_entry = uL; + + /* total number of entries in the central dir */ + number_entry_CD = 0; + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream, &uL)!=ZIP_OK) + err=ZIP_ERRNO; + else + number_entry_CD = uL; + + if ((number_entry_CD!=number_entry) || (number_disk_with_CD!=0) || (number_disk!=0)) + err=ZIP_BADZIPFILE; + + /* size of the central directory */ + size_central_dir = 0; + if (zip64local_getLong(&pziinit->z_filefunc, pziinit->filestream, &uL)!=ZIP_OK) + err=ZIP_ERRNO; + else + size_central_dir = uL; + + /* offset of start of central directory with respect to the starting disk number */ + offset_central_dir = 0; + if (zip64local_getLong(&pziinit->z_filefunc, pziinit->filestream, &uL)!=ZIP_OK) + err=ZIP_ERRNO; + else + offset_central_dir = uL; + + + /* zipfile global comment length */ + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream, &size_comment)!=ZIP_OK) + err=ZIP_ERRNO; + } + + if ((central_pos<offset_central_dir+size_central_dir) && + (err==ZIP_OK)) + err=ZIP_BADZIPFILE; + + if (err!=ZIP_OK) + { + ZCLOSE64(pziinit->z_filefunc, pziinit->filestream); + return ZIP_ERRNO; + } + + if (size_comment>0) + { + pziinit->globalcomment = (char*)ALLOC(size_comment+1); + if (pziinit->globalcomment) + { + size_comment = ZREAD64(pziinit->z_filefunc, pziinit->filestream, pziinit->globalcomment,size_comment); + pziinit->globalcomment[size_comment]=0; + } + } + + byte_before_the_zipfile = central_pos - (offset_central_dir+size_central_dir); + pziinit->add_position_when_writting_offset = byte_before_the_zipfile; + + { + ZPOS64_T size_central_dir_to_read = size_central_dir; + size_t buf_size = SIZEDATA_INDATABLOCK; + void* buf_read = (void*)ALLOC(buf_size); + if (ZSEEK64(pziinit->z_filefunc, pziinit->filestream, offset_central_dir + byte_before_the_zipfile, ZLIB_FILEFUNC_SEEK_SET) != 0) + err=ZIP_ERRNO; + + while ((size_central_dir_to_read>0) && (err==ZIP_OK)) + { + ZPOS64_T read_this = SIZEDATA_INDATABLOCK; + if (read_this > size_central_dir_to_read) + read_this = size_central_dir_to_read; + + if (ZREAD64(pziinit->z_filefunc, pziinit->filestream,buf_read,(uLong)read_this) != read_this) + err=ZIP_ERRNO; + + if (err==ZIP_OK) + err = add_data_in_datablock(&pziinit->central_dir,buf_read, (uLong)read_this); + + size_central_dir_to_read-=read_this; + } + TRYFREE(buf_read); + } + pziinit->begin_pos = byte_before_the_zipfile; + pziinit->number_entry = number_entry_CD; + + if (ZSEEK64(pziinit->z_filefunc, pziinit->filestream, offset_central_dir+byte_before_the_zipfile,ZLIB_FILEFUNC_SEEK_SET) != 0) + err=ZIP_ERRNO; + + return err; +} + + +#endif /* !NO_ADDFILEINEXISTINGZIP*/ + + +/************************************************************/ +extern zipFile ZEXPORT zipOpen3 (const void *pathname, int append, zipcharpc* globalcomment, zlib_filefunc64_32_def* pzlib_filefunc64_32_def) +{ + zip64_internal ziinit; + zip64_internal* zi; + int err=ZIP_OK; + + ziinit.z_filefunc.zseek32_file = NULL; + ziinit.z_filefunc.ztell32_file = NULL; + if (pzlib_filefunc64_32_def==NULL) + fill_fopen64_filefunc(&ziinit.z_filefunc.zfile_func64); + else + ziinit.z_filefunc = *pzlib_filefunc64_32_def; + + ziinit.filestream = ZOPEN64(ziinit.z_filefunc, + pathname, + (append == APPEND_STATUS_CREATE) ? + (ZLIB_FILEFUNC_MODE_READ | ZLIB_FILEFUNC_MODE_WRITE | ZLIB_FILEFUNC_MODE_CREATE) : + (ZLIB_FILEFUNC_MODE_READ | ZLIB_FILEFUNC_MODE_WRITE | ZLIB_FILEFUNC_MODE_EXISTING)); + + if (ziinit.filestream == NULL) + return NULL; + + if (append == APPEND_STATUS_CREATEAFTER) + ZSEEK64(ziinit.z_filefunc,ziinit.filestream,0,SEEK_END); + + ziinit.begin_pos = ZTELL64(ziinit.z_filefunc,ziinit.filestream); + ziinit.in_opened_file_inzip = 0; + ziinit.ci.stream_initialised = 0; + ziinit.number_entry = 0; + ziinit.add_position_when_writting_offset = 0; + init_linkedlist(&(ziinit.central_dir)); + + + + zi = (zip64_internal*)ALLOC(sizeof(zip64_internal)); + if (zi==NULL) + { + ZCLOSE64(ziinit.z_filefunc,ziinit.filestream); + return NULL; + } + + /* now we add file in a zipfile */ +# ifndef NO_ADDFILEINEXISTINGZIP + ziinit.globalcomment = NULL; + if (append == APPEND_STATUS_ADDINZIP) + { + // Read and Cache Central Directory Records + err = LoadCentralDirectoryRecord(&ziinit); + } + + if (globalcomment) + { + *globalcomment = ziinit.globalcomment; + } +# endif /* !NO_ADDFILEINEXISTINGZIP*/ + + if (err != ZIP_OK) + { +# ifndef NO_ADDFILEINEXISTINGZIP + TRYFREE(ziinit.globalcomment); +# endif /* !NO_ADDFILEINEXISTINGZIP*/ + TRYFREE(zi); + return NULL; + } + else + { + *zi = ziinit; + return (zipFile)zi; + } +} + +extern zipFile ZEXPORT zipOpen2 (const char *pathname, int append, zipcharpc* globalcomment, zlib_filefunc_def* pzlib_filefunc32_def) +{ + if (pzlib_filefunc32_def != NULL) + { + zlib_filefunc64_32_def zlib_filefunc64_32_def_fill; + fill_zlib_filefunc64_32_def_from_filefunc32(&zlib_filefunc64_32_def_fill,pzlib_filefunc32_def); + return zipOpen3(pathname, append, globalcomment, &zlib_filefunc64_32_def_fill); + } + else + return zipOpen3(pathname, append, globalcomment, NULL); +} + +extern zipFile ZEXPORT zipOpen2_64 (const void *pathname, int append, zipcharpc* globalcomment, zlib_filefunc64_def* pzlib_filefunc_def) +{ + if (pzlib_filefunc_def != NULL) + { + zlib_filefunc64_32_def zlib_filefunc64_32_def_fill; + zlib_filefunc64_32_def_fill.zfile_func64 = *pzlib_filefunc_def; + zlib_filefunc64_32_def_fill.ztell32_file = NULL; + zlib_filefunc64_32_def_fill.zseek32_file = NULL; + return zipOpen3(pathname, append, globalcomment, &zlib_filefunc64_32_def_fill); + } + else + return zipOpen3(pathname, append, globalcomment, NULL); +} + + + +extern zipFile ZEXPORT zipOpen (const char* pathname, int append) +{ + return zipOpen3((const void*)pathname,append,NULL,NULL); +} + +extern zipFile ZEXPORT zipOpen64 (const void* pathname, int append) +{ + return zipOpen3(pathname,append,NULL,NULL); +} + +int Write_LocalFileHeader(zip64_internal* zi, const char* filename, uInt size_extrafield_local, const void* extrafield_local) +{ + /* write the local header */ + int err; + uInt size_filename = (uInt)strlen(filename); + uInt size_extrafield = size_extrafield_local; + + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)LOCALHEADERMAGIC, 4); + + if (err==ZIP_OK) + { + if(zi->ci.zip64) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)45,2);/* version needed to extract */ + else + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)20,2);/* version needed to extract */ + } + + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)zi->ci.flag,2); + + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)zi->ci.method,2); + + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)zi->ci.dosDate,4); + + // CRC / Compressed size / Uncompressed size will be filled in later and rewritten later + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,4); /* crc 32, unknown */ + if (err==ZIP_OK) + { + if(zi->ci.zip64) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0xFFFFFFFF,4); /* compressed size, unknown */ + else + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,4); /* compressed size, unknown */ + } + if (err==ZIP_OK) + { + if(zi->ci.zip64) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0xFFFFFFFF,4); /* uncompressed size, unknown */ + else + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,4); /* uncompressed size, unknown */ + } + + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)size_filename,2); + + if(zi->ci.zip64) + { + size_extrafield += 20; + } + + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)size_extrafield,2); + + if ((err==ZIP_OK) && (size_filename > 0)) + { + if (ZWRITE64(zi->z_filefunc,zi->filestream,filename,size_filename)!=size_filename) + err = ZIP_ERRNO; + } + + if ((err==ZIP_OK) && (size_extrafield_local > 0)) + { + if (ZWRITE64(zi->z_filefunc, zi->filestream, extrafield_local, size_extrafield_local) != size_extrafield_local) + err = ZIP_ERRNO; + } + + + if ((err==ZIP_OK) && (zi->ci.zip64)) + { + // write the Zip64 extended info + short HeaderID = 1; + short DataSize = 16; + ZPOS64_T CompressedSize = 0; + ZPOS64_T UncompressedSize = 0; + + // Remember position of Zip64 extended info for the local file header. (needed when we update size after done with file) + zi->ci.pos_zip64extrainfo = ZTELL64(zi->z_filefunc,zi->filestream); + + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, (short)HeaderID,2); + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, (short)DataSize,2); + + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, (ZPOS64_T)UncompressedSize,8); + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, (ZPOS64_T)CompressedSize,8); + } + + return err; +} + +/* + NOTE. + When writing RAW the ZIP64 extended information in extrafield_local and extrafield_global needs to be stripped + before calling this function it can be done with zipRemoveExtraInfoBlock + + It is not done here because then we need to realloc a new buffer since parameters are 'const' and I want to minimize + unnecessary allocations. + */ +extern int ZEXPORT zipOpenNewFileInZip4_64 (zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw, + int windowBits,int memLevel, int strategy, + const char* password, uLong crcForCrypting, + uLong versionMadeBy, uLong flagBase, int zip64) +{ + zip64_internal* zi; + uInt size_filename; + uInt size_comment; + uInt i; + int err = ZIP_OK; + +# ifdef NOCRYPT + (crcForCrypting); + if (password != NULL) + return ZIP_PARAMERROR; +# endif + + if (file == NULL) + return ZIP_PARAMERROR; + +#ifdef HAVE_BZIP2 + if ((method!=0) && (method!=Z_DEFLATED) && (method!=Z_BZIP2ED)) + return ZIP_PARAMERROR; +#else + if ((method!=0) && (method!=Z_DEFLATED)) + return ZIP_PARAMERROR; +#endif + + zi = (zip64_internal*)file; + + if (zi->in_opened_file_inzip == 1) + { + err = zipCloseFileInZip (file); + if (err != ZIP_OK) + return err; + } + + if (filename==NULL) + filename="-"; + + if (comment==NULL) + size_comment = 0; + else + size_comment = (uInt)strlen(comment); + + size_filename = (uInt)strlen(filename); + + if (zipfi == NULL) + zi->ci.dosDate = 0; + else + { + if (zipfi->dosDate != 0) + zi->ci.dosDate = zipfi->dosDate; + else + zi->ci.dosDate = zip64local_TmzDateToDosDate(&zipfi->tmz_date); + } + + zi->ci.flag = flagBase; + if ((level==8) || (level==9)) + zi->ci.flag |= 2; + if (level==2) + zi->ci.flag |= 4; + if (level==1) + zi->ci.flag |= 6; + if (password != NULL) + zi->ci.flag |= 1; + + zi->ci.crc32 = 0; + zi->ci.method = method; + zi->ci.encrypt = 0; + zi->ci.stream_initialised = 0; + zi->ci.pos_in_buffered_data = 0; + zi->ci.raw = raw; + zi->ci.pos_local_header = ZTELL64(zi->z_filefunc,zi->filestream); + + zi->ci.size_centralheader = SIZECENTRALHEADER + size_filename + size_extrafield_global + size_comment; + zi->ci.size_centralExtraFree = 32; // Extra space we have reserved in case we need to add ZIP64 extra info data + + zi->ci.central_header = (char*)ALLOC((uInt)zi->ci.size_centralheader + zi->ci.size_centralExtraFree); + + zi->ci.size_centralExtra = size_extrafield_global; + zip64local_putValue_inmemory(zi->ci.central_header,(uLong)CENTRALHEADERMAGIC,4); + /* version info */ + zip64local_putValue_inmemory(zi->ci.central_header+4,(uLong)versionMadeBy,2); + zip64local_putValue_inmemory(zi->ci.central_header+6,(uLong)20,2); + zip64local_putValue_inmemory(zi->ci.central_header+8,(uLong)zi->ci.flag,2); + zip64local_putValue_inmemory(zi->ci.central_header+10,(uLong)zi->ci.method,2); + zip64local_putValue_inmemory(zi->ci.central_header+12,(uLong)zi->ci.dosDate,4); + zip64local_putValue_inmemory(zi->ci.central_header+16,(uLong)0,4); /*crc*/ + zip64local_putValue_inmemory(zi->ci.central_header+20,(uLong)0,4); /*compr size*/ + zip64local_putValue_inmemory(zi->ci.central_header+24,(uLong)0,4); /*uncompr size*/ + zip64local_putValue_inmemory(zi->ci.central_header+28,(uLong)size_filename,2); + zip64local_putValue_inmemory(zi->ci.central_header+30,(uLong)size_extrafield_global,2); + zip64local_putValue_inmemory(zi->ci.central_header+32,(uLong)size_comment,2); + zip64local_putValue_inmemory(zi->ci.central_header+34,(uLong)0,2); /*disk nm start*/ + + if (zipfi==NULL) + zip64local_putValue_inmemory(zi->ci.central_header+36,(uLong)0,2); + else + zip64local_putValue_inmemory(zi->ci.central_header+36,(uLong)zipfi->internal_fa,2); + + if (zipfi==NULL) + zip64local_putValue_inmemory(zi->ci.central_header+38,(uLong)0,4); + else + zip64local_putValue_inmemory(zi->ci.central_header+38,(uLong)zipfi->external_fa,4); + + if(zi->ci.pos_local_header >= 0xffffffff) + zip64local_putValue_inmemory(zi->ci.central_header+42,(uLong)0xffffffff,4); + else + zip64local_putValue_inmemory(zi->ci.central_header+42,(uLong)zi->ci.pos_local_header - zi->add_position_when_writting_offset,4); + + for (i=0;i<size_filename;i++) + *(zi->ci.central_header+SIZECENTRALHEADER+i) = *(filename+i); + + for (i=0;i<size_extrafield_global;i++) + *(zi->ci.central_header+SIZECENTRALHEADER+size_filename+i) = + *(((const char*)extrafield_global)+i); + + for (i=0;i<size_comment;i++) + *(zi->ci.central_header+SIZECENTRALHEADER+size_filename+ + size_extrafield_global+i) = *(comment+i); + if (zi->ci.central_header == NULL) + return ZIP_INTERNALERROR; + + zi->ci.zip64 = zip64; + zi->ci.totalCompressedData = 0; + zi->ci.totalUncompressedData = 0; + zi->ci.pos_zip64extrainfo = 0; + + err = Write_LocalFileHeader(zi, filename, size_extrafield_local, extrafield_local); + +#ifdef HAVE_BZIP2 + zi->ci.bstream.avail_in = (uInt)0; + zi->ci.bstream.avail_out = (uInt)Z_BUFSIZE; + zi->ci.bstream.next_out = (char*)zi->ci.buffered_data; + zi->ci.bstream.total_in_hi32 = 0; + zi->ci.bstream.total_in_lo32 = 0; + zi->ci.bstream.total_out_hi32 = 0; + zi->ci.bstream.total_out_lo32 = 0; +#endif + + zi->ci.stream.avail_in = (uInt)0; + zi->ci.stream.avail_out = (uInt)Z_BUFSIZE; + zi->ci.stream.next_out = zi->ci.buffered_data; + zi->ci.stream.total_in = 0; + zi->ci.stream.total_out = 0; + zi->ci.stream.data_type = Z_BINARY; + +#ifdef HAVE_BZIP2 + if ((err==ZIP_OK) && (zi->ci.method == Z_DEFLATED || zi->ci.method == Z_BZIP2ED) && (!zi->ci.raw)) +#else + if ((err==ZIP_OK) && (zi->ci.method == Z_DEFLATED) && (!zi->ci.raw)) +#endif + { + if(zi->ci.method == Z_DEFLATED) + { + zi->ci.stream.zalloc = (alloc_func)0; + zi->ci.stream.zfree = (free_func)0; + zi->ci.stream.opaque = (voidpf)0; + + if (windowBits>0) + windowBits = -windowBits; + + err = deflateInit2(&zi->ci.stream, level, Z_DEFLATED, windowBits, memLevel, strategy); + + if (err==Z_OK) + zi->ci.stream_initialised = Z_DEFLATED; + } + else if(zi->ci.method == Z_BZIP2ED) + { +#ifdef HAVE_BZIP2 + // Init BZip stuff here + zi->ci.bstream.bzalloc = 0; + zi->ci.bstream.bzfree = 0; + zi->ci.bstream.opaque = (voidpf)0; + + err = BZ2_bzCompressInit(&zi->ci.bstream, level, 0,35); + if(err == BZ_OK) + zi->ci.stream_initialised = Z_BZIP2ED; +#endif + } + + } + +# ifndef NOCRYPT + zi->ci.crypt_header_size = 0; + if ((err==Z_OK) && (password != NULL)) + { + unsigned char bufHead[RAND_HEAD_LEN]; + unsigned int sizeHead; + zi->ci.encrypt = 1; + zi->ci.pcrc_32_tab = get_crc_table(); + /*init_keys(password,zi->ci.keys,zi->ci.pcrc_32_tab);*/ + + sizeHead=crypthead(password,bufHead,RAND_HEAD_LEN,zi->ci.keys,zi->ci.pcrc_32_tab,crcForCrypting); + zi->ci.crypt_header_size = sizeHead; + + if (ZWRITE64(zi->z_filefunc,zi->filestream,bufHead,sizeHead) != sizeHead) + err = ZIP_ERRNO; + } +# endif + + if (err==Z_OK) + zi->in_opened_file_inzip = 1; + return err; +} + +extern int ZEXPORT zipOpenNewFileInZip4 (zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw, + int windowBits,int memLevel, int strategy, + const char* password, uLong crcForCrypting, + uLong versionMadeBy, uLong flagBase) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, raw, + windowBits, memLevel, strategy, + password, crcForCrypting, versionMadeBy, flagBase, 0); +} + +extern int ZEXPORT zipOpenNewFileInZip3 (zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw, + int windowBits,int memLevel, int strategy, + const char* password, uLong crcForCrypting) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, raw, + windowBits, memLevel, strategy, + password, crcForCrypting, VERSIONMADEBY, 0, 0); +} + +extern int ZEXPORT zipOpenNewFileInZip3_64(zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw, + int windowBits,int memLevel, int strategy, + const char* password, uLong crcForCrypting, int zip64) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, raw, + windowBits, memLevel, strategy, + password, crcForCrypting, VERSIONMADEBY, 0, zip64); +} + +extern int ZEXPORT zipOpenNewFileInZip2(zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, raw, + -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, + NULL, 0, VERSIONMADEBY, 0, 0); +} + +extern int ZEXPORT zipOpenNewFileInZip2_64(zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw, int zip64) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, raw, + -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, + NULL, 0, VERSIONMADEBY, 0, zip64); +} + +extern int ZEXPORT zipOpenNewFileInZip64 (zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void*extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int zip64) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, 0, + -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, + NULL, 0, VERSIONMADEBY, 0, zip64); +} + +extern int ZEXPORT zipOpenNewFileInZip (zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void*extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, 0, + -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, + NULL, 0, VERSIONMADEBY, 0, 0); +} + +local int zip64FlushWriteBuffer(zip64_internal* zi) +{ + int err=ZIP_OK; + + if (zi->ci.encrypt != 0) + { +#ifndef NOCRYPT + uInt i; + int t; + for (i=0;i<zi->ci.pos_in_buffered_data;i++) + zi->ci.buffered_data[i] = zencode(zi->ci.keys, zi->ci.pcrc_32_tab, zi->ci.buffered_data[i],t); +#endif + } + + if (ZWRITE64(zi->z_filefunc,zi->filestream,zi->ci.buffered_data,zi->ci.pos_in_buffered_data) != zi->ci.pos_in_buffered_data) + err = ZIP_ERRNO; + + zi->ci.totalCompressedData += zi->ci.pos_in_buffered_data; + +#ifdef HAVE_BZIP2 + if(zi->ci.method == Z_BZIP2ED) + { + zi->ci.totalUncompressedData += zi->ci.bstream.total_in_lo32; + zi->ci.bstream.total_in_lo32 = 0; + zi->ci.bstream.total_in_hi32 = 0; + } + else +#endif + { + zi->ci.totalUncompressedData += zi->ci.stream.total_in; + zi->ci.stream.total_in = 0; + } + + + zi->ci.pos_in_buffered_data = 0; + + return err; +} + +extern int ZEXPORT zipWriteInFileInZip (zipFile file,const void* buf,unsigned int len) +{ + zip64_internal* zi; + int err=ZIP_OK; + + if (file == NULL) + return ZIP_PARAMERROR; + zi = (zip64_internal*)file; + + if (zi->in_opened_file_inzip == 0) + return ZIP_PARAMERROR; + + zi->ci.crc32 = crc32(zi->ci.crc32,buf,(uInt)len); + +#ifdef HAVE_BZIP2 + if(zi->ci.method == Z_BZIP2ED && (!zi->ci.raw)) + { + zi->ci.bstream.next_in = (void*)buf; + zi->ci.bstream.avail_in = len; + err = BZ_RUN_OK; + + while ((err==BZ_RUN_OK) && (zi->ci.bstream.avail_in>0)) + { + if (zi->ci.bstream.avail_out == 0) + { + if (zip64FlushWriteBuffer(zi) == ZIP_ERRNO) + err = ZIP_ERRNO; + zi->ci.bstream.avail_out = (uInt)Z_BUFSIZE; + zi->ci.bstream.next_out = (char*)zi->ci.buffered_data; + } + + + if(err != BZ_RUN_OK) + break; + + if ((zi->ci.method == Z_BZIP2ED) && (!zi->ci.raw)) + { + uLong uTotalOutBefore_lo = zi->ci.bstream.total_out_lo32; +// uLong uTotalOutBefore_hi = zi->ci.bstream.total_out_hi32; + err=BZ2_bzCompress(&zi->ci.bstream, BZ_RUN); + + zi->ci.pos_in_buffered_data += (uInt)(zi->ci.bstream.total_out_lo32 - uTotalOutBefore_lo) ; + } + } + + if(err == BZ_RUN_OK) + err = ZIP_OK; + } + else +#endif + { + zi->ci.stream.next_in = (Bytef*)buf; + zi->ci.stream.avail_in = len; + + while ((err==ZIP_OK) && (zi->ci.stream.avail_in>0)) + { + if (zi->ci.stream.avail_out == 0) + { + if (zip64FlushWriteBuffer(zi) == ZIP_ERRNO) + err = ZIP_ERRNO; + zi->ci.stream.avail_out = (uInt)Z_BUFSIZE; + zi->ci.stream.next_out = zi->ci.buffered_data; + } + + + if(err != ZIP_OK) + break; + + if ((zi->ci.method == Z_DEFLATED) && (!zi->ci.raw)) + { + uLong uTotalOutBefore = zi->ci.stream.total_out; + err=deflate(&zi->ci.stream, Z_NO_FLUSH); + if(uTotalOutBefore > zi->ci.stream.total_out) + { + int bBreak = 0; + bBreak++; + } + + zi->ci.pos_in_buffered_data += (uInt)(zi->ci.stream.total_out - uTotalOutBefore) ; + } + else + { + uInt copy_this,i; + if (zi->ci.stream.avail_in < zi->ci.stream.avail_out) + copy_this = zi->ci.stream.avail_in; + else + copy_this = zi->ci.stream.avail_out; + + for (i = 0; i < copy_this; i++) + *(((char*)zi->ci.stream.next_out)+i) = + *(((const char*)zi->ci.stream.next_in)+i); + { + zi->ci.stream.avail_in -= copy_this; + zi->ci.stream.avail_out-= copy_this; + zi->ci.stream.next_in+= copy_this; + zi->ci.stream.next_out+= copy_this; + zi->ci.stream.total_in+= copy_this; + zi->ci.stream.total_out+= copy_this; + zi->ci.pos_in_buffered_data += copy_this; + } + } + }// while(...) + } + + return err; +} + +extern int ZEXPORT zipCloseFileInZipRaw (zipFile file, uLong uncompressed_size, uLong crc32) +{ + return zipCloseFileInZipRaw64 (file, uncompressed_size, crc32); +} + +extern int ZEXPORT zipCloseFileInZipRaw64 (zipFile file, ZPOS64_T uncompressed_size, uLong crc32) +{ + zip64_internal* zi; + ZPOS64_T compressed_size; + uLong invalidValue = 0xffffffff; + short datasize = 0; + int err=ZIP_OK; + + if (file == NULL) + return ZIP_PARAMERROR; + zi = (zip64_internal*)file; + + if (zi->in_opened_file_inzip == 0) + return ZIP_PARAMERROR; + zi->ci.stream.avail_in = 0; + + if ((zi->ci.method == Z_DEFLATED) && (!zi->ci.raw)) + { + while (err==ZIP_OK) + { + uLong uTotalOutBefore; + if (zi->ci.stream.avail_out == 0) + { + if (zip64FlushWriteBuffer(zi) == ZIP_ERRNO) + err = ZIP_ERRNO; + zi->ci.stream.avail_out = (uInt)Z_BUFSIZE; + zi->ci.stream.next_out = zi->ci.buffered_data; + } + uTotalOutBefore = zi->ci.stream.total_out; + err=deflate(&zi->ci.stream, Z_FINISH); + zi->ci.pos_in_buffered_data += (uInt)(zi->ci.stream.total_out - uTotalOutBefore) ; + } + } + else if ((zi->ci.method == Z_BZIP2ED) && (!zi->ci.raw)) + { +#ifdef HAVE_BZIP2 + err = BZ_FINISH_OK; + while (err==BZ_FINISH_OK) + { + uLong uTotalOutBefore; + if (zi->ci.bstream.avail_out == 0) + { + if (zip64FlushWriteBuffer(zi) == ZIP_ERRNO) + err = ZIP_ERRNO; + zi->ci.bstream.avail_out = (uInt)Z_BUFSIZE; + zi->ci.bstream.next_out = (char*)zi->ci.buffered_data; + } + uTotalOutBefore = zi->ci.bstream.total_out_lo32; + err=BZ2_bzCompress(&zi->ci.bstream, BZ_FINISH); + if(err == BZ_STREAM_END) + err = Z_STREAM_END; + + zi->ci.pos_in_buffered_data += (uInt)(zi->ci.bstream.total_out_lo32 - uTotalOutBefore); + } + + if(err == BZ_FINISH_OK) + err = ZIP_OK; +#endif + } + + if (err==Z_STREAM_END) + err=ZIP_OK; /* this is normal */ + + if ((zi->ci.pos_in_buffered_data>0) && (err==ZIP_OK)) + { + if (zip64FlushWriteBuffer(zi)==ZIP_ERRNO) + err = ZIP_ERRNO; + } + + if ((zi->ci.method == Z_DEFLATED) && (!zi->ci.raw)) + { + int tmp_err = deflateEnd(&zi->ci.stream); + if (err == ZIP_OK) + err = tmp_err; + zi->ci.stream_initialised = 0; + } +#ifdef HAVE_BZIP2 + else if((zi->ci.method == Z_BZIP2ED) && (!zi->ci.raw)) + { + int tmperr = BZ2_bzCompressEnd(&zi->ci.bstream); + if (err==ZIP_OK) + err = tmperr; + zi->ci.stream_initialised = 0; + } +#endif + + if (!zi->ci.raw) + { + crc32 = (uLong)zi->ci.crc32; + uncompressed_size = zi->ci.totalUncompressedData; + } + compressed_size = zi->ci.totalCompressedData; + +# ifndef NOCRYPT + compressed_size += zi->ci.crypt_header_size; +# endif + + // update Current Item crc and sizes, + if(compressed_size >= 0xffffffff || uncompressed_size >= 0xffffffff || zi->ci.pos_local_header >= 0xffffffff) + { + /*version Made by*/ + zip64local_putValue_inmemory(zi->ci.central_header+4,(uLong)45,2); + /*version needed*/ + zip64local_putValue_inmemory(zi->ci.central_header+6,(uLong)45,2); + + } + + zip64local_putValue_inmemory(zi->ci.central_header+16,crc32,4); /*crc*/ + + + if(compressed_size >= 0xffffffff) + zip64local_putValue_inmemory(zi->ci.central_header+20, invalidValue,4); /*compr size*/ + else + zip64local_putValue_inmemory(zi->ci.central_header+20, compressed_size,4); /*compr size*/ + + /// set internal file attributes field + if (zi->ci.stream.data_type == Z_ASCII) + zip64local_putValue_inmemory(zi->ci.central_header+36,(uLong)Z_ASCII,2); + + if(uncompressed_size >= 0xffffffff) + zip64local_putValue_inmemory(zi->ci.central_header+24, invalidValue,4); /*uncompr size*/ + else + zip64local_putValue_inmemory(zi->ci.central_header+24, uncompressed_size,4); /*uncompr size*/ + + // Add ZIP64 extra info field for uncompressed size + if(uncompressed_size >= 0xffffffff) + datasize += 8; + + // Add ZIP64 extra info field for compressed size + if(compressed_size >= 0xffffffff) + datasize += 8; + + // Add ZIP64 extra info field for relative offset to local file header of current file + if(zi->ci.pos_local_header >= 0xffffffff) + datasize += 8; + + if(datasize > 0) + { + char* p = NULL; + + if((uLong)(datasize + 4) > zi->ci.size_centralExtraFree) + { + // we can not write more data to the buffer that we have room for. + return ZIP_BADZIPFILE; + } + + p = zi->ci.central_header + zi->ci.size_centralheader; + + // Add Extra Information Header for 'ZIP64 information' + zip64local_putValue_inmemory(p, 0x0001, 2); // HeaderID + p += 2; + zip64local_putValue_inmemory(p, datasize, 2); // DataSize + p += 2; + + if(uncompressed_size >= 0xffffffff) + { + zip64local_putValue_inmemory(p, uncompressed_size, 8); + p += 8; + } + + if(compressed_size >= 0xffffffff) + { + zip64local_putValue_inmemory(p, compressed_size, 8); + p += 8; + } + + if(zi->ci.pos_local_header >= 0xffffffff) + { + zip64local_putValue_inmemory(p, zi->ci.pos_local_header, 8); + p += 8; + } + + // Update how much extra free space we got in the memory buffer + // and increase the centralheader size so the new ZIP64 fields are included + // ( 4 below is the size of HeaderID and DataSize field ) + zi->ci.size_centralExtraFree -= datasize + 4; + zi->ci.size_centralheader += datasize + 4; + + // Update the extra info size field + zi->ci.size_centralExtra += datasize + 4; + zip64local_putValue_inmemory(zi->ci.central_header+30,(uLong)zi->ci.size_centralExtra,2); + } + + if (err==ZIP_OK) + err = add_data_in_datablock(&zi->central_dir, zi->ci.central_header, (uLong)zi->ci.size_centralheader); + + free(zi->ci.central_header); + + if (err==ZIP_OK) + { + // Update the LocalFileHeader with the new values. + + ZPOS64_T cur_pos_inzip = ZTELL64(zi->z_filefunc,zi->filestream); + + if (ZSEEK64(zi->z_filefunc,zi->filestream, zi->ci.pos_local_header + 14,ZLIB_FILEFUNC_SEEK_SET)!=0) + err = ZIP_ERRNO; + + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,crc32,4); /* crc 32, unknown */ + + if(uncompressed_size >= 0xffffffff || compressed_size >= 0xffffffff ) + { + if(zi->ci.pos_zip64extrainfo > 0) + { + // Update the size in the ZIP64 extended field. + if (ZSEEK64(zi->z_filefunc,zi->filestream, zi->ci.pos_zip64extrainfo + 4,ZLIB_FILEFUNC_SEEK_SET)!=0) + err = ZIP_ERRNO; + + if (err==ZIP_OK) /* compressed size, unknown */ + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, uncompressed_size, 8); + + if (err==ZIP_OK) /* uncompressed size, unknown */ + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, compressed_size, 8); + } + else + err = ZIP_BADZIPFILE; // Caller passed zip64 = 0, so no room for zip64 info -> fatal + } + else + { + if (err==ZIP_OK) /* compressed size, unknown */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,compressed_size,4); + + if (err==ZIP_OK) /* uncompressed size, unknown */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,uncompressed_size,4); + } + + if (ZSEEK64(zi->z_filefunc,zi->filestream, cur_pos_inzip,ZLIB_FILEFUNC_SEEK_SET)!=0) + err = ZIP_ERRNO; + } + + zi->number_entry ++; + zi->in_opened_file_inzip = 0; + + return err; +} + +extern int ZEXPORT zipCloseFileInZip (zipFile file) +{ + return zipCloseFileInZipRaw (file,0,0); +} + +int Write_Zip64EndOfCentralDirectoryLocator(zip64_internal* zi, ZPOS64_T zip64eocd_pos_inzip) +{ + int err = ZIP_OK; + ZPOS64_T pos = zip64eocd_pos_inzip - zi->add_position_when_writting_offset; + + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)ZIP64ENDLOCHEADERMAGIC,4); + + /*num disks*/ + if (err==ZIP_OK) /* number of the disk with the start of the central directory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,4); + + /*relative offset*/ + if (err==ZIP_OK) /* Relative offset to the Zip64EndOfCentralDirectory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream, pos,8); + + /*total disks*/ /* Do not support spawning of disk so always say 1 here*/ + if (err==ZIP_OK) /* number of the disk with the start of the central directory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)1,4); + + return err; +} + +int Write_Zip64EndOfCentralDirectoryRecord(zip64_internal* zi, uLong size_centraldir, ZPOS64_T centraldir_pos_inzip) +{ + int err = ZIP_OK; + + uLong Zip64DataSize = 44; + + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)ZIP64ENDHEADERMAGIC,4); + + if (err==ZIP_OK) /* size of this 'zip64 end of central directory' */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(ZPOS64_T)Zip64DataSize,8); // why ZPOS64_T of this ? + + if (err==ZIP_OK) /* version made by */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)45,2); + + if (err==ZIP_OK) /* version needed */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)45,2); + + if (err==ZIP_OK) /* number of this disk */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,4); + + if (err==ZIP_OK) /* number of the disk with the start of the central directory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,4); + + if (err==ZIP_OK) /* total number of entries in the central dir on this disk */ + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, zi->number_entry, 8); + + if (err==ZIP_OK) /* total number of entries in the central dir */ + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, zi->number_entry, 8); + + if (err==ZIP_OK) /* size of the central directory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(ZPOS64_T)size_centraldir,8); + + if (err==ZIP_OK) /* offset of start of central directory with respect to the starting disk number */ + { + ZPOS64_T pos = centraldir_pos_inzip - zi->add_position_when_writting_offset; + err = zip64local_putValue(&zi->z_filefunc,zi->filestream, (ZPOS64_T)pos,8); + } + return err; +} +int Write_EndOfCentralDirectoryRecord(zip64_internal* zi, uLong size_centraldir, ZPOS64_T centraldir_pos_inzip) +{ + int err = ZIP_OK; + + /*signature*/ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)ENDHEADERMAGIC,4); + + if (err==ZIP_OK) /* number of this disk */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,2); + + if (err==ZIP_OK) /* number of the disk with the start of the central directory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,2); + + if (err==ZIP_OK) /* total number of entries in the central dir on this disk */ + { + { + if(zi->number_entry >= 0xFFFF) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0xffff,2); // use value in ZIP64 record + else + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)zi->number_entry,2); + } + } + + if (err==ZIP_OK) /* total number of entries in the central dir */ + { + if(zi->number_entry >= 0xFFFF) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0xffff,2); // use value in ZIP64 record + else + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)zi->number_entry,2); + } + + if (err==ZIP_OK) /* size of the central directory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)size_centraldir,4); + + if (err==ZIP_OK) /* offset of start of central directory with respect to the starting disk number */ + { + ZPOS64_T pos = centraldir_pos_inzip - zi->add_position_when_writting_offset; + if(pos >= 0xffffffff) + { + err = zip64local_putValue(&zi->z_filefunc,zi->filestream, (uLong)0xffffffff,4); + } + else + err = zip64local_putValue(&zi->z_filefunc,zi->filestream, (uLong)(centraldir_pos_inzip - zi->add_position_when_writting_offset),4); + } + + return err; +} + +int Write_GlobalComment(zip64_internal* zi, const char* global_comment) +{ + int err = ZIP_OK; + uInt size_global_comment = 0; + + if(global_comment != NULL) + size_global_comment = (uInt)strlen(global_comment); + + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)size_global_comment,2); + + if (err == ZIP_OK && size_global_comment > 0) + { + if (ZWRITE64(zi->z_filefunc,zi->filestream, global_comment, size_global_comment) != size_global_comment) + err = ZIP_ERRNO; + } + return err; +} + +extern int ZEXPORT zipClose (zipFile file, const char* global_comment) +{ + zip64_internal* zi; + int err = 0; + uLong size_centraldir = 0; + ZPOS64_T centraldir_pos_inzip; + ZPOS64_T pos; + + if (file == NULL) + return ZIP_PARAMERROR; + + zi = (zip64_internal*)file; + + if (zi->in_opened_file_inzip == 1) + { + err = zipCloseFileInZip (file); + } + +#ifndef NO_ADDFILEINEXISTINGZIP + if (global_comment==NULL) + global_comment = zi->globalcomment; +#endif + + centraldir_pos_inzip = ZTELL64(zi->z_filefunc,zi->filestream); + + if (err==ZIP_OK) + { + linkedlist_datablock_internal* ldi = zi->central_dir.first_block; + while (ldi!=NULL) + { + if ((err==ZIP_OK) && (ldi->filled_in_this_block>0)) + { + if (ZWRITE64(zi->z_filefunc,zi->filestream, ldi->data, ldi->filled_in_this_block) != ldi->filled_in_this_block) + err = ZIP_ERRNO; + } + + size_centraldir += ldi->filled_in_this_block; + ldi = ldi->next_datablock; + } + } + free_linkedlist(&(zi->central_dir)); + + pos = centraldir_pos_inzip - zi->add_position_when_writting_offset; + if(pos >= 0xffffffff || zi->number_entry > 0xFFFF) + { + ZPOS64_T Zip64EOCDpos = ZTELL64(zi->z_filefunc,zi->filestream); + Write_Zip64EndOfCentralDirectoryRecord(zi, size_centraldir, centraldir_pos_inzip); + + Write_Zip64EndOfCentralDirectoryLocator(zi, Zip64EOCDpos); + } + + if (err==ZIP_OK) + err = Write_EndOfCentralDirectoryRecord(zi, size_centraldir, centraldir_pos_inzip); + + if(err == ZIP_OK) + err = Write_GlobalComment(zi, global_comment); + + if (ZCLOSE64(zi->z_filefunc,zi->filestream) != 0) + if (err == ZIP_OK) + err = ZIP_ERRNO; + +#ifndef NO_ADDFILEINEXISTINGZIP + TRYFREE(zi->globalcomment); +#endif + TRYFREE(zi); + + return err; +} + +extern int ZEXPORT zipRemoveExtraInfoBlock (char* pData, int* dataLen, short sHeader) +{ + char* p = pData; + int size = 0; + char* pNewHeader; + char* pTmp; + short header; + short dataSize; + + int retVal = ZIP_OK; + + if(pData == NULL || *dataLen < 4) + return ZIP_PARAMERROR; + + pNewHeader = (char*)ALLOC(*dataLen); + pTmp = pNewHeader; + + while(p < (pData + *dataLen)) + { + header = *(short*)p; + dataSize = *(((short*)p)+1); + + if( header == sHeader ) // Header found. + { + p += dataSize + 4; // skip it. do not copy to temp buffer + } + else + { + // Extra Info block should not be removed, So copy it to the temp buffer. + memcpy(pTmp, p, dataSize + 4); + p += dataSize + 4; + size += dataSize + 4; + } + + } + + if(size < *dataLen) + { + // clean old extra info block. + memset(pData,0, *dataLen); + + // copy the new extra info block over the old + if(size > 0) + memcpy(pData, pNewHeader, size); + + // set the new extra info size + *dataLen = size; + + retVal = ZIP_OK; + } + else + retVal = ZIP_ERRNO; + + TRYFREE(pNewHeader); + + return retVal; +} diff --git a/storage/connect/zip.h b/storage/connect/zip.h new file mode 100644 index 00000000000..8aaebb62343 --- /dev/null +++ b/storage/connect/zip.h @@ -0,0 +1,362 @@ +/* zip.h -- IO on .zip files using zlib + Version 1.1, February 14h, 2010 + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications for Zip64 support + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + + --------------------------------------------------------------------------- + + Condition of use and distribution are the same than zlib : + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + --------------------------------------------------------------------------- + + Changes + + See header of zip.h + +*/ + +#ifndef _zip12_H +#define _zip12_H + +#ifdef __cplusplus +extern "C" { +#endif + +//#define HAVE_BZIP2 + +#ifndef _ZLIB_H +#include "zlib.h" +#endif + +#ifndef _ZLIBIOAPI_H +#include "ioapi.h" +#endif + +#ifdef HAVE_BZIP2 +#include "bzlib.h" +#endif + +#define Z_BZIP2ED 12 + +#if defined(STRICTZIP) || defined(STRICTZIPUNZIP) +/* like the STRICT of WIN32, we define a pointer that cannot be converted + from (void*) without cast */ +typedef struct TagzipFile__ { int unused; } zipFile__; +typedef zipFile__ *zipFile; +#else +typedef voidp zipFile; +#endif + +#define ZIP_OK (0) +#define ZIP_EOF (0) +#define ZIP_ERRNO (Z_ERRNO) +#define ZIP_PARAMERROR (-102) +#define ZIP_BADZIPFILE (-103) +#define ZIP_INTERNALERROR (-104) + +#ifndef DEF_MEM_LEVEL +# if MAX_MEM_LEVEL >= 8 +# define DEF_MEM_LEVEL 8 +# else +# define DEF_MEM_LEVEL MAX_MEM_LEVEL +# endif +#endif +/* default memLevel */ + +/* tm_zip contain date/time info */ +typedef struct tm_zip_s +{ + uInt tm_sec; /* seconds after the minute - [0,59] */ + uInt tm_min; /* minutes after the hour - [0,59] */ + uInt tm_hour; /* hours since midnight - [0,23] */ + uInt tm_mday; /* day of the month - [1,31] */ + uInt tm_mon; /* months since January - [0,11] */ + uInt tm_year; /* years - [1980..2044] */ +} tm_zip; + +typedef struct +{ + tm_zip tmz_date; /* date in understandable format */ + uLong dosDate; /* if dos_date == 0, tmu_date is used */ +/* uLong flag; */ /* general purpose bit flag 2 bytes */ + + uLong internal_fa; /* internal file attributes 2 bytes */ + uLong external_fa; /* external file attributes 4 bytes */ +} zip_fileinfo; + +typedef const char* zipcharpc; + + +#define APPEND_STATUS_CREATE (0) +#define APPEND_STATUS_CREATEAFTER (1) +#define APPEND_STATUS_ADDINZIP (2) + +extern zipFile ZEXPORT zipOpen OF((const char *pathname, int append)); +extern zipFile ZEXPORT zipOpen64 OF((const void *pathname, int append)); +/* + Create a zipfile. + pathname contain on Windows XP a filename like "c:\\zlib\\zlib113.zip" or on + an Unix computer "zlib/zlib113.zip". + if the file pathname exist and append==APPEND_STATUS_CREATEAFTER, the zip + will be created at the end of the file. + (useful if the file contain a self extractor code) + if the file pathname exist and append==APPEND_STATUS_ADDINZIP, we will + add files in existing zip (be sure you don't add file that doesn't exist) + If the zipfile cannot be opened, the return value is NULL. + Else, the return value is a zipFile Handle, usable with other function + of this zip package. +*/ + +/* Note : there is no delete function into a zipfile. + If you want delete file into a zipfile, you must open a zipfile, and create another + Of couse, you can use RAW reading and writing to copy the file you did not want delte +*/ + +extern zipFile ZEXPORT zipOpen2 OF((const char *pathname, + int append, + zipcharpc* globalcomment, + zlib_filefunc_def* pzlib_filefunc_def)); + +extern zipFile ZEXPORT zipOpen2_64 OF((const void *pathname, + int append, + zipcharpc* globalcomment, + zlib_filefunc64_def* pzlib_filefunc_def)); + +extern int ZEXPORT zipOpenNewFileInZip OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level)); + +extern int ZEXPORT zipOpenNewFileInZip64 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int zip64)); + +/* + Open a file in the ZIP for writing. + filename : the filename in zip (if NULL, '-' without quote will be used + *zipfi contain supplemental information + if extrafield_local!=NULL and size_extrafield_local>0, extrafield_local + contains the extrafield data the the local header + if extrafield_global!=NULL and size_extrafield_global>0, extrafield_global + contains the extrafield data the the local header + if comment != NULL, comment contain the comment string + method contain the compression method (0 for store, Z_DEFLATED for deflate) + level contain the level of compression (can be Z_DEFAULT_COMPRESSION) + zip64 is set to 1 if a zip64 extended information block should be added to the local file header. + this MUST be '1' if the uncompressed size is >= 0xffffffff. + +*/ + + +extern int ZEXPORT zipOpenNewFileInZip2 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw)); + + +extern int ZEXPORT zipOpenNewFileInZip2_64 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw, + int zip64)); +/* + Same than zipOpenNewFileInZip, except if raw=1, we write raw file + */ + +extern int ZEXPORT zipOpenNewFileInZip3 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw, + int windowBits, + int memLevel, + int strategy, + const char* password, + uLong crcForCrypting)); + +extern int ZEXPORT zipOpenNewFileInZip3_64 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw, + int windowBits, + int memLevel, + int strategy, + const char* password, + uLong crcForCrypting, + int zip64 + )); + +/* + Same than zipOpenNewFileInZip2, except + windowBits,memLevel,,strategy : see parameter strategy in deflateInit2 + password : crypting password (NULL for no crypting) + crcForCrypting : crc of file to compress (needed for crypting) + */ + +extern int ZEXPORT zipOpenNewFileInZip4 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw, + int windowBits, + int memLevel, + int strategy, + const char* password, + uLong crcForCrypting, + uLong versionMadeBy, + uLong flagBase + )); + + +extern int ZEXPORT zipOpenNewFileInZip4_64 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw, + int windowBits, + int memLevel, + int strategy, + const char* password, + uLong crcForCrypting, + uLong versionMadeBy, + uLong flagBase, + int zip64 + )); +/* + Same than zipOpenNewFileInZip4, except + versionMadeBy : value for Version made by field + flag : value for flag field (compression level info will be added) + */ + + +extern int ZEXPORT zipWriteInFileInZip OF((zipFile file, + const void* buf, + unsigned len)); +/* + Write data in the zipfile +*/ + +extern int ZEXPORT zipCloseFileInZip OF((zipFile file)); +/* + Close the current file in the zipfile +*/ + +extern int ZEXPORT zipCloseFileInZipRaw OF((zipFile file, + uLong uncompressed_size, + uLong crc32)); + +extern int ZEXPORT zipCloseFileInZipRaw64 OF((zipFile file, + ZPOS64_T uncompressed_size, + uLong crc32)); + +/* + Close the current file in the zipfile, for file opened with + parameter raw=1 in zipOpenNewFileInZip2 + uncompressed_size and crc32 are value for the uncompressed size +*/ + +extern int ZEXPORT zipClose OF((zipFile file, + const char* global_comment)); +/* + Close the zipfile +*/ + + +extern int ZEXPORT zipRemoveExtraInfoBlock OF((char* pData, int* dataLen, short sHeader)); +/* + zipRemoveExtraInfoBlock - Added by Mathias Svensson + + Remove extra information block from a extra information data for the local file header or central directory header + + It is needed to remove ZIP64 extra information blocks when before data is written if using RAW mode. + + 0x0001 is the signature header for the ZIP64 extra information blocks + + usage. + Remove ZIP64 Extra information from a central director extra field data + zipRemoveExtraInfoBlock(pCenDirExtraFieldData, &nCenDirExtraFieldDataLen, 0x0001); + + Remove ZIP64 Extra information from a Local File Header extra field data + zipRemoveExtraInfoBlock(pLocalHeaderExtraFieldData, &nLocalHeaderExtraFieldDataLen, 0x0001); +*/ + +#ifdef __cplusplus +} +#endif + +#endif /* _zip64_H */ diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 3b34e750b6c..9373f06e7de 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -1614,6 +1614,7 @@ struct dict_foreign_remove_partial if (table != NULL) { table->referenced_set.erase(foreign); } + dict_foreign_free(foreign); } }; @@ -3557,8 +3558,7 @@ dict_foreign_add_to_cache( } if (for_in_cache) { - /* Free the foreign object */ - mem_heap_free(foreign->heap); + dict_foreign_free(foreign); } else { for_in_cache = foreign; } @@ -3586,7 +3586,7 @@ dict_foreign_add_to_cache( " the ones in table."); if (for_in_cache == foreign) { - mem_heap_free(foreign->heap); + dict_foreign_free(foreign); } return(DB_CANNOT_ADD_CONSTRAINT); @@ -3642,7 +3642,7 @@ dict_foreign_add_to_cache( be one */ } - mem_heap_free(foreign->heap); + dict_foreign_free(foreign); } return(DB_CANNOT_ADD_CONSTRAINT); diff --git a/storage/innobase/dict/dict0load.cc b/storage/innobase/dict/dict0load.cc index e68bbe7d10a..77a31e5de63 100644 --- a/storage/innobase/dict/dict0load.cc +++ b/storage/innobase/dict/dict0load.cc @@ -489,7 +489,7 @@ err_len: } /* This recieves a dict_foreign_t* that points to a stack variable. - So mem_heap_free(foreign->heap) is not used as elsewhere. + So dict_foreign_free(foreign) is not used as elsewhere. Since the heap used here is freed elsewhere, foreign->heap is not assigned. */ foreign->id = mem_heap_strdupl(heap, (const char*) field, len); diff --git a/storage/innobase/dict/dict0stats.cc b/storage/innobase/dict/dict0stats.cc index a4aa43651f8..c0a83c951a5 100644 --- a/storage/innobase/dict/dict0stats.cc +++ b/storage/innobase/dict/dict0stats.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 2009, 2015, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2009, 2016, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1109,7 +1109,8 @@ dict_stats_analyze_index_level( them away) which brings non-determinism. We skip only leaf-level delete marks because delete marks on non-leaf level do not make sense. */ - if (level == 0 && + + if (level == 0 && srv_stats_include_delete_marked? 0: rec_get_deleted_flag( rec, page_is_comp(btr_pcur_get_page(&pcur)))) { @@ -1295,8 +1296,12 @@ enum page_scan_method_t { the given page and count the number of distinct ones, also ignore delete marked records */ - QUIT_ON_FIRST_NON_BORING/* quit when the first record that differs + QUIT_ON_FIRST_NON_BORING,/* quit when the first record that differs from its right neighbor is found */ + COUNT_ALL_NON_BORING_INCLUDE_DEL_MARKED/* scan all records on + the given page and count the number of + distinct ones, include delete marked + records */ }; /* @} */ @@ -1572,6 +1577,8 @@ dict_stats_analyze_index_below_cur( offsets_rec = dict_stats_scan_page( &rec, offsets1, offsets2, index, page, n_prefix, + srv_stats_include_delete_marked ? + COUNT_ALL_NON_BORING_INCLUDE_DEL_MARKED: COUNT_ALL_NON_BORING_AND_SKIP_DEL_MARKED, n_diff, n_external_pages); diff --git a/storage/innobase/fts/fts0opt.cc b/storage/innobase/fts/fts0opt.cc index ea937c20752..455e2669c0d 100644 --- a/storage/innobase/fts/fts0opt.cc +++ b/storage/innobase/fts/fts0opt.cc @@ -578,9 +578,6 @@ fts_zip_read_word( fts_zip_t* zip, /*!< in: Zip state + data */ fts_string_t* word) /*!< out: uncompressed word */ { -#ifdef UNIV_DEBUG - ulint i; -#endif short len = 0; void* null = NULL; byte* ptr = word->f_str; @@ -655,10 +652,9 @@ fts_zip_read_word( } } -#ifdef UNIV_DEBUG /* All blocks must be freed at end of inflate. */ if (zip->status != Z_OK) { - for (i = 0; i < ib_vector_size(zip->blocks); ++i) { + for (ulint i = 0; i < ib_vector_size(zip->blocks); ++i) { if (ib_vector_getp(zip->blocks, i)) { ut_free(ib_vector_getp(zip->blocks, i)); ib_vector_set(zip->blocks, i, &null); @@ -669,7 +665,6 @@ fts_zip_read_word( if (ptr != NULL) { ut_ad(word->f_len == strlen((char*) ptr)); } -#endif /* UNIV_DEBUG */ return(zip->status == Z_OK || zip->status == Z_STREAM_END ? ptr : NULL); } diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 21bb795f5d9..6fe56892ae6 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -2054,11 +2054,16 @@ innobase_get_stmt( THD* thd, /*!< in: MySQL thread handle */ size_t* length) /*!< out: length of the SQL statement */ { - LEX_STRING* stmt; - - stmt = thd_query_string(thd); - *length = stmt->length; - return(stmt->str); + const char* query = NULL; + LEX_STRING *stmt = NULL; + if (thd) { + stmt = thd_query_string(thd); + if (stmt) { + *length = stmt->length; + query = stmt->str; + } + } + return (query); } /**********************************************************************//** @@ -7163,7 +7168,66 @@ build_template_field( UNIV_MEM_INVALID(templ, sizeof *templ); templ->col_no = i; templ->clust_rec_field_no = dict_col_get_clust_pos(col, clust_index); - ut_a(templ->clust_rec_field_no != ULINT_UNDEFINED); + + /* If clustered index record field is not found, lets print out + field names and all the rest to understand why field is not found. */ + if (templ->clust_rec_field_no == ULINT_UNDEFINED) { + const char* tb_col_name = dict_table_get_col_name(clust_index->table, i); + dict_field_t* field=NULL; + size_t size = 0; + + for(ulint j=0; j < clust_index->n_user_defined_cols; j++) { + dict_field_t* ifield = &(clust_index->fields[j]); + if (ifield && !memcmp(tb_col_name, ifield->name, + strlen(tb_col_name))) { + field = ifield; + break; + } + } + + ib_logf(IB_LOG_LEVEL_INFO, + "Looking for field %lu name %s from table %s", + i, + (tb_col_name ? tb_col_name : "NULL"), + clust_index->table->name); + + + for(ulint j=0; j < clust_index->n_user_defined_cols; j++) { + dict_field_t* ifield = &(clust_index->fields[j]); + ib_logf(IB_LOG_LEVEL_INFO, + "InnoDB Table %s field %lu name %s", + clust_index->table->name, + j, + (ifield ? ifield->name : "NULL")); + } + + for(ulint j=0; j < table->s->stored_fields; j++) { + ib_logf(IB_LOG_LEVEL_INFO, + "MySQL table %s field %lu name %s", + table->s->table_name.str, + j, + table->field[j]->field_name); + } + + ib_logf(IB_LOG_LEVEL_ERROR, + "Clustered record field for column %lu" + " not found table n_user_defined %d" + " index n_user_defined %d" + " InnoDB table %s field name %s" + " MySQL table %s field name %s n_fields %d" + " query %s", + i, + clust_index->n_user_defined_cols, + clust_index->table->n_cols - DATA_N_SYS_COLS, + clust_index->table->name, + (field ? field->name : "NULL"), + table->s->table_name.str, + (tb_col_name ? tb_col_name : "NULL"), + table->s->stored_fields, + innobase_get_stmt(current_thd, &size)); + + ut_a(templ->clust_rec_field_no != ULINT_UNDEFINED); + } if (dict_index_is_clust(index)) { templ->rec_field_no = templ->clust_rec_field_no; @@ -15065,6 +15129,37 @@ ha_innobase::get_auto_increment( ulonglong col_max_value = innobase_get_int_col_max_value( table->next_number_field); + /** The following logic is needed to avoid duplicate key error + for autoincrement column. + + (1) InnoDB gives the current autoincrement value with respect + to increment and offset value. + + (2) Basically it does compute_next_insert_id() logic inside InnoDB + to avoid the current auto increment value changed by handler layer. + + (3) It is restricted only for insert operations. */ + + if (increment > 1 && thd_sql_command(user_thd) != SQLCOM_ALTER_TABLE + && autoinc < col_max_value) { + + ulonglong prev_auto_inc = autoinc; + + autoinc = ((autoinc - 1) + increment - offset)/ increment; + + autoinc = autoinc * increment + offset; + + /* If autoinc exceeds the col_max_value then reset + to old autoinc value. Because in case of non-strict + sql mode, boundary value is not considered as error. */ + + if (autoinc >= col_max_value) { + autoinc = prev_auto_inc; + } + + ut_ad(autoinc > 0); + } + /* Called for the first time ? */ if (trx->n_autoinc_rows == 0) { @@ -17924,6 +18019,12 @@ static MYSQL_SYSVAR_BOOL(use_fallocate, innobase_use_fallocate, "Preallocate files fast, using operating system functionality. On POSIX systems, posix_fallocate system call is used.", NULL, NULL, FALSE); +static MYSQL_SYSVAR_BOOL(stats_include_delete_marked, + srv_stats_include_delete_marked, + PLUGIN_VAR_OPCMDARG, + "Scan delete marked records for persistent stat", + NULL, NULL, FALSE); + static MYSQL_SYSVAR_ULONG(io_capacity, srv_io_capacity, PLUGIN_VAR_RQCMDARG, "Number of IOPs the server can do. Tunes the background IO rate", @@ -18789,6 +18890,7 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(doublewrite), MYSQL_SYSVAR(use_atomic_writes), MYSQL_SYSVAR(use_fallocate), + MYSQL_SYSVAR(stats_include_delete_marked), MYSQL_SYSVAR(api_enable_binlog), MYSQL_SYSVAR(api_enable_mdl), MYSQL_SYSVAR(api_disable_rowlock), diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index e164d7e279c..221e9324f3f 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -1842,6 +1842,7 @@ innobase_fts_check_doc_id_index_in_def( return(FTS_NOT_EXIST_DOC_ID_INDEX); } + /*******************************************************************//** Create an index table where indexes are ordered as follows: @@ -1910,26 +1911,11 @@ innobase_create_key_defs( (only prefix/part of the column is indexed), MySQL will treat the index as a PRIMARY KEY unless the table already has one. */ - if (n_add > 0 && !new_primary && got_default_clust - && (key_info[*add].flags & HA_NOSAME) - && !(key_info[*add].flags & HA_KEY_HAS_PART_KEY_SEG)) { - uint key_part = key_info[*add].user_defined_key_parts; - - new_primary = true; - - while (key_part--) { - const uint maybe_null - = key_info[*add].key_part[key_part].key_type - & FIELDFLAG_MAYBE_NULL; - DBUG_ASSERT(!maybe_null - == !key_info[*add].key_part[key_part]. - field->real_maybe_null()); + ut_ad(altered_table->s->primary_key == 0 + || altered_table->s->primary_key == MAX_KEY); - if (maybe_null) { - new_primary = false; - break; - } - } + if (got_default_clust && !new_primary) { + new_primary = (altered_table->s->primary_key != MAX_KEY); } const bool rebuild = new_primary || add_fts_doc_id @@ -1948,8 +1934,14 @@ innobase_create_key_defs( ulint primary_key_number; if (new_primary) { - DBUG_ASSERT(n_add > 0); - primary_key_number = *add; + if (n_add == 0) { + DBUG_ASSERT(got_default_clust); + DBUG_ASSERT(altered_table->s->primary_key + == 0); + primary_key_number = 0; + } else { + primary_key_number = *add; + } } else if (got_default_clust) { /* Create the GEN_CLUST_INDEX */ index_def_t* index = indexdef++; @@ -3057,6 +3049,8 @@ prepare_inplace_alter_table_dict( ctx->add_cols = add_cols; } else { DBUG_ASSERT(!innobase_need_rebuild(ha_alter_info, old_table)); + DBUG_ASSERT(old_table->s->primary_key + == altered_table->s->primary_key); if (!ctx->new_table->fts && innobase_fulltext_exist(altered_table)) { @@ -3894,7 +3888,7 @@ check_if_can_drop_indexes: index->name, TRUE); my_error(ER_INDEX_CORRUPT, MYF(0), index_name); - DBUG_RETURN(true); + goto err_exit; } } } @@ -4077,6 +4071,27 @@ found_col: add_fts_doc_id_idx)); } +/** Get the name of an erroneous key. +@param[in] error_key_num InnoDB number of the erroneus key +@param[in] ha_alter_info changes that were being performed +@param[in] table InnoDB table +@return the name of the erroneous key */ +static +const char* +get_error_key_name( + ulint error_key_num, + const Alter_inplace_info* ha_alter_info, + const dict_table_t* table) +{ + if (error_key_num == ULINT_UNDEFINED) { + return(FTS_DOC_ID_INDEX_NAME); + } else if (ha_alter_info->key_count == 0) { + return(dict_table_get_first_index(table)->name); + } else { + return(ha_alter_info->key_info_buffer[error_key_num].name); + } +} + /** Alter the table structure in-place with operations specified using Alter_inplace_info. The level of concurrency allowed during this operation depends @@ -4194,17 +4209,13 @@ oom: case DB_ONLINE_LOG_TOO_BIG: DBUG_ASSERT(ctx->online); my_error(ER_INNODB_ONLINE_LOG_TOO_BIG, MYF(0), - (prebuilt->trx->error_key_num == ULINT_UNDEFINED) - ? FTS_DOC_ID_INDEX_NAME - : ha_alter_info->key_info_buffer[ - prebuilt->trx->error_key_num].name); + get_error_key_name(prebuilt->trx->error_key_num, + ha_alter_info, prebuilt->table)); break; case DB_INDEX_CORRUPT: my_error(ER_INDEX_CORRUPT, MYF(0), - (prebuilt->trx->error_key_num == ULINT_UNDEFINED) - ? FTS_DOC_ID_INDEX_NAME - : ha_alter_info->key_info_buffer[ - prebuilt->trx->error_key_num].name); + get_error_key_name(prebuilt->trx->error_key_num, + ha_alter_info, prebuilt->table)); break; default: my_error_innodb(error, @@ -5017,7 +5028,6 @@ innobase_update_foreign_cache( "Foreign key constraints for table '%s'" " are loaded with charset check off", user_table->name); - } } @@ -5117,14 +5127,13 @@ commit_try_rebuild( DBUG_RETURN(true); case DB_ONLINE_LOG_TOO_BIG: my_error(ER_INNODB_ONLINE_LOG_TOO_BIG, MYF(0), - ha_alter_info->key_info_buffer[0].name); + get_error_key_name(err_key, ha_alter_info, + rebuilt_table)); DBUG_RETURN(true); case DB_INDEX_CORRUPT: my_error(ER_INDEX_CORRUPT, MYF(0), - (err_key == ULINT_UNDEFINED) - ? FTS_DOC_ID_INDEX_NAME - : ha_alter_info->key_info_buffer[err_key] - .name); + get_error_key_name(err_key, ha_alter_info, + rebuilt_table)); DBUG_RETURN(true); default: my_error_innodb(error, table_name, user_table->flags); diff --git a/storage/innobase/include/os0thread.h b/storage/innobase/include/os0thread.h index 9a1ada8fa0d..54f3d7554bf 100644 --- a/storage/innobase/include/os0thread.h +++ b/storage/innobase/include/os0thread.h @@ -117,14 +117,25 @@ os_thread_create_func( os_thread_id_t* thread_id); /*!< out: id of the created thread, or NULL */ +/** Waits until the specified thread completes and joins it. +Its return value is ignored. +@param[in,out] thread thread to join */ +UNIV_INTERN +void +os_thread_join( + os_thread_t thread); + /*****************************************************************//** Exits the current thread. */ UNIV_INTERN void os_thread_exit( /*===========*/ - void* exit_value) /*!< in: exit value; in Windows this void* + void* exit_value, /*!< in: exit value; in Windows this void* is cast as a DWORD */ + bool detach = true) /*!< in: if true, the thread will be detached + right before exiting. If false, another thread + is responsible for joining this thread. */ UNIV_COLD MY_ATTRIBUTE((noreturn)); /*****************************************************************//** Returns the thread identifier of current thread. diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index 045051adf18..4b9b8281d04 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -368,6 +368,7 @@ extern unsigned long long srv_stats_persistent_sample_pages; extern my_bool srv_stats_auto_recalc; extern unsigned long long srv_stats_modified_counter; extern my_bool srv_stats_sample_traditional; +extern my_bool srv_stats_include_delete_marked; extern ibool srv_use_doublewrite_buf; extern ulong srv_doublewrite_batch_size; diff --git a/storage/innobase/include/univ.i b/storage/innobase/include/univ.i index e6ddf45faba..824a06829f9 100644 --- a/storage/innobase/include/univ.i +++ b/storage/innobase/include/univ.i @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. Copyright (c) 2008, Google Inc. Portions of this file contain modifications contributed and copyrighted by @@ -44,7 +45,7 @@ Created 1/20/1994 Heikki Tuuri #define INNODB_VERSION_MAJOR 5 #define INNODB_VERSION_MINOR 6 -#define INNODB_VERSION_BUGFIX 33 +#define INNODB_VERSION_BUGFIX 35 /* The following is the InnoDB version as shown in SELECT plugin_version FROM information_schema.plugins; @@ -171,7 +172,7 @@ command. Not tested on Windows. */ #define UNIV_COMPILE_TEST_FUNCS */ -#if defined(HAVE_valgrind)&& defined(HAVE_VALGRIND_MEMCHECK_H) +#if defined HAVE_valgrind && defined HAVE_VALGRIND # define UNIV_DEBUG_VALGRIND #endif /* HAVE_VALGRIND */ #if 0 diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 85f4f6ea671..aed94d00834 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -2,6 +2,7 @@ Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -387,12 +388,6 @@ recv_sys_init( } #ifndef UNIV_HOTBACKUP - /* Initialize red-black tree for fast insertions into the - flush_list during recovery process. - As this initialization is done while holding the buffer pool - mutex we perform it before acquiring recv_sys->mutex. */ - buf_flush_init_flush_rbt(); - mutex_enter(&(recv_sys->mutex)); recv_sys->heap = mem_heap_create_typed(256, @@ -3030,6 +3025,11 @@ recv_recovery_from_checkpoint_start_func( byte* buf; byte log_hdr_buf[LOG_FILE_HDR_SIZE]; dberr_t err; + + /* Initialize red-black tree for fast insertions into the + flush_list during recovery process. */ + buf_flush_init_flush_rbt(); + ut_when_dtor<recv_dblwr_t> tmp(recv_sys->dblwr); #ifdef UNIV_LOG_ARCHIVE diff --git a/storage/innobase/mach/mach0data.cc b/storage/innobase/mach/mach0data.cc index df68aab8a18..feeedb01609 100644 --- a/storage/innobase/mach/mach0data.cc +++ b/storage/innobase/mach/mach0data.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1995, 2009, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -55,8 +55,22 @@ mach_parse_compressed( if (flag < 0x80UL) { *val = flag; return(ptr + 1); + } + + /* Workaround GCC bug + https://gcc.gnu.org/bugzilla/show_bug.cgi?id=77673: + the compiler moves mach_read_from_4 right to the beginning of the + function, causing and out-of-bounds read if we are reading a short + integer close to the end of buffer. */ +#if defined(__GNUC__) && (__GNUC__ >= 5) && !defined(__clang__) +#define DEPLOY_FENCE +#endif + +#ifdef DEPLOY_FENCE + __atomic_thread_fence(__ATOMIC_ACQUIRE); +#endif - } else if (flag < 0xC0UL) { + if (flag < 0xC0UL) { if (end_ptr < ptr + 2) { return(NULL); } @@ -64,8 +78,13 @@ mach_parse_compressed( *val = mach_read_from_2(ptr) & 0x7FFFUL; return(ptr + 2); + } + +#ifdef DEPLOY_FENCE + __atomic_thread_fence(__ATOMIC_ACQUIRE); +#endif - } else if (flag < 0xE0UL) { + if (flag < 0xE0UL) { if (end_ptr < ptr + 3) { return(NULL); } @@ -73,7 +92,13 @@ mach_parse_compressed( *val = mach_read_from_3(ptr) & 0x3FFFFFUL; return(ptr + 3); - } else if (flag < 0xF0UL) { + } + +#ifdef DEPLOY_FENCE + __atomic_thread_fence(__ATOMIC_ACQUIRE); +#endif + + if (flag < 0xF0UL) { if (end_ptr < ptr + 4) { return(NULL); } @@ -81,14 +106,20 @@ mach_parse_compressed( *val = mach_read_from_4(ptr) & 0x1FFFFFFFUL; return(ptr + 4); - } else { - ut_ad(flag == 0xF0UL); + } - if (end_ptr < ptr + 5) { - return(NULL); - } +#ifdef DEPLOY_FENCE + __atomic_thread_fence(__ATOMIC_ACQUIRE); +#endif + +#undef DEPLOY_FENCE + + ut_ad(flag == 0xF0UL); - *val = mach_read_from_4(ptr + 1); - return(ptr + 5); + if (end_ptr < ptr + 5) { + return(NULL); } + + *val = mach_read_from_4(ptr + 1); + return(ptr + 5); } diff --git a/storage/innobase/os/os0thread.cc b/storage/innobase/os/os0thread.cc index a5b0f7de6ae..a6bef176ba4 100644 --- a/storage/innobase/os/os0thread.cc +++ b/storage/innobase/os/os0thread.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1995, 2013, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -191,14 +191,38 @@ os_thread_create_func( #endif } +/** Waits until the specified thread completes and joins it. +Its return value is ignored. +@param[in,out] thread thread to join */ +UNIV_INTERN +void +os_thread_join( + os_thread_t thread) +{ +#ifdef __WIN__ + /* Do nothing. */ +#else +#ifdef UNIV_DEBUG + const int ret = +#endif /* UNIV_DEBUG */ + pthread_join(thread, NULL); + + /* Waiting on already-quit threads is allowed. */ + ut_ad(ret == 0 || ret == ESRCH); +#endif /* __WIN__ */ +} + /*****************************************************************//** Exits the current thread. */ UNIV_INTERN void os_thread_exit( /*===========*/ - void* exit_value) /*!< in: exit value; in Windows this void* + void* exit_value, /*!< in: exit value; in Windows this void* is cast as a DWORD */ + bool detach) /*!< in: if true, the thread will be detached + right before exiting. If false, another thread + is responsible for joining this thread. */ { #ifdef UNIV_DEBUG_THREAD_CREATION fprintf(stderr, "Thread exits, id %lu\n", @@ -216,7 +240,9 @@ os_thread_exit( #ifdef __WIN__ ExitThread((DWORD) exit_value); #else - pthread_detach(pthread_self()); + if (detach) { + pthread_detach(pthread_self()); + } pthread_exit(exit_value); #endif } diff --git a/storage/innobase/row/row0ftsort.cc b/storage/innobase/row/row0ftsort.cc index 1ce99acaef8..e4a88047955 100644 --- a/storage/innobase/row/row0ftsort.cc +++ b/storage/innobase/row/row0ftsort.cc @@ -958,7 +958,7 @@ fts_parallel_merge( CloseHandle(psort_info->thread_hdl); #endif /*__WIN__ */ - os_thread_exit(NULL); + os_thread_exit(NULL, false); OS_THREAD_DUMMY_RETURN; } diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index 9cd6e63e031..9518712e160 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -3824,6 +3824,13 @@ wait_again: " exited when creating FTS" " index '%s'", indexes[i]->name); + } else { + for (j = 0; j < FTS_NUM_AUX_INDEX; + j++) { + + os_thread_join(merge_info[j] + .thread_hdl); + } } } else { /* This cannot report duplicates; an diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 07b841de300..6eb93f9e22d 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -1365,6 +1365,8 @@ run_again: row_ins_step(thr); + DEBUG_SYNC_C("ib_after_row_insert_step"); + err = trx->error_state; if (err != DB_SUCCESS) { diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index 7b0b101e96f..ae73fe7df96 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -351,6 +351,7 @@ this many index pages, there are 2 ways to calculate statistics: table/index are not found in the innodb database */ UNIV_INTERN unsigned long long srv_stats_transient_sample_pages = 8; UNIV_INTERN my_bool srv_stats_persistent = TRUE; +UNIV_INTERN my_bool srv_stats_include_delete_marked = FALSE; UNIV_INTERN unsigned long long srv_stats_persistent_sample_pages = 20; UNIV_INTERN my_bool srv_stats_auto_recalc = TRUE; diff --git a/storage/maria/ma_test2.c b/storage/maria/ma_test2.c index 709a190c1a7..c34c45d42fb 100644 --- a/storage/maria/ma_test2.c +++ b/storage/maria/ma_test2.c @@ -27,9 +27,6 @@ #define STANDARD_LENGTH 37 #define MARIA_KEYS 6 #define MAX_PARTS 4 -#if !defined(MSDOS) && !defined(labs) -#define labs(a) abs(a) -#endif static void get_options(int argc, char *argv[]); static uint rnd(uint max_value); diff --git a/storage/myisam/ft_boolean_search.c b/storage/myisam/ft_boolean_search.c index 766f8411ca1..4145ca75483 100644 --- a/storage/myisam/ft_boolean_search.c +++ b/storage/myisam/ft_boolean_search.c @@ -195,12 +195,7 @@ static int ftb_query_add_word(MYSQL_FTPARSER_PARAM *param, switch (info->type) { case FT_TOKEN_WORD: ftbw= (FTB_WORD *)alloc_root(&ftb_param->ftb->mem_root, - sizeof(FTB_WORD) + - (info->trunc ? HA_MAX_KEY_BUFF : - (word_len + 1) * - ftb_param->ftb->charset->mbmaxlen + - HA_FT_WLEN + - ftb_param->ftb->info->s->rec_reflength)); + sizeof(FTB_WORD) + HA_MAX_KEY_BUFF); ftbw->len= word_len + 1; ftbw->flags= 0; ftbw->off= 0; diff --git a/storage/myisam/mi_test2.c b/storage/myisam/mi_test2.c index be58b3c54d0..32dabca0ef5 100644 --- a/storage/myisam/mi_test2.c +++ b/storage/myisam/mi_test2.c @@ -26,9 +26,6 @@ #define STANDARD_LENGTH 37 #define MYISAM_KEYS 6 #define MAX_PARTS 4 -#if !defined(labs) -#define labs(a) abs(a) -#endif static void get_options(int argc, char *argv[]); static uint rnd(uint max_value); diff --git a/storage/perfschema/pfs.cc b/storage/perfschema/pfs.cc index dd61df5f861..ad3faeea618 100644 --- a/storage/perfschema/pfs.cc +++ b/storage/perfschema/pfs.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2008, 2015, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2008, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -2560,10 +2560,7 @@ start_table_io_wait_v1(PSI_table_locker_state *state, if (! pfs_table->m_io_enabled) return NULL; - PFS_thread *pfs_thread= pfs_table->m_thread_owner; - - DBUG_ASSERT(pfs_thread == - my_pthread_getspecific_ptr(PFS_thread*, THR_PFS)); + PFS_thread *pfs_thread= my_pthread_getspecific_ptr(PFS_thread*, THR_PFS); register uint flags; ulonglong timer_start= 0; @@ -2666,7 +2663,7 @@ start_table_lock_wait_v1(PSI_table_locker_state *state, if (! pfs_table->m_lock_enabled) return NULL; - PFS_thread *pfs_thread= pfs_table->m_thread_owner; + PFS_thread *pfs_thread= my_pthread_getspecific_ptr(PFS_thread*, THR_PFS); PFS_TL_LOCK_TYPE lock_type; @@ -3068,7 +3065,12 @@ start_socket_wait_v1(PSI_socket_locker_state *state, if (flag_thread_instrumentation) { - PFS_thread *pfs_thread= pfs_socket->m_thread_owner; + /* + Do not use pfs_socket->m_thread_owner here, + as different threads may use concurrently the same socket, + for example during a KILL. + */ + PFS_thread *pfs_thread= my_pthread_getspecific_ptr(PFS_thread*, THR_PFS); if (unlikely(pfs_thread == NULL)) return NULL; @@ -3436,6 +3438,8 @@ static void end_idle_wait_v1(PSI_idle_locker* locker) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } @@ -3517,6 +3521,8 @@ static void end_mutex_wait_v1(PSI_mutex_locker* locker, int rc) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } } @@ -3596,6 +3602,8 @@ static void end_rwlock_rdwait_v1(PSI_rwlock_locker* locker, int rc) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } } @@ -3668,6 +3676,8 @@ static void end_rwlock_wrwait_v1(PSI_rwlock_locker* locker, int rc) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } } @@ -3732,6 +3742,8 @@ static void end_cond_wait_v1(PSI_cond_locker* locker, int rc) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } } @@ -3826,6 +3838,8 @@ static void end_table_io_wait_v1(PSI_table_locker* locker) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } @@ -3895,6 +3909,8 @@ static void end_table_lock_wait_v1(PSI_table_locker* locker) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } @@ -4143,6 +4159,8 @@ static void end_file_wait_v1(PSI_file_locker *locker, if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } } @@ -5070,6 +5088,8 @@ static void end_socket_wait_v1(PSI_socket_locker *locker, size_t byte_count) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } diff --git a/storage/perfschema/pfs_digest.cc b/storage/perfschema/pfs_digest.cc index bf42ada8e67..3b4842a67de 100644 --- a/storage/perfschema/pfs_digest.cc +++ b/storage/perfschema/pfs_digest.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2008, 2015, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2008, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -45,7 +45,7 @@ bool flag_statements_digest= true; Current index in Stat array where new record is to be inserted. index 0 is reserved for "all else" case when entire array is full. */ -volatile uint32 digest_index; +volatile uint32 PFS_ALIGNED digest_monotonic_index; bool digest_full= false; LF_HASH digest_hash; @@ -63,7 +63,7 @@ int init_digest(const PFS_global_param *param) */ digest_max= param->m_digest_sizing; digest_lost= 0; - digest_index= 1; + PFS_atomic::store_u32(& digest_monotonic_index, 1); digest_full= false; if (digest_max == 0) @@ -105,6 +105,9 @@ int init_digest(const PFS_global_param *param) + index * pfs_max_digest_length, pfs_max_digest_length); } + /* Set record[0] as allocated. */ + statements_digest_stat_array[0].m_lock.set_allocated(); + return 0; } @@ -207,9 +210,10 @@ find_or_create_digest(PFS_thread *thread, memcpy(hash_key.m_schema_name, schema_name, schema_name_length); int res; - ulong safe_index; uint retry_count= 0; const uint retry_max= 3; + size_t safe_index; + size_t attempts= 0; PFS_statements_digest_stat **entry; PFS_statements_digest_stat *pfs= NULL; @@ -245,55 +249,70 @@ search: return & pfs->m_stat; } - safe_index= PFS_atomic::add_u32(& digest_index, 1); - if (safe_index >= digest_max) + while (++attempts <= digest_max) { - /* The digest array is now full. */ - digest_full= true; - pfs= &statements_digest_stat_array[0]; - - if (pfs->m_first_seen == 0) - pfs->m_first_seen= now; - pfs->m_last_seen= now; - return & pfs->m_stat; - } - - /* Add a new record in digest stat array. */ - pfs= &statements_digest_stat_array[safe_index]; - - /* Copy digest hash/LF Hash search key. */ - memcpy(& pfs->m_digest_key, &hash_key, sizeof(PFS_digest_key)); - - /* - Copy digest storage to statement_digest_stat_array so that it could be - used later to generate digest text. - */ - pfs->m_digest_storage.copy(digest_storage); - - pfs->m_first_seen= now; - pfs->m_last_seen= now; + safe_index= PFS_atomic::add_u32(& digest_monotonic_index, 1) % digest_max; + if (safe_index == 0) + { + /* Record [0] is reserved. */ + safe_index= 1; + } - res= lf_hash_insert(&digest_hash, pins, &pfs); - if (likely(res == 0)) - { - return & pfs->m_stat; - } + /* Add a new record in digest stat array. */ + pfs= &statements_digest_stat_array[safe_index]; - if (res > 0) - { - /* Duplicate insert by another thread */ - if (++retry_count > retry_max) + if (pfs->m_lock.is_free()) { - /* Avoid infinite loops */ - digest_lost++; - return NULL; + if (pfs->m_lock.free_to_dirty()) + { + /* Copy digest hash/LF Hash search key. */ + memcpy(& pfs->m_digest_key, &hash_key, sizeof(PFS_digest_key)); + + /* + Copy digest storage to statement_digest_stat_array so that it could be + used later to generate digest text. + */ + pfs->m_digest_storage.copy(digest_storage); + + pfs->m_first_seen= now; + pfs->m_last_seen= now; + + res= lf_hash_insert(&digest_hash, pins, &pfs); + if (likely(res == 0)) + { + pfs->m_lock.dirty_to_allocated(); + return & pfs->m_stat; + } + + pfs->m_lock.dirty_to_free(); + + if (res > 0) + { + /* Duplicate insert by another thread */ + if (++retry_count > retry_max) + { + /* Avoid infinite loops */ + digest_lost++; + return NULL; + } + goto search; + } + + /* OOM in lf_hash_insert */ + digest_lost++; + return NULL; + } } - goto search; } - /* OOM in lf_hash_insert */ - digest_lost++; - return NULL; + /* The digest array is now full. */ + digest_full= true; + pfs= &statements_digest_stat_array[0]; + + if (pfs->m_first_seen == 0) + pfs->m_first_seen= now; + pfs->m_last_seen= now; + return & pfs->m_stat; } void purge_digest(PFS_thread* thread, PFS_digest_key *hash_key) @@ -320,10 +339,12 @@ void purge_digest(PFS_thread* thread, PFS_digest_key *hash_key) void PFS_statements_digest_stat::reset_data(unsigned char *token_array, uint length) { + m_lock.set_dirty(); m_digest_storage.reset(token_array, length); m_stat.reset(); m_first_seen= 0; m_last_seen= 0; + m_lock.dirty_to_free(); } void PFS_statements_digest_stat::reset_index(PFS_thread *thread) @@ -351,11 +372,14 @@ void reset_esms_by_digest() statements_digest_stat_array[index].reset_data(statements_digest_token_array + index * pfs_max_digest_length, pfs_max_digest_length); } + /* Mark record[0] as allocated again. */ + statements_digest_stat_array[0].m_lock.set_allocated(); + /* Reset index which indicates where the next calculated digest information to be inserted in statements_digest_stat_array. */ - digest_index= 1; + PFS_atomic::store_u32(& digest_monotonic_index, 1); digest_full= false; } diff --git a/storage/perfschema/pfs_digest.h b/storage/perfschema/pfs_digest.h index 76d6c33d984..429a9f4250a 100644 --- a/storage/perfschema/pfs_digest.h +++ b/storage/perfschema/pfs_digest.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2011, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -44,6 +44,9 @@ struct PFS_digest_key /** A statement digest stat record. */ struct PFS_ALIGNED PFS_statements_digest_stat { + /** Internal lock. */ + pfs_lock m_lock; + /** Digest Schema + MD5 Hash. */ PFS_digest_key m_digest_key; diff --git a/storage/perfschema/pfs_lock.h b/storage/perfschema/pfs_lock.h index 09efecd1c5f..be84d0f7fb4 100644 --- a/storage/perfschema/pfs_lock.h +++ b/storage/perfschema/pfs_lock.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2009, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -156,6 +156,18 @@ struct pfs_lock } /** + Initialize a lock to dirty. + */ + void set_dirty(void) + { + /* Do not set the version to 0, read the previous value. */ + uint32 copy= PFS_atomic::load_u32(&m_version_state); + /* Increment the version, set the DIRTY state */ + uint32 new_val= (copy & VERSION_MASK) + VERSION_INC + PFS_LOCK_DIRTY; + PFS_atomic::store_u32(&m_version_state, new_val); + } + + /** Execute a dirty to free transition. This transition should be executed by the writer that owns the record. */ diff --git a/storage/perfschema/table_esms_by_digest.cc b/storage/perfschema/table_esms_by_digest.cc index 80fa4077281..5a3145b91d1 100644 --- a/storage/perfschema/table_esms_by_digest.cc +++ b/storage/perfschema/table_esms_by_digest.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2010, 2012, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2010, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -238,11 +238,14 @@ int table_esms_by_digest::rnd_next(void) m_pos.next()) { digest_stat= &statements_digest_stat_array[m_pos.m_index]; - if (digest_stat->m_first_seen != 0) + if (digest_stat->m_lock.is_populated()) { - make_row(digest_stat); - m_next_pos.set_after(&m_pos); - return 0; + if (digest_stat->m_first_seen != 0) + { + make_row(digest_stat); + m_next_pos.set_after(&m_pos); + return 0; + } } } @@ -260,10 +263,13 @@ table_esms_by_digest::rnd_pos(const void *pos) set_position(pos); digest_stat= &statements_digest_stat_array[m_pos.m_index]; - if (digest_stat->m_first_seen != 0) + if (digest_stat->m_lock.is_populated()) { - make_row(digest_stat); - return 0; + if (digest_stat->m_first_seen != 0) + { + make_row(digest_stat); + return 0; + } } return HA_ERR_RECORD_DELETED; diff --git a/storage/perfschema/unittest/pfs-t.cc b/storage/perfschema/unittest/pfs-t.cc index 6121fac098f..f76b1aa2e75 100644 --- a/storage/perfschema/unittest/pfs-t.cc +++ b/storage/perfschema/unittest/pfs-t.cc @@ -1316,6 +1316,7 @@ void test_locker_disabled() /* Pretend the socket does not have a thread owner */ /* ---------------------------------------------- */ + psi->delete_current_thread(); socket_class_A->m_enabled= true; socket_A1= psi->init_socket(socket_key_A, NULL, NULL, 0); ok(socket_A1 != NULL, "instrumented"); diff --git a/storage/sphinx/mysql-test/sphinx/suite.pm b/storage/sphinx/mysql-test/sphinx/suite.pm index e4c3c1b9f74..fc127ffd6c0 100644 --- a/storage/sphinx/mysql-test/sphinx/suite.pm +++ b/storage/sphinx/mysql-test/sphinx/suite.pm @@ -16,13 +16,26 @@ sub locate_sphinx_binary { for (@list) { return $_ if -x $_; } } -# Look for Sphinx binaries. +# Look for Sphinx binaries my $exe_sphinx_indexer = &locate_sphinx_binary('indexer'); + +unless ($exe_sphinx_indexer) { + mtr_report("Sphinx 'indexer' binary not found, sphinx suite will be skipped"); + return "No Sphinx"; +} my $exe_sphinx_searchd = &locate_sphinx_binary('searchd'); -return "No Sphinx" unless $exe_sphinx_indexer and $exe_sphinx_searchd; -return "No SphinxSE" unless $ENV{HA_SPHINX_SO} or - $::mysqld_variables{'sphinx'} eq "ON"; +unless ($exe_sphinx_searchd) { + mtr_report("Sphinx 'searchd' binary not found, sphinx suite will be skipped"); + return "No Sphinx"; +} + +# Check for Sphinx engine + +unless ($ENV{HA_SPHINX_SO} or $::mysqld_variables{'sphinx'} eq "ON") { + mtr_report("Sphinx engine not found, sphinx suite will be skipped"); + return "No SphinxSE"; +} { local $_ = `"$exe_sphinx_searchd" --help`; diff --git a/storage/tokudb/CMakeLists.txt b/storage/tokudb/CMakeLists.txt index dad90fe96eb..3017d3cf68e 100644 --- a/storage/tokudb/CMakeLists.txt +++ b/storage/tokudb/CMakeLists.txt @@ -1,4 +1,4 @@ -SET(TOKUDB_VERSION 5.6.33-79.0) +SET(TOKUDB_VERSION 5.6.34-79.1) # PerconaFT only supports x86-64 and cmake-2.8.9+ IF(CMAKE_VERSION VERSION_LESS "2.8.9") MESSAGE(STATUS "CMake 2.8.9 or higher is required by TokuDB") diff --git a/storage/tokudb/ha_tokudb.cc b/storage/tokudb/ha_tokudb.cc index 0afe9958b85..39a2af3f8eb 100644 --- a/storage/tokudb/ha_tokudb.cc +++ b/storage/tokudb/ha_tokudb.cc @@ -3728,6 +3728,8 @@ int ha_tokudb::do_uniqueness_checks(uchar* record, DB_TXN* txn, THD* thd) { // first do uniqueness checks // if (share->has_unique_keys && do_unique_checks(thd, in_rpl_write_rows)) { + DBUG_EXECUTE_IF("tokudb_crash_if_rpl_does_uniqueness_check", + DBUG_ASSERT(0);); for (uint keynr = 0; keynr < table_share->keys; keynr++) { bool is_unique_key = (table->key_info[keynr].flags & HA_NOSAME) || (keynr == primary_key); bool is_unique = false; @@ -5961,6 +5963,7 @@ int ha_tokudb::rnd_pos(uchar * buf, uchar * pos) { // test rpl slave by inducing a delay before the point query THD *thd = ha_thd(); if (thd->slave_thread && (in_rpl_delete_rows || in_rpl_update_rows)) { + DBUG_EXECUTE_IF("tokudb_crash_if_rpl_looks_up_row", DBUG_ASSERT(0);); uint64_t delay_ms = tokudb::sysvars::rpl_lookup_rows_delay(thd); if (delay_ms) usleep(delay_ms * 1000); diff --git a/storage/tokudb/mysql-test/tokudb/r/cluster_2968-1.result b/storage/tokudb/mysql-test/tokudb/r/cluster_2968-1.result index f79b7aa05aa..0108e33c710 100644 --- a/storage/tokudb/mysql-test/tokudb/r/cluster_2968-1.result +++ b/storage/tokudb/mysql-test/tokudb/r/cluster_2968-1.result @@ -1045,10 +1045,10 @@ id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t ref b b 5 test.s.b 1 alter table s add key(b) clustering=yes; Warnings: -Note 1831 Duplicate index 'b_2' defined on the table 'test.s'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_2`. This is deprecated and will be disallowed in a future release. alter table t add key(b) clustering=yes; Warnings: -Note 1831 Duplicate index 'b_2' defined on the table 'test.t'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_2`. This is deprecated and will be disallowed in a future release. show create table s; Table Create Table s CREATE TABLE `s` ( @@ -1095,10 +1095,10 @@ id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t ref b_2 b_2 5 test.s.b 1 Using index alter table s add key(b); Warnings: -Note 1831 Duplicate index 'b' defined on the table 'test.s'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b`. This is deprecated and will be disallowed in a future release. alter table t add key(b); Warnings: -Note 1831 Duplicate index 'b' defined on the table 'test.t'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b`. This is deprecated and will be disallowed in a future release. show create table s; Table Create Table s CREATE TABLE `s` ( diff --git a/storage/tokudb/mysql-test/tokudb/r/cluster_2968-2.result b/storage/tokudb/mysql-test/tokudb/r/cluster_2968-2.result index a935574ba7f..b14fb676ed2 100644 --- a/storage/tokudb/mysql-test/tokudb/r/cluster_2968-2.result +++ b/storage/tokudb/mysql-test/tokudb/r/cluster_2968-2.result @@ -1069,10 +1069,10 @@ id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t ref b,b_2 b_2 5 test.s.b 1 Using index alter table s add key(b) clustering=yes; Warnings: -Note 1831 Duplicate index 'b_3' defined on the table 'test.s'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_3`. This is deprecated and will be disallowed in a future release. alter table t add key(b) clustering=yes; Warnings: -Note 1831 Duplicate index 'b_3' defined on the table 'test.t'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_3`. This is deprecated and will be disallowed in a future release. show create table s; Table Create Table s CREATE TABLE `s` ( diff --git a/storage/tokudb/mysql-test/tokudb/r/cluster_2968-3.result b/storage/tokudb/mysql-test/tokudb/r/cluster_2968-3.result index 85a318b7163..89641d25239 100644 --- a/storage/tokudb/mysql-test/tokudb/r/cluster_2968-3.result +++ b/storage/tokudb/mysql-test/tokudb/r/cluster_2968-3.result @@ -1066,13 +1066,13 @@ id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE u ref c c 5 test.s.c 1 alter table s add key (b) clustering=yes; Warnings: -Note 1831 Duplicate index 'b_2' defined on the table 'test.s'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_2`. This is deprecated and will be disallowed in a future release. alter table t add key (b) clustering=yes; Warnings: -Note 1831 Duplicate index 'b_2' defined on the table 'test.t'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `b_2`. This is deprecated and will be disallowed in a future release. alter table u add key (c) clustering=yes; Warnings: -Note 1831 Duplicate index 'c_2' defined on the table 'test.u'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `c_2`. This is deprecated and will be disallowed in a future release. show create table s; Table Create Table s CREATE TABLE `s` ( diff --git a/storage/tokudb/mysql-test/tokudb/r/table_index_statistics.result b/storage/tokudb/mysql-test/tokudb/r/table_index_statistics.result new file mode 100644 index 00000000000..d3184bfb07a --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb/r/table_index_statistics.result @@ -0,0 +1,48 @@ +SET @default_storage_engine_old = @@session.default_storage_engine; +SET SESSION default_storage_engine = TOKUDB; +FLUSH INDEX_STATISTICS; +FLUSH TABLE_STATISTICS; +SET @userstat_old= @@userstat; +SET GLOBAL userstat=ON; +CREATE TABLE t1 (id int(10), PRIMARY KEY (id)); +INSERT INTO t1 VALUES (1),(2),(3),(4),(5),(6),(7),(8),(9),(10); +SELECT COUNT(*) FROM t1; +COUNT(*) +10 +SELECT ROWS_READ FROM INFORMATION_SCHEMA.TABLE_STATISTICS WHERE TABLE_NAME='t1'; +ROWS_READ +10 +SELECT ROWS_READ FROM INFORMATION_SCHEMA.INDEX_STATISTICS WHERE TABLE_NAME='t1'; +ROWS_READ +10 +FLUSH TABLE_STATISTICS; +SELECT ROWS_READ FROM INFORMATION_SCHEMA.TABLE_STATISTICS WHERE TABLE_NAME='t1'; +ROWS_READ +SELECT ROWS_READ FROM INFORMATION_SCHEMA.INDEX_STATISTICS WHERE TABLE_NAME='t1'; +ROWS_READ +10 +FLUSH INDEX_STATISTICS; +SELECT ROWS_READ FROM INFORMATION_SCHEMA.INDEX_STATISTICS WHERE TABLE_NAME='t1'; +ROWS_READ +SELECT COUNT(*) FROM t1; +COUNT(*) +10 +SELECT ROWS_READ FROM INFORMATION_SCHEMA.TABLE_STATISTICS WHERE TABLE_NAME='t1'; +ROWS_READ +10 +SELECT ROWS_READ FROM INFORMATION_SCHEMA.INDEX_STATISTICS WHERE TABLE_NAME='t1'; +ROWS_READ +10 +DROP TABLE t1; +CREATE TABLE t2 (c1 INT UNSIGNED); +ALTER TABLE t2 MODIFY c1 FLOAT; +SELECT * FROM INFORMATION_SCHEMA.TABLE_STATISTICS WHERE TABLE_NAME='t2'; +TABLE_SCHEMA TABLE_NAME ROWS_READ ROWS_CHANGED ROWS_CHANGED_X_INDEXES +DROP TABLE t2; +CREATE TABLE t2 (c1 INT UNSIGNED); +ALTER TABLE t2 MODIFY c1 FLOAT; +SELECT * FROM INFORMATION_SCHEMA.TABLE_STATISTICS WHERE TABLE_NAME='t2'; +TABLE_SCHEMA TABLE_NAME ROWS_READ ROWS_CHANGED ROWS_CHANGED_X_INDEXES +DROP TABLE t2; +SET GLOBAL userstat= @userstat_old; +SET SESSION default_storage_engine = @default_storage_engine_old; diff --git a/storage/tokudb/mysql-test/tokudb/r/type_decimal.result b/storage/tokudb/mysql-test/tokudb/r/type_decimal.result index 23f1dd4110f..68e16cf61ed 100644 --- a/storage/tokudb/mysql-test/tokudb/r/type_decimal.result +++ b/storage/tokudb/mysql-test/tokudb/r/type_decimal.result @@ -222,7 +222,6 @@ drop table t1; create table t1 (a decimal(10,2) unsigned); insert into t1 values ("0.0"),("-0.0"),("+0.0"),("01.0"),("+01.0"),("-01.0"); Warnings: -Warning 1264 Out of range value for column 'a' at row 2 Warning 1264 Out of range value for column 'a' at row 6 insert into t1 values ("-.1"),("+.1"),(".1"); Warnings: @@ -281,7 +280,6 @@ drop table t1; create table t1 (a decimal(10,2) zerofill); insert into t1 values ("0.0"),("-0.0"),("+0.0"),("01.0"),("+01.0"),("-01.0"); Warnings: -Warning 1264 Out of range value for column 'a' at row 2 Warning 1264 Out of range value for column 'a' at row 6 insert into t1 values ("-.1"),("+.1"),(".1"); Warnings: diff --git a/storage/tokudb/mysql-test/tokudb/r/type_ranges.result b/storage/tokudb/mysql-test/tokudb/r/type_ranges.result index c1656a663e1..51b9acb9f7a 100644 --- a/storage/tokudb/mysql-test/tokudb/r/type_ranges.result +++ b/storage/tokudb/mysql-test/tokudb/r/type_ranges.result @@ -87,10 +87,10 @@ t1 1 options 2 flags A NA NULL NULL BTREE CREATE UNIQUE INDEX test on t1 ( auto ) ; CREATE INDEX test2 on t1 ( ulonglong,ulong) ; Warnings: -Note 1831 Duplicate index 'test2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `test2`. This is deprecated and will be disallowed in a future release. CREATE INDEX test3 on t1 ( medium ) ; Warnings: -Note 1831 Duplicate index 'test3' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `test3`. This is deprecated and will be disallowed in a future release. DROP INDEX test ON t1; insert into t1 values (10, 1,1,1,1,1,1,1,1,1,1,1,1,1,NULL,0,0,0,1,1,1,1,'one','one'); insert into t1 values (NULL,2,2,2,2,2,2,2,2,2,2,2,2,2,NULL,NULL,NULL,NULL,NULL,NULL,2,2,'two','two,one'); @@ -309,7 +309,7 @@ const int(1) NULL NO 0 # drop table t1,t2,t3; create table t1 ( myfield INT NOT NULL, UNIQUE INDEX (myfield), unique (myfield), index(myfield)); Warnings: -Note 1831 Duplicate index 'myfield_2' defined on the table 'test.t1'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `myfield_2`. This is deprecated and will be disallowed in a future release. drop table t1; create table t1 ( id integer unsigned not null primary key ); create table t2 ( id integer unsigned not null primary key ); diff --git a/storage/tokudb/mysql-test/tokudb/t/table_index_statistics.test b/storage/tokudb/mysql-test/tokudb/t/table_index_statistics.test new file mode 100644 index 00000000000..9a3b28aab95 --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb/t/table_index_statistics.test @@ -0,0 +1,8 @@ +--source include/have_tokudb.inc + +SET @default_storage_engine_old = @@session.default_storage_engine; +SET SESSION default_storage_engine = TOKUDB; + +--source extra/table_index_statistics.inc + +SET SESSION default_storage_engine = @default_storage_engine_old; diff --git a/storage/tokudb/mysql-test/tokudb_alter_table/r/hcad_tmp_tables_56.result b/storage/tokudb/mysql-test/tokudb_alter_table/r/hcad_tmp_tables_56.result index 0ebc7b3d874..ff03e9c81ab 100644 --- a/storage/tokudb/mysql-test/tokudb_alter_table/r/hcad_tmp_tables_56.result +++ b/storage/tokudb/mysql-test/tokudb_alter_table/r/hcad_tmp_tables_56.result @@ -5,7 +5,7 @@ create temporary table bar (a int, key(a))engine=TOkuDB; alter table bar add column c int default 0; create index blah on bar(a); Warnings: -Note 1831 Duplicate index 'blah' defined on the table 'test.bar'. This is deprecated and will be disallowed in a future release. +Note 1831 Duplicate index `blah`. This is deprecated and will be disallowed in a future release. drop index a on bar; set session tokudb_disable_slow_alter=OFF; insert into bar (a) values (1),(2),(3); diff --git a/storage/tokudb/mysql-test/tokudb_rpl/r/rpl_tokudb_rfr_partition_table.result b/storage/tokudb/mysql-test/tokudb_rpl/r/rpl_tokudb_rfr_partition_table.result new file mode 100644 index 00000000000..4594959c6d0 --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb_rpl/r/rpl_tokudb_rfr_partition_table.result @@ -0,0 +1,36 @@ +include/master-slave.inc +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. +[connection master] +call mtr.add_suppression(".*read free replication is disabled for TokuDB table.*continue with rows lookup"); +CREATE TABLE t1 (id int(11) NOT NULL, pid int(11), PRIMARY KEY (id)) ENGINE=TokuDB +PARTITION BY RANGE (id) +(PARTITION p_1 VALUES LESS THAN (10) ENGINE = TokuDB, +PARTITION p_2 VALUES LESS THAN (20) ENGINE = TokuDB, +PARTITION p_all VALUES LESS THAN MAXVALUE ENGINE = TokuDB); +insert into t1 values (1, 1), (2, 2), (3, 3), (11, 11), (12, 12), (13, 13); +CREATE TABLE t2 (id int(11) NOT NULL, pid int(11), key idx_1(id)) ENGINE=TokuDB +PARTITION BY RANGE (id) +(PARTITION p_1 VALUES LESS THAN (10) ENGINE = TokuDB, +PARTITION p_2 VALUES LESS THAN (20) ENGINE = TokuDB, +PARTITION p_all VALUES LESS THAN MAXVALUE ENGINE = TokuDB); +insert into t2 values (1, 1), (2, 2), (3, 3), (11, 11), (12, 12), (13, 13); +include/stop_slave.inc +set global debug= "+d,tokudb_crash_if_rpl_looks_up_row,tokudb_crash_if_rpl_does_uniqueness_check"; +include/start_slave.inc +insert into t1 values(21, 21); +delete from t1 where id = 11; +update t1 set pid = 2 where id = 1; +include/diff_tables.inc [master:test.t1, slave:test.t1] +insert into t2 values(21, 21); +delete from t2 where id = 11; +update t2 set pid = 2 where id = 1; +include/diff_tables.inc [master:test.t2, slave:test.t2] +drop table t1; +drop table t2; +include/stop_slave.inc +set global debug= "-d,tokudb_crash_if_rpl_looks_up_row,tokudb_crash_if_rpl_does_uniqueness_check"; +set global debug= @saved_debug; +include/start_slave.inc +include/rpl_end.inc diff --git a/storage/tokudb/mysql-test/tokudb_rpl/t/rpl_tokudb_rfr_partition_table-slave.opt b/storage/tokudb/mysql-test/tokudb_rpl/t/rpl_tokudb_rfr_partition_table-slave.opt new file mode 100644 index 00000000000..2a7ec2590cc --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb_rpl/t/rpl_tokudb_rfr_partition_table-slave.opt @@ -0,0 +1 @@ +--read-only=ON --loose-tokudb-rpl-unique-checks=OFF --loose-tokudb-rpl-lookup-rows=OFF diff --git a/storage/tokudb/mysql-test/tokudb_rpl/t/rpl_tokudb_rfr_partition_table.test b/storage/tokudb/mysql-test/tokudb_rpl/t/rpl_tokudb_rfr_partition_table.test new file mode 100644 index 00000000000..319e741e12c --- /dev/null +++ b/storage/tokudb/mysql-test/tokudb_rpl/t/rpl_tokudb_rfr_partition_table.test @@ -0,0 +1,74 @@ +# test tokudb read free replication feature with partition table + +--source include/have_debug.inc +--source include/have_tokudb.inc +--source include/have_binlog_format_row.inc +--source include/master-slave.inc + +call mtr.add_suppression(".*read free replication is disabled for TokuDB table.*continue with rows lookup"); + +connection master; + +# partition table with explicit PK +CREATE TABLE t1 (id int(11) NOT NULL, pid int(11), PRIMARY KEY (id)) ENGINE=TokuDB +PARTITION BY RANGE (id) +(PARTITION p_1 VALUES LESS THAN (10) ENGINE = TokuDB, + PARTITION p_2 VALUES LESS THAN (20) ENGINE = TokuDB, + PARTITION p_all VALUES LESS THAN MAXVALUE ENGINE = TokuDB); + +insert into t1 values (1, 1), (2, 2), (3, 3), (11, 11), (12, 12), (13, 13); + +# partition table without explicit PK +CREATE TABLE t2 (id int(11) NOT NULL, pid int(11), key idx_1(id)) ENGINE=TokuDB +PARTITION BY RANGE (id) +(PARTITION p_1 VALUES LESS THAN (10) ENGINE = TokuDB, + PARTITION p_2 VALUES LESS THAN (20) ENGINE = TokuDB, + PARTITION p_all VALUES LESS THAN MAXVALUE ENGINE = TokuDB); + +insert into t2 values (1, 1), (2, 2), (3, 3), (11, 11), (12, 12), (13, 13); + +--sync_slave_with_master + +# set tokudb rfr crash/assert conditions if we enter lookup code +# to make sure no unique checks or row lookups is invoked +connection slave; +--source include/stop_slave.inc +let $saved_debug = `select @@debug`; +set global debug= "+d,tokudb_crash_if_rpl_looks_up_row,tokudb_crash_if_rpl_does_uniqueness_check"; +--source include/start_slave.inc + +connection master; +insert into t1 values(21, 21); +delete from t1 where id = 11; +update t1 set pid = 2 where id = 1; + +sync_slave_with_master; + +connection master; + +--let $diff_tables= master:test.t1, slave:test.t1 +--source include/diff_tables.inc + +# print rfr disabled warning in errlog +connection master; +insert into t2 values(21, 21); +delete from t2 where id = 11; +update t2 set pid = 2 where id = 1; + +sync_slave_with_master; + +--let $diff_tables= master:test.t2, slave:test.t2 +--source include/diff_tables.inc + +connection master; +drop table t1; +drop table t2; +sync_slave_with_master; + +connection slave; +--source include/stop_slave.inc +set global debug= "-d,tokudb_crash_if_rpl_looks_up_row,tokudb_crash_if_rpl_does_uniqueness_check"; +set global debug= @saved_debug; +--source include/start_slave.inc + +--source include/rpl_end.inc diff --git a/storage/xtradb/dict/dict0boot.cc b/storage/xtradb/dict/dict0boot.cc index c0bb0298bea..20ab5d75205 100644 --- a/storage/xtradb/dict/dict0boot.cc +++ b/storage/xtradb/dict/dict0boot.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2012, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -272,10 +273,6 @@ dict_boot(void) ut_ad(DICT_NUM_FIELDS__SYS_FOREIGN_FOR_NAME == 2); ut_ad(DICT_NUM_COLS__SYS_FOREIGN_COLS == 4); ut_ad(DICT_NUM_FIELDS__SYS_FOREIGN_COLS == 6); - ut_ad(DICT_NUM_COLS__SYS_ZIP_DICT == 3); - ut_ad(DICT_NUM_FIELDS__SYS_ZIP_DICT == 5); - ut_ad(DICT_NUM_COLS__SYS_ZIP_DICT_COLS == 3); - ut_ad(DICT_NUM_FIELDS__SYS_ZIP_DICT_COLS == 5); mtr_start(&mtr); diff --git a/storage/xtradb/dict/dict0crea.cc b/storage/xtradb/dict/dict0crea.cc index db0ca638de4..9a4040421d7 100644 --- a/storage/xtradb/dict/dict0crea.cc +++ b/storage/xtradb/dict/dict0crea.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1932,135 +1933,6 @@ dict_create_or_check_sys_tablespace(void) return(err); } -/** Creates the zip_dict system table inside InnoDB -at server bootstrap or server start if it is not found or is -not of the right form. -@return DB_SUCCESS or error code */ -UNIV_INTERN -dberr_t -dict_create_or_check_sys_zip_dict(void) -{ - trx_t* trx; - my_bool srv_file_per_table_backup; - dberr_t err; - dberr_t sys_zip_dict_err; - dberr_t sys_zip_dict_cols_err; - - ut_a(srv_get_active_thread_type() == SRV_NONE); - - /* Note: The master thread has not been started at this point. */ - - sys_zip_dict_err = dict_check_if_system_table_exists( - "SYS_ZIP_DICT", DICT_NUM_FIELDS__SYS_ZIP_DICT + 1, 2); - sys_zip_dict_cols_err = dict_check_if_system_table_exists( - "SYS_ZIP_DICT_COLS", DICT_NUM_FIELDS__SYS_ZIP_DICT_COLS + 1, - 1); - - if (sys_zip_dict_err == DB_SUCCESS && - sys_zip_dict_cols_err == DB_SUCCESS) - return (DB_SUCCESS); - - trx = trx_allocate_for_mysql(); - - trx_set_dict_operation(trx, TRX_DICT_OP_TABLE); - - trx->op_info = "creating zip_dict and zip_dict_cols sys tables"; - - row_mysql_lock_data_dictionary(trx); - - /* Check which incomplete table definition to drop. */ - - if (sys_zip_dict_err == DB_CORRUPTION) { - ib_logf(IB_LOG_LEVEL_WARN, - "Dropping incompletely created " - "SYS_ZIP_DICT table."); - row_drop_table_for_mysql("SYS_ZIP_DICT", trx, TRUE, TRUE); - } - if (sys_zip_dict_cols_err == DB_CORRUPTION) { - ib_logf(IB_LOG_LEVEL_WARN, - "Dropping incompletely created " - "SYS_ZIP_DICT_COLS table."); - row_drop_table_for_mysql("SYS_ZIP_DICT_COLS", trx, TRUE, TRUE); - } - - ib_logf(IB_LOG_LEVEL_INFO, - "Creating zip_dict and zip_dict_cols system tables."); - - /* We always want SYSTEM tables to be created inside the system - tablespace. */ - srv_file_per_table_backup = srv_file_per_table; - srv_file_per_table = 0; - - err = que_eval_sql( - NULL, - "PROCEDURE CREATE_SYS_ZIP_DICT_PROC () IS\n" - "BEGIN\n" - "CREATE TABLE SYS_ZIP_DICT(\n" - " ID INT UNSIGNED NOT NULL,\n" - " NAME CHAR(" - STRINGIFY_ARG(ZIP_DICT_MAX_NAME_LENGTH) - ") NOT NULL,\n" - " DATA BLOB NOT NULL\n" - ");\n" - "CREATE UNIQUE CLUSTERED INDEX SYS_ZIP_DICT_ID" - " ON SYS_ZIP_DICT (ID);\n" - "CREATE UNIQUE INDEX SYS_ZIP_DICT_NAME" - " ON SYS_ZIP_DICT (NAME);\n" - "CREATE TABLE SYS_ZIP_DICT_COLS(\n" - " TABLE_ID INT UNSIGNED NOT NULL,\n" - " COLUMN_POS INT UNSIGNED NOT NULL,\n" - " DICT_ID INT UNSIGNED NOT NULL\n" - ");\n" - "CREATE UNIQUE CLUSTERED INDEX SYS_ZIP_DICT_COLS_COMPOSITE" - " ON SYS_ZIP_DICT_COLS (TABLE_ID, COLUMN_POS);\n" - "END;\n", - FALSE, trx); - - if (err != DB_SUCCESS) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Creation of SYS_ZIP_DICT and SYS_ZIP_DICT_COLS" - "has failed with error %lu. Tablespace is full. " - "Dropping incompletely created tables.", - (ulong) err); - - ut_a(err == DB_OUT_OF_FILE_SPACE - || err == DB_TOO_MANY_CONCURRENT_TRXS); - - row_drop_table_for_mysql("SYS_ZIP_DICT", trx, TRUE, TRUE); - row_drop_table_for_mysql("SYS_ZIP_DICT_COLS", trx, TRUE, TRUE); - - if (err == DB_OUT_OF_FILE_SPACE) { - err = DB_MUST_GET_MORE_FILE_SPACE; - } - } - - trx_commit_for_mysql(trx); - - row_mysql_unlock_data_dictionary(trx); - - trx_free_for_mysql(trx); - - srv_file_per_table = srv_file_per_table_backup; - - if (err == DB_SUCCESS) { - ib_logf(IB_LOG_LEVEL_INFO, - "zip_dict and zip_dict_cols system tables created."); - } - - /* Note: The master thread has not been started at this point. */ - /* Confirm and move to the non-LRU part of the table LRU list. */ - - sys_zip_dict_err = dict_check_if_system_table_exists( - "SYS_ZIP_DICT", DICT_NUM_FIELDS__SYS_ZIP_DICT + 1, 2); - ut_a(sys_zip_dict_err == DB_SUCCESS); - sys_zip_dict_cols_err = dict_check_if_system_table_exists( - "SYS_ZIP_DICT_COLS", - DICT_NUM_FIELDS__SYS_ZIP_DICT_COLS + 1, 1); - ut_a(sys_zip_dict_cols_err == DB_SUCCESS); - - return(err); -} - /********************************************************************//** Add a single tablespace definition to the data dictionary tables in the database. @@ -2114,456 +1986,3 @@ dict_create_add_tablespace_to_dictionary( return(error); } - -/** Add a single compression dictionary definition to the SYS_ZIP_DICT -InnoDB system table. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_add_zip_dict( - const char* name, /*!< in: dict name */ - ulint name_len, /*!< in: dict name length */ - const char* data, /*!< in: dict data */ - ulint data_len, /*!< in: dict data length */ - trx_t* trx) /*!< in/out: transaction */ -{ - ut_ad(name); - ut_ad(data); - - pars_info_t* info = pars_info_create(); - - pars_info_add_literal(info, "name", name, name_len, - DATA_VARCHAR, DATA_ENGLISH); - pars_info_add_literal(info, "data", data, data_len, - DATA_BLOB, DATA_BINARY_TYPE | DATA_NOT_NULL); - - dberr_t error = que_eval_sql(info, - "PROCEDURE P () IS\n" - " max_id INT;\n" - "DECLARE CURSOR cur IS\n" - " SELECT ID FROM SYS_ZIP_DICT\n" - " ORDER BY ID DESC;\n" - "BEGIN\n" - " max_id := 0;\n" - " OPEN cur;\n" - " FETCH cur INTO max_id;\n" - " IF (cur % NOTFOUND) THEN\n" - " max_id := 0;\n" - " END IF;\n" - " CLOSE cur;\n" - " INSERT INTO SYS_ZIP_DICT VALUES" - " (max_id + 1, :name, :data);\n" - "END;\n", - FALSE, trx); - - return error; -} - -/** Fetch callback, just stores extracted zip_dict id in the external -variable. -@return TRUE if all OK */ -static -ibool -dict_create_extract_int_aux( - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: int32 id */ -{ - sel_node_t* node = static_cast<sel_node_t*>(row); - dfield_t* dfield = que_node_get_val(node->select_list); - dtype_t* type = dfield_get_type(dfield); - ulint len = dfield_get_len(dfield); - - ut_a(dtype_get_mtype(type) == DATA_INT); - ut_a(len == sizeof(ib_uint32_t)); - - memcpy(user_arg, dfield_get_data(dfield), sizeof(ib_uint32_t)); - - return(TRUE); -} - -/** Add a single compression dictionary reference to the SYS_ZIP_DICT_COLS -InnoDB system table. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_add_zip_dict_reference( - ulint table_id, /*!< in: table id */ - ulint column_pos, /*!< in: column position */ - ulint dict_id, /*!< in: dict id */ - trx_t* trx) /*!< in/out: transaction */ -{ - pars_info_t* info = pars_info_create(); - - pars_info_add_int4_literal(info, "table_id", table_id); - pars_info_add_int4_literal(info, "column_pos", column_pos); - pars_info_add_int4_literal(info, "dict_id", dict_id); - - dberr_t error = que_eval_sql(info, - "PROCEDURE P () IS\n" - "BEGIN\n" - " INSERT INTO SYS_ZIP_DICT_COLS VALUES" - " (:table_id, :column_pos, :dict_id);\n" - "END;\n", - FALSE, trx); - return error; -} - -/** Get a single compression dictionary id for the given -(table id, column pos) pair. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_get_zip_dict_id_by_reference( - ulint table_id, /*!< in: table id */ - ulint column_pos, /*!< in: column position */ - ulint* dict_id, /*!< out: dict id */ - trx_t* trx) /*!< in/out: transaction */ -{ - ut_ad(dict_id); - - pars_info_t* info = pars_info_create(); - - ib_uint32_t dict_id_buf; - mach_write_to_4(reinterpret_cast<byte*>(&dict_id_buf ), - ULINT32_UNDEFINED); - - pars_info_add_int4_literal(info, "table_id", table_id); - pars_info_add_int4_literal(info, "column_pos", column_pos); - pars_info_bind_function( - info, "my_func", dict_create_extract_int_aux, &dict_id_buf); - - dberr_t error = que_eval_sql(info, - "PROCEDURE P () IS\n" - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR cur IS\n" - " SELECT DICT_ID FROM SYS_ZIP_DICT_COLS\n" - " WHERE TABLE_ID = :table_id AND\n" - " COLUMN_POS = :column_pos;\n" - "BEGIN\n" - " OPEN cur;\n" - " FETCH cur INTO my_func();\n" - " CLOSE cur;\n" - "END;\n", - FALSE, trx); - if (error == DB_SUCCESS) { - ib_uint32_t local_dict_id = mach_read_from_4( - reinterpret_cast<const byte*>(&dict_id_buf)); - if (local_dict_id == ULINT32_UNDEFINED) - error = DB_RECORD_NOT_FOUND; - else - *dict_id = local_dict_id; - } - return error; -} - -/** Get compression dictionary id for the given name. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_get_zip_dict_id_by_name( - const char* dict_name, /*!< in: dict name */ - ulint dict_name_len, /*!< in: dict name length */ - ulint* dict_id, /*!< out: dict id */ - trx_t* trx) /*!< in/out: transaction */ -{ - ut_ad(dict_name); - ut_ad(dict_name_len); - ut_ad(dict_id); - - pars_info_t* info = pars_info_create(); - - pars_info_add_literal(info, "dict_name", dict_name, dict_name_len, - DATA_VARCHAR, DATA_ENGLISH); - - ib_uint32_t dict_id_buf; - mach_write_to_4(reinterpret_cast<byte*>(&dict_id_buf), - ULINT32_UNDEFINED); - pars_info_bind_function( - info, "my_func", dict_create_extract_int_aux, &dict_id_buf); - - dberr_t error = que_eval_sql(info, - "PROCEDURE P () IS\n" - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR cur IS\n" - " SELECT ID FROM SYS_ZIP_DICT\n" - " WHERE NAME = :dict_name;\n" - "BEGIN\n" - " OPEN cur;\n" - " FETCH cur INTO my_func();\n" - " CLOSE cur;\n" - "END;\n", - FALSE, trx); - if (error == DB_SUCCESS) { - ib_uint32_t local_dict_id = mach_read_from_4( - reinterpret_cast<const byte*>(&dict_id_buf)); - if (local_dict_id == ULINT32_UNDEFINED) - error = DB_RECORD_NOT_FOUND; - else - *dict_id = local_dict_id; - } - return error; -} - -/** Auxiliary enum used to indicate zip dict data extraction result code */ -enum zip_dict_info_aux_code { - zip_dict_info_success, /*!< success */ - zip_dict_info_not_found, /*!< zip dict record not found */ - zip_dict_info_oom, /*!< out of memory */ - zip_dict_info_corrupted_name, /*!< corrupted zip dict name */ - zip_dict_info_corrupted_data /*!< corrupted zip dict data */ -}; - -/** Auxiliary struct used to return zip dict info aling with result code */ -struct zip_dict_info_aux { - LEX_STRING name; /*!< zip dict name */ - LEX_STRING data; /*!< zip dict data */ - int code; /*!< result code (0 - success) */ -}; - -/** Fetch callback, just stores extracted zip_dict data in the external -variable. -@return always returns TRUE */ -static -ibool -dict_create_get_zip_dict_info_by_id_aux( - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to zip_dict_info_aux* */ -{ - sel_node_t* node = static_cast<sel_node_t*>(row); - zip_dict_info_aux* result = - static_cast<zip_dict_info_aux*>(user_arg); - - result->code = zip_dict_info_success; - result->name.str = 0; - result->name.length = 0; - result->data.str = 0; - result->data.length = 0; - - /* NAME field */ - que_node_t* exp = node->select_list; - ut_a(exp != 0); - - dfield_t* dfield = que_node_get_val(exp); - dtype_t* type = dfield_get_type(dfield); - ut_a(dtype_get_mtype(type) == DATA_VARCHAR); - - ulint len = dfield_get_len(dfield); - void* data = dfield_get_data(dfield); - - - if (len == UNIV_SQL_NULL) { - result->code = zip_dict_info_corrupted_name; - } - else { - result->name.str = - static_cast<char*>(my_malloc(len + 1, MYF(0))); - if (result->name.str == 0) { - result->code = zip_dict_info_oom; - } - else { - memcpy(result->name.str, data, len); - result->name.str[len] = '\0'; - result->name.length = len; - } - } - - /* DATA field */ - exp = que_node_get_next(exp); - ut_a(exp != 0); - - dfield = que_node_get_val(exp); - type = dfield_get_type(dfield); - ut_a(dtype_get_mtype(type) == DATA_BLOB); - - len = dfield_get_len(dfield); - data = dfield_get_data(dfield); - - if (len == UNIV_SQL_NULL) { - result->code = zip_dict_info_corrupted_data; - } - else { - result->data.str = - static_cast<char*>(my_malloc( - len == 0 ? 1 : len, MYF(0))); - if (result->data.str == 0) { - result->code = zip_dict_info_oom; - } - else { - memcpy(result->data.str, data, len); - result->data.length = len; - } - } - - ut_ad(que_node_get_next(exp) == 0); - - if (result->code != zip_dict_info_success) { - if (result->name.str == 0) { - mem_free(result->name.str); - result->name.str = 0; - result->name.length = 0; - } - if (result->data.str == 0) { - mem_free(result->data.str); - result->data.str = 0; - result->data.length = 0; - } - } - - return TRUE; -} - -/** Get compression dictionary info (name and data) for the given id. -Allocates memory for name and data on success. -Must be freed with mem_free(). -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_get_zip_dict_info_by_id( - ulint dict_id, /*!< in: dict id */ - char** name, /*!< out: dict name */ - ulint* name_len, /*!< out: dict name length*/ - char** data, /*!< out: dict data */ - ulint* data_len, /*!< out: dict data length*/ - trx_t* trx) /*!< in/out: transaction */ -{ - ut_ad(name); - ut_ad(data); - - zip_dict_info_aux rec; - rec.code = zip_dict_info_not_found; - pars_info_t* info = pars_info_create(); - - pars_info_add_int4_literal(info, "id", dict_id); - pars_info_bind_function( - info, "my_func", dict_create_get_zip_dict_info_by_id_aux, - &rec); - - dberr_t error = que_eval_sql(info, - "PROCEDURE P () IS\n" - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR cur IS\n" - " SELECT NAME, DATA FROM SYS_ZIP_DICT\n" - " WHERE ID = :id;\n" - "BEGIN\n" - " OPEN cur;\n" - " FETCH cur INTO my_func();\n" - " CLOSE cur;\n" - "END;\n", - FALSE, trx); - if (error == DB_SUCCESS) { - switch (rec.code) { - case zip_dict_info_success: - *name = rec.name.str; - *name_len = rec.name.length; - *data = rec.data.str; - *data_len = rec.data.length; - break; - case zip_dict_info_not_found: - error = DB_RECORD_NOT_FOUND; - break; - case zip_dict_info_oom: - error = DB_OUT_OF_MEMORY; - break; - case zip_dict_info_corrupted_name: - case zip_dict_info_corrupted_data: - error = DB_INVALID_NULL; - break; - default: - ut_error; - } - } - return error; -} - -/** Remove a single compression dictionary from the data dictionary -tables in the database. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_remove_zip_dict( - const char* name, /*!< in: dict name */ - ulint name_len, /*!< in: dict name length */ - trx_t* trx) /*!< in/out: transaction */ -{ - ut_ad(name); - - pars_info_t* info = pars_info_create(); - - ib_uint32_t dict_id_buf; - mach_write_to_4(reinterpret_cast<byte*>(&dict_id_buf), - ULINT32_UNDEFINED); - ib_uint32_t counter_buf; - mach_write_to_4(reinterpret_cast<byte*>(&counter_buf), - ULINT32_UNDEFINED); - - pars_info_add_literal(info, "name", name, name_len, - DATA_VARCHAR, DATA_ENGLISH); - pars_info_bind_int4_literal(info, "dict_id", &dict_id_buf); - pars_info_bind_function(info, "find_dict_func", - dict_create_extract_int_aux, &dict_id_buf); - pars_info_bind_function(info, "count_func", - dict_create_extract_int_aux, &counter_buf); - - dberr_t error = que_eval_sql(info, - "PROCEDURE P () IS\n" - "DECLARE FUNCTION find_dict_func;\n" - "DECLARE FUNCTION count_func;\n" - "DECLARE CURSOR dict_cur IS\n" - " SELECT ID FROM SYS_ZIP_DICT\n" - " WHERE NAME = :name\n" - " FOR UPDATE;\n" - "DECLARE CURSOR ref_cur IS\n" - " SELECT 1 FROM SYS_ZIP_DICT_COLS\n" - " WHERE DICT_ID = :dict_id;\n" - "BEGIN\n" - " OPEN dict_cur;\n" - " FETCH dict_cur INTO find_dict_func();\n" - " IF NOT (SQL % NOTFOUND) THEN\n" - " OPEN ref_cur;\n" - " FETCH ref_cur INTO count_func();\n" - " IF SQL % NOTFOUND THEN\n" - " DELETE FROM SYS_ZIP_DICT WHERE CURRENT OF dict_cur;\n" - " END IF;\n" - " CLOSE ref_cur;\n" - " END IF;\n" - " CLOSE dict_cur;\n" - "END;\n", - FALSE, trx); - if (error == DB_SUCCESS) { - ib_uint32_t local_dict_id = mach_read_from_4( - reinterpret_cast<const byte*>(&dict_id_buf)); - if (local_dict_id == ULINT32_UNDEFINED) { - error = DB_RECORD_NOT_FOUND; - } - else { - ib_uint32_t local_counter = mach_read_from_4( - reinterpret_cast<const byte*>(&counter_buf)); - if (local_counter != ULINT32_UNDEFINED) - error = DB_ROW_IS_REFERENCED; - } - } - return error; -} - -/** Remove all compression dictionary references for the given table ID from -the data dictionary tables in the database. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_remove_zip_dict_references_for_table( - ulint table_id, /*!< in: table id */ - trx_t* trx) /*!< in/out: transaction */ -{ - pars_info_t* info = pars_info_create(); - - pars_info_add_int4_literal(info, "table_id", table_id); - - dberr_t error = que_eval_sql(info, - "PROCEDURE P () IS\n" - "BEGIN\n" - " DELETE FROM SYS_ZIP_DICT_COLS\n" - " WHERE TABLE_ID = :table_id;\n" - "END;\n", - FALSE, trx); - return error; -} diff --git a/storage/xtradb/dict/dict0dict.cc b/storage/xtradb/dict/dict0dict.cc index 9e2aa8328d1..83e950b9334 100644 --- a/storage/xtradb/dict/dict0dict.cc +++ b/storage/xtradb/dict/dict0dict.cc @@ -2,7 +2,7 @@ Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2014, 2015, MariaDB Corporation. +Copyright (c) 2014, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1614,6 +1614,7 @@ struct dict_foreign_remove_partial if (table != NULL) { table->referenced_set.erase(foreign); } + dict_foreign_free(foreign); } }; @@ -3561,8 +3562,7 @@ dict_foreign_add_to_cache( } if (for_in_cache) { - /* Free the foreign object */ - mem_heap_free(foreign->heap); + dict_foreign_free(foreign); } else { for_in_cache = foreign; } @@ -3586,7 +3586,7 @@ dict_foreign_add_to_cache( " the ones in table."); if (for_in_cache == foreign) { - mem_heap_free(foreign->heap); + dict_foreign_free(foreign); } return(DB_CANNOT_ADD_CONSTRAINT); @@ -3642,7 +3642,7 @@ dict_foreign_add_to_cache( be one */ } - mem_heap_free(foreign->heap); + dict_foreign_free(foreign); } return(DB_CANNOT_ADD_CONSTRAINT); @@ -7310,161 +7310,3 @@ dict_tf_to_row_format_string( return(0); } #endif /* !UNIV_HOTBACKUP */ - -/** Insert a records into SYS_ZIP_DICT. -@retval DB_SUCCESS if OK -@retval dberr_t if the insert failed */ -UNIV_INTERN -dberr_t -dict_create_zip_dict( - const char* name, /*!< in: zip_dict name */ - ulint name_len, /*!< in: zip_dict name length*/ - const char* data, /*!< in: zip_dict data */ - ulint data_len) /*!< in: zip_dict data length */ -{ - dberr_t err = DB_SUCCESS; - trx_t* trx; - - ut_ad(name); - ut_ad(data); - - rw_lock_x_lock(&dict_operation_lock); - dict_mutex_enter_for_mysql(); - - trx = trx_allocate_for_background(); - trx->op_info = "insert zip_dict"; - trx->dict_operation_lock_mode = RW_X_LATCH; - trx_start_if_not_started(trx); - - err = dict_create_add_zip_dict(name, name_len, data, data_len, trx); - - if (err == DB_SUCCESS) { - trx_commit_for_mysql(trx); - } - else { - trx->op_info = "rollback of internal trx on zip_dict table"; - trx_rollback_to_savepoint(trx, NULL); - ut_a(trx->error_state == DB_SUCCESS); - } - trx->op_info = ""; - trx->dict_operation_lock_mode = 0; - trx_free_for_background(trx); - - dict_mutex_exit_for_mysql(); - rw_lock_x_unlock(&dict_operation_lock); - - return err; -} -/** Get single compression dictionary id for the given -(table id, column pos) pair. -@retval DB_SUCCESS if OK -@retval DB_RECORD_NOT_FOUND if not found */ -UNIV_INTERN -dberr_t -dict_get_dictionary_id_by_key( - ulint table_id, /*!< in: table id */ - ulint column_pos, /*!< in: column position */ - ulint* dict_id) /*!< out: zip_dict id */ -{ - dberr_t err = DB_SUCCESS; - trx_t* trx; - - rw_lock_s_lock(&dict_operation_lock); - dict_mutex_enter_for_mysql(); - - trx = trx_allocate_for_background(); - trx->op_info = "get zip dict id by composite key"; - trx->dict_operation_lock_mode = RW_S_LATCH; - trx_start_if_not_started(trx); - - err = dict_create_get_zip_dict_id_by_reference(table_id, column_pos, - dict_id, trx); - - trx_commit_for_mysql(trx); - trx->dict_operation_lock_mode = 0; - trx_free_for_background(trx); - - dict_mutex_exit_for_mysql(); - rw_lock_s_unlock(&dict_operation_lock); - - return err; -} -/** Get compression dictionary info (name and data) for the given id. -Allocates memory in name->str and data->str on success. -Must be freed with mem_free(). -@retval DB_SUCCESS if OK -@retval DB_RECORD_NOT_FOUND if not found */ -UNIV_INTERN -dberr_t -dict_get_dictionary_info_by_id( - ulint dict_id, /*!< in: table name */ - char** name, /*!< out: dictionary name */ - ulint* name_len, /*!< out: dictionary name length*/ - char** data, /*!< out: dictionary data */ - ulint* data_len) /*!< out: dictionary data length*/ -{ - dberr_t err = DB_SUCCESS; - trx_t* trx; - - rw_lock_s_lock(&dict_operation_lock); - dict_mutex_enter_for_mysql(); - - trx = trx_allocate_for_background(); - trx->op_info = "get zip dict name and data by id"; - trx->dict_operation_lock_mode = RW_S_LATCH; - trx_start_if_not_started(trx); - - err = dict_create_get_zip_dict_info_by_id(dict_id, name, name_len, - data, data_len, trx); - - trx_commit_for_mysql(trx); - trx->dict_operation_lock_mode = 0; - trx_free_for_background(trx); - - dict_mutex_exit_for_mysql(); - rw_lock_s_unlock(&dict_operation_lock); - - return err; -} -/** Delete a record in SYS_ZIP_DICT with the given name. -@retval DB_SUCCESS if OK -@retval DB_RECORD_NOT_FOUND if not found -@retval DB_ROW_IS_REFERENCED if in use */ -UNIV_INTERN -dberr_t -dict_drop_zip_dict( - const char* name, /*!< in: zip_dict name */ - ulint name_len) /*!< in: zip_dict name length*/ -{ - dberr_t err = DB_SUCCESS; - trx_t* trx; - - ut_ad(name); - - rw_lock_x_lock(&dict_operation_lock); - dict_mutex_enter_for_mysql(); - - trx = trx_allocate_for_background(); - trx->op_info = "delete zip_dict"; - trx->dict_operation_lock_mode = RW_X_LATCH; - trx_start_if_not_started(trx); - - err = dict_create_remove_zip_dict(name, name_len, trx); - - if (err == DB_SUCCESS) { - trx_commit_for_mysql(trx); - } - else { - trx->op_info = "rollback of internal trx on zip_dict table"; - trx_rollback_to_savepoint(trx, NULL); - ut_a(trx->error_state == DB_SUCCESS); - } - trx->op_info = ""; - trx->dict_operation_lock_mode = 0; - trx_free_for_background(trx); - - dict_mutex_exit_for_mysql(); - rw_lock_x_unlock(&dict_operation_lock); - - return err; -} diff --git a/storage/xtradb/dict/dict0load.cc b/storage/xtradb/dict/dict0load.cc index ca7de72c9b9..6cbd0a3d488 100644 --- a/storage/xtradb/dict/dict0load.cc +++ b/storage/xtradb/dict/dict0load.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -56,9 +57,7 @@ static const char* SYSTEM_TABLE_NAME[] = { "SYS_FOREIGN", "SYS_FOREIGN_COLS", "SYS_TABLESPACES", - "SYS_DATAFILES", - "SYS_ZIP_DICT", - "SYS_ZIP_DICT_COLS" + "SYS_DATAFILES" }; /* If this flag is TRUE, then we will load the cluster index's (and tables') @@ -491,7 +490,7 @@ err_len: } /* This recieves a dict_foreign_t* that points to a stack variable. - So mem_heap_free(foreign->heap) is not used as elsewhere. + So dict_foreign_free(foreign) is not used as elsewhere. Since the heap used here is freed elsewhere, foreign->heap is not assigned. */ foreign->id = mem_heap_strdupl(heap, (const char*) field, len); @@ -730,161 +729,6 @@ err_len: return(NULL); } -/** This function parses a SYS_ZIP_DICT record, extracts necessary -information from the record and returns to caller. -@return error message, or NULL on success */ -UNIV_INTERN -const char* -dict_process_sys_zip_dict( - mem_heap_t* heap, /*!< in/out: heap memory */ - ulint zip_size, /*!< in: nonzero=compressed BLOB page size */ - const rec_t* rec, /*!< in: current SYS_ZIP_DICT rec */ - ulint* id, /*!< out: dict id */ - const char** name, /*!< out: dict name */ - const char** data, /*!< out: dict data */ - ulint* data_len) /*!< out: dict data length */ -{ - ulint len; - const byte* field; - - /* Initialize the output values */ - *id = ULINT_UNDEFINED; - *name = NULL; - *data = NULL; - *data_len = 0; - - if (UNIV_UNLIKELY(rec_get_deleted_flag(rec, 0))) { - return("delete-marked record in SYS_ZIP_DICT"); - } - - if (UNIV_UNLIKELY( - rec_get_n_fields_old(rec)!= DICT_NUM_FIELDS__SYS_ZIP_DICT)) { - return("wrong number of columns in SYS_ZIP_DICT record"); - } - - field = rec_get_nth_field_old( - rec, DICT_FLD__SYS_ZIP_DICT__ID, &len); - if (UNIV_UNLIKELY(len != DICT_FLD_LEN_SPACE)) { - goto err_len; - } - *id = mach_read_from_4(field); - - rec_get_nth_field_offs_old( - rec, DICT_FLD__SYS_ZIP_DICT__DB_TRX_ID, &len); - if (UNIV_UNLIKELY(len != DATA_TRX_ID_LEN && len != UNIV_SQL_NULL)) { - goto err_len; - } - - rec_get_nth_field_offs_old( - rec, DICT_FLD__SYS_ZIP_DICT__DB_ROLL_PTR, &len); - if (UNIV_UNLIKELY(len != DATA_ROLL_PTR_LEN && len != UNIV_SQL_NULL)) { - goto err_len; - } - - field = rec_get_nth_field_old( - rec, DICT_FLD__SYS_ZIP_DICT__NAME, &len); - if (UNIV_UNLIKELY(len == 0 || len == UNIV_SQL_NULL)) { - goto err_len; - } - *name = mem_heap_strdupl(heap, (char*) field, len); - - field = rec_get_nth_field_old( - rec, DICT_FLD__SYS_ZIP_DICT__DATA, &len); - if (UNIV_UNLIKELY(len == UNIV_SQL_NULL)) { - goto err_len; - } - - if (rec_get_1byte_offs_flag(rec) == 0 && - rec_2_is_field_extern(rec, DICT_FLD__SYS_ZIP_DICT__DATA)) { - ut_a(len >= BTR_EXTERN_FIELD_REF_SIZE); - - if (UNIV_UNLIKELY - (!memcmp(field + len - BTR_EXTERN_FIELD_REF_SIZE, - field_ref_zero, - BTR_EXTERN_FIELD_REF_SIZE))) { - goto err_len; - } - *data = reinterpret_cast<char*>( - btr_copy_externally_stored_field(data_len, field, - zip_size, len, heap, 0)); - } - else { - *data_len = len; - *data = static_cast<char*>(mem_heap_dup(heap, field, len)); - } - - return(NULL); - -err_len: - return("incorrect column length in SYS_ZIP_DICT"); -} - -/** This function parses a SYS_ZIP_DICT_COLS record, extracts necessary -information from the record and returns to caller. -@return error message, or NULL on success */ -UNIV_INTERN -const char* -dict_process_sys_zip_dict_cols( - mem_heap_t* heap, /*!< in/out: heap memory */ - const rec_t* rec, /*!< in: current SYS_ZIP_DICT rec */ - ulint* table_id, /*!< out: table id */ - ulint* column_pos, /*!< out: column position */ - ulint* dict_id) /*!< out: dict id */ -{ - ulint len; - const byte* field; - - /* Initialize the output values */ - *table_id = ULINT_UNDEFINED; - *column_pos = ULINT_UNDEFINED; - *dict_id = ULINT_UNDEFINED; - - if (UNIV_UNLIKELY(rec_get_deleted_flag(rec, 0))) { - return("delete-marked record in SYS_ZIP_DICT_COLS"); - } - - if (UNIV_UNLIKELY(rec_get_n_fields_old(rec) != - DICT_NUM_FIELDS__SYS_ZIP_DICT_COLS)) { - return("wrong number of columns in SYS_ZIP_DICT_COLS" - " record"); - } - - field = rec_get_nth_field_old( - rec, DICT_FLD__SYS_ZIP_DICT_COLS__TABLE_ID, &len); - if (UNIV_UNLIKELY(len != DICT_FLD_LEN_SPACE)) { -err_len: - return("incorrect column length in SYS_ZIP_DICT_COLS"); - } - *table_id = mach_read_from_4(field); - - field = rec_get_nth_field_old( - rec, DICT_FLD__SYS_ZIP_DICT_COLS__COLUMN_POS, &len); - if (UNIV_UNLIKELY(len != DICT_FLD_LEN_SPACE)) { - goto err_len; - } - *column_pos = mach_read_from_4(field); - - rec_get_nth_field_offs_old( - rec, DICT_FLD__SYS_ZIP_DICT_COLS__DB_TRX_ID, &len); - if (UNIV_UNLIKELY(len != DATA_TRX_ID_LEN && len != UNIV_SQL_NULL)) { - goto err_len; - } - - rec_get_nth_field_offs_old( - rec, DICT_FLD__SYS_ZIP_DICT_COLS__DB_ROLL_PTR, &len); - if (UNIV_UNLIKELY(len != DATA_ROLL_PTR_LEN && len != UNIV_SQL_NULL)) { - goto err_len; - } - - field = rec_get_nth_field_old( - rec, DICT_FLD__SYS_ZIP_DICT_COLS__DICT_ID, &len); - if (UNIV_UNLIKELY(len != DICT_FLD_LEN_SPACE)) { - goto err_len; - } - *dict_id = mach_read_from_4(field); - - return(NULL); -} /********************************************************************//** Determine the flags of a table as stored in SYS_TABLES.TYPE and N_COLS. @return ULINT_UNDEFINED if error, else a valid dict_table_t::flags. */ diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 66d2ae4f9de..ea6a08fc156 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -4,7 +4,7 @@ Copyright (c) 2000, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, 2009 Google Inc. Copyright (c) 2009, Percona Inc. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -107,14 +107,6 @@ this program; if not, write to the Free Software Foundation, Inc., #define thd_get_trx_isolation(X) ((enum_tx_isolation)thd_tx_isolation(X)) -#ifndef HAVE_PERCONA_COMPRESSED_COLUMNS -#define COLUMN_FORMAT_TYPE_COMPRESSED 0xBADF00D -#define SQLCOM_CREATE_COMPRESSION_DICTIONARY 0xDECAF -#define SQLCOM_DROP_COMPRESSION_DICTIONARY 0xC0FFEE -#define ER_COMPRESSION_DICTIONARY_DOES_NOT_EXIST 0xDEADFACE -const static LEX_CSTRING null_lex_cstr={0,0}; -#endif - #ifdef MYSQL_DYNAMIC_PLUGIN #define tc_size 400 #define tdc_size 400 @@ -1514,30 +1506,6 @@ normalize_table_name_low( ibool set_lower_case); /* in: TRUE if we want to set name to lower case */ -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS -/** Creates a new compression dictionary. */ -static -handler_create_zip_dict_result -innobase_create_zip_dict( - handlerton* hton, /*!< in: innobase handlerton */ - THD* thd, /*!< in: handle to the MySQL thread */ - const char* name, /*!< in: zip dictionary name */ - ulint* name_len, - /*!< in/out: zip dictionary name length */ - const char* data, /*!< in: zip dictionary data */ - ulint* data_len); - /*!< in/out: zip dictionary data length */ - -/** Drops a existing compression dictionary. */ -static -handler_drop_zip_dict_result -innobase_drop_zip_dict( - handlerton* hton, /*!< in: innobase handlerton */ - THD* thd, /*!< in: handle to the MySQL thread */ - const char* name, /*!< in: zip dictionary name */ - ulint* name_len); - /*!< in/out: zip dictionary name length */ -#endif /*************************************************************//** Checks if buffer pool is big enough to enable backoff algorithm. InnoDB empty free list algorithm backoff requires free pages @@ -2411,11 +2379,16 @@ innobase_get_stmt( THD* thd, /*!< in: MySQL thread handle */ size_t* length) /*!< out: length of the SQL statement */ { - LEX_STRING* stmt; - - stmt = thd_query_string(thd); - *length = stmt->length; - return(stmt->str); + const char* query = NULL; + LEX_STRING *stmt = NULL; + if (thd) { + stmt = thd_query_string(thd); + if (stmt) { + *length = stmt->length; + query = stmt->str; + } + } + return (query); } /**********************************************************************//** @@ -3652,10 +3625,6 @@ innobase_init( innobase_hton->wsrep_fake_trx_id=wsrep_fake_trx_id; #endif /* WITH_WSREP */ -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS - innobase_hton->create_zip_dict = innobase_create_zip_dict; - innobase_hton->drop_zip_dict = innobase_drop_zip_dict; -#endif ut_a(DATA_MYSQL_TRUE_VARCHAR == (ulint)MYSQL_TYPE_VARCHAR); #ifndef DBUG_OFF @@ -4349,90 +4318,6 @@ innobase_purge_changed_page_bitmaps( return (my_bool)log_online_purge_changed_page_bitmaps(lsn); } -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS -/** Creates a new compression dictionary. */ -static -handler_create_zip_dict_result -innobase_create_zip_dict( - handlerton* hton, /*!< in: innobase handlerton */ - THD* thd, /*!< in: handle to the MySQL thread */ - const char* name, /*!< in: zip dictionary name */ - ulint* name_len, - /*!< in/out: zip dictionary name length */ - const char* data, /*!< in: zip dictionary data */ - ulint* data_len) - /*!< in/out: zip dictionary data length */ -{ - handler_create_zip_dict_result result = - HA_CREATE_ZIP_DICT_UNKNOWN_ERROR; - - DBUG_ENTER("innobase_create_zip_dict"); - DBUG_ASSERT(hton == innodb_hton_ptr); - - if (UNIV_UNLIKELY(high_level_read_only)) { - DBUG_RETURN(HA_CREATE_ZIP_DICT_READ_ONLY); - } - - if (UNIV_UNLIKELY(*name_len > ZIP_DICT_MAX_NAME_LENGTH)) { - *name_len = ZIP_DICT_MAX_NAME_LENGTH; - DBUG_RETURN(HA_CREATE_ZIP_DICT_NAME_TOO_LONG); - } - - if (UNIV_UNLIKELY(*data_len > ZIP_DICT_MAX_DATA_LENGTH)) { - *data_len = ZIP_DICT_MAX_DATA_LENGTH; - DBUG_RETURN(HA_CREATE_ZIP_DICT_DATA_TOO_LONG); - } - - switch (dict_create_zip_dict(name, *name_len, data, *data_len)) { - case DB_SUCCESS: - result = HA_CREATE_ZIP_DICT_OK; - break; - case DB_DUPLICATE_KEY: - result = HA_CREATE_ZIP_DICT_ALREADY_EXISTS; - break; - default: - ut_ad(0); - result = HA_CREATE_ZIP_DICT_UNKNOWN_ERROR; - } - DBUG_RETURN(result); -} - -/** Drops a existing compression dictionary. */ -static -handler_drop_zip_dict_result -innobase_drop_zip_dict( - handlerton* hton, /*!< in: innobase handlerton */ - THD* thd, /*!< in: handle to the MySQL thread */ - const char* name, /*!< in: zip dictionary name */ - ulint* name_len) - /*!< in/out: zip dictionary name length */ -{ - handler_drop_zip_dict_result result = HA_DROP_ZIP_DICT_UNKNOWN_ERROR; - - DBUG_ENTER("innobase_drop_zip_dict"); - DBUG_ASSERT(hton == innodb_hton_ptr); - - if (UNIV_UNLIKELY(high_level_read_only)) { - DBUG_RETURN(HA_DROP_ZIP_DICT_READ_ONLY); - } - - switch (dict_drop_zip_dict(name, *name_len)) { - case DB_SUCCESS: - result = HA_DROP_ZIP_DICT_OK; - break; - case DB_RECORD_NOT_FOUND: - result = HA_DROP_ZIP_DICT_DOES_NOT_EXIST; - break; - case DB_ROW_IS_REFERENCED: - result = HA_DROP_ZIP_DICT_IS_REFERENCED; - break; - default: - ut_ad(0); - result = HA_DROP_ZIP_DICT_UNKNOWN_ERROR; - } - DBUG_RETURN(result); -} -#endif /*****************************************************************//** Check whether this is a fake change transaction. @return TRUE if a fake change transaction */ @@ -6066,88 +5951,6 @@ func_exit: DBUG_RETURN(ret); } -/** This function checks if all the compression dictionaries referenced -in table->fields exist in SYS_ZIP_DICT InnoDB system table. -@return true if all referenced dictionaries exist */ -UNIV_INTERN -bool -innobase_check_zip_dicts( - const TABLE* table, /*!< in: table in MySQL data - dictionary */ - ulint* dict_ids, /*!< out: identified zip dict ids - (at least n_fields long) */ - trx_t* trx, /*!< in: transaction */ - const char** err_dict_name) /*!< out: the name of the - zip_dict which does not exist. */ -{ - DBUG_ENTER("innobase_check_zip_dicts"); - - bool res = true; -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS - dberr_t err = DB_SUCCESS; - const size_t n_fields = table->s->fields; - - Field* field_ptr; - for (size_t field_idx = 0; err == DB_SUCCESS && field_idx < n_fields; - ++field_idx) - { - field_ptr = table->field[field_idx]; - if (field_ptr->has_associated_compression_dictionary()) { - err = dict_create_get_zip_dict_id_by_name( - field_ptr->zip_dict_name.str, - field_ptr->zip_dict_name.length, - &dict_ids[field_idx], - trx); - ut_a(err == DB_SUCCESS || err == DB_RECORD_NOT_FOUND); - } - else { - dict_ids[field_idx] = ULINT_UNDEFINED; - } - - } - - if (err != DB_SUCCESS) { - res = false; - *err_dict_name = field_ptr->zip_dict_name.str; - } - -#endif - DBUG_RETURN(res); -} - -/** This function creates compression dictionary references in -SYS_ZIP_DICT_COLS InnoDB system table for table_id based on info -in table->fields and provided zip dict ids. */ -UNIV_INTERN -void -innobase_create_zip_dict_references( - const TABLE* table, /*!< in: table in MySQL data - dictionary */ - table_id_t ib_table_id, /*!< in: table ID in Innodb data - dictionary */ - ulint* zip_dict_ids, /*!< in: zip dict ids - (at least n_fields long) */ - trx_t* trx) /*!< in: transaction */ -{ - DBUG_ENTER("innobase_create_zip_dict_references"); - - dberr_t err = DB_SUCCESS; - const size_t n_fields = table->s->fields; - - for (size_t field_idx = 0; err == DB_SUCCESS && field_idx < n_fields; - ++field_idx) - { - if (zip_dict_ids[field_idx] != ULINT_UNDEFINED) { - err = dict_create_add_zip_dict_reference(ib_table_id, - table->field[field_idx]->field_index, - zip_dict_ids[field_idx], trx); - ut_a(err == DB_SUCCESS); - } - } - - DBUG_VOID_RETURN; -} - /*******************************************************************//** This function uses index translation table to quickly locate the requested index structure. @@ -7841,16 +7644,7 @@ ha_innobase::store_key_val_for_row( blob_data = row_mysql_read_blob_ref(&blob_len, (byte*) (record + (ulint) get_field_offset(table, field)), - (ulint) field->pack_length(), -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS - field->column_format() == - COLUMN_FORMAT_TYPE_COMPRESSED, - reinterpret_cast<const byte*>( - field->zip_dict_data.str), - field->zip_dict_data.length, prebuilt); -#else - 0, 0, 0, prebuilt); -#endif + (ulint) field->pack_length()); true_len = blob_len; @@ -8074,7 +7868,66 @@ build_template_field( UNIV_MEM_INVALID(templ, sizeof *templ); templ->col_no = i; templ->clust_rec_field_no = dict_col_get_clust_pos(col, clust_index); - ut_a(templ->clust_rec_field_no != ULINT_UNDEFINED); + + /* If clustered index record field is not found, lets print out + field names and all the rest to understand why field is not found. */ + if (templ->clust_rec_field_no == ULINT_UNDEFINED) { + const char* tb_col_name = dict_table_get_col_name(clust_index->table, i); + dict_field_t* field=NULL; + size_t size = 0; + + for(ulint j=0; j < clust_index->n_user_defined_cols; j++) { + dict_field_t* ifield = &(clust_index->fields[j]); + if (ifield && !memcmp(tb_col_name, ifield->name, + strlen(tb_col_name))) { + field = ifield; + break; + } + } + + ib_logf(IB_LOG_LEVEL_INFO, + "Looking for field %lu name %s from table %s", + i, + (tb_col_name ? tb_col_name : "NULL"), + clust_index->table->name); + + + for(ulint j=0; j < clust_index->n_user_defined_cols; j++) { + dict_field_t* ifield = &(clust_index->fields[j]); + ib_logf(IB_LOG_LEVEL_INFO, + "InnoDB Table %s field %lu name %s", + clust_index->table->name, + j, + (ifield ? ifield->name : "NULL")); + } + + for(ulint j=0; j < table->s->stored_fields; j++) { + ib_logf(IB_LOG_LEVEL_INFO, + "MySQL table %s field %lu name %s", + table->s->table_name.str, + j, + table->field[j]->field_name); + } + + ib_logf(IB_LOG_LEVEL_ERROR, + "Clustered record field for column %lu" + " not found table n_user_defined %d" + " index n_user_defined %d" + " InnoDB table %s field name %s" + " MySQL table %s field name %s n_fields %d" + " query %s", + i, + clust_index->n_user_defined_cols, + clust_index->table->n_cols - DATA_N_SYS_COLS, + clust_index->table->name, + (field ? field->name : "NULL"), + table->s->table_name.str, + (tb_col_name ? tb_col_name : "NULL"), + table->s->stored_fields, + innobase_get_stmt(current_thd, &size)); + + ut_a(templ->clust_rec_field_no != ULINT_UNDEFINED); + } if (dict_index_is_clust(index)) { templ->rec_field_no = templ->clust_rec_field_no; @@ -8106,14 +7959,6 @@ build_template_field( templ->mbminlen = dict_col_get_mbminlen(col); templ->mbmaxlen = dict_col_get_mbmaxlen(col); templ->is_unsigned = col->prtype & DATA_UNSIGNED; -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS - templ->compressed = (field->column_format() - == COLUMN_FORMAT_TYPE_COMPRESSED); - templ->zip_dict_data = field->zip_dict_data; -#else - templ->compressed = 0; - templ->zip_dict_data = null_lex_cstr; -#endif if (!dict_index_is_clust(index) && templ->rec_field_no == ULINT_UNDEFINED) { @@ -8769,12 +8614,6 @@ no_commit: error = row_insert_for_mysql((byte*) record, prebuilt); DEBUG_SYNC(user_thd, "ib_after_row_insert"); -#ifdef EXTENDED_FOR_USERSTAT - if (UNIV_LIKELY(error == DB_SUCCESS && !trx->fake_changes)) { - rows_changed++; - } -#endif - /* Handle duplicate key errors */ if (auto_inc_used) { ulonglong auto_inc; @@ -9022,10 +8861,8 @@ calc_row_difference( case DATA_BLOB: /* Do not compress blob column while comparing*/ - o_ptr = row_mysql_read_blob_ref(&o_len, o_ptr, o_len, - false, 0, 0, prebuilt); - n_ptr = row_mysql_read_blob_ref(&n_len, n_ptr, n_len, - false, 0, 0, prebuilt); + o_ptr = row_mysql_read_blob_ref(&o_len, o_ptr, o_len); + n_ptr = row_mysql_read_blob_ref(&n_len, n_ptr, n_len); break; @@ -9095,17 +8932,7 @@ calc_row_difference( TRUE, new_mysql_row_col, col_pack_len, - dict_table_is_comp(prebuilt->table), -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS - field->column_format() == - COLUMN_FORMAT_TYPE_COMPRESSED, - reinterpret_cast<const byte*>( - field->zip_dict_data.str), - field->zip_dict_data.length, -#else - 0, 0, 0, -#endif - prebuilt); + dict_table_is_comp(prebuilt->table)); dfield_copy(&ufield->new_val, &dfield); } else { dfield_set_null(&ufield->new_val); @@ -9441,12 +9268,6 @@ ha_innobase::update_row( } } -#ifdef EXTENDED_FOR_USERSTAT - if (UNIV_LIKELY(error == DB_SUCCESS && !trx->fake_changes)) { - rows_changed++; - } -#endif - innobase_srv_conc_exit_innodb(trx); func_exit: @@ -9543,12 +9364,6 @@ ha_innobase::delete_row( error = row_update_for_mysql((byte*) record, prebuilt); -#ifdef EXTENDED_FOR_USERSTAT - if (UNIV_LIKELY(error == DB_SUCCESS && !trx->fake_changes)) { - rows_changed++; - } -#endif - innobase_srv_conc_exit_innodb(trx); /* Tell the InnoDB server that there might be work for @@ -9919,11 +9734,6 @@ ha_innobase::index_read( srv_stats.n_rows_read.add( (size_t) prebuilt->trx->id, 1); } -#ifdef EXTENDED_FOR_USERSTAT - rows_read++; - if (active_index < MAX_KEY) - index_rows_read[active_index]++; -#endif break; case DB_RECORD_NOT_FOUND: error = HA_ERR_KEY_NOT_FOUND; @@ -10220,11 +10030,6 @@ ha_innobase::general_fetch( error = 0; table->status = 0; srv_stats.n_rows_read.add((size_t) prebuilt->trx->id, 1); -#ifdef EXTENDED_FOR_USERSTAT - rows_read++; - if (active_index < MAX_KEY) - index_rows_read[active_index]++; -#endif break; case DB_RECORD_NOT_FOUND: error = HA_ERR_END_OF_FILE; @@ -11338,7 +11143,6 @@ create_table_def( ulint unsigned_type; ulint binary_type; ulint long_true_varchar; - ulint compressed; ulint charset_no; ulint i; ulint doc_id_col = 0; @@ -11488,13 +11292,6 @@ create_table_def( } } - /* Check if the the field has COMPRESSED attribute */ - compressed = 0; - if (field->column_format() == - COLUMN_FORMAT_TYPE_COMPRESSED) { - compressed = DATA_COMPRESSED; - } - /* First check whether the column to be added has a system reserved name. */ if (dict_col_name_is_reserved(field->field_name)){ @@ -11515,8 +11312,7 @@ err_col: dtype_form_prtype( (ulint) field->type() | nulls_allowed | unsigned_type - | binary_type | long_true_varchar - | compressed, + | binary_type | long_true_varchar, charset_no), col_len); } @@ -12346,9 +12142,6 @@ ha_innobase::create( const char* stmt; size_t stmt_len; - mem_heap_t* heap = 0; - ulint* zip_dict_ids = 0; - DBUG_ENTER("ha_innobase::create"); DBUG_ASSERT(thd != NULL); @@ -12439,19 +12232,6 @@ ha_innobase::create( row_mysql_lock_data_dictionary(trx); - heap = mem_heap_create(form->s->fields * sizeof(ulint)); - zip_dict_ids = static_cast<ulint*>( - mem_heap_alloc(heap, form->s->fields * sizeof(ulint))); - - const char* err_zip_dict_name = 0; - if (!innobase_check_zip_dicts(form, zip_dict_ids, - trx, &err_zip_dict_name)) { - error = -1; - my_error(ER_COMPRESSION_DICTIONARY_DOES_NOT_EXIST, - MYF(0), err_zip_dict_name); - goto cleanup; - } - error = create_table_def(trx, form, norm_name, temp_path, remote_path, flags, flags2); if (error) { @@ -12559,22 +12339,6 @@ ha_innobase::create( dict_table_get_all_fts_indexes(innobase_table, fts->indexes); } - /* - Adding compression dictionary <-> compressed table column links - to the SYS_ZIP_DICT_COLS table. - */ - ut_a(zip_dict_ids != 0); - { - dict_table_t* local_table = dict_table_open_on_name( - norm_name, TRUE, FALSE, DICT_ERR_IGNORE_NONE); - - ut_a(local_table); - table_id_t table_id = local_table->id; - dict_table_close(local_table, TRUE, FALSE); - innobase_create_zip_dict_references(form, - table_id, zip_dict_ids, trx); - } - stmt = innobase_get_stmt(thd, &stmt_len); if (stmt) { @@ -12691,9 +12455,6 @@ ha_innobase::create( trx_free_for_mysql(trx); - if (heap != 0) - mem_heap_free(heap); - DBUG_RETURN(0); cleanup: @@ -12703,9 +12464,6 @@ cleanup: trx_free_for_mysql(trx); - if (heap != 0) - mem_heap_free(heap); - DBUG_RETURN(error); } @@ -14987,10 +14745,6 @@ ha_innobase::extra( row_mysql_prebuilt_free_blob_heap(prebuilt); } - if (prebuilt->compress_heap) { - row_mysql_prebuilt_free_compress_heap(prebuilt); - } - break; case HA_EXTRA_RESET_STATE: reset_template(); @@ -15042,10 +14796,6 @@ ha_innobase::reset() row_mysql_prebuilt_free_blob_heap(prebuilt); } - if (prebuilt->compress_heap) { - row_mysql_prebuilt_free_compress_heap(prebuilt); - } - reset_template(); ds_mrr.dsmrr_close(); @@ -15252,11 +15002,7 @@ ha_innobase::external_lock( && lock_type == F_WRLCK) || thd_sql_command(thd) == SQLCOM_CREATE_INDEX || thd_sql_command(thd) == SQLCOM_DROP_INDEX - || thd_sql_command(thd) == SQLCOM_DELETE - || thd_sql_command(thd) == - SQLCOM_CREATE_COMPRESSION_DICTIONARY - || thd_sql_command(thd) == - SQLCOM_DROP_COMPRESSION_DICTIONARY)) { + || thd_sql_command(thd) == SQLCOM_DELETE)) { if (thd_sql_command(thd) == SQLCOM_CREATE_TABLE) { @@ -16024,9 +15770,7 @@ ha_innobase::store_lock( && lock_type <= TL_WRITE)) || sql_command == SQLCOM_CREATE_INDEX || sql_command == SQLCOM_DROP_INDEX - || sql_command == SQLCOM_DELETE - || sql_command == SQLCOM_CREATE_COMPRESSION_DICTIONARY - || sql_command == SQLCOM_DROP_COMPRESSION_DICTIONARY)) { + || sql_command == SQLCOM_DELETE)) { ib_senderrf(trx->mysql_thd, IB_LOG_LEVEL_WARN, ER_READ_ONLY_MODE); @@ -16983,84 +16727,6 @@ ha_innobase::check_if_incompatible_data( return(COMPATIBLE_DATA_YES); } -/** This function reads zip dict-related info from SYS_ZIP_DICT -and SYS_ZIP_DICT_COLS for all columns marked with -COLUMN_FORMAT_TYPE_COMPRESSED flag and updates -zip_dict_name / zip_dict_data for those which have associated -compression dictionaries. -*/ -UNIV_INTERN -void -ha_innobase::update_field_defs_with_zip_dict_info() -{ - DBUG_ENTER("update_field_defs_with_zip_dict_info"); - ut_ad(!mutex_own(&dict_sys->mutex)); - - char norm_name[FN_REFLEN]; - normalize_table_name(norm_name, table_share->normalized_path.str); - - dict_table_t* ib_table = dict_table_open_on_name( - norm_name, FALSE, FALSE, DICT_ERR_IGNORE_NONE); - - /* if dict_table_open_on_name() returns NULL, then it means that - TABLE_SHARE is populated for a table being created and we can - skip filling zip dict info here */ - if (ib_table == 0) - DBUG_VOID_RETURN; - -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS - table_id_t ib_table_id = ib_table->id; - dict_table_close(ib_table, FALSE, FALSE); - Field* field; - for (uint i = 0; i < table_share->fields; ++i) { - field = table_share->field[i]; - if (field->column_format() == - COLUMN_FORMAT_TYPE_COMPRESSED) { - bool reference_found = false; - ulint dict_id = 0; - switch (dict_get_dictionary_id_by_key(ib_table_id, i, - &dict_id)) { - case DB_SUCCESS: - reference_found = true; - break; - case DB_RECORD_NOT_FOUND: - reference_found = false; - break; - default: - ut_error; - } - if (reference_found) { - char* local_name = 0; - ulint local_name_len = 0; - char* local_data = 0; - ulint local_data_len = 0; - if (dict_get_dictionary_info_by_id(dict_id, - &local_name, &local_name_len, - &local_data, &local_data_len) != - DB_SUCCESS) { - ut_error; - } - else { - field->zip_dict_name.str = - local_name; - field->zip_dict_name.length = - local_name_len; - field->zip_dict_data.str = - local_data; - field->zip_dict_data.length = - local_data_len; - } - } - else { - field->zip_dict_name = null_lex_cstr; - field->zip_dict_data = null_lex_cstr; - } - } - } -#endif - DBUG_VOID_RETURN; -} - /****************************************************************//** Update the system variable innodb_io_capacity_max using the "saved" value. This function is registered as a callback with MySQL. */ @@ -20799,21 +20465,6 @@ static MYSQL_SYSVAR_BOOL(use_stacktrace, srv_use_stacktrace, "Print stacktrace on long semaphore wait (off by default supported only on linux)", NULL, NULL, FALSE); -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS -static MYSQL_SYSVAR_UINT(compressed_columns_zip_level, - srv_compressed_columns_zip_level, - PLUGIN_VAR_RQCMDARG, - "Compression level used for compressed columns. 0 is no compression" - ", 1 is fastest and 9 is best compression. Default is 6.", - NULL, NULL, DEFAULT_COMPRESSION_LEVEL, 0, 9, 0); - -static MYSQL_SYSVAR_ULONG(compressed_columns_threshold, - srv_compressed_columns_threshold, - PLUGIN_VAR_RQCMDARG, - "Compress column data if its length exceeds this value. Default is 96", - NULL, NULL, 96, 1, ~0UL, 0); -#endif - static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(log_block_size), MYSQL_SYSVAR(additional_mem_pool_size), @@ -21020,10 +20671,6 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(tmpdir), MYSQL_SYSVAR(use_stacktrace), MYSQL_SYSVAR(simulate_comp_failures), -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS - MYSQL_SYSVAR(compressed_columns_zip_level), - MYSQL_SYSVAR(compressed_columns_threshold), -#endif NULL }; @@ -21046,10 +20693,6 @@ maria_declare_plugin(xtradb) i_s_xtradb_read_view, i_s_xtradb_internal_hash_tables, i_s_xtradb_rseg, -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS -i_s_xtradb_zip_dict, -i_s_xtradb_zip_dict_cols, -#endif i_s_innodb_trx, i_s_innodb_locks, i_s_innodb_lock_waits, diff --git a/storage/xtradb/handler/ha_innodb.h b/storage/xtradb/handler/ha_innodb.h index a824f4c680a..672894f6f51 100644 --- a/storage/xtradb/handler/ha_innodb.h +++ b/storage/xtradb/handler/ha_innodb.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2000, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -294,14 +294,6 @@ class ha_innobase: public handler bool check_if_supported_virtual_columns(void) { return TRUE; } - /** This function reads zip dict-related info from SYS_ZIP_DICT - and SYS_ZIP_DICT_COLS for all columns marked with - COLUMN_FORMAT_TYPE_COMPRESSED flag and updates - zip_dict_name / zip_dict_data for those which have associated - compression dictionaries. - */ - virtual void update_field_defs_with_zip_dict_info(); - private: /** Builds a 'template' to the prebuilt struct. @@ -732,31 +724,3 @@ ib_push_frm_error( TABLE* table, /*!< in: MySQL table */ ulint n_keys, /*!< in: InnoDB #keys */ bool push_warning); /*!< in: print warning ? */ - -/** This function checks if all the compression dictionaries referenced -in table->fields exist in SYS_ZIP_DICT InnoDB system table. -@return true if all referenced dictionaries exist */ -UNIV_INTERN -bool -innobase_check_zip_dicts( - const TABLE* table, /*!< in: table in MySQL data - dictionary */ - ulint* dict_ids, /*!< out: identified zip dict ids - (at least n_fields long) */ - trx_t* trx, /*!< in: transaction */ - const char** err_dict_name); /*!< out: the name of the - zip_dict which does not exist. */ - -/** This function creates compression dictionary references in -SYS_ZIP_DICT_COLS InnoDB system table for table_id based on info -in table->fields and provided zip dict ids. */ -UNIV_INTERN -void -innobase_create_zip_dict_references( - const TABLE* table, /*!< in: table in MySQL data - dictionary */ - table_id_t ib_table_id, /*!< in: table ID in Innodb data - dictionary */ - ulint* zip_dict_ids, /*!< in: zip dict ids - (at least n_fields long) */ - trx_t* trx); /*!< in: transaction */ diff --git a/storage/xtradb/handler/handler0alter.cc b/storage/xtradb/handler/handler0alter.cc index 8299b4dafe1..2eea4aa4a1f 100644 --- a/storage/xtradb/handler/handler0alter.cc +++ b/storage/xtradb/handler/handler0alter.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2005, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -21,11 +22,6 @@ this program; if not, write to the Free Software Foundation, Inc., Smart ALTER TABLE *******************************************************/ -#ifndef HAVE_PERCONA_COMPRESSED_COLUMNS -#define COLUMN_FORMAT_TYPE_COMPRESSED 0xBADF00D -#define ER_COMPRESSION_DICTIONARY_DOES_NOT_EXIST 0xDEADFACE -#endif - #include <my_global.h> #include <unireg.h> #include <mysqld_error.h> @@ -1189,15 +1185,6 @@ innobase_col_to_mysql( field->reset(); if (field->type() == MYSQL_TYPE_VARCHAR) { - if (field->column_format() == - COLUMN_FORMAT_TYPE_COMPRESSED) { - /* Skip compressed varchar column when - reporting an erroneous row - during index creation or table rebuild. */ - field->set_null(); - break; - } - /* This is a >= 5.0.3 type true VARCHAR. Store the length of the data to the first byte or the first two bytes of dest. */ @@ -2496,14 +2483,7 @@ innobase_build_col_map_add( byte* buf = static_cast<byte*>(mem_heap_alloc(heap, size)); row_mysql_store_col_in_innobase_format( - dfield, buf, TRUE, field->ptr, size, comp, -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS - field->column_format() == COLUMN_FORMAT_TYPE_COMPRESSED, - reinterpret_cast<const byte*>(field->zip_dict_data.str), - field->zip_dict_data.length, prebuilt); -#else - 0,0,0, prebuilt); -#endif + dfield, buf, TRUE, field->ptr, size, comp); } /** Construct the translation table for reordering, dropping or @@ -2757,7 +2737,6 @@ prepare_inplace_alter_table_dict( ulint num_fts_index; ha_innobase_inplace_ctx*ctx; uint sql_idx; - ulint* zip_dict_ids = 0; DBUG_ENTER("prepare_inplace_alter_table_dict"); @@ -2894,18 +2873,6 @@ prepare_inplace_alter_table_dict( ulint n_cols; dtuple_t* add_cols; - zip_dict_ids = static_cast<ulint*>( - mem_heap_alloc(ctx->heap, - altered_table->s->fields * sizeof(ulint))); - - const char* err_zip_dict_name = 0; - if (!innobase_check_zip_dicts(altered_table, zip_dict_ids, - ctx->trx, &err_zip_dict_name)) { - my_error(ER_COMPRESSION_DICTIONARY_DOES_NOT_EXIST, - MYF(0), err_zip_dict_name); - goto new_clustered_failed; - } - if (innobase_check_foreigns( ha_alter_info, altered_table, old_table, user_table, ctx->drop_fk, ctx->num_to_drop_fk)) { @@ -3012,12 +2979,6 @@ prepare_inplace_alter_table_dict( } } - if (field->column_format() == - COLUMN_FORMAT_TYPE_COMPRESSED) { - field_type |= DATA_COMPRESSED; - } - - if (dict_col_name_is_reserved(field->field_name)) { dict_mem_table_free(ctx->new_table); my_error(ER_WRONG_COLUMN_NAME, MYF(0), @@ -3275,15 +3236,6 @@ op_ok: DBUG_ASSERT(error == DB_SUCCESS); - /* - Adding compression dictionary <-> compressed table column links - to the SYS_ZIP_DICT_COLS table. - */ - if (zip_dict_ids != 0) { - innobase_create_zip_dict_references(altered_table, - ctx->trx->table_id, zip_dict_ids, ctx->trx); - } - /* Commit the data dictionary transaction in order to release the table locks on the system tables. This means that if MySQL crashes while creating a new primary key inside @@ -3956,7 +3908,7 @@ check_if_can_drop_indexes: index->name, TRUE); my_error(ER_INDEX_CORRUPT, MYF(0), index_name); - DBUG_RETURN(true); + goto err_exit; } } } diff --git a/storage/xtradb/handler/xtradb_i_s.cc b/storage/xtradb/handler/xtradb_i_s.cc index eb6637dad03..84c1e5852bc 100644 --- a/storage/xtradb/handler/xtradb_i_s.cc +++ b/storage/xtradb/handler/xtradb_i_s.cc @@ -2,6 +2,7 @@ Copyright (c) 2007, 2012, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2010-2012, Percona Inc. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -36,7 +37,6 @@ this program; if not, write to the Free Software Foundation, Inc., #include <btr0sea.h> /* btr_search_sys */ #include <log0recv.h> /* recv_sys */ #include <fil0fil.h> -#include <dict0crea.h> /* for ZIP_DICT_MAX_* constants */ /* for XTRADB_RSEG table */ #include "trx0trx.h" /* for TRX_QUE_STATE_STR_MAX_LEN */ @@ -629,331 +629,3 @@ UNIV_INTERN struct st_mysql_plugin i_s_xtradb_rseg = STRUCT_FLD(version_info, INNODB_VERSION_STR), STRUCT_FLD(maturity, MariaDB_PLUGIN_MATURITY_STABLE), }; - - -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS -/************************************************************************/ -enum zip_dict_field_type -{ - zip_dict_field_id, - zip_dict_field_name, - zip_dict_field_zip_dict -}; - -static ST_FIELD_INFO xtradb_sys_zip_dict_fields_info[] = -{ - { STRUCT_FLD(field_name, "id"), - STRUCT_FLD(field_length, MY_INT64_NUM_DECIMAL_DIGITS), - STRUCT_FLD(field_type, MYSQL_TYPE_LONGLONG), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, MY_I_S_UNSIGNED), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE) }, - - { STRUCT_FLD(field_name, "name"), - STRUCT_FLD(field_length, ZIP_DICT_MAX_NAME_LENGTH), - STRUCT_FLD(field_type, MYSQL_TYPE_STRING), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, 0), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE) }, - - { STRUCT_FLD(field_name, "zip_dict"), - STRUCT_FLD(field_length, ZIP_DICT_MAX_DATA_LENGTH), - STRUCT_FLD(field_type, MYSQL_TYPE_BLOB), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, 0), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE) }, - - END_OF_ST_FIELD_INFO -}; - -/** Function to fill INFORMATION_SCHEMA.XTRADB_ZIP_DICT with information -collected by scanning SYS_ZIP_DICT table. -@return 0 on success */ -static -int -xtradb_i_s_dict_fill_sys_zip_dict( - THD* thd, /*!< in: thread */ - ulint id, /*!< in: dict ID */ - const char* name, /*!< in: dict name */ - const char* data, /*!< in: dict data */ - ulint data_len, /*!< in: dict data length */ - TABLE* table_to_fill) /*!< in/out: fill this table */ -{ - DBUG_ENTER("xtradb_i_s_dict_fill_sys_zip_dict"); - - Field** fields = table_to_fill->field; - - OK(field_store_ulint(fields[zip_dict_field_id], id)); - OK(field_store_string(fields[zip_dict_field_name], name)); - OK(field_store_blob(fields[zip_dict_field_zip_dict], data, - data_len)); - - OK(schema_table_store_record(thd, table_to_fill)); - - DBUG_RETURN(0); -} - -/** Function to populate INFORMATION_SCHEMA.XTRADB_ZIP_DICT table. -Loop through each record in SYS_ZIP_DICT, and extract the column -information and fill the INFORMATION_SCHEMA.XTRADB_ZIP_DICT table. -@return 0 on success */ -static -int -xtradb_i_s_sys_zip_dict_fill_table( - THD* thd, /*!< in: thread */ - TABLE_LIST* tables, /*!< in/out: tables to fill */ - Item* ) /*!< in: condition (not used) */ -{ - btr_pcur_t pcur; - const rec_t* rec; - mem_heap_t* heap; - mtr_t mtr; - - DBUG_ENTER("xtradb_i_s_sys_zip_dict_fill_table"); - RETURN_IF_INNODB_NOT_STARTED(tables->schema_table_name); - - /* deny access to user without SUPER_ACL privilege */ - if (check_global_access(thd, SUPER_ACL)) { - DBUG_RETURN(0); - } - - heap = mem_heap_create(1000); - mutex_enter(&dict_sys->mutex); - mtr_start(&mtr); - - rec = dict_startscan_system(&pcur, &mtr, SYS_ZIP_DICT); - ulint zip_size = dict_table_zip_size(pcur.btr_cur.index->table); - - while (rec) { - const char* err_msg; - ulint id; - const char* name; - const char* data; - ulint data_len; - - /* Extract necessary information from a SYS_ZIP_DICT row */ - err_msg = dict_process_sys_zip_dict( - heap, zip_size, rec, &id, &name, &data, &data_len); - - mtr_commit(&mtr); - mutex_exit(&dict_sys->mutex); - - if (!err_msg) { - xtradb_i_s_dict_fill_sys_zip_dict( - thd, id, name, data, data_len, - tables->table); - } else { - push_warning_printf(thd, - Sql_condition::WARN_LEVEL_WARN, - ER_CANT_FIND_SYSTEM_REC, "%s", err_msg); - } - - mem_heap_empty(heap); - - /* Get the next record */ - mutex_enter(&dict_sys->mutex); - mtr_start(&mtr); - rec = dict_getnext_system(&pcur, &mtr); - } - - mtr_commit(&mtr); - mutex_exit(&dict_sys->mutex); - mem_heap_free(heap); - - DBUG_RETURN(0); -} - -static int i_s_xtradb_zip_dict_init(void* p) -{ - DBUG_ENTER("i_s_xtradb_zip_dict_init"); - - ST_SCHEMA_TABLE* schema = static_cast<ST_SCHEMA_TABLE*>(p); - - schema->fields_info = xtradb_sys_zip_dict_fields_info; - schema->fill_table = xtradb_i_s_sys_zip_dict_fill_table; - - DBUG_RETURN(0); -} - -UNIV_INTERN struct st_mysql_plugin i_s_xtradb_zip_dict = -{ - STRUCT_FLD(type, MYSQL_INFORMATION_SCHEMA_PLUGIN), - STRUCT_FLD(info, &i_s_info), - STRUCT_FLD(name, "XTRADB_ZIP_DICT"), - STRUCT_FLD(author, PLUGIN_AUTHOR), - STRUCT_FLD(descr, "InnoDB compression dictionaries information"), - STRUCT_FLD(license, PLUGIN_LICENSE_GPL), - STRUCT_FLD(init, i_s_xtradb_zip_dict_init), - STRUCT_FLD(deinit, i_s_common_deinit), - STRUCT_FLD(version, INNODB_VERSION_SHORT), - STRUCT_FLD(status_vars, NULL), - STRUCT_FLD(system_vars, NULL), - STRUCT_FLD(__reserved1, NULL), - STRUCT_FLD(flags, 0UL), -}; - -enum zip_dict_cols_field_type -{ - zip_dict_cols_field_table_id, - zip_dict_cols_field_column_pos, - zip_dict_cols_field_dict_id -}; - -static ST_FIELD_INFO xtradb_sys_zip_dict_cols_fields_info[] = -{ - { STRUCT_FLD(field_name, "table_id"), - STRUCT_FLD(field_length, MY_INT64_NUM_DECIMAL_DIGITS), - STRUCT_FLD(field_type, MYSQL_TYPE_LONGLONG), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, MY_I_S_UNSIGNED), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE) }, - - { STRUCT_FLD(field_name, "column_pos"), - STRUCT_FLD(field_length, MY_INT64_NUM_DECIMAL_DIGITS), - STRUCT_FLD(field_type, MYSQL_TYPE_LONGLONG), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, MY_I_S_UNSIGNED), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE) }, - - { STRUCT_FLD(field_name, "dict_id"), - STRUCT_FLD(field_length, MY_INT64_NUM_DECIMAL_DIGITS), - STRUCT_FLD(field_type, MYSQL_TYPE_LONGLONG), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, MY_I_S_UNSIGNED), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE) }, - - END_OF_ST_FIELD_INFO -}; - -/** Function to fill INFORMATION_SCHEMA.XTRADB_ZIP_DICT_COLS with information -collected by scanning SYS_ZIP_DICT_COLS table. -@return 0 on success */ -static -int -xtradb_i_s_dict_fill_sys_zip_dict_cols( - THD* thd, /*!< in: thread */ - ulint table_id, /*!< in: table ID */ - ulint column_pos, /*!< in: column position */ - ulint dict_id, /*!< in: dict ID */ - TABLE* table_to_fill) /*!< in/out: fill this table */ -{ - DBUG_ENTER("xtradb_i_s_dict_fill_sys_zip_dict_cols"); - - Field** fields = table_to_fill->field; - - OK(field_store_ulint(fields[zip_dict_cols_field_table_id], - table_id)); - OK(field_store_ulint(fields[zip_dict_cols_field_column_pos], - column_pos)); - OK(field_store_ulint(fields[zip_dict_cols_field_dict_id], - dict_id)); - - OK(schema_table_store_record(thd, table_to_fill)); - - DBUG_RETURN(0); -} - -/** Function to populate INFORMATION_SCHEMA.XTRADB_ZIP_DICT_COLS table. -Loop through each record in SYS_ZIP_DICT_COLS, and extract the column -information and fill the INFORMATION_SCHEMA.XTRADB_ZIP_DICT_COLS table. -@return 0 on success */ -static -int -xtradb_i_s_sys_zip_dict_cols_fill_table( - THD* thd, /*!< in: thread */ - TABLE_LIST* tables, /*!< in/out: tables to fill */ - Item* ) /*!< in: condition (not used) */ -{ - btr_pcur_t pcur; - const rec_t* rec; - mem_heap_t* heap; - mtr_t mtr; - - DBUG_ENTER("xtradb_i_s_sys_zip_dict_cols_fill_table"); - RETURN_IF_INNODB_NOT_STARTED(tables->schema_table_name); - - /* deny access to user without SUPER_ACL privilege */ - if (check_global_access(thd, SUPER_ACL)) { - DBUG_RETURN(0); - } - - heap = mem_heap_create(1000); - mutex_enter(&dict_sys->mutex); - mtr_start(&mtr); - - rec = dict_startscan_system(&pcur, &mtr, SYS_ZIP_DICT_COLS); - - while (rec) { - const char* err_msg; - ulint table_id; - ulint column_pos; - ulint dict_id; - - /* Extract necessary information from a SYS_ZIP_DICT_COLS - row */ - err_msg = dict_process_sys_zip_dict_cols( - heap, rec, &table_id, &column_pos, &dict_id); - - mtr_commit(&mtr); - mutex_exit(&dict_sys->mutex); - - if (!err_msg) { - xtradb_i_s_dict_fill_sys_zip_dict_cols( - thd, table_id, column_pos, dict_id, - tables->table); - } else { - push_warning_printf(thd, - Sql_condition::WARN_LEVEL_WARN, - ER_CANT_FIND_SYSTEM_REC, "%s", err_msg); - } - - mem_heap_empty(heap); - - /* Get the next record */ - mutex_enter(&dict_sys->mutex); - mtr_start(&mtr); - rec = dict_getnext_system(&pcur, &mtr); - } - - mtr_commit(&mtr); - mutex_exit(&dict_sys->mutex); - mem_heap_free(heap); - - DBUG_RETURN(0); -} - -static int i_s_xtradb_zip_dict_cols_init(void* p) -{ - DBUG_ENTER("i_s_xtradb_zip_dict_cols_init"); - - ST_SCHEMA_TABLE* schema = static_cast<ST_SCHEMA_TABLE*>(p); - - schema->fields_info = xtradb_sys_zip_dict_cols_fields_info; - schema->fill_table = xtradb_i_s_sys_zip_dict_cols_fill_table; - - DBUG_RETURN(0); -} - -UNIV_INTERN struct st_mysql_plugin i_s_xtradb_zip_dict_cols = -{ - STRUCT_FLD(type, MYSQL_INFORMATION_SCHEMA_PLUGIN), - STRUCT_FLD(info, &i_s_info), - STRUCT_FLD(name, "XTRADB_ZIP_DICT_COLS"), - STRUCT_FLD(author, PLUGIN_AUTHOR), - STRUCT_FLD(descr, "InnoDB compressed columns information"), - STRUCT_FLD(license, PLUGIN_LICENSE_GPL), - STRUCT_FLD(init, i_s_xtradb_zip_dict_cols_init), - STRUCT_FLD(deinit, i_s_common_deinit), - STRUCT_FLD(version, INNODB_VERSION_SHORT), - STRUCT_FLD(status_vars, NULL), - STRUCT_FLD(system_vars, NULL), - STRUCT_FLD(__reserved1, NULL), - STRUCT_FLD(flags, 0UL), -}; -#endif diff --git a/storage/xtradb/handler/xtradb_i_s.h b/storage/xtradb/handler/xtradb_i_s.h index 905d84587af..994bc11c1b8 100644 --- a/storage/xtradb/handler/xtradb_i_s.h +++ b/storage/xtradb/handler/xtradb_i_s.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2010-2012, Percona Inc. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -22,7 +23,5 @@ this program; if not, write to the Free Software Foundation, Inc., extern struct st_mysql_plugin i_s_xtradb_read_view; extern struct st_mysql_plugin i_s_xtradb_internal_hash_tables; extern struct st_mysql_plugin i_s_xtradb_rseg; -extern struct st_mysql_plugin i_s_xtradb_zip_dict; -extern struct st_mysql_plugin i_s_xtradb_zip_dict_cols; #endif /* XTRADB_I_S_H */ diff --git a/storage/xtradb/include/data0type.h b/storage/xtradb/include/data0type.h index f269c266efb..df6b6a41c11 100644 --- a/storage/xtradb/include/data0type.h +++ b/storage/xtradb/include/data0type.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2012, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -170,9 +171,6 @@ be less than 256 */ type when the column is true VARCHAR where MySQL uses 2 bytes to store the data len; for shorter VARCHARs MySQL uses only 1 byte */ -#define DATA_COMPRESSED 16384 /* this is ORed to the precise data - type when the column has COLUMN_FORMAT = - COMPRESSED attribute*/ /*-------------------------------------------*/ /* This many bytes we need to store the type information affecting the @@ -503,17 +501,6 @@ dtype_print( /*========*/ const dtype_t* type); /*!< in: type */ -/** -Calculates the number of extra bytes needed for compression header -depending on precise column type. -@reval 0 if prtype does not include DATA_COMPRESSED flag -@reval ZIP_COLUMN_HEADER_LENGTH if prtype includes DATA_COMPRESSED flag -*/ -UNIV_INLINE -ulint -prtype_get_compression_extra( - ulint prtype); /*!< in: precise type */ - /* Structure for an SQL data type. If you add fields to this structure, be sure to initialize them everywhere. This structure is initialized in the following functions: diff --git a/storage/xtradb/include/data0type.ic b/storage/xtradb/include/data0type.ic index 29dc480a19c..555852474aa 100644 --- a/storage/xtradb/include/data0type.ic +++ b/storage/xtradb/include/data0type.ic @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2012, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -710,18 +711,3 @@ dtype_get_sql_null_size( 0, 0)); #endif /* !UNIV_HOTBACKUP */ } - -/** -Calculates the number of extra bytes needed for compression header -depending on precise column type. -@reval 0 if prtype does not include DATA_COMPRESSED flag -@reval ZIP_COLUMN_HEADER_LENGTH if prtype includes DATA_COMPRESSED flag -*/ -UNIV_INLINE -ulint -prtype_get_compression_extra( - ulint prtype) /*!< in: precise type */ -{ - return (prtype & DATA_COMPRESSED) != 0 ? - ZIP_COLUMN_HEADER_LENGTH : 0; -} diff --git a/storage/xtradb/include/dict0boot.h b/storage/xtradb/include/dict0boot.h index d5bee886cbf..4fd9b0b7f98 100644 --- a/storage/xtradb/include/dict0boot.h +++ b/storage/xtradb/include/dict0boot.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -324,38 +325,6 @@ enum dict_fld_sys_datafiles_enum { DICT_FLD__SYS_DATAFILES__PATH = 3, DICT_NUM_FIELDS__SYS_DATAFILES = 4 }; -/* The columns in SYS_DICT */ -enum dict_col_sys_zip_dict_enum { - DICT_COL__SYS_ZIP_DICT__ID = 0, - DICT_COL__SYS_ZIP_DICT__NAME = 1, - DICT_COL__SYS_ZIP_DICT__DATA = 2, - DICT_NUM_COLS__SYS_ZIP_DICT = 3 -}; -/* The field numbers in the SYS_DICT clustered index */ -enum dict_fld_sys_zip_dict_enum { - DICT_FLD__SYS_ZIP_DICT__ID = 0, - DICT_FLD__SYS_ZIP_DICT__DB_TRX_ID = 1, - DICT_FLD__SYS_ZIP_DICT__DB_ROLL_PTR = 2, - DICT_FLD__SYS_ZIP_DICT__NAME = 3, - DICT_FLD__SYS_ZIP_DICT__DATA = 4, - DICT_NUM_FIELDS__SYS_ZIP_DICT = 5 -}; -/* The columns in SYS_DICT_COLS */ -enum dict_col_sys_zip_dict_cols_enum { - DICT_COL__SYS_ZIP_DICT_COLS__TABLE_ID = 0, - DICT_COL__SYS_ZIP_DICT_COLS__COLUMN_POS = 1, - DICT_COL__SYS_ZIP_DICT_COLS__DICT_ID = 2, - DICT_NUM_COLS__SYS_ZIP_DICT_COLS = 3 -}; -/* The field numbers in the SYS_DICT_COLS clustered index */ -enum dict_fld_sys_zip_dict_cols_enum { - DICT_FLD__SYS_ZIP_DICT_COLS__TABLE_ID = 0, - DICT_FLD__SYS_ZIP_DICT_COLS__COLUMN_POS = 1, - DICT_FLD__SYS_ZIP_DICT_COLS__DB_TRX_ID = 2, - DICT_FLD__SYS_ZIP_DICT_COLS__DB_ROLL_PTR = 3, - DICT_FLD__SYS_ZIP_DICT_COLS__DICT_ID = 4, - DICT_NUM_FIELDS__SYS_ZIP_DICT_COLS = 5 -}; /* A number of the columns above occur in multiple tables. These are the length of thos fields. */ diff --git a/storage/xtradb/include/dict0boot.ic b/storage/xtradb/include/dict0boot.ic index 2b156a4f672..42e91ee930e 100644 --- a/storage/xtradb/include/dict0boot.ic +++ b/storage/xtradb/include/dict0boot.ic @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2012, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -92,5 +93,3 @@ dict_is_sys_table( { return(id < DICT_HDR_FIRST_ID); } - - diff --git a/storage/xtradb/include/dict0crea.h b/storage/xtradb/include/dict0crea.h index 686f56ad58c..e5977cbabf6 100644 --- a/storage/xtradb/include/dict0crea.h +++ b/storage/xtradb/include/dict0crea.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -164,18 +165,6 @@ dberr_t dict_create_or_check_sys_tablespace(void); /*=====================================*/ -#define ZIP_DICT_MAX_NAME_LENGTH 64 -/* Max window size (2^15) minus 262 */ -#define ZIP_DICT_MAX_DATA_LENGTH 32506 - -/** Creates the zip_dict system table inside InnoDB -at server bootstrap or server start if it is not found or is -not of the right form. -@return DB_SUCCESS or error code */ -UNIV_INTERN -dberr_t -dict_create_or_check_sys_zip_dict(void); - /********************************************************************//** Add a single tablespace definition to the data dictionary tables in the database. @@ -192,83 +181,6 @@ dict_create_add_tablespace_to_dictionary( bool commit); /*!< in: if true then commit the transaction */ -/** Add a single compression dictionary definition to the SYS_ZIP_DICT -InnoDB system table. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_add_zip_dict( - const char* name, /*!< in: dict name */ - ulint name_len, /*!< in: dict name length */ - const char* data, /*!< in: dict data */ - ulint data_len, /*!< in: dict data length */ - trx_t* trx); /*!< in/out: transaction */ - -/** Add a single compression dictionary reference to the SYS_ZIP_DICT_COLS -InnoDB system table. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_add_zip_dict_reference( - ulint table_id, /*!< in: table id */ - ulint column_pos, /*!< in: column position */ - ulint dict_id, /*!< in: dict id */ - trx_t* trx); /*!< in/out: transaction */ - -/** Get a single compression dictionary id for the given -(table id, column pos) pair. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_get_zip_dict_id_by_reference( - ulint table_id, /*!< in: table id */ - ulint column_pos, /*!< in: column position */ - ulint* dict_id, /*!< out: dict id */ - trx_t* trx); /*!< in/out: transaction */ - -/** Get compression dictionary id for the given name. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_get_zip_dict_id_by_name( - const char* dict_name, /*!< in: dict name */ - ulint dict_name_len, /*!< in: dict name length */ - ulint* dict_id, /*!< out: dict id */ - trx_t* trx); /*!< in/out: transaction */ - -/** Get compression dictionary info (name and data) for the given id. -Allocates memory for name and data on success. -Must be freed with mem_free(). -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_get_zip_dict_info_by_id( - ulint dict_id, /*!< in: dict id */ - char** name, /*!< out: dict name */ - ulint* name_len, /*!< out: dict name length */ - char** data, /*!< out: dict data */ - ulint* data_len, /*!< out: dict data length */ - trx_t* trx); /*!< in/out: transaction */ - -/** Remove a single compression dictionary from the data dictionary -tables in the database. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_remove_zip_dict( - const char* name, /*!< in: dict name */ - ulint name_len, /*!< in: dict name length */ - trx_t* trx); /*!< in/out: transaction */ - -/** Remove all compression dictionary references for the given table ID from -the data dictionary tables in the database. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_remove_zip_dict_references_for_table( - ulint table_id, /*!< in: table id */ - trx_t* trx); /*!< in/out: transaction */ - /********************************************************************//** Add a foreign key definition to the data dictionary tables. @return error code or DB_SUCCESS */ diff --git a/storage/xtradb/include/dict0dict.h b/storage/xtradb/include/dict0dict.h index 1b6110dd010..a85f6f1eef9 100644 --- a/storage/xtradb/include/dict0dict.h +++ b/storage/xtradb/include/dict0dict.h @@ -2,7 +2,7 @@ Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2014, 2015, MariaDB Corporation. +Copyright (c) 2014, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1871,52 +1871,6 @@ dict_table_set_corrupt_by_space( ulint space_id, ibool need_mutex); -/** Insert a records into SYS_ZIP_DICT. -@retval DB_SUCCESS if OK -@retval dberr_t if the insert failed */ -UNIV_INTERN -dberr_t -dict_create_zip_dict( - const char* name, /*!< in: zip_dict name */ - ulint name_len, /*!< in: zip_dict name length*/ - const char* data, /*!< in: zip_dict data */ - ulint data_len); /*!< in: zip_dict data length */ - -/** Get single compression dictionary id for the given -(table id, column pos) pair. -@retval DB_SUCCESS if OK -@retval DB_RECORD_NOT_FOUND if not found */ -UNIV_INTERN -dberr_t -dict_get_dictionary_id_by_key( - ulint table_id, /*!< in: table id */ - ulint column_pos, /*!< in: column position */ - ulint* dict_id); /*!< out: zip_dict id */ - -/** Get compression dictionary info (name and data) for the given id. -Allocates memory in name->str and data->str on success. -Must be freed with mem_free(). -@retval DB_SUCCESS if OK -@retval DB_RECORD_NOT_FOUND if not found */ -UNIV_INTERN -dberr_t -dict_get_dictionary_info_by_id( - ulint dict_id, /*!< in: table name */ - char** name, /*!< out: dictionary name */ - ulint* name_len, /*!< out: dictionary name length*/ - char** data, /*!< out: dictionary data */ - ulint* data_len); /*!< out: dictionary data length*/ - -/** Delete a record in SYS_ZIP_DICT with the given name. -@retval DB_SUCCESS if OK -@retval DB_RECORD_NOT_FOUND if not found -@retval DB_ROW_IS_REFERENCED if in use */ -UNIV_INTERN -dberr_t -dict_drop_zip_dict( - const char* name, /*!< in: zip_dict name */ - ulint name_len); /*!< in: zip_dict name length*/ - #ifndef UNIV_NONINL #include "dict0dict.ic" #endif diff --git a/storage/xtradb/include/dict0load.h b/storage/xtradb/include/dict0load.h index 85e3e565637..1a720de5bb6 100644 --- a/storage/xtradb/include/dict0load.h +++ b/storage/xtradb/include/dict0load.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -44,8 +45,6 @@ enum dict_system_id_t { SYS_FOREIGN_COLS, SYS_TABLESPACES, SYS_DATAFILES, - SYS_ZIP_DICT, - SYS_ZIP_DICT_COLS, /* This must be last item. Defines the number of system tables. */ SYS_NUM_SYSTEM_TABLES @@ -389,32 +388,6 @@ dict_process_sys_datafiles( ulint* space, /*!< out: pace id */ const char** path); /*!< out: datafile path */ -/** This function parses a SYS_ZIP_DICT record, extracts necessary -information from the record and returns to caller. -@return error message, or NULL on success */ -UNIV_INTERN -const char* -dict_process_sys_zip_dict( - mem_heap_t* heap, /*!< in/out: heap memory */ - ulint zip_size, /*!< in: nonzero=compressed BLOB page size */ - const rec_t* rec, /*!< in: current SYS_ZIP_DICT rec */ - ulint* id, /*!< out: dict id */ - const char** name, /*!< out: dict name */ - const char** data, /*!< out: dict data */ - ulint* data_len); /*!< out: dict data length */ - -/** This function parses a SYS_ZIP_DICT_COLS record, extracts necessary -information from the record and returns to caller. -@return error message, or NULL on success */ -UNIV_INTERN -const char* -dict_process_sys_zip_dict_cols( - mem_heap_t* heap, /*!< in/out: heap memory */ - const rec_t* rec, /*!< in: current SYS_ZIP_DICT rec */ - ulint* table_id, /*!< out: table id */ - ulint* column_pos, /*!< out: column position */ - ulint* dict_id); /*!< out: dict id */ - /********************************************************************//** Get the filepath for a spaceid from SYS_DATAFILES. This function provides a temporary heap which is used for the table lookup, but not for the path. diff --git a/storage/xtradb/include/row0mysql.h b/storage/xtradb/include/row0mysql.h index 70da84640e5..e1148517d99 100644 --- a/storage/xtradb/include/row0mysql.h +++ b/storage/xtradb/include/row0mysql.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2000, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -41,9 +42,6 @@ struct SysIndexCallback; extern ibool row_rollback_on_timeout; -extern uint srv_compressed_columns_zip_level; -extern ulong srv_compressed_columns_threshold; - struct row_prebuilt_t; /*******************************************************************//** @@ -55,48 +53,6 @@ row_mysql_prebuilt_free_blob_heap( row_prebuilt_t* prebuilt); /*!< in: prebuilt struct of a ha_innobase:: table handle */ -/** Frees the compress heap in prebuilt when no longer needed. */ -UNIV_INTERN -void -row_mysql_prebuilt_free_compress_heap( - row_prebuilt_t* prebuilt); /*!< in: prebuilt struct of a - ha_innobase:: table handle */ - -/** Uncompress blob/text/varchar column using zlib -@return pointer to the uncompressed data */ -const byte* -row_decompress_column( - const byte* data, /*!< in: data in innodb(compressed) format */ - ulint *len, /*!< in: data length; out: length of - decompressed data*/ - const byte* dict_data, - /*!< in: optional dictionary data used for - decompression */ - ulint dict_data_len, - /*!< in: optional dictionary data length */ - row_prebuilt_t* prebuilt); - /*!< in: use prebuilt->compress_heap only - here*/ - -/** Compress blob/text/varchar column using zlib -@return pointer to the compressed data */ -byte* -row_compress_column( - const byte* data, /*!< in: data in mysql(uncompressed) - format */ - ulint *len, /*!< in: data length; out: length of - compressed data*/ - ulint lenlen, /*!< in: bytes used to store the length of - data */ - const byte* dict_data, - /*!< in: optional dictionary data used for - compression */ - ulint dict_data_len, - /*!< in: optional dictionary data length */ - row_prebuilt_t* prebuilt); - /*!< in: use prebuilt->compress_heap only - here*/ - /*******************************************************************//** Stores a >= 5.0.3 format true VARCHAR length to dest, in the MySQL row format. @@ -135,21 +91,10 @@ row_mysql_store_blob_ref( to 4 bytes */ const void* data, /*!< in: BLOB data; if the value to store is SQL NULL this should be NULL pointer */ - ulint len, /*!< in: BLOB length; if the value to store + ulint len); /*!< in: BLOB length; if the value to store is SQL NULL this should be 0; remember also to set the NULL bit in the MySQL record header! */ - bool need_decompression, - /*!< in: if the data need to be compressed*/ - const byte* dict_data, - /*!< in: optional compression dictionary - data */ - ulint dict_data_len, - /*!< in: optional compression dictionary data - length */ - row_prebuilt_t* prebuilt); - /*<! in: use prebuilt->compress_heap only - here */ /*******************************************************************//** Reads a reference to a BLOB in the MySQL format. @return pointer to BLOB data */ @@ -160,17 +105,8 @@ row_mysql_read_blob_ref( ulint* len, /*!< out: BLOB length */ const byte* ref, /*!< in: BLOB reference in the MySQL format */ - ulint col_len, /*!< in: BLOB reference length + ulint col_len); /*!< in: BLOB reference length (not BLOB length) */ - bool need_compression, - /*!< in: if the data need to be - compressed*/ - const byte* dict_data, /*!< in: optional compression - dictionary data */ - ulint dict_data_len, /*!< in: optional compression - dictionary data length */ - row_prebuilt_t* prebuilt); /*!< in: use prebuilt->compress_heap - only here */ /**************************************************************//** Pad a column with spaces. */ UNIV_INTERN @@ -218,16 +154,7 @@ row_mysql_store_col_in_innobase_format( necessarily the length of the actual payload data; if the column is a true VARCHAR then this is irrelevant */ - ulint comp, /*!< in: nonzero=compact format */ - bool need_compression, - /*!< in: if the data need to be - compressed */ - const byte* dict_data, /*!< in: optional compression - dictionary data */ - ulint dict_data_len, /*!< in: optional compression - dictionary data length */ - row_prebuilt_t* prebuilt); /*!< in: use prebuilt->compress_heap - only here */ + ulint comp); /*!< in: nonzero=compact format */ /****************************************************************//** Handles user errors and lock waits detected by the database engine. @return true if it was a lock wait and we should continue running the @@ -721,8 +648,6 @@ struct mysql_row_templ_t { ulint is_unsigned; /*!< if a column type is an integer type and this field is != 0, then it is an unsigned integer type */ - bool compressed; /*!< if column format is compressed */ - LEX_CSTRING zip_dict_data; /*!< associated compression dictionary */ }; #define MYSQL_FETCH_CACHE_SIZE 8 @@ -920,8 +845,6 @@ struct row_prebuilt_t { in fetch_cache */ mem_heap_t* blob_heap; /*!< in SELECTS BLOB fields are copied to this heap */ - mem_heap_t* compress_heap; /*!< memory heap used to compress - /decompress blob column*/ mem_heap_t* old_vers_heap; /*!< memory heap where a previous version is built in consistent read */ bool in_fts_query; /*!< Whether we are in a FTS query */ diff --git a/storage/xtradb/include/univ.i b/storage/xtradb/include/univ.i index fb103e2c9b8..3444cbb1e7f 100644 --- a/storage/xtradb/include/univ.i +++ b/storage/xtradb/include/univ.i @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. Copyright (c) 2008, Google Inc. Portions of this file contain modifications contributed and copyrighted by @@ -44,10 +45,10 @@ Created 1/20/1994 Heikki Tuuri #define INNODB_VERSION_MAJOR 5 #define INNODB_VERSION_MINOR 6 -#define INNODB_VERSION_BUGFIX 32 +#define INNODB_VERSION_BUGFIX 34 #ifndef PERCONA_INNODB_VERSION -#define PERCONA_INNODB_VERSION 79.0 +#define PERCONA_INNODB_VERSION 79.1 #endif /* Enable UNIV_LOG_ARCHIVE in XtraDB */ @@ -183,7 +184,7 @@ command. Not tested on Windows. */ #define UNIV_COMPILE_TEST_FUNCS */ -#if defined(HAVE_valgrind)&& defined(HAVE_VALGRIND_MEMCHECK_H) +#if defined HAVE_valgrind && defined HAVE_VALGRIND # define UNIV_DEBUG_VALGRIND #endif #if 0 diff --git a/storage/xtradb/rem/rem0rec.cc b/storage/xtradb/rem/rem0rec.cc index f8eb8076547..f3de279e182 100644 --- a/storage/xtradb/rem/rem0rec.cc +++ b/storage/xtradb/rem/rem0rec.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -323,8 +324,7 @@ rec_init_offsets_comp_ordinary( stored in one byte for 0..127. The length will be encoded in two bytes when it is 128 or more, or when the field is stored externally. */ - if (UNIV_UNLIKELY(col->len > 255 - - prtype_get_compression_extra(col->prtype)) + if (UNIV_UNLIKELY(col->len > 255) || UNIV_UNLIKELY(col->mtype == DATA_BLOB)) { if (len & 0x80) { @@ -849,8 +849,7 @@ rec_get_converted_size_comp_prefix_low( ((col->mtype == DATA_VARCHAR || col->mtype == DATA_BINARY || col->mtype == DATA_VARMYSQL) && (col->len == 0 - || len <= col->len + - prtype_get_compression_extra(col->prtype)))); + || len <= col->len))); fixed_len = field->fixed_len; if (temp && fixed_len @@ -882,8 +881,7 @@ rec_get_converted_size_comp_prefix_low( ut_ad(col->len >= 256 || col->mtype == DATA_BLOB); extra_size += 2; } else if (len < 128 - || (col->len < 256 - - prtype_get_compression_extra(col->prtype) + || (col->len < 256 && col->mtype != DATA_BLOB)) { extra_size++; } else { @@ -1279,16 +1277,12 @@ rec_convert_dtuple_to_rec_comp( *lens-- = (byte) (len >> 8) | 0xc0; *lens-- = (byte) len; } else { - ut_ad(len <= dtype_get_len(type) + - prtype_get_compression_extra( - dtype_get_prtype(type)) + ut_ad(len <= dtype_get_len(type) || dtype_get_mtype(type) == DATA_BLOB || !strcmp(index->name, FTS_INDEX_TABLE_IND_NAME)); if (len < 128 - || (dtype_get_len(type) < 256 - - prtype_get_compression_extra( - dtype_get_prtype(type)) + || (dtype_get_len(type) < 256 && dtype_get_mtype(type) != DATA_BLOB)) { *lens-- = (byte) len; diff --git a/storage/xtradb/row/row0merge.cc b/storage/xtradb/row/row0merge.cc index e397053949e..c1436f9e1b7 100644 --- a/storage/xtradb/row/row0merge.cc +++ b/storage/xtradb/row/row0merge.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2005, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -529,8 +530,7 @@ row_merge_buf_add( ((col->mtype == DATA_VARCHAR || col->mtype == DATA_BINARY || col->mtype == DATA_VARMYSQL) && (col->len == 0 - || len <= col->len + - prtype_get_compression_extra(col->prtype)))); + || len <= col->len))); fixed_len = ifield->fixed_len; if (fixed_len && !dict_table_is_comp(index->table) @@ -559,8 +559,7 @@ row_merge_buf_add( } else if (dfield_is_ext(field)) { extra_size += 2; } else if (len < 128 - || (col->len < 256 - - prtype_get_compression_extra(col->prtype) + || (col->len < 256 && col->mtype != DATA_BLOB)) { extra_size++; } else { diff --git a/storage/xtradb/row/row0mysql.cc b/storage/xtradb/row/row0mysql.cc index 46bf523750c..b366a55b2d9 100644 --- a/storage/xtradb/row/row0mysql.cc +++ b/storage/xtradb/row/row0mysql.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2000, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -69,48 +70,6 @@ Created 9/17/2000 Heikki Tuuri /** Provide optional 4.x backwards compatibility for 5.0 and above */ UNIV_INTERN ibool row_rollback_on_timeout = FALSE; -/** -Z_NO_COMPRESSION = 0 -Z_BEST_SPEED = 1 -Z_BEST_COMPRESSION = 9 -Z_DEFAULT_COMPRESSION = -1 -Compression level to be used by zlib for compressed-blob columns. -Settable by user. -*/ -UNIV_INTERN uint srv_compressed_columns_zip_level = DEFAULT_COMPRESSION_LEVEL; -/** -(Z_FILTERED | Z_HUFFMAN_ONLY | Z_RLE | Z_FIXED | Z_DEFAULT_STRATEGY) - -The strategy parameter is used to tune the compression algorithm. Use the -value Z_DEFAULT_STRATEGY for normal data, Z_FILTERED for data produced by a -filter (or predictor), Z_HUFFMAN_ONLY to force Huffman encoding only -(no string match), or Z_RLE to limit match distances to one -(run-length encoding). Filtered data consists mostly of small values with a -somewhat random distribution. In this case, the compression algorithm is -tuned to compress them better. -The effect of Z_FILTERED is to force more Huffman coding and less string -matching; it is somewhat intermediate between Z_DEFAULT_STRATEGY and -Z_HUFFMAN_ONLY. Z_RLE is designed to be almost as fast as Z_HUFFMAN_ONLY, -but give better compression for PNG image data. The strategy parameter only -affects the compression ratio but not the correctness of the compressed -output even if it is not set appropriately. Z_FIXED prevents the use of -dynamic Huffman codes, allowing for a simpler decoder for special -applications. -*/ -const uint srv_compressed_columns_zlib_strategy = Z_DEFAULT_STRATEGY; -/** Compress the column if the data length exceeds this value. */ -UNIV_INTERN ulong srv_compressed_columns_threshold = 96; -/** -Determine if zlib needs to compute adler32 value for the compressed data. -This variables is similar to page_zip_zlib_wrap, but only used by -compressed blob columns. -*/ -const bool srv_compressed_columns_zlib_wrap = true; -/** -Determine if zlib will use custom memory allocation functions based on -InnoDB memory heap routines (mem_heap_t*). -*/ -const bool srv_compressed_columns_zlib_use_heap = false; /** Chain node of the list of tables to drop in the background. */ struct row_mysql_drop_t{ char* table_name; /*!< table name */ @@ -214,17 +173,6 @@ row_mysql_prebuilt_free_blob_heap( prebuilt->blob_heap = NULL; } -/** Frees the compress heap in prebuilt when no longer needed. */ -UNIV_INTERN -void -row_mysql_prebuilt_free_compress_heap( - row_prebuilt_t* prebuilt) /*!< in: prebuilt struct of a - ha_innobase:: table handle */ -{ - mem_heap_free(prebuilt->compress_heap); - prebuilt->compress_heap = NULL; -} - /*******************************************************************//** Stores a >= 5.0.3 format true VARCHAR length to dest, in the MySQL row format. @@ -281,425 +229,6 @@ row_mysql_read_true_varchar( return(field + 1); } -/** - Compressed BLOB header format: - --------------------------------------------------------------- - | reserved | wrap | algorithm | len-len | compressed | unused | - | [1] | [1] | [5] | [3] | [1] | [5] | - --------------------------------------------------------------- - | 0 0 | 1 1 | 2 6 | 7 9 | 10 10 | 11 15 | - --------------------------------------------------------------- - * 'reserved' bit is planned to be used in future versions of the BLOB - header. In this version it must always be - 'default_zip_column_reserved_value' (0). - * 'wrap' identifies if compression algorithm calculated a checksum - (adler32 in case of zlib) and appended it to the compressed data. - * 'algorithm' identifies which algoritm was used to compress this BLOB. - Currently, the only value 'default_zip_column_algorithm_value' (0) is - supported. - * 'len-len' field identifies the length of the column length data portion - followed by this header (see below). - * If 'compressed' bit is set to 1, then this header is immediately followed - by 1..8 bytes (depending on the value of 'len-len' bitfield) which - determine original (uncompressed) block size. These 'len-len' bytes are - followed by compressed representation of the original data. - * If 'compressed' bit is set to 0, every other bitfield ('wrap', - 'algorithm' and 'le-len') must be ignored. In this case the header is - immediately followed by uncompressed (original) data. -*/ - -/** - Currently the only supported value for the 'reserved' field is - false (0). -*/ -static const bool default_zip_column_reserved_value = false; - -/** - Currently the only supported value for the 'algorithm' field is 0, which - means 'zlib'. -*/ -static const uint default_zip_column_algorithm_value = 0; - -static const size_t zip_column_prefix_max_length = - ZIP_COLUMN_HEADER_LENGTH + 8; -static const size_t zip_column_header_length = ZIP_COLUMN_HEADER_LENGTH; - -/* 'reserved', bit 0 */ -static const uint zip_column_reserved = 0; -/* 0000 0000 0000 0001 */ -static const uint zip_column_reserved_mask = 0x0001; - -/* 'wrap', bit 1 */ -static const uint zip_column_wrap = 1; -/* 0000 0000 0000 0010 */ -static const uint zip_column_wrap_mask = 0x0002; - -/* 'algorithm', bit 2,3,4,5,6 */ -static const uint zip_column_algorithm = 2; -/* 0000 0000 0111 1100 */ -static const uint zip_column_algorithm_mask = 0x007C; - -/* 'len-len', bit 7,8,9 */ -static const uint zip_column_data_length = 7; -/* 0000 0011 1000 0000 */ -static const uint zip_column_data_length_mask = 0x0380; - -/* 'compressed', bit 10 */ -static const uint zip_column_compressed = 10; -/* 0000 0100 0000 0000 */ -static const uint zip_column_compressed_mask = 0x0400; - -/** Updates compressed block header with the given components */ -static void -column_set_compress_header( - byte* data, - bool compressed, - ulint lenlen, - uint alg, - bool wrap, - bool reserved) -{ - ulint header = 0; - header |= (compressed << zip_column_compressed); - header |= (lenlen << zip_column_data_length); - header |= (alg << zip_column_algorithm); - header |= (wrap << zip_column_wrap); - header |= (reserved << zip_column_reserved); - mach_write_to_2(data, header); -} - -/** Parse compressed block header into components */ -static void -column_get_compress_header( - const byte* data, - bool* compressed, - ulint* lenlen, - uint* alg, - bool* wrap, - bool* reserved -) -{ - ulint header = mach_read_from_2(data); - *compressed = ((header & zip_column_compressed_mask) >> - zip_column_compressed); - *lenlen = ((header & zip_column_data_length_mask) >> - zip_column_data_length); - *alg = ((header & zip_column_algorithm_mask) >> - zip_column_algorithm); - *wrap = ((header & zip_column_wrap_mask) >> - zip_column_wrap); - *reserved = ((header & zip_column_reserved_mask) >> - zip_column_reserved); -} - -/** Allocate memory for zlib. */ -static -void* -column_zip_zalloc( - void* opaque, /*!< in/out: memory heap */ - uInt items, /*!< in: number of items to allocate */ - uInt size) /*!< in: size of an item in bytes */ -{ - return(mem_heap_zalloc(static_cast<mem_heap_t*>(opaque), - items * size)); -} - -/** Deallocate memory for zlib. */ -static -void -column_zip_free( - void* opaque MY_ATTRIBUTE((unused)), /*!< in: memory heap */ - void* address MY_ATTRIBUTE((unused))) /*!< in: object to free */ -{ -} - -/** Configure the zlib allocator to use the given memory heap. */ -UNIV_INTERN -void -column_zip_set_alloc( - void* stream, /*!< in/out: zlib stream */ - mem_heap_t* heap) /*!< in: memory heap to use */ -{ - z_stream* strm = static_cast<z_stream*>(stream); - - if (srv_compressed_columns_zlib_use_heap) { - strm->zalloc = column_zip_zalloc; - strm->zfree = column_zip_free; - strm->opaque = heap; - } else { - strm->zalloc = (alloc_func)0; - strm->zfree = (free_func)0; - strm->opaque = (voidpf)0; - } -} - -/** Compress blob/text/varchar column using zlib -@return pointer to the compressed data */ -byte* -row_compress_column( - const byte* data, /*!< in: data in mysql(uncompressed) - format */ - ulint *len, /*!< in: data length; out: length of - compressed data*/ - ulint lenlen, /*!< in: bytes used to store the length of - data */ - const byte* dict_data, - /*!< in: optional dictionary data used for - compression */ - ulint dict_data_len, - /*!< in: optional dictionary data length */ - row_prebuilt_t* prebuilt) - /*!< in: use prebuilt->compress_heap only - here*/ -{ - int err = 0; - ulint comp_len = *len; - ulint buf_len = *len + zip_column_prefix_max_length; - byte* buf; - byte* ptr; - z_stream c_stream; - bool wrap = srv_compressed_columns_zlib_wrap; - - int window_bits = wrap ? MAX_WBITS : -MAX_WBITS; - - if (!prebuilt->compress_heap) { - prebuilt->compress_heap = - mem_heap_create(max(UNIV_PAGE_SIZE, buf_len)); - } - - buf = static_cast<byte*>(mem_heap_zalloc( - prebuilt->compress_heap,buf_len)); - - if (*len < srv_compressed_columns_threshold || - srv_compressed_columns_zip_level == Z_NO_COMPRESSION) - goto do_not_compress; - - ptr = buf + zip_column_header_length + lenlen; - - /*init deflate object*/ - c_stream.next_in = const_cast<Bytef*>(data); - c_stream.avail_in = *len; - c_stream.next_out = ptr; - c_stream.avail_out = comp_len; - - column_zip_set_alloc(&c_stream, prebuilt->compress_heap); - - err = deflateInit2(&c_stream, srv_compressed_columns_zip_level, - Z_DEFLATED, window_bits, MAX_MEM_LEVEL, - srv_compressed_columns_zlib_strategy); - ut_a(err == Z_OK); - - if (dict_data != 0 && dict_data_len != 0) { - err = deflateSetDictionary(&c_stream, dict_data, - dict_data_len); - ut_a(err == Z_OK); - } - - err = deflate(&c_stream, Z_FINISH); - if (err != Z_STREAM_END) { - deflateEnd(&c_stream); - if (err == Z_OK) - err = Z_BUF_ERROR; - } else { - comp_len = c_stream.total_out; - err = deflateEnd(&c_stream); - } - - switch (err) { - case Z_OK: - break; - case Z_BUF_ERROR: - /* data after compress is larger than uncompressed data*/ - break; - default: - ib_logf(IB_LOG_LEVEL_ERROR, - "failed to compress the column, error: %d\n", err); - } - - /* make sure the compressed data size is smaller than - uncompressed data */ - if (err == Z_OK && - *len > (comp_len + zip_column_header_length + lenlen)) { - column_set_compress_header(buf, true, lenlen - 1, - default_zip_column_algorithm_value, wrap, - default_zip_column_reserved_value); - ptr = buf + zip_column_header_length; - /*store the uncompressed data length*/ - switch (lenlen) { - case 1: - mach_write_to_1(ptr, *len); - break; - case 2: - mach_write_to_2(ptr, *len); - break; - case 3: - mach_write_to_3(ptr, *len); - break; - case 4: - mach_write_to_4(ptr, *len); - break; - default: - ut_error; - } - - *len = comp_len + zip_column_header_length + lenlen; - return buf; - } - -do_not_compress: - ptr = buf; - column_set_compress_header(ptr, false, 0, - default_zip_column_algorithm_value, false, - default_zip_column_reserved_value); - ptr += zip_column_header_length; - memcpy(ptr, data, *len); - *len += zip_column_header_length; - return buf; -} - -/** Uncompress blob/text/varchar column using zlib -@return pointer to the uncompressed data */ -const byte* -row_decompress_column( - const byte* data, /*!< in: data in innodb(compressed) format */ - ulint *len, /*!< in: data length; out: length of - decompressed data*/ - const byte* dict_data, - /*!< in: optional dictionary data used for - decompression */ - ulint dict_data_len, - /*!< in: optional dictionary data length */ - row_prebuilt_t* prebuilt) - /*!< in: use prebuilt->compress_heap only - here*/ -{ - ulint buf_len = 0; - byte* buf; - int err = 0; - int window_bits = 0; - z_stream d_stream; - bool is_compressed = false; - bool wrap = false; - bool reserved = false; - ulint lenlen = 0; - uint alg = 0; - - ut_ad(*len != ULINT_UNDEFINED); - ut_ad(*len >= zip_column_header_length); - - column_get_compress_header(data, &is_compressed, &lenlen, &alg, - &wrap, &reserved); - - if (reserved != default_zip_column_reserved_value) { - ib_logf(IB_LOG_LEVEL_FATAL, - "unsupported compressed BLOB header format\n"); - } - - if (alg != default_zip_column_algorithm_value) { - ib_logf(IB_LOG_LEVEL_FATAL, - "unsupported 'algorithm' value in the" - " compressed BLOB header\n"); - } - - ut_a(lenlen < 4); - - data += zip_column_header_length; - if (!is_compressed) { /* column not compressed */ - *len -= zip_column_header_length; - return data; - } - - lenlen++; - - ulint comp_len = *len - zip_column_header_length - lenlen; - - ulint uncomp_len = 0; - switch (lenlen) { - case 1: - uncomp_len = mach_read_from_1(data); - break; - case 2: - uncomp_len = mach_read_from_2(data); - break; - case 3: - uncomp_len = mach_read_from_3(data); - break; - case 4: - uncomp_len = mach_read_from_4(data); - break; - default: - ut_error; - } - - data += lenlen; - - /* data is compressed, decompress it*/ - if (!prebuilt->compress_heap) { - prebuilt->compress_heap = - mem_heap_create(max(UNIV_PAGE_SIZE, uncomp_len)); - } - - buf_len = uncomp_len; - buf = static_cast<byte*>(mem_heap_zalloc( - prebuilt->compress_heap, buf_len)); - - /* init d_stream */ - d_stream.next_in = const_cast<Bytef*>(data); - d_stream.avail_in = comp_len; - d_stream.next_out = buf; - d_stream.avail_out = buf_len; - - column_zip_set_alloc(&d_stream, prebuilt->compress_heap); - - window_bits = wrap ? MAX_WBITS : -MAX_WBITS; - err = inflateInit2(&d_stream, window_bits); - ut_a(err == Z_OK); - - err = inflate(&d_stream, Z_FINISH); - if (err == Z_NEED_DICT) { - ut_a(dict_data != 0 && dict_data_len != 0); - err = inflateSetDictionary(&d_stream, dict_data, - dict_data_len); - ut_a(err == Z_OK); - err = inflate(&d_stream, Z_FINISH); - } - - if (err != Z_STREAM_END) { - inflateEnd(&d_stream); - if (err == Z_BUF_ERROR && d_stream.avail_in == 0) - err = Z_DATA_ERROR; - } else { - buf_len = d_stream.total_out; - err = inflateEnd(&d_stream); - } - - switch (err) { - case Z_OK: - break; - case Z_BUF_ERROR: - ib_logf(IB_LOG_LEVEL_FATAL, - "zlib buf error, this shouldn't happen\n"); - break; - default: - ib_logf(IB_LOG_LEVEL_FATAL, - "failed to decompress column, error: %d\n", err); - } - - if (err == Z_OK) { - if (buf_len != uncomp_len) { - ib_logf(IB_LOG_LEVEL_FATAL, - "failed to decompress blob column, may" - " be corrupted\n"); - } - *len = buf_len; - return buf; - } - - *len -= (zip_column_header_length + lenlen); - return data; -} - - /*******************************************************************//** Stores a reference to a BLOB in the MySQL format. */ UNIV_INTERN @@ -713,21 +242,10 @@ row_mysql_store_blob_ref( to 4 bytes */ const void* data, /*!< in: BLOB data; if the value to store is SQL NULL this should be NULL pointer */ - ulint len, /*!< in: BLOB length; if the value to store + ulint len) /*!< in: BLOB length; if the value to store is SQL NULL this should be 0; remember also to set the NULL bit in the MySQL record header! */ - bool need_decompression, - /*!< in: if the data need to be compressed*/ - const byte* dict_data, - /*!< in: optional compression dictionary - data */ - ulint dict_data_len, - /*!< in: optional compression dictionary data - length */ - row_prebuilt_t* prebuilt) - /*<! in: use prebuilt->compress_heap only - here */ { /* MySQL might assume the field is set to zero except the length and the pointer fields */ @@ -739,27 +257,11 @@ row_mysql_store_blob_ref( In 32-bit architectures we only use the first 4 bytes of the pointer slot. */ - ut_a(col_len - 8 > 1 || - len < 256 + - (need_decompression ? ZIP_COLUMN_HEADER_LENGTH : 0)); - ut_a(col_len - 8 > 2 || - len < 256 * 256 + - (need_decompression ? ZIP_COLUMN_HEADER_LENGTH : 0)); - ut_a(col_len - 8 > 3 || - len < 256 * 256 * 256 + - (need_decompression ? ZIP_COLUMN_HEADER_LENGTH : 0)); - - const byte *ptr = NULL; - - if (need_decompression) - ptr = row_decompress_column((const byte*)data, &len, - dict_data, dict_data_len, prebuilt); - - if (ptr) - memcpy(dest + col_len - 8, &ptr, sizeof ptr); - else - memcpy(dest + col_len - 8, &data, sizeof data); + ut_a(col_len - 8 > 1 || len < 256); + ut_a(col_len - 8 > 2 || len < 256 * 256); + ut_a(col_len - 8 > 3 || len < 256 * 256 * 256); + memcpy(dest + col_len - 8, &data, sizeof data); mach_write_to_n_little_endian(dest, col_len - 8, len); } @@ -773,32 +275,15 @@ row_mysql_read_blob_ref( ulint* len, /*!< out: BLOB length */ const byte* ref, /*!< in: BLOB reference in the MySQL format */ - ulint col_len, /*!< in: BLOB reference length + ulint col_len) /*!< in: BLOB reference length (not BLOB length) */ - bool need_compression, - /*!< in: if the data need to be - compressed*/ - const byte* dict_data, /*!< in: optional compression - dictionary data */ - ulint dict_data_len, /*!< in: optional compression - dictionary data length */ - row_prebuilt_t* prebuilt) /*!< in: use prebuilt->compress_heap - only here */ { byte* data = NULL; - byte* ptr = NULL; *len = mach_read_from_n_little_endian(ref, col_len - 8); memcpy(&data, ref + col_len - 8, sizeof data); - if (need_compression) { - ptr = row_compress_column(data, len, col_len - 8, dict_data, - dict_data_len, prebuilt); - if (ptr) - data = ptr; - } - return(data); } @@ -881,16 +366,7 @@ row_mysql_store_col_in_innobase_format( necessarily the length of the actual payload data; if the column is a true VARCHAR then this is irrelevant */ - ulint comp, /*!< in: nonzero=compact format */ - bool need_compression, - /*!< in: if the data need to be - compressed*/ - const byte* dict_data, /*!< in: optional compression - dictionary data */ - ulint dict_data_len, /*!< in: optional compression - dictionary data length */ - row_prebuilt_t* prebuilt) /*!< in: use prebuilt->compress_heap - only here */ + ulint comp) /*!< in: nonzero=compact format */ { const byte* ptr = mysql_data; const dtype_t* dtype; @@ -943,14 +419,8 @@ row_mysql_store_col_in_innobase_format( lenlen = 2; } - const byte* tmp_ptr = row_mysql_read_true_varchar( + ptr = row_mysql_read_true_varchar( &col_len, mysql_data, lenlen); - if (need_compression) - ptr = row_compress_column(tmp_ptr, &col_len, - lenlen, dict_data, dict_data_len, - prebuilt); - else - ptr = tmp_ptr; } else { /* Remove trailing spaces from old style VARCHAR columns. */ @@ -1032,9 +502,7 @@ row_mysql_store_col_in_innobase_format( } } else if (type == DATA_BLOB && row_format_col) { - ptr = row_mysql_read_blob_ref(&col_len, mysql_data, col_len, - need_compression, dict_data, dict_data_len, - prebuilt); + ptr = row_mysql_read_blob_ref(&col_len, mysql_data, col_len); } dfield_set_data(dfield, ptr, col_len); @@ -1092,11 +560,7 @@ row_mysql_convert_row_to_innobase( TRUE, /* MySQL row format data */ mysql_rec + templ->mysql_col_offset, templ->mysql_col_len, - dict_table_is_comp(prebuilt->table), - templ->compressed, - reinterpret_cast<const byte*>( - templ->zip_dict_data.str), - templ->zip_dict_data.length, prebuilt); + dict_table_is_comp(prebuilt->table)); next_column: ; } @@ -1442,10 +906,6 @@ row_prebuilt_free( mem_heap_free(prebuilt->blob_heap); } - if (prebuilt->compress_heap) { - mem_heap_free(prebuilt->compress_heap); - } - if (prebuilt->old_vers_heap) { mem_heap_free(prebuilt->old_vers_heap); } @@ -1875,9 +1335,6 @@ row_insert_for_mysql( return(DB_READ_ONLY); } - if (UNIV_LIKELY_NULL(prebuilt->compress_heap)) - mem_heap_empty(prebuilt->compress_heap); - trx->op_info = "inserting"; row_mysql_delay_if_needed(); @@ -3443,8 +2900,6 @@ row_mysql_table_id_reassign( " WHERE TABLE_ID = :old_id;\n" "UPDATE SYS_INDEXES SET TABLE_ID = :new_id\n" " WHERE TABLE_ID = :old_id;\n" - "UPDATE SYS_ZIP_DICT_COLS SET TABLE_ID = :new_id_narrow\n" - " WHERE TABLE_ID = :old_id_narrow;\n" "END;\n", FALSE, trx); return(err); @@ -4228,9 +3683,6 @@ next_rec: "UPDATE SYS_INDEXES" " SET TABLE_ID = :new_id, SPACE = :new_space\n" " WHERE TABLE_ID = :old_id;\n" - "UPDATE SYS_ZIP_DICT_COLS\n" - " SET TABLE_ID = :new_id_narrow\n" - " WHERE TABLE_ID = :old_id_narrow;\n" "END;\n" , FALSE, trx); @@ -4858,19 +4310,6 @@ row_drop_table_for_mysql( filepath = fil_make_ibd_name(tablename, false); } - /* Remove all compression dictionary references for the - table */ - err = dict_create_remove_zip_dict_references_for_table( - table->id, trx); - if (err != DB_SUCCESS) { - ib_logf(IB_LOG_LEVEL_ERROR, "Error: (%s) not " - "able to remove compression dictionary " - "references for table %s", ut_strerr(err), - tablename); - - goto funct_exit; - } - if (dict_table_has_fts_index(table) || DICT_TF2_FLAG_IS_SET(table, DICT_TF2_FTS_HAS_DOC_ID)) { ut_ad(table->n_ref_count == 0); diff --git a/storage/xtradb/row/row0sel.cc b/storage/xtradb/row/row0sel.cc index ad1e9e2bf9d..fd23d83e0e5 100644 --- a/storage/xtradb/row/row0sel.cc +++ b/storage/xtradb/row/row0sel.cc @@ -2,6 +2,7 @@ Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, Google Inc. +Copyright (c) 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -2459,8 +2460,7 @@ row_sel_convert_mysql_key_to_innobase( /* MySQL key value format col */ FALSE, key_ptr + data_offset, data_len, - dict_table_is_comp(index->table), - false, 0, 0 ,0); + dict_table_is_comp(index->table)); ut_a(buf <= original_buf + buf_len); } @@ -2554,15 +2554,15 @@ row_sel_store_row_id_to_prebuilt( #ifdef UNIV_DEBUG /** Convert a non-SQL-NULL field from Innobase format to MySQL format. */ # define row_sel_field_store_in_mysql_format( \ - dest,templ,idx,field,src,len,prebuilt) \ + dest,templ,idx,field,src,len) \ row_sel_field_store_in_mysql_format_func \ - (dest,templ,idx,field,src,len, prebuilt) + (dest,templ,idx,field,src,len) #else /* UNIV_DEBUG */ /** Convert a non-SQL-NULL field from Innobase format to MySQL format. */ # define row_sel_field_store_in_mysql_format( \ - dest,templ,idx,field,src,len,prebuilt) \ + dest,templ,idx,field,src,len) \ row_sel_field_store_in_mysql_format_func \ - (dest,templ,src,len, prebuilt) + (dest,templ,src,len) #endif /* UNIV_DEBUG */ /**************************************************************//** @@ -2592,10 +2592,7 @@ row_sel_field_store_in_mysql_format_func( templ->icp_rec_field_no */ #endif /* UNIV_DEBUG */ const byte* data, /*!< in: data to store */ - ulint len, /*!< in: length of the data */ - row_prebuilt_t* prebuilt) - /*!< in: use prebuilt->compress_heap - only here */ + ulint len) /*!< in: length of the data */ { byte* ptr; #ifdef UNIV_DEBUG @@ -2639,15 +2636,6 @@ row_sel_field_store_in_mysql_format_func( field_end = dest + templ->mysql_col_len; if (templ->mysql_type == DATA_MYSQL_TRUE_VARCHAR) { - /* If this is a compressed column, - decompress it first */ - if (templ->compressed) - data = row_decompress_column(data, &len, - reinterpret_cast<const byte*>( - templ->zip_dict_data.str), - templ->zip_dict_data.length, - prebuilt); - /* This is a >= 5.0.3 type true VARCHAR. Store the length of the data to the first byte or the first two bytes of dest. */ @@ -2698,11 +2686,7 @@ row_sel_field_store_in_mysql_format_func( already copied to the buffer in row_sel_store_mysql_rec */ row_mysql_store_blob_ref(dest, templ->mysql_col_len, data, - len, templ->compressed, - reinterpret_cast<const byte*>( - templ->zip_dict_data.str), - templ->zip_dict_data.length, - prebuilt); + len); break; case DATA_MYSQL: @@ -2855,7 +2839,7 @@ row_sel_store_mysql_field_func( row_sel_field_store_in_mysql_format( mysql_rec + templ->mysql_col_offset, - templ, index, field_no, data, len, prebuilt); + templ, index, field_no, data, len); if (heap != prebuilt->blob_heap) { mem_heap_free(heap); @@ -2905,7 +2889,7 @@ row_sel_store_mysql_field_func( row_sel_field_store_in_mysql_format( mysql_rec + templ->mysql_col_offset, - templ, index, field_no, data, len, prebuilt); + templ, index, field_no, data, len); } ut_ad(len != UNIV_SQL_NULL); @@ -2953,9 +2937,6 @@ row_sel_store_mysql_rec( prebuilt->blob_heap = NULL; } - if (UNIV_LIKELY_NULL(prebuilt->compress_heap)) - mem_heap_empty(prebuilt->compress_heap); - for (i = 0; i < prebuilt->n_template; i++) { const mysql_row_templ_t*templ = &prebuilt->mysql_template[i]; const ulint field_no diff --git a/storage/xtradb/srv/srv0start.cc b/storage/xtradb/srv/srv0start.cc index 82beb83192f..270489a1973 100644 --- a/storage/xtradb/srv/srv0start.cc +++ b/storage/xtradb/srv/srv0start.cc @@ -3,6 +3,7 @@ Copyright (c) 1996, 2016, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2008, Google Inc. Copyright (c) 2009, Percona Inc. +Copyright (c) 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -2780,12 +2781,6 @@ files_checked: } } - /* Create the SYS_ZIP_DICT system table */ - err = dict_create_or_check_sys_zip_dict(); - if (err != DB_SUCCESS) { - return(err); - } - srv_is_being_started = FALSE; ut_a(trx_purge_state() == PURGE_STATE_INIT); diff --git a/strings/decimal.c b/strings/decimal.c index 2353a8aefce..c6898cd099f 100644 --- a/strings/decimal.c +++ b/strings/decimal.c @@ -928,6 +928,8 @@ internal_str2dec(const char *from, decimal_t *to, char **end, my_bool fixed) error= decimal_shift(to, (int) exponent); } } + if (to->sign && decimal_is_zero(to)) + to->sign= 0; return error; fatal_error: diff --git a/support-files/CMakeLists.txt b/support-files/CMakeLists.txt index a66adc19b13..2a2e4171c95 100644 --- a/support-files/CMakeLists.txt +++ b/support-files/CMakeLists.txt @@ -67,7 +67,7 @@ IF(UNIX) ENDFOREACH() IF(INSTALL_SUPPORTFILESDIR) INSTALL(FILES magic DESTINATION ${inst_location} COMPONENT SupportFiles) - INSTALL(DIRECTORY RHEL4-SElinux/ DESTINATION ${inst_location}/SELinux/RHEL4 COMPONENT SupportFiles) + ADD_SUBDIRECTORY(SELinux) ENDIF() INSTALL(FILES mysql.m4 DESTINATION ${INSTALL_SHAREDIR}/aclocal COMPONENT Development) diff --git a/support-files/SELinux/CMakeLists.txt b/support-files/SELinux/CMakeLists.txt new file mode 100644 index 00000000000..e3cdb26ca8f --- /dev/null +++ b/support-files/SELinux/CMakeLists.txt @@ -0,0 +1,35 @@ +# Copyright (c) 2017, MariaDB +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +FIND_PROGRAM(CHECKMODULE checkmodule) +FIND_PROGRAM(SEMODULE_PACKAGE semodule_package) +MARK_AS_ADVANCED(CHECKMODULE SEMODULE_PACKAGE) + +SET(params DESTINATION ${INSTALL_SUPPORTFILESDIR}/SELinux COMPONENT SupportFiles) + +IF(CHECKMODULE AND SEMODULE_PACKAGE) + FOREACH(pol centos6-mariadb) + SET(src ${CMAKE_CURRENT_SOURCE_DIR}/${pol}.te) + SET(mod ${CMAKE_CURRENT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/${pol}-pp.dir/${pol}.mod) + SET(out ${CMAKE_CURRENT_BINARY_DIR}/${pol}.pp) + ADD_CUSTOM_COMMAND(OUTPUT ${out} + COMMAND ${CHECKMODULE} -M -m ${src} -o ${mod} + COMMAND ${SEMODULE_PACKAGE} -m ${mod} -o ${out} + DEPENDS ${src}) + ADD_CUSTOM_TARGET(${pol}-pp ALL DEPENDS ${out}) + INSTALL(FILES ${out} ${params}) + ENDFOREACH() +ENDIF() +INSTALL(FILES centos6-mariadb.te rhel4-mysql.fc rhel4-mysql.te ${params}) diff --git a/support-files/SELinux/centos6-mariadb.te b/support-files/SELinux/centos6-mariadb.te new file mode 100644 index 00000000000..1d3de52c700 --- /dev/null +++ b/support-files/SELinux/centos6-mariadb.te @@ -0,0 +1,9 @@ +module mariadb 1.0; + +require { + type mysqld_safe_t; + class capability { setuid setgid }; +} + +#============= mysqld_safe_t ============== +allow mysqld_safe_t self:capability { setuid setgid }; diff --git a/support-files/RHEL4-SElinux/mysql.fc b/support-files/SELinux/rhel4-mysql.fc index aa0fced4bbc..aa0fced4bbc 100644 --- a/support-files/RHEL4-SElinux/mysql.fc +++ b/support-files/SELinux/rhel4-mysql.fc diff --git a/support-files/RHEL4-SElinux/mysql.te b/support-files/SELinux/rhel4-mysql.te index d2609244640..d2609244640 100644 --- a/support-files/RHEL4-SElinux/mysql.te +++ b/support-files/SELinux/rhel4-mysql.te diff --git a/support-files/mysql.server.sh b/support-files/mysql.server.sh index 8d2e78edbd8..fc173e3993d 100644 --- a/support-files/mysql.server.sh +++ b/support-files/mysql.server.sh @@ -160,15 +160,9 @@ parse_server_arguments() { # Get arguments from the my.cnf file, # the only group, which is read from now on is [mysqld] -if test -x ./bin/my_print_defaults -then - print_defaults="./bin/my_print_defaults" -elif test -x $bindir/my_print_defaults +if test -x $bindir/my_print_defaults then print_defaults="$bindir/my_print_defaults" -elif test -x $bindir/mysql_print_defaults -then - print_defaults="$bindir/mysql_print_defaults" else # Try to find basedir in /etc/my.cnf conf=/etc/my.cnf diff --git a/support-files/rpm/server-postin.sh b/support-files/rpm/server-postin.sh index cd2aec4d84a..377a752824d 100644 --- a/support-files/rpm/server-postin.sh +++ b/support-files/rpm/server-postin.sh @@ -79,7 +79,12 @@ if [ -f /etc/redhat-release ] ; then echo ' make load' echo echo - fi + fi + if grep 'CentOS release 6' /etc/redhat-release >/dev/null 2>&1; then + if [ -x /usr/sbin/semodule ] ; then + /usr/sbin/semodule -i /usr/share/mysql/SELinux/centos6-mariadb.pp + fi + fi fi if [ -x sbin/restorecon ] ; then diff --git a/win/packaging/extra.wxs.in b/win/packaging/extra.wxs.in index befe070b404..016f6a8a97f 100644 --- a/win/packaging/extra.wxs.in +++ b/win/packaging/extra.wxs.in @@ -463,9 +463,24 @@ <RegistryValue Root='HKLM' Key='SOFTWARE\Monty Program AB\@CPACK_WIX_PACKAGE_NAME@' Name='DATADIR' Value='[DATADIR]' Type='string' KeyPath='yes'/> + <CreateFolder> + <util:PermissionEx User="NetworkService" GenericAll="yes" /> + </CreateFolder> + </Component> + + <Component Id="C.datadir.permissions" Directory="DATADIR"> + <Condition> + <!-- + Skip setting permissions for LogonUser, if package is installed by + service user (e.g LocalSystem) + --> + <![CDATA[ (UserSID <> "S-1-5-18") AND (UserSID <> "S-1-5-19") AND (UserSID <> "S-1-5-20") ]]> + </Condition> + <RegistryValue Root='HKLM' + Key='SOFTWARE\Monty Program AB\@CPACK_WIX_PACKAGE_NAME@' + Name='InstalledBy' Value='[USER_DOMAIN]\[LogonUser]' Type='string' KeyPath='yes'/> <CreateFolder> <util:PermissionEx User="[LogonUser]" Domain="[USER_DOMAIN]" GenericAll="yes" /> - <util:PermissionEx User="NetworkService" GenericAll="yes" /> </CreateFolder> </Component> diff --git a/win/packaging/heidisql.cmake b/win/packaging/heidisql.cmake index e2636eb4af8..772834e7c7d 100644 --- a/win/packaging/heidisql.cmake +++ b/win/packaging/heidisql.cmake @@ -1,4 +1,4 @@ -SET(HEIDISQL_BASE_NAME "HeidiSQL_9.3_Portable") +SET(HEIDISQL_BASE_NAME "HeidiSQL_9.4_Portable") SET(HEIDISQL_ZIP "${HEIDISQL_BASE_NAME}.zip") SET(HEIDISQL_URL "http://www.heidisql.com/downloads/releases/${HEIDISQL_ZIP}") SET(HEIDISQL_DOWNLOAD_DIR ${THIRD_PARTY_DOWNLOAD_LOCATION}/${HEIDISQL_BASE_NAME}) diff --git a/win/packaging/heidisql.wxi.in b/win/packaging/heidisql.wxi.in index 4f07a07627c..1d5c4430cd2 100644 --- a/win/packaging/heidisql.wxi.in +++ b/win/packaging/heidisql.wxi.in @@ -45,14 +45,20 @@ <Component Id="component.HeidiSQL_ssleay32.dll" Guid="*" Win64="no"> <File Id="heidisql.ssleay32.dll" Name="ssleay32.dll" Source="${HEIDISQL_DOWNLOAD_DIR}\ssleay32.dll" /> </Component> - <Component Id="component.HeidiSQL_libintl.dll" Guid="*" Win64="no"> - <File Id="heidisql.libintl.dll" Name="libintl.dll" Source="${HEIDISQL_DOWNLOAD_DIR}\libintl.dll" /> + <Component Id="component.HeidiSQL_libintl_8.dll" Guid="*" Win64="no"> + <File Id="heidisql.libintl_8.dll" Name="libintl-8.dll" Source="${HEIDISQL_DOWNLOAD_DIR}\libintl-8.dll" /> + </Component> + <Component Id="component.HeidiSQL_libiconv_2.dll" Guid="*" Win64="no"> + <File Id="heidisql.libiconv_2.dll" Name="libiconv-2.dll" Source="${HEIDISQL_DOWNLOAD_DIR}\libiconv-2.dll" /> </Component> <Directory Id="D.HeidiSQL.plugins" Name="plugins"> <Component Id="component.HeidiSQL_dialog.dll" Guid="*" Win64="no"> <File Id="heidisql.dialog.dll" Name="dialog.dll" Source="${HEIDISQL_DOWNLOAD_DIR}\plugins\dialog.dll" /> </Component> + <Component Id="component.HeidiSQL_auth_gssapi_client.dll" Guid="*" Win64="no"> + <File Id="heidisql.auth_gssapi_client.dll" Name="auth_gssapi_client.dll" Source="${HEIDISQL_DOWNLOAD_DIR}\plugins\auth_gssapi_client.dll" /> + </Component> </Directory> <Component Id="component.HeidiSQL_CleanupSettings" Guid="*" Win64="no"> @@ -70,8 +76,10 @@ <ComponentRef Id="component.HeidiSQL_libeay32.dll" /> <ComponentRef Id="component.HeidiSQL_libpq.dll" /> <ComponentRef Id="component.HeidiSQL_ssleay32.dll" /> - <ComponentRef Id="component.HeidiSQL_libintl.dll" /> + <ComponentRef Id="component.HeidiSQL_libintl_8.dll" /> + <ComponentRef Id="component.HeidiSQL_libiconv_2.dll" /> <ComponentRef Id="component.HeidiSQL_dialog.dll" /> + <ComponentRef Id="component.HeidiSQL_auth_gssapi_client.dll" /> <ComponentRef Id="component.HeidiSQL_CleanupSettings"/> </ComponentGroup> </Include> |