diff options
| author | Sergei Golubchik <serg@mariadb.org> | 2017-02-11 09:45:34 +0100 |
|---|---|---|
| committer | Sergei Golubchik <serg@mariadb.org> | 2017-02-11 09:45:34 +0100 |
| commit | f3914d10b61eb4bc4184b2a9aa92a998e1abd779 (patch) | |
| tree | 0f973c0dc2fb1f667ed852e4177b17938e28b6b9 | |
| parent | 559345806216369bad999539018bd3943860a5d5 (diff) | |
| parent | 2195bb4e416232ab807ff67eecf03b1223bf6bff (diff) | |
| download | mariadb-git-f3914d10b61eb4bc4184b2a9aa92a998e1abd779.tar.gz | |
Merge branch 'bb-10.2-serg-merge' into 10.2
219 files changed, 11098 insertions, 2926 deletions
diff --git a/.gitignore b/.gitignore index 737528ffdd4..9fd9637d7ef 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,7 @@ extra/jemalloc/build/ extra/jemalloc/tmp/ extra/my_print_defaults extra/mysql_waitpid +extra/mysqld_safe_helper extra/perror extra/replace extra/resolve_stack_dump @@ -224,6 +225,7 @@ support-files/mysql.spec support-files/mysqld_multi.server support-files/wsrep.cnf support-files/wsrep_notify +support-files/policy/selinux/mysqld-safe.pp tags tests/async_queries tests/bug25714 diff --git a/client/mysqladmin.cc b/client/mysqladmin.cc index d186a4c7fcc..e670079c99e 100644 --- a/client/mysqladmin.cc +++ b/client/mysqladmin.cc @@ -1571,8 +1571,10 @@ static my_bool get_pidfile(MYSQL *mysql, char *pidfile) if (mysql_query(mysql, "SHOW VARIABLES LIKE 'pid_file'")) { - my_printf_error(0, "query failed; error: '%s'", error_flags, - mysql_error(mysql)); + my_printf_error(mysql_errno(mysql), + "The query to get the server's pid file failed," + " error: '%s'. Continuing.", error_flags, + mysql_error(mysql)); } result = mysql_store_result(mysql); if (result) diff --git a/debian/mariadb-server-10.2.install b/debian/mariadb-server-10.2.install index 71aeb1f560b..7ab6ccd7c7a 100644 --- a/debian/mariadb-server-10.2.install +++ b/debian/mariadb-server-10.2.install @@ -27,6 +27,7 @@ usr/bin/mysql_tzinfo_to_sql usr/bin/mysqlbinlog usr/bin/mysqld_multi usr/bin/mysqld_safe +usr/bin/mysqld_safe_helper usr/bin/mysqlhotcopy usr/bin/perror usr/bin/replace diff --git a/debian/patches/33_scripts__mysql_create_system_tables__no_test.dpatch b/debian/patches/33_scripts__mysql_create_system_tables__no_test.dpatch index 6547e4434f1..183212ef678 100755 --- a/debian/patches/33_scripts__mysql_create_system_tables__no_test.dpatch +++ b/debian/patches/33_scripts__mysql_create_system_tables__no_test.dpatch @@ -8,8 +8,8 @@ ## DP: http://bugs.mysql.com/bug.php?id=6901 @DPATCH@ ---- old/scripts/mysql_system_tables_data.sql 2008-12-04 22:59:44.000000000 +0100 -+++ new/scripts/mysql_system_tables_data.sql 2008-12-04 23:00:07.000000000 +0100 +--- a/scripts/mysql_system_tables_data.sql ++++ b/scripts/mysql_system_tables_data.sql @@ -26,16 +26,6 @@ -- a plain character SELECT LOWER( REPLACE((SELECT REPLACE(@@hostname,'_','\_')),'%','\%') )INTO @current_hostname; @@ -26,14 +26,14 @@ - -- Fill "user" table with default users allowing root access -- from local machine if "user" table didn't exist before - CREATE TEMPORARY TABLE tmp_user LIKE user; -@@ -43,8 +33,6 @@ INSERT INTO tmp_user VALUES ('localhost','root','','Y','Y','Y','Y','Y','Y','Y',' - REPLACE INTO tmp_user SELECT @current_hostname,'root','','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'','','N','N','',0 FROM dual WHERE @current_hostname != 'localhost'; - REPLACE INTO tmp_user VALUES ('127.0.0.1','root','','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'','','N','N','',0); - REPLACE INTO tmp_user VALUES ('::1','root','','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'','','N','N', '', 0); --INSERT INTO tmp_user (host,user) VALUES ('localhost',''); --INSERT INTO tmp_user (host,user) SELECT @current_hostname,'' FROM dual WHERE @current_hostname != 'localhost'; - INSERT INTO user SELECT * FROM tmp_user WHERE @had_user_table=0; - DROP TABLE tmp_user; + CREATE TEMPORARY TABLE tmp_user_nopasswd LIKE user; +@@ -48,9 +38,6 @@ REPLACE INTO tmp_user_nopasswd VALUES ('127.0.0.1','root','','Y','Y','Y','Y','Y' + REPLACE INTO tmp_user_nopasswd VALUES ('::1','root','','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'','','N','N', '', 0); + -- More secure root account using unix sucket auth. + INSERT INTO tmp_user_socket VALUES ('localhost',IFNULL(@auth_root_socket, 'root'),'','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'unix_socket','','N', 'N','', 0); +--- Anonymous user with no privileges. +-INSERT INTO tmp_user_anonymous (host,user) VALUES ('localhost',''); +-INSERT INTO tmp_user_anonymous (host,user) SELECT @current_hostname,'' FROM dual WHERE @current_hostname != 'localhost'; - + INSERT INTO user SELECT * FROM tmp_user_nopasswd WHERE @had_user_table=0 AND @skip_auth_root_nopasswd IS NULL; + INSERT INTO user SELECT * FROM tmp_user_socket WHERE @had_user_table=0 AND @auth_root_socket IS NOT NULL; diff --git a/extra/CMakeLists.txt b/extra/CMakeLists.txt index 480cd6e7fbb..9cc5e7faa86 100644 --- a/extra/CMakeLists.txt +++ b/extra/CMakeLists.txt @@ -98,4 +98,7 @@ IF(UNIX) MYSQL_ADD_EXECUTABLE(mysql_waitpid mysql_waitpid.c COMPONENT Client) TARGET_LINK_LIBRARIES(mysql_waitpid mysys) + + MYSQL_ADD_EXECUTABLE(mysqld_safe_helper mysqld_safe_helper.c COMPONENT Server) + TARGET_LINK_LIBRARIES(mysqld_safe_helper mysys) ENDIF() diff --git a/extra/mysqld_safe_helper.c b/extra/mysqld_safe_helper.c new file mode 100644 index 00000000000..09e507c6e1c --- /dev/null +++ b/extra/mysqld_safe_helper.c @@ -0,0 +1,77 @@ +#include <my_global.h> +#include <m_string.h> +#include <my_sys.h> +#include <my_pthread.h> +#ifdef HAVE_PWD_H +#include <pwd.h> +#endif +#include <stdlib.h> +#include <stdio.h> + +void my_exit(int c) +{ + my_end(0); + exit(c); +} + +void do_usage() +{ + printf("Usage:\n" + " %s <user> log <filename>\n" + " %s <user> exec <command> <args>\n", + my_progname, my_progname); + my_exit(1); +} + +void do_log(const char *logfile) +{ + FILE *f; + uchar buf[4096]; + int size; + + if (!logfile) + do_usage(); + + f= my_fopen(logfile, O_WRONLY|O_APPEND|O_CREAT, MYF(MY_WME)); + if (!f) + my_exit(1); + + while ((size= my_fread(stdin, buf, sizeof(buf), MYF(MY_WME))) > 0) + if ((int)my_fwrite(f, buf, size, MYF(MY_WME)) != size) + my_exit(1); + + my_fclose(f, MYF(0)); + my_exit(0); +} + +void do_exec(char *args[]) +{ + if (!args[0]) + do_usage(); + + my_end(0); + execvp(args[0], args); +} + +int main(int argc, char *argv[]) +{ + struct passwd *user_info; + MY_INIT(argv[0]); + + if (argc < 3) + do_usage(argv[0]); + + user_info= my_check_user(argv[1], MYF(0)); + if (user_info ? my_set_user(argv[1], user_info, MYF(MY_WME)) + : my_errno == EINVAL) + my_exit(1); + + if (strcmp(argv[2], "log") == 0) + do_log(argv[3]); + + if (strcmp(argv[2], "exec") == 0) + do_exec(argv+3); + + my_end(0); + return 1; +} diff --git a/include/my_sys.h b/include/my_sys.h index 2e9f842d06e..2e24bfd02d3 100644 --- a/include/my_sys.h +++ b/include/my_sys.h @@ -678,8 +678,12 @@ extern void *my_memmem(const void *haystack, size_t haystacklen, #ifdef _WIN32 extern int my_access(const char *path, int amode); +#define my_check_user(A,B) (NULL) +#define my_set_user(A,B,C) (0) #else #define my_access access +struct passwd *my_check_user(const char *user, myf MyFlags); +int my_set_user(const char *user, struct passwd *user_info, myf MyFlags); #endif extern int check_if_legal_filename(const char *path); diff --git a/man/mysql_secure_installation.1 b/man/mysql_secure_installation.1 index 19522042e87..71e9d67dee4 100644 --- a/man/mysql_secure_installation.1 +++ b/man/mysql_secure_installation.1 @@ -1,6 +1,6 @@ '\" t .\" -.TH "\FBMYSQL_SECURE_INST" "1" "22/3/2016" "MariaDB 10\&.2" "MariaDB Database System" +.TH "\FBMYSQL_SECURE_INST" "1" "3 January 2017" "MariaDB 10\&.2" "MariaDB Database System" .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- @@ -71,9 +71,8 @@ test database, which by default can be accessed by anonymous users\&. .RE .PP -Invoke \fBmysql_secure_installation\fR -without arguments: +can be invoked without arguments: .sp .if n \{\ .RS 4 @@ -86,10 +85,75 @@ shell> \fBmysql_secure_installation\fR .\} .PP The script will prompt you to determine which actions to perform\&. +.PP +\fBmysql_secure_installation\fR +accepts some options: +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +.\" mysql_secure_installation: basedir option +.\" basedir option: mysql_secure_installation +\fB\-\-basedir=\fR\fB\fIdir_name\fR\fR +.sp +Base directory\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +.\" mysql_secure_installation: defaults-extra-file option +.\" defaults-extra-file option: mysql_secure_installation +\fB\-\-defaults\-extra\-file=\fR\fB\fIfile_name\fR\fR +.sp +Additional option file\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +.\" mysql_secure_installation: defaults-file option +.\" defaults-file option: mysql_secure_installation +\fB\-\-defaults\-file=\fR\fB\fIfile_name\fR\fR +.sp +Option file\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +.\" mysql_secure_installation: no-defaults option +.\" no-defaults option: mysql_secure_installation +\fB\-\-no\-defaults\fR +.sp +Don't read any defaults file\&. +.RE +.sp +Other unrecognized options will be passed on to the server\&. .SH "COPYRIGHT" .br .PP -Copyright 2007-2008 MySQL AB, 2008-2010 Sun Microsystems, Inc., 2010-2015 MariaDB Foundation +Copyright 2007-2008 MySQL AB, 2008-2010 Sun Microsystems, Inc., 2010-2017 MariaDB Foundation .PP This documentation is free software; you can redistribute it and/or modify it only under the terms of the GNU General Public License as published by the Free Software Foundation; version 2 of the License. .PP diff --git a/mysql-test/extra/rpl_tests/rpl_special_charset.inc b/mysql-test/extra/rpl_tests/rpl_special_charset.inc index 51119b319d7..641aa483d32 100644 --- a/mysql-test/extra/rpl_tests/rpl_special_charset.inc +++ b/mysql-test/extra/rpl_tests/rpl_special_charset.inc @@ -20,7 +20,7 @@ # then set default's client character set(latin1) as client's character set. ############################################################################### --source include/master-slave.inc -call mtr.add_suppression("Cannot use utf16 as character_set_client"); +call mtr.add_suppression("'utf16' can not be used as client character set"); CREATE TABLE t1(i VARCHAR(20)); INSERT INTO t1 VALUES (0xFFFF); --sync_slave_with_master diff --git a/mysql-test/include/analyze-format.inc b/mysql-test/include/analyze-format.inc new file mode 100644 index 00000000000..65e61b81582 --- /dev/null +++ b/mysql-test/include/analyze-format.inc @@ -0,0 +1,3 @@ +# The time on ANALYSE FORMAT=JSON is rather variable + +--replace_regex /("(r_total_time_ms|r_buffer_size)": )[^, \n]*/\1"REPLACED"/ diff --git a/mysql-test/r/analyze_format_json.result b/mysql-test/r/analyze_format_json.result index db28a2748aa..6ac23c6ce80 100644 --- a/mysql-test/r/analyze_format_json.result +++ b/mysql-test/r/analyze_format_json.result @@ -491,15 +491,15 @@ ANALYZE "query_block": { "select_id": 1, "r_loops": 1, - "volatile parameter": "REPLACED", + "r_total_time_ms": "REPLACED", "having_condition": "TOP > t2.a", "filesort": { "sort_key": "t2.a", "r_loops": 1, - "volatile parameter": "REPLACED", + "r_total_time_ms": "REPLACED", "r_used_priority_queue": false, "r_output_rows": 0, - "volatile parameter": "REPLACED", + "r_buffer_size": "REPLACED", "temporary_table": { "table": { "table_name": "t2", @@ -507,7 +507,7 @@ ANALYZE "r_loops": 1, "rows": 256, "r_rows": 256, - "volatile parameter": "REPLACED", + "r_total_time_ms": "REPLACED", "filtered": 100, "r_filtered": 100 } @@ -522,14 +522,14 @@ ANALYZE "query_block": { "select_id": 1, "r_loops": 1, - "volatile parameter": "REPLACED", + "r_total_time_ms": "REPLACED", "filesort": { "sort_key": "t2.a", "r_loops": 1, - "volatile parameter": "REPLACED", + "r_total_time_ms": "REPLACED", "r_used_priority_queue": false, "r_output_rows": 256, - "volatile parameter": "REPLACED", + "r_buffer_size": "REPLACED", "temporary_table": { "table": { "table_name": "t2", @@ -537,7 +537,7 @@ ANALYZE "r_loops": 1, "rows": 256, "r_rows": 256, - "volatile parameter": "REPLACED", + "r_total_time_ms": "REPLACED", "filtered": 100, "r_filtered": 100 } @@ -563,14 +563,14 @@ ANALYZE "query_block": { "select_id": 1, "r_loops": 1, - "volatile parameter": "REPLACED", + "r_total_time_ms": "REPLACED", "filesort": { "sort_key": "t2.a", "r_loops": 1, - "volatile parameter": "REPLACED", + "r_total_time_ms": "REPLACED", "r_used_priority_queue": false, "r_output_rows": 256, - "volatile parameter": "REPLACED", + "r_buffer_size": "REPLACED", "temporary_table": { "table": { "table_name": "t2", @@ -578,7 +578,7 @@ ANALYZE "r_loops": 1, "rows": 256, "r_rows": 256, - "volatile parameter": "REPLACED", + "r_total_time_ms": "REPLACED", "filtered": 100, "r_filtered": 100 } @@ -600,14 +600,14 @@ ANALYZE "query_block": { "select_id": 1, "r_loops": 1, - "volatile parameter": "REPLACED", + "r_total_time_ms": "REPLACED", "table": { "table_name": "t1", "access_type": "ALL", "r_loops": 1, "rows": 2, "r_rows": 2, - "volatile parameter": "REPLACED", + "r_total_time_ms": "REPLACED", "filtered": 100, "r_filtered": 100 }, @@ -619,7 +619,7 @@ ANALYZE "r_loops": 1, "rows": 2, "r_rows": 2, - "volatile parameter": "REPLACED", + "r_total_time_ms": "REPLACED", "filtered": 100, "r_filtered": 100 }, @@ -632,14 +632,14 @@ ANALYZE "query_block": { "select_id": 2, "r_loops": 1, - "volatile parameter": "REPLACED", + "r_total_time_ms": "REPLACED", "table": { "table_name": "t1", "access_type": "ALL", "r_loops": 1, "rows": 2, "r_rows": 2, - "volatile parameter": "REPLACED", + "r_total_time_ms": "REPLACED", "filtered": 100, "r_filtered": 100 } @@ -653,7 +653,7 @@ ANALYZE "r_loops": 1, "rows": 2, "r_rows": 2, - "volatile parameter": "REPLACED", + "r_total_time_ms": "REPLACED", "filtered": 100, "r_filtered": 100 }, @@ -685,22 +685,22 @@ ANALYZE "query_block": { "select_id": 1, "r_loops": 1, - "volatile parameter": "REPLACED", + "r_total_time_ms": "REPLACED", "filesort": { "sort_key": "group_concat(t3.f3 separator ',')", "r_loops": 1, - "volatile parameter": "REPLACED", + "r_total_time_ms": "REPLACED", "r_used_priority_queue": false, "r_output_rows": 0, - "volatile parameter": "REPLACED", + "r_buffer_size": "REPLACED", "temporary_table": { "filesort": { "sort_key": "(subquery#2)", "r_loops": 1, - "volatile parameter": "REPLACED", + "r_total_time_ms": "REPLACED", "r_used_priority_queue": false, "r_output_rows": 0, - "volatile parameter": "REPLACED", + "r_buffer_size": "REPLACED", "temporary_table": { "table": { "table_name": "t2", @@ -708,7 +708,7 @@ ANALYZE "r_loops": 1, "rows": 2, "r_rows": 2, - "volatile parameter": "REPLACED", + "r_total_time_ms": "REPLACED", "filtered": 100, "r_filtered": 100 }, @@ -719,7 +719,7 @@ ANALYZE "r_loops": 1, "rows": 2, "r_rows": 2, - "volatile parameter": "REPLACED", + "r_total_time_ms": "REPLACED", "filtered": 100, "r_filtered": 0, "attached_condition": "t3.f3 in (1,2)" diff --git a/mysql-test/r/create.result b/mysql-test/r/create.result index 5dc60742e53..81b65c420cd 100644 --- a/mysql-test/r/create.result +++ b/mysql-test/r/create.result @@ -1917,6 +1917,13 @@ end| create table t1 as select f1(); ERROR 42S02: Table 'test.t1' doesn't exist drop function f1; +# +# MDEV-10274 Bundling insert with create statement +# for table with unsigned Decimal primary key issues warning 1194 +# +create table t1(ID decimal(2,1) unsigned NOT NULL, PRIMARY KEY (ID))engine=memory +select 2.1 ID; +drop table t1; End of 5.5 tests create table t1; ERROR 42000: A table must have at least 1 column diff --git a/mysql-test/r/ctype_ucs2_def.result b/mysql-test/r/ctype_ucs2_def.result index 5ca7d1689d2..9d8f5181103 100644 --- a/mysql-test/r/ctype_ucs2_def.result +++ b/mysql-test/r/ctype_ucs2_def.result @@ -1,4 +1,4 @@ -call mtr.add_suppression("Cannot use ucs2 as character_set_client"); +call mtr.add_suppression("'ucs2' can not be used as client character set"); show variables like 'collation_server'; Variable_name Value collation_server ucs2_unicode_ci diff --git a/mysql-test/r/ctype_ucs2_query_cache.result b/mysql-test/r/ctype_ucs2_query_cache.result index 9a7580324c1..eba7f2fb0fc 100644 --- a/mysql-test/r/ctype_ucs2_query_cache.result +++ b/mysql-test/r/ctype_ucs2_query_cache.result @@ -1,4 +1,4 @@ -call mtr.add_suppression("Cannot use ucs2 as character_set_client"); +call mtr.add_suppression("'ucs2' can not be used as client character set"); # # Start of 5.5 tests # diff --git a/mysql-test/r/ctype_utf16_def.result b/mysql-test/r/ctype_utf16_def.result index 9d3d110fc99..98b6f7d913d 100644 --- a/mysql-test/r/ctype_utf16_def.result +++ b/mysql-test/r/ctype_utf16_def.result @@ -1,4 +1,4 @@ -call mtr.add_suppression("Cannot use utf16 as character_set_client"); +call mtr.add_suppression("'utf16' can not be used as client character set"); SHOW VARIABLES LIKE 'collation_server'; Variable_name Value collation_server utf16_general_ci diff --git a/mysql-test/r/events_slowlog.result b/mysql-test/r/events_slowlog.result new file mode 100644 index 00000000000..7de5925bc0f --- /dev/null +++ b/mysql-test/r/events_slowlog.result @@ -0,0 +1,13 @@ +set @event_scheduler_save= @@global.event_scheduler; +set @slow_query_log_save= @@global.slow_query_log; +set global event_scheduler= on; +set global slow_query_log= on; +set global long_query_time=0.2; +create table t1 (i int); +insert into t1 values (0); +create event ev on schedule at CURRENT_TIMESTAMP + INTERVAL 1 second do update t1 set i=1+sleep(0.5); +FOUND /update t1 set i=1/ in mysqld-slow.log +drop table t1; +set global event_scheduler= @event_scheduler_save; +set global slow_query_log= @slow_query_log_save; +set global long_query_time= @@session.long_query_time; diff --git a/mysql-test/r/func_time.result b/mysql-test/r/func_time.result index 814cd2dc5aa..54da823c439 100644 --- a/mysql-test/r/func_time.result +++ b/mysql-test/r/func_time.result @@ -2766,6 +2766,12 @@ id date1 date2 DATE_ADD(a.date1,INTERVAL -10 DAY) TO_DAYS(a.date1)-10 18 2010-10-13 2010-10-03 2010-10-03 734413 DROP TABLE t1; # +# MDEV-10524 Assertion `arg1_int >= 0' failed in Item_func_additive_op::result_precision() +# +SELECT 1 MOD ADDTIME( '13:58:57', '00:00:01' ) + 2; +1 MOD ADDTIME( '13:58:57', '00:00:01' ) + 2 +3 +# # Start of 10.0 tests # # diff --git a/mysql-test/r/index_merge_innodb.result b/mysql-test/r/index_merge_innodb.result index 0450622411a..6a3ea839535 100644 --- a/mysql-test/r/index_merge_innodb.result +++ b/mysql-test/r/index_merge_innodb.result @@ -799,3 +799,32 @@ a b c 9 d d DROP TABLE t1; set optimizer_switch= @optimizer_switch_save; +# +# MDEV-10927: Crash When Using sort_union Optimization +# +set @tmp_optimizer_switch=@@optimizer_switch; +SET optimizer_switch='index_merge_sort_intersection=on'; +SET SESSION sort_buffer_size = 1024; +create table t1 ( +pk int(11) NOT NULL AUTO_INCREMENT, +col1 int(11) NOT NULL, +col2 int(11) NOT NULL, +col3 int(11) NOT NULL, +key2 int(11) NOT NULL, +col4 int(11) NOT NULL, +key1 int(11) NOT NULL, +PRIMARY KEY (pk), +KEY key1 (key1), +KEY key2 (key2) +) ENGINE=InnoDB AUTO_INCREMENT=12860259 DEFAULT CHARSET=utf8 ROW_FORMAT=COMPACT; +create table t2(a int); +insert into t2 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +create table t3(a int); +insert into t3 select A.a + B.a* 10 + C.a * 100 + D.a*1000 from t2 A, t2 B, t2 C, t2 D; +insert into t1 (key1, key2, col1,col2,col3,col4) +select a,a, a,a,a,a from t3; +SELECT sum(col1) FROM t1 FORCE INDEX (key1,key2) WHERE (key1 between 10 and 8191+10) or (key2= 5); +sum(col1) +33632261 +drop table t1,t2,t3; +set optimizer_switch=@tmp_optimizer_switch; diff --git a/mysql-test/r/information_schema_part.result b/mysql-test/r/information_schema_part.result index e0a0f28dec6..91720d12ac4 100644 --- a/mysql-test/r/information_schema_part.result +++ b/mysql-test/r/information_schema_part.result @@ -151,3 +151,11 @@ select create_options from information_schema.tables where table_schema="test"; create_options partitioned drop table t1; +# +# MDEV-11353 - Identical logical conditions +# +CREATE TABLE t1(a INT) CHECKSUM=1 SELECT 1; +SELECT CHECKSUM FROM INFORMATION_SCHEMA.PARTITIONS WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +CHECKSUM +3036305396 +DROP TABLE t1; diff --git a/mysql-test/r/insert_update.result b/mysql-test/r/insert_update.result index 1987c5c0559..e8e6e16fe5a 100644 --- a/mysql-test/r/insert_update.result +++ b/mysql-test/r/insert_update.result @@ -242,14 +242,16 @@ ERROR 42S22: Unknown column 'a' in 'field list' DROP TABLE t1,t2; SET SQL_MODE = 'TRADITIONAL'; CREATE TABLE t1 (a INT PRIMARY KEY, b INT NOT NULL); +INSERT INTO t1 VALUES (1,1); INSERT INTO t1 (a) VALUES (1); ERROR HY000: Field 'b' doesn't have a default value INSERT INTO t1 (a) VALUES (1) ON DUPLICATE KEY UPDATE a = b; ERROR HY000: Field 'b' doesn't have a default value +INSERT INTO t1 (a) VALUES (1) ON DUPLICATE KEY UPDATE b = a; INSERT INTO t1 (a) VALUES (1) ON DUPLICATE KEY UPDATE b = b; -ERROR HY000: Field 'b' doesn't have a default value SELECT * FROM t1; a b +1 1 DROP TABLE t1; CREATE TABLE t1 (f1 INT AUTO_INCREMENT PRIMARY KEY, f2 VARCHAR(5) NOT NULL UNIQUE); diff --git a/mysql-test/r/join_cache.result b/mysql-test/r/join_cache.result index 74e64d92856..6a8581e0605 100644 --- a/mysql-test/r/join_cache.result +++ b/mysql-test/r/join_cache.result @@ -5755,11 +5755,13 @@ LEFT JOIN t2 c1 ON c1.parent_id = t.id AND c1.col2 = "val" LEFT JOIN t2 c23 ON c23.parent_id = t.id AND c23.col2 = "val" LEFT JOIN t2 c24 ON c24.parent_id = t.id AND c24.col2 = "val" LEFT JOIN t2 c25 ON c25.parent_id = t.id AND c25.col2 = "val" + LEFT JOIN t2 c26 ON c26.parent_id = t.id AND c26.col2 = "val" + LEFT JOIN t2 c27 ON c27.parent_id = t.id AND c27.col2 = "val" ORDER BY col1; id col1 -select timestampdiff(second, @init_time, now()) <= 1; -timestampdiff(second, @init_time, now()) <= 1 +select timestampdiff(second, @init_time, now()) <= 5; +timestampdiff(second, @init_time, now()) <= 5 1 set join_cache_level=2; set @init_time:=now(); @@ -5791,11 +5793,13 @@ LEFT JOIN t2 c1 ON c1.parent_id = t.id AND c1.col2 = "val" LEFT JOIN t2 c23 ON c23.parent_id = t.id AND c23.col2 = "val" LEFT JOIN t2 c24 ON c24.parent_id = t.id AND c24.col2 = "val" LEFT JOIN t2 c25 ON c25.parent_id = t.id AND c25.col2 = "val" + LEFT JOIN t2 c26 ON c26.parent_id = t.id AND c26.col2 = "val" + LEFT JOIN t2 c27 ON c27.parent_id = t.id AND c27.col2 = "val" ORDER BY col1; id col1 -select timestampdiff(second, @init_time, now()) <= 1; -timestampdiff(second, @init_time, now()) <= 1 +select timestampdiff(second, @init_time, now()) <= 5; +timestampdiff(second, @init_time, now()) <= 5 1 EXPLAIN SELECT t.* @@ -5826,6 +5830,8 @@ LEFT JOIN t2 c1 ON c1.parent_id = t.id AND c1.col2 = "val" LEFT JOIN t2 c23 ON c23.parent_id = t.id AND c23.col2 = "val" LEFT JOIN t2 c24 ON c24.parent_id = t.id AND c24.col2 = "val" LEFT JOIN t2 c25 ON c25.parent_id = t.id AND c25.col2 = "val" + LEFT JOIN t2 c26 ON c26.parent_id = t.id AND c26.col2 = "val" + LEFT JOIN t2 c27 ON c27.parent_id = t.id AND c27.col2 = "val" ORDER BY col1; id select_type table type possible_keys key key_len ref rows Extra @@ -5855,6 +5861,8 @@ id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE c23 ALL NULL NULL NULL NULL 1 Using where; Using join buffer (incremental, BNL join) 1 SIMPLE c24 ALL NULL NULL NULL NULL 1 Using where; Using join buffer (incremental, BNL join) 1 SIMPLE c25 ALL NULL NULL NULL NULL 1 Using where; Using join buffer (incremental, BNL join) +1 SIMPLE c26 ALL NULL NULL NULL NULL 1 Using where; Using join buffer (incremental, BNL join) +1 SIMPLE c27 ALL NULL NULL NULL NULL 1 Using where; Using join buffer (incremental, BNL join) set join_buffer_size=default; set join_cache_level = default; DROP TABLE t1,t2; diff --git a/mysql-test/r/loaddata.result b/mysql-test/r/loaddata.result index 5a61f044148..b91dd1b87ae 100644 --- a/mysql-test/r/loaddata.result +++ b/mysql-test/r/loaddata.result @@ -537,7 +537,7 @@ disconnect con1; # Bug#11765139 58069: LOAD DATA INFILE: VALGRIND REPORTS INVALID MEMORY READS AND WRITES WITH U # CREATE TABLE t1(f1 INT); -SELECT 0xE1C330 INTO OUTFILE 't1.dat'; +SELECT 0xE1BB30 INTO OUTFILE 't1.dat'; LOAD DATA INFILE 't1.dat' IGNORE INTO TABLE t1 CHARACTER SET utf8; DROP TABLE t1; # @@ -562,34 +562,21 @@ FIELDS TERMINATED BY 't' LINES TERMINATED BY ''; Got one of the listed errors SET @@sql_mode= @old_mode; DROP TABLE t1; - # -# Bug#23080148 - Backport of Bug#20683959. -# Bug#20683959 LOAD DATA INFILE IGNORES A SPECIFIC ROW SILENTLY -# UNDER DB CHARSET IS UTF8. +# MDEV-11079 Regression: LOAD DATA INFILE lost BLOB support using utf8 load files # -CREATE DATABASE d1 CHARSET latin1; -USE d1; -CREATE TABLE t1 (val TEXT); -LOAD DATA INFILE '../../std_data/bug20683959loaddata.txt' INTO TABLE t1; -SELECT COUNT(*) FROM t1; -COUNT(*) -1 -SELECT HEX(val) FROM t1; -HEX(val) -C38322525420406E696F757A656368756E3A20E98198E2889AF58081AEE7B99DE4B88AE383A3E7B99DE69690F58087B3E7B9A7EFBDA8E7B99DEFBDB3E7B99DE78999E880B3E7B8BAEFBDAAE7B9A7E89699E296A1E7B8BAE4BBA3EFBD8CE7B8BAEFBDA9E7B8B2E2889AE38184E7B99DEFBDB3E7B99DE4B88AE383A3E7B99DE69690F58087B3E7B9A7EFBDA8E7B99DEFBDB3E7B99DE5B3A8EFBD84E8ABA0EFBDA8E89C89F580948EE599AAE7B8BAEFBDAAE7B8BAE9A198EFBDA9EFBDB1E7B9A7E581B5E289A0E7B8BAEFBDBEE7B9A7E9A194EFBDA9E882B4EFBDA5EFBDB5E980A7F5808B96E28693E99EABE38287E58F99E7B8BAE58AB1E28691E7B8BAF5808B9AE7828AE98095EFBDB1E7B8BAEFBDAFE7B8B2E288ABE6A89FE89EB3E6BA98F58081ADE88EA0EFBDBAE98095E6BA98F58081AEE89D93EFBDBAE8AD9BEFBDACE980A7F5808B96E28693E7B8BAF580918EE288AAE7B8BAE4B88AEFBC9EE7B8BAE4B99DE28691E7B8BAF5808B96EFBCA0E88DB3E6A68AEFBDB9EFBDB3E981B2E5B3A8E296A1E7B8BAE7A4BCE7828AE88DB3E6A68AEFBDB0EFBDBDE7B8BAA0E7B8BAE88B93EFBDBEE5B899EFBC9E -CREATE DATABASE d2 CHARSET utf8; -USE d2; -CREATE TABLE t1 (val TEXT); -LOAD DATA INFILE '../../std_data/bug20683959loaddata.txt' IGNORE INTO TABLE t1; -Warnings: -Warning 1366 Incorrect string value: '\xF5\x80\x81\xAE\xE7\xB9...' for column 'val' at row 1 -SELECT COUNT(*) FROM t1; -COUNT(*) -1 -SELECT HEX(val) FROM t1; -HEX(val) -C38322525420406E696F757A656368756E3A20E98198E2889A3F3F3F3FE7B99DE4B88AE383A3E7B99DE696903F3F3F3FE7B9A7EFBDA8E7B99DEFBDB3E7B99DE78999E880B3E7B8BAEFBDAAE7B9A7E89699E296A1E7B8BAE4BBA3EFBD8CE7B8BAEFBDA9E7B8B2E2889AE38184E7B99DEFBDB3E7B99DE4B88AE383A3E7B99DE696903F3F3F3FE7B9A7EFBDA8E7B99DEFBDB3E7B99DE5B3A8EFBD84E8ABA0EFBDA8E89C893F3F3F3FE599AAE7B8BAEFBDAAE7B8BAE9A198EFBDA9EFBDB1E7B9A7E581B5E289A0E7B8BAEFBDBEE7B9A7E9A194EFBDA9E882B4EFBDA5EFBDB5E980A73F3F3F3FE28693E99EABE38287E58F99E7B8BAE58AB1E28691E7B8BA3F3F3F3FE7828AE98095EFBDB1E7B8BAEFBDAFE7B8B2E288ABE6A89FE89EB3E6BA983F3F3F3FE88EA0EFBDBAE98095E6BA983F3F3F3FE89D93EFBDBAE8AD9BEFBDACE980A73F3F3F3FE28693E7B8BA3F3F3F3FE288AAE7B8BAE4B88AEFBC9EE7B8BAE4B99DE28691E7B8BA3F3F3F3FEFBCA0E88DB3E6A68AEFBDB9EFBDB3E981B2E5B3A8E296A1E7B8BAE7A4BCE7828AE88DB3E6A68AEFBDB0EFBDBDE7B8BA3FE7B8BAE88B93EFBDBEE5B899EFBC9E -DROP TABLE d1.t1, d2.t1; -DROP DATABASE d1; -DROP DATABASE d2; +CREATE TABLE t1 (a mediumblob NOT NULL) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_general_ci; +LOAD DATA INFILE '../../std_data/loaddata/mdev-11079.txt' INTO TABLE t1 CHARSET utf8 FIELDS TERMINATED BY ';' ENCLOSED BY '"' ESCAPED BY '\\' LINES TERMINATED BY '\n'; +SELECT HEX(a) FROM t1; +HEX(a) +25AAABAC +DROP TABLE t1; +# +# MDEV-11631 LOAD DATA INFILE fails to load data with an escape character followed by a multi-byte character +# +CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET utf8); +LOAD DATA INFILE '../../std_data/loaddata/mdev-11631.txt' INTO TABLE t1 CHARACTER SET utf8; +SELECT HEX(a) FROM t1; +HEX(a) +C3A4 +DROP TABLE t1; diff --git a/mysql-test/r/log_slow.result b/mysql-test/r/log_slow.result index 2e20a78a58c..510bb929d84 100644 --- a/mysql-test/r/log_slow.result +++ b/mysql-test/r/log_slow.result @@ -70,9 +70,9 @@ sleep(0.5) select count(*) FROM mysql.slow_log; count(*) 1 -truncate mysql.slow_log; set @@long_query_time=default; set global slow_query_log= @org_slow_query_log; set @@log_slow_filter=default; set @@log_slow_verbosity=default; set global log_output= default; +truncate mysql.slow_log; diff --git a/mysql-test/r/mysqld--help.result b/mysql-test/r/mysqld--help.result index 1b9d664dbda..2541687a729 100644 --- a/mysql-test/r/mysqld--help.result +++ b/mysql-test/r/mysqld--help.result @@ -477,10 +477,10 @@ The following options may be given as the first argument: --max-sp-recursion-depth[=#] Maximum stored procedure recursion depth --max-statement-time=# - A SELECT query that have taken more than - max_statement_time seconds will be aborted. The argument - will be treated as a decimal value with microsecond - precision. A value of 0 (default) means no timeout + A query that has taken more than max_statement_time + seconds will be aborted. The argument will be treated as + a decimal value with microsecond precision. A value of 0 + (default) means no timeout --max-tmp-tables=# Unused, will be removed. --max-user-connections=# The maximum number of active connections for a single diff --git a/mysql-test/r/order_by.result b/mysql-test/r/order_by.result index 1c1eb359fba..02381ebeed5 100644 --- a/mysql-test/r/order_by.result +++ b/mysql-test/r/order_by.result @@ -2937,6 +2937,17 @@ t2 A, t2 B where A.b = B.b order by A.col2, B.col2 limit 10, 1000000; drop table t1,t2,t3; +# +# mdev-10705 : long order by list that can be skipped +# +SELECT 1 +UNION +( SELECT 2 +ORDER BY NULL, @a0 := 3, @a1 := 3, @a2 := 3, @a3 := 3, @a4 := 3, +@a5 := 3, @a6 := 3, @a7 := 3, @a8 := 3, @a9 := 3, @a10 := 3 ); +1 +1 +2 End of 5.5 tests # # MDEV-5884: EXPLAIN UPDATE ... ORDER BY LIMIT shows wrong #rows diff --git a/mysql-test/r/subselect.result b/mysql-test/r/subselect.result index 1f7edd79029..f3187568428 100644 --- a/mysql-test/r/subselect.result +++ b/mysql-test/r/subselect.result @@ -7166,6 +7166,17 @@ NULL drop view v2; drop table t1,t2; # +# MDEV-10386 Assertion `fixed == 1' failed in virtual String* Item_func_conv_charset::val_str(String*) +# +CREATE TABLE t1 (f1 CHAR(3) CHARACTER SET utf8 NULL, f2 CHAR(3) CHARACTER SET latin1 NULL); +INSERT INTO t1 VALUES ('foo','bar'); +SELECT * FROM t1 WHERE f2 >= SOME ( SELECT f1 FROM t1 ); +f1 f2 +SELECT * FROM t1 WHERE f2 <= SOME ( SELECT f1 FROM t1 ); +f1 f2 +foo bar +DROP TABLE t1; +# # MDEV-9487: Server crashes in Time_and_counter_tracker::incr_loops # with UNION in ALL subquery # diff --git a/mysql-test/r/subselect2.result b/mysql-test/r/subselect2.result index 93204f99808..b8295416e93 100644 --- a/mysql-test/r/subselect2.result +++ b/mysql-test/r/subselect2.result @@ -348,4 +348,49 @@ where t1.a = t2.a and ( t1.a = ( select min(a) from t1 ) or 0 ); a a a FRA FRA FRA drop table t1,t2,t3; +# +# MDEV-10148: Database crashes in the query to the View +# +CREATE TABLE t1 ( +key_code INT(11) NOT NULL, +value_string VARCHAR(50) NULL DEFAULT NULL, +PRIMARY KEY (key_code) +) COLLATE='utf8_general_ci' ENGINE=InnoDB ; +CREATE TABLE t2 ( +key_code INT(11) NOT NULL, +target_date DATE NULL DEFAULT NULL, +PRIMARY KEY (key_code) +) COLLATE='utf8_general_ci' ENGINE=InnoDB ; +CREATE TABLE t3 ( +now_date DATE NOT NULL, +PRIMARY KEY (now_date) +) COLLATE='utf8_general_ci' ENGINE=InnoDB ; +CREATE VIEW v1 +AS +SELECT +B.key_code, +B.target_date +FROM +t2 B INNER JOIN t3 C ON +B.target_date = C.now_date +; +SET @s = 'SELECT A.* FROM t1 A WHERE A.key_code IN (SELECT key_code FROM v1)'; +PREPARE stmt FROM @s; +EXECUTE stmt; +key_code value_string +EXECUTE stmt; +key_code value_string +DEALLOCATE PREPARE stmt; +DROP VIEW v1; +DROP TABLE t1,t2,t3; set optimizer_switch=@subselect2_test_tmp; +create table t1 (a int); +create table t2 (a int); +create table t3(a int); +insert into t1 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +insert into t2 select a from t1; +insert into t3 select a from t1; +select null in (select a from t1 where a < out3.a union select a from t2 where +(select a from t3) +1 < out3.a+1) from t3 out3; +ERROR 21000: Subquery returns more than 1 row +drop table t1, t2, t3; diff --git a/mysql-test/r/subselect4.result b/mysql-test/r/subselect4.result index 792ce657ba0..6bbb80c0662 100644 --- a/mysql-test/r/subselect4.result +++ b/mysql-test/r/subselect4.result @@ -2401,5 +2401,56 @@ SELECT x FROM t1 WHERE id > (SELECT MAX(id) - 1000 FROM t1) ORDER BY x LIMIT 1; x 0 drop table t1; +# +# MDEV-7691: Assertion `outer_context || !*from_field || *from_field == not_found_field' ... +# +set optimizer_switch=default; +CREATE TABLE t1 (a INT) ENGINE=MyISAM; +INSERT INTO t1 VALUES (4),(6); +CREATE TABLE t2 (b INT) ENGINE=MyISAM; +INSERT INTO t2 VALUES (1),(8); +PREPARE stmt FROM " +SELECT * FROM t2 +HAVING 0 IN ( + SELECT a FROM t1 + WHERE a IN ( + SELECT a FROM t1 + WHERE b = a + ) +) +"; +EXECUTE stmt; +b +EXECUTE stmt; +b +# Alternative test case, without HAVING +CREATE TABLE t3 (i INT) ENGINE=MyISAM; +INSERT INTO t3 VALUES (4),(6); +PREPARE stmt FROM " +SELECT * FROM t3 AS t10 +WHERE EXISTS ( + SELECT * FROM t3 AS t20 WHERE t10.i IN ( + SELECT i FROM t3 + ) +)"; +EXECUTE stmt; +i +6 +EXECUTE stmt; +i +6 +drop table t1, t2, t3; SET optimizer_switch= @@global.optimizer_switch; set @@tmp_table_size= @@global.tmp_table_size; +# +# MDEV-10232 Scalar result of subquery changes after adding an outer select stmt +# +create table t1(c1 int, c2 int, primary key(c2)); +insert into t1 values(2,1),(1,2); +select (select c1 from t1 group by c1,c2 order by c1 limit 1) as x; +x +1 +(select c1 from t1 group by c1,c2 order by c1 limit 1); +c1 +1 +drop table t1; diff --git a/mysql-test/r/subselect_no_exists_to_in.result b/mysql-test/r/subselect_no_exists_to_in.result index af023ea12f1..d1cad4f9389 100644 --- a/mysql-test/r/subselect_no_exists_to_in.result +++ b/mysql-test/r/subselect_no_exists_to_in.result @@ -7166,6 +7166,17 @@ NULL drop view v2; drop table t1,t2; # +# MDEV-10386 Assertion `fixed == 1' failed in virtual String* Item_func_conv_charset::val_str(String*) +# +CREATE TABLE t1 (f1 CHAR(3) CHARACTER SET utf8 NULL, f2 CHAR(3) CHARACTER SET latin1 NULL); +INSERT INTO t1 VALUES ('foo','bar'); +SELECT * FROM t1 WHERE f2 >= SOME ( SELECT f1 FROM t1 ); +f1 f2 +SELECT * FROM t1 WHERE f2 <= SOME ( SELECT f1 FROM t1 ); +f1 f2 +foo bar +DROP TABLE t1; +# # MDEV-9487: Server crashes in Time_and_counter_tracker::incr_loops # with UNION in ALL subquery # diff --git a/mysql-test/r/subselect_no_mat.result b/mysql-test/r/subselect_no_mat.result index de116f46079..32d39224e2a 100644 --- a/mysql-test/r/subselect_no_mat.result +++ b/mysql-test/r/subselect_no_mat.result @@ -7159,6 +7159,17 @@ NULL drop view v2; drop table t1,t2; # +# MDEV-10386 Assertion `fixed == 1' failed in virtual String* Item_func_conv_charset::val_str(String*) +# +CREATE TABLE t1 (f1 CHAR(3) CHARACTER SET utf8 NULL, f2 CHAR(3) CHARACTER SET latin1 NULL); +INSERT INTO t1 VALUES ('foo','bar'); +SELECT * FROM t1 WHERE f2 >= SOME ( SELECT f1 FROM t1 ); +f1 f2 +SELECT * FROM t1 WHERE f2 <= SOME ( SELECT f1 FROM t1 ); +f1 f2 +foo bar +DROP TABLE t1; +# # MDEV-9487: Server crashes in Time_and_counter_tracker::incr_loops # with UNION in ALL subquery # diff --git a/mysql-test/r/subselect_no_opts.result b/mysql-test/r/subselect_no_opts.result index dd3fb658f7a..2daf33980d6 100644 --- a/mysql-test/r/subselect_no_opts.result +++ b/mysql-test/r/subselect_no_opts.result @@ -7157,6 +7157,17 @@ NULL drop view v2; drop table t1,t2; # +# MDEV-10386 Assertion `fixed == 1' failed in virtual String* Item_func_conv_charset::val_str(String*) +# +CREATE TABLE t1 (f1 CHAR(3) CHARACTER SET utf8 NULL, f2 CHAR(3) CHARACTER SET latin1 NULL); +INSERT INTO t1 VALUES ('foo','bar'); +SELECT * FROM t1 WHERE f2 >= SOME ( SELECT f1 FROM t1 ); +f1 f2 +SELECT * FROM t1 WHERE f2 <= SOME ( SELECT f1 FROM t1 ); +f1 f2 +foo bar +DROP TABLE t1; +# # MDEV-9487: Server crashes in Time_and_counter_tracker::incr_loops # with UNION in ALL subquery # diff --git a/mysql-test/r/subselect_no_scache.result b/mysql-test/r/subselect_no_scache.result index abe032f7311..09530f3ac0f 100644 --- a/mysql-test/r/subselect_no_scache.result +++ b/mysql-test/r/subselect_no_scache.result @@ -7172,6 +7172,17 @@ NULL drop view v2; drop table t1,t2; # +# MDEV-10386 Assertion `fixed == 1' failed in virtual String* Item_func_conv_charset::val_str(String*) +# +CREATE TABLE t1 (f1 CHAR(3) CHARACTER SET utf8 NULL, f2 CHAR(3) CHARACTER SET latin1 NULL); +INSERT INTO t1 VALUES ('foo','bar'); +SELECT * FROM t1 WHERE f2 >= SOME ( SELECT f1 FROM t1 ); +f1 f2 +SELECT * FROM t1 WHERE f2 <= SOME ( SELECT f1 FROM t1 ); +f1 f2 +foo bar +DROP TABLE t1; +# # MDEV-9487: Server crashes in Time_and_counter_tracker::incr_loops # with UNION in ALL subquery # diff --git a/mysql-test/r/subselect_no_semijoin.result b/mysql-test/r/subselect_no_semijoin.result index 8d2c823a38a..7efb5127d2b 100644 --- a/mysql-test/r/subselect_no_semijoin.result +++ b/mysql-test/r/subselect_no_semijoin.result @@ -7157,6 +7157,17 @@ NULL drop view v2; drop table t1,t2; # +# MDEV-10386 Assertion `fixed == 1' failed in virtual String* Item_func_conv_charset::val_str(String*) +# +CREATE TABLE t1 (f1 CHAR(3) CHARACTER SET utf8 NULL, f2 CHAR(3) CHARACTER SET latin1 NULL); +INSERT INTO t1 VALUES ('foo','bar'); +SELECT * FROM t1 WHERE f2 >= SOME ( SELECT f1 FROM t1 ); +f1 f2 +SELECT * FROM t1 WHERE f2 <= SOME ( SELECT f1 FROM t1 ); +f1 f2 +foo bar +DROP TABLE t1; +# # MDEV-9487: Server crashes in Time_and_counter_tracker::incr_loops # with UNION in ALL subquery # diff --git a/mysql-test/r/trigger_no_defaults-11698.result b/mysql-test/r/trigger_no_defaults-11698.result new file mode 100644 index 00000000000..40546cee41d --- /dev/null +++ b/mysql-test/r/trigger_no_defaults-11698.result @@ -0,0 +1,22 @@ +set sql_mode='strict_all_tables'; +create table t1 (a int not null, b int); +insert t1 (b) values (1); +ERROR HY000: Field 'a' doesn't have a default value +create trigger trgi before insert on t1 for each row +case new.b +when 10 then +set new.a = new.b; +when 30 then +set new.a = new.a; +else +do 1; +end case| +insert t1 (b) values (10); +insert t1 (b) values (20); +ERROR HY000: Field 'a' doesn't have a default value +insert t1 (b) values (30); +select * from t1; +a b +10 10 +0 30 +drop table t1; diff --git a/mysql-test/r/union.result b/mysql-test/r/union.result index 289058f1aab..3f8ad60e452 100644 --- a/mysql-test/r/union.result +++ b/mysql-test/r/union.result @@ -1233,12 +1233,9 @@ a b select * from ((select * from t1 limit 1) union (select * from t1 limit 1)) a; a b 1 a -2 b select * from ((select * from t1 limit 1) union (select * from t1 limit 1) union (select * from t1 limit 1)) a; a b 1 a -2 b -3 c select * from ((((select * from t1))) union (select * from t1) union (select * from t1)) a; a b 1 a @@ -1621,7 +1618,6 @@ NULL UNION RESULT <union1,2> ALL NULL NULL NULL NULL NULL NULL Using filesort Warnings: Note 1003 select NULL AS `a` from `test`.`t1` union select NULL AS `a` from `test`.`t1` order by `a` DROP TABLE t1; -End of 5.0 tests # # Bug#32858: Error: "Incorrect usage of UNION and INTO" does not take # subselects into account @@ -1736,6 +1732,16 @@ a 6 7 8 +(select a from t1 where false) UNION (select a from t1) limit 8; +a +10 +2 +3 +4 +5 +6 +7 +8 drop table t1; # # Bug#11765255 58201: @@ -2031,6 +2037,25 @@ bbbb dddd drop table t1; # +# MDEV-10172: UNION query returns incorrect rows outside +# conditional evaluation +# +create table t1 (d datetime not null primary key); +insert into t1(d) values ('2016-06-01'),('2016-06-02'),('2016-06-03'),('2016-06-04'); +select * from +( +select * from t1 where d between '2016-06-02' and '2016-06-05' + union +(select * from t1 where d < '2016-06-05' order by d desc limit 1) +) onlyJun2toJun4 +order by d; +d +2016-06-02 00:00:00 +2016-06-03 00:00:00 +2016-06-04 00:00:00 +drop table t1; +End of 5.0 tests +# # WL#1763 Avoid creating temporary table in UNION ALL # EXPLAIN SELECT 1 UNION ALL SELECT 1 LIMIT 1 OFFSET 1; diff --git a/mysql-test/std_data/bug20683959loaddata.txt b/mysql-test/std_data/bug20683959loaddata.txt deleted file mode 100644 index 1878cc78879..00000000000 --- a/mysql-test/std_data/bug20683959loaddata.txt +++ /dev/null @@ -1 +0,0 @@ -Ã"RT @niouzechun: é˜âˆšõ€®ç¹ä¸Šãƒ£ç¹æ–õ€‡³ç¹§ï½¨ç¹ï½³ç¹ç‰™è€³ç¸ºï½ªç¹§è–™â–¡ç¸ºä»£ï½Œç¸ºï½©ç¸²âˆšã„ç¹ï½³ç¹ä¸Šãƒ£ç¹æ–õ€‡³ç¹§ï½¨ç¹ï½³ç¹å³¨ï½„è« ï½¨èœ‰õ€”Žå™ªç¸ºï½ªç¸ºé¡˜ï½©ï½±ç¹§åµâ‰ 縺ｾ繧顔ｩ肴･ｵ逧õ€‹–↓鞫ょå™ç¸ºåŠ±â†‘縺õ€‹šç‚Šé€•ï½±ç¸ºï½¯ç¸²âˆ«æ¨Ÿèž³æº˜õ€èŽ ｺ逕溘õ€®è“ｺè›ï½¬é€§õ€‹–↓縺õ€‘Žâˆªç¸ºä¸Šï¼žç¸ºä¹â†‘縺õ€‹–ï¼ è³æ¦Šï½¹ï½³é²å³¨â–¡ç¸ºç¤¼ç‚Šè³æ¦Šï½°ï½½ç¸º 縺苓ｾ帙＞ diff --git a/mysql-test/std_data/loaddata/mdev-11079.txt b/mysql-test/std_data/loaddata/mdev-11079.txt new file mode 100644 index 00000000000..a792f984d5f --- /dev/null +++ b/mysql-test/std_data/loaddata/mdev-11079.txt @@ -0,0 +1 @@ +"%ª«¬" diff --git a/mysql-test/std_data/loaddata/mdev-11631.txt b/mysql-test/std_data/loaddata/mdev-11631.txt new file mode 100644 index 00000000000..87b824b71ae --- /dev/null +++ b/mysql-test/std_data/loaddata/mdev-11631.txt @@ -0,0 +1 @@ +\ä diff --git a/mysql-test/suite/binlog/r/binlog_max_binlog_stmt_cache_size.result b/mysql-test/suite/binlog/r/binlog_max_binlog_stmt_cache_size.result new file mode 100644 index 00000000000..00f01b59732 --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_max_binlog_stmt_cache_size.result @@ -0,0 +1,22 @@ +# +# MDEV-4774: Strangeness with max_binlog_stmt_cache_size Settings +# +CALL mtr.add_suppression("unsigned value 18446744073709547520 adjusted to 4294963200"); +SELECT @@global.max_binlog_stmt_cache_size; +@@global.max_binlog_stmt_cache_size +MAX_BINLOG_STMT_CACHE_SIZE +SET @cache_size= @@max_binlog_stmt_cache_size; +SET @@global.max_binlog_stmt_cache_size= @cache_size+1; +SELECT @@global.max_binlog_stmt_cache_size; +@@global.max_binlog_stmt_cache_size +MAX_BINLOG_STMT_CACHE_SIZE +SET @@global.max_binlog_stmt_cache_size = @cache_size+4095; +SELECT @@global.max_binlog_stmt_cache_size; +@@global.max_binlog_stmt_cache_size +MAX_BINLOG_STMT_CACHE_SIZE +SET @@global.max_binlog_stmt_cache_size= @cache_size-1; +SELECT @@global.max_binlog_stmt_cache_size = @cache_size-4096; +@@global.max_binlog_stmt_cache_size = @cache_size-4096 +1 +SET @@global.max_binlog_stmt_cache_size= @cache_size; +# End of test diff --git a/mysql-test/suite/binlog/t/binlog_max_binlog_stmt_cache_size.opt b/mysql-test/suite/binlog/t/binlog_max_binlog_stmt_cache_size.opt new file mode 100644 index 00000000000..c52ef14d5d0 --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_max_binlog_stmt_cache_size.opt @@ -0,0 +1 @@ +--max_binlog_stmt_cache_size=18446744073709547520 diff --git a/mysql-test/suite/binlog/t/binlog_max_binlog_stmt_cache_size.test b/mysql-test/suite/binlog/t/binlog_max_binlog_stmt_cache_size.test new file mode 100644 index 00000000000..ca3f45c154c --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_max_binlog_stmt_cache_size.test @@ -0,0 +1,36 @@ + +--echo # +--echo # MDEV-4774: Strangeness with max_binlog_stmt_cache_size Settings +--echo # + +CALL mtr.add_suppression("unsigned value 18446744073709547520 adjusted to 4294963200"); + +--replace_result 18446744073709547520 MAX_BINLOG_STMT_CACHE_SIZE 4294963200 MAX_BINLOG_STMT_CACHE_SIZE +SELECT @@global.max_binlog_stmt_cache_size; + +# Save the initial value of @@global.max_binlog_stmt_cache_size. +--replace_result 18446744073709547520 MAX_BINLOG_STMT_CACHE_SIZE 4294963200 MAX_BINLOG_STMT_CACHE_SIZE +SET @cache_size= @@max_binlog_stmt_cache_size; + +--disable_warnings +SET @@global.max_binlog_stmt_cache_size= @cache_size+1; +--enable_warnings +--replace_result 18446744073709547520 MAX_BINLOG_STMT_CACHE_SIZE 4294963200 MAX_BINLOG_STMT_CACHE_SIZE +SELECT @@global.max_binlog_stmt_cache_size; + +--disable_warnings +SET @@global.max_binlog_stmt_cache_size = @cache_size+4095; +--enable_warnings +--replace_result 4294963200 MAX_BINLOG_STMT_CACHE_SIZE 18446744073709547520 MAX_BINLOG_STMT_CACHE_SIZE +SELECT @@global.max_binlog_stmt_cache_size; + +--disable_warnings +SET @@global.max_binlog_stmt_cache_size= @cache_size-1; +--enable_warnings +SELECT @@global.max_binlog_stmt_cache_size = @cache_size-4096; + +# Restore @@global.max_binlog_stmt_cache_size to its initial value. +SET @@global.max_binlog_stmt_cache_size= @cache_size; + +--echo # End of test + diff --git a/mysql-test/suite/binlog_encryption/rpl_special_charset.result b/mysql-test/suite/binlog_encryption/rpl_special_charset.result index 218ced9b8ea..b947cf3484d 100644 --- a/mysql-test/suite/binlog_encryption/rpl_special_charset.result +++ b/mysql-test/suite/binlog_encryption/rpl_special_charset.result @@ -1,6 +1,6 @@ include/master-slave.inc [connection master] -call mtr.add_suppression("Cannot use utf16 as character_set_client"); +call mtr.add_suppression("'utf16' can not be used as client character set"); CREATE TABLE t1(i VARCHAR(20)); INSERT INTO t1 VALUES (0xFFFF); connection slave; diff --git a/mysql-test/suite/encryption/r/innodb-read-only.result b/mysql-test/suite/encryption/r/innodb-read-only.result new file mode 100644 index 00000000000..5d063e129e2 --- /dev/null +++ b/mysql-test/suite/encryption/r/innodb-read-only.result @@ -0,0 +1,3 @@ +# Wait max 10 min for key encryption threads to encrypt all spaces +# Success! +# All done diff --git a/mysql-test/suite/encryption/t/innodb-read-only.opt b/mysql-test/suite/encryption/t/innodb-read-only.opt new file mode 100644 index 00000000000..acedb1a17bb --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb-read-only.opt @@ -0,0 +1,3 @@ +--innodb-encrypt-tables=1 +--innodb-encryption-threads=4 +--innodb-tablespaces-encryption diff --git a/mysql-test/suite/encryption/t/innodb-read-only.test b/mysql-test/suite/encryption/t/innodb-read-only.test new file mode 100644 index 00000000000..10ec87467b6 --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb-read-only.test @@ -0,0 +1,34 @@ +--source suite/encryption/include/have_file_key_management_plugin.inc +--source include/have_innodb.inc +--source include/not_embedded.inc + +--echo # Wait max 10 min for key encryption threads to encrypt all spaces +let $cnt=600; +while ($cnt) +{ + let $success=`SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0`; + if ($success) + { + let $cnt=0; + } + if (!$success) + { + real_sleep 1; + dec $cnt; + } +} +if (!$success) +{ + SELECT * FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION; + SHOW STATUS LIKE 'innodb_encryption%'; + -- die Timeout waiting for encryption threads +} +--echo # Success! + +# +# MDEV-11835: InnoDB: Failing assertion: free_slot != NULL on +# restarting server with encryption and read-only +# +--let $restart_parameters= --innodb-read-only=1 --innodb-encrypt-tables=1 +--source include/restart_mysqld.inc +--echo # All done diff --git a/mysql-test/suite/funcs_2/t/innodb_charset.test b/mysql-test/suite/funcs_2/t/innodb_charset.test index b77bacfc01c..da4dea44ad7 100644 --- a/mysql-test/suite/funcs_2/t/innodb_charset.test +++ b/mysql-test/suite/funcs_2/t/innodb_charset.test @@ -6,6 +6,7 @@ # Checking of other prerequisites is in charset_master.test # ################################################################################ +--source include/no_valgrind_without_big.inc --source include/have_innodb.inc let $engine_type= InnoDB; diff --git a/mysql-test/suite/galera/r/galera_mdev_10812.result b/mysql-test/suite/galera/r/galera_mdev_10812.result new file mode 100644 index 00000000000..fba10004761 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_mdev_10812.result @@ -0,0 +1,11 @@ +# +# MDEV-10812: On COM_STMT_CLOSE/COM_QUIT, when wsrep_conflict_state +# is ABORTED, it causes wrong response to be sent to the client +# +SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; +CREATE TABLE t1(a INT PRIMARY KEY); +INSERT INTO t1 VALUES(1),(2),(3); +START TRANSACTION ; +UPDATE t1 SET a=a+100; +UPDATE t1 SET a=a+100; +DROP TABLE t1; diff --git a/mysql-test/suite/galera/r/galera_var_certify_nonPK_off.result b/mysql-test/suite/galera/r/galera_var_certify_nonPK_off.result index b13302d3ecd..cecec999f37 100644 --- a/mysql-test/suite/galera/r/galera_var_certify_nonPK_off.result +++ b/mysql-test/suite/galera/r/galera_var_certify_nonPK_off.result @@ -24,3 +24,4 @@ connection node_2; SET GLOBAL wsrep_certify_nonPK = 1; DROP TABLE t1; DROP TABLE t2; +call mtr.add_suppression("SQL statement was ineffective"); diff --git a/mysql-test/suite/galera/r/galera_var_cluster_address.result b/mysql-test/suite/galera/r/galera_var_cluster_address.result index ef75cf21a79..251a73a6f9a 100644 --- a/mysql-test/suite/galera/r/galera_var_cluster_address.result +++ b/mysql-test/suite/galera/r/galera_var_cluster_address.result @@ -45,3 +45,4 @@ CALL mtr.add_suppression("gcs connect failed: Connection timed out"); CALL mtr.add_suppression("WSREP: wsrep::connect\\(foo://\\) failed: 7"); disconnect node_2; disconnect node_1; +# End of test diff --git a/mysql-test/suite/galera/r/galera_var_max_ws_rows.result b/mysql-test/suite/galera/r/galera_var_max_ws_rows.result index 6e239c70a3e..a1deb16c5a6 100644 --- a/mysql-test/suite/galera/r/galera_var_max_ws_rows.result +++ b/mysql-test/suite/galera/r/galera_var_max_ws_rows.result @@ -113,3 +113,23 @@ INSERT INTO t1 (f2) VALUES (2); ERROR HY000: wsrep_max_ws_rows exceeded DROP TABLE t1; DROP TABLE ten; +# +# MDEV-11817: Altering a table with more rows than +# wsrep_max_ws_rows causes cluster to break when running +# Galera cluster in TOI mode +# +CREATE TABLE t1(c1 INT)ENGINE = INNODB; +SET GLOBAL wsrep_max_ws_rows= DEFAULT; +INSERT INTO t1 VALUES(1); +INSERT INTO t1 SELECT * FROM t1; +SET GLOBAL wsrep_max_ws_rows= 1; +ALTER TABLE t1 CHANGE COLUMN c1 c1 BIGINT; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c1` bigint(20) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +SELECT COUNT(*) FROM t1; +COUNT(*) +2 +DROP TABLE t1; diff --git a/mysql-test/suite/galera/suite.pm b/mysql-test/suite/galera/suite.pm index 2da2b3ad503..a161301d8ff 100644 --- a/mysql-test/suite/galera/suite.pm +++ b/mysql-test/suite/galera/suite.pm @@ -32,7 +32,6 @@ push @::global_suppressions, qr(WSREP:.*down context.*), qr(WSREP: Failed to send state UUID:), qr(WSREP: last inactive check more than .* skipping check), - qr(WSREP: SQL statement was ineffective), qr(WSREP: Releasing seqno [0-9]* before [0-9]* was assigned.), qr|WSREP: access file\(.*gvwstate.dat\) failed\(No such file or directory\)|, qr(WSREP: Quorum: No node with complete state), @@ -67,6 +66,7 @@ push @::global_suppressions, qr|WSREP: gcs_caused\(\) returned .*|, qr|WSREP: Protocol violation. JOIN message sender .* is not in state transfer \(SYNCED\). Message ignored.|, qr(WSREP: Action message in non-primary configuration from member [0-9]*), + qr(WSREP: discarding established .*), ); diff --git a/mysql-test/suite/galera/t/galera_mdev_10812.test b/mysql-test/suite/galera/t/galera_mdev_10812.test new file mode 100644 index 00000000000..4539ab6c84d --- /dev/null +++ b/mysql-test/suite/galera/t/galera_mdev_10812.test @@ -0,0 +1,27 @@ +--source include/galera_cluster.inc +--source include/have_innodb.inc + +--echo # +--echo # MDEV-10812: On COM_STMT_CLOSE/COM_QUIT, when wsrep_conflict_state +--echo # is ABORTED, it causes wrong response to be sent to the client +--echo # + +# First create a deadlock +--connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1 +SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; +CREATE TABLE t1(a INT PRIMARY KEY); +INSERT INTO t1 VALUES(1),(2),(3); +START TRANSACTION ; +UPDATE t1 SET a=a+100; + +--sleep 2 +--connection node_2 +UPDATE t1 SET a=a+100; + +--sleep 2 +--connection node_1a +# here we get deadlock error +--disconnect node_1a + +--connection node_2 +DROP TABLE t1; diff --git a/mysql-test/suite/galera/t/galera_var_certify_nonPK_off.test b/mysql-test/suite/galera/t/galera_var_certify_nonPK_off.test index 3e8f3d18859..d2d16176e41 100644 --- a/mysql-test/suite/galera/t/galera_var_certify_nonPK_off.test +++ b/mysql-test/suite/galera/t/galera_var_certify_nonPK_off.test @@ -37,3 +37,7 @@ SELECT COUNT(*) = 1 FROM t2 WHERE f1 = 3; DROP TABLE t1; DROP TABLE t2; +--connection node_1 +call mtr.add_suppression("SQL statement was ineffective"); + +--source include/galera_end.inc diff --git a/mysql-test/suite/galera/t/galera_var_cluster_address.test b/mysql-test/suite/galera/t/galera_var_cluster_address.test index 4e5d138ae0a..740c38765f6 100644 --- a/mysql-test/suite/galera/t/galera_var_cluster_address.test +++ b/mysql-test/suite/galera/t/galera_var_cluster_address.test @@ -9,6 +9,7 @@ --let $node_1=node_1 --let $node_2=node_2 --source include/auto_increment_offset_save.inc + # # Set to invalid value # @@ -73,3 +74,4 @@ CALL mtr.add_suppression("WSREP: wsrep::connect\\(foo://\\) failed: 7"); --source include/auto_increment_offset_restore.inc --source include/galera_end.inc +--echo # End of test diff --git a/mysql-test/suite/galera/t/galera_var_max_ws_rows.test b/mysql-test/suite/galera/t/galera_var_max_ws_rows.test index 944238bf1aa..ab6a3390c06 100644 --- a/mysql-test/suite/galera/t/galera_var_max_ws_rows.test +++ b/mysql-test/suite/galera/t/galera_var_max_ws_rows.test @@ -146,10 +146,31 @@ INSERT INTO t1 (f2) VALUES (1); --error ER_ERROR_DURING_COMMIT INSERT INTO t1 (f2) VALUES (2); +DROP TABLE t1; +DROP TABLE ten; + +--echo # +--echo # MDEV-11817: Altering a table with more rows than +--echo # wsrep_max_ws_rows causes cluster to break when running +--echo # Galera cluster in TOI mode +--echo # +--connection node_1 +CREATE TABLE t1(c1 INT)ENGINE = INNODB; +SET GLOBAL wsrep_max_ws_rows= DEFAULT; +INSERT INTO t1 VALUES(1); +INSERT INTO t1 SELECT * FROM t1; +SET GLOBAL wsrep_max_ws_rows= 1; +ALTER TABLE t1 CHANGE COLUMN c1 c1 BIGINT; + +--connection node_2 +SHOW CREATE TABLE t1; +SELECT COUNT(*) FROM t1; +DROP TABLE t1; + +--connection node_1 --disable_query_log --eval SET GLOBAL wsrep_max_ws_rows = $wsrep_max_ws_rows_orig --enable_query_log -DROP TABLE t1; -DROP TABLE ten; +--source include/galera_end.inc diff --git a/mysql-test/suite/innodb/r/innodb-page_compression_default.result b/mysql-test/suite/innodb/r/innodb-page_compression_default.result new file mode 100644 index 00000000000..10e1d6c766c --- /dev/null +++ b/mysql-test/suite/innodb/r/innodb-page_compression_default.result @@ -0,0 +1,20 @@ +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; +create table t1 (c1 int not null primary key auto_increment, b char(200)) engine=innodb page_compressed=1; +insert into t1 values(NULL,'compressed_text_aaaaaaaaabbbbbbbbbbbbbccccccccccccc'); +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +# t1 compressed expected NOT FOUND +NOT FOUND /compressed_text/ in t1.ibd +drop table t1; diff --git a/mysql-test/suite/innodb/t/innodb-page_compression_default.test b/mysql-test/suite/innodb/t/innodb-page_compression_default.test new file mode 100644 index 00000000000..28f184c278c --- /dev/null +++ b/mysql-test/suite/innodb/t/innodb-page_compression_default.test @@ -0,0 +1,51 @@ +--source include/have_innodb.inc + +--disable_query_log +let $innodb_compression_algorithm_orig=`SELECT @@innodb_compression_algorithm`; +let $innodb_file_format_orig = `SELECT @@innodb_file_format`; +let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; +--enable_query_log + +--disable_warnings +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; +--enable_warnings + +create table t1 (c1 int not null primary key auto_increment, b char(200)) engine=innodb page_compressed=1; +insert into t1 values(NULL,'compressed_text_aaaaaaaaabbbbbbbbbbbbbccccccccccccc'); +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; + +let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_PAGE_COMPRESSED'; +--source include/wait_condition.inc + +--let $MYSQLD_DATADIR=`select @@datadir` +--let t1_IBD = $MYSQLD_DATADIR/test/t1.ibd +--let SEARCH_RANGE = 10000000 +--let SEARCH_PATTERN=compressed_text + +--echo # t1 compressed expected NOT FOUND +-- let SEARCH_FILE=$t1_IBD +-- source include/search_pattern_in_file.inc + +drop table t1; + +# reset system +--disable_query_log +--disable_warnings +EVAL SET GLOBAL innodb_compression_algorithm = $innodb_compression_algorithm_orig; +EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; +EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig; +--enable_warnings +--enable_query_log diff --git a/mysql-test/suite/innodb_zip/r/innochecksum_3.result b/mysql-test/suite/innodb_zip/r/innochecksum_3.result index 32e6d3f1d13..da7de031f42 100644 --- a/mysql-test/suite/innodb_zip/r/innochecksum_3.result +++ b/mysql-test/suite/innodb_zip/r/innochecksum_3.result @@ -206,10 +206,10 @@ Filename::tab#.ibd # allow-mismatches,page,start-page,end-page [9]: check the both short and long options "page" and "start-page" when # seek value is larger than file size. -NOT FOUND /Error: Unable to seek to necessary offset: Invalid argument/ in my_restart.err -NOT FOUND /Error: Unable to seek to necessary offset: Invalid argument/ in my_restart.err -NOT FOUND /Error: Unable to seek to necessary offset: Invalid argument/ in my_restart.err -NOT FOUND /Error: Unable to seek to necessary offset: Invalid argument/ in my_restart.err +FOUND /Error: Unable to seek to necessary offset: Invalid argument/ in my_restart.err +FOUND /Error: Unable to seek to necessary offset: Invalid argument/ in my_restart.err +FOUND /Error: Unable to seek to necessary offset: Invalid argument/ in my_restart.err +FOUND /Error: Unable to seek to necessary offset: Invalid argument/ in my_restart.err [34]: check the invalid upper bound values for options, allow-mismatches, end-page, start-page and page. # innochecksum will fail with error code: 1 NOT FOUND /Incorrect unsigned integer value: '18446744073709551616'/ in my_restart.err diff --git a/mysql-test/suite/rpl/r/rpl_row_mysqlbinlog.result b/mysql-test/suite/rpl/r/rpl_row_mysqlbinlog.result index 5c3bcf9070c..b86743507d8 100644 --- a/mysql-test/suite/rpl/r/rpl_row_mysqlbinlog.result +++ b/mysql-test/suite/rpl/r/rpl_row_mysqlbinlog.result @@ -1,15 +1,21 @@ include/master-slave.inc [connection master] - ---Setup Section -- set timestamp=1000000000; -DROP TABLE IF EXISTS t1,t2,t3; -connection master; CREATE TABLE t1(word VARCHAR(20)); CREATE TABLE t2(id INT AUTO_INCREMENT NOT NULL PRIMARY KEY); CREATE TABLE t3(c1 INT NOT NULL PRIMARY KEY, c2 LONGBLOB, c3 TIMESTAMP, c4 TEXT, c5 FLOAT); - ----Test1 check table load -- +INSERT INTO t1 VALUES ("abirvalg"); +LOAD DATA INFILE '../../std_data/words.dat' INTO TABLE t1; +LOAD DATA INFILE '../../std_data/words.dat' INTO TABLE t1; +LOAD DATA INFILE '../../std_data/words.dat' INTO TABLE t1; +LOAD DATA INFILE '../../std_data/words.dat' INTO TABLE t1; +LOAD DATA INFILE '../../std_data/words.dat' INTO TABLE t1; +set @d1 = 'dd1'; +set @d1 = concat(@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1); +set @d1 = concat(@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1); +set @d1 = concat(@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1); +---Test 1 check table load -- SELECT COUNT(*) from t1; COUNT(*) 351 @@ -74,9 +80,7 @@ c1 c3 c4 c5 connection master; insert into t1 values ("Alas"); flush logs; - --- Test 1 Dump binlog to file -- - --- Test 1 delete tables, clean master and slave -- DROP TABLE t1; DROP TABLE t2; @@ -91,9 +95,7 @@ reset slave; start slave; include/wait_for_slave_to_start.inc connection master; - --- Test 1 Load from Dump binlog file -- - --- Test 1 Check Load Results -- SELECT COUNT(*) from t1; COUNT(*) @@ -157,7 +159,6 @@ c1 c3 c4 c5 4 2006-02-22 00:00:00 Tested in Texas 8.8 5 2006-02-22 00:00:00 Tested in Texas 11 connection master; - --- Test 2 position test -- /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/; /*!40019 SET @@session.max_insert_delayed_threads=0*/; @@ -168,7 +169,7 @@ use `test`/*!*/; SET TIMESTAMP=1000000000/*!*/; SET @@session.pseudo_thread_id=999999999/*!*/; SET @@session.foreign_key_checks=1, @@session.sql_auto_is_null=0, @@session.unique_checks=1, @@session.autocommit=1, @@session.check_constraint_checks=1/*!*/; -SET @@session.sql_mode=0/*!*/; +SET @@session.sql_mode=1342177280/*!*/; SET @@session.auto_increment_increment=1, @@session.auto_increment_offset=1/*!*/; /*!\C latin1 *//*!*/; SET @@session.character_set_client=8,@@session.collation_connection=8,@@session.collation_server=8/*!*/; @@ -181,7 +182,6 @@ DELIMITER ; ROLLBACK /* added by mysqlbinlog */; /*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/; /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/; - --- Test 3 First Remote test -- /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/; /*!40019 SET @@session.max_insert_delayed_threads=0*/; @@ -192,15 +192,12 @@ use `test`/*!*/; SET TIMESTAMP=1000000000/*!*/; SET @@session.pseudo_thread_id=999999999/*!*/; SET @@session.foreign_key_checks=1, @@session.sql_auto_is_null=0, @@session.unique_checks=1, @@session.autocommit=1, @@session.check_constraint_checks=1/*!*/; -SET @@session.sql_mode=0/*!*/; +SET @@session.sql_mode=1342177280/*!*/; SET @@session.auto_increment_increment=1, @@session.auto_increment_offset=1/*!*/; /*!\C latin1 *//*!*/; SET @@session.character_set_client=8,@@session.collation_connection=8,@@session.collation_server=8/*!*/; SET @@session.lc_time_names=0/*!*/; SET @@session.collation_database=DEFAULT/*!*/; -DROP TABLE IF EXISTS `t1`,`t2`,`t3` /* generated by server */ -/*!*/; -SET TIMESTAMP=1000000000/*!*/; CREATE TABLE t1(word VARCHAR(20)) /*!*/; SET TIMESTAMP=1000000000/*!*/; @@ -214,7 +211,6 @@ DELIMITER ; ROLLBACK /* added by mysqlbinlog */; /*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/; /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/; - --- Test 4 Second Remote test -- DROP TABLE t1; DROP TABLE t2; @@ -291,7 +287,6 @@ c1 c3 c4 c5 4 2006-02-22 00:00:00 Tested in Texas 8.8 5 2006-02-22 00:00:00 Tested in Texas 11 connection master; - --- Test 5 LOAD DATA -- /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/; /*!40019 SET @@session.max_insert_delayed_threads=0*/; @@ -302,7 +297,6 @@ DELIMITER ; ROLLBACK /* added by mysqlbinlog */; /*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/; /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/; - --- Test 6 reading stdin -- /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/; /*!40019 SET @@session.max_insert_delayed_threads=0*/; @@ -313,15 +307,12 @@ use `test`/*!*/; SET TIMESTAMP=1000000000/*!*/; SET @@session.pseudo_thread_id=999999999/*!*/; SET @@session.foreign_key_checks=1, @@session.sql_auto_is_null=0, @@session.unique_checks=1, @@session.autocommit=1, @@session.check_constraint_checks=1/*!*/; -SET @@session.sql_mode=0/*!*/; +SET @@session.sql_mode=1342177280/*!*/; SET @@session.auto_increment_increment=1, @@session.auto_increment_offset=1/*!*/; /*!\C latin1 *//*!*/; SET @@session.character_set_client=8,@@session.collation_connection=8,@@session.collation_server=8/*!*/; SET @@session.lc_time_names=0/*!*/; SET @@session.collation_database=DEFAULT/*!*/; -DROP TABLE IF EXISTS `t1`,`t2`,`t3` /* generated by server */ -/*!*/; -SET TIMESTAMP=1000000000/*!*/; CREATE TABLE t1(word VARCHAR(20)) /*!*/; SET TIMESTAMP=1000000000/*!*/; @@ -335,7 +326,6 @@ DELIMITER ; ROLLBACK /* added by mysqlbinlog */; /*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/; /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/; - --- Test 7 reading stdin w/position -- /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/; /*!40019 SET @@session.max_insert_delayed_threads=0*/; @@ -346,7 +336,7 @@ use `test`/*!*/; SET TIMESTAMP=1000000000/*!*/; SET @@session.pseudo_thread_id=999999999/*!*/; SET @@session.foreign_key_checks=1, @@session.sql_auto_is_null=0, @@session.unique_checks=1, @@session.autocommit=1, @@session.check_constraint_checks=1/*!*/; -SET @@session.sql_mode=0/*!*/; +SET @@session.sql_mode=1342177280/*!*/; SET @@session.auto_increment_increment=1, @@session.auto_increment_offset=1/*!*/; /*!\C latin1 *//*!*/; SET @@session.character_set_client=8,@@session.collation_connection=8,@@session.collation_server=8/*!*/; @@ -359,7 +349,6 @@ DELIMITER ; ROLLBACK /* added by mysqlbinlog */; /*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/; /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/; - --- Test 8 switch internal charset -- connection slave; stop slave; @@ -400,18 +389,17 @@ HEX(f) select HEX(f) from t5; HEX(f) 835C - --- Test cleanup -- connection master; connection slave; connection master; -DROP TABLE IF EXISTS t1; +DROP TABLE t1, t2, t3, t04, t05, t4, t5; CREATE TABLE t1 (a INT NOT NULL KEY, b INT); INSERT INTO t1 VALUES(1,1); SELECT * FROM t1; a b 1 1 FLUSH LOGS; -DROP TABLE IF EXISTS t1, t2, t3, t04, t05, t4, t5; +DROP TABLE t1; connection slave; include/rpl_end.inc diff --git a/mysql-test/suite/rpl/r/rpl_special_charset.result b/mysql-test/suite/rpl/r/rpl_special_charset.result index 218ced9b8ea..b947cf3484d 100644 --- a/mysql-test/suite/rpl/r/rpl_special_charset.result +++ b/mysql-test/suite/rpl/r/rpl_special_charset.result @@ -1,6 +1,6 @@ include/master-slave.inc [connection master] -call mtr.add_suppression("Cannot use utf16 as character_set_client"); +call mtr.add_suppression("'utf16' can not be used as client character set"); CREATE TABLE t1(i VARCHAR(20)); INSERT INTO t1 VALUES (0xFFFF); connection slave; diff --git a/mysql-test/suite/rpl/r/rpl_strict_password_validation.result b/mysql-test/suite/rpl/r/rpl_strict_password_validation.result new file mode 100644 index 00000000000..579f576d881 --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_strict_password_validation.result @@ -0,0 +1,17 @@ +include/master-slave.inc +[connection master] +connection slave; +install soname "simple_password_check"; +select @@strict_password_validation; +@@strict_password_validation +1 +connection master; +create user foo1 identified by password '11111111111111111111111111111111111111111'; +set password for foo1 = PASSWORD('PLAINtext-password!!99'); +drop user foo1; +connection slave; +connection slave; +create user foo1 identified by password '11111111111111111111111111111111111111111'; +ERROR HY000: The MariaDB server is running with the --strict-password-validation option so it cannot execute this statement +uninstall plugin simple_password_check; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_row_mysqlbinlog.test b/mysql-test/suite/rpl/t/rpl_row_mysqlbinlog.test index a44cc18c0ed..678679f0cf1 100644 --- a/mysql-test/suite/rpl/t/rpl_row_mysqlbinlog.test +++ b/mysql-test/suite/rpl/t/rpl_row_mysqlbinlog.test @@ -4,44 +4,27 @@ # Purpose: To test changes to mysqlbinlog for row based bin logs # # We are using .opt file since we need small binlog size # ################################################################## -# Include Section -# Make sure that we have row based bin log -- source include/have_binlog_format_row.inc -# Embedded server doesn't support binlogging -- source include/not_embedded.inc -# This test requires the cp932 charset compiled in -- source include/have_cp932.inc -# Slow test, don't run during staging part --- source include/not_staging.inc - -- source include/master-slave.inc -# Setup Section -# we need this for getting fixed timestamps inside of this test - ---disable_query_log -select "---Setup Section --" as ""; -set sql_mode=""; ---enable_query_log +--echo ---Setup Section -- +# we need this for getting fixed timestamps inside of this test set timestamp=1000000000; ---disable_warnings -DROP TABLE IF EXISTS t1,t2,t3; ---enable_warnings - -connection master; CREATE TABLE t1(word VARCHAR(20)); CREATE TABLE t2(id INT AUTO_INCREMENT NOT NULL PRIMARY KEY); ---let $position= query_get_value(SHOW MASTER STATUS, Position, 1) +--let position= query_get_value(SHOW MASTER STATUS, Position, 1) CREATE TABLE t3(c1 INT NOT NULL PRIMARY KEY, c2 LONGBLOB, c3 TIMESTAMP, c4 TEXT, c5 FLOAT); ---let $stop_position=query_get_value(SHOW MASTER STATUS, Position, 1) ---let $stop_position1=`select $stop_position - 1` ---let $binlog_start_pos=query_get_value(SHOW BINLOG EVENTS LIMIT 1, End_log_pos, 1) +--let stop_position=query_get_value(SHOW MASTER STATUS, Position, 1) +--let stop_position1=`select $stop_position - 1` +--let binlog_start_pos=query_get_value(SHOW BINLOG EVENTS LIMIT 1, End_log_pos, 1) + # Test Section # Lets start by putting some data into the tables. ---disable_query_log INSERT INTO t1 VALUES ("abirvalg"); LOAD DATA INFILE '../../std_data/words.dat' INTO TABLE t1; LOAD DATA INFILE '../../std_data/words.dat' INTO TABLE t1; @@ -55,7 +38,8 @@ set @d1 = concat(@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1); set @d1 = concat(@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1); set @d1 = concat(@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1); -let $count=500; +--disable_query_log +let count=500; while ($count) { INSERT INTO t2 VALUES (NULL); @@ -64,10 +48,7 @@ while ($count) } --enable_query_log - ---disable_query_log -select "---Test1 check table load --" as ""; ---enable_query_log +--echo ---Test 1 check table load -- # Lets Check the tables on the Master SELECT COUNT(*) from t1; @@ -96,34 +77,26 @@ insert into t1 values ("Alas"); flush logs; # delimiters are for easier debugging in future ---disable_query_log -select "--- Test 1 Dump binlog to file --" as ""; ---enable_query_log +--echo --- Test 1 Dump binlog to file -- # # Prepare local temporary file to recreate what we have currently. -let $MYSQLD_DATADIR= `select @@datadir;`; +let MYSQLD_DATADIR= `select @@datadir;`; --exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 > $MYSQLTEST_VARDIR/tmp/master.sql --exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000002 >> $MYSQLTEST_VARDIR/tmp/master.sql # Now that we have our file, lets get rid of the current database. # Cleanup the master and the slave and try to recreate. ---disable_query_log -select "--- Test 1 delete tables, clean master and slave --" as ""; ---enable_query_log +--echo --- Test 1 delete tables, clean master and slave -- DROP TABLE t1; DROP TABLE t2; DROP TABLE t3; sync_slave_with_master; -#we expect STOP SLAVE to produce a warning as the slave is stopped -#(the server was started with skip-slave-start) ---disable_warnings stop slave; --source include/wait_for_slave_to_stop.inc ---enable_warnings connection master; reset master; connection slave; @@ -133,15 +106,11 @@ start slave; connection master; # We should be clean at this point, now we will run in the file from above. ---disable_query_log -select "--- Test 1 Load from Dump binlog file --" as ""; ---enable_query_log +--echo --- Test 1 Load from Dump binlog file -- --exec $MYSQL -e "source $MYSQLTEST_VARDIR/tmp/master.sql" ---disable_query_log -select "--- Test 1 Check Load Results --" as ""; ---enable_query_log +--echo --- Test 1 Check Load Results -- # Lets Check the tables on the Master SELECT COUNT(*) from t1; @@ -169,28 +138,20 @@ remove_file $MYSQLTEST_VARDIR/tmp/master.sql; # this test for start-position option # By setting this position to 416, we should only get the create of t3 ---disable_query_log -select "--- Test 2 position test --" as ""; ---enable_query_log -let $MYSQLD_DATADIR= `select @@datadir;`; +--echo --- Test 2 position test -- --exec $MYSQL_BINLOG --short-form --local-load=$MYSQLTEST_VARDIR/tmp/ --start-position=$position --stop-position=$stop_position $MYSQLD_DATADIR/master-bin.000001 # These are tests for remote binlog. # They should return the same as previous test. ---disable_query_log -select "--- Test 3 First Remote test --" as ""; ---enable_query_log +--echo --- Test 3 First Remote test -- # This is broken now --replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR --exec $MYSQL_BINLOG --short-form --local-load=$MYSQLTEST_VARDIR/tmp/ --stop-position=$stop_position --read-from-remote-server --user=root --host=127.0.0.1 --port=$MASTER_MYPORT master-bin.000001 ---disable_query_log -select "--- Test 4 Second Remote test --" as ""; ---enable_query_log +--echo --- Test 4 Second Remote test -- --exec $MYSQL_BINLOG --read-from-remote-server --user=root --host=127.0.0.1 --port=$MASTER_MYPORT master-bin.000001 > $MYSQLTEST_VARDIR/tmp/remote.sql - --exec $MYSQL_BINLOG --read-from-remote-server --user=root --host=127.0.0.1 --port=$MASTER_MYPORT master-bin.000002 >> $MYSQLTEST_VARDIR/tmp/remote.sql # Now that we have our file, lets get rid of the current database. @@ -202,13 +163,8 @@ DROP TABLE t3; sync_slave_with_master; -#we expect STOP SLAVE to produce a warning as the slave is stopped -#(the server was started with skip-slave-start) - ---disable_warnings stop slave; --source include/wait_for_slave_to_stop.inc ---enable_warnings connection master; reset master; connection slave; @@ -252,40 +208,26 @@ connection master; # transactions. /Matz # LOAD DATA ---disable_query_log -select "--- Test 5 LOAD DATA --" as ""; ---enable_query_log +--echo --- Test 5 LOAD DATA -- --replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR --exec $MYSQL_BINLOG --short-form --local-load=$MYSQLTEST_VARDIR/tmp/ --stop-position=$binlog_start_pos --read-from-remote-server --user=root --host=127.0.0.1 --port=$MASTER_MYPORT master-bin.000002 # Bug#7853 (mysqlbinlog does not accept input from stdin) ---disable_query_log -select "--- Test 6 reading stdin --" as ""; ---enable_query_log -let $MYSQLD_DATADIR= `select @@datadir;`; +--echo --- Test 6 reading stdin -- --replace_result $MYSQL_TEST_DIR MYSQL_TEST_DIR --exec $MYSQL_BINLOG --short-form --stop-position=$stop_position1 - < $MYSQLD_DATADIR/master-bin.000001 ---disable_query_log -select "--- Test 7 reading stdin w/position --" as ""; ---enable_query_log +--echo --- Test 7 reading stdin w/position -- --replace_result $MYSQL_TEST_DIR MYSQL_TEST_DIR --exec $MYSQL_BINLOG --short-form --start-position=$position --stop-position=$stop_position - < $MYSQLD_DATADIR/master-bin.000001 # Bug#16217 (mysql client did not know how not switch its internal charset) ---disable_query_log -select "--- Test 8 switch internal charset --" as ""; ---enable_query_log +--echo --- Test 8 switch internal charset -- sync_slave_with_master; -#we expect STOP SLAVE to produce a warning as the slave is stopped -#(the server was started with skip-slave-start) - ---disable_warnings stop slave; --source include/wait_for_slave_to_stop.inc ---enable_warnings connection master; reset master; connection slave; @@ -298,7 +240,6 @@ create table t4 (f text character set utf8); create table t5 (f text character set cp932); --exec $MYSQL --default-character-set=utf8 test -e "insert into t4 values(_utf8'ソ')" --exec $MYSQL --default-character-set=cp932 test -e "insert into t5 values(_cp932'ƒ\');" -let $MYSQLD_DATADIR= `select @@datadir;`; flush logs; rename table t4 to t04, t5 to t05; --exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 | $MYSQL --default-character-set=utf8 @@ -315,42 +256,30 @@ select HEX(f) from t4; select HEX(f) from t05; select HEX(f) from t5; ---disable_query_log -select "--- Test cleanup --" as ""; ---enable_query_log +--echo --- Test cleanup -- # clean up connection master; sync_slave_with_master; connection master; +DROP TABLE t1, t2, t3, t04, t05, t4, t5; # BUG#17654 also test mysqlbinlog to ensure it can read the binlog from a remote server # and ensure that the results are the same as if read from a file (the same file). ---disable_warnings -DROP TABLE IF EXISTS t1; ---enable_warnings - CREATE TABLE t1 (a INT NOT NULL KEY, b INT); - INSERT INTO t1 VALUES(1,1); - SELECT * FROM t1; - -let $MYSQLD_DATADIR= `select @@datadir;`; - FLUSH LOGS; --exec $MYSQL_BINLOG --read-from-remote-server --user=root --host=127.0.0.1 --port=$MASTER_MYPORT master-bin.000001 > $MYSQLTEST_VARDIR/tmp/remote.sql --exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 > $MYSQLTEST_VARDIR/tmp/local.sql --diff_files $MYSQLTEST_VARDIR/tmp/local.sql $MYSQLTEST_VARDIR/tmp/remote.sql - --remove_file $MYSQLTEST_VARDIR/tmp/remote.sql - --remove_file $MYSQLTEST_VARDIR/tmp/local.sql +DROP TABLE t1; -DROP TABLE IF EXISTS t1, t2, t3, t04, t05, t4, t5; sync_slave_with_master; # End of 4.1 tests diff --git a/mysql-test/suite/rpl/t/rpl_strict_password_validation.test b/mysql-test/suite/rpl/t/rpl_strict_password_validation.test new file mode 100644 index 00000000000..c4dda1e1269 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_strict_password_validation.test @@ -0,0 +1,24 @@ +if (!$SIMPLE_PASSWORD_CHECK_SO) { + skip No SIMPLE_PASSWORD_CHECK plugin; +} + +--source include/master-slave.inc + + +--connection slave +install soname "simple_password_check"; +select @@strict_password_validation; + +--connection master +create user foo1 identified by password '11111111111111111111111111111111111111111'; +set password for foo1 = PASSWORD('PLAINtext-password!!99'); +drop user foo1; +--sync_slave_with_master + +--connection slave +--error ER_OPTION_PREVENTS_STATEMENT +create user foo1 identified by password '11111111111111111111111111111111111111111'; + +uninstall plugin simple_password_check; + +--source include/rpl_end.inc diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb,32bit.rdiff b/mysql-test/suite/sys_vars/r/sysvars_innodb,32bit.rdiff index ed57382a2a9..2fd22767dce 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb,32bit.rdiff +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb,32bit.rdiff @@ -1,6 +1,6 @@ ---- ./suite/sys_vars/r/sysvars_innodb.result 2017-01-03 12:06:25.401683053 +0200 -+++ ./suite/sys_vars/r/sysvars_innodb,32bit.reject 2017-01-04 18:26:46.741752657 +0200 -@@ -53,7 +53,7 @@ +--- suite/sys_vars/r/sysvars_innodb.result ++++ suite/sys_vars/r/sysvars_innodb.result +@@ -54,7 +54,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 8 VARIABLE_SCOPE GLOBAL @@ -9,7 +9,7 @@ VARIABLE_COMMENT Number of InnoDB Adapative Hash Index Partitions. (default = 8). NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 512 -@@ -67,7 +67,7 @@ +@@ -68,7 +68,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 150000 VARIABLE_SCOPE GLOBAL @@ -18,7 +18,7 @@ VARIABLE_COMMENT The upper limit of the sleep delay in usec. Value of 0 disables it. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 1000000 -@@ -81,7 +81,7 @@ +@@ -82,7 +82,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 5 VARIABLE_SCOPE GLOBAL @@ -27,7 +27,7 @@ VARIABLE_COMMENT Background commit interval in seconds NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 1073741824 -@@ -137,7 +137,7 @@ +@@ -138,7 +138,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -36,7 +36,7 @@ VARIABLE_COMMENT InnoDB API transaction isolation level NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 3 -@@ -151,7 +151,7 @@ +@@ -152,7 +152,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 64 VARIABLE_SCOPE GLOBAL @@ -45,7 +45,7 @@ VARIABLE_COMMENT Data file autoextend increment in megabytes NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 1000 -@@ -165,7 +165,7 @@ +@@ -166,7 +166,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 1 VARIABLE_SCOPE GLOBAL @@ -54,7 +54,7 @@ VARIABLE_COMMENT The AUTOINC lock modes supported by InnoDB: 0 => Old style AUTOINC locking (for backward compatibility); 1 => New style AUTOINC locking; 2 => No AUTOINC locking (unsafe for SBR) NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 2 -@@ -249,10 +249,10 @@ +@@ -250,10 +250,10 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 134217728 VARIABLE_SCOPE GLOBAL @@ -67,7 +67,7 @@ NUMERIC_BLOCK_SIZE 1048576 ENUM_VALUE_LIST NULL READ_ONLY YES -@@ -291,7 +291,7 @@ +@@ -292,7 +292,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 25 VARIABLE_SCOPE GLOBAL @@ -76,7 +76,7 @@ VARIABLE_COMMENT Dump only the hottest N% of each buffer pool, defaults to 25 NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 100 -@@ -333,7 +333,7 @@ +@@ -334,7 +334,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -85,7 +85,7 @@ VARIABLE_COMMENT Number of buffer pool instances, set to higher value on high-end machines to increase scalability NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 64 -@@ -403,7 +403,7 @@ +@@ -404,7 +404,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -94,7 +94,7 @@ VARIABLE_COMMENT A number between [0, 100] that tells how oftern buffer pool dump status in percentages should be printed. E.g. 10 means that buffer pool dump status is printed when every 10% of number of buffer pool pages are dumped. Default is 0 (only start and end status is printed). NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 100 -@@ -515,7 +515,7 @@ +@@ -516,7 +516,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -103,7 +103,7 @@ VARIABLE_COMMENT Helps in performance tuning in heavily concurrent environments. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 1000 -@@ -557,7 +557,7 @@ +@@ -558,7 +558,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 5 VARIABLE_SCOPE GLOBAL @@ -112,7 +112,7 @@ VARIABLE_COMMENT If the compression failure rate of a table is greater than this number more padding is added to the pages to reduce the failures. A value of zero implies no padding NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 100 -@@ -585,7 +585,7 @@ +@@ -586,7 +586,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 50 VARIABLE_SCOPE GLOBAL @@ -121,7 +121,7 @@ VARIABLE_COMMENT Percentage of empty space on a data page that can be reserved to make the page compressible. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 75 -@@ -613,10 +613,10 @@ +@@ -600,10 +600,10 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 5000 VARIABLE_SCOPE GLOBAL @@ -134,7 +134,7 @@ NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -865,7 +865,7 @@ +@@ -852,7 +852,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 120 VARIABLE_SCOPE GLOBAL @@ -143,7 +143,7 @@ VARIABLE_COMMENT Number of pages reserved in doublewrite buffer for batch flushing NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 127 -@@ -949,7 +949,7 @@ +@@ -936,7 +936,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 1 VARIABLE_SCOPE GLOBAL @@ -152,7 +152,7 @@ VARIABLE_COMMENT Speeds up the shutdown process of the InnoDB storage engine. Possible values are 0, 1 (faster) or 2 (fastest - crash-like). NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 2 -@@ -963,7 +963,7 @@ +@@ -950,7 +950,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 600 VARIABLE_SCOPE GLOBAL @@ -161,7 +161,7 @@ VARIABLE_COMMENT Maximum number of seconds that semaphore times out in InnoDB. NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 4294967295 -@@ -1033,7 +1033,7 @@ +@@ -1020,7 +1020,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 100 VARIABLE_SCOPE GLOBAL @@ -170,7 +170,7 @@ VARIABLE_COMMENT Percentage of B-tree page filled during bulk insert NUMERIC_MIN_VALUE 10 NUMERIC_MAX_VALUE 100 -@@ -1047,7 +1047,7 @@ +@@ -1034,7 +1034,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -179,7 +179,7 @@ VARIABLE_COMMENT Make the first page of the given tablespace dirty. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 4294967295 -@@ -1061,7 +1061,7 @@ +@@ -1048,7 +1048,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 30 VARIABLE_SCOPE GLOBAL @@ -188,7 +188,7 @@ VARIABLE_COMMENT Number of iterations over which the background flushing is averaged. NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 1000 -@@ -1089,7 +1089,7 @@ +@@ -1076,7 +1076,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 1 VARIABLE_SCOPE GLOBAL @@ -197,7 +197,7 @@ VARIABLE_COMMENT Controls the durability/speed trade-off for commits. Set to 0 (write and flush redo log to disk only once per second), 1 (flush to disk at each commit), 2 (write to log at commit but flush to disk only once per second) or 3 (flush to disk at prepare and at commit, slower and usually redundant). 1 and 3 guarantees that after a crash, committed transactions will not be lost and will be consistent with the binlog and other transactional engines. 2 can get inconsistent and lose transactions if there is a power failure or kernel crash but not if mysqld crashes. 0 has no guarantees in case of crash. 0 and 2 can be faster than 1 or 3. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 3 -@@ -1117,7 +1117,7 @@ +@@ -1104,7 +1104,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 1 VARIABLE_SCOPE GLOBAL @@ -206,7 +206,7 @@ VARIABLE_COMMENT Set to 0 (don't flush neighbors from buffer pool), 1 (flush contiguous neighbors from buffer pool) or 2 (flush neighbors from buffer pool), when flushing a block NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 2 -@@ -1173,7 +1173,7 @@ +@@ -1160,7 +1160,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -215,16 +215,7 @@ VARIABLE_COMMENT Helps to save your data in case the disk image of the database becomes corrupt. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 6 -@@ -1187,7 +1187,7 @@ - GLOBAL_VALUE_ORIGIN COMPILE-TIME - DEFAULT_VALUE 0 - VARIABLE_SCOPE GLOBAL --VARIABLE_TYPE BIGINT UNSIGNED -+VARIABLE_TYPE INT UNSIGNED - VARIABLE_COMMENT Kills the server during crash recovery. - NUMERIC_MIN_VALUE 0 - NUMERIC_MAX_VALUE 100 -@@ -1215,7 +1215,7 @@ +@@ -1188,7 +1188,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 8000000 VARIABLE_SCOPE GLOBAL @@ -233,7 +224,7 @@ VARIABLE_COMMENT InnoDB Fulltext search cache size in bytes NUMERIC_MIN_VALUE 1600000 NUMERIC_MAX_VALUE 80000000 -@@ -1257,7 +1257,7 @@ +@@ -1230,7 +1230,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 84 VARIABLE_SCOPE GLOBAL @@ -242,7 +233,7 @@ VARIABLE_COMMENT InnoDB Fulltext search maximum token size in characters NUMERIC_MIN_VALUE 10 NUMERIC_MAX_VALUE 84 -@@ -1271,7 +1271,7 @@ +@@ -1244,7 +1244,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 3 VARIABLE_SCOPE GLOBAL @@ -251,7 +242,7 @@ VARIABLE_COMMENT InnoDB Fulltext search minimum token size in characters NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 16 -@@ -1285,7 +1285,7 @@ +@@ -1258,7 +1258,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 2000 VARIABLE_SCOPE GLOBAL @@ -260,7 +251,7 @@ VARIABLE_COMMENT InnoDB Fulltext search number of words to optimize for each optimize table call NUMERIC_MIN_VALUE 1000 NUMERIC_MAX_VALUE 10000 -@@ -1299,7 +1299,7 @@ +@@ -1272,7 +1272,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 2000000000 VARIABLE_SCOPE GLOBAL @@ -269,7 +260,7 @@ VARIABLE_COMMENT InnoDB Fulltext search query result cache limit in bytes NUMERIC_MIN_VALUE 1000000 NUMERIC_MAX_VALUE 4294967295 -@@ -1327,7 +1327,7 @@ +@@ -1300,7 +1300,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 2 VARIABLE_SCOPE GLOBAL @@ -278,7 +269,7 @@ VARIABLE_COMMENT InnoDB Fulltext search parallel sort degree, will round up to nearest power of 2 number NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 16 -@@ -1341,7 +1341,7 @@ +@@ -1314,7 +1314,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 640000000 VARIABLE_SCOPE GLOBAL @@ -287,7 +278,7 @@ VARIABLE_COMMENT Total memory allocated for InnoDB Fulltext Search cache NUMERIC_MIN_VALUE 32000000 NUMERIC_MAX_VALUE 1600000000 -@@ -1369,7 +1369,7 @@ +@@ -1342,7 +1342,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 100 VARIABLE_SCOPE GLOBAL @@ -296,7 +287,7 @@ VARIABLE_COMMENT Up to what percentage of dirty pages should be flushed when innodb finds it has spare resources to do so. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 100 -@@ -1411,10 +1411,10 @@ +@@ -1384,10 +1384,10 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 200 VARIABLE_SCOPE GLOBAL @@ -309,7 +300,7 @@ NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -1423,12 +1423,12 @@ +@@ -1396,12 +1396,12 @@ SESSION_VALUE NULL GLOBAL_VALUE 2000 GLOBAL_VALUE_ORIGIN COMPILE-TIME @@ -325,7 +316,7 @@ NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -1495,7 +1495,7 @@ +@@ -1468,7 +1468,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 50 VARIABLE_SCOPE SESSION @@ -334,7 +325,7 @@ VARIABLE_COMMENT Timeout in seconds an InnoDB transaction may wait for a lock before being rolled back. Values above 100000000 disable the timeout. NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 1073741824 -@@ -1509,10 +1509,10 @@ +@@ -1482,10 +1482,10 @@ GLOBAL_VALUE_ORIGIN CONFIG DEFAULT_VALUE 16777216 VARIABLE_SCOPE GLOBAL @@ -347,7 +338,7 @@ NUMERIC_BLOCK_SIZE 1024 ENUM_VALUE_LIST NULL READ_ONLY YES -@@ -1565,7 +1565,7 @@ +@@ -1538,7 +1538,7 @@ GLOBAL_VALUE_ORIGIN CONFIG DEFAULT_VALUE 2 VARIABLE_SCOPE GLOBAL @@ -356,7 +347,7 @@ VARIABLE_COMMENT Number of log files in the log group. InnoDB writes to the files in a circular fashion. NUMERIC_MIN_VALUE 2 NUMERIC_MAX_VALUE 100 -@@ -1607,7 +1607,7 @@ +@@ -1580,7 +1580,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 8192 VARIABLE_SCOPE GLOBAL @@ -365,7 +356,7 @@ VARIABLE_COMMENT Redo log write ahead unit size to avoid read-on-write, it should match the OS cache block IO size NUMERIC_MIN_VALUE 512 NUMERIC_MAX_VALUE 16384 -@@ -1621,10 +1621,10 @@ +@@ -1594,10 +1594,10 @@ GLOBAL_VALUE_ORIGIN CONFIG DEFAULT_VALUE 1024 VARIABLE_SCOPE GLOBAL @@ -378,7 +369,7 @@ NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -1677,10 +1677,10 @@ +@@ -1650,10 +1650,10 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -391,7 +382,7 @@ NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -1691,7 +1691,7 @@ +@@ -1664,7 +1664,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -400,7 +391,7 @@ VARIABLE_COMMENT Maximum delay of user threads in micro-seconds NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 10000000 -@@ -1789,7 +1789,7 @@ +@@ -1762,7 +1762,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 8 VARIABLE_SCOPE GLOBAL @@ -409,7 +400,7 @@ VARIABLE_COMMENT Number of multi-threaded flush threads NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 64 -@@ -1845,10 +1845,10 @@ +@@ -1818,10 +1818,10 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -422,7 +413,7 @@ NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY YES -@@ -1873,7 +1873,7 @@ +@@ -1846,7 +1846,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 4 VARIABLE_SCOPE GLOBAL @@ -431,7 +422,7 @@ VARIABLE_COMMENT Page cleaner threads can be from 1 to 64. Default is 4. NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 64 -@@ -1901,7 +1901,7 @@ +@@ -1874,7 +1874,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 16 VARIABLE_SCOPE GLOBAL @@ -440,7 +431,7 @@ VARIABLE_COMMENT Number of rw_locks protecting buffer pool page_hash. Rounded up to the next power of 2 NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 1024 -@@ -1915,7 +1915,7 @@ +@@ -1888,7 +1888,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 16384 VARIABLE_SCOPE GLOBAL @@ -449,7 +440,7 @@ VARIABLE_COMMENT Page size to use for all InnoDB tablespaces. NUMERIC_MIN_VALUE 4096 NUMERIC_MAX_VALUE 65536 -@@ -1957,7 +1957,7 @@ +@@ -1930,7 +1930,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 300 VARIABLE_SCOPE GLOBAL @@ -458,7 +449,7 @@ VARIABLE_COMMENT Number of UNDO log pages to purge in one batch from the history list. NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 5000 -@@ -1971,7 +1971,7 @@ +@@ -1944,7 +1944,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 128 VARIABLE_SCOPE GLOBAL @@ -467,7 +458,7 @@ VARIABLE_COMMENT Dictates rate at which UNDO records are purged. Value N means purge rollback segment(s) on every Nth iteration of purge invocation NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 128 -@@ -2013,7 +2013,7 @@ +@@ -1986,7 +1986,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 4 VARIABLE_SCOPE GLOBAL @@ -476,7 +467,7 @@ VARIABLE_COMMENT Purge threads can be from 1 to 32. Default is 4. NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 32 -@@ -2041,7 +2041,7 @@ +@@ -2014,7 +2014,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 56 VARIABLE_SCOPE GLOBAL @@ -485,7 +476,7 @@ VARIABLE_COMMENT Number of pages that must be accessed sequentially for InnoDB to trigger a readahead. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 64 -@@ -2055,7 +2055,7 @@ +@@ -2028,7 +2028,7 @@ GLOBAL_VALUE_ORIGIN CONFIG DEFAULT_VALUE 4 VARIABLE_SCOPE GLOBAL @@ -494,7 +485,7 @@ VARIABLE_COMMENT Number of background read I/O threads in InnoDB. NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 64 -@@ -2083,10 +2083,10 @@ +@@ -2056,10 +2056,10 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -507,7 +498,7 @@ NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -2111,7 +2111,7 @@ +@@ -2084,7 +2084,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 128 VARIABLE_SCOPE GLOBAL @@ -516,7 +507,7 @@ VARIABLE_COMMENT Number of undo logs to use (deprecated). NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 128 -@@ -2125,7 +2125,7 @@ +@@ -2098,7 +2098,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -525,7 +516,7 @@ VARIABLE_COMMENT An InnoDB page number. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 4294967295 -@@ -2181,7 +2181,7 @@ +@@ -2154,7 +2154,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 1048576 VARIABLE_SCOPE GLOBAL @@ -534,7 +525,7 @@ VARIABLE_COMMENT Memory buffer size for index creation NUMERIC_MIN_VALUE 65536 NUMERIC_MAX_VALUE 67108864 -@@ -2391,7 +2391,7 @@ +@@ -2364,7 +2364,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 1 VARIABLE_SCOPE GLOBAL @@ -543,7 +534,7 @@ VARIABLE_COMMENT Size of the mutex/lock wait array. NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 1024 -@@ -2419,10 +2419,10 @@ +@@ -2392,10 +2392,10 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 30 VARIABLE_SCOPE GLOBAL @@ -556,7 +547,7 @@ NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -2461,7 +2461,7 @@ +@@ -2434,7 +2434,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -565,7 +556,7 @@ VARIABLE_COMMENT Helps in performance tuning in heavily concurrent environments. Sets the maximum number of threads allowed inside InnoDB. Value 0 will disable the thread throttling. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 1000 -@@ -2475,7 +2475,7 @@ +@@ -2448,7 +2448,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 10000 VARIABLE_SCOPE GLOBAL @@ -574,7 +565,7 @@ VARIABLE_COMMENT Time of innodb thread sleeping before joining InnoDB queue (usec). Value 0 disable a sleep NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 1000000 -@@ -2545,7 +2545,7 @@ +@@ -2518,7 +2518,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 128 VARIABLE_SCOPE GLOBAL @@ -583,7 +574,7 @@ VARIABLE_COMMENT Number of undo logs to use. NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 128 -@@ -2573,7 +2573,7 @@ +@@ -2546,7 +2546,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -592,7 +583,7 @@ VARIABLE_COMMENT Number of undo tablespaces to use. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 95 -@@ -2657,7 +2657,7 @@ +@@ -2630,7 +2630,7 @@ GLOBAL_VALUE_ORIGIN CONFIG DEFAULT_VALUE 4 VARIABLE_SCOPE GLOBAL diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result index 46b80370912..0816cd7bd31 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result @@ -526,9 +526,9 @@ READ_ONLY NO COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME INNODB_COMPRESSION_ALGORITHM SESSION_VALUE NULL -GLOBAL_VALUE none +GLOBAL_VALUE zlib GLOBAL_VALUE_ORIGIN COMPILE-TIME -DEFAULT_VALUE none +DEFAULT_VALUE zlib VARIABLE_SCOPE GLOBAL VARIABLE_TYPE ENUM VARIABLE_COMMENT Compression algorithm used on page compression. One of: none, zlib, lz4, lzo, lzma, or bzip2 diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result b/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result index 30c4c5a59b0..89a6d8448fc 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result +++ b/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result @@ -2102,7 +2102,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0.000000 VARIABLE_SCOPE SESSION VARIABLE_TYPE DOUBLE -VARIABLE_COMMENT A SELECT query that have taken more than max_statement_time seconds will be aborted. The argument will be treated as a decimal value with microsecond precision. A value of 0 (default) means no timeout +VARIABLE_COMMENT A query that has taken more than max_statement_time seconds will be aborted. The argument will be treated as a decimal value with microsecond precision. A value of 0 (default) means no timeout NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 31536000 NUMERIC_BLOCK_SIZE NULL diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result index 0f048c27caf..c4018b39a8f 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result +++ b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result @@ -2298,7 +2298,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0.000000 VARIABLE_SCOPE SESSION VARIABLE_TYPE DOUBLE -VARIABLE_COMMENT A SELECT query that have taken more than max_statement_time seconds will be aborted. The argument will be treated as a decimal value with microsecond precision. A value of 0 (default) means no timeout +VARIABLE_COMMENT A query that has taken more than max_statement_time seconds will be aborted. The argument will be treated as a decimal value with microsecond precision. A value of 0 (default) means no timeout NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 31536000 NUMERIC_BLOCK_SIZE NULL diff --git a/mysql-test/suite/sys_vars/t/sysvars_innodb.test b/mysql-test/suite/sys_vars/t/sysvars_innodb.test index 38f248cb611..2abafda34c6 100644 --- a/mysql-test/suite/sys_vars/t/sysvars_innodb.test +++ b/mysql-test/suite/sys_vars/t/sysvars_innodb.test @@ -3,6 +3,11 @@ --source include/not_valgrind.inc --source include/word_size.inc +if (`select plugin_auth_version <= "5.6.34-79.1" from information_schema.plugins where plugin_name='innodb'`) +{ + --skip Not fixed in XtraDB as of 10.1.21-MariaDB or earlier +} + --vertical_results --replace_regex /^\/\S+/PATH/ /\.\//PATH/ select * from information_schema.system_variables diff --git a/mysql-test/suite/vcol/inc/vcol_trigger_sp.inc b/mysql-test/suite/vcol/inc/vcol_trigger_sp.inc index 5acea045d0f..f19c0f6d63f 100644 --- a/mysql-test/suite/vcol/inc/vcol_trigger_sp.inc +++ b/mysql-test/suite/vcol/inc/vcol_trigger_sp.inc @@ -110,7 +110,7 @@ drop table t1,t2; drop procedure p1; --echo # ---echo # Bug mdev-3845: values of virtual columns are not computed for triggers +--echo # MDEV-3845 values of virtual columns are not computed for triggers --echo # CREATE TABLE t1 ( @@ -149,6 +149,14 @@ DROP TRIGGER t1_ins_aft; DROP TRIGGER t1_del_bef; DROP TABLE t1,t2; +# +# MDEV-11706 Assertion `is_stat_field || !table || (!table->write_set || bitmap_is_set(table->write_set, field_index) || (table->vcol_set && bitmap_is_set(table->vcol_set, field_index)))' failed in Field_time::store_TIME_with_warning +# +create table t1 (i int, t time not null, vt time(4) as (t) virtual); +create trigger trg before update on t1 for each row set @a = 1; +insert ignore into t1 (i) values (1); +drop table t1; + --echo # --echo # Examine the number of times triggers are recalculated for updates --echo # diff --git a/mysql-test/suite/vcol/r/vcol_misc.result b/mysql-test/suite/vcol/r/vcol_misc.result index ed78731c605..9aadc5ed4bb 100644 --- a/mysql-test/suite/vcol/r/vcol_misc.result +++ b/mysql-test/suite/vcol/r/vcol_misc.result @@ -328,6 +328,13 @@ t1 CREATE TABLE `t1` ( `c1` varchar(50) COLLATE latin1_general_ci DEFAULT NULL ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_general_ci drop table t1; +set sql_mode='no_zero_date'; +create table t1 ( +ts timestamp not null default current_timestamp, +tsv timestamp as (adddate(ts, interval 1 day)) virtual +); +drop table t1; +set sql_mode=default; # # Start of 10.1 tests # diff --git a/mysql-test/suite/vcol/r/vcol_trigger_sp_innodb.result b/mysql-test/suite/vcol/r/vcol_trigger_sp_innodb.result index dae34569d1b..de5447e49eb 100644 --- a/mysql-test/suite/vcol/r/vcol_trigger_sp_innodb.result +++ b/mysql-test/suite/vcol/r/vcol_trigger_sp_innodb.result @@ -86,7 +86,7 @@ a b c drop table t1,t2; drop procedure p1; # -# Bug mdev-3845: values of virtual columns are not computed for triggers +# MDEV-3845 values of virtual columns are not computed for triggers # CREATE TABLE t1 ( a INTEGER UNSIGNED NULL DEFAULT NULL, @@ -125,6 +125,12 @@ c DROP TRIGGER t1_ins_aft; DROP TRIGGER t1_del_bef; DROP TABLE t1,t2; +create table t1 (i int, t time not null, vt time(4) as (t) virtual); +create trigger trg before update on t1 for each row set @a = 1; +insert ignore into t1 (i) values (1); +Warnings: +Warning 1364 Field 't' doesn't have a default value +drop table t1; # # Examine the number of times triggers are recalculated for updates # diff --git a/mysql-test/suite/vcol/r/vcol_trigger_sp_myisam.result b/mysql-test/suite/vcol/r/vcol_trigger_sp_myisam.result index 5cb02f3911c..f5c87838ec9 100644 --- a/mysql-test/suite/vcol/r/vcol_trigger_sp_myisam.result +++ b/mysql-test/suite/vcol/r/vcol_trigger_sp_myisam.result @@ -86,7 +86,7 @@ a b c drop table t1,t2; drop procedure p1; # -# Bug mdev-3845: values of virtual columns are not computed for triggers +# MDEV-3845 values of virtual columns are not computed for triggers # CREATE TABLE t1 ( a INTEGER UNSIGNED NULL DEFAULT NULL, @@ -125,6 +125,12 @@ c DROP TRIGGER t1_ins_aft; DROP TRIGGER t1_del_bef; DROP TABLE t1,t2; +create table t1 (i int, t time not null, vt time(4) as (t) virtual); +create trigger trg before update on t1 for each row set @a = 1; +insert ignore into t1 (i) values (1); +Warnings: +Warning 1364 Field 't' doesn't have a default value +drop table t1; # # Examine the number of times triggers are recalculated for updates # diff --git a/mysql-test/suite/vcol/r/wrong_arena.result b/mysql-test/suite/vcol/r/wrong_arena.result new file mode 100644 index 00000000000..8a07c828288 --- /dev/null +++ b/mysql-test/suite/vcol/r/wrong_arena.result @@ -0,0 +1,67 @@ +create table t1 (a datetime, +# get_datetime_value +b int as (a > 1), # Arg_comparator +c int as (a in (1,2,3)), # in_datetime +d int as ((a,a) in ((1,1),(2,1),(NULL,1))), # cmp_item_datetime +# other issues +e int as ((a,1) in ((1,1),(2,1),(NULL,1))) # cmp_item_row::alloc_comparators() +); +Warnings: +Warning 1292 Incorrect datetime value: '1' +Warning 1292 Incorrect datetime value: '2' +Warning 1292 Incorrect datetime value: '3' +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` datetime DEFAULT NULL, + `b` int(11) GENERATED ALWAYS AS (`a` > 1) VIRTUAL, + `c` int(11) GENERATED ALWAYS AS (`a` in (1,2,3)) VIRTUAL, + `d` int(11) GENERATED ALWAYS AS ((`a`,`a`) in ((1,1),(2,1),(NULL,1))) VIRTUAL, + `e` int(11) GENERATED ALWAYS AS ((`a`,1) in ((1,1),(2,1),(NULL,1))) VIRTUAL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +Warnings: +Warning 1292 Incorrect datetime value: '1' +Warning 1292 Incorrect datetime value: '2' +Warning 1292 Incorrect datetime value: '3' +connect con1, localhost, root; +insert t1 (a) values ('2010-10-10 10:10:10'); +select * from t1; +a b c d e +2010-10-10 10:10:10 1 0 0 NULL +Warnings: +Warning 1292 Incorrect datetime value: '1' +Warning 1292 Incorrect datetime value: '1' +Warning 1292 Incorrect datetime value: '2' +Warning 1292 Incorrect datetime value: '1' +Warning 1292 Incorrect datetime value: '1' +Warning 1292 Incorrect datetime value: '2' +disconnect con1; +connection default; +select * from t1; +a b c d e +2010-10-10 10:10:10 1 0 0 NULL +Warnings: +Warning 1292 Incorrect datetime value: '1' +Warning 1292 Incorrect datetime value: '2' +Warning 1292 Incorrect datetime value: '1' +Warning 1292 Incorrect datetime value: '1' +Warning 1292 Incorrect datetime value: '2' +drop table t1; +connect con1, localhost, root; +create table t1 (a datetime, +b datetime as (least(a,1)) # Item_func_min_max::get_date +); +insert t1 (a) values ('2010-10-10 10:10:10'); +select * from t1; +a b +2010-10-10 10:10:10 0000-00-00 00:00:00 +Warnings: +Warning 1292 Incorrect datetime value: '1' +disconnect con1; +connection default; +select * from t1; +a b +2010-10-10 10:10:10 0000-00-00 00:00:00 +Warnings: +Warning 1292 Incorrect datetime value: '1' +drop table t1; diff --git a/mysql-test/suite/vcol/t/vcol_misc.test b/mysql-test/suite/vcol/t/vcol_misc.test index 1961e2bd974..5bdd927070a 100644 --- a/mysql-test/suite/vcol/t/vcol_misc.test +++ b/mysql-test/suite/vcol/t/vcol_misc.test @@ -294,6 +294,17 @@ create table t1 (v1 varchar(255) as (c1) persistent, c1 varchar(50)) collate=lat show create table t1; drop table t1; +# +# MDEV-11527 Virtual columns do not get along well with NO_ZERO_DATE +# +set sql_mode='no_zero_date'; +create table t1 ( + ts timestamp not null default current_timestamp, + tsv timestamp as (adddate(ts, interval 1 day)) virtual +); +drop table t1; +set sql_mode=default; + --echo # --echo # Start of 10.1 tests diff --git a/mysql-test/suite/vcol/t/wrong_arena.test b/mysql-test/suite/vcol/t/wrong_arena.test new file mode 100644 index 00000000000..484f1fe685d --- /dev/null +++ b/mysql-test/suite/vcol/t/wrong_arena.test @@ -0,0 +1,35 @@ +# +# This tests various issues when vcol items allocate memory (e.g. more items) +# not in the TABLE::expr_arena. +# + +# +# MDEV-9690 concurrent queries with virtual columns crash in temporal code +# +create table t1 (a datetime, + # get_datetime_value + b int as (a > 1), # Arg_comparator + c int as (a in (1,2,3)), # in_datetime + d int as ((a,a) in ((1,1),(2,1),(NULL,1))), # cmp_item_datetime + # other issues + e int as ((a,1) in ((1,1),(2,1),(NULL,1))) # cmp_item_row::alloc_comparators() +); +show create table t1; +connect con1, localhost, root; +insert t1 (a) values ('2010-10-10 10:10:10'); +select * from t1; +disconnect con1; +connection default; +select * from t1; +drop table t1; + +connect con1, localhost, root; +create table t1 (a datetime, + b datetime as (least(a,1)) # Item_func_min_max::get_date +); +insert t1 (a) values ('2010-10-10 10:10:10'); +select * from t1; +disconnect con1; +connection default; +select * from t1; +drop table t1; diff --git a/mysql-test/t/analyze_format_json.test b/mysql-test/t/analyze_format_json.test index 807e02d2334..88a907718d9 100644 --- a/mysql-test/t/analyze_format_json.test +++ b/mysql-test/t/analyze_format_json.test @@ -9,34 +9,34 @@ create table t0 (a int); INSERT INTO t0 VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); --echo # r_filtered=30%, because 3 rows match: 0,1,2 ---replace_regex /"r_total_time_ms": [0-9]*[.]?[0-9]*/"r_total_time_ms": "REPLACED"/ +--source include/analyze-format.inc analyze format=json select * from t0 where a<3; create table t1 (a int, b int, c int, key(a)); insert into t1 select A.a*10 + B.a, A.a*10 + B.a, A.a*10 + B.a from t0 A, t0 B; ---replace_regex /"r_total_time_ms": [0-9]*[.]?[0-9]*/"r_total_time_ms": "REPLACED"/ +--source include/analyze-format.inc analyze select * from t0, t1 where t1.a=t0.a and t0.a > 9; ---replace_regex /"r_total_time_ms": [0-9]*[.]?[0-9]*/"r_total_time_ms": "REPLACED"/ +--source include/analyze-format.inc analyze format=json select * from t0, t1 where t1.a=t0.a and t0.a > 9; analyze select * from t0, t1 where t1.a=t0.a and t1.b<4; ---replace_regex /"r_total_time_ms": [0-9]*[.]?[0-9]*/"r_total_time_ms": "REPLACED"/ +--source include/analyze-format.inc analyze format=json select * from t0, t1 where t1.a=t0.a and t1.b<4; analyze select * from t1 tbl1, t1 tbl2 where tbl1.b<2 and tbl2.b>5; ---replace_regex /"r_total_time_ms": [0-9]*[.]?[0-9]*/"r_total_time_ms": "REPLACED"/ +--source include/analyze-format.inc analyze format=json select * from t1 tbl1, t1 tbl2 where tbl1.b<20 and tbl2.b<60; ---replace_regex /"r_total_time_ms": [0-9]*[.]?[0-9]*/"r_total_time_ms": "REPLACED"/ +--source include/analyze-format.inc analyze format=json select * from t1 tbl1, t1 tbl2 where tbl1.b<20 and tbl2.b<60 and tbl1.c > tbl2.c; @@ -53,7 +53,7 @@ insert into t1 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); create table t2 (a int, key(a)); insert into t2 values (0),(1); ---replace_regex /"r_total_time_ms": [0-9]*[.]?[0-9]*/"r_total_time_ms": "REPLACED"/ +--source include/analyze-format.inc analyze format=json select * from t1 straight_join t2 force index(a) where t2.a=t1.a; drop table t1,t2; @@ -69,7 +69,7 @@ select database(); connect (con1,localhost,root,,*NO-ONE*); connection con1; select database(); ---replace_regex /"r_total_time_ms": [0-9]*[.]?[0-9]*/"r_total_time_ms": "REPLACED"/ +--source include/analyze-format.inc analyze format=json select * from test.t1 where t1.a<5; disconnect con1; connection default; @@ -91,15 +91,15 @@ create table t1 (pk int primary key); insert into t1 select a from t3; alter table t1 add b int; ---replace_regex /"r_total_time_ms": [0-9]*[.]?[0-9]*/"r_total_time_ms": "REPLACED"/ +--source include/analyze-format.inc analyze format=json update t1 set b=pk; ---replace_regex /"r_total_time_ms": [0-9]*[.]?[0-9]*/"r_total_time_ms": "REPLACED"/ +--source include/analyze-format.inc analyze format=json select * from t1 where pk < 10 and b > 4; ---replace_regex /"r_total_time_ms": [0-9]*[.]?[0-9]*/"r_total_time_ms": "REPLACED"/ +--source include/analyze-format.inc analyze format=json delete from t1 where pk < 10 and b > 4; @@ -127,7 +127,7 @@ create table t2 (key1 int, key2 int, key3 int, key4 int, col1 int, insert into t2 select a,a,a,a,a from t3; insert into t2 select 15,15,15,15,15 from t4; ---replace_regex /"r_total_time_ms": [0-9]*[.]?[0-9]*/"r_total_time_ms": "REPLACED"/ +--source include/analyze-format.inc analyze format=json select * from t1, t2 where (t2.key1 between t1.lb1 and t1.rb1) and (t2.key2 between t1.lb2 and t1.rb2) and @@ -144,7 +144,7 @@ INSERT INTO t0 VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); create table t1 (a int); INSERT INTO t1 select * from t0; ---replace_regex /"r_total_time_ms": [0-9]*[.]?[0-9]*/"r_total_time_ms": "REPLACED"/ +--source include/analyze-format.inc analyze format=json (select * from t1 tbl1 where a<5) union (select * from t1 tbl2 where a in (2,3)); drop table t0, t1; @@ -164,16 +164,16 @@ create table t2 ( ); insert into t2 select A.a*1000 + B.a, A.a*1000 + B.a from t0 A, t1 B; --echo # normal HAVING ---replace_regex /"(r_total_time_ms|r_buffer_size)": .*?,/"volatile parameter": "REPLACED",/ +--source include/analyze-format.inc analyze format=json select a, max(b) as TOP from t2 group by a having TOP > a; --echo # HAVING is always TRUE (not printed) ---replace_regex /"(r_total_time_ms|r_buffer_size)": .*?,/"volatile parameter": "REPLACED",/ +--source include/analyze-format.inc analyze format=json select a, max(b) as TOP from t2 group by a having 1<>2; --echo # HAVING is always FALSE (intercepted by message) ---replace_regex /"(r_total_time_ms|r_buffer_size)": .*?,/"volatile parameter": "REPLACED",/ +--source include/analyze-format.inc analyze format=json select a, max(b) as TOP from t2 group by a having 1=2; --echo # HAVING is absent ---replace_regex /"(r_total_time_ms|r_buffer_size)": .*?,/"volatile parameter": "REPLACED",/ +--source include/analyze-format.inc analyze format=json select a, max(b) as TOP from t2 group by a; drop table t0, t1, t2; @@ -186,7 +186,7 @@ INSERT INTO t1 VALUES (1),(2); CREATE TABLE t2 (b INT); INSERT INTO t2 VALUES (3),(4); ---replace_regex /"(r_total_time_ms|r_buffer_size)": .*?,/"volatile parameter": "REPLACED",/ +--source include/analyze-format.inc ANALYZE FORMAT=JSON SELECT STRAIGHT_JOIN * FROM t1, t2 WHERE b IN ( SELECT a FROM t1 ); drop table t1,t2; @@ -203,7 +203,7 @@ INSERT INTO t2 VALUES (2),(3); CREATE TABLE t3 (f3 INT) ENGINE=MyISAM; INSERT INTO t3 VALUES (3),(4); ---replace_regex /"(r_total_time_ms|r_buffer_size)": .*?,/"volatile parameter": "REPLACED",/ +--source include/analyze-format.inc ANALYZE FORMAT=JSON SELECT GROUP_CONCAT(f3) AS gc, ( SELECT MAX(f1) FROM t1, t2 WHERE f2 = f3 ) sq FROM t2, t3 diff --git a/mysql-test/t/analyze_stmt_orderby.test b/mysql-test/t/analyze_stmt_orderby.test index a40f34805d1..a1cfb58f31a 100644 --- a/mysql-test/t/analyze_stmt_orderby.test +++ b/mysql-test/t/analyze_stmt_orderby.test @@ -22,7 +22,7 @@ explain update t2 set b=b+1 order by b limit 5; explain format=json update t2 set b=b+1 order by b limit 5; ---replace_regex /"r_total_time_ms": [0-9]*[.]?[0-9]*/"r_total_time_ms": "REPLACED"/ +--source include/analyze-format.inc analyze format=json update t2 set b=b+1 order by b limit 5; @@ -33,7 +33,7 @@ explain update t2 set a=a+1 where a<10; explain format=json update t2 set a=a+1 where a<10; ---replace_regex /"r_total_time_ms": [0-9]*[.]?[0-9]*/"r_total_time_ms": "REPLACED"/ +--source include/analyze-format.inc analyze format=json update t2 set a=a+1 where a<10; @@ -44,7 +44,7 @@ explain delete from t2 order by b limit 5; explain format=json delete from t2 order by b limit 5; ---replace_regex /"r_total_time_ms": [0-9]*[.]?[0-9]*/"r_total_time_ms": "REPLACED"/ /"r_buffer_size": "[^"]+"/"r_buffer_size": "REPLACED"/ +--source include/analyze-format.inc analyze format=json delete from t2 order by b limit 5; @@ -55,7 +55,7 @@ explain select * from t0,t2 where t2.a=t0.a order by t2.b limit 4; explain format=json select * from t0,t2 where t2.a=t0.a order by t2.b limit 4; ---replace_regex /"r_total_time_ms": [0-9]*[.]?[0-9]*/"r_total_time_ms": "REPLACED"/ +--source include/analyze-format.inc analyze format=json select * from t0,t2 where t2.a=t0.a order by t2.b limit 4; @@ -67,7 +67,7 @@ explain select * from t0,t2 where t2.a=t0.a order by t0.a limit 4; explain format=json select * from t0,t2 where t2.a=t0.a order by t0.a limit 4; ---replace_regex /"r_total_time_ms": [0-9]*[.]?[0-9]*/"r_total_time_ms": "REPLACED"/ /"r_buffer_size": "[^"]+"/"r_buffer_size": "REPLACED"/ +--source include/analyze-format.inc analyze format=json select * from t0,t2 where t2.a=t0.a order by t0.a limit 4; @@ -86,7 +86,7 @@ select c.a from t0 a, t0 b, t0 c; ---replace_regex /"r_total_time_ms": [0-9]*[.]?[0-9]*/"r_total_time_ms": "REPLACED"/ /"r_buffer_size": "[^"]+"/"r_buffer_size": "REPLACED"/ +--source include/analyze-format.inc analyze format=json select MAX(b) from t2 where mod(a,2)=0 group by c; @@ -115,7 +115,7 @@ drop table t3; create table t3 (a int, b int); insert into t3 select a, 123 from t0; ---replace_regex /"r_total_time_ms": [0-9]*[.]?[0-9]*/"r_total_time_ms": "REPLACED"/ /"r_buffer_size": "[^"]+"/"r_buffer_size": "REPLACED"/ +--source include/analyze-format.inc analyze format=json select distinct max(t3.b) Q from t0, t3 where t0.a=t3.a group by t0.a order by null; @@ -136,7 +136,7 @@ insert into t6 values (0, 0), (1, 1), (2, 1), (3, 1), (4, 1); insert into t7 values (3, 3), (2, 2), (1, 1); --echo # TODO: This ANALYZE output doesn't make it clear what is used for what. ---replace_regex /"r_total_time_ms": [0-9]*[.]?[0-9]*/"r_total_time_ms": "REPLACED"/ /"r_buffer_size": "[^"]+"/"r_buffer_size": "REPLACED"/ +--source include/analyze-format.inc analyze format=json select count(distinct t5.b) as sum from t5, t6 where t5.a=t6.a and t6.b > 0 and t5.a <= 5 @@ -168,7 +168,7 @@ explain select col1 f1, col2 f2, col1 f3 from t2 group by f1; analyze select col1 f1, col2 f2, col1 f3 from t2 group by f1; ---replace_regex /"r_total_time_ms": [0-9]*[.]?[0-9]*/"r_total_time_ms": "REPLACED"/ /"r_buffer_size": "[^"]+"/"r_buffer_size": "REPLACED"/ +--source include/analyze-format.inc analyze format=json select col1 f1, col2 f2, col1 f3 from t2 group by f1; drop table t2; diff --git a/mysql-test/t/create.test b/mysql-test/t/create.test index 36fc4e4e2d0..6461204f06e 100644 --- a/mysql-test/t/create.test +++ b/mysql-test/t/create.test @@ -1759,6 +1759,15 @@ DELIMITER ;| create table t1 as select f1(); drop function f1; +--echo # +--echo # MDEV-10274 Bundling insert with create statement +--echo # for table with unsigned Decimal primary key issues warning 1194 +--echo # + +create table t1(ID decimal(2,1) unsigned NOT NULL, PRIMARY KEY (ID))engine=memory + select 2.1 ID; +drop table t1; + --echo End of 5.5 tests # diff --git a/mysql-test/t/cte_recursive.test b/mysql-test/t/cte_recursive.test index 334599f1653..e8954bfdaeb 100644 --- a/mysql-test/t/cte_recursive.test +++ b/mysql-test/t/cte_recursive.test @@ -1510,7 +1510,7 @@ drop table t1; --echo # MDEV-10773: ANALYZE for query with recursive CTE --echo # ---replace_regex /"r_total_time_ms": [0-9]*[.]?[0-9]*(e[-+]?[0-9]+)?/"r_total_time_ms": "REPLACED"/ +--source include/analyze-format.inc analyze format=json with recursive src(counter) as (select 1 diff --git a/mysql-test/t/ctype_ucs2_def.test b/mysql-test/t/ctype_ucs2_def.test index be8e044f2e4..e297fa5ccf1 100644 --- a/mysql-test/t/ctype_ucs2_def.test +++ b/mysql-test/t/ctype_ucs2_def.test @@ -1,6 +1,6 @@ -- source include/have_ucs2.inc -call mtr.add_suppression("Cannot use ucs2 as character_set_client"); +call mtr.add_suppression("'ucs2' can not be used as client character set"); # # MySQL Bug#15276: MySQL ignores collation-server diff --git a/mysql-test/t/ctype_ucs2_query_cache.test b/mysql-test/t/ctype_ucs2_query_cache.test index acb39419751..ace826aec44 100644 --- a/mysql-test/t/ctype_ucs2_query_cache.test +++ b/mysql-test/t/ctype_ucs2_query_cache.test @@ -1,7 +1,7 @@ -- source include/have_query_cache.inc -- source include/have_ucs2.inc -call mtr.add_suppression("Cannot use ucs2 as character_set_client"); +call mtr.add_suppression("'ucs2' can not be used as client character set"); --echo # --echo # Start of 5.5 tests diff --git a/mysql-test/t/ctype_utf16_def.test b/mysql-test/t/ctype_utf16_def.test index fad61b057c3..0829cd53285 100644 --- a/mysql-test/t/ctype_utf16_def.test +++ b/mysql-test/t/ctype_utf16_def.test @@ -1,5 +1,5 @@ --source include/have_utf16.inc -call mtr.add_suppression("Cannot use utf16 as character_set_client"); +call mtr.add_suppression("'utf16' can not be used as client character set"); # # Bug #32391 Character sets: crash with --character-set-server diff --git a/mysql-test/t/events_slowlog.test b/mysql-test/t/events_slowlog.test new file mode 100644 index 00000000000..9679714dba3 --- /dev/null +++ b/mysql-test/t/events_slowlog.test @@ -0,0 +1,28 @@ +--source include/not_embedded.inc +# +# MDEV-11552 Queries executed by event scheduler are written to slow log incorrectly or not written at all +# +set @event_scheduler_save= @@global.event_scheduler; +set @slow_query_log_save= @@global.slow_query_log; + +set global event_scheduler= on; +set global slow_query_log= on; +set global long_query_time=0.2; + +create table t1 (i int); +insert into t1 values (0); +create event ev on schedule at CURRENT_TIMESTAMP + INTERVAL 1 second do update t1 set i=1+sleep(0.5); + +--let wait_condition= select i from t1 where i > 0 +--source include/wait_condition.inc + +--let SEARCH_FILE = `SELECT @@slow_query_log_file` +--let SEARCH_PATTERN= update t1 set i=1 +--let SEARCH_RANGE= -1000 +--source include/search_pattern_in_file.inc + +drop table t1; + +set global event_scheduler= @event_scheduler_save; +set global slow_query_log= @slow_query_log_save; +set global long_query_time= @@session.long_query_time; diff --git a/mysql-test/t/explain_json.test b/mysql-test/t/explain_json.test index 22bfd5aedcd..d253b8380e9 100644 --- a/mysql-test/t/explain_json.test +++ b/mysql-test/t/explain_json.test @@ -219,7 +219,7 @@ create table t1 (a int, b int, c int, d int, key(a,b,c)); insert into t1 select A.a, B.a, C.a, D.a from t2 A, t2 B, t2 C, t2 D; explain select count(distinct b) from t1 group by a; explain format=json select count(distinct b) from t1 group by a; ---replace_regex /"r_total_time_ms": [0-9]*[.]?[0-9]*/"r_total_time_ms": "REPLACED"/ +--source include/analyze-format.inc analyze format=json select count(distinct b) from t1 group by a; drop table t1,t2; @@ -342,7 +342,7 @@ explain select * from t1 left join t2 on t2.pk > 10 and t2.pk < 0; explain format=json select * from t1 left join t2 on t2.pk > 10 and t2.pk < 0; ---replace_regex /"r_total_time_ms": [0-9]*[.]?[0-9]*/"r_total_time_ms": "REPLACED"/ +--source include/analyze-format.inc analyze format=json select * from t1 left join t2 on t2.pk > 10 and t2.pk < 0; @@ -351,7 +351,7 @@ explain select * from t1 left join t2 on t2.pk=t1.a where t2.pk is null; explain format=json select * from t1 left join t2 on t2.pk=t1.a where t2.pk is null; ---replace_regex /"r_total_time_ms": [0-9]*[.]?[0-9]*/"r_total_time_ms": "REPLACED"/ +--source include/analyze-format.inc analyze format=json select * from t1 left join t2 on t2.pk=t1.a where t2.pk is null; @@ -360,7 +360,7 @@ explain select distinct t1.a from t1 join t2 on t2.pk=t1.a; explain format=json select distinct t1.a from t1 join t2 on t2.pk=t1.a; ---replace_regex /"r_total_time_ms": [0-9]*[.]?[0-9]*/"r_total_time_ms": "REPLACED"/ +--source include/analyze-format.inc analyze format=json select distinct t1.a from t1 join t2 on t2.pk=t1.a; drop table t1,t2; @@ -386,7 +386,7 @@ explain select * from t3,t4 where t3.a=t4.a and (t4.b+1 <= t3.b+1); explain format=json select * from t3,t4 where t3.a=t4.a and (t4.b+1 <= t3.b+1); ---replace_regex /"r_total_time_ms": [0-9]*[.]?[0-9]*/"r_total_time_ms": "REPLACED"/ +--source include/analyze-format.inc analyze format=json select * from t3,t4 where t3.a=t4.a and (t4.b+1 <= t3.b+1); set optimizer_switch=@tmp_optimizer_switch; diff --git a/mysql-test/t/explain_json_format_partitions.test b/mysql-test/t/explain_json_format_partitions.test index f3567797019..4c7d3f165d1 100644 --- a/mysql-test/t/explain_json_format_partitions.test +++ b/mysql-test/t/explain_json_format_partitions.test @@ -8,10 +8,10 @@ create table t1 ( insert into t1 select a from t2; explain partitions select * from t1 where a in (2,3,4); explain format=json select * from t1 where a in (2,3,4); ---replace_regex /"r_total_time_ms": [0-9]*[.]?[0-9]*/"r_total_time_ms": "REPLACED"/ +--source include/analyze-format.inc analyze format=json select * from t1 where a in (2,3,4); ---replace_regex /"r_total_time_ms": [0-9]*[.]?[0-9]*/"r_total_time_ms": "REPLACED"/ +--source include/analyze-format.inc analyze format=json update t1 set a=a+10 where a in (2,3,4); ---replace_regex /"r_total_time_ms": [0-9]*[.]?[0-9]*/"r_total_time_ms": "REPLACED"/ +--source include/analyze-format.inc analyze format=json delete from t1 where a in (20,30,40); drop table t1,t2; diff --git a/mysql-test/t/func_time.test b/mysql-test/t/func_time.test index 0905ed48078..6e0169d97ca 100644 --- a/mysql-test/t/func_time.test +++ b/mysql-test/t/func_time.test @@ -1673,6 +1673,11 @@ INSERT INTO t1 VALUES (18, '2010-10-13'); SELECT a.id,a.date1,FROM_DAYS(TO_DAYS(a.date1)-10) as date2, DATE_ADD(a.date1,INTERVAL -10 DAY),TO_DAYS(a.date1)-10 FROM t1 a ORDER BY a.id; DROP TABLE t1; +--echo # +--echo # MDEV-10524 Assertion `arg1_int >= 0' failed in Item_func_additive_op::result_precision() +--echo # +SELECT 1 MOD ADDTIME( '13:58:57', '00:00:01' ) + 2; + --echo # --echo # Start of 10.0 tests diff --git a/mysql-test/t/index_merge_innodb.test b/mysql-test/t/index_merge_innodb.test index 3a2601342ba..53ce3114b49 100644 --- a/mysql-test/t/index_merge_innodb.test +++ b/mysql-test/t/index_merge_innodb.test @@ -171,6 +171,37 @@ WHERE ( tb.b != ta.b OR tb.a = ta.a ) AND ( tb.b = ta.c OR tb.b = ta.b ); DROP TABLE t1; - set optimizer_switch= @optimizer_switch_save; +--echo # +--echo # MDEV-10927: Crash When Using sort_union Optimization +--echo # + +set @tmp_optimizer_switch=@@optimizer_switch; +SET optimizer_switch='index_merge_sort_intersection=on'; +SET SESSION sort_buffer_size = 1024; + +create table t1 ( +pk int(11) NOT NULL AUTO_INCREMENT, +col1 int(11) NOT NULL, +col2 int(11) NOT NULL, +col3 int(11) NOT NULL, +key2 int(11) NOT NULL, +col4 int(11) NOT NULL, +key1 int(11) NOT NULL, +PRIMARY KEY (pk), +KEY key1 (key1), +KEY key2 (key2) +) ENGINE=InnoDB AUTO_INCREMENT=12860259 DEFAULT CHARSET=utf8 ROW_FORMAT=COMPACT; + +create table t2(a int); +insert into t2 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); + +create table t3(a int); +insert into t3 select A.a + B.a* 10 + C.a * 100 + D.a*1000 from t2 A, t2 B, t2 C, t2 D; + +insert into t1 (key1, key2, col1,col2,col3,col4) +select a,a, a,a,a,a from t3; +SELECT sum(col1) FROM t1 FORCE INDEX (key1,key2) WHERE (key1 between 10 and 8191+10) or (key2= 5); +drop table t1,t2,t3; +set optimizer_switch=@tmp_optimizer_switch; diff --git a/mysql-test/t/information_schema_part.test b/mysql-test/t/information_schema_part.test index f1415d12f79..ea88f364c07 100644 --- a/mysql-test/t/information_schema_part.test +++ b/mysql-test/t/information_schema_part.test @@ -131,3 +131,10 @@ drop table if exists t1; create table t1 (f1 int key) partition by key(f1) partitions 2; select create_options from information_schema.tables where table_schema="test"; drop table t1; + +--echo # +--echo # MDEV-11353 - Identical logical conditions +--echo # +CREATE TABLE t1(a INT) CHECKSUM=1 SELECT 1; +SELECT CHECKSUM FROM INFORMATION_SCHEMA.PARTITIONS WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DROP TABLE t1; diff --git a/mysql-test/t/insert_update.test b/mysql-test/t/insert_update.test index de38ae0b0d3..7234973eeb8 100644 --- a/mysql-test/t/insert_update.test +++ b/mysql-test/t/insert_update.test @@ -170,6 +170,7 @@ DROP TABLE t1,t2; SET SQL_MODE = 'TRADITIONAL'; CREATE TABLE t1 (a INT PRIMARY KEY, b INT NOT NULL); +INSERT INTO t1 VALUES (1,1); --error ER_NO_DEFAULT_FOR_FIELD INSERT INTO t1 (a) VALUES (1); @@ -177,7 +178,10 @@ INSERT INTO t1 (a) VALUES (1); --error ER_NO_DEFAULT_FOR_FIELD INSERT INTO t1 (a) VALUES (1) ON DUPLICATE KEY UPDATE a = b; ---error ER_NO_DEFAULT_FOR_FIELD +# this one is ok +INSERT INTO t1 (a) VALUES (1) ON DUPLICATE KEY UPDATE b = a; + +# arguably the statement below should fail INSERT INTO t1 (a) VALUES (1) ON DUPLICATE KEY UPDATE b = b; SELECT * FROM t1; diff --git a/mysql-test/t/join_cache.test b/mysql-test/t/join_cache.test index 019d8edde2f..5ee868b4177 100644 --- a/mysql-test/t/join_cache.test +++ b/mysql-test/t/join_cache.test @@ -3742,9 +3742,11 @@ FROM LEFT JOIN t2 c23 ON c23.parent_id = t.id AND c23.col2 = "val" LEFT JOIN t2 c24 ON c24.parent_id = t.id AND c24.col2 = "val" LEFT JOIN t2 c25 ON c25.parent_id = t.id AND c25.col2 = "val" + LEFT JOIN t2 c26 ON c26.parent_id = t.id AND c26.col2 = "val" + LEFT JOIN t2 c27 ON c27.parent_id = t.id AND c27.col2 = "val" ORDER BY col1; -select timestampdiff(second, @init_time, now()) <= 1; +select timestampdiff(second, @init_time, now()) <= 5; set join_cache_level=2; @@ -3777,9 +3779,11 @@ FROM LEFT JOIN t2 c23 ON c23.parent_id = t.id AND c23.col2 = "val" LEFT JOIN t2 c24 ON c24.parent_id = t.id AND c24.col2 = "val" LEFT JOIN t2 c25 ON c25.parent_id = t.id AND c25.col2 = "val" + LEFT JOIN t2 c26 ON c26.parent_id = t.id AND c26.col2 = "val" + LEFT JOIN t2 c27 ON c27.parent_id = t.id AND c27.col2 = "val" ORDER BY col1; -select timestampdiff(second, @init_time, now()) <= 1; +select timestampdiff(second, @init_time, now()) <= 5; EXPLAIN SELECT t.* @@ -3810,6 +3814,8 @@ FROM LEFT JOIN t2 c23 ON c23.parent_id = t.id AND c23.col2 = "val" LEFT JOIN t2 c24 ON c24.parent_id = t.id AND c24.col2 = "val" LEFT JOIN t2 c25 ON c25.parent_id = t.id AND c25.col2 = "val" + LEFT JOIN t2 c26 ON c26.parent_id = t.id AND c26.col2 = "val" + LEFT JOIN t2 c27 ON c27.parent_id = t.id AND c27.col2 = "val" ORDER BY col1; diff --git a/mysql-test/t/loaddata.test b/mysql-test/t/loaddata.test index f827883af85..88c4a363a57 100644 --- a/mysql-test/t/loaddata.test +++ b/mysql-test/t/loaddata.test @@ -612,7 +612,7 @@ disconnect con1; --echo # CREATE TABLE t1(f1 INT); -EVAL SELECT 0xE1C330 INTO OUTFILE 't1.dat'; +EVAL SELECT 0xE1BB30 INTO OUTFILE 't1.dat'; --disable_warnings LOAD DATA INFILE 't1.dat' IGNORE INTO TABLE t1 CHARACTER SET utf8; --enable_warnings @@ -658,27 +658,21 @@ SET @@sql_mode= @old_mode; --remove_file $MYSQLTEST_VARDIR/mysql DROP TABLE t1; ---echo + +--echo # +--echo # MDEV-11079 Regression: LOAD DATA INFILE lost BLOB support using utf8 load files +--echo # + +CREATE TABLE t1 (a mediumblob NOT NULL) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_general_ci; +LOAD DATA INFILE '../../std_data/loaddata/mdev-11079.txt' INTO TABLE t1 CHARSET utf8 FIELDS TERMINATED BY ';' ENCLOSED BY '"' ESCAPED BY '\\' LINES TERMINATED BY '\n'; +SELECT HEX(a) FROM t1; +DROP TABLE t1; + --echo # ---echo # Bug#23080148 - Backport of Bug#20683959. ---echo # Bug#20683959 LOAD DATA INFILE IGNORES A SPECIFIC ROW SILENTLY ---echo # UNDER DB CHARSET IS UTF8. +--echo # MDEV-11631 LOAD DATA INFILE fails to load data with an escape character followed by a multi-byte character --echo # -CREATE DATABASE d1 CHARSET latin1; -USE d1; -CREATE TABLE t1 (val TEXT); -LOAD DATA INFILE '../../std_data/bug20683959loaddata.txt' INTO TABLE t1; -SELECT COUNT(*) FROM t1; -SELECT HEX(val) FROM t1; - -CREATE DATABASE d2 CHARSET utf8; -USE d2; -CREATE TABLE t1 (val TEXT); -LOAD DATA INFILE '../../std_data/bug20683959loaddata.txt' IGNORE INTO TABLE t1; -SELECT COUNT(*) FROM t1; -SELECT HEX(val) FROM t1; - -DROP TABLE d1.t1, d2.t1; -DROP DATABASE d1; -DROP DATABASE d2; +CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET utf8); +LOAD DATA INFILE '../../std_data/loaddata/mdev-11631.txt' INTO TABLE t1 CHARACTER SET utf8; +SELECT HEX(a) FROM t1; +DROP TABLE t1; diff --git a/mysql-test/t/log_slow.test b/mysql-test/t/log_slow.test index 8d5a09d7a94..56e35bd5a20 100644 --- a/mysql-test/t/log_slow.test +++ b/mysql-test/t/log_slow.test @@ -50,7 +50,6 @@ set global slow_query_log=1; set global log_output='TABLE'; select sleep(0.5); select count(*) FROM mysql.slow_log; -truncate mysql.slow_log; # Reset used variables set @@long_query_time=default; @@ -58,3 +57,4 @@ set global slow_query_log= @org_slow_query_log; set @@log_slow_filter=default; set @@log_slow_verbosity=default; set global log_output= default; +truncate mysql.slow_log; diff --git a/mysql-test/t/order_by.test b/mysql-test/t/order_by.test index 6baa66112da..92c7cdf44b6 100644 --- a/mysql-test/t/order_by.test +++ b/mysql-test/t/order_by.test @@ -1948,6 +1948,16 @@ order by A.col2, B.col2 limit 10, 1000000; drop table t1,t2,t3; +--echo # +--echo # mdev-10705 : long order by list that can be skipped +--echo # + +SELECT 1 +UNION +( SELECT 2 + ORDER BY NULL, @a0 := 3, @a1 := 3, @a2 := 3, @a3 := 3, @a4 := 3, + @a5 := 3, @a6 := 3, @a7 := 3, @a8 := 3, @a9 := 3, @a10 := 3 ); + --echo End of 5.5 tests --echo # diff --git a/mysql-test/t/subselect.test b/mysql-test/t/subselect.test index ffb92222c84..44a400dca20 100644 --- a/mysql-test/t/subselect.test +++ b/mysql-test/t/subselect.test @@ -6017,6 +6017,17 @@ drop view v2; drop table t1,t2; --echo # +--echo # MDEV-10386 Assertion `fixed == 1' failed in virtual String* Item_func_conv_charset::val_str(String*) +--echo # + +CREATE TABLE t1 (f1 CHAR(3) CHARACTER SET utf8 NULL, f2 CHAR(3) CHARACTER SET latin1 NULL); +INSERT INTO t1 VALUES ('foo','bar'); +SELECT * FROM t1 WHERE f2 >= SOME ( SELECT f1 FROM t1 ); +SELECT * FROM t1 WHERE f2 <= SOME ( SELECT f1 FROM t1 ); +DROP TABLE t1; + + +--echo # --echo # MDEV-9487: Server crashes in Time_and_counter_tracker::incr_loops --echo # with UNION in ALL subquery --echo # diff --git a/mysql-test/t/subselect2.test b/mysql-test/t/subselect2.test index bd34546d83f..c106d40b095 100644 --- a/mysql-test/t/subselect2.test +++ b/mysql-test/t/subselect2.test @@ -359,5 +359,55 @@ where t1.a = t2.a and ( t1.a = ( select min(a) from t1 ) or 0 ); drop table t1,t2,t3; +--echo # +--echo # MDEV-10148: Database crashes in the query to the View +--echo # +CREATE TABLE t1 ( + key_code INT(11) NOT NULL, + value_string VARCHAR(50) NULL DEFAULT NULL, + PRIMARY KEY (key_code) +) COLLATE='utf8_general_ci' ENGINE=InnoDB ; + +CREATE TABLE t2 ( + key_code INT(11) NOT NULL, + target_date DATE NULL DEFAULT NULL, + PRIMARY KEY (key_code) +) COLLATE='utf8_general_ci' ENGINE=InnoDB ; + +CREATE TABLE t3 ( + now_date DATE NOT NULL, + PRIMARY KEY (now_date) +) COLLATE='utf8_general_ci' ENGINE=InnoDB ; + +CREATE VIEW v1 +AS +SELECT + B.key_code, + B.target_date +FROM + t2 B INNER JOIN t3 C ON + B.target_date = C.now_date +; +SET @s = 'SELECT A.* FROM t1 A WHERE A.key_code IN (SELECT key_code FROM v1)'; +PREPARE stmt FROM @s; +EXECUTE stmt; #1st time -> success +EXECUTE stmt; #2nd time -> crash +DEALLOCATE PREPARE stmt; +DROP VIEW v1; +DROP TABLE t1,t2,t3; + set optimizer_switch=@subselect2_test_tmp; +# +# Bug #23303485 : HANDLE_FATAL_SIGNAL (SIG=11) IN SUBSELECT_UNION_ENGINE::NO_ROWS +# +create table t1 (a int); +create table t2 (a int); +create table t3(a int); +insert into t1 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +insert into t2 select a from t1; +insert into t3 select a from t1; +--error ER_SUBQUERY_NO_1_ROW +select null in (select a from t1 where a < out3.a union select a from t2 where + (select a from t3) +1 < out3.a+1) from t3 out3; +drop table t1, t2, t3; diff --git a/mysql-test/t/subselect4.test b/mysql-test/t/subselect4.test index 4eb9701ee71..253160c46ac 100644 --- a/mysql-test/t/subselect4.test +++ b/mysql-test/t/subselect4.test @@ -1956,5 +1956,56 @@ SELECT x FROM t1 WHERE id > (SELECT MAX(id) - 1000 FROM t1) ORDER BY x LIMIT 1; drop table t1; +--echo # +--echo # MDEV-7691: Assertion `outer_context || !*from_field || *from_field == not_found_field' ... +--echo # +set optimizer_switch=default; +CREATE TABLE t1 (a INT) ENGINE=MyISAM; +INSERT INTO t1 VALUES (4),(6); + +CREATE TABLE t2 (b INT) ENGINE=MyISAM; +INSERT INTO t2 VALUES (1),(8); + +PREPARE stmt FROM " +SELECT * FROM t2 +HAVING 0 IN ( + SELECT a FROM t1 + WHERE a IN ( + SELECT a FROM t1 + WHERE b = a + ) +) +"; + +EXECUTE stmt; +EXECUTE stmt; + +--echo # Alternative test case, without HAVING +CREATE TABLE t3 (i INT) ENGINE=MyISAM; +INSERT INTO t3 VALUES (4),(6); + +PREPARE stmt FROM " +SELECT * FROM t3 AS t10 +WHERE EXISTS ( + SELECT * FROM t3 AS t20 WHERE t10.i IN ( + SELECT i FROM t3 + ) +)"; + +EXECUTE stmt; +EXECUTE stmt; + +drop table t1, t2, t3; + SET optimizer_switch= @@global.optimizer_switch; set @@tmp_table_size= @@global.tmp_table_size; + +--echo # +--echo # MDEV-10232 Scalar result of subquery changes after adding an outer select stmt +--echo # + +create table t1(c1 int, c2 int, primary key(c2)); +insert into t1 values(2,1),(1,2); +select (select c1 from t1 group by c1,c2 order by c1 limit 1) as x; +(select c1 from t1 group by c1,c2 order by c1 limit 1); +drop table t1; diff --git a/mysql-test/t/subselect_cache.test b/mysql-test/t/subselect_cache.test index 21247541fb6..55da0000f13 100644 --- a/mysql-test/t/subselect_cache.test +++ b/mysql-test/t/subselect_cache.test @@ -24,10 +24,10 @@ select a, (select d from t2 where b=c) from t1; show status like "subquery_cache%"; show status like '%Handler_read%'; ---replace_regex /"r_total_time_ms": [0-9]*[.]?[0-9]*/"r_total_time_ms": "REPLACED"/ +--source include/analyze-format.inc analyze format=json select a, (select d from t2 where b=c) from t1; ---replace_regex /"r_total_time_ms": [0-9]*[.]?[0-9]*/"r_total_time_ms": "REPLACED"/ +--source include/analyze-format.inc analyze format=json select a, (select d from t2 where b=c), (select d from t2 where b=c union select 1 order by 1 limit 1) from t1; explain format=json diff --git a/mysql-test/t/trigger_no_defaults-11698.test b/mysql-test/t/trigger_no_defaults-11698.test new file mode 100644 index 00000000000..fab7845ad7d --- /dev/null +++ b/mysql-test/t/trigger_no_defaults-11698.test @@ -0,0 +1,25 @@ +# +# MDEV-11698 Old Bug possibly not fixed; BEFORE INSERT Trigger on NOT NULL +# +set sql_mode='strict_all_tables'; +create table t1 (a int not null, b int); +--error ER_NO_DEFAULT_FOR_FIELD +insert t1 (b) values (1); +delimiter |; +create trigger trgi before insert on t1 for each row + case new.b + when 10 then + set new.a = new.b; + when 30 then + set new.a = new.a; + else + do 1; + end case| +delimiter ;| +insert t1 (b) values (10); +--error ER_NO_DEFAULT_FOR_FIELD +insert t1 (b) values (20); +# arguably the statement below should fail too +insert t1 (b) values (30); +select * from t1; +drop table t1; diff --git a/mysql-test/t/union.test b/mysql-test/t/union.test index 151512515b9..e0c011e3d20 100644 --- a/mysql-test/t/union.test +++ b/mysql-test/t/union.test @@ -1057,7 +1057,6 @@ ORDER BY a; DROP TABLE t1; ---echo End of 5.0 tests -- echo # -- echo # Bug#32858: Error: "Incorrect usage of UNION and INTO" does not take -- echo # subselects into account @@ -1160,6 +1159,8 @@ create table t1 (a int); insert into t1 values (10),(10),(10),(2),(3),(4),(5),(6),(7),(8),(9),(1),(10); --sorted_result select a from t1 where false UNION select a from t1 limit 8; +--sorted_result +(select a from t1 where false) UNION (select a from t1) limit 8; drop table t1; --echo # @@ -1385,6 +1386,26 @@ UNION drop table t1; + +--echo # +--echo # MDEV-10172: UNION query returns incorrect rows outside +--echo # conditional evaluation +--echo # + +create table t1 (d datetime not null primary key); +insert into t1(d) values ('2016-06-01'),('2016-06-02'),('2016-06-03'),('2016-06-04'); +select * from +( + select * from t1 where d between '2016-06-02' and '2016-06-05' + union + (select * from t1 where d < '2016-06-05' order by d desc limit 1) +) onlyJun2toJun4 +order by d; +drop table t1; + +--echo End of 5.0 tests + + --echo # --echo # WL#1763 Avoid creating temporary table in UNION ALL --echo # @@ -1486,4 +1507,3 @@ SELECT * FROM t1 t1_1 LEFT JOIN t1 t1_2 ON ( t1_2.b = t1_1.a ) WHERE t1_2.b NOT IN ( SELECT 4 UNION ALL SELECT 5 ); DROP TABLE t1; - diff --git a/mysql-test/valgrind.supp b/mysql-test/valgrind.supp index 20d4b124dd7..6bdbcab4530 100644 --- a/mysql-test/valgrind.supp +++ b/mysql-test/valgrind.supp @@ -950,6 +950,14 @@ fun:backtrace } +{ + memory leak in mysqld_exit + Memcheck:Leak + fun:malloc + fun:_dl_close_worker + fun:_dl_close +} + # # Bug in Glibc 2.9: http://sourceware.org/bugzilla/show_bug.cgi?id=10391 # Fixed in latest Glibc, but suppressed here for running tests on hosts diff --git a/mysys/CMakeLists.txt b/mysys/CMakeLists.txt index 6c43c29a758..13ea8dd9f35 100644 --- a/mysys/CMakeLists.txt +++ b/mysys/CMakeLists.txt @@ -50,7 +50,7 @@ IF (WIN32) ENDIF() IF(UNIX) - SET (MYSYS_SOURCES ${MYSYS_SOURCES} my_addr_resolve.c) + SET (MYSYS_SOURCES ${MYSYS_SOURCES} my_addr_resolve.c my_setuser.c) ENDIF() IF(HAVE_ALARM) diff --git a/mysys/my_getopt.c b/mysys/my_getopt.c index 3f75f6553a2..57b28d1fd8a 100644 --- a/mysys/my_getopt.c +++ b/mysys/my_getopt.c @@ -949,15 +949,39 @@ my_bool getopt_compare_strings(register const char *s, register const char *t, /* function: eval_num_suffix + Transforms suffix like k/m/g to their real value. +*/ + +static inline long eval_num_suffix(char *suffix, int *error) +{ + long num= 1; + if (*suffix == 'k' || *suffix == 'K') + num*= 1024L; + else if (*suffix == 'm' || *suffix == 'M') + num*= 1024L * 1024L; + else if (*suffix == 'g' || *suffix == 'G') + num*= 1024L * 1024L * 1024L; + else if (*suffix) + { + *error= 1; + return 0; + } + return num; +} + +/* + function: eval_num_suffix_ll + Transforms a number with a suffix to real number. Suffix can be k|K for kilo, m|M for mega or g|G for giga. */ -static longlong eval_num_suffix(char *argument, int *error, char *option_name) +static longlong eval_num_suffix_ll(char *argument, + int *error, char *option_name) { char *endchar; longlong num; - DBUG_ENTER("eval_num_suffix"); + DBUG_ENTER("eval_num_suffix_ll"); *error= 0; @@ -970,23 +994,47 @@ static longlong eval_num_suffix(char *argument, int *error, char *option_name) *error= 1; DBUG_RETURN(0); } - if (*endchar == 'k' || *endchar == 'K') - num*= 1024L; - else if (*endchar == 'm' || *endchar == 'M') - num*= 1024L * 1024L; - else if (*endchar == 'g' || *endchar == 'G') - num*= 1024L * 1024L * 1024L; - else if (*endchar) - { + num*= eval_num_suffix(endchar, error); + if (*error) fprintf(stderr, "Unknown suffix '%c' used for variable '%s' (value '%s')\n", *endchar, option_name, argument); + DBUG_RETURN(num); +} + +/* + function: eval_num_suffix_ull + + Transforms a number with a suffix to positive Integer. Suffix can + be k|K for kilo, m|M for mega or g|G for giga. +*/ + +static ulonglong eval_num_suffix_ull(char *argument, + int *error, char *option_name) +{ + char *endchar; + ulonglong num; + DBUG_ENTER("eval_num_suffix_ull"); + + *error= 0; + errno= 0; + num= strtoull(argument, &endchar, 10); + if (errno == ERANGE) + { + my_getopt_error_reporter(ERROR_LEVEL, + "Incorrect integer value: '%s'", argument); *error= 1; DBUG_RETURN(0); } + num*= eval_num_suffix(endchar, error); + if (*error) + fprintf(stderr, + "Unknown suffix '%c' used for variable '%s' (value '%s')\n", + *endchar, option_name, argument); DBUG_RETURN(num); } + /* function: getopt_ll @@ -1000,7 +1048,7 @@ static longlong eval_num_suffix(char *argument, int *error, char *option_name) static longlong getopt_ll(char *arg, const struct my_option *optp, int *err) { - longlong num=eval_num_suffix(arg, err, (char*) optp->name); + longlong num=eval_num_suffix_ll(arg, err, (char*) optp->name); return getopt_ll_limit_value(num, optp, NULL); } @@ -1077,7 +1125,7 @@ longlong getopt_ll_limit_value(longlong num, const struct my_option *optp, static ulonglong getopt_ull(char *arg, const struct my_option *optp, int *err) { - ulonglong num= eval_num_suffix(arg, err, (char*) optp->name); + ulonglong num= eval_num_suffix_ull(arg, err, (char*) optp->name); return getopt_ull_limit_value(num, optp, NULL); } diff --git a/mysys/my_setuser.c b/mysys/my_setuser.c new file mode 100644 index 00000000000..14ab04dd10f --- /dev/null +++ b/mysys/my_setuser.c @@ -0,0 +1,82 @@ +#include <my_global.h> +#include <m_string.h> +#include <my_sys.h> +#include <my_pthread.h> +#ifdef HAVE_PWD_H +#include <pwd.h> +#endif +#ifdef HAVE_GRP_H +#include <grp.h> +#endif + +struct passwd *my_check_user(const char *user, myf MyFlags) +{ + struct passwd *user_info; + uid_t user_id= geteuid(); + DBUG_ENTER("my_check_user"); + + // Don't bother if we aren't superuser + if (user_id) + { + if (user) + { + /* Don't give a warning, if real user is same as given with --user */ + user_info= getpwnam(user); + if (!user_info || user_id != user_info->pw_uid) + { + my_errno= EPERM; + if (MyFlags & MY_WME) + my_printf_error(my_errno, "One can only use the --user switch if " + "running as root", MYF(ME_JUST_WARNING|ME_NOREFRESH)); + } + } + DBUG_RETURN(NULL); + } + if (!user) + { + if (MyFlags & MY_FAE) + { + my_errno= EINVAL; + my_printf_error(my_errno, "Please consult the Knowledge Base to find " + "out how to run mysqld as root!", MYF(ME_NOREFRESH)); + } + DBUG_RETURN(NULL); + } + if (!strcmp(user,"root")) + DBUG_RETURN(NULL); + + if (!(user_info= getpwnam(user))) + { + // Allow a numeric uid to be used + int err= 0; + user_id= my_strtoll10(user, NULL, &err); + if (err || !(user_info= getpwuid(user_id))) + { + my_errno= EINVAL; + my_printf_error(my_errno, "Can't change to run as user '%s'. Please " + "check that the user exists!", MYF(ME_NOREFRESH), user); + DBUG_RETURN(NULL); + } + } + DBUG_ASSERT(user_info); + DBUG_RETURN(user_info); +} + +int my_set_user(const char *user, struct passwd *user_info, myf MyFlags) +{ + DBUG_ENTER("my_set_user"); + + DBUG_ASSERT(user_info != 0); +#ifdef HAVE_INITGROUPS + initgroups(user, user_info->pw_gid); +#endif + if (setgid(user_info->pw_gid) == -1 || setuid(user_info->pw_uid) == -1) + { + my_errno= errno; + if (MyFlags & MY_WME) + my_printf_error(errno, "Cannot change uid/gid (errno: %d)", MYF(ME_NOREFRESH), + errno); + DBUG_RETURN(my_errno); + } + DBUG_RETURN(0); +} diff --git a/plugin/server_audit/server_audit.c b/plugin/server_audit/server_audit.c index a06c0409cd6..95b9586a1c5 100644 --- a/plugin/server_audit/server_audit.c +++ b/plugin/server_audit/server_audit.c @@ -2843,6 +2843,7 @@ void __attribute__ ((constructor)) audit_plugin_so_init(void) _mysql_plugin_declarations_[0].info= mysql_v4_descriptor; use_event_data_for_disconnect= 1; } + MYSQL_SYSVAR_NAME(loc_info).flags= PLUGIN_VAR_READONLY | PLUGIN_VAR_MEMALLOC; } memset(locinfo_ini_value, 'O', sizeof(locinfo_ini_value)-1); diff --git a/scripts/mariadb-service-convert b/scripts/mariadb-service-convert index 42654f68420..83c92659144 100755 --- a/scripts/mariadb-service-convert +++ b/scripts/mariadb-service-convert @@ -16,7 +16,7 @@ # # PURPOSE: # -# Used to generate a mariadb.service file based on the curent mysql/maridb settings +# Used to generate a mariadb.service file based on the current mysql/mariadb settings # # This is to assist distro maintainers in migrating to systemd service definations from # a user mysqld_safe settings in the my.cnf files. diff --git a/scripts/mysql_install_db.sh b/scripts/mysql_install_db.sh index 3b48dad649c..f8f4859e627 100644 --- a/scripts/mysql_install_db.sh +++ b/scripts/mysql_install_db.sh @@ -35,11 +35,26 @@ force=0 in_rpm=0 ip_only=0 cross_bootstrap=0 +install_params="" +auth_root_authentication_method=normal +auth_root_socket_user='root' usage() { cat <<EOF Usage: $0 [OPTIONS] + --auth-root-authentication-method=normal|socket + Chooses the authentication method for the created initial + root user. The default is 'normal' to creates a root user + that can login without password, which can be insecure. + The alternative 'socket' allows only the system root user + to login as MariaDB root; this requires the unix socket + authentication plugin. + --auth-root-socket-user=user + Used with --auth-root-authentication-method=socket. It + specifies the name of the MariaDB root account, as well + as of the system account allowed to access it. Defaults + to 'root'. --basedir=path The path to the MariaDB installation directory. --builddir=path If using --srcdir with out-of-directory builds, you will need to set this to the location of the build @@ -60,6 +75,8 @@ Usage: $0 [OPTIONS] --defaults-file=path Read only this configuration file. --rpm For internal use. This option is used by RPM files during the MariaDB installation process. + --skip-auth-anonymous-user + Do not install an unprivileged anonymous user. --skip-name-resolve Use IP addresses rather than hostnames when creating grant table entries. This option can be useful if your DNS does not work. @@ -142,6 +159,17 @@ parse_arguments() # # --windows is a deprecated alias cross_bootstrap=1 ;; + --skip-auth-anonymous-user) + install_params="$install_params +SET @skip_auth_anonymous=1;" ;; + --auth-root-authentication-method=normal) + auth_root_authentication_method=normal ;; + --auth-root-authentication-method=socket) + auth_root_authentication_method=socket ;; + --auth-root-authentication-method=*) + usage ;; + --auth-root-socket-user=*) + auth_root_socket_user="$(parse_arg "$arg")" ;; *) if test -n "$pick_args" @@ -431,7 +459,17 @@ mysqld_install_cmd_line() # Create the system and help tables by passing them to "mysqld --bootstrap" s_echo "Installing MariaDB/MySQL system tables in '$ldata' ..." -if { echo "use mysql;"; cat "$create_system_tables" "$create_system_tables2" "$fill_system_tables" "$fill_help_tables" "$maria_add_gis_sp"; } | eval "$filter_cmd_line" | mysqld_install_cmd_line > /dev/null +case "$auth_root_authentication_method" in + normal) + install_params="$install_params +SET @skip_auth_root_nopasswd=NULL; +SET @auth_root_socket=NULL;" ;; + socket) + install_params="$install_params +SET @skip_auth_root_nopasswd=1; +SET @auth_root_socket='$auth_root_socket_user';" ;; +esac +if { echo "use mysql;$install_params"; cat "$create_system_tables" "$create_system_tables2" "$fill_system_tables" "$fill_help_tables" "$maria_add_gis_sp"; } | eval "$filter_cmd_line" | mysqld_install_cmd_line > /dev/null then s_echo "OK" else diff --git a/scripts/mysql_system_tables_data.sql b/scripts/mysql_system_tables_data.sql index 075aafd5e3b..9556e7ba160 100644 --- a/scripts/mysql_system_tables_data.sql +++ b/scripts/mysql_system_tables_data.sql @@ -38,15 +38,24 @@ DROP TABLE tmp_db; -- Fill "user" table with default users allowing root access -- from local machine if "user" table didn't exist before -CREATE TEMPORARY TABLE tmp_user LIKE user; -INSERT INTO tmp_user VALUES ('localhost','root','','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'','','N', 'N','', 0); -REPLACE INTO tmp_user SELECT @current_hostname,'root','','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'','','N','N','',0 FROM dual WHERE @current_hostname != 'localhost'; -REPLACE INTO tmp_user VALUES ('127.0.0.1','root','','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'','','N','N','',0); -REPLACE INTO tmp_user VALUES ('::1','root','','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'','','N','N', '', 0); -INSERT INTO tmp_user (host,user) VALUES ('localhost',''); -INSERT INTO tmp_user (host,user) SELECT @current_hostname,'' FROM dual WHERE @current_hostname != 'localhost'; -INSERT INTO user SELECT * FROM tmp_user WHERE @had_user_table=0; -DROP TABLE tmp_user; +CREATE TEMPORARY TABLE tmp_user_nopasswd LIKE user; +CREATE TEMPORARY TABLE tmp_user_socket LIKE user; +CREATE TEMPORARY TABLE tmp_user_anonymous LIKE user; +-- Classic passwordless root account. +INSERT INTO tmp_user_nopasswd VALUES ('localhost','root','','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'','','N', 'N','', 0); +REPLACE INTO tmp_user_nopasswd SELECT @current_hostname,'root','','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'','','N','N','',0 FROM dual WHERE @current_hostname != 'localhost'; +REPLACE INTO tmp_user_nopasswd VALUES ('127.0.0.1','root','','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'','','N','N','',0); +REPLACE INTO tmp_user_nopasswd VALUES ('::1','root','','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'','','N','N', '', 0); +-- More secure root account using unix sucket auth. +INSERT INTO tmp_user_socket VALUES ('localhost',IFNULL(@auth_root_socket, 'root'),'','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'unix_socket','','N', 'N','', 0); +-- Anonymous user with no privileges. +INSERT INTO tmp_user_anonymous (host,user) VALUES ('localhost',''); +INSERT INTO tmp_user_anonymous (host,user) SELECT @current_hostname,'' FROM dual WHERE @current_hostname != 'localhost'; + +INSERT INTO user SELECT * FROM tmp_user_nopasswd WHERE @had_user_table=0 AND @skip_auth_root_nopasswd IS NULL; +INSERT INTO user SELECT * FROM tmp_user_socket WHERE @had_user_table=0 AND @auth_root_socket IS NOT NULL; +INSERT INTO user SELECT * FROM tmp_user_anonymous WHERE @had_user_table=0 AND @skip_auth_anonymous IS NULL; +DROP TABLE tmp_user_nopasswd, tmp_user_socket, tmp_user_anonymous; CREATE TEMPORARY TABLE tmp_proxies_priv LIKE proxies_priv; INSERT INTO tmp_proxies_priv VALUES ('localhost', 'root', '', '', TRUE, '', now()); diff --git a/scripts/mysqld_safe.sh b/scripts/mysqld_safe.sh index 000271f7808..48baf61be3a 100644 --- a/scripts/mysqld_safe.sh +++ b/scripts/mysqld_safe.sh @@ -132,6 +132,18 @@ my_which () return $ret # Success } +find_in_bin() { + if test -x "$MY_BASEDIR_VERSION/bin/$1" + then + echo "$MY_BASEDIR_VERSION/bin/$1" + elif test -x "@bindir@/$1" + then + echo "@bindir@/$1" + else + echo "$1" + fi +} + log_generic () { [ $dry_run -eq 1 ] && return priority="$1" @@ -141,7 +153,7 @@ log_generic () { echo "$msg" case $logging in init) ;; # Just echo the message, don't save it anywhere - file) echo "$msg" >> "$err_log" ;; + file) echo "$msg" | "$helper" "$user" log "$err_log" ;; syslog) logger -t "$syslog_tag_mysqld_safe" -p "$priority" "$*" ;; *) echo "Internal program error (non-fatal):" \ @@ -161,7 +173,7 @@ log_notice () { eval_log_error () { local cmd="$1" case $logging in - file) cmd="$cmd >> "`shell_quote_string "$err_log"`" 2>&1" ;; + file) cmd="$cmd 2>&1 | "`shell_quote_string "$helper"`" $user log "`shell_quote_string "$err_log"` ;; syslog) # mysqld often prefixes its messages with a timestamp, which is # redundant when logging to syslog (which adds its own timestamp) @@ -282,7 +294,6 @@ parse_arguments() { for arg do val=`echo "$arg" | sed -e "s;--[^=]*=;;"` case "$arg" in - --crash[-_]script=*) CRASH_SCRIPT="$val" ;; # these get passed explicitly to mysqld --basedir=*) MY_BASEDIR_VERSION="$val" ;; --datadir=*|--data=*) DATADIR="$val" ;; @@ -311,6 +322,7 @@ parse_arguments() { --core[-_]file[-_]size=*) core_file_size="$val" ;; --ledir=*) ledir="$val" ;; --malloc[-_]lib=*) set_malloc_lib "$val" ;; + --crash[-_]script=*) crash_script="$val" ;; --mysqld=*) MYSQLD="$val" ;; --mysqld[-_]version=*) if test -n "$val" @@ -498,15 +510,8 @@ set_malloc_lib() { # First, try to find BASEDIR and ledir (where mysqld is) # -if echo '@pkgdatadir@' | grep '^@prefix@' > /dev/null -then - relpkgdata=`echo '@pkgdatadir@' | sed -e 's,^@prefix@,,' -e 's,^/,,' -e 's,^,./,'` -else - # pkgdatadir is not relative to prefix - relpkgdata='@pkgdatadir@' -fi - -MY_PWD=`pwd` +MY_PWD=`dirname $0` +MY_PWD=`cd "$MY_PWD"/.. && pwd` # Check for the directories we would expect from a binary release install if test -n "$MY_BASEDIR_VERSION" -a -d "$MY_BASEDIR_VERSION" then @@ -522,16 +527,16 @@ then else ledir="$MY_BASEDIR_VERSION/bin" fi -elif test -f "$relpkgdata"/english/errmsg.sys -a -x "$MY_PWD/bin/mysqld" +elif test -x "$MY_PWD/bin/mysqld" then MY_BASEDIR_VERSION="$MY_PWD" # Where bin, share and data are ledir="$MY_PWD/bin" # Where mysqld is # Check for the directories we would expect from a source install -elif test -f "$relpkgdata"/english/errmsg.sys -a -x "$MY_PWD/libexec/mysqld" +elif test -x "$MY_PWD/libexec/mysqld" then MY_BASEDIR_VERSION="$MY_PWD" # Where libexec, share and var are ledir="$MY_PWD/libexec" # Where mysqld is -elif test -f "$relpkgdata"/english/errmsg.sys -a -x "$MY_PWD/sbin/mysqld" +elif test -x "$MY_PWD/sbin/mysqld" then MY_BASEDIR_VERSION="$MY_PWD" # Where sbin, share and var are ledir="$MY_PWD/sbin" # Where mysqld is @@ -541,6 +546,8 @@ else ledir='@libexecdir@' fi +helper=`find_in_bin mysqld_safe_helper` +print_defaults=`find_in_bin my_print_defaults` # # Second, try to find the data directory @@ -571,34 +578,15 @@ $MY_BASEDIR_VERSION/my.cnf" fi export MYSQL_HOME - -# Get first arguments from the my.cnf file, groups [mysqld] and [mysqld_safe] -# and then merge with the command line arguments -if test -x "$MY_BASEDIR_VERSION/bin/my_print_defaults" -then - print_defaults="$MY_BASEDIR_VERSION/bin/my_print_defaults" -elif test -x `dirname $0`/my_print_defaults -then - print_defaults="`dirname $0`/my_print_defaults" -elif test -x ./bin/my_print_defaults -then - print_defaults="./bin/my_print_defaults" -elif test -x @bindir@/my_print_defaults -then - print_defaults="@bindir@/my_print_defaults" -elif test -x @bindir@/mysql_print_defaults -then - print_defaults="@bindir@/mysql_print_defaults" -else - print_defaults="my_print_defaults" -fi - append_arg_to_args () { args="$args "`shell_quote_string "$1"` } args= +# Get first arguments from the my.cnf file, groups [mysqld] and [mysqld_safe] +# and then merge with the command line arguments + SET_USER=2 parse_arguments `$print_defaults $defaults --loose-verbose --mysqld` if test $SET_USER -eq 2 @@ -707,11 +695,6 @@ then log_notice "Logging to '$err_log'." logging=file - if [ ! -f "$err_log" ]; then # if error log already exists, - touch "$err_log" # we just append. otherwise, - chmod "$fmode" "$err_log" # fix the permissions here! - fi - else if [ -n "$syslog_tag" ] then @@ -731,11 +714,6 @@ then then USER_OPTION="--user=$user" fi - # Change the err log to the right user, if it is in use - if [ $want_syslog -eq 0 ]; then - touch "$err_log" - chown $user "$err_log" - fi if test -n "$open_files" then ulimit -n $open_files @@ -973,10 +951,6 @@ cmd="$cmd $args" # Avoid 'nohup: ignoring input' warning test -n "$NOHUP_NICENESS" && cmd="$cmd < /dev/null" -# close stdout and stderr, everything goes to $logging now -exec 1>&- -exec 2>&- - log_notice "Starting $MYSQLD daemon with databases from $DATADIR" # variable to track the current number of "fast" (a.k.a. subsecond) restarts @@ -986,6 +960,15 @@ max_fast_restarts=5 # flag whether a usable sleep command exists have_sleep=1 +# close stdout and stderr, everything goes to $logging now +if expr "${-}" : '.*x' > /dev/null +then + : +else + exec 1>&- + exec 2>&- +fi + # maximum number of wsrep restarts max_wsrep_restarts=0 @@ -1014,13 +997,6 @@ do else eval_log_error "$cmd" fi - - if [ $want_syslog -eq 0 -a ! -f "$err_log" ]; then - touch "$err_log" # hypothetical: log was renamed but not - chown $user "$err_log" # flushed yet. we'd recreate it with - chmod "$fmode" "$err_log" # wrong owner next time we log, so set - fi # it up correctly while we can! - end_time=`date +%M%S` if test ! -f "$pid_file" # This is removed if normal shutdown @@ -1098,12 +1074,11 @@ do fi log_notice "mysqld restarted" - if test -n "$CRASH_SCRIPT" + if test -n "$crash_script" then - crash_script_output=`$CRASH_SCRIPT 2>&1` + crash_script_output=`$crash_script 2>&1` log_error "$crash_script_output" fi done log_notice "mysqld from pid file $pid_file ended" - diff --git a/sql/event_db_repository.cc b/sql/event_db_repository.cc index c96060cd5df..ca4d93eb240 100644 --- a/sql/event_db_repository.cc +++ b/sql/event_db_repository.cc @@ -18,6 +18,7 @@ #include "sql_priv.h" #include "unireg.h" #include "sql_base.h" // close_thread_tables +#include "sql_parse.h" #include "event_db_repository.h" #include "key.h" // key_copy #include "sql_db.h" // get_default_db_collation @@ -702,19 +703,17 @@ Event_db_repository::create_event(THD *thd, Event_parse_data *parse_data, restore_record(table, s->default_values); // Get default values for fields - if (system_charset_info->cset-> - numchars(system_charset_info, parse_data->dbname.str, - parse_data->dbname.str + parse_data->dbname.length) > - table->field[ET_FIELD_DB]->char_length()) + if (check_string_char_length(&parse_data->dbname, 0, + table->field[ET_FIELD_DB]->char_length(), + system_charset_info, 1)) { my_error(ER_TOO_LONG_IDENT, MYF(0), parse_data->dbname.str); goto end; } - if (system_charset_info->cset-> - numchars(system_charset_info, parse_data->name.str, - parse_data->name.str + parse_data->name.length) > - table->field[ET_FIELD_NAME]->char_length()) + if (check_string_char_length(&parse_data->name, 0, + table->field[ET_FIELD_NAME]->char_length(), + system_charset_info, 1)) { my_error(ER_TOO_LONG_IDENT, MYF(0), parse_data->name.str); goto end; diff --git a/sql/field.cc b/sql/field.cc index ff8b948ef62..23a8ecd965d 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -1371,12 +1371,15 @@ void Field_num::prepend_zeros(String *value) const int diff; if ((diff= (int) (field_length - value->length())) > 0) { - bmove_upp((uchar*) value->ptr()+field_length, - (uchar*) value->ptr()+value->length(), - value->length()); - bfill((uchar*) value->ptr(),diff,'0'); - value->length(field_length); - (void) value->c_ptr_quick(); // Avoid warnings in purify + const bool error= value->realloc(field_length); + if (!error) + { + bmove_upp((uchar*) value->ptr()+field_length, + (uchar*) value->ptr()+value->length(), + value->length()); + bfill((uchar*) value->ptr(),diff,'0'); + value->length(field_length); + } } } @@ -2309,9 +2312,10 @@ void Field::set_default() { if (default_value) { - table->in_use->reset_arena_for_cached_items(table->expr_arena); + Query_arena backup_arena; + table->in_use->set_n_backup_active_arena(table->expr_arena, &backup_arena); (void) default_value->expr->save_in_field(this, 0); - table->in_use->reset_arena_for_cached_items(0); + table->in_use->restore_active_arena(table->expr_arena, &backup_arena); return; } /* Copy constant value stored in s->default_values */ @@ -10509,7 +10513,7 @@ Column_definition::Column_definition(THD *thd, Field *old_field, if (length != 4) { char buff[sizeof("YEAR()") + MY_INT64_NUM_DECIMAL_DIGITS + 1]; - my_snprintf(buff, sizeof(buff), "YEAR(%lu)", length); + my_snprintf(buff, sizeof(buff), "YEAR(%llu)", length); push_warning_printf(thd, Sql_condition::WARN_LEVEL_NOTE, ER_WARN_DEPRECATED_SYNTAX, ER_THD(thd, ER_WARN_DEPRECATED_SYNTAX), diff --git a/sql/field.h b/sql/field.h index fd68ade1165..f0c3b48cd6a 100644 --- a/sql/field.h +++ b/sql/field.h @@ -948,14 +948,11 @@ public: */ void set_has_explicit_value() { - if (table->has_value_set) /* If we have default functions */ - bitmap_set_bit(table->has_value_set, field_index); + bitmap_set_bit(&table->has_value_set, field_index); } bool has_explicit_value() { - /* This function is only called when we have default functions */ - DBUG_ASSERT(table->has_value_set); - return bitmap_is_set(table->has_value_set, field_index); + return bitmap_is_set(&table->has_value_set, field_index); } virtual void set_explicit_default(Item *value); diff --git a/sql/filesort.cc b/sql/filesort.cc index 6046693cba1..2283e6e3d0a 100644 --- a/sql/filesort.cc +++ b/sql/filesort.cc @@ -1672,7 +1672,7 @@ int merge_buffers(Sort_param *param, IO_CACHE *from_file, if (!(error= (int) read_to_buffer(from_file, buffpek, rec_length))) { - queue_remove(&queue,0); + (void) queue_remove_top(&queue); reuse_freed_buff(&queue, buffpek, rec_length); } else if (error == -1) diff --git a/sql/handler.cc b/sql/handler.cc index f06c5d71a5e..28fabeea692 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -4549,7 +4549,7 @@ void handler::get_dynamic_partition_info(PARTITION_STATS *stat_info, stat_info->update_time= stats.update_time; stat_info->check_time= stats.check_time; stat_info->check_sum= 0; - if (table_flags() & (HA_HAS_OLD_CHECKSUM | HA_HAS_OLD_CHECKSUM)) + if (table_flags() & (HA_HAS_OLD_CHECKSUM | HA_HAS_NEW_CHECKSUM)) stat_info->check_sum= checksum(); return; } @@ -5773,7 +5773,7 @@ static int write_locked_table_maps(THD *thd) typedef bool Log_func(THD*, TABLE*, bool, const uchar*, const uchar*); - +static int check_wsrep_max_ws_rows(); static int binlog_log_row_internal(TABLE* table, const uchar *before_record, @@ -5802,6 +5802,13 @@ static int binlog_log_row_internal(TABLE* table, bool const has_trans= thd->lex->sql_command == SQLCOM_CREATE_TABLE || table->file->has_transactions(); error= (*log_func)(thd, table, has_trans, before_record, after_record); + + /* + Now that the record has been logged, increment wsrep_affected_rows and + also check whether its within the allowable limits (wsrep_max_ws_rows). + */ + if (error == 0) + error= check_wsrep_max_ws_rows(); } return error ? HA_ERR_RBR_LOGGING_FAILED : 0; } @@ -5967,7 +5974,7 @@ int handler::ha_write_row(uchar *buf) error= binlog_log_row(table, 0, buf, log_func); } DEBUG_SYNC_C("ha_write_row_end"); - DBUG_RETURN(error ? error : check_wsrep_max_ws_rows()); + DBUG_RETURN(error); } @@ -5998,7 +6005,7 @@ int handler::ha_update_row(const uchar *old_data, uchar *new_data) rows_changed++; error= binlog_log_row(table, old_data, new_data, log_func); } - return error ? error : check_wsrep_max_ws_rows(); + return error; } int handler::ha_delete_row(const uchar *buf) @@ -6025,7 +6032,7 @@ int handler::ha_delete_row(const uchar *buf) rows_changed++; error= binlog_log_row(table, buf, 0, log_func); } - return error ? error : check_wsrep_max_ws_rows(); + return error; } diff --git a/sql/item.cc b/sql/item.cc index 369b2df2675..4ce0b318ed0 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -1189,7 +1189,8 @@ Item *Item_cache::safe_charset_converter(THD *thd, CHARSET_INFO *tocs) if (conv == example) return this; Item_cache *cache; - if (!conv || !(cache= new (thd->mem_root) Item_cache_str(thd, conv))) + if (!conv || conv->fix_fields(thd, (Item **) NULL) || + !(cache= new (thd->mem_root) Item_cache_str(thd, conv))) return NULL; // Safe conversion is not possible, or OEM cache->setup(thd, conv); cache->fixed= false; // Make Item::fix_fields() happy @@ -2885,6 +2886,44 @@ void Item_field::fix_after_pullout(st_select_lex *new_parent, Item **ref) depended_from= NULL; if (context) { + bool need_change= false; + /* + Suppose there are nested selects: + + select_id=1 + select_id=2 + select_id=3 <----+ + select_id=4 -+ + select_id=5 --+ + + Suppose, pullout operation has moved anything that had select_id=4 or 5 + in to select_id=3. + + If this Item_field had a name resolution context pointing into select_lex + with id=4 or id=5, it needs a new name resolution context. + + However, it could also be that this object is a part of outer reference: + Item_ref(Item_field(field in select with select_id=1))). + - The Item_ref object has a context with select_id=5, and so needs a new + name resolution context. + - The Item_field object has a context with select_id=1, and doesn't need + a new name resolution context. + + So, the following loop walks from Item_field's current context upwards. + If we find that the select we've been pulled out to is up there, we + create the new name resolution context. Otherwise, we don't. + */ + for (Name_resolution_context *ct= context; ct; ct= ct->outer_context) + { + if (new_parent == ct->select_lex) + { + need_change= true; + break; + } + } + if (!need_change) + return; + Name_resolution_context *ctx= new Name_resolution_context(); if (context->select_lex == new_parent) { diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc index 2856cea0697..a3b9041aed4 100644 --- a/sql/item_cmpfunc.cc +++ b/sql/item_cmpfunc.cc @@ -610,7 +610,7 @@ int Arg_comparator::set_compare_func(Item_func_or_sum *item, Item_result type) int Arg_comparator::set_cmp_func(Item_func_or_sum *owner_arg, Item **a1, Item **a2) { - thd= current_thd; + THD *thd= current_thd; owner= owner_arg; set_null= set_null && owner_arg; a= a1; @@ -742,12 +742,10 @@ get_datetime_value(THD *thd, Item ***item_arg, Item **cache_arg, if (cache_arg && item->const_item() && !(item->type() == Item::CACHE_ITEM && item->cmp_type() == TIME_RESULT)) { - Query_arena backup; - Query_arena *save_arena= thd->switch_to_arena_for_cached_items(&backup); - Item_cache_temporal *cache= new (thd->mem_root) Item_cache_temporal(thd, f_type); - if (save_arena) - thd->set_query_arena(save_arena); + if (!thd) + thd= current_thd; + Item_cache_temporal *cache= new (thd->mem_root) Item_cache_temporal(thd, f_type); cache->store_packed(value, item); *cache_arg= cache; *item_arg= cache_arg; @@ -782,12 +780,12 @@ int Arg_comparator::compare_temporal(enum_field_types type) owner->null_value= 1; /* Get DATE/DATETIME/TIME value of the 'a' item. */ - a_value= get_datetime_value(thd, &a, &a_cache, type, &a_is_null); + a_value= get_datetime_value(0, &a, &a_cache, type, &a_is_null); if (a_is_null) return -1; /* Get DATE/DATETIME/TIME value of the 'b' item. */ - b_value= get_datetime_value(thd, &b, &b_cache, type, &b_is_null); + b_value= get_datetime_value(0, &b, &b_cache, type, &b_is_null); if (b_is_null) return -1; @@ -805,10 +803,10 @@ int Arg_comparator::compare_e_temporal(enum_field_types type) longlong a_value, b_value; /* Get DATE/DATETIME/TIME value of the 'a' item. */ - a_value= get_datetime_value(thd, &a, &a_cache, type, &a_is_null); + a_value= get_datetime_value(0, &a, &a_cache, type, &a_is_null); /* Get DATE/DATETIME/TIME value of the 'b' item. */ - b_value= get_datetime_value(thd, &b, &b_cache, type, &b_is_null); + b_value= get_datetime_value(0, &b, &b_cache, type, &b_is_null); return a_is_null || b_is_null ? a_is_null == b_is_null : a_value == b_value; } @@ -3742,7 +3740,7 @@ uchar *in_datetime::get_value(Item *item) Item **tmp_item= lval_cache ? &lval_cache : &item; enum_field_types f_type= tmp_item[0]->field_type_for_temporal_comparison(warn_item); - tmp.val= get_datetime_value(thd, &tmp_item, &lval_cache, f_type, &is_null); + tmp.val= get_datetime_value(0, &tmp_item, &lval_cache, f_type, &is_null); if (item->null_value) return 0; tmp.unsigned_flag= 1L; @@ -4007,7 +4005,7 @@ void cmp_item_datetime::store_value(Item *item) Item **tmp_item= lval_cache ? &lval_cache : &item; enum_field_types f_type= tmp_item[0]->field_type_for_temporal_comparison(warn_item); - value= get_datetime_value(thd, &tmp_item, &lval_cache, f_type, &is_null); + value= get_datetime_value(0, &tmp_item, &lval_cache, f_type, &is_null); m_null_value= item->null_value; } diff --git a/sql/item_cmpfunc.h b/sql/item_cmpfunc.h index 7774663bd57..cb51b1dda82 100644 --- a/sql/item_cmpfunc.h +++ b/sql/item_cmpfunc.h @@ -55,7 +55,6 @@ class Arg_comparator: public Sql_alloc Arg_comparator *comparators; // used only for compare_row() double precision; /* Fields used in DATE/DATETIME comparison. */ - THD *thd; Item *a_cache, *b_cache; // Cached values of a and b items // when one of arguments is NULL. int set_compare_func(Item_func_or_sum *owner, Item_result type); @@ -70,12 +69,12 @@ public: Arg_comparator(): m_compare_type(STRING_RESULT), m_compare_collation(&my_charset_bin), - set_null(TRUE), comparators(0), thd(0), + set_null(TRUE), comparators(0), a_cache(0), b_cache(0) {}; Arg_comparator(Item **a1, Item **a2): a(a1), b(a2), m_compare_type(STRING_RESULT), m_compare_collation(&my_charset_bin), - set_null(TRUE), comparators(0), thd(0), + set_null(TRUE), comparators(0), a_cache(0), b_cache(0) {}; public: @@ -1268,15 +1267,13 @@ public: class in_datetime :public in_longlong { public: - THD *thd; /* An item used to issue warnings. */ Item *warn_item; /* Cache for the left item. */ Item *lval_cache; in_datetime(THD *thd, Item *warn_item_arg, uint elements) - :in_longlong(thd, elements), thd(current_thd), warn_item(warn_item_arg), - lval_cache(0) {}; + :in_longlong(thd, elements), warn_item(warn_item_arg), lval_cache(0) {}; void set(uint pos,Item *item); uchar *get_value(Item *item); Item *create_item(THD *thd); @@ -1445,14 +1442,13 @@ class cmp_item_datetime : public cmp_item_scalar { longlong value; public: - THD *thd; /* Item used for issuing warnings. */ Item *warn_item; /* Cache for the left item. */ Item *lval_cache; cmp_item_datetime(Item *warn_item_arg) - :thd(current_thd), warn_item(warn_item_arg), lval_cache(0) {} + : warn_item(warn_item_arg), lval_cache(0) {} void store_value(Item *item); int cmp(Item *arg); int compare(cmp_item *ci); diff --git a/sql/item_func.cc b/sql/item_func.cc index f6162b54806..2e0596da2d1 100644 --- a/sql/item_func.cc +++ b/sql/item_func.cc @@ -2725,7 +2725,6 @@ void Item_func_min_max::fix_length_and_dec() decimals=0; max_length=0; maybe_null=0; - thd= current_thd; Item_result tmp_cmp_type= args[0]->cmp_type(); uint string_type_count= 0; uint temporal_type_count= 0; @@ -2867,10 +2866,8 @@ bool Item_func_min_max::get_date(MYSQL_TIME *ltime, ulonglong fuzzy_date) longlong res= args[i]->val_temporal_packed(Item_func_min_max::field_type()); /* Check if we need to stop (because of error or KILL) and stop the loop */ - if (thd->is_error() || args[i]->null_value) - { + if (args[i]->null_value) return (null_value= 1); - } if (i == 0 || (res < min_max ? cmp_sign : -cmp_sign) > 0) min_max= res; diff --git a/sql/item_func.h b/sql/item_func.h index 63b97a1e0c5..08b1421cb1d 100644 --- a/sql/item_func.h +++ b/sql/item_func.h @@ -1163,7 +1163,6 @@ class Item_func_min_max :public Item_hybrid_func { String tmp_value; int cmp_sign; - THD *thd; public: Item_func_min_max(THD *thd, List<Item> &list, int cmp_sign_arg): Item_hybrid_func(thd, list), cmp_sign(cmp_sign_arg) diff --git a/sql/item_subselect.cc b/sql/item_subselect.cc index 89c663b5f16..94bc71ca889 100644 --- a/sql/item_subselect.cc +++ b/sql/item_subselect.cc @@ -1,5 +1,5 @@ -/* Copyright (c) 2002, 2015, Oracle and/or its affiliates. - Copyright (c) 2010, 2015, MariaDB +/* Copyright (c) 2002, 2016, Oracle and/or its affiliates. + Copyright (c) 2010, 2016, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -3520,9 +3520,14 @@ bool subselect_union_engine::is_executed() const bool subselect_union_engine::no_rows() { /* Check if we got any rows when reading UNION result from temp. table: */ - return MY_TEST(!(unit->fake_select_lex ? - unit->fake_select_lex->join->send_records : - ((select_union_direct *)(unit->get_union_result())) + if (unit->fake_select_lex) + { + JOIN *join= unit->fake_select_lex->join; + if (join) + return MY_TEST(!join->send_records); + return false; + } + return MY_TEST(!(((select_union_direct *)(unit->get_union_result())) ->send_records)); } @@ -4897,9 +4902,9 @@ bool subselect_hash_sj_engine::init(List<Item> *tmp_columns, uint subquery_id) result= result_sink; /* - If the subquery has blobs, or the total key lenght is bigger than + If the subquery has blobs, or the total key length is bigger than some length, or the total number of key parts is more than the - allowed maximum (currently MAX_REF_PARTS == 16), then the created + allowed maximum (currently MAX_REF_PARTS == 32), then the created index cannot be used for lookups and we can't use hash semi join. If this is the case, delete the temporary table since it will not be used, and tell the caller we failed to initialize the diff --git a/sql/item_timefunc.cc b/sql/item_timefunc.cc index 27ce83f3f2b..0a564780ac2 100644 --- a/sql/item_timefunc.cc +++ b/sql/item_timefunc.cc @@ -426,7 +426,7 @@ static bool extract_date_time(DATE_TIME_FORMAT *format, { if (!my_isspace(&my_charset_latin1,*val)) { - make_truncated_value_warning(current_thd, + make_truncated_value_warning(current_thd, Sql_condition::WARN_LEVEL_WARN, val_begin, length, cached_timestamp_type, NullS); @@ -711,7 +711,7 @@ static bool get_interval_info(const char *str,uint length,CHARSET_INFO *cs, { longlong value; const char *start= str; - for (value=0; str != end && my_isdigit(cs, *str) ; str++) + for (value= 0; str != end && my_isdigit(cs, *str); str++) value= value*10 + *str - '0'; msec_length= 6 - (str - start); values[i]= value; diff --git a/sql/log_event.cc b/sql/log_event.cc index 893781223fb..291b550353d 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -7922,6 +7922,7 @@ Gtid_list_log_event::do_apply_event(rpl_group_info *rgi) rli->abort_slave= true; rli->stop_for_until= true; } + free_root(thd->mem_root, MYF(MY_KEEP_PREALLOC)); return ret; } diff --git a/sql/my_json_writer.cc b/sql/my_json_writer.cc index d36fdd1192a..135ce353552 100644 --- a/sql/my_json_writer.cc +++ b/sql/my_json_writer.cc @@ -125,7 +125,7 @@ void Json_writer::start_element() void Json_writer::add_ll(longlong val) { char buf[64]; - my_snprintf(buf, sizeof(buf), "%ld", val); + my_snprintf(buf, sizeof(buf), "%lld", val); add_unquoted_str(buf); } @@ -135,16 +135,16 @@ void Json_writer::add_size(longlong val) { char buf[64]; if (val < 1024) - my_snprintf(buf, sizeof(buf), "%ld", val); + my_snprintf(buf, sizeof(buf), "%lld", val); else if (val < 1024*1024*16) { /* Values less than 16MB are specified in KB for precision */ - size_t len= my_snprintf(buf, sizeof(buf), "%ld", val/1024); + size_t len= my_snprintf(buf, sizeof(buf), "%lld", val/1024); strcpy(buf + len, "Kb"); } else { - size_t len= my_snprintf(buf, sizeof(buf), "%ld", val/(1024*1024)); + size_t len= my_snprintf(buf, sizeof(buf), "%lld", val/(1024*1024)); strcpy(buf + len, "Mb"); } add_str(buf); diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 8a859a6f511..e4a1a54749b 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -130,10 +130,7 @@ extern "C" { // Because of SCO 3.2V4.2 #include <sysent.h> #endif #ifdef HAVE_PWD_H -#include <pwd.h> // For getpwent -#endif -#ifdef HAVE_GRP_H -#include <grp.h> +#include <pwd.h> // For struct passwd #endif #include <my_net.h> @@ -485,9 +482,7 @@ ulong opt_binlog_rows_event_max_size; my_bool opt_master_verify_checksum= 0; my_bool opt_slave_sql_verify_checksum= 1; const char *binlog_format_names[]= {"MIXED", "STATEMENT", "ROW", NullS}; -#ifdef HAVE_INITGROUPS volatile sig_atomic_t calling_initgroups= 0; /**< Used in SIGSEGV handler. */ -#endif uint mysqld_port, test_flags, select_errors, dropping_tables, ha_open_options; uint mysqld_extra_port; uint mysqld_port_timeout; @@ -2421,59 +2416,18 @@ static void set_ports() static struct passwd *check_user(const char *user) { -#if !defined(__WIN__) - struct passwd *tmp_user_info; - uid_t user_id= geteuid(); + myf flags= 0; + if (global_system_variables.log_warnings) + flags|= MY_WME; + if (!opt_bootstrap && !opt_help) + flags|= MY_FAE; - // Don't bother if we aren't superuser - if (user_id) - { - if (user) - { - /* Don't give a warning, if real user is same as given with --user */ - /* purecov: begin tested */ - tmp_user_info= getpwnam(user); - if ((!tmp_user_info || user_id != tmp_user_info->pw_uid) && - global_system_variables.log_warnings) - sql_print_warning( - "One can only use the --user switch if running as root\n"); - /* purecov: end */ - } - return NULL; - } - if (!user) - { - if (!opt_bootstrap && !opt_help) - { - sql_print_error("Fatal error: Please consult the Knowledge Base " - "to find out how to run mysqld as root!\n"); - unireg_abort(1); - } - return NULL; - } - /* purecov: begin tested */ - if (!strcmp(user,"root")) - return NULL; // Avoid problem with dynamic libraries + struct passwd *tmp_user_info= my_check_user(user, MYF(flags)); - if (!(tmp_user_info= getpwnam(user))) - { - // Allow a numeric uid to be used - const char *pos; - for (pos= user; my_isdigit(mysqld_charset,*pos); pos++) ; - if (*pos) // Not numeric id - goto err; - if (!(tmp_user_info= getpwuid(atoi(user)))) - goto err; - } + if (!tmp_user_info && my_errno==EINVAL && (flags & MY_FAE)) + unireg_abort(1); return tmp_user_info; - /* purecov: end */ - -err: - sql_print_error("Fatal error: Can't change to run as user '%s' ; Please check that the user exists!\n",user); - unireg_abort(1); -#endif - return NULL; } static inline void allow_coredumps() @@ -2490,10 +2444,6 @@ static inline void allow_coredumps() static void set_user(const char *user, struct passwd *user_info_arg) { - /* purecov: begin tested */ -#if !defined(__WIN__) - DBUG_ASSERT(user_info_arg != 0); -#ifdef HAVE_INITGROUPS /* We can get a SIGSEGV when calling initgroups() on some systems when NSS is configured to use LDAP and the server is statically linked. We set @@ -2501,22 +2451,11 @@ static void set_user(const char *user, struct passwd *user_info_arg) output a specific message to help the user resolve this problem. */ calling_initgroups= 1; - initgroups((char*) user, user_info_arg->pw_gid); + int res= my_set_user(user, user_info_arg, MYF(MY_WME)); calling_initgroups= 0; -#endif - if (setgid(user_info_arg->pw_gid) == -1) - { - sql_perror("setgid"); + if (res) unireg_abort(1); - } - if (setuid(user_info_arg->pw_uid) == -1) - { - sql_perror("setuid"); - unireg_abort(1); - } allow_coredumps(); -#endif - /* purecov: end */ } @@ -4588,19 +4527,24 @@ static int init_common_variables() default_charset_info= default_collation; } /* Set collactions that depends on the default collation */ - global_system_variables.collation_server= default_charset_info; - global_system_variables.collation_database= default_charset_info; - global_system_variables.collation_connection= default_charset_info; - global_system_variables.character_set_results= default_charset_info; - if (default_charset_info->mbminlen > 1) - { - global_system_variables.character_set_client= &my_charset_latin1; - sql_print_warning("Cannot use %s as character_set_client, %s will be used instead", - default_charset_info->csname, - global_system_variables.character_set_client->csname); + global_system_variables.collation_server= default_charset_info; + global_system_variables.collation_database= default_charset_info; + if (is_supported_parser_charset(default_charset_info)) + { + global_system_variables.collation_connection= default_charset_info; + global_system_variables.character_set_results= default_charset_info; + global_system_variables.character_set_client= default_charset_info; } else - global_system_variables.character_set_client= default_charset_info; + { + sql_print_warning("'%s' can not be used as client character set. " + "'%s' will be used as default client character set.", + default_charset_info->csname, + my_charset_latin1.csname); + global_system_variables.collation_connection= &my_charset_latin1; + global_system_variables.character_set_results= &my_charset_latin1; + global_system_variables.character_set_client= &my_charset_latin1; + } if (!(character_set_filesystem= get_charset_by_csname(character_set_filesystem_name, diff --git a/sql/opt_subselect.cc b/sql/opt_subselect.cc index 43b9bdd6255..dc0caa32998 100644 --- a/sql/opt_subselect.cc +++ b/sql/opt_subselect.cc @@ -3978,7 +3978,7 @@ SJ_TMP_TABLE::create_sj_weedout_tmp_table(THD *thd) &tmpname, (uint) strlen(path)+1, &group_buff, (!using_unique_constraint ? uniq_tuple_length_arg : 0), - &bitmaps, bitmap_buffer_size(1)*5, + &bitmaps, bitmap_buffer_size(1)*6, NullS)) { if (temp_pool_slot != MY_BIT_NONE) diff --git a/sql/signal_handler.cc b/sql/signal_handler.cc index ad7fb873c68..dc90fcae576 100644 --- a/sql/signal_handler.cc +++ b/sql/signal_handler.cc @@ -156,7 +156,7 @@ extern "C" sig_handler handle_fatal_signal(int sig) if (opt_stack_trace) { - my_safe_printf_stderr("Thread pointer: 0x%p\n", thd); + my_safe_printf_stderr("Thread pointer: %p\n", thd); my_safe_printf_stderr("%s", "Attempting backtrace. You can use the following " "information to find out\n" diff --git a/sql/sp_head.cc b/sql/sp_head.cc index 0b9b503aef3..251b829fb1d 100644 --- a/sql/sp_head.cc +++ b/sql/sp_head.cc @@ -523,12 +523,8 @@ check_routine_name(LEX_STRING *ident) my_error(ER_SP_WRONG_NAME, MYF(0), ident->str); return TRUE; } - if (check_string_char_length(ident, 0, NAME_CHAR_LEN, - system_charset_info, 1)) - { - my_error(ER_TOO_LONG_IDENT, MYF(0), ident->str); + if (check_ident_length(ident)) return TRUE; - } return FALSE; } @@ -3152,23 +3148,23 @@ sp_instr_stmt::execute(THD *thd, uint *nextp) thd->query_length()) <= 0) { res= m_lex_keeper.reset_lex_and_exec_core(thd, nextp, FALSE, this); + bool log_slow= !res && thd->enable_slow_log; - if (thd->get_stmt_da()->is_eof()) - { - /* Finalize server status flags after executing a statement. */ + /* Finalize server status flags after executing a statement. */ + if (log_slow || thd->get_stmt_da()->is_eof()) thd->update_server_status(); + if (thd->get_stmt_da()->is_eof()) thd->protocol->end_statement(); - } query_cache_end_of_result(thd); mysql_audit_general(thd, MYSQL_AUDIT_GENERAL_STATUS, - thd->get_stmt_da()->is_error() ? - thd->get_stmt_da()->sql_errno() : 0, - command_name[COM_QUERY].str); + thd->get_stmt_da()->is_error() ? + thd->get_stmt_da()->sql_errno() : 0, + command_name[COM_QUERY].str); - if (!res && unlikely(thd->enable_slow_log)) + if (log_slow) log_slow_statement(thd); } else diff --git a/sql/sql_acl.cc b/sql/sql_acl.cc index 4a3c9ae75e7..ada5ab1cd28 100644 --- a/sql/sql_acl.cc +++ b/sql/sql_acl.cc @@ -892,7 +892,7 @@ static my_bool do_validate(THD *, plugin_ref plugin, void *arg) } -static bool validate_password(LEX_USER *user) +static bool validate_password(LEX_USER *user, THD *thd) { if (user->pwtext.length || !user->pwhash.length) { @@ -908,7 +908,8 @@ static bool validate_password(LEX_USER *user) } else { - if (strict_password_validation && has_validation_plugins()) + if (!thd->slave_thread && + strict_password_validation && has_validation_plugins()) { my_error(ER_OPTION_PREVENTS_STATEMENT, MYF(0), "--strict-password-validation"); return true; @@ -2748,7 +2749,7 @@ bool check_change_password(THD *thd, LEX_USER *user) LEX_USER *real_user= get_current_user(thd, user); if (fix_and_copy_user(real_user, user, thd) || - validate_password(real_user)) + validate_password(real_user, thd)) return true; *user= *real_user; @@ -3463,7 +3464,7 @@ static int replace_user_table(THD *thd, TABLE *table, LEX_USER &combo, } if (!old_row_exists || combo.pwtext.length || combo.pwhash.length) - if (!handle_as_role && validate_password(&combo)) + if (!handle_as_role && validate_password(&combo, thd)) goto end; /* Update table columns with new privileges */ diff --git a/sql/sql_class.cc b/sql/sql_class.cc index f78382ded7d..f63ba4ed657 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -918,7 +918,6 @@ THD::THD(my_thread_id id, bool is_wsrep_applier) m_internal_handler= NULL; m_binlog_invoker= INVOKER_NONE; - arena_for_cached_items= 0; memset(&invoker_user, 0, sizeof(invoker_user)); memset(&invoker_host, 0, sizeof(invoker_host)); prepare_derived_at_open= FALSE; diff --git a/sql/sql_class.h b/sql/sql_class.h index 37758f25170..3d9cbc65c22 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -3998,27 +3998,7 @@ public: } } -private: - /* - This reference points to the table arena when the expression - for a virtual column is being evaluated - */ - Query_arena *arena_for_cached_items; - public: - void reset_arena_for_cached_items(Query_arena *new_arena) - { - arena_for_cached_items= new_arena; - } - Query_arena *switch_to_arena_for_cached_items(Query_arena *backup) - { - if (!arena_for_cached_items) - return 0; - set_n_backup_active_arena(arena_for_cached_items, backup); - return backup; - } - - void clear_wakeup_ready() { wakeup_ready= false; } /* Sleep waiting for others to wake us up with signal_wakeup_ready(). diff --git a/sql/sql_connect.cc b/sql/sql_connect.cc index fe6a101f999..5cb4cd87296 100644 --- a/sql/sql_connect.cc +++ b/sql/sql_connect.cc @@ -1,6 +1,6 @@ /* Copyright (c) 2007, 2013, Oracle and/or its affiliates. - Copyright (c) 2008, 2014, SkySQL Ab. + Copyright (c) 2008, 2016, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/sql/sql_delete.cc b/sql/sql_delete.cc index 2f4c2d9a6f8..108fc51c569 100644 --- a/sql/sql_delete.cc +++ b/sql/sql_delete.cc @@ -1346,4 +1346,3 @@ bool multi_delete::send_eof() } return 0; } - diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc index b9c1dcebee9..1e9de5a039a 100644 --- a/sql/sql_insert.cc +++ b/sql/sql_insert.cc @@ -309,6 +309,32 @@ static int check_insert_fields(THD *thd, TABLE_LIST *table_list, DBUG_RETURN(0); } +static bool has_no_default_value(THD *thd, Field *field, TABLE_LIST *table_list) +{ + if ((field->flags & NO_DEFAULT_VALUE_FLAG) && field->real_type() != MYSQL_TYPE_ENUM) + { + bool view= false; + if (table_list) + { + table_list= table_list->top_table(); + view= table_list->view != NULL; + } + if (view) + { + push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, ER_NO_DEFAULT_FOR_VIEW_FIELD, + ER_THD(thd, ER_NO_DEFAULT_FOR_VIEW_FIELD), + table_list->view_db.str, table_list->view_name.str); + } + else + { + push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, ER_NO_DEFAULT_FOR_FIELD, + ER_THD(thd, ER_NO_DEFAULT_FOR_FIELD), field->field_name); + } + return true; + } + return false; +} + /** Check if update fields are correct. @@ -744,13 +770,10 @@ bool mysql_insert(THD *thd,TABLE_LIST *table_list, DBUG_ASSERT(bulk_iterations > 0); if (mysql_prepare_insert(thd, table_list, table, fields, values, update_fields, update_values, duplic, &unused_conds, - FALSE, - (fields.elements || !value_count || - table_list->view != 0), - !ignore && thd->is_strict_mode())) + FALSE)) goto abort; - /* mysql_prepare_insert set table_list->table if it was not set */ + /* mysql_prepare_insert sets table_list->table if it was not set */ table= table_list->table; context= &thd->lex->select_lex.context; @@ -879,6 +902,15 @@ bool mysql_insert(THD *thd,TABLE_LIST *table_list, table->prepare_triggers_for_insert_stmt_or_event(); table->mark_columns_needed_for_insert(); + if (fields.elements || !value_count || table_list->view != 0) + { + if (check_that_all_fields_are_given_values(thd, table, table_list)) + { + error= 1; + goto values_loop_end; + } + } + if (table_list->prepare_where(thd, 0, TRUE) || table_list->prepare_check_option(thd)) error= 1; @@ -973,6 +1005,24 @@ bool mysql_insert(THD *thd,TABLE_LIST *table_list, } } + /* + with triggers a field can get a value *conditionally*, so we have to repeat + has_no_default_value() check for every row + */ + if (table->triggers && + table->triggers->has_triggers(TRG_EVENT_INSERT, TRG_ACTION_BEFORE)) + { + for (Field **f=table->field ; *f ; f++) + { + if (!(*f)->has_explicit_value() && + has_no_default_value(thd, *f, table_list)) + { + error= 1; + goto values_loop_end; + } + } + } + if ((res= table_list->view_check_option(thd, (values_list.elements == 1 ? 0 : @@ -984,6 +1034,7 @@ bool mysql_insert(THD *thd,TABLE_LIST *table_list, error= 1; break; } + #ifndef EMBEDDED_LIBRARY if (lock_type == TL_WRITE_DELAYED) { @@ -1384,10 +1435,6 @@ static void prepare_for_positional_update(TABLE *table, TABLE_LIST *tables) be taken from table_list->table) where Where clause (for insert ... select) select_insert TRUE if INSERT ... SELECT statement - check_fields TRUE if need to check that all INSERT fields are - given values. - abort_on_warning whether to report if some INSERT field is not - assigned as an error (TRUE) or as a warning (FALSE). TODO (in far future) In cases of: @@ -1407,9 +1454,8 @@ static void prepare_for_positional_update(TABLE *table, TABLE_LIST *tables) bool mysql_prepare_insert(THD *thd, TABLE_LIST *table_list, TABLE *table, List<Item> &fields, List_item *values, List<Item> &update_fields, List<Item> &update_values, - enum_duplicates duplic, - COND **where, bool select_insert, - bool check_fields, bool abort_on_warning) + enum_duplicates duplic, COND **where, + bool select_insert) { SELECT_LEX *select_lex= &thd->lex->select_lex; Name_resolution_context *context= &select_lex->context; @@ -1481,17 +1527,6 @@ bool mysql_prepare_insert(THD *thd, TABLE_LIST *table_list, check_insert_fields(thd, context->table_list, fields, *values, !insert_into_view, 0, &map)); - if (!res && check_fields) - { - bool saved_abort_on_warning= thd->abort_on_warning; - thd->abort_on_warning= abort_on_warning; - res= check_that_all_fields_are_given_values(thd, - table ? table : - context->table_list->table, - context->table_list); - thd->abort_on_warning= saved_abort_on_warning; - } - if (!res) res= setup_fields(thd, Ref_ptr_array(), update_values, MARK_COLUMNS_READ, 0, 0); @@ -1927,8 +1962,8 @@ before_trg_err: Check that all fields with arn't null_fields are used ******************************************************************************/ -int check_that_all_fields_are_given_values(THD *thd, TABLE *entry, - TABLE_LIST *table_list) + +int check_that_all_fields_are_given_values(THD *thd, TABLE *entry, TABLE_LIST *table_list) { int err= 0; MY_BITMAP *write_set= entry->write_set; @@ -1936,32 +1971,8 @@ int check_that_all_fields_are_given_values(THD *thd, TABLE *entry, for (Field **field=entry->field ; *field ; field++) { if (!bitmap_is_set(write_set, (*field)->field_index) && - ((*field)->flags & NO_DEFAULT_VALUE_FLAG) && - ((*field)->real_type() != MYSQL_TYPE_ENUM)) - { - bool view= FALSE; - if (table_list) - { - table_list= table_list->top_table(); - view= MY_TEST(table_list->view); - } - if (view) - { - push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, - ER_NO_DEFAULT_FOR_VIEW_FIELD, - ER_THD(thd, ER_NO_DEFAULT_FOR_VIEW_FIELD), - table_list->view_db.str, - table_list->view_name.str); - } - else - { - push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, - ER_NO_DEFAULT_FOR_FIELD, - ER_THD(thd, ER_NO_DEFAULT_FOR_FIELD), - (*field)->field_name); - } - err= 1; - } + has_no_default_value(thd, *field, table_list)) + err=1; } return thd->abort_on_warning ? err : 0; } @@ -2523,10 +2534,7 @@ TABLE *Delayed_insert::get_local_table(THD* client_thd) } if (share->default_fields || share->default_expressions) { - if (!(copy->has_value_set= (MY_BITMAP*) alloc_root(client_thd->mem_root, - sizeof(MY_BITMAP)))) - goto error; - my_bitmap_init(copy->has_value_set, + my_bitmap_init(©->has_value_set, (my_bitmap_map*) (bitmap + bitmaps_used*share->column_bitmap_size), share->fields, FALSE); @@ -3430,9 +3438,8 @@ bool mysql_insert_select_prepare(THD *thd) if (mysql_prepare_insert(thd, lex->query_tables, lex->query_tables->table, lex->field_list, 0, - lex->update_list, lex->value_list, - lex->duplicates, - &select_lex->where, TRUE, FALSE, FALSE)) + lex->update_list, lex->value_list, lex->duplicates, + &select_lex->where, TRUE)) DBUG_RETURN(TRUE); DBUG_ASSERT(select_lex->leaf_tables.elements != 0); @@ -3519,8 +3526,7 @@ select_insert::prepare(List<Item> &values, SELECT_LEX_UNIT *u) { bool saved_abort_on_warning= thd->abort_on_warning; thd->abort_on_warning= !info.ignore && thd->is_strict_mode(); - res= check_that_all_fields_are_given_values(thd, table_list->table, - table_list); + res= check_that_all_fields_are_given_values(thd, table_list->table, table_list); thd->abort_on_warning= saved_abort_on_warning; } diff --git a/sql/sql_insert.h b/sql/sql_insert.h index cbfc1ea9dcd..aea0dac6b0d 100644 --- a/sql/sql_insert.h +++ b/sql/sql_insert.h @@ -27,8 +27,7 @@ bool mysql_prepare_insert(THD *thd, TABLE_LIST *table_list, TABLE *table, List<Item> &fields, List_item *values, List<Item> &update_fields, List<Item> &update_values, enum_duplicates duplic, - COND **where, bool select_insert, - bool check_fields, bool abort_on_warning); + COND **where, bool select_insert); bool mysql_insert(THD *thd,TABLE_LIST *table,List<Item> &fields, List<List_item> &values, List<Item> &update_fields, List<Item> &update_values, enum_duplicates flag, diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc index 4e29c31a774..4c53ba5b93d 100644 --- a/sql/sql_lex.cc +++ b/sql/sql_lex.cc @@ -4594,6 +4594,12 @@ bool st_select_lex::is_merged_child_of(st_select_lex *ancestor) { continue; } + + if (sl->master_unit()->derived && + sl->master_unit()->derived->is_merged_derived()) + { + continue; + } all_merged= FALSE; break; } diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index e413b73f62c..b41f6ffd0f0 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -1570,7 +1570,9 @@ bool dispatch_command(enum enum_server_command command, THD *thd, { wsrep_client_rollback(thd); } - if (thd->wsrep_conflict_state== ABORTED) + /* We let COM_QUIT and COM_STMT_CLOSE to execute even if wsrep aborted. */ + if (thd->wsrep_conflict_state == ABORTED && + command != COM_STMT_CLOSE && command != COM_QUIT) { my_error(ER_LOCK_DEADLOCK, MYF(0), "wsrep aborted transaction"); WSREP_DEBUG("Deadlock error for: %s", thd->query()); @@ -2329,6 +2331,12 @@ com_multi_end: if (WSREP(thd)) { + /* + MDEV-10812 + In the case of COM_QUIT/COM_STMT_CLOSE thread status should be disabled. + */ + DBUG_ASSERT((command != COM_QUIT && command != COM_STMT_CLOSE) + || thd->get_stmt_da()->is_disabled()); /* wsrep BF abort in query exec phase */ mysql_mutex_lock(&thd->LOCK_wsrep_thd); do_end_of_statement= thd->wsrep_conflict_state != REPLAYING && @@ -5975,11 +5983,8 @@ end_with_restore_list: } case SQLCOM_SHOW_CREATE_TRIGGER: { - if (lex->spname->m_name.length > NAME_LEN) - { - my_error(ER_TOO_LONG_IDENT, MYF(0), lex->spname->m_name.str); + if (check_ident_length(&lex->spname->m_name)) goto error; - } #ifdef WITH_WSREP if (WSREP_CLIENT(thd) && wsrep_sync_wait(thd)) goto error; @@ -9679,6 +9684,18 @@ bool check_string_char_length(LEX_STRING *str, uint err_msg, return TRUE; } + +bool check_ident_length(LEX_STRING *ident) +{ + if (check_string_char_length(ident, 0, NAME_CHAR_LEN, system_charset_info, 1)) + { + my_error(ER_TOO_LONG_IDENT, MYF(0), ident->str); + return 1; + } + return 0; +} + + C_MODE_START /* diff --git a/sql/sql_parse.h b/sql/sql_parse.h index ad29bb2cdd3..d669f36acef 100644 --- a/sql/sql_parse.h +++ b/sql/sql_parse.h @@ -77,6 +77,7 @@ bool check_string_byte_length(LEX_STRING *str, uint err_msg, bool check_string_char_length(LEX_STRING *str, uint err_msg, uint max_char_length, CHARSET_INFO *cs, bool no_error); +bool check_ident_length(LEX_STRING *ident); CHARSET_INFO* merge_charset_and_collation(CHARSET_INFO *cs, CHARSET_INFO *cl); CHARSET_INFO *find_bin_collation(CHARSET_INFO *cs); bool check_host_name(LEX_STRING *str); diff --git a/sql/sql_prepare.cc b/sql/sql_prepare.cc index f5e0244f75a..303460a5fb4 100644 --- a/sql/sql_prepare.cc +++ b/sql/sql_prepare.cc @@ -1346,7 +1346,7 @@ static bool mysql_test_insert(Prepared_statement *stmt, if (mysql_prepare_insert(thd, table_list, table_list->table, fields, values, update_fields, update_values, - duplic, &unused_conds, FALSE, FALSE, FALSE)) + duplic, &unused_conds, FALSE)) goto error; value_count= values->elements; diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 38752a153b5..ce9d75e9744 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -788,10 +788,15 @@ JOIN::prepare(TABLE_LIST *tables_init, if (mixed_implicit_grouping && tbl->table) tbl->table->maybe_null= 1; } + + uint real_og_num= og_num; + if (skip_order_by && + select_lex != select_lex->master_unit()->global_parameters()) + real_og_num+= select_lex->order_list.elements; if (setup_wild(thd, tables_list, fields_list, &all_fields, wild_num)) DBUG_RETURN(-1); - if (select_lex->setup_ref_array(thd, og_num)) + if (select_lex->setup_ref_array(thd, real_og_num)) DBUG_RETURN(-1); ref_ptrs= ref_ptr_array_slice(0); @@ -1722,7 +1727,8 @@ JOIN::optimize_inner() <fields> to ORDER BY <fields>. There are three exceptions: - if skip_sort_order is set (see above), then we can simply skip GROUP BY; - - if we are in a subquery, we don't have to maintain order + - if we are in a subquery, we don't have to maintain order unless there + is a limit clause in the subquery. - we can only rewrite ORDER BY if the ORDER BY fields are 'compatible' with the GROUP BY ones, i.e. either one is a prefix of another. We only check if the ORDER BY is a prefix of GROUP BY. In this case @@ -1734,7 +1740,7 @@ JOIN::optimize_inner() if (!order || test_if_subpart(group_list, order)) { if (skip_sort_order || - select_lex->master_unit()->item) // This is a subquery + (select_lex->master_unit()->item && select_limit == HA_POS_ERROR)) // This is a subquery order= NULL; else order= group_list; @@ -13028,9 +13034,12 @@ COND *Item_cond_and::build_equal_items(THD *thd, COND_EQUAL cond_equal; cond_equal.upper_levels= inherited; + if (check_stack_overrun(thd, STACK_MIN_SIZE, NULL)) + return this; // Fatal error flag is set! + List<Item> eq_list; List<Item> *cond_args= argument_list(); - + List_iterator<Item> li(*cond_args); Item *item; @@ -13040,7 +13049,7 @@ COND *Item_cond_and::build_equal_items(THD *thd, that are subject to substitution by multiple equality items and removing each such predicate from the conjunction after having found/created a multiple equality whose inference the predicate is. - */ + */ while ((item= li++)) { /* @@ -16134,6 +16143,9 @@ setup_tmp_table_column_bitmaps(TABLE *table, uchar *bitmaps, uint field_count) bitmaps+= bitmap_size; my_bitmap_init(&table->cond_set, (my_bitmap_map*) bitmaps, field_count, FALSE); + bitmaps+= bitmap_size; + my_bitmap_init(&table->has_value_set, + (my_bitmap_map*) bitmaps, field_count, FALSE); /* write_set and all_set are copies of read_set */ table->def_write_set= table->def_read_set; table->s->all_set= table->def_read_set; @@ -16309,7 +16321,7 @@ create_tmp_table(THD *thd, TMP_TABLE_PARAM *param, List<Item> &fields, &tmpname, (uint) strlen(path)+1, &group_buff, (group && ! using_unique_constraint ? param->group_length : 0), - &bitmaps, bitmap_buffer_size(field_count)*5, + &bitmaps, bitmap_buffer_size(field_count)*6, NullS)) { if (temp_pool_slot != MY_BIT_NONE) @@ -17039,7 +17051,7 @@ bool Virtual_tmp_table::init(uint field_count) &s, sizeof(*s), &field, (field_count + 1) * sizeof(Field*), &blob_field, (field_count + 1) * sizeof(uint), - &bitmaps, bitmap_buffer_size(field_count) * 5, + &bitmaps, bitmap_buffer_size(field_count) * 6, NullS)) return true; bzero(s, sizeof(*s)); @@ -17844,7 +17856,7 @@ do_select(JOIN *join, Procedure *procedure) } join->procedure= procedure; - join->send_records=0; + join->duplicate_rows= join->send_records=0; if (join->only_const_tables() && !join->need_tmp) { Next_select_func end_select= setup_end_select_func(join, NULL); @@ -17907,7 +17919,7 @@ do_select(JOIN *join, Procedure *procedure) error= join->first_select(join,join_tab,1); } - join->thd->limit_found_rows= join->send_records; + join->thd->limit_found_rows= join->send_records - join->duplicate_rows; if (error == NESTED_LOOP_NO_MORE_ROWS || join->thd->killed == ABORT_QUERY) error= NESTED_LOOP_OK; @@ -19524,7 +19536,12 @@ end_send(JOIN *join, JOIN_TAB *join_tab __attribute__((unused)), int error; /* result < 0 if row was not accepted and should not be counted */ if ((error= join->result->send_data(*fields))) - DBUG_RETURN(error < 0 ? NESTED_LOOP_OK : NESTED_LOOP_ERROR); + { + if (error > 0) + DBUG_RETURN(NESTED_LOOP_ERROR); + // error < 0 => duplicate row + join->duplicate_rows++; + } } ++join->send_records; @@ -19670,7 +19687,7 @@ end_send_group(JOIN *join, JOIN_TAB *join_tab __attribute__((unused)), if (error < 0) { /* Duplicate row, don't count */ - join->send_records--; + join->duplicate_rows++; error= 0; } } diff --git a/sql/sql_select.h b/sql/sql_select.h index ab4cc89c9d3..f1d48b700d7 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -1149,6 +1149,10 @@ public: (if sql_calc_found_rows is used, LIMIT is ignored) */ ha_rows select_limit; + /* + Number of duplicate rows found in UNION. + */ + ha_rows duplicate_rows; /** Used to fetch no more than given amount of rows per one fetch operation of server side cursor. @@ -1420,7 +1424,7 @@ public: sort_and_group= 0; first_record= 0; do_send_rows= 1; - send_records= 0; + duplicate_rows= send_records= 0; found_records= 0; fetch_limit= HA_POS_ERROR; thd= thd_arg; diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 2df3f4ef34a..3856cc46031 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -3475,7 +3475,6 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, sql_field->pack_length= dup_field->pack_length; sql_field->key_length= dup_field->key_length; sql_field->decimals= dup_field->decimals; - sql_field->create_length_to_internal_length(); sql_field->unireg_check= dup_field->unireg_check; /* We're making one field from two, the result field will have @@ -3485,6 +3484,7 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, if (!(sql_field->flags & NOT_NULL_FLAG)) null_fields--; sql_field->flags= dup_field->flags; + sql_field->create_length_to_internal_length(); sql_field->interval= dup_field->interval; sql_field->vcol_info= dup_field->vcol_info; it2.remove(); // Remove first (create) definition @@ -3610,12 +3610,8 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, my_error(ER_TOO_MANY_KEY_PARTS,MYF(0),tmp); DBUG_RETURN(TRUE); } - if (check_string_char_length(&key->name, 0, NAME_CHAR_LEN, - system_charset_info, 1)) - { - my_error(ER_TOO_LONG_IDENT, MYF(0), key->name.str); + if (check_ident_length(&key->name)) DBUG_RETURN(TRUE); - } key_iterator2.rewind (); if (key->type != Key::FOREIGN_KEY) { @@ -4130,7 +4126,7 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, } if (thd->variables.sql_mode & MODE_NO_ZERO_DATE && - !sql_field->default_value && + !sql_field->default_value && !sql_field->vcol_info && is_timestamp_type(sql_field->sql_type) && (sql_field->flags & NOT_NULL_FLAG) && (type == Field::NONE || type == Field::TIMESTAMP_UN_FIELD)) diff --git a/sql/sql_trigger.cc b/sql/sql_trigger.cc index 00540399abd..ca9b1431785 100644 --- a/sql/sql_trigger.cc +++ b/sql/sql_trigger.cc @@ -2504,4 +2504,3 @@ bool load_table_name_for_trigger(THD *thd, DBUG_RETURN(FALSE); } - diff --git a/sql/sql_udf.cc b/sql/sql_udf.cc index db2ef93204c..394fa713445 100644 --- a/sql/sql_udf.cc +++ b/sql/sql_udf.cc @@ -501,12 +501,8 @@ int mysql_create_function(THD *thd,udf_func *udf) my_message(ER_UDF_NO_PATHS, ER_THD(thd, ER_UDF_NO_PATHS), MYF(0)); DBUG_RETURN(1); } - if (check_string_char_length(&udf->name, 0, NAME_CHAR_LEN, - system_charset_info, 1)) - { - my_error(ER_TOO_LONG_IDENT, MYF(0), udf->name.str); + if (check_ident_length(&udf->name)) DBUG_RETURN(1); - } tables.init_one_table(STRING_WITH_LEN("mysql"), STRING_WITH_LEN("func"), "func", TL_WRITE); diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index 12423e71f1f..b84944c2ae2 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -5196,6 +5196,8 @@ part_name: { partition_info *part_info= Lex->part_info; partition_element *p_elem= part_info->curr_part_elem; + if (check_ident_length(&$1)) + MYSQL_YYABORT; p_elem->partition_name= $1.str; } ; @@ -5500,7 +5502,11 @@ sub_part_definition: sub_name: ident_or_text - { Lex->part_info->curr_part_elem->partition_name= $1.str; } + { + if (check_ident_length(&$1)) + MYSQL_YYABORT; + Lex->part_info->curr_part_elem->partition_name= $1.str; + } ; opt_part_options: diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 4ec528f1c6b..40cdd84a292 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -642,6 +642,7 @@ static bool check_cs_client(sys_var *self, THD *thd, set_var *var) if (check_charset_not_null(self, thd, var)) return true; + // Currently, UCS-2 cannot be used as a client character set if (!is_supported_parser_charset((CHARSET_INFO *)(var->save_result.ptr))) return true; @@ -1256,9 +1257,9 @@ static bool update_cached_max_statement_time(sys_var *self, THD *thd, static Sys_var_double Sys_max_statement_time( "max_statement_time", - "A SELECT query that have taken more than max_statement_time seconds " + "A query that has taken more than max_statement_time seconds " "will be aborted. The argument will be treated as a decimal value " - "with microsecond precision. A value of 0 (default) means no timeout", + "with microsecond precision. A value of 0 (default) means no timeout", SESSION_VAR(max_statement_time_double), CMD_LINE(REQUIRED_ARG), VALID_RANGE(0, LONG_TIMEOUT), DEFAULT(0), NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(0), diff --git a/sql/table.cc b/sql/table.cc index c839062ea4a..ca1fd3ac262 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -3214,11 +3214,9 @@ partititon_err: /* Allocate bitmaps */ bitmap_size= share->column_bitmap_size; - bitmap_count= 6; + bitmap_count= 7; if (share->virtual_fields) bitmap_count++; - if (outparam->default_field) - bitmap_count++; if (!(bitmaps= (uchar*) alloc_root(&outparam->mem_root, bitmap_size * bitmap_count))) @@ -3231,7 +3229,7 @@ partititon_err: (my_bitmap_map*) bitmaps, share->fields, FALSE); bitmaps+= bitmap_size; - /* Don't allocate vcol_bitmap or explicit_value if we don't need it */ + /* Don't allocate vcol_bitmap if we don't need it */ if (share->virtual_fields) { if (!(outparam->def_vcol_set= (MY_BITMAP*) @@ -3241,16 +3239,10 @@ partititon_err: (my_bitmap_map*) bitmaps, share->fields, FALSE); bitmaps+= bitmap_size; } - if (outparam->default_field) - { - if (!(outparam->has_value_set= (MY_BITMAP*) - alloc_root(&outparam->mem_root, sizeof(*outparam->has_value_set)))) - goto err; - my_bitmap_init(outparam->has_value_set, - (my_bitmap_map*) bitmaps, share->fields, FALSE); - bitmaps+= bitmap_size; - } + my_bitmap_init(&outparam->has_value_set, + (my_bitmap_map*) bitmaps, share->fields, FALSE); + bitmaps+= bitmap_size; my_bitmap_init(&outparam->tmp_set, (my_bitmap_map*) bitmaps, share->fields, FALSE); bitmaps+= bitmap_size; @@ -7348,13 +7340,14 @@ int TABLE::update_virtual_fields(enum_vcol_update_mode update_mode) DBUG_ENTER("TABLE::update_virtual_fields"); DBUG_PRINT("enter", ("update_mode: %d", update_mode)); Field **vfield_ptr, *vf; + Query_arena backup_arena; Turn_errors_to_warnings_handler Suppress_errors; int error; bool handler_pushed= 0; DBUG_ASSERT(vfield); error= 0; - in_use->reset_arena_for_cached_items(expr_arena); + in_use->set_n_backup_active_arena(expr_arena, &backup_arena); /* When reading or deleting row, ignore errors from virtual columns */ if (update_mode == VCOL_UPDATE_FOR_READ || @@ -7364,6 +7357,7 @@ int TABLE::update_virtual_fields(enum_vcol_update_mode update_mode) in_use->push_internal_handler(&Suppress_errors); handler_pushed= 1; } + /* Iterate over virtual fields in the table */ for (vfield_ptr= vfield; *vfield_ptr; vfield_ptr++) { @@ -7437,7 +7431,7 @@ int TABLE::update_virtual_fields(enum_vcol_update_mode update_mode) } if (handler_pushed) in_use->pop_internal_handler(); - in_use->reset_arena_for_cached_items(0); + in_use->restore_active_arena(expr_arena, &backup_arena); /* Return 1 only of we got a fatal error, not a warning */ DBUG_RETURN(in_use->is_error()); @@ -7445,13 +7439,13 @@ int TABLE::update_virtual_fields(enum_vcol_update_mode update_mode) int TABLE::update_virtual_field(Field *vf) { + Query_arena backup_arena; DBUG_ENTER("TABLE::update_virtual_field"); - - in_use->reset_arena_for_cached_items(expr_arena); + in_use->set_n_backup_active_arena(expr_arena, &backup_arena); bitmap_clear_all(&tmp_set); vf->vcol_info->expr->walk(&Item::update_vcol_processor, 0, &tmp_set); vf->vcol_info->expr->save_in_field(vf, 0); - in_use->reset_arena_for_cached_items(0); + in_use->restore_active_arena(expr_arena, &backup_arena); DBUG_RETURN(0); } @@ -7478,12 +7472,13 @@ int TABLE::update_virtual_field(Field *vf) int TABLE::update_default_fields(bool update_command, bool ignore_errors) { - DBUG_ENTER("TABLE::update_default_fields"); + Query_arena backup_arena; Field **field_ptr; int res= 0; + DBUG_ENTER("TABLE::update_default_fields"); DBUG_ASSERT(default_field); - in_use->reset_arena_for_cached_items(expr_arena); + in_use->set_n_backup_active_arena(expr_arena, &backup_arena); /* Iterate over fields with default functions in the table */ for (field_ptr= default_field; *field_ptr ; field_ptr++) @@ -7511,7 +7506,7 @@ int TABLE::update_default_fields(bool update_command, bool ignore_errors) res= 0; } } - in_use->reset_arena_for_cached_items(0); + in_use->restore_active_arena(expr_arena, &backup_arena); DBUG_RETURN(res); } @@ -7522,8 +7517,7 @@ int TABLE::update_default_fields(bool update_command, bool ignore_errors) void TABLE::reset_default_fields() { DBUG_ENTER("reset_default_fields"); - if (has_value_set) - bitmap_clear_all(has_value_set); + bitmap_clear_all(&has_value_set); DBUG_VOID_RETURN; } diff --git a/sql/table.h b/sql/table.h index 3b8086dd09f..5b1c1d0d2ea 100644 --- a/sql/table.h +++ b/sql/table.h @@ -1104,7 +1104,7 @@ public: /* Set if using virtual fields */ MY_BITMAP *vcol_set, *def_vcol_set; /* On INSERT: fields that the user specified a value for */ - MY_BITMAP *has_value_set; + MY_BITMAP has_value_set; /* The ID of the query that opened and is using this table. Has different diff --git a/storage/connect/ApacheInterface.java b/storage/connect/ApacheInterface.java index b4c8a4e9885..47b46dc0506 100644 --- a/storage/connect/ApacheInterface.java +++ b/storage/connect/ApacheInterface.java @@ -35,7 +35,10 @@ public class ApacheInterface extends JdbcInterface { ds.setPassword(parms[3]); pool.put(url, ds); } // endif ds - + + // if (parms.length > 4 && parms[4] != null) + // ds.setConnectionProperties(parms[4]); + // Get a connection from the data source conn = ds.getConnection(); diff --git a/storage/connect/CMakeLists.txt b/storage/connect/CMakeLists.txt index 95d88538119..ce6de424421 100644 --- a/storage/connect/CMakeLists.txt +++ b/storage/connect/CMakeLists.txt @@ -20,25 +20,25 @@ SET(CONNECT_SOURCES ha_connect.cc connect.cc user_connect.cc mycat.cc fmdlex.c osutil.c plugutil.c rcmsg.c rcmsg.h array.cpp blkfil.cpp colblk.cpp csort.cpp -filamap.cpp filamdbf.cpp filamfix.cpp filamtxt.cpp filamvct.cpp filamzip.cpp +filamap.cpp filamdbf.cpp filamfix.cpp filamgz.cpp filamtxt.cpp filter.cpp json.cpp jsonudf.cpp maputil.cpp myconn.cpp myutil.cpp plgdbutl.cpp reldef.cpp tabcol.cpp tabdos.cpp tabfix.cpp tabfmt.cpp tabjson.cpp table.cpp tabmul.cpp tabmysql.cpp taboccur.cpp tabpivot.cpp tabsys.cpp tabtbl.cpp tabutil.cpp -tabvct.cpp tabvir.cpp tabxcl.cpp valblk.cpp value.cpp xindex.cpp xobject.cpp +tabvir.cpp tabxcl.cpp valblk.cpp value.cpp xindex.cpp xobject.cpp array.h blkfil.h block.h catalog.h checklvl.h colblk.h connect.h csort.h -engmsg.h filamap.h filamdbf.h filamfix.h filamtxt.h filamvct.h filamzip.h +engmsg.h filamap.h filamdbf.h filamfix.h filamgz.h filamtxt.h filter.h global.h ha_connect.h inihandl.h json.h jsonudf.h maputil.h msgid.h mycat.h myconn.h myutil.h os.h osutil.h plgcnx.h plgdbsem.h preparse.h reldef.h resource.h tabcol.h tabdos.h tabfix.h tabfmt.h tabjson.h tabmul.h tabmysql.h -taboccur.h tabpivot.h tabsys.h tabtbl.h tabutil.h tabvct.h tabvir.h tabxcl.h +taboccur.h tabpivot.h tabsys.h tabtbl.h tabutil.h tabvir.h tabxcl.h user_connect.h valblk.h value.h xindex.h xobject.h xtable.h) # # Definitions that are shared for all OSes # add_definitions( -DMARIADB -DFORCE_INIT_OF_VARS -Dconnect_EXPORTS) -add_definitions( -DHUGE_SUPPORT -DZIP_SUPPORT -DPIVOT_SUPPORT ) +add_definitions( -DHUGE_SUPPORT -DGZ_SUPPORT -DPIVOT_SUPPORT ) # @@ -90,6 +90,18 @@ ENDIF(UNIX) # +# VCT: the VEC format might be not supported in future versions +# + +OPTION(CONNECT_WITH_VCT "Compile CONNECT storage engine with VCT support" ON) + +IF(CONNECT_WITH_VCT) + SET(CONNECT_SOURCES ${CONNECT_SOURCES} filamvct.cpp tabvct.cpp filamvct.h tabvct.h) + add_definitions(-DVCT_SUPPORT) +ENDIF(CONNECT_WITH_VCT) + + +# # XML # @@ -236,9 +248,9 @@ ENDIF(CONNECT_WITH_ODBC) # JDBC # IF(APPLE) - OPTION(CONNECT_WITH_JDBC "some comment" OFF) + OPTION(CONNECT_WITH_JDBC "Compile CONNECT storage engine without JDBC support" OFF) ELSE() - OPTION(CONNECT_WITH_JDBC "some comment" ON) + OPTION(CONNECT_WITH_JDBC "Compile CONNECT storage engine with JDBC support" ON) ENDIF() IF(CONNECT_WITH_JDBC) @@ -252,18 +264,33 @@ IF(CONNECT_WITH_JDBC) SET(CONNECT_SOURCES ${CONNECT_SOURCES} jdbconn.cpp tabjdbc.cpp jdbconn.h tabjdbc.h jdbccat.h JdbcInterface.java ApacheInterface.java MariadbInterface.java - MysqlInterface.java OracleInterface.java PostgresqlInterface.java) + MysqlInterface.java OracleInterface.java PostgresqlInterface.java + JavaWrappers.jar) # TODO: Find how to compile and install the java wrapper classes # Find required libraries and include directories SET (JAVA_SOURCES JdbcInterface.java) add_jar(JdbcInterface ${JAVA_SOURCES}) - install_jar(JdbcInterface DESTINATION ${INSTALL_PLUGINDIR} COMPONENT connect-engine) + install_jar(JdbcInterface DESTINATION ${INSTALL_PLUGINDIR} COMPONENT connect-engine) + INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/JavaWrappers.jar + DESTINATION ${INSTALL_PLUGINDIR} COMPONENT connect-engine) add_definitions(-DJDBC_SUPPORT) ELSE() SET(JDBC_LIBRARY "") ENDIF() ENDIF(CONNECT_WITH_JDBC) +# +# ZIP +# + +OPTION(CONNECT_WITH_ZIP "Compile CONNECT storage engine with ZIP support" ON) + +IF(CONNECT_WITH_ZIP) + SET(CONNECT_SOURCES ${CONNECT_SOURCES} filamzip.cpp tabzip.cpp unzip.c ioapi.c zip.c + filamzip.h tabzip.h ioapi.h unzip.h zip.h) + add_definitions(-DZIP_SUPPORT -DNOCRYPT) +ENDIF(CONNECT_WITH_ZIP) + # # XMAP diff --git a/storage/connect/JavaWrappers.jar b/storage/connect/JavaWrappers.jar Binary files differnew file mode 100644 index 00000000000..8c01c364a3f --- /dev/null +++ b/storage/connect/JavaWrappers.jar diff --git a/storage/connect/JdbcInterface.java b/storage/connect/JdbcInterface.java index 34af8c4e013..a1b1360e6ea 100644 --- a/storage/connect/JdbcInterface.java +++ b/storage/connect/JdbcInterface.java @@ -82,6 +82,9 @@ public class JdbcInterface { System.out.println("URL=" + parms[1]); CheckURL(parms[1], null); + + // This is required for drivers using context class loaders + Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); if (parms[2] != null && !parms[2].isEmpty()) { if (DEBUG) @@ -220,6 +223,19 @@ public class JdbcInterface { } // end of SetTimestampParm + public int SetNullParm(int i, int typ) { + int rc = 0; + + try { + pstmt.setNull(i, typ); + } catch (Exception e) { + SetErrmsg(e); + rc = -1; + } // end try/catch + + return rc; + } // end of SetNullParm + public int ExecutePrep() { int n = -3; diff --git a/storage/connect/domdoc.cpp b/storage/connect/domdoc.cpp index 64a0a172956..eb9660b439d 100644 --- a/storage/connect/domdoc.cpp +++ b/storage/connect/domdoc.cpp @@ -89,30 +89,43 @@ DOMDOC::DOMDOC(char *nsl, char *nsdf, char *enc, PFBLOCK fp) /******************************************************************/ /* Initialize XML parser and check library compatibility. */ /******************************************************************/ -bool DOMDOC::Initialize(PGLOBAL g) - { - if (TestHr(g, CoInitialize(NULL))) +bool DOMDOC::Initialize(PGLOBAL g, char *entry, bool zipped) +{ + if (zipped && InitZip(g, entry)) + return true; + + if (TestHr(g, CoInitialize(NULL))) return true; if (TestHr(g, Docp.CreateInstance("msxml2.domdocument"))) return true; return MakeNSlist(g); - } // end of Initialize +} // end of Initialize /******************************************************************/ /* Parse the XML file and construct node tree in memory. */ /******************************************************************/ -bool DOMDOC::ParseFile(char *fn) - { - // Load the document +bool DOMDOC::ParseFile(PGLOBAL g, char *fn) +{ + bool b; + Docp->async = false; - if (!(bool)Docp->load((_bstr_t)fn)) + if (zip) { + // Parse an in memory document + char *xdoc = GetMemDoc(g, fn); + + b = (xdoc) ? (bool)Docp->loadXML((_bstr_t)xdoc) : false; + } else + // Load the document + b = (bool)Docp->load((_bstr_t)fn); + + if (!b) return true; return false; - } // end of ParseFile +} // end of ParseFile /******************************************************************/ /* Create or reuse an Xblock for this document. */ @@ -239,6 +252,7 @@ int DOMDOC::DumpDoc(PGLOBAL g, char *ofn) void DOMDOC::CloseDoc(PGLOBAL g, PFBLOCK xp) { CloseXMLFile(g, xp, false); + CloseZip(); } // end of Close /* ----------------------- class DOMNODE ------------------------ */ @@ -616,13 +630,13 @@ PXNODE DOMNODELIST::GetItem(PGLOBAL g, int n, PXNODE np) /* Reset the pointer on the deleted item. */ /******************************************************************/ bool DOMNODELIST::DropItem(PGLOBAL g, int n) - { - if (Listp == NULL || Listp->length <= n) - return true; +{ + if (Listp == NULL || Listp->length < n) + return true; //Listp->item[n] = NULL; La propriété n'a pas de méthode 'set' return false; - } // end of DeleteItem +} // end of DeleteItem /* ----------------------- class DOMATTR ------------------------ */ diff --git a/storage/connect/domdoc.h b/storage/connect/domdoc.h index 2cffec499e2..cfec98a9422 100644 --- a/storage/connect/domdoc.h +++ b/storage/connect/domdoc.h @@ -37,8 +37,8 @@ class DOMDOC : public XMLDOCUMENT { virtual void SetNofree(bool b) {} // Only libxml2 // Methods - virtual bool Initialize(PGLOBAL g); - virtual bool ParseFile(char *fn); + virtual bool Initialize(PGLOBAL g, char *entry, bool zipped); + virtual bool ParseFile(PGLOBAL g, char *fn); virtual bool NewDoc(PGLOBAL g, char *ver); virtual void AddComment(PGLOBAL g, char *com); virtual PXNODE GetRoot(PGLOBAL g); diff --git a/storage/connect/filamap.cpp b/storage/connect/filamap.cpp index c6b4585a839..94c562a9981 100644 --- a/storage/connect/filamap.cpp +++ b/storage/connect/filamap.cpp @@ -87,7 +87,7 @@ int MAPFAM::GetFileLength(PGLOBAL g) { int len; - len = (To_Fb) ? To_Fb->Length : TXTFAM::GetFileLength(g); + len = (To_Fb && To_Fb->Count) ? To_Fb->Length : TXTFAM::GetFileLength(g); if (trace) htrc("Mapped file length=%d\n", len); @@ -319,11 +319,13 @@ int MAPFAM::SkipRecord(PGLOBAL g, bool header) /***********************************************************************/ int MAPFAM::ReadBuffer(PGLOBAL g) { - int len; + int rc, len; // Are we at the end of the memory - if (Mempos >= Top) - return RC_EF; + if (Mempos >= Top) + if ((rc = GetNext(g)) != RC_OK) + return rc; + if (!Placed) { /*******************************************************************/ @@ -341,8 +343,10 @@ int MAPFAM::ReadBuffer(PGLOBAL g) /*******************************************************************/ switch (Tdbp->TestBlock(g)) { case RC_EF: - return RC_EF; - case RC_NF: + if ((rc = GetNext(g)) != RC_OK) + return rc; + + case RC_NF: // Skip this record if ((rc = SkipRecord(g, false)) != RC_OK) return rc; @@ -498,7 +502,7 @@ int MAPFAM::DeleteRecords(PGLOBAL g, int irc) void MAPFAM::CloseTableFile(PGLOBAL g, bool) { PlugCloseFile(g, To_Fb); - To_Fb = NULL; // To get correct file size in Cardinality +//To_Fb = NULL; // To get correct file size in Cardinality if (trace) htrc("MAP Close: closing %s count=%d\n", @@ -569,7 +573,7 @@ int MBKFAM::GetRowID(void) /***********************************************************************/ int MBKFAM::ReadBuffer(PGLOBAL g) { - int len; + int rc, len; /*********************************************************************/ /* Sequential block reading when Placed is not true. */ @@ -577,8 +581,10 @@ int MBKFAM::ReadBuffer(PGLOBAL g) if (Placed) { Placed = false; } else if (Mempos >= Top) { // Are we at the end of the memory - return RC_EF; - } else if (++CurNum < Nrec) { + if ((rc = GetNext(g)) != RC_OK) + return rc; + + } else if (++CurNum < Nrec) { Fpos = Mempos; } else { /*******************************************************************/ @@ -588,7 +594,8 @@ int MBKFAM::ReadBuffer(PGLOBAL g) next: if (++CurBlk >= Block) - return RC_EF; + if ((rc = GetNext(g)) != RC_OK) + return rc; /*******************************************************************/ /* Before reading a new block, check whether block optimization */ @@ -596,8 +603,11 @@ int MBKFAM::ReadBuffer(PGLOBAL g) /*******************************************************************/ switch (Tdbp->TestBlock(g)) { case RC_EF: - return RC_EF; - case RC_NF: + if ((rc = GetNext(g)) != RC_OK) + return rc; + + break; + case RC_NF: goto next; } // endswitch rc @@ -697,14 +707,18 @@ int MPXFAM::InitDelete(PGLOBAL, int fpos, int) /***********************************************************************/ int MPXFAM::ReadBuffer(PGLOBAL g) { + int rc; + /*********************************************************************/ /* Sequential block reading when Placed is not true. */ /*********************************************************************/ if (Placed) { Placed = false; } else if (Mempos >= Top) { // Are we at the end of the memory - return RC_EF; - } else if (++CurNum < Nrec) { + if ((rc = GetNext(g)) != RC_OK) + return rc; + + } else if (++CurNum < Nrec) { Fpos = Mempos; } else { /*******************************************************************/ @@ -714,7 +728,7 @@ int MPXFAM::ReadBuffer(PGLOBAL g) next: if (++CurBlk >= Block) - return RC_EF; + return GetNext(g); /*******************************************************************/ /* Before reading a new block, check whether block optimization */ @@ -722,8 +736,11 @@ int MPXFAM::ReadBuffer(PGLOBAL g) /*******************************************************************/ switch (Tdbp->TestBlock(g)) { case RC_EF: - return RC_EF; - case RC_NF: + if ((rc = GetNext(g)) != RC_OK) + return rc; + + break; + case RC_NF: goto next; } // endswitch rc diff --git a/storage/connect/filamap.h b/storage/connect/filamap.h index b9c8ad965fd..774eb8b91b3 100644 --- a/storage/connect/filamap.h +++ b/storage/connect/filamap.h @@ -41,7 +41,8 @@ class DllExport MAPFAM : public TXTFAM { virtual int SkipRecord(PGLOBAL g, bool header); virtual bool OpenTableFile(PGLOBAL g); virtual bool DeferReading(void) {return false;} - virtual int ReadBuffer(PGLOBAL g); + virtual int GetNext(PGLOBAL g) {return RC_EF;} + virtual int ReadBuffer(PGLOBAL g); virtual int WriteBuffer(PGLOBAL g); virtual int DeleteRecords(PGLOBAL g, int irc); virtual void CloseTableFile(PGLOBAL g, bool abort); diff --git a/storage/connect/filamgz.cpp b/storage/connect/filamgz.cpp new file mode 100644 index 00000000000..07242ea633c --- /dev/null +++ b/storage/connect/filamgz.cpp @@ -0,0 +1,1426 @@ +/************ File AM GZ C++ Program Source Code File (.CPP) ***********/ +/* PROGRAM NAME: FILAMGZ */ +/* ------------- */ +/* Version 1.5 */ +/* */ +/* COPYRIGHT: */ +/* ---------- */ +/* (C) Copyright to the author Olivier BERTRAND 2005-2016 */ +/* */ +/* WHAT THIS PROGRAM DOES: */ +/* ----------------------- */ +/* This program are the ZLIB compressed files classes. */ +/* */ +/***********************************************************************/ + +/***********************************************************************/ +/* Include relevant MariaDB header file. */ +/***********************************************************************/ +#include "my_global.h" +#if defined(__WIN__) +#include <io.h> +#include <fcntl.h> +#if defined(__BORLANDC__) +#define __MFC_COMPAT__ // To define min/max as macro +#endif +//#include <windows.h> +#else // !__WIN__ +#if defined(UNIX) +#include <errno.h> +#else // !UNIX +#include <io.h> +#endif +#include <fcntl.h> +#endif // !__WIN__ + +/***********************************************************************/ +/* Include application header files: */ +/* global.h is header containing all global declarations. */ +/* plgdbsem.h is header containing the DB application declarations. */ +/* tabdos.h is header containing the TABDOS class declarations. */ +/***********************************************************************/ +#include "global.h" +#include "plgdbsem.h" +//#include "catalog.h" +//#include "reldef.h" +//#include "xobject.h" +//#include "kindex.h" +#include "filamtxt.h" +#include "tabdos.h" +#if defined(UNIX) +#include "osutil.h" +#endif + +/***********************************************************************/ +/* This define prepares ZLIB function declarations. */ +/***********************************************************************/ +//#define ZLIB_DLL + +#include "filamgz.h" + +/***********************************************************************/ +/* DB static variables. */ +/***********************************************************************/ +extern int num_read, num_there, num_eq[]; // Statistics + +/* ------------------------------------------------------------------- */ + +/***********************************************************************/ +/* Implementation of the GZFAM class. */ +/***********************************************************************/ +GZFAM::GZFAM(PGZFAM txfp) : TXTFAM(txfp) + { + Zfile = txfp->Zfile; + Zpos = txfp->Zpos; + } // end of GZFAM copy constructor + +/***********************************************************************/ +/* Zerror: Error function for gz calls. */ +/* gzerror returns the error message for the last error which occurred*/ +/* on the given compressed file. errnum is set to zlib error number. */ +/* If an error occurred in the file system and not in the compression */ +/* library, errnum is set to Z_ERRNO and the application may consult */ +/* errno to get the exact error code. */ +/***********************************************************************/ +int GZFAM::Zerror(PGLOBAL g) + { + int errnum; + + strcpy(g->Message, gzerror(Zfile, &errnum)); + + if (errnum == Z_ERRNO) +#if defined(__WIN__) + sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(NULL)); +#else // !__WIN__ + sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(errno)); +#endif // !__WIN__ + + return (errnum == Z_STREAM_END) ? RC_EF : RC_FX; + } // end of Zerror + +/***********************************************************************/ +/* Reset: reset position values at the beginning of file. */ +/***********************************************************************/ +void GZFAM::Reset(void) + { + TXTFAM::Reset(); +//gzrewind(Zfile); // Useful ????? + Zpos = 0; + } // end of Reset + +/***********************************************************************/ +/* GZ GetFileLength: returns an estimate of what would be the */ +/* uncompressed file size in number of bytes. */ +/***********************************************************************/ +int GZFAM::GetFileLength(PGLOBAL g) + { + int len = TXTFAM::GetFileLength(g); + + if (len > 0) + // Estimate size reduction to a max of 6 + len *= 6; + + return len; + } // end of GetFileLength + +/***********************************************************************/ +/* GZ Access Method opening routine. */ +/***********************************************************************/ +bool GZFAM::OpenTableFile(PGLOBAL g) + { + char opmode[4], filename[_MAX_PATH]; + MODE mode = Tdbp->GetMode(); + + switch (mode) { + case MODE_READ: + strcpy(opmode, "r"); + break; + case MODE_UPDATE: + /*****************************************************************/ + /* Updating GZ files not implemented yet. */ + /*****************************************************************/ + strcpy(g->Message, MSG(UPD_ZIP_NOT_IMP)); + return true; + case MODE_DELETE: + if (!Tdbp->GetNext()) { + // Store the number of deleted lines + DelRows = Cardinality(g); + + // This will erase the entire file + strcpy(opmode, "w"); +// Block = 0; // For ZBKFAM +// Last = Nrec; // For ZBKFAM + Tdbp->ResetSize(); + } else { + sprintf(g->Message, MSG(NO_PART_DEL), "GZ"); + return true; + } // endif filter + + break; + case MODE_INSERT: + strcpy(opmode, "a+"); + break; + default: + sprintf(g->Message, MSG(BAD_OPEN_MODE), mode); + return true; + } // endswitch Mode + + /*********************************************************************/ + /* Open according to logical input/output mode required. */ + /* Use specific zlib functions. */ + /* Treat files as binary. */ + /*********************************************************************/ + strcat(opmode, "b"); + Zfile = gzopen(PlugSetPath(filename, To_File, Tdbp->GetPath()), opmode); + + if (Zfile == NULL) { + sprintf(g->Message, MSG(GZOPEN_ERROR), + opmode, (int)errno, filename); + strcat(strcat(g->Message, ": "), strerror(errno)); + return (mode == MODE_READ && errno == ENOENT) + ? PushWarning(g, Tdbp) : true; + } // endif Zfile + + /*********************************************************************/ + /* Something to be done here. >>>>>>>> NOT DONE <<<<<<<< */ + /*********************************************************************/ +//To_Fb = dbuserp->Openlist; // Keep track of File block + + /*********************************************************************/ + /* Allocate the line buffer. */ + /*********************************************************************/ + return AllocateBuffer(g); + } // end of OpenTableFile + +/***********************************************************************/ +/* Allocate the line buffer. For mode Delete a bigger buffer has to */ +/* be allocated because is it also used to move lines into the file. */ +/***********************************************************************/ +bool GZFAM::AllocateBuffer(PGLOBAL g) + { + MODE mode = Tdbp->GetMode(); + + Buflen = Lrecl + 2; // Lrecl does not include CRLF +//Buflen *= ((Mode == MODE_DELETE) ? DOS_BUFF_LEN : 1); NIY + + if (trace) + htrc("SubAllocating a buffer of %d bytes\n", Buflen); + + To_Buf = (char*)PlugSubAlloc(g, NULL, Buflen); + + if (mode == MODE_INSERT) { + /*******************************************************************/ + /* For Insert buffer must be prepared. */ + /*******************************************************************/ + memset(To_Buf, ' ', Buflen); + To_Buf[Buflen - 2] = '\n'; + To_Buf[Buflen - 1] = '\0'; + } // endif Insert + + return false; + } // end of AllocateBuffer + +/***********************************************************************/ +/* GetRowID: return the RowID of last read record. */ +/***********************************************************************/ +int GZFAM::GetRowID(void) + { + return Rows; + } // end of GetRowID + +/***********************************************************************/ +/* GetPos: return the position of last read record. */ +/***********************************************************************/ +int GZFAM::GetPos(void) + { + return (int)Zpos; + } // end of GetPos + +/***********************************************************************/ +/* GetNextPos: return the position of next record. */ +/***********************************************************************/ +int GZFAM::GetNextPos(void) + { + return gztell(Zfile); + } // end of GetNextPos + +/***********************************************************************/ +/* SetPos: Replace the table at the specified position. */ +/***********************************************************************/ +bool GZFAM::SetPos(PGLOBAL g, int pos __attribute__((unused))) + { + sprintf(g->Message, MSG(NO_SETPOS_YET), "GZ"); + return true; +#if 0 + Fpos = pos; + + if (fseek(Stream, Fpos, SEEK_SET)) { + sprintf(g->Message, MSG(FSETPOS_ERROR), Fpos); + return true; + } // endif + + Placed = true; + return false; +#endif // 0 + } // end of SetPos + +/***********************************************************************/ +/* Record file position in case of UPDATE or DELETE. */ +/***********************************************************************/ +bool GZFAM::RecordPos(PGLOBAL) + { + Zpos = gztell(Zfile); + return false; + } // end of RecordPos + +/***********************************************************************/ +/* Skip one record in file. */ +/***********************************************************************/ +int GZFAM::SkipRecord(PGLOBAL g, bool header) + { + // Skip this record + if (gzeof(Zfile)) + return RC_EF; + else if (gzgets(Zfile, To_Buf, Buflen) == Z_NULL) + return Zerror(g); + + if (header) + RecordPos(g); + + return RC_OK; + } // end of SkipRecord + +/***********************************************************************/ +/* ReadBuffer: Read one line from a compressed text file. */ +/***********************************************************************/ +int GZFAM::ReadBuffer(PGLOBAL g) + { + char *p; + int rc; + + if (!Zfile) + return RC_EF; + + if (!Placed) { + /*******************************************************************/ + /* Record file position in case of UPDATE or DELETE. */ + /*******************************************************************/ + next: + if (RecordPos(g)) + return RC_FX; + + CurBlk = Rows++; // Update RowID + + /*******************************************************************/ + /* Check whether optimization on ROWID */ + /* can be done, as well as for join as for local filtering. */ + /*******************************************************************/ + switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + // Skip this record + if ((rc = SkipRecord(g, FALSE)) != RC_OK) + return rc; + + goto next; + } // endswitch rc + + } else + Placed = false; + + if (gzeof(Zfile)) { + rc = RC_EF; + } else if (gzgets(Zfile, To_Buf, Buflen) != Z_NULL) { + p = To_Buf + strlen(To_Buf) - 1; + + if (*p == '\n') + *p = '\0'; // Eliminate ending new-line character + + if (*(--p) == '\r') + *p = '\0'; // Eliminate eventuel carriage return + + strcpy(Tdbp->GetLine(), To_Buf); + IsRead = true; + rc = RC_OK; + num_read++; + } else + rc = Zerror(g); + + if (trace > 1) + htrc(" Read: '%s' rc=%d\n", To_Buf, rc); + + return rc; + } // end of ReadBuffer + +/***********************************************************************/ +/* WriteDB: Data Base write routine for ZDOS access method. */ +/* Update is not possible without using a temporary file (NIY). */ +/***********************************************************************/ +int GZFAM::WriteBuffer(PGLOBAL g) + { + /*********************************************************************/ + /* Prepare the write buffer. */ + /*********************************************************************/ + strcat(strcpy(To_Buf, Tdbp->GetLine()), CrLf); + + /*********************************************************************/ + /* Now start the writing process. */ + /*********************************************************************/ + if (gzputs(Zfile, To_Buf) < 0) + return Zerror(g); + + return RC_OK; + } // end of WriteBuffer + +/***********************************************************************/ +/* Data Base delete line routine for ZDOS access method. (NIY) */ +/***********************************************************************/ +int GZFAM::DeleteRecords(PGLOBAL g, int) + { + strcpy(g->Message, MSG(NO_ZIP_DELETE)); + return RC_FX; + } // end of DeleteRecords + +/***********************************************************************/ +/* Data Base close routine for DOS access method. */ +/***********************************************************************/ +void GZFAM::CloseTableFile(PGLOBAL, bool) + { + int rc = gzclose(Zfile); + + if (trace) + htrc("GZ CloseDB: closing %s rc=%d\n", To_File, rc); + + Zfile = NULL; // So we can know whether table is open +//To_Fb->Count = 0; // Avoid double closing by PlugCloseAll + } // end of CloseTableFile + +/***********************************************************************/ +/* Rewind routine for GZ access method. */ +/***********************************************************************/ +void GZFAM::Rewind(void) + { + gzrewind(Zfile); + } // end of Rewind + +/* ------------------------------------------------------------------- */ + +/***********************************************************************/ +/* Constructors. */ +/***********************************************************************/ +ZBKFAM::ZBKFAM(PDOSDEF tdp) : GZFAM(tdp) + { + Blocked = true; + Block = tdp->GetBlock(); + Last = tdp->GetLast(); + Nrec = tdp->GetElemt(); + CurLine = NULL; + NxtLine = NULL; + Closing = false; + BlkPos = tdp->GetTo_Pos(); + } // end of ZBKFAM standard constructor + +ZBKFAM::ZBKFAM(PZBKFAM txfp) : GZFAM(txfp) + { + CurLine = txfp->CurLine; + NxtLine = txfp->NxtLine; + Closing = txfp->Closing; + } // end of ZBKFAM copy constructor + +/***********************************************************************/ +/* Use BlockTest to reduce the table estimated size. */ +/***********************************************************************/ +int ZBKFAM::MaxBlkSize(PGLOBAL g, int) + { + int rc = RC_OK, savcur = CurBlk; + int size; + + // Roughly estimate the table size as the sum of blocks + // that can contain good rows + for (size = 0, CurBlk = 0; CurBlk < Block; CurBlk++) + if ((rc = Tdbp->TestBlock(g)) == RC_OK) + size += (CurBlk == Block - 1) ? Last : Nrec; + else if (rc == RC_EF) + break; + + CurBlk = savcur; + return size; + } // end of MaxBlkSize + +/***********************************************************************/ +/* ZBK Cardinality: returns table cardinality in number of rows. */ +/* This function can be called with a null argument to test the */ +/* availability of Cardinality implementation (1 yes, 0 no). */ +/***********************************************************************/ +int ZBKFAM::Cardinality(PGLOBAL g) + { + return (g) ? (int)((Block - 1) * Nrec + Last) : 1; + } // end of Cardinality + +/***********************************************************************/ +/* Allocate the line buffer. For mode Delete a bigger buffer has to */ +/* be allocated because is it also used to move lines into the file. */ +/***********************************************************************/ +bool ZBKFAM::AllocateBuffer(PGLOBAL g) + { + Buflen = Nrec * (Lrecl + 2); + CurLine = To_Buf = (char*)PlugSubAlloc(g, NULL, Buflen); + + if (Tdbp->GetMode() == MODE_INSERT) { + // Set values so Block and Last can be recalculated + if (Last == Nrec) { + CurBlk = Block; + Rbuf = Nrec; // To be used by WriteDB + } else { + // The last block must be completed + CurBlk = Block - 1; + Rbuf = Nrec - Last; // To be used by WriteDB + } // endif Last + + } // endif Insert + + return false; + } // end of AllocateBuffer + +/***********************************************************************/ +/* GetRowID: return the RowID of last read record. */ +/***********************************************************************/ +int ZBKFAM::GetRowID(void) + { + return CurNum + Nrec * CurBlk + 1; + } // end of GetRowID + +/***********************************************************************/ +/* GetPos: return the position of last read record. */ +/***********************************************************************/ +int ZBKFAM::GetPos(void) + { + return CurNum + Nrec * CurBlk; // Computed file index + } // end of GetPos + +/***********************************************************************/ +/* Record file position in case of UPDATE or DELETE. */ +/* Not used yet for fixed tables. */ +/***********************************************************************/ +bool ZBKFAM::RecordPos(PGLOBAL /*g*/) + { +//strcpy(g->Message, "RecordPos not implemented for gz blocked tables"); +//return true; + return RC_OK; + } // end of RecordPos + +/***********************************************************************/ +/* Skip one record in file. */ +/***********************************************************************/ +int ZBKFAM::SkipRecord(PGLOBAL /*g*/, bool) + { +//strcpy(g->Message, "SkipRecord not implemented for gz blocked tables"); +//return RC_FX; + return RC_OK; + } // end of SkipRecord + +/***********************************************************************/ +/* ReadBuffer: Read one line from a compressed text file. */ +/***********************************************************************/ +int ZBKFAM::ReadBuffer(PGLOBAL g) + { + int n, skip, rc = RC_OK; + + /*********************************************************************/ + /* Sequential reading when Placed is not true. */ + /*********************************************************************/ + if (++CurNum < Rbuf) { + CurLine = NxtLine; + + // Get the position of the next line in the buffer + while (*NxtLine++ != '\n') ; + + // Set caller line buffer + n = NxtLine - CurLine - Ending; + memcpy(Tdbp->GetLine(), CurLine, n); + Tdbp->GetLine()[n] = '\0'; + return RC_OK; + } else if (Rbuf < Nrec && CurBlk != -1) + return RC_EF; + + /*********************************************************************/ + /* New block. */ + /*********************************************************************/ + CurNum = 0; + skip = 0; + + next: + if (++CurBlk >= Block) + return RC_EF; + + /*********************************************************************/ + /* Before using the new block, check whether block optimization */ + /* can be done, as well as for join as for local filtering. */ + /*********************************************************************/ + switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + skip++; + goto next; + } // endswitch rc + + if (skip) + // Skip blocks rejected by block optimization + for (int i = CurBlk - skip; i < CurBlk; i++) { + BlkLen = BlkPos[i + 1] - BlkPos[i]; + + if (gzseek(Zfile, (z_off_t)BlkLen, SEEK_CUR) < 0) + return Zerror(g); + + } // endfor i + + BlkLen = BlkPos[CurBlk + 1] - BlkPos[CurBlk]; + + if (!(n = gzread(Zfile, To_Buf, BlkLen))) { + rc = RC_EF; + } else if (n > 0) { + // Get the position of the current line + CurLine = To_Buf; + + // Now get the position of the next line + for (NxtLine = CurLine; *NxtLine++ != '\n';) ; + + // Set caller line buffer + n = NxtLine - CurLine - Ending; + memcpy(Tdbp->GetLine(), CurLine, n); + Tdbp->GetLine()[n] = '\0'; + Rbuf = (CurBlk == Block - 1) ? Last : Nrec; + IsRead = true; + rc = RC_OK; + num_read++; + } else + rc = Zerror(g); + + return rc; + } // end of ReadBuffer + +/***********************************************************************/ +/* WriteDB: Data Base write routine for ZDOS access method. */ +/* Update is not possible without using a temporary file (NIY). */ +/***********************************************************************/ +int ZBKFAM::WriteBuffer(PGLOBAL g) + { + /*********************************************************************/ + /* Prepare the write buffer. */ + /*********************************************************************/ + if (!Closing) + strcat(strcpy(CurLine, Tdbp->GetLine()), CrLf); + + /*********************************************************************/ + /* In Insert mode, blocs are added sequentialy to the file end. */ + /* Note: Update mode is not handled for gz files. */ + /*********************************************************************/ + if (++CurNum == Rbuf) { + /*******************************************************************/ + /* New block, start the writing process. */ + /*******************************************************************/ + BlkLen = CurLine + strlen(CurLine) - To_Buf; + + if (gzwrite(Zfile, To_Buf, BlkLen) != BlkLen || + gzflush(Zfile, Z_FULL_FLUSH)) { + Closing = true; + return Zerror(g); + } // endif gzwrite + + Rbuf = Nrec; + CurBlk++; + CurNum = 0; + CurLine = To_Buf; + } else + CurLine += strlen(CurLine); + + return RC_OK; + } // end of WriteBuffer + +/***********************************************************************/ +/* Data Base delete line routine for ZBK access method. */ +/* Implemented only for total deletion of the table, which is done */ +/* by opening the file in mode "wb". */ +/***********************************************************************/ +int ZBKFAM::DeleteRecords(PGLOBAL g, int irc) + { + if (irc == RC_EF) { + LPCSTR name = Tdbp->GetName(); + PDOSDEF defp = (PDOSDEF)Tdbp->GetDef(); + + defp->SetBlock(0); + defp->SetLast(Nrec); + + if (!defp->SetIntCatInfo("Blocks", 0) || + !defp->SetIntCatInfo("Last", 0)) { + sprintf(g->Message, MSG(UPDATE_ERROR), "Header"); + return RC_FX; + } else + return RC_OK; + + } else + return irc; + + } // end of DeleteRecords + +/***********************************************************************/ +/* Data Base close routine for ZBK access method. */ +/***********************************************************************/ +void ZBKFAM::CloseTableFile(PGLOBAL g, bool) + { + int rc = RC_OK; + + if (Tdbp->GetMode() == MODE_INSERT) { + LPCSTR name = Tdbp->GetName(); + PDOSDEF defp = (PDOSDEF)Tdbp->GetDef(); + + if (CurNum && !Closing) { + // Some more inserted lines remain to be written + Last = (Nrec - Rbuf) + CurNum; + Block = CurBlk + 1; + Rbuf = CurNum--; + Closing = true; + rc = WriteBuffer(g); + } else if (Rbuf == Nrec) { + Last = Nrec; + Block = CurBlk; + } // endif CurNum + + if (rc != RC_FX) { + defp->SetBlock(Block); + defp->SetLast(Last); + defp->SetIntCatInfo("Blocks", Block); + defp->SetIntCatInfo("Last", Last); + } // endif + + gzclose(Zfile); + } else if (Tdbp->GetMode() == MODE_DELETE) { + rc = DeleteRecords(g, RC_EF); + gzclose(Zfile); + } else + rc = gzclose(Zfile); + + if (trace) + htrc("GZ CloseDB: closing %s rc=%d\n", To_File, rc); + + Zfile = NULL; // So we can know whether table is open +//To_Fb->Count = 0; // Avoid double closing by PlugCloseAll + } // end of CloseTableFile + +/***********************************************************************/ +/* Rewind routine for ZBK access method. */ +/***********************************************************************/ +void ZBKFAM::Rewind(void) + { + gzrewind(Zfile); + CurBlk = -1; + CurNum = Rbuf; + } // end of Rewind + +/* ------------------------------------------------------------------- */ + +/***********************************************************************/ +/* Constructors. */ +/***********************************************************************/ +ZIXFAM::ZIXFAM(PDOSDEF tdp) : ZBKFAM(tdp) + { +//Block = tdp->GetBlock(); +//Last = tdp->GetLast(); + Nrec = (tdp->GetElemt()) ? tdp->GetElemt() : DOS_BUFF_LEN; + Blksize = Nrec * Lrecl; + } // end of ZIXFAM standard constructor + +/***********************************************************************/ +/* ZIX Cardinality: returns table cardinality in number of rows. */ +/* This function can be called with a null argument to test the */ +/* availability of Cardinality implementation (1 yes, 0 no). */ +/***********************************************************************/ +int ZIXFAM::Cardinality(PGLOBAL g) + { + if (Last) + return (g) ? (int)((Block - 1) * Nrec + Last) : 1; + else // Last and Block not defined, cannot do it yet + return 0; + + } // end of Cardinality + +/***********************************************************************/ +/* Allocate the line buffer. For mode Delete a bigger buffer has to */ +/* be allocated because is it also used to move lines into the file. */ +/***********************************************************************/ +bool ZIXFAM::AllocateBuffer(PGLOBAL g) + { + Buflen = Blksize; + To_Buf = (char*)PlugSubAlloc(g, NULL, Buflen); + + if (Tdbp->GetMode() == MODE_INSERT) { + /*******************************************************************/ + /* For Insert the buffer must be prepared. */ + /*******************************************************************/ + memset(To_Buf, ' ', Buflen); + + if (Tdbp->GetFtype() < 2) + // if not binary, the file is physically a text file + for (int len = Lrecl; len <= Buflen; len += Lrecl) { +#if defined(__WIN__) + To_Buf[len - 2] = '\r'; +#endif // __WIN__ + To_Buf[len - 1] = '\n'; + } // endfor len + + // Set values so Block and Last can be recalculated + if (Last == Nrec) { + CurBlk = Block; + Rbuf = Nrec; // To be used by WriteDB + } else { + // The last block must be completed + CurBlk = Block - 1; + Rbuf = Nrec - Last; // To be used by WriteDB + } // endif Last + + } // endif Insert + + return false; + } // end of AllocateBuffer + +/***********************************************************************/ +/* ReadBuffer: Read one line from a compressed text file. */ +/***********************************************************************/ +int ZIXFAM::ReadBuffer(PGLOBAL g) + { + int n, rc = RC_OK; + + /*********************************************************************/ + /* Sequential reading when Placed is not true. */ + /*********************************************************************/ + if (++CurNum < Rbuf) { + Tdbp->IncLine(Lrecl); // Used by DOSCOL functions + return RC_OK; + } else if (Rbuf < Nrec && CurBlk != -1) + return RC_EF; + + /*********************************************************************/ + /* New block. */ + /*********************************************************************/ + CurNum = 0; + Tdbp->SetLine(To_Buf); + + int skip = 0; + + next: + if (++CurBlk >= Block) + return RC_EF; + + /*********************************************************************/ + /* Before using the new block, check whether block optimization */ + /* can be done, as well as for join as for local filtering. */ + /*********************************************************************/ + switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + skip++; + goto next; + } // endswitch rc + + if (skip) + // Skip blocks rejected by block optimization + for (int i = 0; i < skip; i++) { + if (gzseek(Zfile, (z_off_t)Buflen, SEEK_CUR) < 0) + return Zerror(g); + + } // endfor i + + if (!(n = gzread(Zfile, To_Buf, Buflen))) { + rc = RC_EF; + } else if (n > 0) { + Rbuf = n / Lrecl; + IsRead = true; + rc = RC_OK; + num_read++; + } else + rc = Zerror(g); + + return rc; + } // end of ReadBuffer + +/***********************************************************************/ +/* WriteDB: Data Base write routine for ZDOS access method. */ +/* Update is not possible without using a temporary file (NIY). */ +/***********************************************************************/ +int ZIXFAM::WriteBuffer(PGLOBAL g) + { + /*********************************************************************/ + /* In Insert mode, blocs are added sequentialy to the file end. */ + /* Note: Update mode is not handled for gz files. */ + /*********************************************************************/ + if (++CurNum == Rbuf) { + /*******************************************************************/ + /* New block, start the writing process. */ + /*******************************************************************/ + BlkLen = Rbuf * Lrecl; + + if (gzwrite(Zfile, To_Buf, BlkLen) != BlkLen || + gzflush(Zfile, Z_FULL_FLUSH)) { + Closing = true; + return Zerror(g); + } // endif gzwrite + + Rbuf = Nrec; + CurBlk++; + CurNum = 0; + Tdbp->SetLine(To_Buf); + } else + Tdbp->IncLine(Lrecl); // Used by FIXCOL functions + + return RC_OK; + } // end of WriteBuffer + +/* --------------------------- Class ZLBFAM -------------------------- */ + +/***********************************************************************/ +/* Constructors. */ +/***********************************************************************/ +ZLBFAM::ZLBFAM(PDOSDEF tdp) : BLKFAM(tdp) + { + Zstream = NULL; + Zbuffer = NULL; + Zlenp = NULL; + Optimized = tdp->IsOptimized(); + } // end of ZLBFAM standard constructor + +ZLBFAM::ZLBFAM(PZLBFAM txfp) : BLKFAM(txfp) + { + Zstream = txfp->Zstream; + Zbuffer = txfp->Zbuffer; + Zlenp = txfp->Zlenp; + Optimized = txfp->Optimized; + } // end of ZLBFAM (dummy?) copy constructor + +/***********************************************************************/ +/* ZLB GetFileLength: returns an estimate of what would be the */ +/* uncompressed file size in number of bytes. */ +/***********************************************************************/ +int ZLBFAM::GetFileLength(PGLOBAL g) + { + int len = (Optimized) ? BlkPos[Block] : BLKFAM::GetFileLength(g); + + if (len > 0) + // Estimate size reduction to a max of 5 + len *= 5; + + return len; + } // end of GetFileLength + +/***********************************************************************/ +/* Allocate the line buffer. For mode Delete a bigger buffer has to */ +/* be allocated because is it also used to move lines into the file. */ +/***********************************************************************/ +bool ZLBFAM::AllocateBuffer(PGLOBAL g) + { + char *msg; + int n, zrc; + +#if 0 + if (!Optimized && Tdbp->NeedIndexing(g)) { + strcpy(g->Message, MSG(NOP_ZLIB_INDEX)); + return TRUE; + } // endif indexing +#endif // 0 + +#if defined(NOLIB) + if (!zlib && LoadZlib()) { + sprintf(g->Message, MSG(DLL_LOAD_ERROR), GetLastError(), "zlib.dll"); + return TRUE; + } // endif zlib +#endif + + BLKFAM::AllocateBuffer(g); +//Buflen = Nrec * (Lrecl + 2); +//Rbuf = Nrec; + + // Allocate the compressed buffer + n = Buflen + 16; // ????????????????????????????????? + Zlenp = (int*)PlugSubAlloc(g, NULL, n); + Zbuffer = (Byte*)(Zlenp + 1); + + // Allocate and initialize the Z stream + Zstream = (z_streamp)PlugSubAlloc(g, NULL, sizeof(z_stream)); + Zstream->zalloc = (alloc_func)0; + Zstream->zfree = (free_func)0; + Zstream->opaque = (voidpf)0; + Zstream->next_in = NULL; + Zstream->avail_in = 0; + + if (Tdbp->GetMode() == MODE_READ) { + msg = "inflateInit"; + zrc = inflateInit(Zstream); + } else { + msg = "deflateInit"; + zrc = deflateInit(Zstream, Z_DEFAULT_COMPRESSION); + } // endif Mode + + if (zrc != Z_OK) { + if (Zstream->msg) + sprintf(g->Message, "%s error: %s", msg, Zstream->msg); + else + sprintf(g->Message, "%s error: %d", msg, zrc); + + return TRUE; + } // endif zrc + + if (Tdbp->GetMode() == MODE_INSERT) { + // Write the file header block + if (Last == Nrec) { + CurBlk = Block; + CurNum = 0; + + if (!GetFileLength(g)) { + // Write the zlib header as an extra block + strcpy(To_Buf, "PlugDB"); + BlkLen = strlen("PlugDB") + 1; + + if (WriteCompressedBuffer(g)) + return TRUE; + + } // endif void file + + } else { + // In mode insert, if Last != Nrec, last block must be updated + CurBlk = Block - 1; + CurNum = Last; + + strcpy(g->Message, MSG(NO_PAR_BLK_INS)); + return TRUE; + } // endif Last + + } else { // MODE_READ + // First thing to do is to read the header block + void *rdbuf; + + if (Optimized) { + BlkLen = BlkPos[0]; + rdbuf = Zlenp; + } else { + // Get the stored length from the file itself + if (fread(Zlenp, sizeof(int), 1, Stream) != 1) + return FALSE; // Empty file + + BlkLen = *Zlenp; + rdbuf = Zbuffer; + } // endif Optimized + + switch (ReadCompressedBuffer(g, rdbuf)) { + case RC_EF: + return FALSE; + case RC_FX: +#if defined(UNIX) + sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(errno)); +#else + sprintf(g->Message, MSG(READ_ERROR), To_File, _strerror(NULL)); +#endif + case RC_NF: + return TRUE; + } // endswitch + + // Some old tables can have PlugDB in their header + if (strcmp(To_Buf, "PlugDB")) { + sprintf(g->Message, MSG(BAD_HEADER), Tdbp->GetFile(g)); + return TRUE; + } // endif strcmp + + } // endif Mode + + return FALSE; + } // end of AllocateBuffer + +/***********************************************************************/ +/* GetPos: return the position of last read record. */ +/***********************************************************************/ +int ZLBFAM::GetPos(void) + { + return (Optimized) ? (CurNum + Nrec * CurBlk) : Fpos; + } // end of GetPos + +/***********************************************************************/ +/* GetNextPos: should not be called for this class. */ +/***********************************************************************/ +int ZLBFAM::GetNextPos(void) + { + if (Optimized) { + assert(FALSE); + return 0; + } else + return ftell(Stream); + + } // end of GetNextPos + +/***********************************************************************/ +/* SetPos: Replace the table at the specified position. */ +/***********************************************************************/ +bool ZLBFAM::SetPos(PGLOBAL g, int pos __attribute__((unused))) + { + sprintf(g->Message, MSG(NO_SETPOS_YET), "GZ"); + return true; +#if 0 // All this must be checked + if (pos < 0) { + strcpy(g->Message, MSG(INV_REC_POS)); + return true; + } // endif recpos + + CurBlk = pos / Nrec; + CurNum = pos % Nrec; +#if defined(_DEBUG) + num_eq[(CurBlk == OldBlk) ? 1 : 0]++; +#endif + + // Indicate the table position was externally set + Placed = true; + return false; +#endif // 0 + } // end of SetPos + +/***********************************************************************/ +/* ReadBuffer: Read one line for a text file. */ +/***********************************************************************/ +int ZLBFAM::ReadBuffer(PGLOBAL g) + { + int n; + void *rdbuf; + + /*********************************************************************/ + /* Sequential reading when Placed is not true. */ + /*********************************************************************/ + if (Placed) { + Placed = FALSE; + } else if (++CurNum < Rbuf) { + CurLine = NxtLine; + + // Get the position of the next line in the buffer + if (Tdbp->GetFtype() == RECFM_VAR) + while (*NxtLine++ != '\n') ; + else + NxtLine += Lrecl; + + // Set caller line buffer + n = NxtLine - CurLine - ((Tdbp->GetFtype() == RECFM_BIN) ? 0 : Ending); + memcpy(Tdbp->GetLine(), CurLine, n); + Tdbp->GetLine()[n] = '\0'; + return RC_OK; + } else if (Rbuf < Nrec && CurBlk != -1) { + CurNum--; // To have a correct Last value when optimizing + return RC_EF; + } else { + /*******************************************************************/ + /* New block. */ + /*******************************************************************/ + CurNum = 0; + + next: + if (++CurBlk >= Block) + return RC_EF; + + /*******************************************************************/ + /* Before reading a new block, check whether block optimization */ + /* can be done, as well as for join as for local filtering. */ + /*******************************************************************/ + if (Optimized) switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + goto next; + } // endswitch rc + + } // endif's + + if (OldBlk == CurBlk) + goto ok; // Block is already there + + if (Optimized) { + // Store the position of next block + Fpos = BlkPos[CurBlk]; + + // fseek is required only in non sequential reading + if (CurBlk != OldBlk + 1) + if (fseek(Stream, Fpos, SEEK_SET)) { + sprintf(g->Message, MSG(FSETPOS_ERROR), Fpos); + return RC_FX; + } // endif fseek + + // Calculate the length of block to read + BlkLen = BlkPos[CurBlk + 1] - Fpos; + rdbuf = Zlenp; + } else { // !Optimized + if (CurBlk != OldBlk + 1) { + strcpy(g->Message, MSG(INV_RAND_ACC)); + return RC_FX; + } else + Fpos = ftell(Stream); // Used when optimizing + + // Get the stored length from the file itself + if (fread(Zlenp, sizeof(int), 1, Stream) != 1) { + if (feof(Stream)) + return RC_EF; + + goto err; + } // endif fread + + BlkLen = *Zlenp; + rdbuf = Zbuffer; + } // endif Optimized + + // Read the next block + switch (ReadCompressedBuffer(g, rdbuf)) { + case RC_FX: goto err; + case RC_NF: return RC_FX; + case RC_EF: return RC_EF; + default: Rbuf = (CurBlk == Block - 1) ? Last : Nrec; + } // endswitch ReadCompressedBuffer + + ok: + if (Tdbp->GetFtype() == RECFM_VAR) { + int i; + + // Get the position of the current line + for (i = 0, CurLine = To_Buf; i < CurNum; i++) + while (*CurLine++ != '\n') ; // What about Unix ??? + + // Now get the position of the next line + for (NxtLine = CurLine; *NxtLine++ != '\n';) ; + + // Set caller line buffer + n = NxtLine - CurLine - Ending; + } else { + CurLine = To_Buf + CurNum * Lrecl; + NxtLine = CurLine + Lrecl; + n = Lrecl - ((Tdbp->GetFtype() == RECFM_BIN) ? 0 : Ending); + } // endif Ftype + + memcpy(Tdbp->GetLine(), CurLine, n); + Tdbp->GetLine()[n] = '\0'; + + OldBlk = CurBlk; // Last block actually read + IsRead = TRUE; // Is read indeed + return RC_OK; + + err: +#if defined(UNIX) + sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(errno)); +#else + sprintf(g->Message, MSG(READ_ERROR), To_File, _strerror(NULL)); +#endif + return RC_FX; + } // end of ReadBuffer + +/***********************************************************************/ +/* Read and decompress a block from the stream. */ +/***********************************************************************/ +int ZLBFAM::ReadCompressedBuffer(PGLOBAL g, void *rdbuf) + { + if (fread(rdbuf, 1, (size_t)BlkLen, Stream) == (unsigned)BlkLen) { + int zrc; + + num_read++; + + if (Optimized && BlkLen != signed(*Zlenp + sizeof(int))) { + sprintf(g->Message, MSG(BAD_BLK_SIZE), CurBlk + 1); + return RC_NF; + } // endif BlkLen + + // HERE WE MUST INFLATE THE BLOCK + Zstream->next_in = Zbuffer; + Zstream->avail_in = (uInt)(*Zlenp); + Zstream->next_out = (Byte*)To_Buf; + Zstream->avail_out = Buflen; + zrc = inflate(Zstream, Z_SYNC_FLUSH); + + if (zrc != Z_OK) { + if (Zstream->msg) + sprintf(g->Message, MSG(FUNC_ERR_S), "inflate", Zstream->msg); + else + sprintf(g->Message, MSG(FUNCTION_ERROR), "inflate", (int)zrc); + + return RC_NF; + } // endif zrc + + } else if (feof(Stream)) { + return RC_EF; + } else + return RC_FX; + + return RC_OK; + } // end of ReadCompressedBuffer + +/***********************************************************************/ +/* WriteBuffer: File write routine for DOS access method. */ +/* Update is directly written back into the file, */ +/* with this (fast) method, record size cannot change. */ +/***********************************************************************/ +int ZLBFAM::WriteBuffer(PGLOBAL g) + { + assert (Tdbp->GetMode() == MODE_INSERT); + + /*********************************************************************/ + /* Prepare the write buffer. */ + /*********************************************************************/ + if (!Closing) { + if (Tdbp->GetFtype() == RECFM_BIN) + memcpy(CurLine, Tdbp->GetLine(), Lrecl); + else + strcat(strcpy(CurLine, Tdbp->GetLine()), CrLf); + +#if defined(_DEBUG) + if (Tdbp->GetFtype() == RECFM_FIX && + (signed)strlen(CurLine) != Lrecl + (signed)strlen(CrLf)) { + strcpy(g->Message, MSG(BAD_LINE_LEN)); + Closing = TRUE; + return RC_FX; + } // endif Lrecl +#endif // _DEBUG + } // endif Closing + + /*********************************************************************/ + /* In Insert mode, blocs are added sequentialy to the file end. */ + /*********************************************************************/ + if (++CurNum != Rbuf) { + if (Tdbp->GetFtype() == RECFM_VAR) + CurLine += strlen(CurLine); + else + CurLine += Lrecl; + + return RC_OK; // We write only full blocks + } // endif CurNum + + // HERE WE MUST DEFLATE THE BLOCK + if (Tdbp->GetFtype() == RECFM_VAR) + NxtLine = CurLine + strlen(CurLine); + else + NxtLine = CurLine + Lrecl; + + BlkLen = NxtLine - To_Buf; + + if (WriteCompressedBuffer(g)) { + Closing = TRUE; // To tell CloseDB about a Write error + return RC_FX; + } // endif WriteCompressedBuffer + + CurBlk++; + CurNum = 0; + CurLine = To_Buf; + return RC_OK; + } // end of WriteBuffer + +/***********************************************************************/ +/* Compress the buffer and write the deflated output to stream. */ +/***********************************************************************/ +bool ZLBFAM::WriteCompressedBuffer(PGLOBAL g) + { + int zrc; + + Zstream->next_in = (Byte*)To_Buf; + Zstream->avail_in = (uInt)BlkLen; + Zstream->next_out = Zbuffer; + Zstream->avail_out = Buflen + 16; + Zstream->total_out = 0; + zrc = deflate(Zstream, Z_FULL_FLUSH); + + if (zrc != Z_OK) { + if (Zstream->msg) + sprintf(g->Message, MSG(FUNC_ERR_S), "deflate", Zstream->msg); + else + sprintf(g->Message, MSG(FUNCTION_ERROR), "deflate", (int)zrc); + + return TRUE; + } else + *Zlenp = Zstream->total_out; + + // Now start the writing process. + BlkLen = *Zlenp + sizeof(int); + + if (fwrite(Zlenp, 1, BlkLen, Stream) != (size_t)BlkLen) { + sprintf(g->Message, MSG(FWRITE_ERROR), strerror(errno)); + return TRUE; + } // endif size + + return FALSE; + } // end of WriteCompressedBuffer + +/***********************************************************************/ +/* Table file close routine for DOS access method. */ +/***********************************************************************/ +void ZLBFAM::CloseTableFile(PGLOBAL g, bool) + { + int rc = RC_OK; + + if (Tdbp->GetMode() == MODE_INSERT) { + LPCSTR name = Tdbp->GetName(); + PDOSDEF defp = (PDOSDEF)Tdbp->GetDef(); + + // Closing is True if last Write was in error + if (CurNum && !Closing) { + // Some more inserted lines remain to be written + Last = (Nrec - Rbuf) + CurNum; + Block = CurBlk + 1; + Rbuf = CurNum--; + Closing = TRUE; + rc = WriteBuffer(g); + } else if (Rbuf == Nrec) { + Last = Nrec; + Block = CurBlk; + } // endif CurNum + + if (rc != RC_FX) { + defp->SetBlock(Block); + defp->SetLast(Last); + defp->SetIntCatInfo("Blocks", Block); + defp->SetIntCatInfo("Last", Last); + } // endif + + fclose(Stream); + } else + rc = fclose(Stream); + + if (trace) + htrc("ZLB CloseTableFile: closing %s mode=%d rc=%d\n", + To_File, Tdbp->GetMode(), rc); + + Stream = NULL; // So we can know whether table is open + To_Fb->Count = 0; // Avoid double closing by PlugCloseAll + + if (Tdbp->GetMode() == MODE_READ) + rc = inflateEnd(Zstream); + else + rc = deflateEnd(Zstream); + + } // end of CloseTableFile + +/***********************************************************************/ +/* Rewind routine for ZLIB access method. */ +/***********************************************************************/ +void ZLBFAM::Rewind(void) + { + // We must be positioned after the header block + if (CurBlk >= 0) { // Nothing to do if no block read yet + if (!Optimized) { // If optimized, fseek will be done in ReadBuffer + size_t st; + + rewind(Stream); + + if (!(st = fread(Zlenp, sizeof(int), 1, Stream)) && trace) + htrc("fread error %d in Rewind", errno); + + fseek(Stream, *Zlenp + sizeof(int), SEEK_SET); + OldBlk = -1; + } // endif Optimized + + CurBlk = -1; + CurNum = Rbuf; + } // endif CurBlk + +//OldBlk = -1; +//Rbuf = 0; commented out in case we reuse last read block + } // end of Rewind + +/* ------------------------ End of GzFam ---------------------------- */ diff --git a/storage/connect/filamgz.h b/storage/connect/filamgz.h new file mode 100644 index 00000000000..d667fdddcc2 --- /dev/null +++ b/storage/connect/filamgz.h @@ -0,0 +1,170 @@ +/*************** FilAmGz H Declares Source Code File (.H) **************/ +/* Name: FILAMGZ.H Version 1.3 */ +/* */ +/* (C) Copyright to the author Olivier BERTRAND 2005-2016 */ +/* */ +/* This file contains the GZIP access method classes declares. */ +/***********************************************************************/ +#ifndef __FILAMGZ_H +#define __FILAMGZ_H + +#include "zlib.h" + +typedef class GZFAM *PGZFAM; +typedef class ZBKFAM *PZBKFAM; +typedef class ZIXFAM *PZIXFAM; +typedef class ZLBFAM *PZLBFAM; + +/***********************************************************************/ +/* This is the access method class declaration for not optimized */ +/* variable record length files compressed using the gzip library */ +/* functions. File is accessed record by record (row). */ +/***********************************************************************/ +class DllExport GZFAM : public TXTFAM { +// friend class DOSCOL; + public: + // Constructor + GZFAM(PDOSDEF tdp) : TXTFAM(tdp) {Zfile = NULL; Zpos = 0;} + GZFAM(PGZFAM txfp); + + // Implementation + virtual AMT GetAmType(void) {return TYPE_AM_GZ;} + virtual int GetPos(void); + virtual int GetNextPos(void); + virtual PTXF Duplicate(PGLOBAL g) + {return (PTXF)new(g) GZFAM(this);} + + // Methods + virtual void Reset(void); + virtual int GetFileLength(PGLOBAL g); + virtual int Cardinality(PGLOBAL g) {return (g) ? -1 : 0;} + virtual int MaxBlkSize(PGLOBAL g, int s) {return s;} + virtual bool AllocateBuffer(PGLOBAL g); + virtual int GetRowID(void); + virtual bool RecordPos(PGLOBAL g); + virtual bool SetPos(PGLOBAL g, int recpos); + virtual int SkipRecord(PGLOBAL g, bool header); + virtual bool OpenTableFile(PGLOBAL g); + virtual int ReadBuffer(PGLOBAL g); + virtual int WriteBuffer(PGLOBAL g); + virtual int DeleteRecords(PGLOBAL g, int irc); + virtual void CloseTableFile(PGLOBAL g, bool abort); + virtual void Rewind(void); + + protected: + int Zerror(PGLOBAL g); // GZ error function + + // Members + gzFile Zfile; // Points to GZ file structure + z_off_t Zpos; // Uncompressed file position + }; // end of class GZFAM + +/***********************************************************************/ +/* This is the access method class declaration for optimized variable */ +/* record length files compressed using the gzip library functions. */ +/* The File is accessed by block (requires an opt file). */ +/***********************************************************************/ +class DllExport ZBKFAM : public GZFAM { + public: + // Constructor + ZBKFAM(PDOSDEF tdp); + ZBKFAM(PZBKFAM txfp); + + // Implementation + virtual int GetPos(void); + virtual int GetNextPos(void) {return 0;} + virtual PTXF Duplicate(PGLOBAL g) + {return (PTXF)new(g) ZBKFAM(this);} + + // Methods + virtual int Cardinality(PGLOBAL g); + virtual int MaxBlkSize(PGLOBAL g, int s); + virtual bool AllocateBuffer(PGLOBAL g); + virtual int GetRowID(void); + virtual bool RecordPos(PGLOBAL g); + virtual int SkipRecord(PGLOBAL g, bool header); + virtual int ReadBuffer(PGLOBAL g); + virtual int WriteBuffer(PGLOBAL g); + virtual int DeleteRecords(PGLOBAL g, int irc); + virtual void CloseTableFile(PGLOBAL g, bool abort); + virtual void Rewind(void); + + protected: + // Members + char *CurLine; // Position of current line in buffer + char *NxtLine; // Position of Next line in buffer + bool Closing; // True when closing on Insert + }; // end of class ZBKFAM + +/***********************************************************************/ +/* This is the access method class declaration for fixed record */ +/* length files compressed using the gzip library functions. */ +/* The file is always accessed by block. */ +/***********************************************************************/ +class DllExport ZIXFAM : public ZBKFAM { + public: + // Constructor + ZIXFAM(PDOSDEF tdp); + ZIXFAM(PZIXFAM txfp) : ZBKFAM(txfp) {} + + // Implementation + virtual int GetNextPos(void) {return 0;} + virtual PTXF Duplicate(PGLOBAL g) + {return (PTXF)new(g) ZIXFAM(this);} + + // Methods + virtual int Cardinality(PGLOBAL g); + virtual bool AllocateBuffer(PGLOBAL g); + virtual int ReadBuffer(PGLOBAL g); + virtual int WriteBuffer(PGLOBAL g); + + protected: + // No additional Members + }; // end of class ZIXFAM + +/***********************************************************************/ +/* This is the DOS/UNIX Access Method class declaration for PlugDB */ +/* fixed/variable files compressed using the zlib library functions. */ +/* Physically these are written and read using the same technique */ +/* than blocked variable files, only the contain of each block is */ +/* compressed using the deflate zlib function. The purpose of this */ +/* specific format is to have a fast mechanism for direct access of */ +/* records so blocked optimization is fast and direct access (joins) */ +/* is allowed. Note that the block length is written ahead of each */ +/* block to enable reading when optimization file is not available. */ +/***********************************************************************/ +class DllExport ZLBFAM : public BLKFAM { + public: + // Constructor + ZLBFAM(PDOSDEF tdp); + ZLBFAM(PZLBFAM txfp); + + // Implementation + virtual AMT GetAmType(void) {return TYPE_AM_ZLIB;} + virtual int GetPos(void); + virtual int GetNextPos(void); + virtual PTXF Duplicate(PGLOBAL g) + {return (PTXF)new(g) ZLBFAM(this);} + inline void SetOptimized(bool b) {Optimized = b;} + + // Methods + virtual int GetFileLength(PGLOBAL g); + virtual bool SetPos(PGLOBAL g, int recpos); + virtual bool AllocateBuffer(PGLOBAL g); + virtual int ReadBuffer(PGLOBAL g); + virtual int WriteBuffer(PGLOBAL g); + virtual void CloseTableFile(PGLOBAL g, bool abort); + virtual void Rewind(void); + + protected: + bool WriteCompressedBuffer(PGLOBAL g); + int ReadCompressedBuffer(PGLOBAL g, void *rdbuf); + + // Members + z_streamp Zstream; // Compression/decompression stream + Byte *Zbuffer; // Compressed block buffer + int *Zlenp; // Pointer to block length + bool Optimized; // true when opt file is available + }; // end of class ZLBFAM + +#endif // __FILAMGZ_H diff --git a/storage/connect/filamzip.cpp b/storage/connect/filamzip.cpp index d9834e56dcd..6aca4631f32 100644 --- a/storage/connect/filamzip.cpp +++ b/storage/connect/filamzip.cpp @@ -1,35 +1,29 @@ /*********** File AM Zip C++ Program Source Code File (.CPP) ***********/ /* PROGRAM NAME: FILAMZIP */ /* ------------- */ -/* Version 1.5 */ +/* Version 1.0 */ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 2005-2015 */ +/* (C) Copyright to the author Olivier BERTRAND 2016 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ -/* This program are the ZLIB compressed files classes. */ +/* This program are the ZIP file access method classes. */ /* */ /***********************************************************************/ /***********************************************************************/ -/* Include relevant MariaDB header file. */ +/* Include relevant sections of the System header files. */ /***********************************************************************/ #include "my_global.h" -#if defined(__WIN__) -#include <io.h> -#include <fcntl.h> -#if defined(__BORLANDC__) -#define __MFC_COMPAT__ // To define min/max as macro -#endif -//#include <windows.h> -#else // !__WIN__ +#if !defined(__WIN__) #if defined(UNIX) #include <errno.h> -#else // !UNIX +#include <unistd.h> +#else // !UNIX #include <io.h> -#endif +#endif // !UNIX #include <fcntl.h> #endif // !__WIN__ @@ -37,1390 +31,592 @@ /* Include application header files: */ /* global.h is header containing all global declarations. */ /* plgdbsem.h is header containing the DB application declarations. */ -/* tabdos.h is header containing the TABDOS class declarations. */ /***********************************************************************/ #include "global.h" #include "plgdbsem.h" -//#include "catalog.h" -//#include "reldef.h" -//#include "xobject.h" -//#include "kindex.h" -#include "filamtxt.h" -#include "tabdos.h" -#if defined(UNIX) #include "osutil.h" -#endif - -/***********************************************************************/ -/* This define prepares ZLIB function declarations. */ -/***********************************************************************/ -//#define ZLIB_DLL - +#include "filamtxt.h" +#include "tabfmt.h" +//#include "tabzip.h" #include "filamzip.h" -/***********************************************************************/ -/* DB static variables. */ -/***********************************************************************/ -extern int num_read, num_there, num_eq[]; // Statistics - -/* ------------------------------------------------------------------- */ +/* -------------------------- class ZIPUTIL -------------------------- */ /***********************************************************************/ -/* Implementation of the ZIPFAM class. */ +/* Constructors. */ /***********************************************************************/ -ZIPFAM::ZIPFAM(PZIPFAM txfp) : TXTFAM(txfp) - { - Zfile = txfp->Zfile; - Zpos = txfp->Zpos; - } // end of ZIPFAM copy constructor +ZIPUTIL::ZIPUTIL(PSZ tgt, bool mul) +{ + zipfile = NULL; + target = tgt; + fp = NULL; + memory = NULL; + size = 0; + entryopen = false; + multiple = mul; + memset(fn, 0, sizeof(fn)); + + // Init the case mapping table. +#if defined(__WIN__) + for (int i = 0; i < 256; ++i) mapCaseTable[i] = toupper(i); +#else + for (int i = 0; i < 256; ++i) mapCaseTable[i] = i; +#endif +} // end of ZIPUTIL standard constructor + +#if 0 +ZIPUTIL::ZIPUTIL(PZIPUTIL zutp) +{ + zipfile = zutp->zipfile; + target = zutp->target; + fp = zutp->fp; + finfo = zutp->finfo; + entryopen = zutp->entryopen; + multiple = zutp->multiple; + for (int i = 0; i < 256; ++i) mapCaseTable[i] = zutp->mapCaseTable[i]; +} // end of ZIPUTIL copy constructor +#endif // 0 /***********************************************************************/ -/* Zerror: Error function for gz calls. */ -/* gzerror returns the error message for the last error which occurred*/ -/* on the given compressed file. errnum is set to zlib error number. */ -/* If an error occurred in the file system and not in the compression */ -/* library, errnum is set to Z_ERRNO and the application may consult */ -/* errno to get the exact error code. */ +/* This code is the copyright property of Alessandro Felice Cantatore. */ +/* http://xoomer.virgilio.it/acantato/dev/wildcard/wildmatch.html */ /***********************************************************************/ -int ZIPFAM::Zerror(PGLOBAL g) - { - int errnum; +bool ZIPUTIL::WildMatch(PSZ pat, PSZ str) { + PSZ s, p; + bool star = FALSE; - strcpy(g->Message, gzerror(Zfile, &errnum)); +loopStart: + for (s = str, p = pat; *s; ++s, ++p) { + switch (*p) { + case '?': + if (*s == '.') goto starCheck; + break; + case '*': + star = TRUE; + str = s, pat = p; + if (!*++pat) return TRUE; + goto loopStart; + default: + if (mapCaseTable[(uchar)*s] != mapCaseTable[(uchar)*p]) + goto starCheck; + break; + } /* endswitch */ + } /* endfor */ + if (*p == '*') ++p; + return (!*p); - if (errnum == Z_ERRNO) -#if defined(__WIN__) - sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(NULL)); -#else // !__WIN__ - sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(errno)); -#endif // !__WIN__ - - return (errnum == Z_STREAM_END) ? RC_EF : RC_FX; - } // end of Zerror +starCheck: + if (!star) return FALSE; + str++; + goto loopStart; +} // end of WildMatch /***********************************************************************/ -/* Reset: reset position values at the beginning of file. */ +/* open a zip file. */ +/* param: filename path and the filename of the zip file to open. */ +/* return: true if open, false otherwise. */ /***********************************************************************/ -void ZIPFAM::Reset(void) - { - TXTFAM::Reset(); -//gzrewind(Zfile); // Useful ????? - Zpos = 0; - } // end of Reset +bool ZIPUTIL::open(PGLOBAL g, char *filename) +{ + if (!zipfile && !(zipfile = unzOpen64(filename))) + sprintf(g->Message, "Zipfile open error on %s", filename); + + return (zipfile == NULL); +} // end of open /***********************************************************************/ -/* ZIP GetFileLength: returns an estimate of what would be the */ -/* uncompressed file size in number of bytes. */ +/* Close the zip file. */ /***********************************************************************/ -int ZIPFAM::GetFileLength(PGLOBAL g) - { - int len = TXTFAM::GetFileLength(g); +void ZIPUTIL::close() +{ + if (zipfile) { + closeEntry(); + unzClose(zipfile); + zipfile = NULL; + } // endif zipfile - if (len > 0) - // Estimate size reduction to a max of 6 - len *= 6; - - return len; - } // end of GetFileLength +} // end of close /***********************************************************************/ -/* ZIP Access Method opening routine. */ +/* Find next entry matching target pattern. */ /***********************************************************************/ -bool ZIPFAM::OpenTableFile(PGLOBAL g) - { - char opmode[4], filename[_MAX_PATH]; - MODE mode = Tdbp->GetMode(); - - switch (mode) { - case MODE_READ: - strcpy(opmode, "r"); - break; - case MODE_UPDATE: - /*****************************************************************/ - /* Updating ZIP files not implemented yet. */ - /*****************************************************************/ - strcpy(g->Message, MSG(UPD_ZIP_NOT_IMP)); - return true; - case MODE_DELETE: - if (!Tdbp->GetNext()) { - // Store the number of deleted lines - DelRows = Cardinality(g); - - // This will erase the entire file - strcpy(opmode, "w"); -// Block = 0; // For ZBKFAM -// Last = Nrec; // For ZBKFAM - Tdbp->ResetSize(); - } else { - sprintf(g->Message, MSG(NO_PART_DEL), "ZIP"); - return true; - } // endif filter - - break; - case MODE_INSERT: - strcpy(opmode, "a+"); - break; - default: - sprintf(g->Message, MSG(BAD_OPEN_MODE), mode); - return true; - } // endswitch Mode - - /*********************************************************************/ - /* Open according to logical input/output mode required. */ - /* Use specific zlib functions. */ - /* Treat files as binary. */ - /*********************************************************************/ - strcat(opmode, "b"); - Zfile = gzopen(PlugSetPath(filename, To_File, Tdbp->GetPath()), opmode); - - if (Zfile == NULL) { - sprintf(g->Message, MSG(GZOPEN_ERROR), - opmode, (int)errno, filename); - strcat(strcat(g->Message, ": "), strerror(errno)); - return (mode == MODE_READ && errno == ENOENT) - ? PushWarning(g, Tdbp) : true; - } // endif Zfile - - /*********************************************************************/ - /* Something to be done here. >>>>>>>> NOT DONE <<<<<<<< */ - /*********************************************************************/ -//To_Fb = dbuserp->Openlist; // Keep track of File block - - /*********************************************************************/ - /* Allocate the line buffer. */ - /*********************************************************************/ - return AllocateBuffer(g); - } // end of OpenTableFile +int ZIPUTIL::findEntry(PGLOBAL g, bool next) +{ + int rc; -/***********************************************************************/ -/* Allocate the line buffer. For mode Delete a bigger buffer has to */ -/* be allocated because is it also used to move lines into the file. */ -/***********************************************************************/ -bool ZIPFAM::AllocateBuffer(PGLOBAL g) - { - MODE mode = Tdbp->GetMode(); + do { + if (next) { + rc = unzGoToNextFile(zipfile); - Buflen = Lrecl + 2; // Lrecl does not include CRLF -//Buflen *= ((Mode == MODE_DELETE) ? DOS_BUFF_LEN : 1); NIY + if (rc == UNZ_END_OF_LIST_OF_FILE) + return RC_EF; + else if (rc != UNZ_OK) { + sprintf(g->Message, "unzGoToNextFile rc = %d", rc); + return RC_FX; + } // endif rc - if (trace) - htrc("SubAllocating a buffer of %d bytes\n", Buflen); + } // endif next - To_Buf = (char*)PlugSubAlloc(g, NULL, Buflen); + if (target && *target) { + rc = unzGetCurrentFileInfo(zipfile, NULL, fn, sizeof(fn), + NULL, 0, NULL, 0); + if (rc == UNZ_OK) { + if (WildMatch(target, fn)) + return RC_OK; - if (mode == MODE_INSERT) { - /*******************************************************************/ - /* For Insert buffer must be prepared. */ - /*******************************************************************/ - memset(To_Buf, ' ', Buflen); - To_Buf[Buflen - 2] = '\n'; - To_Buf[Buflen - 1] = '\0'; - } // endif Insert + } else { + sprintf(g->Message, "GetCurrentFileInfo rc = %d", rc); + return RC_FX; + } // endif rc - return false; - } // end of AllocateBuffer + } else + return RC_OK; -/***********************************************************************/ -/* GetRowID: return the RowID of last read record. */ -/***********************************************************************/ -int ZIPFAM::GetRowID(void) - { - return Rows; - } // end of GetRowID + next = true; + } while (true); -/***********************************************************************/ -/* GetPos: return the position of last read record. */ -/***********************************************************************/ -int ZIPFAM::GetPos(void) - { - return (int)Zpos; - } // end of GetPos + strcpy(g->Message, "FindNext logical error"); + return RC_FX; +} // end of FindEntry -/***********************************************************************/ -/* GetNextPos: return the position of next record. */ -/***********************************************************************/ -int ZIPFAM::GetNextPos(void) - { - return gztell(Zfile); - } // end of GetNextPos /***********************************************************************/ -/* SetPos: Replace the table at the specified position. */ +/* Get the next used entry. */ /***********************************************************************/ -bool ZIPFAM::SetPos(PGLOBAL g, int pos __attribute__((unused))) - { - sprintf(g->Message, MSG(NO_SETPOS_YET), "ZIP"); - return true; -#if 0 - Fpos = pos; +int ZIPUTIL::nextEntry(PGLOBAL g) +{ + if (multiple) { + int rc; - if (fseek(Stream, Fpos, SEEK_SET)) { - sprintf(g->Message, MSG(FSETPOS_ERROR), Fpos); - return true; - } // endif + closeEntry(); - Placed = true; - return false; -#endif // 0 - } // end of SetPos + if ((rc = findEntry(g, true)) != RC_OK) + return rc; -/***********************************************************************/ -/* Record file position in case of UPDATE or DELETE. */ -/***********************************************************************/ -bool ZIPFAM::RecordPos(PGLOBAL) - { - Zpos = gztell(Zfile); - return false; - } // end of RecordPos + if (openEntry(g)) + return RC_FX; -/***********************************************************************/ -/* Skip one record in file. */ -/***********************************************************************/ -int ZIPFAM::SkipRecord(PGLOBAL g, bool header) - { - // Skip this record - if (gzeof(Zfile)) - return RC_EF; - else if (gzgets(Zfile, To_Buf, Buflen) == Z_NULL) - return Zerror(g); + return RC_OK; + } else + return RC_EF; - if (header) - RecordPos(g); +} // end of nextEntry - return RC_OK; - } // end of SkipRecord /***********************************************************************/ -/* ReadBuffer: Read one line from a compressed text file. */ +/* OpenTableFile: Open a DOS/UNIX table file from a ZIP file. */ /***********************************************************************/ -int ZIPFAM::ReadBuffer(PGLOBAL g) - { - char *p; - int rc; - - if (!Zfile) - return RC_EF; - - if (!Placed) { - /*******************************************************************/ - /* Record file position in case of UPDATE or DELETE. */ - /*******************************************************************/ - next: - if (RecordPos(g)) - return RC_FX; - - CurBlk = Rows++; // Update RowID - - /*******************************************************************/ - /* Check whether optimization on ROWID */ - /* can be done, as well as for join as for local filtering. */ - /*******************************************************************/ - switch (Tdbp->TestBlock(g)) { - case RC_EF: - return RC_EF; - case RC_NF: - // Skip this record - if ((rc = SkipRecord(g, FALSE)) != RC_OK) - return rc; - - goto next; - } // endswitch rc - - } else - Placed = false; - - if (gzeof(Zfile)) { - rc = RC_EF; - } else if (gzgets(Zfile, To_Buf, Buflen) != Z_NULL) { - p = To_Buf + strlen(To_Buf) - 1; - - if (*p == '\n') - *p = '\0'; // Eliminate ending new-line character - - if (*(--p) == '\r') - *p = '\0'; // Eliminate eventuel carriage return - - strcpy(Tdbp->GetLine(), To_Buf); - IsRead = true; - rc = RC_OK; - num_read++; - } else - rc = Zerror(g); - - if (trace > 1) - htrc(" Read: '%s' rc=%d\n", To_Buf, rc); - - return rc; - } // end of ReadBuffer +bool ZIPUTIL::OpenTable(PGLOBAL g, MODE mode, char *fn) +{ + /*********************************************************************/ + /* The file will be decompressed into virtual memory. */ + /*********************************************************************/ + if (mode == MODE_READ || mode == MODE_ANY) { + bool b = open(g, fn); -/***********************************************************************/ -/* WriteDB: Data Base write routine for ZDOS access method. */ -/* Update is not possible without using a temporary file (NIY). */ -/***********************************************************************/ -int ZIPFAM::WriteBuffer(PGLOBAL g) - { - /*********************************************************************/ - /* Prepare the write buffer. */ - /*********************************************************************/ - strcat(strcpy(To_Buf, Tdbp->GetLine()), CrLf); - - /*********************************************************************/ - /* Now start the writing process. */ - /*********************************************************************/ - if (gzputs(Zfile, To_Buf) < 0) - return Zerror(g); - - return RC_OK; - } // end of WriteBuffer + if (!b) { + int rc; -/***********************************************************************/ -/* Data Base delete line routine for ZDOS access method. (NIY) */ -/***********************************************************************/ -int ZIPFAM::DeleteRecords(PGLOBAL g, int) - { - strcpy(g->Message, MSG(NO_ZIP_DELETE)); - return RC_FX; - } // end of DeleteRecords + if (target && *target) { + if (!multiple) { + rc = unzLocateFile(zipfile, target, 0); -/***********************************************************************/ -/* Data Base close routine for DOS access method. */ -/***********************************************************************/ -void ZIPFAM::CloseTableFile(PGLOBAL, bool) - { - int rc = gzclose(Zfile); + if (rc == UNZ_END_OF_LIST_OF_FILE) { + sprintf(g->Message, "Target file %s not in %s", target, fn); + return true; + } else if (rc != UNZ_OK) { + sprintf(g->Message, "unzLocateFile rc=%d", rc); + return true; + } // endif's rc - if (trace) - htrc("ZIP CloseDB: closing %s rc=%d\n", To_File, rc); + } else { + if ((rc = findEntry(g, false)) == RC_FX) + return true; + else if (rc == RC_EF) { + sprintf(g->Message, "No match of %s in %s", target, fn); + return true; + } // endif rc - Zfile = NULL; // So we can know whether table is open -//To_Fb->Count = 0; // Avoid double closing by PlugCloseAll - } // end of CloseTableFile + } // endif multiple -/***********************************************************************/ -/* Rewind routine for ZIP access method. */ -/***********************************************************************/ -void ZIPFAM::Rewind(void) - { - gzrewind(Zfile); - } // end of Rewind + } // endif target -/* ------------------------------------------------------------------- */ + if (openEntry(g)) + return true; -/***********************************************************************/ -/* Constructors. */ -/***********************************************************************/ -ZBKFAM::ZBKFAM(PDOSDEF tdp) : ZIPFAM(tdp) - { - Blocked = true; - Block = tdp->GetBlock(); - Last = tdp->GetLast(); - Nrec = tdp->GetElemt(); - CurLine = NULL; - NxtLine = NULL; - Closing = false; - BlkPos = tdp->GetTo_Pos(); - } // end of ZBKFAM standard constructor - -ZBKFAM::ZBKFAM(PZBKFAM txfp) : ZIPFAM(txfp) - { - CurLine = txfp->CurLine; - NxtLine = txfp->NxtLine; - Closing = txfp->Closing; - } // end of ZBKFAM copy constructor + if (size > 0) { + /*******************************************************************/ + /* Link a Fblock. This make possible to automatically close it */ + /* in case of error g->jump. */ + /*******************************************************************/ + PDBUSER dbuserp = (PDBUSER)g->Activityp->Aptr; -/***********************************************************************/ -/* Use BlockTest to reduce the table estimated size. */ -/***********************************************************************/ -int ZBKFAM::MaxBlkSize(PGLOBAL g, int) - { - int rc = RC_OK, savcur = CurBlk; - int size; - - // Roughly estimate the table size as the sum of blocks - // that can contain good rows - for (size = 0, CurBlk = 0; CurBlk < Block; CurBlk++) - if ((rc = Tdbp->TestBlock(g)) == RC_OK) - size += (CurBlk == Block - 1) ? Last : Nrec; - else if (rc == RC_EF) - break; - - CurBlk = savcur; - return size; - } // end of MaxBlkSize + fp = (PFBLOCK)PlugSubAlloc(g, NULL, sizeof(FBLOCK)); + fp->Type = TYPE_FB_ZIP; + fp->Fname = PlugDup(g, fn); + fp->Next = dbuserp->Openlist; + dbuserp->Openlist = fp; + fp->Count = 1; + fp->Length = size; + fp->Memory = memory; + fp->Mode = mode; + fp->File = this; + fp->Handle = 0; + } // endif fp -/***********************************************************************/ -/* ZBK Cardinality: returns table cardinality in number of rows. */ -/* This function can be called with a null argument to test the */ -/* availability of Cardinality implementation (1 yes, 0 no). */ -/***********************************************************************/ -int ZBKFAM::Cardinality(PGLOBAL g) - { - return (g) ? (int)((Block - 1) * Nrec + Last) : 1; - } // end of Cardinality + } else + return true; -/***********************************************************************/ -/* Allocate the line buffer. For mode Delete a bigger buffer has to */ -/* be allocated because is it also used to move lines into the file. */ -/***********************************************************************/ -bool ZBKFAM::AllocateBuffer(PGLOBAL g) - { - Buflen = Nrec * (Lrecl + 2); - CurLine = To_Buf = (char*)PlugSubAlloc(g, NULL, Buflen); - - if (Tdbp->GetMode() == MODE_INSERT) { - // Set values so Block and Last can be recalculated - if (Last == Nrec) { - CurBlk = Block; - Rbuf = Nrec; // To be used by WriteDB - } else { - // The last block must be completed - CurBlk = Block - 1; - Rbuf = Nrec - Last; // To be used by WriteDB - } // endif Last - - } // endif Insert - - return false; - } // end of AllocateBuffer + } else { + strcpy(g->Message, "Only READ mode supported for ZIP files"); + return true; + } // endif mode -/***********************************************************************/ -/* GetRowID: return the RowID of last read record. */ -/***********************************************************************/ -int ZBKFAM::GetRowID(void) - { - return CurNum + Nrec * CurBlk + 1; - } // end of GetRowID + return false; +} // end of OpenTableFile /***********************************************************************/ -/* GetPos: return the position of last read record. */ +/* Open target in zip file. */ /***********************************************************************/ -int ZBKFAM::GetPos(void) - { - return CurNum + Nrec * CurBlk; // Computed file index - } // end of GetPos +bool ZIPUTIL::openEntry(PGLOBAL g) +{ + int rc; -/***********************************************************************/ -/* Record file position in case of UPDATE or DELETE. */ -/* Not used yet for fixed tables. */ -/***********************************************************************/ -bool ZBKFAM::RecordPos(PGLOBAL /*g*/) - { -//strcpy(g->Message, "RecordPos not implemented for zip blocked tables"); -//return true; - return RC_OK; - } // end of RecordPos + rc = unzGetCurrentFileInfo(zipfile, &finfo, fn, sizeof(fn), + NULL, 0, NULL, 0); -/***********************************************************************/ -/* Skip one record in file. */ -/***********************************************************************/ -int ZBKFAM::SkipRecord(PGLOBAL /*g*/, bool) - { -//strcpy(g->Message, "SkipRecord not implemented for zip blocked tables"); -//return RC_FX; - return RC_OK; - } // end of SkipRecord + if (rc != UNZ_OK) { + sprintf(g->Message, "unzGetCurrentFileInfo64 rc=%d", rc); + return true; + } else if ((rc = unzOpenCurrentFile(zipfile)) != UNZ_OK) { + sprintf(g->Message, "unzOpen fn=%s rc=%d", fn, rc); + return true; + } // endif rc -/***********************************************************************/ -/* ReadBuffer: Read one line from a compressed text file. */ -/***********************************************************************/ -int ZBKFAM::ReadBuffer(PGLOBAL g) - { - int n, skip, rc = RC_OK; - - /*********************************************************************/ - /* Sequential reading when Placed is not true. */ - /*********************************************************************/ - if (++CurNum < Rbuf) { - CurLine = NxtLine; - - // Get the position of the next line in the buffer - while (*NxtLine++ != '\n') ; - - // Set caller line buffer - n = NxtLine - CurLine - Ending; - memcpy(Tdbp->GetLine(), CurLine, n); - Tdbp->GetLine()[n] = '\0'; - return RC_OK; - } else if (Rbuf < Nrec && CurBlk != -1) - return RC_EF; - - /*********************************************************************/ - /* New block. */ - /*********************************************************************/ - CurNum = 0; - skip = 0; - - next: - if (++CurBlk >= Block) - return RC_EF; - - /*********************************************************************/ - /* Before using the new block, check whether block optimization */ - /* can be done, as well as for join as for local filtering. */ - /*********************************************************************/ - switch (Tdbp->TestBlock(g)) { - case RC_EF: - return RC_EF; - case RC_NF: - skip++; - goto next; - } // endswitch rc - - if (skip) - // Skip blocks rejected by block optimization - for (int i = CurBlk - skip; i < CurBlk; i++) { - BlkLen = BlkPos[i + 1] - BlkPos[i]; - - if (gzseek(Zfile, (z_off_t)BlkLen, SEEK_CUR) < 0) - return Zerror(g); - - } // endfor i - - BlkLen = BlkPos[CurBlk + 1] - BlkPos[CurBlk]; - - if (!(n = gzread(Zfile, To_Buf, BlkLen))) { - rc = RC_EF; - } else if (n > 0) { - // Get the position of the current line - CurLine = To_Buf; - - // Now get the position of the next line - for (NxtLine = CurLine; *NxtLine++ != '\n';) ; - - // Set caller line buffer - n = NxtLine - CurLine - Ending; - memcpy(Tdbp->GetLine(), CurLine, n); - Tdbp->GetLine()[n] = '\0'; - Rbuf = (CurBlk == Block - 1) ? Last : Nrec; - IsRead = true; - rc = RC_OK; - num_read++; - } else - rc = Zerror(g); - - return rc; - } // end of ReadBuffer + size = finfo.uncompressed_size; + memory = new char[size + 1]; -/***********************************************************************/ -/* WriteDB: Data Base write routine for ZDOS access method. */ -/* Update is not possible without using a temporary file (NIY). */ -/***********************************************************************/ -int ZBKFAM::WriteBuffer(PGLOBAL g) - { - /*********************************************************************/ - /* Prepare the write buffer. */ - /*********************************************************************/ - if (!Closing) - strcat(strcpy(CurLine, Tdbp->GetLine()), CrLf); - - /*********************************************************************/ - /* In Insert mode, blocs are added sequentialy to the file end. */ - /* Note: Update mode is not handled for zip files. */ - /*********************************************************************/ - if (++CurNum == Rbuf) { - /*******************************************************************/ - /* New block, start the writing process. */ - /*******************************************************************/ - BlkLen = CurLine + strlen(CurLine) - To_Buf; - - if (gzwrite(Zfile, To_Buf, BlkLen) != BlkLen || - gzflush(Zfile, Z_FULL_FLUSH)) { - Closing = true; - return Zerror(g); - } // endif gzwrite - - Rbuf = Nrec; - CurBlk++; - CurNum = 0; - CurLine = To_Buf; - } else - CurLine += strlen(CurLine); - - return RC_OK; - } // end of WriteBuffer + if ((rc = unzReadCurrentFile(zipfile, memory, size)) < 0) { + sprintf(g->Message, "unzReadCurrentFile rc = %d", rc); + unzCloseCurrentFile(zipfile); + free(memory); + memory = NULL; + entryopen = false; + } else { + memory[size] = 0; // Required by some table types (XML) + entryopen = true; + } // endif rc + + if (trace) + htrc("Openning entry%s %s\n", fn, (entryopen) ? "oked" : "failed"); + + return !entryopen; +} // end of openEntry + +/***********************************************************************/ +/* Close the zip file. */ +/***********************************************************************/ +void ZIPUTIL::closeEntry() +{ + if (entryopen) { + unzCloseCurrentFile(zipfile); + entryopen = false; + } // endif entryopen + + if (memory) { + free(memory); + memory = NULL; + } // endif memory + +} // end of closeEntry + +/* -------------------------- class ZIPFAM --------------------------- */ /***********************************************************************/ -/* Data Base delete line routine for ZBK access method. */ -/* Implemented only for total deletion of the table, which is done */ -/* by opening the file in mode "wb". */ +/* Constructors. */ /***********************************************************************/ -int ZBKFAM::DeleteRecords(PGLOBAL g, int irc) - { - if (irc == RC_EF) { - LPCSTR name = Tdbp->GetName(); - PDOSDEF defp = (PDOSDEF)Tdbp->GetDef(); - - defp->SetBlock(0); - defp->SetLast(Nrec); +ZIPFAM::ZIPFAM(PDOSDEF tdp) : MAPFAM(tdp) +{ + zutp = NULL; + target = tdp->GetEntry(); + mul = tdp->GetMul(); +} // end of ZIPFAM standard constructor - if (!defp->SetIntCatInfo("Blocks", 0) || - !defp->SetIntCatInfo("Last", 0)) { - sprintf(g->Message, MSG(UPDATE_ERROR), "Header"); - return RC_FX; - } else - return RC_OK; +ZIPFAM::ZIPFAM(PZIPFAM txfp) : MAPFAM(txfp) +{ + zutp = txfp->zutp; + target = txfp->target; + mul = txfp->mul; +} // end of ZIPFAM copy constructor - } else - return irc; - - } // end of DeleteRecords +ZIPFAM::ZIPFAM(PDOSDEF tdp, PZPXFAM txfp) : MAPFAM(tdp) +{ + zutp = txfp->zutp; + target = txfp->target; + mul = txfp->mul; +} // end of ZIPFAM constructor used in ResetTableOpt /***********************************************************************/ -/* Data Base close routine for ZBK access method. */ +/* ZIP GetFileLength: returns file size in number of bytes. */ /***********************************************************************/ -void ZBKFAM::CloseTableFile(PGLOBAL g, bool) - { - int rc = RC_OK; - - if (Tdbp->GetMode() == MODE_INSERT) { - LPCSTR name = Tdbp->GetName(); - PDOSDEF defp = (PDOSDEF)Tdbp->GetDef(); - - if (CurNum && !Closing) { - // Some more inserted lines remain to be written - Last = (Nrec - Rbuf) + CurNum; - Block = CurBlk + 1; - Rbuf = CurNum--; - Closing = true; - rc = WriteBuffer(g); - } else if (Rbuf == Nrec) { - Last = Nrec; - Block = CurBlk; - } // endif CurNum - - if (rc != RC_FX) { - defp->SetBlock(Block); - defp->SetLast(Last); - defp->SetIntCatInfo("Blocks", Block); - defp->SetIntCatInfo("Last", Last); - } // endif - - gzclose(Zfile); - } else if (Tdbp->GetMode() == MODE_DELETE) { - rc = DeleteRecords(g, RC_EF); - gzclose(Zfile); - } else - rc = gzclose(Zfile); - - if (trace) - htrc("ZIP CloseDB: closing %s rc=%d\n", To_File, rc); - - Zfile = NULL; // So we can know whether table is open -//To_Fb->Count = 0; // Avoid double closing by PlugCloseAll - } // end of CloseTableFile +int ZIPFAM::GetFileLength(PGLOBAL g) +{ + int len = (zutp && zutp->entryopen) ? Top - Memory + : TXTFAM::GetFileLength(g) * 3; -/***********************************************************************/ -/* Rewind routine for ZBK access method. */ -/***********************************************************************/ -void ZBKFAM::Rewind(void) - { - gzrewind(Zfile); - CurBlk = -1; - CurNum = Rbuf; - } // end of Rewind + if (trace) + htrc("Zipped file length=%d\n", len); -/* ------------------------------------------------------------------- */ + return len; +} // end of GetFileLength /***********************************************************************/ -/* Constructors. */ +/* ZIP Cardinality: return the number of rows if possible. */ /***********************************************************************/ -ZIXFAM::ZIXFAM(PDOSDEF tdp) : ZBKFAM(tdp) - { -//Block = tdp->GetBlock(); -//Last = tdp->GetLast(); - Nrec = (tdp->GetElemt()) ? tdp->GetElemt() : DOS_BUFF_LEN; - Blksize = Nrec * Lrecl; - } // end of ZIXFAM standard constructor +int ZIPFAM::Cardinality(PGLOBAL g) +{ + if (!g) + return 1; -/***********************************************************************/ -/* ZIX Cardinality: returns table cardinality in number of rows. */ -/* This function can be called with a null argument to test the */ -/* availability of Cardinality implementation (1 yes, 0 no). */ -/***********************************************************************/ -int ZIXFAM::Cardinality(PGLOBAL g) - { - if (Last) - return (g) ? (int)((Block - 1) * Nrec + Last) : 1; - else // Last and Block not defined, cannot do it yet - return 0; + int card = -1; + int len = GetFileLength(g); - } // end of Cardinality + card = (len / (int)Lrecl) * 2; // Estimated ??? + return card; +} // end of Cardinality /***********************************************************************/ -/* Allocate the line buffer. For mode Delete a bigger buffer has to */ -/* be allocated because is it also used to move lines into the file. */ +/* OpenTableFile: Open a DOS/UNIX table file from a ZIP file. */ /***********************************************************************/ -bool ZIXFAM::AllocateBuffer(PGLOBAL g) - { - Buflen = Blksize; - To_Buf = (char*)PlugSubAlloc(g, NULL, Buflen); - - if (Tdbp->GetMode() == MODE_INSERT) { - /*******************************************************************/ - /* For Insert the buffer must be prepared. */ - /*******************************************************************/ - memset(To_Buf, ' ', Buflen); - - if (Tdbp->GetFtype() < 2) - // if not binary, the file is physically a text file - for (int len = Lrecl; len <= Buflen; len += Lrecl) { -#if defined(__WIN__) - To_Buf[len - 2] = '\r'; -#endif // __WIN__ - To_Buf[len - 1] = '\n'; - } // endfor len - - // Set values so Block and Last can be recalculated - if (Last == Nrec) { - CurBlk = Block; - Rbuf = Nrec; // To be used by WriteDB - } else { - // The last block must be completed - CurBlk = Block - 1; - Rbuf = Nrec - Last; // To be used by WriteDB - } // endif Last - - } // endif Insert - - return false; - } // end of AllocateBuffer +bool ZIPFAM::OpenTableFile(PGLOBAL g) +{ + char filename[_MAX_PATH]; + MODE mode = Tdbp->GetMode(); -/***********************************************************************/ -/* ReadBuffer: Read one line from a compressed text file. */ -/***********************************************************************/ -int ZIXFAM::ReadBuffer(PGLOBAL g) - { - int n, rc = RC_OK; - - /*********************************************************************/ - /* Sequential reading when Placed is not true. */ - /*********************************************************************/ - if (++CurNum < Rbuf) { - Tdbp->IncLine(Lrecl); // Used by DOSCOL functions - return RC_OK; - } else if (Rbuf < Nrec && CurBlk != -1) - return RC_EF; - - /*********************************************************************/ - /* New block. */ - /*********************************************************************/ - CurNum = 0; - Tdbp->SetLine(To_Buf); - - int skip = 0; - - next: - if (++CurBlk >= Block) - return RC_EF; - - /*********************************************************************/ - /* Before using the new block, check whether block optimization */ - /* can be done, as well as for join as for local filtering. */ - /*********************************************************************/ - switch (Tdbp->TestBlock(g)) { - case RC_EF: - return RC_EF; - case RC_NF: - skip++; - goto next; - } // endswitch rc - - if (skip) - // Skip blocks rejected by block optimization - for (int i = 0; i < skip; i++) { - if (gzseek(Zfile, (z_off_t)Buflen, SEEK_CUR) < 0) - return Zerror(g); - - } // endfor i - - if (!(n = gzread(Zfile, To_Buf, Buflen))) { - rc = RC_EF; - } else if (n > 0) { - Rbuf = n / Lrecl; - IsRead = true; - rc = RC_OK; - num_read++; - } else - rc = Zerror(g); - - return rc; - } // end of ReadBuffer + /*********************************************************************/ + /* Allocate the ZIP utility class. */ + /*********************************************************************/ + zutp = new(g) ZIPUTIL(target, mul); -/***********************************************************************/ -/* WriteDB: Data Base write routine for ZDOS access method. */ -/* Update is not possible without using a temporary file (NIY). */ -/***********************************************************************/ -int ZIXFAM::WriteBuffer(PGLOBAL g) - { - /*********************************************************************/ - /* In Insert mode, blocs are added sequentialy to the file end. */ - /* Note: Update mode is not handled for zip files. */ - /*********************************************************************/ - if (++CurNum == Rbuf) { - /*******************************************************************/ - /* New block, start the writing process. */ - /*******************************************************************/ - BlkLen = Rbuf * Lrecl; - - if (gzwrite(Zfile, To_Buf, BlkLen) != BlkLen || - gzflush(Zfile, Z_FULL_FLUSH)) { - Closing = true; - return Zerror(g); - } // endif gzwrite - - Rbuf = Nrec; - CurBlk++; - CurNum = 0; - Tdbp->SetLine(To_Buf); - } else - Tdbp->IncLine(Lrecl); // Used by FIXCOL functions - - return RC_OK; - } // end of WriteBuffer - -/* --------------------------- Class ZLBFAM -------------------------- */ + // We used the file name relative to recorded datapath + PlugSetPath(filename, To_File, Tdbp->GetPath()); -/***********************************************************************/ -/* Constructors. */ -/***********************************************************************/ -ZLBFAM::ZLBFAM(PDOSDEF tdp) : BLKFAM(tdp) - { - Zstream = NULL; - Zbuffer = NULL; - Zlenp = NULL; - Optimized = tdp->IsOptimized(); - } // end of ZLBFAM standard constructor - -ZLBFAM::ZLBFAM(PZLBFAM txfp) : BLKFAM(txfp) - { - Zstream = txfp->Zstream; - Zbuffer = txfp->Zbuffer; - Zlenp = txfp->Zlenp; - Optimized = txfp->Optimized; - } // end of ZLBFAM (dummy?) copy constructor + if (!zutp->OpenTable(g, mode, filename)) { + // The pseudo "buffer" is here the entire real buffer + Fpos = Mempos = Memory = zutp->memory; + Top = Memory + zutp->size; + To_Fb = zutp->fp; // Useful when closing + } else + return true; + + return false; + } // end of OpenTableFile /***********************************************************************/ -/* ZLB GetFileLength: returns an estimate of what would be the */ -/* uncompressed file size in number of bytes. */ +/* GetNext: go to next entry. */ /***********************************************************************/ -int ZLBFAM::GetFileLength(PGLOBAL g) - { - int len = (Optimized) ? BlkPos[Block] : BLKFAM::GetFileLength(g); +int ZIPFAM::GetNext(PGLOBAL g) +{ + int rc = zutp->nextEntry(g); - if (len > 0) - // Estimate size reduction to a max of 5 - len *= 5; + if (rc != RC_OK) + return rc; - return len; - } // end of GetFileLength + Mempos = Memory = zutp->memory; + Top = Memory + zutp->size; + return RC_OK; +} // end of GetNext +#if 0 /***********************************************************************/ -/* Allocate the line buffer. For mode Delete a bigger buffer has to */ -/* be allocated because is it also used to move lines into the file. */ +/* ReadBuffer: Read one line for a ZIP file. */ /***********************************************************************/ -bool ZLBFAM::AllocateBuffer(PGLOBAL g) - { - char *msg; - int n, zrc; +int ZIPFAM::ReadBuffer(PGLOBAL g) +{ + int rc, len; -#if 0 - if (!Optimized && Tdbp->NeedIndexing(g)) { - strcpy(g->Message, MSG(NOP_ZLIB_INDEX)); - return TRUE; - } // endif indexing -#endif // 0 + // Are we at the end of the memory + if (Mempos >= Top) { + if ((rc = zutp->nextEntry(g)) != RC_OK) + return rc; -#if defined(NOLIB) - if (!zlib && LoadZlib()) { - sprintf(g->Message, MSG(DLL_LOAD_ERROR), GetLastError(), "zlib.dll"); - return TRUE; - } // endif zlib -#endif + Mempos = Memory = zutp->memory; + Top = Memory + zutp->size; + } // endif Mempos - BLKFAM::AllocateBuffer(g); -//Buflen = Nrec * (Lrecl + 2); -//Rbuf = Nrec; - - // Allocate the compressed buffer - n = Buflen + 16; // ????????????????????????????????? - Zlenp = (int*)PlugSubAlloc(g, NULL, n); - Zbuffer = (Byte*)(Zlenp + 1); - - // Allocate and initialize the Z stream - Zstream = (z_streamp)PlugSubAlloc(g, NULL, sizeof(z_stream)); - Zstream->zalloc = (alloc_func)0; - Zstream->zfree = (free_func)0; - Zstream->opaque = (voidpf)0; - Zstream->next_in = NULL; - Zstream->avail_in = 0; - - if (Tdbp->GetMode() == MODE_READ) { - msg = "inflateInit"; - zrc = inflateInit(Zstream); - } else { - msg = "deflateInit"; - zrc = deflateInit(Zstream, Z_DEFAULT_COMPRESSION); - } // endif Mode - - if (zrc != Z_OK) { - if (Zstream->msg) - sprintf(g->Message, "%s error: %s", msg, Zstream->msg); - else - sprintf(g->Message, "%s error: %d", msg, zrc); - - return TRUE; - } // endif zrc - - if (Tdbp->GetMode() == MODE_INSERT) { - // Write the file header block - if (Last == Nrec) { - CurBlk = Block; - CurNum = 0; - - if (!GetFileLength(g)) { - // Write the zlib header as an extra block - strcpy(To_Buf, "PlugDB"); - BlkLen = strlen("PlugDB") + 1; - - if (WriteCompressedBuffer(g)) - return TRUE; - - } // endif void file - - } else { - // In mode insert, if Last != Nrec, last block must be updated - CurBlk = Block - 1; - CurNum = Last; - - strcpy(g->Message, MSG(NO_PAR_BLK_INS)); - return TRUE; - } // endif Last - - } else { // MODE_READ - // First thing to do is to read the header block - void *rdbuf; - - if (Optimized) { - BlkLen = BlkPos[0]; - rdbuf = Zlenp; - } else { - // Get the stored length from the file itself - if (fread(Zlenp, sizeof(int), 1, Stream) != 1) - return FALSE; // Empty file - - BlkLen = *Zlenp; - rdbuf = Zbuffer; - } // endif Optimized - - switch (ReadCompressedBuffer(g, rdbuf)) { - case RC_EF: - return FALSE; - case RC_FX: -#if defined(UNIX) - sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(errno)); +#if 0 + if (!Placed) { + /*******************************************************************/ + /* Record file position in case of UPDATE or DELETE. */ + /*******************************************************************/ + int rc; + + next: + Fpos = Mempos; + CurBlk = (int)Rows++; + + /*******************************************************************/ + /* Check whether optimization on ROWID */ + /* can be done, as well as for join as for local filtering. */ + /*******************************************************************/ + switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + // Skip this record + if ((rc = SkipRecord(g, false)) != RC_OK) + return rc; + + goto next; + } // endswitch rc + + } else + Placed = false; #else - sprintf(g->Message, MSG(READ_ERROR), To_File, _strerror(NULL)); + // Perhaps unuseful + Fpos = Mempos; + CurBlk = (int)Rows++; + Placed = false; #endif - case RC_NF: - return TRUE; - } // endswitch - // Some old tables can have PlugDB in their header - if (strcmp(To_Buf, "PlugDB")) { - sprintf(g->Message, MSG(BAD_HEADER), Tdbp->GetFile(g)); - return TRUE; - } // endif strcmp + // Immediately calculate next position (Used by DeleteDB) + while (*Mempos++ != '\n'); // What about Unix ??? - } // endif Mode + // Set caller line buffer + len = (Mempos - Fpos) - 1; - return FALSE; - } // end of AllocateBuffer + // Don't rely on ENDING setting + if (len > 0 && *(Mempos - 2) == '\r') + len--; // Line ends by CRLF -/***********************************************************************/ -/* GetPos: return the position of last read record. */ -/***********************************************************************/ -int ZLBFAM::GetPos(void) - { - return (Optimized) ? (CurNum + Nrec * CurBlk) : Fpos; - } // end of GetPos + memcpy(Tdbp->GetLine(), Fpos, len); + Tdbp->GetLine()[len] = '\0'; + return RC_OK; +} // end of ReadBuffer /***********************************************************************/ -/* GetNextPos: should not be called for this class. */ +/* Table file close routine for MAP access method. */ /***********************************************************************/ -int ZLBFAM::GetNextPos(void) - { - if (Optimized) { - assert(FALSE); - return 0; - } else - return ftell(Stream); +void ZIPFAM::CloseTableFile(PGLOBAL g, bool) +{ + close(); +} // end of CloseTableFile +#endif // 0 - } // end of GetNextPos +/* -------------------------- class ZPXFAM --------------------------- */ /***********************************************************************/ -/* SetPos: Replace the table at the specified position. */ +/* Constructors. */ /***********************************************************************/ -bool ZLBFAM::SetPos(PGLOBAL g, int pos __attribute__((unused))) - { - sprintf(g->Message, MSG(NO_SETPOS_YET), "ZIP"); - return true; -#if 0 // All this must be checked - if (pos < 0) { - strcpy(g->Message, MSG(INV_REC_POS)); - return true; - } // endif recpos - - CurBlk = pos / Nrec; - CurNum = pos % Nrec; -#if defined(_DEBUG) - num_eq[(CurBlk == OldBlk) ? 1 : 0]++; -#endif +ZPXFAM::ZPXFAM(PDOSDEF tdp) : MPXFAM(tdp) +{ + zutp = NULL; + target = tdp->GetEntry(); + mul = tdp->GetMul(); + //Lrecl = tdp->GetLrecl(); +} // end of ZPXFAM standard constructor - // Indicate the table position was externally set - Placed = true; - return false; -#endif // 0 - } // end of SetPos +ZPXFAM::ZPXFAM(PZPXFAM txfp) : MPXFAM(txfp) +{ + zutp = txfp->zutp; + target = txfp->target; + mul = txfp->mul; +//Lrecl = txfp->Lrecl; +} // end of ZPXFAM copy constructor /***********************************************************************/ -/* ReadBuffer: Read one line for a text file. */ +/* ZIP GetFileLength: returns file size in number of bytes. */ /***********************************************************************/ -int ZLBFAM::ReadBuffer(PGLOBAL g) - { - int n; - void *rdbuf; - - /*********************************************************************/ - /* Sequential reading when Placed is not true. */ - /*********************************************************************/ - if (Placed) { - Placed = FALSE; - } else if (++CurNum < Rbuf) { - CurLine = NxtLine; - - // Get the position of the next line in the buffer - if (Tdbp->GetFtype() == RECFM_VAR) - while (*NxtLine++ != '\n') ; - else - NxtLine += Lrecl; - - // Set caller line buffer - n = NxtLine - CurLine - ((Tdbp->GetFtype() == RECFM_BIN) ? 0 : Ending); - memcpy(Tdbp->GetLine(), CurLine, n); - Tdbp->GetLine()[n] = '\0'; - return RC_OK; - } else if (Rbuf < Nrec && CurBlk != -1) { - CurNum--; // To have a correct Last value when optimizing - return RC_EF; - } else { - /*******************************************************************/ - /* New block. */ - /*******************************************************************/ - CurNum = 0; - - next: - if (++CurBlk >= Block) - return RC_EF; - - /*******************************************************************/ - /* Before reading a new block, check whether block optimization */ - /* can be done, as well as for join as for local filtering. */ - /*******************************************************************/ - if (Optimized) switch (Tdbp->TestBlock(g)) { - case RC_EF: - return RC_EF; - case RC_NF: - goto next; - } // endswitch rc - - } // endif's - - if (OldBlk == CurBlk) - goto ok; // Block is already there - - if (Optimized) { - // Store the position of next block - Fpos = BlkPos[CurBlk]; - - // fseek is required only in non sequential reading - if (CurBlk != OldBlk + 1) - if (fseek(Stream, Fpos, SEEK_SET)) { - sprintf(g->Message, MSG(FSETPOS_ERROR), Fpos); - return RC_FX; - } // endif fseek - - // Calculate the length of block to read - BlkLen = BlkPos[CurBlk + 1] - Fpos; - rdbuf = Zlenp; - } else { // !Optimized - if (CurBlk != OldBlk + 1) { - strcpy(g->Message, MSG(INV_RAND_ACC)); - return RC_FX; - } else - Fpos = ftell(Stream); // Used when optimizing - - // Get the stored length from the file itself - if (fread(Zlenp, sizeof(int), 1, Stream) != 1) { - if (feof(Stream)) - return RC_EF; - - goto err; - } // endif fread - - BlkLen = *Zlenp; - rdbuf = Zbuffer; - } // endif Optimized - - // Read the next block - switch (ReadCompressedBuffer(g, rdbuf)) { - case RC_FX: goto err; - case RC_NF: return RC_FX; - case RC_EF: return RC_EF; - default: Rbuf = (CurBlk == Block - 1) ? Last : Nrec; - } // endswitch ReadCompressedBuffer - - ok: - if (Tdbp->GetFtype() == RECFM_VAR) { - int i; - - // Get the position of the current line - for (i = 0, CurLine = To_Buf; i < CurNum; i++) - while (*CurLine++ != '\n') ; // What about Unix ??? - - // Now get the position of the next line - for (NxtLine = CurLine; *NxtLine++ != '\n';) ; - - // Set caller line buffer - n = NxtLine - CurLine - Ending; - } else { - CurLine = To_Buf + CurNum * Lrecl; - NxtLine = CurLine + Lrecl; - n = Lrecl - ((Tdbp->GetFtype() == RECFM_BIN) ? 0 : Ending); - } // endif Ftype - - memcpy(Tdbp->GetLine(), CurLine, n); - Tdbp->GetLine()[n] = '\0'; - - OldBlk = CurBlk; // Last block actually read - IsRead = TRUE; // Is read indeed - return RC_OK; - - err: -#if defined(UNIX) - sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(errno)); -#else - sprintf(g->Message, MSG(READ_ERROR), To_File, _strerror(NULL)); -#endif - return RC_FX; - } // end of ReadBuffer +int ZPXFAM::GetFileLength(PGLOBAL g) +{ + int len; -/***********************************************************************/ -/* Read and decompress a block from the stream. */ -/***********************************************************************/ -int ZLBFAM::ReadCompressedBuffer(PGLOBAL g, void *rdbuf) - { - if (fread(rdbuf, 1, (size_t)BlkLen, Stream) == (unsigned)BlkLen) { - int zrc; - - num_read++; - - if (Optimized && BlkLen != signed(*Zlenp + sizeof(int))) { - sprintf(g->Message, MSG(BAD_BLK_SIZE), CurBlk + 1); - return RC_NF; - } // endif BlkLen - - // HERE WE MUST INFLATE THE BLOCK - Zstream->next_in = Zbuffer; - Zstream->avail_in = (uInt)(*Zlenp); - Zstream->next_out = (Byte*)To_Buf; - Zstream->avail_out = Buflen; - zrc = inflate(Zstream, Z_SYNC_FLUSH); - - if (zrc != Z_OK) { - if (Zstream->msg) - sprintf(g->Message, MSG(FUNC_ERR_S), "inflate", Zstream->msg); - else - sprintf(g->Message, MSG(FUNCTION_ERROR), "inflate", (int)zrc); - - return RC_NF; - } // endif zrc - - } else if (feof(Stream)) { - return RC_EF; - } else - return RC_FX; - - return RC_OK; - } // end of ReadCompressedBuffer + if (!zutp && OpenTableFile(g)) + return 0; -/***********************************************************************/ -/* WriteBuffer: File write routine for DOS access method. */ -/* Update is directly written back into the file, */ -/* with this (fast) method, record size cannot change. */ -/***********************************************************************/ -int ZLBFAM::WriteBuffer(PGLOBAL g) - { - assert (Tdbp->GetMode() == MODE_INSERT); - - /*********************************************************************/ - /* Prepare the write buffer. */ - /*********************************************************************/ - if (!Closing) { - if (Tdbp->GetFtype() == RECFM_BIN) - memcpy(CurLine, Tdbp->GetLine(), Lrecl); - else - strcat(strcpy(CurLine, Tdbp->GetLine()), CrLf); - -#if defined(_DEBUG) - if (Tdbp->GetFtype() == RECFM_FIX && - (signed)strlen(CurLine) != Lrecl + (signed)strlen(CrLf)) { - strcpy(g->Message, MSG(BAD_LINE_LEN)); - Closing = TRUE; - return RC_FX; - } // endif Lrecl -#endif // _DEBUG - } // endif Closing - - /*********************************************************************/ - /* In Insert mode, blocs are added sequentialy to the file end. */ - /*********************************************************************/ - if (++CurNum != Rbuf) { - if (Tdbp->GetFtype() == RECFM_VAR) - CurLine += strlen(CurLine); - else - CurLine += Lrecl; - - return RC_OK; // We write only full blocks - } // endif CurNum - - // HERE WE MUST DEFLATE THE BLOCK - if (Tdbp->GetFtype() == RECFM_VAR) - NxtLine = CurLine + strlen(CurLine); - else - NxtLine = CurLine + Lrecl; - - BlkLen = NxtLine - To_Buf; - - if (WriteCompressedBuffer(g)) { - Closing = TRUE; // To tell CloseDB about a Write error - return RC_FX; - } // endif WriteCompressedBuffer - - CurBlk++; - CurNum = 0; - CurLine = To_Buf; - return RC_OK; - } // end of WriteBuffer + if (zutp->entryopen) + len = zutp->size; + else + len = 0; + + return len; +} // end of GetFileLength /***********************************************************************/ -/* Compress the buffer and write the deflated output to stream. */ +/* ZIP Cardinality: return the number of rows if possible. */ /***********************************************************************/ -bool ZLBFAM::WriteCompressedBuffer(PGLOBAL g) - { - int zrc; - - Zstream->next_in = (Byte*)To_Buf; - Zstream->avail_in = (uInt)BlkLen; - Zstream->next_out = Zbuffer; - Zstream->avail_out = Buflen + 16; - Zstream->total_out = 0; - zrc = deflate(Zstream, Z_FULL_FLUSH); - - if (zrc != Z_OK) { - if (Zstream->msg) - sprintf(g->Message, MSG(FUNC_ERR_S), "deflate", Zstream->msg); - else - sprintf(g->Message, MSG(FUNCTION_ERROR), "deflate", (int)zrc); - - return TRUE; - } else - *Zlenp = Zstream->total_out; - - // Now start the writing process. - BlkLen = *Zlenp + sizeof(int); - - if (fwrite(Zlenp, 1, BlkLen, Stream) != (size_t)BlkLen) { - sprintf(g->Message, MSG(FWRITE_ERROR), strerror(errno)); - return TRUE; - } // endif size - - return FALSE; - } // end of WriteCompressedBuffer +int ZPXFAM::Cardinality(PGLOBAL g) +{ + if (!g) + return 1; + + int card = -1; + int len = GetFileLength(g); + + if (!(len % Lrecl)) + card = len / (int)Lrecl; // Fixed length file + else + sprintf(g->Message, MSG(NOT_FIXED_LEN), zutp->fn, len, Lrecl); + + // Set number of blocks for later use + Block = (card > 0) ? (card + Nrec - 1) / Nrec : 0; + return card; +} // end of Cardinality /***********************************************************************/ -/* Table file close routine for DOS access method. */ +/* OpenTableFile: Open a DOS/UNIX table file from a ZIP file. */ /***********************************************************************/ -void ZLBFAM::CloseTableFile(PGLOBAL g, bool) - { - int rc = RC_OK; - - if (Tdbp->GetMode() == MODE_INSERT) { - LPCSTR name = Tdbp->GetName(); - PDOSDEF defp = (PDOSDEF)Tdbp->GetDef(); - - // Closing is True if last Write was in error - if (CurNum && !Closing) { - // Some more inserted lines remain to be written - Last = (Nrec - Rbuf) + CurNum; - Block = CurBlk + 1; - Rbuf = CurNum--; - Closing = TRUE; - rc = WriteBuffer(g); - } else if (Rbuf == Nrec) { - Last = Nrec; - Block = CurBlk; - } // endif CurNum - - if (rc != RC_FX) { - defp->SetBlock(Block); - defp->SetLast(Last); - defp->SetIntCatInfo("Blocks", Block); - defp->SetIntCatInfo("Last", Last); - } // endif - - fclose(Stream); - } else - rc = fclose(Stream); - - if (trace) - htrc("ZLB CloseTableFile: closing %s mode=%d rc=%d\n", - To_File, Tdbp->GetMode(), rc); - - Stream = NULL; // So we can know whether table is open - To_Fb->Count = 0; // Avoid double closing by PlugCloseAll - - if (Tdbp->GetMode() == MODE_READ) - rc = inflateEnd(Zstream); - else - rc = deflateEnd(Zstream); - - } // end of CloseTableFile +bool ZPXFAM::OpenTableFile(PGLOBAL g) +{ + // May have been already opened in GetFileLength + if (!zutp || !zutp->zipfile) { + char filename[_MAX_PATH]; + MODE mode = Tdbp->GetMode(); + + /*********************************************************************/ + /* Allocate the ZIP utility class. */ + /*********************************************************************/ + if (!zutp) + zutp = new(g)ZIPUTIL(target, mul); + + // We used the file name relative to recorded datapath + PlugSetPath(filename, To_File, Tdbp->GetPath()); + + if (!zutp->OpenTable(g, mode, filename)) { + // The pseudo "buffer" is here the entire real buffer + Memory = zutp->memory; + Fpos = Mempos = Memory + Headlen; + Top = Memory + zutp->size; + To_Fb = zutp->fp; // Useful when closing + } else + return true; + + } else + Reset(); + + return false; +} // end of OpenTableFile /***********************************************************************/ -/* Rewind routine for ZLIB access method. */ +/* GetNext: go to next entry. */ /***********************************************************************/ -void ZLBFAM::Rewind(void) - { - // We must be positioned after the header block - if (CurBlk >= 0) { // Nothing to do if no block read yet - if (!Optimized) { // If optimized, fseek will be done in ReadBuffer - size_t st; - - rewind(Stream); +int ZPXFAM::GetNext(PGLOBAL g) +{ + int rc = zutp->nextEntry(g); - if (!(st = fread(Zlenp, sizeof(int), 1, Stream)) && trace) - htrc("fread error %d in Rewind", errno); + if (rc != RC_OK) + return rc; - fseek(Stream, *Zlenp + sizeof(int), SEEK_SET); - OldBlk = -1; - } // endif Optimized + int len = zutp->size; - CurBlk = -1; - CurNum = Rbuf; - } // endif CurBlk + if (len % Lrecl) { + sprintf(g->Message, MSG(NOT_FIXED_LEN), zutp->fn, len, Lrecl); + return RC_FX; + } // endif size -//OldBlk = -1; -//Rbuf = 0; commented out in case we reuse last read block - } // end of Rewind + Memory = zutp->memory; + Top = Memory + len; + Rewind(); + return RC_OK; +} // end of GetNext -/* ------------------------ End of ZipFam ---------------------------- */ diff --git a/storage/connect/filamzip.h b/storage/connect/filamzip.h index edb8b5db323..9312fb2f70e 100644 --- a/storage/connect/filamzip.h +++ b/storage/connect/filamzip.h @@ -1,170 +1,117 @@ -/************** FilAmZip H Declares Source Code File (.H) **************/ -/* Name: FILAMZIP.H Version 1.2 */ +/************** filamzip H Declares Source Code File (.H) **************/ +/* Name: filamzip.h Version 1.0 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 2005-2014 */ +/* (C) Copyright to the author Olivier BERTRAND 2016 */ /* */ -/* This file contains the GZIP access method classes declares. */ +/* This file contains the ZIP file access method classes declares. */ /***********************************************************************/ #ifndef __FILAMZIP_H #define __FILAMZIP_H -#include "zlib.h" +#include "block.h" +#include "filamap.h" +#include "unzip.h" -typedef class ZIPFAM *PZIPFAM; -typedef class ZBKFAM *PZBKFAM; -typedef class ZIXFAM *PZIXFAM; -typedef class ZLBFAM *PZLBFAM; +#define DLLEXPORT extern "C" -/***********************************************************************/ -/* This is the access method class declaration for not optimized */ -/* variable record length files compressed using the gzip library */ -/* functions. File is accessed record by record (row). */ -/***********************************************************************/ -class DllExport ZIPFAM : public TXTFAM { -// friend class DOSCOL; - public: - // Constructor - ZIPFAM(PDOSDEF tdp) : TXTFAM(tdp) {Zfile = NULL; Zpos = 0;} - ZIPFAM(PZIPFAM txfp); - - // Implementation - virtual AMT GetAmType(void) {return TYPE_AM_ZIP;} - virtual int GetPos(void); - virtual int GetNextPos(void); - virtual PTXF Duplicate(PGLOBAL g) - {return (PTXF)new(g) ZIPFAM(this);} - - // Methods - virtual void Reset(void); - virtual int GetFileLength(PGLOBAL g); - virtual int Cardinality(PGLOBAL g) {return (g) ? -1 : 0;} - virtual int MaxBlkSize(PGLOBAL g, int s) {return s;} - virtual bool AllocateBuffer(PGLOBAL g); - virtual int GetRowID(void); - virtual bool RecordPos(PGLOBAL g); - virtual bool SetPos(PGLOBAL g, int recpos); - virtual int SkipRecord(PGLOBAL g, bool header); - virtual bool OpenTableFile(PGLOBAL g); - virtual int ReadBuffer(PGLOBAL g); - virtual int WriteBuffer(PGLOBAL g); - virtual int DeleteRecords(PGLOBAL g, int irc); - virtual void CloseTableFile(PGLOBAL g, bool abort); - virtual void Rewind(void); - - protected: - int Zerror(PGLOBAL g); // GZ error function - - // Members - gzFile Zfile; // Points to GZ file structure - z_off_t Zpos; // Uncompressed file position - }; // end of class ZIPFAM +typedef class ZIPFAM *PZIPFAM; +typedef class ZPXFAM *PZPXFAM; /***********************************************************************/ -/* This is the access method class declaration for optimized variable */ -/* record length files compressed using the gzip library functions. */ -/* The File is accessed by block (requires an opt file). */ +/* This is the ZIP utility fonctions class. */ /***********************************************************************/ -class DllExport ZBKFAM : public ZIPFAM { - public: - // Constructor - ZBKFAM(PDOSDEF tdp); - ZBKFAM(PZBKFAM txfp); - - // Implementation - virtual int GetPos(void); - virtual int GetNextPos(void) {return 0;} - virtual PTXF Duplicate(PGLOBAL g) - {return (PTXF)new(g) ZBKFAM(this);} - - // Methods - virtual int Cardinality(PGLOBAL g); - virtual int MaxBlkSize(PGLOBAL g, int s); - virtual bool AllocateBuffer(PGLOBAL g); - virtual int GetRowID(void); - virtual bool RecordPos(PGLOBAL g); - virtual int SkipRecord(PGLOBAL g, bool header); - virtual int ReadBuffer(PGLOBAL g); - virtual int WriteBuffer(PGLOBAL g); - virtual int DeleteRecords(PGLOBAL g, int irc); - virtual void CloseTableFile(PGLOBAL g, bool abort); - virtual void Rewind(void); - - protected: - // Members - char *CurLine; // Position of current line in buffer - char *NxtLine; // Position of Next line in buffer - bool Closing; // True when closing on Insert - }; // end of class ZBKFAM +class DllExport ZIPUTIL : public BLOCK { +public: + // Constructor + ZIPUTIL(PSZ tgt, bool mul); +//ZIPUTIL(ZIPUTIL *zutp); + + // Implementation +//PTXF Duplicate(PGLOBAL g) { return (PTXF) new(g)ZIPFAM(this); } + + // Methods + virtual bool OpenTable(PGLOBAL g, MODE mode, char *fn); + bool open(PGLOBAL g, char *fn); + bool openEntry(PGLOBAL g); + void close(void); + void closeEntry(void); + bool WildMatch(PSZ pat, PSZ str); + int findEntry(PGLOBAL g, bool next); + int nextEntry(PGLOBAL g); + + // Members + unzFile zipfile; // The ZIP container file + PSZ target; // The target file name + unz_file_info finfo; // The current file info + PFBLOCK fp; + char *memory; + uint size; + int multiple; // Multiple targets + bool entryopen; // True when open current entry + char fn[FILENAME_MAX]; // The current entry file name + char mapCaseTable[256]; +}; // end of ZIPFAM /***********************************************************************/ -/* This is the access method class declaration for fixed record */ -/* length files compressed using the gzip library functions. */ -/* The file is always accessed by block. */ +/* This is the ZIP file access method. */ /***********************************************************************/ -class DllExport ZIXFAM : public ZBKFAM { - public: - // Constructor - ZIXFAM(PDOSDEF tdp); - ZIXFAM(PZIXFAM txfp) : ZBKFAM(txfp) {} - - // Implementation - virtual int GetNextPos(void) {return 0;} - virtual PTXF Duplicate(PGLOBAL g) - {return (PTXF)new(g) ZIXFAM(this);} - - // Methods - virtual int Cardinality(PGLOBAL g); - virtual bool AllocateBuffer(PGLOBAL g); - virtual int ReadBuffer(PGLOBAL g); - virtual int WriteBuffer(PGLOBAL g); - - protected: - // No additional Members - }; // end of class ZIXFAM +class DllExport ZIPFAM : public MAPFAM { + friend class ZPXFAM; +public: + // Constructors + ZIPFAM(PDOSDEF tdp); + ZIPFAM(PZIPFAM txfp); + ZIPFAM(PDOSDEF tdp, PZPXFAM txfp); + + // Implementation + virtual AMT GetAmType(void) { return TYPE_AM_ZIP; } + virtual PTXF Duplicate(PGLOBAL g) { return (PTXF) new(g)ZIPFAM(this); } + + // Methods + virtual int Cardinality(PGLOBAL g); + virtual int GetFileLength(PGLOBAL g); +//virtual int MaxBlkSize(PGLOBAL g, int s) {return s;} + virtual bool OpenTableFile(PGLOBAL g); + virtual bool DeferReading(void) { return false; } + virtual int GetNext(PGLOBAL g); +//virtual int ReadBuffer(PGLOBAL g); +//virtual int WriteBuffer(PGLOBAL g); +//virtual int DeleteRecords(PGLOBAL g, int irc); +//virtual void CloseTableFile(PGLOBAL g, bool abort); + +protected: + // Members + ZIPUTIL *zutp; + PSZ target; + bool mul; +}; // end of ZIPFAM /***********************************************************************/ -/* This is the DOS/UNIX Access Method class declaration for PlugDB */ -/* fixed/variable files compressed using the zlib library functions. */ -/* Physically these are written and read using the same technique */ -/* than blocked variable files, only the contain of each block is */ -/* compressed using the deflate zlib function. The purpose of this */ -/* specific format is to have a fast mechanism for direct access of */ -/* records so blocked optimization is fast and direct access (joins) */ -/* is allowed. Note that the block length is written ahead of each */ -/* block to enable reading when optimization file is not available. */ +/* This is the fixed ZIP file access method. */ /***********************************************************************/ -class DllExport ZLBFAM : public BLKFAM { - public: - // Constructor - ZLBFAM(PDOSDEF tdp); - ZLBFAM(PZLBFAM txfp); - - // Implementation - virtual AMT GetAmType(void) {return TYPE_AM_ZLIB;} - virtual int GetPos(void); - virtual int GetNextPos(void); - virtual PTXF Duplicate(PGLOBAL g) - {return (PTXF)new(g) ZLBFAM(this);} - inline void SetOptimized(bool b) {Optimized = b;} - - // Methods - virtual int GetFileLength(PGLOBAL g); - virtual bool SetPos(PGLOBAL g, int recpos); - virtual bool AllocateBuffer(PGLOBAL g); - virtual int ReadBuffer(PGLOBAL g); - virtual int WriteBuffer(PGLOBAL g); - virtual void CloseTableFile(PGLOBAL g, bool abort); - virtual void Rewind(void); - - protected: - bool WriteCompressedBuffer(PGLOBAL g); - int ReadCompressedBuffer(PGLOBAL g, void *rdbuf); - - // Members - z_streamp Zstream; // Compression/decompression stream - Byte *Zbuffer; // Compressed block buffer - int *Zlenp; // Pointer to block length - bool Optimized; // true when opt file is available - }; // end of class ZLBFAM +class DllExport ZPXFAM : public MPXFAM { + friend class ZIPFAM; +public: + // Constructors + ZPXFAM(PDOSDEF tdp); + ZPXFAM(PZPXFAM txfp); + + // Implementation + virtual AMT GetAmType(void) { return TYPE_AM_ZIP; } + virtual PTXF Duplicate(PGLOBAL g) { return (PTXF) new(g)ZPXFAM(this); } + + // Methods + virtual int GetFileLength(PGLOBAL g); + virtual int Cardinality(PGLOBAL g); + virtual bool OpenTableFile(PGLOBAL g); + virtual int GetNext(PGLOBAL g); +//virtual int ReadBuffer(PGLOBAL g); + +protected: + // Members + ZIPUTIL *zutp; + PSZ target; + bool mul; +}; // end of ZPXFAM #endif // __FILAMZIP_H diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index 419a33ed74e..b542ca180c5 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -171,9 +171,9 @@ #define JSONMAX 10 // JSON Default max grp size extern "C" { - char version[]= "Version 1.04.0008 August 10, 2016"; + char version[]= "Version 1.05.0001 December 13, 2016"; #if defined(__WIN__) - char compver[]= "Version 1.04.0008 " __DATE__ " " __TIME__; + char compver[]= "Version 1.05.0001 " __DATE__ " " __TIME__; char slash= '\\'; #else // !__WIN__ char slash= '/'; @@ -512,13 +512,13 @@ ha_create_table_option connect_table_option_list[]= HA_TOPTION_NUMBER("QUOTED", quoted, (ulonglong) -1, 0, 3, 1), HA_TOPTION_NUMBER("ENDING", ending, (ulonglong) -1, 0, INT_MAX32, 1), HA_TOPTION_NUMBER("COMPRESS", compressed, 0, 0, 2, 1), -//HA_TOPTION_BOOL("COMPRESS", compressed, 0), HA_TOPTION_BOOL("MAPPED", mapped, 0), HA_TOPTION_BOOL("HUGE", huge, 0), HA_TOPTION_BOOL("SPLIT", split, 0), HA_TOPTION_BOOL("READONLY", readonly, 0), HA_TOPTION_BOOL("SEPINDEX", sepindex, 0), - HA_TOPTION_END + HA_TOPTION_BOOL("ZIPPED", zipped, 0), + HA_TOPTION_END }; @@ -532,7 +532,6 @@ ha_create_table_option connect_field_option_list[]= { HA_FOPTION_NUMBER("FLAG", offset, (ulonglong) -1, 0, INT_MAX32, 1), HA_FOPTION_NUMBER("MAX_DIST", freq, 0, 0, INT_MAX32, 1), // BLK_INDX -//HA_FOPTION_NUMBER("DISTRIB", opt, 0, 0, 2, 1), // used for BLK_INDX HA_FOPTION_NUMBER("FIELD_LENGTH", fldlen, 0, 0, INT_MAX32, 1), HA_FOPTION_STRING("DATE_FORMAT", dateformat), HA_FOPTION_STRING("FIELD_FORMAT", fieldformat), @@ -678,7 +677,6 @@ static int connect_init_func(void *p) connect_hton= (handlerton *)p; connect_hton->state= SHOW_OPTION_YES; connect_hton->create= connect_create_handler; -//connect_hton->flags= HTON_TEMPORARY_NOT_SUPPORTED | HTON_NO_PARTITION; connect_hton->flags= HTON_TEMPORARY_NOT_SUPPORTED; connect_hton->table_options= connect_table_option_list; connect_hton->field_options= connect_field_option_list; @@ -1135,7 +1133,9 @@ bool GetBooleanTableOption(PGLOBAL g, PTOS options, char *opname, bool bdef) opval= options->sepindex; else if (!stricmp(opname, "Header")) opval= (options->header != 0); // Is Boolean for some table types - else if (options->oplist) + else if (!stricmp(opname, "Zipped")) + opval = options->zipped; + else if (options->oplist) if ((pv= GetListOption(g, opname, options->oplist))) opval= (!*pv || *pv == 'y' || *pv == 'Y' || atoi(pv) != 0); @@ -1242,8 +1242,10 @@ char *ha_connect::GetStringOption(char *opname, char *sdef) if (opval && (!stricmp(opname, "connect") || !stricmp(opname, "tabname") - || !stricmp(opname, "filename"))) - opval = GetRealString(opval); + || !stricmp(opname, "filename") + || !stricmp(opname, "optname") + || !stricmp(opname, "entry"))) + opval = GetRealString(opval); if (!opval) { if (sdef && !strcmp(sdef, "*")) { @@ -4164,7 +4166,8 @@ bool ha_connect::check_privileges(THD *thd, PTOS options, char *dbn, bool quick) case TAB_DIR: case TAB_MAC: case TAB_WMI: - case TAB_OEM: + case TAB_ZIP: + case TAB_OEM: #ifdef NO_EMBEDDED_ACCESS_CHECKS return false; #endif @@ -5172,13 +5175,13 @@ static int connect_assisted_discovery(handlerton *, THD* thd, char v=0, spc= ',', qch= 0; const char *fncn= "?"; const char *user, *fn, *db, *host, *pwd, *sep, *tbl, *src; - const char *col, *ocl, *rnk, *pic, *fcl, *skc; + const char *col, *ocl, *rnk, *pic, *fcl, *skc, *zfn; char *tab, *dsn, *shm, *dpath; #if defined(__WIN__) char *nsp= NULL, *cls= NULL; #endif // __WIN__ - int port= 0, hdr= 0, mxr= 0, mxe= 0, rc= 0; - int cop __attribute__((unused))= 0, lrecl= 0; +//int hdr, mxe; + int port = 0, mxr = 0, rc = 0, mul = 0, lrecl = 0; #if defined(ODBC_SUPPORT) POPARM sop= NULL; char *ucnc= NULL; @@ -5189,7 +5192,8 @@ static int connect_assisted_discovery(handlerton *, THD* thd, PJPARM sjp= NULL; char *driver= NULL; char *url= NULL; - char *tabtyp = NULL; +//char *prop= NULL; + char *tabtyp= NULL; #endif // JDBC_SUPPORT uint tm, fnc= FNC_NO, supfnc= (FNC_NO | FNC_COL); bool bif, ok= false, dbf= false; @@ -5209,7 +5213,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, if (!g) return HA_ERR_INTERNAL_ERROR; - user= host= pwd= tbl= src= col= ocl= pic= fcl= skc= rnk= dsn= NULL; + user= host= pwd= tbl= src= col= ocl= pic= fcl= skc= rnk= zfn= dsn= NULL; // Get the useful create options ttp= GetTypeID(topt->type); @@ -5222,7 +5226,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, sep= topt->separator; spc= (!sep) ? ',' : *sep; qch= topt->qchar ? *topt->qchar : (signed)topt->quoted >= 0 ? '"' : 0; - hdr= (int)topt->header; + mul = (int)topt->multiple; tbl= topt->tablist; col= topt->colist; @@ -5255,13 +5259,16 @@ static int connect_assisted_discovery(handlerton *, THD* thd, #if defined(JDBC_SUPPORT) driver= GetListOption(g, "Driver", topt->oplist, NULL); // url= GetListOption(g, "URL", topt->oplist, NULL); +// prop = GetListOption(g, "Properties", topt->oplist, NULL); tabtyp = GetListOption(g, "Tabtype", topt->oplist, NULL); #endif // JDBC_SUPPORT - mxe= atoi(GetListOption(g,"maxerr", topt->oplist, "0")); #if defined(PROMPT_OK) cop= atoi(GetListOption(g, "checkdsn", topt->oplist, "0")); #endif // PROMPT_OK - } else { +#if defined(ZIP_SUPPORT) + zfn = GetListOption(g, "Zipfile", topt->oplist, NULL); +#endif // ZIP_SUPPORT + } else { host= "localhost"; user= (ttp == TAB_ODBC ? NULL : "root"); } // endif option_list @@ -5365,6 +5372,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, jdef->SetName(create_info->alias); sjp= (PJPARM)PlugSubAlloc(g, NULL, sizeof(JDBCPARM)); sjp->Driver= driver; +// sjp->Properties = prop; sjp->Fsize= 0; sjp->Scrollable= false; @@ -5467,7 +5475,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, case TAB_XML: #endif // LIBXML2_SUPPORT || DOMDOC_SUPPORT case TAB_JSON: - if (!fn) + if (!fn && !zfn && !mul) sprintf(g->Message, "Missing %s file name", topt->type); else ok= true; @@ -5581,7 +5589,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, NULL, port, fnc == FNC_COL); break; case TAB_CSV: - qrp= CSVColumns(g, dpath, fn, spc, qch, hdr, mxe, fnc == FNC_COL); + qrp = CSVColumns(g, dpath, topt, fnc == FNC_COL); break; #if defined(__WIN__) case TAB_WMI: @@ -6946,7 +6954,7 @@ maria_declare_plugin(connect) 0x0104, /* version number (1.04) */ NULL, /* status variables */ connect_system_variables, /* system variables */ - "1.04.0008", /* string version */ + "1.05.0001", /* string version */ MariaDB_PLUGIN_MATURITY_GAMMA /* maturity */ } maria_declare_plugin_end; diff --git a/storage/connect/ha_connect.h b/storage/connect/ha_connect.h index 60194ac0e3c..3d9ff967618 100644 --- a/storage/connect/ha_connect.h +++ b/storage/connect/ha_connect.h @@ -83,42 +83,9 @@ extern handlerton *connect_hton; These can be specified in the CREATE TABLE: CREATE TABLE ( ... ) {...here...} -*/ -#if 0 // moved to mycat.h -typedef struct ha_table_option_struct TOS, *PTOS; - -struct ha_table_option_struct { - const char *type; - const char *filename; - const char *optname; - const char *tabname; - const char *tablist; - const char *dbname; - const char *separator; -//const char *connect; - const char *qchar; - const char *module; - const char *subtype; - const char *catfunc; - const char *srcdef; - const char *colist; - const char *oplist; - const char *data_charset; - ulonglong lrecl; - ulonglong elements; -//ulonglong estimate; - ulonglong multiple; - ulonglong header; - ulonglong quoted; - ulonglong ending; - ulonglong compressed; - bool mapped; - bool huge; - bool split; - bool readonly; - bool sepindex; - }; -#endif // 0 + + ------ Was moved to mycat.h ------ + */ /** structure for CREATE TABLE options (field options) diff --git a/storage/connect/ioapi.c b/storage/connect/ioapi.c new file mode 100644 index 00000000000..7f5c191b2af --- /dev/null +++ b/storage/connect/ioapi.c @@ -0,0 +1,247 @@ +/* ioapi.h -- IO base function header for compress/uncompress .zip + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications for Zip64 support + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + +*/ + +#if defined(_WIN32) && (!(defined(_CRT_SECURE_NO_WARNINGS))) + #define _CRT_SECURE_NO_WARNINGS +#endif + +#if defined(__APPLE__) || defined(IOAPI_NO_64) +// In darwin and perhaps other BSD variants off_t is a 64 bit value, hence no need for specific 64 bit functions +#define FOPEN_FUNC(filename, mode) fopen(filename, mode) +#define FTELLO_FUNC(stream) ftello(stream) +#define FSEEKO_FUNC(stream, offset, origin) fseeko(stream, offset, origin) +#else +#define FOPEN_FUNC(filename, mode) fopen64(filename, mode) +#define FTELLO_FUNC(stream) ftello64(stream) +#define FSEEKO_FUNC(stream, offset, origin) fseeko64(stream, offset, origin) +#endif + + +#include "ioapi.h" + +voidpf call_zopen64 (const zlib_filefunc64_32_def* pfilefunc,const void*filename,int mode) +{ + if (pfilefunc->zfile_func64.zopen64_file != NULL) + return (*(pfilefunc->zfile_func64.zopen64_file)) (pfilefunc->zfile_func64.opaque,filename,mode); + else + { + return (*(pfilefunc->zopen32_file))(pfilefunc->zfile_func64.opaque,(const char*)filename,mode); + } +} + +long call_zseek64 (const zlib_filefunc64_32_def* pfilefunc,voidpf filestream, ZPOS64_T offset, int origin) +{ + if (pfilefunc->zfile_func64.zseek64_file != NULL) + return (*(pfilefunc->zfile_func64.zseek64_file)) (pfilefunc->zfile_func64.opaque,filestream,offset,origin); + else + { + uLong offsetTruncated = (uLong)offset; + if (offsetTruncated != offset) + return -1; + else + return (*(pfilefunc->zseek32_file))(pfilefunc->zfile_func64.opaque,filestream,offsetTruncated,origin); + } +} + +ZPOS64_T call_ztell64 (const zlib_filefunc64_32_def* pfilefunc,voidpf filestream) +{ + if (pfilefunc->zfile_func64.zseek64_file != NULL) + return (*(pfilefunc->zfile_func64.ztell64_file)) (pfilefunc->zfile_func64.opaque,filestream); + else + { + uLong tell_uLong = (*(pfilefunc->ztell32_file))(pfilefunc->zfile_func64.opaque,filestream); + if ((tell_uLong) == MAXU32) + return (ZPOS64_T)-1; + else + return tell_uLong; + } +} + +void fill_zlib_filefunc64_32_def_from_filefunc32(zlib_filefunc64_32_def* p_filefunc64_32,const zlib_filefunc_def* p_filefunc32) +{ + p_filefunc64_32->zfile_func64.zopen64_file = NULL; + p_filefunc64_32->zopen32_file = p_filefunc32->zopen_file; + p_filefunc64_32->zfile_func64.zerror_file = p_filefunc32->zerror_file; + p_filefunc64_32->zfile_func64.zread_file = p_filefunc32->zread_file; + p_filefunc64_32->zfile_func64.zwrite_file = p_filefunc32->zwrite_file; + p_filefunc64_32->zfile_func64.ztell64_file = NULL; + p_filefunc64_32->zfile_func64.zseek64_file = NULL; + p_filefunc64_32->zfile_func64.zclose_file = p_filefunc32->zclose_file; + p_filefunc64_32->zfile_func64.zerror_file = p_filefunc32->zerror_file; + p_filefunc64_32->zfile_func64.opaque = p_filefunc32->opaque; + p_filefunc64_32->zseek32_file = p_filefunc32->zseek_file; + p_filefunc64_32->ztell32_file = p_filefunc32->ztell_file; +} + + + +static voidpf ZCALLBACK fopen_file_func OF((voidpf opaque, const char* filename, int mode)); +static uLong ZCALLBACK fread_file_func OF((voidpf opaque, voidpf stream, void* buf, uLong size)); +static uLong ZCALLBACK fwrite_file_func OF((voidpf opaque, voidpf stream, const void* buf,uLong size)); +static ZPOS64_T ZCALLBACK ftell64_file_func OF((voidpf opaque, voidpf stream)); +static long ZCALLBACK fseek64_file_func OF((voidpf opaque, voidpf stream, ZPOS64_T offset, int origin)); +static int ZCALLBACK fclose_file_func OF((voidpf opaque, voidpf stream)); +static int ZCALLBACK ferror_file_func OF((voidpf opaque, voidpf stream)); + +static voidpf ZCALLBACK fopen_file_func (voidpf opaque, const char* filename, int mode) +{ + FILE* file = NULL; + const char* mode_fopen = NULL; + if ((mode & ZLIB_FILEFUNC_MODE_READWRITEFILTER)==ZLIB_FILEFUNC_MODE_READ) + mode_fopen = "rb"; + else + if (mode & ZLIB_FILEFUNC_MODE_EXISTING) + mode_fopen = "r+b"; + else + if (mode & ZLIB_FILEFUNC_MODE_CREATE) + mode_fopen = "wb"; + + if ((filename!=NULL) && (mode_fopen != NULL)) + file = fopen(filename, mode_fopen); + return file; +} + +static voidpf ZCALLBACK fopen64_file_func (voidpf opaque, const void* filename, int mode) +{ + FILE* file = NULL; + const char* mode_fopen = NULL; + if ((mode & ZLIB_FILEFUNC_MODE_READWRITEFILTER)==ZLIB_FILEFUNC_MODE_READ) + mode_fopen = "rb"; + else + if (mode & ZLIB_FILEFUNC_MODE_EXISTING) + mode_fopen = "r+b"; + else + if (mode & ZLIB_FILEFUNC_MODE_CREATE) + mode_fopen = "wb"; + + if ((filename!=NULL) && (mode_fopen != NULL)) + file = FOPEN_FUNC((const char*)filename, mode_fopen); + return file; +} + + +static uLong ZCALLBACK fread_file_func (voidpf opaque, voidpf stream, void* buf, uLong size) +{ + uLong ret; + ret = (uLong)fread(buf, 1, (size_t)size, (FILE *)stream); + return ret; +} + +static uLong ZCALLBACK fwrite_file_func (voidpf opaque, voidpf stream, const void* buf, uLong size) +{ + uLong ret; + ret = (uLong)fwrite(buf, 1, (size_t)size, (FILE *)stream); + return ret; +} + +static long ZCALLBACK ftell_file_func (voidpf opaque, voidpf stream) +{ + long ret; + ret = ftell((FILE *)stream); + return ret; +} + + +static ZPOS64_T ZCALLBACK ftell64_file_func (voidpf opaque, voidpf stream) +{ + ZPOS64_T ret; + ret = FTELLO_FUNC((FILE *)stream); + return ret; +} + +static long ZCALLBACK fseek_file_func (voidpf opaque, voidpf stream, uLong offset, int origin) +{ + int fseek_origin=0; + long ret; + switch (origin) + { + case ZLIB_FILEFUNC_SEEK_CUR : + fseek_origin = SEEK_CUR; + break; + case ZLIB_FILEFUNC_SEEK_END : + fseek_origin = SEEK_END; + break; + case ZLIB_FILEFUNC_SEEK_SET : + fseek_origin = SEEK_SET; + break; + default: return -1; + } + ret = 0; + if (fseek((FILE *)stream, offset, fseek_origin) != 0) + ret = -1; + return ret; +} + +static long ZCALLBACK fseek64_file_func (voidpf opaque, voidpf stream, ZPOS64_T offset, int origin) +{ + int fseek_origin=0; + long ret; + switch (origin) + { + case ZLIB_FILEFUNC_SEEK_CUR : + fseek_origin = SEEK_CUR; + break; + case ZLIB_FILEFUNC_SEEK_END : + fseek_origin = SEEK_END; + break; + case ZLIB_FILEFUNC_SEEK_SET : + fseek_origin = SEEK_SET; + break; + default: return -1; + } + ret = 0; + + if(FSEEKO_FUNC((FILE *)stream, offset, fseek_origin) != 0) + ret = -1; + + return ret; +} + + +static int ZCALLBACK fclose_file_func (voidpf opaque, voidpf stream) +{ + int ret; + ret = fclose((FILE *)stream); + return ret; +} + +static int ZCALLBACK ferror_file_func (voidpf opaque, voidpf stream) +{ + int ret; + ret = ferror((FILE *)stream); + return ret; +} + +void fill_fopen_filefunc (pzlib_filefunc_def) + zlib_filefunc_def* pzlib_filefunc_def; +{ + pzlib_filefunc_def->zopen_file = fopen_file_func; + pzlib_filefunc_def->zread_file = fread_file_func; + pzlib_filefunc_def->zwrite_file = fwrite_file_func; + pzlib_filefunc_def->ztell_file = ftell_file_func; + pzlib_filefunc_def->zseek_file = fseek_file_func; + pzlib_filefunc_def->zclose_file = fclose_file_func; + pzlib_filefunc_def->zerror_file = ferror_file_func; + pzlib_filefunc_def->opaque = NULL; +} + +void fill_fopen64_filefunc (zlib_filefunc64_def* pzlib_filefunc_def) +{ + pzlib_filefunc_def->zopen64_file = fopen64_file_func; + pzlib_filefunc_def->zread_file = fread_file_func; + pzlib_filefunc_def->zwrite_file = fwrite_file_func; + pzlib_filefunc_def->ztell64_file = ftell64_file_func; + pzlib_filefunc_def->zseek64_file = fseek64_file_func; + pzlib_filefunc_def->zclose_file = fclose_file_func; + pzlib_filefunc_def->zerror_file = ferror_file_func; + pzlib_filefunc_def->opaque = NULL; +} diff --git a/storage/connect/ioapi.h b/storage/connect/ioapi.h new file mode 100644 index 00000000000..4fa73002053 --- /dev/null +++ b/storage/connect/ioapi.h @@ -0,0 +1,209 @@ +/* ioapi.h -- IO base function header for compress/uncompress .zip + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications for Zip64 support + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + + Changes + + Oct-2009 - Defined ZPOS64_T to fpos_t on windows and u_int64_t on linux. (might need to find a better why for this) + Oct-2009 - Change to fseeko64, ftello64 and fopen64 so large files would work on linux. + More if/def section may be needed to support other platforms + Oct-2009 - Defined fxxxx64 calls to normal fopen/ftell/fseek so they would compile on windows. + (but you should use iowin32.c for windows instead) + +*/ + +#ifndef _ZLIBIOAPI64_H +#define _ZLIBIOAPI64_H + +#if (!defined(_WIN32)) && (!defined(WIN32)) && (!defined(__APPLE__)) + + // Linux needs this to support file operation on files larger then 4+GB + // But might need better if/def to select just the platforms that needs them. + + #ifndef __USE_FILE_OFFSET64 + #define __USE_FILE_OFFSET64 + #endif + #ifndef __USE_LARGEFILE64 + #define __USE_LARGEFILE64 + #endif + #ifndef _LARGEFILE64_SOURCE + #define _LARGEFILE64_SOURCE + #endif + #ifndef _FILE_OFFSET_BIT + #define _FILE_OFFSET_BIT 64 + #endif + +#endif + +#include <stdio.h> +#include <stdlib.h> +#include "zlib.h" + +#if defined(USE_FILE32API) +#define fopen64 fopen +#define ftello64 ftell +#define fseeko64 fseek +#else +#ifdef __FreeBSD__ +#define fopen64 fopen +#define ftello64 ftello +#define fseeko64 fseeko +#endif +#ifdef _MSC_VER + #define fopen64 fopen + #if (_MSC_VER >= 1400) && (!(defined(NO_MSCVER_FILE64_FUNC))) + #define ftello64 _ftelli64 + #define fseeko64 _fseeki64 + #else // old MSC + #define ftello64 ftell + #define fseeko64 fseek + #endif +#endif +#endif + +/* +#ifndef ZPOS64_T + #ifdef _WIN32 + #define ZPOS64_T fpos_t + #else + #include <stdint.h> + #define ZPOS64_T uint64_t + #endif +#endif +*/ + +#ifdef HAVE_MINIZIP64_CONF_H +#include "mz64conf.h" +#endif + +/* a type choosen by DEFINE */ +#ifdef HAVE_64BIT_INT_CUSTOM +typedef 64BIT_INT_CUSTOM_TYPE ZPOS64_T; +#else +#ifdef HAS_STDINT_H +#include "stdint.h" +typedef uint64_t ZPOS64_T; +#else + +/* Maximum unsigned 32-bit value used as placeholder for zip64 */ +#define MAXU32 0xffffffff + +#if defined(_MSC_VER) || defined(__BORLANDC__) +typedef unsigned __int64 ZPOS64_T; +#else +typedef unsigned long long int ZPOS64_T; +#endif +#endif +#endif + + + +#ifdef __cplusplus +extern "C" { +#endif + + +#define ZLIB_FILEFUNC_SEEK_CUR (1) +#define ZLIB_FILEFUNC_SEEK_END (2) +#define ZLIB_FILEFUNC_SEEK_SET (0) + +#define ZLIB_FILEFUNC_MODE_READ (1) +#define ZLIB_FILEFUNC_MODE_WRITE (2) +#define ZLIB_FILEFUNC_MODE_READWRITEFILTER (3) + +#define ZLIB_FILEFUNC_MODE_EXISTING (4) +#define ZLIB_FILEFUNC_MODE_CREATE (8) + + +#ifndef ZCALLBACK + #if (defined(WIN32) || defined(_WIN32) || defined (WINDOWS) || defined (_WINDOWS)) && defined(CALLBACK) && defined (USEWINDOWS_CALLBACK) + #define ZCALLBACK CALLBACK + #else + #define ZCALLBACK + #endif +#endif + +#ifndef OF +#define OF(args) args +#endif + +typedef voidpf (ZCALLBACK *open_file_func) OF((voidpf opaque, const char* filename, int mode)); +typedef uLong (ZCALLBACK *read_file_func) OF((voidpf opaque, voidpf stream, void* buf, uLong size)); +typedef uLong (ZCALLBACK *write_file_func) OF((voidpf opaque, voidpf stream, const void* buf, uLong size)); +typedef int (ZCALLBACK *close_file_func) OF((voidpf opaque, voidpf stream)); +typedef int (ZCALLBACK *testerror_file_func) OF((voidpf opaque, voidpf stream)); + +typedef long (ZCALLBACK *tell_file_func) OF((voidpf opaque, voidpf stream)); +typedef long (ZCALLBACK *seek_file_func) OF((voidpf opaque, voidpf stream, uLong offset, int origin)); + + +/* here is the "old" 32 bits structure structure */ +typedef struct zlib_filefunc_def_s +{ + open_file_func zopen_file; + read_file_func zread_file; + write_file_func zwrite_file; + tell_file_func ztell_file; + seek_file_func zseek_file; + close_file_func zclose_file; + testerror_file_func zerror_file; + voidpf opaque; +} zlib_filefunc_def; + +typedef ZPOS64_T (ZCALLBACK *tell64_file_func) OF((voidpf opaque, voidpf stream)); +typedef long (ZCALLBACK *seek64_file_func) OF((voidpf opaque, voidpf stream, ZPOS64_T offset, int origin)); +typedef voidpf (ZCALLBACK *open64_file_func) OF((voidpf opaque, const void* filename, int mode)); + +typedef struct zlib_filefunc64_def_s +{ + open64_file_func zopen64_file; + read_file_func zread_file; + write_file_func zwrite_file; + tell64_file_func ztell64_file; + seek64_file_func zseek64_file; + close_file_func zclose_file; + testerror_file_func zerror_file; + voidpf opaque; +} zlib_filefunc64_def; + +void fill_fopen64_filefunc OF((zlib_filefunc64_def* pzlib_filefunc_def)); +void fill_fopen_filefunc OF((zlib_filefunc_def* pzlib_filefunc_def)); + +/* now internal definition, only for zip.c and unzip.h */ +typedef struct zlib_filefunc64_32_def_s +{ + zlib_filefunc64_def zfile_func64; + open_file_func zopen32_file; + tell_file_func ztell32_file; + seek_file_func zseek32_file; +} zlib_filefunc64_32_def; + + +#define ZREAD64(filefunc,filestream,buf,size) ((*((filefunc).zfile_func64.zread_file)) ((filefunc).zfile_func64.opaque,filestream,buf,size)) +#define ZWRITE64(filefunc,filestream,buf,size) ((*((filefunc).zfile_func64.zwrite_file)) ((filefunc).zfile_func64.opaque,filestream,buf,size)) +//#define ZTELL64(filefunc,filestream) ((*((filefunc).ztell64_file)) ((filefunc).opaque,filestream)) +//#define ZSEEK64(filefunc,filestream,pos,mode) ((*((filefunc).zseek64_file)) ((filefunc).opaque,filestream,pos,mode)) +#define ZCLOSE64(filefunc,filestream) ((*((filefunc).zfile_func64.zclose_file)) ((filefunc).zfile_func64.opaque,filestream)) +#define ZERROR64(filefunc,filestream) ((*((filefunc).zfile_func64.zerror_file)) ((filefunc).zfile_func64.opaque,filestream)) + +voidpf call_zopen64 OF((const zlib_filefunc64_32_def* pfilefunc,const void*filename,int mode)); +long call_zseek64 OF((const zlib_filefunc64_32_def* pfilefunc,voidpf filestream, ZPOS64_T offset, int origin)); +ZPOS64_T call_ztell64 OF((const zlib_filefunc64_32_def* pfilefunc,voidpf filestream)); + +void fill_zlib_filefunc64_32_def_from_filefunc32(zlib_filefunc64_32_def* p_filefunc64_32,const zlib_filefunc_def* p_filefunc32); + +#define ZOPEN64(filefunc,filename,mode) (call_zopen64((&(filefunc)),(filename),(mode))) +#define ZTELL64(filefunc,filestream) (call_ztell64((&(filefunc)),(filestream))) +#define ZSEEK64(filefunc,filestream,pos,mode) (call_zseek64((&(filefunc)),(filestream),(pos),(mode))) + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/storage/connect/jdbccat.h b/storage/connect/jdbccat.h index 37f33d7063d..7108aa376ce 100644 --- a/storage/connect/jdbccat.h +++ b/storage/connect/jdbccat.h @@ -8,6 +8,7 @@ typedef struct jdbc_parms { char *Url; // Driver URL char *User; // User connect info char *Pwd; // Password connect info +//char *Properties; // Connection property list //int Cto; // Connect timeout //int Qto; // Query timeout int Fsize; // Fetch size diff --git a/storage/connect/jdbconn.cpp b/storage/connect/jdbconn.cpp index dca9bd0eac4..a69f84a94a1 100644 --- a/storage/connect/jdbconn.cpp +++ b/storage/connect/jdbconn.cpp @@ -55,9 +55,8 @@ #if defined(__WIN__) extern "C" HINSTANCE s_hModule; // Saved module handle -#else // !__WIN__ +#endif // __WIN__ #define nullptr 0 -#endif // !__WIN__ TYPCONV GetTypeConv(); int GetConvSize(); @@ -525,10 +524,10 @@ JDBConn::JDBConn(PGLOBAL g, TDBJDBC *tdbp) m_Wrap = strcat(strcpy(wn, "wrappers/"), m_Wrap); } // endif m_Wrap - m_Driver = NULL; - m_Url = NULL; - m_User = NULL; - m_Pwd = NULL; +//m_Driver = NULL; +//m_Url = NULL; +//m_User = NULL; +//m_Pwd = NULL; m_Ncol = 0; m_Aff = 0; m_Rows = 0; @@ -772,7 +771,7 @@ bool JDBConn::GetJVM(PGLOBAL g) /***********************************************************************/ int JDBConn::Open(PJPARM sop) { - + int irc = RC_FX; bool err = false; jboolean jt = (trace > 0); PGLOBAL& g = m_G; @@ -865,30 +864,37 @@ int JDBConn::Open(PJPARM sop) switch (rc) { case JNI_OK: strcpy(g->Message, "VM successfully created"); + irc = RC_OK; break; case JNI_ERR: strcpy(g->Message, "Initialising JVM failed: unknown error"); - return RC_FX; + break; case JNI_EDETACHED: strcpy(g->Message, "Thread detached from the VM"); - return RC_FX; + break; case JNI_EVERSION: strcpy(g->Message, "JNI version error"); - return RC_FX; + break; case JNI_ENOMEM: strcpy(g->Message, "Not enough memory"); - return RC_FX; + break; case JNI_EEXIST: strcpy(g->Message, "VM already created"); - return RC_FX; + break; case JNI_EINVAL: strcpy(g->Message, "Invalid arguments"); - return RC_FX; + break; default: - sprintf(g->Message, "Unknown return code %d", rc); - return RC_FX; + sprintf(g->Message, "Unknown return code %d", (int)rc); + break; } // endswitch rc + if (trace) + htrc("%s\n", g->Message); + + if (irc != RC_OK) + return irc; + //=============== Display JVM version =============== jint ver = env->GetVersion(); printf("JVM Version %d.%d\n", ((ver>>16)&0x0f), (ver&0x0f)); @@ -978,10 +984,10 @@ int JDBConn::Open(PJPARM sop) jobjectArray parms = env->NewObjectArray(4, // constructs java array of 4 env->FindClass("java/lang/String"), NULL); // Strings - m_Driver = sop->Driver; - m_Url = sop->Url; - m_User = sop->User; - m_Pwd = sop->Pwd; +//m_Driver = sop->Driver; +//m_Url = sop->Url; +//m_User = sop->User; +//m_Pwd = sop->Pwd; m_Scrollable = sop->Scrollable; m_RowsetSize = sop->Fsize; //m_LoginTimeout = sop->Cto; @@ -989,17 +995,20 @@ int JDBConn::Open(PJPARM sop) //m_UseCnc = sop->UseCnc; // change some elements - if (m_Driver) - env->SetObjectArrayElement(parms, 0, env->NewStringUTF(m_Driver)); + if (sop->Driver) + env->SetObjectArrayElement(parms, 0, env->NewStringUTF(sop->Driver)); - if (m_Url) - env->SetObjectArrayElement(parms, 1, env->NewStringUTF(m_Url)); + if (sop->Url) + env->SetObjectArrayElement(parms, 1, env->NewStringUTF(sop->Url)); - if (m_User) - env->SetObjectArrayElement(parms, 2, env->NewStringUTF(m_User)); + if (sop->User) + env->SetObjectArrayElement(parms, 2, env->NewStringUTF(sop->User)); - if (m_Pwd) - env->SetObjectArrayElement(parms, 3, env->NewStringUTF(m_Pwd)); + if (sop->Pwd) + env->SetObjectArrayElement(parms, 3, env->NewStringUTF(sop->Pwd)); + +//if (sop->Properties) +// env->SetObjectArrayElement(parms, 4, env->NewStringUTF(sop->Properties)); // call method rc = env->CallIntMethod(job, cid, parms, m_RowsetSize, m_Scrollable); @@ -1432,7 +1441,7 @@ bool JDBConn::SetParam(JDBCCOL *colp) PGLOBAL& g = m_G; bool rc = false; PVAL val = colp->GetValue(); - jint n, i = (jint)colp->GetRank(); + jint n, jrc = 0, i = (jint)colp->GetRank(); jshort s; jlong lg; //jfloat f; @@ -1442,69 +1451,74 @@ bool JDBConn::SetParam(JDBCCOL *colp) jstring jst = nullptr; jmethodID dtc, setid = nullptr; - switch (val->GetType()) { - case TYPE_STRING: - if (gmID(g, setid, "SetStringParm", "(ILjava/lang/String;)V")) - return true; - - jst = env->NewStringUTF(val->GetCharValue()); - env->CallVoidMethod(job, setid, i, jst); - break; - case TYPE_INT: - if (gmID(g, setid, "SetIntParm", "(II)V")) + if (val->GetNullable() && val->IsNull()) { + if (gmID(g, setid, "SetNullParm", "(II)I")) return true; - n = (jint)val->GetIntValue(); - env->CallVoidMethod(job, setid, i, n); - break; - case TYPE_TINY: - case TYPE_SHORT: - if (gmID(g, setid, "SetShortParm", "(IS)V")) - return true; + jrc = env->CallIntMethod(job, setid, i, (jint)GetJDBCType(val->GetType())); + } else switch (val->GetType()) { + case TYPE_STRING: + if (gmID(g, setid, "SetStringParm", "(ILjava/lang/String;)V")) + return true; - s = (jshort)val->GetShortValue(); - env->CallVoidMethod(job, setid, i, s); - break; - case TYPE_BIGINT: - if (gmID(g, setid, "SetBigintParm", "(IJ)V")) - return true; + jst = env->NewStringUTF(val->GetCharValue()); + env->CallVoidMethod(job, setid, i, jst); + break; + case TYPE_INT: + if (gmID(g, setid, "SetIntParm", "(II)V")) + return true; - lg = (jlong)val->GetBigintValue(); - env->CallVoidMethod(job, setid, i, lg); - break; - case TYPE_DOUBLE: - case TYPE_DECIM: - if (gmID(g, setid, "SetDoubleParm", "(ID)V")) - return true; + n = (jint)val->GetIntValue(); + env->CallVoidMethod(job, setid, i, n); + break; + case TYPE_TINY: + case TYPE_SHORT: + if (gmID(g, setid, "SetShortParm", "(IS)V")) + return true; - d = (jdouble)val->GetFloatValue(); - env->CallVoidMethod(job, setid, i, d); - break; - case TYPE_DATE: - if ((dat = env->FindClass("java/sql/Timestamp")) == nullptr) { - strcpy(g->Message, "Cannot find Timestamp class"); - return true; - } else if (!(dtc = env->GetMethodID(dat, "<init>", "(J)V"))) { - strcpy(g->Message, "Cannot find Timestamp class constructor"); - return true; - } // endif's + s = (jshort)val->GetShortValue(); + env->CallVoidMethod(job, setid, i, s); + break; + case TYPE_BIGINT: + if (gmID(g, setid, "SetBigintParm", "(IJ)V")) + return true; - lg = (jlong)val->GetBigintValue() * 1000; + lg = (jlong)val->GetBigintValue(); + env->CallVoidMethod(job, setid, i, lg); + break; + case TYPE_DOUBLE: + case TYPE_DECIM: + if (gmID(g, setid, "SetDoubleParm", "(ID)V")) + return true; - if ((datobj = env->NewObject(dat, dtc, lg)) == nullptr) { - strcpy(g->Message, "Cannot make Timestamp object"); - return true; - } else if (gmID(g, setid, "SetTimestampParm", "(ILjava/sql/Timestamp;)V")) + d = (jdouble)val->GetFloatValue(); + env->CallVoidMethod(job, setid, i, d); + break; + case TYPE_DATE: + if ((dat = env->FindClass("java/sql/Timestamp")) == nullptr) { + strcpy(g->Message, "Cannot find Timestamp class"); + return true; + } else if (!(dtc = env->GetMethodID(dat, "<init>", "(J)V"))) { + strcpy(g->Message, "Cannot find Timestamp class constructor"); + return true; + } // endif's + + lg = (jlong)val->GetBigintValue() * 1000; + + if ((datobj = env->NewObject(dat, dtc, lg)) == nullptr) { + strcpy(g->Message, "Cannot make Timestamp object"); + return true; + } else if (gmID(g, setid, "SetTimestampParm", "(ILjava/sql/Timestamp;)V")) + return true; + + env->CallVoidMethod(job, setid, i, datobj); + break; + default: + sprintf(g->Message, "Parm type %d not supported", val->GetType()); return true; + } // endswitch Type - env->CallVoidMethod(job, setid, i, datobj); - break; - default: - sprintf(g->Message, "Parm type %d not supported", val->GetType()); - return true; - } // endswitch Type - - if (Check()) { + if (Check(jrc)) { sprintf(g->Message, "SetParam: col=%s msg=%s", colp->GetName(), Msg); rc = true; } // endif msg diff --git a/storage/connect/jdbconn.h b/storage/connect/jdbconn.h index 0a1c52d4576..9d428142839 100644 --- a/storage/connect/jdbconn.h +++ b/storage/connect/jdbconn.h @@ -180,9 +180,9 @@ protected: char *Msg; char *m_Wrap; char m_IDQuoteChar[2]; - PSZ m_Driver; - PSZ m_Url; - PSZ m_User; +//PSZ m_Driver; +//PSZ m_Url; +//PSZ m_User; PSZ m_Pwd; int m_Ncol; int m_Aff; diff --git a/storage/connect/jsonudf.cpp b/storage/connect/jsonudf.cpp index 88931ea6c13..bc7f231814d 100644 --- a/storage/connect/jsonudf.cpp +++ b/storage/connect/jsonudf.cpp @@ -5263,3 +5263,50 @@ char *envar(UDF_INIT *initid, UDF_ARGS *args, char *result, return str; } // end of envar +/*********************************************************************************/ +/* Returns the distinct number of B occurences in A. */ +/*********************************************************************************/ +my_bool countin_init(UDF_INIT *initid, UDF_ARGS *args, char *message) +{ + if (args->arg_count != 2) { + strcpy(message, "This function must have 2 arguments"); + return true; + } else if (args->arg_type[0] != STRING_RESULT) { + strcpy(message, "First argument must be string"); + return true; + } else if (args->arg_type[1] != STRING_RESULT) { + strcpy(message, "Second argument is not a string"); + return true; + } // endif args + + return false; +} // end of countin_init + +long long countin(UDF_INIT *initid, UDF_ARGS *args, char *result, + unsigned long *res_length, char *is_null, char *) +{ + PSZ str1, str2; + char *s; + long long n = 0; + size_t lg; + + lg = (size_t)args->lengths[0]; + s = str1 = (PSZ)malloc(lg + 1); + memcpy(str1, args->args[0], lg); + str1[lg] = 0; + + lg = (size_t)args->lengths[1]; + str2 = (PSZ)malloc(lg + 1); + memcpy(str2, args->args[1], lg); + str2[lg] = 0; + + while (s = strstr(s, str2)) { + n++; + s += lg; + } // endwhile + + free(str1); + free(str2); + return n; +} // end of countin + diff --git a/storage/connect/jsonudf.h b/storage/connect/jsonudf.h index 1406d9f2f2e..d2890421c62 100644 --- a/storage/connect/jsonudf.h +++ b/storage/connect/jsonudf.h @@ -221,6 +221,9 @@ extern "C" { DllExport my_bool envar_init(UDF_INIT*, UDF_ARGS*, char*); DllExport char *envar(UDF_EXEC_ARGS); + + DllExport my_bool countin_init(UDF_INIT*, UDF_ARGS*, char*); + DllExport long long countin(UDF_EXEC_ARGS); } // extern "C" diff --git a/storage/connect/libdoc.cpp b/storage/connect/libdoc.cpp index c2882fc0d7c..2470d37c353 100644 --- a/storage/connect/libdoc.cpp +++ b/storage/connect/libdoc.cpp @@ -1,6 +1,6 @@ /******************************************************************/ /* Implementation of XML document processing using libxml2 */ -/* Author: Olivier Bertrand 2007-2015 */ +/* Author: Olivier Bertrand 2007-2016 */ /******************************************************************/ #include "my_global.h" #include <string.h> @@ -68,8 +68,8 @@ class LIBXMLDOC : public XMLDOCUMENT { virtual void SetNofree(bool b) {Nofreelist = b;} // Methods - virtual bool Initialize(PGLOBAL g); - virtual bool ParseFile(char *fn); + virtual bool Initialize(PGLOBAL g, char *entry, bool zipped); + virtual bool ParseFile(PGLOBAL g, char *fn); virtual bool NewDoc(PGLOBAL g, char *ver); virtual void AddComment(PGLOBAL g, char *com); virtual PXNODE GetRoot(PGLOBAL g); @@ -373,22 +373,33 @@ LIBXMLDOC::LIBXMLDOC(char *nsl, char *nsdf, char *enc, PFBLOCK fp) /******************************************************************/ /* Initialize XML parser and check library compatibility. */ /******************************************************************/ -bool LIBXMLDOC::Initialize(PGLOBAL g) - { +bool LIBXMLDOC::Initialize(PGLOBAL g, char *entry, bool zipped) +{ + if (zipped && InitZip(g, entry)) + return true; + int n = xmlKeepBlanksDefault(1); return MakeNSlist(g); - } // end of Initialize +} // end of Initialize /******************************************************************/ /* Parse the XML file and construct node tree in memory. */ /******************************************************************/ -bool LIBXMLDOC::ParseFile(char *fn) +bool LIBXMLDOC::ParseFile(PGLOBAL g, char *fn) { if (trace) htrc("ParseFile\n"); - if ((Docp = xmlParseFile(fn))) { - if (Docp->encoding) + if (zip) { + // Parse an in memory document + char *xdoc = GetMemDoc(g, fn); + + Docp = (xdoc) ? xmlParseDoc((const xmlChar *)xdoc) : NULL; + } else + Docp = xmlParseFile(fn); + + if (Docp) { + if (Docp->encoding) Encoding = (char*)Docp->encoding; return false; @@ -609,6 +620,7 @@ void LIBXMLDOC::CloseDoc(PGLOBAL g, PFBLOCK xp) } // endif xp CloseXML2File(g, xp, false); + CloseZip(); } // end of Close /******************************************************************/ diff --git a/storage/connect/mycat.cc b/storage/connect/mycat.cc index b4b03e6ba4a..497fe5e1aa8 100644 --- a/storage/connect/mycat.cc +++ b/storage/connect/mycat.cc @@ -16,7 +16,7 @@ /*************** Mycat CC Program Source Code File (.CC) ***************/ /* PROGRAM NAME: MYCAT */ /* ------------- */ -/* Version 1.4 */ +/* Version 1.5 */ /* */ /* Author: Olivier Bertrand 2012 - 2016 */ /* */ @@ -64,7 +64,9 @@ #include "filamtxt.h" #include "tabdos.h" #include "tabfmt.h" +#if defined(VCT_SUPPORT) #include "tabvct.h" +#endif // VCT_SUPPORT #include "tabsys.h" #if defined(__WIN__) #include "tabmac.h" @@ -93,6 +95,9 @@ #if defined(XML_SUPPORT) #include "tabxml.h" #endif // XML_SUPPORT +#if defined(ZIP_SUPPORT) +#include "tabzip.h" +#endif // ZIP_SUPPORT #include "mycat.h" /***********************************************************************/ @@ -152,7 +157,10 @@ TABTYPE GetTypeID(const char *type) #endif : (!stricmp(type, "VIR")) ? TAB_VIR : (!stricmp(type, "JSON")) ? TAB_JSON - : (!stricmp(type, "OEM")) ? TAB_OEM : TAB_NIY; +#ifdef ZIP_SUPPORT + : (!stricmp(type, "ZIP")) ? TAB_ZIP +#endif + : (!stricmp(type, "OEM")) ? TAB_OEM : TAB_NIY; } // end of GetTypeID /***********************************************************************/ @@ -173,6 +181,7 @@ bool IsFileType(TABTYPE type) case TAB_INI: case TAB_VEC: case TAB_JSON: +// case TAB_ZIP: isfile= true; break; default: @@ -549,7 +558,9 @@ PRELDEF MYCAT::MakeTableDesc(PGLOBAL g, PTABLE tablep, LPCSTR am) #if defined(XML_SUPPORT) case TAB_XML: tdp= new(g) XMLDEF; break; #endif // XML_SUPPORT - case TAB_VEC: tdp= new(g) VCTDEF; break; +#if defined(VCT_SUPPORT) + case TAB_VEC: tdp = new(g)VCTDEF; break; +#endif // VCT_SUPPORT #if defined(ODBC_SUPPORT) case TAB_ODBC: tdp= new(g) ODBCDEF; break; #endif // ODBC_SUPPORT @@ -571,7 +582,10 @@ PRELDEF MYCAT::MakeTableDesc(PGLOBAL g, PTABLE tablep, LPCSTR am) #endif // PIVOT_SUPPORT case TAB_VIR: tdp= new(g) VIRDEF; break; case TAB_JSON: tdp= new(g) JSONDEF; break; - default: +#if defined(ZIP_SUPPORT) + case TAB_ZIP: tdp= new(g) ZIPDEF; break; +#endif // ZIP_SUPPORT + default: sprintf(g->Message, MSG(BAD_TABLE_TYPE), am, name); } // endswitch diff --git a/storage/connect/mycat.h b/storage/connect/mycat.h index 05163f08f1b..663b68fd4b9 100644 --- a/storage/connect/mycat.h +++ b/storage/connect/mycat.h @@ -62,6 +62,7 @@ struct ha_table_option_struct { bool split; bool readonly; bool sepindex; + bool zipped; }; // Possible value for catalog functions diff --git a/storage/connect/mysql-test/connect/disabled.def b/storage/connect/mysql-test/connect/disabled.def index 64d7ece3fe1..0e5a5fc64e3 100644 --- a/storage/connect/mysql-test/connect/disabled.def +++ b/storage/connect/mysql-test/connect/disabled.def @@ -9,8 +9,8 @@ # Do not use any TAB characters for whitespace. # ############################################################################## -jdbc : Variable settings depend on machine configuration -jdbc_new : Variable settings depend on machine configuration +#jdbc : Variable settings depend on machine configuration +#jdbc_new : Variable settings depend on machine configuration jdbc_oracle : Variable settings depend on machine configuration jdbc_postgresql : Variable settings depend on machine configuration json : TABLE_TYPE = JSON conflicts with the SQL syntax diff --git a/storage/connect/mysql-test/connect/r/jdbc_new.result b/storage/connect/mysql-test/connect/r/jdbc_new.result index 14381b0b11f..5cc4826213d 100644 --- a/storage/connect/mysql-test/connect/r/jdbc_new.result +++ b/storage/connect/mysql-test/connect/r/jdbc_new.result @@ -56,7 +56,7 @@ t1 CREATE TABLE `t1` ( ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='jdbc:mysql://127.0.0.1:SLAVE_PORT/test?user=root' `TABLE_TYPE`='JDBC' SELECT * FROM t1; a b -0 NULL +NULL NULL 0 test00 1 test01 2 test02 @@ -72,7 +72,7 @@ t1 CREATE TABLE `t1` ( ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='jdbc:mysql://127.0.0.1:SLAVE_PORT/test?user=root' `TABLE_TYPE`=JDBC `TABNAME`='t1' SELECT * FROM t1; a b -0 NULL +NULL NULL 0 test00 1 test01 2 test02 @@ -104,7 +104,7 @@ t1 CREATE TABLE `t1` ( ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='jdbc:mysql://127.0.0.1:SLAVE_PORT/test?user=root' `TABLE_TYPE`=JDBC SELECT * FROM t1; a b -0 NULL +NULL NULL 0 0 1 0 2 0 diff --git a/storage/connect/mysql-test/connect/std_data/JdbcMariaDB.jar b/storage/connect/mysql-test/connect/std_data/JdbcMariaDB.jar Binary files differindex 81f91e4465a..8525da74018 100644 --- a/storage/connect/mysql-test/connect/std_data/JdbcMariaDB.jar +++ b/storage/connect/mysql-test/connect/std_data/JdbcMariaDB.jar diff --git a/storage/connect/mysql-test/connect/t/jdbc.test b/storage/connect/mysql-test/connect/t/jdbc.test index 41fd298776b..58a527a3e6b 100644 --- a/storage/connect/mysql-test/connect/t/jdbc.test +++ b/storage/connect/mysql-test/connect/t/jdbc.test @@ -1,3 +1,4 @@ +-- source windows.inc -- source jdbconn.inc SET GLOBAL time_zone='+1:00'; diff --git a/storage/connect/mysql-test/connect/t/jdbc_new.test b/storage/connect/mysql-test/connect/t/jdbc_new.test index d1ad5117b72..5586cf8c027 100644 --- a/storage/connect/mysql-test/connect/t/jdbc_new.test +++ b/storage/connect/mysql-test/connect/t/jdbc_new.test @@ -5,6 +5,7 @@ connect (master,127.0.0.1,root,,test,$MASTER_MYPORT,); connect (slave,127.0.0.1,root,,test,$SLAVE_MYPORT,); connection master; +-- source windows.inc -- source jdbconn.inc connection slave; diff --git a/storage/connect/mysql-test/connect/t/windows.inc b/storage/connect/mysql-test/connect/t/windows.inc new file mode 100644 index 00000000000..88553d8aa59 --- /dev/null +++ b/storage/connect/mysql-test/connect/t/windows.inc @@ -0,0 +1,5 @@ +if (`select convert(@@version_compile_os using latin1) IN ("Win32","Win64","Windows") = 0`) +{ + skip Need windows; +} + diff --git a/storage/connect/plgdbsem.h b/storage/connect/plgdbsem.h index 910ce97f48a..cb408494319 100644 --- a/storage/connect/plgdbsem.h +++ b/storage/connect/plgdbsem.h @@ -1,9 +1,9 @@ /************** PlgDBSem H Declares Source Code File (.H) **************/ -/* Name: PLGDBSEM.H Version 3.6 */ +/* Name: PLGDBSEM.H Version 3.7 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 1998-2015 */ +/* (C) Copyright to the author Olivier BERTRAND 1998-2016 */ /* */ -/* This file contains the PlugDB++ application type definitions. */ +/* This file contains the CONNECT storage engine definitions. */ /***********************************************************************/ /***********************************************************************/ @@ -49,7 +49,8 @@ enum BLKTYP {TYPE_TABLE = 50, /* Table Name/Srcdef/... Block */ TYPE_FB_MAP = 23, /* Mapped file block (storage) */ TYPE_FB_HANDLE = 24, /* File block (handle) */ TYPE_FB_XML = 21, /* DOM XML file block */ - TYPE_FB_XML2 = 27}; /* libxml2 XML file block */ + TYPE_FB_XML2 = 27, /* libxml2 XML file block */ + TYPE_FB_ZIP = 28}; /* ZIP file block */ enum TABTYPE {TAB_UNDEF = 0, /* Table of undefined type */ TAB_DOS = 1, /* Fixed column offset, variable LRECL */ @@ -78,7 +79,8 @@ enum TABTYPE {TAB_UNDEF = 0, /* Table of undefined type */ TAB_JCT = 24, /* Junction tables NIY */ TAB_DMY = 25, /* DMY Dummy tables NIY */ TAB_JDBC = 26, /* Table accessed via JDBC */ - TAB_NIY = 27}; /* Table not implemented yet */ + TAB_ZIP = 27, /* ZIP file info table */ + TAB_NIY = 28}; /* Table not implemented yet */ enum AMT {TYPE_AM_ERROR = 0, /* Type not defined */ TYPE_AM_ROWID = 1, /* ROWID type (special column) */ @@ -123,7 +125,7 @@ enum AMT {TYPE_AM_ERROR = 0, /* Type not defined */ TYPE_AM_PRX = 129, /* PROXY access method type no */ TYPE_AM_XTB = 130, /* SYS table access method type */ TYPE_AM_BLK = 131, /* BLK access method type no */ - TYPE_AM_ZIP = 132, /* ZIP access method type no */ + TYPE_AM_GZ = 132, /* GZ access method type no */ TYPE_AM_ZLIB = 133, /* ZLIB access method type no */ TYPE_AM_JSON = 134, /* JSON access method type no */ TYPE_AM_JSN = 135, /* JSN access method type no */ @@ -140,7 +142,8 @@ enum AMT {TYPE_AM_ERROR = 0, /* Type not defined */ TYPE_AM_MYSQL = 192, /* MYSQL access method type no */ TYPE_AM_MYX = 193, /* MYSQL EXEC access method type */ TYPE_AM_CAT = 195, /* Catalog access method type no */ - TYPE_AM_OUT = 200}; /* Output relations (storage) */ + TYPE_AM_ZIP = 198, /* ZIP access method type no */ + TYPE_AM_OUT = 200}; /* Output relations (storage) */ enum RECFM {RECFM_NAF = -2, /* Not a file */ RECFM_OEM = -1, /* OEM file access method */ diff --git a/storage/connect/plgdbutl.cpp b/storage/connect/plgdbutl.cpp index 13c0dfd1e18..83975c6d8fa 100644 --- a/storage/connect/plgdbutl.cpp +++ b/storage/connect/plgdbutl.cpp @@ -68,6 +68,9 @@ #include "tabcol.h" // header of XTAB and COLUMN classes #include "valblk.h" #include "rcmsg.h" +#ifdef ZIP_SUPPORT +#include "filamzip.h" +#endif // ZIP_SUPPORT /***********************************************************************/ /* DB static variables. */ @@ -934,7 +937,16 @@ int PlugCloseFile(PGLOBAL g __attribute__((unused)), PFBLOCK fp, bool all) CloseXML2File(g, fp, all); break; #endif // LIBXML2_SUPPORT - default: +#ifdef ZIP_SUPPORT + case TYPE_FB_ZIP: + ((ZIPUTIL*)fp->File)->close(); + fp->Memory = NULL; + fp->Mode = MODE_ANY; + fp->Count = 0; + fp->File = NULL; + break; +#endif // ZIP_SUPPORT + default: rc = RC_FX; } // endswitch Type diff --git a/storage/connect/plgxml.cpp b/storage/connect/plgxml.cpp index 3061a6d697e..71b72621b06 100644 --- a/storage/connect/plgxml.cpp +++ b/storage/connect/plgxml.cpp @@ -30,19 +30,51 @@ PXDOC GetLibxmlDoc(PGLOBAL g, char *nsl, char *nsdf, /* XMLDOCUMENT constructor. */ /******************************************************************/ XMLDOCUMENT::XMLDOCUMENT(char *nsl, char *nsdf, char *enc) - { - Namespaces = NULL; +{ +#if defined(ZIP_SUPPORT) + zip = NULL; +#else // !ZIP_SUPPORT + zip = false; +#endif // !ZIP_SUPPORT + Namespaces = NULL; Encoding = enc; Nslist = nsl; DefNs = nsdf; - } // end of XMLDOCUMENT constructor +} // end of XMLDOCUMENT constructor + +/******************************************************************/ +/* Initialize zipped file processing. */ +/******************************************************************/ +bool XMLDOCUMENT::InitZip(PGLOBAL g, char *entry) +{ +#if defined(ZIP_SUPPORT) + bool mul = (entry) ? strchr(entry, '*') || strchr(entry, '?') : false; + zip = new(g) ZIPUTIL(entry, mul); + return zip == NULL; +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return true; +#endif // !ZIP_SUPPORT +} // end of InitZip + +/******************************************************************/ +/* Make the namespace structure list. */ +/******************************************************************/ +char* XMLDOCUMENT::GetMemDoc(PGLOBAL g, char *fn) +{ +#if defined(ZIP_SUPPORT) + return (zip->OpenTable(g, MODE_ANY, fn)) ? NULL : zip->memory; +#else // !ZIP_SUPPORT + return NULL; +#endif // !ZIP_SUPPORT +} // end of GetMemDoc /******************************************************************/ /* Make the namespace structure list. */ /******************************************************************/ bool XMLDOCUMENT::MakeNSlist(PGLOBAL g) - { - char *prefix, *href, *next = Nslist; +{ + char *prefix, *href, *next = Nslist; PNS nsp, *ppns = &Namespaces; while (next) { @@ -84,6 +116,19 @@ bool XMLDOCUMENT::MakeNSlist(PGLOBAL g) } // end of MakeNSlist /******************************************************************/ +/* Close ZIP file. */ +/******************************************************************/ +void XMLDOCUMENT::CloseZip(void) +{ +#if defined(ZIP_SUPPORT) + if (zip) { + zip->close(); + zip = NULL; + } // endif zip +#endif // ZIP_SUPPORT +} // end of CloseZip + +/******************************************************************/ /* XMLNODE constructor. */ /******************************************************************/ XMLNODE::XMLNODE(PXDOC dp) diff --git a/storage/connect/plgxml.h b/storage/connect/plgxml.h index b8e914e0bf1..db7dfa6bda5 100644 --- a/storage/connect/plgxml.h +++ b/storage/connect/plgxml.h @@ -1,3 +1,7 @@ +#if defined(ZIP_SUPPORT) +#include "filamzip.h" +#endif // ZIP_SUPPORT + /******************************************************************/ /* Dual XML implementation base classes defines. */ /******************************************************************/ @@ -72,8 +76,8 @@ class XMLDOCUMENT : public BLOCK { virtual void SetNofree(bool b) = 0; // Methods - virtual bool Initialize(PGLOBAL) = 0; - virtual bool ParseFile(char *) = 0; + virtual bool Initialize(PGLOBAL, char *, bool) = 0; + virtual bool ParseFile(PGLOBAL, char *) = 0; virtual bool NewDoc(PGLOBAL, char *) = 0; virtual void AddComment(PGLOBAL, char *) = 0; virtual PXNODE GetRoot(PGLOBAL) = 0; @@ -91,8 +95,16 @@ class XMLDOCUMENT : public BLOCK { // Utility bool MakeNSlist(PGLOBAL g); + bool InitZip(PGLOBAL g, char *entry); + char *GetMemDoc(PGLOBAL g, char *fn); + void CloseZip(void); // Members +#if defined(ZIP_SUPPORT) + ZIPUTIL *zip; /* Used for zipped file */ +#else // !ZIP_SUPPORT + bool zip; /* Always false */ +#endif // !ZIP_SUPPORT PNS Namespaces; /* To the namespaces */ char *Encoding; /* The document encoding */ char *Nslist; /* Namespace list */ diff --git a/storage/connect/reldef.cpp b/storage/connect/reldef.cpp index 56a7ea8c512..ef91414a9ab 100644 --- a/storage/connect/reldef.cpp +++ b/storage/connect/reldef.cpp @@ -1,7 +1,7 @@ /************* RelDef CPP Program Source Code File (.CPP) **************/ /* PROGRAM NAME: RELDEF */ /* ------------- */ -/* Version 1.5 */ +/* Version 1.6 */ /* */ /* COPYRIGHT: */ /* ---------- */ @@ -40,10 +40,12 @@ #include "tabcol.h" #include "filamap.h" #include "filamfix.h" +#if defined(VCT_SUPPORT) #include "filamvct.h" -#if defined(ZIP_SUPPORT) -#include "filamzip.h" -#endif // ZIP_SUPPORT +#endif // VCT_SUPPORT +#if defined(GZ_SUPPORT) +#include "filamgz.h" +#endif // GZ_SUPPORT #include "tabdos.h" #include "valblk.h" #include "tabmul.h" @@ -663,15 +665,15 @@ PTDB OEMDEF::GetTable(PGLOBAL g, MODE mode) /*********************************************************************/ if (!((PTDBDOS)tdbp)->GetTxfp()) { if (cmpr) { -#if defined(ZIP_SUPPORT) +#if defined(GZ_SUPPORT) if (cmpr == 1) - txfp = new(g) ZIPFAM(defp); + txfp = new(g) GZFAM(defp); else txfp = new(g) ZLBFAM(defp); -#else // !ZIP_SUPPORT +#else // !GZ_SUPPORT strcpy(g->Message, "Compress not supported"); return NULL; -#endif // !ZIP_SUPPORT +#endif // !GZ_SUPPORT } else if (rfm == RECFM_VAR) { if (map) txfp = new(g) MAPFAM(defp); @@ -683,16 +685,19 @@ PTDB OEMDEF::GetTable(PGLOBAL g, MODE mode) txfp = new(g) MPXFAM(defp); else txfp = new(g) FIXFAM(defp); - } else if (rfm == RECFM_VCT) { - assert (Pxdef->GetDefType() == TYPE_AM_VCT); +#if defined(VCT_SUPPORT) + assert(Pxdef->GetDefType() == TYPE_AM_VCT); if (map) txfp = new(g) VCMFAM((PVCTDEF)defp); else txfp = new(g) VCTFAM((PVCTDEF)defp); - - } // endif's +#else // !VCT_SUPPORT + strcpy(g->Message, "VCT no more supported"); + return NULL; +#endif // !VCT_SUPPORT + } // endif's ((PTDBDOS)tdbp)->SetTxfp(txfp); } // endif Txfp diff --git a/storage/connect/tabdos.cpp b/storage/connect/tabdos.cpp index 98633f49d23..16cc6c33b44 100644 --- a/storage/connect/tabdos.cpp +++ b/storage/connect/tabdos.cpp @@ -1,11 +1,11 @@ /************* TabDos C++ Program Source Code File (.CPP) **************/ /* PROGRAM NAME: TABDOS */ /* ------------- */ -/* Version 4.9 */ +/* Version 4.9.2 */ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 1998-2015 */ +/* (C) Copyright to the author Olivier BERTRAND 1998-2016 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -51,6 +51,9 @@ #include "filamap.h" #include "filamfix.h" #include "filamdbf.h" +#if defined(GZ_SUPPORT) +#include "filamgz.h" +#endif // GZ_SUPPORT #if defined(ZIP_SUPPORT) #include "filamzip.h" #endif // ZIP_SUPPORT @@ -93,10 +96,13 @@ DOSDEF::DOSDEF(void) Pseudo = 3; Fn = NULL; Ofn = NULL; + Entry = NULL; To_Indx = NULL; Recfm = RECFM_VAR; Mapped = false; - Padded = false; + Zipped = false; + Mulentries = false; + Padded = false; Huge = false; Accept = false; Eof = false; @@ -126,6 +132,11 @@ bool DOSDEF::DefineAM(PGLOBAL g, LPCSTR am, int) : (am && (*am == 'B' || *am == 'b')) ? "B" : (am && !stricmp(am, "DBF")) ? "D" : "V"; + if ((Zipped = GetBoolCatInfo("Zipped", false))) + Mulentries = ((Entry = GetStringCatInfo(g, "Entry", NULL))) + ? strchr(Entry, '*') || strchr(Entry, '?') + : GetBoolCatInfo("Mulentries", false); + Desc = Fn = GetStringCatInfo(g, "Filename", NULL); Ofn = GetStringCatInfo(g, "Optname", Fn); GetCharCatInfo("Recfm", (PSZ)dfm, buf, sizeof(buf)); @@ -333,7 +344,21 @@ PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode) /* Allocate table and file processing class of the proper type. */ /* Column blocks will be allocated only when needed. */ /*********************************************************************/ - if (Recfm == RECFM_DBF) { + if (Zipped) { +#if defined(ZIP_SUPPORT) + if (Recfm == RECFM_VAR) { + txfp = new(g)ZIPFAM(this); + tdbp = new(g)TDBDOS(this, txfp); + } else { + txfp = new(g)ZPXFAM(this); + tdbp = new(g)TDBFIX(this, txfp); + } // endif Recfm + +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif // !ZIP_SUPPORT + } else if (Recfm == RECFM_DBF) { if (Catfunc == FNC_NO) { if (map) txfp = new(g) DBMFAM(this); @@ -350,28 +375,28 @@ PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode) else if (map) txfp = new(g) MPXFAM(this); else if (Compressed) { -#if defined(ZIP_SUPPORT) +#if defined(GZ_SUPPORT) txfp = new(g) ZIXFAM(this); -#else // !ZIP_SUPPORT - sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); +#else // !GZ_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "GZ"); return NULL; -#endif // !ZIP_SUPPORT +#endif // !GZ_SUPPORT } else txfp = new(g) FIXFAM(this); tdbp = new(g) TDBFIX(this, txfp); } else { if (Compressed) { -#if defined(ZIP_SUPPORT) +#if defined(GZ_SUPPORT) if (Compressed == 1) - txfp = new(g) ZIPFAM(this); + txfp = new(g) GZFAM(this); else txfp = new(g) ZLBFAM(this); -#else // !ZIP_SUPPORT - sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); +#else // !GZ_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "GZ"); return NULL; -#endif // !ZIP_SUPPORT +#endif // !GZ_SUPPORT } else if (map) txfp = new(g) MAPFAM(this); else @@ -396,7 +421,7 @@ PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode) if (map) { txfp = new(g) MBKFAM(this); } else if (Compressed) { -#if defined(ZIP_SUPPORT) +#if defined(GZ_SUPPORT) if (Compressed == 1) txfp = new(g) ZBKFAM(this); else { @@ -404,7 +429,7 @@ PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode) ((PZLBFAM)txfp)->SetOptimized(To_Pos != NULL); } // endelse #else - sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "GZ"); return NULL; #endif } else @@ -531,14 +556,14 @@ int TDBDOS::ResetTableOpt(PGLOBAL g, bool dop, bool dox) // except for ZLIB access method. if (Txfp->GetAmType() == TYPE_AM_MAP) { Txfp = new(g) MAPFAM((PDOSDEF)To_Def); -#if defined(ZIP_SUPPORT) - } else if (Txfp->GetAmType() == TYPE_AM_ZIP) { - Txfp = new(g) ZIPFAM((PDOSDEF)To_Def); +#if defined(GZ_SUPPORT) + } else if (Txfp->GetAmType() == TYPE_AM_GZ) { + Txfp = new(g) GZFAM((PDOSDEF)To_Def); } else if (Txfp->GetAmType() == TYPE_AM_ZLIB) { Txfp->Reset(); ((PZLBFAM)Txfp)->SetOptimized(false); -#endif // ZIP_SUPPORT - } else if (Txfp->GetAmType() == TYPE_AM_BLK) +#endif // GZ_SUPPORT + } else if (Txfp->GetAmType() == TYPE_AM_BLK) Txfp = new(g) DOSFAM((PDOSDEF)To_Def); Txfp->SetTdbp(this); @@ -609,7 +634,12 @@ int TDBDOS::MakeBlockValues(PGLOBAL g) defp->SetOptimized(0); // Estimate the number of needed blocks - block = (int)((MaxSize + (int)nrec - 1) / (int)nrec); + if ((block = (int)((MaxSize + (int)nrec - 1) / (int)nrec)) < 2) { + // This may be wrong to do in some cases + defp->RemoveOptValues(g); + strcpy(g->Message, MSG(TABLE_NOT_OPT)); + return RC_INFO; // Not to be optimized + } // endif block // We have to use local variables because Txfp->CurBlk is set // to Rows+1 by unblocked variable length table access methods. @@ -952,13 +982,14 @@ bool TDBDOS::GetBlockValues(PGLOBAL g) PCOLDEF cdp; PDOSDEF defp = (PDOSDEF)To_Def; PCATLG cat = defp->GetCat(); + PDBUSER dup = PlgGetUser(g); #if 0 if (Mode == MODE_INSERT && Txfp->GetAmType() == TYPE_AM_DOS) return false; #endif // __WIN__ - if (defp->Optimized) + if (defp->Optimized || !(dup->Check & CHK_OPT)) return false; // Already done or to be redone if (Ftype == RECFM_VAR || defp->Compressed == 2) { @@ -2079,10 +2110,10 @@ bool TDBDOS::OpenDB(PGLOBAL g) /*******************************************************************/ if (Txfp->GetAmType() == TYPE_AM_MAP && Mode == MODE_DELETE) Txfp = new(g) MAPFAM((PDOSDEF)To_Def); -#if defined(ZIP_SUPPORT) - else if (Txfp->GetAmType() == TYPE_AM_ZIP) - Txfp = new(g) ZIPFAM((PDOSDEF)To_Def); -#endif // ZIP_SUPPORT +#if defined(GZ_SUPPORT) + else if (Txfp->GetAmType() == TYPE_AM_GZ) + Txfp = new(g) GZFAM((PDOSDEF)To_Def); +#endif // GZ_SUPPORT else // if (Txfp->GetAmType() != TYPE_AM_DOS) ??? Txfp = new(g) DOSFAM((PDOSDEF)To_Def); diff --git a/storage/connect/tabdos.h b/storage/connect/tabdos.h index c098886f14b..4c8eb438a26 100644 --- a/storage/connect/tabdos.h +++ b/storage/connect/tabdos.h @@ -28,6 +28,7 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */ friend class TDBFIX; friend class TXTFAM; friend class DBFBASE; + friend class ZIPUTIL; public: // Constructor DOSDEF(void); @@ -40,7 +41,9 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */ virtual bool IsHuge(void) {return Huge;} PSZ GetFn(void) {return Fn;} PSZ GetOfn(void) {return Ofn;} - void SetBlock(int block) {Block = block;} + PSZ GetEntry(void) {return Entry;} + bool GetMul(void) {return Mulentries;} + void SetBlock(int block) {Block = block;} int GetBlock(void) {return Block;} int GetLast(void) {return Last;} void SetLast(int last) {Last = last;} @@ -57,9 +60,9 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */ int *GetTo_Pos(void) {return To_Pos;} // Methods - virtual int Indexable(void) - {return (!Multiple && Compressed != 1) ? 1 : 0;} - virtual bool DeleteIndexFile(PGLOBAL g, PIXDEF pxdf); + virtual int Indexable(void) + {return (!Multiple && !Mulentries && Compressed != 1) ? 1 : 0;} + virtual bool DeleteIndexFile(PGLOBAL g, PIXDEF pxdf); virtual bool DefineAM(PGLOBAL g, LPCSTR am, int poff); virtual PTDB GetTable(PGLOBAL g, MODE mode); bool InvalidateIndex(PGLOBAL g); @@ -72,10 +75,13 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */ // Members PSZ Fn; /* Path/Name of corresponding file */ PSZ Ofn; /* Base Path/Name of matching index files*/ - PIXDEF To_Indx; /* To index definitions blocks */ + PSZ Entry; /* Zip entry name or pattern */ + PIXDEF To_Indx; /* To index definitions blocks */ RECFM Recfm; /* 0:VAR, 1:FIX, 2:BIN, 3:VCT, 6:DBF */ bool Mapped; /* 0: disk file, 1: memory mapped file */ - bool Padded; /* true for padded table file */ + bool Zipped; /* true for zipped table file */ + bool Mulentries; /* true for multiple entries */ + bool Padded; /* true for padded table file */ bool Huge; /* true for files larger than 2GB */ bool Accept; /* true if wrong lines are accepted */ bool Eof; /* true if an EOF (0xA) character exists */ @@ -91,7 +97,7 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */ int Maxerr; /* Maximum number of bad records (DBF) */ int ReadMode; /* Specific to DBF */ int Ending; /* Length of end of lines */ - int Teds; /* Binary table default endian setting */ + char Teds; /* Binary table default endian setting */ }; // end of DOSDEF /***********************************************************************/ diff --git a/storage/connect/tabfix.cpp b/storage/connect/tabfix.cpp index 55c254f41ea..d99f7800f26 100644 --- a/storage/connect/tabfix.cpp +++ b/storage/connect/tabfix.cpp @@ -1,11 +1,11 @@ /************* TabFix C++ Program Source Code File (.CPP) **************/ /* PROGRAM NAME: TABFIX */ /* ------------- */ -/* Version 4.9 */ +/* Version 4.9.1 */ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 1998-2015 */ +/* (C) Copyright to the author Olivier BERTRAND 1998-2016 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -589,9 +589,10 @@ void BINCOL::WriteColumn(PGLOBAL g) switch (Fmt) { case 'X': // Standard not converted values - if (Eds && IsTypeChar(Buf_Type)) - *(longlong *)p = Value->GetBigintValue(); - else if (Value->GetBinValue(p, Long, Status)) { + if (Eds && IsTypeChar(Buf_Type)) { + if (Status) + Value->GetValueNonAligned<longlong>(p, Value->GetBigintValue()); + } else if (Value->GetBinValue(p, Long, Status)) { sprintf(g->Message, MSG(BIN_F_TOO_LONG), Name, Value->GetSize(), Long); longjmp(g->jumper[g->jump_level], 31); @@ -605,7 +606,7 @@ void BINCOL::WriteColumn(PGLOBAL g) sprintf(g->Message, MSG(VALUE_TOO_BIG), n, Name); longjmp(g->jumper[g->jump_level], 31); } else if (Status) - *(short *)p = (short)n; + Value->GetValueNonAligned<short>(p, (short)n); break; case 'T': // Tiny integer @@ -625,7 +626,7 @@ void BINCOL::WriteColumn(PGLOBAL g) sprintf(g->Message, MSG(VALUE_TOO_BIG), n, Name); longjmp(g->jumper[g->jump_level], 31); } else if (Status) - *(int *)p = Value->GetIntValue(); + Value->GetValueNonAligned<int>(p, (int)n); break; case 'G': // Large (great) integer @@ -636,12 +637,12 @@ void BINCOL::WriteColumn(PGLOBAL g) case 'F': // Float case 'R': // Real if (Status) - *(float *)p = (float)Value->GetFloatValue(); + Value->GetValueNonAligned<float>(p, (float)Value->GetFloatValue()); break; case 'D': // Double if (Status) - *(double *)p = Value->GetFloatValue(); + Value->GetValueNonAligned<double>(p, Value->GetFloatValue()); break; case 'C': // Characters diff --git a/storage/connect/tabfmt.cpp b/storage/connect/tabfmt.cpp index b786fb64b56..183bb8f8d65 100644 --- a/storage/connect/tabfmt.cpp +++ b/storage/connect/tabfmt.cpp @@ -1,11 +1,11 @@ /************* TabFmt C++ Program Source Code File (.CPP) **************/ /* PROGRAM NAME: TABFMT */ /* ------------- */ -/* Version 3.9 */ +/* Version 3.9.2 */ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 2001 - 2015 */ +/* (C) Copyright to the author Olivier BERTRAND 2001 - 2016 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -51,6 +51,9 @@ #include "plgdbsem.h" #include "mycat.h" #include "filamap.h" +#if defined(GZ_SUPPORT) +#include "filamgz.h" +#endif // GZ_SUPPORT #if defined(ZIP_SUPPORT) #include "filamzip.h" #endif // ZIP_SUPPORT @@ -78,20 +81,24 @@ USETEMP UseTemp(void); /* of types (TYPE_STRING < TYPE_DOUBLE < TYPE_INT) (1 < 2 < 7). */ /* If these values are changed, this will have to be revisited. */ /***********************************************************************/ -PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, - char q, int hdr, int mxr, bool info) +PQRYRES CSVColumns(PGLOBAL g, char *dp, PTOS topt, bool info) { static int buftyp[] = {TYPE_STRING, TYPE_SHORT, TYPE_STRING, TYPE_INT, TYPE_INT, TYPE_SHORT}; static XFLD fldtyp[] = {FLD_NAME, FLD_TYPE, FLD_TYPENAME, FLD_PREC, FLD_LENGTH, FLD_SCALE}; static unsigned int length[] = {6, 6, 8, 10, 10, 6}; - char *p, *colname[MAXCOL], dechar, filename[_MAX_PATH], buf[4096]; + const char *fn; + char sep, q; + int rc, mxr; + bool hdr; + char *p, *colname[MAXCOL], dechar, buf[8]; int i, imax, hmax, n, nerr, phase, blank, digit, dec, type; int ncol = sizeof(buftyp) / sizeof(int); int num_read = 0, num_max = 10000000; // Statistics int len[MAXCOL], typ[MAXCOL], prc[MAXCOL]; - FILE *infile; + PCSVDEF tdp; + PTDBCSV tdbp; PQRYRES qrp; PCOLRES crp; @@ -101,27 +108,13 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, goto skipit; } // endif info -// num_max = atoi(p+1); // Max num of record to test -#if defined(__WIN__) - if (sep == ',' || strnicmp(setlocale(LC_NUMERIC, NULL), "French", 6)) - dechar = '.'; - else - dechar = ','; -#else // !__WIN__ - dechar = '.'; -#endif // !__WIN__ - - if (trace) - htrc("File %s sep=%c q=%c hdr=%d mxr=%d\n", - SVP(fn), sep, q, hdr, mxr); - - if (!fn) { - strcpy(g->Message, MSG(MISSING_FNAME)); - return NULL; - } // endif fn + if (GetIntegerTableOption(g, topt, "Multiple", 0)) { + strcpy(g->Message, "Cannot find column definition for multiple table"); + return NULL; + } // endif Multiple +// num_max = atoi(p+1); // Max num of record to test imax = hmax = nerr = 0; - mxr = MY_MAX(0, mxr); for (i = 0; i < MAXCOL; i++) { colname[i] = NULL; @@ -131,12 +124,76 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, } // endfor i /*********************************************************************/ - /* Open the input file. */ + /* Get the CSV table description block. */ /*********************************************************************/ - PlugSetPath(filename, fn, dp); + tdp = new(g) CSVDEF; +#if defined(ZIP_SUPPORT) + tdp->Entry = GetStringTableOption(g, topt, "Entry", NULL); + tdp->Zipped = GetBooleanTableOption(g, topt, "Zipped", false); +#endif // ZIP_SUPPORT + fn = tdp->Fn = GetStringTableOption(g, topt, "Filename", NULL); + + if (!tdp->Fn) { + strcpy(g->Message, MSG(MISSING_FNAME)); + return NULL; + } // endif Fn + + if (!(tdp->Lrecl = GetIntegerTableOption(g, topt, "Lrecl", 0))) + tdp->Lrecl = 4096; + + p = GetStringTableOption(g, topt, "Separator", ","); + tdp->Sep = (strlen(p) == 2 && p[0] == '\\' && p[1] == 't') ? '\t' : *p; + +#if defined(__WIN__) + if (tdp->Sep == ',' || strnicmp(setlocale(LC_NUMERIC, NULL), "French", 6)) + dechar = '.'; + else + dechar = ','; +#else // !__WIN__ + dechar = '.'; +#endif // !__WIN__ + + sep = tdp->Sep; + tdp->Quoted = GetIntegerTableOption(g, topt, "Quoted", -1); + p = GetStringTableOption(g, topt, "Qchar", ""); + tdp->Qot = *p; + + if (tdp->Qot && tdp->Quoted < 0) + tdp->Quoted = 0; + else if (!tdp->Qot && tdp->Quoted >= 0) + tdp->Qot = '"'; - if (!(infile= global_fopen(g, MSGID_CANNOT_OPEN, filename, "r"))) - return NULL; + q = tdp->Qot; + hdr = GetBooleanTableOption(g, topt, "Header", false); + tdp->Maxerr = GetIntegerTableOption(g, topt, "Maxerr", 0); + tdp->Accept = GetBooleanTableOption(g, topt, "Accept", false); + + if (tdp->Accept && tdp->Maxerr == 0) + tdp->Maxerr = INT_MAX32; // Accept all bad lines + + mxr = MY_MAX(0, tdp->Maxerr); + + if (trace) + htrc("File %s Sep=%c Qot=%c Header=%d maxerr=%d\n", + SVP(tdp->Fn), tdp->Sep, tdp->Qot, tdp->Header, tdp->Maxerr); + + if (tdp->Zipped) { +#if defined(ZIP_SUPPORT) + tdbp = new(g)TDBCSV(tdp, new(g)ZIPFAM(tdp)); +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif // !ZIP_SUPPORT + } else + tdbp = new(g) TDBCSV(tdp, new(g) DOSFAM(tdp)); + + tdbp->SetMode(MODE_READ); + + /*********************************************************************/ + /* Open the CSV file. */ + /*********************************************************************/ + if (tdbp->OpenDB(g)) + return NULL; if (hdr) { /*******************************************************************/ @@ -144,16 +201,8 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, /*******************************************************************/ phase = 0; - if (fgets(buf, sizeof(buf), infile)) { - n = strlen(buf) + 1; - buf[n - 2] = '\0'; -#if !defined(__WIN__) - // The file can be imported from Windows - if (buf[n - 3] == '\r') - buf[n - 3] = 0; -#endif // UNIX - p = (char*)PlugSubAlloc(g, NULL, n); - memcpy(p, buf, n); + if ((rc = tdbp->ReadDB(g)) == RC_OK) { + p = PlgDBDup(g, tdbp->To_Line); //skip leading blanks for (; *p == ' '; p++) ; @@ -165,10 +214,11 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, } // endif q colname[0] = p; - } else { + } else if (rc == RC_EF) { sprintf(g->Message, MSG(FILE_IS_EMPTY), fn); goto err; - } // endif's + } else + goto err; for (i = 1; *p; p++) if (phase == 1 && *p == q) { @@ -201,15 +251,8 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, /*******************************************************************/ /* Now start the reading process. Read one line. */ /*******************************************************************/ - if (fgets(buf, sizeof(buf), infile)) { - n = strlen(buf); - buf[n - 1] = '\0'; -#if !defined(__WIN__) - // The file can be imported from Windows - if (buf[n - 2] == '\r') - buf[n - 2] = 0; -#endif // UNIX - } else if (feof(infile)) { + if ((rc = tdbp->ReadDB(g)) == RC_OK) { + } else if (rc == RC_EF) { sprintf(g->Message, MSG(EOF_AFTER_LINE), num_read -1); break; } else { @@ -222,7 +265,7 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, /*******************************************************************/ i = n = phase = blank = digit = dec = 0; - for (p = buf; *p; p++) + for (p = tdbp->To_Line; *p; p++) if (*p == sep) { if (phase != 1) { if (i == MAXCOL - 1) { @@ -331,7 +374,7 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, htrc("\n"); } // endif trace - fclose(infile); + tdbp->CloseDB(g); skipit: if (trace) @@ -381,7 +424,7 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, return qrp; err: - fclose(infile); + tdbp->CloseDB(g); return NULL; } // end of CSVCColumns @@ -458,20 +501,27 @@ PTDB CSVDEF::GetTable(PGLOBAL g, MODE mode) /*******************************************************************/ /* Allocate a file processing class of the proper type. */ /*******************************************************************/ - if (map) { + if (Zipped) { +#if defined(ZIP_SUPPORT) + txfp = new(g) ZIPFAM(this); +#else // !ZIP_SUPPORT + strcpy(g->Message, "ZIP not supported"); + return NULL; +#endif // !ZIP_SUPPORT + } else if (map) { // Should be now compatible with UNIX txfp = new(g) MAPFAM(this); } else if (Compressed) { -#if defined(ZIP_SUPPORT) +#if defined(GZ_SUPPORT) if (Compressed == 1) - txfp = new(g) ZIPFAM(this); + txfp = new(g) GZFAM(this); else txfp = new(g) ZLBFAM(this); -#else // !ZIP_SUPPORT +#else // !GZ_SUPPORT strcpy(g->Message, "Compress not supported"); return NULL; -#endif // !ZIP_SUPPORT +#endif // !GZ_SUPPORT } else txfp = new(g) DOSFAM(this); @@ -498,7 +548,7 @@ PTDB CSVDEF::GetTable(PGLOBAL g, MODE mode) if (map) { txfp = new(g) MBKFAM(this); } else if (Compressed) { -#if defined(ZIP_SUPPORT) +#if defined(GZ_SUPPORT) if (Compressed == 1) txfp = new(g) ZBKFAM(this); else { @@ -506,7 +556,7 @@ PTDB CSVDEF::GetTable(PGLOBAL g, MODE mode) ((PZLBFAM)txfp)->SetOptimized(To_Pos != NULL); } // endelse #else - sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "GZ"); return NULL; #endif } else @@ -830,8 +880,9 @@ bool TDBCSV::SkipHeader(PGLOBAL g) /***********************************************************************/ int TDBCSV::ReadBuffer(PGLOBAL g) { - char *p1, *p2, *p = NULL; - int i, n, len, rc = Txfp->ReadBuffer(g); + //char *p1, *p2, *p = NULL; + char *p2, *p = NULL; + int i, n, len, rc = Txfp->ReadBuffer(g); bool bad = false; if (trace > 1) @@ -846,14 +897,23 @@ int TDBCSV::ReadBuffer(PGLOBAL g) for (i = 0; i < Fields; i++) { if (!bad) { if (Qot && *p2 == Qot) { // Quoted field - for (n = 0, p1 = ++p2; (p = strchr(p1, Qot)); p1 = p + 2) - if (*(p + 1) == Qot) - n++; // Doubled internal quotes - else - break; // Final quote + //for (n = 0, p1 = ++p2; (p = strchr(p1, Qot)); p1 = p + 2) + // if (*(p + 1) == Qot) + // n++; // Doubled internal quotes + // else + // break; // Final quote + + for (n = 0, p = ++p2; p; p++) + if (*p == Qot || *p == '\\') { + if (*(++p) == Qot) + n++; // Escaped internal quotes + else if (*(p - 1) == Qot) + break; // Final quote + } // endif *p if (p) { - len = p++ - p2; + //len = p++ - p2; + len = p - p2 - 1;; // if (Sep != ' ') // for (; *p == ' '; p++) ; // Skip blanks @@ -873,10 +933,12 @@ int TDBCSV::ReadBuffer(PGLOBAL g) if (n) { int j, k; - // Suppress the double of internal quotes + // Suppress the escape of internal quotes for (j = k = 0; j < len; j++, k++) { - if (p2[j] == Qot) - j++; // skip first one + if (p2[j] == Qot || (p2[j] == '\\' && p2[j + 1] == Qot)) + j++; // skip escape char + else if (p2[j] == '\\') + p2[k++] = p2[j++]; // avoid \\Qot p2[k] = p2[j]; } // endfor i, j @@ -1464,21 +1526,16 @@ void CSVCOL::WriteColumn(PGLOBAL g) /* TDBCCL class constructor. */ /***********************************************************************/ TDBCCL::TDBCCL(PCSVDEF tdp) : TDBCAT(tdp) - { - Fn = tdp->GetFn(); - Hdr = tdp->Header; - Mxr = tdp->Maxerr; - Qtd = tdp->Quoted; - Sep = tdp->Sep; - } // end of TDBCCL constructor +{ + Topt = tdp->GetTopt(); +} // end of TDBCCL constructor /***********************************************************************/ /* GetResult: Get the list the CSV file columns. */ /***********************************************************************/ PQRYRES TDBCCL::GetResult(PGLOBAL g) { - return CSVColumns(g, ((PTABDEF)To_Def)->GetPath(), - Fn, Sep, Qtd, Hdr, Mxr, false); + return CSVColumns(g, ((PTABDEF)To_Def)->GetPath(), Topt, false); } // end of GetResult /* ------------------------ End of TabFmt ---------------------------- */ diff --git a/storage/connect/tabfmt.h b/storage/connect/tabfmt.h index ce80a276cdc..5ce8d399a64 100644 --- a/storage/connect/tabfmt.h +++ b/storage/connect/tabfmt.h @@ -1,7 +1,7 @@ /*************** TabFmt H Declares Source Code File (.H) ***************/ -/* Name: TABFMT.H Version 2.4 */ +/* Name: TABFMT.H Version 2.5 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 2001-2014 */ +/* (C) Copyright to the author Olivier BERTRAND 2001-2016 */ /* */ /* This file contains the CSV and FMT classes declares. */ /***********************************************************************/ @@ -13,8 +13,7 @@ typedef class TDBFMT *PTDBFMT; /***********************************************************************/ /* Functions used externally. */ /***********************************************************************/ -PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, - char q, int hdr, int mxr, bool info); +PQRYRES CSVColumns(PGLOBAL g, char *dp, PTOS topt, bool info); /***********************************************************************/ /* CSV table. */ @@ -22,7 +21,8 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, class DllExport CSVDEF : public DOSDEF { /* Logical table description */ friend class TDBCSV; friend class TDBCCL; - public: + friend PQRYRES CSVColumns(PGLOBAL, char *, PTOS, bool); +public: // Constructor CSVDEF(void); @@ -50,9 +50,10 @@ class DllExport CSVDEF : public DOSDEF { /* Logical table description */ /* This is the DOS/UNIX Access Method class declaration for files */ /* that are CSV files with columns separated by the Sep character. */ /***********************************************************************/ -class TDBCSV : public TDBDOS { +class DllExport TDBCSV : public TDBDOS { friend class CSVCOL; - public: + friend PQRYRES CSVColumns(PGLOBAL, char *, PTOS, bool); +public: // Constructor TDBCSV(PCSVDEF tdp, PTXF txfp); TDBCSV(PGLOBAL g, PTDBCSV tdbp); @@ -101,7 +102,7 @@ class TDBCSV : public TDBDOS { /* Class CSVCOL: CSV access method column descriptor. */ /* This A.M. is used for Comma Separated V(?) files. */ /***********************************************************************/ -class CSVCOL : public DOSCOL { +class DllExport CSVCOL : public DOSCOL { friend class TDBCSV; friend class TDBFMT; public: @@ -129,7 +130,7 @@ class CSVCOL : public DOSCOL { /* This is the DOS/UNIX Access Method class declaration for files */ /* whose record format is described by a Format keyword. */ /***********************************************************************/ -class TDBFMT : public TDBCSV { +class DllExport TDBFMT : public TDBCSV { friend class CSVCOL; //friend class FMTCOL; public: @@ -173,7 +174,7 @@ class TDBFMT : public TDBCSV { /***********************************************************************/ /* This is the class declaration for the CSV catalog table. */ /***********************************************************************/ -class TDBCCL : public TDBCAT { +class DllExport TDBCCL : public TDBCAT { public: // Constructor TDBCCL(PCSVDEF tdp); @@ -183,11 +184,7 @@ class TDBCCL : public TDBCAT { virtual PQRYRES GetResult(PGLOBAL g); // Members - char *Fn; // The CSV file (path) name - bool Hdr; // true if first line contains headers - int Mxr; // Maximum number of bad records - int Qtd; // Quoting level for quoted fields - char Sep; // Separator for standard CSV files - }; // end of class TDBCCL + PTOS Topt; +}; // end of class TDBCCL /* ------------------------- End of TabFmt.H ------------------------- */ diff --git a/storage/connect/tabjdbc.cpp b/storage/connect/tabjdbc.cpp index 93ba3ca264d..c36aa080956 100644 --- a/storage/connect/tabjdbc.cpp +++ b/storage/connect/tabjdbc.cpp @@ -110,6 +110,7 @@ bool JDBCDEF::SetParms(PJPARM sjp) sjp->Url= Url; sjp->User= Username; sjp->Pwd= Password; +//sjp->Properties = Prop; return true; } // end of SetParms @@ -234,6 +235,7 @@ bool JDBCDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) Read_Only = true; Wrapname = GetStringCatInfo(g, "Wrapper", NULL); +//Prop = GetStringCatInfo(g, "Properties", NULL); Tabcat = GetStringCatInfo(g, "Qualifier", NULL); Tabcat = GetStringCatInfo(g, "Catalog", Tabcat); Tabschema = GetStringCatInfo(g, "Dbname", NULL); @@ -337,6 +339,7 @@ TDBJDBC::TDBJDBC(PJDBCDEF tdp) : TDBASE(tdp) Schema = tdp->Tabschema; Ops.User = tdp->Username; Ops.Pwd = tdp->Password; +// Ops.Properties = tdp->Prop; Catalog = tdp->Tabcat; Srcdef = tdp->Srcdef; Qrystr = tdp->Qrystr; @@ -356,6 +359,7 @@ TDBJDBC::TDBJDBC(PJDBCDEF tdp) : TDBASE(tdp) Ops.Url = NULL; Ops.User = NULL; Ops.Pwd = NULL; +// Ops.Properties = NULL; Catalog = NULL; Srcdef = NULL; Qrystr = NULL; diff --git a/storage/connect/tabjdbc.h b/storage/connect/tabjdbc.h index 7244ebd3832..fee8223abaf 100644 --- a/storage/connect/tabjdbc.h +++ b/storage/connect/tabjdbc.h @@ -58,6 +58,7 @@ protected: PSZ Tabschema; /* External table schema */ PSZ Username; /* User connect name */ PSZ Password; /* Password connect info */ +//PSZ Prop; /* Connection Properties */ PSZ Tabcat; /* External table catalog */ PSZ Tabtype; /* External table type */ PSZ Colpat; /* Catalog column pattern */ diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp index 17260836371..1b9ce8b64c9 100644 --- a/storage/connect/tabjson.cpp +++ b/storage/connect/tabjson.cpp @@ -1,6 +1,6 @@ /************* tabjson C++ Program Source Code File (.CPP) *************/ -/* PROGRAM NAME: tabjson Version 1.1 */ -/* (C) Copyright to the author Olivier BERTRAND 2014 - 2015 */ +/* PROGRAM NAME: tabjson Version 1.3 */ +/* (C) Copyright to the author Olivier BERTRAND 2014 - 2016 */ /* This program are the JSON class DB execution routines. */ /***********************************************************************/ @@ -25,6 +25,9 @@ //#include "resource.h" // for IDS_COLUMNS #include "tabjson.h" #include "filamap.h" +#if defined(GZ_SUPPORT) +#include "filamgz.h" +#endif // GZ_SUPPORT #if defined(ZIP_SUPPORT) #include "filamzip.h" #endif // ZIP_SUPPORT @@ -67,7 +70,7 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info) static XFLD fldtyp[] = {FLD_NAME, FLD_TYPE, FLD_TYPENAME, FLD_PREC, FLD_LENGTH, FLD_SCALE, FLD_NULL, FLD_FORMAT}; static unsigned int length[] = {0, 6, 8, 10, 10, 6, 6, 0}; - char *fn, colname[65], fmt[129]; + char colname[65], fmt[129]; int i, j, lvl, n = 0; int ncol = sizeof(buftyp) / sizeof(int); PVAL valp; @@ -91,19 +94,29 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info) goto skipit; } // endif info - /*********************************************************************/ + if (GetIntegerTableOption(g, topt, "Multiple", 0)) { + strcpy(g->Message, "Cannot find column definition for multiple table"); + return NULL; + } // endif Multiple + + /*********************************************************************/ /* Open the input file. */ /*********************************************************************/ - if (!(fn = GetStringTableOption(g, topt, "Filename", NULL))) { - strcpy(g->Message, MSG(MISSING_FNAME)); - return NULL; - } else { - lvl = GetIntegerTableOption(g, topt, "Level", 0); - lvl = (lvl < 0) ? 0 : (lvl > 16) ? 16 : lvl; - } // endif fn + lvl = GetIntegerTableOption(g, topt, "Level", 0); + lvl = (lvl < 0) ? 0 : (lvl > 16) ? 16 : lvl; tdp = new(g) JSONDEF; - tdp->Fn = fn; +#if defined(ZIP_SUPPORT) + tdp->Entry = GetStringTableOption(g, topt, "Entry", NULL); + tdp->Zipped = GetBooleanTableOption(g, topt, "Zipped", false); +#endif // ZIP_SUPPORT + tdp->Fn = GetStringTableOption(g, topt, "Filename", NULL); + + if (!tdp->Fn) { + strcpy(g->Message, MSG(MISSING_FNAME)); + return NULL; + } // endif Fn + tdp->Database = SetPath(g, db); tdp->Objname = GetStringTableOption(g, topt, "Object", NULL); tdp->Base = GetIntegerTableOption(g, topt, "Base", 0) ? 1 : 0; @@ -114,7 +127,15 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info) tdp->Fn, tdp->Objname, tdp->Pretty, lvl); if (tdp->Pretty == 2) { - tjsp = new(g) TDBJSON(tdp, new(g) MAPFAM(tdp)); + if (tdp->Zipped) { +#if defined(ZIP_SUPPORT) + tjsp = new(g) TDBJSON(tdp, new(g) ZIPFAM(tdp)); +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif // !ZIP_SUPPORT + } else + tjsp = new(g) TDBJSON(tdp, new(g) MAPFAM(tdp)); if (tjsp->MakeDocument(g)) return NULL; @@ -127,10 +148,33 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info) } // endif lrecl tdp->Ending = GetIntegerTableOption(g, topt, "Ending", CRLF); - tjnp = new(g) TDBJSN(tdp, new(g) DOSFAM(tdp)); + + if (tdp->Zipped) { +#if defined(ZIP_SUPPORT) + tjnp = new(g)TDBJSN(tdp, new(g)ZIPFAM(tdp)); +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif // !ZIP_SUPPORT + } else + tjnp = new(g) TDBJSN(tdp, new(g) DOSFAM(tdp)); + tjnp->SetMode(MODE_READ); - if (tjnp->OpenDB(g)) +#if USE_G + // Allocate the parse work memory + PGLOBAL G = (PGLOBAL)PlugSubAlloc(g, NULL, sizeof(GLOBAL)); + memset(G, 0, sizeof(GLOBAL)); + G->Sarea_Size = tdp->Lrecl * 10; + G->Sarea = PlugSubAlloc(g, NULL, G->Sarea_Size); + PlugSubSet(G, G->Sarea, G->Sarea_Size); + G->jump_level = -1; + tjnp->SetG(G); +#else + tjnp->SetG(g); +#endif + + if (tjnp->OpenDB(g)) return NULL; switch (tjnp->ReadDB(g)) { @@ -395,16 +439,23 @@ PTDB JSONDEF::GetTable(PGLOBAL g, MODE m) !(tmp == TMP_FORCE && (m == MODE_UPDATE || m == MODE_DELETE)); - if (Compressed) { + if (Zipped) { #if defined(ZIP_SUPPORT) + txfp = new(g) ZIPFAM(this); +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif // !ZIP_SUPPORT + } else if (Compressed) { +#if defined(GZ_SUPPORT) if (Compressed == 1) - txfp = new(g) ZIPFAM(this); + txfp = new(g) GZFAM(this); else txfp = new(g) ZLBFAM(this); -#else // !ZIP_SUPPORT - sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); +#else // !GZ_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "GZ"); return NULL; -#endif // !ZIP_SUPPORT +#endif // !GZ_SUPPORT } else if (map) txfp = new(g) MAPFAM(this); else @@ -426,7 +477,16 @@ PTDB JSONDEF::GetTable(PGLOBAL g, MODE m) ((TDBJSN*)tdbp)->G = g; #endif } else { - txfp = new(g) MAPFAM(this); + if (Zipped) { +#if defined(ZIP_SUPPORT) + txfp = new(g)ZIPFAM(this); +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif // !ZIP_SUPPORT + } else + txfp = new(g) MAPFAM(this); + tdbp = new(g) TDBJSON(this, txfp); ((TDBJSON*)tdbp)->G = g; } // endif Pretty diff --git a/storage/connect/tabjson.h b/storage/connect/tabjson.h index f7cb74c3c4d..c9d30d48f2a 100644 --- a/storage/connect/tabjson.h +++ b/storage/connect/tabjson.h @@ -78,7 +78,8 @@ public: virtual AMT GetAmType(void) {return TYPE_AM_JSN;} virtual bool SkipHeader(PGLOBAL g); virtual PTDB Duplicate(PGLOBAL g) {return (PTDB)new(g) TDBJSN(this);} - PJSON GetRow(void) {return Row;} + PJSON GetRow(void) {return Row;} + void SetG(PGLOBAL g) {G = g;} // Methods virtual PTDB CopyOne(PTABS t); diff --git a/storage/connect/tabxml.cpp b/storage/connect/tabxml.cpp index 57d204a4286..3b8229fcf51 100644 --- a/storage/connect/tabxml.cpp +++ b/storage/connect/tabxml.cpp @@ -1,9 +1,9 @@ /************* Tabxml C++ Program Source Code File (.CPP) **************/ /* PROGRAM NAME: TABXML */ /* ------------- */ -/* Version 2.8 */ +/* Version 2.9 */ /* */ -/* Author Olivier BERTRAND 2007 - 2015 */ +/* Author Olivier BERTRAND 2007 - 2016 */ /* */ /* This program are the XML tables classes using MS-DOM or libxml2. */ /***********************************************************************/ @@ -136,7 +136,12 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info) goto skipit; } // endif info - /*********************************************************************/ + if (GetIntegerTableOption(g, topt, "Multiple", 0)) { + strcpy(g->Message, "Cannot find column definition for multiple table"); + return NULL; + } // endif Multiple + + /*********************************************************************/ /* Open the input file. */ /*********************************************************************/ if (!(fn = GetStringTableOption(g, topt, "Filename", NULL))) { @@ -154,6 +159,8 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info) tdp->Fn = fn; tdp->Database = SetPath(g, db); tdp->Tabname = tab; + tdp->Zipped = GetBooleanTableOption(g, topt, "Zipped", false); + tdp->Entry = GetStringTableOption(g, topt, "Entry", NULL); if (!(op = GetStringTableOption(g, topt, "Xmlsup", NULL))) #if defined(__WIN__) @@ -204,7 +211,8 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info) while (true) { if (!vp->atp && - !(node = (vp->nl) ? vp->nl->GetItem(g, vp->k++, node) : NULL)) + !(node = (vp->nl) ? vp->nl->GetItem(g, vp->k++, tdp->Usedom ? node : NULL) + : NULL)) if (j) { vp = lvlp[--j]; @@ -254,7 +262,8 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info) if (j < lvl && ok) { vp = lvlp[j+1]; vp->k = 0; - vp->atp = node->GetAttribute(g, NULL); + vp->pn = node; + vp->atp = node->GetAttribute(g, NULL); vp->nl = node->GetChildElements(g); if (tdp->Usedom && vp->nl->GetLength() == 1) { @@ -265,7 +274,7 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info) if (vp->atp || vp->b) { if (!vp->atp) - node = vp->nl->GetItem(g, vp->k++, node); + node = vp->nl->GetItem(g, vp->k++, tdp->Usedom ? node : NULL); strncat(fmt, colname, XLEN(fmt)); strncat(fmt, "/", XLEN(fmt)); @@ -424,11 +433,14 @@ XMLDEF::XMLDEF(void) DefNs = NULL; Attrib = NULL; Hdattr = NULL; + Entry = NULL; Coltype = 1; Limit = 0; Header = 0; Xpand = false; Usedom = false; + Zipped = false; + Mulentries = false; } // end of XMLDEF constructor /***********************************************************************/ @@ -507,7 +519,14 @@ bool XMLDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) // Get eventual table node attribute Attrib = GetStringCatInfo(g, "Attribute", NULL); Hdattr = GetStringCatInfo(g, "HeadAttr", NULL); - return false; + + // Specific for zipped files + if ((Zipped = GetBoolCatInfo("Zipped", false))) + Mulentries = ((Entry = GetStringCatInfo(g, "Entry", NULL))) + ? strchr(Entry, '*') || strchr(Entry, '?') + : GetBoolCatInfo("Mulentries", false); + + return false; } // end of DefineAM /***********************************************************************/ @@ -547,6 +566,7 @@ TDBXML::TDBXML(PXMLDEF tdp) : TDBASE(tdp) Xfile = tdp->Fn; Enc = tdp->Encoding; Tabname = tdp->Tabname; +#if 0 // why all these? Rowname = (tdp->Rowname) ? tdp->Rowname : NULL; Colname = (tdp->Colname) ? tdp->Colname : NULL; Mulnode = (tdp->Mulnode) ? tdp->Mulnode : NULL; @@ -555,10 +575,22 @@ TDBXML::TDBXML(PXMLDEF tdp) : TDBASE(tdp) DefNs = (tdp->DefNs) ? tdp->DefNs : NULL; Attrib = (tdp->Attrib) ? tdp->Attrib : NULL; Hdattr = (tdp->Hdattr) ? tdp->Hdattr : NULL; - Coltype = tdp->Coltype; +#endif // 0 + Rowname = tdp->Rowname; + Colname = tdp->Colname; + Mulnode = tdp->Mulnode; + XmlDB = tdp->XmlDB; + Nslist = tdp->Nslist; + DefNs = tdp->DefNs; + Attrib = tdp->Attrib; + Hdattr = tdp->Hdattr; + Entry = tdp->Entry; + Coltype = tdp->Coltype; Limit = tdp->Limit; Xpand = tdp->Xpand; - Changed = false; + Zipped = tdp->Zipped; + Mulentries = tdp->Mulentries; + Changed = false; Checked = false; NextSame = false; NewRow = false; @@ -600,10 +632,13 @@ TDBXML::TDBXML(PTDBXML tdbp) : TDBASE(tdbp) DefNs = tdbp->DefNs; Attrib = tdbp->Attrib; Hdattr = tdbp->Hdattr; - Coltype = tdbp->Coltype; + Entry = tdbp->Entry; + Coltype = tdbp->Coltype; Limit = tdbp->Limit; Xpand = tdbp->Xpand; - Changed = tdbp->Changed; + Zipped = tdbp->Zipped; + Mulentries = tdbp->Mulentries; + Changed = tdbp->Changed; Checked = tdbp->Checked; NextSame = tdbp->NextSame; NewRow = tdbp->NewRow; @@ -681,7 +716,7 @@ int TDBXML::LoadTableFile(PGLOBAL g, char *filename) /*********************************************************************/ /* Firstly we check whether this file have been already loaded. */ /*********************************************************************/ - if (Mode == MODE_READ || Mode == MODE_ANY) + if ((Mode == MODE_READ || Mode == MODE_ANY) && !Zipped) for (fp = dup->Openlist; fp; fp = fp->Next) if (fp->Type == type && fp->Length && fp->Count) if (!stricmp(fp->Fname, filename)) @@ -703,7 +738,7 @@ int TDBXML::LoadTableFile(PGLOBAL g, char *filename) return RC_FX; // Initialize the implementation - if (Docp->Initialize(g)) { + if (Docp->Initialize(g, Entry, Zipped)) { sprintf(g->Message, MSG(INIT_FAILED), (Usedom) ? "DOM" : "libxml2"); return RC_FX; } // endif init @@ -712,7 +747,7 @@ int TDBXML::LoadTableFile(PGLOBAL g, char *filename) htrc("TDBXML: parsing %s rc=%d\n", filename, rc); // Parse the XML file - if (Docp->ParseFile(filename)) { + if (Docp->ParseFile(g, filename)) { // Does the file exist? int h= global_open(g, MSGID_NONE, filename, _O_RDONLY); diff --git a/storage/connect/tabxml.h b/storage/connect/tabxml.h index 7ba3166881d..6c586d79dec 100644 --- a/storage/connect/tabxml.h +++ b/storage/connect/tabxml.h @@ -1,7 +1,7 @@ /*************** Tabxml H Declares Source Code File (.H) ***************/ -/* Name: TABXML.H Version 1.6 */ +/* Name: TABXML.H Version 1.7 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 2007-2015 */ +/* (C) Copyright to the author Olivier BERTRAND 2007-2016 */ /* */ /* This file contains the XML table classes declares. */ /***********************************************************************/ @@ -42,12 +42,15 @@ class DllExport XMLDEF : public TABDEF { /* Logical table description */ char *DefNs; /* Dummy name of default namespace */ char *Attrib; /* Table node attributes */ char *Hdattr; /* Header node attributes */ - int Coltype; /* Default column type */ + char *Entry; /* Zip entry name or pattern */ + int Coltype; /* Default column type */ int Limit; /* Limit of multiple values */ int Header; /* n first rows are header rows */ bool Xpand; /* Put multiple tags in several rows */ bool Usedom; /* True: DOM, False: libxml2 */ - }; // end of XMLDEF + bool Zipped; /* True: Zipped XML file(s) */ + bool Mulentries; /* True: multiple entries in zip file*/ +}; // end of XMLDEF #if defined(INCLUDE_TDBXML) /***********************************************************************/ @@ -122,7 +125,9 @@ class DllExport TDBXML : public TDBASE { bool Bufdone; // True when column buffers allocated bool Nodedone; // True when column nodes allocated bool Void; // True if the file does not exist - char *Xfile; // The XML file + bool Zipped; // True if Zipped XML file(s) + bool Mulentries; // True if multiple entries in zip file + char *Xfile; // The XML file char *Enc; // New XML table file encoding char *Tabname; // Name of Table node char *Rowname; // Name of first level nodes @@ -133,7 +138,8 @@ class DllExport TDBXML : public TDBASE { char *DefNs; // Dummy name of default namespace char *Attrib; // Table node attribut(s) char *Hdattr; // Header node attribut(s) - int Coltype; // Default column type + char *Entry; // Zip entry name or pattern + int Coltype; // Default column type int Limit; // Limit of multiple values int Header; // n first rows are header rows int Multiple; // If multiple files diff --git a/storage/connect/tabzip.cpp b/storage/connect/tabzip.cpp new file mode 100644 index 00000000000..11f414ee154 --- /dev/null +++ b/storage/connect/tabzip.cpp @@ -0,0 +1,230 @@ +/************* TabZip C++ Program Source Code File (.CPP) **************/ +/* PROGRAM NAME: TABZIP Version 1.0 */ +/* (C) Copyright to the author Olivier BERTRAND 2016 */ +/* This program are the TABZIP class DB execution routines. */ +/***********************************************************************/ + +/***********************************************************************/ +/* Include relevant sections of the MariaDB header file. */ +/***********************************************************************/ +#include <my_global.h> + +/***********************************************************************/ +/* Include application header files: */ +/* global.h is header containing all global declarations. */ +/* plgdbsem.h is header containing the DB application declarations. */ +/* (x)table.h is header containing the TDBASE declarations. */ +/* tabzip.h is header containing the TABZIP classes declarations. */ +/***********************************************************************/ +#include "global.h" +#include "plgdbsem.h" +#include "xtable.h" +#include "filamtxt.h" +#include "filamzip.h" +#include "resource.h" // for IDS_COLUMNS +#include "tabdos.h" +#include "tabzip.h" + +/* -------------------------- Class ZIPDEF --------------------------- */ + +/************************************************************************/ +/* DefineAM: define specific AM block values. */ +/************************************************************************/ +bool ZIPDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) +{ +//target = GetStringCatInfo(g, "Target", NULL); + return DOSDEF::DefineAM(g, "ZIP", poff); +} // end of DefineAM + +/***********************************************************************/ +/* GetTable: makes a new Table Description Block. */ +/***********************************************************************/ +PTDB ZIPDEF::GetTable(PGLOBAL g, MODE m) +{ + return new(g) TDBZIP(this); +} // end of GetTable + +/* ------------------------------------------------------------------- */ + +/***********************************************************************/ +/* Implementation of the TDBZIP class. */ +/***********************************************************************/ +TDBZIP::TDBZIP(PZIPDEF tdp) : TDBASE(tdp) +{ + zipfile = NULL; + zfn = tdp->Fn; +//target = tdp->target; + nexterr = UNZ_OK; +} // end of TDBZIP standard constructor + +/***********************************************************************/ +/* Allocate ZIP column description block. */ +/***********************************************************************/ +PCOL TDBZIP::MakeCol(PGLOBAL g, PCOLDEF cdp, PCOL cprec, int n) +{ + return new(g) ZIPCOL(cdp, this, cprec, n); +} // end of MakeCol + +/***********************************************************************/ +/* open a zip file. */ +/* param: filename path and the filename of the zip file to open. */ +/* return: true if open, false otherwise. */ +/***********************************************************************/ +bool TDBZIP::open(PGLOBAL g, const char *filename) +{ + if (!zipfile && !(zipfile = unzOpen64(filename))) + sprintf(g->Message, "Zipfile open error"); + + return (zipfile == NULL); +} // end of open + +/***********************************************************************/ +/* Close the zip file. */ +/***********************************************************************/ +void TDBZIP::close() +{ + if (zipfile) { + unzClose(zipfile); + zipfile = NULL; + } // endif zipfile + +} // end of close + +/***********************************************************************/ +/* ZIP Cardinality: returns table size in number of rows. */ +/***********************************************************************/ +int TDBZIP::Cardinality(PGLOBAL g) +{ + if (!g) + return 1; + else if (Cardinal < 0) { + if (!open(g, zfn)) { + unz_global_info64 ginfo; + int err = unzGetGlobalInfo64(zipfile, &ginfo); + + Cardinal = (err == UNZ_OK) ? ginfo.number_entry : 0; + } else + Cardinal = 0; + + } // endif Cardinal + + return Cardinal; +} // end of Cardinality + +/***********************************************************************/ +/* ZIP GetMaxSize: returns file size estimate in number of lines. */ +/***********************************************************************/ +int TDBZIP::GetMaxSize(PGLOBAL g) +{ + if (MaxSize < 0) + MaxSize = Cardinality(g); + + return MaxSize; +} // end of GetMaxSize + +/***********************************************************************/ +/* ZIP Access Method opening routine. */ +/***********************************************************************/ +bool TDBZIP::OpenDB(PGLOBAL g) +{ + if (Use == USE_OPEN) + // Table already open + return false; + + Use = USE_OPEN; // To be clean + return open(g, zfn); +} // end of OpenDB + +/***********************************************************************/ +/* ReadDB: Data Base read routine for ZIP access method. */ +/***********************************************************************/ +int TDBZIP::ReadDB(PGLOBAL g) +{ + if (nexterr == UNZ_END_OF_LIST_OF_FILE) + return RC_EF; + else if (nexterr != UNZ_OK) { + sprintf(g->Message, "unzGoToNextFile error %d", nexterr); + return RC_FX; + } // endif nexterr + + int err = unzGetCurrentFileInfo64(zipfile, &finfo, fn, + sizeof(fn), NULL, 0, NULL, 0); + + if (err != UNZ_OK) { + sprintf(g->Message, "unzGetCurrentFileInfo64 error %d", err); + return RC_FX; + } // endif err + + nexterr = unzGoToNextFile(zipfile); + return RC_OK; +} // end of ReadDB + +/***********************************************************************/ +/* WriteDB: Data Base write routine for ZIP access method. */ +/***********************************************************************/ +int TDBZIP::WriteDB(PGLOBAL g) +{ + strcpy(g->Message, "ZIP tables are read only"); + return RC_FX; +} // end of WriteDB + +/***********************************************************************/ +/* Data Base delete line routine for ZIP access method. */ +/***********************************************************************/ +int TDBZIP::DeleteDB(PGLOBAL g, int irc) +{ + strcpy(g->Message, "Delete not enabled for ZIP tables"); + return RC_FX; +} // end of DeleteDB + +/***********************************************************************/ +/* Data Base close routine for ZIP access method. */ +/***********************************************************************/ +void TDBZIP::CloseDB(PGLOBAL g) +{ + close(); + Use = USE_READY; // Just to be clean +} // end of CloseDB + +/* ---------------------------- ZIPCOL ------------------------------- */ + +/***********************************************************************/ +/* ZIPCOL public constructor. */ +/***********************************************************************/ +ZIPCOL::ZIPCOL(PCOLDEF cdp, PTDB tdbp, PCOL cprec, int i, PSZ am) + : COLBLK(cdp, tdbp, i) +{ + if (cprec) { + Next = cprec->GetNext(); + cprec->SetNext(this); + } else { + Next = tdbp->GetColumns(); + tdbp->SetColumns(this); + } // endif cprec + + Tdbz = (TDBZIP*)tdbp; + flag = cdp->GetOffset(); +} // end of ZIPCOL constructor + +/***********************************************************************/ +/* ReadColumn: */ +/***********************************************************************/ +void ZIPCOL::ReadColumn(PGLOBAL g) +{ + switch (flag) { + case 1: + Value->SetValue(Tdbz->finfo.compressed_size); + break; + case 2: + Value->SetValue(Tdbz->finfo.uncompressed_size); + break; + case 3: + Value->SetValue((int)Tdbz->finfo.compression_method); + break; + default: + Value->SetValue_psz((PSZ)Tdbz->fn); + } // endswitch flag + +} // end of ReadColumn + +/* -------------------------- End of tabzip -------------------------- */ diff --git a/storage/connect/tabzip.h b/storage/connect/tabzip.h new file mode 100644 index 00000000000..6f1735258e7 --- /dev/null +++ b/storage/connect/tabzip.h @@ -0,0 +1,100 @@ +/*************** tabzip H Declares Source Code File (.H) ***************/ +/* Name: tabzip.h Version 1.0 */ +/* */ +/* (C) Copyright to the author Olivier BERTRAND 2016 */ +/* */ +/* This file contains the ZIP classe declares. */ +/***********************************************************************/ +#include "osutil.h" +#include "block.h" +#include "colblk.h" +#include "xtable.h" +#include "unzip.h" + +typedef class ZIPDEF *PZIPDEF; +typedef class TDBZIP *PTDBZIP; +typedef class ZIPCOL *PZIPCOL; + +/***********************************************************************/ +/* ZIP table: display info about a ZIP file. */ +/***********************************************************************/ +class DllExport ZIPDEF : public DOSDEF { /* Table description */ + friend class TDBZIP; + friend class ZIPFAM; +public: + // Constructor + ZIPDEF(void) {} + + // Implementation + virtual const char *GetType(void) {return "ZIP";} + + // Methods + virtual bool DefineAM(PGLOBAL g, LPCSTR am, int poff); + virtual PTDB GetTable(PGLOBAL g, MODE m); + +protected: + // Members + PSZ target; // The inside file to query +}; // end of ZIPDEF + +/***********************************************************************/ +/* This is the ZIP Access Method class declaration. */ +/***********************************************************************/ +class DllExport TDBZIP : public TDBASE { + friend class ZIPCOL; +public: + // Constructor + TDBZIP(PZIPDEF tdp); + + // Implementation + virtual AMT GetAmType(void) {return TYPE_AM_ZIP;} + + // Methods + virtual PCOL MakeCol(PGLOBAL g, PCOLDEF cdp, PCOL cprec, int n); + virtual int Cardinality(PGLOBAL g); + virtual int GetMaxSize(PGLOBAL g); + virtual int GetRecpos(void) {return 0;} + + // Database routines + virtual bool OpenDB(PGLOBAL g); + virtual int ReadDB(PGLOBAL g); + virtual int WriteDB(PGLOBAL g); + virtual int DeleteDB(PGLOBAL g, int irc); + virtual void CloseDB(PGLOBAL g); + +protected: + bool open(PGLOBAL g, const char *filename); + void close(void); + + // Members + unzFile zipfile; // The ZIP container file + PSZ zfn; // The ZIP file name +//PSZ target; + unz_file_info64 finfo; // The current file info + char fn[FILENAME_MAX]; // The current file name + int nexterr; // Next file error +}; // end of class TDBZIP + +/***********************************************************************/ +/* Class ZIPCOL: ZIP access method column descriptor. */ +/***********************************************************************/ +class DllExport ZIPCOL : public COLBLK { + friend class TDBZIP; +public: + // Constructors + ZIPCOL(PCOLDEF cdp, PTDB tdbp, PCOL cprec, int i, PSZ am = "ZIP"); + + // Implementation + virtual int GetAmType(void) { return TYPE_AM_ZIP; } + + // Methods + virtual void ReadColumn(PGLOBAL g); + +protected: + // Default constructor not to be used + ZIPCOL(void) {} + + // Members + TDBZIP *Tdbz; + int flag; +}; // end of class ZIPCOL diff --git a/storage/connect/unzip.c b/storage/connect/unzip.c new file mode 100644 index 00000000000..909350435a5 --- /dev/null +++ b/storage/connect/unzip.c @@ -0,0 +1,2125 @@ +/* unzip.c -- IO for uncompress .zip files using zlib + Version 1.1, February 14h, 2010 + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications of Unzip for Zip64 + Copyright (C) 2007-2008 Even Rouault + + Modifications for Zip64 support on both zip and unzip + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + + + ------------------------------------------------------------------------------------ + Decryption code comes from crypt.c by Info-ZIP but has been greatly reduced in terms of + compatibility with older software. The following is from the original crypt.c. + Code woven in by Terry Thorsen 1/2003. + + Copyright (c) 1990-2000 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2000-Apr-09 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, all these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html + + crypt.c (full version) by Info-ZIP. Last revised: [see crypt.h] + + The encryption/decryption parts of this source code (as opposed to the + non-echoing password parts) were originally written in Europe. The + whole source package can be freely distributed, including from the USA. + (Prior to January 2000, re-export from the US was a violation of US law.) + + This encryption code is a direct transcription of the algorithm from + Roger Schlafly, described by Phil Katz in the file appnote.txt. This + file (appnote.txt) is distributed with the PKZIP program (even in the + version without encryption capabilities). + + ------------------------------------------------------------------------------------ + + Changes in unzip.c + + 2007-2008 - Even Rouault - Addition of cpl_unzGetCurrentFileZStreamPos + 2007-2008 - Even Rouault - Decoration of symbol names unz* -> cpl_unz* + 2007-2008 - Even Rouault - Remove old C style function prototypes + 2007-2008 - Even Rouault - Add unzip support for ZIP64 + + Copyright (C) 2007-2008 Even Rouault + + + Oct-2009 - Mathias Svensson - Removed cpl_* from symbol names (Even Rouault added them but since this is now moved to a new project (minizip64) I renamed them again). + Oct-2009 - Mathias Svensson - Fixed problem if uncompressed size was > 4G and compressed size was <4G + should only read the compressed/uncompressed size from the Zip64 format if + the size from normal header was 0xFFFFFFFF + Oct-2009 - Mathias Svensson - Applied some bug fixes from paches recived from Gilles Vollant + Oct-2009 - Mathias Svensson - Applied support to unzip files with compression mathod BZIP2 (bzip2 lib is required) + Patch created by Daniel Borca + + Jan-2010 - back to unzip and minizip 1.0 name scheme, with compatibility layer + + Copyright (C) 1998 - 2010 Gilles Vollant, Even Rouault, Mathias Svensson + +*/ + + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#ifndef NOUNCRYPT + #define NOUNCRYPT +#endif + +#include "zlib.h" +#include "unzip.h" + +#ifdef STDC +# include <stddef.h> +# include <string.h> +# include <stdlib.h> +#endif +#ifdef NO_ERRNO_H + extern int errno; +#else +# include <errno.h> +#endif + + +#ifndef local +# define local static +#endif +/* compile with -Dlocal if your debugger can't find static symbols */ + + +#ifndef CASESENSITIVITYDEFAULT_NO +# if !defined(unix) && !defined(CASESENSITIVITYDEFAULT_YES) +# define CASESENSITIVITYDEFAULT_NO +# endif +#endif + + +#ifndef UNZ_BUFSIZE +#define UNZ_BUFSIZE (16384) +#endif + +#ifndef UNZ_MAXFILENAMEINZIP +#define UNZ_MAXFILENAMEINZIP (256) +#endif + +#ifndef ALLOC +# define ALLOC(size) (malloc(size)) +#endif +#ifndef TRYFREE +# define TRYFREE(p) {if (p) free(p);} +#endif + +#define SIZECENTRALDIRITEM (0x2e) +#define SIZEZIPLOCALHEADER (0x1e) + + +const char unz_copyright[] = + " unzip 1.01 Copyright 1998-2004 Gilles Vollant - http://www.winimage.com/zLibDll"; + +/* unz_file_info_interntal contain internal info about a file in zipfile*/ +typedef struct unz_file_info64_internal_s +{ + ZPOS64_T offset_curfile;/* relative offset of local header 8 bytes */ +} unz_file_info64_internal; + + +/* file_in_zip_read_info_s contain internal information about a file in zipfile, + when reading and decompress it */ +typedef struct +{ + char *read_buffer; /* internal buffer for compressed data */ + z_stream stream; /* zLib stream structure for inflate */ + +#ifdef HAVE_BZIP2 + bz_stream bstream; /* bzLib stream structure for bziped */ +#endif + + ZPOS64_T pos_in_zipfile; /* position in byte on the zipfile, for fseek*/ + uLong stream_initialised; /* flag set if stream structure is initialised*/ + + ZPOS64_T offset_local_extrafield;/* offset of the local extra field */ + uInt size_local_extrafield;/* size of the local extra field */ + ZPOS64_T pos_local_extrafield; /* position in the local extra field in read*/ + ZPOS64_T total_out_64; + + uLong crc32; /* crc32 of all data uncompressed */ + uLong crc32_wait; /* crc32 we must obtain after decompress all */ + ZPOS64_T rest_read_compressed; /* number of byte to be decompressed */ + ZPOS64_T rest_read_uncompressed;/*number of byte to be obtained after decomp*/ + zlib_filefunc64_32_def z_filefunc; + voidpf filestream; /* io structore of the zipfile */ + uLong compression_method; /* compression method (0==store) */ + ZPOS64_T byte_before_the_zipfile;/* byte before the zipfile, (>0 for sfx)*/ + int raw; +} file_in_zip64_read_info_s; + + +/* unz64_s contain internal information about the zipfile +*/ +typedef struct +{ + zlib_filefunc64_32_def z_filefunc; + int is64bitOpenFunction; + voidpf filestream; /* io structore of the zipfile */ + unz_global_info64 gi; /* public global information */ + ZPOS64_T byte_before_the_zipfile;/* byte before the zipfile, (>0 for sfx)*/ + ZPOS64_T num_file; /* number of the current file in the zipfile*/ + ZPOS64_T pos_in_central_dir; /* pos of the current file in the central dir*/ + ZPOS64_T current_file_ok; /* flag about the usability of the current file*/ + ZPOS64_T central_pos; /* position of the beginning of the central dir*/ + + ZPOS64_T size_central_dir; /* size of the central directory */ + ZPOS64_T offset_central_dir; /* offset of start of central directory with + respect to the starting disk number */ + + unz_file_info64 cur_file_info; /* public info about the current file in zip*/ + unz_file_info64_internal cur_file_info_internal; /* private info about it*/ + file_in_zip64_read_info_s* pfile_in_zip_read; /* structure about the current + file if we are decompressing it */ + int encrypted; + + int isZip64; + +# ifndef NOUNCRYPT + unsigned long keys[3]; /* keys defining the pseudo-random sequence */ + const z_crc_t* pcrc_32_tab; +# endif +} unz64_s; + + +#ifndef NOUNCRYPT +#include "crypt.h" +#endif + +/* =========================================================================== + Read a byte from a gz_stream; update next_in and avail_in. Return EOF + for end of file. + IN assertion: the stream s has been sucessfully opened for reading. +*/ + + +local int unz64local_getByte OF(( + const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + int *pi)); + +local int unz64local_getByte(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, int *pi) +{ + unsigned char c; + int err = (int)ZREAD64(*pzlib_filefunc_def,filestream,&c,1); + if (err==1) + { + *pi = (int)c; + return UNZ_OK; + } + else + { + if (ZERROR64(*pzlib_filefunc_def,filestream)) + return UNZ_ERRNO; + else + return UNZ_EOF; + } +} + + +/* =========================================================================== + Reads a long in LSB order from the given gz_stream. Sets +*/ +local int unz64local_getShort OF(( + const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + uLong *pX)); + +local int unz64local_getShort (const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + uLong *pX) +{ + uLong x ; + int i = 0; + int err; + + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x = (uLong)i; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((uLong)i)<<8; + + if (err==UNZ_OK) + *pX = x; + else + *pX = 0; + return err; +} + +local int unz64local_getLong OF(( + const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + uLong *pX)); + +local int unz64local_getLong (const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + uLong *pX) +{ + uLong x ; + int i = 0; + int err; + + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x = (uLong)i; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((uLong)i)<<8; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((uLong)i)<<16; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((uLong)i)<<24; + + if (err==UNZ_OK) + *pX = x; + else + *pX = 0; + return err; +} + +local int unz64local_getLong64 OF(( + const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + ZPOS64_T *pX)); + + +local int unz64local_getLong64 (const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + ZPOS64_T *pX) +{ + ZPOS64_T x ; + int i = 0; + int err; + + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x = (ZPOS64_T)i; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<8; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<16; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<24; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<32; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<40; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<48; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<56; + + if (err==UNZ_OK) + *pX = x; + else + *pX = 0; + return err; +} + +/* My own strcmpi / strcasecmp */ +local int strcmpcasenosensitive_internal (const char* fileName1, const char* fileName2) +{ + for (;;) + { + char c1=*(fileName1++); + char c2=*(fileName2++); + if ((c1>='a') && (c1<='z')) + c1 -= 0x20; + if ((c2>='a') && (c2<='z')) + c2 -= 0x20; + if (c1=='\0') + return ((c2=='\0') ? 0 : -1); + if (c2=='\0') + return 1; + if (c1<c2) + return -1; + if (c1>c2) + return 1; + } +} + + +#ifdef CASESENSITIVITYDEFAULT_NO +#define CASESENSITIVITYDEFAULTVALUE 2 +#else +#define CASESENSITIVITYDEFAULTVALUE 1 +#endif + +#ifndef STRCMPCASENOSENTIVEFUNCTION +#define STRCMPCASENOSENTIVEFUNCTION strcmpcasenosensitive_internal +#endif + +/* + Compare two filename (fileName1,fileName2). + If iCaseSenisivity = 1, comparision is case sensitivity (like strcmp) + If iCaseSenisivity = 2, comparision is not case sensitivity (like strcmpi + or strcasecmp) + If iCaseSenisivity = 0, case sensitivity is defaut of your operating system + (like 1 on Unix, 2 on Windows) + +*/ +extern int ZEXPORT unzStringFileNameCompare (const char* fileName1, + const char* fileName2, + int iCaseSensitivity) + +{ + if (iCaseSensitivity==0) + iCaseSensitivity=CASESENSITIVITYDEFAULTVALUE; + + if (iCaseSensitivity==1) + return strcmp(fileName1,fileName2); + + return STRCMPCASENOSENTIVEFUNCTION(fileName1,fileName2); +} + +#ifndef BUFREADCOMMENT +#define BUFREADCOMMENT (0x400) +#endif + +/* + Locate the Central directory of a zipfile (at the end, just before + the global comment) +*/ +local ZPOS64_T unz64local_SearchCentralDir OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream)); +local ZPOS64_T unz64local_SearchCentralDir(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream) +{ + unsigned char* buf; + ZPOS64_T uSizeFile; + ZPOS64_T uBackRead; + ZPOS64_T uMaxBack=0xffff; /* maximum size of global comment */ + ZPOS64_T uPosFound=0; + + if (ZSEEK64(*pzlib_filefunc_def,filestream,0,ZLIB_FILEFUNC_SEEK_END) != 0) + return 0; + + + uSizeFile = ZTELL64(*pzlib_filefunc_def,filestream); + + if (uMaxBack>uSizeFile) + uMaxBack = uSizeFile; + + buf = (unsigned char*)ALLOC(BUFREADCOMMENT+4); + if (buf==NULL) + return 0; + + uBackRead = 4; + while (uBackRead<uMaxBack) + { + uLong uReadSize; + ZPOS64_T uReadPos ; + int i; + if (uBackRead+BUFREADCOMMENT>uMaxBack) + uBackRead = uMaxBack; + else + uBackRead+=BUFREADCOMMENT; + uReadPos = uSizeFile-uBackRead ; + + uReadSize = ((BUFREADCOMMENT+4) < (uSizeFile-uReadPos)) ? + (BUFREADCOMMENT+4) : (uLong)(uSizeFile-uReadPos); + if (ZSEEK64(*pzlib_filefunc_def,filestream,uReadPos,ZLIB_FILEFUNC_SEEK_SET)!=0) + break; + + if (ZREAD64(*pzlib_filefunc_def,filestream,buf,uReadSize)!=uReadSize) + break; + + for (i=(int)uReadSize-3; (i--)>0;) + if (((*(buf+i))==0x50) && ((*(buf+i+1))==0x4b) && + ((*(buf+i+2))==0x05) && ((*(buf+i+3))==0x06)) + { + uPosFound = uReadPos+i; + break; + } + + if (uPosFound!=0) + break; + } + TRYFREE(buf); + return uPosFound; +} + + +/* + Locate the Central directory 64 of a zipfile (at the end, just before + the global comment) +*/ +local ZPOS64_T unz64local_SearchCentralDir64 OF(( + const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream)); + +local ZPOS64_T unz64local_SearchCentralDir64(const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream) +{ + unsigned char* buf; + ZPOS64_T uSizeFile; + ZPOS64_T uBackRead; + ZPOS64_T uMaxBack=0xffff; /* maximum size of global comment */ + ZPOS64_T uPosFound=0; + uLong uL; + ZPOS64_T relativeOffset; + + if (ZSEEK64(*pzlib_filefunc_def,filestream,0,ZLIB_FILEFUNC_SEEK_END) != 0) + return 0; + + + uSizeFile = ZTELL64(*pzlib_filefunc_def,filestream); + + if (uMaxBack>uSizeFile) + uMaxBack = uSizeFile; + + buf = (unsigned char*)ALLOC(BUFREADCOMMENT+4); + if (buf==NULL) + return 0; + + uBackRead = 4; + while (uBackRead<uMaxBack) + { + uLong uReadSize; + ZPOS64_T uReadPos; + int i; + if (uBackRead+BUFREADCOMMENT>uMaxBack) + uBackRead = uMaxBack; + else + uBackRead+=BUFREADCOMMENT; + uReadPos = uSizeFile-uBackRead ; + + uReadSize = ((BUFREADCOMMENT+4) < (uSizeFile-uReadPos)) ? + (BUFREADCOMMENT+4) : (uLong)(uSizeFile-uReadPos); + if (ZSEEK64(*pzlib_filefunc_def,filestream,uReadPos,ZLIB_FILEFUNC_SEEK_SET)!=0) + break; + + if (ZREAD64(*pzlib_filefunc_def,filestream,buf,uReadSize)!=uReadSize) + break; + + for (i=(int)uReadSize-3; (i--)>0;) + if (((*(buf+i))==0x50) && ((*(buf+i+1))==0x4b) && + ((*(buf+i+2))==0x06) && ((*(buf+i+3))==0x07)) + { + uPosFound = uReadPos+i; + break; + } + + if (uPosFound!=0) + break; + } + TRYFREE(buf); + if (uPosFound == 0) + return 0; + + /* Zip64 end of central directory locator */ + if (ZSEEK64(*pzlib_filefunc_def,filestream, uPosFound,ZLIB_FILEFUNC_SEEK_SET)!=0) + return 0; + + /* the signature, already checked */ + if (unz64local_getLong(pzlib_filefunc_def,filestream,&uL)!=UNZ_OK) + return 0; + + /* number of the disk with the start of the zip64 end of central directory */ + if (unz64local_getLong(pzlib_filefunc_def,filestream,&uL)!=UNZ_OK) + return 0; + if (uL != 0) + return 0; + + /* relative offset of the zip64 end of central directory record */ + if (unz64local_getLong64(pzlib_filefunc_def,filestream,&relativeOffset)!=UNZ_OK) + return 0; + + /* total number of disks */ + if (unz64local_getLong(pzlib_filefunc_def,filestream,&uL)!=UNZ_OK) + return 0; + if (uL != 1) + return 0; + + /* Goto end of central directory record */ + if (ZSEEK64(*pzlib_filefunc_def,filestream, relativeOffset,ZLIB_FILEFUNC_SEEK_SET)!=0) + return 0; + + /* the signature */ + if (unz64local_getLong(pzlib_filefunc_def,filestream,&uL)!=UNZ_OK) + return 0; + + if (uL != 0x06064b50) + return 0; + + return relativeOffset; +} + +/* + Open a Zip file. path contain the full pathname (by example, + on a Windows NT computer "c:\\test\\zlib114.zip" or on an Unix computer + "zlib/zlib114.zip". + If the zipfile cannot be opened (file doesn't exist or in not valid), the + return value is NULL. + Else, the return value is a unzFile Handle, usable with other function + of this unzip package. +*/ +local unzFile unzOpenInternal (const void *path, + zlib_filefunc64_32_def* pzlib_filefunc64_32_def, + int is64bitOpenFunction) +{ + unz64_s us; + unz64_s *s; + ZPOS64_T central_pos; + uLong uL; + + uLong number_disk; /* number of the current dist, used for + spaning ZIP, unsupported, always 0*/ + uLong number_disk_with_CD; /* number the the disk with central dir, used + for spaning ZIP, unsupported, always 0*/ + ZPOS64_T number_entry_CD; /* total number of entries in + the central dir + (same than number_entry on nospan) */ + + int err=UNZ_OK; + + if (unz_copyright[0]!=' ') + return NULL; + + us.z_filefunc.zseek32_file = NULL; + us.z_filefunc.ztell32_file = NULL; + if (pzlib_filefunc64_32_def==NULL) + fill_fopen64_filefunc(&us.z_filefunc.zfile_func64); + else + us.z_filefunc = *pzlib_filefunc64_32_def; + us.is64bitOpenFunction = is64bitOpenFunction; + + + + us.filestream = ZOPEN64(us.z_filefunc, + path, + ZLIB_FILEFUNC_MODE_READ | + ZLIB_FILEFUNC_MODE_EXISTING); + if (us.filestream==NULL) + return NULL; + + central_pos = unz64local_SearchCentralDir64(&us.z_filefunc,us.filestream); + if (central_pos) + { + uLong uS; + ZPOS64_T uL64; + + us.isZip64 = 1; + + if (ZSEEK64(us.z_filefunc, us.filestream, + central_pos,ZLIB_FILEFUNC_SEEK_SET)!=0) + err=UNZ_ERRNO; + + /* the signature, already checked */ + if (unz64local_getLong(&us.z_filefunc, us.filestream,&uL)!=UNZ_OK) + err=UNZ_ERRNO; + + /* size of zip64 end of central directory record */ + if (unz64local_getLong64(&us.z_filefunc, us.filestream,&uL64)!=UNZ_OK) + err=UNZ_ERRNO; + + /* version made by */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&uS)!=UNZ_OK) + err=UNZ_ERRNO; + + /* version needed to extract */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&uS)!=UNZ_OK) + err=UNZ_ERRNO; + + /* number of this disk */ + if (unz64local_getLong(&us.z_filefunc, us.filestream,&number_disk)!=UNZ_OK) + err=UNZ_ERRNO; + + /* number of the disk with the start of the central directory */ + if (unz64local_getLong(&us.z_filefunc, us.filestream,&number_disk_with_CD)!=UNZ_OK) + err=UNZ_ERRNO; + + /* total number of entries in the central directory on this disk */ + if (unz64local_getLong64(&us.z_filefunc, us.filestream,&us.gi.number_entry)!=UNZ_OK) + err=UNZ_ERRNO; + + /* total number of entries in the central directory */ + if (unz64local_getLong64(&us.z_filefunc, us.filestream,&number_entry_CD)!=UNZ_OK) + err=UNZ_ERRNO; + + if ((number_entry_CD!=us.gi.number_entry) || + (number_disk_with_CD!=0) || + (number_disk!=0)) + err=UNZ_BADZIPFILE; + + /* size of the central directory */ + if (unz64local_getLong64(&us.z_filefunc, us.filestream,&us.size_central_dir)!=UNZ_OK) + err=UNZ_ERRNO; + + /* offset of start of central directory with respect to the + starting disk number */ + if (unz64local_getLong64(&us.z_filefunc, us.filestream,&us.offset_central_dir)!=UNZ_OK) + err=UNZ_ERRNO; + + us.gi.size_comment = 0; + } + else + { + central_pos = unz64local_SearchCentralDir(&us.z_filefunc,us.filestream); + if (central_pos==0) + err=UNZ_ERRNO; + + us.isZip64 = 0; + + if (ZSEEK64(us.z_filefunc, us.filestream, + central_pos,ZLIB_FILEFUNC_SEEK_SET)!=0) + err=UNZ_ERRNO; + + /* the signature, already checked */ + if (unz64local_getLong(&us.z_filefunc, us.filestream,&uL)!=UNZ_OK) + err=UNZ_ERRNO; + + /* number of this disk */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&number_disk)!=UNZ_OK) + err=UNZ_ERRNO; + + /* number of the disk with the start of the central directory */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&number_disk_with_CD)!=UNZ_OK) + err=UNZ_ERRNO; + + /* total number of entries in the central dir on this disk */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&uL)!=UNZ_OK) + err=UNZ_ERRNO; + us.gi.number_entry = uL; + + /* total number of entries in the central dir */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&uL)!=UNZ_OK) + err=UNZ_ERRNO; + number_entry_CD = uL; + + if ((number_entry_CD!=us.gi.number_entry) || + (number_disk_with_CD!=0) || + (number_disk!=0)) + err=UNZ_BADZIPFILE; + + /* size of the central directory */ + if (unz64local_getLong(&us.z_filefunc, us.filestream,&uL)!=UNZ_OK) + err=UNZ_ERRNO; + us.size_central_dir = uL; + + /* offset of start of central directory with respect to the + starting disk number */ + if (unz64local_getLong(&us.z_filefunc, us.filestream,&uL)!=UNZ_OK) + err=UNZ_ERRNO; + us.offset_central_dir = uL; + + /* zipfile comment length */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&us.gi.size_comment)!=UNZ_OK) + err=UNZ_ERRNO; + } + + if ((central_pos<us.offset_central_dir+us.size_central_dir) && + (err==UNZ_OK)) + err=UNZ_BADZIPFILE; + + if (err!=UNZ_OK) + { + ZCLOSE64(us.z_filefunc, us.filestream); + return NULL; + } + + us.byte_before_the_zipfile = central_pos - + (us.offset_central_dir+us.size_central_dir); + us.central_pos = central_pos; + us.pfile_in_zip_read = NULL; + us.encrypted = 0; + + + s=(unz64_s*)ALLOC(sizeof(unz64_s)); + if( s != NULL) + { + *s=us; + unzGoToFirstFile((unzFile)s); + } + return (unzFile)s; +} + + +extern unzFile ZEXPORT unzOpen2 (const char *path, + zlib_filefunc_def* pzlib_filefunc32_def) +{ + if (pzlib_filefunc32_def != NULL) + { + zlib_filefunc64_32_def zlib_filefunc64_32_def_fill; + fill_zlib_filefunc64_32_def_from_filefunc32(&zlib_filefunc64_32_def_fill,pzlib_filefunc32_def); + return unzOpenInternal(path, &zlib_filefunc64_32_def_fill, 0); + } + else + return unzOpenInternal(path, NULL, 0); +} + +extern unzFile ZEXPORT unzOpen2_64 (const void *path, + zlib_filefunc64_def* pzlib_filefunc_def) +{ + if (pzlib_filefunc_def != NULL) + { + zlib_filefunc64_32_def zlib_filefunc64_32_def_fill; + zlib_filefunc64_32_def_fill.zfile_func64 = *pzlib_filefunc_def; + zlib_filefunc64_32_def_fill.ztell32_file = NULL; + zlib_filefunc64_32_def_fill.zseek32_file = NULL; + return unzOpenInternal(path, &zlib_filefunc64_32_def_fill, 1); + } + else + return unzOpenInternal(path, NULL, 1); +} + +extern unzFile ZEXPORT unzOpen (const char *path) +{ + return unzOpenInternal(path, NULL, 0); +} + +extern unzFile ZEXPORT unzOpen64 (const void *path) +{ + return unzOpenInternal(path, NULL, 1); +} + +/* + Close a ZipFile opened with unzOpen. + If there is files inside the .Zip opened with unzOpenCurrentFile (see later), + these files MUST be closed with unzCloseCurrentFile before call unzClose. + return UNZ_OK if there is no problem. */ +extern int ZEXPORT unzClose (unzFile file) +{ + unz64_s* s; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + + if (s->pfile_in_zip_read!=NULL) + unzCloseCurrentFile(file); + + ZCLOSE64(s->z_filefunc, s->filestream); + TRYFREE(s); + return UNZ_OK; +} + + +/* + Write info about the ZipFile in the *pglobal_info structure. + No preparation of the structure is needed + return UNZ_OK if there is no problem. */ +extern int ZEXPORT unzGetGlobalInfo64 (unzFile file, unz_global_info64* pglobal_info) +{ + unz64_s* s; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + *pglobal_info=s->gi; + return UNZ_OK; +} + +extern int ZEXPORT unzGetGlobalInfo (unzFile file, unz_global_info* pglobal_info32) +{ + unz64_s* s; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + /* to do : check if number_entry is not truncated */ + pglobal_info32->number_entry = (uLong)s->gi.number_entry; + pglobal_info32->size_comment = s->gi.size_comment; + return UNZ_OK; +} +/* + Translate date/time from Dos format to tm_unz (readable more easilty) +*/ +local void unz64local_DosDateToTmuDate (ZPOS64_T ulDosDate, tm_unz* ptm) +{ + ZPOS64_T uDate; + uDate = (ZPOS64_T)(ulDosDate>>16); + ptm->tm_mday = (uInt)(uDate&0x1f) ; + ptm->tm_mon = (uInt)((((uDate)&0x1E0)/0x20)-1) ; + ptm->tm_year = (uInt)(((uDate&0x0FE00)/0x0200)+1980) ; + + ptm->tm_hour = (uInt) ((ulDosDate &0xF800)/0x800); + ptm->tm_min = (uInt) ((ulDosDate&0x7E0)/0x20) ; + ptm->tm_sec = (uInt) (2*(ulDosDate&0x1f)) ; +} + +/* + Get Info about the current file in the zipfile, with internal only info +*/ +local int unz64local_GetCurrentFileInfoInternal OF((unzFile file, + unz_file_info64 *pfile_info, + unz_file_info64_internal + *pfile_info_internal, + char *szFileName, + uLong fileNameBufferSize, + void *extraField, + uLong extraFieldBufferSize, + char *szComment, + uLong commentBufferSize)); + +local int unz64local_GetCurrentFileInfoInternal (unzFile file, + unz_file_info64 *pfile_info, + unz_file_info64_internal + *pfile_info_internal, + char *szFileName, + uLong fileNameBufferSize, + void *extraField, + uLong extraFieldBufferSize, + char *szComment, + uLong commentBufferSize) +{ + unz64_s* s; + unz_file_info64 file_info; + unz_file_info64_internal file_info_internal; + int err=UNZ_OK; + uLong uMagic; + long lSeek=0; + uLong uL; + + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + if (ZSEEK64(s->z_filefunc, s->filestream, + s->pos_in_central_dir+s->byte_before_the_zipfile, + ZLIB_FILEFUNC_SEEK_SET)!=0) + err=UNZ_ERRNO; + + + /* we check the magic */ + if (err==UNZ_OK) + { + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uMagic) != UNZ_OK) + err=UNZ_ERRNO; + else if (uMagic!=0x02014b50) + err=UNZ_BADZIPFILE; + } + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.version) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.version_needed) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.flag) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.compression_method) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&file_info.dosDate) != UNZ_OK) + err=UNZ_ERRNO; + + unz64local_DosDateToTmuDate(file_info.dosDate,&file_info.tmu_date); + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&file_info.crc) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uL) != UNZ_OK) + err=UNZ_ERRNO; + file_info.compressed_size = uL; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uL) != UNZ_OK) + err=UNZ_ERRNO; + file_info.uncompressed_size = uL; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.size_filename) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.size_file_extra) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.size_file_comment) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.disk_num_start) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.internal_fa) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&file_info.external_fa) != UNZ_OK) + err=UNZ_ERRNO; + + // relative offset of local header + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uL) != UNZ_OK) + err=UNZ_ERRNO; + file_info_internal.offset_curfile = uL; + + lSeek+=file_info.size_filename; + if ((err==UNZ_OK) && (szFileName!=NULL)) + { + uLong uSizeRead ; + if (file_info.size_filename<fileNameBufferSize) + { + *(szFileName+file_info.size_filename)='\0'; + uSizeRead = file_info.size_filename; + } + else + uSizeRead = fileNameBufferSize; + + if ((file_info.size_filename>0) && (fileNameBufferSize>0)) + if (ZREAD64(s->z_filefunc, s->filestream,szFileName,uSizeRead)!=uSizeRead) + err=UNZ_ERRNO; + lSeek -= uSizeRead; + } + + // Read extrafield + if ((err==UNZ_OK) && (extraField!=NULL)) + { + ZPOS64_T uSizeRead ; + if (file_info.size_file_extra<extraFieldBufferSize) + uSizeRead = file_info.size_file_extra; + else + uSizeRead = extraFieldBufferSize; + + if (lSeek!=0) + { + if (ZSEEK64(s->z_filefunc, s->filestream,lSeek,ZLIB_FILEFUNC_SEEK_CUR)==0) + lSeek=0; + else + err=UNZ_ERRNO; + } + + if ((file_info.size_file_extra>0) && (extraFieldBufferSize>0)) + if (ZREAD64(s->z_filefunc, s->filestream,extraField,(uLong)uSizeRead)!=uSizeRead) + err=UNZ_ERRNO; + + lSeek += file_info.size_file_extra - (uLong)uSizeRead; + } + else + lSeek += file_info.size_file_extra; + + + if ((err==UNZ_OK) && (file_info.size_file_extra != 0)) + { + uLong acc = 0; + + // since lSeek now points to after the extra field we need to move back + lSeek -= file_info.size_file_extra; + + if (lSeek!=0) + { + if (ZSEEK64(s->z_filefunc, s->filestream,lSeek,ZLIB_FILEFUNC_SEEK_CUR)==0) + lSeek=0; + else + err=UNZ_ERRNO; + } + + while(acc < file_info.size_file_extra) + { + uLong headerId; + uLong dataSize; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&headerId) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&dataSize) != UNZ_OK) + err=UNZ_ERRNO; + + /* ZIP64 extra fields */ + if (headerId == 0x0001) + { + uLong uL; + + if(file_info.uncompressed_size == MAXU32) + { + if (unz64local_getLong64(&s->z_filefunc, s->filestream,&file_info.uncompressed_size) != UNZ_OK) + err=UNZ_ERRNO; + } + + if(file_info.compressed_size == MAXU32) + { + if (unz64local_getLong64(&s->z_filefunc, s->filestream,&file_info.compressed_size) != UNZ_OK) + err=UNZ_ERRNO; + } + + if(file_info_internal.offset_curfile == MAXU32) + { + /* Relative Header offset */ + if (unz64local_getLong64(&s->z_filefunc, s->filestream,&file_info_internal.offset_curfile) != UNZ_OK) + err=UNZ_ERRNO; + } + + if(file_info.disk_num_start == MAXU32) + { + /* Disk Start Number */ + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uL) != UNZ_OK) + err=UNZ_ERRNO; + } + + } + else + { + if (ZSEEK64(s->z_filefunc, s->filestream,dataSize,ZLIB_FILEFUNC_SEEK_CUR)!=0) + err=UNZ_ERRNO; + } + + acc += 2 + 2 + dataSize; + } + } + + if ((err==UNZ_OK) && (szComment!=NULL)) + { + uLong uSizeRead ; + if (file_info.size_file_comment<commentBufferSize) + { + *(szComment+file_info.size_file_comment)='\0'; + uSizeRead = file_info.size_file_comment; + } + else + uSizeRead = commentBufferSize; + + if (lSeek!=0) + { + if (ZSEEK64(s->z_filefunc, s->filestream,lSeek,ZLIB_FILEFUNC_SEEK_CUR)==0) + lSeek=0; + else + err=UNZ_ERRNO; + } + + if ((file_info.size_file_comment>0) && (commentBufferSize>0)) + if (ZREAD64(s->z_filefunc, s->filestream,szComment,uSizeRead)!=uSizeRead) + err=UNZ_ERRNO; + lSeek+=file_info.size_file_comment - uSizeRead; + } + else + lSeek+=file_info.size_file_comment; + + + if ((err==UNZ_OK) && (pfile_info!=NULL)) + *pfile_info=file_info; + + if ((err==UNZ_OK) && (pfile_info_internal!=NULL)) + *pfile_info_internal=file_info_internal; + + return err; +} + + + +/* + Write info about the ZipFile in the *pglobal_info structure. + No preparation of the structure is needed + return UNZ_OK if there is no problem. +*/ +extern int ZEXPORT unzGetCurrentFileInfo64 (unzFile file, + unz_file_info64 * pfile_info, + char * szFileName, uLong fileNameBufferSize, + void *extraField, uLong extraFieldBufferSize, + char* szComment, uLong commentBufferSize) +{ + return unz64local_GetCurrentFileInfoInternal(file,pfile_info,NULL, + szFileName,fileNameBufferSize, + extraField,extraFieldBufferSize, + szComment,commentBufferSize); +} + +extern int ZEXPORT unzGetCurrentFileInfo (unzFile file, + unz_file_info * pfile_info, + char * szFileName, uLong fileNameBufferSize, + void *extraField, uLong extraFieldBufferSize, + char* szComment, uLong commentBufferSize) +{ + int err; + unz_file_info64 file_info64; + err = unz64local_GetCurrentFileInfoInternal(file,&file_info64,NULL, + szFileName,fileNameBufferSize, + extraField,extraFieldBufferSize, + szComment,commentBufferSize); + if ((err==UNZ_OK) && (pfile_info != NULL)) + { + pfile_info->version = file_info64.version; + pfile_info->version_needed = file_info64.version_needed; + pfile_info->flag = file_info64.flag; + pfile_info->compression_method = file_info64.compression_method; + pfile_info->dosDate = file_info64.dosDate; + pfile_info->crc = file_info64.crc; + + pfile_info->size_filename = file_info64.size_filename; + pfile_info->size_file_extra = file_info64.size_file_extra; + pfile_info->size_file_comment = file_info64.size_file_comment; + + pfile_info->disk_num_start = file_info64.disk_num_start; + pfile_info->internal_fa = file_info64.internal_fa; + pfile_info->external_fa = file_info64.external_fa; + + pfile_info->tmu_date = file_info64.tmu_date, + + + pfile_info->compressed_size = (uLong)file_info64.compressed_size; + pfile_info->uncompressed_size = (uLong)file_info64.uncompressed_size; + + } + return err; +} +/* + Set the current file of the zipfile to the first file. + return UNZ_OK if there is no problem +*/ +extern int ZEXPORT unzGoToFirstFile (unzFile file) +{ + int err=UNZ_OK; + unz64_s* s; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + s->pos_in_central_dir=s->offset_central_dir; + s->num_file=0; + err=unz64local_GetCurrentFileInfoInternal(file,&s->cur_file_info, + &s->cur_file_info_internal, + NULL,0,NULL,0,NULL,0); + s->current_file_ok = (err == UNZ_OK); + return err; +} + +/* + Set the current file of the zipfile to the next file. + return UNZ_OK if there is no problem + return UNZ_END_OF_LIST_OF_FILE if the actual file was the latest. +*/ +extern int ZEXPORT unzGoToNextFile (unzFile file) +{ + unz64_s* s; + int err; + + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + if (!s->current_file_ok) + return UNZ_END_OF_LIST_OF_FILE; + if (s->gi.number_entry != 0xffff) /* 2^16 files overflow hack */ + if (s->num_file+1==s->gi.number_entry) + return UNZ_END_OF_LIST_OF_FILE; + + s->pos_in_central_dir += SIZECENTRALDIRITEM + s->cur_file_info.size_filename + + s->cur_file_info.size_file_extra + s->cur_file_info.size_file_comment ; + s->num_file++; + err = unz64local_GetCurrentFileInfoInternal(file,&s->cur_file_info, + &s->cur_file_info_internal, + NULL,0,NULL,0,NULL,0); + s->current_file_ok = (err == UNZ_OK); + return err; +} + + +/* + Try locate the file szFileName in the zipfile. + For the iCaseSensitivity signification, see unzStringFileNameCompare + + return value : + UNZ_OK if the file is found. It becomes the current file. + UNZ_END_OF_LIST_OF_FILE if the file is not found +*/ +extern int ZEXPORT unzLocateFile (unzFile file, const char *szFileName, int iCaseSensitivity) +{ + unz64_s* s; + int err; + + /* We remember the 'current' position in the file so that we can jump + * back there if we fail. + */ + unz_file_info64 cur_file_infoSaved; + unz_file_info64_internal cur_file_info_internalSaved; + ZPOS64_T num_fileSaved; + ZPOS64_T pos_in_central_dirSaved; + + + if (file==NULL) + return UNZ_PARAMERROR; + + if (strlen(szFileName)>=UNZ_MAXFILENAMEINZIP) + return UNZ_PARAMERROR; + + s=(unz64_s*)file; + if (!s->current_file_ok) + return UNZ_END_OF_LIST_OF_FILE; + + /* Save the current state */ + num_fileSaved = s->num_file; + pos_in_central_dirSaved = s->pos_in_central_dir; + cur_file_infoSaved = s->cur_file_info; + cur_file_info_internalSaved = s->cur_file_info_internal; + + err = unzGoToFirstFile(file); + + while (err == UNZ_OK) + { + char szCurrentFileName[UNZ_MAXFILENAMEINZIP+1]; + err = unzGetCurrentFileInfo64(file,NULL, + szCurrentFileName,sizeof(szCurrentFileName)-1, + NULL,0,NULL,0); + if (err == UNZ_OK) + { + if (unzStringFileNameCompare(szCurrentFileName, + szFileName,iCaseSensitivity)==0) + return UNZ_OK; + err = unzGoToNextFile(file); + } + } + + /* We failed, so restore the state of the 'current file' to where we + * were. + */ + s->num_file = num_fileSaved ; + s->pos_in_central_dir = pos_in_central_dirSaved ; + s->cur_file_info = cur_file_infoSaved; + s->cur_file_info_internal = cur_file_info_internalSaved; + return err; +} + + +/* +/////////////////////////////////////////// +// Contributed by Ryan Haksi (mailto://cryogen@infoserve.net) +// I need random access +// +// Further optimization could be realized by adding an ability +// to cache the directory in memory. The goal being a single +// comprehensive file read to put the file I need in a memory. +*/ + +/* +typedef struct unz_file_pos_s +{ + ZPOS64_T pos_in_zip_directory; // offset in file + ZPOS64_T num_of_file; // # of file +} unz_file_pos; +*/ + +extern int ZEXPORT unzGetFilePos64(unzFile file, unz64_file_pos* file_pos) +{ + unz64_s* s; + + if (file==NULL || file_pos==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + if (!s->current_file_ok) + return UNZ_END_OF_LIST_OF_FILE; + + file_pos->pos_in_zip_directory = s->pos_in_central_dir; + file_pos->num_of_file = s->num_file; + + return UNZ_OK; +} + +extern int ZEXPORT unzGetFilePos( + unzFile file, + unz_file_pos* file_pos) +{ + unz64_file_pos file_pos64; + int err = unzGetFilePos64(file,&file_pos64); + if (err==UNZ_OK) + { + file_pos->pos_in_zip_directory = (uLong)file_pos64.pos_in_zip_directory; + file_pos->num_of_file = (uLong)file_pos64.num_of_file; + } + return err; +} + +extern int ZEXPORT unzGoToFilePos64(unzFile file, const unz64_file_pos* file_pos) +{ + unz64_s* s; + int err; + + if (file==NULL || file_pos==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + + /* jump to the right spot */ + s->pos_in_central_dir = file_pos->pos_in_zip_directory; + s->num_file = file_pos->num_of_file; + + /* set the current file */ + err = unz64local_GetCurrentFileInfoInternal(file,&s->cur_file_info, + &s->cur_file_info_internal, + NULL,0,NULL,0,NULL,0); + /* return results */ + s->current_file_ok = (err == UNZ_OK); + return err; +} + +extern int ZEXPORT unzGoToFilePos( + unzFile file, + unz_file_pos* file_pos) +{ + unz64_file_pos file_pos64; + if (file_pos == NULL) + return UNZ_PARAMERROR; + + file_pos64.pos_in_zip_directory = file_pos->pos_in_zip_directory; + file_pos64.num_of_file = file_pos->num_of_file; + return unzGoToFilePos64(file,&file_pos64); +} + +/* +// Unzip Helper Functions - should be here? +/////////////////////////////////////////// +*/ + +/* + Read the local header of the current zipfile + Check the coherency of the local header and info in the end of central + directory about this file + store in *piSizeVar the size of extra info in local header + (filename and size of extra field data) +*/ +local int unz64local_CheckCurrentFileCoherencyHeader (unz64_s* s, uInt* piSizeVar, + ZPOS64_T * poffset_local_extrafield, + uInt * psize_local_extrafield) +{ + uLong uMagic,uData,uFlags; + uLong size_filename; + uLong size_extra_field; + int err=UNZ_OK; + + *piSizeVar = 0; + *poffset_local_extrafield = 0; + *psize_local_extrafield = 0; + + if (ZSEEK64(s->z_filefunc, s->filestream,s->cur_file_info_internal.offset_curfile + + s->byte_before_the_zipfile,ZLIB_FILEFUNC_SEEK_SET)!=0) + return UNZ_ERRNO; + + + if (err==UNZ_OK) + { + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uMagic) != UNZ_OK) + err=UNZ_ERRNO; + else if (uMagic!=0x04034b50) + err=UNZ_BADZIPFILE; + } + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) + err=UNZ_ERRNO; +/* + else if ((err==UNZ_OK) && (uData!=s->cur_file_info.wVersion)) + err=UNZ_BADZIPFILE; +*/ + if (unz64local_getShort(&s->z_filefunc, s->filestream,&uFlags) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) + err=UNZ_ERRNO; + else if ((err==UNZ_OK) && (uData!=s->cur_file_info.compression_method)) + err=UNZ_BADZIPFILE; + + if ((err==UNZ_OK) && (s->cur_file_info.compression_method!=0) && +/* #ifdef HAVE_BZIP2 */ + (s->cur_file_info.compression_method!=Z_BZIP2ED) && +/* #endif */ + (s->cur_file_info.compression_method!=Z_DEFLATED)) + err=UNZ_BADZIPFILE; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) /* date/time */ + err=UNZ_ERRNO; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) /* crc */ + err=UNZ_ERRNO; + else if ((err==UNZ_OK) && (uData!=s->cur_file_info.crc) && ((uFlags & 8)==0)) + err=UNZ_BADZIPFILE; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) /* size compr */ + err=UNZ_ERRNO; + else if (uData != 0xFFFFFFFF && (err==UNZ_OK) && (uData!=s->cur_file_info.compressed_size) && ((uFlags & 8)==0)) + err=UNZ_BADZIPFILE; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) /* size uncompr */ + err=UNZ_ERRNO; + else if (uData != 0xFFFFFFFF && (err==UNZ_OK) && (uData!=s->cur_file_info.uncompressed_size) && ((uFlags & 8)==0)) + err=UNZ_BADZIPFILE; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&size_filename) != UNZ_OK) + err=UNZ_ERRNO; + else if ((err==UNZ_OK) && (size_filename!=s->cur_file_info.size_filename)) + err=UNZ_BADZIPFILE; + + *piSizeVar += (uInt)size_filename; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&size_extra_field) != UNZ_OK) + err=UNZ_ERRNO; + *poffset_local_extrafield= s->cur_file_info_internal.offset_curfile + + SIZEZIPLOCALHEADER + size_filename; + *psize_local_extrafield = (uInt)size_extra_field; + + *piSizeVar += (uInt)size_extra_field; + + return err; +} + +/* + Open for reading data the current file in the zipfile. + If there is no error and the file is opened, the return value is UNZ_OK. +*/ +extern int ZEXPORT unzOpenCurrentFile3 (unzFile file, int* method, + int* level, int raw, const char* password) +{ + int err=UNZ_OK; + uInt iSizeVar; + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + ZPOS64_T offset_local_extrafield; /* offset of the local extra field */ + uInt size_local_extrafield; /* size of the local extra field */ +# ifndef NOUNCRYPT + char source[12]; +# else + if (password != NULL) + return UNZ_PARAMERROR; +# endif + + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + if (!s->current_file_ok) + return UNZ_PARAMERROR; + + if (s->pfile_in_zip_read != NULL) + unzCloseCurrentFile(file); + + if (unz64local_CheckCurrentFileCoherencyHeader(s,&iSizeVar, &offset_local_extrafield,&size_local_extrafield)!=UNZ_OK) + return UNZ_BADZIPFILE; + + pfile_in_zip_read_info = (file_in_zip64_read_info_s*)ALLOC(sizeof(file_in_zip64_read_info_s)); + if (pfile_in_zip_read_info==NULL) + return UNZ_INTERNALERROR; + + pfile_in_zip_read_info->read_buffer=(char*)ALLOC(UNZ_BUFSIZE); + pfile_in_zip_read_info->offset_local_extrafield = offset_local_extrafield; + pfile_in_zip_read_info->size_local_extrafield = size_local_extrafield; + pfile_in_zip_read_info->pos_local_extrafield=0; + pfile_in_zip_read_info->raw=raw; + + if (pfile_in_zip_read_info->read_buffer==NULL) + { + TRYFREE(pfile_in_zip_read_info); + return UNZ_INTERNALERROR; + } + + pfile_in_zip_read_info->stream_initialised=0; + + if (method!=NULL) + *method = (int)s->cur_file_info.compression_method; + + if (level!=NULL) + { + *level = 6; + switch (s->cur_file_info.flag & 0x06) + { + case 6 : *level = 1; break; + case 4 : *level = 2; break; + case 2 : *level = 9; break; + } + } + + if ((s->cur_file_info.compression_method!=0) && +/* #ifdef HAVE_BZIP2 */ + (s->cur_file_info.compression_method!=Z_BZIP2ED) && +/* #endif */ + (s->cur_file_info.compression_method!=Z_DEFLATED)) + + err=UNZ_BADZIPFILE; + + pfile_in_zip_read_info->crc32_wait=s->cur_file_info.crc; + pfile_in_zip_read_info->crc32=0; + pfile_in_zip_read_info->total_out_64=0; + pfile_in_zip_read_info->compression_method = s->cur_file_info.compression_method; + pfile_in_zip_read_info->filestream=s->filestream; + pfile_in_zip_read_info->z_filefunc=s->z_filefunc; + pfile_in_zip_read_info->byte_before_the_zipfile=s->byte_before_the_zipfile; + + pfile_in_zip_read_info->stream.total_out = 0; + + if ((s->cur_file_info.compression_method==Z_BZIP2ED) && (!raw)) + { +#ifdef HAVE_BZIP2 + pfile_in_zip_read_info->bstream.bzalloc = (void *(*) (void *, int, int))0; + pfile_in_zip_read_info->bstream.bzfree = (free_func)0; + pfile_in_zip_read_info->bstream.opaque = (voidpf)0; + pfile_in_zip_read_info->bstream.state = (voidpf)0; + + pfile_in_zip_read_info->stream.zalloc = (alloc_func)0; + pfile_in_zip_read_info->stream.zfree = (free_func)0; + pfile_in_zip_read_info->stream.opaque = (voidpf)0; + pfile_in_zip_read_info->stream.next_in = (voidpf)0; + pfile_in_zip_read_info->stream.avail_in = 0; + + err=BZ2_bzDecompressInit(&pfile_in_zip_read_info->bstream, 0, 0); + if (err == Z_OK) + pfile_in_zip_read_info->stream_initialised=Z_BZIP2ED; + else + { + TRYFREE(pfile_in_zip_read_info); + return err; + } +#else + pfile_in_zip_read_info->raw=1; +#endif + } + else if ((s->cur_file_info.compression_method==Z_DEFLATED) && (!raw)) + { + pfile_in_zip_read_info->stream.zalloc = (alloc_func)0; + pfile_in_zip_read_info->stream.zfree = (free_func)0; + pfile_in_zip_read_info->stream.opaque = (voidpf)0; + pfile_in_zip_read_info->stream.next_in = 0; + pfile_in_zip_read_info->stream.avail_in = 0; + + err=inflateInit2(&pfile_in_zip_read_info->stream, -MAX_WBITS); + if (err == Z_OK) + pfile_in_zip_read_info->stream_initialised=Z_DEFLATED; + else + { + TRYFREE(pfile_in_zip_read_info); + return err; + } + /* windowBits is passed < 0 to tell that there is no zlib header. + * Note that in this case inflate *requires* an extra "dummy" byte + * after the compressed stream in order to complete decompression and + * return Z_STREAM_END. + * In unzip, i don't wait absolutely Z_STREAM_END because I known the + * size of both compressed and uncompressed data + */ + } + pfile_in_zip_read_info->rest_read_compressed = + s->cur_file_info.compressed_size ; + pfile_in_zip_read_info->rest_read_uncompressed = + s->cur_file_info.uncompressed_size ; + + + pfile_in_zip_read_info->pos_in_zipfile = + s->cur_file_info_internal.offset_curfile + SIZEZIPLOCALHEADER + + iSizeVar; + + pfile_in_zip_read_info->stream.avail_in = (uInt)0; + + s->pfile_in_zip_read = pfile_in_zip_read_info; + s->encrypted = 0; + +# ifndef NOUNCRYPT + if (password != NULL) + { + int i; + s->pcrc_32_tab = get_crc_table(); + init_keys(password,s->keys,s->pcrc_32_tab); + if (ZSEEK64(s->z_filefunc, s->filestream, + s->pfile_in_zip_read->pos_in_zipfile + + s->pfile_in_zip_read->byte_before_the_zipfile, + SEEK_SET)!=0) + return UNZ_INTERNALERROR; + if(ZREAD64(s->z_filefunc, s->filestream,source, 12)<12) + return UNZ_INTERNALERROR; + + for (i = 0; i<12; i++) + zdecode(s->keys,s->pcrc_32_tab,source[i]); + + s->pfile_in_zip_read->pos_in_zipfile+=12; + s->encrypted=1; + } +# endif + + + return UNZ_OK; +} + +extern int ZEXPORT unzOpenCurrentFile (unzFile file) +{ + return unzOpenCurrentFile3(file, NULL, NULL, 0, NULL); +} + +extern int ZEXPORT unzOpenCurrentFilePassword (unzFile file, const char* password) +{ + return unzOpenCurrentFile3(file, NULL, NULL, 0, password); +} + +extern int ZEXPORT unzOpenCurrentFile2 (unzFile file, int* method, int* level, int raw) +{ + return unzOpenCurrentFile3(file, method, level, raw, NULL); +} + +/** Addition for GDAL : START */ + +extern ZPOS64_T ZEXPORT unzGetCurrentFileZStreamPos64( unzFile file) +{ + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + s=(unz64_s*)file; + if (file==NULL) + return 0; //UNZ_PARAMERROR; + pfile_in_zip_read_info=s->pfile_in_zip_read; + if (pfile_in_zip_read_info==NULL) + return 0; //UNZ_PARAMERROR; + return pfile_in_zip_read_info->pos_in_zipfile + + pfile_in_zip_read_info->byte_before_the_zipfile; +} + +/** Addition for GDAL : END */ + +/* + Read bytes from the current file. + buf contain buffer where data must be copied + len the size of buf. + + return the number of byte copied if somes bytes are copied + return 0 if the end of file was reached + return <0 with error code if there is an error + (UNZ_ERRNO for IO error, or zLib error for uncompress error) +*/ +extern int ZEXPORT unzReadCurrentFile (unzFile file, voidp buf, unsigned len) +{ + int err=UNZ_OK; + uInt iRead = 0; + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + pfile_in_zip_read_info=s->pfile_in_zip_read; + + if (pfile_in_zip_read_info==NULL) + return UNZ_PARAMERROR; + + + if (pfile_in_zip_read_info->read_buffer == NULL) + return UNZ_END_OF_LIST_OF_FILE; + if (len==0) + return 0; + + pfile_in_zip_read_info->stream.next_out = (Bytef*)buf; + + pfile_in_zip_read_info->stream.avail_out = (uInt)len; + + if ((len>pfile_in_zip_read_info->rest_read_uncompressed) && + (!(pfile_in_zip_read_info->raw))) + pfile_in_zip_read_info->stream.avail_out = + (uInt)pfile_in_zip_read_info->rest_read_uncompressed; + + if ((len>pfile_in_zip_read_info->rest_read_compressed+ + pfile_in_zip_read_info->stream.avail_in) && + (pfile_in_zip_read_info->raw)) + pfile_in_zip_read_info->stream.avail_out = + (uInt)pfile_in_zip_read_info->rest_read_compressed+ + pfile_in_zip_read_info->stream.avail_in; + + while (pfile_in_zip_read_info->stream.avail_out>0) + { + if ((pfile_in_zip_read_info->stream.avail_in==0) && + (pfile_in_zip_read_info->rest_read_compressed>0)) + { + uInt uReadThis = UNZ_BUFSIZE; + if (pfile_in_zip_read_info->rest_read_compressed<uReadThis) + uReadThis = (uInt)pfile_in_zip_read_info->rest_read_compressed; + if (uReadThis == 0) + return UNZ_EOF; + if (ZSEEK64(pfile_in_zip_read_info->z_filefunc, + pfile_in_zip_read_info->filestream, + pfile_in_zip_read_info->pos_in_zipfile + + pfile_in_zip_read_info->byte_before_the_zipfile, + ZLIB_FILEFUNC_SEEK_SET)!=0) + return UNZ_ERRNO; + if (ZREAD64(pfile_in_zip_read_info->z_filefunc, + pfile_in_zip_read_info->filestream, + pfile_in_zip_read_info->read_buffer, + uReadThis)!=uReadThis) + return UNZ_ERRNO; + + +# ifndef NOUNCRYPT + if(s->encrypted) + { + uInt i; + for(i=0;i<uReadThis;i++) + pfile_in_zip_read_info->read_buffer[i] = + zdecode(s->keys,s->pcrc_32_tab, + pfile_in_zip_read_info->read_buffer[i]); + } +# endif + + + pfile_in_zip_read_info->pos_in_zipfile += uReadThis; + + pfile_in_zip_read_info->rest_read_compressed-=uReadThis; + + pfile_in_zip_read_info->stream.next_in = + (Bytef*)pfile_in_zip_read_info->read_buffer; + pfile_in_zip_read_info->stream.avail_in = (uInt)uReadThis; + } + + if ((pfile_in_zip_read_info->compression_method==0) || (pfile_in_zip_read_info->raw)) + { + uInt uDoCopy,i ; + + if ((pfile_in_zip_read_info->stream.avail_in == 0) && + (pfile_in_zip_read_info->rest_read_compressed == 0)) + return (iRead==0) ? UNZ_EOF : iRead; + + if (pfile_in_zip_read_info->stream.avail_out < + pfile_in_zip_read_info->stream.avail_in) + uDoCopy = pfile_in_zip_read_info->stream.avail_out ; + else + uDoCopy = pfile_in_zip_read_info->stream.avail_in ; + + for (i=0;i<uDoCopy;i++) + *(pfile_in_zip_read_info->stream.next_out+i) = + *(pfile_in_zip_read_info->stream.next_in+i); + + pfile_in_zip_read_info->total_out_64 = pfile_in_zip_read_info->total_out_64 + uDoCopy; + + pfile_in_zip_read_info->crc32 = crc32(pfile_in_zip_read_info->crc32, + pfile_in_zip_read_info->stream.next_out, + uDoCopy); + pfile_in_zip_read_info->rest_read_uncompressed-=uDoCopy; + pfile_in_zip_read_info->stream.avail_in -= uDoCopy; + pfile_in_zip_read_info->stream.avail_out -= uDoCopy; + pfile_in_zip_read_info->stream.next_out += uDoCopy; + pfile_in_zip_read_info->stream.next_in += uDoCopy; + pfile_in_zip_read_info->stream.total_out += uDoCopy; + iRead += uDoCopy; + } + else if (pfile_in_zip_read_info->compression_method==Z_BZIP2ED) + { +#ifdef HAVE_BZIP2 + uLong uTotalOutBefore,uTotalOutAfter; + const Bytef *bufBefore; + uLong uOutThis; + + pfile_in_zip_read_info->bstream.next_in = (char*)pfile_in_zip_read_info->stream.next_in; + pfile_in_zip_read_info->bstream.avail_in = pfile_in_zip_read_info->stream.avail_in; + pfile_in_zip_read_info->bstream.total_in_lo32 = pfile_in_zip_read_info->stream.total_in; + pfile_in_zip_read_info->bstream.total_in_hi32 = 0; + pfile_in_zip_read_info->bstream.next_out = (char*)pfile_in_zip_read_info->stream.next_out; + pfile_in_zip_read_info->bstream.avail_out = pfile_in_zip_read_info->stream.avail_out; + pfile_in_zip_read_info->bstream.total_out_lo32 = pfile_in_zip_read_info->stream.total_out; + pfile_in_zip_read_info->bstream.total_out_hi32 = 0; + + uTotalOutBefore = pfile_in_zip_read_info->bstream.total_out_lo32; + bufBefore = (const Bytef *)pfile_in_zip_read_info->bstream.next_out; + + err=BZ2_bzDecompress(&pfile_in_zip_read_info->bstream); + + uTotalOutAfter = pfile_in_zip_read_info->bstream.total_out_lo32; + uOutThis = uTotalOutAfter-uTotalOutBefore; + + pfile_in_zip_read_info->total_out_64 = pfile_in_zip_read_info->total_out_64 + uOutThis; + + pfile_in_zip_read_info->crc32 = crc32(pfile_in_zip_read_info->crc32,bufBefore, (uInt)(uOutThis)); + pfile_in_zip_read_info->rest_read_uncompressed -= uOutThis; + iRead += (uInt)(uTotalOutAfter - uTotalOutBefore); + + pfile_in_zip_read_info->stream.next_in = (Bytef*)pfile_in_zip_read_info->bstream.next_in; + pfile_in_zip_read_info->stream.avail_in = pfile_in_zip_read_info->bstream.avail_in; + pfile_in_zip_read_info->stream.total_in = pfile_in_zip_read_info->bstream.total_in_lo32; + pfile_in_zip_read_info->stream.next_out = (Bytef*)pfile_in_zip_read_info->bstream.next_out; + pfile_in_zip_read_info->stream.avail_out = pfile_in_zip_read_info->bstream.avail_out; + pfile_in_zip_read_info->stream.total_out = pfile_in_zip_read_info->bstream.total_out_lo32; + + if (err==BZ_STREAM_END) + return (iRead==0) ? UNZ_EOF : iRead; + if (err!=BZ_OK) + break; +#endif + } // end Z_BZIP2ED + else + { + ZPOS64_T uTotalOutBefore,uTotalOutAfter; + const Bytef *bufBefore; + ZPOS64_T uOutThis; + int flush=Z_SYNC_FLUSH; + + uTotalOutBefore = pfile_in_zip_read_info->stream.total_out; + bufBefore = pfile_in_zip_read_info->stream.next_out; + + /* + if ((pfile_in_zip_read_info->rest_read_uncompressed == + pfile_in_zip_read_info->stream.avail_out) && + (pfile_in_zip_read_info->rest_read_compressed == 0)) + flush = Z_FINISH; + */ + err=inflate(&pfile_in_zip_read_info->stream,flush); + + if ((err>=0) && (pfile_in_zip_read_info->stream.msg!=NULL)) + err = Z_DATA_ERROR; + + uTotalOutAfter = pfile_in_zip_read_info->stream.total_out; + uOutThis = uTotalOutAfter-uTotalOutBefore; + + pfile_in_zip_read_info->total_out_64 = pfile_in_zip_read_info->total_out_64 + uOutThis; + + pfile_in_zip_read_info->crc32 = + crc32(pfile_in_zip_read_info->crc32,bufBefore, + (uInt)(uOutThis)); + + pfile_in_zip_read_info->rest_read_uncompressed -= + uOutThis; + + iRead += (uInt)(uTotalOutAfter - uTotalOutBefore); + + if (err==Z_STREAM_END) + return (iRead==0) ? UNZ_EOF : iRead; + if (err!=Z_OK) + break; + } + } + + if (err==Z_OK) + return iRead; + return err; +} + + +/* + Give the current position in uncompressed data +*/ +extern z_off_t ZEXPORT unztell (unzFile file) +{ + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + pfile_in_zip_read_info=s->pfile_in_zip_read; + + if (pfile_in_zip_read_info==NULL) + return UNZ_PARAMERROR; + + return (z_off_t)pfile_in_zip_read_info->stream.total_out; +} + +extern ZPOS64_T ZEXPORT unztell64 (unzFile file) +{ + + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + if (file==NULL) + return (ZPOS64_T)-1; + s=(unz64_s*)file; + pfile_in_zip_read_info=s->pfile_in_zip_read; + + if (pfile_in_zip_read_info==NULL) + return (ZPOS64_T)-1; + + return pfile_in_zip_read_info->total_out_64; +} + + +/* + return 1 if the end of file was reached, 0 elsewhere +*/ +extern int ZEXPORT unzeof (unzFile file) +{ + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + pfile_in_zip_read_info=s->pfile_in_zip_read; + + if (pfile_in_zip_read_info==NULL) + return UNZ_PARAMERROR; + + if (pfile_in_zip_read_info->rest_read_uncompressed == 0) + return 1; + else + return 0; +} + + + +/* +Read extra field from the current file (opened by unzOpenCurrentFile) +This is the local-header version of the extra field (sometimes, there is +more info in the local-header version than in the central-header) + + if buf==NULL, it return the size of the local extra field that can be read + + if buf!=NULL, len is the size of the buffer, the extra header is copied in + buf. + the return value is the number of bytes copied in buf, or (if <0) + the error code +*/ +extern int ZEXPORT unzGetLocalExtrafield (unzFile file, voidp buf, unsigned len) +{ + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + uInt read_now; + ZPOS64_T size_to_read; + + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + pfile_in_zip_read_info=s->pfile_in_zip_read; + + if (pfile_in_zip_read_info==NULL) + return UNZ_PARAMERROR; + + size_to_read = (pfile_in_zip_read_info->size_local_extrafield - + pfile_in_zip_read_info->pos_local_extrafield); + + if (buf==NULL) + return (int)size_to_read; + + if (len>size_to_read) + read_now = (uInt)size_to_read; + else + read_now = (uInt)len ; + + if (read_now==0) + return 0; + + if (ZSEEK64(pfile_in_zip_read_info->z_filefunc, + pfile_in_zip_read_info->filestream, + pfile_in_zip_read_info->offset_local_extrafield + + pfile_in_zip_read_info->pos_local_extrafield, + ZLIB_FILEFUNC_SEEK_SET)!=0) + return UNZ_ERRNO; + + if (ZREAD64(pfile_in_zip_read_info->z_filefunc, + pfile_in_zip_read_info->filestream, + buf,read_now)!=read_now) + return UNZ_ERRNO; + + return (int)read_now; +} + +/* + Close the file in zip opened with unzOpenCurrentFile + Return UNZ_CRCERROR if all the file was read but the CRC is not good +*/ +extern int ZEXPORT unzCloseCurrentFile (unzFile file) +{ + int err=UNZ_OK; + + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + pfile_in_zip_read_info=s->pfile_in_zip_read; + + if (pfile_in_zip_read_info==NULL) + return UNZ_PARAMERROR; + + + if ((pfile_in_zip_read_info->rest_read_uncompressed == 0) && + (!pfile_in_zip_read_info->raw)) + { + if (pfile_in_zip_read_info->crc32 != pfile_in_zip_read_info->crc32_wait) + err=UNZ_CRCERROR; + } + + + TRYFREE(pfile_in_zip_read_info->read_buffer); + pfile_in_zip_read_info->read_buffer = NULL; + if (pfile_in_zip_read_info->stream_initialised == Z_DEFLATED) + inflateEnd(&pfile_in_zip_read_info->stream); +#ifdef HAVE_BZIP2 + else if (pfile_in_zip_read_info->stream_initialised == Z_BZIP2ED) + BZ2_bzDecompressEnd(&pfile_in_zip_read_info->bstream); +#endif + + + pfile_in_zip_read_info->stream_initialised = 0; + TRYFREE(pfile_in_zip_read_info); + + s->pfile_in_zip_read=NULL; + + return err; +} + + +/* + Get the global comment string of the ZipFile, in the szComment buffer. + uSizeBuf is the size of the szComment buffer. + return the number of byte copied or an error code <0 +*/ +extern int ZEXPORT unzGetGlobalComment (unzFile file, char * szComment, uLong uSizeBuf) +{ + unz64_s* s; + uLong uReadThis ; + if (file==NULL) + return (int)UNZ_PARAMERROR; + s=(unz64_s*)file; + + uReadThis = uSizeBuf; + if (uReadThis>s->gi.size_comment) + uReadThis = s->gi.size_comment; + + if (ZSEEK64(s->z_filefunc,s->filestream,s->central_pos+22,ZLIB_FILEFUNC_SEEK_SET)!=0) + return UNZ_ERRNO; + + if (uReadThis>0) + { + *szComment='\0'; + if (ZREAD64(s->z_filefunc,s->filestream,szComment,uReadThis)!=uReadThis) + return UNZ_ERRNO; + } + + if ((szComment != NULL) && (uSizeBuf > s->gi.size_comment)) + *(szComment+s->gi.size_comment)='\0'; + return (int)uReadThis; +} + +/* Additions by RX '2004 */ +extern ZPOS64_T ZEXPORT unzGetOffset64(unzFile file) +{ + unz64_s* s; + + if (file==NULL) + return 0; //UNZ_PARAMERROR; + s=(unz64_s*)file; + if (!s->current_file_ok) + return 0; + if (s->gi.number_entry != 0 && s->gi.number_entry != 0xffff) + if (s->num_file==s->gi.number_entry) + return 0; + return s->pos_in_central_dir; +} + +extern uLong ZEXPORT unzGetOffset (unzFile file) +{ + ZPOS64_T offset64; + + if (file==NULL) + return 0; //UNZ_PARAMERROR; + offset64 = unzGetOffset64(file); + return (uLong)offset64; +} + +extern int ZEXPORT unzSetOffset64(unzFile file, ZPOS64_T pos) +{ + unz64_s* s; + int err; + + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + + s->pos_in_central_dir = pos; + s->num_file = s->gi.number_entry; /* hack */ + err = unz64local_GetCurrentFileInfoInternal(file,&s->cur_file_info, + &s->cur_file_info_internal, + NULL,0,NULL,0,NULL,0); + s->current_file_ok = (err == UNZ_OK); + return err; +} + +extern int ZEXPORT unzSetOffset (unzFile file, uLong pos) +{ + return unzSetOffset64(file,pos); +} diff --git a/storage/connect/unzip.h b/storage/connect/unzip.h new file mode 100644 index 00000000000..2104e391507 --- /dev/null +++ b/storage/connect/unzip.h @@ -0,0 +1,437 @@ +/* unzip.h -- IO for uncompress .zip files using zlib + Version 1.1, February 14h, 2010 + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications of Unzip for Zip64 + Copyright (C) 2007-2008 Even Rouault + + Modifications for Zip64 support on both zip and unzip + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + + --------------------------------------------------------------------------------- + + Condition of use and distribution are the same than zlib : + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + --------------------------------------------------------------------------------- + + Changes + + See header of unzip64.c + +*/ + +#ifndef _unz64_H +#define _unz64_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef _ZLIB_H +#include "zlib.h" +#endif + +#ifndef _ZLIBIOAPI_H +#include "ioapi.h" +#endif + +#ifdef HAVE_BZIP2 +#include "bzlib.h" +#endif + +#define Z_BZIP2ED 12 + +#if defined(STRICTUNZIP) || defined(STRICTZIPUNZIP) +/* like the STRICT of WIN32, we define a pointer that cannot be converted + from (void*) without cast */ +typedef struct TagunzFile__ { int unused; } unzFile__; +typedef unzFile__ *unzFile; +#else +typedef voidp unzFile; +#endif + + +#define UNZ_OK (0) +#define UNZ_END_OF_LIST_OF_FILE (-100) +#define UNZ_ERRNO (Z_ERRNO) +#define UNZ_EOF (0) +#define UNZ_PARAMERROR (-102) +#define UNZ_BADZIPFILE (-103) +#define UNZ_INTERNALERROR (-104) +#define UNZ_CRCERROR (-105) + +/* tm_unz contain date/time info */ +typedef struct tm_unz_s +{ + uInt tm_sec; /* seconds after the minute - [0,59] */ + uInt tm_min; /* minutes after the hour - [0,59] */ + uInt tm_hour; /* hours since midnight - [0,23] */ + uInt tm_mday; /* day of the month - [1,31] */ + uInt tm_mon; /* months since January - [0,11] */ + uInt tm_year; /* years - [1980..2044] */ +} tm_unz; + +/* unz_global_info structure contain global data about the ZIPfile + These data comes from the end of central dir */ +typedef struct unz_global_info64_s +{ + ZPOS64_T number_entry; /* total number of entries in + the central dir on this disk */ + uLong size_comment; /* size of the global comment of the zipfile */ +} unz_global_info64; + +typedef struct unz_global_info_s +{ + uLong number_entry; /* total number of entries in + the central dir on this disk */ + uLong size_comment; /* size of the global comment of the zipfile */ +} unz_global_info; + +/* unz_file_info contain information about a file in the zipfile */ +typedef struct unz_file_info64_s +{ + uLong version; /* version made by 2 bytes */ + uLong version_needed; /* version needed to extract 2 bytes */ + uLong flag; /* general purpose bit flag 2 bytes */ + uLong compression_method; /* compression method 2 bytes */ + uLong dosDate; /* last mod file date in Dos fmt 4 bytes */ + uLong crc; /* crc-32 4 bytes */ + ZPOS64_T compressed_size; /* compressed size 8 bytes */ + ZPOS64_T uncompressed_size; /* uncompressed size 8 bytes */ + uLong size_filename; /* filename length 2 bytes */ + uLong size_file_extra; /* extra field length 2 bytes */ + uLong size_file_comment; /* file comment length 2 bytes */ + + uLong disk_num_start; /* disk number start 2 bytes */ + uLong internal_fa; /* internal file attributes 2 bytes */ + uLong external_fa; /* external file attributes 4 bytes */ + + tm_unz tmu_date; +} unz_file_info64; + +typedef struct unz_file_info_s +{ + uLong version; /* version made by 2 bytes */ + uLong version_needed; /* version needed to extract 2 bytes */ + uLong flag; /* general purpose bit flag 2 bytes */ + uLong compression_method; /* compression method 2 bytes */ + uLong dosDate; /* last mod file date in Dos fmt 4 bytes */ + uLong crc; /* crc-32 4 bytes */ + uLong compressed_size; /* compressed size 4 bytes */ + uLong uncompressed_size; /* uncompressed size 4 bytes */ + uLong size_filename; /* filename length 2 bytes */ + uLong size_file_extra; /* extra field length 2 bytes */ + uLong size_file_comment; /* file comment length 2 bytes */ + + uLong disk_num_start; /* disk number start 2 bytes */ + uLong internal_fa; /* internal file attributes 2 bytes */ + uLong external_fa; /* external file attributes 4 bytes */ + + tm_unz tmu_date; +} unz_file_info; + +extern int ZEXPORT unzStringFileNameCompare OF ((const char* fileName1, + const char* fileName2, + int iCaseSensitivity)); +/* + Compare two filename (fileName1,fileName2). + If iCaseSenisivity = 1, comparision is case sensitivity (like strcmp) + If iCaseSenisivity = 2, comparision is not case sensitivity (like strcmpi + or strcasecmp) + If iCaseSenisivity = 0, case sensitivity is defaut of your operating system + (like 1 on Unix, 2 on Windows) +*/ + + +extern unzFile ZEXPORT unzOpen OF((const char *path)); +extern unzFile ZEXPORT unzOpen64 OF((const void *path)); +/* + Open a Zip file. path contain the full pathname (by example, + on a Windows XP computer "c:\\zlib\\zlib113.zip" or on an Unix computer + "zlib/zlib113.zip". + If the zipfile cannot be opened (file don't exist or in not valid), the + return value is NULL. + Else, the return value is a unzFile Handle, usable with other function + of this unzip package. + the "64" function take a const void* pointer, because the path is just the + value passed to the open64_file_func callback. + Under Windows, if UNICODE is defined, using fill_fopen64_filefunc, the path + is a pointer to a wide unicode string (LPCTSTR is LPCWSTR), so const char* + does not describe the reality +*/ + + +extern unzFile ZEXPORT unzOpen2 OF((const char *path, + zlib_filefunc_def* pzlib_filefunc_def)); +/* + Open a Zip file, like unzOpen, but provide a set of file low level API + for read/write the zip file (see ioapi.h) +*/ + +extern unzFile ZEXPORT unzOpen2_64 OF((const void *path, + zlib_filefunc64_def* pzlib_filefunc_def)); +/* + Open a Zip file, like unz64Open, but provide a set of file low level API + for read/write the zip file (see ioapi.h) +*/ + +extern int ZEXPORT unzClose OF((unzFile file)); +/* + Close a ZipFile opened with unzOpen. + If there is files inside the .Zip opened with unzOpenCurrentFile (see later), + these files MUST be closed with unzCloseCurrentFile before call unzClose. + return UNZ_OK if there is no problem. */ + +extern int ZEXPORT unzGetGlobalInfo OF((unzFile file, + unz_global_info *pglobal_info)); + +extern int ZEXPORT unzGetGlobalInfo64 OF((unzFile file, + unz_global_info64 *pglobal_info)); +/* + Write info about the ZipFile in the *pglobal_info structure. + No preparation of the structure is needed + return UNZ_OK if there is no problem. */ + + +extern int ZEXPORT unzGetGlobalComment OF((unzFile file, + char *szComment, + uLong uSizeBuf)); +/* + Get the global comment string of the ZipFile, in the szComment buffer. + uSizeBuf is the size of the szComment buffer. + return the number of byte copied or an error code <0 +*/ + + +/***************************************************************************/ +/* Unzip package allow you browse the directory of the zipfile */ + +extern int ZEXPORT unzGoToFirstFile OF((unzFile file)); +/* + Set the current file of the zipfile to the first file. + return UNZ_OK if there is no problem +*/ + +extern int ZEXPORT unzGoToNextFile OF((unzFile file)); +/* + Set the current file of the zipfile to the next file. + return UNZ_OK if there is no problem + return UNZ_END_OF_LIST_OF_FILE if the actual file was the latest. +*/ + +extern int ZEXPORT unzLocateFile OF((unzFile file, + const char *szFileName, + int iCaseSensitivity)); +/* + Try locate the file szFileName in the zipfile. + For the iCaseSensitivity signification, see unzStringFileNameCompare + + return value : + UNZ_OK if the file is found. It becomes the current file. + UNZ_END_OF_LIST_OF_FILE if the file is not found +*/ + + +/* ****************************************** */ +/* Ryan supplied functions */ +/* unz_file_info contain information about a file in the zipfile */ +typedef struct unz_file_pos_s +{ + uLong pos_in_zip_directory; /* offset in zip file directory */ + uLong num_of_file; /* # of file */ +} unz_file_pos; + +extern int ZEXPORT unzGetFilePos( + unzFile file, + unz_file_pos* file_pos); + +extern int ZEXPORT unzGoToFilePos( + unzFile file, + unz_file_pos* file_pos); + +typedef struct unz64_file_pos_s +{ + ZPOS64_T pos_in_zip_directory; /* offset in zip file directory */ + ZPOS64_T num_of_file; /* # of file */ +} unz64_file_pos; + +extern int ZEXPORT unzGetFilePos64( + unzFile file, + unz64_file_pos* file_pos); + +extern int ZEXPORT unzGoToFilePos64( + unzFile file, + const unz64_file_pos* file_pos); + +/* ****************************************** */ + +extern int ZEXPORT unzGetCurrentFileInfo64 OF((unzFile file, + unz_file_info64 *pfile_info, + char *szFileName, + uLong fileNameBufferSize, + void *extraField, + uLong extraFieldBufferSize, + char *szComment, + uLong commentBufferSize)); + +extern int ZEXPORT unzGetCurrentFileInfo OF((unzFile file, + unz_file_info *pfile_info, + char *szFileName, + uLong fileNameBufferSize, + void *extraField, + uLong extraFieldBufferSize, + char *szComment, + uLong commentBufferSize)); +/* + Get Info about the current file + if pfile_info!=NULL, the *pfile_info structure will contain somes info about + the current file + if szFileName!=NULL, the filemane string will be copied in szFileName + (fileNameBufferSize is the size of the buffer) + if extraField!=NULL, the extra field information will be copied in extraField + (extraFieldBufferSize is the size of the buffer). + This is the Central-header version of the extra field + if szComment!=NULL, the comment string of the file will be copied in szComment + (commentBufferSize is the size of the buffer) +*/ + + +/** Addition for GDAL : START */ + +extern ZPOS64_T ZEXPORT unzGetCurrentFileZStreamPos64 OF((unzFile file)); + +/** Addition for GDAL : END */ + + +/***************************************************************************/ +/* for reading the content of the current zipfile, you can open it, read data + from it, and close it (you can close it before reading all the file) + */ + +extern int ZEXPORT unzOpenCurrentFile OF((unzFile file)); +/* + Open for reading data the current file in the zipfile. + If there is no error, the return value is UNZ_OK. +*/ + +extern int ZEXPORT unzOpenCurrentFilePassword OF((unzFile file, + const char* password)); +/* + Open for reading data the current file in the zipfile. + password is a crypting password + If there is no error, the return value is UNZ_OK. +*/ + +extern int ZEXPORT unzOpenCurrentFile2 OF((unzFile file, + int* method, + int* level, + int raw)); +/* + Same than unzOpenCurrentFile, but open for read raw the file (not uncompress) + if raw==1 + *method will receive method of compression, *level will receive level of + compression + note : you can set level parameter as NULL (if you did not want known level, + but you CANNOT set method parameter as NULL +*/ + +extern int ZEXPORT unzOpenCurrentFile3 OF((unzFile file, + int* method, + int* level, + int raw, + const char* password)); +/* + Same than unzOpenCurrentFile, but open for read raw the file (not uncompress) + if raw==1 + *method will receive method of compression, *level will receive level of + compression + note : you can set level parameter as NULL (if you did not want known level, + but you CANNOT set method parameter as NULL +*/ + + +extern int ZEXPORT unzCloseCurrentFile OF((unzFile file)); +/* + Close the file in zip opened with unzOpenCurrentFile + Return UNZ_CRCERROR if all the file was read but the CRC is not good +*/ + +extern int ZEXPORT unzReadCurrentFile OF((unzFile file, + voidp buf, + unsigned len)); +/* + Read bytes from the current file (opened by unzOpenCurrentFile) + buf contain buffer where data must be copied + len the size of buf. + + return the number of byte copied if somes bytes are copied + return 0 if the end of file was reached + return <0 with error code if there is an error + (UNZ_ERRNO for IO error, or zLib error for uncompress error) +*/ + +extern z_off_t ZEXPORT unztell OF((unzFile file)); + +extern ZPOS64_T ZEXPORT unztell64 OF((unzFile file)); +/* + Give the current position in uncompressed data +*/ + +extern int ZEXPORT unzeof OF((unzFile file)); +/* + return 1 if the end of file was reached, 0 elsewhere +*/ + +extern int ZEXPORT unzGetLocalExtrafield OF((unzFile file, + voidp buf, + unsigned len)); +/* + Read extra field from the current file (opened by unzOpenCurrentFile) + This is the local-header version of the extra field (sometimes, there is + more info in the local-header version than in the central-header) + + if buf==NULL, it return the size of the local extra field + + if buf!=NULL, len is the size of the buffer, the extra header is copied in + buf. + the return value is the number of bytes copied in buf, or (if <0) + the error code +*/ + +/***************************************************************************/ + +/* Get the current file offset */ +extern ZPOS64_T ZEXPORT unzGetOffset64 (unzFile file); +extern uLong ZEXPORT unzGetOffset (unzFile file); + +/* Set the current file offset */ +extern int ZEXPORT unzSetOffset64 (unzFile file, ZPOS64_T pos); +extern int ZEXPORT unzSetOffset (unzFile file, uLong pos); + + + +#ifdef __cplusplus +} +#endif + +#endif /* _unz64_H */ diff --git a/storage/connect/value.cpp b/storage/connect/value.cpp index 64d0e13e8c4..ced690e77c0 100644 --- a/storage/connect/value.cpp +++ b/storage/connect/value.cpp @@ -1,7 +1,7 @@ /************* Value C++ Functions Source Code File (.CPP) *************/ -/* Name: VALUE.CPP Version 2.5 */ +/* Name: VALUE.CPP Version 2.6 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 2001-2015 */ +/* (C) Copyright to the author Olivier BERTRAND 2001-2016 */ /* */ /* This file contains the VALUE and derived classes family functions. */ /* These classes contain values of different types. They are used so */ @@ -792,19 +792,29 @@ uchar TYPVAL<uchar>::GetTypedValue(PVBLK blk, int n) /***********************************************************************/ /* TYPVAL SetBinValue: with bytes extracted from a line. */ +/* Currently only used reading column of binary files. */ /***********************************************************************/ template <class TYPE> void TYPVAL<TYPE>::SetBinValue(void *p) - { - Tval = *(TYPE *)p; - Null = false; - } // end of SetBinValue +{ +#if defined(UNALIGNED_OK) + // x86 can cast non-aligned memory directly + Tval = *(TYPE *)p; +#else + // Prevent unaligned memory access on MIPS and ArmHF platforms. + // Make use of memcpy instead of straight pointer dereferencing. + // Currently only used by WriteColumn of binary files. + // From original author: Vicentiu Ciorbaru <vicentiu@mariadb.org> + memcpy(&Tval, p, sizeof(TYPE)); +#endif + Null = false; +} // end of SetBinValue /***********************************************************************/ /* GetBinValue: fill a buffer with the internal binary value. */ /* This function checks whether the buffer length is enough and */ /* returns true if not. Actual filling occurs only if go is true. */ -/* Currently used by WriteColumn of binary files. */ +/* Currently only used writing column of binary files. */ /***********************************************************************/ template <class TYPE> bool TYPVAL<TYPE>::GetBinValue(void *buf, int buflen, bool go) @@ -819,7 +829,16 @@ bool TYPVAL<TYPE>::GetBinValue(void *buf, int buflen, bool go) //#endif if (go) - *(TYPE *)buf = Tval; +#if defined(UNALIGNED_OK) + // x86 can cast non-aligned memory directly + *(TYPE *)buf = Tval; +#else + // Prevent unaligned memory access on MIPS and ArmHF platforms. + // Make use of memcpy instead of straight pointer dereferencing. + // Currently only used by WriteColumn of binary files. + // From original author: Vicentiu Ciorbaru <vicentiu@mariadb.org> + memcpy(buf, &Tval, sizeof(TYPE)); +#endif Null = false; return false; diff --git a/storage/connect/value.h b/storage/connect/value.h index c5a381e89da..a670ade4c28 100644 --- a/storage/connect/value.h +++ b/storage/connect/value.h @@ -1,7 +1,7 @@ /**************** Value H Declares Source Code File (.H) ***************/ -/* Name: VALUE.H Version 2.1 */ +/* Name: VALUE.H Version 2.2 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 2001-2014 */ +/* (C) Copyright to the author Olivier BERTRAND 2001-2016 */ /* */ /* This file contains the VALUE and derived classes declares. */ /***********************************************************************/ @@ -17,6 +17,13 @@ #include "block.h" /***********************************************************************/ +/* This should list the processors accepting unaligned numeral values.*/ +/***********************************************************************/ +#if defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) || defined(_M_X64) || defined(_M_AMD64) || defined(_M_IA64) +#define UNALIGNED_OK +#endif + +/***********************************************************************/ /* Types used in some class definitions. */ /***********************************************************************/ enum CONV {CNV_ANY = 0, /* Convert to any type */ @@ -116,27 +123,46 @@ class DllExport VALUE : public BLOCK { virtual bool Compute(PGLOBAL g, PVAL *vp, int np, OPVAL op); virtual bool FormatValue(PVAL vp, char *fmt) = 0; - /** - Set value from a non-aligned in-memory value in the machine byte order. - TYPE can be either of: - - int, short, longlong - - uint, ushort, ulonglong - - float, double - @param - a pointer to a non-aligned value of type TYPE. - */ - template<typename TYPE> - void SetValueNonAligned(const char *p) - { -#if defined(__i386__) || defined(__x86_64__) - SetValue(*((TYPE*) p)); // x86 can cast non-aligned memory directly + /** + Set value from a non-aligned in-memory value in the machine byte order. + TYPE can be either of: + - int, short, longlong + - uint, ushort, ulonglong + - float, double + @param - a pointer to a non-aligned value of type TYPE. + */ + template<typename TYPE> + void SetValueNonAligned(const char *p) + { +#if defined(UNALIGNED_OK) + SetValue(*((TYPE*)p)); // x86 can cast non-aligned memory directly #else - TYPE tmp; // a slower version for non-x86 platforms - memcpy(&tmp, p, sizeof(tmp)); - SetValue(tmp); + TYPE tmp; // a slower version for non-x86 platforms + memcpy(&tmp, p, sizeof(tmp)); + SetValue(tmp); #endif - } + } // end of SetValueNonAligned + + /** + Get value from a non-aligned in-memory value in the machine byte order. + TYPE can be either of: + - int, short, longlong + - uint, ushort, ulonglong + - float, double + @params - a pointer to a non-aligned value of type TYPE, the TYPE value. + */ + template<typename TYPE> + void GetValueNonAligned(char *p, TYPE n) + { +#if defined(UNALIGNED_OK) + *(TYPE *)p = n; // x86 can cast non-aligned memory directly +#else + TYPE tmp = n; // a slower version for non-x86 platforms + memcpy(p, &tmp, sizeof(tmp)); +#endif + } // end of SetValueNonAligned - protected: +protected: virtual bool SetConstFormat(PGLOBAL, FORMAT&) = 0; const char *GetXfmt(void); diff --git a/storage/connect/xindex.cpp b/storage/connect/xindex.cpp index 56312630278..a2cf4e77b80 100755 --- a/storage/connect/xindex.cpp +++ b/storage/connect/xindex.cpp @@ -45,7 +45,9 @@ //nclude "array.h" #include "filamtxt.h" #include "tabdos.h" +#if defined(VCT_SUPPORT) #include "tabvct.h" +#endif // VCT_SUPPORT /***********************************************************************/ /* Macro or external routine definition */ @@ -293,9 +295,11 @@ bool XINDEX::AddColumns(void) return false; // Not applying to static index else if (IsMul()) return false; // Not done yet for multiple index - else if (Tbxp->GetAmType() == TYPE_AM_VCT && ((PTDBVCT)Tbxp)->IsSplit()) +#if defined(VCT_SUPPORT) + else if (Tbxp->GetAmType() == TYPE_AM_VCT && ((PTDBVCT)Tbxp)->IsSplit()) return false; // This would require to read additional files - else +#endif // VCT_SUPPORT + else return true; } // end of AddColumns diff --git a/storage/connect/zip.c b/storage/connect/zip.c new file mode 100644 index 00000000000..ea54853e858 --- /dev/null +++ b/storage/connect/zip.c @@ -0,0 +1,2007 @@ +/* zip.c -- IO on .zip files using zlib + Version 1.1, February 14h, 2010 + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications for Zip64 support + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + + Changes + Oct-2009 - Mathias Svensson - Remove old C style function prototypes + Oct-2009 - Mathias Svensson - Added Zip64 Support when creating new file archives + Oct-2009 - Mathias Svensson - Did some code cleanup and refactoring to get better overview of some functions. + Oct-2009 - Mathias Svensson - Added zipRemoveExtraInfoBlock to strip extra field data from its ZIP64 data + It is used when recreting zip archive with RAW when deleting items from a zip. + ZIP64 data is automaticly added to items that needs it, and existing ZIP64 data need to be removed. + Oct-2009 - Mathias Svensson - Added support for BZIP2 as compression mode (bzip2 lib is required) + Jan-2010 - back to unzip and minizip 1.0 name scheme, with compatibility layer + +*/ + + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include "zlib.h" +#include "zip.h" + +#ifdef STDC +# include <stddef.h> +# include <string.h> +# include <stdlib.h> +#endif +#ifdef NO_ERRNO_H + extern int errno; +#else +# include <errno.h> +#endif + + +#ifndef local +# define local static +#endif +/* compile with -Dlocal if your debugger can't find static symbols */ + +#ifndef VERSIONMADEBY +# define VERSIONMADEBY (0x0) /* platform depedent */ +#endif + +#ifndef Z_BUFSIZE +#define Z_BUFSIZE (64*1024) //(16384) +#endif + +#ifndef Z_MAXFILENAMEINZIP +#define Z_MAXFILENAMEINZIP (256) +#endif + +#ifndef ALLOC +# define ALLOC(size) (malloc(size)) +#endif +#ifndef TRYFREE +# define TRYFREE(p) {if (p) free(p);} +#endif + +/* +#define SIZECENTRALDIRITEM (0x2e) +#define SIZEZIPLOCALHEADER (0x1e) +*/ + +/* I've found an old Unix (a SunOS 4.1.3_U1) without all SEEK_* defined.... */ + + +// NOT sure that this work on ALL platform +#define MAKEULONG64(a, b) ((ZPOS64_T)(((unsigned long)(a)) | ((ZPOS64_T)((unsigned long)(b))) << 32)) + +#ifndef SEEK_CUR +#define SEEK_CUR 1 +#endif + +#ifndef SEEK_END +#define SEEK_END 2 +#endif + +#ifndef SEEK_SET +#define SEEK_SET 0 +#endif + +#ifndef DEF_MEM_LEVEL +#if MAX_MEM_LEVEL >= 8 +# define DEF_MEM_LEVEL 8 +#else +# define DEF_MEM_LEVEL MAX_MEM_LEVEL +#endif +#endif +const char zip_copyright[] =" zip 1.01 Copyright 1998-2004 Gilles Vollant - http://www.winimage.com/zLibDll"; + + +#define SIZEDATA_INDATABLOCK (4096-(4*4)) + +#define LOCALHEADERMAGIC (0x04034b50) +#define CENTRALHEADERMAGIC (0x02014b50) +#define ENDHEADERMAGIC (0x06054b50) +#define ZIP64ENDHEADERMAGIC (0x6064b50) +#define ZIP64ENDLOCHEADERMAGIC (0x7064b50) + +#define FLAG_LOCALHEADER_OFFSET (0x06) +#define CRC_LOCALHEADER_OFFSET (0x0e) + +#define SIZECENTRALHEADER (0x2e) /* 46 */ + +typedef struct linkedlist_datablock_internal_s +{ + struct linkedlist_datablock_internal_s* next_datablock; + uLong avail_in_this_block; + uLong filled_in_this_block; + uLong unused; /* for future use and alignement */ + unsigned char data[SIZEDATA_INDATABLOCK]; +} linkedlist_datablock_internal; + +typedef struct linkedlist_data_s +{ + linkedlist_datablock_internal* first_block; + linkedlist_datablock_internal* last_block; +} linkedlist_data; + + +typedef struct +{ + z_stream stream; /* zLib stream structure for inflate */ +#ifdef HAVE_BZIP2 + bz_stream bstream; /* bzLib stream structure for bziped */ +#endif + + int stream_initialised; /* 1 is stream is initialised */ + uInt pos_in_buffered_data; /* last written byte in buffered_data */ + + ZPOS64_T pos_local_header; /* offset of the local header of the file + currenty writing */ + char* central_header; /* central header data for the current file */ + uLong size_centralExtra; + uLong size_centralheader; /* size of the central header for cur file */ + uLong size_centralExtraFree; /* Extra bytes allocated to the centralheader but that are not used */ + uLong flag; /* flag of the file currently writing */ + + int method; /* compression method of file currenty wr.*/ + int raw; /* 1 for directly writing raw data */ + Byte buffered_data[Z_BUFSIZE];/* buffer contain compressed data to be writ*/ + uLong dosDate; + uLong crc32; + int encrypt; + int zip64; /* Add ZIP64 extened information in the extra field */ + ZPOS64_T pos_zip64extrainfo; + ZPOS64_T totalCompressedData; + ZPOS64_T totalUncompressedData; +#ifndef NOCRYPT + unsigned long keys[3]; /* keys defining the pseudo-random sequence */ + const z_crc_t* pcrc_32_tab; + int crypt_header_size; +#endif +} curfile64_info; + +typedef struct +{ + zlib_filefunc64_32_def z_filefunc; + voidpf filestream; /* io structore of the zipfile */ + linkedlist_data central_dir;/* datablock with central dir in construction*/ + int in_opened_file_inzip; /* 1 if a file in the zip is currently writ.*/ + curfile64_info ci; /* info on the file curretly writing */ + + ZPOS64_T begin_pos; /* position of the beginning of the zipfile */ + ZPOS64_T add_position_when_writting_offset; + ZPOS64_T number_entry; + +#ifndef NO_ADDFILEINEXISTINGZIP + char *globalcomment; +#endif + +} zip64_internal; + + +#ifndef NOCRYPT +#define INCLUDECRYPTINGCODE_IFCRYPTALLOWED +#include "crypt.h" +#endif + +local linkedlist_datablock_internal* allocate_new_datablock() +{ + linkedlist_datablock_internal* ldi; + ldi = (linkedlist_datablock_internal*) + ALLOC(sizeof(linkedlist_datablock_internal)); + if (ldi!=NULL) + { + ldi->next_datablock = NULL ; + ldi->filled_in_this_block = 0 ; + ldi->avail_in_this_block = SIZEDATA_INDATABLOCK ; + } + return ldi; +} + +local void free_datablock(linkedlist_datablock_internal* ldi) +{ + while (ldi!=NULL) + { + linkedlist_datablock_internal* ldinext = ldi->next_datablock; + TRYFREE(ldi); + ldi = ldinext; + } +} + +local void init_linkedlist(linkedlist_data* ll) +{ + ll->first_block = ll->last_block = NULL; +} + +local void free_linkedlist(linkedlist_data* ll) +{ + free_datablock(ll->first_block); + ll->first_block = ll->last_block = NULL; +} + + +local int add_data_in_datablock(linkedlist_data* ll, const void* buf, uLong len) +{ + linkedlist_datablock_internal* ldi; + const unsigned char* from_copy; + + if (ll==NULL) + return ZIP_INTERNALERROR; + + if (ll->last_block == NULL) + { + ll->first_block = ll->last_block = allocate_new_datablock(); + if (ll->first_block == NULL) + return ZIP_INTERNALERROR; + } + + ldi = ll->last_block; + from_copy = (unsigned char*)buf; + + while (len>0) + { + uInt copy_this; + uInt i; + unsigned char* to_copy; + + if (ldi->avail_in_this_block==0) + { + ldi->next_datablock = allocate_new_datablock(); + if (ldi->next_datablock == NULL) + return ZIP_INTERNALERROR; + ldi = ldi->next_datablock ; + ll->last_block = ldi; + } + + if (ldi->avail_in_this_block < len) + copy_this = (uInt)ldi->avail_in_this_block; + else + copy_this = (uInt)len; + + to_copy = &(ldi->data[ldi->filled_in_this_block]); + + for (i=0;i<copy_this;i++) + *(to_copy+i)=*(from_copy+i); + + ldi->filled_in_this_block += copy_this; + ldi->avail_in_this_block -= copy_this; + from_copy += copy_this ; + len -= copy_this; + } + return ZIP_OK; +} + + + +/****************************************************************************/ + +#ifndef NO_ADDFILEINEXISTINGZIP +/* =========================================================================== + Inputs a long in LSB order to the given file + nbByte == 1, 2 ,4 or 8 (byte, short or long, ZPOS64_T) +*/ + +local int zip64local_putValue OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, ZPOS64_T x, int nbByte)); +local int zip64local_putValue (const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, ZPOS64_T x, int nbByte) +{ + unsigned char buf[8]; + int n; + for (n = 0; n < nbByte; n++) + { + buf[n] = (unsigned char)(x & 0xff); + x >>= 8; + } + if (x != 0) + { /* data overflow - hack for ZIP64 (X Roche) */ + for (n = 0; n < nbByte; n++) + { + buf[n] = 0xff; + } + } + + if (ZWRITE64(*pzlib_filefunc_def,filestream,buf,nbByte)!=(uLong)nbByte) + return ZIP_ERRNO; + else + return ZIP_OK; +} + +local void zip64local_putValue_inmemory OF((void* dest, ZPOS64_T x, int nbByte)); +local void zip64local_putValue_inmemory (void* dest, ZPOS64_T x, int nbByte) +{ + unsigned char* buf=(unsigned char*)dest; + int n; + for (n = 0; n < nbByte; n++) { + buf[n] = (unsigned char)(x & 0xff); + x >>= 8; + } + + if (x != 0) + { /* data overflow - hack for ZIP64 */ + for (n = 0; n < nbByte; n++) + { + buf[n] = 0xff; + } + } +} + +/****************************************************************************/ + + +local uLong zip64local_TmzDateToDosDate(const tm_zip* ptm) +{ + uLong year = (uLong)ptm->tm_year; + if (year>=1980) + year-=1980; + else if (year>=80) + year-=80; + return + (uLong) (((ptm->tm_mday) + (32 * (ptm->tm_mon+1)) + (512 * year)) << 16) | + ((ptm->tm_sec/2) + (32* ptm->tm_min) + (2048 * (uLong)ptm->tm_hour)); +} + + +/****************************************************************************/ + +local int zip64local_getByte OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, int *pi)); + +local int zip64local_getByte(const zlib_filefunc64_32_def* pzlib_filefunc_def,voidpf filestream,int* pi) +{ + unsigned char c; + int err = (int)ZREAD64(*pzlib_filefunc_def,filestream,&c,1); + if (err==1) + { + *pi = (int)c; + return ZIP_OK; + } + else + { + if (ZERROR64(*pzlib_filefunc_def,filestream)) + return ZIP_ERRNO; + else + return ZIP_EOF; + } +} + + +/* =========================================================================== + Reads a long in LSB order from the given gz_stream. Sets +*/ +local int zip64local_getShort OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, uLong *pX)); + +local int zip64local_getShort (const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, uLong* pX) +{ + uLong x ; + int i = 0; + int err; + + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x = (uLong)i; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((uLong)i)<<8; + + if (err==ZIP_OK) + *pX = x; + else + *pX = 0; + return err; +} + +local int zip64local_getLong OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, uLong *pX)); + +local int zip64local_getLong (const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, uLong* pX) +{ + uLong x ; + int i = 0; + int err; + + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x = (uLong)i; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((uLong)i)<<8; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((uLong)i)<<16; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((uLong)i)<<24; + + if (err==ZIP_OK) + *pX = x; + else + *pX = 0; + return err; +} + +local int zip64local_getLong64 OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, ZPOS64_T *pX)); + + +local int zip64local_getLong64 (const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, ZPOS64_T *pX) +{ + ZPOS64_T x; + int i = 0; + int err; + + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x = (ZPOS64_T)i; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<8; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<16; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<24; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<32; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<40; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<48; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<56; + + if (err==ZIP_OK) + *pX = x; + else + *pX = 0; + + return err; +} + +#ifndef BUFREADCOMMENT +#define BUFREADCOMMENT (0x400) +#endif +/* + Locate the Central directory of a zipfile (at the end, just before + the global comment) +*/ +local ZPOS64_T zip64local_SearchCentralDir OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream)); + +local ZPOS64_T zip64local_SearchCentralDir(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream) +{ + unsigned char* buf; + ZPOS64_T uSizeFile; + ZPOS64_T uBackRead; + ZPOS64_T uMaxBack=0xffff; /* maximum size of global comment */ + ZPOS64_T uPosFound=0; + + if (ZSEEK64(*pzlib_filefunc_def,filestream,0,ZLIB_FILEFUNC_SEEK_END) != 0) + return 0; + + + uSizeFile = ZTELL64(*pzlib_filefunc_def,filestream); + + if (uMaxBack>uSizeFile) + uMaxBack = uSizeFile; + + buf = (unsigned char*)ALLOC(BUFREADCOMMENT+4); + if (buf==NULL) + return 0; + + uBackRead = 4; + while (uBackRead<uMaxBack) + { + uLong uReadSize; + ZPOS64_T uReadPos ; + int i; + if (uBackRead+BUFREADCOMMENT>uMaxBack) + uBackRead = uMaxBack; + else + uBackRead+=BUFREADCOMMENT; + uReadPos = uSizeFile-uBackRead ; + + uReadSize = ((BUFREADCOMMENT+4) < (uSizeFile-uReadPos)) ? + (BUFREADCOMMENT+4) : (uLong)(uSizeFile-uReadPos); + if (ZSEEK64(*pzlib_filefunc_def,filestream,uReadPos,ZLIB_FILEFUNC_SEEK_SET)!=0) + break; + + if (ZREAD64(*pzlib_filefunc_def,filestream,buf,uReadSize)!=uReadSize) + break; + + for (i=(int)uReadSize-3; (i--)>0;) + if (((*(buf+i))==0x50) && ((*(buf+i+1))==0x4b) && + ((*(buf+i+2))==0x05) && ((*(buf+i+3))==0x06)) + { + uPosFound = uReadPos+i; + break; + } + + if (uPosFound!=0) + break; + } + TRYFREE(buf); + return uPosFound; +} + +/* +Locate the End of Zip64 Central directory locator and from there find the CD of a zipfile (at the end, just before +the global comment) +*/ +local ZPOS64_T zip64local_SearchCentralDir64 OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream)); + +local ZPOS64_T zip64local_SearchCentralDir64(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream) +{ + unsigned char* buf; + ZPOS64_T uSizeFile; + ZPOS64_T uBackRead; + ZPOS64_T uMaxBack=0xffff; /* maximum size of global comment */ + ZPOS64_T uPosFound=0; + uLong uL; + ZPOS64_T relativeOffset; + + if (ZSEEK64(*pzlib_filefunc_def,filestream,0,ZLIB_FILEFUNC_SEEK_END) != 0) + return 0; + + uSizeFile = ZTELL64(*pzlib_filefunc_def,filestream); + + if (uMaxBack>uSizeFile) + uMaxBack = uSizeFile; + + buf = (unsigned char*)ALLOC(BUFREADCOMMENT+4); + if (buf==NULL) + return 0; + + uBackRead = 4; + while (uBackRead<uMaxBack) + { + uLong uReadSize; + ZPOS64_T uReadPos; + int i; + if (uBackRead+BUFREADCOMMENT>uMaxBack) + uBackRead = uMaxBack; + else + uBackRead+=BUFREADCOMMENT; + uReadPos = uSizeFile-uBackRead ; + + uReadSize = ((BUFREADCOMMENT+4) < (uSizeFile-uReadPos)) ? + (BUFREADCOMMENT+4) : (uLong)(uSizeFile-uReadPos); + if (ZSEEK64(*pzlib_filefunc_def,filestream,uReadPos,ZLIB_FILEFUNC_SEEK_SET)!=0) + break; + + if (ZREAD64(*pzlib_filefunc_def,filestream,buf,uReadSize)!=uReadSize) + break; + + for (i=(int)uReadSize-3; (i--)>0;) + { + // Signature "0x07064b50" Zip64 end of central directory locater + if (((*(buf+i))==0x50) && ((*(buf+i+1))==0x4b) && ((*(buf+i+2))==0x06) && ((*(buf+i+3))==0x07)) + { + uPosFound = uReadPos+i; + break; + } + } + + if (uPosFound!=0) + break; + } + + TRYFREE(buf); + if (uPosFound == 0) + return 0; + + /* Zip64 end of central directory locator */ + if (ZSEEK64(*pzlib_filefunc_def,filestream, uPosFound,ZLIB_FILEFUNC_SEEK_SET)!=0) + return 0; + + /* the signature, already checked */ + if (zip64local_getLong(pzlib_filefunc_def,filestream,&uL)!=ZIP_OK) + return 0; + + /* number of the disk with the start of the zip64 end of central directory */ + if (zip64local_getLong(pzlib_filefunc_def,filestream,&uL)!=ZIP_OK) + return 0; + if (uL != 0) + return 0; + + /* relative offset of the zip64 end of central directory record */ + if (zip64local_getLong64(pzlib_filefunc_def,filestream,&relativeOffset)!=ZIP_OK) + return 0; + + /* total number of disks */ + if (zip64local_getLong(pzlib_filefunc_def,filestream,&uL)!=ZIP_OK) + return 0; + if (uL != 1) + return 0; + + /* Goto Zip64 end of central directory record */ + if (ZSEEK64(*pzlib_filefunc_def,filestream, relativeOffset,ZLIB_FILEFUNC_SEEK_SET)!=0) + return 0; + + /* the signature */ + if (zip64local_getLong(pzlib_filefunc_def,filestream,&uL)!=ZIP_OK) + return 0; + + if (uL != 0x06064b50) // signature of 'Zip64 end of central directory' + return 0; + + return relativeOffset; +} + +int LoadCentralDirectoryRecord(zip64_internal* pziinit) +{ + int err=ZIP_OK; + ZPOS64_T byte_before_the_zipfile;/* byte before the zipfile, (>0 for sfx)*/ + + ZPOS64_T size_central_dir; /* size of the central directory */ + ZPOS64_T offset_central_dir; /* offset of start of central directory */ + ZPOS64_T central_pos; + uLong uL; + + uLong number_disk; /* number of the current dist, used for + spaning ZIP, unsupported, always 0*/ + uLong number_disk_with_CD; /* number the the disk with central dir, used + for spaning ZIP, unsupported, always 0*/ + ZPOS64_T number_entry; + ZPOS64_T number_entry_CD; /* total number of entries in + the central dir + (same than number_entry on nospan) */ + uLong VersionMadeBy; + uLong VersionNeeded; + uLong size_comment; + + int hasZIP64Record = 0; + + // check first if we find a ZIP64 record + central_pos = zip64local_SearchCentralDir64(&pziinit->z_filefunc,pziinit->filestream); + if(central_pos > 0) + { + hasZIP64Record = 1; + } + else if(central_pos == 0) + { + central_pos = zip64local_SearchCentralDir(&pziinit->z_filefunc,pziinit->filestream); + } + +/* disable to allow appending to empty ZIP archive + if (central_pos==0) + err=ZIP_ERRNO; +*/ + + if(hasZIP64Record) + { + ZPOS64_T sizeEndOfCentralDirectory; + if (ZSEEK64(pziinit->z_filefunc, pziinit->filestream, central_pos, ZLIB_FILEFUNC_SEEK_SET) != 0) + err=ZIP_ERRNO; + + /* the signature, already checked */ + if (zip64local_getLong(&pziinit->z_filefunc, pziinit->filestream,&uL)!=ZIP_OK) + err=ZIP_ERRNO; + + /* size of zip64 end of central directory record */ + if (zip64local_getLong64(&pziinit->z_filefunc, pziinit->filestream, &sizeEndOfCentralDirectory)!=ZIP_OK) + err=ZIP_ERRNO; + + /* version made by */ + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream, &VersionMadeBy)!=ZIP_OK) + err=ZIP_ERRNO; + + /* version needed to extract */ + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream, &VersionNeeded)!=ZIP_OK) + err=ZIP_ERRNO; + + /* number of this disk */ + if (zip64local_getLong(&pziinit->z_filefunc, pziinit->filestream,&number_disk)!=ZIP_OK) + err=ZIP_ERRNO; + + /* number of the disk with the start of the central directory */ + if (zip64local_getLong(&pziinit->z_filefunc, pziinit->filestream,&number_disk_with_CD)!=ZIP_OK) + err=ZIP_ERRNO; + + /* total number of entries in the central directory on this disk */ + if (zip64local_getLong64(&pziinit->z_filefunc, pziinit->filestream, &number_entry)!=ZIP_OK) + err=ZIP_ERRNO; + + /* total number of entries in the central directory */ + if (zip64local_getLong64(&pziinit->z_filefunc, pziinit->filestream,&number_entry_CD)!=ZIP_OK) + err=ZIP_ERRNO; + + if ((number_entry_CD!=number_entry) || (number_disk_with_CD!=0) || (number_disk!=0)) + err=ZIP_BADZIPFILE; + + /* size of the central directory */ + if (zip64local_getLong64(&pziinit->z_filefunc, pziinit->filestream,&size_central_dir)!=ZIP_OK) + err=ZIP_ERRNO; + + /* offset of start of central directory with respect to the + starting disk number */ + if (zip64local_getLong64(&pziinit->z_filefunc, pziinit->filestream,&offset_central_dir)!=ZIP_OK) + err=ZIP_ERRNO; + + // TODO.. + // read the comment from the standard central header. + size_comment = 0; + } + else + { + // Read End of central Directory info + if (ZSEEK64(pziinit->z_filefunc, pziinit->filestream, central_pos,ZLIB_FILEFUNC_SEEK_SET)!=0) + err=ZIP_ERRNO; + + /* the signature, already checked */ + if (zip64local_getLong(&pziinit->z_filefunc, pziinit->filestream,&uL)!=ZIP_OK) + err=ZIP_ERRNO; + + /* number of this disk */ + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream,&number_disk)!=ZIP_OK) + err=ZIP_ERRNO; + + /* number of the disk with the start of the central directory */ + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream,&number_disk_with_CD)!=ZIP_OK) + err=ZIP_ERRNO; + + /* total number of entries in the central dir on this disk */ + number_entry = 0; + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream, &uL)!=ZIP_OK) + err=ZIP_ERRNO; + else + number_entry = uL; + + /* total number of entries in the central dir */ + number_entry_CD = 0; + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream, &uL)!=ZIP_OK) + err=ZIP_ERRNO; + else + number_entry_CD = uL; + + if ((number_entry_CD!=number_entry) || (number_disk_with_CD!=0) || (number_disk!=0)) + err=ZIP_BADZIPFILE; + + /* size of the central directory */ + size_central_dir = 0; + if (zip64local_getLong(&pziinit->z_filefunc, pziinit->filestream, &uL)!=ZIP_OK) + err=ZIP_ERRNO; + else + size_central_dir = uL; + + /* offset of start of central directory with respect to the starting disk number */ + offset_central_dir = 0; + if (zip64local_getLong(&pziinit->z_filefunc, pziinit->filestream, &uL)!=ZIP_OK) + err=ZIP_ERRNO; + else + offset_central_dir = uL; + + + /* zipfile global comment length */ + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream, &size_comment)!=ZIP_OK) + err=ZIP_ERRNO; + } + + if ((central_pos<offset_central_dir+size_central_dir) && + (err==ZIP_OK)) + err=ZIP_BADZIPFILE; + + if (err!=ZIP_OK) + { + ZCLOSE64(pziinit->z_filefunc, pziinit->filestream); + return ZIP_ERRNO; + } + + if (size_comment>0) + { + pziinit->globalcomment = (char*)ALLOC(size_comment+1); + if (pziinit->globalcomment) + { + size_comment = ZREAD64(pziinit->z_filefunc, pziinit->filestream, pziinit->globalcomment,size_comment); + pziinit->globalcomment[size_comment]=0; + } + } + + byte_before_the_zipfile = central_pos - (offset_central_dir+size_central_dir); + pziinit->add_position_when_writting_offset = byte_before_the_zipfile; + + { + ZPOS64_T size_central_dir_to_read = size_central_dir; + size_t buf_size = SIZEDATA_INDATABLOCK; + void* buf_read = (void*)ALLOC(buf_size); + if (ZSEEK64(pziinit->z_filefunc, pziinit->filestream, offset_central_dir + byte_before_the_zipfile, ZLIB_FILEFUNC_SEEK_SET) != 0) + err=ZIP_ERRNO; + + while ((size_central_dir_to_read>0) && (err==ZIP_OK)) + { + ZPOS64_T read_this = SIZEDATA_INDATABLOCK; + if (read_this > size_central_dir_to_read) + read_this = size_central_dir_to_read; + + if (ZREAD64(pziinit->z_filefunc, pziinit->filestream,buf_read,(uLong)read_this) != read_this) + err=ZIP_ERRNO; + + if (err==ZIP_OK) + err = add_data_in_datablock(&pziinit->central_dir,buf_read, (uLong)read_this); + + size_central_dir_to_read-=read_this; + } + TRYFREE(buf_read); + } + pziinit->begin_pos = byte_before_the_zipfile; + pziinit->number_entry = number_entry_CD; + + if (ZSEEK64(pziinit->z_filefunc, pziinit->filestream, offset_central_dir+byte_before_the_zipfile,ZLIB_FILEFUNC_SEEK_SET) != 0) + err=ZIP_ERRNO; + + return err; +} + + +#endif /* !NO_ADDFILEINEXISTINGZIP*/ + + +/************************************************************/ +extern zipFile ZEXPORT zipOpen3 (const void *pathname, int append, zipcharpc* globalcomment, zlib_filefunc64_32_def* pzlib_filefunc64_32_def) +{ + zip64_internal ziinit; + zip64_internal* zi; + int err=ZIP_OK; + + ziinit.z_filefunc.zseek32_file = NULL; + ziinit.z_filefunc.ztell32_file = NULL; + if (pzlib_filefunc64_32_def==NULL) + fill_fopen64_filefunc(&ziinit.z_filefunc.zfile_func64); + else + ziinit.z_filefunc = *pzlib_filefunc64_32_def; + + ziinit.filestream = ZOPEN64(ziinit.z_filefunc, + pathname, + (append == APPEND_STATUS_CREATE) ? + (ZLIB_FILEFUNC_MODE_READ | ZLIB_FILEFUNC_MODE_WRITE | ZLIB_FILEFUNC_MODE_CREATE) : + (ZLIB_FILEFUNC_MODE_READ | ZLIB_FILEFUNC_MODE_WRITE | ZLIB_FILEFUNC_MODE_EXISTING)); + + if (ziinit.filestream == NULL) + return NULL; + + if (append == APPEND_STATUS_CREATEAFTER) + ZSEEK64(ziinit.z_filefunc,ziinit.filestream,0,SEEK_END); + + ziinit.begin_pos = ZTELL64(ziinit.z_filefunc,ziinit.filestream); + ziinit.in_opened_file_inzip = 0; + ziinit.ci.stream_initialised = 0; + ziinit.number_entry = 0; + ziinit.add_position_when_writting_offset = 0; + init_linkedlist(&(ziinit.central_dir)); + + + + zi = (zip64_internal*)ALLOC(sizeof(zip64_internal)); + if (zi==NULL) + { + ZCLOSE64(ziinit.z_filefunc,ziinit.filestream); + return NULL; + } + + /* now we add file in a zipfile */ +# ifndef NO_ADDFILEINEXISTINGZIP + ziinit.globalcomment = NULL; + if (append == APPEND_STATUS_ADDINZIP) + { + // Read and Cache Central Directory Records + err = LoadCentralDirectoryRecord(&ziinit); + } + + if (globalcomment) + { + *globalcomment = ziinit.globalcomment; + } +# endif /* !NO_ADDFILEINEXISTINGZIP*/ + + if (err != ZIP_OK) + { +# ifndef NO_ADDFILEINEXISTINGZIP + TRYFREE(ziinit.globalcomment); +# endif /* !NO_ADDFILEINEXISTINGZIP*/ + TRYFREE(zi); + return NULL; + } + else + { + *zi = ziinit; + return (zipFile)zi; + } +} + +extern zipFile ZEXPORT zipOpen2 (const char *pathname, int append, zipcharpc* globalcomment, zlib_filefunc_def* pzlib_filefunc32_def) +{ + if (pzlib_filefunc32_def != NULL) + { + zlib_filefunc64_32_def zlib_filefunc64_32_def_fill; + fill_zlib_filefunc64_32_def_from_filefunc32(&zlib_filefunc64_32_def_fill,pzlib_filefunc32_def); + return zipOpen3(pathname, append, globalcomment, &zlib_filefunc64_32_def_fill); + } + else + return zipOpen3(pathname, append, globalcomment, NULL); +} + +extern zipFile ZEXPORT zipOpen2_64 (const void *pathname, int append, zipcharpc* globalcomment, zlib_filefunc64_def* pzlib_filefunc_def) +{ + if (pzlib_filefunc_def != NULL) + { + zlib_filefunc64_32_def zlib_filefunc64_32_def_fill; + zlib_filefunc64_32_def_fill.zfile_func64 = *pzlib_filefunc_def; + zlib_filefunc64_32_def_fill.ztell32_file = NULL; + zlib_filefunc64_32_def_fill.zseek32_file = NULL; + return zipOpen3(pathname, append, globalcomment, &zlib_filefunc64_32_def_fill); + } + else + return zipOpen3(pathname, append, globalcomment, NULL); +} + + + +extern zipFile ZEXPORT zipOpen (const char* pathname, int append) +{ + return zipOpen3((const void*)pathname,append,NULL,NULL); +} + +extern zipFile ZEXPORT zipOpen64 (const void* pathname, int append) +{ + return zipOpen3(pathname,append,NULL,NULL); +} + +int Write_LocalFileHeader(zip64_internal* zi, const char* filename, uInt size_extrafield_local, const void* extrafield_local) +{ + /* write the local header */ + int err; + uInt size_filename = (uInt)strlen(filename); + uInt size_extrafield = size_extrafield_local; + + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)LOCALHEADERMAGIC, 4); + + if (err==ZIP_OK) + { + if(zi->ci.zip64) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)45,2);/* version needed to extract */ + else + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)20,2);/* version needed to extract */ + } + + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)zi->ci.flag,2); + + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)zi->ci.method,2); + + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)zi->ci.dosDate,4); + + // CRC / Compressed size / Uncompressed size will be filled in later and rewritten later + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,4); /* crc 32, unknown */ + if (err==ZIP_OK) + { + if(zi->ci.zip64) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0xFFFFFFFF,4); /* compressed size, unknown */ + else + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,4); /* compressed size, unknown */ + } + if (err==ZIP_OK) + { + if(zi->ci.zip64) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0xFFFFFFFF,4); /* uncompressed size, unknown */ + else + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,4); /* uncompressed size, unknown */ + } + + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)size_filename,2); + + if(zi->ci.zip64) + { + size_extrafield += 20; + } + + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)size_extrafield,2); + + if ((err==ZIP_OK) && (size_filename > 0)) + { + if (ZWRITE64(zi->z_filefunc,zi->filestream,filename,size_filename)!=size_filename) + err = ZIP_ERRNO; + } + + if ((err==ZIP_OK) && (size_extrafield_local > 0)) + { + if (ZWRITE64(zi->z_filefunc, zi->filestream, extrafield_local, size_extrafield_local) != size_extrafield_local) + err = ZIP_ERRNO; + } + + + if ((err==ZIP_OK) && (zi->ci.zip64)) + { + // write the Zip64 extended info + short HeaderID = 1; + short DataSize = 16; + ZPOS64_T CompressedSize = 0; + ZPOS64_T UncompressedSize = 0; + + // Remember position of Zip64 extended info for the local file header. (needed when we update size after done with file) + zi->ci.pos_zip64extrainfo = ZTELL64(zi->z_filefunc,zi->filestream); + + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, (short)HeaderID,2); + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, (short)DataSize,2); + + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, (ZPOS64_T)UncompressedSize,8); + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, (ZPOS64_T)CompressedSize,8); + } + + return err; +} + +/* + NOTE. + When writing RAW the ZIP64 extended information in extrafield_local and extrafield_global needs to be stripped + before calling this function it can be done with zipRemoveExtraInfoBlock + + It is not done here because then we need to realloc a new buffer since parameters are 'const' and I want to minimize + unnecessary allocations. + */ +extern int ZEXPORT zipOpenNewFileInZip4_64 (zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw, + int windowBits,int memLevel, int strategy, + const char* password, uLong crcForCrypting, + uLong versionMadeBy, uLong flagBase, int zip64) +{ + zip64_internal* zi; + uInt size_filename; + uInt size_comment; + uInt i; + int err = ZIP_OK; + +# ifdef NOCRYPT + (crcForCrypting); + if (password != NULL) + return ZIP_PARAMERROR; +# endif + + if (file == NULL) + return ZIP_PARAMERROR; + +#ifdef HAVE_BZIP2 + if ((method!=0) && (method!=Z_DEFLATED) && (method!=Z_BZIP2ED)) + return ZIP_PARAMERROR; +#else + if ((method!=0) && (method!=Z_DEFLATED)) + return ZIP_PARAMERROR; +#endif + + zi = (zip64_internal*)file; + + if (zi->in_opened_file_inzip == 1) + { + err = zipCloseFileInZip (file); + if (err != ZIP_OK) + return err; + } + + if (filename==NULL) + filename="-"; + + if (comment==NULL) + size_comment = 0; + else + size_comment = (uInt)strlen(comment); + + size_filename = (uInt)strlen(filename); + + if (zipfi == NULL) + zi->ci.dosDate = 0; + else + { + if (zipfi->dosDate != 0) + zi->ci.dosDate = zipfi->dosDate; + else + zi->ci.dosDate = zip64local_TmzDateToDosDate(&zipfi->tmz_date); + } + + zi->ci.flag = flagBase; + if ((level==8) || (level==9)) + zi->ci.flag |= 2; + if (level==2) + zi->ci.flag |= 4; + if (level==1) + zi->ci.flag |= 6; + if (password != NULL) + zi->ci.flag |= 1; + + zi->ci.crc32 = 0; + zi->ci.method = method; + zi->ci.encrypt = 0; + zi->ci.stream_initialised = 0; + zi->ci.pos_in_buffered_data = 0; + zi->ci.raw = raw; + zi->ci.pos_local_header = ZTELL64(zi->z_filefunc,zi->filestream); + + zi->ci.size_centralheader = SIZECENTRALHEADER + size_filename + size_extrafield_global + size_comment; + zi->ci.size_centralExtraFree = 32; // Extra space we have reserved in case we need to add ZIP64 extra info data + + zi->ci.central_header = (char*)ALLOC((uInt)zi->ci.size_centralheader + zi->ci.size_centralExtraFree); + + zi->ci.size_centralExtra = size_extrafield_global; + zip64local_putValue_inmemory(zi->ci.central_header,(uLong)CENTRALHEADERMAGIC,4); + /* version info */ + zip64local_putValue_inmemory(zi->ci.central_header+4,(uLong)versionMadeBy,2); + zip64local_putValue_inmemory(zi->ci.central_header+6,(uLong)20,2); + zip64local_putValue_inmemory(zi->ci.central_header+8,(uLong)zi->ci.flag,2); + zip64local_putValue_inmemory(zi->ci.central_header+10,(uLong)zi->ci.method,2); + zip64local_putValue_inmemory(zi->ci.central_header+12,(uLong)zi->ci.dosDate,4); + zip64local_putValue_inmemory(zi->ci.central_header+16,(uLong)0,4); /*crc*/ + zip64local_putValue_inmemory(zi->ci.central_header+20,(uLong)0,4); /*compr size*/ + zip64local_putValue_inmemory(zi->ci.central_header+24,(uLong)0,4); /*uncompr size*/ + zip64local_putValue_inmemory(zi->ci.central_header+28,(uLong)size_filename,2); + zip64local_putValue_inmemory(zi->ci.central_header+30,(uLong)size_extrafield_global,2); + zip64local_putValue_inmemory(zi->ci.central_header+32,(uLong)size_comment,2); + zip64local_putValue_inmemory(zi->ci.central_header+34,(uLong)0,2); /*disk nm start*/ + + if (zipfi==NULL) + zip64local_putValue_inmemory(zi->ci.central_header+36,(uLong)0,2); + else + zip64local_putValue_inmemory(zi->ci.central_header+36,(uLong)zipfi->internal_fa,2); + + if (zipfi==NULL) + zip64local_putValue_inmemory(zi->ci.central_header+38,(uLong)0,4); + else + zip64local_putValue_inmemory(zi->ci.central_header+38,(uLong)zipfi->external_fa,4); + + if(zi->ci.pos_local_header >= 0xffffffff) + zip64local_putValue_inmemory(zi->ci.central_header+42,(uLong)0xffffffff,4); + else + zip64local_putValue_inmemory(zi->ci.central_header+42,(uLong)zi->ci.pos_local_header - zi->add_position_when_writting_offset,4); + + for (i=0;i<size_filename;i++) + *(zi->ci.central_header+SIZECENTRALHEADER+i) = *(filename+i); + + for (i=0;i<size_extrafield_global;i++) + *(zi->ci.central_header+SIZECENTRALHEADER+size_filename+i) = + *(((const char*)extrafield_global)+i); + + for (i=0;i<size_comment;i++) + *(zi->ci.central_header+SIZECENTRALHEADER+size_filename+ + size_extrafield_global+i) = *(comment+i); + if (zi->ci.central_header == NULL) + return ZIP_INTERNALERROR; + + zi->ci.zip64 = zip64; + zi->ci.totalCompressedData = 0; + zi->ci.totalUncompressedData = 0; + zi->ci.pos_zip64extrainfo = 0; + + err = Write_LocalFileHeader(zi, filename, size_extrafield_local, extrafield_local); + +#ifdef HAVE_BZIP2 + zi->ci.bstream.avail_in = (uInt)0; + zi->ci.bstream.avail_out = (uInt)Z_BUFSIZE; + zi->ci.bstream.next_out = (char*)zi->ci.buffered_data; + zi->ci.bstream.total_in_hi32 = 0; + zi->ci.bstream.total_in_lo32 = 0; + zi->ci.bstream.total_out_hi32 = 0; + zi->ci.bstream.total_out_lo32 = 0; +#endif + + zi->ci.stream.avail_in = (uInt)0; + zi->ci.stream.avail_out = (uInt)Z_BUFSIZE; + zi->ci.stream.next_out = zi->ci.buffered_data; + zi->ci.stream.total_in = 0; + zi->ci.stream.total_out = 0; + zi->ci.stream.data_type = Z_BINARY; + +#ifdef HAVE_BZIP2 + if ((err==ZIP_OK) && (zi->ci.method == Z_DEFLATED || zi->ci.method == Z_BZIP2ED) && (!zi->ci.raw)) +#else + if ((err==ZIP_OK) && (zi->ci.method == Z_DEFLATED) && (!zi->ci.raw)) +#endif + { + if(zi->ci.method == Z_DEFLATED) + { + zi->ci.stream.zalloc = (alloc_func)0; + zi->ci.stream.zfree = (free_func)0; + zi->ci.stream.opaque = (voidpf)0; + + if (windowBits>0) + windowBits = -windowBits; + + err = deflateInit2(&zi->ci.stream, level, Z_DEFLATED, windowBits, memLevel, strategy); + + if (err==Z_OK) + zi->ci.stream_initialised = Z_DEFLATED; + } + else if(zi->ci.method == Z_BZIP2ED) + { +#ifdef HAVE_BZIP2 + // Init BZip stuff here + zi->ci.bstream.bzalloc = 0; + zi->ci.bstream.bzfree = 0; + zi->ci.bstream.opaque = (voidpf)0; + + err = BZ2_bzCompressInit(&zi->ci.bstream, level, 0,35); + if(err == BZ_OK) + zi->ci.stream_initialised = Z_BZIP2ED; +#endif + } + + } + +# ifndef NOCRYPT + zi->ci.crypt_header_size = 0; + if ((err==Z_OK) && (password != NULL)) + { + unsigned char bufHead[RAND_HEAD_LEN]; + unsigned int sizeHead; + zi->ci.encrypt = 1; + zi->ci.pcrc_32_tab = get_crc_table(); + /*init_keys(password,zi->ci.keys,zi->ci.pcrc_32_tab);*/ + + sizeHead=crypthead(password,bufHead,RAND_HEAD_LEN,zi->ci.keys,zi->ci.pcrc_32_tab,crcForCrypting); + zi->ci.crypt_header_size = sizeHead; + + if (ZWRITE64(zi->z_filefunc,zi->filestream,bufHead,sizeHead) != sizeHead) + err = ZIP_ERRNO; + } +# endif + + if (err==Z_OK) + zi->in_opened_file_inzip = 1; + return err; +} + +extern int ZEXPORT zipOpenNewFileInZip4 (zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw, + int windowBits,int memLevel, int strategy, + const char* password, uLong crcForCrypting, + uLong versionMadeBy, uLong flagBase) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, raw, + windowBits, memLevel, strategy, + password, crcForCrypting, versionMadeBy, flagBase, 0); +} + +extern int ZEXPORT zipOpenNewFileInZip3 (zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw, + int windowBits,int memLevel, int strategy, + const char* password, uLong crcForCrypting) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, raw, + windowBits, memLevel, strategy, + password, crcForCrypting, VERSIONMADEBY, 0, 0); +} + +extern int ZEXPORT zipOpenNewFileInZip3_64(zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw, + int windowBits,int memLevel, int strategy, + const char* password, uLong crcForCrypting, int zip64) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, raw, + windowBits, memLevel, strategy, + password, crcForCrypting, VERSIONMADEBY, 0, zip64); +} + +extern int ZEXPORT zipOpenNewFileInZip2(zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, raw, + -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, + NULL, 0, VERSIONMADEBY, 0, 0); +} + +extern int ZEXPORT zipOpenNewFileInZip2_64(zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw, int zip64) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, raw, + -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, + NULL, 0, VERSIONMADEBY, 0, zip64); +} + +extern int ZEXPORT zipOpenNewFileInZip64 (zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void*extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int zip64) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, 0, + -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, + NULL, 0, VERSIONMADEBY, 0, zip64); +} + +extern int ZEXPORT zipOpenNewFileInZip (zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void*extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, 0, + -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, + NULL, 0, VERSIONMADEBY, 0, 0); +} + +local int zip64FlushWriteBuffer(zip64_internal* zi) +{ + int err=ZIP_OK; + + if (zi->ci.encrypt != 0) + { +#ifndef NOCRYPT + uInt i; + int t; + for (i=0;i<zi->ci.pos_in_buffered_data;i++) + zi->ci.buffered_data[i] = zencode(zi->ci.keys, zi->ci.pcrc_32_tab, zi->ci.buffered_data[i],t); +#endif + } + + if (ZWRITE64(zi->z_filefunc,zi->filestream,zi->ci.buffered_data,zi->ci.pos_in_buffered_data) != zi->ci.pos_in_buffered_data) + err = ZIP_ERRNO; + + zi->ci.totalCompressedData += zi->ci.pos_in_buffered_data; + +#ifdef HAVE_BZIP2 + if(zi->ci.method == Z_BZIP2ED) + { + zi->ci.totalUncompressedData += zi->ci.bstream.total_in_lo32; + zi->ci.bstream.total_in_lo32 = 0; + zi->ci.bstream.total_in_hi32 = 0; + } + else +#endif + { + zi->ci.totalUncompressedData += zi->ci.stream.total_in; + zi->ci.stream.total_in = 0; + } + + + zi->ci.pos_in_buffered_data = 0; + + return err; +} + +extern int ZEXPORT zipWriteInFileInZip (zipFile file,const void* buf,unsigned int len) +{ + zip64_internal* zi; + int err=ZIP_OK; + + if (file == NULL) + return ZIP_PARAMERROR; + zi = (zip64_internal*)file; + + if (zi->in_opened_file_inzip == 0) + return ZIP_PARAMERROR; + + zi->ci.crc32 = crc32(zi->ci.crc32,buf,(uInt)len); + +#ifdef HAVE_BZIP2 + if(zi->ci.method == Z_BZIP2ED && (!zi->ci.raw)) + { + zi->ci.bstream.next_in = (void*)buf; + zi->ci.bstream.avail_in = len; + err = BZ_RUN_OK; + + while ((err==BZ_RUN_OK) && (zi->ci.bstream.avail_in>0)) + { + if (zi->ci.bstream.avail_out == 0) + { + if (zip64FlushWriteBuffer(zi) == ZIP_ERRNO) + err = ZIP_ERRNO; + zi->ci.bstream.avail_out = (uInt)Z_BUFSIZE; + zi->ci.bstream.next_out = (char*)zi->ci.buffered_data; + } + + + if(err != BZ_RUN_OK) + break; + + if ((zi->ci.method == Z_BZIP2ED) && (!zi->ci.raw)) + { + uLong uTotalOutBefore_lo = zi->ci.bstream.total_out_lo32; +// uLong uTotalOutBefore_hi = zi->ci.bstream.total_out_hi32; + err=BZ2_bzCompress(&zi->ci.bstream, BZ_RUN); + + zi->ci.pos_in_buffered_data += (uInt)(zi->ci.bstream.total_out_lo32 - uTotalOutBefore_lo) ; + } + } + + if(err == BZ_RUN_OK) + err = ZIP_OK; + } + else +#endif + { + zi->ci.stream.next_in = (Bytef*)buf; + zi->ci.stream.avail_in = len; + + while ((err==ZIP_OK) && (zi->ci.stream.avail_in>0)) + { + if (zi->ci.stream.avail_out == 0) + { + if (zip64FlushWriteBuffer(zi) == ZIP_ERRNO) + err = ZIP_ERRNO; + zi->ci.stream.avail_out = (uInt)Z_BUFSIZE; + zi->ci.stream.next_out = zi->ci.buffered_data; + } + + + if(err != ZIP_OK) + break; + + if ((zi->ci.method == Z_DEFLATED) && (!zi->ci.raw)) + { + uLong uTotalOutBefore = zi->ci.stream.total_out; + err=deflate(&zi->ci.stream, Z_NO_FLUSH); + if(uTotalOutBefore > zi->ci.stream.total_out) + { + int bBreak = 0; + bBreak++; + } + + zi->ci.pos_in_buffered_data += (uInt)(zi->ci.stream.total_out - uTotalOutBefore) ; + } + else + { + uInt copy_this,i; + if (zi->ci.stream.avail_in < zi->ci.stream.avail_out) + copy_this = zi->ci.stream.avail_in; + else + copy_this = zi->ci.stream.avail_out; + + for (i = 0; i < copy_this; i++) + *(((char*)zi->ci.stream.next_out)+i) = + *(((const char*)zi->ci.stream.next_in)+i); + { + zi->ci.stream.avail_in -= copy_this; + zi->ci.stream.avail_out-= copy_this; + zi->ci.stream.next_in+= copy_this; + zi->ci.stream.next_out+= copy_this; + zi->ci.stream.total_in+= copy_this; + zi->ci.stream.total_out+= copy_this; + zi->ci.pos_in_buffered_data += copy_this; + } + } + }// while(...) + } + + return err; +} + +extern int ZEXPORT zipCloseFileInZipRaw (zipFile file, uLong uncompressed_size, uLong crc32) +{ + return zipCloseFileInZipRaw64 (file, uncompressed_size, crc32); +} + +extern int ZEXPORT zipCloseFileInZipRaw64 (zipFile file, ZPOS64_T uncompressed_size, uLong crc32) +{ + zip64_internal* zi; + ZPOS64_T compressed_size; + uLong invalidValue = 0xffffffff; + short datasize = 0; + int err=ZIP_OK; + + if (file == NULL) + return ZIP_PARAMERROR; + zi = (zip64_internal*)file; + + if (zi->in_opened_file_inzip == 0) + return ZIP_PARAMERROR; + zi->ci.stream.avail_in = 0; + + if ((zi->ci.method == Z_DEFLATED) && (!zi->ci.raw)) + { + while (err==ZIP_OK) + { + uLong uTotalOutBefore; + if (zi->ci.stream.avail_out == 0) + { + if (zip64FlushWriteBuffer(zi) == ZIP_ERRNO) + err = ZIP_ERRNO; + zi->ci.stream.avail_out = (uInt)Z_BUFSIZE; + zi->ci.stream.next_out = zi->ci.buffered_data; + } + uTotalOutBefore = zi->ci.stream.total_out; + err=deflate(&zi->ci.stream, Z_FINISH); + zi->ci.pos_in_buffered_data += (uInt)(zi->ci.stream.total_out - uTotalOutBefore) ; + } + } + else if ((zi->ci.method == Z_BZIP2ED) && (!zi->ci.raw)) + { +#ifdef HAVE_BZIP2 + err = BZ_FINISH_OK; + while (err==BZ_FINISH_OK) + { + uLong uTotalOutBefore; + if (zi->ci.bstream.avail_out == 0) + { + if (zip64FlushWriteBuffer(zi) == ZIP_ERRNO) + err = ZIP_ERRNO; + zi->ci.bstream.avail_out = (uInt)Z_BUFSIZE; + zi->ci.bstream.next_out = (char*)zi->ci.buffered_data; + } + uTotalOutBefore = zi->ci.bstream.total_out_lo32; + err=BZ2_bzCompress(&zi->ci.bstream, BZ_FINISH); + if(err == BZ_STREAM_END) + err = Z_STREAM_END; + + zi->ci.pos_in_buffered_data += (uInt)(zi->ci.bstream.total_out_lo32 - uTotalOutBefore); + } + + if(err == BZ_FINISH_OK) + err = ZIP_OK; +#endif + } + + if (err==Z_STREAM_END) + err=ZIP_OK; /* this is normal */ + + if ((zi->ci.pos_in_buffered_data>0) && (err==ZIP_OK)) + { + if (zip64FlushWriteBuffer(zi)==ZIP_ERRNO) + err = ZIP_ERRNO; + } + + if ((zi->ci.method == Z_DEFLATED) && (!zi->ci.raw)) + { + int tmp_err = deflateEnd(&zi->ci.stream); + if (err == ZIP_OK) + err = tmp_err; + zi->ci.stream_initialised = 0; + } +#ifdef HAVE_BZIP2 + else if((zi->ci.method == Z_BZIP2ED) && (!zi->ci.raw)) + { + int tmperr = BZ2_bzCompressEnd(&zi->ci.bstream); + if (err==ZIP_OK) + err = tmperr; + zi->ci.stream_initialised = 0; + } +#endif + + if (!zi->ci.raw) + { + crc32 = (uLong)zi->ci.crc32; + uncompressed_size = zi->ci.totalUncompressedData; + } + compressed_size = zi->ci.totalCompressedData; + +# ifndef NOCRYPT + compressed_size += zi->ci.crypt_header_size; +# endif + + // update Current Item crc and sizes, + if(compressed_size >= 0xffffffff || uncompressed_size >= 0xffffffff || zi->ci.pos_local_header >= 0xffffffff) + { + /*version Made by*/ + zip64local_putValue_inmemory(zi->ci.central_header+4,(uLong)45,2); + /*version needed*/ + zip64local_putValue_inmemory(zi->ci.central_header+6,(uLong)45,2); + + } + + zip64local_putValue_inmemory(zi->ci.central_header+16,crc32,4); /*crc*/ + + + if(compressed_size >= 0xffffffff) + zip64local_putValue_inmemory(zi->ci.central_header+20, invalidValue,4); /*compr size*/ + else + zip64local_putValue_inmemory(zi->ci.central_header+20, compressed_size,4); /*compr size*/ + + /// set internal file attributes field + if (zi->ci.stream.data_type == Z_ASCII) + zip64local_putValue_inmemory(zi->ci.central_header+36,(uLong)Z_ASCII,2); + + if(uncompressed_size >= 0xffffffff) + zip64local_putValue_inmemory(zi->ci.central_header+24, invalidValue,4); /*uncompr size*/ + else + zip64local_putValue_inmemory(zi->ci.central_header+24, uncompressed_size,4); /*uncompr size*/ + + // Add ZIP64 extra info field for uncompressed size + if(uncompressed_size >= 0xffffffff) + datasize += 8; + + // Add ZIP64 extra info field for compressed size + if(compressed_size >= 0xffffffff) + datasize += 8; + + // Add ZIP64 extra info field for relative offset to local file header of current file + if(zi->ci.pos_local_header >= 0xffffffff) + datasize += 8; + + if(datasize > 0) + { + char* p = NULL; + + if((uLong)(datasize + 4) > zi->ci.size_centralExtraFree) + { + // we can not write more data to the buffer that we have room for. + return ZIP_BADZIPFILE; + } + + p = zi->ci.central_header + zi->ci.size_centralheader; + + // Add Extra Information Header for 'ZIP64 information' + zip64local_putValue_inmemory(p, 0x0001, 2); // HeaderID + p += 2; + zip64local_putValue_inmemory(p, datasize, 2); // DataSize + p += 2; + + if(uncompressed_size >= 0xffffffff) + { + zip64local_putValue_inmemory(p, uncompressed_size, 8); + p += 8; + } + + if(compressed_size >= 0xffffffff) + { + zip64local_putValue_inmemory(p, compressed_size, 8); + p += 8; + } + + if(zi->ci.pos_local_header >= 0xffffffff) + { + zip64local_putValue_inmemory(p, zi->ci.pos_local_header, 8); + p += 8; + } + + // Update how much extra free space we got in the memory buffer + // and increase the centralheader size so the new ZIP64 fields are included + // ( 4 below is the size of HeaderID and DataSize field ) + zi->ci.size_centralExtraFree -= datasize + 4; + zi->ci.size_centralheader += datasize + 4; + + // Update the extra info size field + zi->ci.size_centralExtra += datasize + 4; + zip64local_putValue_inmemory(zi->ci.central_header+30,(uLong)zi->ci.size_centralExtra,2); + } + + if (err==ZIP_OK) + err = add_data_in_datablock(&zi->central_dir, zi->ci.central_header, (uLong)zi->ci.size_centralheader); + + free(zi->ci.central_header); + + if (err==ZIP_OK) + { + // Update the LocalFileHeader with the new values. + + ZPOS64_T cur_pos_inzip = ZTELL64(zi->z_filefunc,zi->filestream); + + if (ZSEEK64(zi->z_filefunc,zi->filestream, zi->ci.pos_local_header + 14,ZLIB_FILEFUNC_SEEK_SET)!=0) + err = ZIP_ERRNO; + + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,crc32,4); /* crc 32, unknown */ + + if(uncompressed_size >= 0xffffffff || compressed_size >= 0xffffffff ) + { + if(zi->ci.pos_zip64extrainfo > 0) + { + // Update the size in the ZIP64 extended field. + if (ZSEEK64(zi->z_filefunc,zi->filestream, zi->ci.pos_zip64extrainfo + 4,ZLIB_FILEFUNC_SEEK_SET)!=0) + err = ZIP_ERRNO; + + if (err==ZIP_OK) /* compressed size, unknown */ + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, uncompressed_size, 8); + + if (err==ZIP_OK) /* uncompressed size, unknown */ + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, compressed_size, 8); + } + else + err = ZIP_BADZIPFILE; // Caller passed zip64 = 0, so no room for zip64 info -> fatal + } + else + { + if (err==ZIP_OK) /* compressed size, unknown */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,compressed_size,4); + + if (err==ZIP_OK) /* uncompressed size, unknown */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,uncompressed_size,4); + } + + if (ZSEEK64(zi->z_filefunc,zi->filestream, cur_pos_inzip,ZLIB_FILEFUNC_SEEK_SET)!=0) + err = ZIP_ERRNO; + } + + zi->number_entry ++; + zi->in_opened_file_inzip = 0; + + return err; +} + +extern int ZEXPORT zipCloseFileInZip (zipFile file) +{ + return zipCloseFileInZipRaw (file,0,0); +} + +int Write_Zip64EndOfCentralDirectoryLocator(zip64_internal* zi, ZPOS64_T zip64eocd_pos_inzip) +{ + int err = ZIP_OK; + ZPOS64_T pos = zip64eocd_pos_inzip - zi->add_position_when_writting_offset; + + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)ZIP64ENDLOCHEADERMAGIC,4); + + /*num disks*/ + if (err==ZIP_OK) /* number of the disk with the start of the central directory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,4); + + /*relative offset*/ + if (err==ZIP_OK) /* Relative offset to the Zip64EndOfCentralDirectory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream, pos,8); + + /*total disks*/ /* Do not support spawning of disk so always say 1 here*/ + if (err==ZIP_OK) /* number of the disk with the start of the central directory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)1,4); + + return err; +} + +int Write_Zip64EndOfCentralDirectoryRecord(zip64_internal* zi, uLong size_centraldir, ZPOS64_T centraldir_pos_inzip) +{ + int err = ZIP_OK; + + uLong Zip64DataSize = 44; + + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)ZIP64ENDHEADERMAGIC,4); + + if (err==ZIP_OK) /* size of this 'zip64 end of central directory' */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(ZPOS64_T)Zip64DataSize,8); // why ZPOS64_T of this ? + + if (err==ZIP_OK) /* version made by */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)45,2); + + if (err==ZIP_OK) /* version needed */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)45,2); + + if (err==ZIP_OK) /* number of this disk */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,4); + + if (err==ZIP_OK) /* number of the disk with the start of the central directory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,4); + + if (err==ZIP_OK) /* total number of entries in the central dir on this disk */ + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, zi->number_entry, 8); + + if (err==ZIP_OK) /* total number of entries in the central dir */ + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, zi->number_entry, 8); + + if (err==ZIP_OK) /* size of the central directory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(ZPOS64_T)size_centraldir,8); + + if (err==ZIP_OK) /* offset of start of central directory with respect to the starting disk number */ + { + ZPOS64_T pos = centraldir_pos_inzip - zi->add_position_when_writting_offset; + err = zip64local_putValue(&zi->z_filefunc,zi->filestream, (ZPOS64_T)pos,8); + } + return err; +} +int Write_EndOfCentralDirectoryRecord(zip64_internal* zi, uLong size_centraldir, ZPOS64_T centraldir_pos_inzip) +{ + int err = ZIP_OK; + + /*signature*/ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)ENDHEADERMAGIC,4); + + if (err==ZIP_OK) /* number of this disk */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,2); + + if (err==ZIP_OK) /* number of the disk with the start of the central directory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,2); + + if (err==ZIP_OK) /* total number of entries in the central dir on this disk */ + { + { + if(zi->number_entry >= 0xFFFF) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0xffff,2); // use value in ZIP64 record + else + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)zi->number_entry,2); + } + } + + if (err==ZIP_OK) /* total number of entries in the central dir */ + { + if(zi->number_entry >= 0xFFFF) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0xffff,2); // use value in ZIP64 record + else + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)zi->number_entry,2); + } + + if (err==ZIP_OK) /* size of the central directory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)size_centraldir,4); + + if (err==ZIP_OK) /* offset of start of central directory with respect to the starting disk number */ + { + ZPOS64_T pos = centraldir_pos_inzip - zi->add_position_when_writting_offset; + if(pos >= 0xffffffff) + { + err = zip64local_putValue(&zi->z_filefunc,zi->filestream, (uLong)0xffffffff,4); + } + else + err = zip64local_putValue(&zi->z_filefunc,zi->filestream, (uLong)(centraldir_pos_inzip - zi->add_position_when_writting_offset),4); + } + + return err; +} + +int Write_GlobalComment(zip64_internal* zi, const char* global_comment) +{ + int err = ZIP_OK; + uInt size_global_comment = 0; + + if(global_comment != NULL) + size_global_comment = (uInt)strlen(global_comment); + + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)size_global_comment,2); + + if (err == ZIP_OK && size_global_comment > 0) + { + if (ZWRITE64(zi->z_filefunc,zi->filestream, global_comment, size_global_comment) != size_global_comment) + err = ZIP_ERRNO; + } + return err; +} + +extern int ZEXPORT zipClose (zipFile file, const char* global_comment) +{ + zip64_internal* zi; + int err = 0; + uLong size_centraldir = 0; + ZPOS64_T centraldir_pos_inzip; + ZPOS64_T pos; + + if (file == NULL) + return ZIP_PARAMERROR; + + zi = (zip64_internal*)file; + + if (zi->in_opened_file_inzip == 1) + { + err = zipCloseFileInZip (file); + } + +#ifndef NO_ADDFILEINEXISTINGZIP + if (global_comment==NULL) + global_comment = zi->globalcomment; +#endif + + centraldir_pos_inzip = ZTELL64(zi->z_filefunc,zi->filestream); + + if (err==ZIP_OK) + { + linkedlist_datablock_internal* ldi = zi->central_dir.first_block; + while (ldi!=NULL) + { + if ((err==ZIP_OK) && (ldi->filled_in_this_block>0)) + { + if (ZWRITE64(zi->z_filefunc,zi->filestream, ldi->data, ldi->filled_in_this_block) != ldi->filled_in_this_block) + err = ZIP_ERRNO; + } + + size_centraldir += ldi->filled_in_this_block; + ldi = ldi->next_datablock; + } + } + free_linkedlist(&(zi->central_dir)); + + pos = centraldir_pos_inzip - zi->add_position_when_writting_offset; + if(pos >= 0xffffffff || zi->number_entry > 0xFFFF) + { + ZPOS64_T Zip64EOCDpos = ZTELL64(zi->z_filefunc,zi->filestream); + Write_Zip64EndOfCentralDirectoryRecord(zi, size_centraldir, centraldir_pos_inzip); + + Write_Zip64EndOfCentralDirectoryLocator(zi, Zip64EOCDpos); + } + + if (err==ZIP_OK) + err = Write_EndOfCentralDirectoryRecord(zi, size_centraldir, centraldir_pos_inzip); + + if(err == ZIP_OK) + err = Write_GlobalComment(zi, global_comment); + + if (ZCLOSE64(zi->z_filefunc,zi->filestream) != 0) + if (err == ZIP_OK) + err = ZIP_ERRNO; + +#ifndef NO_ADDFILEINEXISTINGZIP + TRYFREE(zi->globalcomment); +#endif + TRYFREE(zi); + + return err; +} + +extern int ZEXPORT zipRemoveExtraInfoBlock (char* pData, int* dataLen, short sHeader) +{ + char* p = pData; + int size = 0; + char* pNewHeader; + char* pTmp; + short header; + short dataSize; + + int retVal = ZIP_OK; + + if(pData == NULL || *dataLen < 4) + return ZIP_PARAMERROR; + + pNewHeader = (char*)ALLOC(*dataLen); + pTmp = pNewHeader; + + while(p < (pData + *dataLen)) + { + header = *(short*)p; + dataSize = *(((short*)p)+1); + + if( header == sHeader ) // Header found. + { + p += dataSize + 4; // skip it. do not copy to temp buffer + } + else + { + // Extra Info block should not be removed, So copy it to the temp buffer. + memcpy(pTmp, p, dataSize + 4); + p += dataSize + 4; + size += dataSize + 4; + } + + } + + if(size < *dataLen) + { + // clean old extra info block. + memset(pData,0, *dataLen); + + // copy the new extra info block over the old + if(size > 0) + memcpy(pData, pNewHeader, size); + + // set the new extra info size + *dataLen = size; + + retVal = ZIP_OK; + } + else + retVal = ZIP_ERRNO; + + TRYFREE(pNewHeader); + + return retVal; +} diff --git a/storage/connect/zip.h b/storage/connect/zip.h new file mode 100644 index 00000000000..8aaebb62343 --- /dev/null +++ b/storage/connect/zip.h @@ -0,0 +1,362 @@ +/* zip.h -- IO on .zip files using zlib + Version 1.1, February 14h, 2010 + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications for Zip64 support + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + + --------------------------------------------------------------------------- + + Condition of use and distribution are the same than zlib : + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + --------------------------------------------------------------------------- + + Changes + + See header of zip.h + +*/ + +#ifndef _zip12_H +#define _zip12_H + +#ifdef __cplusplus +extern "C" { +#endif + +//#define HAVE_BZIP2 + +#ifndef _ZLIB_H +#include "zlib.h" +#endif + +#ifndef _ZLIBIOAPI_H +#include "ioapi.h" +#endif + +#ifdef HAVE_BZIP2 +#include "bzlib.h" +#endif + +#define Z_BZIP2ED 12 + +#if defined(STRICTZIP) || defined(STRICTZIPUNZIP) +/* like the STRICT of WIN32, we define a pointer that cannot be converted + from (void*) without cast */ +typedef struct TagzipFile__ { int unused; } zipFile__; +typedef zipFile__ *zipFile; +#else +typedef voidp zipFile; +#endif + +#define ZIP_OK (0) +#define ZIP_EOF (0) +#define ZIP_ERRNO (Z_ERRNO) +#define ZIP_PARAMERROR (-102) +#define ZIP_BADZIPFILE (-103) +#define ZIP_INTERNALERROR (-104) + +#ifndef DEF_MEM_LEVEL +# if MAX_MEM_LEVEL >= 8 +# define DEF_MEM_LEVEL 8 +# else +# define DEF_MEM_LEVEL MAX_MEM_LEVEL +# endif +#endif +/* default memLevel */ + +/* tm_zip contain date/time info */ +typedef struct tm_zip_s +{ + uInt tm_sec; /* seconds after the minute - [0,59] */ + uInt tm_min; /* minutes after the hour - [0,59] */ + uInt tm_hour; /* hours since midnight - [0,23] */ + uInt tm_mday; /* day of the month - [1,31] */ + uInt tm_mon; /* months since January - [0,11] */ + uInt tm_year; /* years - [1980..2044] */ +} tm_zip; + +typedef struct +{ + tm_zip tmz_date; /* date in understandable format */ + uLong dosDate; /* if dos_date == 0, tmu_date is used */ +/* uLong flag; */ /* general purpose bit flag 2 bytes */ + + uLong internal_fa; /* internal file attributes 2 bytes */ + uLong external_fa; /* external file attributes 4 bytes */ +} zip_fileinfo; + +typedef const char* zipcharpc; + + +#define APPEND_STATUS_CREATE (0) +#define APPEND_STATUS_CREATEAFTER (1) +#define APPEND_STATUS_ADDINZIP (2) + +extern zipFile ZEXPORT zipOpen OF((const char *pathname, int append)); +extern zipFile ZEXPORT zipOpen64 OF((const void *pathname, int append)); +/* + Create a zipfile. + pathname contain on Windows XP a filename like "c:\\zlib\\zlib113.zip" or on + an Unix computer "zlib/zlib113.zip". + if the file pathname exist and append==APPEND_STATUS_CREATEAFTER, the zip + will be created at the end of the file. + (useful if the file contain a self extractor code) + if the file pathname exist and append==APPEND_STATUS_ADDINZIP, we will + add files in existing zip (be sure you don't add file that doesn't exist) + If the zipfile cannot be opened, the return value is NULL. + Else, the return value is a zipFile Handle, usable with other function + of this zip package. +*/ + +/* Note : there is no delete function into a zipfile. + If you want delete file into a zipfile, you must open a zipfile, and create another + Of couse, you can use RAW reading and writing to copy the file you did not want delte +*/ + +extern zipFile ZEXPORT zipOpen2 OF((const char *pathname, + int append, + zipcharpc* globalcomment, + zlib_filefunc_def* pzlib_filefunc_def)); + +extern zipFile ZEXPORT zipOpen2_64 OF((const void *pathname, + int append, + zipcharpc* globalcomment, + zlib_filefunc64_def* pzlib_filefunc_def)); + +extern int ZEXPORT zipOpenNewFileInZip OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level)); + +extern int ZEXPORT zipOpenNewFileInZip64 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int zip64)); + +/* + Open a file in the ZIP for writing. + filename : the filename in zip (if NULL, '-' without quote will be used + *zipfi contain supplemental information + if extrafield_local!=NULL and size_extrafield_local>0, extrafield_local + contains the extrafield data the the local header + if extrafield_global!=NULL and size_extrafield_global>0, extrafield_global + contains the extrafield data the the local header + if comment != NULL, comment contain the comment string + method contain the compression method (0 for store, Z_DEFLATED for deflate) + level contain the level of compression (can be Z_DEFAULT_COMPRESSION) + zip64 is set to 1 if a zip64 extended information block should be added to the local file header. + this MUST be '1' if the uncompressed size is >= 0xffffffff. + +*/ + + +extern int ZEXPORT zipOpenNewFileInZip2 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw)); + + +extern int ZEXPORT zipOpenNewFileInZip2_64 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw, + int zip64)); +/* + Same than zipOpenNewFileInZip, except if raw=1, we write raw file + */ + +extern int ZEXPORT zipOpenNewFileInZip3 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw, + int windowBits, + int memLevel, + int strategy, + const char* password, + uLong crcForCrypting)); + +extern int ZEXPORT zipOpenNewFileInZip3_64 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw, + int windowBits, + int memLevel, + int strategy, + const char* password, + uLong crcForCrypting, + int zip64 + )); + +/* + Same than zipOpenNewFileInZip2, except + windowBits,memLevel,,strategy : see parameter strategy in deflateInit2 + password : crypting password (NULL for no crypting) + crcForCrypting : crc of file to compress (needed for crypting) + */ + +extern int ZEXPORT zipOpenNewFileInZip4 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw, + int windowBits, + int memLevel, + int strategy, + const char* password, + uLong crcForCrypting, + uLong versionMadeBy, + uLong flagBase + )); + + +extern int ZEXPORT zipOpenNewFileInZip4_64 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw, + int windowBits, + int memLevel, + int strategy, + const char* password, + uLong crcForCrypting, + uLong versionMadeBy, + uLong flagBase, + int zip64 + )); +/* + Same than zipOpenNewFileInZip4, except + versionMadeBy : value for Version made by field + flag : value for flag field (compression level info will be added) + */ + + +extern int ZEXPORT zipWriteInFileInZip OF((zipFile file, + const void* buf, + unsigned len)); +/* + Write data in the zipfile +*/ + +extern int ZEXPORT zipCloseFileInZip OF((zipFile file)); +/* + Close the current file in the zipfile +*/ + +extern int ZEXPORT zipCloseFileInZipRaw OF((zipFile file, + uLong uncompressed_size, + uLong crc32)); + +extern int ZEXPORT zipCloseFileInZipRaw64 OF((zipFile file, + ZPOS64_T uncompressed_size, + uLong crc32)); + +/* + Close the current file in the zipfile, for file opened with + parameter raw=1 in zipOpenNewFileInZip2 + uncompressed_size and crc32 are value for the uncompressed size +*/ + +extern int ZEXPORT zipClose OF((zipFile file, + const char* global_comment)); +/* + Close the zipfile +*/ + + +extern int ZEXPORT zipRemoveExtraInfoBlock OF((char* pData, int* dataLen, short sHeader)); +/* + zipRemoveExtraInfoBlock - Added by Mathias Svensson + + Remove extra information block from a extra information data for the local file header or central directory header + + It is needed to remove ZIP64 extra information blocks when before data is written if using RAW mode. + + 0x0001 is the signature header for the ZIP64 extra information blocks + + usage. + Remove ZIP64 Extra information from a central director extra field data + zipRemoveExtraInfoBlock(pCenDirExtraFieldData, &nCenDirExtraFieldDataLen, 0x0001); + + Remove ZIP64 Extra information from a Local File Header extra field data + zipRemoveExtraInfoBlock(pLocalHeaderExtraFieldData, &nLocalHeaderExtraFieldDataLen, 0x0001); +*/ + +#ifdef __cplusplus +} +#endif + +#endif /* _zip64_H */ diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 90ddf473232..d6a2d9ba9e0 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -841,11 +841,15 @@ buf_page_is_corrupted( { ulint checksum_field1; ulint checksum_field2; - bool page_encrypted = false; + bool no_checksum = false; #ifndef UNIV_INNOCHECKSUM ulint space_id = mach_read_from_4( read_buf + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); + ulint page_type = mach_read_from_4( + read_buf + FIL_PAGE_TYPE); + no_checksum = (page_type == FIL_PAGE_PAGE_COMPRESSED || + page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED); fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id); /* Page is encrypted if encryption information is found from @@ -854,19 +858,24 @@ buf_page_is_corrupted( if (crypt_data && crypt_data->type != CRYPT_SCHEME_UNENCRYPTED && fil_page_is_encrypted(read_buf)) { - page_encrypted = true; + no_checksum = true; + } + + /* Return early if there is no checksum or END_LSN */ + if (no_checksum) { + return (FALSE); } #else if (mach_read_from_4(read_buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) != 0 || mach_read_from_2(read_buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - page_encrypted = true; + no_checksum= true; } #endif DBUG_EXECUTE_IF("buf_page_is_corrupt_failure", return(TRUE); ); - if (!page_encrypted && !page_size.is_compressed() + if (!no_checksum && !page_size.is_compressed() && memcmp(read_buf + FIL_PAGE_LSN + 4, read_buf + page_size.logical() - FIL_PAGE_END_LSN_OLD_CHKSUM + 4, 4)) { @@ -934,9 +943,6 @@ buf_page_is_corrupted( page_size.physical())); #endif /* UNIV_INNOCHECKSUM */ } - if (page_encrypted) { - return (FALSE); - } checksum_field1 = mach_read_from_4( read_buf + FIL_PAGE_SPACE_OR_CHKSUM); @@ -1969,7 +1975,7 @@ buf_pool_init_instance( /* Initialize the temporal memory array and slots */ buf_pool->tmp_arr = (buf_tmp_array_t *)ut_malloc_nokey(sizeof(buf_tmp_array_t)); memset(buf_pool->tmp_arr, 0, sizeof(buf_tmp_array_t)); - ulint n_slots = srv_n_read_io_threads * srv_n_write_io_threads * (8 * OS_AIO_N_PENDING_IOS_PER_THREAD); + ulint n_slots = (srv_n_read_io_threads + srv_n_write_io_threads) * (8 * OS_AIO_N_PENDING_IOS_PER_THREAD); buf_pool->tmp_arr->n_slots = n_slots; buf_pool->tmp_arr->slots = (buf_tmp_buffer_t*)ut_malloc_nokey(sizeof(buf_tmp_buffer_t) * n_slots); memset(buf_pool->tmp_arr->slots, 0, (sizeof(buf_tmp_buffer_t) * n_slots)); diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 5df46bc7a8f..34ab49bfd0d 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -21786,7 +21786,7 @@ static MYSQL_SYSVAR_ENUM(compression_algorithm, innodb_compression_algorithm, /* We use here the largest number of supported compression method to enable all those methods that are available. Availability of compression method is verified on innodb_compression_algorithm_validate function. */ - PAGE_UNCOMPRESSED, + PAGE_ZLIB_ALGORITHM, &page_compression_algorithms_typelib); static MYSQL_SYSVAR_LONG(mtflush_threads, srv_mtflush_threads, diff --git a/storage/innobase/os/os0thread.cc b/storage/innobase/os/os0thread.cc index fd88fafa69d..8039c2e4f46 100644 --- a/storage/innobase/os/os0thread.cc +++ b/storage/innobase/os/os0thread.cc @@ -279,4 +279,3 @@ os_thread_free() mutex_destroy(&thread_mutex); } - diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 00acaed1555..06feb5d25ef 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -1427,12 +1427,12 @@ innobase_start_or_create_for_mysql(void) size_t dirnamelen; unsigned i = 0; - /* Reset the start state. */ - srv_start_state = SRV_START_STATE_NONE; - high_level_read_only = srv_read_only_mode || srv_force_recovery > SRV_FORCE_NO_TRX_UNDO; + /* Reset the start state. */ + srv_start_state = SRV_START_STATE_NONE; + if (srv_read_only_mode) { ib::info() << "Started in read only mode"; diff --git a/storage/perfschema/pfs.cc b/storage/perfschema/pfs.cc index 4cd1a35fd77..b58862d4824 100644 --- a/storage/perfschema/pfs.cc +++ b/storage/perfschema/pfs.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2008, 2015, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2008, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -2562,10 +2562,7 @@ start_table_io_wait_v1(PSI_table_locker_state *state, if (! pfs_table->m_io_enabled) return NULL; - PFS_thread *pfs_thread= pfs_table->m_thread_owner; - - DBUG_ASSERT(pfs_thread == - my_pthread_getspecific_ptr(PFS_thread*, THR_PFS)); + PFS_thread *pfs_thread= my_pthread_getspecific_ptr(PFS_thread*, THR_PFS); register uint flags; ulonglong timer_start= 0; @@ -2668,7 +2665,7 @@ start_table_lock_wait_v1(PSI_table_locker_state *state, if (! pfs_table->m_lock_enabled) return NULL; - PFS_thread *pfs_thread= pfs_table->m_thread_owner; + PFS_thread *pfs_thread= my_pthread_getspecific_ptr(PFS_thread*, THR_PFS); PFS_TL_LOCK_TYPE lock_type; @@ -3070,7 +3067,12 @@ start_socket_wait_v1(PSI_socket_locker_state *state, if (flag_thread_instrumentation) { - PFS_thread *pfs_thread= pfs_socket->m_thread_owner; + /* + Do not use pfs_socket->m_thread_owner here, + as different threads may use concurrently the same socket, + for example during a KILL. + */ + PFS_thread *pfs_thread= my_pthread_getspecific_ptr(PFS_thread*, THR_PFS); if (unlikely(pfs_thread == NULL)) return NULL; @@ -3438,6 +3440,8 @@ static void end_idle_wait_v1(PSI_idle_locker* locker) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } @@ -3519,6 +3523,8 @@ static void end_mutex_wait_v1(PSI_mutex_locker* locker, int rc) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } } @@ -3598,6 +3604,8 @@ static void end_rwlock_rdwait_v1(PSI_rwlock_locker* locker, int rc) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } } @@ -3670,6 +3678,8 @@ static void end_rwlock_wrwait_v1(PSI_rwlock_locker* locker, int rc) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } } @@ -3734,6 +3744,8 @@ static void end_cond_wait_v1(PSI_cond_locker* locker, int rc) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } } @@ -3828,6 +3840,8 @@ static void end_table_io_wait_v1(PSI_table_locker* locker) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } @@ -3897,6 +3911,8 @@ static void end_table_lock_wait_v1(PSI_table_locker* locker) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } @@ -4145,6 +4161,8 @@ static void end_file_wait_v1(PSI_file_locker *locker, if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } } @@ -5074,6 +5092,8 @@ static void end_socket_wait_v1(PSI_socket_locker *locker, size_t byte_count) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } diff --git a/storage/perfschema/pfs_digest.cc b/storage/perfschema/pfs_digest.cc index 0a01ee08993..3330c29795f 100644 --- a/storage/perfschema/pfs_digest.cc +++ b/storage/perfschema/pfs_digest.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2008, 2015, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2008, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -47,7 +47,7 @@ bool flag_statements_digest= true; Current index in Stat array where new record is to be inserted. index 0 is reserved for "all else" case when entire array is full. */ -volatile uint32 digest_index; +volatile uint32 PFS_ALIGNED digest_monotonic_index; bool digest_full= false; LF_HASH digest_hash; @@ -65,7 +65,7 @@ int init_digest(const PFS_global_param *param) */ digest_max= param->m_digest_sizing; digest_lost= 0; - digest_index= 1; + PFS_atomic::store_u32(& digest_monotonic_index, 1); digest_full= false; if (digest_max == 0) @@ -107,6 +107,9 @@ int init_digest(const PFS_global_param *param) + index * pfs_max_digest_length, pfs_max_digest_length); } + /* Set record[0] as allocated. */ + statements_digest_stat_array[0].m_lock.set_allocated(); + return 0; } @@ -209,9 +212,10 @@ find_or_create_digest(PFS_thread *thread, memcpy(hash_key.m_schema_name, schema_name, schema_name_length); int res; - ulong safe_index; uint retry_count= 0; const uint retry_max= 3; + size_t safe_index; + size_t attempts= 0; PFS_statements_digest_stat **entry; PFS_statements_digest_stat *pfs= NULL; @@ -247,55 +251,70 @@ search: return & pfs->m_stat; } - safe_index= PFS_atomic::add_u32(& digest_index, 1); - if (safe_index >= digest_max) + while (++attempts <= digest_max) { - /* The digest array is now full. */ - digest_full= true; - pfs= &statements_digest_stat_array[0]; - - if (pfs->m_first_seen == 0) - pfs->m_first_seen= now; - pfs->m_last_seen= now; - return & pfs->m_stat; - } - - /* Add a new record in digest stat array. */ - pfs= &statements_digest_stat_array[safe_index]; - - /* Copy digest hash/LF Hash search key. */ - memcpy(& pfs->m_digest_key, &hash_key, sizeof(PFS_digest_key)); - - /* - Copy digest storage to statement_digest_stat_array so that it could be - used later to generate digest text. - */ - pfs->m_digest_storage.copy(digest_storage); - - pfs->m_first_seen= now; - pfs->m_last_seen= now; + safe_index= PFS_atomic::add_u32(& digest_monotonic_index, 1) % digest_max; + if (safe_index == 0) + { + /* Record [0] is reserved. */ + safe_index= 1; + } - res= lf_hash_insert(&digest_hash, pins, &pfs); - if (likely(res == 0)) - { - return & pfs->m_stat; - } + /* Add a new record in digest stat array. */ + pfs= &statements_digest_stat_array[safe_index]; - if (res > 0) - { - /* Duplicate insert by another thread */ - if (++retry_count > retry_max) + if (pfs->m_lock.is_free()) { - /* Avoid infinite loops */ - digest_lost++; - return NULL; + if (pfs->m_lock.free_to_dirty()) + { + /* Copy digest hash/LF Hash search key. */ + memcpy(& pfs->m_digest_key, &hash_key, sizeof(PFS_digest_key)); + + /* + Copy digest storage to statement_digest_stat_array so that it could be + used later to generate digest text. + */ + pfs->m_digest_storage.copy(digest_storage); + + pfs->m_first_seen= now; + pfs->m_last_seen= now; + + res= lf_hash_insert(&digest_hash, pins, &pfs); + if (likely(res == 0)) + { + pfs->m_lock.dirty_to_allocated(); + return & pfs->m_stat; + } + + pfs->m_lock.dirty_to_free(); + + if (res > 0) + { + /* Duplicate insert by another thread */ + if (++retry_count > retry_max) + { + /* Avoid infinite loops */ + digest_lost++; + return NULL; + } + goto search; + } + + /* OOM in lf_hash_insert */ + digest_lost++; + return NULL; + } } - goto search; } - /* OOM in lf_hash_insert */ - digest_lost++; - return NULL; + /* The digest array is now full. */ + digest_full= true; + pfs= &statements_digest_stat_array[0]; + + if (pfs->m_first_seen == 0) + pfs->m_first_seen= now; + pfs->m_last_seen= now; + return & pfs->m_stat; } void purge_digest(PFS_thread* thread, PFS_digest_key *hash_key) @@ -322,10 +341,12 @@ void purge_digest(PFS_thread* thread, PFS_digest_key *hash_key) void PFS_statements_digest_stat::reset_data(unsigned char *token_array, uint length) { + m_lock.set_dirty(); m_digest_storage.reset(token_array, length); m_stat.reset(); m_first_seen= 0; m_last_seen= 0; + m_lock.dirty_to_free(); } void PFS_statements_digest_stat::reset_index(PFS_thread *thread) @@ -353,11 +374,14 @@ void reset_esms_by_digest() statements_digest_stat_array[index].reset_data(statements_digest_token_array + index * pfs_max_digest_length, pfs_max_digest_length); } + /* Mark record[0] as allocated again. */ + statements_digest_stat_array[0].m_lock.set_allocated(); + /* Reset index which indicates where the next calculated digest information to be inserted in statements_digest_stat_array. */ - digest_index= 1; + PFS_atomic::store_u32(& digest_monotonic_index, 1); digest_full= false; } diff --git a/storage/perfschema/pfs_digest.h b/storage/perfschema/pfs_digest.h index 76d6c33d984..429a9f4250a 100644 --- a/storage/perfschema/pfs_digest.h +++ b/storage/perfschema/pfs_digest.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2011, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -44,6 +44,9 @@ struct PFS_digest_key /** A statement digest stat record. */ struct PFS_ALIGNED PFS_statements_digest_stat { + /** Internal lock. */ + pfs_lock m_lock; + /** Digest Schema + MD5 Hash. */ PFS_digest_key m_digest_key; diff --git a/storage/perfschema/pfs_lock.h b/storage/perfschema/pfs_lock.h index 09efecd1c5f..be84d0f7fb4 100644 --- a/storage/perfschema/pfs_lock.h +++ b/storage/perfschema/pfs_lock.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2009, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -156,6 +156,18 @@ struct pfs_lock } /** + Initialize a lock to dirty. + */ + void set_dirty(void) + { + /* Do not set the version to 0, read the previous value. */ + uint32 copy= PFS_atomic::load_u32(&m_version_state); + /* Increment the version, set the DIRTY state */ + uint32 new_val= (copy & VERSION_MASK) + VERSION_INC + PFS_LOCK_DIRTY; + PFS_atomic::store_u32(&m_version_state, new_val); + } + + /** Execute a dirty to free transition. This transition should be executed by the writer that owns the record. */ diff --git a/storage/perfschema/table_esms_by_digest.cc b/storage/perfschema/table_esms_by_digest.cc index 88860a77c5d..6c2b173d000 100644 --- a/storage/perfschema/table_esms_by_digest.cc +++ b/storage/perfschema/table_esms_by_digest.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2010, 2012, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2010, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -113,11 +113,14 @@ int table_esms_by_digest::rnd_next(void) m_pos.next()) { digest_stat= &statements_digest_stat_array[m_pos.m_index]; - if (digest_stat->m_first_seen != 0) + if (digest_stat->m_lock.is_populated()) { - make_row(digest_stat); - m_next_pos.set_after(&m_pos); - return 0; + if (digest_stat->m_first_seen != 0) + { + make_row(digest_stat); + m_next_pos.set_after(&m_pos); + return 0; + } } } @@ -135,10 +138,13 @@ table_esms_by_digest::rnd_pos(const void *pos) set_position(pos); digest_stat= &statements_digest_stat_array[m_pos.m_index]; - if (digest_stat->m_first_seen != 0) + if (digest_stat->m_lock.is_populated()) { - make_row(digest_stat); - return 0; + if (digest_stat->m_first_seen != 0) + { + make_row(digest_stat); + return 0; + } } return HA_ERR_RECORD_DELETED; diff --git a/storage/perfschema/unittest/pfs-t.cc b/storage/perfschema/unittest/pfs-t.cc index 6121fac098f..f76b1aa2e75 100644 --- a/storage/perfschema/unittest/pfs-t.cc +++ b/storage/perfschema/unittest/pfs-t.cc @@ -1316,6 +1316,7 @@ void test_locker_disabled() /* Pretend the socket does not have a thread owner */ /* ---------------------------------------------- */ + psi->delete_current_thread(); socket_class_A->m_enabled= true; socket_A1= psi->init_socket(socket_key_A, NULL, NULL, 0); ok(socket_A1 != NULL, "instrumented"); diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index 84b64622c78..6d5776dc726 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -2,7 +2,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, Google Inc. -Copyright (c) 2013, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. All Rights Reserved. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -707,8 +707,12 @@ buf_page_is_corrupted( ulint checksum_field2; ulint space_id = mach_read_from_4( read_buf + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); + ulint page_type = mach_read_from_4( + read_buf + FIL_PAGE_TYPE); + bool no_checksum = (page_type == FIL_PAGE_PAGE_COMPRESSED || + page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED); fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id); - bool page_encrypted = false; + /* Page is encrypted if encryption information is found from tablespace and page contains used key_version. This is true @@ -716,10 +720,15 @@ buf_page_is_corrupted( if (crypt_data && crypt_data->type != CRYPT_SCHEME_UNENCRYPTED && fil_page_is_encrypted(read_buf)) { - page_encrypted = true; + no_checksum = true; + } + + /* Return early if there is no checksum or END_LSN */ + if (no_checksum) { + return (FALSE); } - if (!page_encrypted && !zip_size + if (!no_checksum && !zip_size && memcmp(read_buf + FIL_PAGE_LSN + 4, read_buf + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM + 4, 4)) { @@ -778,10 +787,6 @@ buf_page_is_corrupted( return(!page_zip_verify_checksum(read_buf, zip_size)); } - if (page_encrypted) { - return (FALSE); - } - checksum_field1 = mach_read_from_4( read_buf + FIL_PAGE_SPACE_OR_CHKSUM); @@ -1572,7 +1577,7 @@ buf_pool_init_instance( /* Initialize the temporal memory array and slots */ buf_pool->tmp_arr = (buf_tmp_array_t *)mem_zalloc(sizeof(buf_tmp_array_t)); - ulint n_slots = srv_n_read_io_threads * srv_n_write_io_threads * (8 * OS_AIO_N_PENDING_IOS_PER_THREAD); + ulint n_slots = (srv_n_read_io_threads + srv_n_write_io_threads) * (8 * OS_AIO_N_PENDING_IOS_PER_THREAD); buf_pool->tmp_arr->n_slots = n_slots; buf_pool->tmp_arr->slots = (buf_tmp_buffer_t*)mem_zalloc(sizeof(buf_tmp_buffer_t) * n_slots); diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index f4005847ca9..00f13172db1 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -3870,23 +3870,16 @@ innobase_init( srv_log_block_size = 512; } - if (UNIV_PAGE_SIZE != UNIV_PAGE_SIZE_DEF) { + /* The buffer pool needs to be able to accommodate enough many + pages, even for larger pages */ + if (UNIV_PAGE_SIZE > UNIV_PAGE_SIZE_DEF + && innobase_buffer_pool_size < (24 * 1024 * 1024)) { ib_logf(IB_LOG_LEVEL_INFO, - "innodb_page_size has been " - "changed from default value %d to %ld.", - UNIV_PAGE_SIZE_DEF, UNIV_PAGE_SIZE); - - /* There is hang on buffer pool when trying to get a new - page if buffer pool size is too small for large page sizes */ - if (UNIV_PAGE_SIZE > UNIV_PAGE_SIZE_DEF - && innobase_buffer_pool_size < (24 * 1024 * 1024)) { - ib_logf(IB_LOG_LEVEL_ERROR, - "innodb_page_size=%lu requires " - "innodb_buffer_pool_size > 24M current %lld", - UNIV_PAGE_SIZE, innobase_buffer_pool_size); - - goto error; - } + "innodb_page_size= " ULINTPF " requires " + "innodb_buffer_pool_size > 24M current %lld. ", + UNIV_PAGE_SIZE, + innobase_buffer_pool_size); + goto error; } ut_ad (srv_log_block_size >= OS_MIN_LOG_BLOCK_SIZE); @@ -4209,11 +4202,11 @@ innobase_change_buffering_inited_ok: srv_page_size); goto mem_free_and_error; } + if (UNIV_PAGE_SIZE_DEF != srv_page_size) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: innodb-page-size has been changed" - " from the default value %d to %lu.\n", + ib_logf(IB_LOG_LEVEL_INFO, + " innodb-page-size has been changed" + " from the default value %d to " ULINTPF " .", UNIV_PAGE_SIZE_DEF, srv_page_size); } @@ -21262,7 +21255,7 @@ static MYSQL_SYSVAR_ENUM(compression_algorithm, innodb_compression_algorithm, /* We use here the largest number of supported compression method to enable all those methods that are available. Availability of compression method is verified on innodb_compression_algorithm_validate function. */ - PAGE_UNCOMPRESSED, + PAGE_ZLIB_ALGORITHM, &page_compression_algorithms_typelib); static MYSQL_SYSVAR_LONG(mtflush_threads, srv_mtflush_threads, diff --git a/storage/xtradb/log/log0recv.cc b/storage/xtradb/log/log0recv.cc index 1777084e746..11c643afff1 100644 --- a/storage/xtradb/log/log0recv.cc +++ b/storage/xtradb/log/log0recv.cc @@ -2,7 +2,7 @@ Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2013, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -2900,11 +2900,10 @@ recv_scan_log_recs( recv_init_crash_recovery(); } else { - - ib_logf(IB_LOG_LEVEL_WARN, - "Recovery skipped, " - "--innodb-read-only set!"); - + ib_logf(IB_LOG_LEVEL_ERROR, + "innodb_read_only prevents" + " crash recovery"); + recv_needed_recovery = TRUE; return(TRUE); } } @@ -3351,6 +3350,11 @@ recv_recovery_from_checkpoint_start_func( } /* Done with startup scan. Clear the flag. */ recv_log_scan_is_startup_type = FALSE; + + if (srv_read_only_mode && recv_needed_recovery) { + return(DB_READ_ONLY); + } + if (TYPE_CHECKPOINT) { /* NOTE: we always do a 'recovery' at startup, but only if there is something wrong we will print a message to the @@ -3505,15 +3509,6 @@ void recv_recovery_from_checkpoint_finish(void) /*======================================*/ { - /* Apply the hashed log records to the respective file pages */ - - if (srv_force_recovery < SRV_FORCE_NO_LOG_REDO) { - - recv_apply_hashed_log_recs(TRUE); - } - - DBUG_PRINT("ib_log", ("apply completed")); - if (recv_needed_recovery) { trx_sys_print_mysql_master_log_pos(); trx_sys_print_mysql_binlog_offset(); diff --git a/storage/xtradb/srv/srv0start.cc b/storage/xtradb/srv/srv0start.cc index 70aec8767ad..4dd31ad43f6 100644 --- a/storage/xtradb/srv/srv0start.cc +++ b/storage/xtradb/srv/srv0start.cc @@ -631,7 +631,7 @@ create_log_file( fprintf(stderr, "innodb_force_recovery_crash=%lu\n", \ srv_force_recovery_crash); \ fflush(stderr); \ - abort(); \ + exit(3); \ } \ } while (0) #endif @@ -1631,8 +1631,6 @@ innobase_start_or_create_for_mysql(void) lsn_t max_arch_log_no = LSN_MAX; #endif /* UNIV_LOG_ARCHIVE */ ulint sum_of_new_sizes; - ulint sum_of_data_file_sizes; - ulint tablespace_size_in_header; dberr_t err; unsigned i; ulint srv_n_log_files_found = srv_n_log_files; @@ -1645,6 +1643,19 @@ innobase_start_or_create_for_mysql(void) size_t dirnamelen; bool sys_datafiles_created = false; + /* Check that os_fast_mutexes work as expected */ + os_fast_mutex_init(PFS_NOT_INSTRUMENTED, &srv_os_test_mutex); + + ut_a(0 == os_fast_mutex_trylock(&srv_os_test_mutex)); + + os_fast_mutex_unlock(&srv_os_test_mutex); + + os_fast_mutex_lock(&srv_os_test_mutex); + + os_fast_mutex_unlock(&srv_os_test_mutex); + + os_fast_mutex_free(&srv_os_test_mutex); + /* This should be initialized early */ ut_init_timer(); @@ -1690,22 +1701,7 @@ innobase_start_or_create_for_mysql(void) #endif /* PAGE_ATOMIC_REF_COUNT */ ); - - if (sizeof(ulint) != sizeof(void*)) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Error: size of InnoDB's ulint is %lu, " - "but size of void*\n", (ulong) sizeof(ulint)); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: is %lu. The sizes should be the same " - "so that on a 64-bit\n", - (ulong) sizeof(void*)); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: platforms you can allocate more than 4 GB " - "of memory.\n"); - } + compile_time_assert(sizeof(ulint) == sizeof(void*)); /* If stacktrace is used we set up signal handler for SIGUSR2 signal here. If signal handler set fails we report that and disable @@ -2406,7 +2402,6 @@ files_checked: trx_sys_create(); if (create_new_db) { - ut_a(!srv_read_only_mode); init_log_online(); @@ -2488,25 +2483,103 @@ files_checked: min_flushed_lsn, max_flushed_lsn); if (err != DB_SUCCESS) { - - return(DB_ERROR); + return(err); } init_log_online(); - /* Since the insert buffer init is in dict_boot, and the - insert buffer is needed in any disk i/o, first we call - dict_boot(). Note that trx_sys_init_at_db_start() only needs - to access space 0, and the insert buffer at this stage already - works for space 0. */ - + /* Initialize the change buffer. */ err = dict_boot(); if (err != DB_SUCCESS) { return(err); } + /* This must precede recv_apply_hashed_log_recs(TRUE). */ ib_bh = trx_sys_init_at_db_start(); + + if (srv_force_recovery < SRV_FORCE_NO_LOG_REDO) { + /* Apply the hashed log records to the + respective file pages, for the last batch of + recv_group_scan_log_recs(). */ + + err = recv_apply_hashed_log_recs(TRUE); + DBUG_PRINT("ib_log", ("apply completed")); + + if (err != DB_SUCCESS) { + return(err); + } + } + + if (!srv_read_only_mode) { + const ulint flags = FSP_FLAGS_PAGE_SSIZE(); + for (ulint id = 0; id <= srv_undo_tablespaces; id++) { + if (fil_space_get(id)) { + fsp_flags_try_adjust(id, flags); + } + } + + if (sum_of_new_sizes > 0) { + /* New data file(s) were added */ + mtr_start(&mtr); + fsp_header_inc_size(0, sum_of_new_sizes, &mtr); + mtr_commit(&mtr); + /* Immediately write the log record about + increased tablespace size to disk, so that it + is durable even if mysqld would crash + quickly */ + log_buffer_flush_to_disk(); + } + } + + const ulint tablespace_size_in_header + = fsp_header_get_tablespace_size(); + +#ifdef UNIV_DEBUG + /* buf_debug_prints = TRUE; */ +#endif /* UNIV_DEBUG */ + ulint sum_of_data_file_sizes = 0; + + for (ulint d = 0; d < srv_n_data_files; d++) { + sum_of_data_file_sizes += srv_data_file_sizes[d]; + } + + /* Compare the system tablespace file size to what is + stored in FSP_SIZE. In open_or_create_data_files() + we already checked that the file sizes match the + innodb_data_file_path specification. */ + if (srv_read_only_mode + || sum_of_data_file_sizes == tablespace_size_in_header) { + /* Do not complain about the size. */ + } else if (!srv_auto_extend_last_data_file + || sum_of_data_file_sizes + < tablespace_size_in_header) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Tablespace size stored in header is " ULINTPF + " pages, but the sum of data file sizes is " + ULINTPF " pages", + tablespace_size_in_header, + sum_of_data_file_sizes); + + if (srv_force_recovery == 0 + && sum_of_data_file_sizes + < tablespace_size_in_header) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Cannot start InnoDB. The tail of" + " the system tablespace is" + " missing. Have you edited" + " innodb_data_file_path in my.cnf" + " in an inappropriate way, removing" + " data files from there?" + " You can set innodb_force_recovery=1" + " in my.cnf to force" + " a startup if you are trying to" + " recover a badly corrupt database."); + + return(DB_ERROR); + } + } + n_recovered_trx = UT_LIST_GET_LEN(trx_sys->rw_trx_list); /* The purge system needs to create the purge view and @@ -2659,20 +2732,8 @@ files_checked: trx_sys_file_format_tag_init(); } - if (!create_new_db && sum_of_new_sizes > 0) { - /* New data file(s) were added */ - mtr_start(&mtr); - - fsp_header_inc_size(0, sum_of_new_sizes, &mtr); - - mtr_commit(&mtr); - - /* Immediately write the log record about increased tablespace - size to disk, so that it is durable even if mysqld would crash - quickly */ - - log_buffer_flush_to_disk(); - } + ut_ad(err == DB_SUCCESS); + ut_a(sum_of_new_sizes != ULINT_UNDEFINED); #ifdef UNIV_LOG_ARCHIVE if (!srv_read_only_mode) { @@ -2737,13 +2798,6 @@ files_checked: } if (!srv_read_only_mode) { - const ulint flags = FSP_FLAGS_PAGE_SSIZE(); - for (ulint id = 0; id <= srv_undo_tablespaces; id++) { - if (fil_space_get(id)) { - fsp_flags_try_adjust(id, flags); - } - } - /* Create the thread which watches the timeouts for lock waits */ thread_handles[2 + SRV_MAX_N_IO_THREADS] = os_thread_create( @@ -2846,116 +2900,6 @@ files_checked: buf_flush_lru_manager_thread_handle = os_thread_create(buf_flush_lru_manager_thread, NULL, NULL); buf_flush_lru_manager_thread_started = true; -#ifdef UNIV_DEBUG - /* buf_debug_prints = TRUE; */ -#endif /* UNIV_DEBUG */ - sum_of_data_file_sizes = 0; - - for (i = 0; i < srv_n_data_files; i++) { - sum_of_data_file_sizes += srv_data_file_sizes[i]; - } - - tablespace_size_in_header = fsp_header_get_tablespace_size(); - - if (!srv_read_only_mode - && !srv_auto_extend_last_data_file - && sum_of_data_file_sizes != tablespace_size_in_header) { - - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Error: tablespace size" - " stored in header is %lu pages, but\n", - (ulong) tablespace_size_in_header); - ut_print_timestamp(stderr); - fprintf(stderr, - "InnoDB: the sum of data file sizes is %lu pages\n", - (ulong) sum_of_data_file_sizes); - - if (srv_force_recovery == 0 - && sum_of_data_file_sizes < tablespace_size_in_header) { - /* This is a fatal error, the tail of a tablespace is - missing */ - - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Cannot start InnoDB." - " The tail of the system tablespace is\n"); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: missing. Have you edited" - " innodb_data_file_path in my.cnf in an\n"); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: inappropriate way, removing" - " ibdata files from there?\n"); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: You can set innodb_force_recovery=1" - " in my.cnf to force\n"); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: a startup if you are trying" - " to recover a badly corrupt database.\n"); - - return(DB_ERROR); - } - } - - if (!srv_read_only_mode - && srv_auto_extend_last_data_file - && sum_of_data_file_sizes < tablespace_size_in_header) { - - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Error: tablespace size stored in header" - " is %lu pages, but\n", - (ulong) tablespace_size_in_header); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: the sum of data file sizes" - " is only %lu pages\n", - (ulong) sum_of_data_file_sizes); - - if (srv_force_recovery == 0) { - - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Cannot start InnoDB. The tail of" - " the system tablespace is\n"); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: missing. Have you edited" - " innodb_data_file_path in my.cnf in an\n"); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: inappropriate way, removing" - " ibdata files from there?\n"); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: You can set innodb_force_recovery=1" - " in my.cnf to force\n"); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: a startup if you are trying to" - " recover a badly corrupt database.\n"); - - return(DB_ERROR); - } - } - - /* Check that os_fast_mutexes work as expected */ - os_fast_mutex_init(PFS_NOT_INSTRUMENTED, &srv_os_test_mutex); - - ut_a(0 == os_fast_mutex_trylock(&srv_os_test_mutex)); - - os_fast_mutex_unlock(&srv_os_test_mutex); - - os_fast_mutex_lock(&srv_os_test_mutex); - - os_fast_mutex_unlock(&srv_os_test_mutex); - - os_fast_mutex_free(&srv_os_test_mutex); - if (!srv_file_per_table && srv_pass_corrupt_table) { fprintf(stderr, "InnoDB: Warning:" " The option innodb_file_per_table is disabled," diff --git a/support-files/CMakeLists.txt b/support-files/CMakeLists.txt index eceff5fcf18..c80de04d069 100644 --- a/support-files/CMakeLists.txt +++ b/support-files/CMakeLists.txt @@ -56,9 +56,28 @@ IF(UNIX) INSTALL(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${script} DESTINATION ${inst_location} COMPONENT Server_Scripts) ENDFOREACH() + IF(INSTALL_SUPPORTFILESDIR) INSTALL(FILES magic DESTINATION ${inst_location} COMPONENT SupportFiles) INSTALL(DIRECTORY policy DESTINATION ${inst_location} COMPONENT SupportFiles) + FIND_PROGRAM(CHECKMODULE checkmodule) + FIND_PROGRAM(SEMODULE_PACKAGE semodule_package) + MARK_AS_ADVANCED(CHECKMODULE SEMODULE_PACKAGE) + + # Build pp files in policy/selinux + IF(CHECKMODULE AND SEMODULE_PACKAGE) + FOREACH(pol mariadb) + SET(src ${CMAKE_CURRENT_SOURCE_DIR}/policy/selinux/${pol}.te) + SET(tmp ${CMAKE_CURRENT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/${pol}-pp.dir/${pol}.mod) + SET(out ${CMAKE_CURRENT_BINARY_DIR}/${pol}.pp) + ADD_CUSTOM_COMMAND(OUTPUT ${out} + COMMAND ${CHECKMODULE} -M -m ${src} -o ${tmp} + COMMAND ${SEMODULE_PACKAGE} -m ${tmp} -o ${out} + DEPENDS ${src}) + ADD_CUSTOM_TARGET(${pol}-pp ALL DEPENDS ${out}) + INSTALL(FILES ${out} DESTINATION ${inst_location}/policy/selinux COMPONENT SupportFiles) + ENDFOREACH() + ENDIF() ENDIF() CONFIGURE_FILE(mariadb.pc.in ${CMAKE_CURRENT_BINARY_DIR}/mariadb.pc @ONLY) diff --git a/support-files/mysql.server.sh b/support-files/mysql.server.sh index c934f7a47a5..9162cf7d716 100644 --- a/support-files/mysql.server.sh +++ b/support-files/mysql.server.sh @@ -156,15 +156,9 @@ parse_server_arguments() { # Get arguments from the my.cnf file, # the only group, which is read from now on is [mysqld] -if test -x ./bin/my_print_defaults -then - print_defaults="./bin/my_print_defaults" -elif test -x $bindir/my_print_defaults +if test -x $bindir/my_print_defaults then print_defaults="$bindir/my_print_defaults" -elif test -x $bindir/mysql_print_defaults -then - print_defaults="$bindir/mysql_print_defaults" else # Try to find basedir in /etc/my.cnf conf=/etc/my.cnf diff --git a/support-files/policy/selinux/mariadb.te b/support-files/policy/selinux/mariadb.te new file mode 100644 index 00000000000..1d3de52c700 --- /dev/null +++ b/support-files/policy/selinux/mariadb.te @@ -0,0 +1,9 @@ +module mariadb 1.0; + +require { + type mysqld_safe_t; + class capability { setuid setgid }; +} + +#============= mysqld_safe_t ============== +allow mysqld_safe_t self:capability { setuid setgid }; diff --git a/support-files/rpm/server-postin.sh b/support-files/rpm/server-postin.sh index 5b95b751212..9ef9bec3e0d 100644 --- a/support-files/rpm/server-postin.sh +++ b/support-files/rpm/server-postin.sh @@ -93,7 +93,12 @@ if [ -f /etc/redhat-release ] ; then echo ' make load' echo echo - fi + fi + if grep 'CentOS release 6' /etc/redhat-release >/dev/null 2>&1; then + if [ -x /usr/sbin/semodule ] ; then + /usr/sbin/semodule -i /usr/share/mysql/policy/selinux/mariadb.pp + fi + fi fi if [ -x sbin/restorecon ] ; then |