diff options
| author | Vicențiu Ciorbaru <vicentiu@mariadb.org> | 2017-01-16 03:18:14 +0200 |
|---|---|---|
| committer | Vicențiu Ciorbaru <vicentiu@mariadb.org> | 2017-01-16 03:18:14 +0200 |
| commit | 8e15768731c52b6bbb29d7bfe27bdd71c025a3a3 (patch) | |
| tree | 3148e77619dba81f92ab25d604079b46689e58b8 | |
| parent | ebb8c9fb26f86cff8c0d81bd2415f415cef952bb (diff) | |
| parent | 66744f4540c464413055a79111c34449e8381618 (diff) | |
| download | mariadb-git-8e15768731c52b6bbb29d7bfe27bdd71c025a3a3.tar.gz | |
Merge branch '10.0' into 10.1
180 files changed, 10539 insertions, 2460 deletions
diff --git a/.gitignore b/.gitignore index 279d3f59dc5..c76b08477c9 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,7 @@ extra/jemalloc/build/ extra/jemalloc/tmp/ extra/my_print_defaults extra/mysql_waitpid +extra/mysqld_safe_helper extra/perror extra/replace extra/resolve_stack_dump @@ -217,6 +218,7 @@ support-files/mysql.spec support-files/mysqld_multi.server support-files/wsrep.cnf support-files/wsrep_notify +support-files/SELinux/centos6-mariadb.pp tags tests/async_queries tests/bug25714 diff --git a/client/mysqladmin.cc b/client/mysqladmin.cc index fe9db6ea93d..9642903ae79 100644 --- a/client/mysqladmin.cc +++ b/client/mysqladmin.cc @@ -1564,8 +1564,10 @@ static my_bool get_pidfile(MYSQL *mysql, char *pidfile) if (mysql_query(mysql, "SHOW VARIABLES LIKE 'pid_file'")) { - my_printf_error(0, "query failed; error: '%s'", error_flags, - mysql_error(mysql)); + my_printf_error(mysql_errno(mysql), + "The query to get the server's pid file failed," + " error: '%s'. Continuing.", error_flags, + mysql_error(mysql)); } result = mysql_store_result(mysql); if (result) diff --git a/debian/mariadb-server-10.1.files.in b/debian/mariadb-server-10.1.files.in index 3b143c5c9cf..f3c048ad3d7 100644 --- a/debian/mariadb-server-10.1.files.in +++ b/debian/mariadb-server-10.1.files.in @@ -46,6 +46,7 @@ usr/bin/mysql_zap usr/bin/mysqlbinlog usr/bin/mysqld_multi usr/bin/mysqld_safe +usr/bin/mysqld_safe_helper usr/bin/mysqlhotcopy usr/bin/perror usr/bin/replace diff --git a/extra/CMakeLists.txt b/extra/CMakeLists.txt index 24090db6135..5b650413bf3 100644 --- a/extra/CMakeLists.txt +++ b/extra/CMakeLists.txt @@ -102,4 +102,7 @@ IF(UNIX) MYSQL_ADD_EXECUTABLE(mysql_waitpid mysql_waitpid.c COMPONENT Client) TARGET_LINK_LIBRARIES(mysql_waitpid mysys) + + MYSQL_ADD_EXECUTABLE(mysqld_safe_helper mysqld_safe_helper.c COMPONENT Server) + TARGET_LINK_LIBRARIES(mysqld_safe_helper mysys) ENDIF() diff --git a/extra/mysqld_safe_helper.c b/extra/mysqld_safe_helper.c new file mode 100644 index 00000000000..09e507c6e1c --- /dev/null +++ b/extra/mysqld_safe_helper.c @@ -0,0 +1,77 @@ +#include <my_global.h> +#include <m_string.h> +#include <my_sys.h> +#include <my_pthread.h> +#ifdef HAVE_PWD_H +#include <pwd.h> +#endif +#include <stdlib.h> +#include <stdio.h> + +void my_exit(int c) +{ + my_end(0); + exit(c); +} + +void do_usage() +{ + printf("Usage:\n" + " %s <user> log <filename>\n" + " %s <user> exec <command> <args>\n", + my_progname, my_progname); + my_exit(1); +} + +void do_log(const char *logfile) +{ + FILE *f; + uchar buf[4096]; + int size; + + if (!logfile) + do_usage(); + + f= my_fopen(logfile, O_WRONLY|O_APPEND|O_CREAT, MYF(MY_WME)); + if (!f) + my_exit(1); + + while ((size= my_fread(stdin, buf, sizeof(buf), MYF(MY_WME))) > 0) + if ((int)my_fwrite(f, buf, size, MYF(MY_WME)) != size) + my_exit(1); + + my_fclose(f, MYF(0)); + my_exit(0); +} + +void do_exec(char *args[]) +{ + if (!args[0]) + do_usage(); + + my_end(0); + execvp(args[0], args); +} + +int main(int argc, char *argv[]) +{ + struct passwd *user_info; + MY_INIT(argv[0]); + + if (argc < 3) + do_usage(argv[0]); + + user_info= my_check_user(argv[1], MYF(0)); + if (user_info ? my_set_user(argv[1], user_info, MYF(MY_WME)) + : my_errno == EINVAL) + my_exit(1); + + if (strcmp(argv[2], "log") == 0) + do_log(argv[3]); + + if (strcmp(argv[2], "exec") == 0) + do_exec(argv+3); + + my_end(0); + return 1; +} diff --git a/include/my_sys.h b/include/my_sys.h index a89480d3fcc..2b1698ca2bc 100644 --- a/include/my_sys.h +++ b/include/my_sys.h @@ -668,8 +668,12 @@ extern void *my_memmem(const void *haystack, size_t haystacklen, #ifdef _WIN32 extern int my_access(const char *path, int amode); +#define my_check_user(A,B) (NULL) +#define my_set_user(A,B,C) (0) #else #define my_access access +struct passwd *my_check_user(const char *user, myf MyFlags); +int my_set_user(const char *user, struct passwd *user_info, myf MyFlags); #endif extern int check_if_legal_filename(const char *path); diff --git a/man/mysql_secure_installation.1 b/man/mysql_secure_installation.1 index 4b24588ef26..4009e1bee2f 100644 --- a/man/mysql_secure_installation.1 +++ b/man/mysql_secure_installation.1 @@ -1,6 +1,6 @@ '\" t .\" -.TH "\FBMYSQL_SECURE_INST" "1" "14/12/2015" "MariaDB 10\&.1" "MariaDB Database System" +.TH "\FBMYSQL_SECURE_INST" "1" "3 January 2017" "MariaDB 10\&.1" "MariaDB Database System" .\" ----------------------------------------------------------------- .\" * set default formatting .\" ----------------------------------------------------------------- @@ -71,9 +71,8 @@ test database, which by default can be accessed by anonymous users\&. .RE .PP -Invoke \fBmysql_secure_installation\fR -without arguments: +can be invoked without arguments: .sp .if n \{\ .RS 4 @@ -86,10 +85,75 @@ shell> \fBmysql_secure_installation\fR .\} .PP The script will prompt you to determine which actions to perform\&. +.PP +\fBmysql_secure_installation\fR +accepts some options: +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +.\" mysql_secure_installation: basedir option +.\" basedir option: mysql_secure_installation +\fB\-\-basedir=\fR\fB\fIdir_name\fR\fR +.sp +Base directory\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +.\" mysql_secure_installation: defaults-extra-file option +.\" defaults-extra-file option: mysql_secure_installation +\fB\-\-defaults\-extra\-file=\fR\fB\fIfile_name\fR\fR +.sp +Additional option file\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +.\" mysql_secure_installation: defaults-file option +.\" defaults-file option: mysql_secure_installation +\fB\-\-defaults\-file=\fR\fB\fIfile_name\fR\fR +.sp +Option file\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} +.\" mysql_secure_installation: no-defaults option +.\" no-defaults option: mysql_secure_installation +\fB\-\-no\-defaults\fR +.sp +Don't read any defaults file\&. +.RE +.sp +Other unrecognized options will be passed on to the server\&. .SH "COPYRIGHT" .br .PP -Copyright 2007-2008 MySQL AB, 2008-2010 Sun Microsystems, Inc., 2010-2015 MariaDB Foundation +Copyright 2007-2008 MySQL AB, 2008-2010 Sun Microsystems, Inc., 2010-2017 MariaDB Foundation .PP This documentation is free software; you can redistribute it and/or modify it only under the terms of the GNU General Public License as published by the Free Software Foundation; version 2 of the License. .PP diff --git a/mysql-test/extra/rpl_tests/rpl_special_charset.inc b/mysql-test/extra/rpl_tests/rpl_special_charset.inc index 51119b319d7..641aa483d32 100644 --- a/mysql-test/extra/rpl_tests/rpl_special_charset.inc +++ b/mysql-test/extra/rpl_tests/rpl_special_charset.inc @@ -20,7 +20,7 @@ # then set default's client character set(latin1) as client's character set. ############################################################################### --source include/master-slave.inc -call mtr.add_suppression("Cannot use utf16 as character_set_client"); +call mtr.add_suppression("'utf16' can not be used as client character set"); CREATE TABLE t1(i VARCHAR(20)); INSERT INTO t1 VALUES (0xFFFF); --sync_slave_with_master diff --git a/mysql-test/r/create.result b/mysql-test/r/create.result index 370fc4ba57e..14a8788d3ef 100644 --- a/mysql-test/r/create.result +++ b/mysql-test/r/create.result @@ -1905,6 +1905,13 @@ end| create table t1 as select f1(); ERROR 42S02: Table 'test.t1' doesn't exist drop function f1; +# +# MDEV-10274 Bundling insert with create statement +# for table with unsigned Decimal primary key issues warning 1194 +# +create table t1(ID decimal(2,1) unsigned NOT NULL, PRIMARY KEY (ID))engine=memory +select 2.1 ID; +drop table t1; End of 5.5 tests create table t1; ERROR 42000: A table must have at least 1 column diff --git a/mysql-test/r/ctype_ucs.result b/mysql-test/r/ctype_ucs.result index 767e034a0e5..cedb7a4701e 100644 --- a/mysql-test/r/ctype_ucs.result +++ b/mysql-test/r/ctype_ucs.result @@ -4576,6 +4576,16 @@ SELECT CHAR_LENGTH(TRIM(BOTH 0x00 FROM _ucs2 0x0061)); CHAR_LENGTH(TRIM(BOTH 0x00 FROM _ucs2 0x0061)) 1 # +# MDEV-11685: sql_mode can't be set with non-ascii connection charset +# +SET character_set_connection=ucs2; +SET sql_mode='NO_ENGINE_SUBSTITUTION'; +SELECT @@sql_mode; +@@sql_mode +NO_ENGINE_SUBSTITUTION +SET sql_mode=DEFAULT; +SET NAMES utf8; +# # End of 5.5 tests # # diff --git a/mysql-test/r/ctype_ucs2_def.result b/mysql-test/r/ctype_ucs2_def.result index 5ca7d1689d2..9d8f5181103 100644 --- a/mysql-test/r/ctype_ucs2_def.result +++ b/mysql-test/r/ctype_ucs2_def.result @@ -1,4 +1,4 @@ -call mtr.add_suppression("Cannot use ucs2 as character_set_client"); +call mtr.add_suppression("'ucs2' can not be used as client character set"); show variables like 'collation_server'; Variable_name Value collation_server ucs2_unicode_ci diff --git a/mysql-test/r/ctype_ucs2_query_cache.result b/mysql-test/r/ctype_ucs2_query_cache.result index 9a7580324c1..eba7f2fb0fc 100644 --- a/mysql-test/r/ctype_ucs2_query_cache.result +++ b/mysql-test/r/ctype_ucs2_query_cache.result @@ -1,4 +1,4 @@ -call mtr.add_suppression("Cannot use ucs2 as character_set_client"); +call mtr.add_suppression("'ucs2' can not be used as client character set"); # # Start of 5.5 tests # diff --git a/mysql-test/r/ctype_utf16.result b/mysql-test/r/ctype_utf16.result index 3bd37255b59..c8ba1fedab8 100644 --- a/mysql-test/r/ctype_utf16.result +++ b/mysql-test/r/ctype_utf16.result @@ -1584,6 +1584,16 @@ ERROR HY000: Invalid utf16 character string: 'DE9899' DO LPAD(_utf16 0x0061 COLLATE utf16_unicode_ci, 10000, 0x0061DE989999); ERROR HY000: Invalid utf16 character string: 'DE9899' # +# MDEV-11685: sql_mode can't be set with non-ascii connection charset +# +SET character_set_connection=utf16; +SET sql_mode='NO_ENGINE_SUBSTITUTION'; +SELECT @@sql_mode; +@@sql_mode +NO_ENGINE_SUBSTITUTION +SET sql_mode=DEFAULT; +SET NAMES utf8; +# # End of 5.5 tests # # diff --git a/mysql-test/r/ctype_utf16_def.result b/mysql-test/r/ctype_utf16_def.result index 9d3d110fc99..98b6f7d913d 100644 --- a/mysql-test/r/ctype_utf16_def.result +++ b/mysql-test/r/ctype_utf16_def.result @@ -1,4 +1,4 @@ -call mtr.add_suppression("Cannot use utf16 as character_set_client"); +call mtr.add_suppression("'utf16' can not be used as client character set"); SHOW VARIABLES LIKE 'collation_server'; Variable_name Value collation_server utf16_general_ci diff --git a/mysql-test/r/ctype_utf32.result b/mysql-test/r/ctype_utf32.result index 5797a030e73..645db153a3e 100644 --- a/mysql-test/r/ctype_utf32.result +++ b/mysql-test/r/ctype_utf32.result @@ -1668,6 +1668,16 @@ c Warnings: Warning 1300 Invalid utf32 character string: '\xFF\xFF\x00\x00' # +# MDEV-11685: sql_mode can't be set with non-ascii connection charset +# +SET character_set_connection=utf32; +SET sql_mode='NO_ENGINE_SUBSTITUTION'; +SELECT @@sql_mode; +@@sql_mode +NO_ENGINE_SUBSTITUTION +SET sql_mode=DEFAULT; +SET NAMES utf8; +# # End of 5.5 tests # # diff --git a/mysql-test/r/events_slowlog.result b/mysql-test/r/events_slowlog.result new file mode 100644 index 00000000000..7de5925bc0f --- /dev/null +++ b/mysql-test/r/events_slowlog.result @@ -0,0 +1,13 @@ +set @event_scheduler_save= @@global.event_scheduler; +set @slow_query_log_save= @@global.slow_query_log; +set global event_scheduler= on; +set global slow_query_log= on; +set global long_query_time=0.2; +create table t1 (i int); +insert into t1 values (0); +create event ev on schedule at CURRENT_TIMESTAMP + INTERVAL 1 second do update t1 set i=1+sleep(0.5); +FOUND /update t1 set i=1/ in mysqld-slow.log +drop table t1; +set global event_scheduler= @event_scheduler_save; +set global slow_query_log= @slow_query_log_save; +set global long_query_time= @@session.long_query_time; diff --git a/mysql-test/r/func_time.result b/mysql-test/r/func_time.result index 9f82e246a79..76daecf82a5 100644 --- a/mysql-test/r/func_time.result +++ b/mysql-test/r/func_time.result @@ -2765,6 +2765,12 @@ id date1 date2 DATE_ADD(a.date1,INTERVAL -10 DAY) TO_DAYS(a.date1)-10 18 2010-10-13 2010-10-03 2010-10-03 734413 DROP TABLE t1; # +# MDEV-10524 Assertion `arg1_int >= 0' failed in Item_func_additive_op::result_precision() +# +SELECT 1 MOD ADDTIME( '13:58:57', '00:00:01' ) + 2; +1 MOD ADDTIME( '13:58:57', '00:00:01' ) + 2 +3 +# # Start of 10.0 tests # # diff --git a/mysql-test/r/index_merge_innodb.result b/mysql-test/r/index_merge_innodb.result index 5bf56e213ab..b3007408368 100644 --- a/mysql-test/r/index_merge_innodb.result +++ b/mysql-test/r/index_merge_innodb.result @@ -799,3 +799,32 @@ a b c 9 d d DROP TABLE t1; set optimizer_switch= @optimizer_switch_save; +# +# MDEV-10927: Crash When Using sort_union Optimization +# +set @tmp_optimizer_switch=@@optimizer_switch; +SET optimizer_switch='index_merge_sort_intersection=on'; +SET SESSION sort_buffer_size = 1024; +create table t1 ( +pk int(11) NOT NULL AUTO_INCREMENT, +col1 int(11) NOT NULL, +col2 int(11) NOT NULL, +col3 int(11) NOT NULL, +key2 int(11) NOT NULL, +col4 int(11) NOT NULL, +key1 int(11) NOT NULL, +PRIMARY KEY (pk), +KEY key1 (key1), +KEY key2 (key2) +) ENGINE=InnoDB AUTO_INCREMENT=12860259 DEFAULT CHARSET=utf8 ROW_FORMAT=COMPACT; +create table t2(a int); +insert into t2 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +create table t3(a int); +insert into t3 select A.a + B.a* 10 + C.a * 100 + D.a*1000 from t2 A, t2 B, t2 C, t2 D; +insert into t1 (key1, key2, col1,col2,col3,col4) +select a,a, a,a,a,a from t3; +SELECT sum(col1) FROM t1 FORCE INDEX (key1,key2) WHERE (key1 between 10 and 8191+10) or (key2= 5); +sum(col1) +33632261 +drop table t1,t2,t3; +set optimizer_switch=@tmp_optimizer_switch; diff --git a/mysql-test/r/information_schema_part.result b/mysql-test/r/information_schema_part.result index b34183ebdee..77959de256e 100644 --- a/mysql-test/r/information_schema_part.result +++ b/mysql-test/r/information_schema_part.result @@ -151,3 +151,11 @@ select create_options from information_schema.tables where table_schema="test"; create_options partitioned drop table t1; +# +# MDEV-11353 - Identical logical conditions +# +CREATE TABLE t1(a INT) CHECKSUM=1 SELECT 1; +SELECT CHECKSUM FROM INFORMATION_SCHEMA.PARTITIONS WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +CHECKSUM +3036305396 +DROP TABLE t1; diff --git a/mysql-test/r/join_cache.result b/mysql-test/r/join_cache.result index 90d305fb384..a5a24455d9b 100644 --- a/mysql-test/r/join_cache.result +++ b/mysql-test/r/join_cache.result @@ -5752,11 +5752,13 @@ LEFT JOIN t2 c1 ON c1.parent_id = t.id AND c1.col2 = "val" LEFT JOIN t2 c23 ON c23.parent_id = t.id AND c23.col2 = "val" LEFT JOIN t2 c24 ON c24.parent_id = t.id AND c24.col2 = "val" LEFT JOIN t2 c25 ON c25.parent_id = t.id AND c25.col2 = "val" + LEFT JOIN t2 c26 ON c26.parent_id = t.id AND c26.col2 = "val" + LEFT JOIN t2 c27 ON c27.parent_id = t.id AND c27.col2 = "val" ORDER BY col1; id col1 -select timestampdiff(second, @init_time, now()) <= 1; -timestampdiff(second, @init_time, now()) <= 1 +select timestampdiff(second, @init_time, now()) <= 5; +timestampdiff(second, @init_time, now()) <= 5 1 set join_cache_level=2; set @init_time:=now(); @@ -5788,11 +5790,13 @@ LEFT JOIN t2 c1 ON c1.parent_id = t.id AND c1.col2 = "val" LEFT JOIN t2 c23 ON c23.parent_id = t.id AND c23.col2 = "val" LEFT JOIN t2 c24 ON c24.parent_id = t.id AND c24.col2 = "val" LEFT JOIN t2 c25 ON c25.parent_id = t.id AND c25.col2 = "val" + LEFT JOIN t2 c26 ON c26.parent_id = t.id AND c26.col2 = "val" + LEFT JOIN t2 c27 ON c27.parent_id = t.id AND c27.col2 = "val" ORDER BY col1; id col1 -select timestampdiff(second, @init_time, now()) <= 1; -timestampdiff(second, @init_time, now()) <= 1 +select timestampdiff(second, @init_time, now()) <= 5; +timestampdiff(second, @init_time, now()) <= 5 1 EXPLAIN SELECT t.* @@ -5823,6 +5827,8 @@ LEFT JOIN t2 c1 ON c1.parent_id = t.id AND c1.col2 = "val" LEFT JOIN t2 c23 ON c23.parent_id = t.id AND c23.col2 = "val" LEFT JOIN t2 c24 ON c24.parent_id = t.id AND c24.col2 = "val" LEFT JOIN t2 c25 ON c25.parent_id = t.id AND c25.col2 = "val" + LEFT JOIN t2 c26 ON c26.parent_id = t.id AND c26.col2 = "val" + LEFT JOIN t2 c27 ON c27.parent_id = t.id AND c27.col2 = "val" ORDER BY col1; id select_type table type possible_keys key key_len ref rows Extra @@ -5852,6 +5858,8 @@ id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE c23 ALL NULL NULL NULL NULL 1 Using where; Using join buffer (incremental, BNL join) 1 SIMPLE c24 ALL NULL NULL NULL NULL 1 Using where; Using join buffer (incremental, BNL join) 1 SIMPLE c25 ALL NULL NULL NULL NULL 1 Using where; Using join buffer (incremental, BNL join) +1 SIMPLE c26 ALL NULL NULL NULL NULL 1 Using where; Using join buffer (incremental, BNL join) +1 SIMPLE c27 ALL NULL NULL NULL NULL 1 Using where; Using join buffer (incremental, BNL join) set join_buffer_size=default; set join_cache_level = default; DROP TABLE t1,t2; diff --git a/mysql-test/r/loaddata.result b/mysql-test/r/loaddata.result index f985af2ee50..0a038776323 100644 --- a/mysql-test/r/loaddata.result +++ b/mysql-test/r/loaddata.result @@ -531,7 +531,7 @@ DROP TABLE t1; # Bug#11765139 58069: LOAD DATA INFILE: VALGRIND REPORTS INVALID MEMORY READS AND WRITES WITH U # CREATE TABLE t1(f1 INT); -SELECT 0xE1C330 INTO OUTFILE 't1.dat'; +SELECT 0xE1BB30 INTO OUTFILE 't1.dat'; LOAD DATA INFILE 't1.dat' IGNORE INTO TABLE t1 CHARACTER SET utf8; DROP TABLE t1; # @@ -556,28 +556,21 @@ FIELDS TERMINATED BY 't' LINES TERMINATED BY ''; Got one of the listed errors SET @@sql_mode= @old_mode; DROP TABLE t1; - # -# Bug#23080148 - Backport of Bug#20683959. -# Bug#20683959 LOAD DATA INFILE IGNORES A SPECIFIC ROW SILENTLY -# UNDER DB CHARSET IS UTF8. +# MDEV-11079 Regression: LOAD DATA INFILE lost BLOB support using utf8 load files # -CREATE DATABASE d1 CHARSET latin1; -USE d1; -CREATE TABLE t1 (val TEXT); -LOAD DATA INFILE '../../std_data/bug20683959loaddata.txt' INTO TABLE t1; -SELECT COUNT(*) FROM t1; -COUNT(*) -1 -SELECT HEX(val) FROM t1; -HEX(val) -C38322525420406E696F757A656368756E3A20E98198E2889AF58081AEE7B99DE4B88AE383A3E7B99DE69690F58087B3E7B9A7EFBDA8E7B99DEFBDB3E7B99DE78999E880B3E7B8BAEFBDAAE7B9A7E89699E296A1E7B8BAE4BBA3EFBD8CE7B8BAEFBDA9E7B8B2E2889AE38184E7B99DEFBDB3E7B99DE4B88AE383A3E7B99DE69690F58087B3E7B9A7EFBDA8E7B99DEFBDB3E7B99DE5B3A8EFBD84E8ABA0EFBDA8E89C89F580948EE599AAE7B8BAEFBDAAE7B8BAE9A198EFBDA9EFBDB1E7B9A7E581B5E289A0E7B8BAEFBDBEE7B9A7E9A194EFBDA9E882B4EFBDA5EFBDB5E980A7F5808B96E28693E99EABE38287E58F99E7B8BAE58AB1E28691E7B8BAF5808B9AE7828AE98095EFBDB1E7B8BAEFBDAFE7B8B2E288ABE6A89FE89EB3E6BA98F58081ADE88EA0EFBDBAE98095E6BA98F58081AEE89D93EFBDBAE8AD9BEFBDACE980A7F5808B96E28693E7B8BAF580918EE288AAE7B8BAE4B88AEFBC9EE7B8BAE4B99DE28691E7B8BAF5808B96EFBCA0E88DB3E6A68AEFBDB9EFBDB3E981B2E5B3A8E296A1E7B8BAE7A4BCE7828AE88DB3E6A68AEFBDB0EFBDBDE7B8BAA0E7B8BAE88B93EFBDBEE5B899EFBC9E -CREATE DATABASE d2 CHARSET utf8; -USE d2; -CREATE TABLE t1 (val TEXT); -LOAD DATA INFILE '../../std_data/bug20683959loaddata.txt' INTO TABLE t1; -Warnings: -Warning 1366 Incorrect string value: '\xF5\x80\x81\xAE\xE7\xB9...' for column 'val' at row 1 -DROP TABLE d1.t1, d2.t1; -DROP DATABASE d1; -DROP DATABASE d2; +CREATE TABLE t1 (a mediumblob NOT NULL) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_general_ci; +LOAD DATA INFILE '../../std_data/loaddata/mdev-11079.txt' INTO TABLE t1 CHARSET utf8 FIELDS TERMINATED BY ';' ENCLOSED BY '"' ESCAPED BY '\\' LINES TERMINATED BY '\n'; +SELECT HEX(a) FROM t1; +HEX(a) +25AAABAC +DROP TABLE t1; +# +# MDEV-11631 LOAD DATA INFILE fails to load data with an escape character followed by a multi-byte character +# +CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET utf8); +LOAD DATA INFILE '../../std_data/loaddata/mdev-11631.txt' INTO TABLE t1 CHARACTER SET utf8; +SELECT HEX(a) FROM t1; +HEX(a) +C3A4 +DROP TABLE t1; diff --git a/mysql-test/r/log_slow.result b/mysql-test/r/log_slow.result index 6a3f48506e3..02e04cfbb2d 100644 --- a/mysql-test/r/log_slow.result +++ b/mysql-test/r/log_slow.result @@ -70,9 +70,9 @@ sleep(0.5) select count(*) FROM mysql.slow_log; count(*) 1 -truncate mysql.slow_log; set @@long_query_time=default; set global slow_query_log= @org_slow_query_log; set @@log_slow_filter=default; set @@log_slow_verbosity=default; set global log_output= default; +truncate mysql.slow_log; diff --git a/mysql-test/r/order_by.result b/mysql-test/r/order_by.result index 50ac3435596..e24d9a94891 100644 --- a/mysql-test/r/order_by.result +++ b/mysql-test/r/order_by.result @@ -2939,6 +2939,17 @@ t2 A, t2 B where A.b = B.b order by A.col2, B.col2 limit 10, 1000000; drop table t1,t2,t3; +# +# mdev-10705 : long order by list that can be skipped +# +SELECT 1 +UNION +( SELECT 2 +ORDER BY NULL, @a0 := 3, @a1 := 3, @a2 := 3, @a3 := 3, @a4 := 3, +@a5 := 3, @a6 := 3, @a7 := 3, @a8 := 3, @a9 := 3, @a10 := 3 ); +1 +1 +2 End of 5.5 tests # # MDEV-5884: EXPLAIN UPDATE ... ORDER BY LIMIT shows wrong #rows diff --git a/mysql-test/r/subselect.result b/mysql-test/r/subselect.result index f39b276b8e6..49a033626ee 100644 --- a/mysql-test/r/subselect.result +++ b/mysql-test/r/subselect.result @@ -7161,6 +7161,17 @@ NULL drop view v2; drop table t1,t2; # +# MDEV-10386 Assertion `fixed == 1' failed in virtual String* Item_func_conv_charset::val_str(String*) +# +CREATE TABLE t1 (f1 CHAR(3) CHARACTER SET utf8 NULL, f2 CHAR(3) CHARACTER SET latin1 NULL); +INSERT INTO t1 VALUES ('foo','bar'); +SELECT * FROM t1 WHERE f2 >= SOME ( SELECT f1 FROM t1 ); +f1 f2 +SELECT * FROM t1 WHERE f2 <= SOME ( SELECT f1 FROM t1 ); +f1 f2 +foo bar +DROP TABLE t1; +# # MDEV-9487: Server crashes in Time_and_counter_tracker::incr_loops # with UNION in ALL subquery # diff --git a/mysql-test/r/subselect2.result b/mysql-test/r/subselect2.result index 75e8c084026..a54080f6cb4 100644 --- a/mysql-test/r/subselect2.result +++ b/mysql-test/r/subselect2.result @@ -348,4 +348,49 @@ where t1.a = t2.a and ( t1.a = ( select min(a) from t1 ) or 0 ); a a a FRA FRA FRA drop table t1,t2,t3; +# +# MDEV-10148: Database crashes in the query to the View +# +CREATE TABLE t1 ( +key_code INT(11) NOT NULL, +value_string VARCHAR(50) NULL DEFAULT NULL, +PRIMARY KEY (key_code) +) COLLATE='utf8_general_ci' ENGINE=InnoDB ; +CREATE TABLE t2 ( +key_code INT(11) NOT NULL, +target_date DATE NULL DEFAULT NULL, +PRIMARY KEY (key_code) +) COLLATE='utf8_general_ci' ENGINE=InnoDB ; +CREATE TABLE t3 ( +now_date DATE NOT NULL, +PRIMARY KEY (now_date) +) COLLATE='utf8_general_ci' ENGINE=InnoDB ; +CREATE VIEW v1 +AS +SELECT +B.key_code, +B.target_date +FROM +t2 B INNER JOIN t3 C ON +B.target_date = C.now_date +; +SET @s = 'SELECT A.* FROM t1 A WHERE A.key_code IN (SELECT key_code FROM v1)'; +PREPARE stmt FROM @s; +EXECUTE stmt; +key_code value_string +EXECUTE stmt; +key_code value_string +DEALLOCATE PREPARE stmt; +DROP VIEW v1; +DROP TABLE t1,t2,t3; set optimizer_switch=@subselect2_test_tmp; +create table t1 (a int); +create table t2 (a int); +create table t3(a int); +insert into t1 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +insert into t2 select a from t1; +insert into t3 select a from t1; +select null in (select a from t1 where a < out3.a union select a from t2 where +(select a from t3) +1 < out3.a+1) from t3 out3; +ERROR 21000: Subquery returns more than 1 row +drop table t1, t2, t3; diff --git a/mysql-test/r/subselect4.result b/mysql-test/r/subselect4.result index 87645d187f2..8b912e1bfe9 100644 --- a/mysql-test/r/subselect4.result +++ b/mysql-test/r/subselect4.result @@ -2401,5 +2401,44 @@ SELECT x FROM t1 WHERE id > (SELECT MAX(id) - 1000 FROM t1) ORDER BY x LIMIT 1; x 0 drop table t1; +# +# MDEV-7691: Assertion `outer_context || !*from_field || *from_field == not_found_field' ... +# +set optimizer_switch=default; +CREATE TABLE t1 (a INT) ENGINE=MyISAM; +INSERT INTO t1 VALUES (4),(6); +CREATE TABLE t2 (b INT) ENGINE=MyISAM; +INSERT INTO t2 VALUES (1),(8); +PREPARE stmt FROM " +SELECT * FROM t2 +HAVING 0 IN ( + SELECT a FROM t1 + WHERE a IN ( + SELECT a FROM t1 + WHERE b = a + ) +) +"; +EXECUTE stmt; +b +EXECUTE stmt; +b +# Alternative test case, without HAVING +CREATE TABLE t3 (i INT) ENGINE=MyISAM; +INSERT INTO t3 VALUES (4),(6); +PREPARE stmt FROM " +SELECT * FROM t3 AS t10 +WHERE EXISTS ( + SELECT * FROM t3 AS t20 WHERE t10.i IN ( + SELECT i FROM t3 + ) +)"; +EXECUTE stmt; +i +6 +EXECUTE stmt; +i +6 +drop table t1, t2, t3; SET optimizer_switch= @@global.optimizer_switch; set @@tmp_table_size= @@global.tmp_table_size; diff --git a/mysql-test/r/subselect_no_exists_to_in.result b/mysql-test/r/subselect_no_exists_to_in.result index 4e83db1aef7..d968a81c638 100644 --- a/mysql-test/r/subselect_no_exists_to_in.result +++ b/mysql-test/r/subselect_no_exists_to_in.result @@ -7161,6 +7161,17 @@ NULL drop view v2; drop table t1,t2; # +# MDEV-10386 Assertion `fixed == 1' failed in virtual String* Item_func_conv_charset::val_str(String*) +# +CREATE TABLE t1 (f1 CHAR(3) CHARACTER SET utf8 NULL, f2 CHAR(3) CHARACTER SET latin1 NULL); +INSERT INTO t1 VALUES ('foo','bar'); +SELECT * FROM t1 WHERE f2 >= SOME ( SELECT f1 FROM t1 ); +f1 f2 +SELECT * FROM t1 WHERE f2 <= SOME ( SELECT f1 FROM t1 ); +f1 f2 +foo bar +DROP TABLE t1; +# # MDEV-9487: Server crashes in Time_and_counter_tracker::incr_loops # with UNION in ALL subquery # diff --git a/mysql-test/r/subselect_no_mat.result b/mysql-test/r/subselect_no_mat.result index 1938e287633..19a936e86c7 100644 --- a/mysql-test/r/subselect_no_mat.result +++ b/mysql-test/r/subselect_no_mat.result @@ -7154,6 +7154,17 @@ NULL drop view v2; drop table t1,t2; # +# MDEV-10386 Assertion `fixed == 1' failed in virtual String* Item_func_conv_charset::val_str(String*) +# +CREATE TABLE t1 (f1 CHAR(3) CHARACTER SET utf8 NULL, f2 CHAR(3) CHARACTER SET latin1 NULL); +INSERT INTO t1 VALUES ('foo','bar'); +SELECT * FROM t1 WHERE f2 >= SOME ( SELECT f1 FROM t1 ); +f1 f2 +SELECT * FROM t1 WHERE f2 <= SOME ( SELECT f1 FROM t1 ); +f1 f2 +foo bar +DROP TABLE t1; +# # MDEV-9487: Server crashes in Time_and_counter_tracker::incr_loops # with UNION in ALL subquery # diff --git a/mysql-test/r/subselect_no_opts.result b/mysql-test/r/subselect_no_opts.result index a77ac2e5605..1293e1de412 100644 --- a/mysql-test/r/subselect_no_opts.result +++ b/mysql-test/r/subselect_no_opts.result @@ -7152,6 +7152,17 @@ NULL drop view v2; drop table t1,t2; # +# MDEV-10386 Assertion `fixed == 1' failed in virtual String* Item_func_conv_charset::val_str(String*) +# +CREATE TABLE t1 (f1 CHAR(3) CHARACTER SET utf8 NULL, f2 CHAR(3) CHARACTER SET latin1 NULL); +INSERT INTO t1 VALUES ('foo','bar'); +SELECT * FROM t1 WHERE f2 >= SOME ( SELECT f1 FROM t1 ); +f1 f2 +SELECT * FROM t1 WHERE f2 <= SOME ( SELECT f1 FROM t1 ); +f1 f2 +foo bar +DROP TABLE t1; +# # MDEV-9487: Server crashes in Time_and_counter_tracker::incr_loops # with UNION in ALL subquery # diff --git a/mysql-test/r/subselect_no_scache.result b/mysql-test/r/subselect_no_scache.result index 0674fb865f7..cc3d20ef163 100644 --- a/mysql-test/r/subselect_no_scache.result +++ b/mysql-test/r/subselect_no_scache.result @@ -7167,6 +7167,17 @@ NULL drop view v2; drop table t1,t2; # +# MDEV-10386 Assertion `fixed == 1' failed in virtual String* Item_func_conv_charset::val_str(String*) +# +CREATE TABLE t1 (f1 CHAR(3) CHARACTER SET utf8 NULL, f2 CHAR(3) CHARACTER SET latin1 NULL); +INSERT INTO t1 VALUES ('foo','bar'); +SELECT * FROM t1 WHERE f2 >= SOME ( SELECT f1 FROM t1 ); +f1 f2 +SELECT * FROM t1 WHERE f2 <= SOME ( SELECT f1 FROM t1 ); +f1 f2 +foo bar +DROP TABLE t1; +# # MDEV-9487: Server crashes in Time_and_counter_tracker::incr_loops # with UNION in ALL subquery # diff --git a/mysql-test/r/subselect_no_semijoin.result b/mysql-test/r/subselect_no_semijoin.result index 87533769d47..2bbde7b397f 100644 --- a/mysql-test/r/subselect_no_semijoin.result +++ b/mysql-test/r/subselect_no_semijoin.result @@ -7152,6 +7152,17 @@ NULL drop view v2; drop table t1,t2; # +# MDEV-10386 Assertion `fixed == 1' failed in virtual String* Item_func_conv_charset::val_str(String*) +# +CREATE TABLE t1 (f1 CHAR(3) CHARACTER SET utf8 NULL, f2 CHAR(3) CHARACTER SET latin1 NULL); +INSERT INTO t1 VALUES ('foo','bar'); +SELECT * FROM t1 WHERE f2 >= SOME ( SELECT f1 FROM t1 ); +f1 f2 +SELECT * FROM t1 WHERE f2 <= SOME ( SELECT f1 FROM t1 ); +f1 f2 +foo bar +DROP TABLE t1; +# # MDEV-9487: Server crashes in Time_and_counter_tracker::incr_loops # with UNION in ALL subquery # diff --git a/mysql-test/r/union.result b/mysql-test/r/union.result index dd7c8fcc8ee..afeec1104bb 100644 --- a/mysql-test/r/union.result +++ b/mysql-test/r/union.result @@ -1233,12 +1233,9 @@ a b select * from ((select * from t1 limit 1) union (select * from t1 limit 1)) a; a b 1 a -2 b select * from ((select * from t1 limit 1) union (select * from t1 limit 1) union (select * from t1 limit 1)) a; a b 1 a -2 b -3 c select * from ((((select * from t1))) union (select * from t1) union (select * from t1)) a; a b 1 a @@ -1617,7 +1614,6 @@ NULL UNION RESULT <union1,2> ALL NULL NULL NULL NULL NULL NULL Using filesort Warnings: Note 1003 select NULL AS `a` from `test`.`t1` union select NULL AS `a` from `test`.`t1` order by `a` DROP TABLE t1; -End of 5.0 tests # # Bug#32858: Error: "Incorrect usage of UNION and INTO" does not take # subselects into account @@ -1732,6 +1728,16 @@ a 6 7 8 +(select a from t1 where false) UNION (select a from t1) limit 8; +a +10 +2 +3 +4 +5 +6 +7 +8 drop table t1; # # Bug#11765255 58201: @@ -2027,6 +2033,25 @@ bbbb dddd drop table t1; # +# MDEV-10172: UNION query returns incorrect rows outside +# conditional evaluation +# +create table t1 (d datetime not null primary key); +insert into t1(d) values ('2016-06-01'),('2016-06-02'),('2016-06-03'),('2016-06-04'); +select * from +( +select * from t1 where d between '2016-06-02' and '2016-06-05' + union +(select * from t1 where d < '2016-06-05' order by d desc limit 1) +) onlyJun2toJun4 +order by d; +d +2016-06-02 00:00:00 +2016-06-03 00:00:00 +2016-06-04 00:00:00 +drop table t1; +End of 5.0 tests +# # WL#1763 Avoid creating temporary table in UNION ALL # EXPLAIN SELECT 1 UNION ALL SELECT 1 LIMIT 1 OFFSET 1; diff --git a/mysql-test/std_data/bug20683959loaddata.txt b/mysql-test/std_data/bug20683959loaddata.txt deleted file mode 100644 index 1878cc78879..00000000000 --- a/mysql-test/std_data/bug20683959loaddata.txt +++ /dev/null @@ -1 +0,0 @@ -Ã"RT @niouzechun: é˜âˆšõ€®ç¹ä¸Šãƒ£ç¹æ–õ€‡³ç¹§ï½¨ç¹ï½³ç¹ç‰™è€³ç¸ºï½ªç¹§è–™â–¡ç¸ºä»£ï½Œç¸ºï½©ç¸²âˆšã„ç¹ï½³ç¹ä¸Šãƒ£ç¹æ–õ€‡³ç¹§ï½¨ç¹ï½³ç¹å³¨ï½„è« ï½¨èœ‰õ€”Žå™ªç¸ºï½ªç¸ºé¡˜ï½©ï½±ç¹§åµâ‰ 縺ｾ繧顔ｩ肴･ｵ逧õ€‹–↓鞫ょå™ç¸ºåŠ±â†‘縺õ€‹šç‚Šé€•ï½±ç¸ºï½¯ç¸²âˆ«æ¨Ÿèž³æº˜õ€èŽ ｺ逕溘õ€®è“ｺè›ï½¬é€§õ€‹–↓縺õ€‘Žâˆªç¸ºä¸Šï¼žç¸ºä¹â†‘縺õ€‹–ï¼ è³æ¦Šï½¹ï½³é²å³¨â–¡ç¸ºç¤¼ç‚Šè³æ¦Šï½°ï½½ç¸º 縺苓ｾ帙＞ diff --git a/mysql-test/std_data/loaddata/mdev-11079.txt b/mysql-test/std_data/loaddata/mdev-11079.txt new file mode 100644 index 00000000000..a792f984d5f --- /dev/null +++ b/mysql-test/std_data/loaddata/mdev-11079.txt @@ -0,0 +1 @@ +"%ª«¬" diff --git a/mysql-test/std_data/loaddata/mdev-11631.txt b/mysql-test/std_data/loaddata/mdev-11631.txt new file mode 100644 index 00000000000..87b824b71ae --- /dev/null +++ b/mysql-test/std_data/loaddata/mdev-11631.txt @@ -0,0 +1 @@ +\ä diff --git a/mysql-test/suite/binlog_encryption/rpl_special_charset.result b/mysql-test/suite/binlog_encryption/rpl_special_charset.result index 99b8d3a21d1..6efe81bf6cb 100644 --- a/mysql-test/suite/binlog_encryption/rpl_special_charset.result +++ b/mysql-test/suite/binlog_encryption/rpl_special_charset.result @@ -1,6 +1,6 @@ include/master-slave.inc [connection master] -call mtr.add_suppression("Cannot use utf16 as character_set_client"); +call mtr.add_suppression("'utf16' can not be used as client character set"); CREATE TABLE t1(i VARCHAR(20)); INSERT INTO t1 VALUES (0xFFFF); include/diff_tables.inc [master:t1, slave:t1] diff --git a/mysql-test/suite/engines/iuds/r/insert_decimal.result b/mysql-test/suite/engines/iuds/r/insert_decimal.result index 50fde80d81d..eab8592c4ee 100644 --- a/mysql-test/suite/engines/iuds/r/insert_decimal.result +++ b/mysql-test/suite/engines/iuds/r/insert_decimal.result @@ -48,7 +48,6 @@ Warning 1264 Out of range value for column 'c2' at row 1 Note 1265 Data truncated for column 'c3' at row 1 insert into t2 values ("0.0","0.0","0.0",7),("-0.0","-0.0","-0.0",8),("+0.0","+0.0","+0.0",9),("01.0","01.0","01.0",10),("+01.0","+01.0","+01.0",11),("-01.0","-01.0","-01.0",12); Warnings: -Warning 1264 Out of range value for column 'c2' at row 2 Warning 1264 Out of range value for column 'c2' at row 6 insert into t2 values ("-.1","-.1","-.1",13),("+.1","+.1","+.1",14),(".1",".1",".1",15); Warnings: diff --git a/mysql-test/suite/funcs_2/t/innodb_charset.test b/mysql-test/suite/funcs_2/t/innodb_charset.test index b77bacfc01c..da4dea44ad7 100644 --- a/mysql-test/suite/funcs_2/t/innodb_charset.test +++ b/mysql-test/suite/funcs_2/t/innodb_charset.test @@ -6,6 +6,7 @@ # Checking of other prerequisites is in charset_master.test # ################################################################################ +--source include/no_valgrind_without_big.inc --source include/have_innodb.inc let $engine_type= InnoDB; diff --git a/mysql-test/suite/rpl/r/rpl_row_mysqlbinlog.result b/mysql-test/suite/rpl/r/rpl_row_mysqlbinlog.result index 1489af830cc..08a2237df39 100644 --- a/mysql-test/suite/rpl/r/rpl_row_mysqlbinlog.result +++ b/mysql-test/suite/rpl/r/rpl_row_mysqlbinlog.result @@ -1,14 +1,22 @@ include/master-slave.inc [connection master] - +---Setup Section -- ---Setup Section -- set timestamp=1000000000; -DROP TABLE IF EXISTS t1,t2,t3; CREATE TABLE t1(word VARCHAR(20)); CREATE TABLE t2(id INT AUTO_INCREMENT NOT NULL PRIMARY KEY); CREATE TABLE t3(c1 INT NOT NULL PRIMARY KEY, c2 LONGBLOB, c3 TIMESTAMP, c4 TEXT, c5 FLOAT); - ----Test1 check table load -- +INSERT INTO t1 VALUES ("abirvalg"); +LOAD DATA INFILE '../../std_data/words.dat' INTO TABLE t1; +LOAD DATA INFILE '../../std_data/words.dat' INTO TABLE t1; +LOAD DATA INFILE '../../std_data/words.dat' INTO TABLE t1; +LOAD DATA INFILE '../../std_data/words.dat' INTO TABLE t1; +LOAD DATA INFILE '../../std_data/words.dat' INTO TABLE t1; +set @d1 = 'dd1'; +set @d1 = concat(@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1); +set @d1 = concat(@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1); +set @d1 = concat(@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1); +---Test 1 check table load -- SELECT COUNT(*) from t1; COUNT(*) 351 @@ -71,9 +79,7 @@ c1 c3 c4 c5 5 2006-02-22 00:00:00 Tested in Texas 11 insert into t1 values ("Alas"); flush logs; - --- Test 1 Dump binlog to file -- - --- Test 1 delete tables, clean master and slave -- DROP TABLE t1; DROP TABLE t2; @@ -84,9 +90,7 @@ reset master; reset slave; start slave; include/wait_for_slave_to_start.inc - --- Test 1 Load from Dump binlog file -- - --- Test 1 Check Load Results -- SELECT COUNT(*) from t1; COUNT(*) @@ -148,7 +152,6 @@ c1 c3 c4 c5 3 2006-02-22 00:00:00 Tested in Texas 6.6 4 2006-02-22 00:00:00 Tested in Texas 8.8 5 2006-02-22 00:00:00 Tested in Texas 11 - --- Test 2 position test -- /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/; /*!40019 SET @@session.max_insert_delayed_threads=0*/; @@ -159,7 +162,7 @@ use `test`/*!*/; SET TIMESTAMP=1000000000/*!*/; SET @@session.pseudo_thread_id=999999999/*!*/; SET @@session.foreign_key_checks=1, @@session.sql_auto_is_null=0, @@session.unique_checks=1, @@session.autocommit=1/*!*/; -SET @@session.sql_mode=0/*!*/; +SET @@session.sql_mode=1342177280/*!*/; SET @@session.auto_increment_increment=1, @@session.auto_increment_offset=1/*!*/; /*!\C latin1 *//*!*/; SET @@session.character_set_client=8,@@session.collation_connection=8,@@session.collation_server=8/*!*/; @@ -172,7 +175,6 @@ DELIMITER ; ROLLBACK /* added by mysqlbinlog */; /*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/; /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/; - --- Test 3 First Remote test -- /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/; /*!40019 SET @@session.max_insert_delayed_threads=0*/; @@ -183,15 +185,12 @@ use `test`/*!*/; SET TIMESTAMP=1000000000/*!*/; SET @@session.pseudo_thread_id=999999999/*!*/; SET @@session.foreign_key_checks=1, @@session.sql_auto_is_null=0, @@session.unique_checks=1, @@session.autocommit=1/*!*/; -SET @@session.sql_mode=0/*!*/; +SET @@session.sql_mode=1342177280/*!*/; SET @@session.auto_increment_increment=1, @@session.auto_increment_offset=1/*!*/; /*!\C latin1 *//*!*/; SET @@session.character_set_client=8,@@session.collation_connection=8,@@session.collation_server=8/*!*/; SET @@session.lc_time_names=0/*!*/; SET @@session.collation_database=DEFAULT/*!*/; -DROP TABLE IF EXISTS `t1`,`t2`,`t3` /* generated by server */ -/*!*/; -SET TIMESTAMP=1000000000/*!*/; CREATE TABLE t1(word VARCHAR(20)) /*!*/; SET TIMESTAMP=1000000000/*!*/; @@ -205,7 +204,6 @@ DELIMITER ; ROLLBACK /* added by mysqlbinlog */; /*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/; /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/; - --- Test 4 Second Remote test -- DROP TABLE t1; DROP TABLE t2; @@ -276,7 +274,6 @@ c1 c3 c4 c5 3 2006-02-22 00:00:00 Tested in Texas 6.6 4 2006-02-22 00:00:00 Tested in Texas 8.8 5 2006-02-22 00:00:00 Tested in Texas 11 - --- Test 5 LOAD DATA -- /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/; /*!40019 SET @@session.max_insert_delayed_threads=0*/; @@ -287,7 +284,6 @@ DELIMITER ; ROLLBACK /* added by mysqlbinlog */; /*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/; /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/; - --- Test 6 reading stdin -- /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/; /*!40019 SET @@session.max_insert_delayed_threads=0*/; @@ -298,15 +294,12 @@ use `test`/*!*/; SET TIMESTAMP=1000000000/*!*/; SET @@session.pseudo_thread_id=999999999/*!*/; SET @@session.foreign_key_checks=1, @@session.sql_auto_is_null=0, @@session.unique_checks=1, @@session.autocommit=1/*!*/; -SET @@session.sql_mode=0/*!*/; +SET @@session.sql_mode=1342177280/*!*/; SET @@session.auto_increment_increment=1, @@session.auto_increment_offset=1/*!*/; /*!\C latin1 *//*!*/; SET @@session.character_set_client=8,@@session.collation_connection=8,@@session.collation_server=8/*!*/; SET @@session.lc_time_names=0/*!*/; SET @@session.collation_database=DEFAULT/*!*/; -DROP TABLE IF EXISTS `t1`,`t2`,`t3` /* generated by server */ -/*!*/; -SET TIMESTAMP=1000000000/*!*/; CREATE TABLE t1(word VARCHAR(20)) /*!*/; SET TIMESTAMP=1000000000/*!*/; @@ -320,7 +313,6 @@ DELIMITER ; ROLLBACK /* added by mysqlbinlog */; /*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/; /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/; - --- Test 7 reading stdin w/position -- /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/; /*!40019 SET @@session.max_insert_delayed_threads=0*/; @@ -331,7 +323,7 @@ use `test`/*!*/; SET TIMESTAMP=1000000000/*!*/; SET @@session.pseudo_thread_id=999999999/*!*/; SET @@session.foreign_key_checks=1, @@session.sql_auto_is_null=0, @@session.unique_checks=1, @@session.autocommit=1/*!*/; -SET @@session.sql_mode=0/*!*/; +SET @@session.sql_mode=1342177280/*!*/; SET @@session.auto_increment_increment=1, @@session.auto_increment_offset=1/*!*/; /*!\C latin1 *//*!*/; SET @@session.character_set_client=8,@@session.collation_connection=8,@@session.collation_server=8/*!*/; @@ -344,7 +336,6 @@ DELIMITER ; ROLLBACK /* added by mysqlbinlog */; /*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/; /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/; - --- Test 8 switch internal charset -- stop slave; include/wait_for_slave_to_stop.inc @@ -380,14 +371,13 @@ HEX(f) select HEX(f) from t5; HEX(f) 835C - --- Test cleanup -- -DROP TABLE IF EXISTS t1; +DROP TABLE t1, t2, t3, t04, t05, t4, t5; CREATE TABLE t1 (a INT NOT NULL KEY, b INT); INSERT INTO t1 VALUES(1,1); SELECT * FROM t1; a b 1 1 FLUSH LOGS; -DROP TABLE IF EXISTS t1, t2, t3, t04, t05, t4, t5; +DROP TABLE t1; include/rpl_end.inc diff --git a/mysql-test/suite/rpl/r/rpl_special_charset.result b/mysql-test/suite/rpl/r/rpl_special_charset.result index 99b8d3a21d1..6efe81bf6cb 100644 --- a/mysql-test/suite/rpl/r/rpl_special_charset.result +++ b/mysql-test/suite/rpl/r/rpl_special_charset.result @@ -1,6 +1,6 @@ include/master-slave.inc [connection master] -call mtr.add_suppression("Cannot use utf16 as character_set_client"); +call mtr.add_suppression("'utf16' can not be used as client character set"); CREATE TABLE t1(i VARCHAR(20)); INSERT INTO t1 VALUES (0xFFFF); include/diff_tables.inc [master:t1, slave:t1] diff --git a/mysql-test/suite/rpl/t/rpl_row_mysqlbinlog.test b/mysql-test/suite/rpl/t/rpl_row_mysqlbinlog.test index a44cc18c0ed..6f17824f43e 100644 --- a/mysql-test/suite/rpl/t/rpl_row_mysqlbinlog.test +++ b/mysql-test/suite/rpl/t/rpl_row_mysqlbinlog.test @@ -4,44 +4,29 @@ # Purpose: To test changes to mysqlbinlog for row based bin logs # # We are using .opt file since we need small binlog size # ################################################################## -# Include Section -# Make sure that we have row based bin log -- source include/have_binlog_format_row.inc -# Embedded server doesn't support binlogging -- source include/not_embedded.inc -# This test requires the cp932 charset compiled in -- source include/have_cp932.inc -# Slow test, don't run during staging part --- source include/not_staging.inc - -- source include/master-slave.inc -# Setup Section -# we need this for getting fixed timestamps inside of this test +--echo ---Setup Section -- ---disable_query_log -select "---Setup Section --" as ""; -set sql_mode=""; ---enable_query_log +--echo ---Setup Section -- +# we need this for getting fixed timestamps inside of this test set timestamp=1000000000; ---disable_warnings -DROP TABLE IF EXISTS t1,t2,t3; ---enable_warnings - -connection master; CREATE TABLE t1(word VARCHAR(20)); CREATE TABLE t2(id INT AUTO_INCREMENT NOT NULL PRIMARY KEY); ---let $position= query_get_value(SHOW MASTER STATUS, Position, 1) +--let position= query_get_value(SHOW MASTER STATUS, Position, 1) CREATE TABLE t3(c1 INT NOT NULL PRIMARY KEY, c2 LONGBLOB, c3 TIMESTAMP, c4 TEXT, c5 FLOAT); ---let $stop_position=query_get_value(SHOW MASTER STATUS, Position, 1) ---let $stop_position1=`select $stop_position - 1` ---let $binlog_start_pos=query_get_value(SHOW BINLOG EVENTS LIMIT 1, End_log_pos, 1) +--let stop_position=query_get_value(SHOW MASTER STATUS, Position, 1) +--let stop_position1=`select $stop_position - 1` +--let binlog_start_pos=query_get_value(SHOW BINLOG EVENTS LIMIT 1, End_log_pos, 1) + # Test Section # Lets start by putting some data into the tables. ---disable_query_log INSERT INTO t1 VALUES ("abirvalg"); LOAD DATA INFILE '../../std_data/words.dat' INTO TABLE t1; LOAD DATA INFILE '../../std_data/words.dat' INTO TABLE t1; @@ -55,7 +40,8 @@ set @d1 = concat(@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1); set @d1 = concat(@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1); set @d1 = concat(@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1,@d1); -let $count=500; +--disable_query_log +let count=500; while ($count) { INSERT INTO t2 VALUES (NULL); @@ -64,10 +50,7 @@ while ($count) } --enable_query_log - ---disable_query_log -select "---Test1 check table load --" as ""; ---enable_query_log +--echo ---Test 1 check table load -- # Lets Check the tables on the Master SELECT COUNT(*) from t1; @@ -96,34 +79,26 @@ insert into t1 values ("Alas"); flush logs; # delimiters are for easier debugging in future ---disable_query_log -select "--- Test 1 Dump binlog to file --" as ""; ---enable_query_log +--echo --- Test 1 Dump binlog to file -- # # Prepare local temporary file to recreate what we have currently. -let $MYSQLD_DATADIR= `select @@datadir;`; +let MYSQLD_DATADIR= `select @@datadir;`; --exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 > $MYSQLTEST_VARDIR/tmp/master.sql --exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000002 >> $MYSQLTEST_VARDIR/tmp/master.sql # Now that we have our file, lets get rid of the current database. # Cleanup the master and the slave and try to recreate. ---disable_query_log -select "--- Test 1 delete tables, clean master and slave --" as ""; ---enable_query_log +--echo --- Test 1 delete tables, clean master and slave -- DROP TABLE t1; DROP TABLE t2; DROP TABLE t3; sync_slave_with_master; -#we expect STOP SLAVE to produce a warning as the slave is stopped -#(the server was started with skip-slave-start) ---disable_warnings stop slave; --source include/wait_for_slave_to_stop.inc ---enable_warnings connection master; reset master; connection slave; @@ -133,15 +108,11 @@ start slave; connection master; # We should be clean at this point, now we will run in the file from above. ---disable_query_log -select "--- Test 1 Load from Dump binlog file --" as ""; ---enable_query_log +--echo --- Test 1 Load from Dump binlog file -- --exec $MYSQL -e "source $MYSQLTEST_VARDIR/tmp/master.sql" ---disable_query_log -select "--- Test 1 Check Load Results --" as ""; ---enable_query_log +--echo --- Test 1 Check Load Results -- # Lets Check the tables on the Master SELECT COUNT(*) from t1; @@ -169,28 +140,20 @@ remove_file $MYSQLTEST_VARDIR/tmp/master.sql; # this test for start-position option # By setting this position to 416, we should only get the create of t3 ---disable_query_log -select "--- Test 2 position test --" as ""; ---enable_query_log -let $MYSQLD_DATADIR= `select @@datadir;`; +--echo --- Test 2 position test -- --exec $MYSQL_BINLOG --short-form --local-load=$MYSQLTEST_VARDIR/tmp/ --start-position=$position --stop-position=$stop_position $MYSQLD_DATADIR/master-bin.000001 # These are tests for remote binlog. # They should return the same as previous test. ---disable_query_log -select "--- Test 3 First Remote test --" as ""; ---enable_query_log +--echo --- Test 3 First Remote test -- # This is broken now --replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR --exec $MYSQL_BINLOG --short-form --local-load=$MYSQLTEST_VARDIR/tmp/ --stop-position=$stop_position --read-from-remote-server --user=root --host=127.0.0.1 --port=$MASTER_MYPORT master-bin.000001 ---disable_query_log -select "--- Test 4 Second Remote test --" as ""; ---enable_query_log +--echo --- Test 4 Second Remote test -- --exec $MYSQL_BINLOG --read-from-remote-server --user=root --host=127.0.0.1 --port=$MASTER_MYPORT master-bin.000001 > $MYSQLTEST_VARDIR/tmp/remote.sql - --exec $MYSQL_BINLOG --read-from-remote-server --user=root --host=127.0.0.1 --port=$MASTER_MYPORT master-bin.000002 >> $MYSQLTEST_VARDIR/tmp/remote.sql # Now that we have our file, lets get rid of the current database. @@ -202,13 +165,8 @@ DROP TABLE t3; sync_slave_with_master; -#we expect STOP SLAVE to produce a warning as the slave is stopped -#(the server was started with skip-slave-start) - ---disable_warnings stop slave; --source include/wait_for_slave_to_stop.inc ---enable_warnings connection master; reset master; connection slave; @@ -252,40 +210,26 @@ connection master; # transactions. /Matz # LOAD DATA ---disable_query_log -select "--- Test 5 LOAD DATA --" as ""; ---enable_query_log +--echo --- Test 5 LOAD DATA -- --replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR --exec $MYSQL_BINLOG --short-form --local-load=$MYSQLTEST_VARDIR/tmp/ --stop-position=$binlog_start_pos --read-from-remote-server --user=root --host=127.0.0.1 --port=$MASTER_MYPORT master-bin.000002 # Bug#7853 (mysqlbinlog does not accept input from stdin) ---disable_query_log -select "--- Test 6 reading stdin --" as ""; ---enable_query_log -let $MYSQLD_DATADIR= `select @@datadir;`; +--echo --- Test 6 reading stdin -- --replace_result $MYSQL_TEST_DIR MYSQL_TEST_DIR --exec $MYSQL_BINLOG --short-form --stop-position=$stop_position1 - < $MYSQLD_DATADIR/master-bin.000001 ---disable_query_log -select "--- Test 7 reading stdin w/position --" as ""; ---enable_query_log +--echo --- Test 7 reading stdin w/position -- --replace_result $MYSQL_TEST_DIR MYSQL_TEST_DIR --exec $MYSQL_BINLOG --short-form --start-position=$position --stop-position=$stop_position - < $MYSQLD_DATADIR/master-bin.000001 # Bug#16217 (mysql client did not know how not switch its internal charset) ---disable_query_log -select "--- Test 8 switch internal charset --" as ""; ---enable_query_log +--echo --- Test 8 switch internal charset -- sync_slave_with_master; -#we expect STOP SLAVE to produce a warning as the slave is stopped -#(the server was started with skip-slave-start) - ---disable_warnings stop slave; --source include/wait_for_slave_to_stop.inc ---enable_warnings connection master; reset master; connection slave; @@ -298,7 +242,6 @@ create table t4 (f text character set utf8); create table t5 (f text character set cp932); --exec $MYSQL --default-character-set=utf8 test -e "insert into t4 values(_utf8'ソ')" --exec $MYSQL --default-character-set=cp932 test -e "insert into t5 values(_cp932'ƒ\');" -let $MYSQLD_DATADIR= `select @@datadir;`; flush logs; rename table t4 to t04, t5 to t05; --exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 | $MYSQL --default-character-set=utf8 @@ -315,42 +258,30 @@ select HEX(f) from t4; select HEX(f) from t05; select HEX(f) from t5; ---disable_query_log -select "--- Test cleanup --" as ""; ---enable_query_log +--echo --- Test cleanup -- # clean up connection master; sync_slave_with_master; connection master; +DROP TABLE t1, t2, t3, t04, t05, t4, t5; # BUG#17654 also test mysqlbinlog to ensure it can read the binlog from a remote server # and ensure that the results are the same as if read from a file (the same file). ---disable_warnings -DROP TABLE IF EXISTS t1; ---enable_warnings - CREATE TABLE t1 (a INT NOT NULL KEY, b INT); - INSERT INTO t1 VALUES(1,1); - SELECT * FROM t1; - -let $MYSQLD_DATADIR= `select @@datadir;`; - FLUSH LOGS; --exec $MYSQL_BINLOG --read-from-remote-server --user=root --host=127.0.0.1 --port=$MASTER_MYPORT master-bin.000001 > $MYSQLTEST_VARDIR/tmp/remote.sql --exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 > $MYSQLTEST_VARDIR/tmp/local.sql --diff_files $MYSQLTEST_VARDIR/tmp/local.sql $MYSQLTEST_VARDIR/tmp/remote.sql - --remove_file $MYSQLTEST_VARDIR/tmp/remote.sql - --remove_file $MYSQLTEST_VARDIR/tmp/local.sql +DROP TABLE t1; -DROP TABLE IF EXISTS t1, t2, t3, t04, t05, t4, t5; sync_slave_with_master; # End of 4.1 tests diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result index d3e8977c6bb..05b1a07c328 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result @@ -1993,6 +1993,20 @@ NUMERIC_BLOCK_SIZE NULL ENUM_VALUE_LIST OFF,ON READ_ONLY NO COMMAND_LINE_ARGUMENT OPTIONAL +VARIABLE_NAME INNODB_STATS_INCLUDE_DELETE_MARKED +SESSION_VALUE NULL +GLOBAL_VALUE OFF +GLOBAL_VALUE_ORIGIN COMPILE-TIME +DEFAULT_VALUE OFF +VARIABLE_SCOPE GLOBAL +VARIABLE_TYPE BOOLEAN +VARIABLE_COMMENT Scan delete marked records for persistent stat +NUMERIC_MIN_VALUE NULL +NUMERIC_MAX_VALUE NULL +NUMERIC_BLOCK_SIZE NULL +ENUM_VALUE_LIST OFF,ON +READ_ONLY NO +COMMAND_LINE_ARGUMENT OPTIONAL VARIABLE_NAME INNODB_STATS_METHOD SESSION_VALUE NULL GLOBAL_VALUE nulls_equal @@ -2387,7 +2401,7 @@ READ_ONLY NO COMMAND_LINE_ARGUMENT OPTIONAL VARIABLE_NAME INNODB_VERSION SESSION_VALUE NULL -GLOBAL_VALUE 5.6.33 +GLOBAL_VALUE 5.6.35 GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE NULL VARIABLE_SCOPE GLOBAL diff --git a/mysql-test/suite/vcol/inc/vcol_trigger_sp.inc b/mysql-test/suite/vcol/inc/vcol_trigger_sp.inc index eb7e6ad32b9..b9a6ba28a5f 100644 --- a/mysql-test/suite/vcol/inc/vcol_trigger_sp.inc +++ b/mysql-test/suite/vcol/inc/vcol_trigger_sp.inc @@ -110,7 +110,7 @@ drop table t1,t2; drop procedure p1; --echo # ---echo # Bug mdev-3845: values of virtual columns are not computed for triggers +--echo # MDEV-3845 values of virtual columns are not computed for triggers --echo # CREATE TABLE t1 ( @@ -149,3 +149,10 @@ DROP TRIGGER t1_ins_aft; DROP TRIGGER t1_del_bef; DROP TABLE t1,t2; +# +# MDEV-11706 Assertion `is_stat_field || !table || (!table->write_set || bitmap_is_set(table->write_set, field_index) || (table->vcol_set && bitmap_is_set(table->vcol_set, field_index)))' failed in Field_time::store_TIME_with_warning +# +create table t1 (i int, t time not null, vt time(4) as (t) virtual); +create trigger trg before update on t1 for each row set @a = 1; +insert ignore into t1 (i) values (1); +drop table t1; diff --git a/mysql-test/suite/vcol/r/vcol_misc.result b/mysql-test/suite/vcol/r/vcol_misc.result index 451898aa972..659bc00c2b7 100644 --- a/mysql-test/suite/vcol/r/vcol_misc.result +++ b/mysql-test/suite/vcol/r/vcol_misc.result @@ -330,6 +330,13 @@ t1 CREATE TABLE `t1` ( `c1` varchar(50) COLLATE latin1_general_ci DEFAULT NULL ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_general_ci drop table t1; +set sql_mode='no_zero_date'; +create table t1 ( +ts timestamp not null default current_timestamp, +tsv timestamp as (adddate(ts, interval 1 day)) virtual +); +drop table t1; +set sql_mode=default; # # Start of 10.1 tests # diff --git a/mysql-test/suite/vcol/r/vcol_trigger_sp_innodb.result b/mysql-test/suite/vcol/r/vcol_trigger_sp_innodb.result index 1d78bbf50e4..034a67a9bb0 100644 --- a/mysql-test/suite/vcol/r/vcol_trigger_sp_innodb.result +++ b/mysql-test/suite/vcol/r/vcol_trigger_sp_innodb.result @@ -86,7 +86,7 @@ a b c drop table t1,t2; drop procedure p1; # -# Bug mdev-3845: values of virtual columns are not computed for triggers +# MDEV-3845 values of virtual columns are not computed for triggers # CREATE TABLE t1 ( a INTEGER UNSIGNED NULL DEFAULT NULL, @@ -125,3 +125,9 @@ c DROP TRIGGER t1_ins_aft; DROP TRIGGER t1_del_bef; DROP TABLE t1,t2; +create table t1 (i int, t time not null, vt time(4) as (t) virtual); +create trigger trg before update on t1 for each row set @a = 1; +insert ignore into t1 (i) values (1); +Warnings: +Warning 1364 Field 't' doesn't have a default value +drop table t1; diff --git a/mysql-test/suite/vcol/r/vcol_trigger_sp_myisam.result b/mysql-test/suite/vcol/r/vcol_trigger_sp_myisam.result index 77efa8fe6b9..07d011ac64c 100644 --- a/mysql-test/suite/vcol/r/vcol_trigger_sp_myisam.result +++ b/mysql-test/suite/vcol/r/vcol_trigger_sp_myisam.result @@ -86,7 +86,7 @@ a b c drop table t1,t2; drop procedure p1; # -# Bug mdev-3845: values of virtual columns are not computed for triggers +# MDEV-3845 values of virtual columns are not computed for triggers # CREATE TABLE t1 ( a INTEGER UNSIGNED NULL DEFAULT NULL, @@ -125,3 +125,9 @@ c DROP TRIGGER t1_ins_aft; DROP TRIGGER t1_del_bef; DROP TABLE t1,t2; +create table t1 (i int, t time not null, vt time(4) as (t) virtual); +create trigger trg before update on t1 for each row set @a = 1; +insert ignore into t1 (i) values (1); +Warnings: +Warning 1364 Field 't' doesn't have a default value +drop table t1; diff --git a/mysql-test/suite/vcol/t/vcol_misc.test b/mysql-test/suite/vcol/t/vcol_misc.test index c78093bce0f..d9c1f64991c 100644 --- a/mysql-test/suite/vcol/t/vcol_misc.test +++ b/mysql-test/suite/vcol/t/vcol_misc.test @@ -291,6 +291,17 @@ create table t1 (v1 varchar(255) as (c1) persistent, c1 varchar(50)) collate=lat show create table t1; drop table t1; +# +# MDEV-11527 Virtual columns do not get along well with NO_ZERO_DATE +# +set sql_mode='no_zero_date'; +create table t1 ( + ts timestamp not null default current_timestamp, + tsv timestamp as (adddate(ts, interval 1 day)) virtual +); +drop table t1; +set sql_mode=default; + --echo # --echo # Start of 10.1 tests diff --git a/mysql-test/t/create.test b/mysql-test/t/create.test index 0d260085129..7306a819ad7 100644 --- a/mysql-test/t/create.test +++ b/mysql-test/t/create.test @@ -1756,6 +1756,15 @@ DELIMITER ;| create table t1 as select f1(); drop function f1; +--echo # +--echo # MDEV-10274 Bundling insert with create statement +--echo # for table with unsigned Decimal primary key issues warning 1194 +--echo # + +create table t1(ID decimal(2,1) unsigned NOT NULL, PRIMARY KEY (ID))engine=memory + select 2.1 ID; +drop table t1; + --echo End of 5.5 tests # diff --git a/mysql-test/t/ctype_ucs.test b/mysql-test/t/ctype_ucs.test index 2f480621c56..c4794f143af 100644 --- a/mysql-test/t/ctype_ucs.test +++ b/mysql-test/t/ctype_ucs.test @@ -816,6 +816,14 @@ SELECT CHAR_LENGTH(TRIM(BOTH 0x0001 FROM _ucs2 0x0061)); SELECT CHAR_LENGTH(TRIM(BOTH 0x61 FROM _ucs2 0x0061)); SELECT CHAR_LENGTH(TRIM(BOTH 0x00 FROM _ucs2 0x0061)); +--echo # +--echo # MDEV-11685: sql_mode can't be set with non-ascii connection charset +--echo # +SET character_set_connection=ucs2; +SET sql_mode='NO_ENGINE_SUBSTITUTION'; +SELECT @@sql_mode; +SET sql_mode=DEFAULT; +SET NAMES utf8; --echo # --echo # End of 5.5 tests diff --git a/mysql-test/t/ctype_ucs2_def.test b/mysql-test/t/ctype_ucs2_def.test index be8e044f2e4..e297fa5ccf1 100644 --- a/mysql-test/t/ctype_ucs2_def.test +++ b/mysql-test/t/ctype_ucs2_def.test @@ -1,6 +1,6 @@ -- source include/have_ucs2.inc -call mtr.add_suppression("Cannot use ucs2 as character_set_client"); +call mtr.add_suppression("'ucs2' can not be used as client character set"); # # MySQL Bug#15276: MySQL ignores collation-server diff --git a/mysql-test/t/ctype_ucs2_query_cache.test b/mysql-test/t/ctype_ucs2_query_cache.test index acb39419751..ace826aec44 100644 --- a/mysql-test/t/ctype_ucs2_query_cache.test +++ b/mysql-test/t/ctype_ucs2_query_cache.test @@ -1,7 +1,7 @@ -- source include/have_query_cache.inc -- source include/have_ucs2.inc -call mtr.add_suppression("Cannot use ucs2 as character_set_client"); +call mtr.add_suppression("'ucs2' can not be used as client character set"); --echo # --echo # Start of 5.5 tests diff --git a/mysql-test/t/ctype_utf16.test b/mysql-test/t/ctype_utf16.test index bb7eb8cd959..4b95257cbde 100644 --- a/mysql-test/t/ctype_utf16.test +++ b/mysql-test/t/ctype_utf16.test @@ -796,6 +796,15 @@ DO RPAD(_utf16 0x0061 COLLATE utf16_unicode_ci, 10000, 0x0061DE989999); DO LPAD(_utf16 0x0061 COLLATE utf16_unicode_ci, 10000, 0x0061DE989999); --echo # +--echo # MDEV-11685: sql_mode can't be set with non-ascii connection charset +--echo # +SET character_set_connection=utf16; +SET sql_mode='NO_ENGINE_SUBSTITUTION'; +SELECT @@sql_mode; +SET sql_mode=DEFAULT; +SET NAMES utf8; + +--echo # --echo # End of 5.5 tests --echo # diff --git a/mysql-test/t/ctype_utf16_def.test b/mysql-test/t/ctype_utf16_def.test index fad61b057c3..0829cd53285 100644 --- a/mysql-test/t/ctype_utf16_def.test +++ b/mysql-test/t/ctype_utf16_def.test @@ -1,5 +1,5 @@ --source include/have_utf16.inc -call mtr.add_suppression("Cannot use utf16 as character_set_client"); +call mtr.add_suppression("'utf16' can not be used as client character set"); # # Bug #32391 Character sets: crash with --character-set-server diff --git a/mysql-test/t/ctype_utf32.test b/mysql-test/t/ctype_utf32.test index 190b9f3fb50..96e9ecdd805 100644 --- a/mysql-test/t/ctype_utf32.test +++ b/mysql-test/t/ctype_utf32.test @@ -895,6 +895,15 @@ SELECT CHAR_LENGTH(TRIM(BOTH 0x00 FROM _utf32 0x00000061)); select hex(lower(cast(0xffff0000 as char character set utf32))) as c; --echo # +--echo # MDEV-11685: sql_mode can't be set with non-ascii connection charset +--echo # +SET character_set_connection=utf32; +SET sql_mode='NO_ENGINE_SUBSTITUTION'; +SELECT @@sql_mode; +SET sql_mode=DEFAULT; +SET NAMES utf8; + +--echo # --echo # End of 5.5 tests --echo # diff --git a/mysql-test/t/events_slowlog.test b/mysql-test/t/events_slowlog.test new file mode 100644 index 00000000000..9679714dba3 --- /dev/null +++ b/mysql-test/t/events_slowlog.test @@ -0,0 +1,28 @@ +--source include/not_embedded.inc +# +# MDEV-11552 Queries executed by event scheduler are written to slow log incorrectly or not written at all +# +set @event_scheduler_save= @@global.event_scheduler; +set @slow_query_log_save= @@global.slow_query_log; + +set global event_scheduler= on; +set global slow_query_log= on; +set global long_query_time=0.2; + +create table t1 (i int); +insert into t1 values (0); +create event ev on schedule at CURRENT_TIMESTAMP + INTERVAL 1 second do update t1 set i=1+sleep(0.5); + +--let wait_condition= select i from t1 where i > 0 +--source include/wait_condition.inc + +--let SEARCH_FILE = `SELECT @@slow_query_log_file` +--let SEARCH_PATTERN= update t1 set i=1 +--let SEARCH_RANGE= -1000 +--source include/search_pattern_in_file.inc + +drop table t1; + +set global event_scheduler= @event_scheduler_save; +set global slow_query_log= @slow_query_log_save; +set global long_query_time= @@session.long_query_time; diff --git a/mysql-test/t/func_time.test b/mysql-test/t/func_time.test index 40a6c387448..5cfb7f7d05f 100644 --- a/mysql-test/t/func_time.test +++ b/mysql-test/t/func_time.test @@ -1672,6 +1672,11 @@ INSERT INTO t1 VALUES (18, '2010-10-13'); SELECT a.id,a.date1,FROM_DAYS(TO_DAYS(a.date1)-10) as date2, DATE_ADD(a.date1,INTERVAL -10 DAY),TO_DAYS(a.date1)-10 FROM t1 a ORDER BY a.id; DROP TABLE t1; +--echo # +--echo # MDEV-10524 Assertion `arg1_int >= 0' failed in Item_func_additive_op::result_precision() +--echo # +SELECT 1 MOD ADDTIME( '13:58:57', '00:00:01' ) + 2; + --echo # --echo # Start of 10.0 tests diff --git a/mysql-test/t/index_merge_innodb.test b/mysql-test/t/index_merge_innodb.test index 6a1cb53dc40..fb56e44b5ae 100644 --- a/mysql-test/t/index_merge_innodb.test +++ b/mysql-test/t/index_merge_innodb.test @@ -171,6 +171,37 @@ WHERE ( tb.b != ta.b OR tb.a = ta.a ) AND ( tb.b = ta.c OR tb.b = ta.b ); DROP TABLE t1; - set optimizer_switch= @optimizer_switch_save; +--echo # +--echo # MDEV-10927: Crash When Using sort_union Optimization +--echo # + +set @tmp_optimizer_switch=@@optimizer_switch; +SET optimizer_switch='index_merge_sort_intersection=on'; +SET SESSION sort_buffer_size = 1024; + +create table t1 ( +pk int(11) NOT NULL AUTO_INCREMENT, +col1 int(11) NOT NULL, +col2 int(11) NOT NULL, +col3 int(11) NOT NULL, +key2 int(11) NOT NULL, +col4 int(11) NOT NULL, +key1 int(11) NOT NULL, +PRIMARY KEY (pk), +KEY key1 (key1), +KEY key2 (key2) +) ENGINE=InnoDB AUTO_INCREMENT=12860259 DEFAULT CHARSET=utf8 ROW_FORMAT=COMPACT; + +create table t2(a int); +insert into t2 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); + +create table t3(a int); +insert into t3 select A.a + B.a* 10 + C.a * 100 + D.a*1000 from t2 A, t2 B, t2 C, t2 D; + +insert into t1 (key1, key2, col1,col2,col3,col4) +select a,a, a,a,a,a from t3; +SELECT sum(col1) FROM t1 FORCE INDEX (key1,key2) WHERE (key1 between 10 and 8191+10) or (key2= 5); +drop table t1,t2,t3; +set optimizer_switch=@tmp_optimizer_switch; diff --git a/mysql-test/t/information_schema_part.test b/mysql-test/t/information_schema_part.test index f1415d12f79..ea88f364c07 100644 --- a/mysql-test/t/information_schema_part.test +++ b/mysql-test/t/information_schema_part.test @@ -131,3 +131,10 @@ drop table if exists t1; create table t1 (f1 int key) partition by key(f1) partitions 2; select create_options from information_schema.tables where table_schema="test"; drop table t1; + +--echo # +--echo # MDEV-11353 - Identical logical conditions +--echo # +CREATE TABLE t1(a INT) CHECKSUM=1 SELECT 1; +SELECT CHECKSUM FROM INFORMATION_SCHEMA.PARTITIONS WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1'; +DROP TABLE t1; diff --git a/mysql-test/t/join_cache.test b/mysql-test/t/join_cache.test index 019d8edde2f..5ee868b4177 100644 --- a/mysql-test/t/join_cache.test +++ b/mysql-test/t/join_cache.test @@ -3742,9 +3742,11 @@ FROM LEFT JOIN t2 c23 ON c23.parent_id = t.id AND c23.col2 = "val" LEFT JOIN t2 c24 ON c24.parent_id = t.id AND c24.col2 = "val" LEFT JOIN t2 c25 ON c25.parent_id = t.id AND c25.col2 = "val" + LEFT JOIN t2 c26 ON c26.parent_id = t.id AND c26.col2 = "val" + LEFT JOIN t2 c27 ON c27.parent_id = t.id AND c27.col2 = "val" ORDER BY col1; -select timestampdiff(second, @init_time, now()) <= 1; +select timestampdiff(second, @init_time, now()) <= 5; set join_cache_level=2; @@ -3777,9 +3779,11 @@ FROM LEFT JOIN t2 c23 ON c23.parent_id = t.id AND c23.col2 = "val" LEFT JOIN t2 c24 ON c24.parent_id = t.id AND c24.col2 = "val" LEFT JOIN t2 c25 ON c25.parent_id = t.id AND c25.col2 = "val" + LEFT JOIN t2 c26 ON c26.parent_id = t.id AND c26.col2 = "val" + LEFT JOIN t2 c27 ON c27.parent_id = t.id AND c27.col2 = "val" ORDER BY col1; -select timestampdiff(second, @init_time, now()) <= 1; +select timestampdiff(second, @init_time, now()) <= 5; EXPLAIN SELECT t.* @@ -3810,6 +3814,8 @@ FROM LEFT JOIN t2 c23 ON c23.parent_id = t.id AND c23.col2 = "val" LEFT JOIN t2 c24 ON c24.parent_id = t.id AND c24.col2 = "val" LEFT JOIN t2 c25 ON c25.parent_id = t.id AND c25.col2 = "val" + LEFT JOIN t2 c26 ON c26.parent_id = t.id AND c26.col2 = "val" + LEFT JOIN t2 c27 ON c27.parent_id = t.id AND c27.col2 = "val" ORDER BY col1; diff --git a/mysql-test/t/loaddata.test b/mysql-test/t/loaddata.test index 9f2aafc8efd..1bc7eb139b9 100644 --- a/mysql-test/t/loaddata.test +++ b/mysql-test/t/loaddata.test @@ -612,7 +612,7 @@ disconnect con1; --echo # CREATE TABLE t1(f1 INT); -EVAL SELECT 0xE1C330 INTO OUTFILE 't1.dat'; +EVAL SELECT 0xE1BB30 INTO OUTFILE 't1.dat'; --disable_warnings LOAD DATA INFILE 't1.dat' IGNORE INTO TABLE t1 CHARACTER SET utf8; --enable_warnings @@ -658,25 +658,21 @@ SET @@sql_mode= @old_mode; --remove_file $MYSQLTEST_VARDIR/mysql DROP TABLE t1; ---echo + +--echo # +--echo # MDEV-11079 Regression: LOAD DATA INFILE lost BLOB support using utf8 load files +--echo # + +CREATE TABLE t1 (a mediumblob NOT NULL) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_general_ci; +LOAD DATA INFILE '../../std_data/loaddata/mdev-11079.txt' INTO TABLE t1 CHARSET utf8 FIELDS TERMINATED BY ';' ENCLOSED BY '"' ESCAPED BY '\\' LINES TERMINATED BY '\n'; +SELECT HEX(a) FROM t1; +DROP TABLE t1; + --echo # ---echo # Bug#23080148 - Backport of Bug#20683959. ---echo # Bug#20683959 LOAD DATA INFILE IGNORES A SPECIFIC ROW SILENTLY ---echo # UNDER DB CHARSET IS UTF8. +--echo # MDEV-11631 LOAD DATA INFILE fails to load data with an escape character followed by a multi-byte character --echo # -CREATE DATABASE d1 CHARSET latin1; -USE d1; -CREATE TABLE t1 (val TEXT); -LOAD DATA INFILE '../../std_data/bug20683959loaddata.txt' INTO TABLE t1; -SELECT COUNT(*) FROM t1; -SELECT HEX(val) FROM t1; - -CREATE DATABASE d2 CHARSET utf8; -USE d2; -CREATE TABLE t1 (val TEXT); -LOAD DATA INFILE '../../std_data/bug20683959loaddata.txt' INTO TABLE t1; - -DROP TABLE d1.t1, d2.t1; -DROP DATABASE d1; -DROP DATABASE d2; +CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET utf8); +LOAD DATA INFILE '../../std_data/loaddata/mdev-11631.txt' INTO TABLE t1 CHARACTER SET utf8; +SELECT HEX(a) FROM t1; +DROP TABLE t1; diff --git a/mysql-test/t/log_slow.test b/mysql-test/t/log_slow.test index 8d5a09d7a94..56e35bd5a20 100644 --- a/mysql-test/t/log_slow.test +++ b/mysql-test/t/log_slow.test @@ -50,7 +50,6 @@ set global slow_query_log=1; set global log_output='TABLE'; select sleep(0.5); select count(*) FROM mysql.slow_log; -truncate mysql.slow_log; # Reset used variables set @@long_query_time=default; @@ -58,3 +57,4 @@ set global slow_query_log= @org_slow_query_log; set @@log_slow_filter=default; set @@log_slow_verbosity=default; set global log_output= default; +truncate mysql.slow_log; diff --git a/mysql-test/t/order_by.test b/mysql-test/t/order_by.test index 13c8db5a481..2d0c134a2d9 100644 --- a/mysql-test/t/order_by.test +++ b/mysql-test/t/order_by.test @@ -1945,6 +1945,16 @@ order by A.col2, B.col2 limit 10, 1000000; drop table t1,t2,t3; +--echo # +--echo # mdev-10705 : long order by list that can be skipped +--echo # + +SELECT 1 +UNION +( SELECT 2 + ORDER BY NULL, @a0 := 3, @a1 := 3, @a2 := 3, @a3 := 3, @a4 := 3, + @a5 := 3, @a6 := 3, @a7 := 3, @a8 := 3, @a9 := 3, @a10 := 3 ); + --echo End of 5.5 tests --echo # diff --git a/mysql-test/t/subselect.test b/mysql-test/t/subselect.test index 38aafe39775..cfb5042299a 100644 --- a/mysql-test/t/subselect.test +++ b/mysql-test/t/subselect.test @@ -6015,6 +6015,17 @@ drop view v2; drop table t1,t2; --echo # +--echo # MDEV-10386 Assertion `fixed == 1' failed in virtual String* Item_func_conv_charset::val_str(String*) +--echo # + +CREATE TABLE t1 (f1 CHAR(3) CHARACTER SET utf8 NULL, f2 CHAR(3) CHARACTER SET latin1 NULL); +INSERT INTO t1 VALUES ('foo','bar'); +SELECT * FROM t1 WHERE f2 >= SOME ( SELECT f1 FROM t1 ); +SELECT * FROM t1 WHERE f2 <= SOME ( SELECT f1 FROM t1 ); +DROP TABLE t1; + + +--echo # --echo # MDEV-9487: Server crashes in Time_and_counter_tracker::incr_loops --echo # with UNION in ALL subquery --echo # diff --git a/mysql-test/t/subselect2.test b/mysql-test/t/subselect2.test index b3c1322184d..ae210b865a2 100644 --- a/mysql-test/t/subselect2.test +++ b/mysql-test/t/subselect2.test @@ -359,5 +359,55 @@ where t1.a = t2.a and ( t1.a = ( select min(a) from t1 ) or 0 ); drop table t1,t2,t3; +--echo # +--echo # MDEV-10148: Database crashes in the query to the View +--echo # +CREATE TABLE t1 ( + key_code INT(11) NOT NULL, + value_string VARCHAR(50) NULL DEFAULT NULL, + PRIMARY KEY (key_code) +) COLLATE='utf8_general_ci' ENGINE=InnoDB ; + +CREATE TABLE t2 ( + key_code INT(11) NOT NULL, + target_date DATE NULL DEFAULT NULL, + PRIMARY KEY (key_code) +) COLLATE='utf8_general_ci' ENGINE=InnoDB ; + +CREATE TABLE t3 ( + now_date DATE NOT NULL, + PRIMARY KEY (now_date) +) COLLATE='utf8_general_ci' ENGINE=InnoDB ; + +CREATE VIEW v1 +AS +SELECT + B.key_code, + B.target_date +FROM + t2 B INNER JOIN t3 C ON + B.target_date = C.now_date +; +SET @s = 'SELECT A.* FROM t1 A WHERE A.key_code IN (SELECT key_code FROM v1)'; +PREPARE stmt FROM @s; +EXECUTE stmt; #1st time -> success +EXECUTE stmt; #2nd time -> crash +DEALLOCATE PREPARE stmt; +DROP VIEW v1; +DROP TABLE t1,t2,t3; + set optimizer_switch=@subselect2_test_tmp; +# +# Bug #23303485 : HANDLE_FATAL_SIGNAL (SIG=11) IN SUBSELECT_UNION_ENGINE::NO_ROWS +# +create table t1 (a int); +create table t2 (a int); +create table t3(a int); +insert into t1 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +insert into t2 select a from t1; +insert into t3 select a from t1; +--error ER_SUBQUERY_NO_1_ROW +select null in (select a from t1 where a < out3.a union select a from t2 where + (select a from t3) +1 < out3.a+1) from t3 out3; +drop table t1, t2, t3; diff --git a/mysql-test/t/subselect4.test b/mysql-test/t/subselect4.test index 4eb9701ee71..cb102f8319e 100644 --- a/mysql-test/t/subselect4.test +++ b/mysql-test/t/subselect4.test @@ -1956,5 +1956,46 @@ SELECT x FROM t1 WHERE id > (SELECT MAX(id) - 1000 FROM t1) ORDER BY x LIMIT 1; drop table t1; +--echo # +--echo # MDEV-7691: Assertion `outer_context || !*from_field || *from_field == not_found_field' ... +--echo # +set optimizer_switch=default; +CREATE TABLE t1 (a INT) ENGINE=MyISAM; +INSERT INTO t1 VALUES (4),(6); + +CREATE TABLE t2 (b INT) ENGINE=MyISAM; +INSERT INTO t2 VALUES (1),(8); + +PREPARE stmt FROM " +SELECT * FROM t2 +HAVING 0 IN ( + SELECT a FROM t1 + WHERE a IN ( + SELECT a FROM t1 + WHERE b = a + ) +) +"; + +EXECUTE stmt; +EXECUTE stmt; + +--echo # Alternative test case, without HAVING +CREATE TABLE t3 (i INT) ENGINE=MyISAM; +INSERT INTO t3 VALUES (4),(6); + +PREPARE stmt FROM " +SELECT * FROM t3 AS t10 +WHERE EXISTS ( + SELECT * FROM t3 AS t20 WHERE t10.i IN ( + SELECT i FROM t3 + ) +)"; + +EXECUTE stmt; +EXECUTE stmt; + +drop table t1, t2, t3; + SET optimizer_switch= @@global.optimizer_switch; set @@tmp_table_size= @@global.tmp_table_size; diff --git a/mysql-test/t/union.test b/mysql-test/t/union.test index f2026a67bc2..0580ee305c3 100644 --- a/mysql-test/t/union.test +++ b/mysql-test/t/union.test @@ -1057,7 +1057,6 @@ ORDER BY a; DROP TABLE t1; ---echo End of 5.0 tests -- echo # -- echo # Bug#32858: Error: "Incorrect usage of UNION and INTO" does not take -- echo # subselects into account @@ -1160,6 +1159,8 @@ create table t1 (a int); insert into t1 values (10),(10),(10),(2),(3),(4),(5),(6),(7),(8),(9),(1),(10); --sorted_result select a from t1 where false UNION select a from t1 limit 8; +--sorted_result +(select a from t1 where false) UNION (select a from t1) limit 8; drop table t1; --echo # @@ -1385,6 +1386,26 @@ UNION drop table t1; + +--echo # +--echo # MDEV-10172: UNION query returns incorrect rows outside +--echo # conditional evaluation +--echo # + +create table t1 (d datetime not null primary key); +insert into t1(d) values ('2016-06-01'),('2016-06-02'),('2016-06-03'),('2016-06-04'); +select * from +( + select * from t1 where d between '2016-06-02' and '2016-06-05' + union + (select * from t1 where d < '2016-06-05' order by d desc limit 1) +) onlyJun2toJun4 +order by d; +drop table t1; + +--echo End of 5.0 tests + + --echo # --echo # WL#1763 Avoid creating temporary table in UNION ALL --echo # @@ -1486,4 +1507,3 @@ SELECT * FROM t1 t1_1 LEFT JOIN t1 t1_2 ON ( t1_2.b = t1_1.a ) WHERE t1_2.b NOT IN ( SELECT 4 UNION ALL SELECT 5 ); DROP TABLE t1; - diff --git a/mysql-test/valgrind.supp b/mysql-test/valgrind.supp index 8745c6990c3..c9ec5613dda 100644 --- a/mysql-test/valgrind.supp +++ b/mysql-test/valgrind.supp @@ -952,6 +952,14 @@ fun:backtrace } +{ + memory leak in mysqld_exit + Memcheck:Leak + fun:malloc + fun:_dl_close_worker + fun:_dl_close +} + # # Bug in Glibc 2.9: http://sourceware.org/bugzilla/show_bug.cgi?id=10391 # Fixed in latest Glibc, but suppressed here for running tests on hosts diff --git a/mysys/CMakeLists.txt b/mysys/CMakeLists.txt index 0d404586569..3cd243f9f14 100644 --- a/mysys/CMakeLists.txt +++ b/mysys/CMakeLists.txt @@ -48,7 +48,7 @@ IF (WIN32) ENDIF() IF(UNIX) - SET (MYSYS_SOURCES ${MYSYS_SOURCES} my_addr_resolve.c) + SET (MYSYS_SOURCES ${MYSYS_SOURCES} my_addr_resolve.c my_setuser.c) ENDIF() IF(HAVE_ALARM) diff --git a/mysys/my_setuser.c b/mysys/my_setuser.c new file mode 100644 index 00000000000..14ab04dd10f --- /dev/null +++ b/mysys/my_setuser.c @@ -0,0 +1,82 @@ +#include <my_global.h> +#include <m_string.h> +#include <my_sys.h> +#include <my_pthread.h> +#ifdef HAVE_PWD_H +#include <pwd.h> +#endif +#ifdef HAVE_GRP_H +#include <grp.h> +#endif + +struct passwd *my_check_user(const char *user, myf MyFlags) +{ + struct passwd *user_info; + uid_t user_id= geteuid(); + DBUG_ENTER("my_check_user"); + + // Don't bother if we aren't superuser + if (user_id) + { + if (user) + { + /* Don't give a warning, if real user is same as given with --user */ + user_info= getpwnam(user); + if (!user_info || user_id != user_info->pw_uid) + { + my_errno= EPERM; + if (MyFlags & MY_WME) + my_printf_error(my_errno, "One can only use the --user switch if " + "running as root", MYF(ME_JUST_WARNING|ME_NOREFRESH)); + } + } + DBUG_RETURN(NULL); + } + if (!user) + { + if (MyFlags & MY_FAE) + { + my_errno= EINVAL; + my_printf_error(my_errno, "Please consult the Knowledge Base to find " + "out how to run mysqld as root!", MYF(ME_NOREFRESH)); + } + DBUG_RETURN(NULL); + } + if (!strcmp(user,"root")) + DBUG_RETURN(NULL); + + if (!(user_info= getpwnam(user))) + { + // Allow a numeric uid to be used + int err= 0; + user_id= my_strtoll10(user, NULL, &err); + if (err || !(user_info= getpwuid(user_id))) + { + my_errno= EINVAL; + my_printf_error(my_errno, "Can't change to run as user '%s'. Please " + "check that the user exists!", MYF(ME_NOREFRESH), user); + DBUG_RETURN(NULL); + } + } + DBUG_ASSERT(user_info); + DBUG_RETURN(user_info); +} + +int my_set_user(const char *user, struct passwd *user_info, myf MyFlags) +{ + DBUG_ENTER("my_set_user"); + + DBUG_ASSERT(user_info != 0); +#ifdef HAVE_INITGROUPS + initgroups(user, user_info->pw_gid); +#endif + if (setgid(user_info->pw_gid) == -1 || setuid(user_info->pw_uid) == -1) + { + my_errno= errno; + if (MyFlags & MY_WME) + my_printf_error(errno, "Cannot change uid/gid (errno: %d)", MYF(ME_NOREFRESH), + errno); + DBUG_RETURN(my_errno); + } + DBUG_RETURN(0); +} diff --git a/plugin/server_audit/server_audit.c b/plugin/server_audit/server_audit.c index d48b6c37728..4c5e6992a32 100644 --- a/plugin/server_audit/server_audit.c +++ b/plugin/server_audit/server_audit.c @@ -2843,6 +2843,7 @@ void __attribute__ ((constructor)) audit_plugin_so_init(void) _mysql_plugin_declarations_[0].info= mysql_v4_descriptor; use_event_data_for_disconnect= 1; } + MYSQL_SYSVAR_NAME(loc_info).flags= PLUGIN_VAR_READONLY | PLUGIN_VAR_MEMALLOC; } memset(locinfo_ini_value, 'O', sizeof(locinfo_ini_value)-1); diff --git a/scripts/mysqld_safe.sh b/scripts/mysqld_safe.sh index 0a9f411717c..4decf4120c5 100644 --- a/scripts/mysqld_safe.sh +++ b/scripts/mysqld_safe.sh @@ -20,6 +20,7 @@ mysqld_ld_preload= mysqld_ld_library_path= flush_caches=0 numa_interleave=0 +unsafe_my_cnf=0 wsrep_on=0 dry_run=0 @@ -132,6 +133,18 @@ my_which () return $ret # Success } +find_in_bin() { + if test -x "$MY_BASEDIR_VERSION/bin/$1" + then + echo "$MY_BASEDIR_VERSION/bin/$1" + elif test -x "@bindir@/$1" + then + echo "@bindir@/$1" + else + echo "$1" + fi +} + log_generic () { [ $dry_run -eq 1 ] && return priority="$1" @@ -141,7 +154,7 @@ log_generic () { echo "$msg" case $logging in init) ;; # Just echo the message, don't save it anywhere - file) echo "$msg" >> "$err_log" ;; + file) echo "$msg" | "$helper" "$user" log "$err_log" ;; syslog) logger -t "$syslog_tag_mysqld_safe" -p "$priority" "$*" ;; *) echo "Internal program error (non-fatal):" \ @@ -161,7 +174,7 @@ log_notice () { eval_log_error () { local cmd="$1" case $logging in - file) cmd="$cmd >> "`shell_quote_string "$err_log"`" 2>&1" ;; + file) cmd="$cmd 2>&1 | "`shell_quote_string "$helper"`" $user log "`shell_quote_string "$err_log"` ;; syslog) # mysqld often prefixes its messages with a timestamp, which is # redundant when logging to syslog (which adds its own timestamp) @@ -195,6 +208,13 @@ shell_quote_string() { echo "$1" | sed -e 's,\([^a-zA-Z0-9/_.=-]\),\\\1,g' } +check_executable_location() { + if test "$unsafe_my_cnf" = 1 -a "$unrecognized_handling" != collect; then + log_error "Cannot accept $1 from a config file, when my.cnf is in the datadir" + exit 1 + fi +} + wsrep_pick_url() { [ $# -eq 0 ] && return 0 @@ -282,7 +302,6 @@ parse_arguments() { for arg do val=`echo "$arg" | sed -e "s;--[^=]*=;;"` case "$arg" in - --crash[-_]script=*) CRASH_SCRIPT="$val" ;; # these get passed explicitly to mysqld --basedir=*) MY_BASEDIR_VERSION="$val" ;; --datadir=*|--data=*) DATADIR="$val" ;; @@ -309,12 +328,14 @@ parse_arguments() { # mysqld_safe-specific options - must be set in my.cnf ([mysqld_safe])! --core[-_]file[-_]size=*) core_file_size="$val" ;; - --ledir=*) ledir="$val" ;; - --malloc[-_]lib=*) set_malloc_lib "$val" ;; - --mysqld=*) MYSQLD="$val" ;; + --ledir=*) check_executable_location "$arg" ; ledir="$val" ;; + --malloc[-_]lib=*) check_executable_location "$arg"; set_malloc_lib "$val" ;; + --crash[-_]script=*) check_executable_location "$arg"; crash_script="$val" ;; + --mysqld=*) check_executable_location "$arg"; MYSQLD="$val" ;; --mysqld[-_]version=*) if test -n "$val" then + check_executable_location "$arg" MYSQLD="mysqld-$val" PLUGIN_VARIANT="/$val" else @@ -498,15 +519,8 @@ set_malloc_lib() { # First, try to find BASEDIR and ledir (where mysqld is) # -if echo '@pkgdatadir@' | grep '^@prefix@' > /dev/null -then - relpkgdata=`echo '@pkgdatadir@' | sed -e 's,^@prefix@,,' -e 's,^/,,' -e 's,^,./,'` -else - # pkgdatadir is not relative to prefix - relpkgdata='@pkgdatadir@' -fi - -MY_PWD=`pwd` +MY_PWD=`dirname $0` +MY_PWD=`cd "$MY_PWD"/.. && pwd` # Check for the directories we would expect from a binary release install if test -n "$MY_BASEDIR_VERSION" -a -d "$MY_BASEDIR_VERSION" then @@ -522,16 +536,16 @@ then else ledir="$MY_BASEDIR_VERSION/bin" fi -elif test -f "$relpkgdata"/english/errmsg.sys -a -x "$MY_PWD/bin/mysqld" +elif test -x "$MY_PWD/bin/mysqld" then MY_BASEDIR_VERSION="$MY_PWD" # Where bin, share and data are ledir="$MY_PWD/bin" # Where mysqld is # Check for the directories we would expect from a source install -elif test -f "$relpkgdata"/english/errmsg.sys -a -x "$MY_PWD/libexec/mysqld" +elif test -x "$MY_PWD/libexec/mysqld" then MY_BASEDIR_VERSION="$MY_PWD" # Where libexec, share and var are ledir="$MY_PWD/libexec" # Where mysqld is -elif test -f "$relpkgdata"/english/errmsg.sys -a -x "$MY_PWD/sbin/mysqld" +elif test -x "$MY_PWD/sbin/mysqld" then MY_BASEDIR_VERSION="$MY_PWD" # Where sbin, share and var are ledir="$MY_PWD/sbin" # Where mysqld is @@ -541,6 +555,8 @@ else ledir='@libexecdir@' fi +helper=`find_in_bin mysqld_safe_helper` +print_defaults=`find_in_bin my_print_defaults` # # Second, try to find the data directory @@ -578,6 +594,7 @@ IGNORING $DATADIR/my.cnf" log_error "WARNING: Found $DATADIR/my.cnf The data directory is a deprecated location for my.cnf, please move it to $MY_BASEDIR_VERSION/my.cnf" + unsafe_my_cnf=1 MYSQL_HOME=$DATADIR else MYSQL_HOME=$MY_BASEDIR_VERSION @@ -585,34 +602,15 @@ $MY_BASEDIR_VERSION/my.cnf" fi export MYSQL_HOME - -# Get first arguments from the my.cnf file, groups [mysqld] and [mysqld_safe] -# and then merge with the command line arguments -if test -x "$MY_BASEDIR_VERSION/bin/my_print_defaults" -then - print_defaults="$MY_BASEDIR_VERSION/bin/my_print_defaults" -elif test -x `dirname $0`/my_print_defaults -then - print_defaults="`dirname $0`/my_print_defaults" -elif test -x ./bin/my_print_defaults -then - print_defaults="./bin/my_print_defaults" -elif test -x @bindir@/my_print_defaults -then - print_defaults="@bindir@/my_print_defaults" -elif test -x @bindir@/mysql_print_defaults -then - print_defaults="@bindir@/mysql_print_defaults" -else - print_defaults="my_print_defaults" -fi - append_arg_to_args () { args="$args "`shell_quote_string "$1"` } args= +# Get first arguments from the my.cnf file, groups [mysqld] and [mysqld_safe] +# and then merge with the command line arguments + SET_USER=2 parse_arguments `$print_defaults $defaults --loose-verbose --mysqld` if test $SET_USER -eq 2 @@ -721,11 +719,6 @@ then log_notice "Logging to '$err_log'." logging=file - if [ ! -f "$err_log" ]; then # if error log already exists, - touch "$err_log" # we just append. otherwise, - chmod "$fmode" "$err_log" # fix the permissions here! - fi - else if [ -n "$syslog_tag" ] then @@ -745,11 +738,6 @@ then then USER_OPTION="--user=$user" fi - # Change the err log to the right user, if it is in use - if [ $want_syslog -eq 0 ]; then - touch "$err_log" - chown $user "$err_log" - fi if test -n "$open_files" then ulimit -n $open_files @@ -1000,6 +988,16 @@ max_fast_restarts=5 # flag whether a usable sleep command exists have_sleep=1 +# close stdout and stderr, everything goes to $logging now +if expr "${-}" : '.*x' > /dev/null +then + : +else + exec 1>&- + exec 2>&- +fi + + # maximum number of wsrep restarts max_wsrep_restarts=0 @@ -1028,13 +1026,6 @@ do else eval_log_error "$cmd" fi - - if [ $want_syslog -eq 0 -a ! -f "$err_log" ]; then - touch "$err_log" # hypothetical: log was renamed but not - chown $user "$err_log" # flushed yet. we'd recreate it with - chmod "$fmode" "$err_log" # wrong owner next time we log, so set - fi # it up correctly while we can! - end_time=`date +%M%S` if test ! -f "$pid_file" # This is removed if normal shutdown @@ -1112,9 +1103,9 @@ do fi log_notice "mysqld restarted" - if test -n "$CRASH_SCRIPT" + if test -n "$crash_script" then - crash_script_output=`$CRASH_SCRIPT 2>&1` + crash_script_output=`$crash_script 2>&1` log_error "$crash_script_output" fi done diff --git a/sql/event_db_repository.cc b/sql/event_db_repository.cc index ba782a19f8c..3afd2659a29 100644 --- a/sql/event_db_repository.cc +++ b/sql/event_db_repository.cc @@ -18,6 +18,7 @@ #include "sql_priv.h" #include "unireg.h" #include "sql_base.h" // close_thread_tables +#include "sql_parse.h" #include "event_db_repository.h" #include "key.h" // key_copy #include "sql_db.h" // get_default_db_collation @@ -701,19 +702,17 @@ Event_db_repository::create_event(THD *thd, Event_parse_data *parse_data, restore_record(table, s->default_values); // Get default values for fields - if (system_charset_info->cset-> - numchars(system_charset_info, parse_data->dbname.str, - parse_data->dbname.str + parse_data->dbname.length) > - table->field[ET_FIELD_DB]->char_length()) + if (check_string_char_length(&parse_data->dbname, 0, + table->field[ET_FIELD_DB]->char_length(), + system_charset_info, 1)) { my_error(ER_TOO_LONG_IDENT, MYF(0), parse_data->dbname.str); goto end; } - if (system_charset_info->cset-> - numchars(system_charset_info, parse_data->name.str, - parse_data->name.str + parse_data->name.length) > - table->field[ET_FIELD_NAME]->char_length()) + if (check_string_char_length(&parse_data->name, 0, + table->field[ET_FIELD_NAME]->char_length(), + system_charset_info, 1)) { my_error(ER_TOO_LONG_IDENT, MYF(0), parse_data->name.str); goto end; diff --git a/sql/field.cc b/sql/field.cc index b909d14ec8f..58def13f5f3 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -1345,12 +1345,15 @@ void Field_num::prepend_zeros(String *value) const int diff; if ((diff= (int) (field_length - value->length())) > 0) { - bmove_upp((uchar*) value->ptr()+field_length, - (uchar*) value->ptr()+value->length(), - value->length()); - bfill((uchar*) value->ptr(),diff,'0'); - value->length(field_length); - (void) value->c_ptr_quick(); // Avoid warnings in purify + const bool error= value->realloc(field_length); + if (!error) + { + bmove_upp((uchar*) value->ptr()+field_length, + (uchar*) value->ptr()+value->length(), + value->length()); + bfill((uchar*) value->ptr(),diff,'0'); + value->length(field_length); + } } } diff --git a/sql/filesort.cc b/sql/filesort.cc index 917b9de6038..23674b937fc 100644 --- a/sql/filesort.cc +++ b/sql/filesort.cc @@ -1639,7 +1639,7 @@ int merge_buffers(Sort_param *param, IO_CACHE *from_file, if (!(error= (int) read_to_buffer(from_file, buffpek, rec_length))) { - queue_remove(&queue,0); + (void) queue_remove_top(&queue); reuse_freed_buff(&queue, buffpek, rec_length); } else if (error == -1) diff --git a/sql/handler.cc b/sql/handler.cc index 802364cb6e5..af06427ff19 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -4517,7 +4517,7 @@ void handler::get_dynamic_partition_info(PARTITION_STATS *stat_info, stat_info->update_time= stats.update_time; stat_info->check_time= stats.check_time; stat_info->check_sum= 0; - if (table_flags() & (HA_HAS_OLD_CHECKSUM | HA_HAS_OLD_CHECKSUM)) + if (table_flags() & (HA_HAS_OLD_CHECKSUM | HA_HAS_NEW_CHECKSUM)) stat_info->check_sum= checksum(); return; } diff --git a/sql/item.cc b/sql/item.cc index 015f5591c5d..2e7bc8e20c0 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -1133,7 +1133,8 @@ Item *Item_cache::safe_charset_converter(THD *thd, CHARSET_INFO *tocs) if (conv == example) return this; Item_cache *cache; - if (!conv || !(cache= new (thd->mem_root) Item_cache_str(thd, conv))) + if (!conv || conv->fix_fields(current_thd, (Item **) NULL) || + !(cache= new (thd->mem_root) Item_cache_str(thd, conv))) return NULL; // Safe conversion is not possible, or OEM cache->setup(thd, conv); cache->fixed= false; // Make Item::fix_fields() happy @@ -2679,6 +2680,44 @@ void Item_field::fix_after_pullout(st_select_lex *new_parent, Item **ref) depended_from= NULL; if (context) { + bool need_change= false; + /* + Suppose there are nested selects: + + select_id=1 + select_id=2 + select_id=3 <----+ + select_id=4 -+ + select_id=5 --+ + + Suppose, pullout operation has moved anything that had select_id=4 or 5 + in to select_id=3. + + If this Item_field had a name resolution context pointing into select_lex + with id=4 or id=5, it needs a new name resolution context. + + However, it could also be that this object is a part of outer reference: + Item_ref(Item_field(field in select with select_id=1))). + - The Item_ref object has a context with select_id=5, and so needs a new + name resolution context. + - The Item_field object has a context with select_id=1, and doesn't need + a new name resolution context. + + So, the following loop walks from Item_field's current context upwards. + If we find that the select we've been pulled out to is up there, we + create the new name resolution context. Otherwise, we don't. + */ + for (Name_resolution_context *ct= context; ct; ct= ct->outer_context) + { + if (new_parent == ct->select_lex) + { + need_change= true; + break; + } + } + if (!need_change) + return; + Name_resolution_context *ctx= new Name_resolution_context(); if (context->select_lex == new_parent) { diff --git a/sql/item_subselect.cc b/sql/item_subselect.cc index 98f7c470037..3157d666e9b 100644 --- a/sql/item_subselect.cc +++ b/sql/item_subselect.cc @@ -1,5 +1,5 @@ -/* Copyright (c) 2002, 2015, Oracle and/or its affiliates. - Copyright (c) 2010, 2015, MariaDB +/* Copyright (c) 2002, 2016, Oracle and/or its affiliates. + Copyright (c) 2010, 2016, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -3511,9 +3511,14 @@ bool subselect_union_engine::is_executed() const bool subselect_union_engine::no_rows() { /* Check if we got any rows when reading UNION result from temp. table: */ - return MY_TEST(!(unit->fake_select_lex ? - unit->fake_select_lex->join->send_records : - ((select_union_direct *)(unit->get_union_result())) + if (unit->fake_select_lex) + { + JOIN *join= unit->fake_select_lex->join; + if (join) + return MY_TEST(!join->send_records); + return false; + } + return MY_TEST(!(((select_union_direct *)(unit->get_union_result())) ->send_records)); } @@ -4897,9 +4902,9 @@ bool subselect_hash_sj_engine::init(List<Item> *tmp_columns, uint subquery_id) result= result_sink; /* - If the subquery has blobs, or the total key lenght is bigger than + If the subquery has blobs, or the total key length is bigger than some length, or the total number of key parts is more than the - allowed maximum (currently MAX_REF_PARTS == 16), then the created + allowed maximum (currently MAX_REF_PARTS == 32), then the created index cannot be used for lookups and we can't use hash semi join. If this is the case, delete the temporary table since it will not be used, and tell the caller we failed to initialize the diff --git a/sql/item_timefunc.cc b/sql/item_timefunc.cc index ff13e707ac4..49bf8036ea1 100644 --- a/sql/item_timefunc.cc +++ b/sql/item_timefunc.cc @@ -426,7 +426,7 @@ static bool extract_date_time(DATE_TIME_FORMAT *format, { if (!my_isspace(&my_charset_latin1,*val)) { - make_truncated_value_warning(current_thd, + make_truncated_value_warning(current_thd, Sql_condition::WARN_LEVEL_WARN, val_begin, length, cached_timestamp_type, NullS); @@ -711,7 +711,7 @@ static bool get_interval_info(const char *str,uint length,CHARSET_INFO *cs, { longlong value; const char *start= str; - for (value=0; str != end && my_isdigit(cs, *str) ; str++) + for (value= 0; str != end && my_isdigit(cs, *str); str++) value= value*10 + *str - '0'; msec_length= 6 - (str - start); values[i]= value; @@ -1464,6 +1464,7 @@ void Item_temporal_func::fix_length_and_dec() time can get us to return NULL. */ maybe_null= 1; + if (decimals) { if (decimals == NOT_FIXED_DEC) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index adb1b273b80..d540cc03d2a 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -130,10 +130,7 @@ extern "C" { // Because of SCO 3.2V4.2 #include <sysent.h> #endif #ifdef HAVE_PWD_H -#include <pwd.h> // For getpwent -#endif -#ifdef HAVE_GRP_H -#include <grp.h> +#include <pwd.h> // For struct passwd #endif #include <my_net.h> @@ -483,9 +480,7 @@ ulong opt_binlog_rows_event_max_size; my_bool opt_master_verify_checksum= 0; my_bool opt_slave_sql_verify_checksum= 1; const char *binlog_format_names[]= {"MIXED", "STATEMENT", "ROW", NullS}; -#ifdef HAVE_INITGROUPS volatile sig_atomic_t calling_initgroups= 0; /**< Used in SIGSEGV handler. */ -#endif uint mysqld_port, test_flags, select_errors, dropping_tables, ha_open_options; uint mysqld_extra_port; uint mysqld_port_timeout; @@ -2331,59 +2326,18 @@ static void set_ports() static struct passwd *check_user(const char *user) { -#if !defined(__WIN__) - struct passwd *tmp_user_info; - uid_t user_id= geteuid(); + myf flags= 0; + if (global_system_variables.log_warnings) + flags|= MY_WME; + if (!opt_bootstrap && !opt_help) + flags|= MY_FAE; - // Don't bother if we aren't superuser - if (user_id) - { - if (user) - { - /* Don't give a warning, if real user is same as given with --user */ - /* purecov: begin tested */ - tmp_user_info= getpwnam(user); - if ((!tmp_user_info || user_id != tmp_user_info->pw_uid) && - global_system_variables.log_warnings) - sql_print_warning( - "One can only use the --user switch if running as root\n"); - /* purecov: end */ - } - return NULL; - } - if (!user) - { - if (!opt_bootstrap && !opt_help) - { - sql_print_error("Fatal error: Please consult the Knowledge Base " - "to find out how to run mysqld as root!\n"); - unireg_abort(1); - } - return NULL; - } - /* purecov: begin tested */ - if (!strcmp(user,"root")) - return NULL; // Avoid problem with dynamic libraries + struct passwd *tmp_user_info= my_check_user(user, MYF(flags)); - if (!(tmp_user_info= getpwnam(user))) - { - // Allow a numeric uid to be used - const char *pos; - for (pos= user; my_isdigit(mysqld_charset,*pos); pos++) ; - if (*pos) // Not numeric id - goto err; - if (!(tmp_user_info= getpwuid(atoi(user)))) - goto err; - } + if (!tmp_user_info && my_errno==EINVAL && (flags & MY_FAE)) + unireg_abort(1); return tmp_user_info; - /* purecov: end */ - -err: - sql_print_error("Fatal error: Can't change to run as user '%s' ; Please check that the user exists!\n",user); - unireg_abort(1); -#endif - return NULL; } static inline void allow_coredumps() @@ -2400,10 +2354,6 @@ static inline void allow_coredumps() static void set_user(const char *user, struct passwd *user_info_arg) { - /* purecov: begin tested */ -#if !defined(__WIN__) - DBUG_ASSERT(user_info_arg != 0); -#ifdef HAVE_INITGROUPS /* We can get a SIGSEGV when calling initgroups() on some systems when NSS is configured to use LDAP and the server is statically linked. We set @@ -2411,22 +2361,11 @@ static void set_user(const char *user, struct passwd *user_info_arg) output a specific message to help the user resolve this problem. */ calling_initgroups= 1; - initgroups((char*) user, user_info_arg->pw_gid); + int res= my_set_user(user, user_info_arg, MYF(MY_WME)); calling_initgroups= 0; -#endif - if (setgid(user_info_arg->pw_gid) == -1) - { - sql_perror("setgid"); + if (res) unireg_abort(1); - } - if (setuid(user_info_arg->pw_uid) == -1) - { - sql_perror("setuid"); - unireg_abort(1); - } allow_coredumps(); -#endif - /* purecov: end */ } @@ -4475,19 +4414,24 @@ static int init_common_variables() default_charset_info= default_collation; } /* Set collactions that depends on the default collation */ - global_system_variables.collation_server= default_charset_info; - global_system_variables.collation_database= default_charset_info; - global_system_variables.collation_connection= default_charset_info; - global_system_variables.character_set_results= default_charset_info; - if (default_charset_info->mbminlen > 1) - { - global_system_variables.character_set_client= &my_charset_latin1; - sql_print_warning("Cannot use %s as character_set_client, %s will be used instead", - default_charset_info->csname, - global_system_variables.character_set_client->csname); + global_system_variables.collation_server= default_charset_info; + global_system_variables.collation_database= default_charset_info; + if (is_supported_parser_charset(default_charset_info)) + { + global_system_variables.collation_connection= default_charset_info; + global_system_variables.character_set_results= default_charset_info; + global_system_variables.character_set_client= default_charset_info; } else - global_system_variables.character_set_client= default_charset_info; + { + sql_print_warning("'%s' can not be used as client character set. " + "'%s' will be used as default client character set.", + default_charset_info->csname, + my_charset_latin1.csname); + global_system_variables.collation_connection= &my_charset_latin1; + global_system_variables.character_set_results= &my_charset_latin1; + global_system_variables.character_set_client= &my_charset_latin1; + } if (!(character_set_filesystem= get_charset_by_csname(character_set_filesystem_name, diff --git a/sql/signal_handler.cc b/sql/signal_handler.cc index 6593c3f3dbc..76af7733fb9 100644 --- a/sql/signal_handler.cc +++ b/sql/signal_handler.cc @@ -156,7 +156,7 @@ extern "C" sig_handler handle_fatal_signal(int sig) if (opt_stack_trace) { - my_safe_printf_stderr("Thread pointer: 0x%p\n", thd); + my_safe_printf_stderr("Thread pointer: %p\n", thd); my_safe_printf_stderr("%s", "Attempting backtrace. You can use the following " "information to find out\n" diff --git a/sql/sp_head.cc b/sql/sp_head.cc index 29427c3834a..58f5082ab14 100644 --- a/sql/sp_head.cc +++ b/sql/sp_head.cc @@ -542,12 +542,8 @@ check_routine_name(LEX_STRING *ident) my_error(ER_SP_WRONG_NAME, MYF(0), ident->str); return TRUE; } - if (check_string_char_length(ident, 0, NAME_CHAR_LEN, - system_charset_info, 1)) - { - my_error(ER_TOO_LONG_IDENT, MYF(0), ident->str); + if (check_ident_length(ident)) return TRUE; - } return FALSE; } @@ -3129,23 +3125,23 @@ sp_instr_stmt::execute(THD *thd, uint *nextp) thd->query_length()) <= 0) { res= m_lex_keeper.reset_lex_and_exec_core(thd, nextp, FALSE, this); + bool log_slow= !res && thd->enable_slow_log; - if (thd->get_stmt_da()->is_eof()) - { - /* Finalize server status flags after executing a statement. */ + /* Finalize server status flags after executing a statement. */ + if (log_slow || thd->get_stmt_da()->is_eof()) thd->update_server_status(); + if (thd->get_stmt_da()->is_eof()) thd->protocol->end_statement(); - } query_cache_end_of_result(thd); mysql_audit_general(thd, MYSQL_AUDIT_GENERAL_STATUS, - thd->get_stmt_da()->is_error() ? - thd->get_stmt_da()->sql_errno() : 0, - command_name[COM_QUERY].str); + thd->get_stmt_da()->is_error() ? + thd->get_stmt_da()->sql_errno() : 0, + command_name[COM_QUERY].str); - if (!res && unlikely(thd->enable_slow_log)) + if (log_slow) log_slow_statement(thd); } else diff --git a/sql/sql_base.cc b/sql/sql_base.cc index e3f755be7ae..860a2c52ab5 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -8761,9 +8761,7 @@ fill_record(THD *thd, TABLE *table_arg, List<Item> &fields, List<Item> &values, /* Update virtual fields*/ thd->abort_on_warning= FALSE; if (vcol_table && vcol_table->vfield && - update_virtual_fields(thd, vcol_table, - vcol_table->triggers ? VCOL_UPDATE_ALL : - VCOL_UPDATE_FOR_WRITE)) + update_virtual_fields(thd, vcol_table, VCOL_UPDATE_FOR_WRITE)) goto err; thd->abort_on_warning= save_abort_on_warning; thd->no_errors= save_no_errors; @@ -8882,9 +8880,7 @@ fill_record_n_invoke_before_triggers(THD *thd, TABLE *table, List<Item> &fields, if (item_field && item_field->field && table && table->vfield) { DBUG_ASSERT(table == item_field->field->table); - result= update_virtual_fields(thd, table, - table->triggers ? VCOL_UPDATE_ALL : - VCOL_UPDATE_FOR_WRITE); + result= update_virtual_fields(thd, table, VCOL_UPDATE_FOR_WRITE); } } } @@ -8972,9 +8968,7 @@ fill_record(THD *thd, TABLE *table, Field **ptr, List<Item> &values, /* Update virtual fields*/ thd->abort_on_warning= FALSE; if (table->vfield && - update_virtual_fields(thd, table, - table->triggers ? VCOL_UPDATE_ALL : - VCOL_UPDATE_FOR_WRITE)) + update_virtual_fields(thd, table, VCOL_UPDATE_FOR_WRITE)) goto err; thd->abort_on_warning= abort_on_warning_saved; DBUG_RETURN(thd->is_error()); @@ -9028,9 +9022,7 @@ fill_record_n_invoke_before_triggers(THD *thd, TABLE *table, Field **ptr, { DBUG_ASSERT(table == (*ptr)->table); if (table->vfield) - result= update_virtual_fields(thd, table, - table->triggers ? VCOL_UPDATE_ALL : - VCOL_UPDATE_FOR_WRITE); + result= update_virtual_fields(thd, table, VCOL_UPDATE_FOR_WRITE); } return result; diff --git a/sql/sql_connect.cc b/sql/sql_connect.cc index 80ac420937a..7befde1c1f3 100644 --- a/sql/sql_connect.cc +++ b/sql/sql_connect.cc @@ -1,6 +1,6 @@ /* Copyright (c) 2007, 2013, Oracle and/or its affiliates. - Copyright (c) 2008, 2014, SkySQL Ab. + Copyright (c) 2008, 2016, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -789,6 +789,7 @@ void update_global_user_stats(THD *thd, bool create_user, time_t now) bool thd_init_client_charset(THD *thd, uint cs_number) { + SV *gv=&global_system_variables; CHARSET_INFO *cs; /* Use server character set and collation if @@ -799,12 +800,10 @@ bool thd_init_client_charset(THD *thd, uint cs_number) if (!opt_character_set_client_handshake || !(cs= get_charset(cs_number, MYF(0)))) { - thd->variables.character_set_client= - global_system_variables.character_set_client; - thd->variables.collation_connection= - global_system_variables.collation_connection; - thd->variables.character_set_results= - global_system_variables.character_set_results; + DBUG_ASSERT(is_supported_parser_charset(gv->character_set_client)); + thd->variables.character_set_client= gv->character_set_client; + thd->variables.collation_connection= gv->collation_connection; + thd->variables.character_set_results= gv->character_set_results; } else { diff --git a/sql/sql_delete.cc b/sql/sql_delete.cc index 5b8896aa37f..c1011410970 100644 --- a/sql/sql_delete.cc +++ b/sql/sql_delete.cc @@ -572,9 +572,7 @@ bool mysql_delete(THD *thd, TABLE_LIST *table_list, COND *conds, { explain->tracker.on_record_read(); if (table->vfield) - update_virtual_fields(thd, table, - table->triggers ? VCOL_UPDATE_ALL : - VCOL_UPDATE_FOR_READ); + update_virtual_fields(thd, table, VCOL_UPDATE_FOR_READ); thd->inc_examined_row_count(1); // thd->is_error() is tested to disallow delete row on error if (!select || select->skip_record(thd) > 0) @@ -1343,4 +1341,3 @@ bool multi_delete::send_eof() } return 0; } - diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc index 19281f44771..bc0b88a7be7 100644 --- a/sql/sql_lex.cc +++ b/sql/sql_lex.cc @@ -4509,6 +4509,12 @@ bool st_select_lex::is_merged_child_of(st_select_lex *ancestor) { continue; } + + if (sl->master_unit()->derived && + sl->master_unit()->derived->is_merged_derived()) + { + continue; + } all_merged= FALSE; break; } diff --git a/sql/sql_load.cc b/sql/sql_load.cc index 630c1e0d21e..231d45b9016 100644 --- a/sql/sql_load.cc +++ b/sql/sql_load.cc @@ -1464,7 +1464,7 @@ READ_INFO::READ_INFO(File file_par, uint tot_length, CHARSET_INFO *cs, stack=stack_pos=(int*) sql_alloc(sizeof(int)*length); if (!(buffer=(uchar*) my_malloc(buff_length+1,MYF(MY_WME | MY_THREAD_SPECIFIC)))) - error=1; /* purecov: inspected */ + error= 1; /* purecov: inspected */ else { end_of_buff=buffer+buff_length; @@ -1724,7 +1724,7 @@ int READ_INFO::read_field() ** We come here if buffer is too small. Enlarge it and continue */ if (!(new_buffer=(uchar*) my_realloc((char*) buffer,buff_length+1+IO_SIZE, - MYF(MY_WME | MY_THREAD_SPECIFIC)))) + MYF(MY_WME | MY_THREAD_SPECIFIC)))) return (error=1); to=new_buffer + (to-buffer); buffer=new_buffer; @@ -1960,15 +1960,7 @@ int READ_INFO::read_value(int delim, String *val) for (chr= GET; my_tospace(chr) != delim && chr != my_b_EOF;) { #ifdef USE_MB - uint ml= my_mbcharlen(read_charset, chr); - if (ml == 0) - { - chr= my_b_EOF; - val->length(0); - return chr; - } - - if (ml > 1) + if (my_mbcharlen(read_charset, chr) > 1) { DBUG_PRINT("read_xml",("multi byte")); int i, ml= my_mbcharlen(read_charset, chr); diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 4f630b68264..033e88aa6e1 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -5470,11 +5470,8 @@ end_with_restore_list: } case SQLCOM_SHOW_CREATE_TRIGGER: { - if (lex->spname->m_name.length > NAME_LEN) - { - my_error(ER_TOO_LONG_IDENT, MYF(0), lex->spname->m_name.str); + if (check_ident_length(&lex->spname->m_name)) goto error; - } #ifdef WITH_WSREP if (WSREP_CLIENT(thd) && wsrep_sync_wait(thd)) goto error; @@ -9101,6 +9098,18 @@ bool check_string_char_length(LEX_STRING *str, uint err_msg, return TRUE; } + +bool check_ident_length(LEX_STRING *ident) +{ + if (check_string_char_length(ident, 0, NAME_CHAR_LEN, system_charset_info, 1)) + { + my_error(ER_TOO_LONG_IDENT, MYF(0), ident->str); + return 1; + } + return 0; +} + + C_MODE_START /* diff --git a/sql/sql_parse.h b/sql/sql_parse.h index 7a67df55ac3..c50090a95cc 100644 --- a/sql/sql_parse.h +++ b/sql/sql_parse.h @@ -76,6 +76,7 @@ bool check_string_byte_length(LEX_STRING *str, uint err_msg, bool check_string_char_length(LEX_STRING *str, uint err_msg, uint max_char_length, CHARSET_INFO *cs, bool no_error); +bool check_ident_length(LEX_STRING *ident); CHARSET_INFO* merge_charset_and_collation(CHARSET_INFO *cs, CHARSET_INFO *cl); CHARSET_INFO *find_bin_collation(CHARSET_INFO *cs); bool check_host_name(LEX_STRING *str); diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 3f06ec846d4..f299e863bf2 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -787,10 +787,15 @@ JOIN::prepare(Item ***rref_pointer_array, if (mixed_implicit_grouping && tbl->table) tbl->table->maybe_null= 1; } + + uint real_og_num= og_num; + if (skip_order_by && + select_lex != select_lex->master_unit()->global_parameters()) + real_og_num+= select_lex->order_list.elements; if ((wild_num && setup_wild(thd, tables_list, fields_list, &all_fields, wild_num)) || - select_lex->setup_ref_array(thd, og_num) || + select_lex->setup_ref_array(thd, real_og_num) || setup_fields(thd, (*rref_pointer_array), fields_list, MARK_COLUMNS_READ, &all_fields, 1) || setup_without_group(thd, (*rref_pointer_array), tables_list, @@ -13071,9 +13076,12 @@ COND *Item_cond_and::build_equal_items(THD *thd, COND_EQUAL cond_equal; cond_equal.upper_levels= inherited; + if (check_stack_overrun(thd, STACK_MIN_SIZE, NULL)) + return this; // Fatal error flag is set! + List<Item> eq_list; List<Item> *cond_args= argument_list(); - + List_iterator<Item> li(*cond_args); Item *item; @@ -13083,7 +13091,7 @@ COND *Item_cond_and::build_equal_items(THD *thd, that are subject to substitution by multiple equality items and removing each such predicate from the conjunction after having found/created a multiple equality whose inference the predicate is. - */ + */ while ((item= li++)) { /* @@ -17899,7 +17907,7 @@ do_select(JOIN *join,List<Item> *fields,TABLE *table,Procedure *procedure) join->join_tab[join->top_join_tab_count - 1].next_select= end_select; join_tab=join->join_tab+join->const_tables; } - join->send_records=0; + join->duplicate_rows= join->send_records=0; if (join->table_count == join->const_tables) { /* @@ -19438,7 +19446,12 @@ end_send(JOIN *join, JOIN_TAB *join_tab __attribute__((unused)), int error; /* result < 0 if row was not accepted and should not be counted */ if ((error= join->result->send_data(*join->fields))) - DBUG_RETURN(error < 0 ? NESTED_LOOP_OK : NESTED_LOOP_ERROR); + { + if (error > 0) + DBUG_RETURN(NESTED_LOOP_ERROR); + // error < 0 => duplicate row + join->duplicate_rows++; + } } ++join->send_records; @@ -19580,7 +19593,7 @@ end_send_group(JOIN *join, JOIN_TAB *join_tab __attribute__((unused)), if (error < 0) { /* Duplicate row, don't count */ - join->send_records--; + join->duplicate_rows++; error= 0; } } diff --git a/sql/sql_select.h b/sql/sql_select.h index dfa96f1c81c..50168f6592c 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -1071,6 +1071,10 @@ public: (if sql_calc_found_rows is used, LIMIT is ignored) */ ha_rows select_limit; + /* + Number of duplicate rows found in UNION. + */ + ha_rows duplicate_rows; /** Used to fetch no more than given amount of rows per one fetch operation of server side cursor. @@ -1347,7 +1351,7 @@ public: sort_and_group= 0; first_record= 0; do_send_rows= 1; - send_records= 0; + duplicate_rows= send_records= 0; found_records= 0; fetch_limit= HA_POS_ERROR; join_examined_rows= 0; diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 836e2bda70c..ebbec8c4c83 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -3489,7 +3489,6 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, sql_field->pack_length= dup_field->pack_length; sql_field->key_length= dup_field->key_length; sql_field->decimals= dup_field->decimals; - sql_field->create_length_to_internal_length(); sql_field->unireg_check= dup_field->unireg_check; /* We're making one field from two, the result field will have @@ -3499,6 +3498,7 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, if (!(sql_field->flags & NOT_NULL_FLAG)) null_fields--; sql_field->flags= dup_field->flags; + sql_field->create_length_to_internal_length(); sql_field->interval= dup_field->interval; sql_field->vcol_info= dup_field->vcol_info; sql_field->stored_in_db= dup_field->stored_in_db; @@ -3625,12 +3625,8 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, my_error(ER_TOO_MANY_KEY_PARTS,MYF(0),tmp); DBUG_RETURN(TRUE); } - if (check_string_char_length(&key->name, 0, NAME_CHAR_LEN, - system_charset_info, 1)) - { - my_error(ER_TOO_LONG_IDENT, MYF(0), key->name.str); + if (check_ident_length(&key->name)) DBUG_RETURN(TRUE); - } key_iterator2.rewind (); if (key->type != Key::FOREIGN_KEY) { @@ -4142,7 +4138,7 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, } if (thd->variables.sql_mode & MODE_NO_ZERO_DATE && - !sql_field->def && + !sql_field->def && !sql_field->vcol_info && is_timestamp_type(sql_field->sql_type) && (sql_field->flags & NOT_NULL_FLAG) && (type == Field::NONE || type == Field::TIMESTAMP_UN_FIELD)) diff --git a/sql/sql_trigger.cc b/sql/sql_trigger.cc index 7a61279fc9c..7514d2af739 100644 --- a/sql/sql_trigger.cc +++ b/sql/sql_trigger.cc @@ -2298,6 +2298,9 @@ void Table_triggers_list::mark_fields_used(trg_event_type event) bitmap_set_bit(trigger_table->read_set, trg_field->field_idx); if (trg_field->get_settable_routine_parameter()) bitmap_set_bit(trigger_table->write_set, trg_field->field_idx); + if (trigger_table->field[trg_field->field_idx]->vcol_info) + trigger_table->mark_virtual_col(trigger_table-> + field[trg_field->field_idx]); } } } diff --git a/sql/sql_udf.cc b/sql/sql_udf.cc index 0b294b5af8c..aa4859b6639 100644 --- a/sql/sql_udf.cc +++ b/sql/sql_udf.cc @@ -500,12 +500,8 @@ int mysql_create_function(THD *thd,udf_func *udf) my_message(ER_UDF_NO_PATHS, ER_THD(thd, ER_UDF_NO_PATHS), MYF(0)); DBUG_RETURN(1); } - if (check_string_char_length(&udf->name, 0, NAME_CHAR_LEN, - system_charset_info, 1)) - { - my_error(ER_TOO_LONG_IDENT, MYF(0), udf->name.str); + if (check_ident_length(&udf->name)) DBUG_RETURN(1); - } tables.init_one_table(STRING_WITH_LEN("mysql"), STRING_WITH_LEN("func"), "func", TL_WRITE); diff --git a/sql/sql_update.cc b/sql/sql_update.cc index 04aea060543..5ccc2384b84 100644 --- a/sql/sql_update.cc +++ b/sql/sql_update.cc @@ -640,9 +640,7 @@ int mysql_update(THD *thd, { explain->buf_tracker.on_record_read(); if (table->vfield) - update_virtual_fields(thd, table, - table->triggers ? VCOL_UPDATE_ALL : - VCOL_UPDATE_FOR_READ); + update_virtual_fields(thd, table, VCOL_UPDATE_FOR_READ); thd->inc_examined_row_count(1); if (!select || (error= select->skip_record(thd)) > 0) { @@ -758,9 +756,7 @@ int mysql_update(THD *thd, { explain->tracker.on_record_read(); if (table->vfield) - update_virtual_fields(thd, table, - table->triggers ? VCOL_UPDATE_ALL : - VCOL_UPDATE_FOR_READ); + update_virtual_fields(thd, table, VCOL_UPDATE_FOR_READ); thd->inc_examined_row_count(1); if (!select || select->skip_record(thd) > 0) { @@ -2437,8 +2433,7 @@ int multi_update::do_updates() if (table->default_field && (error= table->update_default_fields())) goto err2; if (table->vfield && - update_virtual_fields(thd, table, - (table->triggers ? VCOL_UPDATE_ALL : VCOL_UPDATE_FOR_WRITE))) + update_virtual_fields(thd, table, VCOL_UPDATE_FOR_WRITE)) goto err2; if ((error= cur_table->view_check_option(thd, ignore)) != VIEW_CHECK_OK) diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index e7db46d1c85..09dda0809de 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -5086,6 +5086,8 @@ part_name: { partition_info *part_info= Lex->part_info; partition_element *p_elem= part_info->curr_part_elem; + if (check_ident_length(&$1)) + MYSQL_YYABORT; p_elem->partition_name= $1.str; } ; @@ -5369,7 +5371,11 @@ sub_part_definition: sub_name: ident_or_text - { Lex->part_info->curr_part_elem->partition_name= $1.str; } + { + if (check_ident_length(&$1)) + MYSQL_YYABORT; + Lex->part_info->curr_part_elem->partition_name= $1.str; + } ; opt_part_options: diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index bb349e78724..1690bc40e33 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -639,6 +639,7 @@ static bool check_cs_client(sys_var *self, THD *thd, set_var *var) if (check_charset_not_null(self, thd, var)) return true; + // Currently, UCS-2 cannot be used as a client character set if (!is_supported_parser_charset((CHARSET_INFO *)(var->save_result.ptr))) return true; diff --git a/sql/sys_vars.ic b/sql/sys_vars.ic index 334dd318643..cbc10c85351 100644 --- a/sql/sys_vars.ic +++ b/sql/sys_vars.ic @@ -1247,7 +1247,7 @@ public: if (var->value->result_type() == STRING_RESULT) { - if (!(res=var->value->val_str(&str))) + if (!(res=var->value->val_str_ascii(&str))) return true; else { diff --git a/sql/table.cc b/sql/table.cc index 686b3569a97..8f93f8ae286 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -6872,11 +6872,9 @@ bool is_simple_order(ORDER *order) @details The function computes the values of the virtual columns of the table and stores them in the table record buffer. - If vcol_update_mode is set to VCOL_UPDATE_ALL then all virtual column are - computed. Otherwise, only fields from vcol_set are computed: all of them, - if vcol_update_mode is set to VCOL_UPDATE_FOR_WRITE, and, only those with - the stored_in_db flag set to false, if vcol_update_mode is equal to - VCOL_UPDATE_FOR_READ. + Only fields from vcol_set are computed: all of them, if vcol_update_mode is + set to VCOL_UPDATE_FOR_WRITE, and, only those with the stored_in_db flag + set to false, if vcol_update_mode is equal to VCOL_UPDATE_FOR_READ. @retval 0 Success @@ -6898,9 +6896,8 @@ int update_virtual_fields(THD *thd, TABLE *table, { vfield= (*vfield_ptr); DBUG_ASSERT(vfield->vcol_info && vfield->vcol_info->expr_item); - if ((bitmap_is_set(table->vcol_set, vfield->field_index) && - (vcol_update_mode == VCOL_UPDATE_FOR_WRITE || !vfield->stored_in_db)) || - vcol_update_mode == VCOL_UPDATE_ALL) + if (bitmap_is_set(table->vcol_set, vfield->field_index) && + (vcol_update_mode == VCOL_UPDATE_FOR_WRITE || !vfield->stored_in_db)) { /* Compute the actual value of the virtual fields */ error= vfield->vcol_info->expr_item->save_in_field(vfield, 0); diff --git a/sql/table.h b/sql/table.h index 093c357e575..876f496a312 100644 --- a/sql/table.h +++ b/sql/table.h @@ -317,8 +317,7 @@ enum release_type { RELEASE_NORMAL, RELEASE_WAIT_FOR_DROP }; enum enum_vcol_update_mode { VCOL_UPDATE_FOR_READ= 0, - VCOL_UPDATE_FOR_WRITE, - VCOL_UPDATE_ALL + VCOL_UPDATE_FOR_WRITE }; class Filesort_info diff --git a/storage/connect/ApacheInterface.java b/storage/connect/ApacheInterface.java index b4c8a4e9885..47b46dc0506 100644 --- a/storage/connect/ApacheInterface.java +++ b/storage/connect/ApacheInterface.java @@ -35,7 +35,10 @@ public class ApacheInterface extends JdbcInterface { ds.setPassword(parms[3]); pool.put(url, ds); } // endif ds - + + // if (parms.length > 4 && parms[4] != null) + // ds.setConnectionProperties(parms[4]); + // Get a connection from the data source conn = ds.getConnection(); diff --git a/storage/connect/CMakeLists.txt b/storage/connect/CMakeLists.txt index 95d88538119..ce6de424421 100644 --- a/storage/connect/CMakeLists.txt +++ b/storage/connect/CMakeLists.txt @@ -20,25 +20,25 @@ SET(CONNECT_SOURCES ha_connect.cc connect.cc user_connect.cc mycat.cc fmdlex.c osutil.c plugutil.c rcmsg.c rcmsg.h array.cpp blkfil.cpp colblk.cpp csort.cpp -filamap.cpp filamdbf.cpp filamfix.cpp filamtxt.cpp filamvct.cpp filamzip.cpp +filamap.cpp filamdbf.cpp filamfix.cpp filamgz.cpp filamtxt.cpp filter.cpp json.cpp jsonudf.cpp maputil.cpp myconn.cpp myutil.cpp plgdbutl.cpp reldef.cpp tabcol.cpp tabdos.cpp tabfix.cpp tabfmt.cpp tabjson.cpp table.cpp tabmul.cpp tabmysql.cpp taboccur.cpp tabpivot.cpp tabsys.cpp tabtbl.cpp tabutil.cpp -tabvct.cpp tabvir.cpp tabxcl.cpp valblk.cpp value.cpp xindex.cpp xobject.cpp +tabvir.cpp tabxcl.cpp valblk.cpp value.cpp xindex.cpp xobject.cpp array.h blkfil.h block.h catalog.h checklvl.h colblk.h connect.h csort.h -engmsg.h filamap.h filamdbf.h filamfix.h filamtxt.h filamvct.h filamzip.h +engmsg.h filamap.h filamdbf.h filamfix.h filamgz.h filamtxt.h filter.h global.h ha_connect.h inihandl.h json.h jsonudf.h maputil.h msgid.h mycat.h myconn.h myutil.h os.h osutil.h plgcnx.h plgdbsem.h preparse.h reldef.h resource.h tabcol.h tabdos.h tabfix.h tabfmt.h tabjson.h tabmul.h tabmysql.h -taboccur.h tabpivot.h tabsys.h tabtbl.h tabutil.h tabvct.h tabvir.h tabxcl.h +taboccur.h tabpivot.h tabsys.h tabtbl.h tabutil.h tabvir.h tabxcl.h user_connect.h valblk.h value.h xindex.h xobject.h xtable.h) # # Definitions that are shared for all OSes # add_definitions( -DMARIADB -DFORCE_INIT_OF_VARS -Dconnect_EXPORTS) -add_definitions( -DHUGE_SUPPORT -DZIP_SUPPORT -DPIVOT_SUPPORT ) +add_definitions( -DHUGE_SUPPORT -DGZ_SUPPORT -DPIVOT_SUPPORT ) # @@ -90,6 +90,18 @@ ENDIF(UNIX) # +# VCT: the VEC format might be not supported in future versions +# + +OPTION(CONNECT_WITH_VCT "Compile CONNECT storage engine with VCT support" ON) + +IF(CONNECT_WITH_VCT) + SET(CONNECT_SOURCES ${CONNECT_SOURCES} filamvct.cpp tabvct.cpp filamvct.h tabvct.h) + add_definitions(-DVCT_SUPPORT) +ENDIF(CONNECT_WITH_VCT) + + +# # XML # @@ -236,9 +248,9 @@ ENDIF(CONNECT_WITH_ODBC) # JDBC # IF(APPLE) - OPTION(CONNECT_WITH_JDBC "some comment" OFF) + OPTION(CONNECT_WITH_JDBC "Compile CONNECT storage engine without JDBC support" OFF) ELSE() - OPTION(CONNECT_WITH_JDBC "some comment" ON) + OPTION(CONNECT_WITH_JDBC "Compile CONNECT storage engine with JDBC support" ON) ENDIF() IF(CONNECT_WITH_JDBC) @@ -252,18 +264,33 @@ IF(CONNECT_WITH_JDBC) SET(CONNECT_SOURCES ${CONNECT_SOURCES} jdbconn.cpp tabjdbc.cpp jdbconn.h tabjdbc.h jdbccat.h JdbcInterface.java ApacheInterface.java MariadbInterface.java - MysqlInterface.java OracleInterface.java PostgresqlInterface.java) + MysqlInterface.java OracleInterface.java PostgresqlInterface.java + JavaWrappers.jar) # TODO: Find how to compile and install the java wrapper classes # Find required libraries and include directories SET (JAVA_SOURCES JdbcInterface.java) add_jar(JdbcInterface ${JAVA_SOURCES}) - install_jar(JdbcInterface DESTINATION ${INSTALL_PLUGINDIR} COMPONENT connect-engine) + install_jar(JdbcInterface DESTINATION ${INSTALL_PLUGINDIR} COMPONENT connect-engine) + INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/JavaWrappers.jar + DESTINATION ${INSTALL_PLUGINDIR} COMPONENT connect-engine) add_definitions(-DJDBC_SUPPORT) ELSE() SET(JDBC_LIBRARY "") ENDIF() ENDIF(CONNECT_WITH_JDBC) +# +# ZIP +# + +OPTION(CONNECT_WITH_ZIP "Compile CONNECT storage engine with ZIP support" ON) + +IF(CONNECT_WITH_ZIP) + SET(CONNECT_SOURCES ${CONNECT_SOURCES} filamzip.cpp tabzip.cpp unzip.c ioapi.c zip.c + filamzip.h tabzip.h ioapi.h unzip.h zip.h) + add_definitions(-DZIP_SUPPORT -DNOCRYPT) +ENDIF(CONNECT_WITH_ZIP) + # # XMAP diff --git a/storage/connect/JavaWrappers.jar b/storage/connect/JavaWrappers.jar Binary files differnew file mode 100644 index 00000000000..8c01c364a3f --- /dev/null +++ b/storage/connect/JavaWrappers.jar diff --git a/storage/connect/JdbcInterface.java b/storage/connect/JdbcInterface.java index 34af8c4e013..a1b1360e6ea 100644 --- a/storage/connect/JdbcInterface.java +++ b/storage/connect/JdbcInterface.java @@ -82,6 +82,9 @@ public class JdbcInterface { System.out.println("URL=" + parms[1]); CheckURL(parms[1], null); + + // This is required for drivers using context class loaders + Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); if (parms[2] != null && !parms[2].isEmpty()) { if (DEBUG) @@ -220,6 +223,19 @@ public class JdbcInterface { } // end of SetTimestampParm + public int SetNullParm(int i, int typ) { + int rc = 0; + + try { + pstmt.setNull(i, typ); + } catch (Exception e) { + SetErrmsg(e); + rc = -1; + } // end try/catch + + return rc; + } // end of SetNullParm + public int ExecutePrep() { int n = -3; diff --git a/storage/connect/domdoc.cpp b/storage/connect/domdoc.cpp index 64a0a172956..eb9660b439d 100644 --- a/storage/connect/domdoc.cpp +++ b/storage/connect/domdoc.cpp @@ -89,30 +89,43 @@ DOMDOC::DOMDOC(char *nsl, char *nsdf, char *enc, PFBLOCK fp) /******************************************************************/ /* Initialize XML parser and check library compatibility. */ /******************************************************************/ -bool DOMDOC::Initialize(PGLOBAL g) - { - if (TestHr(g, CoInitialize(NULL))) +bool DOMDOC::Initialize(PGLOBAL g, char *entry, bool zipped) +{ + if (zipped && InitZip(g, entry)) + return true; + + if (TestHr(g, CoInitialize(NULL))) return true; if (TestHr(g, Docp.CreateInstance("msxml2.domdocument"))) return true; return MakeNSlist(g); - } // end of Initialize +} // end of Initialize /******************************************************************/ /* Parse the XML file and construct node tree in memory. */ /******************************************************************/ -bool DOMDOC::ParseFile(char *fn) - { - // Load the document +bool DOMDOC::ParseFile(PGLOBAL g, char *fn) +{ + bool b; + Docp->async = false; - if (!(bool)Docp->load((_bstr_t)fn)) + if (zip) { + // Parse an in memory document + char *xdoc = GetMemDoc(g, fn); + + b = (xdoc) ? (bool)Docp->loadXML((_bstr_t)xdoc) : false; + } else + // Load the document + b = (bool)Docp->load((_bstr_t)fn); + + if (!b) return true; return false; - } // end of ParseFile +} // end of ParseFile /******************************************************************/ /* Create or reuse an Xblock for this document. */ @@ -239,6 +252,7 @@ int DOMDOC::DumpDoc(PGLOBAL g, char *ofn) void DOMDOC::CloseDoc(PGLOBAL g, PFBLOCK xp) { CloseXMLFile(g, xp, false); + CloseZip(); } // end of Close /* ----------------------- class DOMNODE ------------------------ */ @@ -616,13 +630,13 @@ PXNODE DOMNODELIST::GetItem(PGLOBAL g, int n, PXNODE np) /* Reset the pointer on the deleted item. */ /******************************************************************/ bool DOMNODELIST::DropItem(PGLOBAL g, int n) - { - if (Listp == NULL || Listp->length <= n) - return true; +{ + if (Listp == NULL || Listp->length < n) + return true; //Listp->item[n] = NULL; La propriété n'a pas de méthode 'set' return false; - } // end of DeleteItem +} // end of DeleteItem /* ----------------------- class DOMATTR ------------------------ */ diff --git a/storage/connect/domdoc.h b/storage/connect/domdoc.h index 2cffec499e2..cfec98a9422 100644 --- a/storage/connect/domdoc.h +++ b/storage/connect/domdoc.h @@ -37,8 +37,8 @@ class DOMDOC : public XMLDOCUMENT { virtual void SetNofree(bool b) {} // Only libxml2 // Methods - virtual bool Initialize(PGLOBAL g); - virtual bool ParseFile(char *fn); + virtual bool Initialize(PGLOBAL g, char *entry, bool zipped); + virtual bool ParseFile(PGLOBAL g, char *fn); virtual bool NewDoc(PGLOBAL g, char *ver); virtual void AddComment(PGLOBAL g, char *com); virtual PXNODE GetRoot(PGLOBAL g); diff --git a/storage/connect/filamap.cpp b/storage/connect/filamap.cpp index c6b4585a839..94c562a9981 100644 --- a/storage/connect/filamap.cpp +++ b/storage/connect/filamap.cpp @@ -87,7 +87,7 @@ int MAPFAM::GetFileLength(PGLOBAL g) { int len; - len = (To_Fb) ? To_Fb->Length : TXTFAM::GetFileLength(g); + len = (To_Fb && To_Fb->Count) ? To_Fb->Length : TXTFAM::GetFileLength(g); if (trace) htrc("Mapped file length=%d\n", len); @@ -319,11 +319,13 @@ int MAPFAM::SkipRecord(PGLOBAL g, bool header) /***********************************************************************/ int MAPFAM::ReadBuffer(PGLOBAL g) { - int len; + int rc, len; // Are we at the end of the memory - if (Mempos >= Top) - return RC_EF; + if (Mempos >= Top) + if ((rc = GetNext(g)) != RC_OK) + return rc; + if (!Placed) { /*******************************************************************/ @@ -341,8 +343,10 @@ int MAPFAM::ReadBuffer(PGLOBAL g) /*******************************************************************/ switch (Tdbp->TestBlock(g)) { case RC_EF: - return RC_EF; - case RC_NF: + if ((rc = GetNext(g)) != RC_OK) + return rc; + + case RC_NF: // Skip this record if ((rc = SkipRecord(g, false)) != RC_OK) return rc; @@ -498,7 +502,7 @@ int MAPFAM::DeleteRecords(PGLOBAL g, int irc) void MAPFAM::CloseTableFile(PGLOBAL g, bool) { PlugCloseFile(g, To_Fb); - To_Fb = NULL; // To get correct file size in Cardinality +//To_Fb = NULL; // To get correct file size in Cardinality if (trace) htrc("MAP Close: closing %s count=%d\n", @@ -569,7 +573,7 @@ int MBKFAM::GetRowID(void) /***********************************************************************/ int MBKFAM::ReadBuffer(PGLOBAL g) { - int len; + int rc, len; /*********************************************************************/ /* Sequential block reading when Placed is not true. */ @@ -577,8 +581,10 @@ int MBKFAM::ReadBuffer(PGLOBAL g) if (Placed) { Placed = false; } else if (Mempos >= Top) { // Are we at the end of the memory - return RC_EF; - } else if (++CurNum < Nrec) { + if ((rc = GetNext(g)) != RC_OK) + return rc; + + } else if (++CurNum < Nrec) { Fpos = Mempos; } else { /*******************************************************************/ @@ -588,7 +594,8 @@ int MBKFAM::ReadBuffer(PGLOBAL g) next: if (++CurBlk >= Block) - return RC_EF; + if ((rc = GetNext(g)) != RC_OK) + return rc; /*******************************************************************/ /* Before reading a new block, check whether block optimization */ @@ -596,8 +603,11 @@ int MBKFAM::ReadBuffer(PGLOBAL g) /*******************************************************************/ switch (Tdbp->TestBlock(g)) { case RC_EF: - return RC_EF; - case RC_NF: + if ((rc = GetNext(g)) != RC_OK) + return rc; + + break; + case RC_NF: goto next; } // endswitch rc @@ -697,14 +707,18 @@ int MPXFAM::InitDelete(PGLOBAL, int fpos, int) /***********************************************************************/ int MPXFAM::ReadBuffer(PGLOBAL g) { + int rc; + /*********************************************************************/ /* Sequential block reading when Placed is not true. */ /*********************************************************************/ if (Placed) { Placed = false; } else if (Mempos >= Top) { // Are we at the end of the memory - return RC_EF; - } else if (++CurNum < Nrec) { + if ((rc = GetNext(g)) != RC_OK) + return rc; + + } else if (++CurNum < Nrec) { Fpos = Mempos; } else { /*******************************************************************/ @@ -714,7 +728,7 @@ int MPXFAM::ReadBuffer(PGLOBAL g) next: if (++CurBlk >= Block) - return RC_EF; + return GetNext(g); /*******************************************************************/ /* Before reading a new block, check whether block optimization */ @@ -722,8 +736,11 @@ int MPXFAM::ReadBuffer(PGLOBAL g) /*******************************************************************/ switch (Tdbp->TestBlock(g)) { case RC_EF: - return RC_EF; - case RC_NF: + if ((rc = GetNext(g)) != RC_OK) + return rc; + + break; + case RC_NF: goto next; } // endswitch rc diff --git a/storage/connect/filamap.h b/storage/connect/filamap.h index b9c8ad965fd..774eb8b91b3 100644 --- a/storage/connect/filamap.h +++ b/storage/connect/filamap.h @@ -41,7 +41,8 @@ class DllExport MAPFAM : public TXTFAM { virtual int SkipRecord(PGLOBAL g, bool header); virtual bool OpenTableFile(PGLOBAL g); virtual bool DeferReading(void) {return false;} - virtual int ReadBuffer(PGLOBAL g); + virtual int GetNext(PGLOBAL g) {return RC_EF;} + virtual int ReadBuffer(PGLOBAL g); virtual int WriteBuffer(PGLOBAL g); virtual int DeleteRecords(PGLOBAL g, int irc); virtual void CloseTableFile(PGLOBAL g, bool abort); diff --git a/storage/connect/filamgz.cpp b/storage/connect/filamgz.cpp new file mode 100644 index 00000000000..07242ea633c --- /dev/null +++ b/storage/connect/filamgz.cpp @@ -0,0 +1,1426 @@ +/************ File AM GZ C++ Program Source Code File (.CPP) ***********/ +/* PROGRAM NAME: FILAMGZ */ +/* ------------- */ +/* Version 1.5 */ +/* */ +/* COPYRIGHT: */ +/* ---------- */ +/* (C) Copyright to the author Olivier BERTRAND 2005-2016 */ +/* */ +/* WHAT THIS PROGRAM DOES: */ +/* ----------------------- */ +/* This program are the ZLIB compressed files classes. */ +/* */ +/***********************************************************************/ + +/***********************************************************************/ +/* Include relevant MariaDB header file. */ +/***********************************************************************/ +#include "my_global.h" +#if defined(__WIN__) +#include <io.h> +#include <fcntl.h> +#if defined(__BORLANDC__) +#define __MFC_COMPAT__ // To define min/max as macro +#endif +//#include <windows.h> +#else // !__WIN__ +#if defined(UNIX) +#include <errno.h> +#else // !UNIX +#include <io.h> +#endif +#include <fcntl.h> +#endif // !__WIN__ + +/***********************************************************************/ +/* Include application header files: */ +/* global.h is header containing all global declarations. */ +/* plgdbsem.h is header containing the DB application declarations. */ +/* tabdos.h is header containing the TABDOS class declarations. */ +/***********************************************************************/ +#include "global.h" +#include "plgdbsem.h" +//#include "catalog.h" +//#include "reldef.h" +//#include "xobject.h" +//#include "kindex.h" +#include "filamtxt.h" +#include "tabdos.h" +#if defined(UNIX) +#include "osutil.h" +#endif + +/***********************************************************************/ +/* This define prepares ZLIB function declarations. */ +/***********************************************************************/ +//#define ZLIB_DLL + +#include "filamgz.h" + +/***********************************************************************/ +/* DB static variables. */ +/***********************************************************************/ +extern int num_read, num_there, num_eq[]; // Statistics + +/* ------------------------------------------------------------------- */ + +/***********************************************************************/ +/* Implementation of the GZFAM class. */ +/***********************************************************************/ +GZFAM::GZFAM(PGZFAM txfp) : TXTFAM(txfp) + { + Zfile = txfp->Zfile; + Zpos = txfp->Zpos; + } // end of GZFAM copy constructor + +/***********************************************************************/ +/* Zerror: Error function for gz calls. */ +/* gzerror returns the error message for the last error which occurred*/ +/* on the given compressed file. errnum is set to zlib error number. */ +/* If an error occurred in the file system and not in the compression */ +/* library, errnum is set to Z_ERRNO and the application may consult */ +/* errno to get the exact error code. */ +/***********************************************************************/ +int GZFAM::Zerror(PGLOBAL g) + { + int errnum; + + strcpy(g->Message, gzerror(Zfile, &errnum)); + + if (errnum == Z_ERRNO) +#if defined(__WIN__) + sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(NULL)); +#else // !__WIN__ + sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(errno)); +#endif // !__WIN__ + + return (errnum == Z_STREAM_END) ? RC_EF : RC_FX; + } // end of Zerror + +/***********************************************************************/ +/* Reset: reset position values at the beginning of file. */ +/***********************************************************************/ +void GZFAM::Reset(void) + { + TXTFAM::Reset(); +//gzrewind(Zfile); // Useful ????? + Zpos = 0; + } // end of Reset + +/***********************************************************************/ +/* GZ GetFileLength: returns an estimate of what would be the */ +/* uncompressed file size in number of bytes. */ +/***********************************************************************/ +int GZFAM::GetFileLength(PGLOBAL g) + { + int len = TXTFAM::GetFileLength(g); + + if (len > 0) + // Estimate size reduction to a max of 6 + len *= 6; + + return len; + } // end of GetFileLength + +/***********************************************************************/ +/* GZ Access Method opening routine. */ +/***********************************************************************/ +bool GZFAM::OpenTableFile(PGLOBAL g) + { + char opmode[4], filename[_MAX_PATH]; + MODE mode = Tdbp->GetMode(); + + switch (mode) { + case MODE_READ: + strcpy(opmode, "r"); + break; + case MODE_UPDATE: + /*****************************************************************/ + /* Updating GZ files not implemented yet. */ + /*****************************************************************/ + strcpy(g->Message, MSG(UPD_ZIP_NOT_IMP)); + return true; + case MODE_DELETE: + if (!Tdbp->GetNext()) { + // Store the number of deleted lines + DelRows = Cardinality(g); + + // This will erase the entire file + strcpy(opmode, "w"); +// Block = 0; // For ZBKFAM +// Last = Nrec; // For ZBKFAM + Tdbp->ResetSize(); + } else { + sprintf(g->Message, MSG(NO_PART_DEL), "GZ"); + return true; + } // endif filter + + break; + case MODE_INSERT: + strcpy(opmode, "a+"); + break; + default: + sprintf(g->Message, MSG(BAD_OPEN_MODE), mode); + return true; + } // endswitch Mode + + /*********************************************************************/ + /* Open according to logical input/output mode required. */ + /* Use specific zlib functions. */ + /* Treat files as binary. */ + /*********************************************************************/ + strcat(opmode, "b"); + Zfile = gzopen(PlugSetPath(filename, To_File, Tdbp->GetPath()), opmode); + + if (Zfile == NULL) { + sprintf(g->Message, MSG(GZOPEN_ERROR), + opmode, (int)errno, filename); + strcat(strcat(g->Message, ": "), strerror(errno)); + return (mode == MODE_READ && errno == ENOENT) + ? PushWarning(g, Tdbp) : true; + } // endif Zfile + + /*********************************************************************/ + /* Something to be done here. >>>>>>>> NOT DONE <<<<<<<< */ + /*********************************************************************/ +//To_Fb = dbuserp->Openlist; // Keep track of File block + + /*********************************************************************/ + /* Allocate the line buffer. */ + /*********************************************************************/ + return AllocateBuffer(g); + } // end of OpenTableFile + +/***********************************************************************/ +/* Allocate the line buffer. For mode Delete a bigger buffer has to */ +/* be allocated because is it also used to move lines into the file. */ +/***********************************************************************/ +bool GZFAM::AllocateBuffer(PGLOBAL g) + { + MODE mode = Tdbp->GetMode(); + + Buflen = Lrecl + 2; // Lrecl does not include CRLF +//Buflen *= ((Mode == MODE_DELETE) ? DOS_BUFF_LEN : 1); NIY + + if (trace) + htrc("SubAllocating a buffer of %d bytes\n", Buflen); + + To_Buf = (char*)PlugSubAlloc(g, NULL, Buflen); + + if (mode == MODE_INSERT) { + /*******************************************************************/ + /* For Insert buffer must be prepared. */ + /*******************************************************************/ + memset(To_Buf, ' ', Buflen); + To_Buf[Buflen - 2] = '\n'; + To_Buf[Buflen - 1] = '\0'; + } // endif Insert + + return false; + } // end of AllocateBuffer + +/***********************************************************************/ +/* GetRowID: return the RowID of last read record. */ +/***********************************************************************/ +int GZFAM::GetRowID(void) + { + return Rows; + } // end of GetRowID + +/***********************************************************************/ +/* GetPos: return the position of last read record. */ +/***********************************************************************/ +int GZFAM::GetPos(void) + { + return (int)Zpos; + } // end of GetPos + +/***********************************************************************/ +/* GetNextPos: return the position of next record. */ +/***********************************************************************/ +int GZFAM::GetNextPos(void) + { + return gztell(Zfile); + } // end of GetNextPos + +/***********************************************************************/ +/* SetPos: Replace the table at the specified position. */ +/***********************************************************************/ +bool GZFAM::SetPos(PGLOBAL g, int pos __attribute__((unused))) + { + sprintf(g->Message, MSG(NO_SETPOS_YET), "GZ"); + return true; +#if 0 + Fpos = pos; + + if (fseek(Stream, Fpos, SEEK_SET)) { + sprintf(g->Message, MSG(FSETPOS_ERROR), Fpos); + return true; + } // endif + + Placed = true; + return false; +#endif // 0 + } // end of SetPos + +/***********************************************************************/ +/* Record file position in case of UPDATE or DELETE. */ +/***********************************************************************/ +bool GZFAM::RecordPos(PGLOBAL) + { + Zpos = gztell(Zfile); + return false; + } // end of RecordPos + +/***********************************************************************/ +/* Skip one record in file. */ +/***********************************************************************/ +int GZFAM::SkipRecord(PGLOBAL g, bool header) + { + // Skip this record + if (gzeof(Zfile)) + return RC_EF; + else if (gzgets(Zfile, To_Buf, Buflen) == Z_NULL) + return Zerror(g); + + if (header) + RecordPos(g); + + return RC_OK; + } // end of SkipRecord + +/***********************************************************************/ +/* ReadBuffer: Read one line from a compressed text file. */ +/***********************************************************************/ +int GZFAM::ReadBuffer(PGLOBAL g) + { + char *p; + int rc; + + if (!Zfile) + return RC_EF; + + if (!Placed) { + /*******************************************************************/ + /* Record file position in case of UPDATE or DELETE. */ + /*******************************************************************/ + next: + if (RecordPos(g)) + return RC_FX; + + CurBlk = Rows++; // Update RowID + + /*******************************************************************/ + /* Check whether optimization on ROWID */ + /* can be done, as well as for join as for local filtering. */ + /*******************************************************************/ + switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + // Skip this record + if ((rc = SkipRecord(g, FALSE)) != RC_OK) + return rc; + + goto next; + } // endswitch rc + + } else + Placed = false; + + if (gzeof(Zfile)) { + rc = RC_EF; + } else if (gzgets(Zfile, To_Buf, Buflen) != Z_NULL) { + p = To_Buf + strlen(To_Buf) - 1; + + if (*p == '\n') + *p = '\0'; // Eliminate ending new-line character + + if (*(--p) == '\r') + *p = '\0'; // Eliminate eventuel carriage return + + strcpy(Tdbp->GetLine(), To_Buf); + IsRead = true; + rc = RC_OK; + num_read++; + } else + rc = Zerror(g); + + if (trace > 1) + htrc(" Read: '%s' rc=%d\n", To_Buf, rc); + + return rc; + } // end of ReadBuffer + +/***********************************************************************/ +/* WriteDB: Data Base write routine for ZDOS access method. */ +/* Update is not possible without using a temporary file (NIY). */ +/***********************************************************************/ +int GZFAM::WriteBuffer(PGLOBAL g) + { + /*********************************************************************/ + /* Prepare the write buffer. */ + /*********************************************************************/ + strcat(strcpy(To_Buf, Tdbp->GetLine()), CrLf); + + /*********************************************************************/ + /* Now start the writing process. */ + /*********************************************************************/ + if (gzputs(Zfile, To_Buf) < 0) + return Zerror(g); + + return RC_OK; + } // end of WriteBuffer + +/***********************************************************************/ +/* Data Base delete line routine for ZDOS access method. (NIY) */ +/***********************************************************************/ +int GZFAM::DeleteRecords(PGLOBAL g, int) + { + strcpy(g->Message, MSG(NO_ZIP_DELETE)); + return RC_FX; + } // end of DeleteRecords + +/***********************************************************************/ +/* Data Base close routine for DOS access method. */ +/***********************************************************************/ +void GZFAM::CloseTableFile(PGLOBAL, bool) + { + int rc = gzclose(Zfile); + + if (trace) + htrc("GZ CloseDB: closing %s rc=%d\n", To_File, rc); + + Zfile = NULL; // So we can know whether table is open +//To_Fb->Count = 0; // Avoid double closing by PlugCloseAll + } // end of CloseTableFile + +/***********************************************************************/ +/* Rewind routine for GZ access method. */ +/***********************************************************************/ +void GZFAM::Rewind(void) + { + gzrewind(Zfile); + } // end of Rewind + +/* ------------------------------------------------------------------- */ + +/***********************************************************************/ +/* Constructors. */ +/***********************************************************************/ +ZBKFAM::ZBKFAM(PDOSDEF tdp) : GZFAM(tdp) + { + Blocked = true; + Block = tdp->GetBlock(); + Last = tdp->GetLast(); + Nrec = tdp->GetElemt(); + CurLine = NULL; + NxtLine = NULL; + Closing = false; + BlkPos = tdp->GetTo_Pos(); + } // end of ZBKFAM standard constructor + +ZBKFAM::ZBKFAM(PZBKFAM txfp) : GZFAM(txfp) + { + CurLine = txfp->CurLine; + NxtLine = txfp->NxtLine; + Closing = txfp->Closing; + } // end of ZBKFAM copy constructor + +/***********************************************************************/ +/* Use BlockTest to reduce the table estimated size. */ +/***********************************************************************/ +int ZBKFAM::MaxBlkSize(PGLOBAL g, int) + { + int rc = RC_OK, savcur = CurBlk; + int size; + + // Roughly estimate the table size as the sum of blocks + // that can contain good rows + for (size = 0, CurBlk = 0; CurBlk < Block; CurBlk++) + if ((rc = Tdbp->TestBlock(g)) == RC_OK) + size += (CurBlk == Block - 1) ? Last : Nrec; + else if (rc == RC_EF) + break; + + CurBlk = savcur; + return size; + } // end of MaxBlkSize + +/***********************************************************************/ +/* ZBK Cardinality: returns table cardinality in number of rows. */ +/* This function can be called with a null argument to test the */ +/* availability of Cardinality implementation (1 yes, 0 no). */ +/***********************************************************************/ +int ZBKFAM::Cardinality(PGLOBAL g) + { + return (g) ? (int)((Block - 1) * Nrec + Last) : 1; + } // end of Cardinality + +/***********************************************************************/ +/* Allocate the line buffer. For mode Delete a bigger buffer has to */ +/* be allocated because is it also used to move lines into the file. */ +/***********************************************************************/ +bool ZBKFAM::AllocateBuffer(PGLOBAL g) + { + Buflen = Nrec * (Lrecl + 2); + CurLine = To_Buf = (char*)PlugSubAlloc(g, NULL, Buflen); + + if (Tdbp->GetMode() == MODE_INSERT) { + // Set values so Block and Last can be recalculated + if (Last == Nrec) { + CurBlk = Block; + Rbuf = Nrec; // To be used by WriteDB + } else { + // The last block must be completed + CurBlk = Block - 1; + Rbuf = Nrec - Last; // To be used by WriteDB + } // endif Last + + } // endif Insert + + return false; + } // end of AllocateBuffer + +/***********************************************************************/ +/* GetRowID: return the RowID of last read record. */ +/***********************************************************************/ +int ZBKFAM::GetRowID(void) + { + return CurNum + Nrec * CurBlk + 1; + } // end of GetRowID + +/***********************************************************************/ +/* GetPos: return the position of last read record. */ +/***********************************************************************/ +int ZBKFAM::GetPos(void) + { + return CurNum + Nrec * CurBlk; // Computed file index + } // end of GetPos + +/***********************************************************************/ +/* Record file position in case of UPDATE or DELETE. */ +/* Not used yet for fixed tables. */ +/***********************************************************************/ +bool ZBKFAM::RecordPos(PGLOBAL /*g*/) + { +//strcpy(g->Message, "RecordPos not implemented for gz blocked tables"); +//return true; + return RC_OK; + } // end of RecordPos + +/***********************************************************************/ +/* Skip one record in file. */ +/***********************************************************************/ +int ZBKFAM::SkipRecord(PGLOBAL /*g*/, bool) + { +//strcpy(g->Message, "SkipRecord not implemented for gz blocked tables"); +//return RC_FX; + return RC_OK; + } // end of SkipRecord + +/***********************************************************************/ +/* ReadBuffer: Read one line from a compressed text file. */ +/***********************************************************************/ +int ZBKFAM::ReadBuffer(PGLOBAL g) + { + int n, skip, rc = RC_OK; + + /*********************************************************************/ + /* Sequential reading when Placed is not true. */ + /*********************************************************************/ + if (++CurNum < Rbuf) { + CurLine = NxtLine; + + // Get the position of the next line in the buffer + while (*NxtLine++ != '\n') ; + + // Set caller line buffer + n = NxtLine - CurLine - Ending; + memcpy(Tdbp->GetLine(), CurLine, n); + Tdbp->GetLine()[n] = '\0'; + return RC_OK; + } else if (Rbuf < Nrec && CurBlk != -1) + return RC_EF; + + /*********************************************************************/ + /* New block. */ + /*********************************************************************/ + CurNum = 0; + skip = 0; + + next: + if (++CurBlk >= Block) + return RC_EF; + + /*********************************************************************/ + /* Before using the new block, check whether block optimization */ + /* can be done, as well as for join as for local filtering. */ + /*********************************************************************/ + switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + skip++; + goto next; + } // endswitch rc + + if (skip) + // Skip blocks rejected by block optimization + for (int i = CurBlk - skip; i < CurBlk; i++) { + BlkLen = BlkPos[i + 1] - BlkPos[i]; + + if (gzseek(Zfile, (z_off_t)BlkLen, SEEK_CUR) < 0) + return Zerror(g); + + } // endfor i + + BlkLen = BlkPos[CurBlk + 1] - BlkPos[CurBlk]; + + if (!(n = gzread(Zfile, To_Buf, BlkLen))) { + rc = RC_EF; + } else if (n > 0) { + // Get the position of the current line + CurLine = To_Buf; + + // Now get the position of the next line + for (NxtLine = CurLine; *NxtLine++ != '\n';) ; + + // Set caller line buffer + n = NxtLine - CurLine - Ending; + memcpy(Tdbp->GetLine(), CurLine, n); + Tdbp->GetLine()[n] = '\0'; + Rbuf = (CurBlk == Block - 1) ? Last : Nrec; + IsRead = true; + rc = RC_OK; + num_read++; + } else + rc = Zerror(g); + + return rc; + } // end of ReadBuffer + +/***********************************************************************/ +/* WriteDB: Data Base write routine for ZDOS access method. */ +/* Update is not possible without using a temporary file (NIY). */ +/***********************************************************************/ +int ZBKFAM::WriteBuffer(PGLOBAL g) + { + /*********************************************************************/ + /* Prepare the write buffer. */ + /*********************************************************************/ + if (!Closing) + strcat(strcpy(CurLine, Tdbp->GetLine()), CrLf); + + /*********************************************************************/ + /* In Insert mode, blocs are added sequentialy to the file end. */ + /* Note: Update mode is not handled for gz files. */ + /*********************************************************************/ + if (++CurNum == Rbuf) { + /*******************************************************************/ + /* New block, start the writing process. */ + /*******************************************************************/ + BlkLen = CurLine + strlen(CurLine) - To_Buf; + + if (gzwrite(Zfile, To_Buf, BlkLen) != BlkLen || + gzflush(Zfile, Z_FULL_FLUSH)) { + Closing = true; + return Zerror(g); + } // endif gzwrite + + Rbuf = Nrec; + CurBlk++; + CurNum = 0; + CurLine = To_Buf; + } else + CurLine += strlen(CurLine); + + return RC_OK; + } // end of WriteBuffer + +/***********************************************************************/ +/* Data Base delete line routine for ZBK access method. */ +/* Implemented only for total deletion of the table, which is done */ +/* by opening the file in mode "wb". */ +/***********************************************************************/ +int ZBKFAM::DeleteRecords(PGLOBAL g, int irc) + { + if (irc == RC_EF) { + LPCSTR name = Tdbp->GetName(); + PDOSDEF defp = (PDOSDEF)Tdbp->GetDef(); + + defp->SetBlock(0); + defp->SetLast(Nrec); + + if (!defp->SetIntCatInfo("Blocks", 0) || + !defp->SetIntCatInfo("Last", 0)) { + sprintf(g->Message, MSG(UPDATE_ERROR), "Header"); + return RC_FX; + } else + return RC_OK; + + } else + return irc; + + } // end of DeleteRecords + +/***********************************************************************/ +/* Data Base close routine for ZBK access method. */ +/***********************************************************************/ +void ZBKFAM::CloseTableFile(PGLOBAL g, bool) + { + int rc = RC_OK; + + if (Tdbp->GetMode() == MODE_INSERT) { + LPCSTR name = Tdbp->GetName(); + PDOSDEF defp = (PDOSDEF)Tdbp->GetDef(); + + if (CurNum && !Closing) { + // Some more inserted lines remain to be written + Last = (Nrec - Rbuf) + CurNum; + Block = CurBlk + 1; + Rbuf = CurNum--; + Closing = true; + rc = WriteBuffer(g); + } else if (Rbuf == Nrec) { + Last = Nrec; + Block = CurBlk; + } // endif CurNum + + if (rc != RC_FX) { + defp->SetBlock(Block); + defp->SetLast(Last); + defp->SetIntCatInfo("Blocks", Block); + defp->SetIntCatInfo("Last", Last); + } // endif + + gzclose(Zfile); + } else if (Tdbp->GetMode() == MODE_DELETE) { + rc = DeleteRecords(g, RC_EF); + gzclose(Zfile); + } else + rc = gzclose(Zfile); + + if (trace) + htrc("GZ CloseDB: closing %s rc=%d\n", To_File, rc); + + Zfile = NULL; // So we can know whether table is open +//To_Fb->Count = 0; // Avoid double closing by PlugCloseAll + } // end of CloseTableFile + +/***********************************************************************/ +/* Rewind routine for ZBK access method. */ +/***********************************************************************/ +void ZBKFAM::Rewind(void) + { + gzrewind(Zfile); + CurBlk = -1; + CurNum = Rbuf; + } // end of Rewind + +/* ------------------------------------------------------------------- */ + +/***********************************************************************/ +/* Constructors. */ +/***********************************************************************/ +ZIXFAM::ZIXFAM(PDOSDEF tdp) : ZBKFAM(tdp) + { +//Block = tdp->GetBlock(); +//Last = tdp->GetLast(); + Nrec = (tdp->GetElemt()) ? tdp->GetElemt() : DOS_BUFF_LEN; + Blksize = Nrec * Lrecl; + } // end of ZIXFAM standard constructor + +/***********************************************************************/ +/* ZIX Cardinality: returns table cardinality in number of rows. */ +/* This function can be called with a null argument to test the */ +/* availability of Cardinality implementation (1 yes, 0 no). */ +/***********************************************************************/ +int ZIXFAM::Cardinality(PGLOBAL g) + { + if (Last) + return (g) ? (int)((Block - 1) * Nrec + Last) : 1; + else // Last and Block not defined, cannot do it yet + return 0; + + } // end of Cardinality + +/***********************************************************************/ +/* Allocate the line buffer. For mode Delete a bigger buffer has to */ +/* be allocated because is it also used to move lines into the file. */ +/***********************************************************************/ +bool ZIXFAM::AllocateBuffer(PGLOBAL g) + { + Buflen = Blksize; + To_Buf = (char*)PlugSubAlloc(g, NULL, Buflen); + + if (Tdbp->GetMode() == MODE_INSERT) { + /*******************************************************************/ + /* For Insert the buffer must be prepared. */ + /*******************************************************************/ + memset(To_Buf, ' ', Buflen); + + if (Tdbp->GetFtype() < 2) + // if not binary, the file is physically a text file + for (int len = Lrecl; len <= Buflen; len += Lrecl) { +#if defined(__WIN__) + To_Buf[len - 2] = '\r'; +#endif // __WIN__ + To_Buf[len - 1] = '\n'; + } // endfor len + + // Set values so Block and Last can be recalculated + if (Last == Nrec) { + CurBlk = Block; + Rbuf = Nrec; // To be used by WriteDB + } else { + // The last block must be completed + CurBlk = Block - 1; + Rbuf = Nrec - Last; // To be used by WriteDB + } // endif Last + + } // endif Insert + + return false; + } // end of AllocateBuffer + +/***********************************************************************/ +/* ReadBuffer: Read one line from a compressed text file. */ +/***********************************************************************/ +int ZIXFAM::ReadBuffer(PGLOBAL g) + { + int n, rc = RC_OK; + + /*********************************************************************/ + /* Sequential reading when Placed is not true. */ + /*********************************************************************/ + if (++CurNum < Rbuf) { + Tdbp->IncLine(Lrecl); // Used by DOSCOL functions + return RC_OK; + } else if (Rbuf < Nrec && CurBlk != -1) + return RC_EF; + + /*********************************************************************/ + /* New block. */ + /*********************************************************************/ + CurNum = 0; + Tdbp->SetLine(To_Buf); + + int skip = 0; + + next: + if (++CurBlk >= Block) + return RC_EF; + + /*********************************************************************/ + /* Before using the new block, check whether block optimization */ + /* can be done, as well as for join as for local filtering. */ + /*********************************************************************/ + switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + skip++; + goto next; + } // endswitch rc + + if (skip) + // Skip blocks rejected by block optimization + for (int i = 0; i < skip; i++) { + if (gzseek(Zfile, (z_off_t)Buflen, SEEK_CUR) < 0) + return Zerror(g); + + } // endfor i + + if (!(n = gzread(Zfile, To_Buf, Buflen))) { + rc = RC_EF; + } else if (n > 0) { + Rbuf = n / Lrecl; + IsRead = true; + rc = RC_OK; + num_read++; + } else + rc = Zerror(g); + + return rc; + } // end of ReadBuffer + +/***********************************************************************/ +/* WriteDB: Data Base write routine for ZDOS access method. */ +/* Update is not possible without using a temporary file (NIY). */ +/***********************************************************************/ +int ZIXFAM::WriteBuffer(PGLOBAL g) + { + /*********************************************************************/ + /* In Insert mode, blocs are added sequentialy to the file end. */ + /* Note: Update mode is not handled for gz files. */ + /*********************************************************************/ + if (++CurNum == Rbuf) { + /*******************************************************************/ + /* New block, start the writing process. */ + /*******************************************************************/ + BlkLen = Rbuf * Lrecl; + + if (gzwrite(Zfile, To_Buf, BlkLen) != BlkLen || + gzflush(Zfile, Z_FULL_FLUSH)) { + Closing = true; + return Zerror(g); + } // endif gzwrite + + Rbuf = Nrec; + CurBlk++; + CurNum = 0; + Tdbp->SetLine(To_Buf); + } else + Tdbp->IncLine(Lrecl); // Used by FIXCOL functions + + return RC_OK; + } // end of WriteBuffer + +/* --------------------------- Class ZLBFAM -------------------------- */ + +/***********************************************************************/ +/* Constructors. */ +/***********************************************************************/ +ZLBFAM::ZLBFAM(PDOSDEF tdp) : BLKFAM(tdp) + { + Zstream = NULL; + Zbuffer = NULL; + Zlenp = NULL; + Optimized = tdp->IsOptimized(); + } // end of ZLBFAM standard constructor + +ZLBFAM::ZLBFAM(PZLBFAM txfp) : BLKFAM(txfp) + { + Zstream = txfp->Zstream; + Zbuffer = txfp->Zbuffer; + Zlenp = txfp->Zlenp; + Optimized = txfp->Optimized; + } // end of ZLBFAM (dummy?) copy constructor + +/***********************************************************************/ +/* ZLB GetFileLength: returns an estimate of what would be the */ +/* uncompressed file size in number of bytes. */ +/***********************************************************************/ +int ZLBFAM::GetFileLength(PGLOBAL g) + { + int len = (Optimized) ? BlkPos[Block] : BLKFAM::GetFileLength(g); + + if (len > 0) + // Estimate size reduction to a max of 5 + len *= 5; + + return len; + } // end of GetFileLength + +/***********************************************************************/ +/* Allocate the line buffer. For mode Delete a bigger buffer has to */ +/* be allocated because is it also used to move lines into the file. */ +/***********************************************************************/ +bool ZLBFAM::AllocateBuffer(PGLOBAL g) + { + char *msg; + int n, zrc; + +#if 0 + if (!Optimized && Tdbp->NeedIndexing(g)) { + strcpy(g->Message, MSG(NOP_ZLIB_INDEX)); + return TRUE; + } // endif indexing +#endif // 0 + +#if defined(NOLIB) + if (!zlib && LoadZlib()) { + sprintf(g->Message, MSG(DLL_LOAD_ERROR), GetLastError(), "zlib.dll"); + return TRUE; + } // endif zlib +#endif + + BLKFAM::AllocateBuffer(g); +//Buflen = Nrec * (Lrecl + 2); +//Rbuf = Nrec; + + // Allocate the compressed buffer + n = Buflen + 16; // ????????????????????????????????? + Zlenp = (int*)PlugSubAlloc(g, NULL, n); + Zbuffer = (Byte*)(Zlenp + 1); + + // Allocate and initialize the Z stream + Zstream = (z_streamp)PlugSubAlloc(g, NULL, sizeof(z_stream)); + Zstream->zalloc = (alloc_func)0; + Zstream->zfree = (free_func)0; + Zstream->opaque = (voidpf)0; + Zstream->next_in = NULL; + Zstream->avail_in = 0; + + if (Tdbp->GetMode() == MODE_READ) { + msg = "inflateInit"; + zrc = inflateInit(Zstream); + } else { + msg = "deflateInit"; + zrc = deflateInit(Zstream, Z_DEFAULT_COMPRESSION); + } // endif Mode + + if (zrc != Z_OK) { + if (Zstream->msg) + sprintf(g->Message, "%s error: %s", msg, Zstream->msg); + else + sprintf(g->Message, "%s error: %d", msg, zrc); + + return TRUE; + } // endif zrc + + if (Tdbp->GetMode() == MODE_INSERT) { + // Write the file header block + if (Last == Nrec) { + CurBlk = Block; + CurNum = 0; + + if (!GetFileLength(g)) { + // Write the zlib header as an extra block + strcpy(To_Buf, "PlugDB"); + BlkLen = strlen("PlugDB") + 1; + + if (WriteCompressedBuffer(g)) + return TRUE; + + } // endif void file + + } else { + // In mode insert, if Last != Nrec, last block must be updated + CurBlk = Block - 1; + CurNum = Last; + + strcpy(g->Message, MSG(NO_PAR_BLK_INS)); + return TRUE; + } // endif Last + + } else { // MODE_READ + // First thing to do is to read the header block + void *rdbuf; + + if (Optimized) { + BlkLen = BlkPos[0]; + rdbuf = Zlenp; + } else { + // Get the stored length from the file itself + if (fread(Zlenp, sizeof(int), 1, Stream) != 1) + return FALSE; // Empty file + + BlkLen = *Zlenp; + rdbuf = Zbuffer; + } // endif Optimized + + switch (ReadCompressedBuffer(g, rdbuf)) { + case RC_EF: + return FALSE; + case RC_FX: +#if defined(UNIX) + sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(errno)); +#else + sprintf(g->Message, MSG(READ_ERROR), To_File, _strerror(NULL)); +#endif + case RC_NF: + return TRUE; + } // endswitch + + // Some old tables can have PlugDB in their header + if (strcmp(To_Buf, "PlugDB")) { + sprintf(g->Message, MSG(BAD_HEADER), Tdbp->GetFile(g)); + return TRUE; + } // endif strcmp + + } // endif Mode + + return FALSE; + } // end of AllocateBuffer + +/***********************************************************************/ +/* GetPos: return the position of last read record. */ +/***********************************************************************/ +int ZLBFAM::GetPos(void) + { + return (Optimized) ? (CurNum + Nrec * CurBlk) : Fpos; + } // end of GetPos + +/***********************************************************************/ +/* GetNextPos: should not be called for this class. */ +/***********************************************************************/ +int ZLBFAM::GetNextPos(void) + { + if (Optimized) { + assert(FALSE); + return 0; + } else + return ftell(Stream); + + } // end of GetNextPos + +/***********************************************************************/ +/* SetPos: Replace the table at the specified position. */ +/***********************************************************************/ +bool ZLBFAM::SetPos(PGLOBAL g, int pos __attribute__((unused))) + { + sprintf(g->Message, MSG(NO_SETPOS_YET), "GZ"); + return true; +#if 0 // All this must be checked + if (pos < 0) { + strcpy(g->Message, MSG(INV_REC_POS)); + return true; + } // endif recpos + + CurBlk = pos / Nrec; + CurNum = pos % Nrec; +#if defined(_DEBUG) + num_eq[(CurBlk == OldBlk) ? 1 : 0]++; +#endif + + // Indicate the table position was externally set + Placed = true; + return false; +#endif // 0 + } // end of SetPos + +/***********************************************************************/ +/* ReadBuffer: Read one line for a text file. */ +/***********************************************************************/ +int ZLBFAM::ReadBuffer(PGLOBAL g) + { + int n; + void *rdbuf; + + /*********************************************************************/ + /* Sequential reading when Placed is not true. */ + /*********************************************************************/ + if (Placed) { + Placed = FALSE; + } else if (++CurNum < Rbuf) { + CurLine = NxtLine; + + // Get the position of the next line in the buffer + if (Tdbp->GetFtype() == RECFM_VAR) + while (*NxtLine++ != '\n') ; + else + NxtLine += Lrecl; + + // Set caller line buffer + n = NxtLine - CurLine - ((Tdbp->GetFtype() == RECFM_BIN) ? 0 : Ending); + memcpy(Tdbp->GetLine(), CurLine, n); + Tdbp->GetLine()[n] = '\0'; + return RC_OK; + } else if (Rbuf < Nrec && CurBlk != -1) { + CurNum--; // To have a correct Last value when optimizing + return RC_EF; + } else { + /*******************************************************************/ + /* New block. */ + /*******************************************************************/ + CurNum = 0; + + next: + if (++CurBlk >= Block) + return RC_EF; + + /*******************************************************************/ + /* Before reading a new block, check whether block optimization */ + /* can be done, as well as for join as for local filtering. */ + /*******************************************************************/ + if (Optimized) switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + goto next; + } // endswitch rc + + } // endif's + + if (OldBlk == CurBlk) + goto ok; // Block is already there + + if (Optimized) { + // Store the position of next block + Fpos = BlkPos[CurBlk]; + + // fseek is required only in non sequential reading + if (CurBlk != OldBlk + 1) + if (fseek(Stream, Fpos, SEEK_SET)) { + sprintf(g->Message, MSG(FSETPOS_ERROR), Fpos); + return RC_FX; + } // endif fseek + + // Calculate the length of block to read + BlkLen = BlkPos[CurBlk + 1] - Fpos; + rdbuf = Zlenp; + } else { // !Optimized + if (CurBlk != OldBlk + 1) { + strcpy(g->Message, MSG(INV_RAND_ACC)); + return RC_FX; + } else + Fpos = ftell(Stream); // Used when optimizing + + // Get the stored length from the file itself + if (fread(Zlenp, sizeof(int), 1, Stream) != 1) { + if (feof(Stream)) + return RC_EF; + + goto err; + } // endif fread + + BlkLen = *Zlenp; + rdbuf = Zbuffer; + } // endif Optimized + + // Read the next block + switch (ReadCompressedBuffer(g, rdbuf)) { + case RC_FX: goto err; + case RC_NF: return RC_FX; + case RC_EF: return RC_EF; + default: Rbuf = (CurBlk == Block - 1) ? Last : Nrec; + } // endswitch ReadCompressedBuffer + + ok: + if (Tdbp->GetFtype() == RECFM_VAR) { + int i; + + // Get the position of the current line + for (i = 0, CurLine = To_Buf; i < CurNum; i++) + while (*CurLine++ != '\n') ; // What about Unix ??? + + // Now get the position of the next line + for (NxtLine = CurLine; *NxtLine++ != '\n';) ; + + // Set caller line buffer + n = NxtLine - CurLine - Ending; + } else { + CurLine = To_Buf + CurNum * Lrecl; + NxtLine = CurLine + Lrecl; + n = Lrecl - ((Tdbp->GetFtype() == RECFM_BIN) ? 0 : Ending); + } // endif Ftype + + memcpy(Tdbp->GetLine(), CurLine, n); + Tdbp->GetLine()[n] = '\0'; + + OldBlk = CurBlk; // Last block actually read + IsRead = TRUE; // Is read indeed + return RC_OK; + + err: +#if defined(UNIX) + sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(errno)); +#else + sprintf(g->Message, MSG(READ_ERROR), To_File, _strerror(NULL)); +#endif + return RC_FX; + } // end of ReadBuffer + +/***********************************************************************/ +/* Read and decompress a block from the stream. */ +/***********************************************************************/ +int ZLBFAM::ReadCompressedBuffer(PGLOBAL g, void *rdbuf) + { + if (fread(rdbuf, 1, (size_t)BlkLen, Stream) == (unsigned)BlkLen) { + int zrc; + + num_read++; + + if (Optimized && BlkLen != signed(*Zlenp + sizeof(int))) { + sprintf(g->Message, MSG(BAD_BLK_SIZE), CurBlk + 1); + return RC_NF; + } // endif BlkLen + + // HERE WE MUST INFLATE THE BLOCK + Zstream->next_in = Zbuffer; + Zstream->avail_in = (uInt)(*Zlenp); + Zstream->next_out = (Byte*)To_Buf; + Zstream->avail_out = Buflen; + zrc = inflate(Zstream, Z_SYNC_FLUSH); + + if (zrc != Z_OK) { + if (Zstream->msg) + sprintf(g->Message, MSG(FUNC_ERR_S), "inflate", Zstream->msg); + else + sprintf(g->Message, MSG(FUNCTION_ERROR), "inflate", (int)zrc); + + return RC_NF; + } // endif zrc + + } else if (feof(Stream)) { + return RC_EF; + } else + return RC_FX; + + return RC_OK; + } // end of ReadCompressedBuffer + +/***********************************************************************/ +/* WriteBuffer: File write routine for DOS access method. */ +/* Update is directly written back into the file, */ +/* with this (fast) method, record size cannot change. */ +/***********************************************************************/ +int ZLBFAM::WriteBuffer(PGLOBAL g) + { + assert (Tdbp->GetMode() == MODE_INSERT); + + /*********************************************************************/ + /* Prepare the write buffer. */ + /*********************************************************************/ + if (!Closing) { + if (Tdbp->GetFtype() == RECFM_BIN) + memcpy(CurLine, Tdbp->GetLine(), Lrecl); + else + strcat(strcpy(CurLine, Tdbp->GetLine()), CrLf); + +#if defined(_DEBUG) + if (Tdbp->GetFtype() == RECFM_FIX && + (signed)strlen(CurLine) != Lrecl + (signed)strlen(CrLf)) { + strcpy(g->Message, MSG(BAD_LINE_LEN)); + Closing = TRUE; + return RC_FX; + } // endif Lrecl +#endif // _DEBUG + } // endif Closing + + /*********************************************************************/ + /* In Insert mode, blocs are added sequentialy to the file end. */ + /*********************************************************************/ + if (++CurNum != Rbuf) { + if (Tdbp->GetFtype() == RECFM_VAR) + CurLine += strlen(CurLine); + else + CurLine += Lrecl; + + return RC_OK; // We write only full blocks + } // endif CurNum + + // HERE WE MUST DEFLATE THE BLOCK + if (Tdbp->GetFtype() == RECFM_VAR) + NxtLine = CurLine + strlen(CurLine); + else + NxtLine = CurLine + Lrecl; + + BlkLen = NxtLine - To_Buf; + + if (WriteCompressedBuffer(g)) { + Closing = TRUE; // To tell CloseDB about a Write error + return RC_FX; + } // endif WriteCompressedBuffer + + CurBlk++; + CurNum = 0; + CurLine = To_Buf; + return RC_OK; + } // end of WriteBuffer + +/***********************************************************************/ +/* Compress the buffer and write the deflated output to stream. */ +/***********************************************************************/ +bool ZLBFAM::WriteCompressedBuffer(PGLOBAL g) + { + int zrc; + + Zstream->next_in = (Byte*)To_Buf; + Zstream->avail_in = (uInt)BlkLen; + Zstream->next_out = Zbuffer; + Zstream->avail_out = Buflen + 16; + Zstream->total_out = 0; + zrc = deflate(Zstream, Z_FULL_FLUSH); + + if (zrc != Z_OK) { + if (Zstream->msg) + sprintf(g->Message, MSG(FUNC_ERR_S), "deflate", Zstream->msg); + else + sprintf(g->Message, MSG(FUNCTION_ERROR), "deflate", (int)zrc); + + return TRUE; + } else + *Zlenp = Zstream->total_out; + + // Now start the writing process. + BlkLen = *Zlenp + sizeof(int); + + if (fwrite(Zlenp, 1, BlkLen, Stream) != (size_t)BlkLen) { + sprintf(g->Message, MSG(FWRITE_ERROR), strerror(errno)); + return TRUE; + } // endif size + + return FALSE; + } // end of WriteCompressedBuffer + +/***********************************************************************/ +/* Table file close routine for DOS access method. */ +/***********************************************************************/ +void ZLBFAM::CloseTableFile(PGLOBAL g, bool) + { + int rc = RC_OK; + + if (Tdbp->GetMode() == MODE_INSERT) { + LPCSTR name = Tdbp->GetName(); + PDOSDEF defp = (PDOSDEF)Tdbp->GetDef(); + + // Closing is True if last Write was in error + if (CurNum && !Closing) { + // Some more inserted lines remain to be written + Last = (Nrec - Rbuf) + CurNum; + Block = CurBlk + 1; + Rbuf = CurNum--; + Closing = TRUE; + rc = WriteBuffer(g); + } else if (Rbuf == Nrec) { + Last = Nrec; + Block = CurBlk; + } // endif CurNum + + if (rc != RC_FX) { + defp->SetBlock(Block); + defp->SetLast(Last); + defp->SetIntCatInfo("Blocks", Block); + defp->SetIntCatInfo("Last", Last); + } // endif + + fclose(Stream); + } else + rc = fclose(Stream); + + if (trace) + htrc("ZLB CloseTableFile: closing %s mode=%d rc=%d\n", + To_File, Tdbp->GetMode(), rc); + + Stream = NULL; // So we can know whether table is open + To_Fb->Count = 0; // Avoid double closing by PlugCloseAll + + if (Tdbp->GetMode() == MODE_READ) + rc = inflateEnd(Zstream); + else + rc = deflateEnd(Zstream); + + } // end of CloseTableFile + +/***********************************************************************/ +/* Rewind routine for ZLIB access method. */ +/***********************************************************************/ +void ZLBFAM::Rewind(void) + { + // We must be positioned after the header block + if (CurBlk >= 0) { // Nothing to do if no block read yet + if (!Optimized) { // If optimized, fseek will be done in ReadBuffer + size_t st; + + rewind(Stream); + + if (!(st = fread(Zlenp, sizeof(int), 1, Stream)) && trace) + htrc("fread error %d in Rewind", errno); + + fseek(Stream, *Zlenp + sizeof(int), SEEK_SET); + OldBlk = -1; + } // endif Optimized + + CurBlk = -1; + CurNum = Rbuf; + } // endif CurBlk + +//OldBlk = -1; +//Rbuf = 0; commented out in case we reuse last read block + } // end of Rewind + +/* ------------------------ End of GzFam ---------------------------- */ diff --git a/storage/connect/filamgz.h b/storage/connect/filamgz.h new file mode 100644 index 00000000000..d667fdddcc2 --- /dev/null +++ b/storage/connect/filamgz.h @@ -0,0 +1,170 @@ +/*************** FilAmGz H Declares Source Code File (.H) **************/ +/* Name: FILAMGZ.H Version 1.3 */ +/* */ +/* (C) Copyright to the author Olivier BERTRAND 2005-2016 */ +/* */ +/* This file contains the GZIP access method classes declares. */ +/***********************************************************************/ +#ifndef __FILAMGZ_H +#define __FILAMGZ_H + +#include "zlib.h" + +typedef class GZFAM *PGZFAM; +typedef class ZBKFAM *PZBKFAM; +typedef class ZIXFAM *PZIXFAM; +typedef class ZLBFAM *PZLBFAM; + +/***********************************************************************/ +/* This is the access method class declaration for not optimized */ +/* variable record length files compressed using the gzip library */ +/* functions. File is accessed record by record (row). */ +/***********************************************************************/ +class DllExport GZFAM : public TXTFAM { +// friend class DOSCOL; + public: + // Constructor + GZFAM(PDOSDEF tdp) : TXTFAM(tdp) {Zfile = NULL; Zpos = 0;} + GZFAM(PGZFAM txfp); + + // Implementation + virtual AMT GetAmType(void) {return TYPE_AM_GZ;} + virtual int GetPos(void); + virtual int GetNextPos(void); + virtual PTXF Duplicate(PGLOBAL g) + {return (PTXF)new(g) GZFAM(this);} + + // Methods + virtual void Reset(void); + virtual int GetFileLength(PGLOBAL g); + virtual int Cardinality(PGLOBAL g) {return (g) ? -1 : 0;} + virtual int MaxBlkSize(PGLOBAL g, int s) {return s;} + virtual bool AllocateBuffer(PGLOBAL g); + virtual int GetRowID(void); + virtual bool RecordPos(PGLOBAL g); + virtual bool SetPos(PGLOBAL g, int recpos); + virtual int SkipRecord(PGLOBAL g, bool header); + virtual bool OpenTableFile(PGLOBAL g); + virtual int ReadBuffer(PGLOBAL g); + virtual int WriteBuffer(PGLOBAL g); + virtual int DeleteRecords(PGLOBAL g, int irc); + virtual void CloseTableFile(PGLOBAL g, bool abort); + virtual void Rewind(void); + + protected: + int Zerror(PGLOBAL g); // GZ error function + + // Members + gzFile Zfile; // Points to GZ file structure + z_off_t Zpos; // Uncompressed file position + }; // end of class GZFAM + +/***********************************************************************/ +/* This is the access method class declaration for optimized variable */ +/* record length files compressed using the gzip library functions. */ +/* The File is accessed by block (requires an opt file). */ +/***********************************************************************/ +class DllExport ZBKFAM : public GZFAM { + public: + // Constructor + ZBKFAM(PDOSDEF tdp); + ZBKFAM(PZBKFAM txfp); + + // Implementation + virtual int GetPos(void); + virtual int GetNextPos(void) {return 0;} + virtual PTXF Duplicate(PGLOBAL g) + {return (PTXF)new(g) ZBKFAM(this);} + + // Methods + virtual int Cardinality(PGLOBAL g); + virtual int MaxBlkSize(PGLOBAL g, int s); + virtual bool AllocateBuffer(PGLOBAL g); + virtual int GetRowID(void); + virtual bool RecordPos(PGLOBAL g); + virtual int SkipRecord(PGLOBAL g, bool header); + virtual int ReadBuffer(PGLOBAL g); + virtual int WriteBuffer(PGLOBAL g); + virtual int DeleteRecords(PGLOBAL g, int irc); + virtual void CloseTableFile(PGLOBAL g, bool abort); + virtual void Rewind(void); + + protected: + // Members + char *CurLine; // Position of current line in buffer + char *NxtLine; // Position of Next line in buffer + bool Closing; // True when closing on Insert + }; // end of class ZBKFAM + +/***********************************************************************/ +/* This is the access method class declaration for fixed record */ +/* length files compressed using the gzip library functions. */ +/* The file is always accessed by block. */ +/***********************************************************************/ +class DllExport ZIXFAM : public ZBKFAM { + public: + // Constructor + ZIXFAM(PDOSDEF tdp); + ZIXFAM(PZIXFAM txfp) : ZBKFAM(txfp) {} + + // Implementation + virtual int GetNextPos(void) {return 0;} + virtual PTXF Duplicate(PGLOBAL g) + {return (PTXF)new(g) ZIXFAM(this);} + + // Methods + virtual int Cardinality(PGLOBAL g); + virtual bool AllocateBuffer(PGLOBAL g); + virtual int ReadBuffer(PGLOBAL g); + virtual int WriteBuffer(PGLOBAL g); + + protected: + // No additional Members + }; // end of class ZIXFAM + +/***********************************************************************/ +/* This is the DOS/UNIX Access Method class declaration for PlugDB */ +/* fixed/variable files compressed using the zlib library functions. */ +/* Physically these are written and read using the same technique */ +/* than blocked variable files, only the contain of each block is */ +/* compressed using the deflate zlib function. The purpose of this */ +/* specific format is to have a fast mechanism for direct access of */ +/* records so blocked optimization is fast and direct access (joins) */ +/* is allowed. Note that the block length is written ahead of each */ +/* block to enable reading when optimization file is not available. */ +/***********************************************************************/ +class DllExport ZLBFAM : public BLKFAM { + public: + // Constructor + ZLBFAM(PDOSDEF tdp); + ZLBFAM(PZLBFAM txfp); + + // Implementation + virtual AMT GetAmType(void) {return TYPE_AM_ZLIB;} + virtual int GetPos(void); + virtual int GetNextPos(void); + virtual PTXF Duplicate(PGLOBAL g) + {return (PTXF)new(g) ZLBFAM(this);} + inline void SetOptimized(bool b) {Optimized = b;} + + // Methods + virtual int GetFileLength(PGLOBAL g); + virtual bool SetPos(PGLOBAL g, int recpos); + virtual bool AllocateBuffer(PGLOBAL g); + virtual int ReadBuffer(PGLOBAL g); + virtual int WriteBuffer(PGLOBAL g); + virtual void CloseTableFile(PGLOBAL g, bool abort); + virtual void Rewind(void); + + protected: + bool WriteCompressedBuffer(PGLOBAL g); + int ReadCompressedBuffer(PGLOBAL g, void *rdbuf); + + // Members + z_streamp Zstream; // Compression/decompression stream + Byte *Zbuffer; // Compressed block buffer + int *Zlenp; // Pointer to block length + bool Optimized; // true when opt file is available + }; // end of class ZLBFAM + +#endif // __FILAMGZ_H diff --git a/storage/connect/filamzip.cpp b/storage/connect/filamzip.cpp index d9834e56dcd..65013e170e4 100644 --- a/storage/connect/filamzip.cpp +++ b/storage/connect/filamzip.cpp @@ -1,35 +1,29 @@ /*********** File AM Zip C++ Program Source Code File (.CPP) ***********/ /* PROGRAM NAME: FILAMZIP */ /* ------------- */ -/* Version 1.5 */ +/* Version 1.0 */ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 2005-2015 */ +/* (C) Copyright to the author Olivier BERTRAND 2016 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ -/* This program are the ZLIB compressed files classes. */ +/* This program are the ZIP file access method classes. */ /* */ /***********************************************************************/ /***********************************************************************/ -/* Include relevant MariaDB header file. */ +/* Include relevant sections of the System header files. */ /***********************************************************************/ #include "my_global.h" -#if defined(__WIN__) -#include <io.h> -#include <fcntl.h> -#if defined(__BORLANDC__) -#define __MFC_COMPAT__ // To define min/max as macro -#endif -//#include <windows.h> -#else // !__WIN__ +#if !defined(__WIN__) #if defined(UNIX) #include <errno.h> -#else // !UNIX +#include <unistd.h> +#else // !UNIX #include <io.h> -#endif +#endif // !UNIX #include <fcntl.h> #endif // !__WIN__ @@ -37,1390 +31,592 @@ /* Include application header files: */ /* global.h is header containing all global declarations. */ /* plgdbsem.h is header containing the DB application declarations. */ -/* tabdos.h is header containing the TABDOS class declarations. */ /***********************************************************************/ #include "global.h" #include "plgdbsem.h" -//#include "catalog.h" -//#include "reldef.h" -//#include "xobject.h" -//#include "kindex.h" -#include "filamtxt.h" -#include "tabdos.h" -#if defined(UNIX) #include "osutil.h" -#endif - -/***********************************************************************/ -/* This define prepares ZLIB function declarations. */ -/***********************************************************************/ -//#define ZLIB_DLL - +#include "filamtxt.h" +#include "tabfmt.h" +//#include "tabzip.h" #include "filamzip.h" -/***********************************************************************/ -/* DB static variables. */ -/***********************************************************************/ -extern int num_read, num_there, num_eq[]; // Statistics - -/* ------------------------------------------------------------------- */ +/* -------------------------- class ZIPUTIL -------------------------- */ /***********************************************************************/ -/* Implementation of the ZIPFAM class. */ +/* Constructors. */ /***********************************************************************/ -ZIPFAM::ZIPFAM(PZIPFAM txfp) : TXTFAM(txfp) - { - Zfile = txfp->Zfile; - Zpos = txfp->Zpos; - } // end of ZIPFAM copy constructor +ZIPUTIL::ZIPUTIL(PSZ tgt, bool mul) +{ + zipfile = NULL; + target = tgt; + fp = NULL; + memory = NULL; + size = 0; + entryopen = false; + multiple = mul; + memset(fn, 0, sizeof(fn)); + + // Init the case mapping table. +#if defined(__WIN__) + for (int i = 0; i < 256; ++i) mapCaseTable[i] = toupper(i); +#else + for (int i = 0; i < 256; ++i) mapCaseTable[i] = i; +#endif +} // end of ZIPUTIL standard constructor + +#if 0 +ZIPUTIL::ZIPUTIL(PZIPUTIL zutp) +{ + zipfile = zutp->zipfile; + target = zutp->target; + fp = zutp->fp; + finfo = zutp->finfo; + entryopen = zutp->entryopen; + multiple = zutp->multiple; + for (int i = 0; i < 256; ++i) mapCaseTable[i] = zutp->mapCaseTable[i]; +} // end of ZIPUTIL copy constructor +#endif // 0 /***********************************************************************/ -/* Zerror: Error function for gz calls. */ -/* gzerror returns the error message for the last error which occurred*/ -/* on the given compressed file. errnum is set to zlib error number. */ -/* If an error occurred in the file system and not in the compression */ -/* library, errnum is set to Z_ERRNO and the application may consult */ -/* errno to get the exact error code. */ +/* This code is the copyright property of Alessandro Felice Cantatore. */ +/* http://xoomer.virgilio.it/acantato/dev/wildcard/wildmatch.html */ /***********************************************************************/ -int ZIPFAM::Zerror(PGLOBAL g) - { - int errnum; +bool ZIPUTIL::WildMatch(PSZ pat, PSZ str) { + PSZ s, p; + bool star = FALSE; - strcpy(g->Message, gzerror(Zfile, &errnum)); +loopStart: + for (s = str, p = pat; *s; ++s, ++p) { + switch (*p) { + case '?': + if (*s == '.') goto starCheck; + break; + case '*': + star = TRUE; + str = s, pat = p; + if (!*++pat) return TRUE; + goto loopStart; + default: + if (mapCaseTable[*s] != mapCaseTable[*p]) + goto starCheck; + break; + } /* endswitch */ + } /* endfor */ + if (*p == '*') ++p; + return (!*p); - if (errnum == Z_ERRNO) -#if defined(__WIN__) - sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(NULL)); -#else // !__WIN__ - sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(errno)); -#endif // !__WIN__ - - return (errnum == Z_STREAM_END) ? RC_EF : RC_FX; - } // end of Zerror +starCheck: + if (!star) return FALSE; + str++; + goto loopStart; +} // end of WildMatch /***********************************************************************/ -/* Reset: reset position values at the beginning of file. */ +/* open a zip file. */ +/* param: filename path and the filename of the zip file to open. */ +/* return: true if open, false otherwise. */ /***********************************************************************/ -void ZIPFAM::Reset(void) - { - TXTFAM::Reset(); -//gzrewind(Zfile); // Useful ????? - Zpos = 0; - } // end of Reset +bool ZIPUTIL::open(PGLOBAL g, char *filename) +{ + if (!zipfile && !(zipfile = unzOpen64(filename))) + sprintf(g->Message, "Zipfile open error on %s", filename); + + return (zipfile == NULL); +} // end of open /***********************************************************************/ -/* ZIP GetFileLength: returns an estimate of what would be the */ -/* uncompressed file size in number of bytes. */ +/* Close the zip file. */ /***********************************************************************/ -int ZIPFAM::GetFileLength(PGLOBAL g) - { - int len = TXTFAM::GetFileLength(g); +void ZIPUTIL::close() +{ + if (zipfile) { + closeEntry(); + unzClose(zipfile); + zipfile = NULL; + } // endif zipfile - if (len > 0) - // Estimate size reduction to a max of 6 - len *= 6; - - return len; - } // end of GetFileLength +} // end of close /***********************************************************************/ -/* ZIP Access Method opening routine. */ +/* Find next entry matching target pattern. */ /***********************************************************************/ -bool ZIPFAM::OpenTableFile(PGLOBAL g) - { - char opmode[4], filename[_MAX_PATH]; - MODE mode = Tdbp->GetMode(); - - switch (mode) { - case MODE_READ: - strcpy(opmode, "r"); - break; - case MODE_UPDATE: - /*****************************************************************/ - /* Updating ZIP files not implemented yet. */ - /*****************************************************************/ - strcpy(g->Message, MSG(UPD_ZIP_NOT_IMP)); - return true; - case MODE_DELETE: - if (!Tdbp->GetNext()) { - // Store the number of deleted lines - DelRows = Cardinality(g); - - // This will erase the entire file - strcpy(opmode, "w"); -// Block = 0; // For ZBKFAM -// Last = Nrec; // For ZBKFAM - Tdbp->ResetSize(); - } else { - sprintf(g->Message, MSG(NO_PART_DEL), "ZIP"); - return true; - } // endif filter - - break; - case MODE_INSERT: - strcpy(opmode, "a+"); - break; - default: - sprintf(g->Message, MSG(BAD_OPEN_MODE), mode); - return true; - } // endswitch Mode - - /*********************************************************************/ - /* Open according to logical input/output mode required. */ - /* Use specific zlib functions. */ - /* Treat files as binary. */ - /*********************************************************************/ - strcat(opmode, "b"); - Zfile = gzopen(PlugSetPath(filename, To_File, Tdbp->GetPath()), opmode); - - if (Zfile == NULL) { - sprintf(g->Message, MSG(GZOPEN_ERROR), - opmode, (int)errno, filename); - strcat(strcat(g->Message, ": "), strerror(errno)); - return (mode == MODE_READ && errno == ENOENT) - ? PushWarning(g, Tdbp) : true; - } // endif Zfile - - /*********************************************************************/ - /* Something to be done here. >>>>>>>> NOT DONE <<<<<<<< */ - /*********************************************************************/ -//To_Fb = dbuserp->Openlist; // Keep track of File block - - /*********************************************************************/ - /* Allocate the line buffer. */ - /*********************************************************************/ - return AllocateBuffer(g); - } // end of OpenTableFile +int ZIPUTIL::findEntry(PGLOBAL g, bool next) +{ + int rc; -/***********************************************************************/ -/* Allocate the line buffer. For mode Delete a bigger buffer has to */ -/* be allocated because is it also used to move lines into the file. */ -/***********************************************************************/ -bool ZIPFAM::AllocateBuffer(PGLOBAL g) - { - MODE mode = Tdbp->GetMode(); + do { + if (next) { + rc = unzGoToNextFile(zipfile); - Buflen = Lrecl + 2; // Lrecl does not include CRLF -//Buflen *= ((Mode == MODE_DELETE) ? DOS_BUFF_LEN : 1); NIY + if (rc == UNZ_END_OF_LIST_OF_FILE) + return RC_EF; + else if (rc != UNZ_OK) { + sprintf(g->Message, "unzGoToNextFile rc = ", rc); + return RC_FX; + } // endif rc - if (trace) - htrc("SubAllocating a buffer of %d bytes\n", Buflen); + } // endif next - To_Buf = (char*)PlugSubAlloc(g, NULL, Buflen); + if (target && *target) { + rc = unzGetCurrentFileInfo(zipfile, NULL, fn, sizeof(fn), + NULL, 0, NULL, 0); + if (rc == UNZ_OK) { + if (WildMatch(target, fn)) + return RC_OK; - if (mode == MODE_INSERT) { - /*******************************************************************/ - /* For Insert buffer must be prepared. */ - /*******************************************************************/ - memset(To_Buf, ' ', Buflen); - To_Buf[Buflen - 2] = '\n'; - To_Buf[Buflen - 1] = '\0'; - } // endif Insert + } else { + sprintf(g->Message, "GetCurrentFileInfo rc = %d", rc); + return RC_FX; + } // endif rc - return false; - } // end of AllocateBuffer + } else + return RC_OK; -/***********************************************************************/ -/* GetRowID: return the RowID of last read record. */ -/***********************************************************************/ -int ZIPFAM::GetRowID(void) - { - return Rows; - } // end of GetRowID + next = true; + } while (true); -/***********************************************************************/ -/* GetPos: return the position of last read record. */ -/***********************************************************************/ -int ZIPFAM::GetPos(void) - { - return (int)Zpos; - } // end of GetPos + strcpy(g->Message, "FindNext logical error"); + return RC_FX; +} // end of FindEntry -/***********************************************************************/ -/* GetNextPos: return the position of next record. */ -/***********************************************************************/ -int ZIPFAM::GetNextPos(void) - { - return gztell(Zfile); - } // end of GetNextPos /***********************************************************************/ -/* SetPos: Replace the table at the specified position. */ +/* Get the next used entry. */ /***********************************************************************/ -bool ZIPFAM::SetPos(PGLOBAL g, int pos __attribute__((unused))) - { - sprintf(g->Message, MSG(NO_SETPOS_YET), "ZIP"); - return true; -#if 0 - Fpos = pos; +int ZIPUTIL::nextEntry(PGLOBAL g) +{ + if (multiple) { + int rc; - if (fseek(Stream, Fpos, SEEK_SET)) { - sprintf(g->Message, MSG(FSETPOS_ERROR), Fpos); - return true; - } // endif + closeEntry(); - Placed = true; - return false; -#endif // 0 - } // end of SetPos + if ((rc = findEntry(g, true)) != RC_OK) + return rc; -/***********************************************************************/ -/* Record file position in case of UPDATE or DELETE. */ -/***********************************************************************/ -bool ZIPFAM::RecordPos(PGLOBAL) - { - Zpos = gztell(Zfile); - return false; - } // end of RecordPos + if (openEntry(g)) + return RC_FX; -/***********************************************************************/ -/* Skip one record in file. */ -/***********************************************************************/ -int ZIPFAM::SkipRecord(PGLOBAL g, bool header) - { - // Skip this record - if (gzeof(Zfile)) - return RC_EF; - else if (gzgets(Zfile, To_Buf, Buflen) == Z_NULL) - return Zerror(g); + return RC_OK; + } else + return RC_EF; - if (header) - RecordPos(g); +} // end of nextEntry - return RC_OK; - } // end of SkipRecord /***********************************************************************/ -/* ReadBuffer: Read one line from a compressed text file. */ +/* OpenTableFile: Open a DOS/UNIX table file from a ZIP file. */ /***********************************************************************/ -int ZIPFAM::ReadBuffer(PGLOBAL g) - { - char *p; - int rc; - - if (!Zfile) - return RC_EF; - - if (!Placed) { - /*******************************************************************/ - /* Record file position in case of UPDATE or DELETE. */ - /*******************************************************************/ - next: - if (RecordPos(g)) - return RC_FX; - - CurBlk = Rows++; // Update RowID - - /*******************************************************************/ - /* Check whether optimization on ROWID */ - /* can be done, as well as for join as for local filtering. */ - /*******************************************************************/ - switch (Tdbp->TestBlock(g)) { - case RC_EF: - return RC_EF; - case RC_NF: - // Skip this record - if ((rc = SkipRecord(g, FALSE)) != RC_OK) - return rc; - - goto next; - } // endswitch rc - - } else - Placed = false; - - if (gzeof(Zfile)) { - rc = RC_EF; - } else if (gzgets(Zfile, To_Buf, Buflen) != Z_NULL) { - p = To_Buf + strlen(To_Buf) - 1; - - if (*p == '\n') - *p = '\0'; // Eliminate ending new-line character - - if (*(--p) == '\r') - *p = '\0'; // Eliminate eventuel carriage return - - strcpy(Tdbp->GetLine(), To_Buf); - IsRead = true; - rc = RC_OK; - num_read++; - } else - rc = Zerror(g); - - if (trace > 1) - htrc(" Read: '%s' rc=%d\n", To_Buf, rc); - - return rc; - } // end of ReadBuffer +bool ZIPUTIL::OpenTable(PGLOBAL g, MODE mode, char *fn) +{ + /*********************************************************************/ + /* The file will be decompressed into virtual memory. */ + /*********************************************************************/ + if (mode == MODE_READ || mode == MODE_ANY) { + bool b = open(g, fn); -/***********************************************************************/ -/* WriteDB: Data Base write routine for ZDOS access method. */ -/* Update is not possible without using a temporary file (NIY). */ -/***********************************************************************/ -int ZIPFAM::WriteBuffer(PGLOBAL g) - { - /*********************************************************************/ - /* Prepare the write buffer. */ - /*********************************************************************/ - strcat(strcpy(To_Buf, Tdbp->GetLine()), CrLf); - - /*********************************************************************/ - /* Now start the writing process. */ - /*********************************************************************/ - if (gzputs(Zfile, To_Buf) < 0) - return Zerror(g); - - return RC_OK; - } // end of WriteBuffer + if (!b) { + int rc; -/***********************************************************************/ -/* Data Base delete line routine for ZDOS access method. (NIY) */ -/***********************************************************************/ -int ZIPFAM::DeleteRecords(PGLOBAL g, int) - { - strcpy(g->Message, MSG(NO_ZIP_DELETE)); - return RC_FX; - } // end of DeleteRecords + if (target && *target) { + if (!multiple) { + rc = unzLocateFile(zipfile, target, 0); -/***********************************************************************/ -/* Data Base close routine for DOS access method. */ -/***********************************************************************/ -void ZIPFAM::CloseTableFile(PGLOBAL, bool) - { - int rc = gzclose(Zfile); + if (rc == UNZ_END_OF_LIST_OF_FILE) { + sprintf(g->Message, "Target file %s not in %s", target, fn); + return true; + } else if (rc != UNZ_OK) { + sprintf(g->Message, "unzLocateFile rc=%d", rc); + return true; + } // endif's rc - if (trace) - htrc("ZIP CloseDB: closing %s rc=%d\n", To_File, rc); + } else { + if ((rc = findEntry(g, false)) == RC_FX) + return true; + else if (rc == RC_EF) { + sprintf(g->Message, "No match of %s in %s", target, fn); + return true; + } // endif rc - Zfile = NULL; // So we can know whether table is open -//To_Fb->Count = 0; // Avoid double closing by PlugCloseAll - } // end of CloseTableFile + } // endif multiple -/***********************************************************************/ -/* Rewind routine for ZIP access method. */ -/***********************************************************************/ -void ZIPFAM::Rewind(void) - { - gzrewind(Zfile); - } // end of Rewind + } // endif target -/* ------------------------------------------------------------------- */ + if (openEntry(g)) + return true; -/***********************************************************************/ -/* Constructors. */ -/***********************************************************************/ -ZBKFAM::ZBKFAM(PDOSDEF tdp) : ZIPFAM(tdp) - { - Blocked = true; - Block = tdp->GetBlock(); - Last = tdp->GetLast(); - Nrec = tdp->GetElemt(); - CurLine = NULL; - NxtLine = NULL; - Closing = false; - BlkPos = tdp->GetTo_Pos(); - } // end of ZBKFAM standard constructor - -ZBKFAM::ZBKFAM(PZBKFAM txfp) : ZIPFAM(txfp) - { - CurLine = txfp->CurLine; - NxtLine = txfp->NxtLine; - Closing = txfp->Closing; - } // end of ZBKFAM copy constructor + if (size > 0) { + /*******************************************************************/ + /* Link a Fblock. This make possible to automatically close it */ + /* in case of error g->jump. */ + /*******************************************************************/ + PDBUSER dbuserp = (PDBUSER)g->Activityp->Aptr; -/***********************************************************************/ -/* Use BlockTest to reduce the table estimated size. */ -/***********************************************************************/ -int ZBKFAM::MaxBlkSize(PGLOBAL g, int) - { - int rc = RC_OK, savcur = CurBlk; - int size; - - // Roughly estimate the table size as the sum of blocks - // that can contain good rows - for (size = 0, CurBlk = 0; CurBlk < Block; CurBlk++) - if ((rc = Tdbp->TestBlock(g)) == RC_OK) - size += (CurBlk == Block - 1) ? Last : Nrec; - else if (rc == RC_EF) - break; - - CurBlk = savcur; - return size; - } // end of MaxBlkSize + fp = (PFBLOCK)PlugSubAlloc(g, NULL, sizeof(FBLOCK)); + fp->Type = TYPE_FB_ZIP; + fp->Fname = PlugDup(g, fn); + fp->Next = dbuserp->Openlist; + dbuserp->Openlist = fp; + fp->Count = 1; + fp->Length = size; + fp->Memory = memory; + fp->Mode = mode; + fp->File = this; + fp->Handle = NULL; + } // endif fp -/***********************************************************************/ -/* ZBK Cardinality: returns table cardinality in number of rows. */ -/* This function can be called with a null argument to test the */ -/* availability of Cardinality implementation (1 yes, 0 no). */ -/***********************************************************************/ -int ZBKFAM::Cardinality(PGLOBAL g) - { - return (g) ? (int)((Block - 1) * Nrec + Last) : 1; - } // end of Cardinality + } else + return true; -/***********************************************************************/ -/* Allocate the line buffer. For mode Delete a bigger buffer has to */ -/* be allocated because is it also used to move lines into the file. */ -/***********************************************************************/ -bool ZBKFAM::AllocateBuffer(PGLOBAL g) - { - Buflen = Nrec * (Lrecl + 2); - CurLine = To_Buf = (char*)PlugSubAlloc(g, NULL, Buflen); - - if (Tdbp->GetMode() == MODE_INSERT) { - // Set values so Block and Last can be recalculated - if (Last == Nrec) { - CurBlk = Block; - Rbuf = Nrec; // To be used by WriteDB - } else { - // The last block must be completed - CurBlk = Block - 1; - Rbuf = Nrec - Last; // To be used by WriteDB - } // endif Last - - } // endif Insert - - return false; - } // end of AllocateBuffer + } else { + strcpy(g->Message, "Only READ mode supported for ZIP files"); + return true; + } // endif mode -/***********************************************************************/ -/* GetRowID: return the RowID of last read record. */ -/***********************************************************************/ -int ZBKFAM::GetRowID(void) - { - return CurNum + Nrec * CurBlk + 1; - } // end of GetRowID + return false; +} // end of OpenTableFile /***********************************************************************/ -/* GetPos: return the position of last read record. */ +/* Open target in zip file. */ /***********************************************************************/ -int ZBKFAM::GetPos(void) - { - return CurNum + Nrec * CurBlk; // Computed file index - } // end of GetPos +bool ZIPUTIL::openEntry(PGLOBAL g) +{ + int rc; -/***********************************************************************/ -/* Record file position in case of UPDATE or DELETE. */ -/* Not used yet for fixed tables. */ -/***********************************************************************/ -bool ZBKFAM::RecordPos(PGLOBAL /*g*/) - { -//strcpy(g->Message, "RecordPos not implemented for zip blocked tables"); -//return true; - return RC_OK; - } // end of RecordPos + rc = unzGetCurrentFileInfo(zipfile, &finfo, fn, sizeof(fn), + NULL, 0, NULL, 0); -/***********************************************************************/ -/* Skip one record in file. */ -/***********************************************************************/ -int ZBKFAM::SkipRecord(PGLOBAL /*g*/, bool) - { -//strcpy(g->Message, "SkipRecord not implemented for zip blocked tables"); -//return RC_FX; - return RC_OK; - } // end of SkipRecord + if (rc != UNZ_OK) { + sprintf(g->Message, "unzGetCurrentFileInfo64 rc=%d", rc); + return true; + } else if ((rc = unzOpenCurrentFile(zipfile)) != UNZ_OK) { + sprintf(g->Message, "unzOpen fn=%s rc=%d", fn, rc); + return true; + } // endif rc -/***********************************************************************/ -/* ReadBuffer: Read one line from a compressed text file. */ -/***********************************************************************/ -int ZBKFAM::ReadBuffer(PGLOBAL g) - { - int n, skip, rc = RC_OK; - - /*********************************************************************/ - /* Sequential reading when Placed is not true. */ - /*********************************************************************/ - if (++CurNum < Rbuf) { - CurLine = NxtLine; - - // Get the position of the next line in the buffer - while (*NxtLine++ != '\n') ; - - // Set caller line buffer - n = NxtLine - CurLine - Ending; - memcpy(Tdbp->GetLine(), CurLine, n); - Tdbp->GetLine()[n] = '\0'; - return RC_OK; - } else if (Rbuf < Nrec && CurBlk != -1) - return RC_EF; - - /*********************************************************************/ - /* New block. */ - /*********************************************************************/ - CurNum = 0; - skip = 0; - - next: - if (++CurBlk >= Block) - return RC_EF; - - /*********************************************************************/ - /* Before using the new block, check whether block optimization */ - /* can be done, as well as for join as for local filtering. */ - /*********************************************************************/ - switch (Tdbp->TestBlock(g)) { - case RC_EF: - return RC_EF; - case RC_NF: - skip++; - goto next; - } // endswitch rc - - if (skip) - // Skip blocks rejected by block optimization - for (int i = CurBlk - skip; i < CurBlk; i++) { - BlkLen = BlkPos[i + 1] - BlkPos[i]; - - if (gzseek(Zfile, (z_off_t)BlkLen, SEEK_CUR) < 0) - return Zerror(g); - - } // endfor i - - BlkLen = BlkPos[CurBlk + 1] - BlkPos[CurBlk]; - - if (!(n = gzread(Zfile, To_Buf, BlkLen))) { - rc = RC_EF; - } else if (n > 0) { - // Get the position of the current line - CurLine = To_Buf; - - // Now get the position of the next line - for (NxtLine = CurLine; *NxtLine++ != '\n';) ; - - // Set caller line buffer - n = NxtLine - CurLine - Ending; - memcpy(Tdbp->GetLine(), CurLine, n); - Tdbp->GetLine()[n] = '\0'; - Rbuf = (CurBlk == Block - 1) ? Last : Nrec; - IsRead = true; - rc = RC_OK; - num_read++; - } else - rc = Zerror(g); - - return rc; - } // end of ReadBuffer + size = finfo.uncompressed_size; + memory = new char[size + 1]; -/***********************************************************************/ -/* WriteDB: Data Base write routine for ZDOS access method. */ -/* Update is not possible without using a temporary file (NIY). */ -/***********************************************************************/ -int ZBKFAM::WriteBuffer(PGLOBAL g) - { - /*********************************************************************/ - /* Prepare the write buffer. */ - /*********************************************************************/ - if (!Closing) - strcat(strcpy(CurLine, Tdbp->GetLine()), CrLf); - - /*********************************************************************/ - /* In Insert mode, blocs are added sequentialy to the file end. */ - /* Note: Update mode is not handled for zip files. */ - /*********************************************************************/ - if (++CurNum == Rbuf) { - /*******************************************************************/ - /* New block, start the writing process. */ - /*******************************************************************/ - BlkLen = CurLine + strlen(CurLine) - To_Buf; - - if (gzwrite(Zfile, To_Buf, BlkLen) != BlkLen || - gzflush(Zfile, Z_FULL_FLUSH)) { - Closing = true; - return Zerror(g); - } // endif gzwrite - - Rbuf = Nrec; - CurBlk++; - CurNum = 0; - CurLine = To_Buf; - } else - CurLine += strlen(CurLine); - - return RC_OK; - } // end of WriteBuffer + if ((rc = unzReadCurrentFile(zipfile, memory, size)) < 0) { + sprintf(g->Message, "unzReadCurrentFile rc = ", rc); + unzCloseCurrentFile(zipfile); + free(memory); + memory = NULL; + entryopen = false; + } else { + memory[size] = 0; // Required by some table types (XML) + entryopen = true; + } // endif rc + + if (trace) + htrc("Openning entry%s %s\n", fn, (entryopen) ? "oked" : "failed"); + + return !entryopen; +} // end of openEntry + +/***********************************************************************/ +/* Close the zip file. */ +/***********************************************************************/ +void ZIPUTIL::closeEntry() +{ + if (entryopen) { + unzCloseCurrentFile(zipfile); + entryopen = false; + } // endif entryopen + + if (memory) { + free(memory); + memory = NULL; + } // endif memory + +} // end of closeEntry + +/* -------------------------- class ZIPFAM --------------------------- */ /***********************************************************************/ -/* Data Base delete line routine for ZBK access method. */ -/* Implemented only for total deletion of the table, which is done */ -/* by opening the file in mode "wb". */ +/* Constructors. */ /***********************************************************************/ -int ZBKFAM::DeleteRecords(PGLOBAL g, int irc) - { - if (irc == RC_EF) { - LPCSTR name = Tdbp->GetName(); - PDOSDEF defp = (PDOSDEF)Tdbp->GetDef(); - - defp->SetBlock(0); - defp->SetLast(Nrec); +ZIPFAM::ZIPFAM(PDOSDEF tdp) : MAPFAM(tdp) +{ + zutp = NULL; + target = tdp->GetEntry(); + mul = tdp->GetMul(); +} // end of ZIPFAM standard constructor - if (!defp->SetIntCatInfo("Blocks", 0) || - !defp->SetIntCatInfo("Last", 0)) { - sprintf(g->Message, MSG(UPDATE_ERROR), "Header"); - return RC_FX; - } else - return RC_OK; +ZIPFAM::ZIPFAM(PZIPFAM txfp) : MAPFAM(txfp) +{ + zutp = txfp->zutp; + target = txfp->target; + mul = txfp->mul; +} // end of ZIPFAM copy constructor - } else - return irc; - - } // end of DeleteRecords +ZIPFAM::ZIPFAM(PDOSDEF tdp, PZPXFAM txfp) : MAPFAM(tdp) +{ + zutp = txfp->zutp; + target = txfp->target; + mul = txfp->mul; +} // end of ZIPFAM constructor used in ResetTableOpt /***********************************************************************/ -/* Data Base close routine for ZBK access method. */ +/* ZIP GetFileLength: returns file size in number of bytes. */ /***********************************************************************/ -void ZBKFAM::CloseTableFile(PGLOBAL g, bool) - { - int rc = RC_OK; - - if (Tdbp->GetMode() == MODE_INSERT) { - LPCSTR name = Tdbp->GetName(); - PDOSDEF defp = (PDOSDEF)Tdbp->GetDef(); - - if (CurNum && !Closing) { - // Some more inserted lines remain to be written - Last = (Nrec - Rbuf) + CurNum; - Block = CurBlk + 1; - Rbuf = CurNum--; - Closing = true; - rc = WriteBuffer(g); - } else if (Rbuf == Nrec) { - Last = Nrec; - Block = CurBlk; - } // endif CurNum - - if (rc != RC_FX) { - defp->SetBlock(Block); - defp->SetLast(Last); - defp->SetIntCatInfo("Blocks", Block); - defp->SetIntCatInfo("Last", Last); - } // endif - - gzclose(Zfile); - } else if (Tdbp->GetMode() == MODE_DELETE) { - rc = DeleteRecords(g, RC_EF); - gzclose(Zfile); - } else - rc = gzclose(Zfile); - - if (trace) - htrc("ZIP CloseDB: closing %s rc=%d\n", To_File, rc); - - Zfile = NULL; // So we can know whether table is open -//To_Fb->Count = 0; // Avoid double closing by PlugCloseAll - } // end of CloseTableFile +int ZIPFAM::GetFileLength(PGLOBAL g) +{ + int len = (zutp && zutp->entryopen) ? Top - Memory + : TXTFAM::GetFileLength(g) * 3; -/***********************************************************************/ -/* Rewind routine for ZBK access method. */ -/***********************************************************************/ -void ZBKFAM::Rewind(void) - { - gzrewind(Zfile); - CurBlk = -1; - CurNum = Rbuf; - } // end of Rewind + if (trace) + htrc("Zipped file length=%d\n", len); -/* ------------------------------------------------------------------- */ + return len; +} // end of GetFileLength /***********************************************************************/ -/* Constructors. */ +/* ZIP Cardinality: return the number of rows if possible. */ /***********************************************************************/ -ZIXFAM::ZIXFAM(PDOSDEF tdp) : ZBKFAM(tdp) - { -//Block = tdp->GetBlock(); -//Last = tdp->GetLast(); - Nrec = (tdp->GetElemt()) ? tdp->GetElemt() : DOS_BUFF_LEN; - Blksize = Nrec * Lrecl; - } // end of ZIXFAM standard constructor +int ZIPFAM::Cardinality(PGLOBAL g) +{ + if (!g) + return 1; -/***********************************************************************/ -/* ZIX Cardinality: returns table cardinality in number of rows. */ -/* This function can be called with a null argument to test the */ -/* availability of Cardinality implementation (1 yes, 0 no). */ -/***********************************************************************/ -int ZIXFAM::Cardinality(PGLOBAL g) - { - if (Last) - return (g) ? (int)((Block - 1) * Nrec + Last) : 1; - else // Last and Block not defined, cannot do it yet - return 0; + int card = -1; + int len = GetFileLength(g); - } // end of Cardinality + card = (len / (int)Lrecl) * 2; // Estimated ??? + return card; +} // end of Cardinality /***********************************************************************/ -/* Allocate the line buffer. For mode Delete a bigger buffer has to */ -/* be allocated because is it also used to move lines into the file. */ +/* OpenTableFile: Open a DOS/UNIX table file from a ZIP file. */ /***********************************************************************/ -bool ZIXFAM::AllocateBuffer(PGLOBAL g) - { - Buflen = Blksize; - To_Buf = (char*)PlugSubAlloc(g, NULL, Buflen); - - if (Tdbp->GetMode() == MODE_INSERT) { - /*******************************************************************/ - /* For Insert the buffer must be prepared. */ - /*******************************************************************/ - memset(To_Buf, ' ', Buflen); - - if (Tdbp->GetFtype() < 2) - // if not binary, the file is physically a text file - for (int len = Lrecl; len <= Buflen; len += Lrecl) { -#if defined(__WIN__) - To_Buf[len - 2] = '\r'; -#endif // __WIN__ - To_Buf[len - 1] = '\n'; - } // endfor len - - // Set values so Block and Last can be recalculated - if (Last == Nrec) { - CurBlk = Block; - Rbuf = Nrec; // To be used by WriteDB - } else { - // The last block must be completed - CurBlk = Block - 1; - Rbuf = Nrec - Last; // To be used by WriteDB - } // endif Last - - } // endif Insert - - return false; - } // end of AllocateBuffer +bool ZIPFAM::OpenTableFile(PGLOBAL g) +{ + char filename[_MAX_PATH]; + MODE mode = Tdbp->GetMode(); -/***********************************************************************/ -/* ReadBuffer: Read one line from a compressed text file. */ -/***********************************************************************/ -int ZIXFAM::ReadBuffer(PGLOBAL g) - { - int n, rc = RC_OK; - - /*********************************************************************/ - /* Sequential reading when Placed is not true. */ - /*********************************************************************/ - if (++CurNum < Rbuf) { - Tdbp->IncLine(Lrecl); // Used by DOSCOL functions - return RC_OK; - } else if (Rbuf < Nrec && CurBlk != -1) - return RC_EF; - - /*********************************************************************/ - /* New block. */ - /*********************************************************************/ - CurNum = 0; - Tdbp->SetLine(To_Buf); - - int skip = 0; - - next: - if (++CurBlk >= Block) - return RC_EF; - - /*********************************************************************/ - /* Before using the new block, check whether block optimization */ - /* can be done, as well as for join as for local filtering. */ - /*********************************************************************/ - switch (Tdbp->TestBlock(g)) { - case RC_EF: - return RC_EF; - case RC_NF: - skip++; - goto next; - } // endswitch rc - - if (skip) - // Skip blocks rejected by block optimization - for (int i = 0; i < skip; i++) { - if (gzseek(Zfile, (z_off_t)Buflen, SEEK_CUR) < 0) - return Zerror(g); - - } // endfor i - - if (!(n = gzread(Zfile, To_Buf, Buflen))) { - rc = RC_EF; - } else if (n > 0) { - Rbuf = n / Lrecl; - IsRead = true; - rc = RC_OK; - num_read++; - } else - rc = Zerror(g); - - return rc; - } // end of ReadBuffer + /*********************************************************************/ + /* Allocate the ZIP utility class. */ + /*********************************************************************/ + zutp = new(g) ZIPUTIL(target, mul); -/***********************************************************************/ -/* WriteDB: Data Base write routine for ZDOS access method. */ -/* Update is not possible without using a temporary file (NIY). */ -/***********************************************************************/ -int ZIXFAM::WriteBuffer(PGLOBAL g) - { - /*********************************************************************/ - /* In Insert mode, blocs are added sequentialy to the file end. */ - /* Note: Update mode is not handled for zip files. */ - /*********************************************************************/ - if (++CurNum == Rbuf) { - /*******************************************************************/ - /* New block, start the writing process. */ - /*******************************************************************/ - BlkLen = Rbuf * Lrecl; - - if (gzwrite(Zfile, To_Buf, BlkLen) != BlkLen || - gzflush(Zfile, Z_FULL_FLUSH)) { - Closing = true; - return Zerror(g); - } // endif gzwrite - - Rbuf = Nrec; - CurBlk++; - CurNum = 0; - Tdbp->SetLine(To_Buf); - } else - Tdbp->IncLine(Lrecl); // Used by FIXCOL functions - - return RC_OK; - } // end of WriteBuffer - -/* --------------------------- Class ZLBFAM -------------------------- */ + // We used the file name relative to recorded datapath + PlugSetPath(filename, To_File, Tdbp->GetPath()); -/***********************************************************************/ -/* Constructors. */ -/***********************************************************************/ -ZLBFAM::ZLBFAM(PDOSDEF tdp) : BLKFAM(tdp) - { - Zstream = NULL; - Zbuffer = NULL; - Zlenp = NULL; - Optimized = tdp->IsOptimized(); - } // end of ZLBFAM standard constructor - -ZLBFAM::ZLBFAM(PZLBFAM txfp) : BLKFAM(txfp) - { - Zstream = txfp->Zstream; - Zbuffer = txfp->Zbuffer; - Zlenp = txfp->Zlenp; - Optimized = txfp->Optimized; - } // end of ZLBFAM (dummy?) copy constructor + if (!zutp->OpenTable(g, mode, filename)) { + // The pseudo "buffer" is here the entire real buffer + Fpos = Mempos = Memory = zutp->memory; + Top = Memory + zutp->size; + To_Fb = zutp->fp; // Useful when closing + } else + return true; + + return false; + } // end of OpenTableFile /***********************************************************************/ -/* ZLB GetFileLength: returns an estimate of what would be the */ -/* uncompressed file size in number of bytes. */ +/* GetNext: go to next entry. */ /***********************************************************************/ -int ZLBFAM::GetFileLength(PGLOBAL g) - { - int len = (Optimized) ? BlkPos[Block] : BLKFAM::GetFileLength(g); +int ZIPFAM::GetNext(PGLOBAL g) +{ + int rc = zutp->nextEntry(g); - if (len > 0) - // Estimate size reduction to a max of 5 - len *= 5; + if (rc != RC_OK) + return rc; - return len; - } // end of GetFileLength + Mempos = Memory = zutp->memory; + Top = Memory + zutp->size; + return RC_OK; +} // end of GetNext +#if 0 /***********************************************************************/ -/* Allocate the line buffer. For mode Delete a bigger buffer has to */ -/* be allocated because is it also used to move lines into the file. */ +/* ReadBuffer: Read one line for a ZIP file. */ /***********************************************************************/ -bool ZLBFAM::AllocateBuffer(PGLOBAL g) - { - char *msg; - int n, zrc; +int ZIPFAM::ReadBuffer(PGLOBAL g) +{ + int rc, len; -#if 0 - if (!Optimized && Tdbp->NeedIndexing(g)) { - strcpy(g->Message, MSG(NOP_ZLIB_INDEX)); - return TRUE; - } // endif indexing -#endif // 0 + // Are we at the end of the memory + if (Mempos >= Top) { + if ((rc = zutp->nextEntry(g)) != RC_OK) + return rc; -#if defined(NOLIB) - if (!zlib && LoadZlib()) { - sprintf(g->Message, MSG(DLL_LOAD_ERROR), GetLastError(), "zlib.dll"); - return TRUE; - } // endif zlib -#endif + Mempos = Memory = zutp->memory; + Top = Memory + zutp->size; + } // endif Mempos - BLKFAM::AllocateBuffer(g); -//Buflen = Nrec * (Lrecl + 2); -//Rbuf = Nrec; - - // Allocate the compressed buffer - n = Buflen + 16; // ????????????????????????????????? - Zlenp = (int*)PlugSubAlloc(g, NULL, n); - Zbuffer = (Byte*)(Zlenp + 1); - - // Allocate and initialize the Z stream - Zstream = (z_streamp)PlugSubAlloc(g, NULL, sizeof(z_stream)); - Zstream->zalloc = (alloc_func)0; - Zstream->zfree = (free_func)0; - Zstream->opaque = (voidpf)0; - Zstream->next_in = NULL; - Zstream->avail_in = 0; - - if (Tdbp->GetMode() == MODE_READ) { - msg = "inflateInit"; - zrc = inflateInit(Zstream); - } else { - msg = "deflateInit"; - zrc = deflateInit(Zstream, Z_DEFAULT_COMPRESSION); - } // endif Mode - - if (zrc != Z_OK) { - if (Zstream->msg) - sprintf(g->Message, "%s error: %s", msg, Zstream->msg); - else - sprintf(g->Message, "%s error: %d", msg, zrc); - - return TRUE; - } // endif zrc - - if (Tdbp->GetMode() == MODE_INSERT) { - // Write the file header block - if (Last == Nrec) { - CurBlk = Block; - CurNum = 0; - - if (!GetFileLength(g)) { - // Write the zlib header as an extra block - strcpy(To_Buf, "PlugDB"); - BlkLen = strlen("PlugDB") + 1; - - if (WriteCompressedBuffer(g)) - return TRUE; - - } // endif void file - - } else { - // In mode insert, if Last != Nrec, last block must be updated - CurBlk = Block - 1; - CurNum = Last; - - strcpy(g->Message, MSG(NO_PAR_BLK_INS)); - return TRUE; - } // endif Last - - } else { // MODE_READ - // First thing to do is to read the header block - void *rdbuf; - - if (Optimized) { - BlkLen = BlkPos[0]; - rdbuf = Zlenp; - } else { - // Get the stored length from the file itself - if (fread(Zlenp, sizeof(int), 1, Stream) != 1) - return FALSE; // Empty file - - BlkLen = *Zlenp; - rdbuf = Zbuffer; - } // endif Optimized - - switch (ReadCompressedBuffer(g, rdbuf)) { - case RC_EF: - return FALSE; - case RC_FX: -#if defined(UNIX) - sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(errno)); +#if 0 + if (!Placed) { + /*******************************************************************/ + /* Record file position in case of UPDATE or DELETE. */ + /*******************************************************************/ + int rc; + + next: + Fpos = Mempos; + CurBlk = (int)Rows++; + + /*******************************************************************/ + /* Check whether optimization on ROWID */ + /* can be done, as well as for join as for local filtering. */ + /*******************************************************************/ + switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + // Skip this record + if ((rc = SkipRecord(g, false)) != RC_OK) + return rc; + + goto next; + } // endswitch rc + + } else + Placed = false; #else - sprintf(g->Message, MSG(READ_ERROR), To_File, _strerror(NULL)); + // Perhaps unuseful + Fpos = Mempos; + CurBlk = (int)Rows++; + Placed = false; #endif - case RC_NF: - return TRUE; - } // endswitch - // Some old tables can have PlugDB in their header - if (strcmp(To_Buf, "PlugDB")) { - sprintf(g->Message, MSG(BAD_HEADER), Tdbp->GetFile(g)); - return TRUE; - } // endif strcmp + // Immediately calculate next position (Used by DeleteDB) + while (*Mempos++ != '\n'); // What about Unix ??? - } // endif Mode + // Set caller line buffer + len = (Mempos - Fpos) - 1; - return FALSE; - } // end of AllocateBuffer + // Don't rely on ENDING setting + if (len > 0 && *(Mempos - 2) == '\r') + len--; // Line ends by CRLF -/***********************************************************************/ -/* GetPos: return the position of last read record. */ -/***********************************************************************/ -int ZLBFAM::GetPos(void) - { - return (Optimized) ? (CurNum + Nrec * CurBlk) : Fpos; - } // end of GetPos + memcpy(Tdbp->GetLine(), Fpos, len); + Tdbp->GetLine()[len] = '\0'; + return RC_OK; +} // end of ReadBuffer /***********************************************************************/ -/* GetNextPos: should not be called for this class. */ +/* Table file close routine for MAP access method. */ /***********************************************************************/ -int ZLBFAM::GetNextPos(void) - { - if (Optimized) { - assert(FALSE); - return 0; - } else - return ftell(Stream); +void ZIPFAM::CloseTableFile(PGLOBAL g, bool) +{ + close(); +} // end of CloseTableFile +#endif // 0 - } // end of GetNextPos +/* -------------------------- class ZPXFAM --------------------------- */ /***********************************************************************/ -/* SetPos: Replace the table at the specified position. */ +/* Constructors. */ /***********************************************************************/ -bool ZLBFAM::SetPos(PGLOBAL g, int pos __attribute__((unused))) - { - sprintf(g->Message, MSG(NO_SETPOS_YET), "ZIP"); - return true; -#if 0 // All this must be checked - if (pos < 0) { - strcpy(g->Message, MSG(INV_REC_POS)); - return true; - } // endif recpos - - CurBlk = pos / Nrec; - CurNum = pos % Nrec; -#if defined(_DEBUG) - num_eq[(CurBlk == OldBlk) ? 1 : 0]++; -#endif +ZPXFAM::ZPXFAM(PDOSDEF tdp) : MPXFAM(tdp) +{ + zutp = NULL; + target = tdp->GetEntry(); + mul = tdp->GetMul(); + //Lrecl = tdp->GetLrecl(); +} // end of ZPXFAM standard constructor - // Indicate the table position was externally set - Placed = true; - return false; -#endif // 0 - } // end of SetPos +ZPXFAM::ZPXFAM(PZPXFAM txfp) : MPXFAM(txfp) +{ + zutp = txfp->zutp; + target = txfp->target; + mul = txfp->mul; +//Lrecl = txfp->Lrecl; +} // end of ZPXFAM copy constructor /***********************************************************************/ -/* ReadBuffer: Read one line for a text file. */ +/* ZIP GetFileLength: returns file size in number of bytes. */ /***********************************************************************/ -int ZLBFAM::ReadBuffer(PGLOBAL g) - { - int n; - void *rdbuf; - - /*********************************************************************/ - /* Sequential reading when Placed is not true. */ - /*********************************************************************/ - if (Placed) { - Placed = FALSE; - } else if (++CurNum < Rbuf) { - CurLine = NxtLine; - - // Get the position of the next line in the buffer - if (Tdbp->GetFtype() == RECFM_VAR) - while (*NxtLine++ != '\n') ; - else - NxtLine += Lrecl; - - // Set caller line buffer - n = NxtLine - CurLine - ((Tdbp->GetFtype() == RECFM_BIN) ? 0 : Ending); - memcpy(Tdbp->GetLine(), CurLine, n); - Tdbp->GetLine()[n] = '\0'; - return RC_OK; - } else if (Rbuf < Nrec && CurBlk != -1) { - CurNum--; // To have a correct Last value when optimizing - return RC_EF; - } else { - /*******************************************************************/ - /* New block. */ - /*******************************************************************/ - CurNum = 0; - - next: - if (++CurBlk >= Block) - return RC_EF; - - /*******************************************************************/ - /* Before reading a new block, check whether block optimization */ - /* can be done, as well as for join as for local filtering. */ - /*******************************************************************/ - if (Optimized) switch (Tdbp->TestBlock(g)) { - case RC_EF: - return RC_EF; - case RC_NF: - goto next; - } // endswitch rc - - } // endif's - - if (OldBlk == CurBlk) - goto ok; // Block is already there - - if (Optimized) { - // Store the position of next block - Fpos = BlkPos[CurBlk]; - - // fseek is required only in non sequential reading - if (CurBlk != OldBlk + 1) - if (fseek(Stream, Fpos, SEEK_SET)) { - sprintf(g->Message, MSG(FSETPOS_ERROR), Fpos); - return RC_FX; - } // endif fseek - - // Calculate the length of block to read - BlkLen = BlkPos[CurBlk + 1] - Fpos; - rdbuf = Zlenp; - } else { // !Optimized - if (CurBlk != OldBlk + 1) { - strcpy(g->Message, MSG(INV_RAND_ACC)); - return RC_FX; - } else - Fpos = ftell(Stream); // Used when optimizing - - // Get the stored length from the file itself - if (fread(Zlenp, sizeof(int), 1, Stream) != 1) { - if (feof(Stream)) - return RC_EF; - - goto err; - } // endif fread - - BlkLen = *Zlenp; - rdbuf = Zbuffer; - } // endif Optimized - - // Read the next block - switch (ReadCompressedBuffer(g, rdbuf)) { - case RC_FX: goto err; - case RC_NF: return RC_FX; - case RC_EF: return RC_EF; - default: Rbuf = (CurBlk == Block - 1) ? Last : Nrec; - } // endswitch ReadCompressedBuffer - - ok: - if (Tdbp->GetFtype() == RECFM_VAR) { - int i; - - // Get the position of the current line - for (i = 0, CurLine = To_Buf; i < CurNum; i++) - while (*CurLine++ != '\n') ; // What about Unix ??? - - // Now get the position of the next line - for (NxtLine = CurLine; *NxtLine++ != '\n';) ; - - // Set caller line buffer - n = NxtLine - CurLine - Ending; - } else { - CurLine = To_Buf + CurNum * Lrecl; - NxtLine = CurLine + Lrecl; - n = Lrecl - ((Tdbp->GetFtype() == RECFM_BIN) ? 0 : Ending); - } // endif Ftype - - memcpy(Tdbp->GetLine(), CurLine, n); - Tdbp->GetLine()[n] = '\0'; - - OldBlk = CurBlk; // Last block actually read - IsRead = TRUE; // Is read indeed - return RC_OK; - - err: -#if defined(UNIX) - sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(errno)); -#else - sprintf(g->Message, MSG(READ_ERROR), To_File, _strerror(NULL)); -#endif - return RC_FX; - } // end of ReadBuffer +int ZPXFAM::GetFileLength(PGLOBAL g) +{ + int len; -/***********************************************************************/ -/* Read and decompress a block from the stream. */ -/***********************************************************************/ -int ZLBFAM::ReadCompressedBuffer(PGLOBAL g, void *rdbuf) - { - if (fread(rdbuf, 1, (size_t)BlkLen, Stream) == (unsigned)BlkLen) { - int zrc; - - num_read++; - - if (Optimized && BlkLen != signed(*Zlenp + sizeof(int))) { - sprintf(g->Message, MSG(BAD_BLK_SIZE), CurBlk + 1); - return RC_NF; - } // endif BlkLen - - // HERE WE MUST INFLATE THE BLOCK - Zstream->next_in = Zbuffer; - Zstream->avail_in = (uInt)(*Zlenp); - Zstream->next_out = (Byte*)To_Buf; - Zstream->avail_out = Buflen; - zrc = inflate(Zstream, Z_SYNC_FLUSH); - - if (zrc != Z_OK) { - if (Zstream->msg) - sprintf(g->Message, MSG(FUNC_ERR_S), "inflate", Zstream->msg); - else - sprintf(g->Message, MSG(FUNCTION_ERROR), "inflate", (int)zrc); - - return RC_NF; - } // endif zrc - - } else if (feof(Stream)) { - return RC_EF; - } else - return RC_FX; - - return RC_OK; - } // end of ReadCompressedBuffer + if (!zutp && OpenTableFile(g)) + return 0; -/***********************************************************************/ -/* WriteBuffer: File write routine for DOS access method. */ -/* Update is directly written back into the file, */ -/* with this (fast) method, record size cannot change. */ -/***********************************************************************/ -int ZLBFAM::WriteBuffer(PGLOBAL g) - { - assert (Tdbp->GetMode() == MODE_INSERT); - - /*********************************************************************/ - /* Prepare the write buffer. */ - /*********************************************************************/ - if (!Closing) { - if (Tdbp->GetFtype() == RECFM_BIN) - memcpy(CurLine, Tdbp->GetLine(), Lrecl); - else - strcat(strcpy(CurLine, Tdbp->GetLine()), CrLf); - -#if defined(_DEBUG) - if (Tdbp->GetFtype() == RECFM_FIX && - (signed)strlen(CurLine) != Lrecl + (signed)strlen(CrLf)) { - strcpy(g->Message, MSG(BAD_LINE_LEN)); - Closing = TRUE; - return RC_FX; - } // endif Lrecl -#endif // _DEBUG - } // endif Closing - - /*********************************************************************/ - /* In Insert mode, blocs are added sequentialy to the file end. */ - /*********************************************************************/ - if (++CurNum != Rbuf) { - if (Tdbp->GetFtype() == RECFM_VAR) - CurLine += strlen(CurLine); - else - CurLine += Lrecl; - - return RC_OK; // We write only full blocks - } // endif CurNum - - // HERE WE MUST DEFLATE THE BLOCK - if (Tdbp->GetFtype() == RECFM_VAR) - NxtLine = CurLine + strlen(CurLine); - else - NxtLine = CurLine + Lrecl; - - BlkLen = NxtLine - To_Buf; - - if (WriteCompressedBuffer(g)) { - Closing = TRUE; // To tell CloseDB about a Write error - return RC_FX; - } // endif WriteCompressedBuffer - - CurBlk++; - CurNum = 0; - CurLine = To_Buf; - return RC_OK; - } // end of WriteBuffer + if (zutp->entryopen) + len = zutp->size; + else + len = 0; + + return len; +} // end of GetFileLength /***********************************************************************/ -/* Compress the buffer and write the deflated output to stream. */ +/* ZIP Cardinality: return the number of rows if possible. */ /***********************************************************************/ -bool ZLBFAM::WriteCompressedBuffer(PGLOBAL g) - { - int zrc; - - Zstream->next_in = (Byte*)To_Buf; - Zstream->avail_in = (uInt)BlkLen; - Zstream->next_out = Zbuffer; - Zstream->avail_out = Buflen + 16; - Zstream->total_out = 0; - zrc = deflate(Zstream, Z_FULL_FLUSH); - - if (zrc != Z_OK) { - if (Zstream->msg) - sprintf(g->Message, MSG(FUNC_ERR_S), "deflate", Zstream->msg); - else - sprintf(g->Message, MSG(FUNCTION_ERROR), "deflate", (int)zrc); - - return TRUE; - } else - *Zlenp = Zstream->total_out; - - // Now start the writing process. - BlkLen = *Zlenp + sizeof(int); - - if (fwrite(Zlenp, 1, BlkLen, Stream) != (size_t)BlkLen) { - sprintf(g->Message, MSG(FWRITE_ERROR), strerror(errno)); - return TRUE; - } // endif size - - return FALSE; - } // end of WriteCompressedBuffer +int ZPXFAM::Cardinality(PGLOBAL g) +{ + if (!g) + return 1; + + int card = -1; + int len = GetFileLength(g); + + if (!(len % Lrecl)) + card = len / (int)Lrecl; // Fixed length file + else + sprintf(g->Message, MSG(NOT_FIXED_LEN), zutp->fn, len, Lrecl); + + // Set number of blocks for later use + Block = (card > 0) ? (card + Nrec - 1) / Nrec : 0; + return card; +} // end of Cardinality /***********************************************************************/ -/* Table file close routine for DOS access method. */ +/* OpenTableFile: Open a DOS/UNIX table file from a ZIP file. */ /***********************************************************************/ -void ZLBFAM::CloseTableFile(PGLOBAL g, bool) - { - int rc = RC_OK; - - if (Tdbp->GetMode() == MODE_INSERT) { - LPCSTR name = Tdbp->GetName(); - PDOSDEF defp = (PDOSDEF)Tdbp->GetDef(); - - // Closing is True if last Write was in error - if (CurNum && !Closing) { - // Some more inserted lines remain to be written - Last = (Nrec - Rbuf) + CurNum; - Block = CurBlk + 1; - Rbuf = CurNum--; - Closing = TRUE; - rc = WriteBuffer(g); - } else if (Rbuf == Nrec) { - Last = Nrec; - Block = CurBlk; - } // endif CurNum - - if (rc != RC_FX) { - defp->SetBlock(Block); - defp->SetLast(Last); - defp->SetIntCatInfo("Blocks", Block); - defp->SetIntCatInfo("Last", Last); - } // endif - - fclose(Stream); - } else - rc = fclose(Stream); - - if (trace) - htrc("ZLB CloseTableFile: closing %s mode=%d rc=%d\n", - To_File, Tdbp->GetMode(), rc); - - Stream = NULL; // So we can know whether table is open - To_Fb->Count = 0; // Avoid double closing by PlugCloseAll - - if (Tdbp->GetMode() == MODE_READ) - rc = inflateEnd(Zstream); - else - rc = deflateEnd(Zstream); - - } // end of CloseTableFile +bool ZPXFAM::OpenTableFile(PGLOBAL g) +{ + // May have been already opened in GetFileLength + if (!zutp || !zutp->zipfile) { + char filename[_MAX_PATH]; + MODE mode = Tdbp->GetMode(); + + /*********************************************************************/ + /* Allocate the ZIP utility class. */ + /*********************************************************************/ + if (!zutp) + zutp = new(g)ZIPUTIL(target, mul); + + // We used the file name relative to recorded datapath + PlugSetPath(filename, To_File, Tdbp->GetPath()); + + if (!zutp->OpenTable(g, mode, filename)) { + // The pseudo "buffer" is here the entire real buffer + Memory = zutp->memory; + Fpos = Mempos = Memory + Headlen; + Top = Memory + zutp->size; + To_Fb = zutp->fp; // Useful when closing + } else + return true; + + } else + Reset(); + + return false; +} // end of OpenTableFile /***********************************************************************/ -/* Rewind routine for ZLIB access method. */ +/* GetNext: go to next entry. */ /***********************************************************************/ -void ZLBFAM::Rewind(void) - { - // We must be positioned after the header block - if (CurBlk >= 0) { // Nothing to do if no block read yet - if (!Optimized) { // If optimized, fseek will be done in ReadBuffer - size_t st; - - rewind(Stream); +int ZPXFAM::GetNext(PGLOBAL g) +{ + int rc = zutp->nextEntry(g); - if (!(st = fread(Zlenp, sizeof(int), 1, Stream)) && trace) - htrc("fread error %d in Rewind", errno); + if (rc != RC_OK) + return rc; - fseek(Stream, *Zlenp + sizeof(int), SEEK_SET); - OldBlk = -1; - } // endif Optimized + int len = zutp->size; - CurBlk = -1; - CurNum = Rbuf; - } // endif CurBlk + if (len % Lrecl) { + sprintf(g->Message, MSG(NOT_FIXED_LEN), zutp->fn, len, Lrecl); + return RC_FX; + } // endif size -//OldBlk = -1; -//Rbuf = 0; commented out in case we reuse last read block - } // end of Rewind + Memory = zutp->memory; + Top = Memory + len; + Rewind(); + return RC_OK; +} // end of GetNext -/* ------------------------ End of ZipFam ---------------------------- */ diff --git a/storage/connect/filamzip.h b/storage/connect/filamzip.h index edb8b5db323..9312fb2f70e 100644 --- a/storage/connect/filamzip.h +++ b/storage/connect/filamzip.h @@ -1,170 +1,117 @@ -/************** FilAmZip H Declares Source Code File (.H) **************/ -/* Name: FILAMZIP.H Version 1.2 */ +/************** filamzip H Declares Source Code File (.H) **************/ +/* Name: filamzip.h Version 1.0 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 2005-2014 */ +/* (C) Copyright to the author Olivier BERTRAND 2016 */ /* */ -/* This file contains the GZIP access method classes declares. */ +/* This file contains the ZIP file access method classes declares. */ /***********************************************************************/ #ifndef __FILAMZIP_H #define __FILAMZIP_H -#include "zlib.h" +#include "block.h" +#include "filamap.h" +#include "unzip.h" -typedef class ZIPFAM *PZIPFAM; -typedef class ZBKFAM *PZBKFAM; -typedef class ZIXFAM *PZIXFAM; -typedef class ZLBFAM *PZLBFAM; +#define DLLEXPORT extern "C" -/***********************************************************************/ -/* This is the access method class declaration for not optimized */ -/* variable record length files compressed using the gzip library */ -/* functions. File is accessed record by record (row). */ -/***********************************************************************/ -class DllExport ZIPFAM : public TXTFAM { -// friend class DOSCOL; - public: - // Constructor - ZIPFAM(PDOSDEF tdp) : TXTFAM(tdp) {Zfile = NULL; Zpos = 0;} - ZIPFAM(PZIPFAM txfp); - - // Implementation - virtual AMT GetAmType(void) {return TYPE_AM_ZIP;} - virtual int GetPos(void); - virtual int GetNextPos(void); - virtual PTXF Duplicate(PGLOBAL g) - {return (PTXF)new(g) ZIPFAM(this);} - - // Methods - virtual void Reset(void); - virtual int GetFileLength(PGLOBAL g); - virtual int Cardinality(PGLOBAL g) {return (g) ? -1 : 0;} - virtual int MaxBlkSize(PGLOBAL g, int s) {return s;} - virtual bool AllocateBuffer(PGLOBAL g); - virtual int GetRowID(void); - virtual bool RecordPos(PGLOBAL g); - virtual bool SetPos(PGLOBAL g, int recpos); - virtual int SkipRecord(PGLOBAL g, bool header); - virtual bool OpenTableFile(PGLOBAL g); - virtual int ReadBuffer(PGLOBAL g); - virtual int WriteBuffer(PGLOBAL g); - virtual int DeleteRecords(PGLOBAL g, int irc); - virtual void CloseTableFile(PGLOBAL g, bool abort); - virtual void Rewind(void); - - protected: - int Zerror(PGLOBAL g); // GZ error function - - // Members - gzFile Zfile; // Points to GZ file structure - z_off_t Zpos; // Uncompressed file position - }; // end of class ZIPFAM +typedef class ZIPFAM *PZIPFAM; +typedef class ZPXFAM *PZPXFAM; /***********************************************************************/ -/* This is the access method class declaration for optimized variable */ -/* record length files compressed using the gzip library functions. */ -/* The File is accessed by block (requires an opt file). */ +/* This is the ZIP utility fonctions class. */ /***********************************************************************/ -class DllExport ZBKFAM : public ZIPFAM { - public: - // Constructor - ZBKFAM(PDOSDEF tdp); - ZBKFAM(PZBKFAM txfp); - - // Implementation - virtual int GetPos(void); - virtual int GetNextPos(void) {return 0;} - virtual PTXF Duplicate(PGLOBAL g) - {return (PTXF)new(g) ZBKFAM(this);} - - // Methods - virtual int Cardinality(PGLOBAL g); - virtual int MaxBlkSize(PGLOBAL g, int s); - virtual bool AllocateBuffer(PGLOBAL g); - virtual int GetRowID(void); - virtual bool RecordPos(PGLOBAL g); - virtual int SkipRecord(PGLOBAL g, bool header); - virtual int ReadBuffer(PGLOBAL g); - virtual int WriteBuffer(PGLOBAL g); - virtual int DeleteRecords(PGLOBAL g, int irc); - virtual void CloseTableFile(PGLOBAL g, bool abort); - virtual void Rewind(void); - - protected: - // Members - char *CurLine; // Position of current line in buffer - char *NxtLine; // Position of Next line in buffer - bool Closing; // True when closing on Insert - }; // end of class ZBKFAM +class DllExport ZIPUTIL : public BLOCK { +public: + // Constructor + ZIPUTIL(PSZ tgt, bool mul); +//ZIPUTIL(ZIPUTIL *zutp); + + // Implementation +//PTXF Duplicate(PGLOBAL g) { return (PTXF) new(g)ZIPFAM(this); } + + // Methods + virtual bool OpenTable(PGLOBAL g, MODE mode, char *fn); + bool open(PGLOBAL g, char *fn); + bool openEntry(PGLOBAL g); + void close(void); + void closeEntry(void); + bool WildMatch(PSZ pat, PSZ str); + int findEntry(PGLOBAL g, bool next); + int nextEntry(PGLOBAL g); + + // Members + unzFile zipfile; // The ZIP container file + PSZ target; // The target file name + unz_file_info finfo; // The current file info + PFBLOCK fp; + char *memory; + uint size; + int multiple; // Multiple targets + bool entryopen; // True when open current entry + char fn[FILENAME_MAX]; // The current entry file name + char mapCaseTable[256]; +}; // end of ZIPFAM /***********************************************************************/ -/* This is the access method class declaration for fixed record */ -/* length files compressed using the gzip library functions. */ -/* The file is always accessed by block. */ +/* This is the ZIP file access method. */ /***********************************************************************/ -class DllExport ZIXFAM : public ZBKFAM { - public: - // Constructor - ZIXFAM(PDOSDEF tdp); - ZIXFAM(PZIXFAM txfp) : ZBKFAM(txfp) {} - - // Implementation - virtual int GetNextPos(void) {return 0;} - virtual PTXF Duplicate(PGLOBAL g) - {return (PTXF)new(g) ZIXFAM(this);} - - // Methods - virtual int Cardinality(PGLOBAL g); - virtual bool AllocateBuffer(PGLOBAL g); - virtual int ReadBuffer(PGLOBAL g); - virtual int WriteBuffer(PGLOBAL g); - - protected: - // No additional Members - }; // end of class ZIXFAM +class DllExport ZIPFAM : public MAPFAM { + friend class ZPXFAM; +public: + // Constructors + ZIPFAM(PDOSDEF tdp); + ZIPFAM(PZIPFAM txfp); + ZIPFAM(PDOSDEF tdp, PZPXFAM txfp); + + // Implementation + virtual AMT GetAmType(void) { return TYPE_AM_ZIP; } + virtual PTXF Duplicate(PGLOBAL g) { return (PTXF) new(g)ZIPFAM(this); } + + // Methods + virtual int Cardinality(PGLOBAL g); + virtual int GetFileLength(PGLOBAL g); +//virtual int MaxBlkSize(PGLOBAL g, int s) {return s;} + virtual bool OpenTableFile(PGLOBAL g); + virtual bool DeferReading(void) { return false; } + virtual int GetNext(PGLOBAL g); +//virtual int ReadBuffer(PGLOBAL g); +//virtual int WriteBuffer(PGLOBAL g); +//virtual int DeleteRecords(PGLOBAL g, int irc); +//virtual void CloseTableFile(PGLOBAL g, bool abort); + +protected: + // Members + ZIPUTIL *zutp; + PSZ target; + bool mul; +}; // end of ZIPFAM /***********************************************************************/ -/* This is the DOS/UNIX Access Method class declaration for PlugDB */ -/* fixed/variable files compressed using the zlib library functions. */ -/* Physically these are written and read using the same technique */ -/* than blocked variable files, only the contain of each block is */ -/* compressed using the deflate zlib function. The purpose of this */ -/* specific format is to have a fast mechanism for direct access of */ -/* records so blocked optimization is fast and direct access (joins) */ -/* is allowed. Note that the block length is written ahead of each */ -/* block to enable reading when optimization file is not available. */ +/* This is the fixed ZIP file access method. */ /***********************************************************************/ -class DllExport ZLBFAM : public BLKFAM { - public: - // Constructor - ZLBFAM(PDOSDEF tdp); - ZLBFAM(PZLBFAM txfp); - - // Implementation - virtual AMT GetAmType(void) {return TYPE_AM_ZLIB;} - virtual int GetPos(void); - virtual int GetNextPos(void); - virtual PTXF Duplicate(PGLOBAL g) - {return (PTXF)new(g) ZLBFAM(this);} - inline void SetOptimized(bool b) {Optimized = b;} - - // Methods - virtual int GetFileLength(PGLOBAL g); - virtual bool SetPos(PGLOBAL g, int recpos); - virtual bool AllocateBuffer(PGLOBAL g); - virtual int ReadBuffer(PGLOBAL g); - virtual int WriteBuffer(PGLOBAL g); - virtual void CloseTableFile(PGLOBAL g, bool abort); - virtual void Rewind(void); - - protected: - bool WriteCompressedBuffer(PGLOBAL g); - int ReadCompressedBuffer(PGLOBAL g, void *rdbuf); - - // Members - z_streamp Zstream; // Compression/decompression stream - Byte *Zbuffer; // Compressed block buffer - int *Zlenp; // Pointer to block length - bool Optimized; // true when opt file is available - }; // end of class ZLBFAM +class DllExport ZPXFAM : public MPXFAM { + friend class ZIPFAM; +public: + // Constructors + ZPXFAM(PDOSDEF tdp); + ZPXFAM(PZPXFAM txfp); + + // Implementation + virtual AMT GetAmType(void) { return TYPE_AM_ZIP; } + virtual PTXF Duplicate(PGLOBAL g) { return (PTXF) new(g)ZPXFAM(this); } + + // Methods + virtual int GetFileLength(PGLOBAL g); + virtual int Cardinality(PGLOBAL g); + virtual bool OpenTableFile(PGLOBAL g); + virtual int GetNext(PGLOBAL g); +//virtual int ReadBuffer(PGLOBAL g); + +protected: + // Members + ZIPUTIL *zutp; + PSZ target; + bool mul; +}; // end of ZPXFAM #endif // __FILAMZIP_H diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index cf945a73f46..33f02d8338c 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -171,9 +171,9 @@ #define JSONMAX 10 // JSON Default max grp size extern "C" { - char version[]= "Version 1.04.0008 August 10, 2016"; + char version[]= "Version 1.05.0001 December 13, 2016"; #if defined(__WIN__) - char compver[]= "Version 1.04.0008 " __DATE__ " " __TIME__; + char compver[]= "Version 1.05.0001 " __DATE__ " " __TIME__; char slash= '\\'; #else // !__WIN__ char slash= '/'; @@ -512,13 +512,13 @@ ha_create_table_option connect_table_option_list[]= HA_TOPTION_NUMBER("QUOTED", quoted, (ulonglong) -1, 0, 3, 1), HA_TOPTION_NUMBER("ENDING", ending, (ulonglong) -1, 0, INT_MAX32, 1), HA_TOPTION_NUMBER("COMPRESS", compressed, 0, 0, 2, 1), -//HA_TOPTION_BOOL("COMPRESS", compressed, 0), HA_TOPTION_BOOL("MAPPED", mapped, 0), HA_TOPTION_BOOL("HUGE", huge, 0), HA_TOPTION_BOOL("SPLIT", split, 0), HA_TOPTION_BOOL("READONLY", readonly, 0), HA_TOPTION_BOOL("SEPINDEX", sepindex, 0), - HA_TOPTION_END + HA_TOPTION_BOOL("ZIPPED", zipped, 0), + HA_TOPTION_END }; @@ -532,7 +532,6 @@ ha_create_table_option connect_field_option_list[]= { HA_FOPTION_NUMBER("FLAG", offset, (ulonglong) -1, 0, INT_MAX32, 1), HA_FOPTION_NUMBER("MAX_DIST", freq, 0, 0, INT_MAX32, 1), // BLK_INDX -//HA_FOPTION_NUMBER("DISTRIB", opt, 0, 0, 2, 1), // used for BLK_INDX HA_FOPTION_NUMBER("FIELD_LENGTH", fldlen, 0, 0, INT_MAX32, 1), HA_FOPTION_STRING("DATE_FORMAT", dateformat), HA_FOPTION_STRING("FIELD_FORMAT", fieldformat), @@ -678,7 +677,6 @@ static int connect_init_func(void *p) connect_hton= (handlerton *)p; connect_hton->state= SHOW_OPTION_YES; connect_hton->create= connect_create_handler; -//connect_hton->flags= HTON_TEMPORARY_NOT_SUPPORTED | HTON_NO_PARTITION; connect_hton->flags= HTON_TEMPORARY_NOT_SUPPORTED; connect_hton->table_options= connect_table_option_list; connect_hton->field_options= connect_field_option_list; @@ -1135,7 +1133,9 @@ bool GetBooleanTableOption(PGLOBAL g, PTOS options, char *opname, bool bdef) opval= options->sepindex; else if (!stricmp(opname, "Header")) opval= (options->header != 0); // Is Boolean for some table types - else if (options->oplist) + else if (!stricmp(opname, "Zipped")) + opval = options->zipped; + else if (options->oplist) if ((pv= GetListOption(g, opname, options->oplist))) opval= (!*pv || *pv == 'y' || *pv == 'Y' || atoi(pv) != 0); @@ -1242,8 +1242,10 @@ char *ha_connect::GetStringOption(char *opname, char *sdef) if (opval && (!stricmp(opname, "connect") || !stricmp(opname, "tabname") - || !stricmp(opname, "filename"))) - opval = GetRealString(opval); + || !stricmp(opname, "filename") + || !stricmp(opname, "optname") + || !stricmp(opname, "entry"))) + opval = GetRealString(opval); if (!opval) { if (sdef && !strcmp(sdef, "*")) { @@ -4165,7 +4167,8 @@ bool ha_connect::check_privileges(THD *thd, PTOS options, char *dbn, bool quick) case TAB_DIR: case TAB_MAC: case TAB_WMI: - case TAB_OEM: + case TAB_ZIP: + case TAB_OEM: #ifdef NO_EMBEDDED_ACCESS_CHECKS return false; #endif @@ -5173,13 +5176,13 @@ static int connect_assisted_discovery(handlerton *, THD* thd, char v=0, spc= ',', qch= 0; const char *fncn= "?"; const char *user, *fn, *db, *host, *pwd, *sep, *tbl, *src; - const char *col, *ocl, *rnk, *pic, *fcl, *skc; + const char *col, *ocl, *rnk, *pic, *fcl, *skc, *zfn; char *tab, *dsn, *shm, *dpath; #if defined(__WIN__) char *nsp= NULL, *cls= NULL; #endif // __WIN__ - int port= 0, hdr= 0, mxr= 0, mxe= 0, rc= 0; - int cop __attribute__((unused))= 0, lrecl= 0; +//int hdr, mxe; + int port = 0, mxr = 0, rc = 0, mul = 0, lrecl = 0; #if defined(ODBC_SUPPORT) POPARM sop= NULL; char *ucnc= NULL; @@ -5190,7 +5193,8 @@ static int connect_assisted_discovery(handlerton *, THD* thd, PJPARM sjp= NULL; char *driver= NULL; char *url= NULL; - char *tabtyp = NULL; +//char *prop= NULL; + char *tabtyp= NULL; #endif // JDBC_SUPPORT uint tm, fnc= FNC_NO, supfnc= (FNC_NO | FNC_COL); bool bif, ok= false, dbf= false; @@ -5210,7 +5214,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, if (!g) return HA_ERR_INTERNAL_ERROR; - user= host= pwd= tbl= src= col= ocl= pic= fcl= skc= rnk= dsn= NULL; + user= host= pwd= tbl= src= col= ocl= pic= fcl= skc= rnk= zfn= dsn= NULL; // Get the useful create options ttp= GetTypeID(topt->type); @@ -5223,7 +5227,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, sep= topt->separator; spc= (!sep) ? ',' : *sep; qch= topt->qchar ? *topt->qchar : (signed)topt->quoted >= 0 ? '"' : 0; - hdr= (int)topt->header; + mul = (int)topt->multiple; tbl= topt->tablist; col= topt->colist; @@ -5256,13 +5260,16 @@ static int connect_assisted_discovery(handlerton *, THD* thd, #if defined(JDBC_SUPPORT) driver= GetListOption(g, "Driver", topt->oplist, NULL); // url= GetListOption(g, "URL", topt->oplist, NULL); +// prop = GetListOption(g, "Properties", topt->oplist, NULL); tabtyp = GetListOption(g, "Tabtype", topt->oplist, NULL); #endif // JDBC_SUPPORT - mxe= atoi(GetListOption(g,"maxerr", topt->oplist, "0")); #if defined(PROMPT_OK) cop= atoi(GetListOption(g, "checkdsn", topt->oplist, "0")); #endif // PROMPT_OK - } else { +#if defined(ZIP_SUPPORT) + zfn = GetListOption(g, "Zipfile", topt->oplist, NULL); +#endif // ZIP_SUPPORT + } else { host= "localhost"; user= (ttp == TAB_ODBC ? NULL : "root"); } // endif option_list @@ -5366,6 +5373,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, jdef->SetName(create_info->alias); sjp= (PJPARM)PlugSubAlloc(g, NULL, sizeof(JDBCPARM)); sjp->Driver= driver; +// sjp->Properties = prop; sjp->Fsize= 0; sjp->Scrollable= false; @@ -5468,7 +5476,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, case TAB_XML: #endif // LIBXML2_SUPPORT || DOMDOC_SUPPORT case TAB_JSON: - if (!fn) + if (!fn && !zfn && !mul) sprintf(g->Message, "Missing %s file name", topt->type); else ok= true; @@ -5582,7 +5590,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, NULL, port, fnc == FNC_COL); break; case TAB_CSV: - qrp= CSVColumns(g, dpath, fn, spc, qch, hdr, mxe, fnc == FNC_COL); + qrp = CSVColumns(g, dpath, topt, fnc == FNC_COL); break; #if defined(__WIN__) case TAB_WMI: @@ -6947,7 +6955,7 @@ maria_declare_plugin(connect) 0x0104, /* version number (1.04) */ NULL, /* status variables */ connect_system_variables, /* system variables */ - "1.04.0008", /* string version */ + "1.05.0001", /* string version */ MariaDB_PLUGIN_MATURITY_GAMMA /* maturity */ } maria_declare_plugin_end; diff --git a/storage/connect/ha_connect.h b/storage/connect/ha_connect.h index 60194ac0e3c..3d9ff967618 100644 --- a/storage/connect/ha_connect.h +++ b/storage/connect/ha_connect.h @@ -83,42 +83,9 @@ extern handlerton *connect_hton; These can be specified in the CREATE TABLE: CREATE TABLE ( ... ) {...here...} -*/ -#if 0 // moved to mycat.h -typedef struct ha_table_option_struct TOS, *PTOS; - -struct ha_table_option_struct { - const char *type; - const char *filename; - const char *optname; - const char *tabname; - const char *tablist; - const char *dbname; - const char *separator; -//const char *connect; - const char *qchar; - const char *module; - const char *subtype; - const char *catfunc; - const char *srcdef; - const char *colist; - const char *oplist; - const char *data_charset; - ulonglong lrecl; - ulonglong elements; -//ulonglong estimate; - ulonglong multiple; - ulonglong header; - ulonglong quoted; - ulonglong ending; - ulonglong compressed; - bool mapped; - bool huge; - bool split; - bool readonly; - bool sepindex; - }; -#endif // 0 + + ------ Was moved to mycat.h ------ + */ /** structure for CREATE TABLE options (field options) diff --git a/storage/connect/ioapi.c b/storage/connect/ioapi.c new file mode 100644 index 00000000000..7f5c191b2af --- /dev/null +++ b/storage/connect/ioapi.c @@ -0,0 +1,247 @@ +/* ioapi.h -- IO base function header for compress/uncompress .zip + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications for Zip64 support + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + +*/ + +#if defined(_WIN32) && (!(defined(_CRT_SECURE_NO_WARNINGS))) + #define _CRT_SECURE_NO_WARNINGS +#endif + +#if defined(__APPLE__) || defined(IOAPI_NO_64) +// In darwin and perhaps other BSD variants off_t is a 64 bit value, hence no need for specific 64 bit functions +#define FOPEN_FUNC(filename, mode) fopen(filename, mode) +#define FTELLO_FUNC(stream) ftello(stream) +#define FSEEKO_FUNC(stream, offset, origin) fseeko(stream, offset, origin) +#else +#define FOPEN_FUNC(filename, mode) fopen64(filename, mode) +#define FTELLO_FUNC(stream) ftello64(stream) +#define FSEEKO_FUNC(stream, offset, origin) fseeko64(stream, offset, origin) +#endif + + +#include "ioapi.h" + +voidpf call_zopen64 (const zlib_filefunc64_32_def* pfilefunc,const void*filename,int mode) +{ + if (pfilefunc->zfile_func64.zopen64_file != NULL) + return (*(pfilefunc->zfile_func64.zopen64_file)) (pfilefunc->zfile_func64.opaque,filename,mode); + else + { + return (*(pfilefunc->zopen32_file))(pfilefunc->zfile_func64.opaque,(const char*)filename,mode); + } +} + +long call_zseek64 (const zlib_filefunc64_32_def* pfilefunc,voidpf filestream, ZPOS64_T offset, int origin) +{ + if (pfilefunc->zfile_func64.zseek64_file != NULL) + return (*(pfilefunc->zfile_func64.zseek64_file)) (pfilefunc->zfile_func64.opaque,filestream,offset,origin); + else + { + uLong offsetTruncated = (uLong)offset; + if (offsetTruncated != offset) + return -1; + else + return (*(pfilefunc->zseek32_file))(pfilefunc->zfile_func64.opaque,filestream,offsetTruncated,origin); + } +} + +ZPOS64_T call_ztell64 (const zlib_filefunc64_32_def* pfilefunc,voidpf filestream) +{ + if (pfilefunc->zfile_func64.zseek64_file != NULL) + return (*(pfilefunc->zfile_func64.ztell64_file)) (pfilefunc->zfile_func64.opaque,filestream); + else + { + uLong tell_uLong = (*(pfilefunc->ztell32_file))(pfilefunc->zfile_func64.opaque,filestream); + if ((tell_uLong) == MAXU32) + return (ZPOS64_T)-1; + else + return tell_uLong; + } +} + +void fill_zlib_filefunc64_32_def_from_filefunc32(zlib_filefunc64_32_def* p_filefunc64_32,const zlib_filefunc_def* p_filefunc32) +{ + p_filefunc64_32->zfile_func64.zopen64_file = NULL; + p_filefunc64_32->zopen32_file = p_filefunc32->zopen_file; + p_filefunc64_32->zfile_func64.zerror_file = p_filefunc32->zerror_file; + p_filefunc64_32->zfile_func64.zread_file = p_filefunc32->zread_file; + p_filefunc64_32->zfile_func64.zwrite_file = p_filefunc32->zwrite_file; + p_filefunc64_32->zfile_func64.ztell64_file = NULL; + p_filefunc64_32->zfile_func64.zseek64_file = NULL; + p_filefunc64_32->zfile_func64.zclose_file = p_filefunc32->zclose_file; + p_filefunc64_32->zfile_func64.zerror_file = p_filefunc32->zerror_file; + p_filefunc64_32->zfile_func64.opaque = p_filefunc32->opaque; + p_filefunc64_32->zseek32_file = p_filefunc32->zseek_file; + p_filefunc64_32->ztell32_file = p_filefunc32->ztell_file; +} + + + +static voidpf ZCALLBACK fopen_file_func OF((voidpf opaque, const char* filename, int mode)); +static uLong ZCALLBACK fread_file_func OF((voidpf opaque, voidpf stream, void* buf, uLong size)); +static uLong ZCALLBACK fwrite_file_func OF((voidpf opaque, voidpf stream, const void* buf,uLong size)); +static ZPOS64_T ZCALLBACK ftell64_file_func OF((voidpf opaque, voidpf stream)); +static long ZCALLBACK fseek64_file_func OF((voidpf opaque, voidpf stream, ZPOS64_T offset, int origin)); +static int ZCALLBACK fclose_file_func OF((voidpf opaque, voidpf stream)); +static int ZCALLBACK ferror_file_func OF((voidpf opaque, voidpf stream)); + +static voidpf ZCALLBACK fopen_file_func (voidpf opaque, const char* filename, int mode) +{ + FILE* file = NULL; + const char* mode_fopen = NULL; + if ((mode & ZLIB_FILEFUNC_MODE_READWRITEFILTER)==ZLIB_FILEFUNC_MODE_READ) + mode_fopen = "rb"; + else + if (mode & ZLIB_FILEFUNC_MODE_EXISTING) + mode_fopen = "r+b"; + else + if (mode & ZLIB_FILEFUNC_MODE_CREATE) + mode_fopen = "wb"; + + if ((filename!=NULL) && (mode_fopen != NULL)) + file = fopen(filename, mode_fopen); + return file; +} + +static voidpf ZCALLBACK fopen64_file_func (voidpf opaque, const void* filename, int mode) +{ + FILE* file = NULL; + const char* mode_fopen = NULL; + if ((mode & ZLIB_FILEFUNC_MODE_READWRITEFILTER)==ZLIB_FILEFUNC_MODE_READ) + mode_fopen = "rb"; + else + if (mode & ZLIB_FILEFUNC_MODE_EXISTING) + mode_fopen = "r+b"; + else + if (mode & ZLIB_FILEFUNC_MODE_CREATE) + mode_fopen = "wb"; + + if ((filename!=NULL) && (mode_fopen != NULL)) + file = FOPEN_FUNC((const char*)filename, mode_fopen); + return file; +} + + +static uLong ZCALLBACK fread_file_func (voidpf opaque, voidpf stream, void* buf, uLong size) +{ + uLong ret; + ret = (uLong)fread(buf, 1, (size_t)size, (FILE *)stream); + return ret; +} + +static uLong ZCALLBACK fwrite_file_func (voidpf opaque, voidpf stream, const void* buf, uLong size) +{ + uLong ret; + ret = (uLong)fwrite(buf, 1, (size_t)size, (FILE *)stream); + return ret; +} + +static long ZCALLBACK ftell_file_func (voidpf opaque, voidpf stream) +{ + long ret; + ret = ftell((FILE *)stream); + return ret; +} + + +static ZPOS64_T ZCALLBACK ftell64_file_func (voidpf opaque, voidpf stream) +{ + ZPOS64_T ret; + ret = FTELLO_FUNC((FILE *)stream); + return ret; +} + +static long ZCALLBACK fseek_file_func (voidpf opaque, voidpf stream, uLong offset, int origin) +{ + int fseek_origin=0; + long ret; + switch (origin) + { + case ZLIB_FILEFUNC_SEEK_CUR : + fseek_origin = SEEK_CUR; + break; + case ZLIB_FILEFUNC_SEEK_END : + fseek_origin = SEEK_END; + break; + case ZLIB_FILEFUNC_SEEK_SET : + fseek_origin = SEEK_SET; + break; + default: return -1; + } + ret = 0; + if (fseek((FILE *)stream, offset, fseek_origin) != 0) + ret = -1; + return ret; +} + +static long ZCALLBACK fseek64_file_func (voidpf opaque, voidpf stream, ZPOS64_T offset, int origin) +{ + int fseek_origin=0; + long ret; + switch (origin) + { + case ZLIB_FILEFUNC_SEEK_CUR : + fseek_origin = SEEK_CUR; + break; + case ZLIB_FILEFUNC_SEEK_END : + fseek_origin = SEEK_END; + break; + case ZLIB_FILEFUNC_SEEK_SET : + fseek_origin = SEEK_SET; + break; + default: return -1; + } + ret = 0; + + if(FSEEKO_FUNC((FILE *)stream, offset, fseek_origin) != 0) + ret = -1; + + return ret; +} + + +static int ZCALLBACK fclose_file_func (voidpf opaque, voidpf stream) +{ + int ret; + ret = fclose((FILE *)stream); + return ret; +} + +static int ZCALLBACK ferror_file_func (voidpf opaque, voidpf stream) +{ + int ret; + ret = ferror((FILE *)stream); + return ret; +} + +void fill_fopen_filefunc (pzlib_filefunc_def) + zlib_filefunc_def* pzlib_filefunc_def; +{ + pzlib_filefunc_def->zopen_file = fopen_file_func; + pzlib_filefunc_def->zread_file = fread_file_func; + pzlib_filefunc_def->zwrite_file = fwrite_file_func; + pzlib_filefunc_def->ztell_file = ftell_file_func; + pzlib_filefunc_def->zseek_file = fseek_file_func; + pzlib_filefunc_def->zclose_file = fclose_file_func; + pzlib_filefunc_def->zerror_file = ferror_file_func; + pzlib_filefunc_def->opaque = NULL; +} + +void fill_fopen64_filefunc (zlib_filefunc64_def* pzlib_filefunc_def) +{ + pzlib_filefunc_def->zopen64_file = fopen64_file_func; + pzlib_filefunc_def->zread_file = fread_file_func; + pzlib_filefunc_def->zwrite_file = fwrite_file_func; + pzlib_filefunc_def->ztell64_file = ftell64_file_func; + pzlib_filefunc_def->zseek64_file = fseek64_file_func; + pzlib_filefunc_def->zclose_file = fclose_file_func; + pzlib_filefunc_def->zerror_file = ferror_file_func; + pzlib_filefunc_def->opaque = NULL; +} diff --git a/storage/connect/ioapi.h b/storage/connect/ioapi.h new file mode 100644 index 00000000000..8dcbdb06e35 --- /dev/null +++ b/storage/connect/ioapi.h @@ -0,0 +1,208 @@ +/* ioapi.h -- IO base function header for compress/uncompress .zip + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications for Zip64 support + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + + Changes + + Oct-2009 - Defined ZPOS64_T to fpos_t on windows and u_int64_t on linux. (might need to find a better why for this) + Oct-2009 - Change to fseeko64, ftello64 and fopen64 so large files would work on linux. + More if/def section may be needed to support other platforms + Oct-2009 - Defined fxxxx64 calls to normal fopen/ftell/fseek so they would compile on windows. + (but you should use iowin32.c for windows instead) + +*/ + +#ifndef _ZLIBIOAPI64_H +#define _ZLIBIOAPI64_H + +#if (!defined(_WIN32)) && (!defined(WIN32)) && (!defined(__APPLE__)) + + // Linux needs this to support file operation on files larger then 4+GB + // But might need better if/def to select just the platforms that needs them. + + #ifndef __USE_FILE_OFFSET64 + #define __USE_FILE_OFFSET64 + #endif + #ifndef __USE_LARGEFILE64 + #define __USE_LARGEFILE64 + #endif + #ifndef _LARGEFILE64_SOURCE + #define _LARGEFILE64_SOURCE + #endif + #ifndef _FILE_OFFSET_BIT + #define _FILE_OFFSET_BIT 64 + #endif + +#endif + +#include <stdio.h> +#include <stdlib.h> +#include "zlib.h" + +#if defined(USE_FILE32API) +#define fopen64 fopen +#define ftello64 ftell +#define fseeko64 fseek +#else +#ifdef __FreeBSD__ +#define fopen64 fopen +#define ftello64 ftello +#define fseeko64 fseeko +#endif +#ifdef _MSC_VER + #define fopen64 fopen + #if (_MSC_VER >= 1400) && (!(defined(NO_MSCVER_FILE64_FUNC))) + #define ftello64 _ftelli64 + #define fseeko64 _fseeki64 + #else // old MSC + #define ftello64 ftell + #define fseeko64 fseek + #endif +#endif +#endif + +/* +#ifndef ZPOS64_T + #ifdef _WIN32 + #define ZPOS64_T fpos_t + #else + #include <stdint.h> + #define ZPOS64_T uint64_t + #endif +#endif +*/ + +#ifdef HAVE_MINIZIP64_CONF_H +#include "mz64conf.h" +#endif + +/* a type choosen by DEFINE */ +#ifdef HAVE_64BIT_INT_CUSTOM +typedef 64BIT_INT_CUSTOM_TYPE ZPOS64_T; +#else +#ifdef HAS_STDINT_H +#include "stdint.h" +typedef uint64_t ZPOS64_T; +#else + +/* Maximum unsigned 32-bit value used as placeholder for zip64 */ +#define MAXU32 0xffffffff + +#if defined(_MSC_VER) || defined(__BORLANDC__) +typedef unsigned __int64 ZPOS64_T; +#else +typedef unsigned long long int ZPOS64_T; +#endif +#endif +#endif + + + +#ifdef __cplusplus +extern "C" { +#endif + + +#define ZLIB_FILEFUNC_SEEK_CUR (1) +#define ZLIB_FILEFUNC_SEEK_END (2) +#define ZLIB_FILEFUNC_SEEK_SET (0) + +#define ZLIB_FILEFUNC_MODE_READ (1) +#define ZLIB_FILEFUNC_MODE_WRITE (2) +#define ZLIB_FILEFUNC_MODE_READWRITEFILTER (3) + +#define ZLIB_FILEFUNC_MODE_EXISTING (4) +#define ZLIB_FILEFUNC_MODE_CREATE (8) + + +#ifndef ZCALLBACK + #if (defined(WIN32) || defined(_WIN32) || defined (WINDOWS) || defined (_WINDOWS)) && defined(CALLBACK) && defined (USEWINDOWS_CALLBACK) + #define ZCALLBACK CALLBACK + #else + #define ZCALLBACK + #endif +#endif + + + + +typedef voidpf (ZCALLBACK *open_file_func) OF((voidpf opaque, const char* filename, int mode)); +typedef uLong (ZCALLBACK *read_file_func) OF((voidpf opaque, voidpf stream, void* buf, uLong size)); +typedef uLong (ZCALLBACK *write_file_func) OF((voidpf opaque, voidpf stream, const void* buf, uLong size)); +typedef int (ZCALLBACK *close_file_func) OF((voidpf opaque, voidpf stream)); +typedef int (ZCALLBACK *testerror_file_func) OF((voidpf opaque, voidpf stream)); + +typedef long (ZCALLBACK *tell_file_func) OF((voidpf opaque, voidpf stream)); +typedef long (ZCALLBACK *seek_file_func) OF((voidpf opaque, voidpf stream, uLong offset, int origin)); + + +/* here is the "old" 32 bits structure structure */ +typedef struct zlib_filefunc_def_s +{ + open_file_func zopen_file; + read_file_func zread_file; + write_file_func zwrite_file; + tell_file_func ztell_file; + seek_file_func zseek_file; + close_file_func zclose_file; + testerror_file_func zerror_file; + voidpf opaque; +} zlib_filefunc_def; + +typedef ZPOS64_T (ZCALLBACK *tell64_file_func) OF((voidpf opaque, voidpf stream)); +typedef long (ZCALLBACK *seek64_file_func) OF((voidpf opaque, voidpf stream, ZPOS64_T offset, int origin)); +typedef voidpf (ZCALLBACK *open64_file_func) OF((voidpf opaque, const void* filename, int mode)); + +typedef struct zlib_filefunc64_def_s +{ + open64_file_func zopen64_file; + read_file_func zread_file; + write_file_func zwrite_file; + tell64_file_func ztell64_file; + seek64_file_func zseek64_file; + close_file_func zclose_file; + testerror_file_func zerror_file; + voidpf opaque; +} zlib_filefunc64_def; + +void fill_fopen64_filefunc OF((zlib_filefunc64_def* pzlib_filefunc_def)); +void fill_fopen_filefunc OF((zlib_filefunc_def* pzlib_filefunc_def)); + +/* now internal definition, only for zip.c and unzip.h */ +typedef struct zlib_filefunc64_32_def_s +{ + zlib_filefunc64_def zfile_func64; + open_file_func zopen32_file; + tell_file_func ztell32_file; + seek_file_func zseek32_file; +} zlib_filefunc64_32_def; + + +#define ZREAD64(filefunc,filestream,buf,size) ((*((filefunc).zfile_func64.zread_file)) ((filefunc).zfile_func64.opaque,filestream,buf,size)) +#define ZWRITE64(filefunc,filestream,buf,size) ((*((filefunc).zfile_func64.zwrite_file)) ((filefunc).zfile_func64.opaque,filestream,buf,size)) +//#define ZTELL64(filefunc,filestream) ((*((filefunc).ztell64_file)) ((filefunc).opaque,filestream)) +//#define ZSEEK64(filefunc,filestream,pos,mode) ((*((filefunc).zseek64_file)) ((filefunc).opaque,filestream,pos,mode)) +#define ZCLOSE64(filefunc,filestream) ((*((filefunc).zfile_func64.zclose_file)) ((filefunc).zfile_func64.opaque,filestream)) +#define ZERROR64(filefunc,filestream) ((*((filefunc).zfile_func64.zerror_file)) ((filefunc).zfile_func64.opaque,filestream)) + +voidpf call_zopen64 OF((const zlib_filefunc64_32_def* pfilefunc,const void*filename,int mode)); +long call_zseek64 OF((const zlib_filefunc64_32_def* pfilefunc,voidpf filestream, ZPOS64_T offset, int origin)); +ZPOS64_T call_ztell64 OF((const zlib_filefunc64_32_def* pfilefunc,voidpf filestream)); + +void fill_zlib_filefunc64_32_def_from_filefunc32(zlib_filefunc64_32_def* p_filefunc64_32,const zlib_filefunc_def* p_filefunc32); + +#define ZOPEN64(filefunc,filename,mode) (call_zopen64((&(filefunc)),(filename),(mode))) +#define ZTELL64(filefunc,filestream) (call_ztell64((&(filefunc)),(filestream))) +#define ZSEEK64(filefunc,filestream,pos,mode) (call_zseek64((&(filefunc)),(filestream),(pos),(mode))) + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/storage/connect/jdbccat.h b/storage/connect/jdbccat.h index 37f33d7063d..7108aa376ce 100644 --- a/storage/connect/jdbccat.h +++ b/storage/connect/jdbccat.h @@ -8,6 +8,7 @@ typedef struct jdbc_parms { char *Url; // Driver URL char *User; // User connect info char *Pwd; // Password connect info +//char *Properties; // Connection property list //int Cto; // Connect timeout //int Qto; // Query timeout int Fsize; // Fetch size diff --git a/storage/connect/jdbconn.cpp b/storage/connect/jdbconn.cpp index dca9bd0eac4..a69f84a94a1 100644 --- a/storage/connect/jdbconn.cpp +++ b/storage/connect/jdbconn.cpp @@ -55,9 +55,8 @@ #if defined(__WIN__) extern "C" HINSTANCE s_hModule; // Saved module handle -#else // !__WIN__ +#endif // __WIN__ #define nullptr 0 -#endif // !__WIN__ TYPCONV GetTypeConv(); int GetConvSize(); @@ -525,10 +524,10 @@ JDBConn::JDBConn(PGLOBAL g, TDBJDBC *tdbp) m_Wrap = strcat(strcpy(wn, "wrappers/"), m_Wrap); } // endif m_Wrap - m_Driver = NULL; - m_Url = NULL; - m_User = NULL; - m_Pwd = NULL; +//m_Driver = NULL; +//m_Url = NULL; +//m_User = NULL; +//m_Pwd = NULL; m_Ncol = 0; m_Aff = 0; m_Rows = 0; @@ -772,7 +771,7 @@ bool JDBConn::GetJVM(PGLOBAL g) /***********************************************************************/ int JDBConn::Open(PJPARM sop) { - + int irc = RC_FX; bool err = false; jboolean jt = (trace > 0); PGLOBAL& g = m_G; @@ -865,30 +864,37 @@ int JDBConn::Open(PJPARM sop) switch (rc) { case JNI_OK: strcpy(g->Message, "VM successfully created"); + irc = RC_OK; break; case JNI_ERR: strcpy(g->Message, "Initialising JVM failed: unknown error"); - return RC_FX; + break; case JNI_EDETACHED: strcpy(g->Message, "Thread detached from the VM"); - return RC_FX; + break; case JNI_EVERSION: strcpy(g->Message, "JNI version error"); - return RC_FX; + break; case JNI_ENOMEM: strcpy(g->Message, "Not enough memory"); - return RC_FX; + break; case JNI_EEXIST: strcpy(g->Message, "VM already created"); - return RC_FX; + break; case JNI_EINVAL: strcpy(g->Message, "Invalid arguments"); - return RC_FX; + break; default: - sprintf(g->Message, "Unknown return code %d", rc); - return RC_FX; + sprintf(g->Message, "Unknown return code %d", (int)rc); + break; } // endswitch rc + if (trace) + htrc("%s\n", g->Message); + + if (irc != RC_OK) + return irc; + //=============== Display JVM version =============== jint ver = env->GetVersion(); printf("JVM Version %d.%d\n", ((ver>>16)&0x0f), (ver&0x0f)); @@ -978,10 +984,10 @@ int JDBConn::Open(PJPARM sop) jobjectArray parms = env->NewObjectArray(4, // constructs java array of 4 env->FindClass("java/lang/String"), NULL); // Strings - m_Driver = sop->Driver; - m_Url = sop->Url; - m_User = sop->User; - m_Pwd = sop->Pwd; +//m_Driver = sop->Driver; +//m_Url = sop->Url; +//m_User = sop->User; +//m_Pwd = sop->Pwd; m_Scrollable = sop->Scrollable; m_RowsetSize = sop->Fsize; //m_LoginTimeout = sop->Cto; @@ -989,17 +995,20 @@ int JDBConn::Open(PJPARM sop) //m_UseCnc = sop->UseCnc; // change some elements - if (m_Driver) - env->SetObjectArrayElement(parms, 0, env->NewStringUTF(m_Driver)); + if (sop->Driver) + env->SetObjectArrayElement(parms, 0, env->NewStringUTF(sop->Driver)); - if (m_Url) - env->SetObjectArrayElement(parms, 1, env->NewStringUTF(m_Url)); + if (sop->Url) + env->SetObjectArrayElement(parms, 1, env->NewStringUTF(sop->Url)); - if (m_User) - env->SetObjectArrayElement(parms, 2, env->NewStringUTF(m_User)); + if (sop->User) + env->SetObjectArrayElement(parms, 2, env->NewStringUTF(sop->User)); - if (m_Pwd) - env->SetObjectArrayElement(parms, 3, env->NewStringUTF(m_Pwd)); + if (sop->Pwd) + env->SetObjectArrayElement(parms, 3, env->NewStringUTF(sop->Pwd)); + +//if (sop->Properties) +// env->SetObjectArrayElement(parms, 4, env->NewStringUTF(sop->Properties)); // call method rc = env->CallIntMethod(job, cid, parms, m_RowsetSize, m_Scrollable); @@ -1432,7 +1441,7 @@ bool JDBConn::SetParam(JDBCCOL *colp) PGLOBAL& g = m_G; bool rc = false; PVAL val = colp->GetValue(); - jint n, i = (jint)colp->GetRank(); + jint n, jrc = 0, i = (jint)colp->GetRank(); jshort s; jlong lg; //jfloat f; @@ -1442,69 +1451,74 @@ bool JDBConn::SetParam(JDBCCOL *colp) jstring jst = nullptr; jmethodID dtc, setid = nullptr; - switch (val->GetType()) { - case TYPE_STRING: - if (gmID(g, setid, "SetStringParm", "(ILjava/lang/String;)V")) - return true; - - jst = env->NewStringUTF(val->GetCharValue()); - env->CallVoidMethod(job, setid, i, jst); - break; - case TYPE_INT: - if (gmID(g, setid, "SetIntParm", "(II)V")) + if (val->GetNullable() && val->IsNull()) { + if (gmID(g, setid, "SetNullParm", "(II)I")) return true; - n = (jint)val->GetIntValue(); - env->CallVoidMethod(job, setid, i, n); - break; - case TYPE_TINY: - case TYPE_SHORT: - if (gmID(g, setid, "SetShortParm", "(IS)V")) - return true; + jrc = env->CallIntMethod(job, setid, i, (jint)GetJDBCType(val->GetType())); + } else switch (val->GetType()) { + case TYPE_STRING: + if (gmID(g, setid, "SetStringParm", "(ILjava/lang/String;)V")) + return true; - s = (jshort)val->GetShortValue(); - env->CallVoidMethod(job, setid, i, s); - break; - case TYPE_BIGINT: - if (gmID(g, setid, "SetBigintParm", "(IJ)V")) - return true; + jst = env->NewStringUTF(val->GetCharValue()); + env->CallVoidMethod(job, setid, i, jst); + break; + case TYPE_INT: + if (gmID(g, setid, "SetIntParm", "(II)V")) + return true; - lg = (jlong)val->GetBigintValue(); - env->CallVoidMethod(job, setid, i, lg); - break; - case TYPE_DOUBLE: - case TYPE_DECIM: - if (gmID(g, setid, "SetDoubleParm", "(ID)V")) - return true; + n = (jint)val->GetIntValue(); + env->CallVoidMethod(job, setid, i, n); + break; + case TYPE_TINY: + case TYPE_SHORT: + if (gmID(g, setid, "SetShortParm", "(IS)V")) + return true; - d = (jdouble)val->GetFloatValue(); - env->CallVoidMethod(job, setid, i, d); - break; - case TYPE_DATE: - if ((dat = env->FindClass("java/sql/Timestamp")) == nullptr) { - strcpy(g->Message, "Cannot find Timestamp class"); - return true; - } else if (!(dtc = env->GetMethodID(dat, "<init>", "(J)V"))) { - strcpy(g->Message, "Cannot find Timestamp class constructor"); - return true; - } // endif's + s = (jshort)val->GetShortValue(); + env->CallVoidMethod(job, setid, i, s); + break; + case TYPE_BIGINT: + if (gmID(g, setid, "SetBigintParm", "(IJ)V")) + return true; - lg = (jlong)val->GetBigintValue() * 1000; + lg = (jlong)val->GetBigintValue(); + env->CallVoidMethod(job, setid, i, lg); + break; + case TYPE_DOUBLE: + case TYPE_DECIM: + if (gmID(g, setid, "SetDoubleParm", "(ID)V")) + return true; - if ((datobj = env->NewObject(dat, dtc, lg)) == nullptr) { - strcpy(g->Message, "Cannot make Timestamp object"); - return true; - } else if (gmID(g, setid, "SetTimestampParm", "(ILjava/sql/Timestamp;)V")) + d = (jdouble)val->GetFloatValue(); + env->CallVoidMethod(job, setid, i, d); + break; + case TYPE_DATE: + if ((dat = env->FindClass("java/sql/Timestamp")) == nullptr) { + strcpy(g->Message, "Cannot find Timestamp class"); + return true; + } else if (!(dtc = env->GetMethodID(dat, "<init>", "(J)V"))) { + strcpy(g->Message, "Cannot find Timestamp class constructor"); + return true; + } // endif's + + lg = (jlong)val->GetBigintValue() * 1000; + + if ((datobj = env->NewObject(dat, dtc, lg)) == nullptr) { + strcpy(g->Message, "Cannot make Timestamp object"); + return true; + } else if (gmID(g, setid, "SetTimestampParm", "(ILjava/sql/Timestamp;)V")) + return true; + + env->CallVoidMethod(job, setid, i, datobj); + break; + default: + sprintf(g->Message, "Parm type %d not supported", val->GetType()); return true; + } // endswitch Type - env->CallVoidMethod(job, setid, i, datobj); - break; - default: - sprintf(g->Message, "Parm type %d not supported", val->GetType()); - return true; - } // endswitch Type - - if (Check()) { + if (Check(jrc)) { sprintf(g->Message, "SetParam: col=%s msg=%s", colp->GetName(), Msg); rc = true; } // endif msg diff --git a/storage/connect/jdbconn.h b/storage/connect/jdbconn.h index 0a1c52d4576..9d428142839 100644 --- a/storage/connect/jdbconn.h +++ b/storage/connect/jdbconn.h @@ -180,9 +180,9 @@ protected: char *Msg; char *m_Wrap; char m_IDQuoteChar[2]; - PSZ m_Driver; - PSZ m_Url; - PSZ m_User; +//PSZ m_Driver; +//PSZ m_Url; +//PSZ m_User; PSZ m_Pwd; int m_Ncol; int m_Aff; diff --git a/storage/connect/jsonudf.cpp b/storage/connect/jsonudf.cpp index 8bddc68e2ae..f9034f25739 100644 --- a/storage/connect/jsonudf.cpp +++ b/storage/connect/jsonudf.cpp @@ -5264,3 +5264,50 @@ char *envar(UDF_INIT *initid, UDF_ARGS *args, char *result, return str; } // end of envar +/*********************************************************************************/ +/* Returns the distinct number of B occurences in A. */ +/*********************************************************************************/ +my_bool countin_init(UDF_INIT *initid, UDF_ARGS *args, char *message) +{ + if (args->arg_count != 2) { + strcpy(message, "This function must have 2 arguments"); + return true; + } else if (args->arg_type[0] != STRING_RESULT) { + strcpy(message, "First argument must be string"); + return true; + } else if (args->arg_type[1] != STRING_RESULT) { + strcpy(message, "Second argument is not a string"); + return true; + } // endif args + + return false; +} // end of countin_init + +long long countin(UDF_INIT *initid, UDF_ARGS *args, char *result, + unsigned long *res_length, char *is_null, char *) +{ + PSZ str1, str2; + char *s; + long long n = 0; + size_t lg; + + lg = (size_t)args->lengths[0]; + s = str1 = (PSZ)malloc(lg + 1); + memcpy(str1, args->args[0], lg); + str1[lg] = 0; + + lg = (size_t)args->lengths[1]; + str2 = (PSZ)malloc(lg + 1); + memcpy(str2, args->args[1], lg); + str2[lg] = 0; + + while (s = strstr(s, str2)) { + n++; + s += lg; + } // endwhile + + free(str1); + free(str2); + return n; +} // end of countin + diff --git a/storage/connect/jsonudf.h b/storage/connect/jsonudf.h index 1406d9f2f2e..d2890421c62 100644 --- a/storage/connect/jsonudf.h +++ b/storage/connect/jsonudf.h @@ -221,6 +221,9 @@ extern "C" { DllExport my_bool envar_init(UDF_INIT*, UDF_ARGS*, char*); DllExport char *envar(UDF_EXEC_ARGS); + + DllExport my_bool countin_init(UDF_INIT*, UDF_ARGS*, char*); + DllExport long long countin(UDF_EXEC_ARGS); } // extern "C" diff --git a/storage/connect/libdoc.cpp b/storage/connect/libdoc.cpp index c2882fc0d7c..2470d37c353 100644 --- a/storage/connect/libdoc.cpp +++ b/storage/connect/libdoc.cpp @@ -1,6 +1,6 @@ /******************************************************************/ /* Implementation of XML document processing using libxml2 */ -/* Author: Olivier Bertrand 2007-2015 */ +/* Author: Olivier Bertrand 2007-2016 */ /******************************************************************/ #include "my_global.h" #include <string.h> @@ -68,8 +68,8 @@ class LIBXMLDOC : public XMLDOCUMENT { virtual void SetNofree(bool b) {Nofreelist = b;} // Methods - virtual bool Initialize(PGLOBAL g); - virtual bool ParseFile(char *fn); + virtual bool Initialize(PGLOBAL g, char *entry, bool zipped); + virtual bool ParseFile(PGLOBAL g, char *fn); virtual bool NewDoc(PGLOBAL g, char *ver); virtual void AddComment(PGLOBAL g, char *com); virtual PXNODE GetRoot(PGLOBAL g); @@ -373,22 +373,33 @@ LIBXMLDOC::LIBXMLDOC(char *nsl, char *nsdf, char *enc, PFBLOCK fp) /******************************************************************/ /* Initialize XML parser and check library compatibility. */ /******************************************************************/ -bool LIBXMLDOC::Initialize(PGLOBAL g) - { +bool LIBXMLDOC::Initialize(PGLOBAL g, char *entry, bool zipped) +{ + if (zipped && InitZip(g, entry)) + return true; + int n = xmlKeepBlanksDefault(1); return MakeNSlist(g); - } // end of Initialize +} // end of Initialize /******************************************************************/ /* Parse the XML file and construct node tree in memory. */ /******************************************************************/ -bool LIBXMLDOC::ParseFile(char *fn) +bool LIBXMLDOC::ParseFile(PGLOBAL g, char *fn) { if (trace) htrc("ParseFile\n"); - if ((Docp = xmlParseFile(fn))) { - if (Docp->encoding) + if (zip) { + // Parse an in memory document + char *xdoc = GetMemDoc(g, fn); + + Docp = (xdoc) ? xmlParseDoc((const xmlChar *)xdoc) : NULL; + } else + Docp = xmlParseFile(fn); + + if (Docp) { + if (Docp->encoding) Encoding = (char*)Docp->encoding; return false; @@ -609,6 +620,7 @@ void LIBXMLDOC::CloseDoc(PGLOBAL g, PFBLOCK xp) } // endif xp CloseXML2File(g, xp, false); + CloseZip(); } // end of Close /******************************************************************/ diff --git a/storage/connect/mycat.cc b/storage/connect/mycat.cc index b4b03e6ba4a..497fe5e1aa8 100644 --- a/storage/connect/mycat.cc +++ b/storage/connect/mycat.cc @@ -16,7 +16,7 @@ /*************** Mycat CC Program Source Code File (.CC) ***************/ /* PROGRAM NAME: MYCAT */ /* ------------- */ -/* Version 1.4 */ +/* Version 1.5 */ /* */ /* Author: Olivier Bertrand 2012 - 2016 */ /* */ @@ -64,7 +64,9 @@ #include "filamtxt.h" #include "tabdos.h" #include "tabfmt.h" +#if defined(VCT_SUPPORT) #include "tabvct.h" +#endif // VCT_SUPPORT #include "tabsys.h" #if defined(__WIN__) #include "tabmac.h" @@ -93,6 +95,9 @@ #if defined(XML_SUPPORT) #include "tabxml.h" #endif // XML_SUPPORT +#if defined(ZIP_SUPPORT) +#include "tabzip.h" +#endif // ZIP_SUPPORT #include "mycat.h" /***********************************************************************/ @@ -152,7 +157,10 @@ TABTYPE GetTypeID(const char *type) #endif : (!stricmp(type, "VIR")) ? TAB_VIR : (!stricmp(type, "JSON")) ? TAB_JSON - : (!stricmp(type, "OEM")) ? TAB_OEM : TAB_NIY; +#ifdef ZIP_SUPPORT + : (!stricmp(type, "ZIP")) ? TAB_ZIP +#endif + : (!stricmp(type, "OEM")) ? TAB_OEM : TAB_NIY; } // end of GetTypeID /***********************************************************************/ @@ -173,6 +181,7 @@ bool IsFileType(TABTYPE type) case TAB_INI: case TAB_VEC: case TAB_JSON: +// case TAB_ZIP: isfile= true; break; default: @@ -549,7 +558,9 @@ PRELDEF MYCAT::MakeTableDesc(PGLOBAL g, PTABLE tablep, LPCSTR am) #if defined(XML_SUPPORT) case TAB_XML: tdp= new(g) XMLDEF; break; #endif // XML_SUPPORT - case TAB_VEC: tdp= new(g) VCTDEF; break; +#if defined(VCT_SUPPORT) + case TAB_VEC: tdp = new(g)VCTDEF; break; +#endif // VCT_SUPPORT #if defined(ODBC_SUPPORT) case TAB_ODBC: tdp= new(g) ODBCDEF; break; #endif // ODBC_SUPPORT @@ -571,7 +582,10 @@ PRELDEF MYCAT::MakeTableDesc(PGLOBAL g, PTABLE tablep, LPCSTR am) #endif // PIVOT_SUPPORT case TAB_VIR: tdp= new(g) VIRDEF; break; case TAB_JSON: tdp= new(g) JSONDEF; break; - default: +#if defined(ZIP_SUPPORT) + case TAB_ZIP: tdp= new(g) ZIPDEF; break; +#endif // ZIP_SUPPORT + default: sprintf(g->Message, MSG(BAD_TABLE_TYPE), am, name); } // endswitch diff --git a/storage/connect/mycat.h b/storage/connect/mycat.h index 05163f08f1b..663b68fd4b9 100644 --- a/storage/connect/mycat.h +++ b/storage/connect/mycat.h @@ -62,6 +62,7 @@ struct ha_table_option_struct { bool split; bool readonly; bool sepindex; + bool zipped; }; // Possible value for catalog functions diff --git a/storage/connect/mysql-test/connect/disabled.def b/storage/connect/mysql-test/connect/disabled.def index 4e07b5c0576..5e15e0806ba 100644 --- a/storage/connect/mysql-test/connect/disabled.def +++ b/storage/connect/mysql-test/connect/disabled.def @@ -9,7 +9,7 @@ # Do not use any TAB characters for whitespace. # ############################################################################## -jdbc : Variable settings depend on machine configuration -jdbc_new : Variable settings depend on machine configuration +#jdbc : Variable settings depend on machine configuration +#jdbc_new : Variable settings depend on machine configuration jdbc_oracle : Variable settings depend on machine configuration jdbc_postgresql : Variable settings depend on machine configuration diff --git a/storage/connect/mysql-test/connect/r/jdbc_new.result b/storage/connect/mysql-test/connect/r/jdbc_new.result index 14381b0b11f..5cc4826213d 100644 --- a/storage/connect/mysql-test/connect/r/jdbc_new.result +++ b/storage/connect/mysql-test/connect/r/jdbc_new.result @@ -56,7 +56,7 @@ t1 CREATE TABLE `t1` ( ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='jdbc:mysql://127.0.0.1:SLAVE_PORT/test?user=root' `TABLE_TYPE`='JDBC' SELECT * FROM t1; a b -0 NULL +NULL NULL 0 test00 1 test01 2 test02 @@ -72,7 +72,7 @@ t1 CREATE TABLE `t1` ( ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='jdbc:mysql://127.0.0.1:SLAVE_PORT/test?user=root' `TABLE_TYPE`=JDBC `TABNAME`='t1' SELECT * FROM t1; a b -0 NULL +NULL NULL 0 test00 1 test01 2 test02 @@ -104,7 +104,7 @@ t1 CREATE TABLE `t1` ( ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='jdbc:mysql://127.0.0.1:SLAVE_PORT/test?user=root' `TABLE_TYPE`=JDBC SELECT * FROM t1; a b -0 NULL +NULL NULL 0 0 1 0 2 0 diff --git a/storage/connect/mysql-test/connect/std_data/JdbcMariaDB.jar b/storage/connect/mysql-test/connect/std_data/JdbcMariaDB.jar Binary files differindex 81f91e4465a..8525da74018 100644 --- a/storage/connect/mysql-test/connect/std_data/JdbcMariaDB.jar +++ b/storage/connect/mysql-test/connect/std_data/JdbcMariaDB.jar diff --git a/storage/connect/mysql-test/connect/t/jdbc.test b/storage/connect/mysql-test/connect/t/jdbc.test index 41fd298776b..58a527a3e6b 100644 --- a/storage/connect/mysql-test/connect/t/jdbc.test +++ b/storage/connect/mysql-test/connect/t/jdbc.test @@ -1,3 +1,4 @@ +-- source windows.inc -- source jdbconn.inc SET GLOBAL time_zone='+1:00'; diff --git a/storage/connect/mysql-test/connect/t/jdbc_new.test b/storage/connect/mysql-test/connect/t/jdbc_new.test index d1ad5117b72..5586cf8c027 100644 --- a/storage/connect/mysql-test/connect/t/jdbc_new.test +++ b/storage/connect/mysql-test/connect/t/jdbc_new.test @@ -5,6 +5,7 @@ connect (master,127.0.0.1,root,,test,$MASTER_MYPORT,); connect (slave,127.0.0.1,root,,test,$SLAVE_MYPORT,); connection master; +-- source windows.inc -- source jdbconn.inc connection slave; diff --git a/storage/connect/mysql-test/connect/t/windows.inc b/storage/connect/mysql-test/connect/t/windows.inc new file mode 100644 index 00000000000..88553d8aa59 --- /dev/null +++ b/storage/connect/mysql-test/connect/t/windows.inc @@ -0,0 +1,5 @@ +if (`select convert(@@version_compile_os using latin1) IN ("Win32","Win64","Windows") = 0`) +{ + skip Need windows; +} + diff --git a/storage/connect/plgdbsem.h b/storage/connect/plgdbsem.h index 910ce97f48a..cb408494319 100644 --- a/storage/connect/plgdbsem.h +++ b/storage/connect/plgdbsem.h @@ -1,9 +1,9 @@ /************** PlgDBSem H Declares Source Code File (.H) **************/ -/* Name: PLGDBSEM.H Version 3.6 */ +/* Name: PLGDBSEM.H Version 3.7 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 1998-2015 */ +/* (C) Copyright to the author Olivier BERTRAND 1998-2016 */ /* */ -/* This file contains the PlugDB++ application type definitions. */ +/* This file contains the CONNECT storage engine definitions. */ /***********************************************************************/ /***********************************************************************/ @@ -49,7 +49,8 @@ enum BLKTYP {TYPE_TABLE = 50, /* Table Name/Srcdef/... Block */ TYPE_FB_MAP = 23, /* Mapped file block (storage) */ TYPE_FB_HANDLE = 24, /* File block (handle) */ TYPE_FB_XML = 21, /* DOM XML file block */ - TYPE_FB_XML2 = 27}; /* libxml2 XML file block */ + TYPE_FB_XML2 = 27, /* libxml2 XML file block */ + TYPE_FB_ZIP = 28}; /* ZIP file block */ enum TABTYPE {TAB_UNDEF = 0, /* Table of undefined type */ TAB_DOS = 1, /* Fixed column offset, variable LRECL */ @@ -78,7 +79,8 @@ enum TABTYPE {TAB_UNDEF = 0, /* Table of undefined type */ TAB_JCT = 24, /* Junction tables NIY */ TAB_DMY = 25, /* DMY Dummy tables NIY */ TAB_JDBC = 26, /* Table accessed via JDBC */ - TAB_NIY = 27}; /* Table not implemented yet */ + TAB_ZIP = 27, /* ZIP file info table */ + TAB_NIY = 28}; /* Table not implemented yet */ enum AMT {TYPE_AM_ERROR = 0, /* Type not defined */ TYPE_AM_ROWID = 1, /* ROWID type (special column) */ @@ -123,7 +125,7 @@ enum AMT {TYPE_AM_ERROR = 0, /* Type not defined */ TYPE_AM_PRX = 129, /* PROXY access method type no */ TYPE_AM_XTB = 130, /* SYS table access method type */ TYPE_AM_BLK = 131, /* BLK access method type no */ - TYPE_AM_ZIP = 132, /* ZIP access method type no */ + TYPE_AM_GZ = 132, /* GZ access method type no */ TYPE_AM_ZLIB = 133, /* ZLIB access method type no */ TYPE_AM_JSON = 134, /* JSON access method type no */ TYPE_AM_JSN = 135, /* JSN access method type no */ @@ -140,7 +142,8 @@ enum AMT {TYPE_AM_ERROR = 0, /* Type not defined */ TYPE_AM_MYSQL = 192, /* MYSQL access method type no */ TYPE_AM_MYX = 193, /* MYSQL EXEC access method type */ TYPE_AM_CAT = 195, /* Catalog access method type no */ - TYPE_AM_OUT = 200}; /* Output relations (storage) */ + TYPE_AM_ZIP = 198, /* ZIP access method type no */ + TYPE_AM_OUT = 200}; /* Output relations (storage) */ enum RECFM {RECFM_NAF = -2, /* Not a file */ RECFM_OEM = -1, /* OEM file access method */ diff --git a/storage/connect/plgdbutl.cpp b/storage/connect/plgdbutl.cpp index 13c0dfd1e18..83975c6d8fa 100644 --- a/storage/connect/plgdbutl.cpp +++ b/storage/connect/plgdbutl.cpp @@ -68,6 +68,9 @@ #include "tabcol.h" // header of XTAB and COLUMN classes #include "valblk.h" #include "rcmsg.h" +#ifdef ZIP_SUPPORT +#include "filamzip.h" +#endif // ZIP_SUPPORT /***********************************************************************/ /* DB static variables. */ @@ -934,7 +937,16 @@ int PlugCloseFile(PGLOBAL g __attribute__((unused)), PFBLOCK fp, bool all) CloseXML2File(g, fp, all); break; #endif // LIBXML2_SUPPORT - default: +#ifdef ZIP_SUPPORT + case TYPE_FB_ZIP: + ((ZIPUTIL*)fp->File)->close(); + fp->Memory = NULL; + fp->Mode = MODE_ANY; + fp->Count = 0; + fp->File = NULL; + break; +#endif // ZIP_SUPPORT + default: rc = RC_FX; } // endswitch Type diff --git a/storage/connect/plgxml.cpp b/storage/connect/plgxml.cpp index 3061a6d697e..71b72621b06 100644 --- a/storage/connect/plgxml.cpp +++ b/storage/connect/plgxml.cpp @@ -30,19 +30,51 @@ PXDOC GetLibxmlDoc(PGLOBAL g, char *nsl, char *nsdf, /* XMLDOCUMENT constructor. */ /******************************************************************/ XMLDOCUMENT::XMLDOCUMENT(char *nsl, char *nsdf, char *enc) - { - Namespaces = NULL; +{ +#if defined(ZIP_SUPPORT) + zip = NULL; +#else // !ZIP_SUPPORT + zip = false; +#endif // !ZIP_SUPPORT + Namespaces = NULL; Encoding = enc; Nslist = nsl; DefNs = nsdf; - } // end of XMLDOCUMENT constructor +} // end of XMLDOCUMENT constructor + +/******************************************************************/ +/* Initialize zipped file processing. */ +/******************************************************************/ +bool XMLDOCUMENT::InitZip(PGLOBAL g, char *entry) +{ +#if defined(ZIP_SUPPORT) + bool mul = (entry) ? strchr(entry, '*') || strchr(entry, '?') : false; + zip = new(g) ZIPUTIL(entry, mul); + return zip == NULL; +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return true; +#endif // !ZIP_SUPPORT +} // end of InitZip + +/******************************************************************/ +/* Make the namespace structure list. */ +/******************************************************************/ +char* XMLDOCUMENT::GetMemDoc(PGLOBAL g, char *fn) +{ +#if defined(ZIP_SUPPORT) + return (zip->OpenTable(g, MODE_ANY, fn)) ? NULL : zip->memory; +#else // !ZIP_SUPPORT + return NULL; +#endif // !ZIP_SUPPORT +} // end of GetMemDoc /******************************************************************/ /* Make the namespace structure list. */ /******************************************************************/ bool XMLDOCUMENT::MakeNSlist(PGLOBAL g) - { - char *prefix, *href, *next = Nslist; +{ + char *prefix, *href, *next = Nslist; PNS nsp, *ppns = &Namespaces; while (next) { @@ -84,6 +116,19 @@ bool XMLDOCUMENT::MakeNSlist(PGLOBAL g) } // end of MakeNSlist /******************************************************************/ +/* Close ZIP file. */ +/******************************************************************/ +void XMLDOCUMENT::CloseZip(void) +{ +#if defined(ZIP_SUPPORT) + if (zip) { + zip->close(); + zip = NULL; + } // endif zip +#endif // ZIP_SUPPORT +} // end of CloseZip + +/******************************************************************/ /* XMLNODE constructor. */ /******************************************************************/ XMLNODE::XMLNODE(PXDOC dp) diff --git a/storage/connect/plgxml.h b/storage/connect/plgxml.h index b8e914e0bf1..db7dfa6bda5 100644 --- a/storage/connect/plgxml.h +++ b/storage/connect/plgxml.h @@ -1,3 +1,7 @@ +#if defined(ZIP_SUPPORT) +#include "filamzip.h" +#endif // ZIP_SUPPORT + /******************************************************************/ /* Dual XML implementation base classes defines. */ /******************************************************************/ @@ -72,8 +76,8 @@ class XMLDOCUMENT : public BLOCK { virtual void SetNofree(bool b) = 0; // Methods - virtual bool Initialize(PGLOBAL) = 0; - virtual bool ParseFile(char *) = 0; + virtual bool Initialize(PGLOBAL, char *, bool) = 0; + virtual bool ParseFile(PGLOBAL, char *) = 0; virtual bool NewDoc(PGLOBAL, char *) = 0; virtual void AddComment(PGLOBAL, char *) = 0; virtual PXNODE GetRoot(PGLOBAL) = 0; @@ -91,8 +95,16 @@ class XMLDOCUMENT : public BLOCK { // Utility bool MakeNSlist(PGLOBAL g); + bool InitZip(PGLOBAL g, char *entry); + char *GetMemDoc(PGLOBAL g, char *fn); + void CloseZip(void); // Members +#if defined(ZIP_SUPPORT) + ZIPUTIL *zip; /* Used for zipped file */ +#else // !ZIP_SUPPORT + bool zip; /* Always false */ +#endif // !ZIP_SUPPORT PNS Namespaces; /* To the namespaces */ char *Encoding; /* The document encoding */ char *Nslist; /* Namespace list */ diff --git a/storage/connect/reldef.cpp b/storage/connect/reldef.cpp index ac2327212e0..30e4d49d249 100644 --- a/storage/connect/reldef.cpp +++ b/storage/connect/reldef.cpp @@ -1,7 +1,7 @@ /************* RelDef CPP Program Source Code File (.CPP) **************/ /* PROGRAM NAME: RELDEF */ /* ------------- */ -/* Version 1.5 */ +/* Version 1.6 */ /* */ /* COPYRIGHT: */ /* ---------- */ @@ -40,10 +40,12 @@ #include "tabcol.h" #include "filamap.h" #include "filamfix.h" +#if defined(VCT_SUPPORT) #include "filamvct.h" -#if defined(ZIP_SUPPORT) -#include "filamzip.h" -#endif // ZIP_SUPPORT +#endif // VCT_SUPPORT +#if defined(GZ_SUPPORT) +#include "filamgz.h" +#endif // GZ_SUPPORT #include "tabdos.h" #include "valblk.h" #include "tabmul.h" @@ -663,15 +665,15 @@ PTDB OEMDEF::GetTable(PGLOBAL g, MODE mode) /*********************************************************************/ if (!((PTDBDOS)tdbp)->GetTxfp()) { if (cmpr) { -#if defined(ZIP_SUPPORT) +#if defined(GZ_SUPPORT) if (cmpr == 1) - txfp = new(g) ZIPFAM(defp); + txfp = new(g) GZFAM(defp); else txfp = new(g) ZLBFAM(defp); -#else // !ZIP_SUPPORT +#else // !GZ_SUPPORT strcpy(g->Message, "Compress not supported"); return NULL; -#endif // !ZIP_SUPPORT +#endif // !GZ_SUPPORT } else if (rfm == RECFM_VAR) { if (map) txfp = new(g) MAPFAM(defp); @@ -683,16 +685,19 @@ PTDB OEMDEF::GetTable(PGLOBAL g, MODE mode) txfp = new(g) MPXFAM(defp); else txfp = new(g) FIXFAM(defp); - } else if (rfm == RECFM_VCT) { - assert (Pxdef->GetDefType() == TYPE_AM_VCT); +#if defined(VCT_SUPPORT) + assert(Pxdef->GetDefType() == TYPE_AM_VCT); if (map) txfp = new(g) VCMFAM((PVCTDEF)defp); else txfp = new(g) VCTFAM((PVCTDEF)defp); - - } // endif's +#else // !VCT_SUPPORT + strcpy(g->Message, "VCT no more supported"); + return NULL; +#endif // !VCT_SUPPORT + } // endif's ((PTDBDOS)tdbp)->SetTxfp(txfp); } // endif Txfp diff --git a/storage/connect/tabdos.cpp b/storage/connect/tabdos.cpp index 98633f49d23..16cc6c33b44 100644 --- a/storage/connect/tabdos.cpp +++ b/storage/connect/tabdos.cpp @@ -1,11 +1,11 @@ /************* TabDos C++ Program Source Code File (.CPP) **************/ /* PROGRAM NAME: TABDOS */ /* ------------- */ -/* Version 4.9 */ +/* Version 4.9.2 */ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 1998-2015 */ +/* (C) Copyright to the author Olivier BERTRAND 1998-2016 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -51,6 +51,9 @@ #include "filamap.h" #include "filamfix.h" #include "filamdbf.h" +#if defined(GZ_SUPPORT) +#include "filamgz.h" +#endif // GZ_SUPPORT #if defined(ZIP_SUPPORT) #include "filamzip.h" #endif // ZIP_SUPPORT @@ -93,10 +96,13 @@ DOSDEF::DOSDEF(void) Pseudo = 3; Fn = NULL; Ofn = NULL; + Entry = NULL; To_Indx = NULL; Recfm = RECFM_VAR; Mapped = false; - Padded = false; + Zipped = false; + Mulentries = false; + Padded = false; Huge = false; Accept = false; Eof = false; @@ -126,6 +132,11 @@ bool DOSDEF::DefineAM(PGLOBAL g, LPCSTR am, int) : (am && (*am == 'B' || *am == 'b')) ? "B" : (am && !stricmp(am, "DBF")) ? "D" : "V"; + if ((Zipped = GetBoolCatInfo("Zipped", false))) + Mulentries = ((Entry = GetStringCatInfo(g, "Entry", NULL))) + ? strchr(Entry, '*') || strchr(Entry, '?') + : GetBoolCatInfo("Mulentries", false); + Desc = Fn = GetStringCatInfo(g, "Filename", NULL); Ofn = GetStringCatInfo(g, "Optname", Fn); GetCharCatInfo("Recfm", (PSZ)dfm, buf, sizeof(buf)); @@ -333,7 +344,21 @@ PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode) /* Allocate table and file processing class of the proper type. */ /* Column blocks will be allocated only when needed. */ /*********************************************************************/ - if (Recfm == RECFM_DBF) { + if (Zipped) { +#if defined(ZIP_SUPPORT) + if (Recfm == RECFM_VAR) { + txfp = new(g)ZIPFAM(this); + tdbp = new(g)TDBDOS(this, txfp); + } else { + txfp = new(g)ZPXFAM(this); + tdbp = new(g)TDBFIX(this, txfp); + } // endif Recfm + +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif // !ZIP_SUPPORT + } else if (Recfm == RECFM_DBF) { if (Catfunc == FNC_NO) { if (map) txfp = new(g) DBMFAM(this); @@ -350,28 +375,28 @@ PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode) else if (map) txfp = new(g) MPXFAM(this); else if (Compressed) { -#if defined(ZIP_SUPPORT) +#if defined(GZ_SUPPORT) txfp = new(g) ZIXFAM(this); -#else // !ZIP_SUPPORT - sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); +#else // !GZ_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "GZ"); return NULL; -#endif // !ZIP_SUPPORT +#endif // !GZ_SUPPORT } else txfp = new(g) FIXFAM(this); tdbp = new(g) TDBFIX(this, txfp); } else { if (Compressed) { -#if defined(ZIP_SUPPORT) +#if defined(GZ_SUPPORT) if (Compressed == 1) - txfp = new(g) ZIPFAM(this); + txfp = new(g) GZFAM(this); else txfp = new(g) ZLBFAM(this); -#else // !ZIP_SUPPORT - sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); +#else // !GZ_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "GZ"); return NULL; -#endif // !ZIP_SUPPORT +#endif // !GZ_SUPPORT } else if (map) txfp = new(g) MAPFAM(this); else @@ -396,7 +421,7 @@ PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode) if (map) { txfp = new(g) MBKFAM(this); } else if (Compressed) { -#if defined(ZIP_SUPPORT) +#if defined(GZ_SUPPORT) if (Compressed == 1) txfp = new(g) ZBKFAM(this); else { @@ -404,7 +429,7 @@ PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode) ((PZLBFAM)txfp)->SetOptimized(To_Pos != NULL); } // endelse #else - sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "GZ"); return NULL; #endif } else @@ -531,14 +556,14 @@ int TDBDOS::ResetTableOpt(PGLOBAL g, bool dop, bool dox) // except for ZLIB access method. if (Txfp->GetAmType() == TYPE_AM_MAP) { Txfp = new(g) MAPFAM((PDOSDEF)To_Def); -#if defined(ZIP_SUPPORT) - } else if (Txfp->GetAmType() == TYPE_AM_ZIP) { - Txfp = new(g) ZIPFAM((PDOSDEF)To_Def); +#if defined(GZ_SUPPORT) + } else if (Txfp->GetAmType() == TYPE_AM_GZ) { + Txfp = new(g) GZFAM((PDOSDEF)To_Def); } else if (Txfp->GetAmType() == TYPE_AM_ZLIB) { Txfp->Reset(); ((PZLBFAM)Txfp)->SetOptimized(false); -#endif // ZIP_SUPPORT - } else if (Txfp->GetAmType() == TYPE_AM_BLK) +#endif // GZ_SUPPORT + } else if (Txfp->GetAmType() == TYPE_AM_BLK) Txfp = new(g) DOSFAM((PDOSDEF)To_Def); Txfp->SetTdbp(this); @@ -609,7 +634,12 @@ int TDBDOS::MakeBlockValues(PGLOBAL g) defp->SetOptimized(0); // Estimate the number of needed blocks - block = (int)((MaxSize + (int)nrec - 1) / (int)nrec); + if ((block = (int)((MaxSize + (int)nrec - 1) / (int)nrec)) < 2) { + // This may be wrong to do in some cases + defp->RemoveOptValues(g); + strcpy(g->Message, MSG(TABLE_NOT_OPT)); + return RC_INFO; // Not to be optimized + } // endif block // We have to use local variables because Txfp->CurBlk is set // to Rows+1 by unblocked variable length table access methods. @@ -952,13 +982,14 @@ bool TDBDOS::GetBlockValues(PGLOBAL g) PCOLDEF cdp; PDOSDEF defp = (PDOSDEF)To_Def; PCATLG cat = defp->GetCat(); + PDBUSER dup = PlgGetUser(g); #if 0 if (Mode == MODE_INSERT && Txfp->GetAmType() == TYPE_AM_DOS) return false; #endif // __WIN__ - if (defp->Optimized) + if (defp->Optimized || !(dup->Check & CHK_OPT)) return false; // Already done or to be redone if (Ftype == RECFM_VAR || defp->Compressed == 2) { @@ -2079,10 +2110,10 @@ bool TDBDOS::OpenDB(PGLOBAL g) /*******************************************************************/ if (Txfp->GetAmType() == TYPE_AM_MAP && Mode == MODE_DELETE) Txfp = new(g) MAPFAM((PDOSDEF)To_Def); -#if defined(ZIP_SUPPORT) - else if (Txfp->GetAmType() == TYPE_AM_ZIP) - Txfp = new(g) ZIPFAM((PDOSDEF)To_Def); -#endif // ZIP_SUPPORT +#if defined(GZ_SUPPORT) + else if (Txfp->GetAmType() == TYPE_AM_GZ) + Txfp = new(g) GZFAM((PDOSDEF)To_Def); +#endif // GZ_SUPPORT else // if (Txfp->GetAmType() != TYPE_AM_DOS) ??? Txfp = new(g) DOSFAM((PDOSDEF)To_Def); diff --git a/storage/connect/tabdos.h b/storage/connect/tabdos.h index c098886f14b..4c8eb438a26 100644 --- a/storage/connect/tabdos.h +++ b/storage/connect/tabdos.h @@ -28,6 +28,7 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */ friend class TDBFIX; friend class TXTFAM; friend class DBFBASE; + friend class ZIPUTIL; public: // Constructor DOSDEF(void); @@ -40,7 +41,9 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */ virtual bool IsHuge(void) {return Huge;} PSZ GetFn(void) {return Fn;} PSZ GetOfn(void) {return Ofn;} - void SetBlock(int block) {Block = block;} + PSZ GetEntry(void) {return Entry;} + bool GetMul(void) {return Mulentries;} + void SetBlock(int block) {Block = block;} int GetBlock(void) {return Block;} int GetLast(void) {return Last;} void SetLast(int last) {Last = last;} @@ -57,9 +60,9 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */ int *GetTo_Pos(void) {return To_Pos;} // Methods - virtual int Indexable(void) - {return (!Multiple && Compressed != 1) ? 1 : 0;} - virtual bool DeleteIndexFile(PGLOBAL g, PIXDEF pxdf); + virtual int Indexable(void) + {return (!Multiple && !Mulentries && Compressed != 1) ? 1 : 0;} + virtual bool DeleteIndexFile(PGLOBAL g, PIXDEF pxdf); virtual bool DefineAM(PGLOBAL g, LPCSTR am, int poff); virtual PTDB GetTable(PGLOBAL g, MODE mode); bool InvalidateIndex(PGLOBAL g); @@ -72,10 +75,13 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */ // Members PSZ Fn; /* Path/Name of corresponding file */ PSZ Ofn; /* Base Path/Name of matching index files*/ - PIXDEF To_Indx; /* To index definitions blocks */ + PSZ Entry; /* Zip entry name or pattern */ + PIXDEF To_Indx; /* To index definitions blocks */ RECFM Recfm; /* 0:VAR, 1:FIX, 2:BIN, 3:VCT, 6:DBF */ bool Mapped; /* 0: disk file, 1: memory mapped file */ - bool Padded; /* true for padded table file */ + bool Zipped; /* true for zipped table file */ + bool Mulentries; /* true for multiple entries */ + bool Padded; /* true for padded table file */ bool Huge; /* true for files larger than 2GB */ bool Accept; /* true if wrong lines are accepted */ bool Eof; /* true if an EOF (0xA) character exists */ @@ -91,7 +97,7 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */ int Maxerr; /* Maximum number of bad records (DBF) */ int ReadMode; /* Specific to DBF */ int Ending; /* Length of end of lines */ - int Teds; /* Binary table default endian setting */ + char Teds; /* Binary table default endian setting */ }; // end of DOSDEF /***********************************************************************/ diff --git a/storage/connect/tabfix.cpp b/storage/connect/tabfix.cpp index 55c254f41ea..d99f7800f26 100644 --- a/storage/connect/tabfix.cpp +++ b/storage/connect/tabfix.cpp @@ -1,11 +1,11 @@ /************* TabFix C++ Program Source Code File (.CPP) **************/ /* PROGRAM NAME: TABFIX */ /* ------------- */ -/* Version 4.9 */ +/* Version 4.9.1 */ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 1998-2015 */ +/* (C) Copyright to the author Olivier BERTRAND 1998-2016 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -589,9 +589,10 @@ void BINCOL::WriteColumn(PGLOBAL g) switch (Fmt) { case 'X': // Standard not converted values - if (Eds && IsTypeChar(Buf_Type)) - *(longlong *)p = Value->GetBigintValue(); - else if (Value->GetBinValue(p, Long, Status)) { + if (Eds && IsTypeChar(Buf_Type)) { + if (Status) + Value->GetValueNonAligned<longlong>(p, Value->GetBigintValue()); + } else if (Value->GetBinValue(p, Long, Status)) { sprintf(g->Message, MSG(BIN_F_TOO_LONG), Name, Value->GetSize(), Long); longjmp(g->jumper[g->jump_level], 31); @@ -605,7 +606,7 @@ void BINCOL::WriteColumn(PGLOBAL g) sprintf(g->Message, MSG(VALUE_TOO_BIG), n, Name); longjmp(g->jumper[g->jump_level], 31); } else if (Status) - *(short *)p = (short)n; + Value->GetValueNonAligned<short>(p, (short)n); break; case 'T': // Tiny integer @@ -625,7 +626,7 @@ void BINCOL::WriteColumn(PGLOBAL g) sprintf(g->Message, MSG(VALUE_TOO_BIG), n, Name); longjmp(g->jumper[g->jump_level], 31); } else if (Status) - *(int *)p = Value->GetIntValue(); + Value->GetValueNonAligned<int>(p, (int)n); break; case 'G': // Large (great) integer @@ -636,12 +637,12 @@ void BINCOL::WriteColumn(PGLOBAL g) case 'F': // Float case 'R': // Real if (Status) - *(float *)p = (float)Value->GetFloatValue(); + Value->GetValueNonAligned<float>(p, (float)Value->GetFloatValue()); break; case 'D': // Double if (Status) - *(double *)p = Value->GetFloatValue(); + Value->GetValueNonAligned<double>(p, Value->GetFloatValue()); break; case 'C': // Characters diff --git a/storage/connect/tabfmt.cpp b/storage/connect/tabfmt.cpp index 015f8d93b15..b24375443f6 100644 --- a/storage/connect/tabfmt.cpp +++ b/storage/connect/tabfmt.cpp @@ -1,11 +1,11 @@ /************* TabFmt C++ Program Source Code File (.CPP) **************/ /* PROGRAM NAME: TABFMT */ /* ------------- */ -/* Version 3.9 */ +/* Version 3.9.2 */ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 2001 - 2015 */ +/* (C) Copyright to the author Olivier BERTRAND 2001 - 2016 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -51,6 +51,9 @@ #include "plgdbsem.h" #include "mycat.h" #include "filamap.h" +#if defined(GZ_SUPPORT) +#include "filamgz.h" +#endif // GZ_SUPPORT #if defined(ZIP_SUPPORT) #include "filamzip.h" #endif // ZIP_SUPPORT @@ -78,20 +81,24 @@ USETEMP UseTemp(void); /* of types (TYPE_STRING < TYPE_DOUBLE < TYPE_INT) (1 < 2 < 7). */ /* If these values are changed, this will have to be revisited. */ /***********************************************************************/ -PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, - char q, int hdr, int mxr, bool info) +PQRYRES CSVColumns(PGLOBAL g, char *dp, PTOS topt, bool info) { static int buftyp[] = {TYPE_STRING, TYPE_SHORT, TYPE_STRING, TYPE_INT, TYPE_INT, TYPE_SHORT}; static XFLD fldtyp[] = {FLD_NAME, FLD_TYPE, FLD_TYPENAME, FLD_PREC, FLD_LENGTH, FLD_SCALE}; static unsigned int length[] = {6, 6, 8, 10, 10, 6}; - char *p, *colname[MAXCOL], dechar, filename[_MAX_PATH], buf[4096]; + const char *fn; + char sep, q; + int rc, mxr; + bool hdr; + char *p, *colname[MAXCOL], dechar, buf[8]; int i, imax, hmax, n, nerr, phase, blank, digit, dec, type; int ncol = sizeof(buftyp) / sizeof(int); int num_read = 0, num_max = 10000000; // Statistics int len[MAXCOL], typ[MAXCOL], prc[MAXCOL]; - FILE *infile; + PCSVDEF tdp; + PTDBCSV tdbp; PQRYRES qrp; PCOLRES crp; @@ -101,27 +108,13 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, goto skipit; } // endif info -// num_max = atoi(p+1); // Max num of record to test -#if defined(__WIN__) - if (sep == ',' || strnicmp(setlocale(LC_NUMERIC, NULL), "French", 6)) - dechar = '.'; - else - dechar = ','; -#else // !__WIN__ - dechar = '.'; -#endif // !__WIN__ - - if (trace) - htrc("File %s sep=%c q=%c hdr=%d mxr=%d\n", - SVP(fn), sep, q, hdr, mxr); - - if (!fn) { - strcpy(g->Message, MSG(MISSING_FNAME)); - return NULL; - } // endif fn + if (GetIntegerTableOption(g, topt, "Multiple", 0)) { + strcpy(g->Message, "Cannot find column definition for multiple table"); + return NULL; + } // endif Multiple +// num_max = atoi(p+1); // Max num of record to test imax = hmax = nerr = 0; - mxr = MY_MAX(0, mxr); for (i = 0; i < MAXCOL; i++) { colname[i] = NULL; @@ -131,12 +124,76 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, } // endfor i /*********************************************************************/ - /* Open the input file. */ + /* Get the CSV table description block. */ /*********************************************************************/ - PlugSetPath(filename, fn, dp); + tdp = new(g) CSVDEF; +#if defined(ZIP_SUPPORT) + tdp->Entry = GetStringTableOption(g, topt, "Entry", NULL); + tdp->Zipped = GetBooleanTableOption(g, topt, "Zipped", false); +#endif // ZIP_SUPPORT + fn = tdp->Fn = GetStringTableOption(g, topt, "Filename", NULL); + + if (!tdp->Fn) { + strcpy(g->Message, MSG(MISSING_FNAME)); + return NULL; + } // endif Fn + + if (!(tdp->Lrecl = GetIntegerTableOption(g, topt, "Lrecl", 0))) + tdp->Lrecl = 4096; + + p = GetStringTableOption(g, topt, "Separator", ","); + tdp->Sep = (strlen(p) == 2 && p[0] == '\\' && p[1] == 't') ? '\t' : *p; + +#if defined(__WIN__) + if (tdp->Sep == ',' || strnicmp(setlocale(LC_NUMERIC, NULL), "French", 6)) + dechar = '.'; + else + dechar = ','; +#else // !__WIN__ + dechar = '.'; +#endif // !__WIN__ + + sep = tdp->Sep; + tdp->Quoted = GetIntegerTableOption(g, topt, "Quoted", -1); + p = GetStringTableOption(g, topt, "Qchar", ""); + tdp->Qot = *p; + + if (tdp->Qot && tdp->Quoted < 0) + tdp->Quoted = 0; + else if (!tdp->Qot && tdp->Quoted >= 0) + tdp->Qot = '"'; - if (!(infile= global_fopen(g, MSGID_CANNOT_OPEN, filename, "r"))) - return NULL; + q = tdp->Qot; + hdr = GetBooleanTableOption(g, topt, "Header", false); + tdp->Maxerr = GetIntegerTableOption(g, topt, "Maxerr", 0); + tdp->Accept = GetBooleanTableOption(g, topt, "Accept", false); + + if (tdp->Accept && tdp->Maxerr == 0) + tdp->Maxerr = INT_MAX32; // Accept all bad lines + + mxr = MY_MAX(0, tdp->Maxerr); + + if (trace) + htrc("File %s Sep=%c Qot=%c Header=%d maxerr=%d\n", + SVP(tdp->Fn), tdp->Sep, tdp->Qot, tdp->Header, tdp->Maxerr); + + if (tdp->Zipped) { +#if defined(ZIP_SUPPORT) + tdbp = new(g)TDBCSV(tdp, new(g)ZIPFAM(tdp)); +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif // !ZIP_SUPPORT + } else + tdbp = new(g) TDBCSV(tdp, new(g) DOSFAM(tdp)); + + tdbp->SetMode(MODE_READ); + + /*********************************************************************/ + /* Open the CSV file. */ + /*********************************************************************/ + if (tdbp->OpenDB(g)) + return NULL; if (hdr) { /*******************************************************************/ @@ -144,16 +201,8 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, /*******************************************************************/ phase = 0; - if (fgets(buf, sizeof(buf), infile)) { - n = strlen(buf) + 1; - buf[n - 2] = '\0'; -#if !defined(__WIN__) - // The file can be imported from Windows - if (buf[n - 3] == '\r') - buf[n - 3] = 0; -#endif // UNIX - p = (char*)PlugSubAlloc(g, NULL, n); - memcpy(p, buf, n); + if ((rc = tdbp->ReadDB(g)) == RC_OK) { + p = PlgDBDup(g, tdbp->To_Line); //skip leading blanks for (; *p == ' '; p++) ; @@ -165,10 +214,11 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, } // endif q colname[0] = p; - } else { + } else if (rc == RC_EF) { sprintf(g->Message, MSG(FILE_IS_EMPTY), fn); goto err; - } // endif's + } else + goto err; for (i = 1; *p; p++) if (phase == 1 && *p == q) { @@ -201,15 +251,8 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, /*******************************************************************/ /* Now start the reading process. Read one line. */ /*******************************************************************/ - if (fgets(buf, sizeof(buf), infile)) { - n = strlen(buf); - buf[n - 1] = '\0'; -#if !defined(__WIN__) - // The file can be imported from Windows - if (buf[n - 2] == '\r') - buf[n - 2] = 0; -#endif // UNIX - } else if (feof(infile)) { + if ((rc = tdbp->ReadDB(g)) == RC_OK) { + } else if (rc == RC_EF) { sprintf(g->Message, MSG(EOF_AFTER_LINE), num_read -1); break; } else { @@ -222,7 +265,7 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, /*******************************************************************/ i = n = phase = blank = digit = dec = 0; - for (p = buf; *p; p++) + for (p = tdbp->To_Line; *p; p++) if (*p == sep) { if (phase != 1) { if (i == MAXCOL - 1) { @@ -331,7 +374,7 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, htrc("\n"); } // endif trace - fclose(infile); + tdbp->CloseDB(g); skipit: if (trace) @@ -381,7 +424,7 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, return qrp; err: - fclose(infile); + tdbp->CloseDB(g); return NULL; } // end of CSVCColumns @@ -458,20 +501,27 @@ PTDB CSVDEF::GetTable(PGLOBAL g, MODE mode) /*******************************************************************/ /* Allocate a file processing class of the proper type. */ /*******************************************************************/ - if (map) { + if (Zipped) { +#if defined(ZIP_SUPPORT) + txfp = new(g) ZIPFAM(this); +#else // !ZIP_SUPPORT + strcpy(g->Message, "ZIP not supported"); + return NULL; +#endif // !ZIP_SUPPORT + } else if (map) { // Should be now compatible with UNIX txfp = new(g) MAPFAM(this); } else if (Compressed) { -#if defined(ZIP_SUPPORT) +#if defined(GZ_SUPPORT) if (Compressed == 1) - txfp = new(g) ZIPFAM(this); + txfp = new(g) GZFAM(this); else txfp = new(g) ZLBFAM(this); -#else // !ZIP_SUPPORT +#else // !GZ_SUPPORT strcpy(g->Message, "Compress not supported"); return NULL; -#endif // !ZIP_SUPPORT +#endif // !GZ_SUPPORT } else txfp = new(g) DOSFAM(this); @@ -498,7 +548,7 @@ PTDB CSVDEF::GetTable(PGLOBAL g, MODE mode) if (map) { txfp = new(g) MBKFAM(this); } else if (Compressed) { -#if defined(ZIP_SUPPORT) +#if defined(GZ_SUPPORT) if (Compressed == 1) txfp = new(g) ZBKFAM(this); else { @@ -506,7 +556,7 @@ PTDB CSVDEF::GetTable(PGLOBAL g, MODE mode) ((PZLBFAM)txfp)->SetOptimized(To_Pos != NULL); } // endelse #else - sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "GZ"); return NULL; #endif } else @@ -830,8 +880,9 @@ bool TDBCSV::SkipHeader(PGLOBAL g) /***********************************************************************/ int TDBCSV::ReadBuffer(PGLOBAL g) { - char *p1, *p2, *p = NULL; - int i, n, len, rc = Txfp->ReadBuffer(g); + //char *p1, *p2, *p = NULL; + char *p2, *p = NULL; + int i, n, len, rc = Txfp->ReadBuffer(g); bool bad = false; if (trace > 1) @@ -846,14 +897,23 @@ int TDBCSV::ReadBuffer(PGLOBAL g) for (i = 0; i < Fields; i++) { if (!bad) { if (Qot && *p2 == Qot) { // Quoted field - for (n = 0, p1 = ++p2; (p = strchr(p1, Qot)); p1 = p + 2) - if (*(p + 1) == Qot) - n++; // Doubled internal quotes - else - break; // Final quote + //for (n = 0, p1 = ++p2; (p = strchr(p1, Qot)); p1 = p + 2) + // if (*(p + 1) == Qot) + // n++; // Doubled internal quotes + // else + // break; // Final quote + + for (n = 0, p = ++p2; p; p++) + if (*p == Qot || *p == '\\') { + if (*(++p) == Qot) + n++; // Escaped internal quotes + else if (*(p - 1) == Qot) + break; // Final quote + } // endif *p if (p) { - len = p++ - p2; + //len = p++ - p2; + len = p - p2 - 1;; // if (Sep != ' ') // for (; *p == ' '; p++) ; // Skip blanks @@ -873,10 +933,12 @@ int TDBCSV::ReadBuffer(PGLOBAL g) if (n) { int j, k; - // Suppress the double of internal quotes + // Suppress the escape of internal quotes for (j = k = 0; j < len; j++, k++) { - if (p2[j] == Qot) - j++; // skip first one + if (p2[j] == Qot || (p2[j] == '\\' && p2[j + 1] == Qot)) + j++; // skip escape char + else if (p2[j] == '\\') + p2[k++] = p2[j++]; // avoid \\Qot p2[k] = p2[j]; } // endfor i, j @@ -1464,21 +1526,16 @@ void CSVCOL::WriteColumn(PGLOBAL g) /* TDBCCL class constructor. */ /***********************************************************************/ TDBCCL::TDBCCL(PCSVDEF tdp) : TDBCAT(tdp) - { - Fn = tdp->GetFn(); - Hdr = tdp->Header; - Mxr = tdp->Maxerr; - Qtd = tdp->Quoted; - Sep = tdp->Sep; - } // end of TDBCCL constructor +{ + Topt = tdp->GetTopt(); +} // end of TDBCCL constructor /***********************************************************************/ /* GetResult: Get the list the CSV file columns. */ /***********************************************************************/ PQRYRES TDBCCL::GetResult(PGLOBAL g) { - return CSVColumns(g, ((PTABDEF)To_Def)->GetPath(), - Fn, Sep, Qtd, Hdr, Mxr, false); + return CSVColumns(g, ((PTABDEF)To_Def)->GetPath(), Topt, false); } // end of GetResult /* ------------------------ End of TabFmt ---------------------------- */ diff --git a/storage/connect/tabfmt.h b/storage/connect/tabfmt.h index ce80a276cdc..5ce8d399a64 100644 --- a/storage/connect/tabfmt.h +++ b/storage/connect/tabfmt.h @@ -1,7 +1,7 @@ /*************** TabFmt H Declares Source Code File (.H) ***************/ -/* Name: TABFMT.H Version 2.4 */ +/* Name: TABFMT.H Version 2.5 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 2001-2014 */ +/* (C) Copyright to the author Olivier BERTRAND 2001-2016 */ /* */ /* This file contains the CSV and FMT classes declares. */ /***********************************************************************/ @@ -13,8 +13,7 @@ typedef class TDBFMT *PTDBFMT; /***********************************************************************/ /* Functions used externally. */ /***********************************************************************/ -PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, - char q, int hdr, int mxr, bool info); +PQRYRES CSVColumns(PGLOBAL g, char *dp, PTOS topt, bool info); /***********************************************************************/ /* CSV table. */ @@ -22,7 +21,8 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, class DllExport CSVDEF : public DOSDEF { /* Logical table description */ friend class TDBCSV; friend class TDBCCL; - public: + friend PQRYRES CSVColumns(PGLOBAL, char *, PTOS, bool); +public: // Constructor CSVDEF(void); @@ -50,9 +50,10 @@ class DllExport CSVDEF : public DOSDEF { /* Logical table description */ /* This is the DOS/UNIX Access Method class declaration for files */ /* that are CSV files with columns separated by the Sep character. */ /***********************************************************************/ -class TDBCSV : public TDBDOS { +class DllExport TDBCSV : public TDBDOS { friend class CSVCOL; - public: + friend PQRYRES CSVColumns(PGLOBAL, char *, PTOS, bool); +public: // Constructor TDBCSV(PCSVDEF tdp, PTXF txfp); TDBCSV(PGLOBAL g, PTDBCSV tdbp); @@ -101,7 +102,7 @@ class TDBCSV : public TDBDOS { /* Class CSVCOL: CSV access method column descriptor. */ /* This A.M. is used for Comma Separated V(?) files. */ /***********************************************************************/ -class CSVCOL : public DOSCOL { +class DllExport CSVCOL : public DOSCOL { friend class TDBCSV; friend class TDBFMT; public: @@ -129,7 +130,7 @@ class CSVCOL : public DOSCOL { /* This is the DOS/UNIX Access Method class declaration for files */ /* whose record format is described by a Format keyword. */ /***********************************************************************/ -class TDBFMT : public TDBCSV { +class DllExport TDBFMT : public TDBCSV { friend class CSVCOL; //friend class FMTCOL; public: @@ -173,7 +174,7 @@ class TDBFMT : public TDBCSV { /***********************************************************************/ /* This is the class declaration for the CSV catalog table. */ /***********************************************************************/ -class TDBCCL : public TDBCAT { +class DllExport TDBCCL : public TDBCAT { public: // Constructor TDBCCL(PCSVDEF tdp); @@ -183,11 +184,7 @@ class TDBCCL : public TDBCAT { virtual PQRYRES GetResult(PGLOBAL g); // Members - char *Fn; // The CSV file (path) name - bool Hdr; // true if first line contains headers - int Mxr; // Maximum number of bad records - int Qtd; // Quoting level for quoted fields - char Sep; // Separator for standard CSV files - }; // end of class TDBCCL + PTOS Topt; +}; // end of class TDBCCL /* ------------------------- End of TabFmt.H ------------------------- */ diff --git a/storage/connect/tabjdbc.cpp b/storage/connect/tabjdbc.cpp index e398523892f..912e6c7d530 100644 --- a/storage/connect/tabjdbc.cpp +++ b/storage/connect/tabjdbc.cpp @@ -110,6 +110,7 @@ bool JDBCDEF::SetParms(PJPARM sjp) sjp->Url= Url; sjp->User= Username; sjp->Pwd= Password; +//sjp->Properties = Prop; return true; } // end of SetParms @@ -234,6 +235,7 @@ bool JDBCDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) Read_Only = true; Wrapname = GetStringCatInfo(g, "Wrapper", NULL); +//Prop = GetStringCatInfo(g, "Properties", NULL); Tabcat = GetStringCatInfo(g, "Qualifier", NULL); Tabcat = GetStringCatInfo(g, "Catalog", Tabcat); Tabschema = GetStringCatInfo(g, "Dbname", NULL); @@ -337,6 +339,7 @@ TDBJDBC::TDBJDBC(PJDBCDEF tdp) : TDBASE(tdp) Schema = tdp->Tabschema; Ops.User = tdp->Username; Ops.Pwd = tdp->Password; +// Ops.Properties = tdp->Prop; Catalog = tdp->Tabcat; Srcdef = tdp->Srcdef; Qrystr = tdp->Qrystr; @@ -356,6 +359,7 @@ TDBJDBC::TDBJDBC(PJDBCDEF tdp) : TDBASE(tdp) Ops.Url = NULL; Ops.User = NULL; Ops.Pwd = NULL; +// Ops.Properties = NULL; Catalog = NULL; Srcdef = NULL; Qrystr = NULL; diff --git a/storage/connect/tabjdbc.h b/storage/connect/tabjdbc.h index 7244ebd3832..fee8223abaf 100644 --- a/storage/connect/tabjdbc.h +++ b/storage/connect/tabjdbc.h @@ -58,6 +58,7 @@ protected: PSZ Tabschema; /* External table schema */ PSZ Username; /* User connect name */ PSZ Password; /* Password connect info */ +//PSZ Prop; /* Connection Properties */ PSZ Tabcat; /* External table catalog */ PSZ Tabtype; /* External table type */ PSZ Colpat; /* Catalog column pattern */ diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp index 17260836371..1b9ce8b64c9 100644 --- a/storage/connect/tabjson.cpp +++ b/storage/connect/tabjson.cpp @@ -1,6 +1,6 @@ /************* tabjson C++ Program Source Code File (.CPP) *************/ -/* PROGRAM NAME: tabjson Version 1.1 */ -/* (C) Copyright to the author Olivier BERTRAND 2014 - 2015 */ +/* PROGRAM NAME: tabjson Version 1.3 */ +/* (C) Copyright to the author Olivier BERTRAND 2014 - 2016 */ /* This program are the JSON class DB execution routines. */ /***********************************************************************/ @@ -25,6 +25,9 @@ //#include "resource.h" // for IDS_COLUMNS #include "tabjson.h" #include "filamap.h" +#if defined(GZ_SUPPORT) +#include "filamgz.h" +#endif // GZ_SUPPORT #if defined(ZIP_SUPPORT) #include "filamzip.h" #endif // ZIP_SUPPORT @@ -67,7 +70,7 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info) static XFLD fldtyp[] = {FLD_NAME, FLD_TYPE, FLD_TYPENAME, FLD_PREC, FLD_LENGTH, FLD_SCALE, FLD_NULL, FLD_FORMAT}; static unsigned int length[] = {0, 6, 8, 10, 10, 6, 6, 0}; - char *fn, colname[65], fmt[129]; + char colname[65], fmt[129]; int i, j, lvl, n = 0; int ncol = sizeof(buftyp) / sizeof(int); PVAL valp; @@ -91,19 +94,29 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info) goto skipit; } // endif info - /*********************************************************************/ + if (GetIntegerTableOption(g, topt, "Multiple", 0)) { + strcpy(g->Message, "Cannot find column definition for multiple table"); + return NULL; + } // endif Multiple + + /*********************************************************************/ /* Open the input file. */ /*********************************************************************/ - if (!(fn = GetStringTableOption(g, topt, "Filename", NULL))) { - strcpy(g->Message, MSG(MISSING_FNAME)); - return NULL; - } else { - lvl = GetIntegerTableOption(g, topt, "Level", 0); - lvl = (lvl < 0) ? 0 : (lvl > 16) ? 16 : lvl; - } // endif fn + lvl = GetIntegerTableOption(g, topt, "Level", 0); + lvl = (lvl < 0) ? 0 : (lvl > 16) ? 16 : lvl; tdp = new(g) JSONDEF; - tdp->Fn = fn; +#if defined(ZIP_SUPPORT) + tdp->Entry = GetStringTableOption(g, topt, "Entry", NULL); + tdp->Zipped = GetBooleanTableOption(g, topt, "Zipped", false); +#endif // ZIP_SUPPORT + tdp->Fn = GetStringTableOption(g, topt, "Filename", NULL); + + if (!tdp->Fn) { + strcpy(g->Message, MSG(MISSING_FNAME)); + return NULL; + } // endif Fn + tdp->Database = SetPath(g, db); tdp->Objname = GetStringTableOption(g, topt, "Object", NULL); tdp->Base = GetIntegerTableOption(g, topt, "Base", 0) ? 1 : 0; @@ -114,7 +127,15 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info) tdp->Fn, tdp->Objname, tdp->Pretty, lvl); if (tdp->Pretty == 2) { - tjsp = new(g) TDBJSON(tdp, new(g) MAPFAM(tdp)); + if (tdp->Zipped) { +#if defined(ZIP_SUPPORT) + tjsp = new(g) TDBJSON(tdp, new(g) ZIPFAM(tdp)); +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif // !ZIP_SUPPORT + } else + tjsp = new(g) TDBJSON(tdp, new(g) MAPFAM(tdp)); if (tjsp->MakeDocument(g)) return NULL; @@ -127,10 +148,33 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info) } // endif lrecl tdp->Ending = GetIntegerTableOption(g, topt, "Ending", CRLF); - tjnp = new(g) TDBJSN(tdp, new(g) DOSFAM(tdp)); + + if (tdp->Zipped) { +#if defined(ZIP_SUPPORT) + tjnp = new(g)TDBJSN(tdp, new(g)ZIPFAM(tdp)); +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif // !ZIP_SUPPORT + } else + tjnp = new(g) TDBJSN(tdp, new(g) DOSFAM(tdp)); + tjnp->SetMode(MODE_READ); - if (tjnp->OpenDB(g)) +#if USE_G + // Allocate the parse work memory + PGLOBAL G = (PGLOBAL)PlugSubAlloc(g, NULL, sizeof(GLOBAL)); + memset(G, 0, sizeof(GLOBAL)); + G->Sarea_Size = tdp->Lrecl * 10; + G->Sarea = PlugSubAlloc(g, NULL, G->Sarea_Size); + PlugSubSet(G, G->Sarea, G->Sarea_Size); + G->jump_level = -1; + tjnp->SetG(G); +#else + tjnp->SetG(g); +#endif + + if (tjnp->OpenDB(g)) return NULL; switch (tjnp->ReadDB(g)) { @@ -395,16 +439,23 @@ PTDB JSONDEF::GetTable(PGLOBAL g, MODE m) !(tmp == TMP_FORCE && (m == MODE_UPDATE || m == MODE_DELETE)); - if (Compressed) { + if (Zipped) { #if defined(ZIP_SUPPORT) + txfp = new(g) ZIPFAM(this); +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif // !ZIP_SUPPORT + } else if (Compressed) { +#if defined(GZ_SUPPORT) if (Compressed == 1) - txfp = new(g) ZIPFAM(this); + txfp = new(g) GZFAM(this); else txfp = new(g) ZLBFAM(this); -#else // !ZIP_SUPPORT - sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); +#else // !GZ_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "GZ"); return NULL; -#endif // !ZIP_SUPPORT +#endif // !GZ_SUPPORT } else if (map) txfp = new(g) MAPFAM(this); else @@ -426,7 +477,16 @@ PTDB JSONDEF::GetTable(PGLOBAL g, MODE m) ((TDBJSN*)tdbp)->G = g; #endif } else { - txfp = new(g) MAPFAM(this); + if (Zipped) { +#if defined(ZIP_SUPPORT) + txfp = new(g)ZIPFAM(this); +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif // !ZIP_SUPPORT + } else + txfp = new(g) MAPFAM(this); + tdbp = new(g) TDBJSON(this, txfp); ((TDBJSON*)tdbp)->G = g; } // endif Pretty diff --git a/storage/connect/tabjson.h b/storage/connect/tabjson.h index f7cb74c3c4d..c9d30d48f2a 100644 --- a/storage/connect/tabjson.h +++ b/storage/connect/tabjson.h @@ -78,7 +78,8 @@ public: virtual AMT GetAmType(void) {return TYPE_AM_JSN;} virtual bool SkipHeader(PGLOBAL g); virtual PTDB Duplicate(PGLOBAL g) {return (PTDB)new(g) TDBJSN(this);} - PJSON GetRow(void) {return Row;} + PJSON GetRow(void) {return Row;} + void SetG(PGLOBAL g) {G = g;} // Methods virtual PTDB CopyOne(PTABS t); diff --git a/storage/connect/tabxml.cpp b/storage/connect/tabxml.cpp index 57d204a4286..3b8229fcf51 100644 --- a/storage/connect/tabxml.cpp +++ b/storage/connect/tabxml.cpp @@ -1,9 +1,9 @@ /************* Tabxml C++ Program Source Code File (.CPP) **************/ /* PROGRAM NAME: TABXML */ /* ------------- */ -/* Version 2.8 */ +/* Version 2.9 */ /* */ -/* Author Olivier BERTRAND 2007 - 2015 */ +/* Author Olivier BERTRAND 2007 - 2016 */ /* */ /* This program are the XML tables classes using MS-DOM or libxml2. */ /***********************************************************************/ @@ -136,7 +136,12 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info) goto skipit; } // endif info - /*********************************************************************/ + if (GetIntegerTableOption(g, topt, "Multiple", 0)) { + strcpy(g->Message, "Cannot find column definition for multiple table"); + return NULL; + } // endif Multiple + + /*********************************************************************/ /* Open the input file. */ /*********************************************************************/ if (!(fn = GetStringTableOption(g, topt, "Filename", NULL))) { @@ -154,6 +159,8 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info) tdp->Fn = fn; tdp->Database = SetPath(g, db); tdp->Tabname = tab; + tdp->Zipped = GetBooleanTableOption(g, topt, "Zipped", false); + tdp->Entry = GetStringTableOption(g, topt, "Entry", NULL); if (!(op = GetStringTableOption(g, topt, "Xmlsup", NULL))) #if defined(__WIN__) @@ -204,7 +211,8 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info) while (true) { if (!vp->atp && - !(node = (vp->nl) ? vp->nl->GetItem(g, vp->k++, node) : NULL)) + !(node = (vp->nl) ? vp->nl->GetItem(g, vp->k++, tdp->Usedom ? node : NULL) + : NULL)) if (j) { vp = lvlp[--j]; @@ -254,7 +262,8 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info) if (j < lvl && ok) { vp = lvlp[j+1]; vp->k = 0; - vp->atp = node->GetAttribute(g, NULL); + vp->pn = node; + vp->atp = node->GetAttribute(g, NULL); vp->nl = node->GetChildElements(g); if (tdp->Usedom && vp->nl->GetLength() == 1) { @@ -265,7 +274,7 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info) if (vp->atp || vp->b) { if (!vp->atp) - node = vp->nl->GetItem(g, vp->k++, node); + node = vp->nl->GetItem(g, vp->k++, tdp->Usedom ? node : NULL); strncat(fmt, colname, XLEN(fmt)); strncat(fmt, "/", XLEN(fmt)); @@ -424,11 +433,14 @@ XMLDEF::XMLDEF(void) DefNs = NULL; Attrib = NULL; Hdattr = NULL; + Entry = NULL; Coltype = 1; Limit = 0; Header = 0; Xpand = false; Usedom = false; + Zipped = false; + Mulentries = false; } // end of XMLDEF constructor /***********************************************************************/ @@ -507,7 +519,14 @@ bool XMLDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) // Get eventual table node attribute Attrib = GetStringCatInfo(g, "Attribute", NULL); Hdattr = GetStringCatInfo(g, "HeadAttr", NULL); - return false; + + // Specific for zipped files + if ((Zipped = GetBoolCatInfo("Zipped", false))) + Mulentries = ((Entry = GetStringCatInfo(g, "Entry", NULL))) + ? strchr(Entry, '*') || strchr(Entry, '?') + : GetBoolCatInfo("Mulentries", false); + + return false; } // end of DefineAM /***********************************************************************/ @@ -547,6 +566,7 @@ TDBXML::TDBXML(PXMLDEF tdp) : TDBASE(tdp) Xfile = tdp->Fn; Enc = tdp->Encoding; Tabname = tdp->Tabname; +#if 0 // why all these? Rowname = (tdp->Rowname) ? tdp->Rowname : NULL; Colname = (tdp->Colname) ? tdp->Colname : NULL; Mulnode = (tdp->Mulnode) ? tdp->Mulnode : NULL; @@ -555,10 +575,22 @@ TDBXML::TDBXML(PXMLDEF tdp) : TDBASE(tdp) DefNs = (tdp->DefNs) ? tdp->DefNs : NULL; Attrib = (tdp->Attrib) ? tdp->Attrib : NULL; Hdattr = (tdp->Hdattr) ? tdp->Hdattr : NULL; - Coltype = tdp->Coltype; +#endif // 0 + Rowname = tdp->Rowname; + Colname = tdp->Colname; + Mulnode = tdp->Mulnode; + XmlDB = tdp->XmlDB; + Nslist = tdp->Nslist; + DefNs = tdp->DefNs; + Attrib = tdp->Attrib; + Hdattr = tdp->Hdattr; + Entry = tdp->Entry; + Coltype = tdp->Coltype; Limit = tdp->Limit; Xpand = tdp->Xpand; - Changed = false; + Zipped = tdp->Zipped; + Mulentries = tdp->Mulentries; + Changed = false; Checked = false; NextSame = false; NewRow = false; @@ -600,10 +632,13 @@ TDBXML::TDBXML(PTDBXML tdbp) : TDBASE(tdbp) DefNs = tdbp->DefNs; Attrib = tdbp->Attrib; Hdattr = tdbp->Hdattr; - Coltype = tdbp->Coltype; + Entry = tdbp->Entry; + Coltype = tdbp->Coltype; Limit = tdbp->Limit; Xpand = tdbp->Xpand; - Changed = tdbp->Changed; + Zipped = tdbp->Zipped; + Mulentries = tdbp->Mulentries; + Changed = tdbp->Changed; Checked = tdbp->Checked; NextSame = tdbp->NextSame; NewRow = tdbp->NewRow; @@ -681,7 +716,7 @@ int TDBXML::LoadTableFile(PGLOBAL g, char *filename) /*********************************************************************/ /* Firstly we check whether this file have been already loaded. */ /*********************************************************************/ - if (Mode == MODE_READ || Mode == MODE_ANY) + if ((Mode == MODE_READ || Mode == MODE_ANY) && !Zipped) for (fp = dup->Openlist; fp; fp = fp->Next) if (fp->Type == type && fp->Length && fp->Count) if (!stricmp(fp->Fname, filename)) @@ -703,7 +738,7 @@ int TDBXML::LoadTableFile(PGLOBAL g, char *filename) return RC_FX; // Initialize the implementation - if (Docp->Initialize(g)) { + if (Docp->Initialize(g, Entry, Zipped)) { sprintf(g->Message, MSG(INIT_FAILED), (Usedom) ? "DOM" : "libxml2"); return RC_FX; } // endif init @@ -712,7 +747,7 @@ int TDBXML::LoadTableFile(PGLOBAL g, char *filename) htrc("TDBXML: parsing %s rc=%d\n", filename, rc); // Parse the XML file - if (Docp->ParseFile(filename)) { + if (Docp->ParseFile(g, filename)) { // Does the file exist? int h= global_open(g, MSGID_NONE, filename, _O_RDONLY); diff --git a/storage/connect/tabxml.h b/storage/connect/tabxml.h index 7ba3166881d..6c586d79dec 100644 --- a/storage/connect/tabxml.h +++ b/storage/connect/tabxml.h @@ -1,7 +1,7 @@ /*************** Tabxml H Declares Source Code File (.H) ***************/ -/* Name: TABXML.H Version 1.6 */ +/* Name: TABXML.H Version 1.7 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 2007-2015 */ +/* (C) Copyright to the author Olivier BERTRAND 2007-2016 */ /* */ /* This file contains the XML table classes declares. */ /***********************************************************************/ @@ -42,12 +42,15 @@ class DllExport XMLDEF : public TABDEF { /* Logical table description */ char *DefNs; /* Dummy name of default namespace */ char *Attrib; /* Table node attributes */ char *Hdattr; /* Header node attributes */ - int Coltype; /* Default column type */ + char *Entry; /* Zip entry name or pattern */ + int Coltype; /* Default column type */ int Limit; /* Limit of multiple values */ int Header; /* n first rows are header rows */ bool Xpand; /* Put multiple tags in several rows */ bool Usedom; /* True: DOM, False: libxml2 */ - }; // end of XMLDEF + bool Zipped; /* True: Zipped XML file(s) */ + bool Mulentries; /* True: multiple entries in zip file*/ +}; // end of XMLDEF #if defined(INCLUDE_TDBXML) /***********************************************************************/ @@ -122,7 +125,9 @@ class DllExport TDBXML : public TDBASE { bool Bufdone; // True when column buffers allocated bool Nodedone; // True when column nodes allocated bool Void; // True if the file does not exist - char *Xfile; // The XML file + bool Zipped; // True if Zipped XML file(s) + bool Mulentries; // True if multiple entries in zip file + char *Xfile; // The XML file char *Enc; // New XML table file encoding char *Tabname; // Name of Table node char *Rowname; // Name of first level nodes @@ -133,7 +138,8 @@ class DllExport TDBXML : public TDBASE { char *DefNs; // Dummy name of default namespace char *Attrib; // Table node attribut(s) char *Hdattr; // Header node attribut(s) - int Coltype; // Default column type + char *Entry; // Zip entry name or pattern + int Coltype; // Default column type int Limit; // Limit of multiple values int Header; // n first rows are header rows int Multiple; // If multiple files diff --git a/storage/connect/tabzip.cpp b/storage/connect/tabzip.cpp new file mode 100644 index 00000000000..11f414ee154 --- /dev/null +++ b/storage/connect/tabzip.cpp @@ -0,0 +1,230 @@ +/************* TabZip C++ Program Source Code File (.CPP) **************/ +/* PROGRAM NAME: TABZIP Version 1.0 */ +/* (C) Copyright to the author Olivier BERTRAND 2016 */ +/* This program are the TABZIP class DB execution routines. */ +/***********************************************************************/ + +/***********************************************************************/ +/* Include relevant sections of the MariaDB header file. */ +/***********************************************************************/ +#include <my_global.h> + +/***********************************************************************/ +/* Include application header files: */ +/* global.h is header containing all global declarations. */ +/* plgdbsem.h is header containing the DB application declarations. */ +/* (x)table.h is header containing the TDBASE declarations. */ +/* tabzip.h is header containing the TABZIP classes declarations. */ +/***********************************************************************/ +#include "global.h" +#include "plgdbsem.h" +#include "xtable.h" +#include "filamtxt.h" +#include "filamzip.h" +#include "resource.h" // for IDS_COLUMNS +#include "tabdos.h" +#include "tabzip.h" + +/* -------------------------- Class ZIPDEF --------------------------- */ + +/************************************************************************/ +/* DefineAM: define specific AM block values. */ +/************************************************************************/ +bool ZIPDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) +{ +//target = GetStringCatInfo(g, "Target", NULL); + return DOSDEF::DefineAM(g, "ZIP", poff); +} // end of DefineAM + +/***********************************************************************/ +/* GetTable: makes a new Table Description Block. */ +/***********************************************************************/ +PTDB ZIPDEF::GetTable(PGLOBAL g, MODE m) +{ + return new(g) TDBZIP(this); +} // end of GetTable + +/* ------------------------------------------------------------------- */ + +/***********************************************************************/ +/* Implementation of the TDBZIP class. */ +/***********************************************************************/ +TDBZIP::TDBZIP(PZIPDEF tdp) : TDBASE(tdp) +{ + zipfile = NULL; + zfn = tdp->Fn; +//target = tdp->target; + nexterr = UNZ_OK; +} // end of TDBZIP standard constructor + +/***********************************************************************/ +/* Allocate ZIP column description block. */ +/***********************************************************************/ +PCOL TDBZIP::MakeCol(PGLOBAL g, PCOLDEF cdp, PCOL cprec, int n) +{ + return new(g) ZIPCOL(cdp, this, cprec, n); +} // end of MakeCol + +/***********************************************************************/ +/* open a zip file. */ +/* param: filename path and the filename of the zip file to open. */ +/* return: true if open, false otherwise. */ +/***********************************************************************/ +bool TDBZIP::open(PGLOBAL g, const char *filename) +{ + if (!zipfile && !(zipfile = unzOpen64(filename))) + sprintf(g->Message, "Zipfile open error"); + + return (zipfile == NULL); +} // end of open + +/***********************************************************************/ +/* Close the zip file. */ +/***********************************************************************/ +void TDBZIP::close() +{ + if (zipfile) { + unzClose(zipfile); + zipfile = NULL; + } // endif zipfile + +} // end of close + +/***********************************************************************/ +/* ZIP Cardinality: returns table size in number of rows. */ +/***********************************************************************/ +int TDBZIP::Cardinality(PGLOBAL g) +{ + if (!g) + return 1; + else if (Cardinal < 0) { + if (!open(g, zfn)) { + unz_global_info64 ginfo; + int err = unzGetGlobalInfo64(zipfile, &ginfo); + + Cardinal = (err == UNZ_OK) ? ginfo.number_entry : 0; + } else + Cardinal = 0; + + } // endif Cardinal + + return Cardinal; +} // end of Cardinality + +/***********************************************************************/ +/* ZIP GetMaxSize: returns file size estimate in number of lines. */ +/***********************************************************************/ +int TDBZIP::GetMaxSize(PGLOBAL g) +{ + if (MaxSize < 0) + MaxSize = Cardinality(g); + + return MaxSize; +} // end of GetMaxSize + +/***********************************************************************/ +/* ZIP Access Method opening routine. */ +/***********************************************************************/ +bool TDBZIP::OpenDB(PGLOBAL g) +{ + if (Use == USE_OPEN) + // Table already open + return false; + + Use = USE_OPEN; // To be clean + return open(g, zfn); +} // end of OpenDB + +/***********************************************************************/ +/* ReadDB: Data Base read routine for ZIP access method. */ +/***********************************************************************/ +int TDBZIP::ReadDB(PGLOBAL g) +{ + if (nexterr == UNZ_END_OF_LIST_OF_FILE) + return RC_EF; + else if (nexterr != UNZ_OK) { + sprintf(g->Message, "unzGoToNextFile error %d", nexterr); + return RC_FX; + } // endif nexterr + + int err = unzGetCurrentFileInfo64(zipfile, &finfo, fn, + sizeof(fn), NULL, 0, NULL, 0); + + if (err != UNZ_OK) { + sprintf(g->Message, "unzGetCurrentFileInfo64 error %d", err); + return RC_FX; + } // endif err + + nexterr = unzGoToNextFile(zipfile); + return RC_OK; +} // end of ReadDB + +/***********************************************************************/ +/* WriteDB: Data Base write routine for ZIP access method. */ +/***********************************************************************/ +int TDBZIP::WriteDB(PGLOBAL g) +{ + strcpy(g->Message, "ZIP tables are read only"); + return RC_FX; +} // end of WriteDB + +/***********************************************************************/ +/* Data Base delete line routine for ZIP access method. */ +/***********************************************************************/ +int TDBZIP::DeleteDB(PGLOBAL g, int irc) +{ + strcpy(g->Message, "Delete not enabled for ZIP tables"); + return RC_FX; +} // end of DeleteDB + +/***********************************************************************/ +/* Data Base close routine for ZIP access method. */ +/***********************************************************************/ +void TDBZIP::CloseDB(PGLOBAL g) +{ + close(); + Use = USE_READY; // Just to be clean +} // end of CloseDB + +/* ---------------------------- ZIPCOL ------------------------------- */ + +/***********************************************************************/ +/* ZIPCOL public constructor. */ +/***********************************************************************/ +ZIPCOL::ZIPCOL(PCOLDEF cdp, PTDB tdbp, PCOL cprec, int i, PSZ am) + : COLBLK(cdp, tdbp, i) +{ + if (cprec) { + Next = cprec->GetNext(); + cprec->SetNext(this); + } else { + Next = tdbp->GetColumns(); + tdbp->SetColumns(this); + } // endif cprec + + Tdbz = (TDBZIP*)tdbp; + flag = cdp->GetOffset(); +} // end of ZIPCOL constructor + +/***********************************************************************/ +/* ReadColumn: */ +/***********************************************************************/ +void ZIPCOL::ReadColumn(PGLOBAL g) +{ + switch (flag) { + case 1: + Value->SetValue(Tdbz->finfo.compressed_size); + break; + case 2: + Value->SetValue(Tdbz->finfo.uncompressed_size); + break; + case 3: + Value->SetValue((int)Tdbz->finfo.compression_method); + break; + default: + Value->SetValue_psz((PSZ)Tdbz->fn); + } // endswitch flag + +} // end of ReadColumn + +/* -------------------------- End of tabzip -------------------------- */ diff --git a/storage/connect/tabzip.h b/storage/connect/tabzip.h new file mode 100644 index 00000000000..6f1735258e7 --- /dev/null +++ b/storage/connect/tabzip.h @@ -0,0 +1,100 @@ +/*************** tabzip H Declares Source Code File (.H) ***************/ +/* Name: tabzip.h Version 1.0 */ +/* */ +/* (C) Copyright to the author Olivier BERTRAND 2016 */ +/* */ +/* This file contains the ZIP classe declares. */ +/***********************************************************************/ +#include "osutil.h" +#include "block.h" +#include "colblk.h" +#include "xtable.h" +#include "unzip.h" + +typedef class ZIPDEF *PZIPDEF; +typedef class TDBZIP *PTDBZIP; +typedef class ZIPCOL *PZIPCOL; + +/***********************************************************************/ +/* ZIP table: display info about a ZIP file. */ +/***********************************************************************/ +class DllExport ZIPDEF : public DOSDEF { /* Table description */ + friend class TDBZIP; + friend class ZIPFAM; +public: + // Constructor + ZIPDEF(void) {} + + // Implementation + virtual const char *GetType(void) {return "ZIP";} + + // Methods + virtual bool DefineAM(PGLOBAL g, LPCSTR am, int poff); + virtual PTDB GetTable(PGLOBAL g, MODE m); + +protected: + // Members + PSZ target; // The inside file to query +}; // end of ZIPDEF + +/***********************************************************************/ +/* This is the ZIP Access Method class declaration. */ +/***********************************************************************/ +class DllExport TDBZIP : public TDBASE { + friend class ZIPCOL; +public: + // Constructor + TDBZIP(PZIPDEF tdp); + + // Implementation + virtual AMT GetAmType(void) {return TYPE_AM_ZIP;} + + // Methods + virtual PCOL MakeCol(PGLOBAL g, PCOLDEF cdp, PCOL cprec, int n); + virtual int Cardinality(PGLOBAL g); + virtual int GetMaxSize(PGLOBAL g); + virtual int GetRecpos(void) {return 0;} + + // Database routines + virtual bool OpenDB(PGLOBAL g); + virtual int ReadDB(PGLOBAL g); + virtual int WriteDB(PGLOBAL g); + virtual int DeleteDB(PGLOBAL g, int irc); + virtual void CloseDB(PGLOBAL g); + +protected: + bool open(PGLOBAL g, const char *filename); + void close(void); + + // Members + unzFile zipfile; // The ZIP container file + PSZ zfn; // The ZIP file name +//PSZ target; + unz_file_info64 finfo; // The current file info + char fn[FILENAME_MAX]; // The current file name + int nexterr; // Next file error +}; // end of class TDBZIP + +/***********************************************************************/ +/* Class ZIPCOL: ZIP access method column descriptor. */ +/***********************************************************************/ +class DllExport ZIPCOL : public COLBLK { + friend class TDBZIP; +public: + // Constructors + ZIPCOL(PCOLDEF cdp, PTDB tdbp, PCOL cprec, int i, PSZ am = "ZIP"); + + // Implementation + virtual int GetAmType(void) { return TYPE_AM_ZIP; } + + // Methods + virtual void ReadColumn(PGLOBAL g); + +protected: + // Default constructor not to be used + ZIPCOL(void) {} + + // Members + TDBZIP *Tdbz; + int flag; +}; // end of class ZIPCOL diff --git a/storage/connect/unzip.c b/storage/connect/unzip.c new file mode 100644 index 00000000000..909350435a5 --- /dev/null +++ b/storage/connect/unzip.c @@ -0,0 +1,2125 @@ +/* unzip.c -- IO for uncompress .zip files using zlib + Version 1.1, February 14h, 2010 + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications of Unzip for Zip64 + Copyright (C) 2007-2008 Even Rouault + + Modifications for Zip64 support on both zip and unzip + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + + + ------------------------------------------------------------------------------------ + Decryption code comes from crypt.c by Info-ZIP but has been greatly reduced in terms of + compatibility with older software. The following is from the original crypt.c. + Code woven in by Terry Thorsen 1/2003. + + Copyright (c) 1990-2000 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2000-Apr-09 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, all these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html + + crypt.c (full version) by Info-ZIP. Last revised: [see crypt.h] + + The encryption/decryption parts of this source code (as opposed to the + non-echoing password parts) were originally written in Europe. The + whole source package can be freely distributed, including from the USA. + (Prior to January 2000, re-export from the US was a violation of US law.) + + This encryption code is a direct transcription of the algorithm from + Roger Schlafly, described by Phil Katz in the file appnote.txt. This + file (appnote.txt) is distributed with the PKZIP program (even in the + version without encryption capabilities). + + ------------------------------------------------------------------------------------ + + Changes in unzip.c + + 2007-2008 - Even Rouault - Addition of cpl_unzGetCurrentFileZStreamPos + 2007-2008 - Even Rouault - Decoration of symbol names unz* -> cpl_unz* + 2007-2008 - Even Rouault - Remove old C style function prototypes + 2007-2008 - Even Rouault - Add unzip support for ZIP64 + + Copyright (C) 2007-2008 Even Rouault + + + Oct-2009 - Mathias Svensson - Removed cpl_* from symbol names (Even Rouault added them but since this is now moved to a new project (minizip64) I renamed them again). + Oct-2009 - Mathias Svensson - Fixed problem if uncompressed size was > 4G and compressed size was <4G + should only read the compressed/uncompressed size from the Zip64 format if + the size from normal header was 0xFFFFFFFF + Oct-2009 - Mathias Svensson - Applied some bug fixes from paches recived from Gilles Vollant + Oct-2009 - Mathias Svensson - Applied support to unzip files with compression mathod BZIP2 (bzip2 lib is required) + Patch created by Daniel Borca + + Jan-2010 - back to unzip and minizip 1.0 name scheme, with compatibility layer + + Copyright (C) 1998 - 2010 Gilles Vollant, Even Rouault, Mathias Svensson + +*/ + + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#ifndef NOUNCRYPT + #define NOUNCRYPT +#endif + +#include "zlib.h" +#include "unzip.h" + +#ifdef STDC +# include <stddef.h> +# include <string.h> +# include <stdlib.h> +#endif +#ifdef NO_ERRNO_H + extern int errno; +#else +# include <errno.h> +#endif + + +#ifndef local +# define local static +#endif +/* compile with -Dlocal if your debugger can't find static symbols */ + + +#ifndef CASESENSITIVITYDEFAULT_NO +# if !defined(unix) && !defined(CASESENSITIVITYDEFAULT_YES) +# define CASESENSITIVITYDEFAULT_NO +# endif +#endif + + +#ifndef UNZ_BUFSIZE +#define UNZ_BUFSIZE (16384) +#endif + +#ifndef UNZ_MAXFILENAMEINZIP +#define UNZ_MAXFILENAMEINZIP (256) +#endif + +#ifndef ALLOC +# define ALLOC(size) (malloc(size)) +#endif +#ifndef TRYFREE +# define TRYFREE(p) {if (p) free(p);} +#endif + +#define SIZECENTRALDIRITEM (0x2e) +#define SIZEZIPLOCALHEADER (0x1e) + + +const char unz_copyright[] = + " unzip 1.01 Copyright 1998-2004 Gilles Vollant - http://www.winimage.com/zLibDll"; + +/* unz_file_info_interntal contain internal info about a file in zipfile*/ +typedef struct unz_file_info64_internal_s +{ + ZPOS64_T offset_curfile;/* relative offset of local header 8 bytes */ +} unz_file_info64_internal; + + +/* file_in_zip_read_info_s contain internal information about a file in zipfile, + when reading and decompress it */ +typedef struct +{ + char *read_buffer; /* internal buffer for compressed data */ + z_stream stream; /* zLib stream structure for inflate */ + +#ifdef HAVE_BZIP2 + bz_stream bstream; /* bzLib stream structure for bziped */ +#endif + + ZPOS64_T pos_in_zipfile; /* position in byte on the zipfile, for fseek*/ + uLong stream_initialised; /* flag set if stream structure is initialised*/ + + ZPOS64_T offset_local_extrafield;/* offset of the local extra field */ + uInt size_local_extrafield;/* size of the local extra field */ + ZPOS64_T pos_local_extrafield; /* position in the local extra field in read*/ + ZPOS64_T total_out_64; + + uLong crc32; /* crc32 of all data uncompressed */ + uLong crc32_wait; /* crc32 we must obtain after decompress all */ + ZPOS64_T rest_read_compressed; /* number of byte to be decompressed */ + ZPOS64_T rest_read_uncompressed;/*number of byte to be obtained after decomp*/ + zlib_filefunc64_32_def z_filefunc; + voidpf filestream; /* io structore of the zipfile */ + uLong compression_method; /* compression method (0==store) */ + ZPOS64_T byte_before_the_zipfile;/* byte before the zipfile, (>0 for sfx)*/ + int raw; +} file_in_zip64_read_info_s; + + +/* unz64_s contain internal information about the zipfile +*/ +typedef struct +{ + zlib_filefunc64_32_def z_filefunc; + int is64bitOpenFunction; + voidpf filestream; /* io structore of the zipfile */ + unz_global_info64 gi; /* public global information */ + ZPOS64_T byte_before_the_zipfile;/* byte before the zipfile, (>0 for sfx)*/ + ZPOS64_T num_file; /* number of the current file in the zipfile*/ + ZPOS64_T pos_in_central_dir; /* pos of the current file in the central dir*/ + ZPOS64_T current_file_ok; /* flag about the usability of the current file*/ + ZPOS64_T central_pos; /* position of the beginning of the central dir*/ + + ZPOS64_T size_central_dir; /* size of the central directory */ + ZPOS64_T offset_central_dir; /* offset of start of central directory with + respect to the starting disk number */ + + unz_file_info64 cur_file_info; /* public info about the current file in zip*/ + unz_file_info64_internal cur_file_info_internal; /* private info about it*/ + file_in_zip64_read_info_s* pfile_in_zip_read; /* structure about the current + file if we are decompressing it */ + int encrypted; + + int isZip64; + +# ifndef NOUNCRYPT + unsigned long keys[3]; /* keys defining the pseudo-random sequence */ + const z_crc_t* pcrc_32_tab; +# endif +} unz64_s; + + +#ifndef NOUNCRYPT +#include "crypt.h" +#endif + +/* =========================================================================== + Read a byte from a gz_stream; update next_in and avail_in. Return EOF + for end of file. + IN assertion: the stream s has been sucessfully opened for reading. +*/ + + +local int unz64local_getByte OF(( + const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + int *pi)); + +local int unz64local_getByte(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, int *pi) +{ + unsigned char c; + int err = (int)ZREAD64(*pzlib_filefunc_def,filestream,&c,1); + if (err==1) + { + *pi = (int)c; + return UNZ_OK; + } + else + { + if (ZERROR64(*pzlib_filefunc_def,filestream)) + return UNZ_ERRNO; + else + return UNZ_EOF; + } +} + + +/* =========================================================================== + Reads a long in LSB order from the given gz_stream. Sets +*/ +local int unz64local_getShort OF(( + const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + uLong *pX)); + +local int unz64local_getShort (const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + uLong *pX) +{ + uLong x ; + int i = 0; + int err; + + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x = (uLong)i; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((uLong)i)<<8; + + if (err==UNZ_OK) + *pX = x; + else + *pX = 0; + return err; +} + +local int unz64local_getLong OF(( + const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + uLong *pX)); + +local int unz64local_getLong (const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + uLong *pX) +{ + uLong x ; + int i = 0; + int err; + + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x = (uLong)i; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((uLong)i)<<8; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((uLong)i)<<16; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((uLong)i)<<24; + + if (err==UNZ_OK) + *pX = x; + else + *pX = 0; + return err; +} + +local int unz64local_getLong64 OF(( + const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + ZPOS64_T *pX)); + + +local int unz64local_getLong64 (const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + ZPOS64_T *pX) +{ + ZPOS64_T x ; + int i = 0; + int err; + + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x = (ZPOS64_T)i; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<8; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<16; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<24; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<32; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<40; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<48; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<56; + + if (err==UNZ_OK) + *pX = x; + else + *pX = 0; + return err; +} + +/* My own strcmpi / strcasecmp */ +local int strcmpcasenosensitive_internal (const char* fileName1, const char* fileName2) +{ + for (;;) + { + char c1=*(fileName1++); + char c2=*(fileName2++); + if ((c1>='a') && (c1<='z')) + c1 -= 0x20; + if ((c2>='a') && (c2<='z')) + c2 -= 0x20; + if (c1=='\0') + return ((c2=='\0') ? 0 : -1); + if (c2=='\0') + return 1; + if (c1<c2) + return -1; + if (c1>c2) + return 1; + } +} + + +#ifdef CASESENSITIVITYDEFAULT_NO +#define CASESENSITIVITYDEFAULTVALUE 2 +#else +#define CASESENSITIVITYDEFAULTVALUE 1 +#endif + +#ifndef STRCMPCASENOSENTIVEFUNCTION +#define STRCMPCASENOSENTIVEFUNCTION strcmpcasenosensitive_internal +#endif + +/* + Compare two filename (fileName1,fileName2). + If iCaseSenisivity = 1, comparision is case sensitivity (like strcmp) + If iCaseSenisivity = 2, comparision is not case sensitivity (like strcmpi + or strcasecmp) + If iCaseSenisivity = 0, case sensitivity is defaut of your operating system + (like 1 on Unix, 2 on Windows) + +*/ +extern int ZEXPORT unzStringFileNameCompare (const char* fileName1, + const char* fileName2, + int iCaseSensitivity) + +{ + if (iCaseSensitivity==0) + iCaseSensitivity=CASESENSITIVITYDEFAULTVALUE; + + if (iCaseSensitivity==1) + return strcmp(fileName1,fileName2); + + return STRCMPCASENOSENTIVEFUNCTION(fileName1,fileName2); +} + +#ifndef BUFREADCOMMENT +#define BUFREADCOMMENT (0x400) +#endif + +/* + Locate the Central directory of a zipfile (at the end, just before + the global comment) +*/ +local ZPOS64_T unz64local_SearchCentralDir OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream)); +local ZPOS64_T unz64local_SearchCentralDir(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream) +{ + unsigned char* buf; + ZPOS64_T uSizeFile; + ZPOS64_T uBackRead; + ZPOS64_T uMaxBack=0xffff; /* maximum size of global comment */ + ZPOS64_T uPosFound=0; + + if (ZSEEK64(*pzlib_filefunc_def,filestream,0,ZLIB_FILEFUNC_SEEK_END) != 0) + return 0; + + + uSizeFile = ZTELL64(*pzlib_filefunc_def,filestream); + + if (uMaxBack>uSizeFile) + uMaxBack = uSizeFile; + + buf = (unsigned char*)ALLOC(BUFREADCOMMENT+4); + if (buf==NULL) + return 0; + + uBackRead = 4; + while (uBackRead<uMaxBack) + { + uLong uReadSize; + ZPOS64_T uReadPos ; + int i; + if (uBackRead+BUFREADCOMMENT>uMaxBack) + uBackRead = uMaxBack; + else + uBackRead+=BUFREADCOMMENT; + uReadPos = uSizeFile-uBackRead ; + + uReadSize = ((BUFREADCOMMENT+4) < (uSizeFile-uReadPos)) ? + (BUFREADCOMMENT+4) : (uLong)(uSizeFile-uReadPos); + if (ZSEEK64(*pzlib_filefunc_def,filestream,uReadPos,ZLIB_FILEFUNC_SEEK_SET)!=0) + break; + + if (ZREAD64(*pzlib_filefunc_def,filestream,buf,uReadSize)!=uReadSize) + break; + + for (i=(int)uReadSize-3; (i--)>0;) + if (((*(buf+i))==0x50) && ((*(buf+i+1))==0x4b) && + ((*(buf+i+2))==0x05) && ((*(buf+i+3))==0x06)) + { + uPosFound = uReadPos+i; + break; + } + + if (uPosFound!=0) + break; + } + TRYFREE(buf); + return uPosFound; +} + + +/* + Locate the Central directory 64 of a zipfile (at the end, just before + the global comment) +*/ +local ZPOS64_T unz64local_SearchCentralDir64 OF(( + const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream)); + +local ZPOS64_T unz64local_SearchCentralDir64(const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream) +{ + unsigned char* buf; + ZPOS64_T uSizeFile; + ZPOS64_T uBackRead; + ZPOS64_T uMaxBack=0xffff; /* maximum size of global comment */ + ZPOS64_T uPosFound=0; + uLong uL; + ZPOS64_T relativeOffset; + + if (ZSEEK64(*pzlib_filefunc_def,filestream,0,ZLIB_FILEFUNC_SEEK_END) != 0) + return 0; + + + uSizeFile = ZTELL64(*pzlib_filefunc_def,filestream); + + if (uMaxBack>uSizeFile) + uMaxBack = uSizeFile; + + buf = (unsigned char*)ALLOC(BUFREADCOMMENT+4); + if (buf==NULL) + return 0; + + uBackRead = 4; + while (uBackRead<uMaxBack) + { + uLong uReadSize; + ZPOS64_T uReadPos; + int i; + if (uBackRead+BUFREADCOMMENT>uMaxBack) + uBackRead = uMaxBack; + else + uBackRead+=BUFREADCOMMENT; + uReadPos = uSizeFile-uBackRead ; + + uReadSize = ((BUFREADCOMMENT+4) < (uSizeFile-uReadPos)) ? + (BUFREADCOMMENT+4) : (uLong)(uSizeFile-uReadPos); + if (ZSEEK64(*pzlib_filefunc_def,filestream,uReadPos,ZLIB_FILEFUNC_SEEK_SET)!=0) + break; + + if (ZREAD64(*pzlib_filefunc_def,filestream,buf,uReadSize)!=uReadSize) + break; + + for (i=(int)uReadSize-3; (i--)>0;) + if (((*(buf+i))==0x50) && ((*(buf+i+1))==0x4b) && + ((*(buf+i+2))==0x06) && ((*(buf+i+3))==0x07)) + { + uPosFound = uReadPos+i; + break; + } + + if (uPosFound!=0) + break; + } + TRYFREE(buf); + if (uPosFound == 0) + return 0; + + /* Zip64 end of central directory locator */ + if (ZSEEK64(*pzlib_filefunc_def,filestream, uPosFound,ZLIB_FILEFUNC_SEEK_SET)!=0) + return 0; + + /* the signature, already checked */ + if (unz64local_getLong(pzlib_filefunc_def,filestream,&uL)!=UNZ_OK) + return 0; + + /* number of the disk with the start of the zip64 end of central directory */ + if (unz64local_getLong(pzlib_filefunc_def,filestream,&uL)!=UNZ_OK) + return 0; + if (uL != 0) + return 0; + + /* relative offset of the zip64 end of central directory record */ + if (unz64local_getLong64(pzlib_filefunc_def,filestream,&relativeOffset)!=UNZ_OK) + return 0; + + /* total number of disks */ + if (unz64local_getLong(pzlib_filefunc_def,filestream,&uL)!=UNZ_OK) + return 0; + if (uL != 1) + return 0; + + /* Goto end of central directory record */ + if (ZSEEK64(*pzlib_filefunc_def,filestream, relativeOffset,ZLIB_FILEFUNC_SEEK_SET)!=0) + return 0; + + /* the signature */ + if (unz64local_getLong(pzlib_filefunc_def,filestream,&uL)!=UNZ_OK) + return 0; + + if (uL != 0x06064b50) + return 0; + + return relativeOffset; +} + +/* + Open a Zip file. path contain the full pathname (by example, + on a Windows NT computer "c:\\test\\zlib114.zip" or on an Unix computer + "zlib/zlib114.zip". + If the zipfile cannot be opened (file doesn't exist or in not valid), the + return value is NULL. + Else, the return value is a unzFile Handle, usable with other function + of this unzip package. +*/ +local unzFile unzOpenInternal (const void *path, + zlib_filefunc64_32_def* pzlib_filefunc64_32_def, + int is64bitOpenFunction) +{ + unz64_s us; + unz64_s *s; + ZPOS64_T central_pos; + uLong uL; + + uLong number_disk; /* number of the current dist, used for + spaning ZIP, unsupported, always 0*/ + uLong number_disk_with_CD; /* number the the disk with central dir, used + for spaning ZIP, unsupported, always 0*/ + ZPOS64_T number_entry_CD; /* total number of entries in + the central dir + (same than number_entry on nospan) */ + + int err=UNZ_OK; + + if (unz_copyright[0]!=' ') + return NULL; + + us.z_filefunc.zseek32_file = NULL; + us.z_filefunc.ztell32_file = NULL; + if (pzlib_filefunc64_32_def==NULL) + fill_fopen64_filefunc(&us.z_filefunc.zfile_func64); + else + us.z_filefunc = *pzlib_filefunc64_32_def; + us.is64bitOpenFunction = is64bitOpenFunction; + + + + us.filestream = ZOPEN64(us.z_filefunc, + path, + ZLIB_FILEFUNC_MODE_READ | + ZLIB_FILEFUNC_MODE_EXISTING); + if (us.filestream==NULL) + return NULL; + + central_pos = unz64local_SearchCentralDir64(&us.z_filefunc,us.filestream); + if (central_pos) + { + uLong uS; + ZPOS64_T uL64; + + us.isZip64 = 1; + + if (ZSEEK64(us.z_filefunc, us.filestream, + central_pos,ZLIB_FILEFUNC_SEEK_SET)!=0) + err=UNZ_ERRNO; + + /* the signature, already checked */ + if (unz64local_getLong(&us.z_filefunc, us.filestream,&uL)!=UNZ_OK) + err=UNZ_ERRNO; + + /* size of zip64 end of central directory record */ + if (unz64local_getLong64(&us.z_filefunc, us.filestream,&uL64)!=UNZ_OK) + err=UNZ_ERRNO; + + /* version made by */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&uS)!=UNZ_OK) + err=UNZ_ERRNO; + + /* version needed to extract */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&uS)!=UNZ_OK) + err=UNZ_ERRNO; + + /* number of this disk */ + if (unz64local_getLong(&us.z_filefunc, us.filestream,&number_disk)!=UNZ_OK) + err=UNZ_ERRNO; + + /* number of the disk with the start of the central directory */ + if (unz64local_getLong(&us.z_filefunc, us.filestream,&number_disk_with_CD)!=UNZ_OK) + err=UNZ_ERRNO; + + /* total number of entries in the central directory on this disk */ + if (unz64local_getLong64(&us.z_filefunc, us.filestream,&us.gi.number_entry)!=UNZ_OK) + err=UNZ_ERRNO; + + /* total number of entries in the central directory */ + if (unz64local_getLong64(&us.z_filefunc, us.filestream,&number_entry_CD)!=UNZ_OK) + err=UNZ_ERRNO; + + if ((number_entry_CD!=us.gi.number_entry) || + (number_disk_with_CD!=0) || + (number_disk!=0)) + err=UNZ_BADZIPFILE; + + /* size of the central directory */ + if (unz64local_getLong64(&us.z_filefunc, us.filestream,&us.size_central_dir)!=UNZ_OK) + err=UNZ_ERRNO; + + /* offset of start of central directory with respect to the + starting disk number */ + if (unz64local_getLong64(&us.z_filefunc, us.filestream,&us.offset_central_dir)!=UNZ_OK) + err=UNZ_ERRNO; + + us.gi.size_comment = 0; + } + else + { + central_pos = unz64local_SearchCentralDir(&us.z_filefunc,us.filestream); + if (central_pos==0) + err=UNZ_ERRNO; + + us.isZip64 = 0; + + if (ZSEEK64(us.z_filefunc, us.filestream, + central_pos,ZLIB_FILEFUNC_SEEK_SET)!=0) + err=UNZ_ERRNO; + + /* the signature, already checked */ + if (unz64local_getLong(&us.z_filefunc, us.filestream,&uL)!=UNZ_OK) + err=UNZ_ERRNO; + + /* number of this disk */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&number_disk)!=UNZ_OK) + err=UNZ_ERRNO; + + /* number of the disk with the start of the central directory */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&number_disk_with_CD)!=UNZ_OK) + err=UNZ_ERRNO; + + /* total number of entries in the central dir on this disk */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&uL)!=UNZ_OK) + err=UNZ_ERRNO; + us.gi.number_entry = uL; + + /* total number of entries in the central dir */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&uL)!=UNZ_OK) + err=UNZ_ERRNO; + number_entry_CD = uL; + + if ((number_entry_CD!=us.gi.number_entry) || + (number_disk_with_CD!=0) || + (number_disk!=0)) + err=UNZ_BADZIPFILE; + + /* size of the central directory */ + if (unz64local_getLong(&us.z_filefunc, us.filestream,&uL)!=UNZ_OK) + err=UNZ_ERRNO; + us.size_central_dir = uL; + + /* offset of start of central directory with respect to the + starting disk number */ + if (unz64local_getLong(&us.z_filefunc, us.filestream,&uL)!=UNZ_OK) + err=UNZ_ERRNO; + us.offset_central_dir = uL; + + /* zipfile comment length */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&us.gi.size_comment)!=UNZ_OK) + err=UNZ_ERRNO; + } + + if ((central_pos<us.offset_central_dir+us.size_central_dir) && + (err==UNZ_OK)) + err=UNZ_BADZIPFILE; + + if (err!=UNZ_OK) + { + ZCLOSE64(us.z_filefunc, us.filestream); + return NULL; + } + + us.byte_before_the_zipfile = central_pos - + (us.offset_central_dir+us.size_central_dir); + us.central_pos = central_pos; + us.pfile_in_zip_read = NULL; + us.encrypted = 0; + + + s=(unz64_s*)ALLOC(sizeof(unz64_s)); + if( s != NULL) + { + *s=us; + unzGoToFirstFile((unzFile)s); + } + return (unzFile)s; +} + + +extern unzFile ZEXPORT unzOpen2 (const char *path, + zlib_filefunc_def* pzlib_filefunc32_def) +{ + if (pzlib_filefunc32_def != NULL) + { + zlib_filefunc64_32_def zlib_filefunc64_32_def_fill; + fill_zlib_filefunc64_32_def_from_filefunc32(&zlib_filefunc64_32_def_fill,pzlib_filefunc32_def); + return unzOpenInternal(path, &zlib_filefunc64_32_def_fill, 0); + } + else + return unzOpenInternal(path, NULL, 0); +} + +extern unzFile ZEXPORT unzOpen2_64 (const void *path, + zlib_filefunc64_def* pzlib_filefunc_def) +{ + if (pzlib_filefunc_def != NULL) + { + zlib_filefunc64_32_def zlib_filefunc64_32_def_fill; + zlib_filefunc64_32_def_fill.zfile_func64 = *pzlib_filefunc_def; + zlib_filefunc64_32_def_fill.ztell32_file = NULL; + zlib_filefunc64_32_def_fill.zseek32_file = NULL; + return unzOpenInternal(path, &zlib_filefunc64_32_def_fill, 1); + } + else + return unzOpenInternal(path, NULL, 1); +} + +extern unzFile ZEXPORT unzOpen (const char *path) +{ + return unzOpenInternal(path, NULL, 0); +} + +extern unzFile ZEXPORT unzOpen64 (const void *path) +{ + return unzOpenInternal(path, NULL, 1); +} + +/* + Close a ZipFile opened with unzOpen. + If there is files inside the .Zip opened with unzOpenCurrentFile (see later), + these files MUST be closed with unzCloseCurrentFile before call unzClose. + return UNZ_OK if there is no problem. */ +extern int ZEXPORT unzClose (unzFile file) +{ + unz64_s* s; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + + if (s->pfile_in_zip_read!=NULL) + unzCloseCurrentFile(file); + + ZCLOSE64(s->z_filefunc, s->filestream); + TRYFREE(s); + return UNZ_OK; +} + + +/* + Write info about the ZipFile in the *pglobal_info structure. + No preparation of the structure is needed + return UNZ_OK if there is no problem. */ +extern int ZEXPORT unzGetGlobalInfo64 (unzFile file, unz_global_info64* pglobal_info) +{ + unz64_s* s; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + *pglobal_info=s->gi; + return UNZ_OK; +} + +extern int ZEXPORT unzGetGlobalInfo (unzFile file, unz_global_info* pglobal_info32) +{ + unz64_s* s; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + /* to do : check if number_entry is not truncated */ + pglobal_info32->number_entry = (uLong)s->gi.number_entry; + pglobal_info32->size_comment = s->gi.size_comment; + return UNZ_OK; +} +/* + Translate date/time from Dos format to tm_unz (readable more easilty) +*/ +local void unz64local_DosDateToTmuDate (ZPOS64_T ulDosDate, tm_unz* ptm) +{ + ZPOS64_T uDate; + uDate = (ZPOS64_T)(ulDosDate>>16); + ptm->tm_mday = (uInt)(uDate&0x1f) ; + ptm->tm_mon = (uInt)((((uDate)&0x1E0)/0x20)-1) ; + ptm->tm_year = (uInt)(((uDate&0x0FE00)/0x0200)+1980) ; + + ptm->tm_hour = (uInt) ((ulDosDate &0xF800)/0x800); + ptm->tm_min = (uInt) ((ulDosDate&0x7E0)/0x20) ; + ptm->tm_sec = (uInt) (2*(ulDosDate&0x1f)) ; +} + +/* + Get Info about the current file in the zipfile, with internal only info +*/ +local int unz64local_GetCurrentFileInfoInternal OF((unzFile file, + unz_file_info64 *pfile_info, + unz_file_info64_internal + *pfile_info_internal, + char *szFileName, + uLong fileNameBufferSize, + void *extraField, + uLong extraFieldBufferSize, + char *szComment, + uLong commentBufferSize)); + +local int unz64local_GetCurrentFileInfoInternal (unzFile file, + unz_file_info64 *pfile_info, + unz_file_info64_internal + *pfile_info_internal, + char *szFileName, + uLong fileNameBufferSize, + void *extraField, + uLong extraFieldBufferSize, + char *szComment, + uLong commentBufferSize) +{ + unz64_s* s; + unz_file_info64 file_info; + unz_file_info64_internal file_info_internal; + int err=UNZ_OK; + uLong uMagic; + long lSeek=0; + uLong uL; + + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + if (ZSEEK64(s->z_filefunc, s->filestream, + s->pos_in_central_dir+s->byte_before_the_zipfile, + ZLIB_FILEFUNC_SEEK_SET)!=0) + err=UNZ_ERRNO; + + + /* we check the magic */ + if (err==UNZ_OK) + { + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uMagic) != UNZ_OK) + err=UNZ_ERRNO; + else if (uMagic!=0x02014b50) + err=UNZ_BADZIPFILE; + } + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.version) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.version_needed) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.flag) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.compression_method) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&file_info.dosDate) != UNZ_OK) + err=UNZ_ERRNO; + + unz64local_DosDateToTmuDate(file_info.dosDate,&file_info.tmu_date); + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&file_info.crc) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uL) != UNZ_OK) + err=UNZ_ERRNO; + file_info.compressed_size = uL; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uL) != UNZ_OK) + err=UNZ_ERRNO; + file_info.uncompressed_size = uL; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.size_filename) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.size_file_extra) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.size_file_comment) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.disk_num_start) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.internal_fa) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&file_info.external_fa) != UNZ_OK) + err=UNZ_ERRNO; + + // relative offset of local header + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uL) != UNZ_OK) + err=UNZ_ERRNO; + file_info_internal.offset_curfile = uL; + + lSeek+=file_info.size_filename; + if ((err==UNZ_OK) && (szFileName!=NULL)) + { + uLong uSizeRead ; + if (file_info.size_filename<fileNameBufferSize) + { + *(szFileName+file_info.size_filename)='\0'; + uSizeRead = file_info.size_filename; + } + else + uSizeRead = fileNameBufferSize; + + if ((file_info.size_filename>0) && (fileNameBufferSize>0)) + if (ZREAD64(s->z_filefunc, s->filestream,szFileName,uSizeRead)!=uSizeRead) + err=UNZ_ERRNO; + lSeek -= uSizeRead; + } + + // Read extrafield + if ((err==UNZ_OK) && (extraField!=NULL)) + { + ZPOS64_T uSizeRead ; + if (file_info.size_file_extra<extraFieldBufferSize) + uSizeRead = file_info.size_file_extra; + else + uSizeRead = extraFieldBufferSize; + + if (lSeek!=0) + { + if (ZSEEK64(s->z_filefunc, s->filestream,lSeek,ZLIB_FILEFUNC_SEEK_CUR)==0) + lSeek=0; + else + err=UNZ_ERRNO; + } + + if ((file_info.size_file_extra>0) && (extraFieldBufferSize>0)) + if (ZREAD64(s->z_filefunc, s->filestream,extraField,(uLong)uSizeRead)!=uSizeRead) + err=UNZ_ERRNO; + + lSeek += file_info.size_file_extra - (uLong)uSizeRead; + } + else + lSeek += file_info.size_file_extra; + + + if ((err==UNZ_OK) && (file_info.size_file_extra != 0)) + { + uLong acc = 0; + + // since lSeek now points to after the extra field we need to move back + lSeek -= file_info.size_file_extra; + + if (lSeek!=0) + { + if (ZSEEK64(s->z_filefunc, s->filestream,lSeek,ZLIB_FILEFUNC_SEEK_CUR)==0) + lSeek=0; + else + err=UNZ_ERRNO; + } + + while(acc < file_info.size_file_extra) + { + uLong headerId; + uLong dataSize; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&headerId) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&dataSize) != UNZ_OK) + err=UNZ_ERRNO; + + /* ZIP64 extra fields */ + if (headerId == 0x0001) + { + uLong uL; + + if(file_info.uncompressed_size == MAXU32) + { + if (unz64local_getLong64(&s->z_filefunc, s->filestream,&file_info.uncompressed_size) != UNZ_OK) + err=UNZ_ERRNO; + } + + if(file_info.compressed_size == MAXU32) + { + if (unz64local_getLong64(&s->z_filefunc, s->filestream,&file_info.compressed_size) != UNZ_OK) + err=UNZ_ERRNO; + } + + if(file_info_internal.offset_curfile == MAXU32) + { + /* Relative Header offset */ + if (unz64local_getLong64(&s->z_filefunc, s->filestream,&file_info_internal.offset_curfile) != UNZ_OK) + err=UNZ_ERRNO; + } + + if(file_info.disk_num_start == MAXU32) + { + /* Disk Start Number */ + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uL) != UNZ_OK) + err=UNZ_ERRNO; + } + + } + else + { + if (ZSEEK64(s->z_filefunc, s->filestream,dataSize,ZLIB_FILEFUNC_SEEK_CUR)!=0) + err=UNZ_ERRNO; + } + + acc += 2 + 2 + dataSize; + } + } + + if ((err==UNZ_OK) && (szComment!=NULL)) + { + uLong uSizeRead ; + if (file_info.size_file_comment<commentBufferSize) + { + *(szComment+file_info.size_file_comment)='\0'; + uSizeRead = file_info.size_file_comment; + } + else + uSizeRead = commentBufferSize; + + if (lSeek!=0) + { + if (ZSEEK64(s->z_filefunc, s->filestream,lSeek,ZLIB_FILEFUNC_SEEK_CUR)==0) + lSeek=0; + else + err=UNZ_ERRNO; + } + + if ((file_info.size_file_comment>0) && (commentBufferSize>0)) + if (ZREAD64(s->z_filefunc, s->filestream,szComment,uSizeRead)!=uSizeRead) + err=UNZ_ERRNO; + lSeek+=file_info.size_file_comment - uSizeRead; + } + else + lSeek+=file_info.size_file_comment; + + + if ((err==UNZ_OK) && (pfile_info!=NULL)) + *pfile_info=file_info; + + if ((err==UNZ_OK) && (pfile_info_internal!=NULL)) + *pfile_info_internal=file_info_internal; + + return err; +} + + + +/* + Write info about the ZipFile in the *pglobal_info structure. + No preparation of the structure is needed + return UNZ_OK if there is no problem. +*/ +extern int ZEXPORT unzGetCurrentFileInfo64 (unzFile file, + unz_file_info64 * pfile_info, + char * szFileName, uLong fileNameBufferSize, + void *extraField, uLong extraFieldBufferSize, + char* szComment, uLong commentBufferSize) +{ + return unz64local_GetCurrentFileInfoInternal(file,pfile_info,NULL, + szFileName,fileNameBufferSize, + extraField,extraFieldBufferSize, + szComment,commentBufferSize); +} + +extern int ZEXPORT unzGetCurrentFileInfo (unzFile file, + unz_file_info * pfile_info, + char * szFileName, uLong fileNameBufferSize, + void *extraField, uLong extraFieldBufferSize, + char* szComment, uLong commentBufferSize) +{ + int err; + unz_file_info64 file_info64; + err = unz64local_GetCurrentFileInfoInternal(file,&file_info64,NULL, + szFileName,fileNameBufferSize, + extraField,extraFieldBufferSize, + szComment,commentBufferSize); + if ((err==UNZ_OK) && (pfile_info != NULL)) + { + pfile_info->version = file_info64.version; + pfile_info->version_needed = file_info64.version_needed; + pfile_info->flag = file_info64.flag; + pfile_info->compression_method = file_info64.compression_method; + pfile_info->dosDate = file_info64.dosDate; + pfile_info->crc = file_info64.crc; + + pfile_info->size_filename = file_info64.size_filename; + pfile_info->size_file_extra = file_info64.size_file_extra; + pfile_info->size_file_comment = file_info64.size_file_comment; + + pfile_info->disk_num_start = file_info64.disk_num_start; + pfile_info->internal_fa = file_info64.internal_fa; + pfile_info->external_fa = file_info64.external_fa; + + pfile_info->tmu_date = file_info64.tmu_date, + + + pfile_info->compressed_size = (uLong)file_info64.compressed_size; + pfile_info->uncompressed_size = (uLong)file_info64.uncompressed_size; + + } + return err; +} +/* + Set the current file of the zipfile to the first file. + return UNZ_OK if there is no problem +*/ +extern int ZEXPORT unzGoToFirstFile (unzFile file) +{ + int err=UNZ_OK; + unz64_s* s; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + s->pos_in_central_dir=s->offset_central_dir; + s->num_file=0; + err=unz64local_GetCurrentFileInfoInternal(file,&s->cur_file_info, + &s->cur_file_info_internal, + NULL,0,NULL,0,NULL,0); + s->current_file_ok = (err == UNZ_OK); + return err; +} + +/* + Set the current file of the zipfile to the next file. + return UNZ_OK if there is no problem + return UNZ_END_OF_LIST_OF_FILE if the actual file was the latest. +*/ +extern int ZEXPORT unzGoToNextFile (unzFile file) +{ + unz64_s* s; + int err; + + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + if (!s->current_file_ok) + return UNZ_END_OF_LIST_OF_FILE; + if (s->gi.number_entry != 0xffff) /* 2^16 files overflow hack */ + if (s->num_file+1==s->gi.number_entry) + return UNZ_END_OF_LIST_OF_FILE; + + s->pos_in_central_dir += SIZECENTRALDIRITEM + s->cur_file_info.size_filename + + s->cur_file_info.size_file_extra + s->cur_file_info.size_file_comment ; + s->num_file++; + err = unz64local_GetCurrentFileInfoInternal(file,&s->cur_file_info, + &s->cur_file_info_internal, + NULL,0,NULL,0,NULL,0); + s->current_file_ok = (err == UNZ_OK); + return err; +} + + +/* + Try locate the file szFileName in the zipfile. + For the iCaseSensitivity signification, see unzStringFileNameCompare + + return value : + UNZ_OK if the file is found. It becomes the current file. + UNZ_END_OF_LIST_OF_FILE if the file is not found +*/ +extern int ZEXPORT unzLocateFile (unzFile file, const char *szFileName, int iCaseSensitivity) +{ + unz64_s* s; + int err; + + /* We remember the 'current' position in the file so that we can jump + * back there if we fail. + */ + unz_file_info64 cur_file_infoSaved; + unz_file_info64_internal cur_file_info_internalSaved; + ZPOS64_T num_fileSaved; + ZPOS64_T pos_in_central_dirSaved; + + + if (file==NULL) + return UNZ_PARAMERROR; + + if (strlen(szFileName)>=UNZ_MAXFILENAMEINZIP) + return UNZ_PARAMERROR; + + s=(unz64_s*)file; + if (!s->current_file_ok) + return UNZ_END_OF_LIST_OF_FILE; + + /* Save the current state */ + num_fileSaved = s->num_file; + pos_in_central_dirSaved = s->pos_in_central_dir; + cur_file_infoSaved = s->cur_file_info; + cur_file_info_internalSaved = s->cur_file_info_internal; + + err = unzGoToFirstFile(file); + + while (err == UNZ_OK) + { + char szCurrentFileName[UNZ_MAXFILENAMEINZIP+1]; + err = unzGetCurrentFileInfo64(file,NULL, + szCurrentFileName,sizeof(szCurrentFileName)-1, + NULL,0,NULL,0); + if (err == UNZ_OK) + { + if (unzStringFileNameCompare(szCurrentFileName, + szFileName,iCaseSensitivity)==0) + return UNZ_OK; + err = unzGoToNextFile(file); + } + } + + /* We failed, so restore the state of the 'current file' to where we + * were. + */ + s->num_file = num_fileSaved ; + s->pos_in_central_dir = pos_in_central_dirSaved ; + s->cur_file_info = cur_file_infoSaved; + s->cur_file_info_internal = cur_file_info_internalSaved; + return err; +} + + +/* +/////////////////////////////////////////// +// Contributed by Ryan Haksi (mailto://cryogen@infoserve.net) +// I need random access +// +// Further optimization could be realized by adding an ability +// to cache the directory in memory. The goal being a single +// comprehensive file read to put the file I need in a memory. +*/ + +/* +typedef struct unz_file_pos_s +{ + ZPOS64_T pos_in_zip_directory; // offset in file + ZPOS64_T num_of_file; // # of file +} unz_file_pos; +*/ + +extern int ZEXPORT unzGetFilePos64(unzFile file, unz64_file_pos* file_pos) +{ + unz64_s* s; + + if (file==NULL || file_pos==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + if (!s->current_file_ok) + return UNZ_END_OF_LIST_OF_FILE; + + file_pos->pos_in_zip_directory = s->pos_in_central_dir; + file_pos->num_of_file = s->num_file; + + return UNZ_OK; +} + +extern int ZEXPORT unzGetFilePos( + unzFile file, + unz_file_pos* file_pos) +{ + unz64_file_pos file_pos64; + int err = unzGetFilePos64(file,&file_pos64); + if (err==UNZ_OK) + { + file_pos->pos_in_zip_directory = (uLong)file_pos64.pos_in_zip_directory; + file_pos->num_of_file = (uLong)file_pos64.num_of_file; + } + return err; +} + +extern int ZEXPORT unzGoToFilePos64(unzFile file, const unz64_file_pos* file_pos) +{ + unz64_s* s; + int err; + + if (file==NULL || file_pos==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + + /* jump to the right spot */ + s->pos_in_central_dir = file_pos->pos_in_zip_directory; + s->num_file = file_pos->num_of_file; + + /* set the current file */ + err = unz64local_GetCurrentFileInfoInternal(file,&s->cur_file_info, + &s->cur_file_info_internal, + NULL,0,NULL,0,NULL,0); + /* return results */ + s->current_file_ok = (err == UNZ_OK); + return err; +} + +extern int ZEXPORT unzGoToFilePos( + unzFile file, + unz_file_pos* file_pos) +{ + unz64_file_pos file_pos64; + if (file_pos == NULL) + return UNZ_PARAMERROR; + + file_pos64.pos_in_zip_directory = file_pos->pos_in_zip_directory; + file_pos64.num_of_file = file_pos->num_of_file; + return unzGoToFilePos64(file,&file_pos64); +} + +/* +// Unzip Helper Functions - should be here? +/////////////////////////////////////////// +*/ + +/* + Read the local header of the current zipfile + Check the coherency of the local header and info in the end of central + directory about this file + store in *piSizeVar the size of extra info in local header + (filename and size of extra field data) +*/ +local int unz64local_CheckCurrentFileCoherencyHeader (unz64_s* s, uInt* piSizeVar, + ZPOS64_T * poffset_local_extrafield, + uInt * psize_local_extrafield) +{ + uLong uMagic,uData,uFlags; + uLong size_filename; + uLong size_extra_field; + int err=UNZ_OK; + + *piSizeVar = 0; + *poffset_local_extrafield = 0; + *psize_local_extrafield = 0; + + if (ZSEEK64(s->z_filefunc, s->filestream,s->cur_file_info_internal.offset_curfile + + s->byte_before_the_zipfile,ZLIB_FILEFUNC_SEEK_SET)!=0) + return UNZ_ERRNO; + + + if (err==UNZ_OK) + { + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uMagic) != UNZ_OK) + err=UNZ_ERRNO; + else if (uMagic!=0x04034b50) + err=UNZ_BADZIPFILE; + } + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) + err=UNZ_ERRNO; +/* + else if ((err==UNZ_OK) && (uData!=s->cur_file_info.wVersion)) + err=UNZ_BADZIPFILE; +*/ + if (unz64local_getShort(&s->z_filefunc, s->filestream,&uFlags) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) + err=UNZ_ERRNO; + else if ((err==UNZ_OK) && (uData!=s->cur_file_info.compression_method)) + err=UNZ_BADZIPFILE; + + if ((err==UNZ_OK) && (s->cur_file_info.compression_method!=0) && +/* #ifdef HAVE_BZIP2 */ + (s->cur_file_info.compression_method!=Z_BZIP2ED) && +/* #endif */ + (s->cur_file_info.compression_method!=Z_DEFLATED)) + err=UNZ_BADZIPFILE; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) /* date/time */ + err=UNZ_ERRNO; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) /* crc */ + err=UNZ_ERRNO; + else if ((err==UNZ_OK) && (uData!=s->cur_file_info.crc) && ((uFlags & 8)==0)) + err=UNZ_BADZIPFILE; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) /* size compr */ + err=UNZ_ERRNO; + else if (uData != 0xFFFFFFFF && (err==UNZ_OK) && (uData!=s->cur_file_info.compressed_size) && ((uFlags & 8)==0)) + err=UNZ_BADZIPFILE; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) /* size uncompr */ + err=UNZ_ERRNO; + else if (uData != 0xFFFFFFFF && (err==UNZ_OK) && (uData!=s->cur_file_info.uncompressed_size) && ((uFlags & 8)==0)) + err=UNZ_BADZIPFILE; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&size_filename) != UNZ_OK) + err=UNZ_ERRNO; + else if ((err==UNZ_OK) && (size_filename!=s->cur_file_info.size_filename)) + err=UNZ_BADZIPFILE; + + *piSizeVar += (uInt)size_filename; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&size_extra_field) != UNZ_OK) + err=UNZ_ERRNO; + *poffset_local_extrafield= s->cur_file_info_internal.offset_curfile + + SIZEZIPLOCALHEADER + size_filename; + *psize_local_extrafield = (uInt)size_extra_field; + + *piSizeVar += (uInt)size_extra_field; + + return err; +} + +/* + Open for reading data the current file in the zipfile. + If there is no error and the file is opened, the return value is UNZ_OK. +*/ +extern int ZEXPORT unzOpenCurrentFile3 (unzFile file, int* method, + int* level, int raw, const char* password) +{ + int err=UNZ_OK; + uInt iSizeVar; + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + ZPOS64_T offset_local_extrafield; /* offset of the local extra field */ + uInt size_local_extrafield; /* size of the local extra field */ +# ifndef NOUNCRYPT + char source[12]; +# else + if (password != NULL) + return UNZ_PARAMERROR; +# endif + + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + if (!s->current_file_ok) + return UNZ_PARAMERROR; + + if (s->pfile_in_zip_read != NULL) + unzCloseCurrentFile(file); + + if (unz64local_CheckCurrentFileCoherencyHeader(s,&iSizeVar, &offset_local_extrafield,&size_local_extrafield)!=UNZ_OK) + return UNZ_BADZIPFILE; + + pfile_in_zip_read_info = (file_in_zip64_read_info_s*)ALLOC(sizeof(file_in_zip64_read_info_s)); + if (pfile_in_zip_read_info==NULL) + return UNZ_INTERNALERROR; + + pfile_in_zip_read_info->read_buffer=(char*)ALLOC(UNZ_BUFSIZE); + pfile_in_zip_read_info->offset_local_extrafield = offset_local_extrafield; + pfile_in_zip_read_info->size_local_extrafield = size_local_extrafield; + pfile_in_zip_read_info->pos_local_extrafield=0; + pfile_in_zip_read_info->raw=raw; + + if (pfile_in_zip_read_info->read_buffer==NULL) + { + TRYFREE(pfile_in_zip_read_info); + return UNZ_INTERNALERROR; + } + + pfile_in_zip_read_info->stream_initialised=0; + + if (method!=NULL) + *method = (int)s->cur_file_info.compression_method; + + if (level!=NULL) + { + *level = 6; + switch (s->cur_file_info.flag & 0x06) + { + case 6 : *level = 1; break; + case 4 : *level = 2; break; + case 2 : *level = 9; break; + } + } + + if ((s->cur_file_info.compression_method!=0) && +/* #ifdef HAVE_BZIP2 */ + (s->cur_file_info.compression_method!=Z_BZIP2ED) && +/* #endif */ + (s->cur_file_info.compression_method!=Z_DEFLATED)) + + err=UNZ_BADZIPFILE; + + pfile_in_zip_read_info->crc32_wait=s->cur_file_info.crc; + pfile_in_zip_read_info->crc32=0; + pfile_in_zip_read_info->total_out_64=0; + pfile_in_zip_read_info->compression_method = s->cur_file_info.compression_method; + pfile_in_zip_read_info->filestream=s->filestream; + pfile_in_zip_read_info->z_filefunc=s->z_filefunc; + pfile_in_zip_read_info->byte_before_the_zipfile=s->byte_before_the_zipfile; + + pfile_in_zip_read_info->stream.total_out = 0; + + if ((s->cur_file_info.compression_method==Z_BZIP2ED) && (!raw)) + { +#ifdef HAVE_BZIP2 + pfile_in_zip_read_info->bstream.bzalloc = (void *(*) (void *, int, int))0; + pfile_in_zip_read_info->bstream.bzfree = (free_func)0; + pfile_in_zip_read_info->bstream.opaque = (voidpf)0; + pfile_in_zip_read_info->bstream.state = (voidpf)0; + + pfile_in_zip_read_info->stream.zalloc = (alloc_func)0; + pfile_in_zip_read_info->stream.zfree = (free_func)0; + pfile_in_zip_read_info->stream.opaque = (voidpf)0; + pfile_in_zip_read_info->stream.next_in = (voidpf)0; + pfile_in_zip_read_info->stream.avail_in = 0; + + err=BZ2_bzDecompressInit(&pfile_in_zip_read_info->bstream, 0, 0); + if (err == Z_OK) + pfile_in_zip_read_info->stream_initialised=Z_BZIP2ED; + else + { + TRYFREE(pfile_in_zip_read_info); + return err; + } +#else + pfile_in_zip_read_info->raw=1; +#endif + } + else if ((s->cur_file_info.compression_method==Z_DEFLATED) && (!raw)) + { + pfile_in_zip_read_info->stream.zalloc = (alloc_func)0; + pfile_in_zip_read_info->stream.zfree = (free_func)0; + pfile_in_zip_read_info->stream.opaque = (voidpf)0; + pfile_in_zip_read_info->stream.next_in = 0; + pfile_in_zip_read_info->stream.avail_in = 0; + + err=inflateInit2(&pfile_in_zip_read_info->stream, -MAX_WBITS); + if (err == Z_OK) + pfile_in_zip_read_info->stream_initialised=Z_DEFLATED; + else + { + TRYFREE(pfile_in_zip_read_info); + return err; + } + /* windowBits is passed < 0 to tell that there is no zlib header. + * Note that in this case inflate *requires* an extra "dummy" byte + * after the compressed stream in order to complete decompression and + * return Z_STREAM_END. + * In unzip, i don't wait absolutely Z_STREAM_END because I known the + * size of both compressed and uncompressed data + */ + } + pfile_in_zip_read_info->rest_read_compressed = + s->cur_file_info.compressed_size ; + pfile_in_zip_read_info->rest_read_uncompressed = + s->cur_file_info.uncompressed_size ; + + + pfile_in_zip_read_info->pos_in_zipfile = + s->cur_file_info_internal.offset_curfile + SIZEZIPLOCALHEADER + + iSizeVar; + + pfile_in_zip_read_info->stream.avail_in = (uInt)0; + + s->pfile_in_zip_read = pfile_in_zip_read_info; + s->encrypted = 0; + +# ifndef NOUNCRYPT + if (password != NULL) + { + int i; + s->pcrc_32_tab = get_crc_table(); + init_keys(password,s->keys,s->pcrc_32_tab); + if (ZSEEK64(s->z_filefunc, s->filestream, + s->pfile_in_zip_read->pos_in_zipfile + + s->pfile_in_zip_read->byte_before_the_zipfile, + SEEK_SET)!=0) + return UNZ_INTERNALERROR; + if(ZREAD64(s->z_filefunc, s->filestream,source, 12)<12) + return UNZ_INTERNALERROR; + + for (i = 0; i<12; i++) + zdecode(s->keys,s->pcrc_32_tab,source[i]); + + s->pfile_in_zip_read->pos_in_zipfile+=12; + s->encrypted=1; + } +# endif + + + return UNZ_OK; +} + +extern int ZEXPORT unzOpenCurrentFile (unzFile file) +{ + return unzOpenCurrentFile3(file, NULL, NULL, 0, NULL); +} + +extern int ZEXPORT unzOpenCurrentFilePassword (unzFile file, const char* password) +{ + return unzOpenCurrentFile3(file, NULL, NULL, 0, password); +} + +extern int ZEXPORT unzOpenCurrentFile2 (unzFile file, int* method, int* level, int raw) +{ + return unzOpenCurrentFile3(file, method, level, raw, NULL); +} + +/** Addition for GDAL : START */ + +extern ZPOS64_T ZEXPORT unzGetCurrentFileZStreamPos64( unzFile file) +{ + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + s=(unz64_s*)file; + if (file==NULL) + return 0; //UNZ_PARAMERROR; + pfile_in_zip_read_info=s->pfile_in_zip_read; + if (pfile_in_zip_read_info==NULL) + return 0; //UNZ_PARAMERROR; + return pfile_in_zip_read_info->pos_in_zipfile + + pfile_in_zip_read_info->byte_before_the_zipfile; +} + +/** Addition for GDAL : END */ + +/* + Read bytes from the current file. + buf contain buffer where data must be copied + len the size of buf. + + return the number of byte copied if somes bytes are copied + return 0 if the end of file was reached + return <0 with error code if there is an error + (UNZ_ERRNO for IO error, or zLib error for uncompress error) +*/ +extern int ZEXPORT unzReadCurrentFile (unzFile file, voidp buf, unsigned len) +{ + int err=UNZ_OK; + uInt iRead = 0; + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + pfile_in_zip_read_info=s->pfile_in_zip_read; + + if (pfile_in_zip_read_info==NULL) + return UNZ_PARAMERROR; + + + if (pfile_in_zip_read_info->read_buffer == NULL) + return UNZ_END_OF_LIST_OF_FILE; + if (len==0) + return 0; + + pfile_in_zip_read_info->stream.next_out = (Bytef*)buf; + + pfile_in_zip_read_info->stream.avail_out = (uInt)len; + + if ((len>pfile_in_zip_read_info->rest_read_uncompressed) && + (!(pfile_in_zip_read_info->raw))) + pfile_in_zip_read_info->stream.avail_out = + (uInt)pfile_in_zip_read_info->rest_read_uncompressed; + + if ((len>pfile_in_zip_read_info->rest_read_compressed+ + pfile_in_zip_read_info->stream.avail_in) && + (pfile_in_zip_read_info->raw)) + pfile_in_zip_read_info->stream.avail_out = + (uInt)pfile_in_zip_read_info->rest_read_compressed+ + pfile_in_zip_read_info->stream.avail_in; + + while (pfile_in_zip_read_info->stream.avail_out>0) + { + if ((pfile_in_zip_read_info->stream.avail_in==0) && + (pfile_in_zip_read_info->rest_read_compressed>0)) + { + uInt uReadThis = UNZ_BUFSIZE; + if (pfile_in_zip_read_info->rest_read_compressed<uReadThis) + uReadThis = (uInt)pfile_in_zip_read_info->rest_read_compressed; + if (uReadThis == 0) + return UNZ_EOF; + if (ZSEEK64(pfile_in_zip_read_info->z_filefunc, + pfile_in_zip_read_info->filestream, + pfile_in_zip_read_info->pos_in_zipfile + + pfile_in_zip_read_info->byte_before_the_zipfile, + ZLIB_FILEFUNC_SEEK_SET)!=0) + return UNZ_ERRNO; + if (ZREAD64(pfile_in_zip_read_info->z_filefunc, + pfile_in_zip_read_info->filestream, + pfile_in_zip_read_info->read_buffer, + uReadThis)!=uReadThis) + return UNZ_ERRNO; + + +# ifndef NOUNCRYPT + if(s->encrypted) + { + uInt i; + for(i=0;i<uReadThis;i++) + pfile_in_zip_read_info->read_buffer[i] = + zdecode(s->keys,s->pcrc_32_tab, + pfile_in_zip_read_info->read_buffer[i]); + } +# endif + + + pfile_in_zip_read_info->pos_in_zipfile += uReadThis; + + pfile_in_zip_read_info->rest_read_compressed-=uReadThis; + + pfile_in_zip_read_info->stream.next_in = + (Bytef*)pfile_in_zip_read_info->read_buffer; + pfile_in_zip_read_info->stream.avail_in = (uInt)uReadThis; + } + + if ((pfile_in_zip_read_info->compression_method==0) || (pfile_in_zip_read_info->raw)) + { + uInt uDoCopy,i ; + + if ((pfile_in_zip_read_info->stream.avail_in == 0) && + (pfile_in_zip_read_info->rest_read_compressed == 0)) + return (iRead==0) ? UNZ_EOF : iRead; + + if (pfile_in_zip_read_info->stream.avail_out < + pfile_in_zip_read_info->stream.avail_in) + uDoCopy = pfile_in_zip_read_info->stream.avail_out ; + else + uDoCopy = pfile_in_zip_read_info->stream.avail_in ; + + for (i=0;i<uDoCopy;i++) + *(pfile_in_zip_read_info->stream.next_out+i) = + *(pfile_in_zip_read_info->stream.next_in+i); + + pfile_in_zip_read_info->total_out_64 = pfile_in_zip_read_info->total_out_64 + uDoCopy; + + pfile_in_zip_read_info->crc32 = crc32(pfile_in_zip_read_info->crc32, + pfile_in_zip_read_info->stream.next_out, + uDoCopy); + pfile_in_zip_read_info->rest_read_uncompressed-=uDoCopy; + pfile_in_zip_read_info->stream.avail_in -= uDoCopy; + pfile_in_zip_read_info->stream.avail_out -= uDoCopy; + pfile_in_zip_read_info->stream.next_out += uDoCopy; + pfile_in_zip_read_info->stream.next_in += uDoCopy; + pfile_in_zip_read_info->stream.total_out += uDoCopy; + iRead += uDoCopy; + } + else if (pfile_in_zip_read_info->compression_method==Z_BZIP2ED) + { +#ifdef HAVE_BZIP2 + uLong uTotalOutBefore,uTotalOutAfter; + const Bytef *bufBefore; + uLong uOutThis; + + pfile_in_zip_read_info->bstream.next_in = (char*)pfile_in_zip_read_info->stream.next_in; + pfile_in_zip_read_info->bstream.avail_in = pfile_in_zip_read_info->stream.avail_in; + pfile_in_zip_read_info->bstream.total_in_lo32 = pfile_in_zip_read_info->stream.total_in; + pfile_in_zip_read_info->bstream.total_in_hi32 = 0; + pfile_in_zip_read_info->bstream.next_out = (char*)pfile_in_zip_read_info->stream.next_out; + pfile_in_zip_read_info->bstream.avail_out = pfile_in_zip_read_info->stream.avail_out; + pfile_in_zip_read_info->bstream.total_out_lo32 = pfile_in_zip_read_info->stream.total_out; + pfile_in_zip_read_info->bstream.total_out_hi32 = 0; + + uTotalOutBefore = pfile_in_zip_read_info->bstream.total_out_lo32; + bufBefore = (const Bytef *)pfile_in_zip_read_info->bstream.next_out; + + err=BZ2_bzDecompress(&pfile_in_zip_read_info->bstream); + + uTotalOutAfter = pfile_in_zip_read_info->bstream.total_out_lo32; + uOutThis = uTotalOutAfter-uTotalOutBefore; + + pfile_in_zip_read_info->total_out_64 = pfile_in_zip_read_info->total_out_64 + uOutThis; + + pfile_in_zip_read_info->crc32 = crc32(pfile_in_zip_read_info->crc32,bufBefore, (uInt)(uOutThis)); + pfile_in_zip_read_info->rest_read_uncompressed -= uOutThis; + iRead += (uInt)(uTotalOutAfter - uTotalOutBefore); + + pfile_in_zip_read_info->stream.next_in = (Bytef*)pfile_in_zip_read_info->bstream.next_in; + pfile_in_zip_read_info->stream.avail_in = pfile_in_zip_read_info->bstream.avail_in; + pfile_in_zip_read_info->stream.total_in = pfile_in_zip_read_info->bstream.total_in_lo32; + pfile_in_zip_read_info->stream.next_out = (Bytef*)pfile_in_zip_read_info->bstream.next_out; + pfile_in_zip_read_info->stream.avail_out = pfile_in_zip_read_info->bstream.avail_out; + pfile_in_zip_read_info->stream.total_out = pfile_in_zip_read_info->bstream.total_out_lo32; + + if (err==BZ_STREAM_END) + return (iRead==0) ? UNZ_EOF : iRead; + if (err!=BZ_OK) + break; +#endif + } // end Z_BZIP2ED + else + { + ZPOS64_T uTotalOutBefore,uTotalOutAfter; + const Bytef *bufBefore; + ZPOS64_T uOutThis; + int flush=Z_SYNC_FLUSH; + + uTotalOutBefore = pfile_in_zip_read_info->stream.total_out; + bufBefore = pfile_in_zip_read_info->stream.next_out; + + /* + if ((pfile_in_zip_read_info->rest_read_uncompressed == + pfile_in_zip_read_info->stream.avail_out) && + (pfile_in_zip_read_info->rest_read_compressed == 0)) + flush = Z_FINISH; + */ + err=inflate(&pfile_in_zip_read_info->stream,flush); + + if ((err>=0) && (pfile_in_zip_read_info->stream.msg!=NULL)) + err = Z_DATA_ERROR; + + uTotalOutAfter = pfile_in_zip_read_info->stream.total_out; + uOutThis = uTotalOutAfter-uTotalOutBefore; + + pfile_in_zip_read_info->total_out_64 = pfile_in_zip_read_info->total_out_64 + uOutThis; + + pfile_in_zip_read_info->crc32 = + crc32(pfile_in_zip_read_info->crc32,bufBefore, + (uInt)(uOutThis)); + + pfile_in_zip_read_info->rest_read_uncompressed -= + uOutThis; + + iRead += (uInt)(uTotalOutAfter - uTotalOutBefore); + + if (err==Z_STREAM_END) + return (iRead==0) ? UNZ_EOF : iRead; + if (err!=Z_OK) + break; + } + } + + if (err==Z_OK) + return iRead; + return err; +} + + +/* + Give the current position in uncompressed data +*/ +extern z_off_t ZEXPORT unztell (unzFile file) +{ + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + pfile_in_zip_read_info=s->pfile_in_zip_read; + + if (pfile_in_zip_read_info==NULL) + return UNZ_PARAMERROR; + + return (z_off_t)pfile_in_zip_read_info->stream.total_out; +} + +extern ZPOS64_T ZEXPORT unztell64 (unzFile file) +{ + + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + if (file==NULL) + return (ZPOS64_T)-1; + s=(unz64_s*)file; + pfile_in_zip_read_info=s->pfile_in_zip_read; + + if (pfile_in_zip_read_info==NULL) + return (ZPOS64_T)-1; + + return pfile_in_zip_read_info->total_out_64; +} + + +/* + return 1 if the end of file was reached, 0 elsewhere +*/ +extern int ZEXPORT unzeof (unzFile file) +{ + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + pfile_in_zip_read_info=s->pfile_in_zip_read; + + if (pfile_in_zip_read_info==NULL) + return UNZ_PARAMERROR; + + if (pfile_in_zip_read_info->rest_read_uncompressed == 0) + return 1; + else + return 0; +} + + + +/* +Read extra field from the current file (opened by unzOpenCurrentFile) +This is the local-header version of the extra field (sometimes, there is +more info in the local-header version than in the central-header) + + if buf==NULL, it return the size of the local extra field that can be read + + if buf!=NULL, len is the size of the buffer, the extra header is copied in + buf. + the return value is the number of bytes copied in buf, or (if <0) + the error code +*/ +extern int ZEXPORT unzGetLocalExtrafield (unzFile file, voidp buf, unsigned len) +{ + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + uInt read_now; + ZPOS64_T size_to_read; + + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + pfile_in_zip_read_info=s->pfile_in_zip_read; + + if (pfile_in_zip_read_info==NULL) + return UNZ_PARAMERROR; + + size_to_read = (pfile_in_zip_read_info->size_local_extrafield - + pfile_in_zip_read_info->pos_local_extrafield); + + if (buf==NULL) + return (int)size_to_read; + + if (len>size_to_read) + read_now = (uInt)size_to_read; + else + read_now = (uInt)len ; + + if (read_now==0) + return 0; + + if (ZSEEK64(pfile_in_zip_read_info->z_filefunc, + pfile_in_zip_read_info->filestream, + pfile_in_zip_read_info->offset_local_extrafield + + pfile_in_zip_read_info->pos_local_extrafield, + ZLIB_FILEFUNC_SEEK_SET)!=0) + return UNZ_ERRNO; + + if (ZREAD64(pfile_in_zip_read_info->z_filefunc, + pfile_in_zip_read_info->filestream, + buf,read_now)!=read_now) + return UNZ_ERRNO; + + return (int)read_now; +} + +/* + Close the file in zip opened with unzOpenCurrentFile + Return UNZ_CRCERROR if all the file was read but the CRC is not good +*/ +extern int ZEXPORT unzCloseCurrentFile (unzFile file) +{ + int err=UNZ_OK; + + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + pfile_in_zip_read_info=s->pfile_in_zip_read; + + if (pfile_in_zip_read_info==NULL) + return UNZ_PARAMERROR; + + + if ((pfile_in_zip_read_info->rest_read_uncompressed == 0) && + (!pfile_in_zip_read_info->raw)) + { + if (pfile_in_zip_read_info->crc32 != pfile_in_zip_read_info->crc32_wait) + err=UNZ_CRCERROR; + } + + + TRYFREE(pfile_in_zip_read_info->read_buffer); + pfile_in_zip_read_info->read_buffer = NULL; + if (pfile_in_zip_read_info->stream_initialised == Z_DEFLATED) + inflateEnd(&pfile_in_zip_read_info->stream); +#ifdef HAVE_BZIP2 + else if (pfile_in_zip_read_info->stream_initialised == Z_BZIP2ED) + BZ2_bzDecompressEnd(&pfile_in_zip_read_info->bstream); +#endif + + + pfile_in_zip_read_info->stream_initialised = 0; + TRYFREE(pfile_in_zip_read_info); + + s->pfile_in_zip_read=NULL; + + return err; +} + + +/* + Get the global comment string of the ZipFile, in the szComment buffer. + uSizeBuf is the size of the szComment buffer. + return the number of byte copied or an error code <0 +*/ +extern int ZEXPORT unzGetGlobalComment (unzFile file, char * szComment, uLong uSizeBuf) +{ + unz64_s* s; + uLong uReadThis ; + if (file==NULL) + return (int)UNZ_PARAMERROR; + s=(unz64_s*)file; + + uReadThis = uSizeBuf; + if (uReadThis>s->gi.size_comment) + uReadThis = s->gi.size_comment; + + if (ZSEEK64(s->z_filefunc,s->filestream,s->central_pos+22,ZLIB_FILEFUNC_SEEK_SET)!=0) + return UNZ_ERRNO; + + if (uReadThis>0) + { + *szComment='\0'; + if (ZREAD64(s->z_filefunc,s->filestream,szComment,uReadThis)!=uReadThis) + return UNZ_ERRNO; + } + + if ((szComment != NULL) && (uSizeBuf > s->gi.size_comment)) + *(szComment+s->gi.size_comment)='\0'; + return (int)uReadThis; +} + +/* Additions by RX '2004 */ +extern ZPOS64_T ZEXPORT unzGetOffset64(unzFile file) +{ + unz64_s* s; + + if (file==NULL) + return 0; //UNZ_PARAMERROR; + s=(unz64_s*)file; + if (!s->current_file_ok) + return 0; + if (s->gi.number_entry != 0 && s->gi.number_entry != 0xffff) + if (s->num_file==s->gi.number_entry) + return 0; + return s->pos_in_central_dir; +} + +extern uLong ZEXPORT unzGetOffset (unzFile file) +{ + ZPOS64_T offset64; + + if (file==NULL) + return 0; //UNZ_PARAMERROR; + offset64 = unzGetOffset64(file); + return (uLong)offset64; +} + +extern int ZEXPORT unzSetOffset64(unzFile file, ZPOS64_T pos) +{ + unz64_s* s; + int err; + + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + + s->pos_in_central_dir = pos; + s->num_file = s->gi.number_entry; /* hack */ + err = unz64local_GetCurrentFileInfoInternal(file,&s->cur_file_info, + &s->cur_file_info_internal, + NULL,0,NULL,0,NULL,0); + s->current_file_ok = (err == UNZ_OK); + return err; +} + +extern int ZEXPORT unzSetOffset (unzFile file, uLong pos) +{ + return unzSetOffset64(file,pos); +} diff --git a/storage/connect/unzip.h b/storage/connect/unzip.h new file mode 100644 index 00000000000..2104e391507 --- /dev/null +++ b/storage/connect/unzip.h @@ -0,0 +1,437 @@ +/* unzip.h -- IO for uncompress .zip files using zlib + Version 1.1, February 14h, 2010 + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications of Unzip for Zip64 + Copyright (C) 2007-2008 Even Rouault + + Modifications for Zip64 support on both zip and unzip + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + + --------------------------------------------------------------------------------- + + Condition of use and distribution are the same than zlib : + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + --------------------------------------------------------------------------------- + + Changes + + See header of unzip64.c + +*/ + +#ifndef _unz64_H +#define _unz64_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef _ZLIB_H +#include "zlib.h" +#endif + +#ifndef _ZLIBIOAPI_H +#include "ioapi.h" +#endif + +#ifdef HAVE_BZIP2 +#include "bzlib.h" +#endif + +#define Z_BZIP2ED 12 + +#if defined(STRICTUNZIP) || defined(STRICTZIPUNZIP) +/* like the STRICT of WIN32, we define a pointer that cannot be converted + from (void*) without cast */ +typedef struct TagunzFile__ { int unused; } unzFile__; +typedef unzFile__ *unzFile; +#else +typedef voidp unzFile; +#endif + + +#define UNZ_OK (0) +#define UNZ_END_OF_LIST_OF_FILE (-100) +#define UNZ_ERRNO (Z_ERRNO) +#define UNZ_EOF (0) +#define UNZ_PARAMERROR (-102) +#define UNZ_BADZIPFILE (-103) +#define UNZ_INTERNALERROR (-104) +#define UNZ_CRCERROR (-105) + +/* tm_unz contain date/time info */ +typedef struct tm_unz_s +{ + uInt tm_sec; /* seconds after the minute - [0,59] */ + uInt tm_min; /* minutes after the hour - [0,59] */ + uInt tm_hour; /* hours since midnight - [0,23] */ + uInt tm_mday; /* day of the month - [1,31] */ + uInt tm_mon; /* months since January - [0,11] */ + uInt tm_year; /* years - [1980..2044] */ +} tm_unz; + +/* unz_global_info structure contain global data about the ZIPfile + These data comes from the end of central dir */ +typedef struct unz_global_info64_s +{ + ZPOS64_T number_entry; /* total number of entries in + the central dir on this disk */ + uLong size_comment; /* size of the global comment of the zipfile */ +} unz_global_info64; + +typedef struct unz_global_info_s +{ + uLong number_entry; /* total number of entries in + the central dir on this disk */ + uLong size_comment; /* size of the global comment of the zipfile */ +} unz_global_info; + +/* unz_file_info contain information about a file in the zipfile */ +typedef struct unz_file_info64_s +{ + uLong version; /* version made by 2 bytes */ + uLong version_needed; /* version needed to extract 2 bytes */ + uLong flag; /* general purpose bit flag 2 bytes */ + uLong compression_method; /* compression method 2 bytes */ + uLong dosDate; /* last mod file date in Dos fmt 4 bytes */ + uLong crc; /* crc-32 4 bytes */ + ZPOS64_T compressed_size; /* compressed size 8 bytes */ + ZPOS64_T uncompressed_size; /* uncompressed size 8 bytes */ + uLong size_filename; /* filename length 2 bytes */ + uLong size_file_extra; /* extra field length 2 bytes */ + uLong size_file_comment; /* file comment length 2 bytes */ + + uLong disk_num_start; /* disk number start 2 bytes */ + uLong internal_fa; /* internal file attributes 2 bytes */ + uLong external_fa; /* external file attributes 4 bytes */ + + tm_unz tmu_date; +} unz_file_info64; + +typedef struct unz_file_info_s +{ + uLong version; /* version made by 2 bytes */ + uLong version_needed; /* version needed to extract 2 bytes */ + uLong flag; /* general purpose bit flag 2 bytes */ + uLong compression_method; /* compression method 2 bytes */ + uLong dosDate; /* last mod file date in Dos fmt 4 bytes */ + uLong crc; /* crc-32 4 bytes */ + uLong compressed_size; /* compressed size 4 bytes */ + uLong uncompressed_size; /* uncompressed size 4 bytes */ + uLong size_filename; /* filename length 2 bytes */ + uLong size_file_extra; /* extra field length 2 bytes */ + uLong size_file_comment; /* file comment length 2 bytes */ + + uLong disk_num_start; /* disk number start 2 bytes */ + uLong internal_fa; /* internal file attributes 2 bytes */ + uLong external_fa; /* external file attributes 4 bytes */ + + tm_unz tmu_date; +} unz_file_info; + +extern int ZEXPORT unzStringFileNameCompare OF ((const char* fileName1, + const char* fileName2, + int iCaseSensitivity)); +/* + Compare two filename (fileName1,fileName2). + If iCaseSenisivity = 1, comparision is case sensitivity (like strcmp) + If iCaseSenisivity = 2, comparision is not case sensitivity (like strcmpi + or strcasecmp) + If iCaseSenisivity = 0, case sensitivity is defaut of your operating system + (like 1 on Unix, 2 on Windows) +*/ + + +extern unzFile ZEXPORT unzOpen OF((const char *path)); +extern unzFile ZEXPORT unzOpen64 OF((const void *path)); +/* + Open a Zip file. path contain the full pathname (by example, + on a Windows XP computer "c:\\zlib\\zlib113.zip" or on an Unix computer + "zlib/zlib113.zip". + If the zipfile cannot be opened (file don't exist or in not valid), the + return value is NULL. + Else, the return value is a unzFile Handle, usable with other function + of this unzip package. + the "64" function take a const void* pointer, because the path is just the + value passed to the open64_file_func callback. + Under Windows, if UNICODE is defined, using fill_fopen64_filefunc, the path + is a pointer to a wide unicode string (LPCTSTR is LPCWSTR), so const char* + does not describe the reality +*/ + + +extern unzFile ZEXPORT unzOpen2 OF((const char *path, + zlib_filefunc_def* pzlib_filefunc_def)); +/* + Open a Zip file, like unzOpen, but provide a set of file low level API + for read/write the zip file (see ioapi.h) +*/ + +extern unzFile ZEXPORT unzOpen2_64 OF((const void *path, + zlib_filefunc64_def* pzlib_filefunc_def)); +/* + Open a Zip file, like unz64Open, but provide a set of file low level API + for read/write the zip file (see ioapi.h) +*/ + +extern int ZEXPORT unzClose OF((unzFile file)); +/* + Close a ZipFile opened with unzOpen. + If there is files inside the .Zip opened with unzOpenCurrentFile (see later), + these files MUST be closed with unzCloseCurrentFile before call unzClose. + return UNZ_OK if there is no problem. */ + +extern int ZEXPORT unzGetGlobalInfo OF((unzFile file, + unz_global_info *pglobal_info)); + +extern int ZEXPORT unzGetGlobalInfo64 OF((unzFile file, + unz_global_info64 *pglobal_info)); +/* + Write info about the ZipFile in the *pglobal_info structure. + No preparation of the structure is needed + return UNZ_OK if there is no problem. */ + + +extern int ZEXPORT unzGetGlobalComment OF((unzFile file, + char *szComment, + uLong uSizeBuf)); +/* + Get the global comment string of the ZipFile, in the szComment buffer. + uSizeBuf is the size of the szComment buffer. + return the number of byte copied or an error code <0 +*/ + + +/***************************************************************************/ +/* Unzip package allow you browse the directory of the zipfile */ + +extern int ZEXPORT unzGoToFirstFile OF((unzFile file)); +/* + Set the current file of the zipfile to the first file. + return UNZ_OK if there is no problem +*/ + +extern int ZEXPORT unzGoToNextFile OF((unzFile file)); +/* + Set the current file of the zipfile to the next file. + return UNZ_OK if there is no problem + return UNZ_END_OF_LIST_OF_FILE if the actual file was the latest. +*/ + +extern int ZEXPORT unzLocateFile OF((unzFile file, + const char *szFileName, + int iCaseSensitivity)); +/* + Try locate the file szFileName in the zipfile. + For the iCaseSensitivity signification, see unzStringFileNameCompare + + return value : + UNZ_OK if the file is found. It becomes the current file. + UNZ_END_OF_LIST_OF_FILE if the file is not found +*/ + + +/* ****************************************** */ +/* Ryan supplied functions */ +/* unz_file_info contain information about a file in the zipfile */ +typedef struct unz_file_pos_s +{ + uLong pos_in_zip_directory; /* offset in zip file directory */ + uLong num_of_file; /* # of file */ +} unz_file_pos; + +extern int ZEXPORT unzGetFilePos( + unzFile file, + unz_file_pos* file_pos); + +extern int ZEXPORT unzGoToFilePos( + unzFile file, + unz_file_pos* file_pos); + +typedef struct unz64_file_pos_s +{ + ZPOS64_T pos_in_zip_directory; /* offset in zip file directory */ + ZPOS64_T num_of_file; /* # of file */ +} unz64_file_pos; + +extern int ZEXPORT unzGetFilePos64( + unzFile file, + unz64_file_pos* file_pos); + +extern int ZEXPORT unzGoToFilePos64( + unzFile file, + const unz64_file_pos* file_pos); + +/* ****************************************** */ + +extern int ZEXPORT unzGetCurrentFileInfo64 OF((unzFile file, + unz_file_info64 *pfile_info, + char *szFileName, + uLong fileNameBufferSize, + void *extraField, + uLong extraFieldBufferSize, + char *szComment, + uLong commentBufferSize)); + +extern int ZEXPORT unzGetCurrentFileInfo OF((unzFile file, + unz_file_info *pfile_info, + char *szFileName, + uLong fileNameBufferSize, + void *extraField, + uLong extraFieldBufferSize, + char *szComment, + uLong commentBufferSize)); +/* + Get Info about the current file + if pfile_info!=NULL, the *pfile_info structure will contain somes info about + the current file + if szFileName!=NULL, the filemane string will be copied in szFileName + (fileNameBufferSize is the size of the buffer) + if extraField!=NULL, the extra field information will be copied in extraField + (extraFieldBufferSize is the size of the buffer). + This is the Central-header version of the extra field + if szComment!=NULL, the comment string of the file will be copied in szComment + (commentBufferSize is the size of the buffer) +*/ + + +/** Addition for GDAL : START */ + +extern ZPOS64_T ZEXPORT unzGetCurrentFileZStreamPos64 OF((unzFile file)); + +/** Addition for GDAL : END */ + + +/***************************************************************************/ +/* for reading the content of the current zipfile, you can open it, read data + from it, and close it (you can close it before reading all the file) + */ + +extern int ZEXPORT unzOpenCurrentFile OF((unzFile file)); +/* + Open for reading data the current file in the zipfile. + If there is no error, the return value is UNZ_OK. +*/ + +extern int ZEXPORT unzOpenCurrentFilePassword OF((unzFile file, + const char* password)); +/* + Open for reading data the current file in the zipfile. + password is a crypting password + If there is no error, the return value is UNZ_OK. +*/ + +extern int ZEXPORT unzOpenCurrentFile2 OF((unzFile file, + int* method, + int* level, + int raw)); +/* + Same than unzOpenCurrentFile, but open for read raw the file (not uncompress) + if raw==1 + *method will receive method of compression, *level will receive level of + compression + note : you can set level parameter as NULL (if you did not want known level, + but you CANNOT set method parameter as NULL +*/ + +extern int ZEXPORT unzOpenCurrentFile3 OF((unzFile file, + int* method, + int* level, + int raw, + const char* password)); +/* + Same than unzOpenCurrentFile, but open for read raw the file (not uncompress) + if raw==1 + *method will receive method of compression, *level will receive level of + compression + note : you can set level parameter as NULL (if you did not want known level, + but you CANNOT set method parameter as NULL +*/ + + +extern int ZEXPORT unzCloseCurrentFile OF((unzFile file)); +/* + Close the file in zip opened with unzOpenCurrentFile + Return UNZ_CRCERROR if all the file was read but the CRC is not good +*/ + +extern int ZEXPORT unzReadCurrentFile OF((unzFile file, + voidp buf, + unsigned len)); +/* + Read bytes from the current file (opened by unzOpenCurrentFile) + buf contain buffer where data must be copied + len the size of buf. + + return the number of byte copied if somes bytes are copied + return 0 if the end of file was reached + return <0 with error code if there is an error + (UNZ_ERRNO for IO error, or zLib error for uncompress error) +*/ + +extern z_off_t ZEXPORT unztell OF((unzFile file)); + +extern ZPOS64_T ZEXPORT unztell64 OF((unzFile file)); +/* + Give the current position in uncompressed data +*/ + +extern int ZEXPORT unzeof OF((unzFile file)); +/* + return 1 if the end of file was reached, 0 elsewhere +*/ + +extern int ZEXPORT unzGetLocalExtrafield OF((unzFile file, + voidp buf, + unsigned len)); +/* + Read extra field from the current file (opened by unzOpenCurrentFile) + This is the local-header version of the extra field (sometimes, there is + more info in the local-header version than in the central-header) + + if buf==NULL, it return the size of the local extra field + + if buf!=NULL, len is the size of the buffer, the extra header is copied in + buf. + the return value is the number of bytes copied in buf, or (if <0) + the error code +*/ + +/***************************************************************************/ + +/* Get the current file offset */ +extern ZPOS64_T ZEXPORT unzGetOffset64 (unzFile file); +extern uLong ZEXPORT unzGetOffset (unzFile file); + +/* Set the current file offset */ +extern int ZEXPORT unzSetOffset64 (unzFile file, ZPOS64_T pos); +extern int ZEXPORT unzSetOffset (unzFile file, uLong pos); + + + +#ifdef __cplusplus +} +#endif + +#endif /* _unz64_H */ diff --git a/storage/connect/value.cpp b/storage/connect/value.cpp index 64d0e13e8c4..ced690e77c0 100644 --- a/storage/connect/value.cpp +++ b/storage/connect/value.cpp @@ -1,7 +1,7 @@ /************* Value C++ Functions Source Code File (.CPP) *************/ -/* Name: VALUE.CPP Version 2.5 */ +/* Name: VALUE.CPP Version 2.6 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 2001-2015 */ +/* (C) Copyright to the author Olivier BERTRAND 2001-2016 */ /* */ /* This file contains the VALUE and derived classes family functions. */ /* These classes contain values of different types. They are used so */ @@ -792,19 +792,29 @@ uchar TYPVAL<uchar>::GetTypedValue(PVBLK blk, int n) /***********************************************************************/ /* TYPVAL SetBinValue: with bytes extracted from a line. */ +/* Currently only used reading column of binary files. */ /***********************************************************************/ template <class TYPE> void TYPVAL<TYPE>::SetBinValue(void *p) - { - Tval = *(TYPE *)p; - Null = false; - } // end of SetBinValue +{ +#if defined(UNALIGNED_OK) + // x86 can cast non-aligned memory directly + Tval = *(TYPE *)p; +#else + // Prevent unaligned memory access on MIPS and ArmHF platforms. + // Make use of memcpy instead of straight pointer dereferencing. + // Currently only used by WriteColumn of binary files. + // From original author: Vicentiu Ciorbaru <vicentiu@mariadb.org> + memcpy(&Tval, p, sizeof(TYPE)); +#endif + Null = false; +} // end of SetBinValue /***********************************************************************/ /* GetBinValue: fill a buffer with the internal binary value. */ /* This function checks whether the buffer length is enough and */ /* returns true if not. Actual filling occurs only if go is true. */ -/* Currently used by WriteColumn of binary files. */ +/* Currently only used writing column of binary files. */ /***********************************************************************/ template <class TYPE> bool TYPVAL<TYPE>::GetBinValue(void *buf, int buflen, bool go) @@ -819,7 +829,16 @@ bool TYPVAL<TYPE>::GetBinValue(void *buf, int buflen, bool go) //#endif if (go) - *(TYPE *)buf = Tval; +#if defined(UNALIGNED_OK) + // x86 can cast non-aligned memory directly + *(TYPE *)buf = Tval; +#else + // Prevent unaligned memory access on MIPS and ArmHF platforms. + // Make use of memcpy instead of straight pointer dereferencing. + // Currently only used by WriteColumn of binary files. + // From original author: Vicentiu Ciorbaru <vicentiu@mariadb.org> + memcpy(buf, &Tval, sizeof(TYPE)); +#endif Null = false; return false; diff --git a/storage/connect/value.h b/storage/connect/value.h index c5a381e89da..a670ade4c28 100644 --- a/storage/connect/value.h +++ b/storage/connect/value.h @@ -1,7 +1,7 @@ /**************** Value H Declares Source Code File (.H) ***************/ -/* Name: VALUE.H Version 2.1 */ +/* Name: VALUE.H Version 2.2 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 2001-2014 */ +/* (C) Copyright to the author Olivier BERTRAND 2001-2016 */ /* */ /* This file contains the VALUE and derived classes declares. */ /***********************************************************************/ @@ -17,6 +17,13 @@ #include "block.h" /***********************************************************************/ +/* This should list the processors accepting unaligned numeral values.*/ +/***********************************************************************/ +#if defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) || defined(_M_X64) || defined(_M_AMD64) || defined(_M_IA64) +#define UNALIGNED_OK +#endif + +/***********************************************************************/ /* Types used in some class definitions. */ /***********************************************************************/ enum CONV {CNV_ANY = 0, /* Convert to any type */ @@ -116,27 +123,46 @@ class DllExport VALUE : public BLOCK { virtual bool Compute(PGLOBAL g, PVAL *vp, int np, OPVAL op); virtual bool FormatValue(PVAL vp, char *fmt) = 0; - /** - Set value from a non-aligned in-memory value in the machine byte order. - TYPE can be either of: - - int, short, longlong - - uint, ushort, ulonglong - - float, double - @param - a pointer to a non-aligned value of type TYPE. - */ - template<typename TYPE> - void SetValueNonAligned(const char *p) - { -#if defined(__i386__) || defined(__x86_64__) - SetValue(*((TYPE*) p)); // x86 can cast non-aligned memory directly + /** + Set value from a non-aligned in-memory value in the machine byte order. + TYPE can be either of: + - int, short, longlong + - uint, ushort, ulonglong + - float, double + @param - a pointer to a non-aligned value of type TYPE. + */ + template<typename TYPE> + void SetValueNonAligned(const char *p) + { +#if defined(UNALIGNED_OK) + SetValue(*((TYPE*)p)); // x86 can cast non-aligned memory directly #else - TYPE tmp; // a slower version for non-x86 platforms - memcpy(&tmp, p, sizeof(tmp)); - SetValue(tmp); + TYPE tmp; // a slower version for non-x86 platforms + memcpy(&tmp, p, sizeof(tmp)); + SetValue(tmp); #endif - } + } // end of SetValueNonAligned + + /** + Get value from a non-aligned in-memory value in the machine byte order. + TYPE can be either of: + - int, short, longlong + - uint, ushort, ulonglong + - float, double + @params - a pointer to a non-aligned value of type TYPE, the TYPE value. + */ + template<typename TYPE> + void GetValueNonAligned(char *p, TYPE n) + { +#if defined(UNALIGNED_OK) + *(TYPE *)p = n; // x86 can cast non-aligned memory directly +#else + TYPE tmp = n; // a slower version for non-x86 platforms + memcpy(p, &tmp, sizeof(tmp)); +#endif + } // end of SetValueNonAligned - protected: +protected: virtual bool SetConstFormat(PGLOBAL, FORMAT&) = 0; const char *GetXfmt(void); diff --git a/storage/connect/xindex.cpp b/storage/connect/xindex.cpp index 56312630278..a2cf4e77b80 100755 --- a/storage/connect/xindex.cpp +++ b/storage/connect/xindex.cpp @@ -45,7 +45,9 @@ //nclude "array.h" #include "filamtxt.h" #include "tabdos.h" +#if defined(VCT_SUPPORT) #include "tabvct.h" +#endif // VCT_SUPPORT /***********************************************************************/ /* Macro or external routine definition */ @@ -293,9 +295,11 @@ bool XINDEX::AddColumns(void) return false; // Not applying to static index else if (IsMul()) return false; // Not done yet for multiple index - else if (Tbxp->GetAmType() == TYPE_AM_VCT && ((PTDBVCT)Tbxp)->IsSplit()) +#if defined(VCT_SUPPORT) + else if (Tbxp->GetAmType() == TYPE_AM_VCT && ((PTDBVCT)Tbxp)->IsSplit()) return false; // This would require to read additional files - else +#endif // VCT_SUPPORT + else return true; } // end of AddColumns diff --git a/storage/connect/zip.c b/storage/connect/zip.c new file mode 100644 index 00000000000..ea54853e858 --- /dev/null +++ b/storage/connect/zip.c @@ -0,0 +1,2007 @@ +/* zip.c -- IO on .zip files using zlib + Version 1.1, February 14h, 2010 + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications for Zip64 support + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + + Changes + Oct-2009 - Mathias Svensson - Remove old C style function prototypes + Oct-2009 - Mathias Svensson - Added Zip64 Support when creating new file archives + Oct-2009 - Mathias Svensson - Did some code cleanup and refactoring to get better overview of some functions. + Oct-2009 - Mathias Svensson - Added zipRemoveExtraInfoBlock to strip extra field data from its ZIP64 data + It is used when recreting zip archive with RAW when deleting items from a zip. + ZIP64 data is automaticly added to items that needs it, and existing ZIP64 data need to be removed. + Oct-2009 - Mathias Svensson - Added support for BZIP2 as compression mode (bzip2 lib is required) + Jan-2010 - back to unzip and minizip 1.0 name scheme, with compatibility layer + +*/ + + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include "zlib.h" +#include "zip.h" + +#ifdef STDC +# include <stddef.h> +# include <string.h> +# include <stdlib.h> +#endif +#ifdef NO_ERRNO_H + extern int errno; +#else +# include <errno.h> +#endif + + +#ifndef local +# define local static +#endif +/* compile with -Dlocal if your debugger can't find static symbols */ + +#ifndef VERSIONMADEBY +# define VERSIONMADEBY (0x0) /* platform depedent */ +#endif + +#ifndef Z_BUFSIZE +#define Z_BUFSIZE (64*1024) //(16384) +#endif + +#ifndef Z_MAXFILENAMEINZIP +#define Z_MAXFILENAMEINZIP (256) +#endif + +#ifndef ALLOC +# define ALLOC(size) (malloc(size)) +#endif +#ifndef TRYFREE +# define TRYFREE(p) {if (p) free(p);} +#endif + +/* +#define SIZECENTRALDIRITEM (0x2e) +#define SIZEZIPLOCALHEADER (0x1e) +*/ + +/* I've found an old Unix (a SunOS 4.1.3_U1) without all SEEK_* defined.... */ + + +// NOT sure that this work on ALL platform +#define MAKEULONG64(a, b) ((ZPOS64_T)(((unsigned long)(a)) | ((ZPOS64_T)((unsigned long)(b))) << 32)) + +#ifndef SEEK_CUR +#define SEEK_CUR 1 +#endif + +#ifndef SEEK_END +#define SEEK_END 2 +#endif + +#ifndef SEEK_SET +#define SEEK_SET 0 +#endif + +#ifndef DEF_MEM_LEVEL +#if MAX_MEM_LEVEL >= 8 +# define DEF_MEM_LEVEL 8 +#else +# define DEF_MEM_LEVEL MAX_MEM_LEVEL +#endif +#endif +const char zip_copyright[] =" zip 1.01 Copyright 1998-2004 Gilles Vollant - http://www.winimage.com/zLibDll"; + + +#define SIZEDATA_INDATABLOCK (4096-(4*4)) + +#define LOCALHEADERMAGIC (0x04034b50) +#define CENTRALHEADERMAGIC (0x02014b50) +#define ENDHEADERMAGIC (0x06054b50) +#define ZIP64ENDHEADERMAGIC (0x6064b50) +#define ZIP64ENDLOCHEADERMAGIC (0x7064b50) + +#define FLAG_LOCALHEADER_OFFSET (0x06) +#define CRC_LOCALHEADER_OFFSET (0x0e) + +#define SIZECENTRALHEADER (0x2e) /* 46 */ + +typedef struct linkedlist_datablock_internal_s +{ + struct linkedlist_datablock_internal_s* next_datablock; + uLong avail_in_this_block; + uLong filled_in_this_block; + uLong unused; /* for future use and alignement */ + unsigned char data[SIZEDATA_INDATABLOCK]; +} linkedlist_datablock_internal; + +typedef struct linkedlist_data_s +{ + linkedlist_datablock_internal* first_block; + linkedlist_datablock_internal* last_block; +} linkedlist_data; + + +typedef struct +{ + z_stream stream; /* zLib stream structure for inflate */ +#ifdef HAVE_BZIP2 + bz_stream bstream; /* bzLib stream structure for bziped */ +#endif + + int stream_initialised; /* 1 is stream is initialised */ + uInt pos_in_buffered_data; /* last written byte in buffered_data */ + + ZPOS64_T pos_local_header; /* offset of the local header of the file + currenty writing */ + char* central_header; /* central header data for the current file */ + uLong size_centralExtra; + uLong size_centralheader; /* size of the central header for cur file */ + uLong size_centralExtraFree; /* Extra bytes allocated to the centralheader but that are not used */ + uLong flag; /* flag of the file currently writing */ + + int method; /* compression method of file currenty wr.*/ + int raw; /* 1 for directly writing raw data */ + Byte buffered_data[Z_BUFSIZE];/* buffer contain compressed data to be writ*/ + uLong dosDate; + uLong crc32; + int encrypt; + int zip64; /* Add ZIP64 extened information in the extra field */ + ZPOS64_T pos_zip64extrainfo; + ZPOS64_T totalCompressedData; + ZPOS64_T totalUncompressedData; +#ifndef NOCRYPT + unsigned long keys[3]; /* keys defining the pseudo-random sequence */ + const z_crc_t* pcrc_32_tab; + int crypt_header_size; +#endif +} curfile64_info; + +typedef struct +{ + zlib_filefunc64_32_def z_filefunc; + voidpf filestream; /* io structore of the zipfile */ + linkedlist_data central_dir;/* datablock with central dir in construction*/ + int in_opened_file_inzip; /* 1 if a file in the zip is currently writ.*/ + curfile64_info ci; /* info on the file curretly writing */ + + ZPOS64_T begin_pos; /* position of the beginning of the zipfile */ + ZPOS64_T add_position_when_writting_offset; + ZPOS64_T number_entry; + +#ifndef NO_ADDFILEINEXISTINGZIP + char *globalcomment; +#endif + +} zip64_internal; + + +#ifndef NOCRYPT +#define INCLUDECRYPTINGCODE_IFCRYPTALLOWED +#include "crypt.h" +#endif + +local linkedlist_datablock_internal* allocate_new_datablock() +{ + linkedlist_datablock_internal* ldi; + ldi = (linkedlist_datablock_internal*) + ALLOC(sizeof(linkedlist_datablock_internal)); + if (ldi!=NULL) + { + ldi->next_datablock = NULL ; + ldi->filled_in_this_block = 0 ; + ldi->avail_in_this_block = SIZEDATA_INDATABLOCK ; + } + return ldi; +} + +local void free_datablock(linkedlist_datablock_internal* ldi) +{ + while (ldi!=NULL) + { + linkedlist_datablock_internal* ldinext = ldi->next_datablock; + TRYFREE(ldi); + ldi = ldinext; + } +} + +local void init_linkedlist(linkedlist_data* ll) +{ + ll->first_block = ll->last_block = NULL; +} + +local void free_linkedlist(linkedlist_data* ll) +{ + free_datablock(ll->first_block); + ll->first_block = ll->last_block = NULL; +} + + +local int add_data_in_datablock(linkedlist_data* ll, const void* buf, uLong len) +{ + linkedlist_datablock_internal* ldi; + const unsigned char* from_copy; + + if (ll==NULL) + return ZIP_INTERNALERROR; + + if (ll->last_block == NULL) + { + ll->first_block = ll->last_block = allocate_new_datablock(); + if (ll->first_block == NULL) + return ZIP_INTERNALERROR; + } + + ldi = ll->last_block; + from_copy = (unsigned char*)buf; + + while (len>0) + { + uInt copy_this; + uInt i; + unsigned char* to_copy; + + if (ldi->avail_in_this_block==0) + { + ldi->next_datablock = allocate_new_datablock(); + if (ldi->next_datablock == NULL) + return ZIP_INTERNALERROR; + ldi = ldi->next_datablock ; + ll->last_block = ldi; + } + + if (ldi->avail_in_this_block < len) + copy_this = (uInt)ldi->avail_in_this_block; + else + copy_this = (uInt)len; + + to_copy = &(ldi->data[ldi->filled_in_this_block]); + + for (i=0;i<copy_this;i++) + *(to_copy+i)=*(from_copy+i); + + ldi->filled_in_this_block += copy_this; + ldi->avail_in_this_block -= copy_this; + from_copy += copy_this ; + len -= copy_this; + } + return ZIP_OK; +} + + + +/****************************************************************************/ + +#ifndef NO_ADDFILEINEXISTINGZIP +/* =========================================================================== + Inputs a long in LSB order to the given file + nbByte == 1, 2 ,4 or 8 (byte, short or long, ZPOS64_T) +*/ + +local int zip64local_putValue OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, ZPOS64_T x, int nbByte)); +local int zip64local_putValue (const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, ZPOS64_T x, int nbByte) +{ + unsigned char buf[8]; + int n; + for (n = 0; n < nbByte; n++) + { + buf[n] = (unsigned char)(x & 0xff); + x >>= 8; + } + if (x != 0) + { /* data overflow - hack for ZIP64 (X Roche) */ + for (n = 0; n < nbByte; n++) + { + buf[n] = 0xff; + } + } + + if (ZWRITE64(*pzlib_filefunc_def,filestream,buf,nbByte)!=(uLong)nbByte) + return ZIP_ERRNO; + else + return ZIP_OK; +} + +local void zip64local_putValue_inmemory OF((void* dest, ZPOS64_T x, int nbByte)); +local void zip64local_putValue_inmemory (void* dest, ZPOS64_T x, int nbByte) +{ + unsigned char* buf=(unsigned char*)dest; + int n; + for (n = 0; n < nbByte; n++) { + buf[n] = (unsigned char)(x & 0xff); + x >>= 8; + } + + if (x != 0) + { /* data overflow - hack for ZIP64 */ + for (n = 0; n < nbByte; n++) + { + buf[n] = 0xff; + } + } +} + +/****************************************************************************/ + + +local uLong zip64local_TmzDateToDosDate(const tm_zip* ptm) +{ + uLong year = (uLong)ptm->tm_year; + if (year>=1980) + year-=1980; + else if (year>=80) + year-=80; + return + (uLong) (((ptm->tm_mday) + (32 * (ptm->tm_mon+1)) + (512 * year)) << 16) | + ((ptm->tm_sec/2) + (32* ptm->tm_min) + (2048 * (uLong)ptm->tm_hour)); +} + + +/****************************************************************************/ + +local int zip64local_getByte OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, int *pi)); + +local int zip64local_getByte(const zlib_filefunc64_32_def* pzlib_filefunc_def,voidpf filestream,int* pi) +{ + unsigned char c; + int err = (int)ZREAD64(*pzlib_filefunc_def,filestream,&c,1); + if (err==1) + { + *pi = (int)c; + return ZIP_OK; + } + else + { + if (ZERROR64(*pzlib_filefunc_def,filestream)) + return ZIP_ERRNO; + else + return ZIP_EOF; + } +} + + +/* =========================================================================== + Reads a long in LSB order from the given gz_stream. Sets +*/ +local int zip64local_getShort OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, uLong *pX)); + +local int zip64local_getShort (const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, uLong* pX) +{ + uLong x ; + int i = 0; + int err; + + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x = (uLong)i; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((uLong)i)<<8; + + if (err==ZIP_OK) + *pX = x; + else + *pX = 0; + return err; +} + +local int zip64local_getLong OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, uLong *pX)); + +local int zip64local_getLong (const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, uLong* pX) +{ + uLong x ; + int i = 0; + int err; + + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x = (uLong)i; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((uLong)i)<<8; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((uLong)i)<<16; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((uLong)i)<<24; + + if (err==ZIP_OK) + *pX = x; + else + *pX = 0; + return err; +} + +local int zip64local_getLong64 OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, ZPOS64_T *pX)); + + +local int zip64local_getLong64 (const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, ZPOS64_T *pX) +{ + ZPOS64_T x; + int i = 0; + int err; + + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x = (ZPOS64_T)i; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<8; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<16; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<24; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<32; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<40; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<48; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<56; + + if (err==ZIP_OK) + *pX = x; + else + *pX = 0; + + return err; +} + +#ifndef BUFREADCOMMENT +#define BUFREADCOMMENT (0x400) +#endif +/* + Locate the Central directory of a zipfile (at the end, just before + the global comment) +*/ +local ZPOS64_T zip64local_SearchCentralDir OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream)); + +local ZPOS64_T zip64local_SearchCentralDir(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream) +{ + unsigned char* buf; + ZPOS64_T uSizeFile; + ZPOS64_T uBackRead; + ZPOS64_T uMaxBack=0xffff; /* maximum size of global comment */ + ZPOS64_T uPosFound=0; + + if (ZSEEK64(*pzlib_filefunc_def,filestream,0,ZLIB_FILEFUNC_SEEK_END) != 0) + return 0; + + + uSizeFile = ZTELL64(*pzlib_filefunc_def,filestream); + + if (uMaxBack>uSizeFile) + uMaxBack = uSizeFile; + + buf = (unsigned char*)ALLOC(BUFREADCOMMENT+4); + if (buf==NULL) + return 0; + + uBackRead = 4; + while (uBackRead<uMaxBack) + { + uLong uReadSize; + ZPOS64_T uReadPos ; + int i; + if (uBackRead+BUFREADCOMMENT>uMaxBack) + uBackRead = uMaxBack; + else + uBackRead+=BUFREADCOMMENT; + uReadPos = uSizeFile-uBackRead ; + + uReadSize = ((BUFREADCOMMENT+4) < (uSizeFile-uReadPos)) ? + (BUFREADCOMMENT+4) : (uLong)(uSizeFile-uReadPos); + if (ZSEEK64(*pzlib_filefunc_def,filestream,uReadPos,ZLIB_FILEFUNC_SEEK_SET)!=0) + break; + + if (ZREAD64(*pzlib_filefunc_def,filestream,buf,uReadSize)!=uReadSize) + break; + + for (i=(int)uReadSize-3; (i--)>0;) + if (((*(buf+i))==0x50) && ((*(buf+i+1))==0x4b) && + ((*(buf+i+2))==0x05) && ((*(buf+i+3))==0x06)) + { + uPosFound = uReadPos+i; + break; + } + + if (uPosFound!=0) + break; + } + TRYFREE(buf); + return uPosFound; +} + +/* +Locate the End of Zip64 Central directory locator and from there find the CD of a zipfile (at the end, just before +the global comment) +*/ +local ZPOS64_T zip64local_SearchCentralDir64 OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream)); + +local ZPOS64_T zip64local_SearchCentralDir64(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream) +{ + unsigned char* buf; + ZPOS64_T uSizeFile; + ZPOS64_T uBackRead; + ZPOS64_T uMaxBack=0xffff; /* maximum size of global comment */ + ZPOS64_T uPosFound=0; + uLong uL; + ZPOS64_T relativeOffset; + + if (ZSEEK64(*pzlib_filefunc_def,filestream,0,ZLIB_FILEFUNC_SEEK_END) != 0) + return 0; + + uSizeFile = ZTELL64(*pzlib_filefunc_def,filestream); + + if (uMaxBack>uSizeFile) + uMaxBack = uSizeFile; + + buf = (unsigned char*)ALLOC(BUFREADCOMMENT+4); + if (buf==NULL) + return 0; + + uBackRead = 4; + while (uBackRead<uMaxBack) + { + uLong uReadSize; + ZPOS64_T uReadPos; + int i; + if (uBackRead+BUFREADCOMMENT>uMaxBack) + uBackRead = uMaxBack; + else + uBackRead+=BUFREADCOMMENT; + uReadPos = uSizeFile-uBackRead ; + + uReadSize = ((BUFREADCOMMENT+4) < (uSizeFile-uReadPos)) ? + (BUFREADCOMMENT+4) : (uLong)(uSizeFile-uReadPos); + if (ZSEEK64(*pzlib_filefunc_def,filestream,uReadPos,ZLIB_FILEFUNC_SEEK_SET)!=0) + break; + + if (ZREAD64(*pzlib_filefunc_def,filestream,buf,uReadSize)!=uReadSize) + break; + + for (i=(int)uReadSize-3; (i--)>0;) + { + // Signature "0x07064b50" Zip64 end of central directory locater + if (((*(buf+i))==0x50) && ((*(buf+i+1))==0x4b) && ((*(buf+i+2))==0x06) && ((*(buf+i+3))==0x07)) + { + uPosFound = uReadPos+i; + break; + } + } + + if (uPosFound!=0) + break; + } + + TRYFREE(buf); + if (uPosFound == 0) + return 0; + + /* Zip64 end of central directory locator */ + if (ZSEEK64(*pzlib_filefunc_def,filestream, uPosFound,ZLIB_FILEFUNC_SEEK_SET)!=0) + return 0; + + /* the signature, already checked */ + if (zip64local_getLong(pzlib_filefunc_def,filestream,&uL)!=ZIP_OK) + return 0; + + /* number of the disk with the start of the zip64 end of central directory */ + if (zip64local_getLong(pzlib_filefunc_def,filestream,&uL)!=ZIP_OK) + return 0; + if (uL != 0) + return 0; + + /* relative offset of the zip64 end of central directory record */ + if (zip64local_getLong64(pzlib_filefunc_def,filestream,&relativeOffset)!=ZIP_OK) + return 0; + + /* total number of disks */ + if (zip64local_getLong(pzlib_filefunc_def,filestream,&uL)!=ZIP_OK) + return 0; + if (uL != 1) + return 0; + + /* Goto Zip64 end of central directory record */ + if (ZSEEK64(*pzlib_filefunc_def,filestream, relativeOffset,ZLIB_FILEFUNC_SEEK_SET)!=0) + return 0; + + /* the signature */ + if (zip64local_getLong(pzlib_filefunc_def,filestream,&uL)!=ZIP_OK) + return 0; + + if (uL != 0x06064b50) // signature of 'Zip64 end of central directory' + return 0; + + return relativeOffset; +} + +int LoadCentralDirectoryRecord(zip64_internal* pziinit) +{ + int err=ZIP_OK; + ZPOS64_T byte_before_the_zipfile;/* byte before the zipfile, (>0 for sfx)*/ + + ZPOS64_T size_central_dir; /* size of the central directory */ + ZPOS64_T offset_central_dir; /* offset of start of central directory */ + ZPOS64_T central_pos; + uLong uL; + + uLong number_disk; /* number of the current dist, used for + spaning ZIP, unsupported, always 0*/ + uLong number_disk_with_CD; /* number the the disk with central dir, used + for spaning ZIP, unsupported, always 0*/ + ZPOS64_T number_entry; + ZPOS64_T number_entry_CD; /* total number of entries in + the central dir + (same than number_entry on nospan) */ + uLong VersionMadeBy; + uLong VersionNeeded; + uLong size_comment; + + int hasZIP64Record = 0; + + // check first if we find a ZIP64 record + central_pos = zip64local_SearchCentralDir64(&pziinit->z_filefunc,pziinit->filestream); + if(central_pos > 0) + { + hasZIP64Record = 1; + } + else if(central_pos == 0) + { + central_pos = zip64local_SearchCentralDir(&pziinit->z_filefunc,pziinit->filestream); + } + +/* disable to allow appending to empty ZIP archive + if (central_pos==0) + err=ZIP_ERRNO; +*/ + + if(hasZIP64Record) + { + ZPOS64_T sizeEndOfCentralDirectory; + if (ZSEEK64(pziinit->z_filefunc, pziinit->filestream, central_pos, ZLIB_FILEFUNC_SEEK_SET) != 0) + err=ZIP_ERRNO; + + /* the signature, already checked */ + if (zip64local_getLong(&pziinit->z_filefunc, pziinit->filestream,&uL)!=ZIP_OK) + err=ZIP_ERRNO; + + /* size of zip64 end of central directory record */ + if (zip64local_getLong64(&pziinit->z_filefunc, pziinit->filestream, &sizeEndOfCentralDirectory)!=ZIP_OK) + err=ZIP_ERRNO; + + /* version made by */ + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream, &VersionMadeBy)!=ZIP_OK) + err=ZIP_ERRNO; + + /* version needed to extract */ + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream, &VersionNeeded)!=ZIP_OK) + err=ZIP_ERRNO; + + /* number of this disk */ + if (zip64local_getLong(&pziinit->z_filefunc, pziinit->filestream,&number_disk)!=ZIP_OK) + err=ZIP_ERRNO; + + /* number of the disk with the start of the central directory */ + if (zip64local_getLong(&pziinit->z_filefunc, pziinit->filestream,&number_disk_with_CD)!=ZIP_OK) + err=ZIP_ERRNO; + + /* total number of entries in the central directory on this disk */ + if (zip64local_getLong64(&pziinit->z_filefunc, pziinit->filestream, &number_entry)!=ZIP_OK) + err=ZIP_ERRNO; + + /* total number of entries in the central directory */ + if (zip64local_getLong64(&pziinit->z_filefunc, pziinit->filestream,&number_entry_CD)!=ZIP_OK) + err=ZIP_ERRNO; + + if ((number_entry_CD!=number_entry) || (number_disk_with_CD!=0) || (number_disk!=0)) + err=ZIP_BADZIPFILE; + + /* size of the central directory */ + if (zip64local_getLong64(&pziinit->z_filefunc, pziinit->filestream,&size_central_dir)!=ZIP_OK) + err=ZIP_ERRNO; + + /* offset of start of central directory with respect to the + starting disk number */ + if (zip64local_getLong64(&pziinit->z_filefunc, pziinit->filestream,&offset_central_dir)!=ZIP_OK) + err=ZIP_ERRNO; + + // TODO.. + // read the comment from the standard central header. + size_comment = 0; + } + else + { + // Read End of central Directory info + if (ZSEEK64(pziinit->z_filefunc, pziinit->filestream, central_pos,ZLIB_FILEFUNC_SEEK_SET)!=0) + err=ZIP_ERRNO; + + /* the signature, already checked */ + if (zip64local_getLong(&pziinit->z_filefunc, pziinit->filestream,&uL)!=ZIP_OK) + err=ZIP_ERRNO; + + /* number of this disk */ + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream,&number_disk)!=ZIP_OK) + err=ZIP_ERRNO; + + /* number of the disk with the start of the central directory */ + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream,&number_disk_with_CD)!=ZIP_OK) + err=ZIP_ERRNO; + + /* total number of entries in the central dir on this disk */ + number_entry = 0; + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream, &uL)!=ZIP_OK) + err=ZIP_ERRNO; + else + number_entry = uL; + + /* total number of entries in the central dir */ + number_entry_CD = 0; + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream, &uL)!=ZIP_OK) + err=ZIP_ERRNO; + else + number_entry_CD = uL; + + if ((number_entry_CD!=number_entry) || (number_disk_with_CD!=0) || (number_disk!=0)) + err=ZIP_BADZIPFILE; + + /* size of the central directory */ + size_central_dir = 0; + if (zip64local_getLong(&pziinit->z_filefunc, pziinit->filestream, &uL)!=ZIP_OK) + err=ZIP_ERRNO; + else + size_central_dir = uL; + + /* offset of start of central directory with respect to the starting disk number */ + offset_central_dir = 0; + if (zip64local_getLong(&pziinit->z_filefunc, pziinit->filestream, &uL)!=ZIP_OK) + err=ZIP_ERRNO; + else + offset_central_dir = uL; + + + /* zipfile global comment length */ + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream, &size_comment)!=ZIP_OK) + err=ZIP_ERRNO; + } + + if ((central_pos<offset_central_dir+size_central_dir) && + (err==ZIP_OK)) + err=ZIP_BADZIPFILE; + + if (err!=ZIP_OK) + { + ZCLOSE64(pziinit->z_filefunc, pziinit->filestream); + return ZIP_ERRNO; + } + + if (size_comment>0) + { + pziinit->globalcomment = (char*)ALLOC(size_comment+1); + if (pziinit->globalcomment) + { + size_comment = ZREAD64(pziinit->z_filefunc, pziinit->filestream, pziinit->globalcomment,size_comment); + pziinit->globalcomment[size_comment]=0; + } + } + + byte_before_the_zipfile = central_pos - (offset_central_dir+size_central_dir); + pziinit->add_position_when_writting_offset = byte_before_the_zipfile; + + { + ZPOS64_T size_central_dir_to_read = size_central_dir; + size_t buf_size = SIZEDATA_INDATABLOCK; + void* buf_read = (void*)ALLOC(buf_size); + if (ZSEEK64(pziinit->z_filefunc, pziinit->filestream, offset_central_dir + byte_before_the_zipfile, ZLIB_FILEFUNC_SEEK_SET) != 0) + err=ZIP_ERRNO; + + while ((size_central_dir_to_read>0) && (err==ZIP_OK)) + { + ZPOS64_T read_this = SIZEDATA_INDATABLOCK; + if (read_this > size_central_dir_to_read) + read_this = size_central_dir_to_read; + + if (ZREAD64(pziinit->z_filefunc, pziinit->filestream,buf_read,(uLong)read_this) != read_this) + err=ZIP_ERRNO; + + if (err==ZIP_OK) + err = add_data_in_datablock(&pziinit->central_dir,buf_read, (uLong)read_this); + + size_central_dir_to_read-=read_this; + } + TRYFREE(buf_read); + } + pziinit->begin_pos = byte_before_the_zipfile; + pziinit->number_entry = number_entry_CD; + + if (ZSEEK64(pziinit->z_filefunc, pziinit->filestream, offset_central_dir+byte_before_the_zipfile,ZLIB_FILEFUNC_SEEK_SET) != 0) + err=ZIP_ERRNO; + + return err; +} + + +#endif /* !NO_ADDFILEINEXISTINGZIP*/ + + +/************************************************************/ +extern zipFile ZEXPORT zipOpen3 (const void *pathname, int append, zipcharpc* globalcomment, zlib_filefunc64_32_def* pzlib_filefunc64_32_def) +{ + zip64_internal ziinit; + zip64_internal* zi; + int err=ZIP_OK; + + ziinit.z_filefunc.zseek32_file = NULL; + ziinit.z_filefunc.ztell32_file = NULL; + if (pzlib_filefunc64_32_def==NULL) + fill_fopen64_filefunc(&ziinit.z_filefunc.zfile_func64); + else + ziinit.z_filefunc = *pzlib_filefunc64_32_def; + + ziinit.filestream = ZOPEN64(ziinit.z_filefunc, + pathname, + (append == APPEND_STATUS_CREATE) ? + (ZLIB_FILEFUNC_MODE_READ | ZLIB_FILEFUNC_MODE_WRITE | ZLIB_FILEFUNC_MODE_CREATE) : + (ZLIB_FILEFUNC_MODE_READ | ZLIB_FILEFUNC_MODE_WRITE | ZLIB_FILEFUNC_MODE_EXISTING)); + + if (ziinit.filestream == NULL) + return NULL; + + if (append == APPEND_STATUS_CREATEAFTER) + ZSEEK64(ziinit.z_filefunc,ziinit.filestream,0,SEEK_END); + + ziinit.begin_pos = ZTELL64(ziinit.z_filefunc,ziinit.filestream); + ziinit.in_opened_file_inzip = 0; + ziinit.ci.stream_initialised = 0; + ziinit.number_entry = 0; + ziinit.add_position_when_writting_offset = 0; + init_linkedlist(&(ziinit.central_dir)); + + + + zi = (zip64_internal*)ALLOC(sizeof(zip64_internal)); + if (zi==NULL) + { + ZCLOSE64(ziinit.z_filefunc,ziinit.filestream); + return NULL; + } + + /* now we add file in a zipfile */ +# ifndef NO_ADDFILEINEXISTINGZIP + ziinit.globalcomment = NULL; + if (append == APPEND_STATUS_ADDINZIP) + { + // Read and Cache Central Directory Records + err = LoadCentralDirectoryRecord(&ziinit); + } + + if (globalcomment) + { + *globalcomment = ziinit.globalcomment; + } +# endif /* !NO_ADDFILEINEXISTINGZIP*/ + + if (err != ZIP_OK) + { +# ifndef NO_ADDFILEINEXISTINGZIP + TRYFREE(ziinit.globalcomment); +# endif /* !NO_ADDFILEINEXISTINGZIP*/ + TRYFREE(zi); + return NULL; + } + else + { + *zi = ziinit; + return (zipFile)zi; + } +} + +extern zipFile ZEXPORT zipOpen2 (const char *pathname, int append, zipcharpc* globalcomment, zlib_filefunc_def* pzlib_filefunc32_def) +{ + if (pzlib_filefunc32_def != NULL) + { + zlib_filefunc64_32_def zlib_filefunc64_32_def_fill; + fill_zlib_filefunc64_32_def_from_filefunc32(&zlib_filefunc64_32_def_fill,pzlib_filefunc32_def); + return zipOpen3(pathname, append, globalcomment, &zlib_filefunc64_32_def_fill); + } + else + return zipOpen3(pathname, append, globalcomment, NULL); +} + +extern zipFile ZEXPORT zipOpen2_64 (const void *pathname, int append, zipcharpc* globalcomment, zlib_filefunc64_def* pzlib_filefunc_def) +{ + if (pzlib_filefunc_def != NULL) + { + zlib_filefunc64_32_def zlib_filefunc64_32_def_fill; + zlib_filefunc64_32_def_fill.zfile_func64 = *pzlib_filefunc_def; + zlib_filefunc64_32_def_fill.ztell32_file = NULL; + zlib_filefunc64_32_def_fill.zseek32_file = NULL; + return zipOpen3(pathname, append, globalcomment, &zlib_filefunc64_32_def_fill); + } + else + return zipOpen3(pathname, append, globalcomment, NULL); +} + + + +extern zipFile ZEXPORT zipOpen (const char* pathname, int append) +{ + return zipOpen3((const void*)pathname,append,NULL,NULL); +} + +extern zipFile ZEXPORT zipOpen64 (const void* pathname, int append) +{ + return zipOpen3(pathname,append,NULL,NULL); +} + +int Write_LocalFileHeader(zip64_internal* zi, const char* filename, uInt size_extrafield_local, const void* extrafield_local) +{ + /* write the local header */ + int err; + uInt size_filename = (uInt)strlen(filename); + uInt size_extrafield = size_extrafield_local; + + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)LOCALHEADERMAGIC, 4); + + if (err==ZIP_OK) + { + if(zi->ci.zip64) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)45,2);/* version needed to extract */ + else + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)20,2);/* version needed to extract */ + } + + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)zi->ci.flag,2); + + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)zi->ci.method,2); + + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)zi->ci.dosDate,4); + + // CRC / Compressed size / Uncompressed size will be filled in later and rewritten later + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,4); /* crc 32, unknown */ + if (err==ZIP_OK) + { + if(zi->ci.zip64) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0xFFFFFFFF,4); /* compressed size, unknown */ + else + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,4); /* compressed size, unknown */ + } + if (err==ZIP_OK) + { + if(zi->ci.zip64) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0xFFFFFFFF,4); /* uncompressed size, unknown */ + else + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,4); /* uncompressed size, unknown */ + } + + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)size_filename,2); + + if(zi->ci.zip64) + { + size_extrafield += 20; + } + + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)size_extrafield,2); + + if ((err==ZIP_OK) && (size_filename > 0)) + { + if (ZWRITE64(zi->z_filefunc,zi->filestream,filename,size_filename)!=size_filename) + err = ZIP_ERRNO; + } + + if ((err==ZIP_OK) && (size_extrafield_local > 0)) + { + if (ZWRITE64(zi->z_filefunc, zi->filestream, extrafield_local, size_extrafield_local) != size_extrafield_local) + err = ZIP_ERRNO; + } + + + if ((err==ZIP_OK) && (zi->ci.zip64)) + { + // write the Zip64 extended info + short HeaderID = 1; + short DataSize = 16; + ZPOS64_T CompressedSize = 0; + ZPOS64_T UncompressedSize = 0; + + // Remember position of Zip64 extended info for the local file header. (needed when we update size after done with file) + zi->ci.pos_zip64extrainfo = ZTELL64(zi->z_filefunc,zi->filestream); + + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, (short)HeaderID,2); + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, (short)DataSize,2); + + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, (ZPOS64_T)UncompressedSize,8); + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, (ZPOS64_T)CompressedSize,8); + } + + return err; +} + +/* + NOTE. + When writing RAW the ZIP64 extended information in extrafield_local and extrafield_global needs to be stripped + before calling this function it can be done with zipRemoveExtraInfoBlock + + It is not done here because then we need to realloc a new buffer since parameters are 'const' and I want to minimize + unnecessary allocations. + */ +extern int ZEXPORT zipOpenNewFileInZip4_64 (zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw, + int windowBits,int memLevel, int strategy, + const char* password, uLong crcForCrypting, + uLong versionMadeBy, uLong flagBase, int zip64) +{ + zip64_internal* zi; + uInt size_filename; + uInt size_comment; + uInt i; + int err = ZIP_OK; + +# ifdef NOCRYPT + (crcForCrypting); + if (password != NULL) + return ZIP_PARAMERROR; +# endif + + if (file == NULL) + return ZIP_PARAMERROR; + +#ifdef HAVE_BZIP2 + if ((method!=0) && (method!=Z_DEFLATED) && (method!=Z_BZIP2ED)) + return ZIP_PARAMERROR; +#else + if ((method!=0) && (method!=Z_DEFLATED)) + return ZIP_PARAMERROR; +#endif + + zi = (zip64_internal*)file; + + if (zi->in_opened_file_inzip == 1) + { + err = zipCloseFileInZip (file); + if (err != ZIP_OK) + return err; + } + + if (filename==NULL) + filename="-"; + + if (comment==NULL) + size_comment = 0; + else + size_comment = (uInt)strlen(comment); + + size_filename = (uInt)strlen(filename); + + if (zipfi == NULL) + zi->ci.dosDate = 0; + else + { + if (zipfi->dosDate != 0) + zi->ci.dosDate = zipfi->dosDate; + else + zi->ci.dosDate = zip64local_TmzDateToDosDate(&zipfi->tmz_date); + } + + zi->ci.flag = flagBase; + if ((level==8) || (level==9)) + zi->ci.flag |= 2; + if (level==2) + zi->ci.flag |= 4; + if (level==1) + zi->ci.flag |= 6; + if (password != NULL) + zi->ci.flag |= 1; + + zi->ci.crc32 = 0; + zi->ci.method = method; + zi->ci.encrypt = 0; + zi->ci.stream_initialised = 0; + zi->ci.pos_in_buffered_data = 0; + zi->ci.raw = raw; + zi->ci.pos_local_header = ZTELL64(zi->z_filefunc,zi->filestream); + + zi->ci.size_centralheader = SIZECENTRALHEADER + size_filename + size_extrafield_global + size_comment; + zi->ci.size_centralExtraFree = 32; // Extra space we have reserved in case we need to add ZIP64 extra info data + + zi->ci.central_header = (char*)ALLOC((uInt)zi->ci.size_centralheader + zi->ci.size_centralExtraFree); + + zi->ci.size_centralExtra = size_extrafield_global; + zip64local_putValue_inmemory(zi->ci.central_header,(uLong)CENTRALHEADERMAGIC,4); + /* version info */ + zip64local_putValue_inmemory(zi->ci.central_header+4,(uLong)versionMadeBy,2); + zip64local_putValue_inmemory(zi->ci.central_header+6,(uLong)20,2); + zip64local_putValue_inmemory(zi->ci.central_header+8,(uLong)zi->ci.flag,2); + zip64local_putValue_inmemory(zi->ci.central_header+10,(uLong)zi->ci.method,2); + zip64local_putValue_inmemory(zi->ci.central_header+12,(uLong)zi->ci.dosDate,4); + zip64local_putValue_inmemory(zi->ci.central_header+16,(uLong)0,4); /*crc*/ + zip64local_putValue_inmemory(zi->ci.central_header+20,(uLong)0,4); /*compr size*/ + zip64local_putValue_inmemory(zi->ci.central_header+24,(uLong)0,4); /*uncompr size*/ + zip64local_putValue_inmemory(zi->ci.central_header+28,(uLong)size_filename,2); + zip64local_putValue_inmemory(zi->ci.central_header+30,(uLong)size_extrafield_global,2); + zip64local_putValue_inmemory(zi->ci.central_header+32,(uLong)size_comment,2); + zip64local_putValue_inmemory(zi->ci.central_header+34,(uLong)0,2); /*disk nm start*/ + + if (zipfi==NULL) + zip64local_putValue_inmemory(zi->ci.central_header+36,(uLong)0,2); + else + zip64local_putValue_inmemory(zi->ci.central_header+36,(uLong)zipfi->internal_fa,2); + + if (zipfi==NULL) + zip64local_putValue_inmemory(zi->ci.central_header+38,(uLong)0,4); + else + zip64local_putValue_inmemory(zi->ci.central_header+38,(uLong)zipfi->external_fa,4); + + if(zi->ci.pos_local_header >= 0xffffffff) + zip64local_putValue_inmemory(zi->ci.central_header+42,(uLong)0xffffffff,4); + else + zip64local_putValue_inmemory(zi->ci.central_header+42,(uLong)zi->ci.pos_local_header - zi->add_position_when_writting_offset,4); + + for (i=0;i<size_filename;i++) + *(zi->ci.central_header+SIZECENTRALHEADER+i) = *(filename+i); + + for (i=0;i<size_extrafield_global;i++) + *(zi->ci.central_header+SIZECENTRALHEADER+size_filename+i) = + *(((const char*)extrafield_global)+i); + + for (i=0;i<size_comment;i++) + *(zi->ci.central_header+SIZECENTRALHEADER+size_filename+ + size_extrafield_global+i) = *(comment+i); + if (zi->ci.central_header == NULL) + return ZIP_INTERNALERROR; + + zi->ci.zip64 = zip64; + zi->ci.totalCompressedData = 0; + zi->ci.totalUncompressedData = 0; + zi->ci.pos_zip64extrainfo = 0; + + err = Write_LocalFileHeader(zi, filename, size_extrafield_local, extrafield_local); + +#ifdef HAVE_BZIP2 + zi->ci.bstream.avail_in = (uInt)0; + zi->ci.bstream.avail_out = (uInt)Z_BUFSIZE; + zi->ci.bstream.next_out = (char*)zi->ci.buffered_data; + zi->ci.bstream.total_in_hi32 = 0; + zi->ci.bstream.total_in_lo32 = 0; + zi->ci.bstream.total_out_hi32 = 0; + zi->ci.bstream.total_out_lo32 = 0; +#endif + + zi->ci.stream.avail_in = (uInt)0; + zi->ci.stream.avail_out = (uInt)Z_BUFSIZE; + zi->ci.stream.next_out = zi->ci.buffered_data; + zi->ci.stream.total_in = 0; + zi->ci.stream.total_out = 0; + zi->ci.stream.data_type = Z_BINARY; + +#ifdef HAVE_BZIP2 + if ((err==ZIP_OK) && (zi->ci.method == Z_DEFLATED || zi->ci.method == Z_BZIP2ED) && (!zi->ci.raw)) +#else + if ((err==ZIP_OK) && (zi->ci.method == Z_DEFLATED) && (!zi->ci.raw)) +#endif + { + if(zi->ci.method == Z_DEFLATED) + { + zi->ci.stream.zalloc = (alloc_func)0; + zi->ci.stream.zfree = (free_func)0; + zi->ci.stream.opaque = (voidpf)0; + + if (windowBits>0) + windowBits = -windowBits; + + err = deflateInit2(&zi->ci.stream, level, Z_DEFLATED, windowBits, memLevel, strategy); + + if (err==Z_OK) + zi->ci.stream_initialised = Z_DEFLATED; + } + else if(zi->ci.method == Z_BZIP2ED) + { +#ifdef HAVE_BZIP2 + // Init BZip stuff here + zi->ci.bstream.bzalloc = 0; + zi->ci.bstream.bzfree = 0; + zi->ci.bstream.opaque = (voidpf)0; + + err = BZ2_bzCompressInit(&zi->ci.bstream, level, 0,35); + if(err == BZ_OK) + zi->ci.stream_initialised = Z_BZIP2ED; +#endif + } + + } + +# ifndef NOCRYPT + zi->ci.crypt_header_size = 0; + if ((err==Z_OK) && (password != NULL)) + { + unsigned char bufHead[RAND_HEAD_LEN]; + unsigned int sizeHead; + zi->ci.encrypt = 1; + zi->ci.pcrc_32_tab = get_crc_table(); + /*init_keys(password,zi->ci.keys,zi->ci.pcrc_32_tab);*/ + + sizeHead=crypthead(password,bufHead,RAND_HEAD_LEN,zi->ci.keys,zi->ci.pcrc_32_tab,crcForCrypting); + zi->ci.crypt_header_size = sizeHead; + + if (ZWRITE64(zi->z_filefunc,zi->filestream,bufHead,sizeHead) != sizeHead) + err = ZIP_ERRNO; + } +# endif + + if (err==Z_OK) + zi->in_opened_file_inzip = 1; + return err; +} + +extern int ZEXPORT zipOpenNewFileInZip4 (zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw, + int windowBits,int memLevel, int strategy, + const char* password, uLong crcForCrypting, + uLong versionMadeBy, uLong flagBase) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, raw, + windowBits, memLevel, strategy, + password, crcForCrypting, versionMadeBy, flagBase, 0); +} + +extern int ZEXPORT zipOpenNewFileInZip3 (zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw, + int windowBits,int memLevel, int strategy, + const char* password, uLong crcForCrypting) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, raw, + windowBits, memLevel, strategy, + password, crcForCrypting, VERSIONMADEBY, 0, 0); +} + +extern int ZEXPORT zipOpenNewFileInZip3_64(zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw, + int windowBits,int memLevel, int strategy, + const char* password, uLong crcForCrypting, int zip64) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, raw, + windowBits, memLevel, strategy, + password, crcForCrypting, VERSIONMADEBY, 0, zip64); +} + +extern int ZEXPORT zipOpenNewFileInZip2(zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, raw, + -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, + NULL, 0, VERSIONMADEBY, 0, 0); +} + +extern int ZEXPORT zipOpenNewFileInZip2_64(zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw, int zip64) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, raw, + -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, + NULL, 0, VERSIONMADEBY, 0, zip64); +} + +extern int ZEXPORT zipOpenNewFileInZip64 (zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void*extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int zip64) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, 0, + -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, + NULL, 0, VERSIONMADEBY, 0, zip64); +} + +extern int ZEXPORT zipOpenNewFileInZip (zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void*extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, 0, + -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, + NULL, 0, VERSIONMADEBY, 0, 0); +} + +local int zip64FlushWriteBuffer(zip64_internal* zi) +{ + int err=ZIP_OK; + + if (zi->ci.encrypt != 0) + { +#ifndef NOCRYPT + uInt i; + int t; + for (i=0;i<zi->ci.pos_in_buffered_data;i++) + zi->ci.buffered_data[i] = zencode(zi->ci.keys, zi->ci.pcrc_32_tab, zi->ci.buffered_data[i],t); +#endif + } + + if (ZWRITE64(zi->z_filefunc,zi->filestream,zi->ci.buffered_data,zi->ci.pos_in_buffered_data) != zi->ci.pos_in_buffered_data) + err = ZIP_ERRNO; + + zi->ci.totalCompressedData += zi->ci.pos_in_buffered_data; + +#ifdef HAVE_BZIP2 + if(zi->ci.method == Z_BZIP2ED) + { + zi->ci.totalUncompressedData += zi->ci.bstream.total_in_lo32; + zi->ci.bstream.total_in_lo32 = 0; + zi->ci.bstream.total_in_hi32 = 0; + } + else +#endif + { + zi->ci.totalUncompressedData += zi->ci.stream.total_in; + zi->ci.stream.total_in = 0; + } + + + zi->ci.pos_in_buffered_data = 0; + + return err; +} + +extern int ZEXPORT zipWriteInFileInZip (zipFile file,const void* buf,unsigned int len) +{ + zip64_internal* zi; + int err=ZIP_OK; + + if (file == NULL) + return ZIP_PARAMERROR; + zi = (zip64_internal*)file; + + if (zi->in_opened_file_inzip == 0) + return ZIP_PARAMERROR; + + zi->ci.crc32 = crc32(zi->ci.crc32,buf,(uInt)len); + +#ifdef HAVE_BZIP2 + if(zi->ci.method == Z_BZIP2ED && (!zi->ci.raw)) + { + zi->ci.bstream.next_in = (void*)buf; + zi->ci.bstream.avail_in = len; + err = BZ_RUN_OK; + + while ((err==BZ_RUN_OK) && (zi->ci.bstream.avail_in>0)) + { + if (zi->ci.bstream.avail_out == 0) + { + if (zip64FlushWriteBuffer(zi) == ZIP_ERRNO) + err = ZIP_ERRNO; + zi->ci.bstream.avail_out = (uInt)Z_BUFSIZE; + zi->ci.bstream.next_out = (char*)zi->ci.buffered_data; + } + + + if(err != BZ_RUN_OK) + break; + + if ((zi->ci.method == Z_BZIP2ED) && (!zi->ci.raw)) + { + uLong uTotalOutBefore_lo = zi->ci.bstream.total_out_lo32; +// uLong uTotalOutBefore_hi = zi->ci.bstream.total_out_hi32; + err=BZ2_bzCompress(&zi->ci.bstream, BZ_RUN); + + zi->ci.pos_in_buffered_data += (uInt)(zi->ci.bstream.total_out_lo32 - uTotalOutBefore_lo) ; + } + } + + if(err == BZ_RUN_OK) + err = ZIP_OK; + } + else +#endif + { + zi->ci.stream.next_in = (Bytef*)buf; + zi->ci.stream.avail_in = len; + + while ((err==ZIP_OK) && (zi->ci.stream.avail_in>0)) + { + if (zi->ci.stream.avail_out == 0) + { + if (zip64FlushWriteBuffer(zi) == ZIP_ERRNO) + err = ZIP_ERRNO; + zi->ci.stream.avail_out = (uInt)Z_BUFSIZE; + zi->ci.stream.next_out = zi->ci.buffered_data; + } + + + if(err != ZIP_OK) + break; + + if ((zi->ci.method == Z_DEFLATED) && (!zi->ci.raw)) + { + uLong uTotalOutBefore = zi->ci.stream.total_out; + err=deflate(&zi->ci.stream, Z_NO_FLUSH); + if(uTotalOutBefore > zi->ci.stream.total_out) + { + int bBreak = 0; + bBreak++; + } + + zi->ci.pos_in_buffered_data += (uInt)(zi->ci.stream.total_out - uTotalOutBefore) ; + } + else + { + uInt copy_this,i; + if (zi->ci.stream.avail_in < zi->ci.stream.avail_out) + copy_this = zi->ci.stream.avail_in; + else + copy_this = zi->ci.stream.avail_out; + + for (i = 0; i < copy_this; i++) + *(((char*)zi->ci.stream.next_out)+i) = + *(((const char*)zi->ci.stream.next_in)+i); + { + zi->ci.stream.avail_in -= copy_this; + zi->ci.stream.avail_out-= copy_this; + zi->ci.stream.next_in+= copy_this; + zi->ci.stream.next_out+= copy_this; + zi->ci.stream.total_in+= copy_this; + zi->ci.stream.total_out+= copy_this; + zi->ci.pos_in_buffered_data += copy_this; + } + } + }// while(...) + } + + return err; +} + +extern int ZEXPORT zipCloseFileInZipRaw (zipFile file, uLong uncompressed_size, uLong crc32) +{ + return zipCloseFileInZipRaw64 (file, uncompressed_size, crc32); +} + +extern int ZEXPORT zipCloseFileInZipRaw64 (zipFile file, ZPOS64_T uncompressed_size, uLong crc32) +{ + zip64_internal* zi; + ZPOS64_T compressed_size; + uLong invalidValue = 0xffffffff; + short datasize = 0; + int err=ZIP_OK; + + if (file == NULL) + return ZIP_PARAMERROR; + zi = (zip64_internal*)file; + + if (zi->in_opened_file_inzip == 0) + return ZIP_PARAMERROR; + zi->ci.stream.avail_in = 0; + + if ((zi->ci.method == Z_DEFLATED) && (!zi->ci.raw)) + { + while (err==ZIP_OK) + { + uLong uTotalOutBefore; + if (zi->ci.stream.avail_out == 0) + { + if (zip64FlushWriteBuffer(zi) == ZIP_ERRNO) + err = ZIP_ERRNO; + zi->ci.stream.avail_out = (uInt)Z_BUFSIZE; + zi->ci.stream.next_out = zi->ci.buffered_data; + } + uTotalOutBefore = zi->ci.stream.total_out; + err=deflate(&zi->ci.stream, Z_FINISH); + zi->ci.pos_in_buffered_data += (uInt)(zi->ci.stream.total_out - uTotalOutBefore) ; + } + } + else if ((zi->ci.method == Z_BZIP2ED) && (!zi->ci.raw)) + { +#ifdef HAVE_BZIP2 + err = BZ_FINISH_OK; + while (err==BZ_FINISH_OK) + { + uLong uTotalOutBefore; + if (zi->ci.bstream.avail_out == 0) + { + if (zip64FlushWriteBuffer(zi) == ZIP_ERRNO) + err = ZIP_ERRNO; + zi->ci.bstream.avail_out = (uInt)Z_BUFSIZE; + zi->ci.bstream.next_out = (char*)zi->ci.buffered_data; + } + uTotalOutBefore = zi->ci.bstream.total_out_lo32; + err=BZ2_bzCompress(&zi->ci.bstream, BZ_FINISH); + if(err == BZ_STREAM_END) + err = Z_STREAM_END; + + zi->ci.pos_in_buffered_data += (uInt)(zi->ci.bstream.total_out_lo32 - uTotalOutBefore); + } + + if(err == BZ_FINISH_OK) + err = ZIP_OK; +#endif + } + + if (err==Z_STREAM_END) + err=ZIP_OK; /* this is normal */ + + if ((zi->ci.pos_in_buffered_data>0) && (err==ZIP_OK)) + { + if (zip64FlushWriteBuffer(zi)==ZIP_ERRNO) + err = ZIP_ERRNO; + } + + if ((zi->ci.method == Z_DEFLATED) && (!zi->ci.raw)) + { + int tmp_err = deflateEnd(&zi->ci.stream); + if (err == ZIP_OK) + err = tmp_err; + zi->ci.stream_initialised = 0; + } +#ifdef HAVE_BZIP2 + else if((zi->ci.method == Z_BZIP2ED) && (!zi->ci.raw)) + { + int tmperr = BZ2_bzCompressEnd(&zi->ci.bstream); + if (err==ZIP_OK) + err = tmperr; + zi->ci.stream_initialised = 0; + } +#endif + + if (!zi->ci.raw) + { + crc32 = (uLong)zi->ci.crc32; + uncompressed_size = zi->ci.totalUncompressedData; + } + compressed_size = zi->ci.totalCompressedData; + +# ifndef NOCRYPT + compressed_size += zi->ci.crypt_header_size; +# endif + + // update Current Item crc and sizes, + if(compressed_size >= 0xffffffff || uncompressed_size >= 0xffffffff || zi->ci.pos_local_header >= 0xffffffff) + { + /*version Made by*/ + zip64local_putValue_inmemory(zi->ci.central_header+4,(uLong)45,2); + /*version needed*/ + zip64local_putValue_inmemory(zi->ci.central_header+6,(uLong)45,2); + + } + + zip64local_putValue_inmemory(zi->ci.central_header+16,crc32,4); /*crc*/ + + + if(compressed_size >= 0xffffffff) + zip64local_putValue_inmemory(zi->ci.central_header+20, invalidValue,4); /*compr size*/ + else + zip64local_putValue_inmemory(zi->ci.central_header+20, compressed_size,4); /*compr size*/ + + /// set internal file attributes field + if (zi->ci.stream.data_type == Z_ASCII) + zip64local_putValue_inmemory(zi->ci.central_header+36,(uLong)Z_ASCII,2); + + if(uncompressed_size >= 0xffffffff) + zip64local_putValue_inmemory(zi->ci.central_header+24, invalidValue,4); /*uncompr size*/ + else + zip64local_putValue_inmemory(zi->ci.central_header+24, uncompressed_size,4); /*uncompr size*/ + + // Add ZIP64 extra info field for uncompressed size + if(uncompressed_size >= 0xffffffff) + datasize += 8; + + // Add ZIP64 extra info field for compressed size + if(compressed_size >= 0xffffffff) + datasize += 8; + + // Add ZIP64 extra info field for relative offset to local file header of current file + if(zi->ci.pos_local_header >= 0xffffffff) + datasize += 8; + + if(datasize > 0) + { + char* p = NULL; + + if((uLong)(datasize + 4) > zi->ci.size_centralExtraFree) + { + // we can not write more data to the buffer that we have room for. + return ZIP_BADZIPFILE; + } + + p = zi->ci.central_header + zi->ci.size_centralheader; + + // Add Extra Information Header for 'ZIP64 information' + zip64local_putValue_inmemory(p, 0x0001, 2); // HeaderID + p += 2; + zip64local_putValue_inmemory(p, datasize, 2); // DataSize + p += 2; + + if(uncompressed_size >= 0xffffffff) + { + zip64local_putValue_inmemory(p, uncompressed_size, 8); + p += 8; + } + + if(compressed_size >= 0xffffffff) + { + zip64local_putValue_inmemory(p, compressed_size, 8); + p += 8; + } + + if(zi->ci.pos_local_header >= 0xffffffff) + { + zip64local_putValue_inmemory(p, zi->ci.pos_local_header, 8); + p += 8; + } + + // Update how much extra free space we got in the memory buffer + // and increase the centralheader size so the new ZIP64 fields are included + // ( 4 below is the size of HeaderID and DataSize field ) + zi->ci.size_centralExtraFree -= datasize + 4; + zi->ci.size_centralheader += datasize + 4; + + // Update the extra info size field + zi->ci.size_centralExtra += datasize + 4; + zip64local_putValue_inmemory(zi->ci.central_header+30,(uLong)zi->ci.size_centralExtra,2); + } + + if (err==ZIP_OK) + err = add_data_in_datablock(&zi->central_dir, zi->ci.central_header, (uLong)zi->ci.size_centralheader); + + free(zi->ci.central_header); + + if (err==ZIP_OK) + { + // Update the LocalFileHeader with the new values. + + ZPOS64_T cur_pos_inzip = ZTELL64(zi->z_filefunc,zi->filestream); + + if (ZSEEK64(zi->z_filefunc,zi->filestream, zi->ci.pos_local_header + 14,ZLIB_FILEFUNC_SEEK_SET)!=0) + err = ZIP_ERRNO; + + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,crc32,4); /* crc 32, unknown */ + + if(uncompressed_size >= 0xffffffff || compressed_size >= 0xffffffff ) + { + if(zi->ci.pos_zip64extrainfo > 0) + { + // Update the size in the ZIP64 extended field. + if (ZSEEK64(zi->z_filefunc,zi->filestream, zi->ci.pos_zip64extrainfo + 4,ZLIB_FILEFUNC_SEEK_SET)!=0) + err = ZIP_ERRNO; + + if (err==ZIP_OK) /* compressed size, unknown */ + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, uncompressed_size, 8); + + if (err==ZIP_OK) /* uncompressed size, unknown */ + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, compressed_size, 8); + } + else + err = ZIP_BADZIPFILE; // Caller passed zip64 = 0, so no room for zip64 info -> fatal + } + else + { + if (err==ZIP_OK) /* compressed size, unknown */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,compressed_size,4); + + if (err==ZIP_OK) /* uncompressed size, unknown */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,uncompressed_size,4); + } + + if (ZSEEK64(zi->z_filefunc,zi->filestream, cur_pos_inzip,ZLIB_FILEFUNC_SEEK_SET)!=0) + err = ZIP_ERRNO; + } + + zi->number_entry ++; + zi->in_opened_file_inzip = 0; + + return err; +} + +extern int ZEXPORT zipCloseFileInZip (zipFile file) +{ + return zipCloseFileInZipRaw (file,0,0); +} + +int Write_Zip64EndOfCentralDirectoryLocator(zip64_internal* zi, ZPOS64_T zip64eocd_pos_inzip) +{ + int err = ZIP_OK; + ZPOS64_T pos = zip64eocd_pos_inzip - zi->add_position_when_writting_offset; + + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)ZIP64ENDLOCHEADERMAGIC,4); + + /*num disks*/ + if (err==ZIP_OK) /* number of the disk with the start of the central directory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,4); + + /*relative offset*/ + if (err==ZIP_OK) /* Relative offset to the Zip64EndOfCentralDirectory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream, pos,8); + + /*total disks*/ /* Do not support spawning of disk so always say 1 here*/ + if (err==ZIP_OK) /* number of the disk with the start of the central directory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)1,4); + + return err; +} + +int Write_Zip64EndOfCentralDirectoryRecord(zip64_internal* zi, uLong size_centraldir, ZPOS64_T centraldir_pos_inzip) +{ + int err = ZIP_OK; + + uLong Zip64DataSize = 44; + + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)ZIP64ENDHEADERMAGIC,4); + + if (err==ZIP_OK) /* size of this 'zip64 end of central directory' */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(ZPOS64_T)Zip64DataSize,8); // why ZPOS64_T of this ? + + if (err==ZIP_OK) /* version made by */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)45,2); + + if (err==ZIP_OK) /* version needed */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)45,2); + + if (err==ZIP_OK) /* number of this disk */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,4); + + if (err==ZIP_OK) /* number of the disk with the start of the central directory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,4); + + if (err==ZIP_OK) /* total number of entries in the central dir on this disk */ + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, zi->number_entry, 8); + + if (err==ZIP_OK) /* total number of entries in the central dir */ + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, zi->number_entry, 8); + + if (err==ZIP_OK) /* size of the central directory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(ZPOS64_T)size_centraldir,8); + + if (err==ZIP_OK) /* offset of start of central directory with respect to the starting disk number */ + { + ZPOS64_T pos = centraldir_pos_inzip - zi->add_position_when_writting_offset; + err = zip64local_putValue(&zi->z_filefunc,zi->filestream, (ZPOS64_T)pos,8); + } + return err; +} +int Write_EndOfCentralDirectoryRecord(zip64_internal* zi, uLong size_centraldir, ZPOS64_T centraldir_pos_inzip) +{ + int err = ZIP_OK; + + /*signature*/ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)ENDHEADERMAGIC,4); + + if (err==ZIP_OK) /* number of this disk */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,2); + + if (err==ZIP_OK) /* number of the disk with the start of the central directory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,2); + + if (err==ZIP_OK) /* total number of entries in the central dir on this disk */ + { + { + if(zi->number_entry >= 0xFFFF) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0xffff,2); // use value in ZIP64 record + else + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)zi->number_entry,2); + } + } + + if (err==ZIP_OK) /* total number of entries in the central dir */ + { + if(zi->number_entry >= 0xFFFF) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0xffff,2); // use value in ZIP64 record + else + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)zi->number_entry,2); + } + + if (err==ZIP_OK) /* size of the central directory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)size_centraldir,4); + + if (err==ZIP_OK) /* offset of start of central directory with respect to the starting disk number */ + { + ZPOS64_T pos = centraldir_pos_inzip - zi->add_position_when_writting_offset; + if(pos >= 0xffffffff) + { + err = zip64local_putValue(&zi->z_filefunc,zi->filestream, (uLong)0xffffffff,4); + } + else + err = zip64local_putValue(&zi->z_filefunc,zi->filestream, (uLong)(centraldir_pos_inzip - zi->add_position_when_writting_offset),4); + } + + return err; +} + +int Write_GlobalComment(zip64_internal* zi, const char* global_comment) +{ + int err = ZIP_OK; + uInt size_global_comment = 0; + + if(global_comment != NULL) + size_global_comment = (uInt)strlen(global_comment); + + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)size_global_comment,2); + + if (err == ZIP_OK && size_global_comment > 0) + { + if (ZWRITE64(zi->z_filefunc,zi->filestream, global_comment, size_global_comment) != size_global_comment) + err = ZIP_ERRNO; + } + return err; +} + +extern int ZEXPORT zipClose (zipFile file, const char* global_comment) +{ + zip64_internal* zi; + int err = 0; + uLong size_centraldir = 0; + ZPOS64_T centraldir_pos_inzip; + ZPOS64_T pos; + + if (file == NULL) + return ZIP_PARAMERROR; + + zi = (zip64_internal*)file; + + if (zi->in_opened_file_inzip == 1) + { + err = zipCloseFileInZip (file); + } + +#ifndef NO_ADDFILEINEXISTINGZIP + if (global_comment==NULL) + global_comment = zi->globalcomment; +#endif + + centraldir_pos_inzip = ZTELL64(zi->z_filefunc,zi->filestream); + + if (err==ZIP_OK) + { + linkedlist_datablock_internal* ldi = zi->central_dir.first_block; + while (ldi!=NULL) + { + if ((err==ZIP_OK) && (ldi->filled_in_this_block>0)) + { + if (ZWRITE64(zi->z_filefunc,zi->filestream, ldi->data, ldi->filled_in_this_block) != ldi->filled_in_this_block) + err = ZIP_ERRNO; + } + + size_centraldir += ldi->filled_in_this_block; + ldi = ldi->next_datablock; + } + } + free_linkedlist(&(zi->central_dir)); + + pos = centraldir_pos_inzip - zi->add_position_when_writting_offset; + if(pos >= 0xffffffff || zi->number_entry > 0xFFFF) + { + ZPOS64_T Zip64EOCDpos = ZTELL64(zi->z_filefunc,zi->filestream); + Write_Zip64EndOfCentralDirectoryRecord(zi, size_centraldir, centraldir_pos_inzip); + + Write_Zip64EndOfCentralDirectoryLocator(zi, Zip64EOCDpos); + } + + if (err==ZIP_OK) + err = Write_EndOfCentralDirectoryRecord(zi, size_centraldir, centraldir_pos_inzip); + + if(err == ZIP_OK) + err = Write_GlobalComment(zi, global_comment); + + if (ZCLOSE64(zi->z_filefunc,zi->filestream) != 0) + if (err == ZIP_OK) + err = ZIP_ERRNO; + +#ifndef NO_ADDFILEINEXISTINGZIP + TRYFREE(zi->globalcomment); +#endif + TRYFREE(zi); + + return err; +} + +extern int ZEXPORT zipRemoveExtraInfoBlock (char* pData, int* dataLen, short sHeader) +{ + char* p = pData; + int size = 0; + char* pNewHeader; + char* pTmp; + short header; + short dataSize; + + int retVal = ZIP_OK; + + if(pData == NULL || *dataLen < 4) + return ZIP_PARAMERROR; + + pNewHeader = (char*)ALLOC(*dataLen); + pTmp = pNewHeader; + + while(p < (pData + *dataLen)) + { + header = *(short*)p; + dataSize = *(((short*)p)+1); + + if( header == sHeader ) // Header found. + { + p += dataSize + 4; // skip it. do not copy to temp buffer + } + else + { + // Extra Info block should not be removed, So copy it to the temp buffer. + memcpy(pTmp, p, dataSize + 4); + p += dataSize + 4; + size += dataSize + 4; + } + + } + + if(size < *dataLen) + { + // clean old extra info block. + memset(pData,0, *dataLen); + + // copy the new extra info block over the old + if(size > 0) + memcpy(pData, pNewHeader, size); + + // set the new extra info size + *dataLen = size; + + retVal = ZIP_OK; + } + else + retVal = ZIP_ERRNO; + + TRYFREE(pNewHeader); + + return retVal; +} diff --git a/storage/connect/zip.h b/storage/connect/zip.h new file mode 100644 index 00000000000..8aaebb62343 --- /dev/null +++ b/storage/connect/zip.h @@ -0,0 +1,362 @@ +/* zip.h -- IO on .zip files using zlib + Version 1.1, February 14h, 2010 + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications for Zip64 support + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + + --------------------------------------------------------------------------- + + Condition of use and distribution are the same than zlib : + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + --------------------------------------------------------------------------- + + Changes + + See header of zip.h + +*/ + +#ifndef _zip12_H +#define _zip12_H + +#ifdef __cplusplus +extern "C" { +#endif + +//#define HAVE_BZIP2 + +#ifndef _ZLIB_H +#include "zlib.h" +#endif + +#ifndef _ZLIBIOAPI_H +#include "ioapi.h" +#endif + +#ifdef HAVE_BZIP2 +#include "bzlib.h" +#endif + +#define Z_BZIP2ED 12 + +#if defined(STRICTZIP) || defined(STRICTZIPUNZIP) +/* like the STRICT of WIN32, we define a pointer that cannot be converted + from (void*) without cast */ +typedef struct TagzipFile__ { int unused; } zipFile__; +typedef zipFile__ *zipFile; +#else +typedef voidp zipFile; +#endif + +#define ZIP_OK (0) +#define ZIP_EOF (0) +#define ZIP_ERRNO (Z_ERRNO) +#define ZIP_PARAMERROR (-102) +#define ZIP_BADZIPFILE (-103) +#define ZIP_INTERNALERROR (-104) + +#ifndef DEF_MEM_LEVEL +# if MAX_MEM_LEVEL >= 8 +# define DEF_MEM_LEVEL 8 +# else +# define DEF_MEM_LEVEL MAX_MEM_LEVEL +# endif +#endif +/* default memLevel */ + +/* tm_zip contain date/time info */ +typedef struct tm_zip_s +{ + uInt tm_sec; /* seconds after the minute - [0,59] */ + uInt tm_min; /* minutes after the hour - [0,59] */ + uInt tm_hour; /* hours since midnight - [0,23] */ + uInt tm_mday; /* day of the month - [1,31] */ + uInt tm_mon; /* months since January - [0,11] */ + uInt tm_year; /* years - [1980..2044] */ +} tm_zip; + +typedef struct +{ + tm_zip tmz_date; /* date in understandable format */ + uLong dosDate; /* if dos_date == 0, tmu_date is used */ +/* uLong flag; */ /* general purpose bit flag 2 bytes */ + + uLong internal_fa; /* internal file attributes 2 bytes */ + uLong external_fa; /* external file attributes 4 bytes */ +} zip_fileinfo; + +typedef const char* zipcharpc; + + +#define APPEND_STATUS_CREATE (0) +#define APPEND_STATUS_CREATEAFTER (1) +#define APPEND_STATUS_ADDINZIP (2) + +extern zipFile ZEXPORT zipOpen OF((const char *pathname, int append)); +extern zipFile ZEXPORT zipOpen64 OF((const void *pathname, int append)); +/* + Create a zipfile. + pathname contain on Windows XP a filename like "c:\\zlib\\zlib113.zip" or on + an Unix computer "zlib/zlib113.zip". + if the file pathname exist and append==APPEND_STATUS_CREATEAFTER, the zip + will be created at the end of the file. + (useful if the file contain a self extractor code) + if the file pathname exist and append==APPEND_STATUS_ADDINZIP, we will + add files in existing zip (be sure you don't add file that doesn't exist) + If the zipfile cannot be opened, the return value is NULL. + Else, the return value is a zipFile Handle, usable with other function + of this zip package. +*/ + +/* Note : there is no delete function into a zipfile. + If you want delete file into a zipfile, you must open a zipfile, and create another + Of couse, you can use RAW reading and writing to copy the file you did not want delte +*/ + +extern zipFile ZEXPORT zipOpen2 OF((const char *pathname, + int append, + zipcharpc* globalcomment, + zlib_filefunc_def* pzlib_filefunc_def)); + +extern zipFile ZEXPORT zipOpen2_64 OF((const void *pathname, + int append, + zipcharpc* globalcomment, + zlib_filefunc64_def* pzlib_filefunc_def)); + +extern int ZEXPORT zipOpenNewFileInZip OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level)); + +extern int ZEXPORT zipOpenNewFileInZip64 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int zip64)); + +/* + Open a file in the ZIP for writing. + filename : the filename in zip (if NULL, '-' without quote will be used + *zipfi contain supplemental information + if extrafield_local!=NULL and size_extrafield_local>0, extrafield_local + contains the extrafield data the the local header + if extrafield_global!=NULL and size_extrafield_global>0, extrafield_global + contains the extrafield data the the local header + if comment != NULL, comment contain the comment string + method contain the compression method (0 for store, Z_DEFLATED for deflate) + level contain the level of compression (can be Z_DEFAULT_COMPRESSION) + zip64 is set to 1 if a zip64 extended information block should be added to the local file header. + this MUST be '1' if the uncompressed size is >= 0xffffffff. + +*/ + + +extern int ZEXPORT zipOpenNewFileInZip2 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw)); + + +extern int ZEXPORT zipOpenNewFileInZip2_64 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw, + int zip64)); +/* + Same than zipOpenNewFileInZip, except if raw=1, we write raw file + */ + +extern int ZEXPORT zipOpenNewFileInZip3 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw, + int windowBits, + int memLevel, + int strategy, + const char* password, + uLong crcForCrypting)); + +extern int ZEXPORT zipOpenNewFileInZip3_64 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw, + int windowBits, + int memLevel, + int strategy, + const char* password, + uLong crcForCrypting, + int zip64 + )); + +/* + Same than zipOpenNewFileInZip2, except + windowBits,memLevel,,strategy : see parameter strategy in deflateInit2 + password : crypting password (NULL for no crypting) + crcForCrypting : crc of file to compress (needed for crypting) + */ + +extern int ZEXPORT zipOpenNewFileInZip4 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw, + int windowBits, + int memLevel, + int strategy, + const char* password, + uLong crcForCrypting, + uLong versionMadeBy, + uLong flagBase + )); + + +extern int ZEXPORT zipOpenNewFileInZip4_64 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw, + int windowBits, + int memLevel, + int strategy, + const char* password, + uLong crcForCrypting, + uLong versionMadeBy, + uLong flagBase, + int zip64 + )); +/* + Same than zipOpenNewFileInZip4, except + versionMadeBy : value for Version made by field + flag : value for flag field (compression level info will be added) + */ + + +extern int ZEXPORT zipWriteInFileInZip OF((zipFile file, + const void* buf, + unsigned len)); +/* + Write data in the zipfile +*/ + +extern int ZEXPORT zipCloseFileInZip OF((zipFile file)); +/* + Close the current file in the zipfile +*/ + +extern int ZEXPORT zipCloseFileInZipRaw OF((zipFile file, + uLong uncompressed_size, + uLong crc32)); + +extern int ZEXPORT zipCloseFileInZipRaw64 OF((zipFile file, + ZPOS64_T uncompressed_size, + uLong crc32)); + +/* + Close the current file in the zipfile, for file opened with + parameter raw=1 in zipOpenNewFileInZip2 + uncompressed_size and crc32 are value for the uncompressed size +*/ + +extern int ZEXPORT zipClose OF((zipFile file, + const char* global_comment)); +/* + Close the zipfile +*/ + + +extern int ZEXPORT zipRemoveExtraInfoBlock OF((char* pData, int* dataLen, short sHeader)); +/* + zipRemoveExtraInfoBlock - Added by Mathias Svensson + + Remove extra information block from a extra information data for the local file header or central directory header + + It is needed to remove ZIP64 extra information blocks when before data is written if using RAW mode. + + 0x0001 is the signature header for the ZIP64 extra information blocks + + usage. + Remove ZIP64 Extra information from a central director extra field data + zipRemoveExtraInfoBlock(pCenDirExtraFieldData, &nCenDirExtraFieldDataLen, 0x0001); + + Remove ZIP64 Extra information from a Local File Header extra field data + zipRemoveExtraInfoBlock(pLocalHeaderExtraFieldData, &nLocalHeaderExtraFieldDataLen, 0x0001); +*/ + +#ifdef __cplusplus +} +#endif + +#endif /* _zip64_H */ diff --git a/storage/innobase/dict/dict0stats.cc b/storage/innobase/dict/dict0stats.cc index c13d4583fef..6a28f3cdf8f 100644 --- a/storage/innobase/dict/dict0stats.cc +++ b/storage/innobase/dict/dict0stats.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 2009, 2015, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2009, 2016, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1161,7 +1161,8 @@ dict_stats_analyze_index_level( them away) which brings non-determinism. We skip only leaf-level delete marks because delete marks on non-leaf level do not make sense. */ - if (level == 0 && + + if (level == 0 && srv_stats_include_delete_marked? 0: rec_get_deleted_flag( rec, page_is_comp(btr_pcur_get_page(&pcur)))) { @@ -1347,8 +1348,12 @@ enum page_scan_method_t { the given page and count the number of distinct ones, also ignore delete marked records */ - QUIT_ON_FIRST_NON_BORING/* quit when the first record that differs + QUIT_ON_FIRST_NON_BORING,/* quit when the first record that differs from its right neighbor is found */ + COUNT_ALL_NON_BORING_INCLUDE_DEL_MARKED/* scan all records on + the given page and count the number of + distinct ones, include delete marked + records */ }; /* @} */ @@ -1624,6 +1629,8 @@ dict_stats_analyze_index_below_cur( offsets_rec = dict_stats_scan_page( &rec, offsets1, offsets2, index, page, n_prefix, + srv_stats_include_delete_marked ? + COUNT_ALL_NON_BORING_INCLUDE_DEL_MARKED: COUNT_ALL_NON_BORING_AND_SKIP_DEL_MARKED, n_diff, n_external_pages); diff --git a/storage/innobase/fts/fts0opt.cc b/storage/innobase/fts/fts0opt.cc index ed882d33548..cb30122adcb 100644 --- a/storage/innobase/fts/fts0opt.cc +++ b/storage/innobase/fts/fts0opt.cc @@ -579,9 +579,6 @@ fts_zip_read_word( fts_zip_t* zip, /*!< in: Zip state + data */ fts_string_t* word) /*!< out: uncompressed word */ { -#ifdef UNIV_DEBUG - ulint i; -#endif short len = 0; void* null = NULL; byte* ptr = word->f_str; @@ -656,10 +653,9 @@ fts_zip_read_word( } } -#ifdef UNIV_DEBUG /* All blocks must be freed at end of inflate. */ if (zip->status != Z_OK) { - for (i = 0; i < ib_vector_size(zip->blocks); ++i) { + for (ulint i = 0; i < ib_vector_size(zip->blocks); ++i) { if (ib_vector_getp(zip->blocks, i)) { ut_free(ib_vector_getp(zip->blocks, i)); ib_vector_set(zip->blocks, i, &null); @@ -670,7 +666,6 @@ fts_zip_read_word( if (ptr != NULL) { ut_ad(word->f_len == strlen((char*) ptr)); } -#endif /* UNIV_DEBUG */ return(zip->status == Z_OK || zip->status == Z_STREAM_END ? ptr : NULL); } diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 227018fd3b6..297ca0d162d 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -15835,6 +15835,37 @@ ha_innobase::get_auto_increment( ulonglong col_max_value = innobase_get_int_col_max_value( table->next_number_field); + /** The following logic is needed to avoid duplicate key error + for autoincrement column. + + (1) InnoDB gives the current autoincrement value with respect + to increment and offset value. + + (2) Basically it does compute_next_insert_id() logic inside InnoDB + to avoid the current auto increment value changed by handler layer. + + (3) It is restricted only for insert operations. */ + + if (increment > 1 && thd_sql_command(user_thd) != SQLCOM_ALTER_TABLE + && autoinc < col_max_value) { + + ulonglong prev_auto_inc = autoinc; + + autoinc = ((autoinc - 1) + increment - offset)/ increment; + + autoinc = autoinc * increment + offset; + + /* If autoinc exceeds the col_max_value then reset + to old autoinc value. Because in case of non-strict + sql mode, boundary value is not considered as error. */ + + if (autoinc >= col_max_value) { + autoinc = prev_auto_inc; + } + + ut_ad(autoinc > 0); + } + /* Called for the first time ? */ if (trx->n_autoinc_rows == 0) { @@ -18784,6 +18815,12 @@ static MYSQL_SYSVAR_BOOL(use_fallocate, innobase_use_fallocate, "Preallocate files fast, using operating system functionality. On POSIX systems, posix_fallocate system call is used.", NULL, NULL, FALSE); +static MYSQL_SYSVAR_BOOL(stats_include_delete_marked, + srv_stats_include_delete_marked, + PLUGIN_VAR_OPCMDARG, + "Scan delete marked records for persistent stat", + NULL, NULL, FALSE); + static MYSQL_SYSVAR_ULONG(io_capacity, srv_io_capacity, PLUGIN_VAR_RQCMDARG, "Number of IOPs the server can do. Tunes the background IO rate", @@ -19910,6 +19947,7 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(doublewrite), MYSQL_SYSVAR(use_atomic_writes), MYSQL_SYSVAR(use_fallocate), + MYSQL_SYSVAR(stats_include_delete_marked), MYSQL_SYSVAR(api_enable_binlog), MYSQL_SYSVAR(api_enable_mdl), MYSQL_SYSVAR(api_disable_rowlock), diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 345dcd0ef2c..b12e5223a34 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -1864,6 +1864,7 @@ innobase_fts_check_doc_id_index_in_def( return(FTS_NOT_EXIST_DOC_ID_INDEX); } + /*******************************************************************//** Create an index table where indexes are ordered as follows: @@ -1932,26 +1933,11 @@ innobase_create_key_defs( (only prefix/part of the column is indexed), MySQL will treat the index as a PRIMARY KEY unless the table already has one. */ - if (n_add > 0 && !new_primary && got_default_clust - && (key_info[*add].flags & HA_NOSAME) - && !(key_info[*add].flags & HA_KEY_HAS_PART_KEY_SEG)) { - uint key_part = key_info[*add].user_defined_key_parts; - - new_primary = true; + ut_ad(altered_table->s->primary_key == 0 + || altered_table->s->primary_key == MAX_KEY); - while (key_part--) { - const uint maybe_null - = key_info[*add].key_part[key_part].key_type - & FIELDFLAG_MAYBE_NULL; - DBUG_ASSERT(!maybe_null - == !key_info[*add].key_part[key_part]. - field->real_maybe_null()); - - if (maybe_null) { - new_primary = false; - break; - } - } + if (got_default_clust && !new_primary) { + new_primary = (altered_table->s->primary_key != MAX_KEY); } const bool rebuild = new_primary || add_fts_doc_id @@ -1970,8 +1956,14 @@ innobase_create_key_defs( ulint primary_key_number; if (new_primary) { - DBUG_ASSERT(n_add > 0); - primary_key_number = *add; + if (n_add == 0) { + DBUG_ASSERT(got_default_clust); + DBUG_ASSERT(altered_table->s->primary_key + == 0); + primary_key_number = 0; + } else { + primary_key_number = *add; + } } else if (got_default_clust) { /* Create the GEN_CLUST_INDEX */ index_def_t* index = indexdef++; @@ -3089,6 +3081,8 @@ prepare_inplace_alter_table_dict( ctx->add_cols = add_cols; } else { DBUG_ASSERT(!innobase_need_rebuild(ha_alter_info, old_table)); + DBUG_ASSERT(old_table->s->primary_key + == altered_table->s->primary_key); if (!ctx->new_table->fts && innobase_fulltext_exist(altered_table)) { @@ -4125,6 +4119,27 @@ found_col: add_fts_doc_id_idx)); } +/** Get the name of an erroneous key. +@param[in] error_key_num InnoDB number of the erroneus key +@param[in] ha_alter_info changes that were being performed +@param[in] table InnoDB table +@return the name of the erroneous key */ +static +const char* +get_error_key_name( + ulint error_key_num, + const Alter_inplace_info* ha_alter_info, + const dict_table_t* table) +{ + if (error_key_num == ULINT_UNDEFINED) { + return(FTS_DOC_ID_INDEX_NAME); + } else if (ha_alter_info->key_count == 0) { + return(dict_table_get_first_index(table)->name); + } else { + return(ha_alter_info->key_info_buffer[error_key_num].name); + } +} + /** Alter the table structure in-place with operations specified using Alter_inplace_info. The level of concurrency allowed during this operation depends @@ -4247,17 +4262,13 @@ oom: case DB_ONLINE_LOG_TOO_BIG: DBUG_ASSERT(ctx->online); my_error(ER_INNODB_ONLINE_LOG_TOO_BIG, MYF(0), - (prebuilt->trx->error_key_num == ULINT_UNDEFINED) - ? FTS_DOC_ID_INDEX_NAME - : ha_alter_info->key_info_buffer[ - prebuilt->trx->error_key_num].name); + get_error_key_name(prebuilt->trx->error_key_num, + ha_alter_info, prebuilt->table)); break; case DB_INDEX_CORRUPT: my_error(ER_INDEX_CORRUPT, MYF(0), - (prebuilt->trx->error_key_num == ULINT_UNDEFINED) - ? FTS_DOC_ID_INDEX_NAME - : ha_alter_info->key_info_buffer[ - prebuilt->trx->error_key_num].name); + get_error_key_name(prebuilt->trx->error_key_num, + ha_alter_info, prebuilt->table)); break; case DB_DECRYPTION_FAILED: { String str; @@ -5077,7 +5088,6 @@ innobase_update_foreign_cache( "Foreign key constraints for table '%s'" " are loaded with charset check off", user_table->name); - } } @@ -5177,14 +5187,13 @@ commit_try_rebuild( DBUG_RETURN(true); case DB_ONLINE_LOG_TOO_BIG: my_error(ER_INNODB_ONLINE_LOG_TOO_BIG, MYF(0), - ha_alter_info->key_info_buffer[0].name); + get_error_key_name(err_key, ha_alter_info, + rebuilt_table)); DBUG_RETURN(true); case DB_INDEX_CORRUPT: my_error(ER_INDEX_CORRUPT, MYF(0), - (err_key == ULINT_UNDEFINED) - ? FTS_DOC_ID_INDEX_NAME - : ha_alter_info->key_info_buffer[err_key] - .name); + get_error_key_name(err_key, ha_alter_info, + rebuilt_table)); DBUG_RETURN(true); default: my_error_innodb(error, table_name, user_table->flags); diff --git a/storage/innobase/include/os0thread.h b/storage/innobase/include/os0thread.h index 9a1ada8fa0d..54f3d7554bf 100644 --- a/storage/innobase/include/os0thread.h +++ b/storage/innobase/include/os0thread.h @@ -117,14 +117,25 @@ os_thread_create_func( os_thread_id_t* thread_id); /*!< out: id of the created thread, or NULL */ +/** Waits until the specified thread completes and joins it. +Its return value is ignored. +@param[in,out] thread thread to join */ +UNIV_INTERN +void +os_thread_join( + os_thread_t thread); + /*****************************************************************//** Exits the current thread. */ UNIV_INTERN void os_thread_exit( /*===========*/ - void* exit_value) /*!< in: exit value; in Windows this void* + void* exit_value, /*!< in: exit value; in Windows this void* is cast as a DWORD */ + bool detach = true) /*!< in: if true, the thread will be detached + right before exiting. If false, another thread + is responsible for joining this thread. */ UNIV_COLD MY_ATTRIBUTE((noreturn)); /*****************************************************************//** Returns the thread identifier of current thread. diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index f404668390f..9dd5b3efa9a 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -458,6 +458,7 @@ extern unsigned long long srv_stats_persistent_sample_pages; extern my_bool srv_stats_auto_recalc; extern unsigned long long srv_stats_modified_counter; extern my_bool srv_stats_sample_traditional; +extern my_bool srv_stats_include_delete_marked; extern ibool srv_use_doublewrite_buf; extern ulong srv_doublewrite_batch_size; diff --git a/storage/innobase/include/univ.i b/storage/innobase/include/univ.i index 53be1cf1ffe..6fe7950db0f 100644 --- a/storage/innobase/include/univ.i +++ b/storage/innobase/include/univ.i @@ -45,7 +45,7 @@ Created 1/20/1994 Heikki Tuuri #define INNODB_VERSION_MAJOR 5 #define INNODB_VERSION_MINOR 6 -#define INNODB_VERSION_BUGFIX 33 +#define INNODB_VERSION_BUGFIX 35 /* The following is the InnoDB version as shown in SELECT plugin_version FROM information_schema.plugins; diff --git a/storage/innobase/mach/mach0data.cc b/storage/innobase/mach/mach0data.cc index df68aab8a18..feeedb01609 100644 --- a/storage/innobase/mach/mach0data.cc +++ b/storage/innobase/mach/mach0data.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1995, 2009, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -55,8 +55,22 @@ mach_parse_compressed( if (flag < 0x80UL) { *val = flag; return(ptr + 1); + } + + /* Workaround GCC bug + https://gcc.gnu.org/bugzilla/show_bug.cgi?id=77673: + the compiler moves mach_read_from_4 right to the beginning of the + function, causing and out-of-bounds read if we are reading a short + integer close to the end of buffer. */ +#if defined(__GNUC__) && (__GNUC__ >= 5) && !defined(__clang__) +#define DEPLOY_FENCE +#endif + +#ifdef DEPLOY_FENCE + __atomic_thread_fence(__ATOMIC_ACQUIRE); +#endif - } else if (flag < 0xC0UL) { + if (flag < 0xC0UL) { if (end_ptr < ptr + 2) { return(NULL); } @@ -64,8 +78,13 @@ mach_parse_compressed( *val = mach_read_from_2(ptr) & 0x7FFFUL; return(ptr + 2); + } + +#ifdef DEPLOY_FENCE + __atomic_thread_fence(__ATOMIC_ACQUIRE); +#endif - } else if (flag < 0xE0UL) { + if (flag < 0xE0UL) { if (end_ptr < ptr + 3) { return(NULL); } @@ -73,7 +92,13 @@ mach_parse_compressed( *val = mach_read_from_3(ptr) & 0x3FFFFFUL; return(ptr + 3); - } else if (flag < 0xF0UL) { + } + +#ifdef DEPLOY_FENCE + __atomic_thread_fence(__ATOMIC_ACQUIRE); +#endif + + if (flag < 0xF0UL) { if (end_ptr < ptr + 4) { return(NULL); } @@ -81,14 +106,20 @@ mach_parse_compressed( *val = mach_read_from_4(ptr) & 0x1FFFFFFFUL; return(ptr + 4); - } else { - ut_ad(flag == 0xF0UL); + } - if (end_ptr < ptr + 5) { - return(NULL); - } +#ifdef DEPLOY_FENCE + __atomic_thread_fence(__ATOMIC_ACQUIRE); +#endif + +#undef DEPLOY_FENCE + + ut_ad(flag == 0xF0UL); - *val = mach_read_from_4(ptr + 1); - return(ptr + 5); + if (end_ptr < ptr + 5) { + return(NULL); } + + *val = mach_read_from_4(ptr + 1); + return(ptr + 5); } diff --git a/storage/innobase/os/os0thread.cc b/storage/innobase/os/os0thread.cc index 88f4292d6fe..205bef02079 100644 --- a/storage/innobase/os/os0thread.cc +++ b/storage/innobase/os/os0thread.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1995, 2013, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -187,14 +187,38 @@ os_thread_create_func( #endif } +/** Waits until the specified thread completes and joins it. +Its return value is ignored. +@param[in,out] thread thread to join */ +UNIV_INTERN +void +os_thread_join( + os_thread_t thread) +{ +#ifdef __WIN__ + /* Do nothing. */ +#else +#ifdef UNIV_DEBUG + const int ret = +#endif /* UNIV_DEBUG */ + pthread_join(thread, NULL); + + /* Waiting on already-quit threads is allowed. */ + ut_ad(ret == 0 || ret == ESRCH); +#endif /* __WIN__ */ +} + /*****************************************************************//** Exits the current thread. */ UNIV_INTERN void os_thread_exit( /*===========*/ - void* exit_value) /*!< in: exit value; in Windows this void* + void* exit_value, /*!< in: exit value; in Windows this void* is cast as a DWORD */ + bool detach) /*!< in: if true, the thread will be detached + right before exiting. If false, another thread + is responsible for joining this thread. */ { #ifdef UNIV_DEBUG_THREAD_CREATION fprintf(stderr, "Thread exits, id %lu\n", @@ -212,7 +236,9 @@ os_thread_exit( #ifdef __WIN__ ExitThread((DWORD) exit_value); #else - pthread_detach(pthread_self()); + if (detach) { + pthread_detach(pthread_self()); + } pthread_exit(exit_value); #endif } diff --git a/storage/innobase/row/row0ftsort.cc b/storage/innobase/row/row0ftsort.cc index cc676af4ef3..83a6bd5a2e7 100644 --- a/storage/innobase/row/row0ftsort.cc +++ b/storage/innobase/row/row0ftsort.cc @@ -1017,7 +1017,7 @@ fts_parallel_merge( CloseHandle(psort_info->thread_hdl); #endif /*__WIN__ */ - os_thread_exit(NULL); + os_thread_exit(NULL, false); OS_THREAD_DUMMY_RETURN; } diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index 4a2a8116ec7..926e8a44152 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -4165,6 +4165,13 @@ wait_again: " exited when creating FTS" " index '%s'", indexes[i]->name); + } else { + for (j = 0; j < FTS_NUM_AUX_INDEX; + j++) { + + os_thread_join(merge_info[j] + .thread_hdl); + } } } else { /* This cannot report duplicates; an diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 1c02590933f..4e109391374 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -1373,6 +1373,8 @@ run_again: row_ins_step(thr); + DEBUG_SYNC_C("ib_after_row_insert_step"); + err = trx->error_state; if (err != DB_SUCCESS) { diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index bf2a0a7fa36..147279dd0d1 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -378,6 +378,7 @@ this many index pages, there are 2 ways to calculate statistics: table/index are not found in the innodb database */ UNIV_INTERN unsigned long long srv_stats_transient_sample_pages = 8; UNIV_INTERN my_bool srv_stats_persistent = TRUE; +UNIV_INTERN my_bool srv_stats_include_delete_marked = FALSE; UNIV_INTERN unsigned long long srv_stats_persistent_sample_pages = 20; UNIV_INTERN my_bool srv_stats_auto_recalc = TRUE; diff --git a/storage/perfschema/pfs.cc b/storage/perfschema/pfs.cc index dd61df5f861..ad3faeea618 100644 --- a/storage/perfschema/pfs.cc +++ b/storage/perfschema/pfs.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2008, 2015, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2008, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -2560,10 +2560,7 @@ start_table_io_wait_v1(PSI_table_locker_state *state, if (! pfs_table->m_io_enabled) return NULL; - PFS_thread *pfs_thread= pfs_table->m_thread_owner; - - DBUG_ASSERT(pfs_thread == - my_pthread_getspecific_ptr(PFS_thread*, THR_PFS)); + PFS_thread *pfs_thread= my_pthread_getspecific_ptr(PFS_thread*, THR_PFS); register uint flags; ulonglong timer_start= 0; @@ -2666,7 +2663,7 @@ start_table_lock_wait_v1(PSI_table_locker_state *state, if (! pfs_table->m_lock_enabled) return NULL; - PFS_thread *pfs_thread= pfs_table->m_thread_owner; + PFS_thread *pfs_thread= my_pthread_getspecific_ptr(PFS_thread*, THR_PFS); PFS_TL_LOCK_TYPE lock_type; @@ -3068,7 +3065,12 @@ start_socket_wait_v1(PSI_socket_locker_state *state, if (flag_thread_instrumentation) { - PFS_thread *pfs_thread= pfs_socket->m_thread_owner; + /* + Do not use pfs_socket->m_thread_owner here, + as different threads may use concurrently the same socket, + for example during a KILL. + */ + PFS_thread *pfs_thread= my_pthread_getspecific_ptr(PFS_thread*, THR_PFS); if (unlikely(pfs_thread == NULL)) return NULL; @@ -3436,6 +3438,8 @@ static void end_idle_wait_v1(PSI_idle_locker* locker) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } @@ -3517,6 +3521,8 @@ static void end_mutex_wait_v1(PSI_mutex_locker* locker, int rc) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } } @@ -3596,6 +3602,8 @@ static void end_rwlock_rdwait_v1(PSI_rwlock_locker* locker, int rc) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } } @@ -3668,6 +3676,8 @@ static void end_rwlock_wrwait_v1(PSI_rwlock_locker* locker, int rc) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } } @@ -3732,6 +3742,8 @@ static void end_cond_wait_v1(PSI_cond_locker* locker, int rc) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } } @@ -3826,6 +3838,8 @@ static void end_table_io_wait_v1(PSI_table_locker* locker) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } @@ -3895,6 +3909,8 @@ static void end_table_lock_wait_v1(PSI_table_locker* locker) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } @@ -4143,6 +4159,8 @@ static void end_file_wait_v1(PSI_file_locker *locker, if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } } @@ -5070,6 +5088,8 @@ static void end_socket_wait_v1(PSI_socket_locker *locker, size_t byte_count) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } diff --git a/storage/perfschema/pfs_digest.cc b/storage/perfschema/pfs_digest.cc index 0a01ee08993..3330c29795f 100644 --- a/storage/perfschema/pfs_digest.cc +++ b/storage/perfschema/pfs_digest.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2008, 2015, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2008, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -47,7 +47,7 @@ bool flag_statements_digest= true; Current index in Stat array where new record is to be inserted. index 0 is reserved for "all else" case when entire array is full. */ -volatile uint32 digest_index; +volatile uint32 PFS_ALIGNED digest_monotonic_index; bool digest_full= false; LF_HASH digest_hash; @@ -65,7 +65,7 @@ int init_digest(const PFS_global_param *param) */ digest_max= param->m_digest_sizing; digest_lost= 0; - digest_index= 1; + PFS_atomic::store_u32(& digest_monotonic_index, 1); digest_full= false; if (digest_max == 0) @@ -107,6 +107,9 @@ int init_digest(const PFS_global_param *param) + index * pfs_max_digest_length, pfs_max_digest_length); } + /* Set record[0] as allocated. */ + statements_digest_stat_array[0].m_lock.set_allocated(); + return 0; } @@ -209,9 +212,10 @@ find_or_create_digest(PFS_thread *thread, memcpy(hash_key.m_schema_name, schema_name, schema_name_length); int res; - ulong safe_index; uint retry_count= 0; const uint retry_max= 3; + size_t safe_index; + size_t attempts= 0; PFS_statements_digest_stat **entry; PFS_statements_digest_stat *pfs= NULL; @@ -247,55 +251,70 @@ search: return & pfs->m_stat; } - safe_index= PFS_atomic::add_u32(& digest_index, 1); - if (safe_index >= digest_max) + while (++attempts <= digest_max) { - /* The digest array is now full. */ - digest_full= true; - pfs= &statements_digest_stat_array[0]; - - if (pfs->m_first_seen == 0) - pfs->m_first_seen= now; - pfs->m_last_seen= now; - return & pfs->m_stat; - } - - /* Add a new record in digest stat array. */ - pfs= &statements_digest_stat_array[safe_index]; - - /* Copy digest hash/LF Hash search key. */ - memcpy(& pfs->m_digest_key, &hash_key, sizeof(PFS_digest_key)); - - /* - Copy digest storage to statement_digest_stat_array so that it could be - used later to generate digest text. - */ - pfs->m_digest_storage.copy(digest_storage); - - pfs->m_first_seen= now; - pfs->m_last_seen= now; + safe_index= PFS_atomic::add_u32(& digest_monotonic_index, 1) % digest_max; + if (safe_index == 0) + { + /* Record [0] is reserved. */ + safe_index= 1; + } - res= lf_hash_insert(&digest_hash, pins, &pfs); - if (likely(res == 0)) - { - return & pfs->m_stat; - } + /* Add a new record in digest stat array. */ + pfs= &statements_digest_stat_array[safe_index]; - if (res > 0) - { - /* Duplicate insert by another thread */ - if (++retry_count > retry_max) + if (pfs->m_lock.is_free()) { - /* Avoid infinite loops */ - digest_lost++; - return NULL; + if (pfs->m_lock.free_to_dirty()) + { + /* Copy digest hash/LF Hash search key. */ + memcpy(& pfs->m_digest_key, &hash_key, sizeof(PFS_digest_key)); + + /* + Copy digest storage to statement_digest_stat_array so that it could be + used later to generate digest text. + */ + pfs->m_digest_storage.copy(digest_storage); + + pfs->m_first_seen= now; + pfs->m_last_seen= now; + + res= lf_hash_insert(&digest_hash, pins, &pfs); + if (likely(res == 0)) + { + pfs->m_lock.dirty_to_allocated(); + return & pfs->m_stat; + } + + pfs->m_lock.dirty_to_free(); + + if (res > 0) + { + /* Duplicate insert by another thread */ + if (++retry_count > retry_max) + { + /* Avoid infinite loops */ + digest_lost++; + return NULL; + } + goto search; + } + + /* OOM in lf_hash_insert */ + digest_lost++; + return NULL; + } } - goto search; } - /* OOM in lf_hash_insert */ - digest_lost++; - return NULL; + /* The digest array is now full. */ + digest_full= true; + pfs= &statements_digest_stat_array[0]; + + if (pfs->m_first_seen == 0) + pfs->m_first_seen= now; + pfs->m_last_seen= now; + return & pfs->m_stat; } void purge_digest(PFS_thread* thread, PFS_digest_key *hash_key) @@ -322,10 +341,12 @@ void purge_digest(PFS_thread* thread, PFS_digest_key *hash_key) void PFS_statements_digest_stat::reset_data(unsigned char *token_array, uint length) { + m_lock.set_dirty(); m_digest_storage.reset(token_array, length); m_stat.reset(); m_first_seen= 0; m_last_seen= 0; + m_lock.dirty_to_free(); } void PFS_statements_digest_stat::reset_index(PFS_thread *thread) @@ -353,11 +374,14 @@ void reset_esms_by_digest() statements_digest_stat_array[index].reset_data(statements_digest_token_array + index * pfs_max_digest_length, pfs_max_digest_length); } + /* Mark record[0] as allocated again. */ + statements_digest_stat_array[0].m_lock.set_allocated(); + /* Reset index which indicates where the next calculated digest information to be inserted in statements_digest_stat_array. */ - digest_index= 1; + PFS_atomic::store_u32(& digest_monotonic_index, 1); digest_full= false; } diff --git a/storage/perfschema/pfs_digest.h b/storage/perfschema/pfs_digest.h index 76d6c33d984..429a9f4250a 100644 --- a/storage/perfschema/pfs_digest.h +++ b/storage/perfschema/pfs_digest.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2011, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -44,6 +44,9 @@ struct PFS_digest_key /** A statement digest stat record. */ struct PFS_ALIGNED PFS_statements_digest_stat { + /** Internal lock. */ + pfs_lock m_lock; + /** Digest Schema + MD5 Hash. */ PFS_digest_key m_digest_key; diff --git a/storage/perfschema/pfs_lock.h b/storage/perfschema/pfs_lock.h index 09efecd1c5f..be84d0f7fb4 100644 --- a/storage/perfschema/pfs_lock.h +++ b/storage/perfschema/pfs_lock.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2009, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -156,6 +156,18 @@ struct pfs_lock } /** + Initialize a lock to dirty. + */ + void set_dirty(void) + { + /* Do not set the version to 0, read the previous value. */ + uint32 copy= PFS_atomic::load_u32(&m_version_state); + /* Increment the version, set the DIRTY state */ + uint32 new_val= (copy & VERSION_MASK) + VERSION_INC + PFS_LOCK_DIRTY; + PFS_atomic::store_u32(&m_version_state, new_val); + } + + /** Execute a dirty to free transition. This transition should be executed by the writer that owns the record. */ diff --git a/storage/perfschema/table_esms_by_digest.cc b/storage/perfschema/table_esms_by_digest.cc index 88860a77c5d..6c2b173d000 100644 --- a/storage/perfschema/table_esms_by_digest.cc +++ b/storage/perfschema/table_esms_by_digest.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2010, 2012, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2010, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -113,11 +113,14 @@ int table_esms_by_digest::rnd_next(void) m_pos.next()) { digest_stat= &statements_digest_stat_array[m_pos.m_index]; - if (digest_stat->m_first_seen != 0) + if (digest_stat->m_lock.is_populated()) { - make_row(digest_stat); - m_next_pos.set_after(&m_pos); - return 0; + if (digest_stat->m_first_seen != 0) + { + make_row(digest_stat); + m_next_pos.set_after(&m_pos); + return 0; + } } } @@ -135,10 +138,13 @@ table_esms_by_digest::rnd_pos(const void *pos) set_position(pos); digest_stat= &statements_digest_stat_array[m_pos.m_index]; - if (digest_stat->m_first_seen != 0) + if (digest_stat->m_lock.is_populated()) { - make_row(digest_stat); - return 0; + if (digest_stat->m_first_seen != 0) + { + make_row(digest_stat); + return 0; + } } return HA_ERR_RECORD_DELETED; diff --git a/storage/perfschema/unittest/pfs-t.cc b/storage/perfschema/unittest/pfs-t.cc index 6121fac098f..f76b1aa2e75 100644 --- a/storage/perfschema/unittest/pfs-t.cc +++ b/storage/perfschema/unittest/pfs-t.cc @@ -1316,6 +1316,7 @@ void test_locker_disabled() /* Pretend the socket does not have a thread owner */ /* ---------------------------------------------- */ + psi->delete_current_thread(); socket_class_A->m_enabled= true; socket_A1= psi->init_socket(socket_key_A, NULL, NULL, 0); ok(socket_A1 != NULL, "instrumented"); diff --git a/support-files/CMakeLists.txt b/support-files/CMakeLists.txt index eceff5fcf18..e2c449d3b80 100644 --- a/support-files/CMakeLists.txt +++ b/support-files/CMakeLists.txt @@ -59,6 +59,7 @@ IF(UNIX) IF(INSTALL_SUPPORTFILESDIR) INSTALL(FILES magic DESTINATION ${inst_location} COMPONENT SupportFiles) INSTALL(DIRECTORY policy DESTINATION ${inst_location} COMPONENT SupportFiles) + ADD_SUBDIRECTORY(SELinux) ENDIF() CONFIGURE_FILE(mariadb.pc.in ${CMAKE_CURRENT_BINARY_DIR}/mariadb.pc @ONLY) diff --git a/support-files/SELinux/CMakeLists.txt b/support-files/SELinux/CMakeLists.txt new file mode 100644 index 00000000000..e3cdb26ca8f --- /dev/null +++ b/support-files/SELinux/CMakeLists.txt @@ -0,0 +1,35 @@ +# Copyright (c) 2017, MariaDB +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +FIND_PROGRAM(CHECKMODULE checkmodule) +FIND_PROGRAM(SEMODULE_PACKAGE semodule_package) +MARK_AS_ADVANCED(CHECKMODULE SEMODULE_PACKAGE) + +SET(params DESTINATION ${INSTALL_SUPPORTFILESDIR}/SELinux COMPONENT SupportFiles) + +IF(CHECKMODULE AND SEMODULE_PACKAGE) + FOREACH(pol centos6-mariadb) + SET(src ${CMAKE_CURRENT_SOURCE_DIR}/${pol}.te) + SET(mod ${CMAKE_CURRENT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/${pol}-pp.dir/${pol}.mod) + SET(out ${CMAKE_CURRENT_BINARY_DIR}/${pol}.pp) + ADD_CUSTOM_COMMAND(OUTPUT ${out} + COMMAND ${CHECKMODULE} -M -m ${src} -o ${mod} + COMMAND ${SEMODULE_PACKAGE} -m ${mod} -o ${out} + DEPENDS ${src}) + ADD_CUSTOM_TARGET(${pol}-pp ALL DEPENDS ${out}) + INSTALL(FILES ${out} ${params}) + ENDFOREACH() +ENDIF() +INSTALL(FILES centos6-mariadb.te rhel4-mysql.fc rhel4-mysql.te ${params}) diff --git a/support-files/SELinux/centos6-mariadb.te b/support-files/SELinux/centos6-mariadb.te new file mode 100644 index 00000000000..1d3de52c700 --- /dev/null +++ b/support-files/SELinux/centos6-mariadb.te @@ -0,0 +1,9 @@ +module mariadb 1.0; + +require { + type mysqld_safe_t; + class capability { setuid setgid }; +} + +#============= mysqld_safe_t ============== +allow mysqld_safe_t self:capability { setuid setgid }; diff --git a/support-files/SELinux/rhel4-mysql.fc b/support-files/SELinux/rhel4-mysql.fc new file mode 100644 index 00000000000..aa0fced4bbc --- /dev/null +++ b/support-files/SELinux/rhel4-mysql.fc @@ -0,0 +1,41 @@ +# Copyright (C) 2007 MySQL AB +# Use is subject to license terms +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +# MySQL Database Server + +# +# /etc +# +/etc/my\.cnf -- gen_context(system_u:object_r:mysqld_etc_t,s0) +/etc/mysql(/.*)? gen_context(system_u:object_r:mysqld_etc_t,s0) + +# +# /usr +# Red Hat compatibility +/usr/libexec/mysqld -- gen_context(system_u:object_r:mysqld_exec_t,s0) + +# MySQL AB compatibility +/usr/sbin/mysqld(-max)? -- gen_context(system_u:object_r:mysqld_exec_t,s0) + +# +# /var +# +/var/lib/mysql(/.*)? gen_context(system_u:object_r:mysqld_db_t,s0) +/var/lib/mysql/mysql\.sock -s gen_context(system_u:object_r:mysqld_var_run_t,s0) + +/var/log/mysql.* -- gen_context(system_u:object_r:mysqld_log_t,s0) + +/var/run/mysqld(/.*)? gen_context(system_u:object_r:mysqld_var_run_t,s0) diff --git a/support-files/SELinux/rhel4-mysql.te b/support-files/SELinux/rhel4-mysql.te new file mode 100644 index 00000000000..d2609244640 --- /dev/null +++ b/support-files/SELinux/rhel4-mysql.te @@ -0,0 +1,147 @@ +# Copyright (C) 2007 MySQL AB +# Use is subject to license terms +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +policy_module(mysql,1.0.0) + +######################################## +# +# Declarations +# + +type mysqld_t; +type mysqld_exec_t; +init_daemon_domain(mysqld_t,mysqld_exec_t) + +type mysqld_var_run_t; +files_pid_file(mysqld_var_run_t) + +type mysqld_db_t; +files_type(mysqld_db_t) + +type mysqld_etc_t alias etc_mysqld_t; +files_config_file(mysqld_etc_t) + +type mysqld_log_t; +logging_log_file(mysqld_log_t) + +type mysqld_tmp_t; +files_tmp_file(mysqld_tmp_t) + +######################################## +# +# Local policy +# + +allow mysqld_t self:capability { dac_override setgid setuid sys_resource net_bind_service }; +dontaudit mysqld_t self:capability sys_tty_config; +allow mysqld_t self:process { setsched getsched setrlimit signal_perms rlimitinh }; +allow mysqld_t self:fifo_file { read write }; +allow mysqld_t self:unix_stream_socket create_stream_socket_perms; +allow mysqld_t self:tcp_socket create_stream_socket_perms; +allow mysqld_t self:udp_socket create_socket_perms; + +allow mysqld_t mysqld_db_t:dir create_dir_perms; +allow mysqld_t mysqld_db_t:file create_file_perms; +allow mysqld_t mysqld_db_t:lnk_file create_lnk_perms; +files_var_lib_filetrans(mysqld_t,mysqld_db_t,{ dir file }) + +allow mysqld_t mysqld_etc_t:file { getattr read }; +allow mysqld_t mysqld_etc_t:lnk_file { getattr read }; +allow mysqld_t mysqld_etc_t:dir list_dir_perms; + +allow mysqld_t mysqld_log_t:file create_file_perms; +logging_log_filetrans(mysqld_t,mysqld_log_t,file) + +allow mysqld_t mysqld_tmp_t:dir create_dir_perms; +allow mysqld_t mysqld_tmp_t:file create_file_perms; +files_tmp_filetrans(mysqld_t, mysqld_tmp_t, { file dir }) + +allow mysqld_t mysqld_var_run_t:dir rw_dir_perms; +allow mysqld_t mysqld_var_run_t:sock_file create_file_perms; +allow mysqld_t mysqld_var_run_t:file create_file_perms; +files_pid_filetrans(mysqld_t,mysqld_var_run_t,file) + +kernel_read_system_state(mysqld_t) +kernel_read_kernel_sysctls(mysqld_t) + +corenet_non_ipsec_sendrecv(mysqld_t) +corenet_tcp_sendrecv_all_if(mysqld_t) +corenet_udp_sendrecv_all_if(mysqld_t) +corenet_tcp_sendrecv_all_nodes(mysqld_t) +corenet_udp_sendrecv_all_nodes(mysqld_t) +corenet_tcp_sendrecv_all_ports(mysqld_t) +corenet_udp_sendrecv_all_ports(mysqld_t) +corenet_tcp_bind_all_nodes(mysqld_t) +corenet_tcp_bind_mysqld_port(mysqld_t) +corenet_tcp_connect_mysqld_port(mysqld_t) +corenet_sendrecv_mysqld_client_packets(mysqld_t) +corenet_sendrecv_mysqld_server_packets(mysqld_t) + +dev_read_sysfs(mysqld_t) + +fs_getattr_all_fs(mysqld_t) +fs_search_auto_mountpoints(mysqld_t) + +term_dontaudit_use_console(mysqld_t) + +domain_use_interactive_fds(mysqld_t) + +files_getattr_var_lib_dirs(mysqld_t) +files_read_etc_runtime_files(mysqld_t) +files_read_etc_files(mysqld_t) +files_read_usr_files(mysqld_t) +files_search_var_lib(mysqld_t) + +auth_use_nsswitch(mysqld_t) + +init_use_fds(mysqld_t) +init_use_script_ptys(mysqld_t) + +libs_use_ld_so(mysqld_t) +libs_use_shared_libs(mysqld_t) + +logging_send_syslog_msg(mysqld_t) + +miscfiles_read_localization(mysqld_t) + +sysnet_read_config(mysqld_t) + +userdom_dontaudit_use_unpriv_user_fds(mysqld_t) +# for /root/.my.cnf - should not be needed: +userdom_read_sysadm_home_content_files(mysqld_t) + +ifdef(`distro_redhat',` + # because Fedora has the sock_file in the database directory + type_transition mysqld_t mysqld_db_t:sock_file mysqld_var_run_t; +') + +ifdef(`targeted_policy',` + term_dontaudit_use_unallocated_ttys(mysqld_t) + term_dontaudit_use_generic_ptys(mysqld_t) + files_dontaudit_read_root_files(mysqld_t) +') + +optional_policy(` + daemontools_service_domain(mysqld_t, mysqld_exec_t) +') + +optional_policy(` + seutil_sigchld_newrole(mysqld_t) +') + +optional_policy(` + udev_read_db(mysqld_t) +') diff --git a/support-files/mysql.server.sh b/support-files/mysql.server.sh index 145daff6962..28bfd19be60 100644 --- a/support-files/mysql.server.sh +++ b/support-files/mysql.server.sh @@ -156,15 +156,9 @@ parse_server_arguments() { # Get arguments from the my.cnf file, # the only group, which is read from now on is [mysqld] -if test -x ./bin/my_print_defaults -then - print_defaults="./bin/my_print_defaults" -elif test -x $bindir/my_print_defaults +if test -x $bindir/my_print_defaults then print_defaults="$bindir/my_print_defaults" -elif test -x $bindir/mysql_print_defaults -then - print_defaults="$bindir/mysql_print_defaults" else # Try to find basedir in /etc/my.cnf conf=/etc/my.cnf diff --git a/support-files/rpm/server-postin.sh b/support-files/rpm/server-postin.sh index 5b95b751212..fca66a96c04 100644 --- a/support-files/rpm/server-postin.sh +++ b/support-files/rpm/server-postin.sh @@ -93,7 +93,12 @@ if [ -f /etc/redhat-release ] ; then echo ' make load' echo echo - fi + fi + if grep 'CentOS release 6' /etc/redhat-release >/dev/null 2>&1; then + if [ -x /usr/sbin/semodule ] ; then + /usr/sbin/semodule -i /usr/share/mysql/SELinux/centos6-mariadb.pp + fi + fi fi if [ -x sbin/restorecon ] ; then |