diff options
author | Vicențiu Ciorbaru <vicentiu@mariadb.org> | 2017-01-12 03:37:35 +0200 |
---|---|---|
committer | Vicențiu Ciorbaru <vicentiu@mariadb.org> | 2017-01-12 03:37:35 +0200 |
commit | 4f5338467807f099b67f07d0bfb415358222a2b1 (patch) | |
tree | 9be13fd51713f08b9f5952e73f673cda824377d6 /storage | |
parent | 833fda8f1a0c415ced1f2b8c1851809383bcc639 (diff) | |
parent | 1c5ca7c1837b9db071cb2e791d6605f8806af0c1 (diff) | |
download | mariadb-git-4f5338467807f099b67f07d0bfb415358222a2b1.tar.gz |
Merge branch 'bb-10.0-vicentiu' into 10.0mariadb-10.0.29
Extra merge commit due to intermediate commits pushed to 10.0 while
merge was done.
Diffstat (limited to 'storage')
74 files changed, 9116 insertions, 1991 deletions
diff --git a/storage/connect/ApacheInterface.java b/storage/connect/ApacheInterface.java index b4c8a4e9885..47b46dc0506 100644 --- a/storage/connect/ApacheInterface.java +++ b/storage/connect/ApacheInterface.java @@ -35,7 +35,10 @@ public class ApacheInterface extends JdbcInterface { ds.setPassword(parms[3]); pool.put(url, ds); } // endif ds - + + // if (parms.length > 4 && parms[4] != null) + // ds.setConnectionProperties(parms[4]); + // Get a connection from the data source conn = ds.getConnection(); diff --git a/storage/connect/CMakeLists.txt b/storage/connect/CMakeLists.txt index 95d88538119..ce6de424421 100644 --- a/storage/connect/CMakeLists.txt +++ b/storage/connect/CMakeLists.txt @@ -20,25 +20,25 @@ SET(CONNECT_SOURCES ha_connect.cc connect.cc user_connect.cc mycat.cc fmdlex.c osutil.c plugutil.c rcmsg.c rcmsg.h array.cpp blkfil.cpp colblk.cpp csort.cpp -filamap.cpp filamdbf.cpp filamfix.cpp filamtxt.cpp filamvct.cpp filamzip.cpp +filamap.cpp filamdbf.cpp filamfix.cpp filamgz.cpp filamtxt.cpp filter.cpp json.cpp jsonudf.cpp maputil.cpp myconn.cpp myutil.cpp plgdbutl.cpp reldef.cpp tabcol.cpp tabdos.cpp tabfix.cpp tabfmt.cpp tabjson.cpp table.cpp tabmul.cpp tabmysql.cpp taboccur.cpp tabpivot.cpp tabsys.cpp tabtbl.cpp tabutil.cpp -tabvct.cpp tabvir.cpp tabxcl.cpp valblk.cpp value.cpp xindex.cpp xobject.cpp +tabvir.cpp tabxcl.cpp valblk.cpp value.cpp xindex.cpp xobject.cpp array.h blkfil.h block.h catalog.h checklvl.h colblk.h connect.h csort.h -engmsg.h filamap.h filamdbf.h filamfix.h filamtxt.h filamvct.h filamzip.h +engmsg.h filamap.h filamdbf.h filamfix.h filamgz.h filamtxt.h filter.h global.h ha_connect.h inihandl.h json.h jsonudf.h maputil.h msgid.h mycat.h myconn.h myutil.h os.h osutil.h plgcnx.h plgdbsem.h preparse.h reldef.h resource.h tabcol.h tabdos.h tabfix.h tabfmt.h tabjson.h tabmul.h tabmysql.h -taboccur.h tabpivot.h tabsys.h tabtbl.h tabutil.h tabvct.h tabvir.h tabxcl.h +taboccur.h tabpivot.h tabsys.h tabtbl.h tabutil.h tabvir.h tabxcl.h user_connect.h valblk.h value.h xindex.h xobject.h xtable.h) # # Definitions that are shared for all OSes # add_definitions( -DMARIADB -DFORCE_INIT_OF_VARS -Dconnect_EXPORTS) -add_definitions( -DHUGE_SUPPORT -DZIP_SUPPORT -DPIVOT_SUPPORT ) +add_definitions( -DHUGE_SUPPORT -DGZ_SUPPORT -DPIVOT_SUPPORT ) # @@ -90,6 +90,18 @@ ENDIF(UNIX) # +# VCT: the VEC format might be not supported in future versions +# + +OPTION(CONNECT_WITH_VCT "Compile CONNECT storage engine with VCT support" ON) + +IF(CONNECT_WITH_VCT) + SET(CONNECT_SOURCES ${CONNECT_SOURCES} filamvct.cpp tabvct.cpp filamvct.h tabvct.h) + add_definitions(-DVCT_SUPPORT) +ENDIF(CONNECT_WITH_VCT) + + +# # XML # @@ -236,9 +248,9 @@ ENDIF(CONNECT_WITH_ODBC) # JDBC # IF(APPLE) - OPTION(CONNECT_WITH_JDBC "some comment" OFF) + OPTION(CONNECT_WITH_JDBC "Compile CONNECT storage engine without JDBC support" OFF) ELSE() - OPTION(CONNECT_WITH_JDBC "some comment" ON) + OPTION(CONNECT_WITH_JDBC "Compile CONNECT storage engine with JDBC support" ON) ENDIF() IF(CONNECT_WITH_JDBC) @@ -252,18 +264,33 @@ IF(CONNECT_WITH_JDBC) SET(CONNECT_SOURCES ${CONNECT_SOURCES} jdbconn.cpp tabjdbc.cpp jdbconn.h tabjdbc.h jdbccat.h JdbcInterface.java ApacheInterface.java MariadbInterface.java - MysqlInterface.java OracleInterface.java PostgresqlInterface.java) + MysqlInterface.java OracleInterface.java PostgresqlInterface.java + JavaWrappers.jar) # TODO: Find how to compile and install the java wrapper classes # Find required libraries and include directories SET (JAVA_SOURCES JdbcInterface.java) add_jar(JdbcInterface ${JAVA_SOURCES}) - install_jar(JdbcInterface DESTINATION ${INSTALL_PLUGINDIR} COMPONENT connect-engine) + install_jar(JdbcInterface DESTINATION ${INSTALL_PLUGINDIR} COMPONENT connect-engine) + INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/JavaWrappers.jar + DESTINATION ${INSTALL_PLUGINDIR} COMPONENT connect-engine) add_definitions(-DJDBC_SUPPORT) ELSE() SET(JDBC_LIBRARY "") ENDIF() ENDIF(CONNECT_WITH_JDBC) +# +# ZIP +# + +OPTION(CONNECT_WITH_ZIP "Compile CONNECT storage engine with ZIP support" ON) + +IF(CONNECT_WITH_ZIP) + SET(CONNECT_SOURCES ${CONNECT_SOURCES} filamzip.cpp tabzip.cpp unzip.c ioapi.c zip.c + filamzip.h tabzip.h ioapi.h unzip.h zip.h) + add_definitions(-DZIP_SUPPORT -DNOCRYPT) +ENDIF(CONNECT_WITH_ZIP) + # # XMAP diff --git a/storage/connect/JavaWrappers.jar b/storage/connect/JavaWrappers.jar Binary files differnew file mode 100644 index 00000000000..8c01c364a3f --- /dev/null +++ b/storage/connect/JavaWrappers.jar diff --git a/storage/connect/JdbcInterface.java b/storage/connect/JdbcInterface.java index 34af8c4e013..a1b1360e6ea 100644 --- a/storage/connect/JdbcInterface.java +++ b/storage/connect/JdbcInterface.java @@ -82,6 +82,9 @@ public class JdbcInterface { System.out.println("URL=" + parms[1]); CheckURL(parms[1], null); + + // This is required for drivers using context class loaders + Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); if (parms[2] != null && !parms[2].isEmpty()) { if (DEBUG) @@ -220,6 +223,19 @@ public class JdbcInterface { } // end of SetTimestampParm + public int SetNullParm(int i, int typ) { + int rc = 0; + + try { + pstmt.setNull(i, typ); + } catch (Exception e) { + SetErrmsg(e); + rc = -1; + } // end try/catch + + return rc; + } // end of SetNullParm + public int ExecutePrep() { int n = -3; diff --git a/storage/connect/domdoc.cpp b/storage/connect/domdoc.cpp index 64a0a172956..eb9660b439d 100644 --- a/storage/connect/domdoc.cpp +++ b/storage/connect/domdoc.cpp @@ -89,30 +89,43 @@ DOMDOC::DOMDOC(char *nsl, char *nsdf, char *enc, PFBLOCK fp) /******************************************************************/ /* Initialize XML parser and check library compatibility. */ /******************************************************************/ -bool DOMDOC::Initialize(PGLOBAL g) - { - if (TestHr(g, CoInitialize(NULL))) +bool DOMDOC::Initialize(PGLOBAL g, char *entry, bool zipped) +{ + if (zipped && InitZip(g, entry)) + return true; + + if (TestHr(g, CoInitialize(NULL))) return true; if (TestHr(g, Docp.CreateInstance("msxml2.domdocument"))) return true; return MakeNSlist(g); - } // end of Initialize +} // end of Initialize /******************************************************************/ /* Parse the XML file and construct node tree in memory. */ /******************************************************************/ -bool DOMDOC::ParseFile(char *fn) - { - // Load the document +bool DOMDOC::ParseFile(PGLOBAL g, char *fn) +{ + bool b; + Docp->async = false; - if (!(bool)Docp->load((_bstr_t)fn)) + if (zip) { + // Parse an in memory document + char *xdoc = GetMemDoc(g, fn); + + b = (xdoc) ? (bool)Docp->loadXML((_bstr_t)xdoc) : false; + } else + // Load the document + b = (bool)Docp->load((_bstr_t)fn); + + if (!b) return true; return false; - } // end of ParseFile +} // end of ParseFile /******************************************************************/ /* Create or reuse an Xblock for this document. */ @@ -239,6 +252,7 @@ int DOMDOC::DumpDoc(PGLOBAL g, char *ofn) void DOMDOC::CloseDoc(PGLOBAL g, PFBLOCK xp) { CloseXMLFile(g, xp, false); + CloseZip(); } // end of Close /* ----------------------- class DOMNODE ------------------------ */ @@ -616,13 +630,13 @@ PXNODE DOMNODELIST::GetItem(PGLOBAL g, int n, PXNODE np) /* Reset the pointer on the deleted item. */ /******************************************************************/ bool DOMNODELIST::DropItem(PGLOBAL g, int n) - { - if (Listp == NULL || Listp->length <= n) - return true; +{ + if (Listp == NULL || Listp->length < n) + return true; //Listp->item[n] = NULL; La propriété n'a pas de méthode 'set' return false; - } // end of DeleteItem +} // end of DeleteItem /* ----------------------- class DOMATTR ------------------------ */ diff --git a/storage/connect/domdoc.h b/storage/connect/domdoc.h index 2cffec499e2..cfec98a9422 100644 --- a/storage/connect/domdoc.h +++ b/storage/connect/domdoc.h @@ -37,8 +37,8 @@ class DOMDOC : public XMLDOCUMENT { virtual void SetNofree(bool b) {} // Only libxml2 // Methods - virtual bool Initialize(PGLOBAL g); - virtual bool ParseFile(char *fn); + virtual bool Initialize(PGLOBAL g, char *entry, bool zipped); + virtual bool ParseFile(PGLOBAL g, char *fn); virtual bool NewDoc(PGLOBAL g, char *ver); virtual void AddComment(PGLOBAL g, char *com); virtual PXNODE GetRoot(PGLOBAL g); diff --git a/storage/connect/filamap.cpp b/storage/connect/filamap.cpp index 5cf9a4d945c..94c562a9981 100644 --- a/storage/connect/filamap.cpp +++ b/storage/connect/filamap.cpp @@ -87,7 +87,7 @@ int MAPFAM::GetFileLength(PGLOBAL g) { int len; - len = (To_Fb) ? To_Fb->Length : TXTFAM::GetFileLength(g); + len = (To_Fb && To_Fb->Count) ? To_Fb->Length : TXTFAM::GetFileLength(g); if (trace) htrc("Mapped file length=%d\n", len); @@ -319,11 +319,13 @@ int MAPFAM::SkipRecord(PGLOBAL g, bool header) /***********************************************************************/ int MAPFAM::ReadBuffer(PGLOBAL g) { - int len; + int rc, len; // Are we at the end of the memory - if (Mempos >= Top) - return RC_EF; + if (Mempos >= Top) + if ((rc = GetNext(g)) != RC_OK) + return rc; + if (!Placed) { /*******************************************************************/ @@ -341,8 +343,10 @@ int MAPFAM::ReadBuffer(PGLOBAL g) /*******************************************************************/ switch (Tdbp->TestBlock(g)) { case RC_EF: - return RC_EF; - case RC_NF: + if ((rc = GetNext(g)) != RC_OK) + return rc; + + case RC_NF: // Skip this record if ((rc = SkipRecord(g, false)) != RC_OK) return rc; @@ -413,7 +417,7 @@ int MAPFAM::DeleteRecords(PGLOBAL g, int irc) if (Tpos == Spos) { /*******************************************************************/ - /* First line to delete. Move of eventual preceeding lines is */ + /* First line to delete. Move of eventual preceding lines is */ /* not required here, just setting of future Spos and Tpos. */ /*******************************************************************/ Tpos = Spos = Fpos; @@ -498,7 +502,7 @@ int MAPFAM::DeleteRecords(PGLOBAL g, int irc) void MAPFAM::CloseTableFile(PGLOBAL g, bool) { PlugCloseFile(g, To_Fb); - To_Fb = NULL; // To get correct file size in Cardinality +//To_Fb = NULL; // To get correct file size in Cardinality if (trace) htrc("MAP Close: closing %s count=%d\n", @@ -569,7 +573,7 @@ int MBKFAM::GetRowID(void) /***********************************************************************/ int MBKFAM::ReadBuffer(PGLOBAL g) { - int len; + int rc, len; /*********************************************************************/ /* Sequential block reading when Placed is not true. */ @@ -577,8 +581,10 @@ int MBKFAM::ReadBuffer(PGLOBAL g) if (Placed) { Placed = false; } else if (Mempos >= Top) { // Are we at the end of the memory - return RC_EF; - } else if (++CurNum < Nrec) { + if ((rc = GetNext(g)) != RC_OK) + return rc; + + } else if (++CurNum < Nrec) { Fpos = Mempos; } else { /*******************************************************************/ @@ -588,7 +594,8 @@ int MBKFAM::ReadBuffer(PGLOBAL g) next: if (++CurBlk >= Block) - return RC_EF; + if ((rc = GetNext(g)) != RC_OK) + return rc; /*******************************************************************/ /* Before reading a new block, check whether block optimization */ @@ -596,8 +603,11 @@ int MBKFAM::ReadBuffer(PGLOBAL g) /*******************************************************************/ switch (Tdbp->TestBlock(g)) { case RC_EF: - return RC_EF; - case RC_NF: + if ((rc = GetNext(g)) != RC_OK) + return rc; + + break; + case RC_NF: goto next; } // endswitch rc @@ -697,14 +707,18 @@ int MPXFAM::InitDelete(PGLOBAL, int fpos, int) /***********************************************************************/ int MPXFAM::ReadBuffer(PGLOBAL g) { + int rc; + /*********************************************************************/ /* Sequential block reading when Placed is not true. */ /*********************************************************************/ if (Placed) { Placed = false; } else if (Mempos >= Top) { // Are we at the end of the memory - return RC_EF; - } else if (++CurNum < Nrec) { + if ((rc = GetNext(g)) != RC_OK) + return rc; + + } else if (++CurNum < Nrec) { Fpos = Mempos; } else { /*******************************************************************/ @@ -714,7 +728,7 @@ int MPXFAM::ReadBuffer(PGLOBAL g) next: if (++CurBlk >= Block) - return RC_EF; + return GetNext(g); /*******************************************************************/ /* Before reading a new block, check whether block optimization */ @@ -722,8 +736,11 @@ int MPXFAM::ReadBuffer(PGLOBAL g) /*******************************************************************/ switch (Tdbp->TestBlock(g)) { case RC_EF: - return RC_EF; - case RC_NF: + if ((rc = GetNext(g)) != RC_OK) + return rc; + + break; + case RC_NF: goto next; } // endswitch rc diff --git a/storage/connect/filamap.h b/storage/connect/filamap.h index b9c8ad965fd..774eb8b91b3 100644 --- a/storage/connect/filamap.h +++ b/storage/connect/filamap.h @@ -41,7 +41,8 @@ class DllExport MAPFAM : public TXTFAM { virtual int SkipRecord(PGLOBAL g, bool header); virtual bool OpenTableFile(PGLOBAL g); virtual bool DeferReading(void) {return false;} - virtual int ReadBuffer(PGLOBAL g); + virtual int GetNext(PGLOBAL g) {return RC_EF;} + virtual int ReadBuffer(PGLOBAL g); virtual int WriteBuffer(PGLOBAL g); virtual int DeleteRecords(PGLOBAL g, int irc); virtual void CloseTableFile(PGLOBAL g, bool abort); diff --git a/storage/connect/filamgz.cpp b/storage/connect/filamgz.cpp new file mode 100644 index 00000000000..07242ea633c --- /dev/null +++ b/storage/connect/filamgz.cpp @@ -0,0 +1,1426 @@ +/************ File AM GZ C++ Program Source Code File (.CPP) ***********/ +/* PROGRAM NAME: FILAMGZ */ +/* ------------- */ +/* Version 1.5 */ +/* */ +/* COPYRIGHT: */ +/* ---------- */ +/* (C) Copyright to the author Olivier BERTRAND 2005-2016 */ +/* */ +/* WHAT THIS PROGRAM DOES: */ +/* ----------------------- */ +/* This program are the ZLIB compressed files classes. */ +/* */ +/***********************************************************************/ + +/***********************************************************************/ +/* Include relevant MariaDB header file. */ +/***********************************************************************/ +#include "my_global.h" +#if defined(__WIN__) +#include <io.h> +#include <fcntl.h> +#if defined(__BORLANDC__) +#define __MFC_COMPAT__ // To define min/max as macro +#endif +//#include <windows.h> +#else // !__WIN__ +#if defined(UNIX) +#include <errno.h> +#else // !UNIX +#include <io.h> +#endif +#include <fcntl.h> +#endif // !__WIN__ + +/***********************************************************************/ +/* Include application header files: */ +/* global.h is header containing all global declarations. */ +/* plgdbsem.h is header containing the DB application declarations. */ +/* tabdos.h is header containing the TABDOS class declarations. */ +/***********************************************************************/ +#include "global.h" +#include "plgdbsem.h" +//#include "catalog.h" +//#include "reldef.h" +//#include "xobject.h" +//#include "kindex.h" +#include "filamtxt.h" +#include "tabdos.h" +#if defined(UNIX) +#include "osutil.h" +#endif + +/***********************************************************************/ +/* This define prepares ZLIB function declarations. */ +/***********************************************************************/ +//#define ZLIB_DLL + +#include "filamgz.h" + +/***********************************************************************/ +/* DB static variables. */ +/***********************************************************************/ +extern int num_read, num_there, num_eq[]; // Statistics + +/* ------------------------------------------------------------------- */ + +/***********************************************************************/ +/* Implementation of the GZFAM class. */ +/***********************************************************************/ +GZFAM::GZFAM(PGZFAM txfp) : TXTFAM(txfp) + { + Zfile = txfp->Zfile; + Zpos = txfp->Zpos; + } // end of GZFAM copy constructor + +/***********************************************************************/ +/* Zerror: Error function for gz calls. */ +/* gzerror returns the error message for the last error which occurred*/ +/* on the given compressed file. errnum is set to zlib error number. */ +/* If an error occurred in the file system and not in the compression */ +/* library, errnum is set to Z_ERRNO and the application may consult */ +/* errno to get the exact error code. */ +/***********************************************************************/ +int GZFAM::Zerror(PGLOBAL g) + { + int errnum; + + strcpy(g->Message, gzerror(Zfile, &errnum)); + + if (errnum == Z_ERRNO) +#if defined(__WIN__) + sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(NULL)); +#else // !__WIN__ + sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(errno)); +#endif // !__WIN__ + + return (errnum == Z_STREAM_END) ? RC_EF : RC_FX; + } // end of Zerror + +/***********************************************************************/ +/* Reset: reset position values at the beginning of file. */ +/***********************************************************************/ +void GZFAM::Reset(void) + { + TXTFAM::Reset(); +//gzrewind(Zfile); // Useful ????? + Zpos = 0; + } // end of Reset + +/***********************************************************************/ +/* GZ GetFileLength: returns an estimate of what would be the */ +/* uncompressed file size in number of bytes. */ +/***********************************************************************/ +int GZFAM::GetFileLength(PGLOBAL g) + { + int len = TXTFAM::GetFileLength(g); + + if (len > 0) + // Estimate size reduction to a max of 6 + len *= 6; + + return len; + } // end of GetFileLength + +/***********************************************************************/ +/* GZ Access Method opening routine. */ +/***********************************************************************/ +bool GZFAM::OpenTableFile(PGLOBAL g) + { + char opmode[4], filename[_MAX_PATH]; + MODE mode = Tdbp->GetMode(); + + switch (mode) { + case MODE_READ: + strcpy(opmode, "r"); + break; + case MODE_UPDATE: + /*****************************************************************/ + /* Updating GZ files not implemented yet. */ + /*****************************************************************/ + strcpy(g->Message, MSG(UPD_ZIP_NOT_IMP)); + return true; + case MODE_DELETE: + if (!Tdbp->GetNext()) { + // Store the number of deleted lines + DelRows = Cardinality(g); + + // This will erase the entire file + strcpy(opmode, "w"); +// Block = 0; // For ZBKFAM +// Last = Nrec; // For ZBKFAM + Tdbp->ResetSize(); + } else { + sprintf(g->Message, MSG(NO_PART_DEL), "GZ"); + return true; + } // endif filter + + break; + case MODE_INSERT: + strcpy(opmode, "a+"); + break; + default: + sprintf(g->Message, MSG(BAD_OPEN_MODE), mode); + return true; + } // endswitch Mode + + /*********************************************************************/ + /* Open according to logical input/output mode required. */ + /* Use specific zlib functions. */ + /* Treat files as binary. */ + /*********************************************************************/ + strcat(opmode, "b"); + Zfile = gzopen(PlugSetPath(filename, To_File, Tdbp->GetPath()), opmode); + + if (Zfile == NULL) { + sprintf(g->Message, MSG(GZOPEN_ERROR), + opmode, (int)errno, filename); + strcat(strcat(g->Message, ": "), strerror(errno)); + return (mode == MODE_READ && errno == ENOENT) + ? PushWarning(g, Tdbp) : true; + } // endif Zfile + + /*********************************************************************/ + /* Something to be done here. >>>>>>>> NOT DONE <<<<<<<< */ + /*********************************************************************/ +//To_Fb = dbuserp->Openlist; // Keep track of File block + + /*********************************************************************/ + /* Allocate the line buffer. */ + /*********************************************************************/ + return AllocateBuffer(g); + } // end of OpenTableFile + +/***********************************************************************/ +/* Allocate the line buffer. For mode Delete a bigger buffer has to */ +/* be allocated because is it also used to move lines into the file. */ +/***********************************************************************/ +bool GZFAM::AllocateBuffer(PGLOBAL g) + { + MODE mode = Tdbp->GetMode(); + + Buflen = Lrecl + 2; // Lrecl does not include CRLF +//Buflen *= ((Mode == MODE_DELETE) ? DOS_BUFF_LEN : 1); NIY + + if (trace) + htrc("SubAllocating a buffer of %d bytes\n", Buflen); + + To_Buf = (char*)PlugSubAlloc(g, NULL, Buflen); + + if (mode == MODE_INSERT) { + /*******************************************************************/ + /* For Insert buffer must be prepared. */ + /*******************************************************************/ + memset(To_Buf, ' ', Buflen); + To_Buf[Buflen - 2] = '\n'; + To_Buf[Buflen - 1] = '\0'; + } // endif Insert + + return false; + } // end of AllocateBuffer + +/***********************************************************************/ +/* GetRowID: return the RowID of last read record. */ +/***********************************************************************/ +int GZFAM::GetRowID(void) + { + return Rows; + } // end of GetRowID + +/***********************************************************************/ +/* GetPos: return the position of last read record. */ +/***********************************************************************/ +int GZFAM::GetPos(void) + { + return (int)Zpos; + } // end of GetPos + +/***********************************************************************/ +/* GetNextPos: return the position of next record. */ +/***********************************************************************/ +int GZFAM::GetNextPos(void) + { + return gztell(Zfile); + } // end of GetNextPos + +/***********************************************************************/ +/* SetPos: Replace the table at the specified position. */ +/***********************************************************************/ +bool GZFAM::SetPos(PGLOBAL g, int pos __attribute__((unused))) + { + sprintf(g->Message, MSG(NO_SETPOS_YET), "GZ"); + return true; +#if 0 + Fpos = pos; + + if (fseek(Stream, Fpos, SEEK_SET)) { + sprintf(g->Message, MSG(FSETPOS_ERROR), Fpos); + return true; + } // endif + + Placed = true; + return false; +#endif // 0 + } // end of SetPos + +/***********************************************************************/ +/* Record file position in case of UPDATE or DELETE. */ +/***********************************************************************/ +bool GZFAM::RecordPos(PGLOBAL) + { + Zpos = gztell(Zfile); + return false; + } // end of RecordPos + +/***********************************************************************/ +/* Skip one record in file. */ +/***********************************************************************/ +int GZFAM::SkipRecord(PGLOBAL g, bool header) + { + // Skip this record + if (gzeof(Zfile)) + return RC_EF; + else if (gzgets(Zfile, To_Buf, Buflen) == Z_NULL) + return Zerror(g); + + if (header) + RecordPos(g); + + return RC_OK; + } // end of SkipRecord + +/***********************************************************************/ +/* ReadBuffer: Read one line from a compressed text file. */ +/***********************************************************************/ +int GZFAM::ReadBuffer(PGLOBAL g) + { + char *p; + int rc; + + if (!Zfile) + return RC_EF; + + if (!Placed) { + /*******************************************************************/ + /* Record file position in case of UPDATE or DELETE. */ + /*******************************************************************/ + next: + if (RecordPos(g)) + return RC_FX; + + CurBlk = Rows++; // Update RowID + + /*******************************************************************/ + /* Check whether optimization on ROWID */ + /* can be done, as well as for join as for local filtering. */ + /*******************************************************************/ + switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + // Skip this record + if ((rc = SkipRecord(g, FALSE)) != RC_OK) + return rc; + + goto next; + } // endswitch rc + + } else + Placed = false; + + if (gzeof(Zfile)) { + rc = RC_EF; + } else if (gzgets(Zfile, To_Buf, Buflen) != Z_NULL) { + p = To_Buf + strlen(To_Buf) - 1; + + if (*p == '\n') + *p = '\0'; // Eliminate ending new-line character + + if (*(--p) == '\r') + *p = '\0'; // Eliminate eventuel carriage return + + strcpy(Tdbp->GetLine(), To_Buf); + IsRead = true; + rc = RC_OK; + num_read++; + } else + rc = Zerror(g); + + if (trace > 1) + htrc(" Read: '%s' rc=%d\n", To_Buf, rc); + + return rc; + } // end of ReadBuffer + +/***********************************************************************/ +/* WriteDB: Data Base write routine for ZDOS access method. */ +/* Update is not possible without using a temporary file (NIY). */ +/***********************************************************************/ +int GZFAM::WriteBuffer(PGLOBAL g) + { + /*********************************************************************/ + /* Prepare the write buffer. */ + /*********************************************************************/ + strcat(strcpy(To_Buf, Tdbp->GetLine()), CrLf); + + /*********************************************************************/ + /* Now start the writing process. */ + /*********************************************************************/ + if (gzputs(Zfile, To_Buf) < 0) + return Zerror(g); + + return RC_OK; + } // end of WriteBuffer + +/***********************************************************************/ +/* Data Base delete line routine for ZDOS access method. (NIY) */ +/***********************************************************************/ +int GZFAM::DeleteRecords(PGLOBAL g, int) + { + strcpy(g->Message, MSG(NO_ZIP_DELETE)); + return RC_FX; + } // end of DeleteRecords + +/***********************************************************************/ +/* Data Base close routine for DOS access method. */ +/***********************************************************************/ +void GZFAM::CloseTableFile(PGLOBAL, bool) + { + int rc = gzclose(Zfile); + + if (trace) + htrc("GZ CloseDB: closing %s rc=%d\n", To_File, rc); + + Zfile = NULL; // So we can know whether table is open +//To_Fb->Count = 0; // Avoid double closing by PlugCloseAll + } // end of CloseTableFile + +/***********************************************************************/ +/* Rewind routine for GZ access method. */ +/***********************************************************************/ +void GZFAM::Rewind(void) + { + gzrewind(Zfile); + } // end of Rewind + +/* ------------------------------------------------------------------- */ + +/***********************************************************************/ +/* Constructors. */ +/***********************************************************************/ +ZBKFAM::ZBKFAM(PDOSDEF tdp) : GZFAM(tdp) + { + Blocked = true; + Block = tdp->GetBlock(); + Last = tdp->GetLast(); + Nrec = tdp->GetElemt(); + CurLine = NULL; + NxtLine = NULL; + Closing = false; + BlkPos = tdp->GetTo_Pos(); + } // end of ZBKFAM standard constructor + +ZBKFAM::ZBKFAM(PZBKFAM txfp) : GZFAM(txfp) + { + CurLine = txfp->CurLine; + NxtLine = txfp->NxtLine; + Closing = txfp->Closing; + } // end of ZBKFAM copy constructor + +/***********************************************************************/ +/* Use BlockTest to reduce the table estimated size. */ +/***********************************************************************/ +int ZBKFAM::MaxBlkSize(PGLOBAL g, int) + { + int rc = RC_OK, savcur = CurBlk; + int size; + + // Roughly estimate the table size as the sum of blocks + // that can contain good rows + for (size = 0, CurBlk = 0; CurBlk < Block; CurBlk++) + if ((rc = Tdbp->TestBlock(g)) == RC_OK) + size += (CurBlk == Block - 1) ? Last : Nrec; + else if (rc == RC_EF) + break; + + CurBlk = savcur; + return size; + } // end of MaxBlkSize + +/***********************************************************************/ +/* ZBK Cardinality: returns table cardinality in number of rows. */ +/* This function can be called with a null argument to test the */ +/* availability of Cardinality implementation (1 yes, 0 no). */ +/***********************************************************************/ +int ZBKFAM::Cardinality(PGLOBAL g) + { + return (g) ? (int)((Block - 1) * Nrec + Last) : 1; + } // end of Cardinality + +/***********************************************************************/ +/* Allocate the line buffer. For mode Delete a bigger buffer has to */ +/* be allocated because is it also used to move lines into the file. */ +/***********************************************************************/ +bool ZBKFAM::AllocateBuffer(PGLOBAL g) + { + Buflen = Nrec * (Lrecl + 2); + CurLine = To_Buf = (char*)PlugSubAlloc(g, NULL, Buflen); + + if (Tdbp->GetMode() == MODE_INSERT) { + // Set values so Block and Last can be recalculated + if (Last == Nrec) { + CurBlk = Block; + Rbuf = Nrec; // To be used by WriteDB + } else { + // The last block must be completed + CurBlk = Block - 1; + Rbuf = Nrec - Last; // To be used by WriteDB + } // endif Last + + } // endif Insert + + return false; + } // end of AllocateBuffer + +/***********************************************************************/ +/* GetRowID: return the RowID of last read record. */ +/***********************************************************************/ +int ZBKFAM::GetRowID(void) + { + return CurNum + Nrec * CurBlk + 1; + } // end of GetRowID + +/***********************************************************************/ +/* GetPos: return the position of last read record. */ +/***********************************************************************/ +int ZBKFAM::GetPos(void) + { + return CurNum + Nrec * CurBlk; // Computed file index + } // end of GetPos + +/***********************************************************************/ +/* Record file position in case of UPDATE or DELETE. */ +/* Not used yet for fixed tables. */ +/***********************************************************************/ +bool ZBKFAM::RecordPos(PGLOBAL /*g*/) + { +//strcpy(g->Message, "RecordPos not implemented for gz blocked tables"); +//return true; + return RC_OK; + } // end of RecordPos + +/***********************************************************************/ +/* Skip one record in file. */ +/***********************************************************************/ +int ZBKFAM::SkipRecord(PGLOBAL /*g*/, bool) + { +//strcpy(g->Message, "SkipRecord not implemented for gz blocked tables"); +//return RC_FX; + return RC_OK; + } // end of SkipRecord + +/***********************************************************************/ +/* ReadBuffer: Read one line from a compressed text file. */ +/***********************************************************************/ +int ZBKFAM::ReadBuffer(PGLOBAL g) + { + int n, skip, rc = RC_OK; + + /*********************************************************************/ + /* Sequential reading when Placed is not true. */ + /*********************************************************************/ + if (++CurNum < Rbuf) { + CurLine = NxtLine; + + // Get the position of the next line in the buffer + while (*NxtLine++ != '\n') ; + + // Set caller line buffer + n = NxtLine - CurLine - Ending; + memcpy(Tdbp->GetLine(), CurLine, n); + Tdbp->GetLine()[n] = '\0'; + return RC_OK; + } else if (Rbuf < Nrec && CurBlk != -1) + return RC_EF; + + /*********************************************************************/ + /* New block. */ + /*********************************************************************/ + CurNum = 0; + skip = 0; + + next: + if (++CurBlk >= Block) + return RC_EF; + + /*********************************************************************/ + /* Before using the new block, check whether block optimization */ + /* can be done, as well as for join as for local filtering. */ + /*********************************************************************/ + switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + skip++; + goto next; + } // endswitch rc + + if (skip) + // Skip blocks rejected by block optimization + for (int i = CurBlk - skip; i < CurBlk; i++) { + BlkLen = BlkPos[i + 1] - BlkPos[i]; + + if (gzseek(Zfile, (z_off_t)BlkLen, SEEK_CUR) < 0) + return Zerror(g); + + } // endfor i + + BlkLen = BlkPos[CurBlk + 1] - BlkPos[CurBlk]; + + if (!(n = gzread(Zfile, To_Buf, BlkLen))) { + rc = RC_EF; + } else if (n > 0) { + // Get the position of the current line + CurLine = To_Buf; + + // Now get the position of the next line + for (NxtLine = CurLine; *NxtLine++ != '\n';) ; + + // Set caller line buffer + n = NxtLine - CurLine - Ending; + memcpy(Tdbp->GetLine(), CurLine, n); + Tdbp->GetLine()[n] = '\0'; + Rbuf = (CurBlk == Block - 1) ? Last : Nrec; + IsRead = true; + rc = RC_OK; + num_read++; + } else + rc = Zerror(g); + + return rc; + } // end of ReadBuffer + +/***********************************************************************/ +/* WriteDB: Data Base write routine for ZDOS access method. */ +/* Update is not possible without using a temporary file (NIY). */ +/***********************************************************************/ +int ZBKFAM::WriteBuffer(PGLOBAL g) + { + /*********************************************************************/ + /* Prepare the write buffer. */ + /*********************************************************************/ + if (!Closing) + strcat(strcpy(CurLine, Tdbp->GetLine()), CrLf); + + /*********************************************************************/ + /* In Insert mode, blocs are added sequentialy to the file end. */ + /* Note: Update mode is not handled for gz files. */ + /*********************************************************************/ + if (++CurNum == Rbuf) { + /*******************************************************************/ + /* New block, start the writing process. */ + /*******************************************************************/ + BlkLen = CurLine + strlen(CurLine) - To_Buf; + + if (gzwrite(Zfile, To_Buf, BlkLen) != BlkLen || + gzflush(Zfile, Z_FULL_FLUSH)) { + Closing = true; + return Zerror(g); + } // endif gzwrite + + Rbuf = Nrec; + CurBlk++; + CurNum = 0; + CurLine = To_Buf; + } else + CurLine += strlen(CurLine); + + return RC_OK; + } // end of WriteBuffer + +/***********************************************************************/ +/* Data Base delete line routine for ZBK access method. */ +/* Implemented only for total deletion of the table, which is done */ +/* by opening the file in mode "wb". */ +/***********************************************************************/ +int ZBKFAM::DeleteRecords(PGLOBAL g, int irc) + { + if (irc == RC_EF) { + LPCSTR name = Tdbp->GetName(); + PDOSDEF defp = (PDOSDEF)Tdbp->GetDef(); + + defp->SetBlock(0); + defp->SetLast(Nrec); + + if (!defp->SetIntCatInfo("Blocks", 0) || + !defp->SetIntCatInfo("Last", 0)) { + sprintf(g->Message, MSG(UPDATE_ERROR), "Header"); + return RC_FX; + } else + return RC_OK; + + } else + return irc; + + } // end of DeleteRecords + +/***********************************************************************/ +/* Data Base close routine for ZBK access method. */ +/***********************************************************************/ +void ZBKFAM::CloseTableFile(PGLOBAL g, bool) + { + int rc = RC_OK; + + if (Tdbp->GetMode() == MODE_INSERT) { + LPCSTR name = Tdbp->GetName(); + PDOSDEF defp = (PDOSDEF)Tdbp->GetDef(); + + if (CurNum && !Closing) { + // Some more inserted lines remain to be written + Last = (Nrec - Rbuf) + CurNum; + Block = CurBlk + 1; + Rbuf = CurNum--; + Closing = true; + rc = WriteBuffer(g); + } else if (Rbuf == Nrec) { + Last = Nrec; + Block = CurBlk; + } // endif CurNum + + if (rc != RC_FX) { + defp->SetBlock(Block); + defp->SetLast(Last); + defp->SetIntCatInfo("Blocks", Block); + defp->SetIntCatInfo("Last", Last); + } // endif + + gzclose(Zfile); + } else if (Tdbp->GetMode() == MODE_DELETE) { + rc = DeleteRecords(g, RC_EF); + gzclose(Zfile); + } else + rc = gzclose(Zfile); + + if (trace) + htrc("GZ CloseDB: closing %s rc=%d\n", To_File, rc); + + Zfile = NULL; // So we can know whether table is open +//To_Fb->Count = 0; // Avoid double closing by PlugCloseAll + } // end of CloseTableFile + +/***********************************************************************/ +/* Rewind routine for ZBK access method. */ +/***********************************************************************/ +void ZBKFAM::Rewind(void) + { + gzrewind(Zfile); + CurBlk = -1; + CurNum = Rbuf; + } // end of Rewind + +/* ------------------------------------------------------------------- */ + +/***********************************************************************/ +/* Constructors. */ +/***********************************************************************/ +ZIXFAM::ZIXFAM(PDOSDEF tdp) : ZBKFAM(tdp) + { +//Block = tdp->GetBlock(); +//Last = tdp->GetLast(); + Nrec = (tdp->GetElemt()) ? tdp->GetElemt() : DOS_BUFF_LEN; + Blksize = Nrec * Lrecl; + } // end of ZIXFAM standard constructor + +/***********************************************************************/ +/* ZIX Cardinality: returns table cardinality in number of rows. */ +/* This function can be called with a null argument to test the */ +/* availability of Cardinality implementation (1 yes, 0 no). */ +/***********************************************************************/ +int ZIXFAM::Cardinality(PGLOBAL g) + { + if (Last) + return (g) ? (int)((Block - 1) * Nrec + Last) : 1; + else // Last and Block not defined, cannot do it yet + return 0; + + } // end of Cardinality + +/***********************************************************************/ +/* Allocate the line buffer. For mode Delete a bigger buffer has to */ +/* be allocated because is it also used to move lines into the file. */ +/***********************************************************************/ +bool ZIXFAM::AllocateBuffer(PGLOBAL g) + { + Buflen = Blksize; + To_Buf = (char*)PlugSubAlloc(g, NULL, Buflen); + + if (Tdbp->GetMode() == MODE_INSERT) { + /*******************************************************************/ + /* For Insert the buffer must be prepared. */ + /*******************************************************************/ + memset(To_Buf, ' ', Buflen); + + if (Tdbp->GetFtype() < 2) + // if not binary, the file is physically a text file + for (int len = Lrecl; len <= Buflen; len += Lrecl) { +#if defined(__WIN__) + To_Buf[len - 2] = '\r'; +#endif // __WIN__ + To_Buf[len - 1] = '\n'; + } // endfor len + + // Set values so Block and Last can be recalculated + if (Last == Nrec) { + CurBlk = Block; + Rbuf = Nrec; // To be used by WriteDB + } else { + // The last block must be completed + CurBlk = Block - 1; + Rbuf = Nrec - Last; // To be used by WriteDB + } // endif Last + + } // endif Insert + + return false; + } // end of AllocateBuffer + +/***********************************************************************/ +/* ReadBuffer: Read one line from a compressed text file. */ +/***********************************************************************/ +int ZIXFAM::ReadBuffer(PGLOBAL g) + { + int n, rc = RC_OK; + + /*********************************************************************/ + /* Sequential reading when Placed is not true. */ + /*********************************************************************/ + if (++CurNum < Rbuf) { + Tdbp->IncLine(Lrecl); // Used by DOSCOL functions + return RC_OK; + } else if (Rbuf < Nrec && CurBlk != -1) + return RC_EF; + + /*********************************************************************/ + /* New block. */ + /*********************************************************************/ + CurNum = 0; + Tdbp->SetLine(To_Buf); + + int skip = 0; + + next: + if (++CurBlk >= Block) + return RC_EF; + + /*********************************************************************/ + /* Before using the new block, check whether block optimization */ + /* can be done, as well as for join as for local filtering. */ + /*********************************************************************/ + switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + skip++; + goto next; + } // endswitch rc + + if (skip) + // Skip blocks rejected by block optimization + for (int i = 0; i < skip; i++) { + if (gzseek(Zfile, (z_off_t)Buflen, SEEK_CUR) < 0) + return Zerror(g); + + } // endfor i + + if (!(n = gzread(Zfile, To_Buf, Buflen))) { + rc = RC_EF; + } else if (n > 0) { + Rbuf = n / Lrecl; + IsRead = true; + rc = RC_OK; + num_read++; + } else + rc = Zerror(g); + + return rc; + } // end of ReadBuffer + +/***********************************************************************/ +/* WriteDB: Data Base write routine for ZDOS access method. */ +/* Update is not possible without using a temporary file (NIY). */ +/***********************************************************************/ +int ZIXFAM::WriteBuffer(PGLOBAL g) + { + /*********************************************************************/ + /* In Insert mode, blocs are added sequentialy to the file end. */ + /* Note: Update mode is not handled for gz files. */ + /*********************************************************************/ + if (++CurNum == Rbuf) { + /*******************************************************************/ + /* New block, start the writing process. */ + /*******************************************************************/ + BlkLen = Rbuf * Lrecl; + + if (gzwrite(Zfile, To_Buf, BlkLen) != BlkLen || + gzflush(Zfile, Z_FULL_FLUSH)) { + Closing = true; + return Zerror(g); + } // endif gzwrite + + Rbuf = Nrec; + CurBlk++; + CurNum = 0; + Tdbp->SetLine(To_Buf); + } else + Tdbp->IncLine(Lrecl); // Used by FIXCOL functions + + return RC_OK; + } // end of WriteBuffer + +/* --------------------------- Class ZLBFAM -------------------------- */ + +/***********************************************************************/ +/* Constructors. */ +/***********************************************************************/ +ZLBFAM::ZLBFAM(PDOSDEF tdp) : BLKFAM(tdp) + { + Zstream = NULL; + Zbuffer = NULL; + Zlenp = NULL; + Optimized = tdp->IsOptimized(); + } // end of ZLBFAM standard constructor + +ZLBFAM::ZLBFAM(PZLBFAM txfp) : BLKFAM(txfp) + { + Zstream = txfp->Zstream; + Zbuffer = txfp->Zbuffer; + Zlenp = txfp->Zlenp; + Optimized = txfp->Optimized; + } // end of ZLBFAM (dummy?) copy constructor + +/***********************************************************************/ +/* ZLB GetFileLength: returns an estimate of what would be the */ +/* uncompressed file size in number of bytes. */ +/***********************************************************************/ +int ZLBFAM::GetFileLength(PGLOBAL g) + { + int len = (Optimized) ? BlkPos[Block] : BLKFAM::GetFileLength(g); + + if (len > 0) + // Estimate size reduction to a max of 5 + len *= 5; + + return len; + } // end of GetFileLength + +/***********************************************************************/ +/* Allocate the line buffer. For mode Delete a bigger buffer has to */ +/* be allocated because is it also used to move lines into the file. */ +/***********************************************************************/ +bool ZLBFAM::AllocateBuffer(PGLOBAL g) + { + char *msg; + int n, zrc; + +#if 0 + if (!Optimized && Tdbp->NeedIndexing(g)) { + strcpy(g->Message, MSG(NOP_ZLIB_INDEX)); + return TRUE; + } // endif indexing +#endif // 0 + +#if defined(NOLIB) + if (!zlib && LoadZlib()) { + sprintf(g->Message, MSG(DLL_LOAD_ERROR), GetLastError(), "zlib.dll"); + return TRUE; + } // endif zlib +#endif + + BLKFAM::AllocateBuffer(g); +//Buflen = Nrec * (Lrecl + 2); +//Rbuf = Nrec; + + // Allocate the compressed buffer + n = Buflen + 16; // ????????????????????????????????? + Zlenp = (int*)PlugSubAlloc(g, NULL, n); + Zbuffer = (Byte*)(Zlenp + 1); + + // Allocate and initialize the Z stream + Zstream = (z_streamp)PlugSubAlloc(g, NULL, sizeof(z_stream)); + Zstream->zalloc = (alloc_func)0; + Zstream->zfree = (free_func)0; + Zstream->opaque = (voidpf)0; + Zstream->next_in = NULL; + Zstream->avail_in = 0; + + if (Tdbp->GetMode() == MODE_READ) { + msg = "inflateInit"; + zrc = inflateInit(Zstream); + } else { + msg = "deflateInit"; + zrc = deflateInit(Zstream, Z_DEFAULT_COMPRESSION); + } // endif Mode + + if (zrc != Z_OK) { + if (Zstream->msg) + sprintf(g->Message, "%s error: %s", msg, Zstream->msg); + else + sprintf(g->Message, "%s error: %d", msg, zrc); + + return TRUE; + } // endif zrc + + if (Tdbp->GetMode() == MODE_INSERT) { + // Write the file header block + if (Last == Nrec) { + CurBlk = Block; + CurNum = 0; + + if (!GetFileLength(g)) { + // Write the zlib header as an extra block + strcpy(To_Buf, "PlugDB"); + BlkLen = strlen("PlugDB") + 1; + + if (WriteCompressedBuffer(g)) + return TRUE; + + } // endif void file + + } else { + // In mode insert, if Last != Nrec, last block must be updated + CurBlk = Block - 1; + CurNum = Last; + + strcpy(g->Message, MSG(NO_PAR_BLK_INS)); + return TRUE; + } // endif Last + + } else { // MODE_READ + // First thing to do is to read the header block + void *rdbuf; + + if (Optimized) { + BlkLen = BlkPos[0]; + rdbuf = Zlenp; + } else { + // Get the stored length from the file itself + if (fread(Zlenp, sizeof(int), 1, Stream) != 1) + return FALSE; // Empty file + + BlkLen = *Zlenp; + rdbuf = Zbuffer; + } // endif Optimized + + switch (ReadCompressedBuffer(g, rdbuf)) { + case RC_EF: + return FALSE; + case RC_FX: +#if defined(UNIX) + sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(errno)); +#else + sprintf(g->Message, MSG(READ_ERROR), To_File, _strerror(NULL)); +#endif + case RC_NF: + return TRUE; + } // endswitch + + // Some old tables can have PlugDB in their header + if (strcmp(To_Buf, "PlugDB")) { + sprintf(g->Message, MSG(BAD_HEADER), Tdbp->GetFile(g)); + return TRUE; + } // endif strcmp + + } // endif Mode + + return FALSE; + } // end of AllocateBuffer + +/***********************************************************************/ +/* GetPos: return the position of last read record. */ +/***********************************************************************/ +int ZLBFAM::GetPos(void) + { + return (Optimized) ? (CurNum + Nrec * CurBlk) : Fpos; + } // end of GetPos + +/***********************************************************************/ +/* GetNextPos: should not be called for this class. */ +/***********************************************************************/ +int ZLBFAM::GetNextPos(void) + { + if (Optimized) { + assert(FALSE); + return 0; + } else + return ftell(Stream); + + } // end of GetNextPos + +/***********************************************************************/ +/* SetPos: Replace the table at the specified position. */ +/***********************************************************************/ +bool ZLBFAM::SetPos(PGLOBAL g, int pos __attribute__((unused))) + { + sprintf(g->Message, MSG(NO_SETPOS_YET), "GZ"); + return true; +#if 0 // All this must be checked + if (pos < 0) { + strcpy(g->Message, MSG(INV_REC_POS)); + return true; + } // endif recpos + + CurBlk = pos / Nrec; + CurNum = pos % Nrec; +#if defined(_DEBUG) + num_eq[(CurBlk == OldBlk) ? 1 : 0]++; +#endif + + // Indicate the table position was externally set + Placed = true; + return false; +#endif // 0 + } // end of SetPos + +/***********************************************************************/ +/* ReadBuffer: Read one line for a text file. */ +/***********************************************************************/ +int ZLBFAM::ReadBuffer(PGLOBAL g) + { + int n; + void *rdbuf; + + /*********************************************************************/ + /* Sequential reading when Placed is not true. */ + /*********************************************************************/ + if (Placed) { + Placed = FALSE; + } else if (++CurNum < Rbuf) { + CurLine = NxtLine; + + // Get the position of the next line in the buffer + if (Tdbp->GetFtype() == RECFM_VAR) + while (*NxtLine++ != '\n') ; + else + NxtLine += Lrecl; + + // Set caller line buffer + n = NxtLine - CurLine - ((Tdbp->GetFtype() == RECFM_BIN) ? 0 : Ending); + memcpy(Tdbp->GetLine(), CurLine, n); + Tdbp->GetLine()[n] = '\0'; + return RC_OK; + } else if (Rbuf < Nrec && CurBlk != -1) { + CurNum--; // To have a correct Last value when optimizing + return RC_EF; + } else { + /*******************************************************************/ + /* New block. */ + /*******************************************************************/ + CurNum = 0; + + next: + if (++CurBlk >= Block) + return RC_EF; + + /*******************************************************************/ + /* Before reading a new block, check whether block optimization */ + /* can be done, as well as for join as for local filtering. */ + /*******************************************************************/ + if (Optimized) switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + goto next; + } // endswitch rc + + } // endif's + + if (OldBlk == CurBlk) + goto ok; // Block is already there + + if (Optimized) { + // Store the position of next block + Fpos = BlkPos[CurBlk]; + + // fseek is required only in non sequential reading + if (CurBlk != OldBlk + 1) + if (fseek(Stream, Fpos, SEEK_SET)) { + sprintf(g->Message, MSG(FSETPOS_ERROR), Fpos); + return RC_FX; + } // endif fseek + + // Calculate the length of block to read + BlkLen = BlkPos[CurBlk + 1] - Fpos; + rdbuf = Zlenp; + } else { // !Optimized + if (CurBlk != OldBlk + 1) { + strcpy(g->Message, MSG(INV_RAND_ACC)); + return RC_FX; + } else + Fpos = ftell(Stream); // Used when optimizing + + // Get the stored length from the file itself + if (fread(Zlenp, sizeof(int), 1, Stream) != 1) { + if (feof(Stream)) + return RC_EF; + + goto err; + } // endif fread + + BlkLen = *Zlenp; + rdbuf = Zbuffer; + } // endif Optimized + + // Read the next block + switch (ReadCompressedBuffer(g, rdbuf)) { + case RC_FX: goto err; + case RC_NF: return RC_FX; + case RC_EF: return RC_EF; + default: Rbuf = (CurBlk == Block - 1) ? Last : Nrec; + } // endswitch ReadCompressedBuffer + + ok: + if (Tdbp->GetFtype() == RECFM_VAR) { + int i; + + // Get the position of the current line + for (i = 0, CurLine = To_Buf; i < CurNum; i++) + while (*CurLine++ != '\n') ; // What about Unix ??? + + // Now get the position of the next line + for (NxtLine = CurLine; *NxtLine++ != '\n';) ; + + // Set caller line buffer + n = NxtLine - CurLine - Ending; + } else { + CurLine = To_Buf + CurNum * Lrecl; + NxtLine = CurLine + Lrecl; + n = Lrecl - ((Tdbp->GetFtype() == RECFM_BIN) ? 0 : Ending); + } // endif Ftype + + memcpy(Tdbp->GetLine(), CurLine, n); + Tdbp->GetLine()[n] = '\0'; + + OldBlk = CurBlk; // Last block actually read + IsRead = TRUE; // Is read indeed + return RC_OK; + + err: +#if defined(UNIX) + sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(errno)); +#else + sprintf(g->Message, MSG(READ_ERROR), To_File, _strerror(NULL)); +#endif + return RC_FX; + } // end of ReadBuffer + +/***********************************************************************/ +/* Read and decompress a block from the stream. */ +/***********************************************************************/ +int ZLBFAM::ReadCompressedBuffer(PGLOBAL g, void *rdbuf) + { + if (fread(rdbuf, 1, (size_t)BlkLen, Stream) == (unsigned)BlkLen) { + int zrc; + + num_read++; + + if (Optimized && BlkLen != signed(*Zlenp + sizeof(int))) { + sprintf(g->Message, MSG(BAD_BLK_SIZE), CurBlk + 1); + return RC_NF; + } // endif BlkLen + + // HERE WE MUST INFLATE THE BLOCK + Zstream->next_in = Zbuffer; + Zstream->avail_in = (uInt)(*Zlenp); + Zstream->next_out = (Byte*)To_Buf; + Zstream->avail_out = Buflen; + zrc = inflate(Zstream, Z_SYNC_FLUSH); + + if (zrc != Z_OK) { + if (Zstream->msg) + sprintf(g->Message, MSG(FUNC_ERR_S), "inflate", Zstream->msg); + else + sprintf(g->Message, MSG(FUNCTION_ERROR), "inflate", (int)zrc); + + return RC_NF; + } // endif zrc + + } else if (feof(Stream)) { + return RC_EF; + } else + return RC_FX; + + return RC_OK; + } // end of ReadCompressedBuffer + +/***********************************************************************/ +/* WriteBuffer: File write routine for DOS access method. */ +/* Update is directly written back into the file, */ +/* with this (fast) method, record size cannot change. */ +/***********************************************************************/ +int ZLBFAM::WriteBuffer(PGLOBAL g) + { + assert (Tdbp->GetMode() == MODE_INSERT); + + /*********************************************************************/ + /* Prepare the write buffer. */ + /*********************************************************************/ + if (!Closing) { + if (Tdbp->GetFtype() == RECFM_BIN) + memcpy(CurLine, Tdbp->GetLine(), Lrecl); + else + strcat(strcpy(CurLine, Tdbp->GetLine()), CrLf); + +#if defined(_DEBUG) + if (Tdbp->GetFtype() == RECFM_FIX && + (signed)strlen(CurLine) != Lrecl + (signed)strlen(CrLf)) { + strcpy(g->Message, MSG(BAD_LINE_LEN)); + Closing = TRUE; + return RC_FX; + } // endif Lrecl +#endif // _DEBUG + } // endif Closing + + /*********************************************************************/ + /* In Insert mode, blocs are added sequentialy to the file end. */ + /*********************************************************************/ + if (++CurNum != Rbuf) { + if (Tdbp->GetFtype() == RECFM_VAR) + CurLine += strlen(CurLine); + else + CurLine += Lrecl; + + return RC_OK; // We write only full blocks + } // endif CurNum + + // HERE WE MUST DEFLATE THE BLOCK + if (Tdbp->GetFtype() == RECFM_VAR) + NxtLine = CurLine + strlen(CurLine); + else + NxtLine = CurLine + Lrecl; + + BlkLen = NxtLine - To_Buf; + + if (WriteCompressedBuffer(g)) { + Closing = TRUE; // To tell CloseDB about a Write error + return RC_FX; + } // endif WriteCompressedBuffer + + CurBlk++; + CurNum = 0; + CurLine = To_Buf; + return RC_OK; + } // end of WriteBuffer + +/***********************************************************************/ +/* Compress the buffer and write the deflated output to stream. */ +/***********************************************************************/ +bool ZLBFAM::WriteCompressedBuffer(PGLOBAL g) + { + int zrc; + + Zstream->next_in = (Byte*)To_Buf; + Zstream->avail_in = (uInt)BlkLen; + Zstream->next_out = Zbuffer; + Zstream->avail_out = Buflen + 16; + Zstream->total_out = 0; + zrc = deflate(Zstream, Z_FULL_FLUSH); + + if (zrc != Z_OK) { + if (Zstream->msg) + sprintf(g->Message, MSG(FUNC_ERR_S), "deflate", Zstream->msg); + else + sprintf(g->Message, MSG(FUNCTION_ERROR), "deflate", (int)zrc); + + return TRUE; + } else + *Zlenp = Zstream->total_out; + + // Now start the writing process. + BlkLen = *Zlenp + sizeof(int); + + if (fwrite(Zlenp, 1, BlkLen, Stream) != (size_t)BlkLen) { + sprintf(g->Message, MSG(FWRITE_ERROR), strerror(errno)); + return TRUE; + } // endif size + + return FALSE; + } // end of WriteCompressedBuffer + +/***********************************************************************/ +/* Table file close routine for DOS access method. */ +/***********************************************************************/ +void ZLBFAM::CloseTableFile(PGLOBAL g, bool) + { + int rc = RC_OK; + + if (Tdbp->GetMode() == MODE_INSERT) { + LPCSTR name = Tdbp->GetName(); + PDOSDEF defp = (PDOSDEF)Tdbp->GetDef(); + + // Closing is True if last Write was in error + if (CurNum && !Closing) { + // Some more inserted lines remain to be written + Last = (Nrec - Rbuf) + CurNum; + Block = CurBlk + 1; + Rbuf = CurNum--; + Closing = TRUE; + rc = WriteBuffer(g); + } else if (Rbuf == Nrec) { + Last = Nrec; + Block = CurBlk; + } // endif CurNum + + if (rc != RC_FX) { + defp->SetBlock(Block); + defp->SetLast(Last); + defp->SetIntCatInfo("Blocks", Block); + defp->SetIntCatInfo("Last", Last); + } // endif + + fclose(Stream); + } else + rc = fclose(Stream); + + if (trace) + htrc("ZLB CloseTableFile: closing %s mode=%d rc=%d\n", + To_File, Tdbp->GetMode(), rc); + + Stream = NULL; // So we can know whether table is open + To_Fb->Count = 0; // Avoid double closing by PlugCloseAll + + if (Tdbp->GetMode() == MODE_READ) + rc = inflateEnd(Zstream); + else + rc = deflateEnd(Zstream); + + } // end of CloseTableFile + +/***********************************************************************/ +/* Rewind routine for ZLIB access method. */ +/***********************************************************************/ +void ZLBFAM::Rewind(void) + { + // We must be positioned after the header block + if (CurBlk >= 0) { // Nothing to do if no block read yet + if (!Optimized) { // If optimized, fseek will be done in ReadBuffer + size_t st; + + rewind(Stream); + + if (!(st = fread(Zlenp, sizeof(int), 1, Stream)) && trace) + htrc("fread error %d in Rewind", errno); + + fseek(Stream, *Zlenp + sizeof(int), SEEK_SET); + OldBlk = -1; + } // endif Optimized + + CurBlk = -1; + CurNum = Rbuf; + } // endif CurBlk + +//OldBlk = -1; +//Rbuf = 0; commented out in case we reuse last read block + } // end of Rewind + +/* ------------------------ End of GzFam ---------------------------- */ diff --git a/storage/connect/filamgz.h b/storage/connect/filamgz.h new file mode 100644 index 00000000000..d667fdddcc2 --- /dev/null +++ b/storage/connect/filamgz.h @@ -0,0 +1,170 @@ +/*************** FilAmGz H Declares Source Code File (.H) **************/ +/* Name: FILAMGZ.H Version 1.3 */ +/* */ +/* (C) Copyright to the author Olivier BERTRAND 2005-2016 */ +/* */ +/* This file contains the GZIP access method classes declares. */ +/***********************************************************************/ +#ifndef __FILAMGZ_H +#define __FILAMGZ_H + +#include "zlib.h" + +typedef class GZFAM *PGZFAM; +typedef class ZBKFAM *PZBKFAM; +typedef class ZIXFAM *PZIXFAM; +typedef class ZLBFAM *PZLBFAM; + +/***********************************************************************/ +/* This is the access method class declaration for not optimized */ +/* variable record length files compressed using the gzip library */ +/* functions. File is accessed record by record (row). */ +/***********************************************************************/ +class DllExport GZFAM : public TXTFAM { +// friend class DOSCOL; + public: + // Constructor + GZFAM(PDOSDEF tdp) : TXTFAM(tdp) {Zfile = NULL; Zpos = 0;} + GZFAM(PGZFAM txfp); + + // Implementation + virtual AMT GetAmType(void) {return TYPE_AM_GZ;} + virtual int GetPos(void); + virtual int GetNextPos(void); + virtual PTXF Duplicate(PGLOBAL g) + {return (PTXF)new(g) GZFAM(this);} + + // Methods + virtual void Reset(void); + virtual int GetFileLength(PGLOBAL g); + virtual int Cardinality(PGLOBAL g) {return (g) ? -1 : 0;} + virtual int MaxBlkSize(PGLOBAL g, int s) {return s;} + virtual bool AllocateBuffer(PGLOBAL g); + virtual int GetRowID(void); + virtual bool RecordPos(PGLOBAL g); + virtual bool SetPos(PGLOBAL g, int recpos); + virtual int SkipRecord(PGLOBAL g, bool header); + virtual bool OpenTableFile(PGLOBAL g); + virtual int ReadBuffer(PGLOBAL g); + virtual int WriteBuffer(PGLOBAL g); + virtual int DeleteRecords(PGLOBAL g, int irc); + virtual void CloseTableFile(PGLOBAL g, bool abort); + virtual void Rewind(void); + + protected: + int Zerror(PGLOBAL g); // GZ error function + + // Members + gzFile Zfile; // Points to GZ file structure + z_off_t Zpos; // Uncompressed file position + }; // end of class GZFAM + +/***********************************************************************/ +/* This is the access method class declaration for optimized variable */ +/* record length files compressed using the gzip library functions. */ +/* The File is accessed by block (requires an opt file). */ +/***********************************************************************/ +class DllExport ZBKFAM : public GZFAM { + public: + // Constructor + ZBKFAM(PDOSDEF tdp); + ZBKFAM(PZBKFAM txfp); + + // Implementation + virtual int GetPos(void); + virtual int GetNextPos(void) {return 0;} + virtual PTXF Duplicate(PGLOBAL g) + {return (PTXF)new(g) ZBKFAM(this);} + + // Methods + virtual int Cardinality(PGLOBAL g); + virtual int MaxBlkSize(PGLOBAL g, int s); + virtual bool AllocateBuffer(PGLOBAL g); + virtual int GetRowID(void); + virtual bool RecordPos(PGLOBAL g); + virtual int SkipRecord(PGLOBAL g, bool header); + virtual int ReadBuffer(PGLOBAL g); + virtual int WriteBuffer(PGLOBAL g); + virtual int DeleteRecords(PGLOBAL g, int irc); + virtual void CloseTableFile(PGLOBAL g, bool abort); + virtual void Rewind(void); + + protected: + // Members + char *CurLine; // Position of current line in buffer + char *NxtLine; // Position of Next line in buffer + bool Closing; // True when closing on Insert + }; // end of class ZBKFAM + +/***********************************************************************/ +/* This is the access method class declaration for fixed record */ +/* length files compressed using the gzip library functions. */ +/* The file is always accessed by block. */ +/***********************************************************************/ +class DllExport ZIXFAM : public ZBKFAM { + public: + // Constructor + ZIXFAM(PDOSDEF tdp); + ZIXFAM(PZIXFAM txfp) : ZBKFAM(txfp) {} + + // Implementation + virtual int GetNextPos(void) {return 0;} + virtual PTXF Duplicate(PGLOBAL g) + {return (PTXF)new(g) ZIXFAM(this);} + + // Methods + virtual int Cardinality(PGLOBAL g); + virtual bool AllocateBuffer(PGLOBAL g); + virtual int ReadBuffer(PGLOBAL g); + virtual int WriteBuffer(PGLOBAL g); + + protected: + // No additional Members + }; // end of class ZIXFAM + +/***********************************************************************/ +/* This is the DOS/UNIX Access Method class declaration for PlugDB */ +/* fixed/variable files compressed using the zlib library functions. */ +/* Physically these are written and read using the same technique */ +/* than blocked variable files, only the contain of each block is */ +/* compressed using the deflate zlib function. The purpose of this */ +/* specific format is to have a fast mechanism for direct access of */ +/* records so blocked optimization is fast and direct access (joins) */ +/* is allowed. Note that the block length is written ahead of each */ +/* block to enable reading when optimization file is not available. */ +/***********************************************************************/ +class DllExport ZLBFAM : public BLKFAM { + public: + // Constructor + ZLBFAM(PDOSDEF tdp); + ZLBFAM(PZLBFAM txfp); + + // Implementation + virtual AMT GetAmType(void) {return TYPE_AM_ZLIB;} + virtual int GetPos(void); + virtual int GetNextPos(void); + virtual PTXF Duplicate(PGLOBAL g) + {return (PTXF)new(g) ZLBFAM(this);} + inline void SetOptimized(bool b) {Optimized = b;} + + // Methods + virtual int GetFileLength(PGLOBAL g); + virtual bool SetPos(PGLOBAL g, int recpos); + virtual bool AllocateBuffer(PGLOBAL g); + virtual int ReadBuffer(PGLOBAL g); + virtual int WriteBuffer(PGLOBAL g); + virtual void CloseTableFile(PGLOBAL g, bool abort); + virtual void Rewind(void); + + protected: + bool WriteCompressedBuffer(PGLOBAL g); + int ReadCompressedBuffer(PGLOBAL g, void *rdbuf); + + // Members + z_streamp Zstream; // Compression/decompression stream + Byte *Zbuffer; // Compressed block buffer + int *Zlenp; // Pointer to block length + bool Optimized; // true when opt file is available + }; // end of class ZLBFAM + +#endif // __FILAMGZ_H diff --git a/storage/connect/filamzip.cpp b/storage/connect/filamzip.cpp index d9834e56dcd..65013e170e4 100644 --- a/storage/connect/filamzip.cpp +++ b/storage/connect/filamzip.cpp @@ -1,35 +1,29 @@ /*********** File AM Zip C++ Program Source Code File (.CPP) ***********/ /* PROGRAM NAME: FILAMZIP */ /* ------------- */ -/* Version 1.5 */ +/* Version 1.0 */ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 2005-2015 */ +/* (C) Copyright to the author Olivier BERTRAND 2016 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ -/* This program are the ZLIB compressed files classes. */ +/* This program are the ZIP file access method classes. */ /* */ /***********************************************************************/ /***********************************************************************/ -/* Include relevant MariaDB header file. */ +/* Include relevant sections of the System header files. */ /***********************************************************************/ #include "my_global.h" -#if defined(__WIN__) -#include <io.h> -#include <fcntl.h> -#if defined(__BORLANDC__) -#define __MFC_COMPAT__ // To define min/max as macro -#endif -//#include <windows.h> -#else // !__WIN__ +#if !defined(__WIN__) #if defined(UNIX) #include <errno.h> -#else // !UNIX +#include <unistd.h> +#else // !UNIX #include <io.h> -#endif +#endif // !UNIX #include <fcntl.h> #endif // !__WIN__ @@ -37,1390 +31,592 @@ /* Include application header files: */ /* global.h is header containing all global declarations. */ /* plgdbsem.h is header containing the DB application declarations. */ -/* tabdos.h is header containing the TABDOS class declarations. */ /***********************************************************************/ #include "global.h" #include "plgdbsem.h" -//#include "catalog.h" -//#include "reldef.h" -//#include "xobject.h" -//#include "kindex.h" -#include "filamtxt.h" -#include "tabdos.h" -#if defined(UNIX) #include "osutil.h" -#endif - -/***********************************************************************/ -/* This define prepares ZLIB function declarations. */ -/***********************************************************************/ -//#define ZLIB_DLL - +#include "filamtxt.h" +#include "tabfmt.h" +//#include "tabzip.h" #include "filamzip.h" -/***********************************************************************/ -/* DB static variables. */ -/***********************************************************************/ -extern int num_read, num_there, num_eq[]; // Statistics - -/* ------------------------------------------------------------------- */ +/* -------------------------- class ZIPUTIL -------------------------- */ /***********************************************************************/ -/* Implementation of the ZIPFAM class. */ +/* Constructors. */ /***********************************************************************/ -ZIPFAM::ZIPFAM(PZIPFAM txfp) : TXTFAM(txfp) - { - Zfile = txfp->Zfile; - Zpos = txfp->Zpos; - } // end of ZIPFAM copy constructor +ZIPUTIL::ZIPUTIL(PSZ tgt, bool mul) +{ + zipfile = NULL; + target = tgt; + fp = NULL; + memory = NULL; + size = 0; + entryopen = false; + multiple = mul; + memset(fn, 0, sizeof(fn)); + + // Init the case mapping table. +#if defined(__WIN__) + for (int i = 0; i < 256; ++i) mapCaseTable[i] = toupper(i); +#else + for (int i = 0; i < 256; ++i) mapCaseTable[i] = i; +#endif +} // end of ZIPUTIL standard constructor + +#if 0 +ZIPUTIL::ZIPUTIL(PZIPUTIL zutp) +{ + zipfile = zutp->zipfile; + target = zutp->target; + fp = zutp->fp; + finfo = zutp->finfo; + entryopen = zutp->entryopen; + multiple = zutp->multiple; + for (int i = 0; i < 256; ++i) mapCaseTable[i] = zutp->mapCaseTable[i]; +} // end of ZIPUTIL copy constructor +#endif // 0 /***********************************************************************/ -/* Zerror: Error function for gz calls. */ -/* gzerror returns the error message for the last error which occurred*/ -/* on the given compressed file. errnum is set to zlib error number. */ -/* If an error occurred in the file system and not in the compression */ -/* library, errnum is set to Z_ERRNO and the application may consult */ -/* errno to get the exact error code. */ +/* This code is the copyright property of Alessandro Felice Cantatore. */ +/* http://xoomer.virgilio.it/acantato/dev/wildcard/wildmatch.html */ /***********************************************************************/ -int ZIPFAM::Zerror(PGLOBAL g) - { - int errnum; +bool ZIPUTIL::WildMatch(PSZ pat, PSZ str) { + PSZ s, p; + bool star = FALSE; - strcpy(g->Message, gzerror(Zfile, &errnum)); +loopStart: + for (s = str, p = pat; *s; ++s, ++p) { + switch (*p) { + case '?': + if (*s == '.') goto starCheck; + break; + case '*': + star = TRUE; + str = s, pat = p; + if (!*++pat) return TRUE; + goto loopStart; + default: + if (mapCaseTable[*s] != mapCaseTable[*p]) + goto starCheck; + break; + } /* endswitch */ + } /* endfor */ + if (*p == '*') ++p; + return (!*p); - if (errnum == Z_ERRNO) -#if defined(__WIN__) - sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(NULL)); -#else // !__WIN__ - sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(errno)); -#endif // !__WIN__ - - return (errnum == Z_STREAM_END) ? RC_EF : RC_FX; - } // end of Zerror +starCheck: + if (!star) return FALSE; + str++; + goto loopStart; +} // end of WildMatch /***********************************************************************/ -/* Reset: reset position values at the beginning of file. */ +/* open a zip file. */ +/* param: filename path and the filename of the zip file to open. */ +/* return: true if open, false otherwise. */ /***********************************************************************/ -void ZIPFAM::Reset(void) - { - TXTFAM::Reset(); -//gzrewind(Zfile); // Useful ????? - Zpos = 0; - } // end of Reset +bool ZIPUTIL::open(PGLOBAL g, char *filename) +{ + if (!zipfile && !(zipfile = unzOpen64(filename))) + sprintf(g->Message, "Zipfile open error on %s", filename); + + return (zipfile == NULL); +} // end of open /***********************************************************************/ -/* ZIP GetFileLength: returns an estimate of what would be the */ -/* uncompressed file size in number of bytes. */ +/* Close the zip file. */ /***********************************************************************/ -int ZIPFAM::GetFileLength(PGLOBAL g) - { - int len = TXTFAM::GetFileLength(g); +void ZIPUTIL::close() +{ + if (zipfile) { + closeEntry(); + unzClose(zipfile); + zipfile = NULL; + } // endif zipfile - if (len > 0) - // Estimate size reduction to a max of 6 - len *= 6; - - return len; - } // end of GetFileLength +} // end of close /***********************************************************************/ -/* ZIP Access Method opening routine. */ +/* Find next entry matching target pattern. */ /***********************************************************************/ -bool ZIPFAM::OpenTableFile(PGLOBAL g) - { - char opmode[4], filename[_MAX_PATH]; - MODE mode = Tdbp->GetMode(); - - switch (mode) { - case MODE_READ: - strcpy(opmode, "r"); - break; - case MODE_UPDATE: - /*****************************************************************/ - /* Updating ZIP files not implemented yet. */ - /*****************************************************************/ - strcpy(g->Message, MSG(UPD_ZIP_NOT_IMP)); - return true; - case MODE_DELETE: - if (!Tdbp->GetNext()) { - // Store the number of deleted lines - DelRows = Cardinality(g); - - // This will erase the entire file - strcpy(opmode, "w"); -// Block = 0; // For ZBKFAM -// Last = Nrec; // For ZBKFAM - Tdbp->ResetSize(); - } else { - sprintf(g->Message, MSG(NO_PART_DEL), "ZIP"); - return true; - } // endif filter - - break; - case MODE_INSERT: - strcpy(opmode, "a+"); - break; - default: - sprintf(g->Message, MSG(BAD_OPEN_MODE), mode); - return true; - } // endswitch Mode - - /*********************************************************************/ - /* Open according to logical input/output mode required. */ - /* Use specific zlib functions. */ - /* Treat files as binary. */ - /*********************************************************************/ - strcat(opmode, "b"); - Zfile = gzopen(PlugSetPath(filename, To_File, Tdbp->GetPath()), opmode); - - if (Zfile == NULL) { - sprintf(g->Message, MSG(GZOPEN_ERROR), - opmode, (int)errno, filename); - strcat(strcat(g->Message, ": "), strerror(errno)); - return (mode == MODE_READ && errno == ENOENT) - ? PushWarning(g, Tdbp) : true; - } // endif Zfile - - /*********************************************************************/ - /* Something to be done here. >>>>>>>> NOT DONE <<<<<<<< */ - /*********************************************************************/ -//To_Fb = dbuserp->Openlist; // Keep track of File block - - /*********************************************************************/ - /* Allocate the line buffer. */ - /*********************************************************************/ - return AllocateBuffer(g); - } // end of OpenTableFile +int ZIPUTIL::findEntry(PGLOBAL g, bool next) +{ + int rc; -/***********************************************************************/ -/* Allocate the line buffer. For mode Delete a bigger buffer has to */ -/* be allocated because is it also used to move lines into the file. */ -/***********************************************************************/ -bool ZIPFAM::AllocateBuffer(PGLOBAL g) - { - MODE mode = Tdbp->GetMode(); + do { + if (next) { + rc = unzGoToNextFile(zipfile); - Buflen = Lrecl + 2; // Lrecl does not include CRLF -//Buflen *= ((Mode == MODE_DELETE) ? DOS_BUFF_LEN : 1); NIY + if (rc == UNZ_END_OF_LIST_OF_FILE) + return RC_EF; + else if (rc != UNZ_OK) { + sprintf(g->Message, "unzGoToNextFile rc = ", rc); + return RC_FX; + } // endif rc - if (trace) - htrc("SubAllocating a buffer of %d bytes\n", Buflen); + } // endif next - To_Buf = (char*)PlugSubAlloc(g, NULL, Buflen); + if (target && *target) { + rc = unzGetCurrentFileInfo(zipfile, NULL, fn, sizeof(fn), + NULL, 0, NULL, 0); + if (rc == UNZ_OK) { + if (WildMatch(target, fn)) + return RC_OK; - if (mode == MODE_INSERT) { - /*******************************************************************/ - /* For Insert buffer must be prepared. */ - /*******************************************************************/ - memset(To_Buf, ' ', Buflen); - To_Buf[Buflen - 2] = '\n'; - To_Buf[Buflen - 1] = '\0'; - } // endif Insert + } else { + sprintf(g->Message, "GetCurrentFileInfo rc = %d", rc); + return RC_FX; + } // endif rc - return false; - } // end of AllocateBuffer + } else + return RC_OK; -/***********************************************************************/ -/* GetRowID: return the RowID of last read record. */ -/***********************************************************************/ -int ZIPFAM::GetRowID(void) - { - return Rows; - } // end of GetRowID + next = true; + } while (true); -/***********************************************************************/ -/* GetPos: return the position of last read record. */ -/***********************************************************************/ -int ZIPFAM::GetPos(void) - { - return (int)Zpos; - } // end of GetPos + strcpy(g->Message, "FindNext logical error"); + return RC_FX; +} // end of FindEntry -/***********************************************************************/ -/* GetNextPos: return the position of next record. */ -/***********************************************************************/ -int ZIPFAM::GetNextPos(void) - { - return gztell(Zfile); - } // end of GetNextPos /***********************************************************************/ -/* SetPos: Replace the table at the specified position. */ +/* Get the next used entry. */ /***********************************************************************/ -bool ZIPFAM::SetPos(PGLOBAL g, int pos __attribute__((unused))) - { - sprintf(g->Message, MSG(NO_SETPOS_YET), "ZIP"); - return true; -#if 0 - Fpos = pos; +int ZIPUTIL::nextEntry(PGLOBAL g) +{ + if (multiple) { + int rc; - if (fseek(Stream, Fpos, SEEK_SET)) { - sprintf(g->Message, MSG(FSETPOS_ERROR), Fpos); - return true; - } // endif + closeEntry(); - Placed = true; - return false; -#endif // 0 - } // end of SetPos + if ((rc = findEntry(g, true)) != RC_OK) + return rc; -/***********************************************************************/ -/* Record file position in case of UPDATE or DELETE. */ -/***********************************************************************/ -bool ZIPFAM::RecordPos(PGLOBAL) - { - Zpos = gztell(Zfile); - return false; - } // end of RecordPos + if (openEntry(g)) + return RC_FX; -/***********************************************************************/ -/* Skip one record in file. */ -/***********************************************************************/ -int ZIPFAM::SkipRecord(PGLOBAL g, bool header) - { - // Skip this record - if (gzeof(Zfile)) - return RC_EF; - else if (gzgets(Zfile, To_Buf, Buflen) == Z_NULL) - return Zerror(g); + return RC_OK; + } else + return RC_EF; - if (header) - RecordPos(g); +} // end of nextEntry - return RC_OK; - } // end of SkipRecord /***********************************************************************/ -/* ReadBuffer: Read one line from a compressed text file. */ +/* OpenTableFile: Open a DOS/UNIX table file from a ZIP file. */ /***********************************************************************/ -int ZIPFAM::ReadBuffer(PGLOBAL g) - { - char *p; - int rc; - - if (!Zfile) - return RC_EF; - - if (!Placed) { - /*******************************************************************/ - /* Record file position in case of UPDATE or DELETE. */ - /*******************************************************************/ - next: - if (RecordPos(g)) - return RC_FX; - - CurBlk = Rows++; // Update RowID - - /*******************************************************************/ - /* Check whether optimization on ROWID */ - /* can be done, as well as for join as for local filtering. */ - /*******************************************************************/ - switch (Tdbp->TestBlock(g)) { - case RC_EF: - return RC_EF; - case RC_NF: - // Skip this record - if ((rc = SkipRecord(g, FALSE)) != RC_OK) - return rc; - - goto next; - } // endswitch rc - - } else - Placed = false; - - if (gzeof(Zfile)) { - rc = RC_EF; - } else if (gzgets(Zfile, To_Buf, Buflen) != Z_NULL) { - p = To_Buf + strlen(To_Buf) - 1; - - if (*p == '\n') - *p = '\0'; // Eliminate ending new-line character - - if (*(--p) == '\r') - *p = '\0'; // Eliminate eventuel carriage return - - strcpy(Tdbp->GetLine(), To_Buf); - IsRead = true; - rc = RC_OK; - num_read++; - } else - rc = Zerror(g); - - if (trace > 1) - htrc(" Read: '%s' rc=%d\n", To_Buf, rc); - - return rc; - } // end of ReadBuffer +bool ZIPUTIL::OpenTable(PGLOBAL g, MODE mode, char *fn) +{ + /*********************************************************************/ + /* The file will be decompressed into virtual memory. */ + /*********************************************************************/ + if (mode == MODE_READ || mode == MODE_ANY) { + bool b = open(g, fn); -/***********************************************************************/ -/* WriteDB: Data Base write routine for ZDOS access method. */ -/* Update is not possible without using a temporary file (NIY). */ -/***********************************************************************/ -int ZIPFAM::WriteBuffer(PGLOBAL g) - { - /*********************************************************************/ - /* Prepare the write buffer. */ - /*********************************************************************/ - strcat(strcpy(To_Buf, Tdbp->GetLine()), CrLf); - - /*********************************************************************/ - /* Now start the writing process. */ - /*********************************************************************/ - if (gzputs(Zfile, To_Buf) < 0) - return Zerror(g); - - return RC_OK; - } // end of WriteBuffer + if (!b) { + int rc; -/***********************************************************************/ -/* Data Base delete line routine for ZDOS access method. (NIY) */ -/***********************************************************************/ -int ZIPFAM::DeleteRecords(PGLOBAL g, int) - { - strcpy(g->Message, MSG(NO_ZIP_DELETE)); - return RC_FX; - } // end of DeleteRecords + if (target && *target) { + if (!multiple) { + rc = unzLocateFile(zipfile, target, 0); -/***********************************************************************/ -/* Data Base close routine for DOS access method. */ -/***********************************************************************/ -void ZIPFAM::CloseTableFile(PGLOBAL, bool) - { - int rc = gzclose(Zfile); + if (rc == UNZ_END_OF_LIST_OF_FILE) { + sprintf(g->Message, "Target file %s not in %s", target, fn); + return true; + } else if (rc != UNZ_OK) { + sprintf(g->Message, "unzLocateFile rc=%d", rc); + return true; + } // endif's rc - if (trace) - htrc("ZIP CloseDB: closing %s rc=%d\n", To_File, rc); + } else { + if ((rc = findEntry(g, false)) == RC_FX) + return true; + else if (rc == RC_EF) { + sprintf(g->Message, "No match of %s in %s", target, fn); + return true; + } // endif rc - Zfile = NULL; // So we can know whether table is open -//To_Fb->Count = 0; // Avoid double closing by PlugCloseAll - } // end of CloseTableFile + } // endif multiple -/***********************************************************************/ -/* Rewind routine for ZIP access method. */ -/***********************************************************************/ -void ZIPFAM::Rewind(void) - { - gzrewind(Zfile); - } // end of Rewind + } // endif target -/* ------------------------------------------------------------------- */ + if (openEntry(g)) + return true; -/***********************************************************************/ -/* Constructors. */ -/***********************************************************************/ -ZBKFAM::ZBKFAM(PDOSDEF tdp) : ZIPFAM(tdp) - { - Blocked = true; - Block = tdp->GetBlock(); - Last = tdp->GetLast(); - Nrec = tdp->GetElemt(); - CurLine = NULL; - NxtLine = NULL; - Closing = false; - BlkPos = tdp->GetTo_Pos(); - } // end of ZBKFAM standard constructor - -ZBKFAM::ZBKFAM(PZBKFAM txfp) : ZIPFAM(txfp) - { - CurLine = txfp->CurLine; - NxtLine = txfp->NxtLine; - Closing = txfp->Closing; - } // end of ZBKFAM copy constructor + if (size > 0) { + /*******************************************************************/ + /* Link a Fblock. This make possible to automatically close it */ + /* in case of error g->jump. */ + /*******************************************************************/ + PDBUSER dbuserp = (PDBUSER)g->Activityp->Aptr; -/***********************************************************************/ -/* Use BlockTest to reduce the table estimated size. */ -/***********************************************************************/ -int ZBKFAM::MaxBlkSize(PGLOBAL g, int) - { - int rc = RC_OK, savcur = CurBlk; - int size; - - // Roughly estimate the table size as the sum of blocks - // that can contain good rows - for (size = 0, CurBlk = 0; CurBlk < Block; CurBlk++) - if ((rc = Tdbp->TestBlock(g)) == RC_OK) - size += (CurBlk == Block - 1) ? Last : Nrec; - else if (rc == RC_EF) - break; - - CurBlk = savcur; - return size; - } // end of MaxBlkSize + fp = (PFBLOCK)PlugSubAlloc(g, NULL, sizeof(FBLOCK)); + fp->Type = TYPE_FB_ZIP; + fp->Fname = PlugDup(g, fn); + fp->Next = dbuserp->Openlist; + dbuserp->Openlist = fp; + fp->Count = 1; + fp->Length = size; + fp->Memory = memory; + fp->Mode = mode; + fp->File = this; + fp->Handle = NULL; + } // endif fp -/***********************************************************************/ -/* ZBK Cardinality: returns table cardinality in number of rows. */ -/* This function can be called with a null argument to test the */ -/* availability of Cardinality implementation (1 yes, 0 no). */ -/***********************************************************************/ -int ZBKFAM::Cardinality(PGLOBAL g) - { - return (g) ? (int)((Block - 1) * Nrec + Last) : 1; - } // end of Cardinality + } else + return true; -/***********************************************************************/ -/* Allocate the line buffer. For mode Delete a bigger buffer has to */ -/* be allocated because is it also used to move lines into the file. */ -/***********************************************************************/ -bool ZBKFAM::AllocateBuffer(PGLOBAL g) - { - Buflen = Nrec * (Lrecl + 2); - CurLine = To_Buf = (char*)PlugSubAlloc(g, NULL, Buflen); - - if (Tdbp->GetMode() == MODE_INSERT) { - // Set values so Block and Last can be recalculated - if (Last == Nrec) { - CurBlk = Block; - Rbuf = Nrec; // To be used by WriteDB - } else { - // The last block must be completed - CurBlk = Block - 1; - Rbuf = Nrec - Last; // To be used by WriteDB - } // endif Last - - } // endif Insert - - return false; - } // end of AllocateBuffer + } else { + strcpy(g->Message, "Only READ mode supported for ZIP files"); + return true; + } // endif mode -/***********************************************************************/ -/* GetRowID: return the RowID of last read record. */ -/***********************************************************************/ -int ZBKFAM::GetRowID(void) - { - return CurNum + Nrec * CurBlk + 1; - } // end of GetRowID + return false; +} // end of OpenTableFile /***********************************************************************/ -/* GetPos: return the position of last read record. */ +/* Open target in zip file. */ /***********************************************************************/ -int ZBKFAM::GetPos(void) - { - return CurNum + Nrec * CurBlk; // Computed file index - } // end of GetPos +bool ZIPUTIL::openEntry(PGLOBAL g) +{ + int rc; -/***********************************************************************/ -/* Record file position in case of UPDATE or DELETE. */ -/* Not used yet for fixed tables. */ -/***********************************************************************/ -bool ZBKFAM::RecordPos(PGLOBAL /*g*/) - { -//strcpy(g->Message, "RecordPos not implemented for zip blocked tables"); -//return true; - return RC_OK; - } // end of RecordPos + rc = unzGetCurrentFileInfo(zipfile, &finfo, fn, sizeof(fn), + NULL, 0, NULL, 0); -/***********************************************************************/ -/* Skip one record in file. */ -/***********************************************************************/ -int ZBKFAM::SkipRecord(PGLOBAL /*g*/, bool) - { -//strcpy(g->Message, "SkipRecord not implemented for zip blocked tables"); -//return RC_FX; - return RC_OK; - } // end of SkipRecord + if (rc != UNZ_OK) { + sprintf(g->Message, "unzGetCurrentFileInfo64 rc=%d", rc); + return true; + } else if ((rc = unzOpenCurrentFile(zipfile)) != UNZ_OK) { + sprintf(g->Message, "unzOpen fn=%s rc=%d", fn, rc); + return true; + } // endif rc -/***********************************************************************/ -/* ReadBuffer: Read one line from a compressed text file. */ -/***********************************************************************/ -int ZBKFAM::ReadBuffer(PGLOBAL g) - { - int n, skip, rc = RC_OK; - - /*********************************************************************/ - /* Sequential reading when Placed is not true. */ - /*********************************************************************/ - if (++CurNum < Rbuf) { - CurLine = NxtLine; - - // Get the position of the next line in the buffer - while (*NxtLine++ != '\n') ; - - // Set caller line buffer - n = NxtLine - CurLine - Ending; - memcpy(Tdbp->GetLine(), CurLine, n); - Tdbp->GetLine()[n] = '\0'; - return RC_OK; - } else if (Rbuf < Nrec && CurBlk != -1) - return RC_EF; - - /*********************************************************************/ - /* New block. */ - /*********************************************************************/ - CurNum = 0; - skip = 0; - - next: - if (++CurBlk >= Block) - return RC_EF; - - /*********************************************************************/ - /* Before using the new block, check whether block optimization */ - /* can be done, as well as for join as for local filtering. */ - /*********************************************************************/ - switch (Tdbp->TestBlock(g)) { - case RC_EF: - return RC_EF; - case RC_NF: - skip++; - goto next; - } // endswitch rc - - if (skip) - // Skip blocks rejected by block optimization - for (int i = CurBlk - skip; i < CurBlk; i++) { - BlkLen = BlkPos[i + 1] - BlkPos[i]; - - if (gzseek(Zfile, (z_off_t)BlkLen, SEEK_CUR) < 0) - return Zerror(g); - - } // endfor i - - BlkLen = BlkPos[CurBlk + 1] - BlkPos[CurBlk]; - - if (!(n = gzread(Zfile, To_Buf, BlkLen))) { - rc = RC_EF; - } else if (n > 0) { - // Get the position of the current line - CurLine = To_Buf; - - // Now get the position of the next line - for (NxtLine = CurLine; *NxtLine++ != '\n';) ; - - // Set caller line buffer - n = NxtLine - CurLine - Ending; - memcpy(Tdbp->GetLine(), CurLine, n); - Tdbp->GetLine()[n] = '\0'; - Rbuf = (CurBlk == Block - 1) ? Last : Nrec; - IsRead = true; - rc = RC_OK; - num_read++; - } else - rc = Zerror(g); - - return rc; - } // end of ReadBuffer + size = finfo.uncompressed_size; + memory = new char[size + 1]; -/***********************************************************************/ -/* WriteDB: Data Base write routine for ZDOS access method. */ -/* Update is not possible without using a temporary file (NIY). */ -/***********************************************************************/ -int ZBKFAM::WriteBuffer(PGLOBAL g) - { - /*********************************************************************/ - /* Prepare the write buffer. */ - /*********************************************************************/ - if (!Closing) - strcat(strcpy(CurLine, Tdbp->GetLine()), CrLf); - - /*********************************************************************/ - /* In Insert mode, blocs are added sequentialy to the file end. */ - /* Note: Update mode is not handled for zip files. */ - /*********************************************************************/ - if (++CurNum == Rbuf) { - /*******************************************************************/ - /* New block, start the writing process. */ - /*******************************************************************/ - BlkLen = CurLine + strlen(CurLine) - To_Buf; - - if (gzwrite(Zfile, To_Buf, BlkLen) != BlkLen || - gzflush(Zfile, Z_FULL_FLUSH)) { - Closing = true; - return Zerror(g); - } // endif gzwrite - - Rbuf = Nrec; - CurBlk++; - CurNum = 0; - CurLine = To_Buf; - } else - CurLine += strlen(CurLine); - - return RC_OK; - } // end of WriteBuffer + if ((rc = unzReadCurrentFile(zipfile, memory, size)) < 0) { + sprintf(g->Message, "unzReadCurrentFile rc = ", rc); + unzCloseCurrentFile(zipfile); + free(memory); + memory = NULL; + entryopen = false; + } else { + memory[size] = 0; // Required by some table types (XML) + entryopen = true; + } // endif rc + + if (trace) + htrc("Openning entry%s %s\n", fn, (entryopen) ? "oked" : "failed"); + + return !entryopen; +} // end of openEntry + +/***********************************************************************/ +/* Close the zip file. */ +/***********************************************************************/ +void ZIPUTIL::closeEntry() +{ + if (entryopen) { + unzCloseCurrentFile(zipfile); + entryopen = false; + } // endif entryopen + + if (memory) { + free(memory); + memory = NULL; + } // endif memory + +} // end of closeEntry + +/* -------------------------- class ZIPFAM --------------------------- */ /***********************************************************************/ -/* Data Base delete line routine for ZBK access method. */ -/* Implemented only for total deletion of the table, which is done */ -/* by opening the file in mode "wb". */ +/* Constructors. */ /***********************************************************************/ -int ZBKFAM::DeleteRecords(PGLOBAL g, int irc) - { - if (irc == RC_EF) { - LPCSTR name = Tdbp->GetName(); - PDOSDEF defp = (PDOSDEF)Tdbp->GetDef(); - - defp->SetBlock(0); - defp->SetLast(Nrec); +ZIPFAM::ZIPFAM(PDOSDEF tdp) : MAPFAM(tdp) +{ + zutp = NULL; + target = tdp->GetEntry(); + mul = tdp->GetMul(); +} // end of ZIPFAM standard constructor - if (!defp->SetIntCatInfo("Blocks", 0) || - !defp->SetIntCatInfo("Last", 0)) { - sprintf(g->Message, MSG(UPDATE_ERROR), "Header"); - return RC_FX; - } else - return RC_OK; +ZIPFAM::ZIPFAM(PZIPFAM txfp) : MAPFAM(txfp) +{ + zutp = txfp->zutp; + target = txfp->target; + mul = txfp->mul; +} // end of ZIPFAM copy constructor - } else - return irc; - - } // end of DeleteRecords +ZIPFAM::ZIPFAM(PDOSDEF tdp, PZPXFAM txfp) : MAPFAM(tdp) +{ + zutp = txfp->zutp; + target = txfp->target; + mul = txfp->mul; +} // end of ZIPFAM constructor used in ResetTableOpt /***********************************************************************/ -/* Data Base close routine for ZBK access method. */ +/* ZIP GetFileLength: returns file size in number of bytes. */ /***********************************************************************/ -void ZBKFAM::CloseTableFile(PGLOBAL g, bool) - { - int rc = RC_OK; - - if (Tdbp->GetMode() == MODE_INSERT) { - LPCSTR name = Tdbp->GetName(); - PDOSDEF defp = (PDOSDEF)Tdbp->GetDef(); - - if (CurNum && !Closing) { - // Some more inserted lines remain to be written - Last = (Nrec - Rbuf) + CurNum; - Block = CurBlk + 1; - Rbuf = CurNum--; - Closing = true; - rc = WriteBuffer(g); - } else if (Rbuf == Nrec) { - Last = Nrec; - Block = CurBlk; - } // endif CurNum - - if (rc != RC_FX) { - defp->SetBlock(Block); - defp->SetLast(Last); - defp->SetIntCatInfo("Blocks", Block); - defp->SetIntCatInfo("Last", Last); - } // endif - - gzclose(Zfile); - } else if (Tdbp->GetMode() == MODE_DELETE) { - rc = DeleteRecords(g, RC_EF); - gzclose(Zfile); - } else - rc = gzclose(Zfile); - - if (trace) - htrc("ZIP CloseDB: closing %s rc=%d\n", To_File, rc); - - Zfile = NULL; // So we can know whether table is open -//To_Fb->Count = 0; // Avoid double closing by PlugCloseAll - } // end of CloseTableFile +int ZIPFAM::GetFileLength(PGLOBAL g) +{ + int len = (zutp && zutp->entryopen) ? Top - Memory + : TXTFAM::GetFileLength(g) * 3; -/***********************************************************************/ -/* Rewind routine for ZBK access method. */ -/***********************************************************************/ -void ZBKFAM::Rewind(void) - { - gzrewind(Zfile); - CurBlk = -1; - CurNum = Rbuf; - } // end of Rewind + if (trace) + htrc("Zipped file length=%d\n", len); -/* ------------------------------------------------------------------- */ + return len; +} // end of GetFileLength /***********************************************************************/ -/* Constructors. */ +/* ZIP Cardinality: return the number of rows if possible. */ /***********************************************************************/ -ZIXFAM::ZIXFAM(PDOSDEF tdp) : ZBKFAM(tdp) - { -//Block = tdp->GetBlock(); -//Last = tdp->GetLast(); - Nrec = (tdp->GetElemt()) ? tdp->GetElemt() : DOS_BUFF_LEN; - Blksize = Nrec * Lrecl; - } // end of ZIXFAM standard constructor +int ZIPFAM::Cardinality(PGLOBAL g) +{ + if (!g) + return 1; -/***********************************************************************/ -/* ZIX Cardinality: returns table cardinality in number of rows. */ -/* This function can be called with a null argument to test the */ -/* availability of Cardinality implementation (1 yes, 0 no). */ -/***********************************************************************/ -int ZIXFAM::Cardinality(PGLOBAL g) - { - if (Last) - return (g) ? (int)((Block - 1) * Nrec + Last) : 1; - else // Last and Block not defined, cannot do it yet - return 0; + int card = -1; + int len = GetFileLength(g); - } // end of Cardinality + card = (len / (int)Lrecl) * 2; // Estimated ??? + return card; +} // end of Cardinality /***********************************************************************/ -/* Allocate the line buffer. For mode Delete a bigger buffer has to */ -/* be allocated because is it also used to move lines into the file. */ +/* OpenTableFile: Open a DOS/UNIX table file from a ZIP file. */ /***********************************************************************/ -bool ZIXFAM::AllocateBuffer(PGLOBAL g) - { - Buflen = Blksize; - To_Buf = (char*)PlugSubAlloc(g, NULL, Buflen); - - if (Tdbp->GetMode() == MODE_INSERT) { - /*******************************************************************/ - /* For Insert the buffer must be prepared. */ - /*******************************************************************/ - memset(To_Buf, ' ', Buflen); - - if (Tdbp->GetFtype() < 2) - // if not binary, the file is physically a text file - for (int len = Lrecl; len <= Buflen; len += Lrecl) { -#if defined(__WIN__) - To_Buf[len - 2] = '\r'; -#endif // __WIN__ - To_Buf[len - 1] = '\n'; - } // endfor len - - // Set values so Block and Last can be recalculated - if (Last == Nrec) { - CurBlk = Block; - Rbuf = Nrec; // To be used by WriteDB - } else { - // The last block must be completed - CurBlk = Block - 1; - Rbuf = Nrec - Last; // To be used by WriteDB - } // endif Last - - } // endif Insert - - return false; - } // end of AllocateBuffer +bool ZIPFAM::OpenTableFile(PGLOBAL g) +{ + char filename[_MAX_PATH]; + MODE mode = Tdbp->GetMode(); -/***********************************************************************/ -/* ReadBuffer: Read one line from a compressed text file. */ -/***********************************************************************/ -int ZIXFAM::ReadBuffer(PGLOBAL g) - { - int n, rc = RC_OK; - - /*********************************************************************/ - /* Sequential reading when Placed is not true. */ - /*********************************************************************/ - if (++CurNum < Rbuf) { - Tdbp->IncLine(Lrecl); // Used by DOSCOL functions - return RC_OK; - } else if (Rbuf < Nrec && CurBlk != -1) - return RC_EF; - - /*********************************************************************/ - /* New block. */ - /*********************************************************************/ - CurNum = 0; - Tdbp->SetLine(To_Buf); - - int skip = 0; - - next: - if (++CurBlk >= Block) - return RC_EF; - - /*********************************************************************/ - /* Before using the new block, check whether block optimization */ - /* can be done, as well as for join as for local filtering. */ - /*********************************************************************/ - switch (Tdbp->TestBlock(g)) { - case RC_EF: - return RC_EF; - case RC_NF: - skip++; - goto next; - } // endswitch rc - - if (skip) - // Skip blocks rejected by block optimization - for (int i = 0; i < skip; i++) { - if (gzseek(Zfile, (z_off_t)Buflen, SEEK_CUR) < 0) - return Zerror(g); - - } // endfor i - - if (!(n = gzread(Zfile, To_Buf, Buflen))) { - rc = RC_EF; - } else if (n > 0) { - Rbuf = n / Lrecl; - IsRead = true; - rc = RC_OK; - num_read++; - } else - rc = Zerror(g); - - return rc; - } // end of ReadBuffer + /*********************************************************************/ + /* Allocate the ZIP utility class. */ + /*********************************************************************/ + zutp = new(g) ZIPUTIL(target, mul); -/***********************************************************************/ -/* WriteDB: Data Base write routine for ZDOS access method. */ -/* Update is not possible without using a temporary file (NIY). */ -/***********************************************************************/ -int ZIXFAM::WriteBuffer(PGLOBAL g) - { - /*********************************************************************/ - /* In Insert mode, blocs are added sequentialy to the file end. */ - /* Note: Update mode is not handled for zip files. */ - /*********************************************************************/ - if (++CurNum == Rbuf) { - /*******************************************************************/ - /* New block, start the writing process. */ - /*******************************************************************/ - BlkLen = Rbuf * Lrecl; - - if (gzwrite(Zfile, To_Buf, BlkLen) != BlkLen || - gzflush(Zfile, Z_FULL_FLUSH)) { - Closing = true; - return Zerror(g); - } // endif gzwrite - - Rbuf = Nrec; - CurBlk++; - CurNum = 0; - Tdbp->SetLine(To_Buf); - } else - Tdbp->IncLine(Lrecl); // Used by FIXCOL functions - - return RC_OK; - } // end of WriteBuffer - -/* --------------------------- Class ZLBFAM -------------------------- */ + // We used the file name relative to recorded datapath + PlugSetPath(filename, To_File, Tdbp->GetPath()); -/***********************************************************************/ -/* Constructors. */ -/***********************************************************************/ -ZLBFAM::ZLBFAM(PDOSDEF tdp) : BLKFAM(tdp) - { - Zstream = NULL; - Zbuffer = NULL; - Zlenp = NULL; - Optimized = tdp->IsOptimized(); - } // end of ZLBFAM standard constructor - -ZLBFAM::ZLBFAM(PZLBFAM txfp) : BLKFAM(txfp) - { - Zstream = txfp->Zstream; - Zbuffer = txfp->Zbuffer; - Zlenp = txfp->Zlenp; - Optimized = txfp->Optimized; - } // end of ZLBFAM (dummy?) copy constructor + if (!zutp->OpenTable(g, mode, filename)) { + // The pseudo "buffer" is here the entire real buffer + Fpos = Mempos = Memory = zutp->memory; + Top = Memory + zutp->size; + To_Fb = zutp->fp; // Useful when closing + } else + return true; + + return false; + } // end of OpenTableFile /***********************************************************************/ -/* ZLB GetFileLength: returns an estimate of what would be the */ -/* uncompressed file size in number of bytes. */ +/* GetNext: go to next entry. */ /***********************************************************************/ -int ZLBFAM::GetFileLength(PGLOBAL g) - { - int len = (Optimized) ? BlkPos[Block] : BLKFAM::GetFileLength(g); +int ZIPFAM::GetNext(PGLOBAL g) +{ + int rc = zutp->nextEntry(g); - if (len > 0) - // Estimate size reduction to a max of 5 - len *= 5; + if (rc != RC_OK) + return rc; - return len; - } // end of GetFileLength + Mempos = Memory = zutp->memory; + Top = Memory + zutp->size; + return RC_OK; +} // end of GetNext +#if 0 /***********************************************************************/ -/* Allocate the line buffer. For mode Delete a bigger buffer has to */ -/* be allocated because is it also used to move lines into the file. */ +/* ReadBuffer: Read one line for a ZIP file. */ /***********************************************************************/ -bool ZLBFAM::AllocateBuffer(PGLOBAL g) - { - char *msg; - int n, zrc; +int ZIPFAM::ReadBuffer(PGLOBAL g) +{ + int rc, len; -#if 0 - if (!Optimized && Tdbp->NeedIndexing(g)) { - strcpy(g->Message, MSG(NOP_ZLIB_INDEX)); - return TRUE; - } // endif indexing -#endif // 0 + // Are we at the end of the memory + if (Mempos >= Top) { + if ((rc = zutp->nextEntry(g)) != RC_OK) + return rc; -#if defined(NOLIB) - if (!zlib && LoadZlib()) { - sprintf(g->Message, MSG(DLL_LOAD_ERROR), GetLastError(), "zlib.dll"); - return TRUE; - } // endif zlib -#endif + Mempos = Memory = zutp->memory; + Top = Memory + zutp->size; + } // endif Mempos - BLKFAM::AllocateBuffer(g); -//Buflen = Nrec * (Lrecl + 2); -//Rbuf = Nrec; - - // Allocate the compressed buffer - n = Buflen + 16; // ????????????????????????????????? - Zlenp = (int*)PlugSubAlloc(g, NULL, n); - Zbuffer = (Byte*)(Zlenp + 1); - - // Allocate and initialize the Z stream - Zstream = (z_streamp)PlugSubAlloc(g, NULL, sizeof(z_stream)); - Zstream->zalloc = (alloc_func)0; - Zstream->zfree = (free_func)0; - Zstream->opaque = (voidpf)0; - Zstream->next_in = NULL; - Zstream->avail_in = 0; - - if (Tdbp->GetMode() == MODE_READ) { - msg = "inflateInit"; - zrc = inflateInit(Zstream); - } else { - msg = "deflateInit"; - zrc = deflateInit(Zstream, Z_DEFAULT_COMPRESSION); - } // endif Mode - - if (zrc != Z_OK) { - if (Zstream->msg) - sprintf(g->Message, "%s error: %s", msg, Zstream->msg); - else - sprintf(g->Message, "%s error: %d", msg, zrc); - - return TRUE; - } // endif zrc - - if (Tdbp->GetMode() == MODE_INSERT) { - // Write the file header block - if (Last == Nrec) { - CurBlk = Block; - CurNum = 0; - - if (!GetFileLength(g)) { - // Write the zlib header as an extra block - strcpy(To_Buf, "PlugDB"); - BlkLen = strlen("PlugDB") + 1; - - if (WriteCompressedBuffer(g)) - return TRUE; - - } // endif void file - - } else { - // In mode insert, if Last != Nrec, last block must be updated - CurBlk = Block - 1; - CurNum = Last; - - strcpy(g->Message, MSG(NO_PAR_BLK_INS)); - return TRUE; - } // endif Last - - } else { // MODE_READ - // First thing to do is to read the header block - void *rdbuf; - - if (Optimized) { - BlkLen = BlkPos[0]; - rdbuf = Zlenp; - } else { - // Get the stored length from the file itself - if (fread(Zlenp, sizeof(int), 1, Stream) != 1) - return FALSE; // Empty file - - BlkLen = *Zlenp; - rdbuf = Zbuffer; - } // endif Optimized - - switch (ReadCompressedBuffer(g, rdbuf)) { - case RC_EF: - return FALSE; - case RC_FX: -#if defined(UNIX) - sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(errno)); +#if 0 + if (!Placed) { + /*******************************************************************/ + /* Record file position in case of UPDATE or DELETE. */ + /*******************************************************************/ + int rc; + + next: + Fpos = Mempos; + CurBlk = (int)Rows++; + + /*******************************************************************/ + /* Check whether optimization on ROWID */ + /* can be done, as well as for join as for local filtering. */ + /*******************************************************************/ + switch (Tdbp->TestBlock(g)) { + case RC_EF: + return RC_EF; + case RC_NF: + // Skip this record + if ((rc = SkipRecord(g, false)) != RC_OK) + return rc; + + goto next; + } // endswitch rc + + } else + Placed = false; #else - sprintf(g->Message, MSG(READ_ERROR), To_File, _strerror(NULL)); + // Perhaps unuseful + Fpos = Mempos; + CurBlk = (int)Rows++; + Placed = false; #endif - case RC_NF: - return TRUE; - } // endswitch - // Some old tables can have PlugDB in their header - if (strcmp(To_Buf, "PlugDB")) { - sprintf(g->Message, MSG(BAD_HEADER), Tdbp->GetFile(g)); - return TRUE; - } // endif strcmp + // Immediately calculate next position (Used by DeleteDB) + while (*Mempos++ != '\n'); // What about Unix ??? - } // endif Mode + // Set caller line buffer + len = (Mempos - Fpos) - 1; - return FALSE; - } // end of AllocateBuffer + // Don't rely on ENDING setting + if (len > 0 && *(Mempos - 2) == '\r') + len--; // Line ends by CRLF -/***********************************************************************/ -/* GetPos: return the position of last read record. */ -/***********************************************************************/ -int ZLBFAM::GetPos(void) - { - return (Optimized) ? (CurNum + Nrec * CurBlk) : Fpos; - } // end of GetPos + memcpy(Tdbp->GetLine(), Fpos, len); + Tdbp->GetLine()[len] = '\0'; + return RC_OK; +} // end of ReadBuffer /***********************************************************************/ -/* GetNextPos: should not be called for this class. */ +/* Table file close routine for MAP access method. */ /***********************************************************************/ -int ZLBFAM::GetNextPos(void) - { - if (Optimized) { - assert(FALSE); - return 0; - } else - return ftell(Stream); +void ZIPFAM::CloseTableFile(PGLOBAL g, bool) +{ + close(); +} // end of CloseTableFile +#endif // 0 - } // end of GetNextPos +/* -------------------------- class ZPXFAM --------------------------- */ /***********************************************************************/ -/* SetPos: Replace the table at the specified position. */ +/* Constructors. */ /***********************************************************************/ -bool ZLBFAM::SetPos(PGLOBAL g, int pos __attribute__((unused))) - { - sprintf(g->Message, MSG(NO_SETPOS_YET), "ZIP"); - return true; -#if 0 // All this must be checked - if (pos < 0) { - strcpy(g->Message, MSG(INV_REC_POS)); - return true; - } // endif recpos - - CurBlk = pos / Nrec; - CurNum = pos % Nrec; -#if defined(_DEBUG) - num_eq[(CurBlk == OldBlk) ? 1 : 0]++; -#endif +ZPXFAM::ZPXFAM(PDOSDEF tdp) : MPXFAM(tdp) +{ + zutp = NULL; + target = tdp->GetEntry(); + mul = tdp->GetMul(); + //Lrecl = tdp->GetLrecl(); +} // end of ZPXFAM standard constructor - // Indicate the table position was externally set - Placed = true; - return false; -#endif // 0 - } // end of SetPos +ZPXFAM::ZPXFAM(PZPXFAM txfp) : MPXFAM(txfp) +{ + zutp = txfp->zutp; + target = txfp->target; + mul = txfp->mul; +//Lrecl = txfp->Lrecl; +} // end of ZPXFAM copy constructor /***********************************************************************/ -/* ReadBuffer: Read one line for a text file. */ +/* ZIP GetFileLength: returns file size in number of bytes. */ /***********************************************************************/ -int ZLBFAM::ReadBuffer(PGLOBAL g) - { - int n; - void *rdbuf; - - /*********************************************************************/ - /* Sequential reading when Placed is not true. */ - /*********************************************************************/ - if (Placed) { - Placed = FALSE; - } else if (++CurNum < Rbuf) { - CurLine = NxtLine; - - // Get the position of the next line in the buffer - if (Tdbp->GetFtype() == RECFM_VAR) - while (*NxtLine++ != '\n') ; - else - NxtLine += Lrecl; - - // Set caller line buffer - n = NxtLine - CurLine - ((Tdbp->GetFtype() == RECFM_BIN) ? 0 : Ending); - memcpy(Tdbp->GetLine(), CurLine, n); - Tdbp->GetLine()[n] = '\0'; - return RC_OK; - } else if (Rbuf < Nrec && CurBlk != -1) { - CurNum--; // To have a correct Last value when optimizing - return RC_EF; - } else { - /*******************************************************************/ - /* New block. */ - /*******************************************************************/ - CurNum = 0; - - next: - if (++CurBlk >= Block) - return RC_EF; - - /*******************************************************************/ - /* Before reading a new block, check whether block optimization */ - /* can be done, as well as for join as for local filtering. */ - /*******************************************************************/ - if (Optimized) switch (Tdbp->TestBlock(g)) { - case RC_EF: - return RC_EF; - case RC_NF: - goto next; - } // endswitch rc - - } // endif's - - if (OldBlk == CurBlk) - goto ok; // Block is already there - - if (Optimized) { - // Store the position of next block - Fpos = BlkPos[CurBlk]; - - // fseek is required only in non sequential reading - if (CurBlk != OldBlk + 1) - if (fseek(Stream, Fpos, SEEK_SET)) { - sprintf(g->Message, MSG(FSETPOS_ERROR), Fpos); - return RC_FX; - } // endif fseek - - // Calculate the length of block to read - BlkLen = BlkPos[CurBlk + 1] - Fpos; - rdbuf = Zlenp; - } else { // !Optimized - if (CurBlk != OldBlk + 1) { - strcpy(g->Message, MSG(INV_RAND_ACC)); - return RC_FX; - } else - Fpos = ftell(Stream); // Used when optimizing - - // Get the stored length from the file itself - if (fread(Zlenp, sizeof(int), 1, Stream) != 1) { - if (feof(Stream)) - return RC_EF; - - goto err; - } // endif fread - - BlkLen = *Zlenp; - rdbuf = Zbuffer; - } // endif Optimized - - // Read the next block - switch (ReadCompressedBuffer(g, rdbuf)) { - case RC_FX: goto err; - case RC_NF: return RC_FX; - case RC_EF: return RC_EF; - default: Rbuf = (CurBlk == Block - 1) ? Last : Nrec; - } // endswitch ReadCompressedBuffer - - ok: - if (Tdbp->GetFtype() == RECFM_VAR) { - int i; - - // Get the position of the current line - for (i = 0, CurLine = To_Buf; i < CurNum; i++) - while (*CurLine++ != '\n') ; // What about Unix ??? - - // Now get the position of the next line - for (NxtLine = CurLine; *NxtLine++ != '\n';) ; - - // Set caller line buffer - n = NxtLine - CurLine - Ending; - } else { - CurLine = To_Buf + CurNum * Lrecl; - NxtLine = CurLine + Lrecl; - n = Lrecl - ((Tdbp->GetFtype() == RECFM_BIN) ? 0 : Ending); - } // endif Ftype - - memcpy(Tdbp->GetLine(), CurLine, n); - Tdbp->GetLine()[n] = '\0'; - - OldBlk = CurBlk; // Last block actually read - IsRead = TRUE; // Is read indeed - return RC_OK; - - err: -#if defined(UNIX) - sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(errno)); -#else - sprintf(g->Message, MSG(READ_ERROR), To_File, _strerror(NULL)); -#endif - return RC_FX; - } // end of ReadBuffer +int ZPXFAM::GetFileLength(PGLOBAL g) +{ + int len; -/***********************************************************************/ -/* Read and decompress a block from the stream. */ -/***********************************************************************/ -int ZLBFAM::ReadCompressedBuffer(PGLOBAL g, void *rdbuf) - { - if (fread(rdbuf, 1, (size_t)BlkLen, Stream) == (unsigned)BlkLen) { - int zrc; - - num_read++; - - if (Optimized && BlkLen != signed(*Zlenp + sizeof(int))) { - sprintf(g->Message, MSG(BAD_BLK_SIZE), CurBlk + 1); - return RC_NF; - } // endif BlkLen - - // HERE WE MUST INFLATE THE BLOCK - Zstream->next_in = Zbuffer; - Zstream->avail_in = (uInt)(*Zlenp); - Zstream->next_out = (Byte*)To_Buf; - Zstream->avail_out = Buflen; - zrc = inflate(Zstream, Z_SYNC_FLUSH); - - if (zrc != Z_OK) { - if (Zstream->msg) - sprintf(g->Message, MSG(FUNC_ERR_S), "inflate", Zstream->msg); - else - sprintf(g->Message, MSG(FUNCTION_ERROR), "inflate", (int)zrc); - - return RC_NF; - } // endif zrc - - } else if (feof(Stream)) { - return RC_EF; - } else - return RC_FX; - - return RC_OK; - } // end of ReadCompressedBuffer + if (!zutp && OpenTableFile(g)) + return 0; -/***********************************************************************/ -/* WriteBuffer: File write routine for DOS access method. */ -/* Update is directly written back into the file, */ -/* with this (fast) method, record size cannot change. */ -/***********************************************************************/ -int ZLBFAM::WriteBuffer(PGLOBAL g) - { - assert (Tdbp->GetMode() == MODE_INSERT); - - /*********************************************************************/ - /* Prepare the write buffer. */ - /*********************************************************************/ - if (!Closing) { - if (Tdbp->GetFtype() == RECFM_BIN) - memcpy(CurLine, Tdbp->GetLine(), Lrecl); - else - strcat(strcpy(CurLine, Tdbp->GetLine()), CrLf); - -#if defined(_DEBUG) - if (Tdbp->GetFtype() == RECFM_FIX && - (signed)strlen(CurLine) != Lrecl + (signed)strlen(CrLf)) { - strcpy(g->Message, MSG(BAD_LINE_LEN)); - Closing = TRUE; - return RC_FX; - } // endif Lrecl -#endif // _DEBUG - } // endif Closing - - /*********************************************************************/ - /* In Insert mode, blocs are added sequentialy to the file end. */ - /*********************************************************************/ - if (++CurNum != Rbuf) { - if (Tdbp->GetFtype() == RECFM_VAR) - CurLine += strlen(CurLine); - else - CurLine += Lrecl; - - return RC_OK; // We write only full blocks - } // endif CurNum - - // HERE WE MUST DEFLATE THE BLOCK - if (Tdbp->GetFtype() == RECFM_VAR) - NxtLine = CurLine + strlen(CurLine); - else - NxtLine = CurLine + Lrecl; - - BlkLen = NxtLine - To_Buf; - - if (WriteCompressedBuffer(g)) { - Closing = TRUE; // To tell CloseDB about a Write error - return RC_FX; - } // endif WriteCompressedBuffer - - CurBlk++; - CurNum = 0; - CurLine = To_Buf; - return RC_OK; - } // end of WriteBuffer + if (zutp->entryopen) + len = zutp->size; + else + len = 0; + + return len; +} // end of GetFileLength /***********************************************************************/ -/* Compress the buffer and write the deflated output to stream. */ +/* ZIP Cardinality: return the number of rows if possible. */ /***********************************************************************/ -bool ZLBFAM::WriteCompressedBuffer(PGLOBAL g) - { - int zrc; - - Zstream->next_in = (Byte*)To_Buf; - Zstream->avail_in = (uInt)BlkLen; - Zstream->next_out = Zbuffer; - Zstream->avail_out = Buflen + 16; - Zstream->total_out = 0; - zrc = deflate(Zstream, Z_FULL_FLUSH); - - if (zrc != Z_OK) { - if (Zstream->msg) - sprintf(g->Message, MSG(FUNC_ERR_S), "deflate", Zstream->msg); - else - sprintf(g->Message, MSG(FUNCTION_ERROR), "deflate", (int)zrc); - - return TRUE; - } else - *Zlenp = Zstream->total_out; - - // Now start the writing process. - BlkLen = *Zlenp + sizeof(int); - - if (fwrite(Zlenp, 1, BlkLen, Stream) != (size_t)BlkLen) { - sprintf(g->Message, MSG(FWRITE_ERROR), strerror(errno)); - return TRUE; - } // endif size - - return FALSE; - } // end of WriteCompressedBuffer +int ZPXFAM::Cardinality(PGLOBAL g) +{ + if (!g) + return 1; + + int card = -1; + int len = GetFileLength(g); + + if (!(len % Lrecl)) + card = len / (int)Lrecl; // Fixed length file + else + sprintf(g->Message, MSG(NOT_FIXED_LEN), zutp->fn, len, Lrecl); + + // Set number of blocks for later use + Block = (card > 0) ? (card + Nrec - 1) / Nrec : 0; + return card; +} // end of Cardinality /***********************************************************************/ -/* Table file close routine for DOS access method. */ +/* OpenTableFile: Open a DOS/UNIX table file from a ZIP file. */ /***********************************************************************/ -void ZLBFAM::CloseTableFile(PGLOBAL g, bool) - { - int rc = RC_OK; - - if (Tdbp->GetMode() == MODE_INSERT) { - LPCSTR name = Tdbp->GetName(); - PDOSDEF defp = (PDOSDEF)Tdbp->GetDef(); - - // Closing is True if last Write was in error - if (CurNum && !Closing) { - // Some more inserted lines remain to be written - Last = (Nrec - Rbuf) + CurNum; - Block = CurBlk + 1; - Rbuf = CurNum--; - Closing = TRUE; - rc = WriteBuffer(g); - } else if (Rbuf == Nrec) { - Last = Nrec; - Block = CurBlk; - } // endif CurNum - - if (rc != RC_FX) { - defp->SetBlock(Block); - defp->SetLast(Last); - defp->SetIntCatInfo("Blocks", Block); - defp->SetIntCatInfo("Last", Last); - } // endif - - fclose(Stream); - } else - rc = fclose(Stream); - - if (trace) - htrc("ZLB CloseTableFile: closing %s mode=%d rc=%d\n", - To_File, Tdbp->GetMode(), rc); - - Stream = NULL; // So we can know whether table is open - To_Fb->Count = 0; // Avoid double closing by PlugCloseAll - - if (Tdbp->GetMode() == MODE_READ) - rc = inflateEnd(Zstream); - else - rc = deflateEnd(Zstream); - - } // end of CloseTableFile +bool ZPXFAM::OpenTableFile(PGLOBAL g) +{ + // May have been already opened in GetFileLength + if (!zutp || !zutp->zipfile) { + char filename[_MAX_PATH]; + MODE mode = Tdbp->GetMode(); + + /*********************************************************************/ + /* Allocate the ZIP utility class. */ + /*********************************************************************/ + if (!zutp) + zutp = new(g)ZIPUTIL(target, mul); + + // We used the file name relative to recorded datapath + PlugSetPath(filename, To_File, Tdbp->GetPath()); + + if (!zutp->OpenTable(g, mode, filename)) { + // The pseudo "buffer" is here the entire real buffer + Memory = zutp->memory; + Fpos = Mempos = Memory + Headlen; + Top = Memory + zutp->size; + To_Fb = zutp->fp; // Useful when closing + } else + return true; + + } else + Reset(); + + return false; +} // end of OpenTableFile /***********************************************************************/ -/* Rewind routine for ZLIB access method. */ +/* GetNext: go to next entry. */ /***********************************************************************/ -void ZLBFAM::Rewind(void) - { - // We must be positioned after the header block - if (CurBlk >= 0) { // Nothing to do if no block read yet - if (!Optimized) { // If optimized, fseek will be done in ReadBuffer - size_t st; - - rewind(Stream); +int ZPXFAM::GetNext(PGLOBAL g) +{ + int rc = zutp->nextEntry(g); - if (!(st = fread(Zlenp, sizeof(int), 1, Stream)) && trace) - htrc("fread error %d in Rewind", errno); + if (rc != RC_OK) + return rc; - fseek(Stream, *Zlenp + sizeof(int), SEEK_SET); - OldBlk = -1; - } // endif Optimized + int len = zutp->size; - CurBlk = -1; - CurNum = Rbuf; - } // endif CurBlk + if (len % Lrecl) { + sprintf(g->Message, MSG(NOT_FIXED_LEN), zutp->fn, len, Lrecl); + return RC_FX; + } // endif size -//OldBlk = -1; -//Rbuf = 0; commented out in case we reuse last read block - } // end of Rewind + Memory = zutp->memory; + Top = Memory + len; + Rewind(); + return RC_OK; +} // end of GetNext -/* ------------------------ End of ZipFam ---------------------------- */ diff --git a/storage/connect/filamzip.h b/storage/connect/filamzip.h index edb8b5db323..9312fb2f70e 100644 --- a/storage/connect/filamzip.h +++ b/storage/connect/filamzip.h @@ -1,170 +1,117 @@ -/************** FilAmZip H Declares Source Code File (.H) **************/ -/* Name: FILAMZIP.H Version 1.2 */ +/************** filamzip H Declares Source Code File (.H) **************/ +/* Name: filamzip.h Version 1.0 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 2005-2014 */ +/* (C) Copyright to the author Olivier BERTRAND 2016 */ /* */ -/* This file contains the GZIP access method classes declares. */ +/* This file contains the ZIP file access method classes declares. */ /***********************************************************************/ #ifndef __FILAMZIP_H #define __FILAMZIP_H -#include "zlib.h" +#include "block.h" +#include "filamap.h" +#include "unzip.h" -typedef class ZIPFAM *PZIPFAM; -typedef class ZBKFAM *PZBKFAM; -typedef class ZIXFAM *PZIXFAM; -typedef class ZLBFAM *PZLBFAM; +#define DLLEXPORT extern "C" -/***********************************************************************/ -/* This is the access method class declaration for not optimized */ -/* variable record length files compressed using the gzip library */ -/* functions. File is accessed record by record (row). */ -/***********************************************************************/ -class DllExport ZIPFAM : public TXTFAM { -// friend class DOSCOL; - public: - // Constructor - ZIPFAM(PDOSDEF tdp) : TXTFAM(tdp) {Zfile = NULL; Zpos = 0;} - ZIPFAM(PZIPFAM txfp); - - // Implementation - virtual AMT GetAmType(void) {return TYPE_AM_ZIP;} - virtual int GetPos(void); - virtual int GetNextPos(void); - virtual PTXF Duplicate(PGLOBAL g) - {return (PTXF)new(g) ZIPFAM(this);} - - // Methods - virtual void Reset(void); - virtual int GetFileLength(PGLOBAL g); - virtual int Cardinality(PGLOBAL g) {return (g) ? -1 : 0;} - virtual int MaxBlkSize(PGLOBAL g, int s) {return s;} - virtual bool AllocateBuffer(PGLOBAL g); - virtual int GetRowID(void); - virtual bool RecordPos(PGLOBAL g); - virtual bool SetPos(PGLOBAL g, int recpos); - virtual int SkipRecord(PGLOBAL g, bool header); - virtual bool OpenTableFile(PGLOBAL g); - virtual int ReadBuffer(PGLOBAL g); - virtual int WriteBuffer(PGLOBAL g); - virtual int DeleteRecords(PGLOBAL g, int irc); - virtual void CloseTableFile(PGLOBAL g, bool abort); - virtual void Rewind(void); - - protected: - int Zerror(PGLOBAL g); // GZ error function - - // Members - gzFile Zfile; // Points to GZ file structure - z_off_t Zpos; // Uncompressed file position - }; // end of class ZIPFAM +typedef class ZIPFAM *PZIPFAM; +typedef class ZPXFAM *PZPXFAM; /***********************************************************************/ -/* This is the access method class declaration for optimized variable */ -/* record length files compressed using the gzip library functions. */ -/* The File is accessed by block (requires an opt file). */ +/* This is the ZIP utility fonctions class. */ /***********************************************************************/ -class DllExport ZBKFAM : public ZIPFAM { - public: - // Constructor - ZBKFAM(PDOSDEF tdp); - ZBKFAM(PZBKFAM txfp); - - // Implementation - virtual int GetPos(void); - virtual int GetNextPos(void) {return 0;} - virtual PTXF Duplicate(PGLOBAL g) - {return (PTXF)new(g) ZBKFAM(this);} - - // Methods - virtual int Cardinality(PGLOBAL g); - virtual int MaxBlkSize(PGLOBAL g, int s); - virtual bool AllocateBuffer(PGLOBAL g); - virtual int GetRowID(void); - virtual bool RecordPos(PGLOBAL g); - virtual int SkipRecord(PGLOBAL g, bool header); - virtual int ReadBuffer(PGLOBAL g); - virtual int WriteBuffer(PGLOBAL g); - virtual int DeleteRecords(PGLOBAL g, int irc); - virtual void CloseTableFile(PGLOBAL g, bool abort); - virtual void Rewind(void); - - protected: - // Members - char *CurLine; // Position of current line in buffer - char *NxtLine; // Position of Next line in buffer - bool Closing; // True when closing on Insert - }; // end of class ZBKFAM +class DllExport ZIPUTIL : public BLOCK { +public: + // Constructor + ZIPUTIL(PSZ tgt, bool mul); +//ZIPUTIL(ZIPUTIL *zutp); + + // Implementation +//PTXF Duplicate(PGLOBAL g) { return (PTXF) new(g)ZIPFAM(this); } + + // Methods + virtual bool OpenTable(PGLOBAL g, MODE mode, char *fn); + bool open(PGLOBAL g, char *fn); + bool openEntry(PGLOBAL g); + void close(void); + void closeEntry(void); + bool WildMatch(PSZ pat, PSZ str); + int findEntry(PGLOBAL g, bool next); + int nextEntry(PGLOBAL g); + + // Members + unzFile zipfile; // The ZIP container file + PSZ target; // The target file name + unz_file_info finfo; // The current file info + PFBLOCK fp; + char *memory; + uint size; + int multiple; // Multiple targets + bool entryopen; // True when open current entry + char fn[FILENAME_MAX]; // The current entry file name + char mapCaseTable[256]; +}; // end of ZIPFAM /***********************************************************************/ -/* This is the access method class declaration for fixed record */ -/* length files compressed using the gzip library functions. */ -/* The file is always accessed by block. */ +/* This is the ZIP file access method. */ /***********************************************************************/ -class DllExport ZIXFAM : public ZBKFAM { - public: - // Constructor - ZIXFAM(PDOSDEF tdp); - ZIXFAM(PZIXFAM txfp) : ZBKFAM(txfp) {} - - // Implementation - virtual int GetNextPos(void) {return 0;} - virtual PTXF Duplicate(PGLOBAL g) - {return (PTXF)new(g) ZIXFAM(this);} - - // Methods - virtual int Cardinality(PGLOBAL g); - virtual bool AllocateBuffer(PGLOBAL g); - virtual int ReadBuffer(PGLOBAL g); - virtual int WriteBuffer(PGLOBAL g); - - protected: - // No additional Members - }; // end of class ZIXFAM +class DllExport ZIPFAM : public MAPFAM { + friend class ZPXFAM; +public: + // Constructors + ZIPFAM(PDOSDEF tdp); + ZIPFAM(PZIPFAM txfp); + ZIPFAM(PDOSDEF tdp, PZPXFAM txfp); + + // Implementation + virtual AMT GetAmType(void) { return TYPE_AM_ZIP; } + virtual PTXF Duplicate(PGLOBAL g) { return (PTXF) new(g)ZIPFAM(this); } + + // Methods + virtual int Cardinality(PGLOBAL g); + virtual int GetFileLength(PGLOBAL g); +//virtual int MaxBlkSize(PGLOBAL g, int s) {return s;} + virtual bool OpenTableFile(PGLOBAL g); + virtual bool DeferReading(void) { return false; } + virtual int GetNext(PGLOBAL g); +//virtual int ReadBuffer(PGLOBAL g); +//virtual int WriteBuffer(PGLOBAL g); +//virtual int DeleteRecords(PGLOBAL g, int irc); +//virtual void CloseTableFile(PGLOBAL g, bool abort); + +protected: + // Members + ZIPUTIL *zutp; + PSZ target; + bool mul; +}; // end of ZIPFAM /***********************************************************************/ -/* This is the DOS/UNIX Access Method class declaration for PlugDB */ -/* fixed/variable files compressed using the zlib library functions. */ -/* Physically these are written and read using the same technique */ -/* than blocked variable files, only the contain of each block is */ -/* compressed using the deflate zlib function. The purpose of this */ -/* specific format is to have a fast mechanism for direct access of */ -/* records so blocked optimization is fast and direct access (joins) */ -/* is allowed. Note that the block length is written ahead of each */ -/* block to enable reading when optimization file is not available. */ +/* This is the fixed ZIP file access method. */ /***********************************************************************/ -class DllExport ZLBFAM : public BLKFAM { - public: - // Constructor - ZLBFAM(PDOSDEF tdp); - ZLBFAM(PZLBFAM txfp); - - // Implementation - virtual AMT GetAmType(void) {return TYPE_AM_ZLIB;} - virtual int GetPos(void); - virtual int GetNextPos(void); - virtual PTXF Duplicate(PGLOBAL g) - {return (PTXF)new(g) ZLBFAM(this);} - inline void SetOptimized(bool b) {Optimized = b;} - - // Methods - virtual int GetFileLength(PGLOBAL g); - virtual bool SetPos(PGLOBAL g, int recpos); - virtual bool AllocateBuffer(PGLOBAL g); - virtual int ReadBuffer(PGLOBAL g); - virtual int WriteBuffer(PGLOBAL g); - virtual void CloseTableFile(PGLOBAL g, bool abort); - virtual void Rewind(void); - - protected: - bool WriteCompressedBuffer(PGLOBAL g); - int ReadCompressedBuffer(PGLOBAL g, void *rdbuf); - - // Members - z_streamp Zstream; // Compression/decompression stream - Byte *Zbuffer; // Compressed block buffer - int *Zlenp; // Pointer to block length - bool Optimized; // true when opt file is available - }; // end of class ZLBFAM +class DllExport ZPXFAM : public MPXFAM { + friend class ZIPFAM; +public: + // Constructors + ZPXFAM(PDOSDEF tdp); + ZPXFAM(PZPXFAM txfp); + + // Implementation + virtual AMT GetAmType(void) { return TYPE_AM_ZIP; } + virtual PTXF Duplicate(PGLOBAL g) { return (PTXF) new(g)ZPXFAM(this); } + + // Methods + virtual int GetFileLength(PGLOBAL g); + virtual int Cardinality(PGLOBAL g); + virtual bool OpenTableFile(PGLOBAL g); + virtual int GetNext(PGLOBAL g); +//virtual int ReadBuffer(PGLOBAL g); + +protected: + // Members + ZIPUTIL *zutp; + PSZ target; + bool mul; +}; // end of ZPXFAM #endif // __FILAMZIP_H diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index cf945a73f46..33f02d8338c 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -171,9 +171,9 @@ #define JSONMAX 10 // JSON Default max grp size extern "C" { - char version[]= "Version 1.04.0008 August 10, 2016"; + char version[]= "Version 1.05.0001 December 13, 2016"; #if defined(__WIN__) - char compver[]= "Version 1.04.0008 " __DATE__ " " __TIME__; + char compver[]= "Version 1.05.0001 " __DATE__ " " __TIME__; char slash= '\\'; #else // !__WIN__ char slash= '/'; @@ -512,13 +512,13 @@ ha_create_table_option connect_table_option_list[]= HA_TOPTION_NUMBER("QUOTED", quoted, (ulonglong) -1, 0, 3, 1), HA_TOPTION_NUMBER("ENDING", ending, (ulonglong) -1, 0, INT_MAX32, 1), HA_TOPTION_NUMBER("COMPRESS", compressed, 0, 0, 2, 1), -//HA_TOPTION_BOOL("COMPRESS", compressed, 0), HA_TOPTION_BOOL("MAPPED", mapped, 0), HA_TOPTION_BOOL("HUGE", huge, 0), HA_TOPTION_BOOL("SPLIT", split, 0), HA_TOPTION_BOOL("READONLY", readonly, 0), HA_TOPTION_BOOL("SEPINDEX", sepindex, 0), - HA_TOPTION_END + HA_TOPTION_BOOL("ZIPPED", zipped, 0), + HA_TOPTION_END }; @@ -532,7 +532,6 @@ ha_create_table_option connect_field_option_list[]= { HA_FOPTION_NUMBER("FLAG", offset, (ulonglong) -1, 0, INT_MAX32, 1), HA_FOPTION_NUMBER("MAX_DIST", freq, 0, 0, INT_MAX32, 1), // BLK_INDX -//HA_FOPTION_NUMBER("DISTRIB", opt, 0, 0, 2, 1), // used for BLK_INDX HA_FOPTION_NUMBER("FIELD_LENGTH", fldlen, 0, 0, INT_MAX32, 1), HA_FOPTION_STRING("DATE_FORMAT", dateformat), HA_FOPTION_STRING("FIELD_FORMAT", fieldformat), @@ -678,7 +677,6 @@ static int connect_init_func(void *p) connect_hton= (handlerton *)p; connect_hton->state= SHOW_OPTION_YES; connect_hton->create= connect_create_handler; -//connect_hton->flags= HTON_TEMPORARY_NOT_SUPPORTED | HTON_NO_PARTITION; connect_hton->flags= HTON_TEMPORARY_NOT_SUPPORTED; connect_hton->table_options= connect_table_option_list; connect_hton->field_options= connect_field_option_list; @@ -1135,7 +1133,9 @@ bool GetBooleanTableOption(PGLOBAL g, PTOS options, char *opname, bool bdef) opval= options->sepindex; else if (!stricmp(opname, "Header")) opval= (options->header != 0); // Is Boolean for some table types - else if (options->oplist) + else if (!stricmp(opname, "Zipped")) + opval = options->zipped; + else if (options->oplist) if ((pv= GetListOption(g, opname, options->oplist))) opval= (!*pv || *pv == 'y' || *pv == 'Y' || atoi(pv) != 0); @@ -1242,8 +1242,10 @@ char *ha_connect::GetStringOption(char *opname, char *sdef) if (opval && (!stricmp(opname, "connect") || !stricmp(opname, "tabname") - || !stricmp(opname, "filename"))) - opval = GetRealString(opval); + || !stricmp(opname, "filename") + || !stricmp(opname, "optname") + || !stricmp(opname, "entry"))) + opval = GetRealString(opval); if (!opval) { if (sdef && !strcmp(sdef, "*")) { @@ -4165,7 +4167,8 @@ bool ha_connect::check_privileges(THD *thd, PTOS options, char *dbn, bool quick) case TAB_DIR: case TAB_MAC: case TAB_WMI: - case TAB_OEM: + case TAB_ZIP: + case TAB_OEM: #ifdef NO_EMBEDDED_ACCESS_CHECKS return false; #endif @@ -5173,13 +5176,13 @@ static int connect_assisted_discovery(handlerton *, THD* thd, char v=0, spc= ',', qch= 0; const char *fncn= "?"; const char *user, *fn, *db, *host, *pwd, *sep, *tbl, *src; - const char *col, *ocl, *rnk, *pic, *fcl, *skc; + const char *col, *ocl, *rnk, *pic, *fcl, *skc, *zfn; char *tab, *dsn, *shm, *dpath; #if defined(__WIN__) char *nsp= NULL, *cls= NULL; #endif // __WIN__ - int port= 0, hdr= 0, mxr= 0, mxe= 0, rc= 0; - int cop __attribute__((unused))= 0, lrecl= 0; +//int hdr, mxe; + int port = 0, mxr = 0, rc = 0, mul = 0, lrecl = 0; #if defined(ODBC_SUPPORT) POPARM sop= NULL; char *ucnc= NULL; @@ -5190,7 +5193,8 @@ static int connect_assisted_discovery(handlerton *, THD* thd, PJPARM sjp= NULL; char *driver= NULL; char *url= NULL; - char *tabtyp = NULL; +//char *prop= NULL; + char *tabtyp= NULL; #endif // JDBC_SUPPORT uint tm, fnc= FNC_NO, supfnc= (FNC_NO | FNC_COL); bool bif, ok= false, dbf= false; @@ -5210,7 +5214,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, if (!g) return HA_ERR_INTERNAL_ERROR; - user= host= pwd= tbl= src= col= ocl= pic= fcl= skc= rnk= dsn= NULL; + user= host= pwd= tbl= src= col= ocl= pic= fcl= skc= rnk= zfn= dsn= NULL; // Get the useful create options ttp= GetTypeID(topt->type); @@ -5223,7 +5227,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, sep= topt->separator; spc= (!sep) ? ',' : *sep; qch= topt->qchar ? *topt->qchar : (signed)topt->quoted >= 0 ? '"' : 0; - hdr= (int)topt->header; + mul = (int)topt->multiple; tbl= topt->tablist; col= topt->colist; @@ -5256,13 +5260,16 @@ static int connect_assisted_discovery(handlerton *, THD* thd, #if defined(JDBC_SUPPORT) driver= GetListOption(g, "Driver", topt->oplist, NULL); // url= GetListOption(g, "URL", topt->oplist, NULL); +// prop = GetListOption(g, "Properties", topt->oplist, NULL); tabtyp = GetListOption(g, "Tabtype", topt->oplist, NULL); #endif // JDBC_SUPPORT - mxe= atoi(GetListOption(g,"maxerr", topt->oplist, "0")); #if defined(PROMPT_OK) cop= atoi(GetListOption(g, "checkdsn", topt->oplist, "0")); #endif // PROMPT_OK - } else { +#if defined(ZIP_SUPPORT) + zfn = GetListOption(g, "Zipfile", topt->oplist, NULL); +#endif // ZIP_SUPPORT + } else { host= "localhost"; user= (ttp == TAB_ODBC ? NULL : "root"); } // endif option_list @@ -5366,6 +5373,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, jdef->SetName(create_info->alias); sjp= (PJPARM)PlugSubAlloc(g, NULL, sizeof(JDBCPARM)); sjp->Driver= driver; +// sjp->Properties = prop; sjp->Fsize= 0; sjp->Scrollable= false; @@ -5468,7 +5476,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, case TAB_XML: #endif // LIBXML2_SUPPORT || DOMDOC_SUPPORT case TAB_JSON: - if (!fn) + if (!fn && !zfn && !mul) sprintf(g->Message, "Missing %s file name", topt->type); else ok= true; @@ -5582,7 +5590,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, NULL, port, fnc == FNC_COL); break; case TAB_CSV: - qrp= CSVColumns(g, dpath, fn, spc, qch, hdr, mxe, fnc == FNC_COL); + qrp = CSVColumns(g, dpath, topt, fnc == FNC_COL); break; #if defined(__WIN__) case TAB_WMI: @@ -6947,7 +6955,7 @@ maria_declare_plugin(connect) 0x0104, /* version number (1.04) */ NULL, /* status variables */ connect_system_variables, /* system variables */ - "1.04.0008", /* string version */ + "1.05.0001", /* string version */ MariaDB_PLUGIN_MATURITY_GAMMA /* maturity */ } maria_declare_plugin_end; diff --git a/storage/connect/ha_connect.h b/storage/connect/ha_connect.h index 60194ac0e3c..3d9ff967618 100644 --- a/storage/connect/ha_connect.h +++ b/storage/connect/ha_connect.h @@ -83,42 +83,9 @@ extern handlerton *connect_hton; These can be specified in the CREATE TABLE: CREATE TABLE ( ... ) {...here...} -*/ -#if 0 // moved to mycat.h -typedef struct ha_table_option_struct TOS, *PTOS; - -struct ha_table_option_struct { - const char *type; - const char *filename; - const char *optname; - const char *tabname; - const char *tablist; - const char *dbname; - const char *separator; -//const char *connect; - const char *qchar; - const char *module; - const char *subtype; - const char *catfunc; - const char *srcdef; - const char *colist; - const char *oplist; - const char *data_charset; - ulonglong lrecl; - ulonglong elements; -//ulonglong estimate; - ulonglong multiple; - ulonglong header; - ulonglong quoted; - ulonglong ending; - ulonglong compressed; - bool mapped; - bool huge; - bool split; - bool readonly; - bool sepindex; - }; -#endif // 0 + + ------ Was moved to mycat.h ------ + */ /** structure for CREATE TABLE options (field options) diff --git a/storage/connect/ioapi.c b/storage/connect/ioapi.c new file mode 100644 index 00000000000..7f5c191b2af --- /dev/null +++ b/storage/connect/ioapi.c @@ -0,0 +1,247 @@ +/* ioapi.h -- IO base function header for compress/uncompress .zip + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications for Zip64 support + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + +*/ + +#if defined(_WIN32) && (!(defined(_CRT_SECURE_NO_WARNINGS))) + #define _CRT_SECURE_NO_WARNINGS +#endif + +#if defined(__APPLE__) || defined(IOAPI_NO_64) +// In darwin and perhaps other BSD variants off_t is a 64 bit value, hence no need for specific 64 bit functions +#define FOPEN_FUNC(filename, mode) fopen(filename, mode) +#define FTELLO_FUNC(stream) ftello(stream) +#define FSEEKO_FUNC(stream, offset, origin) fseeko(stream, offset, origin) +#else +#define FOPEN_FUNC(filename, mode) fopen64(filename, mode) +#define FTELLO_FUNC(stream) ftello64(stream) +#define FSEEKO_FUNC(stream, offset, origin) fseeko64(stream, offset, origin) +#endif + + +#include "ioapi.h" + +voidpf call_zopen64 (const zlib_filefunc64_32_def* pfilefunc,const void*filename,int mode) +{ + if (pfilefunc->zfile_func64.zopen64_file != NULL) + return (*(pfilefunc->zfile_func64.zopen64_file)) (pfilefunc->zfile_func64.opaque,filename,mode); + else + { + return (*(pfilefunc->zopen32_file))(pfilefunc->zfile_func64.opaque,(const char*)filename,mode); + } +} + +long call_zseek64 (const zlib_filefunc64_32_def* pfilefunc,voidpf filestream, ZPOS64_T offset, int origin) +{ + if (pfilefunc->zfile_func64.zseek64_file != NULL) + return (*(pfilefunc->zfile_func64.zseek64_file)) (pfilefunc->zfile_func64.opaque,filestream,offset,origin); + else + { + uLong offsetTruncated = (uLong)offset; + if (offsetTruncated != offset) + return -1; + else + return (*(pfilefunc->zseek32_file))(pfilefunc->zfile_func64.opaque,filestream,offsetTruncated,origin); + } +} + +ZPOS64_T call_ztell64 (const zlib_filefunc64_32_def* pfilefunc,voidpf filestream) +{ + if (pfilefunc->zfile_func64.zseek64_file != NULL) + return (*(pfilefunc->zfile_func64.ztell64_file)) (pfilefunc->zfile_func64.opaque,filestream); + else + { + uLong tell_uLong = (*(pfilefunc->ztell32_file))(pfilefunc->zfile_func64.opaque,filestream); + if ((tell_uLong) == MAXU32) + return (ZPOS64_T)-1; + else + return tell_uLong; + } +} + +void fill_zlib_filefunc64_32_def_from_filefunc32(zlib_filefunc64_32_def* p_filefunc64_32,const zlib_filefunc_def* p_filefunc32) +{ + p_filefunc64_32->zfile_func64.zopen64_file = NULL; + p_filefunc64_32->zopen32_file = p_filefunc32->zopen_file; + p_filefunc64_32->zfile_func64.zerror_file = p_filefunc32->zerror_file; + p_filefunc64_32->zfile_func64.zread_file = p_filefunc32->zread_file; + p_filefunc64_32->zfile_func64.zwrite_file = p_filefunc32->zwrite_file; + p_filefunc64_32->zfile_func64.ztell64_file = NULL; + p_filefunc64_32->zfile_func64.zseek64_file = NULL; + p_filefunc64_32->zfile_func64.zclose_file = p_filefunc32->zclose_file; + p_filefunc64_32->zfile_func64.zerror_file = p_filefunc32->zerror_file; + p_filefunc64_32->zfile_func64.opaque = p_filefunc32->opaque; + p_filefunc64_32->zseek32_file = p_filefunc32->zseek_file; + p_filefunc64_32->ztell32_file = p_filefunc32->ztell_file; +} + + + +static voidpf ZCALLBACK fopen_file_func OF((voidpf opaque, const char* filename, int mode)); +static uLong ZCALLBACK fread_file_func OF((voidpf opaque, voidpf stream, void* buf, uLong size)); +static uLong ZCALLBACK fwrite_file_func OF((voidpf opaque, voidpf stream, const void* buf,uLong size)); +static ZPOS64_T ZCALLBACK ftell64_file_func OF((voidpf opaque, voidpf stream)); +static long ZCALLBACK fseek64_file_func OF((voidpf opaque, voidpf stream, ZPOS64_T offset, int origin)); +static int ZCALLBACK fclose_file_func OF((voidpf opaque, voidpf stream)); +static int ZCALLBACK ferror_file_func OF((voidpf opaque, voidpf stream)); + +static voidpf ZCALLBACK fopen_file_func (voidpf opaque, const char* filename, int mode) +{ + FILE* file = NULL; + const char* mode_fopen = NULL; + if ((mode & ZLIB_FILEFUNC_MODE_READWRITEFILTER)==ZLIB_FILEFUNC_MODE_READ) + mode_fopen = "rb"; + else + if (mode & ZLIB_FILEFUNC_MODE_EXISTING) + mode_fopen = "r+b"; + else + if (mode & ZLIB_FILEFUNC_MODE_CREATE) + mode_fopen = "wb"; + + if ((filename!=NULL) && (mode_fopen != NULL)) + file = fopen(filename, mode_fopen); + return file; +} + +static voidpf ZCALLBACK fopen64_file_func (voidpf opaque, const void* filename, int mode) +{ + FILE* file = NULL; + const char* mode_fopen = NULL; + if ((mode & ZLIB_FILEFUNC_MODE_READWRITEFILTER)==ZLIB_FILEFUNC_MODE_READ) + mode_fopen = "rb"; + else + if (mode & ZLIB_FILEFUNC_MODE_EXISTING) + mode_fopen = "r+b"; + else + if (mode & ZLIB_FILEFUNC_MODE_CREATE) + mode_fopen = "wb"; + + if ((filename!=NULL) && (mode_fopen != NULL)) + file = FOPEN_FUNC((const char*)filename, mode_fopen); + return file; +} + + +static uLong ZCALLBACK fread_file_func (voidpf opaque, voidpf stream, void* buf, uLong size) +{ + uLong ret; + ret = (uLong)fread(buf, 1, (size_t)size, (FILE *)stream); + return ret; +} + +static uLong ZCALLBACK fwrite_file_func (voidpf opaque, voidpf stream, const void* buf, uLong size) +{ + uLong ret; + ret = (uLong)fwrite(buf, 1, (size_t)size, (FILE *)stream); + return ret; +} + +static long ZCALLBACK ftell_file_func (voidpf opaque, voidpf stream) +{ + long ret; + ret = ftell((FILE *)stream); + return ret; +} + + +static ZPOS64_T ZCALLBACK ftell64_file_func (voidpf opaque, voidpf stream) +{ + ZPOS64_T ret; + ret = FTELLO_FUNC((FILE *)stream); + return ret; +} + +static long ZCALLBACK fseek_file_func (voidpf opaque, voidpf stream, uLong offset, int origin) +{ + int fseek_origin=0; + long ret; + switch (origin) + { + case ZLIB_FILEFUNC_SEEK_CUR : + fseek_origin = SEEK_CUR; + break; + case ZLIB_FILEFUNC_SEEK_END : + fseek_origin = SEEK_END; + break; + case ZLIB_FILEFUNC_SEEK_SET : + fseek_origin = SEEK_SET; + break; + default: return -1; + } + ret = 0; + if (fseek((FILE *)stream, offset, fseek_origin) != 0) + ret = -1; + return ret; +} + +static long ZCALLBACK fseek64_file_func (voidpf opaque, voidpf stream, ZPOS64_T offset, int origin) +{ + int fseek_origin=0; + long ret; + switch (origin) + { + case ZLIB_FILEFUNC_SEEK_CUR : + fseek_origin = SEEK_CUR; + break; + case ZLIB_FILEFUNC_SEEK_END : + fseek_origin = SEEK_END; + break; + case ZLIB_FILEFUNC_SEEK_SET : + fseek_origin = SEEK_SET; + break; + default: return -1; + } + ret = 0; + + if(FSEEKO_FUNC((FILE *)stream, offset, fseek_origin) != 0) + ret = -1; + + return ret; +} + + +static int ZCALLBACK fclose_file_func (voidpf opaque, voidpf stream) +{ + int ret; + ret = fclose((FILE *)stream); + return ret; +} + +static int ZCALLBACK ferror_file_func (voidpf opaque, voidpf stream) +{ + int ret; + ret = ferror((FILE *)stream); + return ret; +} + +void fill_fopen_filefunc (pzlib_filefunc_def) + zlib_filefunc_def* pzlib_filefunc_def; +{ + pzlib_filefunc_def->zopen_file = fopen_file_func; + pzlib_filefunc_def->zread_file = fread_file_func; + pzlib_filefunc_def->zwrite_file = fwrite_file_func; + pzlib_filefunc_def->ztell_file = ftell_file_func; + pzlib_filefunc_def->zseek_file = fseek_file_func; + pzlib_filefunc_def->zclose_file = fclose_file_func; + pzlib_filefunc_def->zerror_file = ferror_file_func; + pzlib_filefunc_def->opaque = NULL; +} + +void fill_fopen64_filefunc (zlib_filefunc64_def* pzlib_filefunc_def) +{ + pzlib_filefunc_def->zopen64_file = fopen64_file_func; + pzlib_filefunc_def->zread_file = fread_file_func; + pzlib_filefunc_def->zwrite_file = fwrite_file_func; + pzlib_filefunc_def->ztell64_file = ftell64_file_func; + pzlib_filefunc_def->zseek64_file = fseek64_file_func; + pzlib_filefunc_def->zclose_file = fclose_file_func; + pzlib_filefunc_def->zerror_file = ferror_file_func; + pzlib_filefunc_def->opaque = NULL; +} diff --git a/storage/connect/ioapi.h b/storage/connect/ioapi.h new file mode 100644 index 00000000000..8dcbdb06e35 --- /dev/null +++ b/storage/connect/ioapi.h @@ -0,0 +1,208 @@ +/* ioapi.h -- IO base function header for compress/uncompress .zip + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications for Zip64 support + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + + Changes + + Oct-2009 - Defined ZPOS64_T to fpos_t on windows and u_int64_t on linux. (might need to find a better why for this) + Oct-2009 - Change to fseeko64, ftello64 and fopen64 so large files would work on linux. + More if/def section may be needed to support other platforms + Oct-2009 - Defined fxxxx64 calls to normal fopen/ftell/fseek so they would compile on windows. + (but you should use iowin32.c for windows instead) + +*/ + +#ifndef _ZLIBIOAPI64_H +#define _ZLIBIOAPI64_H + +#if (!defined(_WIN32)) && (!defined(WIN32)) && (!defined(__APPLE__)) + + // Linux needs this to support file operation on files larger then 4+GB + // But might need better if/def to select just the platforms that needs them. + + #ifndef __USE_FILE_OFFSET64 + #define __USE_FILE_OFFSET64 + #endif + #ifndef __USE_LARGEFILE64 + #define __USE_LARGEFILE64 + #endif + #ifndef _LARGEFILE64_SOURCE + #define _LARGEFILE64_SOURCE + #endif + #ifndef _FILE_OFFSET_BIT + #define _FILE_OFFSET_BIT 64 + #endif + +#endif + +#include <stdio.h> +#include <stdlib.h> +#include "zlib.h" + +#if defined(USE_FILE32API) +#define fopen64 fopen +#define ftello64 ftell +#define fseeko64 fseek +#else +#ifdef __FreeBSD__ +#define fopen64 fopen +#define ftello64 ftello +#define fseeko64 fseeko +#endif +#ifdef _MSC_VER + #define fopen64 fopen + #if (_MSC_VER >= 1400) && (!(defined(NO_MSCVER_FILE64_FUNC))) + #define ftello64 _ftelli64 + #define fseeko64 _fseeki64 + #else // old MSC + #define ftello64 ftell + #define fseeko64 fseek + #endif +#endif +#endif + +/* +#ifndef ZPOS64_T + #ifdef _WIN32 + #define ZPOS64_T fpos_t + #else + #include <stdint.h> + #define ZPOS64_T uint64_t + #endif +#endif +*/ + +#ifdef HAVE_MINIZIP64_CONF_H +#include "mz64conf.h" +#endif + +/* a type choosen by DEFINE */ +#ifdef HAVE_64BIT_INT_CUSTOM +typedef 64BIT_INT_CUSTOM_TYPE ZPOS64_T; +#else +#ifdef HAS_STDINT_H +#include "stdint.h" +typedef uint64_t ZPOS64_T; +#else + +/* Maximum unsigned 32-bit value used as placeholder for zip64 */ +#define MAXU32 0xffffffff + +#if defined(_MSC_VER) || defined(__BORLANDC__) +typedef unsigned __int64 ZPOS64_T; +#else +typedef unsigned long long int ZPOS64_T; +#endif +#endif +#endif + + + +#ifdef __cplusplus +extern "C" { +#endif + + +#define ZLIB_FILEFUNC_SEEK_CUR (1) +#define ZLIB_FILEFUNC_SEEK_END (2) +#define ZLIB_FILEFUNC_SEEK_SET (0) + +#define ZLIB_FILEFUNC_MODE_READ (1) +#define ZLIB_FILEFUNC_MODE_WRITE (2) +#define ZLIB_FILEFUNC_MODE_READWRITEFILTER (3) + +#define ZLIB_FILEFUNC_MODE_EXISTING (4) +#define ZLIB_FILEFUNC_MODE_CREATE (8) + + +#ifndef ZCALLBACK + #if (defined(WIN32) || defined(_WIN32) || defined (WINDOWS) || defined (_WINDOWS)) && defined(CALLBACK) && defined (USEWINDOWS_CALLBACK) + #define ZCALLBACK CALLBACK + #else + #define ZCALLBACK + #endif +#endif + + + + +typedef voidpf (ZCALLBACK *open_file_func) OF((voidpf opaque, const char* filename, int mode)); +typedef uLong (ZCALLBACK *read_file_func) OF((voidpf opaque, voidpf stream, void* buf, uLong size)); +typedef uLong (ZCALLBACK *write_file_func) OF((voidpf opaque, voidpf stream, const void* buf, uLong size)); +typedef int (ZCALLBACK *close_file_func) OF((voidpf opaque, voidpf stream)); +typedef int (ZCALLBACK *testerror_file_func) OF((voidpf opaque, voidpf stream)); + +typedef long (ZCALLBACK *tell_file_func) OF((voidpf opaque, voidpf stream)); +typedef long (ZCALLBACK *seek_file_func) OF((voidpf opaque, voidpf stream, uLong offset, int origin)); + + +/* here is the "old" 32 bits structure structure */ +typedef struct zlib_filefunc_def_s +{ + open_file_func zopen_file; + read_file_func zread_file; + write_file_func zwrite_file; + tell_file_func ztell_file; + seek_file_func zseek_file; + close_file_func zclose_file; + testerror_file_func zerror_file; + voidpf opaque; +} zlib_filefunc_def; + +typedef ZPOS64_T (ZCALLBACK *tell64_file_func) OF((voidpf opaque, voidpf stream)); +typedef long (ZCALLBACK *seek64_file_func) OF((voidpf opaque, voidpf stream, ZPOS64_T offset, int origin)); +typedef voidpf (ZCALLBACK *open64_file_func) OF((voidpf opaque, const void* filename, int mode)); + +typedef struct zlib_filefunc64_def_s +{ + open64_file_func zopen64_file; + read_file_func zread_file; + write_file_func zwrite_file; + tell64_file_func ztell64_file; + seek64_file_func zseek64_file; + close_file_func zclose_file; + testerror_file_func zerror_file; + voidpf opaque; +} zlib_filefunc64_def; + +void fill_fopen64_filefunc OF((zlib_filefunc64_def* pzlib_filefunc_def)); +void fill_fopen_filefunc OF((zlib_filefunc_def* pzlib_filefunc_def)); + +/* now internal definition, only for zip.c and unzip.h */ +typedef struct zlib_filefunc64_32_def_s +{ + zlib_filefunc64_def zfile_func64; + open_file_func zopen32_file; + tell_file_func ztell32_file; + seek_file_func zseek32_file; +} zlib_filefunc64_32_def; + + +#define ZREAD64(filefunc,filestream,buf,size) ((*((filefunc).zfile_func64.zread_file)) ((filefunc).zfile_func64.opaque,filestream,buf,size)) +#define ZWRITE64(filefunc,filestream,buf,size) ((*((filefunc).zfile_func64.zwrite_file)) ((filefunc).zfile_func64.opaque,filestream,buf,size)) +//#define ZTELL64(filefunc,filestream) ((*((filefunc).ztell64_file)) ((filefunc).opaque,filestream)) +//#define ZSEEK64(filefunc,filestream,pos,mode) ((*((filefunc).zseek64_file)) ((filefunc).opaque,filestream,pos,mode)) +#define ZCLOSE64(filefunc,filestream) ((*((filefunc).zfile_func64.zclose_file)) ((filefunc).zfile_func64.opaque,filestream)) +#define ZERROR64(filefunc,filestream) ((*((filefunc).zfile_func64.zerror_file)) ((filefunc).zfile_func64.opaque,filestream)) + +voidpf call_zopen64 OF((const zlib_filefunc64_32_def* pfilefunc,const void*filename,int mode)); +long call_zseek64 OF((const zlib_filefunc64_32_def* pfilefunc,voidpf filestream, ZPOS64_T offset, int origin)); +ZPOS64_T call_ztell64 OF((const zlib_filefunc64_32_def* pfilefunc,voidpf filestream)); + +void fill_zlib_filefunc64_32_def_from_filefunc32(zlib_filefunc64_32_def* p_filefunc64_32,const zlib_filefunc_def* p_filefunc32); + +#define ZOPEN64(filefunc,filename,mode) (call_zopen64((&(filefunc)),(filename),(mode))) +#define ZTELL64(filefunc,filestream) (call_ztell64((&(filefunc)),(filestream))) +#define ZSEEK64(filefunc,filestream,pos,mode) (call_zseek64((&(filefunc)),(filestream),(pos),(mode))) + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/storage/connect/jdbccat.h b/storage/connect/jdbccat.h index 37f33d7063d..7108aa376ce 100644 --- a/storage/connect/jdbccat.h +++ b/storage/connect/jdbccat.h @@ -8,6 +8,7 @@ typedef struct jdbc_parms { char *Url; // Driver URL char *User; // User connect info char *Pwd; // Password connect info +//char *Properties; // Connection property list //int Cto; // Connect timeout //int Qto; // Query timeout int Fsize; // Fetch size diff --git a/storage/connect/jdbconn.cpp b/storage/connect/jdbconn.cpp index dca9bd0eac4..a69f84a94a1 100644 --- a/storage/connect/jdbconn.cpp +++ b/storage/connect/jdbconn.cpp @@ -55,9 +55,8 @@ #if defined(__WIN__) extern "C" HINSTANCE s_hModule; // Saved module handle -#else // !__WIN__ +#endif // __WIN__ #define nullptr 0 -#endif // !__WIN__ TYPCONV GetTypeConv(); int GetConvSize(); @@ -525,10 +524,10 @@ JDBConn::JDBConn(PGLOBAL g, TDBJDBC *tdbp) m_Wrap = strcat(strcpy(wn, "wrappers/"), m_Wrap); } // endif m_Wrap - m_Driver = NULL; - m_Url = NULL; - m_User = NULL; - m_Pwd = NULL; +//m_Driver = NULL; +//m_Url = NULL; +//m_User = NULL; +//m_Pwd = NULL; m_Ncol = 0; m_Aff = 0; m_Rows = 0; @@ -772,7 +771,7 @@ bool JDBConn::GetJVM(PGLOBAL g) /***********************************************************************/ int JDBConn::Open(PJPARM sop) { - + int irc = RC_FX; bool err = false; jboolean jt = (trace > 0); PGLOBAL& g = m_G; @@ -865,30 +864,37 @@ int JDBConn::Open(PJPARM sop) switch (rc) { case JNI_OK: strcpy(g->Message, "VM successfully created"); + irc = RC_OK; break; case JNI_ERR: strcpy(g->Message, "Initialising JVM failed: unknown error"); - return RC_FX; + break; case JNI_EDETACHED: strcpy(g->Message, "Thread detached from the VM"); - return RC_FX; + break; case JNI_EVERSION: strcpy(g->Message, "JNI version error"); - return RC_FX; + break; case JNI_ENOMEM: strcpy(g->Message, "Not enough memory"); - return RC_FX; + break; case JNI_EEXIST: strcpy(g->Message, "VM already created"); - return RC_FX; + break; case JNI_EINVAL: strcpy(g->Message, "Invalid arguments"); - return RC_FX; + break; default: - sprintf(g->Message, "Unknown return code %d", rc); - return RC_FX; + sprintf(g->Message, "Unknown return code %d", (int)rc); + break; } // endswitch rc + if (trace) + htrc("%s\n", g->Message); + + if (irc != RC_OK) + return irc; + //=============== Display JVM version =============== jint ver = env->GetVersion(); printf("JVM Version %d.%d\n", ((ver>>16)&0x0f), (ver&0x0f)); @@ -978,10 +984,10 @@ int JDBConn::Open(PJPARM sop) jobjectArray parms = env->NewObjectArray(4, // constructs java array of 4 env->FindClass("java/lang/String"), NULL); // Strings - m_Driver = sop->Driver; - m_Url = sop->Url; - m_User = sop->User; - m_Pwd = sop->Pwd; +//m_Driver = sop->Driver; +//m_Url = sop->Url; +//m_User = sop->User; +//m_Pwd = sop->Pwd; m_Scrollable = sop->Scrollable; m_RowsetSize = sop->Fsize; //m_LoginTimeout = sop->Cto; @@ -989,17 +995,20 @@ int JDBConn::Open(PJPARM sop) //m_UseCnc = sop->UseCnc; // change some elements - if (m_Driver) - env->SetObjectArrayElement(parms, 0, env->NewStringUTF(m_Driver)); + if (sop->Driver) + env->SetObjectArrayElement(parms, 0, env->NewStringUTF(sop->Driver)); - if (m_Url) - env->SetObjectArrayElement(parms, 1, env->NewStringUTF(m_Url)); + if (sop->Url) + env->SetObjectArrayElement(parms, 1, env->NewStringUTF(sop->Url)); - if (m_User) - env->SetObjectArrayElement(parms, 2, env->NewStringUTF(m_User)); + if (sop->User) + env->SetObjectArrayElement(parms, 2, env->NewStringUTF(sop->User)); - if (m_Pwd) - env->SetObjectArrayElement(parms, 3, env->NewStringUTF(m_Pwd)); + if (sop->Pwd) + env->SetObjectArrayElement(parms, 3, env->NewStringUTF(sop->Pwd)); + +//if (sop->Properties) +// env->SetObjectArrayElement(parms, 4, env->NewStringUTF(sop->Properties)); // call method rc = env->CallIntMethod(job, cid, parms, m_RowsetSize, m_Scrollable); @@ -1432,7 +1441,7 @@ bool JDBConn::SetParam(JDBCCOL *colp) PGLOBAL& g = m_G; bool rc = false; PVAL val = colp->GetValue(); - jint n, i = (jint)colp->GetRank(); + jint n, jrc = 0, i = (jint)colp->GetRank(); jshort s; jlong lg; //jfloat f; @@ -1442,69 +1451,74 @@ bool JDBConn::SetParam(JDBCCOL *colp) jstring jst = nullptr; jmethodID dtc, setid = nullptr; - switch (val->GetType()) { - case TYPE_STRING: - if (gmID(g, setid, "SetStringParm", "(ILjava/lang/String;)V")) - return true; - - jst = env->NewStringUTF(val->GetCharValue()); - env->CallVoidMethod(job, setid, i, jst); - break; - case TYPE_INT: - if (gmID(g, setid, "SetIntParm", "(II)V")) + if (val->GetNullable() && val->IsNull()) { + if (gmID(g, setid, "SetNullParm", "(II)I")) return true; - n = (jint)val->GetIntValue(); - env->CallVoidMethod(job, setid, i, n); - break; - case TYPE_TINY: - case TYPE_SHORT: - if (gmID(g, setid, "SetShortParm", "(IS)V")) - return true; + jrc = env->CallIntMethod(job, setid, i, (jint)GetJDBCType(val->GetType())); + } else switch (val->GetType()) { + case TYPE_STRING: + if (gmID(g, setid, "SetStringParm", "(ILjava/lang/String;)V")) + return true; - s = (jshort)val->GetShortValue(); - env->CallVoidMethod(job, setid, i, s); - break; - case TYPE_BIGINT: - if (gmID(g, setid, "SetBigintParm", "(IJ)V")) - return true; + jst = env->NewStringUTF(val->GetCharValue()); + env->CallVoidMethod(job, setid, i, jst); + break; + case TYPE_INT: + if (gmID(g, setid, "SetIntParm", "(II)V")) + return true; - lg = (jlong)val->GetBigintValue(); - env->CallVoidMethod(job, setid, i, lg); - break; - case TYPE_DOUBLE: - case TYPE_DECIM: - if (gmID(g, setid, "SetDoubleParm", "(ID)V")) - return true; + n = (jint)val->GetIntValue(); + env->CallVoidMethod(job, setid, i, n); + break; + case TYPE_TINY: + case TYPE_SHORT: + if (gmID(g, setid, "SetShortParm", "(IS)V")) + return true; - d = (jdouble)val->GetFloatValue(); - env->CallVoidMethod(job, setid, i, d); - break; - case TYPE_DATE: - if ((dat = env->FindClass("java/sql/Timestamp")) == nullptr) { - strcpy(g->Message, "Cannot find Timestamp class"); - return true; - } else if (!(dtc = env->GetMethodID(dat, "<init>", "(J)V"))) { - strcpy(g->Message, "Cannot find Timestamp class constructor"); - return true; - } // endif's + s = (jshort)val->GetShortValue(); + env->CallVoidMethod(job, setid, i, s); + break; + case TYPE_BIGINT: + if (gmID(g, setid, "SetBigintParm", "(IJ)V")) + return true; - lg = (jlong)val->GetBigintValue() * 1000; + lg = (jlong)val->GetBigintValue(); + env->CallVoidMethod(job, setid, i, lg); + break; + case TYPE_DOUBLE: + case TYPE_DECIM: + if (gmID(g, setid, "SetDoubleParm", "(ID)V")) + return true; - if ((datobj = env->NewObject(dat, dtc, lg)) == nullptr) { - strcpy(g->Message, "Cannot make Timestamp object"); - return true; - } else if (gmID(g, setid, "SetTimestampParm", "(ILjava/sql/Timestamp;)V")) + d = (jdouble)val->GetFloatValue(); + env->CallVoidMethod(job, setid, i, d); + break; + case TYPE_DATE: + if ((dat = env->FindClass("java/sql/Timestamp")) == nullptr) { + strcpy(g->Message, "Cannot find Timestamp class"); + return true; + } else if (!(dtc = env->GetMethodID(dat, "<init>", "(J)V"))) { + strcpy(g->Message, "Cannot find Timestamp class constructor"); + return true; + } // endif's + + lg = (jlong)val->GetBigintValue() * 1000; + + if ((datobj = env->NewObject(dat, dtc, lg)) == nullptr) { + strcpy(g->Message, "Cannot make Timestamp object"); + return true; + } else if (gmID(g, setid, "SetTimestampParm", "(ILjava/sql/Timestamp;)V")) + return true; + + env->CallVoidMethod(job, setid, i, datobj); + break; + default: + sprintf(g->Message, "Parm type %d not supported", val->GetType()); return true; + } // endswitch Type - env->CallVoidMethod(job, setid, i, datobj); - break; - default: - sprintf(g->Message, "Parm type %d not supported", val->GetType()); - return true; - } // endswitch Type - - if (Check()) { + if (Check(jrc)) { sprintf(g->Message, "SetParam: col=%s msg=%s", colp->GetName(), Msg); rc = true; } // endif msg diff --git a/storage/connect/jdbconn.h b/storage/connect/jdbconn.h index 0a1c52d4576..9d428142839 100644 --- a/storage/connect/jdbconn.h +++ b/storage/connect/jdbconn.h @@ -180,9 +180,9 @@ protected: char *Msg; char *m_Wrap; char m_IDQuoteChar[2]; - PSZ m_Driver; - PSZ m_Url; - PSZ m_User; +//PSZ m_Driver; +//PSZ m_Url; +//PSZ m_User; PSZ m_Pwd; int m_Ncol; int m_Aff; diff --git a/storage/connect/jsonudf.cpp b/storage/connect/jsonudf.cpp index 8bddc68e2ae..f9034f25739 100644 --- a/storage/connect/jsonudf.cpp +++ b/storage/connect/jsonudf.cpp @@ -5264,3 +5264,50 @@ char *envar(UDF_INIT *initid, UDF_ARGS *args, char *result, return str; } // end of envar +/*********************************************************************************/ +/* Returns the distinct number of B occurences in A. */ +/*********************************************************************************/ +my_bool countin_init(UDF_INIT *initid, UDF_ARGS *args, char *message) +{ + if (args->arg_count != 2) { + strcpy(message, "This function must have 2 arguments"); + return true; + } else if (args->arg_type[0] != STRING_RESULT) { + strcpy(message, "First argument must be string"); + return true; + } else if (args->arg_type[1] != STRING_RESULT) { + strcpy(message, "Second argument is not a string"); + return true; + } // endif args + + return false; +} // end of countin_init + +long long countin(UDF_INIT *initid, UDF_ARGS *args, char *result, + unsigned long *res_length, char *is_null, char *) +{ + PSZ str1, str2; + char *s; + long long n = 0; + size_t lg; + + lg = (size_t)args->lengths[0]; + s = str1 = (PSZ)malloc(lg + 1); + memcpy(str1, args->args[0], lg); + str1[lg] = 0; + + lg = (size_t)args->lengths[1]; + str2 = (PSZ)malloc(lg + 1); + memcpy(str2, args->args[1], lg); + str2[lg] = 0; + + while (s = strstr(s, str2)) { + n++; + s += lg; + } // endwhile + + free(str1); + free(str2); + return n; +} // end of countin + diff --git a/storage/connect/jsonudf.h b/storage/connect/jsonudf.h index 1406d9f2f2e..d2890421c62 100644 --- a/storage/connect/jsonudf.h +++ b/storage/connect/jsonudf.h @@ -221,6 +221,9 @@ extern "C" { DllExport my_bool envar_init(UDF_INIT*, UDF_ARGS*, char*); DllExport char *envar(UDF_EXEC_ARGS); + + DllExport my_bool countin_init(UDF_INIT*, UDF_ARGS*, char*); + DllExport long long countin(UDF_EXEC_ARGS); } // extern "C" diff --git a/storage/connect/libdoc.cpp b/storage/connect/libdoc.cpp index c2882fc0d7c..2470d37c353 100644 --- a/storage/connect/libdoc.cpp +++ b/storage/connect/libdoc.cpp @@ -1,6 +1,6 @@ /******************************************************************/ /* Implementation of XML document processing using libxml2 */ -/* Author: Olivier Bertrand 2007-2015 */ +/* Author: Olivier Bertrand 2007-2016 */ /******************************************************************/ #include "my_global.h" #include <string.h> @@ -68,8 +68,8 @@ class LIBXMLDOC : public XMLDOCUMENT { virtual void SetNofree(bool b) {Nofreelist = b;} // Methods - virtual bool Initialize(PGLOBAL g); - virtual bool ParseFile(char *fn); + virtual bool Initialize(PGLOBAL g, char *entry, bool zipped); + virtual bool ParseFile(PGLOBAL g, char *fn); virtual bool NewDoc(PGLOBAL g, char *ver); virtual void AddComment(PGLOBAL g, char *com); virtual PXNODE GetRoot(PGLOBAL g); @@ -373,22 +373,33 @@ LIBXMLDOC::LIBXMLDOC(char *nsl, char *nsdf, char *enc, PFBLOCK fp) /******************************************************************/ /* Initialize XML parser and check library compatibility. */ /******************************************************************/ -bool LIBXMLDOC::Initialize(PGLOBAL g) - { +bool LIBXMLDOC::Initialize(PGLOBAL g, char *entry, bool zipped) +{ + if (zipped && InitZip(g, entry)) + return true; + int n = xmlKeepBlanksDefault(1); return MakeNSlist(g); - } // end of Initialize +} // end of Initialize /******************************************************************/ /* Parse the XML file and construct node tree in memory. */ /******************************************************************/ -bool LIBXMLDOC::ParseFile(char *fn) +bool LIBXMLDOC::ParseFile(PGLOBAL g, char *fn) { if (trace) htrc("ParseFile\n"); - if ((Docp = xmlParseFile(fn))) { - if (Docp->encoding) + if (zip) { + // Parse an in memory document + char *xdoc = GetMemDoc(g, fn); + + Docp = (xdoc) ? xmlParseDoc((const xmlChar *)xdoc) : NULL; + } else + Docp = xmlParseFile(fn); + + if (Docp) { + if (Docp->encoding) Encoding = (char*)Docp->encoding; return false; @@ -609,6 +620,7 @@ void LIBXMLDOC::CloseDoc(PGLOBAL g, PFBLOCK xp) } // endif xp CloseXML2File(g, xp, false); + CloseZip(); } // end of Close /******************************************************************/ diff --git a/storage/connect/mycat.cc b/storage/connect/mycat.cc index b4b03e6ba4a..497fe5e1aa8 100644 --- a/storage/connect/mycat.cc +++ b/storage/connect/mycat.cc @@ -16,7 +16,7 @@ /*************** Mycat CC Program Source Code File (.CC) ***************/ /* PROGRAM NAME: MYCAT */ /* ------------- */ -/* Version 1.4 */ +/* Version 1.5 */ /* */ /* Author: Olivier Bertrand 2012 - 2016 */ /* */ @@ -64,7 +64,9 @@ #include "filamtxt.h" #include "tabdos.h" #include "tabfmt.h" +#if defined(VCT_SUPPORT) #include "tabvct.h" +#endif // VCT_SUPPORT #include "tabsys.h" #if defined(__WIN__) #include "tabmac.h" @@ -93,6 +95,9 @@ #if defined(XML_SUPPORT) #include "tabxml.h" #endif // XML_SUPPORT +#if defined(ZIP_SUPPORT) +#include "tabzip.h" +#endif // ZIP_SUPPORT #include "mycat.h" /***********************************************************************/ @@ -152,7 +157,10 @@ TABTYPE GetTypeID(const char *type) #endif : (!stricmp(type, "VIR")) ? TAB_VIR : (!stricmp(type, "JSON")) ? TAB_JSON - : (!stricmp(type, "OEM")) ? TAB_OEM : TAB_NIY; +#ifdef ZIP_SUPPORT + : (!stricmp(type, "ZIP")) ? TAB_ZIP +#endif + : (!stricmp(type, "OEM")) ? TAB_OEM : TAB_NIY; } // end of GetTypeID /***********************************************************************/ @@ -173,6 +181,7 @@ bool IsFileType(TABTYPE type) case TAB_INI: case TAB_VEC: case TAB_JSON: +// case TAB_ZIP: isfile= true; break; default: @@ -549,7 +558,9 @@ PRELDEF MYCAT::MakeTableDesc(PGLOBAL g, PTABLE tablep, LPCSTR am) #if defined(XML_SUPPORT) case TAB_XML: tdp= new(g) XMLDEF; break; #endif // XML_SUPPORT - case TAB_VEC: tdp= new(g) VCTDEF; break; +#if defined(VCT_SUPPORT) + case TAB_VEC: tdp = new(g)VCTDEF; break; +#endif // VCT_SUPPORT #if defined(ODBC_SUPPORT) case TAB_ODBC: tdp= new(g) ODBCDEF; break; #endif // ODBC_SUPPORT @@ -571,7 +582,10 @@ PRELDEF MYCAT::MakeTableDesc(PGLOBAL g, PTABLE tablep, LPCSTR am) #endif // PIVOT_SUPPORT case TAB_VIR: tdp= new(g) VIRDEF; break; case TAB_JSON: tdp= new(g) JSONDEF; break; - default: +#if defined(ZIP_SUPPORT) + case TAB_ZIP: tdp= new(g) ZIPDEF; break; +#endif // ZIP_SUPPORT + default: sprintf(g->Message, MSG(BAD_TABLE_TYPE), am, name); } // endswitch diff --git a/storage/connect/mycat.h b/storage/connect/mycat.h index 05163f08f1b..663b68fd4b9 100644 --- a/storage/connect/mycat.h +++ b/storage/connect/mycat.h @@ -62,6 +62,7 @@ struct ha_table_option_struct { bool split; bool readonly; bool sepindex; + bool zipped; }; // Possible value for catalog functions diff --git a/storage/connect/mysql-test/connect/disabled.def b/storage/connect/mysql-test/connect/disabled.def index 9b4570915c7..5e15e0806ba 100644 --- a/storage/connect/mysql-test/connect/disabled.def +++ b/storage/connect/mysql-test/connect/disabled.def @@ -9,8 +9,7 @@ # Do not use any TAB characters for whitespace. # ############################################################################## -#json_udf_bin : broken upstream in --ps (fixed) -jdbc : Variable settings depend on machine configuration -jdbc_new : Variable settings depend on machine configuration +#jdbc : Variable settings depend on machine configuration +#jdbc_new : Variable settings depend on machine configuration jdbc_oracle : Variable settings depend on machine configuration jdbc_postgresql : Variable settings depend on machine configuration diff --git a/storage/connect/mysql-test/connect/r/jdbc_new.result b/storage/connect/mysql-test/connect/r/jdbc_new.result index 14381b0b11f..5cc4826213d 100644 --- a/storage/connect/mysql-test/connect/r/jdbc_new.result +++ b/storage/connect/mysql-test/connect/r/jdbc_new.result @@ -56,7 +56,7 @@ t1 CREATE TABLE `t1` ( ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='jdbc:mysql://127.0.0.1:SLAVE_PORT/test?user=root' `TABLE_TYPE`='JDBC' SELECT * FROM t1; a b -0 NULL +NULL NULL 0 test00 1 test01 2 test02 @@ -72,7 +72,7 @@ t1 CREATE TABLE `t1` ( ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='jdbc:mysql://127.0.0.1:SLAVE_PORT/test?user=root' `TABLE_TYPE`=JDBC `TABNAME`='t1' SELECT * FROM t1; a b -0 NULL +NULL NULL 0 test00 1 test01 2 test02 @@ -104,7 +104,7 @@ t1 CREATE TABLE `t1` ( ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='jdbc:mysql://127.0.0.1:SLAVE_PORT/test?user=root' `TABLE_TYPE`=JDBC SELECT * FROM t1; a b -0 NULL +NULL NULL 0 0 1 0 2 0 diff --git a/storage/connect/mysql-test/connect/std_data/JdbcMariaDB.jar b/storage/connect/mysql-test/connect/std_data/JdbcMariaDB.jar Binary files differindex 81f91e4465a..8525da74018 100644 --- a/storage/connect/mysql-test/connect/std_data/JdbcMariaDB.jar +++ b/storage/connect/mysql-test/connect/std_data/JdbcMariaDB.jar diff --git a/storage/connect/mysql-test/connect/t/jdbc.test b/storage/connect/mysql-test/connect/t/jdbc.test index 41fd298776b..58a527a3e6b 100644 --- a/storage/connect/mysql-test/connect/t/jdbc.test +++ b/storage/connect/mysql-test/connect/t/jdbc.test @@ -1,3 +1,4 @@ +-- source windows.inc -- source jdbconn.inc SET GLOBAL time_zone='+1:00'; diff --git a/storage/connect/mysql-test/connect/t/jdbc_new.test b/storage/connect/mysql-test/connect/t/jdbc_new.test index d1ad5117b72..5586cf8c027 100644 --- a/storage/connect/mysql-test/connect/t/jdbc_new.test +++ b/storage/connect/mysql-test/connect/t/jdbc_new.test @@ -5,6 +5,7 @@ connect (master,127.0.0.1,root,,test,$MASTER_MYPORT,); connect (slave,127.0.0.1,root,,test,$SLAVE_MYPORT,); connection master; +-- source windows.inc -- source jdbconn.inc connection slave; diff --git a/storage/connect/mysql-test/connect/t/windows.inc b/storage/connect/mysql-test/connect/t/windows.inc new file mode 100644 index 00000000000..88553d8aa59 --- /dev/null +++ b/storage/connect/mysql-test/connect/t/windows.inc @@ -0,0 +1,5 @@ +if (`select convert(@@version_compile_os using latin1) IN ("Win32","Win64","Windows") = 0`) +{ + skip Need windows; +} + diff --git a/storage/connect/plgdbsem.h b/storage/connect/plgdbsem.h index 910ce97f48a..cb408494319 100644 --- a/storage/connect/plgdbsem.h +++ b/storage/connect/plgdbsem.h @@ -1,9 +1,9 @@ /************** PlgDBSem H Declares Source Code File (.H) **************/ -/* Name: PLGDBSEM.H Version 3.6 */ +/* Name: PLGDBSEM.H Version 3.7 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 1998-2015 */ +/* (C) Copyright to the author Olivier BERTRAND 1998-2016 */ /* */ -/* This file contains the PlugDB++ application type definitions. */ +/* This file contains the CONNECT storage engine definitions. */ /***********************************************************************/ /***********************************************************************/ @@ -49,7 +49,8 @@ enum BLKTYP {TYPE_TABLE = 50, /* Table Name/Srcdef/... Block */ TYPE_FB_MAP = 23, /* Mapped file block (storage) */ TYPE_FB_HANDLE = 24, /* File block (handle) */ TYPE_FB_XML = 21, /* DOM XML file block */ - TYPE_FB_XML2 = 27}; /* libxml2 XML file block */ + TYPE_FB_XML2 = 27, /* libxml2 XML file block */ + TYPE_FB_ZIP = 28}; /* ZIP file block */ enum TABTYPE {TAB_UNDEF = 0, /* Table of undefined type */ TAB_DOS = 1, /* Fixed column offset, variable LRECL */ @@ -78,7 +79,8 @@ enum TABTYPE {TAB_UNDEF = 0, /* Table of undefined type */ TAB_JCT = 24, /* Junction tables NIY */ TAB_DMY = 25, /* DMY Dummy tables NIY */ TAB_JDBC = 26, /* Table accessed via JDBC */ - TAB_NIY = 27}; /* Table not implemented yet */ + TAB_ZIP = 27, /* ZIP file info table */ + TAB_NIY = 28}; /* Table not implemented yet */ enum AMT {TYPE_AM_ERROR = 0, /* Type not defined */ TYPE_AM_ROWID = 1, /* ROWID type (special column) */ @@ -123,7 +125,7 @@ enum AMT {TYPE_AM_ERROR = 0, /* Type not defined */ TYPE_AM_PRX = 129, /* PROXY access method type no */ TYPE_AM_XTB = 130, /* SYS table access method type */ TYPE_AM_BLK = 131, /* BLK access method type no */ - TYPE_AM_ZIP = 132, /* ZIP access method type no */ + TYPE_AM_GZ = 132, /* GZ access method type no */ TYPE_AM_ZLIB = 133, /* ZLIB access method type no */ TYPE_AM_JSON = 134, /* JSON access method type no */ TYPE_AM_JSN = 135, /* JSN access method type no */ @@ -140,7 +142,8 @@ enum AMT {TYPE_AM_ERROR = 0, /* Type not defined */ TYPE_AM_MYSQL = 192, /* MYSQL access method type no */ TYPE_AM_MYX = 193, /* MYSQL EXEC access method type */ TYPE_AM_CAT = 195, /* Catalog access method type no */ - TYPE_AM_OUT = 200}; /* Output relations (storage) */ + TYPE_AM_ZIP = 198, /* ZIP access method type no */ + TYPE_AM_OUT = 200}; /* Output relations (storage) */ enum RECFM {RECFM_NAF = -2, /* Not a file */ RECFM_OEM = -1, /* OEM file access method */ diff --git a/storage/connect/plgdbutl.cpp b/storage/connect/plgdbutl.cpp index 13c0dfd1e18..83975c6d8fa 100644 --- a/storage/connect/plgdbutl.cpp +++ b/storage/connect/plgdbutl.cpp @@ -68,6 +68,9 @@ #include "tabcol.h" // header of XTAB and COLUMN classes #include "valblk.h" #include "rcmsg.h" +#ifdef ZIP_SUPPORT +#include "filamzip.h" +#endif // ZIP_SUPPORT /***********************************************************************/ /* DB static variables. */ @@ -934,7 +937,16 @@ int PlugCloseFile(PGLOBAL g __attribute__((unused)), PFBLOCK fp, bool all) CloseXML2File(g, fp, all); break; #endif // LIBXML2_SUPPORT - default: +#ifdef ZIP_SUPPORT + case TYPE_FB_ZIP: + ((ZIPUTIL*)fp->File)->close(); + fp->Memory = NULL; + fp->Mode = MODE_ANY; + fp->Count = 0; + fp->File = NULL; + break; +#endif // ZIP_SUPPORT + default: rc = RC_FX; } // endswitch Type diff --git a/storage/connect/plgxml.cpp b/storage/connect/plgxml.cpp index 3061a6d697e..71b72621b06 100644 --- a/storage/connect/plgxml.cpp +++ b/storage/connect/plgxml.cpp @@ -30,19 +30,51 @@ PXDOC GetLibxmlDoc(PGLOBAL g, char *nsl, char *nsdf, /* XMLDOCUMENT constructor. */ /******************************************************************/ XMLDOCUMENT::XMLDOCUMENT(char *nsl, char *nsdf, char *enc) - { - Namespaces = NULL; +{ +#if defined(ZIP_SUPPORT) + zip = NULL; +#else // !ZIP_SUPPORT + zip = false; +#endif // !ZIP_SUPPORT + Namespaces = NULL; Encoding = enc; Nslist = nsl; DefNs = nsdf; - } // end of XMLDOCUMENT constructor +} // end of XMLDOCUMENT constructor + +/******************************************************************/ +/* Initialize zipped file processing. */ +/******************************************************************/ +bool XMLDOCUMENT::InitZip(PGLOBAL g, char *entry) +{ +#if defined(ZIP_SUPPORT) + bool mul = (entry) ? strchr(entry, '*') || strchr(entry, '?') : false; + zip = new(g) ZIPUTIL(entry, mul); + return zip == NULL; +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return true; +#endif // !ZIP_SUPPORT +} // end of InitZip + +/******************************************************************/ +/* Make the namespace structure list. */ +/******************************************************************/ +char* XMLDOCUMENT::GetMemDoc(PGLOBAL g, char *fn) +{ +#if defined(ZIP_SUPPORT) + return (zip->OpenTable(g, MODE_ANY, fn)) ? NULL : zip->memory; +#else // !ZIP_SUPPORT + return NULL; +#endif // !ZIP_SUPPORT +} // end of GetMemDoc /******************************************************************/ /* Make the namespace structure list. */ /******************************************************************/ bool XMLDOCUMENT::MakeNSlist(PGLOBAL g) - { - char *prefix, *href, *next = Nslist; +{ + char *prefix, *href, *next = Nslist; PNS nsp, *ppns = &Namespaces; while (next) { @@ -84,6 +116,19 @@ bool XMLDOCUMENT::MakeNSlist(PGLOBAL g) } // end of MakeNSlist /******************************************************************/ +/* Close ZIP file. */ +/******************************************************************/ +void XMLDOCUMENT::CloseZip(void) +{ +#if defined(ZIP_SUPPORT) + if (zip) { + zip->close(); + zip = NULL; + } // endif zip +#endif // ZIP_SUPPORT +} // end of CloseZip + +/******************************************************************/ /* XMLNODE constructor. */ /******************************************************************/ XMLNODE::XMLNODE(PXDOC dp) diff --git a/storage/connect/plgxml.h b/storage/connect/plgxml.h index b8e914e0bf1..db7dfa6bda5 100644 --- a/storage/connect/plgxml.h +++ b/storage/connect/plgxml.h @@ -1,3 +1,7 @@ +#if defined(ZIP_SUPPORT) +#include "filamzip.h" +#endif // ZIP_SUPPORT + /******************************************************************/ /* Dual XML implementation base classes defines. */ /******************************************************************/ @@ -72,8 +76,8 @@ class XMLDOCUMENT : public BLOCK { virtual void SetNofree(bool b) = 0; // Methods - virtual bool Initialize(PGLOBAL) = 0; - virtual bool ParseFile(char *) = 0; + virtual bool Initialize(PGLOBAL, char *, bool) = 0; + virtual bool ParseFile(PGLOBAL, char *) = 0; virtual bool NewDoc(PGLOBAL, char *) = 0; virtual void AddComment(PGLOBAL, char *) = 0; virtual PXNODE GetRoot(PGLOBAL) = 0; @@ -91,8 +95,16 @@ class XMLDOCUMENT : public BLOCK { // Utility bool MakeNSlist(PGLOBAL g); + bool InitZip(PGLOBAL g, char *entry); + char *GetMemDoc(PGLOBAL g, char *fn); + void CloseZip(void); // Members +#if defined(ZIP_SUPPORT) + ZIPUTIL *zip; /* Used for zipped file */ +#else // !ZIP_SUPPORT + bool zip; /* Always false */ +#endif // !ZIP_SUPPORT PNS Namespaces; /* To the namespaces */ char *Encoding; /* The document encoding */ char *Nslist; /* Namespace list */ diff --git a/storage/connect/reldef.cpp b/storage/connect/reldef.cpp index ac2327212e0..30e4d49d249 100644 --- a/storage/connect/reldef.cpp +++ b/storage/connect/reldef.cpp @@ -1,7 +1,7 @@ /************* RelDef CPP Program Source Code File (.CPP) **************/ /* PROGRAM NAME: RELDEF */ /* ------------- */ -/* Version 1.5 */ +/* Version 1.6 */ /* */ /* COPYRIGHT: */ /* ---------- */ @@ -40,10 +40,12 @@ #include "tabcol.h" #include "filamap.h" #include "filamfix.h" +#if defined(VCT_SUPPORT) #include "filamvct.h" -#if defined(ZIP_SUPPORT) -#include "filamzip.h" -#endif // ZIP_SUPPORT +#endif // VCT_SUPPORT +#if defined(GZ_SUPPORT) +#include "filamgz.h" +#endif // GZ_SUPPORT #include "tabdos.h" #include "valblk.h" #include "tabmul.h" @@ -663,15 +665,15 @@ PTDB OEMDEF::GetTable(PGLOBAL g, MODE mode) /*********************************************************************/ if (!((PTDBDOS)tdbp)->GetTxfp()) { if (cmpr) { -#if defined(ZIP_SUPPORT) +#if defined(GZ_SUPPORT) if (cmpr == 1) - txfp = new(g) ZIPFAM(defp); + txfp = new(g) GZFAM(defp); else txfp = new(g) ZLBFAM(defp); -#else // !ZIP_SUPPORT +#else // !GZ_SUPPORT strcpy(g->Message, "Compress not supported"); return NULL; -#endif // !ZIP_SUPPORT +#endif // !GZ_SUPPORT } else if (rfm == RECFM_VAR) { if (map) txfp = new(g) MAPFAM(defp); @@ -683,16 +685,19 @@ PTDB OEMDEF::GetTable(PGLOBAL g, MODE mode) txfp = new(g) MPXFAM(defp); else txfp = new(g) FIXFAM(defp); - } else if (rfm == RECFM_VCT) { - assert (Pxdef->GetDefType() == TYPE_AM_VCT); +#if defined(VCT_SUPPORT) + assert(Pxdef->GetDefType() == TYPE_AM_VCT); if (map) txfp = new(g) VCMFAM((PVCTDEF)defp); else txfp = new(g) VCTFAM((PVCTDEF)defp); - - } // endif's +#else // !VCT_SUPPORT + strcpy(g->Message, "VCT no more supported"); + return NULL; +#endif // !VCT_SUPPORT + } // endif's ((PTDBDOS)tdbp)->SetTxfp(txfp); } // endif Txfp diff --git a/storage/connect/tabdos.cpp b/storage/connect/tabdos.cpp index 98633f49d23..16cc6c33b44 100644 --- a/storage/connect/tabdos.cpp +++ b/storage/connect/tabdos.cpp @@ -1,11 +1,11 @@ /************* TabDos C++ Program Source Code File (.CPP) **************/ /* PROGRAM NAME: TABDOS */ /* ------------- */ -/* Version 4.9 */ +/* Version 4.9.2 */ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 1998-2015 */ +/* (C) Copyright to the author Olivier BERTRAND 1998-2016 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -51,6 +51,9 @@ #include "filamap.h" #include "filamfix.h" #include "filamdbf.h" +#if defined(GZ_SUPPORT) +#include "filamgz.h" +#endif // GZ_SUPPORT #if defined(ZIP_SUPPORT) #include "filamzip.h" #endif // ZIP_SUPPORT @@ -93,10 +96,13 @@ DOSDEF::DOSDEF(void) Pseudo = 3; Fn = NULL; Ofn = NULL; + Entry = NULL; To_Indx = NULL; Recfm = RECFM_VAR; Mapped = false; - Padded = false; + Zipped = false; + Mulentries = false; + Padded = false; Huge = false; Accept = false; Eof = false; @@ -126,6 +132,11 @@ bool DOSDEF::DefineAM(PGLOBAL g, LPCSTR am, int) : (am && (*am == 'B' || *am == 'b')) ? "B" : (am && !stricmp(am, "DBF")) ? "D" : "V"; + if ((Zipped = GetBoolCatInfo("Zipped", false))) + Mulentries = ((Entry = GetStringCatInfo(g, "Entry", NULL))) + ? strchr(Entry, '*') || strchr(Entry, '?') + : GetBoolCatInfo("Mulentries", false); + Desc = Fn = GetStringCatInfo(g, "Filename", NULL); Ofn = GetStringCatInfo(g, "Optname", Fn); GetCharCatInfo("Recfm", (PSZ)dfm, buf, sizeof(buf)); @@ -333,7 +344,21 @@ PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode) /* Allocate table and file processing class of the proper type. */ /* Column blocks will be allocated only when needed. */ /*********************************************************************/ - if (Recfm == RECFM_DBF) { + if (Zipped) { +#if defined(ZIP_SUPPORT) + if (Recfm == RECFM_VAR) { + txfp = new(g)ZIPFAM(this); + tdbp = new(g)TDBDOS(this, txfp); + } else { + txfp = new(g)ZPXFAM(this); + tdbp = new(g)TDBFIX(this, txfp); + } // endif Recfm + +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif // !ZIP_SUPPORT + } else if (Recfm == RECFM_DBF) { if (Catfunc == FNC_NO) { if (map) txfp = new(g) DBMFAM(this); @@ -350,28 +375,28 @@ PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode) else if (map) txfp = new(g) MPXFAM(this); else if (Compressed) { -#if defined(ZIP_SUPPORT) +#if defined(GZ_SUPPORT) txfp = new(g) ZIXFAM(this); -#else // !ZIP_SUPPORT - sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); +#else // !GZ_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "GZ"); return NULL; -#endif // !ZIP_SUPPORT +#endif // !GZ_SUPPORT } else txfp = new(g) FIXFAM(this); tdbp = new(g) TDBFIX(this, txfp); } else { if (Compressed) { -#if defined(ZIP_SUPPORT) +#if defined(GZ_SUPPORT) if (Compressed == 1) - txfp = new(g) ZIPFAM(this); + txfp = new(g) GZFAM(this); else txfp = new(g) ZLBFAM(this); -#else // !ZIP_SUPPORT - sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); +#else // !GZ_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "GZ"); return NULL; -#endif // !ZIP_SUPPORT +#endif // !GZ_SUPPORT } else if (map) txfp = new(g) MAPFAM(this); else @@ -396,7 +421,7 @@ PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode) if (map) { txfp = new(g) MBKFAM(this); } else if (Compressed) { -#if defined(ZIP_SUPPORT) +#if defined(GZ_SUPPORT) if (Compressed == 1) txfp = new(g) ZBKFAM(this); else { @@ -404,7 +429,7 @@ PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode) ((PZLBFAM)txfp)->SetOptimized(To_Pos != NULL); } // endelse #else - sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "GZ"); return NULL; #endif } else @@ -531,14 +556,14 @@ int TDBDOS::ResetTableOpt(PGLOBAL g, bool dop, bool dox) // except for ZLIB access method. if (Txfp->GetAmType() == TYPE_AM_MAP) { Txfp = new(g) MAPFAM((PDOSDEF)To_Def); -#if defined(ZIP_SUPPORT) - } else if (Txfp->GetAmType() == TYPE_AM_ZIP) { - Txfp = new(g) ZIPFAM((PDOSDEF)To_Def); +#if defined(GZ_SUPPORT) + } else if (Txfp->GetAmType() == TYPE_AM_GZ) { + Txfp = new(g) GZFAM((PDOSDEF)To_Def); } else if (Txfp->GetAmType() == TYPE_AM_ZLIB) { Txfp->Reset(); ((PZLBFAM)Txfp)->SetOptimized(false); -#endif // ZIP_SUPPORT - } else if (Txfp->GetAmType() == TYPE_AM_BLK) +#endif // GZ_SUPPORT + } else if (Txfp->GetAmType() == TYPE_AM_BLK) Txfp = new(g) DOSFAM((PDOSDEF)To_Def); Txfp->SetTdbp(this); @@ -609,7 +634,12 @@ int TDBDOS::MakeBlockValues(PGLOBAL g) defp->SetOptimized(0); // Estimate the number of needed blocks - block = (int)((MaxSize + (int)nrec - 1) / (int)nrec); + if ((block = (int)((MaxSize + (int)nrec - 1) / (int)nrec)) < 2) { + // This may be wrong to do in some cases + defp->RemoveOptValues(g); + strcpy(g->Message, MSG(TABLE_NOT_OPT)); + return RC_INFO; // Not to be optimized + } // endif block // We have to use local variables because Txfp->CurBlk is set // to Rows+1 by unblocked variable length table access methods. @@ -952,13 +982,14 @@ bool TDBDOS::GetBlockValues(PGLOBAL g) PCOLDEF cdp; PDOSDEF defp = (PDOSDEF)To_Def; PCATLG cat = defp->GetCat(); + PDBUSER dup = PlgGetUser(g); #if 0 if (Mode == MODE_INSERT && Txfp->GetAmType() == TYPE_AM_DOS) return false; #endif // __WIN__ - if (defp->Optimized) + if (defp->Optimized || !(dup->Check & CHK_OPT)) return false; // Already done or to be redone if (Ftype == RECFM_VAR || defp->Compressed == 2) { @@ -2079,10 +2110,10 @@ bool TDBDOS::OpenDB(PGLOBAL g) /*******************************************************************/ if (Txfp->GetAmType() == TYPE_AM_MAP && Mode == MODE_DELETE) Txfp = new(g) MAPFAM((PDOSDEF)To_Def); -#if defined(ZIP_SUPPORT) - else if (Txfp->GetAmType() == TYPE_AM_ZIP) - Txfp = new(g) ZIPFAM((PDOSDEF)To_Def); -#endif // ZIP_SUPPORT +#if defined(GZ_SUPPORT) + else if (Txfp->GetAmType() == TYPE_AM_GZ) + Txfp = new(g) GZFAM((PDOSDEF)To_Def); +#endif // GZ_SUPPORT else // if (Txfp->GetAmType() != TYPE_AM_DOS) ??? Txfp = new(g) DOSFAM((PDOSDEF)To_Def); diff --git a/storage/connect/tabdos.h b/storage/connect/tabdos.h index c098886f14b..4c8eb438a26 100644 --- a/storage/connect/tabdos.h +++ b/storage/connect/tabdos.h @@ -28,6 +28,7 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */ friend class TDBFIX; friend class TXTFAM; friend class DBFBASE; + friend class ZIPUTIL; public: // Constructor DOSDEF(void); @@ -40,7 +41,9 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */ virtual bool IsHuge(void) {return Huge;} PSZ GetFn(void) {return Fn;} PSZ GetOfn(void) {return Ofn;} - void SetBlock(int block) {Block = block;} + PSZ GetEntry(void) {return Entry;} + bool GetMul(void) {return Mulentries;} + void SetBlock(int block) {Block = block;} int GetBlock(void) {return Block;} int GetLast(void) {return Last;} void SetLast(int last) {Last = last;} @@ -57,9 +60,9 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */ int *GetTo_Pos(void) {return To_Pos;} // Methods - virtual int Indexable(void) - {return (!Multiple && Compressed != 1) ? 1 : 0;} - virtual bool DeleteIndexFile(PGLOBAL g, PIXDEF pxdf); + virtual int Indexable(void) + {return (!Multiple && !Mulentries && Compressed != 1) ? 1 : 0;} + virtual bool DeleteIndexFile(PGLOBAL g, PIXDEF pxdf); virtual bool DefineAM(PGLOBAL g, LPCSTR am, int poff); virtual PTDB GetTable(PGLOBAL g, MODE mode); bool InvalidateIndex(PGLOBAL g); @@ -72,10 +75,13 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */ // Members PSZ Fn; /* Path/Name of corresponding file */ PSZ Ofn; /* Base Path/Name of matching index files*/ - PIXDEF To_Indx; /* To index definitions blocks */ + PSZ Entry; /* Zip entry name or pattern */ + PIXDEF To_Indx; /* To index definitions blocks */ RECFM Recfm; /* 0:VAR, 1:FIX, 2:BIN, 3:VCT, 6:DBF */ bool Mapped; /* 0: disk file, 1: memory mapped file */ - bool Padded; /* true for padded table file */ + bool Zipped; /* true for zipped table file */ + bool Mulentries; /* true for multiple entries */ + bool Padded; /* true for padded table file */ bool Huge; /* true for files larger than 2GB */ bool Accept; /* true if wrong lines are accepted */ bool Eof; /* true if an EOF (0xA) character exists */ @@ -91,7 +97,7 @@ class DllExport DOSDEF : public TABDEF { /* Logical table description */ int Maxerr; /* Maximum number of bad records (DBF) */ int ReadMode; /* Specific to DBF */ int Ending; /* Length of end of lines */ - int Teds; /* Binary table default endian setting */ + char Teds; /* Binary table default endian setting */ }; // end of DOSDEF /***********************************************************************/ diff --git a/storage/connect/tabfix.cpp b/storage/connect/tabfix.cpp index 55c254f41ea..d99f7800f26 100644 --- a/storage/connect/tabfix.cpp +++ b/storage/connect/tabfix.cpp @@ -1,11 +1,11 @@ /************* TabFix C++ Program Source Code File (.CPP) **************/ /* PROGRAM NAME: TABFIX */ /* ------------- */ -/* Version 4.9 */ +/* Version 4.9.1 */ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 1998-2015 */ +/* (C) Copyright to the author Olivier BERTRAND 1998-2016 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -589,9 +589,10 @@ void BINCOL::WriteColumn(PGLOBAL g) switch (Fmt) { case 'X': // Standard not converted values - if (Eds && IsTypeChar(Buf_Type)) - *(longlong *)p = Value->GetBigintValue(); - else if (Value->GetBinValue(p, Long, Status)) { + if (Eds && IsTypeChar(Buf_Type)) { + if (Status) + Value->GetValueNonAligned<longlong>(p, Value->GetBigintValue()); + } else if (Value->GetBinValue(p, Long, Status)) { sprintf(g->Message, MSG(BIN_F_TOO_LONG), Name, Value->GetSize(), Long); longjmp(g->jumper[g->jump_level], 31); @@ -605,7 +606,7 @@ void BINCOL::WriteColumn(PGLOBAL g) sprintf(g->Message, MSG(VALUE_TOO_BIG), n, Name); longjmp(g->jumper[g->jump_level], 31); } else if (Status) - *(short *)p = (short)n; + Value->GetValueNonAligned<short>(p, (short)n); break; case 'T': // Tiny integer @@ -625,7 +626,7 @@ void BINCOL::WriteColumn(PGLOBAL g) sprintf(g->Message, MSG(VALUE_TOO_BIG), n, Name); longjmp(g->jumper[g->jump_level], 31); } else if (Status) - *(int *)p = Value->GetIntValue(); + Value->GetValueNonAligned<int>(p, (int)n); break; case 'G': // Large (great) integer @@ -636,12 +637,12 @@ void BINCOL::WriteColumn(PGLOBAL g) case 'F': // Float case 'R': // Real if (Status) - *(float *)p = (float)Value->GetFloatValue(); + Value->GetValueNonAligned<float>(p, (float)Value->GetFloatValue()); break; case 'D': // Double if (Status) - *(double *)p = Value->GetFloatValue(); + Value->GetValueNonAligned<double>(p, Value->GetFloatValue()); break; case 'C': // Characters diff --git a/storage/connect/tabfmt.cpp b/storage/connect/tabfmt.cpp index 015f8d93b15..b24375443f6 100644 --- a/storage/connect/tabfmt.cpp +++ b/storage/connect/tabfmt.cpp @@ -1,11 +1,11 @@ /************* TabFmt C++ Program Source Code File (.CPP) **************/ /* PROGRAM NAME: TABFMT */ /* ------------- */ -/* Version 3.9 */ +/* Version 3.9.2 */ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 2001 - 2015 */ +/* (C) Copyright to the author Olivier BERTRAND 2001 - 2016 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -51,6 +51,9 @@ #include "plgdbsem.h" #include "mycat.h" #include "filamap.h" +#if defined(GZ_SUPPORT) +#include "filamgz.h" +#endif // GZ_SUPPORT #if defined(ZIP_SUPPORT) #include "filamzip.h" #endif // ZIP_SUPPORT @@ -78,20 +81,24 @@ USETEMP UseTemp(void); /* of types (TYPE_STRING < TYPE_DOUBLE < TYPE_INT) (1 < 2 < 7). */ /* If these values are changed, this will have to be revisited. */ /***********************************************************************/ -PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, - char q, int hdr, int mxr, bool info) +PQRYRES CSVColumns(PGLOBAL g, char *dp, PTOS topt, bool info) { static int buftyp[] = {TYPE_STRING, TYPE_SHORT, TYPE_STRING, TYPE_INT, TYPE_INT, TYPE_SHORT}; static XFLD fldtyp[] = {FLD_NAME, FLD_TYPE, FLD_TYPENAME, FLD_PREC, FLD_LENGTH, FLD_SCALE}; static unsigned int length[] = {6, 6, 8, 10, 10, 6}; - char *p, *colname[MAXCOL], dechar, filename[_MAX_PATH], buf[4096]; + const char *fn; + char sep, q; + int rc, mxr; + bool hdr; + char *p, *colname[MAXCOL], dechar, buf[8]; int i, imax, hmax, n, nerr, phase, blank, digit, dec, type; int ncol = sizeof(buftyp) / sizeof(int); int num_read = 0, num_max = 10000000; // Statistics int len[MAXCOL], typ[MAXCOL], prc[MAXCOL]; - FILE *infile; + PCSVDEF tdp; + PTDBCSV tdbp; PQRYRES qrp; PCOLRES crp; @@ -101,27 +108,13 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, goto skipit; } // endif info -// num_max = atoi(p+1); // Max num of record to test -#if defined(__WIN__) - if (sep == ',' || strnicmp(setlocale(LC_NUMERIC, NULL), "French", 6)) - dechar = '.'; - else - dechar = ','; -#else // !__WIN__ - dechar = '.'; -#endif // !__WIN__ - - if (trace) - htrc("File %s sep=%c q=%c hdr=%d mxr=%d\n", - SVP(fn), sep, q, hdr, mxr); - - if (!fn) { - strcpy(g->Message, MSG(MISSING_FNAME)); - return NULL; - } // endif fn + if (GetIntegerTableOption(g, topt, "Multiple", 0)) { + strcpy(g->Message, "Cannot find column definition for multiple table"); + return NULL; + } // endif Multiple +// num_max = atoi(p+1); // Max num of record to test imax = hmax = nerr = 0; - mxr = MY_MAX(0, mxr); for (i = 0; i < MAXCOL; i++) { colname[i] = NULL; @@ -131,12 +124,76 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, } // endfor i /*********************************************************************/ - /* Open the input file. */ + /* Get the CSV table description block. */ /*********************************************************************/ - PlugSetPath(filename, fn, dp); + tdp = new(g) CSVDEF; +#if defined(ZIP_SUPPORT) + tdp->Entry = GetStringTableOption(g, topt, "Entry", NULL); + tdp->Zipped = GetBooleanTableOption(g, topt, "Zipped", false); +#endif // ZIP_SUPPORT + fn = tdp->Fn = GetStringTableOption(g, topt, "Filename", NULL); + + if (!tdp->Fn) { + strcpy(g->Message, MSG(MISSING_FNAME)); + return NULL; + } // endif Fn + + if (!(tdp->Lrecl = GetIntegerTableOption(g, topt, "Lrecl", 0))) + tdp->Lrecl = 4096; + + p = GetStringTableOption(g, topt, "Separator", ","); + tdp->Sep = (strlen(p) == 2 && p[0] == '\\' && p[1] == 't') ? '\t' : *p; + +#if defined(__WIN__) + if (tdp->Sep == ',' || strnicmp(setlocale(LC_NUMERIC, NULL), "French", 6)) + dechar = '.'; + else + dechar = ','; +#else // !__WIN__ + dechar = '.'; +#endif // !__WIN__ + + sep = tdp->Sep; + tdp->Quoted = GetIntegerTableOption(g, topt, "Quoted", -1); + p = GetStringTableOption(g, topt, "Qchar", ""); + tdp->Qot = *p; + + if (tdp->Qot && tdp->Quoted < 0) + tdp->Quoted = 0; + else if (!tdp->Qot && tdp->Quoted >= 0) + tdp->Qot = '"'; - if (!(infile= global_fopen(g, MSGID_CANNOT_OPEN, filename, "r"))) - return NULL; + q = tdp->Qot; + hdr = GetBooleanTableOption(g, topt, "Header", false); + tdp->Maxerr = GetIntegerTableOption(g, topt, "Maxerr", 0); + tdp->Accept = GetBooleanTableOption(g, topt, "Accept", false); + + if (tdp->Accept && tdp->Maxerr == 0) + tdp->Maxerr = INT_MAX32; // Accept all bad lines + + mxr = MY_MAX(0, tdp->Maxerr); + + if (trace) + htrc("File %s Sep=%c Qot=%c Header=%d maxerr=%d\n", + SVP(tdp->Fn), tdp->Sep, tdp->Qot, tdp->Header, tdp->Maxerr); + + if (tdp->Zipped) { +#if defined(ZIP_SUPPORT) + tdbp = new(g)TDBCSV(tdp, new(g)ZIPFAM(tdp)); +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif // !ZIP_SUPPORT + } else + tdbp = new(g) TDBCSV(tdp, new(g) DOSFAM(tdp)); + + tdbp->SetMode(MODE_READ); + + /*********************************************************************/ + /* Open the CSV file. */ + /*********************************************************************/ + if (tdbp->OpenDB(g)) + return NULL; if (hdr) { /*******************************************************************/ @@ -144,16 +201,8 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, /*******************************************************************/ phase = 0; - if (fgets(buf, sizeof(buf), infile)) { - n = strlen(buf) + 1; - buf[n - 2] = '\0'; -#if !defined(__WIN__) - // The file can be imported from Windows - if (buf[n - 3] == '\r') - buf[n - 3] = 0; -#endif // UNIX - p = (char*)PlugSubAlloc(g, NULL, n); - memcpy(p, buf, n); + if ((rc = tdbp->ReadDB(g)) == RC_OK) { + p = PlgDBDup(g, tdbp->To_Line); //skip leading blanks for (; *p == ' '; p++) ; @@ -165,10 +214,11 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, } // endif q colname[0] = p; - } else { + } else if (rc == RC_EF) { sprintf(g->Message, MSG(FILE_IS_EMPTY), fn); goto err; - } // endif's + } else + goto err; for (i = 1; *p; p++) if (phase == 1 && *p == q) { @@ -201,15 +251,8 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, /*******************************************************************/ /* Now start the reading process. Read one line. */ /*******************************************************************/ - if (fgets(buf, sizeof(buf), infile)) { - n = strlen(buf); - buf[n - 1] = '\0'; -#if !defined(__WIN__) - // The file can be imported from Windows - if (buf[n - 2] == '\r') - buf[n - 2] = 0; -#endif // UNIX - } else if (feof(infile)) { + if ((rc = tdbp->ReadDB(g)) == RC_OK) { + } else if (rc == RC_EF) { sprintf(g->Message, MSG(EOF_AFTER_LINE), num_read -1); break; } else { @@ -222,7 +265,7 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, /*******************************************************************/ i = n = phase = blank = digit = dec = 0; - for (p = buf; *p; p++) + for (p = tdbp->To_Line; *p; p++) if (*p == sep) { if (phase != 1) { if (i == MAXCOL - 1) { @@ -331,7 +374,7 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, htrc("\n"); } // endif trace - fclose(infile); + tdbp->CloseDB(g); skipit: if (trace) @@ -381,7 +424,7 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, return qrp; err: - fclose(infile); + tdbp->CloseDB(g); return NULL; } // end of CSVCColumns @@ -458,20 +501,27 @@ PTDB CSVDEF::GetTable(PGLOBAL g, MODE mode) /*******************************************************************/ /* Allocate a file processing class of the proper type. */ /*******************************************************************/ - if (map) { + if (Zipped) { +#if defined(ZIP_SUPPORT) + txfp = new(g) ZIPFAM(this); +#else // !ZIP_SUPPORT + strcpy(g->Message, "ZIP not supported"); + return NULL; +#endif // !ZIP_SUPPORT + } else if (map) { // Should be now compatible with UNIX txfp = new(g) MAPFAM(this); } else if (Compressed) { -#if defined(ZIP_SUPPORT) +#if defined(GZ_SUPPORT) if (Compressed == 1) - txfp = new(g) ZIPFAM(this); + txfp = new(g) GZFAM(this); else txfp = new(g) ZLBFAM(this); -#else // !ZIP_SUPPORT +#else // !GZ_SUPPORT strcpy(g->Message, "Compress not supported"); return NULL; -#endif // !ZIP_SUPPORT +#endif // !GZ_SUPPORT } else txfp = new(g) DOSFAM(this); @@ -498,7 +548,7 @@ PTDB CSVDEF::GetTable(PGLOBAL g, MODE mode) if (map) { txfp = new(g) MBKFAM(this); } else if (Compressed) { -#if defined(ZIP_SUPPORT) +#if defined(GZ_SUPPORT) if (Compressed == 1) txfp = new(g) ZBKFAM(this); else { @@ -506,7 +556,7 @@ PTDB CSVDEF::GetTable(PGLOBAL g, MODE mode) ((PZLBFAM)txfp)->SetOptimized(To_Pos != NULL); } // endelse #else - sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "GZ"); return NULL; #endif } else @@ -830,8 +880,9 @@ bool TDBCSV::SkipHeader(PGLOBAL g) /***********************************************************************/ int TDBCSV::ReadBuffer(PGLOBAL g) { - char *p1, *p2, *p = NULL; - int i, n, len, rc = Txfp->ReadBuffer(g); + //char *p1, *p2, *p = NULL; + char *p2, *p = NULL; + int i, n, len, rc = Txfp->ReadBuffer(g); bool bad = false; if (trace > 1) @@ -846,14 +897,23 @@ int TDBCSV::ReadBuffer(PGLOBAL g) for (i = 0; i < Fields; i++) { if (!bad) { if (Qot && *p2 == Qot) { // Quoted field - for (n = 0, p1 = ++p2; (p = strchr(p1, Qot)); p1 = p + 2) - if (*(p + 1) == Qot) - n++; // Doubled internal quotes - else - break; // Final quote + //for (n = 0, p1 = ++p2; (p = strchr(p1, Qot)); p1 = p + 2) + // if (*(p + 1) == Qot) + // n++; // Doubled internal quotes + // else + // break; // Final quote + + for (n = 0, p = ++p2; p; p++) + if (*p == Qot || *p == '\\') { + if (*(++p) == Qot) + n++; // Escaped internal quotes + else if (*(p - 1) == Qot) + break; // Final quote + } // endif *p if (p) { - len = p++ - p2; + //len = p++ - p2; + len = p - p2 - 1;; // if (Sep != ' ') // for (; *p == ' '; p++) ; // Skip blanks @@ -873,10 +933,12 @@ int TDBCSV::ReadBuffer(PGLOBAL g) if (n) { int j, k; - // Suppress the double of internal quotes + // Suppress the escape of internal quotes for (j = k = 0; j < len; j++, k++) { - if (p2[j] == Qot) - j++; // skip first one + if (p2[j] == Qot || (p2[j] == '\\' && p2[j + 1] == Qot)) + j++; // skip escape char + else if (p2[j] == '\\') + p2[k++] = p2[j++]; // avoid \\Qot p2[k] = p2[j]; } // endfor i, j @@ -1464,21 +1526,16 @@ void CSVCOL::WriteColumn(PGLOBAL g) /* TDBCCL class constructor. */ /***********************************************************************/ TDBCCL::TDBCCL(PCSVDEF tdp) : TDBCAT(tdp) - { - Fn = tdp->GetFn(); - Hdr = tdp->Header; - Mxr = tdp->Maxerr; - Qtd = tdp->Quoted; - Sep = tdp->Sep; - } // end of TDBCCL constructor +{ + Topt = tdp->GetTopt(); +} // end of TDBCCL constructor /***********************************************************************/ /* GetResult: Get the list the CSV file columns. */ /***********************************************************************/ PQRYRES TDBCCL::GetResult(PGLOBAL g) { - return CSVColumns(g, ((PTABDEF)To_Def)->GetPath(), - Fn, Sep, Qtd, Hdr, Mxr, false); + return CSVColumns(g, ((PTABDEF)To_Def)->GetPath(), Topt, false); } // end of GetResult /* ------------------------ End of TabFmt ---------------------------- */ diff --git a/storage/connect/tabfmt.h b/storage/connect/tabfmt.h index ce80a276cdc..5ce8d399a64 100644 --- a/storage/connect/tabfmt.h +++ b/storage/connect/tabfmt.h @@ -1,7 +1,7 @@ /*************** TabFmt H Declares Source Code File (.H) ***************/ -/* Name: TABFMT.H Version 2.4 */ +/* Name: TABFMT.H Version 2.5 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 2001-2014 */ +/* (C) Copyright to the author Olivier BERTRAND 2001-2016 */ /* */ /* This file contains the CSV and FMT classes declares. */ /***********************************************************************/ @@ -13,8 +13,7 @@ typedef class TDBFMT *PTDBFMT; /***********************************************************************/ /* Functions used externally. */ /***********************************************************************/ -PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, - char q, int hdr, int mxr, bool info); +PQRYRES CSVColumns(PGLOBAL g, char *dp, PTOS topt, bool info); /***********************************************************************/ /* CSV table. */ @@ -22,7 +21,8 @@ PQRYRES CSVColumns(PGLOBAL g, char *dp, const char *fn, char sep, class DllExport CSVDEF : public DOSDEF { /* Logical table description */ friend class TDBCSV; friend class TDBCCL; - public: + friend PQRYRES CSVColumns(PGLOBAL, char *, PTOS, bool); +public: // Constructor CSVDEF(void); @@ -50,9 +50,10 @@ class DllExport CSVDEF : public DOSDEF { /* Logical table description */ /* This is the DOS/UNIX Access Method class declaration for files */ /* that are CSV files with columns separated by the Sep character. */ /***********************************************************************/ -class TDBCSV : public TDBDOS { +class DllExport TDBCSV : public TDBDOS { friend class CSVCOL; - public: + friend PQRYRES CSVColumns(PGLOBAL, char *, PTOS, bool); +public: // Constructor TDBCSV(PCSVDEF tdp, PTXF txfp); TDBCSV(PGLOBAL g, PTDBCSV tdbp); @@ -101,7 +102,7 @@ class TDBCSV : public TDBDOS { /* Class CSVCOL: CSV access method column descriptor. */ /* This A.M. is used for Comma Separated V(?) files. */ /***********************************************************************/ -class CSVCOL : public DOSCOL { +class DllExport CSVCOL : public DOSCOL { friend class TDBCSV; friend class TDBFMT; public: @@ -129,7 +130,7 @@ class CSVCOL : public DOSCOL { /* This is the DOS/UNIX Access Method class declaration for files */ /* whose record format is described by a Format keyword. */ /***********************************************************************/ -class TDBFMT : public TDBCSV { +class DllExport TDBFMT : public TDBCSV { friend class CSVCOL; //friend class FMTCOL; public: @@ -173,7 +174,7 @@ class TDBFMT : public TDBCSV { /***********************************************************************/ /* This is the class declaration for the CSV catalog table. */ /***********************************************************************/ -class TDBCCL : public TDBCAT { +class DllExport TDBCCL : public TDBCAT { public: // Constructor TDBCCL(PCSVDEF tdp); @@ -183,11 +184,7 @@ class TDBCCL : public TDBCAT { virtual PQRYRES GetResult(PGLOBAL g); // Members - char *Fn; // The CSV file (path) name - bool Hdr; // true if first line contains headers - int Mxr; // Maximum number of bad records - int Qtd; // Quoting level for quoted fields - char Sep; // Separator for standard CSV files - }; // end of class TDBCCL + PTOS Topt; +}; // end of class TDBCCL /* ------------------------- End of TabFmt.H ------------------------- */ diff --git a/storage/connect/tabjdbc.cpp b/storage/connect/tabjdbc.cpp index e398523892f..912e6c7d530 100644 --- a/storage/connect/tabjdbc.cpp +++ b/storage/connect/tabjdbc.cpp @@ -110,6 +110,7 @@ bool JDBCDEF::SetParms(PJPARM sjp) sjp->Url= Url; sjp->User= Username; sjp->Pwd= Password; +//sjp->Properties = Prop; return true; } // end of SetParms @@ -234,6 +235,7 @@ bool JDBCDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) Read_Only = true; Wrapname = GetStringCatInfo(g, "Wrapper", NULL); +//Prop = GetStringCatInfo(g, "Properties", NULL); Tabcat = GetStringCatInfo(g, "Qualifier", NULL); Tabcat = GetStringCatInfo(g, "Catalog", Tabcat); Tabschema = GetStringCatInfo(g, "Dbname", NULL); @@ -337,6 +339,7 @@ TDBJDBC::TDBJDBC(PJDBCDEF tdp) : TDBASE(tdp) Schema = tdp->Tabschema; Ops.User = tdp->Username; Ops.Pwd = tdp->Password; +// Ops.Properties = tdp->Prop; Catalog = tdp->Tabcat; Srcdef = tdp->Srcdef; Qrystr = tdp->Qrystr; @@ -356,6 +359,7 @@ TDBJDBC::TDBJDBC(PJDBCDEF tdp) : TDBASE(tdp) Ops.Url = NULL; Ops.User = NULL; Ops.Pwd = NULL; +// Ops.Properties = NULL; Catalog = NULL; Srcdef = NULL; Qrystr = NULL; diff --git a/storage/connect/tabjdbc.h b/storage/connect/tabjdbc.h index 7244ebd3832..fee8223abaf 100644 --- a/storage/connect/tabjdbc.h +++ b/storage/connect/tabjdbc.h @@ -58,6 +58,7 @@ protected: PSZ Tabschema; /* External table schema */ PSZ Username; /* User connect name */ PSZ Password; /* Password connect info */ +//PSZ Prop; /* Connection Properties */ PSZ Tabcat; /* External table catalog */ PSZ Tabtype; /* External table type */ PSZ Colpat; /* Catalog column pattern */ diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp index 17260836371..1b9ce8b64c9 100644 --- a/storage/connect/tabjson.cpp +++ b/storage/connect/tabjson.cpp @@ -1,6 +1,6 @@ /************* tabjson C++ Program Source Code File (.CPP) *************/ -/* PROGRAM NAME: tabjson Version 1.1 */ -/* (C) Copyright to the author Olivier BERTRAND 2014 - 2015 */ +/* PROGRAM NAME: tabjson Version 1.3 */ +/* (C) Copyright to the author Olivier BERTRAND 2014 - 2016 */ /* This program are the JSON class DB execution routines. */ /***********************************************************************/ @@ -25,6 +25,9 @@ //#include "resource.h" // for IDS_COLUMNS #include "tabjson.h" #include "filamap.h" +#if defined(GZ_SUPPORT) +#include "filamgz.h" +#endif // GZ_SUPPORT #if defined(ZIP_SUPPORT) #include "filamzip.h" #endif // ZIP_SUPPORT @@ -67,7 +70,7 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info) static XFLD fldtyp[] = {FLD_NAME, FLD_TYPE, FLD_TYPENAME, FLD_PREC, FLD_LENGTH, FLD_SCALE, FLD_NULL, FLD_FORMAT}; static unsigned int length[] = {0, 6, 8, 10, 10, 6, 6, 0}; - char *fn, colname[65], fmt[129]; + char colname[65], fmt[129]; int i, j, lvl, n = 0; int ncol = sizeof(buftyp) / sizeof(int); PVAL valp; @@ -91,19 +94,29 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info) goto skipit; } // endif info - /*********************************************************************/ + if (GetIntegerTableOption(g, topt, "Multiple", 0)) { + strcpy(g->Message, "Cannot find column definition for multiple table"); + return NULL; + } // endif Multiple + + /*********************************************************************/ /* Open the input file. */ /*********************************************************************/ - if (!(fn = GetStringTableOption(g, topt, "Filename", NULL))) { - strcpy(g->Message, MSG(MISSING_FNAME)); - return NULL; - } else { - lvl = GetIntegerTableOption(g, topt, "Level", 0); - lvl = (lvl < 0) ? 0 : (lvl > 16) ? 16 : lvl; - } // endif fn + lvl = GetIntegerTableOption(g, topt, "Level", 0); + lvl = (lvl < 0) ? 0 : (lvl > 16) ? 16 : lvl; tdp = new(g) JSONDEF; - tdp->Fn = fn; +#if defined(ZIP_SUPPORT) + tdp->Entry = GetStringTableOption(g, topt, "Entry", NULL); + tdp->Zipped = GetBooleanTableOption(g, topt, "Zipped", false); +#endif // ZIP_SUPPORT + tdp->Fn = GetStringTableOption(g, topt, "Filename", NULL); + + if (!tdp->Fn) { + strcpy(g->Message, MSG(MISSING_FNAME)); + return NULL; + } // endif Fn + tdp->Database = SetPath(g, db); tdp->Objname = GetStringTableOption(g, topt, "Object", NULL); tdp->Base = GetIntegerTableOption(g, topt, "Base", 0) ? 1 : 0; @@ -114,7 +127,15 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info) tdp->Fn, tdp->Objname, tdp->Pretty, lvl); if (tdp->Pretty == 2) { - tjsp = new(g) TDBJSON(tdp, new(g) MAPFAM(tdp)); + if (tdp->Zipped) { +#if defined(ZIP_SUPPORT) + tjsp = new(g) TDBJSON(tdp, new(g) ZIPFAM(tdp)); +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif // !ZIP_SUPPORT + } else + tjsp = new(g) TDBJSON(tdp, new(g) MAPFAM(tdp)); if (tjsp->MakeDocument(g)) return NULL; @@ -127,10 +148,33 @@ PQRYRES JSONColumns(PGLOBAL g, char *db, PTOS topt, bool info) } // endif lrecl tdp->Ending = GetIntegerTableOption(g, topt, "Ending", CRLF); - tjnp = new(g) TDBJSN(tdp, new(g) DOSFAM(tdp)); + + if (tdp->Zipped) { +#if defined(ZIP_SUPPORT) + tjnp = new(g)TDBJSN(tdp, new(g)ZIPFAM(tdp)); +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif // !ZIP_SUPPORT + } else + tjnp = new(g) TDBJSN(tdp, new(g) DOSFAM(tdp)); + tjnp->SetMode(MODE_READ); - if (tjnp->OpenDB(g)) +#if USE_G + // Allocate the parse work memory + PGLOBAL G = (PGLOBAL)PlugSubAlloc(g, NULL, sizeof(GLOBAL)); + memset(G, 0, sizeof(GLOBAL)); + G->Sarea_Size = tdp->Lrecl * 10; + G->Sarea = PlugSubAlloc(g, NULL, G->Sarea_Size); + PlugSubSet(G, G->Sarea, G->Sarea_Size); + G->jump_level = -1; + tjnp->SetG(G); +#else + tjnp->SetG(g); +#endif + + if (tjnp->OpenDB(g)) return NULL; switch (tjnp->ReadDB(g)) { @@ -395,16 +439,23 @@ PTDB JSONDEF::GetTable(PGLOBAL g, MODE m) !(tmp == TMP_FORCE && (m == MODE_UPDATE || m == MODE_DELETE)); - if (Compressed) { + if (Zipped) { #if defined(ZIP_SUPPORT) + txfp = new(g) ZIPFAM(this); +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif // !ZIP_SUPPORT + } else if (Compressed) { +#if defined(GZ_SUPPORT) if (Compressed == 1) - txfp = new(g) ZIPFAM(this); + txfp = new(g) GZFAM(this); else txfp = new(g) ZLBFAM(this); -#else // !ZIP_SUPPORT - sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); +#else // !GZ_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "GZ"); return NULL; -#endif // !ZIP_SUPPORT +#endif // !GZ_SUPPORT } else if (map) txfp = new(g) MAPFAM(this); else @@ -426,7 +477,16 @@ PTDB JSONDEF::GetTable(PGLOBAL g, MODE m) ((TDBJSN*)tdbp)->G = g; #endif } else { - txfp = new(g) MAPFAM(this); + if (Zipped) { +#if defined(ZIP_SUPPORT) + txfp = new(g)ZIPFAM(this); +#else // !ZIP_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; +#endif // !ZIP_SUPPORT + } else + txfp = new(g) MAPFAM(this); + tdbp = new(g) TDBJSON(this, txfp); ((TDBJSON*)tdbp)->G = g; } // endif Pretty diff --git a/storage/connect/tabjson.h b/storage/connect/tabjson.h index f7cb74c3c4d..c9d30d48f2a 100644 --- a/storage/connect/tabjson.h +++ b/storage/connect/tabjson.h @@ -78,7 +78,8 @@ public: virtual AMT GetAmType(void) {return TYPE_AM_JSN;} virtual bool SkipHeader(PGLOBAL g); virtual PTDB Duplicate(PGLOBAL g) {return (PTDB)new(g) TDBJSN(this);} - PJSON GetRow(void) {return Row;} + PJSON GetRow(void) {return Row;} + void SetG(PGLOBAL g) {G = g;} // Methods virtual PTDB CopyOne(PTABS t); diff --git a/storage/connect/tabxml.cpp b/storage/connect/tabxml.cpp index 57d204a4286..3b8229fcf51 100644 --- a/storage/connect/tabxml.cpp +++ b/storage/connect/tabxml.cpp @@ -1,9 +1,9 @@ /************* Tabxml C++ Program Source Code File (.CPP) **************/ /* PROGRAM NAME: TABXML */ /* ------------- */ -/* Version 2.8 */ +/* Version 2.9 */ /* */ -/* Author Olivier BERTRAND 2007 - 2015 */ +/* Author Olivier BERTRAND 2007 - 2016 */ /* */ /* This program are the XML tables classes using MS-DOM or libxml2. */ /***********************************************************************/ @@ -136,7 +136,12 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info) goto skipit; } // endif info - /*********************************************************************/ + if (GetIntegerTableOption(g, topt, "Multiple", 0)) { + strcpy(g->Message, "Cannot find column definition for multiple table"); + return NULL; + } // endif Multiple + + /*********************************************************************/ /* Open the input file. */ /*********************************************************************/ if (!(fn = GetStringTableOption(g, topt, "Filename", NULL))) { @@ -154,6 +159,8 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info) tdp->Fn = fn; tdp->Database = SetPath(g, db); tdp->Tabname = tab; + tdp->Zipped = GetBooleanTableOption(g, topt, "Zipped", false); + tdp->Entry = GetStringTableOption(g, topt, "Entry", NULL); if (!(op = GetStringTableOption(g, topt, "Xmlsup", NULL))) #if defined(__WIN__) @@ -204,7 +211,8 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info) while (true) { if (!vp->atp && - !(node = (vp->nl) ? vp->nl->GetItem(g, vp->k++, node) : NULL)) + !(node = (vp->nl) ? vp->nl->GetItem(g, vp->k++, tdp->Usedom ? node : NULL) + : NULL)) if (j) { vp = lvlp[--j]; @@ -254,7 +262,8 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info) if (j < lvl && ok) { vp = lvlp[j+1]; vp->k = 0; - vp->atp = node->GetAttribute(g, NULL); + vp->pn = node; + vp->atp = node->GetAttribute(g, NULL); vp->nl = node->GetChildElements(g); if (tdp->Usedom && vp->nl->GetLength() == 1) { @@ -265,7 +274,7 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info) if (vp->atp || vp->b) { if (!vp->atp) - node = vp->nl->GetItem(g, vp->k++, node); + node = vp->nl->GetItem(g, vp->k++, tdp->Usedom ? node : NULL); strncat(fmt, colname, XLEN(fmt)); strncat(fmt, "/", XLEN(fmt)); @@ -424,11 +433,14 @@ XMLDEF::XMLDEF(void) DefNs = NULL; Attrib = NULL; Hdattr = NULL; + Entry = NULL; Coltype = 1; Limit = 0; Header = 0; Xpand = false; Usedom = false; + Zipped = false; + Mulentries = false; } // end of XMLDEF constructor /***********************************************************************/ @@ -507,7 +519,14 @@ bool XMLDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) // Get eventual table node attribute Attrib = GetStringCatInfo(g, "Attribute", NULL); Hdattr = GetStringCatInfo(g, "HeadAttr", NULL); - return false; + + // Specific for zipped files + if ((Zipped = GetBoolCatInfo("Zipped", false))) + Mulentries = ((Entry = GetStringCatInfo(g, "Entry", NULL))) + ? strchr(Entry, '*') || strchr(Entry, '?') + : GetBoolCatInfo("Mulentries", false); + + return false; } // end of DefineAM /***********************************************************************/ @@ -547,6 +566,7 @@ TDBXML::TDBXML(PXMLDEF tdp) : TDBASE(tdp) Xfile = tdp->Fn; Enc = tdp->Encoding; Tabname = tdp->Tabname; +#if 0 // why all these? Rowname = (tdp->Rowname) ? tdp->Rowname : NULL; Colname = (tdp->Colname) ? tdp->Colname : NULL; Mulnode = (tdp->Mulnode) ? tdp->Mulnode : NULL; @@ -555,10 +575,22 @@ TDBXML::TDBXML(PXMLDEF tdp) : TDBASE(tdp) DefNs = (tdp->DefNs) ? tdp->DefNs : NULL; Attrib = (tdp->Attrib) ? tdp->Attrib : NULL; Hdattr = (tdp->Hdattr) ? tdp->Hdattr : NULL; - Coltype = tdp->Coltype; +#endif // 0 + Rowname = tdp->Rowname; + Colname = tdp->Colname; + Mulnode = tdp->Mulnode; + XmlDB = tdp->XmlDB; + Nslist = tdp->Nslist; + DefNs = tdp->DefNs; + Attrib = tdp->Attrib; + Hdattr = tdp->Hdattr; + Entry = tdp->Entry; + Coltype = tdp->Coltype; Limit = tdp->Limit; Xpand = tdp->Xpand; - Changed = false; + Zipped = tdp->Zipped; + Mulentries = tdp->Mulentries; + Changed = false; Checked = false; NextSame = false; NewRow = false; @@ -600,10 +632,13 @@ TDBXML::TDBXML(PTDBXML tdbp) : TDBASE(tdbp) DefNs = tdbp->DefNs; Attrib = tdbp->Attrib; Hdattr = tdbp->Hdattr; - Coltype = tdbp->Coltype; + Entry = tdbp->Entry; + Coltype = tdbp->Coltype; Limit = tdbp->Limit; Xpand = tdbp->Xpand; - Changed = tdbp->Changed; + Zipped = tdbp->Zipped; + Mulentries = tdbp->Mulentries; + Changed = tdbp->Changed; Checked = tdbp->Checked; NextSame = tdbp->NextSame; NewRow = tdbp->NewRow; @@ -681,7 +716,7 @@ int TDBXML::LoadTableFile(PGLOBAL g, char *filename) /*********************************************************************/ /* Firstly we check whether this file have been already loaded. */ /*********************************************************************/ - if (Mode == MODE_READ || Mode == MODE_ANY) + if ((Mode == MODE_READ || Mode == MODE_ANY) && !Zipped) for (fp = dup->Openlist; fp; fp = fp->Next) if (fp->Type == type && fp->Length && fp->Count) if (!stricmp(fp->Fname, filename)) @@ -703,7 +738,7 @@ int TDBXML::LoadTableFile(PGLOBAL g, char *filename) return RC_FX; // Initialize the implementation - if (Docp->Initialize(g)) { + if (Docp->Initialize(g, Entry, Zipped)) { sprintf(g->Message, MSG(INIT_FAILED), (Usedom) ? "DOM" : "libxml2"); return RC_FX; } // endif init @@ -712,7 +747,7 @@ int TDBXML::LoadTableFile(PGLOBAL g, char *filename) htrc("TDBXML: parsing %s rc=%d\n", filename, rc); // Parse the XML file - if (Docp->ParseFile(filename)) { + if (Docp->ParseFile(g, filename)) { // Does the file exist? int h= global_open(g, MSGID_NONE, filename, _O_RDONLY); diff --git a/storage/connect/tabxml.h b/storage/connect/tabxml.h index 7ba3166881d..6c586d79dec 100644 --- a/storage/connect/tabxml.h +++ b/storage/connect/tabxml.h @@ -1,7 +1,7 @@ /*************** Tabxml H Declares Source Code File (.H) ***************/ -/* Name: TABXML.H Version 1.6 */ +/* Name: TABXML.H Version 1.7 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 2007-2015 */ +/* (C) Copyright to the author Olivier BERTRAND 2007-2016 */ /* */ /* This file contains the XML table classes declares. */ /***********************************************************************/ @@ -42,12 +42,15 @@ class DllExport XMLDEF : public TABDEF { /* Logical table description */ char *DefNs; /* Dummy name of default namespace */ char *Attrib; /* Table node attributes */ char *Hdattr; /* Header node attributes */ - int Coltype; /* Default column type */ + char *Entry; /* Zip entry name or pattern */ + int Coltype; /* Default column type */ int Limit; /* Limit of multiple values */ int Header; /* n first rows are header rows */ bool Xpand; /* Put multiple tags in several rows */ bool Usedom; /* True: DOM, False: libxml2 */ - }; // end of XMLDEF + bool Zipped; /* True: Zipped XML file(s) */ + bool Mulentries; /* True: multiple entries in zip file*/ +}; // end of XMLDEF #if defined(INCLUDE_TDBXML) /***********************************************************************/ @@ -122,7 +125,9 @@ class DllExport TDBXML : public TDBASE { bool Bufdone; // True when column buffers allocated bool Nodedone; // True when column nodes allocated bool Void; // True if the file does not exist - char *Xfile; // The XML file + bool Zipped; // True if Zipped XML file(s) + bool Mulentries; // True if multiple entries in zip file + char *Xfile; // The XML file char *Enc; // New XML table file encoding char *Tabname; // Name of Table node char *Rowname; // Name of first level nodes @@ -133,7 +138,8 @@ class DllExport TDBXML : public TDBASE { char *DefNs; // Dummy name of default namespace char *Attrib; // Table node attribut(s) char *Hdattr; // Header node attribut(s) - int Coltype; // Default column type + char *Entry; // Zip entry name or pattern + int Coltype; // Default column type int Limit; // Limit of multiple values int Header; // n first rows are header rows int Multiple; // If multiple files diff --git a/storage/connect/tabzip.cpp b/storage/connect/tabzip.cpp new file mode 100644 index 00000000000..11f414ee154 --- /dev/null +++ b/storage/connect/tabzip.cpp @@ -0,0 +1,230 @@ +/************* TabZip C++ Program Source Code File (.CPP) **************/ +/* PROGRAM NAME: TABZIP Version 1.0 */ +/* (C) Copyright to the author Olivier BERTRAND 2016 */ +/* This program are the TABZIP class DB execution routines. */ +/***********************************************************************/ + +/***********************************************************************/ +/* Include relevant sections of the MariaDB header file. */ +/***********************************************************************/ +#include <my_global.h> + +/***********************************************************************/ +/* Include application header files: */ +/* global.h is header containing all global declarations. */ +/* plgdbsem.h is header containing the DB application declarations. */ +/* (x)table.h is header containing the TDBASE declarations. */ +/* tabzip.h is header containing the TABZIP classes declarations. */ +/***********************************************************************/ +#include "global.h" +#include "plgdbsem.h" +#include "xtable.h" +#include "filamtxt.h" +#include "filamzip.h" +#include "resource.h" // for IDS_COLUMNS +#include "tabdos.h" +#include "tabzip.h" + +/* -------------------------- Class ZIPDEF --------------------------- */ + +/************************************************************************/ +/* DefineAM: define specific AM block values. */ +/************************************************************************/ +bool ZIPDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) +{ +//target = GetStringCatInfo(g, "Target", NULL); + return DOSDEF::DefineAM(g, "ZIP", poff); +} // end of DefineAM + +/***********************************************************************/ +/* GetTable: makes a new Table Description Block. */ +/***********************************************************************/ +PTDB ZIPDEF::GetTable(PGLOBAL g, MODE m) +{ + return new(g) TDBZIP(this); +} // end of GetTable + +/* ------------------------------------------------------------------- */ + +/***********************************************************************/ +/* Implementation of the TDBZIP class. */ +/***********************************************************************/ +TDBZIP::TDBZIP(PZIPDEF tdp) : TDBASE(tdp) +{ + zipfile = NULL; + zfn = tdp->Fn; +//target = tdp->target; + nexterr = UNZ_OK; +} // end of TDBZIP standard constructor + +/***********************************************************************/ +/* Allocate ZIP column description block. */ +/***********************************************************************/ +PCOL TDBZIP::MakeCol(PGLOBAL g, PCOLDEF cdp, PCOL cprec, int n) +{ + return new(g) ZIPCOL(cdp, this, cprec, n); +} // end of MakeCol + +/***********************************************************************/ +/* open a zip file. */ +/* param: filename path and the filename of the zip file to open. */ +/* return: true if open, false otherwise. */ +/***********************************************************************/ +bool TDBZIP::open(PGLOBAL g, const char *filename) +{ + if (!zipfile && !(zipfile = unzOpen64(filename))) + sprintf(g->Message, "Zipfile open error"); + + return (zipfile == NULL); +} // end of open + +/***********************************************************************/ +/* Close the zip file. */ +/***********************************************************************/ +void TDBZIP::close() +{ + if (zipfile) { + unzClose(zipfile); + zipfile = NULL; + } // endif zipfile + +} // end of close + +/***********************************************************************/ +/* ZIP Cardinality: returns table size in number of rows. */ +/***********************************************************************/ +int TDBZIP::Cardinality(PGLOBAL g) +{ + if (!g) + return 1; + else if (Cardinal < 0) { + if (!open(g, zfn)) { + unz_global_info64 ginfo; + int err = unzGetGlobalInfo64(zipfile, &ginfo); + + Cardinal = (err == UNZ_OK) ? ginfo.number_entry : 0; + } else + Cardinal = 0; + + } // endif Cardinal + + return Cardinal; +} // end of Cardinality + +/***********************************************************************/ +/* ZIP GetMaxSize: returns file size estimate in number of lines. */ +/***********************************************************************/ +int TDBZIP::GetMaxSize(PGLOBAL g) +{ + if (MaxSize < 0) + MaxSize = Cardinality(g); + + return MaxSize; +} // end of GetMaxSize + +/***********************************************************************/ +/* ZIP Access Method opening routine. */ +/***********************************************************************/ +bool TDBZIP::OpenDB(PGLOBAL g) +{ + if (Use == USE_OPEN) + // Table already open + return false; + + Use = USE_OPEN; // To be clean + return open(g, zfn); +} // end of OpenDB + +/***********************************************************************/ +/* ReadDB: Data Base read routine for ZIP access method. */ +/***********************************************************************/ +int TDBZIP::ReadDB(PGLOBAL g) +{ + if (nexterr == UNZ_END_OF_LIST_OF_FILE) + return RC_EF; + else if (nexterr != UNZ_OK) { + sprintf(g->Message, "unzGoToNextFile error %d", nexterr); + return RC_FX; + } // endif nexterr + + int err = unzGetCurrentFileInfo64(zipfile, &finfo, fn, + sizeof(fn), NULL, 0, NULL, 0); + + if (err != UNZ_OK) { + sprintf(g->Message, "unzGetCurrentFileInfo64 error %d", err); + return RC_FX; + } // endif err + + nexterr = unzGoToNextFile(zipfile); + return RC_OK; +} // end of ReadDB + +/***********************************************************************/ +/* WriteDB: Data Base write routine for ZIP access method. */ +/***********************************************************************/ +int TDBZIP::WriteDB(PGLOBAL g) +{ + strcpy(g->Message, "ZIP tables are read only"); + return RC_FX; +} // end of WriteDB + +/***********************************************************************/ +/* Data Base delete line routine for ZIP access method. */ +/***********************************************************************/ +int TDBZIP::DeleteDB(PGLOBAL g, int irc) +{ + strcpy(g->Message, "Delete not enabled for ZIP tables"); + return RC_FX; +} // end of DeleteDB + +/***********************************************************************/ +/* Data Base close routine for ZIP access method. */ +/***********************************************************************/ +void TDBZIP::CloseDB(PGLOBAL g) +{ + close(); + Use = USE_READY; // Just to be clean +} // end of CloseDB + +/* ---------------------------- ZIPCOL ------------------------------- */ + +/***********************************************************************/ +/* ZIPCOL public constructor. */ +/***********************************************************************/ +ZIPCOL::ZIPCOL(PCOLDEF cdp, PTDB tdbp, PCOL cprec, int i, PSZ am) + : COLBLK(cdp, tdbp, i) +{ + if (cprec) { + Next = cprec->GetNext(); + cprec->SetNext(this); + } else { + Next = tdbp->GetColumns(); + tdbp->SetColumns(this); + } // endif cprec + + Tdbz = (TDBZIP*)tdbp; + flag = cdp->GetOffset(); +} // end of ZIPCOL constructor + +/***********************************************************************/ +/* ReadColumn: */ +/***********************************************************************/ +void ZIPCOL::ReadColumn(PGLOBAL g) +{ + switch (flag) { + case 1: + Value->SetValue(Tdbz->finfo.compressed_size); + break; + case 2: + Value->SetValue(Tdbz->finfo.uncompressed_size); + break; + case 3: + Value->SetValue((int)Tdbz->finfo.compression_method); + break; + default: + Value->SetValue_psz((PSZ)Tdbz->fn); + } // endswitch flag + +} // end of ReadColumn + +/* -------------------------- End of tabzip -------------------------- */ diff --git a/storage/connect/tabzip.h b/storage/connect/tabzip.h new file mode 100644 index 00000000000..6f1735258e7 --- /dev/null +++ b/storage/connect/tabzip.h @@ -0,0 +1,100 @@ +/*************** tabzip H Declares Source Code File (.H) ***************/ +/* Name: tabzip.h Version 1.0 */ +/* */ +/* (C) Copyright to the author Olivier BERTRAND 2016 */ +/* */ +/* This file contains the ZIP classe declares. */ +/***********************************************************************/ +#include "osutil.h" +#include "block.h" +#include "colblk.h" +#include "xtable.h" +#include "unzip.h" + +typedef class ZIPDEF *PZIPDEF; +typedef class TDBZIP *PTDBZIP; +typedef class ZIPCOL *PZIPCOL; + +/***********************************************************************/ +/* ZIP table: display info about a ZIP file. */ +/***********************************************************************/ +class DllExport ZIPDEF : public DOSDEF { /* Table description */ + friend class TDBZIP; + friend class ZIPFAM; +public: + // Constructor + ZIPDEF(void) {} + + // Implementation + virtual const char *GetType(void) {return "ZIP";} + + // Methods + virtual bool DefineAM(PGLOBAL g, LPCSTR am, int poff); + virtual PTDB GetTable(PGLOBAL g, MODE m); + +protected: + // Members + PSZ target; // The inside file to query +}; // end of ZIPDEF + +/***********************************************************************/ +/* This is the ZIP Access Method class declaration. */ +/***********************************************************************/ +class DllExport TDBZIP : public TDBASE { + friend class ZIPCOL; +public: + // Constructor + TDBZIP(PZIPDEF tdp); + + // Implementation + virtual AMT GetAmType(void) {return TYPE_AM_ZIP;} + + // Methods + virtual PCOL MakeCol(PGLOBAL g, PCOLDEF cdp, PCOL cprec, int n); + virtual int Cardinality(PGLOBAL g); + virtual int GetMaxSize(PGLOBAL g); + virtual int GetRecpos(void) {return 0;} + + // Database routines + virtual bool OpenDB(PGLOBAL g); + virtual int ReadDB(PGLOBAL g); + virtual int WriteDB(PGLOBAL g); + virtual int DeleteDB(PGLOBAL g, int irc); + virtual void CloseDB(PGLOBAL g); + +protected: + bool open(PGLOBAL g, const char *filename); + void close(void); + + // Members + unzFile zipfile; // The ZIP container file + PSZ zfn; // The ZIP file name +//PSZ target; + unz_file_info64 finfo; // The current file info + char fn[FILENAME_MAX]; // The current file name + int nexterr; // Next file error +}; // end of class TDBZIP + +/***********************************************************************/ +/* Class ZIPCOL: ZIP access method column descriptor. */ +/***********************************************************************/ +class DllExport ZIPCOL : public COLBLK { + friend class TDBZIP; +public: + // Constructors + ZIPCOL(PCOLDEF cdp, PTDB tdbp, PCOL cprec, int i, PSZ am = "ZIP"); + + // Implementation + virtual int GetAmType(void) { return TYPE_AM_ZIP; } + + // Methods + virtual void ReadColumn(PGLOBAL g); + +protected: + // Default constructor not to be used + ZIPCOL(void) {} + + // Members + TDBZIP *Tdbz; + int flag; +}; // end of class ZIPCOL diff --git a/storage/connect/unzip.c b/storage/connect/unzip.c new file mode 100644 index 00000000000..909350435a5 --- /dev/null +++ b/storage/connect/unzip.c @@ -0,0 +1,2125 @@ +/* unzip.c -- IO for uncompress .zip files using zlib + Version 1.1, February 14h, 2010 + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications of Unzip for Zip64 + Copyright (C) 2007-2008 Even Rouault + + Modifications for Zip64 support on both zip and unzip + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + + + ------------------------------------------------------------------------------------ + Decryption code comes from crypt.c by Info-ZIP but has been greatly reduced in terms of + compatibility with older software. The following is from the original crypt.c. + Code woven in by Terry Thorsen 1/2003. + + Copyright (c) 1990-2000 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2000-Apr-09 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, all these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html + + crypt.c (full version) by Info-ZIP. Last revised: [see crypt.h] + + The encryption/decryption parts of this source code (as opposed to the + non-echoing password parts) were originally written in Europe. The + whole source package can be freely distributed, including from the USA. + (Prior to January 2000, re-export from the US was a violation of US law.) + + This encryption code is a direct transcription of the algorithm from + Roger Schlafly, described by Phil Katz in the file appnote.txt. This + file (appnote.txt) is distributed with the PKZIP program (even in the + version without encryption capabilities). + + ------------------------------------------------------------------------------------ + + Changes in unzip.c + + 2007-2008 - Even Rouault - Addition of cpl_unzGetCurrentFileZStreamPos + 2007-2008 - Even Rouault - Decoration of symbol names unz* -> cpl_unz* + 2007-2008 - Even Rouault - Remove old C style function prototypes + 2007-2008 - Even Rouault - Add unzip support for ZIP64 + + Copyright (C) 2007-2008 Even Rouault + + + Oct-2009 - Mathias Svensson - Removed cpl_* from symbol names (Even Rouault added them but since this is now moved to a new project (minizip64) I renamed them again). + Oct-2009 - Mathias Svensson - Fixed problem if uncompressed size was > 4G and compressed size was <4G + should only read the compressed/uncompressed size from the Zip64 format if + the size from normal header was 0xFFFFFFFF + Oct-2009 - Mathias Svensson - Applied some bug fixes from paches recived from Gilles Vollant + Oct-2009 - Mathias Svensson - Applied support to unzip files with compression mathod BZIP2 (bzip2 lib is required) + Patch created by Daniel Borca + + Jan-2010 - back to unzip and minizip 1.0 name scheme, with compatibility layer + + Copyright (C) 1998 - 2010 Gilles Vollant, Even Rouault, Mathias Svensson + +*/ + + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#ifndef NOUNCRYPT + #define NOUNCRYPT +#endif + +#include "zlib.h" +#include "unzip.h" + +#ifdef STDC +# include <stddef.h> +# include <string.h> +# include <stdlib.h> +#endif +#ifdef NO_ERRNO_H + extern int errno; +#else +# include <errno.h> +#endif + + +#ifndef local +# define local static +#endif +/* compile with -Dlocal if your debugger can't find static symbols */ + + +#ifndef CASESENSITIVITYDEFAULT_NO +# if !defined(unix) && !defined(CASESENSITIVITYDEFAULT_YES) +# define CASESENSITIVITYDEFAULT_NO +# endif +#endif + + +#ifndef UNZ_BUFSIZE +#define UNZ_BUFSIZE (16384) +#endif + +#ifndef UNZ_MAXFILENAMEINZIP +#define UNZ_MAXFILENAMEINZIP (256) +#endif + +#ifndef ALLOC +# define ALLOC(size) (malloc(size)) +#endif +#ifndef TRYFREE +# define TRYFREE(p) {if (p) free(p);} +#endif + +#define SIZECENTRALDIRITEM (0x2e) +#define SIZEZIPLOCALHEADER (0x1e) + + +const char unz_copyright[] = + " unzip 1.01 Copyright 1998-2004 Gilles Vollant - http://www.winimage.com/zLibDll"; + +/* unz_file_info_interntal contain internal info about a file in zipfile*/ +typedef struct unz_file_info64_internal_s +{ + ZPOS64_T offset_curfile;/* relative offset of local header 8 bytes */ +} unz_file_info64_internal; + + +/* file_in_zip_read_info_s contain internal information about a file in zipfile, + when reading and decompress it */ +typedef struct +{ + char *read_buffer; /* internal buffer for compressed data */ + z_stream stream; /* zLib stream structure for inflate */ + +#ifdef HAVE_BZIP2 + bz_stream bstream; /* bzLib stream structure for bziped */ +#endif + + ZPOS64_T pos_in_zipfile; /* position in byte on the zipfile, for fseek*/ + uLong stream_initialised; /* flag set if stream structure is initialised*/ + + ZPOS64_T offset_local_extrafield;/* offset of the local extra field */ + uInt size_local_extrafield;/* size of the local extra field */ + ZPOS64_T pos_local_extrafield; /* position in the local extra field in read*/ + ZPOS64_T total_out_64; + + uLong crc32; /* crc32 of all data uncompressed */ + uLong crc32_wait; /* crc32 we must obtain after decompress all */ + ZPOS64_T rest_read_compressed; /* number of byte to be decompressed */ + ZPOS64_T rest_read_uncompressed;/*number of byte to be obtained after decomp*/ + zlib_filefunc64_32_def z_filefunc; + voidpf filestream; /* io structore of the zipfile */ + uLong compression_method; /* compression method (0==store) */ + ZPOS64_T byte_before_the_zipfile;/* byte before the zipfile, (>0 for sfx)*/ + int raw; +} file_in_zip64_read_info_s; + + +/* unz64_s contain internal information about the zipfile +*/ +typedef struct +{ + zlib_filefunc64_32_def z_filefunc; + int is64bitOpenFunction; + voidpf filestream; /* io structore of the zipfile */ + unz_global_info64 gi; /* public global information */ + ZPOS64_T byte_before_the_zipfile;/* byte before the zipfile, (>0 for sfx)*/ + ZPOS64_T num_file; /* number of the current file in the zipfile*/ + ZPOS64_T pos_in_central_dir; /* pos of the current file in the central dir*/ + ZPOS64_T current_file_ok; /* flag about the usability of the current file*/ + ZPOS64_T central_pos; /* position of the beginning of the central dir*/ + + ZPOS64_T size_central_dir; /* size of the central directory */ + ZPOS64_T offset_central_dir; /* offset of start of central directory with + respect to the starting disk number */ + + unz_file_info64 cur_file_info; /* public info about the current file in zip*/ + unz_file_info64_internal cur_file_info_internal; /* private info about it*/ + file_in_zip64_read_info_s* pfile_in_zip_read; /* structure about the current + file if we are decompressing it */ + int encrypted; + + int isZip64; + +# ifndef NOUNCRYPT + unsigned long keys[3]; /* keys defining the pseudo-random sequence */ + const z_crc_t* pcrc_32_tab; +# endif +} unz64_s; + + +#ifndef NOUNCRYPT +#include "crypt.h" +#endif + +/* =========================================================================== + Read a byte from a gz_stream; update next_in and avail_in. Return EOF + for end of file. + IN assertion: the stream s has been sucessfully opened for reading. +*/ + + +local int unz64local_getByte OF(( + const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + int *pi)); + +local int unz64local_getByte(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, int *pi) +{ + unsigned char c; + int err = (int)ZREAD64(*pzlib_filefunc_def,filestream,&c,1); + if (err==1) + { + *pi = (int)c; + return UNZ_OK; + } + else + { + if (ZERROR64(*pzlib_filefunc_def,filestream)) + return UNZ_ERRNO; + else + return UNZ_EOF; + } +} + + +/* =========================================================================== + Reads a long in LSB order from the given gz_stream. Sets +*/ +local int unz64local_getShort OF(( + const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + uLong *pX)); + +local int unz64local_getShort (const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + uLong *pX) +{ + uLong x ; + int i = 0; + int err; + + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x = (uLong)i; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((uLong)i)<<8; + + if (err==UNZ_OK) + *pX = x; + else + *pX = 0; + return err; +} + +local int unz64local_getLong OF(( + const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + uLong *pX)); + +local int unz64local_getLong (const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + uLong *pX) +{ + uLong x ; + int i = 0; + int err; + + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x = (uLong)i; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((uLong)i)<<8; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((uLong)i)<<16; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((uLong)i)<<24; + + if (err==UNZ_OK) + *pX = x; + else + *pX = 0; + return err; +} + +local int unz64local_getLong64 OF(( + const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + ZPOS64_T *pX)); + + +local int unz64local_getLong64 (const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + ZPOS64_T *pX) +{ + ZPOS64_T x ; + int i = 0; + int err; + + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x = (ZPOS64_T)i; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<8; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<16; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<24; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<32; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<40; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<48; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<56; + + if (err==UNZ_OK) + *pX = x; + else + *pX = 0; + return err; +} + +/* My own strcmpi / strcasecmp */ +local int strcmpcasenosensitive_internal (const char* fileName1, const char* fileName2) +{ + for (;;) + { + char c1=*(fileName1++); + char c2=*(fileName2++); + if ((c1>='a') && (c1<='z')) + c1 -= 0x20; + if ((c2>='a') && (c2<='z')) + c2 -= 0x20; + if (c1=='\0') + return ((c2=='\0') ? 0 : -1); + if (c2=='\0') + return 1; + if (c1<c2) + return -1; + if (c1>c2) + return 1; + } +} + + +#ifdef CASESENSITIVITYDEFAULT_NO +#define CASESENSITIVITYDEFAULTVALUE 2 +#else +#define CASESENSITIVITYDEFAULTVALUE 1 +#endif + +#ifndef STRCMPCASENOSENTIVEFUNCTION +#define STRCMPCASENOSENTIVEFUNCTION strcmpcasenosensitive_internal +#endif + +/* + Compare two filename (fileName1,fileName2). + If iCaseSenisivity = 1, comparision is case sensitivity (like strcmp) + If iCaseSenisivity = 2, comparision is not case sensitivity (like strcmpi + or strcasecmp) + If iCaseSenisivity = 0, case sensitivity is defaut of your operating system + (like 1 on Unix, 2 on Windows) + +*/ +extern int ZEXPORT unzStringFileNameCompare (const char* fileName1, + const char* fileName2, + int iCaseSensitivity) + +{ + if (iCaseSensitivity==0) + iCaseSensitivity=CASESENSITIVITYDEFAULTVALUE; + + if (iCaseSensitivity==1) + return strcmp(fileName1,fileName2); + + return STRCMPCASENOSENTIVEFUNCTION(fileName1,fileName2); +} + +#ifndef BUFREADCOMMENT +#define BUFREADCOMMENT (0x400) +#endif + +/* + Locate the Central directory of a zipfile (at the end, just before + the global comment) +*/ +local ZPOS64_T unz64local_SearchCentralDir OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream)); +local ZPOS64_T unz64local_SearchCentralDir(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream) +{ + unsigned char* buf; + ZPOS64_T uSizeFile; + ZPOS64_T uBackRead; + ZPOS64_T uMaxBack=0xffff; /* maximum size of global comment */ + ZPOS64_T uPosFound=0; + + if (ZSEEK64(*pzlib_filefunc_def,filestream,0,ZLIB_FILEFUNC_SEEK_END) != 0) + return 0; + + + uSizeFile = ZTELL64(*pzlib_filefunc_def,filestream); + + if (uMaxBack>uSizeFile) + uMaxBack = uSizeFile; + + buf = (unsigned char*)ALLOC(BUFREADCOMMENT+4); + if (buf==NULL) + return 0; + + uBackRead = 4; + while (uBackRead<uMaxBack) + { + uLong uReadSize; + ZPOS64_T uReadPos ; + int i; + if (uBackRead+BUFREADCOMMENT>uMaxBack) + uBackRead = uMaxBack; + else + uBackRead+=BUFREADCOMMENT; + uReadPos = uSizeFile-uBackRead ; + + uReadSize = ((BUFREADCOMMENT+4) < (uSizeFile-uReadPos)) ? + (BUFREADCOMMENT+4) : (uLong)(uSizeFile-uReadPos); + if (ZSEEK64(*pzlib_filefunc_def,filestream,uReadPos,ZLIB_FILEFUNC_SEEK_SET)!=0) + break; + + if (ZREAD64(*pzlib_filefunc_def,filestream,buf,uReadSize)!=uReadSize) + break; + + for (i=(int)uReadSize-3; (i--)>0;) + if (((*(buf+i))==0x50) && ((*(buf+i+1))==0x4b) && + ((*(buf+i+2))==0x05) && ((*(buf+i+3))==0x06)) + { + uPosFound = uReadPos+i; + break; + } + + if (uPosFound!=0) + break; + } + TRYFREE(buf); + return uPosFound; +} + + +/* + Locate the Central directory 64 of a zipfile (at the end, just before + the global comment) +*/ +local ZPOS64_T unz64local_SearchCentralDir64 OF(( + const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream)); + +local ZPOS64_T unz64local_SearchCentralDir64(const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream) +{ + unsigned char* buf; + ZPOS64_T uSizeFile; + ZPOS64_T uBackRead; + ZPOS64_T uMaxBack=0xffff; /* maximum size of global comment */ + ZPOS64_T uPosFound=0; + uLong uL; + ZPOS64_T relativeOffset; + + if (ZSEEK64(*pzlib_filefunc_def,filestream,0,ZLIB_FILEFUNC_SEEK_END) != 0) + return 0; + + + uSizeFile = ZTELL64(*pzlib_filefunc_def,filestream); + + if (uMaxBack>uSizeFile) + uMaxBack = uSizeFile; + + buf = (unsigned char*)ALLOC(BUFREADCOMMENT+4); + if (buf==NULL) + return 0; + + uBackRead = 4; + while (uBackRead<uMaxBack) + { + uLong uReadSize; + ZPOS64_T uReadPos; + int i; + if (uBackRead+BUFREADCOMMENT>uMaxBack) + uBackRead = uMaxBack; + else + uBackRead+=BUFREADCOMMENT; + uReadPos = uSizeFile-uBackRead ; + + uReadSize = ((BUFREADCOMMENT+4) < (uSizeFile-uReadPos)) ? + (BUFREADCOMMENT+4) : (uLong)(uSizeFile-uReadPos); + if (ZSEEK64(*pzlib_filefunc_def,filestream,uReadPos,ZLIB_FILEFUNC_SEEK_SET)!=0) + break; + + if (ZREAD64(*pzlib_filefunc_def,filestream,buf,uReadSize)!=uReadSize) + break; + + for (i=(int)uReadSize-3; (i--)>0;) + if (((*(buf+i))==0x50) && ((*(buf+i+1))==0x4b) && + ((*(buf+i+2))==0x06) && ((*(buf+i+3))==0x07)) + { + uPosFound = uReadPos+i; + break; + } + + if (uPosFound!=0) + break; + } + TRYFREE(buf); + if (uPosFound == 0) + return 0; + + /* Zip64 end of central directory locator */ + if (ZSEEK64(*pzlib_filefunc_def,filestream, uPosFound,ZLIB_FILEFUNC_SEEK_SET)!=0) + return 0; + + /* the signature, already checked */ + if (unz64local_getLong(pzlib_filefunc_def,filestream,&uL)!=UNZ_OK) + return 0; + + /* number of the disk with the start of the zip64 end of central directory */ + if (unz64local_getLong(pzlib_filefunc_def,filestream,&uL)!=UNZ_OK) + return 0; + if (uL != 0) + return 0; + + /* relative offset of the zip64 end of central directory record */ + if (unz64local_getLong64(pzlib_filefunc_def,filestream,&relativeOffset)!=UNZ_OK) + return 0; + + /* total number of disks */ + if (unz64local_getLong(pzlib_filefunc_def,filestream,&uL)!=UNZ_OK) + return 0; + if (uL != 1) + return 0; + + /* Goto end of central directory record */ + if (ZSEEK64(*pzlib_filefunc_def,filestream, relativeOffset,ZLIB_FILEFUNC_SEEK_SET)!=0) + return 0; + + /* the signature */ + if (unz64local_getLong(pzlib_filefunc_def,filestream,&uL)!=UNZ_OK) + return 0; + + if (uL != 0x06064b50) + return 0; + + return relativeOffset; +} + +/* + Open a Zip file. path contain the full pathname (by example, + on a Windows NT computer "c:\\test\\zlib114.zip" or on an Unix computer + "zlib/zlib114.zip". + If the zipfile cannot be opened (file doesn't exist or in not valid), the + return value is NULL. + Else, the return value is a unzFile Handle, usable with other function + of this unzip package. +*/ +local unzFile unzOpenInternal (const void *path, + zlib_filefunc64_32_def* pzlib_filefunc64_32_def, + int is64bitOpenFunction) +{ + unz64_s us; + unz64_s *s; + ZPOS64_T central_pos; + uLong uL; + + uLong number_disk; /* number of the current dist, used for + spaning ZIP, unsupported, always 0*/ + uLong number_disk_with_CD; /* number the the disk with central dir, used + for spaning ZIP, unsupported, always 0*/ + ZPOS64_T number_entry_CD; /* total number of entries in + the central dir + (same than number_entry on nospan) */ + + int err=UNZ_OK; + + if (unz_copyright[0]!=' ') + return NULL; + + us.z_filefunc.zseek32_file = NULL; + us.z_filefunc.ztell32_file = NULL; + if (pzlib_filefunc64_32_def==NULL) + fill_fopen64_filefunc(&us.z_filefunc.zfile_func64); + else + us.z_filefunc = *pzlib_filefunc64_32_def; + us.is64bitOpenFunction = is64bitOpenFunction; + + + + us.filestream = ZOPEN64(us.z_filefunc, + path, + ZLIB_FILEFUNC_MODE_READ | + ZLIB_FILEFUNC_MODE_EXISTING); + if (us.filestream==NULL) + return NULL; + + central_pos = unz64local_SearchCentralDir64(&us.z_filefunc,us.filestream); + if (central_pos) + { + uLong uS; + ZPOS64_T uL64; + + us.isZip64 = 1; + + if (ZSEEK64(us.z_filefunc, us.filestream, + central_pos,ZLIB_FILEFUNC_SEEK_SET)!=0) + err=UNZ_ERRNO; + + /* the signature, already checked */ + if (unz64local_getLong(&us.z_filefunc, us.filestream,&uL)!=UNZ_OK) + err=UNZ_ERRNO; + + /* size of zip64 end of central directory record */ + if (unz64local_getLong64(&us.z_filefunc, us.filestream,&uL64)!=UNZ_OK) + err=UNZ_ERRNO; + + /* version made by */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&uS)!=UNZ_OK) + err=UNZ_ERRNO; + + /* version needed to extract */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&uS)!=UNZ_OK) + err=UNZ_ERRNO; + + /* number of this disk */ + if (unz64local_getLong(&us.z_filefunc, us.filestream,&number_disk)!=UNZ_OK) + err=UNZ_ERRNO; + + /* number of the disk with the start of the central directory */ + if (unz64local_getLong(&us.z_filefunc, us.filestream,&number_disk_with_CD)!=UNZ_OK) + err=UNZ_ERRNO; + + /* total number of entries in the central directory on this disk */ + if (unz64local_getLong64(&us.z_filefunc, us.filestream,&us.gi.number_entry)!=UNZ_OK) + err=UNZ_ERRNO; + + /* total number of entries in the central directory */ + if (unz64local_getLong64(&us.z_filefunc, us.filestream,&number_entry_CD)!=UNZ_OK) + err=UNZ_ERRNO; + + if ((number_entry_CD!=us.gi.number_entry) || + (number_disk_with_CD!=0) || + (number_disk!=0)) + err=UNZ_BADZIPFILE; + + /* size of the central directory */ + if (unz64local_getLong64(&us.z_filefunc, us.filestream,&us.size_central_dir)!=UNZ_OK) + err=UNZ_ERRNO; + + /* offset of start of central directory with respect to the + starting disk number */ + if (unz64local_getLong64(&us.z_filefunc, us.filestream,&us.offset_central_dir)!=UNZ_OK) + err=UNZ_ERRNO; + + us.gi.size_comment = 0; + } + else + { + central_pos = unz64local_SearchCentralDir(&us.z_filefunc,us.filestream); + if (central_pos==0) + err=UNZ_ERRNO; + + us.isZip64 = 0; + + if (ZSEEK64(us.z_filefunc, us.filestream, + central_pos,ZLIB_FILEFUNC_SEEK_SET)!=0) + err=UNZ_ERRNO; + + /* the signature, already checked */ + if (unz64local_getLong(&us.z_filefunc, us.filestream,&uL)!=UNZ_OK) + err=UNZ_ERRNO; + + /* number of this disk */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&number_disk)!=UNZ_OK) + err=UNZ_ERRNO; + + /* number of the disk with the start of the central directory */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&number_disk_with_CD)!=UNZ_OK) + err=UNZ_ERRNO; + + /* total number of entries in the central dir on this disk */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&uL)!=UNZ_OK) + err=UNZ_ERRNO; + us.gi.number_entry = uL; + + /* total number of entries in the central dir */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&uL)!=UNZ_OK) + err=UNZ_ERRNO; + number_entry_CD = uL; + + if ((number_entry_CD!=us.gi.number_entry) || + (number_disk_with_CD!=0) || + (number_disk!=0)) + err=UNZ_BADZIPFILE; + + /* size of the central directory */ + if (unz64local_getLong(&us.z_filefunc, us.filestream,&uL)!=UNZ_OK) + err=UNZ_ERRNO; + us.size_central_dir = uL; + + /* offset of start of central directory with respect to the + starting disk number */ + if (unz64local_getLong(&us.z_filefunc, us.filestream,&uL)!=UNZ_OK) + err=UNZ_ERRNO; + us.offset_central_dir = uL; + + /* zipfile comment length */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&us.gi.size_comment)!=UNZ_OK) + err=UNZ_ERRNO; + } + + if ((central_pos<us.offset_central_dir+us.size_central_dir) && + (err==UNZ_OK)) + err=UNZ_BADZIPFILE; + + if (err!=UNZ_OK) + { + ZCLOSE64(us.z_filefunc, us.filestream); + return NULL; + } + + us.byte_before_the_zipfile = central_pos - + (us.offset_central_dir+us.size_central_dir); + us.central_pos = central_pos; + us.pfile_in_zip_read = NULL; + us.encrypted = 0; + + + s=(unz64_s*)ALLOC(sizeof(unz64_s)); + if( s != NULL) + { + *s=us; + unzGoToFirstFile((unzFile)s); + } + return (unzFile)s; +} + + +extern unzFile ZEXPORT unzOpen2 (const char *path, + zlib_filefunc_def* pzlib_filefunc32_def) +{ + if (pzlib_filefunc32_def != NULL) + { + zlib_filefunc64_32_def zlib_filefunc64_32_def_fill; + fill_zlib_filefunc64_32_def_from_filefunc32(&zlib_filefunc64_32_def_fill,pzlib_filefunc32_def); + return unzOpenInternal(path, &zlib_filefunc64_32_def_fill, 0); + } + else + return unzOpenInternal(path, NULL, 0); +} + +extern unzFile ZEXPORT unzOpen2_64 (const void *path, + zlib_filefunc64_def* pzlib_filefunc_def) +{ + if (pzlib_filefunc_def != NULL) + { + zlib_filefunc64_32_def zlib_filefunc64_32_def_fill; + zlib_filefunc64_32_def_fill.zfile_func64 = *pzlib_filefunc_def; + zlib_filefunc64_32_def_fill.ztell32_file = NULL; + zlib_filefunc64_32_def_fill.zseek32_file = NULL; + return unzOpenInternal(path, &zlib_filefunc64_32_def_fill, 1); + } + else + return unzOpenInternal(path, NULL, 1); +} + +extern unzFile ZEXPORT unzOpen (const char *path) +{ + return unzOpenInternal(path, NULL, 0); +} + +extern unzFile ZEXPORT unzOpen64 (const void *path) +{ + return unzOpenInternal(path, NULL, 1); +} + +/* + Close a ZipFile opened with unzOpen. + If there is files inside the .Zip opened with unzOpenCurrentFile (see later), + these files MUST be closed with unzCloseCurrentFile before call unzClose. + return UNZ_OK if there is no problem. */ +extern int ZEXPORT unzClose (unzFile file) +{ + unz64_s* s; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + + if (s->pfile_in_zip_read!=NULL) + unzCloseCurrentFile(file); + + ZCLOSE64(s->z_filefunc, s->filestream); + TRYFREE(s); + return UNZ_OK; +} + + +/* + Write info about the ZipFile in the *pglobal_info structure. + No preparation of the structure is needed + return UNZ_OK if there is no problem. */ +extern int ZEXPORT unzGetGlobalInfo64 (unzFile file, unz_global_info64* pglobal_info) +{ + unz64_s* s; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + *pglobal_info=s->gi; + return UNZ_OK; +} + +extern int ZEXPORT unzGetGlobalInfo (unzFile file, unz_global_info* pglobal_info32) +{ + unz64_s* s; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + /* to do : check if number_entry is not truncated */ + pglobal_info32->number_entry = (uLong)s->gi.number_entry; + pglobal_info32->size_comment = s->gi.size_comment; + return UNZ_OK; +} +/* + Translate date/time from Dos format to tm_unz (readable more easilty) +*/ +local void unz64local_DosDateToTmuDate (ZPOS64_T ulDosDate, tm_unz* ptm) +{ + ZPOS64_T uDate; + uDate = (ZPOS64_T)(ulDosDate>>16); + ptm->tm_mday = (uInt)(uDate&0x1f) ; + ptm->tm_mon = (uInt)((((uDate)&0x1E0)/0x20)-1) ; + ptm->tm_year = (uInt)(((uDate&0x0FE00)/0x0200)+1980) ; + + ptm->tm_hour = (uInt) ((ulDosDate &0xF800)/0x800); + ptm->tm_min = (uInt) ((ulDosDate&0x7E0)/0x20) ; + ptm->tm_sec = (uInt) (2*(ulDosDate&0x1f)) ; +} + +/* + Get Info about the current file in the zipfile, with internal only info +*/ +local int unz64local_GetCurrentFileInfoInternal OF((unzFile file, + unz_file_info64 *pfile_info, + unz_file_info64_internal + *pfile_info_internal, + char *szFileName, + uLong fileNameBufferSize, + void *extraField, + uLong extraFieldBufferSize, + char *szComment, + uLong commentBufferSize)); + +local int unz64local_GetCurrentFileInfoInternal (unzFile file, + unz_file_info64 *pfile_info, + unz_file_info64_internal + *pfile_info_internal, + char *szFileName, + uLong fileNameBufferSize, + void *extraField, + uLong extraFieldBufferSize, + char *szComment, + uLong commentBufferSize) +{ + unz64_s* s; + unz_file_info64 file_info; + unz_file_info64_internal file_info_internal; + int err=UNZ_OK; + uLong uMagic; + long lSeek=0; + uLong uL; + + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + if (ZSEEK64(s->z_filefunc, s->filestream, + s->pos_in_central_dir+s->byte_before_the_zipfile, + ZLIB_FILEFUNC_SEEK_SET)!=0) + err=UNZ_ERRNO; + + + /* we check the magic */ + if (err==UNZ_OK) + { + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uMagic) != UNZ_OK) + err=UNZ_ERRNO; + else if (uMagic!=0x02014b50) + err=UNZ_BADZIPFILE; + } + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.version) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.version_needed) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.flag) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.compression_method) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&file_info.dosDate) != UNZ_OK) + err=UNZ_ERRNO; + + unz64local_DosDateToTmuDate(file_info.dosDate,&file_info.tmu_date); + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&file_info.crc) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uL) != UNZ_OK) + err=UNZ_ERRNO; + file_info.compressed_size = uL; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uL) != UNZ_OK) + err=UNZ_ERRNO; + file_info.uncompressed_size = uL; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.size_filename) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.size_file_extra) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.size_file_comment) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.disk_num_start) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.internal_fa) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&file_info.external_fa) != UNZ_OK) + err=UNZ_ERRNO; + + // relative offset of local header + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uL) != UNZ_OK) + err=UNZ_ERRNO; + file_info_internal.offset_curfile = uL; + + lSeek+=file_info.size_filename; + if ((err==UNZ_OK) && (szFileName!=NULL)) + { + uLong uSizeRead ; + if (file_info.size_filename<fileNameBufferSize) + { + *(szFileName+file_info.size_filename)='\0'; + uSizeRead = file_info.size_filename; + } + else + uSizeRead = fileNameBufferSize; + + if ((file_info.size_filename>0) && (fileNameBufferSize>0)) + if (ZREAD64(s->z_filefunc, s->filestream,szFileName,uSizeRead)!=uSizeRead) + err=UNZ_ERRNO; + lSeek -= uSizeRead; + } + + // Read extrafield + if ((err==UNZ_OK) && (extraField!=NULL)) + { + ZPOS64_T uSizeRead ; + if (file_info.size_file_extra<extraFieldBufferSize) + uSizeRead = file_info.size_file_extra; + else + uSizeRead = extraFieldBufferSize; + + if (lSeek!=0) + { + if (ZSEEK64(s->z_filefunc, s->filestream,lSeek,ZLIB_FILEFUNC_SEEK_CUR)==0) + lSeek=0; + else + err=UNZ_ERRNO; + } + + if ((file_info.size_file_extra>0) && (extraFieldBufferSize>0)) + if (ZREAD64(s->z_filefunc, s->filestream,extraField,(uLong)uSizeRead)!=uSizeRead) + err=UNZ_ERRNO; + + lSeek += file_info.size_file_extra - (uLong)uSizeRead; + } + else + lSeek += file_info.size_file_extra; + + + if ((err==UNZ_OK) && (file_info.size_file_extra != 0)) + { + uLong acc = 0; + + // since lSeek now points to after the extra field we need to move back + lSeek -= file_info.size_file_extra; + + if (lSeek!=0) + { + if (ZSEEK64(s->z_filefunc, s->filestream,lSeek,ZLIB_FILEFUNC_SEEK_CUR)==0) + lSeek=0; + else + err=UNZ_ERRNO; + } + + while(acc < file_info.size_file_extra) + { + uLong headerId; + uLong dataSize; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&headerId) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&dataSize) != UNZ_OK) + err=UNZ_ERRNO; + + /* ZIP64 extra fields */ + if (headerId == 0x0001) + { + uLong uL; + + if(file_info.uncompressed_size == MAXU32) + { + if (unz64local_getLong64(&s->z_filefunc, s->filestream,&file_info.uncompressed_size) != UNZ_OK) + err=UNZ_ERRNO; + } + + if(file_info.compressed_size == MAXU32) + { + if (unz64local_getLong64(&s->z_filefunc, s->filestream,&file_info.compressed_size) != UNZ_OK) + err=UNZ_ERRNO; + } + + if(file_info_internal.offset_curfile == MAXU32) + { + /* Relative Header offset */ + if (unz64local_getLong64(&s->z_filefunc, s->filestream,&file_info_internal.offset_curfile) != UNZ_OK) + err=UNZ_ERRNO; + } + + if(file_info.disk_num_start == MAXU32) + { + /* Disk Start Number */ + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uL) != UNZ_OK) + err=UNZ_ERRNO; + } + + } + else + { + if (ZSEEK64(s->z_filefunc, s->filestream,dataSize,ZLIB_FILEFUNC_SEEK_CUR)!=0) + err=UNZ_ERRNO; + } + + acc += 2 + 2 + dataSize; + } + } + + if ((err==UNZ_OK) && (szComment!=NULL)) + { + uLong uSizeRead ; + if (file_info.size_file_comment<commentBufferSize) + { + *(szComment+file_info.size_file_comment)='\0'; + uSizeRead = file_info.size_file_comment; + } + else + uSizeRead = commentBufferSize; + + if (lSeek!=0) + { + if (ZSEEK64(s->z_filefunc, s->filestream,lSeek,ZLIB_FILEFUNC_SEEK_CUR)==0) + lSeek=0; + else + err=UNZ_ERRNO; + } + + if ((file_info.size_file_comment>0) && (commentBufferSize>0)) + if (ZREAD64(s->z_filefunc, s->filestream,szComment,uSizeRead)!=uSizeRead) + err=UNZ_ERRNO; + lSeek+=file_info.size_file_comment - uSizeRead; + } + else + lSeek+=file_info.size_file_comment; + + + if ((err==UNZ_OK) && (pfile_info!=NULL)) + *pfile_info=file_info; + + if ((err==UNZ_OK) && (pfile_info_internal!=NULL)) + *pfile_info_internal=file_info_internal; + + return err; +} + + + +/* + Write info about the ZipFile in the *pglobal_info structure. + No preparation of the structure is needed + return UNZ_OK if there is no problem. +*/ +extern int ZEXPORT unzGetCurrentFileInfo64 (unzFile file, + unz_file_info64 * pfile_info, + char * szFileName, uLong fileNameBufferSize, + void *extraField, uLong extraFieldBufferSize, + char* szComment, uLong commentBufferSize) +{ + return unz64local_GetCurrentFileInfoInternal(file,pfile_info,NULL, + szFileName,fileNameBufferSize, + extraField,extraFieldBufferSize, + szComment,commentBufferSize); +} + +extern int ZEXPORT unzGetCurrentFileInfo (unzFile file, + unz_file_info * pfile_info, + char * szFileName, uLong fileNameBufferSize, + void *extraField, uLong extraFieldBufferSize, + char* szComment, uLong commentBufferSize) +{ + int err; + unz_file_info64 file_info64; + err = unz64local_GetCurrentFileInfoInternal(file,&file_info64,NULL, + szFileName,fileNameBufferSize, + extraField,extraFieldBufferSize, + szComment,commentBufferSize); + if ((err==UNZ_OK) && (pfile_info != NULL)) + { + pfile_info->version = file_info64.version; + pfile_info->version_needed = file_info64.version_needed; + pfile_info->flag = file_info64.flag; + pfile_info->compression_method = file_info64.compression_method; + pfile_info->dosDate = file_info64.dosDate; + pfile_info->crc = file_info64.crc; + + pfile_info->size_filename = file_info64.size_filename; + pfile_info->size_file_extra = file_info64.size_file_extra; + pfile_info->size_file_comment = file_info64.size_file_comment; + + pfile_info->disk_num_start = file_info64.disk_num_start; + pfile_info->internal_fa = file_info64.internal_fa; + pfile_info->external_fa = file_info64.external_fa; + + pfile_info->tmu_date = file_info64.tmu_date, + + + pfile_info->compressed_size = (uLong)file_info64.compressed_size; + pfile_info->uncompressed_size = (uLong)file_info64.uncompressed_size; + + } + return err; +} +/* + Set the current file of the zipfile to the first file. + return UNZ_OK if there is no problem +*/ +extern int ZEXPORT unzGoToFirstFile (unzFile file) +{ + int err=UNZ_OK; + unz64_s* s; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + s->pos_in_central_dir=s->offset_central_dir; + s->num_file=0; + err=unz64local_GetCurrentFileInfoInternal(file,&s->cur_file_info, + &s->cur_file_info_internal, + NULL,0,NULL,0,NULL,0); + s->current_file_ok = (err == UNZ_OK); + return err; +} + +/* + Set the current file of the zipfile to the next file. + return UNZ_OK if there is no problem + return UNZ_END_OF_LIST_OF_FILE if the actual file was the latest. +*/ +extern int ZEXPORT unzGoToNextFile (unzFile file) +{ + unz64_s* s; + int err; + + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + if (!s->current_file_ok) + return UNZ_END_OF_LIST_OF_FILE; + if (s->gi.number_entry != 0xffff) /* 2^16 files overflow hack */ + if (s->num_file+1==s->gi.number_entry) + return UNZ_END_OF_LIST_OF_FILE; + + s->pos_in_central_dir += SIZECENTRALDIRITEM + s->cur_file_info.size_filename + + s->cur_file_info.size_file_extra + s->cur_file_info.size_file_comment ; + s->num_file++; + err = unz64local_GetCurrentFileInfoInternal(file,&s->cur_file_info, + &s->cur_file_info_internal, + NULL,0,NULL,0,NULL,0); + s->current_file_ok = (err == UNZ_OK); + return err; +} + + +/* + Try locate the file szFileName in the zipfile. + For the iCaseSensitivity signification, see unzStringFileNameCompare + + return value : + UNZ_OK if the file is found. It becomes the current file. + UNZ_END_OF_LIST_OF_FILE if the file is not found +*/ +extern int ZEXPORT unzLocateFile (unzFile file, const char *szFileName, int iCaseSensitivity) +{ + unz64_s* s; + int err; + + /* We remember the 'current' position in the file so that we can jump + * back there if we fail. + */ + unz_file_info64 cur_file_infoSaved; + unz_file_info64_internal cur_file_info_internalSaved; + ZPOS64_T num_fileSaved; + ZPOS64_T pos_in_central_dirSaved; + + + if (file==NULL) + return UNZ_PARAMERROR; + + if (strlen(szFileName)>=UNZ_MAXFILENAMEINZIP) + return UNZ_PARAMERROR; + + s=(unz64_s*)file; + if (!s->current_file_ok) + return UNZ_END_OF_LIST_OF_FILE; + + /* Save the current state */ + num_fileSaved = s->num_file; + pos_in_central_dirSaved = s->pos_in_central_dir; + cur_file_infoSaved = s->cur_file_info; + cur_file_info_internalSaved = s->cur_file_info_internal; + + err = unzGoToFirstFile(file); + + while (err == UNZ_OK) + { + char szCurrentFileName[UNZ_MAXFILENAMEINZIP+1]; + err = unzGetCurrentFileInfo64(file,NULL, + szCurrentFileName,sizeof(szCurrentFileName)-1, + NULL,0,NULL,0); + if (err == UNZ_OK) + { + if (unzStringFileNameCompare(szCurrentFileName, + szFileName,iCaseSensitivity)==0) + return UNZ_OK; + err = unzGoToNextFile(file); + } + } + + /* We failed, so restore the state of the 'current file' to where we + * were. + */ + s->num_file = num_fileSaved ; + s->pos_in_central_dir = pos_in_central_dirSaved ; + s->cur_file_info = cur_file_infoSaved; + s->cur_file_info_internal = cur_file_info_internalSaved; + return err; +} + + +/* +/////////////////////////////////////////// +// Contributed by Ryan Haksi (mailto://cryogen@infoserve.net) +// I need random access +// +// Further optimization could be realized by adding an ability +// to cache the directory in memory. The goal being a single +// comprehensive file read to put the file I need in a memory. +*/ + +/* +typedef struct unz_file_pos_s +{ + ZPOS64_T pos_in_zip_directory; // offset in file + ZPOS64_T num_of_file; // # of file +} unz_file_pos; +*/ + +extern int ZEXPORT unzGetFilePos64(unzFile file, unz64_file_pos* file_pos) +{ + unz64_s* s; + + if (file==NULL || file_pos==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + if (!s->current_file_ok) + return UNZ_END_OF_LIST_OF_FILE; + + file_pos->pos_in_zip_directory = s->pos_in_central_dir; + file_pos->num_of_file = s->num_file; + + return UNZ_OK; +} + +extern int ZEXPORT unzGetFilePos( + unzFile file, + unz_file_pos* file_pos) +{ + unz64_file_pos file_pos64; + int err = unzGetFilePos64(file,&file_pos64); + if (err==UNZ_OK) + { + file_pos->pos_in_zip_directory = (uLong)file_pos64.pos_in_zip_directory; + file_pos->num_of_file = (uLong)file_pos64.num_of_file; + } + return err; +} + +extern int ZEXPORT unzGoToFilePos64(unzFile file, const unz64_file_pos* file_pos) +{ + unz64_s* s; + int err; + + if (file==NULL || file_pos==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + + /* jump to the right spot */ + s->pos_in_central_dir = file_pos->pos_in_zip_directory; + s->num_file = file_pos->num_of_file; + + /* set the current file */ + err = unz64local_GetCurrentFileInfoInternal(file,&s->cur_file_info, + &s->cur_file_info_internal, + NULL,0,NULL,0,NULL,0); + /* return results */ + s->current_file_ok = (err == UNZ_OK); + return err; +} + +extern int ZEXPORT unzGoToFilePos( + unzFile file, + unz_file_pos* file_pos) +{ + unz64_file_pos file_pos64; + if (file_pos == NULL) + return UNZ_PARAMERROR; + + file_pos64.pos_in_zip_directory = file_pos->pos_in_zip_directory; + file_pos64.num_of_file = file_pos->num_of_file; + return unzGoToFilePos64(file,&file_pos64); +} + +/* +// Unzip Helper Functions - should be here? +/////////////////////////////////////////// +*/ + +/* + Read the local header of the current zipfile + Check the coherency of the local header and info in the end of central + directory about this file + store in *piSizeVar the size of extra info in local header + (filename and size of extra field data) +*/ +local int unz64local_CheckCurrentFileCoherencyHeader (unz64_s* s, uInt* piSizeVar, + ZPOS64_T * poffset_local_extrafield, + uInt * psize_local_extrafield) +{ + uLong uMagic,uData,uFlags; + uLong size_filename; + uLong size_extra_field; + int err=UNZ_OK; + + *piSizeVar = 0; + *poffset_local_extrafield = 0; + *psize_local_extrafield = 0; + + if (ZSEEK64(s->z_filefunc, s->filestream,s->cur_file_info_internal.offset_curfile + + s->byte_before_the_zipfile,ZLIB_FILEFUNC_SEEK_SET)!=0) + return UNZ_ERRNO; + + + if (err==UNZ_OK) + { + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uMagic) != UNZ_OK) + err=UNZ_ERRNO; + else if (uMagic!=0x04034b50) + err=UNZ_BADZIPFILE; + } + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) + err=UNZ_ERRNO; +/* + else if ((err==UNZ_OK) && (uData!=s->cur_file_info.wVersion)) + err=UNZ_BADZIPFILE; +*/ + if (unz64local_getShort(&s->z_filefunc, s->filestream,&uFlags) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) + err=UNZ_ERRNO; + else if ((err==UNZ_OK) && (uData!=s->cur_file_info.compression_method)) + err=UNZ_BADZIPFILE; + + if ((err==UNZ_OK) && (s->cur_file_info.compression_method!=0) && +/* #ifdef HAVE_BZIP2 */ + (s->cur_file_info.compression_method!=Z_BZIP2ED) && +/* #endif */ + (s->cur_file_info.compression_method!=Z_DEFLATED)) + err=UNZ_BADZIPFILE; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) /* date/time */ + err=UNZ_ERRNO; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) /* crc */ + err=UNZ_ERRNO; + else if ((err==UNZ_OK) && (uData!=s->cur_file_info.crc) && ((uFlags & 8)==0)) + err=UNZ_BADZIPFILE; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) /* size compr */ + err=UNZ_ERRNO; + else if (uData != 0xFFFFFFFF && (err==UNZ_OK) && (uData!=s->cur_file_info.compressed_size) && ((uFlags & 8)==0)) + err=UNZ_BADZIPFILE; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) /* size uncompr */ + err=UNZ_ERRNO; + else if (uData != 0xFFFFFFFF && (err==UNZ_OK) && (uData!=s->cur_file_info.uncompressed_size) && ((uFlags & 8)==0)) + err=UNZ_BADZIPFILE; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&size_filename) != UNZ_OK) + err=UNZ_ERRNO; + else if ((err==UNZ_OK) && (size_filename!=s->cur_file_info.size_filename)) + err=UNZ_BADZIPFILE; + + *piSizeVar += (uInt)size_filename; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&size_extra_field) != UNZ_OK) + err=UNZ_ERRNO; + *poffset_local_extrafield= s->cur_file_info_internal.offset_curfile + + SIZEZIPLOCALHEADER + size_filename; + *psize_local_extrafield = (uInt)size_extra_field; + + *piSizeVar += (uInt)size_extra_field; + + return err; +} + +/* + Open for reading data the current file in the zipfile. + If there is no error and the file is opened, the return value is UNZ_OK. +*/ +extern int ZEXPORT unzOpenCurrentFile3 (unzFile file, int* method, + int* level, int raw, const char* password) +{ + int err=UNZ_OK; + uInt iSizeVar; + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + ZPOS64_T offset_local_extrafield; /* offset of the local extra field */ + uInt size_local_extrafield; /* size of the local extra field */ +# ifndef NOUNCRYPT + char source[12]; +# else + if (password != NULL) + return UNZ_PARAMERROR; +# endif + + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + if (!s->current_file_ok) + return UNZ_PARAMERROR; + + if (s->pfile_in_zip_read != NULL) + unzCloseCurrentFile(file); + + if (unz64local_CheckCurrentFileCoherencyHeader(s,&iSizeVar, &offset_local_extrafield,&size_local_extrafield)!=UNZ_OK) + return UNZ_BADZIPFILE; + + pfile_in_zip_read_info = (file_in_zip64_read_info_s*)ALLOC(sizeof(file_in_zip64_read_info_s)); + if (pfile_in_zip_read_info==NULL) + return UNZ_INTERNALERROR; + + pfile_in_zip_read_info->read_buffer=(char*)ALLOC(UNZ_BUFSIZE); + pfile_in_zip_read_info->offset_local_extrafield = offset_local_extrafield; + pfile_in_zip_read_info->size_local_extrafield = size_local_extrafield; + pfile_in_zip_read_info->pos_local_extrafield=0; + pfile_in_zip_read_info->raw=raw; + + if (pfile_in_zip_read_info->read_buffer==NULL) + { + TRYFREE(pfile_in_zip_read_info); + return UNZ_INTERNALERROR; + } + + pfile_in_zip_read_info->stream_initialised=0; + + if (method!=NULL) + *method = (int)s->cur_file_info.compression_method; + + if (level!=NULL) + { + *level = 6; + switch (s->cur_file_info.flag & 0x06) + { + case 6 : *level = 1; break; + case 4 : *level = 2; break; + case 2 : *level = 9; break; + } + } + + if ((s->cur_file_info.compression_method!=0) && +/* #ifdef HAVE_BZIP2 */ + (s->cur_file_info.compression_method!=Z_BZIP2ED) && +/* #endif */ + (s->cur_file_info.compression_method!=Z_DEFLATED)) + + err=UNZ_BADZIPFILE; + + pfile_in_zip_read_info->crc32_wait=s->cur_file_info.crc; + pfile_in_zip_read_info->crc32=0; + pfile_in_zip_read_info->total_out_64=0; + pfile_in_zip_read_info->compression_method = s->cur_file_info.compression_method; + pfile_in_zip_read_info->filestream=s->filestream; + pfile_in_zip_read_info->z_filefunc=s->z_filefunc; + pfile_in_zip_read_info->byte_before_the_zipfile=s->byte_before_the_zipfile; + + pfile_in_zip_read_info->stream.total_out = 0; + + if ((s->cur_file_info.compression_method==Z_BZIP2ED) && (!raw)) + { +#ifdef HAVE_BZIP2 + pfile_in_zip_read_info->bstream.bzalloc = (void *(*) (void *, int, int))0; + pfile_in_zip_read_info->bstream.bzfree = (free_func)0; + pfile_in_zip_read_info->bstream.opaque = (voidpf)0; + pfile_in_zip_read_info->bstream.state = (voidpf)0; + + pfile_in_zip_read_info->stream.zalloc = (alloc_func)0; + pfile_in_zip_read_info->stream.zfree = (free_func)0; + pfile_in_zip_read_info->stream.opaque = (voidpf)0; + pfile_in_zip_read_info->stream.next_in = (voidpf)0; + pfile_in_zip_read_info->stream.avail_in = 0; + + err=BZ2_bzDecompressInit(&pfile_in_zip_read_info->bstream, 0, 0); + if (err == Z_OK) + pfile_in_zip_read_info->stream_initialised=Z_BZIP2ED; + else + { + TRYFREE(pfile_in_zip_read_info); + return err; + } +#else + pfile_in_zip_read_info->raw=1; +#endif + } + else if ((s->cur_file_info.compression_method==Z_DEFLATED) && (!raw)) + { + pfile_in_zip_read_info->stream.zalloc = (alloc_func)0; + pfile_in_zip_read_info->stream.zfree = (free_func)0; + pfile_in_zip_read_info->stream.opaque = (voidpf)0; + pfile_in_zip_read_info->stream.next_in = 0; + pfile_in_zip_read_info->stream.avail_in = 0; + + err=inflateInit2(&pfile_in_zip_read_info->stream, -MAX_WBITS); + if (err == Z_OK) + pfile_in_zip_read_info->stream_initialised=Z_DEFLATED; + else + { + TRYFREE(pfile_in_zip_read_info); + return err; + } + /* windowBits is passed < 0 to tell that there is no zlib header. + * Note that in this case inflate *requires* an extra "dummy" byte + * after the compressed stream in order to complete decompression and + * return Z_STREAM_END. + * In unzip, i don't wait absolutely Z_STREAM_END because I known the + * size of both compressed and uncompressed data + */ + } + pfile_in_zip_read_info->rest_read_compressed = + s->cur_file_info.compressed_size ; + pfile_in_zip_read_info->rest_read_uncompressed = + s->cur_file_info.uncompressed_size ; + + + pfile_in_zip_read_info->pos_in_zipfile = + s->cur_file_info_internal.offset_curfile + SIZEZIPLOCALHEADER + + iSizeVar; + + pfile_in_zip_read_info->stream.avail_in = (uInt)0; + + s->pfile_in_zip_read = pfile_in_zip_read_info; + s->encrypted = 0; + +# ifndef NOUNCRYPT + if (password != NULL) + { + int i; + s->pcrc_32_tab = get_crc_table(); + init_keys(password,s->keys,s->pcrc_32_tab); + if (ZSEEK64(s->z_filefunc, s->filestream, + s->pfile_in_zip_read->pos_in_zipfile + + s->pfile_in_zip_read->byte_before_the_zipfile, + SEEK_SET)!=0) + return UNZ_INTERNALERROR; + if(ZREAD64(s->z_filefunc, s->filestream,source, 12)<12) + return UNZ_INTERNALERROR; + + for (i = 0; i<12; i++) + zdecode(s->keys,s->pcrc_32_tab,source[i]); + + s->pfile_in_zip_read->pos_in_zipfile+=12; + s->encrypted=1; + } +# endif + + + return UNZ_OK; +} + +extern int ZEXPORT unzOpenCurrentFile (unzFile file) +{ + return unzOpenCurrentFile3(file, NULL, NULL, 0, NULL); +} + +extern int ZEXPORT unzOpenCurrentFilePassword (unzFile file, const char* password) +{ + return unzOpenCurrentFile3(file, NULL, NULL, 0, password); +} + +extern int ZEXPORT unzOpenCurrentFile2 (unzFile file, int* method, int* level, int raw) +{ + return unzOpenCurrentFile3(file, method, level, raw, NULL); +} + +/** Addition for GDAL : START */ + +extern ZPOS64_T ZEXPORT unzGetCurrentFileZStreamPos64( unzFile file) +{ + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + s=(unz64_s*)file; + if (file==NULL) + return 0; //UNZ_PARAMERROR; + pfile_in_zip_read_info=s->pfile_in_zip_read; + if (pfile_in_zip_read_info==NULL) + return 0; //UNZ_PARAMERROR; + return pfile_in_zip_read_info->pos_in_zipfile + + pfile_in_zip_read_info->byte_before_the_zipfile; +} + +/** Addition for GDAL : END */ + +/* + Read bytes from the current file. + buf contain buffer where data must be copied + len the size of buf. + + return the number of byte copied if somes bytes are copied + return 0 if the end of file was reached + return <0 with error code if there is an error + (UNZ_ERRNO for IO error, or zLib error for uncompress error) +*/ +extern int ZEXPORT unzReadCurrentFile (unzFile file, voidp buf, unsigned len) +{ + int err=UNZ_OK; + uInt iRead = 0; + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + pfile_in_zip_read_info=s->pfile_in_zip_read; + + if (pfile_in_zip_read_info==NULL) + return UNZ_PARAMERROR; + + + if (pfile_in_zip_read_info->read_buffer == NULL) + return UNZ_END_OF_LIST_OF_FILE; + if (len==0) + return 0; + + pfile_in_zip_read_info->stream.next_out = (Bytef*)buf; + + pfile_in_zip_read_info->stream.avail_out = (uInt)len; + + if ((len>pfile_in_zip_read_info->rest_read_uncompressed) && + (!(pfile_in_zip_read_info->raw))) + pfile_in_zip_read_info->stream.avail_out = + (uInt)pfile_in_zip_read_info->rest_read_uncompressed; + + if ((len>pfile_in_zip_read_info->rest_read_compressed+ + pfile_in_zip_read_info->stream.avail_in) && + (pfile_in_zip_read_info->raw)) + pfile_in_zip_read_info->stream.avail_out = + (uInt)pfile_in_zip_read_info->rest_read_compressed+ + pfile_in_zip_read_info->stream.avail_in; + + while (pfile_in_zip_read_info->stream.avail_out>0) + { + if ((pfile_in_zip_read_info->stream.avail_in==0) && + (pfile_in_zip_read_info->rest_read_compressed>0)) + { + uInt uReadThis = UNZ_BUFSIZE; + if (pfile_in_zip_read_info->rest_read_compressed<uReadThis) + uReadThis = (uInt)pfile_in_zip_read_info->rest_read_compressed; + if (uReadThis == 0) + return UNZ_EOF; + if (ZSEEK64(pfile_in_zip_read_info->z_filefunc, + pfile_in_zip_read_info->filestream, + pfile_in_zip_read_info->pos_in_zipfile + + pfile_in_zip_read_info->byte_before_the_zipfile, + ZLIB_FILEFUNC_SEEK_SET)!=0) + return UNZ_ERRNO; + if (ZREAD64(pfile_in_zip_read_info->z_filefunc, + pfile_in_zip_read_info->filestream, + pfile_in_zip_read_info->read_buffer, + uReadThis)!=uReadThis) + return UNZ_ERRNO; + + +# ifndef NOUNCRYPT + if(s->encrypted) + { + uInt i; + for(i=0;i<uReadThis;i++) + pfile_in_zip_read_info->read_buffer[i] = + zdecode(s->keys,s->pcrc_32_tab, + pfile_in_zip_read_info->read_buffer[i]); + } +# endif + + + pfile_in_zip_read_info->pos_in_zipfile += uReadThis; + + pfile_in_zip_read_info->rest_read_compressed-=uReadThis; + + pfile_in_zip_read_info->stream.next_in = + (Bytef*)pfile_in_zip_read_info->read_buffer; + pfile_in_zip_read_info->stream.avail_in = (uInt)uReadThis; + } + + if ((pfile_in_zip_read_info->compression_method==0) || (pfile_in_zip_read_info->raw)) + { + uInt uDoCopy,i ; + + if ((pfile_in_zip_read_info->stream.avail_in == 0) && + (pfile_in_zip_read_info->rest_read_compressed == 0)) + return (iRead==0) ? UNZ_EOF : iRead; + + if (pfile_in_zip_read_info->stream.avail_out < + pfile_in_zip_read_info->stream.avail_in) + uDoCopy = pfile_in_zip_read_info->stream.avail_out ; + else + uDoCopy = pfile_in_zip_read_info->stream.avail_in ; + + for (i=0;i<uDoCopy;i++) + *(pfile_in_zip_read_info->stream.next_out+i) = + *(pfile_in_zip_read_info->stream.next_in+i); + + pfile_in_zip_read_info->total_out_64 = pfile_in_zip_read_info->total_out_64 + uDoCopy; + + pfile_in_zip_read_info->crc32 = crc32(pfile_in_zip_read_info->crc32, + pfile_in_zip_read_info->stream.next_out, + uDoCopy); + pfile_in_zip_read_info->rest_read_uncompressed-=uDoCopy; + pfile_in_zip_read_info->stream.avail_in -= uDoCopy; + pfile_in_zip_read_info->stream.avail_out -= uDoCopy; + pfile_in_zip_read_info->stream.next_out += uDoCopy; + pfile_in_zip_read_info->stream.next_in += uDoCopy; + pfile_in_zip_read_info->stream.total_out += uDoCopy; + iRead += uDoCopy; + } + else if (pfile_in_zip_read_info->compression_method==Z_BZIP2ED) + { +#ifdef HAVE_BZIP2 + uLong uTotalOutBefore,uTotalOutAfter; + const Bytef *bufBefore; + uLong uOutThis; + + pfile_in_zip_read_info->bstream.next_in = (char*)pfile_in_zip_read_info->stream.next_in; + pfile_in_zip_read_info->bstream.avail_in = pfile_in_zip_read_info->stream.avail_in; + pfile_in_zip_read_info->bstream.total_in_lo32 = pfile_in_zip_read_info->stream.total_in; + pfile_in_zip_read_info->bstream.total_in_hi32 = 0; + pfile_in_zip_read_info->bstream.next_out = (char*)pfile_in_zip_read_info->stream.next_out; + pfile_in_zip_read_info->bstream.avail_out = pfile_in_zip_read_info->stream.avail_out; + pfile_in_zip_read_info->bstream.total_out_lo32 = pfile_in_zip_read_info->stream.total_out; + pfile_in_zip_read_info->bstream.total_out_hi32 = 0; + + uTotalOutBefore = pfile_in_zip_read_info->bstream.total_out_lo32; + bufBefore = (const Bytef *)pfile_in_zip_read_info->bstream.next_out; + + err=BZ2_bzDecompress(&pfile_in_zip_read_info->bstream); + + uTotalOutAfter = pfile_in_zip_read_info->bstream.total_out_lo32; + uOutThis = uTotalOutAfter-uTotalOutBefore; + + pfile_in_zip_read_info->total_out_64 = pfile_in_zip_read_info->total_out_64 + uOutThis; + + pfile_in_zip_read_info->crc32 = crc32(pfile_in_zip_read_info->crc32,bufBefore, (uInt)(uOutThis)); + pfile_in_zip_read_info->rest_read_uncompressed -= uOutThis; + iRead += (uInt)(uTotalOutAfter - uTotalOutBefore); + + pfile_in_zip_read_info->stream.next_in = (Bytef*)pfile_in_zip_read_info->bstream.next_in; + pfile_in_zip_read_info->stream.avail_in = pfile_in_zip_read_info->bstream.avail_in; + pfile_in_zip_read_info->stream.total_in = pfile_in_zip_read_info->bstream.total_in_lo32; + pfile_in_zip_read_info->stream.next_out = (Bytef*)pfile_in_zip_read_info->bstream.next_out; + pfile_in_zip_read_info->stream.avail_out = pfile_in_zip_read_info->bstream.avail_out; + pfile_in_zip_read_info->stream.total_out = pfile_in_zip_read_info->bstream.total_out_lo32; + + if (err==BZ_STREAM_END) + return (iRead==0) ? UNZ_EOF : iRead; + if (err!=BZ_OK) + break; +#endif + } // end Z_BZIP2ED + else + { + ZPOS64_T uTotalOutBefore,uTotalOutAfter; + const Bytef *bufBefore; + ZPOS64_T uOutThis; + int flush=Z_SYNC_FLUSH; + + uTotalOutBefore = pfile_in_zip_read_info->stream.total_out; + bufBefore = pfile_in_zip_read_info->stream.next_out; + + /* + if ((pfile_in_zip_read_info->rest_read_uncompressed == + pfile_in_zip_read_info->stream.avail_out) && + (pfile_in_zip_read_info->rest_read_compressed == 0)) + flush = Z_FINISH; + */ + err=inflate(&pfile_in_zip_read_info->stream,flush); + + if ((err>=0) && (pfile_in_zip_read_info->stream.msg!=NULL)) + err = Z_DATA_ERROR; + + uTotalOutAfter = pfile_in_zip_read_info->stream.total_out; + uOutThis = uTotalOutAfter-uTotalOutBefore; + + pfile_in_zip_read_info->total_out_64 = pfile_in_zip_read_info->total_out_64 + uOutThis; + + pfile_in_zip_read_info->crc32 = + crc32(pfile_in_zip_read_info->crc32,bufBefore, + (uInt)(uOutThis)); + + pfile_in_zip_read_info->rest_read_uncompressed -= + uOutThis; + + iRead += (uInt)(uTotalOutAfter - uTotalOutBefore); + + if (err==Z_STREAM_END) + return (iRead==0) ? UNZ_EOF : iRead; + if (err!=Z_OK) + break; + } + } + + if (err==Z_OK) + return iRead; + return err; +} + + +/* + Give the current position in uncompressed data +*/ +extern z_off_t ZEXPORT unztell (unzFile file) +{ + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + pfile_in_zip_read_info=s->pfile_in_zip_read; + + if (pfile_in_zip_read_info==NULL) + return UNZ_PARAMERROR; + + return (z_off_t)pfile_in_zip_read_info->stream.total_out; +} + +extern ZPOS64_T ZEXPORT unztell64 (unzFile file) +{ + + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + if (file==NULL) + return (ZPOS64_T)-1; + s=(unz64_s*)file; + pfile_in_zip_read_info=s->pfile_in_zip_read; + + if (pfile_in_zip_read_info==NULL) + return (ZPOS64_T)-1; + + return pfile_in_zip_read_info->total_out_64; +} + + +/* + return 1 if the end of file was reached, 0 elsewhere +*/ +extern int ZEXPORT unzeof (unzFile file) +{ + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + pfile_in_zip_read_info=s->pfile_in_zip_read; + + if (pfile_in_zip_read_info==NULL) + return UNZ_PARAMERROR; + + if (pfile_in_zip_read_info->rest_read_uncompressed == 0) + return 1; + else + return 0; +} + + + +/* +Read extra field from the current file (opened by unzOpenCurrentFile) +This is the local-header version of the extra field (sometimes, there is +more info in the local-header version than in the central-header) + + if buf==NULL, it return the size of the local extra field that can be read + + if buf!=NULL, len is the size of the buffer, the extra header is copied in + buf. + the return value is the number of bytes copied in buf, or (if <0) + the error code +*/ +extern int ZEXPORT unzGetLocalExtrafield (unzFile file, voidp buf, unsigned len) +{ + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + uInt read_now; + ZPOS64_T size_to_read; + + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + pfile_in_zip_read_info=s->pfile_in_zip_read; + + if (pfile_in_zip_read_info==NULL) + return UNZ_PARAMERROR; + + size_to_read = (pfile_in_zip_read_info->size_local_extrafield - + pfile_in_zip_read_info->pos_local_extrafield); + + if (buf==NULL) + return (int)size_to_read; + + if (len>size_to_read) + read_now = (uInt)size_to_read; + else + read_now = (uInt)len ; + + if (read_now==0) + return 0; + + if (ZSEEK64(pfile_in_zip_read_info->z_filefunc, + pfile_in_zip_read_info->filestream, + pfile_in_zip_read_info->offset_local_extrafield + + pfile_in_zip_read_info->pos_local_extrafield, + ZLIB_FILEFUNC_SEEK_SET)!=0) + return UNZ_ERRNO; + + if (ZREAD64(pfile_in_zip_read_info->z_filefunc, + pfile_in_zip_read_info->filestream, + buf,read_now)!=read_now) + return UNZ_ERRNO; + + return (int)read_now; +} + +/* + Close the file in zip opened with unzOpenCurrentFile + Return UNZ_CRCERROR if all the file was read but the CRC is not good +*/ +extern int ZEXPORT unzCloseCurrentFile (unzFile file) +{ + int err=UNZ_OK; + + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + pfile_in_zip_read_info=s->pfile_in_zip_read; + + if (pfile_in_zip_read_info==NULL) + return UNZ_PARAMERROR; + + + if ((pfile_in_zip_read_info->rest_read_uncompressed == 0) && + (!pfile_in_zip_read_info->raw)) + { + if (pfile_in_zip_read_info->crc32 != pfile_in_zip_read_info->crc32_wait) + err=UNZ_CRCERROR; + } + + + TRYFREE(pfile_in_zip_read_info->read_buffer); + pfile_in_zip_read_info->read_buffer = NULL; + if (pfile_in_zip_read_info->stream_initialised == Z_DEFLATED) + inflateEnd(&pfile_in_zip_read_info->stream); +#ifdef HAVE_BZIP2 + else if (pfile_in_zip_read_info->stream_initialised == Z_BZIP2ED) + BZ2_bzDecompressEnd(&pfile_in_zip_read_info->bstream); +#endif + + + pfile_in_zip_read_info->stream_initialised = 0; + TRYFREE(pfile_in_zip_read_info); + + s->pfile_in_zip_read=NULL; + + return err; +} + + +/* + Get the global comment string of the ZipFile, in the szComment buffer. + uSizeBuf is the size of the szComment buffer. + return the number of byte copied or an error code <0 +*/ +extern int ZEXPORT unzGetGlobalComment (unzFile file, char * szComment, uLong uSizeBuf) +{ + unz64_s* s; + uLong uReadThis ; + if (file==NULL) + return (int)UNZ_PARAMERROR; + s=(unz64_s*)file; + + uReadThis = uSizeBuf; + if (uReadThis>s->gi.size_comment) + uReadThis = s->gi.size_comment; + + if (ZSEEK64(s->z_filefunc,s->filestream,s->central_pos+22,ZLIB_FILEFUNC_SEEK_SET)!=0) + return UNZ_ERRNO; + + if (uReadThis>0) + { + *szComment='\0'; + if (ZREAD64(s->z_filefunc,s->filestream,szComment,uReadThis)!=uReadThis) + return UNZ_ERRNO; + } + + if ((szComment != NULL) && (uSizeBuf > s->gi.size_comment)) + *(szComment+s->gi.size_comment)='\0'; + return (int)uReadThis; +} + +/* Additions by RX '2004 */ +extern ZPOS64_T ZEXPORT unzGetOffset64(unzFile file) +{ + unz64_s* s; + + if (file==NULL) + return 0; //UNZ_PARAMERROR; + s=(unz64_s*)file; + if (!s->current_file_ok) + return 0; + if (s->gi.number_entry != 0 && s->gi.number_entry != 0xffff) + if (s->num_file==s->gi.number_entry) + return 0; + return s->pos_in_central_dir; +} + +extern uLong ZEXPORT unzGetOffset (unzFile file) +{ + ZPOS64_T offset64; + + if (file==NULL) + return 0; //UNZ_PARAMERROR; + offset64 = unzGetOffset64(file); + return (uLong)offset64; +} + +extern int ZEXPORT unzSetOffset64(unzFile file, ZPOS64_T pos) +{ + unz64_s* s; + int err; + + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + + s->pos_in_central_dir = pos; + s->num_file = s->gi.number_entry; /* hack */ + err = unz64local_GetCurrentFileInfoInternal(file,&s->cur_file_info, + &s->cur_file_info_internal, + NULL,0,NULL,0,NULL,0); + s->current_file_ok = (err == UNZ_OK); + return err; +} + +extern int ZEXPORT unzSetOffset (unzFile file, uLong pos) +{ + return unzSetOffset64(file,pos); +} diff --git a/storage/connect/unzip.h b/storage/connect/unzip.h new file mode 100644 index 00000000000..2104e391507 --- /dev/null +++ b/storage/connect/unzip.h @@ -0,0 +1,437 @@ +/* unzip.h -- IO for uncompress .zip files using zlib + Version 1.1, February 14h, 2010 + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications of Unzip for Zip64 + Copyright (C) 2007-2008 Even Rouault + + Modifications for Zip64 support on both zip and unzip + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + + --------------------------------------------------------------------------------- + + Condition of use and distribution are the same than zlib : + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + --------------------------------------------------------------------------------- + + Changes + + See header of unzip64.c + +*/ + +#ifndef _unz64_H +#define _unz64_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef _ZLIB_H +#include "zlib.h" +#endif + +#ifndef _ZLIBIOAPI_H +#include "ioapi.h" +#endif + +#ifdef HAVE_BZIP2 +#include "bzlib.h" +#endif + +#define Z_BZIP2ED 12 + +#if defined(STRICTUNZIP) || defined(STRICTZIPUNZIP) +/* like the STRICT of WIN32, we define a pointer that cannot be converted + from (void*) without cast */ +typedef struct TagunzFile__ { int unused; } unzFile__; +typedef unzFile__ *unzFile; +#else +typedef voidp unzFile; +#endif + + +#define UNZ_OK (0) +#define UNZ_END_OF_LIST_OF_FILE (-100) +#define UNZ_ERRNO (Z_ERRNO) +#define UNZ_EOF (0) +#define UNZ_PARAMERROR (-102) +#define UNZ_BADZIPFILE (-103) +#define UNZ_INTERNALERROR (-104) +#define UNZ_CRCERROR (-105) + +/* tm_unz contain date/time info */ +typedef struct tm_unz_s +{ + uInt tm_sec; /* seconds after the minute - [0,59] */ + uInt tm_min; /* minutes after the hour - [0,59] */ + uInt tm_hour; /* hours since midnight - [0,23] */ + uInt tm_mday; /* day of the month - [1,31] */ + uInt tm_mon; /* months since January - [0,11] */ + uInt tm_year; /* years - [1980..2044] */ +} tm_unz; + +/* unz_global_info structure contain global data about the ZIPfile + These data comes from the end of central dir */ +typedef struct unz_global_info64_s +{ + ZPOS64_T number_entry; /* total number of entries in + the central dir on this disk */ + uLong size_comment; /* size of the global comment of the zipfile */ +} unz_global_info64; + +typedef struct unz_global_info_s +{ + uLong number_entry; /* total number of entries in + the central dir on this disk */ + uLong size_comment; /* size of the global comment of the zipfile */ +} unz_global_info; + +/* unz_file_info contain information about a file in the zipfile */ +typedef struct unz_file_info64_s +{ + uLong version; /* version made by 2 bytes */ + uLong version_needed; /* version needed to extract 2 bytes */ + uLong flag; /* general purpose bit flag 2 bytes */ + uLong compression_method; /* compression method 2 bytes */ + uLong dosDate; /* last mod file date in Dos fmt 4 bytes */ + uLong crc; /* crc-32 4 bytes */ + ZPOS64_T compressed_size; /* compressed size 8 bytes */ + ZPOS64_T uncompressed_size; /* uncompressed size 8 bytes */ + uLong size_filename; /* filename length 2 bytes */ + uLong size_file_extra; /* extra field length 2 bytes */ + uLong size_file_comment; /* file comment length 2 bytes */ + + uLong disk_num_start; /* disk number start 2 bytes */ + uLong internal_fa; /* internal file attributes 2 bytes */ + uLong external_fa; /* external file attributes 4 bytes */ + + tm_unz tmu_date; +} unz_file_info64; + +typedef struct unz_file_info_s +{ + uLong version; /* version made by 2 bytes */ + uLong version_needed; /* version needed to extract 2 bytes */ + uLong flag; /* general purpose bit flag 2 bytes */ + uLong compression_method; /* compression method 2 bytes */ + uLong dosDate; /* last mod file date in Dos fmt 4 bytes */ + uLong crc; /* crc-32 4 bytes */ + uLong compressed_size; /* compressed size 4 bytes */ + uLong uncompressed_size; /* uncompressed size 4 bytes */ + uLong size_filename; /* filename length 2 bytes */ + uLong size_file_extra; /* extra field length 2 bytes */ + uLong size_file_comment; /* file comment length 2 bytes */ + + uLong disk_num_start; /* disk number start 2 bytes */ + uLong internal_fa; /* internal file attributes 2 bytes */ + uLong external_fa; /* external file attributes 4 bytes */ + + tm_unz tmu_date; +} unz_file_info; + +extern int ZEXPORT unzStringFileNameCompare OF ((const char* fileName1, + const char* fileName2, + int iCaseSensitivity)); +/* + Compare two filename (fileName1,fileName2). + If iCaseSenisivity = 1, comparision is case sensitivity (like strcmp) + If iCaseSenisivity = 2, comparision is not case sensitivity (like strcmpi + or strcasecmp) + If iCaseSenisivity = 0, case sensitivity is defaut of your operating system + (like 1 on Unix, 2 on Windows) +*/ + + +extern unzFile ZEXPORT unzOpen OF((const char *path)); +extern unzFile ZEXPORT unzOpen64 OF((const void *path)); +/* + Open a Zip file. path contain the full pathname (by example, + on a Windows XP computer "c:\\zlib\\zlib113.zip" or on an Unix computer + "zlib/zlib113.zip". + If the zipfile cannot be opened (file don't exist or in not valid), the + return value is NULL. + Else, the return value is a unzFile Handle, usable with other function + of this unzip package. + the "64" function take a const void* pointer, because the path is just the + value passed to the open64_file_func callback. + Under Windows, if UNICODE is defined, using fill_fopen64_filefunc, the path + is a pointer to a wide unicode string (LPCTSTR is LPCWSTR), so const char* + does not describe the reality +*/ + + +extern unzFile ZEXPORT unzOpen2 OF((const char *path, + zlib_filefunc_def* pzlib_filefunc_def)); +/* + Open a Zip file, like unzOpen, but provide a set of file low level API + for read/write the zip file (see ioapi.h) +*/ + +extern unzFile ZEXPORT unzOpen2_64 OF((const void *path, + zlib_filefunc64_def* pzlib_filefunc_def)); +/* + Open a Zip file, like unz64Open, but provide a set of file low level API + for read/write the zip file (see ioapi.h) +*/ + +extern int ZEXPORT unzClose OF((unzFile file)); +/* + Close a ZipFile opened with unzOpen. + If there is files inside the .Zip opened with unzOpenCurrentFile (see later), + these files MUST be closed with unzCloseCurrentFile before call unzClose. + return UNZ_OK if there is no problem. */ + +extern int ZEXPORT unzGetGlobalInfo OF((unzFile file, + unz_global_info *pglobal_info)); + +extern int ZEXPORT unzGetGlobalInfo64 OF((unzFile file, + unz_global_info64 *pglobal_info)); +/* + Write info about the ZipFile in the *pglobal_info structure. + No preparation of the structure is needed + return UNZ_OK if there is no problem. */ + + +extern int ZEXPORT unzGetGlobalComment OF((unzFile file, + char *szComment, + uLong uSizeBuf)); +/* + Get the global comment string of the ZipFile, in the szComment buffer. + uSizeBuf is the size of the szComment buffer. + return the number of byte copied or an error code <0 +*/ + + +/***************************************************************************/ +/* Unzip package allow you browse the directory of the zipfile */ + +extern int ZEXPORT unzGoToFirstFile OF((unzFile file)); +/* + Set the current file of the zipfile to the first file. + return UNZ_OK if there is no problem +*/ + +extern int ZEXPORT unzGoToNextFile OF((unzFile file)); +/* + Set the current file of the zipfile to the next file. + return UNZ_OK if there is no problem + return UNZ_END_OF_LIST_OF_FILE if the actual file was the latest. +*/ + +extern int ZEXPORT unzLocateFile OF((unzFile file, + const char *szFileName, + int iCaseSensitivity)); +/* + Try locate the file szFileName in the zipfile. + For the iCaseSensitivity signification, see unzStringFileNameCompare + + return value : + UNZ_OK if the file is found. It becomes the current file. + UNZ_END_OF_LIST_OF_FILE if the file is not found +*/ + + +/* ****************************************** */ +/* Ryan supplied functions */ +/* unz_file_info contain information about a file in the zipfile */ +typedef struct unz_file_pos_s +{ + uLong pos_in_zip_directory; /* offset in zip file directory */ + uLong num_of_file; /* # of file */ +} unz_file_pos; + +extern int ZEXPORT unzGetFilePos( + unzFile file, + unz_file_pos* file_pos); + +extern int ZEXPORT unzGoToFilePos( + unzFile file, + unz_file_pos* file_pos); + +typedef struct unz64_file_pos_s +{ + ZPOS64_T pos_in_zip_directory; /* offset in zip file directory */ + ZPOS64_T num_of_file; /* # of file */ +} unz64_file_pos; + +extern int ZEXPORT unzGetFilePos64( + unzFile file, + unz64_file_pos* file_pos); + +extern int ZEXPORT unzGoToFilePos64( + unzFile file, + const unz64_file_pos* file_pos); + +/* ****************************************** */ + +extern int ZEXPORT unzGetCurrentFileInfo64 OF((unzFile file, + unz_file_info64 *pfile_info, + char *szFileName, + uLong fileNameBufferSize, + void *extraField, + uLong extraFieldBufferSize, + char *szComment, + uLong commentBufferSize)); + +extern int ZEXPORT unzGetCurrentFileInfo OF((unzFile file, + unz_file_info *pfile_info, + char *szFileName, + uLong fileNameBufferSize, + void *extraField, + uLong extraFieldBufferSize, + char *szComment, + uLong commentBufferSize)); +/* + Get Info about the current file + if pfile_info!=NULL, the *pfile_info structure will contain somes info about + the current file + if szFileName!=NULL, the filemane string will be copied in szFileName + (fileNameBufferSize is the size of the buffer) + if extraField!=NULL, the extra field information will be copied in extraField + (extraFieldBufferSize is the size of the buffer). + This is the Central-header version of the extra field + if szComment!=NULL, the comment string of the file will be copied in szComment + (commentBufferSize is the size of the buffer) +*/ + + +/** Addition for GDAL : START */ + +extern ZPOS64_T ZEXPORT unzGetCurrentFileZStreamPos64 OF((unzFile file)); + +/** Addition for GDAL : END */ + + +/***************************************************************************/ +/* for reading the content of the current zipfile, you can open it, read data + from it, and close it (you can close it before reading all the file) + */ + +extern int ZEXPORT unzOpenCurrentFile OF((unzFile file)); +/* + Open for reading data the current file in the zipfile. + If there is no error, the return value is UNZ_OK. +*/ + +extern int ZEXPORT unzOpenCurrentFilePassword OF((unzFile file, + const char* password)); +/* + Open for reading data the current file in the zipfile. + password is a crypting password + If there is no error, the return value is UNZ_OK. +*/ + +extern int ZEXPORT unzOpenCurrentFile2 OF((unzFile file, + int* method, + int* level, + int raw)); +/* + Same than unzOpenCurrentFile, but open for read raw the file (not uncompress) + if raw==1 + *method will receive method of compression, *level will receive level of + compression + note : you can set level parameter as NULL (if you did not want known level, + but you CANNOT set method parameter as NULL +*/ + +extern int ZEXPORT unzOpenCurrentFile3 OF((unzFile file, + int* method, + int* level, + int raw, + const char* password)); +/* + Same than unzOpenCurrentFile, but open for read raw the file (not uncompress) + if raw==1 + *method will receive method of compression, *level will receive level of + compression + note : you can set level parameter as NULL (if you did not want known level, + but you CANNOT set method parameter as NULL +*/ + + +extern int ZEXPORT unzCloseCurrentFile OF((unzFile file)); +/* + Close the file in zip opened with unzOpenCurrentFile + Return UNZ_CRCERROR if all the file was read but the CRC is not good +*/ + +extern int ZEXPORT unzReadCurrentFile OF((unzFile file, + voidp buf, + unsigned len)); +/* + Read bytes from the current file (opened by unzOpenCurrentFile) + buf contain buffer where data must be copied + len the size of buf. + + return the number of byte copied if somes bytes are copied + return 0 if the end of file was reached + return <0 with error code if there is an error + (UNZ_ERRNO for IO error, or zLib error for uncompress error) +*/ + +extern z_off_t ZEXPORT unztell OF((unzFile file)); + +extern ZPOS64_T ZEXPORT unztell64 OF((unzFile file)); +/* + Give the current position in uncompressed data +*/ + +extern int ZEXPORT unzeof OF((unzFile file)); +/* + return 1 if the end of file was reached, 0 elsewhere +*/ + +extern int ZEXPORT unzGetLocalExtrafield OF((unzFile file, + voidp buf, + unsigned len)); +/* + Read extra field from the current file (opened by unzOpenCurrentFile) + This is the local-header version of the extra field (sometimes, there is + more info in the local-header version than in the central-header) + + if buf==NULL, it return the size of the local extra field + + if buf!=NULL, len is the size of the buffer, the extra header is copied in + buf. + the return value is the number of bytes copied in buf, or (if <0) + the error code +*/ + +/***************************************************************************/ + +/* Get the current file offset */ +extern ZPOS64_T ZEXPORT unzGetOffset64 (unzFile file); +extern uLong ZEXPORT unzGetOffset (unzFile file); + +/* Set the current file offset */ +extern int ZEXPORT unzSetOffset64 (unzFile file, ZPOS64_T pos); +extern int ZEXPORT unzSetOffset (unzFile file, uLong pos); + + + +#ifdef __cplusplus +} +#endif + +#endif /* _unz64_H */ diff --git a/storage/connect/value.cpp b/storage/connect/value.cpp index 64d0e13e8c4..ced690e77c0 100644 --- a/storage/connect/value.cpp +++ b/storage/connect/value.cpp @@ -1,7 +1,7 @@ /************* Value C++ Functions Source Code File (.CPP) *************/ -/* Name: VALUE.CPP Version 2.5 */ +/* Name: VALUE.CPP Version 2.6 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 2001-2015 */ +/* (C) Copyright to the author Olivier BERTRAND 2001-2016 */ /* */ /* This file contains the VALUE and derived classes family functions. */ /* These classes contain values of different types. They are used so */ @@ -792,19 +792,29 @@ uchar TYPVAL<uchar>::GetTypedValue(PVBLK blk, int n) /***********************************************************************/ /* TYPVAL SetBinValue: with bytes extracted from a line. */ +/* Currently only used reading column of binary files. */ /***********************************************************************/ template <class TYPE> void TYPVAL<TYPE>::SetBinValue(void *p) - { - Tval = *(TYPE *)p; - Null = false; - } // end of SetBinValue +{ +#if defined(UNALIGNED_OK) + // x86 can cast non-aligned memory directly + Tval = *(TYPE *)p; +#else + // Prevent unaligned memory access on MIPS and ArmHF platforms. + // Make use of memcpy instead of straight pointer dereferencing. + // Currently only used by WriteColumn of binary files. + // From original author: Vicentiu Ciorbaru <vicentiu@mariadb.org> + memcpy(&Tval, p, sizeof(TYPE)); +#endif + Null = false; +} // end of SetBinValue /***********************************************************************/ /* GetBinValue: fill a buffer with the internal binary value. */ /* This function checks whether the buffer length is enough and */ /* returns true if not. Actual filling occurs only if go is true. */ -/* Currently used by WriteColumn of binary files. */ +/* Currently only used writing column of binary files. */ /***********************************************************************/ template <class TYPE> bool TYPVAL<TYPE>::GetBinValue(void *buf, int buflen, bool go) @@ -819,7 +829,16 @@ bool TYPVAL<TYPE>::GetBinValue(void *buf, int buflen, bool go) //#endif if (go) - *(TYPE *)buf = Tval; +#if defined(UNALIGNED_OK) + // x86 can cast non-aligned memory directly + *(TYPE *)buf = Tval; +#else + // Prevent unaligned memory access on MIPS and ArmHF platforms. + // Make use of memcpy instead of straight pointer dereferencing. + // Currently only used by WriteColumn of binary files. + // From original author: Vicentiu Ciorbaru <vicentiu@mariadb.org> + memcpy(buf, &Tval, sizeof(TYPE)); +#endif Null = false; return false; diff --git a/storage/connect/value.h b/storage/connect/value.h index c5a381e89da..a670ade4c28 100644 --- a/storage/connect/value.h +++ b/storage/connect/value.h @@ -1,7 +1,7 @@ /**************** Value H Declares Source Code File (.H) ***************/ -/* Name: VALUE.H Version 2.1 */ +/* Name: VALUE.H Version 2.2 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 2001-2014 */ +/* (C) Copyright to the author Olivier BERTRAND 2001-2016 */ /* */ /* This file contains the VALUE and derived classes declares. */ /***********************************************************************/ @@ -17,6 +17,13 @@ #include "block.h" /***********************************************************************/ +/* This should list the processors accepting unaligned numeral values.*/ +/***********************************************************************/ +#if defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) || defined(_M_X64) || defined(_M_AMD64) || defined(_M_IA64) +#define UNALIGNED_OK +#endif + +/***********************************************************************/ /* Types used in some class definitions. */ /***********************************************************************/ enum CONV {CNV_ANY = 0, /* Convert to any type */ @@ -116,27 +123,46 @@ class DllExport VALUE : public BLOCK { virtual bool Compute(PGLOBAL g, PVAL *vp, int np, OPVAL op); virtual bool FormatValue(PVAL vp, char *fmt) = 0; - /** - Set value from a non-aligned in-memory value in the machine byte order. - TYPE can be either of: - - int, short, longlong - - uint, ushort, ulonglong - - float, double - @param - a pointer to a non-aligned value of type TYPE. - */ - template<typename TYPE> - void SetValueNonAligned(const char *p) - { -#if defined(__i386__) || defined(__x86_64__) - SetValue(*((TYPE*) p)); // x86 can cast non-aligned memory directly + /** + Set value from a non-aligned in-memory value in the machine byte order. + TYPE can be either of: + - int, short, longlong + - uint, ushort, ulonglong + - float, double + @param - a pointer to a non-aligned value of type TYPE. + */ + template<typename TYPE> + void SetValueNonAligned(const char *p) + { +#if defined(UNALIGNED_OK) + SetValue(*((TYPE*)p)); // x86 can cast non-aligned memory directly #else - TYPE tmp; // a slower version for non-x86 platforms - memcpy(&tmp, p, sizeof(tmp)); - SetValue(tmp); + TYPE tmp; // a slower version for non-x86 platforms + memcpy(&tmp, p, sizeof(tmp)); + SetValue(tmp); #endif - } + } // end of SetValueNonAligned + + /** + Get value from a non-aligned in-memory value in the machine byte order. + TYPE can be either of: + - int, short, longlong + - uint, ushort, ulonglong + - float, double + @params - a pointer to a non-aligned value of type TYPE, the TYPE value. + */ + template<typename TYPE> + void GetValueNonAligned(char *p, TYPE n) + { +#if defined(UNALIGNED_OK) + *(TYPE *)p = n; // x86 can cast non-aligned memory directly +#else + TYPE tmp = n; // a slower version for non-x86 platforms + memcpy(p, &tmp, sizeof(tmp)); +#endif + } // end of SetValueNonAligned - protected: +protected: virtual bool SetConstFormat(PGLOBAL, FORMAT&) = 0; const char *GetXfmt(void); diff --git a/storage/connect/xindex.cpp b/storage/connect/xindex.cpp index 56312630278..a2cf4e77b80 100755 --- a/storage/connect/xindex.cpp +++ b/storage/connect/xindex.cpp @@ -45,7 +45,9 @@ //nclude "array.h" #include "filamtxt.h" #include "tabdos.h" +#if defined(VCT_SUPPORT) #include "tabvct.h" +#endif // VCT_SUPPORT /***********************************************************************/ /* Macro or external routine definition */ @@ -293,9 +295,11 @@ bool XINDEX::AddColumns(void) return false; // Not applying to static index else if (IsMul()) return false; // Not done yet for multiple index - else if (Tbxp->GetAmType() == TYPE_AM_VCT && ((PTDBVCT)Tbxp)->IsSplit()) +#if defined(VCT_SUPPORT) + else if (Tbxp->GetAmType() == TYPE_AM_VCT && ((PTDBVCT)Tbxp)->IsSplit()) return false; // This would require to read additional files - else +#endif // VCT_SUPPORT + else return true; } // end of AddColumns diff --git a/storage/connect/zip.c b/storage/connect/zip.c new file mode 100644 index 00000000000..ea54853e858 --- /dev/null +++ b/storage/connect/zip.c @@ -0,0 +1,2007 @@ +/* zip.c -- IO on .zip files using zlib + Version 1.1, February 14h, 2010 + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications for Zip64 support + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + + Changes + Oct-2009 - Mathias Svensson - Remove old C style function prototypes + Oct-2009 - Mathias Svensson - Added Zip64 Support when creating new file archives + Oct-2009 - Mathias Svensson - Did some code cleanup and refactoring to get better overview of some functions. + Oct-2009 - Mathias Svensson - Added zipRemoveExtraInfoBlock to strip extra field data from its ZIP64 data + It is used when recreting zip archive with RAW when deleting items from a zip. + ZIP64 data is automaticly added to items that needs it, and existing ZIP64 data need to be removed. + Oct-2009 - Mathias Svensson - Added support for BZIP2 as compression mode (bzip2 lib is required) + Jan-2010 - back to unzip and minizip 1.0 name scheme, with compatibility layer + +*/ + + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include "zlib.h" +#include "zip.h" + +#ifdef STDC +# include <stddef.h> +# include <string.h> +# include <stdlib.h> +#endif +#ifdef NO_ERRNO_H + extern int errno; +#else +# include <errno.h> +#endif + + +#ifndef local +# define local static +#endif +/* compile with -Dlocal if your debugger can't find static symbols */ + +#ifndef VERSIONMADEBY +# define VERSIONMADEBY (0x0) /* platform depedent */ +#endif + +#ifndef Z_BUFSIZE +#define Z_BUFSIZE (64*1024) //(16384) +#endif + +#ifndef Z_MAXFILENAMEINZIP +#define Z_MAXFILENAMEINZIP (256) +#endif + +#ifndef ALLOC +# define ALLOC(size) (malloc(size)) +#endif +#ifndef TRYFREE +# define TRYFREE(p) {if (p) free(p);} +#endif + +/* +#define SIZECENTRALDIRITEM (0x2e) +#define SIZEZIPLOCALHEADER (0x1e) +*/ + +/* I've found an old Unix (a SunOS 4.1.3_U1) without all SEEK_* defined.... */ + + +// NOT sure that this work on ALL platform +#define MAKEULONG64(a, b) ((ZPOS64_T)(((unsigned long)(a)) | ((ZPOS64_T)((unsigned long)(b))) << 32)) + +#ifndef SEEK_CUR +#define SEEK_CUR 1 +#endif + +#ifndef SEEK_END +#define SEEK_END 2 +#endif + +#ifndef SEEK_SET +#define SEEK_SET 0 +#endif + +#ifndef DEF_MEM_LEVEL +#if MAX_MEM_LEVEL >= 8 +# define DEF_MEM_LEVEL 8 +#else +# define DEF_MEM_LEVEL MAX_MEM_LEVEL +#endif +#endif +const char zip_copyright[] =" zip 1.01 Copyright 1998-2004 Gilles Vollant - http://www.winimage.com/zLibDll"; + + +#define SIZEDATA_INDATABLOCK (4096-(4*4)) + +#define LOCALHEADERMAGIC (0x04034b50) +#define CENTRALHEADERMAGIC (0x02014b50) +#define ENDHEADERMAGIC (0x06054b50) +#define ZIP64ENDHEADERMAGIC (0x6064b50) +#define ZIP64ENDLOCHEADERMAGIC (0x7064b50) + +#define FLAG_LOCALHEADER_OFFSET (0x06) +#define CRC_LOCALHEADER_OFFSET (0x0e) + +#define SIZECENTRALHEADER (0x2e) /* 46 */ + +typedef struct linkedlist_datablock_internal_s +{ + struct linkedlist_datablock_internal_s* next_datablock; + uLong avail_in_this_block; + uLong filled_in_this_block; + uLong unused; /* for future use and alignement */ + unsigned char data[SIZEDATA_INDATABLOCK]; +} linkedlist_datablock_internal; + +typedef struct linkedlist_data_s +{ + linkedlist_datablock_internal* first_block; + linkedlist_datablock_internal* last_block; +} linkedlist_data; + + +typedef struct +{ + z_stream stream; /* zLib stream structure for inflate */ +#ifdef HAVE_BZIP2 + bz_stream bstream; /* bzLib stream structure for bziped */ +#endif + + int stream_initialised; /* 1 is stream is initialised */ + uInt pos_in_buffered_data; /* last written byte in buffered_data */ + + ZPOS64_T pos_local_header; /* offset of the local header of the file + currenty writing */ + char* central_header; /* central header data for the current file */ + uLong size_centralExtra; + uLong size_centralheader; /* size of the central header for cur file */ + uLong size_centralExtraFree; /* Extra bytes allocated to the centralheader but that are not used */ + uLong flag; /* flag of the file currently writing */ + + int method; /* compression method of file currenty wr.*/ + int raw; /* 1 for directly writing raw data */ + Byte buffered_data[Z_BUFSIZE];/* buffer contain compressed data to be writ*/ + uLong dosDate; + uLong crc32; + int encrypt; + int zip64; /* Add ZIP64 extened information in the extra field */ + ZPOS64_T pos_zip64extrainfo; + ZPOS64_T totalCompressedData; + ZPOS64_T totalUncompressedData; +#ifndef NOCRYPT + unsigned long keys[3]; /* keys defining the pseudo-random sequence */ + const z_crc_t* pcrc_32_tab; + int crypt_header_size; +#endif +} curfile64_info; + +typedef struct +{ + zlib_filefunc64_32_def z_filefunc; + voidpf filestream; /* io structore of the zipfile */ + linkedlist_data central_dir;/* datablock with central dir in construction*/ + int in_opened_file_inzip; /* 1 if a file in the zip is currently writ.*/ + curfile64_info ci; /* info on the file curretly writing */ + + ZPOS64_T begin_pos; /* position of the beginning of the zipfile */ + ZPOS64_T add_position_when_writting_offset; + ZPOS64_T number_entry; + +#ifndef NO_ADDFILEINEXISTINGZIP + char *globalcomment; +#endif + +} zip64_internal; + + +#ifndef NOCRYPT +#define INCLUDECRYPTINGCODE_IFCRYPTALLOWED +#include "crypt.h" +#endif + +local linkedlist_datablock_internal* allocate_new_datablock() +{ + linkedlist_datablock_internal* ldi; + ldi = (linkedlist_datablock_internal*) + ALLOC(sizeof(linkedlist_datablock_internal)); + if (ldi!=NULL) + { + ldi->next_datablock = NULL ; + ldi->filled_in_this_block = 0 ; + ldi->avail_in_this_block = SIZEDATA_INDATABLOCK ; + } + return ldi; +} + +local void free_datablock(linkedlist_datablock_internal* ldi) +{ + while (ldi!=NULL) + { + linkedlist_datablock_internal* ldinext = ldi->next_datablock; + TRYFREE(ldi); + ldi = ldinext; + } +} + +local void init_linkedlist(linkedlist_data* ll) +{ + ll->first_block = ll->last_block = NULL; +} + +local void free_linkedlist(linkedlist_data* ll) +{ + free_datablock(ll->first_block); + ll->first_block = ll->last_block = NULL; +} + + +local int add_data_in_datablock(linkedlist_data* ll, const void* buf, uLong len) +{ + linkedlist_datablock_internal* ldi; + const unsigned char* from_copy; + + if (ll==NULL) + return ZIP_INTERNALERROR; + + if (ll->last_block == NULL) + { + ll->first_block = ll->last_block = allocate_new_datablock(); + if (ll->first_block == NULL) + return ZIP_INTERNALERROR; + } + + ldi = ll->last_block; + from_copy = (unsigned char*)buf; + + while (len>0) + { + uInt copy_this; + uInt i; + unsigned char* to_copy; + + if (ldi->avail_in_this_block==0) + { + ldi->next_datablock = allocate_new_datablock(); + if (ldi->next_datablock == NULL) + return ZIP_INTERNALERROR; + ldi = ldi->next_datablock ; + ll->last_block = ldi; + } + + if (ldi->avail_in_this_block < len) + copy_this = (uInt)ldi->avail_in_this_block; + else + copy_this = (uInt)len; + + to_copy = &(ldi->data[ldi->filled_in_this_block]); + + for (i=0;i<copy_this;i++) + *(to_copy+i)=*(from_copy+i); + + ldi->filled_in_this_block += copy_this; + ldi->avail_in_this_block -= copy_this; + from_copy += copy_this ; + len -= copy_this; + } + return ZIP_OK; +} + + + +/****************************************************************************/ + +#ifndef NO_ADDFILEINEXISTINGZIP +/* =========================================================================== + Inputs a long in LSB order to the given file + nbByte == 1, 2 ,4 or 8 (byte, short or long, ZPOS64_T) +*/ + +local int zip64local_putValue OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, ZPOS64_T x, int nbByte)); +local int zip64local_putValue (const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, ZPOS64_T x, int nbByte) +{ + unsigned char buf[8]; + int n; + for (n = 0; n < nbByte; n++) + { + buf[n] = (unsigned char)(x & 0xff); + x >>= 8; + } + if (x != 0) + { /* data overflow - hack for ZIP64 (X Roche) */ + for (n = 0; n < nbByte; n++) + { + buf[n] = 0xff; + } + } + + if (ZWRITE64(*pzlib_filefunc_def,filestream,buf,nbByte)!=(uLong)nbByte) + return ZIP_ERRNO; + else + return ZIP_OK; +} + +local void zip64local_putValue_inmemory OF((void* dest, ZPOS64_T x, int nbByte)); +local void zip64local_putValue_inmemory (void* dest, ZPOS64_T x, int nbByte) +{ + unsigned char* buf=(unsigned char*)dest; + int n; + for (n = 0; n < nbByte; n++) { + buf[n] = (unsigned char)(x & 0xff); + x >>= 8; + } + + if (x != 0) + { /* data overflow - hack for ZIP64 */ + for (n = 0; n < nbByte; n++) + { + buf[n] = 0xff; + } + } +} + +/****************************************************************************/ + + +local uLong zip64local_TmzDateToDosDate(const tm_zip* ptm) +{ + uLong year = (uLong)ptm->tm_year; + if (year>=1980) + year-=1980; + else if (year>=80) + year-=80; + return + (uLong) (((ptm->tm_mday) + (32 * (ptm->tm_mon+1)) + (512 * year)) << 16) | + ((ptm->tm_sec/2) + (32* ptm->tm_min) + (2048 * (uLong)ptm->tm_hour)); +} + + +/****************************************************************************/ + +local int zip64local_getByte OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, int *pi)); + +local int zip64local_getByte(const zlib_filefunc64_32_def* pzlib_filefunc_def,voidpf filestream,int* pi) +{ + unsigned char c; + int err = (int)ZREAD64(*pzlib_filefunc_def,filestream,&c,1); + if (err==1) + { + *pi = (int)c; + return ZIP_OK; + } + else + { + if (ZERROR64(*pzlib_filefunc_def,filestream)) + return ZIP_ERRNO; + else + return ZIP_EOF; + } +} + + +/* =========================================================================== + Reads a long in LSB order from the given gz_stream. Sets +*/ +local int zip64local_getShort OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, uLong *pX)); + +local int zip64local_getShort (const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, uLong* pX) +{ + uLong x ; + int i = 0; + int err; + + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x = (uLong)i; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((uLong)i)<<8; + + if (err==ZIP_OK) + *pX = x; + else + *pX = 0; + return err; +} + +local int zip64local_getLong OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, uLong *pX)); + +local int zip64local_getLong (const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, uLong* pX) +{ + uLong x ; + int i = 0; + int err; + + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x = (uLong)i; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((uLong)i)<<8; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((uLong)i)<<16; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((uLong)i)<<24; + + if (err==ZIP_OK) + *pX = x; + else + *pX = 0; + return err; +} + +local int zip64local_getLong64 OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, ZPOS64_T *pX)); + + +local int zip64local_getLong64 (const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, ZPOS64_T *pX) +{ + ZPOS64_T x; + int i = 0; + int err; + + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x = (ZPOS64_T)i; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<8; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<16; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<24; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<32; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<40; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<48; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<56; + + if (err==ZIP_OK) + *pX = x; + else + *pX = 0; + + return err; +} + +#ifndef BUFREADCOMMENT +#define BUFREADCOMMENT (0x400) +#endif +/* + Locate the Central directory of a zipfile (at the end, just before + the global comment) +*/ +local ZPOS64_T zip64local_SearchCentralDir OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream)); + +local ZPOS64_T zip64local_SearchCentralDir(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream) +{ + unsigned char* buf; + ZPOS64_T uSizeFile; + ZPOS64_T uBackRead; + ZPOS64_T uMaxBack=0xffff; /* maximum size of global comment */ + ZPOS64_T uPosFound=0; + + if (ZSEEK64(*pzlib_filefunc_def,filestream,0,ZLIB_FILEFUNC_SEEK_END) != 0) + return 0; + + + uSizeFile = ZTELL64(*pzlib_filefunc_def,filestream); + + if (uMaxBack>uSizeFile) + uMaxBack = uSizeFile; + + buf = (unsigned char*)ALLOC(BUFREADCOMMENT+4); + if (buf==NULL) + return 0; + + uBackRead = 4; + while (uBackRead<uMaxBack) + { + uLong uReadSize; + ZPOS64_T uReadPos ; + int i; + if (uBackRead+BUFREADCOMMENT>uMaxBack) + uBackRead = uMaxBack; + else + uBackRead+=BUFREADCOMMENT; + uReadPos = uSizeFile-uBackRead ; + + uReadSize = ((BUFREADCOMMENT+4) < (uSizeFile-uReadPos)) ? + (BUFREADCOMMENT+4) : (uLong)(uSizeFile-uReadPos); + if (ZSEEK64(*pzlib_filefunc_def,filestream,uReadPos,ZLIB_FILEFUNC_SEEK_SET)!=0) + break; + + if (ZREAD64(*pzlib_filefunc_def,filestream,buf,uReadSize)!=uReadSize) + break; + + for (i=(int)uReadSize-3; (i--)>0;) + if (((*(buf+i))==0x50) && ((*(buf+i+1))==0x4b) && + ((*(buf+i+2))==0x05) && ((*(buf+i+3))==0x06)) + { + uPosFound = uReadPos+i; + break; + } + + if (uPosFound!=0) + break; + } + TRYFREE(buf); + return uPosFound; +} + +/* +Locate the End of Zip64 Central directory locator and from there find the CD of a zipfile (at the end, just before +the global comment) +*/ +local ZPOS64_T zip64local_SearchCentralDir64 OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream)); + +local ZPOS64_T zip64local_SearchCentralDir64(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream) +{ + unsigned char* buf; + ZPOS64_T uSizeFile; + ZPOS64_T uBackRead; + ZPOS64_T uMaxBack=0xffff; /* maximum size of global comment */ + ZPOS64_T uPosFound=0; + uLong uL; + ZPOS64_T relativeOffset; + + if (ZSEEK64(*pzlib_filefunc_def,filestream,0,ZLIB_FILEFUNC_SEEK_END) != 0) + return 0; + + uSizeFile = ZTELL64(*pzlib_filefunc_def,filestream); + + if (uMaxBack>uSizeFile) + uMaxBack = uSizeFile; + + buf = (unsigned char*)ALLOC(BUFREADCOMMENT+4); + if (buf==NULL) + return 0; + + uBackRead = 4; + while (uBackRead<uMaxBack) + { + uLong uReadSize; + ZPOS64_T uReadPos; + int i; + if (uBackRead+BUFREADCOMMENT>uMaxBack) + uBackRead = uMaxBack; + else + uBackRead+=BUFREADCOMMENT; + uReadPos = uSizeFile-uBackRead ; + + uReadSize = ((BUFREADCOMMENT+4) < (uSizeFile-uReadPos)) ? + (BUFREADCOMMENT+4) : (uLong)(uSizeFile-uReadPos); + if (ZSEEK64(*pzlib_filefunc_def,filestream,uReadPos,ZLIB_FILEFUNC_SEEK_SET)!=0) + break; + + if (ZREAD64(*pzlib_filefunc_def,filestream,buf,uReadSize)!=uReadSize) + break; + + for (i=(int)uReadSize-3; (i--)>0;) + { + // Signature "0x07064b50" Zip64 end of central directory locater + if (((*(buf+i))==0x50) && ((*(buf+i+1))==0x4b) && ((*(buf+i+2))==0x06) && ((*(buf+i+3))==0x07)) + { + uPosFound = uReadPos+i; + break; + } + } + + if (uPosFound!=0) + break; + } + + TRYFREE(buf); + if (uPosFound == 0) + return 0; + + /* Zip64 end of central directory locator */ + if (ZSEEK64(*pzlib_filefunc_def,filestream, uPosFound,ZLIB_FILEFUNC_SEEK_SET)!=0) + return 0; + + /* the signature, already checked */ + if (zip64local_getLong(pzlib_filefunc_def,filestream,&uL)!=ZIP_OK) + return 0; + + /* number of the disk with the start of the zip64 end of central directory */ + if (zip64local_getLong(pzlib_filefunc_def,filestream,&uL)!=ZIP_OK) + return 0; + if (uL != 0) + return 0; + + /* relative offset of the zip64 end of central directory record */ + if (zip64local_getLong64(pzlib_filefunc_def,filestream,&relativeOffset)!=ZIP_OK) + return 0; + + /* total number of disks */ + if (zip64local_getLong(pzlib_filefunc_def,filestream,&uL)!=ZIP_OK) + return 0; + if (uL != 1) + return 0; + + /* Goto Zip64 end of central directory record */ + if (ZSEEK64(*pzlib_filefunc_def,filestream, relativeOffset,ZLIB_FILEFUNC_SEEK_SET)!=0) + return 0; + + /* the signature */ + if (zip64local_getLong(pzlib_filefunc_def,filestream,&uL)!=ZIP_OK) + return 0; + + if (uL != 0x06064b50) // signature of 'Zip64 end of central directory' + return 0; + + return relativeOffset; +} + +int LoadCentralDirectoryRecord(zip64_internal* pziinit) +{ + int err=ZIP_OK; + ZPOS64_T byte_before_the_zipfile;/* byte before the zipfile, (>0 for sfx)*/ + + ZPOS64_T size_central_dir; /* size of the central directory */ + ZPOS64_T offset_central_dir; /* offset of start of central directory */ + ZPOS64_T central_pos; + uLong uL; + + uLong number_disk; /* number of the current dist, used for + spaning ZIP, unsupported, always 0*/ + uLong number_disk_with_CD; /* number the the disk with central dir, used + for spaning ZIP, unsupported, always 0*/ + ZPOS64_T number_entry; + ZPOS64_T number_entry_CD; /* total number of entries in + the central dir + (same than number_entry on nospan) */ + uLong VersionMadeBy; + uLong VersionNeeded; + uLong size_comment; + + int hasZIP64Record = 0; + + // check first if we find a ZIP64 record + central_pos = zip64local_SearchCentralDir64(&pziinit->z_filefunc,pziinit->filestream); + if(central_pos > 0) + { + hasZIP64Record = 1; + } + else if(central_pos == 0) + { + central_pos = zip64local_SearchCentralDir(&pziinit->z_filefunc,pziinit->filestream); + } + +/* disable to allow appending to empty ZIP archive + if (central_pos==0) + err=ZIP_ERRNO; +*/ + + if(hasZIP64Record) + { + ZPOS64_T sizeEndOfCentralDirectory; + if (ZSEEK64(pziinit->z_filefunc, pziinit->filestream, central_pos, ZLIB_FILEFUNC_SEEK_SET) != 0) + err=ZIP_ERRNO; + + /* the signature, already checked */ + if (zip64local_getLong(&pziinit->z_filefunc, pziinit->filestream,&uL)!=ZIP_OK) + err=ZIP_ERRNO; + + /* size of zip64 end of central directory record */ + if (zip64local_getLong64(&pziinit->z_filefunc, pziinit->filestream, &sizeEndOfCentralDirectory)!=ZIP_OK) + err=ZIP_ERRNO; + + /* version made by */ + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream, &VersionMadeBy)!=ZIP_OK) + err=ZIP_ERRNO; + + /* version needed to extract */ + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream, &VersionNeeded)!=ZIP_OK) + err=ZIP_ERRNO; + + /* number of this disk */ + if (zip64local_getLong(&pziinit->z_filefunc, pziinit->filestream,&number_disk)!=ZIP_OK) + err=ZIP_ERRNO; + + /* number of the disk with the start of the central directory */ + if (zip64local_getLong(&pziinit->z_filefunc, pziinit->filestream,&number_disk_with_CD)!=ZIP_OK) + err=ZIP_ERRNO; + + /* total number of entries in the central directory on this disk */ + if (zip64local_getLong64(&pziinit->z_filefunc, pziinit->filestream, &number_entry)!=ZIP_OK) + err=ZIP_ERRNO; + + /* total number of entries in the central directory */ + if (zip64local_getLong64(&pziinit->z_filefunc, pziinit->filestream,&number_entry_CD)!=ZIP_OK) + err=ZIP_ERRNO; + + if ((number_entry_CD!=number_entry) || (number_disk_with_CD!=0) || (number_disk!=0)) + err=ZIP_BADZIPFILE; + + /* size of the central directory */ + if (zip64local_getLong64(&pziinit->z_filefunc, pziinit->filestream,&size_central_dir)!=ZIP_OK) + err=ZIP_ERRNO; + + /* offset of start of central directory with respect to the + starting disk number */ + if (zip64local_getLong64(&pziinit->z_filefunc, pziinit->filestream,&offset_central_dir)!=ZIP_OK) + err=ZIP_ERRNO; + + // TODO.. + // read the comment from the standard central header. + size_comment = 0; + } + else + { + // Read End of central Directory info + if (ZSEEK64(pziinit->z_filefunc, pziinit->filestream, central_pos,ZLIB_FILEFUNC_SEEK_SET)!=0) + err=ZIP_ERRNO; + + /* the signature, already checked */ + if (zip64local_getLong(&pziinit->z_filefunc, pziinit->filestream,&uL)!=ZIP_OK) + err=ZIP_ERRNO; + + /* number of this disk */ + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream,&number_disk)!=ZIP_OK) + err=ZIP_ERRNO; + + /* number of the disk with the start of the central directory */ + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream,&number_disk_with_CD)!=ZIP_OK) + err=ZIP_ERRNO; + + /* total number of entries in the central dir on this disk */ + number_entry = 0; + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream, &uL)!=ZIP_OK) + err=ZIP_ERRNO; + else + number_entry = uL; + + /* total number of entries in the central dir */ + number_entry_CD = 0; + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream, &uL)!=ZIP_OK) + err=ZIP_ERRNO; + else + number_entry_CD = uL; + + if ((number_entry_CD!=number_entry) || (number_disk_with_CD!=0) || (number_disk!=0)) + err=ZIP_BADZIPFILE; + + /* size of the central directory */ + size_central_dir = 0; + if (zip64local_getLong(&pziinit->z_filefunc, pziinit->filestream, &uL)!=ZIP_OK) + err=ZIP_ERRNO; + else + size_central_dir = uL; + + /* offset of start of central directory with respect to the starting disk number */ + offset_central_dir = 0; + if (zip64local_getLong(&pziinit->z_filefunc, pziinit->filestream, &uL)!=ZIP_OK) + err=ZIP_ERRNO; + else + offset_central_dir = uL; + + + /* zipfile global comment length */ + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream, &size_comment)!=ZIP_OK) + err=ZIP_ERRNO; + } + + if ((central_pos<offset_central_dir+size_central_dir) && + (err==ZIP_OK)) + err=ZIP_BADZIPFILE; + + if (err!=ZIP_OK) + { + ZCLOSE64(pziinit->z_filefunc, pziinit->filestream); + return ZIP_ERRNO; + } + + if (size_comment>0) + { + pziinit->globalcomment = (char*)ALLOC(size_comment+1); + if (pziinit->globalcomment) + { + size_comment = ZREAD64(pziinit->z_filefunc, pziinit->filestream, pziinit->globalcomment,size_comment); + pziinit->globalcomment[size_comment]=0; + } + } + + byte_before_the_zipfile = central_pos - (offset_central_dir+size_central_dir); + pziinit->add_position_when_writting_offset = byte_before_the_zipfile; + + { + ZPOS64_T size_central_dir_to_read = size_central_dir; + size_t buf_size = SIZEDATA_INDATABLOCK; + void* buf_read = (void*)ALLOC(buf_size); + if (ZSEEK64(pziinit->z_filefunc, pziinit->filestream, offset_central_dir + byte_before_the_zipfile, ZLIB_FILEFUNC_SEEK_SET) != 0) + err=ZIP_ERRNO; + + while ((size_central_dir_to_read>0) && (err==ZIP_OK)) + { + ZPOS64_T read_this = SIZEDATA_INDATABLOCK; + if (read_this > size_central_dir_to_read) + read_this = size_central_dir_to_read; + + if (ZREAD64(pziinit->z_filefunc, pziinit->filestream,buf_read,(uLong)read_this) != read_this) + err=ZIP_ERRNO; + + if (err==ZIP_OK) + err = add_data_in_datablock(&pziinit->central_dir,buf_read, (uLong)read_this); + + size_central_dir_to_read-=read_this; + } + TRYFREE(buf_read); + } + pziinit->begin_pos = byte_before_the_zipfile; + pziinit->number_entry = number_entry_CD; + + if (ZSEEK64(pziinit->z_filefunc, pziinit->filestream, offset_central_dir+byte_before_the_zipfile,ZLIB_FILEFUNC_SEEK_SET) != 0) + err=ZIP_ERRNO; + + return err; +} + + +#endif /* !NO_ADDFILEINEXISTINGZIP*/ + + +/************************************************************/ +extern zipFile ZEXPORT zipOpen3 (const void *pathname, int append, zipcharpc* globalcomment, zlib_filefunc64_32_def* pzlib_filefunc64_32_def) +{ + zip64_internal ziinit; + zip64_internal* zi; + int err=ZIP_OK; + + ziinit.z_filefunc.zseek32_file = NULL; + ziinit.z_filefunc.ztell32_file = NULL; + if (pzlib_filefunc64_32_def==NULL) + fill_fopen64_filefunc(&ziinit.z_filefunc.zfile_func64); + else + ziinit.z_filefunc = *pzlib_filefunc64_32_def; + + ziinit.filestream = ZOPEN64(ziinit.z_filefunc, + pathname, + (append == APPEND_STATUS_CREATE) ? + (ZLIB_FILEFUNC_MODE_READ | ZLIB_FILEFUNC_MODE_WRITE | ZLIB_FILEFUNC_MODE_CREATE) : + (ZLIB_FILEFUNC_MODE_READ | ZLIB_FILEFUNC_MODE_WRITE | ZLIB_FILEFUNC_MODE_EXISTING)); + + if (ziinit.filestream == NULL) + return NULL; + + if (append == APPEND_STATUS_CREATEAFTER) + ZSEEK64(ziinit.z_filefunc,ziinit.filestream,0,SEEK_END); + + ziinit.begin_pos = ZTELL64(ziinit.z_filefunc,ziinit.filestream); + ziinit.in_opened_file_inzip = 0; + ziinit.ci.stream_initialised = 0; + ziinit.number_entry = 0; + ziinit.add_position_when_writting_offset = 0; + init_linkedlist(&(ziinit.central_dir)); + + + + zi = (zip64_internal*)ALLOC(sizeof(zip64_internal)); + if (zi==NULL) + { + ZCLOSE64(ziinit.z_filefunc,ziinit.filestream); + return NULL; + } + + /* now we add file in a zipfile */ +# ifndef NO_ADDFILEINEXISTINGZIP + ziinit.globalcomment = NULL; + if (append == APPEND_STATUS_ADDINZIP) + { + // Read and Cache Central Directory Records + err = LoadCentralDirectoryRecord(&ziinit); + } + + if (globalcomment) + { + *globalcomment = ziinit.globalcomment; + } +# endif /* !NO_ADDFILEINEXISTINGZIP*/ + + if (err != ZIP_OK) + { +# ifndef NO_ADDFILEINEXISTINGZIP + TRYFREE(ziinit.globalcomment); +# endif /* !NO_ADDFILEINEXISTINGZIP*/ + TRYFREE(zi); + return NULL; + } + else + { + *zi = ziinit; + return (zipFile)zi; + } +} + +extern zipFile ZEXPORT zipOpen2 (const char *pathname, int append, zipcharpc* globalcomment, zlib_filefunc_def* pzlib_filefunc32_def) +{ + if (pzlib_filefunc32_def != NULL) + { + zlib_filefunc64_32_def zlib_filefunc64_32_def_fill; + fill_zlib_filefunc64_32_def_from_filefunc32(&zlib_filefunc64_32_def_fill,pzlib_filefunc32_def); + return zipOpen3(pathname, append, globalcomment, &zlib_filefunc64_32_def_fill); + } + else + return zipOpen3(pathname, append, globalcomment, NULL); +} + +extern zipFile ZEXPORT zipOpen2_64 (const void *pathname, int append, zipcharpc* globalcomment, zlib_filefunc64_def* pzlib_filefunc_def) +{ + if (pzlib_filefunc_def != NULL) + { + zlib_filefunc64_32_def zlib_filefunc64_32_def_fill; + zlib_filefunc64_32_def_fill.zfile_func64 = *pzlib_filefunc_def; + zlib_filefunc64_32_def_fill.ztell32_file = NULL; + zlib_filefunc64_32_def_fill.zseek32_file = NULL; + return zipOpen3(pathname, append, globalcomment, &zlib_filefunc64_32_def_fill); + } + else + return zipOpen3(pathname, append, globalcomment, NULL); +} + + + +extern zipFile ZEXPORT zipOpen (const char* pathname, int append) +{ + return zipOpen3((const void*)pathname,append,NULL,NULL); +} + +extern zipFile ZEXPORT zipOpen64 (const void* pathname, int append) +{ + return zipOpen3(pathname,append,NULL,NULL); +} + +int Write_LocalFileHeader(zip64_internal* zi, const char* filename, uInt size_extrafield_local, const void* extrafield_local) +{ + /* write the local header */ + int err; + uInt size_filename = (uInt)strlen(filename); + uInt size_extrafield = size_extrafield_local; + + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)LOCALHEADERMAGIC, 4); + + if (err==ZIP_OK) + { + if(zi->ci.zip64) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)45,2);/* version needed to extract */ + else + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)20,2);/* version needed to extract */ + } + + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)zi->ci.flag,2); + + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)zi->ci.method,2); + + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)zi->ci.dosDate,4); + + // CRC / Compressed size / Uncompressed size will be filled in later and rewritten later + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,4); /* crc 32, unknown */ + if (err==ZIP_OK) + { + if(zi->ci.zip64) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0xFFFFFFFF,4); /* compressed size, unknown */ + else + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,4); /* compressed size, unknown */ + } + if (err==ZIP_OK) + { + if(zi->ci.zip64) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0xFFFFFFFF,4); /* uncompressed size, unknown */ + else + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,4); /* uncompressed size, unknown */ + } + + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)size_filename,2); + + if(zi->ci.zip64) + { + size_extrafield += 20; + } + + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)size_extrafield,2); + + if ((err==ZIP_OK) && (size_filename > 0)) + { + if (ZWRITE64(zi->z_filefunc,zi->filestream,filename,size_filename)!=size_filename) + err = ZIP_ERRNO; + } + + if ((err==ZIP_OK) && (size_extrafield_local > 0)) + { + if (ZWRITE64(zi->z_filefunc, zi->filestream, extrafield_local, size_extrafield_local) != size_extrafield_local) + err = ZIP_ERRNO; + } + + + if ((err==ZIP_OK) && (zi->ci.zip64)) + { + // write the Zip64 extended info + short HeaderID = 1; + short DataSize = 16; + ZPOS64_T CompressedSize = 0; + ZPOS64_T UncompressedSize = 0; + + // Remember position of Zip64 extended info for the local file header. (needed when we update size after done with file) + zi->ci.pos_zip64extrainfo = ZTELL64(zi->z_filefunc,zi->filestream); + + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, (short)HeaderID,2); + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, (short)DataSize,2); + + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, (ZPOS64_T)UncompressedSize,8); + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, (ZPOS64_T)CompressedSize,8); + } + + return err; +} + +/* + NOTE. + When writing RAW the ZIP64 extended information in extrafield_local and extrafield_global needs to be stripped + before calling this function it can be done with zipRemoveExtraInfoBlock + + It is not done here because then we need to realloc a new buffer since parameters are 'const' and I want to minimize + unnecessary allocations. + */ +extern int ZEXPORT zipOpenNewFileInZip4_64 (zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw, + int windowBits,int memLevel, int strategy, + const char* password, uLong crcForCrypting, + uLong versionMadeBy, uLong flagBase, int zip64) +{ + zip64_internal* zi; + uInt size_filename; + uInt size_comment; + uInt i; + int err = ZIP_OK; + +# ifdef NOCRYPT + (crcForCrypting); + if (password != NULL) + return ZIP_PARAMERROR; +# endif + + if (file == NULL) + return ZIP_PARAMERROR; + +#ifdef HAVE_BZIP2 + if ((method!=0) && (method!=Z_DEFLATED) && (method!=Z_BZIP2ED)) + return ZIP_PARAMERROR; +#else + if ((method!=0) && (method!=Z_DEFLATED)) + return ZIP_PARAMERROR; +#endif + + zi = (zip64_internal*)file; + + if (zi->in_opened_file_inzip == 1) + { + err = zipCloseFileInZip (file); + if (err != ZIP_OK) + return err; + } + + if (filename==NULL) + filename="-"; + + if (comment==NULL) + size_comment = 0; + else + size_comment = (uInt)strlen(comment); + + size_filename = (uInt)strlen(filename); + + if (zipfi == NULL) + zi->ci.dosDate = 0; + else + { + if (zipfi->dosDate != 0) + zi->ci.dosDate = zipfi->dosDate; + else + zi->ci.dosDate = zip64local_TmzDateToDosDate(&zipfi->tmz_date); + } + + zi->ci.flag = flagBase; + if ((level==8) || (level==9)) + zi->ci.flag |= 2; + if (level==2) + zi->ci.flag |= 4; + if (level==1) + zi->ci.flag |= 6; + if (password != NULL) + zi->ci.flag |= 1; + + zi->ci.crc32 = 0; + zi->ci.method = method; + zi->ci.encrypt = 0; + zi->ci.stream_initialised = 0; + zi->ci.pos_in_buffered_data = 0; + zi->ci.raw = raw; + zi->ci.pos_local_header = ZTELL64(zi->z_filefunc,zi->filestream); + + zi->ci.size_centralheader = SIZECENTRALHEADER + size_filename + size_extrafield_global + size_comment; + zi->ci.size_centralExtraFree = 32; // Extra space we have reserved in case we need to add ZIP64 extra info data + + zi->ci.central_header = (char*)ALLOC((uInt)zi->ci.size_centralheader + zi->ci.size_centralExtraFree); + + zi->ci.size_centralExtra = size_extrafield_global; + zip64local_putValue_inmemory(zi->ci.central_header,(uLong)CENTRALHEADERMAGIC,4); + /* version info */ + zip64local_putValue_inmemory(zi->ci.central_header+4,(uLong)versionMadeBy,2); + zip64local_putValue_inmemory(zi->ci.central_header+6,(uLong)20,2); + zip64local_putValue_inmemory(zi->ci.central_header+8,(uLong)zi->ci.flag,2); + zip64local_putValue_inmemory(zi->ci.central_header+10,(uLong)zi->ci.method,2); + zip64local_putValue_inmemory(zi->ci.central_header+12,(uLong)zi->ci.dosDate,4); + zip64local_putValue_inmemory(zi->ci.central_header+16,(uLong)0,4); /*crc*/ + zip64local_putValue_inmemory(zi->ci.central_header+20,(uLong)0,4); /*compr size*/ + zip64local_putValue_inmemory(zi->ci.central_header+24,(uLong)0,4); /*uncompr size*/ + zip64local_putValue_inmemory(zi->ci.central_header+28,(uLong)size_filename,2); + zip64local_putValue_inmemory(zi->ci.central_header+30,(uLong)size_extrafield_global,2); + zip64local_putValue_inmemory(zi->ci.central_header+32,(uLong)size_comment,2); + zip64local_putValue_inmemory(zi->ci.central_header+34,(uLong)0,2); /*disk nm start*/ + + if (zipfi==NULL) + zip64local_putValue_inmemory(zi->ci.central_header+36,(uLong)0,2); + else + zip64local_putValue_inmemory(zi->ci.central_header+36,(uLong)zipfi->internal_fa,2); + + if (zipfi==NULL) + zip64local_putValue_inmemory(zi->ci.central_header+38,(uLong)0,4); + else + zip64local_putValue_inmemory(zi->ci.central_header+38,(uLong)zipfi->external_fa,4); + + if(zi->ci.pos_local_header >= 0xffffffff) + zip64local_putValue_inmemory(zi->ci.central_header+42,(uLong)0xffffffff,4); + else + zip64local_putValue_inmemory(zi->ci.central_header+42,(uLong)zi->ci.pos_local_header - zi->add_position_when_writting_offset,4); + + for (i=0;i<size_filename;i++) + *(zi->ci.central_header+SIZECENTRALHEADER+i) = *(filename+i); + + for (i=0;i<size_extrafield_global;i++) + *(zi->ci.central_header+SIZECENTRALHEADER+size_filename+i) = + *(((const char*)extrafield_global)+i); + + for (i=0;i<size_comment;i++) + *(zi->ci.central_header+SIZECENTRALHEADER+size_filename+ + size_extrafield_global+i) = *(comment+i); + if (zi->ci.central_header == NULL) + return ZIP_INTERNALERROR; + + zi->ci.zip64 = zip64; + zi->ci.totalCompressedData = 0; + zi->ci.totalUncompressedData = 0; + zi->ci.pos_zip64extrainfo = 0; + + err = Write_LocalFileHeader(zi, filename, size_extrafield_local, extrafield_local); + +#ifdef HAVE_BZIP2 + zi->ci.bstream.avail_in = (uInt)0; + zi->ci.bstream.avail_out = (uInt)Z_BUFSIZE; + zi->ci.bstream.next_out = (char*)zi->ci.buffered_data; + zi->ci.bstream.total_in_hi32 = 0; + zi->ci.bstream.total_in_lo32 = 0; + zi->ci.bstream.total_out_hi32 = 0; + zi->ci.bstream.total_out_lo32 = 0; +#endif + + zi->ci.stream.avail_in = (uInt)0; + zi->ci.stream.avail_out = (uInt)Z_BUFSIZE; + zi->ci.stream.next_out = zi->ci.buffered_data; + zi->ci.stream.total_in = 0; + zi->ci.stream.total_out = 0; + zi->ci.stream.data_type = Z_BINARY; + +#ifdef HAVE_BZIP2 + if ((err==ZIP_OK) && (zi->ci.method == Z_DEFLATED || zi->ci.method == Z_BZIP2ED) && (!zi->ci.raw)) +#else + if ((err==ZIP_OK) && (zi->ci.method == Z_DEFLATED) && (!zi->ci.raw)) +#endif + { + if(zi->ci.method == Z_DEFLATED) + { + zi->ci.stream.zalloc = (alloc_func)0; + zi->ci.stream.zfree = (free_func)0; + zi->ci.stream.opaque = (voidpf)0; + + if (windowBits>0) + windowBits = -windowBits; + + err = deflateInit2(&zi->ci.stream, level, Z_DEFLATED, windowBits, memLevel, strategy); + + if (err==Z_OK) + zi->ci.stream_initialised = Z_DEFLATED; + } + else if(zi->ci.method == Z_BZIP2ED) + { +#ifdef HAVE_BZIP2 + // Init BZip stuff here + zi->ci.bstream.bzalloc = 0; + zi->ci.bstream.bzfree = 0; + zi->ci.bstream.opaque = (voidpf)0; + + err = BZ2_bzCompressInit(&zi->ci.bstream, level, 0,35); + if(err == BZ_OK) + zi->ci.stream_initialised = Z_BZIP2ED; +#endif + } + + } + +# ifndef NOCRYPT + zi->ci.crypt_header_size = 0; + if ((err==Z_OK) && (password != NULL)) + { + unsigned char bufHead[RAND_HEAD_LEN]; + unsigned int sizeHead; + zi->ci.encrypt = 1; + zi->ci.pcrc_32_tab = get_crc_table(); + /*init_keys(password,zi->ci.keys,zi->ci.pcrc_32_tab);*/ + + sizeHead=crypthead(password,bufHead,RAND_HEAD_LEN,zi->ci.keys,zi->ci.pcrc_32_tab,crcForCrypting); + zi->ci.crypt_header_size = sizeHead; + + if (ZWRITE64(zi->z_filefunc,zi->filestream,bufHead,sizeHead) != sizeHead) + err = ZIP_ERRNO; + } +# endif + + if (err==Z_OK) + zi->in_opened_file_inzip = 1; + return err; +} + +extern int ZEXPORT zipOpenNewFileInZip4 (zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw, + int windowBits,int memLevel, int strategy, + const char* password, uLong crcForCrypting, + uLong versionMadeBy, uLong flagBase) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, raw, + windowBits, memLevel, strategy, + password, crcForCrypting, versionMadeBy, flagBase, 0); +} + +extern int ZEXPORT zipOpenNewFileInZip3 (zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw, + int windowBits,int memLevel, int strategy, + const char* password, uLong crcForCrypting) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, raw, + windowBits, memLevel, strategy, + password, crcForCrypting, VERSIONMADEBY, 0, 0); +} + +extern int ZEXPORT zipOpenNewFileInZip3_64(zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw, + int windowBits,int memLevel, int strategy, + const char* password, uLong crcForCrypting, int zip64) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, raw, + windowBits, memLevel, strategy, + password, crcForCrypting, VERSIONMADEBY, 0, zip64); +} + +extern int ZEXPORT zipOpenNewFileInZip2(zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, raw, + -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, + NULL, 0, VERSIONMADEBY, 0, 0); +} + +extern int ZEXPORT zipOpenNewFileInZip2_64(zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw, int zip64) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, raw, + -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, + NULL, 0, VERSIONMADEBY, 0, zip64); +} + +extern int ZEXPORT zipOpenNewFileInZip64 (zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void*extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int zip64) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, 0, + -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, + NULL, 0, VERSIONMADEBY, 0, zip64); +} + +extern int ZEXPORT zipOpenNewFileInZip (zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void*extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, 0, + -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, + NULL, 0, VERSIONMADEBY, 0, 0); +} + +local int zip64FlushWriteBuffer(zip64_internal* zi) +{ + int err=ZIP_OK; + + if (zi->ci.encrypt != 0) + { +#ifndef NOCRYPT + uInt i; + int t; + for (i=0;i<zi->ci.pos_in_buffered_data;i++) + zi->ci.buffered_data[i] = zencode(zi->ci.keys, zi->ci.pcrc_32_tab, zi->ci.buffered_data[i],t); +#endif + } + + if (ZWRITE64(zi->z_filefunc,zi->filestream,zi->ci.buffered_data,zi->ci.pos_in_buffered_data) != zi->ci.pos_in_buffered_data) + err = ZIP_ERRNO; + + zi->ci.totalCompressedData += zi->ci.pos_in_buffered_data; + +#ifdef HAVE_BZIP2 + if(zi->ci.method == Z_BZIP2ED) + { + zi->ci.totalUncompressedData += zi->ci.bstream.total_in_lo32; + zi->ci.bstream.total_in_lo32 = 0; + zi->ci.bstream.total_in_hi32 = 0; + } + else +#endif + { + zi->ci.totalUncompressedData += zi->ci.stream.total_in; + zi->ci.stream.total_in = 0; + } + + + zi->ci.pos_in_buffered_data = 0; + + return err; +} + +extern int ZEXPORT zipWriteInFileInZip (zipFile file,const void* buf,unsigned int len) +{ + zip64_internal* zi; + int err=ZIP_OK; + + if (file == NULL) + return ZIP_PARAMERROR; + zi = (zip64_internal*)file; + + if (zi->in_opened_file_inzip == 0) + return ZIP_PARAMERROR; + + zi->ci.crc32 = crc32(zi->ci.crc32,buf,(uInt)len); + +#ifdef HAVE_BZIP2 + if(zi->ci.method == Z_BZIP2ED && (!zi->ci.raw)) + { + zi->ci.bstream.next_in = (void*)buf; + zi->ci.bstream.avail_in = len; + err = BZ_RUN_OK; + + while ((err==BZ_RUN_OK) && (zi->ci.bstream.avail_in>0)) + { + if (zi->ci.bstream.avail_out == 0) + { + if (zip64FlushWriteBuffer(zi) == ZIP_ERRNO) + err = ZIP_ERRNO; + zi->ci.bstream.avail_out = (uInt)Z_BUFSIZE; + zi->ci.bstream.next_out = (char*)zi->ci.buffered_data; + } + + + if(err != BZ_RUN_OK) + break; + + if ((zi->ci.method == Z_BZIP2ED) && (!zi->ci.raw)) + { + uLong uTotalOutBefore_lo = zi->ci.bstream.total_out_lo32; +// uLong uTotalOutBefore_hi = zi->ci.bstream.total_out_hi32; + err=BZ2_bzCompress(&zi->ci.bstream, BZ_RUN); + + zi->ci.pos_in_buffered_data += (uInt)(zi->ci.bstream.total_out_lo32 - uTotalOutBefore_lo) ; + } + } + + if(err == BZ_RUN_OK) + err = ZIP_OK; + } + else +#endif + { + zi->ci.stream.next_in = (Bytef*)buf; + zi->ci.stream.avail_in = len; + + while ((err==ZIP_OK) && (zi->ci.stream.avail_in>0)) + { + if (zi->ci.stream.avail_out == 0) + { + if (zip64FlushWriteBuffer(zi) == ZIP_ERRNO) + err = ZIP_ERRNO; + zi->ci.stream.avail_out = (uInt)Z_BUFSIZE; + zi->ci.stream.next_out = zi->ci.buffered_data; + } + + + if(err != ZIP_OK) + break; + + if ((zi->ci.method == Z_DEFLATED) && (!zi->ci.raw)) + { + uLong uTotalOutBefore = zi->ci.stream.total_out; + err=deflate(&zi->ci.stream, Z_NO_FLUSH); + if(uTotalOutBefore > zi->ci.stream.total_out) + { + int bBreak = 0; + bBreak++; + } + + zi->ci.pos_in_buffered_data += (uInt)(zi->ci.stream.total_out - uTotalOutBefore) ; + } + else + { + uInt copy_this,i; + if (zi->ci.stream.avail_in < zi->ci.stream.avail_out) + copy_this = zi->ci.stream.avail_in; + else + copy_this = zi->ci.stream.avail_out; + + for (i = 0; i < copy_this; i++) + *(((char*)zi->ci.stream.next_out)+i) = + *(((const char*)zi->ci.stream.next_in)+i); + { + zi->ci.stream.avail_in -= copy_this; + zi->ci.stream.avail_out-= copy_this; + zi->ci.stream.next_in+= copy_this; + zi->ci.stream.next_out+= copy_this; + zi->ci.stream.total_in+= copy_this; + zi->ci.stream.total_out+= copy_this; + zi->ci.pos_in_buffered_data += copy_this; + } + } + }// while(...) + } + + return err; +} + +extern int ZEXPORT zipCloseFileInZipRaw (zipFile file, uLong uncompressed_size, uLong crc32) +{ + return zipCloseFileInZipRaw64 (file, uncompressed_size, crc32); +} + +extern int ZEXPORT zipCloseFileInZipRaw64 (zipFile file, ZPOS64_T uncompressed_size, uLong crc32) +{ + zip64_internal* zi; + ZPOS64_T compressed_size; + uLong invalidValue = 0xffffffff; + short datasize = 0; + int err=ZIP_OK; + + if (file == NULL) + return ZIP_PARAMERROR; + zi = (zip64_internal*)file; + + if (zi->in_opened_file_inzip == 0) + return ZIP_PARAMERROR; + zi->ci.stream.avail_in = 0; + + if ((zi->ci.method == Z_DEFLATED) && (!zi->ci.raw)) + { + while (err==ZIP_OK) + { + uLong uTotalOutBefore; + if (zi->ci.stream.avail_out == 0) + { + if (zip64FlushWriteBuffer(zi) == ZIP_ERRNO) + err = ZIP_ERRNO; + zi->ci.stream.avail_out = (uInt)Z_BUFSIZE; + zi->ci.stream.next_out = zi->ci.buffered_data; + } + uTotalOutBefore = zi->ci.stream.total_out; + err=deflate(&zi->ci.stream, Z_FINISH); + zi->ci.pos_in_buffered_data += (uInt)(zi->ci.stream.total_out - uTotalOutBefore) ; + } + } + else if ((zi->ci.method == Z_BZIP2ED) && (!zi->ci.raw)) + { +#ifdef HAVE_BZIP2 + err = BZ_FINISH_OK; + while (err==BZ_FINISH_OK) + { + uLong uTotalOutBefore; + if (zi->ci.bstream.avail_out == 0) + { + if (zip64FlushWriteBuffer(zi) == ZIP_ERRNO) + err = ZIP_ERRNO; + zi->ci.bstream.avail_out = (uInt)Z_BUFSIZE; + zi->ci.bstream.next_out = (char*)zi->ci.buffered_data; + } + uTotalOutBefore = zi->ci.bstream.total_out_lo32; + err=BZ2_bzCompress(&zi->ci.bstream, BZ_FINISH); + if(err == BZ_STREAM_END) + err = Z_STREAM_END; + + zi->ci.pos_in_buffered_data += (uInt)(zi->ci.bstream.total_out_lo32 - uTotalOutBefore); + } + + if(err == BZ_FINISH_OK) + err = ZIP_OK; +#endif + } + + if (err==Z_STREAM_END) + err=ZIP_OK; /* this is normal */ + + if ((zi->ci.pos_in_buffered_data>0) && (err==ZIP_OK)) + { + if (zip64FlushWriteBuffer(zi)==ZIP_ERRNO) + err = ZIP_ERRNO; + } + + if ((zi->ci.method == Z_DEFLATED) && (!zi->ci.raw)) + { + int tmp_err = deflateEnd(&zi->ci.stream); + if (err == ZIP_OK) + err = tmp_err; + zi->ci.stream_initialised = 0; + } +#ifdef HAVE_BZIP2 + else if((zi->ci.method == Z_BZIP2ED) && (!zi->ci.raw)) + { + int tmperr = BZ2_bzCompressEnd(&zi->ci.bstream); + if (err==ZIP_OK) + err = tmperr; + zi->ci.stream_initialised = 0; + } +#endif + + if (!zi->ci.raw) + { + crc32 = (uLong)zi->ci.crc32; + uncompressed_size = zi->ci.totalUncompressedData; + } + compressed_size = zi->ci.totalCompressedData; + +# ifndef NOCRYPT + compressed_size += zi->ci.crypt_header_size; +# endif + + // update Current Item crc and sizes, + if(compressed_size >= 0xffffffff || uncompressed_size >= 0xffffffff || zi->ci.pos_local_header >= 0xffffffff) + { + /*version Made by*/ + zip64local_putValue_inmemory(zi->ci.central_header+4,(uLong)45,2); + /*version needed*/ + zip64local_putValue_inmemory(zi->ci.central_header+6,(uLong)45,2); + + } + + zip64local_putValue_inmemory(zi->ci.central_header+16,crc32,4); /*crc*/ + + + if(compressed_size >= 0xffffffff) + zip64local_putValue_inmemory(zi->ci.central_header+20, invalidValue,4); /*compr size*/ + else + zip64local_putValue_inmemory(zi->ci.central_header+20, compressed_size,4); /*compr size*/ + + /// set internal file attributes field + if (zi->ci.stream.data_type == Z_ASCII) + zip64local_putValue_inmemory(zi->ci.central_header+36,(uLong)Z_ASCII,2); + + if(uncompressed_size >= 0xffffffff) + zip64local_putValue_inmemory(zi->ci.central_header+24, invalidValue,4); /*uncompr size*/ + else + zip64local_putValue_inmemory(zi->ci.central_header+24, uncompressed_size,4); /*uncompr size*/ + + // Add ZIP64 extra info field for uncompressed size + if(uncompressed_size >= 0xffffffff) + datasize += 8; + + // Add ZIP64 extra info field for compressed size + if(compressed_size >= 0xffffffff) + datasize += 8; + + // Add ZIP64 extra info field for relative offset to local file header of current file + if(zi->ci.pos_local_header >= 0xffffffff) + datasize += 8; + + if(datasize > 0) + { + char* p = NULL; + + if((uLong)(datasize + 4) > zi->ci.size_centralExtraFree) + { + // we can not write more data to the buffer that we have room for. + return ZIP_BADZIPFILE; + } + + p = zi->ci.central_header + zi->ci.size_centralheader; + + // Add Extra Information Header for 'ZIP64 information' + zip64local_putValue_inmemory(p, 0x0001, 2); // HeaderID + p += 2; + zip64local_putValue_inmemory(p, datasize, 2); // DataSize + p += 2; + + if(uncompressed_size >= 0xffffffff) + { + zip64local_putValue_inmemory(p, uncompressed_size, 8); + p += 8; + } + + if(compressed_size >= 0xffffffff) + { + zip64local_putValue_inmemory(p, compressed_size, 8); + p += 8; + } + + if(zi->ci.pos_local_header >= 0xffffffff) + { + zip64local_putValue_inmemory(p, zi->ci.pos_local_header, 8); + p += 8; + } + + // Update how much extra free space we got in the memory buffer + // and increase the centralheader size so the new ZIP64 fields are included + // ( 4 below is the size of HeaderID and DataSize field ) + zi->ci.size_centralExtraFree -= datasize + 4; + zi->ci.size_centralheader += datasize + 4; + + // Update the extra info size field + zi->ci.size_centralExtra += datasize + 4; + zip64local_putValue_inmemory(zi->ci.central_header+30,(uLong)zi->ci.size_centralExtra,2); + } + + if (err==ZIP_OK) + err = add_data_in_datablock(&zi->central_dir, zi->ci.central_header, (uLong)zi->ci.size_centralheader); + + free(zi->ci.central_header); + + if (err==ZIP_OK) + { + // Update the LocalFileHeader with the new values. + + ZPOS64_T cur_pos_inzip = ZTELL64(zi->z_filefunc,zi->filestream); + + if (ZSEEK64(zi->z_filefunc,zi->filestream, zi->ci.pos_local_header + 14,ZLIB_FILEFUNC_SEEK_SET)!=0) + err = ZIP_ERRNO; + + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,crc32,4); /* crc 32, unknown */ + + if(uncompressed_size >= 0xffffffff || compressed_size >= 0xffffffff ) + { + if(zi->ci.pos_zip64extrainfo > 0) + { + // Update the size in the ZIP64 extended field. + if (ZSEEK64(zi->z_filefunc,zi->filestream, zi->ci.pos_zip64extrainfo + 4,ZLIB_FILEFUNC_SEEK_SET)!=0) + err = ZIP_ERRNO; + + if (err==ZIP_OK) /* compressed size, unknown */ + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, uncompressed_size, 8); + + if (err==ZIP_OK) /* uncompressed size, unknown */ + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, compressed_size, 8); + } + else + err = ZIP_BADZIPFILE; // Caller passed zip64 = 0, so no room for zip64 info -> fatal + } + else + { + if (err==ZIP_OK) /* compressed size, unknown */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,compressed_size,4); + + if (err==ZIP_OK) /* uncompressed size, unknown */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,uncompressed_size,4); + } + + if (ZSEEK64(zi->z_filefunc,zi->filestream, cur_pos_inzip,ZLIB_FILEFUNC_SEEK_SET)!=0) + err = ZIP_ERRNO; + } + + zi->number_entry ++; + zi->in_opened_file_inzip = 0; + + return err; +} + +extern int ZEXPORT zipCloseFileInZip (zipFile file) +{ + return zipCloseFileInZipRaw (file,0,0); +} + +int Write_Zip64EndOfCentralDirectoryLocator(zip64_internal* zi, ZPOS64_T zip64eocd_pos_inzip) +{ + int err = ZIP_OK; + ZPOS64_T pos = zip64eocd_pos_inzip - zi->add_position_when_writting_offset; + + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)ZIP64ENDLOCHEADERMAGIC,4); + + /*num disks*/ + if (err==ZIP_OK) /* number of the disk with the start of the central directory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,4); + + /*relative offset*/ + if (err==ZIP_OK) /* Relative offset to the Zip64EndOfCentralDirectory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream, pos,8); + + /*total disks*/ /* Do not support spawning of disk so always say 1 here*/ + if (err==ZIP_OK) /* number of the disk with the start of the central directory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)1,4); + + return err; +} + +int Write_Zip64EndOfCentralDirectoryRecord(zip64_internal* zi, uLong size_centraldir, ZPOS64_T centraldir_pos_inzip) +{ + int err = ZIP_OK; + + uLong Zip64DataSize = 44; + + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)ZIP64ENDHEADERMAGIC,4); + + if (err==ZIP_OK) /* size of this 'zip64 end of central directory' */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(ZPOS64_T)Zip64DataSize,8); // why ZPOS64_T of this ? + + if (err==ZIP_OK) /* version made by */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)45,2); + + if (err==ZIP_OK) /* version needed */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)45,2); + + if (err==ZIP_OK) /* number of this disk */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,4); + + if (err==ZIP_OK) /* number of the disk with the start of the central directory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,4); + + if (err==ZIP_OK) /* total number of entries in the central dir on this disk */ + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, zi->number_entry, 8); + + if (err==ZIP_OK) /* total number of entries in the central dir */ + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, zi->number_entry, 8); + + if (err==ZIP_OK) /* size of the central directory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(ZPOS64_T)size_centraldir,8); + + if (err==ZIP_OK) /* offset of start of central directory with respect to the starting disk number */ + { + ZPOS64_T pos = centraldir_pos_inzip - zi->add_position_when_writting_offset; + err = zip64local_putValue(&zi->z_filefunc,zi->filestream, (ZPOS64_T)pos,8); + } + return err; +} +int Write_EndOfCentralDirectoryRecord(zip64_internal* zi, uLong size_centraldir, ZPOS64_T centraldir_pos_inzip) +{ + int err = ZIP_OK; + + /*signature*/ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)ENDHEADERMAGIC,4); + + if (err==ZIP_OK) /* number of this disk */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,2); + + if (err==ZIP_OK) /* number of the disk with the start of the central directory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,2); + + if (err==ZIP_OK) /* total number of entries in the central dir on this disk */ + { + { + if(zi->number_entry >= 0xFFFF) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0xffff,2); // use value in ZIP64 record + else + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)zi->number_entry,2); + } + } + + if (err==ZIP_OK) /* total number of entries in the central dir */ + { + if(zi->number_entry >= 0xFFFF) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0xffff,2); // use value in ZIP64 record + else + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)zi->number_entry,2); + } + + if (err==ZIP_OK) /* size of the central directory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)size_centraldir,4); + + if (err==ZIP_OK) /* offset of start of central directory with respect to the starting disk number */ + { + ZPOS64_T pos = centraldir_pos_inzip - zi->add_position_when_writting_offset; + if(pos >= 0xffffffff) + { + err = zip64local_putValue(&zi->z_filefunc,zi->filestream, (uLong)0xffffffff,4); + } + else + err = zip64local_putValue(&zi->z_filefunc,zi->filestream, (uLong)(centraldir_pos_inzip - zi->add_position_when_writting_offset),4); + } + + return err; +} + +int Write_GlobalComment(zip64_internal* zi, const char* global_comment) +{ + int err = ZIP_OK; + uInt size_global_comment = 0; + + if(global_comment != NULL) + size_global_comment = (uInt)strlen(global_comment); + + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)size_global_comment,2); + + if (err == ZIP_OK && size_global_comment > 0) + { + if (ZWRITE64(zi->z_filefunc,zi->filestream, global_comment, size_global_comment) != size_global_comment) + err = ZIP_ERRNO; + } + return err; +} + +extern int ZEXPORT zipClose (zipFile file, const char* global_comment) +{ + zip64_internal* zi; + int err = 0; + uLong size_centraldir = 0; + ZPOS64_T centraldir_pos_inzip; + ZPOS64_T pos; + + if (file == NULL) + return ZIP_PARAMERROR; + + zi = (zip64_internal*)file; + + if (zi->in_opened_file_inzip == 1) + { + err = zipCloseFileInZip (file); + } + +#ifndef NO_ADDFILEINEXISTINGZIP + if (global_comment==NULL) + global_comment = zi->globalcomment; +#endif + + centraldir_pos_inzip = ZTELL64(zi->z_filefunc,zi->filestream); + + if (err==ZIP_OK) + { + linkedlist_datablock_internal* ldi = zi->central_dir.first_block; + while (ldi!=NULL) + { + if ((err==ZIP_OK) && (ldi->filled_in_this_block>0)) + { + if (ZWRITE64(zi->z_filefunc,zi->filestream, ldi->data, ldi->filled_in_this_block) != ldi->filled_in_this_block) + err = ZIP_ERRNO; + } + + size_centraldir += ldi->filled_in_this_block; + ldi = ldi->next_datablock; + } + } + free_linkedlist(&(zi->central_dir)); + + pos = centraldir_pos_inzip - zi->add_position_when_writting_offset; + if(pos >= 0xffffffff || zi->number_entry > 0xFFFF) + { + ZPOS64_T Zip64EOCDpos = ZTELL64(zi->z_filefunc,zi->filestream); + Write_Zip64EndOfCentralDirectoryRecord(zi, size_centraldir, centraldir_pos_inzip); + + Write_Zip64EndOfCentralDirectoryLocator(zi, Zip64EOCDpos); + } + + if (err==ZIP_OK) + err = Write_EndOfCentralDirectoryRecord(zi, size_centraldir, centraldir_pos_inzip); + + if(err == ZIP_OK) + err = Write_GlobalComment(zi, global_comment); + + if (ZCLOSE64(zi->z_filefunc,zi->filestream) != 0) + if (err == ZIP_OK) + err = ZIP_ERRNO; + +#ifndef NO_ADDFILEINEXISTINGZIP + TRYFREE(zi->globalcomment); +#endif + TRYFREE(zi); + + return err; +} + +extern int ZEXPORT zipRemoveExtraInfoBlock (char* pData, int* dataLen, short sHeader) +{ + char* p = pData; + int size = 0; + char* pNewHeader; + char* pTmp; + short header; + short dataSize; + + int retVal = ZIP_OK; + + if(pData == NULL || *dataLen < 4) + return ZIP_PARAMERROR; + + pNewHeader = (char*)ALLOC(*dataLen); + pTmp = pNewHeader; + + while(p < (pData + *dataLen)) + { + header = *(short*)p; + dataSize = *(((short*)p)+1); + + if( header == sHeader ) // Header found. + { + p += dataSize + 4; // skip it. do not copy to temp buffer + } + else + { + // Extra Info block should not be removed, So copy it to the temp buffer. + memcpy(pTmp, p, dataSize + 4); + p += dataSize + 4; + size += dataSize + 4; + } + + } + + if(size < *dataLen) + { + // clean old extra info block. + memset(pData,0, *dataLen); + + // copy the new extra info block over the old + if(size > 0) + memcpy(pData, pNewHeader, size); + + // set the new extra info size + *dataLen = size; + + retVal = ZIP_OK; + } + else + retVal = ZIP_ERRNO; + + TRYFREE(pNewHeader); + + return retVal; +} diff --git a/storage/connect/zip.h b/storage/connect/zip.h new file mode 100644 index 00000000000..8aaebb62343 --- /dev/null +++ b/storage/connect/zip.h @@ -0,0 +1,362 @@ +/* zip.h -- IO on .zip files using zlib + Version 1.1, February 14h, 2010 + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications for Zip64 support + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + + --------------------------------------------------------------------------- + + Condition of use and distribution are the same than zlib : + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + --------------------------------------------------------------------------- + + Changes + + See header of zip.h + +*/ + +#ifndef _zip12_H +#define _zip12_H + +#ifdef __cplusplus +extern "C" { +#endif + +//#define HAVE_BZIP2 + +#ifndef _ZLIB_H +#include "zlib.h" +#endif + +#ifndef _ZLIBIOAPI_H +#include "ioapi.h" +#endif + +#ifdef HAVE_BZIP2 +#include "bzlib.h" +#endif + +#define Z_BZIP2ED 12 + +#if defined(STRICTZIP) || defined(STRICTZIPUNZIP) +/* like the STRICT of WIN32, we define a pointer that cannot be converted + from (void*) without cast */ +typedef struct TagzipFile__ { int unused; } zipFile__; +typedef zipFile__ *zipFile; +#else +typedef voidp zipFile; +#endif + +#define ZIP_OK (0) +#define ZIP_EOF (0) +#define ZIP_ERRNO (Z_ERRNO) +#define ZIP_PARAMERROR (-102) +#define ZIP_BADZIPFILE (-103) +#define ZIP_INTERNALERROR (-104) + +#ifndef DEF_MEM_LEVEL +# if MAX_MEM_LEVEL >= 8 +# define DEF_MEM_LEVEL 8 +# else +# define DEF_MEM_LEVEL MAX_MEM_LEVEL +# endif +#endif +/* default memLevel */ + +/* tm_zip contain date/time info */ +typedef struct tm_zip_s +{ + uInt tm_sec; /* seconds after the minute - [0,59] */ + uInt tm_min; /* minutes after the hour - [0,59] */ + uInt tm_hour; /* hours since midnight - [0,23] */ + uInt tm_mday; /* day of the month - [1,31] */ + uInt tm_mon; /* months since January - [0,11] */ + uInt tm_year; /* years - [1980..2044] */ +} tm_zip; + +typedef struct +{ + tm_zip tmz_date; /* date in understandable format */ + uLong dosDate; /* if dos_date == 0, tmu_date is used */ +/* uLong flag; */ /* general purpose bit flag 2 bytes */ + + uLong internal_fa; /* internal file attributes 2 bytes */ + uLong external_fa; /* external file attributes 4 bytes */ +} zip_fileinfo; + +typedef const char* zipcharpc; + + +#define APPEND_STATUS_CREATE (0) +#define APPEND_STATUS_CREATEAFTER (1) +#define APPEND_STATUS_ADDINZIP (2) + +extern zipFile ZEXPORT zipOpen OF((const char *pathname, int append)); +extern zipFile ZEXPORT zipOpen64 OF((const void *pathname, int append)); +/* + Create a zipfile. + pathname contain on Windows XP a filename like "c:\\zlib\\zlib113.zip" or on + an Unix computer "zlib/zlib113.zip". + if the file pathname exist and append==APPEND_STATUS_CREATEAFTER, the zip + will be created at the end of the file. + (useful if the file contain a self extractor code) + if the file pathname exist and append==APPEND_STATUS_ADDINZIP, we will + add files in existing zip (be sure you don't add file that doesn't exist) + If the zipfile cannot be opened, the return value is NULL. + Else, the return value is a zipFile Handle, usable with other function + of this zip package. +*/ + +/* Note : there is no delete function into a zipfile. + If you want delete file into a zipfile, you must open a zipfile, and create another + Of couse, you can use RAW reading and writing to copy the file you did not want delte +*/ + +extern zipFile ZEXPORT zipOpen2 OF((const char *pathname, + int append, + zipcharpc* globalcomment, + zlib_filefunc_def* pzlib_filefunc_def)); + +extern zipFile ZEXPORT zipOpen2_64 OF((const void *pathname, + int append, + zipcharpc* globalcomment, + zlib_filefunc64_def* pzlib_filefunc_def)); + +extern int ZEXPORT zipOpenNewFileInZip OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level)); + +extern int ZEXPORT zipOpenNewFileInZip64 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int zip64)); + +/* + Open a file in the ZIP for writing. + filename : the filename in zip (if NULL, '-' without quote will be used + *zipfi contain supplemental information + if extrafield_local!=NULL and size_extrafield_local>0, extrafield_local + contains the extrafield data the the local header + if extrafield_global!=NULL and size_extrafield_global>0, extrafield_global + contains the extrafield data the the local header + if comment != NULL, comment contain the comment string + method contain the compression method (0 for store, Z_DEFLATED for deflate) + level contain the level of compression (can be Z_DEFAULT_COMPRESSION) + zip64 is set to 1 if a zip64 extended information block should be added to the local file header. + this MUST be '1' if the uncompressed size is >= 0xffffffff. + +*/ + + +extern int ZEXPORT zipOpenNewFileInZip2 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw)); + + +extern int ZEXPORT zipOpenNewFileInZip2_64 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw, + int zip64)); +/* + Same than zipOpenNewFileInZip, except if raw=1, we write raw file + */ + +extern int ZEXPORT zipOpenNewFileInZip3 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw, + int windowBits, + int memLevel, + int strategy, + const char* password, + uLong crcForCrypting)); + +extern int ZEXPORT zipOpenNewFileInZip3_64 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw, + int windowBits, + int memLevel, + int strategy, + const char* password, + uLong crcForCrypting, + int zip64 + )); + +/* + Same than zipOpenNewFileInZip2, except + windowBits,memLevel,,strategy : see parameter strategy in deflateInit2 + password : crypting password (NULL for no crypting) + crcForCrypting : crc of file to compress (needed for crypting) + */ + +extern int ZEXPORT zipOpenNewFileInZip4 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw, + int windowBits, + int memLevel, + int strategy, + const char* password, + uLong crcForCrypting, + uLong versionMadeBy, + uLong flagBase + )); + + +extern int ZEXPORT zipOpenNewFileInZip4_64 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw, + int windowBits, + int memLevel, + int strategy, + const char* password, + uLong crcForCrypting, + uLong versionMadeBy, + uLong flagBase, + int zip64 + )); +/* + Same than zipOpenNewFileInZip4, except + versionMadeBy : value for Version made by field + flag : value for flag field (compression level info will be added) + */ + + +extern int ZEXPORT zipWriteInFileInZip OF((zipFile file, + const void* buf, + unsigned len)); +/* + Write data in the zipfile +*/ + +extern int ZEXPORT zipCloseFileInZip OF((zipFile file)); +/* + Close the current file in the zipfile +*/ + +extern int ZEXPORT zipCloseFileInZipRaw OF((zipFile file, + uLong uncompressed_size, + uLong crc32)); + +extern int ZEXPORT zipCloseFileInZipRaw64 OF((zipFile file, + ZPOS64_T uncompressed_size, + uLong crc32)); + +/* + Close the current file in the zipfile, for file opened with + parameter raw=1 in zipOpenNewFileInZip2 + uncompressed_size and crc32 are value for the uncompressed size +*/ + +extern int ZEXPORT zipClose OF((zipFile file, + const char* global_comment)); +/* + Close the zipfile +*/ + + +extern int ZEXPORT zipRemoveExtraInfoBlock OF((char* pData, int* dataLen, short sHeader)); +/* + zipRemoveExtraInfoBlock - Added by Mathias Svensson + + Remove extra information block from a extra information data for the local file header or central directory header + + It is needed to remove ZIP64 extra information blocks when before data is written if using RAW mode. + + 0x0001 is the signature header for the ZIP64 extra information blocks + + usage. + Remove ZIP64 Extra information from a central director extra field data + zipRemoveExtraInfoBlock(pCenDirExtraFieldData, &nCenDirExtraFieldDataLen, 0x0001); + + Remove ZIP64 Extra information from a Local File Header extra field data + zipRemoveExtraInfoBlock(pLocalHeaderExtraFieldData, &nLocalHeaderExtraFieldDataLen, 0x0001); +*/ + +#ifdef __cplusplus +} +#endif + +#endif /* _zip64_H */ diff --git a/storage/innobase/dict/dict0stats.cc b/storage/innobase/dict/dict0stats.cc index a4aa43651f8..c0a83c951a5 100644 --- a/storage/innobase/dict/dict0stats.cc +++ b/storage/innobase/dict/dict0stats.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 2009, 2015, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2009, 2016, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1109,7 +1109,8 @@ dict_stats_analyze_index_level( them away) which brings non-determinism. We skip only leaf-level delete marks because delete marks on non-leaf level do not make sense. */ - if (level == 0 && + + if (level == 0 && srv_stats_include_delete_marked? 0: rec_get_deleted_flag( rec, page_is_comp(btr_pcur_get_page(&pcur)))) { @@ -1295,8 +1296,12 @@ enum page_scan_method_t { the given page and count the number of distinct ones, also ignore delete marked records */ - QUIT_ON_FIRST_NON_BORING/* quit when the first record that differs + QUIT_ON_FIRST_NON_BORING,/* quit when the first record that differs from its right neighbor is found */ + COUNT_ALL_NON_BORING_INCLUDE_DEL_MARKED/* scan all records on + the given page and count the number of + distinct ones, include delete marked + records */ }; /* @} */ @@ -1572,6 +1577,8 @@ dict_stats_analyze_index_below_cur( offsets_rec = dict_stats_scan_page( &rec, offsets1, offsets2, index, page, n_prefix, + srv_stats_include_delete_marked ? + COUNT_ALL_NON_BORING_INCLUDE_DEL_MARKED: COUNT_ALL_NON_BORING_AND_SKIP_DEL_MARKED, n_diff, n_external_pages); diff --git a/storage/innobase/fts/fts0opt.cc b/storage/innobase/fts/fts0opt.cc index ea937c20752..455e2669c0d 100644 --- a/storage/innobase/fts/fts0opt.cc +++ b/storage/innobase/fts/fts0opt.cc @@ -578,9 +578,6 @@ fts_zip_read_word( fts_zip_t* zip, /*!< in: Zip state + data */ fts_string_t* word) /*!< out: uncompressed word */ { -#ifdef UNIV_DEBUG - ulint i; -#endif short len = 0; void* null = NULL; byte* ptr = word->f_str; @@ -655,10 +652,9 @@ fts_zip_read_word( } } -#ifdef UNIV_DEBUG /* All blocks must be freed at end of inflate. */ if (zip->status != Z_OK) { - for (i = 0; i < ib_vector_size(zip->blocks); ++i) { + for (ulint i = 0; i < ib_vector_size(zip->blocks); ++i) { if (ib_vector_getp(zip->blocks, i)) { ut_free(ib_vector_getp(zip->blocks, i)); ib_vector_set(zip->blocks, i, &null); @@ -669,7 +665,6 @@ fts_zip_read_word( if (ptr != NULL) { ut_ad(word->f_len == strlen((char*) ptr)); } -#endif /* UNIV_DEBUG */ return(zip->status == Z_OK || zip->status == Z_STREAM_END ? ptr : NULL); } diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 59ebc411db0..f94c564c8b3 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -13938,6 +13938,37 @@ ha_innobase::get_auto_increment( ulonglong col_max_value = innobase_get_int_col_max_value( table->next_number_field); + /** The following logic is needed to avoid duplicate key error + for autoincrement column. + + (1) InnoDB gives the current autoincrement value with respect + to increment and offset value. + + (2) Basically it does compute_next_insert_id() logic inside InnoDB + to avoid the current auto increment value changed by handler layer. + + (3) It is restricted only for insert operations. */ + + if (increment > 1 && thd_sql_command(user_thd) != SQLCOM_ALTER_TABLE + && autoinc < col_max_value) { + + ulonglong prev_auto_inc = autoinc; + + autoinc = ((autoinc - 1) + increment - offset)/ increment; + + autoinc = autoinc * increment + offset; + + /* If autoinc exceeds the col_max_value then reset + to old autoinc value. Because in case of non-strict + sql mode, boundary value is not considered as error. */ + + if (autoinc >= col_max_value) { + autoinc = prev_auto_inc; + } + + ut_ad(autoinc > 0); + } + /* Called for the first time ? */ if (trx->n_autoinc_rows == 0) { @@ -16462,6 +16493,12 @@ static MYSQL_SYSVAR_BOOL(use_fallocate, innobase_use_fallocate, "Preallocate files fast, using operating system functionality. On POSIX systems, posix_fallocate system call is used.", NULL, NULL, FALSE); +static MYSQL_SYSVAR_BOOL(stats_include_delete_marked, + srv_stats_include_delete_marked, + PLUGIN_VAR_OPCMDARG, + "Scan delete marked records for persistent stat", + NULL, NULL, FALSE); + static MYSQL_SYSVAR_ULONG(io_capacity, srv_io_capacity, PLUGIN_VAR_RQCMDARG, "Number of IOPs the server can do. Tunes the background IO rate", @@ -17293,6 +17330,7 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(doublewrite), MYSQL_SYSVAR(use_atomic_writes), MYSQL_SYSVAR(use_fallocate), + MYSQL_SYSVAR(stats_include_delete_marked), MYSQL_SYSVAR(api_enable_binlog), MYSQL_SYSVAR(api_enable_mdl), MYSQL_SYSVAR(api_disable_rowlock), diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 89a58d83a60..cc00b841579 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -1838,6 +1838,7 @@ innobase_fts_check_doc_id_index_in_def( return(FTS_NOT_EXIST_DOC_ID_INDEX); } + /*******************************************************************//** Create an index table where indexes are ordered as follows: @@ -1906,26 +1907,11 @@ innobase_create_key_defs( (only prefix/part of the column is indexed), MySQL will treat the index as a PRIMARY KEY unless the table already has one. */ - if (n_add > 0 && !new_primary && got_default_clust - && (key_info[*add].flags & HA_NOSAME) - && !(key_info[*add].flags & HA_KEY_HAS_PART_KEY_SEG)) { - uint key_part = key_info[*add].user_defined_key_parts; - - new_primary = true; + ut_ad(altered_table->s->primary_key == 0 + || altered_table->s->primary_key == MAX_KEY); - while (key_part--) { - const uint maybe_null - = key_info[*add].key_part[key_part].key_type - & FIELDFLAG_MAYBE_NULL; - DBUG_ASSERT(!maybe_null - == !key_info[*add].key_part[key_part]. - field->real_maybe_null()); - - if (maybe_null) { - new_primary = false; - break; - } - } + if (got_default_clust && !new_primary) { + new_primary = (altered_table->s->primary_key != MAX_KEY); } const bool rebuild = new_primary || add_fts_doc_id @@ -1944,8 +1930,14 @@ innobase_create_key_defs( ulint primary_key_number; if (new_primary) { - DBUG_ASSERT(n_add > 0); - primary_key_number = *add; + if (n_add == 0) { + DBUG_ASSERT(got_default_clust); + DBUG_ASSERT(altered_table->s->primary_key + == 0); + primary_key_number = 0; + } else { + primary_key_number = *add; + } } else if (got_default_clust) { /* Create the GEN_CLUST_INDEX */ index_def_t* index = indexdef++; @@ -3053,6 +3045,8 @@ prepare_inplace_alter_table_dict( ctx->add_cols = add_cols; } else { DBUG_ASSERT(!innobase_need_rebuild(ha_alter_info, old_table)); + DBUG_ASSERT(old_table->s->primary_key + == altered_table->s->primary_key); if (!ctx->new_table->fts && innobase_fulltext_exist(altered_table)) { @@ -4073,6 +4067,27 @@ found_col: add_fts_doc_id_idx)); } +/** Get the name of an erroneous key. +@param[in] error_key_num InnoDB number of the erroneus key +@param[in] ha_alter_info changes that were being performed +@param[in] table InnoDB table +@return the name of the erroneous key */ +static +const char* +get_error_key_name( + ulint error_key_num, + const Alter_inplace_info* ha_alter_info, + const dict_table_t* table) +{ + if (error_key_num == ULINT_UNDEFINED) { + return(FTS_DOC_ID_INDEX_NAME); + } else if (ha_alter_info->key_count == 0) { + return(dict_table_get_first_index(table)->name); + } else { + return(ha_alter_info->key_info_buffer[error_key_num].name); + } +} + /** Alter the table structure in-place with operations specified using Alter_inplace_info. The level of concurrency allowed during this operation depends @@ -4190,17 +4205,13 @@ oom: case DB_ONLINE_LOG_TOO_BIG: DBUG_ASSERT(ctx->online); my_error(ER_INNODB_ONLINE_LOG_TOO_BIG, MYF(0), - (prebuilt->trx->error_key_num == ULINT_UNDEFINED) - ? FTS_DOC_ID_INDEX_NAME - : ha_alter_info->key_info_buffer[ - prebuilt->trx->error_key_num].name); + get_error_key_name(prebuilt->trx->error_key_num, + ha_alter_info, prebuilt->table)); break; case DB_INDEX_CORRUPT: my_error(ER_INDEX_CORRUPT, MYF(0), - (prebuilt->trx->error_key_num == ULINT_UNDEFINED) - ? FTS_DOC_ID_INDEX_NAME - : ha_alter_info->key_info_buffer[ - prebuilt->trx->error_key_num].name); + get_error_key_name(prebuilt->trx->error_key_num, + ha_alter_info, prebuilt->table)); break; default: my_error_innodb(error, @@ -5013,7 +5024,6 @@ innobase_update_foreign_cache( "Foreign key constraints for table '%s'" " are loaded with charset check off", user_table->name); - } } @@ -5113,14 +5123,13 @@ commit_try_rebuild( DBUG_RETURN(true); case DB_ONLINE_LOG_TOO_BIG: my_error(ER_INNODB_ONLINE_LOG_TOO_BIG, MYF(0), - ha_alter_info->key_info_buffer[0].name); + get_error_key_name(err_key, ha_alter_info, + rebuilt_table)); DBUG_RETURN(true); case DB_INDEX_CORRUPT: my_error(ER_INDEX_CORRUPT, MYF(0), - (err_key == ULINT_UNDEFINED) - ? FTS_DOC_ID_INDEX_NAME - : ha_alter_info->key_info_buffer[err_key] - .name); + get_error_key_name(err_key, ha_alter_info, + rebuilt_table)); DBUG_RETURN(true); default: my_error_innodb(error, table_name, user_table->flags); diff --git a/storage/innobase/include/os0thread.h b/storage/innobase/include/os0thread.h index 9a1ada8fa0d..54f3d7554bf 100644 --- a/storage/innobase/include/os0thread.h +++ b/storage/innobase/include/os0thread.h @@ -117,14 +117,25 @@ os_thread_create_func( os_thread_id_t* thread_id); /*!< out: id of the created thread, or NULL */ +/** Waits until the specified thread completes and joins it. +Its return value is ignored. +@param[in,out] thread thread to join */ +UNIV_INTERN +void +os_thread_join( + os_thread_t thread); + /*****************************************************************//** Exits the current thread. */ UNIV_INTERN void os_thread_exit( /*===========*/ - void* exit_value) /*!< in: exit value; in Windows this void* + void* exit_value, /*!< in: exit value; in Windows this void* is cast as a DWORD */ + bool detach = true) /*!< in: if true, the thread will be detached + right before exiting. If false, another thread + is responsible for joining this thread. */ UNIV_COLD MY_ATTRIBUTE((noreturn)); /*****************************************************************//** Returns the thread identifier of current thread. diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index b8e1fcbbaf6..6389efa7b84 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -364,6 +364,7 @@ extern unsigned long long srv_stats_persistent_sample_pages; extern my_bool srv_stats_auto_recalc; extern unsigned long long srv_stats_modified_counter; extern my_bool srv_stats_sample_traditional; +extern my_bool srv_stats_include_delete_marked; extern ibool srv_use_doublewrite_buf; extern ulong srv_doublewrite_batch_size; diff --git a/storage/innobase/include/univ.i b/storage/innobase/include/univ.i index 7b867edfb22..824a06829f9 100644 --- a/storage/innobase/include/univ.i +++ b/storage/innobase/include/univ.i @@ -45,7 +45,7 @@ Created 1/20/1994 Heikki Tuuri #define INNODB_VERSION_MAJOR 5 #define INNODB_VERSION_MINOR 6 -#define INNODB_VERSION_BUGFIX 33 +#define INNODB_VERSION_BUGFIX 35 /* The following is the InnoDB version as shown in SELECT plugin_version FROM information_schema.plugins; diff --git a/storage/innobase/mach/mach0data.cc b/storage/innobase/mach/mach0data.cc index df68aab8a18..feeedb01609 100644 --- a/storage/innobase/mach/mach0data.cc +++ b/storage/innobase/mach/mach0data.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1995, 2009, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -55,8 +55,22 @@ mach_parse_compressed( if (flag < 0x80UL) { *val = flag; return(ptr + 1); + } + + /* Workaround GCC bug + https://gcc.gnu.org/bugzilla/show_bug.cgi?id=77673: + the compiler moves mach_read_from_4 right to the beginning of the + function, causing and out-of-bounds read if we are reading a short + integer close to the end of buffer. */ +#if defined(__GNUC__) && (__GNUC__ >= 5) && !defined(__clang__) +#define DEPLOY_FENCE +#endif + +#ifdef DEPLOY_FENCE + __atomic_thread_fence(__ATOMIC_ACQUIRE); +#endif - } else if (flag < 0xC0UL) { + if (flag < 0xC0UL) { if (end_ptr < ptr + 2) { return(NULL); } @@ -64,8 +78,13 @@ mach_parse_compressed( *val = mach_read_from_2(ptr) & 0x7FFFUL; return(ptr + 2); + } + +#ifdef DEPLOY_FENCE + __atomic_thread_fence(__ATOMIC_ACQUIRE); +#endif - } else if (flag < 0xE0UL) { + if (flag < 0xE0UL) { if (end_ptr < ptr + 3) { return(NULL); } @@ -73,7 +92,13 @@ mach_parse_compressed( *val = mach_read_from_3(ptr) & 0x3FFFFFUL; return(ptr + 3); - } else if (flag < 0xF0UL) { + } + +#ifdef DEPLOY_FENCE + __atomic_thread_fence(__ATOMIC_ACQUIRE); +#endif + + if (flag < 0xF0UL) { if (end_ptr < ptr + 4) { return(NULL); } @@ -81,14 +106,20 @@ mach_parse_compressed( *val = mach_read_from_4(ptr) & 0x1FFFFFFFUL; return(ptr + 4); - } else { - ut_ad(flag == 0xF0UL); + } - if (end_ptr < ptr + 5) { - return(NULL); - } +#ifdef DEPLOY_FENCE + __atomic_thread_fence(__ATOMIC_ACQUIRE); +#endif + +#undef DEPLOY_FENCE + + ut_ad(flag == 0xF0UL); - *val = mach_read_from_4(ptr + 1); - return(ptr + 5); + if (end_ptr < ptr + 5) { + return(NULL); } + + *val = mach_read_from_4(ptr + 1); + return(ptr + 5); } diff --git a/storage/innobase/os/os0thread.cc b/storage/innobase/os/os0thread.cc index a5b0f7de6ae..a6bef176ba4 100644 --- a/storage/innobase/os/os0thread.cc +++ b/storage/innobase/os/os0thread.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1995, 2013, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -191,14 +191,38 @@ os_thread_create_func( #endif } +/** Waits until the specified thread completes and joins it. +Its return value is ignored. +@param[in,out] thread thread to join */ +UNIV_INTERN +void +os_thread_join( + os_thread_t thread) +{ +#ifdef __WIN__ + /* Do nothing. */ +#else +#ifdef UNIV_DEBUG + const int ret = +#endif /* UNIV_DEBUG */ + pthread_join(thread, NULL); + + /* Waiting on already-quit threads is allowed. */ + ut_ad(ret == 0 || ret == ESRCH); +#endif /* __WIN__ */ +} + /*****************************************************************//** Exits the current thread. */ UNIV_INTERN void os_thread_exit( /*===========*/ - void* exit_value) /*!< in: exit value; in Windows this void* + void* exit_value, /*!< in: exit value; in Windows this void* is cast as a DWORD */ + bool detach) /*!< in: if true, the thread will be detached + right before exiting. If false, another thread + is responsible for joining this thread. */ { #ifdef UNIV_DEBUG_THREAD_CREATION fprintf(stderr, "Thread exits, id %lu\n", @@ -216,7 +240,9 @@ os_thread_exit( #ifdef __WIN__ ExitThread((DWORD) exit_value); #else - pthread_detach(pthread_self()); + if (detach) { + pthread_detach(pthread_self()); + } pthread_exit(exit_value); #endif } diff --git a/storage/innobase/row/row0ftsort.cc b/storage/innobase/row/row0ftsort.cc index 1ce99acaef8..e4a88047955 100644 --- a/storage/innobase/row/row0ftsort.cc +++ b/storage/innobase/row/row0ftsort.cc @@ -958,7 +958,7 @@ fts_parallel_merge( CloseHandle(psort_info->thread_hdl); #endif /*__WIN__ */ - os_thread_exit(NULL); + os_thread_exit(NULL, false); OS_THREAD_DUMMY_RETURN; } diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index 9cd6e63e031..9518712e160 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -3824,6 +3824,13 @@ wait_again: " exited when creating FTS" " index '%s'", indexes[i]->name); + } else { + for (j = 0; j < FTS_NUM_AUX_INDEX; + j++) { + + os_thread_join(merge_info[j] + .thread_hdl); + } } } else { /* This cannot report duplicates; an diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 07b841de300..6eb93f9e22d 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -1365,6 +1365,8 @@ run_again: row_ins_step(thr); + DEBUG_SYNC_C("ib_after_row_insert_step"); + err = trx->error_state; if (err != DB_SUCCESS) { diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index e9eeda9acb1..4c58a3e84fb 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -343,6 +343,7 @@ this many index pages, there are 2 ways to calculate statistics: table/index are not found in the innodb database */ UNIV_INTERN unsigned long long srv_stats_transient_sample_pages = 8; UNIV_INTERN my_bool srv_stats_persistent = TRUE; +UNIV_INTERN my_bool srv_stats_include_delete_marked = FALSE; UNIV_INTERN unsigned long long srv_stats_persistent_sample_pages = 20; UNIV_INTERN my_bool srv_stats_auto_recalc = TRUE; diff --git a/storage/perfschema/pfs.cc b/storage/perfschema/pfs.cc index dd61df5f861..ad3faeea618 100644 --- a/storage/perfschema/pfs.cc +++ b/storage/perfschema/pfs.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2008, 2015, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2008, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -2560,10 +2560,7 @@ start_table_io_wait_v1(PSI_table_locker_state *state, if (! pfs_table->m_io_enabled) return NULL; - PFS_thread *pfs_thread= pfs_table->m_thread_owner; - - DBUG_ASSERT(pfs_thread == - my_pthread_getspecific_ptr(PFS_thread*, THR_PFS)); + PFS_thread *pfs_thread= my_pthread_getspecific_ptr(PFS_thread*, THR_PFS); register uint flags; ulonglong timer_start= 0; @@ -2666,7 +2663,7 @@ start_table_lock_wait_v1(PSI_table_locker_state *state, if (! pfs_table->m_lock_enabled) return NULL; - PFS_thread *pfs_thread= pfs_table->m_thread_owner; + PFS_thread *pfs_thread= my_pthread_getspecific_ptr(PFS_thread*, THR_PFS); PFS_TL_LOCK_TYPE lock_type; @@ -3068,7 +3065,12 @@ start_socket_wait_v1(PSI_socket_locker_state *state, if (flag_thread_instrumentation) { - PFS_thread *pfs_thread= pfs_socket->m_thread_owner; + /* + Do not use pfs_socket->m_thread_owner here, + as different threads may use concurrently the same socket, + for example during a KILL. + */ + PFS_thread *pfs_thread= my_pthread_getspecific_ptr(PFS_thread*, THR_PFS); if (unlikely(pfs_thread == NULL)) return NULL; @@ -3436,6 +3438,8 @@ static void end_idle_wait_v1(PSI_idle_locker* locker) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } @@ -3517,6 +3521,8 @@ static void end_mutex_wait_v1(PSI_mutex_locker* locker, int rc) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } } @@ -3596,6 +3602,8 @@ static void end_rwlock_rdwait_v1(PSI_rwlock_locker* locker, int rc) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } } @@ -3668,6 +3676,8 @@ static void end_rwlock_wrwait_v1(PSI_rwlock_locker* locker, int rc) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } } @@ -3732,6 +3742,8 @@ static void end_cond_wait_v1(PSI_cond_locker* locker, int rc) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } } @@ -3826,6 +3838,8 @@ static void end_table_io_wait_v1(PSI_table_locker* locker) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } @@ -3895,6 +3909,8 @@ static void end_table_lock_wait_v1(PSI_table_locker* locker) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } @@ -4143,6 +4159,8 @@ static void end_file_wait_v1(PSI_file_locker *locker, if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } } @@ -5070,6 +5088,8 @@ static void end_socket_wait_v1(PSI_socket_locker *locker, size_t byte_count) if (flag_events_waits_history_long) insert_events_waits_history_long(wait); thread->m_events_waits_current--; + + DBUG_ASSERT(wait == thread->m_events_waits_current); } } diff --git a/storage/perfschema/pfs_digest.cc b/storage/perfschema/pfs_digest.cc index bf42ada8e67..3b4842a67de 100644 --- a/storage/perfschema/pfs_digest.cc +++ b/storage/perfschema/pfs_digest.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2008, 2015, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2008, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -45,7 +45,7 @@ bool flag_statements_digest= true; Current index in Stat array where new record is to be inserted. index 0 is reserved for "all else" case when entire array is full. */ -volatile uint32 digest_index; +volatile uint32 PFS_ALIGNED digest_monotonic_index; bool digest_full= false; LF_HASH digest_hash; @@ -63,7 +63,7 @@ int init_digest(const PFS_global_param *param) */ digest_max= param->m_digest_sizing; digest_lost= 0; - digest_index= 1; + PFS_atomic::store_u32(& digest_monotonic_index, 1); digest_full= false; if (digest_max == 0) @@ -105,6 +105,9 @@ int init_digest(const PFS_global_param *param) + index * pfs_max_digest_length, pfs_max_digest_length); } + /* Set record[0] as allocated. */ + statements_digest_stat_array[0].m_lock.set_allocated(); + return 0; } @@ -207,9 +210,10 @@ find_or_create_digest(PFS_thread *thread, memcpy(hash_key.m_schema_name, schema_name, schema_name_length); int res; - ulong safe_index; uint retry_count= 0; const uint retry_max= 3; + size_t safe_index; + size_t attempts= 0; PFS_statements_digest_stat **entry; PFS_statements_digest_stat *pfs= NULL; @@ -245,55 +249,70 @@ search: return & pfs->m_stat; } - safe_index= PFS_atomic::add_u32(& digest_index, 1); - if (safe_index >= digest_max) + while (++attempts <= digest_max) { - /* The digest array is now full. */ - digest_full= true; - pfs= &statements_digest_stat_array[0]; - - if (pfs->m_first_seen == 0) - pfs->m_first_seen= now; - pfs->m_last_seen= now; - return & pfs->m_stat; - } - - /* Add a new record in digest stat array. */ - pfs= &statements_digest_stat_array[safe_index]; - - /* Copy digest hash/LF Hash search key. */ - memcpy(& pfs->m_digest_key, &hash_key, sizeof(PFS_digest_key)); - - /* - Copy digest storage to statement_digest_stat_array so that it could be - used later to generate digest text. - */ - pfs->m_digest_storage.copy(digest_storage); - - pfs->m_first_seen= now; - pfs->m_last_seen= now; + safe_index= PFS_atomic::add_u32(& digest_monotonic_index, 1) % digest_max; + if (safe_index == 0) + { + /* Record [0] is reserved. */ + safe_index= 1; + } - res= lf_hash_insert(&digest_hash, pins, &pfs); - if (likely(res == 0)) - { - return & pfs->m_stat; - } + /* Add a new record in digest stat array. */ + pfs= &statements_digest_stat_array[safe_index]; - if (res > 0) - { - /* Duplicate insert by another thread */ - if (++retry_count > retry_max) + if (pfs->m_lock.is_free()) { - /* Avoid infinite loops */ - digest_lost++; - return NULL; + if (pfs->m_lock.free_to_dirty()) + { + /* Copy digest hash/LF Hash search key. */ + memcpy(& pfs->m_digest_key, &hash_key, sizeof(PFS_digest_key)); + + /* + Copy digest storage to statement_digest_stat_array so that it could be + used later to generate digest text. + */ + pfs->m_digest_storage.copy(digest_storage); + + pfs->m_first_seen= now; + pfs->m_last_seen= now; + + res= lf_hash_insert(&digest_hash, pins, &pfs); + if (likely(res == 0)) + { + pfs->m_lock.dirty_to_allocated(); + return & pfs->m_stat; + } + + pfs->m_lock.dirty_to_free(); + + if (res > 0) + { + /* Duplicate insert by another thread */ + if (++retry_count > retry_max) + { + /* Avoid infinite loops */ + digest_lost++; + return NULL; + } + goto search; + } + + /* OOM in lf_hash_insert */ + digest_lost++; + return NULL; + } } - goto search; } - /* OOM in lf_hash_insert */ - digest_lost++; - return NULL; + /* The digest array is now full. */ + digest_full= true; + pfs= &statements_digest_stat_array[0]; + + if (pfs->m_first_seen == 0) + pfs->m_first_seen= now; + pfs->m_last_seen= now; + return & pfs->m_stat; } void purge_digest(PFS_thread* thread, PFS_digest_key *hash_key) @@ -320,10 +339,12 @@ void purge_digest(PFS_thread* thread, PFS_digest_key *hash_key) void PFS_statements_digest_stat::reset_data(unsigned char *token_array, uint length) { + m_lock.set_dirty(); m_digest_storage.reset(token_array, length); m_stat.reset(); m_first_seen= 0; m_last_seen= 0; + m_lock.dirty_to_free(); } void PFS_statements_digest_stat::reset_index(PFS_thread *thread) @@ -351,11 +372,14 @@ void reset_esms_by_digest() statements_digest_stat_array[index].reset_data(statements_digest_token_array + index * pfs_max_digest_length, pfs_max_digest_length); } + /* Mark record[0] as allocated again. */ + statements_digest_stat_array[0].m_lock.set_allocated(); + /* Reset index which indicates where the next calculated digest information to be inserted in statements_digest_stat_array. */ - digest_index= 1; + PFS_atomic::store_u32(& digest_monotonic_index, 1); digest_full= false; } diff --git a/storage/perfschema/pfs_digest.h b/storage/perfschema/pfs_digest.h index 76d6c33d984..429a9f4250a 100644 --- a/storage/perfschema/pfs_digest.h +++ b/storage/perfschema/pfs_digest.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2011, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -44,6 +44,9 @@ struct PFS_digest_key /** A statement digest stat record. */ struct PFS_ALIGNED PFS_statements_digest_stat { + /** Internal lock. */ + pfs_lock m_lock; + /** Digest Schema + MD5 Hash. */ PFS_digest_key m_digest_key; diff --git a/storage/perfschema/pfs_lock.h b/storage/perfschema/pfs_lock.h index 09efecd1c5f..be84d0f7fb4 100644 --- a/storage/perfschema/pfs_lock.h +++ b/storage/perfschema/pfs_lock.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2009, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -156,6 +156,18 @@ struct pfs_lock } /** + Initialize a lock to dirty. + */ + void set_dirty(void) + { + /* Do not set the version to 0, read the previous value. */ + uint32 copy= PFS_atomic::load_u32(&m_version_state); + /* Increment the version, set the DIRTY state */ + uint32 new_val= (copy & VERSION_MASK) + VERSION_INC + PFS_LOCK_DIRTY; + PFS_atomic::store_u32(&m_version_state, new_val); + } + + /** Execute a dirty to free transition. This transition should be executed by the writer that owns the record. */ diff --git a/storage/perfschema/table_esms_by_digest.cc b/storage/perfschema/table_esms_by_digest.cc index 80fa4077281..5a3145b91d1 100644 --- a/storage/perfschema/table_esms_by_digest.cc +++ b/storage/perfschema/table_esms_by_digest.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2010, 2012, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2010, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -238,11 +238,14 @@ int table_esms_by_digest::rnd_next(void) m_pos.next()) { digest_stat= &statements_digest_stat_array[m_pos.m_index]; - if (digest_stat->m_first_seen != 0) + if (digest_stat->m_lock.is_populated()) { - make_row(digest_stat); - m_next_pos.set_after(&m_pos); - return 0; + if (digest_stat->m_first_seen != 0) + { + make_row(digest_stat); + m_next_pos.set_after(&m_pos); + return 0; + } } } @@ -260,10 +263,13 @@ table_esms_by_digest::rnd_pos(const void *pos) set_position(pos); digest_stat= &statements_digest_stat_array[m_pos.m_index]; - if (digest_stat->m_first_seen != 0) + if (digest_stat->m_lock.is_populated()) { - make_row(digest_stat); - return 0; + if (digest_stat->m_first_seen != 0) + { + make_row(digest_stat); + return 0; + } } return HA_ERR_RECORD_DELETED; diff --git a/storage/perfschema/unittest/pfs-t.cc b/storage/perfschema/unittest/pfs-t.cc index 6121fac098f..f76b1aa2e75 100644 --- a/storage/perfschema/unittest/pfs-t.cc +++ b/storage/perfschema/unittest/pfs-t.cc @@ -1316,6 +1316,7 @@ void test_locker_disabled() /* Pretend the socket does not have a thread owner */ /* ---------------------------------------------- */ + psi->delete_current_thread(); socket_class_A->m_enabled= true; socket_A1= psi->init_socket(socket_key_A, NULL, NULL, 0); ok(socket_A1 != NULL, "instrumented"); |