From 99ab562a92c05896ffb54216602507c56ea4e101 Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Thu, 1 Oct 2020 19:18:26 +0200 Subject: - Make possible to allocate work space larger than 4GB All variables handling sizes that were uint are now size_t. The variable connect_work_size is now ulong (was uint); Also make Json functiosn to allocate a larger memory (M=9 was 7) modified: storage/connect/global.h modified: storage/connect/ha_connect.cc modified: storage/connect/json.cpp modified: storage/connect/jsonudf.cpp modified: storage/connect/plgdbutl.cpp modified: storage/connect/plugutil.cpp modified: storage/connect/user_connect.cc - Fix uninitialised variable (pretty) in Json_File. Make Jbin_file accept the same arguments as Json_File ones. modified: storage/connect/jsonudf.cpp - Change the Level option to Depth (the word currently used) (Level being still accepted) modified: storage/connect/mongo.cpp modified: storage/connect/tabjson.cpp modified: storage/connect/tabxml.cpp - Suppress 2nd argument default value for MYSQLtoPLG function modified: storage/connect/myutil.h - Allow REST tables to be create not specifying a file_name modified: storage/connect/tabrest.cpp --- storage/connect/global.h | 20 +++++++++---------- storage/connect/ha_connect.cc | 21 ++++++++++++-------- storage/connect/json.cpp | 2 +- storage/connect/jsonudf.cpp | 44 +++++++++++++++++++++++------------------ storage/connect/mongo.cpp | 2 +- storage/connect/myutil.h | 4 ++-- storage/connect/plgdbutl.cpp | 20 +++++++++---------- storage/connect/plugutil.cpp | 24 +++++++++++----------- storage/connect/tabjson.cpp | 4 ++-- storage/connect/tabrest.cpp | 29 +++++++++++++++++++-------- storage/connect/tabxml.cpp | 3 ++- storage/connect/user_connect.cc | 17 ++++++++-------- 12 files changed, 108 insertions(+), 82 deletions(-) (limited to 'storage') diff --git a/storage/connect/global.h b/storage/connect/global.h index fd26c87b800..548d047ccc9 100644 --- a/storage/connect/global.h +++ b/storage/connect/global.h @@ -1,7 +1,7 @@ /***********************************************************************/ /* GLOBAL.H: Declaration file used by all CONNECT implementations. */ /* (C) Copyright MariaDB Corporation Ab */ -/* Author Olivier Bertrand 1993-2018 */ +/* Author Olivier Bertrand 1993-2020 */ /***********************************************************************/ /***********************************************************************/ @@ -112,8 +112,8 @@ extern "C" { /***********************************************************************/ #include "os.h" -typedef uint OFFSET; -typedef char NAME[9]; +typedef size_t OFFSET; +typedef char NAME[9]; typedef struct { ushort Length; @@ -136,7 +136,7 @@ typedef struct _parm *PPARM; /***********************************************************************/ typedef struct { /* Plug Area SubAlloc header */ OFFSET To_Free; /* Offset of next free block */ - uint FreeBlk; /* Size of remaining free memory */ + size_t FreeBlk; /* Size of remaining free memory */ } POOLHEADER, *PPOOLHEADER; /***********************************************************************/ @@ -188,7 +188,7 @@ typedef struct _parm { /***********************************************************************/ typedef struct _global { /* Global structure */ void *Sarea; /* Points to work area */ - uint Sarea_Size; /* Work area size */ + size_t Sarea_Size; /* Work area size */ PACTIVITY Activityp; char Message[MAX_STR]; ulong More; /* Used by jsonudf */ @@ -210,16 +210,16 @@ DllExport char *PlugReadMessage(PGLOBAL, int, char *); DllExport char *PlugGetMessage(PGLOBAL, int); #endif // XMSG || NEWMSG #if defined(__WIN__) -DllExport short GetLineLength(PGLOBAL); // Console line length +DllExport short GetLineLength(PGLOBAL); // Console line length #endif // __WIN__ -DllExport PGLOBAL PlugInit(LPCSTR, uint); // Plug global initialization -DllExport int PlugExit(PGLOBAL); // Plug global termination +DllExport PGLOBAL PlugInit(LPCSTR, size_t); // Plug global initialization +DllExport int PlugExit(PGLOBAL); // Plug global termination DllExport LPSTR PlugRemoveType(LPSTR, LPCSTR); DllExport LPCSTR PlugSetPath(LPSTR to, LPCSTR prefix, LPCSTR name, LPCSTR dir); DllExport BOOL PlugIsAbsolutePath(LPCSTR path); -DllExport bool AllocSarea(PGLOBAL, uint); +DllExport bool AllocSarea(PGLOBAL, size_t); DllExport void FreeSarea(PGLOBAL); -DllExport BOOL PlugSubSet(void *, uint); +DllExport BOOL PlugSubSet(void *, size_t); DllExport void *PlugSubAlloc(PGLOBAL, void *, size_t); DllExport char *PlugDup(PGLOBAL g, const char *str); DllExport void *MakePtr(void *, OFFSET); diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index 7d53f287f74..e39b5cebaf4 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -170,7 +170,7 @@ #define JSONMAX 10 // JSON Default max grp size extern "C" { - char version[]= "Version 1.07.0001 November 12, 2019"; + char version[]= "Version 1.07.0001 September 30, 2020"; #if defined(__WIN__) char compver[]= "Version 1.07.0001 " __DATE__ " " __TIME__; char slash= '\\'; @@ -254,8 +254,8 @@ TYPCONV GetTypeConv(void); char *GetJsonNull(void); uint GetJsonGrpSize(void); char *GetJavaWrapper(void); -uint GetWorkSize(void); -void SetWorkSize(uint); +ulong GetWorkSize(void); +void SetWorkSize(ulong); extern "C" const char *msglang(void); static char *strz(PGLOBAL g, LEX_STRING &ls); @@ -348,10 +348,10 @@ static MYSQL_THDVAR_ENUM( &usetemp_typelib); // typelib // Size used for g->Sarea_Size -static MYSQL_THDVAR_UINT(work_size, +static MYSQL_THDVAR_ULONG(work_size, PLUGIN_VAR_RQCMDARG, "Size of the CONNECT work area.", - NULL, NULL, SZWORK, SZWMIN, UINT_MAX, 1); + NULL, NULL, SZWORK, SZWMIN, ULONG_MAX, 1); // Size used when converting TEXT columns to VARCHAR static MYSQL_THDVAR_INT(conv_size, @@ -461,8 +461,8 @@ char *GetJsonNull(void) {return connect_hton ? THDVAR(current_thd, json_null) : NULL;} uint GetJsonGrpSize(void) {return connect_hton ? THDVAR(current_thd, json_grp_size) : 10;} -uint GetWorkSize(void) {return THDVAR(current_thd, work_size);} -void SetWorkSize(uint) +ulong GetWorkSize(void) {return THDVAR(current_thd, work_size);} +void SetWorkSize(ulong) { // Changing the session variable value seems to be impossible here // and should be done in a check function @@ -5326,7 +5326,12 @@ static bool add_field(String *sql, const char *field_name, int typ, int len, int dec, char *key, uint tm, const char *rem, char *dft, char *xtra, char *fmt, int flag, bool dbf, char v) { - char var= (len > 255) ? 'V' : v; +#if defined(DEVELOPMENT) + // Some client programs regard CHAR(36) as GUID + char var= (len > 255 || len == 36) ? 'V' : v; +#else + char var = (len > 255) ? 'V' : v; +#endif bool q, error= false; const char *type= PLGtoMYSQLtype(typ, dbf, var); diff --git a/storage/connect/json.cpp b/storage/connect/json.cpp index 98a4659cea8..a38e6487025 100644 --- a/storage/connect/json.cpp +++ b/storage/connect/json.cpp @@ -425,7 +425,7 @@ char *ParseString(PGLOBAL g, int& i, STRG& src) int n = 0, len = src.len; // Be sure of memory availability - if (len + 1 - i > (signed)((PPOOLHEADER)g->Sarea)->FreeBlk) { + if ((unsigned)(len + 1 - i) > ((PPOOLHEADER)g->Sarea)->FreeBlk) { strcpy(g->Message, "ParseString: Out of memory"); return NULL; } // endif len diff --git a/storage/connect/jsonudf.cpp b/storage/connect/jsonudf.cpp index dad86d51040..a1c6a77d1f2 100644 --- a/storage/connect/jsonudf.cpp +++ b/storage/connect/jsonudf.cpp @@ -25,7 +25,7 @@ #else #define PUSH_WARNING(M) htrc(M) #endif -#define M 7 +#define M 9 bool IsNum(PSZ s); char *NextChr(PSZ s, char sep); @@ -4423,13 +4423,15 @@ char *json_file(UDF_INIT *initid, UDF_ARGS *args, char *result, fn = MakePSZ(g, args, 0); if (args->arg_count > 1) { - int len, pretty, pty = 3; + int len, pretty = 3, pty = 3; PJSON jsp; PJVAL jvp = NULL; - pretty = (args->arg_type[1] == INT_RESULT) ? (int)*(longlong*)args->args[1] - : (args->arg_count > 2 && args->arg_type[2] == INT_RESULT) - ? (int)*(longlong*)args->args[2] : 3; + for (unsigned int i = 1; i < args->arg_count; i++) + if (args->arg_type[i] == INT_RESULT && *(longlong*)args->args[i] < 4) { + pretty = (int) * (longlong*)args->args[i]; + break; + } // endif type /*******************************************************************************/ /* Parse the json file and allocate its tree structure. */ @@ -5626,20 +5628,19 @@ my_bool jbin_file_init(UDF_INIT *initid, UDF_ARGS *args, char *message) } else if (args->arg_type[0] != STRING_RESULT || !args->args[0]) { strcpy(message, "First argument must be a constant string (file name)"); return true; - } else if (args->arg_count > 1 && args->arg_type[1] != STRING_RESULT) { - strcpy(message, "Second argument is not a string (path)"); - return true; - } else if (args->arg_count > 2 && args->arg_type[2] != INT_RESULT) { - strcpy(message, "Third argument is not an integer (pretty)"); - return true; - } else if (args->arg_count > 3) { - if (args->arg_type[3] != INT_RESULT) { - strcpy(message, "Fourth argument is not an integer (memory)"); + } // endifs + + for (unsigned int i = 1; i < args->arg_count; i++) { + if (!(args->arg_type[i] == INT_RESULT || args->arg_type[i] == STRING_RESULT)) { + sprintf(message, "Argument %d is not an integer or a string (pretty or path)", i); return true; - } else - more += (ulong)*(longlong*)args->args[3]; + } // endif arg_type - } // endifs + // Take care of eventual memory argument + if (args->arg_type[i] == INT_RESULT && args->args[i]) + more += (ulong) * (longlong*)args->args[i]; + + } // endfor i initid->maybe_null = 1; CalcLen(args, false, reslen, memlen); @@ -5654,7 +5655,7 @@ char *jbin_file(UDF_INIT *initid, UDF_ARGS *args, char *result, unsigned long *res_length, char *is_null, char *error) { char *fn; - int pretty, len = 0, pty = 3; + int pretty = 3, len = 0, pty = 3; PJSON jsp; PJVAL jvp = NULL; PGLOBAL g = (PGLOBAL)initid->ptr; @@ -5666,7 +5667,12 @@ char *jbin_file(UDF_INIT *initid, UDF_ARGS *args, char *result, PlugSubSet(g->Sarea, g->Sarea_Size); g->Xchk = NULL; fn = MakePSZ(g, args, 0); - pretty = (args->arg_count > 2 && args->args[2]) ? (int)*(longlong*)args->args[2] : 3; + + for (unsigned int i = 1; i < args->arg_count; i++) + if (args->arg_type[i] == INT_RESULT && *(longlong*)args->args[i] < 4) { + pretty = (int) * (longlong*)args->args[i]; + break; + } // endif type /*********************************************************************************/ /* Parse the json file and allocate its tree structure. */ diff --git a/storage/connect/mongo.cpp b/storage/connect/mongo.cpp index bd3d3b893c1..e821440a0c3 100644 --- a/storage/connect/mongo.cpp +++ b/storage/connect/mongo.cpp @@ -251,7 +251,7 @@ int MGODISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ uri, PTOS topt) PCSZ level = GetStringTableOption(g, topt, "Level", NULL); PMGODEF tdp; - if (level) { + if (level = GetStringTableOption(g, topt, "Depth", level)) { lvl = atoi(level); lvl = (lvl > 16) ? 16 : lvl; } else diff --git a/storage/connect/myutil.h b/storage/connect/myutil.h index 6991172b39e..fa41fa47d61 100644 --- a/storage/connect/myutil.h +++ b/storage/connect/myutil.h @@ -6,8 +6,8 @@ enum enum_field_types PLGtoMYSQL(int type, bool dbf, char var = 0); const char *PLGtoMYSQLtype(int type, bool dbf, char var = 0); -int MYSQLtoPLG(char *typname, char *var = NULL); -int MYSQLtoPLG(int mytype, char *var = NULL); +int MYSQLtoPLG(char *typname, char *var); +int MYSQLtoPLG(int mytype, char *var); PCSZ MyDateFmt(int mytype); PCSZ MyDateFmt(char *typname); diff --git a/storage/connect/plgdbutl.cpp b/storage/connect/plgdbutl.cpp index e296553d8e2..7bc7c5f592b 100644 --- a/storage/connect/plgdbutl.cpp +++ b/storage/connect/plgdbutl.cpp @@ -5,7 +5,7 @@ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 1998-2018 */ +/* (C) Copyright to the author Olivier BERTRAND 1998-2020 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -1242,7 +1242,7 @@ void *PlgDBalloc(PGLOBAL g, void *area, MBLOCK& mp) mp.Sub = mp.Size <= ((mp.Sub) ? maxsub : (maxsub >> 2)); if (trace(2)) - htrc("PlgDBalloc: in %p size=%d used=%d free=%d sub=%d\n", + htrc("PlgDBalloc: in %p size=%zd used=%zd free=%zd sub=%d\n", arp, mp.Size, pph->To_Free, pph->FreeBlk, mp.Sub); if (!mp.Sub) { @@ -1258,7 +1258,7 @@ void *PlgDBalloc(PGLOBAL g, void *area, MBLOCK& mp) mp.Memp = malloc(mp.Size); if (trace(8)) - htrc("PlgDBalloc: %s(%d) at %p\n", v, mp.Size, mp.Memp); + htrc("PlgDBalloc: %s(%zd) at %p\n", v, mp.Size, mp.Memp); if (!mp.Inlist && mp.Memp) { // New allocated block, put it in the memory block chain. @@ -1290,7 +1290,7 @@ void *PlgDBrealloc(PGLOBAL g, void *area, MBLOCK& mp, size_t newsize) #endif if (trace(2)) - htrc("PlgDBrealloc: %p size=%d sub=%d\n", mp.Memp, mp.Size, mp.Sub); + htrc("PlgDBrealloc: %p size=%zd sub=%d\n", mp.Memp, mp.Size, mp.Sub); if (newsize == mp.Size) return mp.Memp; // Nothing to do @@ -1340,7 +1340,7 @@ void *PlgDBrealloc(PGLOBAL g, void *area, MBLOCK& mp, size_t newsize) } // endif's if (trace(8)) - htrc(" newsize=%d newp=%p sub=%d\n", mp.Size, mp.Memp, mp.Sub); + htrc(" newsize=%zd newp=%p sub=%d\n", mp.Size, mp.Memp, mp.Sub); return mp.Memp; } // end of PlgDBrealloc @@ -1392,13 +1392,13 @@ void *PlgDBSubAlloc(PGLOBAL g, void *memp, size_t size) pph = (PPOOLHEADER)memp; if (trace(16)) - htrc("PlgDBSubAlloc: memp=%p size=%d used=%d free=%d\n", + htrc("PlgDBSubAlloc: memp=%p size=%zd used=%zd free=%zd\n", memp, size, pph->To_Free, pph->FreeBlk); - if ((uint)size > pph->FreeBlk) { /* Not enough memory left in pool */ + if (size > pph->FreeBlk) { /* Not enough memory left in pool */ sprintf(g->Message, - "Not enough memory in Work area for request of %d (used=%d free=%d)", - (int) size, pph->To_Free, pph->FreeBlk); + "Not enough memory in Work area for request of %zd (used=%zd free=%zd)", + size, pph->To_Free, pph->FreeBlk); if (trace(1)) htrc("%s\n", g->Message); @@ -1414,7 +1414,7 @@ void *PlgDBSubAlloc(PGLOBAL g, void *memp, size_t size) pph->FreeBlk -= size; // New size of pool free block if (trace(16)) - htrc("Done memp=%p used=%d free=%d\n", + htrc("Done memp=%p used=%zd free=%zd\n", memp, pph->To_Free, pph->FreeBlk); return (memp); diff --git a/storage/connect/plugutil.cpp b/storage/connect/plugutil.cpp index e74937b942a..2517b76cd3e 100644 --- a/storage/connect/plugutil.cpp +++ b/storage/connect/plugutil.cpp @@ -6,7 +6,7 @@ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 1993-2019 */ +/* (C) Copyright to the author Olivier BERTRAND 1993-2020 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -142,7 +142,7 @@ void htrc(char const* fmt, ...) /* Language points on initial language name and eventual path. */ /* Return value is the pointer to the Global structure. */ /***********************************************************************/ -PGLOBAL PlugInit(LPCSTR Language, uint worksize) +PGLOBAL PlugInit(LPCSTR Language, size_t worksize) { PGLOBAL g; @@ -459,7 +459,7 @@ short GetLineLength(PGLOBAL g) /***********************************************************************/ /* Program for memory allocation of work and language areas. */ /***********************************************************************/ -bool AllocSarea(PGLOBAL g, uint size) +bool AllocSarea(PGLOBAL g, size_t size) { /*********************************************************************/ /* This is the allocation routine for the WIN32/UNIX/AIX version. */ @@ -483,7 +483,7 @@ bool AllocSarea(PGLOBAL g, uint size) if (trace(8)) { #endif if (g->Sarea) - htrc("Work area of %u allocated at %p\n", size, g->Sarea); + htrc("Work area of %zd allocated at %p\n", size, g->Sarea); else htrc("SareaAlloc: %s\n", g->Message); @@ -510,7 +510,7 @@ void FreeSarea(PGLOBAL g) #else if (trace(8)) #endif - htrc("Freeing Sarea at %p size = %d\n", g->Sarea, g->Sarea_Size); + htrc("Freeing Sarea at %p size = %zd\n", g->Sarea, g->Sarea_Size); g->Sarea = NULL; g->Sarea_Size = 0; @@ -524,7 +524,7 @@ void FreeSarea(PGLOBAL g) /* Here there should be some verification done such as validity of */ /* the address and size not larger than memory size. */ /***********************************************************************/ -BOOL PlugSubSet(void *memp, uint size) +BOOL PlugSubSet(void *memp, size_t size) { PPOOLHEADER pph = (PPOOLHEADER)memp; @@ -560,15 +560,15 @@ void *PlugSubAlloc(PGLOBAL g, void *memp, size_t size) pph = (PPOOLHEADER)memp; if (trace(16)) - htrc("SubAlloc in %p size=%d used=%d free=%d\n", + htrc("SubAlloc in %p size=%zd used=%zd free=%zd\n", memp, size, pph->To_Free, pph->FreeBlk); - if ((uint)size > pph->FreeBlk) { /* Not enough memory left in pool */ + if (size > pph->FreeBlk) { /* Not enough memory left in pool */ PCSZ pname = "Work"; sprintf(g->Message, - "Not enough memory in %s area for request of %u (used=%d free=%d)", - pname, (uint)size, pph->To_Free, pph->FreeBlk); + "Not enough memory in %s area for request of %zd (used=%zd free=%zd)", + pname, size, pph->To_Free, pph->FreeBlk); if (trace(1)) htrc("PlugSubAlloc: %s\n", g->Message); @@ -581,10 +581,10 @@ void *PlugSubAlloc(PGLOBAL g, void *memp, size_t size) /*********************************************************************/ memp = MakePtr(memp, pph->To_Free); /* Points to suballocated block */ pph->To_Free += (OFFSET)size; /* New offset of pool free block */ - pph->FreeBlk -= (uint)size; /* New size of pool free block */ + pph->FreeBlk -= size; /* New size of pool free block */ if (trace(16)) - htrc("Done memp=%p used=%d free=%d\n", + htrc("Done memp=%p used=%zd free=%zd\n", memp, pph->To_Free, pph->FreeBlk); return (memp); diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp index 3b0d458a7a6..08040884449 100644 --- a/storage/connect/tabjson.cpp +++ b/storage/connect/tabjson.cpp @@ -177,8 +177,8 @@ int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt) bool mgo = (GetTypeID(topt->type) == TAB_MONGO); PCSZ level = GetStringTableOption(g, topt, "Level", NULL); - if (level) { - lvl = atoi(level); + if (level = GetStringTableOption(g, topt, "Depth", level)) { + lvl = atoi(level); lvl = (lvl > 16) ? 16 : lvl; } else lvl = 0; diff --git a/storage/connect/tabrest.cpp b/storage/connect/tabrest.cpp index 9c6b724973f..b1bdeffc880 100644 --- a/storage/connect/tabrest.cpp +++ b/storage/connect/tabrest.cpp @@ -158,17 +158,32 @@ PQRYRES __stdcall ColREST(PGLOBAL g, PTOS tp, char *tab, char *db, bool info) http = GetStringTableOption(g, tp, "Http", NULL); uri = GetStringTableOption(g, tp, "Uri", NULL); - fn = GetStringTableOption(g, tp, "Filename", "rest.json"); #if defined(MARIADB) ftype = GetStringTableOption(g, tp, "Type", "JSON"); #else // !MARIADB // OEM tables must specify the file type ftype = GetStringTableOption(g, tp, "Ftype", "JSON"); #endif // !MARIADB + fn = GetStringTableOption(g, tp, "Filename", NULL); + + if (!fn) { + int n, m = strlen(ftype) + 1; + + strcat(strcpy(filename, tab), "."); + n = strlen(filename); + + // Fold ftype to lower case + for (int i = 0; i < m; i++) + filename[n + i] = tolower(ftype[i]); + + fn = filename; + tp->filename = PlugDup(g, fn); + } // endif fn // We used the file name relative to recorded datapath - strcat(strcat(strcat(strcpy(filename, "."), slash), db), slash); - strncat(filename, fn, _MAX_PATH - strlen(filename)); + PlugSetPath(filename, fn, db); + //strcat(strcat(strcat(strcpy(filename, "."), slash), db), slash); + //strncat(filename, fn, _MAX_PATH - strlen(filename)); // Retrieve the file from the web and copy it locally if (http && grf(g->Message, trace(515), http, uri, filename)) { @@ -227,12 +242,10 @@ bool RESTDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) Http = GetStringCatInfo(g, "Http", NULL); Uri = GetStringCatInfo(g, "Uri", NULL); - Fn = GetStringCatInfo(g, "Filename", "rest.json"); + Fn = GetStringCatInfo(g, "Filename", NULL); // We used the file name relative to recorded datapath - //PlugSetPath(filename, Fn, GetPath()); - strcpy(filename, GetPath()); - strncat(filename, Fn, _MAX_PATH - strlen(filename)); + PlugSetPath(filename, Fn, GetPath()); // Retrieve the file from the web and copy it locally rc = grf(g->Message, xt, Http, Uri, filename); @@ -270,7 +283,7 @@ PTDB RESTDEF::GetTable(PGLOBAL g, MODE m) if (trace(515)) htrc("REST GetTable mode=%d\n", m); - if (m != MODE_READ && m != MODE_READX) { + if (m != MODE_READ && m != MODE_READX && m != MODE_ANY) { strcpy(g->Message, "REST tables are currently read only"); return NULL; } // endif m diff --git a/storage/connect/tabxml.cpp b/storage/connect/tabxml.cpp index 68941c67be8..fa140ea0699 100644 --- a/storage/connect/tabxml.cpp +++ b/storage/connect/tabxml.cpp @@ -150,7 +150,8 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info) return NULL; } else { lvl = GetIntegerTableOption(g, topt, "Level", 0); - lvl = (lvl < 0) ? 0 : (lvl > 16) ? 16 : lvl; + lvl = GetIntegerTableOption(g, topt, "Depth", lvl); + lvl = (lvl < 0) ? 0 : (lvl > 16) ? 16 : lvl; } // endif fn if (trace(1)) diff --git a/storage/connect/user_connect.cc b/storage/connect/user_connect.cc index c25443ef7ef..c061294c57e 100644 --- a/storage/connect/user_connect.cc +++ b/storage/connect/user_connect.cc @@ -28,7 +28,7 @@ */ /****************************************************************************/ -/* Author: Olivier Bertrand -- bertrandop@gmail.com -- 2004-2015 */ +/* Author: Olivier Bertrand -- bertrandop@gmail.com -- 2004-2020 */ /****************************************************************************/ #ifdef USE_PRAGMA_IMPLEMENTATION #pragma implementation // gcc: Class implementation @@ -58,8 +58,8 @@ PCONNECT user_connect::to_users= NULL; /****************************************************************************/ /* Get the work_size SESSION variable value . */ /****************************************************************************/ -uint GetWorkSize(void); -void SetWorkSize(uint); +ulong GetWorkSize(void); +void SetWorkSize(ulong); /* -------------------------- class user_connect -------------------------- */ @@ -97,14 +97,14 @@ user_connect::~user_connect() bool user_connect::user_init() { // Initialize Plug-like environment - uint worksize= GetWorkSize(); + ulong worksize= GetWorkSize(); PACTIVITY ap= NULL; PDBUSER dup= NULL; // Areasize= 64M because of VEC tables. Should be parameterisable //g= PlugInit(NULL, 67108864); //g= PlugInit(NULL, 134217728); // 128M was because of old embedded tests - g= PlugInit(NULL, worksize); + g= PlugInit(NULL, (size_t)worksize); // Check whether the initialization is complete if (!g || !g->Sarea || PlugSubSet(g->Sarea, g->Sarea_Size) @@ -157,15 +157,16 @@ void user_connect::SetHandler(ha_connect *hc) bool user_connect::CheckCleanup(bool force) { if (thdp->query_id > last_query_id || force) { - uint worksize= GetWorkSize(), size = g->Sarea_Size; + ulong worksize = GetWorkSize(); + size_t size = g->Sarea_Size; PlugCleanup(g, true); - if (size != worksize) { + if (size != (size_t)worksize) { FreeSarea(g); // Check whether the work area could be allocated - if (AllocSarea(g, worksize)) { + if (AllocSarea(g, (size_t)worksize)) { AllocSarea(g, size); SetWorkSize(g->Sarea_Size); // Was too big } // endif sarea -- cgit v1.2.1 From c6eb127ca88eb998fc3a7be122845d869a8d3ffc Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Sat, 3 Oct 2020 19:06:05 +0200 Subject: - Fix allocating work space larger than 4GB The variable connect_work_size is now ulong or ulonglong for 64bit machines. modified: storage/connect/ha_connect.cc modified: storage/connect/user_connect.cc --- storage/connect/ha_connect.cc | 31 ++++++++++++++++++++----------- storage/connect/user_connect.cc | 12 ++++++------ 2 files changed, 26 insertions(+), 17 deletions(-) (limited to 'storage') diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index e39b5cebaf4..cf2a86185ef 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -170,9 +170,9 @@ #define JSONMAX 10 // JSON Default max grp size extern "C" { - char version[]= "Version 1.07.0001 September 30, 2020"; + char version[]= "Version 1.07.0002 October 03, 2020"; #if defined(__WIN__) - char compver[]= "Version 1.07.0001 " __DATE__ " " __TIME__; + char compver[]= "Version 1.07.0002 " __DATE__ " " __TIME__; char slash= '\\'; #else // !__WIN__ char slash= '/'; @@ -254,8 +254,8 @@ TYPCONV GetTypeConv(void); char *GetJsonNull(void); uint GetJsonGrpSize(void); char *GetJavaWrapper(void); -ulong GetWorkSize(void); -void SetWorkSize(ulong); +size_t GetWorkSize(void); +void SetWorkSize(size_t); extern "C" const char *msglang(void); static char *strz(PGLOBAL g, LEX_STRING &ls); @@ -347,11 +347,19 @@ static MYSQL_THDVAR_ENUM( 1, // def (AUTO) &usetemp_typelib); // typelib +#ifdef _WIN64 +// Size used for g->Sarea_Size +static MYSQL_THDVAR_ULONGLONG(work_size, + PLUGIN_VAR_RQCMDARG, + "Size of the CONNECT work area.", + NULL, NULL, SZWORK, SZWMIN, ULONGLONG_MAX, 1); +#else // Size used for g->Sarea_Size static MYSQL_THDVAR_ULONG(work_size, - PLUGIN_VAR_RQCMDARG, - "Size of the CONNECT work area.", - NULL, NULL, SZWORK, SZWMIN, ULONG_MAX, 1); + PLUGIN_VAR_RQCMDARG, + "Size of the CONNECT work area.", + NULL, NULL, SZWORK, SZWMIN, ULONG_MAX, 1); +#endif // Size used when converting TEXT columns to VARCHAR static MYSQL_THDVAR_INT(conv_size, @@ -461,8 +469,8 @@ char *GetJsonNull(void) {return connect_hton ? THDVAR(current_thd, json_null) : NULL;} uint GetJsonGrpSize(void) {return connect_hton ? THDVAR(current_thd, json_grp_size) : 10;} -ulong GetWorkSize(void) {return THDVAR(current_thd, work_size);} -void SetWorkSize(ulong) +size_t GetWorkSize(void) {return (size_t)THDVAR(current_thd, work_size);} +void SetWorkSize(size_t) { // Changing the session variable value seems to be impossible here // and should be done in a check function @@ -472,7 +480,8 @@ void SetWorkSize(ulong) #if defined(JAVA_SUPPORT) char *GetJavaWrapper(void) -{return connect_hton ? THDVAR(current_thd, java_wrapper) : (char*)"wrappers/JdbcInterface";} +{return connect_hton ? THDVAR(current_thd, java_wrapper) + : (char*)"wrappers/JdbcInterface";} #endif // JAVA_SUPPORT #if defined(JAVA_SUPPORT) || defined(CMGO_SUPPORT) @@ -7376,7 +7385,7 @@ maria_declare_plugin(connect) 0x0107, /* version number (1.07) */ NULL, /* status variables */ connect_system_variables, /* system variables */ - "1.07.0001", /* string version */ + "1.07.0002", /* string version */ MariaDB_PLUGIN_MATURITY_STABLE /* maturity */ } maria_declare_plugin_end; diff --git a/storage/connect/user_connect.cc b/storage/connect/user_connect.cc index c061294c57e..5dbdb8f56f0 100644 --- a/storage/connect/user_connect.cc +++ b/storage/connect/user_connect.cc @@ -58,8 +58,8 @@ PCONNECT user_connect::to_users= NULL; /****************************************************************************/ /* Get the work_size SESSION variable value . */ /****************************************************************************/ -ulong GetWorkSize(void); -void SetWorkSize(ulong); +size_t GetWorkSize(void); +void SetWorkSize(size_t); /* -------------------------- class user_connect -------------------------- */ @@ -97,7 +97,7 @@ user_connect::~user_connect() bool user_connect::user_init() { // Initialize Plug-like environment - ulong worksize= GetWorkSize(); + size_t worksize= GetWorkSize(); PACTIVITY ap= NULL; PDBUSER dup= NULL; @@ -157,16 +157,16 @@ void user_connect::SetHandler(ha_connect *hc) bool user_connect::CheckCleanup(bool force) { if (thdp->query_id > last_query_id || force) { - ulong worksize = GetWorkSize(); + size_t worksize = GetWorkSize(); size_t size = g->Sarea_Size; PlugCleanup(g, true); - if (size != (size_t)worksize) { + if (size != worksize) { FreeSarea(g); // Check whether the work area could be allocated - if (AllocSarea(g, (size_t)worksize)) { + if (AllocSarea(g, worksize)) { AllocSarea(g, size); SetWorkSize(g->Sarea_Size); // Was too big } // endif sarea -- cgit v1.2.1 From 307258c8eec73522805345178aad906aa86a61af Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Mon, 5 Oct 2020 12:29:51 +0200 Subject: - Use BIN type when charset='binary' modified: storage/connect/ha_connect.cc - Allow JSON columns to be "binary" By setting their type as VARBINAY(132) and their name begin with Jbin_ modified: storage/connect/json.h modified: storage/connect/jsonudf.cpp modified: storage/connect/tabjson.cpp modified: storage/connect/value.cpp modified: storage/connect/value.h - CHARSET BINARY cannot be used for text columns modified: storage/connect/mysql-test/connect/r/updelx.result modified: storage/connect/mysql-test/connect/t/updelx.test --- storage/connect/ha_connect.cc | 9 +++++--- storage/connect/json.h | 24 +++++++++++++++++++++- storage/connect/jsonudf.cpp | 10 +++++---- storage/connect/mysql-test/connect/r/updelx.result | 4 ++-- storage/connect/mysql-test/connect/t/updelx.test | 4 ++-- storage/connect/tabjson.cpp | 13 +++++++++++- storage/connect/value.cpp | 9 ++++++++ storage/connect/value.h | 7 ++++--- 8 files changed, 64 insertions(+), 16 deletions(-) (limited to 'storage') diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index cf2a86185ef..fd00b4878a9 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -1586,6 +1586,9 @@ void *ha_connect::GetColumnOption(PGLOBAL g, void *field, PCOLINFO pcf) chset= (char *)fp->charset()->name; + if (!strcmp(chset, "binary")) + v = 'B'; // Binary string + switch (fp->type()) { case MYSQL_TYPE_BLOB: case MYSQL_TYPE_VARCHAR: @@ -1595,7 +1598,7 @@ void *ha_connect::GetColumnOption(PGLOBAL g, void *field, PCOLINFO pcf) default: pcf->Type= MYSQLtoPLG(fp->type(), &v); break; - } // endswitch SQL type + } // endswitch SQL type switch (pcf->Type) { case TYPE_STRING: @@ -1649,7 +1652,7 @@ void *ha_connect::GetColumnOption(PGLOBAL g, void *field, PCOLINFO pcf) break; default: break; - } // endswitch type + } // endswitch type if (fp->flags & UNSIGNED_FLAG) pcf->Flags |= U_UNSIGNED; @@ -2222,7 +2225,7 @@ int ha_connect::MakeRecord(char *buf) case TYPE_BIN: p= value->GetCharValue(); charset= &my_charset_bin; - rc= fp->store(p, strlen(p), charset, CHECK_FIELD_WARN); + rc= fp->store(p, value->GetSize(), charset, CHECK_FIELD_WARN); break; case TYPE_DOUBLE: rc= fp->store(value->GetFloatValue()); diff --git a/storage/connect/json.h b/storage/connect/json.h index 1d058ad575f..d949f244e21 100644 --- a/storage/connect/json.h +++ b/storage/connect/json.h @@ -1,10 +1,11 @@ /**************** json H Declares Source Code File (.H) ****************/ /* Name: json.h Version 1.2 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 2014 - 2017 */ +/* (C) Copyright to the author Olivier BERTRAND 2014 - 2020 */ /* */ /* This file contains the JSON classes declares. */ /***********************************************************************/ +#include #include "value.h" #if defined(_DEBUG) @@ -44,6 +45,27 @@ typedef struct { int len; } STRG, *PSG; +// BSON size should be equal on Linux and Windows +#define BMX 255 +typedef struct BSON* PBSON; + +/***********************************************************************/ +/* Structure used to return binary json to Json UDF functions. */ +/***********************************************************************/ +struct BSON { + char Msg[BMX + 1]; + char *Filename; + PGLOBAL G; + int Pretty; + ulong Reslen; + my_bool Changed; + PJSON Top; + PJSON Jsp; + PBSON Bsp; +}; // end of struct BSON + +PBSON JbinAlloc(PGLOBAL g, UDF_ARGS* args, ulong len, PJSON jsp); + char *NextChr(PSZ s, char sep); char *GetJsonNull(void); diff --git a/storage/connect/jsonudf.cpp b/storage/connect/jsonudf.cpp index a1c6a77d1f2..0a862d2917d 100644 --- a/storage/connect/jsonudf.cpp +++ b/storage/connect/jsonudf.cpp @@ -1076,6 +1076,7 @@ my_bool JSNX::AddPath(void) /* --------------------------------- JSON UDF ---------------------------------- */ +#if 0 // Moved to json.h // BSON size should be equal on Linux and Windows #define BMX 255 typedef struct BSON *PBSON; @@ -1094,11 +1095,12 @@ struct BSON { PJSON Jsp; PBSON Bsp; }; // end of struct BSON +#endif // 0 /*********************************************************************************/ /* Allocate and initialize a BSON structure. */ /*********************************************************************************/ -static PBSON JbinAlloc(PGLOBAL g, UDF_ARGS *args, ulong len, PJSON jsp) +PBSON JbinAlloc(PGLOBAL g, UDF_ARGS *args, ulong len, PJSON jsp) { PBSON bsp = (PBSON)PlgDBSubAlloc(g, NULL, sizeof(BSON)); @@ -1111,7 +1113,7 @@ static PBSON JbinAlloc(PGLOBAL g, UDF_ARGS *args, ulong len, PJSON jsp) bsp->Reslen = len; bsp->Changed = false; bsp->Top = bsp->Jsp = jsp; - bsp->Bsp = (IsJson(args, 0) == 3) ? (PBSON)args->args[0] : NULL; + bsp->Bsp = (args && IsJson(args, 0) == 3) ? (PBSON)args->args[0] : NULL; } else PUSH_WARNING(g->Message); @@ -1422,7 +1424,7 @@ static int IsJson(UDF_ARGS *args, uint i, bool b) n = 2; // arg is a json file name } else if (b) { char *sap; - PGLOBAL g = PlugInit(NULL, args->lengths[i] * M + 1024); + PGLOBAL g = PlugInit(NULL, (size_t)args->lengths[i] * M + 1024); JsonSubSet(g); sap = MakePSZ(g, args, i); @@ -5763,7 +5765,7 @@ char *json_serialize(UDF_INIT *initid, UDF_ARGS *args, char *result, // Keep result of constant function g->Xchk = (initid->const_item) ? str : NULL; } else { - *error = 1; + // *error = 1; str = strcpy(result, "Argument is not a Jbin tree"); } // endif diff --git a/storage/connect/mysql-test/connect/r/updelx.result b/storage/connect/mysql-test/connect/r/updelx.result index 2aed1e06928..bb82afcc1a8 100644 --- a/storage/connect/mysql-test/connect/r/updelx.result +++ b/storage/connect/mysql-test/connect/r/updelx.result @@ -978,7 +978,7 @@ DROP TABLE t1; # FIX table CREATE TABLE t1 ( id INT(4) KEY NOT NULL, -msg VARCHAR(16) CHARSET BINARY DISTRIB=CLUSTERED) +msg VARCHAR(16) DISTRIB=CLUSTERED) ENGINE=CONNECT TABLE_TYPE=FIX BLOCK_SIZE=4; Warnings: Warning 1105 No file name. Table will use t1.fix @@ -1345,7 +1345,7 @@ DROP TABLE t1; # BIN table CREATE TABLE t1 ( id INT(4) KEY NOT NULL, -msg VARCHAR(16) CHARSET BINARY DISTRIB=CLUSTERED) +msg VARCHAR(16) DISTRIB=CLUSTERED) ENGINE=CONNECT TABLE_TYPE=BIN BLOCK_SIZE=8; Warnings: Warning 1105 No file name. Table will use t1.bin diff --git a/storage/connect/mysql-test/connect/t/updelx.test b/storage/connect/mysql-test/connect/t/updelx.test index 19d0d790a30..f6291432e48 100644 --- a/storage/connect/mysql-test/connect/t/updelx.test +++ b/storage/connect/mysql-test/connect/t/updelx.test @@ -36,7 +36,7 @@ DROP TABLE t1; --echo # FIX table CREATE TABLE t1 ( id INT(4) KEY NOT NULL, -msg VARCHAR(16) CHARSET BINARY DISTRIB=CLUSTERED) +msg VARCHAR(16) DISTRIB=CLUSTERED) ENGINE=CONNECT TABLE_TYPE=FIX BLOCK_SIZE=4; -- source updelx.inc ALTER TABLE t1 MAPPED=YES; @@ -48,7 +48,7 @@ DROP TABLE t1; --echo # BIN table CREATE TABLE t1 ( id INT(4) KEY NOT NULL, -msg VARCHAR(16) CHARSET BINARY DISTRIB=CLUSTERED) +msg VARCHAR(16) DISTRIB=CLUSTERED) ENGINE=CONNECT TABLE_TYPE=BIN BLOCK_SIZE=8; -- source updelx.inc ALTER TABLE t1 MAPPED=YES; diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp index 08040884449..3ab750e67f8 100644 --- a/storage/connect/tabjson.cpp +++ b/storage/connect/tabjson.cpp @@ -1486,7 +1486,18 @@ PVAL JSONCOL::MakeJson(PGLOBAL g, PJSON jsp) if (Value->IsTypeNum()) { strcpy(g->Message, "Cannot make Json for a numeric column"); Value->Reset(); - } else + } else if (Value->GetType() == TYPE_BIN) { + if (Value->GetClen() >= sizeof(BSON)) { + ULONG len = Tjp->Lrecl ? Tjp->Lrecl : 500; + PBSON bsp = JbinAlloc(g, NULL, len, jsp); + + strcat(bsp->Msg, " column"); + ((BINVAL*)Value)->SetBinValue(bsp, sizeof(BSON)); + } else { + strcpy(g->Message, "Column size too small"); + Value->SetValue_char(NULL, 0); + } // endif Clen + } else Value->SetValue_psz(Serialize(g, jsp, NULL, 0)); return Value; diff --git a/storage/connect/value.cpp b/storage/connect/value.cpp index df75722d0e8..de04f7678f9 100644 --- a/storage/connect/value.cpp +++ b/storage/connect/value.cpp @@ -2250,6 +2250,15 @@ void BINVAL::SetBinValue(void *p) Len = Clen; } // end of SetBinValue +/***********************************************************************/ +/* BINVAL SetBinValue: fill string with len bytes. */ +/***********************************************************************/ +void BINVAL::SetBinValue(void* p, ulong len) +{ + memcpy(Binp, p, len); + Len = len; +} // end of SetBinValue + /***********************************************************************/ /* GetBinValue: fill a buffer with the internal binary value. */ /* This function checks whether the buffer length is enough and */ diff --git a/storage/connect/value.h b/storage/connect/value.h index 4f7d9a440fa..ee7a1c8032f 100644 --- a/storage/connect/value.h +++ b/storage/connect/value.h @@ -115,8 +115,8 @@ class DllExport VALUE : public BLOCK { virtual void SetValue(ulonglong) {assert(false);} virtual void SetValue(double) {assert(false);} virtual void SetValue_pvblk(PVBLK blk, int n) = 0; - virtual void SetBinValue(void *p) = 0; - virtual bool GetBinValue(void *buf, int buflen, bool go) = 0; + virtual void SetBinValue(void* p) = 0; + virtual bool GetBinValue(void *buf, int buflen, bool go) = 0; virtual int ShowValue(char *buf, int len) = 0; virtual char *GetCharString(char *p) = 0; virtual bool IsEqual(PVAL vp, bool chktype) = 0; @@ -385,7 +385,8 @@ class DllExport BINVAL: public VALUE { virtual void SetValue(ulonglong n); virtual void SetValue(double f); virtual void SetBinValue(void *p); - virtual bool GetBinValue(void *buf, int buflen, bool go); + virtual void SetBinValue(void* p, ulong len); + virtual bool GetBinValue(void *buf, int buflen, bool go); virtual int CompareValue(PVAL) {assert(false); return 0;} virtual int ShowValue(char *buf, int len); virtual char *GetCharString(char *p); -- cgit v1.2.1 From d4138e7eedada2e079f76dca797bf7137088753a Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Mon, 5 Oct 2020 15:21:58 +0200 Subject: Fix compile error in tabjson.cpp (ULONG -> ulong) --- storage/connect/tabjson.cpp | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) (limited to 'storage') diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp index 3ab750e67f8..5fa2021331b 100644 --- a/storage/connect/tabjson.cpp +++ b/storage/connect/tabjson.cpp @@ -177,7 +177,7 @@ int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt) bool mgo = (GetTypeID(topt->type) == TAB_MONGO); PCSZ level = GetStringTableOption(g, topt, "Level", NULL); - if (level = GetStringTableOption(g, topt, "Depth", level)) { + if ((level = GetStringTableOption(g, topt, "Depth", level))) { lvl = atoi(level); lvl = (lvl > 16) ? 16 : lvl; } else @@ -254,12 +254,14 @@ int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt) jsp = (tjsp->GetDoc()) ? tjsp->GetDoc()->GetValue(0) : NULL; } else { - if (!(tdp->Lrecl = GetIntegerTableOption(g, topt, "Lrecl", 0))) - if (!mgo) { - sprintf(g->Message, "LRECL must be specified for pretty=%d", tdp->Pretty); - return 0; - } else - tdp->Lrecl = 8192; // Should be enough + if (!((tdp->Lrecl = GetIntegerTableOption(g, topt, "Lrecl", 0)))) { + if (!mgo) { + sprintf(g->Message, "LRECL must be specified for pretty=%d", tdp->Pretty); + return 0; + } else + tdp->Lrecl = 8192; // Should be enough + + } // endif Lrecl tdp->Ending = GetIntegerTableOption(g, topt, "Ending", CRLF); @@ -1329,7 +1331,7 @@ bool JSONCOL::ParseJpath(PGLOBAL g) { char *p, *p1 = NULL, *p2 = NULL, *pbuf = NULL; int i; - bool a, mul = false; + bool a; if (Parsed) return false; // Already done @@ -1487,8 +1489,8 @@ PVAL JSONCOL::MakeJson(PGLOBAL g, PJSON jsp) strcpy(g->Message, "Cannot make Json for a numeric column"); Value->Reset(); } else if (Value->GetType() == TYPE_BIN) { - if (Value->GetClen() >= sizeof(BSON)) { - ULONG len = Tjp->Lrecl ? Tjp->Lrecl : 500; + if ((unsigned)Value->GetClen() >= sizeof(BSON)) { + ulong len = Tjp->Lrecl ? Tjp->Lrecl : 500; PBSON bsp = JbinAlloc(g, NULL, len, jsp); strcat(bsp->Msg, " column"); @@ -1570,7 +1572,6 @@ void JSONCOL::ReadColumn(PGLOBAL g) PVAL JSONCOL::GetColumnValue(PGLOBAL g, PJSON row, int i) { int n = Nod - 1; - bool expd = false; PJAR arp; PJVAL val = NULL; @@ -2130,13 +2131,15 @@ int TDBJSON::Cardinality(PGLOBAL g) { if (!g) return (Xcol || Multiple) ? 0 : 1; - else if (Cardinal < 0) - if (!Multiple) { - if (MakeDocument(g) == RC_OK) - Cardinal = Doc->size(); + else if (Cardinal < 0) { + if (!Multiple) { + if (MakeDocument(g) == RC_OK) + Cardinal = Doc->size(); - } else - return 10; + } else + return 10; + + } // endif Cardinal return Cardinal; } // end of Cardinality -- cgit v1.2.1 From 5d2ddef26e3fce9784e4befb9e2e0960ff2434cd Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Tue, 6 Oct 2020 12:50:12 +0200 Subject: Fix search for json subtable in tabjson.cpp --- storage/connect/tabjson.cpp | 48 +++++++++++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 15 deletions(-) (limited to 'storage') diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp index 5fa2021331b..fcf7f4d182c 100644 --- a/storage/connect/tabjson.cpp +++ b/storage/connect/tabjson.cpp @@ -1995,8 +1995,9 @@ int TDBJSON::MakeNewDoc(PGLOBAL g) /***********************************************************************/ int TDBJSON::MakeDocument(PGLOBAL g) { - char *p, *memory, *objpath, *key = NULL; + char *p, *p1, *p2, *memory, *objpath, *key = NULL; int len, i = 0; + my_bool a; MODE mode = Mode; PJSON jsp; PJOB objp = NULL; @@ -2039,22 +2040,39 @@ int TDBJSON::MakeDocument(PGLOBAL g) if ((objpath = PlugDup(g, Objname))) { if (*objpath == '$') objpath++; if (*objpath == '.') objpath++; + p1 = p2 = NULL; /*********************************************************************/ /* Find the table in the tree structure. */ /*********************************************************************/ - for (; jsp && objpath; objpath = p) { - if ((p = strchr(objpath, Sep))) - *p++ = 0; - - if (*objpath != '[' && !IsNum(objpath)) { - // objpass is a key + for (p = objpath; jsp && p; p = (p2 ? p2 : NULL)) { + a = (p1 != NULL); + p1 = strchr(p, '['); + p2 = strchr(p, '.'); + + if (!p2) + p2 = p1; + else if (p1) { + if (p1 < p2) + p2 = p1; + else if (p1 == p2 + 1) + *p2++ = 0; // Old syntax .[ + else + p1 = NULL; + + } // endif p1 + + if (p2) + *p2++ = 0; + + if (!a && *p && *p != '[' && !IsNum(p)) { + // obj is a key if (jsp->GetType() != TYPE_JOB) { strcpy(g->Message, "Table path does not match the json file"); return RC_FX; } // endif Type - key = objpath; + key = p; objp = jsp->GetObject(); arp = NULL; val = objp->GetValue(key); @@ -2065,15 +2083,15 @@ int TDBJSON::MakeDocument(PGLOBAL g) } // endif val } else { - if (*objpath == '[') { + if (*p == '[') { // Old style - if (objpath[strlen(objpath) - 1] != ']') { - sprintf(g->Message, "Invalid Table path %s", Objname); + if (p[strlen(p) - 1] != ']') { + sprintf(g->Message, "Invalid Table path near %s", p); return RC_FX; } else - objpath++; + p++; - } // endif objpath + } // endif p if (jsp->GetType() != TYPE_JAR) { strcpy(g->Message, "Table path does not match the json file"); @@ -2082,7 +2100,7 @@ int TDBJSON::MakeDocument(PGLOBAL g) arp = jsp->GetArray(); objp = NULL; - i = atoi(objpath) - B; + i = atoi(p) - B; val = arp->GetValue(i); if (!val) { @@ -2093,7 +2111,7 @@ int TDBJSON::MakeDocument(PGLOBAL g) } // endif jsp = val->GetJson(); - } // endfor objpath + } // endfor p } // endif objpath -- cgit v1.2.1 From dc3a693b7039cb4b1d46d797ca86364ac3173be7 Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Sun, 18 Oct 2020 17:20:44 +0200 Subject: - Inline MakePtr and MakeOff with OFFSET as size_t Also add a new member Saved_Size in the Global structure. modified: storage/connect/global.h modified: storage/connect/plugutil.cpp modified: storage/connect/user_connect.cc modified: storage/connect/jsonudf.cpp - Add session variables json_all_path and default_depth modified: storage/connect/ha_connect.cc modified: storage/connect/mongo.cpp modified: storage/connect/tabjson.cpp modified: storage/connect/tabxml.cpp - ADD column options JPATH and XPATH Work as FIELD_FORMAT but are more readable modified: storage/connect/ha_connect.cc modified: storage/connect/ha_connect.h modified: storage/connect/mysql-test/connect/r/json_java_2.result modified: storage/connect/mysql-test/connect/r/json_java_3.result modified: storage/connect/mysql-test/connect/r/json_mongo_c.result - Handle negative numbes in the option list modified: storage/connect/ha_connect.cc - Fix Json parse that could crash the server. Was because it could use THROW out of the TRY block. Also handle all error by THROW. It is now done by a new class JSON. modified: storage/connect/json.cpp modified: storage/connect/json.h - Add a new UDF function jfile_translate. It translate a Json file to pretty = 0. Fast because it does not a real parse of the file. modified: storage/connect/jsonudf.cpp modified: storage/connect/jsonudf.h - Add a now options JSIZE and STRINGIFY to Json tables. STRINGIFY makes Objects or Arrays to be returned by their json representation instead of by their concatenated values. JSIZE allows to specify the LRECL (was 256) defaults to 1024. Also fix a bug about locating the sub-table by its path. modified: storage/connect/tabjson.cpp modified: storage/connect/tabjson.h --- storage/connect/global.h | 40 +- storage/connect/ha_connect.cc | 210 +++++---- storage/connect/ha_connect.h | 4 +- storage/connect/json.cpp | 164 +++---- storage/connect/json.h | 33 +- storage/connect/jsonudf.cpp | 500 ++++++++++++++++++++- storage/connect/jsonudf.h | 39 ++ storage/connect/mongo.cpp | 10 +- .../mysql-test/connect/r/json_java_2.result | 36 +- .../mysql-test/connect/r/json_java_3.result | 36 +- .../mysql-test/connect/r/json_mongo_c.result | 36 +- storage/connect/plugutil.cpp | 43 +- storage/connect/tabjson.cpp | 51 ++- storage/connect/tabjson.h | 4 +- storage/connect/tabxml.cpp | 6 +- storage/connect/user_connect.cc | 9 +- 16 files changed, 848 insertions(+), 373 deletions(-) (limited to 'storage') diff --git a/storage/connect/global.h b/storage/connect/global.h index 548d047ccc9..d17620861fa 100644 --- a/storage/connect/global.h +++ b/storage/connect/global.h @@ -89,14 +89,10 @@ extern "C" { #define PAT_LOG "log" #if defined(UNIX) || defined(LINUX) || defined(UNIV_LINUX) - /*********************************************************************/ - /* printf does not accept null pointer for %s target. */ - /*********************************************************************/ + // printf does not accept null pointer for %s target #define SVP(S) ((S) ? S : "") #else - /*********************************************************************/ - /* printf accepts null pointer for %s target. */ - /*********************************************************************/ + // printf accepts null pointer for %s target #define SVP(S) S #endif @@ -112,9 +108,6 @@ extern "C" { /***********************************************************************/ #include "os.h" -typedef size_t OFFSET; -typedef char NAME[9]; - typedef struct { ushort Length; char String[2]; @@ -127,6 +120,7 @@ typedef struct _global *PGLOBAL; typedef struct _globplg *PGS; typedef struct _activity *PACTIVITY; typedef struct _parm *PPARM; +typedef char NAME[9]; /***********************************************************************/ /* Segment Sub-Allocation block structure declares. */ @@ -135,7 +129,7 @@ typedef struct _parm *PPARM; /* restore them if needed. This scheme implies that no SubFree be used */ /***********************************************************************/ typedef struct { /* Plug Area SubAlloc header */ - OFFSET To_Free; /* Offset of next free block */ + size_t To_Free; /* Offset of next free block */ size_t FreeBlk; /* Size of remaining free memory */ } POOLHEADER, *PPOOLHEADER; @@ -190,9 +184,10 @@ typedef struct _global { /* Global structure */ void *Sarea; /* Points to work area */ size_t Sarea_Size; /* Work area size */ PACTIVITY Activityp; - char Message[MAX_STR]; + char Message[MAX_STR]; /* Message (result, error, trace) */ ulong More; /* Used by jsonudf */ - int Createas; /* To pass multi to ext tables */ + size_t Saved_Size; /* Saved work area to_free */ + bool Createas; /* To pass multi to ext tables */ void *Xchk; /* indexes in create/alter */ short Alchecked; /* Checked for ALTER */ short Mrr; /* True when doing mrr */ @@ -222,7 +217,6 @@ DllExport void FreeSarea(PGLOBAL); DllExport BOOL PlugSubSet(void *, size_t); DllExport void *PlugSubAlloc(PGLOBAL, void *, size_t); DllExport char *PlugDup(PGLOBAL g, const char *str); -DllExport void *MakePtr(void *, OFFSET); DllExport void htrc(char const *fmt, ...); DllExport void xtrc(uint, char const* fmt, ...); DllExport uint GetTraceValue(void); @@ -232,8 +226,24 @@ DllExport uint GetTraceValue(void); #endif /***********************************************************************/ -/* Non exported routine declarations. */ +/* Inline routine definitions. */ +/***********************************************************************/ +/***********************************************************************/ +/* This routine makes a pointer from an offset to a memory pointer. */ +/***********************************************************************/ +inline void* MakePtr(void* memp, size_t offset) { + // return ((offset == 0) ? NULL : &((char*)memp)[offset]); + return (!offset) ? NULL : (char *)memp + offset; +} /* end of MakePtr */ + +/***********************************************************************/ +/* This routine makes an offset from a pointer new format. */ /***********************************************************************/ -//void *PlugSubAlloc(PGLOBAL, void *, size_t); // Does throw +inline size_t MakeOff(void* memp, void* ptr) { +#if defined(_DEBUG) + assert(ptr > memp); +#endif // _DEBUG + return ((!ptr) ? 0 : (size_t)((char*)ptr - (size_t)memp)); +} /* end of MakeOff */ /*-------------------------- End of Global.H --------------------------*/ diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index fd00b4878a9..a3dfc50562d 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -170,7 +170,7 @@ #define JSONMAX 10 // JSON Default max grp size extern "C" { - char version[]= "Version 1.07.0002 October 03, 2020"; + char version[]= "Version 1.07.0002 October 18, 2020"; #if defined(__WIN__) char compver[]= "Version 1.07.0002 " __DATE__ " " __TIME__; char slash= '\\'; @@ -251,6 +251,7 @@ bool ExactInfo(void); USETEMP UseTemp(void); int GetConvSize(void); TYPCONV GetTypeConv(void); +bool JsonAllPath(void); char *GetJsonNull(void); uint GetJsonGrpSize(void); char *GetJavaWrapper(void); @@ -394,6 +395,11 @@ static MYSQL_THDVAR_ENUM( 1, // def (yes) &xconv_typelib); // typelib +// Adding JPATH to all Json table columns +static MYSQL_THDVAR_BOOL(json_all_path, PLUGIN_VAR_RQCMDARG, + "Adding JPATH to all Json table columns", + NULL, NULL, 0); // NO by default + // Null representation for JSON values static MYSQL_THDVAR_STR(json_null, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, @@ -401,6 +407,12 @@ static MYSQL_THDVAR_STR(json_null, // check_json_null, update_json_null, NULL, NULL, ""); +// Default Json, XML or Mongo depth +static MYSQL_THDVAR_INT(default_depth, + PLUGIN_VAR_RQCMDARG, + "Default depth used by Json, XML and Mongo discovery", + NULL, NULL, 0, -1, 16, 1); + // Estimate max number of rows for JSON aggregate functions static MYSQL_THDVAR_UINT(json_grp_size, PLUGIN_VAR_RQCMDARG, // opt @@ -462,11 +474,13 @@ uint GetTraceValue(void) {return (uint)(connect_hton ? THDVAR(current_thd, xtrace) : 0);} bool ExactInfo(void) {return THDVAR(current_thd, exact_info);} static bool CondPushEnabled(void) {return THDVAR(current_thd, cond_push);} +bool JsonAllPath(void) {return THDVAR(current_thd, json_all_path);} USETEMP UseTemp(void) {return (USETEMP)THDVAR(current_thd, use_tempfile);} int GetConvSize(void) {return THDVAR(current_thd, conv_size);} TYPCONV GetTypeConv(void) {return (TYPCONV)THDVAR(current_thd, type_conv);} char *GetJsonNull(void) {return connect_hton ? THDVAR(current_thd, json_null) : NULL;} +int GetDefaultDepth(void) {return THDVAR(current_thd, default_depth);} uint GetJsonGrpSize(void) {return connect_hton ? THDVAR(current_thd, json_grp_size) : 10;} size_t GetWorkSize(void) {return (size_t)THDVAR(current_thd, work_size);} @@ -630,8 +644,10 @@ ha_create_table_option connect_field_option_list[]= HA_FOPTION_NUMBER("FIELD_LENGTH", fldlen, 0, 0, INT_MAX32, 1), HA_FOPTION_STRING("DATE_FORMAT", dateformat), HA_FOPTION_STRING("FIELD_FORMAT", fieldformat), - HA_FOPTION_STRING("SPECIAL", special), - HA_FOPTION_ENUM("DISTRIB", opt, "scattered,clustered,sorted", 0), + HA_FOPTION_STRING("JPATH", jsonpath), + HA_FOPTION_STRING("XPATH", xmlpath), + HA_FOPTION_STRING("SPECIAL", special), + HA_FOPTION_ENUM("DISTRIB", opt, "scattered,clustered,sorted", 0), HA_FOPTION_END }; @@ -1322,9 +1338,10 @@ int GetIntegerTableOption(PGLOBAL g, PTOS options, PCSZ opname, int idef) if ((ulonglong) opval == (ulonglong)NO_IVAL) { PCSZ pv; - if ((pv= GetListOption(g, opname, options->oplist))) - opval= CharToNumber((char*)pv, strlen(pv), ULONGLONG_MAX, true); - else + if ((pv = GetListOption(g, opname, options->oplist))) { + // opval = CharToNumber((char*)pv, strlen(pv), ULONGLONG_MAX, false); + return atoi(pv); + } else return idef; } // endif opval @@ -1576,8 +1593,9 @@ void *ha_connect::GetColumnOption(PGLOBAL g, void *field, PCOLINFO pcf) pcf->Offset= (int)fop->offset; pcf->Freq= (int)fop->freq; pcf->Datefmt= (char*)fop->dateformat; - pcf->Fieldfmt= (char*)fop->fieldformat; - } else { + pcf->Fieldfmt = fop->fieldformat ? (char*)fop->fieldformat + : fop->jsonpath ? (char*)fop->jsonpath : (char*)fop->xmlpath; + } else { pcf->Offset= -1; pcf->Freq= 0; pcf->Datefmt= NULL; @@ -4984,7 +5002,7 @@ int ha_connect::check_stmt(PGLOBAL g, MODE newmode, bool cras) } // endif CheckCleanup if (cras) - g->Createas= 1; // To tell external tables of a multi-table command + g->Createas= true; // To tell external tables of a multi-table command if (trace(1)) htrc("Calling CntCheckDB db=%s cras=%d\n", GetDBName(NULL), cras); @@ -5334,96 +5352,100 @@ static char *encode(PGLOBAL g, const char *cnm) @return Return 0 if ok */ -static bool add_field(String *sql, const char *field_name, int typ, int len, - int dec, char *key, uint tm, const char *rem, char *dft, - char *xtra, char *fmt, int flag, bool dbf, char v) -{ +static bool add_field(String* sql, TABTYPE ttp, const char* field_name, int typ, + int len, int dec, char* key, uint tm, const char* rem, + char* dft, char* xtra, char* fmt, int flag, bool dbf, char v) { #if defined(DEVELOPMENT) // Some client programs regard CHAR(36) as GUID - char var= (len > 255 || len == 36) ? 'V' : v; + char var = (len > 255 || len == 36) ? 'V' : v; #else char var = (len > 255) ? 'V' : v; #endif - bool q, error= false; - const char *type= PLGtoMYSQLtype(typ, dbf, var); + bool q, error = false; + const char* type = PLGtoMYSQLtype(typ, dbf, var); - error|= sql->append('`'); - error|= sql->append(field_name); - error|= sql->append("` "); - error|= sql->append(type); + error |= sql->append('`'); + error |= sql->append(field_name); + error |= sql->append("` "); + error |= sql->append(type); - if (typ == TYPE_STRING || - (len && typ != TYPE_DATE && (typ != TYPE_DOUBLE || dec >= 0))) { - error|= sql->append('('); - error|= sql->append_ulonglong(len); + if (typ == TYPE_STRING || + (len && typ != TYPE_DATE && (typ != TYPE_DOUBLE || dec >= 0))) { + error |= sql->append('('); + error |= sql->append_ulonglong(len); if (typ == TYPE_DOUBLE) { - error|= sql->append(','); - // dec must be < len and < 31 - error|= sql->append_ulonglong(MY_MIN(dec, (MY_MIN(len, 31) - 1))); - } else if (dec > 0 && !strcmp(type, "DECIMAL")) { - error|= sql->append(','); - // dec must be < len - error|= sql->append_ulonglong(MY_MIN(dec, len - 1)); - } // endif dec - - error|= sql->append(')'); - } // endif len - - if (v == 'U') - error|= sql->append(" UNSIGNED"); - else if (v == 'Z') - error|= sql->append(" ZEROFILL"); - - if (key && *key) { - error|= sql->append(" "); - error|= sql->append(key); - } // endif key - - if (tm) - error|= sql->append(STRING_WITH_LEN(" NOT NULL"), system_charset_info); - - if (dft && *dft) { - error|= sql->append(" DEFAULT "); - - if (typ == TYPE_DATE) - q= (strspn(dft, "0123456789 -:/") == strlen(dft)); - else - q= !IsTypeNum(typ); + error |= sql->append(','); + // dec must be < len and < 31 + error |= sql->append_ulonglong(MY_MIN(dec, (MY_MIN(len, 31) - 1))); + } else if (dec > 0 && !strcmp(type, "DECIMAL")) { + error |= sql->append(','); + // dec must be < len + error |= sql->append_ulonglong(MY_MIN(dec, len - 1)); + } // endif dec + + error |= sql->append(')'); + } // endif len + + if (v == 'U') + error |= sql->append(" UNSIGNED"); + else if (v == 'Z') + error |= sql->append(" ZEROFILL"); + + if (key && *key) { + error |= sql->append(" "); + error |= sql->append(key); + } // endif key + + if (tm) + error |= sql->append(STRING_WITH_LEN(" NOT NULL"), system_charset_info); + + if (dft && *dft) { + error |= sql->append(" DEFAULT "); + + if (typ == TYPE_DATE) + q = (strspn(dft, "0123456789 -:/") == strlen(dft)); + else + q = !IsTypeNum(typ); - if (q) { - error|= sql->append("'"); - error|= sql->append_for_single_quote(dft, strlen(dft)); - error|= sql->append("'"); - } else - error|= sql->append(dft); - - } // endif dft - - if (xtra && *xtra) { - error|= sql->append(" "); - error|= sql->append(xtra); - } // endif rem - - if (rem && *rem) { - error|= sql->append(" COMMENT '"); - error|= sql->append_for_single_quote(rem, strlen(rem)); - error|= sql->append("'"); - } // endif rem - - if (fmt && *fmt) { - error|= sql->append(" FIELD_FORMAT='"); - error|= sql->append_for_single_quote(fmt, strlen(fmt)); - error|= sql->append("'"); - } // endif flag - - if (flag) { - error|= sql->append(" FLAG="); - error|= sql->append_ulonglong(flag); - } // endif flag - - error|= sql->append(','); - return error; + if (q) { + error |= sql->append("'"); + error |= sql->append_for_single_quote(dft, strlen(dft)); + error |= sql->append("'"); + } else + error |= sql->append(dft); + + } // endif dft + + if (xtra && *xtra) { + error |= sql->append(" "); + error |= sql->append(xtra); + } // endif rem + + if (rem && *rem) { + error |= sql->append(" COMMENT '"); + error |= sql->append_for_single_quote(rem, strlen(rem)); + error |= sql->append("'"); + } // endif rem + + if (fmt && *fmt) { + switch (ttp) { + case TAB_JSON: error |= sql->append(" JPATH='"); break; + case TAB_XML: error |= sql->append(" XPATH='"); break; + default: error |= sql->append(" FIELD_FORMAT='"); + } // endswitch ttp + + error |= sql->append_for_single_quote(fmt, strlen(fmt)); + error |= sql->append("'"); + } // endif flag + + if (flag) { + error |= sql->append(" FLAG="); + error |= sql->append_ulonglong(flag); + } // endif flag + + error |= sql->append(','); + return error; } // end of add_field /** @@ -6045,7 +6067,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, len= 256; // STRBLK's have 0 length // Now add the field - if (add_field(&sql, cnm, typ, len, dec, NULL, tm, + if (add_field(&sql, ttp, cnm, typ, len, dec, NULL, tm, NULL, NULL, NULL, NULL, flg, dbf, v)) rc= HA_ERR_OUT_OF_MEM; } // endfor crp @@ -6239,7 +6261,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, prec= 0; // Now add the field - if (add_field(&sql, cnm, typ, prec, dec, key, tm, rem, dft, xtra, + if (add_field(&sql, ttp, cnm, typ, prec, dec, key, tm, rem, dft, xtra, fmt, flg, dbf, v)) rc= HA_ERR_OUT_OF_MEM; } // endfor i @@ -6992,7 +7014,7 @@ bool ha_connect::NoFieldOptionChange(TABLE *tab) fop1->fldlen == fop2->fldlen && CheckString(fop1->dateformat, fop2->dateformat) && CheckString(fop1->fieldformat, fop2->fieldformat) && - CheckString(fop1->special, fop2->special)); + CheckString(fop1->special, fop2->special)); } // endfor fld return rc; @@ -7362,7 +7384,9 @@ static struct st_mysql_sys_var* connect_system_variables[]= { MYSQL_SYSVAR(errmsg_dir_path), #endif // XMSG MYSQL_SYSVAR(json_null), - MYSQL_SYSVAR(json_grp_size), + MYSQL_SYSVAR(json_all_path), + MYSQL_SYSVAR(default_depth), + MYSQL_SYSVAR(json_grp_size), #if defined(JAVA_SUPPORT) MYSQL_SYSVAR(jvm_path), MYSQL_SYSVAR(class_path), diff --git a/storage/connect/ha_connect.h b/storage/connect/ha_connect.h index 9a12ef94431..21e81b141a8 100644 --- a/storage/connect/ha_connect.h +++ b/storage/connect/ha_connect.h @@ -104,7 +104,9 @@ struct ha_field_option_struct uint opt; const char *dateformat; const char *fieldformat; - char *special; + const char* jsonpath; + const char* xmlpath; + char *special; }; /* diff --git a/storage/connect/json.cpp b/storage/connect/json.cpp index a38e6487025..ea3ea18da0b 100644 --- a/storage/connect/json.cpp +++ b/storage/connect/json.cpp @@ -93,9 +93,8 @@ char *NextChr(PSZ s, char sep) PJSON ParseJson(PGLOBAL g, char *s, int len, int *ptyp, bool *comma) { int i, pretty = (ptyp) ? *ptyp : 3; - bool b = false, pty[3] = {true, true, true}; - PJSON jsp = NULL; - STRG src; + bool b = false, pty[3] = {true,true,true}; + PJSON jsp = NULL, jp = NULL; if (trace(1)) htrc("ParseJson: s=%.10s len=%d\n", s, len); @@ -106,27 +105,29 @@ PJSON ParseJson(PGLOBAL g, char *s, int len, int *ptyp, bool *comma) } else if (comma) *comma = false; - src.str = s; - src.len = len; - // Trying to guess the pretty format if (s[0] == '[' && (s[1] == '\n' || (s[1] == '\r' && s[2] == '\n'))) pty[0] = false; try { - for (i = 0; i < len; i++) + jp = new(g) JSON(); + jp->s = s; + jp->len = len; + jp->pty = pty; + + for (i = 0; i < jp->len; i++) switch (s[i]) { case '[': if (jsp) - goto tryit; - else if (!(jsp = ParseArray(g, ++i, src, pty))) - throw 1; + jsp = jp->ParseAsArray(g, i, pretty, ptyp); + else + jsp = jp->ParseArray(g, ++i); break; case '{': if (jsp) - goto tryit; - else if (!(jsp = ParseObject(g, ++i, src, pty))) + jsp = jp->ParseAsArray(g, i, pretty, ptyp); + else if (!(jsp = jp->ParseObject(g, ++i))) throw 2; break; @@ -157,8 +158,8 @@ PJSON ParseJson(PGLOBAL g, char *s, int len, int *ptyp, bool *comma) default: if (jsp) - goto tryit; - else if (!(jsp = ParseValue(g, i, src, pty))) + jsp = jp->ParseAsArray(g, i, pretty, ptyp); + else if (!(jsp = jp->ParseValue(g, i))) throw 4; break; @@ -187,10 +188,17 @@ PJSON ParseJson(PGLOBAL g, char *s, int len, int *ptyp, bool *comma) } // end catch return jsp; +} // end of ParseJson -tryit: +/***********************************************************************/ +/* Parse several items as being in an array. */ +/***********************************************************************/ +PJAR JSON::ParseAsArray(PGLOBAL g, int& i, int pretty, int *ptyp) +{ if (pty[0] && (!pretty || pretty > 2)) { - if ((jsp = ParseArray(g, (i = 0), src, pty)) && ptyp && pretty == 3) + PJAR jsp; + + if ((jsp = ParseArray(g, (i = 0))) && ptyp && pretty == 3) *ptyp = (pty[0]) ? 0 : 3; return jsp; @@ -198,26 +206,23 @@ tryit: strcpy(g->Message, "More than one item in file"); return NULL; -} // end of ParseJson +} // end of ParseAsArray /***********************************************************************/ /* Parse a JSON Array. */ /***********************************************************************/ -PJAR ParseArray(PGLOBAL g, int& i, STRG& src, bool *pty) +PJAR JSON::ParseArray(PGLOBAL g, int& i) { - char *s = src.str; - int len = src.len; - int level = 0; - bool b = (!i); - PJAR jarp = new(g) JARRAY; - PJVAL jvp = NULL; + int level = 0; + bool b = (!i); + PJAR jarp = new(g) JARRAY; for (; i < len; i++) switch (s[i]) { case ',': if (level < 2) { sprintf(g->Message, "Unexpected ',' near %.*s",ARGS); - return NULL; + throw 1; } else level = 1; @@ -225,8 +230,8 @@ PJAR ParseArray(PGLOBAL g, int& i, STRG& src, bool *pty) case ']': if (level == 1) { sprintf(g->Message, "Unexpected ',]' near %.*s", ARGS); - return NULL; - } // endif level + throw 1; + } // endif level jarp->InitArray(g); return jarp; @@ -240,11 +245,9 @@ PJAR ParseArray(PGLOBAL g, int& i, STRG& src, bool *pty) default: if (level == 2) { sprintf(g->Message, "Unexpected value near %.*s", ARGS); - return NULL; - } else if ((jvp = ParseValue(g, i, src, pty))) - jarp->AddValue(g, jvp); - else - return NULL; + throw 1; + } else + jarp->AddValue(g, ParseValue(g, i)); level = (b) ? 1 : 2; break; @@ -256,18 +259,15 @@ PJAR ParseArray(PGLOBAL g, int& i, STRG& src, bool *pty) return jarp; } // endif b - strcpy(g->Message, "Unexpected EOF in array"); - return NULL; + throw ("Unexpected EOF in array"); } // end of ParseArray /***********************************************************************/ /* Parse a JSON Object. */ /***********************************************************************/ -PJOB ParseObject(PGLOBAL g, int& i, STRG& src, bool *pty) +PJOB JSON::ParseObject(PGLOBAL g, int& i) { PSZ key; - char *s = src.str; - int len = src.len; int level = 0; PJOB jobp = new(g) JOBJECT; PJPR jpp = NULL; @@ -276,42 +276,37 @@ PJOB ParseObject(PGLOBAL g, int& i, STRG& src, bool *pty) switch (s[i]) { case '"': if (level < 2) { - if ((key = ParseString(g, ++i, src))) { - jpp = jobp->AddPair(g, key); - level = 1; - } else - return NULL; - + key = ParseString(g, ++i); + jpp = jobp->AddPair(g, key); + level = 1; } else { sprintf(g->Message, "misplaced string near %.*s", ARGS); - return NULL; + throw 2; } // endif level break; case ':': if (level == 1) { - if (!(jpp->Val = ParseValue(g, ++i, src, pty))) - return NULL; - + jpp->Val = ParseValue(g, ++i); level = 2; } else { sprintf(g->Message, "Unexpected ':' near %.*s", ARGS); - return NULL; + throw 2; } // endif level break; case ',': if (level < 2) { sprintf(g->Message, "Unexpected ',' near %.*s", ARGS); - return NULL; + throw 2; } else - level = 1; + level = 0; break; case '}': - if (level == 1) { + if (level < 2) { sprintf(g->Message, "Unexpected '}' near %.*s", ARGS); - return NULL; + throw 2; } // endif level return jobp; @@ -324,20 +319,19 @@ PJOB ParseObject(PGLOBAL g, int& i, STRG& src, bool *pty) default: sprintf(g->Message, "Unexpected character '%c' near %.*s", s[i], ARGS); - return NULL; + throw 2; }; // endswitch s[i] strcpy(g->Message, "Unexpected EOF in Object"); - return NULL; + throw 2; } // end of ParseObject /***********************************************************************/ /* Parse a JSON Value. */ /***********************************************************************/ -PJVAL ParseValue(PGLOBAL g, int& i, STRG& src, bool *pty) +PJVAL JSON::ParseValue(PGLOBAL g, int& i) { - char *strval, *s = src.str; - int n, len = src.len; + int n; PJVAL jvp = new(g) JVALUE; for (; i < len; i++) @@ -355,21 +349,13 @@ PJVAL ParseValue(PGLOBAL g, int& i, STRG& src, bool *pty) suite: switch (s[i]) { case '[': - if (!(jvp->Jsp = ParseArray(g, ++i, src, pty))) - return NULL; - + jvp->Jsp = ParseArray(g, ++i); break; case '{': - if (!(jvp->Jsp = ParseObject(g, ++i, src, pty))) - return NULL; - + jvp->Jsp = ParseObject(g, ++i); break; case '"': - if ((strval = ParseString(g, ++i, src))) - jvp->Value = AllocateValue(g, strval, TYPE_STRING); - else - return NULL; - + jvp->Value = AllocateValue(g, ParseString(g, ++i), TYPE_STRING); break; case 't': if (!strncmp(s + i, "true", 4)) { @@ -398,11 +384,9 @@ PJVAL ParseValue(PGLOBAL g, int& i, STRG& src, bool *pty) break; case '-': default: - if (s[i] == '-' || isdigit(s[i])) { - if (!(jvp->Value = ParseNumeric(g, i, src))) - goto err; - - } else + if (s[i] == '-' || isdigit(s[i])) + jvp->Value = ParseNumeric(g, i); + else goto err; }; // endswitch s[i] @@ -410,25 +394,21 @@ PJVAL ParseValue(PGLOBAL g, int& i, STRG& src, bool *pty) return jvp; err: - sprintf(g->Message, "Unexpected character '%c' near %.*s", - s[i], ARGS); - return NULL; + sprintf(g->Message, "Unexpected character '%c' near %.*s", s[i], ARGS); + throw 3; } // end of ParseValue /***********************************************************************/ /* Unescape and parse a JSON string. */ /***********************************************************************/ -char *ParseString(PGLOBAL g, int& i, STRG& src) +char *JSON::ParseString(PGLOBAL g, int& i) { - char *s = src.str; uchar *p; - int n = 0, len = src.len; + int n = 0; // Be sure of memory availability - if ((unsigned)(len + 1 - i) > ((PPOOLHEADER)g->Sarea)->FreeBlk) { - strcpy(g->Message, "ParseString: Out of memory"); - return NULL; - } // endif len + if (((size_t)len + 1 - i) > ((PPOOLHEADER)g->Sarea)->FreeBlk) + throw("ParseString: Out of memory"); // The size to allocate is not known yet p = (uchar*)PlugSubAlloc(g, NULL, 0); @@ -502,17 +482,16 @@ char *ParseString(PGLOBAL g, int& i, STRG& src) }; // endswitch s[i] err: - strcpy(g->Message, "Unexpected EOF in String"); - return NULL; + throw("Unexpected EOF in String"); } // end of ParseString /***********************************************************************/ /* Parse a JSON numeric value. */ /***********************************************************************/ -PVAL ParseNumeric(PGLOBAL g, int& i, STRG& src) +PVAL JSON::ParseNumeric(PGLOBAL g, int& i) { - char *s = src.str, buf[50]; - int n = 0, len = src.len; + char buf[50]; + int n = 0; short nd = 0; bool has_dot = false; bool has_e = false; @@ -575,14 +554,11 @@ PVAL ParseNumeric(PGLOBAL g, int& i, STRG& src) i--; // Unstack following character return valp; - } else { - strcpy(g->Message, "No digit found"); - return NULL; - } // endif found_digit + } else + throw("No digit found"); err: - strcpy(g->Message, "Unexpected EOF in number"); - return NULL; + throw("Unexpected EOF in number"); } // end of ParseNumeric /***********************************************************************/ diff --git a/storage/connect/json.h b/storage/connect/json.h index d949f244e21..bc94b372133 100644 --- a/storage/connect/json.h +++ b/storage/connect/json.h @@ -69,12 +69,7 @@ PBSON JbinAlloc(PGLOBAL g, UDF_ARGS* args, ulong len, PJSON jsp); char *NextChr(PSZ s, char sep); char *GetJsonNull(void); -PJSON ParseJson(PGLOBAL g, char *s, int n, int *prty = NULL, bool *b = NULL); -PJAR ParseArray(PGLOBAL g, int& i, STRG& src, bool *pty); -PJOB ParseObject(PGLOBAL g, int& i, STRG& src, bool *pty); -PJVAL ParseValue(PGLOBAL g, int& i, STRG& src, bool *pty); -char *ParseString(PGLOBAL g, int& i, STRG& src); -PVAL ParseNumeric(PGLOBAL g, int& i, STRG& src); +PJSON ParseJson(PGLOBAL g, char* s, int n, int* prty = NULL, bool* b = NULL); PSZ Serialize(PGLOBAL g, PJSON jsp, char *fn, int pretty); bool SerializeArray(JOUT *js, PJAR jarp, bool b); bool SerializeObject(JOUT *js, PJOB jobp); @@ -152,7 +147,7 @@ class JOUTPRT : public JOUTFILE { class JPAIR : public BLOCK { friend class JOBJECT; friend class JSNX; - friend PJOB ParseObject(PGLOBAL, int&, STRG&, bool*); + friend class JSON; friend bool SerializeObject(JOUT *, PJOB); public: JPAIR(PCSZ key) : BLOCK() {Key = key; Val = NULL; Next = NULL;} @@ -171,8 +166,9 @@ class JPAIR : public BLOCK { /* Class JSON. The base class for all other json classes. */ /***********************************************************************/ class JSON : public BLOCK { + friend PJSON ParseJson(PGLOBAL, char*, int, int*, bool*); public: - JSON(void) {Size = 0;} + JSON(void) : s(NULL), len(0), pty(NULL) {Size = 0;} int size(void) {return Size;} virtual int GetSize(bool b) {return Size;} @@ -209,14 +205,27 @@ class JSON : public BLOCK { virtual bool IsNull(void) {X return true;} protected: - int Size; + PJAR ParseArray(PGLOBAL g, int& i); + PJOB ParseObject(PGLOBAL g, int& i); + PJVAL ParseValue(PGLOBAL g, int& i); + char *ParseString(PGLOBAL g, int& i); + PVAL ParseNumeric(PGLOBAL g, int& i); + PJAR ParseAsArray(PGLOBAL g, int& i, int pretty, int *ptyp); + + // Members + int Size; + + // Only used when parsing + private: + char *s; + int len; + bool *pty; }; // end of class JSON /***********************************************************************/ /* Class JOBJECT: contains a list of value pairs. */ /***********************************************************************/ class JOBJECT : public JSON { - friend PJOB ParseObject(PGLOBAL, int&, STRG&, bool*); friend bool SerializeObject(JOUT *, PJOB); friend class JSNX; public: @@ -282,8 +291,8 @@ class JVALUE : public JSON { friend class JARRAY; friend class JSNX; friend class JSONCOL; - friend PJVAL ParseValue(PGLOBAL, int&, STRG&, bool*); - friend bool SerializeValue(JOUT *, PJVAL); + friend class JSON; + friend bool SerializeValue(JOUT*, PJVAL); public: JVALUE(void) : JSON() {Clear();} JVALUE(PJSON jsp); diff --git a/storage/connect/jsonudf.cpp b/storage/connect/jsonudf.cpp index 0a862d2917d..06164f4ed78 100644 --- a/storage/connect/jsonudf.cpp +++ b/storage/connect/jsonudf.cpp @@ -1076,27 +1076,6 @@ my_bool JSNX::AddPath(void) /* --------------------------------- JSON UDF ---------------------------------- */ -#if 0 // Moved to json.h -// BSON size should be equal on Linux and Windows -#define BMX 255 -typedef struct BSON *PBSON; - -/*********************************************************************************/ -/* Structure used to return binary json. */ -/*********************************************************************************/ -struct BSON { - char Msg[BMX + 1]; - char *Filename; - PGLOBAL G; - int Pretty; - ulong Reslen; - my_bool Changed; - PJSON Top; - PJSON Jsp; - PBSON Bsp; -}; // end of struct BSON -#endif // 0 - /*********************************************************************************/ /* Allocate and initialize a BSON structure. */ /*********************************************************************************/ @@ -1146,7 +1125,7 @@ static my_bool JsonSubSet(PGLOBAL g) { PPOOLHEADER pph = (PPOOLHEADER)g->Sarea; - pph->To_Free = (OFFSET)((g->Createas) ? g->Createas : sizeof(POOLHEADER)); + pph->To_Free = (g->Saved_Size) ? g->Saved_Size : (size_t)sizeof(POOLHEADER); pph->FreeBlk = g->Sarea_Size - pph->To_Free; return FALSE; } /* end of JsonSubSet */ @@ -1156,7 +1135,7 @@ static my_bool JsonSubSet(PGLOBAL g) /*********************************************************************************/ inline void JsonMemSave(PGLOBAL g) { - g->Createas = (int)((PPOOLHEADER)g->Sarea)->To_Free; + g->Saved_Size = ((PPOOLHEADER)g->Sarea)->To_Free; } /* end of JsonMemSave */ /*********************************************************************************/ @@ -1627,7 +1606,7 @@ static my_bool CheckMemory(PGLOBAL g, UDF_INIT *initid, UDF_ARGS *args, uint n, return true; } // endif SareaAlloc - g->Createas = 0; + g->Saved_Size = 0; g->Xchk = NULL; initid->max_length = rl; } // endif Size @@ -4501,6 +4480,7 @@ my_bool jfile_make_init(UDF_INIT *initid, UDF_ARGS *args, char *message) } // endif CalcLen(args, false, reslen, memlen); + memlen = memlen + 5000; // To take care of not pretty files return JsonInit(initid, args, message, true, reslen, memlen); } // end of jfile_make_init @@ -5781,6 +5761,478 @@ void json_serialize_deinit(UDF_INIT* initid) JsonFreeMem((PGLOBAL)initid->ptr); } // end of json_serialize_deinit +/*********************************************************************************/ +/* Convert a prettiest Json file to Pretty=0. */ +/*********************************************************************************/ +my_bool jfile_convert_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { + unsigned long reslen, memlen; + + if (args->arg_count != 3) { + strcpy(message, "This function must have 3 arguments"); + return true; + } else if (args->arg_type[2] != INT_RESULT) { + strcpy(message, "Third Argument must be an integer (LRECL)"); + return true; + } else for (int i = 0; i < 2; i++) + if (args->arg_type[i] != STRING_RESULT) { + sprintf(message, "Arguments %d must be a string (file name)", i+1); + return true; + } // endif args + + CalcLen(args, false, reslen, memlen); + return JsonInit(initid, args, message, false, reslen, memlen); +} // end of jfile_convert_init + +char *jfile_convert(UDF_INIT* initid, UDF_ARGS* args, char* result, + unsigned long *res_length, char *, char *error) { + char *str, *fn, *ofn; + int lrecl = (int)*(longlong*)args->args[2]; + PGLOBAL g = (PGLOBAL)initid->ptr; + + PlugSubSet(g->Sarea, g->Sarea_Size); + fn = MakePSZ(g, args, 0); + ofn = MakePSZ(g, args, 1); + + if (!g->Xchk) { + JUP* jup = new(g) JUP(g); + + str = jup->UnprettyJsonFile(g, fn, ofn, lrecl); + g->Xchk = str; + } else + str = (char*)g->Xchk; + + if (!str) { + if (g->Message) + str = PlugDup(g, g->Message); + else + str = PlugDup(g, "Unexpected error"); + + } // endif str + + *res_length = strlen(str); + return str; +} // end of jfile_convert + +void jfile_convert_deinit(UDF_INIT* initid) { + JsonFreeMem((PGLOBAL)initid->ptr); +} // end of jfile_convert_deinit + +/* --------------------------------- Class JUP --------------------------------- */ + +#define ARGS MY_MIN(24,len-i),s+MY_MAX(i-3,0) + +/*********************************************************************************/ +/* JUP public constructor. */ +/*********************************************************************************/ +JUP::JUP(PGLOBAL g) { + fs = NULL; + s = buff = NULL; + i = k = len = recl = 0; +} // end of JUP constructor + +/*********************************************************************************/ +/* Copy a json file to another with pretty = 0. */ +/*********************************************************************************/ +char* JUP::UnprettyJsonFile(PGLOBAL g, char *fn, char *outfn, int lrecl) { + char *ret = NULL; + HANDLE hFile; + MEMMAP mm; + + /*******************************************************************************/ + /* Create the mapping file object. */ + /*******************************************************************************/ + hFile = CreateFileMap(g, fn, &mm, MODE_READ, false); + + if (hFile == INVALID_HANDLE_VALUE) { + DWORD rc = GetLastError(); + + if (!(*g->Message)) + sprintf(g->Message, MSG(OPEN_MODE_ERROR), "map", (int)rc, fn); + + return NULL; + } // endif hFile + + /*******************************************************************************/ + /* Get the file size (assuming file is smaller than 4 GB) */ + /*******************************************************************************/ + if (!mm.lenL) { // Empty or deleted file + CloseFileHandle(hFile); + return NULL; + } else + len = (int)mm.lenL; + + if (!mm.memory) { + CloseFileHandle(hFile); + sprintf(g->Message, MSG(MAP_VIEW_ERROR), fn, GetLastError()); + return NULL; + } else + s = (char*)mm.memory; + + CloseFileHandle(hFile); // Not used anymore + + /*********************************************************************************/ + /* Parse the json file and allocate its tree structure. */ + /*********************************************************************************/ + if (!(fs = fopen(outfn, "wb"))) { + sprintf(g->Message, MSG(OPEN_MODE_ERROR), + "w", (int)errno, outfn); + strcat(strcat(g->Message, ": "), strerror(errno)); + CloseMemMap(mm.memory, (size_t)mm.lenL); + return NULL; + } // endif fs + + g->Message[0] = 0; + + if (!unPretty(g, lrecl)) + ret = outfn; + + CloseMemMap(mm.memory, (size_t)mm.lenL); + fclose(fs); + return ret; +} // end of UnprettyJsonFile + +/***********************************************************************/ +/* Translate a json file to pretty = 0. */ +/***********************************************************************/ +bool JUP::unPretty(PGLOBAL g, int lrecl) { + bool go, next, rc = false; + + if (trace(1)) + htrc("UnPretty: s=%.10s len=%zd lrecl=%d\n", s, len, lrecl); + + if (!s || !len) { + strcpy(g->Message, "Void JSON file"); + return true; + } else if (*s != '[') { + // strcpy(g->Message, "JSON file is not an array"); + s = strchr(s, '['); + // return true; + } // endif s + + i = 1; + go = next = true; + + try { + // Allocate the record + buff = (char*)PlugSubAlloc(g, NULL, (size_t)lrecl + 3); + recl = lrecl; + + do { + for (k = 0; go && i < len; i++) + switch (s[i]) { + case '{': + buff[k++] = s[i++]; + CopyObject(g); + break; + case '[': + throw "JSON file is not an array of objects"; + break; + case ' ': + case '\t': + case '\n': + case '\r': + break; + case ',': + go = false; + break; + case ']': + go = next = false; + break; + default: + sprintf(g->Message, "Unexpected '%c' near %.*s", s[i], ARGS); + throw 4; + break; + }; // endswitch s[i] + + // Write the record +#ifdef __win_ + buff[k++] = '\r'; +#endif + buff[k++] = '\n'; + buff[k] = 0; + + if ((fputs(buff, fs)) == EOF) { + sprintf(g->Message, MSG(FPUTS_ERROR), strerror(errno)); + throw 5; + } // endif EOF + + go = true; + } while (next); + + } catch (int n) { + if (trace(1)) + htrc("Exception %d: %s\n", n, g->Message); + rc = true; + } catch (const char* msg) { + strcpy(g->Message, msg); + rc = true; + } // end catch + + return rc; +} // end of unPretty + +/***********************************************************************/ +/* Copy a JSON Object. */ +/***********************************************************************/ +void JUP::CopyObject(PGLOBAL g) { + int level = 0; + + for (; i < len; i++) + switch (s[i]) { + case '"': + AddBuff(s[i++]); + + if (level < 2) { + CopyString(g); + level = 1; + } else { + sprintf(g->Message, "misplaced string near %.*s", ARGS); + throw 3; + } // endif level + + break; + case ':': + AddBuff(s[i++]); + + if (level == 1) { + CopyValue(g); + level = 2; + } else { + sprintf(g->Message, "Unexpected ':' near %.*s", ARGS); + throw 3; + } // endif level + + break; + case ',': + AddBuff(s[i]); + + if (level < 2) { + sprintf(g->Message, "Unexpected ',' near %.*s", ARGS); + throw 3; + } else + level = 0; + + break; + case '}': + AddBuff(s[i]); + + if (level == 1) { + sprintf(g->Message, "Unexpected '}' near %.*s", ARGS); + throw 3; + } // endif level + + return; + case '\n': + case '\r': + case ' ': + case '\t': + break; + default: + sprintf(g->Message, "Unexpected character '%c' near %.*s", s[i], ARGS); + throw 3; + }; // endswitch s[i] + + throw "Unexpected EOF in Object"; +} // end of CopyObject + +/***********************************************************************/ +/* Copy a JSON Array. */ +/***********************************************************************/ +void JUP::CopyArray(PGLOBAL g) { + int level = 0; + + for (; i < len; i++) + switch (s[i]) { + case ',': + if (level < 2) { + sprintf(g->Message, "Unexpected ',' near %.*s", ARGS); + throw 2; + } else + level = 1; + + AddBuff(s[i]); + break; + case ']': + if (level == 1) { + sprintf(g->Message, "Unexpected ',]' near %.*s", ARGS); + throw 2; + } // endif level + + AddBuff(s[i]); + return; + case '\n': + case '\r': + case ' ': + case '\t': + break; + default: + if (level == 2) { + sprintf(g->Message, "Unexpected value near %.*s", ARGS); + throw 2; + } // endif level + + CopyValue(g); + level = 2; + break; + }; // endswitch s[i] + + throw "Unexpected EOF in array"; +} // end of CopyArray + +/***********************************************************************/ +/* Copy a JSON Value. */ +/***********************************************************************/ +void JUP::CopyValue(PGLOBAL g) { + for (; i < len; i++) + switch (s[i]) { + case '\n': + case '\r': + case ' ': + case '\t': + break; + default: + goto suite; + } // endswitch + +suite: + switch (s[i]) { + case '[': + AddBuff(s[i++]); + CopyArray(g); + break; + case '{': + AddBuff(s[i++]); + CopyObject(g); + break; + case '"': + AddBuff(s[i++]); + CopyString(g); + break; + case 't': + if (!strncmp(s + i, "true", 4)) { + AddBuff(s[i++]); + AddBuff(s[i++]); + AddBuff(s[i++]); + AddBuff(s[i]); + } else + goto err; + + break; + case 'f': + if (!strncmp(s + i, "false", 5)) { + AddBuff(s[i++]); + AddBuff(s[i++]); + AddBuff(s[i++]); + AddBuff(s[i++]); + AddBuff(s[i]); + } else + goto err; + + break; + case 'n': + if (!strncmp(s + i, "null", 4)) { + AddBuff(s[i++]); + AddBuff(s[i++]); + AddBuff(s[i++]); + AddBuff(s[i]); + } else + goto err; + + break; + default: + if (s[i] == '-' || isdigit(s[i])) + CopyNumeric(g); + else + goto err; + + }; // endswitch s[i] + + return; + +err: + sprintf(g->Message, "Unexpected character '%c' near %.*s", s[i], ARGS); + throw 1; +} // end of CopyValue + +/***********************************************************************/ +/* Unescape and parse a JSON string. */ +/***********************************************************************/ +void JUP::CopyString(PGLOBAL g) { + for (; i < len; i++) { + AddBuff(s[i]); + + switch (s[i]) { + case '"': + return; + case '\\': + AddBuff(s[++i]); + break; + default: + break; + }; // endswitch s[i] + + } // endfor i + + throw "Unexpected EOF in String"; +} // end of CopyString + +/***********************************************************************/ +/* Copy a JSON numeric value. */ +/***********************************************************************/ +void JUP::CopyNumeric(PGLOBAL g) { + bool has_dot = false; + bool has_e = false; + bool found_digit = false; + + for (; i < len; i++) { + switch (s[i]) { + case '.': + if (!found_digit || has_dot || has_e) + goto err; + + has_dot = true; + break; + case 'e': + case 'E': + if (!found_digit || has_e) + goto err; + + has_e = true; + found_digit = false; + break; + case '+': + if (!has_e) + goto err; + + // fall through + case '-': + if (found_digit) + goto err; + + break; + default: + if (isdigit(s[i])) { + found_digit = true; + } else + goto fin; + + }; // endswitch s[i] + + AddBuff(s[i]); + } // endfor i + +fin: + if (!found_digit) + throw "No digit found"; + else + i--; + + return; + +err: + throw "Unexpected EOF in number"; +} // end of CopyNumeric + /*********************************************************************************/ /* Utility function returning an environment variable value. */ /*********************************************************************************/ diff --git a/storage/connect/jsonudf.h b/storage/connect/jsonudf.h index ee56869a111..897b0fe9919 100644 --- a/storage/connect/jsonudf.h +++ b/storage/connect/jsonudf.h @@ -235,6 +235,10 @@ extern "C" { DllExport char *json_serialize(UDF_EXEC_ARGS); DllExport void json_serialize_deinit(UDF_INIT*); + DllExport my_bool jfile_convert_init(UDF_INIT*, UDF_ARGS*, char*); + DllExport char* jfile_convert(UDF_EXEC_ARGS); + DllExport void jfile_convert_deinit(UDF_INIT*); + DllExport my_bool envar_init(UDF_INIT*, UDF_ARGS*, char*); DllExport char *envar(UDF_EXEC_ARGS); @@ -324,3 +328,38 @@ protected: my_bool Wr; // Write mode my_bool Jb; // Must return json item }; // end of class JSNX + +/*********************************************************************************/ +/* Class JUP: used by jfile_convert to make a json file pretty = 0. */ +/*********************************************************************************/ +class JUP : public BLOCK { +public: + // Constructor + JUP(PGLOBAL g); + + // Implementation + void AddBuff(char c) { + if (k < recl) + buff[k++] = c; + else + throw "Record size is too small"; + } // end of AddBuff + + // Methods + char *UnprettyJsonFile(PGLOBAL g, char* fn, char* outfn, int lrecl); + bool unPretty(PGLOBAL g, int lrecl); + void CopyObject(PGLOBAL g); + void CopyArray(PGLOBAL g); + void CopyValue(PGLOBAL g); + void CopyString(PGLOBAL g); + void CopyNumeric(PGLOBAL g); + + // Members + FILE* fs; + char* s; + char* buff; + int len; + int recl; + int i, k; +}; // end of class JUP + diff --git a/storage/connect/mongo.cpp b/storage/connect/mongo.cpp index e821440a0c3..5f10a89ee67 100644 --- a/storage/connect/mongo.cpp +++ b/storage/connect/mongo.cpp @@ -35,6 +35,7 @@ bool MakeSelector(PGLOBAL g, PFIL fp, PSTRG s); bool IsNum(PSZ s); +int GetDefaultDepth(void); /***********************************************************************/ /* Make selector json representation for Mongo tables. */ @@ -248,15 +249,10 @@ MGODISC::MGODISC(PGLOBAL g, int *lg) { /***********************************************************************/ int MGODISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ uri, PTOS topt) { - PCSZ level = GetStringTableOption(g, topt, "Level", NULL); PMGODEF tdp; - if (level = GetStringTableOption(g, topt, "Depth", level)) { - lvl = atoi(level); - lvl = (lvl > 16) ? 16 : lvl; - } else - lvl = 0; - + lvl = GetIntegerTableOption(g, topt, "Level", GetDefaultDepth()); + lvl = GetIntegerTableOption(g, topt, "Depth", lvl); all = GetBooleanTableOption(g, topt, "Fullarray", false); /*********************************************************************/ diff --git a/storage/connect/mysql-test/connect/r/json_java_2.result b/storage/connect/mysql-test/connect/r/json_java_2.result index 4bbac236200..47fc4abbd28 100644 --- a/storage/connect/mysql-test/connect/r/json_java_2.result +++ b/storage/connect/mysql-test/connect/r/json_java_2.result @@ -20,12 +20,12 @@ SELECT * from t1; Column_Name Data_Type Type_Name Column_Size Buffer_Length Decimal_Digits Nullable Jpath _id 1 CHAR 24 24 0 0 _id address_building 1 CHAR 10 10 0 0 address.building -address_coord 1 CHAR 256 256 0 1 address.coord +address_coord 1 CHAR 1024 1024 0 1 address.coord address_street 1 CHAR 38 38 0 0 address.street address_zipcode 1 CHAR 5 5 0 0 address.zipcode borough 1 CHAR 13 13 0 0 cuisine 1 CHAR 64 64 0 0 -grades_date 1 CHAR 256 256 0 1 grades.0.date +grades_date 1 CHAR 1024 1024 0 1 grades.0.date grades_grade 1 CHAR 14 14 0 1 grades.0.grade grades_score 5 BIGINT 2 2 0 1 grades.0.score name 1 CHAR 98 98 0 0 @@ -64,16 +64,16 @@ OPTION_LIST='Level=1,Driver=Java,Version=2' CONNECTION='mongodb://localhost:2701 SHOW CREATE TABLE t1; Table Create Table t1 CREATE TABLE `t1` ( - `_id` char(24) NOT NULL `FIELD_FORMAT`='_id', - `address_building` char(10) NOT NULL `FIELD_FORMAT`='address.building', - `address_coord` varchar(256) DEFAULT NULL `FIELD_FORMAT`='address.coord', - `address_street` char(38) NOT NULL `FIELD_FORMAT`='address.street', - `address_zipcode` char(5) NOT NULL `FIELD_FORMAT`='address.zipcode', + `_id` char(24) NOT NULL `JPATH`='_id', + `address_building` char(10) NOT NULL `JPATH`='address.building', + `address_coord` varchar(1024) DEFAULT NULL `JPATH`='address.coord', + `address_street` char(38) NOT NULL `JPATH`='address.street', + `address_zipcode` char(5) NOT NULL `JPATH`='address.zipcode', `borough` char(13) NOT NULL, `cuisine` char(64) NOT NULL, - `grades_date` varchar(256) DEFAULT NULL `FIELD_FORMAT`='grades.0.date', - `grades_grade` char(14) DEFAULT NULL `FIELD_FORMAT`='grades.0.grade', - `grades_score` bigint(2) DEFAULT NULL `FIELD_FORMAT`='grades.0.score', + `grades_date` varchar(1024) DEFAULT NULL `JPATH`='grades.0.date', + `grades_grade` char(14) DEFAULT NULL `JPATH`='grades.0.grade', + `grades_score` bigint(2) DEFAULT NULL `JPATH`='grades.0.score', `name` char(98) NOT NULL, `restaurant_id` char(8) NOT NULL ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='mongodb://localhost:27017' `TABLE_TYPE`='JSON' `TABNAME`='restaurants' `OPTION_LIST`='Level=1,Driver=Java,Version=2' `DATA_CHARSET`='utf8' `LRECL`=4096 @@ -251,15 +251,15 @@ OPTION_LIST='Driver=Java,level=2,version=2'; SHOW CREATE TABLE t1; Table Create Table t1 CREATE TABLE `t1` ( - `_id` char(24) NOT NULL `FIELD_FORMAT`='_id', - `address_building` char(10) NOT NULL `FIELD_FORMAT`='address.building', - `address_coord` double(18,16) DEFAULT NULL `FIELD_FORMAT`='address.coord.0', - `address_street` char(38) NOT NULL `FIELD_FORMAT`='address.street', - `address_zipcode` char(5) NOT NULL `FIELD_FORMAT`='address.zipcode', + `_id` char(24) NOT NULL `JPATH`='_id', + `address_building` char(10) NOT NULL `JPATH`='address.building', + `address_coord` double(18,16) DEFAULT NULL `JPATH`='address.coord.0', + `address_street` char(38) NOT NULL `JPATH`='address.street', + `address_zipcode` char(5) NOT NULL `JPATH`='address.zipcode', `borough` char(13) NOT NULL, - `grades_date` char(24) DEFAULT NULL `FIELD_FORMAT`='grades.0.date', - `grades_grade` char(14) DEFAULT NULL `FIELD_FORMAT`='grades.0.grade', - `grades_score` bigint(2) DEFAULT NULL `FIELD_FORMAT`='grades.0.score', + `grades_date` char(24) DEFAULT NULL `JPATH`='grades.0.date', + `grades_grade` char(14) DEFAULT NULL `JPATH`='grades.0.grade', + `grades_score` bigint(2) DEFAULT NULL `JPATH`='grades.0.score', `name` char(98) NOT NULL, `restaurant_id` char(8) NOT NULL ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='mongodb://localhost:27017' `TABLE_TYPE`='JSON' `TABNAME`='restaurants' `COLIST`='{"cuisine":0}' `FILTER`='{"cuisine":"French","borough":{"$ne":"Manhattan"}}' `OPTION_LIST`='Driver=Java,level=2,version=2' `LRECL`=4096 diff --git a/storage/connect/mysql-test/connect/r/json_java_3.result b/storage/connect/mysql-test/connect/r/json_java_3.result index eb8bfc022d6..720c82cd7f9 100644 --- a/storage/connect/mysql-test/connect/r/json_java_3.result +++ b/storage/connect/mysql-test/connect/r/json_java_3.result @@ -20,12 +20,12 @@ SELECT * from t1; Column_Name Data_Type Type_Name Column_Size Buffer_Length Decimal_Digits Nullable Jpath _id 1 CHAR 24 24 0 0 _id address_building 1 CHAR 10 10 0 0 address.building -address_coord 1 CHAR 256 256 0 1 address.coord +address_coord 1 CHAR 1024 1024 0 1 address.coord address_street 1 CHAR 38 38 0 0 address.street address_zipcode 1 CHAR 5 5 0 0 address.zipcode borough 1 CHAR 13 13 0 0 cuisine 1 CHAR 64 64 0 0 -grades_date 1 CHAR 256 256 0 1 grades.0.date +grades_date 1 CHAR 1024 1024 0 1 grades.0.date grades_grade 1 CHAR 14 14 0 1 grades.0.grade grades_score 5 BIGINT 2 2 0 1 grades.0.score name 1 CHAR 98 98 0 0 @@ -64,16 +64,16 @@ OPTION_LIST='Level=1,Driver=Java,Version=3' CONNECTION='mongodb://localhost:2701 SHOW CREATE TABLE t1; Table Create Table t1 CREATE TABLE `t1` ( - `_id` char(24) NOT NULL `FIELD_FORMAT`='_id', - `address_building` char(10) NOT NULL `FIELD_FORMAT`='address.building', - `address_coord` varchar(256) DEFAULT NULL `FIELD_FORMAT`='address.coord', - `address_street` char(38) NOT NULL `FIELD_FORMAT`='address.street', - `address_zipcode` char(5) NOT NULL `FIELD_FORMAT`='address.zipcode', + `_id` char(24) NOT NULL `JPATH`='_id', + `address_building` char(10) NOT NULL `JPATH`='address.building', + `address_coord` varchar(1024) DEFAULT NULL `JPATH`='address.coord', + `address_street` char(38) NOT NULL `JPATH`='address.street', + `address_zipcode` char(5) NOT NULL `JPATH`='address.zipcode', `borough` char(13) NOT NULL, `cuisine` char(64) NOT NULL, - `grades_date` varchar(256) DEFAULT NULL `FIELD_FORMAT`='grades.0.date', - `grades_grade` char(14) DEFAULT NULL `FIELD_FORMAT`='grades.0.grade', - `grades_score` bigint(2) DEFAULT NULL `FIELD_FORMAT`='grades.0.score', + `grades_date` varchar(1024) DEFAULT NULL `JPATH`='grades.0.date', + `grades_grade` char(14) DEFAULT NULL `JPATH`='grades.0.grade', + `grades_score` bigint(2) DEFAULT NULL `JPATH`='grades.0.score', `name` char(98) NOT NULL, `restaurant_id` char(8) NOT NULL ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='mongodb://localhost:27017' `TABLE_TYPE`='JSON' `TABNAME`='restaurants' `OPTION_LIST`='Level=1,Driver=Java,Version=3' `DATA_CHARSET`='utf8' `LRECL`=4096 @@ -251,15 +251,15 @@ OPTION_LIST='Driver=Java,level=2,version=3'; SHOW CREATE TABLE t1; Table Create Table t1 CREATE TABLE `t1` ( - `_id` char(24) NOT NULL `FIELD_FORMAT`='_id', - `address_building` char(10) NOT NULL `FIELD_FORMAT`='address.building', - `address_coord` double(18,16) DEFAULT NULL `FIELD_FORMAT`='address.coord.0', - `address_street` char(38) NOT NULL `FIELD_FORMAT`='address.street', - `address_zipcode` char(5) NOT NULL `FIELD_FORMAT`='address.zipcode', + `_id` char(24) NOT NULL `JPATH`='_id', + `address_building` char(10) NOT NULL `JPATH`='address.building', + `address_coord` double(18,16) DEFAULT NULL `JPATH`='address.coord.0', + `address_street` char(38) NOT NULL `JPATH`='address.street', + `address_zipcode` char(5) NOT NULL `JPATH`='address.zipcode', `borough` char(13) NOT NULL, - `grades_date` bigint(13) DEFAULT NULL `FIELD_FORMAT`='grades.0.date', - `grades_grade` char(14) DEFAULT NULL `FIELD_FORMAT`='grades.0.grade', - `grades_score` bigint(2) DEFAULT NULL `FIELD_FORMAT`='grades.0.score', + `grades_date` bigint(13) DEFAULT NULL `JPATH`='grades.0.date', + `grades_grade` char(14) DEFAULT NULL `JPATH`='grades.0.grade', + `grades_score` bigint(2) DEFAULT NULL `JPATH`='grades.0.score', `name` char(98) NOT NULL, `restaurant_id` char(8) NOT NULL ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='mongodb://localhost:27017' `TABLE_TYPE`='JSON' `TABNAME`='restaurants' `COLIST`='{"cuisine":0}' `FILTER`='{"cuisine":"French","borough":{"$ne":"Manhattan"}}' `OPTION_LIST`='Driver=Java,level=2,version=3' `LRECL`=4096 diff --git a/storage/connect/mysql-test/connect/r/json_mongo_c.result b/storage/connect/mysql-test/connect/r/json_mongo_c.result index 550e94f286e..f9bfc01763e 100644 --- a/storage/connect/mysql-test/connect/r/json_mongo_c.result +++ b/storage/connect/mysql-test/connect/r/json_mongo_c.result @@ -20,12 +20,12 @@ SELECT * from t1; Column_Name Data_Type Type_Name Column_Size Buffer_Length Decimal_Digits Nullable Jpath _id 1 CHAR 24 24 0 0 _id address_building 1 CHAR 10 10 0 0 address.building -address_coord 1 CHAR 256 256 0 1 address.coord +address_coord 1 CHAR 1024 1024 0 1 address.coord address_street 1 CHAR 38 38 0 0 address.street address_zipcode 1 CHAR 5 5 0 0 address.zipcode borough 1 CHAR 13 13 0 0 cuisine 1 CHAR 64 64 0 0 -grades_date 1 CHAR 256 256 0 1 grades.0.date +grades_date 1 CHAR 1024 1024 0 1 grades.0.date grades_grade 1 CHAR 14 14 0 1 grades.0.grade grades_score 5 BIGINT 2 2 0 1 grades.0.score name 1 CHAR 98 98 0 0 @@ -64,16 +64,16 @@ OPTION_LIST='Level=1,Driver=C,Version=0' CONNECTION='mongodb://localhost:27017' SHOW CREATE TABLE t1; Table Create Table t1 CREATE TABLE `t1` ( - `_id` char(24) NOT NULL `FIELD_FORMAT`='_id', - `address_building` char(10) NOT NULL `FIELD_FORMAT`='address.building', - `address_coord` varchar(256) DEFAULT NULL `FIELD_FORMAT`='address.coord', - `address_street` char(38) NOT NULL `FIELD_FORMAT`='address.street', - `address_zipcode` char(5) NOT NULL `FIELD_FORMAT`='address.zipcode', + `_id` char(24) NOT NULL `JPATH`='_id', + `address_building` char(10) NOT NULL `JPATH`='address.building', + `address_coord` varchar(1024) DEFAULT NULL `JPATH`='address.coord', + `address_street` char(38) NOT NULL `JPATH`='address.street', + `address_zipcode` char(5) NOT NULL `JPATH`='address.zipcode', `borough` char(13) NOT NULL, `cuisine` char(64) NOT NULL, - `grades_date` varchar(256) DEFAULT NULL `FIELD_FORMAT`='grades.0.date', - `grades_grade` char(14) DEFAULT NULL `FIELD_FORMAT`='grades.0.grade', - `grades_score` bigint(2) DEFAULT NULL `FIELD_FORMAT`='grades.0.score', + `grades_date` varchar(1024) DEFAULT NULL `JPATH`='grades.0.date', + `grades_grade` char(14) DEFAULT NULL `JPATH`='grades.0.grade', + `grades_score` bigint(2) DEFAULT NULL `JPATH`='grades.0.score', `name` char(98) NOT NULL, `restaurant_id` char(8) NOT NULL ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='mongodb://localhost:27017' `TABLE_TYPE`='JSON' `TABNAME`='restaurants' `OPTION_LIST`='Level=1,Driver=C,Version=0' `DATA_CHARSET`='utf8' `LRECL`=1024 @@ -251,15 +251,15 @@ OPTION_LIST='Driver=C,level=2,version=0'; SHOW CREATE TABLE t1; Table Create Table t1 CREATE TABLE `t1` ( - `_id` char(24) NOT NULL `FIELD_FORMAT`='_id', - `address_building` char(10) NOT NULL `FIELD_FORMAT`='address.building', - `address_coord` double(23,20) DEFAULT NULL `FIELD_FORMAT`='address.coord.0', - `address_street` char(38) NOT NULL `FIELD_FORMAT`='address.street', - `address_zipcode` char(5) NOT NULL `FIELD_FORMAT`='address.zipcode', + `_id` char(24) NOT NULL `JPATH`='_id', + `address_building` char(10) NOT NULL `JPATH`='address.building', + `address_coord` double(23,20) DEFAULT NULL `JPATH`='address.coord.0', + `address_street` char(38) NOT NULL `JPATH`='address.street', + `address_zipcode` char(5) NOT NULL `JPATH`='address.zipcode', `borough` char(13) NOT NULL, - `grades_date` bigint(13) DEFAULT NULL `FIELD_FORMAT`='grades.0.date', - `grades_grade` char(14) DEFAULT NULL `FIELD_FORMAT`='grades.0.grade', - `grades_score` bigint(2) DEFAULT NULL `FIELD_FORMAT`='grades.0.score', + `grades_date` bigint(13) DEFAULT NULL `JPATH`='grades.0.date', + `grades_grade` char(14) DEFAULT NULL `JPATH`='grades.0.grade', + `grades_score` bigint(2) DEFAULT NULL `JPATH`='grades.0.score', `name` char(98) NOT NULL, `restaurant_id` char(8) NOT NULL ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='mongodb://localhost:27017' `TABLE_TYPE`='JSON' `TABNAME`='restaurants' `COLIST`='{"projection":{"cuisine":0}}' `FILTER`='{"cuisine":"French","borough":{"$ne":"Manhattan"}}' `OPTION_LIST`='Driver=C,level=2,version=0' `LRECL`=1024 diff --git a/storage/connect/plugutil.cpp b/storage/connect/plugutil.cpp index 2517b76cd3e..0ab594f5533 100644 --- a/storage/connect/plugutil.cpp +++ b/storage/connect/plugutil.cpp @@ -158,13 +158,14 @@ PGLOBAL PlugInit(LPCSTR Language, size_t worksize) } // end try/catch g->Sarea = NULL; - g->Createas = 0; + g->Createas = false; g->Alchecked = 0; g->Mrr = 0; g->Activityp = NULL; g->Xchk = NULL; g->N = 0; g->More = 0; + g->Saved_Size = 0; strcpy(g->Message, ""); /*******************************************************************/ @@ -528,7 +529,7 @@ BOOL PlugSubSet(void *memp, size_t size) { PPOOLHEADER pph = (PPOOLHEADER)memp; - pph->To_Free = (OFFSET)sizeof(POOLHEADER); + pph->To_Free = (size_t)sizeof(POOLHEADER); pph->FreeBlk = size - pph->To_Free; return FALSE; } /* end of PlugSubSet */ @@ -580,7 +581,7 @@ void *PlugSubAlloc(PGLOBAL g, void *memp, size_t size) /* Do the suballocation the simplest way. */ /*********************************************************************/ memp = MakePtr(memp, pph->To_Free); /* Points to suballocated block */ - pph->To_Free += (OFFSET)size; /* New offset of pool free block */ + pph->To_Free += size; /* New offset of pool free block */ pph->FreeBlk -= size; /* New size of pool free block */ if (trace(16)) @@ -605,40 +606,4 @@ char *PlugDup(PGLOBAL g, const char *str) } // end of PlugDup -#if 0 -/***********************************************************************/ -/* This routine suballocate a copy of the passed string. */ -/***********************************************************************/ -char *PlugDup(PGLOBAL g, const char *str) - { - char *buf; - size_t len; - - if (str && (len = strlen(str))) { - buf = (char*)PlugSubAlloc(g, NULL, len + 1); - strcpy(buf, str); - } else - buf = NULL; - - return(buf); - } /* end of PlugDup */ -#endif // 0 - -/***********************************************************************/ -/* This routine makes a pointer from an offset to a memory pointer. */ -/***********************************************************************/ -void *MakePtr(void *memp, OFFSET offset) - { - return ((offset == 0) ? NULL : &((char *)memp)[offset]); - } /* end of MakePtr */ - -/***********************************************************************/ -/* This routine makes an offset from a pointer new format. */ -/***********************************************************************/ -#if 0 -OFFSET MakeOff(void *memp, void *ptr) - { - return ((!ptr) ? 0 : (OFFSET)((char *)ptr - (char *)memp)); - } /* end of MakeOff */ -#endif /*--------------------- End of PLUGUTIL program -----------------------*/ diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp index fcf7f4d182c..cdf9e40f97c 100644 --- a/storage/connect/tabjson.cpp +++ b/storage/connect/tabjson.cpp @@ -52,19 +52,10 @@ /* External functions. */ /***********************************************************************/ USETEMP UseTemp(void); +bool JsonAllPath(void); +int GetDefaultDepth(void); char *GetJsonNull(void); -//typedef struct _jncol { -// struct _jncol *Next; -// char *Name; -// char *Fmt; -// int Type; -// int Len; -// int Scale; -// bool Cbn; -// bool Found; -//} JCOL, *PJCL; - /***********************************************************************/ /* JSONColumns: construct the result blocks containing the description */ /* of all the columns of a table contained inside a JSON file. */ @@ -167,23 +158,20 @@ JSONDISC::JSONDISC(PGLOBAL g, uint *lg) jsp = NULL; row = NULL; sep = NULL; - i = n = bf = ncol = lvl = 0; - all = false; + i = n = bf = ncol = lvl = sz = 0; + all = strfy = false; } // end of JSONDISC constructor int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt) { char filename[_MAX_PATH]; bool mgo = (GetTypeID(topt->type) == TAB_MONGO); - PCSZ level = GetStringTableOption(g, topt, "Level", NULL); - - if ((level = GetStringTableOption(g, topt, "Depth", level))) { - lvl = atoi(level); - lvl = (lvl > 16) ? 16 : lvl; - } else - lvl = 0; - sep = GetStringTableOption(g, topt, "Separator", "."); + lvl = GetIntegerTableOption(g, topt, "Level", GetDefaultDepth()); + lvl = GetIntegerTableOption(g, topt, "Depth", lvl); + sep = GetStringTableOption(g, topt, "Separator", "."); + sz = GetIntegerTableOption(g, topt, "Jsize", 1024); + strfy = GetBooleanTableOption(g, topt, "Stringify", false); /*********************************************************************/ /* Open the input file. */ @@ -306,7 +294,7 @@ int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt) // Allocate the parse work memory PGLOBAL G = (PGLOBAL)PlugSubAlloc(g, NULL, sizeof(GLOBAL)); memset(G, 0, sizeof(GLOBAL)); - G->Sarea_Size = tdp->Lrecl * 10; + G->Sarea_Size = (size_t)tdp->Lrecl * 10; G->Sarea = PlugSubAlloc(g, NULL, G->Sarea_Size); PlugSubSet(G->Sarea, G->Sarea_Size); G->jump_level = 0; @@ -403,7 +391,10 @@ bool JSONDISC::Find(PGLOBAL g, PJVAL jvp, PCSZ key, int j) PJAR jar; if ((valp = jvp ? jvp->GetValue() : NULL)) { - jcol.Type = valp->GetType(); + if (JsonAllPath() && !fmt[bf]) + strcat(fmt, colname); + + jcol.Type = valp->GetType(); jcol.Len = valp->GetValLen(); jcol.Scale = valp->GetValPrec(); jcol.Cbn = valp->IsNull(); @@ -482,8 +473,16 @@ bool JSONDISC::Find(PGLOBAL g, PJVAL jvp, PCSZ key, int j) } // endswitch Type } else if (lvl >= 0) { - jcol.Type = TYPE_STRING; - jcol.Len = 256; + if (strfy) { + if (!fmt[bf]) + strcat(fmt, colname); + + strcat(fmt, ".*"); + } else if (JsonAllPath() && !fmt[bf]) + strcat(fmt, colname); + + jcol.Type = TYPE_STRING; + jcol.Len = sz; jcol.Scale = 0; jcol.Cbn = true; } else @@ -2040,7 +2039,7 @@ int TDBJSON::MakeDocument(PGLOBAL g) if ((objpath = PlugDup(g, Objname))) { if (*objpath == '$') objpath++; if (*objpath == '.') objpath++; - p1 = p2 = NULL; + p1 = (*objpath == '[') ? objpath++ : NULL; /*********************************************************************/ /* Find the table in the tree structure. */ diff --git a/storage/connect/tabjson.h b/storage/connect/tabjson.h index 8c3f1013919..88aa5e2ee8b 100644 --- a/storage/connect/tabjson.h +++ b/storage/connect/tabjson.h @@ -68,8 +68,8 @@ public: PCSZ sep; char colname[65], fmt[129], buf[16]; uint *length; - int i, n, bf, ncol, lvl; - bool all; + int i, n, bf, ncol, lvl, sz; + bool all, strfy; }; // end of JSONDISC /***********************************************************************/ diff --git a/storage/connect/tabxml.cpp b/storage/connect/tabxml.cpp index fa140ea0699..6c9e9597cec 100644 --- a/storage/connect/tabxml.cpp +++ b/storage/connect/tabxml.cpp @@ -3,7 +3,7 @@ /* ------------- */ /* Version 3.0 */ /* */ -/* Author Olivier BERTRAND 2007 - 2017 */ +/* Author Olivier BERTRAND 2007 - 2020 */ /* */ /* This program are the XML tables classes using MS-DOM or libxml2. */ /***********************************************************************/ @@ -62,6 +62,8 @@ extern "C" char version[]; #define TYPE_UNKNOWN 12 /* Must be greater than other types */ #define XLEN(M) sizeof(M) - strlen(M) - 1 /* To avoid overflow*/ +int GetDefaultDepth(void); + /***********************************************************************/ /* Class and structure used by XMLColumns. */ /***********************************************************************/ @@ -149,7 +151,7 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info) strcpy(g->Message, MSG(MISSING_FNAME)); return NULL; } else { - lvl = GetIntegerTableOption(g, topt, "Level", 0); + lvl = GetIntegerTableOption(g, topt, "Level", GetDefaultDepth()); lvl = GetIntegerTableOption(g, topt, "Depth", lvl); lvl = (lvl < 0) ? 0 : (lvl > 16) ? 16 : lvl; } // endif fn diff --git a/storage/connect/user_connect.cc b/storage/connect/user_connect.cc index 5dbdb8f56f0..09d6db1ad27 100644 --- a/storage/connect/user_connect.cc +++ b/storage/connect/user_connect.cc @@ -158,16 +158,16 @@ bool user_connect::CheckCleanup(bool force) { if (thdp->query_id > last_query_id || force) { size_t worksize = GetWorkSize(); - size_t size = g->Sarea_Size; PlugCleanup(g, true); - if (size != worksize) { + if (worksize != g->Sarea_Size) { FreeSarea(g); + g->Saved_Size = g->Sarea_Size; // Check whether the work area could be allocated if (AllocSarea(g, worksize)) { - AllocSarea(g, size); + AllocSarea(g, g->Saved_Size); SetWorkSize(g->Sarea_Size); // Was too big } // endif sarea @@ -175,10 +175,11 @@ bool user_connect::CheckCleanup(bool force) PlugSubSet(g->Sarea, g->Sarea_Size); g->Xchk = NULL; - g->Createas = 0; + g->Createas = false; g->Alchecked = 0; g->Mrr = 0; g->More = 0; + g->Saved_Size = 0; last_query_id= thdp->query_id; if (trace(65) && !force) -- cgit v1.2.1