diff options
| author | Oleksandr Byelkin <sanja@mariadb.com> | 2020-10-26 09:08:44 +0100 |
|---|---|---|
| committer | Oleksandr Byelkin <sanja@mariadb.com> | 2020-10-26 09:08:44 +0100 |
| commit | 784473b986625c25a7ab5b019bbbb9192102c731 (patch) | |
| tree | 25794505166a168a2511f3882d676ccba454f2ff | |
| parent | 2fdc50367cb31526a8fe2f5b2c89e71350cd4b32 (diff) | |
| parent | dc3a693b7039cb4b1d46d797ca86364ac3173be7 (diff) | |
| download | mariadb-git-784473b986625c25a7ab5b019bbbb9192102c731.tar.gz | |
Merge remote-tracking branch 'connect/10.2' into 10.2
23 files changed, 1071 insertions, 490 deletions
diff --git a/storage/connect/global.h b/storage/connect/global.h index fd26c87b800..d17620861fa 100644 --- a/storage/connect/global.h +++ b/storage/connect/global.h @@ -1,7 +1,7 @@ /***********************************************************************/ /* GLOBAL.H: Declaration file used by all CONNECT implementations. */ /* (C) Copyright MariaDB Corporation Ab */ -/* Author Olivier Bertrand 1993-2018 */ +/* Author Olivier Bertrand 1993-2020 */ /***********************************************************************/ /***********************************************************************/ @@ -89,14 +89,10 @@ extern "C" { #define PAT_LOG "log" #if defined(UNIX) || defined(LINUX) || defined(UNIV_LINUX) - /*********************************************************************/ - /* printf does not accept null pointer for %s target. */ - /*********************************************************************/ + // printf does not accept null pointer for %s target #define SVP(S) ((S) ? S : "<null>") #else - /*********************************************************************/ - /* printf accepts null pointer for %s target. */ - /*********************************************************************/ + // printf accepts null pointer for %s target #define SVP(S) S #endif @@ -112,9 +108,6 @@ extern "C" { /***********************************************************************/ #include "os.h" -typedef uint OFFSET; -typedef char NAME[9]; - typedef struct { ushort Length; char String[2]; @@ -127,6 +120,7 @@ typedef struct _global *PGLOBAL; typedef struct _globplg *PGS; typedef struct _activity *PACTIVITY; typedef struct _parm *PPARM; +typedef char NAME[9]; /***********************************************************************/ /* Segment Sub-Allocation block structure declares. */ @@ -135,8 +129,8 @@ typedef struct _parm *PPARM; /* restore them if needed. This scheme implies that no SubFree be used */ /***********************************************************************/ typedef struct { /* Plug Area SubAlloc header */ - OFFSET To_Free; /* Offset of next free block */ - uint FreeBlk; /* Size of remaining free memory */ + size_t To_Free; /* Offset of next free block */ + size_t FreeBlk; /* Size of remaining free memory */ } POOLHEADER, *PPOOLHEADER; /***********************************************************************/ @@ -188,11 +182,12 @@ typedef struct _parm { /***********************************************************************/ typedef struct _global { /* Global structure */ void *Sarea; /* Points to work area */ - uint Sarea_Size; /* Work area size */ + size_t Sarea_Size; /* Work area size */ PACTIVITY Activityp; - char Message[MAX_STR]; + char Message[MAX_STR]; /* Message (result, error, trace) */ ulong More; /* Used by jsonudf */ - int Createas; /* To pass multi to ext tables */ + size_t Saved_Size; /* Saved work area to_free */ + bool Createas; /* To pass multi to ext tables */ void *Xchk; /* indexes in create/alter */ short Alchecked; /* Checked for ALTER */ short Mrr; /* True when doing mrr */ @@ -210,19 +205,18 @@ DllExport char *PlugReadMessage(PGLOBAL, int, char *); DllExport char *PlugGetMessage(PGLOBAL, int); #endif // XMSG || NEWMSG #if defined(__WIN__) -DllExport short GetLineLength(PGLOBAL); // Console line length +DllExport short GetLineLength(PGLOBAL); // Console line length #endif // __WIN__ -DllExport PGLOBAL PlugInit(LPCSTR, uint); // Plug global initialization -DllExport int PlugExit(PGLOBAL); // Plug global termination +DllExport PGLOBAL PlugInit(LPCSTR, size_t); // Plug global initialization +DllExport int PlugExit(PGLOBAL); // Plug global termination DllExport LPSTR PlugRemoveType(LPSTR, LPCSTR); DllExport LPCSTR PlugSetPath(LPSTR to, LPCSTR prefix, LPCSTR name, LPCSTR dir); DllExport BOOL PlugIsAbsolutePath(LPCSTR path); -DllExport bool AllocSarea(PGLOBAL, uint); +DllExport bool AllocSarea(PGLOBAL, size_t); DllExport void FreeSarea(PGLOBAL); -DllExport BOOL PlugSubSet(void *, uint); +DllExport BOOL PlugSubSet(void *, size_t); DllExport void *PlugSubAlloc(PGLOBAL, void *, size_t); DllExport char *PlugDup(PGLOBAL g, const char *str); -DllExport void *MakePtr(void *, OFFSET); DllExport void htrc(char const *fmt, ...); DllExport void xtrc(uint, char const* fmt, ...); DllExport uint GetTraceValue(void); @@ -232,8 +226,24 @@ DllExport uint GetTraceValue(void); #endif /***********************************************************************/ -/* Non exported routine declarations. */ +/* Inline routine definitions. */ +/***********************************************************************/ +/***********************************************************************/ +/* This routine makes a pointer from an offset to a memory pointer. */ +/***********************************************************************/ +inline void* MakePtr(void* memp, size_t offset) { + // return ((offset == 0) ? NULL : &((char*)memp)[offset]); + return (!offset) ? NULL : (char *)memp + offset; +} /* end of MakePtr */ + +/***********************************************************************/ +/* This routine makes an offset from a pointer new format. */ /***********************************************************************/ -//void *PlugSubAlloc(PGLOBAL, void *, size_t); // Does throw +inline size_t MakeOff(void* memp, void* ptr) { +#if defined(_DEBUG) + assert(ptr > memp); +#endif // _DEBUG + return ((!ptr) ? 0 : (size_t)((char*)ptr - (size_t)memp)); +} /* end of MakeOff */ /*-------------------------- End of Global.H --------------------------*/ diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index 7d53f287f74..738e3d671a0 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -170,9 +170,9 @@ #define JSONMAX 10 // JSON Default max grp size extern "C" { - char version[]= "Version 1.07.0001 November 12, 2019"; + char version[]= "Version 1.07.0002 October 18, 2020"; #if defined(__WIN__) - char compver[]= "Version 1.07.0001 " __DATE__ " " __TIME__; + char compver[]= "Version 1.07.0002 " __DATE__ " " __TIME__; char slash= '\\'; #else // !__WIN__ char slash= '/'; @@ -251,11 +251,13 @@ bool ExactInfo(void); USETEMP UseTemp(void); int GetConvSize(void); TYPCONV GetTypeConv(void); +bool JsonAllPath(void); char *GetJsonNull(void); +int GetDefaultDepth(void); uint GetJsonGrpSize(void); char *GetJavaWrapper(void); -uint GetWorkSize(void); -void SetWorkSize(uint); +size_t GetWorkSize(void); +void SetWorkSize(size_t); extern "C" const char *msglang(void); static char *strz(PGLOBAL g, LEX_STRING &ls); @@ -347,11 +349,19 @@ static MYSQL_THDVAR_ENUM( 1, // def (AUTO) &usetemp_typelib); // typelib +#ifdef _WIN64 // Size used for g->Sarea_Size -static MYSQL_THDVAR_UINT(work_size, - PLUGIN_VAR_RQCMDARG, - "Size of the CONNECT work area.", - NULL, NULL, SZWORK, SZWMIN, UINT_MAX, 1); +static MYSQL_THDVAR_ULONGLONG(work_size, + PLUGIN_VAR_RQCMDARG, + "Size of the CONNECT work area.", + NULL, NULL, SZWORK, SZWMIN, ULONGLONG_MAX, 1); +#else +// Size used for g->Sarea_Size +static MYSQL_THDVAR_ULONG(work_size, + PLUGIN_VAR_RQCMDARG, + "Size of the CONNECT work area.", + NULL, NULL, SZWORK, SZWMIN, ULONG_MAX, 1); +#endif // Size used when converting TEXT columns to VARCHAR static MYSQL_THDVAR_INT(conv_size, @@ -386,6 +396,11 @@ static MYSQL_THDVAR_ENUM( 1, // def (yes) &xconv_typelib); // typelib +// Adding JPATH to all Json table columns +static MYSQL_THDVAR_BOOL(json_all_path, PLUGIN_VAR_RQCMDARG, + "Adding JPATH to all Json table columns", + NULL, NULL, 0); // NO by default + // Null representation for JSON values static MYSQL_THDVAR_STR(json_null, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, @@ -393,6 +408,12 @@ static MYSQL_THDVAR_STR(json_null, // check_json_null, update_json_null, NULL, NULL, "<null>"); +// Default Json, XML or Mongo depth +static MYSQL_THDVAR_INT(default_depth, + PLUGIN_VAR_RQCMDARG, + "Default depth used by Json, XML and Mongo discovery", + NULL, NULL, 0, -1, 16, 1); + // Estimate max number of rows for JSON aggregate functions static MYSQL_THDVAR_UINT(json_grp_size, PLUGIN_VAR_RQCMDARG, // opt @@ -454,15 +475,17 @@ uint GetTraceValue(void) {return (uint)(connect_hton ? THDVAR(current_thd, xtrace) : 0);} bool ExactInfo(void) {return THDVAR(current_thd, exact_info);} static bool CondPushEnabled(void) {return THDVAR(current_thd, cond_push);} +bool JsonAllPath(void) {return THDVAR(current_thd, json_all_path);} USETEMP UseTemp(void) {return (USETEMP)THDVAR(current_thd, use_tempfile);} int GetConvSize(void) {return THDVAR(current_thd, conv_size);} TYPCONV GetTypeConv(void) {return (TYPCONV)THDVAR(current_thd, type_conv);} char *GetJsonNull(void) {return connect_hton ? THDVAR(current_thd, json_null) : NULL;} +int GetDefaultDepth(void) {return THDVAR(current_thd, default_depth);} uint GetJsonGrpSize(void) {return connect_hton ? THDVAR(current_thd, json_grp_size) : 10;} -uint GetWorkSize(void) {return THDVAR(current_thd, work_size);} -void SetWorkSize(uint) +size_t GetWorkSize(void) {return (size_t)THDVAR(current_thd, work_size);} +void SetWorkSize(size_t) { // Changing the session variable value seems to be impossible here // and should be done in a check function @@ -472,7 +495,8 @@ void SetWorkSize(uint) #if defined(JAVA_SUPPORT) char *GetJavaWrapper(void) -{return connect_hton ? THDVAR(current_thd, java_wrapper) : (char*)"wrappers/JdbcInterface";} +{return connect_hton ? THDVAR(current_thd, java_wrapper) + : (char*)"wrappers/JdbcInterface";} #endif // JAVA_SUPPORT #if defined(JAVA_SUPPORT) || defined(CMGO_SUPPORT) @@ -621,8 +645,10 @@ ha_create_table_option connect_field_option_list[]= HA_FOPTION_NUMBER("FIELD_LENGTH", fldlen, 0, 0, INT_MAX32, 1), HA_FOPTION_STRING("DATE_FORMAT", dateformat), HA_FOPTION_STRING("FIELD_FORMAT", fieldformat), - HA_FOPTION_STRING("SPECIAL", special), - HA_FOPTION_ENUM("DISTRIB", opt, "scattered,clustered,sorted", 0), + HA_FOPTION_STRING("JPATH", jsonpath), + HA_FOPTION_STRING("XPATH", xmlpath), + HA_FOPTION_STRING("SPECIAL", special), + HA_FOPTION_ENUM("DISTRIB", opt, "scattered,clustered,sorted", 0), HA_FOPTION_END }; @@ -1313,9 +1339,10 @@ int GetIntegerTableOption(PGLOBAL g, PTOS options, PCSZ opname, int idef) if ((ulonglong) opval == (ulonglong)NO_IVAL) { PCSZ pv; - if ((pv= GetListOption(g, opname, options->oplist))) - opval= CharToNumber((char*)pv, strlen(pv), ULONGLONG_MAX, true); - else + if ((pv = GetListOption(g, opname, options->oplist))) { + // opval = CharToNumber((char*)pv, strlen(pv), ULONGLONG_MAX, false); + return atoi(pv); + } else return idef; } // endif opval @@ -1567,8 +1594,9 @@ void *ha_connect::GetColumnOption(PGLOBAL g, void *field, PCOLINFO pcf) pcf->Offset= (int)fop->offset; pcf->Freq= (int)fop->freq; pcf->Datefmt= (char*)fop->dateformat; - pcf->Fieldfmt= (char*)fop->fieldformat; - } else { + pcf->Fieldfmt = fop->fieldformat ? (char*)fop->fieldformat + : fop->jsonpath ? (char*)fop->jsonpath : (char*)fop->xmlpath; + } else { pcf->Offset= -1; pcf->Freq= 0; pcf->Datefmt= NULL; @@ -1577,6 +1605,9 @@ void *ha_connect::GetColumnOption(PGLOBAL g, void *field, PCOLINFO pcf) chset= (char *)fp->charset()->name; + if (!strcmp(chset, "binary")) + v = 'B'; // Binary string + switch (fp->type()) { case MYSQL_TYPE_BLOB: case MYSQL_TYPE_VARCHAR: @@ -1586,7 +1617,7 @@ void *ha_connect::GetColumnOption(PGLOBAL g, void *field, PCOLINFO pcf) default: pcf->Type= MYSQLtoPLG(fp->type(), &v); break; - } // endswitch SQL type + } // endswitch SQL type switch (pcf->Type) { case TYPE_STRING: @@ -1640,7 +1671,7 @@ void *ha_connect::GetColumnOption(PGLOBAL g, void *field, PCOLINFO pcf) break; default: break; - } // endswitch type + } // endswitch type if (fp->flags & UNSIGNED_FLAG) pcf->Flags |= U_UNSIGNED; @@ -2213,7 +2244,7 @@ int ha_connect::MakeRecord(char *buf) case TYPE_BIN: p= value->GetCharValue(); charset= &my_charset_bin; - rc= fp->store(p, strlen(p), charset, CHECK_FIELD_WARN); + rc= fp->store(p, value->GetSize(), charset, CHECK_FIELD_WARN); break; case TYPE_DOUBLE: rc= fp->store(value->GetFloatValue()); @@ -4972,7 +5003,7 @@ int ha_connect::check_stmt(PGLOBAL g, MODE newmode, bool cras) } // endif CheckCleanup if (cras) - g->Createas= 1; // To tell external tables of a multi-table command + g->Createas= true; // To tell external tables of a multi-table command if (trace(1)) htrc("Calling CntCheckDB db=%s cras=%d\n", GetDBName(NULL), cras); @@ -5322,91 +5353,100 @@ static char *encode(PGLOBAL g, const char *cnm) @return Return 0 if ok */ -static bool add_field(String *sql, const char *field_name, int typ, int len, - int dec, char *key, uint tm, const char *rem, char *dft, - char *xtra, char *fmt, int flag, bool dbf, char v) -{ - char var= (len > 255) ? 'V' : v; - bool q, error= false; - const char *type= PLGtoMYSQLtype(typ, dbf, var); +static bool add_field(String* sql, TABTYPE ttp, const char* field_name, int typ, + int len, int dec, char* key, uint tm, const char* rem, + char* dft, char* xtra, char* fmt, int flag, bool dbf, char v) { +#if defined(DEVELOPMENT) + // Some client programs regard CHAR(36) as GUID + char var = (len > 255 || len == 36) ? 'V' : v; +#else + char var = (len > 255) ? 'V' : v; +#endif + bool q, error = false; + const char* type = PLGtoMYSQLtype(typ, dbf, var); - error|= sql->append('`'); - error|= sql->append(field_name); - error|= sql->append("` "); - error|= sql->append(type); + error |= sql->append('`'); + error |= sql->append(field_name); + error |= sql->append("` "); + error |= sql->append(type); - if (typ == TYPE_STRING || - (len && typ != TYPE_DATE && (typ != TYPE_DOUBLE || dec >= 0))) { - error|= sql->append('('); - error|= sql->append_ulonglong(len); + if (typ == TYPE_STRING || + (len && typ != TYPE_DATE && (typ != TYPE_DOUBLE || dec >= 0))) { + error |= sql->append('('); + error |= sql->append_ulonglong(len); if (typ == TYPE_DOUBLE) { - error|= sql->append(','); - // dec must be < len and < 31 - error|= sql->append_ulonglong(MY_MIN(dec, (MY_MIN(len, 31) - 1))); - } else if (dec > 0 && !strcmp(type, "DECIMAL")) { - error|= sql->append(','); - // dec must be < len - error|= sql->append_ulonglong(MY_MIN(dec, len - 1)); - } // endif dec - - error|= sql->append(')'); - } // endif len - - if (v == 'U') - error|= sql->append(" UNSIGNED"); - else if (v == 'Z') - error|= sql->append(" ZEROFILL"); - - if (key && *key) { - error|= sql->append(" "); - error|= sql->append(key); - } // endif key - - if (tm) - error|= sql->append(STRING_WITH_LEN(" NOT NULL"), system_charset_info); - - if (dft && *dft) { - error|= sql->append(" DEFAULT "); - - if (typ == TYPE_DATE) - q= (strspn(dft, "0123456789 -:/") == strlen(dft)); - else - q= !IsTypeNum(typ); + error |= sql->append(','); + // dec must be < len and < 31 + error |= sql->append_ulonglong(MY_MIN(dec, (MY_MIN(len, 31) - 1))); + } else if (dec > 0 && !strcmp(type, "DECIMAL")) { + error |= sql->append(','); + // dec must be < len + error |= sql->append_ulonglong(MY_MIN(dec, len - 1)); + } // endif dec + + error |= sql->append(')'); + } // endif len + + if (v == 'U') + error |= sql->append(" UNSIGNED"); + else if (v == 'Z') + error |= sql->append(" ZEROFILL"); + + if (key && *key) { + error |= sql->append(" "); + error |= sql->append(key); + } // endif key + + if (tm) + error |= sql->append(STRING_WITH_LEN(" NOT NULL"), system_charset_info); + + if (dft && *dft) { + error |= sql->append(" DEFAULT "); + + if (typ == TYPE_DATE) + q = (strspn(dft, "0123456789 -:/") == strlen(dft)); + else + q = !IsTypeNum(typ); - if (q) { - error|= sql->append("'"); - error|= sql->append_for_single_quote(dft, strlen(dft)); - error|= sql->append("'"); - } else - error|= sql->append(dft); - - } // endif dft - - if (xtra && *xtra) { - error|= sql->append(" "); - error|= sql->append(xtra); - } // endif rem - - if (rem && *rem) { - error|= sql->append(" COMMENT '"); - error|= sql->append_for_single_quote(rem, strlen(rem)); - error|= sql->append("'"); - } // endif rem - - if (fmt && *fmt) { - error|= sql->append(" FIELD_FORMAT='"); - error|= sql->append_for_single_quote(fmt, strlen(fmt)); - error|= sql->append("'"); - } // endif flag - - if (flag) { - error|= sql->append(" FLAG="); - error|= sql->append_ulonglong(flag); - } // endif flag - - error|= sql->append(','); - return error; + if (q) { + error |= sql->append("'"); + error |= sql->append_for_single_quote(dft, strlen(dft)); + error |= sql->append("'"); + } else + error |= sql->append(dft); + + } // endif dft + + if (xtra && *xtra) { + error |= sql->append(" "); + error |= sql->append(xtra); + } // endif rem + + if (rem && *rem) { + error |= sql->append(" COMMENT '"); + error |= sql->append_for_single_quote(rem, strlen(rem)); + error |= sql->append("'"); + } // endif rem + + if (fmt && *fmt) { + switch (ttp) { + case TAB_JSON: error |= sql->append(" JPATH='"); break; + case TAB_XML: error |= sql->append(" XPATH='"); break; + default: error |= sql->append(" FIELD_FORMAT='"); + } // endswitch ttp + + error |= sql->append_for_single_quote(fmt, strlen(fmt)); + error |= sql->append("'"); + } // endif flag + + if (flag) { + error |= sql->append(" FLAG="); + error |= sql->append_ulonglong(flag); + } // endif flag + + error |= sql->append(','); + return error; } // end of add_field /** @@ -6028,7 +6068,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, len= 256; // STRBLK's have 0 length // Now add the field - if (add_field(&sql, cnm, typ, len, dec, NULL, tm, + if (add_field(&sql, ttp, cnm, typ, len, dec, NULL, tm, NULL, NULL, NULL, NULL, flg, dbf, v)) rc= HA_ERR_OUT_OF_MEM; } // endfor crp @@ -6222,7 +6262,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, prec= 0; // Now add the field - if (add_field(&sql, cnm, typ, prec, dec, key, tm, rem, dft, xtra, + if (add_field(&sql, ttp, cnm, typ, prec, dec, key, tm, rem, dft, xtra, fmt, flg, dbf, v)) rc= HA_ERR_OUT_OF_MEM; } // endfor i @@ -6975,7 +7015,7 @@ bool ha_connect::NoFieldOptionChange(TABLE *tab) fop1->fldlen == fop2->fldlen && CheckString(fop1->dateformat, fop2->dateformat) && CheckString(fop1->fieldformat, fop2->fieldformat) && - CheckString(fop1->special, fop2->special)); + CheckString(fop1->special, fop2->special)); } // endfor fld return rc; @@ -7345,7 +7385,9 @@ static struct st_mysql_sys_var* connect_system_variables[]= { MYSQL_SYSVAR(errmsg_dir_path), #endif // XMSG MYSQL_SYSVAR(json_null), - MYSQL_SYSVAR(json_grp_size), + MYSQL_SYSVAR(json_all_path), + MYSQL_SYSVAR(default_depth), + MYSQL_SYSVAR(json_grp_size), #if defined(JAVA_SUPPORT) MYSQL_SYSVAR(jvm_path), MYSQL_SYSVAR(class_path), @@ -7371,7 +7413,7 @@ maria_declare_plugin(connect) 0x0107, /* version number (1.07) */ NULL, /* status variables */ connect_system_variables, /* system variables */ - "1.07.0001", /* string version */ + "1.07.0002", /* string version */ MariaDB_PLUGIN_MATURITY_STABLE /* maturity */ } maria_declare_plugin_end; diff --git a/storage/connect/ha_connect.h b/storage/connect/ha_connect.h index 9a12ef94431..21e81b141a8 100644 --- a/storage/connect/ha_connect.h +++ b/storage/connect/ha_connect.h @@ -104,7 +104,9 @@ struct ha_field_option_struct uint opt; const char *dateformat; const char *fieldformat; - char *special; + const char* jsonpath; + const char* xmlpath; + char *special; }; /* diff --git a/storage/connect/json.cpp b/storage/connect/json.cpp index 98a4659cea8..ea3ea18da0b 100644 --- a/storage/connect/json.cpp +++ b/storage/connect/json.cpp @@ -93,9 +93,8 @@ char *NextChr(PSZ s, char sep) PJSON ParseJson(PGLOBAL g, char *s, int len, int *ptyp, bool *comma) { int i, pretty = (ptyp) ? *ptyp : 3; - bool b = false, pty[3] = {true, true, true}; - PJSON jsp = NULL; - STRG src; + bool b = false, pty[3] = {true,true,true}; + PJSON jsp = NULL, jp = NULL; if (trace(1)) htrc("ParseJson: s=%.10s len=%d\n", s, len); @@ -106,27 +105,29 @@ PJSON ParseJson(PGLOBAL g, char *s, int len, int *ptyp, bool *comma) } else if (comma) *comma = false; - src.str = s; - src.len = len; - // Trying to guess the pretty format if (s[0] == '[' && (s[1] == '\n' || (s[1] == '\r' && s[2] == '\n'))) pty[0] = false; try { - for (i = 0; i < len; i++) + jp = new(g) JSON(); + jp->s = s; + jp->len = len; + jp->pty = pty; + + for (i = 0; i < jp->len; i++) switch (s[i]) { case '[': if (jsp) - goto tryit; - else if (!(jsp = ParseArray(g, ++i, src, pty))) - throw 1; + jsp = jp->ParseAsArray(g, i, pretty, ptyp); + else + jsp = jp->ParseArray(g, ++i); break; case '{': if (jsp) - goto tryit; - else if (!(jsp = ParseObject(g, ++i, src, pty))) + jsp = jp->ParseAsArray(g, i, pretty, ptyp); + else if (!(jsp = jp->ParseObject(g, ++i))) throw 2; break; @@ -157,8 +158,8 @@ PJSON ParseJson(PGLOBAL g, char *s, int len, int *ptyp, bool *comma) default: if (jsp) - goto tryit; - else if (!(jsp = ParseValue(g, i, src, pty))) + jsp = jp->ParseAsArray(g, i, pretty, ptyp); + else if (!(jsp = jp->ParseValue(g, i))) throw 4; break; @@ -187,10 +188,17 @@ PJSON ParseJson(PGLOBAL g, char *s, int len, int *ptyp, bool *comma) } // end catch return jsp; +} // end of ParseJson -tryit: +/***********************************************************************/ +/* Parse several items as being in an array. */ +/***********************************************************************/ +PJAR JSON::ParseAsArray(PGLOBAL g, int& i, int pretty, int *ptyp) +{ if (pty[0] && (!pretty || pretty > 2)) { - if ((jsp = ParseArray(g, (i = 0), src, pty)) && ptyp && pretty == 3) + PJAR jsp; + + if ((jsp = ParseArray(g, (i = 0))) && ptyp && pretty == 3) *ptyp = (pty[0]) ? 0 : 3; return jsp; @@ -198,26 +206,23 @@ tryit: strcpy(g->Message, "More than one item in file"); return NULL; -} // end of ParseJson +} // end of ParseAsArray /***********************************************************************/ /* Parse a JSON Array. */ /***********************************************************************/ -PJAR ParseArray(PGLOBAL g, int& i, STRG& src, bool *pty) +PJAR JSON::ParseArray(PGLOBAL g, int& i) { - char *s = src.str; - int len = src.len; - int level = 0; - bool b = (!i); - PJAR jarp = new(g) JARRAY; - PJVAL jvp = NULL; + int level = 0; + bool b = (!i); + PJAR jarp = new(g) JARRAY; for (; i < len; i++) switch (s[i]) { case ',': if (level < 2) { sprintf(g->Message, "Unexpected ',' near %.*s",ARGS); - return NULL; + throw 1; } else level = 1; @@ -225,8 +230,8 @@ PJAR ParseArray(PGLOBAL g, int& i, STRG& src, bool *pty) case ']': if (level == 1) { sprintf(g->Message, "Unexpected ',]' near %.*s", ARGS); - return NULL; - } // endif level + throw 1; + } // endif level jarp->InitArray(g); return jarp; @@ -240,11 +245,9 @@ PJAR ParseArray(PGLOBAL g, int& i, STRG& src, bool *pty) default: if (level == 2) { sprintf(g->Message, "Unexpected value near %.*s", ARGS); - return NULL; - } else if ((jvp = ParseValue(g, i, src, pty))) - jarp->AddValue(g, jvp); - else - return NULL; + throw 1; + } else + jarp->AddValue(g, ParseValue(g, i)); level = (b) ? 1 : 2; break; @@ -256,18 +259,15 @@ PJAR ParseArray(PGLOBAL g, int& i, STRG& src, bool *pty) return jarp; } // endif b - strcpy(g->Message, "Unexpected EOF in array"); - return NULL; + throw ("Unexpected EOF in array"); } // end of ParseArray /***********************************************************************/ /* Parse a JSON Object. */ /***********************************************************************/ -PJOB ParseObject(PGLOBAL g, int& i, STRG& src, bool *pty) +PJOB JSON::ParseObject(PGLOBAL g, int& i) { PSZ key; - char *s = src.str; - int len = src.len; int level = 0; PJOB jobp = new(g) JOBJECT; PJPR jpp = NULL; @@ -276,42 +276,37 @@ PJOB ParseObject(PGLOBAL g, int& i, STRG& src, bool *pty) switch (s[i]) { case '"': if (level < 2) { - if ((key = ParseString(g, ++i, src))) { - jpp = jobp->AddPair(g, key); - level = 1; - } else - return NULL; - + key = ParseString(g, ++i); + jpp = jobp->AddPair(g, key); + level = 1; } else { sprintf(g->Message, "misplaced string near %.*s", ARGS); - return NULL; + throw 2; } // endif level break; case ':': if (level == 1) { - if (!(jpp->Val = ParseValue(g, ++i, src, pty))) - return NULL; - + jpp->Val = ParseValue(g, ++i); level = 2; } else { sprintf(g->Message, "Unexpected ':' near %.*s", ARGS); - return NULL; + throw 2; } // endif level break; case ',': if (level < 2) { sprintf(g->Message, "Unexpected ',' near %.*s", ARGS); - return NULL; + throw 2; } else - level = 1; + level = 0; break; case '}': - if (level == 1) { + if (level < 2) { sprintf(g->Message, "Unexpected '}' near %.*s", ARGS); - return NULL; + throw 2; } // endif level return jobp; @@ -324,20 +319,19 @@ PJOB ParseObject(PGLOBAL g, int& i, STRG& src, bool *pty) default: sprintf(g->Message, "Unexpected character '%c' near %.*s", s[i], ARGS); - return NULL; + throw 2; }; // endswitch s[i] strcpy(g->Message, "Unexpected EOF in Object"); - return NULL; + throw 2; } // end of ParseObject /***********************************************************************/ /* Parse a JSON Value. */ /***********************************************************************/ -PJVAL ParseValue(PGLOBAL g, int& i, STRG& src, bool *pty) +PJVAL JSON::ParseValue(PGLOBAL g, int& i) { - char *strval, *s = src.str; - int n, len = src.len; + int n; PJVAL jvp = new(g) JVALUE; for (; i < len; i++) @@ -355,21 +349,13 @@ PJVAL ParseValue(PGLOBAL g, int& i, STRG& src, bool *pty) suite: switch (s[i]) { case '[': - if (!(jvp->Jsp = ParseArray(g, ++i, src, pty))) - return NULL; - + jvp->Jsp = ParseArray(g, ++i); break; case '{': - if (!(jvp->Jsp = ParseObject(g, ++i, src, pty))) - return NULL; - + jvp->Jsp = ParseObject(g, ++i); break; case '"': - if ((strval = ParseString(g, ++i, src))) - jvp->Value = AllocateValue(g, strval, TYPE_STRING); - else - return NULL; - + jvp->Value = AllocateValue(g, ParseString(g, ++i), TYPE_STRING); break; case 't': if (!strncmp(s + i, "true", 4)) { @@ -398,11 +384,9 @@ PJVAL ParseValue(PGLOBAL g, int& i, STRG& src, bool *pty) break; case '-': default: - if (s[i] == '-' || isdigit(s[i])) { - if (!(jvp->Value = ParseNumeric(g, i, src))) - goto err; - - } else + if (s[i] == '-' || isdigit(s[i])) + jvp->Value = ParseNumeric(g, i); + else goto err; }; // endswitch s[i] @@ -410,25 +394,21 @@ PJVAL ParseValue(PGLOBAL g, int& i, STRG& src, bool *pty) return jvp; err: - sprintf(g->Message, "Unexpected character '%c' near %.*s", - s[i], ARGS); - return NULL; + sprintf(g->Message, "Unexpected character '%c' near %.*s", s[i], ARGS); + throw 3; } // end of ParseValue /***********************************************************************/ /* Unescape and parse a JSON string. */ /***********************************************************************/ -char *ParseString(PGLOBAL g, int& i, STRG& src) +char *JSON::ParseString(PGLOBAL g, int& i) { - char *s = src.str; uchar *p; - int n = 0, len = src.len; + int n = 0; // Be sure of memory availability - if (len + 1 - i > (signed)((PPOOLHEADER)g->Sarea)->FreeBlk) { - strcpy(g->Message, "ParseString: Out of memory"); - return NULL; - } // endif len + if (((size_t)len + 1 - i) > ((PPOOLHEADER)g->Sarea)->FreeBlk) + throw("ParseString: Out of memory"); // The size to allocate is not known yet p = (uchar*)PlugSubAlloc(g, NULL, 0); @@ -502,17 +482,16 @@ char *ParseString(PGLOBAL g, int& i, STRG& src) }; // endswitch s[i] err: - strcpy(g->Message, "Unexpected EOF in String"); - return NULL; + throw("Unexpected EOF in String"); } // end of ParseString /***********************************************************************/ /* Parse a JSON numeric value. */ /***********************************************************************/ -PVAL ParseNumeric(PGLOBAL g, int& i, STRG& src) +PVAL JSON::ParseNumeric(PGLOBAL g, int& i) { - char *s = src.str, buf[50]; - int n = 0, len = src.len; + char buf[50]; + int n = 0; short nd = 0; bool has_dot = false; bool has_e = false; @@ -575,14 +554,11 @@ PVAL ParseNumeric(PGLOBAL g, int& i, STRG& src) i--; // Unstack following character return valp; - } else { - strcpy(g->Message, "No digit found"); - return NULL; - } // endif found_digit + } else + throw("No digit found"); err: - strcpy(g->Message, "Unexpected EOF in number"); - return NULL; + throw("Unexpected EOF in number"); } // end of ParseNumeric /***********************************************************************/ diff --git a/storage/connect/json.h b/storage/connect/json.h index 1d058ad575f..bc94b372133 100644 --- a/storage/connect/json.h +++ b/storage/connect/json.h @@ -1,10 +1,11 @@ /**************** json H Declares Source Code File (.H) ****************/ /* Name: json.h Version 1.2 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 2014 - 2017 */ +/* (C) Copyright to the author Olivier BERTRAND 2014 - 2020 */ /* */ /* This file contains the JSON classes declares. */ /***********************************************************************/ +#include <mysql_com.h> #include "value.h" #if defined(_DEBUG) @@ -44,15 +45,31 @@ typedef struct { int len; } STRG, *PSG; +// BSON size should be equal on Linux and Windows +#define BMX 255 +typedef struct BSON* PBSON; + +/***********************************************************************/ +/* Structure used to return binary json to Json UDF functions. */ +/***********************************************************************/ +struct BSON { + char Msg[BMX + 1]; + char *Filename; + PGLOBAL G; + int Pretty; + ulong Reslen; + my_bool Changed; + PJSON Top; + PJSON Jsp; + PBSON Bsp; +}; // end of struct BSON + +PBSON JbinAlloc(PGLOBAL g, UDF_ARGS* args, ulong len, PJSON jsp); + char *NextChr(PSZ s, char sep); char *GetJsonNull(void); -PJSON ParseJson(PGLOBAL g, char *s, int n, int *prty = NULL, bool *b = NULL); -PJAR ParseArray(PGLOBAL g, int& i, STRG& src, bool *pty); -PJOB ParseObject(PGLOBAL g, int& i, STRG& src, bool *pty); -PJVAL ParseValue(PGLOBAL g, int& i, STRG& src, bool *pty); -char *ParseString(PGLOBAL g, int& i, STRG& src); -PVAL ParseNumeric(PGLOBAL g, int& i, STRG& src); +PJSON ParseJson(PGLOBAL g, char* s, int n, int* prty = NULL, bool* b = NULL); PSZ Serialize(PGLOBAL g, PJSON jsp, char *fn, int pretty); bool SerializeArray(JOUT *js, PJAR jarp, bool b); bool SerializeObject(JOUT *js, PJOB jobp); @@ -130,7 +147,7 @@ class JOUTPRT : public JOUTFILE { class JPAIR : public BLOCK { friend class JOBJECT; friend class JSNX; - friend PJOB ParseObject(PGLOBAL, int&, STRG&, bool*); + friend class JSON; friend bool SerializeObject(JOUT *, PJOB); public: JPAIR(PCSZ key) : BLOCK() {Key = key; Val = NULL; Next = NULL;} @@ -149,8 +166,9 @@ class JPAIR : public BLOCK { /* Class JSON. The base class for all other json classes. */ /***********************************************************************/ class JSON : public BLOCK { + friend PJSON ParseJson(PGLOBAL, char*, int, int*, bool*); public: - JSON(void) {Size = 0;} + JSON(void) : s(NULL), len(0), pty(NULL) {Size = 0;} int size(void) {return Size;} virtual int GetSize(bool b) {return Size;} @@ -187,14 +205,27 @@ class JSON : public BLOCK { virtual bool IsNull(void) {X return true;} protected: - int Size; + PJAR ParseArray(PGLOBAL g, int& i); + PJOB ParseObject(PGLOBAL g, int& i); + PJVAL ParseValue(PGLOBAL g, int& i); + char *ParseString(PGLOBAL g, int& i); + PVAL ParseNumeric(PGLOBAL g, int& i); + PJAR ParseAsArray(PGLOBAL g, int& i, int pretty, int *ptyp); + + // Members + int Size; + + // Only used when parsing + private: + char *s; + int len; + bool *pty; }; // end of class JSON /***********************************************************************/ /* Class JOBJECT: contains a list of value pairs. */ /***********************************************************************/ class JOBJECT : public JSON { - friend PJOB ParseObject(PGLOBAL, int&, STRG&, bool*); friend bool SerializeObject(JOUT *, PJOB); friend class JSNX; public: @@ -260,8 +291,8 @@ class JVALUE : public JSON { friend class JARRAY; friend class JSNX; friend class JSONCOL; - friend PJVAL ParseValue(PGLOBAL, int&, STRG&, bool*); - friend bool SerializeValue(JOUT *, PJVAL); + friend class JSON; + friend bool SerializeValue(JOUT*, PJVAL); public: JVALUE(void) : JSON() {Clear();} JVALUE(PJSON jsp); diff --git a/storage/connect/jsonudf.cpp b/storage/connect/jsonudf.cpp index dad86d51040..06164f4ed78 100644 --- a/storage/connect/jsonudf.cpp +++ b/storage/connect/jsonudf.cpp @@ -25,7 +25,7 @@ #else #define PUSH_WARNING(M) htrc(M) #endif -#define M 7 +#define M 9 bool IsNum(PSZ s); char *NextChr(PSZ s, char sep); @@ -1076,29 +1076,10 @@ my_bool JSNX::AddPath(void) /* --------------------------------- JSON UDF ---------------------------------- */ -// BSON size should be equal on Linux and Windows -#define BMX 255 -typedef struct BSON *PBSON; - -/*********************************************************************************/ -/* Structure used to return binary json. */ -/*********************************************************************************/ -struct BSON { - char Msg[BMX + 1]; - char *Filename; - PGLOBAL G; - int Pretty; - ulong Reslen; - my_bool Changed; - PJSON Top; - PJSON Jsp; - PBSON Bsp; -}; // end of struct BSON - /*********************************************************************************/ /* Allocate and initialize a BSON structure. */ /*********************************************************************************/ -static PBSON JbinAlloc(PGLOBAL g, UDF_ARGS *args, ulong len, PJSON jsp) +PBSON JbinAlloc(PGLOBAL g, UDF_ARGS *args, ulong len, PJSON jsp) { PBSON bsp = (PBSON)PlgDBSubAlloc(g, NULL, sizeof(BSON)); @@ -1111,7 +1092,7 @@ static PBSON JbinAlloc(PGLOBAL g, UDF_ARGS *args, ulong len, PJSON jsp) bsp->Reslen = len; bsp->Changed = false; bsp->Top = bsp->Jsp = jsp; - bsp->Bsp = (IsJson(args, 0) == 3) ? (PBSON)args->args[0] : NULL; + bsp->Bsp = (args && IsJson(args, 0) == 3) ? (PBSON)args->args[0] : NULL; } else PUSH_WARNING(g->Message); @@ -1144,7 +1125,7 @@ static my_bool JsonSubSet(PGLOBAL g) { PPOOLHEADER pph = (PPOOLHEADER)g->Sarea; - pph->To_Free = (OFFSET)((g->Createas) ? g->Createas : sizeof(POOLHEADER)); + pph->To_Free = (g->Saved_Size) ? g->Saved_Size : (size_t)sizeof(POOLHEADER); pph->FreeBlk = g->Sarea_Size - pph->To_Free; return FALSE; } /* end of JsonSubSet */ @@ -1154,7 +1135,7 @@ static my_bool JsonSubSet(PGLOBAL g) /*********************************************************************************/ inline void JsonMemSave(PGLOBAL g) { - g->Createas = (int)((PPOOLHEADER)g->Sarea)->To_Free; + g->Saved_Size = ((PPOOLHEADER)g->Sarea)->To_Free; } /* end of JsonMemSave */ /*********************************************************************************/ @@ -1422,7 +1403,7 @@ static int IsJson(UDF_ARGS *args, uint i, bool b) n = 2; // arg is a json file name } else if (b) { char *sap; - PGLOBAL g = PlugInit(NULL, args->lengths[i] * M + 1024); + PGLOBAL g = PlugInit(NULL, (size_t)args->lengths[i] * M + 1024); JsonSubSet(g); sap = MakePSZ(g, args, i); @@ -1625,7 +1606,7 @@ static my_bool CheckMemory(PGLOBAL g, UDF_INIT *initid, UDF_ARGS *args, uint n, return true; } // endif SareaAlloc - g->Createas = 0; + g->Saved_Size = 0; g->Xchk = NULL; initid->max_length = rl; } // endif Size @@ -4423,13 +4404,15 @@ char *json_file(UDF_INIT *initid, UDF_ARGS *args, char *result, fn = MakePSZ(g, args, 0); if (args->arg_count > 1) { - int len, pretty, pty = 3; + int len, pretty = 3, pty = 3; PJSON jsp; PJVAL jvp = NULL; - pretty = (args->arg_type[1] == INT_RESULT) ? (int)*(longlong*)args->args[1] - : (args->arg_count > 2 && args->arg_type[2] == INT_RESULT) - ? (int)*(longlong*)args->args[2] : 3; + for (unsigned int i = 1; i < args->arg_count; i++) + if (args->arg_type[i] == INT_RESULT && *(longlong*)args->args[i] < 4) { + pretty = (int) * (longlong*)args->args[i]; + break; + } // endif type /*******************************************************************************/ /* Parse the json file and allocate its tree structure. */ @@ -4497,6 +4480,7 @@ my_bool jfile_make_init(UDF_INIT *initid, UDF_ARGS *args, char *message) } // endif CalcLen(args, false, reslen, memlen); + memlen = memlen + 5000; // To take care of not pretty files return JsonInit(initid, args, message, true, reslen, memlen); } // end of jfile_make_init @@ -5626,20 +5610,19 @@ my_bool jbin_file_init(UDF_INIT *initid, UDF_ARGS *args, char *message) } else if (args->arg_type[0] != STRING_RESULT || !args->args[0]) { strcpy(message, "First argument must be a constant string (file name)"); return true; - } else if (args->arg_count > 1 && args->arg_type[1] != STRING_RESULT) { - strcpy(message, "Second argument is not a string (path)"); - return true; - } else if (args->arg_count > 2 && args->arg_type[2] != INT_RESULT) { - strcpy(message, "Third argument is not an integer (pretty)"); - return true; - } else if (args->arg_count > 3) { - if (args->arg_type[3] != INT_RESULT) { - strcpy(message, "Fourth argument is not an integer (memory)"); + } // endifs + + for (unsigned int i = 1; i < args->arg_count; i++) { + if (!(args->arg_type[i] == INT_RESULT || args->arg_type[i] == STRING_RESULT)) { + sprintf(message, "Argument %d is not an integer or a string (pretty or path)", i); return true; - } else - more += (ulong)*(longlong*)args->args[3]; + } // endif arg_type - } // endifs + // Take care of eventual memory argument + if (args->arg_type[i] == INT_RESULT && args->args[i]) + more += (ulong) * (longlong*)args->args[i]; + + } // endfor i initid->maybe_null = 1; CalcLen(args, false, reslen, memlen); @@ -5654,7 +5637,7 @@ char *jbin_file(UDF_INIT *initid, UDF_ARGS *args, char *result, unsigned long *res_length, char *is_null, char *error) { char *fn; - int pretty, len = 0, pty = 3; + int pretty = 3, len = 0, pty = 3; PJSON jsp; PJVAL jvp = NULL; PGLOBAL g = (PGLOBAL)initid->ptr; @@ -5666,7 +5649,12 @@ char *jbin_file(UDF_INIT *initid, UDF_ARGS *args, char *result, PlugSubSet(g->Sarea, g->Sarea_Size); g->Xchk = NULL; fn = MakePSZ(g, args, 0); - pretty = (args->arg_count > 2 && args->args[2]) ? (int)*(longlong*)args->args[2] : 3; + + for (unsigned int i = 1; i < args->arg_count; i++) + if (args->arg_type[i] == INT_RESULT && *(longlong*)args->args[i] < 4) { + pretty = (int) * (longlong*)args->args[i]; + break; + } // endif type /*********************************************************************************/ /* Parse the json file and allocate its tree structure. */ @@ -5757,7 +5745,7 @@ char *json_serialize(UDF_INIT *initid, UDF_ARGS *args, char *result, // Keep result of constant function g->Xchk = (initid->const_item) ? str : NULL; } else { - *error = 1; + // *error = 1; str = strcpy(result, "Argument is not a Jbin tree"); } // endif @@ -5774,6 +5762,478 @@ void json_serialize_deinit(UDF_INIT* initid) } // end of json_serialize_deinit /*********************************************************************************/ +/* Convert a prettiest Json file to Pretty=0. */ +/*********************************************************************************/ +my_bool jfile_convert_init(UDF_INIT* initid, UDF_ARGS* args, char* message) { + unsigned long reslen, memlen; + + if (args->arg_count != 3) { + strcpy(message, "This function must have 3 arguments"); + return true; + } else if (args->arg_type[2] != INT_RESULT) { + strcpy(message, "Third Argument must be an integer (LRECL)"); + return true; + } else for (int i = 0; i < 2; i++) + if (args->arg_type[i] != STRING_RESULT) { + sprintf(message, "Arguments %d must be a string (file name)", i+1); + return true; + } // endif args + + CalcLen(args, false, reslen, memlen); + return JsonInit(initid, args, message, false, reslen, memlen); +} // end of jfile_convert_init + +char *jfile_convert(UDF_INIT* initid, UDF_ARGS* args, char* result, + unsigned long *res_length, char *, char *error) { + char *str, *fn, *ofn; + int lrecl = (int)*(longlong*)args->args[2]; + PGLOBAL g = (PGLOBAL)initid->ptr; + + PlugSubSet(g->Sarea, g->Sarea_Size); + fn = MakePSZ(g, args, 0); + ofn = MakePSZ(g, args, 1); + + if (!g->Xchk) { + JUP* jup = new(g) JUP(g); + + str = jup->UnprettyJsonFile(g, fn, ofn, lrecl); + g->Xchk = str; + } else + str = (char*)g->Xchk; + + if (!str) { + if (g->Message) + str = PlugDup(g, g->Message); + else + str = PlugDup(g, "Unexpected error"); + + } // endif str + + *res_length = strlen(str); + return str; +} // end of jfile_convert + +void jfile_convert_deinit(UDF_INIT* initid) { + JsonFreeMem((PGLOBAL)initid->ptr); +} // end of jfile_convert_deinit + +/* --------------------------------- Class JUP --------------------------------- */ + +#define ARGS MY_MIN(24,len-i),s+MY_MAX(i-3,0) + +/*********************************************************************************/ +/* JUP public constructor. */ +/*********************************************************************************/ +JUP::JUP(PGLOBAL g) { + fs = NULL; + s = buff = NULL; + i = k = len = recl = 0; +} // end of JUP constructor + +/*********************************************************************************/ +/* Copy a json file to another with pretty = 0. */ +/*********************************************************************************/ +char* JUP::UnprettyJsonFile(PGLOBAL g, char *fn, char *outfn, int lrecl) { + char *ret = NULL; + HANDLE hFile; + MEMMAP mm; + + /*******************************************************************************/ + /* Create the mapping file object. */ + /*******************************************************************************/ + hFile = CreateFileMap(g, fn, &mm, MODE_READ, false); + + if (hFile == INVALID_HANDLE_VALUE) { + DWORD rc = GetLastError(); + + if (!(*g->Message)) + sprintf(g->Message, MSG(OPEN_MODE_ERROR), "map", (int)rc, fn); + + return NULL; + } // endif hFile + + /*******************************************************************************/ + /* Get the file size (assuming file is smaller than 4 GB) */ + /*******************************************************************************/ + if (!mm.lenL) { // Empty or deleted file + CloseFileHandle(hFile); + return NULL; + } else + len = (int)mm.lenL; + + if (!mm.memory) { + CloseFileHandle(hFile); + sprintf(g->Message, MSG(MAP_VIEW_ERROR), fn, GetLastError()); + return NULL; + } else + s = (char*)mm.memory; + + CloseFileHandle(hFile); // Not used anymore + + /*********************************************************************************/ + /* Parse the json file and allocate its tree structure. */ + /*********************************************************************************/ + if (!(fs = fopen(outfn, "wb"))) { + sprintf(g->Message, MSG(OPEN_MODE_ERROR), + "w", (int)errno, outfn); + strcat(strcat(g->Message, ": "), strerror(errno)); + CloseMemMap(mm.memory, (size_t)mm.lenL); + return NULL; + } // endif fs + + g->Message[0] = 0; + + if (!unPretty(g, lrecl)) + ret = outfn; + + CloseMemMap(mm.memory, (size_t)mm.lenL); + fclose(fs); + return ret; +} // end of UnprettyJsonFile + +/***********************************************************************/ +/* Translate a json file to pretty = 0. */ +/***********************************************************************/ +bool JUP::unPretty(PGLOBAL g, int lrecl) { + bool go, next, rc = false; + + if (trace(1)) + htrc("UnPretty: s=%.10s len=%zd lrecl=%d\n", s, len, lrecl); + + if (!s || !len) { + strcpy(g->Message, "Void JSON file"); + return true; + } else if (*s != '[') { + // strcpy(g->Message, "JSON file is not an array"); + s = strchr(s, '['); + // return true; + } // endif s + + i = 1; + go = next = true; + + try { + // Allocate the record + buff = (char*)PlugSubAlloc(g, NULL, (size_t)lrecl + 3); + recl = lrecl; + + do { + for (k = 0; go && i < len; i++) + switch (s[i]) { + case '{': + buff[k++] = s[i++]; + CopyObject(g); + break; + case '[': + throw "JSON file is not an array of objects"; + break; + case ' ': + case '\t': + case '\n': + case '\r': + break; + case ',': + go = false; + break; + case ']': + go = next = false; + break; + default: + sprintf(g->Message, "Unexpected '%c' near %.*s", s[i], ARGS); + throw 4; + break; + }; // endswitch s[i] + + // Write the record +#ifdef __win_ + buff[k++] = '\r'; +#endif + buff[k++] = '\n'; + buff[k] = 0; + + if ((fputs(buff, fs)) == EOF) { + sprintf(g->Message, MSG(FPUTS_ERROR), strerror(errno)); + throw 5; + } // endif EOF + + go = true; + } while (next); + + } catch (int n) { + if (trace(1)) + htrc("Exception %d: %s\n", n, g->Message); + rc = true; + } catch (const char* msg) { + strcpy(g->Message, msg); + rc = true; + } // end catch + + return rc; +} // end of unPretty + +/***********************************************************************/ +/* Copy a JSON Object. */ +/***********************************************************************/ +void JUP::CopyObject(PGLOBAL g) { + int level = 0; + + for (; i < len; i++) + switch (s[i]) { + case '"': + AddBuff(s[i++]); + + if (level < 2) { + CopyString(g); + level = 1; + } else { + sprintf(g->Message, "misplaced string near %.*s", ARGS); + throw 3; + } // endif level + + break; + case ':': + AddBuff(s[i++]); + + if (level == 1) { + CopyValue(g); + level = 2; + } else { + sprintf(g->Message, "Unexpected ':' near %.*s", ARGS); + throw 3; + } // endif level + + break; + case ',': + AddBuff(s[i]); + + if (level < 2) { + sprintf(g->Message, "Unexpected ',' near %.*s", ARGS); + throw 3; + } else + level = 0; + + break; + case '}': + AddBuff(s[i]); + + if (level == 1) { + sprintf(g->Message, "Unexpected '}' near %.*s", ARGS); + throw 3; + } // endif level + + return; + case '\n': + case '\r': + case ' ': + case '\t': + break; + default: + sprintf(g->Message, "Unexpected character '%c' near %.*s", s[i], ARGS); + throw 3; + }; // endswitch s[i] + + throw "Unexpected EOF in Object"; +} // end of CopyObject + +/***********************************************************************/ +/* Copy a JSON Array. */ +/***********************************************************************/ +void JUP::CopyArray(PGLOBAL g) { + int level = 0; + + for (; i < len; i++) + switch (s[i]) { + case ',': + if (level < 2) { + sprintf(g->Message, "Unexpected ',' near %.*s", ARGS); + throw 2; + } else + level = 1; + + AddBuff(s[i]); + break; + case ']': + if (level == 1) { + sprintf(g->Message, "Unexpected ',]' near %.*s", ARGS); + throw 2; + } // endif level + + AddBuff(s[i]); + return; + case '\n': + case '\r': + case ' ': + case '\t': + break; + default: + if (level == 2) { + sprintf(g->Message, "Unexpected value near %.*s", ARGS); + throw 2; + } // endif level + + CopyValue(g); + level = 2; + break; + }; // endswitch s[i] + + throw "Unexpected EOF in array"; +} // end of CopyArray + +/***********************************************************************/ +/* Copy a JSON Value. */ +/***********************************************************************/ +void JUP::CopyValue(PGLOBAL g) { + for (; i < len; i++) + switch (s[i]) { + case '\n': + case '\r': + case ' ': + case '\t': + break; + default: + goto suite; + } // endswitch + +suite: + switch (s[i]) { + case '[': + AddBuff(s[i++]); + CopyArray(g); + break; + case '{': + AddBuff(s[i++]); + CopyObject(g); + break; + case '"': + AddBuff(s[i++]); + CopyString(g); + break; + case 't': + if (!strncmp(s + i, "true", 4)) { + AddBuff(s[i++]); + AddBuff(s[i++]); + AddBuff(s[i++]); + AddBuff(s[i]); + } else + goto err; + + break; + case 'f': + if (!strncmp(s + i, "false", 5)) { + AddBuff(s[i++]); + AddBuff(s[i++]); + AddBuff(s[i++]); + AddBuff(s[i++]); + AddBuff(s[i]); + } else + goto err; + + break; + case 'n': + if (!strncmp(s + i, "null", 4)) { + AddBuff(s[i++]); + AddBuff(s[i++]); + AddBuff(s[i++]); + AddBuff(s[i]); + } else + goto err; + + break; + default: + if (s[i] == '-' || isdigit(s[i])) + CopyNumeric(g); + else + goto err; + + }; // endswitch s[i] + + return; + +err: + sprintf(g->Message, "Unexpected character '%c' near %.*s", s[i], ARGS); + throw 1; +} // end of CopyValue + +/***********************************************************************/ +/* Unescape and parse a JSON string. */ +/***********************************************************************/ +void JUP::CopyString(PGLOBAL g) { + for (; i < len; i++) { + AddBuff(s[i]); + + switch (s[i]) { + case '"': + return; + case '\\': + AddBuff(s[++i]); + break; + default: + break; + }; // endswitch s[i] + + } // endfor i + + throw "Unexpected EOF in String"; +} // end of CopyString + +/***********************************************************************/ +/* Copy a JSON numeric value. */ +/***********************************************************************/ +void JUP::CopyNumeric(PGLOBAL g) { + bool has_dot = false; + bool has_e = false; + bool found_digit = false; + + for (; i < len; i++) { + switch (s[i]) { + case '.': + if (!found_digit || has_dot || has_e) + goto err; + + has_dot = true; + break; + case 'e': + case 'E': + if (!found_digit || has_e) + goto err; + + has_e = true; + found_digit = false; + break; + case '+': + if (!has_e) + goto err; + + // fall through + case '-': + if (found_digit) + goto err; + + break; + default: + if (isdigit(s[i])) { + found_digit = true; + } else + goto fin; + + }; // endswitch s[i] + + AddBuff(s[i]); + } // endfor i + +fin: + if (!found_digit) + throw "No digit found"; + else + i--; + + return; + +err: + throw "Unexpected EOF in number"; +} // end of CopyNumeric + +/*********************************************************************************/ /* Utility function returning an environment variable value. */ /*********************************************************************************/ my_bool envar_init(UDF_INIT *initid, UDF_ARGS *args, char *message) diff --git a/storage/connect/jsonudf.h b/storage/connect/jsonudf.h index ee56869a111..897b0fe9919 100644 --- a/storage/connect/jsonudf.h +++ b/storage/connect/jsonudf.h @@ -235,6 +235,10 @@ extern "C" { DllExport char *json_serialize(UDF_EXEC_ARGS); DllExport void json_serialize_deinit(UDF_INIT*); + DllExport my_bool jfile_convert_init(UDF_INIT*, UDF_ARGS*, char*); + DllExport char* jfile_convert(UDF_EXEC_ARGS); + DllExport void jfile_convert_deinit(UDF_INIT*); + DllExport my_bool envar_init(UDF_INIT*, UDF_ARGS*, char*); DllExport char *envar(UDF_EXEC_ARGS); @@ -324,3 +328,38 @@ protected: my_bool Wr; // Write mode my_bool Jb; // Must return json item }; // end of class JSNX + +/*********************************************************************************/ +/* Class JUP: used by jfile_convert to make a json file pretty = 0. */ +/*********************************************************************************/ +class JUP : public BLOCK { +public: + // Constructor + JUP(PGLOBAL g); + + // Implementation + void AddBuff(char c) { + if (k < recl) + buff[k++] = c; + else + throw "Record size is too small"; + } // end of AddBuff + + // Methods + char *UnprettyJsonFile(PGLOBAL g, char* fn, char* outfn, int lrecl); + bool unPretty(PGLOBAL g, int lrecl); + void CopyObject(PGLOBAL g); + void CopyArray(PGLOBAL g); + void CopyValue(PGLOBAL g); + void CopyString(PGLOBAL g); + void CopyNumeric(PGLOBAL g); + + // Members + FILE* fs; + char* s; + char* buff; + int len; + int recl; + int i, k; +}; // end of class JUP + diff --git a/storage/connect/mongo.cpp b/storage/connect/mongo.cpp index bd3d3b893c1..5f10a89ee67 100644 --- a/storage/connect/mongo.cpp +++ b/storage/connect/mongo.cpp @@ -35,6 +35,7 @@ bool MakeSelector(PGLOBAL g, PFIL fp, PSTRG s); bool IsNum(PSZ s); +int GetDefaultDepth(void); /***********************************************************************/ /* Make selector json representation for Mongo tables. */ @@ -248,15 +249,10 @@ MGODISC::MGODISC(PGLOBAL g, int *lg) { /***********************************************************************/ int MGODISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ uri, PTOS topt) { - PCSZ level = GetStringTableOption(g, topt, "Level", NULL); PMGODEF tdp; - if (level) { - lvl = atoi(level); - lvl = (lvl > 16) ? 16 : lvl; - } else - lvl = 0; - + lvl = GetIntegerTableOption(g, topt, "Level", GetDefaultDepth()); + lvl = GetIntegerTableOption(g, topt, "Depth", lvl); all = GetBooleanTableOption(g, topt, "Fullarray", false); /*********************************************************************/ diff --git a/storage/connect/mysql-test/connect/r/json_java_2.result b/storage/connect/mysql-test/connect/r/json_java_2.result index 4bbac236200..47fc4abbd28 100644 --- a/storage/connect/mysql-test/connect/r/json_java_2.result +++ b/storage/connect/mysql-test/connect/r/json_java_2.result @@ -20,12 +20,12 @@ SELECT * from t1; Column_Name Data_Type Type_Name Column_Size Buffer_Length Decimal_Digits Nullable Jpath _id 1 CHAR 24 24 0 0 _id address_building 1 CHAR 10 10 0 0 address.building -address_coord 1 CHAR 256 256 0 1 address.coord +address_coord 1 CHAR 1024 1024 0 1 address.coord address_street 1 CHAR 38 38 0 0 address.street address_zipcode 1 CHAR 5 5 0 0 address.zipcode borough 1 CHAR 13 13 0 0 cuisine 1 CHAR 64 64 0 0 -grades_date 1 CHAR 256 256 0 1 grades.0.date +grades_date 1 CHAR 1024 1024 0 1 grades.0.date grades_grade 1 CHAR 14 14 0 1 grades.0.grade grades_score 5 BIGINT 2 2 0 1 grades.0.score name 1 CHAR 98 98 0 0 @@ -64,16 +64,16 @@ OPTION_LIST='Level=1,Driver=Java,Version=2' CONNECTION='mongodb://localhost:2701 SHOW CREATE TABLE t1; Table Create Table t1 CREATE TABLE `t1` ( - `_id` char(24) NOT NULL `FIELD_FORMAT`='_id', - `address_building` char(10) NOT NULL `FIELD_FORMAT`='address.building', - `address_coord` varchar(256) DEFAULT NULL `FIELD_FORMAT`='address.coord', - `address_street` char(38) NOT NULL `FIELD_FORMAT`='address.street', - `address_zipcode` char(5) NOT NULL `FIELD_FORMAT`='address.zipcode', + `_id` char(24) NOT NULL `JPATH`='_id', + `address_building` char(10) NOT NULL `JPATH`='address.building', + `address_coord` varchar(1024) DEFAULT NULL `JPATH`='address.coord', + `address_street` char(38) NOT NULL `JPATH`='address.street', + `address_zipcode` char(5) NOT NULL `JPATH`='address.zipcode', `borough` char(13) NOT NULL, `cuisine` char(64) NOT NULL, - `grades_date` varchar(256) DEFAULT NULL `FIELD_FORMAT`='grades.0.date', - `grades_grade` char(14) DEFAULT NULL `FIELD_FORMAT`='grades.0.grade', - `grades_score` bigint(2) DEFAULT NULL `FIELD_FORMAT`='grades.0.score', + `grades_date` varchar(1024) DEFAULT NULL `JPATH`='grades.0.date', + `grades_grade` char(14) DEFAULT NULL `JPATH`='grades.0.grade', + `grades_score` bigint(2) DEFAULT NULL `JPATH`='grades.0.score', `name` char(98) NOT NULL, `restaurant_id` char(8) NOT NULL ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='mongodb://localhost:27017' `TABLE_TYPE`='JSON' `TABNAME`='restaurants' `OPTION_LIST`='Level=1,Driver=Java,Version=2' `DATA_CHARSET`='utf8' `LRECL`=4096 @@ -251,15 +251,15 @@ OPTION_LIST='Driver=Java,level=2,version=2'; SHOW CREATE TABLE t1; Table Create Table t1 CREATE TABLE `t1` ( - `_id` char(24) NOT NULL `FIELD_FORMAT`='_id', - `address_building` char(10) NOT NULL `FIELD_FORMAT`='address.building', - `address_coord` double(18,16) DEFAULT NULL `FIELD_FORMAT`='address.coord.0', - `address_street` char(38) NOT NULL `FIELD_FORMAT`='address.street', - `address_zipcode` char(5) NOT NULL `FIELD_FORMAT`='address.zipcode', + `_id` char(24) NOT NULL `JPATH`='_id', + `address_building` char(10) NOT NULL `JPATH`='address.building', + `address_coord` double(18,16) DEFAULT NULL `JPATH`='address.coord.0', + `address_street` char(38) NOT NULL `JPATH`='address.street', + `address_zipcode` char(5) NOT NULL `JPATH`='address.zipcode', `borough` char(13) NOT NULL, - `grades_date` char(24) DEFAULT NULL `FIELD_FORMAT`='grades.0.date', - `grades_grade` char(14) DEFAULT NULL `FIELD_FORMAT`='grades.0.grade', - `grades_score` bigint(2) DEFAULT NULL `FIELD_FORMAT`='grades.0.score', + `grades_date` char(24) DEFAULT NULL `JPATH`='grades.0.date', + `grades_grade` char(14) DEFAULT NULL `JPATH`='grades.0.grade', + `grades_score` bigint(2) DEFAULT NULL `JPATH`='grades.0.score', `name` char(98) NOT NULL, `restaurant_id` char(8) NOT NULL ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='mongodb://localhost:27017' `TABLE_TYPE`='JSON' `TABNAME`='restaurants' `COLIST`='{"cuisine":0}' `FILTER`='{"cuisine":"French","borough":{"$ne":"Manhattan"}}' `OPTION_LIST`='Driver=Java,level=2,version=2' `LRECL`=4096 diff --git a/storage/connect/mysql-test/connect/r/json_java_3.result b/storage/connect/mysql-test/connect/r/json_java_3.result index eb8bfc022d6..720c82cd7f9 100644 --- a/storage/connect/mysql-test/connect/r/json_java_3.result +++ b/storage/connect/mysql-test/connect/r/json_java_3.result @@ -20,12 +20,12 @@ SELECT * from t1; Column_Name Data_Type Type_Name Column_Size Buffer_Length Decimal_Digits Nullable Jpath _id 1 CHAR 24 24 0 0 _id address_building 1 CHAR 10 10 0 0 address.building -address_coord 1 CHAR 256 256 0 1 address.coord +address_coord 1 CHAR 1024 1024 0 1 address.coord address_street 1 CHAR 38 38 0 0 address.street address_zipcode 1 CHAR 5 5 0 0 address.zipcode borough 1 CHAR 13 13 0 0 cuisine 1 CHAR 64 64 0 0 -grades_date 1 CHAR 256 256 0 1 grades.0.date +grades_date 1 CHAR 1024 1024 0 1 grades.0.date grades_grade 1 CHAR 14 14 0 1 grades.0.grade grades_score 5 BIGINT 2 2 0 1 grades.0.score name 1 CHAR 98 98 0 0 @@ -64,16 +64,16 @@ OPTION_LIST='Level=1,Driver=Java,Version=3' CONNECTION='mongodb://localhost:2701 SHOW CREATE TABLE t1; Table Create Table t1 CREATE TABLE `t1` ( - `_id` char(24) NOT NULL `FIELD_FORMAT`='_id', - `address_building` char(10) NOT NULL `FIELD_FORMAT`='address.building', - `address_coord` varchar(256) DEFAULT NULL `FIELD_FORMAT`='address.coord', - `address_street` char(38) NOT NULL `FIELD_FORMAT`='address.street', - `address_zipcode` char(5) NOT NULL `FIELD_FORMAT`='address.zipcode', + `_id` char(24) NOT NULL `JPATH`='_id', + `address_building` char(10) NOT NULL `JPATH`='address.building', + `address_coord` varchar(1024) DEFAULT NULL `JPATH`='address.coord', + `address_street` char(38) NOT NULL `JPATH`='address.street', + `address_zipcode` char(5) NOT NULL `JPATH`='address.zipcode', `borough` char(13) NOT NULL, `cuisine` char(64) NOT NULL, - `grades_date` varchar(256) DEFAULT NULL `FIELD_FORMAT`='grades.0.date', - `grades_grade` char(14) DEFAULT NULL `FIELD_FORMAT`='grades.0.grade', - `grades_score` bigint(2) DEFAULT NULL `FIELD_FORMAT`='grades.0.score', + `grades_date` varchar(1024) DEFAULT NULL `JPATH`='grades.0.date', + `grades_grade` char(14) DEFAULT NULL `JPATH`='grades.0.grade', + `grades_score` bigint(2) DEFAULT NULL `JPATH`='grades.0.score', `name` char(98) NOT NULL, `restaurant_id` char(8) NOT NULL ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='mongodb://localhost:27017' `TABLE_TYPE`='JSON' `TABNAME`='restaurants' `OPTION_LIST`='Level=1,Driver=Java,Version=3' `DATA_CHARSET`='utf8' `LRECL`=4096 @@ -251,15 +251,15 @@ OPTION_LIST='Driver=Java,level=2,version=3'; SHOW CREATE TABLE t1; Table Create Table t1 CREATE TABLE `t1` ( - `_id` char(24) NOT NULL `FIELD_FORMAT`='_id', - `address_building` char(10) NOT NULL `FIELD_FORMAT`='address.building', - `address_coord` double(18,16) DEFAULT NULL `FIELD_FORMAT`='address.coord.0', - `address_street` char(38) NOT NULL `FIELD_FORMAT`='address.street', - `address_zipcode` char(5) NOT NULL `FIELD_FORMAT`='address.zipcode', + `_id` char(24) NOT NULL `JPATH`='_id', + `address_building` char(10) NOT NULL `JPATH`='address.building', + `address_coord` double(18,16) DEFAULT NULL `JPATH`='address.coord.0', + `address_street` char(38) NOT NULL `JPATH`='address.street', + `address_zipcode` char(5) NOT NULL `JPATH`='address.zipcode', `borough` char(13) NOT NULL, - `grades_date` bigint(13) DEFAULT NULL `FIELD_FORMAT`='grades.0.date', - `grades_grade` char(14) DEFAULT NULL `FIELD_FORMAT`='grades.0.grade', - `grades_score` bigint(2) DEFAULT NULL `FIELD_FORMAT`='grades.0.score', + `grades_date` bigint(13) DEFAULT NULL `JPATH`='grades.0.date', + `grades_grade` char(14) DEFAULT NULL `JPATH`='grades.0.grade', + `grades_score` bigint(2) DEFAULT NULL `JPATH`='grades.0.score', `name` char(98) NOT NULL, `restaurant_id` char(8) NOT NULL ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='mongodb://localhost:27017' `TABLE_TYPE`='JSON' `TABNAME`='restaurants' `COLIST`='{"cuisine":0}' `FILTER`='{"cuisine":"French","borough":{"$ne":"Manhattan"}}' `OPTION_LIST`='Driver=Java,level=2,version=3' `LRECL`=4096 diff --git a/storage/connect/mysql-test/connect/r/json_mongo_c.result b/storage/connect/mysql-test/connect/r/json_mongo_c.result index 550e94f286e..f9bfc01763e 100644 --- a/storage/connect/mysql-test/connect/r/json_mongo_c.result +++ b/storage/connect/mysql-test/connect/r/json_mongo_c.result @@ -20,12 +20,12 @@ SELECT * from t1; Column_Name Data_Type Type_Name Column_Size Buffer_Length Decimal_Digits Nullable Jpath _id 1 CHAR 24 24 0 0 _id address_building 1 CHAR 10 10 0 0 address.building -address_coord 1 CHAR 256 256 0 1 address.coord +address_coord 1 CHAR 1024 1024 0 1 address.coord address_street 1 CHAR 38 38 0 0 address.street address_zipcode 1 CHAR 5 5 0 0 address.zipcode borough 1 CHAR 13 13 0 0 cuisine 1 CHAR 64 64 0 0 -grades_date 1 CHAR 256 256 0 1 grades.0.date +grades_date 1 CHAR 1024 1024 0 1 grades.0.date grades_grade 1 CHAR 14 14 0 1 grades.0.grade grades_score 5 BIGINT 2 2 0 1 grades.0.score name 1 CHAR 98 98 0 0 @@ -64,16 +64,16 @@ OPTION_LIST='Level=1,Driver=C,Version=0' CONNECTION='mongodb://localhost:27017' SHOW CREATE TABLE t1; Table Create Table t1 CREATE TABLE `t1` ( - `_id` char(24) NOT NULL `FIELD_FORMAT`='_id', - `address_building` char(10) NOT NULL `FIELD_FORMAT`='address.building', - `address_coord` varchar(256) DEFAULT NULL `FIELD_FORMAT`='address.coord', - `address_street` char(38) NOT NULL `FIELD_FORMAT`='address.street', - `address_zipcode` char(5) NOT NULL `FIELD_FORMAT`='address.zipcode', + `_id` char(24) NOT NULL `JPATH`='_id', + `address_building` char(10) NOT NULL `JPATH`='address.building', + `address_coord` varchar(1024) DEFAULT NULL `JPATH`='address.coord', + `address_street` char(38) NOT NULL `JPATH`='address.street', + `address_zipcode` char(5) NOT NULL `JPATH`='address.zipcode', `borough` char(13) NOT NULL, `cuisine` char(64) NOT NULL, - `grades_date` varchar(256) DEFAULT NULL `FIELD_FORMAT`='grades.0.date', - `grades_grade` char(14) DEFAULT NULL `FIELD_FORMAT`='grades.0.grade', - `grades_score` bigint(2) DEFAULT NULL `FIELD_FORMAT`='grades.0.score', + `grades_date` varchar(1024) DEFAULT NULL `JPATH`='grades.0.date', + `grades_grade` char(14) DEFAULT NULL `JPATH`='grades.0.grade', + `grades_score` bigint(2) DEFAULT NULL `JPATH`='grades.0.score', `name` char(98) NOT NULL, `restaurant_id` char(8) NOT NULL ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='mongodb://localhost:27017' `TABLE_TYPE`='JSON' `TABNAME`='restaurants' `OPTION_LIST`='Level=1,Driver=C,Version=0' `DATA_CHARSET`='utf8' `LRECL`=1024 @@ -251,15 +251,15 @@ OPTION_LIST='Driver=C,level=2,version=0'; SHOW CREATE TABLE t1; Table Create Table t1 CREATE TABLE `t1` ( - `_id` char(24) NOT NULL `FIELD_FORMAT`='_id', - `address_building` char(10) NOT NULL `FIELD_FORMAT`='address.building', - `address_coord` double(23,20) DEFAULT NULL `FIELD_FORMAT`='address.coord.0', - `address_street` char(38) NOT NULL `FIELD_FORMAT`='address.street', - `address_zipcode` char(5) NOT NULL `FIELD_FORMAT`='address.zipcode', + `_id` char(24) NOT NULL `JPATH`='_id', + `address_building` char(10) NOT NULL `JPATH`='address.building', + `address_coord` double(23,20) DEFAULT NULL `JPATH`='address.coord.0', + `address_street` char(38) NOT NULL `JPATH`='address.street', + `address_zipcode` char(5) NOT NULL `JPATH`='address.zipcode', `borough` char(13) NOT NULL, - `grades_date` bigint(13) DEFAULT NULL `FIELD_FORMAT`='grades.0.date', - `grades_grade` char(14) DEFAULT NULL `FIELD_FORMAT`='grades.0.grade', - `grades_score` bigint(2) DEFAULT NULL `FIELD_FORMAT`='grades.0.score', + `grades_date` bigint(13) DEFAULT NULL `JPATH`='grades.0.date', + `grades_grade` char(14) DEFAULT NULL `JPATH`='grades.0.grade', + `grades_score` bigint(2) DEFAULT NULL `JPATH`='grades.0.score', `name` char(98) NOT NULL, `restaurant_id` char(8) NOT NULL ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='mongodb://localhost:27017' `TABLE_TYPE`='JSON' `TABNAME`='restaurants' `COLIST`='{"projection":{"cuisine":0}}' `FILTER`='{"cuisine":"French","borough":{"$ne":"Manhattan"}}' `OPTION_LIST`='Driver=C,level=2,version=0' `LRECL`=1024 diff --git a/storage/connect/mysql-test/connect/r/updelx.result b/storage/connect/mysql-test/connect/r/updelx.result index 2aed1e06928..bb82afcc1a8 100644 --- a/storage/connect/mysql-test/connect/r/updelx.result +++ b/storage/connect/mysql-test/connect/r/updelx.result @@ -978,7 +978,7 @@ DROP TABLE t1; # FIX table CREATE TABLE t1 ( id INT(4) KEY NOT NULL, -msg VARCHAR(16) CHARSET BINARY DISTRIB=CLUSTERED) +msg VARCHAR(16) DISTRIB=CLUSTERED) ENGINE=CONNECT TABLE_TYPE=FIX BLOCK_SIZE=4; Warnings: Warning 1105 No file name. Table will use t1.fix @@ -1345,7 +1345,7 @@ DROP TABLE t1; # BIN table CREATE TABLE t1 ( id INT(4) KEY NOT NULL, -msg VARCHAR(16) CHARSET BINARY DISTRIB=CLUSTERED) +msg VARCHAR(16) DISTRIB=CLUSTERED) ENGINE=CONNECT TABLE_TYPE=BIN BLOCK_SIZE=8; Warnings: Warning 1105 No file name. Table will use t1.bin diff --git a/storage/connect/mysql-test/connect/t/updelx.test b/storage/connect/mysql-test/connect/t/updelx.test index 19d0d790a30..f6291432e48 100644 --- a/storage/connect/mysql-test/connect/t/updelx.test +++ b/storage/connect/mysql-test/connect/t/updelx.test @@ -36,7 +36,7 @@ DROP TABLE t1; --echo # FIX table CREATE TABLE t1 ( id INT(4) KEY NOT NULL, -msg VARCHAR(16) CHARSET BINARY DISTRIB=CLUSTERED) +msg VARCHAR(16) DISTRIB=CLUSTERED) ENGINE=CONNECT TABLE_TYPE=FIX BLOCK_SIZE=4; -- source updelx.inc ALTER TABLE t1 MAPPED=YES; @@ -48,7 +48,7 @@ DROP TABLE t1; --echo # BIN table CREATE TABLE t1 ( id INT(4) KEY NOT NULL, -msg VARCHAR(16) CHARSET BINARY DISTRIB=CLUSTERED) +msg VARCHAR(16) DISTRIB=CLUSTERED) ENGINE=CONNECT TABLE_TYPE=BIN BLOCK_SIZE=8; -- source updelx.inc ALTER TABLE t1 MAPPED=YES; diff --git a/storage/connect/myutil.h b/storage/connect/myutil.h index 6991172b39e..fa41fa47d61 100644 --- a/storage/connect/myutil.h +++ b/storage/connect/myutil.h @@ -6,8 +6,8 @@ enum enum_field_types PLGtoMYSQL(int type, bool dbf, char var = 0); const char *PLGtoMYSQLtype(int type, bool dbf, char var = 0); -int MYSQLtoPLG(char *typname, char *var = NULL); -int MYSQLtoPLG(int mytype, char *var = NULL); +int MYSQLtoPLG(char *typname, char *var); +int MYSQLtoPLG(int mytype, char *var); PCSZ MyDateFmt(int mytype); PCSZ MyDateFmt(char *typname); diff --git a/storage/connect/plgdbutl.cpp b/storage/connect/plgdbutl.cpp index e296553d8e2..7bc7c5f592b 100644 --- a/storage/connect/plgdbutl.cpp +++ b/storage/connect/plgdbutl.cpp @@ -5,7 +5,7 @@ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 1998-2018 */ +/* (C) Copyright to the author Olivier BERTRAND 1998-2020 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -1242,7 +1242,7 @@ void *PlgDBalloc(PGLOBAL g, void *area, MBLOCK& mp) mp.Sub = mp.Size <= ((mp.Sub) ? maxsub : (maxsub >> 2)); if (trace(2)) - htrc("PlgDBalloc: in %p size=%d used=%d free=%d sub=%d\n", + htrc("PlgDBalloc: in %p size=%zd used=%zd free=%zd sub=%d\n", arp, mp.Size, pph->To_Free, pph->FreeBlk, mp.Sub); if (!mp.Sub) { @@ -1258,7 +1258,7 @@ void *PlgDBalloc(PGLOBAL g, void *area, MBLOCK& mp) mp.Memp = malloc(mp.Size); if (trace(8)) - htrc("PlgDBalloc: %s(%d) at %p\n", v, mp.Size, mp.Memp); + htrc("PlgDBalloc: %s(%zd) at %p\n", v, mp.Size, mp.Memp); if (!mp.Inlist && mp.Memp) { // New allocated block, put it in the memory block chain. @@ -1290,7 +1290,7 @@ void *PlgDBrealloc(PGLOBAL g, void *area, MBLOCK& mp, size_t newsize) #endif if (trace(2)) - htrc("PlgDBrealloc: %p size=%d sub=%d\n", mp.Memp, mp.Size, mp.Sub); + htrc("PlgDBrealloc: %p size=%zd sub=%d\n", mp.Memp, mp.Size, mp.Sub); if (newsize == mp.Size) return mp.Memp; // Nothing to do @@ -1340,7 +1340,7 @@ void *PlgDBrealloc(PGLOBAL g, void *area, MBLOCK& mp, size_t newsize) } // endif's if (trace(8)) - htrc(" newsize=%d newp=%p sub=%d\n", mp.Size, mp.Memp, mp.Sub); + htrc(" newsize=%zd newp=%p sub=%d\n", mp.Size, mp.Memp, mp.Sub); return mp.Memp; } // end of PlgDBrealloc @@ -1392,13 +1392,13 @@ void *PlgDBSubAlloc(PGLOBAL g, void *memp, size_t size) pph = (PPOOLHEADER)memp; if (trace(16)) - htrc("PlgDBSubAlloc: memp=%p size=%d used=%d free=%d\n", + htrc("PlgDBSubAlloc: memp=%p size=%zd used=%zd free=%zd\n", memp, size, pph->To_Free, pph->FreeBlk); - if ((uint)size > pph->FreeBlk) { /* Not enough memory left in pool */ + if (size > pph->FreeBlk) { /* Not enough memory left in pool */ sprintf(g->Message, - "Not enough memory in Work area for request of %d (used=%d free=%d)", - (int) size, pph->To_Free, pph->FreeBlk); + "Not enough memory in Work area for request of %zd (used=%zd free=%zd)", + size, pph->To_Free, pph->FreeBlk); if (trace(1)) htrc("%s\n", g->Message); @@ -1414,7 +1414,7 @@ void *PlgDBSubAlloc(PGLOBAL g, void *memp, size_t size) pph->FreeBlk -= size; // New size of pool free block if (trace(16)) - htrc("Done memp=%p used=%d free=%d\n", + htrc("Done memp=%p used=%zd free=%zd\n", memp, pph->To_Free, pph->FreeBlk); return (memp); diff --git a/storage/connect/plugutil.cpp b/storage/connect/plugutil.cpp index e74937b942a..0ab594f5533 100644 --- a/storage/connect/plugutil.cpp +++ b/storage/connect/plugutil.cpp @@ -6,7 +6,7 @@ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 1993-2019 */ +/* (C) Copyright to the author Olivier BERTRAND 1993-2020 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -142,7 +142,7 @@ void htrc(char const* fmt, ...) /* Language points on initial language name and eventual path. */ /* Return value is the pointer to the Global structure. */ /***********************************************************************/ -PGLOBAL PlugInit(LPCSTR Language, uint worksize) +PGLOBAL PlugInit(LPCSTR Language, size_t worksize) { PGLOBAL g; @@ -158,13 +158,14 @@ PGLOBAL PlugInit(LPCSTR Language, uint worksize) } // end try/catch g->Sarea = NULL; - g->Createas = 0; + g->Createas = false; g->Alchecked = 0; g->Mrr = 0; g->Activityp = NULL; g->Xchk = NULL; g->N = 0; g->More = 0; + g->Saved_Size = 0; strcpy(g->Message, ""); /*******************************************************************/ @@ -459,7 +460,7 @@ short GetLineLength(PGLOBAL g) /***********************************************************************/ /* Program for memory allocation of work and language areas. */ /***********************************************************************/ -bool AllocSarea(PGLOBAL g, uint size) +bool AllocSarea(PGLOBAL g, size_t size) { /*********************************************************************/ /* This is the allocation routine for the WIN32/UNIX/AIX version. */ @@ -483,7 +484,7 @@ bool AllocSarea(PGLOBAL g, uint size) if (trace(8)) { #endif if (g->Sarea) - htrc("Work area of %u allocated at %p\n", size, g->Sarea); + htrc("Work area of %zd allocated at %p\n", size, g->Sarea); else htrc("SareaAlloc: %s\n", g->Message); @@ -510,7 +511,7 @@ void FreeSarea(PGLOBAL g) #else if (trace(8)) #endif - htrc("Freeing Sarea at %p size = %d\n", g->Sarea, g->Sarea_Size); + htrc("Freeing Sarea at %p size = %zd\n", g->Sarea, g->Sarea_Size); g->Sarea = NULL; g->Sarea_Size = 0; @@ -524,11 +525,11 @@ void FreeSarea(PGLOBAL g) /* Here there should be some verification done such as validity of */ /* the address and size not larger than memory size. */ /***********************************************************************/ -BOOL PlugSubSet(void *memp, uint size) +BOOL PlugSubSet(void *memp, size_t size) { PPOOLHEADER pph = (PPOOLHEADER)memp; - pph->To_Free = (OFFSET)sizeof(POOLHEADER); + pph->To_Free = (size_t)sizeof(POOLHEADER); pph->FreeBlk = size - pph->To_Free; return FALSE; } /* end of PlugSubSet */ @@ -560,15 +561,15 @@ void *PlugSubAlloc(PGLOBAL g, void *memp, size_t size) pph = (PPOOLHEADER)memp; if (trace(16)) - htrc("SubAlloc in %p size=%d used=%d free=%d\n", + htrc("SubAlloc in %p size=%zd used=%zd free=%zd\n", memp, size, pph->To_Free, pph->FreeBlk); - if ((uint)size > pph->FreeBlk) { /* Not enough memory left in pool */ + if (size > pph->FreeBlk) { /* Not enough memory left in pool */ PCSZ pname = "Work"; sprintf(g->Message, - "Not enough memory in %s area for request of %u (used=%d free=%d)", - pname, (uint)size, pph->To_Free, pph->FreeBlk); + "Not enough memory in %s area for request of %zd (used=%zd free=%zd)", + pname, size, pph->To_Free, pph->FreeBlk); if (trace(1)) htrc("PlugSubAlloc: %s\n", g->Message); @@ -580,11 +581,11 @@ void *PlugSubAlloc(PGLOBAL g, void *memp, size_t size) /* Do the suballocation the simplest way. */ /*********************************************************************/ memp = MakePtr(memp, pph->To_Free); /* Points to suballocated block */ - pph->To_Free += (OFFSET)size; /* New offset of pool free block */ - pph->FreeBlk -= (uint)size; /* New size of pool free block */ + pph->To_Free += size; /* New offset of pool free block */ + pph->FreeBlk -= size; /* New size of pool free block */ if (trace(16)) - htrc("Done memp=%p used=%d free=%d\n", + htrc("Done memp=%p used=%zd free=%zd\n", memp, pph->To_Free, pph->FreeBlk); return (memp); @@ -605,40 +606,4 @@ char *PlugDup(PGLOBAL g, const char *str) } // end of PlugDup -#if 0 -/***********************************************************************/ -/* This routine suballocate a copy of the passed string. */ -/***********************************************************************/ -char *PlugDup(PGLOBAL g, const char *str) - { - char *buf; - size_t len; - - if (str && (len = strlen(str))) { - buf = (char*)PlugSubAlloc(g, NULL, len + 1); - strcpy(buf, str); - } else - buf = NULL; - - return(buf); - } /* end of PlugDup */ -#endif // 0 - -/***********************************************************************/ -/* This routine makes a pointer from an offset to a memory pointer. */ -/***********************************************************************/ -void *MakePtr(void *memp, OFFSET offset) - { - return ((offset == 0) ? NULL : &((char *)memp)[offset]); - } /* end of MakePtr */ - -/***********************************************************************/ -/* This routine makes an offset from a pointer new format. */ -/***********************************************************************/ -#if 0 -OFFSET MakeOff(void *memp, void *ptr) - { - return ((!ptr) ? 0 : (OFFSET)((char *)ptr - (char *)memp)); - } /* end of MakeOff */ -#endif /*--------------------- End of PLUGUTIL program -----------------------*/ diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp index 3b0d458a7a6..cdf9e40f97c 100644 --- a/storage/connect/tabjson.cpp +++ b/storage/connect/tabjson.cpp @@ -52,19 +52,10 @@ /* External functions. */ /***********************************************************************/ USETEMP UseTemp(void); +bool JsonAllPath(void); +int GetDefaultDepth(void); char *GetJsonNull(void); -//typedef struct _jncol { -// struct _jncol *Next; -// char *Name; -// char *Fmt; -// int Type; -// int Len; -// int Scale; -// bool Cbn; -// bool Found; -//} JCOL, *PJCL; - /***********************************************************************/ /* JSONColumns: construct the result blocks containing the description */ /* of all the columns of a table contained inside a JSON file. */ @@ -167,23 +158,20 @@ JSONDISC::JSONDISC(PGLOBAL g, uint *lg) jsp = NULL; row = NULL; sep = NULL; - i = n = bf = ncol = lvl = 0; - all = false; + i = n = bf = ncol = lvl = sz = 0; + all = strfy = false; } // end of JSONDISC constructor int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt) { char filename[_MAX_PATH]; bool mgo = (GetTypeID(topt->type) == TAB_MONGO); - PCSZ level = GetStringTableOption(g, topt, "Level", NULL); - - if (level) { - lvl = atoi(level); - lvl = (lvl > 16) ? 16 : lvl; - } else - lvl = 0; - sep = GetStringTableOption(g, topt, "Separator", "."); + lvl = GetIntegerTableOption(g, topt, "Level", GetDefaultDepth()); + lvl = GetIntegerTableOption(g, topt, "Depth", lvl); + sep = GetStringTableOption(g, topt, "Separator", "."); + sz = GetIntegerTableOption(g, topt, "Jsize", 1024); + strfy = GetBooleanTableOption(g, topt, "Stringify", false); /*********************************************************************/ /* Open the input file. */ @@ -254,12 +242,14 @@ int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt) jsp = (tjsp->GetDoc()) ? tjsp->GetDoc()->GetValue(0) : NULL; } else { - if (!(tdp->Lrecl = GetIntegerTableOption(g, topt, "Lrecl", 0))) - if (!mgo) { - sprintf(g->Message, "LRECL must be specified for pretty=%d", tdp->Pretty); - return 0; - } else - tdp->Lrecl = 8192; // Should be enough + if (!((tdp->Lrecl = GetIntegerTableOption(g, topt, "Lrecl", 0)))) { + if (!mgo) { + sprintf(g->Message, "LRECL must be specified for pretty=%d", tdp->Pretty); + return 0; + } else + tdp->Lrecl = 8192; // Should be enough + + } // endif Lrecl tdp->Ending = GetIntegerTableOption(g, topt, "Ending", CRLF); @@ -304,7 +294,7 @@ int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt) // Allocate the parse work memory PGLOBAL G = (PGLOBAL)PlugSubAlloc(g, NULL, sizeof(GLOBAL)); memset(G, 0, sizeof(GLOBAL)); - G->Sarea_Size = tdp->Lrecl * 10; + G->Sarea_Size = (size_t)tdp->Lrecl * 10; G->Sarea = PlugSubAlloc(g, NULL, G->Sarea_Size); PlugSubSet(G->Sarea, G->Sarea_Size); G->jump_level = 0; @@ -401,7 +391,10 @@ bool JSONDISC::Find(PGLOBAL g, PJVAL jvp, PCSZ key, int j) PJAR jar; if ((valp = jvp ? jvp->GetValue() : NULL)) { - jcol.Type = valp->GetType(); + if (JsonAllPath() && !fmt[bf]) + strcat(fmt, colname); + + jcol.Type = valp->GetType(); jcol.Len = valp->GetValLen(); jcol.Scale = valp->GetValPrec(); jcol.Cbn = valp->IsNull(); @@ -480,8 +473,16 @@ bool JSONDISC::Find(PGLOBAL g, PJVAL jvp, PCSZ key, int j) } // endswitch Type } else if (lvl >= 0) { - jcol.Type = TYPE_STRING; - jcol.Len = 256; + if (strfy) { + if (!fmt[bf]) + strcat(fmt, colname); + + strcat(fmt, ".*"); + } else if (JsonAllPath() && !fmt[bf]) + strcat(fmt, colname); + + jcol.Type = TYPE_STRING; + jcol.Len = sz; jcol.Scale = 0; jcol.Cbn = true; } else @@ -1329,7 +1330,7 @@ bool JSONCOL::ParseJpath(PGLOBAL g) { char *p, *p1 = NULL, *p2 = NULL, *pbuf = NULL; int i; - bool a, mul = false; + bool a; if (Parsed) return false; // Already done @@ -1486,7 +1487,18 @@ PVAL JSONCOL::MakeJson(PGLOBAL g, PJSON jsp) if (Value->IsTypeNum()) { strcpy(g->Message, "Cannot make Json for a numeric column"); Value->Reset(); - } else + } else if (Value->GetType() == TYPE_BIN) { + if ((unsigned)Value->GetClen() >= sizeof(BSON)) { + ulong len = Tjp->Lrecl ? Tjp->Lrecl : 500; + PBSON bsp = JbinAlloc(g, NULL, len, jsp); + + strcat(bsp->Msg, " column"); + ((BINVAL*)Value)->SetBinValue(bsp, sizeof(BSON)); + } else { + strcpy(g->Message, "Column size too small"); + Value->SetValue_char(NULL, 0); + } // endif Clen + } else Value->SetValue_psz(Serialize(g, jsp, NULL, 0)); return Value; @@ -1559,7 +1571,6 @@ void JSONCOL::ReadColumn(PGLOBAL g) PVAL JSONCOL::GetColumnValue(PGLOBAL g, PJSON row, int i) { int n = Nod - 1; - bool expd = false; PJAR arp; PJVAL val = NULL; @@ -1983,8 +1994,9 @@ int TDBJSON::MakeNewDoc(PGLOBAL g) /***********************************************************************/ int TDBJSON::MakeDocument(PGLOBAL g) { - char *p, *memory, *objpath, *key = NULL; + char *p, *p1, *p2, *memory, *objpath, *key = NULL; int len, i = 0; + my_bool a; MODE mode = Mode; PJSON jsp; PJOB objp = NULL; @@ -2027,22 +2039,39 @@ int TDBJSON::MakeDocument(PGLOBAL g) if ((objpath = PlugDup(g, Objname))) { if (*objpath == '$') objpath++; if (*objpath == '.') objpath++; + p1 = (*objpath == '[') ? objpath++ : NULL; /*********************************************************************/ /* Find the table in the tree structure. */ /*********************************************************************/ - for (; jsp && objpath; objpath = p) { - if ((p = strchr(objpath, Sep))) - *p++ = 0; - - if (*objpath != '[' && !IsNum(objpath)) { - // objpass is a key + for (p = objpath; jsp && p; p = (p2 ? p2 : NULL)) { + a = (p1 != NULL); + p1 = strchr(p, '['); + p2 = strchr(p, '.'); + + if (!p2) + p2 = p1; + else if (p1) { + if (p1 < p2) + p2 = p1; + else if (p1 == p2 + 1) + *p2++ = 0; // Old syntax .[ + else + p1 = NULL; + + } // endif p1 + + if (p2) + *p2++ = 0; + + if (!a && *p && *p != '[' && !IsNum(p)) { + // obj is a key if (jsp->GetType() != TYPE_JOB) { strcpy(g->Message, "Table path does not match the json file"); return RC_FX; } // endif Type - key = objpath; + key = p; objp = jsp->GetObject(); arp = NULL; val = objp->GetValue(key); @@ -2053,15 +2082,15 @@ int TDBJSON::MakeDocument(PGLOBAL g) } // endif val } else { - if (*objpath == '[') { + if (*p == '[') { // Old style - if (objpath[strlen(objpath) - 1] != ']') { - sprintf(g->Message, "Invalid Table path %s", Objname); + if (p[strlen(p) - 1] != ']') { + sprintf(g->Message, "Invalid Table path near %s", p); return RC_FX; } else - objpath++; + p++; - } // endif objpath + } // endif p if (jsp->GetType() != TYPE_JAR) { strcpy(g->Message, "Table path does not match the json file"); @@ -2070,7 +2099,7 @@ int TDBJSON::MakeDocument(PGLOBAL g) arp = jsp->GetArray(); objp = NULL; - i = atoi(objpath) - B; + i = atoi(p) - B; val = arp->GetValue(i); if (!val) { @@ -2081,7 +2110,7 @@ int TDBJSON::MakeDocument(PGLOBAL g) } // endif jsp = val->GetJson(); - } // endfor objpath + } // endfor p } // endif objpath @@ -2119,13 +2148,15 @@ int TDBJSON::Cardinality(PGLOBAL g) { if (!g) return (Xcol || Multiple) ? 0 : 1; - else if (Cardinal < 0) - if (!Multiple) { - if (MakeDocument(g) == RC_OK) - Cardinal = Doc->size(); + else if (Cardinal < 0) { + if (!Multiple) { + if (MakeDocument(g) == RC_OK) + Cardinal = Doc->size(); - } else - return 10; + } else + return 10; + + } // endif Cardinal return Cardinal; } // end of Cardinality diff --git a/storage/connect/tabjson.h b/storage/connect/tabjson.h index 8c3f1013919..88aa5e2ee8b 100644 --- a/storage/connect/tabjson.h +++ b/storage/connect/tabjson.h @@ -68,8 +68,8 @@ public: PCSZ sep; char colname[65], fmt[129], buf[16]; uint *length; - int i, n, bf, ncol, lvl; - bool all; + int i, n, bf, ncol, lvl, sz; + bool all, strfy; }; // end of JSONDISC /***********************************************************************/ diff --git a/storage/connect/tabrest.cpp b/storage/connect/tabrest.cpp index 3ef2a460b9d..b1bdeffc880 100644 --- a/storage/connect/tabrest.cpp +++ b/storage/connect/tabrest.cpp @@ -158,16 +158,32 @@ PQRYRES __stdcall ColREST(PGLOBAL g, PTOS tp, char *tab, char *db, bool info) http = GetStringTableOption(g, tp, "Http", NULL); uri = GetStringTableOption(g, tp, "Uri", NULL); - fn = GetStringTableOption(g, tp, "Filename", "rest.json"); #if defined(MARIADB) ftype = GetStringTableOption(g, tp, "Type", "JSON"); #else // !MARIADB // OEM tables must specify the file type ftype = GetStringTableOption(g, tp, "Ftype", "JSON"); #endif // !MARIADB + fn = GetStringTableOption(g, tp, "Filename", NULL); + + if (!fn) { + int n, m = strlen(ftype) + 1; + + strcat(strcpy(filename, tab), "."); + n = strlen(filename); + + // Fold ftype to lower case + for (int i = 0; i < m; i++) + filename[n + i] = tolower(ftype[i]); + + fn = filename; + tp->filename = PlugDup(g, fn); + } // endif fn // We used the file name relative to recorded datapath - snprintf(filename, sizeof filename, IF_WIN(".\\%s\\%s","./%s/%s"), db, fn); + PlugSetPath(filename, fn, db); + //strcat(strcat(strcat(strcpy(filename, "."), slash), db), slash); + //strncat(filename, fn, _MAX_PATH - strlen(filename)); // Retrieve the file from the web and copy it locally if (http && grf(g->Message, trace(515), http, uri, filename)) { @@ -226,12 +242,10 @@ bool RESTDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) Http = GetStringCatInfo(g, "Http", NULL); Uri = GetStringCatInfo(g, "Uri", NULL); - Fn = GetStringCatInfo(g, "Filename", "rest.json"); + Fn = GetStringCatInfo(g, "Filename", NULL); // We used the file name relative to recorded datapath - //PlugSetPath(filename, Fn, GetPath()); - strcpy(filename, GetPath()); - strncat(filename, Fn, _MAX_PATH - strlen(filename)); + PlugSetPath(filename, Fn, GetPath()); // Retrieve the file from the web and copy it locally rc = grf(g->Message, xt, Http, Uri, filename); @@ -269,7 +283,7 @@ PTDB RESTDEF::GetTable(PGLOBAL g, MODE m) if (trace(515)) htrc("REST GetTable mode=%d\n", m); - if (m != MODE_READ && m != MODE_READX) { + if (m != MODE_READ && m != MODE_READX && m != MODE_ANY) { strcpy(g->Message, "REST tables are currently read only"); return NULL; } // endif m diff --git a/storage/connect/tabxml.cpp b/storage/connect/tabxml.cpp index 68941c67be8..6c9e9597cec 100644 --- a/storage/connect/tabxml.cpp +++ b/storage/connect/tabxml.cpp @@ -3,7 +3,7 @@ /* ------------- */ /* Version 3.0 */ /* */ -/* Author Olivier BERTRAND 2007 - 2017 */ +/* Author Olivier BERTRAND 2007 - 2020 */ /* */ /* This program are the XML tables classes using MS-DOM or libxml2. */ /***********************************************************************/ @@ -62,6 +62,8 @@ extern "C" char version[]; #define TYPE_UNKNOWN 12 /* Must be greater than other types */ #define XLEN(M) sizeof(M) - strlen(M) - 1 /* To avoid overflow*/ +int GetDefaultDepth(void); + /***********************************************************************/ /* Class and structure used by XMLColumns. */ /***********************************************************************/ @@ -149,8 +151,9 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info) strcpy(g->Message, MSG(MISSING_FNAME)); return NULL; } else { - lvl = GetIntegerTableOption(g, topt, "Level", 0); - lvl = (lvl < 0) ? 0 : (lvl > 16) ? 16 : lvl; + lvl = GetIntegerTableOption(g, topt, "Level", GetDefaultDepth()); + lvl = GetIntegerTableOption(g, topt, "Depth", lvl); + lvl = (lvl < 0) ? 0 : (lvl > 16) ? 16 : lvl; } // endif fn if (trace(1)) diff --git a/storage/connect/user_connect.cc b/storage/connect/user_connect.cc index c25443ef7ef..09d6db1ad27 100644 --- a/storage/connect/user_connect.cc +++ b/storage/connect/user_connect.cc @@ -28,7 +28,7 @@ */ /****************************************************************************/ -/* Author: Olivier Bertrand -- bertrandop@gmail.com -- 2004-2015 */ +/* Author: Olivier Bertrand -- bertrandop@gmail.com -- 2004-2020 */ /****************************************************************************/ #ifdef USE_PRAGMA_IMPLEMENTATION #pragma implementation // gcc: Class implementation @@ -58,8 +58,8 @@ PCONNECT user_connect::to_users= NULL; /****************************************************************************/ /* Get the work_size SESSION variable value . */ /****************************************************************************/ -uint GetWorkSize(void); -void SetWorkSize(uint); +size_t GetWorkSize(void); +void SetWorkSize(size_t); /* -------------------------- class user_connect -------------------------- */ @@ -97,14 +97,14 @@ user_connect::~user_connect() bool user_connect::user_init() { // Initialize Plug-like environment - uint worksize= GetWorkSize(); + size_t worksize= GetWorkSize(); PACTIVITY ap= NULL; PDBUSER dup= NULL; // Areasize= 64M because of VEC tables. Should be parameterisable //g= PlugInit(NULL, 67108864); //g= PlugInit(NULL, 134217728); // 128M was because of old embedded tests - g= PlugInit(NULL, worksize); + g= PlugInit(NULL, (size_t)worksize); // Check whether the initialization is complete if (!g || !g->Sarea || PlugSubSet(g->Sarea, g->Sarea_Size) @@ -157,16 +157,17 @@ void user_connect::SetHandler(ha_connect *hc) bool user_connect::CheckCleanup(bool force) { if (thdp->query_id > last_query_id || force) { - uint worksize= GetWorkSize(), size = g->Sarea_Size; + size_t worksize = GetWorkSize(); PlugCleanup(g, true); - if (size != worksize) { + if (worksize != g->Sarea_Size) { FreeSarea(g); + g->Saved_Size = g->Sarea_Size; // Check whether the work area could be allocated if (AllocSarea(g, worksize)) { - AllocSarea(g, size); + AllocSarea(g, g->Saved_Size); SetWorkSize(g->Sarea_Size); // Was too big } // endif sarea @@ -174,10 +175,11 @@ bool user_connect::CheckCleanup(bool force) PlugSubSet(g->Sarea, g->Sarea_Size); g->Xchk = NULL; - g->Createas = 0; + g->Createas = false; g->Alchecked = 0; g->Mrr = 0; g->More = 0; + g->Saved_Size = 0; last_query_id= thdp->query_id; if (trace(65) && !force) diff --git a/storage/connect/value.cpp b/storage/connect/value.cpp index df75722d0e8..de04f7678f9 100644 --- a/storage/connect/value.cpp +++ b/storage/connect/value.cpp @@ -2251,6 +2251,15 @@ void BINVAL::SetBinValue(void *p) } // end of SetBinValue /***********************************************************************/ +/* BINVAL SetBinValue: fill string with len bytes. */ +/***********************************************************************/ +void BINVAL::SetBinValue(void* p, ulong len) +{ + memcpy(Binp, p, len); + Len = len; +} // end of SetBinValue + +/***********************************************************************/ /* GetBinValue: fill a buffer with the internal binary value. */ /* This function checks whether the buffer length is enough and */ /* returns true if not. Actual filling occurs only if go is true. */ diff --git a/storage/connect/value.h b/storage/connect/value.h index 4f7d9a440fa..ee7a1c8032f 100644 --- a/storage/connect/value.h +++ b/storage/connect/value.h @@ -115,8 +115,8 @@ class DllExport VALUE : public BLOCK { virtual void SetValue(ulonglong) {assert(false);} virtual void SetValue(double) {assert(false);} virtual void SetValue_pvblk(PVBLK blk, int n) = 0; - virtual void SetBinValue(void *p) = 0; - virtual bool GetBinValue(void *buf, int buflen, bool go) = 0; + virtual void SetBinValue(void* p) = 0; + virtual bool GetBinValue(void *buf, int buflen, bool go) = 0; virtual int ShowValue(char *buf, int len) = 0; virtual char *GetCharString(char *p) = 0; virtual bool IsEqual(PVAL vp, bool chktype) = 0; @@ -385,7 +385,8 @@ class DllExport BINVAL: public VALUE { virtual void SetValue(ulonglong n); virtual void SetValue(double f); virtual void SetBinValue(void *p); - virtual bool GetBinValue(void *buf, int buflen, bool go); + virtual void SetBinValue(void* p, ulong len); + virtual bool GetBinValue(void *buf, int buflen, bool go); virtual int CompareValue(PVAL) {assert(false); return 0;} virtual int ShowValue(char *buf, int len); virtual char *GetCharString(char *p); |