From e4797a991f7597892608e9327adfc1ca7841e62d Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Tue, 30 Jul 2019 22:45:04 +0200 Subject: In CONNECT version 1.6.10 NOSQL facility is enhanced by a new way to retrieve NOSQL data. In addition to files and Mongo collections, JSON as well as XML and CSV data can be retrieved from the net as answers from REST queries. Because it uses and external package (cpprestsdk) this is currently available only to MariaDB servers compiled from source. -- Add the REST support when Microsoft Casablanca package (cpprestsdk) is installed. -- Also include some changes specific to MariaDB 10.3. modified: storage/connect/CMakeLists.txt -- Add conditional REST support -- Added string options HTTP and URI. -- Added added internal table type TAB_REST. modified: storage/connect/ha_connect.cc modified: storage/connect/mycat.cc modified: storage/connect/mycat.h modified: storage/connect/plgdbsem.h -- Fix MDEV-19648 Variable connect_conv_size doesn't change -- Change Variable wrong block parameter from 8169 to 1. -- Also change connect_conv_size default value to 1024. modified: storage/connect/ha_connect.cc -- Avoid possible buffer overflow -- In particular by the function ShowValue. modified: storage/connect/tabdos.cpp modified: storage/connect/tabfmt.cpp modified: storage/connect/value.cpp modified: storage/connect/value.h -- Add some cast to avoid some compiler warnings modified: storage/connect/filamdbf.cpp -- Fix some C++ error modified: storage/connect/javaconn.cpp modified: storage/connect/jmgoconn.cpp modified: storage/connect/plugutil.cpp -- Miscellaneous Typo and warning suppressing changes modified: storage/connect/connect.cpp modified: storage/connect/connect.h modified: storage/connect/filamvct.cpp modified: storage/connect/inihandl.cpp modified: storage/connect/jsonudf.cpp modified: storage/connect/libdoc.cpp modified: storage/connect/tabjson.cpp modified: storage/connect/tabtbl.cpp modified: storage/connect/tabxml.cpp modified: storage/connect/user_connect.cc modified: storage/connect/user_connect.h -- Update failing test results and disbling modified: storage/connect/mysql-test/connect/disabled.def modified: storage/connect/mysql-test/connect/r/dir.result modified: storage/connect/mysql-test/connect/r/grant.result modified: storage/connect/mysql-test/connect/r/jdbc.result modified: storage/connect/mysql-test/connect/r/jdbc_postgresql.result modified: storage/connect/mysql-test/connect/r/xml.result modified: storage/connect/mysql-test/connect/r/xml2.result modified: storage/connect/mysql-test/connect/r/xml2_mult.result modified: storage/connect/mysql-test/connect/r/xml_mult.result -- Add an option modified: storage/connect/mysql-test/connect/t/grant.test --- storage/connect/CMakeLists.txt | 33 ++- storage/connect/connect.h | 2 +- storage/connect/ha_connect.cc | 635 +++++++++++++++++++++------------------- storage/connect/ha_connect.h | 6 +- storage/connect/inihandl.cpp | 4 +- storage/connect/jmgoconn.cpp | 5 +- storage/connect/jsonudf.cpp | 8 +- storage/connect/libdoc.cpp | 2 +- storage/connect/mycat.cc | 31 +- storage/connect/mycat.h | 6 +- storage/connect/plgdbsem.h | 4 +- storage/connect/restget.cpp | 87 ++++++ storage/connect/tabjdbc.cpp | 4 +- storage/connect/tabjson.cpp | 2 +- storage/connect/tabrest.cpp | 133 +++++++++ storage/connect/tabrest.h | 29 ++ storage/connect/tabtbl.cpp | 4 +- storage/connect/tabxml.cpp | 2 +- storage/connect/user_connect.cc | 2 +- storage/connect/user_connect.h | 2 +- 20 files changed, 665 insertions(+), 336 deletions(-) create mode 100644 storage/connect/restget.cpp create mode 100644 storage/connect/tabrest.cpp create mode 100644 storage/connect/tabrest.h (limited to 'storage') diff --git a/storage/connect/CMakeLists.txt b/storage/connect/CMakeLists.txt index c8747f889c6..8fc7cf695e7 100644 --- a/storage/connect/CMakeLists.txt +++ b/storage/connect/CMakeLists.txt @@ -11,7 +11,7 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA SET(CONNECT_PLUGIN_STATIC "connect") SET(CONNECT_PLUGIN_DYNAMIC "connect") @@ -69,6 +69,10 @@ ELSE(NOT UNIX) # Add exception handling to the CONNECT project) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc") SET(IPHLPAPI_LIBRARY iphlpapi.lib) + IF(MSVC AND (CMAKE_CXX_COMPILER_ID MATCHES Clang)) + # Connect does not work with clang-cl + RETURN() + ENDIF() ENDIF(UNIX) @@ -109,7 +113,6 @@ IF(CONNECT_WITH_LIBXML2) FIND_PACKAGE(LibXml2) IF (LIBXML2_FOUND) INCLUDE_DIRECTORIES(${LIBXML2_INCLUDE_DIR}) - SET(ZLIB_LIBRARY "z") # see ZLIB_INCLUDE_DIR below SET(XML_LIBRARY ${LIBXML2_LIBRARIES}) SET(CONNECT_SOURCES ${CONNECT_SOURCES} libdoc.cpp libdoc.h) add_definitions(-DLIBXML2_SUPPORT) @@ -301,6 +304,24 @@ IF(CONNECT_WITH_MONGO) ENDIF(CONNECT_WITH_MONGO) +# +# REST +# + +OPTION(CONNECT_WITH_REST "Compile CONNECT storage engine with REST support" ON) + +IF(CONNECT_WITH_REST) + MESSAGE(STATUS "=====> REST support is ON") + FIND_PACKAGE(cpprestsdk) + IF (cpprestsdk_FOUND) + MESSAGE(STATUS "=====> cpprestsdk found") + SET(CONNECT_SOURCES ${CONNECT_SOURCES} tabrest.cpp restget.cpp tabrest.h) + add_definitions(-DREST_SUPPORT) + ELSE(NOT cpprestsdk_FOUND) + MESSAGE(STATUS "=====> cpprestsdk package not found") + ENDIF (cpprestsdk_FOUND) +ENDIF(CONNECT_WITH_REST) + # # XMAP # @@ -326,14 +347,6 @@ IF(NOT TARGET connect) RETURN() ENDIF() -# Don't link with bundled zlib and systel libxml2 at the same time. -# System libxml2 uses system zlib, might conflict with the bundled one. -IF (XML_LIBRARY AND BUILD_BUNDLED_ZLIB) - GET_PROPERTY(INCS TARGET connect PROPERTY INCLUDE_DIRECTORIES) - LIST(REMOVE_ITEM INCS ${ZLIB_INCLUDE_DIR}) - SET_PROPERTY(TARGET connect PROPERTY INCLUDE_DIRECTORIES ${INCS}) -ENDIF() - IF(WIN32) IF (libmongoc-1.0_FOUND) SET_TARGET_PROPERTIES(connect PROPERTIES LINK_FLAGS diff --git a/storage/connect/connect.h b/storage/connect/connect.h index 2bca8bf54cb..3a60cd40160 100644 --- a/storage/connect/connect.h +++ b/storage/connect/connect.h @@ -11,7 +11,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ /**************** Cnt H Declares Source Code File (.H) *****************/ /* Name: CONNECT.H Version 2.4 */ diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index 191230cb03c..e7d1b5f0e86 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -11,7 +11,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ /** @file ha_connect.cc @@ -164,15 +164,15 @@ /***********************************************************************/ /* Initialize the ha_connect static members. */ /***********************************************************************/ -#define SZCONV 8192 +#define SZCONV 1024 // Default converted text size #define SZWORK 67108864 // Default work area size 64M #define SZWMIN 4194304 // Minimum work area size 4M #define JSONMAX 10 // JSON Default max grp size extern "C" { - char version[]= "Version 1.06.0009 January 27, 2019"; + char version[]= "Version 1.06.0010 June 01, 2019"; #if defined(__WIN__) - char compver[]= "Version 1.06.0009 " __DATE__ " " __TIME__; + char compver[]= "Version 1.06.0010 " __DATE__ " " __TIME__; char slash= '\\'; #else // !__WIN__ char slash= '/'; @@ -211,14 +211,14 @@ char *GetUserVariable(PGLOBAL g, const uchar *varname) { char buf[1024]; bool b; - THD *thd = current_thd; - CHARSET_INFO *cs = system_charset_info; - String *str = NULL, tmp(buf, sizeof(buf), cs); - HASH uvars = thd->user_vars; - user_var_entry *uvar = (user_var_entry*)my_hash_search(&uvars, varname, 0); + THD *thd= current_thd; + CHARSET_INFO *cs= system_charset_info; + String *str= NULL, tmp(buf, sizeof(buf), cs); + HASH uvars= thd->user_vars; + user_var_entry *uvar= (user_var_entry*)my_hash_search(&uvars, varname, 0); if (uvar) - str = uvar->val_str(&b, &tmp, NOT_FIXED_DEC); + str= uvar->val_str(&b, &tmp, NOT_FIXED_DEC); return str ? PlugDup(g, str->ptr()) : NULL; }; // end of GetUserVariable @@ -231,6 +231,9 @@ PQRYRES OEMColumns(PGLOBAL g, PTOS topt, char *tab, char *db, bool info); PQRYRES VirColumns(PGLOBAL g, bool info); PQRYRES JSONColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt, bool info); PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info); +#if defined(REST_SUPPORT) +PQRYRES RESTColumns(PGLOBAL g, PTOS topt, char *tab, char *db, bool info); +#endif // REST_SUPPORT #if defined(JAVA_SUPPORT) PQRYRES MGOColumns(PGLOBAL g, PCSZ db, PCSZ url, PTOS topt, bool info); #endif // JAVA_SUPPORT @@ -352,7 +355,7 @@ static MYSQL_THDVAR_UINT(work_size, static MYSQL_THDVAR_INT(conv_size, PLUGIN_VAR_RQCMDARG, // opt "Size used when converting TEXT columns.", - NULL, NULL, SZCONV, 0, 65500, 8192); + NULL, NULL, SZCONV, 0, 65500, 1); /** Type conversion: @@ -583,7 +586,9 @@ ha_create_table_option connect_table_option_list[]= HA_TOPTION_STRING("FILTER", filter), HA_TOPTION_STRING("OPTION_LIST", oplist), HA_TOPTION_STRING("DATA_CHARSET", data_charset), - HA_TOPTION_NUMBER("LRECL", lrecl, 0, 0, INT_MAX32, 1), + HA_TOPTION_STRING("HTTP", http), + HA_TOPTION_STRING("URI", uri), + HA_TOPTION_NUMBER("LRECL", lrecl, 0, 0, INT_MAX32, 1), HA_TOPTION_NUMBER("BLOCK_SIZE", elements, 0, 0, INT_MAX32, 1), //HA_TOPTION_NUMBER("ESTIMATE", estimate, 0, 0, INT_MAX32, 1), HA_TOPTION_NUMBER("MULTIPLE", multiple, 0, 0, 3, 1), @@ -991,11 +996,11 @@ static PCONNECT GetUser(THD *thd, PCONNECT xp) pthread_mutex_unlock(&usrmut); if (!xp) { - xp = new user_connect(thd); + xp= new user_connect(thd); if (xp->user_init()) { delete xp; - xp = NULL; + xp= NULL; } // endif user_init } // endif xp @@ -1027,6 +1032,19 @@ TABTYPE ha_connect::GetRealType(PTOS pos) if (type == TAB_UNDEF) type= pos->srcdef ? TAB_MYSQL : pos->tabname ? TAB_PRX : TAB_DOS; +#if defined(REST_SUPPORT) + else if (pos->http) + switch (type) { + case TAB_JSON: + case TAB_XML: + case TAB_CSV: + type = TAB_REST; + break; + case TAB_REST: + type = TAB_NIY; + break; + } // endswitch type +#endif // REST_SUPPORT } else type= TAB_UNDEF; @@ -1130,48 +1148,48 @@ PCSZ GetListOption(PGLOBAL g, PCSZ opname, PCSZ oplist, PCSZ def) return (char*)def; char key[16], val[256]; - char *pv, *pn, *pk = (char*)oplist; - PCSZ opval = def; + char *pv, *pn, *pk= (char*)oplist; + PCSZ opval= def; int n; while (*pk == ' ') pk++; - for (; pk; pk = pn) { - pn = strchr(pk, ','); - pv = strchr(pk, '='); + for (; pk; pk= pn) { + pn= strchr(pk, ','); + pv= strchr(pk, '='); if (pv && (!pn || pv < pn)) { - n = MY_MIN(static_cast(pv - pk), sizeof(key) - 1); + n= MY_MIN(static_cast(pv - pk), sizeof(key) - 1); memcpy(key, pk, n); while (n && key[n - 1] == ' ') n--; - key[n] = 0; + key[n]= 0; while (*(++pv) == ' '); - n = MY_MIN((pn ? pn - pv : strlen(pv)), sizeof(val) - 1); + n= MY_MIN((pn ? pn - pv : strlen(pv)), sizeof(val) - 1); memcpy(val, pv, n); while (n && val[n - 1] == ' ') n--; - val[n] = 0; + val[n]= 0; } else { - n = MY_MIN((pn ? pn - pk : strlen(pk)), sizeof(key) - 1); + n= MY_MIN((pn ? pn - pk : strlen(pk)), sizeof(key) - 1); memcpy(key, pk, n); while (n && key[n - 1] == ' ') n--; - key[n] = 0; - val[0] = 0; + key[n]= 0; + val[0]= 0; } // endif pv if (!stricmp(opname, key)) { - opval = PlugDup(g, val); + opval= PlugDup(g, val); break; } else if (!pn) break; @@ -1219,9 +1237,13 @@ PCSZ GetStringTableOption(PGLOBAL g, PTOS options, PCSZ opname, PCSZ sdef) else if (!stricmp(opname, "Colist")) opval= options->colist; else if (!stricmp(opname, "Filter")) - opval = options->filter; + opval= options->filter; else if (!stricmp(opname, "Data_charset")) opval= options->data_charset; + else if (!stricmp(opname, "Http") || !stricmp(opname, "URL")) + opval = options->http; + else if (!stricmp(opname, "Uri")) + opval = options->uri; if (!opval && options->oplist) opval= GetListOption(g, opname, options->oplist); @@ -1252,7 +1274,7 @@ bool GetBooleanTableOption(PGLOBAL g, PTOS options, PCSZ opname, bool bdef) else if (!stricmp(opname, "Header")) opval= (options->header != 0); // Is Boolean for some table types else if (!stricmp(opname, "Zipped")) - opval = options->zipped; + opval= options->zipped; else if (options->oplist) if ((pv= GetListOption(g, opname, options->oplist))) opval= (!*pv || *pv == 'y' || *pv == 'Y' || atoi(pv) != 0); @@ -1346,8 +1368,8 @@ PCSZ ha_connect::GetStringOption(PCSZ opname, PCSZ sdef) } else if (!stricmp(opname, "Query_String")) { // This escapes everything and returns a wrong query -// opval = thd_query_string(table->in_use)->str; - opval = (PCSZ)PlugSubAlloc(xp->g, NULL, +// opval= thd_query_string(table->in_use)->str; + opval= (PCSZ)PlugSubAlloc(xp->g, NULL, thd_query_string(table->in_use)->length + 1); strcpy((char*)opval, thd_query_string(table->in_use)->str); // sprintf((char*)opval, "%s", thd_query_string(table->in_use)->str); @@ -1368,7 +1390,7 @@ PCSZ ha_connect::GetStringOption(PCSZ opname, PCSZ sdef) || !stricmp(opname, "filename") || !stricmp(opname, "optname") || !stricmp(opname, "entry"))) - opval = GetRealString(opval); + opval= GetRealString(opval); if (!opval) { if (sdef && !strcmp(sdef, "*")) { @@ -1497,7 +1519,7 @@ PFOS ha_connect::GetFieldOptionStruct(Field *fdp) void *ha_connect::GetColumnOption(PGLOBAL g, void *field, PCOLINFO pcf) { const char *cp; - char *chset, v = 0; + char *chset, v= 0; ha_field_option_struct *fop; Field* fp; Field* *fldp; @@ -1549,7 +1571,7 @@ void *ha_connect::GetColumnOption(PGLOBAL g, void *field, PCOLINFO pcf) pcf->Fieldfmt= NULL; } // endif fop - chset = (char *)fp->charset()->name; + chset= (char *)fp->charset()->name; switch (fp->type()) { case MYSQL_TYPE_BLOB: @@ -1817,7 +1839,7 @@ const char *ha_connect::GetTableName(void) { const char *path= tshp ? tshp->path.str : table_share->path.str; const char *name= strrchr(path, slash); - return name ? name+1 : path; + return name ? name + 1 : path; } // end of GetTableName char *ha_connect::GetPartName(void) @@ -2055,10 +2077,10 @@ bool ha_connect::CheckColumnList(PGLOBAL g) } catch (int n) { if (trace(1)) htrc("Exception %d: %s\n", n, g->Message); - brc = true; + brc= true; } catch (const char *msg) { strcpy(g->Message, msg); - brc = true; + brc= true; } // end catch return brc; @@ -2185,9 +2207,9 @@ int ha_connect::MakeRecord(char *buf) rc= fp->store(p, strlen(p), charset, CHECK_FIELD_WARN); break; case TYPE_BIN: - p = value->GetCharValue(); - charset = &my_charset_bin; - rc = fp->store(p, strlen(p), charset, CHECK_FIELD_WARN); + p= value->GetCharValue(); + charset= &my_charset_bin; + rc= fp->store(p, strlen(p), charset, CHECK_FIELD_WARN); break; case TYPE_DOUBLE: rc= fp->store(value->GetFloatValue()); @@ -2436,7 +2458,7 @@ bool ha_connect::MakeKeyWhere(PGLOBAL g, PSTRG qry, OPVAL vop, char q, kfp= &table->key_info[active_index]; old_map= dbug_tmp_use_all_columns(table, table->write_set); - for (i = 0; i <= 1; i++) { + for (i= 0; i <= 1; i++) { if (ranges[i] == NULL) continue; @@ -2547,7 +2569,7 @@ const char *ha_connect::GetValStr(OPVAL vop, bool neg) switch (vop) { case OP_EQ: - val= " = "; + val= "= "; break; case OP_NE: val= " <> "; @@ -2817,7 +2839,7 @@ PFIL ha_connect::CondFilter(PGLOBAL g, Item *cond) /***********************************************************************/ PCFIL ha_connect::CheckCond(PGLOBAL g, PCFIL filp, const Item *cond) { - AMT tty = filp->Type; + AMT tty= filp->Type; char *body= filp->Body; char *havg= filp->Having; unsigned int i; @@ -2834,7 +2856,7 @@ PCFIL ha_connect::CheckCond(PGLOBAL g, PCFIL filp, const Item *cond) if (cond->type() == COND::COND_ITEM) { char *pb0, *pb1, *pb2, *ph0= 0, *ph1= 0, *ph2= 0; - bool bb = false, bh = false; + bool bb= false, bh= false; Item_cond *cond_item= (Item_cond *)cond; if (x) @@ -2893,13 +2915,13 @@ PCFIL ha_connect::CheckCond(PGLOBAL g, PCFIL filp, const Item *cond) bb |= filp->Bd; bh |= filp->Hv; - filp->Bd = filp->Hv = false; + filp->Bd= filp->Hv= false; } else return NULL; if (bb) { strcpy(pb1, ")"); - filp->Bd = bb; + filp->Bd= bb; } else *pb0= 0; @@ -2907,13 +2929,13 @@ PCFIL ha_connect::CheckCond(PGLOBAL g, PCFIL filp, const Item *cond) if (bb && bh && vop == OP_OR) { // Cannot or'ed a where clause with a having clause bb= bh= 0; - *pb0 = 0; - *ph0 = 0; + *pb0= 0; + *ph0= 0; } else if (bh) { strcpy(ph1, ")"); filp->Hv= bh; } else - *ph0 = 0; + *ph0= 0; } // endif havg @@ -2926,7 +2948,7 @@ PCFIL ha_connect::CheckCond(PGLOBAL g, PCFIL filp, const Item *cond) Item_func *condf= (Item_func *)cond; Item* *args= condf->arguments(); - filp->Bd = filp->Hv = false; + filp->Bd= filp->Hv= false; if (trace(1)) htrc("Func type=%d argnum=%d\n", condf->functype(), @@ -2942,10 +2964,10 @@ PCFIL ha_connect::CheckCond(PGLOBAL g, PCFIL filp, const Item *cond) case Item_func::GT_FUNC: vop= OP_GT; break; case Item_func::LIKE_FUNC: vop= OP_LIKE; - neg = ((Item_func_opt_neg *)condf)->negated; + neg= ((Item_func_opt_neg *)condf)->negated; break; case Item_func::ISNOTNULL_FUNC: - neg = true; + neg= true; // fall through case Item_func::ISNULL_FUNC: vop= OP_NULL; break; case Item_func::IN_FUNC: vop= OP_IN; /* fall through */ @@ -3003,12 +3025,12 @@ PCFIL ha_connect::CheckCond(PGLOBAL g, PCFIL filp, const Item *cond) } else { bool h; - fnm = filp->Chk(pField->field->field_name, &h); + fnm= filp->Chk(pField->field->field_name, &h); if (h && i && !ishav) return NULL; // Having should be col VOP arg else - ishav = h; + ishav= h; } // endif's @@ -3059,7 +3081,7 @@ PCFIL ha_connect::CheckCond(PGLOBAL g, PCFIL filp, const Item *cond) if (!x) { const char *p; - char *s = (ishav) ? havg : body; + char *s= (ishav) ? havg : body; uint j, k, n; // Append the value to the filter @@ -3116,37 +3138,37 @@ PCFIL ha_connect::CheckCond(PGLOBAL g, PCFIL filp, const Item *cond) strcat(s, "'}"); break; default: - j = strlen(s); - s[j++] = '\''; - p = res->ptr(); - n = res->length(); + j= strlen(s); + s[j++]= '\''; + p= res->ptr(); + n= res->length(); - for (k = 0; k < n; k++) { + for (k= 0; k < n; k++) { if (p[k] == '\'') - s[j++] = '\''; + s[j++]= '\''; - s[j++] = p[k]; + s[j++]= p[k]; } // endfor k - s[j++] = '\''; - s[j] = 0; + s[j++]= '\''; + s[j]= 0; } // endswitch field type } else { - j = strlen(s); - s[j++] = '\''; - p = res->ptr(); - n = res->length(); + j= strlen(s); + s[j++]= '\''; + p= res->ptr(); + n= res->length(); - for (k = 0; k < n; k++) { + for (k= 0; k < n; k++) { if (p[k] == '\'') - s[j++] = '\''; + s[j++]= '\''; - s[j++] = p[k]; + s[j++]= p[k]; } // endfor k - s[j++] = '\''; - s[j] = 0; + s[j++]= '\''; + s[j]= 0; } // endif tty break; @@ -3170,7 +3192,7 @@ PCFIL ha_connect::CheckCond(PGLOBAL g, PCFIL filp, const Item *cond) } // endif's Type if (!x) { - char *s = (ishav) ? havg : body; + char *s= (ishav) ? havg : body; if (!i) strcat(s, GetValStr(vop, neg)); @@ -3184,11 +3206,11 @@ PCFIL ha_connect::CheckCond(PGLOBAL g, PCFIL filp, const Item *cond) } // endfor i if (x) - filp->Op = vop; + filp->Op= vop; else if (ishav) - filp->Hv = true; + filp->Hv= true; else - filp->Bd = true; + filp->Bd= true; } else { if (trace(1)) @@ -3238,21 +3260,21 @@ const COND *ha_connect::cond_push(const COND *cond) PCFIL filp; int rc; - if ((filp = tdbp->GetCondFil()) && tdbp->GetCond() == cond && + if ((filp= tdbp->GetCondFil()) && tdbp->GetCond() == cond && filp->Idx == active_index && filp->Type == tty) goto fin; - filp = new(g) CONDFIL(active_index, tty); - rc = filp->Init(g, this); + filp= new(g) CONDFIL(active_index, tty); + rc= filp->Init(g, this); if (rc == RC_INFO) { - filp->Having = (char*)PlugSubAlloc(g, NULL, 256); - *filp->Having = 0; + filp->Having= (char*)PlugSubAlloc(g, NULL, 256); + *filp->Having= 0; } else if (rc == RC_FX) goto fin; - filp->Body = (char*)PlugSubAlloc(g, NULL, (x) ? 128 : 0); - *filp->Body = 0; + filp->Body= (char*)PlugSubAlloc(g, NULL, (x) ? 128 : 0); + *filp->Body= 0; if (CheckCond(g, filp, cond)) { if (filp->Having && strlen(filp->Having) > 255) @@ -3266,7 +3288,7 @@ const COND *ha_connect::cond_push(const COND *cond) if (!x) PlugSubAlloc(g, NULL, strlen(filp->Body) + 1); else - cond = NULL; // Does this work? + cond= NULL; // Does this work? tdbp->SetCondFil(filp); } else if (x && cond) @@ -3316,8 +3338,8 @@ ha_rows ha_connect::records() int ha_connect::check(THD* thd, HA_CHECK_OPT* check_opt) { - int rc = HA_ADMIN_OK; - PGLOBAL g = ((table && table->in_use) ? GetPlug(table->in_use, xp) : + int rc= HA_ADMIN_OK; + PGLOBAL g= ((table && table->in_use) ? GetPlug(table->in_use, xp) : (xp) ? xp->g : NULL); DBUG_ENTER("ha_connect::check"); @@ -3327,32 +3349,32 @@ int ha_connect::check(THD* thd, HA_CHECK_OPT* check_opt) // Do not close the table if it was opened yet (possible?) if (IsOpened()) { if (IsPartitioned() && CheckColumnList(g)) // map can have been changed - rc = HA_ADMIN_CORRUPT; + rc= HA_ADMIN_CORRUPT; else if (tdbp->OpenDB(g)) // Rewind table - rc = HA_ADMIN_CORRUPT; + rc= HA_ADMIN_CORRUPT; } else if (xp->CheckQuery(valid_query_id)) { - tdbp = NULL; // Not valid anymore + tdbp= NULL; // Not valid anymore if (OpenTable(g, false)) - rc = HA_ADMIN_CORRUPT; + rc= HA_ADMIN_CORRUPT; } else // possible? DBUG_RETURN(HA_ADMIN_INTERNAL_ERROR); if (rc == HA_ADMIN_OK) { - TABTYPE type = GetTypeID(GetStringOption("Type", "*")); + TABTYPE type= GetTypeID(GetStringOption("Type", "*")); if (IsFileType(type)) { if (check_opt->flags & T_MEDIUM) { // TO DO do { - if ((rc = CntReadNext(g, tdbp)) == RC_FX) + if ((rc= CntReadNext(g, tdbp)) == RC_FX) break; } while (rc != RC_EF); - rc = (rc == RC_EF) ? HA_ADMIN_OK : HA_ADMIN_CORRUPT; + rc= (rc == RC_EF) ? HA_ADMIN_OK : HA_ADMIN_CORRUPT; } else if (check_opt->flags & T_EXTEND) { // TO DO } // endif's flags @@ -3380,7 +3402,7 @@ bool ha_connect::get_error_message(int error, String* buf) DBUG_ENTER("ha_connect::get_error_message"); if (xp && xp->g) { - PGLOBAL g = xp->g; + PGLOBAL g= xp->g; if (trace(1)) htrc("GEM(%d): %s\n", error, g->Message); @@ -3489,32 +3511,32 @@ int ha_connect::optimize(THD* thd, HA_CHECK_OPT*) try { // Ignore error on the opt file dup->Check &= ~CHK_OPT; - tdbp = GetTDB(g); + tdbp= GetTDB(g); dup->Check |= CHK_OPT; if (tdbp && !tdbp->IsRemote()) { - bool dop = IsTypeIndexable(GetRealType(NULL)); - bool dox = (tdbp->GetDef()->Indexable() == 1); + bool dop= IsTypeIndexable(GetRealType(NULL)); + bool dox= (tdbp->GetDef()->Indexable() == 1); - if ((rc = ((PTDBASE)tdbp)->ResetTableOpt(g, dop, dox))) { + if ((rc= ((PTDBASE)tdbp)->ResetTableOpt(g, dop, dox))) { if (rc == RC_INFO) { push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 0, g->Message); - rc = 0; + rc= 0; } else - rc = HA_ERR_CRASHED_ON_USAGE; // Table must be repaired + rc= HA_ERR_CRASHED_ON_USAGE; // Table must be repaired } // endif rc } else if (!tdbp) - rc = HA_ERR_INTERNAL_ERROR; + rc= HA_ERR_INTERNAL_ERROR; } catch (int n) { if (trace(1)) htrc("Exception %d: %s\n", n, g->Message); - rc = HA_ERR_INTERNAL_ERROR; + rc= HA_ERR_INTERNAL_ERROR; } catch (const char *msg) { strcpy(g->Message, msg); - rc = HA_ERR_INTERNAL_ERROR; + rc= HA_ERR_INTERNAL_ERROR; } // end catch if (rc) @@ -4214,7 +4236,7 @@ int ha_connect::rnd_pos(uchar *buf, uchar *pos) tdbp->SetFilter(NULL); rc= rnd_next(buf); } else { - PGLOBAL g = GetPlug((table) ? table->in_use : NULL, xp); + PGLOBAL g= GetPlug((table) ? table->in_use : NULL, xp); // strcpy(g->Message, "Not supported by this table type"); my_message(ER_ILLEGAL_HA, g->Message, MYF(0)); rc= HA_ERR_INTERNAL_ERROR; @@ -4298,12 +4320,12 @@ int ha_connect::info(uint flag) } else DBUG_RETURN(HA_ERR_INTERNAL_ERROR); // Should never happen - if (!(tdbp = GetTDB(g))) { + if (!(tdbp= GetTDB(g))) { my_message(ER_UNKNOWN_ERROR, g->Message, MYF(0)); DBUG_RETURN(HA_ERR_INTERNAL_ERROR); } // endif tdbp - valid_info = false; + valid_info= false; } // endif tdbp if (!valid_info) { @@ -4446,6 +4468,7 @@ bool ha_connect::check_privileges(THD *thd, PTOS options, char *dbn, bool quick) case TAB_XML: case TAB_INI: case TAB_VEC: + case TAB_REST: case TAB_JSON: if (options->filename && *options->filename) { if (!quick) { @@ -4582,13 +4605,13 @@ MODE ha_connect::CheckMode(PGLOBAL g, THD *thd, // newmode= MODE_UPDATE; // To be checked // break; case SQLCOM_DELETE_MULTI: - *cras = true; + *cras= true; case SQLCOM_DELETE: case SQLCOM_TRUNCATE: newmode= MODE_DELETE; break; case SQLCOM_UPDATE_MULTI: - *cras = true; + *cras= true; case SQLCOM_UPDATE: newmode= MODE_UPDATE; break; @@ -4616,7 +4639,7 @@ MODE ha_connect::CheckMode(PGLOBAL g, THD *thd, break; // } // endif partitioned case SQLCOM_REPAIR: // TODO implement it - newmode = MODE_UPDATE; + newmode= MODE_UPDATE; break; default: htrc("Unsupported sql_command=%d\n", thd_sql_command(thd)); @@ -4731,15 +4754,15 @@ int ha_connect::start_stmt(THD *thd, thr_lock_type lock_type) if (CloseTable(g)) { // Make error a warning to avoid crash push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 0, g->Message); - rc = 0; + rc= 0; } // endif Close - locked = 0; - xmod = MODE_ANY; // For info commands + locked= 0; + xmod= MODE_ANY; // For info commands DBUG_RETURN(rc); } // endif MODE_ANY - newmode = CheckMode(g, thd, newmode, &chk, &cras); + newmode= CheckMode(g, thd, newmode, &chk, &cras); if (newmode == MODE_ERROR) DBUG_RETURN(HA_ERR_INTERNAL_ERROR); @@ -4815,7 +4838,7 @@ int ha_connect::external_lock(THD *thd, int lock_type) DBUG_RETURN(0); } else if (g->Xchk) { if (!tdbp) { - if (!(tdbp = GetTDB(g))) { + if (!(tdbp= GetTDB(g))) { // DBUG_RETURN(HA_ERR_INTERNAL_ERROR); causes assert error push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 0, g->Message); DBUG_RETURN(0); @@ -4952,7 +4975,7 @@ int ha_connect::external_lock(THD *thd, int lock_type) int ha_connect::check_stmt(PGLOBAL g, MODE newmode, bool cras) { - int rc = 0; + int rc= 0; DBUG_ENTER("ha_connect::check_stmt"); // If this is the start of a new query, cleanup the previous one @@ -5039,7 +5062,7 @@ THR_LOCK_DATA **ha_connect::store_lock(THD *, { if (lock_type != TL_IGNORE && lock.type == TL_UNLOCK) lock.type=lock_type; - *to++ = &lock; + *to++= &lock; return to; } @@ -5316,7 +5339,7 @@ static bool add_field(String *sql, const char *field_name, int typ, int len, int dec, char *key, uint tm, const char *rem, char *dft, char *xtra, char *fmt, int flag, bool dbf, char v) { - char var = (len > 255) ? 'V' : v; + char var= (len > 255) ? 'V' : v; bool q, error= false; const char *type= PLGtoMYSQLtype(typ, dbf, var); @@ -5360,9 +5383,9 @@ static bool add_field(String *sql, const char *field_name, int typ, int len, error|= sql->append(" DEFAULT "); if (typ == TYPE_DATE) - q = (strspn(dft, "0123456789 -:/") == strlen(dft)); + q= (strspn(dft, "0123456789 -:/") == strlen(dft)); else - q = !IsTypeNum(typ); + q= !IsTypeNum(typ); if (q) { error|= sql->append("'"); @@ -5514,13 +5537,13 @@ static int connect_assisted_discovery(handlerton *, THD* thd, PCSZ fncn= "?"; PCSZ user, fn, db, host, pwd, sep, tbl, src; PCSZ col, ocl, rnk, pic, fcl, skc, zfn; - char *tab, *dsn, *shm, *dpath; + char *tab, *dsn, *shm, *dpath, *url; #if defined(__WIN__) PCSZ nsp= NULL, cls= NULL; #endif // __WIN__ //int hdr, mxe; - int port = 0, mxr = 0, rc = 0, mul = 0, lrecl = 0; -//PCSZ tabtyp = NULL; + int port= 0, mxr= 0, rc= 0, mul= 0, lrecl= 0; +//PCSZ tabtyp= NULL; #if defined(ODBC_SUPPORT) POPARM sop= NULL; PCSZ ucnc= NULL; @@ -5530,7 +5553,6 @@ static int connect_assisted_discovery(handlerton *, THD* thd, #if defined(JAVA_SUPPORT) PJPARM sjp= NULL; PCSZ driver= NULL; - char *url= NULL; #endif // JAVA_SUPPORT uint tm, fnc= FNC_NO, supfnc= (FNC_NO | FNC_COL); bool bif, ok= false, dbf= false; @@ -5550,7 +5572,8 @@ static int connect_assisted_discovery(handlerton *, THD* thd, String sql(buf, sizeof(buf), system_charset_info); sql.copy(STRING_WITH_LEN("CREATE TABLE whatever ("), system_charset_info); - user= host= pwd= tbl= src= col= ocl= pic= fcl= skc= rnk= zfn= dsn= NULL; + user = host = pwd = tbl = src = col = ocl = pic = fcl = skc = rnk = zfn = NULL; + dsn = url = NULL; // Get the useful create options ttp= GetTypeID(topt->type); @@ -5561,7 +5584,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, fncn= topt->catfunc; fnc= GetFuncID(fncn); sep= topt->separator; - mul = (int)topt->multiple; + mul= (int)topt->multiple; tbl= topt->tablist; col= topt->colist; @@ -5584,7 +5607,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, #endif // __WIN__ port= atoi(GetListOption(g, "port", topt->oplist, "0")); #if defined(ODBC_SUPPORT) -// tabtyp = GetListOption(g, "Tabtype", topt->oplist, NULL); +// tabtyp= GetListOption(g, "Tabtype", topt->oplist, NULL); mxr= atoi(GetListOption(g,"maxres", topt->oplist, "0")); cto= atoi(GetListOption(g,"ConnectTimeout", topt->oplist, "-1")); qto= atoi(GetListOption(g,"QueryTimeout", topt->oplist, "-1")); @@ -5599,7 +5622,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, cop= atoi(GetListOption(g, "checkdsn", topt->oplist, "0")); #endif // PROMPT_OK #if defined(ZIP_SUPPORT) - zfn = GetListOption(g, "Zipfile", topt->oplist, NULL); + zfn= GetListOption(g, "Zipfile", topt->oplist, NULL); #endif // ZIP_SUPPORT } else { host= "localhost"; @@ -5612,14 +5635,24 @@ static int connect_assisted_discovery(handlerton *, THD* thd, try { // Check table type if (ttp == TAB_UNDEF) { - topt->type = (src) ? "MYSQL" : (tab) ? "PROXY" : "DOS"; - ttp = GetTypeID(topt->type); + topt->type= (src) ? "MYSQL" : (tab) ? "PROXY" : "DOS"; + ttp= GetTypeID(topt->type); sprintf(g->Message, "No table_type. Was set to %s", topt->type); push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 0, g->Message); } else if (ttp == TAB_NIY) { sprintf(g->Message, "Unsupported table type %s", topt->type); - rc = HA_ERR_INTERNAL_ERROR; + rc= HA_ERR_INTERNAL_ERROR; goto err; +#if defined(REST_SUPPORT) + } else if (topt->http) { + switch (ttp) { + case TAB_JSON: + case TAB_XML: + case TAB_CSV: + ttp = TAB_REST; + break; + } // endswitch type +#endif // REST_SUPPORT } // endif ttp if (!tab) { @@ -5629,39 +5662,39 @@ static int connect_assisted_discovery(handlerton *, THD* thd, if (!tbl) { strcpy(g->Message, "Missing table list"); - rc = HA_ERR_INTERNAL_ERROR; + rc= HA_ERR_INTERNAL_ERROR; goto err; } // endif tbl - tab = PlugDup(g, tbl); + tab= PlugDup(g, tbl); - if ((p = strchr(tab, ','))) - *p = 0; + if ((p= strchr(tab, ','))) + *p= 0; - if ((p = strchr(tab, '.'))) { - *p = 0; - db = tab; - tab = p + 1; + if ((p= strchr(tab, '.'))) { + *p= 0; + db= tab; + tab= p + 1; } // endif p } else if (ttp != TAB_ODBC || !(fnc & (FNC_TABLE | FNC_COL))) - tab = (char*)table_s->table_name.str; // Default value + tab= (char*)table_s->table_name.str; // Default value } // endif tab switch (ttp) { #if defined(ODBC_SUPPORT) case TAB_ODBC: - dsn = strz(g, create_info->connect_string); + dsn= strz(g, create_info->connect_string); if (fnc & (FNC_DSN | FNC_DRIVER)) { - ok = true; + ok= true; #if defined(PROMPT_OK) } else if (!stricmp(thd->main_security_ctx.host, "localhost") && cop == 1) { - if ((dsn = ODBCCheckConnection(g, dsn, cop)) != NULL) { + if ((dsn= ODBCCheckConnection(g, dsn, cop)) != NULL) { thd->make_lex_string(&create_info->connect_string, dsn, strlen(dsn)); - ok = true; + ok= true; } // endif dsn #endif // PROMPT_OK @@ -5669,13 +5702,13 @@ static int connect_assisted_discovery(handlerton *, THD* thd, sprintf(g->Message, "Missing %s connection string", topt->type); } else { // Store ODBC additional parameters - sop = (POPARM)PlugSubAlloc(g, NULL, sizeof(ODBCPARM)); - sop->User = (char*)user; - sop->Pwd = (char*)pwd; - sop->Cto = cto; - sop->Qto = qto; - sop->UseCnc = cnc; - ok = true; + sop= (POPARM)PlugSubAlloc(g, NULL, sizeof(ODBCPARM)); + sop->User= (char*)user; + sop->Pwd= (char*)pwd; + sop->Cto= cto; + sop->Qto= qto; + sop->UseCnc= cnc; + ok= true; } // endif's supfnc |= (FNC_TABLE | FNC_DSN | FNC_DRIVER); @@ -5684,31 +5717,31 @@ static int connect_assisted_discovery(handlerton *, THD* thd, #if defined(JAVA_SUPPORT) case TAB_JDBC: if (fnc & FNC_DRIVER) { - ok = true; - } else if (!(url = strz(g, create_info->connect_string))) { + ok= true; + } else if (!(url= strz(g, create_info->connect_string))) { strcpy(g->Message, "Missing URL"); } else { // Store JDBC additional parameters int rc; - PJDBCDEF jdef = new(g) JDBCDEF(); + PJDBCDEF jdef= new(g) JDBCDEF(); jdef->SetName(create_info->alias); - sjp = (PJPARM)PlugSubAlloc(g, NULL, sizeof(JDBCPARM)); - sjp->Driver = driver; - // sjp->Properties = prop; - sjp->Fsize = 0; - sjp->Scrollable = false; - - if ((rc = jdef->ParseURL(g, url, false)) == RC_OK) { - sjp->Url = url; - sjp->User = (char*)user; - sjp->Pwd = (char*)pwd; - ok = true; + sjp= (PJPARM)PlugSubAlloc(g, NULL, sizeof(JDBCPARM)); + sjp->Driver= driver; + // sjp->Properties= prop; + sjp->Fsize= 0; + sjp->Scrollable= false; + + if ((rc= jdef->ParseURL(g, url, false)) == RC_OK) { + sjp->Url= url; + sjp->User= (char*)user; + sjp->Pwd= (char*)pwd; + ok= true; } else if (rc == RC_NF) { if (jdef->GetTabname()) - tab = (char*)jdef->GetTabname(); + tab= (char*)jdef->GetTabname(); - ok = jdef->SetParms(sjp); + ok= jdef->SetParms(sjp); } // endif rc } // endif's @@ -5717,7 +5750,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, break; #endif // JAVA_SUPPORT case TAB_DBF: - dbf = true; + dbf= true; // fall through case TAB_CSV: if (!fn && fnc != FNC_NO) @@ -5725,55 +5758,55 @@ static int connect_assisted_discovery(handlerton *, THD* thd, else if (sep && strlen(sep) > 1) sprintf(g->Message, "Invalid separator %s", sep); else - ok = true; + ok= true; break; case TAB_MYSQL: - ok = true; + ok= true; if (create_info->connect_string.str && create_info->connect_string.length) { - PMYDEF mydef = new(g) MYSQLDEF(); + PMYDEF mydef= new(g) MYSQLDEF(); - dsn = strz(g, create_info->connect_string); + dsn= strz(g, create_info->connect_string); mydef->SetName(create_info->alias); if (!mydef->ParseURL(g, dsn, false)) { if (mydef->GetHostname()) - host = mydef->GetHostname(); + host= mydef->GetHostname(); if (mydef->GetUsername()) - user = mydef->GetUsername(); + user= mydef->GetUsername(); if (mydef->GetPassword()) - pwd = mydef->GetPassword(); + pwd= mydef->GetPassword(); if (mydef->GetTabschema()) - db = mydef->GetTabschema(); + db= mydef->GetTabschema(); if (mydef->GetTabname()) - tab = (char*)mydef->GetTabname(); + tab= (char*)mydef->GetTabname(); if (mydef->GetPortnumber()) - port = mydef->GetPortnumber(); + port= mydef->GetPortnumber(); } else - ok = false; + ok= false; } else if (!user) - user = "root"; + user= "root"; if (ok && CheckSelf(g, table_s, host, db, tab, src, port)) - ok = false; + ok= false; break; #if defined(__WIN__) case TAB_WMI: - ok = true; + ok= true; break; #endif // __WIN__ case TAB_PIVOT: - supfnc = FNC_NO; + supfnc= FNC_NO; case TAB_PRX: case TAB_TBL: case TAB_XCL: @@ -5782,12 +5815,12 @@ static int connect_assisted_discovery(handlerton *, THD* thd, (!db || !stricmp(db, table_s->db.str))) sprintf(g->Message, "A %s table cannot refer to itself", topt->type); else - ok = true; + ok= true; break; case TAB_OEM: if (topt->module && topt->subtype) - ok = true; + ok= true; else strcpy(g->Message, "Missing OEM module or subtype"); @@ -5796,24 +5829,33 @@ static int connect_assisted_discovery(handlerton *, THD* thd, case TAB_XML: #endif // LIBXML2_SUPPORT || DOMDOC_SUPPORT case TAB_JSON: - dsn = strz(g, create_info->connect_string); + dsn= strz(g, create_info->connect_string); if (!fn && !zfn && !mul && !dsn) sprintf(g->Message, "Missing %s file name", topt->type); else - ok = true; + ok= true; break; #if defined(JAVA_SUPPORT) case TAB_MONGO: if (!topt->tabname) - topt->tabname = tab; + topt->tabname= tab; - ok = true; + ok= true; break; #endif // JAVA_SUPPORT +#if defined(REST_SUPPORT) + case TAB_REST: + if (!topt->http) + sprintf(g->Message, "Missing %s HTTP address", topt->type); + else + ok = true; + + break; +#endif // REST_SUPPORT case TAB_VIR: - ok = true; + ok= true; break; default: sprintf(g->Message, "Cannot get column info for table type %s", topt->type); @@ -5824,12 +5866,12 @@ static int connect_assisted_discovery(handlerton *, THD* thd, if (ok && !(supfnc & fnc)) { sprintf(g->Message, "Unsupported catalog function %s for table type %s", fncn, topt->type); - ok = false; + ok= false; } // endif supfnc if (src && fnc != FNC_NO) { strcpy(g->Message, "Cannot make catalog table from srcdef"); - ok = false; + ok= false; } // endif src if (ok) { @@ -5837,23 +5879,23 @@ static int connect_assisted_discovery(handlerton *, THD* thd, char *dft, *xtra, *key, *fmt; int i, len, prec, dec, typ, flg; - if (!(dpath = SetPath(g, table_s->db.str))) { - rc = HA_ERR_INTERNAL_ERROR; + if (!(dpath= SetPath(g, table_s->db.str))) { + rc= HA_ERR_INTERNAL_ERROR; goto err; } // endif dpath if (src && ttp != TAB_PIVOT && ttp != TAB_ODBC && ttp != TAB_JDBC) { - qrp = SrcColumns(g, host, db, user, pwd, src, port); + qrp= SrcColumns(g, host, db, user, pwd, src, port); if (qrp && ttp == TAB_OCCUR) if (OcrSrcCols(g, qrp, col, ocl, rnk)) { - rc = HA_ERR_INTERNAL_ERROR; + rc= HA_ERR_INTERNAL_ERROR; goto err; } // endif OcrSrcCols } else switch (ttp) { case TAB_DBF: - qrp = DBFColumns(g, dpath, fn, fnc == FNC_COL); + qrp= DBFColumns(g, dpath, fn, fnc == FNC_COL); break; #if defined(ODBC_SUPPORT) case TAB_ODBC: @@ -5861,21 +5903,21 @@ static int connect_assisted_discovery(handlerton *, THD* thd, case FNC_NO: case FNC_COL: if (src) { - qrp = ODBCSrcCols(g, dsn, (char*)src, sop); - src = NULL; // for next tests + qrp= ODBCSrcCols(g, dsn, (char*)src, sop); + src= NULL; // for next tests } else - qrp = ODBCColumns(g, dsn, shm, tab, NULL, + qrp= ODBCColumns(g, dsn, shm, tab, NULL, mxr, fnc == FNC_COL, sop); break; case FNC_TABLE: - qrp = ODBCTables(g, dsn, shm, tab, NULL, mxr, true, sop); + qrp= ODBCTables(g, dsn, shm, tab, NULL, mxr, true, sop); break; case FNC_DSN: - qrp = ODBCDataSources(g, mxr, true); + qrp= ODBCDataSources(g, mxr, true); break; case FNC_DRIVER: - qrp = ODBCDrivers(g, mxr, true); + qrp= ODBCDrivers(g, mxr, true); break; default: sprintf(g->Message, "invalid catfunc %s", fncn); @@ -5890,23 +5932,23 @@ static int connect_assisted_discovery(handlerton *, THD* thd, case FNC_NO: case FNC_COL: if (src) { - qrp = JDBCSrcCols(g, (char*)src, sjp); - src = NULL; // for next tests + qrp= JDBCSrcCols(g, (char*)src, sjp); + src= NULL; // for next tests } else - qrp = JDBCColumns(g, shm, tab, NULL, mxr, fnc == FNC_COL, sjp); + qrp= JDBCColumns(g, shm, tab, NULL, mxr, fnc == FNC_COL, sjp); break; case FNC_TABLE: -// qrp = JDBCTables(g, shm, tab, tabtyp, mxr, true, sjp); - qrp = JDBCTables(g, shm, tab, NULL, mxr, true, sjp); +// qrp= JDBCTables(g, shm, tab, tabtyp, mxr, true, sjp); + qrp= JDBCTables(g, shm, tab, NULL, mxr, true, sjp); break; #if 0 case FNC_DSN: - qrp = JDBCDataSources(g, mxr, true); + qrp= JDBCDataSources(g, mxr, true); break; #endif // 0 case FNC_DRIVER: - qrp = JDBCDrivers(g, mxr, true); + qrp= JDBCDrivers(g, mxr, true); break; default: sprintf(g->Message, "invalid catfunc %s", fncn); @@ -5916,56 +5958,61 @@ static int connect_assisted_discovery(handlerton *, THD* thd, break; #endif // JAVA_SUPPORT case TAB_MYSQL: - qrp = MyColumns(g, thd, host, db, user, pwd, tab, + qrp= MyColumns(g, thd, host, db, user, pwd, tab, NULL, port, fnc == FNC_COL); break; case TAB_CSV: - qrp = CSVColumns(g, dpath, topt, fnc == FNC_COL); + qrp= CSVColumns(g, dpath, topt, fnc == FNC_COL); break; #if defined(__WIN__) case TAB_WMI: - qrp = WMIColumns(g, nsp, cls, fnc == FNC_COL); + qrp= WMIColumns(g, nsp, cls, fnc == FNC_COL); break; #endif // __WIN__ case TAB_PRX: case TAB_TBL: case TAB_XCL: case TAB_OCCUR: - bif = fnc == FNC_COL; - qrp = TabColumns(g, thd, db, tab, bif); + bif= fnc == FNC_COL; + qrp= TabColumns(g, thd, db, tab, bif); if (!qrp && bif && fnc != FNC_COL) // tab is a view - qrp = MyColumns(g, thd, host, db, user, pwd, tab, NULL, port, false); + qrp= MyColumns(g, thd, host, db, user, pwd, tab, NULL, port, false); if (qrp && ttp == TAB_OCCUR && fnc != FNC_COL) if (OcrColumns(g, qrp, col, ocl, rnk)) { - rc = HA_ERR_INTERNAL_ERROR; + rc= HA_ERR_INTERNAL_ERROR; goto err; } // endif OcrColumns break; case TAB_PIVOT: - qrp = PivotColumns(g, tab, src, pic, fcl, skc, host, db, user, pwd, port); + qrp= PivotColumns(g, tab, src, pic, fcl, skc, host, db, user, pwd, port); break; case TAB_VIR: - qrp = VirColumns(g, fnc == FNC_COL); + qrp= VirColumns(g, fnc == FNC_COL); break; case TAB_JSON: - qrp = JSONColumns(g, db, dsn, topt, fnc == FNC_COL); + qrp= JSONColumns(g, db, dsn, topt, fnc == FNC_COL); break; #if defined(JAVA_SUPPORT) case TAB_MONGO: - url = strz(g, create_info->connect_string); - qrp = MGOColumns(g, db, url, topt, fnc == FNC_COL); + url= strz(g, create_info->connect_string); + qrp= MGOColumns(g, db, url, topt, fnc == FNC_COL); break; #endif // JAVA_SUPPORT #if defined(LIBXML2_SUPPORT) || defined(DOMDOC_SUPPORT) case TAB_XML: - qrp = XMLColumns(g, (char*)db, tab, topt, fnc == FNC_COL); + qrp= XMLColumns(g, (char*)db, tab, topt, fnc == FNC_COL); break; #endif // LIBXML2_SUPPORT || DOMDOC_SUPPORT +#if defined(REST_SUPPORT) + case TAB_REST: + qrp = RESTColumns(g, topt, tab, (char *)db, fnc == FNC_COL); + break; +#endif // REST_SUPPORT case TAB_OEM: - qrp = OEMColumns(g, topt, tab, (char*)db, fnc == FNC_COL); + qrp= OEMColumns(g, topt, tab, (char*)db, fnc == FNC_COL); break; default: strcpy(g->Message, "System error during assisted discovery"); @@ -5973,33 +6020,33 @@ static int connect_assisted_discovery(handlerton *, THD* thd, } // endswitch ttp if (!qrp) { - rc = HA_ERR_INTERNAL_ERROR; + rc= HA_ERR_INTERNAL_ERROR; goto err; } // endif !qrp if (fnc != FNC_NO || src || ttp == TAB_PIVOT) { // Catalog like table - for (crp = qrp->Colresp; !rc && crp; crp = crp->Next) { - cnm = (ttp == TAB_PIVOT) ? crp->Name : encode(g, crp->Name); - typ = crp->Type; - len = crp->Length; - dec = crp->Prec; - flg = crp->Flag; - v = (crp->Kdata->IsUnsigned()) ? 'U' : crp->Var; - tm = (crp->Kdata->IsNullable()) ? 0 : NOT_NULL_FLAG; + for (crp= qrp->Colresp; !rc && crp; crp= crp->Next) { + cnm= (ttp == TAB_PIVOT) ? crp->Name : encode(g, crp->Name); + typ= crp->Type; + len= crp->Length; + dec= crp->Prec; + flg= crp->Flag; + v= (crp->Kdata->IsUnsigned()) ? 'U' : crp->Var; + tm= (crp->Kdata->IsNullable()) ? 0 : NOT_NULL_FLAG; if (!len && typ == TYPE_STRING) - len = 256; // STRBLK's have 0 length + len= 256; // STRBLK's have 0 length // Now add the field if (add_field(&sql, cnm, typ, len, dec, NULL, tm, NULL, NULL, NULL, NULL, flg, dbf, v)) - rc = HA_ERR_OUT_OF_MEM; + rc= HA_ERR_OUT_OF_MEM; } // endfor crp } else { - char *schem = NULL; - char *tn = NULL; + char *schem= NULL; + char *tn= NULL; // Not a catalog table if (!qrp->Nblin) { @@ -6008,57 +6055,57 @@ static int connect_assisted_discovery(handlerton *, THD* thd, else strcpy(g->Message, "Fail to retrieve columns"); - rc = HA_ERR_INTERNAL_ERROR; + rc= HA_ERR_INTERNAL_ERROR; goto err; } // endif !nblin - for (i = 0; !rc && i < qrp->Nblin; i++) { - typ = len = prec = dec = 0; - tm = NOT_NULL_FLAG; - cnm = (char*)"noname"; - dft = xtra = key = fmt = tn = NULL; - v = ' '; - rem = NULL; + for (i= 0; !rc && i < qrp->Nblin; i++) { + typ= len= prec= dec= 0; + tm= NOT_NULL_FLAG; + cnm= (char*)"noname"; + dft= xtra= key= fmt= tn= NULL; + v= ' '; + rem= NULL; - for (crp = qrp->Colresp; crp; crp = crp->Next) + for (crp= qrp->Colresp; crp; crp= crp->Next) switch (crp->Fld) { case FLD_NAME: if (ttp == TAB_PRX || (ttp == TAB_CSV && topt->data_charset && (!stricmp(topt->data_charset, "UTF8") || !stricmp(topt->data_charset, "UTF-8")))) - cnm = crp->Kdata->GetCharValue(i); + cnm= crp->Kdata->GetCharValue(i); else - cnm = encode(g, crp->Kdata->GetCharValue(i)); + cnm= encode(g, crp->Kdata->GetCharValue(i)); break; case FLD_TYPE: - typ = crp->Kdata->GetIntValue(i); - v = (crp->Nulls) ? crp->Nulls[i] : 0; + typ= crp->Kdata->GetIntValue(i); + v= (crp->Nulls) ? crp->Nulls[i] : 0; break; case FLD_TYPENAME: - tn = crp->Kdata->GetCharValue(i); + tn= crp->Kdata->GetCharValue(i); break; case FLD_PREC: // PREC must be always before LENGTH - len = prec = crp->Kdata->GetIntValue(i); + len= prec= crp->Kdata->GetIntValue(i); break; case FLD_LENGTH: - len = crp->Kdata->GetIntValue(i); + len= crp->Kdata->GetIntValue(i); break; case FLD_SCALE: - dec = (!crp->Kdata->IsNull(i)) ? crp->Kdata->GetIntValue(i) : -1; + dec= (!crp->Kdata->IsNull(i)) ? crp->Kdata->GetIntValue(i) : -1; break; case FLD_NULL: if (crp->Kdata->GetIntValue(i)) - tm = 0; // Nullable + tm= 0; // Nullable break; case FLD_FORMAT: - fmt = (crp->Kdata) ? crp->Kdata->GetCharValue(i) : NULL; + fmt= (crp->Kdata) ? crp->Kdata->GetCharValue(i) : NULL; break; case FLD_REM: - rem = crp->Kdata->GetCharValue(i); + rem= crp->Kdata->GetCharValue(i); break; // case FLD_CHARSET: // No good because remote table is already translated @@ -6067,19 +6114,19 @@ static int connect_assisted_discovery(handlerton *, THD* thd, // break; case FLD_DEFAULT: - dft = crp->Kdata->GetCharValue(i); + dft= crp->Kdata->GetCharValue(i); break; case FLD_EXTRA: - xtra = crp->Kdata->GetCharValue(i); + xtra= crp->Kdata->GetCharValue(i); // Auto_increment is not supported yet if (!stricmp(xtra, "AUTO_INCREMENT")) - xtra = NULL; + xtra= NULL; break; case FLD_KEY: if (ttp == TAB_VIR) - key = crp->Kdata->GetCharValue(i); + key= crp->Kdata->GetCharValue(i); break; case FLD_SCHEM: @@ -6088,10 +6135,10 @@ static int connect_assisted_discovery(handlerton *, THD* thd, if (schem && stricmp(schem, crp->Kdata->GetCharValue(i))) { sprintf(g->Message, "Several %s tables found, specify DBNAME", tab); - rc = HA_ERR_INTERNAL_ERROR; + rc= HA_ERR_INTERNAL_ERROR; goto err; } else if (!schem) - schem = crp->Kdata->GetCharValue(i); + schem= crp->Kdata->GetCharValue(i); } // endif ttp #endif // ODBC_SUPPORT || JAVA_SUPPORT @@ -6102,10 +6149,10 @@ static int connect_assisted_discovery(handlerton *, THD* thd, #if defined(ODBC_SUPPORT) if (ttp == TAB_ODBC) { int plgtyp; - bool w = false; // Wide character type + bool w= false; // Wide character type // typ must be PLG type, not SQL type - if (!(plgtyp = TranslateSQLType(typ, dec, prec, v, w))) { + if (!(plgtyp= TranslateSQLType(typ, dec, prec, v, w))) { if (GetTypeConv() == TPC_SKIP) { // Skip this column sprintf(g->Message, "Column %s skipped (unsupported type %d)", @@ -6114,12 +6161,12 @@ static int connect_assisted_discovery(handlerton *, THD* thd, continue; } else { sprintf(g->Message, "Unsupported SQL type %d", typ); - rc = HA_ERR_INTERNAL_ERROR; + rc= HA_ERR_INTERNAL_ERROR; goto err; } // endif type_conv } else - typ = plgtyp; + typ= plgtyp; switch (typ) { case TYPE_STRING: @@ -6134,10 +6181,10 @@ static int connect_assisted_discovery(handlerton *, THD* thd, prec += (dec + 2); // To be safe break; case TYPE_DECIM: - prec = len; + prec= len; break; default: - dec = 0; + dec= 0; } // endswitch typ } else @@ -6147,7 +6194,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, int plgtyp; // typ must be PLG type, not SQL type - if (!(plgtyp = TranslateJDBCType(typ, tn, dec, prec, v))) { + if (!(plgtyp= TranslateJDBCType(typ, tn, dec, prec, v))) { if (GetTypeConv() == TPC_SKIP) { // Skip this column sprintf(g->Message, "Column %s skipped (unsupported type %d)", @@ -6156,12 +6203,12 @@ static int connect_assisted_discovery(handlerton *, THD* thd, continue; } else { sprintf(g->Message, "Unsupported SQL type %d", typ); - rc = HA_ERR_INTERNAL_ERROR; + rc= HA_ERR_INTERNAL_ERROR; goto err; } // endif type_conv } else - typ = plgtyp; + typ= plgtyp; switch (typ) { case TYPE_DOUBLE: @@ -6170,43 +6217,43 @@ static int connect_assisted_discovery(handlerton *, THD* thd, prec += (dec + 2); // To be safe break; default: - dec = 0; + dec= 0; } // endswitch typ } else #endif // ODBC_SUPPORT // Make the arguments as required by add_fields if (typ == TYPE_DOUBLE) - prec = len; + prec= len; if (typ == TYPE_DATE) - prec = 0; + prec= 0; // Now add the field if (add_field(&sql, cnm, typ, prec, dec, key, tm, rem, dft, xtra, fmt, 0, dbf, v)) - rc = HA_ERR_OUT_OF_MEM; + rc= HA_ERR_OUT_OF_MEM; } // endfor i } // endif fnc if (!rc) - rc = init_table_share(thd, table_s, create_info, &sql); + rc= init_table_share(thd, table_s, create_info, &sql); //g->jump_level--; //PopUser(xp); //return rc; } else { - rc = HA_ERR_UNSUPPORTED; + rc= HA_ERR_UNSUPPORTED; } // endif ok } catch (int n) { if (trace(1)) htrc("Exception %d: %s\n", n, g->Message); - rc = HA_ERR_INTERNAL_ERROR; + rc= HA_ERR_INTERNAL_ERROR; } catch (const char *msg) { strcpy(g->Message, msg); - rc = HA_ERR_INTERNAL_ERROR; + rc= HA_ERR_INTERNAL_ERROR; } // end catch err: @@ -6274,9 +6321,9 @@ int ha_connect::create(const char *name, TABLE *table_arg, TABTYPE type; TABLE *st= table; // Probably unuseful THD *thd= ha_thd(); - LEX_STRING cnc = table_arg->s->connect_string; -#ifdef WITH_PARTITION_STORAGE_ENGINE - partition_info *part_info= table_arg->part_info; + LEX_STRING cnc= table_arg->s->connect_string; +#if defined(WITH_PARTITION_STORAGE_ENGINE) + partition_info *part_info= table_arg->part_info; #else // !WITH_PARTITION_STORAGE_ENGINE #define part_info 0 #endif // !WITH_PARTITION_STORAGE_ENGINE @@ -6401,7 +6448,7 @@ int ha_connect::create(const char *name, TABLE *table_arg, host= mydef->GetHostname(); if (mydef->GetTabschema()) - db = mydef->GetTabschema(); + db= mydef->GetTabschema(); if (mydef->GetTabname()) tab= mydef->GetTabname(); @@ -6484,7 +6531,7 @@ int ha_connect::create(const char *name, TABLE *table_arg, } // endif type JSON if (type == TAB_CSV) { - const char *sep = options->separator; + const char *sep= options->separator; if (sep && strlen(sep) > 1) { sprintf(g->Message, "Invalid separator %s", sep); @@ -6685,15 +6732,15 @@ int ha_connect::create(const char *name, TABLE *table_arg, if (options->zipped) { // Check whether the zip entry must be made from a file - PCSZ fn = GetListOption(g, "Load", options->oplist, NULL); + PCSZ fn= GetListOption(g, "Load", options->oplist, NULL); if (fn) { char zbuf[_MAX_PATH], buf[_MAX_PATH], dbpath[_MAX_PATH]; - PCSZ entry = GetListOption(g, "Entry", options->oplist, NULL); - PCSZ a = GetListOption(g, "Append", options->oplist, "NO"); - bool append = *a == '1' || *a == 'Y' || *a == 'y' || !stricmp(a, "ON"); - PCSZ m = GetListOption(g, "Mulentries", options->oplist, "NO"); - bool mul = *m == '1' || *m == 'Y' || *m == 'y' || !stricmp(m, "ON"); + PCSZ entry= GetListOption(g, "Entry", options->oplist, NULL); + PCSZ a= GetListOption(g, "Append", options->oplist, "NO"); + bool append= *a == '1' || *a == 'Y' || *a == 'y' || !stricmp(a, "ON"); + PCSZ m= GetListOption(g, "Mulentries", options->oplist, "NO"); + bool mul= *m == '1' || *m == 'Y' || *m == 'y' || !stricmp(m, "ON"); if (!entry && !mul) { my_message(ER_UNKNOWN_ERROR, "Missing entry name", MYF(0)); @@ -6761,7 +6808,7 @@ int ha_connect::create(const char *name, TABLE *table_arg, if (SetDataPath(g, table_arg->s->db.str)) { my_message(ER_UNKNOWN_ERROR, g->Message, MYF(0)); - rc = HA_ERR_INTERNAL_ERROR; + rc= HA_ERR_INTERNAL_ERROR; } else if (cat) { if (part_info) strncpy(partname, @@ -7103,7 +7150,7 @@ ha_connect::check_if_supported_inplace_alter(TABLE *altered_table, /* ALTER TABLE tbl_name CONVERT TO CHARACTER SET .. and - ALTER TABLE table_name DEFAULT CHARSET = .. most likely + ALTER TABLE table_name DEFAULT CHARSET= .. most likely change column charsets and so not supported in-place through old API. @@ -7335,7 +7382,7 @@ maria_declare_plugin(connect) 0x0106, /* version number (1.06) */ NULL, /* status variables */ connect_system_variables, /* system variables */ - "1.06.0009", /* string version */ + "1.06.0010", /* string version */ MariaDB_PLUGIN_MATURITY_STABLE /* maturity */ } maria_declare_plugin_end; diff --git a/storage/connect/ha_connect.h b/storage/connect/ha_connect.h index ea495557fcf..9a12ef94431 100644 --- a/storage/connect/ha_connect.h +++ b/storage/connect/ha_connect.h @@ -11,7 +11,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ /** @file ha_connect.h Author Olivier Bertrand @@ -32,6 +32,10 @@ /****************************************************************************/ #include "mycat.h" +#if defined(JAVA_SUPPORT) || defined(CMGO_SUPPORT) +bool MongoEnabled(void); +#endif // JAVA_SUPPORT || CMGO_SUPPORT + /****************************************************************************/ /* Structures used to pass info between CONNECT and ha_connect. */ /****************************************************************************/ diff --git a/storage/connect/inihandl.cpp b/storage/connect/inihandl.cpp index 8a911cb05c0..8e79aeac7ef 100644 --- a/storage/connect/inihandl.cpp +++ b/storage/connect/inihandl.cpp @@ -16,7 +16,7 @@ * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ #include "my_global.h" @@ -194,7 +194,7 @@ static void PROFILE_Save( FILE *file, PROFILESECTION *section ) } for (key = section->key; key; key = key->next) - if (key->name[0]) { + if (key->name && key->name[0]) { fprintf(file, "%s", SVP(key->name)); if (key->value) diff --git a/storage/connect/jmgoconn.cpp b/storage/connect/jmgoconn.cpp index bd1ddadd80d..c80800bd897 100644 --- a/storage/connect/jmgoconn.cpp +++ b/storage/connect/jmgoconn.cpp @@ -272,7 +272,7 @@ bool JMgoConn::MakeCursor(PGLOBAL g, PTDB tdbp, PCSZ options, if (MakeSelector(g, filp, s)) { strcpy(g->Message, "Failed making selector"); - return NULL; + return true; } else s->Append('}'); @@ -340,7 +340,7 @@ bool JMgoConn::MakeCursor(PGLOBAL g, PTDB tdbp, PCSZ options, if (MakeSelector(g, filp, s)) { strcpy(g->Message, "Failed making selector"); - return NULL; + return true; } // endif Selector tdbp->SetFilter(NULL); // Not needed anymore @@ -813,4 +813,3 @@ PSZ JMgoConn::GetColumnValue(PSZ path) return fld; } // end of GetColumnValue - diff --git a/storage/connect/jsonudf.cpp b/storage/connect/jsonudf.cpp index df9c7200416..dad86d51040 100644 --- a/storage/connect/jsonudf.cpp +++ b/storage/connect/jsonudf.cpp @@ -1,6 +1,6 @@ /****************** jsonudf C++ Program Source Code File (.CPP) ******************/ -/* PROGRAM NAME: jsonudf Version 1.7 */ -/* (C) Copyright to the author Olivier BERTRAND 2015-2018 */ +/* PROGRAM NAME: jsonudf Version 1.8 */ +/* (C) Copyright to the author Olivier BERTRAND 2015-2019 */ /* This program are the JSON User Defined Functions . */ /*********************************************************************************/ @@ -1686,7 +1686,7 @@ static PCSZ MakeKey(PGLOBAL g, UDF_ARGS *args, int i) } // endif *s if (n < 1) - return "Key"; + return (PCSZ) "Key"; if (!b) { if ((p = (PSZ)PlgDBSubAlloc(g, NULL, n + 1))) { @@ -1703,7 +1703,7 @@ static PCSZ MakeKey(PGLOBAL g, UDF_ARGS *args, int i) return s; } // endif count - return "Key"; + return (PCSZ) "Key"; } // end of MakeKey /*********************************************************************************/ diff --git a/storage/connect/libdoc.cpp b/storage/connect/libdoc.cpp index 9b30b315441..58b0267bd6d 100644 --- a/storage/connect/libdoc.cpp +++ b/storage/connect/libdoc.cpp @@ -1035,7 +1035,7 @@ PXNODE XML2NODE::AddChildNode(PGLOBAL g, PCSZ name, PXNODE np) // If name has the format m[n] only m is taken as node name if ((p = strchr(pn, '['))) - p = BufAlloc(g, pn, p - pn); + p = BufAlloc(g, pn, int(p - pn)); else p = pn; diff --git a/storage/connect/mycat.cc b/storage/connect/mycat.cc index 5aef6d9c660..5c67b92ec36 100644 --- a/storage/connect/mycat.cc +++ b/storage/connect/mycat.cc @@ -11,14 +11,14 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ /*************** Mycat CC Program Source Code File (.CC) ***************/ /* PROGRAM NAME: MYCAT */ /* ------------- */ -/* Version 1.6 */ +/* Version 1.7 */ /* */ -/* Author: Olivier Bertrand 2012 - 2018 */ +/* Author: Olivier Bertrand 2012 - 2019 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -93,6 +93,9 @@ #if defined(ZIP_SUPPORT) #include "tabzip.h" #endif // ZIP_SUPPORT +#if defined(REST_SUPPORT) +#include "tabrest.h" +#endif // REST_SUPPORT #include "mycat.h" /***********************************************************************/ @@ -160,7 +163,7 @@ TABTYPE GetTypeID(const char *type) #if defined(ZIP_SUPPORT) : (!stricmp(type, "ZIP")) ? TAB_ZIP #endif - : (!stricmp(type, "OEM")) ? TAB_OEM : TAB_NIY; + : (!stricmp(type, "OEM")) ? TAB_OEM : TAB_NIY; } // end of GetTypeID /***********************************************************************/ @@ -181,6 +184,7 @@ bool IsFileType(TABTYPE type) case TAB_INI: case TAB_VEC: case TAB_JSON: + case TAB_REST: // case TAB_ZIP: isfile= true; break; @@ -488,8 +492,8 @@ PRELDEF MYCAT::GetTableDesc(PGLOBAL g, PTABLE tablep, printf("GetTableDesc: name=%s am=%s\n", tablep->GetName(), SVP(type)); // If not specified get the type of this table - if (!type) - type= Hc->GetStringOption("Type","*"); + //if (!type) + // type= Hc->GetStringOption("Type","*"); return MakeTableDesc(g, tablep, type); } // end of GetTableDesc @@ -501,8 +505,8 @@ PRELDEF MYCAT::GetTableDesc(PGLOBAL g, PTABLE tablep, PRELDEF MYCAT::MakeTableDesc(PGLOBAL g, PTABLE tablep, LPCSTR am) { TABTYPE tc; - LPCSTR name = (PSZ)PlugDup(g, tablep->GetName()); - LPCSTR schema = (PSZ)PlugDup(g, tablep->GetSchema()); + LPCSTR name= (PSZ)PlugDup(g, tablep->GetName()); + LPCSTR schema= (PSZ)PlugDup(g, tablep->GetSchema()); PRELDEF tdp= NULL; if (trace(1)) @@ -512,7 +516,11 @@ PRELDEF MYCAT::MakeTableDesc(PGLOBAL g, PTABLE tablep, LPCSTR am) /*********************************************************************/ /* Get a unique enum identifier for types. */ /*********************************************************************/ - tc= GetTypeID(am); + if (!am) { + tc = Hc->GetRealType(); + am = Hc->GetStringOption("Type", "*"); + } else + tc = GetTypeID(am); switch (tc) { case TAB_FIX: @@ -549,8 +557,11 @@ PRELDEF MYCAT::MakeTableDesc(PGLOBAL g, PTABLE tablep, LPCSTR am) case TAB_VIR: tdp= new(g) VIRDEF; break; case TAB_JSON: tdp= new(g) JSONDEF; break; #if defined(ZIP_SUPPORT) - case TAB_ZIP: tdp = new(g) ZIPDEF; break; + case TAB_ZIP: tdp= new(g) ZIPDEF; break; #endif // ZIP_SUPPORT +#if defined(REST_SUPPORT) + case TAB_REST: tdp= new(g) RESTDEF; break; +#endif // REST_SUPPORT #if defined(JAVA_SUPPORT) || defined(CMGO_SUPPORT) case TAB_MONGO: if (MongoEnabled()) { diff --git a/storage/connect/mycat.h b/storage/connect/mycat.h index f0f889722dd..b306d4acbe0 100644 --- a/storage/connect/mycat.h +++ b/storage/connect/mycat.h @@ -11,7 +11,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ /**************** MYCAT H Declares Source Code File (.H) ***************/ /* Name: MYCAT.H Version 2.3 */ @@ -50,7 +50,9 @@ struct ha_table_option_struct { const char *filter; const char *oplist; const char *data_charset; - ulonglong lrecl; + const char *http; + const char *uri; + ulonglong lrecl; ulonglong elements; //ulonglong estimate; ulonglong multiple; diff --git a/storage/connect/plgdbsem.h b/storage/connect/plgdbsem.h index 5446e0d2a07..14b8549beb0 100644 --- a/storage/connect/plgdbsem.h +++ b/storage/connect/plgdbsem.h @@ -82,7 +82,8 @@ enum TABTYPE {TAB_UNDEF = 0, /* Table of undefined type */ TAB_JDBC = 26, /* Table accessed via JDBC */ TAB_ZIP = 27, /* ZIP file info table */ TAB_MONGO = 28, /* Table retrieved from MongoDB */ - TAB_NIY = 30}; /* Table not implemented yet */ + TAB_REST = 29, /* Table retrieved from Rest query */ + TAB_NIY = 30}; /* Table not implemented yet */ enum AMT {TYPE_AM_ERROR = 0, /* Type not defined */ TYPE_AM_ROWID = 1, /* ROWID type (special column) */ @@ -400,6 +401,7 @@ typedef class VCTDEF *PVCTDEF; typedef class PIVOTDEF *PPIVOTDEF; typedef class DOMDEF *PDOMDEF; typedef class DIRDEF *PDIRDEF; +typedef class RESTDEF *PRESTDEF; typedef class OEMDEF *POEMDEF; typedef class COLCRT *PCOLCRT; typedef class COLDEF *PCOLDEF; diff --git a/storage/connect/restget.cpp b/storage/connect/restget.cpp new file mode 100644 index 00000000000..955e5f97fbc --- /dev/null +++ b/storage/connect/restget.cpp @@ -0,0 +1,87 @@ +/************* Restget C++ Program Source Code File (.CPP) *************/ +/* Adapted from the sample program of the Casablanca tutorial. */ +/* Copyright Olivier Bertrand 2019. */ +/***********************************************************************/ +#include +#include +#if defined(MARIADB) +#include +#else +#include "mini_global.h" +#endif + +using namespace utility::conversions; // String conversions utilities +using namespace web; // Common features like URIs. +using namespace web::http; // Common HTTP functionality +using namespace web::http::client; // HTTP client features +using namespace concurrency::streams; // Asynchronous streams + +#include "global.h" + + +/***********************************************************************/ +/* Make a local copy of the requested file. */ +/***********************************************************************/ +int restGetFile(PGLOBAL g, PCSZ http, PCSZ uri, PCSZ fn) +{ + int rc= 0; + auto fileStream= std::make_shared(); + + if (!http || !fn) { + strcpy(g->Message, "Missing http or filename"); + return 2; + } // endif + + //std::string sfn(fn); + //auto wfn= to_string_t(sfn); + //rc= 0; + + // Open stream to output file. + pplx::task requestTask= + fstream::open_ostream(to_string_t(fn)) + .then([=](ostream outFile) { + *fileStream= outFile; + + // Create http_client to send the request. + http_client client(to_string_t(http)); + + if (uri) + { + // Build request URI and start the request. + uri_builder builder(to_string_t(uri)); + return client.request(methods::GET, builder.to_string()); + } + else + return client.request(methods::GET); + }) + + // Handle response headers arriving. + .then([=](http_response response) { +#if defined(DEVELOPMENT) + fprintf(stderr, "Received response status code:%u\n", + response.status_code()); +#endif // DEVELOPMENT + + // Write response body into the file. + return response.body().read_to_end(fileStream->streambuf()); + }) + + // Close the file stream. + .then([=](size_t) { return fileStream->close(); }); + + // Wait for all the outstanding I/O to complete and handle any exceptions + try + { + requestTask.wait(); + } + catch (const std::exception &e) + { +#if defined(DEVELOPMENT) + fprintf(stderr, "Error exception: %s\n", e.what()); +#endif // DEVELOPMENT + sprintf(g->Message, "Error exception: %s", e.what()); + rc= 1; + } // end try/catch + + return rc; +} // end of restGetFile \ No newline at end of file diff --git a/storage/connect/tabjdbc.cpp b/storage/connect/tabjdbc.cpp index 789daff6fcd..06b6b3a9730 100644 --- a/storage/connect/tabjdbc.cpp +++ b/storage/connect/tabjdbc.cpp @@ -641,7 +641,9 @@ bool TDBJDBC::OpenDB(PGLOBAL g) Cnp->InitValue(g); if ((n = Jcp->GetResultSize(Query->GetStr(), Cnp)) < 0) { - sprintf(g->Message, "Cannot get result size rc=%d", n); + char* msg = PlugDup(g, g->Message); + + sprintf(g->Message, "Get result size: %s (rc=%d)", msg, n); return true; } else if (n) { Jcp->m_Rows = n; diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp index afab52aa282..a8f76900cb4 100644 --- a/storage/connect/tabjson.cpp +++ b/storage/connect/tabjson.cpp @@ -572,7 +572,7 @@ bool JSONDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) Sep = *GetStringCatInfo(g, "Separator", "."); Accept = GetBoolCatInfo("Accept", false); - // Don't use url as uri when called from REST OEM module + // Don't use url as MONGO uri when called from REST if (stricmp(am, "REST") && (Uri = GetStringCatInfo(g, "Connect", NULL))) { #if defined(JAVA_SUPPORT) || defined(CMGO_SUPPORT) Collname = GetStringCatInfo(g, "Name", diff --git a/storage/connect/tabrest.cpp b/storage/connect/tabrest.cpp new file mode 100644 index 00000000000..e800411a36d --- /dev/null +++ b/storage/connect/tabrest.cpp @@ -0,0 +1,133 @@ +/*************** Rest C++ Program Source Code File (.CPP) **************/ +/* PROGRAM NAME: Rest Version 1.3 */ +/* (C) Copyright to the author Olivier BERTRAND 2018 - 2019 */ +/* This program is the REST OEM (Web API support) module definition. */ +/***********************************************************************/ + +/***********************************************************************/ +/* Definitions needed by the included files. */ +/***********************************************************************/ +#include // All MariaDB stuff + +/***********************************************************************/ +/* Include application header files: */ +/* global.h is header containing all global declarations. */ +/* plgdbsem.h is header containing the DB application declarations. */ +/* (x)table.h is header containing the TDBASE declarations. */ +/***********************************************************************/ +#include "global.h" +#include "plgdbsem.h" +#include "xtable.h" +#include "filamtxt.h" +#include "plgxml.h" +#include "tabdos.h" +#include "tabfmt.h" +#include "tabjson.h" +#include "tabrest.h" +#include "tabxml.h" + +/***********************************************************************/ +/* Get the file from the Web. */ +/***********************************************************************/ +int restGetFile(PGLOBAL g, PCSZ http, PCSZ uri, PCSZ fn); + +#if defined(__WIN__) +static PCSZ slash= "\\"; +#else // !__WIN__ +static PCSZ slash= "/"; +#define stricmp strcasecmp +#endif // !__WIN__ + +/***********************************************************************/ +/* Return the columns definition to MariaDB. */ +/***********************************************************************/ +PQRYRES RESTColumns(PGLOBAL g, PTOS tp, char *tab, char *db, bool info) +{ + PQRYRES qrp= NULL; + char filename[_MAX_PATH]; + PCSZ http, uri, fn, ftype; + + http= GetStringTableOption(g, tp, "Http", NULL); + uri= GetStringTableOption(g, tp, "Uri", NULL); + fn= GetStringTableOption(g, tp, "Filename", "rest.json"); + ftype = GetStringTableOption(g, tp, "Type", "JSON"); + + // We used the file name relative to recorded datapath + strcat(strcat(strcat(strcpy(filename, "."), slash), db), slash); + strncat(filename, fn, _MAX_PATH); + + // Retrieve the file from the web and copy it locally + if (http && restGetFile(g, http, uri, filename)) { + // sprintf(g->Message, "Failed to get file at %s", http); + } else if (!stricmp(ftype, "XML")) + qrp= XMLColumns(g, db, tab, tp, info); + else if (!stricmp(ftype, "JSON")) + qrp= JSONColumns(g, db, NULL, tp, info); + else if (!stricmp(ftype, "CSV")) + qrp= CSVColumns(g, NULL, tp, info); + else + sprintf(g->Message, "Usupported file type %s", ftype); + + return qrp; +} // end of RESTColumns + +/* -------------------------- Class RESTDEF -------------------------- */ + +/***********************************************************************/ +/* DefineAM: define specific AM block values. */ +/***********************************************************************/ +bool RESTDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) +{ + char filename[_MAX_PATH]; + TABTYPE type= GetTypeID(am); + + switch (type) { + case TAB_JSON: + case TAB_XML: + case TAB_CSV: + break; + default: + sprintf(g->Message, "Unsupported REST table type %s", am); + return true; + } // endswitch type + + Http= GetStringCatInfo(g, "Http", NULL); + Uri= GetStringCatInfo(g, "Uri", NULL); + Fn= GetStringCatInfo(g, "Filename", "rest.json"); + + // We used the file name relative to recorded datapath + PlugSetPath(filename, Fn, GetPath()); + + // Retrieve the file from the web and copy it locally + if (Http && restGetFile(g, Http, Uri, filename)) {} + else if (type == TAB_JSON) + Tdp= new (g) JSONDEF; + else if (type == TAB_XML) + Tdp= new (g) XMLDEF; + else if (type == TAB_CSV) + Tdp= new (g) CSVDEF; + else + sprintf(g->Message, "Unsupported REST table type %s", am); + + // Do make the table/view definition + if (Tdp && Tdp->Define(g, Cat, Name, Schema, "REST")) + Tdp= NULL; // Error occured + + // Return true in case of error + return (Tdp == NULL); +} // end of DefineAM + +/***********************************************************************/ +/* GetTable: makes a new Table Description Block. */ +/***********************************************************************/ +PTDB RESTDEF::GetTable(PGLOBAL g, MODE m) +{ + if (m != MODE_READ && m != MODE_READX) { + strcpy(g->Message, "REST tables are currently read only"); + return NULL; + } // endif m + + return Tdp->GetTable(g, m); // Leave file type do the job +} // end of GetTable + +/* ---------------------- End of Class RESTDEF ----------------------- */ diff --git a/storage/connect/tabrest.h b/storage/connect/tabrest.h new file mode 100644 index 00000000000..1725f256079 --- /dev/null +++ b/storage/connect/tabrest.h @@ -0,0 +1,29 @@ +/*************** TabRest H Declares Source Code File (.H) **************/ +/* Name: tabrest.h Version 1.0 */ +/* (C) Copyright to the author Olivier BERTRAND 2019 */ +/* This file contains the common tabrest classes declares. */ +/***********************************************************************/ +#pragma once + +/***********************************************************************/ +/* Restest table. */ +/***********************************************************************/ +class RESTDEF : public TABDEF { /* Table description */ +public: + // Constructor + RESTDEF(void) { Tdp = NULL; Http = Uri = Fn = NULL; } + + // Implementation + virtual const char *GetType(void) { return "REST"; } + + // Methods + virtual bool DefineAM(PGLOBAL g, LPCSTR am, int poff); + virtual PTDB GetTable(PGLOBAL g, MODE m); + +protected: + // Members + PRELDEF Tdp; + PCSZ Http; /* Web connection HTTP */ + PCSZ Uri; /* Web connection URI */ + PCSZ Fn; /* The intermediate file name */ +}; // end of class RESTDEF diff --git a/storage/connect/tabtbl.cpp b/storage/connect/tabtbl.cpp index e194568ccf8..1150824464f 100644 --- a/storage/connect/tabtbl.cpp +++ b/storage/connect/tabtbl.cpp @@ -232,7 +232,7 @@ bool TDBTBL::InitTableList(PGLOBAL g) { int n; uint sln; - char *scs; + const char *scs; PTABLE tp, tabp; PCOL colp; PTBLDEF tdp = (PTBLDEF)To_Def; @@ -281,7 +281,7 @@ bool TDBTBL::InitTableList(PGLOBAL g) } // endfor tp - hc->get_table()->s->connect_string.str = scs; + hc->get_table()->s->connect_string.str = (char*)scs; hc->get_table()->s->connect_string.length = sln; //NumTables = n; diff --git a/storage/connect/tabxml.cpp b/storage/connect/tabxml.cpp index d808bd5ecd4..19490d350e8 100644 --- a/storage/connect/tabxml.cpp +++ b/storage/connect/tabxml.cpp @@ -1880,7 +1880,7 @@ void XMULCOL::ReadColumn(PGLOBAL g) if (N > Tdbp->Limit) { N = Tdbp->Limit; - sprintf(g->Message, "Mutiple values limited to %d", Tdbp->Limit); + sprintf(g->Message, "Multiple values limited to %d", Tdbp->Limit); PushWarning(g, Tdbp); } // endif N diff --git a/storage/connect/user_connect.cc b/storage/connect/user_connect.cc index d366e0222df..c25443ef7ef 100644 --- a/storage/connect/user_connect.cc +++ b/storage/connect/user_connect.cc @@ -11,7 +11,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ /** @file user_connect.cc diff --git a/storage/connect/user_connect.h b/storage/connect/user_connect.h index 983d9adc478..2670c5bcebc 100644 --- a/storage/connect/user_connect.h +++ b/storage/connect/user_connect.h @@ -11,7 +11,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ /** @file user_connect.h -- cgit v1.2.1 From d302cb3534f4b4e8a5ae6e293b6ef2c2c9180862 Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Wed, 31 Jul 2019 11:17:59 +0200 Subject: Typo --- storage/connect/mysql-test/connect/r/jdbc_postgresql.result | 2 +- storage/connect/mysql-test/connect/r/xml2_mult.result | 8 ++++---- storage/connect/mysql-test/connect/r/xml_mult.result | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) (limited to 'storage') diff --git a/storage/connect/mysql-test/connect/r/jdbc_postgresql.result b/storage/connect/mysql-test/connect/r/jdbc_postgresql.result index bec1dc8725b..07cc3c465ea 100644 --- a/storage/connect/mysql-test/connect/r/jdbc_postgresql.result +++ b/storage/connect/mysql-test/connect/r/jdbc_postgresql.result @@ -1,4 +1,4 @@ -SET GLOBAL connect_class_path='C:/MariaDB-10.0/MariaDB/storage/connect/mysql-test/connect/std_data/JavaWrappers.jar;C:/Jconnectors/postgresql-42.2.1.jar'; +SET GLOBAL connect_class_path='C:/MariaDB-10.2/MariaDB/storage/connect/mysql-test/connect/std_data/JavaWrappers.jar;C:/Jconnectors/postgresql-42.2.1.jar'; CREATE TABLE t2 ( command varchar(128) not null, number int(5) not null flag=1, diff --git a/storage/connect/mysql-test/connect/r/xml2_mult.result b/storage/connect/mysql-test/connect/r/xml2_mult.result index a9592e986c0..87d1118edd5 100644 --- a/storage/connect/mysql-test/connect/r/xml2_mult.result +++ b/storage/connect/mysql-test/connect/r/xml2_mult.result @@ -51,14 +51,14 @@ WEB XQuery Kick Start en Per Bothner 2003 49.99 WEB XQuery Kick Start en Kurt Cagle 2003 49.99 WEB Learning XML en Erik T. Ray 2003 39.95 Warnings: -Warning 1105 Mutiple values limited to 3 +Warning 1105 Multiple values limited to 3 # One line lost because the where clause is applied only on the first 3 rows SELECT category, title, author, price FROM bookstore WHERE author LIKE 'J%'; category title author price CHILDREN Harry Potter J K. Rowling 29.99 WEB XQuery Kick Start James McGovern 49.99 Warnings: -Warning 1105 Mutiple values limited to 3 +Warning 1105 Multiple values limited to 3 # # Testing concatenated values # @@ -91,12 +91,12 @@ CHILDREN Harry Potter en J K. Rowling 2005 29.99 WEB XQuery Kick Start en James McGovern, Per Bothner, Kurt Cagle, James Linn 2003 49.99 WEB Learning XML en Erik T. Ray 2003 39.95 Warnings: -Warning 1105 Mutiple values limited to 4 +Warning 1105 Multiple values limited to 4 # The where clause is applied on the concatenated column result SELECT category, title, author, price FROM bookstore WHERE author LIKE 'J%'; category title author price CHILDREN Harry Potter J K. Rowling 29.99 WEB XQuery Kick Start James McGovern, Per Bothner, Kurt Cagle, James Linn 49.99 Warnings: -Warning 1105 Mutiple values limited to 4 +Warning 1105 Multiple values limited to 4 DROP TABLE bookstore; diff --git a/storage/connect/mysql-test/connect/r/xml_mult.result b/storage/connect/mysql-test/connect/r/xml_mult.result index d89debadfab..9922b40060c 100644 --- a/storage/connect/mysql-test/connect/r/xml_mult.result +++ b/storage/connect/mysql-test/connect/r/xml_mult.result @@ -49,14 +49,14 @@ WEB XQuery Kick Start en Per Bothner 2003 49.99 WEB XQuery Kick Start en Kurt Cagle 2003 49.99 WEB Learning XML en Erik T. Ray 2003 39.95 Warnings: -Warning 1105 Mutiple values limited to 3 +Warning 1105 Multiple values limited to 3 # One line lost because the where clause is applied only on the first 3 rows SELECT category, title, author, price FROM bookstore WHERE author LIKE 'J%'; category title author price CHILDREN Harry Potter J K. Rowling 29.99 WEB XQuery Kick Start James McGovern 49.99 Warnings: -Warning 1105 Mutiple values limited to 3 +Warning 1105 Multiple values limited to 3 # # Testing concatenated values # @@ -89,12 +89,12 @@ CHILDREN Harry Potter en J K. Rowling 2005 29.99 WEB XQuery Kick Start en James McGovern, Per Bothner, Kurt Cagle, James Linn 2003 49.99 WEB Learning XML en Erik T. Ray 2003 39.95 Warnings: -Warning 1105 Mutiple values limited to 4 +Warning 1105 Multiple values limited to 4 # The where clause is applied on the concatenated column result SELECT category, title, author, price FROM bookstore WHERE author LIKE 'J%'; category title author price CHILDREN Harry Potter J K. Rowling 29.99 WEB XQuery Kick Start James McGovern, Per Bothner, Kurt Cagle, James Linn 49.99 Warnings: -Warning 1105 Mutiple values limited to 4 +Warning 1105 Multiple values limited to 4 DROP TABLE bookstore; -- cgit v1.2.1 From dc91372de3ab4b9b849a0710b4b086619d87b6cd Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Fri, 16 Aug 2019 18:11:32 +0530 Subject: Problem: ======== During ibd file creation, InnoDB flushes the page0 without crypt information. During recovery, InnoDB encounters encrypted page read before initialising the crypt data of the tablespace. So it leads t corruption of page and doesn't allow innodb to start. Solution: ========= Write crypt_data information in page0 while creating .ibd file creation. During recovery, crypt_data will be initialised while processing MLOG_FILE_NAME redo log record. --- storage/innobase/fil/fil0crypt.cc | 27 +++++++++++++++++++++++++++ storage/innobase/fil/fil0fil.cc | 20 +++++++++++++------- storage/innobase/include/fil0crypt.h | 6 ++++++ 3 files changed, 46 insertions(+), 7 deletions(-) (limited to 'storage') diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 9b8373a4d55..90b9aba70aa 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -356,6 +356,33 @@ fil_space_destroy_crypt_data( } } +/** Fill crypt data information to the give page. +It should be called during ibd file creation. +@param[in] flags tablespace flags +@param[in,out] page first page of the tablespace */ +void +fil_space_crypt_t::fill_page0( + ulint flags, + byte* page) +{ + const uint len = sizeof(iv); + const ulint offset = FSP_HEADER_OFFSET + + fsp_header_get_encryption_offset(page_size_t(flags)); + page0_offset = offset; + + memcpy(page + offset, CRYPT_MAGIC, MAGIC_SZ); + mach_write_to_1(page + offset + MAGIC_SZ, type); + mach_write_to_1(page + offset + MAGIC_SZ + 1, len); + memcpy(page + offset + MAGIC_SZ + 2, &iv, len); + + mach_write_to_4(page + offset + MAGIC_SZ + 2 + len, + min_key_version); + mach_write_to_4(page + offset + MAGIC_SZ + 2 + len + 4, + key_id); + mach_write_to_1(page + offset + MAGIC_SZ + 2 + len + 8, + encryption); +} + /****************************************************************** Write crypt data to a page (0) @param[in] space tablespace diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index f06179bf094..e37cb963c28 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -3671,6 +3671,19 @@ fil_ibd_create( fsp_header_init_fields(page, space_id, flags); mach_write_to_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, space_id); + /* Create crypt data if the tablespace is either encrypted or user has + requested it to remain unencrypted. */ + if (mode == FIL_ENCRYPTION_ON || mode == FIL_ENCRYPTION_OFF || + srv_encrypt_tables) { + crypt_data = fil_space_create_crypt_data(mode, key_id); + } + + if (crypt_data) { + /* Write crypt data information in page0 while creating + ibd file. */ + crypt_data->fill_page0(flags, page); + } + const page_size_t page_size(flags); IORequest request(IORequest::WRITE); @@ -3732,13 +3745,6 @@ fil_ibd_create( } } - /* Create crypt data if the tablespace is either encrypted or user has - requested it to remain unencrypted. */ - if (mode == FIL_ENCRYPTION_ON || mode == FIL_ENCRYPTION_OFF || - srv_encrypt_tables) { - crypt_data = fil_space_create_crypt_data(mode, key_id); - } - space = fil_space_create(name, space_id, flags, FIL_TYPE_TABLESPACE, crypt_data, mode); if (!space) { diff --git a/storage/innobase/include/fil0crypt.h b/storage/innobase/include/fil0crypt.h index 9b9f0da6fdd..24108f95167 100644 --- a/storage/innobase/include/fil0crypt.h +++ b/storage/innobase/include/fil0crypt.h @@ -180,6 +180,12 @@ struct fil_space_crypt_t : st_encryption_scheme return (encryption == FIL_ENCRYPTION_OFF); } + /** Fill crypt data information to the give page. + It should be called during ibd file creation. + @param[in] flags tablespace flags + @param[in,out] page first page of the tablespace */ + void fill_page0(ulint flags, byte* page); + /** Write crypt data to a page (0) @param[in] space tablespace @param[in,out] page0 first page of the tablespace -- cgit v1.2.1 From 6626b10a7a49eb91c2fcde2e3baef788b0ebe291 Mon Sep 17 00:00:00 2001 From: Kentoku SHIBA Date: Sat, 17 Aug 2019 00:40:38 +0900 Subject: fix for a compiler warning (#1372) --- storage/spider/spd_db_mysql.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'storage') diff --git a/storage/spider/spd_db_mysql.cc b/storage/spider/spd_db_mysql.cc index 57a0c5e6573..1b62b4a9bd5 100644 --- a/storage/spider/spd_db_mysql.cc +++ b/storage/spider/spd_db_mysql.cc @@ -5282,7 +5282,7 @@ int spider_db_mbase_util::append_tables_top_down( int error_num; uint outer_join_backup; TABLE_LIST *cur_table_list, *prev_table_list = NULL, *cond_table_list = NULL; - bool first; + bool first = TRUE; DBUG_ENTER("spider_db_mbase_util::append_tables_top_down"); DBUG_PRINT("info",("spider this=%p", this)); if ( -- cgit v1.2.1 From 4d93c7f3b0bfc59da369702c2eca725f1e362305 Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Sat, 17 Aug 2019 16:58:58 +0200 Subject: In CONNECT version 1.6.10 NOSQL facility is enhanced by a new way to retrieve NOSQL data. In addition to files and Mongo collections, JSON as well as XML and CSV data can be retrieved from the net as answers from REST queries. Because it uses and external package (cpprestsdk) this is currently available only to MariaDB servers compiled from source. -- Add compile flags needed on Windows /MD or /MDd (debug) -- Also include some changes needed on Linux modified: storage/connect/CMakeLists.txt - Add the xtrc tracing function modified: storage/connect/global.h modified: storage/connect/plugutil.cpp - Modify tracing to use xtrc and some typo modified: storage/connect/array.cpp modified: storage/connect/block.h modified: storage/connect/restget.cpp - Fix compilation error when ZIP is not supported modified: storage/connect/ha_connect.cc modified: storage/connect/tabfmt.cpp - Add some tracing + typo modified: storage/connect/mycat.cc modified: storage/connect/tabjson.cpp - Add conditional code based on MARIADB This to be able to use the same code in CONNECT and EOM modules modified: storage/connect/osutil.h modified: storage/connect/tabrest.cpp - Replace PlugSetPath by some concat (crashed on Fedora) + typo modified: storage/connect/reldef.cpp - Try to fix test failures modified: zlib/CMakeLists.txt --- storage/connect/CMakeLists.txt | 17 +- storage/connect/array.cpp | 127 ++- storage/connect/block.h | 4 +- storage/connect/global.h | 1 + storage/connect/ha_connect.cc | 4 + storage/connect/mini-global.h | 33 + storage/connect/mycat.cc | 161 ++-- storage/connect/osutil.h | 4 + storage/connect/plugutil.cpp | 36 +- storage/connect/reldef.cpp | 299 ++++--- storage/connect/rest.def | 4 + storage/connect/rest.h | 33 + storage/connect/restget.cpp | 104 +-- storage/connect/tabfmt.cpp | 2 + storage/connect/tabjson.cpp | 1929 ++++++++++++++++++++-------------------- storage/connect/tabrest.cpp | 157 +++- 16 files changed, 1540 insertions(+), 1375 deletions(-) create mode 100644 storage/connect/mini-global.h create mode 100644 storage/connect/rest.def create mode 100644 storage/connect/rest.h (limited to 'storage') diff --git a/storage/connect/CMakeLists.txt b/storage/connect/CMakeLists.txt index 8fc7cf695e7..6fd3b5e3018 100644 --- a/storage/connect/CMakeLists.txt +++ b/storage/connect/CMakeLists.txt @@ -68,6 +68,10 @@ ELSE(NOT UNIX) tabwmi.cpp tabwmi.h tabmac.cpp tabmac.h macutil.cpp macutil.h) # Add exception handling to the CONNECT project) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc") + SET(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /MD") + SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MDd") + SET(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} /MD") + SET(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL} /MD") SET(IPHLPAPI_LIBRARY iphlpapi.lib) IF(MSVC AND (CMAKE_CXX_COMPILER_ID MATCHES Clang)) # Connect does not work with clang-cl @@ -315,6 +319,12 @@ IF(CONNECT_WITH_REST) FIND_PACKAGE(cpprestsdk) IF (cpprestsdk_FOUND) MESSAGE(STATUS "=====> cpprestsdk found") + IF(UNIX) +# INCLUDE_DIRECTORIES(${CPPRESTSDK_INCLUDE_DIR}) +# Edit next line to set the path to libcpprest.so + SET(REST_LIBRARY -L/usr/lib64 -lcpprest) + MESSAGE (STATUS ${REST_LIBRARY}) + ENDIF(UNIX) SET(CONNECT_SOURCES ${CONNECT_SOURCES} tabrest.cpp restget.cpp tabrest.h) add_definitions(-DREST_SUPPORT) ELSE(NOT cpprestsdk_FOUND) @@ -341,7 +351,7 @@ MYSQL_ADD_PLUGIN(connect ${CONNECT_SOURCES} COMPONENT connect-engine RECOMPILE_FOR_EMBEDDED LINK_LIBRARIES ${ZLIB_LIBRARY} ${XML_LIBRARY} ${ICONV_LIBRARY} - ${ODBC_LIBRARY} ${JDBC_LIBRARY} ${MONGOC_LIBRARY} ${IPHLPAPI_LIBRARY}) + ${ODBC_LIBRARY} ${JDBC_LIBRARY} ${MONGOC_LIBRARY} ${IPHLPAPI_LIBRARY} ${REST_LIBRARY}) IF(NOT TARGET connect) RETURN() @@ -349,8 +359,8 @@ ENDIF() IF(WIN32) IF (libmongoc-1.0_FOUND) - SET_TARGET_PROPERTIES(connect PROPERTIES LINK_FLAGS - "/DELAYLOAD:libbson-1.0.dll /DELAYLOAD:libmongoc-1.0.dll") + SET_TARGET_PROPERTIES(connect PROPERTIES LINK_FLAGS + "/DELAYLOAD:libbson-1.0.dll /DELAYLOAD:libmongoc-1.0.dll") ENDIF(libmongoc-1.0_FOUND) ENDIF(WIN32) @@ -377,3 +387,4 @@ IF(CONNECT_WITH_JDBC AND JAVA_FOUND AND JNI_FOUND) ${CMAKE_CURRENT_BINARY_DIR}/JdbcInterface.jar DESTINATION ${INSTALL_PLUGINDIR} COMPONENT connect-engine) ENDIF() + diff --git a/storage/connect/array.cpp b/storage/connect/array.cpp index cd1785b48ac..3478342a788 100644 --- a/storage/connect/array.cpp +++ b/storage/connect/array.cpp @@ -1,7 +1,7 @@ /************* Array C++ Functions Source Code File (.CPP) *************/ /* Name: ARRAY.CPP Version 2.3 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 2005-2017 */ +/* (C) Copyright to the author Olivier BERTRAND 2005-2019 */ /* */ /* This file contains the XOBJECT derived class ARRAY functions. */ /* ARRAY is used for elaborate type of processing, such as sorting */ @@ -67,7 +67,7 @@ PARRAY MakeValueArray(PGLOBAL g, PPARM pp); // avoid gcc warning /* MakeValueArray: Makes a value array from a value list. */ /***********************************************************************/ PARRAY MakeValueArray(PGLOBAL g, PPARM pp) - { +{ int n, valtyp = 0; size_t len = 0; PARRAY par; @@ -82,8 +82,7 @@ PARRAY MakeValueArray(PGLOBAL g, PPARM pp) if ((valtyp = pp->Type) != TYPE_STRING) len = 1; - if (trace(1)) - htrc("valtyp=%d len=%d\n", valtyp, len); + xtrc(1, "valtyp=%d len=%d\n", valtyp, len); /*********************************************************************/ /* Firstly check the list and count the number of values in it. */ @@ -127,13 +126,13 @@ PARRAY MakeValueArray(PGLOBAL g, PPARM pp) // Integer stored inside pp->Value par->AddValue(g, parmp->Intval); break; - } // endswitch valtyp + } // endswitch valtyp /*********************************************************************/ /* Send back resulting array. */ /*********************************************************************/ return par; - } // end of MakeValueArray +} // end of MakeValueArray /* -------------------------- Class ARRAY ---------------------------- */ @@ -151,6 +150,9 @@ ARRAY::ARRAY(PGLOBAL g, int type, int size, int length, int prec) Type = type; Xsize = -1; Len = 1; + X = 0; + Inf = 0; + Sup = 0; switch (type) { case TYPE_STRING: @@ -281,130 +283,109 @@ void ARRAY::Empty(void) /* Add a string element to an array. */ /***********************************************************************/ bool ARRAY::AddValue(PGLOBAL g, PSZ strp) - { +{ if (Type != TYPE_STRING) { sprintf(g->Message, MSG(ADD_BAD_TYPE), GetTypeName(Type), "CHAR"); return true; - } // endif Type - - if (trace(1)) - htrc(" adding string(%d): '%s'\n", Nval, strp); + } // endif Type -//Value->SetValue_psz(strp); -//Vblp->SetValue(valp, Nval++); + xtrc(1, " adding string(%d): '%s'\n", Nval, strp); Vblp->SetValue(strp, Nval++); return false; - } // end of AddValue +} // end of AddValue /***********************************************************************/ /* Add a char pointer element to an array. */ /***********************************************************************/ bool ARRAY::AddValue(PGLOBAL g, void *p) - { +{ if (Type != TYPE_PCHAR) { sprintf(g->Message, MSG(ADD_BAD_TYPE), GetTypeName(Type), "PCHAR"); return true; - } // endif Type - - if (trace(1)) - htrc(" adding pointer(%d): %p\n", Nval, p); + } // endif Type + xtrc(1, " adding pointer(%d): %p\n", Nval, p); Vblp->SetValue((PSZ)p, Nval++); return false; - } // end of AddValue +} // end of AddValue /***********************************************************************/ /* Add a short integer element to an array. */ /***********************************************************************/ bool ARRAY::AddValue(PGLOBAL g, short n) - { +{ if (Type != TYPE_SHORT) { sprintf(g->Message, MSG(ADD_BAD_TYPE), GetTypeName(Type), "SHORT"); return true; - } // endif Type - - if (trace(1)) - htrc(" adding SHORT(%d): %hd\n", Nval, n); + } // endif Type -//Value->SetValue(n); -//Vblp->SetValue(valp, Nval++); + xtrc(1, " adding SHORT(%d): %hd\n", Nval, n); Vblp->SetValue(n, Nval++); return false; - } // end of AddValue +} // end of AddValue /***********************************************************************/ /* Add an integer element to an array. */ /***********************************************************************/ bool ARRAY::AddValue(PGLOBAL g, int n) - { +{ if (Type != TYPE_INT) { sprintf(g->Message, MSG(ADD_BAD_TYPE), GetTypeName(Type), "INTEGER"); return true; - } // endif Type + } // endif Type - if (trace(1)) - htrc(" adding int(%d): %d\n", Nval, n); - -//Value->SetValue(n); -//Vblp->SetValue(valp, Nval++); + xtrc(1, " adding int(%d): %d\n", Nval, n); Vblp->SetValue(n, Nval++); return false; - } // end of AddValue +} // end of AddValue /***********************************************************************/ /* Add a double float element to an array. */ /***********************************************************************/ bool ARRAY::AddValue(PGLOBAL g, double d) - { +{ if (Type != TYPE_DOUBLE) { sprintf(g->Message, MSG(ADD_BAD_TYPE), GetTypeName(Type), "DOUBLE"); return true; - } // endif Type - - if (trace(1)) - htrc(" adding float(%d): %lf\n", Nval, d); + } // endif Type + xtrc(1, " adding float(%d): %lf\n", Nval, d); Value->SetValue(d); Vblp->SetValue(Value, Nval++); return false; - } // end of AddValue +} // end of AddValue /***********************************************************************/ /* Add the value of a XOBJECT block to an array. */ /***********************************************************************/ bool ARRAY::AddValue(PGLOBAL g, PXOB xp) - { - if (Type != xp->GetResultType()) { - sprintf(g->Message, MSG(ADD_BAD_TYPE), - GetTypeName(xp->GetResultType()), GetTypeName(Type)); - return true; - } // endif Type - - if (trace(1)) - htrc(" adding (%d) from xp=%p\n", Nval, xp); +{ + if (Type != xp->GetResultType()) { + sprintf(g->Message, MSG(ADD_BAD_TYPE), + GetTypeName(xp->GetResultType()), GetTypeName(Type)); + return true; + } // endif Type -//AddValue(xp->GetValue()); - Vblp->SetValue(xp->GetValue(), Nval++); - return false; - } // end of AddValue + xtrc(1, " adding (%d) from xp=%p\n", Nval, xp); + Vblp->SetValue(xp->GetValue(), Nval++); + return false; +} // end of AddValue /***********************************************************************/ /* Add a value to an array. */ /***********************************************************************/ bool ARRAY::AddValue(PGLOBAL g, PVAL vp) - { +{ if (Type != vp->GetType()) { sprintf(g->Message, MSG(ADD_BAD_TYPE), GetTypeName(vp->GetType()), GetTypeName(Type)); return true; - } // endif Type - - if (trace(1)) - htrc(" adding (%d) from vp=%p\n", Nval, vp); + } // endif Type + xtrc(1, " adding (%d) from vp=%p\n", Nval, vp); Vblp->SetValue(vp, Nval++); return false; - } // end of AddValue +} // end of AddValue /***********************************************************************/ /* Retrieve the nth value of the array. */ @@ -973,7 +954,7 @@ int ARRAY::BlockTest(PGLOBAL, int opc, int opm, /* MakeArrayList: Makes a value list from an SQL IN array (in work). */ /***********************************************************************/ PSZ ARRAY::MakeArrayList(PGLOBAL g) - { +{ char *p, *tp; int i; size_t z, len = 2; @@ -988,11 +969,9 @@ PSZ ARRAY::MakeArrayList(PGLOBAL g) Value->SetValue_pvblk(Vblp, i); Value->Prints(g, tp, z); len += strlen(tp); - } // enfor i - - if (trace(1)) - htrc("Arraylist: len=%d\n", len); + } // enfor i + xtrc(1, "Arraylist: len=%d\n", len); p = (char *)PlugSubAlloc(g, NULL, len); strcpy(p, "("); @@ -1001,19 +980,17 @@ PSZ ARRAY::MakeArrayList(PGLOBAL g) Value->Prints(g, tp, z); strcat(p, tp); strcat(p, (++i == Nval) ? ")" : ","); - } // enfor i - - if (trace(1)) - htrc("Arraylist: newlen=%d\n", strlen(p)); + } // enfor i + xtrc(1, "Arraylist: newlen=%d\n", strlen(p)); return p; - } // end of MakeArrayList +} // end of MakeArrayList /***********************************************************************/ /* Make file output of ARRAY contents. */ /***********************************************************************/ void ARRAY::Printf(PGLOBAL g, FILE *f, uint n) - { +{ char m[64]; int lim = MY_MIN(Nval,10); @@ -1035,19 +1012,19 @@ void ARRAY::Printf(PGLOBAL g, FILE *f, uint n) } else fprintf(f, "%sVALLST: numval=%d\n", m, Nval); - } // end of Printf +} // end of Printf /***********************************************************************/ /* Make string output of ARRAY contents. */ /***********************************************************************/ void ARRAY::Prints(PGLOBAL, char *ps, uint z) - { +{ if (z < 16) return; sprintf(ps, "ARRAY: type=%d\n", Type); // More to be implemented later - } // end of Prints +} // end of Prints /* -------------------------- Class MULAR ---------------------------- */ diff --git a/storage/connect/block.h b/storage/connect/block.h index 737c74c1293..2ca9586ee3f 100644 --- a/storage/connect/block.h +++ b/storage/connect/block.h @@ -38,9 +38,7 @@ typedef class BLOCK *PBLOCK; class DllExport BLOCK { public: void * operator new(size_t size, PGLOBAL g, void *p = NULL) { - if (trace(256)) - htrc("New BLOCK: size=%d g=%p p=%p\n", size, g, p); - + xtrc(256, "New BLOCK: size=%d g=%p p=%p\n", size, g, p); return (PlugSubAlloc(g, p, size)); } // end of new diff --git a/storage/connect/global.h b/storage/connect/global.h index dc1e149745f..fd26c87b800 100644 --- a/storage/connect/global.h +++ b/storage/connect/global.h @@ -224,6 +224,7 @@ DllExport void *PlugSubAlloc(PGLOBAL, void *, size_t); DllExport char *PlugDup(PGLOBAL g, const char *str); DllExport void *MakePtr(void *, OFFSET); DllExport void htrc(char const *fmt, ...); +DllExport void xtrc(uint, char const* fmt, ...); DllExport uint GetTraceValue(void); #if defined(__cplusplus) diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index e7d1b5f0e86..be7c6548181 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -241,7 +241,9 @@ int TranslateJDBCType(int stp, char *tn, int prec, int& len, char& v); void PushWarning(PGLOBAL g, THD *thd, int level); bool CheckSelf(PGLOBAL g, TABLE_SHARE *s, PCSZ host, PCSZ db, PCSZ tab, PCSZ src, int port); +#if defined(ZIP_SUPPORT) bool ZipLoadFile(PGLOBAL, PCSZ, PCSZ, PCSZ, bool, bool); +#endif // ZIP_SUPPORT bool ExactInfo(void); #if defined(CMGO_SUPPORT) //void mongo_init(bool); @@ -6730,6 +6732,7 @@ int ha_connect::create(const char *name, TABLE *table_arg, if (trace(1)) htrc("xchk=%p createas=%d\n", g->Xchk, g->Createas); +#if defined(ZIP_SUPPORT) if (options->zipped) { // Check whether the zip entry must be made from a file PCSZ fn= GetListOption(g, "Load", options->oplist, NULL); @@ -6759,6 +6762,7 @@ int ha_connect::create(const char *name, TABLE *table_arg, } // endif fn } // endif zipped +#endif // ZIP_SUPPORT // To check whether indexes have to be made or remade if (!g->Xchk) { diff --git a/storage/connect/mini-global.h b/storage/connect/mini-global.h new file mode 100644 index 00000000000..f712795827c --- /dev/null +++ b/storage/connect/mini-global.h @@ -0,0 +1,33 @@ +/***********************************************************************/ +/* Definitions needed by the included files. */ +/***********************************************************************/ +#if !defined(MY_GLOBAL_H) +#define MY_GLOBAL_H +typedef unsigned int uint; +typedef unsigned int uint32; +typedef unsigned short ushort; +typedef unsigned long ulong; +typedef unsigned long DWORD; +typedef char *LPSTR; +typedef const char *LPCSTR; +typedef int BOOL; +#if defined(_WINDOWS) +typedef void *HANDLE; +#else +typedef int HANDLE; +#endif +typedef char *PSZ; +typedef const char *PCSZ; +typedef unsigned char BYTE; +typedef unsigned char uchar; +typedef long long longlong; +typedef unsigned long long ulonglong; +typedef char my_bool; +struct charset_info_st {}; +typedef const charset_info_st CHARSET_INFO; +#define FALSE 0 +#define TRUE 1 +#define Item char +#define MY_MAX(a,b) ((a>b)?(a):(b)) +#define MY_MIN(a,b) ((aGetName(), SVP(type)); +{ + PRELDEF tdp= NULL; - // If not specified get the type of this table + if (trace(1)) + htrc("GetTableDesc: name=%s am=%s\n", tablep->GetName(), SVP(type)); + + // If not specified get the type of this table //if (!type) // type= Hc->GetStringOption("Type","*"); - return MakeTableDesc(g, tablep, type); - } // end of GetTableDesc + tdp= MakeTableDesc(g, tablep, type); + + if (trace(1)) + htrc("GetTableDesc: tdp=%p\n", tdp); + + return tdp; +} // end of GetTableDesc /***********************************************************************/ /* MakeTableDesc: make a table/view description. */ @@ -505,22 +510,22 @@ PRELDEF MYCAT::GetTableDesc(PGLOBAL g, PTABLE tablep, PRELDEF MYCAT::MakeTableDesc(PGLOBAL g, PTABLE tablep, LPCSTR am) { TABTYPE tc; - LPCSTR name= (PSZ)PlugDup(g, tablep->GetName()); - LPCSTR schema= (PSZ)PlugDup(g, tablep->GetSchema()); + LPCSTR name= (PSZ)PlugDup(g, tablep->GetName()); + LPCSTR schema= (PSZ)PlugDup(g, tablep->GetSchema()); PRELDEF tdp= NULL; - if (trace(1)) - printf("MakeTableDesc: name=%s schema=%s am=%s\n", - name, SVP(schema), SVP(am)); + if (trace(1)) + htrc("MakeTableDesc: name=%s schema=%s am=%s\n", + name, SVP(schema), SVP(am)); /*********************************************************************/ /* Get a unique enum identifier for types. */ /*********************************************************************/ - if (!am) { - tc = Hc->GetRealType(); - am = Hc->GetStringOption("Type", "*"); - } else - tc = GetTypeID(am); + if (!am) { + tc= Hc->GetRealType(); + am= Hc->GetStringOption("Type","*"); + } else + tc= GetTypeID(am); switch (tc) { case TAB_FIX: @@ -535,49 +540,52 @@ PRELDEF MYCAT::MakeTableDesc(PGLOBAL g, PTABLE tablep, LPCSTR am) case TAB_XML: tdp= new(g) XMLDEF; break; #endif // XML_SUPPORT #if defined(VCT_SUPPORT) - case TAB_VEC: tdp = new(g) VCTDEF; break; + case TAB_VEC: tdp = new(g) VCTDEF; break; #endif // VCT_SUPPORT #if defined(ODBC_SUPPORT) case TAB_ODBC: tdp= new(g) ODBCDEF; break; #endif // ODBC_SUPPORT #if defined(JAVA_SUPPORT) - case TAB_JDBC: tdp= new(g) JDBCDEF; break; + case TAB_JDBC: tdp= new(g) JDBCDEF; break; #endif // JAVA_SUPPORT #if defined(__WIN__) case TAB_MAC: tdp= new(g) MACDEF; break; case TAB_WMI: tdp= new(g) WMIDEF; break; #endif // __WIN__ case TAB_OEM: tdp= new(g) OEMDEF; break; - case TAB_TBL: tdp= new(g) TBLDEF; break; - case TAB_XCL: tdp= new(g) XCLDEF; break; - case TAB_PRX: tdp= new(g) PRXDEF; break; - case TAB_OCCUR: tdp= new(g) OCCURDEF; break; - case TAB_MYSQL: tdp= new(g) MYSQLDEF; break; + case TAB_TBL: tdp= new(g) TBLDEF; break; + case TAB_XCL: tdp= new(g) XCLDEF; break; + case TAB_PRX: tdp= new(g) PRXDEF; break; + case TAB_OCCUR: tdp= new(g) OCCURDEF; break; + case TAB_MYSQL: tdp= new(g) MYSQLDEF; break; case TAB_PIVOT: tdp= new(g) PIVOTDEF; break; case TAB_VIR: tdp= new(g) VIRDEF; break; case TAB_JSON: tdp= new(g) JSONDEF; break; #if defined(ZIP_SUPPORT) - case TAB_ZIP: tdp= new(g) ZIPDEF; break; + case TAB_ZIP: tdp = new(g) ZIPDEF; break; #endif // ZIP_SUPPORT #if defined(REST_SUPPORT) - case TAB_REST: tdp= new(g) RESTDEF; break; + case TAB_REST: tdp= new (g) RESTDEF; break; #endif // REST_SUPPORT #if defined(JAVA_SUPPORT) || defined(CMGO_SUPPORT) - case TAB_MONGO: - if (MongoEnabled()) { - tdp = new(g) MGODEF; - break; - } // endif enabled - // fall through + case TAB_MONGO: + if (MongoEnabled()) { + tdp = new(g) MGODEF; + break; + } // endif enabled + // fall through #endif // JAVA_SUPPORT || CMGO_SUPPORT - default: - sprintf(g->Message, MSG(BAD_TABLE_TYPE), am, name); + default: + sprintf(g->Message, MSG(BAD_TABLE_TYPE), am, name); } // endswitch // Do make the table/view definition if (tdp && tdp->Define(g, this, name, schema, am)) tdp= NULL; + if (trace(1)) + htrc("Table %s made\n", am); + return tdp; } // end of MakeTableDesc @@ -590,26 +598,29 @@ PTDB MYCAT::GetTable(PGLOBAL g, PTABLE tablep, MODE mode, LPCSTR type) PTDB tdbp= NULL; // LPCSTR name= tablep->GetName(); - if (trace(1)) - printf("GetTableDB: name=%s\n", tablep->GetName()); + if (trace(1)) + htrc("GetTableDB: name=%s\n", tablep->GetName()); // Look for the description of the requested table tdp= GetTableDesc(g, tablep, type); if (tdp) { - if (trace(1)) - printf("tdb=%p type=%s\n", tdp, tdp->GetType()); + if (trace(1)) + htrc("tdb=%p type=%s\n", tdp, tdp->GetType()); + + if (tablep->GetSchema()) + tdp->Database = SetPath(g, tablep->GetSchema()); + + if (trace(2)) + htrc("Going to get table...\n"); - if (tablep->GetSchema()) - tdp->Database = SetPath(g, tablep->GetSchema()); - tdbp= tdp->GetTable(g, mode); - } // endif tdp + } // endif tdp if (tdbp) { - if (trace(1)) - printf("tdbp=%p name=%s amtype=%d\n", tdbp, tdbp->GetName(), - tdbp->GetAmType()); + if (trace(1)) + htrc("tdbp=%p name=%s amtype=%d\n", tdbp, tdbp->GetName(), + tdbp->GetAmType()); tablep->SetTo_Tdb(tdbp); tdbp->SetTable(tablep); tdbp->SetMode(mode); diff --git a/storage/connect/osutil.h b/storage/connect/osutil.h index 7e6b8823b9b..380e7bebb22 100644 --- a/storage/connect/osutil.h +++ b/storage/connect/osutil.h @@ -3,7 +3,11 @@ #define __OSUTIL_H__ #if defined(UNIX) || defined(UNIV_LINUX) +#if defined(MARIADB) #include "my_global.h" +#else +#include "mini-global.h" +#endif #include #include #include "os.h" diff --git a/storage/connect/plugutil.cpp b/storage/connect/plugutil.cpp index 6790e7eb45c..e74937b942a 100644 --- a/storage/connect/plugutil.cpp +++ b/storage/connect/plugutil.cpp @@ -2,11 +2,11 @@ /* */ /* PROGRAM NAME: PLUGUTIL */ /* ------------- */ -/* Version 3.0 */ +/* Version 3.1 */ /* */ /* COPYRIGHT: */ /* ---------- */ -/* (C) Copyright to the author Olivier BERTRAND 1993-2017 */ +/* (C) Copyright to the author Olivier BERTRAND 1993-2019 */ /* */ /* WHAT THIS PROGRAM DOES: */ /* ----------------------- */ @@ -111,21 +111,31 @@ ACTIVITY defActivity = { /* Describes activity and language */ #endif // UNIX /**************************************************************************/ -/* Tracing output function. */ +/* Conditional tracing output function. */ /**************************************************************************/ -void htrc(char const *fmt, ...) - { - va_list ap; - va_start (ap, fmt); +void xtrc(uint x, char const *fmt, ...) +{ + if (GetTraceValue() & x) { + va_list ap; + va_start(ap, fmt); + + vfprintf(stderr, fmt, ap); + va_end(ap); + } // endif x +} // end of xtrc -//if (trace == 1) -// vfprintf(debug, fmt, ap); -//else - vfprintf(stderr, fmt, ap); +/**************************************************************************/ +/* Tracing output function. */ +/**************************************************************************/ +void htrc(char const* fmt, ...) +{ + va_list ap; + va_start(ap, fmt); - va_end (ap); - } // end of htrc + vfprintf(stderr, fmt, ap); + va_end(ap); +} // end of htrc /***********************************************************************/ /* Plug initialization routine. */ diff --git a/storage/connect/reldef.cpp b/storage/connect/reldef.cpp index 30d8063d1a6..8ba8aac3621 100644 --- a/storage/connect/reldef.cpp +++ b/storage/connect/reldef.cpp @@ -81,51 +81,51 @@ RELDEF::RELDEF(void) /* This function return a pointer to the Table Option Struct. */ /***********************************************************************/ PTOS RELDEF::GetTopt(void) - { - return Hc->GetTableOptionStruct(); - } // end of GetTopt + { + return Hc->GetTableOptionStruct(); + } // end of GetTopt /***********************************************************************/ /* This function sets an integer table information. */ /***********************************************************************/ bool RELDEF::SetIntCatInfo(PCSZ what, int n) - { - return Hc->SetIntegerOption(what, n); - } // end of SetIntCatInfo + { + return Hc->SetIntegerOption(what, n); + } // end of SetIntCatInfo /***********************************************************************/ /* This function returns integer table information. */ /***********************************************************************/ int RELDEF::GetIntCatInfo(PCSZ what, int idef) - { - int n= Hc->GetIntegerOption(what); + { + int n= Hc->GetIntegerOption(what); - return (n == NO_IVAL) ? idef : n; - } // end of GetIntCatInfo + return (n == NO_IVAL) ? idef : n; + } // end of GetIntCatInfo /***********************************************************************/ /* This function returns Boolean table information. */ /***********************************************************************/ bool RELDEF::GetBoolCatInfo(PCSZ what, bool bdef) - { - bool b= Hc->GetBooleanOption(what, bdef); + { + bool b= Hc->GetBooleanOption(what, bdef); - return b; - } // end of GetBoolCatInfo + return b; + } // end of GetBoolCatInfo /***********************************************************************/ /* This function returns size catalog information. */ /***********************************************************************/ int RELDEF::GetSizeCatInfo(PCSZ what, PCSZ sdef) - { - char c; - PCSZ s; + { + char c; + PCSZ s; int i, n= 0; - if (!(s= Hc->GetStringOption(what))) - s= sdef; + if (!(s= Hc->GetStringOption(what))) + s= sdef; - if ((i= sscanf(s, " %d %c ", &n, &c)) == 2) + if ((i= sscanf(s, " %d %c ", &n, &c)) == 2) switch (toupper(c)) { case 'M': n *= 1024; @@ -141,41 +141,41 @@ int RELDEF::GetSizeCatInfo(PCSZ what, PCSZ sdef) /* This function sets char table information in buf. */ /***********************************************************************/ int RELDEF::GetCharCatInfo(PCSZ what, PCSZ sdef, char *buf, int size) - { - PCSZ s= Hc->GetStringOption(what); + { + PCSZ s= Hc->GetStringOption(what); - strncpy(buf, ((s) ? s : sdef), size); - return size; - } // end of GetCharCatInfo + strncpy(buf, ((s) ? s : sdef), size); + return size; + } // end of GetCharCatInfo /***********************************************************************/ /* To be used by any TDB's. */ /***********************************************************************/ bool RELDEF::Partitioned(void) - { - return Hc->IsPartitioned(); - } // end of Partitioned + { + return Hc->IsPartitioned(); + } // end of Partitioned /***********************************************************************/ /* This function returns string table information. */ /* Default parameter is "*" to get the handler default. */ /***********************************************************************/ char *RELDEF::GetStringCatInfo(PGLOBAL g, PCSZ what, PCSZ sdef) - { - char *sval = NULL; - PCSZ name, s= Hc->GetStringOption(what, sdef); - - if (s) { + { + char *sval = NULL; + PCSZ name, s= Hc->GetStringOption(what, sdef); + + if (s) { if (!Hc->IsPartitioned() || (stricmp(what, "filename") && stricmp(what, "tabname") && stricmp(what, "connect"))) - sval= PlugDup(g, s); + sval= PlugDup(g, s); else sval= (char*)s; } else if (!stricmp(what, "filename")) { // Return default file name - PCSZ ftype= Hc->GetStringOption("Type", "*"); + PCSZ ftype= Hc->GetStringOption("Type", "*"); int i, n; if (IsFileType(GetTypeID(ftype))) { @@ -183,7 +183,7 @@ char *RELDEF::GetStringCatInfo(PGLOBAL g, PCSZ what, PCSZ sdef) sval= (char*)PlugSubAlloc(g, NULL, strlen(name) + 12); strcat(strcpy(sval, name), "."); n= strlen(sval); - + // Fold ftype to lower case for (i= 0; i < 12; i++) if (!ftype[i]) { @@ -196,8 +196,8 @@ char *RELDEF::GetStringCatInfo(PGLOBAL g, PCSZ what, PCSZ sdef) } // endif s - return sval; - } // end of GetStringCatInfo + return sval; + } // end of GetStringCatInfo /* --------------------------- Class TABDEF -------------------------- */ @@ -223,14 +223,14 @@ TABDEF::TABDEF(void) /***********************************************************************/ /* Define: initialize the table definition block from XDB file. */ /***********************************************************************/ -bool TABDEF::Define(PGLOBAL g, PCATLG cat, - LPCSTR name, LPCSTR schema, LPCSTR am) +bool TABDEF::Define(PGLOBAL g, PCATLG cat, + LPCSTR name, LPCSTR schema, LPCSTR am) { int poff = 0; - Hc = ((MYCAT*)cat)->GetHandler(); - Name = (PSZ)name; - Schema = (PSZ)Hc->GetDBName(schema); + Hc = ((MYCAT*)cat)->GetHandler(); + Name = (PSZ)name; + Schema = (PSZ)Hc->GetDBName(schema); Cat = cat; Catfunc = GetFuncID(GetStringCatInfo(g, "Catfunc", NULL)); Elemt = GetIntCatInfo("Elements", 0); @@ -263,14 +263,14 @@ PCSZ TABDEF::GetPath(void) /* This function returns column table information. */ /***********************************************************************/ int TABDEF::GetColCatInfo(PGLOBAL g) - { - char *type= GetStringCatInfo(g, "Type", "*"); + { + char *type= GetStringCatInfo(g, "Type", "*"); char c, fty, eds; - int i, n, loff, poff, nof, nlg; - void *field= NULL; + int i, n, loff, poff, nof, nlg; + void *field= NULL; TABTYPE tc; PCOLDEF cdp, lcdp= NULL, tocols= NULL; - PCOLINFO pcf= (PCOLINFO)PlugSubAlloc(g, NULL, sizeof(COLINFO)); + PCOLINFO pcf= (PCOLINFO)PlugSubAlloc(g, NULL, sizeof(COLINFO)); memset(pcf, 0, sizeof(COLINFO)); @@ -278,33 +278,33 @@ int TABDEF::GetColCatInfo(PGLOBAL g) tc= (Catfunc == FNC_NO) ? GetTypeID(type) : TAB_PRX; // Take care of the column definitions - i= poff= nof= nlg= 0; + i= poff= nof= nlg= 0; #if defined(__WIN__) - // Offsets of HTML and DIR tables start from 0, DBF at 1 - loff= (tc == TAB_DBF) ? 1 : (tc == TAB_XML || tc == TAB_DIR) ? -1 : 0; + // Offsets of HTML and DIR tables start from 0, DBF at 1 + loff= (tc == TAB_DBF) ? 1 : (tc == TAB_XML || tc == TAB_DIR) ? -1 : 0; #else // !__WIN__ - // Offsets of HTML tables start from 0, DIR and DBF at 1 - loff = (tc == TAB_DBF || tc == TAB_DIR) ? 1 : (tc == TAB_XML) ? -1 : 0; + // Offsets of HTML tables start from 0, DIR and DBF at 1 + loff = (tc == TAB_DBF || tc == TAB_DIR) ? 1 : (tc == TAB_XML) ? -1 : 0; #endif // !__WIN__ while (true) { - // Default Offset depends on table type - switch (tc) { + // Default Offset depends on table type + switch (tc) { case TAB_DOS: case TAB_FIX: case TAB_BIN: case TAB_VEC: case TAB_DBF: - poff= loff + nof; // Default next offset - nlg= MY_MAX(nlg, poff); // Default lrecl + poff= loff + nof; // Default next offset + nlg= MY_MAX(nlg, poff); // Default lrecl break; case TAB_CSV: case TAB_FMT: - nlg+= nof; + nlg+= nof; case TAB_DIR: case TAB_XML: - poff= loff + (pcf->Flags & U_VIRTUAL ? 0 : 1); + poff= loff + (pcf->Flags & U_VIRTUAL ? 0 : 1); break; case TAB_INI: case TAB_MAC: @@ -316,39 +316,39 @@ int TABDEF::GetColCatInfo(PGLOBAL g) poff = 0; // Offset represents an independant flag break; default: // VCT PLG ODBC JDBC MYSQL WMI... - poff = 0; // NA + poff = 0; // NA break; - } // endswitch tc + } // endswitch tc -// do { - field= Hc->GetColumnOption(g, field, pcf); +// do { + field= Hc->GetColumnOption(g, field, pcf); // } while (field && (*pcf->Name =='*' /*|| pcf->Flags & U_VIRTUAL*/)); - if (tc == TAB_DBF && pcf->Type == TYPE_DATE && !pcf->Datefmt) { - // DBF date format defaults to 'YYYMMDD' - pcf->Datefmt= "YYYYMMDD"; - pcf->Length= 8; - } // endif tc + if (tc == TAB_DBF && pcf->Type == TYPE_DATE && !pcf->Datefmt) { + // DBF date format defaults to 'YYYMMDD' + pcf->Datefmt= "YYYYMMDD"; + pcf->Length= 8; + } // endif tc - if (!field) - break; + if (!field) + break; // Allocate the column description block cdp= new(g) COLDEF; if ((nof= cdp->Define(g, NULL, pcf, poff)) < 0) - return -1; // Error, probably unhandled type - else - loff= cdp->GetOffset(); - - switch (tc) { - case TAB_VEC: - cdp->SetOffset(0); // Not to have shift - case TAB_BIN: - // BIN/VEC are packed by default + return -1; // Error, probably unhandled type + else + loff= cdp->GetOffset(); + + switch (tc) { + case TAB_VEC: + cdp->SetOffset(0); // Not to have shift + case TAB_BIN: + // BIN/VEC are packed by default if (nof) { - // Field width is the internal representation width - // that can also depend on the column format + // Field width is the internal representation width + // that can also depend on the column format fty = cdp->Decode ? 'C' : 'X'; eds = 0; n = 0; @@ -371,38 +371,38 @@ int TABDEF::GetColCatInfo(PGLOBAL g) if (n) nof = n; else switch (fty) { - case 'X': + case 'X': if (eds && IsTypeChar(cdp->Buf_Type)) nof = sizeof(longlong); else nof= cdp->Clen; break; - case 'C': break; - case 'R': - case 'F': nof = sizeof(float); break; - case 'I': nof = sizeof(int); break; - case 'D': nof = sizeof(double); break; - case 'S': nof = sizeof(short); break; - case 'T': nof = sizeof(char); break; - case 'G': nof = sizeof(longlong); break; - default: /* Wrong format */ + case 'C': break; + case 'R': + case 'F': nof = sizeof(float); break; + case 'I': nof = sizeof(int); break; + case 'D': nof = sizeof(double); break; + case 'S': nof = sizeof(short); break; + case 'T': nof = sizeof(char); break; + case 'G': nof = sizeof(longlong); break; + default: /* Wrong format */ sprintf(g->Message, "Invalid format %c", fty); return -1; - } // endswitch fty + } // endswitch fty } // endif nof default: - break; - } // endswitch tc + break; + } // endswitch tc - if (lcdp) - lcdp->SetNext(cdp); - else - tocols= cdp; + if (lcdp) + lcdp->SetNext(cdp); + else + tocols= cdp; - lcdp= cdp; + lcdp= cdp; i++; } // endwhile @@ -410,31 +410,31 @@ int TABDEF::GetColCatInfo(PGLOBAL g) if (i != GetDegree()) SetDegree(i); - if (GetDefType() == TYPE_AM_DOS) { - int ending, recln= 0; + if (GetDefType() == TYPE_AM_DOS) { + int ending, recln= 0; - // Was commented because sometimes ending is 0 even when - // not specified (for instance if quoted is specified) -// if ((ending= Hc->GetIntegerOption("Ending")) < 0) { - if ((ending= Hc->GetIntegerOption("Ending")) <= 0) { + // Was commented because sometimes ending is 0 even when + // not specified (for instance if quoted is specified) +// if ((ending= Hc->GetIntegerOption("Ending")) < 0) { + if ((ending= Hc->GetIntegerOption("Ending")) <= 0) { ending= (tc == TAB_BIN || tc == TAB_VEC) ? 0 : CRLF; - Hc->SetIntegerOption("Ending", ending); - } // endif ending + Hc->SetIntegerOption("Ending", ending); + } // endif ending - // Calculate the default record size - switch (tc) { + // Calculate the default record size + switch (tc) { case TAB_FIX: case TAB_BIN: recln= nlg + ending; // + length of line ending break; case TAB_VEC: recln= nlg; - + // if ((k= (pak < 0) ? 8 : pak) > 1) // See above for detailed comment // Round up lrecl to multiple of 8 or pak // recln= ((recln + k - 1) / k) * k; - + break; case TAB_DOS: case TAB_DBF: @@ -443,26 +443,30 @@ int TABDEF::GetColCatInfo(PGLOBAL g) case TAB_CSV: case TAB_FMT: // The number of separators (assuming an extra one can exist) -// recln= poff * ((qotd) ? 3 : 1); to be investigated - recln= nlg + poff * 3; // To be safe +// recln= poff * ((qotd) ? 3 : 1); to be investigated + recln= nlg + poff * 3; // To be safe default: break; } // endswitch tc - // lrecl must be at least recln to avoid buffer overflow - if (trace(1)) - htrc("Lrecl: Calculated=%d defined=%d\n", - recln, Hc->GetIntegerOption("Lrecl")); + // lrecl must be at least recln to avoid buffer overflow + if (trace(1)) + htrc("Lrecl: Calculated=%d defined=%d\n", + recln, Hc->GetIntegerOption("Lrecl")); - recln = MY_MAX(recln, Hc->GetIntegerOption("Lrecl")); - Hc->SetIntegerOption("Lrecl", recln); - ((PDOSDEF)this)->SetLrecl(recln); - } // endif Lrecl + recln = MY_MAX(recln, Hc->GetIntegerOption("Lrecl")); + Hc->SetIntegerOption("Lrecl", recln); + ((PDOSDEF)this)->SetLrecl(recln); - // Attach the column definition to the tabdef - SetCols(tocols); - return poff; - } // end of GetColCatInfo + if (trace(1)) + htrc("Lrecl set to %d\n", recln); + + } // endif Lrecl + + // Attach the column definition to the tabdef + SetCols(tocols); + return poff; + } // end of GetColCatInfo /***********************************************************************/ /* SetIndexInfo: retrieve index description from the table structure. */ @@ -487,16 +491,17 @@ PTABDEF OEMDEF::GetXdef(PGLOBAL g) PCATLG cat = Cat; /*********************************************************************/ - /* Ensure that the .dll doesn't have a path. */ - /* This is done to ensure that only approved dll from the system */ + /* Ensure that the module name doesn't have a path. */ + /* This is done to ensure that only approved libs from the system */ /* directories are used (to make this even remotely secure). */ /*********************************************************************/ if (check_valid_path(Module, strlen(Module))) { strcpy(g->Message, "Module cannot contain a path"); return NULL; } else - PlugSetPath(soname, Module, GetPluginDir()); - +// PlugSetPath(soname, Module, GetPluginDir()); // Crashes on Fedora + strncat(strcpy(soname, GetPluginDir()), Module, _MAX_PATH); + #if defined(__WIN__) // Is the DLL already loaded? if (!Hdll && !(Hdll = GetModuleHandle(soname))) @@ -522,31 +527,31 @@ PTABDEF OEMDEF::GetXdef(PGLOBAL g) // Get the function returning an instance of the external DEF class if (!(getdef = (XGETDEF)GetProcAddress((HINSTANCE)Hdll, getname))) { - char buf[256]; - DWORD rc = GetLastError(); - - sprintf(g->Message, MSG(PROCADD_ERROR), rc, getname); - FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM | - FORMAT_MESSAGE_IGNORE_INSERTS, NULL, rc, 0, - (LPTSTR)buf, sizeof(buf), NULL); - strcat(strcat(g->Message, ": "), buf); - FreeLibrary((HMODULE)Hdll); + char buf[256]; + DWORD rc = GetLastError(); + + sprintf(g->Message, MSG(PROCADD_ERROR), rc, getname); + FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, NULL, rc, 0, + (LPTSTR)buf, sizeof(buf), NULL); + strcat(strcat(g->Message, ": "), buf); + FreeLibrary((HMODULE)Hdll); return NULL; } // endif getdef #else // !__WIN__ const char *error = NULL; - + #if 0 // Don't know what all this stuff does - Dl_info dl_info; + Dl_info dl_info; - // The OEM lib must retrieve exported CONNECT variables + // The OEM lib must retrieve exported CONNECT variables if (dladdr(&connect_hton, &dl_info)) { if (dlopen(dl_info.dli_fname, RTLD_NOLOAD | RTLD_NOW | RTLD_GLOBAL) == 0) { error = dlerror(); sprintf(g->Message, "dlopen failed: %s, OEM not supported", SVP(error)); return NULL; } // endif dlopen - + } else { error = dlerror(); sprintf(g->Message, "dladdr failed: %s, OEM not supported", SVP(error)); @@ -626,7 +631,7 @@ bool OEMDEF::DefineAM(PGLOBAL g, LPCSTR, int) char *desc = (char*)PlugSubAlloc(g, NULL, strlen(Module) + strlen(Subtype) + 3); sprintf(desc, "%s(%s)", Module, Subtype); - Desc = desc; + Desc = desc; return false; } // end of DefineAM @@ -701,17 +706,17 @@ PTDB OEMDEF::GetTable(PGLOBAL g, MODE mode) txfp = new(g) FIXFAM(defp); } else if (rfm == RECFM_VCT) { #if defined(VCT_SUPPORT) - assert(Pxdef->GetDefType() == TYPE_AM_VCT); + assert(Pxdef->GetDefType() == TYPE_AM_VCT); if (map) txfp = new(g) VCMFAM((PVCTDEF)defp); else txfp = new(g) VCTFAM((PVCTDEF)defp); #else // !VCT_SUPPORT - strcpy(g->Message, "VCT no more supported"); - return NULL; + strcpy(g->Message, "VCT no more supported"); + return NULL; #endif // !VCT_SUPPORT - } // endif's + } // endif's ((PTDBDOS)tdbp)->SetTxfp(txfp); } // endif Txfp diff --git a/storage/connect/rest.def b/storage/connect/rest.def new file mode 100644 index 00000000000..71c76740af9 --- /dev/null +++ b/storage/connect/rest.def @@ -0,0 +1,4 @@ +LIBRARY REST2 +EXPORTS + GetREST @1 + ColREST @2 diff --git a/storage/connect/rest.h b/storage/connect/rest.h new file mode 100644 index 00000000000..f1d77e0a279 --- /dev/null +++ b/storage/connect/rest.h @@ -0,0 +1,33 @@ +/***********************************************************************/ +/* Definitions needed by the included files. */ +/***********************************************************************/ +#if !defined(MY_GLOBAL_H) +#define MY_GLOBAL_H +typedef unsigned int uint; +typedef unsigned int uint32; +typedef unsigned short ushort; +typedef unsigned long ulong; +typedef unsigned long DWORD; +typedef char *LPSTR; +typedef const char *LPCSTR; +typedef int BOOL; +#if defined(_WINDOWS) +typedef void *HANDLE; +#else +typedef int HANDLE; +#endif +typedef char *PSZ; +typedef const char *PCSZ; +typedef unsigned char BYTE; +typedef unsigned char uchar; +typedef long long longlong; +typedef unsigned long long ulonglong; +typedef char my_bool; +struct charset_info_st {}; +typedef const charset_info_st CHARSET_INFO; +#define FALSE 0 +#define TRUE 1 +#define Item char +#define MY_MAX(a,b) ((a>b)?(a):(b)) +#define MY_MIN(a,b) ((a #else -#include "mini_global.h" +#include "mini-global.h" +#define _OS_H_INCLUDED // Prevent os.h to be called #endif using namespace utility::conversions; // String conversions utilities @@ -18,70 +19,71 @@ using namespace concurrency::streams; // Asynchronous streams #include "global.h" +static uint xt = 0; // Used by lamda expressions /***********************************************************************/ /* Make a local copy of the requested file. */ /***********************************************************************/ int restGetFile(PGLOBAL g, PCSZ http, PCSZ uri, PCSZ fn) { - int rc= 0; - auto fileStream= std::make_shared(); + int rc = 0; + auto fileStream = std::make_shared(); - if (!http || !fn) { - strcpy(g->Message, "Missing http or filename"); - return 2; - } // endif + if (!http || !fn) { + strcpy(g->Message, "Missing http or filename"); + return 2; + } // endif - //std::string sfn(fn); - //auto wfn= to_string_t(sfn); - //rc= 0; + xt = GetTraceValue(); + xtrc(515, "restGetFile: fn=%s\n", fn); // Open stream to output file. - pplx::task requestTask= - fstream::open_ostream(to_string_t(fn)) - .then([=](ostream outFile) { - *fileStream= outFile; - - // Create http_client to send the request. - http_client client(to_string_t(http)); - - if (uri) - { - // Build request URI and start the request. - uri_builder builder(to_string_t(uri)); - return client.request(methods::GET, builder.to_string()); - } - else - return client.request(methods::GET); - }) - - // Handle response headers arriving. - .then([=](http_response response) { -#if defined(DEVELOPMENT) - fprintf(stderr, "Received response status code:%u\n", - response.status_code()); -#endif // DEVELOPMENT - - // Write response body into the file. - return response.body().read_to_end(fileStream->streambuf()); - }) - - // Close the file stream. - .then([=](size_t) { return fileStream->close(); }); + pplx::task requestTask = fstream::open_ostream(to_string_t(fn)) + .then([=](ostream outFile) { + *fileStream= outFile; + + if (xt & 515) + htrc("Outfile isopen=%d\n", outFile.is_open()); + + // Create http_client to send the request. + http_client client(to_string_t(http)); + + if (uri) { + // Build request URI and start the request. + uri_builder builder(to_string_t(uri)); + return client.request(methods::GET, builder.to_string()); + } else + return client.request(methods::GET); + }) + + // Handle response headers arriving. + .then([=](http_response response) { + if (xt & 515) + htrc("Received response status code:%u\n", + response.status_code()); + + // Write response body into the file. + return response.body().read_to_end(fileStream->streambuf()); + }) + + // Close the file stream. + .then([=](size_t n) { + if (xt & 515) + htrc("Return size=%u\n", n); + + return fileStream->close(); + }); // Wait for all the outstanding I/O to complete and handle any exceptions - try - { + try { requestTask.wait(); - } - catch (const std::exception &e) - { -#if defined(DEVELOPMENT) - fprintf(stderr, "Error exception: %s\n", e.what()); -#endif // DEVELOPMENT - sprintf(g->Message, "Error exception: %s", e.what()); + xtrc(515, "In Wait\n"); + } catch (const std::exception &e) { + xtrc(515, "Error exception: %s\n", e.what()); + sprintf(g->Message, "Error exception: %s", e.what()); rc= 1; } // end try/catch + xtrc(515, "restget done: rc=%d\n", rc); return rc; -} // end of restGetFile \ No newline at end of file +} // end of restGetFile diff --git a/storage/connect/tabfmt.cpp b/storage/connect/tabfmt.cpp index 02720a3089a..746382178fb 100644 --- a/storage/connect/tabfmt.cpp +++ b/storage/connect/tabfmt.cpp @@ -189,9 +189,11 @@ PQRYRES CSVColumns(PGLOBAL g, PCSZ dp, PTOS topt, bool info) htrc("File %s Sep=%c Qot=%c Header=%d maxerr=%d\n", SVP(tdp->Fn), tdp->Sep, tdp->Qot, tdp->Header, tdp->Maxerr); +#if defined(ZIP_SUPPORT) if (tdp->Zipped) tcvp = new(g)TDBCSV(tdp, new(g)UNZFAM(tdp)); else +#endif // ZIP_SUPPORT tcvp = new(g) TDBCSV(tdp, new(g) DOSFAM(tdp)); tcvp->SetMode(MODE_READ); diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp index a8f76900cb4..0b282345c8a 100644 --- a/storage/connect/tabjson.cpp +++ b/storage/connect/tabjson.cpp @@ -71,15 +71,15 @@ char *GetJsonNull(void); /***********************************************************************/ PQRYRES JSONColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt, bool info) { - static int buftyp[] = {TYPE_STRING, TYPE_SHORT, TYPE_STRING, TYPE_INT, + static int buftyp[] = {TYPE_STRING, TYPE_SHORT, TYPE_STRING, TYPE_INT, TYPE_INT, TYPE_SHORT, TYPE_SHORT, TYPE_STRING}; - static XFLD fldtyp[] = {FLD_NAME, FLD_TYPE, FLD_TYPENAME, FLD_PREC, + static XFLD fldtyp[] = {FLD_NAME, FLD_TYPE, FLD_TYPENAME, FLD_PREC, FLD_LENGTH, FLD_SCALE, FLD_NULL, FLD_FORMAT}; static unsigned int length[] = {0, 6, 8, 10, 10, 6, 6, 0}; - int i, n = 0; + int i, n = 0; int ncol = sizeof(buftyp) / sizeof(int); - PJCL jcp; - JSONDISC *pjdc = NULL; + PJCL jcp; + JSONDISC *pjdc = NULL; PQRYRES qrp; PCOLRES crp; @@ -89,15 +89,15 @@ PQRYRES JSONColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt, bool info) goto skipit; } // endif info - if (GetIntegerTableOption(g, topt, "Multiple", 0)) { - strcpy(g->Message, "Cannot find column definition for multiple table"); - return NULL; - } // endif Multiple + if (GetIntegerTableOption(g, topt, "Multiple", 0)) { + strcpy(g->Message, "Cannot find column definition for multiple table"); + return NULL; + } // endif Multiple - pjdc = new(g) JSONDISC(g, length); + pjdc = new(g) JSONDISC(g, length); - if (!(n = pjdc->GetColumns(g, db, dsn, topt))) - return NULL; + if (!(n = pjdc->GetColumns(g, db, dsn, topt))) + return NULL; skipit: if (trace(1)) @@ -110,8 +110,8 @@ PQRYRES JSONColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt, bool info) buftyp, fldtyp, length, false, false); crp = qrp->Colresp->Next->Next->Next->Next->Next->Next; - crp->Name = PlugDup(g, "Nullable"); - crp->Next->Name = PlugDup(g, "Jpath"); + crp->Name = PlugDup(g, "Nullable"); + crp->Next->Name = PlugDup(g, "Jpath"); if (info || !qrp) return qrp; @@ -122,8 +122,8 @@ PQRYRES JSONColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt, bool info) /* Now get the results into blocks. */ /*********************************************************************/ for (i = 0, jcp = pjdc->fjcp; jcp; i++, jcp = jcp->Next) { - if (jcp->Type == TYPE_UNKNOWN) - jcp->Type = TYPE_STRING; // Void column + if (jcp->Type == TYPE_UNKNOWN) + jcp->Type = TYPE_STRING; // Void column crp = qrp->Colresp; // Column Name crp->Kdata->SetValue(jcp->Name, i); @@ -159,380 +159,380 @@ PQRYRES JSONColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt, bool info) /***********************************************************************/ JSONDISC::JSONDISC(PGLOBAL g, uint *lg) { - length = lg; - jcp = fjcp = pjcp = NULL; - tjnp = NULL; - jpp = NULL; - tjsp = NULL; - jsp = NULL; - row = NULL; - sep = NULL; - i = n = bf = ncol = lvl = 0; - all = false; -} // end of JSONDISC constructor + length = lg; + jcp = fjcp = pjcp = NULL; + tjnp = NULL; + jpp = NULL; + tjsp = NULL; + jsp = NULL; + row = NULL; + sep = NULL; + i = n = bf = ncol = lvl = 0; + all = false; +} // end of JSONDISC constructor int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt) { - char filename[_MAX_PATH]; - bool mgo = (GetTypeID(topt->type) == TAB_MONGO); - PCSZ level = GetStringTableOption(g, topt, "Level", NULL); - - if (level) { - lvl = atoi(level); - lvl = (lvl > 16) ? 16 : lvl; - } else - lvl = 0; - - sep = GetStringTableOption(g, topt, "Separator", "."); - - /*********************************************************************/ - /* Open the input file. */ - /*********************************************************************/ - tdp = new(g) JSONDEF; + char filename[_MAX_PATH]; + bool mgo = (GetTypeID(topt->type) == TAB_MONGO); + PCSZ level = GetStringTableOption(g, topt, "Level", NULL); + + if (level) { + lvl = atoi(level); + lvl = (lvl > 16) ? 16 : lvl; + } else + lvl = 0; + + sep = GetStringTableOption(g, topt, "Separator", "."); + + /*********************************************************************/ + /* Open the input file. */ + /*********************************************************************/ + tdp = new(g) JSONDEF; #if defined(ZIP_SUPPORT) - tdp->Entry = GetStringTableOption(g, topt, "Entry", NULL); - tdp->Zipped = GetBooleanTableOption(g, topt, "Zipped", false); + tdp->Entry = GetStringTableOption(g, topt, "Entry", NULL); + tdp->Zipped = GetBooleanTableOption(g, topt, "Zipped", false); #endif // ZIP_SUPPORT - tdp->Fn = GetStringTableOption(g, topt, "Filename", NULL); - - if (!(tdp->Database = SetPath(g, db))) - return 0; - - tdp->Objname = GetStringTableOption(g, topt, "Object", NULL); - tdp->Base = GetIntegerTableOption(g, topt, "Base", 0) ? 1 : 0; - tdp->Pretty = GetIntegerTableOption(g, topt, "Pretty", 2); - tdp->Xcol = GetStringTableOption(g, topt, "Expand", NULL); - tdp->Accept = GetBooleanTableOption(g, topt, "Accept", false); - tdp->Uri = (dsn && *dsn ? dsn : NULL); - - if (!tdp->Fn && !tdp->Uri) { - strcpy(g->Message, MSG(MISSING_FNAME)); - return 0; - } // endif Fn - - if (tdp->Fn) { - // We used the file name relative to recorded datapath - PlugSetPath(filename, tdp->Fn, tdp->GetPath()); - tdp->Fn = PlugDup(g, filename); - } // endif Fn - - if (trace(1)) - htrc("File %s objname=%s pretty=%d lvl=%d\n", - tdp->Fn, tdp->Objname, tdp->Pretty, lvl); - - if (tdp->Uri) { + tdp->Fn = GetStringTableOption(g, topt, "Filename", NULL); + + if (!(tdp->Database = SetPath(g, db))) + return 0; + + tdp->Objname = GetStringTableOption(g, topt, "Object", NULL); + tdp->Base = GetIntegerTableOption(g, topt, "Base", 0) ? 1 : 0; + tdp->Pretty = GetIntegerTableOption(g, topt, "Pretty", 2); + tdp->Xcol = GetStringTableOption(g, topt, "Expand", NULL); + tdp->Accept = GetBooleanTableOption(g, topt, "Accept", false); + tdp->Uri = (dsn && *dsn ? dsn : NULL); + + if (!tdp->Fn && !tdp->Uri) { + strcpy(g->Message, MSG(MISSING_FNAME)); + return 0; + } // endif Fn + + if (tdp->Fn) { + // We used the file name relative to recorded datapath + PlugSetPath(filename, tdp->Fn, tdp->GetPath()); + tdp->Fn = PlugDup(g, filename); + } // endif Fn + + if (trace(1)) + htrc("File %s objname=%s pretty=%d lvl=%d\n", + tdp->Fn, tdp->Objname, tdp->Pretty, lvl); + + if (tdp->Uri) { #if defined(JAVA_SUPPORT) || defined(CMGO_SUPPORT) - tdp->Collname = GetStringTableOption(g, topt, "Name", NULL); - tdp->Collname = GetStringTableOption(g, topt, "Tabname", tdp->Collname); - tdp->Schema = GetStringTableOption(g, topt, "Dbname", "test"); - tdp->Options = (PSZ)GetStringTableOption(g, topt, "Colist", "all"); - tdp->Pipe = GetBooleanTableOption(g, topt, "Pipeline", false); - tdp->Driver = (PSZ)GetStringTableOption(g, topt, "Driver", NULL); - tdp->Version = GetIntegerTableOption(g, topt, "Version", 3); - tdp->Wrapname = (PSZ)GetStringTableOption(g, topt, "Wrapper", - (tdp->Version == 2) ? "Mongo2Interface" : "Mongo3Interface"); - tdp->Pretty = 0; + tdp->Collname = GetStringTableOption(g, topt, "Name", NULL); + tdp->Collname = GetStringTableOption(g, topt, "Tabname", tdp->Collname); + tdp->Schema = GetStringTableOption(g, topt, "Dbname", "test"); + tdp->Options = (PSZ)GetStringTableOption(g, topt, "Colist", "all"); + tdp->Pipe = GetBooleanTableOption(g, topt, "Pipeline", false); + tdp->Driver = (PSZ)GetStringTableOption(g, topt, "Driver", NULL); + tdp->Version = GetIntegerTableOption(g, topt, "Version", 3); + tdp->Wrapname = (PSZ)GetStringTableOption(g, topt, "Wrapper", + (tdp->Version == 2) ? "Mongo2Interface" : "Mongo3Interface"); + tdp->Pretty = 0; #else // !MONGO_SUPPORT - sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "MONGO"); - return 0; + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "MONGO"); + return 0; #endif // !MONGO_SUPPORT - } // endif Uri + } // endif Uri - if (tdp->Pretty == 2) { - if (tdp->Zipped) { + if (tdp->Pretty == 2) { + if (tdp->Zipped) { #if defined(ZIP_SUPPORT) - tjsp = new(g) TDBJSON(tdp, new(g) UNZFAM(tdp)); + tjsp = new(g) TDBJSON(tdp, new(g) UNZFAM(tdp)); #else // !ZIP_SUPPORT - sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); - return 0; + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return 0; #endif // !ZIP_SUPPORT - } else - tjsp = new(g) TDBJSON(tdp, new(g) MAPFAM(tdp)); + } else + tjsp = new(g) TDBJSON(tdp, new(g) MAPFAM(tdp)); - if (tjsp->MakeDocument(g)) - return 0; + if (tjsp->MakeDocument(g)) + return 0; - jsp = (tjsp->GetDoc()) ? tjsp->GetDoc()->GetValue(0) : NULL; - } else { - if (!(tdp->Lrecl = GetIntegerTableOption(g, topt, "Lrecl", 0))) - if (!mgo) { - sprintf(g->Message, "LRECL must be specified for pretty=%d", tdp->Pretty); - return 0; - } else - tdp->Lrecl = 8192; // Should be enough + jsp = (tjsp->GetDoc()) ? tjsp->GetDoc()->GetValue(0) : NULL; + } else { + if (!(tdp->Lrecl = GetIntegerTableOption(g, topt, "Lrecl", 0))) + if (!mgo) { + sprintf(g->Message, "LRECL must be specified for pretty=%d", tdp->Pretty); + return 0; + } else + tdp->Lrecl = 8192; // Should be enough - tdp->Ending = GetIntegerTableOption(g, topt, "Ending", CRLF); + tdp->Ending = GetIntegerTableOption(g, topt, "Ending", CRLF); - if (tdp->Zipped) { + if (tdp->Zipped) { #if defined(ZIP_SUPPORT) - tjnp = new(g)TDBJSN(tdp, new(g) UNZFAM(tdp)); + tjnp = new(g)TDBJSN(tdp, new(g) UNZFAM(tdp)); #else // !ZIP_SUPPORT - sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); - return NULL; + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; #endif // !ZIP_SUPPORT - } else if (tdp->Uri) { - if (tdp->Driver && toupper(*tdp->Driver) == 'C') { + } else if (tdp->Uri) { + if (tdp->Driver && toupper(*tdp->Driver) == 'C') { #if defined(CMGO_SUPPORT) - tjnp = new(g) TDBJSN(tdp, new(g) CMGFAM(tdp)); + tjnp = new(g) TDBJSN(tdp, new(g) CMGFAM(tdp)); #else - sprintf(g->Message, "Mongo %s Driver not available", "C"); - return 0; + sprintf(g->Message, "Mongo %s Driver not available", "C"); + return 0; #endif - } else if (tdp->Driver && toupper(*tdp->Driver) == 'J') { + } else if (tdp->Driver && toupper(*tdp->Driver) == 'J') { #if defined(JAVA_SUPPORT) - tjnp = new(g) TDBJSN(tdp, new(g) JMGFAM(tdp)); + tjnp = new(g) TDBJSN(tdp, new(g) JMGFAM(tdp)); #else - sprintf(g->Message, "Mongo %s Driver not available", "Java"); - return 0; + sprintf(g->Message, "Mongo %s Driver not available", "Java"); + return 0; #endif - } else { // Driver not specified + } else { // Driver not specified #if defined(CMGO_SUPPORT) - tjnp = new(g) TDBJSN(tdp, new(g) CMGFAM(tdp)); + tjnp = new(g) TDBJSN(tdp, new(g) CMGFAM(tdp)); #elif defined(JAVA_SUPPORT) - tjnp = new(g) TDBJSN(tdp, new(g) JMGFAM(tdp)); + tjnp = new(g) TDBJSN(tdp, new(g) JMGFAM(tdp)); #else - sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "MONGO"); - return 0; + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "MONGO"); + return 0; #endif - } // endif Driver - - } else - tjnp = new(g) TDBJSN(tdp, new(g) DOSFAM(tdp)); - - tjnp->SetMode(MODE_READ); - - // Allocate the parse work memory - PGLOBAL G = (PGLOBAL)PlugSubAlloc(g, NULL, sizeof(GLOBAL)); - memset(G, 0, sizeof(GLOBAL)); - G->Sarea_Size = tdp->Lrecl * 10; - G->Sarea = PlugSubAlloc(g, NULL, G->Sarea_Size); - PlugSubSet(G->Sarea, G->Sarea_Size); - G->jump_level = 0; - tjnp->SetG(G); - - if (tjnp->OpenDB(g)) - return 0; - - switch (tjnp->ReadDB(g)) { - case RC_EF: - strcpy(g->Message, "Void json table"); - case RC_FX: - goto err; - default: - jsp = tjnp->GetRow(); - } // endswitch ReadDB - - } // endif pretty - - if (!(row = (jsp) ? jsp->GetObject() : NULL)) { - strcpy(g->Message, "Can only retrieve columns from object rows"); - goto err; - } // endif row - - all = GetBooleanTableOption(g, topt, "Fullarray", false); - jcol.Name = jcol.Fmt = NULL; - jcol.Next = NULL; - jcol.Found = true; - colname[0] = 0; - - if (!tdp->Uri) { - fmt[0] = '$'; - fmt[1] = '.'; - bf = 2; - } // endif Uri - - /*********************************************************************/ - /* Analyse the JSON tree and define columns. */ - /*********************************************************************/ - for (i = 1; ; i++) { - for (jpp = row->GetFirst(); jpp; jpp = jpp->GetNext()) { - strncpy(colname, jpp->GetKey(), 64); - fmt[bf] = 0; - - if (Find(g, jpp->GetVal(), colname, MY_MIN(lvl, 0))) - goto err; - - } // endfor jpp - - // Missing column can be null - for (jcp = fjcp; jcp; jcp = jcp->Next) { - jcp->Cbn |= !jcp->Found; - jcp->Found = false; - } // endfor jcp - - if (tdp->Pretty != 2) { - // Read next record - switch (tjnp->ReadDB(g)) { - case RC_EF: - jsp = NULL; - break; - case RC_FX: - goto err; - default: - jsp = tjnp->GetRow(); - } // endswitch ReadDB - - } else - jsp = tjsp->GetDoc()->GetValue(i); - - if (!(row = (jsp) ? jsp->GetObject() : NULL)) - break; - - } // endfor i - - if (tdp->Pretty != 2) - tjnp->CloseDB(g); - - return n; + } // endif Driver + + } else + tjnp = new(g) TDBJSN(tdp, new(g) DOSFAM(tdp)); + + tjnp->SetMode(MODE_READ); + + // Allocate the parse work memory + PGLOBAL G = (PGLOBAL)PlugSubAlloc(g, NULL, sizeof(GLOBAL)); + memset(G, 0, sizeof(GLOBAL)); + G->Sarea_Size = tdp->Lrecl * 10; + G->Sarea = PlugSubAlloc(g, NULL, G->Sarea_Size); + PlugSubSet(G->Sarea, G->Sarea_Size); + G->jump_level = 0; + tjnp->SetG(G); + + if (tjnp->OpenDB(g)) + return 0; + + switch (tjnp->ReadDB(g)) { + case RC_EF: + strcpy(g->Message, "Void json table"); + case RC_FX: + goto err; + default: + jsp = tjnp->GetRow(); + } // endswitch ReadDB + + } // endif pretty + + if (!(row = (jsp) ? jsp->GetObject() : NULL)) { + strcpy(g->Message, "Can only retrieve columns from object rows"); + goto err; + } // endif row + + all = GetBooleanTableOption(g, topt, "Fullarray", false); + jcol.Name = jcol.Fmt = NULL; + jcol.Next = NULL; + jcol.Found = true; + colname[0] = 0; + + if (!tdp->Uri) { + fmt[0] = '$'; + fmt[1] = '.'; + bf = 2; + } // endif Uri + + /*********************************************************************/ + /* Analyse the JSON tree and define columns. */ + /*********************************************************************/ + for (i = 1; ; i++) { + for (jpp = row->GetFirst(); jpp; jpp = jpp->GetNext()) { + strncpy(colname, jpp->GetKey(), 64); + fmt[bf] = 0; + + if (Find(g, jpp->GetVal(), colname, MY_MIN(lvl, 0))) + goto err; + + } // endfor jpp + + // Missing column can be null + for (jcp = fjcp; jcp; jcp = jcp->Next) { + jcp->Cbn |= !jcp->Found; + jcp->Found = false; + } // endfor jcp + + if (tdp->Pretty != 2) { + // Read next record + switch (tjnp->ReadDB(g)) { + case RC_EF: + jsp = NULL; + break; + case RC_FX: + goto err; + default: + jsp = tjnp->GetRow(); + } // endswitch ReadDB + + } else + jsp = tjsp->GetDoc()->GetValue(i); + + if (!(row = (jsp) ? jsp->GetObject() : NULL)) + break; + + } // endfor i + + if (tdp->Pretty != 2) + tjnp->CloseDB(g); + + return n; err: - if (tdp->Pretty != 2) - tjnp->CloseDB(g); + if (tdp->Pretty != 2) + tjnp->CloseDB(g); - return 0; -} // end of GetColumns + return 0; +} // end of GetColumns bool JSONDISC::Find(PGLOBAL g, PJVAL jvp, PCSZ key, int j) { - char *p, *pc = colname + strlen(colname); - int ars; - PJOB job; - PJAR jar; - - if ((valp = jvp ? jvp->GetValue() : NULL)) { - jcol.Type = valp->GetType(); - jcol.Len = valp->GetValLen(); - jcol.Scale = valp->GetValPrec(); - jcol.Cbn = valp->IsNull(); - } else if (!jvp || jvp->IsNull()) { - jcol.Type = TYPE_UNKNOWN; - jcol.Len = jcol.Scale = 0; - jcol.Cbn = true; - } else if (j < lvl) { - if (!fmt[bf]) - strcat(fmt, colname); - - p = fmt + strlen(fmt); - jsp = jvp->GetJson(); - - switch (jsp->GetType()) { - case TYPE_JOB: - job = (PJOB)jsp; - - for (PJPR jrp = job->GetFirst(); jrp; jrp = jrp->GetNext()) { - PCSZ k = jrp->GetKey(); - - if (*k != '$') { - strncat(strncat(fmt, sep, 128), k, 128); - strncat(strncat(colname, "_", 64), k, 64); - } // endif Key - - if (Find(g, jrp->GetVal(), k, j + 1)) - return true; - - *p = *pc = 0; - } // endfor jrp - - return false; - case TYPE_JAR: - jar = (PJAR)jsp; - - if (all || (tdp->Xcol && !stricmp(tdp->Xcol, key))) - ars = jar->GetSize(false); - else - ars = MY_MIN(jar->GetSize(false), 1); - - for (int k = 0; k < ars; k++) { - if (!tdp->Xcol || stricmp(tdp->Xcol, key)) { - sprintf(buf, "%d", k); - - if (tdp->Uri) - strncat(strncat(fmt, sep, 128), buf, 128); - else - strncat(strncat(strncat(fmt, "[", 128), buf, 128), "]", 128); - - if (all) - strncat(strncat(colname, "_", 64), buf, 64); - - } else - strncat(fmt, (tdp->Uri ? sep : "[*]"), 128); - - if (Find(g, jar->GetValue(k), "", j)) - return true; - - *p = *pc = 0; - } // endfor k - - return false; - default: - sprintf(g->Message, "Logical error after %s", fmt); - return true; - } // endswitch Type - - } else if (lvl >= 0) { - jcol.Type = TYPE_STRING; - jcol.Len = 256; - jcol.Scale = 0; - jcol.Cbn = true; - } else - return false; - - AddColumn(g); - return false; -} // end of Find + char *p, *pc = colname + strlen(colname); + int ars; + PJOB job; + PJAR jar; + + if ((valp = jvp ? jvp->GetValue() : NULL)) { + jcol.Type = valp->GetType(); + jcol.Len = valp->GetValLen(); + jcol.Scale = valp->GetValPrec(); + jcol.Cbn = valp->IsNull(); + } else if (!jvp || jvp->IsNull()) { + jcol.Type = TYPE_UNKNOWN; + jcol.Len = jcol.Scale = 0; + jcol.Cbn = true; + } else if (j < lvl) { + if (!fmt[bf]) + strcat(fmt, colname); + + p = fmt + strlen(fmt); + jsp = jvp->GetJson(); + + switch (jsp->GetType()) { + case TYPE_JOB: + job = (PJOB)jsp; + + for (PJPR jrp = job->GetFirst(); jrp; jrp = jrp->GetNext()) { + PCSZ k = jrp->GetKey(); + + if (*k != '$') { + strncat(strncat(fmt, sep, 128), k, 128); + strncat(strncat(colname, "_", 64), k, 64); + } // endif Key + + if (Find(g, jrp->GetVal(), k, j + 1)) + return true; + + *p = *pc = 0; + } // endfor jrp + + return false; + case TYPE_JAR: + jar = (PJAR)jsp; + + if (all || (tdp->Xcol && !stricmp(tdp->Xcol, key))) + ars = jar->GetSize(false); + else + ars = MY_MIN(jar->GetSize(false), 1); + + for (int k = 0; k < ars; k++) { + if (!tdp->Xcol || stricmp(tdp->Xcol, key)) { + sprintf(buf, "%d", k); + + if (tdp->Uri) + strncat(strncat(fmt, sep, 128), buf, 128); + else + strncat(strncat(strncat(fmt, "[", 128), buf, 128), "]", 128); + + if (all) + strncat(strncat(colname, "_", 64), buf, 64); + + } else + strncat(fmt, (tdp->Uri ? sep : "[*]"), 128); + + if (Find(g, jar->GetValue(k), "", j)) + return true; + + *p = *pc = 0; + } // endfor k + + return false; + default: + sprintf(g->Message, "Logical error after %s", fmt); + return true; + } // endswitch Type + + } else if (lvl >= 0) { + jcol.Type = TYPE_STRING; + jcol.Len = 256; + jcol.Scale = 0; + jcol.Cbn = true; + } else + return false; + + AddColumn(g); + return false; +} // end of Find void JSONDISC::AddColumn(PGLOBAL g) { - bool b = fmt[bf] != 0; // True if formatted - - // Check whether this column was already found - for (jcp = fjcp; jcp; jcp = jcp->Next) - if (!strcmp(colname, jcp->Name)) - break; - - if (jcp) { - if (jcp->Type != jcol.Type) { - if (jcp->Type == TYPE_UNKNOWN) - jcp->Type = jcol.Type; - else if (jcol.Type != TYPE_UNKNOWN) - jcp->Type = TYPE_STRING; - - } // endif Type - - if (b && (!jcp->Fmt || strlen(jcp->Fmt) < strlen(fmt))) { - jcp->Fmt = PlugDup(g, fmt); - length[7] = MY_MAX(length[7], strlen(fmt)); - } // endif fmt - - jcp->Len = MY_MAX(jcp->Len, jcol.Len); - jcp->Scale = MY_MAX(jcp->Scale, jcol.Scale); - jcp->Cbn |= jcol.Cbn; - jcp->Found = true; - } else if (jcol.Type != TYPE_UNKNOWN || tdp->Accept) { - // New column - jcp = (PJCL)PlugSubAlloc(g, NULL, sizeof(JCOL)); - *jcp = jcol; - jcp->Cbn |= (i > 1); - jcp->Name = PlugDup(g, colname); - length[0] = MY_MAX(length[0], strlen(colname)); - - if (b) { - jcp->Fmt = PlugDup(g, fmt); - length[7] = MY_MAX(length[7], strlen(fmt)); - } else - jcp->Fmt = NULL; - - if (pjcp) { - jcp->Next = pjcp->Next; - pjcp->Next = jcp; - } else - fjcp = jcp; - - n++; - } // endif jcp - - if (jcp) - pjcp = jcp; + bool b = fmt[bf] != 0; // True if formatted + + // Check whether this column was already found + for (jcp = fjcp; jcp; jcp = jcp->Next) + if (!strcmp(colname, jcp->Name)) + break; + + if (jcp) { + if (jcp->Type != jcol.Type) { + if (jcp->Type == TYPE_UNKNOWN) + jcp->Type = jcol.Type; + else if (jcol.Type != TYPE_UNKNOWN) + jcp->Type = TYPE_STRING; + + } // endif Type + + if (b && (!jcp->Fmt || strlen(jcp->Fmt) < strlen(fmt))) { + jcp->Fmt = PlugDup(g, fmt); + length[7] = MY_MAX(length[7], strlen(fmt)); + } // endif fmt + + jcp->Len = MY_MAX(jcp->Len, jcol.Len); + jcp->Scale = MY_MAX(jcp->Scale, jcol.Scale); + jcp->Cbn |= jcol.Cbn; + jcp->Found = true; + } else if (jcol.Type != TYPE_UNKNOWN || tdp->Accept) { + // New column + jcp = (PJCL)PlugSubAlloc(g, NULL, sizeof(JCOL)); + *jcp = jcol; + jcp->Cbn |= (i > 1); + jcp->Name = PlugDup(g, colname); + length[0] = MY_MAX(length[0], strlen(colname)); + + if (b) { + jcp->Fmt = PlugDup(g, fmt); + length[7] = MY_MAX(length[7], strlen(fmt)); + } else + jcp->Fmt = NULL; + + if (pjcp) { + jcp->Next = pjcp->Next; + pjcp->Next = jcp; + } else + fjcp = jcp; + + n++; + } // endif jcp + + if (jcp) + pjcp = jcp; } // end of AddColumn @@ -548,13 +548,13 @@ JSONDEF::JSONDEF(void) Limit = 1; Base = 0; Strict = false; - Sep = '.'; - Uri = NULL; - Collname = Options = Filter = NULL; - Pipe = false; - Driver = NULL; - Version = 0; - Wrapname = NULL; + Sep = '.'; + Uri = NULL; + Collname = Options = Filter = NULL; + Pipe = false; + Driver = NULL; + Version = 0; + Wrapname = NULL; } // end of JSONDEF constructor /***********************************************************************/ @@ -562,41 +562,41 @@ JSONDEF::JSONDEF(void) /***********************************************************************/ bool JSONDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) { - Schema = GetStringCatInfo(g, "DBname", Schema); - Jmode = (JMODE)GetIntCatInfo("Jmode", MODE_OBJECT); + Schema = GetStringCatInfo(g, "DBname", Schema); + Jmode = (JMODE)GetIntCatInfo("Jmode", MODE_OBJECT); Objname = GetStringCatInfo(g, "Object", NULL); Xcol = GetStringCatInfo(g, "Expand", NULL); Pretty = GetIntCatInfo("Pretty", 2); Limit = GetIntCatInfo("Limit", 10); Base = GetIntCatInfo("Base", 0) ? 1 : 0; - Sep = *GetStringCatInfo(g, "Separator", "."); - Accept = GetBoolCatInfo("Accept", false); + Sep = *GetStringCatInfo(g, "Separator", "."); + Accept = GetBoolCatInfo("Accept", false); - // Don't use url as MONGO uri when called from REST - if (stricmp(am, "REST") && (Uri = GetStringCatInfo(g, "Connect", NULL))) { + // Don't use url as MONGO uri when called from REST + if (stricmp(am, "REST") && (Uri = GetStringCatInfo(g, "Connect", NULL))) { #if defined(JAVA_SUPPORT) || defined(CMGO_SUPPORT) - Collname = GetStringCatInfo(g, "Name", - (Catfunc & (FNC_TABLE | FNC_COL)) ? NULL : Name); - Collname = GetStringCatInfo(g, "Tabname", Collname); - Options = GetStringCatInfo(g, "Colist", NULL); - Filter = GetStringCatInfo(g, "Filter", NULL); - Pipe = GetBoolCatInfo("Pipeline", false); - Driver = GetStringCatInfo(g, "Driver", NULL); - Version = GetIntCatInfo("Version", 3); - Pretty = 0; + Collname = GetStringCatInfo(g, "Name", + (Catfunc & (FNC_TABLE | FNC_COL)) ? NULL : Name); + Collname = GetStringCatInfo(g, "Tabname", Collname); + Options = GetStringCatInfo(g, "Colist", NULL); + Filter = GetStringCatInfo(g, "Filter", NULL); + Pipe = GetBoolCatInfo("Pipeline", false); + Driver = GetStringCatInfo(g, "Driver", NULL); + Version = GetIntCatInfo("Version", 3); + Pretty = 0; #if defined(JAVA_SUPPORT) - if (Version == 2) - Wrapname = GetStringCatInfo(g, "Wrapper", "Mongo2Interface"); - else - Wrapname = GetStringCatInfo(g, "Wrapper", "Mongo3Interface"); + if (Version == 2) + Wrapname = GetStringCatInfo(g, "Wrapper", "Mongo2Interface"); + else + Wrapname = GetStringCatInfo(g, "Wrapper", "Mongo3Interface"); #endif // JAVA_SUPPORT #else // !MONGO_SUPPORT - sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "MONGO"); - return true; + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "MONGO"); + return true; #endif // !MONGO_SUPPORT - } // endif Uri + } // endif Uri - return DOSDEF::DefineAM(g, (Uri ? "XMGO" : "DOS"), poff); + return DOSDEF::DefineAM(g, (Uri ? "XMGO" : "DOS"), poff); } // end of DefineAM /***********************************************************************/ @@ -604,6 +604,9 @@ bool JSONDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) /***********************************************************************/ PTDB JSONDEF::GetTable(PGLOBAL g, MODE m) { + if (trace(1)) + htrc("JSON GetTable Pretty=%d Uri=%s\n", Pretty, SVP(Uri)); + if (Catfunc == FNC_COL) return new(g)TDBJCL(this); @@ -618,47 +621,47 @@ PTDB JSONDEF::GetTable(PGLOBAL g, MODE m) !(tmp == TMP_FORCE && (m == MODE_UPDATE || m == MODE_DELETE)); - if (Uri) { - if (Driver && toupper(*Driver) == 'C') { + if (Uri) { + if (Driver && toupper(*Driver) == 'C') { #if defined(CMGO_SUPPORT) - txfp = new(g) CMGFAM(this); + txfp = new(g) CMGFAM(this); #else - sprintf(g->Message, "Mongo %s Driver not available", "C"); - return NULL; + sprintf(g->Message, "Mongo %s Driver not available", "C"); + return NULL; #endif - } else if (Driver && toupper(*Driver) == 'J') { + } else if (Driver && toupper(*Driver) == 'J') { #if defined(JAVA_SUPPORT) - txfp = new(g) JMGFAM(this); + txfp = new(g) JMGFAM(this); #else - sprintf(g->Message, "Mongo %s Driver not available", "Java"); - return NULL; + sprintf(g->Message, "Mongo %s Driver not available", "Java"); + return NULL; #endif - } else { // Driver not specified + } else { // Driver not specified #if defined(CMGO_SUPPORT) - txfp = new(g) CMGFAM(this); + txfp = new(g) CMGFAM(this); #elif defined(JAVA_SUPPORT) - txfp = new(g) JMGFAM(this); -#else // !MONGO_SUPPORT - sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "MONGO"); - return NULL; + txfp = new(g) JMGFAM(this); +#else // !MONGO_SUPPORT + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "MONGO"); + return NULL; #endif // !MONGO_SUPPORT - } // endif Driver + } // endif Driver - } else if (Zipped) { + } else if (Zipped) { #if defined(ZIP_SUPPORT) - if (m == MODE_READ || m == MODE_ANY || m == MODE_ALTER) { - txfp = new(g) UNZFAM(this); - } else if (m == MODE_INSERT) { - txfp = new(g) ZIPFAM(this); - } else { - strcpy(g->Message, "UPDATE/DELETE not supported for ZIP"); - return NULL; - } // endif's m + if (m == MODE_READ || m == MODE_ANY || m == MODE_ALTER) { + txfp = new(g) UNZFAM(this); + } else if (m == MODE_INSERT) { + txfp = new(g) ZIPFAM(this); + } else { + strcpy(g->Message, "UPDATE/DELETE not supported for ZIP"); + return NULL; + } // endif's m #else // !ZIP_SUPPORT - sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); - return NULL; + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; #endif // !ZIP_SUPPORT - } else if (Compressed) { + } else if (Compressed) { #if defined(GZ_SUPPORT) if (Compressed == 1) txfp = new(g) GZFAM(this); @@ -668,7 +671,7 @@ PTDB JSONDEF::GetTable(PGLOBAL g, MODE m) sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "GZ"); return NULL; #endif // !GZ_SUPPORT - } else if (map) + } else if (map) txfp = new(g) MAPFAM(this); else txfp = new(g) DOSFAM(this); @@ -676,41 +679,41 @@ PTDB JSONDEF::GetTable(PGLOBAL g, MODE m) // Txfp must be set for TDBDOS tdbp = new(g) TDBJSN(this, txfp); - if (Lrecl) { - // Allocate the parse work memory - PGLOBAL G = (PGLOBAL)PlugSubAlloc(g, NULL, sizeof(GLOBAL)); - memset(G, 0, sizeof(GLOBAL)); - G->Sarea_Size = Lrecl * 10; - G->Sarea = PlugSubAlloc(g, NULL, G->Sarea_Size); - PlugSubSet(G->Sarea, G->Sarea_Size); - G->jump_level = 0; - ((TDBJSN*)tdbp)->G = G; - } else { - strcpy(g->Message, "LRECL is not defined"); - return NULL; - } // endif Lrecl - - } else { - if (Zipped) { + if (Lrecl) { + // Allocate the parse work memory + PGLOBAL G = (PGLOBAL)PlugSubAlloc(g, NULL, sizeof(GLOBAL)); + memset(G, 0, sizeof(GLOBAL)); + G->Sarea_Size = Lrecl * 10; + G->Sarea = PlugSubAlloc(g, NULL, G->Sarea_Size); + PlugSubSet(G->Sarea, G->Sarea_Size); + G->jump_level = 0; + ((TDBJSN*)tdbp)->G = G; + } else { + strcpy(g->Message, "LRECL is not defined"); + return NULL; + } // endif Lrecl + + } else { + if (Zipped) { #if defined(ZIP_SUPPORT) - if (m == MODE_READ || m == MODE_ANY || m == MODE_ALTER) { - txfp = new(g) UNZFAM(this); - } else if (m == MODE_INSERT) { - strcpy(g->Message, "INSERT supported only for zipped JSON when pretty=0"); - return NULL; - } else { - strcpy(g->Message, "UPDATE/DELETE not supported for ZIP"); - return NULL; - } // endif's m + if (m == MODE_READ || m == MODE_ANY || m == MODE_ALTER) { + txfp = new(g) UNZFAM(this); + } else if (m == MODE_INSERT) { + strcpy(g->Message, "INSERT supported only for zipped JSON when pretty=0"); + return NULL; + } else { + strcpy(g->Message, "UPDATE/DELETE not supported for ZIP"); + return NULL; + } // endif's m #else // !ZIP_SUPPORT - sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); - return NULL; + sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP"); + return NULL; #endif // !ZIP_SUPPORT - } else - txfp = new(g) MAPFAM(this); + } else + txfp = new(g) MAPFAM(this); tdbp = new(g) TDBJSON(this, txfp); - ((TDBJSON*)tdbp)->G = g; + ((TDBJSON*)tdbp)->G = g; } // endif Pretty if (Multiple) @@ -738,16 +741,16 @@ TDBJSN::TDBJSN(PJDEF tdp, PTXF txfp) : TDBDOS(tdp, txfp) Limit = tdp->Limit; Pretty = tdp->Pretty; B = tdp->Base ? 1 : 0; - Sep = tdp->Sep; + Sep = tdp->Sep; Strict = tdp->Strict; } else { Jmode = MODE_OBJECT; Objname = NULL; - Xcol = NULL; + Xcol = NULL; Limit = 1; Pretty = 0; B = 0; - Sep = '.'; + Sep = '.'; Strict = false; } // endif tdp @@ -761,7 +764,7 @@ TDBJSN::TDBJSN(PJDEF tdp, PTXF txfp) : TDBDOS(tdp, txfp) TDBJSN::TDBJSN(TDBJSN *tdbp) : TDBDOS(NULL, tdbp) { - G = NULL; + G = NULL; Top = tdbp->Top; Row = tdbp->Row; Val = tdbp->Val; @@ -777,7 +780,7 @@ TDBJSN::TDBJSN(TDBJSN *tdbp) : TDBDOS(NULL, tdbp) SameRow = tdbp->SameRow; Xval = tdbp->Xval; B = tdbp->B; - Sep = tdbp->Sep; + Sep = tdbp->Sep; Pretty = tdbp->Pretty; Strict = tdbp->Strict; Comma = tdbp->Comma; @@ -786,7 +789,7 @@ TDBJSN::TDBJSN(TDBJSN *tdbp) : TDBDOS(NULL, tdbp) // Used for update PTDB TDBJSN::Clone(PTABS t) { - G = NULL; + G = NULL; PTDB tp; PJCOL cp1, cp2; PGLOBAL g = t->G; @@ -860,33 +863,33 @@ PJSON TDBJSN::FindRow(PGLOBAL g) PJSON jsp = Row; PJVAL val = NULL; - for (objpath = PlugDup(g, Objname); jsp && objpath; objpath = p) { - if ((p = strchr(objpath, Sep))) - *p++ = 0; + for (objpath = PlugDup(g, Objname); jsp && objpath; objpath = p) { + if ((p = strchr(objpath, Sep))) + *p++ = 0; - if (*objpath != '[' && !IsNum(objpath)) { // objpass is a key - val = (jsp->GetType() == TYPE_JOB) ? - jsp->GetObject()->GetValue(objpath) : NULL; - } else { - if (*objpath == '[') { - if (objpath[strlen(objpath) - 1] == ']') - objpath++; - else - return NULL; - } // endif [ + if (*objpath != '[' && !IsNum(objpath)) { // objpass is a key + val = (jsp->GetType() == TYPE_JOB) ? + jsp->GetObject()->GetValue(objpath) : NULL; + } else { + if (*objpath == '[') { + if (objpath[strlen(objpath) - 1] == ']') + objpath++; + else + return NULL; + } // endif [ - val = (jsp->GetType() == TYPE_JAR) ? - jsp->GetArray()->GetValue(atoi(objpath) - B) : NULL; - } // endif objpath + val = (jsp->GetType() == TYPE_JAR) ? + jsp->GetArray()->GetValue(atoi(objpath) - B) : NULL; + } // endif objpath - jsp = (val) ? val->GetJson() : NULL; - } // endfor objpath + jsp = (val) ? val->GetJson() : NULL; + } // endfor objpath return jsp; } // end of FindRow /***********************************************************************/ -/* OpenDB: Data Base open routine for JSN access method. */ +/* OpenDB: Data Base open routine for JSN access method. */ /***********************************************************************/ bool TDBJSN::OpenDB(PGLOBAL g) { @@ -911,15 +914,15 @@ bool TDBJSN::OpenDB(PGLOBAL g) return true; } // endswitch Jmode - } // endif Use + } // endif Use - if (TDBDOS::OpenDB(g)) - return true; + if (TDBDOS::OpenDB(g)) + return true; - if (Xcol) - To_Filter = NULL; // Imcompatible + if (Xcol) + To_Filter = NULL; // Imcompatible - return false; + return false; } // end of OpenDB /***********************************************************************/ @@ -969,27 +972,27 @@ int TDBJSN::ReadDB(PGLOBAL g) NextSame = 0; M++; return RC_OK; - } else if ((rc = TDBDOS::ReadDB(g)) == RC_OK) { - if (!IsRead() && ((rc = ReadBuffer(g)) != RC_OK)) - // Deferred reading failed - return rc; - - // Recover the memory used for parsing - PlugSubSet(G->Sarea, G->Sarea_Size); - - if ((Row = ParseJson(G, To_Line, strlen(To_Line), &Pretty, &Comma))) { - Row = FindRow(g); - SameRow = 0; - Fpos++; - M = 1; - rc = RC_OK; - } else if (Pretty != 1 || strcmp(To_Line, "]")) { - strcpy(g->Message, G->Message); - rc = RC_FX; - } else - rc = RC_EF; - - } // endif ReadDB + } else if ((rc = TDBDOS::ReadDB(g)) == RC_OK) { + if (!IsRead() && ((rc = ReadBuffer(g)) != RC_OK)) + // Deferred reading failed + return rc; + + // Recover the memory used for parsing + PlugSubSet(G->Sarea, G->Sarea_Size); + + if ((Row = ParseJson(G, To_Line, strlen(To_Line), &Pretty, &Comma))) { + Row = FindRow(g); + SameRow = 0; + Fpos++; + M = 1; + rc = RC_OK; + } else if (Pretty != 1 || strcmp(To_Line, "]")) { + strcpy(g->Message, G->Message); + rc = RC_FX; + } else + rc = RC_EF; + + } // endif ReadDB return rc; } // end of ReadDB @@ -999,68 +1002,68 @@ int TDBJSN::ReadDB(PGLOBAL g) /***********************************************************************/ int TDBJSN::MakeTopTree(PGLOBAL g, PJSON jsp) { - if (Objname) { - if (!Val) { - // Parse and allocate Objname item(s) - char *p; - char *objpath = PlugDup(g, Objname); - int i; - PJOB objp; - PJAR arp; - PJVAL val = NULL; + if (Objname) { + if (!Val) { + // Parse and allocate Objname item(s) + char *p; + char *objpath = PlugDup(g, Objname); + int i; + PJOB objp; + PJAR arp; + PJVAL val = NULL; - Top = NULL; + Top = NULL; - for (; objpath; objpath = p) { - if ((p = strchr(objpath, Sep))) - *p++ = 0; + for (; objpath; objpath = p) { + if ((p = strchr(objpath, Sep))) + *p++ = 0; - if (*objpath != '[' && !IsNum(objpath)) { - objp = new(g) JOBJECT; + if (*objpath != '[' && !IsNum(objpath)) { + objp = new(g) JOBJECT; - if (!Top) - Top = objp; + if (!Top) + Top = objp; - if (val) - val->SetValue(objp); + if (val) + val->SetValue(objp); - val = new(g) JVALUE; - objp->SetValue(g, val, objpath); - } else { - if (*objpath == '[') { - // Old style - if (objpath[strlen(objpath) - 1] != ']') { - sprintf(g->Message, "Invalid Table path %s", Objname); - return RC_FX; - } else - objpath++; + val = new(g) JVALUE; + objp->SetValue(g, val, objpath); + } else { + if (*objpath == '[') { + // Old style + if (objpath[strlen(objpath) - 1] != ']') { + sprintf(g->Message, "Invalid Table path %s", Objname); + return RC_FX; + } else + objpath++; - } // endif objpath + } // endif objpath - arp = new(g) JARRAY; + arp = new(g) JARRAY; - if (!Top) - Top = arp; + if (!Top) + Top = arp; - if (val) - val->SetValue(arp); + if (val) + val->SetValue(arp); - val = new(g) JVALUE; - i = atoi(objpath) - B; - arp->SetValue(g, val, i); - arp->InitArray(g); - } // endif objpath + val = new(g) JVALUE; + i = atoi(objpath) - B; + arp->SetValue(g, val, i); + arp->InitArray(g); + } // endif objpath - } // endfor p + } // endfor p - Val = val; - } // endif Val + Val = val; + } // endif Val - Val->SetValue(jsp); - } else - Top = jsp; + Val->SetValue(jsp); + } else + Top = jsp; - return RC_OK; + return RC_OK; } // end of MakeTopTree /***********************************************************************/ @@ -1095,11 +1098,11 @@ int TDBJSN::MakeTopTree(PGLOBAL g, PJSON jsp) /***********************************************************************/ int TDBJSN::WriteDB(PGLOBAL g) { - int rc = TDBDOS::WriteDB(g); + int rc = TDBDOS::WriteDB(g); - PlugSubSet(G->Sarea, G->Sarea_Size); - Row->Clear(); - return rc; + PlugSubSet(G->Sarea, G->Sarea_Size); + Row->Clear(); + return rc; } // end of WriteDB /* ---------------------------- JSONCOL ------------------------------ */ @@ -1111,12 +1114,12 @@ JSONCOL::JSONCOL(PGLOBAL g, PCOLDEF cdp, PTDB tdbp, PCOL cprec, int i) : DOSCOL(g, cdp, tdbp, cprec, i, "DOS") { Tjp = (TDBJSN *)(tdbp->GetOrig() ? tdbp->GetOrig() : tdbp); - G = Tjp->G; + G = Tjp->G; Jpath = cdp->GetFmt(); MulVal = NULL; Nodes = NULL; Nod = 0; - Sep = Tjp->Sep; + Sep = Tjp->Sep; Xnod = -1; Xpd = false; Parsed = false; @@ -1128,14 +1131,14 @@ JSONCOL::JSONCOL(PGLOBAL g, PCOLDEF cdp, PTDB tdbp, PCOL cprec, int i) /***********************************************************************/ JSONCOL::JSONCOL(JSONCOL *col1, PTDB tdbp) : DOSCOL(col1, tdbp) { - G = col1->G; + G = col1->G; Tjp = col1->Tjp; Jpath = col1->Jpath; MulVal = col1->MulVal; Nodes = col1->Nodes; Nod = col1->Nod; - Sep = col1->Sep; - Xnod = col1->Xnod; + Sep = col1->Sep; + Xnod = col1->Xnod; Xpd = col1->Xpd; Parsed = col1->Parsed; } // end of JSONCOL copy constructor @@ -1153,7 +1156,7 @@ bool JSONCOL::SetBuffer(PGLOBAL g, PVAL value, bool ok, bool check) return true; Tjp = (TDBJSN*)To_Tdb; - G = Tjp->G; + G = Tjp->G; return false; } // end of SetBuffer @@ -1179,130 +1182,130 @@ bool JSONCOL::CheckExpand(PGLOBAL g, int i, PSZ nm, bool b) /***********************************************************************/ bool JSONCOL::SetArrayOptions(PGLOBAL g, char *p, int i, PSZ nm) { - int n; - bool dg = true, b = false; - PJNODE jnp = &Nodes[i]; - - //if (*p == '[') p++; // Old syntax .[ or :[ - n = (int)strlen(p); - - if (*p) { - if (p[n - 1] == ']') { - p[--n] = 0; - } else if (!IsNum(p)) { - // Wrong array specification - sprintf(g->Message, "Invalid array specification %s for %s", p, Name); - return true; - } // endif p - - } else - b = true; - - // To check whether a numeric Rank was specified - dg = IsNum(p); - - if (!n) { - // Default specifications - if (CheckExpand(g, i, nm, false)) - return true; - else if (jnp->Op != OP_EXP) { - if (b) { - // Return 1st value (B is the index base) - jnp->Rank = Tjp->B; - jnp->Op = OP_EQ; - } else if (!Value->IsTypeNum()) { - jnp->CncVal = AllocateValue(g, (void*)", ", TYPE_STRING); - jnp->Op = OP_CNC; - } else - jnp->Op = OP_ADD; - - } // endif OP - - } else if (dg) { - // Return nth value - jnp->Rank = atoi(p) - Tjp->B; - jnp->Op = OP_EQ; - } else if (n == 1) { - // Set the Op value; - if (Sep == ':') - switch (*p) { - case '*': *p = 'x'; break; - case 'x': - case 'X': *p = '*'; break; // Expand this array - default: break; - } // endswitch p - - switch (*p) { - case '+': jnp->Op = OP_ADD; break; - case 'x': jnp->Op = OP_MULT; break; - case '>': jnp->Op = OP_MAX; break; - case '<': jnp->Op = OP_MIN; break; - case '!': jnp->Op = OP_SEP; break; // Average - case '#': jnp->Op = OP_NUM; break; - case '*': // Expand this array - if (!Tjp->Xcol && nm) { - Xpd = true; - jnp->Op = OP_EXP; - Tjp->Xval = i; - Tjp->Xcol = nm; - } else if (CheckExpand(g, i, nm, true)) - return true; - - break; - default: - sprintf(g->Message, - "Invalid function specification %c for %s", *p, Name); - return true; - } // endswitch *p - - } else if (*p == '"' && p[n - 1] == '"') { - // This is a concat specification - jnp->Op = OP_CNC; - - if (n > 2) { - // Set concat intermediate string - p[n - 1] = 0; - jnp->CncVal = AllocateValue(g, p + 1, TYPE_STRING); - } // endif n - - } else { - sprintf(g->Message, "Wrong array specification for %s", Name); - return true; - } // endif's - - // For calculated arrays, a local Value must be used - switch (jnp->Op) { - case OP_NUM: - jnp->Valp = AllocateValue(g, TYPE_INT); - break; - case OP_ADD: - case OP_MULT: - case OP_SEP: - if (!IsTypeChar(Buf_Type)) - jnp->Valp = AllocateValue(g, Buf_Type, 0, GetPrecision()); - else - jnp->Valp = AllocateValue(g, TYPE_DOUBLE, 0, 2); - - break; - case OP_MIN: - case OP_MAX: - jnp->Valp = AllocateValue(g, Buf_Type, Long, GetPrecision()); - break; - case OP_CNC: - if (IsTypeChar(Buf_Type)) - jnp->Valp = AllocateValue(g, TYPE_STRING, Long, GetPrecision()); - else - jnp->Valp = AllocateValue(g, TYPE_STRING, 512); - - break; - default: - break; - } // endswitch Op - - if (jnp->Valp) - MulVal = AllocateValue(g, jnp->Valp); - - return false; + int n; + bool dg = true, b = false; + PJNODE jnp = &Nodes[i]; + + //if (*p == '[') p++; // Old syntax .[ or :[ + n = (int)strlen(p); + + if (*p) { + if (p[n - 1] == ']') { + p[--n] = 0; + } else if (!IsNum(p)) { + // Wrong array specification + sprintf(g->Message, "Invalid array specification %s for %s", p, Name); + return true; + } // endif p + + } else + b = true; + + // To check whether a numeric Rank was specified + dg = IsNum(p); + + if (!n) { + // Default specifications + if (CheckExpand(g, i, nm, false)) + return true; + else if (jnp->Op != OP_EXP) { + if (b) { + // Return 1st value (B is the index base) + jnp->Rank = Tjp->B; + jnp->Op = OP_EQ; + } else if (!Value->IsTypeNum()) { + jnp->CncVal = AllocateValue(g, (void*)", ", TYPE_STRING); + jnp->Op = OP_CNC; + } else + jnp->Op = OP_ADD; + + } // endif OP + + } else if (dg) { + // Return nth value + jnp->Rank = atoi(p) - Tjp->B; + jnp->Op = OP_EQ; + } else if (n == 1) { + // Set the Op value; + if (Sep == ':') + switch (*p) { + case '*': *p = 'x'; break; + case 'x': + case 'X': *p = '*'; break; // Expand this array + default: break; + } // endswitch p + + switch (*p) { + case '+': jnp->Op = OP_ADD; break; + case 'x': jnp->Op = OP_MULT; break; + case '>': jnp->Op = OP_MAX; break; + case '<': jnp->Op = OP_MIN; break; + case '!': jnp->Op = OP_SEP; break; // Average + case '#': jnp->Op = OP_NUM; break; + case '*': // Expand this array + if (!Tjp->Xcol && nm) { + Xpd = true; + jnp->Op = OP_EXP; + Tjp->Xval = i; + Tjp->Xcol = nm; + } else if (CheckExpand(g, i, nm, true)) + return true; + + break; + default: + sprintf(g->Message, + "Invalid function specification %c for %s", *p, Name); + return true; + } // endswitch *p + + } else if (*p == '"' && p[n - 1] == '"') { + // This is a concat specification + jnp->Op = OP_CNC; + + if (n > 2) { + // Set concat intermediate string + p[n - 1] = 0; + jnp->CncVal = AllocateValue(g, p + 1, TYPE_STRING); + } // endif n + + } else { + sprintf(g->Message, "Wrong array specification for %s", Name); + return true; + } // endif's + + // For calculated arrays, a local Value must be used + switch (jnp->Op) { + case OP_NUM: + jnp->Valp = AllocateValue(g, TYPE_INT); + break; + case OP_ADD: + case OP_MULT: + case OP_SEP: + if (!IsTypeChar(Buf_Type)) + jnp->Valp = AllocateValue(g, Buf_Type, 0, GetPrecision()); + else + jnp->Valp = AllocateValue(g, TYPE_DOUBLE, 0, 2); + + break; + case OP_MIN: + case OP_MAX: + jnp->Valp = AllocateValue(g, Buf_Type, Long, GetPrecision()); + break; + case OP_CNC: + if (IsTypeChar(Buf_Type)) + jnp->Valp = AllocateValue(g, TYPE_STRING, Long, GetPrecision()); + else + jnp->Valp = AllocateValue(g, TYPE_STRING, 512); + + break; + default: + break; + } // endswitch Op + + if (jnp->Valp) + MulVal = AllocateValue(g, jnp->Valp); + + return false; } // end of SetArrayOptions /***********************************************************************/ @@ -1313,87 +1316,87 @@ bool JSONCOL::SetArrayOptions(PGLOBAL g, char *p, int i, PSZ nm) /***********************************************************************/ bool JSONCOL::ParseJpath(PGLOBAL g) { - char *p, *p1 = NULL, *p2 = NULL, *pbuf = NULL; - int i; - bool a, mul = false; - - if (Parsed) - return false; // Already done - else if (InitValue(g)) - return true; - else if (!Jpath) - Jpath = Name; - - if (To_Tdb->GetOrig()) { - // This is an updated column, get nodes from origin - for (PJCOL colp = (PJCOL)Tjp->GetColumns(); colp; - colp = (PJCOL)colp->GetNext()) - if (!stricmp(Name, colp->GetName())) { - Nod = colp->Nod; - Nodes = colp->Nodes; - Xpd = colp->Xpd; - goto fin; - } // endif Name - - sprintf(g->Message, "Cannot parse updated column %s", Name); - return true; - } // endif To_Orig - - pbuf = PlugDup(g, Jpath); - if (*pbuf == '$') pbuf++; - if (*pbuf == Sep) pbuf++; - if (*pbuf == '[') p1 = pbuf++; - - // Estimate the required number of nodes - for (i = 0, p = pbuf; (p = NextChr(p, Sep)); i++, p++) - Nod++; // One path node found - - Nodes = (PJNODE)PlugSubAlloc(g, NULL, (++Nod) * sizeof(JNODE)); - memset(Nodes, 0, (Nod) * sizeof(JNODE)); - - // Analyze the Jpath for this column - for (i = 0, p = pbuf; p && i < Nod; i++, p = (p2 ? p2 : NULL)) { - a = (p1 != NULL); - p1 = strchr(p, '['); - p2 = strchr(p, Sep); - - if (!p2) - p2 = p1; - else if (p1) { - if (p1 < p2) - p2 = p1; - else if (p1 == p2 + 1) - *p2++ = 0; // Old syntax .[ or :[ - else - p1 = NULL; - - } // endif p1 - - if (p2) - *p2++ = 0; - - // Jpath must be explicit - if (a || *p == 0 || *p == '[' || IsNum(p)) { - // Analyse intermediate array processing - if (SetArrayOptions(g, p, i, Nodes[i - 1].Key)) - return true; - - } else if (*p == '*') { - // Return JSON - Nodes[i].Op = OP_XX; - } else { - Nodes[i].Key = p; - Nodes[i].Op = OP_EXIST; - } // endif's - - } // endfor i, p - - Nod = i; + char *p, *p1 = NULL, *p2 = NULL, *pbuf = NULL; + int i; + bool a, mul = false; + + if (Parsed) + return false; // Already done + else if (InitValue(g)) + return true; + else if (!Jpath) + Jpath = Name; + + if (To_Tdb->GetOrig()) { + // This is an updated column, get nodes from origin + for (PJCOL colp = (PJCOL)Tjp->GetColumns(); colp; + colp = (PJCOL)colp->GetNext()) + if (!stricmp(Name, colp->GetName())) { + Nod = colp->Nod; + Nodes = colp->Nodes; + Xpd = colp->Xpd; + goto fin; + } // endif Name + + sprintf(g->Message, "Cannot parse updated column %s", Name); + return true; + } // endif To_Orig + + pbuf = PlugDup(g, Jpath); + if (*pbuf == '$') pbuf++; + if (*pbuf == Sep) pbuf++; + if (*pbuf == '[') p1 = pbuf++; + + // Estimate the required number of nodes + for (i = 0, p = pbuf; (p = NextChr(p, Sep)); i++, p++) + Nod++; // One path node found + + Nodes = (PJNODE)PlugSubAlloc(g, NULL, (++Nod) * sizeof(JNODE)); + memset(Nodes, 0, (Nod) * sizeof(JNODE)); + + // Analyze the Jpath for this column + for (i = 0, p = pbuf; p && i < Nod; i++, p = (p2 ? p2 : NULL)) { + a = (p1 != NULL); + p1 = strchr(p, '['); + p2 = strchr(p, Sep); + + if (!p2) + p2 = p1; + else if (p1) { + if (p1 < p2) + p2 = p1; + else if (p1 == p2 + 1) + *p2++ = 0; // Old syntax .[ or :[ + else + p1 = NULL; + + } // endif p1 + + if (p2) + *p2++ = 0; + + // Jpath must be explicit + if (a || *p == 0 || *p == '[' || IsNum(p)) { + // Analyse intermediate array processing + if (SetArrayOptions(g, p, i, Nodes[i - 1].Key)) + return true; + + } else if (*p == '*') { + // Return JSON + Nodes[i].Op = OP_XX; + } else { + Nodes[i].Key = p; + Nodes[i].Op = OP_EXIST; + } // endif's + + } // endfor i, p + + Nod = i; fin: - MulVal = AllocateValue(g, Value); - Parsed = true; - return false; + MulVal = AllocateValue(g, Value); + Parsed = true; + return false; } // end of ParseJpath /***********************************************************************/ @@ -1401,66 +1404,66 @@ fin: /***********************************************************************/ PSZ JSONCOL::GetJpath(PGLOBAL g, bool proj) { - if (Jpath) { - char *p1, *p2, *mgopath; - int i = 0; - - if (strcmp(Jpath, "*")) { - p1 = Jpath; - if (*p1 == '$') p1++; - if (*p1 == '.') p1++; - mgopath = PlugDup(g, p1); - } else - return NULL; - - for (p1 = p2 = mgopath; *p1; p1++) - if (i) { // Inside [] - if (isdigit(*p1)) { - if (!proj) - *p2++ = *p1; - - } else if (*p1 == ']' && i == 1) { - if (proj && p1[1] == '.') - p1++; - - i = 0; - } else if (*p1 == '.' && i == 2) { - if (!proj) - *p2++ = '.'; - - i = 0; - } else if (!proj) - return NULL; - - } else switch (*p1) { - case ':': - case '.': - if (isdigit(p1[1])) - i = 2; - - *p2++ = '.'; - break; - case '[': - if (*(p2 - 1) != '.') - *p2++ = '.'; - - i = 1; - break; - case '*': - if (*(p2 - 1) == '.' && !*(p1 + 1)) { - p2--; // Suppress last :* - break; - } // endif p2 - - default: - *p2++ = *p1; - break; - } // endswitch p1; - - *p2 = 0; - return mgopath; - } else - return NULL; + if (Jpath) { + char *p1, *p2, *mgopath; + int i = 0; + + if (strcmp(Jpath, "*")) { + p1 = Jpath; + if (*p1 == '$') p1++; + if (*p1 == '.') p1++; + mgopath = PlugDup(g, p1); + } else + return NULL; + + for (p1 = p2 = mgopath; *p1; p1++) + if (i) { // Inside [] + if (isdigit(*p1)) { + if (!proj) + *p2++ = *p1; + + } else if (*p1 == ']' && i == 1) { + if (proj && p1[1] == '.') + p1++; + + i = 0; + } else if (*p1 == '.' && i == 2) { + if (!proj) + *p2++ = '.'; + + i = 0; + } else if (!proj) + return NULL; + + } else switch (*p1) { + case ':': + case '.': + if (isdigit(p1[1])) + i = 2; + + *p2++ = '.'; + break; + case '[': + if (*(p2 - 1) != '.') + *p2++ = '.'; + + i = 1; + break; + case '*': + if (*(p2 - 1) == '.' && !*(p1 + 1)) { + p2--; // Suppress last :* + break; + } // endif p2 + + default: + *p2++ = *p1; + break; + } // endswitch p1; + + *p2 = 0; + return mgopath; + } else + return NULL; } // end of GetJpath @@ -1469,7 +1472,7 @@ PSZ JSONCOL::GetJpath(PGLOBAL g, bool proj) /***********************************************************************/ PVAL JSONCOL::MakeJson(PGLOBAL g, PJSON jsp) { - if (Value->IsTypeNum()) { + if (Value->IsTypeNum()) { strcpy(g->Message, "Cannot make Json for a numeric column"); Value->Reset(); } else @@ -1484,22 +1487,22 @@ PVAL JSONCOL::MakeJson(PGLOBAL g, PJSON jsp) void JSONCOL::SetJsonValue(PGLOBAL g, PVAL vp, PJVAL val, int n) { if (val) { - vp->SetNull(false); + vp->SetNull(false); switch (val->GetValType()) { case TYPE_STRG: case TYPE_INTG: - case TYPE_BINT: - case TYPE_DBL: - case TYPE_DTM: - vp->SetValue_pval(val->GetValue()); + case TYPE_BINT: + case TYPE_DBL: + case TYPE_DTM: + vp->SetValue_pval(val->GetValue()); break; case TYPE_BOOL: if (vp->IsTypeNum()) vp->SetValue(val->GetInteger() ? 1 : 0); else vp->SetValue_psz((PSZ)(val->GetInteger() ? "true" : "false")); - + break; case TYPE_JAR: SetJsonValue(g, vp, val->GetArray()->GetValue(0), n); @@ -1509,16 +1512,16 @@ void JSONCOL::SetJsonValue(PGLOBAL g, PVAL vp, PJVAL val, int n) vp->SetValue_psz(val->GetObject()->GetText(g, NULL)); break; // } // endif Type - + default: - vp->Reset(); - vp->SetNull(true); - } // endswitch Type + vp->Reset(); + vp->SetNull(true); + } // endswitch Type - } else { - vp->Reset(); - vp->SetNull(true); - } // endif val + } else { + vp->Reset(); + vp->SetNull(true); + } // endif val } // end of SetJsonValue @@ -1530,8 +1533,8 @@ void JSONCOL::ReadColumn(PGLOBAL g) if (!Tjp->SameRow || Xnod >= Tjp->SameRow) Value->SetValue_pval(GetColumnValue(g, Tjp->Row, 0)); - if (Xpd && Value->IsNull() && !((PJDEF)Tjp->To_Def)->Accept) - throw("Null expandable JSON value"); + if (Xpd && Value->IsNull() && !((PJDEF)Tjp->To_Def)->Accept) + throw("Null expandable JSON value"); // Set null when applicable if (!Nullable) @@ -1579,11 +1582,11 @@ PVAL JSONCOL::GetColumnValue(PGLOBAL g, PJSON row, int i) else return CalculateArray(g, arp, i); - } else { - // Unexpected array, unwrap it as [0] - val = arp->GetValue(0); - i--; - } // endif's + } else { + // Unexpected array, unwrap it as [0] + val = arp->GetValue(0); + i--; + } // endif's break; case TYPE_JVAL: @@ -1612,17 +1615,17 @@ PVAL JSONCOL::ExpandArray(PGLOBAL g, PJAR arp, int n) PJVAL jvp; JVALUE jval; - if (!ars) { - Value->Reset(); - Value->SetNull(true); - Tjp->NextSame = 0; - return Value; - } // endif ars + if (!ars) { + Value->Reset(); + Value->SetNull(true); + Tjp->NextSame = 0; + return Value; + } // endif ars if (!(jvp = arp->GetValue((Nodes[n].Rx = Nodes[n].Nx)))) { strcpy(g->Message, "Logical error expanding array"); - throw 666; - } // endif jvp + throw 666; + } // endif jvp if (n < Nod - 1 && jvp->GetJson()) { jval.SetValue(GetColumnValue(g, jvp->GetJson(), n + 1)); @@ -1637,7 +1640,7 @@ PVAL JSONCOL::ExpandArray(PGLOBAL g, PJAR arp, int n) Xnod = n; Tjp->NextSame = Xnod; - } // endif NextSame + } // endif NextSame SetJsonValue(g, Value, jvp, n); return Value; @@ -1648,58 +1651,58 @@ PVAL JSONCOL::ExpandArray(PGLOBAL g, PJAR arp, int n) /***********************************************************************/ PVAL JSONCOL::CalculateArray(PGLOBAL g, PJAR arp, int n) { - int i, ars, nv = 0, nextsame = Tjp->NextSame; - bool err; + int i, ars, nv = 0, nextsame = Tjp->NextSame; + bool err; OPVAL op = Nodes[n].Op; PVAL val[2], vp = Nodes[n].Valp; PJVAL jvrp, jvp; JVALUE jval; vp->Reset(); - ars = MY_MIN(Tjp->Limit, arp->size()); + ars = MY_MIN(Tjp->Limit, arp->size()); - if (trace(1)) - htrc("CalculateArray: size=%d op=%d nextsame=%d\n", - ars, op, nextsame); + if (trace(1)) + htrc("CalculateArray: size=%d op=%d nextsame=%d\n", + ars, op, nextsame); - for (i = 0; i < ars; i++) { - jvrp = arp->GetValue(i); + for (i = 0; i < ars; i++) { + jvrp = arp->GetValue(i); - if (trace(1)) - htrc("i=%d nv=%d\n", i, nv); + if (trace(1)) + htrc("i=%d nv=%d\n", i, nv); - if (!jvrp->IsNull() || (op == OP_CNC && GetJsonNull())) do { - if (jvrp->IsNull()) { - jvrp->Value = AllocateValue(g, GetJsonNull(), TYPE_STRING); - jvp = jvrp; - } else if (n < Nod - 1 && jvrp->GetJson()) { + if (!jvrp->IsNull() || (op == OP_CNC && GetJsonNull())) do { + if (jvrp->IsNull()) { + jvrp->Value = AllocateValue(g, GetJsonNull(), TYPE_STRING); + jvp = jvrp; + } else if (n < Nod - 1 && jvrp->GetJson()) { Tjp->NextSame = nextsame; jval.SetValue(GetColumnValue(g, jvrp->GetJson(), n + 1)); jvp = &jval; } else jvp = jvrp; - - if (trace(1)) - htrc("jvp=%s null=%d\n", - jvp->GetString(g), jvp->IsNull() ? 1 : 0); - if (!nv++) { + if (trace(1)) + htrc("jvp=%s null=%d\n", + jvp->GetString(g), jvp->IsNull() ? 1 : 0); + + if (!nv++) { SetJsonValue(g, vp, jvp, n); continue; } else SetJsonValue(g, MulVal, jvp, n); - if (!MulVal->IsNull()) { - switch (op) { + if (!MulVal->IsNull()) { + switch (op) { case OP_CNC: if (Nodes[n].CncVal) { val[0] = Nodes[n].CncVal; err = vp->Compute(g, val, 1, op); } // endif CncVal - + val[0] = MulVal; err = vp->Compute(g, val, 1, op); - break; + break; // case OP_NUM: case OP_SEP: val[0] = Nodes[n].Valp; @@ -1714,16 +1717,16 @@ PVAL JSONCOL::CalculateArray(PGLOBAL g, PJAR arp, int n) if (err) vp->Reset(); - - if (trace(1)) { - char buf(32); - htrc("vp='%s' err=%d\n", - vp->GetCharString(&buf), err ? 1 : 0); + if (trace(1)) { + char buf(32); - } // endif trace + htrc("vp='%s' err=%d\n", + vp->GetCharString(&buf), err ? 1 : 0); - } // endif Null + } // endif trace + + } // endif Null } while (Tjp->NextSame > nextsame); @@ -1753,8 +1756,8 @@ PJSON JSONCOL::GetRow(PGLOBAL g) PJAR arp; PJSON nwr, row = Tjp->Row; - for (int i = 0; i < Nod && row; i++) { - if (Nodes[i+1].Op == OP_XX) + for (int i = 0; i < Nod && row; i++) { + if (Nodes[i+1].Op == OP_XX) break; else switch (row->GetType()) { case TYPE_JOB: @@ -1765,19 +1768,19 @@ PJSON JSONCOL::GetRow(PGLOBAL g) val = ((PJOB)row)->GetValue(Nodes[i].Key); break; case TYPE_JAR: - arp = (PJAR)row; + arp = (PJAR)row; - if (!Nodes[i].Key) { + if (!Nodes[i].Key) { if (Nodes[i].Op == OP_EQ) val = arp->GetValue(Nodes[i].Rank); else val = arp->GetValue(Nodes[i].Rx); } else { - // Unexpected array, unwrap it as [0] - val = arp->GetValue(0); - i--; - } // endif Nodes + // Unexpected array, unwrap it as [0] + val = arp->GetValue(0); + i--; + } // endif Nodes break; case TYPE_JVAL: @@ -1827,10 +1830,10 @@ PJSON JSONCOL::GetRow(PGLOBAL g) /***********************************************************************/ void JSONCOL::WriteColumn(PGLOBAL g) { - if (Xpd && Tjp->Pretty < 2) { - strcpy(g->Message, "Cannot write expanded column when Pretty is not 2"); - throw 666; - } // endif Xpd + if (Xpd && Tjp->Pretty < 2) { + strcpy(g->Message, "Cannot write expanded column when Pretty is not 2"); + throw 666; + } // endif Xpd /*********************************************************************/ /* Check whether this node must be written. */ @@ -1864,8 +1867,8 @@ void JSONCOL::WriteColumn(PGLOBAL g) if (!(jsp = ParseJson(G, s, (int)strlen(s)))) { strcpy(g->Message, s); - throw 666; - } // endif jsp + throw 666; + } // endif jsp if (arp) { if (Nod > 1 && Nodes[Nod-2].Op == OP_EQ) @@ -1887,10 +1890,10 @@ void JSONCOL::WriteColumn(PGLOBAL g) // fall through case TYPE_DATE: case TYPE_INT: - case TYPE_TINY: - case TYPE_SHORT: - case TYPE_BIGINT: - case TYPE_DOUBLE: + case TYPE_TINY: + case TYPE_SHORT: + case TYPE_BIGINT: + case TYPE_DOUBLE: if (arp) { if (Nodes[Nod-1].Op == OP_EQ) arp->SetValue(G, new(G) JVALUE(G, Value), Nodes[Nod-1].Rank); @@ -1981,7 +1984,7 @@ int TDBJSON::MakeDocument(PGLOBAL g) return RC_OK; /*********************************************************************/ - /* Create the mapping file object in mode read. */ + /* Create the mapping file object in mode read. */ /*********************************************************************/ Mode = MODE_READ; @@ -2010,70 +2013,70 @@ int TDBJSON::MakeDocument(PGLOBAL g) if (!jsp && g->Message[0]) return RC_FX; - if ((objpath = PlugDup(g, Objname))) { - if (*objpath == '$') objpath++; - if (*objpath == '.') objpath++; - - /*********************************************************************/ - /* Find the table in the tree structure. */ - /*********************************************************************/ - for (; jsp && objpath; objpath = p) { - if ((p = strchr(objpath, Sep))) - *p++ = 0; - - if (*objpath != '[' && !IsNum(objpath)) { - // objpass is a key - if (jsp->GetType() != TYPE_JOB) { - strcpy(g->Message, "Table path does not match the json file"); - return RC_FX; - } // endif Type - - key = objpath; - objp = jsp->GetObject(); - arp = NULL; - val = objp->GetValue(key); - - if (!val || !(jsp = val->GetJson())) { - sprintf(g->Message, "Cannot find object key %s", key); - return RC_FX; - } // endif val - - } else { - if (*objpath == '[') { - // Old style - if (objpath[strlen(objpath) - 1] != ']') { - sprintf(g->Message, "Invalid Table path %s", Objname); - return RC_FX; - } else - objpath++; - - } // endif objpath - - if (jsp->GetType() != TYPE_JAR) { - strcpy(g->Message, "Table path does not match the json file"); - return RC_FX; - } // endif Type - - arp = jsp->GetArray(); - objp = NULL; - i = atoi(objpath) - B; - val = arp->GetValue(i); - - if (!val) { - sprintf(g->Message, "Cannot find array value %d", i); - return RC_FX; - } // endif val - - } // endif - - jsp = val->GetJson(); - } // endfor objpath - - } // endif objpath + if ((objpath = PlugDup(g, Objname))) { + if (*objpath == '$') objpath++; + if (*objpath == '.') objpath++; + + /*********************************************************************/ + /* Find the table in the tree structure. */ + /*********************************************************************/ + for (; jsp && objpath; objpath = p) { + if ((p = strchr(objpath, Sep))) + *p++ = 0; + + if (*objpath != '[' && !IsNum(objpath)) { + // objpass is a key + if (jsp->GetType() != TYPE_JOB) { + strcpy(g->Message, "Table path does not match the json file"); + return RC_FX; + } // endif Type + + key = objpath; + objp = jsp->GetObject(); + arp = NULL; + val = objp->GetValue(key); + + if (!val || !(jsp = val->GetJson())) { + sprintf(g->Message, "Cannot find object key %s", key); + return RC_FX; + } // endif val + + } else { + if (*objpath == '[') { + // Old style + if (objpath[strlen(objpath) - 1] != ']') { + sprintf(g->Message, "Invalid Table path %s", Objname); + return RC_FX; + } else + objpath++; + + } // endif objpath + + if (jsp->GetType() != TYPE_JAR) { + strcpy(g->Message, "Table path does not match the json file"); + return RC_FX; + } // endif Type + + arp = jsp->GetArray(); + objp = NULL; + i = atoi(objpath) - B; + val = arp->GetValue(i); + + if (!val) { + sprintf(g->Message, "Cannot find array value %d", i); + return RC_FX; + } // endif val + + } // endif + + jsp = val->GetJson(); + } // endfor objpath + + } // endif objpath if (jsp && jsp->GetType() == TYPE_JAR) Doc = jsp->GetArray(); - else { + else { // The table is void or is just one object or one value Doc = new(g) JARRAY; @@ -2149,7 +2152,7 @@ int TDBJSON::MakeIndex(PGLOBAL g, PIXDEF pxdf, bool) } else return RC_OK; - } // end of MakeIndex + } // end of MakeIndex /***********************************************************************/ /* Return the position in the table. */ @@ -2227,11 +2230,11 @@ bool TDBJSON::OpenDB(PGLOBAL g) return true; } // endswitch Jmode - if (Xcol) - To_Filter = NULL; // Imcompatible + if (Xcol) + To_Filter = NULL; // Imcompatible - Use = USE_OPEN; - return false; + Use = USE_OPEN; + return false; } // end of OpenDB /***********************************************************************/ @@ -2241,7 +2244,7 @@ int TDBJSON::ReadDB(PGLOBAL) { int rc; - N++; + N++; if (NextSame) { SameRow = NextSame; @@ -2352,8 +2355,8 @@ void TDBJSON::CloseDB(PGLOBAL g) TDBJCL::TDBJCL(PJDEF tdp) : TDBCAT(tdp) { Topt = tdp->GetTopt(); - Db = tdp->Schema; - Dsn = tdp->Uri; + Db = tdp->Schema; + Dsn = tdp->Uri; } // end of TDBJCL constructor /***********************************************************************/ diff --git a/storage/connect/tabrest.cpp b/storage/connect/tabrest.cpp index e800411a36d..eef90f0dc1d 100644 --- a/storage/connect/tabrest.cpp +++ b/storage/connect/tabrest.cpp @@ -1,13 +1,23 @@ /*************** Rest C++ Program Source Code File (.CPP) **************/ -/* PROGRAM NAME: Rest Version 1.3 */ +/* PROGRAM NAME: Rest Version 1.5 */ /* (C) Copyright to the author Olivier BERTRAND 2018 - 2019 */ -/* This program is the REST OEM (Web API support) module definition. */ +/* This program is the REST Web API support for MariaDB. */ +/* When compiled without MARIADB defined, it is the EOM module code. */ /***********************************************************************/ /***********************************************************************/ /* Definitions needed by the included files. */ /***********************************************************************/ +#if defined(MARIADB) #include // All MariaDB stuff +#else // !MARIADB OEM module +#include "mini-global.h" +#define _MAX_PATH 260 +#if !defined(__WIN__) +#define __stdcall +#endif // !__WIN__ +#define _OS_H_INCLUDED // Prevent os.h to be called +#endif // !MARIADB /***********************************************************************/ /* Include application header files: */ @@ -19,12 +29,12 @@ #include "plgdbsem.h" #include "xtable.h" #include "filamtxt.h" -#include "plgxml.h" #include "tabdos.h" -#include "tabfmt.h" +#include "plgxml.h" +#include "tabxml.h" #include "tabjson.h" +#include "tabfmt.h" #include "tabrest.h" -#include "tabxml.h" /***********************************************************************/ /* Get the file from the Web. */ @@ -32,25 +42,61 @@ int restGetFile(PGLOBAL g, PCSZ http, PCSZ uri, PCSZ fn); #if defined(__WIN__) -static PCSZ slash= "\\"; +static PCSZ slash = "\\"; #else // !__WIN__ -static PCSZ slash= "/"; +static PCSZ slash = "/"; #define stricmp strcasecmp #endif // !__WIN__ +#if !defined(MARIADB) +/***********************************************************************/ +/* DB static variables. */ +/***********************************************************************/ +int TDB::Tnum; +int DTVAL::Shift; +int CSORT::Limit = 0; +double CSORT::Lg2 = log(2.0); +size_t CSORT::Cpn[1000] = { 0 }; + +/***********************************************************************/ +/* These functions are exported from the REST library. */ +/***********************************************************************/ +extern "C" { + PTABDEF __stdcall GetREST(PGLOBAL, void*); + PQRYRES __stdcall ColREST(PGLOBAL, PTOS, char*, char*, bool); +} // extern "C" + +/***********************************************************************/ +/* This function returns a table definition class. */ +/***********************************************************************/ +PTABDEF __stdcall GetREST(PGLOBAL g, void *memp) +{ + return new(g, memp) RESTDEF; +} // end of GetREST +#endif // !MARIADB + /***********************************************************************/ /* Return the columns definition to MariaDB. */ /***********************************************************************/ +#if defined(MARIADB) PQRYRES RESTColumns(PGLOBAL g, PTOS tp, char *tab, char *db, bool info) +#else // !MARIADB +PQRYRES __stdcall ColREST(PGLOBAL g, PTOS tp, char *tab, char *db, bool info) +#endif // !MARIADB { PQRYRES qrp= NULL; - char filename[_MAX_PATH]; + char filename[_MAX_PATH + 1]; // MAX PATH ??? PCSZ http, uri, fn, ftype; - http= GetStringTableOption(g, tp, "Http", NULL); - uri= GetStringTableOption(g, tp, "Uri", NULL); - fn= GetStringTableOption(g, tp, "Filename", "rest.json"); + http = GetStringTableOption(g, tp, "Http", NULL); + uri = GetStringTableOption(g, tp, "Uri", NULL); + fn = GetStringTableOption(g, tp, "Filename", "rest.json"); +#if defined(MARIADB) ftype = GetStringTableOption(g, tp, "Type", "JSON"); +#else // !MARIADB + // OEM tables must specify the file type + ftype = GetStringTableOption(g, tp, "Ftype", "JSON"); +#endif // !MARIADB // We used the file name relative to recorded datapath strcat(strcat(strcat(strcpy(filename, "."), slash), db), slash); @@ -60,11 +106,11 @@ PQRYRES RESTColumns(PGLOBAL g, PTOS tp, char *tab, char *db, bool info) if (http && restGetFile(g, http, uri, filename)) { // sprintf(g->Message, "Failed to get file at %s", http); } else if (!stricmp(ftype, "XML")) - qrp= XMLColumns(g, db, tab, tp, info); + qrp = XMLColumns(g, db, tab, tp, info); else if (!stricmp(ftype, "JSON")) - qrp= JSONColumns(g, db, NULL, tp, info); + qrp = JSONColumns(g, db, NULL, tp, info); else if (!stricmp(ftype, "CSV")) - qrp= CSVColumns(g, NULL, tp, info); + qrp = CSVColumns(g, NULL, tp, info); else sprintf(g->Message, "Usupported file type %s", ftype); @@ -78,40 +124,59 @@ PQRYRES RESTColumns(PGLOBAL g, PTOS tp, char *tab, char *db, bool info) /***********************************************************************/ bool RESTDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) { - char filename[_MAX_PATH]; - TABTYPE type= GetTypeID(am); - - switch (type) { - case TAB_JSON: - case TAB_XML: - case TAB_CSV: - break; - default: - sprintf(g->Message, "Unsupported REST table type %s", am); - return true; - } // endswitch type - - Http= GetStringCatInfo(g, "Http", NULL); - Uri= GetStringCatInfo(g, "Uri", NULL); - Fn= GetStringCatInfo(g, "Filename", "rest.json"); + char filename[_MAX_PATH + 1]; + int rc = 0, n; + LPCSTR ftype; + +#if defined(MARIADB) + ftype = GetStringCatInfo(g, "Type", "JSON"); +#else // !MARIADB + // OEM tables must specify the file type + ftype = GetStringCatInfo(g, "Ftype", "JSON"); +#endif // !MARIADB + + if (trace(1)) + htrc("ftype = %s am = %s\n", ftype, SVP(am)); + + n = (!stricmp(ftype, "JSON")) ? 1 + : (!stricmp(ftype, "XML")) ? 2 + : (!stricmp(ftype, "CSV")) ? 3 : 0; + + if (n == 0) { + htrc("DefineAM: Unsupported REST table type %s", am); + sprintf(g->Message, "Unsupported REST table type %s", am); + return true; + } // endif n + + Http = GetStringCatInfo(g, "Http", NULL); + Uri = GetStringCatInfo(g, "Uri", NULL); + Fn = GetStringCatInfo(g, "Filename", "rest.json"); // We used the file name relative to recorded datapath - PlugSetPath(filename, Fn, GetPath()); + //PlugSetPath(filename, Fn, GetPath()); + strncat(strcpy(filename, GetPath()), Fn, _MAX_PATH); // Retrieve the file from the web and copy it locally - if (Http && restGetFile(g, Http, Uri, filename)) {} - else if (type == TAB_JSON) - Tdp= new (g) JSONDEF; - else if (type == TAB_XML) - Tdp= new (g) XMLDEF; - else if (type == TAB_CSV) - Tdp= new (g) CSVDEF; - else - sprintf(g->Message, "Unsupported REST table type %s", am); + rc = restGetFile(g, Http, Uri, filename); + + if (trace(1)) + htrc("Return from restGetFile: rc=%d\n", rc); + + if (rc) + return true; + else switch (n) { + case 1: Tdp = new (g) JSONDEF; break; + case 2: Tdp = new (g) XMLDEF; break; + case 3: Tdp = new (g) CSVDEF; break; + default: Tdp = NULL; + } // endswitch n // Do make the table/view definition if (Tdp && Tdp->Define(g, Cat, Name, Schema, "REST")) - Tdp= NULL; // Error occured + Tdp = NULL; // Error occured + + if (trace(1)) + htrc("Tdp defined\n", rc); // Return true in case of error return (Tdp == NULL); @@ -122,10 +187,12 @@ bool RESTDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) /***********************************************************************/ PTDB RESTDEF::GetTable(PGLOBAL g, MODE m) { - if (m != MODE_READ && m != MODE_READX) { - strcpy(g->Message, "REST tables are currently read only"); - return NULL; - } // endif m + xtrc(515, "REST GetTable mode=%d\n", m); + + if (m != MODE_READ && m != MODE_READX) { + strcpy(g->Message, "REST tables are currently read only"); + return NULL; + } // endif m return Tdp->GetTable(g, m); // Leave file type do the job } // end of GetTable -- cgit v1.2.1 From 44150a770fd861c3837df34357c794432baf8882 Mon Sep 17 00:00:00 2001 From: Monty Date: Sun, 18 Aug 2019 13:18:09 +0300 Subject: Updated result for rocksdb tests after merge --- storage/rocksdb/mysql-test/rocksdb/r/bulk_load.result | 1 + storage/rocksdb/mysql-test/rocksdb/r/bulk_load_rev_cf.result | 1 + storage/rocksdb/mysql-test/rocksdb/r/bulk_load_rev_cf_and_data.result | 1 + storage/rocksdb/mysql-test/rocksdb/r/bulk_load_rev_data.result | 1 + 4 files changed, 4 insertions(+) (limited to 'storage') diff --git a/storage/rocksdb/mysql-test/rocksdb/r/bulk_load.result b/storage/rocksdb/mysql-test/rocksdb/r/bulk_load.result index d37c8097622..889207040b6 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/bulk_load.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/bulk_load.result @@ -26,6 +26,7 @@ start transaction with consistent snapshot; select VALUE > 0 as 'Has opened snapshots' from information_schema.rocksdb_dbstats where stat_type='DB_NUM_SNAPSHOTS'; Has opened snapshots 1 +connection default; SET @@GLOBAL.ROCKSDB_UPDATE_CF_OPTIONS= 'cf1={write_buffer_size=8m;target_file_size_base=1m};'; set rocksdb_bulk_load=1; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/bulk_load_rev_cf.result b/storage/rocksdb/mysql-test/rocksdb/r/bulk_load_rev_cf.result index 270d372f343..2cf2bdb91a6 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/bulk_load_rev_cf.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/bulk_load_rev_cf.result @@ -26,6 +26,7 @@ start transaction with consistent snapshot; select VALUE > 0 as 'Has opened snapshots' from information_schema.rocksdb_dbstats where stat_type='DB_NUM_SNAPSHOTS'; Has opened snapshots 1 +connection default; SET @@GLOBAL.ROCKSDB_UPDATE_CF_OPTIONS= 'cf1={write_buffer_size=8m;target_file_size_base=1m};'; set rocksdb_bulk_load=1; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/bulk_load_rev_cf_and_data.result b/storage/rocksdb/mysql-test/rocksdb/r/bulk_load_rev_cf_and_data.result index 2fe9c551fb0..1d2497dc7e4 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/bulk_load_rev_cf_and_data.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/bulk_load_rev_cf_and_data.result @@ -26,6 +26,7 @@ start transaction with consistent snapshot; select VALUE > 0 as 'Has opened snapshots' from information_schema.rocksdb_dbstats where stat_type='DB_NUM_SNAPSHOTS'; Has opened snapshots 1 +connection default; SET @@GLOBAL.ROCKSDB_UPDATE_CF_OPTIONS= 'cf1={write_buffer_size=8m;target_file_size_base=1m};'; set rocksdb_bulk_load=1; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/bulk_load_rev_data.result b/storage/rocksdb/mysql-test/rocksdb/r/bulk_load_rev_data.result index e534be48921..8b5576b8f5f 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/bulk_load_rev_data.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/bulk_load_rev_data.result @@ -26,6 +26,7 @@ start transaction with consistent snapshot; select VALUE > 0 as 'Has opened snapshots' from information_schema.rocksdb_dbstats where stat_type='DB_NUM_SNAPSHOTS'; Has opened snapshots 1 +connection default; SET @@GLOBAL.ROCKSDB_UPDATE_CF_OPTIONS= 'cf1={write_buffer_size=8m;target_file_size_base=1m};'; set rocksdb_bulk_load=1; -- cgit v1.2.1 From 6dd3f24090ce2d237037eb09cf7db083ebbc92f9 Mon Sep 17 00:00:00 2001 From: Aleksey Midenkov Date: Sun, 18 Aug 2019 23:18:44 +0300 Subject: MDEV-19740 Debug build of 10.3.15 FTBFS * Replace LINT_INIT for non-struct types with ctor initializers; * Check BUILD_DEPS list is not empty so REMOVE_DUPLICATES won't throw error. --- storage/innobase/fil/fil0crypt.cc | 2 ++ storage/innobase/row/row0mysql.cc | 4 ++++ storage/maria/ma_ft_nlq_search.c | 2 +- storage/myisam/ft_nlq_search.c | 2 +- 4 files changed, 8 insertions(+), 2 deletions(-) (limited to 'storage') diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 5df38391e3d..35b70501002 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -2268,9 +2268,11 @@ static void fil_crypt_rotation_list_fill() /* Protect the tablespace while we may release fil_system.mutex. */ space->n_pending_ops++; +#ifndef DBUG_OFF fil_space_t* s= fil_system.read_page0( space->id); ut_ad(!s || s == space); +#endif space->n_pending_ops--; if (!space->size) { /* Page 0 was not loaded. diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index aed2f87349d..43808f157d5 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -1418,11 +1418,13 @@ row_insert_for_mysql( &blob_heap); if (ins_mode != ROW_INS_NORMAL) { +#ifndef DBUG_OFF ut_ad(table->vers_start != table->vers_end); const mysql_row_templ_t* t = prebuilt->get_template_by_col(table->vers_end); ut_ad(t); ut_ad(t->mysql_col_len == 8); +#endif if (ins_mode == ROW_INS_HISTORICAL) { set_tuple_col_8(node->row, table->vers_end, trx->id, @@ -1430,9 +1432,11 @@ row_insert_for_mysql( } else /* ROW_INS_VERSIONED */ { set_tuple_col_8(node->row, table->vers_end, TRX_ID_MAX, node->vers_end_buf); +#ifndef DBUG_OFF t = prebuilt->get_template_by_col(table->vers_start); ut_ad(t); ut_ad(t->mysql_col_len == 8); +#endif set_tuple_col_8(node->row, table->vers_start, trx->id, node->vers_start_buf); } diff --git a/storage/maria/ma_ft_nlq_search.c b/storage/maria/ma_ft_nlq_search.c index 7becf1d7136..f7aa3afec9a 100644 --- a/storage/maria/ma_ft_nlq_search.c +++ b/storage/maria/ma_ft_nlq_search.c @@ -79,13 +79,13 @@ static int walk_and_match(FT_WORD *word, uint32 count, ALL_IN_ONE *aio) MARIA_KEY key; float tmp_weight; DBUG_ENTER("walk_and_match"); - LINT_INIT_STRUCT(subkeys); word->weight=LWS_FOR_QUERY; _ma_ft_make_key(info, &key, aio->keynr, keybuff, word, 0); key.data_length-= HA_FT_WLEN; doc_cnt=0; + subkeys.i= 0; if (share->lock_key_trees) mysql_rwlock_rdlock(&share->keyinfo[aio->keynr].root_lock); diff --git a/storage/myisam/ft_nlq_search.c b/storage/myisam/ft_nlq_search.c index 6c36c327d81..3e433b71761 100644 --- a/storage/myisam/ft_nlq_search.c +++ b/storage/myisam/ft_nlq_search.c @@ -77,13 +77,13 @@ static int walk_and_match(FT_WORD *word, uint32 count, ALL_IN_ONE *aio) uint extra= HA_FT_WLEN + info->s->rec_reflength; float tmp_weight; DBUG_ENTER("walk_and_match"); - LINT_INIT_STRUCT(subkeys); word->weight=LWS_FOR_QUERY; keylen=_ft_make_key(info,aio->keynr,keybuff,word,0); keylen-=HA_FT_WLEN; doc_cnt=0; + subkeys.i= 0; if (share->concurrent_insert) mysql_rwlock_rdlock(&share->key_root_lock[aio->keynr]); -- cgit v1.2.1 From fcae2a6364abba5eccfe85e65b109a88a978f7cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 19 Aug 2019 16:29:36 +0300 Subject: MDEV-20383 Use of uninitialized value in Datafile::find_space_id() for ROW_FORMAT=COMPRESSED Datafile::find_space_id(): Fix a regression that was introduced in c0f47a4a58424c621204dacb8016a94b66cb2bce for MDEV-12026. Because the function buf_page_is_corrupted() now determines the physical page size from the fsp_flags, our buffer size must agree with the fsp_flags. buf_page_is_corrupted(): Use the correct accessor fil_space_t::zip_size() for convering the tablespace flags. ROW_FORMAT=COMPRESSED files never use innodb_checksum_algorithm=full_crc32. --- storage/innobase/buf/buf0buf.cc | 8 +------- storage/innobase/fsp/fsp0file.cc | 5 +++-- 2 files changed, 4 insertions(+), 9 deletions(-) (limited to 'storage') diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 86b669209cb..d9af896e5c0 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -1068,14 +1068,8 @@ buf_page_is_corrupted( size_t checksum_field2 = 0; uint32_t crc32 = 0; bool crc32_inited = false; - ulint zip_size = 0; bool crc32_chksum = false; - - zip_size = FSP_FLAGS_GET_ZIP_SSIZE(fsp_flags); - if (zip_size) { - zip_size = (UNIV_ZIP_SIZE_MIN >> 1) << zip_size; - } - + const ulint zip_size = fil_space_t::zip_size(fsp_flags); ulint page_type = mach_read_from_2(read_buf + FIL_PAGE_TYPE); /* We can trust page type if page compression is set on tablespace diff --git a/storage/innobase/fsp/fsp0file.cc b/storage/innobase/fsp/fsp0file.cc index 4143e246f99..4869160b883 100644 --- a/storage/innobase/fsp/fsp0file.cc +++ b/storage/innobase/fsp/fsp0file.cc @@ -699,7 +699,8 @@ Datafile::find_space_id() /* For noncompressed pages, the page size must be equal to srv_page_size. */ - if (page_size == srv_page_size) { + if (page_size == srv_page_size + && !fil_space_t::zip_size(fsp_flags)) { noncompressed_ok = !buf_page_is_corrupted( false, page, fsp_flags); } @@ -707,7 +708,7 @@ Datafile::find_space_id() bool compressed_ok = false; if (srv_page_size <= UNIV_PAGE_SIZE_DEF - && page_size <= srv_page_size) { + && page_size == fil_space_t::zip_size(fsp_flags)) { compressed_ok = !buf_page_is_corrupted( false, page, fsp_flags); } -- cgit v1.2.1 From c0f9042500b5034b3f6fff96321dfa9eeb36957d Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Mon, 19 Aug 2019 18:06:34 +0200 Subject: Some small changes. =================== - Modify tracing to use htrc to be compatible with old versions when this code is used to make an EOM module. modified: storage/connect/restget.cpp modified: storage/connect/tabrest.cpp - Path apparently not needed for the cpprest lib on Linux modified: storage/connect/CMakeLists.txt --- storage/connect/CMakeLists.txt | 4 ++-- storage/connect/restget.cpp | 25 ++++++++++++++---------- storage/connect/tabrest.cpp | 43 +++++++++++++++++++++--------------------- 3 files changed, 39 insertions(+), 33 deletions(-) (limited to 'storage') diff --git a/storage/connect/CMakeLists.txt b/storage/connect/CMakeLists.txt index 6fd3b5e3018..fd006e104c1 100644 --- a/storage/connect/CMakeLists.txt +++ b/storage/connect/CMakeLists.txt @@ -321,8 +321,8 @@ IF(CONNECT_WITH_REST) MESSAGE(STATUS "=====> cpprestsdk found") IF(UNIX) # INCLUDE_DIRECTORIES(${CPPRESTSDK_INCLUDE_DIR}) -# Edit next line to set the path to libcpprest.so - SET(REST_LIBRARY -L/usr/lib64 -lcpprest) +# If needed edit next line to set the path to libcpprest.so + SET(REST_LIBRARY -lcpprest) MESSAGE (STATUS ${REST_LIBRARY}) ENDIF(UNIX) SET(CONNECT_SOURCES ${CONNECT_SOURCES} tabrest.cpp restget.cpp tabrest.h) diff --git a/storage/connect/restget.cpp b/storage/connect/restget.cpp index d5ceb7b8a39..6b184ae6926 100644 --- a/storage/connect/restget.cpp +++ b/storage/connect/restget.cpp @@ -19,14 +19,13 @@ using namespace concurrency::streams; // Asynchronous streams #include "global.h" -static uint xt = 0; // Used by lamda expressions - /***********************************************************************/ /* Make a local copy of the requested file. */ /***********************************************************************/ int restGetFile(PGLOBAL g, PCSZ http, PCSZ uri, PCSZ fn) { int rc = 0; + bool xt = trace(515); auto fileStream = std::make_shared(); if (!http || !fn) { @@ -34,15 +33,15 @@ int restGetFile(PGLOBAL g, PCSZ http, PCSZ uri, PCSZ fn) return 2; } // endif - xt = GetTraceValue(); - xtrc(515, "restGetFile: fn=%s\n", fn); + if (xt) + htrc("restGetFile: fn=%s\n", fn); // Open stream to output file. pplx::task requestTask = fstream::open_ostream(to_string_t(fn)) .then([=](ostream outFile) { *fileStream= outFile; - if (xt & 515) + if (xt) htrc("Outfile isopen=%d\n", outFile.is_open()); // Create http_client to send the request. @@ -58,7 +57,7 @@ int restGetFile(PGLOBAL g, PCSZ http, PCSZ uri, PCSZ fn) // Handle response headers arriving. .then([=](http_response response) { - if (xt & 515) + if (xt) htrc("Received response status code:%u\n", response.status_code()); @@ -68,7 +67,7 @@ int restGetFile(PGLOBAL g, PCSZ http, PCSZ uri, PCSZ fn) // Close the file stream. .then([=](size_t n) { - if (xt & 515) + if (xt) htrc("Return size=%u\n", n); return fileStream->close(); @@ -77,13 +76,19 @@ int restGetFile(PGLOBAL g, PCSZ http, PCSZ uri, PCSZ fn) // Wait for all the outstanding I/O to complete and handle any exceptions try { requestTask.wait(); - xtrc(515, "In Wait\n"); + + if (xt) + htrc("In Wait\n"); + } catch (const std::exception &e) { - xtrc(515, "Error exception: %s\n", e.what()); + if (xt) + htrc("Error exception: %s\n", e.what()); sprintf(g->Message, "Error exception: %s", e.what()); rc= 1; } // end try/catch - xtrc(515, "restget done: rc=%d\n", rc); + if (xt) + htrc("restget done: rc=%d\n", rc); + return rc; } // end of restGetFile diff --git a/storage/connect/tabrest.cpp b/storage/connect/tabrest.cpp index eef90f0dc1d..9e1a643c89f 100644 --- a/storage/connect/tabrest.cpp +++ b/storage/connect/tabrest.cpp @@ -92,10 +92,10 @@ PQRYRES __stdcall ColREST(PGLOBAL g, PTOS tp, char *tab, char *db, bool info) uri = GetStringTableOption(g, tp, "Uri", NULL); fn = GetStringTableOption(g, tp, "Filename", "rest.json"); #if defined(MARIADB) - ftype = GetStringTableOption(g, tp, "Type", "JSON"); + ftype = GetStringTableOption(g, tp, "Type", "JSON"); #else // !MARIADB - // OEM tables must specify the file type - ftype = GetStringTableOption(g, tp, "Ftype", "JSON"); + // OEM tables must specify the file type + ftype = GetStringTableOption(g, tp, "Ftype", "JSON"); #endif // !MARIADB // We used the file name relative to recorded datapath @@ -126,21 +126,21 @@ bool RESTDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) { char filename[_MAX_PATH + 1]; int rc = 0, n; - LPCSTR ftype; + LPCSTR ftype; #if defined(MARIADB) - ftype = GetStringCatInfo(g, "Type", "JSON"); + ftype = GetStringCatInfo(g, "Type", "JSON"); #else // !MARIADB // OEM tables must specify the file type ftype = GetStringCatInfo(g, "Ftype", "JSON"); #endif // !MARIADB - if (trace(1)) - htrc("ftype = %s am = %s\n", ftype, SVP(am)); + if (trace(515)) + htrc("ftype = %s am = %s\n", ftype, SVP(am)); - n = (!stricmp(ftype, "JSON")) ? 1 - : (!stricmp(ftype, "XML")) ? 2 - : (!stricmp(ftype, "CSV")) ? 3 : 0; + n = (!stricmp(ftype, "JSON")) ? 1 + : (!stricmp(ftype, "XML")) ? 2 + : (!stricmp(ftype, "CSV")) ? 3 : 0; if (n == 0) { htrc("DefineAM: Unsupported REST table type %s", am); @@ -159,24 +159,24 @@ bool RESTDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) // Retrieve the file from the web and copy it locally rc = restGetFile(g, Http, Uri, filename); - if (trace(1)) - htrc("Return from restGetFile: rc=%d\n", rc); + if (trace(515)) + htrc("Return from restGetFile: rc=%d\n", rc); if (rc) return true; - else switch (n) { - case 1: Tdp = new (g) JSONDEF; break; - case 2: Tdp = new (g) XMLDEF; break; - case 3: Tdp = new (g) CSVDEF; break; - default: Tdp = NULL; - } // endswitch n + else switch (n) { + case 1: Tdp = new (g) JSONDEF; break; + case 2: Tdp = new (g) XMLDEF; break; + case 3: Tdp = new (g) CSVDEF; break; + default: Tdp = NULL; + } // endswitch n // Do make the table/view definition if (Tdp && Tdp->Define(g, Cat, Name, Schema, "REST")) Tdp = NULL; // Error occured - if (trace(1)) - htrc("Tdp defined\n", rc); + if (trace(515)) + htrc("Tdp defined\n", rc); // Return true in case of error return (Tdp == NULL); @@ -187,7 +187,8 @@ bool RESTDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) /***********************************************************************/ PTDB RESTDEF::GetTable(PGLOBAL g, MODE m) { - xtrc(515, "REST GetTable mode=%d\n", m); + if (trace(515)) + htrc("REST GetTable mode=%d\n", m); if (m != MODE_READ && m != MODE_READX) { strcpy(g->Message, "REST tables are currently read only"); -- cgit v1.2.1 From 262927a9e5e49d57332e2123d667a33c5faa3f1a Mon Sep 17 00:00:00 2001 From: Monty Date: Tue, 20 Aug 2019 12:57:52 +0300 Subject: Fixes a problem with heap when scanning and insert rows at the same time This causes failures in versioning.update-big in 10.3 and above --- storage/heap/hp_scan.c | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) (limited to 'storage') diff --git a/storage/heap/hp_scan.c b/storage/heap/hp_scan.c index 3315cb05b3f..361a1cd7397 100644 --- a/storage/heap/hp_scan.c +++ b/storage/heap/hp_scan.c @@ -44,6 +44,12 @@ int heap_scan(register HP_INFO *info, uchar *record) DBUG_ENTER("heap_scan"); pos= ++info->current_record; + if (pos >= share->records+share->deleted) + { + info->update= 0; /* No active row */ + DBUG_RETURN(my_errno= HA_ERR_END_OF_FILE); + } + if (pos < info->next_block) { info->current_ptr+=share->block.recbuffer; @@ -51,15 +57,6 @@ int heap_scan(register HP_INFO *info, uchar *record) else { info->next_block+=share->block.records_in_block; - if (info->next_block >= share->records+share->deleted) - { - info->next_block= share->records+share->deleted; - if (pos >= info->next_block) - { - info->update= 0; - DBUG_RETURN(my_errno= HA_ERR_END_OF_FILE); - } - } hp_find_record(info, pos); } if (!info->current_ptr[share->visible]) -- cgit v1.2.1 From 6dd1d6cb90ce942de0a3648b2440d76e376bb013 Mon Sep 17 00:00:00 2001 From: Monty Date: Tue, 20 Aug 2019 15:28:01 +0300 Subject: MDEV-20379 Mroonga has memory leak in ha_mroonga::is_foreign_key_field --- storage/mroonga/ha_mroonga.cpp | 3 +++ 1 file changed, 3 insertions(+) (limited to 'storage') diff --git a/storage/mroonga/ha_mroonga.cpp b/storage/mroonga/ha_mroonga.cpp index f7533fd4600..06ff71c1b5a 100644 --- a/storage/mroonga/ha_mroonga.cpp +++ b/storage/mroonga/ha_mroonga.cpp @@ -8994,10 +8994,12 @@ bool ha_mroonga::is_foreign_key_field(const char *table_name, grn_obj *range = grn_ctx_at(ctx, grn_obj_get_range(ctx, column)); if (!range) { + grn_obj_unlink(ctx, column); DBUG_RETURN(false); } if (!mrn::grn::is_table(range)) { + grn_obj_unlink(ctx, column); DBUG_RETURN(false); } @@ -9011,6 +9013,7 @@ bool ha_mroonga::is_foreign_key_field(const char *table_name, DBUG_RETURN(true); } + grn_obj_unlink(ctx, column); DBUG_RETURN(false); } -- cgit v1.2.1 From 1ad70bf2fe380354ae1719d91644f2ec335e4b12 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 20 Aug 2019 13:06:53 +0200 Subject: Revert "Fixes a problem with heap when scanning and insert rows at the same time" This reverts commit 262927a9e5e49d57332e2123d667a33c5faa3f1a. --- storage/heap/hp_scan.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) (limited to 'storage') diff --git a/storage/heap/hp_scan.c b/storage/heap/hp_scan.c index 361a1cd7397..3315cb05b3f 100644 --- a/storage/heap/hp_scan.c +++ b/storage/heap/hp_scan.c @@ -44,12 +44,6 @@ int heap_scan(register HP_INFO *info, uchar *record) DBUG_ENTER("heap_scan"); pos= ++info->current_record; - if (pos >= share->records+share->deleted) - { - info->update= 0; /* No active row */ - DBUG_RETURN(my_errno= HA_ERR_END_OF_FILE); - } - if (pos < info->next_block) { info->current_ptr+=share->block.recbuffer; @@ -57,6 +51,15 @@ int heap_scan(register HP_INFO *info, uchar *record) else { info->next_block+=share->block.records_in_block; + if (info->next_block >= share->records+share->deleted) + { + info->next_block= share->records+share->deleted; + if (pos >= info->next_block) + { + info->update= 0; + DBUG_RETURN(my_errno= HA_ERR_END_OF_FILE); + } + } hp_find_record(info, pos); } if (!info->current_ptr[share->visible]) -- cgit v1.2.1 From 6c06defb5f6ddccba1a3d7b03ba34dbb83c7ef69 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 19 Aug 2019 14:17:38 +0200 Subject: MDEV-15458 Segfault in heap_scan() upon UPDATE after ADD SYSTEM VERSIONING heap_scan() makes info->next_block to be either an integer number of share->block.records_in_block's or the total number of records in the table. So when this total number or records changes, info->next_block needs to be recalculated to take it into account. This is a different fix for "Fixes a problem with heap when scanning and insert rows at the same time" --- storage/heap/hp_scan.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'storage') diff --git a/storage/heap/hp_scan.c b/storage/heap/hp_scan.c index 3315cb05b3f..f07efe6cf67 100644 --- a/storage/heap/hp_scan.c +++ b/storage/heap/hp_scan.c @@ -50,7 +50,9 @@ int heap_scan(register HP_INFO *info, uchar *record) } else { - info->next_block+=share->block.records_in_block; + /* increase next_block to the next records_in_block boundary */ + ulong rem= info->next_block % share->block.records_in_block; + info->next_block+=share->block.records_in_block - rem; if (info->next_block >= share->records+share->deleted) { info->next_block= share->records+share->deleted; -- cgit v1.2.1 From ddaebdd21006a6ef418a161bce919c5e850109dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 20 Aug 2019 14:55:49 +0300 Subject: dict_table_open_on_index_id(): Remove a redundant parameter --- storage/innobase/btr/btr0scrub.cc | 7 +------ storage/innobase/dict/dict0load.cc | 19 ++----------------- storage/innobase/include/dict0dict.h | 7 +------ 3 files changed, 4 insertions(+), 29 deletions(-) (limited to 'storage') diff --git a/storage/innobase/btr/btr0scrub.cc b/storage/innobase/btr/btr0scrub.cc index 376a106bf8a..bb1e5276a12 100644 --- a/storage/innobase/btr/btr0scrub.cc +++ b/storage/innobase/btr/btr0scrub.cc @@ -632,13 +632,8 @@ btr_scrub_get_table_and_index( scrub_data->current_table = NULL; } - /* argument to dict_table_open_on_index_id */ - bool dict_locked = true; - /* open table based on index_id */ - dict_table_t* table = dict_table_open_on_index_id( - index_id, - dict_locked); + dict_table_t* table = dict_table_open_on_index_id(index_id); if (table != NULL) { /* mark table as being scrubbed */ diff --git a/storage/innobase/dict/dict0load.cc b/storage/innobase/dict/dict0load.cc index ae5bbac9118..a8ed8dd3e29 100644 --- a/storage/innobase/dict/dict0load.cc +++ b/storage/innobase/dict/dict0load.cc @@ -3810,29 +3810,14 @@ dict_load_table_id_on_index_id( return(found); } -UNIV_INTERN -dict_table_t* -dict_table_open_on_index_id( -/*========================*/ - index_id_t index_id, /*!< in: index id */ - bool dict_locked) /*!< in: dict locked */ +dict_table_t* dict_table_open_on_index_id(index_id_t index_id) { - if (!dict_locked) { - mutex_enter(&dict_sys->mutex); - } - - ut_ad(mutex_own(&dict_sys->mutex)); table_id_t table_id; dict_table_t * table = NULL; if (dict_load_table_id_on_index_id(index_id, &table_id)) { - bool local_dict_locked = true; - table = dict_table_open_on_id(table_id, - local_dict_locked, + table = dict_table_open_on_id(table_id, true, DICT_TABLE_OP_LOAD_TABLESPACE); } - if (!dict_locked) { - mutex_exit(&dict_sys->mutex); - } return table; } diff --git a/storage/innobase/include/dict0dict.h b/storage/innobase/include/dict0dict.h index cdcdef75917..fbaef118b70 100644 --- a/storage/innobase/include/dict0dict.h +++ b/storage/innobase/include/dict0dict.h @@ -115,12 +115,7 @@ dict_table_open_on_id( /**********************************************************************//** Returns a table object based on table id. @return table, NULL if does not exist */ -UNIV_INTERN -dict_table_t* -dict_table_open_on_index_id( -/*==================*/ - table_id_t table_id, /*!< in: table id */ - bool dict_locked) /*!< in: TRUE=data dictionary locked */ +dict_table_t* dict_table_open_on_index_id(index_id_t index_id) __attribute__((warn_unused_result)); /********************************************************************//** Decrements the count of open handles to a table. */ -- cgit v1.2.1 From e279c0076d59fdc840931b3bfa497b38893adf93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 20 Aug 2019 12:52:57 +0300 Subject: MDEV-17187: Code cleanup fkerr_t: Errors for the foreign key checks. Replaces ulint, which used #define that looked like dberr_t literals. wsrep_dict_foreign_find_index(): Remove. Use dict_foreign_find_index() instead, with default parameters. dict_foreign_push_index_error(): Do not add redundant quotes around quoted table names. --- storage/innobase/dict/dict0dict.cc | 104 ++++++++++------------------------ storage/innobase/handler/ha_innodb.cc | 15 +---- storage/innobase/include/dict0dict.h | 23 ++++++-- 3 files changed, 51 insertions(+), 91 deletions(-) (limited to 'storage') diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 3fcc1bce177..3e5e8ca2fec 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -3211,11 +3211,6 @@ dict_index_build_internal_fts( } /*====================== FOREIGN KEY PROCESSING ========================*/ -#define DB_FOREIGN_KEY_IS_PREFIX_INDEX 200 -#define DB_FOREIGN_KEY_COL_NOT_NULL 201 -#define DB_FOREIGN_KEY_COLS_NOT_EQUAL 202 -#define DB_FOREIGN_KEY_INDEX_NOT_FOUND 203 - /** Check whether the dict_table_t is a partition. A partitioned table on the SQL level is composed of InnoDB tables, where each InnoDB table is a [sub]partition including its secondary indexes @@ -3322,7 +3317,7 @@ dict_foreign_find_index( /*!< in: nonzero if none of the columns must be declared NOT NULL */ - ulint* error, /*!< out: error code */ + fkerr_t* error, /*!< out: error code */ ulint* err_col_no, /*!< out: column number where error happened */ @@ -3330,17 +3325,15 @@ dict_foreign_find_index( /*!< out: index where error happened */ { - dict_index_t* index; - ut_ad(mutex_own(&dict_sys->mutex)); if (error) { - *error = DB_FOREIGN_KEY_INDEX_NOT_FOUND; + *error = FK_INDEX_NOT_FOUND; } - index = dict_table_get_first_index(table); - - while (index != NULL) { + for (dict_index_t* index = dict_table_get_first_index(table); + index; + index = dict_table_get_next_index(index)) { if (types_idx != index && !index->to_be_dropped && !dict_index_is_online_ddl(index) @@ -3348,42 +3341,17 @@ dict_foreign_find_index( table, col_names, columns, n_cols, index, types_idx, check_charsets, check_null, - error, err_col_no,err_index)) { + error, err_col_no, err_index)) { if (error) { - *error = DB_SUCCESS; + *error = FK_SUCCESS; } return(index); } - - index = dict_table_get_next_index(index); } return(NULL); } -#ifdef WITH_WSREP -dict_index_t* -wsrep_dict_foreign_find_index( -/*====================*/ - dict_table_t* table, /*!< in: table */ - const char** col_names, /*!< in: column names, or NULL - to use table->col_names */ - const char** columns,/*!< in: array of column names */ - ulint n_cols, /*!< in: number of columns */ - dict_index_t* types_idx, /*!< in: NULL or an index to whose types the - column types must match */ - ibool check_charsets, - /*!< in: whether to check charsets. - only has an effect if types_idx != NULL */ - ulint check_null) - /*!< in: nonzero if none of the columns must - be declared NOT NULL */ -{ - return dict_foreign_find_index( - table, col_names, columns, n_cols, types_idx, check_charsets, - check_null, NULL, NULL, NULL); -} -#endif /* WITH_WSREP */ /**********************************************************************//** Report an error in a foreign key definition. */ static @@ -3480,15 +3448,11 @@ dict_foreign_add_to_cache( } if (ref_table && !for_in_cache->referenced_table) { - ulint index_error; - ulint err_col; - dict_index_t *err_index=NULL; - index = dict_foreign_find_index( ref_table, NULL, for_in_cache->referenced_col_names, for_in_cache->n_fields, for_in_cache->foreign_index, - check_charsets, false, &index_error, &err_col, &err_index); + check_charsets, false); if (index == NULL && !(ignore_err & DICT_ERR_IGNORE_FK_NOKEY)) { @@ -3520,10 +3484,6 @@ dict_foreign_add_to_cache( } if (for_table && !for_in_cache->foreign_table) { - ulint index_error; - ulint err_col; - dict_index_t *err_index=NULL; - index = dict_foreign_find_index( for_table, col_names, for_in_cache->foreign_col_names, @@ -3531,8 +3491,7 @@ dict_foreign_add_to_cache( for_in_cache->referenced_index, check_charsets, for_in_cache->type & (DICT_FOREIGN_ON_DELETE_SET_NULL - | DICT_FOREIGN_ON_UPDATE_SET_NULL), - &index_error, &err_col, &err_index); + | DICT_FOREIGN_ON_UPDATE_SET_NULL)); if (index == NULL && !(ignore_err & DICT_ERR_IGNORE_FK_NOKEY)) { @@ -4244,7 +4203,7 @@ dict_foreign_push_index_error( const char* latest_foreign, /*!< in: start of latest foreign key constraint name */ const char** columns, /*!< in: foreign key columns */ - ulint index_error, /*!< in: error code */ + fkerr_t index_error, /*!< in: error code */ ulint err_col, /*!< in: column where error happened */ dict_index_t* err_index, /*!< in: index where error happened @@ -4253,37 +4212,37 @@ dict_foreign_push_index_error( FILE* ef) /*!< in: output stream */ { switch (index_error) { - case DB_FOREIGN_KEY_INDEX_NOT_FOUND: { + case FK_SUCCESS: + break; + case FK_INDEX_NOT_FOUND: fprintf(ef, - "%s table '%s' with foreign key constraint" + "%s table %s with foreign key constraint" " failed. There is no index in the referenced" " table where the referenced columns appear" " as the first columns near '%s'.\n", operation, create_name, latest_foreign); ib_push_warning(trx, DB_CANNOT_ADD_CONSTRAINT, - "%s table '%s' with foreign key constraint" + "%s table %s with foreign key constraint" " failed. There is no index in the referenced" " table where the referenced columns appear" " as the first columns near '%s'.", operation, create_name, latest_foreign); - break; - } - case DB_FOREIGN_KEY_IS_PREFIX_INDEX: { + return; + case FK_IS_PREFIX_INDEX: fprintf(ef, - "%s table '%s' with foreign key constraint" + "%s table %s with foreign key constraint" " failed. There is only prefix index in the referenced" " table where the referenced columns appear" " as the first columns near '%s'.\n", operation, create_name, latest_foreign); ib_push_warning(trx, DB_CANNOT_ADD_CONSTRAINT, - "%s table '%s' with foreign key constraint" + "%s table %s with foreign key constraint" " failed. There is only prefix index in the referenced" " table where the referenced columns appear" " as the first columns near '%s'.", operation, create_name, latest_foreign); - break; - } - case DB_FOREIGN_KEY_COL_NOT_NULL: { + return; + case FK_COL_NOT_NULL: fprintf(ef, "%s table %s with foreign key constraint" " failed. You have defined a SET NULL condition but " @@ -4294,9 +4253,8 @@ dict_foreign_push_index_error( " failed. You have defined a SET NULL condition but " "column '%s' on index is defined as NOT NULL near '%s'.", operation, create_name, columns[err_col], latest_foreign); - break; - } - case DB_FOREIGN_KEY_COLS_NOT_EQUAL: { + return; + case FK_COLS_NOT_EQUAL: dict_field_t* field; const char* col_name; field = dict_index_get_nth_field(err_index, err_col); @@ -4315,11 +4273,9 @@ dict_foreign_push_index_error( " failed. Field type or character set for column '%s' " "does not mach referenced column '%s' near '%s'.", operation, create_name, columns[err_col], col_name, latest_foreign); - break; - } - default: - ut_error; + return; } + DBUG_ASSERT(!"unknown error"); } /*********************************************************************//** @@ -4351,7 +4307,7 @@ dict_create_foreign_constraints_low( const char* start_of_latest_foreign = sql_string; const char* start_of_latest_set = NULL; FILE* ef = dict_foreign_err_file; - ulint index_error = DB_SUCCESS; + fkerr_t index_error = FK_SUCCESS; dict_index_t* err_index = NULL; ulint err_col; const char* constraint_name; @@ -6832,7 +6788,7 @@ dict_foreign_qualify_index( /*!< in: nonzero if none of the columns must be declared NOT NULL */ - ulint* error, /*!< out: error code */ + fkerr_t* error, /*!< out: error code */ ulint* err_col_no, /*!< out: column number where error happened */ @@ -6860,7 +6816,7 @@ dict_foreign_qualify_index( /* We do not accept column prefix indexes here */ if (error && err_col_no && err_index) { - *error = DB_FOREIGN_KEY_IS_PREFIX_INDEX; + *error = FK_IS_PREFIX_INDEX; *err_col_no = i; *err_index = (dict_index_t*)index; } @@ -6870,7 +6826,7 @@ dict_foreign_qualify_index( if (check_null && (field->col->prtype & DATA_NOT_NULL)) { if (error && err_col_no && err_index) { - *error = DB_FOREIGN_KEY_COL_NOT_NULL; + *error = FK_COL_NOT_NULL; *err_col_no = i; *err_index = (dict_index_t*)index; } @@ -6900,7 +6856,7 @@ dict_foreign_qualify_index( dict_index_get_nth_col(types_idx, i), check_charsets)) { if (error && err_col_no && err_index) { - *error = DB_FOREIGN_KEY_COLS_NOT_EQUAL; + *error = FK_COLS_NOT_EQUAL; *err_col_no = i; *err_index = (dict_index_t*)index; } diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 39adcea5f9d..21639fd63b6 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -10331,17 +10331,6 @@ next_record: } #ifdef WITH_WSREP -extern dict_index_t* -wsrep_dict_foreign_find_index( -/*==========================*/ - dict_table_t* table, - const char** col_names, - const char** columns, - ulint n_cols, - dict_index_t* types_idx, - ibool check_charsets, - ulint check_null); - inline const char* wsrep_key_type_to_str(wsrep_key_type type) @@ -10404,7 +10393,7 @@ wsrep_append_foreign_key( foreign->referenced_table_name_lookup); if (foreign->referenced_table) { foreign->referenced_index = - wsrep_dict_foreign_find_index( + dict_foreign_find_index( foreign->referenced_table, NULL, foreign->referenced_col_names, foreign->n_fields, @@ -10418,7 +10407,7 @@ wsrep_append_foreign_key( if (foreign->foreign_table) { foreign->foreign_index = - wsrep_dict_foreign_find_index( + dict_foreign_find_index( foreign->foreign_table, NULL, foreign->foreign_col_names, foreign->n_fields, diff --git a/storage/innobase/include/dict0dict.h b/storage/innobase/include/dict0dict.h index fbaef118b70..4e90c9398ba 100644 --- a/storage/innobase/include/dict0dict.h +++ b/storage/innobase/include/dict0dict.h @@ -527,6 +527,21 @@ dict_table_open_on_name( dict_err_ignore_t ignore_err) MY_ATTRIBUTE((warn_unused_result)); +/** Outcome of dict_foreign_find_index() or dict_foreign_qualify_index() */ +enum fkerr_t +{ + /** A backing index was found for a FOREIGN KEY constraint */ + FK_SUCCESS = 0, + /** There is no index that covers the columns in the constraint. */ + FK_INDEX_NOT_FOUND, + /** The index is for a prefix index, not a full column. */ + FK_IS_PREFIX_INDEX, + /** A condition of SET NULL conflicts with a NOT NULL column. */ + FK_COL_NOT_NULL, + /** The column types do not match */ + FK_COLS_NOT_EQUAL +}; + /*********************************************************************//** Tries to find an index whose first fields are the columns in the array, in the same order and is not marked for deletion and is not the same @@ -553,11 +568,11 @@ dict_foreign_find_index( /*!< in: nonzero if none of the columns must be declared NOT NULL */ - ulint* error, /*!< out: error code */ - ulint* err_col_no, + fkerr_t* error = NULL, /*!< out: error code */ + ulint* err_col_no = NULL, /*!< out: column number where error happened */ - dict_index_t** err_index) + dict_index_t** err_index = NULL) /*!< out: index where error happened */ @@ -643,7 +658,7 @@ dict_foreign_qualify_index( /*!< in: nonzero if none of the columns must be declared NOT NULL */ - ulint* error, /*!< out: error code */ + fkerr_t* error, /*!< out: error code */ ulint* err_col_no, /*!< out: column number where error happened */ -- cgit v1.2.1 From 9de2e60d7491fcf3cd1f20a4be715ef0bedc316f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 21 Aug 2019 11:38:17 +0300 Subject: MDEV-17187 table doesn't exist in engine after ALTER of FOREIGN KEY ha_innobase::open(): Always ignore problems with FOREIGN KEY constraints (pass DICT_ERR_IGNORE_FK_NOKEY), no matter whether foreign_key_checks is enabled. Instead, we must report errors when enforcing the FOREIGN KEY constraints. As a result of ignoring these errors, the tables will be loaded with dict_foreign_t objects whose foreign_index or referenced_index will be NULL. Also, pass DICT_ERR_IGNORE_FK_NOKEY instead of DICT_ERR_IGNORE_NONE to dict_table_open_on_id_low() in many other cases. Notably, on CREATE TABLE and ALTER TABLE, we will keep validating the FOREIGN KEY constraints as before. dict_table_open_on_name(): If no other flags than DICT_ERR_IGNORE_FK_NOKEY are set, refuse access to unreadable tables. Some encryption tests rely on this code path. For the DML code path, we used to have the problem that when one of the indexes was missing in dict_foreign_t, we would ignore the FOREIGN KEY constraint altogether. The following changes address that. row_ins_check_foreign_constraints(): Add the parameter pk. For the primary key, consider also foreign key constraints for which foreign->foreign_index=NULL (no underlying index is available). row_ins_check_foreign_constraint(): Report errors also for !check_ref. Remove a redundant check for srv_read_only_mode. row_ins_foreign_report_add_err(): Tolerate foreign->foreign_index=NULL. --- storage/innobase/dict/dict0dict.cc | 6 +-- storage/innobase/dict/dict0load.cc | 2 +- storage/innobase/handler/ha_innodb.cc | 21 +++------- storage/innobase/include/dict0types.h | 8 ++-- storage/innobase/row/row0ins.cc | 79 +++++++++++++++++++++++------------ storage/innobase/row/row0mysql.cc | 8 ++-- 6 files changed, 70 insertions(+), 54 deletions(-) (limited to 'storage') diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 3e5e8ca2fec..32fcfe68d32 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -432,7 +432,7 @@ dict_table_try_drop_aborted( if (table == NULL) { table = dict_table_open_on_id_low( - table_id, DICT_ERR_IGNORE_NONE, FALSE); + table_id, DICT_ERR_IGNORE_FK_NOKEY, FALSE); } else { ut_ad(table->id == table_id); } @@ -1005,7 +1005,7 @@ dict_table_open_on_id( table_id, table_op == DICT_TABLE_OP_LOAD_TABLESPACE ? DICT_ERR_IGNORE_RECOVER_LOCK - : DICT_ERR_IGNORE_NONE, + : DICT_ERR_IGNORE_FK_NOKEY, table_op == DICT_TABLE_OP_OPEN_ONLY_IF_CACHED); if (table != NULL) { @@ -1167,7 +1167,7 @@ dict_table_open_on_name( if (table != NULL) { /* If table is encrypted or corrupted */ - if (ignore_err == DICT_ERR_IGNORE_NONE + if (!(ignore_err & ~DICT_ERR_IGNORE_FK_NOKEY) && !table->is_readable()) { /* Make life easy for drop table. */ dict_table_prevent_eviction(table); diff --git a/storage/innobase/dict/dict0load.cc b/storage/innobase/dict/dict0load.cc index a8ed8dd3e29..6900d62b225 100644 --- a/storage/innobase/dict/dict0load.cc +++ b/storage/innobase/dict/dict0load.cc @@ -3121,7 +3121,7 @@ func_exit: mem_heap_free(heap); ut_ad(!table - || ignore_err != DICT_ERR_IGNORE_NONE + || (ignore_err & ~DICT_ERR_IGNORE_FK_NOKEY) || !table->is_readable() || !table->corrupted); diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 21639fd63b6..7c9e4148886 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -3185,7 +3185,7 @@ static bool innobase_query_caching_table_check( const char* norm_name) { dict_table_t* table = dict_table_open_on_name( - norm_name, FALSE, FALSE, DICT_ERR_IGNORE_NONE); + norm_name, FALSE, FALSE, DICT_ERR_IGNORE_FK_NOKEY); if (table == NULL) { return false; @@ -6209,9 +6209,7 @@ initialize_auto_increment(dict_table_t* table, const Field* field) int ha_innobase::open(const char* name, int, uint) { - dict_table_t* ib_table; char norm_name[FN_REFLEN]; - dict_err_ignore_t ignore_err = DICT_ERR_IGNORE_NONE; DBUG_ENTER("ha_innobase::open"); @@ -6225,15 +6223,8 @@ ha_innobase::open(const char* name, int, uint) char* is_part = is_partition(norm_name); THD* thd = ha_thd(); - - /* Check whether FOREIGN_KEY_CHECKS is set to 0. If so, the table - can be opened even if some FK indexes are missing. If not, the table - can't be opened in the same situation */ - if (thd_test_options(thd, OPTION_NO_FOREIGN_KEY_CHECKS)) { - ignore_err = DICT_ERR_IGNORE_FK_NOKEY; - } - - ib_table = open_dict_table(name, norm_name, is_part, ignore_err); + dict_table_t* ib_table = open_dict_table(name, norm_name, is_part, + DICT_ERR_IGNORE_FK_NOKEY); DEBUG_SYNC(thd, "ib_open_after_dict_open"); @@ -13404,8 +13395,8 @@ innobase_rename_table( row_mysql_lock_data_dictionary(trx); } - dict_table_t* table = dict_table_open_on_name(norm_from, TRUE, FALSE, - DICT_ERR_IGNORE_NONE); + dict_table_t* table = dict_table_open_on_name( + norm_from, TRUE, FALSE, DICT_ERR_IGNORE_FK_NOKEY); /* Since DICT_BG_YIELD has sleep for 250 milliseconds, Convert lock_wait_timeout unit from second to 250 milliseconds */ @@ -14582,7 +14573,7 @@ ha_innobase::defragment_table( normalize_table_name(norm_name, name); table = dict_table_open_on_name(norm_name, FALSE, - FALSE, DICT_ERR_IGNORE_NONE); + FALSE, DICT_ERR_IGNORE_FK_NOKEY); for (index = dict_table_get_first_index(table); index; index = dict_table_get_next_index(index)) { diff --git a/storage/innobase/include/dict0types.h b/storage/innobase/include/dict0types.h index 39addf697e8..93c2f570e54 100644 --- a/storage/innobase/include/dict0types.h +++ b/storage/innobase/include/dict0types.h @@ -59,11 +59,11 @@ Note: please define the IGNORE_ERR_* as bits, so their value can be or-ed together */ enum dict_err_ignore_t { DICT_ERR_IGNORE_NONE = 0, /*!< no error to ignore */ - DICT_ERR_IGNORE_INDEX_ROOT = 1, /*!< ignore error if index root - page is FIL_NULL or incorrect value */ - DICT_ERR_IGNORE_CORRUPT = 2, /*!< skip corrupted indexes */ - DICT_ERR_IGNORE_FK_NOKEY = 4, /*!< ignore error if any foreign + DICT_ERR_IGNORE_FK_NOKEY = 1, /*!< ignore error if any foreign key is missing */ + DICT_ERR_IGNORE_INDEX_ROOT = 2, /*!< ignore error if index root + page is FIL_NULL or incorrect value */ + DICT_ERR_IGNORE_CORRUPT = 4, /*!< skip corrupted indexes */ DICT_ERR_IGNORE_RECOVER_LOCK = 8, /*!< Used when recovering table locks for resurrected transactions. diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc index ece291378b6..2cda0e3d03c 100644 --- a/storage/innobase/row/row0ins.cc +++ b/storage/innobase/row/row0ins.cc @@ -875,8 +875,12 @@ row_ins_foreign_report_add_err( fk_str = dict_print_info_on_foreign_key_in_create_format(trx, foreign, TRUE); fputs(fk_str.c_str(), ef); - fprintf(ef, " in parent table, in index %s", - foreign->foreign_index->name()); + if (foreign->foreign_index) { + fprintf(ef, " in parent table, in index %s", + foreign->foreign_index->name()); + } else { + fputs(" in parent table", ef); + } if (entry) { fputs(" tuple:\n", ef); /* TODO: DB_TRX_ID and DB_ROLL_PTR may be uninitialized. @@ -1628,34 +1632,51 @@ row_ins_check_foreign_constraint( || check_index == NULL || fil_space_get(check_table->space)->is_being_truncated) { - if (!srv_read_only_mode && check_ref) { - FILE* ef = dict_foreign_err_file; - std::string fk_str; - - row_ins_set_detailed(trx, foreign); - - row_ins_foreign_trx_print(trx); - - fputs("Foreign key constraint fails for table ", ef); - ut_print_name(ef, trx, - foreign->foreign_table_name); - fputs(":\n", ef); - fk_str = dict_print_info_on_foreign_key_in_create_format( - trx, foreign, TRUE); - fputs(fk_str.c_str(), ef); - fprintf(ef, "\nTrying to add to index %s tuple:\n", - foreign->foreign_index->name()); + FILE* ef = dict_foreign_err_file; + std::string fk_str; + + row_ins_set_detailed(trx, foreign); + row_ins_foreign_trx_print(trx); + + fputs("Foreign key constraint fails for table ", ef); + ut_print_name(ef, trx, check_ref + ? foreign->foreign_table_name + : foreign->referenced_table_name); + fputs(":\n", ef); + fk_str = dict_print_info_on_foreign_key_in_create_format( + trx, foreign, TRUE); + fputs(fk_str.c_str(), ef); + if (check_ref) { + if (foreign->foreign_index) { + fprintf(ef, "\nTrying to add to index %s" + " tuple:\n", + foreign->foreign_index->name()); + } else { + fputs("\nTrying to add tuple:\n", ef); + } dtuple_print(ef, entry); fputs("\nBut the parent table ", ef); - ut_print_name(ef, trx, - foreign->referenced_table_name); - fputs("\nor its .ibd file does" + ut_print_name(ef, trx, foreign->referenced_table_name); + fputs("\nor its .ibd file or the required index does" " not currently exist!\n", ef); - mutex_exit(&dict_foreign_err_mutex); - err = DB_NO_REFERENCED_ROW; + } else { + if (foreign->referenced_index) { + fprintf(ef, "\nTrying to modify index %s" + " tuple:\n", + foreign->referenced_index->name()); + } else { + fputs("\nTrying to modify tuple:\n", ef); + } + dtuple_print(ef, entry); + fputs("\nBut the referencing table ", ef); + ut_print_name(ef, trx, foreign->foreign_table_name); + fputs("\nor its .ibd file or the required index does" + " not currently exist!\n", ef); + err = DB_ROW_IS_REFERENCED; } + mutex_exit(&dict_foreign_err_mutex); goto exit_func; } @@ -1923,6 +1944,7 @@ row_ins_check_foreign_constraints( /*==============================*/ dict_table_t* table, /*!< in: table */ dict_index_t* index, /*!< in: index */ + bool pk, /*!< in: index->is_primary() */ dtuple_t* entry, /*!< in: index entry for index */ que_thr_t* thr) /*!< in: query thread */ { @@ -1931,6 +1953,8 @@ row_ins_check_foreign_constraints( trx_t* trx; ibool got_s_lock = FALSE; + DBUG_ASSERT(index->is_primary() == pk); + trx = thr_get_trx(thr); DEBUG_SYNC_C_IF_THD(thr_get_trx(thr)->mysql_thd, @@ -1942,7 +1966,8 @@ row_ins_check_foreign_constraints( foreign = *it; - if (foreign->foreign_index == index) { + if (foreign->foreign_index == index + || (pk && !foreign->foreign_index)) { dict_table_t* ref_table = NULL; dict_table_t* referenced_table = foreign->referenced_table; @@ -3119,7 +3144,7 @@ row_ins_clust_index_entry( if (!index->table->foreign_set.empty()) { err = row_ins_check_foreign_constraints( - index->table, index, entry, thr); + index->table, index, true, entry, thr); if (err != DB_SUCCESS) { DBUG_RETURN(err); @@ -3193,7 +3218,7 @@ row_ins_sec_index_entry( if (!index->table->foreign_set.empty()) { err = row_ins_check_foreign_constraints(index->table, index, - entry, thr); + false, entry, thr); if (err != DB_SUCCESS) { return(err); diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index f3d48edfb80..4b82fb9c2c4 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -2799,7 +2799,7 @@ row_discard_tablespace_begin( dict_table_t* table; table = dict_table_open_on_name( - name, TRUE, FALSE, DICT_ERR_IGNORE_NONE); + name, TRUE, FALSE, DICT_ERR_IGNORE_FK_NOKEY); if (table) { dict_stats_wait_bg_to_stop_using_table(table, trx); @@ -3199,7 +3199,7 @@ row_drop_table_from_cache( dict_table_remove_from_cache(table); - if (dict_load_table(tablename, true, DICT_ERR_IGNORE_NONE)) { + if (dict_load_table(tablename, true, DICT_ERR_IGNORE_FK_NOKEY)) { ib::error() << "Not able to remove table " << ut_get_name(trx, tablename) << " from the dictionary cache!"; @@ -4164,7 +4164,7 @@ row_rename_table_for_mysql( dict_locked = trx->dict_operation_lock_mode == RW_X_LATCH; table = dict_table_open_on_name(old_name, dict_locked, FALSE, - DICT_ERR_IGNORE_NONE); + DICT_ERR_IGNORE_FK_NOKEY); /* We look for pattern #P# to see if the table is partitioned MySQL table. */ @@ -4212,7 +4212,7 @@ row_rename_table_for_mysql( par_case_name, old_name, FALSE); #endif table = dict_table_open_on_name(par_case_name, dict_locked, FALSE, - DICT_ERR_IGNORE_NONE); + DICT_ERR_IGNORE_FK_NOKEY); } if (!table) { -- cgit v1.2.1 From 5b29820d800180816abd67f24414eaa0d51dd6b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 22 Aug 2019 11:11:22 +0300 Subject: Fix -Wstringop-truncation For the Sphinx storage engine, this is a functional change (bug fix): we will ensure that the message buffer is always NUL-terminated. --- storage/perfschema/pfs_instr.cc | 5 ++--- storage/sphinx/ha_sphinx.cc | 9 ++++++--- 2 files changed, 8 insertions(+), 6 deletions(-) (limited to 'storage') diff --git a/storage/perfschema/pfs_instr.cc b/storage/perfschema/pfs_instr.cc index 44b170469bf..1ea4fcd901b 100644 --- a/storage/perfschema/pfs_instr.cc +++ b/storage/perfschema/pfs_instr.cc @@ -1265,7 +1265,6 @@ find_or_create_file(PFS_thread *thread, PFS_file_class *klass, char dirbuffer[FN_REFLEN]; size_t dirlen; const char *normalized_filename; - int normalized_length; dirlen= dirname_length(safe_filename); if (dirlen == 0) @@ -1296,7 +1295,7 @@ find_or_create_file(PFS_thread *thread, PFS_file_class *klass, *buf_end= '\0'; normalized_filename= buffer; - normalized_length= strlen(normalized_filename); + size_t normalized_length= strlen(normalized_filename); PFS_file **entry; uint retry_count= 0; @@ -1345,7 +1344,7 @@ search: pfs->m_class= klass; pfs->m_enabled= klass->m_enabled && flag_global_instrumentation; pfs->m_timed= klass->m_timed; - strncpy(pfs->m_filename, normalized_filename, normalized_length); + strncpy(pfs->m_filename, normalized_filename, normalized_length + 1); pfs->m_filename[normalized_length]= '\0'; pfs->m_filename_length= normalized_length; pfs->m_file_stat.m_open_count= 1; diff --git a/storage/sphinx/ha_sphinx.cc b/storage/sphinx/ha_sphinx.cc index b496450c5d6..96d6bd1b10f 100644 --- a/storage/sphinx/ha_sphinx.cc +++ b/storage/sphinx/ha_sphinx.cc @@ -2291,7 +2291,8 @@ int ha_sphinx::HandleMysqlError ( MYSQL * pConn, int iErrCode ) CSphSEThreadTable * pTable = GetTls (); if ( pTable ) { - strncpy ( pTable->m_tStats.m_sLastMessage, mysql_error ( pConn ), sizeof ( pTable->m_tStats.m_sLastMessage ) ); + strncpy ( pTable->m_tStats.m_sLastMessage, mysql_error ( pConn ), sizeof pTable->m_tStats.m_sLastMessage - 1 ); + pTable->m_tStats.m_sLastMessage[sizeof pTable->m_tStats.m_sLastMessage - 1] = '\0'; pTable->m_tStats.m_bLastError = true; } @@ -2558,7 +2559,8 @@ bool ha_sphinx::UnpackSchema () CSphSEThreadTable * pTable = GetTls (); if ( pTable ) { - strncpy ( pTable->m_tStats.m_sLastMessage, sMessage, sizeof(pTable->m_tStats.m_sLastMessage) ); + strncpy ( pTable->m_tStats.m_sLastMessage, sMessage, sizeof pTable->m_tStats.m_sLastMessage - 1 ); + pTable->m_tStats.m_sLastMessage[sizeof pTable->m_tStats.m_sLastMessage - 1] = '\0'; pTable->m_tStats.m_bLastError = ( uStatus==SEARCHD_ERROR ); } @@ -2982,7 +2984,8 @@ int ha_sphinx::index_read ( byte * buf, const byte * key, uint key_len, enum ha_ SPH_RET ( HA_ERR_END_OF_FILE ); } - strncpy ( pTable->m_tStats.m_sLastMessage, sMessage, sizeof(pTable->m_tStats.m_sLastMessage) ); + strncpy ( pTable->m_tStats.m_sLastMessage, sMessage, sizeof pTable->m_tStats.m_sLastMessage - 1 ); + pTable->m_tStats.m_sLastMessage[sizeof pTable->m_tStats.m_sLastMessage - 1] = '\0'; SafeDeleteArray ( sMessage ); if ( uRespStatus!=SEARCHD_WARNING ) -- cgit v1.2.1 From b01a95f6fc47151345bf298b606492a962e28150 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 22 Aug 2019 17:37:13 +0300 Subject: row_undo_mod_remove_clust_low(): Remove duplicated code Some code was duplicated near the start of the function, only for InnoDB, not XtraDB. This was noticed by comparing the InnoDB between MariaDB and MySQL. --- storage/innobase/row/row0umod.cc | 32 +------------------------------- 1 file changed, 1 insertion(+), 31 deletions(-) (limited to 'storage') diff --git a/storage/innobase/row/row0umod.cc b/storage/innobase/row/row0umod.cc index 1d4a5b9b40f..90fd3c79c57 100644 --- a/storage/innobase/row/row0umod.cc +++ b/storage/innobase/row/row0umod.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2019, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -212,36 +212,6 @@ row_undo_mod_remove_clust_low( return(DB_SUCCESS); } - trx_id_offset = btr_cur_get_index(btr_cur)->trx_id_offset; - - if (!trx_id_offset) { - mem_heap_t* heap = NULL; - ulint trx_id_col; - const ulint* offsets; - ulint len; - - trx_id_col = dict_index_get_sys_col_pos( - btr_cur_get_index(btr_cur), DATA_TRX_ID); - ut_ad(trx_id_col > 0); - ut_ad(trx_id_col != ULINT_UNDEFINED); - - offsets = rec_get_offsets( - btr_cur_get_rec(btr_cur), btr_cur_get_index(btr_cur), - NULL, trx_id_col + 1, &heap); - - trx_id_offset = rec_get_nth_field_offs( - offsets, trx_id_col, &len); - ut_ad(len == DATA_TRX_ID_LEN); - mem_heap_free(heap); - } - - if (trx_read_trx_id(btr_cur_get_rec(btr_cur) + trx_id_offset) - != node->new_trx_id) { - /* The record must have been purged and then replaced - with a different one. */ - return(DB_SUCCESS); - } - /* We are about to remove an old, delete-marked version of the record that may have been delete-marked by a different transaction than the rolling-back one. */ -- cgit v1.2.1 From 6c593cd35868cb47eb9d4a712575668a243f3186 Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Thu, 22 Aug 2019 23:34:42 +0200 Subject: Typo --- storage/connect/connect.cc | 3 ++- storage/connect/mycat.h | 10 +++++----- storage/connect/plgdbsem.h | 8 ++++---- 3 files changed, 11 insertions(+), 10 deletions(-) (limited to 'storage') diff --git a/storage/connect/connect.cc b/storage/connect/connect.cc index 21bca637eab..b1078de8eaa 100644 --- a/storage/connect/connect.cc +++ b/storage/connect/connect.cc @@ -13,7 +13,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ /***********************************************************************/ /* Author Olivier BERTRAND bertrandop@gmail.com 2004-2017 */ @@ -33,6 +33,7 @@ /* plgdbsem.h is header containing the DB applic. declarations. */ /***********************************************************************/ #define DONT_DEFINE_VOID +#include #include "handler.h" #undef OFFSET diff --git a/storage/connect/mycat.h b/storage/connect/mycat.h index b306d4acbe0..818e535b32d 100644 --- a/storage/connect/mycat.h +++ b/storage/connect/mycat.h @@ -11,10 +11,10 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ /**************** MYCAT H Declares Source Code File (.H) ***************/ -/* Name: MYCAT.H Version 2.3 */ +/* Name: MYCAT.H Version 2.4 */ /* Author: Olivier Bertrand */ /* This file contains the CONNECT plugin MYCAT class definitions. */ /***********************************************************************/ @@ -50,9 +50,9 @@ struct ha_table_option_struct { const char *filter; const char *oplist; const char *data_charset; - const char *http; - const char *uri; - ulonglong lrecl; + const char *http; + const char *uri; + ulonglong lrecl; ulonglong elements; //ulonglong estimate; ulonglong multiple; diff --git a/storage/connect/plgdbsem.h b/storage/connect/plgdbsem.h index 14b8549beb0..f10ae209e9d 100644 --- a/storage/connect/plgdbsem.h +++ b/storage/connect/plgdbsem.h @@ -1,7 +1,7 @@ /************** PlgDBSem H Declares Source Code File (.H) **************/ -/* Name: PLGDBSEM.H Version 3.7 */ +/* Name: PLGDBSEM.H Version 3.8 */ /* */ -/* (C) Copyright to the author Olivier BERTRAND 1998-2017 */ +/* (C) Copyright to the author Olivier BERTRAND 1998-2019 */ /* */ /* This file contains the CONNECT storage engine definitions. */ /***********************************************************************/ @@ -82,8 +82,8 @@ enum TABTYPE {TAB_UNDEF = 0, /* Table of undefined type */ TAB_JDBC = 26, /* Table accessed via JDBC */ TAB_ZIP = 27, /* ZIP file info table */ TAB_MONGO = 28, /* Table retrieved from MongoDB */ - TAB_REST = 29, /* Table retrieved from Rest query */ - TAB_NIY = 30}; /* Table not implemented yet */ + TAB_REST = 29, /* Table retrieved from Rest */ + TAB_NIY = 30}; /* Table not implemented yet */ enum AMT {TYPE_AM_ERROR = 0, /* Type not defined */ TYPE_AM_ROWID = 1, /* ROWID type (special column) */ -- cgit v1.2.1 From e7b71e0daa49ca9b5becbf2739ed13d7c8c4fcd6 Mon Sep 17 00:00:00 2001 From: Sujatha Date: Tue, 27 Aug 2019 13:05:04 +0530 Subject: MDEV-19925: Column ... cannot be converted from type 'varchar(20)' to type 'varchar(20)' Cherry picking: Bug#25135304: RBR: WRONG FIELD LENGTH IN ERROR MESSAGE commit 47bd3f7cf3c8518f62b1580ec65af2ba7ac13b95 Description: ============ In row based replication, when replicating from a table with a field with character set set to UTF8mb3 to the same table with the same field set to character set UTF8mb4 I get a confusing error message: For VARCHAR: VARCHAR(1) 'utf8mb3' to VARCHAR(1) 'utf8mb4' "Column 0 of table 'test.t1' cannot be converted from type 'varchar(3)' to type 'varchar(1)'" Similar issue with CHAR type as well. Issue with respect to BLOB types: For BLOB: LONGBLOB to TINYBLOB - Error message displays incorrect blob type. "Column 0 of table 'test.t1' cannot be converted from type 'tinyblob' to type 'tinyblob'" For BINARY to BINARY - Error message displays incorrect type for master side field. "Column 0 of table 'test.t' cannot be converted from type 'char(1)' to type 'binary(10)'" Similar issue exists for VARBINARY type. It is displayed as 'VARCHAR'. Analysis: ========= In Row based replication charset information is not sent as part of metadata from master to slave. For VARCHAR field its character length is converted into equivalent octets/bytes and stored internally. At the time of displaying the data to user it is converted back to original character length. For example: VARCHAR(2)- utf8mb3 is stored as:2*3 = VARCHAR(6) At the time of displaying it to user VARCHAR(6)- charset utf8mb3:6/3= VARCHAR(2). At present the internally converted octect length is sent from master to slave with out providing the charset information. On slave side if the type conversion fails 'show_sql_type' function is used to get the type specific information from metadata. Since there is no charset information is available the filed type is displayed as VARCHAR(6). This results in confused error message. For CHAR fields CHAR(1)- utf8mb3 - CHAR(3) CHAR(1)- utf8mb4 - CHAR(4) 'show_sql_type' function which retrieves type information from metadata uses (bytes/local charset length) to get actual character length. If slave's chaset is 'utf8mb4' then CHAR(3/4)-->CHAR(0) CHAR(4/4)-->CHAR(1). This results in confused error message. Analysis for BLOB type issue: BLOB's length is represented in two forms. 1. Actual length i.e (length < 256) type= MYSQL_TYPE_TINY_BLOB; (length < 65536) type= MYSQL_TYPE_BLOB; ... 2. packlength - The number of bytes used to represent the length of the blob 1- tinyblob 2- blob ... In row based replication only the packlength is written in the binary log. On the slave side this packlength is interpreted as actual length of the blob. Hence the length is always < 256 and the type is displayed as tiny blob. Analysis for BINARY to BINARY type issue: The character set information is needed to identify a filed's type as char or binary. Since master side character set information is not available on the slave side both binary and char fields are displayed as char. Fix: === For CHAR and VARCHAR fields display their length in octets for both source and target fields. For target field display the charset information if it is relevant. For blob type changed the code to use the packlength and display appropriate blob type in error message. For binary and varbinary fields use the slave side character set as reference to map them to binary or varbinary fields. --- .../mysql-test/rpl/r/rpl_extra_col_master_tokudb.result | 4 ++-- .../mysql-test/rpl/r/rpl_extra_col_slave_tokudb.result | 12 ++++++------ storage/tokudb/mysql-test/rpl/r/rpl_row_basic_3tokudb.result | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) (limited to 'storage') diff --git a/storage/tokudb/mysql-test/rpl/r/rpl_extra_col_master_tokudb.result b/storage/tokudb/mysql-test/rpl/r/rpl_extra_col_master_tokudb.result index cca18c7c1ab..6982078d2b8 100644 --- a/storage/tokudb/mysql-test/rpl/r/rpl_extra_col_master_tokudb.result +++ b/storage/tokudb/mysql-test/rpl/r/rpl_extra_col_master_tokudb.result @@ -468,7 +468,7 @@ INSERT INTO t10 () VALUES(1,@b1,DEFAULT,'Kyle',DEFAULT), connection slave; include/wait_for_slave_sql_error_and_skip.inc [errno=1677] -Last_SQL_Error = 'Column 2 of table 'test.t10' cannot be converted from type 'double' to type 'char(5)'' +Last_SQL_Error = 'Column 2 of table 'test.t10' cannot be converted from type 'double' to type 'char(5 octets) character set latin1'' *** Drop t10 *** connection master; @@ -510,7 +510,7 @@ INSERT INTO t11 () VALUES(1,@b1,'Testing is fun','Kyle',DEFAULT), connection slave; include/wait_for_slave_sql_error_and_skip.inc [errno=1677] -Last_SQL_Error = 'Column 2 of table 'test.t11' cannot be converted from type 'tinyblob' to type 'varchar(254)'' +Last_SQL_Error = 'Column 2 of table 'test.t11' cannot be converted from type 'blob' to type 'varchar(254 octets) character set latin1'' *** Drop t11 *** connection master; diff --git a/storage/tokudb/mysql-test/rpl/r/rpl_extra_col_slave_tokudb.result b/storage/tokudb/mysql-test/rpl/r/rpl_extra_col_slave_tokudb.result index 8906cf31d74..318d5496255 100644 --- a/storage/tokudb/mysql-test/rpl/r/rpl_extra_col_slave_tokudb.result +++ b/storage/tokudb/mysql-test/rpl/r/rpl_extra_col_slave_tokudb.result @@ -64,7 +64,7 @@ a b c connection slave; START SLAVE; include/wait_for_slave_sql_error.inc [errno=1677] -Last_SQL_Error = 'Column 2 of table 'test.t2' cannot be converted from type 'char(10)' to type 'char(5)'' +Last_SQL_Error = 'Column 2 of table 'test.t2' cannot be converted from type 'char(10 octets)' to type 'char(5 octets) character set latin1'' STOP SLAVE; RESET SLAVE; SELECT * FROM t2 ORDER BY a; @@ -102,7 +102,7 @@ INSERT INTO t3 () VALUES(@b1,2,'Kyle, TEX'),(@b1,1,'JOE AUSTIN'),(@b1,4,'QA TEST ******************************************** connection slave; include/wait_for_slave_sql_error_and_skip.inc [errno=1677] -Last_SQL_Error = 'Column 0 of table 'test.t3' cannot be converted from type 'tinyblob' to type 'int(11)'' +Last_SQL_Error = 'Column 0 of table 'test.t3' cannot be converted from type 'blob' to type 'int(11)'' *** Drop t3 *** connection master; DROP TABLE t3; @@ -160,7 +160,7 @@ INSERT INTO t5 () VALUES(1,'Kyle',200.23,1,'b1b1',23.00098), ******************************************** connection slave; include/wait_for_slave_sql_error_and_skip.inc [errno=1677] -Last_SQL_Error = 'Column 1 of table 'test.t5' cannot be converted from type 'varchar(6)' to type 'char(5)'' +Last_SQL_Error = 'Column 1 of table 'test.t5' cannot be converted from type 'varchar(6 octets)' to type 'char(5 octets) character set latin1'' *** Drop t5 *** connection master; DROP TABLE t5; @@ -188,7 +188,7 @@ INSERT INTO t6 () VALUES(1,'Kyle',200.23,1), ******************************************** connection slave; include/wait_for_slave_sql_error.inc [errno=1677] -Last_SQL_Error = 'Column 1 of table 'test.t6' cannot be converted from type 'varchar(6)' to type 'char(5)'' +Last_SQL_Error = 'Column 1 of table 'test.t6' cannot be converted from type 'varchar(6 octets)' to type 'char(5 octets) character set latin1'' *** Drop t6 *** include/rpl_reset.inc connection master; @@ -310,7 +310,7 @@ INSERT INTO t10 () VALUES(1,@b1,'Kyle'),(2,@b1,'JOE'),(3,@b1,'QA'); ******************************************** connection slave; include/wait_for_slave_sql_error_and_skip.inc [errno=1677] -Last_SQL_Error = 'Column 2 of table 'test.t10' cannot be converted from type 'char(5)' to type 'double'' +Last_SQL_Error = 'Column 2 of table 'test.t10' cannot be converted from type 'char(5 octets)' to type 'double'' *** Drop t10 *** connection master; DROP TABLE t10; @@ -338,7 +338,7 @@ INSERT INTO t11 () VALUES(1,@b1,'Kyle'),(2,@b1,'JOE'),(3,@b1,'QA'); ******************************************** connection slave; include/wait_for_slave_sql_error_and_skip.inc [errno=1677] -Last_SQL_Error = 'Column 2 of table 'test.t11' cannot be converted from type 'varchar(254)' to type 'int(11)'' +Last_SQL_Error = 'Column 2 of table 'test.t11' cannot be converted from type 'varchar(254 octets)' to type 'int(11)'' *** Drop t11 *** connection master; DROP TABLE t11; diff --git a/storage/tokudb/mysql-test/rpl/r/rpl_row_basic_3tokudb.result b/storage/tokudb/mysql-test/rpl/r/rpl_row_basic_3tokudb.result index 32a42143180..e638a1aab12 100644 --- a/storage/tokudb/mysql-test/rpl/r/rpl_row_basic_3tokudb.result +++ b/storage/tokudb/mysql-test/rpl/r/rpl_row_basic_3tokudb.result @@ -565,7 +565,7 @@ INSERT INTO t5 VALUES (1, "", 1); INSERT INTO t5 VALUES (2, repeat(_utf8'a', 255), 2); connection slave; include/wait_for_slave_sql_error.inc [errno=1677] -Last_SQL_Error = 'Column 1 of table 'test.t5' cannot be converted from type 'char(255)' to type 'char(16)'' +Last_SQL_Error = 'Column 1 of table 'test.t5' cannot be converted from type 'char(765 octets)' to type 'char(48 octets) character set utf8'' include/rpl_reset.inc [expecting slave to stop] connection master; @@ -573,7 +573,7 @@ INSERT INTO t6 VALUES (1, "", 1); INSERT INTO t6 VALUES (2, repeat(_utf8'a', 255), 2); connection slave; include/wait_for_slave_sql_error.inc [errno=1677] -Last_SQL_Error = 'Column 1 of table 'test.t6' cannot be converted from type 'char(255)' to type 'char(128)'' +Last_SQL_Error = 'Column 1 of table 'test.t6' cannot be converted from type 'char(765 octets)' to type 'char(384 octets) character set utf8'' include/rpl_reset.inc [expecting slave to replicate correctly] connection master; -- cgit v1.2.1 From 25af2a183b03f1d50da2466de84559a09a3e182c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 27 Aug 2019 15:48:46 +0300 Subject: MDEV-15326/MDEV-16136 dead code removal Revert part of fa2a74e08d00bcf8ddb1d98dba767e9bd1ed802d. trx_reference(): Remove, and merge the relevant part to the only caller trx_rw_is_active(). If the statements trx = NULL; were ever executed, the function would have dereferenced a NULL pointer and crashed in trx_mutex_exit(trx). Hence, those statements must have been unreachable, and they can be replaced with debug assertions. trx_rw_is_active(): Avoid unnecessary acquisition and release of trx->mutex when do_ref_count=false. lock_trx_release_locks(): Do not reset trx->id=0. Had the statement been necessary, we would have experienced crashes in trx_reference(). --- storage/innobase/include/trx0sys.ic | 9 +++++++-- storage/innobase/include/trx0trx.h | 26 -------------------------- storage/innobase/lock/lock0lock.cc | 2 -- storage/innobase/trx/trx0trx.cc | 7 ++++--- 4 files changed, 11 insertions(+), 33 deletions(-) (limited to 'storage') diff --git a/storage/innobase/include/trx0sys.ic b/storage/innobase/include/trx0sys.ic index c552af9ab47..bfd345a27db 100644 --- a/storage/innobase/include/trx0sys.ic +++ b/storage/innobase/include/trx0sys.ic @@ -364,8 +364,13 @@ trx_rw_is_active( trx_t* trx = trx_rw_is_active_low(trx_id, corrupt); - if (trx) { - trx = trx_reference(do_ref_count ? trx_id : 0, trx); + if (trx && do_ref_count) { + trx_mutex_enter(trx); + ut_ad(!trx_state_eq(trx, TRX_STATE_COMMITTED_IN_MEMORY)); + ut_ad(trx->id == trx_id); + ut_ad(trx->n_ref >= 0); + ++trx->n_ref; + trx_mutex_exit(trx); } trx_sys_mutex_exit(); diff --git a/storage/innobase/include/trx0trx.h b/storage/innobase/include/trx0trx.h index 4ab29e08e0f..62f7f172453 100644 --- a/storage/innobase/include/trx0trx.h +++ b/storage/innobase/include/trx0trx.h @@ -1270,32 +1270,6 @@ struct commit_node_t{ mutex_exit(&t->mutex); \ } while (0) -/** -Increase the reference count. If the transaction is in state -TRX_STATE_COMMITTED_IN_MEMORY then the transaction is considered -committed and the reference count is not incremented. -@param id the transaction ID; 0 if not to increment the reference count -@param trx Transaction that is being referenced -@return trx -@retval NULL if the transaction is no longer active */ -inline trx_t* trx_reference(trx_id_t id, trx_t* trx) -{ - trx_mutex_enter(trx); - - if (trx_state_eq(trx, TRX_STATE_COMMITTED_IN_MEMORY)) { - trx = NULL; - } else if (!id) { - } else if (trx->id != id) { - trx = NULL; - } else { - ut_ad(trx->n_ref >= 0); - ++trx->n_ref; - } - - trx_mutex_exit(trx); - return(trx); -} - #include "trx0trx.ic" #endif diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index 21171bad2bb..a674a38df2d 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -6647,8 +6647,6 @@ lock_trx_release_locks( the is_recovered flag. */ trx->is_recovered = false; - /* Ensure that trx_reference() will not find this transaction. */ - trx->id = 0; trx_mutex_exit(trx); diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index 196ae5db36e..1f6ed7a48d4 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -557,6 +557,7 @@ trx_free_prepared( transaction was never committed and therefore lock_trx_release() was not called. */ trx->lock.table_locks.clear(); + trx->id = 0; trx_free(trx); } @@ -1660,10 +1661,8 @@ trx_commit_in_memory( trx_erase_lists(trx, serialised); } - /* trx->id will be cleared in lock_trx_release_locks(trx). */ - ut_ad(trx->read_only || !trx->rsegs.m_redo.rseg || trx->id); lock_trx_release_locks(trx); - ut_ad(trx->id == 0); + ut_ad(trx->read_only || !trx->rsegs.m_redo.rseg || trx->id); /* Remove the transaction from the list of active transactions now that it no longer holds any user locks. */ @@ -1682,6 +1681,8 @@ trx_commit_in_memory( } else { MONITOR_INC(MONITOR_TRX_RW_COMMIT); } + + trx->id = 0; } ut_ad(!trx->rsegs.m_redo.update_undo); -- cgit v1.2.1 From 947b0b5722117350c83656ee0b23502be59b7d2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 23 Aug 2019 17:43:55 +0300 Subject: Implement innodb_evict_tables_on_commit_debug Some bugs are detected only after a table definition has been evicted and then reloaded to the InnoDB data dictionary cache. For debug builds, introduce the settable Boolean configuration parameter innodb_evict_tables_on_commit_debug that can be set to request InnoDB to attempt to evict table definitions from the data dictionary cache whenever a transaction is committed. This has been tested on 10.3 and 10.4 with the following: ./mysql-test-run.pl --mysqld=--loose-innodb-evict-tables-on-commit-debug You can also use the following: SET GLOBAL innodb_evict_tables_on_commit_debug=ON; SET GLOBAL innodb_evict_tables_on_commit_debug=OFF; The parameter affects the commit (or rollback or abort) of transactions that have modified persistent InnoDB tables. --- storage/innobase/handler/ha_innodb.cc | 7 +++++++ storage/innobase/include/srv0srv.h | 3 ++- storage/innobase/trx/trx0trx.cc | 38 ++++++++++++++++++++++++++++++++++- 3 files changed, 46 insertions(+), 2 deletions(-) (limited to 'storage') diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 59a7c8641f8..538a542bdf8 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -235,6 +235,7 @@ extern my_bool srv_background_scrub_data_compressed; extern uint srv_background_scrub_data_interval; extern uint srv_background_scrub_data_check_interval; #ifdef UNIV_DEBUG +my_bool innodb_evict_tables_on_commit_debug; extern my_bool srv_scrub_force_testing; #endif @@ -19900,6 +19901,11 @@ static MYSQL_SYSVAR_BOOL(trx_purge_view_update_only_debug, " but the each purges were not done yet.", NULL, NULL, FALSE); +static MYSQL_SYSVAR_BOOL(evict_tables_on_commit_debug, + innodb_evict_tables_on_commit_debug, PLUGIN_VAR_OPCMDARG, + "On transaction commit, try to evict tables from the data dictionary cache.", + NULL, NULL, FALSE); + static MYSQL_SYSVAR_UINT(data_file_size_debug, srv_sys_space_size_debug, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, @@ -20265,6 +20271,7 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(trx_rseg_n_slots_debug), MYSQL_SYSVAR(limit_optimistic_insert_debug), MYSQL_SYSVAR(trx_purge_view_update_only_debug), + MYSQL_SYSVAR(evict_tables_on_commit_debug), MYSQL_SYSVAR(data_file_size_debug), MYSQL_SYSVAR(fil_make_page_dirty_debug), MYSQL_SYSVAR(saved_page_number_debug), diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index d1596764bd1..db2218633b3 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -3,7 +3,7 @@ Copyright (c) 1995, 2017, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2008, 2009, Google Inc. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2018, MariaDB Corporation. +Copyright (c) 2013, 2019, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -536,6 +536,7 @@ extern my_bool srv_ibuf_disable_background_merge; #endif /* UNIV_DEBUG || UNIV_IBUF_DEBUG */ #ifdef UNIV_DEBUG +extern my_bool innodb_evict_tables_on_commit_debug; extern my_bool srv_sync_debug; extern my_bool srv_purge_view_update_only_debug; diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index 89ac734652e..723e8458c6d 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -1230,6 +1230,22 @@ trx_update_mod_tables_timestamp( const time_t now = time(NULL); trx_mod_tables_t::const_iterator end = trx->mod_tables.end(); +#ifdef UNIV_DEBUG +# if MYSQL_VERSION_ID >= 100405 +# define dict_sys_mutex dict_sys.mutex +# else +# define dict_sys_mutex dict_sys->mutex +# endif + + const bool preserve_tables = !innodb_evict_tables_on_commit_debug + || trx->is_recovered /* avoid trouble with XA recovery */ +# if 1 /* if dict_stats_exec_sql() were not playing dirty tricks */ + || mutex_own(&dict_sys_mutex) +# else /* this would be more proper way to do it */ + || trx->dict_operation_lock_mode || trx->dict_operation +# endif + ; +#endif for (trx_mod_tables_t::const_iterator it = trx->mod_tables.begin(); it != end; @@ -1243,7 +1259,27 @@ trx_update_mod_tables_timestamp( "garbage" in table->update_time is justified because protecting it with a latch here would be too performance intrusive. */ - it->first->update_time = now; + dict_table_t* table = it->first; + table->update_time = now; +#ifdef UNIV_DEBUG + if (preserve_tables || table->get_ref_count()) { + /* do not evict when committing DDL operations + or if some other transaction is holding the + table handle */ + continue; + } + /* recheck while holding the mutex that blocks + table->acquire() */ + mutex_enter(&dict_sys_mutex); + if (!table->get_ref_count()) { +# if MYSQL_VERSION_ID >= 100405 + dict_sys.remove(table, true); +# else + dict_table_remove_from_cache_low(table, true); +# endif + } + mutex_exit(&dict_sys_mutex); +#endif } trx->mod_tables.clear(); -- cgit v1.2.1 From d4866c7d0db3183b6b3d828130ccd8fb9a98600b Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Mon, 26 Aug 2019 19:19:12 +0300 Subject: fix clang warnings --- storage/connect/ha_connect.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'storage') diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index fc65c578e8d..40f9e90ff35 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -1317,7 +1317,7 @@ char *ha_connect::GetRealString(PCSZ s) { char *sv; - if (IsPartitioned() && s && partname && *partname) { + if (IsPartitioned() && s && *partname) { sv= (char*)PlugSubAlloc(xp->g, NULL, 0); sprintf(sv, s, partname); PlugSubAlloc(xp->g, NULL, strlen(sv) + 1); -- cgit v1.2.1 From 2842ae03bc9e17eb107041be208d53b117243a86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 28 Aug 2019 15:27:35 +0300 Subject: Remove a bogus comment Changes of PAGE_MAX_TRX_ID must be redo-logged for correctness. That was fixed in the InnoDB Plugin for MySQL 5.1 already. --- storage/innobase/lock/lock0lock.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'storage') diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index a674a38df2d..8cb9f19d93d 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -1230,9 +1230,7 @@ lock_sec_rec_some_has_impl( /* Some transaction may have an implicit x-lock on the record only if the max trx id for the page >= min trx id for the trx list, or - database recovery is running. We do not write the changes of a page - max trx id to the log, and therefore during recovery, this value - for a page may be incorrect. */ + database recovery is running. */ if (max_trx_id < trx_rw_min_trx_id() && !recv_recovery_is_on()) { -- cgit v1.2.1 From d4246e25e5057167a32506e3bd823db3e1731aa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 29 Aug 2019 18:22:41 +0300 Subject: After-merge fix Merge a part of commit 25af2a183b03f1d50da2466de84559a09a3e182c. This was forgotten in the merge commit e41eb044f19e3ee9b881344565924dd6b9d20e1a. --- storage/innobase/include/trx0sys.h | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'storage') diff --git a/storage/innobase/include/trx0sys.h b/storage/innobase/include/trx0sys.h index 64e97829043..8705809a373 100644 --- a/storage/innobase/include/trx0sys.h +++ b/storage/innobase/include/trx0sys.h @@ -644,11 +644,8 @@ public: { mutex_enter(&element->mutex); lf_hash_search_unpin(pins); - trx= element->trx; - if (!trx); - else if (UNIV_UNLIKELY(trx_id != trx->id)) - trx= NULL; - else { + if ((trx= element->trx)) { + DBUG_ASSERT(trx_id == trx->id); if (do_ref_count) trx->reference(); ut_d(validate_element(trx)); -- cgit v1.2.1 From f42a23178edee5dd1fb6e0f76bbe88ea2a98a045 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 30 Aug 2019 12:51:37 +0300 Subject: MDEV-20425: Fix -Wimplicit-fallthrough With --skip-debug-assert, DBUG_ASSERT(false) will allow execution to continue. Hence, we will need /* fall through */ after them. Some DBUG_ASSERT(0) were replaced by break; when the switch () statement was followed by DBUG_ASSERT(0). --- storage/innobase/fil/fil0fil.cc | 1 + storage/innobase/handler/handler0alter.cc | 2 +- storage/innobase/mtr/mtr0mtr.cc | 2 +- storage/innobase/row/row0log.cc | 2 ++ storage/innobase/row/row0uins.cc | 3 ++- 5 files changed, 7 insertions(+), 3 deletions(-) (limited to 'storage') diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 5bf7fb9df58..e22b1c4e314 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -826,6 +826,7 @@ static void fil_flush_low(fil_space_t* space, bool metadata = false) switch (space->purpose) { case FIL_TYPE_TEMPORARY: ut_ad(0); // we already checked for this + /* fall through */ case FIL_TYPE_TABLESPACE: case FIL_TYPE_IMPORT: fil_n_pending_tablespace_flushes++; diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index d119fca82b6..a4ba9651dbe 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -2135,7 +2135,7 @@ innobase_col_to_mysql( case DATA_SYS: /* These column types should never be shipped to MySQL. */ ut_ad(0); - + /* fall through */ case DATA_FLOAT: case DATA_DOUBLE: case DATA_DECIMAL: diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc index 751b7ec36ec..5504eb618fe 100644 --- a/storage/innobase/mtr/mtr0mtr.cc +++ b/storage/innobase/mtr/mtr0mtr.cc @@ -754,7 +754,7 @@ mtr_t::Command::prepare_write() switch (m_impl->m_log_mode) { case MTR_LOG_SHORT_INSERTS: ut_ad(0); - /* fall through (write no redo log) */ + /* fall through */ case MTR_LOG_NO_REDO: case MTR_LOG_NONE: ut_ad(m_impl->m_log.size() == 0); diff --git a/storage/innobase/row/row0log.cc b/storage/innobase/row/row0log.cc index 71fb7fc5f7f..2b1c2d9e0be 100644 --- a/storage/innobase/row/row0log.cc +++ b/storage/innobase/row/row0log.cc @@ -1790,6 +1790,7 @@ row_log_table_apply_insert( break; default: ut_ad(0); + /* fall through */ case DB_INVALID_NULL: ut_ad(row == NULL); return(error); @@ -2078,6 +2079,7 @@ row_log_table_apply_update( break; default: ut_ad(0); + /* fall through */ case DB_INVALID_NULL: ut_ad(row == NULL); return(error); diff --git a/storage/innobase/row/row0uins.cc b/storage/innobase/row/row0uins.cc index 173233f1cd5..77c54aade87 100644 --- a/storage/innobase/row/row0uins.cc +++ b/storage/innobase/row/row0uins.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1997, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, 2018, MariaDB Corporation. +Copyright (c) 2017, 2019, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -583,6 +583,7 @@ row_undo_ins( switch (node->rec_type) { default: ut_ad(!"wrong undo record type"); + /* fall through */ case TRX_UNDO_INSERT_REC: /* Skip the clustered index (the first index) */ node->index = dict_table_get_next_index(node->index); -- cgit v1.2.1 From 1688a22612e1fed02056cdf37574ffa2ef33aa2d Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Fri, 30 Aug 2019 14:13:00 +0300 Subject: MDEV-18384: rocksdb.index_merge_rocksdb2 test fails Merge the changes to include/index_merge*inc from the upstream. The changes add this command in many places: +if ($engine_type == RocksDB) +{ + set global rocksdb_force_flush_memtable_now=1; +} also add it in one more place to make the test truly stable. --- .../mysql-test/rocksdb/include/index_merge1.inc | 60 ++++++++++++++++++++++ .../mysql-test/rocksdb/include/index_merge2.inc | 28 ++++++++++ .../mysql-test/rocksdb/include/index_merge_ror.inc | 36 +++++++++++++ .../rocksdb/include/index_merge_ror_cpk.inc | 8 +++ .../rocksdb/r/index_merge_rocksdb2.result | 32 ++++++------ 5 files changed, 148 insertions(+), 16 deletions(-) (limited to 'storage') diff --git a/storage/rocksdb/mysql-test/rocksdb/include/index_merge1.inc b/storage/rocksdb/mysql-test/rocksdb/include/index_merge1.inc index 91bcc68adb4..b5cf7bff763 100644 --- a/storage/rocksdb/mysql-test/rocksdb/include/index_merge1.inc +++ b/storage/rocksdb/mysql-test/rocksdb/include/index_merge1.inc @@ -58,6 +58,10 @@ while ($1) eval set @d=@d*2; dec $1; } +if ($engine_type == RocksDB) +{ + set global rocksdb_force_flush_memtable_now=1; +} --enable_query_log analyze table t0; @@ -329,6 +333,10 @@ alter table t2 add index i321(key3, key2, key1); -- disable_query_log -- disable_result_log analyze table t2; +if ($engine_type == RocksDB) +{ + set global rocksdb_force_flush_memtable_now=1; +} -- enable_result_log -- enable_query_log @@ -371,6 +379,10 @@ insert into t4 select key1,key1,key1 div 10, key1 % 10, key1 % 10, key1 from t0; -- disable_query_log -- disable_result_log +if ($engine_type == RocksDB) +{ + set global rocksdb_force_flush_memtable_now=1; +} analyze table t4; -- enable_result_log -- enable_query_log @@ -397,6 +409,10 @@ insert into t1 select * from t0; -- disable_query_log -- disable_result_log +if ($engine_type == RocksDB) +{ + set global rocksdb_force_flush_memtable_now=1; +} analyze table t1; -- enable_result_log -- enable_query_log @@ -463,6 +479,10 @@ update t3 set key9=key1,keyA=key1,keyB=key1,keyC=key1; -- disable_query_log -- disable_result_log +if ($engine_type == RocksDB) +{ + set global rocksdb_force_flush_memtable_now=1; +} analyze table t3; -- enable_result_log -- enable_query_log @@ -486,6 +506,10 @@ update t0 set key8=123 where key1 < 3 or key2 < 4; -- disable_query_log -- disable_result_log +if ($engine_type == RocksDB) +{ + set global rocksdb_force_flush_memtable_now=1; +} analyze table t0; -- enable_result_log -- enable_query_log @@ -496,6 +520,10 @@ select * from t0 where key1 < 3 or key2 < 4; delete from t0 where key1 < 3 or key2 < 4; -- disable_query_log -- disable_result_log +if ($engine_type == RocksDB) +{ + set global rocksdb_force_flush_memtable_now=1; +} analyze table t0; -- enable_result_log -- enable_query_log @@ -509,6 +537,10 @@ create table t4 (a int); insert into t4 values (1),(4),(3); -- disable_query_log -- disable_result_log +if ($engine_type == RocksDB) +{ + set global rocksdb_force_flush_memtable_now=1; +} analyze table t4; -- enable_result_log -- enable_query_log @@ -533,6 +565,10 @@ select max(A.key1 + B.key1 + A.key2 + B.key2 + A.key3 + B.key3 + A.key4 + B.key4 update t0 set key1=1; -- disable_query_log -- disable_result_log +if ($engine_type == RocksDB) +{ + set global rocksdb_force_flush_memtable_now=1; +} analyze table t0; -- enable_result_log -- enable_query_log @@ -556,6 +592,10 @@ update t0 set key2=1, key3=1, key4=1, key5=1,key6=1,key7=1 where key7 < 500; -- disable_query_log -- disable_result_log +if ($engine_type == RocksDB) +{ + set global rocksdb_force_flush_memtable_now=1; +} analyze table t0; -- enable_result_log -- enable_query_log @@ -612,6 +652,10 @@ select count(*) from t1; -- disable_query_log -- disable_result_log +if ($engine_type == RocksDB) +{ + set global rocksdb_force_flush_memtable_now=1; +} analyze table t1; -- enable_result_log -- enable_query_log @@ -652,6 +696,10 @@ create table t3 ( -- disable_query_log -- disable_result_log +if ($engine_type == RocksDB) +{ + set global rocksdb_force_flush_memtable_now=1; +} analyze table t0; analyze table t1; analyze table t2; @@ -733,6 +781,10 @@ insert into t2 select * from t1; -- disable_query_log -- disable_result_log +if ($engine_type == RocksDB) +{ + set global rocksdb_force_flush_memtable_now=1; +} analyze table t1; analyze table t2; -- enable_result_log @@ -787,6 +839,10 @@ insert into t3 values (1,1,'data'); -- disable_query_log -- disable_result_log +if ($engine_type == RocksDB) +{ + set global rocksdb_force_flush_memtable_now=1; +} analyze table t0; analyze table t1; analyze table t2; @@ -822,6 +878,10 @@ INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; -- disable_query_log -- disable_result_log +if ($engine_type == RocksDB) +{ + set global rocksdb_force_flush_memtable_now=1; +} analyze table t1; -- enable_result_log -- enable_query_log diff --git a/storage/rocksdb/mysql-test/rocksdb/include/index_merge2.inc b/storage/rocksdb/mysql-test/rocksdb/include/index_merge2.inc index 9fd2390c401..7e5cec40a80 100644 --- a/storage/rocksdb/mysql-test/rocksdb/include/index_merge2.inc +++ b/storage/rocksdb/mysql-test/rocksdb/include/index_merge2.inc @@ -43,6 +43,10 @@ while ($1) -- disable_query_log -- disable_result_log +if ($engine_type == RocksDB) +{ + set global rocksdb_force_flush_memtable_now=1; +} analyze table t1; -- enable_result_log -- enable_query_log @@ -68,6 +72,10 @@ alter table t1 add primary key (str1, zeroval, str2, str3); -- disable_query_log -- disable_result_log +if ($engine_type == RocksDB) +{ + set global rocksdb_force_flush_memtable_now=1; +} analyze table t1; -- enable_result_log -- enable_query_log @@ -101,6 +109,10 @@ while ($1) -- disable_query_log -- disable_result_log +if ($engine_type == RocksDB) +{ + set global rocksdb_force_flush_memtable_now=1; +} analyze table t1; -- enable_result_log -- enable_query_log @@ -143,6 +155,10 @@ select count(*) from t1; -- disable_query_log -- disable_result_log +if ($engine_type == RocksDB) +{ + set global rocksdb_force_flush_memtable_now=1; +} analyze table t2; -- enable_result_log -- enable_query_log @@ -379,6 +395,10 @@ update t1 set key2=key1,key3=key1; -- disable_query_log -- disable_result_log +if ($engine_type == RocksDB) +{ + set global rocksdb_force_flush_memtable_now=1; +} analyze table t1; -- enable_result_log -- enable_query_log @@ -417,6 +437,10 @@ INSERT INTO t1 VALUES -- disable_query_log -- disable_result_log +if ($engine_type == RocksDB) +{ + set global rocksdb_force_flush_memtable_now=1; +} analyze table t1; -- enable_result_log -- enable_query_log @@ -472,6 +496,10 @@ INSERT INTO t1 VALUES (1,1,'a'), (2,2,'b'); -- disable_query_log -- disable_result_log +if ($engine_type == RocksDB) +{ + set global rocksdb_force_flush_memtable_now=1; +} analyze table t1; -- enable_result_log -- enable_query_log diff --git a/storage/rocksdb/mysql-test/rocksdb/include/index_merge_ror.inc b/storage/rocksdb/mysql-test/rocksdb/include/index_merge_ror.inc index ed270802db1..21219d1aa95 100644 --- a/storage/rocksdb/mysql-test/rocksdb/include/index_merge_ror.inc +++ b/storage/rocksdb/mysql-test/rocksdb/include/index_merge_ror.inc @@ -119,6 +119,10 @@ select count(*) from t1; -- disable_query_log -- disable_result_log +if ($engine_type == RocksDB) +{ + set global rocksdb_force_flush_memtable_now=1; +} analyze table t0; analyze table t1; -- enable_result_log @@ -141,6 +145,10 @@ insert into t1 (key1, key2, key3, key4, filler1) values (-1, -1, 100, 100, 'key4 -- disable_query_log -- disable_result_log +if ($engine_type == RocksDB) +{ + set global rocksdb_force_flush_memtable_now=1; +} analyze table t1; -- enable_result_log -- enable_query_log @@ -193,6 +201,10 @@ update t1 set key1=200,key2=200 where key1=100 and key2=100; delete from t1 where key1=200 and key2=200; -- disable_query_log -- disable_result_log +if ($engine_type == RocksDB) +{ + set global rocksdb_force_flush_memtable_now=1; +} analyze table t1; -- enable_result_log -- enable_query_log @@ -210,6 +222,10 @@ delete from t1 where key3=100 and key4=100; -- disable_query_log -- disable_result_log +if ($engine_type == RocksDB) +{ + set global rocksdb_force_flush_memtable_now=1; +} analyze table t1; -- enable_result_log -- enable_query_log @@ -236,6 +252,10 @@ insert into t1 (key1, key2, key3, key4, filler1) values (100, 100, 200, 200,'key -- disable_query_log -- disable_result_log +if ($engine_type == RocksDB) +{ + set global rocksdb_force_flush_memtable_now=1; +} analyze table t1; -- enable_result_log -- enable_query_log @@ -250,6 +270,10 @@ insert into t1 (key1, key2, key3, key4, filler1) values (-1, -1, -1, 200,'key4') -- disable_query_log -- disable_result_log +if ($engine_type == RocksDB) +{ + set global rocksdb_force_flush_memtable_now=1; +} analyze table t1; -- enable_result_log -- enable_query_log @@ -264,6 +288,10 @@ insert into t1 (key1, key2, key3, key4, filler1) values (-1, -1, 200, -1,'key3') -- disable_query_log -- disable_result_log +if ($engine_type == RocksDB) +{ + set global rocksdb_force_flush_memtable_now=1; +} analyze table t1; -- enable_result_log -- enable_query_log @@ -392,6 +420,10 @@ select count(a) from t2 where b='BBBBBBBB'; -- disable_query_log -- disable_result_log +if ($engine_type == RocksDB) +{ + set global rocksdb_force_flush_memtable_now=1; +} analyze table t2; -- enable_result_log -- enable_query_log @@ -405,6 +437,10 @@ select count(a) from t2 ignore index(a,b) where a='AAAAAAAA' and b='AAAAAAAA'; insert into t2 values ('ab', 'ab', 'uh', 'oh'); -- disable_query_log -- disable_result_log +if ($engine_type == RocksDB) +{ + set global rocksdb_force_flush_memtable_now=1; +} analyze table t2; -- enable_result_log -- enable_query_log diff --git a/storage/rocksdb/mysql-test/rocksdb/include/index_merge_ror_cpk.inc b/storage/rocksdb/mysql-test/rocksdb/include/index_merge_ror_cpk.inc index 9f1e2d77621..f0d18a50bff 100644 --- a/storage/rocksdb/mysql-test/rocksdb/include/index_merge_ror_cpk.inc +++ b/storage/rocksdb/mysql-test/rocksdb/include/index_merge_ror_cpk.inc @@ -68,6 +68,10 @@ set autocommit=1; -- disable_query_log -- disable_result_log +if ($engine_type == RocksDB) +{ + set global rocksdb_force_flush_memtable_now=1; +} analyze table t1; -- enable_result_log -- enable_query_log @@ -159,6 +163,10 @@ WHERE (SELECT COUNT(*) FROM t2 WHERE t2.f3 = 'h' AND t2.f2 = t1.f1) = 0 AND t1.f -- disable_query_log -- disable_result_log +if ($engine_type == RocksDB) +{ + set global rocksdb_force_flush_memtable_now=1; +} analyze table t1; analyze table t2; -- enable_result_log diff --git a/storage/rocksdb/mysql-test/rocksdb/r/index_merge_rocksdb2.result b/storage/rocksdb/mysql-test/rocksdb/r/index_merge_rocksdb2.result index e0248737381..4603e049724 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/index_merge_rocksdb2.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/index_merge_rocksdb2.result @@ -26,7 +26,7 @@ Table Op Msg_type Msg_text test.t0 analyze status OK explain select * from t0 where key1 < 3 or key1 > 1020; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t0 range i1 i1 4 NULL 2 Using index condition; Using where +1 SIMPLE t0 range i1 i1 4 NULL 4 Using index condition; Using where explain select * from t0 where key1 < 3 or key2 > 1020; id select_type table type possible_keys key key_len ref rows Extra @@ -267,12 +267,12 @@ select * from t0,t1 where (t0.key1=t1.key1) and (t0.key1=3 or t0.key2<4) and t1.key1=2; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t0 ref i1,i2 i1 4 const 2 Using where -1 SIMPLE t1 ref i1 i1 4 const 1 +1 SIMPLE t1 ref i1 i1 4 const 2 explain select * from t0,t1 where t0.key1 = 5 and (t1.key1 = t0.key1 or t1.key8 = t0.key1); id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t0 ref i1 i1 4 const 1 -1 SIMPLE t1 index_merge i1,i8 i1,i8 4,4 NULL 2 Using union(i1,i8); Using where; Using join buffer (flat, BNL join) +1 SIMPLE t0 ref i1 i1 4 const 2 +1 SIMPLE t1 index_merge i1,i8 i1,i8 4,4 NULL 4 Using union(i1,i8); Using where; Using join buffer (flat, BNL join) explain select * from t0,t1 where t0.key1 < 3 and (t1.key1 = t0.key1 or t1.key8 = t0.key1); id select_type table type possible_keys key key_len ref rows Extra @@ -286,7 +286,7 @@ id select_type table type possible_keys key key_len ref rows Extra NULL UNION RESULT ALL NULL NULL NULL NULL NULL explain select * from (select * from t1 where key1 = 3 or key2 =3) as Z where key8 >5; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 index_merge i1,i2,i8 i1,i2 4,4 NULL 2 Using union(i1,i2); Using where +1 SIMPLE t1 range i1,i2,i8 i8 4 NULL 2 Using index condition; Using where create table t3 like t0; insert into t3 select * from t0; alter table t3 add key9 int not null, add index i9(key9); @@ -392,10 +392,10 @@ count(*) 8704 explain select * from t1 WHERE cola = 'foo' AND colb = 'bar'; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 index_merge cola,colb cola,colb 3,3 NULL # Using intersect(cola,colb); Using where +1 SIMPLE t1 ref cola,colb cola 3 const # Using index condition; Using where explain select * from t1 force index(cola,colb) WHERE cola = 'foo' AND colb = 'bar'; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 index_merge cola,colb cola,colb 3,3 NULL # Using intersect(cola,colb); Using where +1 SIMPLE t1 ref cola,colb cola 3 const # Using index condition; Using where drop table t1; CREATE TABLE t1(a INT); INSERT INTO t1 VALUES(1); @@ -603,7 +603,7 @@ count(*) 64801 explain select key1,key2 from t1 where key1=100 and key2=100; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 index_merge key1,key2 key2,key1 5,5 NULL # Using intersect(key2,key1); Using where; Using index +1 SIMPLE t1 index_merge key1,key2 key1,key2 5,5 NULL # Using intersect(key1,key2); Using where; Using index select key1,key2 from t1 where key1=100 and key2=100; key1 key2 100 100 @@ -789,7 +789,7 @@ show warnings; Level Code Message explain select pk from t1 where key1 = 1 and key2 = 1; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 ref key1,key2 key1 5 const 2 Using where +1 SIMPLE t1 index_merge key1,key2 key2,key1 4,5 NULL 1 Using intersect(key2,key1); Using where; Using index select pk from t1 where key2 = 1 and key1 = 1; pk 26 @@ -1057,7 +1057,7 @@ SELECT a FROM t1 WHERE c = 1 AND b = 1 AND d = 1; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 ref c,bd c 5 const 2 Using where +1 SIMPLE t1 index_merge c,bd c,bd 5,10 NULL 1 Using intersect(c,bd); Using where; Using index CREATE TABLE t2 ( a INT ) SELECT a FROM t1 @@ -1288,7 +1288,7 @@ primary key (pk1, pk2) ); explain select * from t1 where pk1 = 1 and pk2 < 80 and key1=0; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range PRIMARY,key1 key1 12 NULL ROWS Using index condition +1 SIMPLE t1 range PRIMARY,key1 PRIMARY 8 NULL ROWS Using where select * from t1 where pk1 = 1 and pk2 < 80 and key1=0; pk1 pk2 key1 key2 pktail1ok pktail2ok pktail3bad pktail4bad pktail5bad pk2copy badkey filler1 filler2 1 10 0 0 0 0 0 0 0 10 0 filler-data-10 filler2 @@ -1321,13 +1321,13 @@ id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 ref key1 key1 4 const ROWS Using where explain select * from t1 where pk1 < 7500 and key1 = 10; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range PRIMARY,key1 key1 8 NULL ROWS Using index condition +1 SIMPLE t1 range PRIMARY,key1 PRIMARY 4 NULL ROWS Using where explain select * from t1 where pktail1ok=1 and key1=10; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 index_merge key1,pktail1ok key1,pktail1ok 4,4 NULL 1 Using intersect(key1,pktail1ok); Using where +1 SIMPLE t1 ref key1,pktail1ok key1 4 const 2 Using where explain select * from t1 where pktail2ok=1 and key1=10; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 index_merge key1,pktail2ok key1,pktail2ok 4,4 NULL 1 Using intersect(key1,pktail2ok); Using where +1 SIMPLE t1 ref key1,pktail2ok key1 4 const 2 Using where explain select * from t1 where (pktail2ok=1 and pk1< 50000) or key1=10; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 index_merge PRIMARY,key1,pktail2ok PRIMARY,key1 4,4 NULL ROWS Using union(PRIMARY,key1); Using where @@ -1393,7 +1393,7 @@ EXPLAIN SELECT t1.f1 FROM t1 WHERE (SELECT COUNT(*) FROM t2 WHERE t2.f3 = 'h' AND t2.f2 = t1.f1) = 0 AND t1.f1 = 2; id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t1 const PRIMARY PRIMARY 4 const 1 Using index -2 SUBQUERY t2 ref f2,f3 f2 5 const 1 Using where +2 SUBQUERY t2 index_merge f2,f3 f3,f2 2,5 NULL 1 Using intersect(f3,f2); Using where; Using index DROP TABLE t1,t2; set global rocksdb_force_flush_memtable_now=1; # @@ -1412,5 +1412,5 @@ INSERT INTO t1 SELECT id + 8, id2 + 8, id3 +8 FROM t1; INSERT INTO t1 SELECT id + 16, 7, 0 FROM t1; EXPLAIN SELECT SQL_NO_CACHE count(*) FROM t1 WHERE id2=7 AND id3=0; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 ref id2,id3,covering_index covering_index 8 const,const 1 Using index +1 SIMPLE t1 ref id2,id3,covering_index covering_index 8 const,const 2 Using index DROP TABLE t1; -- cgit v1.2.1 From 17336f6d3063616443a8f60091fd1f8c694b34f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 30 Aug 2019 14:37:50 +0300 Subject: MDEV-20066 Wrong value on instantly added column after DELETE and UPDATE row_upd_build_difference_binary(): Correctly handle the case where columns (or clustered index fields) have been added since the 'entry' was originally created. In this case, the update vector must replace any missing columns with the default values of the instantly added columns. --- storage/innobase/row/row0upd.cc | 62 +++++++++++++++++++---------------------- 1 file changed, 29 insertions(+), 33 deletions(-) (limited to 'storage') diff --git a/storage/innobase/row/row0upd.cc b/storage/innobase/row/row0upd.cc index c22d422ed86..71494f4f12f 100644 --- a/storage/innobase/row/row0upd.cc +++ b/storage/innobase/row/row0upd.cc @@ -1049,22 +1049,21 @@ row_upd_build_difference_binary( TABLE* mysql_table, dberr_t* error) { - upd_field_t* upd_field; ulint len; upd_t* update; ulint n_diff; ulint trx_id_pos; - ulint i; ulint offsets_[REC_OFFS_NORMAL_SIZE]; - ulint n_fld = dtuple_get_n_fields(entry); - ulint n_v_fld = dtuple_get_n_v_fields(entry); + const ulint n_v_fld = dtuple_get_n_v_fields(entry); rec_offs_init(offsets_); /* This function is used only for a clustered index */ ut_a(dict_index_is_clust(index)); ut_ad(!index->table->skip_alter_undo); + ut_ad(entry->n_fields <= index->n_fields); + ut_ad(entry->n_fields >= index->n_core_fields); - update = upd_create(n_fld + n_v_fld, heap); + update = upd_create(index->n_fields + n_v_fld, heap); n_diff = 0; @@ -1079,7 +1078,7 @@ row_upd_build_difference_binary( ut_ad(rec_offs_validate(rec, index, offsets)); } - for (i = 0; i < n_fld; i++) { + for (ulint i = 0; i < entry->n_fields; i++) { const byte* data = rec_get_nth_cfield(rec, index, offsets, i, &len); const dfield_t* dfield = dtuple_get_nth_field(entry, i); @@ -1101,17 +1100,22 @@ row_upd_build_difference_binary( if (!dfield_is_ext(dfield) != !rec_offs_nth_extern(offsets, i) || !dfield_data_is_binary_equal(dfield, len, data)) { - - upd_field = upd_get_nth_field(update, n_diff); - - dfield_copy(&(upd_field->new_val), dfield); - - upd_field_set_field_no(upd_field, i, index); - - n_diff++; + upd_field_t* uf = upd_get_nth_field(update, n_diff++); + dfield_copy(&uf->new_val, dfield); + upd_field_set_field_no(uf, i, index); } } + for (ulint i = entry->n_fields; i < index->n_fields; i++) { + upd_field_t* uf = upd_get_nth_field(update, n_diff++); + const dict_col_t* col = dict_index_get_nth_col(index, i); + /* upd_create() zero-initialized uf */ + uf->new_val.data = const_cast(col->instant_value(&len)); + uf->new_val.len = static_cast(len); + dict_col_copy_type(col, &uf->new_val.type); + upd_field_set_field_no(uf, i, index); + } + /* Check the virtual columns updates. Even if there is no non-virtual column (base columns) change, we will still need to build the indexed virtual column value so that undo log would log them ( @@ -1136,7 +1140,7 @@ row_upd_build_difference_binary( &mysql_table, &record, &vcol_storage); - for (i = 0; i < n_v_fld; i++) { + for (ulint i = 0; i < n_v_fld; i++) { const dict_v_col_t* col = dict_table_get_nth_v_col(index->table, i); @@ -1164,24 +1168,16 @@ row_upd_build_difference_binary( entry, i); if (!dfield_data_is_binary_equal( - dfield, vfield->len, - static_cast(vfield->data))) { - upd_field = upd_get_nth_field(update, n_diff); - - upd_field->old_v_val = static_cast( - mem_heap_alloc( - heap, - sizeof *upd_field->old_v_val)); - - dfield_copy(upd_field->old_v_val, vfield); - - dfield_copy(&(upd_field->new_val), dfield); - - upd_field_set_v_field_no( - upd_field, i, index); - - n_diff++; - + dfield, vfield->len, + static_cast(vfield->data))) { + upd_field_t* uf = upd_get_nth_field(update, + n_diff++); + uf->old_v_val = static_cast( + mem_heap_alloc(heap, + sizeof *uf->old_v_val)); + dfield_copy(uf->old_v_val, vfield); + dfield_copy(&uf->new_val, dfield); + upd_field_set_v_field_no(uf, i, index); } } -- cgit v1.2.1 From 396da1a70548f80ecf781cf352b0d4a5740c54f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 30 Aug 2019 14:49:21 +0300 Subject: Enable some RocksDB tests that are enabled on 10.2 FIXME: rocksdb.drop_table hangs on shutdown, but is not disabled in 10.2 --- storage/rocksdb/mysql-test/rocksdb/t/disabled.def | 4 ---- 1 file changed, 4 deletions(-) (limited to 'storage') diff --git a/storage/rocksdb/mysql-test/rocksdb/t/disabled.def b/storage/rocksdb/mysql-test/rocksdb/t/disabled.def index 90050cc8cec..37710482e54 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/disabled.def +++ b/storage/rocksdb/mysql-test/rocksdb/t/disabled.def @@ -85,8 +85,6 @@ rpl_row_triggers : Requires read-free slave. compact_deletes: MDEV-12663 : rocksdb.compact_deletes times out and causes other tests to fail blind_delete_without_tx_api: MDEV-12286: rocksdb.blind_delete_without_tx_api test fails -unique_check: wrong error number -autoinc_vars_thread: debug sync point wait timed out information_schema: MDEV-14372: unstable testcase ## @@ -94,6 +92,4 @@ information_schema: MDEV-14372: unstable testcase ## mysqlbinlog_gtid_skip_empty_trans_rocksdb : MariaRocks: requires GTIDs -autoinc_debug: Fails with wrong results drop_table: Hangs on shutdown -allow_to_start_after_corruption : result difference and assertion failure -- cgit v1.2.1 From 7bccd2910f6cd02a49d62a739dfe64c25c6a8462 Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Tue, 3 Sep 2019 16:28:30 +0300 Subject: MDEV-20479 assertion failure in dict_table_get_nth_col() after INSTANT DROP COLUMN get_col_list_to_be_dropped() incorrectly returned uninteresting instantly dropped column which was missing in a new dict_index_t get_col_list_to_be_dropped(): rename to collect_columns_from_dropped_indexes and stop return dropped columns --- storage/innobase/handler/handler0alter.cc | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'storage') diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 722d75960bb..14d6527b764 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -9935,8 +9935,7 @@ commit_cache_rebuild( /** Set of column numbers */ typedef std::set, ut_allocator > col_set; -/** Store the column number of the columns in a list belonging -to indexes which are not being dropped. +/** Collect (not instantly dropped) columns from dropped indexes @param[in] ctx In-place ALTER TABLE context @param[in, out] drop_col_list list which will be set, containing columns which is part of index being dropped @@ -9945,7 +9944,7 @@ to indexes which are not being dropped. being dropped */ static void -get_col_list_to_be_dropped( +collect_columns_from_dropped_indexes( const ha_innobase_inplace_ctx* ctx, col_set& drop_col_list, col_set& drop_v_col_list) @@ -9966,6 +9965,12 @@ get_col_list_to_be_dropped( } else { ulint col_no = dict_col_get_no(idx_col); + if (ctx->col_map + && ctx->col_map[col_no] + == ULINT_UNDEFINED) { + // this column was instantly dropped + continue; + } drop_col_list.insert(col_no); } } @@ -10287,7 +10292,7 @@ commit_cache_norebuild( /* Check if the column, part of an index to be dropped is part of any other index which is not being dropped. If it so, then set the ord_part of the column to 0. */ - get_col_list_to_be_dropped(ctx, drop_list, v_drop_list); + collect_columns_from_dropped_indexes(ctx, drop_list, v_drop_list); for (col_it = drop_list.begin(); col_it != drop_list.end(); ++col_it) { if (!check_col_exists_in_indexes(ctx->new_table, -- cgit v1.2.1 From 18af13b88ba580562981a190c25da128a2e9db26 Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Tue, 3 Sep 2019 16:31:10 +0300 Subject: [NFC] range-forify loops --- storage/innobase/handler/handler0alter.cc | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) (limited to 'storage') diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 14d6527b764..f280c735e05 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -10287,25 +10287,21 @@ commit_cache_norebuild( col_set drop_list; col_set v_drop_list; - col_set::const_iterator col_it; /* Check if the column, part of an index to be dropped is part of any other index which is not being dropped. If it so, then set the ord_part of the column to 0. */ collect_columns_from_dropped_indexes(ctx, drop_list, v_drop_list); - for (col_it = drop_list.begin(); col_it != drop_list.end(); ++col_it) { - if (!check_col_exists_in_indexes(ctx->new_table, - *col_it, false)) { - ctx->new_table->cols[*col_it].ord_part = 0; + for (ulint col : drop_list) { + if (!check_col_exists_in_indexes(ctx->new_table, col, false)) { + ctx->new_table->cols[col].ord_part = 0; } } - for (col_it = v_drop_list.begin(); - col_it != v_drop_list.end(); ++col_it) { - if (!check_col_exists_in_indexes(ctx->new_table, - *col_it, true)) { - ctx->new_table->v_cols[*col_it].m_col.ord_part = 0; + for (ulint col : v_drop_list) { + if (!check_col_exists_in_indexes(ctx->new_table, col, true)) { + ctx->new_table->v_cols[col].m_col.ord_part = 0; } } -- cgit v1.2.1