- Fix bug making REST table fail using CURL

  This when the HTTP contains & characters
  modified:   storage/connect/tabbson.cpp
  modified:   storage/connect/tabjson.cpp

- Make stringfy option work on only one Json item
  modified:   storage/connect/tabbson.cpp
  modified:   storage/connect/tabbson.h
  modified:   storage/connect/tabjson.cpp
  modified:   storage/connect/tabjson.h

- Make Json/Bson DATE columns accept JSON date syntax
  modified:   storage/connect/tabbson.cpp
  modified:   storage/connect/tabjson.cpp

- Fix bug making REST table default file not being
  erased when dropping the table
  modified:   storage/connect/tabbson.cpp
  modified:   storage/connect/tabjson.cpp
  modified:   storage/connect/tabrest.cpp
  modified:   storage/connect/tabxml.cpp

- Suppress CHAR(36) --> VARCHAR(36) when DEVELOPMENT
  This was fixed in MyClient
  modified:   storage/connect/ha_connect.cc

7 files changed, 87 insertions, 61 deletions

diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc
index 69646e22e30..7682063c410 100644
--- a/storage/connect/ha_connect.cc
+++ b/storage/connect/ha_connect.cc
@@ -5391,12 +5391,7 @@ static bool add_field(String* sql, TABTYPE ttp, const char* field_name, int typ,
 	                    int len, int dec, char* key, uint tm, const char* rem,
 	                    char* dft, char* xtra, char* fmt, int flag, bool dbf, char v)
 {
-#if defined(DEVELOPMENT)
-	// Some client programs regard CHAR(36) as GUID
-	char var = (len > 255 || len == 36) ? 'V' : v;
-#else
 	char var = (len > 255) ? 'V' : v;
-#endif
 	bool q, error = false;
 	const char* type = PLGtoMYSQLtype(typ, dbf, var);
 
diff --git a/storage/connect/tabbson.cpp b/storage/connect/tabbson.cpp
index bbc2c54c80f..8477d22d364 100644
--- a/storage/connect/tabbson.cpp
+++ b/storage/connect/tabbson.cpp
@@ -1,6 +1,6 @@
 /************* tabbson C++ Program Source Code File (.CPP) *************/
-/* PROGRAM NAME: tabbson     Version 1.0                               */
-/*  (C) Copyright to the author Olivier BERTRAND          2020         */
+/* PROGRAM NAME: tabbson     Version 1.1                               */
+/*  (C) Copyright to the author Olivier BERTRAND          2020 - 2021  */
 /*  This program are the BSON class DB execution routines.             */
 /***********************************************************************/
 
@@ -158,8 +158,9 @@ BSONDISC::BSONDISC(PGLOBAL g, uint* lg)
   bp = NULL;
   row = NULL;
   sep = NULL;
+  strfy = NULL;
   i = n = bf = ncol = lvl = sz = limit = 0;
-  all = strfy = false;
+  all = false;
 } // end of BSONDISC constructor
 
 int BSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt)
@@ -173,7 +174,7 @@ int BSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt)
   sep = GetStringTableOption(g, topt, "Separator", ".");
   sz = GetIntegerTableOption(g, topt, "Jsize", 1024);
   limit = GetIntegerTableOption(g, topt, "Limit", 10);
-  strfy = GetBooleanTableOption(g, topt, "Stringify", false);
+  strfy = GetStringTableOption(g, topt, "Stringify", NULL);
 
   /*********************************************************************/
   /*  Open the input file.                                             */
@@ -186,6 +187,9 @@ int BSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt)
 #endif   // ZIP_SUPPORT
   tdp->Fn = GetStringTableOption(g, topt, "Filename", NULL);
 
+  if (!tdp->Fn && topt->http)
+    tdp->Fn = GetStringTableOption(g, topt, "Subtype", NULL);
+
   if (!(tdp->Database = SetPath(g, db)))
     return 0;
 
@@ -199,7 +203,8 @@ int BSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt)
   if (!tdp->Fn && !tdp->Uri) {
     strcpy(g->Message, MSG(MISSING_FNAME));
     return 0;
-  } // endif Fn
+  } else
+    topt->subtype = NULL;
 
   if (tdp->Fn) {
     //  We used the file name relative to recorded datapath
@@ -428,7 +433,7 @@ bool BSONDISC::Find(PGLOBAL g, PBVAL jvp, PCSZ key, int j)
     jcol.Type = TYPE_UNKNOWN;
     jcol.Len = jcol.Scale = 0;
     jcol.Cbn = true;
-  } else  if (j < lvl) {
+  } else  if (j < lvl && !(strfy && !stricmp(strfy, colname))) {
     if (!fmt[bf])
       strcat(fmt, colname);
 
@@ -499,7 +504,7 @@ bool BSONDISC::Find(PGLOBAL g, PBVAL jvp, PCSZ key, int j)
     } // endswitch Type
 
   } else if (lvl >= 0) {
-    if (strfy) {
+    if (strfy && !stricmp(strfy, colname)) {
       if (!fmt[bf])
         strcat(fmt, colname);
 
@@ -731,7 +736,6 @@ void BCUTIL::SetJsonValue(PGLOBAL g, PVAL vp, PBVAL jvp)
     case TYPE_FLOAT:
       switch (vp->GetType()) {
         case TYPE_STRING:
-        case TYPE_DATE:
         case TYPE_DECIM:
           vp->SetValue_psz(GetString(jvp));
           break;
@@ -750,6 +754,16 @@ void BCUTIL::SetJsonValue(PGLOBAL g, PVAL vp, PBVAL jvp)
             vp->SetPrec(jvp->Nd);
 
           break;
+        case TYPE_DATE:
+          if (jvp->Type == TYPE_STRG) {
+            if (!((DTVAL*)vp)->IsFormatted())
+              ((DTVAL*)vp)->SetFormat(g, "YYYY-MM-DDThh:mm:ssZ", 20, 0);
+
+            vp->SetValue_psz(GetString(jvp));
+          } else
+            vp->SetValue(GetInteger(jvp));
+
+          break;
         default:
           sprintf(G->Message, "Unsupported column type %d", vp->GetType());
           throw 888;
diff --git a/storage/connect/tabbson.h b/storage/connect/tabbson.h
index adb02dd28e4..e9c5cc6477f 100644
--- a/storage/connect/tabbson.h
+++ b/storage/connect/tabbson.h
@@ -44,10 +44,11 @@ public:
   PBPR    row;
   PBTUT   bp;
   PCSZ    sep;
+  PCSZ    strfy;
   char    colname[65], fmt[129], buf[16];
   uint   *length;
   int     i, n, bf, ncol, lvl, sz, limit;
-  bool    all, strfy;
+  bool    all;
 }; // end of BSONDISC
 
 /***********************************************************************/
diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp
index 96aa5db683a..185ad3ebc8b 100644
--- a/storage/connect/tabjson.cpp
+++ b/storage/connect/tabjson.cpp
@@ -1,6 +1,6 @@
 /************* tabjson C++ Program Source Code File (.CPP) *************/
 /* PROGRAM NAME: tabjson     Version 1.8                               */
-/*  (C) Copyright to the author Olivier BERTRAND          2014 - 2020  */
+/*  (C) Copyright to the author Olivier BERTRAND          2014 - 2021  */
 /*  This program are the JSON class DB execution routines.             */
 /***********************************************************************/
 #undef BSON_SUPPORT
@@ -160,8 +160,9 @@ JSONDISC::JSONDISC(PGLOBAL g, uint *lg)
   jsp = NULL;
   row = NULL;
   sep = NULL;
+  strfy = NULL;
   i = n = bf = ncol = lvl = sz = limit = 0;
-  all = strfy = false;
+  all = false;
 } // end of JSONDISC constructor
 
 int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt)
@@ -173,9 +174,9 @@ int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt)
 	lvl = GetIntegerTableOption(g, topt, "Level", GetDefaultDepth());
 	lvl = GetIntegerTableOption(g, topt, "Depth", lvl);
 	sep = GetStringTableOption(g, topt, "Separator", ".");
-	sz = GetIntegerTableOption(g, topt, "Jsize", 1024);
+  strfy = GetStringTableOption(g, topt, "Stringify", NULL);
+  sz = GetIntegerTableOption(g, topt, "Jsize", 250);
   limit = GetIntegerTableOption(g, topt, "Limit", 10);
-  strfy = GetBooleanTableOption(g, topt, "Stringify", false);
 
   /*********************************************************************/
   /*  Open the input file.                                             */
@@ -187,6 +188,9 @@ int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt)
 #endif   // ZIP_SUPPORT
   tdp->Fn = GetStringTableOption(g, topt, "Filename", NULL);
 
+  if (!tdp->Fn && topt->http)
+    tdp->Fn = GetStringTableOption(g, topt, "Subtype", NULL);
+
   if (!(tdp->Database = SetPath(g, db)))
     return 0;
 
@@ -200,7 +204,8 @@ int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt)
   if (!tdp->Fn && !tdp->Uri) {
     strcpy(g->Message, MSG(MISSING_FNAME));
     return 0;
-  } // endif Fn
+  } else
+    topt->subtype = NULL;
 
   if (tdp->Fn) {
     //  We used the file name relative to recorded datapath
@@ -426,7 +431,7 @@ bool JSONDISC::Find(PGLOBAL g, PJVAL jvp, PCSZ key, int j)
     jcol.Type = TYPE_UNKNOWN;
     jcol.Len = jcol.Scale = 0;
     jcol.Cbn = true;
-  } else  if (j < lvl) {
+  } else if (j < lvl && !(strfy && !stricmp(strfy, colname))) {
     if (!fmt[bf])
       strcat(fmt, colname);
 
@@ -480,9 +485,8 @@ bool JSONDISC::Find(PGLOBAL g, PJVAL jvp, PCSZ key, int j)
 							strncat(strncat(colname, "_", n), buf, n - 1);
 						} // endif all
 
-					} else {
+					} else
 						strncat(fmt, (tdp->Uri ? sep : "[*]"), n);
-					}
 
           if (Find(g, jar->GetArrayValue(k), "", j))
             return true;
@@ -497,7 +501,7 @@ bool JSONDISC::Find(PGLOBAL g, PJVAL jvp, PCSZ key, int j)
     } // endswitch Type
 
   } else if (lvl >= 0) {
-		if (strfy) {
+		if (strfy && !stricmp(strfy, colname)) {
 			if (!fmt[bf])
 				strcat(fmt, colname);
 
@@ -1710,7 +1714,6 @@ void JSONCOL::SetJsonValue(PGLOBAL g, PVAL vp, PJVAL jvp)
       case TYPE_DTM:
 				switch (vp->GetType()) {
 				case TYPE_STRING:
-				case TYPE_DATE:
 					vp->SetValue_psz(jvp->GetString(g));
 					break;
 				case TYPE_INT:
@@ -1728,7 +1731,17 @@ void JSONCOL::SetJsonValue(PGLOBAL g, PVAL vp, PJVAL jvp)
 						vp->SetPrec(jvp->Nd);
 
 					break;
-				default:
+        case TYPE_DATE:
+          if (jvp->GetValType() == TYPE_STRG) {
+            if (!((DTVAL*)vp)->IsFormatted())
+              ((DTVAL*)vp)->SetFormat(g, "YYYY-MM-DDThh:mm:ssZ", 20, 0);
+
+            vp->SetValue_psz(jvp->GetString(g));
+          } else
+            vp->SetValue(jvp->GetInteger());
+
+          break;
+        default:
 					sprintf(g->Message, "Unsupported column type %d\n", vp->GetType());
 					throw 888;
 				} // endswitch Type
diff --git a/storage/connect/tabjson.h b/storage/connect/tabjson.h
index 147bef484a6..1062928d410 100644
--- a/storage/connect/tabjson.h
+++ b/storage/connect/tabjson.h
@@ -1,7 +1,7 @@
 /*************** tabjson H Declares Source Code File (.H) **************/
 /*  Name: tabjson.h   Version 1.3                                      */
 /*                                                                     */
-/*  (C) Copyright to the author Olivier BERTRAND          2014 - 2018  */
+/*  (C) Copyright to the author Olivier BERTRAND          2014 - 2021  */
 /*                                                                     */
 /*  This file contains the JSON classes declares.                      */
 /***********************************************************************/
@@ -67,10 +67,11 @@ public:
 	PJSON   jsp;
 	PJOB    row;
 	PCSZ    sep;
+  PCSZ    strfy;
 	char    colname[65], fmt[129], buf[16];
 	uint   *length;
 	int     i, n, bf, ncol, lvl, sz, limit;
-	bool    all, strfy;
+	bool    all;
 }; // end of JSONDISC
 
 /***********************************************************************/
diff --git a/storage/connect/tabrest.cpp b/storage/connect/tabrest.cpp
index 1efda6e3bca..f0230ef7229 100644
--- a/storage/connect/tabrest.cpp
+++ b/storage/connect/tabrest.cpp
@@ -99,12 +99,12 @@ int Xcurl(PGLOBAL g, PCSZ Http, PCSZ Uri, PCSZ filename)
 
 	if (Uri) {
 		if (*Uri == '/' || Http[strlen(Http) - 1] == '/')
-			sprintf(buf, "curl %s%s -o %s", Http, Uri, filename);
+			sprintf(buf, "curl \"%s%s\" -o %s", Http, Uri, filename);
 		else
-			sprintf(buf, "curl %s/%s -o %s", Http, Uri, filename);
+			sprintf(buf, "curl \"%s/%s\" -o %s", Http, Uri, filename);
 
 	} else
-		sprintf(buf, "curl %s -o %s", Http, filename);
+		sprintf(buf, "curl \"%s\" -o %s", Http, filename);
 
 	if ((pipe = popen(buf, "rt"))) {
 		if (trace(515))
@@ -202,11 +202,11 @@ PQRYRES __stdcall ColREST(PGLOBAL g, PTOS tp, char *tab, char *db, bool info)
   PQRYRES  qrp= NULL;
   char     filename[_MAX_PATH + 1];  // MAX PATH ???
 	int      rc;
-	bool     curl = false;
   PCSZ     http, uri, fn, ftype;
-	XGETREST grf = GetRestFunction(g);
+	XGETREST grf = NULL;
+	bool     curl = GetBooleanTableOption(g, tp, "Curl", false);
 
-	if (!grf)
+	if (!curl && !(grf = GetRestFunction(g)))
 		curl = true;
 
   http = GetStringTableOption(g, tp, "Http", NULL);
@@ -230,28 +230,25 @@ PQRYRES __stdcall ColREST(PGLOBAL g, PTOS tp, char *tab, char *db, bool info)
 			filename[n + i] = tolower(ftype[i]);
 
 		fn = filename;
-		tp->filename = PlugDup(g, fn);
+		tp->subtype = PlugDup(g, fn);
 		sprintf(g->Message, "No file name. Table will use %s", fn);
 		PUSH_WARNING(g->Message);
 	}	// endif fn
 
   //  We used the file name relative to recorded datapath
 	PlugSetPath(filename, fn, db);
-	curl = GetBooleanTableOption(g, tp, "Curl", curl);
+	remove(filename);
 
   // Retrieve the file from the web and copy it locally
 	if (curl)
 		rc = Xcurl(g, http, uri, filename);
-	else if (grf)
+	else
 		rc = grf(g->Message, trace(515), http, uri, filename);
-	else {
-		strcpy(g->Message, "Cannot access to curl nor casablanca");
-		rc = 1;
-	}	// endif !grf
 
-	if (rc)
+	if (rc) {
+		strcpy(g->Message, "Cannot access to curl nor casablanca");
 		return NULL;
-  else if (!stricmp(ftype, "JSON"))
+	} else if (!stricmp(ftype, "JSON"))
     qrp = JSONColumns(g, db, NULL, tp, info);
   else if (!stricmp(ftype, "CSV"))
     qrp = CSVColumns(g, NULL, tp, info);
@@ -274,11 +271,12 @@ bool RESTDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff)
 {
 	char     filename[_MAX_PATH + 1];
   int      rc = 0, n;
-	bool     curl = false, xt = trace(515);
+	bool     xt = trace(515);
 	LPCSTR   ftype;
-	XGETREST grf = GetRestFunction(g);
+	XGETREST grf = NULL;
+	bool     curl = GetBoolCatInfo("Curl", false);
 
-	if (!grf)
+	if (!curl && !(grf = GetRestFunction(g)))
 		curl = true;
 
 #if defined(MARIADB)
@@ -309,24 +307,21 @@ bool RESTDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff)
 
   //  We used the file name relative to recorded datapath
   PlugSetPath(filename, Fn, GetPath());
-
-	curl = GetBoolCatInfo("Curl", curl);
+	remove(filename);
 
   // Retrieve the file from the web and copy it locally
 	if (curl) {
 		rc = Xcurl(g, Http, Uri, filename);
 		xtrc(515, "Return from Xcurl: rc=%d\n", rc);
-	} else if (grf) {
+	} else {
 		rc = grf(g->Message, xt, Http, Uri, filename);
 		xtrc(515, "Return from restGetFile: rc=%d\n", rc);
-	} else {
-		strcpy(g->Message, "Cannot access to curl nor casablanca");
-		rc = 1;
-	}	// endif !grf
+	} // endelse
 
-  if (rc)
-    return true;
-  else switch (n) {
+	if (rc) {
+		strcpy(g->Message, "Cannot access to curl nor casablanca");
+		return true;
+	} else switch (n) {
     case 1: Tdp = new (g) JSONDEF; break;
 #if defined(XML_SUPPORT)
 		case 2: Tdp = new (g) XMLDEF;  break;
diff --git a/storage/connect/tabxml.cpp b/storage/connect/tabxml.cpp
index 6c9e9597cec..6065bb1b5d2 100644
--- a/storage/connect/tabxml.cpp
+++ b/storage/connect/tabxml.cpp
@@ -148,14 +148,21 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info)
   /*  Open the input file.                                             */
   /*********************************************************************/
   if (!(fn = GetStringTableOption(g, topt, "Filename", NULL))) {
-    strcpy(g->Message, MSG(MISSING_FNAME));
-    return NULL;
-  } else {
-    lvl = GetIntegerTableOption(g, topt, "Level", GetDefaultDepth());
-		lvl = GetIntegerTableOption(g, topt, "Depth", lvl);
-		lvl = (lvl < 0) ? 0 : (lvl > 16) ? 16 : lvl;
+    if (topt->http) // REST table can have default filename
+      fn = GetStringTableOption(g, topt, "Subtype", NULL);
+    
+    if (!fn) {
+      strcpy(g->Message, MSG(MISSING_FNAME));
+      return NULL;
+    } else
+      topt->subtype = NULL;
+
   } // endif fn
 
+  lvl = GetIntegerTableOption(g, topt, "Level", GetDefaultDepth());
+  lvl = GetIntegerTableOption(g, topt, "Depth", lvl);
+  lvl = (lvl < 0) ? 0 : (lvl > 16) ? 16 : lvl;
+
   if (trace(1))
     htrc("File %s lvl=%d\n", topt->filename, lvl);