- Fix a few bug mainly concerning discovery and call from OEM

  (and prepare new table types)

  modified:   storage/connect/tabjson.cpp
  modified:   storage/connect/tabjson.h
  modified:   storage/connect/tabxml.cpp
  modified:   storage/connect/tabxml.h

- Fix wrong line estimate
  modified:   storage/connect/mysql-test/connect/r/part_table.result
  modified:   storage/connect/mysql-test/connect/t/part_table.test

6 files changed, 185 insertions, 145 deletions

diff --git a/storage/connect/mysql-test/connect/r/part_table.result b/storage/connect/mysql-test/connect/r/part_table.result
index f3a556ae784..ee17a1d32b9 100644
--- a/storage/connect/mysql-test/connect/r/part_table.result
+++ b/storage/connect/mysql-test/connect/r/part_table.result
@@ -23,7 +23,7 @@ id	msg
 CREATE TABLE xt3 (
 id INT KEY NOT NULL,
 msg VARCHAR(32))
-ENGINE=CONNECT TABLE_TYPE=CSV AVG_ROW_LENGTH=10;
+ENGINE=CONNECT TABLE_TYPE=CSV AVG_ROW_LENGTH=6;
 Warnings:
 Warning	1105	No file name. Table will use xt3.csv
 INSERT INTO xt3 VALUES(60,'sixty'),(81,'eighty one'),(72,'seventy two');
@@ -92,7 +92,7 @@ id	msg
 EXPLAIN PARTITIONS
 SELECT * FROM t1 WHERE id = 81;
 id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t1	3	ALL	NULL	NULL	NULL	NULL	4	Using where
+1	SIMPLE	t1	3	ALL	NULL	NULL	NULL	NULL	6	Using where
 DELETE FROM t1;
 Warnings:
 Note	1105	xt1: 4 affected rows
diff --git a/storage/connect/mysql-test/connect/t/part_table.test b/storage/connect/mysql-test/connect/t/part_table.test
index 5edd5766bd6..0fb2a11f0f9 100644
--- a/storage/connect/mysql-test/connect/t/part_table.test
+++ b/storage/connect/mysql-test/connect/t/part_table.test
@@ -22,7 +22,7 @@ SELECT * FROM xt2;
 CREATE TABLE xt3 (
 id INT KEY NOT NULL,
 msg VARCHAR(32))
-ENGINE=CONNECT TABLE_TYPE=CSV AVG_ROW_LENGTH=10;
+ENGINE=CONNECT TABLE_TYPE=CSV AVG_ROW_LENGTH=6;
 INSERT INTO xt3 VALUES(60,'sixty'),(81,'eighty one'),(72,'seventy two');
 SELECT * FROM xt3;
 
diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp
index d20e793ff88..c0d36efcf42 100644
--- a/storage/connect/tabjson.cpp
+++ b/storage/connect/tabjson.cpp
@@ -1,6 +1,6 @@
 /************* tabjson C++ Program Source Code File (.CPP) *************/
-/* PROGRAM NAME: tabjson     Version 1.5                               */
-/*  (C) Copyright to the author Olivier BERTRAND          2014 - 2017  */
+/* PROGRAM NAME: tabjson     Version 1.6                               */
+/*  (C) Copyright to the author Olivier BERTRAND          2014 - 2018  */
 /*  This program are the JSON class DB execution routines.             */
 /***********************************************************************/
 
@@ -173,6 +173,7 @@ JSONDISC::JSONDISC(PGLOBAL g, uint *lg)
 
 int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt)
 {
+	char filename[_MAX_PATH];
 	bool mgo = (GetTypeID(topt->type) == TAB_MONGO);
 	PCSZ level = GetStringTableOption(g, topt, "Level", NULL);
 
@@ -209,6 +210,12 @@ int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt)
 		return 0;
 	} // endif Fn
 
+	if (tdp->Fn) {
+		//  We used the file name relative to recorded datapath
+		PlugSetPath(filename, tdp->Fn, tdp->GetPath());
+		tdp->Fn = PlugDup(g, filename);
+	} // endif Fn
+
 	if (trace(1))
 		htrc("File %s objname=%s pretty=%d lvl=%d\n",
 			tdp->Fn, tdp->Objname, tdp->Pretty, lvl);
@@ -342,7 +349,7 @@ int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt)
 			strncpy(colname, jpp->GetKey(), 64);
 			fmt[bf] = 0;
 
-			if (Find(g, jpp->GetVal(), MY_MIN(lvl, 0)))
+			if (Find(g, jpp->GetVal(), colname, MY_MIN(lvl, 0)))
 				goto err;
 
 		} // endfor jpp
@@ -385,7 +392,7 @@ err:
 	return 0;
 }	// end of GetColumns
 
-bool JSONDISC::Find(PGLOBAL g, PJVAL jvp, int j)
+bool JSONDISC::Find(PGLOBAL g, PJVAL jvp, PCSZ key, int j)
 {
 	char *p, *pc = colname + strlen(colname);
 	int   ars;
@@ -413,12 +420,14 @@ bool JSONDISC::Find(PGLOBAL g, PJVAL jvp, int j)
 				job = (PJOB)jsp;
 
 				for (PJPR jrp = job->GetFirst(); jrp; jrp = jrp->GetNext()) {
-					if (*jrp->GetKey() != '$') {
-						strncat(strncat(fmt, sep, 128), jrp->GetKey(), 128);
-						strncat(strncat(colname, "_", 64), jrp->GetKey(), 64);
+					PCSZ k = jrp->GetKey();
+
+					if (*k != '$') {
+						strncat(strncat(fmt, sep, 128), k, 128);
+						strncat(strncat(colname, "_", 64), k, 64);
 					} // endif Key
 
-					if (Find(g, jrp->GetVal(), j + 1))
+					if (Find(g, jrp->GetVal(), k, j + 1))
 						return true;
 
 					*p = *pc = 0;
@@ -428,13 +437,13 @@ bool JSONDISC::Find(PGLOBAL g, PJVAL jvp, int j)
 			case TYPE_JAR:
 				jar = (PJAR)jsp;
 
-				if (all || (tdp->Xcol && !stricmp(tdp->Xcol, colname)))
+				if (all || (tdp->Xcol && !stricmp(tdp->Xcol, key)))
 					ars = jar->GetSize(false);
 				else
 					ars = MY_MIN(jar->GetSize(false), 1);
 
 				for (int k = 0; k < ars; k++) {
-					if (!tdp->Xcol || stricmp(tdp->Xcol, colname)) {
+					if (!tdp->Xcol || stricmp(tdp->Xcol, key)) {
 						sprintf(buf, "%d", k);
 
 						if (tdp->Uri)
@@ -448,7 +457,7 @@ bool JSONDISC::Find(PGLOBAL g, PJVAL jvp, int j)
 					} else
 						strncat(fmt, (tdp->Uri ? sep : "[*]"), 128);
 
-					if (Find(g, jar->GetValue(k), j))
+					if (Find(g, jar->GetValue(k), "", j))
 						return true;
 
 					*p = *pc = 0;
@@ -522,7 +531,9 @@ void JSONDISC::AddColumn(PGLOBAL g)
 		n++;
 	} // endif jcp
 
-	pjcp = jcp;
+	if (jcp)
+	  pjcp = jcp;
+
 } // end of AddColumn
 
 
@@ -549,7 +560,7 @@ JSONDEF::JSONDEF(void)
 /***********************************************************************/
 /*  DefineAM: define specific AM block values.                         */
 /***********************************************************************/
-bool JSONDEF::DefineAM(PGLOBAL g, LPCSTR, int poff)
+bool JSONDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff)
 {
 	Schema = GetStringCatInfo(g, "DBname", Schema);
 	Jmode = (JMODE)GetIntCatInfo("Jmode", MODE_OBJECT);
@@ -561,7 +572,8 @@ bool JSONDEF::DefineAM(PGLOBAL g, LPCSTR, int poff)
 	Sep = *GetStringCatInfo(g, "Separator", ".");
 	Accept = GetBoolCatInfo("Accept", false);
 
-	if (Uri = GetStringCatInfo(g, "Connect", NULL)) {
+	// Don't use url as uri when called from REST OEM module
+	if (stricmp(am, "REST") && (Uri = GetStringCatInfo(g, "Connect", NULL))) {
 #if defined(JAVA_SUPPORT) || defined(CMGO_SUPPORT)
 		Collname = GetStringCatInfo(g, "Name",
 			(Catfunc & (FNC_TABLE | FNC_COL)) ? NULL : Name);
@@ -2340,7 +2352,7 @@ void TDBJSON::CloseDB(PGLOBAL g)
 TDBJCL::TDBJCL(PJDEF tdp) : TDBCAT(tdp)
   {
   Topt = tdp->GetTopt();
-  Db = tdp->Schema;
+	Db = tdp->Schema;
 	Dsn = tdp->Uri;
   } // end of TDBJCL constructor
 
diff --git a/storage/connect/tabjson.h b/storage/connect/tabjson.h
index fcbfe4ed1ec..8721a2a5ab7 100644
--- a/storage/connect/tabjson.h
+++ b/storage/connect/tabjson.h
@@ -52,7 +52,7 @@ public:
 
 	// Functions
 	int  GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt);
-	bool Find(PGLOBAL g, PJVAL jvp, int j);
+	bool Find(PGLOBAL g, PJVAL jvp, PCSZ key, int j);
 	void AddColumn(PGLOBAL g);
 
 	// Members
diff --git a/storage/connect/tabxml.cpp b/storage/connect/tabxml.cpp
index fa4854bb618..d808bd5ecd4 100644
--- a/storage/connect/tabxml.cpp
+++ b/storage/connect/tabxml.cpp
@@ -163,8 +163,11 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info)
 		return NULL;
 
   tdp->Tabname = tab;
+	tdp->Tabname = (char*)GetStringTableOption(g, topt, "Tabname", tab);
+	tdp->Rowname = (char*)GetStringTableOption(g, topt, "Rownode", NULL);
 	tdp->Zipped = GetBooleanTableOption(g, topt, "Zipped", false);
 	tdp->Entry = GetStringTableOption(g, topt, "Entry", NULL);
+	tdp->Skip = GetBooleanTableOption(g, topt, "Skipnull", false);
 
   if (!(op = GetStringTableOption(g, topt, "Xmlsup", NULL)))
 #if defined(__WIN__)
@@ -280,7 +283,9 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info)
             if (!vp->atp)
 							node = vp->nl->GetItem(g, vp->k++, tdp->Usedom ? node : NULL);
 
-						strncat(fmt, colname, XLEN(fmt));
+						if (!j)
+							strncat(fmt, colname, XLEN(fmt));
+
 						strncat(fmt, "/", XLEN(fmt));
 						strncat(xcol->Name, "_", XLEN(xcol->Name));
 						j++;
@@ -302,6 +307,7 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info)
           case RC_INFO:
             PushWarning(g, txmp);
           case RC_OK:
+						xcol->Cbn = !strlen(buf);
             break;
           default:
             goto err;
@@ -327,9 +333,9 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info)
 
         xcp->Len = MY_MAX(xcp->Len, xcol->Len);
         xcp->Scale = MY_MAX(xcp->Scale, xcol->Scale);
-        xcp->Cbn |= xcol->Cbn;
+        xcp->Cbn |= (xcol->Cbn || !xcol->Len);
         xcp->Found = true;
-      } else {
+      } else if(xcol->Len || !tdp->Skip) {
         // New column
         xcp = new(g) XMCOL(g, xcol, fmt, i);
         length[0] = MY_MAX(length[0], strlen(xcol->Name));
@@ -344,7 +350,8 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info)
         n++;
       } // endif xcp
 
-      pxcp = xcp;
+			if (xcp)
+				pxcp = xcp;
 
       if (vp->atp)
         vp->atp = vp->atp->GetNext(g);
@@ -445,6 +452,7 @@ XMLDEF::XMLDEF(void)
   Usedom = false;
 	Zipped = false;
 	Mulentries = false;
+	Skip = false;
   } // end of XMLDEF constructor
 
 /***********************************************************************/
@@ -814,127 +822,141 @@ bool TDBXML::Initialize(PGLOBAL g)
     } // endif Bufdone
 
 #if !defined(UNIX)
-  if (!Root) try {
+	if (!Root) try {
 #else
-  if (!Root) {
+	if (!Root) {
 #endif
-    char tabpath[64], filename[_MAX_PATH];
-
-    //  We used the file name relative to recorded datapath
-    PlugSetPath(filename, Xfile, GetPath());
-
-    // Load or re-use the table file
-    rc = LoadTableFile(g, filename);
-
-    if (rc == RC_OK) {
-      // Get root node
-      if (!(Root = Docp->GetRoot(g))) {
-        // This should never happen as load should have failed
-        strcpy(g->Message, MSG(EMPTY_DOC));
-        goto error;
-        } // endif Root
-
-      // If tabname is not an Xpath,
-      // construct one that will find it anywhere
-      if (!strchr(Tabname, '/'))
-        strcat(strcpy(tabpath, "//"), Tabname);
-      else
-        strcpy(tabpath, Tabname);
-
-      // Evaluate table xpath
-      if ((TabNode = Root->SelectSingleNode(g, tabpath))) {
-        if (TabNode->GetType() != XML_ELEMENT_NODE) {
-          sprintf(g->Message, MSG(BAD_NODE_TYPE), TabNode->GetType());
-          goto error;
-          } // endif Type
-
-      } else if (Mode == MODE_INSERT && XmlDB) {
-        // We are adding a new table to a multi-table file
-
-        // If XmlDB is not an Xpath,
-        // construct one that will find it anywhere
-        if (!strchr(XmlDB, '/'))
-          strcat(strcpy(tabpath, "//"), XmlDB);
-        else
-          strcpy(tabpath, XmlDB);
-
-        if (!(DBnode = Root->SelectSingleNode(g, tabpath))) {
-          // DB node does not exist yet; we cannot create it
-          // because we don't know where it should be placed
-          sprintf(g->Message, MSG(MISSING_NODE), XmlDB, Xfile);
-          goto error;
-          } // endif DBnode
-
-        if (!(TabNode = DBnode->AddChildNode(g, Tabname))) {
-          sprintf(g->Message, MSG(FAIL_ADD_NODE), Tabname);
-          goto error;
-          } // endif TabNode
-
-        DBnode->AddText(g, "\n");
-      } else
-        TabNode = Root;              // Try this ?
-
-    } else if (rc == RC_NF || rc == RC_EF) {
-      // The XML file does not exist or is void
-      if (Mode == MODE_INSERT) {
-        // New Document
-        char buf[64];
-
-        // Create the XML node
-        if (Docp->NewDoc(g, "1.0")) {
-          strcpy(g->Message, MSG(NEW_DOC_FAILED));
-          goto error;
-          } // endif NewDoc
-
-        //  Now we can link the Xblock
-        To_Xb = Docp->LinkXblock(g, Mode, rc, filename);
-
-        // Add a CONNECT comment node
-        strcpy(buf, " Created by the MariaDB CONNECT Storage Engine");
-        Docp->AddComment(g, buf);
-
-        if (XmlDB) {
-          // This is a multi-table file
-          DBnode = Root = Docp->NewRoot(g, XmlDB);
-          DBnode->AddText(g, "\n");
-          TabNode = DBnode->AddChildNode(g, Tabname);
-          DBnode->AddText(g, "\n");
-        } else
-          TabNode = Root = Docp->NewRoot(g, Tabname);
-
-        if (TabNode == NULL || Root == NULL) {
-          strcpy(g->Message, MSG(XML_INIT_ERROR));
-          goto error;
-        } else if (SetTabNode(g))
-          goto error;
-
-      } else {
-        sprintf(g->Message, MSG(FILE_UNFOUND), Xfile);
-
-        if (Mode == MODE_READ) {
-          PushWarning(g, this);
-          Void = true;
-          } // endif Mode
-
-        goto error;
-      } // endif Mode
-
-    } else if (rc == RC_INFO) {
-      // Loading failed
-      sprintf(g->Message, MSG(LOADING_FAILED), Xfile);
-      goto error;
-    } else // (rc == RC_FX)
-      goto error;
-
-    // Get row node list
-    if (Rowname)
-      Nlist = TabNode->SelectNodes(g, Rowname);
-    else
-      Nlist = TabNode->GetChildElements(g);
-
-    Docp->SetNofree(true);       // For libxml2
+		char tabpath[64], filename[_MAX_PATH];
+
+		//  We used the file name relative to recorded datapath
+		PlugSetPath(filename, Xfile, GetPath());
+
+		// Load or re-use the table file
+		rc = LoadTableFile(g, filename);
+
+		if (rc == RC_OK) {
+			// Get root node
+			if (!(Root = Docp->GetRoot(g))) {
+				// This should never happen as load should have failed
+				strcpy(g->Message, MSG(EMPTY_DOC));
+				goto error;
+			} // endif Root
+
+		// If tabname is not an Xpath,
+		// construct one that will find it anywhere
+			if (!strchr(Tabname, '/'))
+				strcat(strcpy(tabpath, "//"), Tabname);
+			else
+				strcpy(tabpath, Tabname);
+
+			// Evaluate table xpath
+			if ((TabNode = Root->SelectSingleNode(g, tabpath))) {
+				if (TabNode->GetType() != XML_ELEMENT_NODE) {
+					sprintf(g->Message, MSG(BAD_NODE_TYPE), TabNode->GetType());
+					goto error;
+				} // endif Type
+
+			} else if (Mode == MODE_INSERT && XmlDB) {
+				// We are adding a new table to a multi-table file
+
+				// If XmlDB is not an Xpath,
+				// construct one that will find it anywhere
+				if (!strchr(XmlDB, '/'))
+					strcat(strcpy(tabpath, "//"), XmlDB);
+				else
+					strcpy(tabpath, XmlDB);
+
+				if (!(DBnode = Root->SelectSingleNode(g, tabpath))) {
+					// DB node does not exist yet; we cannot create it
+					// because we don't know where it should be placed
+					sprintf(g->Message, MSG(MISSING_NODE), XmlDB, Xfile);
+					goto error;
+				} // endif DBnode
+
+				if (!(TabNode = DBnode->AddChildNode(g, Tabname))) {
+					sprintf(g->Message, MSG(FAIL_ADD_NODE), Tabname);
+					goto error;
+				} // endif TabNode
+
+				DBnode->AddText(g, "\n");
+			} else {
+				TabNode = Root;              // Try this ?
+				Tabname = TabNode->GetName(g);
+			} // endif's
+
+		} else if (rc == RC_NF || rc == RC_EF) {
+			// The XML file does not exist or is void
+			if (Mode == MODE_INSERT) {
+				// New Document
+				char buf[64];
+
+				// Create the XML node
+				if (Docp->NewDoc(g, "1.0")) {
+					strcpy(g->Message, MSG(NEW_DOC_FAILED));
+					goto error;
+				} // endif NewDoc
+
+			//  Now we can link the Xblock
+				To_Xb = Docp->LinkXblock(g, Mode, rc, filename);
+
+				// Add a CONNECT comment node
+				strcpy(buf, " Created by the MariaDB CONNECT Storage Engine");
+				Docp->AddComment(g, buf);
+
+				if (XmlDB) {
+					// This is a multi-table file
+					DBnode = Root = Docp->NewRoot(g, XmlDB);
+					DBnode->AddText(g, "\n");
+					TabNode = DBnode->AddChildNode(g, Tabname);
+					DBnode->AddText(g, "\n");
+				} else
+					TabNode = Root = Docp->NewRoot(g, Tabname);
+
+				if (TabNode == NULL || Root == NULL) {
+					strcpy(g->Message, MSG(XML_INIT_ERROR));
+					goto error;
+				} else if (SetTabNode(g))
+					goto error;
+
+			} else {
+				sprintf(g->Message, MSG(FILE_UNFOUND), Xfile);
+
+				if (Mode == MODE_READ) {
+					PushWarning(g, this);
+					Void = true;
+				} // endif Mode
+
+				goto error;
+			} // endif Mode
+
+		} else if (rc == RC_INFO) {
+			// Loading failed
+			sprintf(g->Message, MSG(LOADING_FAILED), Xfile);
+			goto error;
+		} else // (rc == RC_FX)
+			goto error;
+
+		if (!Rowname) {
+			for (PXNODE n = TabNode->GetChild(g); n; n = n->GetNext(g))
+				if (n->GetType() == XML_ELEMENT_NODE) {
+					Rowname = n->GetName(g);
+					break;
+				} // endif Type
+
+			if (!Rowname)
+				Rowname = TabNode->GetName(g);
+		} // endif Rowname
+
+			// Get row node list
+		if (strcmp(Rowname, Tabname))
+			Nlist = TabNode->SelectNodes(g, Rowname);
+		else
+			Nrow = 1;
+
+
+		Docp->SetNofree(true);       // For libxml2
 #if defined(__WIN__)
-  } catch(_com_error e) {
+	} catch (_com_error e) {
     // We come here if a DOM command threw an error
     char   buf[128];
 
@@ -1221,10 +1243,14 @@ int TDBXML::ReadDB(PGLOBAL g)
       htrc("TDBXML ReadDB: Irow=%d RowNode=%p\n", Irow, RowNode);
 
     // Get the new row node
-    if ((RowNode = Nlist->GetItem(g, Irow, RowNode)) == NULL) {
-      sprintf(g->Message, MSG(MISSING_ROWNODE), Irow);
-      return RC_FX;
-      } // endif RowNode
+		if (Nlist) {
+			if ((RowNode = Nlist->GetItem(g, Irow, RowNode)) == NULL) {
+				sprintf(g->Message, MSG(MISSING_ROWNODE), Irow);
+				return RC_FX;
+			} // endif RowNode
+
+		} else
+			RowNode = TabNode;
 
     if (Colname && Coltype == 2)
       Clist = RowNode->SelectNodes(g, Colname, Clist);
@@ -1279,6 +1305,7 @@ int TDBXML::WriteDB(PGLOBAL g)
 /***********************************************************************/
 int TDBXML::DeleteDB(PGLOBAL g, int irc)
   {
+	// TODO: Handle null Nlist
   if (irc == RC_FX) {
     // Delete all rows
     for (Irow = 0; Irow < Nrow; Irow++)
diff --git a/storage/connect/tabxml.h b/storage/connect/tabxml.h
index 102767e965a..fb3913f08ea 100644
--- a/storage/connect/tabxml.h
+++ b/storage/connect/tabxml.h
@@ -52,6 +52,7 @@ class DllExport XMLDEF : public TABDEF {  /* Logical table description */
   bool    Usedom;                 /* True: DOM, False: libxml2         */
 	bool    Zipped;                 /* True: Zipped XML file(s)          */
 	bool    Mulentries;             /* True: multiple entries in zip file*/
+	bool    Skip;                   /* Skip null columns                 */
 }; // end of XMLDEF
 
 #if defined(INCLUDE_TDBXML)