- Fix a bug causing CONNECT to loop when expanding a JSON column

  when the expanded column value is null or void array.
- Adding the FullArray option to JSON tables.
- Skipping expanded JSON lines when the expanded column value is null.
  modified:   storage/connect/connect.cc
  modified:   storage/connect/tabdos.h
  modified:   storage/connect/tabjson.cpp
  modified:   storage/connect/tabjson.h

1 files changed, 321 insertions, 266 deletions

diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp
index 45c5ef3afc9..3acc2389975 100644
--- a/storage/connect/tabjson.cpp
+++ b/storage/connect/tabjson.cpp
@@ -54,16 +54,16 @@
 USETEMP UseTemp(void);
 char   *GetJsonNull(void);
 
-typedef struct _jncol {
-  struct _jncol *Next;
-  char *Name;
-  char *Fmt;
-  int   Type;
-  int   Len;
-  int   Scale;
-  bool  Cbn;
-  bool  Found;
-} JCOL, *PJCL;
+//typedef struct _jncol {
+//  struct _jncol *Next;
+//  char *Name;
+//  char *Fmt;
+//  int   Type;
+//  int   Len;
+//  int   Scale;
+//  bool  Cbn;
+//  bool  Found;
+//} JCOL, *PJCL;
 
 /***********************************************************************/
 /* JSONColumns: construct the result blocks containing the description */
@@ -76,26 +76,13 @@ PQRYRES JSONColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt, bool info)
   static XFLD fldtyp[] = {FLD_NAME, FLD_TYPE, FLD_TYPENAME, FLD_PREC, 
                           FLD_LENGTH, FLD_SCALE, FLD_NULL, FLD_FORMAT};
   static unsigned int length[] = {0, 6, 8, 10, 10, 6, 6, 0};
-  char   *p, colname[65], fmt[129];
-  int     i, j, lvl, n = 0;
+	int     i, n = 0;
   int     ncol = sizeof(buftyp) / sizeof(int);
-	bool    mgo = (GetTypeID(topt->type) == TAB_MONGO);
-	PCSZ    sep, level;
-  PVAL    valp;
-  JCOL    jcol;
-  PJCL    jcp, fjcp = NULL, pjcp = NULL;
-  PJPR   *jrp, jpp;
-  PJSON   jsp;
-  PJVAL   jvp;
-  PJOB    row;
-  PJDEF   tdp;
-  TDBJSN *tjnp = NULL;
-  PJTDB   tjsp = NULL;
+	PJCL    jcp;
+	JSONDISC *pjdc = NULL;
   PQRYRES qrp;
   PCOLRES crp;
 
-  jcol.Name = jcol.Fmt = NULL;
-
   if (info) {
     length[0] = 128;
     length[7] = 256;
@@ -107,20 +94,100 @@ PQRYRES JSONColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt, bool info)
 		return NULL;
 	}	// endif Multiple
 
-	/*********************************************************************/
-  /*  Open the input file.                                             */
+	pjdc = new(g) JSONDISC(g, (int*)length);
+
+	if (!(n = pjdc->GetColumns(g, db, dsn, topt)))
+		return NULL;
+
+ skipit:
+  if (trace(1))
+    htrc("JSONColumns: n=%d len=%d\n", n, length[0]);
+
+  /*********************************************************************/
+  /*  Allocate the structures used to refer to the result set.         */
+  /*********************************************************************/
+  qrp = PlgAllocResult(g, ncol, n, IDS_COLUMNS + 3,
+                          buftyp, fldtyp, length, false, false);
+
+  crp = qrp->Colresp->Next->Next->Next->Next->Next->Next;
+  crp->Name = "Nullable";
+  crp->Next->Name = "Jpath";
+
+  if (info || !qrp)
+    return qrp;
+
+  qrp->Nblin = n;
+
+  /*********************************************************************/
+  /*  Now get the results into blocks.                                 */
   /*********************************************************************/
-	level = GetStringTableOption(g, topt, "Level", NULL);
+  for (i = 0, jcp = pjdc->fjcp; jcp; i++, jcp = jcp->Next) {
+		if (jcp->Type == TYPE_UNKNOWN)
+			jcp->Type = TYPE_STRING;             // Void column
+
+    crp = qrp->Colresp;                    // Column Name
+    crp->Kdata->SetValue(jcp->Name, i);
+    crp = crp->Next;                       // Data Type
+    crp->Kdata->SetValue(jcp->Type, i);
+    crp = crp->Next;                       // Type Name
+    crp->Kdata->SetValue(GetTypeName(jcp->Type), i);
+    crp = crp->Next;                       // Precision
+    crp->Kdata->SetValue(jcp->Len, i);
+    crp = crp->Next;                       // Length
+    crp->Kdata->SetValue(jcp->Len, i);
+    crp = crp->Next;                       // Scale (precision)
+    crp->Kdata->SetValue(jcp->Scale, i);
+    crp = crp->Next;                       // Nullable
+    crp->Kdata->SetValue(jcp->Cbn ? 1 : 0, i);
+    crp = crp->Next;                       // Field format
+
+    if (crp->Kdata)
+      crp->Kdata->SetValue(jcp->Fmt, i);
+
+    } // endfor i
+
+  /*********************************************************************/
+  /*  Return the result pointer.                                       */
+  /*********************************************************************/
+  return qrp;
+  } // end of JSONColumns
+
+/* -------------------------- Class JSONDISC ------------------------- */
+
+/***********************************************************************/
+/*  Class used to get the columns of a JSON table.                     */
+/***********************************************************************/
+JSONDISC::JSONDISC(PGLOBAL g, int *lg)
+{
+	length = lg;
+	jcp = fjcp = pjcp = NULL;
+	tjnp = NULL;
+	jpp = NULL;
+	tjsp = NULL;
+	jsp = NULL;
+	row = NULL;
+	sep = NULL;
+	i = n = bf = ncol = lvl = 0;
+	all = false;
+}	// end of JSONDISC constructor
+
+int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt)
+{
+	bool mgo = (GetTypeID(topt->type) == TAB_MONGO);
+	PCSZ level = GetStringTableOption(g, topt, "Level", NULL);
 
 	if (level) {
 		lvl = atoi(level);
 		lvl = (lvl > 16) ? 16 : lvl;
-	} else
+	}	else
 		lvl = 0;
 
 	sep = GetStringTableOption(g, topt, "Separator", ".");
 
-  tdp = new(g) JSONDEF;
+	/*********************************************************************/
+	/*  Open the input file.                                             */
+	/*********************************************************************/
+	tdp = new(g) JSONDEF;
 #if defined(ZIP_SUPPORT)
 	tdp->Entry = GetStringTableOption(g, topt, "Entry", NULL);
 	tdp->Zipped = GetBooleanTableOption(g, topt, "Zipped", false);
@@ -130,21 +197,21 @@ PQRYRES JSONColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt, bool info)
 	if (!(tdp->Database = SetPath(g, db)))
 		return NULL;
 
-  tdp->Objname = GetStringTableOption(g, topt, "Object", NULL);
-  tdp->Base = GetIntegerTableOption(g, topt, "Base", 0) ? 1 : 0;
-  tdp->Pretty = GetIntegerTableOption(g, topt, "Pretty", 2);
+	tdp->Objname = GetStringTableOption(g, topt, "Object", NULL);
+	tdp->Base = GetIntegerTableOption(g, topt, "Base", 0) ? 1 : 0;
+	tdp->Pretty = GetIntegerTableOption(g, topt, "Pretty", 2);
 	tdp->Xcol = GetStringTableOption(g, topt, "Expand", NULL);
 	tdp->Accept = GetBooleanTableOption(g, topt, "Accept", false);
 	tdp->Uri = (dsn && *dsn ? dsn : NULL);
 
 	if (!tdp->Fn && !tdp->Uri) {
 		strcpy(g->Message, MSG(MISSING_FNAME));
-		return NULL;
+		return 0;
 	} // endif Fn
 
 	if (trace(1))
-    htrc("File %s objname=%s pretty=%d lvl=%d\n", 
-          tdp->Fn, tdp->Objname, tdp->Pretty, lvl);
+		htrc("File %s objname=%s pretty=%d lvl=%d\n",
+			tdp->Fn, tdp->Objname, tdp->Pretty, lvl);
 
 	if (tdp->Uri) {
 #if defined(JAVA_SUPPORT) || defined(CMGO_SUPPORT)
@@ -160,34 +227,34 @@ PQRYRES JSONColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt, bool info)
 		tdp->Pretty = 0;
 #else   // !MONGO_SUPPORT
 		sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "MONGO");
-		return NULL;
+		return 0;
 #endif  // !MONGO_SUPPORT
 	}	// endif Uri
 
-  if (tdp->Pretty == 2) {
+	if (tdp->Pretty == 2) {
 		if (tdp->Zipped) {
 #if defined(ZIP_SUPPORT)
 			tjsp = new(g) TDBJSON(tdp, new(g) UNZFAM(tdp));
 #else   // !ZIP_SUPPORT
 			sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP");
-			return NULL;
+			return 0;
 #endif  // !ZIP_SUPPORT
-		} else
-		  tjsp = new(g) TDBJSON(tdp, new(g) MAPFAM(tdp));
+		}	else
+			tjsp = new(g) TDBJSON(tdp, new(g) MAPFAM(tdp));
 
-    if (tjsp->MakeDocument(g))
-      return NULL;
+		if (tjsp->MakeDocument(g))
+			return NULL;
 
-    jsp = (tjsp->GetDoc()) ? tjsp->GetDoc()->GetValue(0) : NULL;
-  } else {
-    if (!(tdp->Lrecl = GetIntegerTableOption(g, topt, "Lrecl", 0)))
+		jsp = (tjsp->GetDoc()) ? tjsp->GetDoc()->GetValue(0) : NULL;
+	}	else {
+		if (!(tdp->Lrecl = GetIntegerTableOption(g, topt, "Lrecl", 0)))
 			if (!mgo) {
 				sprintf(g->Message, "LRECL must be specified for pretty=%d", tdp->Pretty);
 				return NULL;
-			} else
+			}	else
 				tdp->Lrecl = 8192;			 // Should be enough
 
-    tdp->Ending = GetIntegerTableOption(g, topt, "Ending", CRLF);
+		tdp->Ending = GetIntegerTableOption(g, topt, "Ending", CRLF);
 
 		if (tdp->Zipped) {
 #if defined(ZIP_SUPPORT)
@@ -196,7 +263,7 @@ PQRYRES JSONColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt, bool info)
 			sprintf(g->Message, MSG(NO_FEAT_SUPPORT), "ZIP");
 			return NULL;
 #endif  // !ZIP_SUPPORT
-		} else if (tdp->Uri) {
+		}	else if (tdp->Uri) {
 			if (tdp->Driver && toupper(*tdp->Driver) == 'C') {
 #if defined(CMGO_SUPPORT)
 				tjnp = new(g) TDBJSN(tdp, new(g) CMGFAM(tdp));
@@ -204,14 +271,14 @@ PQRYRES JSONColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt, bool info)
 				sprintf(g->Message, "Mongo %s Driver not available", "C");
 				return NULL;
 #endif
-			} else if (tdp->Driver && toupper(*tdp->Driver) == 'J') {
+			}	else if (tdp->Driver && toupper(*tdp->Driver) == 'J') {
 #if defined(JAVA_SUPPORT)
 				tjnp = new(g) TDBJSN(tdp, new(g) JMGFAM(tdp));
 #else
 				sprintf(g->Message, "Mongo %s Driver not available", "Java");
 				return NULL;
 #endif
-			} else {						 // Driver not specified
+			}	else {						 // Driver not specified
 #if defined(CMGO_SUPPORT)
 				tjnp = new(g) TDBJSN(tdp, new(g) CMGFAM(tdp));
 #elif defined(JAVA_SUPPORT)
@@ -222,10 +289,10 @@ PQRYRES JSONColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt, bool info)
 #endif
 			}	// endif Driver
 
-		} else
+		}	else
 			tjnp = new(g) TDBJSN(tdp, new(g) DOSFAM(tdp));
 
-    tjnp->SetMode(MODE_READ);
+		tjnp->SetMode(MODE_READ);
 
 		// Allocate the parse work memory
 		PGLOBAL G = (PGLOBAL)PlugSubAlloc(g, NULL, sizeof(GLOBAL));
@@ -237,248 +304,227 @@ PQRYRES JSONColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt, bool info)
 		tjnp->SetG(G);
 
 		if (tjnp->OpenDB(g))
-      return NULL;
+			return NULL;
 
-    switch (tjnp->ReadDB(g)) {
-      case RC_EF:
-        strcpy(g->Message, "Void json table");
-      case RC_FX:
-        goto err;
-      default:
-        jsp = tjnp->GetRow();
-      } // endswitch ReadDB
+		switch (tjnp->ReadDB(g)) {
+		case RC_EF:
+			strcpy(g->Message, "Void json table");
+		case RC_FX:
+			goto err;
+		default:
+			jsp = tjnp->GetRow();
+		} // endswitch ReadDB
 
-  } // endif pretty
+	} // endif pretty
 
-  if (!(row = (jsp) ? jsp->GetObject() : NULL)) {
-    strcpy(g->Message, "Can only retrieve columns from object rows");
-    goto err;
-    } // endif row
+	if (!(row = (jsp) ? jsp->GetObject() : NULL)) {
+		strcpy(g->Message, "Can only retrieve columns from object rows");
+		goto err;
+	} // endif row
 
-  jcol.Next = NULL;
-  jcol.Found = true;
-  colname[64] = 0;
-  fmt[128] = 0;
+	all = GetBooleanTableOption(g, topt, "Fullarray", false);
+	jcol.Name = jcol.Fmt = NULL;
+	jcol.Next = NULL;
+	jcol.Found = true;
+	colname[0] = 0;
 
 	if (!tdp->Uri) {
-		*fmt = '$';
+		fmt[0] = '$';
 		fmt[1] = '.';
-		p = fmt + 2;
-	} else
-		p = fmt;
-
-  jrp = (PJPR*)PlugSubAlloc(g, NULL, sizeof(PJPR) * MY_MAX(lvl, 0));
-
-  /*********************************************************************/
-  /*  Analyse the JSON tree and define columns.                        */
-  /*********************************************************************/
-  for (i = 1; ; i++) {
-    for (jpp = row->GetFirst(); jpp; jpp = jpp->GetNext()) {
-      for (j = 0; j < lvl; j++)
-        jrp[j] = NULL;
-
-     more:
-      strncpy(colname, jpp->GetKey(), 64);
-			*p = 0;
-      j = 0;
-      jvp = jpp->GetVal();
-
-     retry:
-      if ((valp = jvp ? jvp->GetValue() : NULL)) {
-        jcol.Type = valp->GetType();
-        jcol.Len = valp->GetValLen();
-        jcol.Scale = valp->GetValPrec();
-        jcol.Cbn = valp->IsNull();
-      } else if (!jvp || jvp->IsNull()) {
-        jcol.Type = TYPE_UNKNOWN;
-        jcol.Len = jcol.Scale = 0;
-        jcol.Cbn = true;
-      } else  if (j < lvl) {
-        if (!*p)
-          strcat(fmt, colname);
-
-        jsp = jvp->GetJson();
-
-        switch (jsp->GetType()) {
-          case TYPE_JOB:
-            if (!jrp[j])
-              jrp[j] = jsp->GetFirst();
-
-						if (*jrp[j]->GetKey() != '$') {
-							strncat(strncat(fmt, sep, 128), jrp[j]->GetKey(), 128);
-							strncat(strncat(colname, "_", 64), jrp[j]->GetKey(), 64);
-						} // endif Key
-
-						jvp = jrp[j]->GetVal();
-						j++;
-            break;
-          case TYPE_JAR:
-						if (!tdp->Xcol || stricmp(tdp->Xcol, colname)) {
-							if (tdp->Uri)
-								strncat(strncat(fmt, sep, 128), "0", 128);
-							else
-								strncat(fmt, "[0]", 128);
-
-						} else
-							strncat(fmt, (tdp->Uri ? sep : "[]"), 128);
-
-            jvp = jsp->GetValue(0);
-            break;
-          default:
-            sprintf(g->Message, "Logical error after %s", fmt);
-            goto err;
-          } // endswitch jsp
-
-        goto retry;
-      } else if (lvl >= 0) {
-        jcol.Type = TYPE_STRING;
-        jcol.Len = 256;
-        jcol.Scale = 0;
-        jcol.Cbn = true;
-			} else
-				continue;
+		bf = 2;
+	}	// endif Uri
 
-      // Check whether this column was already found
-      for (jcp = fjcp; jcp; jcp = jcp->Next)
-        if (!strcmp(colname, jcp->Name))
-          break;
-  
-      if (jcp) {
-				if (jcp->Type != jcol.Type) {
-					if (jcp->Type == TYPE_UNKNOWN)
-						jcp->Type = jcol.Type;
-					else if (jcol.Type != TYPE_UNKNOWN)
-					  jcp->Type = TYPE_STRING;
+	/*********************************************************************/
+	/*  Analyse the JSON tree and define columns.                        */
+	/*********************************************************************/
+	for (i = 1; ; i++) {
+		for (jpp = row->GetFirst(); jpp; jpp = jpp->GetNext()) {
+			strncpy(colname, jpp->GetKey(), 64);
+			fmt[bf] = 0;
+
+			if (Find(g, jpp->GetVal(), MY_MIN(lvl, 0)))
+				goto err;
+
+		} // endfor jpp
+
+		// Missing column can be null
+		for (jcp = fjcp; jcp; jcp = jcp->Next) {
+			jcp->Cbn |= !jcp->Found;
+			jcp->Found = false;
+		} // endfor jcp
+
+		if (tdp->Pretty != 2) {
+			// Read next record
+			switch (tjnp->ReadDB(g)) {
+			case RC_EF:
+				jsp = NULL;
+				break;
+			case RC_FX:
+				goto err;
+			default:
+				jsp = tjnp->GetRow();
+			} // endswitch ReadDB
 
-				} // endif Type
+		}	else
+			jsp = tjsp->GetDoc()->GetValue(i);
 
-        if (*p && (!jcp->Fmt || strlen(jcp->Fmt) < strlen(fmt))) {
-          jcp->Fmt = PlugDup(g, fmt);
-          length[7] = MY_MAX(length[7], strlen(fmt));
-          } // endif fmt
-
-        jcp->Len = MY_MAX(jcp->Len, jcol.Len);
-        jcp->Scale = MY_MAX(jcp->Scale, jcol.Scale);
-        jcp->Cbn |= jcol.Cbn;
-        jcp->Found = true;
-      } else if (jcol.Type != TYPE_UNKNOWN || tdp->Accept) {
-        // New column
-        jcp = (PJCL)PlugSubAlloc(g, NULL, sizeof(JCOL));
-        *jcp = jcol;
-        jcp->Cbn |= (i > 1);
-        jcp->Name = PlugDup(g, colname);
-        length[0] = MY_MAX(length[0], strlen(colname));
-  
-        if (*p) {
-          jcp->Fmt = PlugDup(g, fmt);
-          length[7] = MY_MAX(length[7], strlen(fmt));
-        } else
-          jcp->Fmt = NULL;
-  
-        if (pjcp) {
-          jcp->Next = pjcp->Next;
-          pjcp->Next = jcp;
-        } else
-          fjcp = jcp;
+		if (!(row = (jsp) ? jsp->GetObject() : NULL))
+			break;
 
-        n++;
-      } // endif jcp
+	} // endfor i
 
-      pjcp = jcp;
+	if (tdp->Pretty != 2)
+		tjnp->CloseDB(g);
 
-      for (j = lvl - 1; j >= 0; j--)
-        if (jrp[j] && (jrp[j] = jrp[j]->GetNext()))
-          goto more;
+	return n;
 
-      } // endfor jpp
+err:
+	if (tdp->Pretty != 2)
+		tjnp->CloseDB(g);
 
-    // Missing column can be null
-    for (jcp = fjcp; jcp; jcp = jcp->Next) {
-      jcp->Cbn |= !jcp->Found;
-      jcp->Found = false;
-      } // endfor jcp
+	return 0;
+}	// end of GetColumns
 
-    if (tdp->Pretty != 2) {
-      // Read next record
-      switch (tjnp->ReadDB(g)) {
-        case RC_EF:
-          jsp = NULL;
-          break;
-        case RC_FX:
-          goto err;
-        default:
-          jsp = tjnp->GetRow();
-        } // endswitch ReadDB
+bool JSONDISC::Find(PGLOBAL g, PJVAL jvp, int j)
+{
+	char *p, *pc = colname + strlen(colname);
+	int   ars;
+	PJOB  job;
+	PJAR  jar;
+
+	if ((valp = jvp ? jvp->GetValue() : NULL)) {
+		jcol.Type = valp->GetType();
+		jcol.Len = valp->GetValLen();
+		jcol.Scale = valp->GetValPrec();
+		jcol.Cbn = valp->IsNull();
+	} else if (!jvp || jvp->IsNull()) {
+		jcol.Type = TYPE_UNKNOWN;
+		jcol.Len = jcol.Scale = 0;
+		jcol.Cbn = true;
+	} else  if (j < lvl) {
+		if (!fmt[bf])
+			strcat(fmt, colname);
+
+		p = fmt + strlen(fmt);
+		jsp = jvp->GetJson();
+
+		switch (jsp->GetType()) {
+			case TYPE_JOB:
+				job = (PJOB)jsp;
+
+				for (PJPR jrp = job->GetFirst(); jrp; jrp = jrp->GetNext()) {
+					if (*jrp->GetKey() != '$') {
+						strncat(strncat(fmt, sep, 128), jrp->GetKey(), 128);
+						strncat(strncat(colname, "_", 64), jrp->GetKey(), 64);
+					} // endif Key
+
+					if (Find(g, jrp->GetVal(), j + 1))
+						return true;
+
+					*p = *pc = 0;
+				} // endfor jrp
+
+				return false;
+			case TYPE_JAR:
+				jar = (PJAR)jsp;
+
+				if (all || (tdp->Xcol && !stricmp(tdp->Xcol, colname)))
+					ars = jar->GetSize(false);
+				else
+					ars = MY_MIN(jar->GetSize(false), 1);
 
-    } else
-      jsp = tjsp->GetDoc()->GetValue(i);
+				for (int k = 0; k < ars; k++) {
+					if (!tdp->Xcol || stricmp(tdp->Xcol, colname)) {
+						sprintf(buf, "%d", k);
 
-    if (!(row = (jsp) ? jsp->GetObject() : NULL))
-      break;
+						if (tdp->Uri)
+							strncat(strncat(fmt, sep, 128), buf, 128);
+						else
+							strncat(strncat(strncat(fmt, "[", 128), buf, 128), "]", 128);
 
-    } // endor i
+						if (all)
+							strncat(strncat(colname, "_", 64), buf, 64);
 
-  if (tdp->Pretty != 2)
-    tjnp->CloseDB(g);
+					} else
+						strncat(fmt, (tdp->Uri ? sep : "[*]"), 128);
 
- skipit:
-  if (trace(1))
-    htrc("JSONColumns: n=%d len=%d\n", n, length[0]);
+					if (Find(g, jar->GetValue(k), j))
+						return true;
 
-  /*********************************************************************/
-  /*  Allocate the structures used to refer to the result set.         */
-  /*********************************************************************/
-  qrp = PlgAllocResult(g, ncol, n, IDS_COLUMNS + 3,
-                          buftyp, fldtyp, length, false, false);
+					*p = *pc = 0;
+				} // endfor k
 
-  crp = qrp->Colresp->Next->Next->Next->Next->Next->Next;
-  crp->Name = "Nullable";
-  crp->Next->Name = "Jpath";
+				return false;
+			default:
+				sprintf(g->Message, "Logical error after %s", fmt);
+				return true;
+		} // endswitch Type
 
-  if (info || !qrp)
-    return qrp;
+	} else if (lvl >= 0) {
+		jcol.Type = TYPE_STRING;
+		jcol.Len = 256;
+		jcol.Scale = 0;
+		jcol.Cbn = true;
+	} else
+		return false;
 
-  qrp->Nblin = n;
+	AddColumn(g);
+	return false;
+}	// end of Find
 
-  /*********************************************************************/
-  /*  Now get the results into blocks.                                 */
-  /*********************************************************************/
-  for (i = 0, jcp = fjcp; jcp; i++, jcp = jcp->Next) {
-		if (jcp->Type == TYPE_UNKNOWN)
-			jcp->Type = TYPE_STRING;             // Void column
+void JSONDISC::AddColumn(PGLOBAL g)
+{
+	bool b = fmt[bf] != 0;		 // True if formatted
 
-    crp = qrp->Colresp;                    // Column Name
-    crp->Kdata->SetValue(jcp->Name, i);
-    crp = crp->Next;                       // Data Type
-    crp->Kdata->SetValue(jcp->Type, i);
-    crp = crp->Next;                       // Type Name
-    crp->Kdata->SetValue(GetTypeName(jcp->Type), i);
-    crp = crp->Next;                       // Precision
-    crp->Kdata->SetValue(jcp->Len, i);
-    crp = crp->Next;                       // Length
-    crp->Kdata->SetValue(jcp->Len, i);
-    crp = crp->Next;                       // Scale (precision)
-    crp->Kdata->SetValue(jcp->Scale, i);
-    crp = crp->Next;                       // Nullable
-    crp->Kdata->SetValue(jcp->Cbn ? 1 : 0, i);
-    crp = crp->Next;                       // Field format
+	// Check whether this column was already found
+	for (jcp = fjcp; jcp; jcp = jcp->Next)
+		if (!strcmp(colname, jcp->Name))
+			break;
 
-    if (crp->Kdata)
-      crp->Kdata->SetValue(jcp->Fmt, i);
+	if (jcp) {
+		if (jcp->Type != jcol.Type) {
+			if (jcp->Type == TYPE_UNKNOWN)
+				jcp->Type = jcol.Type;
+			else if (jcol.Type != TYPE_UNKNOWN)
+				jcp->Type = TYPE_STRING;
+
+		} // endif Type
+
+		if (b && (!jcp->Fmt || strlen(jcp->Fmt) < strlen(fmt))) {
+			jcp->Fmt = PlugDup(g, fmt);
+			length[7] = MY_MAX(length[7], strlen(fmt));
+		} // endif fmt
+
+		jcp->Len = MY_MAX(jcp->Len, jcol.Len);
+		jcp->Scale = MY_MAX(jcp->Scale, jcol.Scale);
+		jcp->Cbn |= jcol.Cbn;
+		jcp->Found = true;
+	} else if (jcol.Type != TYPE_UNKNOWN || tdp->Accept) {
+		// New column
+		jcp = (PJCL)PlugSubAlloc(g, NULL, sizeof(JCOL));
+		*jcp = jcol;
+		jcp->Cbn |= (i > 1);
+		jcp->Name = PlugDup(g, colname);
+		length[0] = MY_MAX(length[0], strlen(colname));
+
+		if (b) {
+			jcp->Fmt = PlugDup(g, fmt);
+			length[7] = MY_MAX(length[7], strlen(fmt));
+		} else
+			jcp->Fmt = NULL;
 
-    } // endfor i
+		if (pjcp) {
+			jcp->Next = pjcp->Next;
+			pjcp->Next = jcp;
+		} else
+			fjcp = jcp;
 
-  /*********************************************************************/
-  /*  Return the result pointer.                                       */
-  /*********************************************************************/
-  return qrp;
+		n++;
+	} // endif jcp
 
-err:
-  if (tdp->Pretty != 2)
-    tjnp->CloseDB(g);
+	pjcp = jcp;
+} // end of AddColumn
 
-  return NULL;
-  } // end of JSONColumns
 
 /* -------------------------- Class JSONDEF -------------------------- */
 
@@ -513,6 +559,7 @@ bool JSONDEF::DefineAM(PGLOBAL g, LPCSTR, int poff)
   Limit = GetIntCatInfo("Limit", 10);
   Base = GetIntCatInfo("Base", 0) ? 1 : 0;
 	Sep = *GetStringCatInfo(g, "Separator", ".");
+	Accept = GetBoolCatInfo("Accept", false);
 
 	if (Uri = GetStringCatInfo(g, "Connect", NULL)) {
 #if defined(JAVA_SUPPORT) || defined(CMGO_SUPPORT)
@@ -1471,6 +1518,9 @@ void JSONCOL::ReadColumn(PGLOBAL g)
   if (!Tjp->SameRow || Xnod >= Tjp->SameRow)
     Value->SetValue_pval(GetColumnValue(g, Tjp->Row, 0));
 
+	if (Xpd && Value->IsNull() && !((PJDEF)Tjp->To_Def)->Accept)
+		throw("Null expandable JSON value");
+
   // Set null when applicable
   if (!Nullable)
     Value->SetNull(false);
@@ -1546,11 +1596,16 @@ PVAL JSONCOL::GetColumnValue(PGLOBAL g, PJSON row, int i)
 /***********************************************************************/
 PVAL JSONCOL::ExpandArray(PGLOBAL g, PJAR arp, int n)
   {
-  int    ars;
+  int    ars = MY_MIN(Tjp->Limit, arp->size());
   PJVAL  jvp;
   JVALUE jval;
 
-  ars = MY_MIN(Tjp->Limit, arp->size());
+	if (!ars) {
+		Value->Reset();
+		Value->SetNull(true);
+		Tjp->NextSame = 0;
+		return Value;
+	} // endif ars
 
   if (!(jvp = arp->GetValue((Nodes[n].Rx = Nodes[n].Nx)))) {
     strcpy(g->Message, "Logical error expanding array");