- Eliminate virtual columns from CSV and FMT table fields

modified:
  storage/connect/colblk.h
  storage/connect/reldef.h
  storage/connect/tabfmt.cpp

- Fix length specification and writing (when using FIELD_FORMAT) of DECIMAL columns
modified:
  storage/connect/ha_connect.cc
  storage/connect/tabdos.cpp

- Add the D field_format option (specifying the decimal separator character)
modified:
  storage/connect/tabdos.cpp
  storage/connect/tabdos.h
  storage/connect/tabfmt.cpp

6 files changed, 68 insertions, 36 deletions

diff --git a/storage/connect/colblk.h b/storage/connect/colblk.h
index e1d4fc7e948..a340ee4450a 100644
--- a/storage/connect/colblk.h
+++ b/storage/connect/colblk.h
@@ -55,6 +55,7 @@ class DllExport COLBLK : public XOBJECT {
           PSZ     GetFmt(void) {return (Cdp) ? Cdp->Fmt : NULL;}
           bool    IsUnsigned(void) {return Unsigned;}
           bool    IsNullable(void) {return Nullable;}
+          bool    IsVirtual(void) {return Cdp->IsVirtual();}
           void    SetNullable(bool b) {Nullable = b;}
                   
   // Methods      
diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc
index 76cb0c5c62a..ff15b27ca50 100644
--- a/storage/connect/ha_connect.cc
+++ b/storage/connect/ha_connect.cc
@@ -1058,6 +1058,7 @@ void *ha_connect::GetColumnOption(PGLOBAL g, void *field, PCOLINFO pcf)
       break;
     case TYPE_DECIM:
       pcf->Precision= ((Field_new_decimal*)fp)->precision;
+      pcf->Length= pcf->Precision;
       pcf->Scale= fp->decimals();
       break;
     case TYPE_DATE:
diff --git a/storage/connect/reldef.h b/storage/connect/reldef.h
index eb559864a34..29e4bf77f44 100644
--- a/storage/connect/reldef.h
+++ b/storage/connect/reldef.h
@@ -196,6 +196,7 @@ class DllExport COLDEF : public COLCRT { /* Column description block
   int     Define(PGLOBAL g, void *memp, PCOLINFO cfp, int poff);
   void    Define(PGLOBAL g, PCOL colp);
   bool    IsSpecial(void) {return (Flags & U_SPECIAL) ? true : false;} 
+  bool    IsVirtual(void) {return (Flags & U_VIRTUAL) ? true : false;} 
 
  protected:
   int     Buf_Type;            /* Internal data type                   */
diff --git a/storage/connect/tabdos.cpp b/storage/connect/tabdos.cpp
index 86ce92160ab..e66a84f2fa4 100644
--- a/storage/connect/tabdos.cpp
+++ b/storage/connect/tabdos.cpp
@@ -924,10 +924,12 @@ DOSCOL::DOSCOL(PGLOBAL g, PCOLDEF cdp, PTDB tp, PCOL cp, int i, PSZ am)
   Long = cdp->GetLong();
   To_Val = NULL;
   OldVal = NULL;                  // Currently used only in MinMax
+  Dsp = 0;
   Ldz = false;
   Nod = false;
   Dcm = -1;
   p = cdp->GetFmt();
+  Buf = NULL;
 
   if (p && IsTypeNum(Buf_Type)) {
     // Formatted numeric value
@@ -939,6 +941,9 @@ DOSCOL::DOSCOL(PGLOBAL g, PCOLDEF cdp, PTDB tp, PCOL cp, int i, PSZ am)
         case 'N':                 // Have no decimal point
           Nod = true;
           break;
+        case 'D':                 // Decimal separator
+          Dsp = *(++p);
+          break;
         } // endswitch p
 
     // Set number of decimal digits
@@ -960,6 +965,7 @@ DOSCOL::DOSCOL(DOSCOL *col1, PTDB tdbp) : COLBLK(col1, tdbp)
   Long = col1->Long;
   To_Val = col1->To_Val;
   Ldz = col1->Ldz;
+  Dsp = col1->Dsp;
   Nod = col1->Nod;
   Dcm = col1->Dcm;
   OldVal = col1->OldVal;
@@ -1003,7 +1009,7 @@ bool DOSCOL::SetBuffer(PGLOBAL g, PVAL value, bool ok, bool check)
   } // endif's Value, Buf_Type
 
   // Allocate the buffer used in WriteColumn for numeric columns
-  if (IsTypeNum(Buf_Type))
+  if (!Buf && IsTypeNum(Buf_Type))
     Buf = (char*)PlugSubAlloc(g, NULL, MY_MAX(32, Long + Dcm + 1));
 
   // Because Colblk's have been made from a copy of the original TDB in
@@ -1048,14 +1054,18 @@ void DOSCOL::ReadColumn(PGLOBAL g)
   p = tdbp->To_Line + Deplac;
   field = Long;
 
+  /*********************************************************************/
+  /*  For a variable length file, check if the field exists.           */
+  /*********************************************************************/
+  if (tdbp->Ftype == RECFM_VAR && strlen(tdbp->To_Line) < (unsigned)Deplac)
+    field = 0;
+  else if (Dsp)
+    for(i = 0; i < field; i++)
+      if (p[i] == Dsp)
+        p[i] = '.';
+
   switch (tdbp->Ftype) {
     case RECFM_VAR:
-      /*****************************************************************/
-      /*  For a variable length file, check if the field exists.       */
-      /*****************************************************************/
-      if (strlen(tdbp->To_Line) < (unsigned)Deplac)
-        field = 0;
-
     case RECFM_FIX:            // Fixed length text file
     case RECFM_DBF:            // Fixed length DBase file
       if (Nod) switch (Buf_Type) {
@@ -1184,6 +1194,7 @@ void DOSCOL::WriteColumn(PGLOBAL g)
           len = sprintf(Buf, fmt, field - i, Value->GetTinyValue());
           break;
         case TYPE_DOUBLE:
+        case TYPE_DECIM:
           strcpy(fmt, (Ldz) ? "%0*.*lf" : "%*.*lf");
           sprintf(Buf, fmt, field + ((Nod && Dcm) ? 1 : 0),
                   Dcm, Value->GetFloatValue());
@@ -1192,7 +1203,7 @@ void DOSCOL::WriteColumn(PGLOBAL g)
           if (Nod && Dcm)
             for (i = k = 0; i < len; i++, k++)
               if (Buf[i] != ' ') {
-                if (Buf[i] == '.' || Buf[i] == ',')
+                if (Buf[i] == '.')
                   k++;
 
                 Buf[i] = Buf[k];
@@ -1200,10 +1211,13 @@ void DOSCOL::WriteColumn(PGLOBAL g)
 
           len = strlen(Buf);
           break;
+        default:
+          sprintf(g->Message, "Invalid field format for column %s", Name);
+          longjmp(g->jumper[g->jump_level], 31);
         } // endswitch BufType
 
       p2 = Buf;
-    } else                 // Standard PlugDB format
+    } else                 // Standard CONNECT format
       p2 = Value->ShowValue(Buf, field);
 
     if (trace)
@@ -1212,7 +1226,10 @@ void DOSCOL::WriteColumn(PGLOBAL g)
     if ((len = strlen(p2)) > field) {
       sprintf(g->Message, MSG(VALUE_TOO_LONG), p2, Name, field);
       longjmp(g->jumper[g->jump_level], 31);
-      } // endif
+    } else if (Dsp)
+      for (i = 0; i < len; i++)
+        if (p2[i] == '.')
+          p2[i] = Dsp; 
 
     if (trace > 1)
       htrc("buffer=%s\n", p2);
diff --git a/storage/connect/tabdos.h b/storage/connect/tabdos.h
index d6cbc8f8d92..79a2659fb70 100644
--- a/storage/connect/tabdos.h
+++ b/storage/connect/tabdos.h
@@ -193,7 +193,8 @@ class DllExport DOSCOL : public COLBLK {
   // Members
   PVAL  To_Val;       // To value used for Update/Insert
   PVAL  OldVal;       // The previous value of the object.
-  char *Buf;          // Buffer used in write operations
+  char *Buf;          // Buffer used in read/write operations
+  char  Dsp;          // The decimal separator
   bool  Ldz;          // True if field contains leading zeros
   bool  Nod;          // True if no decimal point
   int   Dcm;          // Last Dcm digits are decimals
diff --git a/storage/connect/tabfmt.cpp b/storage/connect/tabfmt.cpp
index 80bdee05b73..7665395167d 100644
--- a/storage/connect/tabfmt.cpp
+++ b/storage/connect/tabfmt.cpp
@@ -598,7 +598,7 @@ int TDBCSV::EstimatedLength(PGLOBAL g)
     PCSVCOL colp;
 
     for (colp = (PCSVCOL)Columns; colp; colp = (PCSVCOL)colp->Next)
-      if (!colp->IsSpecial())  // Not a pseudo column
+      if (!colp->IsSpecial() && !colp->IsVirtual())  // A true column
         Fields = MY_MAX(Fields, (int)colp->Fldnum);
 
     if (Columns)
@@ -641,7 +641,7 @@ bool TDBCSV::OpenDB(PGLOBAL g)
     if (!Fields)              // May have been set in TABFMT::OpenDB
       if (Mode != MODE_UPDATE && Mode != MODE_INSERT) {
         for (colp = (PCSVCOL)Columns; colp; colp = (PCSVCOL)colp->Next)
-          if (!colp->IsSpecial())  // Not a pseudo column
+          if (!colp->IsSpecial() && !colp->IsVirtual())
             Fields = MY_MAX(Fields, (int)colp->Fldnum);
 
         if (Columns)
@@ -649,7 +649,8 @@ bool TDBCSV::OpenDB(PGLOBAL g)
 
       } else
         for (cdp = tdp->GetCols(); cdp; cdp = cdp->GetNext())
-          Fields++;
+          if (!cdp->IsVirtual())
+            Fields++;
 
     Offset = (int*)PlugSubAlloc(g, NULL, sizeof(int) * Fields);
     Fldlen = (int*)PlugSubAlloc(g, NULL, sizeof(int) * Fields);
@@ -672,25 +673,27 @@ bool TDBCSV::OpenDB(PGLOBAL g)
 
     if (Field)
       // Prepare writing fields
-      if (Mode != MODE_UPDATE)
-        for (colp = (PCSVCOL)Columns; colp; colp = (PCSVCOL)colp->Next) {
-          i = colp->Fldnum;
-          len = colp->GetLength();
-          Field[i] = (PSZ)PlugSubAlloc(g, NULL, len + 1);
-          Field[i][len] = '\0';
-          Fldlen[i] = len;
-          Fldtyp[i] = IsTypeNum(colp->GetResultType());
-          } // endfor colp
-
-      else     // MODE_UPDATE
-        for (cdp = tdp->GetCols(); cdp; cdp = cdp->GetNext()) {
-          i = cdp->GetOffset() - 1;
-          len = cdp->GetLength();
-          Field[i] = (PSZ)PlugSubAlloc(g, NULL, len + 1);
-          Field[i][len] = '\0';
-          Fldlen[i] = len;
-          Fldtyp[i] = IsTypeNum(cdp->GetType());
-          } // endfor colp
+      if (Mode != MODE_UPDATE) {
+        for (colp = (PCSVCOL)Columns; colp; colp = (PCSVCOL)colp->Next)
+          if (!colp->IsSpecial() && !colp->IsVirtual()) {
+            i = colp->Fldnum;
+            len = colp->GetLength();
+            Field[i] = (PSZ)PlugSubAlloc(g, NULL, len + 1);
+            Field[i][len] = '\0';
+            Fldlen[i] = len;
+            Fldtyp[i] = IsTypeNum(colp->GetResultType());
+            } // endif colp
+
+      } else     // MODE_UPDATE
+        for (cdp = tdp->GetCols(); cdp; cdp = cdp->GetNext())
+          if (!cdp->IsVirtual()) {
+            i = cdp->GetOffset() - 1;
+            len = cdp->GetLength();
+            Field[i] = (PSZ)PlugSubAlloc(g, NULL, len + 1);
+            Field[i][len] = '\0';
+            Fldlen[i] = len;
+            Fldtyp[i] = IsTypeNum(cdp->GetType());
+            } // endif cdp
 
     } // endif Use
 
@@ -1101,7 +1104,7 @@ bool TDBFMT::OpenDB(PGLOBAL g)
     PDOSDEF tdp = (PDOSDEF)To_Def;
 
     for (colp = (PCSVCOL)Columns; colp; colp = (PCSVCOL)colp->Next)
-      if (!colp->IsSpecial())  // Not a pseudo column
+      if (!colp->IsSpecial() && !colp->IsVirtual())  // a true column
         Fields = MY_MAX(Fields, (int)colp->Fldnum);
 
     if (Columns)
@@ -1115,7 +1118,7 @@ bool TDBFMT::OpenDB(PGLOBAL g)
 
     // Get the column formats
     for (cdp = tdp->GetCols(); cdp; cdp = cdp->GetNext())
-      if ((i = cdp->GetOffset() - 1) < Fields) {
+      if (!cdp->IsVirtual() && (i = cdp->GetOffset() - 1) < Fields) {
         if (!(pfm = cdp->GetFmt())) {
           sprintf(g->Message, MSG(NO_FLD_FORMAT), i + 1, Name);
           return true;
@@ -1318,6 +1321,11 @@ void CSVCOL::ReadColumn(PGLOBAL g)
     // Field have been copied in TDB Field array
     PSZ fp = tdbp->Field[Fldnum];
 
+    if (Dsp)
+      for (int i = 0; fp[i]; i++)
+        if (fp[i] == Dsp)
+          fp[i] = '.';
+
     Value->SetValue_psz(fp);
 
     // Set null when applicable
@@ -1365,7 +1373,10 @@ void CSVCOL::WriteColumn(PGLOBAL g)
     sprintf(g->Message, MSG(BAD_FLD_LENGTH), Name, p, flen,
                         tdbp->RowNumber(g), tdbp->GetFile(g));
     longjmp(g->jumper[g->jump_level], 34);
-    } // endif
+  } else if (Dsp)
+    for (int i = 0; p[i]; i++)
+      if (p[i] == '.')
+        p[i] = Dsp; 
 
   if (trace > 1)
     htrc("buffer=%s\n", p);