This is a major update that fixes most of the issues and bugs that

have been created by the last addition of new CONNECT features.
The version previous to this one is a preliminary test version and
should not be distributed.

- Handle indexed UPDATE/DELETE. Previously this was just tested and
  an error message send when it could not be done. Now CONNECT can
  do it in all the cases. It is done by a MRR like tchnique by making
  a list of all update or delete to do, sort them, then execute them.
modified:
  storage/connect/array.cpp
  storage/connect/array.h
  storage/connect/filamap.cpp
  storage/connect/filamap.h
  storage/connect/filamdbf.cpp
  storage/connect/filamfix.cpp
  storage/connect/filamfix.h
  storage/connect/filamtxt.cpp
  storage/connect/filamtxt.h
  storage/connect/filamvct.cpp
  storage/connect/filamvct.h
  storage/connect/filamzip.cpp
  storage/connect/filamzip.h
  storage/connect/global.h
  storage/connect/ha_connect.cc
  storage/connect/ha_connect.h

- Differenciate Cardinality that returns a true or estimated table size
  and GetMaxSize that return a value equal or greater than the table
  row number. This fixes the errors of non matching opt files.
modified:
  storage/connect/connect.cc
  storage/connect/tabdos.cpp
  storage/connect/tabdos.h
  storage/connect/tabfix.cpp
  storage/connect/table.cpp
  storage/connect/tabmac.h
  storage/connect/tabmysql.cpp
  storage/connect/tabmysql.h
  storage/connect/tabodbc.cpp
  storage/connect/tabodbc.h
  storage/connect/tabpivot.h
  storage/connect/tabtbl.cpp
  storage/connect/tabtbl.h
  storage/connect/tabutil.cpp
  storage/connect/tabutil.h
  storage/connect/tabwmi.h
  storage/connect/xtable.h

- Fix some errors and issues when making index and opt files.
  Erase opt and index files for void tables.
  Fix wrong calculation of Block and Last in MakeBlockValues.
  Invalidate indexes before making opt file.
  Fully handle blocked variable tables. Make opt file for blocked
  variable tables even when they have no optimised colums.
modified:
  storage/connect/tabdos.cpp
  storage/connect/xindex.h

- Fix some errors making index
  Return an error when the allocation is too small (should not
  really occur now that GetMaxSize is sure)
  Don't use XXROW index for DBF tables because of soft deleted lines.
modified:
  storage/connect/xindex.cpp

- Typo
modified:
  storage/connect/macutil.cpp
  storage/connect/tabdos.h
  storage/connect/tabsys.cpp
  storage/connect/tabsys.h

1 files changed, 181 insertions, 29 deletions

diff --git a/storage/connect/filamfix.cpp b/storage/connect/filamfix.cpp
index e37dff3b90e..98e6520b8e0 100644
--- a/storage/connect/filamfix.cpp
+++ b/storage/connect/filamfix.cpp
@@ -1,7 +1,7 @@
 /*********** File AM Fix C++ Program Source Code File (.CPP) ***********/
 /* PROGRAM NAME: FILAMFIX                                              */
 /* -------------                                                       */
-/*  Version 1.5                                                        */
+/*  Version 1.6                                                        */
 /*                                                                     */
 /* COPYRIGHT:                                                          */
 /* ----------                                                          */
@@ -55,6 +55,11 @@
 extern "C" int  trace;
 extern int num_read, num_there, num_eq[2];               // Statistics
 
+/***********************************************************************/
+/*  Routine called externally by BGXFAM MakeDeletedFile function.      */
+/***********************************************************************/
+PARRAY MakeValueArray(PGLOBAL g, PPARM pp);
+
 /* --------------------------- Class FIXFAM -------------------------- */
 
 /***********************************************************************/
@@ -80,6 +85,27 @@ FIXFAM::FIXFAM(PFIXFAM txfp) : BLKFAM(txfp)
   } // end of FIXFAM copy constructor
 
 /***********************************************************************/
+/*  SetPos: Replace the table at the specified position.               */
+/***********************************************************************/
+bool FIXFAM::SetPos(PGLOBAL g, int pos)
+  {
+  if (pos < 0) {
+    strcpy(g->Message, MSG(INV_REC_POS));
+    return true;
+    } // endif recpos
+
+  CurBlk = pos / Nrec;
+  CurNum = pos % Nrec;
+#if defined(_DEBUG)
+  num_eq[(CurBlk == OldBlk) ? 1 : 0]++;
+#endif
+
+  // Indicate the table position was externally set
+  Placed = true;
+  return false;
+  } // end of SetPos
+
+/***********************************************************************/
 /*  Allocate the block buffer for the table.                           */
 /***********************************************************************/
 bool FIXFAM::AllocateBuffer(PGLOBAL g)
@@ -128,12 +154,9 @@ void FIXFAM::ResetBuffer(PGLOBAL g)
   /*********************************************************************/
   /*  If access is random, performances can be much better when the    */
   /*  reads are done on only one row, except for small tables that can */
-  /*  be entirely read in one block. If the index is just used as a    */
-  /*  bitmap filter as for Update or Delete reading will be sequential */
-  /*  and we better keep block reading.                                */
+  /*  be entirely read in one block.                                   */
   /*********************************************************************/
-  if (Tdbp->GetMode() == MODE_READ && ReadBlks != 1 && !Padded &&
-      Tdbp->GetKindex() /*&& Tdbp->GetKindex()->IsRandom()*/) {
+  if (Tdbp->GetKindex() && ReadBlks != 1 && !Padded) {
     Nrec = 1;                       // Better for random access
     Rbuf = 0;
     Blksize = Lrecl;
@@ -316,13 +339,16 @@ int FIXFAM::WriteBuffer(PGLOBAL g)
   } else {                           // Mode == MODE_UPDATE
     // T_Stream is the temporary stream or the table file stream itself
     if (!T_Stream) {
-      if (UseTemp /*&& Tdbp->GetMode() == MODE_UPDATE*/) {
-        if (OpenTempFile(g))
+      if (UseTemp) {
+        if ((Indxd = Tdbp->GetKindex() != NULL)) {
+          strcpy(g->Message, "FIX indexed udate using temp file NIY");
           return RC_FX;
-
-        if (CopyHeader(g))           // For DBF tables
+        } else if (OpenTempFile(g))
+          return RC_FX;
+        else if (CopyHeader(g))           // For DBF tables
           return RC_FX;
 
+//      Indxd = Tdbp->GetKindex() != NULL;
       } else
         T_Stream = Stream;
 
@@ -387,12 +413,17 @@ int FIXFAM::DeleteRecords(PGLOBAL g, int irc)
       Spos = Tpos = Fpos;
     } // endif UseTemp
 
+    Indxd = Tdbp->GetKindex() != NULL;
     } // endif Tpos == Spos
 
   /*********************************************************************/
   /*  Move any intermediate lines.                                     */
   /*********************************************************************/
-  if (MoveIntermediateLines(g, &moved))
+  if (Indxd) {
+    // Moving will be done later, must be done in sequential order
+    (void)AddListValue(g, TYPE_INT, &Fpos, &To_Pos);
+    moved = false;
+  } else if (MoveIntermediateLines(g, &moved))
     return RC_FX;
 
   if (irc == RC_OK) {
@@ -421,10 +452,13 @@ int FIXFAM::DeleteRecords(PGLOBAL g, int irc)
       /*****************************************************************/
       /*  Ok, now delete old file and rename new temp file.            */
       /*****************************************************************/
-      if (RenameTempFile(g, false))
+      if (RenameTempFile(g))
         return RC_FX;
 
     } else {
+      if (Indxd && (Abort = MakeDeletedFile(g)))
+        return RC_FX;
+
       /*****************************************************************/
       /*  Because the chsize functionality is only accessible with a   */
       /*  system call we must close the file and reopen it with the    */
@@ -526,6 +560,59 @@ bool FIXFAM::MoveIntermediateLines(PGLOBAL g, bool *b)
   } // end of MoveIntermediate Lines
 
 /***********************************************************************/
+/*  MakeDeletedFile. When deleting using indexing, the issue is that   */
+/*  record are not necessarily deleted in sequential order. Moving     */
+/*  intermediate lines cannot be done while deleing them because       */
+/*  this can cause extra wrong records to be included in the new file. */
+/*  What we do here is to reorder the deleted record and make the new  */
+/*  deleted file from the ordered deleted records.                     */
+/***********************************************************************/
+bool FIXFAM::MakeDeletedFile(PGLOBAL g)
+  {
+  char *crlf = "\n", *mode = UseTemp ? "rb" : "r+b";
+  int  *ix, i;
+  bool  moved;
+
+  /*********************************************************************/
+  /*  Open the temporary file, Spos is at the beginning of file.       */
+  /*********************************************************************/
+  if (!(Posar = MakeValueArray(g, To_Pos))) {
+    strcpy(g->Message, "Position array is null");
+    goto err;
+  } else if (!(ix = (int*)Posar->GetSortIndex(g))) { 
+    strcpy(g->Message, "Error getting array sort index");
+    goto err;
+  } // endif's
+
+  Spos = 0;
+
+  for (i = 0; i < Posar->GetNval(); i++) {
+    Fpos = Posar->GetIntValue(ix[i]);
+
+    if (i || UseTemp) {
+      // Copy all not updated lines preceding this one
+      if (MoveIntermediateLines(g, &moved))
+        goto err;
+
+    } else
+      Tpos = Fpos;
+
+    // New start position
+    Spos = Fpos + 1; 
+    } // endfor i
+
+  if (!PlugCloseFile(g, To_Fbt) && !PlugCloseFile(g, To_Fb))
+    return false;
+
+err:
+  if (trace)
+    htrc("%s\n", g->Message);
+
+  PlugCloseFile(g, To_Fbt);
+  return true;
+  } // end of MakeDeletedFile
+
+/***********************************************************************/
 /*  Table file close routine for FIX access method.                    */
 /***********************************************************************/
 void FIXFAM::CloseTableFile(PGLOBAL g, bool abort)
@@ -533,6 +620,8 @@ void FIXFAM::CloseTableFile(PGLOBAL g, bool abort)
   int rc = RC_OK, wrc = RC_OK;
   MODE mode = Tdbp->GetMode();
 
+  Abort = abort;
+
   // Closing is True if last Write was in error
   if (mode == MODE_INSERT && CurNum && !Closing) {
     // Some more inserted lines remain to be written
@@ -547,16 +636,17 @@ void FIXFAM::CloseTableFile(PGLOBAL g, bool abort)
       } // endif Modif
 
     if (UseTemp && T_Stream && wrc == RC_OK) {
-      if (!abort) {
+      if (!Abort) {
         // Copy any remaining lines
         bool b;
     
+        // Note: Indxd is not implemented yet
         Fpos = Tdbp->Cardinality(g);
-        abort = MoveIntermediateLines(g, &b) != RC_OK;
-        } // endif // abort
+        Abort = MoveIntermediateLines(g, &b) != RC_OK;
+        } // endif Abort
 
       // Delete the old file and rename the new temp file.
-      RenameTempFile(g, abort);
+      RenameTempFile(g);
       goto fin;
       } // endif UseTemp
 
@@ -1143,7 +1233,10 @@ int BGXFAM::WriteBuffer(PGLOBAL g)
     if (Tfile == INVALID_HANDLE_VALUE)
     {
       if (UseTemp /*&& Tdbp->GetMode() == MODE_UPDATE*/) {
-        if (OpenTempFile(g))
+        if ((Indxd = Tdbp->GetKindex() != NULL)) {
+          strcpy(g->Message, "FIX indexed udate using temp file NIY");
+          return RC_FX;
+        } else if (OpenTempFile(g))
           return RC_FX;
 
       } else
@@ -1210,15 +1303,19 @@ int BGXFAM::DeleteRecords(PGLOBAL g, int irc)
       Spos = Tpos = Fpos;
     } // endif UseTemp
 
+    Indxd = Tdbp->GetKindex() != NULL;
     } // endif Tpos == Spos
 
   /*********************************************************************/
   /*  Move any intermediate lines.                                     */
   /*********************************************************************/
-  if (MoveIntermediateLines(g, &moved))
+  if (Indxd)
+    // Moving will be done later, must be done in sequential order
+    (void)AddListValue(g, TYPE_INT, &Fpos, &To_Pos);
+  else if (MoveIntermediateLines(g, &moved))
     return RC_FX;
 
-  if (irc == RC_OK) {
+  if (irc == RC_OK && !Indxd) {
     if (trace)
       assert(Spos == Fpos);
 
@@ -1234,22 +1331,21 @@ int BGXFAM::DeleteRecords(PGLOBAL g, int irc)
     if (trace > 1)
       htrc("after: Tpos=%d Spos=%d\n", Tpos, Spos);
 
-  } else {
+  } else if (irc != RC_OK) {
     /*******************************************************************/
     /*  Last call after EOF has been reached.                          */
     /*******************************************************************/
-    char filename[_MAX_PATH];
-
-    PlugSetPath(filename, To_File, Tdbp->GetPath());
-
     if (UseTemp) {
       /*****************************************************************/
       /*  Ok, now delete old file and rename new temp file.            */
       /*****************************************************************/
-      if (RenameTempFile(g, false))
+      if (RenameTempFile(g))
         return RC_FX;
 
     } else {
+      if (Indxd && (Abort = MakeDeletedFile(g)))
+        return RC_FX;
+
       /*****************************************************************/
       /*  Remove extra records.                                        */
       /*****************************************************************/
@@ -1374,6 +1470,59 @@ bool BGXFAM::MoveIntermediateLines(PGLOBAL g, bool *b)
   } // end of MoveIntermediateLines
 
 /***********************************************************************/
+/*  MakeDeletedFile. When deleting using indexing, the issue is that   */
+/*  record are not necessarily deleted in sequential order. Moving     */
+/*  intermediate lines cannot be done while deleing them because       */
+/*  this can cause extra wrong records to be included in the new file. */
+/*  What we do here is to reorder the deleted record and make the new  */
+/*  deleted file from the ordered deleted records.                     */
+/***********************************************************************/
+bool BGXFAM::MakeDeletedFile(PGLOBAL g)
+  {
+  char *crlf = "\n", *mode = UseTemp ? "rb" : "r+b";
+  int  *ix, i;
+  bool  moved;
+
+  /*********************************************************************/
+  /*  Open the temporary file, Spos is at the beginning of file.       */
+  /*********************************************************************/
+  if (!(Posar = MakeValueArray(g, To_Pos))) {
+    strcpy(g->Message, "Position array is null");
+    goto err;
+  } else if (!(ix = (int*)Posar->GetSortIndex(g))) { 
+    strcpy(g->Message, "Error getting array sort index");
+    goto err;
+  } // endif's
+
+  Spos = 0;
+
+  for (i = 0; i < Posar->GetNval(); i++) {
+    Fpos = Posar->GetIntValue(ix[i]);
+
+    if (i || UseTemp) {
+      // Copy all not updated lines preceding this one
+      if (MoveIntermediateLines(g, &moved))
+        goto err;
+
+    } else
+      Tpos = Fpos;
+
+    // New start position
+    Spos = Fpos + 1; 
+    } // endfor i
+
+  if (!PlugCloseFile(g, To_Fbt))
+    return false;
+
+err:
+  if (trace)
+    htrc("%s\n", g->Message);
+
+  PlugCloseFile(g, To_Fbt);
+  return true;
+  } // end of MakeDeletedFile
+
+/***********************************************************************/
 /*  Data Base close routine for BIGFIX access method.                  */
 /***********************************************************************/
 void BGXFAM::CloseTableFile(PGLOBAL g, bool abort)
@@ -1381,6 +1530,8 @@ void BGXFAM::CloseTableFile(PGLOBAL g, bool abort)
   int rc = RC_OK, wrc = RC_OK;
   MODE mode = Tdbp->GetMode();
 
+  Abort = abort;
+
   // Closing is True if last Write was in error
   if (mode == MODE_INSERT && CurNum && !Closing) {
     // Some more inserted lines remain to be written
@@ -1394,16 +1545,17 @@ void BGXFAM::CloseTableFile(PGLOBAL g, bool abort)
       } // endif Modif
 
     if (UseTemp && Tfile && wrc == RC_OK) {
-      if (!abort) {
+      if (!Abort) {
         // Copy any remaining lines
         bool b;
     
+        // Indxd is not implemented yet
         Fpos = Tdbp->Cardinality(g);
-        abort = MoveIntermediateLines(g, &b) != RC_OK;
-        } // endif abort
+        Abort = MoveIntermediateLines(g, &b) != RC_OK;
+        } // endif Abort
 
       // Delete the old file and rename the new temp file.
-      RenameTempFile(g, abort);
+      RenameTempFile(g);
       goto fin;
       } // endif UseTemp