Merge branch 'bb-10.3-release' into bb-10.4-release

Note, the fix for "MDEV-23328 Server hang due to Galera lock conflict resolution"
was null-merged. 10.4 version of the fix is coming up separately

1 files changed, 459 insertions, 5 deletions

diff --git a/storage/connect/filamtxt.cpp b/storage/connect/filamtxt.cpp
index 67ab120c499..84eab272cc5 100644
--- a/storage/connect/filamtxt.cpp
+++ b/storage/connect/filamtxt.cpp
@@ -1,11 +1,11 @@
 /*********** File AM Txt C++ Program Source Code File (.CPP) ***********/
 /* PROGRAM NAME: FILAMTXT                                              */
 /* -------------                                                       */
-/*  Version 1.7                                                        */
+/*  Version 1.8                                                        */
 /*                                                                     */
 /* COPYRIGHT:                                                          */
 /* ----------                                                          */
-/*  (C) Copyright to the author Olivier BERTRAND          2005-2017    */
+/*  (C) Copyright to the author Olivier BERTRAND          2005-2020    */
 /*                                                                     */
 /* WHAT THIS PROGRAM DOES:                                             */
 /* -----------------------                                             */
@@ -48,6 +48,7 @@
 #include "plgdbsem.h"
 #include "filamtxt.h"
 #include "tabdos.h"
+#include "tabjson.h"
 
 #if defined(UNIX) || defined(UNIV_LINUX)
 #include "osutil.h"
@@ -804,14 +805,14 @@ int DOSFAM::ReadBuffer(PGLOBAL g)
     Placed = false;
 
   if (trace(2))
-    htrc(" About to read: stream=%p To_Buf=%p Buflen=%d\n",
-                          Stream, To_Buf, Buflen);
+    htrc(" About to read: stream=%p To_Buf=%p Buflen=%d Fpos=%d\n",
+                          Stream, To_Buf, Buflen, Fpos);
 
   if (fgets(To_Buf, Buflen, Stream)) {
     p = To_Buf + strlen(To_Buf) - 1;
 
     if (trace(2))
-      htrc(" Read: To_Buf=%p p=%c\n", To_Buf, To_Buf, p);
+      htrc(" Read: To_Buf=%p p=%c\n", To_Buf, p);
 
 #if defined(__WIN__)
     if (Bin) {
@@ -1663,3 +1664,456 @@ void BLKFAM::Rewind(void)
 //Rbuf = 0;        commented out in case we reuse last read block
   } // end of Rewind
 
+/* --------------------------- Class BINFAM -------------------------- */
+
+#if 0
+/***********************************************************************/
+/*  BIN GetFileLength: returns file size in number of bytes.           */
+/***********************************************************************/
+int BINFAM::GetFileLength(PGLOBAL g)
+{
+	int len;
+
+	if (!Stream)
+		len = TXTFAM::GetFileLength(g);
+	else
+		if ((len = _filelength(_fileno(Stream))) < 0)
+			sprintf(g->Message, MSG(FILELEN_ERROR), "_filelength", To_File);
+
+	xtrc(1, "File length=%d\n", len);
+	return len;
+} // end of GetFileLength
+
+/***********************************************************************/
+/*  Cardinality: returns table cardinality in number of rows.          */
+/*  This function can be called with a null argument to test the       */
+/*  availability of Cardinality implementation (1 yes, 0 no).          */
+/***********************************************************************/
+int BINFAM::Cardinality(PGLOBAL g)
+{
+	return (g) ? -1 : 0;
+} // end of Cardinality
+
+/***********************************************************************/
+/*  OpenTableFile: Open a DOS/UNIX table file using C standard I/Os.   */
+/***********************************************************************/
+bool BINFAM::OpenTableFile(PGLOBAL g) {
+	char    opmode[4], filename[_MAX_PATH];
+	MODE    mode = Tdbp->GetMode();
+	PDBUSER dbuserp = PlgGetUser(g);
+
+	switch (mode) {
+	case MODE_READ:
+		strcpy(opmode, "rb");
+		break;
+	case MODE_WRITE:
+		strcpy(opmode, "wb");
+		break;
+	default:
+		sprintf(g->Message, MSG(BAD_OPEN_MODE), mode);
+		return true;
+	} // endswitch Mode
+
+	// Now open the file stream
+	PlugSetPath(filename, To_File, Tdbp->GetPath());
+
+	if (!(Stream = PlugOpenFile(g, filename, opmode))) {
+		if (trace(1))
+			htrc("%s\n", g->Message);
+
+		return (mode == MODE_READ && errno == ENOENT)
+			? PushWarning(g, Tdbp) : true;
+	} // endif Stream
+
+	if (trace(1))
+		htrc("File %s open Stream=%p mode=%s\n", filename, Stream, opmode);
+
+	To_Fb = dbuserp->Openlist;     // Keep track of File block
+
+	/*********************************************************************/
+	/*  Allocate the line buffer.                                        */
+	/*********************************************************************/
+	return AllocateBuffer(g);
+} // end of OpenTableFile
+#endif // 0
+
+/***********************************************************************/
+/*  Allocate the line buffer. For mode Delete a bigger buffer has to   */
+/*  be allocated because is it also used to move lines into the file.  */
+/***********************************************************************/
+bool BINFAM::AllocateBuffer(PGLOBAL g)
+{
+  MODE mode = Tdbp->GetMode();
+
+  // Lrecl is Ok
+  Buflen = Lrecl;
+
+  // Buffer will be allocated separately
+  if (mode == MODE_ANY) {
+    xtrc(1, "SubAllocating a buffer of %d bytes\n", Buflen);
+    To_Buf = (char*)PlugSubAlloc(g, NULL, Buflen);
+  } else if (UseTemp || mode == MODE_DELETE) {
+    // Have a big buffer to move lines
+    Dbflen = Buflen * DOS_BUFF_LEN;
+    DelBuf = PlugSubAlloc(g, NULL, Dbflen);
+  } // endif mode
+
+  return false;
+#if 0
+  MODE mode = Tdbp->GetMode();
+
+  // Lrecl is Ok
+  Dbflen = Buflen = Lrecl;
+
+  if (trace(1))
+    htrc("SubAllocating a buffer of %d bytes\n", Buflen);
+
+  DelBuf = To_Buf = (char*)PlugSubAlloc(g, NULL, Buflen);
+  return false;
+#endif // 0
+} // end of AllocateBuffer
+
+#if 0
+/***********************************************************************/
+/*  GetRowID: return the RowID of last read record.                    */
+/***********************************************************************/
+int BINFAM::GetRowID(void) {
+	return Rows;
+} // end of GetRowID
+
+/***********************************************************************/
+/*  GetPos: return the position of last read record.                   */
+/***********************************************************************/
+int BINFAM::GetPos(void) {
+	return Fpos;
+} // end of GetPos
+
+/***********************************************************************/
+/*  GetNextPos: return the position of next record.                    */
+/***********************************************************************/
+int BINFAM::GetNextPos(void) {
+	return ftell(Stream);
+} // end of GetNextPos
+
+/***********************************************************************/
+/*  SetPos: Replace the table at the specified position.               */
+/***********************************************************************/
+bool BINFAM::SetPos(PGLOBAL g, int pos) {
+	Fpos = pos;
+
+	if (fseek(Stream, Fpos, SEEK_SET)) {
+		sprintf(g->Message, MSG(FSETPOS_ERROR), Fpos);
+		return true;
+	} // endif
+
+	Placed = true;
+	return false;
+} // end of SetPos
+
+/***********************************************************************/
+/*  Record file position in case of UPDATE or DELETE.                  */
+/***********************************************************************/
+bool BINFAM::RecordPos(PGLOBAL g) {
+	if ((Fpos = ftell(Stream)) < 0) {
+		sprintf(g->Message, MSG(FTELL_ERROR), 0, strerror(errno));
+		//  strcat(g->Message, " (possible wrong ENDING option value)");
+		return true;
+	} // endif Fpos
+
+	return false;
+} // end of RecordPos
+#endif // 0
+
+/***********************************************************************/
+/*  ReadBuffer: Read one line for a text file.                         */
+/***********************************************************************/
+int BINFAM::ReadBuffer(PGLOBAL g)
+{
+	int rc;
+
+	if (!Stream)
+		return RC_EF;
+
+	xtrc(2, "ReadBuffer: Tdbp=%p To_Line=%p Placed=%d\n",
+					Tdbp, Tdbp->GetLine(), Placed);
+
+	if (!Placed) {
+		/*******************************************************************/
+		/*  Record file position in case of UPDATE or DELETE.              */
+		/*******************************************************************/
+		if (RecordPos(g))
+			return RC_FX;
+
+		CurBlk = (int)Rows++;
+		xtrc(2, "ReadBuffer: CurBlk=%d\n", CurBlk);
+	} else
+		Placed = false;
+
+	xtrc(2, " About to read: bstream=%p To_Buf=%p Buflen=%d Fpos=%d\n",
+						Stream, To_Buf, Buflen, Fpos);
+
+	// Read the prefix giving the row length
+	if (!fread(&Recsize, sizeof(size_t), 1, Stream)) {
+		if (!feof(Stream)) {
+			strcpy(g->Message, "Error reading line prefix\n");
+			return RC_FX;
+		} else
+			return RC_EF;
+
+	} else if (Recsize > (unsigned)Buflen) {
+		sprintf(g->Message, "Record too big (Recsize=%zd Buflen=%d)\n", Recsize, Buflen);
+		return RC_FX;
+	}	// endif Recsize
+
+	if (fread(To_Buf, Recsize, 1, Stream)) {
+		xtrc(2, " Read: To_Buf=%p Recsize=%zd\n", To_Buf, Recsize);
+		num_read++;
+		rc = RC_OK;
+	} else if (feof(Stream)) {
+		rc = RC_EF;
+	} else {
+#if defined(__WIN__)
+		sprintf(g->Message, MSG(READ_ERROR), To_File, _strerror(NULL));
+#else
+		sprintf(g->Message, MSG(READ_ERROR), To_File, strerror(0));
+#endif
+		xtrc(2, "%s\n", g->Message);
+		rc = RC_FX;
+	} // endif's fread
+
+	xtrc(2, "ReadBuffer: rc=%d\n", rc);
+	IsRead = true;
+	return rc;
+} // end of ReadBuffer
+
+/***********************************************************************/
+/*  WriteBuffer: File write routine for BIN access method.             */
+/***********************************************************************/
+int BINFAM::WriteBuffer(PGLOBAL g)
+{
+	int   curpos = 0;
+	bool  moved = true;
+
+  // T_Stream is the temporary stream or the table file stream itself
+  if (!T_Stream) {
+    if (UseTemp && Tdbp->GetMode() == MODE_UPDATE) {
+      if (OpenTempFile(g))
+        return RC_FX;
+
+    } else
+      T_Stream = Stream;
+
+  } // endif T_Stream
+
+  if (Tdbp->GetMode() == MODE_UPDATE) {
+    /*******************************************************************/
+    /*  Here we simply rewrite a record on itself. There are two cases */
+    /*  were another method should be used, a/ when Update apply to    */
+    /*  the whole file, b/ when updating the last field of a variable  */
+    /*  length file. The method could be to rewrite a new file, then   */
+    /*  to erase the old one and rename the new updated file.          */
+    /*******************************************************************/
+    curpos = ftell(Stream);
+
+    if (trace(1))
+      htrc("Last : %d cur: %d\n", Fpos, curpos);
+
+    if (UseTemp) {
+      /*****************************************************************/
+      /*  We are using a temporary file.                               */
+      /*  Before writing the updated record, we must eventually copy   */
+      /*  all the intermediate records that have not been updated.     */
+      /*****************************************************************/
+      if (MoveIntermediateLines(g, &moved))
+        return RC_FX;
+
+      Spos = curpos;                            // New start position
+    } else
+      // Update is directly written back into the file,
+      //   with this (fast) method, record size cannot change.
+      if (fseek(Stream, Fpos, SEEK_SET)) {
+        sprintf(g->Message, MSG(FSETPOS_ERROR), 0);
+        return RC_FX;
+      } // endif
+
+  } // endif mode
+
+  /*********************************************************************/
+	/*  Prepare writing the line.                                        */
+	/*********************************************************************/
+//memcpy(To_Buf, Tdbp->GetLine(), Recsize);
+
+	/*********************************************************************/
+	/*  Now start the writing process.                                   */
+	/*********************************************************************/
+	if (fwrite(&Recsize, sizeof(size_t), 1, T_Stream) != 1) {
+		sprintf(g->Message, "Error %d writing prefix to %s",
+			errno, To_File);
+		return RC_FX;
+	} else if (fwrite(To_Buf, Recsize, 1, T_Stream) != 1) {
+		sprintf(g->Message, "Error %d writing %zd bytes to %s",
+			errno, Recsize, To_File);
+		return RC_FX;
+	} // endif fwrite
+
+  if (Tdbp->GetMode() == MODE_UPDATE && moved)
+    if (fseek(Stream, curpos, SEEK_SET)) {
+      sprintf(g->Message, MSG(FSEEK_ERROR), strerror(errno));
+      return RC_FX;
+    } // endif
+
+  xtrc(1, "Binary write done\n");
+	return RC_OK;
+} // end of WriteBuffer
+
+#if 0
+/***********************************************************************/
+/*  Data Base delete line routine for DOS and BLK access methods.      */
+/***********************************************************************/
+int DOSFAM::DeleteRecords(PGLOBAL g, int irc)
+{
+  bool moved;
+  int  curpos = ftell(Stream);
+
+  /*********************************************************************/
+  /*  There is an alternative here:                                    */
+  /*  1 - use a temporary file in which are copied all not deleted     */
+  /*      lines, at the end the original file will be deleted and      */
+  /*      the temporary file renamed to the original file name.        */
+  /*  2 - directly move the not deleted lines inside the original      */
+  /*      file, and at the end erase all trailing records.             */
+  /*  This will be experimented.                                       */
+  /*********************************************************************/
+  if (trace(1))
+    htrc(
+      "DOS DeleteDB: rc=%d UseTemp=%d curpos=%d Fpos=%d Tpos=%d Spos=%d\n",
+      irc, UseTemp, curpos, Fpos, Tpos, Spos);
+
+  if (irc != RC_OK) {
+    /*******************************************************************/
+    /*  EOF: position Fpos at the end-of-file position.                */
+    /*******************************************************************/
+    fseek(Stream, 0, SEEK_END);
+    Fpos = ftell(Stream);
+
+    if (trace(1))
+      htrc("Fpos placed at file end=%d\n", Fpos);
+
+  } // endif irc
+
+  if (Tpos == Spos) {
+    /*******************************************************************/
+    /*  First line to delete, Open temporary file.                     */
+    /*******************************************************************/
+    if (UseTemp) {
+      if (OpenTempFile(g))
+        return RC_FX;
+
+    } else {
+      /*****************************************************************/
+      /*  Move of eventual preceding lines is not required here.       */
+      /*  Set the target file as being the source file itself.         */
+      /*  Set the future Tpos, and give Spos a value to block copying. */
+      /*****************************************************************/
+      T_Stream = Stream;
+      Spos = Tpos = Fpos;
+    } // endif UseTemp
+
+  } // endif Tpos == Spos
+
+    /*********************************************************************/
+    /*  Move any intermediate lines.                                     */
+    /*********************************************************************/
+  if (MoveIntermediateLines(g, &moved))
+    return RC_FX;
+
+  if (irc == RC_OK) {
+    /*******************************************************************/
+    /*  Reposition the file pointer and set Spos.                      */
+    /*******************************************************************/
+    if (!UseTemp || moved)
+      if (fseek(Stream, curpos, SEEK_SET)) {
+        sprintf(g->Message, MSG(FSETPOS_ERROR), 0);
+        return RC_FX;
+      } // endif
+
+    Spos = GetNextPos();                     // New start position
+
+    if (trace(1))
+      htrc("after: Tpos=%d Spos=%d\n", Tpos, Spos);
+
+  } else {
+    /*******************************************************************/
+    /*  Last call after EOF has been reached.                          */
+    /*  The UseTemp case is treated in CloseTableFile.                 */
+    /*******************************************************************/
+    if (!UseTemp & !Abort) {
+      /*****************************************************************/
+      /*  Because the chsize functionality is only accessible with a   */
+      /*  system call we must close the file and reopen it with the    */
+      /*  open function (_fopen for MS ??) this is still to be checked */
+      /*  for compatibility with Text files and other OS's.            */
+      /*****************************************************************/
+      char filename[_MAX_PATH];
+      int  h;                           // File handle, return code
+
+      PlugSetPath(filename, To_File, Tdbp->GetPath());
+      /*rc=*/ PlugCloseFile(g, To_Fb);
+
+      if ((h= global_open(g, MSGID_OPEN_STRERROR, filename, O_WRONLY)) <= 0)
+        return RC_FX;
+
+      /*****************************************************************/
+      /*  Remove extra records.                                        */
+      /*****************************************************************/
+#if defined(__WIN__)
+      if (chsize(h, Tpos)) {
+        sprintf(g->Message, MSG(CHSIZE_ERROR), strerror(errno));
+        close(h);
+        return RC_FX;
+      } // endif
+#else
+      if (ftruncate(h, (off_t)Tpos)) {
+        sprintf(g->Message, MSG(TRUNCATE_ERROR), strerror(errno));
+        close(h);
+        return RC_FX;
+      } // endif
+#endif
+
+      close(h);
+
+      if (trace(1))
+        htrc("done, h=%d irc=%d\n", h, irc);
+
+    } // endif !UseTemp
+
+  } // endif irc
+
+  return RC_OK;                                      // All is correct
+} // end of DeleteRecords
+
+/***********************************************************************/
+/*  Table file close routine for DOS access method.                    */
+/***********************************************************************/
+void BINFAM::CloseTableFile(PGLOBAL g, bool abort)
+{
+  int rc;
+
+  Abort = abort;
+  rc = PlugCloseFile(g, To_Fb);
+  xtrc(1, "BIN Close: closing %s rc=%d\n", To_File, rc);
+  Stream = NULL;           // So we can know whether table is open
+} // end of CloseTableFile
+
+/***********************************************************************/
+/*  Rewind routine for BIN access method.                              */
+/***********************************************************************/
+void BINFAM::Rewind(void)
+{
+	if (Stream)  // Can be NULL when making index on void table
+		rewind(Stream);
+
+	Rows = 0;
+	OldBlk = CurBlk = -1;
+} // end of Rewind
+#endif // 0