Don't choke on files that are removed while pg_rewind runs.

If a file is removed from the source server, while pg_rewind is running, the
invocation of pg_read_binary_file() will fail. Use the just-added missing_ok
option to that function, to have it return NULL instead, and handle that
gracefully. And similarly for pg_ls_dir and pg_stat_file.

Reported by Fujii Masao, fix by Michael Paquier.

1 files changed, 30 insertions, 7 deletions

diff --git a/src/bin/pg_rewind/libpq_fetch.c b/src/bin/pg_rewind/libpq_fetch.c
index 37e515bc98..05aa133cf3 100644
--- a/src/bin/pg_rewind/libpq_fetch.c
+++ b/src/bin/pg_rewind/libpq_fetch.c
@@ -149,14 +149,14 @@ libpqProcessFileList(void)
 	sql =
 		"WITH RECURSIVE files (path, filename, size, isdir) AS (\n"
 		"  SELECT '' AS path, filename, size, isdir FROM\n"
-		"  (SELECT pg_ls_dir('.') AS filename) AS fn,\n"
-		"        pg_stat_file(fn.filename) AS this\n"
+		"  (SELECT pg_ls_dir('.', true, false) AS filename) AS fn,\n"
+		"        pg_stat_file(fn.filename, true) AS this\n"
 		"  UNION ALL\n"
 		"  SELECT parent.path || parent.filename || '/' AS path,\n"
 		"         fn, this.size, this.isdir\n"
 		"  FROM files AS parent,\n"
-		"       pg_ls_dir(parent.path || parent.filename) AS fn,\n"
-		"       pg_stat_file(parent.path || parent.filename || '/' || fn) AS this\n"
+		"       pg_ls_dir(parent.path || parent.filename, true, false) AS fn,\n"
+		"       pg_stat_file(parent.path || parent.filename || '/' || fn, true) AS this\n"
 		"       WHERE parent.isdir = 't'\n"
 		")\n"
 		"SELECT path || filename, size, isdir,\n"
@@ -183,6 +183,15 @@ libpqProcessFileList(void)
 		char	   *link_target = PQgetvalue(res, i, 3);
 		file_type_t type;
 
+		if (PQgetisnull(res, 0, 1))
+		{
+			/*
+			 * The file was removed from the server while the query was
+			 * running. Ignore it.
+			 */
+			continue;
+		}
+
 		if (link_target[0])
 			type = FILE_TYPE_SYMLINK;
 		else if (isdir)
@@ -259,8 +268,7 @@ receiveFileChunks(const char *sql)
 		}
 
 		if (PQgetisnull(res, 0, 0) ||
-			PQgetisnull(res, 0, 1) ||
-			PQgetisnull(res, 0, 2))
+			PQgetisnull(res, 0, 1))
 		{
 			pg_fatal("unexpected null values in result while fetching remote files\n");
 		}
@@ -280,6 +288,21 @@ receiveFileChunks(const char *sql)
 
 		chunk = PQgetvalue(res, 0, 2);
 
+		/*
+		 * It's possible that the file was deleted on remote side after we
+		 * created the file map. In this case simply ignore it, as if it was
+		 * not there in the first place, and move on.
+		 */
+		if (PQgetisnull(res, 0, 2))
+		{
+			pg_log(PG_DEBUG,
+				   "received NULL chunk for file \"%s\", file has been deleted\n",
+				   filename);
+			pg_free(filename);
+			PQclear(res);
+			continue;
+		}
+
 		pg_log(PG_DEBUG, "received chunk for file \"%s\", offset %d, size %d\n",
 			   filename, chunkoff, chunksize);
 
@@ -445,7 +468,7 @@ libpq_executeFileMap(filemap_t *map)
 	 */
 	sql =
 		"SELECT path, begin, \n"
-		"  pg_read_binary_file(path, begin, len) AS chunk\n"
+		"  pg_read_binary_file(path, begin, len, true) AS chunk\n"
 		"FROM fetchchunks\n";
 
 	receiveFileChunks(sql);