libext2fs: don't old the CACHE_MTX while doing I/O

A report a deadlock problem caused by I/O errors (caused by e2fsck's
error handler trying to write to a bad block to perform a forced
rewrite) uncovered that we were holding the CACHE_MTX while doing read
operations.  This serialized read operations which destroyed the
performance benefits from doing parallel bitmap loading (or the
parallel e2fsck processing under development).

So restructure the code in unix_read_blk64() so that the read is
always done into the user-provided buffer, and then copied into the
cache afterwards.

Signed-off-by: Theodore Ts'o <tytso@mit.edu>

1 files changed, 13 insertions, 25 deletions

diff --git a/lib/ext2fs/unix_io.c b/lib/ext2fs/unix_io.c
index 50ac737f..e53db333 100644
--- a/lib/ext2fs/unix_io.c
+++ b/lib/ext2fs/unix_io.c
@@ -977,8 +977,8 @@ static errcode_t unix_read_blk64(io_channel channel, unsigned long long block,
 			       int count, void *buf)
 {
 	struct unix_private_data *data;
-	struct unix_cache *cache, *reuse[READ_DIRECT_SIZE];
-	errcode_t	retval = 0;
+	struct unix_cache *cache;
+	errcode_t	retval;
 	char		*cp;
 	int		i, j;
 
@@ -1005,7 +1005,7 @@ static errcode_t unix_read_blk64(io_channel channel, unsigned long long block,
 	mutex_lock(data, CACHE_MTX);
 	while (count > 0) {
 		/* If it's in the cache, use it! */
-		if ((cache = find_cached_block(data, block, &reuse[0]))) {
+		if ((cache = find_cached_block(data, block, NULL))) {
 #ifdef DEBUG
 			printf("Using cached block %lu\n", block);
 #endif
@@ -1015,47 +1015,35 @@ static errcode_t unix_read_blk64(io_channel channel, unsigned long long block,
 			cp += channel->block_size;
 			continue;
 		}
-		if (count == 1) {
-			/*
-			 * Special case where we read directly into the
-			 * cache buffer; important in the O_DIRECT case
-			 */
-			cache = reuse[0];
-			reuse_cache(channel, data, cache, block);
-			if ((retval = raw_read_blk(channel, data, block, 1,
-						   cache->buf))) {
-				cache->in_use = 0;
-				break;
-			}
-			memcpy(cp, cache->buf, channel->block_size);
-			retval = 0;
-			break;
-		}
 
 		/*
 		 * Find the number of uncached blocks so we can do a
 		 * single read request
 		 */
 		for (i=1; i < count; i++)
-			if (find_cached_block(data, block+i, &reuse[i]))
+			if (find_cached_block(data, block+i, NULL))
 				break;
 #ifdef DEBUG
 		printf("Reading %d blocks starting at %lu\n", i, block);
 #endif
+		mutex_unlock(data, CACHE_MTX);
 		if ((retval = raw_read_blk(channel, data, block, i, cp)))
-			break;
+			return retval;
+		mutex_lock(data, CACHE_MTX);
 
 		/* Save the results in the cache */
 		for (j=0; j < i; j++) {
+			if (!find_cached_block(data, block, &cache)) {
+				reuse_cache(channel, data, cache, block);
+				memcpy(cache->buf, cp, channel->block_size);
+			}
 			count--;
-			cache = reuse[j];
-			reuse_cache(channel, data, cache, block++);
-			memcpy(cache->buf, cp, channel->block_size);
+			block++;
 			cp += channel->block_size;
 		}
 	}
 	mutex_unlock(data, CACHE_MTX);
-	return retval;
+	return 0;
 #endif /* NO_IO_CACHE */
 }