MDEV-21348 - column compression memory leak

The Storage-Engine Independent Column Compression does not call
deflateEnd() when deflate() does not return Z_STREAM_END.

This for instance happens when the data is already (externally)
compressed and deflate() needs more space than the original data.

This patch is based on contribution by Martijn Broenland.

3 files changed, 33 insertions, 9 deletions

diff --git a/mysql-test/main/column_compression.result b/mysql-test/main/column_compression.result
index 121e74ee5eb..e19a4fb75e0 100644
--- a/mysql-test/main/column_compression.result
+++ b/mysql-test/main/column_compression.result
@@ -2663,3 +2663,12 @@ CREATE TABLE t1 (a NVARCHAR(10) COMPRESSED BINARY COMPRESSED);
 ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'COMPRESSED)' at line 1
 CREATE TABLE t1 (a NVARCHAR(10) COMPRESSED DEFAULT '' COMPRESSED);
 ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'COMPRESSED)' at line 1
+#
+# MDEV-21348 - Memory leak in Storage-Engine Independent Column
+#              Compression
+#
+CREATE TABLE t1(a BLOB COMPRESSED);
+SET column_compression_threshold=0;
+INSERT INTO t1 VALUES('aa');
+SET column_compression_threshold=DEFAULT;
+DROP TABLE t1;
diff --git a/mysql-test/main/column_compression.test b/mysql-test/main/column_compression.test
index 092125735cf..1560b23ad3e 100644
--- a/mysql-test/main/column_compression.test
+++ b/mysql-test/main/column_compression.test
@@ -255,3 +255,14 @@ DROP TABLE t1;
 CREATE TABLE t1 (a NVARCHAR(10) COMPRESSED BINARY COMPRESSED);
 --error ER_PARSE_ERROR
 CREATE TABLE t1 (a NVARCHAR(10) COMPRESSED DEFAULT '' COMPRESSED);
+
+
+--echo #
+--echo # MDEV-21348 - Memory leak in Storage-Engine Independent Column
+--echo #              Compression
+--echo #
+CREATE TABLE t1(a BLOB COMPRESSED);
+SET column_compression_threshold=0;
+INSERT INTO t1 VALUES('aa');
+SET column_compression_threshold=DEFAULT;
+DROP TABLE t1;
diff --git a/sql/field_comp.cc b/sql/field_comp.cc
index eb4ae42aa4d..ab97c8ccf08 100644
--- a/sql/field_comp.cc
+++ b/sql/field_comp.cc
@@ -67,10 +67,12 @@ static uint compress_zlib(THD *thd, char *to, const char *from, uint length)
     stream.zfree= 0;
     stream.opaque= 0;
 
-    if (deflateInit2(&stream, level, Z_DEFLATED, wbits, 8, strategy) == Z_OK &&
-        deflate(&stream, Z_FINISH) == Z_STREAM_END &&
-        deflateEnd(&stream) == Z_OK)
-      return (uint) (stream.next_out - (Bytef*) to);
+    if (deflateInit2(&stream, level, Z_DEFLATED, wbits, 8, strategy) == Z_OK)
+    {
+      int res= deflate(&stream, Z_FINISH);
+      if (deflateEnd(&stream) == Z_OK && res == Z_STREAM_END)
+        return (uint) (stream.next_out - (Bytef*) to);
+    }
   }
   return 0;
 }
@@ -117,12 +119,14 @@ static int uncompress_zlib(String *to, const uchar *from, uint from_length,
   stream.zfree= 0;
   stream.opaque= 0;
 
-  if (inflateInit2(&stream, wbits) == Z_OK &&
-      inflate(&stream, Z_FINISH) == Z_STREAM_END &&
-      inflateEnd(&stream) == Z_OK)
+  if (inflateInit2(&stream, wbits) == Z_OK)
   {
-    to->length(stream.total_out);
-    return 0;
+    int res= inflate(&stream, Z_FINISH);
+    if (inflateEnd(&stream) == Z_OK && res == Z_STREAM_END)
+    {
+      to->length(stream.total_out);
+      return 0;
+    }
   }
   my_error(ER_ZLIB_Z_DATA_ERROR, MYF(0));
   return 1;