Added support for LZO compression method.

Removed: innodb_use_lz4 configuration parameter

Added: innodb_compression_algorithm configuration parameter
0 = no compression, 1 = ZLIB, 2 = LZ4, 3 = LZO

Fixed issue with incorrect trim calculations

7 files changed, 126 insertions, 61 deletions

diff --git a/storage/xtradb/CMakeLists.txt b/storage/xtradb/CMakeLists.txt
index 10118cce0c1..7e6e5a048e2 100644
--- a/storage/xtradb/CMakeLists.txt
+++ b/storage/xtradb/CMakeLists.txt
@@ -19,8 +19,10 @@ INCLUDE(CheckFunctionExists)
 INCLUDE(CheckCSourceCompiles)
 INCLUDE(CheckCSourceRuns)
 INCLUDE(lz4)
+INCLUDE(lzo)
 
-MYSQL_CHECK_LZ4()
+MYSQL_CHECK_LZ4_STATIC()
+MYSQL_CHECK_LZO_STATIC()
 
 # OS tests
 IF(UNIX)
diff --git a/storage/xtradb/fil/fil0pagecompress.cc b/storage/xtradb/fil/fil0pagecompress.cc
index 96c019e3723..e06a789e37b 100644
--- a/storage/xtradb/fil/fil0pagecompress.cc
+++ b/storage/xtradb/fil/fil0pagecompress.cc
@@ -66,6 +66,10 @@ static ulint srv_data_read, srv_data_written;
 #ifdef HAVE_LZ4
 #include "lz4.h"
 #endif
+#ifdef HAVE_LZO
+#include "lzo/lzo1x.h"
+#endif
+
 
 /****************************************************************//**
 For page compressed pages compress the page before actual write
@@ -81,7 +85,9 @@ fil_compress_page(
         byte*           out_buf,       /*!< out: compressed buffer */
         ulint           len,           /*!< in: length of input buffer.*/
         ulint           compression_level, /* in: compression level */
-	ulint*          out_len)       /*!< out: actual length of compressed page */
+	ulint*          out_len,       /*!< out: actual length of compressed
+				       page */
+	byte*		lzo_mem)       /*!< in: temporal memory used by LZO */
 {
         int err = Z_OK;
         int level = 0;
@@ -114,9 +120,11 @@ fil_compress_page(
 
 	write_size = UNIV_PAGE_SIZE - header_len;
 
+	switch(innodb_compression_algorithm) {
 #ifdef HAVE_LZ4
-	if (srv_use_lz4) {
-		err = LZ4_compress_limitedOutput((const char *)buf, (char *)out_buf+header_len, len, write_size);
+	case PAGE_LZ4_ALGORITHM:
+		err = LZ4_compress_limitedOutput((const char *)buf,
+			(char *)out_buf+header_len, len, write_size);
 		write_size = err;
 
 		if (err == 0) {
@@ -130,9 +138,26 @@ fil_compress_page(
 			*out_len = len;
 			return (buf);
 		}
-	} else {
+		break;
 #endif /* HAVE_LZ4 */
-		err = compress2(out_buf+header_len, (ulong *)&write_size, buf, len, level);
+#ifdef HAVE_LZO
+	case PAGE_LZO_ALGORITHM:
+		err = lzo1x_1_15_compress(
+			buf, len, out_buf+header_len, &write_size, lzo_mem);
+
+		if (err != LZO_E_OK || write_size > len) {
+			fprintf(stderr,
+				"InnoDB: Warning: Compression failed for space %lu name %s len %lu err %d write_size %lu",
+				space_id, fil_space_name(space), len, err, write_size);
+			srv_stats.pages_page_compression_error.inc();
+			*out_len = len;
+			return (buf);
+		}
+
+		break;
+#endif /* HAVE_LZO */
+	case PAGE_ZLIB_ALGORITHM:
+		err = compress2(out_buf+header_len, (ulong*)&write_size, buf, len, level);
 
 		if (err != Z_OK) {
 			/* If error we leave the actual page as it was */
@@ -145,9 +170,12 @@ fil_compress_page(
 			*out_len = len;
 			return (buf);
 		}
-#ifdef HAVE_LZ4
+		break;
+
+	default:
+		ut_error;
+		break;
 	}
-#endif /* HAVE_LZ4 */
 
 	/* Set up the page header */
 	memcpy(out_buf, buf, FIL_PAGE_DATA);
@@ -156,15 +184,7 @@ fil_compress_page(
 	/* Set up the correct page type */
 	mach_write_to_2(out_buf+FIL_PAGE_TYPE, FIL_PAGE_PAGE_COMPRESSED);
 	/* Set up the flush lsn to be compression algorithm */
-#ifdef HAVE_LZ4
-	if (srv_use_lz4) {
-		mach_write_to_8(out_buf+FIL_PAGE_FILE_FLUSH_LSN, FIL_PAGE_COMPRESSION_LZ4);
-	} else {
-#endif /* HAVE_LZ4 */
-		mach_write_to_8(out_buf+FIL_PAGE_FILE_FLUSH_LSN, FIL_PAGE_COMPRESSION_ZLIB);
-#ifdef HAVE_LZ4
-	}
-#endif /* HAVE_LZ4 */
+	mach_write_to_8(out_buf+FIL_PAGE_FILE_FLUSH_LSN, innodb_compression_algorithm);
 	/* Set up the actual payload lenght */
 	mach_write_to_2(out_buf+FIL_PAGE_DATA, write_size);
 
@@ -173,16 +193,7 @@ fil_compress_page(
 	ut_ad(fil_page_is_compressed(out_buf));
 	ut_ad(mach_read_from_4(out_buf+FIL_PAGE_SPACE_OR_CHKSUM) == BUF_NO_CHECKSUM_MAGIC);
 	ut_ad(mach_read_from_2(out_buf+FIL_PAGE_DATA) == write_size);
-
-#ifdef HAVE_LZ4
-	if (srv_use_lz4) {
-		ut_ad(mach_read_from_8(out_buf+FIL_PAGE_FILE_FLUSH_LSN) == FIL_PAGE_COMPRESSION_LZ4);
-	} else {
-#endif /* HAVE_LZ4 */
-		ut_ad(mach_read_from_8(out_buf+FIL_PAGE_FILE_FLUSH_LSN) == FIL_PAGE_COMPRESSION_ZLIB);
-#ifdef HAVE_LZ4
-	}
-#endif /* HAVE_LZ4 */
+	ut_ad(mach_read_from_8(out_buf+FIL_PAGE_FILE_FLUSH_LSN) == innodb_compression_algorithm);
 #endif /* UNIV_DEBUG */
 
 	write_size+=header_len;
@@ -203,11 +214,6 @@ fil_compress_page(
 
 
 	srv_stats.page_compression_saved.add((len - write_size));
-	if ((len - write_size) > 0) {
-		srv_stats.page_compression_trim_sect512.add(((len - write_size) / SECT_SIZE));
-		srv_stats.page_compression_trim_sect4096.add(((len - write_size) / (SECT_SIZE*8)));
-	}
-
 	srv_stats.pages_page_compressed.inc();
 	*out_len = write_size;
 
@@ -232,6 +238,7 @@ fil_decompress_page(
         ulint actual_size = 0;
 	ulint compression_alg = 0;
 	byte *in_buf;
+	ulint olen=0;
 
 	ut_ad(buf);
 	ut_ad(len);
@@ -258,7 +265,7 @@ fil_decompress_page(
 	if (page_buf == NULL) {
 #ifdef UNIV_PAGECOMPRESS_DEBUG
 		fprintf(stderr,
-			"InnoDB: FIL: Note: Compression buffer not given, allocating...\n");
+			"InnoDB: Note: FIL: Compression buffer not given, allocating...\n");
 #endif /* UNIV_PAGECOMPRESS_DEBUG */
 		in_buf = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE));
 	} else {
@@ -283,14 +290,15 @@ fil_decompress_page(
 		*write_size = actual_size;
 	}
 
-	if (compression_alg == FIL_PAGE_COMPRESSION_ZLIB) {
-
 #ifdef UNIV_PAGECOMPRESS_DEBUG
-		fprintf(stderr,
-			"InnoDB: Note: Preparing for decompress for len %lu\n",
-			actual_size);
+	fprintf(stderr,
+		"InnoDB: Note: Preparing for decompress for len %lu\n",
+		actual_size);
 #endif /* UNIV_PAGECOMPRESS_DEBUG */
 
+
+	switch(compression_alg) {
+	case PAGE_ZLIB_ALGORITHM:
 		err= uncompress(in_buf, &len, buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE, (unsigned long)actual_size);
 
 		/* If uncompress fails it means that page is corrupted */
@@ -306,14 +314,10 @@ fil_decompress_page(
 
 			ut_error;
 		}
+		break;
 
-#ifdef UNIV_PAGECOMPRESS_DEBUG
-		fprintf(stderr,
-			"InnoDB: Note: Decompression succeeded for len %lu \n",
-			len);
-#endif /* UNIV_PAGECOMPRESS_DEBUG */
 #ifdef HAVE_LZ4
-	} else if (compression_alg == FIL_PAGE_COMPRESSION_LZ4) {
+	case PAGE_LZ4_ALGORITHM:
 		err = LZ4_decompress_fast((const char *)buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE, (char *)in_buf, UNIV_PAGE_SIZE);
 
 		if (err != (int)actual_size) {
@@ -326,8 +330,26 @@ fil_decompress_page(
 
 			ut_error;
 		}
+		break;
 #endif /* HAVE_LZ4 */
-	} else {
+#ifdef HAVE_LZO
+	case PAGE_LZO_ALGORITHM:
+		err = lzo1x_decompress((const unsigned char *)buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE,
+			actual_size,(unsigned char *)in_buf, &olen, NULL);
+
+		if (err != LZO_E_OK || (olen == 0 || olen > UNIV_PAGE_SIZE)) {
+			fprintf(stderr,
+				"InnoDB: Corruption: Page is marked as compressed\n"
+				"InnoDB: but decompression read only %d bytes.\n"
+				"InnoDB: size %lu len %lu\n",
+				olen, actual_size, len);
+			fflush(stderr);
+
+			ut_error;
+		}
+		break;
+#endif
+	default:
 		fprintf(stderr,
 			"InnoDB: Corruption: Page is marked as compressed\n"
 			"InnoDB: but compression algorithm %s\n"
@@ -336,8 +358,15 @@ fil_decompress_page(
 
 		fflush(stderr);
 		ut_error;
+		break;
 	}
 
+#ifdef UNIV_PAGECOMPRESS_DEBUG
+	fprintf(stderr,
+		"InnoDB: Note: Decompression succeeded for len %lu \n",
+		len);
+#endif /* UNIV_PAGECOMPRESS_DEBUG */
+
 	srv_stats.pages_page_decompressed.inc();
 
 	/* Copy the uncompressed page to the buffer pool, not
diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc
index 83fd8b28394..25b96be43b7 100644
--- a/storage/xtradb/handler/ha_innodb.cc
+++ b/storage/xtradb/handler/ha_innodb.cc
@@ -103,6 +103,7 @@ this program; if not, write to the Free Software Foundation, Inc.,
 #endif /* UNIV_DEBUG */
 #include "fts0priv.h"
 #include "page0zip.h"
+#include "fil0pagecompress.h"
 
 #define thd_get_trx_isolation(X) ((enum_tx_isolation)thd_tx_isolation(X))
 
@@ -17947,12 +17948,20 @@ static MYSQL_SYSVAR_BOOL(use_trim, srv_use_trim,
   "Use trim. Default FALSE.",
   NULL, NULL, FALSE);
 
-#ifdef HAVE_LZ4
-static MYSQL_SYSVAR_BOOL(use_lz4, srv_use_lz4,
-  PLUGIN_VAR_OPCMDARG ,
-  "Use LZ4 for page compression",
-  NULL, NULL, FALSE);
-#endif /* HAVE_LZ4 */
+static MYSQL_SYSVAR_LONG(compression_algorithm, innodb_compression_algorithm,
+  PLUGIN_VAR_OPCMDARG,
+  "Compression algorithm used on page compression. 1 for zlib, 2 for lz3, 3 for lzo",
+  NULL, NULL,
+  PAGE_ZLIB_ALGORITHM,
+  0,
+#if defined(HAVE_LZO) && defined(HAVE_LZ4)
+  PAGE_ALGORITHM_LAST,
+#elif defined(HAVE_LZ4) && !defined(HAVE_LZO)
+  PAGE_ALGORITHM_LZ4,
+#else
+  PAGE_ALGORITHM_ZLIB,
+#endif
+  0);
 
 static MYSQL_SYSVAR_LONG(mtflush_threads, srv_mtflush_threads,
   PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
@@ -18159,9 +18168,7 @@ static struct st_mysql_sys_var* innobase_system_variables[]= {
   MYSQL_SYSVAR(locking_fake_changes),
   MYSQL_SYSVAR(use_stacktrace),
   MYSQL_SYSVAR(use_trim),
-#ifdef HAVE_LZ4
-  MYSQL_SYSVAR(use_lz4),
-#endif
+  MYSQL_SYSVAR(compression_algorithm),
   MYSQL_SYSVAR(mtflush_threads),
   MYSQL_SYSVAR(use_mtflush),
   NULL
diff --git a/storage/xtradb/include/fil0pagecompress.h b/storage/xtradb/include/fil0pagecompress.h
index c362c0ddcd2..0cc5aeb4678 100644
--- a/storage/xtradb/include/fil0pagecompress.h
+++ b/storage/xtradb/include/fil0pagecompress.h
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (C) 2013 SkySQL Ab. All Rights Reserved.
+Copyright (C) 2013, 2014 SkySQL Ab. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -22,6 +22,12 @@ this program; if not, write to the Free Software Foundation, Inc.,
 #include "fsp0fsp.h"
 #include "fsp0pagecompress.h"
 
+#define PAGE_UNCOMPRESSED   0
+#define PAGE_ZLIB_ALGORITHM 1
+#define PAGE_LZ4_ALGORITHM  2
+#define PAGE_LZO_ALGORITHM  3
+#define PAGE_ALGORITHM_LAST PAGE_LZO_ALGORITHM
+
 /******************************************************************//**
 @file include/fil0pagecompress.h
 Helper functions for extracting/storing page compression and
@@ -85,7 +91,9 @@ fil_compress_page(
         byte*           out_buf,       /*!< out: compressed buffer */
         ulint           len,           /*!< in: length of input buffer.*/
         ulint           compression_level, /*!< in: compression level */
-	ulint*          out_len);       /*!< out: actual length of compressed page */
+	ulint*          out_len,       /*!< out: actual length of compressed
+				       page */
+	byte*		lzo_mem);      /*!< in: temporal memory used by LZO */
 
 /****************************************************************//**
 For page compressed pages decompress the page after actual read
diff --git a/storage/xtradb/include/srv0srv.h b/storage/xtradb/include/srv0srv.h
index be16dfddc72..ea8afd450dd 100644
--- a/storage/xtradb/include/srv0srv.h
+++ b/storage/xtradb/include/srv0srv.h
@@ -269,8 +269,8 @@ extern my_bool srv_use_posix_fallocate;
 /* Use atomic writes i.e disable doublewrite buffer */
 extern my_bool srv_use_atomic_writes;
 
-/* If this flag IS TRUE, then we use lz4 to compress/decompress pages */
-extern my_bool srv_use_lz4;
+/* Compression algorithm*/
+extern long innodb_compression_algorithm;
 
 /* Number of flush threads */
 #define MTFLUSH_MAX_WORKER       64
diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc
index 646f8a87cbc..a3307fa0ba2 100644
--- a/storage/xtradb/os/os0file.cc
+++ b/storage/xtradb/os/os0file.cc
@@ -2,7 +2,7 @@
 
 Copyright (c) 1995, 2013, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2009, Percona Inc.
-Copyright (c) 2013, SkySQL Ab. All Rights Reserved.
+Copyright (c) 2013, 2014, SkySQL Ab. All Rights Reserved.
 
 Portions of this file contain modifications contributed and copyrighted
 by Percona Inc.. Those modifications are
@@ -80,6 +80,10 @@ Created 10/21/1995 Heikki Tuuri
 # endif
 #endif
 
+#ifdef HAVE_LZO
+#include "lzo/lzo1x.h"
+#endif
+
 /** Insert buffer segment id */
 static const ulint IO_IBUF_SEGMENT = 0;
 
@@ -230,6 +234,12 @@ struct os_aio_slot_t{
 	int		n_bytes;	/* bytes written/read. */
 	int		ret;		/* AIO return code */
 #endif /* WIN_ASYNC_IO */
+#ifdef HAVE_LZO
+	byte		lzo_mem[LZO1X_1_15_MEM_COMPRESS];
+#else
+	byte		lzo_mem;	/* Temporal memory used by LZO */
+#endif
+
 };
 
 /** The asynchronous i/o array structure */
@@ -4596,7 +4606,15 @@ found:
 
 		ut_ad(slot->page_buf);
 
-		tmp = fil_compress_page(fil_node_get_space_id(slot->message1), (byte *)buf, slot->page_buf, len, page_compression_level, &real_len);
+		/* Call page compression */
+		tmp = fil_compress_page(fil_node_get_space_id(slot->message1),
+			(byte *)buf,
+			slot->page_buf,
+			len,
+			page_compression_level,
+			&real_len,
+			slot->lzo_mem
+		);
 
 		/* If compression succeeded, set up the length and buffer */
 		if (tmp != buf) {
diff --git a/storage/xtradb/srv/srv0srv.cc b/storage/xtradb/srv/srv0srv.cc
index 386dbfddf0b..e70a2bd0dab 100644
--- a/storage/xtradb/srv/srv0srv.cc
+++ b/storage/xtradb/srv/srv0srv.cc
@@ -73,6 +73,7 @@ Created 10/8/1995 Heikki Tuuri
 
 #include "mysql/plugin.h"
 #include "mysql/service_thd_wait.h"
+#include "fil0pagecompress.h"
 
 /* prototypes of new functions added to ha_innodb.cc for kill_idle_transaction */
 ibool		innobase_thd_is_idle(const void* thd);
@@ -172,7 +173,7 @@ UNIV_INTERN my_bool     srv_use_posix_fallocate         = FALSE;
 /* If this flag is TRUE, then we disable doublewrite buffer */
 UNIV_INTERN my_bool     srv_use_atomic_writes           = FALSE;
 /* If this flag IS TRUE, then we use lz4 to compress/decompress pages */
-UNIV_INTERN my_bool	srv_use_lz4                     = FALSE;
+UNIV_INTERN long	innodb_compression_algorithm = PAGE_ZLIB_ALGORITHM;
 /* Number of threads used for multi-threaded flush */
 UNIV_INTERN long srv_mtflush_threads = MTFLUSH_DEFAULT_WORKER;
 /* If this flag is TRUE, then we will use multi threaded flush. */