diff options
author | Jan Lindström <jan.lindstrom@skysql.com> | 2015-02-19 17:42:18 +0200 |
---|---|---|
committer | Jan Lindström <jan.lindstrom@skysql.com> | 2015-02-19 17:42:18 +0200 |
commit | 9152b83973419ea034bb6040703c18b3ee87e084 (patch) | |
tree | 63a491777c10b1bc4afb3141838676d85df704f3 | |
parent | bab1c686a464afab46caef7639086acefa43b63c (diff) | |
download | mariadb-git-9152b83973419ea034bb6040703c18b3ee87e084.tar.gz |
Merged from 10.0-FusionIO:
Added support for compression method snappy for page compression.
21 files changed, 999 insertions, 55 deletions
diff --git a/cmake/snappy.cmake b/cmake/snappy.cmake new file mode 100644 index 00000000000..cb0839a3480 --- /dev/null +++ b/cmake/snappy.cmake @@ -0,0 +1,32 @@ +# Copyright (C) 2015, MariaDB Corporation. All Rights Reserved. +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free Software +# Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +SET(WITH_INNODB_SNAPPY AUTO CACHE STRING + "Build with snappy. Possible values are 'ON', 'OFF', 'AUTO' and default is 'AUTO'") + +MACRO (MYSQL_CHECK_SNAPPY) + IF (WITH_INNODB_SNAPPY STREQUAL "ON" OR WITH_INNODB_SNAPPY STREQUAL "AUTO") + CHECK_INCLUDE_FILES(snappy-c.h HAVE_SNAPPY_H) + CHECK_LIBRARY_EXISTS(snappy snappy_uncompress "" HAVE_SNAPPY_SHARED_LIB) + + IF(HAVE_SNAPPY_SHARED_LIB AND HAVE_SNAPPY_H) + ADD_DEFINITIONS(-DHAVE_SNAPPY=1) + LINK_LIBRARIES(snappy) + ELSE() + IF (WITH_INNODB_SNAPPY STREQUAL "ON") + MESSAGE(FATAL_ERROR "Required snappy library is not found") + ENDIF() + ENDIF() + ENDIF() +ENDMACRO() diff --git a/mysql-test/include/have_innodb_snappy.inc b/mysql-test/include/have_innodb_snappy.inc new file mode 100644 index 00000000000..c4dca4c19ee --- /dev/null +++ b/mysql-test/include/have_innodb_snappy.inc @@ -0,0 +1,4 @@ +if (! `SELECT COUNT(*) FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE LOWER(variable_name) = 'innodb_have_snappy' AND variable_value = 'ON'`) +{ + --skip Test requires InnoDB compiled with libsnappy +} diff --git a/mysql-test/suite/innodb/r/innodb-page_compression_snappy.result b/mysql-test/suite/innodb/r/innodb-page_compression_snappy.result new file mode 100644 index 00000000000..1709d8e9d2b --- /dev/null +++ b/mysql-test/suite/innodb/r/innodb-page_compression_snappy.result @@ -0,0 +1,438 @@ +call mtr.add_suppression("InnoDB: Warning: Compression failed for space*"); +set global innodb_file_format = `barracuda`; +set global innodb_file_per_table = on; +set global innodb_compression_algorithm = 6; +create table innodb_compressed(c1 int, b char(20)) engine=innodb row_format=compressed key_block_size=8; +show warnings; +Level Code Message +create table innodb_normal (c1 int, b char(20)) engine=innodb; +show warnings; +Level Code Message +create table innodb_page_compressed1 (c1 int, b char(20)) engine=innodb page_compressed=1 page_compression_level=1; +show warnings; +Level Code Message +show create table innodb_page_compressed1; +Table Create Table +innodb_page_compressed1 CREATE TABLE `innodb_page_compressed1` ( + `c1` int(11) DEFAULT NULL, + `b` char(20) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 `page_compressed`=1 `page_compression_level`=1 +create table innodb_page_compressed2 (c1 int, b char(20)) engine=innodb page_compressed=1 page_compression_level=2; +show warnings; +Level Code Message +show create table innodb_page_compressed2; +Table Create Table +innodb_page_compressed2 CREATE TABLE `innodb_page_compressed2` ( + `c1` int(11) DEFAULT NULL, + `b` char(20) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 `page_compressed`=1 `page_compression_level`=2 +create table innodb_page_compressed3 (c1 int, b char(20)) engine=innodb page_compressed=1 page_compression_level=3; +show warnings; +Level Code Message +show create table innodb_page_compressed3; +Table Create Table +innodb_page_compressed3 CREATE TABLE `innodb_page_compressed3` ( + `c1` int(11) DEFAULT NULL, + `b` char(20) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 `page_compressed`=1 `page_compression_level`=3 +create table innodb_page_compressed4 (c1 int, b char(20)) engine=innodb page_compressed=1 page_compression_level=4; +show warnings; +Level Code Message +show create table innodb_page_compressed4; +Table Create Table +innodb_page_compressed4 CREATE TABLE `innodb_page_compressed4` ( + `c1` int(11) DEFAULT NULL, + `b` char(20) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 `page_compressed`=1 `page_compression_level`=4 +create table innodb_page_compressed5 (c1 int, b char(20)) engine=innodb page_compressed=1 page_compression_level=5; +show warnings; +Level Code Message +show create table innodb_page_compressed5; +Table Create Table +innodb_page_compressed5 CREATE TABLE `innodb_page_compressed5` ( + `c1` int(11) DEFAULT NULL, + `b` char(20) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 `page_compressed`=1 `page_compression_level`=5 +create table innodb_page_compressed6 (c1 int, b char(20)) engine=innodb page_compressed=1 page_compression_level=6; +show warnings; +Level Code Message +show create table innodb_page_compressed6; +Table Create Table +innodb_page_compressed6 CREATE TABLE `innodb_page_compressed6` ( + `c1` int(11) DEFAULT NULL, + `b` char(20) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 `page_compressed`=1 `page_compression_level`=6 +create table innodb_page_compressed7 (c1 int, b char(20)) engine=innodb page_compressed=1 page_compression_level=7; +show warnings; +Level Code Message +show create table innodb_page_compressed7; +Table Create Table +innodb_page_compressed7 CREATE TABLE `innodb_page_compressed7` ( + `c1` int(11) DEFAULT NULL, + `b` char(20) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 `page_compressed`=1 `page_compression_level`=7 +create table innodb_page_compressed8 (c1 int, b char(20)) engine=innodb page_compressed=1 page_compression_level=8; +show warnings; +Level Code Message +show create table innodb_page_compressed8; +Table Create Table +innodb_page_compressed8 CREATE TABLE `innodb_page_compressed8` ( + `c1` int(11) DEFAULT NULL, + `b` char(20) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 `page_compressed`=1 `page_compression_level`=8 +create table innodb_page_compressed9 (c1 int, b char(20)) engine=innodb page_compressed=1 page_compression_level=9; +show warnings; +Level Code Message +show create table innodb_page_compressed9; +Table Create Table +innodb_page_compressed9 CREATE TABLE `innodb_page_compressed9` ( + `c1` int(11) DEFAULT NULL, + `b` char(20) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 `page_compressed`=1 `page_compression_level`=9 +create procedure innodb_insert_proc (repeat_count int) +begin +declare current_num int; +set current_num = 0; +while current_num < repeat_count do +insert into innodb_normal values(current_num,'testing..'); +set current_num = current_num + 1; +end while; +end// +commit; +set autocommit=0; +call innodb_insert_proc(5000); +commit; +set autocommit=1; +select count(*) from innodb_normal; +count(*) +5000 +insert into innodb_compressed select * from innodb_normal; +insert into innodb_page_compressed1 select * from innodb_normal; +insert into innodb_page_compressed2 select * from innodb_normal; +insert into innodb_page_compressed3 select * from innodb_normal; +insert into innodb_page_compressed4 select * from innodb_normal; +insert into innodb_page_compressed5 select * from innodb_normal; +insert into innodb_page_compressed6 select * from innodb_normal; +insert into innodb_page_compressed7 select * from innodb_normal; +insert into innodb_page_compressed8 select * from innodb_normal; +insert into innodb_page_compressed9 select * from innodb_normal; +commit; +select count(*) from innodb_compressed; +count(*) +5000 +select count(*) from innodb_page_compressed1; +count(*) +5000 +select count(*) from innodb_page_compressed1 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed2 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed3 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed4 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed5 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed6 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed7 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed8 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed9 where c1 < 500000; +count(*) +5000 +alter table innodb_normal page_compressed=1 page_compression_level=8; +show warnings; +Level Code Message +show create table innodb_normal; +Table Create Table +innodb_normal CREATE TABLE `innodb_normal` ( + `c1` int(11) DEFAULT NULL, + `b` char(20) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 `page_compressed`=1 `page_compression_level`=8 +alter table innodb_compressed row_format=default page_compressed=1 page_compression_level=8 key_block_size=0; +show warnings; +Level Code Message +show create table innodb_compressed; +Table Create Table +innodb_compressed CREATE TABLE `innodb_compressed` ( + `c1` int(11) DEFAULT NULL, + `b` char(20) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 `page_compressed`=1 `page_compression_level`=8 +update innodb_page_compressed1 set c1 = c1 + 1; +update innodb_page_compressed2 set c1 = c1 + 1; +update innodb_page_compressed3 set c1 = c1 + 1; +update innodb_page_compressed4 set c1 = c1 + 1; +update innodb_page_compressed5 set c1 = c1 + 1; +update innodb_page_compressed6 set c1 = c1 + 1; +update innodb_page_compressed7 set c1 = c1 + 1; +update innodb_page_compressed8 set c1 = c1 + 1; +update innodb_page_compressed9 set c1 = c1 + 1; +select count(*) from innodb_compressed; +count(*) +5000 +select count(*) from innodb_page_compressed1; +count(*) +5000 +select count(*) from innodb_page_compressed1 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed2 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed3 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed4 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed5 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed6 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed7 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed8 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed9 where c1 < 500000; +count(*) +5000 +update innodb_page_compressed1 set c1 = c1 + 1; +update innodb_page_compressed2 set c1 = c1 + 1; +update innodb_page_compressed3 set c1 = c1 + 1; +update innodb_page_compressed4 set c1 = c1 + 1; +update innodb_page_compressed5 set c1 = c1 + 1; +update innodb_page_compressed6 set c1 = c1 + 1; +update innodb_page_compressed7 set c1 = c1 + 1; +update innodb_page_compressed8 set c1 = c1 + 1; +update innodb_page_compressed9 set c1 = c1 + 1; +select count(*) from innodb_compressed; +count(*) +5000 +select count(*) from innodb_page_compressed1; +count(*) +5000 +select count(*) from innodb_page_compressed1 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed2 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed3 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed4 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed5 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed6 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed7 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed8 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed9 where c1 < 500000; +count(*) +5000 +set global innodb_compression_algorithm = 1; +update innodb_page_compressed1 set c1 = c1 + 1; +update innodb_page_compressed2 set c1 = c1 + 1; +update innodb_page_compressed3 set c1 = c1 + 1; +update innodb_page_compressed4 set c1 = c1 + 1; +update innodb_page_compressed5 set c1 = c1 + 1; +update innodb_page_compressed6 set c1 = c1 + 1; +update innodb_page_compressed7 set c1 = c1 + 1; +update innodb_page_compressed8 set c1 = c1 + 1; +update innodb_page_compressed9 set c1 = c1 + 1; +commit; +select count(*) from innodb_compressed; +count(*) +5000 +select count(*) from innodb_page_compressed1; +count(*) +5000 +select count(*) from innodb_page_compressed1 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed2 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed3 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed4 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed5 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed6 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed7 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed8 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed9 where c1 < 500000; +count(*) +5000 +update innodb_page_compressed1 set c1 = c1 + 1; +update innodb_page_compressed2 set c1 = c1 + 1; +update innodb_page_compressed3 set c1 = c1 + 1; +update innodb_page_compressed4 set c1 = c1 + 1; +update innodb_page_compressed5 set c1 = c1 + 1; +update innodb_page_compressed6 set c1 = c1 + 1; +update innodb_page_compressed7 set c1 = c1 + 1; +update innodb_page_compressed8 set c1 = c1 + 1; +update innodb_page_compressed9 set c1 = c1 + 1; +select count(*) from innodb_compressed; +count(*) +5000 +select count(*) from innodb_page_compressed1; +count(*) +5000 +select count(*) from innodb_page_compressed1 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed2 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed3 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed4 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed5 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed6 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed7 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed8 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed9 where c1 < 500000; +count(*) +5000 +set global innodb_compression_algorithm = 0; +update innodb_page_compressed1 set c1 = c1 + 1; +update innodb_page_compressed2 set c1 = c1 + 1; +update innodb_page_compressed3 set c1 = c1 + 1; +update innodb_page_compressed4 set c1 = c1 + 1; +update innodb_page_compressed5 set c1 = c1 + 1; +update innodb_page_compressed6 set c1 = c1 + 1; +update innodb_page_compressed7 set c1 = c1 + 1; +update innodb_page_compressed8 set c1 = c1 + 1; +update innodb_page_compressed9 set c1 = c1 + 1; +commit; +select count(*) from innodb_compressed; +count(*) +5000 +select count(*) from innodb_page_compressed1; +count(*) +5000 +select count(*) from innodb_page_compressed1 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed2 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed3 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed4 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed5 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed6 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed7 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed8 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed9 where c1 < 500000; +count(*) +5000 +update innodb_page_compressed1 set c1 = c1 + 1; +update innodb_page_compressed2 set c1 = c1 + 1; +update innodb_page_compressed3 set c1 = c1 + 1; +update innodb_page_compressed4 set c1 = c1 + 1; +update innodb_page_compressed5 set c1 = c1 + 1; +update innodb_page_compressed6 set c1 = c1 + 1; +update innodb_page_compressed7 set c1 = c1 + 1; +update innodb_page_compressed8 set c1 = c1 + 1; +update innodb_page_compressed9 set c1 = c1 + 1; +select count(*) from innodb_compressed; +count(*) +5000 +select count(*) from innodb_page_compressed1; +count(*) +5000 +select count(*) from innodb_page_compressed1 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed2 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed3 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed4 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed5 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed6 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed7 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed8 where c1 < 500000; +count(*) +5000 +select count(*) from innodb_page_compressed9 where c1 < 500000; +count(*) +5000 +drop procedure innodb_insert_proc; +drop table innodb_normal; +drop table innodb_compressed; +drop table innodb_page_compressed1; +drop table innodb_page_compressed2; +drop table innodb_page_compressed3; +drop table innodb_page_compressed4; +drop table innodb_page_compressed5; +drop table innodb_page_compressed6; +drop table innodb_page_compressed7; +drop table innodb_page_compressed8; +drop table innodb_page_compressed9; diff --git a/mysql-test/suite/innodb/t/innodb-page_compression_snappy.test b/mysql-test/suite/innodb/t/innodb-page_compression_snappy.test new file mode 100644 index 00000000000..8c4980ff479 --- /dev/null +++ b/mysql-test/suite/innodb/t/innodb-page_compression_snappy.test @@ -0,0 +1,253 @@ +-- source include/have_innodb.inc +-- source include/have_innodb_snappy.inc + +call mtr.add_suppression("InnoDB: Warning: Compression failed for space*"); + +--disable_query_log +let $innodb_compression_algorithm_orig=`select @@innodb_compression_algorithm`; +let $innodb_file_format_orig = `select @@innodb_file_format`; +let $innodb_file_per_table_orig = `select @@innodb_file_per_table`; +--enable_query_log + +set global innodb_file_format = `barracuda`; +set global innodb_file_per_table = on; + +# snappy +set global innodb_compression_algorithm = 6; + +create table innodb_compressed(c1 int, b char(20)) engine=innodb row_format=compressed key_block_size=8; +show warnings; +create table innodb_normal (c1 int, b char(20)) engine=innodb; +show warnings; +create table innodb_page_compressed1 (c1 int, b char(20)) engine=innodb page_compressed=1 page_compression_level=1; +show warnings; +show create table innodb_page_compressed1; +create table innodb_page_compressed2 (c1 int, b char(20)) engine=innodb page_compressed=1 page_compression_level=2; +show warnings; +show create table innodb_page_compressed2; +create table innodb_page_compressed3 (c1 int, b char(20)) engine=innodb page_compressed=1 page_compression_level=3; +show warnings; +show create table innodb_page_compressed3; +create table innodb_page_compressed4 (c1 int, b char(20)) engine=innodb page_compressed=1 page_compression_level=4; +show warnings; +show create table innodb_page_compressed4; +create table innodb_page_compressed5 (c1 int, b char(20)) engine=innodb page_compressed=1 page_compression_level=5; +show warnings; +show create table innodb_page_compressed5; +create table innodb_page_compressed6 (c1 int, b char(20)) engine=innodb page_compressed=1 page_compression_level=6; +show warnings; +show create table innodb_page_compressed6; +create table innodb_page_compressed7 (c1 int, b char(20)) engine=innodb page_compressed=1 page_compression_level=7; +show warnings; +show create table innodb_page_compressed7; +create table innodb_page_compressed8 (c1 int, b char(20)) engine=innodb page_compressed=1 page_compression_level=8; +show warnings; +show create table innodb_page_compressed8; +create table innodb_page_compressed9 (c1 int, b char(20)) engine=innodb page_compressed=1 page_compression_level=9; +show warnings; +show create table innodb_page_compressed9; +delimiter //; +create procedure innodb_insert_proc (repeat_count int) +begin + declare current_num int; + set current_num = 0; + while current_num < repeat_count do + insert into innodb_normal values(current_num,'testing..'); + set current_num = current_num + 1; + end while; +end// +delimiter ;// +commit; + +set autocommit=0; +call innodb_insert_proc(5000); +commit; +set autocommit=1; +select count(*) from innodb_normal; +insert into innodb_compressed select * from innodb_normal; +insert into innodb_page_compressed1 select * from innodb_normal; +insert into innodb_page_compressed2 select * from innodb_normal; +insert into innodb_page_compressed3 select * from innodb_normal; +insert into innodb_page_compressed4 select * from innodb_normal; +insert into innodb_page_compressed5 select * from innodb_normal; +insert into innodb_page_compressed6 select * from innodb_normal; +insert into innodb_page_compressed7 select * from innodb_normal; +insert into innodb_page_compressed8 select * from innodb_normal; +insert into innodb_page_compressed9 select * from innodb_normal; +commit; +select count(*) from innodb_compressed; +select count(*) from innodb_page_compressed1; +select count(*) from innodb_page_compressed1 where c1 < 500000; +select count(*) from innodb_page_compressed2 where c1 < 500000; +select count(*) from innodb_page_compressed3 where c1 < 500000; +select count(*) from innodb_page_compressed4 where c1 < 500000; +select count(*) from innodb_page_compressed5 where c1 < 500000; +select count(*) from innodb_page_compressed6 where c1 < 500000; +select count(*) from innodb_page_compressed7 where c1 < 500000; +select count(*) from innodb_page_compressed8 where c1 < 500000; +select count(*) from innodb_page_compressed9 where c1 < 500000; + +alter table innodb_normal page_compressed=1 page_compression_level=8; +show warnings; +show create table innodb_normal; +alter table innodb_compressed row_format=default page_compressed=1 page_compression_level=8 key_block_size=0; +show warnings; +show create table innodb_compressed; + +update innodb_page_compressed1 set c1 = c1 + 1; +update innodb_page_compressed2 set c1 = c1 + 1; +update innodb_page_compressed3 set c1 = c1 + 1; +update innodb_page_compressed4 set c1 = c1 + 1; +update innodb_page_compressed5 set c1 = c1 + 1; +update innodb_page_compressed6 set c1 = c1 + 1; +update innodb_page_compressed7 set c1 = c1 + 1; +update innodb_page_compressed8 set c1 = c1 + 1; +update innodb_page_compressed9 set c1 = c1 + 1; +select count(*) from innodb_compressed; +select count(*) from innodb_page_compressed1; +select count(*) from innodb_page_compressed1 where c1 < 500000; +select count(*) from innodb_page_compressed2 where c1 < 500000; +select count(*) from innodb_page_compressed3 where c1 < 500000; +select count(*) from innodb_page_compressed4 where c1 < 500000; +select count(*) from innodb_page_compressed5 where c1 < 500000; +select count(*) from innodb_page_compressed6 where c1 < 500000; +select count(*) from innodb_page_compressed7 where c1 < 500000; +select count(*) from innodb_page_compressed8 where c1 < 500000; +select count(*) from innodb_page_compressed9 where c1 < 500000; + +--source include/restart_mysqld.inc + +update innodb_page_compressed1 set c1 = c1 + 1; +update innodb_page_compressed2 set c1 = c1 + 1; +update innodb_page_compressed3 set c1 = c1 + 1; +update innodb_page_compressed4 set c1 = c1 + 1; +update innodb_page_compressed5 set c1 = c1 + 1; +update innodb_page_compressed6 set c1 = c1 + 1; +update innodb_page_compressed7 set c1 = c1 + 1; +update innodb_page_compressed8 set c1 = c1 + 1; +update innodb_page_compressed9 set c1 = c1 + 1; +select count(*) from innodb_compressed; +select count(*) from innodb_page_compressed1; +select count(*) from innodb_page_compressed1 where c1 < 500000; +select count(*) from innodb_page_compressed2 where c1 < 500000; +select count(*) from innodb_page_compressed3 where c1 < 500000; +select count(*) from innodb_page_compressed4 where c1 < 500000; +select count(*) from innodb_page_compressed5 where c1 < 500000; +select count(*) from innodb_page_compressed6 where c1 < 500000; +select count(*) from innodb_page_compressed7 where c1 < 500000; +select count(*) from innodb_page_compressed8 where c1 < 500000; +select count(*) from innodb_page_compressed9 where c1 < 500000; + +# zlib +set global innodb_compression_algorithm = 1; +update innodb_page_compressed1 set c1 = c1 + 1; +update innodb_page_compressed2 set c1 = c1 + 1; +update innodb_page_compressed3 set c1 = c1 + 1; +update innodb_page_compressed4 set c1 = c1 + 1; +update innodb_page_compressed5 set c1 = c1 + 1; +update innodb_page_compressed6 set c1 = c1 + 1; +update innodb_page_compressed7 set c1 = c1 + 1; +update innodb_page_compressed8 set c1 = c1 + 1; +update innodb_page_compressed9 set c1 = c1 + 1; +commit; +select count(*) from innodb_compressed; +select count(*) from innodb_page_compressed1; +select count(*) from innodb_page_compressed1 where c1 < 500000; +select count(*) from innodb_page_compressed2 where c1 < 500000; +select count(*) from innodb_page_compressed3 where c1 < 500000; +select count(*) from innodb_page_compressed4 where c1 < 500000; +select count(*) from innodb_page_compressed5 where c1 < 500000; +select count(*) from innodb_page_compressed6 where c1 < 500000; +select count(*) from innodb_page_compressed7 where c1 < 500000; +select count(*) from innodb_page_compressed8 where c1 < 500000; +select count(*) from innodb_page_compressed9 where c1 < 500000; + +--source include/restart_mysqld.inc + +update innodb_page_compressed1 set c1 = c1 + 1; +update innodb_page_compressed2 set c1 = c1 + 1; +update innodb_page_compressed3 set c1 = c1 + 1; +update innodb_page_compressed4 set c1 = c1 + 1; +update innodb_page_compressed5 set c1 = c1 + 1; +update innodb_page_compressed6 set c1 = c1 + 1; +update innodb_page_compressed7 set c1 = c1 + 1; +update innodb_page_compressed8 set c1 = c1 + 1; +update innodb_page_compressed9 set c1 = c1 + 1; +select count(*) from innodb_compressed; +select count(*) from innodb_page_compressed1; +select count(*) from innodb_page_compressed1 where c1 < 500000; +select count(*) from innodb_page_compressed2 where c1 < 500000; +select count(*) from innodb_page_compressed3 where c1 < 500000; +select count(*) from innodb_page_compressed4 where c1 < 500000; +select count(*) from innodb_page_compressed5 where c1 < 500000; +select count(*) from innodb_page_compressed6 where c1 < 500000; +select count(*) from innodb_page_compressed7 where c1 < 500000; +select count(*) from innodb_page_compressed8 where c1 < 500000; +select count(*) from innodb_page_compressed9 where c1 < 500000; + +# none +set global innodb_compression_algorithm = 0; +update innodb_page_compressed1 set c1 = c1 + 1; +update innodb_page_compressed2 set c1 = c1 + 1; +update innodb_page_compressed3 set c1 = c1 + 1; +update innodb_page_compressed4 set c1 = c1 + 1; +update innodb_page_compressed5 set c1 = c1 + 1; +update innodb_page_compressed6 set c1 = c1 + 1; +update innodb_page_compressed7 set c1 = c1 + 1; +update innodb_page_compressed8 set c1 = c1 + 1; +update innodb_page_compressed9 set c1 = c1 + 1; +commit; +select count(*) from innodb_compressed; +select count(*) from innodb_page_compressed1; +select count(*) from innodb_page_compressed1 where c1 < 500000; +select count(*) from innodb_page_compressed2 where c1 < 500000; +select count(*) from innodb_page_compressed3 where c1 < 500000; +select count(*) from innodb_page_compressed4 where c1 < 500000; +select count(*) from innodb_page_compressed5 where c1 < 500000; +select count(*) from innodb_page_compressed6 where c1 < 500000; +select count(*) from innodb_page_compressed7 where c1 < 500000; +select count(*) from innodb_page_compressed8 where c1 < 500000; +select count(*) from innodb_page_compressed9 where c1 < 500000; + +--source include/restart_mysqld.inc + +update innodb_page_compressed1 set c1 = c1 + 1; +update innodb_page_compressed2 set c1 = c1 + 1; +update innodb_page_compressed3 set c1 = c1 + 1; +update innodb_page_compressed4 set c1 = c1 + 1; +update innodb_page_compressed5 set c1 = c1 + 1; +update innodb_page_compressed6 set c1 = c1 + 1; +update innodb_page_compressed7 set c1 = c1 + 1; +update innodb_page_compressed8 set c1 = c1 + 1; +update innodb_page_compressed9 set c1 = c1 + 1; +select count(*) from innodb_compressed; +select count(*) from innodb_page_compressed1; +select count(*) from innodb_page_compressed1 where c1 < 500000; +select count(*) from innodb_page_compressed2 where c1 < 500000; +select count(*) from innodb_page_compressed3 where c1 < 500000; +select count(*) from innodb_page_compressed4 where c1 < 500000; +select count(*) from innodb_page_compressed5 where c1 < 500000; +select count(*) from innodb_page_compressed6 where c1 < 500000; +select count(*) from innodb_page_compressed7 where c1 < 500000; +select count(*) from innodb_page_compressed8 where c1 < 500000; +select count(*) from innodb_page_compressed9 where c1 < 500000; + +drop procedure innodb_insert_proc; +drop table innodb_normal; +drop table innodb_compressed; +drop table innodb_page_compressed1; +drop table innodb_page_compressed2; +drop table innodb_page_compressed3; +drop table innodb_page_compressed4; +drop table innodb_page_compressed5; +drop table innodb_page_compressed6; +drop table innodb_page_compressed7; +drop table innodb_page_compressed8; +drop table innodb_page_compressed9; + +# reset system +--disable_query_log +EVAL SET GLOBAL innodb_compression_algorithm = $innodb_compression_algorithm_orig; +EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; +EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig; +--enable_query_log diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result index 553ca6f4042..b3e974a75f6 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result @@ -478,7 +478,7 @@ VARIABLE_COMMENT Compression algorithm used on page compression. One of: none, z NUMERIC_MIN_VALUE NULL NUMERIC_MAX_VALUE NULL NUMERIC_BLOCK_SIZE NULL -ENUM_VALUE_LIST none,zlib,lz4,lzo,lzma,bzip2 +ENUM_VALUE_LIST none,zlib,lz4,lzo,lzma,bzip2,snappy READ_ONLY NO COMMAND_LINE_ARGUMENT OPTIONAL VARIABLE_NAME INNODB_COMPRESSION_FAILURE_THRESHOLD_PCT diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index 628aa5f6d45..87c6a1467e6 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -22,11 +22,13 @@ INCLUDE(lz4) INCLUDE(lzo) INCLUDE(lzma) INCLUDE(bzip2) +INCLUDE(snappy) MYSQL_CHECK_LZ4() MYSQL_CHECK_LZO() MYSQL_CHECK_LZMA() MYSQL_CHECK_BZIP2() +MYSQL_CHECK_SNAPPY() # OS tests IF(UNIX) diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 7c46bdbca6b..a5327d9f2aa 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -1246,6 +1246,7 @@ fil_space_create( space->flags = flags; space->magic_n = FIL_SPACE_MAGIC_N; + space->printed_compression_failure = false; rw_lock_create(fil_space_latch_key, &space->latch, SYNC_FSP); diff --git a/storage/innobase/fil/fil0pagecompress.cc b/storage/innobase/fil/fil0pagecompress.cc index 670af29e7b5..80a71d23c9b 100644 --- a/storage/innobase/fil/fil0pagecompress.cc +++ b/storage/innobase/fil/fil0pagecompress.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2014, SkySQL Ab. All Rights Reserved. +Copyright (C) 2013, 2015, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -20,7 +20,8 @@ this program; if not, write to the Free Software Foundation, Inc., @file fil/fil0pagecompress.cc Implementation for page compressed file spaces. -Created 11/12/2013 Jan Lindström jan.lindstrom@skysql.com +Created 11/12/2013 Jan Lindström jan.lindstrom@mariadb.com +Updated 14/02/2015 ***********************************************************************/ #include "fil0fil.h" @@ -74,6 +75,9 @@ static ulint srv_data_read, srv_data_written; #ifdef HAVE_BZIP2 #include "bzlib.h" #endif +#ifdef HAVE_SNAPPY +#include "snappy-c.h" +#endif /* Used for debugging */ //#define UNIV_PAGECOMPRESS_DEBUG 1 @@ -325,9 +329,12 @@ fil_compress_page( if (err == 0) { /* If error we leave the actual page as it was */ - fprintf(stderr, - "InnoDB: Warning: Compression failed for space %lu name %s len %lu rt %d write %lu\n", - space_id, fil_space_name(space), len, err, write_size); + if (space->printed_compression_failure == false) { + fprintf(stderr, + "InnoDB: Warning: Compression failed for space %lu name %s len %lu rt %d write %lu\n", + space_id, fil_space_name(space), len, err, write_size); + space->printed_compression_failure = true; + } srv_stats.pages_page_compression_error.inc(); *out_len = len; @@ -341,9 +348,12 @@ fil_compress_page( buf, len, out_buf+header_len, &write_size, lzo_mem); if (err != LZO_E_OK || write_size > UNIV_PAGE_SIZE-header_len) { - fprintf(stderr, - "InnoDB: Warning: Compression failed for space %lu name %s len %lu err %d write_size %lu\n", - space_id, fil_space_name(space), len, err, write_size); + if (space->printed_compression_failure == false) { + fprintf(stderr, + "InnoDB: Warning: Compression failed for space %lu name %s len %lu err %d write_size %lu\n", + space_id, fil_space_name(space), len, err, write_size); + space->printed_compression_failure = true; + } srv_stats.pages_page_compression_error.inc(); *out_len = len; return (buf); @@ -366,9 +376,12 @@ fil_compress_page( (size_t)write_size); if (err != LZMA_OK || out_pos > UNIV_PAGE_SIZE-header_len) { - fprintf(stderr, - "InnoDB: Warning: Compression failed for space %lu name %s len %lu err %d write_size %lu\n", - space_id, fil_space_name(space), len, err, out_pos); + if (space->printed_compression_failure == false) { + fprintf(stderr, + "InnoDB: Warning: Compression failed for space %lu name %s len %lu err %d write_size %lu\n", + space_id, fil_space_name(space), len, err, out_pos); + space->printed_compression_failure = true; + } srv_stats.pages_page_compression_error.inc(); *out_len = len; @@ -394,9 +407,12 @@ fil_compress_page( 0); if (err != BZ_OK || write_size > UNIV_PAGE_SIZE-header_len) { - fprintf(stderr, - "InnoDB: Warning: Compression failed for space %lu name %s len %lu err %d write_size %lu\n", - space_id, fil_space_name(space), len, err, write_size); + if (space->printed_compression_failure == false) { + fprintf(stderr, + "InnoDB: Warning: Compression failed for space %lu name %s len %lu err %d write_size %lu\n", + space_id, fil_space_name(space), len, err, write_size); + space->printed_compression_failure = true; + } srv_stats.pages_page_compression_error.inc(); *out_len = len; return (buf); @@ -405,15 +421,40 @@ fil_compress_page( } #endif /* HAVE_BZIP2 */ +#ifdef HAVE_SNAPPY + case PAGE_SNAPPY_ALGORITHM: + { + snappy_status cstatus; + + cstatus = snappy_compress((const char *)buf, len, (char *)(out_buf+header_len), &write_size); + + if (cstatus != SNAPPY_OK || write_size > UNIV_PAGE_SIZE-header_len) { + if (space->printed_compression_failure == false) { + fprintf(stderr, + "InnoDB: Warning: Compression failed for space %lu name %s len %lu err %d write_size %lu\n", + space_id, fil_space_name(space), len, (int)cstatus, write_size); + space->printed_compression_failure = true; + } + srv_stats.pages_page_compression_error.inc(); + *out_len = len; + return (buf); + } + break; + } +#endif /* HAVE_SNAPPY */ + case PAGE_ZLIB_ALGORITHM: err = compress2(out_buf+header_len, (ulong*)&write_size, buf, len, level); if (err != Z_OK) { /* If error we leave the actual page as it was */ - fprintf(stderr, - "InnoDB: Warning: Compression failed for space %lu name %s len %lu rt %d write %lu\n", - space_id, fil_space_name(space), len, err, write_size); + if (space->printed_compression_failure == false) { + fprintf(stderr, + "InnoDB: Warning: Compression failed for space %lu name %s len %lu rt %d write %lu\n", + space_id, fil_space_name(space), len, err, write_size); + space->printed_compression_failure = true; + } srv_stats.pages_page_compression_error.inc(); *out_len = len; @@ -454,8 +495,8 @@ fil_compress_page( byte *comp_page; byte *uncomp_page; - comp_page = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE*2)); - uncomp_page = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE*2)); + comp_page = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE*3)); + uncomp_page = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE*3)); memcpy(comp_page, out_buf, UNIV_PAGE_SIZE); fil_decompress_page(uncomp_page, comp_page, len, NULL); @@ -540,7 +581,7 @@ fil_decompress_page( fprintf(stderr, "InnoDB: Note: FIL: Compression buffer not given, allocating...\n"); #endif /* UNIV_PAGECOMPRESS_DEBUG */ - in_buf = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE*2)); + in_buf = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE*3)); } else { in_buf = page_buf; } @@ -712,7 +753,31 @@ fil_decompress_page( break; } #endif /* HAVE_BZIP2 */ +#ifdef HAVE_SNAPPY + case PAGE_SNAPPY_ALGORITHM: + { + snappy_status cstatus; + ulint olen = 0; + + cstatus = snappy_uncompress( + (const char *)(buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE), + actual_size, + (char *)in_buf, + &olen); + + if (cstatus != SNAPPY_OK || (olen == 0 || olen > UNIV_PAGE_SIZE)) { + fprintf(stderr, + "InnoDB: Corruption: Page is marked as compressed\n" + "InnoDB: but decompression read only %lu bytes.\n" + "InnoDB: size %lu len %lu err %d\n", + olen, actual_size, len, (int)cstatus); + fflush(stderr); + ut_error; + } + break; + } +#endif /* HAVE_SNAPPY */ default: fprintf(stderr, "InnoDB: Corruption: Page is marked as compressed\n" diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index dc395b50d3a..a8cece74666 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -631,6 +631,7 @@ static ibool innodb_have_lzo=IF_LZO(1, 0); static ibool innodb_have_lz4=IF_LZ4(1, 0); static ibool innodb_have_lzma=IF_LZMA(1, 0); static ibool innodb_have_bzip2=IF_BZIP2(1, 0); +static ibool innodb_have_snappy=IF_SNAPPY(1, 0); static const char innobase_hton_name[]= "InnoDB"; @@ -828,6 +829,8 @@ static SHOW_VAR innodb_status_variables[]= { (char*) &innodb_have_lzma, SHOW_BOOL}, {"have_bzip2", (char*) &innodb_have_bzip2, SHOW_BOOL}, + {"have_snappy", + (char*) &innodb_have_snappy, SHOW_BOOL}, /* Defragmentation */ {"defragment_compression_failures", @@ -3229,6 +3232,15 @@ innobase_init( } #endif +#ifndef HAVE_SNAPPY + if (innodb_compression_algorithm == PAGE_SNAPPY_ALGORITHM) { + sql_print_error("InnoDB: innodb_compression_algorithm = %lu unsupported.\n" + "InnoDB: libsnappy is not installed. \n", + innodb_compression_algorithm); + goto error; + } +#endif + os_innodb_umask = (ulint) my_umask; /* First calculate the default path for innodb_data_home_dir etc., @@ -19061,7 +19073,7 @@ static MYSQL_SYSVAR_BOOL(use_trim, srv_use_trim, "Use trim. Default FALSE.", NULL, NULL, FALSE); -static const char *page_compression_algorithms[]= { "none", "zlib", "lz4", "lzo", "lzma", "bzip2", 0 }; +static const char *page_compression_algorithms[]= { "none", "zlib", "lz4", "lzo", "lzma", "bzip2", "snappy", 0 }; static TYPELIB page_compression_algorithms_typelib= { array_elements(page_compression_algorithms) - 1, 0, @@ -20023,5 +20035,15 @@ innodb_compression_algorithm_validate( } #endif +#ifndef HAVE_SNAPPY + if (compression_algorithm == PAGE_SNAPPY_ALGORITHM) { + push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, + HA_ERR_UNSUPPORTED, + "InnoDB: innodb_compression_algorithm = %lu unsupported.\n" + "InnoDB: libsnappy is not installed. \n", + compression_algorithm); + DBUG_RETURN(1); + } +#endif DBUG_RETURN(0); } diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index bd7f39939a3..e4f16b24392 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -343,6 +343,9 @@ struct fil_space_t { bool is_in_unflushed_spaces; /*!< true if this space is currently in unflushed_spaces */ + bool printed_compression_failure; + /*!< true if we have already printed + compression failure */ UT_LIST_NODE_T(fil_space_t) space_list; /*!< list of all spaces */ fil_space_crypt_t* crypt_data; diff --git a/storage/innobase/include/fsp0pagecompress.h b/storage/innobase/include/fsp0pagecompress.h index 15212227829..5f943ee2b83 100644 --- a/storage/innobase/include/fsp0pagecompress.h +++ b/storage/innobase/include/fsp0pagecompress.h @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2014 SkySQL Ab. All Rights Reserved. +Copyright (C) 2013, 2015, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -35,7 +35,8 @@ Created 11/12/2013 Jan Lindström jan.lindstrom@skysql.com #define PAGE_LZO_ALGORITHM 3 #define PAGE_LZMA_ALGORITHM 4 #define PAGE_BZIP2_ALGORITHM 5 -#define PAGE_ALGORITHM_LAST PAGE_BZIP2_ALGORITHM +#define PAGE_SNAPPY_ALGORITHM 6 +#define PAGE_ALGORITHM_LAST PAGE_SNAPPY_ALGORITHM /**********************************************************************//** Reads the page compression level from the first page of a tablespace. diff --git a/storage/innobase/include/univ.i b/storage/innobase/include/univ.i index b2f486d059e..89c62a14958 100644 --- a/storage/innobase/include/univ.i +++ b/storage/innobase/include/univ.i @@ -2,7 +2,7 @@ Copyright (c) 1994, 2013, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, Google Inc. -Copyright (c) 2013, 2014 SkySQL Ab. +Copyright (c) 2013, 2015, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -356,6 +356,12 @@ typedef enum innodb_file_formats_enum innodb_file_formats_t; #define IF_BZIP2(A,B) B #endif +#ifdef HAVE_SNAPPY +#define IF_SNAPPY(A,B) A +#else +#define IF_SNAPPY(A,B) B +#endif + /** The universal page size of the database */ #define UNIV_PAGE_SIZE ((ulint) srv_page_size) diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 7495e0d570c..e552aa7b697 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -2,7 +2,7 @@ Copyright (c) 1995, 2013, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2014, MariaDB Corporation. +Copyright (c) 2013, 2015, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Percona Inc.. Those modifications are @@ -97,6 +97,10 @@ Created 10/21/1995 Heikki Tuuri #include "lzo/lzo1x.h" #endif +#ifdef HAVE_SNAPPY +#include "snappy-c.h" +#endif + /** Insert buffer segment id */ static const ulint IO_IBUF_SEGMENT = 0; @@ -6667,13 +6671,18 @@ os_slot_alloc_page_buf( if (slot->page_buf == NULL) { byte* cbuf2; byte* cbuf; + ulint asize = UNIV_PAGE_SIZE; +#ifdef HAVE_SNAPPY + asize += snappy_max_compressed_length(asize) - UNIV_PAGE_SIZE; +#endif + /* We allocate extra to avoid memory overwrite on compression */ - cbuf2 = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE*2)); + cbuf2 = static_cast<byte *>(ut_malloc(asize*2)); cbuf = static_cast<byte *>(ut_align(cbuf2, UNIV_PAGE_SIZE)); slot->page_compression_page = static_cast<byte *>(cbuf2); slot->page_buf = static_cast<byte *>(cbuf); ut_a(slot->page_buf != NULL); - memset(slot->page_compression_page, 0, UNIV_PAGE_SIZE*2); + memset(slot->page_compression_page, 0, asize*2); } } diff --git a/storage/xtradb/CMakeLists.txt b/storage/xtradb/CMakeLists.txt index 4ba7a73e527..da2f988e9b3 100644 --- a/storage/xtradb/CMakeLists.txt +++ b/storage/xtradb/CMakeLists.txt @@ -22,11 +22,13 @@ INCLUDE(lz4) INCLUDE(lzo) INCLUDE(lzma) INCLUDE(bzip2) +INCLUDE(snappy) MYSQL_CHECK_LZ4() MYSQL_CHECK_LZO() MYSQL_CHECK_LZMA() MYSQL_CHECK_BZIP2() +MYSQL_CHECK_SNAPPY() # OS tests IF(UNIX) diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 8a01dc7d7e6..4a8586b97ee 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -1281,6 +1281,7 @@ fil_space_create( space->flags = flags; space->magic_n = FIL_SPACE_MAGIC_N; + space->printed_compression_failure = false; rw_lock_create(fil_space_latch_key, &space->latch, SYNC_FSP); diff --git a/storage/xtradb/fil/fil0pagecompress.cc b/storage/xtradb/fil/fil0pagecompress.cc index 987493c8d26..a48fcf0f71a 100644 --- a/storage/xtradb/fil/fil0pagecompress.cc +++ b/storage/xtradb/fil/fil0pagecompress.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2014, SkySQL Ab. All Rights Reserved. +Copyright (C) 2013, 2015, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -20,7 +20,8 @@ this program; if not, write to the Free Software Foundation, Inc., @file fil/fil0pagecompress.cc Implementation for page compressed file spaces. -Created 11/12/2013 Jan Lindström jan.lindstrom@skysql.com +Created 11/12/2013 Jan Lindström jan.lindstrom@mariadb.com +Updated 14/02/2015 ***********************************************************************/ #include "fil0fil.h" @@ -74,6 +75,9 @@ static ulint srv_data_read, srv_data_written; #ifdef HAVE_BZIP2 #include "bzlib.h" #endif +#ifdef HAVE_SNAPPY +#include "snappy-c.h" +#endif /* Used for debugging */ //#define UNIV_PAGECOMPRESS_DEBUG 1 @@ -325,9 +329,12 @@ fil_compress_page( if (err == 0) { /* If error we leave the actual page as it was */ - fprintf(stderr, - "InnoDB: Warning: Compression failed for space %lu name %s len %lu rt %d write %lu\n", - space_id, fil_space_name(space), len, err, write_size); + if (space->printed_compression_failure == false) { + fprintf(stderr, + "InnoDB: Warning: Compression failed for space %lu name %s len %lu rt %d write %lu\n", + space_id, fil_space_name(space), len, err, write_size); + space->printed_compression_failure = true; + } srv_stats.pages_page_compression_error.inc(); *out_len = len; @@ -341,9 +348,12 @@ fil_compress_page( buf, len, out_buf+header_len, &write_size, lzo_mem); if (err != LZO_E_OK || write_size > UNIV_PAGE_SIZE-header_len) { - fprintf(stderr, - "InnoDB: Warning: Compression failed for space %lu name %s len %lu err %d write_size %lu\n", - space_id, fil_space_name(space), len, err, write_size); + if (space->printed_compression_failure == false) { + fprintf(stderr, + "InnoDB: Warning: Compression failed for space %lu name %s len %lu err %d write_size %lu\n", + space_id, fil_space_name(space), len, err, write_size); + space->printed_compression_failure = true; + } srv_stats.pages_page_compression_error.inc(); *out_len = len; return (buf); @@ -366,9 +376,12 @@ fil_compress_page( (size_t)write_size); if (err != LZMA_OK || out_pos > UNIV_PAGE_SIZE-header_len) { - fprintf(stderr, - "InnoDB: Warning: Compression failed for space %lu name %s len %lu err %d write_size %lu\n", - space_id, fil_space_name(space), len, err, out_pos); + if (space->printed_compression_failure == false) { + fprintf(stderr, + "InnoDB: Warning: Compression failed for space %lu name %s len %lu err %d write_size %lu\n", + space_id, fil_space_name(space), len, err, out_pos); + space->printed_compression_failure = true; + } srv_stats.pages_page_compression_error.inc(); *out_len = len; return (buf); @@ -393,9 +406,12 @@ fil_compress_page( 0); if (err != BZ_OK || write_size > UNIV_PAGE_SIZE-header_len) { - fprintf(stderr, - "InnoDB: Warning: Compression failed for space %lu name %s len %lu err %d write_size %lu\n", - space_id, fil_space_name(space), len, err, write_size); + if (space->printed_compression_failure == false) { + fprintf(stderr, + "InnoDB: Warning: Compression failed for space %lu name %s len %lu err %d write_size %lu\n", + space_id, fil_space_name(space), len, err, write_size); + space->printed_compression_failure = true; + } srv_stats.pages_page_compression_error.inc(); *out_len = len; return (buf); @@ -404,15 +420,40 @@ fil_compress_page( } #endif /* HAVE_BZIP2 */ +#ifdef HAVE_SNAPPY + case PAGE_SNAPPY_ALGORITHM: + { + snappy_status cstatus; + + cstatus = snappy_compress((const char *)buf, len, (char *)(out_buf+header_len), &write_size); + + if (cstatus != SNAPPY_OK || write_size > UNIV_PAGE_SIZE-header_len) { + if (space->printed_compression_failure == false) { + fprintf(stderr, + "InnoDB: Warning: Compression failed for space %lu name %s len %lu err %d write_size %lu\n", + space_id, fil_space_name(space), len, (int)cstatus, write_size); + space->printed_compression_failure = true; + } + srv_stats.pages_page_compression_error.inc(); + *out_len = len; + return (buf); + } + break; + } +#endif /* HAVE_SNAPPY */ + case PAGE_ZLIB_ALGORITHM: err = compress2(out_buf+header_len, (ulong*)&write_size, buf, len, level); if (err != Z_OK) { /* If error we leave the actual page as it was */ - fprintf(stderr, - "InnoDB: Warning: Compression failed for space %lu name %s len %lu rt %d write %lu\n", - space_id, fil_space_name(space), len, err, write_size); + if (space->printed_compression_failure == false) { + fprintf(stderr, + "InnoDB: Warning: Compression failed for space %lu name %s len %lu rt %d write %lu\n", + space_id, fil_space_name(space), len, err, write_size); + space->printed_compression_failure = true; + } srv_stats.pages_page_compression_error.inc(); *out_len = len; @@ -452,8 +493,8 @@ fil_compress_page( byte *comp_page; byte *uncomp_page; - comp_page = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE*2)); - uncomp_page = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE*2)); + comp_page = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE*3)); + uncomp_page = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE*3)); memcpy(comp_page, out_buf, UNIV_PAGE_SIZE); fil_decompress_page(uncomp_page, comp_page, len, NULL); @@ -537,7 +578,7 @@ fil_decompress_page( fprintf(stderr, "InnoDB: Note: FIL: Compression buffer not given, allocating...\n"); #endif /* UNIV_PAGECOMPRESS_DEBUG */ - in_buf = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE*2)); + in_buf = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE*3)); } else { in_buf = page_buf; } @@ -708,7 +749,31 @@ fil_decompress_page( break; } #endif /* HAVE_BZIP2 */ +#ifdef HAVE_SNAPPY + case PAGE_SNAPPY_ALGORITHM: + { + snappy_status cstatus; + ulint olen = 0; + + cstatus = snappy_uncompress( + (const char *)(buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE), + actual_size, + (char *)in_buf, + &olen); + + if (cstatus != SNAPPY_OK || (olen == 0 || olen > UNIV_PAGE_SIZE)) { + fprintf(stderr, + "InnoDB: Corruption: Page is marked as compressed\n" + "InnoDB: but decompression read only %lu bytes.\n" + "InnoDB: size %lu len %lu err %d\n", + olen, actual_size, len, (int)cstatus); + fflush(stderr); + ut_error; + } + break; + } +#endif /* HAVE_SNAPPY */ default: fprintf(stderr, "InnoDB: Corruption: Page is marked as compressed\n" diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 9a760eb62d5..ee62dbc2d6a 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -777,6 +777,7 @@ static ibool innodb_have_lzo=IF_LZO(1, 0); static ibool innodb_have_lz4=IF_LZ4(1, 0); static ibool innodb_have_lzma=IF_LZMA(1, 0); static ibool innodb_have_bzip2=IF_BZIP2(1, 0); +static ibool innodb_have_snappy=IF_SNAPPY(1, 0); static SHOW_VAR innodb_status_variables[]= { {"available_undo_logs", @@ -1027,6 +1028,8 @@ static SHOW_VAR innodb_status_variables[]= { (char*) &innodb_have_lzma, SHOW_BOOL}, {"have_bzip2", (char*) &innodb_have_bzip2, SHOW_BOOL}, + {"have_snappy", + (char*) &innodb_have_snappy, SHOW_BOOL}, /* Defragment */ {"defragment_compression_failures", @@ -3620,6 +3623,15 @@ innobase_init( } #endif +#ifndef HAVE_SNAPPY + if (innodb_compression_algorithm == PAGE_SNAPPY_ALGORITHM) { + sql_print_error("InnoDB: innodb_compression_algorithm = %lu unsupported.\n" + "InnoDB: libsnappy is not installed. \n", + innodb_compression_algorithm); + goto error; + } +#endif + os_innodb_umask = (ulint) my_umask; /* First calculate the default path for innodb_data_home_dir etc., @@ -20252,7 +20264,7 @@ static MYSQL_SYSVAR_BOOL(use_trim, srv_use_trim, "Use trim. Default FALSE.", NULL, NULL, FALSE); -static const char *page_compression_algorithms[]= { "none", "zlib", "lz4", "lzo", "lzma", "bzip2", 0 }; +static const char *page_compression_algorithms[]= { "none", "zlib", "lz4", "lzo", "lzma", "bzip2", "snappy", 0 }; static TYPELIB page_compression_algorithms_typelib= { array_elements(page_compression_algorithms) - 1, 0, @@ -21260,5 +21272,15 @@ innodb_compression_algorithm_validate( } #endif +#ifndef HAVE_SNAPPY + if (compression_algorithm == PAGE_SNAPPY_ALGORITHM) { + push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, + HA_ERR_UNSUPPORTED, + "InnoDB: innodb_compression_algorithm = %lu unsupported.\n" + "InnoDB: libsnappy is not installed. \n", + compression_algorithm); + DBUG_RETURN(1); + } +#endif DBUG_RETURN(0); } diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index 1847a314d06..2e92c2b09b2 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -337,6 +337,9 @@ struct fil_space_t { /*!< true if this space is currently in unflushed_spaces */ ibool is_corrupt; + bool printed_compression_failure; + /*!< true if we have already printed + compression failure */ UT_LIST_NODE_T(fil_space_t) space_list; /*!< list of all spaces */ fil_space_crypt_t* crypt_data; diff --git a/storage/xtradb/include/fsp0pagecompress.h b/storage/xtradb/include/fsp0pagecompress.h index 15212227829..5f943ee2b83 100644 --- a/storage/xtradb/include/fsp0pagecompress.h +++ b/storage/xtradb/include/fsp0pagecompress.h @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2014 SkySQL Ab. All Rights Reserved. +Copyright (C) 2013, 2015, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -35,7 +35,8 @@ Created 11/12/2013 Jan Lindström jan.lindstrom@skysql.com #define PAGE_LZO_ALGORITHM 3 #define PAGE_LZMA_ALGORITHM 4 #define PAGE_BZIP2_ALGORITHM 5 -#define PAGE_ALGORITHM_LAST PAGE_BZIP2_ALGORITHM +#define PAGE_SNAPPY_ALGORITHM 6 +#define PAGE_ALGORITHM_LAST PAGE_SNAPPY_ALGORITHM /**********************************************************************//** Reads the page compression level from the first page of a tablespace. diff --git a/storage/xtradb/include/univ.i b/storage/xtradb/include/univ.i index 0cc56b121ed..37d63489363 100644 --- a/storage/xtradb/include/univ.i +++ b/storage/xtradb/include/univ.i @@ -2,7 +2,7 @@ Copyright (c) 1994, 2013, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, Google Inc. -Copyright (c) 2013, 2014 SkySQL Ab. +Copyright (c) 2013, 2015, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -375,6 +375,12 @@ typedef enum innodb_file_formats_enum innodb_file_formats_t; #define IF_BZIP2(A,B) B #endif +#ifdef HAVE_SNAPPY +#define IF_SNAPPY(A,B) A +#else +#define IF_SNAPPY(A,B) B +#endif + /** The universal page size of the database */ #define UNIV_PAGE_SIZE ((ulint) srv_page_size) diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index a737ebb1fa0..156816166f8 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -2,7 +2,7 @@ Copyright (c) 1995, 2013, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2014, MariaDB Corporation. +Copyright (c) 2013, 2015, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Percona Inc.. Those modifications are @@ -103,6 +103,10 @@ Created 10/21/1995 Heikki Tuuri #include "lzo/lzo1x.h" #endif +#ifdef HAVE_SNAPPY +#include "snappy-c.h" +#endif + /** Insert buffer segment id */ static const ulint IO_IBUF_SEGMENT = 0; @@ -6752,14 +6756,18 @@ os_slot_alloc_page_buf( if (slot->page_buf == NULL) { byte* cbuf2; byte* cbuf; + ulint asize = UNIV_PAGE_SIZE; +#ifdef HAVE_SNAPPY + asize += snappy_max_compressed_length(asize) - UNIV_PAGE_SIZE; +#endif /* We allocate extra to avoid memory overwrite on compression */ - cbuf2 = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE*2)); + cbuf2 = static_cast<byte *>(ut_malloc(asize*2)); cbuf = static_cast<byte *>(ut_align(cbuf2, UNIV_PAGE_SIZE)); slot->page_compression_page = static_cast<byte *>(cbuf2); slot->page_buf = static_cast<byte *>(cbuf); ut_a(slot->page_buf != NULL); - memset(slot->page_compression_page, 0, UNIV_PAGE_SIZE*2); + memset(slot->page_compression_page, 0, asize*2); } } |