From d9340d6c8e1b227044fc90bc40c5da1d1f6b0dcc Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 27 May 2015 00:18:20 +0200 Subject: MDEV-8126 encryption for temp files IO_CACHE tempfiles encryption --- include/my_sys.h | 2 +- libmysqld/CMakeLists.txt | 2 +- mysql-test/include/have_sequence.inc | 4 + mysql-test/include/have_sequence.opt | 1 + mysql-test/r/mysqld--help.result | 8 +- mysql-test/suite/encryption/r/tempfiles.result | 46 ++++ mysql-test/suite/encryption/t/tempfiles.test | 74 ++++++ mysql-test/suite/sys_vars/r/all_vars.result | 1 + .../sys_vars/r/sysvars_server_embedded.result | 16 +- .../sys_vars/r/sysvars_server_notembedded.result | 16 +- mysys/mf_iocache.c | 59 ++++- mysys/mf_iocache2.c | 7 + mysys/mysys_priv.h | 20 ++ sql/CMakeLists.txt | 5 +- sql/encryption.cc | 5 + sql/mf_iocache_encr.cc | 254 +++++++++++++++++++++ sql/mysqld.cc | 2 +- sql/mysqld.h | 2 +- sql/sys_vars.cc | 8 +- storage/maria/ma_cache.c | 1 + storage/myisam/mi_cache.c | 1 + 21 files changed, 518 insertions(+), 16 deletions(-) create mode 100644 mysql-test/include/have_sequence.inc create mode 100644 mysql-test/include/have_sequence.opt create mode 100644 mysql-test/suite/encryption/r/tempfiles.result create mode 100644 mysql-test/suite/encryption/t/tempfiles.test create mode 100644 sql/mf_iocache_encr.cc diff --git a/include/my_sys.h b/include/my_sys.h index dfd45b18f26..0aaf4e90ea8 100644 --- a/include/my_sys.h +++ b/include/my_sys.h @@ -67,7 +67,7 @@ typedef struct my_aio_result { #define MY_WME 16 /* Write message on error */ #define MY_WAIT_IF_FULL 32 /* Wait and try again if disk full error */ #define MY_IGNORE_BADFD 32 /* my_sync: ignore 'bad descriptor' errors */ -#define MY_UNUSED 64 /* Unused (was support for RAID) */ +#define MY_ENCRYPT 64 /* Encrypt IO_CACHE temporary files */ #define MY_FULL_IO 512 /* For my_read - loop intil I/O is complete */ #define MY_DONT_CHECK_FILESIZE 128 /* Option to init_io_cache() */ #define MY_LINK_WARNING 32 /* my_redel() gives warning if links */ diff --git a/libmysqld/CMakeLists.txt b/libmysqld/CMakeLists.txt index 57df0da36ec..5e0927395c2 100644 --- a/libmysqld/CMakeLists.txt +++ b/libmysqld/CMakeLists.txt @@ -104,7 +104,7 @@ SET(SQL_EMBEDDED_SOURCES emb_qcache.cc libmysqld.c lib_sql.cc ../sql/sql_explain.cc ../sql/sql_explain.h ../sql/sql_analyze_stmt.cc ../sql/sql_analyze_stmt.h ../sql/compat56.cc - ../sql/table_cache.cc + ../sql/table_cache.cc ../sql/mf_iocache_encr.cc ../sql/item_inetfunc.cc ../sql/wsrep_dummy.cc ../sql/encryption.cc ${GEN_SOURCES} diff --git a/mysql-test/include/have_sequence.inc b/mysql-test/include/have_sequence.inc new file mode 100644 index 00000000000..b509d605177 --- /dev/null +++ b/mysql-test/include/have_sequence.inc @@ -0,0 +1,4 @@ +if (`SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'sequence' AND support IN ('YES', 'DEFAULT', 'ENABLED')`) +{ + --skip Test requires Sequence engine +} diff --git a/mysql-test/include/have_sequence.opt b/mysql-test/include/have_sequence.opt new file mode 100644 index 00000000000..d08d8f112cd --- /dev/null +++ b/mysql-test/include/have_sequence.opt @@ -0,0 +1 @@ +--loose-sequence diff --git a/mysql-test/r/mysqld--help.result b/mysql-test/r/mysqld--help.result index 830821a1a96..8abda849494 100644 --- a/mysql-test/r/mysqld--help.result +++ b/mysql-test/r/mysqld--help.result @@ -163,8 +163,11 @@ The following options may be given as the first argument: Precision of the result of '/' operator will be increased on that value --encrypt-tmp-disk-tables - Encrypt tmp disk tables (created as part of query - execution) + Encrypt temporary on-disk tables (created as part of + query execution) + --encrypt-tmp-files Encrypt temporary files (created for filesort, binary log + cache, etc) + (Defaults to on; use --skip-encrypt-tmp-files to disable.) --enforce-storage-engine=name Force the use of a storage engine for new tables --event-scheduler[=name] @@ -1149,6 +1152,7 @@ delayed-insert-timeout 300 delayed-queue-size 1000 div-precision-increment 4 encrypt-tmp-disk-tables FALSE +encrypt-tmp-files TRUE enforce-storage-engine (No default value) event-scheduler OFF expensive-subquery-limit 100 diff --git a/mysql-test/suite/encryption/r/tempfiles.result b/mysql-test/suite/encryption/r/tempfiles.result new file mode 100644 index 00000000000..a0b7596dd5a --- /dev/null +++ b/mysql-test/suite/encryption/r/tempfiles.result @@ -0,0 +1,46 @@ +CREATE TABLE t1(a INT); +INSERT INTO t1 VALUES(1),(2); +DELETE FROM t1 WHERE a=1; +OPTIMIZE TABLE t1; +Table Op Msg_type Msg_text +test.t1 optimize status OK +CHECK TABLE t1; +Table Op Msg_type Msg_text +test.t1 check status OK +DROP TABLE t1; +create table t1 (v varchar(10), c char(10), t text, key(v), key(c), key(t(10))); +insert into t1 (v) select concat(char(ascii('a')+s2.seq),repeat(' ',s1.seq)) +from seq_0_to_9 as s1, seq_0_to_26 as s2; +update t1 set c=v, t=v; +select sql_big_result t,count(t) from t1 group by t limit 10; +t count(t) +a 10 +b 10 +c 10 +d 10 +e 10 +f 10 +g 10 +h 10 +i 10 +j 10 +drop table t1; +set global binlog_cache_size=8192; +create table t1 (a text) engine=innodb; +start transaction; +insert t1 select repeat(seq, 1000) from seq_1_to_15; +commit; +start transaction; +insert t1 select repeat(seq, 1000) from seq_1_to_8; +commit; +drop table t1; +create table t1 (a text) engine=innodb; +start transaction; +insert t1 select repeat(seq, 1000) from seq_1_to_15; +savepoint foo; +insert t1 select repeat(seq, 1000) from seq_16_to_30; +rollback to savepoint foo; +insert t1 select repeat(seq, 1000) from seq_31_to_40; +commit; +drop table t1; +set global binlog_cache_size=default; diff --git a/mysql-test/suite/encryption/t/tempfiles.test b/mysql-test/suite/encryption/t/tempfiles.test new file mode 100644 index 00000000000..6395a15d8a5 --- /dev/null +++ b/mysql-test/suite/encryption/t/tempfiles.test @@ -0,0 +1,74 @@ +# +# Various test cases for IO_CACHE tempfiles (file==-1) encryption +# +source include/have_example_key_management_plugin.inc; +source include/have_sequence.inc; + +# Row binlog format to fill binlog cache faster +source include/have_binlog_format_row.inc; + +# Nothing XtraDB specific in this test, it just needs *some* transactional +# engine. But there's no need to run it twice for InnoDB and XtraDB. +source include/have_xtradb.inc; + +# +# MyISAM messing around with IO_CACHE::file +# +CREATE TABLE t1(a INT); +INSERT INTO t1 VALUES(1),(2); +DELETE FROM t1 WHERE a=1; +OPTIMIZE TABLE t1; +CHECK TABLE t1; +DROP TABLE t1; + +# +# filesort, my_b_pread, seeks in READ_CACHE +# +create table t1 (v varchar(10), c char(10), t text, key(v), key(c), key(t(10))); +insert into t1 (v) select concat(char(ascii('a')+s2.seq),repeat(' ',s1.seq)) + from seq_0_to_9 as s1, seq_0_to_26 as s2; +update t1 set c=v, t=v; +select sql_big_result t,count(t) from t1 group by t limit 10; +drop table t1; + +set global binlog_cache_size=8192; + +connect con1, localhost, root; + +# +# Test the last half-filled block: +# first write 3 blocks, then reinit the file and write one full and one +# partial block. reading the second time must stop in the middle of the +# second block, and NOT read till EOF. +# +create table t1 (a text) engine=innodb; +start transaction; +insert t1 select repeat(seq, 1000) from seq_1_to_15; +commit; +start transaction; +insert t1 select repeat(seq, 1000) from seq_1_to_8; +commit; +drop table t1; + +disconnect con1; +connect con2, localhost, root; + +# +# Test reinit_io_cache(WRITE_CACHE) with non-zero seek_offset: +# Start a transaction, write until the cache goes to disk, +# create a savepoint, write more blocks to disk, rollback to savepoint. +# +create table t1 (a text) engine=innodb; +start transaction; +insert t1 select repeat(seq, 1000) from seq_1_to_15; +savepoint foo; +insert t1 select repeat(seq, 1000) from seq_16_to_30; +rollback to savepoint foo; +insert t1 select repeat(seq, 1000) from seq_31_to_40; +commit; +drop table t1; + +disconnect con2; +connection default; + +set global binlog_cache_size=default; diff --git a/mysql-test/suite/sys_vars/r/all_vars.result b/mysql-test/suite/sys_vars/r/all_vars.result index de462475acc..ca821f870fb 100644 --- a/mysql-test/suite/sys_vars/r/all_vars.result +++ b/mysql-test/suite/sys_vars/r/all_vars.result @@ -10,6 +10,7 @@ there should be *no* long test name listed below: select distinct variable_name as `there should be *no* variables listed below:` from t2 left join t1 on variable_name=test_name where test_name is null; there should be *no* variables listed below: +encrypt_tmp_files innodb_default_encryption_key_id max_digest_length strict_password_validation diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result b/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result index 4c63e258574..d3bb7c2bf80 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result +++ b/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result @@ -688,13 +688,27 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE OFF VARIABLE_SCOPE GLOBAL VARIABLE_TYPE BOOLEAN -VARIABLE_COMMENT Encrypt tmp disk tables (created as part of query execution) +VARIABLE_COMMENT Encrypt temporary on-disk tables (created as part of query execution) NUMERIC_MIN_VALUE NULL NUMERIC_MAX_VALUE NULL NUMERIC_BLOCK_SIZE NULL ENUM_VALUE_LIST OFF,ON READ_ONLY NO COMMAND_LINE_ARGUMENT OPTIONAL +VARIABLE_NAME ENCRYPT_TMP_FILES +SESSION_VALUE NULL +GLOBAL_VALUE ON +GLOBAL_VALUE_ORIGIN COMPILE-TIME +DEFAULT_VALUE ON +VARIABLE_SCOPE GLOBAL +VARIABLE_TYPE BOOLEAN +VARIABLE_COMMENT Encrypt temporary files (created for filesort, binary log cache, etc) +NUMERIC_MIN_VALUE NULL +NUMERIC_MAX_VALUE NULL +NUMERIC_BLOCK_SIZE NULL +ENUM_VALUE_LIST OFF,ON +READ_ONLY YES +COMMAND_LINE_ARGUMENT OPTIONAL VARIABLE_NAME ENFORCE_STORAGE_ENGINE SESSION_VALUE GLOBAL_VALUE NULL diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result index 09e220417b1..10b3b12dabb 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result +++ b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result @@ -702,13 +702,27 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE OFF VARIABLE_SCOPE GLOBAL VARIABLE_TYPE BOOLEAN -VARIABLE_COMMENT Encrypt tmp disk tables (created as part of query execution) +VARIABLE_COMMENT Encrypt temporary on-disk tables (created as part of query execution) NUMERIC_MIN_VALUE NULL NUMERIC_MAX_VALUE NULL NUMERIC_BLOCK_SIZE NULL ENUM_VALUE_LIST OFF,ON READ_ONLY NO COMMAND_LINE_ARGUMENT OPTIONAL +VARIABLE_NAME ENCRYPT_TMP_FILES +SESSION_VALUE NULL +GLOBAL_VALUE ON +GLOBAL_VALUE_ORIGIN COMPILE-TIME +DEFAULT_VALUE ON +VARIABLE_SCOPE GLOBAL +VARIABLE_TYPE BOOLEAN +VARIABLE_COMMENT Encrypt temporary files (created for filesort, binary log cache, etc) +NUMERIC_MIN_VALUE NULL +NUMERIC_MAX_VALUE NULL +NUMERIC_BLOCK_SIZE NULL +ENUM_VALUE_LIST OFF,ON +READ_ONLY YES +COMMAND_LINE_ARGUMENT OPTIONAL VARIABLE_NAME ENFORCE_STORAGE_ENGINE SESSION_VALUE GLOBAL_VALUE NULL diff --git a/mysys/mf_iocache.c b/mysys/mf_iocache.c index 9160f607448..28e5e72130d 100644 --- a/mysys/mf_iocache.c +++ b/mysys/mf_iocache.c @@ -70,6 +70,10 @@ static int _my_b_seq_read(IO_CACHE *info, uchar *Buffer, size_t Count); static int _my_b_cache_write(IO_CACHE *info, const uchar *Buffer, size_t Count); static int _my_b_cache_write_r(IO_CACHE *info, const uchar *Buffer, size_t Count); +int (*_my_b_encr_read)(IO_CACHE *info,uchar *Buffer,size_t Count)= 0; +int (*_my_b_encr_write)(IO_CACHE *info,const uchar *Buffer,size_t Count)= 0; + + /* Setup internal pointers inside IO_CACHE @@ -114,18 +118,35 @@ init_functions(IO_CACHE* info) programs that link against mysys but know nothing about THD, such as myisamchk */ + DBUG_ASSERT(!(info->myflags & MY_ENCRYPT)); break; case SEQ_READ_APPEND: info->read_function = _my_b_seq_read; + DBUG_ASSERT(!(info->myflags & MY_ENCRYPT)); break; case READ_CACHE: + if (info->myflags & MY_ENCRYPT) + { + DBUG_ASSERT(info->share == 0); + info->read_function = _my_b_encr_read; + break; + } + /* fall through */ case WRITE_CACHE: + if (info->myflags & MY_ENCRYPT) + { + info->write_function = _my_b_encr_write; + break; + } + /* fall through */ case READ_FIFO: + DBUG_ASSERT(!(info->myflags & MY_ENCRYPT)); info->read_function = info->share ? _my_b_cache_read_r : _my_b_cache_read; info->write_function = info->share ? _my_b_cache_write_r : _my_b_cache_write; break; case TYPE_NOT_SET: DBUG_ASSERT(0); + break; } setup_io_cache(info); @@ -175,6 +196,7 @@ int init_io_cache(IO_CACHE *info, File file, size_t cachesize, if (file >= 0) { + DBUG_ASSERT(!(cache_myflags & MY_ENCRYPT)); pos= mysql_file_tell(file, MYF(0)); if ((pos == (my_off_t) -1) && (my_errno == ESPIPE)) { @@ -191,6 +213,12 @@ int init_io_cache(IO_CACHE *info, File file, size_t cachesize, else info->seek_not_done= MY_TEST(seek_offset != pos); } + else + if (type == WRITE_CACHE && _my_b_encr_read) + { + cache_myflags|= MY_ENCRYPT; + DBUG_ASSERT(seek_offset == 0); + } info->disk_writes= 0; info->share=0; @@ -200,6 +228,7 @@ int init_io_cache(IO_CACHE *info, File file, size_t cachesize, min_cache=use_async_io ? IO_SIZE*4 : IO_SIZE*2; if (type == READ_CACHE || type == SEQ_READ_APPEND) { /* Assume file isn't growing */ + DBUG_ASSERT(!(cache_myflags & MY_ENCRYPT)); if (!(cache_myflags & MY_DONT_CHECK_FILESIZE)) { /* Calculate end of file to avoid allocating oversized buffers */ @@ -235,6 +264,8 @@ int init_io_cache(IO_CACHE *info, File file, size_t cachesize, buffer_block= cachesize; if (type == SEQ_READ_APPEND) buffer_block *= 2; + else if (cache_myflags & MY_ENCRYPT) + buffer_block= 2*(buffer_block + MY_AES_BLOCK_SIZE) + sizeof(IO_CACHE_CRYPT); if (cachesize == min_cache) flags|= (myf) MY_WME; @@ -288,6 +319,7 @@ int init_io_cache(IO_CACHE *info, File file, size_t cachesize, if (use_async_io && ! my_disable_async_io) { DBUG_PRINT("info",("Using async io")); + DBUG_ASSERT(!(cache_myflags & MY_ENCRYPT)); info->read_length/=2; info->read_function=_my_b_async_read; } @@ -400,8 +432,22 @@ my_bool reinit_io_cache(IO_CACHE *info, enum cache_type type, } else { - info->write_end=(info->buffer + info->buffer_length - - (seek_offset & (IO_SIZE-1))); + if (info->myflags & MY_ENCRYPT) + { + info->write_end = info->write_buffer + info->buffer_length; + if (seek_offset && info->file != -1) + { + info->read_end= info->buffer; + _my_b_encr_read(info, 0, 0); /* prefill the buffer */ + info->write_pos= info->read_pos; + info->pos_in_file+= info->buffer_length; + } + } + else + { + info->write_end=(info->buffer + info->buffer_length - + (seek_offset & (IO_SIZE-1))); + } info->end_of_file= ~(my_off_t) 0; } } @@ -414,6 +460,7 @@ my_bool reinit_io_cache(IO_CACHE *info, enum cache_type type, ((ulong) info->buffer_length < (ulong) (info->end_of_file - seek_offset))) { + DBUG_ASSERT(!(cache_myflags & MY_ENCRYPT)); info->read_length=info->buffer_length/2; info->read_function=_my_b_async_read; } @@ -514,7 +561,7 @@ int _my_b_write(IO_CACHE *info, const uchar *Buffer, size_t Count) Otherwise info->error contains the number of bytes in Buffer. */ -static int _my_b_cache_read(IO_CACHE *info, uchar *Buffer, size_t Count) +int _my_b_cache_read(IO_CACHE *info, uchar *Buffer, size_t Count) { size_t length, diff_length, left_length= 0, max_length; my_off_t pos_in_file; @@ -1057,6 +1104,7 @@ static int _my_b_cache_read_r(IO_CACHE *cache, uchar *Buffer, size_t Count) size_t length, diff_length, left_length= 0; IO_CACHE_SHARE *cshare= cache->share; DBUG_ENTER("_my_b_cache_read_r"); + DBUG_ASSERT(!(cache->myflags & MY_ENCRYPT)); while (Count) { @@ -1560,7 +1608,7 @@ int _my_b_get(IO_CACHE *info) -1 On error; my_errno contains error code. */ -static int _my_b_cache_write(IO_CACHE *info, const uchar *Buffer, size_t Count) +int _my_b_cache_write(IO_CACHE *info, const uchar *Buffer, size_t Count) { if (Buffer != info->write_buffer) { @@ -1611,6 +1659,7 @@ static int _my_b_cache_write_r(IO_CACHE *info, const uchar *Buffer, size_t Count if (res) return res; + DBUG_ASSERT(!(info->myflags & MY_ENCRYPT)); DBUG_ASSERT(info->share); copy_to_read_buffer(info, Buffer, old_pos_in_file); @@ -1633,6 +1682,7 @@ int my_b_append(IO_CACHE *info, const uchar *Buffer, size_t Count) day, we might need to add a call to copy_to_read_buffer(). */ DBUG_ASSERT(!info->share); + DBUG_ASSERT(!(info->myflags & MY_ENCRYPT)); lock_append_buffer(info); rest_length= (size_t) (info->write_end - info->write_pos); @@ -1699,6 +1749,7 @@ int my_block_write(IO_CACHE *info, const uchar *Buffer, size_t Count, day, we might need to add a call to copy_to_read_buffer(). */ DBUG_ASSERT(!info->share); + DBUG_ASSERT(!(info->myflags & MY_ENCRYPT)); if (pos < info->pos_in_file) { diff --git a/mysys/mf_iocache2.c b/mysys/mf_iocache2.c index 5443d5c21c5..93caa7bc74a 100644 --- a/mysys/mf_iocache2.c +++ b/mysys/mf_iocache2.c @@ -182,6 +182,13 @@ void my_b_seek(IO_CACHE *info,my_off_t pos) int my_b_pread(IO_CACHE *info, uchar *Buffer, size_t Count, my_off_t pos) { + if (info->myflags & MY_ENCRYPT) + { + my_b_seek(info, pos); + return my_b_read(info, Buffer, Count); + } + + /* backward compatibility behavior. XXX remove it? */ if (mysql_file_pread(info->file, Buffer, Count, pos, info->myflags | MY_NABP)) return info->error= -1; return 0; diff --git a/mysys/mysys_priv.h b/mysys/mysys_priv.h index 4ea6d081107..d080aca7404 100644 --- a/mysys/mysys_priv.h +++ b/mysys/mysys_priv.h @@ -13,8 +13,14 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ +#ifndef MYSYS_PRIV_INCLUDED +#define MYSYS_PRIV_INCLUDED + #include #include +#include + +C_MODE_START #ifdef USE_SYSTEM_WRAPPERS #include "system_wrappers.h" @@ -71,6 +77,16 @@ extern PSI_file_key key_file_proc_meminfo; extern PSI_file_key key_file_charset, key_file_cnf; #endif /* HAVE_PSI_INTERFACE */ +typedef struct { + ulonglong counter; + uint block_length, last_block_length; + uchar key[MY_AES_BLOCK_SIZE]; + ulonglong inbuf_counter; +} IO_CACHE_CRYPT; + +extern int (*_my_b_encr_read)(IO_CACHE *info,uchar *Buffer,size_t Count); +extern int (*_my_b_encr_write)(IO_CACHE *info,const uchar *Buffer,size_t Count); + #ifdef SAFEMALLOC void *sf_malloc(size_t size, myf my_flags); void *sf_realloc(void *ptr, size_t size, myf my_flags); @@ -116,3 +132,7 @@ extern File my_win_dup(File fd); extern File my_win_sopen(const char *path, int oflag, int shflag, int perm); extern File my_open_osfhandle(HANDLE handle, int oflag); #endif + +C_MODE_END + +#endif diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 522e8076275..2eb65837992 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -130,9 +130,8 @@ SET (SQL_SOURCE opt_index_cond_pushdown.cc opt_subselect.cc opt_table_elimination.cc sql_expression_cache.cc gcalc_slicescan.cc gcalc_tools.cc - threadpool_common.cc - ../sql-common/mysql_async.c - my_apc.cc my_apc.h + threadpool_common.cc ../sql-common/mysql_async.c + my_apc.cc my_apc.h mf_iocache_encr.cc my_json_writer.cc my_json_writer.h rpl_gtid.cc rpl_parallel.cc ${WSREP_SOURCES} diff --git a/sql/encryption.cc b/sql/encryption.cc index b108eb6a25c..520eb8b898f 100644 --- a/sql/encryption.cc +++ b/sql/encryption.cc @@ -19,6 +19,8 @@ #include "sql_plugin.h" #include +void init_io_cache_encryption(); + /* there can be only one encryption plugin enabled */ static plugin_ref encryption_manager= 0; struct encryption_service_st encryption_handler; @@ -79,6 +81,8 @@ int initialize_encryption_plugin(st_plugin_int *plugin) encryption_handler.encryption_key_get_latest_version_func= handle->get_latest_key_version; // must be the last + init_io_cache_encryption(); + return 0; } @@ -100,6 +104,7 @@ int finalize_encryption_plugin(st_plugin_int *plugin) if (encryption_manager) plugin_unlock(NULL, encryption_manager); encryption_manager= 0; + init_io_cache_encryption(); return 0; } diff --git a/sql/mf_iocache_encr.cc b/sql/mf_iocache_encr.cc new file mode 100644 index 00000000000..0a72492d743 --- /dev/null +++ b/sql/mf_iocache_encr.cc @@ -0,0 +1,254 @@ +/* + Copyright (c) 2015, MariaDB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +/************************************************************************* + Limitation of encrypted IO_CACHEs + 1. Designed to support temporary files only (open_cached_file, fd=-1) + 2. Created with WRITE_CACHE, later can be reinit_io_cache'ed to + READ_CACHE and WRITE_CACHE in any order arbitrary number of times. + 3. no seeks for writes, but reinit_io_cache(WRITE_CACHE, seek_offset) + is allowed (there's a special hack in reinit_io_cache() for that) +*/ + +#include "../mysys/mysys_priv.h" +#include "log.h" +#include "mysqld.h" +#include "sql_class.h" + +static uint keyid, keyver; + +#define set_iv(IV, N1, N2) \ + do { \ + compile_time_assert(sizeof(IV) >= sizeof(N1) + sizeof(N2)); \ + memcpy(IV, &(N1), sizeof(N1)); \ + memcpy(IV + sizeof(N1), &(N2), sizeof(N2)); \ + } while(0) + +static int my_b_encr_read(IO_CACHE *info, uchar *Buffer, size_t Count) +{ + my_off_t pos_in_file= info->pos_in_file + (info->read_end - info->buffer); + my_off_t old_pos_in_file= pos_in_file, pos_offset= 0; + IO_CACHE_CRYPT *crypt_data= + (IO_CACHE_CRYPT *)(info->buffer + info->buffer_length + MY_AES_BLOCK_SIZE); + uchar *wbuffer= (uchar*)&(crypt_data->inbuf_counter); + uchar *ebuffer= (uchar*)(crypt_data + 1); + DBUG_ENTER("my_b_encr_read"); + + if (pos_in_file == info->end_of_file) + { + info->read_pos= info->read_end= info->buffer; + info->pos_in_file= pos_in_file; + info->error= 0; + DBUG_RETURN(MY_TEST(Count)); + } + + if (info->seek_not_done) + { + size_t wpos; + + pos_offset= pos_in_file % info->buffer_length; + pos_in_file-= pos_offset; + + wpos= pos_in_file / info->buffer_length * crypt_data->block_length; + + if ((mysql_file_seek(info->file, wpos, MY_SEEK_SET, MYF(0)) + == MY_FILEPOS_ERROR)) + { + info->error= -1; + DBUG_RETURN(1); + } + info->seek_not_done= 0; + } + + do + { + size_t copied; + uint elength, wlength, length; + uchar iv[MY_AES_BLOCK_SIZE]= {0}; + + DBUG_ASSERT(pos_in_file % info->buffer_length == 0); + + if (info->end_of_file - pos_in_file >= info->buffer_length) + wlength= crypt_data->block_length; + else + wlength= crypt_data->last_block_length; + + if (mysql_file_read(info->file, wbuffer, wlength, info->myflags | MY_NABP)) + { + info->error= -1; + DBUG_RETURN(1); + } + + elength= wlength - (ebuffer - wbuffer); + set_iv(iv, pos_in_file, crypt_data->inbuf_counter); + + if (encryption_decrypt(ebuffer, elength, info->buffer, &length, + crypt_data->key, sizeof(crypt_data->key), + iv, sizeof(iv), 0, keyid, keyver)) + { + my_errno= 1; + DBUG_RETURN(info->error= -1); + } + + DBUG_ASSERT(length <= info->buffer_length); + + copied= MY_MIN(Count, length - pos_offset); + + memcpy(Buffer, info->buffer + pos_offset, copied); + Count-= copied; + Buffer+= copied; + + info->read_pos= info->buffer + pos_offset + copied; + info->read_end= info->buffer + length; + info->pos_in_file= pos_in_file; + pos_in_file+= length; + pos_offset= 0; + + if (wlength < crypt_data->block_length && pos_in_file < info->end_of_file) + { + info->error= pos_in_file - old_pos_in_file; + DBUG_RETURN(1); + } + } while (Count); + + DBUG_RETURN(0); +} + +static int my_b_encr_write(IO_CACHE *info, const uchar *Buffer, size_t Count) +{ + IO_CACHE_CRYPT *crypt_data= + (IO_CACHE_CRYPT *)(info->buffer + info->buffer_length + MY_AES_BLOCK_SIZE); + uchar *wbuffer= (uchar*)&(crypt_data->inbuf_counter); + uchar *ebuffer= (uchar*)(crypt_data + 1); + DBUG_ENTER("my_b_encr_write"); + + if (Buffer != info->write_buffer) + { + Count-= Count % info->buffer_length; + if (!Count) + DBUG_RETURN(0); + } + + if (info->seek_not_done) + { + DBUG_ASSERT(info->pos_in_file == 0); + + if ((mysql_file_seek(info->file, 0, MY_SEEK_SET, MYF(0)) == MY_FILEPOS_ERROR)) + { + info->error= -1; + DBUG_RETURN(1); + } + info->seek_not_done= 0; + } + + if (info->pos_in_file == 0) + { + if (my_random_bytes(crypt_data->key, sizeof(crypt_data->key))) + { + my_errno= 1; + DBUG_RETURN(info->error= -1); + } + crypt_data->counter= 0; + + IF_DBUG(crypt_data->block_length= 0,); + } + + do + { + size_t length= MY_MIN(info->buffer_length, Count); + uint elength, wlength; + uchar iv[MY_AES_BLOCK_SIZE]= {0}; + + crypt_data->inbuf_counter= crypt_data->counter; + set_iv(iv, info->pos_in_file, crypt_data->inbuf_counter); + + if (encryption_encrypt(Buffer, length, ebuffer, &elength, + crypt_data->key, sizeof(crypt_data->key), + iv, sizeof(iv), 0, keyid, keyver)) + { + my_errno= 1; + DBUG_RETURN(info->error= -1); + } + wlength= elength + ebuffer - wbuffer; + + if (length == info->buffer_length) + { + /* + block_length should be always the same. that is, encrypting + buffer_length bytes should *always* produce block_length bytes + */ + DBUG_ASSERT(crypt_data->block_length == 0 || crypt_data->block_length == wlength); + DBUG_ASSERT(elength <= length + MY_AES_BLOCK_SIZE); + crypt_data->block_length= wlength; + } + else + { + /* if we write a partial block, it *must* be the last write */ + IF_DBUG(info->write_function= 0,); + crypt_data->last_block_length= wlength; + } + + if (mysql_file_write(info->file, wbuffer, wlength, info->myflags | MY_NABP)) + DBUG_RETURN(info->error= -1); + + Buffer+= length; + Count-= length; + info->pos_in_file+= length; + crypt_data->counter++; + } while (Count); + DBUG_RETURN(0); +} + +/** + determine what key id and key version to use for IO_CACHE temp files + + First, try key id 2, if it doesn't exist, use key id 1. + + (key id 1 is the default system key id, used pretty much everywhere, it must + exist. key id 2 is for tempfiles, it can be used, for example, to set a + faster encryption algorithm for temporary files) + + This looks like it might have a bug: if an encryption plugin is unloaded when + there's an open IO_CACHE, that IO_CACHE will become unreadable after reinit. + But in fact it is safe, as an encryption plugin can only be unloaded on + server shutdown. + + Note that encrypt_tmp_files variable is read-only. +*/ +void init_io_cache_encryption() +{ + if (encrypt_tmp_files) + { + keyver= encryption_key_get_latest_version(keyid= 2); + if (keyver == ENCRYPTION_KEY_VERSION_INVALID) + keyver= encryption_key_get_latest_version(keyid= 1); + } + else + keyver= ENCRYPTION_KEY_VERSION_INVALID; + + if (keyver != ENCRYPTION_KEY_VERSION_INVALID) + { + sql_print_information("Using encryption key id %d for temporary files", keyid); + _my_b_encr_read= my_b_encr_read; + _my_b_encr_write= my_b_encr_write; + } + else + { + _my_b_encr_read= 0; + _my_b_encr_write= 0; + } +} + diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 37ae81f896b..2bde88189fb 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -629,7 +629,7 @@ char server_version[SERVER_VERSION_LENGTH]; char *mysqld_unix_port, *opt_mysql_tmpdir; ulong thread_handling; -my_bool encrypt_tmp_disk_tables; +my_bool encrypt_tmp_disk_tables, encrypt_tmp_files; /** name of reference on left expression in rewritten IN subquery */ const char *in_left_expr_name= ""; diff --git a/sql/mysqld.h b/sql/mysqld.h index e1c39431015..ebf55d8fd8a 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -254,7 +254,7 @@ extern ulong connection_errors_internal; extern ulong connection_errors_max_connection; extern ulong connection_errors_peer_addr; extern ulong log_warnings; -extern my_bool encrypt_tmp_disk_tables; +extern my_bool encrypt_tmp_disk_tables, encrypt_tmp_files; extern ulong encryption_algorithm; extern const char *encryption_algorithm_names[]; diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 09c743434f5..88e5f1b392b 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -5163,10 +5163,16 @@ static Sys_var_harows Sys_expensive_subquery_limit( static Sys_var_mybool Sys_encrypt_tmp_disk_tables( "encrypt_tmp_disk_tables", - "Encrypt tmp disk tables (created as part of query execution)", + "Encrypt temporary on-disk tables (created as part of query execution)", GLOBAL_VAR(encrypt_tmp_disk_tables), CMD_LINE(OPT_ARG), DEFAULT(FALSE)); +static Sys_var_mybool Sys_encrypt_tmp_files( + "encrypt_tmp_files", + "Encrypt temporary files (created for filesort, binary log cache, etc)", + READ_ONLY GLOBAL_VAR(encrypt_tmp_files), + CMD_LINE(OPT_ARG), DEFAULT(TRUE)); + static bool check_pseudo_slave_mode(sys_var *self, THD *thd, set_var *var) { longlong previous_val= thd->variables.pseudo_slave_mode; diff --git a/storage/maria/ma_cache.c b/storage/maria/ma_cache.c index e32ba73f0a5..24739671be6 100644 --- a/storage/maria/ma_cache.c +++ b/storage/maria/ma_cache.c @@ -42,6 +42,7 @@ my_bool _ma_read_cache(MARIA_HA *handler, IO_CACHE *info, uchar *buff, my_off_t offset; uchar *in_buff_pos; DBUG_ENTER("_ma_read_cache"); + DBUG_ASSERT(!(info->myflags & MY_ENCRYPT)); if (pos < info->pos_in_file) { diff --git a/storage/myisam/mi_cache.c b/storage/myisam/mi_cache.c index d86dcb4d918..edcc3520c35 100644 --- a/storage/myisam/mi_cache.c +++ b/storage/myisam/mi_cache.c @@ -43,6 +43,7 @@ int _mi_read_cache(IO_CACHE *info, uchar *buff, my_off_t pos, uint length, my_off_t offset; uchar *in_buff_pos; DBUG_ENTER("_mi_read_cache"); + DBUG_ASSERT(!(info->myflags & MY_ENCRYPT)); if (pos < info->pos_in_file) { -- cgit v1.2.1