diff options
| author | Sergei Golubchik <serg@mariadb.org> | 2015-12-15 17:23:58 +0100 |
|---|---|---|
| committer | Sergei Golubchik <serg@mariadb.org> | 2015-12-15 17:23:58 +0100 |
| commit | dcc501aec364f27f03aabfd5db6d5362a966be86 (patch) | |
| tree | 9ad055032ef798b5b38836b73dff4bb3924774a2 | |
| parent | 2c8c65297865d9f8da501761f46e2a34e29af603 (diff) | |
| download | mariadb-git-dcc501aec364f27f03aabfd5db6d5362a966be86.tar.gz | |
5.6.27-76.0
206 files changed, 15619 insertions, 5969 deletions
diff --git a/mysql-test/suite/tokudb.add_index/t/suite.opt b/mysql-test/suite/tokudb.add_index/t/suite.opt new file mode 100644 index 00000000000..23511b05020 --- /dev/null +++ b/mysql-test/suite/tokudb.add_index/t/suite.opt @@ -0,0 +1 @@ +$TOKUDB_OPT $TOKUDB_LOAD_ADD --loose-tokudb-check-jemalloc=0 diff --git a/mysql-test/suite/tokudb.alter_table/t/suite.opt b/mysql-test/suite/tokudb.alter_table/t/suite.opt new file mode 100644 index 00000000000..23511b05020 --- /dev/null +++ b/mysql-test/suite/tokudb.alter_table/t/suite.opt @@ -0,0 +1 @@ +$TOKUDB_OPT $TOKUDB_LOAD_ADD --loose-tokudb-check-jemalloc=0 diff --git a/mysql-test/suite/tokudb.bugs/r/db756_card_part_hash.result b/mysql-test/suite/tokudb.bugs/r/db756_card_part_hash.result index c1ca24b0551..70bc86e1abc 100644 --- a/mysql-test/suite/tokudb.bugs/r/db756_card_part_hash.result +++ b/mysql-test/suite/tokudb.bugs/r/db756_card_part_hash.result @@ -4,18 +4,18 @@ create table t (id int, x int, primary key (id), key (x)) partition by hash(id) show indexes from t; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment t 0 PRIMARY 1 id A 2 NULL NULL BTREE -t 1 x 1 x A NULL NULL NULL YES BTREE +t 1 x 1 x A 2 NULL NULL YES BTREE insert into t values (1,1),(3,1),(5,1); insert into t values (2,1),(4,1),(6,1); show indexes from t; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment t 0 PRIMARY 1 id A 6 NULL NULL BTREE -t 1 x 1 x A NULL NULL NULL YES BTREE +t 1 x 1 x A 6 NULL NULL YES BTREE analyze table t; Table Op Msg_type Msg_text test.t analyze status OK show indexes from t; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment t 0 PRIMARY 1 id A 6 NULL NULL BTREE -t 1 x 1 x A 2 NULL NULL YES BTREE +t 1 x 1 x A 6 NULL NULL YES BTREE drop table t; diff --git a/mysql-test/suite/tokudb.bugs/r/db756_card_part_hash_1.result b/mysql-test/suite/tokudb.bugs/r/db756_card_part_hash_1.result index c82c922a87a..b6d9fd7da85 100644 --- a/mysql-test/suite/tokudb.bugs/r/db756_card_part_hash_1.result +++ b/mysql-test/suite/tokudb.bugs/r/db756_card_part_hash_1.result @@ -4,17 +4,17 @@ create table t (id int, x int, primary key (id), key (x)) partition by hash(id) show indexes from t; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment t 0 PRIMARY 1 id A 2 NULL NULL BTREE -t 1 x 1 x A NULL NULL NULL YES BTREE +t 1 x 1 x A 2 NULL NULL YES BTREE insert into t values (1,1),(3,1),(5,1); show indexes from t; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment t 0 PRIMARY 1 id A 4 NULL NULL BTREE -t 1 x 1 x A NULL NULL NULL YES BTREE +t 1 x 1 x A 4 NULL NULL YES BTREE analyze table t; Table Op Msg_type Msg_text test.t analyze status OK show indexes from t; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment t 0 PRIMARY 1 id A 4 NULL NULL BTREE -t 1 x 1 x A 1 NULL NULL YES BTREE +t 1 x 1 x A 4 NULL NULL YES BTREE drop table t; diff --git a/mysql-test/suite/tokudb.bugs/r/db756_card_part_hash_1_pick.result b/mysql-test/suite/tokudb.bugs/r/db756_card_part_hash_1_pick.result index 576d1adee6c..caaa963c325 100644 --- a/mysql-test/suite/tokudb.bugs/r/db756_card_part_hash_1_pick.result +++ b/mysql-test/suite/tokudb.bugs/r/db756_card_part_hash_1_pick.result @@ -4,18 +4,18 @@ create table t (id int, x int, primary key (id), key (x)) partition by hash(id) show indexes from t; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment t 0 PRIMARY 1 id A 2 NULL NULL BTREE -t 1 x 1 x A NULL NULL NULL YES BTREE +t 1 x 1 x A 2 NULL NULL YES BTREE insert into t values (1,1),(3,2),(5,3); insert into t values (2,1),(4,1),(6,1),(8,1); show indexes from t; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment t 0 PRIMARY 1 id A 7 NULL NULL BTREE -t 1 x 1 x A NULL NULL NULL YES BTREE +t 1 x 1 x A 7 NULL NULL YES BTREE analyze table t; Table Op Msg_type Msg_text test.t analyze status OK show indexes from t; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment t 0 PRIMARY 1 id A 7 NULL NULL BTREE -t 1 x 1 x A 1 NULL NULL YES BTREE +t 1 x 1 x A 7 NULL NULL YES BTREE drop table t; diff --git a/mysql-test/suite/tokudb.bugs/r/db756_card_part_hash_2.result b/mysql-test/suite/tokudb.bugs/r/db756_card_part_hash_2.result index 8937ddabbe3..6d345d98c95 100644 --- a/mysql-test/suite/tokudb.bugs/r/db756_card_part_hash_2.result +++ b/mysql-test/suite/tokudb.bugs/r/db756_card_part_hash_2.result @@ -4,17 +4,17 @@ create table t (id int, x int, primary key (id), key (x)) partition by hash(id) show indexes from t; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment t 0 PRIMARY 1 id A 2 NULL NULL BTREE -t 1 x 1 x A NULL NULL NULL YES BTREE +t 1 x 1 x A 2 NULL NULL YES BTREE insert into t values (2,1),(4,1),(6,1); show indexes from t; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment t 0 PRIMARY 1 id A 4 NULL NULL BTREE -t 1 x 1 x A NULL NULL NULL YES BTREE +t 1 x 1 x A 4 NULL NULL YES BTREE analyze table t; Table Op Msg_type Msg_text test.t analyze status OK show indexes from t; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment t 0 PRIMARY 1 id A 4 NULL NULL BTREE -t 1 x 1 x A 1 NULL NULL YES BTREE +t 1 x 1 x A 4 NULL NULL YES BTREE drop table t; diff --git a/mysql-test/suite/tokudb.bugs/r/db756_card_part_hash_2_pick.result b/mysql-test/suite/tokudb.bugs/r/db756_card_part_hash_2_pick.result index 89b51a5a6b0..06639c311cf 100644 --- a/mysql-test/suite/tokudb.bugs/r/db756_card_part_hash_2_pick.result +++ b/mysql-test/suite/tokudb.bugs/r/db756_card_part_hash_2_pick.result @@ -4,13 +4,13 @@ create table t (id int, x int, primary key (id), key (x)) partition by hash(id) show indexes from t; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment t 0 PRIMARY 1 id A 2 NULL NULL BTREE -t 1 x 1 x A NULL NULL NULL YES BTREE +t 1 x 1 x A 2 NULL NULL YES BTREE insert into t values (1,1),(3,2),(5,3),(7,4); insert into t values (2,1),(4,1),(6,1); show indexes from t; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment t 0 PRIMARY 1 id A 7 NULL NULL BTREE -t 1 x 1 x A NULL NULL NULL YES BTREE +t 1 x 1 x A 7 NULL NULL YES BTREE analyze table t; Table Op Msg_type Msg_text test.t analyze status OK diff --git a/mysql-test/suite/tokudb.bugs/r/db757_part_alter_analyze.result b/mysql-test/suite/tokudb.bugs/r/db757_part_alter_analyze.result index d80e3e4eac8..4b82cb06026 100644 --- a/mysql-test/suite/tokudb.bugs/r/db757_part_alter_analyze.result +++ b/mysql-test/suite/tokudb.bugs/r/db757_part_alter_analyze.result @@ -7,15 +7,15 @@ insert into t values (1,1,1),(2,1,2),(3,1,3),(4,1,4); show indexes from t; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment t 0 PRIMARY 1 id A 5 NULL NULL BTREE -t 1 x 1 x A NULL NULL NULL YES BTREE -t 1 y 1 y A NULL NULL NULL YES BTREE +t 1 x 1 x A 5 NULL NULL YES BTREE +t 1 y 1 y A 5 NULL NULL YES BTREE alter table t analyze partition p0; Table Op Msg_type Msg_text test.t analyze status OK show indexes from t; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment t 0 PRIMARY 1 id A 5 NULL NULL BTREE -t 1 x 1 x A 1 NULL NULL YES BTREE +t 1 x 1 x A 5 NULL NULL YES BTREE t 1 y 1 y A 5 NULL NULL YES BTREE alter table t analyze partition p1; Table Op Msg_type Msg_text @@ -23,13 +23,13 @@ test.t analyze status OK show indexes from t; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment t 0 PRIMARY 1 id A 5 NULL NULL BTREE -t 1 x 1 x A 1 NULL NULL YES BTREE +t 1 x 1 x A 5 NULL NULL YES BTREE t 1 y 1 y A 5 NULL NULL YES BTREE insert into t values (100,1,1),(200,2,1),(300,3,1),(400,4,1),(500,5,1); show indexes from t; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment t 0 PRIMARY 1 id A 9 NULL NULL BTREE -t 1 x 1 x A 2 NULL NULL YES BTREE +t 1 x 1 x A 9 NULL NULL YES BTREE t 1 y 1 y A 9 NULL NULL YES BTREE alter table t analyze partition p0; Table Op Msg_type Msg_text @@ -37,8 +37,8 @@ test.t analyze status OK show indexes from t; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment t 0 PRIMARY 1 id A 9 NULL NULL BTREE -t 1 x 1 x A NULL NULL NULL YES BTREE -t 1 y 1 y A NULL NULL NULL YES BTREE +t 1 x 1 x A 9 NULL NULL YES BTREE +t 1 y 1 y A 9 NULL NULL YES BTREE alter table t analyze partition p1; Table Op Msg_type Msg_text test.t analyze status OK @@ -46,5 +46,5 @@ show indexes from t; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment t 0 PRIMARY 1 id A 9 NULL NULL BTREE t 1 x 1 x A 9 NULL NULL YES BTREE -t 1 y 1 y A 1 NULL NULL YES BTREE +t 1 y 1 y A 4 NULL NULL YES BTREE drop table t; diff --git a/mysql-test/suite/tokudb.bugs/t/5585-master.opt b/mysql-test/suite/tokudb.bugs/t/5585-master.opt index 017432e797d..acad193fd76 100644 --- a/mysql-test/suite/tokudb.bugs/t/5585-master.opt +++ b/mysql-test/suite/tokudb.bugs/t/5585-master.opt @@ -1 +1 @@ ---tokudb-cache-size=1000000000 --innodb-buffer-pool-size=1000000000 +--loose-tokudb-cache-size=1000000000 --innodb-buffer-pool-size=1000000000 diff --git a/mysql-test/suite/tokudb.bugs/t/5585.test b/mysql-test/suite/tokudb.bugs/t/5585.test index bf0df681e7a..6cc5fb223c0 100644 --- a/mysql-test/suite/tokudb.bugs/t/5585.test +++ b/mysql-test/suite/tokudb.bugs/t/5585.test @@ -1,5 +1,6 @@ --source include/have_innodb.inc --source include/have_tokudb.inc +--source include/big_test.inc --disable_warnings drop table if exists t1, t2; @@ -64,4 +65,4 @@ select t1.file_id, (select hits from t2 where t2.file_id = t1.file_id and t2.insert_ts = date(date_sub(now(),interval 1 day))) as d from t1; -drop table if exists t1, t2;
\ No newline at end of file +drop table if exists t1, t2; diff --git a/mysql-test/suite/tokudb.bugs/t/disabled.def b/mysql-test/suite/tokudb.bugs/t/disabled.def index 7e67d159d7a..8c755dde8f5 100644 --- a/mysql-test/suite/tokudb.bugs/t/disabled.def +++ b/mysql-test/suite/tokudb.bugs/t/disabled.def @@ -11,3 +11,6 @@ checkpoint_lock_2: test can not work when the checkpoint_safe_lock is a fair rwl 6053: tokudb is not the default storage engine 1883: tokutek's auto inc singleton patch missing 3083: no patch to find_shortest_key to prefer PK over CK +db768 : https://tokutek.atlassian.net/browse/DB-768 +dict_leak_3518 : https://tokutek.atlassian.net/browse/DB-635 +1872 : https://tokutek.atlassian.net/browse/DB-750 diff --git a/mysql-test/suite/tokudb.bugs/t/suite.opt b/mysql-test/suite/tokudb.bugs/t/suite.opt new file mode 100644 index 00000000000..23511b05020 --- /dev/null +++ b/mysql-test/suite/tokudb.bugs/t/suite.opt @@ -0,0 +1 @@ +$TOKUDB_OPT $TOKUDB_LOAD_ADD --loose-tokudb-check-jemalloc=0 diff --git a/mysql-test/suite/tokudb.parts/r/partition_alter4_tokudb.result b/mysql-test/suite/tokudb.parts/r/partition_alter4_tokudb.result index 644c4815a36..b4e8e47b7d9 100644 --- a/mysql-test/suite/tokudb.parts/r/partition_alter4_tokudb.result +++ b/mysql-test/suite/tokudb.parts/r/partition_alter4_tokudb.result @@ -7560,7 +7560,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 ANALYZE PARTITION part_1,part_2,part_5,part_6,part_10; Table Op Msg_type Msg_text -test.t1 analyze error Error in list of partitions to test.t1 +test.t1 analyze Error Error in list of partitions to test.t1 +test.t1 analyze status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -8019,7 +8020,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 ANALYZE PARTITION part_1,part_2,part_5,part_6,part_10; Table Op Msg_type Msg_text -test.t1 analyze error Error in list of partitions to test.t1 +test.t1 analyze Error Error in list of partitions to test.t1 +test.t1 analyze status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -8489,7 +8491,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 ANALYZE PARTITION part_1,part_2,part_5,part_6,part_10; Table Op Msg_type Msg_text -test.t1 analyze error Error in list of partitions to test.t1 +test.t1 analyze Error Error in list of partitions to test.t1 +test.t1 analyze status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -8960,7 +8963,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 ANALYZE PARTITION part_1,part_2,part_5,part_6,part_10; Table Op Msg_type Msg_text -test.t1 analyze error Error in list of partitions to test.t1 +test.t1 analyze Error Error in list of partitions to test.t1 +test.t1 analyze status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -9425,7 +9429,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 ANALYZE PARTITION part_1,part_2,part_5,part_6,part_10; Table Op Msg_type Msg_text -test.t1 analyze error Error in list of partitions to test.t1 +test.t1 analyze Error Error in list of partitions to test.t1 +test.t1 analyze status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -9896,7 +9901,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 ANALYZE PARTITION part_1,part_2,part_5,part_6,part_10; Table Op Msg_type Msg_text -test.t1 analyze error Error in list of partitions to test.t1 +test.t1 analyze Error Error in list of partitions to test.t1 +test.t1 analyze status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -10372,7 +10378,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 ANALYZE PARTITION part_1,part_2,part_5,part_6,part_10; Table Op Msg_type Msg_text -test.t1 analyze error Error in list of partitions to test.t1 +test.t1 analyze Error Error in list of partitions to test.t1 +test.t1 analyze status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -10846,7 +10853,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 ANALYZE PARTITION part_1,part_2,part_5,part_6,part_10; Table Op Msg_type Msg_text -test.t1 analyze error Error in list of partitions to test.t1 +test.t1 analyze Error Error in list of partitions to test.t1 +test.t1 analyze status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -11310,7 +11318,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 ANALYZE PARTITION part_1,part_1,part_1; Table Op Msg_type Msg_text -test.t1 analyze error Error in list of partitions to test.t1 +test.t1 analyze Error Error in list of partitions to test.t1 +test.t1 analyze status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -11769,7 +11778,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 ANALYZE PARTITION part_1,part_1,part_1; Table Op Msg_type Msg_text -test.t1 analyze error Error in list of partitions to test.t1 +test.t1 analyze Error Error in list of partitions to test.t1 +test.t1 analyze status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -12239,7 +12249,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 ANALYZE PARTITION part_1,part_1,part_1; Table Op Msg_type Msg_text -test.t1 analyze error Error in list of partitions to test.t1 +test.t1 analyze Error Error in list of partitions to test.t1 +test.t1 analyze status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -12710,7 +12721,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 ANALYZE PARTITION part_1,part_1,part_1; Table Op Msg_type Msg_text -test.t1 analyze error Error in list of partitions to test.t1 +test.t1 analyze Error Error in list of partitions to test.t1 +test.t1 analyze status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -13175,7 +13187,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 ANALYZE PARTITION part_1,part_1,part_1; Table Op Msg_type Msg_text -test.t1 analyze error Error in list of partitions to test.t1 +test.t1 analyze Error Error in list of partitions to test.t1 +test.t1 analyze status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -13646,7 +13659,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 ANALYZE PARTITION part_1,part_1,part_1; Table Op Msg_type Msg_text -test.t1 analyze error Error in list of partitions to test.t1 +test.t1 analyze Error Error in list of partitions to test.t1 +test.t1 analyze status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -14122,7 +14136,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 ANALYZE PARTITION part_1,part_1,part_1; Table Op Msg_type Msg_text -test.t1 analyze error Error in list of partitions to test.t1 +test.t1 analyze Error Error in list of partitions to test.t1 +test.t1 analyze status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -14596,7 +14611,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 ANALYZE PARTITION part_1,part_1,part_1; Table Op Msg_type Msg_text -test.t1 analyze error Error in list of partitions to test.t1 +test.t1 analyze Error Error in list of partitions to test.t1 +test.t1 analyze status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -26313,7 +26329,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 CHECK PARTITION part_1,part_2,part_5,part_6,part_10; Table Op Msg_type Msg_text -test.t1 check error Error in list of partitions to test.t1 +test.t1 check Error Error in list of partitions to test.t1 +test.t1 check status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -26772,7 +26789,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 CHECK PARTITION part_1,part_2,part_5,part_6,part_10; Table Op Msg_type Msg_text -test.t1 check error Error in list of partitions to test.t1 +test.t1 check Error Error in list of partitions to test.t1 +test.t1 check status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -27242,7 +27260,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 CHECK PARTITION part_1,part_2,part_5,part_6,part_10; Table Op Msg_type Msg_text -test.t1 check error Error in list of partitions to test.t1 +test.t1 check Error Error in list of partitions to test.t1 +test.t1 check status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -27713,7 +27732,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 CHECK PARTITION part_1,part_2,part_5,part_6,part_10; Table Op Msg_type Msg_text -test.t1 check error Error in list of partitions to test.t1 +test.t1 check Error Error in list of partitions to test.t1 +test.t1 check status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -28178,7 +28198,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 CHECK PARTITION part_1,part_2,part_5,part_6,part_10; Table Op Msg_type Msg_text -test.t1 check error Error in list of partitions to test.t1 +test.t1 check Error Error in list of partitions to test.t1 +test.t1 check status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -28649,7 +28670,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 CHECK PARTITION part_1,part_2,part_5,part_6,part_10; Table Op Msg_type Msg_text -test.t1 check error Error in list of partitions to test.t1 +test.t1 check Error Error in list of partitions to test.t1 +test.t1 check status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -29125,7 +29147,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 CHECK PARTITION part_1,part_2,part_5,part_6,part_10; Table Op Msg_type Msg_text -test.t1 check error Error in list of partitions to test.t1 +test.t1 check Error Error in list of partitions to test.t1 +test.t1 check status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -29599,7 +29622,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 CHECK PARTITION part_1,part_2,part_5,part_6,part_10; Table Op Msg_type Msg_text -test.t1 check error Error in list of partitions to test.t1 +test.t1 check Error Error in list of partitions to test.t1 +test.t1 check status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -30063,7 +30087,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 CHECK PARTITION part_1,part_1,part_1; Table Op Msg_type Msg_text -test.t1 check error Error in list of partitions to test.t1 +test.t1 check Error Error in list of partitions to test.t1 +test.t1 check status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -30522,7 +30547,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 CHECK PARTITION part_1,part_1,part_1; Table Op Msg_type Msg_text -test.t1 check error Error in list of partitions to test.t1 +test.t1 check Error Error in list of partitions to test.t1 +test.t1 check status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -30992,7 +31018,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 CHECK PARTITION part_1,part_1,part_1; Table Op Msg_type Msg_text -test.t1 check error Error in list of partitions to test.t1 +test.t1 check Error Error in list of partitions to test.t1 +test.t1 check status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -31463,7 +31490,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 CHECK PARTITION part_1,part_1,part_1; Table Op Msg_type Msg_text -test.t1 check error Error in list of partitions to test.t1 +test.t1 check Error Error in list of partitions to test.t1 +test.t1 check status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -31928,7 +31956,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 CHECK PARTITION part_1,part_1,part_1; Table Op Msg_type Msg_text -test.t1 check error Error in list of partitions to test.t1 +test.t1 check Error Error in list of partitions to test.t1 +test.t1 check status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -32399,7 +32428,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 CHECK PARTITION part_1,part_1,part_1; Table Op Msg_type Msg_text -test.t1 check error Error in list of partitions to test.t1 +test.t1 check Error Error in list of partitions to test.t1 +test.t1 check status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -32875,7 +32905,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 CHECK PARTITION part_1,part_1,part_1; Table Op Msg_type Msg_text -test.t1 check error Error in list of partitions to test.t1 +test.t1 check Error Error in list of partitions to test.t1 +test.t1 check status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -33349,7 +33380,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 CHECK PARTITION part_1,part_1,part_1; Table Op Msg_type Msg_text -test.t1 check error Error in list of partitions to test.t1 +test.t1 check Error Error in list of partitions to test.t1 +test.t1 check status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -45082,7 +45114,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 OPTIMIZE PARTITION part_1,part_2,part_5,part_6,part_10; Table Op Msg_type Msg_text -test.t1 optimize error Error in list of partitions to test.t1 +test.t1 optimize Error Error in list of partitions to test.t1 +test.t1 optimize status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -45541,7 +45574,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 OPTIMIZE PARTITION part_1,part_2,part_5,part_6,part_10; Table Op Msg_type Msg_text -test.t1 optimize error Error in list of partitions to test.t1 +test.t1 optimize Error Error in list of partitions to test.t1 +test.t1 optimize status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -46011,7 +46045,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 OPTIMIZE PARTITION part_1,part_2,part_5,part_6,part_10; Table Op Msg_type Msg_text -test.t1 optimize error Error in list of partitions to test.t1 +test.t1 optimize Error Error in list of partitions to test.t1 +test.t1 optimize status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -46482,7 +46517,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 OPTIMIZE PARTITION part_1,part_2,part_5,part_6,part_10; Table Op Msg_type Msg_text -test.t1 optimize error Error in list of partitions to test.t1 +test.t1 optimize Error Error in list of partitions to test.t1 +test.t1 optimize status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -46947,7 +46983,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 OPTIMIZE PARTITION part_1,part_2,part_5,part_6,part_10; Table Op Msg_type Msg_text -test.t1 optimize error Error in list of partitions to test.t1 +test.t1 optimize Error Error in list of partitions to test.t1 +test.t1 optimize status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -47418,7 +47455,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 OPTIMIZE PARTITION part_1,part_2,part_5,part_6,part_10; Table Op Msg_type Msg_text -test.t1 optimize error Error in list of partitions to test.t1 +test.t1 optimize Error Error in list of partitions to test.t1 +test.t1 optimize status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -47894,7 +47932,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 OPTIMIZE PARTITION part_1,part_2,part_5,part_6,part_10; Table Op Msg_type Msg_text -test.t1 optimize error Error in list of partitions to test.t1 +test.t1 optimize Error Error in list of partitions to test.t1 +test.t1 optimize status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -48368,7 +48407,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 OPTIMIZE PARTITION part_1,part_2,part_5,part_6,part_10; Table Op Msg_type Msg_text -test.t1 optimize error Error in list of partitions to test.t1 +test.t1 optimize Error Error in list of partitions to test.t1 +test.t1 optimize status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -48832,7 +48872,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 OPTIMIZE PARTITION part_1,part_1,part_1; Table Op Msg_type Msg_text -test.t1 optimize error Error in list of partitions to test.t1 +test.t1 optimize Error Error in list of partitions to test.t1 +test.t1 optimize status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -49291,7 +49332,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 OPTIMIZE PARTITION part_1,part_1,part_1; Table Op Msg_type Msg_text -test.t1 optimize error Error in list of partitions to test.t1 +test.t1 optimize Error Error in list of partitions to test.t1 +test.t1 optimize status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -49761,7 +49803,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 OPTIMIZE PARTITION part_1,part_1,part_1; Table Op Msg_type Msg_text -test.t1 optimize error Error in list of partitions to test.t1 +test.t1 optimize Error Error in list of partitions to test.t1 +test.t1 optimize status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -50232,7 +50275,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 OPTIMIZE PARTITION part_1,part_1,part_1; Table Op Msg_type Msg_text -test.t1 optimize error Error in list of partitions to test.t1 +test.t1 optimize Error Error in list of partitions to test.t1 +test.t1 optimize status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -50697,7 +50741,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 OPTIMIZE PARTITION part_1,part_1,part_1; Table Op Msg_type Msg_text -test.t1 optimize error Error in list of partitions to test.t1 +test.t1 optimize Error Error in list of partitions to test.t1 +test.t1 optimize status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -51168,7 +51213,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 OPTIMIZE PARTITION part_1,part_1,part_1; Table Op Msg_type Msg_text -test.t1 optimize error Error in list of partitions to test.t1 +test.t1 optimize Error Error in list of partitions to test.t1 +test.t1 optimize status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -51644,7 +51690,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 OPTIMIZE PARTITION part_1,part_1,part_1; Table Op Msg_type Msg_text -test.t1 optimize error Error in list of partitions to test.t1 +test.t1 optimize Error Error in list of partitions to test.t1 +test.t1 optimize status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -52118,7 +52165,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 OPTIMIZE PARTITION part_1,part_1,part_1; Table Op Msg_type Msg_text -test.t1 optimize error Error in list of partitions to test.t1 +test.t1 optimize Error Error in list of partitions to test.t1 +test.t1 optimize status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -75368,7 +75416,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 REPAIR PARTITION part_1,part_2,part_5,part_6,part_10; Table Op Msg_type Msg_text -test.t1 repair error Error in list of partitions to test.t1 +test.t1 repair Error Error in list of partitions to test.t1 +test.t1 repair status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -75827,7 +75876,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 REPAIR PARTITION part_1,part_2,part_5,part_6,part_10; Table Op Msg_type Msg_text -test.t1 repair error Error in list of partitions to test.t1 +test.t1 repair Error Error in list of partitions to test.t1 +test.t1 repair status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -76297,7 +76347,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 REPAIR PARTITION part_1,part_2,part_5,part_6,part_10; Table Op Msg_type Msg_text -test.t1 repair error Error in list of partitions to test.t1 +test.t1 repair Error Error in list of partitions to test.t1 +test.t1 repair status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -76768,7 +76819,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 REPAIR PARTITION part_1,part_2,part_5,part_6,part_10; Table Op Msg_type Msg_text -test.t1 repair error Error in list of partitions to test.t1 +test.t1 repair Error Error in list of partitions to test.t1 +test.t1 repair status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -77233,7 +77285,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 REPAIR PARTITION part_1,part_2,part_5,part_6,part_10; Table Op Msg_type Msg_text -test.t1 repair error Error in list of partitions to test.t1 +test.t1 repair Error Error in list of partitions to test.t1 +test.t1 repair status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -77704,7 +77757,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 REPAIR PARTITION part_1,part_2,part_5,part_6,part_10; Table Op Msg_type Msg_text -test.t1 repair error Error in list of partitions to test.t1 +test.t1 repair Error Error in list of partitions to test.t1 +test.t1 repair status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -78180,7 +78234,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 REPAIR PARTITION part_1,part_2,part_5,part_6,part_10; Table Op Msg_type Msg_text -test.t1 repair error Error in list of partitions to test.t1 +test.t1 repair Error Error in list of partitions to test.t1 +test.t1 repair status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -78654,7 +78709,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 REPAIR PARTITION part_1,part_2,part_5,part_6,part_10; Table Op Msg_type Msg_text -test.t1 repair error Error in list of partitions to test.t1 +test.t1 repair Error Error in list of partitions to test.t1 +test.t1 repair status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -79118,7 +79174,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 REPAIR PARTITION part_1,part_1,part_1; Table Op Msg_type Msg_text -test.t1 repair error Error in list of partitions to test.t1 +test.t1 repair Error Error in list of partitions to test.t1 +test.t1 repair status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -79577,7 +79634,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 REPAIR PARTITION part_1,part_1,part_1; Table Op Msg_type Msg_text -test.t1 repair error Error in list of partitions to test.t1 +test.t1 repair Error Error in list of partitions to test.t1 +test.t1 repair status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -80047,7 +80105,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 REPAIR PARTITION part_1,part_1,part_1; Table Op Msg_type Msg_text -test.t1 repair error Error in list of partitions to test.t1 +test.t1 repair Error Error in list of partitions to test.t1 +test.t1 repair status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -80518,7 +80577,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 REPAIR PARTITION part_1,part_1,part_1; Table Op Msg_type Msg_text -test.t1 repair error Error in list of partitions to test.t1 +test.t1 repair Error Error in list of partitions to test.t1 +test.t1 repair status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -80983,7 +81043,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 REPAIR PARTITION part_1,part_1,part_1; Table Op Msg_type Msg_text -test.t1 repair error Error in list of partitions to test.t1 +test.t1 repair Error Error in list of partitions to test.t1 +test.t1 repair status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -81454,7 +81515,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 REPAIR PARTITION part_1,part_1,part_1; Table Op Msg_type Msg_text -test.t1 repair error Error in list of partitions to test.t1 +test.t1 repair Error Error in list of partitions to test.t1 +test.t1 repair status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -81930,7 +81992,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 REPAIR PARTITION part_1,part_1,part_1; Table Op Msg_type Msg_text -test.t1 repair error Error in list of partitions to test.t1 +test.t1 repair Error Error in list of partitions to test.t1 +test.t1 repair status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; @@ -82404,7 +82467,8 @@ SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN 1 AND @max_row_div2 - 1; ALTER TABLE t1 REPAIR PARTITION part_1,part_1,part_1; Table Op Msg_type Msg_text -test.t1 repair error Error in list of partitions to test.t1 +test.t1 repair Error Error in list of partitions to test.t1 +test.t1 repair status Operation failed INSERT INTO t1(f_int1,f_int2,f_char1,f_char2,f_charbig) SELECT f_int1,f_int2,f_char1,f_char2,f_charbig FROM t0_template WHERE f_int1 BETWEEN @max_row_div2 AND @max_row; diff --git a/mysql-test/suite/tokudb.parts/r/partition_auto_increment_tokudb.result b/mysql-test/suite/tokudb.parts/r/partition_auto_increment_tokudb.result index a40493a18f5..b4ae303adff 100644 --- a/mysql-test/suite/tokudb.parts/r/partition_auto_increment_tokudb.result +++ b/mysql-test/suite/tokudb.parts/r/partition_auto_increment_tokudb.result @@ -1,3 +1,6 @@ +SET @tokudb_prelock_empty_saved = @@GLOBAL.tokudb_prelock_empty; +SET GLOBAL tokudb_prelock_empty = 0; +SET SESSION tokudb_prelock_empty = 0; DROP TABLE IF EXISTS t1; # test without partitioning for reference CREATE TABLE t1 ( @@ -1113,3 +1116,4 @@ a 0 DROP TABLE t1; ############################################################################## +SET GLOBAL tokudb_prelock_empty = @tokudb_prelock_empty_saved; diff --git a/mysql-test/suite/tokudb.parts/t/disabled.def b/mysql-test/suite/tokudb.parts/t/disabled.def index 90e599cd035..68d7693612f 100644 --- a/mysql-test/suite/tokudb.parts/t/disabled.def +++ b/mysql-test/suite/tokudb.parts/t/disabled.def @@ -1,2 +1,4 @@ partition_basic_symlink_tokudb : tokudb_file_per_table is not supported partition_reorganize_tokudb : tokudb_file_per_table is not supported +partition_mgm_lc0_tokudb : https://tokutek.atlassian.net/browse/DB-637 +partition_mgm_lc1_tokudb : https://tokutek.atlassian.net/browse/DB-637 diff --git a/mysql-test/suite/tokudb.parts/t/partition_auto_increment_tokudb-master.opt b/mysql-test/suite/tokudb.parts/t/partition_auto_increment_tokudb-master.opt deleted file mode 100644 index 857da664d10..00000000000 --- a/mysql-test/suite/tokudb.parts/t/partition_auto_increment_tokudb-master.opt +++ /dev/null @@ -1 +0,0 @@ ---tokudb-prelock-empty=0 diff --git a/mysql-test/suite/tokudb.parts/t/partition_auto_increment_tokudb.test b/mysql-test/suite/tokudb.parts/t/partition_auto_increment_tokudb.test index a13ddf65d4b..ceec2fda9ee 100644 --- a/mysql-test/suite/tokudb.parts/t/partition_auto_increment_tokudb.test +++ b/mysql-test/suite/tokudb.parts/t/partition_auto_increment_tokudb.test @@ -29,7 +29,12 @@ let $engine= 'TokuDB'; --source include/have_tokudb.inc +SET @tokudb_prelock_empty_saved = @@GLOBAL.tokudb_prelock_empty; +SET GLOBAL tokudb_prelock_empty = 0; +SET SESSION tokudb_prelock_empty = 0; + #------------------------------------------------------------------------------# # Execute the tests to be applied to all storage engines --source suite/parts/inc/partition_auto_increment.inc +SET GLOBAL tokudb_prelock_empty = @tokudb_prelock_empty_saved; diff --git a/mysql-test/suite/tokudb.parts/t/suite.opt b/mysql-test/suite/tokudb.parts/t/suite.opt new file mode 100644 index 00000000000..23511b05020 --- /dev/null +++ b/mysql-test/suite/tokudb.parts/t/suite.opt @@ -0,0 +1 @@ +$TOKUDB_OPT $TOKUDB_LOAD_ADD --loose-tokudb-check-jemalloc=0 diff --git a/mysql-test/suite/tokudb.rpl/combinations b/mysql-test/suite/tokudb.rpl/combinations new file mode 100644 index 00000000000..07042c2cbec --- /dev/null +++ b/mysql-test/suite/tokudb.rpl/combinations @@ -0,0 +1,8 @@ +[row] +binlog-format=row + +[stmt] +binlog-format=statement + +[mix] +binlog-format=mixed diff --git a/mysql-test/suite/tokudb.rpl/r/rpl_parallel_tokudb_delete_pk.result b/mysql-test/suite/tokudb.rpl/r/rpl_parallel_tokudb_delete_pk.result index 3021675bbda..afbc4b50da8 100644 --- a/mysql-test/suite/tokudb.rpl/r/rpl_parallel_tokudb_delete_pk.result +++ b/mysql-test/suite/tokudb.rpl/r/rpl_parallel_tokudb_delete_pk.result @@ -6,6 +6,13 @@ Warnings: Note 1753 slave_transaction_retries is not supported in multi-threaded slave mode. In the event of a transient failure, the slave will not retry the transaction and will stop. [connection master] drop table if exists t; +show variables like 'tokudb_rpl_%'; +Variable_name Value +tokudb_rpl_check_readonly ON +tokudb_rpl_lookup_rows OFF +tokudb_rpl_lookup_rows_delay 10000 +tokudb_rpl_unique_checks OFF +tokudb_rpl_unique_checks_delay 10000 create table t (a bigint not null, primary key(a)) engine=tokudb; insert into t values (1); insert into t values (2),(3); diff --git a/mysql-test/suite/tokudb.rpl/r/rpl_parallel_tokudb_update_pk_uc0_lookup0.result b/mysql-test/suite/tokudb.rpl/r/rpl_parallel_tokudb_update_pk_uc0_lookup0.result index 39a6856d684..7aab8947940 100644 --- a/mysql-test/suite/tokudb.rpl/r/rpl_parallel_tokudb_update_pk_uc0_lookup0.result +++ b/mysql-test/suite/tokudb.rpl/r/rpl_parallel_tokudb_update_pk_uc0_lookup0.result @@ -6,6 +6,13 @@ Warnings: Note 1753 slave_transaction_retries is not supported in multi-threaded slave mode. In the event of a transient failure, the slave will not retry the transaction and will stop. [connection master] drop table if exists t; +show variables like 'tokudb_rpl_%'; +Variable_name Value +tokudb_rpl_check_readonly ON +tokudb_rpl_lookup_rows OFF +tokudb_rpl_lookup_rows_delay 10000 +tokudb_rpl_unique_checks OFF +tokudb_rpl_unique_checks_delay 10000 create table t (a bigint not null, b bigint not null, primary key(a)) engine=tokudb; insert into t values (1,0); insert into t values (2,0),(3,0); diff --git a/mysql-test/suite/tokudb.rpl/r/rpl_parallel_tokudb_write_pk.result b/mysql-test/suite/tokudb.rpl/r/rpl_parallel_tokudb_write_pk.result index 314ccaedaf7..64b495350c2 100644 --- a/mysql-test/suite/tokudb.rpl/r/rpl_parallel_tokudb_write_pk.result +++ b/mysql-test/suite/tokudb.rpl/r/rpl_parallel_tokudb_write_pk.result @@ -6,6 +6,10 @@ Warnings: Note 1753 slave_transaction_retries is not supported in multi-threaded slave mode. In the event of a transient failure, the slave will not retry the transaction and will stop. [connection master] drop table if exists t; +show variables like 'tokudb_rpl_unique_checks%'; +Variable_name Value +tokudb_rpl_unique_checks OFF +tokudb_rpl_unique_checks_delay 5000 create table t (a bigint not null, primary key(a)) engine=tokudb; select unix_timestamp() into @tstart; insert into t values (1); diff --git a/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_delete_pk.result b/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_delete_pk.result index c77cbbc71c9..6fab29177d7 100644 --- a/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_delete_pk.result +++ b/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_delete_pk.result @@ -4,6 +4,13 @@ Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. [connection master] drop table if exists t; +show variables like 'tokudb_rpl_%'; +Variable_name Value +tokudb_rpl_check_readonly ON +tokudb_rpl_lookup_rows OFF +tokudb_rpl_lookup_rows_delay 10000 +tokudb_rpl_unique_checks OFF +tokudb_rpl_unique_checks_delay 10000 create table t (a bigint not null, primary key(a)) engine=tokudb; insert into t values (1); insert into t values (2),(3); diff --git a/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_delete_pk_lookup1.result b/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_delete_pk_lookup1.result index e178b8ad137..f8efd5e04ee 100644 --- a/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_delete_pk_lookup1.result +++ b/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_delete_pk_lookup1.result @@ -4,6 +4,13 @@ Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. [connection master] drop table if exists t; +show variables like 'tokudb_rpl_%'; +Variable_name Value +tokudb_rpl_check_readonly ON +tokudb_rpl_lookup_rows ON +tokudb_rpl_lookup_rows_delay 10000 +tokudb_rpl_unique_checks ON +tokudb_rpl_unique_checks_delay 0 create table t (a bigint not null, primary key(a)) engine=tokudb; insert into t values (1); insert into t values (2),(3); diff --git a/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_mixed_dml.result b/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_mixed_dml.result index 9300d29bb29..c11ae61b3ca 100644 --- a/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_mixed_dml.result +++ b/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_mixed_dml.result @@ -1,3 +1,4 @@ +SET SESSION tokudb_pk_insert_mode = 2; include/master-slave.inc Warnings: Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. diff --git a/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_read_only_ff.result b/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_read_only_ff.result index 5926cdda565..76a5e31b969 100644 --- a/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_read_only_ff.result +++ b/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_read_only_ff.result @@ -4,6 +4,13 @@ Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. [connection master] drop table if exists t; +show variables like 'tokudb_rpl%'; +Variable_name Value +tokudb_rpl_check_readonly OFF +tokudb_rpl_lookup_rows ON +tokudb_rpl_lookup_rows_delay 0 +tokudb_rpl_unique_checks OFF +tokudb_rpl_unique_checks_delay 5000 create table t (a bigint not null, primary key(a)) engine=tokudb; select unix_timestamp() into @tstart; insert into t values (1); diff --git a/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_read_only_ft.result b/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_read_only_ft.result index 0b5ac77481f..2930cb019b4 100644 --- a/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_read_only_ft.result +++ b/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_read_only_ft.result @@ -4,6 +4,13 @@ Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. [connection master] drop table if exists t; +show variables like 'tokudb_rpl%'; +Variable_name Value +tokudb_rpl_check_readonly ON +tokudb_rpl_lookup_rows ON +tokudb_rpl_lookup_rows_delay 0 +tokudb_rpl_unique_checks OFF +tokudb_rpl_unique_checks_delay 5000 create table t (a bigint not null, primary key(a)) engine=tokudb; select unix_timestamp() into @tstart; insert into t values (1); diff --git a/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_read_only_tf.result b/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_read_only_tf.result index 5926cdda565..76a5e31b969 100644 --- a/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_read_only_tf.result +++ b/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_read_only_tf.result @@ -4,6 +4,13 @@ Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. [connection master] drop table if exists t; +show variables like 'tokudb_rpl%'; +Variable_name Value +tokudb_rpl_check_readonly OFF +tokudb_rpl_lookup_rows ON +tokudb_rpl_lookup_rows_delay 0 +tokudb_rpl_unique_checks OFF +tokudb_rpl_unique_checks_delay 5000 create table t (a bigint not null, primary key(a)) engine=tokudb; select unix_timestamp() into @tstart; insert into t values (1); diff --git a/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_read_only_tt.result b/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_read_only_tt.result index 5926cdda565..0cba2a1cddb 100644 --- a/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_read_only_tt.result +++ b/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_read_only_tt.result @@ -4,6 +4,13 @@ Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. [connection master] drop table if exists t; +show variables like 'tokudb_rpl%'; +Variable_name Value +tokudb_rpl_check_readonly ON +tokudb_rpl_lookup_rows ON +tokudb_rpl_lookup_rows_delay 0 +tokudb_rpl_unique_checks OFF +tokudb_rpl_unique_checks_delay 5000 create table t (a bigint not null, primary key(a)) engine=tokudb; select unix_timestamp() into @tstart; insert into t values (1); diff --git a/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_update_pk_uc0_lookup0.result b/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_update_pk_uc0_lookup0.result index 162655f9896..50f43ebe5bf 100644 --- a/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_update_pk_uc0_lookup0.result +++ b/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_update_pk_uc0_lookup0.result @@ -4,6 +4,13 @@ Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. [connection master] drop table if exists t; +show variables like 'tokudb_rpl_%'; +Variable_name Value +tokudb_rpl_check_readonly ON +tokudb_rpl_lookup_rows OFF +tokudb_rpl_lookup_rows_delay 10000 +tokudb_rpl_unique_checks OFF +tokudb_rpl_unique_checks_delay 10000 create table t (a bigint not null, b bigint not null, primary key(a)) engine=tokudb; insert into t values (1,0); insert into t values (2,0),(3,0); diff --git a/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_update_pk_uc0_lookup1.result b/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_update_pk_uc0_lookup1.result index 3c909718426..9e7f932a3c9 100644 --- a/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_update_pk_uc0_lookup1.result +++ b/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_update_pk_uc0_lookup1.result @@ -4,6 +4,13 @@ Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. [connection master] drop table if exists t; +show variables like 'tokudb_rpl_%'; +Variable_name Value +tokudb_rpl_check_readonly ON +tokudb_rpl_lookup_rows ON +tokudb_rpl_lookup_rows_delay 10000 +tokudb_rpl_unique_checks OFF +tokudb_rpl_unique_checks_delay 10000 create table t (a bigint not null, b bigint not null, primary key(a)) engine=tokudb; insert into t values (1,0); insert into t values (2,0),(3,0); diff --git a/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_update_pk_uc1_lookup0.result b/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_update_pk_uc1_lookup0.result index 3c909718426..348734b206d 100644 --- a/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_update_pk_uc1_lookup0.result +++ b/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_update_pk_uc1_lookup0.result @@ -4,6 +4,13 @@ Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. [connection master] drop table if exists t; +show variables like 'tokudb_rpl_%'; +Variable_name Value +tokudb_rpl_check_readonly ON +tokudb_rpl_lookup_rows OFF +tokudb_rpl_lookup_rows_delay 10000 +tokudb_rpl_unique_checks ON +tokudb_rpl_unique_checks_delay 10000 create table t (a bigint not null, b bigint not null, primary key(a)) engine=tokudb; insert into t values (1,0); insert into t values (2,0),(3,0); diff --git a/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_update_pk_uc1_lookup1.result b/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_update_pk_uc1_lookup1.result index 3c909718426..bfd640e52e9 100644 --- a/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_update_pk_uc1_lookup1.result +++ b/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_update_pk_uc1_lookup1.result @@ -4,6 +4,13 @@ Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. [connection master] drop table if exists t; +show variables like 'tokudb_rpl_%'; +Variable_name Value +tokudb_rpl_check_readonly ON +tokudb_rpl_lookup_rows ON +tokudb_rpl_lookup_rows_delay 10000 +tokudb_rpl_unique_checks ON +tokudb_rpl_unique_checks_delay 10000 create table t (a bigint not null, b bigint not null, primary key(a)) engine=tokudb; insert into t values (1,0); insert into t values (2,0),(3,0); diff --git a/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_update_unique_uc0_lookup0.result b/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_update_unique_uc0_lookup0.result index 941010071ae..faf969f851a 100644 --- a/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_update_unique_uc0_lookup0.result +++ b/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_update_unique_uc0_lookup0.result @@ -4,6 +4,13 @@ Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. [connection master] drop table if exists t; +show variables like 'tokudb_rpl_%'; +Variable_name Value +tokudb_rpl_check_readonly ON +tokudb_rpl_lookup_rows OFF +tokudb_rpl_lookup_rows_delay 10000 +tokudb_rpl_unique_checks OFF +tokudb_rpl_unique_checks_delay 10000 create table t (a bigint not null, b bigint not null, c bigint not null, primary key(a), unique key(c)) engine=tokudb; insert into t values (1,0,-1); insert into t values (2,0,-2),(3,0,-3); diff --git a/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_update_unique_uc0_lookup1.result b/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_update_unique_uc0_lookup1.result index 6a0b1126710..9ac87512f80 100644 --- a/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_update_unique_uc0_lookup1.result +++ b/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_update_unique_uc0_lookup1.result @@ -4,6 +4,13 @@ Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. [connection master] drop table if exists t; +show variables like 'tokudb_rpl_%'; +Variable_name Value +tokudb_rpl_check_readonly ON +tokudb_rpl_lookup_rows ON +tokudb_rpl_lookup_rows_delay 10000 +tokudb_rpl_unique_checks OFF +tokudb_rpl_unique_checks_delay 10000 create table t (a bigint not null, b bigint not null, c bigint not null, primary key(a), unique key(c)) engine=tokudb; insert into t values (1,0,-1); insert into t values (2,0,-2),(3,0,-3); diff --git a/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_write_pk.result b/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_write_pk.result index 5926cdda565..ea1b84f8138 100644 --- a/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_write_pk.result +++ b/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_write_pk.result @@ -4,6 +4,10 @@ Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. [connection master] drop table if exists t; +show variables like 'tokudb_rpl_unique_checks%'; +Variable_name Value +tokudb_rpl_unique_checks OFF +tokudb_rpl_unique_checks_delay 5000 create table t (a bigint not null, primary key(a)) engine=tokudb; select unix_timestamp() into @tstart; insert into t values (1); diff --git a/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_write_pk_uc1.result b/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_write_pk_uc1.result index 0b5ac77481f..c846ec81612 100644 --- a/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_write_pk_uc1.result +++ b/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_write_pk_uc1.result @@ -4,6 +4,10 @@ Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. [connection master] drop table if exists t; +show variables like 'tokudb_rpl_unique_checks%'; +Variable_name Value +tokudb_rpl_unique_checks ON +tokudb_rpl_unique_checks_delay 10000 create table t (a bigint not null, primary key(a)) engine=tokudb; select unix_timestamp() into @tstart; insert into t values (1); diff --git a/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_write_unique.result b/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_write_unique.result index 60ffc0d0530..808303387d5 100644 --- a/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_write_unique.result +++ b/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_write_unique.result @@ -4,6 +4,10 @@ Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. [connection master] drop table if exists t; +show variables like 'tokudb_rpl_unique_checks%'; +Variable_name Value +tokudb_rpl_unique_checks OFF +tokudb_rpl_unique_checks_delay 5000 create table t (a bigint not null, b bigint not null, primary key(a), unique key(b)) engine=tokudb; select unix_timestamp() into @tstart; insert into t values (1,2); diff --git a/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_write_unique_uc1.result b/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_write_unique_uc1.result index 2f2cf58032e..a40548cec62 100644 --- a/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_write_unique_uc1.result +++ b/mysql-test/suite/tokudb.rpl/r/rpl_tokudb_write_unique_uc1.result @@ -4,6 +4,10 @@ Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. [connection master] drop table if exists t; +show variables like 'tokudb_rpl_unique_checks%'; +Variable_name Value +tokudb_rpl_unique_checks ON +tokudb_rpl_unique_checks_delay 5000 create table t (a bigint not null, b bigint not null, primary key(a), unique key(b)) engine=tokudb; select unix_timestamp() into @tstart; insert into t values (1,2); diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_parallel_tokudb.test b/mysql-test/suite/tokudb.rpl/t/rpl_parallel_tokudb.test index efb60fac2ba..23d2d6cdf51 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_parallel_tokudb.test +++ b/mysql-test/suite/tokudb.rpl/t/rpl_parallel_tokudb.test @@ -1,3 +1,4 @@ +--source include/have_tokudb.inc #Want to skip this test from daily Valgrind execution --source include/no_valgrind_without_big.inc # diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_parallel_tokudb_delete_pk-slave.opt b/mysql-test/suite/tokudb.rpl/t/rpl_parallel_tokudb_delete_pk-slave.opt index 2c35be69b98..042156e63d1 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_parallel_tokudb_delete_pk-slave.opt +++ b/mysql-test/suite/tokudb.rpl/t/rpl_parallel_tokudb_delete_pk-slave.opt @@ -1,2 +1,2 @@ ---read-only=ON --tokudb-rpl-unique-checks-delay=10000 --tokudb-rpl-unique-checks=OFF --tokudb-rpl-lookup-rows-delay=10000 --tokudb-rpl-lookup-rows=OFF --slave-parallel-workers=2 +--read-only=ON --loose-tokudb-rpl-unique-checks-delay=10000 --loose-tokudb-rpl-unique-checks=OFF --loose-tokudb-rpl-lookup-rows-delay=10000 --loose-tokudb-rpl-lookup-rows=OFF --slave-parallel-workers=2 diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_parallel_tokudb_delete_pk.test b/mysql-test/suite/tokudb.rpl/t/rpl_parallel_tokudb_delete_pk.test index fb42f40bb62..bedeb9513be 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_parallel_tokudb_delete_pk.test +++ b/mysql-test/suite/tokudb.rpl/t/rpl_parallel_tokudb_delete_pk.test @@ -14,7 +14,7 @@ enable_warnings; connection slave; # show variables like 'read_only'; -# show variables like 'tokudb_rpl_%'; +show variables like 'tokudb_rpl_%'; # insert some rows connection master; diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_parallel_tokudb_update_pk_uc0_lookup0-slave.opt b/mysql-test/suite/tokudb.rpl/t/rpl_parallel_tokudb_update_pk_uc0_lookup0-slave.opt index d84d7111bd6..fffefff5bcd 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_parallel_tokudb_update_pk_uc0_lookup0-slave.opt +++ b/mysql-test/suite/tokudb.rpl/t/rpl_parallel_tokudb_update_pk_uc0_lookup0-slave.opt @@ -1 +1 @@ ---read-only=ON --tokudb-rpl-unique-checks-delay=10000 --tokudb-rpl-unique-checks=OFF --tokudb-rpl-lookup-rows-delay=10000 --tokudb-rpl-lookup-rows=OFF --slave-parallel-workers=2 +--read-only=ON --loose-tokudb-rpl-unique-checks-delay=10000 --loose-tokudb-rpl-unique-checks=OFF --loose-tokudb-rpl-lookup-rows-delay=10000 --loose-tokudb-rpl-lookup-rows=OFF --slave-parallel-workers=2 diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_parallel_tokudb_update_pk_uc0_lookup0.test b/mysql-test/suite/tokudb.rpl/t/rpl_parallel_tokudb_update_pk_uc0_lookup0.test index 998987349c7..6dd9b660eed 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_parallel_tokudb_update_pk_uc0_lookup0.test +++ b/mysql-test/suite/tokudb.rpl/t/rpl_parallel_tokudb_update_pk_uc0_lookup0.test @@ -13,7 +13,7 @@ enable_warnings; connection slave; # show variables like 'read_only'; -# show variables like 'tokudb_rpl_%'; +show variables like 'tokudb_rpl_%'; # insert some rows connection master; diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_parallel_tokudb_write_pk-slave.opt b/mysql-test/suite/tokudb.rpl/t/rpl_parallel_tokudb_write_pk-slave.opt index 9e4f2f7048d..f2ed0286136 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_parallel_tokudb_write_pk-slave.opt +++ b/mysql-test/suite/tokudb.rpl/t/rpl_parallel_tokudb_write_pk-slave.opt @@ -1,2 +1,2 @@ ---read-only=ON --tokudb-rpl-unique-checks-delay=5000 --tokudb-rpl-unique-checks=OFF --slave-parallel-workers=2 +--read-only=ON --loose-tokudb-rpl-unique-checks-delay=5000 --loose-tokudb-rpl-unique-checks=OFF --slave-parallel-workers=2 diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_parallel_tokudb_write_pk.test b/mysql-test/suite/tokudb.rpl/t/rpl_parallel_tokudb_write_pk.test index c77e4b49605..0ed12b34e1f 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_parallel_tokudb_write_pk.test +++ b/mysql-test/suite/tokudb.rpl/t/rpl_parallel_tokudb_write_pk.test @@ -14,7 +14,7 @@ enable_warnings; connection slave; # show variables like 'read_only'; -# show variables like 'tokudb_rpl_unique_checks%'; +show variables like 'tokudb_rpl_unique_checks%'; # insert some rows connection master; diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_delete_pk-slave.opt b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_delete_pk-slave.opt index dc139282dc4..93a2685e847 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_delete_pk-slave.opt +++ b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_delete_pk-slave.opt @@ -1 +1 @@ ---read-only=ON --tokudb-rpl-unique-checks-delay=10000 --tokudb-rpl-unique-checks=OFF --tokudb-rpl-lookup-rows-delay=10000 --tokudb-rpl-lookup-rows=OFF +--read-only=ON --loose-tokudb-rpl-unique-checks-delay=10000 --loose-tokudb-rpl-unique-checks=OFF --loose-tokudb-rpl-lookup-rows-delay=10000 --loose-tokudb-rpl-lookup-rows=OFF diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_delete_pk.test b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_delete_pk.test index fb42f40bb62..bedeb9513be 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_delete_pk.test +++ b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_delete_pk.test @@ -14,7 +14,7 @@ enable_warnings; connection slave; # show variables like 'read_only'; -# show variables like 'tokudb_rpl_%'; +show variables like 'tokudb_rpl_%'; # insert some rows connection master; diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_delete_pk_lookup1-slave.opt b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_delete_pk_lookup1-slave.opt index 4675b07763d..9a2fec628f9 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_delete_pk_lookup1-slave.opt +++ b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_delete_pk_lookup1-slave.opt @@ -1 +1 @@ ---read-only=ON --tokudb-rpl-unique-checks-delay=0 --tokudb-rpl-unique-checks=ON --tokudb-rpl-lookup-rows-delay=10000 --tokudb-rpl-lookup-rows=ON +--read-only=ON --loose-tokudb-rpl-unique-checks-delay=0 --loose-tokudb-rpl-unique-checks=ON --loose-tokudb-rpl-lookup-rows-delay=10000 --loose-tokudb-rpl-lookup-rows=ON diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_delete_pk_lookup1.test b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_delete_pk_lookup1.test index bf5edbd2c1b..9e9aaede416 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_delete_pk_lookup1.test +++ b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_delete_pk_lookup1.test @@ -15,7 +15,7 @@ enable_warnings; connection slave; # show variables like 'read_only'; -# show variables like 'tokudb_rpl_%'; +show variables like 'tokudb_rpl_%'; # insert some rows connection master; diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_mixed_dml-master.opt b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_mixed_dml-master.opt deleted file mode 100644 index fd3de58d816..00000000000 --- a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_mixed_dml-master.opt +++ /dev/null @@ -1,2 +0,0 @@ ---tokudb_pk_insert_mode=2 - diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_mixed_dml.test b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_mixed_dml.test index c27625abd7f..6147d321784 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_mixed_dml.test +++ b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_mixed_dml.test @@ -6,4 +6,7 @@ --source include/have_binlog_format_mixed.inc --source include/have_tokudb.inc let $engine_type=TokuDB; + +SET SESSION tokudb_pk_insert_mode = 2; + --source suite/rpl/include/rpl_mixed_dml.inc diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_read_only_ff-slave.opt b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_read_only_ff-slave.opt index b9eb687d8d5..0240c9d6ae4 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_read_only_ff-slave.opt +++ b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_read_only_ff-slave.opt @@ -1 +1 @@ ---read-only=OFF --tokudb-rpl-check-readonly=OFF --tokudb-rpl-unique-checks-delay=5000 --tokudb-rpl-unique-checks=OFF +--read-only=OFF --loose-tokudb-rpl-check-readonly=OFF --loose-tokudb-rpl-unique-checks-delay=5000 --loose-tokudb-rpl-unique-checks=OFF diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_read_only_ff.test b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_read_only_ff.test index c77e4b49605..7b3e8f0c0d3 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_read_only_ff.test +++ b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_read_only_ff.test @@ -14,7 +14,7 @@ enable_warnings; connection slave; # show variables like 'read_only'; -# show variables like 'tokudb_rpl_unique_checks%'; +show variables like 'tokudb_rpl%'; # insert some rows connection master; diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_read_only_ft-slave.opt b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_read_only_ft-slave.opt index 8283875e8a7..e1ee193d4c1 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_read_only_ft-slave.opt +++ b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_read_only_ft-slave.opt @@ -1 +1 @@ ---read-only=OFF --tokudb-rpl-check-readonly=ON --tokudb-rpl-unique-checks-delay=5000 --tokudb-rpl-unique-checks=OFF +--read-only=OFF --loose-tokudb-rpl-check-readonly=ON --loose-tokudb-rpl-unique-checks-delay=5000 --loose-tokudb-rpl-unique-checks=OFF diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_read_only_ft.test b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_read_only_ft.test index c77e4b49605..7b3e8f0c0d3 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_read_only_ft.test +++ b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_read_only_ft.test @@ -14,7 +14,7 @@ enable_warnings; connection slave; # show variables like 'read_only'; -# show variables like 'tokudb_rpl_unique_checks%'; +show variables like 'tokudb_rpl%'; # insert some rows connection master; diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_read_only_tf-slave.opt b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_read_only_tf-slave.opt index 21e57d27c17..e60afd0380a 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_read_only_tf-slave.opt +++ b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_read_only_tf-slave.opt @@ -1 +1 @@ ---read-only=ON --tokudb-rpl-check-readonly=OFF --tokudb-rpl-unique-checks-delay=5000 --tokudb-rpl-unique-checks=OFF +--read-only=ON --loose-tokudb-rpl-check-readonly=OFF --loose-tokudb-rpl-unique-checks-delay=5000 --loose-tokudb-rpl-unique-checks=OFF diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_read_only_tf.test b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_read_only_tf.test index c77e4b49605..7b3e8f0c0d3 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_read_only_tf.test +++ b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_read_only_tf.test @@ -14,7 +14,7 @@ enable_warnings; connection slave; # show variables like 'read_only'; -# show variables like 'tokudb_rpl_unique_checks%'; +show variables like 'tokudb_rpl%'; # insert some rows connection master; diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_read_only_tt-slave.opt b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_read_only_tt-slave.opt index fd77ee0da9c..f6658646e65 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_read_only_tt-slave.opt +++ b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_read_only_tt-slave.opt @@ -1 +1 @@ ---read-only=ON --tokudb-rpl-check-readonly=ON --tokudb-rpl-unique-checks-delay=5000 --tokudb-rpl-unique-checks=OFF +--read-only=ON --loose-tokudb-rpl-check-readonly=ON --loose-tokudb-rpl-unique-checks-delay=5000 --loose-tokudb-rpl-unique-checks=OFF diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_read_only_tt.test b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_read_only_tt.test index c77e4b49605..7b3e8f0c0d3 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_read_only_tt.test +++ b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_read_only_tt.test @@ -14,7 +14,7 @@ enable_warnings; connection slave; # show variables like 'read_only'; -# show variables like 'tokudb_rpl_unique_checks%'; +show variables like 'tokudb_rpl%'; # insert some rows connection master; diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_pk_uc0_lookup0-slave.opt b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_pk_uc0_lookup0-slave.opt index dc139282dc4..93a2685e847 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_pk_uc0_lookup0-slave.opt +++ b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_pk_uc0_lookup0-slave.opt @@ -1 +1 @@ ---read-only=ON --tokudb-rpl-unique-checks-delay=10000 --tokudb-rpl-unique-checks=OFF --tokudb-rpl-lookup-rows-delay=10000 --tokudb-rpl-lookup-rows=OFF +--read-only=ON --loose-tokudb-rpl-unique-checks-delay=10000 --loose-tokudb-rpl-unique-checks=OFF --loose-tokudb-rpl-lookup-rows-delay=10000 --loose-tokudb-rpl-lookup-rows=OFF diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_pk_uc0_lookup0.test b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_pk_uc0_lookup0.test index 998987349c7..6dd9b660eed 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_pk_uc0_lookup0.test +++ b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_pk_uc0_lookup0.test @@ -13,7 +13,7 @@ enable_warnings; connection slave; # show variables like 'read_only'; -# show variables like 'tokudb_rpl_%'; +show variables like 'tokudb_rpl_%'; # insert some rows connection master; diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_pk_uc0_lookup1-slave.opt b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_pk_uc0_lookup1-slave.opt index d546dd00669..a4ca1104425 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_pk_uc0_lookup1-slave.opt +++ b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_pk_uc0_lookup1-slave.opt @@ -1 +1 @@ ---read-only=ON --tokudb-rpl-unique-checks-delay=10000 --tokudb-rpl-unique-checks=OFF --tokudb-rpl-lookup-rows-delay=10000 --tokudb-rpl-lookup-rows=ON +--read-only=ON --loose-tokudb-rpl-unique-checks-delay=10000 --loose-tokudb-rpl-unique-checks=OFF --loose-tokudb-rpl-lookup-rows-delay=10000 --loose-tokudb-rpl-lookup-rows=ON diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_pk_uc0_lookup1.test b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_pk_uc0_lookup1.test index 998987349c7..6dd9b660eed 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_pk_uc0_lookup1.test +++ b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_pk_uc0_lookup1.test @@ -13,7 +13,7 @@ enable_warnings; connection slave; # show variables like 'read_only'; -# show variables like 'tokudb_rpl_%'; +show variables like 'tokudb_rpl_%'; # insert some rows connection master; diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_pk_uc1_lookup0-slave.opt b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_pk_uc1_lookup0-slave.opt index 5cfe5f83a91..4b1d21daf5f 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_pk_uc1_lookup0-slave.opt +++ b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_pk_uc1_lookup0-slave.opt @@ -1 +1 @@ ---read-only=ON --tokudb-rpl-unique-checks-delay=10000 --tokudb-rpl-unique-checks=ON --tokudb-rpl-lookup-rows-delay=10000 --tokudb-rpl-lookup-rows=OFF +--read-only=ON --loose-tokudb-rpl-unique-checks-delay=10000 --loose-tokudb-rpl-unique-checks=ON --loose-tokudb-rpl-lookup-rows-delay=10000 --loose-tokudb-rpl-lookup-rows=OFF diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_pk_uc1_lookup0.test b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_pk_uc1_lookup0.test index 998987349c7..6dd9b660eed 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_pk_uc1_lookup0.test +++ b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_pk_uc1_lookup0.test @@ -13,7 +13,7 @@ enable_warnings; connection slave; # show variables like 'read_only'; -# show variables like 'tokudb_rpl_%'; +show variables like 'tokudb_rpl_%'; # insert some rows connection master; diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_pk_uc1_lookup1-slave.opt b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_pk_uc1_lookup1-slave.opt index 7cd575c52bb..239e19ac040 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_pk_uc1_lookup1-slave.opt +++ b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_pk_uc1_lookup1-slave.opt @@ -1 +1 @@ ---read-only=ON --tokudb-rpl-unique-checks-delay=10000 --tokudb-rpl-unique-checks=ON --tokudb-rpl-lookup-rows-delay=10000 --tokudb-rpl-lookup-rows=ON +--read-only=ON --loose-tokudb-rpl-unique-checks-delay=10000 --loose-tokudb-rpl-unique-checks=ON --loose-tokudb-rpl-lookup-rows-delay=10000 --loose-tokudb-rpl-lookup-rows=ON diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_pk_uc1_lookup1.test b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_pk_uc1_lookup1.test index 998987349c7..6dd9b660eed 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_pk_uc1_lookup1.test +++ b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_pk_uc1_lookup1.test @@ -13,7 +13,7 @@ enable_warnings; connection slave; # show variables like 'read_only'; -# show variables like 'tokudb_rpl_%'; +show variables like 'tokudb_rpl_%'; # insert some rows connection master; diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_unique_uc0_lookup0-slave.opt b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_unique_uc0_lookup0-slave.opt index dc139282dc4..93a2685e847 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_unique_uc0_lookup0-slave.opt +++ b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_unique_uc0_lookup0-slave.opt @@ -1 +1 @@ ---read-only=ON --tokudb-rpl-unique-checks-delay=10000 --tokudb-rpl-unique-checks=OFF --tokudb-rpl-lookup-rows-delay=10000 --tokudb-rpl-lookup-rows=OFF +--read-only=ON --loose-tokudb-rpl-unique-checks-delay=10000 --loose-tokudb-rpl-unique-checks=OFF --loose-tokudb-rpl-lookup-rows-delay=10000 --loose-tokudb-rpl-lookup-rows=OFF diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_unique_uc0_lookup0.test b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_unique_uc0_lookup0.test index 11401ac0ce0..93fef3699d9 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_unique_uc0_lookup0.test +++ b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_unique_uc0_lookup0.test @@ -13,7 +13,7 @@ enable_warnings; connection slave; # show variables like 'read_only'; -# show variables like 'tokudb_rpl_%'; +show variables like 'tokudb_rpl_%'; # insert some rows connection master; diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_unique_uc0_lookup1-slave.opt b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_unique_uc0_lookup1-slave.opt index d546dd00669..a4ca1104425 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_unique_uc0_lookup1-slave.opt +++ b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_unique_uc0_lookup1-slave.opt @@ -1 +1 @@ ---read-only=ON --tokudb-rpl-unique-checks-delay=10000 --tokudb-rpl-unique-checks=OFF --tokudb-rpl-lookup-rows-delay=10000 --tokudb-rpl-lookup-rows=ON +--read-only=ON --loose-tokudb-rpl-unique-checks-delay=10000 --loose-tokudb-rpl-unique-checks=OFF --loose-tokudb-rpl-lookup-rows-delay=10000 --loose-tokudb-rpl-lookup-rows=ON diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_unique_uc0_lookup1.test b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_unique_uc0_lookup1.test index ea77447bc75..c8976db8ccd 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_unique_uc0_lookup1.test +++ b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_update_unique_uc0_lookup1.test @@ -13,7 +13,7 @@ enable_warnings; connection slave; # show variables like 'read_only'; -# show variables like 'tokudb_rpl_%'; +show variables like 'tokudb_rpl_%'; # insert some rows connection master; diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_write_pk-slave.opt b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_write_pk-slave.opt index 9baf0d65ecf..19b40f86454 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_write_pk-slave.opt +++ b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_write_pk-slave.opt @@ -1 +1 @@ ---read-only=ON --tokudb-rpl-unique-checks-delay=5000 --tokudb-rpl-unique-checks=OFF +--read-only=ON --loose-tokudb-rpl-unique-checks-delay=5000 --loose-tokudb-rpl-unique-checks=OFF diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_write_pk.test b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_write_pk.test index c77e4b49605..0ed12b34e1f 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_write_pk.test +++ b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_write_pk.test @@ -14,7 +14,7 @@ enable_warnings; connection slave; # show variables like 'read_only'; -# show variables like 'tokudb_rpl_unique_checks%'; +show variables like 'tokudb_rpl_unique_checks%'; # insert some rows connection master; diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_write_pk_uc1-slave.opt b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_write_pk_uc1-slave.opt index b1df0b6daf0..646a9991753 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_write_pk_uc1-slave.opt +++ b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_write_pk_uc1-slave.opt @@ -1 +1 @@ ---read-only=ON --tokudb-rpl-unique-checks-delay=10000 --tokudb-rpl-unique-checks=ON +--read-only=ON --loose-tokudb-rpl-unique-checks-delay=10000 --loose-tokudb-rpl-unique-checks=ON diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_write_pk_uc1.test b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_write_pk_uc1.test index c77e4b49605..0ed12b34e1f 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_write_pk_uc1.test +++ b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_write_pk_uc1.test @@ -14,7 +14,7 @@ enable_warnings; connection slave; # show variables like 'read_only'; -# show variables like 'tokudb_rpl_unique_checks%'; +show variables like 'tokudb_rpl_unique_checks%'; # insert some rows connection master; diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_write_unique-slave.opt b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_write_unique-slave.opt index 9baf0d65ecf..19b40f86454 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_write_unique-slave.opt +++ b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_write_unique-slave.opt @@ -1 +1 @@ ---read-only=ON --tokudb-rpl-unique-checks-delay=5000 --tokudb-rpl-unique-checks=OFF +--read-only=ON --loose-tokudb-rpl-unique-checks-delay=5000 --loose-tokudb-rpl-unique-checks=OFF diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_write_unique.test b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_write_unique.test index cf6a26b423d..fc4c9597dac 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_write_unique.test +++ b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_write_unique.test @@ -14,7 +14,7 @@ enable_warnings; connection slave; # show variables like 'read_only'; -# show variables like 'tokudb_rpl_unique_checks%'; +show variables like 'tokudb_rpl_unique_checks%'; # insert some rows connection master; diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_write_unique_uc1-slave.opt b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_write_unique_uc1-slave.opt index 0518efd3da5..9139a370e57 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_write_unique_uc1-slave.opt +++ b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_write_unique_uc1-slave.opt @@ -1 +1 @@ ---read-only=ON --tokudb-rpl-unique-checks-delay=5000 --tokudb-rpl-unique-checks=ON +--read-only=ON --loose-tokudb-rpl-unique-checks-delay=5000 --loose-tokudb-rpl-unique-checks=ON diff --git a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_write_unique_uc1.test b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_write_unique_uc1.test index cf6a26b423d..fc4c9597dac 100644 --- a/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_write_unique_uc1.test +++ b/mysql-test/suite/tokudb.rpl/t/rpl_tokudb_write_unique_uc1.test @@ -14,7 +14,7 @@ enable_warnings; connection slave; # show variables like 'read_only'; -# show variables like 'tokudb_rpl_unique_checks%'; +show variables like 'tokudb_rpl_unique_checks%'; # insert some rows connection master; diff --git a/mysql-test/suite/tokudb.rpl/t/suite.opt b/mysql-test/suite/tokudb.rpl/t/suite.opt new file mode 100644 index 00000000000..23511b05020 --- /dev/null +++ b/mysql-test/suite/tokudb.rpl/t/suite.opt @@ -0,0 +1 @@ +$TOKUDB_OPT $TOKUDB_LOAD_ADD --loose-tokudb-check-jemalloc=0 diff --git a/mysql-test/suite/tokudb.sys_vars/r/tokudb_analyze_delete_fraction.result b/mysql-test/suite/tokudb.sys_vars/r/tokudb_analyze_delete_fraction.result new file mode 100644 index 00000000000..41918883dd6 --- /dev/null +++ b/mysql-test/suite/tokudb.sys_vars/r/tokudb_analyze_delete_fraction.result @@ -0,0 +1,73 @@ +SET @orig_global = @@global.tokudb_analyze_delete_fraction; +SELECT @orig_global; +@orig_global +1 +SET @orig_session = @@session.tokudb_analyze_delete_fraction; +SELECT @orig_session; +@orig_session +1 +SET GLOBAL tokudb_analyze_delete_fraction = .5; +SELECT @@global.tokudb_analyze_delete_fraction; +@@global.tokudb_analyze_delete_fraction +0.500000 +SET GLOBAL tokudb_analyze_delete_fraction = 0; +SELECT @@global.tokudb_analyze_delete_fraction; +@@global.tokudb_analyze_delete_fraction +0.000000 +SET GLOBAL tokudb_analyze_delete_fraction = DEFAULT; +SELECT @@global.tokudb_analyze_delete_fraction; +@@global.tokudb_analyze_delete_fraction +1.000000 +SET GLOBAL tokudb_analyze_delete_fraction = 'foobar'; +ERROR 42000: Incorrect argument type to variable 'tokudb_analyze_delete_fraction' +SELECT @@global.tokudb_analyze_delete_fraction; +@@global.tokudb_analyze_delete_fraction +1.000000 +SET GLOBAL tokudb_analyze_delete_fraction = 3.75; +Warnings: +Warning 1292 Truncated incorrect tokudb_analyze_delete_fraction value: '3.75' +SELECT @@global.tokudb_analyze_delete_fraction; +@@global.tokudb_analyze_delete_fraction +1.000000 +SET SESSION tokudb_analyze_delete_fraction = .5; +SELECT @@session.tokudb_analyze_delete_fraction; +@@session.tokudb_analyze_delete_fraction +0.500000 +SET SESSION tokudb_analyze_delete_fraction = 0; +SELECT @@session.tokudb_analyze_delete_fraction; +@@session.tokudb_analyze_delete_fraction +0.000000 +SET SESSION tokudb_analyze_delete_fraction = DEFAULT; +SELECT @@session.tokudb_analyze_delete_fraction; +@@session.tokudb_analyze_delete_fraction +1.000000 +SET SESSION tokudb_analyze_delete_fraction = 'foobar'; +ERROR 42000: Incorrect argument type to variable 'tokudb_analyze_delete_fraction' +SELECT @@session.tokudb_analyze_delete_fraction; +@@session.tokudb_analyze_delete_fraction +1.000000 +SET SESSION tokudb_analyze_delete_fraction = 3.75; +Warnings: +Warning 1292 Truncated incorrect tokudb_analyze_delete_fraction value: '3.75' +SELECT @@session.tokudb_analyze_delete_fraction; +@@session.tokudb_analyze_delete_fraction +1.000000 +SET GLOBAL tokudb_analyze_delete_fraction = .2; +SET SESSION tokudb_analyze_delete_fraction = .3; +SELECT @@global.tokudb_analyze_delete_fraction; +@@global.tokudb_analyze_delete_fraction +0.200000 +SELECT @@session.tokudb_analyze_delete_fraction; +@@session.tokudb_analyze_delete_fraction +0.300000 +SHOW VARIABLES LIKE 'tokudb_analyze_delete_fraction'; +Variable_name Value +tokudb_analyze_delete_fraction 0.300000 +SET SESSION tokudb_analyze_delete_fraction = @orig_session; +SELECT @@session.tokudb_analyze_delete_fraction; +@@session.tokudb_analyze_delete_fraction +1.000000 +SET GLOBAL tokudb_analyze_delete_fraction = @orig_global; +SELECT @@global.tokudb_analyze_delete_fraction; +@@global.tokudb_analyze_delete_fraction +1.000000 diff --git a/mysql-test/suite/tokudb.sys_vars/r/tokudb_analyze_in_background_basic.result b/mysql-test/suite/tokudb.sys_vars/r/tokudb_analyze_in_background_basic.result new file mode 100644 index 00000000000..53e96810eda --- /dev/null +++ b/mysql-test/suite/tokudb.sys_vars/r/tokudb_analyze_in_background_basic.result @@ -0,0 +1,99 @@ +SET @orig_global = @@global.tokudb_analyze_in_background; +SELECT @orig_global; +@orig_global +0 +SET @orig_session = @@session.tokudb_analyze_in_background; +SELECT @orig_session; +@orig_session +0 +SET GLOBAL tokudb_analyze_in_background = 0; +SELECT @@global.tokudb_analyze_in_background; +@@global.tokudb_analyze_in_background +0 +SET GLOBAL tokudb_analyze_in_background = 1; +SELECT @@global.tokudb_analyze_in_background; +@@global.tokudb_analyze_in_background +1 +SET GLOBAL tokudb_analyze_in_background = DEFAULT; +SELECT @@global.tokudb_analyze_in_background; +@@global.tokudb_analyze_in_background +0 +SET GLOBAL tokudb_analyze_in_background = -6; +SELECT @@global.tokudb_analyze_in_background; +@@global.tokudb_analyze_in_background +1 +SET GLOBAL tokudb_analyze_in_background = 1.6; +ERROR 42000: Incorrect argument type to variable 'tokudb_analyze_in_background' +SELECT @@global.tokudb_analyze_in_background; +@@global.tokudb_analyze_in_background +1 +SET GLOBAL tokudb_analyze_in_background = "T"; +ERROR 42000: Variable 'tokudb_analyze_in_background' can't be set to the value of 'T' +SELECT @@global.tokudb_analyze_in_background; +@@global.tokudb_analyze_in_background +1 +SET GLOBAL tokudb_analyze_in_background = "Y"; +ERROR 42000: Variable 'tokudb_analyze_in_background' can't be set to the value of 'Y' +SELECT @@global.tokudb_analyze_in_background; +@@global.tokudb_analyze_in_background +1 +SET GLOBAL tokudb_analyze_in_background = 'foobar'; +ERROR 42000: Variable 'tokudb_analyze_in_background' can't be set to the value of 'foobar' +SELECT @@global.tokudb_analyze_in_background; +@@global.tokudb_analyze_in_background +1 +SET SESSION tokudb_analyze_in_background = 0; +SELECT @@session.tokudb_analyze_in_background; +@@session.tokudb_analyze_in_background +0 +SET SESSION tokudb_analyze_in_background = 1; +SELECT @@session.tokudb_analyze_in_background; +@@session.tokudb_analyze_in_background +1 +SET SESSION tokudb_analyze_in_background = DEFAULT; +SELECT @@session.tokudb_analyze_in_background; +@@session.tokudb_analyze_in_background +1 +SET SESSION tokudb_analyze_in_background = -6; +SELECT @@session.tokudb_analyze_in_background; +@@session.tokudb_analyze_in_background +1 +SET SESSION tokudb_analyze_in_background = 1.6; +ERROR 42000: Incorrect argument type to variable 'tokudb_analyze_in_background' +SELECT @@session.tokudb_analyze_in_background; +@@session.tokudb_analyze_in_background +1 +SET SESSION tokudb_analyze_in_background = "T"; +ERROR 42000: Variable 'tokudb_analyze_in_background' can't be set to the value of 'T' +SELECT @@session.tokudb_analyze_in_background; +@@session.tokudb_analyze_in_background +1 +SET SESSION tokudb_analyze_in_background = "Y"; +ERROR 42000: Variable 'tokudb_analyze_in_background' can't be set to the value of 'Y' +SELECT @@session.tokudb_analyze_in_background; +@@session.tokudb_analyze_in_background +1 +SET SESSION tokudb_analyze_in_background = 'foobar'; +ERROR 42000: Variable 'tokudb_analyze_in_background' can't be set to the value of 'foobar' +SELECT @@session.tokudb_analyze_in_background; +@@session.tokudb_analyze_in_background +1 +SET GLOBAL tokudb_analyze_in_background = 0; +SET SESSION tokudb_analyze_in_background = 1; +SELECT @@global.tokudb_analyze_in_background; +@@global.tokudb_analyze_in_background +0 +SELECT @@session.tokudb_analyze_in_background; +@@session.tokudb_analyze_in_background +1 +SHOW VARIABLES LIKE 'tokudb_analyze_in_background'; +Variable_name Value +tokudb_analyze_in_background ON +SET SESSION tokudb_analyze_in_background = @orig_session; +SELECT @@session.tokudb_analyze_in_background; +@@session.tokudb_analyze_in_background +0 +SET GLOBAL tokudb_analyze_in_background = @orig_global; +SELECT @@global.tokudb_analyze_in_background; +@@global.tokudb_analyze_in_background +0 diff --git a/mysql-test/suite/tokudb.sys_vars/r/tokudb_analyze_mode_basic.result b/mysql-test/suite/tokudb.sys_vars/r/tokudb_analyze_mode_basic.result new file mode 100644 index 00000000000..e2a3059a5aa --- /dev/null +++ b/mysql-test/suite/tokudb.sys_vars/r/tokudb_analyze_mode_basic.result @@ -0,0 +1,89 @@ +SET @orig_global = @@global.tokudb_analyze_mode; +SELECT @orig_global; +@orig_global +TOKUDB_ANALYZE_STANDARD +SET @orig_session = @@session.tokudb_analyze_mode; +SELECT @orig_session; +@orig_session +TOKUDB_ANALYZE_STANDARD +SET GLOBAL tokudb_analyze_mode = 'tokudb_analyze_standard'; +SELECT @@global.tokudb_analyze_mode; +@@global.tokudb_analyze_mode +TOKUDB_ANALYZE_STANDARD +SET GLOBAL tokudb_analyze_mode = 'tokudb_analyze_recount_rows'; +SELECT @@global.tokudb_analyze_mode; +@@global.tokudb_analyze_mode +TOKUDB_ANALYZE_RECOUNT_ROWS +SET GLOBAL tokudb_analyze_mode = 'tokudb_analyze_cancel'; +SELECT @@global.tokudb_analyze_mode; +@@global.tokudb_analyze_mode +TOKUDB_ANALYZE_CANCEL +SET GLOBAL tokudb_analyze_mode = DEFAULT; +SELECT @@global.tokudb_analyze_mode; +@@global.tokudb_analyze_mode +TOKUDB_ANALYZE_STANDARD +SET GLOBAL tokudb_analyze_mode = ''; +ERROR 42000: Variable 'tokudb_analyze_mode' can't be set to the value of '' +SELECT @@global.tokudb_analyze_mode; +@@global.tokudb_analyze_mode +TOKUDB_ANALYZE_STANDARD +SET GLOBAL tokudb_analyze_mode = 'foobar'; +ERROR 42000: Variable 'tokudb_analyze_mode' can't be set to the value of 'foobar' +SELECT @@global.tokudb_analyze_mode; +@@global.tokudb_analyze_mode +TOKUDB_ANALYZE_STANDARD +SET GLOBAL tokudb_analyze_mode = 123; +ERROR 42000: Variable 'tokudb_analyze_mode' can't be set to the value of '123' +SELECT @@global.tokudb_analyze_mode; +@@global.tokudb_analyze_mode +TOKUDB_ANALYZE_STANDARD +SET SESSION tokudb_analyze_mode = 'tokudb_analyze_standard'; +SELECT @@session.tokudb_analyze_mode; +@@session.tokudb_analyze_mode +TOKUDB_ANALYZE_STANDARD +SET SESSION tokudb_analyze_mode = 'tokudb_analyze_recount_rows'; +SELECT @@session.tokudb_analyze_mode; +@@session.tokudb_analyze_mode +TOKUDB_ANALYZE_RECOUNT_ROWS +SET SESSION tokudb_analyze_mode = 'tokudb_analyze_cancel'; +SELECT @@session.tokudb_analyze_mode; +@@session.tokudb_analyze_mode +TOKUDB_ANALYZE_CANCEL +SET SESSION tokudb_analyze_mode = DEFAULT; +SELECT @@session.tokudb_analyze_mode; +@@session.tokudb_analyze_mode +TOKUDB_ANALYZE_STANDARD +SET SESSION tokudb_analyze_mode = ''; +ERROR 42000: Variable 'tokudb_analyze_mode' can't be set to the value of '' +SELECT @@session.tokudb_analyze_mode; +@@session.tokudb_analyze_mode +TOKUDB_ANALYZE_STANDARD +SET SESSION tokudb_analyze_mode = 'foobar'; +ERROR 42000: Variable 'tokudb_analyze_mode' can't be set to the value of 'foobar' +SELECT @@session.tokudb_analyze_mode; +@@session.tokudb_analyze_mode +TOKUDB_ANALYZE_STANDARD +SET SESSION tokudb_analyze_mode = 123; +ERROR 42000: Variable 'tokudb_analyze_mode' can't be set to the value of '123' +SELECT @@session.tokudb_analyze_mode; +@@session.tokudb_analyze_mode +TOKUDB_ANALYZE_STANDARD +SET GLOBAL tokudb_analyze_mode = 'tokudb_analyze_standard'; +SET SESSION tokudb_analyze_mode = 'tokudb_analyze_recount_rows'; +SELECT @@global.tokudb_analyze_mode; +@@global.tokudb_analyze_mode +TOKUDB_ANALYZE_STANDARD +SELECT @@session.tokudb_analyze_mode; +@@session.tokudb_analyze_mode +TOKUDB_ANALYZE_RECOUNT_ROWS +SHOW VARIABLES LIKE 'tokudb_analyze_mode'; +Variable_name Value +tokudb_analyze_mode TOKUDB_ANALYZE_RECOUNT_ROWS +SET SESSION tokudb_analyze_mode = @orig_session; +SELECT @@session.tokudb_analyze_mode; +@@session.tokudb_analyze_mode +TOKUDB_ANALYZE_STANDARD +SET GLOBAL tokudb_analyze_mode = @orig_global; +SELECT @@global.tokudb_analyze_mode; +@@global.tokudb_analyze_mode +TOKUDB_ANALYZE_STANDARD diff --git a/mysql-test/suite/tokudb.sys_vars/r/tokudb_analyze_throttle_basic.result b/mysql-test/suite/tokudb.sys_vars/r/tokudb_analyze_throttle_basic.result new file mode 100644 index 00000000000..34317c7cb7b --- /dev/null +++ b/mysql-test/suite/tokudb.sys_vars/r/tokudb_analyze_throttle_basic.result @@ -0,0 +1,61 @@ +SET @orig_global = @@global.tokudb_analyze_throttle; +SELECT @orig_global; +@orig_global +0 +SET @orig_session = @@session.tokudb_analyze_throttle; +SELECT @orig_session; +@orig_session +0 +SET GLOBAL tokudb_analyze_throttle = 10; +SELECT @@global.tokudb_analyze_throttle; +@@global.tokudb_analyze_throttle +10 +SET GLOBAL tokudb_analyze_throttle = 0; +SELECT @@global.tokudb_analyze_throttle; +@@global.tokudb_analyze_throttle +0 +SET GLOBAL tokudb_analyze_throttle = DEFAULT; +SELECT @@global.tokudb_analyze_throttle; +@@global.tokudb_analyze_throttle +0 +SET GLOBAL tokudb_analyze_throttle = 'foobar'; +ERROR 42000: Incorrect argument type to variable 'tokudb_analyze_throttle' +SELECT @@global.tokudb_analyze_throttle; +@@global.tokudb_analyze_throttle +0 +SET SESSION tokudb_analyze_throttle = 10; +SELECT @@session.tokudb_analyze_throttle; +@@session.tokudb_analyze_throttle +10 +SET SESSION tokudb_analyze_throttle = 0; +SELECT @@session.tokudb_analyze_throttle; +@@session.tokudb_analyze_throttle +0 +SET SESSION tokudb_analyze_throttle = DEFAULT; +SELECT @@session.tokudb_analyze_throttle; +@@session.tokudb_analyze_throttle +0 +SET SESSION tokudb_analyze_throttle = 'foobar'; +ERROR 42000: Incorrect argument type to variable 'tokudb_analyze_throttle' +SELECT @@session.tokudb_analyze_throttle; +@@session.tokudb_analyze_throttle +0 +SET GLOBAL tokudb_analyze_throttle = 12; +SET SESSION tokudb_analyze_throttle = 13; +SELECT @@global.tokudb_analyze_throttle; +@@global.tokudb_analyze_throttle +12 +SELECT @@session.tokudb_analyze_throttle; +@@session.tokudb_analyze_throttle +13 +SHOW VARIABLES LIKE 'tokudb_analyze_throttle'; +Variable_name Value +tokudb_analyze_throttle 13 +SET SESSION tokudb_analyze_throttle = @orig_session; +SELECT @@session.tokudb_analyze_throttle; +@@session.tokudb_analyze_throttle +0 +SET GLOBAL tokudb_analyze_throttle = @orig_global; +SELECT @@global.tokudb_analyze_throttle; +@@global.tokudb_analyze_throttle +0 diff --git a/mysql-test/suite/tokudb.sys_vars/r/tokudb_analyze_time_basic.result b/mysql-test/suite/tokudb.sys_vars/r/tokudb_analyze_time_basic.result new file mode 100644 index 00000000000..2eac1fcc3a1 --- /dev/null +++ b/mysql-test/suite/tokudb.sys_vars/r/tokudb_analyze_time_basic.result @@ -0,0 +1,61 @@ +SET @orig_global = @@global.tokudb_analyze_time; +SELECT @orig_global; +@orig_global +5 +SET @orig_session = @@session.tokudb_analyze_time; +SELECT @orig_session; +@orig_session +5 +SET GLOBAL tokudb_analyze_time = 10; +SELECT @@global.tokudb_analyze_time; +@@global.tokudb_analyze_time +10 +SET GLOBAL tokudb_analyze_time = 0; +SELECT @@global.tokudb_analyze_time; +@@global.tokudb_analyze_time +0 +SET GLOBAL tokudb_analyze_time = DEFAULT; +SELECT @@global.tokudb_analyze_time; +@@global.tokudb_analyze_time +5 +SET GLOBAL tokudb_analyze_time = 'foobar'; +ERROR 42000: Incorrect argument type to variable 'tokudb_analyze_time' +SELECT @@global.tokudb_analyze_time; +@@global.tokudb_analyze_time +5 +SET SESSION tokudb_analyze_time = 10; +SELECT @@session.tokudb_analyze_time; +@@session.tokudb_analyze_time +10 +SET SESSION tokudb_analyze_time = 0; +SELECT @@session.tokudb_analyze_time; +@@session.tokudb_analyze_time +0 +SET SESSION tokudb_analyze_time = DEFAULT; +SELECT @@session.tokudb_analyze_time; +@@session.tokudb_analyze_time +5 +SET SESSION tokudb_analyze_time = 'foobar'; +ERROR 42000: Incorrect argument type to variable 'tokudb_analyze_time' +SELECT @@session.tokudb_analyze_time; +@@session.tokudb_analyze_time +5 +SET GLOBAL tokudb_analyze_time = 12; +SET SESSION tokudb_analyze_time = 13; +SELECT @@global.tokudb_analyze_time; +@@global.tokudb_analyze_time +12 +SELECT @@session.tokudb_analyze_time; +@@session.tokudb_analyze_time +13 +SHOW VARIABLES LIKE 'tokudb_analyze_time'; +Variable_name Value +tokudb_analyze_time 13 +SET SESSION tokudb_analyze_time = @orig_session; +SELECT @@session.tokudb_analyze_time; +@@session.tokudb_analyze_time +5 +SET GLOBAL tokudb_analyze_time = @orig_global; +SELECT @@global.tokudb_analyze_time; +@@global.tokudb_analyze_time +5 diff --git a/mysql-test/suite/tokudb.sys_vars/r/tokudb_auto_analyze.result b/mysql-test/suite/tokudb.sys_vars/r/tokudb_auto_analyze.result new file mode 100644 index 00000000000..5e22bc489cb --- /dev/null +++ b/mysql-test/suite/tokudb.sys_vars/r/tokudb_auto_analyze.result @@ -0,0 +1,61 @@ +SET @orig_global = @@global.tokudb_auto_analyze; +SELECT @orig_global; +@orig_global +0 +SET @orig_session = @@session.tokudb_auto_analyze; +SELECT @orig_session; +@orig_session +0 +SET GLOBAL tokudb_auto_analyze = 10; +SELECT @@global.tokudb_auto_analyze; +@@global.tokudb_auto_analyze +10 +SET GLOBAL tokudb_auto_analyze = 0; +SELECT @@global.tokudb_auto_analyze; +@@global.tokudb_auto_analyze +0 +SET GLOBAL tokudb_auto_analyze = DEFAULT; +SELECT @@global.tokudb_auto_analyze; +@@global.tokudb_auto_analyze +0 +SET GLOBAL tokudb_auto_analyze = 'foobar'; +ERROR 42000: Incorrect argument type to variable 'tokudb_auto_analyze' +SELECT @@global.tokudb_auto_analyze; +@@global.tokudb_auto_analyze +0 +SET SESSION tokudb_auto_analyze = 10; +SELECT @@session.tokudb_auto_analyze; +@@session.tokudb_auto_analyze +10 +SET SESSION tokudb_auto_analyze = 0; +SELECT @@session.tokudb_auto_analyze; +@@session.tokudb_auto_analyze +0 +SET SESSION tokudb_auto_analyze = DEFAULT; +SELECT @@session.tokudb_auto_analyze; +@@session.tokudb_auto_analyze +0 +SET SESSION tokudb_auto_analyze = 'foobar'; +ERROR 42000: Incorrect argument type to variable 'tokudb_auto_analyze' +SELECT @@session.tokudb_auto_analyze; +@@session.tokudb_auto_analyze +0 +SET GLOBAL tokudb_auto_analyze = 12; +SET SESSION tokudb_auto_analyze = 13; +SELECT @@global.tokudb_auto_analyze; +@@global.tokudb_auto_analyze +12 +SELECT @@session.tokudb_auto_analyze; +@@session.tokudb_auto_analyze +13 +SHOW VARIABLES LIKE 'tokudb_auto_analyze'; +Variable_name Value +tokudb_auto_analyze 13 +SET SESSION tokudb_auto_analyze = @orig_session; +SELECT @@session.tokudb_auto_analyze; +@@session.tokudb_auto_analyze +0 +SET GLOBAL tokudb_auto_analyze = @orig_global; +SELECT @@global.tokudb_auto_analyze; +@@global.tokudb_auto_analyze +0 diff --git a/mysql-test/suite/tokudb.sys_vars/r/tokudb_cardinality_scale_percent_basic.result b/mysql-test/suite/tokudb.sys_vars/r/tokudb_cardinality_scale_percent_basic.result new file mode 100644 index 00000000000..cac5d8b0dc7 --- /dev/null +++ b/mysql-test/suite/tokudb.sys_vars/r/tokudb_cardinality_scale_percent_basic.result @@ -0,0 +1,36 @@ +SET @orig_global = @@global.tokudb_cardinality_scale_percent; +SELECT @orig_global; +@orig_global +50 +SET GLOBAL tokudb_cardinality_scale_percent = 10; +SELECT @@global.tokudb_cardinality_scale_percent; +@@global.tokudb_cardinality_scale_percent +10 +SET GLOBAL tokudb_cardinality_scale_percent = 0; +SELECT @@global.tokudb_cardinality_scale_percent; +@@global.tokudb_cardinality_scale_percent +0 +SET GLOBAL tokudb_cardinality_scale_percent = DEFAULT; +SELECT @@global.tokudb_cardinality_scale_percent; +@@global.tokudb_cardinality_scale_percent +50 +SET GLOBAL tokudb_cardinality_scale_percent = 'foobar'; +ERROR 42000: Incorrect argument type to variable 'tokudb_cardinality_scale_percent' +SELECT @@global.tokudb_cardinality_scale_percent; +@@global.tokudb_cardinality_scale_percent +50 +SET GLOBAL tokudb_cardinality_scale_percent = 12; +SET SESSION tokudb_cardinality_scale_percent = 13; +ERROR HY000: Variable 'tokudb_cardinality_scale_percent' is a GLOBAL variable and should be set with SET GLOBAL +SELECT @@global.tokudb_cardinality_scale_percent; +@@global.tokudb_cardinality_scale_percent +12 +SELECT @@session.tokudb_cardinality_scale_percent; +ERROR HY000: Variable 'tokudb_cardinality_scale_percent' is a GLOBAL variable +SHOW VARIABLES LIKE 'tokudb_cardinality_scale_percent'; +Variable_name Value +tokudb_cardinality_scale_percent 12 +SET GLOBAL tokudb_cardinality_scale_percent = @orig_global; +SELECT @@global.tokudb_cardinality_scale_percent; +@@global.tokudb_cardinality_scale_percent +50 diff --git a/mysql-test/suite/tokudb.sys_vars/t/suite.opt b/mysql-test/suite/tokudb.sys_vars/t/suite.opt new file mode 100644 index 00000000000..23511b05020 --- /dev/null +++ b/mysql-test/suite/tokudb.sys_vars/t/suite.opt @@ -0,0 +1 @@ +$TOKUDB_OPT $TOKUDB_LOAD_ADD --loose-tokudb-check-jemalloc=0 diff --git a/mysql-test/suite/tokudb.sys_vars/t/tokudb_analyze_delete_fraction.test b/mysql-test/suite/tokudb.sys_vars/t/tokudb_analyze_delete_fraction.test new file mode 100644 index 00000000000..d5bd382740f --- /dev/null +++ b/mysql-test/suite/tokudb.sys_vars/t/tokudb_analyze_delete_fraction.test @@ -0,0 +1,56 @@ +--source include/have_tokudb.inc + +# Check the default value +SET @orig_global = @@global.tokudb_analyze_delete_fraction; +SELECT @orig_global; + +SET @orig_session = @@session.tokudb_analyze_delete_fraction; +SELECT @orig_session; + +# Test global +SET GLOBAL tokudb_analyze_delete_fraction = .5; +SELECT @@global.tokudb_analyze_delete_fraction; + +SET GLOBAL tokudb_analyze_delete_fraction = 0; +SELECT @@global.tokudb_analyze_delete_fraction; + +SET GLOBAL tokudb_analyze_delete_fraction = DEFAULT; +SELECT @@global.tokudb_analyze_delete_fraction; + +-- error ER_WRONG_TYPE_FOR_VAR +SET GLOBAL tokudb_analyze_delete_fraction = 'foobar'; +SELECT @@global.tokudb_analyze_delete_fraction; + +SET GLOBAL tokudb_analyze_delete_fraction = 3.75; +SELECT @@global.tokudb_analyze_delete_fraction; + +# Test session +SET SESSION tokudb_analyze_delete_fraction = .5; +SELECT @@session.tokudb_analyze_delete_fraction; + +SET SESSION tokudb_analyze_delete_fraction = 0; +SELECT @@session.tokudb_analyze_delete_fraction; + +SET SESSION tokudb_analyze_delete_fraction = DEFAULT; +SELECT @@session.tokudb_analyze_delete_fraction; + +-- error ER_WRONG_TYPE_FOR_VAR +SET SESSION tokudb_analyze_delete_fraction = 'foobar'; +SELECT @@session.tokudb_analyze_delete_fraction; + +SET SESSION tokudb_analyze_delete_fraction = 3.75; +SELECT @@session.tokudb_analyze_delete_fraction; + +# both +SET GLOBAL tokudb_analyze_delete_fraction = .2; +SET SESSION tokudb_analyze_delete_fraction = .3; +SELECT @@global.tokudb_analyze_delete_fraction; +SELECT @@session.tokudb_analyze_delete_fraction; +SHOW VARIABLES LIKE 'tokudb_analyze_delete_fraction'; + +# Clean up +SET SESSION tokudb_analyze_delete_fraction = @orig_session; +SELECT @@session.tokudb_analyze_delete_fraction; + +SET GLOBAL tokudb_analyze_delete_fraction = @orig_global; +SELECT @@global.tokudb_analyze_delete_fraction; diff --git a/mysql-test/suite/tokudb.sys_vars/t/tokudb_analyze_in_background_basic.test b/mysql-test/suite/tokudb.sys_vars/t/tokudb_analyze_in_background_basic.test new file mode 100644 index 00000000000..dfb2a0e416d --- /dev/null +++ b/mysql-test/suite/tokudb.sys_vars/t/tokudb_analyze_in_background_basic.test @@ -0,0 +1,80 @@ +--source include/have_tokudb.inc + +# Check the default value +SET @orig_global = @@global.tokudb_analyze_in_background; +SELECT @orig_global; + +SET @orig_session = @@session.tokudb_analyze_in_background; +SELECT @orig_session; + +# Test global +SET GLOBAL tokudb_analyze_in_background = 0; +SELECT @@global.tokudb_analyze_in_background; + +SET GLOBAL tokudb_analyze_in_background = 1; +SELECT @@global.tokudb_analyze_in_background; + +SET GLOBAL tokudb_analyze_in_background = DEFAULT; +SELECT @@global.tokudb_analyze_in_background; + +SET GLOBAL tokudb_analyze_in_background = -6; +SELECT @@global.tokudb_analyze_in_background; + +-- error ER_WRONG_TYPE_FOR_VAR +SET GLOBAL tokudb_analyze_in_background = 1.6; +SELECT @@global.tokudb_analyze_in_background; + +-- error ER_WRONG_VALUE_FOR_VAR +SET GLOBAL tokudb_analyze_in_background = "T"; +SELECT @@global.tokudb_analyze_in_background; + +-- error ER_WRONG_VALUE_FOR_VAR +SET GLOBAL tokudb_analyze_in_background = "Y"; +SELECT @@global.tokudb_analyze_in_background; + +-- error ER_WRONG_VALUE_FOR_VAR +SET GLOBAL tokudb_analyze_in_background = 'foobar'; +SELECT @@global.tokudb_analyze_in_background; + +# Test session +SET SESSION tokudb_analyze_in_background = 0; +SELECT @@session.tokudb_analyze_in_background; + +SET SESSION tokudb_analyze_in_background = 1; +SELECT @@session.tokudb_analyze_in_background; + +SET SESSION tokudb_analyze_in_background = DEFAULT; +SELECT @@session.tokudb_analyze_in_background; + +SET SESSION tokudb_analyze_in_background = -6; +SELECT @@session.tokudb_analyze_in_background; + +-- error ER_WRONG_TYPE_FOR_VAR +SET SESSION tokudb_analyze_in_background = 1.6; +SELECT @@session.tokudb_analyze_in_background; + +-- error ER_WRONG_VALUE_FOR_VAR +SET SESSION tokudb_analyze_in_background = "T"; +SELECT @@session.tokudb_analyze_in_background; + +-- error ER_WRONG_VALUE_FOR_VAR +SET SESSION tokudb_analyze_in_background = "Y"; +SELECT @@session.tokudb_analyze_in_background; + +-- error ER_WRONG_VALUE_FOR_VAR +SET SESSION tokudb_analyze_in_background = 'foobar'; +SELECT @@session.tokudb_analyze_in_background; + +# both +SET GLOBAL tokudb_analyze_in_background = 0; +SET SESSION tokudb_analyze_in_background = 1; +SELECT @@global.tokudb_analyze_in_background; +SELECT @@session.tokudb_analyze_in_background; +SHOW VARIABLES LIKE 'tokudb_analyze_in_background'; + +# Clean up +SET SESSION tokudb_analyze_in_background = @orig_session; +SELECT @@session.tokudb_analyze_in_background; + +SET GLOBAL tokudb_analyze_in_background = @orig_global; +SELECT @@global.tokudb_analyze_in_background; diff --git a/mysql-test/suite/tokudb.sys_vars/t/tokudb_analyze_mode_basic.test b/mysql-test/suite/tokudb.sys_vars/t/tokudb_analyze_mode_basic.test new file mode 100644 index 00000000000..69def75bd3d --- /dev/null +++ b/mysql-test/suite/tokudb.sys_vars/t/tokudb_analyze_mode_basic.test @@ -0,0 +1,72 @@ +--source include/have_tokudb.inc + +# Check the default value +SET @orig_global = @@global.tokudb_analyze_mode; +SELECT @orig_global; + +SET @orig_session = @@session.tokudb_analyze_mode; +SELECT @orig_session; + +# Test global +SET GLOBAL tokudb_analyze_mode = 'tokudb_analyze_standard'; +SELECT @@global.tokudb_analyze_mode; + +SET GLOBAL tokudb_analyze_mode = 'tokudb_analyze_recount_rows'; +SELECT @@global.tokudb_analyze_mode; + +SET GLOBAL tokudb_analyze_mode = 'tokudb_analyze_cancel'; +SELECT @@global.tokudb_analyze_mode; + +SET GLOBAL tokudb_analyze_mode = DEFAULT; +SELECT @@global.tokudb_analyze_mode; + +-- error ER_WRONG_VALUE_FOR_VAR +SET GLOBAL tokudb_analyze_mode = ''; +SELECT @@global.tokudb_analyze_mode; + +-- error ER_WRONG_VALUE_FOR_VAR +SET GLOBAL tokudb_analyze_mode = 'foobar'; +SELECT @@global.tokudb_analyze_mode; + +-- error ER_WRONG_VALUE_FOR_VAR +SET GLOBAL tokudb_analyze_mode = 123; +SELECT @@global.tokudb_analyze_mode; + +# Test session +SET SESSION tokudb_analyze_mode = 'tokudb_analyze_standard'; +SELECT @@session.tokudb_analyze_mode; + +SET SESSION tokudb_analyze_mode = 'tokudb_analyze_recount_rows'; +SELECT @@session.tokudb_analyze_mode; + +SET SESSION tokudb_analyze_mode = 'tokudb_analyze_cancel'; +SELECT @@session.tokudb_analyze_mode; + +SET SESSION tokudb_analyze_mode = DEFAULT; +SELECT @@session.tokudb_analyze_mode; + +-- error ER_WRONG_VALUE_FOR_VAR +SET SESSION tokudb_analyze_mode = ''; +SELECT @@session.tokudb_analyze_mode; + +-- error ER_WRONG_VALUE_FOR_VAR +SET SESSION tokudb_analyze_mode = 'foobar'; +SELECT @@session.tokudb_analyze_mode; + +-- error ER_WRONG_VALUE_FOR_VAR +SET SESSION tokudb_analyze_mode = 123; +SELECT @@session.tokudb_analyze_mode; + +# both +SET GLOBAL tokudb_analyze_mode = 'tokudb_analyze_standard'; +SET SESSION tokudb_analyze_mode = 'tokudb_analyze_recount_rows'; +SELECT @@global.tokudb_analyze_mode; +SELECT @@session.tokudb_analyze_mode; +SHOW VARIABLES LIKE 'tokudb_analyze_mode'; + +# Clean up +SET SESSION tokudb_analyze_mode = @orig_session; +SELECT @@session.tokudb_analyze_mode; + +SET GLOBAL tokudb_analyze_mode = @orig_global; +SELECT @@global.tokudb_analyze_mode; diff --git a/mysql-test/suite/tokudb.sys_vars/t/tokudb_analyze_throttle_basic.test b/mysql-test/suite/tokudb.sys_vars/t/tokudb_analyze_throttle_basic.test new file mode 100644 index 00000000000..a3660b0626a --- /dev/null +++ b/mysql-test/suite/tokudb.sys_vars/t/tokudb_analyze_throttle_basic.test @@ -0,0 +1,50 @@ +--source include/have_tokudb.inc + +# Check the default value +SET @orig_global = @@global.tokudb_analyze_throttle; +SELECT @orig_global; + +SET @orig_session = @@session.tokudb_analyze_throttle; +SELECT @orig_session; + +# Test global +SET GLOBAL tokudb_analyze_throttle = 10; +SELECT @@global.tokudb_analyze_throttle; + +SET GLOBAL tokudb_analyze_throttle = 0; +SELECT @@global.tokudb_analyze_throttle; + +SET GLOBAL tokudb_analyze_throttle = DEFAULT; +SELECT @@global.tokudb_analyze_throttle; + +-- error ER_WRONG_TYPE_FOR_VAR +SET GLOBAL tokudb_analyze_throttle = 'foobar'; +SELECT @@global.tokudb_analyze_throttle; + +# Test session +SET SESSION tokudb_analyze_throttle = 10; +SELECT @@session.tokudb_analyze_throttle; + +SET SESSION tokudb_analyze_throttle = 0; +SELECT @@session.tokudb_analyze_throttle; + +SET SESSION tokudb_analyze_throttle = DEFAULT; +SELECT @@session.tokudb_analyze_throttle; + +-- error ER_WRONG_TYPE_FOR_VAR +SET SESSION tokudb_analyze_throttle = 'foobar'; +SELECT @@session.tokudb_analyze_throttle; + +# both +SET GLOBAL tokudb_analyze_throttle = 12; +SET SESSION tokudb_analyze_throttle = 13; +SELECT @@global.tokudb_analyze_throttle; +SELECT @@session.tokudb_analyze_throttle; +SHOW VARIABLES LIKE 'tokudb_analyze_throttle'; + +# Clean up +SET SESSION tokudb_analyze_throttle = @orig_session; +SELECT @@session.tokudb_analyze_throttle; + +SET GLOBAL tokudb_analyze_throttle = @orig_global; +SELECT @@global.tokudb_analyze_throttle; diff --git a/mysql-test/suite/tokudb.sys_vars/t/tokudb_analyze_time_basic.test b/mysql-test/suite/tokudb.sys_vars/t/tokudb_analyze_time_basic.test new file mode 100644 index 00000000000..3098934ee8c --- /dev/null +++ b/mysql-test/suite/tokudb.sys_vars/t/tokudb_analyze_time_basic.test @@ -0,0 +1,50 @@ +--source include/have_tokudb.inc + +# Check the default value +SET @orig_global = @@global.tokudb_analyze_time; +SELECT @orig_global; + +SET @orig_session = @@session.tokudb_analyze_time; +SELECT @orig_session; + +# Test global +SET GLOBAL tokudb_analyze_time = 10; +SELECT @@global.tokudb_analyze_time; + +SET GLOBAL tokudb_analyze_time = 0; +SELECT @@global.tokudb_analyze_time; + +SET GLOBAL tokudb_analyze_time = DEFAULT; +SELECT @@global.tokudb_analyze_time; + +-- error ER_WRONG_TYPE_FOR_VAR +SET GLOBAL tokudb_analyze_time = 'foobar'; +SELECT @@global.tokudb_analyze_time; + +# Test session +SET SESSION tokudb_analyze_time = 10; +SELECT @@session.tokudb_analyze_time; + +SET SESSION tokudb_analyze_time = 0; +SELECT @@session.tokudb_analyze_time; + +SET SESSION tokudb_analyze_time = DEFAULT; +SELECT @@session.tokudb_analyze_time; + +-- error ER_WRONG_TYPE_FOR_VAR +SET SESSION tokudb_analyze_time = 'foobar'; +SELECT @@session.tokudb_analyze_time; + +# both +SET GLOBAL tokudb_analyze_time = 12; +SET SESSION tokudb_analyze_time = 13; +SELECT @@global.tokudb_analyze_time; +SELECT @@session.tokudb_analyze_time; +SHOW VARIABLES LIKE 'tokudb_analyze_time'; + +# Clean up +SET SESSION tokudb_analyze_time = @orig_session; +SELECT @@session.tokudb_analyze_time; + +SET GLOBAL tokudb_analyze_time = @orig_global; +SELECT @@global.tokudb_analyze_time; diff --git a/mysql-test/suite/tokudb.sys_vars/t/tokudb_auto_analyze.test b/mysql-test/suite/tokudb.sys_vars/t/tokudb_auto_analyze.test new file mode 100644 index 00000000000..d9998508ae1 --- /dev/null +++ b/mysql-test/suite/tokudb.sys_vars/t/tokudb_auto_analyze.test @@ -0,0 +1,50 @@ +--source include/have_tokudb.inc + +# Check the default value +SET @orig_global = @@global.tokudb_auto_analyze; +SELECT @orig_global; + +SET @orig_session = @@session.tokudb_auto_analyze; +SELECT @orig_session; + +# Test global +SET GLOBAL tokudb_auto_analyze = 10; +SELECT @@global.tokudb_auto_analyze; + +SET GLOBAL tokudb_auto_analyze = 0; +SELECT @@global.tokudb_auto_analyze; + +SET GLOBAL tokudb_auto_analyze = DEFAULT; +SELECT @@global.tokudb_auto_analyze; + +-- error ER_WRONG_TYPE_FOR_VAR +SET GLOBAL tokudb_auto_analyze = 'foobar'; +SELECT @@global.tokudb_auto_analyze; + +# Test session +SET SESSION tokudb_auto_analyze = 10; +SELECT @@session.tokudb_auto_analyze; + +SET SESSION tokudb_auto_analyze = 0; +SELECT @@session.tokudb_auto_analyze; + +SET SESSION tokudb_auto_analyze = DEFAULT; +SELECT @@session.tokudb_auto_analyze; + +-- error ER_WRONG_TYPE_FOR_VAR +SET SESSION tokudb_auto_analyze = 'foobar'; +SELECT @@session.tokudb_auto_analyze; + +# both +SET GLOBAL tokudb_auto_analyze = 12; +SET SESSION tokudb_auto_analyze = 13; +SELECT @@global.tokudb_auto_analyze; +SELECT @@session.tokudb_auto_analyze; +SHOW VARIABLES LIKE 'tokudb_auto_analyze'; + +# Clean up +SET SESSION tokudb_auto_analyze = @orig_session; +SELECT @@session.tokudb_auto_analyze; + +SET GLOBAL tokudb_auto_analyze = @orig_global; +SELECT @@global.tokudb_auto_analyze; diff --git a/mysql-test/suite/tokudb.sys_vars/t/tokudb_cardinality_scale_percent_basic.test b/mysql-test/suite/tokudb.sys_vars/t/tokudb_cardinality_scale_percent_basic.test new file mode 100644 index 00000000000..83063f04248 --- /dev/null +++ b/mysql-test/suite/tokudb.sys_vars/t/tokudb_cardinality_scale_percent_basic.test @@ -0,0 +1,32 @@ +--source include/have_tokudb.inc + +# Check the default value +SET @orig_global = @@global.tokudb_cardinality_scale_percent; +SELECT @orig_global; + +# Test global +SET GLOBAL tokudb_cardinality_scale_percent = 10; +SELECT @@global.tokudb_cardinality_scale_percent; + +SET GLOBAL tokudb_cardinality_scale_percent = 0; +SELECT @@global.tokudb_cardinality_scale_percent; + +SET GLOBAL tokudb_cardinality_scale_percent = DEFAULT; +SELECT @@global.tokudb_cardinality_scale_percent; + +-- error ER_WRONG_TYPE_FOR_VAR +SET GLOBAL tokudb_cardinality_scale_percent = 'foobar'; +SELECT @@global.tokudb_cardinality_scale_percent; + +# both +SET GLOBAL tokudb_cardinality_scale_percent = 12; +-- error ER_GLOBAL_VARIABLE +SET SESSION tokudb_cardinality_scale_percent = 13; +SELECT @@global.tokudb_cardinality_scale_percent; +-- error ER_INCORRECT_GLOBAL_LOCAL_VAR +SELECT @@session.tokudb_cardinality_scale_percent; +SHOW VARIABLES LIKE 'tokudb_cardinality_scale_percent'; + +# Clean up +SET GLOBAL tokudb_cardinality_scale_percent = @orig_global; +SELECT @@global.tokudb_cardinality_scale_percent; diff --git a/mysql-test/suite/tokudb/r/background_job_manager.result b/mysql-test/suite/tokudb/r/background_job_manager.result Binary files differnew file mode 100644 index 00000000000..c66872d7b0f --- /dev/null +++ b/mysql-test/suite/tokudb/r/background_job_manager.result diff --git a/mysql-test/suite/tokudb/r/card_add_drop.result b/mysql-test/suite/tokudb/r/card_add_drop.result index 431f0200a7a..71a39eb1f3e 100644 --- a/mysql-test/suite/tokudb/r/card_add_drop.result +++ b/mysql-test/suite/tokudb/r/card_add_drop.result @@ -4,35 +4,35 @@ create table tt (a int, b int, c int, d int, key(a), key(b), key(c)); insert into tt values (0,0,0,0),(1,0,0,0),(2,0,1,0),(3,0,1,0); show indexes from tt; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment -tt 1 a 1 a A NULL NULL NULL YES BTREE -tt 1 b 1 b A NULL NULL NULL YES BTREE -tt 1 c 1 c A NULL NULL NULL YES BTREE +tt 1 a 1 a A 4 NULL NULL YES BTREE +tt 1 b 1 b A 4 NULL NULL YES BTREE +tt 1 c 1 c A 4 NULL NULL YES BTREE analyze table tt; Table Op Msg_type Msg_text test.tt analyze status OK show indexes from tt; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment tt 1 a 1 a A 4 NULL NULL YES BTREE -tt 1 b 1 b A 1 NULL NULL YES BTREE -tt 1 c 1 c A 2 NULL NULL YES BTREE +tt 1 b 1 b A 2 NULL NULL YES BTREE +tt 1 c 1 c A 4 NULL NULL YES BTREE alter table tt drop key b, add key (d); show indexes from tt; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment tt 1 a 1 a A 4 NULL NULL YES BTREE -tt 1 c 1 c A 2 NULL NULL YES BTREE -tt 1 d 1 d A NULL NULL NULL YES BTREE +tt 1 c 1 c A 4 NULL NULL YES BTREE +tt 1 d 1 d A 4 NULL NULL YES BTREE analyze table tt; Table Op Msg_type Msg_text test.tt analyze status OK show indexes from tt; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment tt 1 a 1 a A 4 NULL NULL YES BTREE -tt 1 c 1 c A 2 NULL NULL YES BTREE -tt 1 d 1 d A 1 NULL NULL YES BTREE +tt 1 c 1 c A 4 NULL NULL YES BTREE +tt 1 d 1 d A 2 NULL NULL YES BTREE flush tables; show indexes from tt; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment tt 1 a 1 a A 4 NULL NULL YES BTREE -tt 1 c 1 c A 2 NULL NULL YES BTREE -tt 1 d 1 d A 1 NULL NULL YES BTREE +tt 1 c 1 c A 4 NULL NULL YES BTREE +tt 1 d 1 d A 2 NULL NULL YES BTREE drop table tt; diff --git a/mysql-test/suite/tokudb/r/card_add_index.result b/mysql-test/suite/tokudb/r/card_add_index.result index f5ba5b58bed..9a929b19a80 100644 --- a/mysql-test/suite/tokudb/r/card_add_index.result +++ b/mysql-test/suite/tokudb/r/card_add_index.result @@ -15,32 +15,32 @@ alter table tt add key (b); show indexes from tt; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment tt 0 PRIMARY 1 a A 4 NULL NULL BTREE -tt 1 b 1 b A NULL NULL NULL YES BTREE +tt 1 b 1 b A 4 NULL NULL YES BTREE analyze table tt; Table Op Msg_type Msg_text test.tt analyze status OK show indexes from tt; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment tt 0 PRIMARY 1 a A 4 NULL NULL BTREE -tt 1 b 1 b A 1 NULL NULL YES BTREE +tt 1 b 1 b A 2 NULL NULL YES BTREE alter table tt add key (c); show indexes from tt; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment tt 0 PRIMARY 1 a A 4 NULL NULL BTREE -tt 1 b 1 b A 1 NULL NULL YES BTREE -tt 1 c 1 c A NULL NULL NULL YES BTREE +tt 1 b 1 b A 2 NULL NULL YES BTREE +tt 1 c 1 c A 4 NULL NULL YES BTREE analyze table tt; Table Op Msg_type Msg_text test.tt analyze status OK show indexes from tt; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment tt 0 PRIMARY 1 a A 4 NULL NULL BTREE -tt 1 b 1 b A 1 NULL NULL YES BTREE -tt 1 c 1 c A 2 NULL NULL YES BTREE +tt 1 b 1 b A 2 NULL NULL YES BTREE +tt 1 c 1 c A 4 NULL NULL YES BTREE flush tables; show indexes from tt; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment tt 0 PRIMARY 1 a A 4 NULL NULL BTREE -tt 1 b 1 b A 1 NULL NULL YES BTREE -tt 1 c 1 c A 2 NULL NULL YES BTREE +tt 1 b 1 b A 2 NULL NULL YES BTREE +tt 1 c 1 c A 4 NULL NULL YES BTREE drop table tt; diff --git a/mysql-test/suite/tokudb/r/card_auto_analyze_lots.result b/mysql-test/suite/tokudb/r/card_auto_analyze_lots.result new file mode 100644 index 00000000000..c665ef758a4 --- /dev/null +++ b/mysql-test/suite/tokudb/r/card_auto_analyze_lots.result @@ -0,0 +1,800 @@ +SHOW INDEX FROM ar_200; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_200 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_200 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_199; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_199 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_199 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_198; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_198 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_198 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_197; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_197 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_197 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_196; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_196 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_196 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_195; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_195 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_195 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_194; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_194 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_194 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_193; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_193 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_193 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_192; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_192 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_192 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_191; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_191 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_191 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_190; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_190 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_190 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_189; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_189 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_189 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_188; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_188 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_188 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_187; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_187 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_187 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_186; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_186 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_186 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_185; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_185 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_185 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_184; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_184 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_184 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_183; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_183 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_183 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_182; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_182 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_182 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_181; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_181 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_181 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_180; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_180 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_180 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_179; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_179 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_179 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_178; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_178 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_178 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_177; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_177 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_177 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_176; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_176 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_176 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_175; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_175 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_175 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_174; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_174 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_174 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_173; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_173 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_173 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_172; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_172 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_172 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_171; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_171 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_171 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_170; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_170 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_170 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_169; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_169 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_169 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_168; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_168 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_168 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_167; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_167 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_167 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_166; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_166 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_166 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_165; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_165 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_165 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_164; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_164 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_164 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_163; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_163 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_163 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_162; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_162 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_162 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_161; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_161 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_161 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_160; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_160 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_160 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_159; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_159 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_159 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_158; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_158 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_158 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_157; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_157 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_157 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_156; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_156 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_156 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_155; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_155 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_155 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_154; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_154 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_154 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_153; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_153 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_153 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_152; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_152 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_152 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_151; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_151 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_151 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_150; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_150 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_150 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_149; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_149 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_149 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_148; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_148 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_148 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_147; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_147 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_147 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_146; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_146 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_146 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_145; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_145 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_145 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_144; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_144 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_144 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_143; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_143 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_143 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_142; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_142 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_142 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_141; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_141 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_141 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_140; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_140 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_140 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_139; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_139 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_139 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_138; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_138 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_138 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_137; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_137 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_137 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_136; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_136 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_136 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_135; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_135 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_135 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_134; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_134 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_134 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_133; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_133 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_133 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_132; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_132 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_132 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_131; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_131 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_131 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_130; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_130 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_130 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_129; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_129 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_129 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_128; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_128 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_128 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_127; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_127 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_127 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_126; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_126 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_126 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_125; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_125 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_125 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_124; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_124 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_124 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_123; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_123 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_123 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_122; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_122 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_122 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_121; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_121 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_121 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_120; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_120 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_120 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_119; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_119 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_119 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_118; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_118 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_118 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_117; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_117 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_117 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_116; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_116 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_116 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_115; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_115 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_115 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_114; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_114 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_114 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_113; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_113 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_113 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_112; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_112 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_112 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_111; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_111 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_111 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_110; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_110 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_110 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_109; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_109 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_109 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_108; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_108 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_108 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_107; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_107 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_107 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_106; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_106 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_106 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_105; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_105 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_105 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_104; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_104 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_104 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_103; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_103 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_103 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_102; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_102 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_102 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_101; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_101 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_101 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_100; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_100 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_100 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_99; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_99 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_99 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_98; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_98 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_98 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_97; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_97 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_97 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_96; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_96 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_96 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_95; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_95 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_95 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_94; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_94 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_94 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_93; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_93 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_93 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_92; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_92 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_92 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_91; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_91 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_91 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_90; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_90 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_90 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_89; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_89 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_89 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_88; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_88 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_88 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_87; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_87 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_87 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_86; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_86 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_86 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_85; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_85 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_85 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_84; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_84 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_84 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_83; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_83 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_83 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_82; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_82 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_82 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_81; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_81 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_81 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_80; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_80 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_80 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_79; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_79 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_79 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_78; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_78 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_78 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_77; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_77 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_77 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_76; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_76 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_76 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_75; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_75 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_75 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_74; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_74 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_74 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_73; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_73 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_73 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_72; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_72 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_72 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_71; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_71 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_71 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_70; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_70 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_70 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_69; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_69 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_69 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_68; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_68 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_68 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_67; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_67 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_67 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_66; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_66 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_66 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_65; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_65 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_65 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_64; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_64 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_64 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_63; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_63 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_63 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_62; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_62 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_62 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_61; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_61 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_61 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_60; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_60 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_60 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_59; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_59 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_59 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_58; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_58 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_58 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_57; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_57 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_57 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_56; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_56 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_56 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_55; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_55 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_55 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_54; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_54 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_54 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_53; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_53 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_53 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_52; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_52 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_52 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_51; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_51 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_51 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_50; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_50 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_50 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_49; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_49 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_49 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_48; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_48 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_48 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_47; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_47 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_47 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_46; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_46 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_46 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_45; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_45 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_45 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_44; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_44 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_44 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_43; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_43 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_43 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_42; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_42 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_42 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_41; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_41 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_41 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_40; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_40 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_40 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_39; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_39 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_39 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_38; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_38 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_38 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_37; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_37 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_37 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_36; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_36 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_36 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_35; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_35 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_35 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_34; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_34 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_34 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_33; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_33 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_33 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_32; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_32 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_32 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_31; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_31 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_31 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_30; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_30 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_30 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_29; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_29 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_29 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_28; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_28 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_28 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_27; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_27 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_27 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_26; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_26 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_26 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_25; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_25 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_25 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_24; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_24 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_24 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_23; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_23 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_23 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_22; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_22 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_22 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_21; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_21 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_21 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_20; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_20 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_20 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_19; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_19 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_19 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_18; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_18 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_18 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_17; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_17 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_17 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_16; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_16 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_16 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_15; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_15 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_15 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_14; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_14 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_14 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_13; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_13 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_13 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_12; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_12 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_12 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_11; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_11 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_11 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_10; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_10 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_10 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_9; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_9 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_9 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_8; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_8 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_8 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_7; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_7 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_7 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_6; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_6 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_6 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_5; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_5 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_5 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_4; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_4 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_4 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_3; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_3 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_3 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_2; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_2 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_2 1 bkey 1 b A 1 NULL NULL YES BTREE +SHOW INDEX FROM ar_1; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +ar_1 0 PRIMARY 1 a A 1 NULL NULL BTREE +ar_1 1 bkey 1 b A 1 NULL NULL YES BTREE diff --git a/mysql-test/suite/tokudb/r/card_drop_index.result b/mysql-test/suite/tokudb/r/card_drop_index.result index 9fc8fb6a6b8..2cfdfe11296 100644 --- a/mysql-test/suite/tokudb/r/card_drop_index.result +++ b/mysql-test/suite/tokudb/r/card_drop_index.result @@ -5,21 +5,21 @@ insert into tt values (1,0,0),(2,0,0),(3,0,1),(4,0,1); show indexes from tt; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment tt 0 PRIMARY 1 a A 4 NULL NULL BTREE -tt 1 b 1 b A NULL NULL NULL YES BTREE -tt 1 c 1 c A NULL NULL NULL YES BTREE +tt 1 b 1 b A 4 NULL NULL YES BTREE +tt 1 c 1 c A 4 NULL NULL YES BTREE analyze table tt; Table Op Msg_type Msg_text test.tt analyze status OK show indexes from tt; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment tt 0 PRIMARY 1 a A 4 NULL NULL BTREE -tt 1 b 1 b A 1 NULL NULL YES BTREE -tt 1 c 1 c A 2 NULL NULL YES BTREE +tt 1 b 1 b A 2 NULL NULL YES BTREE +tt 1 c 1 c A 4 NULL NULL YES BTREE alter table tt drop key b; show indexes from tt; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment tt 0 PRIMARY 1 a A 4 NULL NULL BTREE -tt 1 c 1 c A 2 NULL NULL YES BTREE +tt 1 c 1 c A 4 NULL NULL YES BTREE alter table tt drop key c; show indexes from tt; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment diff --git a/mysql-test/suite/tokudb/r/card_drop_index_2.result b/mysql-test/suite/tokudb/r/card_drop_index_2.result index 4103a37a6ed..ed28d2a3226 100644 --- a/mysql-test/suite/tokudb/r/card_drop_index_2.result +++ b/mysql-test/suite/tokudb/r/card_drop_index_2.result @@ -132,21 +132,21 @@ count(*) show indexes from tt; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment tt 0 PRIMARY 1 a A 500 NULL NULL BTREE -tt 1 b 1 b A NULL NULL NULL YES BTREE -tt 1 c 1 c A NULL NULL NULL YES BTREE +tt 1 b 1 b A 500 NULL NULL YES BTREE +tt 1 c 1 c A 500 NULL NULL YES BTREE analyze table tt; Table Op Msg_type Msg_text test.tt analyze status OK show indexes from tt; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment tt 0 PRIMARY 1 a A 500 NULL NULL BTREE -tt 1 b 1 b A 125 NULL NULL YES BTREE -tt 1 c 1 c A 1 NULL NULL YES BTREE +tt 1 b 1 b A 250 NULL NULL YES BTREE +tt 1 c 1 c A 2 NULL NULL YES BTREE alter table tt drop key b; show indexes from tt; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment tt 0 PRIMARY 1 a A 500 NULL NULL BTREE -tt 1 c 1 c A 1 NULL NULL YES BTREE +tt 1 c 1 c A 2 NULL NULL YES BTREE alter table tt drop key c; show indexes from tt; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment diff --git a/mysql-test/suite/tokudb/r/card_drop_pk.result b/mysql-test/suite/tokudb/r/card_drop_pk.result index bdbd9a3f097..2369d88c274 100644 --- a/mysql-test/suite/tokudb/r/card_drop_pk.result +++ b/mysql-test/suite/tokudb/r/card_drop_pk.result @@ -5,24 +5,24 @@ insert into tt values (1,0,0),(2,0,0),(3,0,1),(4,0,1); show indexes from tt; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment tt 0 PRIMARY 1 a A 4 NULL NULL BTREE -tt 1 b 1 b A NULL NULL NULL YES BTREE -tt 1 c 1 c A NULL NULL NULL YES BTREE +tt 1 b 1 b A 4 NULL NULL YES BTREE +tt 1 c 1 c A 4 NULL NULL YES BTREE analyze table tt; Table Op Msg_type Msg_text test.tt analyze status OK show indexes from tt; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment tt 0 PRIMARY 1 a A 4 NULL NULL BTREE -tt 1 b 1 b A 1 NULL NULL YES BTREE -tt 1 c 1 c A 2 NULL NULL YES BTREE +tt 1 b 1 b A 2 NULL NULL YES BTREE +tt 1 c 1 c A 4 NULL NULL YES BTREE alter table tt drop primary key; show indexes from tt; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment -tt 1 b 1 b A NULL NULL NULL YES BTREE -tt 1 c 1 c A NULL NULL NULL YES BTREE +tt 1 b 1 b A 4 NULL NULL YES BTREE +tt 1 c 1 c A 4 NULL NULL YES BTREE flush tables; show indexes from tt; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment -tt 1 b 1 b A NULL NULL NULL YES BTREE -tt 1 c 1 c A NULL NULL NULL YES BTREE +tt 1 b 1 b A 4 NULL NULL YES BTREE +tt 1 c 1 c A 4 NULL NULL YES BTREE drop table tt; diff --git a/mysql-test/suite/tokudb/r/card_pk_2.result b/mysql-test/suite/tokudb/r/card_pk_2.result index dd850df9a89..3c1b652db15 100644 --- a/mysql-test/suite/tokudb/r/card_pk_2.result +++ b/mysql-test/suite/tokudb/r/card_pk_2.result @@ -4,18 +4,18 @@ create table tt (a int, b int, primary key(a,b)); insert into tt values (0,0),(0,1),(1,0),(1,1); show indexes from tt; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment -tt 0 PRIMARY 1 a A NULL NULL NULL BTREE +tt 0 PRIMARY 1 a A 4 NULL NULL BTREE tt 0 PRIMARY 2 b A 4 NULL NULL BTREE analyze table tt; Table Op Msg_type Msg_text test.tt analyze status OK show indexes from tt; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment -tt 0 PRIMARY 1 a A 2 NULL NULL BTREE +tt 0 PRIMARY 1 a A 4 NULL NULL BTREE tt 0 PRIMARY 2 b A 4 NULL NULL BTREE flush tables; show indexes from tt; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment -tt 0 PRIMARY 1 a A 2 NULL NULL BTREE +tt 0 PRIMARY 1 a A 4 NULL NULL BTREE tt 0 PRIMARY 2 b A 4 NULL NULL BTREE drop table tt; diff --git a/mysql-test/suite/tokudb/r/card_pk_sk.result b/mysql-test/suite/tokudb/r/card_pk_sk.result index 5458f19de32..02c8d1f8218 100644 --- a/mysql-test/suite/tokudb/r/card_pk_sk.result +++ b/mysql-test/suite/tokudb/r/card_pk_sk.result @@ -1004,7 +1004,7 @@ insert into tt values (4*999,4*999+1),(4*999+1,4*999+2),(4*999+2,4*999+3),(4*999 show indexes from tt; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment tt 0 PRIMARY 1 a A 4000 NULL NULL BTREE -tt 1 b 1 b A NULL NULL NULL YES BTREE +tt 1 b 1 b A 4000 NULL NULL YES BTREE analyze table tt; Table Op Msg_type Msg_text test.tt analyze status OK @@ -2022,17 +2022,17 @@ insert into tt values (4*999,0),(4*999+1,0),(4*999+2,0),(4*999+3,0); show indexes from tt; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment tt 0 PRIMARY 1 a A 4000 NULL NULL BTREE -tt 1 b 1 b A NULL NULL NULL YES BTREE +tt 1 b 1 b A 4000 NULL NULL YES BTREE analyze table tt; Table Op Msg_type Msg_text test.tt analyze status OK show indexes from tt; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment tt 0 PRIMARY 1 a A 4000 NULL NULL BTREE -tt 1 b 1 b A 1 NULL NULL YES BTREE +tt 1 b 1 b A 2 NULL NULL YES BTREE flush tables; show indexes from tt; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment tt 0 PRIMARY 1 a A 4000 NULL NULL BTREE -tt 1 b 1 b A 1 NULL NULL YES BTREE +tt 1 b 1 b A 2 NULL NULL YES BTREE drop table tt; diff --git a/mysql-test/suite/tokudb/r/card_scale_percent.result b/mysql-test/suite/tokudb/r/card_scale_percent.result new file mode 100644 index 00000000000..cfd7e38179c --- /dev/null +++ b/mysql-test/suite/tokudb/r/card_scale_percent.result @@ -0,0 +1,42 @@ +set global tokudb_cardinality_scale_percent = 10; +analyze table tt; +Table Op Msg_type Msg_text +test.tt analyze status OK +show indexes from tt; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +tt 0 PRIMARY 1 a A 4000 NULL NULL BTREE +tt 1 b 1 b A 4000 NULL NULL YES BTREE +tt 1 c 1 c A 4000 NULL NULL YES BTREE +tt 1 d 1 d A 4000 NULL NULL YES BTREE +set global tokudb_cardinality_scale_percent = 50; +analyze table tt; +Table Op Msg_type Msg_text +test.tt analyze status OK +show indexes from tt; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +tt 0 PRIMARY 1 a A 4000 NULL NULL BTREE +tt 1 b 1 b A 4000 NULL NULL YES BTREE +tt 1 c 1 c A 4000 NULL NULL YES BTREE +tt 1 d 1 d A 2000 NULL NULL YES BTREE +set global tokudb_cardinality_scale_percent = 100; +analyze table tt; +Table Op Msg_type Msg_text +test.tt analyze status OK +show indexes from tt; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +tt 0 PRIMARY 1 a A 4000 NULL NULL BTREE +tt 1 b 1 b A 4000 NULL NULL YES BTREE +tt 1 c 1 c A 2000 NULL NULL YES BTREE +tt 1 d 1 d A 1000 NULL NULL YES BTREE +set global tokudb_cardinality_scale_percent = 200; +Warnings: +Warning 1292 Truncated incorrect tokudb_cardinality_scale_percent value: '200' +analyze table tt; +Table Op Msg_type Msg_text +test.tt analyze status OK +show indexes from tt; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +tt 0 PRIMARY 1 a A 4000 NULL NULL BTREE +tt 1 b 1 b A 4000 NULL NULL YES BTREE +tt 1 c 1 c A 2000 NULL NULL YES BTREE +tt 1 d 1 d A 1000 NULL NULL YES BTREE diff --git a/mysql-test/suite/tokudb/r/card_sk.result b/mysql-test/suite/tokudb/r/card_sk.result index 1846b4e82bc..310fc863a9b 100644 --- a/mysql-test/suite/tokudb/r/card_sk.result +++ b/mysql-test/suite/tokudb/r/card_sk.result @@ -5,15 +5,15 @@ insert into tt values (1,0),(2,1),(3,2),(4,3); insert into tt values (5,0),(6,1),(7,2),(8,3); show indexes from tt; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment -tt 1 b 1 b A NULL NULL NULL YES BTREE +tt 1 b 1 b A 8 NULL NULL YES BTREE analyze table tt; Table Op Msg_type Msg_text test.tt analyze status OK show indexes from tt; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment -tt 1 b 1 b A 4 NULL NULL YES BTREE +tt 1 b 1 b A 8 NULL NULL YES BTREE flush tables; show indexes from tt; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment -tt 1 b 1 b A 4 NULL NULL YES BTREE +tt 1 b 1 b A 8 NULL NULL YES BTREE drop table tt; diff --git a/mysql-test/suite/tokudb/r/card_sk_2.result b/mysql-test/suite/tokudb/r/card_sk_2.result index c087bad3b18..8ff57b63e5d 100644 --- a/mysql-test/suite/tokudb/r/card_sk_2.result +++ b/mysql-test/suite/tokudb/r/card_sk_2.result @@ -4,18 +4,18 @@ create table tt (a int, b int, key(a,b)); insert into tt values (0,0),(0,1),(1,0),(1,1); show indexes from tt; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment -tt 1 a 1 a A NULL NULL NULL YES BTREE -tt 1 a 2 b A NULL NULL NULL YES BTREE +tt 1 a 1 a A 4 NULL NULL YES BTREE +tt 1 a 2 b A 4 NULL NULL YES BTREE analyze table tt; Table Op Msg_type Msg_text test.tt analyze status OK show indexes from tt; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment -tt 1 a 1 a A 2 NULL NULL YES BTREE +tt 1 a 1 a A 4 NULL NULL YES BTREE tt 1 a 2 b A 4 NULL NULL YES BTREE flush tables; show indexes from tt; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment -tt 1 a 1 a A 2 NULL NULL YES BTREE +tt 1 a 1 a A 4 NULL NULL YES BTREE tt 1 a 2 b A 4 NULL NULL YES BTREE drop table tt; diff --git a/mysql-test/suite/tokudb/r/cluster_2968-0.result b/mysql-test/suite/tokudb/r/cluster_2968-0.result index 954d93b5f73..e1effd0aba8 100644 --- a/mysql-test/suite/tokudb/r/cluster_2968-0.result +++ b/mysql-test/suite/tokudb/r/cluster_2968-0.result @@ -1042,5 +1042,5 @@ t CREATE TABLE `t` ( explain select straight_join * from s,t where s.b = t.b; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE s index b b 5 NULL 1000 Using where -1 SIMPLE t ref b b 5 test.s.b 11 NULL +1 SIMPLE t ref b b 5 test.s.b 1 NULL drop table s,t; diff --git a/mysql-test/suite/tokudb/r/cluster_2968-1.result b/mysql-test/suite/tokudb/r/cluster_2968-1.result index 903504db389..8c0b64217ca 100644 --- a/mysql-test/suite/tokudb/r/cluster_2968-1.result +++ b/mysql-test/suite/tokudb/r/cluster_2968-1.result @@ -1042,7 +1042,7 @@ t CREATE TABLE `t` ( explain select straight_join * from s,t where s.b = t.b; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE s ALL b NULL NULL NULL 1000 Using where -1 SIMPLE t ref b b 5 test.s.b 11 NULL +1 SIMPLE t ref b b 5 test.s.b 1 NULL alter table s add clustering key(b); alter table t add clustering key(b); show create table s; @@ -1066,7 +1066,7 @@ t CREATE TABLE `t` ( explain select straight_join * from s,t where s.b = t.b; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE s index b,b_2 b_2 5 NULL 1000 Using where -1 SIMPLE t ref b,b_2 b_2 5 test.s.b 11 NULL +1 SIMPLE t ref b,b_2 b 5 test.s.b 1 NULL alter table s drop key b; alter table t drop key b; show create table s; @@ -1088,7 +1088,7 @@ t CREATE TABLE `t` ( explain select straight_join * from s,t where s.b = t.b; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE s index b_2 b_2 5 NULL 1000 Using where -1 SIMPLE t ref b_2 b_2 5 test.s.b 11 NULL +1 SIMPLE t ref b_2 b_2 5 test.s.b 1 NULL alter table s add key(b); alter table t add key(b); show create table s; @@ -1112,5 +1112,5 @@ t CREATE TABLE `t` ( explain select straight_join * from s,t where s.b = t.b; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE s index b_2,b b_2 5 NULL 1000 Using where -1 SIMPLE t ref b_2,b b_2 5 test.s.b 11 NULL +1 SIMPLE t ref b_2,b b 5 test.s.b 1 NULL drop table s,t; diff --git a/mysql-test/suite/tokudb/r/cluster_2968-2.result b/mysql-test/suite/tokudb/r/cluster_2968-2.result index 854226a14f8..63718aa2fda 100644 --- a/mysql-test/suite/tokudb/r/cluster_2968-2.result +++ b/mysql-test/suite/tokudb/r/cluster_2968-2.result @@ -1042,7 +1042,7 @@ t CREATE TABLE `t` ( explain select straight_join s.a,t.a from s,t where s.b = t.b; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE s ALL b NULL NULL NULL 1000 Using where -1 SIMPLE t ref b b 5 test.s.b 11 NULL +1 SIMPLE t ref b b 5 test.s.b 1 NULL alter table s add key(b,a); alter table t add key(b,a); show create table s; @@ -1066,7 +1066,7 @@ t CREATE TABLE `t` ( explain select straight_join s.a,t.a from s,t where s.b = t.b; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE s index b,b_2 b_2 10 NULL 1000 Using where; Using index -1 SIMPLE t ref b,b_2 b_2 5 test.s.b 11 Using index +1 SIMPLE t ref b,b_2 b 5 test.s.b 1 NULL alter table s add clustering key(b); alter table t add clustering key(b); show create table s; @@ -1092,7 +1092,7 @@ t CREATE TABLE `t` ( explain select straight_join s.a,t.a from s,t where s.b = t.b; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE s index b,b_2,b_3 b_2 10 NULL 1000 Using where; Using index -1 SIMPLE t ref b,b_2,b_3 b_2 5 test.s.b 11 Using index +1 SIMPLE t ref b,b_2,b_3 b 5 test.s.b 1 NULL alter table s drop key b_2; alter table t drop key b_2; show create table s; @@ -1116,5 +1116,5 @@ t CREATE TABLE `t` ( explain select straight_join s.a,t.a from s,t where s.b = t.b; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE s index b,b_3 b_3 5 NULL 1000 Using where -1 SIMPLE t ref b,b_3 b_3 5 test.s.b 11 NULL +1 SIMPLE t ref b,b_3 b 5 test.s.b 1 NULL drop table s,t; diff --git a/mysql-test/suite/tokudb/r/cluster_2968-3.result b/mysql-test/suite/tokudb/r/cluster_2968-3.result index 5a2fbf2ccc2..adc542cd14b 100644 --- a/mysql-test/suite/tokudb/r/cluster_2968-3.result +++ b/mysql-test/suite/tokudb/r/cluster_2968-3.result @@ -1062,8 +1062,8 @@ u CREATE TABLE `u` ( explain select straight_join * from s,t,u where s.b = t.b and s.c = u.c; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE s ALL b NULL NULL NULL 1000 Using where -1 SIMPLE t ref b b 5 test.s.b 11 NULL -1 SIMPLE u ref c c 5 test.s.c 11 NULL +1 SIMPLE t ref b b 5 test.s.b 1 NULL +1 SIMPLE u ref c c 5 test.s.c 1 NULL alter table s add clustering key (b); alter table t add clustering key (b); alter table u add clustering key (c); @@ -1097,6 +1097,6 @@ u CREATE TABLE `u` ( explain select straight_join * from s,t,u where s.b = t.b and s.c = u.c; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE s index b,b_2 b_2 5 NULL 1000 Using where -1 SIMPLE t ref b,b_2 b_2 5 test.s.b 11 NULL -1 SIMPLE u ref c,c_2 c_2 5 test.s.c 11 NULL +1 SIMPLE t ref b,b_2 b 5 test.s.b 1 NULL +1 SIMPLE u ref c,c_2 c 5 test.s.c 1 NULL drop table s,t,u; diff --git a/mysql-test/suite/tokudb/r/type_bit.result b/mysql-test/suite/tokudb/r/type_bit.result index d6a82e4cf29..f3285b8c3e5 100644 --- a/mysql-test/suite/tokudb/r/type_bit.result +++ b/mysql-test/suite/tokudb/r/type_bit.result @@ -675,7 +675,7 @@ INSERT INTO t1(a) VALUES (65535),(65525),(65535),(65535),(65535),(65535),(65535),(65535),(65535),(65535); EXPLAIN SELECT 1 FROM t1 GROUP BY a; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range a a 3 NULL 6 Using index for group-by +1 SIMPLE t1 index a a 3 NULL 10 Using index SELECT 1 FROM t1 GROUP BY a; 1 1 diff --git a/mysql-test/suite/tokudb/t/background_job_manager.test b/mysql-test/suite/tokudb/t/background_job_manager.test new file mode 100644 index 00000000000..933814442e0 --- /dev/null +++ b/mysql-test/suite/tokudb/t/background_job_manager.test @@ -0,0 +1,139 @@ +# This is a comprehensive test for the background job manager and +# the information_schema.tokudb_background_job_status table +# +# This test validates that analyze table in various modes operate as expected +# for both foreground and background jobs. +# +# This test is NOT intended to test the actual results of an analysis. +# +# This test makes use of a global, debug only tokudb variable +# tokudb_debug_pause_background_job_manager in order to control the bjm and +# prevent it from acting on any queued jobs. +# This variable was necessary since the debug_sync facility requires any thread +# that is syncing to have a valid THD associated with it, which a background +# thread would not have. This variable is compiled out of release builds. + +-- source include/have_tokudb.inc +-- source include/have_debug.inc + +-- enable_query_log + +set @orig_auto_analyze = @@session.tokudb_auto_analyze; +set @orig_in_background = @@session.tokudb_analyze_in_background; +set @orig_mode = @@session.tokudb_analyze_mode; +set @orig_throttle = @@session.tokudb_analyze_throttle; +set @orig_time = @@session.tokudb_analyze_time; +set @orig_scale_percent = @@global.tokudb_cardinality_scale_percent; +set @orig_default_storage_engine = @@session.default_storage_engine; +set @orig_pause_background_job_manager = @@global.tokudb_debug_pause_background_job_manager; + +# first, lets set up to auto analyze in the background with about any activity +set session default_storage_engine='tokudb'; +set session tokudb_auto_analyze=1; +set session tokudb_analyze_in_background=1; +set session tokudb_analyze_mode=tokudb_analyze_standard; +set session tokudb_analyze_throttle=0; +set session tokudb_analyze_time=0; +set global tokudb_cardinality_scale_percent=DEFAULT; + +# in debug build, we can prevent the background job manager from running, +# let's do it so we can see that there was an analyze scheduled on the first +# insert +set global tokudb_debug_pause_background_job_manager=TRUE; + +# let's see what the i_s table is laid out like +show create table information_schema.tokudb_background_job_status; + +create table t1 (a int not null auto_increment, b int, c int, primary key(a), key kb(b), key kc(c), key kabc(a,b,c), key kab(a,b), key kbc(b,c)); + +insert into t1(b,c) values(0,0), (1,1), (2,2), (3,3); + +# insert above should have triggered an analyze, but since the bjm is paused, +# we will see it sitting in the queue +select database_name, table_name, job_type, job_params, scheduler from information_schema.tokudb_background_job_status; + +# some more tables +create table t2 like t1; +create table t3 like t1; +create table t4 like t1; + +# manually analyze, the t1 should be rejected because there is already a job +# pending. t2, t3 and t4 should get queued. +analyze table t1; +analyze table t2; +analyze table t3; +analyze table t4; + +select database_name, table_name, job_type, job_params, scheduler from information_schema.tokudb_background_job_status; + +# let the bjm go to clear the jobs +set global tokudb_debug_pause_background_job_manager=FALSE; + +# wait for the bjm queue to empty +-- disable_query_log +let $wait_condition=select count(*)=0 from information_schema.tokudb_background_job_status; +-- source include/wait_condition.inc +-- enable_query_log + +# pause the bjm again +set global tokudb_debug_pause_background_job_manager=TRUE; + +# add some new jobs +analyze table t1; +analyze table t2; +analyze table t3; +analyze table t4; + +select database_name, table_name, job_type, job_params, scheduler from information_schema.tokudb_background_job_status; + +# alter a table, should kill the job for t1 +alter table t1 add column d int; + +select database_name, table_name, job_type, job_params, scheduler from information_schema.tokudb_background_job_status; + +# try an explicit cancel on t2 +set session tokudb_analyze_mode=tokudb_analyze_cancel; +analyze table t2; + +select database_name, table_name, job_type, job_params, scheduler from information_schema.tokudb_background_job_status; + +# try a recount on t1, should reschedule a new job +set session tokudb_analyze_mode=tokudb_analyze_recount_rows; +analyze table t1; + +select database_name, table_name, job_type, job_params, scheduler from information_schema.tokudb_background_job_status; + +# do a foreground analysis that clashes with a background job, it should +# kill the background job for t3 and perform the analysis immediately +set session tokudb_analyze_mode=tokudb_analyze_standard; +set session tokudb_analyze_in_background=0; +analyze table t3; + +select database_name, table_name, job_type, job_params, scheduler from information_schema.tokudb_background_job_status; + +# drop the tables, should kill the remaining jobs for t1, and t4 +drop table t1; +drop table t2; +drop table t3; +drop table t4; + +select database_name, table_name, job_type, job_params, scheduler from information_schema.tokudb_background_job_status; + +# let the bjm go +set global tokudb_debug_pause_background_job_manager=FALSE; + +#cleanup +-- disable_query_log +let $wait_condition=select count(*)=0 from information_schema.tokudb_background_job_status; +-- source include/wait_condition.inc + +set session tokudb_auto_analyze = @orig_auto_analyze; +set session tokudb_analyze_in_background = @orig_in_background; +set session tokudb_analyze_mode = @orig_mode; +set session tokudb_analyze_throttle = @orig_throttle; +set session tokudb_analyze_time = @orig_time; +set global tokudb_cardinality_scale_percent = @orig_scale_percent; +set session default_storage_engine = @orig_default_storage_engine; +set global tokudb_debug_pause_background_job_manager = @orig_pause_background_job_manager; + +-- enable_query_log diff --git a/mysql-test/suite/tokudb/t/card_auto_analyze_lots.test b/mysql-test/suite/tokudb/t/card_auto_analyze_lots.test new file mode 100644 index 00000000000..ec74a4a28bc --- /dev/null +++ b/mysql-test/suite/tokudb/t/card_auto_analyze_lots.test @@ -0,0 +1,82 @@ +# Test the auto analyze on lots of tables +-- source include/have_tokudb.inc + +-- disable_query_log +let $max = 200; + +SET @orig_auto_analyze = @@session.tokudb_auto_analyze; +SET @orig_in_background = @@session.tokudb_analyze_in_background; +SET @orig_mode = @@session.tokudb_analyze_mode; +SET @orig_throttle = @@session.tokudb_analyze_throttle; +SET @orig_time = @@session.tokudb_analyze_time; + +SET SESSION tokudb_auto_analyze = 1; +SET SESSION tokudb_analyze_in_background = 0; +SET SESSION tokudb_analyze_mode = TOKUDB_ANALYZE_STANDARD; +SET SESSION tokudb_analyze_throttle = 0; +SET SESSION tokudb_analyze_time = 0; + +let $i = $max; +while ($i > 0) { + eval CREATE TABLE ar_$i (a INT, b INT, PRIMARY KEY (a), KEY bkey (b)) ENGINE=TOKUDB; + dec $i; +} + +# check that the one row insertion triggered auto analyze within the calling +# client context, the cardinality should go from NULL to 1 +let $i = $max; +while ($i > 0) { + eval INSERT INTO ar_$i VALUES (0, 0); + dec $i; +} +-- enable_query_log +let $i = $max; +while ($i > 0) { + eval SHOW INDEX FROM ar_$i; + dec $i; +} + + +-- disable_query_log +# check that lots of background analysis get scheduled and run +# cleanly and serially in the background +SET SESSION tokudb_auto_analyze = 1; +SET SESSION tokudb_analyze_in_background = 1; +SET SESSION tokudb_analyze_mode = TOKUDB_ANALYZE_STANDARD; +SET SESSION tokudb_analyze_throttle = 0; +SET SESSION tokudb_analyze_time = 0; + +let $i = $max; +while ($i > 0) { + eval INSERT INTO ar_$i VALUES (1, 1), (2, 1), (3, 2), (4, 2); + dec $i; +} + +let $i = $max; +while ($i > 0) { + eval INSERT INTO ar_$i VALUES (5, 3), (6, 3), (7, 4), (8, 4); + dec $i; +} + +# would be too long to wait for stats to become up to date here and +# checking is quite non-deterministic, InnoDB test does same thing + +# dropping tables should cancel any running background jobs +let $i = $max; +while ($i > 0) { + eval DROP TABLE ar_$i; + dec $i; +} + +# wait for the bjm queue to empty +-- disable_query_log +let $wait_condition=select count(*)=0 from information_schema.tokudb_background_job_status; +-- source include/wait_condition.inc + +SET SESSION tokudb_auto_analyze = @orig_auto_analyze; +SET SESSION tokudb_analyze_in_background = @orig_in_background; +SET SESSION tokudb_analyze_mode = @orig_mode; +SET SESSION tokudb_analyze_throttle = @orig_throttle; +SET SESSION tokudb_analyze_time = @orig_time; + +-- enable_query_log diff --git a/mysql-test/suite/tokudb/t/card_scale_percent.test b/mysql-test/suite/tokudb/t/card_scale_percent.test new file mode 100644 index 00000000000..47f1eb37989 --- /dev/null +++ b/mysql-test/suite/tokudb/t/card_scale_percent.test @@ -0,0 +1,56 @@ +-- source include/have_tokudb.inc + +-- disable_query_log + +set @orig_throttle = @@session.tokudb_analyze_throttle; +set @orig_time = @@session.tokudb_analyze_time; +set @orig_scale_percent = @@global.tokudb_cardinality_scale_percent; + +create table tt (a int, b int, c int, d int, primary key(a), key(b), key(c), key(d)) engine=tokudb; +let $i=0; +while ($i < 1000) { + eval insert into tt values ($i, $i, $i, $i); + inc $i; +} +while ($i < 2000) { + eval insert into tt values ($i, $i, $i, 0); + inc $i; +} +while ($i < 3000) { + eval insert into tt values ($i, $i, 0, 0); + inc $i; +} +while ($i < 4000) { + eval insert into tt values ($i, 0, 0, 0); + inc $i; +} + +set session tokudb_analyze_time=0; +set session tokudb_analyze_throttle=0; + +-- enable_query_log + +set global tokudb_cardinality_scale_percent = 10; +analyze table tt; +show indexes from tt; + +set global tokudb_cardinality_scale_percent = 50; +analyze table tt; +show indexes from tt; + +set global tokudb_cardinality_scale_percent = 100; +analyze table tt; +show indexes from tt; + +set global tokudb_cardinality_scale_percent = 200; +analyze table tt; +show indexes from tt; + +-- disable_query_log + +drop table tt; +set session tokudb_analyze_throttle = @orig_throttle; +set session tokudb_analyze_time = @orig_time; +set global tokudb_cardinality_scale_percent = @orig_scale_percent; + +-- enable_query_log diff --git a/mysql-test/suite/tokudb/t/disabled.def b/mysql-test/suite/tokudb/t/disabled.def index 751794ac690..f65151ecd54 100644 --- a/mysql-test/suite/tokudb/t/disabled.def +++ b/mysql-test/suite/tokudb/t/disabled.def @@ -2,3 +2,26 @@ mvcc-19: tokutek mvcc-20: tokutek mvcc-27: tokutek storage_engine_default: tokudb is not the default storage engine +cluster_key_part : https://tokutek.atlassian.net/browse/DB-720 +fast_update_blobs : https://tokutek.atlassian.net/browse/DB-871 +fast_update_blobs_fixed_varchar : https://tokutek.atlassian.net/browse/DB-871 +fast_update_blobs_with_varchar : https://tokutek.atlassian.net/browse/DB-871 +fast_update_char : https://tokutek.atlassian.net/browse/DB-871 +fast_update_decr_floor : https://tokutek.atlassian.net/browse/DB-871 +fast_update_int : https://tokutek.atlassian.net/browse/DB-871 +fast_update_int_bounds : https://tokutek.atlassian.net/browse/DB-871 +fast_update_uint_bounds : https://tokutek.atlassian.net/browse/DB-871 +fast_update_varchar : https://tokutek.atlassian.net/browse/DB-871 +fast_upsert_char : https://tokutek.atlassian.net/browse/DB-871 +fast_upsert_int : https://tokutek.atlassian.net/browse/DB-871 +fast_update_binlog_statement : https://tokutek.atlassian.net/browse/DB-871 +fast_update_deadlock : https://tokutek.atlassian.net/browse/DB-871 +fast_update_error : https://tokutek.atlassian.net/browse/DB-871 +fast_update_disable_slow_update : https://tokutek.atlassian.net/browse/DB-871 +fast_update_key : https://tokutek.atlassian.net/browse/DB-871 +fast_update_sqlmode : https://tokutek.atlassian.net/browse/DB-871 +fast_upsert_bin_pad : https://tokutek.atlassian.net/browse/DB-871 +fast_upsert_deadlock : https://tokutek.atlassian.net/browse/DB-871 +fast_upsert_key : https://tokutek.atlassian.net/browse/DB-871 +fast_upsert_sqlmode : https://tokutek.atlassian.net/browse/DB-871 +fast_upsert_values : https://tokutek.atlassian.net/browse/DB-871 diff --git a/mysql-test/suite/tokudb/t/rows-32m-seq-insert.test b/mysql-test/suite/tokudb/t/rows-32m-seq-insert.test index 5b1f5440452..7ee84c69eca 100644 --- a/mysql-test/suite/tokudb/t/rows-32m-seq-insert.test +++ b/mysql-test/suite/tokudb/t/rows-32m-seq-insert.test @@ -1,3 +1,4 @@ +source include/big_test.inc; source include/have_tokudb.inc; # do a lot of longblob insertions up to 32MB-4 in size with seq primary key @@ -34,4 +35,4 @@ optimize table t; check table t; -drop table t;
\ No newline at end of file +drop table t; diff --git a/mysql-test/suite/tokudb/t/suite.opt b/mysql-test/suite/tokudb/t/suite.opt new file mode 100644 index 00000000000..23511b05020 --- /dev/null +++ b/mysql-test/suite/tokudb/t/suite.opt @@ -0,0 +1 @@ +$TOKUDB_OPT $TOKUDB_LOAD_ADD --loose-tokudb-check-jemalloc=0 diff --git a/storage/tokudb/CMakeLists.txt b/storage/tokudb/CMakeLists.txt index 7b6a024f63e..37de19c3c19 100644 --- a/storage/tokudb/CMakeLists.txt +++ b/storage/tokudb/CMakeLists.txt @@ -1,8 +1,7 @@ -SET(TOKUDB_VERSION 5.6.26-74.0) +SET(TOKUDB_VERSION 5.6.27-76.0) # PerconaFT only supports x86-64 and cmake-2.8.9+ IF(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND - NOT CMAKE_VERSION VERSION_LESS "2.8.9" AND - NOT WITHOUT_TOKUDB AND NOT WITHOUT_TOKUDB_STORAGE_ENGINE) + NOT CMAKE_VERSION VERSION_LESS "2.8.9") CHECK_CXX_SOURCE_COMPILES( " struct a {int b; int c; }; @@ -11,14 +10,30 @@ int main() { return 0; } " TOKUDB_OK) ENDIF() -IF(NOT TOKUDB_OK) +IF(NOT TOKUDB_OK OR WITHOUT_TOKUDB OR WITHOUT_TOKUDB_STORAGE_ENGINE) RETURN() ENDIF() +## adds a compiler flag if the compiler supports it +include(CheckCCompilerFlag) +include(CheckCXXCompilerFlag) + +# pick language dialect +check_cxx_compiler_flag(-std=c++11 HAVE_STDCXX11) +if (HAVE_STDCXX11) + set(CMAKE_CXX_FLAGS "-std=c++11 ${CMAKE_CXX_FLAGS}") +else () + message(FATAL_ERROR "${CMAKE_CXX_COMPILER} doesn't support -std=c++11, you need one that does.") +endif () + + SET(BUILD_TESTING OFF CACHE BOOL "") SET(USE_VALGRIND OFF CACHE BOOL "") SET(TOKU_DEBUG_PARANOID OFF CACHE BOOL "") +# Enable TokuDB's TOKUDB_DEBUG in debug builds +SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DTOKUDB_DEBUG") + IF(NOT DEFINED TOKUDB_VERSION) IF(DEFINED ENV{TOKUDB_VERSION}) SET(TOKUDB_VERSION $ENV{TOKUDB_VERSION}) @@ -41,10 +56,6 @@ IF(DEFINED TOKUDB_CHECK_JEMALLOC) ADD_DEFINITIONS("-DTOKUDB_CHECK_JEMALLOC=${TOKUDB_CHECK_JEMALLOC}") ENDIF() -## adds a compiler flag if the compiler supports it -include(CheckCCompilerFlag) -include(CheckCXXCompilerFlag) - macro(set_cflags_if_supported) foreach(flag ${ARGN}) string(REGEX REPLACE "-" "_" temp_flag ${flag}) @@ -99,8 +110,13 @@ INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}/${TOKU_FT_DIR_NAME}/portability) SET(TOKUDB_PLUGIN_DYNAMIC "ha_tokudb") -SET(TOKUDB_SOURCES ha_tokudb.cc) +SET(TOKUDB_SOURCES + ha_tokudb.cc + tokudb_background.cc + tokudb_information_schema.cc + tokudb_sysvars.cc + tokudb_thread.cc) MYSQL_ADD_PLUGIN(tokudb ${TOKUDB_SOURCES} STORAGE_ENGINE MODULE_ONLY LINK_LIBRARIES tokufractaltree_static tokuportability_static ${ZLIB_LIBRARY} stdc++) -SET_PROPERTY(TARGET tokudb APPEND PROPERTY LINK_FLAGS_RELEASE "-flto -fuse-linker-plugin") -SET_PROPERTY(TARGET tokudb APPEND PROPERTY LINK_FLAGS_RELWITHDEBINFO "-flto -fuse-linker-plugin") +SET(CMAKE_MODULE_LINKER_FLAGS_RELEASE "${CMAKE_MODULE_LINKER_FLAGS_RELEASE} -flto -fuse-linker-plugin") +SET(CMAKE_MODULE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_MODULE_LINKER_FLAGS_RELWITHDEBINFO} -flto -fuse-linker-plugin") diff --git a/storage/tokudb/PerconaFT/CMakeLists.txt b/storage/tokudb/PerconaFT/CMakeLists.txt index 788b3ef3d9d..59e0d145ce0 100644 --- a/storage/tokudb/PerconaFT/CMakeLists.txt +++ b/storage/tokudb/PerconaFT/CMakeLists.txt @@ -1,4 +1,6 @@ -cmake_minimum_required(VERSION 2.8.8 FATAL_ERROR) +if (CMAKE_PROJECT_NAME STREQUAL TokuDB) + cmake_minimum_required(VERSION 2.8.8 FATAL_ERROR) +endif() set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules") project(TokuDB) diff --git a/storage/tokudb/PerconaFT/CTestCustom.cmake b/storage/tokudb/PerconaFT/CTestCustom.cmake.in index 54170b2b903..54170b2b903 100644 --- a/storage/tokudb/PerconaFT/CTestCustom.cmake +++ b/storage/tokudb/PerconaFT/CTestCustom.cmake.in diff --git a/storage/tokudb/PerconaFT/buildheader/CMakeLists.txt b/storage/tokudb/PerconaFT/buildheader/CMakeLists.txt index d9629f2cc68..5da3c98ff48 100644 --- a/storage/tokudb/PerconaFT/buildheader/CMakeLists.txt +++ b/storage/tokudb/PerconaFT/buildheader/CMakeLists.txt @@ -4,7 +4,7 @@ file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/runcat.sh" "#!/bin/bash out=$1; shift exec \"$@\" >$out") -add_executable(make_tdb make_tdb) +add_executable(make_tdb make_tdb.cc) set_property(TARGET make_tdb APPEND PROPERTY COMPILE_DEFINITIONS _GNU_SOURCE) add_custom_command( OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/db.h" @@ -26,4 +26,4 @@ if (NOT DEFINED MYSQL_PROJECT_NAME_DOCSTRING) DESTINATION include COMPONENT tokukv_headers ) -endif ()
\ No newline at end of file +endif () diff --git a/storage/tokudb/PerconaFT/buildheader/make_tdb.cc b/storage/tokudb/PerconaFT/buildheader/make_tdb.cc index 958f00a8706..5c29209e19d 100644 --- a/storage/tokudb/PerconaFT/buildheader/make_tdb.cc +++ b/storage/tokudb/PerconaFT/buildheader/make_tdb.cc @@ -510,8 +510,9 @@ static void print_db_struct (void) { "int (*update_broadcast)(DB *, DB_TXN*, const DBT *extra, uint32_t flags)", "int (*get_fractal_tree_info64)(DB*,uint64_t*,uint64_t*,uint64_t*,uint64_t*)", "int (*iterate_fractal_tree_block_map)(DB*,int(*)(uint64_t,int64_t,int64_t,int64_t,int64_t,void*),void*)", - "const char *(*get_dname)(DB *db)", - "int (*get_last_key)(DB *db, YDB_CALLBACK_FUNCTION func, void* extra)", + "const char *(*get_dname)(DB *db)", + "int (*get_last_key)(DB *db, YDB_CALLBACK_FUNCTION func, void* extra)", + "int (*recount_rows)(DB* db, int (*progress_callback)(uint64_t count, uint64_t deleted, void* progress_extra), void* progress_extra)", NULL}; sort_and_dump_fields("db", true, extra); } diff --git a/storage/tokudb/PerconaFT/cmake_modules/FindValgrind.cmake b/storage/tokudb/PerconaFT/cmake_modules/FindValgrind.cmake index 03335c5bbe8..73841723ce9 100644 --- a/storage/tokudb/PerconaFT/cmake_modules/FindValgrind.cmake +++ b/storage/tokudb/PerconaFT/cmake_modules/FindValgrind.cmake @@ -11,7 +11,7 @@ find_path(VALGRIND_INCLUDE_DIR valgrind/memcheck.h) find_program(VALGRIND_PROGRAM NAMES valgrind) -find_package_handle_standard_args(VALGRIND DEFAULT_MSG +find_package_handle_standard_args(Valgrind DEFAULT_MSG VALGRIND_INCLUDE_DIR VALGRIND_PROGRAM) diff --git a/storage/tokudb/PerconaFT/cmake_modules/TokuMergeLibs.cmake b/storage/tokudb/PerconaFT/cmake_modules/TokuMergeLibs.cmake index 9dc1c493513..e1da095fc00 100644 --- a/storage/tokudb/PerconaFT/cmake_modules/TokuMergeLibs.cmake +++ b/storage/tokudb/PerconaFT/cmake_modules/TokuMergeLibs.cmake @@ -48,7 +48,8 @@ MACRO(TOKU_MERGE_STATIC_LIBS TARGET OUTPUT_NAME LIBS_TO_MERGE) ENDIF() ENDFOREACH() IF(OSLIBS) - LIST(REMOVE_DUPLICATES OSLIBS) + # REMOVE_DUPLICATES destroys the order of the libs so disabled + # LIST(REMOVE_DUPLICATES OSLIBS) TARGET_LINK_LIBRARIES(${TARGET} LINK_PUBLIC ${OSLIBS}) ENDIF() diff --git a/storage/tokudb/PerconaFT/cmake_modules/TokuSetupCTest.cmake b/storage/tokudb/PerconaFT/cmake_modules/TokuSetupCTest.cmake index 1912dd33a07..5b6882cc4a1 100644 --- a/storage/tokudb/PerconaFT/cmake_modules/TokuSetupCTest.cmake +++ b/storage/tokudb/PerconaFT/cmake_modules/TokuSetupCTest.cmake @@ -9,6 +9,7 @@ macro(real_executable_name filename_input out) execute_process( COMMAND which ${filename} RESULT_VARIABLE res + ERROR_QUIET OUTPUT_VARIABLE full_filename OUTPUT_STRIP_TRAILING_WHITESPACE) if(NOT(res)) @@ -59,7 +60,7 @@ macro(hostname out) COMMAND hostname OUTPUT_VARIABLE fullhostname OUTPUT_STRIP_TRAILING_WHITESPACE) - string(REGEX REPLACE "\\.tokutek\\.com$" "" ${out} ${fullhostname}) + string(REGEX REPLACE "\\.tokutek\\.com$" "" ${out} "${fullhostname}") endmacro(hostname) ## gather machine info @@ -150,5 +151,5 @@ if (BUILD_TESTING OR BUILD_FT_TESTS OR BUILD_SRC_TESTS) option(RUN_STRESS_TESTS "If set, run the stress tests." OFF) option(RUN_PERF_TESTS "If set, run the perf tests." OFF) - configure_file(CTestCustom.cmake . @ONLY) + configure_file(CTestCustom.cmake.in CTestCustom.cmake @ONLY) endif (BUILD_TESTING OR BUILD_FT_TESTS OR BUILD_SRC_TESTS) diff --git a/storage/tokudb/PerconaFT/cmake_modules/TokuSetupCompiler.cmake b/storage/tokudb/PerconaFT/cmake_modules/TokuSetupCompiler.cmake index c011349c714..87947bb47d2 100644 --- a/storage/tokudb/PerconaFT/cmake_modules/TokuSetupCompiler.cmake +++ b/storage/tokudb/PerconaFT/cmake_modules/TokuSetupCompiler.cmake @@ -24,12 +24,12 @@ endif () ## add TOKU_PTHREAD_DEBUG for debug builds if (CMAKE_VERSION VERSION_LESS 3.0) - set_property(DIRECTORY APPEND PROPERTY COMPILE_DEFINITIONS_DEBUG TOKU_PTHREAD_DEBUG=1) - set_property(DIRECTORY APPEND PROPERTY COMPILE_DEFINITIONS_DRD TOKU_PTHREAD_DEBUG=1) + set_property(DIRECTORY APPEND PROPERTY COMPILE_DEFINITIONS_DEBUG TOKU_PTHREAD_DEBUG=1 TOKU_DEBUG_TXN_SYNC=1) + set_property(DIRECTORY APPEND PROPERTY COMPILE_DEFINITIONS_DRD TOKU_PTHREAD_DEBUG=1 TOKU_DEBUG_TXN_SYNC=1) set_property(DIRECTORY APPEND PROPERTY COMPILE_DEFINITIONS_DRD _FORTIFY_SOURCE=2) else () set_property(DIRECTORY APPEND PROPERTY COMPILE_DEFINITIONS - $<$<OR:$<CONFIG:DEBUG>,$<CONFIG:DRD>>:TOKU_PTHREAD_DEBUG=1> + $<$<OR:$<CONFIG:DEBUG>,$<CONFIG:DRD>>:TOKU_PTHREAD_DEBUG=1 TOKU_DEBUG_TXN_SYNC=1> $<$<CONFIG:DRD>:_FORTIFY_SOURCE=2> ) endif () diff --git a/storage/tokudb/PerconaFT/cmake_modules/TokuThirdParty.cmake b/storage/tokudb/PerconaFT/cmake_modules/TokuThirdParty.cmake index 1e7e47f9bf4..db94b19f08c 100644 --- a/storage/tokudb/PerconaFT/cmake_modules/TokuThirdParty.cmake +++ b/storage/tokudb/PerconaFT/cmake_modules/TokuThirdParty.cmake @@ -41,7 +41,7 @@ if (APPLE) list(APPEND xz_configure_opts --disable-assembler) endif () -list(APPEND xz_configure_opts CC=${CMAKE_C_COMPILER}) +list(APPEND xz_configure_opts "CC=${CMAKE_C_COMPILER} ${CMAKE_C_COMPILER_ARG1}") if (CMAKE_BUILD_TYPE STREQUAL Debug OR CMAKE_BUILD_TYPE STREQUAL drd) list(APPEND xz_configure_opts --enable-debug) endif () @@ -68,6 +68,7 @@ ExternalProject_Add(build_lzma CONFIGURE_COMMAND "<SOURCE_DIR>/configure" ${xz_configure_opts} "--prefix=${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_CFG_INTDIR}/xz" + "--libdir=${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_CFG_INTDIR}/xz/lib" BUILD_COMMAND ${SUBMAKE_COMMAND} -C src/liblzma INSTALL_COMMAND diff --git a/storage/tokudb/PerconaFT/ft/CMakeLists.txt b/storage/tokudb/PerconaFT/ft/CMakeLists.txt index 744b9c9a9e1..11091073ac2 100644 --- a/storage/tokudb/PerconaFT/ft/CMakeLists.txt +++ b/storage/tokudb/PerconaFT/ft/CMakeLists.txt @@ -36,6 +36,7 @@ set(FT_SOURCES ft-flusher ft-hot-flusher ft-ops + ft-recount-rows ft-status ft-test-helpers ft-verify diff --git a/storage/tokudb/PerconaFT/ft/ft-flusher.cc b/storage/tokudb/PerconaFT/ft/ft-flusher.cc index 530947fe868..fb456ea6a18 100644 --- a/storage/tokudb/PerconaFT/ft/ft-flusher.cc +++ b/storage/tokudb/PerconaFT/ft/ft-flusher.cc @@ -1572,6 +1572,7 @@ void toku_bnc_flush_to_child(FT ft, NONLEAF_CHILDINFO bnc, FTNODE child, TXNID p txn_gc_info *gc_info; STAT64INFO_S stats_delta; + int64_t logical_rows_delta = 0; size_t remaining_memsize = bnc->msg_buffer.buffer_size_in_use(); flush_msg_fn(FT t, FTNODE n, NONLEAF_CHILDINFO nl, txn_gc_info *g) : @@ -1599,8 +1600,8 @@ void toku_bnc_flush_to_child(FT ft, NONLEAF_CHILDINFO bnc, FTNODE child, TXNID p is_fresh, gc_info, flow_deltas, - &stats_delta - ); + &stats_delta, + &logical_rows_delta); remaining_memsize -= memsize_in_buffer; return 0; } @@ -1613,6 +1614,7 @@ void toku_bnc_flush_to_child(FT ft, NONLEAF_CHILDINFO bnc, FTNODE child, TXNID p if (flush_fn.stats_delta.numbytes || flush_fn.stats_delta.numrows) { toku_ft_update_stats(&ft->in_memory_stats, flush_fn.stats_delta); } + toku_ft_adjust_logical_row_count(ft, flush_fn.logical_rows_delta); if (do_garbage_collection) { size_t buffsize = bnc->msg_buffer.buffer_size_in_use(); // may be misleading if there's a broadcast message in there diff --git a/storage/tokudb/PerconaFT/ft/ft-internal.h b/storage/tokudb/PerconaFT/ft/ft-internal.h index 6bf7029245b..eec591d1744 100644 --- a/storage/tokudb/PerconaFT/ft/ft-internal.h +++ b/storage/tokudb/PerconaFT/ft/ft-internal.h @@ -143,6 +143,10 @@ struct ft_header { MSN msn_at_start_of_last_completed_optimize; STAT64INFO_S on_disk_stats; + + // This represents the balance of inserts - deletes and should be + // closer to a logical representation of the number of records in an index + uint64_t on_disk_logical_rows; }; typedef struct ft_header *FT_HEADER; @@ -176,6 +180,7 @@ struct ft { // protected by atomic builtins STAT64INFO_S in_memory_stats; + uint64_t in_memory_logical_rows; // transient, not serialized to disk. updated when we do write to // disk. tells us whether we can do partial eviction (we can't if diff --git a/storage/tokudb/PerconaFT/ft/ft-ops.cc b/storage/tokudb/PerconaFT/ft/ft-ops.cc index f5da82ee000..8f61bc67339 100644 --- a/storage/tokudb/PerconaFT/ft/ft-ops.cc +++ b/storage/tokudb/PerconaFT/ft/ft-ops.cc @@ -1371,7 +1371,8 @@ static void inject_message_in_locked_node( ft_msg msg_with_msn(msg.kdbt(), msg.vdbt(), msg.type(), msg_msn, msg.xids()); paranoid_invariant(msg_with_msn.msn().msn > node->max_msn_applied_to_node_on_disk.msn); - STAT64INFO_S stats_delta = {0,0}; + STAT64INFO_S stats_delta = { 0,0 }; + int64_t logical_rows_delta = 0; toku_ftnode_put_msg( ft->cmp, ft->update_fun, @@ -1381,11 +1382,12 @@ static void inject_message_in_locked_node( true, gc_info, flow_deltas, - &stats_delta - ); + &stats_delta, + &logical_rows_delta); if (stats_delta.numbytes || stats_delta.numrows) { toku_ft_update_stats(&ft->in_memory_stats, stats_delta); } + toku_ft_adjust_logical_row_count(ft, logical_rows_delta); // // assumption is that toku_ftnode_put_msg will // mark the node as dirty. @@ -2169,6 +2171,7 @@ int toku_ft_insert_unique(FT_HANDLE ft_h, DBT *key, DBT *val, TOKUTXN txn, bool if (r == 0) { ft_txn_log_insert(ft_h->ft, key, val, txn, do_logging, FT_INSERT); + toku_ft_adjust_logical_row_count(ft_h->ft, 1); } return r; } @@ -2344,6 +2347,7 @@ void toku_ft_maybe_insert (FT_HANDLE ft_h, DBT *key, DBT *val, TOKUTXN txn, bool if (r != 0) { toku_ft_send_insert(ft_h, key, val, message_xids, type, &gc_info); } + toku_ft_adjust_logical_row_count(ft_h->ft, 1); } } @@ -2513,6 +2517,7 @@ void toku_ft_maybe_delete(FT_HANDLE ft_h, DBT *key, TOKUTXN txn, bool oplsn_vali oldest_referenced_xid_estimate, txn != nullptr ? !txn->for_recovery : false); toku_ft_send_delete(ft_h, key, message_xids, &gc_info); + toku_ft_adjust_logical_row_count(ft_h->ft, -1); } } diff --git a/storage/tokudb/PerconaFT/ft/ft-ops.h b/storage/tokudb/PerconaFT/ft/ft-ops.h index 7d0b165b70c..313a74628ea 100644 --- a/storage/tokudb/PerconaFT/ft/ft-ops.h +++ b/storage/tokudb/PerconaFT/ft/ft-ops.h @@ -207,6 +207,15 @@ extern int toku_ft_debug_mode; int toku_verify_ft (FT_HANDLE ft_h) __attribute__ ((warn_unused_result)); int toku_verify_ft_with_progress (FT_HANDLE ft_h, int (*progress_callback)(void *extra, float progress), void *extra, int verbose, int keep_going) __attribute__ ((warn_unused_result)); +int toku_ft_recount_rows( + FT_HANDLE ft, + int (*progress_callback)( + uint64_t count, + uint64_t deleted, + void* progress_extra), + void* progress_extra); + + DICTIONARY_ID toku_ft_get_dictionary_id(FT_HANDLE); enum ft_flags { diff --git a/storage/tokudb/PerconaFT/ft/ft-recount-rows.cc b/storage/tokudb/PerconaFT/ft/ft-recount-rows.cc new file mode 100644 index 00000000000..adac96f4882 --- /dev/null +++ b/storage/tokudb/PerconaFT/ft/ft-recount-rows.cc @@ -0,0 +1,115 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +#ident "$Id$" +/*====== +This file is part of PerconaFT. + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. + +---------------------------------------- + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License, version 3, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#include "ft/serialize/block_table.h" +#include "ft/ft.h" +#include "ft/ft-internal.h" +#include "ft/cursor.h" + +struct recount_rows_extra_t { + int (*_progress_callback)( + uint64_t count, + uint64_t deleted, + void* progress_extra); + void* _progress_extra; + uint64_t _keys; + bool _cancelled; +}; + +static int recount_rows_found( + uint32_t UU(keylen), + const void* key, + uint32_t UU(vallen), + const void* UU(val), + void* extra, + bool UU(lock_only)) { + + recount_rows_extra_t* rre = (recount_rows_extra_t*)extra; + + if (FT_LIKELY(key != nullptr)) { + rre->_keys++; + } + return rre->_cancelled + = rre->_progress_callback(rre->_keys, 0, rre->_progress_extra); +} +static bool recount_rows_interrupt(void* extra, uint64_t deleted_rows) { + recount_rows_extra_t* rre = (recount_rows_extra_t*)extra; + + return rre->_cancelled = + rre->_progress_callback(rre->_keys, deleted_rows, rre->_progress_extra); +} +int toku_ft_recount_rows( + FT_HANDLE ft, + int (*progress_callback)( + uint64_t count, + uint64_t deleted, + void* progress_extra), + void* progress_extra) { + + int ret = 0; + recount_rows_extra_t rre = { + progress_callback, + progress_extra, + 0, + false + }; + + ft_cursor c; + ret = toku_ft_cursor_create(ft, &c, nullptr, C_READ_ANY, false, false); + if (ret) return ret; + + toku_ft_cursor_set_check_interrupt_cb( + &c, + recount_rows_interrupt, + &rre); + + ret = toku_ft_cursor_first(&c, recount_rows_found, &rre); + while (FT_LIKELY(ret == 0)) { + ret = toku_ft_cursor_next(&c, recount_rows_found, &rre); + } + + toku_ft_cursor_destroy(&c); + + if (rre._cancelled == false) { + // update ft count + toku_unsafe_set(&ft->ft->in_memory_logical_rows, rre._keys); + ret = 0; + } + + return ret; +} diff --git a/storage/tokudb/PerconaFT/ft/ft-status.cc b/storage/tokudb/PerconaFT/ft/ft-status.cc index 982df1822c4..19a378c22bf 100644 --- a/storage/tokudb/PerconaFT/ft/ft-status.cc +++ b/storage/tokudb/PerconaFT/ft/ft-status.cc @@ -128,24 +128,24 @@ void CACHETABLE_STATUS_S::init() { CT_STATUS_INIT(CT_LONG_WAIT_PRESSURE_COUNT, CACHETABLE_LONG_WAIT_PRESSURE_COUNT, UINT64, "number of long waits on cache pressure"); CT_STATUS_INIT(CT_LONG_WAIT_PRESSURE_TIME, CACHETABLE_LONG_WAIT_PRESSURE_TIME, UINT64, "long time waiting on cache pressure"); - CT_STATUS_INIT(CT_POOL_CLIENT_NUM_THREADS, CACHETABLE_POOL_CLIENT_NUM_THREADS, UINT64, "number of threads in pool"); - CT_STATUS_INIT(CT_POOL_CLIENT_NUM_THREADS_ACTIVE, CACHETABLE_POOL_CLIENT_NUM_THREADS_ACTIVE, UINT64, "number of currently active threads in pool"); - CT_STATUS_INIT(CT_POOL_CLIENT_QUEUE_SIZE, CACHETABLE_POOL_CLIENT_QUEUE_SIZE, UINT64, "number of currently queued work items"); - CT_STATUS_INIT(CT_POOL_CLIENT_MAX_QUEUE_SIZE, CACHETABLE_POOL_CLIENT_MAX_QUEUE_SIZE, UINT64, "largest number of queued work items"); - CT_STATUS_INIT(CT_POOL_CLIENT_TOTAL_ITEMS_PROCESSED, CACHETABLE_POOL_CLIENT_TOTAL_ITEMS_PROCESSED, UINT64, "total number of work items processed"); - CT_STATUS_INIT(CT_POOL_CLIENT_TOTAL_EXECUTION_TIME, CACHETABLE_POOL_CLIENT_TOTAL_EXECUTION_TIME, UINT64, "total execution time of processing work items"); - CT_STATUS_INIT(CT_POOL_CACHETABLE_NUM_THREADS, CACHETABLE_POOL_CACHETABLE_NUM_THREADS, UINT64, "number of threads in pool"); - CT_STATUS_INIT(CT_POOL_CACHETABLE_NUM_THREADS_ACTIVE, CACHETABLE_POOL_CACHETABLE_NUM_THREADS_ACTIVE, UINT64, "number of currently active threads in pool"); - CT_STATUS_INIT(CT_POOL_CACHETABLE_QUEUE_SIZE, CACHETABLE_POOL_CACHETABLE_QUEUE_SIZE, UINT64, "number of currently queued work items"); - CT_STATUS_INIT(CT_POOL_CACHETABLE_MAX_QUEUE_SIZE, CACHETABLE_POOL_CACHETABLE_MAX_QUEUE_SIZE, UINT64, "largest number of queued work items"); - CT_STATUS_INIT(CT_POOL_CACHETABLE_TOTAL_ITEMS_PROCESSED, CACHETABLE_POOL_CACHETABLE_TOTAL_ITEMS_PROCESSED, UINT64, "total number of work items processed"); - CT_STATUS_INIT(CT_POOL_CACHETABLE_TOTAL_EXECUTION_TIME, CACHETABLE_POOL_CACHETABLE_TOTAL_EXECUTION_TIME, UINT64, "total execution time of processing work items"); - CT_STATUS_INIT(CT_POOL_CHECKPOINT_NUM_THREADS, CACHETABLE_POOL_CHECKPOINT_NUM_THREADS, UINT64, "number of threads in pool"); - CT_STATUS_INIT(CT_POOL_CHECKPOINT_NUM_THREADS_ACTIVE, CACHETABLE_POOL_CHECKPOINT_NUM_THREADS_ACTIVE, UINT64, "number of currently active threads in pool"); - CT_STATUS_INIT(CT_POOL_CHECKPOINT_QUEUE_SIZE, CACHETABLE_POOL_CHECKPOINT_QUEUE_SIZE, UINT64, "number of currently queued work items"); - CT_STATUS_INIT(CT_POOL_CHECKPOINT_MAX_QUEUE_SIZE, CACHETABLE_POOL_CHECKPOINT_MAX_QUEUE_SIZE, UINT64, "largest number of queued work items"); - CT_STATUS_INIT(CT_POOL_CHECKPOINT_TOTAL_ITEMS_PROCESSED, CACHETABLE_POOL_CHECKPOINT_TOTAL_ITEMS_PROCESSED, UINT64, "total number of work items processed"); - CT_STATUS_INIT(CT_POOL_CHECKPOINT_TOTAL_EXECUTION_TIME, CACHETABLE_POOL_CHECKPOINT_TOTAL_EXECUTION_TIME, UINT64, "total execution time of processing work items"); + CT_STATUS_INIT(CT_POOL_CLIENT_NUM_THREADS, CACHETABLE_POOL_CLIENT_NUM_THREADS, UINT64, "client pool: number of threads in pool"); + CT_STATUS_INIT(CT_POOL_CLIENT_NUM_THREADS_ACTIVE, CACHETABLE_POOL_CLIENT_NUM_THREADS_ACTIVE, UINT64, "client pool: number of currently active threads in pool"); + CT_STATUS_INIT(CT_POOL_CLIENT_QUEUE_SIZE, CACHETABLE_POOL_CLIENT_QUEUE_SIZE, UINT64, "client pool: number of currently queued work items"); + CT_STATUS_INIT(CT_POOL_CLIENT_MAX_QUEUE_SIZE, CACHETABLE_POOL_CLIENT_MAX_QUEUE_SIZE, UINT64, "client pool: largest number of queued work items"); + CT_STATUS_INIT(CT_POOL_CLIENT_TOTAL_ITEMS_PROCESSED, CACHETABLE_POOL_CLIENT_TOTAL_ITEMS_PROCESSED, UINT64, "client pool: total number of work items processed"); + CT_STATUS_INIT(CT_POOL_CLIENT_TOTAL_EXECUTION_TIME, CACHETABLE_POOL_CLIENT_TOTAL_EXECUTION_TIME, UINT64, "client pool: total execution time of processing work items"); + CT_STATUS_INIT(CT_POOL_CACHETABLE_NUM_THREADS, CACHETABLE_POOL_CACHETABLE_NUM_THREADS, UINT64, "cachetable pool: number of threads in pool"); + CT_STATUS_INIT(CT_POOL_CACHETABLE_NUM_THREADS_ACTIVE, CACHETABLE_POOL_CACHETABLE_NUM_THREADS_ACTIVE, UINT64, "cachetable pool: number of currently active threads in pool"); + CT_STATUS_INIT(CT_POOL_CACHETABLE_QUEUE_SIZE, CACHETABLE_POOL_CACHETABLE_QUEUE_SIZE, UINT64, "cachetable pool: number of currently queued work items"); + CT_STATUS_INIT(CT_POOL_CACHETABLE_MAX_QUEUE_SIZE, CACHETABLE_POOL_CACHETABLE_MAX_QUEUE_SIZE, UINT64, "cachetable pool: largest number of queued work items"); + CT_STATUS_INIT(CT_POOL_CACHETABLE_TOTAL_ITEMS_PROCESSED, CACHETABLE_POOL_CACHETABLE_TOTAL_ITEMS_PROCESSED, UINT64, "cachetable pool: total number of work items processed"); + CT_STATUS_INIT(CT_POOL_CACHETABLE_TOTAL_EXECUTION_TIME, CACHETABLE_POOL_CACHETABLE_TOTAL_EXECUTION_TIME, UINT64, "cachetable pool: total execution time of processing work items"); + CT_STATUS_INIT(CT_POOL_CHECKPOINT_NUM_THREADS, CACHETABLE_POOL_CHECKPOINT_NUM_THREADS, UINT64, "checkpoint pool: number of threads in pool"); + CT_STATUS_INIT(CT_POOL_CHECKPOINT_NUM_THREADS_ACTIVE, CACHETABLE_POOL_CHECKPOINT_NUM_THREADS_ACTIVE, UINT64, "checkpoint pool: number of currently active threads in pool"); + CT_STATUS_INIT(CT_POOL_CHECKPOINT_QUEUE_SIZE, CACHETABLE_POOL_CHECKPOINT_QUEUE_SIZE, UINT64, "checkpoint pool: number of currently queued work items"); + CT_STATUS_INIT(CT_POOL_CHECKPOINT_MAX_QUEUE_SIZE, CACHETABLE_POOL_CHECKPOINT_MAX_QUEUE_SIZE, UINT64, "checkpoint pool: largest number of queued work items"); + CT_STATUS_INIT(CT_POOL_CHECKPOINT_TOTAL_ITEMS_PROCESSED, CACHETABLE_POOL_CHECKPOINT_TOTAL_ITEMS_PROCESSED, UINT64, "checkpoint pool: total number of work items processed"); + CT_STATUS_INIT(CT_POOL_CHECKPOINT_TOTAL_EXECUTION_TIME, CACHETABLE_POOL_CHECKPOINT_TOTAL_EXECUTION_TIME, UINT64, "checkpoint pool: total execution time of processing work items"); m_initialized = true; #undef CT_STATUS_INIT diff --git a/storage/tokudb/PerconaFT/ft/ft-test-helpers.cc b/storage/tokudb/PerconaFT/ft/ft-test-helpers.cc index 7ca36c23780..6fcdbbdc9e3 100644 --- a/storage/tokudb/PerconaFT/ft/ft-test-helpers.cc +++ b/storage/tokudb/PerconaFT/ft/ft-test-helpers.cc @@ -172,21 +172,26 @@ int toku_testsetup_insert_to_leaf (FT_HANDLE ft_handle, BLOCKNUM blocknum, const assert(node->height==0); DBT kdbt, vdbt; - ft_msg msg(toku_fill_dbt(&kdbt, key, keylen), toku_fill_dbt(&vdbt, val, vallen), - FT_INSERT, next_dummymsn(), toku_xids_get_root_xids()); + ft_msg msg( + toku_fill_dbt(&kdbt, key, keylen), + toku_fill_dbt(&vdbt, val, vallen), + FT_INSERT, + next_dummymsn(), + toku_xids_get_root_xids()); static size_t zero_flow_deltas[] = { 0, 0 }; txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, true); - toku_ftnode_put_msg(ft_handle->ft->cmp, - ft_handle->ft->update_fun, - node, - -1, - msg, - true, - &gc_info, - zero_flow_deltas, - NULL - ); + toku_ftnode_put_msg( + ft_handle->ft->cmp, + ft_handle->ft->update_fun, + node, + -1, + msg, + true, + &gc_info, + zero_flow_deltas, + NULL, + NULL); toku_verify_or_set_counts(node); diff --git a/storage/tokudb/PerconaFT/ft/ft.cc b/storage/tokudb/PerconaFT/ft/ft.cc index 2a0fb6f6800..93d21233bf7 100644 --- a/storage/tokudb/PerconaFT/ft/ft.cc +++ b/storage/tokudb/PerconaFT/ft/ft.cc @@ -198,6 +198,8 @@ static void ft_checkpoint (CACHEFILE cf, int fd, void *header_v) { ch->time_of_last_modification = now; ch->checkpoint_count++; ft_hack_highest_unused_msn_for_upgrade_for_checkpoint(ft); + ch->on_disk_logical_rows = + ft->h->on_disk_logical_rows = ft->in_memory_logical_rows; // write translation and header to disk (or at least to OS internal buffer) toku_serialize_ft_to(fd, ch, &ft->blocktable, ft->cf); @@ -383,7 +385,8 @@ ft_header_create(FT_OPTIONS options, BLOCKNUM root_blocknum, TXNID root_xid_that .count_of_optimize_in_progress = 0, .count_of_optimize_in_progress_read_from_disk = 0, .msn_at_start_of_last_completed_optimize = ZERO_MSN, - .on_disk_stats = ZEROSTATS + .on_disk_stats = ZEROSTATS, + .on_disk_logical_rows = 0 }; return (FT_HEADER) toku_xmemdup(&h, sizeof h); } @@ -802,7 +805,14 @@ toku_ft_stat64 (FT ft, struct ftstat64_s *s) { s->fsize = toku_cachefile_size(ft->cf); // just use the in memory stats from the header // prevent appearance of negative numbers for numrows, numbytes - int64_t n = ft->in_memory_stats.numrows; + // if the logical count was never properly re-counted on an upgrade, + // return the existing physical count instead. + int64_t n; + if (ft->in_memory_logical_rows == (uint64_t)-1) { + n = ft->in_memory_stats.numrows; + } else { + n = ft->in_memory_logical_rows; + } if (n < 0) { n = 0; } @@ -871,20 +881,38 @@ DESCRIPTOR toku_ft_get_cmp_descriptor(FT_HANDLE ft_handle) { return &ft_handle->ft->cmp_descriptor; } -void -toku_ft_update_stats(STAT64INFO headerstats, STAT64INFO_S delta) { +void toku_ft_update_stats(STAT64INFO headerstats, STAT64INFO_S delta) { (void) toku_sync_fetch_and_add(&(headerstats->numrows), delta.numrows); (void) toku_sync_fetch_and_add(&(headerstats->numbytes), delta.numbytes); } -void -toku_ft_decrease_stats(STAT64INFO headerstats, STAT64INFO_S delta) { +void toku_ft_decrease_stats(STAT64INFO headerstats, STAT64INFO_S delta) { (void) toku_sync_fetch_and_sub(&(headerstats->numrows), delta.numrows); (void) toku_sync_fetch_and_sub(&(headerstats->numbytes), delta.numbytes); } -void -toku_ft_remove_reference(FT ft, bool oplsn_valid, LSN oplsn, remove_ft_ref_callback remove_ref, void *extra) { +void toku_ft_adjust_logical_row_count(FT ft, int64_t delta) { + // In order to make sure that the correct count is returned from + // toku_ft_stat64, the ft->(in_memory|on_disk)_logical_rows _MUST_NOT_ be + // modified from anywhere else from here with the exceptions of + // serializing in a header, initializing a new header and analyzing + // an index for a logical_row count. + // The gist is that on an index upgrade, all logical_rows values + // in the ft header are set to -1 until an analyze can reset it to an + // accurate value. Until then, the physical count from in_memory_stats + // must be returned in toku_ft_stat64. + if (delta != 0 && ft->in_memory_logical_rows != (uint64_t)-1) { + toku_sync_fetch_and_add(&(ft->in_memory_logical_rows), delta); + } +} + +void toku_ft_remove_reference( + FT ft, + bool oplsn_valid, + LSN oplsn, + remove_ft_ref_callback remove_ref, + void *extra) { + toku_ft_grab_reflock(ft); if (toku_ft_has_one_reference_unlocked(ft)) { toku_ft_release_reflock(ft); diff --git a/storage/tokudb/PerconaFT/ft/ft.h b/storage/tokudb/PerconaFT/ft/ft.h index cc64bdfc6d3..d600e093bdc 100644 --- a/storage/tokudb/PerconaFT/ft/ft.h +++ b/storage/tokudb/PerconaFT/ft/ft.h @@ -127,13 +127,17 @@ DESCRIPTOR toku_ft_get_cmp_descriptor(FT_HANDLE ft_handle); typedef struct { // delta versions in basements could be negative + // These represent the physical leaf entries and do not account + // for pending deletes or other in-flight messages that have not been + // applied to a leaf entry. int64_t numrows; int64_t numbytes; } STAT64INFO_S, *STAT64INFO; -static const STAT64INFO_S ZEROSTATS = { .numrows = 0, .numbytes = 0}; +static const STAT64INFO_S ZEROSTATS = { .numrows = 0, .numbytes = 0 }; void toku_ft_update_stats(STAT64INFO headerstats, STAT64INFO_S delta); void toku_ft_decrease_stats(STAT64INFO headerstats, STAT64INFO_S delta); +void toku_ft_adjust_logical_row_count(FT ft, int64_t delta); typedef void (*remove_ft_ref_callback)(FT ft, void *extra); void toku_ft_remove_reference(FT ft, diff --git a/storage/tokudb/PerconaFT/ft/leafentry.h b/storage/tokudb/PerconaFT/ft/leafentry.h index 9cb81ef7cd6..7274a1480e2 100644 --- a/storage/tokudb/PerconaFT/ft/leafentry.h +++ b/storage/tokudb/PerconaFT/ft/leafentry.h @@ -180,43 +180,57 @@ uint64_t le_outermost_uncommitted_xid (LEAFENTRY le); // r|r!=0&&r!=TOKUDB_ACCEPT: Quit early, return r, because something unexpected went wrong (error case) typedef int(*LE_ITERATE_CALLBACK)(TXNID id, TOKUTXN context, bool is_provisional); -int le_iterate_val(LEAFENTRY le, LE_ITERATE_CALLBACK f, void** valpp, uint32_t *vallenp, TOKUTXN context); - -void le_extract_val(LEAFENTRY le, - // should we return the entire leafentry as the val? - bool is_leaf_mode, enum cursor_read_type read_type, - TOKUTXN ttxn, uint32_t *vallen, void **val); - -size_t -leafentry_disksize_13(LEAFENTRY_13 le); - -int -toku_le_upgrade_13_14(LEAFENTRY_13 old_leafentry, // NULL if there was no stored data. - void** keyp, - uint32_t* keylen, - size_t *new_leafentry_memorysize, - LEAFENTRY *new_leafentry_p); +int le_iterate_val( + LEAFENTRY le, + LE_ITERATE_CALLBACK f, + void** valpp, + uint32_t* vallenp, + TOKUTXN context); + +void le_extract_val( + LEAFENTRY le, + // should we return the entire leafentry as the val? + bool is_leaf_mode, + enum cursor_read_type read_type, + TOKUTXN ttxn, + uint32_t* vallen, + void** val); + +size_t leafentry_disksize_13(LEAFENTRY_13 le); + +int toku_le_upgrade_13_14( + // NULL if there was no stored data. + LEAFENTRY_13 old_leafentry, + void** keyp, + uint32_t* keylen, + size_t* new_leafentry_memorysize, + LEAFENTRY *new_leafentry_p); class bn_data; -void -toku_le_apply_msg(const ft_msg &msg, - LEAFENTRY old_leafentry, // NULL if there was no stored data. - bn_data* data_buffer, // bn_data storing leafentry, if NULL, means there is no bn_data - uint32_t idx, // index in data_buffer where leafentry is stored (and should be replaced - uint32_t old_keylen, - txn_gc_info *gc_info, - LEAFENTRY *new_leafentry_p, - int64_t * numbytes_delta_p); - -bool toku_le_worth_running_garbage_collection(LEAFENTRY le, txn_gc_info *gc_info); - -void -toku_le_garbage_collect(LEAFENTRY old_leaf_entry, - bn_data* data_buffer, - uint32_t idx, - void* keyp, - uint32_t keylen, - txn_gc_info *gc_info, - LEAFENTRY *new_leaf_entry, - int64_t * numbytes_delta_p); +int64_t toku_le_apply_msg( + const ft_msg &msg, + // NULL if there was no stored data. + LEAFENTRY old_leafentry, + // bn_data storing leafentry, if NULL, means there is no bn_data + bn_data* data_buffer, + // index in data_buffer where leafentry is stored (and should be replaced + uint32_t idx, + uint32_t old_keylen, + txn_gc_info* gc_info, + LEAFENTRY *new_leafentry_p, + int64_t* numbytes_delta_p); + +bool toku_le_worth_running_garbage_collection( + LEAFENTRY le, + txn_gc_info* gc_info); + +void toku_le_garbage_collect( + LEAFENTRY old_leaf_entry, + bn_data* data_buffer, + uint32_t idx, + void* keyp, + uint32_t keylen, + txn_gc_info* gc_info, + LEAFENTRY* new_leaf_entry, + int64_t* numbytes_delta_p); diff --git a/storage/tokudb/PerconaFT/ft/loader/loader.cc b/storage/tokudb/PerconaFT/ft/loader/loader.cc index 5ff0d69af46..20f9363da1e 100644 --- a/storage/tokudb/PerconaFT/ft/loader/loader.cc +++ b/storage/tokudb/PerconaFT/ft/loader/loader.cc @@ -2312,11 +2312,42 @@ static struct leaf_buf *start_leaf (struct dbout *out, const DESCRIPTOR UU(desc) return lbuf; } -static void finish_leafnode (struct dbout *out, struct leaf_buf *lbuf, int progress_allocation, FTLOADER bl, uint32_t target_basementnodesize, enum toku_compression_method target_compression_method); -static int write_nonleaves (FTLOADER bl, FIDX pivots_fidx, struct dbout *out, struct subtrees_info *sts, const DESCRIPTOR descriptor, uint32_t target_nodesize, uint32_t target_basementnodesize, enum toku_compression_method target_compression_method); -static void add_pair_to_leafnode (struct leaf_buf *lbuf, unsigned char *key, int keylen, unsigned char *val, int vallen, int this_leafentry_size, STAT64INFO stats_to_update); -static int write_translation_table (struct dbout *out, long long *off_of_translation_p); -static int write_header (struct dbout *out, long long translation_location_on_disk, long long translation_size_on_disk); +static void finish_leafnode( + struct dbout* out, + struct leaf_buf* lbuf, + int progress_allocation, + FTLOADER bl, + uint32_t target_basementnodesize, + enum toku_compression_method target_compression_method); + +static int write_nonleaves( + FTLOADER bl, + FIDX pivots_fidx, + struct dbout* out, + struct subtrees_info* sts, + const DESCRIPTOR descriptor, + uint32_t target_nodesize, + uint32_t target_basementnodesize, + enum toku_compression_method target_compression_method); + +static void add_pair_to_leafnode( + struct leaf_buf* lbuf, + unsigned char* key, + int keylen, + unsigned char* val, + int vallen, + int this_leafentry_size, + STAT64INFO stats_to_update, + int64_t* logical_rows_delta); + +static int write_translation_table( + struct dbout* out, + long long* off_of_translation_p); + +static int write_header( + struct dbout* out, + long long translation_location_on_disk, + long long translation_size_on_disk); static void drain_writer_q(QUEUE q) { void *item; @@ -2448,6 +2479,12 @@ static int toku_loader_write_ft_from_q (FTLOADER bl, DBT maxkey = make_dbt(0, 0); // keep track of the max key of the current node STAT64INFO_S deltas = ZEROSTATS; + // This is just a placeholder and not used in the loader, the real/accurate + // stats will come out of 'deltas' because this loader is not pushing + // messages down into the top of a fractal tree where the logical row count + // is done, it is directly creating leaf entries so it must also take on + // performing the logical row counting on its own + int64_t logical_rows_delta = 0; while (result == 0) { void *item; { @@ -2506,7 +2543,15 @@ static int toku_loader_write_ft_from_q (FTLOADER bl, lbuf = start_leaf(&out, descriptor, lblock, le_xid, target_nodesize); } - add_pair_to_leafnode(lbuf, (unsigned char *) key.data, key.size, (unsigned char *) val.data, val.size, this_leafentry_size, &deltas); + add_pair_to_leafnode( + lbuf, + (unsigned char*)key.data, + key.size, + (unsigned char*)val.data, + val.size, + this_leafentry_size, + &deltas, + &logical_rows_delta); n_rows_remaining--; update_maxkey(&maxkey, &key); // set the new maxkey to the current key @@ -2526,6 +2571,13 @@ static int toku_loader_write_ft_from_q (FTLOADER bl, toku_ft_update_stats(&ft.in_memory_stats, deltas); } + // As noted above, the loader directly creates a tree structure without + // going through the higher level ft API and tus bypasses the logical row + // counting performed at that level. So, we must manually update the logical + // row count with the info we have from the physical delta that comes out of + // add_pair_to_leafnode. + toku_ft_adjust_logical_row_count(&ft, deltas.numrows); + cleanup_maxkey(&maxkey); if (lbuf) { @@ -2878,7 +2930,16 @@ int toku_ft_loader_get_error(FTLOADER bl, int *error) { return 0; } -static void add_pair_to_leafnode (struct leaf_buf *lbuf, unsigned char *key, int keylen, unsigned char *val, int vallen, int this_leafentry_size, STAT64INFO stats_to_update) { +static void add_pair_to_leafnode( + struct leaf_buf* lbuf, + unsigned char* key, + int keylen, + unsigned char* val, + int vallen, + int this_leafentry_size, + STAT64INFO stats_to_update, + int64_t* logical_rows_delta) { + lbuf->nkeys++; lbuf->ndata++; lbuf->dsize += keylen + vallen; @@ -2890,11 +2951,25 @@ static void add_pair_to_leafnode (struct leaf_buf *lbuf, unsigned char *key, int FTNODE leafnode = lbuf->node; uint32_t idx = BLB_DATA(leafnode, 0)->num_klpairs(); DBT kdbt, vdbt; - ft_msg msg(toku_fill_dbt(&kdbt, key, keylen), toku_fill_dbt(&vdbt, val, vallen), FT_INSERT, ZERO_MSN, lbuf->xids); + ft_msg msg( + toku_fill_dbt(&kdbt, key, keylen), + toku_fill_dbt(&vdbt, val, vallen), + FT_INSERT, + ZERO_MSN, + lbuf->xids); uint64_t workdone = 0; // there's no mvcc garbage in a bulk-loaded FT, so there's no need to pass useful gc info txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, true); - toku_ft_bn_apply_msg_once(BLB(leafnode,0), msg, idx, keylen, NULL, &gc_info, &workdone, stats_to_update); + toku_ft_bn_apply_msg_once( + BLB(leafnode, 0), + msg, + idx, + keylen, + NULL, + &gc_info, + &workdone, + stats_to_update, + logical_rows_delta); } static int write_literal(struct dbout *out, void*data, size_t len) { @@ -2905,7 +2980,14 @@ static int write_literal(struct dbout *out, void*data, size_t len) { return result; } -static void finish_leafnode (struct dbout *out, struct leaf_buf *lbuf, int progress_allocation, FTLOADER bl, uint32_t target_basementnodesize, enum toku_compression_method target_compression_method) { +static void finish_leafnode( + struct dbout* out, + struct leaf_buf* lbuf, + int progress_allocation, + FTLOADER bl, + uint32_t target_basementnodesize, + enum toku_compression_method target_compression_method) { + int result = 0; // serialize leaf to buffer @@ -2913,7 +2995,16 @@ static void finish_leafnode (struct dbout *out, struct leaf_buf *lbuf, int progr size_t uncompressed_serialized_leaf_size = 0; char *serialized_leaf = NULL; FTNODE_DISK_DATA ndd = NULL; - result = toku_serialize_ftnode_to_memory(lbuf->node, &ndd, target_basementnodesize, target_compression_method, true, true, &serialized_leaf_size, &uncompressed_serialized_leaf_size, &serialized_leaf); + result = toku_serialize_ftnode_to_memory( + lbuf->node, + &ndd, + target_basementnodesize, + target_compression_method, + true, + true, + &serialized_leaf_size, + &uncompressed_serialized_leaf_size, + &serialized_leaf); // write it out if (result == 0) { @@ -2979,8 +3070,11 @@ static int write_translation_table (struct dbout *out, long long *off_of_transla return result; } -static int -write_header (struct dbout *out, long long translation_location_on_disk, long long translation_size_on_disk) { +static int write_header( + struct dbout* out, + long long translation_location_on_disk, + long long translation_size_on_disk) { + int result = 0; size_t size = toku_serialize_ft_size(out->ft->h); size_t alloced_size = roundup_to_multiple(512, size); @@ -2991,6 +3085,7 @@ write_header (struct dbout *out, long long translation_location_on_disk, long lo } else { wbuf_init(&wbuf, buf, size); out->ft->h->on_disk_stats = out->ft->in_memory_stats; + out->ft->h->on_disk_logical_rows = out->ft->in_memory_logical_rows; toku_serialize_ft_to_wbuf(&wbuf, out->ft->h, translation_location_on_disk, translation_size_on_disk); for (size_t i=size; i<alloced_size; i++) buf[i]=0; // initialize all those unused spots to zero if (wbuf.ndone != size) diff --git a/storage/tokudb/PerconaFT/ft/logger/log_upgrade.cc b/storage/tokudb/PerconaFT/ft/logger/log_upgrade.cc index 6dca8c25378..cbe3cc328b4 100644 --- a/storage/tokudb/PerconaFT/ft/logger/log_upgrade.cc +++ b/storage/tokudb/PerconaFT/ft/logger/log_upgrade.cc @@ -265,8 +265,9 @@ toku_maybe_upgrade_log(const char *env_dir, const char *log_dir, LSN * lsn_of_cl TXNID last_xid = TXNID_NONE; r = verify_clean_shutdown_of_log_version(log_dir, version_of_logs_on_disk, &last_lsn, &last_xid); if (r != 0) { - if (TOKU_LOG_VERSION_25 <= version_of_logs_on_disk && version_of_logs_on_disk <= TOKU_LOG_VERSION_27 - && TOKU_LOG_VERSION_28 == TOKU_LOG_VERSION) { + if (TOKU_LOG_VERSION_25 <= version_of_logs_on_disk && + version_of_logs_on_disk <= TOKU_LOG_VERSION_27 && + TOKU_LOG_VERSION_29 == TOKU_LOG_VERSION) { r = 0; // can do recovery on dirty shutdown } else { fprintf(stderr, "Cannot upgrade PerconaFT version %d database.", version_of_logs_on_disk); diff --git a/storage/tokudb/PerconaFT/ft/logger/logger.h b/storage/tokudb/PerconaFT/ft/logger/logger.h index 1f15f59fb3f..d9595d71065 100644 --- a/storage/tokudb/PerconaFT/ft/logger/logger.h +++ b/storage/tokudb/PerconaFT/ft/logger/logger.h @@ -54,6 +54,7 @@ enum { TOKU_LOG_VERSION_26 = 26, // no change from 25 TOKU_LOG_VERSION_27 = 27, // no change from 26 TOKU_LOG_VERSION_28 = 28, // no change from 27 + TOKU_LOG_VERSION_29 = 29, // no change from 28 TOKU_LOG_VERSION = FT_LAYOUT_VERSION, TOKU_LOG_MIN_SUPPORTED_VERSION = FT_LAYOUT_MIN_SUPPORTED_VERSION, }; diff --git a/storage/tokudb/PerconaFT/ft/node.cc b/storage/tokudb/PerconaFT/ft/node.cc index 44dbc73ba2b..58ba675eb7c 100644 --- a/storage/tokudb/PerconaFT/ft/node.cc +++ b/storage/tokudb/PerconaFT/ft/node.cc @@ -206,12 +206,20 @@ int msg_buffer_offset_msn_cmp(message_buffer &msg_buffer, const int32_t &ao, con } /** - * Given a message buffer and and offset, apply the message with toku_ft_bn_apply_msg, or discard it, + * Given a message buffer and and offset, apply the message with + * toku_ft_bn_apply_msg, or discard it, * based on its MSN and the MSN of the basement node. */ -static void -do_bn_apply_msg(FT_HANDLE ft_handle, BASEMENTNODE bn, message_buffer *msg_buffer, int32_t offset, - txn_gc_info *gc_info, uint64_t *workdone, STAT64INFO stats_to_update) { +static void do_bn_apply_msg( + FT_HANDLE ft_handle, + BASEMENTNODE bn, + message_buffer* msg_buffer, + int32_t offset, + txn_gc_info* gc_info, + uint64_t* workdone, + STAT64INFO stats_to_update, + int64_t* logical_rows_delta) { + DBT k, v; ft_msg msg = msg_buffer->get_message(offset, &k, &v); @@ -227,16 +235,17 @@ do_bn_apply_msg(FT_HANDLE ft_handle, BASEMENTNODE bn, message_buffer *msg_buffer msg, gc_info, workdone, - stats_to_update - ); + stats_to_update, + logical_rows_delta); } else { toku_ft_status_note_msn_discard(); } // We must always mark message as stale since it has been marked // (using omt::iterate_and_mark_range) - // It is possible to call do_bn_apply_msg even when it won't apply the message because - // the node containing it could have been evicted and brought back in. + // It is possible to call do_bn_apply_msg even when it won't apply the + // message because the node containing it could have been evicted and + // brought back in. msg_buffer->set_freshness(offset, false); } @@ -248,12 +257,29 @@ struct iterate_do_bn_apply_msg_extra { txn_gc_info *gc_info; uint64_t *workdone; STAT64INFO stats_to_update; + int64_t *logical_rows_delta; }; -int iterate_do_bn_apply_msg(const int32_t &offset, const uint32_t UU(idx), struct iterate_do_bn_apply_msg_extra *const e) __attribute__((nonnull(3))); -int iterate_do_bn_apply_msg(const int32_t &offset, const uint32_t UU(idx), struct iterate_do_bn_apply_msg_extra *const e) +int iterate_do_bn_apply_msg( + const int32_t &offset, + const uint32_t UU(idx), + struct iterate_do_bn_apply_msg_extra* const e) + __attribute__((nonnull(3))); + +int iterate_do_bn_apply_msg( + const int32_t &offset, + const uint32_t UU(idx), + struct iterate_do_bn_apply_msg_extra* const e) { - do_bn_apply_msg(e->t, e->bn, &e->bnc->msg_buffer, offset, e->gc_info, e->workdone, e->stats_to_update); + do_bn_apply_msg( + e->t, + e->bn, + &e->bnc->msg_buffer, + offset, + e->gc_info, + e->workdone, + e->stats_to_update, + e->logical_rows_delta); return 0; } @@ -354,17 +380,15 @@ find_bounds_within_message_tree( * or plus infinity respectively if they are NULL. Do not mark the node * as dirty (preserve previous state of 'dirty' bit). */ -static void -bnc_apply_messages_to_basement_node( +static void bnc_apply_messages_to_basement_node( FT_HANDLE t, // used for comparison function BASEMENTNODE bn, // where to apply messages FTNODE ancestor, // the ancestor node where we can find messages to apply int childnum, // which child buffer of ancestor contains messages we want const pivot_bounds &bounds, // contains pivot key bounds of this basement node - txn_gc_info *gc_info, - bool* msgs_applied - ) -{ + txn_gc_info* gc_info, + bool* msgs_applied) { + int r; NONLEAF_CHILDINFO bnc = BNC(ancestor, childnum); @@ -372,16 +396,29 @@ bnc_apply_messages_to_basement_node( // apply messages from this buffer STAT64INFO_S stats_delta = {0,0}; uint64_t workdone_this_ancestor = 0; + int64_t logical_rows_delta = 0; uint32_t stale_lbi, stale_ube; if (!bn->stale_ancestor_messages_applied) { - find_bounds_within_message_tree(t->ft->cmp, bnc->stale_message_tree, &bnc->msg_buffer, bounds, &stale_lbi, &stale_ube); + find_bounds_within_message_tree( + t->ft->cmp, + bnc->stale_message_tree, + &bnc->msg_buffer, + bounds, + &stale_lbi, + &stale_ube); } else { stale_lbi = 0; stale_ube = 0; } uint32_t fresh_lbi, fresh_ube; - find_bounds_within_message_tree(t->ft->cmp, bnc->fresh_message_tree, &bnc->msg_buffer, bounds, &fresh_lbi, &fresh_ube); + find_bounds_within_message_tree( + t->ft->cmp, + bnc->fresh_message_tree, + &bnc->msg_buffer, + bounds, + &fresh_lbi, + &fresh_ube); // We now know where all the messages we must apply are, so one of the // following 4 cases will do the application, depending on which of @@ -395,7 +432,9 @@ bnc_apply_messages_to_basement_node( // We have messages in multiple trees, so we grab all // the relevant messages' offsets and sort them by MSN, then apply // them in MSN order. - const int buffer_size = ((stale_ube - stale_lbi) + (fresh_ube - fresh_lbi) + bnc->broadcast_list.size()); + const int buffer_size = ((stale_ube - stale_lbi) + + (fresh_ube - fresh_lbi) + + bnc->broadcast_list.size()); toku::scoped_malloc offsets_buf(buffer_size * sizeof(int32_t)); int32_t *offsets = reinterpret_cast<int32_t *>(offsets_buf.get()); struct store_msg_buffer_offset_extra sfo_extra = { .offsets = offsets, .i = 0 }; @@ -419,11 +458,27 @@ bnc_apply_messages_to_basement_node( // Apply the messages in MSN order. for (int i = 0; i < buffer_size; ++i) { *msgs_applied = true; - do_bn_apply_msg(t, bn, &bnc->msg_buffer, offsets[i], gc_info, &workdone_this_ancestor, &stats_delta); + do_bn_apply_msg( + t, + bn, + &bnc->msg_buffer, + offsets[i], + gc_info, + &workdone_this_ancestor, + &stats_delta, + &logical_rows_delta); } } else if (stale_lbi == stale_ube) { // No stale messages to apply, we just apply fresh messages, and mark them to be moved to stale later. - struct iterate_do_bn_apply_msg_extra iter_extra = { .t = t, .bn = bn, .bnc = bnc, .gc_info = gc_info, .workdone = &workdone_this_ancestor, .stats_to_update = &stats_delta }; + struct iterate_do_bn_apply_msg_extra iter_extra = { + .t = t, + .bn = bn, + .bnc = bnc, + .gc_info = gc_info, + .workdone = &workdone_this_ancestor, + .stats_to_update = &stats_delta, + .logical_rows_delta = &logical_rows_delta + }; if (fresh_ube - fresh_lbi > 0) *msgs_applied = true; r = bnc->fresh_message_tree.iterate_and_mark_range<struct iterate_do_bn_apply_msg_extra, iterate_do_bn_apply_msg>(fresh_lbi, fresh_ube, &iter_extra); assert_zero(r); @@ -432,7 +487,15 @@ bnc_apply_messages_to_basement_node( // No fresh messages to apply, we just apply stale messages. if (stale_ube - stale_lbi > 0) *msgs_applied = true; - struct iterate_do_bn_apply_msg_extra iter_extra = { .t = t, .bn = bn, .bnc = bnc, .gc_info = gc_info, .workdone = &workdone_this_ancestor, .stats_to_update = &stats_delta }; + struct iterate_do_bn_apply_msg_extra iter_extra = { + .t = t, + .bn = bn, + .bnc = bnc, + .gc_info = gc_info, + .workdone = &workdone_this_ancestor, + .stats_to_update = &stats_delta, + .logical_rows_delta = &logical_rows_delta + }; r = bnc->stale_message_tree.iterate_on_range<struct iterate_do_bn_apply_msg_extra, iterate_do_bn_apply_msg>(stale_lbi, stale_ube, &iter_extra); assert_zero(r); @@ -446,6 +509,7 @@ bnc_apply_messages_to_basement_node( if (stats_delta.numbytes || stats_delta.numrows) { toku_ft_update_stats(&t->ft->in_memory_stats, stats_delta); } + toku_ft_adjust_logical_row_count(t->ft, logical_rows_delta); } static void @@ -1073,7 +1137,8 @@ toku_ft_bn_apply_msg_once ( LEAFENTRY le, txn_gc_info *gc_info, uint64_t *workdone, - STAT64INFO stats_to_update + STAT64INFO stats_to_update, + int64_t *logical_rows_delta ) // Effect: Apply msg to leafentry (msn is ignored) // Calculate work done by message on leafentry and add it to caller's workdone counter. @@ -1082,26 +1147,34 @@ toku_ft_bn_apply_msg_once ( { size_t newsize=0, oldsize=0, workdone_this_le=0; LEAFENTRY new_le=0; - int64_t numbytes_delta = 0; // how many bytes of user data (not including overhead) were added or deleted from this row - int64_t numrows_delta = 0; // will be +1 or -1 or 0 (if row was added or deleted or not) + // how many bytes of user data (not including overhead) were added or + // deleted from this row + int64_t numbytes_delta = 0; + // will be +1 or -1 or 0 (if row was added or deleted or not) + int64_t numrows_delta = 0; + // will be +1, -1 or 0 if a message that was accounted for logically has + // changed in meaning such as an insert changed to an update or a delete + // changed to a noop + int64_t logical_rows_delta_le = 0; uint32_t key_storage_size = msg.kdbt()->size + sizeof(uint32_t); if (le) { oldsize = leafentry_memsize(le) + key_storage_size; } - // toku_le_apply_msg() may call bn_data::mempool_malloc_and_update_dmt() to allocate more space. - // That means le is guaranteed to not cause a sigsegv but it may point to a mempool that is - // no longer in use. We'll have to release the old mempool later. - toku_le_apply_msg( - msg, + // toku_le_apply_msg() may call bn_data::mempool_malloc_and_update_dmt() + // to allocate more space. That means le is guaranteed to not cause a + // sigsegv but it may point to a mempool that is no longer in use. + // We'll have to release the old mempool later. + logical_rows_delta_le = toku_le_apply_msg( + msg, le, &bn->data_buffer, idx, le_keylen, - gc_info, - &new_le, - &numbytes_delta - ); + gc_info, + &new_le, + &numbytes_delta); + // at this point, we cannot trust cmd->u.id.key to be valid. // The dmt may have realloced its mempool and freed the one containing key. @@ -1121,37 +1194,42 @@ toku_ft_bn_apply_msg_once ( numrows_delta = 1; } } - if (workdone) { // test programs may call with NULL + if (FT_LIKELY(workdone != NULL)) { // test programs may call with NULL *workdone += workdone_this_le; } + if (FT_LIKELY(logical_rows_delta != NULL)) { + *logical_rows_delta += logical_rows_delta_le; + } // now update stat64 statistics bn->stat64_delta.numrows += numrows_delta; bn->stat64_delta.numbytes += numbytes_delta; // the only reason stats_to_update may be null is for tests - if (stats_to_update) { + if (FT_LIKELY(stats_to_update != NULL)) { stats_to_update->numrows += numrows_delta; stats_to_update->numbytes += numbytes_delta; } - } static const uint32_t setval_tag = 0xee0ccb99; // this was gotten by doing "cat /dev/random|head -c4|od -x" to get a random number. We want to make sure that the user actually passes us the setval_extra_s that we passed in. struct setval_extra_s { uint32_t tag; bool did_set_val; - int setval_r; // any error code that setval_fun wants to return goes here. + // any error code that setval_fun wants to return goes here. + int setval_r; // need arguments for toku_ft_bn_apply_msg_once BASEMENTNODE bn; - MSN msn; // captured from original message, not currently used + // captured from original message, not currently used + MSN msn; XIDS xids; - const DBT *key; + const DBT* key; uint32_t idx; uint32_t le_keylen; LEAFENTRY le; - txn_gc_info *gc_info; - uint64_t * workdone; // set by toku_ft_bn_apply_msg_once() + txn_gc_info* gc_info; + uint64_t* workdone; // set by toku_ft_bn_apply_msg_once() STAT64INFO stats_to_update; + int64_t* logical_rows_delta; }; /* @@ -1170,29 +1248,45 @@ static void setval_fun (const DBT *new_val, void *svextra_v) { // can't leave scope until toku_ft_bn_apply_msg_once if // this is a delete DBT val; - ft_msg msg(svextra->key, - new_val ? new_val : toku_init_dbt(&val), - new_val ? FT_INSERT : FT_DELETE_ANY, - svextra->msn, svextra->xids); - toku_ft_bn_apply_msg_once(svextra->bn, msg, - svextra->idx, svextra->le_keylen, svextra->le, - svextra->gc_info, - svextra->workdone, svextra->stats_to_update); + ft_msg msg( + svextra->key, + new_val ? new_val : toku_init_dbt(&val), + new_val ? FT_INSERT : FT_DELETE_ANY, + svextra->msn, + svextra->xids); + toku_ft_bn_apply_msg_once( + svextra->bn, + msg, + svextra->idx, + svextra->le_keylen, + svextra->le, + svextra->gc_info, + svextra->workdone, + svextra->stats_to_update, + svextra->logical_rows_delta); svextra->setval_r = 0; } } -// We are already past the msn filter (in toku_ft_bn_apply_msg(), which calls do_update()), -// so capturing the msn in the setval_extra_s is not strictly required. The alternative -// would be to put a dummy msn in the messages created by setval_fun(), but preserving -// the original msn seems cleaner and it preserves accountability at a lower layer. -static int do_update(ft_update_func update_fun, const DESCRIPTOR_S *desc, BASEMENTNODE bn, const ft_msg &msg, uint32_t idx, - LEAFENTRY le, - void* keydata, - uint32_t keylen, - txn_gc_info *gc_info, - uint64_t * workdone, - STAT64INFO stats_to_update) { +// We are already past the msn filter (in toku_ft_bn_apply_msg(), which calls +// do_update()), so capturing the msn in the setval_extra_s is not strictly +// required. The alternative would be to put a dummy msn in the messages +// created by setval_fun(), but preserving the original msn seems cleaner and +// it preserves accountability at a lower layer. +static int do_update( + ft_update_func update_fun, + const DESCRIPTOR_S* desc, + BASEMENTNODE bn, + const ft_msg &msg, + uint32_t idx, + LEAFENTRY le, + void* keydata, + uint32_t keylen, + txn_gc_info* gc_info, + uint64_t* workdone, + STAT64INFO stats_to_update, + int64_t* logical_rows_delta) { + LEAFENTRY le_for_update; DBT key; const DBT *keyp; @@ -1232,39 +1326,52 @@ static int do_update(ft_update_func update_fun, const DESCRIPTOR_S *desc, BASEME } le_for_update = le; - struct setval_extra_s setval_extra = {setval_tag, false, 0, bn, msg.msn(), msg.xids(), - keyp, idx, keylen, le_for_update, gc_info, - workdone, stats_to_update}; - // call handlerton's ft->update_fun(), which passes setval_extra to setval_fun() + struct setval_extra_s setval_extra = { + setval_tag, + false, + 0, + bn, + msg.msn(), + msg.xids(), + keyp, + idx, + keylen, + le_for_update, + gc_info, + workdone, + stats_to_update, + logical_rows_delta + }; + // call handlerton's ft->update_fun(), which passes setval_extra + // to setval_fun() FAKE_DB(db, desc); int r = update_fun( &db, keyp, vdbtp, update_function_extra, - setval_fun, &setval_extra - ); + setval_fun, + &setval_extra); if (r == 0) { r = setval_extra.setval_r; } return r; } // Should be renamed as something like "apply_msg_to_basement()." -void -toku_ft_bn_apply_msg ( - const toku::comparator &cmp, +void toku_ft_bn_apply_msg( + const toku::comparator& cmp, ft_update_func update_fun, BASEMENTNODE bn, - const ft_msg &msg, - txn_gc_info *gc_info, - uint64_t *workdone, - STAT64INFO stats_to_update - ) + const ft_msg& msg, + txn_gc_info* gc_info, + uint64_t* workdone, + STAT64INFO stats_to_update, + int64_t* logical_rows_delta) { // Effect: // Put a msg into a leaf. -// Calculate work done by message on leafnode and add it to caller's workdone counter. +// Calculate work done by message on leafnode and add it to caller's +// workdone counter. // The leaf could end up "too big" or "too small". The caller must fix that up. -{ LEAFENTRY storeddata; void* key = NULL; uint32_t keylen = 0; @@ -1303,7 +1410,16 @@ toku_ft_bn_apply_msg ( } else { assert_zero(r); } - toku_ft_bn_apply_msg_once(bn, msg, idx, keylen, storeddata, gc_info, workdone, stats_to_update); + toku_ft_bn_apply_msg_once( + bn, + msg, + idx, + keylen, + storeddata, + gc_info, + workdone, + stats_to_update, + logical_rows_delta); // if the insertion point is within a window of the right edge of // the leaf then it is sequential @@ -1331,12 +1447,19 @@ toku_ft_bn_apply_msg ( &storeddata, &key, &keylen, - &idx - ); + &idx); if (r == DB_NOTFOUND) break; assert_zero(r); - toku_ft_bn_apply_msg_once(bn, msg, idx, keylen, storeddata, gc_info, workdone, stats_to_update); - + toku_ft_bn_apply_msg_once( + bn, + msg, + idx, + keylen, + storeddata, + gc_info, + workdone, + stats_to_update, + logical_rows_delta); break; } case FT_OPTIMIZE_FOR_UPGRADE: @@ -1352,13 +1475,27 @@ toku_ft_bn_apply_msg ( assert_zero(r); int deleted = 0; if (!le_is_clean(storeddata)) { //If already clean, nothing to do. - // message application code needs a key in order to determine how much - // work was done by this message. since this is a broadcast message, - // we have to create a new message whose key is the current le's key. + // message application code needs a key in order to determine + // how much work was done by this message. since this is a + // broadcast message, we have to create a new message whose + // key is the current le's key. DBT curr_keydbt; - ft_msg curr_msg(toku_fill_dbt(&curr_keydbt, curr_keyp, curr_keylen), - msg.vdbt(), msg.type(), msg.msn(), msg.xids()); - toku_ft_bn_apply_msg_once(bn, curr_msg, idx, curr_keylen, storeddata, gc_info, workdone, stats_to_update); + ft_msg curr_msg( + toku_fill_dbt(&curr_keydbt, curr_keyp, curr_keylen), + msg.vdbt(), + msg.type(), + msg.msn(), + msg.xids()); + toku_ft_bn_apply_msg_once( + bn, + curr_msg, + idx, + curr_keylen, + storeddata, + gc_info, + workdone, + stats_to_update, + logical_rows_delta); // at this point, we cannot trust msg.kdbt to be valid. uint32_t new_dmt_size = bn->data_buffer.num_klpairs(); if (new_dmt_size != num_klpairs) { @@ -1386,13 +1523,27 @@ toku_ft_bn_apply_msg ( assert_zero(r); int deleted = 0; if (le_has_xids(storeddata, msg.xids())) { - // message application code needs a key in order to determine how much - // work was done by this message. since this is a broadcast message, - // we have to create a new message whose key is the current le's key. + // message application code needs a key in order to determine + // how much work was done by this message. since this is a + // broadcast message, we have to create a new message whose key + // is the current le's key. DBT curr_keydbt; - ft_msg curr_msg(toku_fill_dbt(&curr_keydbt, curr_keyp, curr_keylen), - msg.vdbt(), msg.type(), msg.msn(), msg.xids()); - toku_ft_bn_apply_msg_once(bn, curr_msg, idx, curr_keylen, storeddata, gc_info, workdone, stats_to_update); + ft_msg curr_msg( + toku_fill_dbt(&curr_keydbt, curr_keyp, curr_keylen), + msg.vdbt(), + msg.type(), + msg.msn(), + msg.xids()); + toku_ft_bn_apply_msg_once( + bn, + curr_msg, + idx, + curr_keylen, + storeddata, + gc_info, + workdone, + stats_to_update, + logical_rows_delta); uint32_t new_dmt_size = bn->data_buffer.num_klpairs(); if (new_dmt_size != num_klpairs) { paranoid_invariant(new_dmt_size + 1 == num_klpairs); @@ -1424,9 +1575,33 @@ toku_ft_bn_apply_msg ( key = msg.kdbt()->data; keylen = msg.kdbt()->size; } - r = do_update(update_fun, cmp.get_descriptor(), bn, msg, idx, NULL, NULL, 0, gc_info, workdone, stats_to_update); + r = do_update( + update_fun, + cmp.get_descriptor(), + bn, + msg, + idx, + NULL, + NULL, + 0, + gc_info, + workdone, + stats_to_update, + logical_rows_delta); } else if (r==0) { - r = do_update(update_fun, cmp.get_descriptor(), bn, msg, idx, storeddata, key, keylen, gc_info, workdone, stats_to_update); + r = do_update( + update_fun, + cmp.get_descriptor(), + bn, + msg, + idx, + storeddata, + key, + keylen, + gc_info, + workdone, + stats_to_update, + logical_rows_delta); } // otherwise, a worse error, just return it break; } @@ -1434,6 +1609,12 @@ toku_ft_bn_apply_msg ( // apply to all leafentries. uint32_t idx = 0; uint32_t num_leafentries_before; + // This is used to avoid having the logical row count changed on apply + // of this message since it will return a negative number of the number + // of leaf entries visited and cause the ft header value to go to 0; + // This message will not change the number of rows, so just use the + // bogus value. + int64_t temp_logical_rows_delta = 0; while (idx < (num_leafentries_before = bn->data_buffer.num_klpairs())) { void* curr_key = nullptr; uint32_t curr_keylen = 0; @@ -1449,7 +1630,19 @@ toku_ft_bn_apply_msg ( // This is broken below. Have a compilation error checked // in as a reminder - r = do_update(update_fun, cmp.get_descriptor(), bn, msg, idx, storeddata, curr_key, curr_keylen, gc_info, workdone, stats_to_update); + r = do_update( + update_fun, + cmp.get_descriptor(), + bn, + msg, + idx, + storeddata, + curr_key, + curr_keylen, + gc_info, + workdone, + stats_to_update, + &temp_logical_rows_delta); assert_zero(r); if (num_leafentries_before == bn->data_buffer.num_klpairs()) { @@ -1810,24 +2003,22 @@ void toku_ftnode_leaf_run_gc(FT ft, FTNODE node) { } } -void -toku_ftnode_put_msg ( +void toku_ftnode_put_msg( const toku::comparator &cmp, ft_update_func update_fun, FTNODE node, int target_childnum, const ft_msg &msg, bool is_fresh, - txn_gc_info *gc_info, + txn_gc_info* gc_info, size_t flow_deltas[], - STAT64INFO stats_to_update - ) + STAT64INFO stats_to_update, + int64_t* logical_rows_delta) { // Effect: Push message into the subtree rooted at NODE. // If NODE is a leaf, then // put message into leaf, applying it to the leafentries // If NODE is a nonleaf, then push the message into the message buffer(s) of the relevent child(ren). // The node may become overfull. That's not our problem. -{ toku_ftnode_assert_fully_in_memory(node); // // see comments in toku_ft_leaf_apply_msg @@ -1836,26 +2027,40 @@ toku_ftnode_put_msg ( // and instead defer to these functions // if (node->height==0) { - toku_ft_leaf_apply_msg(cmp, update_fun, node, target_childnum, msg, gc_info, nullptr, stats_to_update); + toku_ft_leaf_apply_msg( + cmp, + update_fun, + node, + target_childnum, msg, + gc_info, + nullptr, + stats_to_update, + logical_rows_delta); } else { - ft_nonleaf_put_msg(cmp, node, target_childnum, msg, is_fresh, flow_deltas); + ft_nonleaf_put_msg( + cmp, + node, + target_childnum, + msg, + is_fresh, + flow_deltas); } } -// Effect: applies the message to the leaf if the appropriate basement node is in memory. -// This function is called during message injection and/or flushing, so the entire -// node MUST be in memory. +// Effect: applies the message to the leaf if the appropriate basement node is +// in memory. This function is called during message injection and/or +// flushing, so the entire node MUST be in memory. void toku_ft_leaf_apply_msg( - const toku::comparator &cmp, + const toku::comparator& cmp, ft_update_func update_fun, FTNODE node, int target_childnum, // which child to inject to, or -1 if unknown - const ft_msg &msg, - txn_gc_info *gc_info, - uint64_t *workdone, - STAT64INFO stats_to_update - ) -{ + const ft_msg& msg, + txn_gc_info* gc_info, + uint64_t* workdone, + STAT64INFO stats_to_update, + int64_t* logical_rows_delta) { + VERIFY_NODE(t, node); toku_ftnode_assert_fully_in_memory(node); @@ -1891,34 +2096,36 @@ void toku_ft_leaf_apply_msg( BASEMENTNODE bn = BLB(node, childnum); if (msg.msn().msn > bn->max_msn_applied.msn) { bn->max_msn_applied = msg.msn(); - toku_ft_bn_apply_msg(cmp, - update_fun, - bn, - msg, - gc_info, - workdone, - stats_to_update); + toku_ft_bn_apply_msg( + cmp, + update_fun, + bn, + msg, + gc_info, + workdone, + stats_to_update, + logical_rows_delta); } else { toku_ft_status_note_msn_discard(); } - } - else if (ft_msg_type_applies_all(msg.type())) { + } else if (ft_msg_type_applies_all(msg.type())) { for (int childnum=0; childnum<node->n_children; childnum++) { if (msg.msn().msn > BLB(node, childnum)->max_msn_applied.msn) { BLB(node, childnum)->max_msn_applied = msg.msn(); - toku_ft_bn_apply_msg(cmp, - update_fun, - BLB(node, childnum), - msg, - gc_info, - workdone, - stats_to_update); + toku_ft_bn_apply_msg( + cmp, + update_fun, + BLB(node, childnum), + msg, + gc_info, + workdone, + stats_to_update, + logical_rows_delta); } else { toku_ft_status_note_msn_discard(); } } - } - else if (!ft_msg_type_does_nothing(msg.type())) { + } else if (!ft_msg_type_does_nothing(msg.type())) { invariant(ft_msg_type_does_nothing(msg.type())); } VERIFY_NODE(t, node); diff --git a/storage/tokudb/PerconaFT/ft/node.h b/storage/tokudb/PerconaFT/ft/node.h index 9d910491682..ad0298e81c5 100644 --- a/storage/tokudb/PerconaFT/ft/node.h +++ b/storage/tokudb/PerconaFT/ft/node.h @@ -382,25 +382,54 @@ enum reactivity toku_ftnode_get_leaf_reactivity(FTNODE node, uint32_t nodesize); * If k is equal to some pivot, then we return the next (to the right) * childnum. */ -int toku_ftnode_hot_next_child(FTNODE node, const DBT *k, const toku::comparator &cmp); - -void toku_ftnode_put_msg(const toku::comparator &cmp, ft_update_func update_fun, - FTNODE node, int target_childnum, - const ft_msg &msg, bool is_fresh, txn_gc_info *gc_info, - size_t flow_deltas[], STAT64INFO stats_to_update); - -void toku_ft_bn_apply_msg_once(BASEMENTNODE bn, const ft_msg &msg, uint32_t idx, - uint32_t le_keylen, LEAFENTRY le, txn_gc_info *gc_info, - uint64_t *workdonep, STAT64INFO stats_to_update); - -void toku_ft_bn_apply_msg(const toku::comparator &cmp, ft_update_func update_fun, - BASEMENTNODE bn, const ft_msg &msg, txn_gc_info *gc_info, - uint64_t *workdone, STAT64INFO stats_to_update); - -void toku_ft_leaf_apply_msg(const toku::comparator &cmp, ft_update_func update_fun, - FTNODE node, int target_childnum, - const ft_msg &msg, txn_gc_info *gc_info, - uint64_t *workdone, STAT64INFO stats_to_update); +int toku_ftnode_hot_next_child( + FTNODE node, + const DBT* k, + const toku::comparator &cmp); + +void toku_ftnode_put_msg( + const toku::comparator& cmp, + ft_update_func update_fun, + FTNODE node, + int target_childnum, + const ft_msg& msg, + bool is_fresh, + txn_gc_info* gc_info, + size_t flow_deltas[], + STAT64INFO stats_to_update, + int64_t* logical_rows_delta); + +void toku_ft_bn_apply_msg_once( + BASEMENTNODE bn, + const ft_msg& msg, + uint32_t idx, + uint32_t le_keylen, + LEAFENTRY le, + txn_gc_info* gc_info, + uint64_t* workdonep, + STAT64INFO stats_to_update, + int64_t* logical_rows_delta); + +void toku_ft_bn_apply_msg( + const toku::comparator& cmp, + ft_update_func update_fun, + BASEMENTNODE bn, + const ft_msg& msg, + txn_gc_info* gc_info, + uint64_t* workdone, + STAT64INFO stats_to_update, + int64_t* logical_rows_delta); + +void toku_ft_leaf_apply_msg( + const toku::comparator& cmp, + ft_update_func update_fun, + FTNODE node, + int target_childnum, + const ft_msg& msg, + txn_gc_info* gc_info, + uint64_t* workdone, + STAT64INFO stats_to_update, + int64_t* logical_rows_delta); // // Message management for orthopush diff --git a/storage/tokudb/PerconaFT/ft/serialize/ft-serialize.cc b/storage/tokudb/PerconaFT/ft/serialize/ft-serialize.cc index a7bc2949276..49d4368a3ab 100644 --- a/storage/tokudb/PerconaFT/ft/serialize/ft-serialize.cc +++ b/storage/tokudb/PerconaFT/ft/serialize/ft-serialize.cc @@ -323,6 +323,13 @@ int deserialize_ft_versioned(int fd, struct rbuf *rb, FT *ftp, uint32_t version) fanout = rbuf_int(rb); } + uint64_t on_disk_logical_rows; + on_disk_logical_rows = (uint64_t)-1; + if (ft->layout_version_read_from_disk >= FT_LAYOUT_VERSION_29) { + on_disk_logical_rows = rbuf_ulonglong(rb); + } + ft->in_memory_logical_rows = on_disk_logical_rows; + (void) rbuf_int(rb); //Read in checksum and ignore (already verified). if (rb->ndone != rb->size) { fprintf(stderr, "Header size did not match contents.\n"); @@ -357,7 +364,8 @@ int deserialize_ft_versioned(int fd, struct rbuf *rb, FT *ftp, uint32_t version) .count_of_optimize_in_progress = count_of_optimize_in_progress, .count_of_optimize_in_progress_read_from_disk = count_of_optimize_in_progress, .msn_at_start_of_last_completed_optimize = msn_at_start_of_last_completed_optimize, - .on_disk_stats = on_disk_stats + .on_disk_stats = on_disk_stats, + .on_disk_logical_rows = on_disk_logical_rows }; XMEMDUP(ft->h, &h); } @@ -408,6 +416,8 @@ serialize_ft_min_size (uint32_t version) { size_t size = 0; switch(version) { + case FT_LAYOUT_VERSION_29: + size += sizeof(uint64_t); // logrows in ft case FT_LAYOUT_VERSION_28: size += sizeof(uint32_t); // fanout in ft case FT_LAYOUT_VERSION_27: @@ -754,6 +764,7 @@ void toku_serialize_ft_to_wbuf ( wbuf_MSN(wbuf, h->highest_unused_msn_for_upgrade); wbuf_MSN(wbuf, h->max_msn_in_ft); wbuf_int(wbuf, h->fanout); + wbuf_ulonglong(wbuf, h->on_disk_logical_rows); uint32_t checksum = toku_x1764_finish(&wbuf->checksum); wbuf_int(wbuf, checksum); lazy_assert(wbuf->ndone == wbuf->size); diff --git a/storage/tokudb/PerconaFT/ft/serialize/ft_layout_version.h b/storage/tokudb/PerconaFT/ft/serialize/ft_layout_version.h index 72b6882bc06..9407a568337 100644 --- a/storage/tokudb/PerconaFT/ft/serialize/ft_layout_version.h +++ b/storage/tokudb/PerconaFT/ft/serialize/ft_layout_version.h @@ -68,6 +68,7 @@ enum ft_layout_version_e { FT_LAYOUT_VERSION_26 = 26, // Hojo: basements store key/vals separately on disk for fixed klpair length BNs FT_LAYOUT_VERSION_27 = 27, // serialize message trees with nonleaf buffers to avoid key, msn sort on deserialize FT_LAYOUT_VERSION_28 = 28, // Add fanout to ft_header + FT_LAYOUT_VERSION_29 = 29, // Add logrows to ft_header FT_NEXT_VERSION, // the version after the current version FT_LAYOUT_VERSION = FT_NEXT_VERSION-1, // A hack so I don't have to change this line. FT_LAYOUT_MIN_SUPPORTED_VERSION = FT_LAYOUT_VERSION_13, // Minimum version supported diff --git a/storage/tokudb/PerconaFT/ft/tests/CMakeLists.txt b/storage/tokudb/PerconaFT/ft/tests/CMakeLists.txt index d9f0da6a1b0..0098b6091be 100644 --- a/storage/tokudb/PerconaFT/ft/tests/CMakeLists.txt +++ b/storage/tokudb/PerconaFT/ft/tests/CMakeLists.txt @@ -29,7 +29,7 @@ if(BUILD_TESTING OR BUILD_FT_TESTS) add_test(ft/logcursor-bw echo "logcursor-bw must be run manually (needs logs to iterate over).") foreach(test ${tests}) - add_executable(${test} ${test}) + add_executable(${test} ${test}.cc) target_link_libraries(${test} ft ${LIBTOKUPORTABILITY}) set_target_properties(${test} PROPERTIES POSITION_INDEPENDENT_CODE ON) add_space_separated_property(TARGET ${test} COMPILE_FLAGS -fvisibility=hidden) diff --git a/storage/tokudb/PerconaFT/ft/tests/make-tree.cc b/storage/tokudb/PerconaFT/ft/tests/make-tree.cc index 663bbf3beb2..c83517b5f64 100644 --- a/storage/tokudb/PerconaFT/ft/tests/make-tree.cc +++ b/storage/tokudb/PerconaFT/ft/tests/make-tree.cc @@ -74,7 +74,16 @@ append_leaf(FTNODE leafnode, void *key, size_t keylen, void *val, size_t vallen) // apply an insert to the leaf node txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, false); ft_msg msg(&thekey, &theval, FT_INSERT, msn, toku_xids_get_root_xids()); - toku_ft_bn_apply_msg_once(BLB(leafnode,0), msg, idx, keylen, NULL, &gc_info, NULL, NULL); + toku_ft_bn_apply_msg_once( + BLB(leafnode, 0), + msg, + idx, + keylen, + NULL, + &gc_info, + NULL, + NULL, + NULL); leafnode->max_msn_applied_to_node_on_disk = msn; diff --git a/storage/tokudb/PerconaFT/ft/tests/msnfilter.cc b/storage/tokudb/PerconaFT/ft/tests/msnfilter.cc index 737c3556ad6..d9608258054 100644 --- a/storage/tokudb/PerconaFT/ft/tests/msnfilter.cc +++ b/storage/tokudb/PerconaFT/ft/tests/msnfilter.cc @@ -82,46 +82,82 @@ append_leaf(FT_HANDLE ft, FTNODE leafnode, void *key, uint32_t keylen, void *val ft_msg msg(&thekey, &theval, FT_INSERT, msn, toku_xids_get_root_xids()); txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, false); - toku_ft_leaf_apply_msg(ft->ft->cmp, ft->ft->update_fun, leafnode, -1, msg, &gc_info, nullptr, nullptr); + toku_ft_leaf_apply_msg( + ft->ft->cmp, + ft->ft->update_fun, + leafnode, + -1, + msg, + &gc_info, + nullptr, + nullptr, + nullptr); { - int r = toku_ft_lookup(ft, &thekey, lookup_checkf, &pair); - assert(r==0); - assert(pair.call_count==1); + int r = toku_ft_lookup(ft, &thekey, lookup_checkf, &pair); + assert(r==0); + assert(pair.call_count==1); } ft_msg badmsg(&thekey, &badval, FT_INSERT, msn, toku_xids_get_root_xids()); - toku_ft_leaf_apply_msg(ft->ft->cmp, ft->ft->update_fun, leafnode, -1, badmsg, &gc_info, nullptr, nullptr); + toku_ft_leaf_apply_msg( + ft->ft->cmp, + ft->ft->update_fun, + leafnode, + -1, + badmsg, + &gc_info, + nullptr, + nullptr, + nullptr); // message should be rejected for duplicate msn, row should still have original val { - int r = toku_ft_lookup(ft, &thekey, lookup_checkf, &pair); - assert(r==0); - assert(pair.call_count==2); + int r = toku_ft_lookup(ft, &thekey, lookup_checkf, &pair); + assert(r==0); + assert(pair.call_count==2); } // now verify that message with proper msn gets through msn = next_dummymsn(); ft->ft->h->max_msn_in_ft = msn; ft_msg msg2(&thekey, &val2, FT_INSERT, msn, toku_xids_get_root_xids()); - toku_ft_leaf_apply_msg(ft->ft->cmp, ft->ft->update_fun, leafnode, -1, msg2, &gc_info, nullptr, nullptr); + toku_ft_leaf_apply_msg( + ft->ft->cmp, + ft->ft->update_fun, + leafnode, + -1, + msg2, + &gc_info, + nullptr, + nullptr, + nullptr); // message should be accepted, val should have new value { - int r = toku_ft_lookup(ft, &thekey, lookup_checkf, &pair2); - assert(r==0); - assert(pair2.call_count==1); + int r = toku_ft_lookup(ft, &thekey, lookup_checkf, &pair2); + assert(r==0); + assert(pair2.call_count==1); } // now verify that message with lesser (older) msn is rejected msn.msn = msn.msn - 10; ft_msg msg3(&thekey, &badval, FT_INSERT, msn, toku_xids_get_root_xids()); - toku_ft_leaf_apply_msg(ft->ft->cmp, ft->ft->update_fun, leafnode, -1, msg3, &gc_info, nullptr, nullptr); + toku_ft_leaf_apply_msg( + ft->ft->cmp, + ft->ft->update_fun, + leafnode, + -1, + msg3, + &gc_info, + nullptr, + nullptr, + nullptr); // message should be rejected, val should still have value in pair2 { - int r = toku_ft_lookup(ft, &thekey, lookup_checkf, &pair2); - assert(r==0); - assert(pair2.call_count==2); + int r = toku_ft_lookup(ft, &thekey, lookup_checkf, &pair2); + assert(r==0); + assert(pair2.call_count==2); } // dont forget to dirty the node diff --git a/storage/tokudb/PerconaFT/ft/tests/orthopush-flush.cc b/storage/tokudb/PerconaFT/ft/tests/orthopush-flush.cc index 055a38e5f6d..393fb88ac2e 100644 --- a/storage/tokudb/PerconaFT/ft/tests/orthopush-flush.cc +++ b/storage/tokudb/PerconaFT/ft/tests/orthopush-flush.cc @@ -137,8 +137,24 @@ insert_random_message_to_bn( *keyp = toku_xmemdup(keydbt->data, keydbt->size); ft_msg msg(keydbt, valdbt, FT_INSERT, msn, xids); int64_t numbytes; - toku_le_apply_msg(msg, NULL, NULL, 0, keydbt->size, &non_mvcc_gc_info, save, &numbytes); - toku_ft_bn_apply_msg(t->ft->cmp, t->ft->update_fun, blb, msg, &non_mvcc_gc_info, NULL, NULL); + toku_le_apply_msg( + msg, + NULL, + NULL, + 0, + keydbt->size, + &non_mvcc_gc_info, + save, + &numbytes); + toku_ft_bn_apply_msg( + t->ft->cmp, + t->ft->update_fun, + blb, + msg, + &non_mvcc_gc_info, + NULL, + NULL, + NULL); if (msn.msn > blb->max_msn_applied.msn) { blb->max_msn_applied = msn; } @@ -182,12 +198,36 @@ insert_same_message_to_bns( *keyp = toku_xmemdup(keydbt->data, keydbt->size); ft_msg msg(keydbt, valdbt, FT_INSERT, msn, xids); int64_t numbytes; - toku_le_apply_msg(msg, NULL, NULL, 0, keydbt->size, &non_mvcc_gc_info, save, &numbytes); - toku_ft_bn_apply_msg(t->ft->cmp, t->ft->update_fun, blb1, msg, &non_mvcc_gc_info, NULL, NULL); + toku_le_apply_msg( + msg, + NULL, + NULL, + 0, + keydbt->size, + &non_mvcc_gc_info, + save, + &numbytes); + toku_ft_bn_apply_msg( + t->ft->cmp, + t->ft->update_fun, + blb1, + msg, + &non_mvcc_gc_info, + NULL, + NULL, + NULL); if (msn.msn > blb1->max_msn_applied.msn) { blb1->max_msn_applied = msn; } - toku_ft_bn_apply_msg(t->ft->cmp, t->ft->update_fun, blb2, msg, &non_mvcc_gc_info, NULL, NULL); + toku_ft_bn_apply_msg( + t->ft->cmp, + t->ft->update_fun, + blb2, + msg, + &non_mvcc_gc_info, + NULL, + NULL, + NULL); if (msn.msn > blb2->max_msn_applied.msn) { blb2->max_msn_applied = msn; } @@ -619,7 +659,16 @@ flush_to_leaf(FT_HANDLE t, bool make_leaf_up_to_date, bool use_flush) { if (make_leaf_up_to_date) { for (i = 0; i < num_parent_messages; ++i) { if (!parent_messages_is_fresh[i]) { - toku_ft_leaf_apply_msg(t->ft->cmp, t->ft->update_fun, child, -1, *parent_messages[i], &non_mvcc_gc_info, NULL, NULL); + toku_ft_leaf_apply_msg( + t->ft->cmp, + t->ft->update_fun, + child, + -1, + *parent_messages[i], + &non_mvcc_gc_info, + NULL, + NULL, + NULL); } } for (i = 0; i < 8; ++i) { @@ -842,7 +891,16 @@ flush_to_leaf_with_keyrange(FT_HANDLE t, bool make_leaf_up_to_date) { for (i = 0; i < num_parent_messages; ++i) { if (dummy_cmp(parent_messages[i]->kdbt(), &childkeys[7]) <= 0 && !parent_messages_is_fresh[i]) { - toku_ft_leaf_apply_msg(t->ft->cmp, t->ft->update_fun, child, -1, *parent_messages[i], &non_mvcc_gc_info, NULL, NULL); + toku_ft_leaf_apply_msg( + t->ft->cmp, + t->ft->update_fun, + child, + -1, + *parent_messages[i], + &non_mvcc_gc_info, + NULL, + NULL, + NULL); } } for (i = 0; i < 8; ++i) { @@ -1045,8 +1103,26 @@ compare_apply_and_flush(FT_HANDLE t, bool make_leaf_up_to_date) { if (make_leaf_up_to_date) { for (i = 0; i < num_parent_messages; ++i) { if (!parent_messages_is_fresh[i]) { - toku_ft_leaf_apply_msg(t->ft->cmp, t->ft->update_fun, child1, -1, *parent_messages[i], &non_mvcc_gc_info, NULL, NULL); - toku_ft_leaf_apply_msg(t->ft->cmp, t->ft->update_fun, child2, -1, *parent_messages[i], &non_mvcc_gc_info, NULL, NULL); + toku_ft_leaf_apply_msg( + t->ft->cmp, + t->ft->update_fun, + child1, + -1, + *parent_messages[i], + &non_mvcc_gc_info, + NULL, + NULL, + NULL); + toku_ft_leaf_apply_msg( + t->ft->cmp, + t->ft->update_fun, + child2, + -1, + *parent_messages[i], + &non_mvcc_gc_info, + NULL, + NULL, + NULL); } } for (i = 0; i < 8; ++i) { diff --git a/storage/tokudb/PerconaFT/ft/tests/verify-bad-msn.cc b/storage/tokudb/PerconaFT/ft/tests/verify-bad-msn.cc index 68fac0e6a9c..40af5dab7ad 100644 --- a/storage/tokudb/PerconaFT/ft/tests/verify-bad-msn.cc +++ b/storage/tokudb/PerconaFT/ft/tests/verify-bad-msn.cc @@ -78,7 +78,16 @@ append_leaf(FTNODE leafnode, void *key, size_t keylen, void *val, size_t vallen) // apply an insert to the leaf node ft_msg msg(&thekey, &theval, FT_INSERT, msn, toku_xids_get_root_xids()); txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, false); - toku_ft_bn_apply_msg_once(BLB(leafnode, 0), msg, idx, keylen, NULL, &gc_info, NULL, NULL); + toku_ft_bn_apply_msg_once( + BLB(leafnode, 0), + msg, + idx, + keylen, + NULL, + &gc_info, + NULL, + NULL, + NULL); // Create bad tree (don't do following): // leafnode->max_msn_applied_to_node = msn; diff --git a/storage/tokudb/PerconaFT/ft/tests/verify-bad-pivots.cc b/storage/tokudb/PerconaFT/ft/tests/verify-bad-pivots.cc index 49b2b8a6c21..37054eb119a 100644 --- a/storage/tokudb/PerconaFT/ft/tests/verify-bad-pivots.cc +++ b/storage/tokudb/PerconaFT/ft/tests/verify-bad-pivots.cc @@ -65,7 +65,16 @@ append_leaf(FTNODE leafnode, void *key, size_t keylen, void *val, size_t vallen) MSN msn = next_dummymsn(); ft_msg msg(&thekey, &theval, FT_INSERT, msn, toku_xids_get_root_xids()); txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, false); - toku_ft_bn_apply_msg_once(BLB(leafnode, 0), msg, idx, keylen, NULL, &gc_info, NULL, NULL); + toku_ft_bn_apply_msg_once( + BLB(leafnode, 0), + msg, + idx, + keylen, + NULL, + &gc_info, + NULL, + NULL, + NULL); // dont forget to dirty the node leafnode->dirty = 1; diff --git a/storage/tokudb/PerconaFT/ft/tests/verify-dup-in-leaf.cc b/storage/tokudb/PerconaFT/ft/tests/verify-dup-in-leaf.cc index 72c4063f51f..42e82884432 100644 --- a/storage/tokudb/PerconaFT/ft/tests/verify-dup-in-leaf.cc +++ b/storage/tokudb/PerconaFT/ft/tests/verify-dup-in-leaf.cc @@ -66,7 +66,16 @@ append_leaf(FTNODE leafnode, void *key, size_t keylen, void *val, size_t vallen) MSN msn = next_dummymsn(); ft_msg msg(&thekey, &theval, FT_INSERT, msn, toku_xids_get_root_xids()); txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, false); - toku_ft_bn_apply_msg_once(BLB(leafnode, 0), msg, idx, keylen, NULL, &gc_info, NULL, NULL); + toku_ft_bn_apply_msg_once( + BLB(leafnode, 0), + msg, + idx, + keylen, + NULL, + &gc_info, + NULL, + NULL, + NULL); // dont forget to dirty the node leafnode->dirty = 1; diff --git a/storage/tokudb/PerconaFT/ft/tests/verify-dup-pivots.cc b/storage/tokudb/PerconaFT/ft/tests/verify-dup-pivots.cc index f569f502dc8..b3e8663ed3b 100644 --- a/storage/tokudb/PerconaFT/ft/tests/verify-dup-pivots.cc +++ b/storage/tokudb/PerconaFT/ft/tests/verify-dup-pivots.cc @@ -65,7 +65,16 @@ append_leaf(FTNODE leafnode, void *key, size_t keylen, void *val, size_t vallen) MSN msn = next_dummymsn(); ft_msg msg(&thekey, &theval, FT_INSERT, msn, toku_xids_get_root_xids()); txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, false); - toku_ft_bn_apply_msg_once(BLB(leafnode, 0), msg, idx, keylen, NULL, &gc_info, NULL, NULL); + toku_ft_bn_apply_msg_once( + BLB(leafnode, 0), + msg, + idx, + keylen, + NULL, + &gc_info, + NULL, + NULL, + NULL); // dont forget to dirty the node leafnode->dirty = 1; diff --git a/storage/tokudb/PerconaFT/ft/tests/verify-misrouted-msgs.cc b/storage/tokudb/PerconaFT/ft/tests/verify-misrouted-msgs.cc index 3a6db8ee4de..df5c21ca64e 100644 --- a/storage/tokudb/PerconaFT/ft/tests/verify-misrouted-msgs.cc +++ b/storage/tokudb/PerconaFT/ft/tests/verify-misrouted-msgs.cc @@ -66,7 +66,16 @@ append_leaf(FTNODE leafnode, void *key, size_t keylen, void *val, size_t vallen) MSN msn = next_dummymsn(); ft_msg msg(&thekey, &theval, FT_INSERT, msn, toku_xids_get_root_xids()); txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, false); - toku_ft_bn_apply_msg_once(BLB(leafnode,0), msg, idx, keylen, NULL, &gc_info, NULL, NULL); + toku_ft_bn_apply_msg_once( + BLB(leafnode, 0), + msg, + idx, + keylen, + NULL, + &gc_info, + NULL, + NULL, + NULL); // dont forget to dirty the node leafnode->dirty = 1; diff --git a/storage/tokudb/PerconaFT/ft/tests/verify-unsorted-leaf.cc b/storage/tokudb/PerconaFT/ft/tests/verify-unsorted-leaf.cc index 4392887718f..4eccb06c1f3 100644 --- a/storage/tokudb/PerconaFT/ft/tests/verify-unsorted-leaf.cc +++ b/storage/tokudb/PerconaFT/ft/tests/verify-unsorted-leaf.cc @@ -68,7 +68,16 @@ append_leaf(FTNODE leafnode, void *key, size_t keylen, void *val, size_t vallen) MSN msn = next_dummymsn(); ft_msg msg(&thekey, &theval, FT_INSERT, msn, toku_xids_get_root_xids()); txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, false); - toku_ft_bn_apply_msg_once(BLB(leafnode, 0), msg, idx, keylen, NULL, &gc_info, NULL, NULL); + toku_ft_bn_apply_msg_once( + BLB(leafnode, 0), + msg, + idx, + keylen, + NULL, + &gc_info, + NULL, + NULL, + NULL); // dont forget to dirty the node leafnode->dirty = 1; diff --git a/storage/tokudb/PerconaFT/ft/tests/verify-unsorted-pivots.cc b/storage/tokudb/PerconaFT/ft/tests/verify-unsorted-pivots.cc index e3167bd3dc1..4492ea9364a 100644 --- a/storage/tokudb/PerconaFT/ft/tests/verify-unsorted-pivots.cc +++ b/storage/tokudb/PerconaFT/ft/tests/verify-unsorted-pivots.cc @@ -65,7 +65,16 @@ append_leaf(FTNODE leafnode, void *key, size_t keylen, void *val, size_t vallen) MSN msn = next_dummymsn(); ft_msg msg(&thekey, &theval, FT_INSERT, msn, toku_xids_get_root_xids()); txn_gc_info gc_info(nullptr, TXNID_NONE, TXNID_NONE, false); - toku_ft_bn_apply_msg_once(BLB(leafnode, 0), msg, idx, keylen, NULL, &gc_info, NULL, NULL); + toku_ft_bn_apply_msg_once( + BLB(leafnode, 0), + msg, + idx, + keylen, + NULL, + &gc_info, + NULL, + NULL, + NULL); // dont forget to dirty the node leafnode->dirty = 1; diff --git a/storage/tokudb/PerconaFT/ft/txn/rollback-apply.cc b/storage/tokudb/PerconaFT/ft/txn/rollback-apply.cc index 6a8c0d45b45..df830afd0df 100644 --- a/storage/tokudb/PerconaFT/ft/txn/rollback-apply.cc +++ b/storage/tokudb/PerconaFT/ft/txn/rollback-apply.cc @@ -186,6 +186,7 @@ int toku_rollback_commit(TOKUTXN txn, LSN lsn) { // Append the list to the front of the parent. if (child_log->oldest_logentry) { // There are some entries, so link them in. + parent_log->dirty = true; child_log->oldest_logentry->prev = parent_log->newest_logentry; if (!parent_log->oldest_logentry) { parent_log->oldest_logentry = child_log->oldest_logentry; diff --git a/storage/tokudb/PerconaFT/ft/txn/txn.cc b/storage/tokudb/PerconaFT/ft/txn/txn.cc index 57f3c105628..dd03073a3ec 100644 --- a/storage/tokudb/PerconaFT/ft/txn/txn.cc +++ b/storage/tokudb/PerconaFT/ft/txn/txn.cc @@ -248,11 +248,24 @@ static txn_child_manager tcm; .xa_xid = {0, 0, 0, ""}, .progress_poll_fun = NULL, .progress_poll_fun_extra = NULL, - .txn_lock = TOKU_MUTEX_INITIALIZER, + + // You cannot initialize txn_lock a TOKU_MUTEX_INITIALIZER, because we + // will initialize it in the code below, and it cannot already + // be initialized at that point. Also, in general, you don't + // get to use PTHREAD_MUTEX_INITALIZER (which is what is inside + // TOKU_MUTEX_INITIALIZER) except in static variables, and this + // is initializing an auto variable. + // + // And we cannot simply avoid initializing these fields + // because, although it avoids -Wmissing-field-initializer + // errors under gcc, it gets other errors about non-trivial + // designated initializers not being supported. + + .txn_lock = ZERO_MUTEX_INITIALIZER, // Not TOKU_MUTEX_INITIALIZER .open_fts = open_fts, .roll_info = roll_info, - .state_lock = TOKU_MUTEX_INITIALIZER, - .state_cond = TOKU_COND_INITIALIZER, + .state_lock = ZERO_MUTEX_INITIALIZER, // Not TOKU_MUTEX_INITIALIZER + .state_cond = ZERO_COND_INITIALIZER, // Not TOKU_COND_INITIALIZER .state = TOKUTXN_LIVE, .num_pin = 0, .client_id = 0, diff --git a/storage/tokudb/PerconaFT/ft/txn/txn_manager.cc b/storage/tokudb/PerconaFT/ft/txn/txn_manager.cc index 551fd32b8d5..805c60d30be 100644 --- a/storage/tokudb/PerconaFT/ft/txn/txn_manager.cc +++ b/storage/tokudb/PerconaFT/ft/txn/txn_manager.cc @@ -45,7 +45,15 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #include "ft/txn/txn_manager.h" #include "ft/txn/rollback.h" #include "util/omt.h" +//this is only for testing +static void (* test_txn_sync_callback) (uint64_t, void *) = NULL; +static void * test_txn_sync_callback_extra = NULL; + +void set_test_txn_sync_callback(void (*cb) (uint64_t, void *), void *extra) { + test_txn_sync_callback = cb; + test_txn_sync_callback_extra = extra; +} bool garbage_collection_debug = false; static bool txn_records_snapshot(TXN_SNAPSHOT_TYPE snapshot_type, struct tokutxn *parent) { @@ -525,14 +533,19 @@ void toku_txn_manager_handle_snapshot_create_for_child_txn( XMALLOC(txn->live_root_txn_list); txn_manager_lock(txn_manager); txn_manager_create_snapshot_unlocked(txn_manager, txn); - txn_manager_unlock(txn_manager); } else { inherit_snapshot_from_parent(txn); } - if (copies_snapshot) { - setup_live_root_txn_list(&txn_manager->live_root_ids, txn->live_root_txn_list); - } + + toku_debug_txn_sync(pthread_self()); + + if (copies_snapshot) { + if(!records_snapshot) + txn_manager_lock(txn_manager); + setup_live_root_txn_list(&txn_manager->live_root_ids, txn->live_root_txn_list); + txn_manager_unlock(txn_manager); + } } void toku_txn_manager_handle_snapshot_destroy_for_child_txn( diff --git a/storage/tokudb/PerconaFT/ft/txn/txn_manager.h b/storage/tokudb/PerconaFT/ft/txn/txn_manager.h index 658c6f9aecd..28fa1ac10b6 100644 --- a/storage/tokudb/PerconaFT/ft/txn/txn_manager.h +++ b/storage/tokudb/PerconaFT/ft/txn/txn_manager.h @@ -43,6 +43,15 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #include "ft/txn/txn.h" +void set_test_txn_sync_callback(void (*) (uint64_t, void*), void*); +#define toku_test_txn_sync_callback(a) ((test_txn_sync_callback)? test_txn_sync_callback( a,test_txn_sync_callback_extra) : (void) 0) + +#if TOKU_DEBUG_TXN_SYNC +#define toku_debug_txn_sync(a) toku_test_txn_sync_callback(a) +#else +#define toku_debug_txn_sync(a) ((void) 0) +#endif + typedef struct txn_manager *TXN_MANAGER; struct referenced_xid_tuple { diff --git a/storage/tokudb/PerconaFT/ft/ule.cc b/storage/tokudb/PerconaFT/ft/ule.cc index 573c4488f70..ac393fbf179 100644 --- a/storage/tokudb/PerconaFT/ft/ule.cc +++ b/storage/tokudb/PerconaFT/ft/ule.cc @@ -73,12 +73,11 @@ void toku_le_get_status(LE_STATUS statp) { *statp = le_status; } -/////////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// // Accessor functions used by outside world (e.g. indexer) // -ULEHANDLE -toku_ule_create(LEAFENTRY le) { +ULEHANDLE toku_ule_create(LEAFENTRY le) { ULE XMALLOC(ule_p); le_unpack(ule_p, le); return (ULEHANDLE) ule_p; @@ -89,7 +88,7 @@ void toku_ule_free(ULEHANDLE ule_p) { toku_free(ule_p); } -/////////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// // // Question: Can any software outside this file modify or read a leafentry? // If so, is it worthwhile to put it all here? @@ -117,27 +116,43 @@ const UXR_S committed_delete = { // Local functions: -static void msg_init_empty_ule(ULE ule); -static void msg_modify_ule(ULE ule, const ft_msg &msg); -static void ule_init_empty_ule(ULE ule); +static inline void msg_init_empty_ule(ULE ule); +static int64_t msg_modify_ule(ULE ule, const ft_msg &msg); +static inline void ule_init_empty_ule(ULE ule); static void ule_do_implicit_promotions(ULE ule, XIDS xids); -static void ule_try_promote_provisional_outermost(ULE ule, TXNID oldest_possible_live_xid); +static void ule_try_promote_provisional_outermost( + ULE ule, + TXNID oldest_possible_live_xid); static void ule_promote_provisional_innermost_to_index(ULE ule, uint32_t index); static void ule_promote_provisional_innermost_to_committed(ULE ule); -static void ule_apply_insert(ULE ule, XIDS xids, uint32_t vallen, void * valp); -static void ule_apply_delete(ULE ule, XIDS xids); -static void ule_prepare_for_new_uxr(ULE ule, XIDS xids); -static void ule_apply_abort(ULE ule, XIDS xids); +static inline int64_t ule_apply_insert_no_overwrite( + ULE ule, + XIDS xids, + uint32_t vallen, + void* valp); +static inline int64_t ule_apply_insert( + ULE ule, + XIDS xids, + uint32_t vallen, + void* valp); +static inline int64_t ule_apply_delete(ULE ule, XIDS xids); +static inline void ule_prepare_for_new_uxr(ULE ule, XIDS xids); +static inline int64_t ule_apply_abort(ULE ule, XIDS xids); static void ule_apply_broadcast_commit_all (ULE ule); static void ule_apply_commit(ULE ule, XIDS xids); -static void ule_push_insert_uxr(ULE ule, bool is_committed, TXNID xid, uint32_t vallen, void * valp); -static void ule_push_delete_uxr(ULE ule, bool is_committed, TXNID xid); -static void ule_push_placeholder_uxr(ULE ule, TXNID xid); -static UXR ule_get_innermost_uxr(ULE ule); -static UXR ule_get_first_empty_uxr(ULE ule); -static void ule_remove_innermost_uxr(ULE ule); -static TXNID ule_get_innermost_xid(ULE ule); -static TXNID ule_get_xid(ULE ule, uint32_t index); +static inline void ule_push_insert_uxr( + ULE ule, + bool is_committed, + TXNID xid, + uint32_t vallen, + void* valp); +static inline void ule_push_delete_uxr(ULE ule, bool is_committed, TXNID xid); +static inline void ule_push_placeholder_uxr(ULE ule, TXNID xid); +static inline UXR ule_get_innermost_uxr(ULE ule); +static inline UXR ule_get_first_empty_uxr(ULE ule); +static inline void ule_remove_innermost_uxr(ULE ule); +static inline TXNID ule_get_innermost_xid(ULE ule); +static inline TXNID ule_get_xid(ULE ule, uint32_t index); static void ule_remove_innermost_placeholders(ULE ule); static void ule_add_placeholders(ULE ule, XIDS xids); static void ule_optimize(ULE ule, XIDS xids); @@ -153,6 +168,30 @@ static inline size_t uxr_unpack_type_and_length(UXR uxr, uint8_t *p); static inline size_t uxr_unpack_length_and_bit(UXR uxr, uint8_t *p); static inline size_t uxr_unpack_data(UXR uxr, uint8_t *p); +#if 0 +static void ule_print(ULE ule, const char* note) { + fprintf(stderr, "%s : ULE[0x%p]\n", note, ule); + fprintf(stderr, " num_puxrs[%u]\n", ule->num_puxrs); + fprintf(stderr, " num_cuxrs[%u]\n", ule->num_cuxrs); + fprintf(stderr, " innermost[%u]\n", ule->num_cuxrs + ule->num_puxrs - 1); + fprintf(stderr, " first_empty[%u]\n", ule->num_cuxrs + ule->num_puxrs); + + uint32_t num_uxrs = ule->num_cuxrs + ule->num_puxrs - 1; + for (uint32_t uxr_num = 0; uxr_num <= num_uxrs; uxr_num++) { + UXR uxr = &(ule->uxrs[uxr_num]); + fprintf(stderr, " uxr[%u]\n", uxr_num); + switch (uxr->type) { + case 0: fprintf(stderr, " type[NONE]\n"); break; + case 1: fprintf(stderr, " type[INSERT]\n"); break; + case 2: fprintf(stderr, " type[DELETE]\n"); break; + case 3: fprintf(stderr, " type[PLACEHOLDER]\n"); break; + default: fprintf(stderr, " type[WHAT??]\n"); break; + } + fprintf(stderr, " xid[%lu]\n", uxr->xid); + } +} +#endif + static void get_space_for_le( bn_data* data_buffer, uint32_t idx, @@ -162,21 +201,30 @@ static void get_space_for_le( uint32_t old_le_size, size_t size, LEAFENTRY* new_le_space, - void **const maybe_free - ) -{ + void** const maybe_free) { + if (data_buffer == nullptr) { CAST_FROM_VOIDP(*new_le_space, toku_xmalloc(size)); - } - else { + } else if (old_le_size > 0) { // this means we are overwriting something - if (old_le_size > 0) { - data_buffer->get_space_for_overwrite(idx, keyp, keylen, old_keylen, old_le_size, size, new_le_space, maybe_free); - } + data_buffer->get_space_for_overwrite( + idx, + keyp, + keylen, + old_keylen, + old_le_size, + size, + new_le_space, + maybe_free); + } else { // this means we are inserting something new - else { - data_buffer->get_space_for_insert(idx, keyp, keylen, size, new_le_space, maybe_free); - } + data_buffer->get_space_for_insert( + idx, + keyp, + keylen, + size, + new_le_space, + maybe_free); } } @@ -185,15 +233,13 @@ static void get_space_for_le( // Garbage collection related functions // -static TXNID -get_next_older_txnid(TXNID xc, const xid_omt_t &omt) { +static TXNID get_next_older_txnid(TXNID xc, const xid_omt_t &omt) { int r; TXNID xid; r = omt.find<TXNID, toku_find_xid_by_xid>(xc, -1, &xid, nullptr); if (r==0) { invariant(xid < xc); //sanity check - } - else { + } else { invariant(r==DB_NOTFOUND); xid = TXNID_NONE; } @@ -201,17 +247,32 @@ get_next_older_txnid(TXNID xc, const xid_omt_t &omt) { } // -// This function returns true if live transaction TL1 is allowed to read a value committed by -// transaction xc, false otherwise. +// This function returns true if live transaction TL1 is allowed to read a +// value committed by transaction xc, false otherwise. // -static bool -xid_reads_committed_xid(TXNID tl1, TXNID xc, const xid_omt_t &snapshot_txnids, const rx_omt_t &referenced_xids) { +static bool xid_reads_committed_xid( + TXNID tl1, + TXNID xc, + const xid_omt_t& snapshot_txnids, + const rx_omt_t& referenced_xids) { + bool rval; - if (tl1 < xc) rval = false; //cannot read a newer txn - else { - TXNID x = toku_get_youngest_live_list_txnid_for(xc, snapshot_txnids, referenced_xids); - if (x == TXNID_NONE) rval = true; //Not in ANY live list, tl1 can read it. - else rval = tl1 > x; //Newer than the 'newest one that has it in live list' + if (tl1 < xc) { + rval = false; //cannot read a newer txn + } else { + TXNID x = + toku_get_youngest_live_list_txnid_for( + xc, + snapshot_txnids, + referenced_xids); + + if (x == TXNID_NONE) { + //Not in ANY live list, tl1 can read it. + rval = true; + } else { + //Newer than the 'newest one that has it in live list' + rval = tl1 > x; + } // we know tl1 > xc // we know x > xc // if tl1 == x, then we do not read, because tl1 is in xc's live list @@ -228,8 +289,7 @@ xid_reads_committed_xid(TXNID tl1, TXNID xc, const xid_omt_t &snapshot_txnids, c // than oldest_referenced_xid. All elements below this entry are garbage, // so we get rid of them. // -static void -ule_simple_garbage_collection(ULE ule, txn_gc_info *gc_info) { +static void ule_simple_garbage_collection(ULE ule, txn_gc_info *gc_info) { if (ule->num_cuxrs == 1) { return; } @@ -240,7 +300,8 @@ ule_simple_garbage_collection(ULE ule, txn_gc_info *gc_info) { // uxr with a txnid that is less than oldest_referenced_xid for (uint32_t i = 0; i < ule->num_cuxrs; i++) { curr_index = ule->num_cuxrs - i - 1; - if (ule->uxrs[curr_index].xid < gc_info->oldest_referenced_xid_for_simple_gc) { + if (ule->uxrs[curr_index].xid < + gc_info->oldest_referenced_xid_for_simple_gc) { break; } } @@ -250,12 +311,15 @@ ule_simple_garbage_collection(ULE ule, txn_gc_info *gc_info) { curr_index = ule->num_cuxrs - 1; } - // curr_index is now set to the youngest uxr older than oldest_referenced_xid - // so if it's not the bottom of the stack.. + // curr_index is now set to the youngest uxr older than + // oldest_referenced_xid so if it's not the bottom of the stack.. if (curr_index != 0) { // ..then we need to get rid of the entries below curr_index uint32_t num_entries = ule->num_cuxrs + ule->num_puxrs - curr_index; - memmove(&ule->uxrs[0], &ule->uxrs[curr_index], num_entries * sizeof(ule->uxrs[0])); + memmove( + &ule->uxrs[0], + &ule->uxrs[curr_index], + num_entries * sizeof(ule->uxrs[0])); ule->uxrs[0].xid = TXNID_NONE; // New 'bottom of stack' loses its TXNID ule->num_cuxrs -= curr_index; } @@ -264,8 +328,12 @@ ule_simple_garbage_collection(ULE ule, txn_gc_info *gc_info) { // TODO: Clean this up extern bool garbage_collection_debug; -static void -ule_garbage_collect(ULE ule, const xid_omt_t &snapshot_xids, const rx_omt_t &referenced_xids, const xid_omt_t &live_root_txns) { +static void ule_garbage_collect( + ULE ule, + const xid_omt_t& snapshot_xids, + const rx_omt_t& referenced_xids, + const xid_omt_t& live_root_txns) { + if (ule->num_cuxrs == 1) { return; } @@ -289,10 +357,12 @@ ule_garbage_collect(ULE ule, const xid_omt_t &snapshot_xids, const rx_omt_t &ref // If we find that the committed transaction is in the live list, // then xc is really in the process of being committed. It has not // been fully committed. As a result, our assumption that transactions - // newer than what is currently in these OMTs will read the top of the stack - // is not necessarily accurate. Transactions may read what is just below xc. - // As a result, we must mark what is just below xc as necessary and move on. - // This issue was found while testing flusher threads, and was fixed for #3979 + // newer than what is currently in these OMTs will read the top of the + // stack is not necessarily accurate. Transactions may read what is + // just below xc. + // As a result, we must mark what is just below xc as necessary and + // move on. This issue was found while testing flusher threads, and was + // fixed for #3979 // bool is_xc_live = toku_is_txn_in_live_root_txn_list(live_root_txns, xc); if (is_xc_live) { @@ -300,13 +370,19 @@ ule_garbage_collect(ULE ule, const xid_omt_t &snapshot_xids, const rx_omt_t &ref continue; } - tl1 = toku_get_youngest_live_list_txnid_for(xc, snapshot_xids, referenced_xids); + tl1 = + toku_get_youngest_live_list_txnid_for( + xc, + snapshot_xids, + referenced_xids); - // if tl1 == xc, that means xc should be live and show up in live_root_txns, which we check above. + // if tl1 == xc, that means xc should be live and show up in + // live_root_txns, which we check above. invariant(tl1 != xc); if (tl1 == TXNID_NONE) { - // set tl1 to youngest live transaction older than ule->uxrs[curr_committed_entry]->xid + // set tl1 to youngest live transaction older than + // ule->uxrs[curr_committed_entry]->xid tl1 = get_next_older_txnid(xc, snapshot_xids); if (tl1 == TXNID_NONE) { // remainder is garbage, we're done @@ -314,8 +390,13 @@ ule_garbage_collect(ULE ule, const xid_omt_t &snapshot_xids, const rx_omt_t &ref } } if (garbage_collection_debug) { - int r = snapshot_xids.find_zero<TXNID, toku_find_xid_by_xid>(tl1, nullptr, nullptr); - invariant_zero(r); // make sure that the txn you are claiming is live is actually live + int r = + snapshot_xids.find_zero<TXNID, toku_find_xid_by_xid>( + tl1, + nullptr, + nullptr); + // make sure that the txn you are claiming is live is actually live + invariant_zero(r); } // // tl1 should now be set @@ -323,7 +404,11 @@ ule_garbage_collect(ULE ule, const xid_omt_t &snapshot_xids, const rx_omt_t &ref curr_committed_entry--; while (curr_committed_entry > 0) { xc = ule->uxrs[curr_committed_entry].xid; - if (xid_reads_committed_xid(tl1, xc, snapshot_xids, referenced_xids)) { + if (xid_reads_committed_xid( + tl1, + xc, + snapshot_xids, + referenced_xids)) { break; } curr_committed_entry--; @@ -343,7 +428,10 @@ ule_garbage_collect(ULE ule, const xid_omt_t &snapshot_xids, const rx_omt_t &ref ule->uxrs[0].xid = TXNID_NONE; //New 'bottom of stack' loses its TXNID if (first_free != ule->num_cuxrs) { // Shift provisional values - memmove(&ule->uxrs[first_free], &ule->uxrs[ule->num_cuxrs], ule->num_puxrs * sizeof(ule->uxrs[0])); + memmove( + &ule->uxrs[first_free], + &ule->uxrs[ule->num_cuxrs], + ule->num_puxrs * sizeof(ule->uxrs[0])); } ule->num_cuxrs = saved; } @@ -367,29 +455,42 @@ enum { ULE_MIN_MEMSIZE_TO_FORCE_GC = 1024 * 1024 }; -///////////////////////////////////////////////////////////////////////////////// -// This is the big enchilada. (Bring Tums.) Note that this level of abstraction -// has no knowledge of the inner structure of either leafentry or msg. It makes -// calls into the next lower layer (msg_xxx) which handles messages. +//////////////////////////////////////////////////////////////////////////////// +// This is the big enchilada. (Bring Tums.) Note that this level of +// abstraction has no knowledge of the inner structure of either leafentry or +// msg. It makes calls into the next lower layer (msg_xxx) which handles +// messages. // // NOTE: This is the only function (at least in this body of code) that modifies // a leafentry. // NOTE: It is the responsibility of the caller to make sure that the key is set // in the FT_MSG, as it will be used to store the data in the data_buffer // -// Return 0 on success. -// If the leafentry is destroyed it sets *new_leafentry_p to NULL. -// Otehrwise the new_leafentry_p points at the new leaf entry. -// As of October 2011, this function always returns 0. -void -toku_le_apply_msg(const ft_msg &msg, - LEAFENTRY old_leafentry, // NULL if there was no stored data. - bn_data* data_buffer, // bn_data storing leafentry, if NULL, means there is no bn_data - uint32_t idx, // index in data_buffer where leafentry is stored (and should be replaced - uint32_t old_keylen, // length of the any key in data_buffer - txn_gc_info *gc_info, - LEAFENTRY *new_leafentry_p, - int64_t * numbytes_delta_p) { // change in total size of key and val, not including any overhead +// Returns -1, 0, or 1 that identifies the change in logical row count needed +// based on the results of the message application. For example, if a delete +// finds no logical leafentry or if an insert finds a duplicate and is +// converted to an update. +// +// old_leafentry - NULL if there was no stored data. +// data_buffer - bn_data storing leafentry, if NULL, means there is no bn_data +// idx - index in data_buffer where leafentry is stored +// (and should be replaced) +// old_keylen - length of the any key in data_buffer +// new_leafentry_p - If the leafentry is destroyed it sets *new_leafentry_p +// to NULL. Otherwise the new_leafentry_p points at the new +// leaf entry. +// numbytes_delta_p - change in total size of key and val, not including any +// overhead +int64_t toku_le_apply_msg( + const ft_msg& msg, + LEAFENTRY old_leafentry, + bn_data* data_buffer, + uint32_t idx, + uint32_t old_keylen, + txn_gc_info* gc_info, + LEAFENTRY* new_leafentry_p, + int64_t* numbytes_delta_p) { + invariant_notnull(gc_info); paranoid_invariant_notnull(new_leafentry_p); ULE_S ule; @@ -397,6 +498,7 @@ toku_le_apply_msg(const ft_msg &msg, int64_t newnumbytes = 0; uint64_t oldmemsize = 0; uint32_t keylen = msg.kdbt()->size; + int32_t rowcountdelta = 0; if (old_leafentry == NULL) { msg_init_empty_ule(&ule); @@ -405,49 +507,62 @@ toku_le_apply_msg(const ft_msg &msg, le_unpack(&ule, old_leafentry); // otherwise unpack leafentry oldnumbytes = ule_get_innermost_numbytes(&ule, keylen); } - msg_modify_ule(&ule, msg); // modify unpacked leafentry - // - we may be able to immediately promote the newly-apllied outermost provisonal uxr - // - either way, run simple gc first, and then full gc if there are still some committed uxrs. - ule_try_promote_provisional_outermost(&ule, gc_info->oldest_referenced_xid_for_implicit_promotion); + // modify unpacked leafentry + rowcountdelta = msg_modify_ule(&ule, msg); + + // - we may be able to immediately promote the newly-apllied outermost + // provisonal uxr + // - either way, run simple gc first, and then full gc if there are still + // some committed uxrs. + ule_try_promote_provisional_outermost( + &ule, + gc_info->oldest_referenced_xid_for_implicit_promotion); ule_simple_garbage_collection(&ule, gc_info); txn_manager_state *txn_state_for_gc = gc_info->txn_state_for_gc; size_t size_before_gc = 0; - if (ule.num_cuxrs > 1 && txn_state_for_gc != nullptr && // there is garbage to clean, and our caller gave us state.. - // ..and either the state is pre-initialized, or the committed stack is large enough - (txn_state_for_gc->initialized || ule.num_cuxrs >= ULE_MIN_STACK_SIZE_TO_FORCE_GC || - // ..or the ule's raw memsize is sufficiently large - (size_before_gc = ule_packed_memsize(&ule)) >= ULE_MIN_MEMSIZE_TO_FORCE_GC)) { - // ..then it's worth running gc, possibly initializing the txn manager state, if it isn't already + // there is garbage to clean, and our caller gave us state.. + // ..and either the state is pre-initialized, or the committed stack is + // large enough + // ..or the ule's raw memsize is sufficiently large + // ..then it's worth running gc, possibly initializing the txn manager + // state, if it isn't already + if (ule.num_cuxrs > 1 && txn_state_for_gc != nullptr && + (txn_state_for_gc->initialized || + ule.num_cuxrs >= ULE_MIN_STACK_SIZE_TO_FORCE_GC || + (size_before_gc = ule_packed_memsize(&ule)) >= + ULE_MIN_MEMSIZE_TO_FORCE_GC)) { if (!txn_state_for_gc->initialized) { txn_state_for_gc->init(); } - - size_before_gc = size_before_gc != 0 ? size_before_gc : // it's already been calculated above - ule_packed_memsize(&ule); - ule_garbage_collect(&ule, - txn_state_for_gc->snapshot_xids, - txn_state_for_gc->referenced_xids, - txn_state_for_gc->live_root_txns - ); + // it's already been calculated above + size_before_gc = + size_before_gc != 0 ? size_before_gc : ule_packed_memsize(&ule); + ule_garbage_collect( + &ule, + txn_state_for_gc->snapshot_xids, + txn_state_for_gc->referenced_xids, + txn_state_for_gc->live_root_txns); size_t size_after_gc = ule_packed_memsize(&ule); LE_STATUS_INC(LE_APPLY_GC_BYTES_IN, size_before_gc); LE_STATUS_INC(LE_APPLY_GC_BYTES_OUT, size_after_gc); } - void *maybe_free = nullptr; - int r = le_pack( - &ule, // create packed leafentry - data_buffer, - idx, - msg.kdbt()->data, // contract of this function is caller has this set, always - keylen, // contract of this function is caller has this set, always - old_keylen, - oldmemsize, - new_leafentry_p, - &maybe_free - ); + void* maybe_free = nullptr; + // create packed leafentry + // contract of this function is caller has keyp and keylen set, always + int r = + le_pack( + &ule, + data_buffer, + idx, + msg.kdbt()->data, + keylen, + old_keylen, + oldmemsize, + new_leafentry_p, + &maybe_free); invariant_zero(r); if (*new_leafentry_p) { newnumbytes = ule_get_innermost_numbytes(&ule, keylen); @@ -458,16 +573,22 @@ toku_le_apply_msg(const ft_msg &msg, if (maybe_free != nullptr) { toku_free(maybe_free); } + return rowcountdelta; } -bool toku_le_worth_running_garbage_collection(LEAFENTRY le, txn_gc_info *gc_info) { -// Effect: Quickly determines if it's worth trying to run garbage collection on a leafentry +bool toku_le_worth_running_garbage_collection( + LEAFENTRY le, + txn_gc_info* gc_info) { +// Effect: Quickly determines if it's worth trying to run garbage collection +// on a leafentry // Return: True if it makes sense to try garbage collection, false otherwise. // Rationale: Garbage collection is likely to clean up under two circumstances: -// 1.) There are multiple committed entries. Some may never be read by new txns. -// 2.) There is only one committed entry, but the outermost provisional entry -// is older than the oldest known referenced xid, so it must have commited. -// Therefor we can promote it to committed and get rid of the old commited entry. +// 1.) There are multiple committed entries. Some may never be read +// by new txns. +// 2.) There is only one committed entry, but the outermost +// provisional entry is older than the oldest known referenced +// xid, so it must have commited. Therefor we can promote it to +// committed and get rid of the old commited entry. if (le->type != LE_MVCC) { return false; } @@ -477,7 +598,8 @@ bool toku_le_worth_running_garbage_collection(LEAFENTRY le, txn_gc_info *gc_info paranoid_invariant(le->u.mvcc.num_cxrs == 1); } return le->u.mvcc.num_pxrs > 0 && - le_outermost_uncommitted_xid(le) < gc_info->oldest_referenced_xid_for_implicit_promotion; + le_outermost_uncommitted_xid(le) < + gc_info->oldest_referenced_xid_for_implicit_promotion; } // Garbage collect one leaf entry, using the given OMT's. @@ -498,16 +620,18 @@ bool toku_le_worth_running_garbage_collection(LEAFENTRY le, txn_gc_info *gc_info // -- referenced_xids : list of in memory active transactions. // NOTE: it is not a good idea to garbage collect a leaf // entry with only one committed value. -void -toku_le_garbage_collect(LEAFENTRY old_leaf_entry, - bn_data* data_buffer, - uint32_t idx, - void* keyp, - uint32_t keylen, - txn_gc_info *gc_info, - LEAFENTRY *new_leaf_entry, - int64_t * numbytes_delta_p) { - // We shouldn't want to run gc without having provided a snapshot of the txn system. +void toku_le_garbage_collect( + LEAFENTRY old_leaf_entry, + bn_data* data_buffer, + uint32_t idx, + void* keyp, + uint32_t keylen, + txn_gc_info* gc_info, + LEAFENTRY* new_leaf_entry, + int64_t* numbytes_delta_p) { + + // We shouldn't want to run gc without having provided a snapshot of the + // txn system. invariant_notnull(gc_info); invariant_notnull(gc_info->txn_state_for_gc); paranoid_invariant_notnull(new_leaf_entry); @@ -520,20 +644,24 @@ toku_le_garbage_collect(LEAFENTRY old_leaf_entry, oldnumbytes = ule_get_innermost_numbytes(&ule, keylen); uint32_t old_mem_size = leafentry_memsize(old_leaf_entry); - // Before running garbage collection, try to promote the outermost provisional - // entries to committed if its xid is older than the oldest possible live xid. + // Before running garbage collection, try to promote the outermost + // provisional entries to committed if its xid is older than the oldest + // possible live xid. // // The oldest known refeferenced xid is a lower bound on the oldest possible // live xid, so we use that. It's usually close enough to get rid of most // garbage in leafentries. - ule_try_promote_provisional_outermost(&ule, gc_info->oldest_referenced_xid_for_implicit_promotion); + ule_try_promote_provisional_outermost( + &ule, + gc_info->oldest_referenced_xid_for_implicit_promotion); // No need to run simple gc here if we're going straight for full gc. if (ule.num_cuxrs > 1) { size_t size_before_gc = ule_packed_memsize(&ule); - ule_garbage_collect(&ule, - gc_info->txn_state_for_gc->snapshot_xids, - gc_info->txn_state_for_gc->referenced_xids, - gc_info->txn_state_for_gc->live_root_txns); + ule_garbage_collect( + &ule, + gc_info->txn_state_for_gc->snapshot_xids, + gc_info->txn_state_for_gc->referenced_xids, + gc_info->txn_state_for_gc->live_root_txns); size_t size_after_gc = ule_packed_memsize(&ule); LE_STATUS_INC(LE_APPLY_GC_BYTES_IN, size_before_gc); @@ -541,17 +669,18 @@ toku_le_garbage_collect(LEAFENTRY old_leaf_entry, } void *maybe_free = nullptr; - int r = le_pack( - &ule, - data_buffer, - idx, - keyp, - keylen, - keylen, // old_keylen, same because the key isn't going to change for gc - old_mem_size, - new_leaf_entry, - &maybe_free - ); + // old_keylen, same because the key isn't going to change for gc + int r = + le_pack( + &ule, + data_buffer, + idx, + keyp, + keylen, + keylen, + old_mem_size, + new_leaf_entry, + &maybe_free); invariant_zero(r); if (*new_leaf_entry) { newnumbytes = ule_get_innermost_numbytes(&ule, keylen); @@ -564,49 +693,54 @@ toku_le_garbage_collect(LEAFENTRY old_leaf_entry, } } -///////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// // This layer of abstraction (msg_xxx) // knows the accessors of msg, but not of leafentry or unpacked leaf entry. // It makes calls into the lower layer (le_xxx) which handles leafentries. // Purpose is to init the ule with given key and no transaction records // -static void -msg_init_empty_ule(ULE ule) { +static inline void msg_init_empty_ule(ULE ule) { ule_init_empty_ule(ule); } // Purpose is to modify the unpacked leafentry in our private workspace. // -static void -msg_modify_ule(ULE ule, const ft_msg &msg) { +// Returns -1, 0, or 1 that identifies the change in logical row count needed +// based on the results of the message application. For example, if a delete +// finds no logical leafentry or if an insert finds a duplicate and is +// converted to an update. +static int64_t msg_modify_ule(ULE ule, const ft_msg &msg) { + int64_t retval = 0; XIDS xids = msg.xids(); invariant(toku_xids_get_num_xids(xids) < MAX_TRANSACTION_RECORDS); enum ft_msg_type type = msg.type(); - if (type != FT_OPTIMIZE && type != FT_OPTIMIZE_FOR_UPGRADE) { + if (FT_LIKELY(type != FT_OPTIMIZE && type != FT_OPTIMIZE_FOR_UPGRADE)) { ule_do_implicit_promotions(ule, xids); } switch (type) { - case FT_INSERT_NO_OVERWRITE: { - UXR old_innermost_uxr = ule_get_innermost_uxr(ule); - //If something exists, quit (no overwrite). - if (uxr_is_insert(old_innermost_uxr)) break; - //else it is just an insert, so - //fall through to FT_INSERT on purpose. - } - case FT_INSERT: { - uint32_t vallen = msg.vdbt()->size; - invariant(IS_VALID_LEN(vallen)); - void * valp = msg.vdbt()->data; - ule_apply_insert(ule, xids, vallen, valp); + case FT_INSERT_NO_OVERWRITE: + retval = + ule_apply_insert_no_overwrite( + ule, + xids, + msg.vdbt()->size, + msg.vdbt()->data); + break; + case FT_INSERT: + retval = + ule_apply_insert( + ule, + xids, + msg.vdbt()->size, + msg.vdbt()->data); break; - } case FT_DELETE_ANY: - ule_apply_delete(ule, xids); + retval = ule_apply_delete(ule, xids); break; case FT_ABORT_ANY: case FT_ABORT_BROADCAST_TXN: - ule_apply_abort(ule, xids); + retval = ule_apply_abort(ule, xids); break; case FT_COMMIT_BROADCAST_ALL: ule_apply_broadcast_commit_all(ule); @@ -621,34 +755,40 @@ msg_modify_ule(ULE ule, const ft_msg &msg) { break; case FT_UPDATE: case FT_UPDATE_BROADCAST_ALL: - assert(false); // These messages don't get this far. Instead they get translated (in setval_fun in do_update) into FT_INSERT messages. + // These messages don't get this far. Instead they get translated (in + // setval_fun in do_update) into FT_INSERT messages. + assert(false); break; default: - assert(false); /* illegal ft msg type */ + // illegal ft msg type + assert(false); break; } + return retval; } -void test_msg_modify_ule(ULE ule, const ft_msg &msg){ +void test_msg_modify_ule(ULE ule, const ft_msg &msg) { msg_modify_ule(ule,msg); } static void ule_optimize(ULE ule, XIDS xids) { if (ule->num_puxrs) { - TXNID uncommitted = ule->uxrs[ule->num_cuxrs].xid; // outermost uncommitted + // outermost uncommitted + TXNID uncommitted = ule->uxrs[ule->num_cuxrs].xid; TXNID oldest_living_xid = TXNID_NONE; uint32_t num_xids = toku_xids_get_num_xids(xids); if (num_xids > 0) { invariant(num_xids==1); oldest_living_xid = toku_xids_get_xid(xids, 0); } - if (oldest_living_xid == TXNID_NONE || uncommitted < oldest_living_xid) { + if (oldest_living_xid == TXNID_NONE || + uncommitted < oldest_living_xid) { ule_promote_provisional_innermost_to_committed(ule); } } } -///////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// // This layer of abstraction (le_xxx) understands the structure of the leafentry // and of the unpacked leafentry. It is the only layer that understands the // structure of leafentry. It has no knowledge of any other data structures. @@ -657,8 +797,7 @@ static void ule_optimize(ULE ule, XIDS xids) { // // required for every le_unpack that is done // -void -ule_cleanup(ULE ule) { +void ule_cleanup(ULE ule) { invariant(ule->uxrs); if (ule->uxrs != ule->uxrs_static) { toku_free(ule->uxrs); @@ -668,8 +807,7 @@ ule_cleanup(ULE ule) { // populate an unpacked leafentry using pointers into the given leafentry. // thus, the memory referenced by 'le' must live as long as the ULE. -void -le_unpack(ULE ule, LEAFENTRY le) { +void le_unpack(ULE ule, LEAFENTRY le) { uint8_t type = le->type; uint8_t *p; uint32_t i; @@ -694,9 +832,10 @@ le_unpack(ULE ule, LEAFENTRY le) { //Dynamic memory if (ule->num_cuxrs < MAX_TRANSACTION_RECORDS) { ule->uxrs = ule->uxrs_static; - } - else { - XMALLOC_N(ule->num_cuxrs + 1 + MAX_TRANSACTION_RECORDS, ule->uxrs); + } else { + XMALLOC_N( + ule->num_cuxrs + 1 + MAX_TRANSACTION_RECORDS, + ule->uxrs); } p = le->u.mvcc.xrs; @@ -717,9 +856,12 @@ le_unpack(ULE ule, LEAFENTRY le) { p += uxr_unpack_length_and_bit(innermost, p); } for (i = 0; i < ule->num_cuxrs; i++) { - p += uxr_unpack_length_and_bit(ule->uxrs + ule->num_cuxrs - 1 - i, p); + p += + uxr_unpack_length_and_bit( + ule->uxrs + ule->num_cuxrs - 1 - i, + p); } - + //unpack interesting values inner to outer if (ule->num_puxrs!=0) { UXR innermost = ule->uxrs + ule->num_cuxrs + ule->num_puxrs - 1; @@ -761,14 +903,12 @@ le_unpack(ULE ule, LEAFENTRY le) { #endif } -static inline size_t -uxr_pack_txnid(UXR uxr, uint8_t *p) { +static inline size_t uxr_pack_txnid(UXR uxr, uint8_t *p) { *(TXNID*)p = toku_htod64(uxr->xid); return sizeof(TXNID); } -static inline size_t -uxr_pack_type_and_length(UXR uxr, uint8_t *p) { +static inline size_t uxr_pack_type_and_length(UXR uxr, uint8_t *p) { size_t rval = 1; *p = uxr->type; if (uxr_is_insert(uxr)) { @@ -778,21 +918,18 @@ uxr_pack_type_and_length(UXR uxr, uint8_t *p) { return rval; } -static inline size_t -uxr_pack_length_and_bit(UXR uxr, uint8_t *p) { +static inline size_t uxr_pack_length_and_bit(UXR uxr, uint8_t *p) { uint32_t length_and_bit; if (uxr_is_insert(uxr)) { length_and_bit = INSERT_LENGTH(uxr->vallen); - } - else { + } else { length_and_bit = DELETE_LENGTH(uxr->vallen); } *(uint32_t*)p = toku_htod32(length_and_bit); return sizeof(uint32_t); } -static inline size_t -uxr_pack_data(UXR uxr, uint8_t *p) { +static inline size_t uxr_pack_data(UXR uxr, uint8_t *p) { if (uxr_is_insert(uxr)) { memcpy(p, uxr->valp, uxr->vallen); return uxr->vallen; @@ -800,14 +937,12 @@ uxr_pack_data(UXR uxr, uint8_t *p) { return 0; } -static inline size_t -uxr_unpack_txnid(UXR uxr, uint8_t *p) { +static inline size_t uxr_unpack_txnid(UXR uxr, uint8_t *p) { uxr->xid = toku_dtoh64(*(TXNID*)p); return sizeof(TXNID); } -static inline size_t -uxr_unpack_type_and_length(UXR uxr, uint8_t *p) { +static inline size_t uxr_unpack_type_and_length(UXR uxr, uint8_t *p) { size_t rval = 1; uxr->type = *p; if (uxr_is_insert(uxr)) { @@ -817,22 +952,19 @@ uxr_unpack_type_and_length(UXR uxr, uint8_t *p) { return rval; } -static inline size_t -uxr_unpack_length_and_bit(UXR uxr, uint8_t *p) { +static inline size_t uxr_unpack_length_and_bit(UXR uxr, uint8_t *p) { uint32_t length_and_bit = toku_dtoh32(*(uint32_t*)p); if (IS_INSERT(length_and_bit)) { uxr->type = XR_INSERT; uxr->vallen = GET_LENGTH(length_and_bit); - } - else { + } else { uxr->type = XR_DELETE; uxr->vallen = 0; } return sizeof(uint32_t); } -static inline size_t -uxr_unpack_data(UXR uxr, uint8_t *p) { +static inline size_t uxr_unpack_data(UXR uxr, uint8_t *p) { if (uxr_is_insert(uxr)) { uxr->valp = p; return uxr->vallen; @@ -841,8 +973,7 @@ uxr_unpack_data(UXR uxr, uint8_t *p) { } // executed too often to be worth making threadsafe -static inline void -update_le_status(ULE ule, size_t memsize) { +static inline void update_le_status(ULE ule, size_t memsize) { if (ule->num_cuxrs > LE_STATUS_VAL(LE_MAX_COMMITTED_XR)) LE_STATUS_VAL(LE_MAX_COMMITTED_XR) = ule->num_cuxrs; if (ule->num_puxrs > LE_STATUS_VAL(LE_MAX_PROVISIONAL_XR)) @@ -856,21 +987,22 @@ update_le_status(ULE ule, size_t memsize) { // Purpose is to return a newly allocated leaf entry in packed format, or // return null if leaf entry should be destroyed (if no transaction records // are for inserts). -// Transaction records in packed le are stored inner to outer (first xr is innermost), -// with some information extracted out of the transaction records into the header. +// Transaction records in packed le are stored inner to outer (first xr is +// innermost), with some information extracted out of the transaction records +// into the header. // Transaction records in ule are stored outer to inner (uxr[0] is outermost). -int -le_pack(ULE ule, // data to be packed into new leafentry - bn_data* data_buffer, - uint32_t idx, - void* keyp, - uint32_t keylen, - uint32_t old_keylen, - uint32_t old_le_size, - LEAFENTRY * const new_leafentry_p, // this is what this function creates - void **const maybe_free - ) -{ +// Takes 'ule' and creates 'new_leafentry_p +int le_pack( + ULE ule, + bn_data* data_buffer, + uint32_t idx, + void* keyp, + uint32_t keylen, + uint32_t old_keylen, + uint32_t old_le_size, + LEAFENTRY* const new_leafentry_p, + void** const maybe_free) { + invariant(ule->num_cuxrs > 0); invariant(ule->uxrs[0].xid == TXNID_NONE); int rval; @@ -888,7 +1020,8 @@ le_pack(ULE ule, // data to be packed into new leafentry } } if (data_buffer && old_le_size > 0) { - // must pass old_keylen and old_le_size, since that's what is actually stored in data_buffer + // must pass old_keylen and old_le_size, since that's what is + // actually stored in data_buffer data_buffer->delete_leafentry(idx, old_keylen, old_le_size); } *new_leafentry_p = NULL; @@ -898,14 +1031,24 @@ le_pack(ULE ule, // data to be packed into new leafentry found_insert: memsize = le_memsize_from_ule(ule); LEAFENTRY new_leafentry; - get_space_for_le(data_buffer, idx, keyp, keylen, old_keylen, old_le_size, memsize, &new_leafentry, maybe_free); + get_space_for_le( + data_buffer, + idx, + keyp, + keylen, + old_keylen, + old_le_size, + memsize, + &new_leafentry, + maybe_free); //p always points to first unused byte after leafentry we are packing uint8_t *p; invariant(ule->num_cuxrs>0); //Type specific data if (ule->num_cuxrs == 1 && ule->num_puxrs == 0) { - //Pack a 'clean leafentry' (no uncommitted transactions, only one committed value) + //Pack a 'clean leafentry' (no uncommitted transactions, only one + //committed value) new_leafentry->type = LE_CLEAN; uint32_t vallen = ule->uxrs[0].vallen; @@ -917,8 +1060,7 @@ found_insert: //Set p to after leafentry p = new_leafentry->u.clean.val + vallen; - } - else { + } else { uint32_t i; //Pack an 'mvcc leafentry' new_leafentry->type = LE_MVCC; @@ -969,7 +1111,9 @@ found_insert: p += uxr_pack_data(outermost, p); } //pack txnid, length, bit, data for non-outermost, non-innermost - for (i = ule->num_cuxrs + 1; i < ule->num_cuxrs + ule->num_puxrs - 1; i++) { + for (i = ule->num_cuxrs + 1; + i < ule->num_cuxrs + ule->num_puxrs - 1; + i++) { UXR uxr = ule->uxrs + i; p += uxr_pack_txnid(uxr, p); p += uxr_pack_type_and_length(uxr, p); @@ -1022,13 +1166,13 @@ cleanup: return rval; } -////////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// // Following functions provide convenient access to a packed leafentry. //Requires: -// Leafentry that ule represents should not be destroyed (is not just all deletes) -size_t -le_memsize_from_ule (ULE ule) { +// Leafentry that ule represents should not be destroyed (is not just all +// deletes) +size_t le_memsize_from_ule (ULE ule) { invariant(ule->num_cuxrs); size_t rval; if (ule->num_cuxrs == 1 && ule->num_puxrs == 0) { @@ -1037,13 +1181,13 @@ le_memsize_from_ule (ULE ule) { rval = 1 //type +4 //vallen +committed->vallen; //actual val - } - else { + } else { rval = 1 //type +4 //num_cuxrs +1 //num_puxrs +4*(ule->num_cuxrs) //types+lengths for committed - +8*(ule->num_cuxrs + ule->num_puxrs - 1); //txnids (excluding superroot) + +8*(ule->num_cuxrs + ule->num_puxrs - 1); //txnids (excluding + //superroot) uint32_t i; //Count data from committed uxrs and innermost puxr for (i = 0; i < ule->num_cuxrs; i++) { @@ -1072,8 +1216,11 @@ le_memsize_from_ule (ULE ule) { } // TODO: rename -size_t -leafentry_rest_memsize(uint32_t num_puxrs, uint32_t num_cuxrs, uint8_t* start) { +size_t leafentry_rest_memsize( + uint32_t num_puxrs, + uint32_t num_cuxrs, + uint8_t* start) { + UXR_S uxr; size_t lengths = 0; uint8_t* p = start; @@ -1122,8 +1269,7 @@ leafentry_rest_memsize(uint32_t num_puxrs, uint32_t num_cuxrs, uint8_t* start) { return rval; } -size_t -leafentry_memsize (LEAFENTRY le) { +size_t leafentry_memsize (LEAFENTRY le) { size_t rval = 0; uint8_t type = le->type; @@ -1162,13 +1308,11 @@ leafentry_memsize (LEAFENTRY le) { return rval; } -size_t -leafentry_disksize (LEAFENTRY le) { +size_t leafentry_disksize (LEAFENTRY le) { return leafentry_memsize(le); } -bool -le_is_clean(LEAFENTRY le) { +bool le_is_clean(LEAFENTRY le) { uint8_t type = le->type; uint32_t rval; switch (type) { @@ -1228,13 +1372,14 @@ int le_latest_is_del(LEAFENTRY le) { // -// returns true if the outermost provisional transaction id on the leafentry's stack matches -// the outermost transaction id in xids -// It is used to determine if a broadcast commit/abort message (look in ft-ops.c) should be applied to this leafentry -// If the outermost transactions match, then the broadcast commit/abort should be applied +// returns true if the outermost provisional transaction id on the leafentry's +// stack matches the outermost transaction id in xids +// It is used to determine if a broadcast commit/abort message (look in ft-ops.c) +// should be applied to this leafentry +// If the outermost transactions match, then the broadcast commit/abort should +// be applied // -bool -le_has_xids(LEAFENTRY le, XIDS xids) { +bool le_has_xids(LEAFENTRY le, XIDS xids) { //Read num_uxrs uint32_t num_xids = toku_xids_get_num_xids(xids); invariant(num_xids > 0); //Disallow checking for having TXNID_NONE @@ -1245,8 +1390,7 @@ le_has_xids(LEAFENTRY le, XIDS xids) { return rval; } -void* -le_latest_val_and_len (LEAFENTRY le, uint32_t *len) { +void* le_latest_val_and_len (LEAFENTRY le, uint32_t *len) { uint8_t type = le->type; void *valp; @@ -1277,8 +1421,7 @@ le_latest_val_and_len (LEAFENTRY le, uint32_t *len) { if (uxr_is_insert(&uxr)) { *len = uxr.vallen; valp = p + (num_cuxrs - 1 + (num_puxrs!=0))*sizeof(uint32_t); - } - else { + } else { *len = 0; valp = NULL; } @@ -1295,8 +1438,7 @@ le_latest_val_and_len (LEAFENTRY le, uint32_t *len) { if (uxr_is_insert(uxr)) { slow_valp = uxr->valp; slow_len = uxr->vallen; - } - else { + } else { slow_valp = NULL; slow_len = 0; } @@ -1310,8 +1452,7 @@ le_latest_val_and_len (LEAFENTRY le, uint32_t *len) { } //DEBUG ONLY can be slow -void* -le_latest_val (LEAFENTRY le) { +void* le_latest_val (LEAFENTRY le) { ULE_S ule; le_unpack(&ule, le); UXR uxr = ule_get_innermost_uxr(&ule); @@ -1325,8 +1466,7 @@ le_latest_val (LEAFENTRY le) { } //needed to be fast for statistics. -uint32_t -le_latest_vallen (LEAFENTRY le) { +uint32_t le_latest_vallen (LEAFENTRY le) { uint32_t rval; uint8_t type = le->type; uint8_t *p; @@ -1354,8 +1494,7 @@ le_latest_vallen (LEAFENTRY le) { uxr_unpack_length_and_bit(&uxr, p); if (uxr_is_insert(&uxr)) { rval = uxr.vallen; - } - else { + } else { rval = 0; } break; @@ -1377,8 +1516,7 @@ le_latest_vallen (LEAFENTRY le) { return rval; } -uint64_t -le_outermost_uncommitted_xid (LEAFENTRY le) { +uint64_t le_outermost_uncommitted_xid (LEAFENTRY le) { uint64_t rval = TXNID_NONE; uint8_t type = le->type; @@ -1412,8 +1550,7 @@ le_outermost_uncommitted_xid (LEAFENTRY le) { //Optimization not required. This is a debug only function. //Print a leafentry out in human-readable format -int -print_klpair (FILE *outf, const void* keyp, uint32_t keylen, LEAFENTRY le) { +int print_klpair (FILE *outf, const void* keyp, uint32_t keylen, LEAFENTRY le) { ULE_S ule; le_unpack(&ule, le); uint32_t i; @@ -1444,23 +1581,21 @@ print_klpair (FILE *outf, const void* keyp, uint32_t keylen, LEAFENTRY le) { return 0; } -///////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// // This layer of abstraction (ule_xxx) knows the structure of the unpacked // leafentry and no other structure. // // ule constructor // Note that transaction 0 is explicit in the ule -static void -ule_init_empty_ule(ULE ule) { +static inline void ule_init_empty_ule(ULE ule) { ule->num_cuxrs = 1; ule->num_puxrs = 0; ule->uxrs = ule->uxrs_static; ule->uxrs[0] = committed_delete; } -static inline int32_t -min_i32(int32_t a, int32_t b) { +static inline int32_t min_i32(int32_t a, int32_t b) { int32_t rval = a < b ? a : b; return rval; } @@ -1470,8 +1605,8 @@ min_i32(int32_t a, int32_t b) { // // If the leafentry has already been promoted, there is nothing to do. // We have two transaction stacks (one from message, one from leaf entry). -// We want to implicitly promote transactions newer than (but not including) -// the innermost common ancestor (ICA) of the two stacks of transaction ids. We +// We want to implicitly promote transactions newer than (but not including) +// the innermost common ancestor (ICA) of the two stacks of transaction ids. We // know that this is the right thing to do because each transaction with an id // greater (later) than the ICA must have been either committed or aborted. // If it was aborted then we would have seen an abort message and removed the @@ -1483,8 +1618,7 @@ min_i32(int32_t a, int32_t b) { // record of ICA, keeping the transaction id of the ICA. // Outermost xid is zero for both ule and xids<> // -static void -ule_do_implicit_promotions(ULE ule, XIDS xids) { +static void ule_do_implicit_promotions(ULE ule, XIDS xids) { //Optimization for (most) common case. //No commits necessary if everything is already committed. if (ule->num_puxrs > 0) { @@ -1506,17 +1640,16 @@ ule_do_implicit_promotions(ULE ule, XIDS xids) { if (ica_index < ule->num_cuxrs) { invariant(ica_index == ule->num_cuxrs - 1); ule_promote_provisional_innermost_to_committed(ule); - } - else if (ica_index < ule->num_cuxrs + ule->num_puxrs - 1) { - //If ica is the innermost uxr in the leafentry, no commits are necessary. + } else if (ica_index < ule->num_cuxrs + ule->num_puxrs - 1) { + //If ica is the innermost uxr in the leafentry, no commits are + //necessary. ule_promote_provisional_innermost_to_index(ule, ica_index); } } } -static void -ule_promote_provisional_innermost_to_committed(ULE ule) { +static void ule_promote_provisional_innermost_to_committed(ULE ule) { //Must be something to promote. invariant(ule->num_puxrs); //Take value (or delete flag) from innermost. @@ -1532,8 +1665,7 @@ ule_promote_provisional_innermost_to_committed(ULE ule) { ule->num_puxrs = 0; //Discard all provisional uxrs. if (uxr_is_delete(old_innermost_uxr)) { ule_push_delete_uxr(ule, true, old_outermost_uncommitted_uxr->xid); - } - else { + } else { ule_push_insert_uxr(ule, true, old_outermost_uncommitted_uxr->xid, old_innermost_uxr->vallen, @@ -1541,11 +1673,13 @@ ule_promote_provisional_innermost_to_committed(ULE ule) { } } -static void -ule_try_promote_provisional_outermost(ULE ule, TXNID oldest_possible_live_xid) { +static void ule_try_promote_provisional_outermost( + ULE ule, + TXNID oldest_possible_live_xid) { // Effect: If there is a provisional record whose outermost xid is older than // the oldest known referenced_xid, promote it to committed. - if (ule->num_puxrs > 0 && ule_get_xid(ule, ule->num_cuxrs) < oldest_possible_live_xid) { + if (ule->num_puxrs > 0 && + ule_get_xid(ule, ule->num_cuxrs) < oldest_possible_live_xid) { ule_promote_provisional_innermost_to_committed(ule); } } @@ -1553,8 +1687,9 @@ ule_try_promote_provisional_outermost(ULE ule, TXNID oldest_possible_live_xid) { // Purpose is to promote the value (and type) of the innermost transaction // record to the uxr at the specified index (keeping the txnid of the uxr at // specified index.) -static void -ule_promote_provisional_innermost_to_index(ULE ule, uint32_t index) { +static void ule_promote_provisional_innermost_to_index( + ULE ule, + uint32_t index) { //Must not promote to committed portion of stack. invariant(index >= ule->num_cuxrs); //Must actually be promoting. @@ -1562,15 +1697,17 @@ ule_promote_provisional_innermost_to_index(ULE ule, uint32_t index) { UXR old_innermost_uxr = ule_get_innermost_uxr(ule); assert(!uxr_is_placeholder(old_innermost_uxr)); TXNID new_innermost_xid = ule->uxrs[index].xid; - ule->num_puxrs = index - ule->num_cuxrs; //Discard old uxr at index (and everything inner) + //Discard old uxr at index (and everything inner) + ule->num_puxrs = index - ule->num_cuxrs; if (uxr_is_delete(old_innermost_uxr)) { ule_push_delete_uxr(ule, false, new_innermost_xid); - } - else { - ule_push_insert_uxr(ule, false, - new_innermost_xid, - old_innermost_uxr->vallen, - old_innermost_uxr->valp); + } else { + ule_push_insert_uxr( + ule, + false, + new_innermost_xid, + old_innermost_uxr->vallen, + old_innermost_uxr->valp); } } @@ -1581,19 +1718,60 @@ ule_promote_provisional_innermost_to_index(ULE ule, uint32_t index) { // Purpose is to apply an insert message to this leafentry: -static void -ule_apply_insert(ULE ule, XIDS xids, uint32_t vallen, void * valp) { +static inline int64_t ule_apply_insert_no_overwrite( + ULE ule, + XIDS xids, + uint32_t vallen, + void* valp) { + + invariant(IS_VALID_LEN(vallen)); + int64_t retval = 0; + UXR old_innermost_uxr = ule_get_innermost_uxr(ule); + // If something exists, don't overwrite + if (uxr_is_insert(old_innermost_uxr)) { + retval = -1; + return retval; + } ule_prepare_for_new_uxr(ule, xids); - TXNID this_xid = toku_xids_get_innermost_xid(xids); // xid of transaction doing this insert + // xid of transaction doing this insert + TXNID this_xid = toku_xids_get_innermost_xid(xids); ule_push_insert_uxr(ule, this_xid == TXNID_NONE, this_xid, vallen, valp); + return retval; +} + +// Purpose is to apply an insert message to this leafentry: +static inline int64_t ule_apply_insert( + ULE ule, + XIDS xids, + uint32_t vallen, + void* valp) { + + invariant(IS_VALID_LEN(vallen)); + int64_t retval = 0; + UXR old_innermost_uxr = ule_get_innermost_uxr(ule); + // If something exists, overwrite + if (uxr_is_insert(old_innermost_uxr)) { + retval = -1; + } + ule_prepare_for_new_uxr(ule, xids); + // xid of transaction doing this insert + TXNID this_xid = toku_xids_get_innermost_xid(xids); + ule_push_insert_uxr(ule, this_xid == TXNID_NONE, this_xid, vallen, valp); + return retval; } // Purpose is to apply a delete message to this leafentry: -static void -ule_apply_delete(ULE ule, XIDS xids) { +static inline int64_t ule_apply_delete(ULE ule, XIDS xids) { + int64_t retval = 0; + UXR old_innermost_uxr = ule_get_innermost_uxr(ule); + if (FT_UNLIKELY(uxr_is_delete(old_innermost_uxr))) { + retval = 1; + } ule_prepare_for_new_uxr(ule, xids); - TXNID this_xid = toku_xids_get_innermost_xid(xids); // xid of transaction doing this delete + // xid of transaction doing this delete + TXNID this_xid = toku_xids_get_innermost_xid(xids); ule_push_delete_uxr(ule, this_xid == TXNID_NONE, this_xid); + return retval; } // First, discard anything done earlier by this transaction. @@ -1601,20 +1779,18 @@ ule_apply_delete(ULE ule, XIDS xids) { // outer transactions that are newer than then newest (innermost) transaction in // the leafentry. If so, record those outer transactions in the leafentry // with placeholders. -static void -ule_prepare_for_new_uxr(ULE ule, XIDS xids) { +static inline void ule_prepare_for_new_uxr(ULE ule, XIDS xids) { TXNID this_xid = toku_xids_get_innermost_xid(xids); //This is for LOADER_USE_PUTS or transactionless environment //where messages use XIDS of 0 if (this_xid == TXNID_NONE && ule_get_innermost_xid(ule) == TXNID_NONE) { ule_remove_innermost_uxr(ule); - } - // case where we are transactional and xids stack matches ule stack - else if (ule->num_puxrs > 0 && ule_get_innermost_xid(ule) == this_xid) { + } else if (ule->num_puxrs > 0 && ule_get_innermost_xid(ule) == this_xid) { + // case where we are transactional and xids stack matches ule stack ule_remove_innermost_uxr(ule); - } - // case where we are transactional and xids stack does not match ule stack - else { + } else { + // case where we are transactional and xids stack does not match ule + // stack ule_add_placeholders(ule, xids); } } @@ -1625,10 +1801,12 @@ ule_prepare_for_new_uxr(ULE ule, XIDS xids) { // then there is nothing to be done. // If this transaction did modify the leafentry, then undo whatever it did (by // removing the transaction record (uxr) and any placeholders underneath. -// Remember, the innermost uxr can only be an insert or a delete, not a placeholder. -static void -ule_apply_abort(ULE ule, XIDS xids) { - TXNID this_xid = toku_xids_get_innermost_xid(xids); // xid of transaction doing this abort +// Remember, the innermost uxr can only be an insert or a delete, not a +// placeholder. +static inline int64_t ule_apply_abort(ULE ule, XIDS xids) { + int64_t retval = 0; + // xid of transaction doing this abort + TXNID this_xid = toku_xids_get_innermost_xid(xids); invariant(this_xid!=TXNID_NONE); UXR innermost = ule_get_innermost_uxr(ule); // need to check for provisional entries in ule, otherwise @@ -1636,15 +1814,34 @@ ule_apply_abort(ULE ule, XIDS xids) { // in a bug where the most recently committed has same xid // as the XID's innermost if (ule->num_puxrs > 0 && innermost->xid == this_xid) { + // if this is a rollback of a delete of a new ule, return 0 + // (i.e. double delete) + if (uxr_is_delete(innermost)) { + if (ule->num_puxrs == 1 && ule->num_cuxrs == 1 && + uxr_is_delete(&(ule->uxrs[0]))) { + retval = 0; + } else { + retval = 1; + } + } else if (uxr_is_insert(innermost)) { + if (ule->num_puxrs == 1 && ule->num_cuxrs == 1 && + uxr_is_insert(&(ule->uxrs[0]))) { + retval = 0; + } else { + retval = -1; + } + } + // if this is a rollback of a insert of an exising ule, return 0 + // (i.e. double insert) invariant(ule->num_puxrs>0); ule_remove_innermost_uxr(ule); ule_remove_innermost_placeholders(ule); } invariant(ule->num_cuxrs > 0); + return retval; } -static void -ule_apply_broadcast_commit_all (ULE ule) { +static void ule_apply_broadcast_commit_all (ULE ule) { ule->uxrs[0] = ule->uxrs[ule->num_puxrs + ule->num_cuxrs - 1]; ule->uxrs[0].xid = TXNID_NONE; ule->num_puxrs = 0; @@ -1657,9 +1854,11 @@ ule_apply_broadcast_commit_all (ULE ule) { // then there is nothing to be done. // Also, if there are no uncommitted transaction records there is nothing to do. // If this transaction did modify the leafentry, then promote whatever it did. -// Remember, the innermost uxr can only be an insert or a delete, not a placeholder. +// Remember, the innermost uxr can only be an insert or a delete, not a +// placeholder. void ule_apply_commit(ULE ule, XIDS xids) { - TXNID this_xid = toku_xids_get_innermost_xid(xids); // xid of transaction committing + // xid of transaction committing + TXNID this_xid = toku_xids_get_innermost_xid(xids); invariant(this_xid!=TXNID_NONE); // need to check for provisional entries in ule, otherwise // there is nothing to abort, not checking this may result @@ -1668,16 +1867,19 @@ void ule_apply_commit(ULE ule, XIDS xids) { if (ule->num_puxrs > 0 && ule_get_innermost_xid(ule) == this_xid) { // 3 cases: //1- it's already a committed value (do nothing) (num_puxrs==0) - //2- it's provisional but root level (make a new committed value (num_puxrs==1) + //2- it's provisional but root level (make a new committed value + // (num_puxrs==1) //3- it's provisional and not root (promote); (num_puxrs>1) if (ule->num_puxrs == 1) { //new committed value ule_promote_provisional_innermost_to_committed(ule); - } - else if (ule->num_puxrs > 1) { - //ule->uxrs[ule->num_cuxrs+ule->num_puxrs-1] is the innermost (this transaction) + } else if (ule->num_puxrs > 1) { + //ule->uxrs[ule->num_cuxrs+ule->num_puxrs-1] is the innermost + // (this transaction) //ule->uxrs[ule->num_cuxrs+ule->num_puxrs-2] is the 2nd innermost //We want to promote the innermost uxr one level out. - ule_promote_provisional_innermost_to_index(ule, ule->num_cuxrs+ule->num_puxrs-2); + ule_promote_provisional_innermost_to_index( + ule, + ule->num_cuxrs+ule->num_puxrs-2); } } } @@ -1687,14 +1889,17 @@ void ule_apply_commit(ULE ule, XIDS xids) { // // Purpose is to record an insert for this transaction (and set type correctly). -static void -ule_push_insert_uxr(ULE ule, bool is_committed, TXNID xid, uint32_t vallen, void * valp) { - UXR uxr = ule_get_first_empty_uxr(ule); +static inline void ule_push_insert_uxr( + ULE ule, + bool is_committed, TXNID xid, + uint32_t vallen, + void* valp) { + + UXR uxr = ule_get_first_empty_uxr(ule); if (is_committed) { invariant(ule->num_puxrs==0); ule->num_cuxrs++; - } - else { + } else { ule->num_puxrs++; } uxr->xid = xid; @@ -1706,23 +1911,21 @@ ule_push_insert_uxr(ULE ule, bool is_committed, TXNID xid, uint32_t vallen, void // Purpose is to record a delete for this transaction. If this transaction // is the root transaction, then truly delete the leafentry by marking the // ule as empty. -static void -ule_push_delete_uxr(ULE ule, bool is_committed, TXNID xid) { +static inline void ule_push_delete_uxr(ULE ule, bool is_committed, TXNID xid) { UXR uxr = ule_get_first_empty_uxr(ule); if (is_committed) { invariant(ule->num_puxrs==0); ule->num_cuxrs++; - } - else { + } else { ule->num_puxrs++; } uxr->xid = xid; uxr->type = XR_DELETE; } -// Purpose is to push a placeholder on the top of the leafentry's transaction stack. -static void -ule_push_placeholder_uxr(ULE ule, TXNID xid) { +// Purpose is to push a placeholder on the top of the leafentry's transaction +// stack. +static inline void ule_push_placeholder_uxr(ULE ule, TXNID xid) { invariant(ule->num_cuxrs>0); UXR uxr = ule_get_first_empty_uxr(ule); uxr->xid = xid; @@ -1731,16 +1934,14 @@ ule_push_placeholder_uxr(ULE ule, TXNID xid) { } // Return innermost transaction record. -static UXR -ule_get_innermost_uxr(ULE ule) { +static inline UXR ule_get_innermost_uxr(ULE ule) { invariant(ule->num_cuxrs > 0); UXR rval = &(ule->uxrs[ule->num_cuxrs + ule->num_puxrs - 1]); return rval; } // Return first empty transaction record -static UXR -ule_get_first_empty_uxr(ULE ule) { +static inline UXR ule_get_first_empty_uxr(ULE ule) { invariant(ule->num_puxrs < MAX_TRANSACTION_RECORDS-1); UXR rval = &(ule->uxrs[ule->num_cuxrs+ule->num_puxrs]); return rval; @@ -1748,14 +1949,12 @@ ule_get_first_empty_uxr(ULE ule) { // Remove the innermost transaction (pop the leafentry's stack), undoing // whatever the innermost transaction did. -static void -ule_remove_innermost_uxr(ULE ule) { +static inline void ule_remove_innermost_uxr(ULE ule) { //It is possible to remove the committed delete at first insert. invariant(ule->num_cuxrs > 0); if (ule->num_puxrs) { ule->num_puxrs--; - } - else { + } else { //This is for LOADER_USE_PUTS or transactionless environment //where messages use XIDS of 0 invariant(ule->num_cuxrs == 1); @@ -1764,14 +1963,12 @@ ule_remove_innermost_uxr(ULE ule) { } } -static TXNID -ule_get_innermost_xid(ULE ule) { +static inline TXNID ule_get_innermost_xid(ULE ule) { TXNID rval = ule_get_xid(ule, ule->num_cuxrs + ule->num_puxrs - 1); return rval; } -static TXNID -ule_get_xid(ULE ule, uint32_t index) { +static inline TXNID ule_get_xid(ULE ule, uint32_t index) { invariant(index < ule->num_cuxrs + ule->num_puxrs); TXNID rval = ule->uxrs[index].xid; return rval; @@ -1781,8 +1978,7 @@ ule_get_xid(ULE ule, uint32_t index) { // innermost recorded transactions), if necessary. This function is idempotent. // It makes no logical sense for a placeholder to be the innermost recorded // transaction record, so placeholders at the top of the stack are not legal. -static void -ule_remove_innermost_placeholders(ULE ule) { +static void ule_remove_innermost_placeholders(ULE ule) { UXR uxr = ule_get_innermost_uxr(ule); while (uxr_is_placeholder(uxr)) { invariant(ule->num_puxrs>0); @@ -1796,8 +1992,7 @@ ule_remove_innermost_placeholders(ULE ule) { // Note, after placeholders are added, an insert or delete will be added. This // function temporarily leaves the transaction stack in an illegal state (having // placeholders on top). -static void -ule_add_placeholders(ULE ule, XIDS xids) { +static void ule_add_placeholders(ULE ule, XIDS xids) { //Placeholders can be placed on top of the committed uxr. invariant(ule->num_cuxrs > 0); @@ -1819,47 +2014,40 @@ ule_add_placeholders(ULE ule, XIDS xids) { } } -uint64_t -ule_num_uxrs(ULE ule) { +uint64_t ule_num_uxrs(ULE ule) { return ule->num_cuxrs + ule->num_puxrs; } -UXR -ule_get_uxr(ULE ule, uint64_t ith) { +UXR ule_get_uxr(ULE ule, uint64_t ith) { invariant(ith < ule_num_uxrs(ule)); return &ule->uxrs[ith]; } -uint32_t -ule_get_num_committed(ULE ule) { +uint32_t ule_get_num_committed(ULE ule) { return ule->num_cuxrs; } -uint32_t -ule_get_num_provisional(ULE ule) { +uint32_t ule_get_num_provisional(ULE ule) { return ule->num_puxrs; } -int -ule_is_committed(ULE ule, uint64_t ith) { +int ule_is_committed(ULE ule, uint64_t ith) { invariant(ith < ule_num_uxrs(ule)); return ith < ule->num_cuxrs; } -int -ule_is_provisional(ULE ule, uint64_t ith) { +int ule_is_provisional(ULE ule, uint64_t ith) { invariant(ith < ule_num_uxrs(ule)); return ith >= ule->num_cuxrs; } // return size of data for innermost uxr, the size of val -uint32_t -ule_get_innermost_numbytes(ULE ule, uint32_t keylen) { +uint32_t ule_get_innermost_numbytes(ULE ule, uint32_t keylen) { uint32_t rval; UXR uxr = ule_get_innermost_uxr(ule); - if (uxr_is_delete(uxr)) + if (uxr_is_delete(uxr)) { rval = 0; - else { + } else { rval = uxr_get_vallen(uxr) + keylen; } return rval; @@ -1870,68 +2058,65 @@ ule_get_innermost_numbytes(ULE ule, uint32_t keylen) { // This layer of abstraction (uxr_xxx) understands uxr and nothing else. // -static inline bool -uxr_type_is_insert(uint8_t type) { +static inline bool uxr_type_is_insert(uint8_t type) { bool rval = (bool)(type == XR_INSERT); return rval; } -bool -uxr_is_insert(UXR uxr) { +bool uxr_is_insert(UXR uxr) { return uxr_type_is_insert(uxr->type); } -static inline bool -uxr_type_is_delete(uint8_t type) { +static inline bool uxr_type_is_delete(uint8_t type) { bool rval = (bool)(type == XR_DELETE); return rval; } -bool -uxr_is_delete(UXR uxr) { +bool uxr_is_delete(UXR uxr) { return uxr_type_is_delete(uxr->type); } -static inline bool -uxr_type_is_placeholder(uint8_t type) { +static inline bool uxr_type_is_placeholder(uint8_t type) { bool rval = (bool)(type == XR_PLACEHOLDER); return rval; } -bool -uxr_is_placeholder(UXR uxr) { +bool uxr_is_placeholder(UXR uxr) { return uxr_type_is_placeholder(uxr->type); } -void * -uxr_get_val(UXR uxr) { +void* uxr_get_val(UXR uxr) { return uxr->valp; } -uint32_t -uxr_get_vallen(UXR uxr) { +uint32_t uxr_get_vallen(UXR uxr) { return uxr->vallen; } -TXNID -uxr_get_txnid(UXR uxr) { +TXNID uxr_get_txnid(UXR uxr) { return uxr->xid; } -static int -le_iterate_get_accepted_index(TXNID *xids, uint32_t *index, uint32_t num_xids, LE_ITERATE_CALLBACK f, TOKUTXN context, bool top_is_provisional) { +static int le_iterate_get_accepted_index( + TXNID* xids, + uint32_t* index, + uint32_t num_xids, + LE_ITERATE_CALLBACK f, + TOKUTXN context, + bool top_is_provisional) { + uint32_t i; int r = 0; - // if this for loop does not return anything, we return num_xids-1, which should map to T_0 + // if this for loop does not return anything, we return num_xids-1, which + // should map to T_0 for (i = 0; i < num_xids - 1; i++) { TXNID xid = toku_dtoh64(xids[i]); r = f(xid, context, (i == 0 && top_is_provisional)); if (r==TOKUDB_ACCEPT) { r = 0; break; //or goto something - } - else if (r!=0) { + } else if (r!=0) { break; } } @@ -1940,8 +2125,7 @@ le_iterate_get_accepted_index(TXNID *xids, uint32_t *index, uint32_t num_xids, L } #if ULE_DEBUG -static void -ule_verify_xids(ULE ule, uint32_t interesting, TXNID *xids) { +static void ule_verify_xids(ULE ule, uint32_t interesting, TXNID *xids) { int has_p = (ule->num_puxrs != 0); invariant(ule->num_cuxrs + has_p == interesting); uint32_t i; @@ -1953,21 +2137,29 @@ ule_verify_xids(ULE ule, uint32_t interesting, TXNID *xids) { #endif // -// Iterates over "possible" TXNIDs in a leafentry's stack, until one is accepted by 'f'. If the value -// associated with the accepted TXNID is not an insert, then set *is_emptyp to true, otherwise false +// Iterates over "possible" TXNIDs in a leafentry's stack, until one is +// accepted by 'f'. If the value associated with the accepted TXNID is not an +// insert, then set *is_emptyp to true, otherwise false // The "possible" TXNIDs are: -// if provisionals exist, then the first possible TXNID is the outermost provisional. -// The next possible TXNIDs are the committed TXNIDs, from most recently committed to T_0. -// If provisionals exist, and the outermost provisional is accepted by 'f', +// If provisionals exist, then the first possible TXNID is the outermost +// provisional. +// The next possible TXNIDs are the committed TXNIDs, from most recently +// committed to T_0. +// If provisionals exist, and the outermost provisional is accepted by 'f', // the associated value checked is the innermost provisional's value. // Parameters: // le - leafentry to iterate over -// f - callback function that checks if a TXNID in le is accepted, and its associated value should be examined. +// f - callback function that checks if a TXNID in le is accepted, and its +// associated value should be examined. // is_delp - output parameter that returns answer // context - parameter for f // -static int -le_iterate_is_del(LEAFENTRY le, LE_ITERATE_CALLBACK f, bool *is_delp, TOKUTXN context) { +static int le_iterate_is_del( + LEAFENTRY le, + LE_ITERATE_CALLBACK f, + bool* is_delp, + TOKUTXN context) { + #if ULE_DEBUG ULE_S ule; le_unpack(&ule, le); @@ -2002,8 +2194,17 @@ le_iterate_is_del(LEAFENTRY le, LE_ITERATE_CALLBACK f, bool *is_delp, TOKUTXN co #if ULE_DEBUG ule_verify_xids(&ule, num_interesting, xids); #endif - r = le_iterate_get_accepted_index(xids, &index, num_interesting, f, context, (num_puxrs != 0)); - if (r!=0) goto cleanup; + r = + le_iterate_get_accepted_index( + xids, + &index, + num_interesting, + f, + context, + (num_puxrs != 0)); + if (r != 0) { + goto cleanup; + } invariant(index < num_interesting); //Skip TXNIDs @@ -2017,7 +2218,9 @@ le_iterate_is_del(LEAFENTRY le, LE_ITERATE_CALLBACK f, bool *is_delp, TOKUTXN co #if ULE_DEBUG { uint32_t has_p = (ule.num_puxrs != 0); - uint32_t ule_index = (index==0) ? ule.num_cuxrs + ule.num_puxrs - 1 : ule.num_cuxrs - 1 + has_p - index; + uint32_t ule_index = (index==0) ? + ule.num_cuxrs + ule.num_puxrs - 1 : + ule.num_cuxrs - 1 + has_p - index; UXR uxr = ule.uxrs + ule_index; invariant(uxr_is_delete(uxr) == is_del); } @@ -2034,7 +2237,11 @@ cleanup: return r; } -static int le_iterate_read_committed_callback(TXNID txnid, TOKUTXN txn, bool is_provisional UU()) { +static int le_iterate_read_committed_callback( + TXNID txnid, + TOKUTXN txn, + bool is_provisional UU()) { + if (is_provisional) { return toku_txn_reads_txnid(txnid, txn, is_provisional); } @@ -2058,33 +2265,40 @@ int le_val_is_del(LEAFENTRY le, enum cursor_read_type read_type, TOKUTXN txn) { txn ); rval = is_del; - } - else if (read_type == C_READ_ANY) { + } else if (read_type == C_READ_ANY) { rval = le_latest_is_del(le); - } - else { + } else { invariant(false); } return rval; } // -// Iterates over "possible" TXNIDs in a leafentry's stack, until one is accepted by 'f'. Set -// valpp and vallenp to value and length associated with accepted TXNID +// Iterates over "possible" TXNIDs in a leafentry's stack, until one is accepted +// by 'f'. Set valpp and vallenp to value and length associated with accepted +// TXNID // The "possible" TXNIDs are: -// if provisionals exist, then the first possible TXNID is the outermost provisional. -// The next possible TXNIDs are the committed TXNIDs, from most recently committed to T_0. -// If provisionals exist, and the outermost provisional is accepted by 'f', +// If provisionals exist, then the first possible TXNID is the outermost +// provisional. +// The next possible TXNIDs are the committed TXNIDs, from most recently +// committed to T_0. +// If provisionals exist, and the outermost provisional is accepted by 'f', // the associated length value is the innermost provisional's length and value. // Parameters: // le - leafentry to iterate over -// f - callback function that checks if a TXNID in le is accepted, and its associated value should be examined. +// f - callback function that checks if a TXNID in le is accepted, and its +// associated value should be examined. // valpp - output parameter that returns pointer to value // vallenp - output parameter that returns length of value // context - parameter for f // -int -le_iterate_val(LEAFENTRY le, LE_ITERATE_CALLBACK f, void** valpp, uint32_t *vallenp, TOKUTXN context) { +int le_iterate_val( + LEAFENTRY le, + LE_ITERATE_CALLBACK f, + void** valpp, + uint32_t* vallenp, + TOKUTXN context) { + #if ULE_DEBUG ULE_S ule; le_unpack(&ule, le); @@ -2124,8 +2338,17 @@ le_iterate_val(LEAFENTRY le, LE_ITERATE_CALLBACK f, void** valpp, uint32_t *vall #if ULE_DEBUG ule_verify_xids(&ule, num_interesting, xids); #endif - r = le_iterate_get_accepted_index(xids, &index, num_interesting, f, context, (num_puxrs != 0)); - if (r!=0) goto cleanup; + r = + le_iterate_get_accepted_index( + xids, + &index, + num_interesting, + f, + context, + (num_puxrs != 0)); + if (r != 0) { + goto cleanup; + } invariant(index < num_interesting); //Skip TXNIDs @@ -2158,7 +2381,9 @@ le_iterate_val(LEAFENTRY le, LE_ITERATE_CALLBACK f, void** valpp, uint32_t *vall #if ULE_DEBUG { uint32_t has_p = (ule.num_puxrs != 0); - uint32_t ule_index = (index==0) ? ule.num_cuxrs + ule.num_puxrs - 1 : ule.num_cuxrs - 1 + has_p - index; + uint32_t ule_index = (index==0) ? + ule.num_cuxrs + ule.num_puxrs - 1 : + ule.num_cuxrs - 1 + has_p - index; UXR uxr = ule.uxrs + ule_index; invariant(uxr_is_insert(uxr)); invariant(uxr->vallen == vallen); @@ -2188,10 +2413,15 @@ cleanup: return r; } -void le_extract_val(LEAFENTRY le, - // should we return the entire leafentry as the val? - bool is_leaf_mode, enum cursor_read_type read_type, - TOKUTXN ttxn, uint32_t *vallen, void **val) { +void le_extract_val( + LEAFENTRY le, + // should we return the entire leafentry as the val? + bool is_leaf_mode, + enum cursor_read_type read_type, + TOKUTXN ttxn, + uint32_t* vallen, + void** val) { + if (is_leaf_mode) { *val = le; *vallen = leafentry_memsize(le); @@ -2199,18 +2429,11 @@ void le_extract_val(LEAFENTRY le, LE_ITERATE_CALLBACK f = (read_type == C_READ_SNAPSHOT) ? toku_txn_reads_txnid : le_iterate_read_committed_callback; - int r = le_iterate_val( - le, - f, - val, - vallen, - ttxn - ); + int r = le_iterate_val(le, f, val, vallen, ttxn); lazy_assert_zero(r); } else if (read_type == C_READ_ANY){ *val = le_latest_val_and_len(le, vallen); - } - else { + } else { assert(false); } } @@ -2244,9 +2467,9 @@ static_assert(18 == sizeof(leafentry_13), "wrong size"); static_assert(9 == __builtin_offsetof(leafentry_13, u), "wrong offset"); //Requires: -// Leafentry that ule represents should not be destroyed (is not just all deletes) -static size_t -le_memsize_from_ule_13 (ULE ule, LEAFENTRY_13 le) { +// Leafentry that ule represents should not be destroyed (is not just all +// deletes) +static size_t le_memsize_from_ule_13 (ULE ule, LEAFENTRY_13 le) { uint32_t num_uxrs = ule->num_cuxrs + ule->num_puxrs; assert(num_uxrs); size_t rval; @@ -2257,8 +2480,7 @@ le_memsize_from_ule_13 (ULE ule, LEAFENTRY_13 le) { +4 //vallen +le->keylen //actual key +ule->uxrs[0].vallen; //actual val - } - else { + } else { rval = 1 //num_uxrs +4 //keylen +le->keylen //actual key @@ -2276,16 +2498,20 @@ le_memsize_from_ule_13 (ULE ule, LEAFENTRY_13 le) { return rval; } -//This function is mostly copied from 4.1.1 (which is version 12, same as 13 except that only 13 is upgradable). -// Note, number of transaction records in version 13 has been replaced by separate counters in version 14 (MVCC), -// one counter for committed transaction records and one counter for provisional transaction records. When -// upgrading a version 13 le to version 14, the number of committed transaction records is always set to one (1) -// and the number of provisional transaction records is set to the original number of transaction records -// minus one. The bottom transaction record is assumed to be a committed value. (If there is no committed -// value then the bottom transaction record of version 13 is a committed delete.) -// This is the only change from the 4.1.1 code. The rest of the leafentry is read as is. -static void -le_unpack_13(ULE ule, LEAFENTRY_13 le) { +// This function is mostly copied from 4.1.1 (which is version 12, same as 13 +// except that only 13 is upgradable). +// Note, number of transaction records in version 13 has been replaced by +// separate counters in version 14 (MVCC), one counter for committed transaction +// records and one counter for provisional transaction records. When upgrading +// a version 13 le to version 14, the number of committed transaction records is +// always set to one (1) and the number of provisional transaction records is +// set to the original number of transaction records minus one. The bottom +// transaction record is assumed to be a committed value. (If there is no +// committed value then the bottom transaction record of version 13 is a +// committed delete.) +// This is the only change from the 4.1.1 code. The rest of the leafentry is +// read as is. +static void le_unpack_13(ULE ule, LEAFENTRY_13 le) { //Read num_uxrs uint8_t num_xrs = le->num_xrs; assert(num_xrs > 0); @@ -2302,15 +2528,15 @@ le_unpack_13(ULE ule, LEAFENTRY_13 le) { uint8_t *p; if (num_xrs == 1) { //Unpack a 'committed leafentry' (No uncommitted transactions exist) - ule->uxrs[0].type = XR_INSERT; //Must be or the leafentry would not exist + //Must be or the leafentry would not exist + ule->uxrs[0].type = XR_INSERT; ule->uxrs[0].vallen = vallen_of_innermost_insert; ule->uxrs[0].valp = &le->u.comm.key_val[keylen]; ule->uxrs[0].xid = 0; //Required. //Set p to immediately after leafentry p = &le->u.comm.key_val[keylen + vallen_of_innermost_insert]; - } - else { + } else { //Unpack a 'provisional leafentry' (Uncommitted transactions exist) //Read in type. @@ -2337,8 +2563,7 @@ le_unpack_13(ULE ule, LEAFENTRY_13 le) { //Not innermost, so load the type. uxr->type = *p; p += 1; - } - else { + } else { //Innermost, load the type previously read from header uxr->type = innermost_type; } @@ -2349,12 +2574,11 @@ le_unpack_13(ULE ule, LEAFENTRY_13 le) { //Not committed nor outermost uncommitted, so load the xid. uxr->xid = toku_dtoh64(*(TXNID*)p); p += 8; - } - else if (i == 1) { - //Outermost uncommitted, load the xid previously read from header + } else if (i == 1) { + //Outermost uncommitted, load the xid previously read from + //header uxr->xid = xid_outermost_uncommitted; - } - else { + } else { // i == 0, committed entry uxr->xid = 0; } @@ -2367,9 +2591,9 @@ le_unpack_13(ULE ule, LEAFENTRY_13 le) { uxr->valp = p; p += uxr->vallen; - } - else { - //Innermost insert, load the vallen/valp previously read from header + } else { + //Innermost insert, load the vallen/valp previously read + //from header uxr->vallen = vallen_of_innermost_insert; uxr->valp = valp_of_innermost_insert; found_innermost_insert = true; @@ -2384,8 +2608,7 @@ le_unpack_13(ULE ule, LEAFENTRY_13 le) { #endif } -size_t -leafentry_disksize_13(LEAFENTRY_13 le) { +size_t leafentry_disksize_13(LEAFENTRY_13 le) { ULE_S ule; le_unpack_13(&ule, le); size_t memsize = le_memsize_from_ule_13(&ule, le); @@ -2393,13 +2616,13 @@ leafentry_disksize_13(LEAFENTRY_13 le) { return memsize; } -int -toku_le_upgrade_13_14(LEAFENTRY_13 old_leafentry, - void** keyp, - uint32_t* keylen, - size_t *new_leafentry_memorysize, - LEAFENTRY *new_leafentry_p - ) { +int toku_le_upgrade_13_14( + LEAFENTRY_13 old_leafentry, + void** keyp, + uint32_t* keylen, + size_t* new_leafentry_memorysize, + LEAFENTRY* new_leafentry_p) { + ULE_S ule; int rval; invariant(old_leafentry); @@ -2408,23 +2631,23 @@ toku_le_upgrade_13_14(LEAFENTRY_13 old_leafentry, *keylen = old_leafentry->keylen; if (old_leafentry->num_xrs == 1) { *keyp = old_leafentry->u.comm.key_val; - } - else { + } else { *keyp = old_leafentry->u.prov.key_val_xrs; } // We used to pass NULL for omt and mempool, so that we would use // malloc instead of a mempool. However after supporting upgrade, // we need to use mempools and the OMT. - rval = le_pack(&ule, // create packed leafentry - nullptr, - 0, //only matters if we are passing in a bn_data - nullptr, //only matters if we are passing in a bn_data - 0, //only matters if we are passing in a bn_data - 0, //only matters if we are passing in a bn_data - 0, //only matters if we are passing in a bn_data - new_leafentry_p, - nullptr //only matters if we are passing in a bn_data - ); + rval = + le_pack( + &ule, // create packed leafentry + nullptr, + 0, //only matters if we are passing in a bn_data + nullptr, //only matters if we are passing in a bn_data + 0, //only matters if we are passing in a bn_data + 0, //only matters if we are passing in a bn_data + 0, //only matters if we are passing in a bn_data + new_leafentry_p, + nullptr); //only matters if we are passing in a bn_data ule_cleanup(&ule); *new_leafentry_memorysize = leafentry_memsize(*new_leafentry_p); return rval; diff --git a/storage/tokudb/PerconaFT/locktree/tests/CMakeLists.txt b/storage/tokudb/PerconaFT/locktree/tests/CMakeLists.txt index 47ec4d884fb..20ab682ae12 100644 --- a/storage/tokudb/PerconaFT/locktree/tests/CMakeLists.txt +++ b/storage/tokudb/PerconaFT/locktree/tests/CMakeLists.txt @@ -7,7 +7,7 @@ if(BUILD_TESTING) foreach(src ${srcs}) get_filename_component(base ${src} NAME_WE) - add_executable(${base} ${base}) + add_executable(${base} ${base}.cc) add_space_separated_property(TARGET ${base} COMPILE_FLAGS -fvisibility=hidden) target_link_libraries(${base} locktree ft ${LIBTOKUPORTABILITY}) add_locktree_test(${base}) diff --git a/storage/tokudb/PerconaFT/portability/tests/CMakeLists.txt b/storage/tokudb/PerconaFT/portability/tests/CMakeLists.txt index 7fea21fc8e3..ff233327436 100644 --- a/storage/tokudb/PerconaFT/portability/tests/CMakeLists.txt +++ b/storage/tokudb/PerconaFT/portability/tests/CMakeLists.txt @@ -10,7 +10,7 @@ if(BUILD_TESTING) foreach(src ${srcs}) get_filename_component(test ${src} NAME_WE) - add_executable(${test} ${test}) + add_executable(${test} ${test}.cc) target_link_libraries(${test} ${LIBTOKUPORTABILITY}) set_target_properties(${test} PROPERTIES POSITION_INDEPENDENT_CODE ON) add_space_separated_property(TARGET ${test} COMPILE_FLAGS -fvisibility=hidden) diff --git a/storage/tokudb/PerconaFT/portability/toku_pthread.h b/storage/tokudb/PerconaFT/portability/toku_pthread.h index 25cf48dfd8c..84c27736201 100644 --- a/storage/tokudb/PerconaFT/portability/toku_pthread.h +++ b/storage/tokudb/PerconaFT/portability/toku_pthread.h @@ -72,15 +72,18 @@ typedef struct toku_mutex_aligned { toku_mutex_t aligned_mutex __attribute__((__aligned__(64))); } toku_mutex_aligned_t; -// Different OSes implement mutexes as different amounts of nested structs. -// C++ will fill out all missing values with zeroes if you provide at least one zero, but it needs the right amount of nesting. -#if defined(__FreeBSD__) -# define ZERO_MUTEX_INITIALIZER {0} -#elif defined(__APPLE__) -# define ZERO_MUTEX_INITIALIZER {{0}} -#else // __linux__, at least -# define ZERO_MUTEX_INITIALIZER {{{0}}} -#endif +// Initializing with {} will fill in a struct with all zeros. +// But you may also need a pragma to suppress the warnings, as follows +// +// #pragma GCC diagnostic push +// #pragma GCC diagnostic ignored "-Wmissing-field-initializers" +// toku_mutex_t foo = ZERO_MUTEX_INITIALIZER; +// #pragma GCC diagnostic pop +// +// In general it will be a lot of busy work to make this codebase compile +// cleanly with -Wmissing-field-initializers + +# define ZERO_MUTEX_INITIALIZER {} #if TOKU_PTHREAD_DEBUG # define TOKU_MUTEX_INITIALIZER { .pmutex = PTHREAD_MUTEX_INITIALIZER, .owner = 0, .locked = false, .valid = true } @@ -223,15 +226,9 @@ typedef struct toku_cond { pthread_cond_t pcond; } toku_cond_t; -// Different OSes implement mutexes as different amounts of nested structs. -// C++ will fill out all missing values with zeroes if you provide at least one zero, but it needs the right amount of nesting. -#if defined(__FreeBSD__) -# define ZERO_COND_INITIALIZER {0} -#elif defined(__APPLE__) -# define ZERO_COND_INITIALIZER {{0}} -#else // __linux__, at least -# define ZERO_COND_INITIALIZER {{{0}}} -#endif +// Same considerations as for ZERO_MUTEX_INITIALIZER apply +#define ZERO_COND_INITIALIZER {} + #define TOKU_COND_INITIALIZER {PTHREAD_COND_INITIALIZER} static inline void diff --git a/storage/tokudb/PerconaFT/portability/toku_time.h b/storage/tokudb/PerconaFT/portability/toku_time.h index c476b64a212..11a3f3aa2b9 100644 --- a/storage/tokudb/PerconaFT/portability/toku_time.h +++ b/storage/tokudb/PerconaFT/portability/toku_time.h @@ -108,3 +108,13 @@ static inline uint64_t toku_current_time_microsec(void) { gettimeofday(&t, NULL); return t.tv_sec * (1UL * 1000 * 1000) + t.tv_usec; } + +// sleep microseconds +static inline void toku_sleep_microsec(uint64_t ms) { + struct timeval t; + + t.tv_sec = ms / 1000000; + t.tv_usec = ms % 1000000; + + select(0, NULL, NULL, NULL, &t); +} diff --git a/storage/tokudb/PerconaFT/src/export.map b/storage/tokudb/PerconaFT/src/export.map index 3f2c7569ea4..fc2be5f41a5 100644 --- a/storage/tokudb/PerconaFT/src/export.map +++ b/storage/tokudb/PerconaFT/src/export.map @@ -82,6 +82,7 @@ toku_test_db_redirect_dictionary; toku_test_get_latest_lsn; toku_test_get_checkpointing_user_data_status; + toku_set_test_txn_sync_callback; toku_indexer_set_test_only_flags; toku_increase_last_xid; diff --git a/storage/tokudb/PerconaFT/src/tests/CMakeLists.txt b/storage/tokudb/PerconaFT/src/tests/CMakeLists.txt index 457dd9b96a9..47f6aa44a75 100644 --- a/storage/tokudb/PerconaFT/src/tests/CMakeLists.txt +++ b/storage/tokudb/PerconaFT/src/tests/CMakeLists.txt @@ -49,15 +49,15 @@ if(BUILD_TESTING OR BUILD_SRC_TESTS) ## #5138 only reproduces when using the static library. list(REMOVE_ITEM tdb_bins test-5138.tdb) - add_executable(test-5138.tdb test-5138) + add_executable(test-5138.tdb test-5138.cc) target_link_libraries(test-5138.tdb ${LIBTOKUDB}_static z ${LIBTOKUPORTABILITY}_static ${CMAKE_THREAD_LIBS_INIT} ${EXTRA_SYSTEM_LIBS}) add_space_separated_property(TARGET test-5138.tdb COMPILE_FLAGS -fvisibility=hidden) add_ydb_test(test-5138.tdb) - + add_ydb_test(rollback-inconsistency.tdb) foreach(bin ${tdb_bins}) get_filename_component(base ${bin} NAME_WE) - add_executable(${base}.tdb ${base}) + add_executable(${base}.tdb ${base}.cc) # Some of the symbols in util may not be exported properly by # libtokudb.so. # We link the test with util directly so that the test code itself can use diff --git a/storage/tokudb/PerconaFT/src/tests/rollback-inconsistency.cc b/storage/tokudb/PerconaFT/src/tests/rollback-inconsistency.cc new file mode 100644 index 00000000000..2fc05b23f0d --- /dev/null +++ b/storage/tokudb/PerconaFT/src/tests/rollback-inconsistency.cc @@ -0,0 +1,161 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +#ident "$Id$" +/*====== +This file is part of PerconaFT. + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. + +---------------------------------------- + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License, version 3, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#include "test.h" + +// insert enough rows with a child txn and then force an eviction to verify the rollback +// log node is in valid state. +// the test fails without the fix (and of course passes with it). +// The test basically simulates the test script of George's. + + +static void +populate_table(int start, int end, DB_TXN * parent, DB_ENV * env, DB * db) { + DB_TXN *txn = NULL; + int r = env->txn_begin(env, parent, &txn, 0); assert_zero(r); + for (int i = start; i < end; i++) { + int k = htonl(i); + char kk[4]; + char str[220]; + memset(kk, 0, sizeof kk); + memcpy(kk, &k, sizeof k); + memset(str,'a', sizeof str-1); + DBT key = { .data = kk, .size = sizeof kk }; + DBT val = { .data = str, .size = 220 }; + r = db->put(db, txn, &key, &val, 0); + assert_zero(r); + } + r = txn->commit(txn, 0); + assert_zero(r); +} + +static void +populate_and_test(DB_ENV *env, DB *db) { + int r; + DB_TXN *parent = NULL; + r = env->txn_begin(env, NULL, &parent, 0); assert_zero(r); + + populate_table(0, 128, parent, env, db); + + //we know the eviction is going to happen here and the log node of parent txn is going to be evicted + //due to the extremely low cachesize. + populate_table(128, 256, parent, env, db); + + //again eviction due to the memory pressure. 256 rows is the point when that rollback log spills out. The spilled node + //will be written back but will not be dirtied by including rollback nodes from child txn(in which case the bug is bypassed). + populate_table(256, 512, parent, env, db); + + r = parent->abort(parent); assert_zero(r); + + //try to search anything in the lost range + int k = htonl(200); + char kk[4]; + memset(kk, 0, sizeof kk); + memcpy(kk, &k, sizeof k); + DBT key = { .data = kk, .size = sizeof kk }; + DBT val; + r = db->get(db, NULL, &key, &val, 0); + assert(r==DB_NOTFOUND); + +} + +static void +run_test(void) { + int r; + DB_ENV *env = NULL; + r = db_env_create(&env, 0); + assert_zero(r); + env->set_errfile(env, stderr); + + //setting up the cachetable size 64k + uint32_t cachesize = 64*1024; + r = env->set_cachesize(env, 0, cachesize, 1); + assert_zero(r); + + //setting up the log write block size to 4k so the rollback log nodes spill in accordance with the node size + r = env->set_lg_bsize(env, 4096); + assert_zero(r); + + r = env->open(env, TOKU_TEST_FILENAME, DB_INIT_MPOOL|DB_CREATE|DB_THREAD |DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_TXN|DB_PRIVATE, S_IRWXU+S_IRWXG+S_IRWXO); + assert_zero(r); + + DB *db = NULL; + r = db_create(&db, env, 0); + assert_zero(r); + + r = db->set_pagesize(db, 4096); + assert_zero(r); + + r = db->set_readpagesize(db, 1024); + assert_zero(r); + + r = db->open(db, NULL, "test.tdb", NULL, DB_BTREE, DB_AUTO_COMMIT+DB_CREATE, S_IRWXU+S_IRWXG+S_IRWXO); + assert_zero(r); + + populate_and_test(env, db); + + r = db->close(db, 0); assert_zero(r); + + r = env->close(env, 0); assert_zero(r); +} + +int +test_main(int argc, char * const argv[]) { + int r; + + // parse_args(argc, argv); + for (int i = 1; i < argc; i++) { + char * const arg = argv[i]; + if (strcmp(arg, "-v") == 0) { + verbose++; + continue; + } + if (strcmp(arg, "-q") == 0) { + verbose = 0; + continue; + } + } + + toku_os_recursive_delete(TOKU_TEST_FILENAME); + r = toku_os_mkdir(TOKU_TEST_FILENAME, S_IRWXU+S_IRWXG+S_IRWXO); assert_zero(r); + + run_test(); + + return 0; +} + diff --git a/storage/tokudb/PerconaFT/src/tests/stat64-root-changes.cc b/storage/tokudb/PerconaFT/src/tests/stat64-root-changes.cc index 0c70b0669ad..a2b48e443cd 100644 --- a/storage/tokudb/PerconaFT/src/tests/stat64-root-changes.cc +++ b/storage/tokudb/PerconaFT/src/tests/stat64-root-changes.cc @@ -166,7 +166,7 @@ run_test (void) { DB_BTREE_STAT64 s; r = db->stat64(db, NULL, &s); CKERR(r); - assert(s.bt_nkeys == 1 && s.bt_dsize == sizeof key + sizeof val); + assert(s.bt_nkeys == 0); r = db->close(db, 0); CKERR(r); @@ -176,7 +176,7 @@ run_test (void) { r = txn->commit(txn, 0); CKERR(r); r = db->stat64(db, NULL, &s); CKERR(r); - assert(s.bt_nkeys == 1 && s.bt_dsize == sizeof key + sizeof val); + assert(s.bt_nkeys == 0); } // verify update callback overwrites the row diff --git a/storage/tokudb/PerconaFT/src/tests/test_db_rowcount.cc b/storage/tokudb/PerconaFT/src/tests/test_db_rowcount.cc new file mode 100644 index 00000000000..c3ebbd811bb --- /dev/null +++ b/storage/tokudb/PerconaFT/src/tests/test_db_rowcount.cc @@ -0,0 +1,528 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +#ident "$Id$" +/*====== +This file is part of PerconaFT. + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. + +---------------------------------------- + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License, version 3, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#include "test.h" +#include <stdio.h> + +#include <sys/stat.h> +#include <db.h> + +// Tests that the logical row counts are correct and not subject to variance +// due to normal insert/delete messages within the tree with the few exceptions +// of 1) rollback messages not yet applied; 2) inserts messages turned to +// updates on apply; and 3) missing leafentries on delete messages on apply. + +static DB_TXN* const null_txn = 0; +static const uint64_t num_records = 4*1024; + +#define CHECK_NUM_ROWS(_expected, _stats) assert(_stats.bt_ndata == _expected) + +static DB* create_db(const char* fname, DB_ENV* env) { + int r; + DB* db; + + r = db_create(&db, env, 0); + assert(r == 0); + db->set_errfile(db, stderr); + + r = db->set_pagesize(db, 8192); + assert(r == 0); + + r = db->set_readpagesize(db, 1024); + assert(r == 0); + + r = db->set_fanout(db, 4); + assert(r == 0); + + r = db->set_compression_method(db, TOKU_NO_COMPRESSION); + assert(r == 0); + + r = db->open(db, null_txn, fname, "main", DB_BTREE, DB_CREATE, + 0666); + assert(r == 0); + + return db; +} +static void add_records(DB* db, DB_TXN* txn, uint64_t start_id, uint64_t num) { + int r; + for (uint64_t i = 0, j=start_id; i < num; i++,j++) { + char key[100], val[256]; + DBT k,v; + snprintf(key, 100, "%08lu", j); + snprintf(val, 256, "%*s", 200, key); + r = + db->put( + db, + txn, + dbt_init(&k, key, 1+strlen(key)), + dbt_init(&v, val, 1+strlen(val)), + 0); + assert(r == 0); + } +} +static void delete_records( + DB* db, + DB_TXN* txn, + uint64_t start_id, + uint64_t num) { + + int r; + for (uint64_t i = 0, j=start_id; i < num; i++,j++) { + char key[100]; + DBT k; + snprintf(key, 100, "%08lu", j); + r = + db->del( + db, + txn, + dbt_init(&k, key, 1+strlen(key)), + 0); + assert(r == 0); + } +} +static void full_optimize(DB* db) { + int r; + uint64_t loops_run = 0; + + r = db->optimize(db); + assert(r == 0); + + r = db->hot_optimize(db, NULL, NULL, NULL, NULL, &loops_run); + assert(r == 0); +} +static void test_insert_commit(DB_ENV* env) { + int r; + DB* db; + DB_TXN* txn; + DB_BTREE_STAT64 stats; + + db = create_db(__FUNCTION__, env); + + r = env->txn_begin(env, null_txn, &txn, 0); + assert(r == 0); + + add_records(db, txn, 0, num_records); + + r = db->stat64(db, null_txn, &stats); + assert(r == 0); + + CHECK_NUM_ROWS(num_records, stats); + if (verbose) + printf("%s : before commit %lu rows\n", __FUNCTION__, stats.bt_ndata); + + r = txn->commit(txn, 0); + assert(r == 0); + + r = db->stat64(db, null_txn, &stats); + assert(r == 0); + + CHECK_NUM_ROWS(num_records, stats); + if (verbose) + printf("%s : after commit %lu rows\n", __FUNCTION__, stats.bt_ndata); + + db->close(db, 0); +} +static void test_insert_delete_commit(DB_ENV* env) { + int r; + DB* db; + DB_TXN* txn; + DB_BTREE_STAT64 stats; + + db = create_db(__FUNCTION__, env); + + r = env->txn_begin(env, null_txn, &txn, 0); + assert(r == 0); + + add_records(db, txn, 0, num_records); + + r = db->stat64(db, null_txn, &stats); + assert(r == 0); + + CHECK_NUM_ROWS(num_records, stats); + if (verbose) + printf("%s : before delete %lu rows\n", __FUNCTION__, stats.bt_ndata); + + delete_records(db, txn, 0, num_records); + + r = db->stat64(db, null_txn, &stats); + assert(r == 0); + + CHECK_NUM_ROWS(0, stats); + if (verbose) + printf("%s : after delete %lu rows\n", __FUNCTION__, stats.bt_ndata); + + r = txn->commit(txn, 0); + assert(r == 0); + + r = db->stat64(db, null_txn, &stats); + assert(r == 0); + + CHECK_NUM_ROWS(0, stats); + if (verbose) + printf("%s : after commit %lu rows\n", __FUNCTION__, stats.bt_ndata); + + db->close(db, 0); +} +static void test_insert_commit_delete_commit(DB_ENV* env) { + int r; + DB* db; + DB_TXN* txn; + DB_BTREE_STAT64 stats; + + db = create_db(__FUNCTION__, env); + + r = env->txn_begin(env, null_txn, &txn, 0); + assert(r == 0); + + add_records(db, txn, 0, num_records); + + r = db->stat64(db, null_txn, &stats); + assert(r == 0); + + CHECK_NUM_ROWS(num_records, stats); + if (verbose) + printf( + "%s : before insert commit %lu rows\n", + __FUNCTION__, + stats.bt_ndata); + + r = txn->commit(txn, 0); + assert(r == 0); + + r = db->stat64(db, null_txn, &stats); + assert(r == 0); + + CHECK_NUM_ROWS(num_records, stats); + if (verbose) + printf( + "%s : after insert commit %lu rows\n", + __FUNCTION__, + stats.bt_ndata); + + r = env->txn_begin(env, null_txn, &txn, 0); + assert(r == 0); + + delete_records(db, txn, 0, num_records); + + r = db->stat64(db, null_txn, &stats); + assert(r == 0); + + CHECK_NUM_ROWS(0, stats); + if (verbose) + printf("%s : after delete %lu rows\n", __FUNCTION__, stats.bt_ndata); + + r = txn->commit(txn, 0); + assert(r == 0); + + r = db->stat64(db, null_txn, &stats); + assert(r == 0); + + CHECK_NUM_ROWS(0, stats); + if (verbose) + printf( + "%s : after delete commit %lu rows\n", + __FUNCTION__, + stats.bt_ndata); + + db->close(db, 0); +} +static void test_insert_rollback(DB_ENV* env) { + int r; + DB* db; + DB_TXN* txn; + DB_BTREE_STAT64 stats; + + db = create_db(__FUNCTION__, env); + + r = env->txn_begin(env, null_txn, &txn, 0); + assert(r == 0); + + add_records(db, txn, 0, num_records); + + r = db->stat64(db, null_txn, &stats); + assert(r == 0); + + CHECK_NUM_ROWS(num_records, stats); + if (verbose) + printf("%s : before rollback %lu rows\n", __FUNCTION__, stats.bt_ndata); + + r = txn->abort(txn); + assert(r == 0); + + r = db->stat64(db, null_txn, &stats); + assert(r == 0); + + // CAN NOT TEST stats HERE AS THEY ARE SOMEWHAT NON_DETERMINISTIC UNTIL + // optimize + hot_optimize HAVE BEEN RUN DUE TO THE FACT THAT ROLLBACK + // MESSAGES ARE "IN-FLIGHT" IN THE TREE AND MUST BE APPLIED IN ORDER TO + // CORRECT THE RUNNING LOGICAL COUNT + if (verbose) + printf("%s : after rollback %lu rows\n", __FUNCTION__, stats.bt_ndata); + + full_optimize(db); + + r = db->stat64(db, null_txn, &stats); + assert(r == 0); + + CHECK_NUM_ROWS(0, stats); + if (verbose) + printf( + "%s : after rollback optimize %lu rows\n", + __FUNCTION__, + stats.bt_ndata); + + db->close(db, 0); +} +static void test_insert_delete_rollback(DB_ENV* env) { + int r; + DB* db; + DB_TXN* txn; + DB_BTREE_STAT64 stats; + + db = create_db(__FUNCTION__, env); + + r = env->txn_begin(env, null_txn, &txn, 0); + assert(r == 0); + + add_records(db, txn, 0, num_records); + + r = db->stat64(db, null_txn, &stats); + assert(r == 0); + + CHECK_NUM_ROWS(num_records, stats); + if (verbose) + printf("%s : before delete %lu rows\n", __FUNCTION__, stats.bt_ndata); + + delete_records(db, txn, 0, num_records); + + r = db->stat64(db, null_txn, &stats); + assert(r == 0); + + CHECK_NUM_ROWS(0, stats); + if (verbose) + printf("%s : after delete %lu rows\n", __FUNCTION__, stats.bt_ndata); + + r = txn->abort(txn); + assert(r == 0); + + r = db->stat64(db, null_txn, &stats); + assert(r == 0); + + CHECK_NUM_ROWS(0, stats); + if (verbose) + printf("%s : after commit %lu rows\n", __FUNCTION__, stats.bt_ndata); + + db->close(db, 0); +} +static void test_insert_commit_delete_rollback(DB_ENV* env) { + int r; + DB* db; + DB_TXN* txn; + DB_BTREE_STAT64 stats; + + db = create_db(__FUNCTION__, env); + + r = env->txn_begin(env, null_txn, &txn, 0); + assert(r == 0); + + add_records(db, txn, 0, num_records); + + r = db->stat64(db, null_txn, &stats); + assert(r == 0); + + CHECK_NUM_ROWS(num_records, stats); + if (verbose) + printf( + "%s : before insert commit %lu rows\n", + __FUNCTION__, + stats.bt_ndata); + + r = txn->commit(txn, 0); + assert(r == 0); + + r = db->stat64(db, null_txn, &stats); + assert(r == 0); + + CHECK_NUM_ROWS(num_records, stats); + if (verbose) + printf( + "%s : after insert commit %lu rows\n", + __FUNCTION__, + stats.bt_ndata); + + r = env->txn_begin(env, null_txn, &txn, 0); + assert(r == 0); + + delete_records(db, txn, 0, num_records); + + r = db->stat64(db, null_txn, &stats); + assert(r == 0); + + CHECK_NUM_ROWS(0, stats); + if (verbose) + printf("%s : after delete %lu rows\n", __FUNCTION__, stats.bt_ndata); + + r = txn->abort(txn); + assert(r == 0); + + r = db->stat64(db, null_txn, &stats); + assert(r == 0); + + // CAN NOT TEST stats HERE AS THEY ARE SOMEWHAT NON_DETERMINISTIC UNTIL + // optimize + hot_optimize HAVE BEEN RUN DUE TO THE FACT THAT ROLLBACK + // MESSAGES ARE "IN-FLIGHT" IN THE TREE AND MUST BE APPLIED IN ORDER TO + // CORRECT THE RUNNING LOGICAL COUNT + if (verbose) + printf( + "%s : after delete rollback %lu rows\n", + __FUNCTION__, + stats.bt_ndata); + + full_optimize(db); + + r = db->stat64(db, null_txn, &stats); + assert(r == 0); + + CHECK_NUM_ROWS(num_records, stats); + if (verbose) + printf( + "%s : after delete rollback optimize %lu rows\n", + __FUNCTION__, + stats.bt_ndata); + + db->close(db, 0); +} +static inline uint64_t time_in_microsec() { + struct timeval t; + gettimeofday(&t, NULL); + return t.tv_sec * (1UL * 1000 * 1000) + t.tv_usec; +} + +static int test_recount_insert_commit_progress( + uint64_t count, + uint64_t deleted, + void*) { + + if (verbose) + printf( + "%s : count[%lu] deleted[%lu]\n", + __FUNCTION__, + count, + deleted); + return 0; +} +static int test_recount_cancel_progress(uint64_t, uint64_t, void*) { + return 1; +} + +static void test_recount_insert_commit(DB_ENV* env) { + int r; + DB* db; + DB_TXN* txn; + DB_BTREE_STAT64 stats; + + db = create_db(__FUNCTION__, env); + + r = env->txn_begin(env, null_txn, &txn, 0); + assert(r == 0); + + add_records(db, txn, 0, num_records); + + r = db->stat64(db, null_txn, &stats); + assert(r == 0); + + CHECK_NUM_ROWS(num_records, stats); + if (verbose) + printf( + "%s : before commit %lu rows\n", + __FUNCTION__, + stats.bt_ndata); + + r = txn->commit(txn, 0); + assert(r == 0); + + r = db->stat64(db, null_txn, &stats); + assert(r == 0); + + CHECK_NUM_ROWS(num_records, stats); + if (verbose) + printf("%s : after commit %lu rows\n", __FUNCTION__, stats.bt_ndata); + + // test that recount counted correct # of rows + r = db->recount_rows(db, test_recount_insert_commit_progress, NULL); + assert(r == 0); + CHECK_NUM_ROWS(num_records, stats); + + // test that recount callback cancel returns + r = db->recount_rows(db, test_recount_cancel_progress, NULL); + assert(r == 1); + CHECK_NUM_ROWS(num_records, stats); + + db->close(db, 0); +} +int test_main(int UU(argc), char UU(*const argv[])) { + int r; + DB_ENV* env; + + toku_os_recursive_delete(TOKU_TEST_FILENAME); + toku_os_mkdir(TOKU_TEST_FILENAME, S_IRWXU + S_IRWXG + S_IRWXO); + + r = db_env_create(&env, 0); + assert(r == 0); + + r = + env->open( + env, + TOKU_TEST_FILENAME, + DB_INIT_MPOOL + DB_INIT_LOG + DB_INIT_TXN + DB_PRIVATE + DB_CREATE, + S_IRWXU + S_IRWXG + S_IRWXO); + assert(r == 0); + + test_insert_commit(env); + test_insert_delete_commit(env); + test_insert_commit_delete_commit(env); + test_insert_rollback(env); + test_insert_delete_rollback(env); + test_insert_commit_delete_rollback(env); + test_recount_insert_commit(env); + + r = env->close(env, 0); + assert(r == 0); + + return 0; +} diff --git a/storage/tokudb/PerconaFT/src/tests/txn_manager_handle_snapshot_atomicity.cc b/storage/tokudb/PerconaFT/src/tests/txn_manager_handle_snapshot_atomicity.cc new file mode 100644 index 00000000000..b55610757e2 --- /dev/null +++ b/storage/tokudb/PerconaFT/src/tests/txn_manager_handle_snapshot_atomicity.cc @@ -0,0 +1,217 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +#ident "$Id$" +/*====== +This file is part of PerconaFT. + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. + +---------------------------------------- + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License, version 3, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with PerconaFT. If not, see <http://www.gnu.org/licenses/>. +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +//In response to the read-commit crash bug in the sysbench, this test is created to test +//the atomicity of the txn manager when handling the child txn snapshot. +//The test is supposed to fail before the read-commit-fix. + +#include "test.h" +#include "toku_pthread.h" +#include "ydb.h" +struct test_sync { + int state; + toku_mutex_t lock; + toku_cond_t cv; +}; + +static void test_sync_init(struct test_sync *UU(sync)) { +#if TOKU_DEBUG_TXN_SYNC + sync->state = 0; + toku_mutex_init(&sync->lock, NULL); + toku_cond_init(&sync->cv, NULL); +#endif +} + +static void test_sync_destroy(struct test_sync *UU(sync)) { +#if TOKU_DEBUG_TXN_SYNC + toku_mutex_destroy(&sync->lock); + toku_cond_destroy(&sync->cv); +#endif +} + +static void test_sync_sleep(struct test_sync *UU(sync), int UU(new_state)) { +#if TOKU_DEBUG_TXN_SYNC + toku_mutex_lock(&sync->lock); + while (sync->state != new_state) { + toku_cond_wait(&sync->cv, &sync->lock); + } + toku_mutex_unlock(&sync->lock); +#endif +} + +static void test_sync_next_state(struct test_sync *UU(sync)) { +#if TOKU_DEBUG_TXN_SYNC + toku_mutex_lock(&sync->lock); + sync->state++; + toku_cond_broadcast(&sync->cv); + toku_mutex_unlock(&sync->lock); +#endif +} + + +struct start_txn_arg { + DB_ENV *env; + DB *db; + DB_TXN * parent; +}; + +static struct test_sync sync_s; + +static void test_callback(uint64_t self_tid, void * extra) { + pthread_t **p = (pthread_t **) extra; + pthread_t tid_1 = *p[0]; + pthread_t tid_2 = *p[1]; + assert(self_tid == tid_2); + printf("%s: the thread[%" PRIu64 "] is going to wait...\n", __func__, tid_1); + test_sync_next_state(&sync_s); + sleep(3); + //test_sync_sleep(&sync_s,3); + //using test_sync_sleep/test_sync_next_state pair can sync threads better, however + //after the fix, this might cause a deadlock. just simply use sleep to do a proof- + //of-concept test. + printf("%s: the thread[%" PRIu64 "] is resuming...\n", __func__, tid_1); + return; +} + +static void * start_txn2(void * extra) { + struct start_txn_arg * args = (struct start_txn_arg *) extra; + DB_ENV * env = args -> env; + DB * db = args->db; + DB_TXN * parent = args->parent; + test_sync_sleep(&sync_s, 1); + printf("start %s [thread %" PRIu64 "]\n", __func__, pthread_self()); + DB_TXN *txn; + int r = env->txn_begin(env, parent, &txn, DB_READ_COMMITTED); + assert(r == 0); + //do some random things... + DBT key, data; + dbt_init(&key, "hello", 6); + dbt_init(&data, "world", 6); + db->put(db, txn, &key, &data, 0); + db->get(db, txn, &key, &data, 0); + + r = txn->commit(txn, 0); + assert(r == 0); + printf("%s done[thread %" PRIu64 "]\n", __func__, pthread_self()); + return extra; +} + +static void * start_txn1(void * extra) { + struct start_txn_arg * args = (struct start_txn_arg *) extra; + DB_ENV * env = args -> env; + DB * db = args->db; + printf("start %s: [thread %" PRIu64 "]\n", __func__, pthread_self()); + DB_TXN *txn; + int r = env->txn_begin(env, NULL, &txn, DB_READ_COMMITTED); + assert(r == 0); + printf("%s: txn began by [thread %" PRIu64 "], will wait\n", __func__, pthread_self()); + test_sync_next_state(&sync_s); + test_sync_sleep(&sync_s,2); + printf("%s: [thread %" PRIu64 "] resumed\n", __func__, pthread_self()); + //do some random things... + DBT key, data; + dbt_init(&key, "hello", 6); + dbt_init(&data, "world", 6); + db->put(db, txn, &key, &data, 0); + db->get(db, txn, &key, &data, 0); + r = txn->commit(txn, 0); + assert(r == 0); + printf("%s: done[thread %" PRIu64 "]\n", __func__, pthread_self()); + //test_sync_next_state(&sync_s); + return extra; +} + +int test_main (int UU(argc), char * const UU(argv[])) { + int r; + toku_os_recursive_delete(TOKU_TEST_FILENAME); + r = toku_os_mkdir(TOKU_TEST_FILENAME, S_IRWXU+S_IRWXG+S_IRWXO); + assert(r == 0); + + DB_ENV *env; + r = db_env_create(&env, 0); + assert(r == 0); + + r = env->open(env, TOKU_TEST_FILENAME, DB_INIT_MPOOL|DB_CREATE|DB_THREAD |DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_TXN|DB_PRIVATE, S_IRWXU+S_IRWXG+S_IRWXO); + assert(r == 0); + + DB *db = NULL; + r = db_create(&db, env, 0); + assert(r == 0); + + r = db->open(db, NULL, "testit", NULL, DB_BTREE, DB_AUTO_COMMIT+DB_CREATE, S_IRWXU+S_IRWXG+S_IRWXO); + assert(r == 0); + + DB_TXN * parent = NULL; + r = env->txn_begin(env, 0, &parent, DB_READ_COMMITTED); + assert(r == 0); + + ZERO_STRUCT(sync_s); + test_sync_init(&sync_s); + + pthread_t tid_1 = 0; + pthread_t tid_2 = 0; + pthread_t* callback_extra[2] = {&tid_1, &tid_2}; + toku_set_test_txn_sync_callback(test_callback, callback_extra); + + struct start_txn_arg args = {env, db, parent}; + + r = pthread_create(&tid_1, NULL, start_txn1, &args); + assert(r==0); + + r= pthread_create(&tid_2, NULL, start_txn2, &args); + assert(r==0); + + void * ret; + r = pthread_join(tid_1, &ret); + assert(r == 0); + r = pthread_join(tid_2, &ret); + assert(r == 0); + + r = parent->commit(parent, 0); + assert(r ==0); + + test_sync_destroy(&sync_s); + r = db->close(db, 0); + assert(r == 0); + + r = env->close(env, 0); + assert(r == 0); + + return 0; +} + diff --git a/storage/tokudb/PerconaFT/src/ydb.cc b/storage/tokudb/PerconaFT/src/ydb.cc index 88c6c86f214..61ce5a8476e 100644 --- a/storage/tokudb/PerconaFT/src/ydb.cc +++ b/storage/tokudb/PerconaFT/src/ydb.cc @@ -3148,6 +3148,10 @@ toku_test_get_latest_lsn(DB_ENV *env) { return rval.lsn; } +void toku_set_test_txn_sync_callback(void (* cb) (uint64_t, void *), void * extra) { + set_test_txn_sync_callback(cb, extra); +} + int toku_test_get_checkpointing_user_data_status (void) { return toku_cachetable_get_checkpointing_user_data_status(); diff --git a/storage/tokudb/PerconaFT/src/ydb.h b/storage/tokudb/PerconaFT/src/ydb.h index 9d4e94c6f30..bd2902e6c6e 100644 --- a/storage/tokudb/PerconaFT/src/ydb.h +++ b/storage/tokudb/PerconaFT/src/ydb.h @@ -58,3 +58,6 @@ extern "C" uint64_t toku_test_get_latest_lsn(DB_ENV *env) __attribute__((__visib // test-only function extern "C" int toku_test_get_checkpointing_user_data_status(void) __attribute__((__visibility__("default"))); + +// test-only function +extern "C" void toku_set_test_txn_sync_callback(void (* ) (uint64_t, void *), void * extra) __attribute__((__visibility__("default"))); diff --git a/storage/tokudb/PerconaFT/src/ydb_db.cc b/storage/tokudb/PerconaFT/src/ydb_db.cc index 25b24467684..e5bd4e7d089 100644 --- a/storage/tokudb/PerconaFT/src/ydb_db.cc +++ b/storage/tokudb/PerconaFT/src/ydb_db.cc @@ -1015,6 +1015,25 @@ toku_db_verify_with_progress(DB *db, int (*progress_callback)(void *extra, float return r; } + +static int +toku_db_recount_rows(DB* db, int (*progress_callback)(uint64_t count, + uint64_t deleted, + void* progress_extra), + void* progress_extra) { + + HANDLE_PANICKED_DB(db); + int r = 0; + r = + toku_ft_recount_rows( + db->i->ft_handle, + progress_callback, + progress_extra); + + return r; +} + + int toku_setup_db_internal (DB **dbp, DB_ENV *env, uint32_t flags, FT_HANDLE ft_handle, bool is_open) { if (flags || env == NULL) return EINVAL; @@ -1098,6 +1117,7 @@ toku_db_create(DB ** db, DB_ENV * env, uint32_t flags) { USDB(dbt_pos_infty); USDB(dbt_neg_infty); USDB(get_fragmentation); + USDB(recount_rows); #undef USDB result->get_indexer = db_get_indexer; result->del = autotxn_db_del; diff --git a/storage/tokudb/PerconaFT/tools/CMakeLists.txt b/storage/tokudb/PerconaFT/tools/CMakeLists.txt index 6d17210eeb0..d3808483fea 100644 --- a/storage/tokudb/PerconaFT/tools/CMakeLists.txt +++ b/storage/tokudb/PerconaFT/tools/CMakeLists.txt @@ -2,7 +2,7 @@ set_property(DIRECTORY APPEND PROPERTY COMPILE_DEFINITIONS _GNU_SOURCE DONT_DEPR set(tools tokudb_dump tokuftdump tokuft_logprint tdb-recover ftverify ba_replay) foreach(tool ${tools}) - add_executable(${tool} ${tool}) + add_executable(${tool} ${tool}.cc) add_dependencies(${tool} install_tdb_h) target_link_libraries(${tool} ${LIBTOKUDB}_static ft_static z lzma snappy ${LIBTOKUPORTABILITY}_static ${CMAKE_THREAD_LIBS_INIT} ${EXTRA_SYSTEM_LIBS}) diff --git a/storage/tokudb/PerconaFT/util/tests/CMakeLists.txt b/storage/tokudb/PerconaFT/util/tests/CMakeLists.txt index f3768c41956..8d53dd89a27 100644 --- a/storage/tokudb/PerconaFT/util/tests/CMakeLists.txt +++ b/storage/tokudb/PerconaFT/util/tests/CMakeLists.txt @@ -6,7 +6,7 @@ if(BUILD_TESTING) endforeach(src) foreach(test ${tests}) - add_executable(${test} ${test}) + add_executable(${test} ${test}.cc) target_link_libraries(${test} util ${LIBTOKUPORTABILITY}) endforeach(test) diff --git a/storage/tokudb/ha_tokudb.cc b/storage/tokudb/ha_tokudb.cc index 5e8c3819aff..3979b360d8f 100644 --- a/storage/tokudb/ha_tokudb.cc +++ b/storage/tokudb/ha_tokudb.cc @@ -23,60 +23,18 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." -#ifdef USE_PRAGMA_IMPLEMENTATION -#pragma implementation // gcc: Class implementation -#endif - -#include <my_config.h> -extern "C" { -#include "stdint.h" -#define __STDC_FORMAT_MACROS -#include "inttypes.h" -#if defined(_WIN32) -#include "misc.h" -#endif -} - -#define MYSQL_SERVER 1 -#include "mysql_version.h" -#include "sql_table.h" -#include "handler.h" -#include "table.h" -#include "log.h" -#include "sql_class.h" -#include "sql_show.h" -#include "discover.h" - -#if (50600 <= MYSQL_VERSION_ID && MYSQL_VERSION_ID <= 50699) || (50700 <= MYSQL_VERSION_ID && MYSQL_VERSION_ID <= 50799) -#include <binlog.h> -#endif - -#include "db.h" -#include "toku_os.h" -#include "hatoku_defines.h" +#include "hatoku_hton.h" #include "hatoku_cmp.h" - -static inline uint get_key_parts(const KEY *key); - -#undef PACKAGE -#undef VERSION -#undef HAVE_DTRACE -#undef _DTRACE_VERSION - -/* We define DTRACE after mysql_priv.h in case it disabled dtrace in the main server */ -#ifdef HAVE_DTRACE -#define _DTRACE_VERSION 1 -#else -#endif - #include "tokudb_buffer.h" #include "tokudb_status.h" #include "tokudb_card.h" #include "ha_tokudb.h" -#include "hatoku_hton.h" -#include <mysql/plugin.h> -static const char *ha_tokudb_exts[] = { + +HASH TOKUDB_SHARE::_open_tables; +tokudb::thread::mutex_t TOKUDB_SHARE::_open_tables_mutex; + +static const char* ha_tokudb_exts[] = { ha_tokudb_ext, NullS }; @@ -84,10 +42,15 @@ static const char *ha_tokudb_exts[] = { // // This offset is calculated starting from AFTER the NULL bytes // -static inline uint32_t get_fixed_field_size(KEY_AND_COL_INFO* kc_info, TABLE_SHARE* table_share, uint keynr) { +static inline uint32_t get_fixed_field_size( + KEY_AND_COL_INFO* kc_info, + TABLE_SHARE* table_share, + uint keynr) { + uint offset = 0; for (uint i = 0; i < table_share->fields; i++) { - if (is_fixed_field(kc_info, i) && !bitmap_is_set(&kc_info->key_filters[keynr],i)) { + if (is_fixed_field(kc_info, i) && + !bitmap_is_set(&kc_info->key_filters[keynr], i)) { offset += kc_info->field_lengths[i]; } } @@ -95,10 +58,15 @@ static inline uint32_t get_fixed_field_size(KEY_AND_COL_INFO* kc_info, TABLE_SHA } -static inline uint32_t get_len_of_offsets(KEY_AND_COL_INFO* kc_info, TABLE_SHARE* table_share, uint keynr) { +static inline uint32_t get_len_of_offsets( + KEY_AND_COL_INFO* kc_info, + TABLE_SHARE* table_share, + uint keynr) { + uint len = 0; for (uint i = 0; i < table_share->fields; i++) { - if (is_variable_field(kc_info, i) && !bitmap_is_set(&kc_info->key_filters[keynr],i)) { + if (is_variable_field(kc_info, i) && + !bitmap_is_set(&kc_info->key_filters[keynr], i)) { len += kc_info->num_offset_bytes; } } @@ -106,32 +74,36 @@ static inline uint32_t get_len_of_offsets(KEY_AND_COL_INFO* kc_info, TABLE_SHARE } -static int allocate_key_and_col_info ( TABLE_SHARE* table_share, KEY_AND_COL_INFO* kc_info) { +static int allocate_key_and_col_info( + TABLE_SHARE* table_share, + KEY_AND_COL_INFO* kc_info) { + int error; // // initialize all of the bitmaps // for (uint i = 0; i < MAX_KEY + 1; i++) { - error = bitmap_init( - &kc_info->key_filters[i], - NULL, - table_share->fields, - false - ); + error = + bitmap_init( + &kc_info->key_filters[i], + NULL, + table_share->fields, + false); if (error) { goto exit; } } - + // // create the field lengths // - kc_info->multi_ptr = tokudb_my_multi_malloc(MYF(MY_WME+MY_ZEROFILL), - &kc_info->field_types, (uint)(table_share->fields * sizeof (uint8_t)), - &kc_info->field_lengths, (uint)(table_share->fields * sizeof (uint16_t)), - &kc_info->length_bytes, (uint)(table_share->fields * sizeof (uint8_t)), - &kc_info->blob_fields, (uint)(table_share->fields * sizeof (uint32_t)), - NullS); + kc_info->multi_ptr = tokudb::memory::multi_malloc( + MYF(MY_WME+MY_ZEROFILL), + &kc_info->field_types, (uint)(table_share->fields * sizeof (uint8_t)), + &kc_info->field_lengths, (uint)(table_share->fields * sizeof (uint16_t)), + &kc_info->length_bytes, (uint)(table_share->fields * sizeof (uint8_t)), + &kc_info->blob_fields, (uint)(table_share->fields * sizeof (uint32_t)), + NullS); if (kc_info->multi_ptr == NULL) { error = ENOMEM; goto exit; @@ -141,7 +113,7 @@ exit: for (uint i = 0; MAX_KEY + 1; i++) { bitmap_free(&kc_info->key_filters[i]); } - tokudb_my_free(kc_info->multi_ptr); + tokudb::memory::free(kc_info->multi_ptr); } return error; } @@ -150,136 +122,263 @@ static void free_key_and_col_info (KEY_AND_COL_INFO* kc_info) { for (uint i = 0; i < MAX_KEY+1; i++) { bitmap_free(&kc_info->key_filters[i]); } - + for (uint i = 0; i < MAX_KEY+1; i++) { - tokudb_my_free(kc_info->cp_info[i]); + tokudb::memory::free(kc_info->cp_info[i]); kc_info->cp_info[i] = NULL; // 3144 } - tokudb_my_free(kc_info->multi_ptr); + tokudb::memory::free(kc_info->multi_ptr); kc_info->field_types = NULL; kc_info->field_lengths = NULL; kc_info->length_bytes = NULL; kc_info->blob_fields = NULL; } -void TOKUDB_SHARE::init(void) { - use_count = 0; - thr_lock_init(&lock); - tokudb_pthread_mutex_init(&mutex, MY_MUTEX_INIT_FAST); - my_rwlock_init(&num_DBs_lock, 0); - tokudb_pthread_cond_init(&m_openclose_cond, NULL); - m_state = CLOSED; -} -void TOKUDB_SHARE::destroy(void) { - assert(m_state == CLOSED); - thr_lock_delete(&lock); - tokudb_pthread_mutex_destroy(&mutex); - rwlock_destroy(&num_DBs_lock); - tokudb_pthread_cond_destroy(&m_openclose_cond); - tokudb_my_free(rec_per_key); - rec_per_key = NULL; -} +uchar* TOKUDB_SHARE::hash_get_key( + TOKUDB_SHARE* share, + size_t* length, + TOKUDB_UNUSED(my_bool not_used)) { -// MUST have tokudb_mutex locked on input -static TOKUDB_SHARE *get_share(const char *table_name, TABLE_SHARE* table_share) { - TOKUDB_SHARE *share = NULL; + *length = share->_full_table_name.length(); + return (uchar *) share->_full_table_name.c_ptr(); +} +void TOKUDB_SHARE::hash_free_element(TOKUDB_SHARE* share) { + share->destroy(); + delete share; +} +void TOKUDB_SHARE::static_init() { + my_hash_init( + &_open_tables, + table_alias_charset, + 32, + 0, + 0, + (my_hash_get_key)hash_get_key, + (my_hash_free_key)hash_free_element, 0); +} +void TOKUDB_SHARE::static_destroy() { + my_hash_free(&_open_tables); +} +void* TOKUDB_SHARE::operator new(size_t sz) { + return tokudb::memory::malloc(sz, MYF(MY_WME|MY_ZEROFILL|MY_FAE)); +} +void TOKUDB_SHARE::operator delete(void* p) { + tokudb::memory::free(p); +} +TOKUDB_SHARE::TOKUDB_SHARE() : + _num_DBs_lock(), + _mutex() { +} +void TOKUDB_SHARE::init(const char* table_name) { + _use_count = 0; + thr_lock_init(&_thr_lock); + _state = CLOSED; + _row_delta_activity = 0; + _allow_auto_analysis = true; + + _full_table_name.append(table_name); + + String tmp_dictionary_name; + tokudb_split_dname( + table_name, + _database_name, + _table_name, + tmp_dictionary_name); +} +void TOKUDB_SHARE::destroy() { + assert_always(_use_count == 0); + assert_always( + _state == TOKUDB_SHARE::CLOSED || _state == TOKUDB_SHARE::ERROR); + thr_lock_delete(&_thr_lock); +} +TOKUDB_SHARE* TOKUDB_SHARE::get_share( + const char* table_name, + TABLE_SHARE* table_share, + THR_LOCK_DATA* data, + bool create_new) { + + _open_tables_mutex.lock(); int error = 0; uint length = (uint) strlen(table_name); - if (!(share = (TOKUDB_SHARE *) my_hash_search(&tokudb_open_tables, (uchar *) table_name, length))) { - char *tmp_name; - + TOKUDB_SHARE* share = + (TOKUDB_SHARE*)my_hash_search( + &_open_tables, + (uchar*)table_name, + length); + if (!share) { + if (create_new == false) + goto exit; // create share and fill it with all zeroes // hence, all pointers are initialized to NULL - share = (TOKUDB_SHARE *) tokudb_my_multi_malloc(MYF(MY_WME | MY_ZEROFILL), - &share, sizeof(*share), - &tmp_name, length + 1, - NullS - ); - assert(share); + share = new TOKUDB_SHARE; + assert_always(share); - share->init(); + share->init(table_name); - share->table_name_length = length; - share->table_name = tmp_name; - strmov(share->table_name, table_name); - - error = my_hash_insert(&tokudb_open_tables, (uchar *) share); + error = my_hash_insert(&_open_tables, (uchar*)share); if (error) { free_key_and_col_info(&share->kc_info); + share->destroy(); + tokudb::memory::free((uchar*)share); + share = NULL; goto exit; } } + share->addref(); + + if (data) + thr_lock_data_init(&(share->_thr_lock), data, NULL); + exit: - if (error) { - share->destroy(); - tokudb_my_free((uchar *) share); - share = NULL; - } + _open_tables_mutex.unlock(); return share; } +void TOKUDB_SHARE::drop_share(TOKUDB_SHARE* share) { + _open_tables_mutex.lock(); + my_hash_delete(&_open_tables, (uchar*)share); + _open_tables_mutex.unlock(); +} +TOKUDB_SHARE::share_state_t TOKUDB_SHARE::addref() { + lock(); + _use_count++; -static int free_share(TOKUDB_SHARE * share) { - int error, result = 0; + DBUG_PRINT("info", ("0x%p share->_use_count %u", this, _use_count)); - tokudb_pthread_mutex_lock(&share->mutex); - DBUG_PRINT("info", ("share->use_count %u", share->use_count)); - if (!--share->use_count) { - share->m_state = TOKUDB_SHARE::CLOSING; - tokudb_pthread_mutex_unlock(&share->mutex); + return _state; +} +int TOKUDB_SHARE::release() { + int error, result = 0; - // - // number of open DB's may not be equal to number of keys we have because add_index - // may have added some. So, we loop through entire array and close any non-NULL value - // It is imperative that we reset a DB to NULL once we are done with it. - // - for (uint i = 0; i < sizeof(share->key_file)/sizeof(share->key_file[0]); i++) { - if (share->key_file[i]) { - if (tokudb_debug & TOKUDB_DEBUG_OPEN) { - TOKUDB_TRACE("dbclose:%p", share->key_file[i]); - } - error = share->key_file[i]->close(share->key_file[i], 0); - assert(error == 0); + _mutex.lock(); + assert_always(_use_count != 0); + _use_count--; + DBUG_PRINT("info", ("0x%p share->_use_count %u", this, _use_count)); + if (_use_count == 0 && _state == TOKUDB_SHARE::OPENED) { + // number of open DB's may not be equal to number of keys we have + // because add_index may have added some. So, we loop through entire + // array and close any non-NULL value. It is imperative that we reset + // a DB to NULL once we are done with it. + for (uint i = 0; i < sizeof(key_file)/sizeof(key_file[0]); i++) { + if (key_file[i]) { + TOKUDB_TRACE_FOR_FLAGS( + TOKUDB_DEBUG_OPEN, + "dbclose:%p", + key_file[i]); + error = key_file[i]->close(key_file[i], 0); + assert_always(error == 0); if (error) { result = error; } - if (share->key_file[i] == share->file) - share->file = NULL; - share->key_file[i] = NULL; + if (key_file[i] == file) + file = NULL; + key_file[i] = NULL; } } - error = tokudb::close_status(&share->status_block); - assert(error == 0); + error = tokudb::metadata::close(&status_block); + assert_always(error == 0); - free_key_and_col_info(&share->kc_info); + free_key_and_col_info(&kc_info); - tokudb_pthread_mutex_lock(&tokudb_mutex); - tokudb_pthread_mutex_lock(&share->mutex); - share->m_state = TOKUDB_SHARE::CLOSED; - if (share->use_count > 0) { - tokudb_pthread_cond_broadcast(&share->m_openclose_cond); - tokudb_pthread_mutex_unlock(&share->mutex); - tokudb_pthread_mutex_unlock(&tokudb_mutex); - } else { - - my_hash_delete(&tokudb_open_tables, (uchar *) share); - - tokudb_pthread_mutex_unlock(&share->mutex); - tokudb_pthread_mutex_unlock(&tokudb_mutex); + if (_rec_per_key) { + tokudb::memory::free(_rec_per_key); + _rec_per_key = NULL; + _rec_per_keys = 0; + } - share->destroy(); - tokudb_my_free((uchar *) share); + for (uint i = 0; i < _keys; i++) { + tokudb::memory::free(_key_descriptors[i]._name); } - } else { - tokudb_pthread_mutex_unlock(&share->mutex); + tokudb::memory::free(_key_descriptors); + _keys = _max_key_parts = 0; _key_descriptors = NULL; + + _state = TOKUDB_SHARE::CLOSED; } + _mutex.unlock(); return result; } +void TOKUDB_SHARE::update_row_count( + THD* thd, + uint64_t added, + uint64_t deleted, + uint64_t updated) { + + uint64_t delta = added + deleted + updated; + lock(); + if (deleted > added && _rows < (deleted - added)) { + _rows = 0; + } else { + _rows += added - deleted; + } + _row_delta_activity += delta; + if (_row_delta_activity == (uint64_t)~0) + _row_delta_activity = 1; + + ulonglong auto_threshold = tokudb::sysvars::auto_analyze(thd); + if (delta && auto_threshold > 0 && _allow_auto_analysis) { + ulonglong pct_of_rows_changed_to_trigger; + pct_of_rows_changed_to_trigger = ((_rows * auto_threshold) / 100); + if (_row_delta_activity >= pct_of_rows_changed_to_trigger) { + char msg[200]; + snprintf( + msg, + sizeof(msg), + "TokuDB: Auto %s background analysis for %s, delta_activity " + "%llu is greater than %llu percent of %llu rows.", + tokudb::sysvars::analyze_in_background(thd) > 0 ? + "scheduling" : "running", + full_table_name(), + _row_delta_activity, + auto_threshold, + (ulonglong)(_rows)); + + // analyze_standard will unlock _mutex regardless of success/failure + int ret = analyze_standard(thd, NULL); + if (ret == 0) { + sql_print_information("%s - succeeded.", msg); + } else { + sql_print_information( + "%s - failed, likely a job already running.", + msg); + } + } + } + unlock(); +} +void TOKUDB_SHARE::set_cardinality_counts_in_table(TABLE* table) { + // if there is nothing new to report, just skip it. + if (_card_changed) { + lock(); + uint32_t next_key_part = 0; + for (uint32_t i = 0; i < table->s->keys; i++) { + bool is_unique_key = + (i == table->s->primary_key) || + (table->key_info[i].flags & HA_NOSAME); + + uint32_t num_key_parts = get_key_parts(&table->key_info[i]); + for (uint32_t j = 0; j < num_key_parts; j++) { + assert_always(next_key_part < _rec_per_keys); + ulong val = _rec_per_key[next_key_part++]; + if (is_unique_key && j == num_key_parts-1) { + val = 1; + } else { + val = + (val*tokudb::sysvars::cardinality_scale_percent)/100; + if (val == 0) + val = 1; + } + table->key_info[i].rec_per_key[j] = val; + } + } + _card_changed = false; + unlock(); + } +} #define HANDLE_INVALID_CURSOR() \ if (cursor == NULL) { \ @@ -288,7 +387,6 @@ static int free_share(TOKUDB_SHARE * share) { } const char *ha_tokudb::table_type() const { - extern const char * const tokudb_hton_name; return tokudb_hton_name; } @@ -296,7 +394,7 @@ const char *ha_tokudb::index_type(uint inx) { return "BTREE"; } -/* +/* * returns NULL terminated file extension string */ const char **ha_tokudb::bas_ext() const { @@ -315,13 +413,19 @@ static inline bool is_replace_into(THD* thd) { return thd->lex->duplicates == DUP_REPLACE; } -static inline bool do_ignore_flag_optimization(THD* thd, TABLE* table, bool opt_eligible) { +static inline bool do_ignore_flag_optimization( + THD* thd, + TABLE* table, + bool opt_eligible) { + bool do_opt = false; if (opt_eligible) { if (is_replace_into(thd) || is_insert_ignore(thd)) { - uint pk_insert_mode = get_pk_insert_mode(thd); - if ((!table->triggers && pk_insert_mode < 2) || pk_insert_mode == 0) { - if (mysql_bin_log.is_open() && thd->variables.binlog_format != BINLOG_FORMAT_STMT) { + uint pk_insert_mode = tokudb::sysvars::pk_insert_mode(thd); + if ((!table->triggers && pk_insert_mode < 2) || + pk_insert_mode == 0) { + if (mysql_bin_log.is_open() && + thd->variables.binlog_format != BINLOG_FORMAT_STMT) { do_opt = false; } else { do_opt = true; @@ -332,16 +436,6 @@ static inline bool do_ignore_flag_optimization(THD* thd, TABLE* table, bool opt_ return do_opt; } -static inline uint get_key_parts(const KEY *key) { -#if (50609 <= MYSQL_VERSION_ID && MYSQL_VERSION_ID <= 50699) || \ - (50700 <= MYSQL_VERSION_ID && MYSQL_VERSION_ID <= 50799) || \ - (100009 <= MYSQL_VERSION_ID && MYSQL_VERSION_ID <= 100099) - return key->user_defined_key_parts; -#else - return key->key_parts; -#endif -} - #if TOKU_INCLUDE_EXTENDED_KEYS static inline uint get_ext_key_parts(const KEY *key) { #if (50609 <= MYSQL_VERSION_ID && MYSQL_VERSION_ID <= 50699) || \ @@ -365,8 +459,9 @@ ulonglong ha_tokudb::table_flags() const { // ulong ha_tokudb::index_flags(uint idx, uint part, bool all_parts) const { TOKUDB_HANDLER_DBUG_ENTER(""); - assert(table_share); - ulong flags = (HA_READ_NEXT | HA_READ_PREV | HA_READ_ORDER | HA_KEYREAD_ONLY | HA_READ_RANGE); + assert_always(table_share); + ulong flags = (HA_READ_NEXT | HA_READ_PREV | HA_READ_ORDER | + HA_KEYREAD_ONLY | HA_READ_RANGE); #if defined(MARIADB_BASE_VERSION) || (50600 <= MYSQL_VERSION_ID && MYSQL_VERSION_ID <= 50699) flags |= HA_DO_INDEX_COND_PUSHDOWN; #endif @@ -434,13 +529,13 @@ static int loader_poll_fun(void *extra, float progress) { static void loader_ai_err_fun(DB *db, int i, int err, DBT *key, DBT *val, void *error_extra) { LOADER_CONTEXT context = (LOADER_CONTEXT)error_extra; - assert(context->ha); + assert_always(context->ha); context->ha->set_loader_error(err); } static void loader_dup_fun(DB *db, int i, int err, DBT *key, DBT *val, void *error_extra) { LOADER_CONTEXT context = (LOADER_CONTEXT)error_extra; - assert(context->ha); + assert_always(context->ha); context->ha->set_loader_error(err); if (err == DB_KEYEXIST) { context->ha->set_dup_value_for_pk(key); @@ -620,8 +715,7 @@ static ulonglong retrieve_auto_increment(uint16 type, uint32 offset,const uchar break; default: - DBUG_ASSERT(0); - unsigned_autoinc = 0; + assert_unreachable(); } if (signed_autoinc < 0) { @@ -632,21 +726,6 @@ static ulonglong retrieve_auto_increment(uint16 type, uint32 offset,const uchar unsigned_autoinc : (ulonglong) signed_autoinc; } -static inline bool -is_null_field( TABLE* table, Field* field, const uchar* record) { - uint null_offset; - bool ret_val; - if (!field->real_maybe_null()) { - ret_val = false; - goto exitpt; - } - null_offset = get_null_offset(table,field); - ret_val = (record[null_offset] & field->null_bit) ? true: false; - -exitpt: - return ret_val; -} - static inline ulong field_offset(Field* field, TABLE* table) { return((ulong) (field->ptr - table->record[0])); } @@ -830,7 +909,7 @@ static inline uchar* write_var_field( int2store(to_tokudb_offset_ptr,offset); break; default: - assert(false); + assert_unreachable(); break; } return to_tokudb_data + data_length; @@ -850,8 +929,7 @@ static inline uint32_t get_var_data_length( data_length = uint2korr(from_mysql); break; default: - assert(false); - break; + assert_unreachable(); } return data_length; } @@ -894,8 +972,7 @@ static inline void unpack_var_field( int2store(to_mysql, from_tokudb_data_len); break; default: - assert(false); - break; + assert_unreachable(); } // // store the data @@ -928,7 +1005,7 @@ static uchar* pack_toku_field_blob( length = uint4korr(from_mysql); break; default: - assert(false); + assert_unreachable(); } if (length > 0) { @@ -940,7 +1017,9 @@ static uchar* pack_toku_field_blob( static int create_tokudb_trx_data_instance(tokudb_trx_data** out_trx) { int error; - tokudb_trx_data* trx = (tokudb_trx_data *) tokudb_my_malloc(sizeof(*trx), MYF(MY_ZEROFILL)); + tokudb_trx_data* trx = (tokudb_trx_data *) tokudb::memory::malloc( + sizeof(*trx), + MYF(MY_ZEROFILL)); if (!trx) { error = ENOMEM; goto cleanup; @@ -1011,32 +1090,27 @@ static inline int tokudb_generate_row( void* old_ptr = dest_key->data; void* new_ptr = NULL; new_ptr = realloc(old_ptr, max_key_len); - assert(new_ptr); + assert_always(new_ptr); dest_key->data = new_ptr; dest_key->ulen = max_key_len; } buff = (uchar *)dest_key->data; - assert(buff != NULL && max_key_len > 0); - } - else { - assert(false); + assert_always(buff != NULL && max_key_len > 0); + } else { + assert_unreachable(); } - dest_key->size = pack_key_from_desc( - buff, - row_desc, - desc_size, - src_key, - src_val - ); - assert(dest_key->ulen >= dest_key->size); - if (tokudb_debug & TOKUDB_DEBUG_CHECK_KEY && !max_key_len) { + dest_key->size = pack_key_from_desc(buff, row_desc, desc_size, src_key, + src_val); + assert_always(dest_key->ulen >= dest_key->size); + if (TOKUDB_UNLIKELY(TOKUDB_DEBUG_FLAGS(TOKUDB_DEBUG_CHECK_KEY)) && + !max_key_len) { max_key_len = max_key_size_from_desc(row_desc, desc_size); max_key_len += src_key->size; } if (max_key_len) { - assert(max_key_len >= dest_key->size); + assert_always(max_key_len >= dest_key->size); } row_desc += desc_size; @@ -1045,8 +1119,7 @@ static inline int tokudb_generate_row( if (dest_val != NULL) { if (!is_key_clustering(row_desc, desc_size) || src_val->size == 0) { dest_val->size = 0; - } - else { + } else { uchar* buff = NULL; if (dest_val->flags == 0) { dest_val->ulen = 0; @@ -1059,23 +1132,21 @@ static inline int tokudb_generate_row( void* old_ptr = dest_val->data; void* new_ptr = NULL; new_ptr = realloc(old_ptr, src_val->size); - assert(new_ptr); + assert_always(new_ptr); dest_val->data = new_ptr; dest_val->ulen = src_val->size; } buff = (uchar *)dest_val->data; - assert(buff != NULL); - } - else { - assert(false); + assert_always(buff != NULL); + } else { + assert_unreachable(); } dest_val->size = pack_clustering_val_from_desc( buff, row_desc, desc_size, - src_val - ); - assert(dest_val->ulen >= dest_val->size); + src_val); + assert_always(dest_val->ulen >= dest_val->size); } } error = 0; @@ -1143,6 +1214,7 @@ ha_tokudb::ha_tokudb(handlerton * hton, TABLE_SHARE * table_arg):handler(hton, t read_key = false; added_rows = 0; deleted_rows = 0; + updated_rows = 0; last_dup_key = UINT_MAX; using_ignore = false; using_ignore_no_key = false; @@ -1224,41 +1296,42 @@ bool ha_tokudb::has_auto_increment_flag(uint* index) { static int open_status_dictionary(DB** ptr, const char* name, DB_TXN* txn) { int error; char* newname = NULL; - newname = (char *)tokudb_my_malloc( - get_max_dict_name_path_length(name), - MYF(MY_WME)); + size_t newname_len = get_max_dict_name_path_length(name); + newname = (char*)tokudb::memory::malloc(newname_len, MYF(MY_WME)); if (newname == NULL) { error = ENOMEM; goto cleanup; } - make_name(newname, name, "status"); - if (tokudb_debug & TOKUDB_DEBUG_OPEN) { - TOKUDB_TRACE("open:%s", newname); - } + make_name(newname, newname_len, name, "status"); + TOKUDB_TRACE_FOR_FLAGS(TOKUDB_DEBUG_OPEN, "open:%s", newname); - error = tokudb::open_status(db_env, ptr, newname, txn); + error = tokudb::metadata::open(db_env, ptr, newname, txn); cleanup: - tokudb_my_free(newname); + tokudb::memory::free(newname); return error; } -int ha_tokudb::open_main_dictionary(const char* name, bool is_read_only, DB_TXN* txn) { +int ha_tokudb::open_main_dictionary( + const char* name, + bool is_read_only, + DB_TXN* txn) { + int error; char* newname = NULL; + size_t newname_len = 0; uint open_flags = (is_read_only ? DB_RDONLY : 0) | DB_THREAD; - assert(share->file == NULL); - assert(share->key_file[primary_key] == NULL); - - newname = (char *)tokudb_my_malloc( - get_max_dict_name_path_length(name), - MYF(MY_WME|MY_ZEROFILL) - ); + assert_always(share->file == NULL); + assert_always(share->key_file[primary_key] == NULL); + newname_len = get_max_dict_name_path_length(name); + newname = (char*)tokudb::memory::malloc( + newname_len, + MYF(MY_WME|MY_ZEROFILL)); if (newname == NULL) { error = ENOMEM; goto exit; } - make_name(newname, name, "main"); + make_name(newname, newname_len, name, "main"); error = db_create(&share->file, db_env, 0); if (error) { @@ -1266,14 +1339,24 @@ int ha_tokudb::open_main_dictionary(const char* name, bool is_read_only, DB_TXN* } share->key_file[primary_key] = share->file; - error = share->file->open(share->file, txn, newname, NULL, DB_BTREE, open_flags, 0); + error = + share->file->open( + share->file, + txn, + newname, + NULL, + DB_BTREE, + open_flags, + 0); if (error) { goto exit; } - - if (tokudb_debug & TOKUDB_DEBUG_OPEN) { - TOKUDB_HANDLER_TRACE("open:%s:file=%p", newname, share->file); - } + + TOKUDB_HANDLER_TRACE_FOR_FLAGS( + TOKUDB_DEBUG_OPEN, + "open:%s:file=%p", + newname, + share->file); error = 0; exit: @@ -1283,34 +1366,42 @@ exit: share->file, 0 ); - assert(r==0); + assert_always(r==0); share->file = NULL; share->key_file[primary_key] = NULL; } } - tokudb_my_free(newname); + tokudb::memory::free(newname); return error; } // -// Open a secondary table, the key will be a secondary index, the data will be a primary key +// Open a secondary table, the key will be a secondary index, the data will +// be a primary key // -int ha_tokudb::open_secondary_dictionary(DB** ptr, KEY* key_info, const char* name, bool is_read_only, DB_TXN* txn) { +int ha_tokudb::open_secondary_dictionary( + DB** ptr, + KEY* key_info, + const char* name, + bool is_read_only, + DB_TXN* txn) { + int error = ENOSYS; char dict_name[MAX_DICT_NAME_LEN]; uint open_flags = (is_read_only ? DB_RDONLY : 0) | DB_THREAD; char* newname = NULL; - uint newname_len = 0; - + size_t newname_len = 0; + sprintf(dict_name, "key-%s", key_info->name); newname_len = get_max_dict_name_path_length(name); - newname = (char *)tokudb_my_malloc(newname_len, MYF(MY_WME|MY_ZEROFILL)); + newname = + (char*)tokudb::memory::malloc(newname_len, MYF(MY_WME|MY_ZEROFILL)); if (newname == NULL) { error = ENOMEM; goto cleanup; } - make_name(newname, name, dict_name); + make_name(newname, newname_len, name, dict_name); if ((error = db_create(ptr, db_env, 0))) { @@ -1319,22 +1410,25 @@ int ha_tokudb::open_secondary_dictionary(DB** ptr, KEY* key_info, const char* na } - if ((error = (*ptr)->open(*ptr, txn, newname, NULL, DB_BTREE, open_flags, 0))) { + error = (*ptr)->open(*ptr, txn, newname, NULL, DB_BTREE, open_flags, 0); + if (error) { my_errno = error; goto cleanup; } - if (tokudb_debug & TOKUDB_DEBUG_OPEN) { - TOKUDB_HANDLER_TRACE("open:%s:file=%p", newname, *ptr); - } + TOKUDB_HANDLER_TRACE_FOR_FLAGS( + TOKUDB_DEBUG_OPEN, + "open:%s:file=%p", + newname, + *ptr); cleanup: if (error) { if (*ptr) { int r = (*ptr)->close(*ptr, 0); - assert(r==0); + assert_always(r==0); *ptr = NULL; } } - tokudb_my_free(newname); + tokudb::memory::free(newname); return error; } @@ -1343,11 +1437,10 @@ static int initialize_col_pack_info(KEY_AND_COL_INFO* kc_info, TABLE_SHARE* tabl // // set up the cp_info // - assert(kc_info->cp_info[keynr] == NULL); - kc_info->cp_info[keynr] = (COL_PACK_INFO *)tokudb_my_malloc( - table_share->fields*sizeof(COL_PACK_INFO), - MYF(MY_WME | MY_ZEROFILL) - ); + assert_always(kc_info->cp_info[keynr] == NULL); + kc_info->cp_info[keynr] = (COL_PACK_INFO*)tokudb::memory::malloc( + table_share->fields * sizeof(COL_PACK_INFO), + MYF(MY_WME | MY_ZEROFILL)); if (kc_info->cp_info[keynr] == NULL) { error = ENOMEM; goto exit; @@ -1396,12 +1489,18 @@ exit: // reset the kc_info state at keynr static void reset_key_and_col_info(KEY_AND_COL_INFO *kc_info, uint keynr) { bitmap_clear_all(&kc_info->key_filters[keynr]); - tokudb_my_free(kc_info->cp_info[keynr]); + tokudb::memory::free(kc_info->cp_info[keynr]); kc_info->cp_info[keynr] = NULL; kc_info->mcp_info[keynr] = (MULTI_COL_PACK_INFO) { 0, 0 }; } -static int initialize_key_and_col_info(TABLE_SHARE* table_share, TABLE* table, KEY_AND_COL_INFO* kc_info, uint hidden_primary_key, uint primary_key) { +static int initialize_key_and_col_info( + TABLE_SHARE* table_share, + TABLE* table, + KEY_AND_COL_INFO* kc_info, + uint hidden_primary_key, + uint primary_key) { + int error = 0; uint32_t curr_blob_field_index = 0; uint32_t max_var_bytes = 0; @@ -1420,7 +1519,7 @@ static int initialize_key_and_col_info(TABLE_SHARE* table_share, TABLE* table, K case toku_type_fixbinary: case toku_type_fixstring: pack_length = field->pack_length(); - assert(pack_length < 1<<16); + assert_always(pack_length < 1<<16); kc_info->field_types[i] = KEY_AND_COL_INFO::TOKUDB_FIXED_FIELD; kc_info->field_lengths[i] = (uint16_t)pack_length; kc_info->length_bytes[i] = 0; @@ -1436,11 +1535,12 @@ static int initialize_key_and_col_info(TABLE_SHARE* table_share, TABLE* table, K case toku_type_varbinary: kc_info->field_types[i] = KEY_AND_COL_INFO::TOKUDB_VARIABLE_FIELD; kc_info->field_lengths[i] = 0; - kc_info->length_bytes[i] = (uchar)((Field_varstring *)field)->length_bytes; + kc_info->length_bytes[i] = + (uchar)((Field_varstring*)field)->length_bytes; max_var_bytes += field->field_length; break; default: - assert(false); + assert_unreachable(); } } kc_info->num_blobs = curr_blob_field_index; @@ -1452,48 +1552,44 @@ static int initialize_key_and_col_info(TABLE_SHARE* table_share, TABLE* table, K // if (max_var_bytes < 256) { kc_info->num_offset_bytes = 1; - } - else { + } else { kc_info->num_offset_bytes = 2; } - for (uint i = 0; i < table_share->keys + tokudb_test(hidden_primary_key); i++) { + for (uint i = 0; + i < table_share->keys + tokudb_test(hidden_primary_key); + i++) { // // do the cluster/primary key filtering calculations // - if (! (i==primary_key && hidden_primary_key) ){ - if ( i == primary_key ) { + if (!(i==primary_key && hidden_primary_key)) { + if (i == primary_key) { set_key_filter( &kc_info->key_filters[primary_key], &table_share->key_info[primary_key], table, - true - ); - } - else { + true); + } else { set_key_filter( &kc_info->key_filters[i], &table_share->key_info[i], table, - true - ); + true); if (!hidden_primary_key) { set_key_filter( &kc_info->key_filters[i], &table_share->key_info[primary_key], table, - true - ); + true); } } } if (i == primary_key || key_is_clustering(&table_share->key_info[i])) { - error = initialize_col_pack_info(kc_info,table_share,i); + error = initialize_col_pack_info(kc_info, table_share, i); if (error) { goto exit; } } - } exit: return error; @@ -1544,8 +1640,6 @@ int ha_tokudb::initialize_share(const char* name, int mode) { if (error) { goto exit; } } - DBUG_PRINT("info", ("share->use_count %u", share->use_count)); - share->m_initialize_count++; error = get_status(txn); if (error) { @@ -1574,43 +1668,63 @@ int ha_tokudb::initialize_share(const char* name, int mode) { goto exit; #endif - error = initialize_key_and_col_info( - table_share, - table, - &share->kc_info, - hidden_primary_key, - primary_key - ); + error = + initialize_key_and_col_info( + table_share, + table, + &share->kc_info, + hidden_primary_key, + primary_key); if (error) { goto exit; } - + error = open_main_dictionary(name, mode == O_RDONLY, txn); - if (error) { goto exit; } + if (error) { + goto exit; + } share->has_unique_keys = false; + share->_keys = table_share->keys; + share->_max_key_parts = table_share->key_parts; + share->_key_descriptors = + (TOKUDB_SHARE::key_descriptor_t*)tokudb::memory::malloc( + sizeof(TOKUDB_SHARE::key_descriptor_t) * share->_keys, + MYF(MY_ZEROFILL)); + /* Open other keys; These are part of the share structure */ for (uint i = 0; i < table_share->keys; i++) { + share->_key_descriptors[i]._parts = get_key_parts(&table_share->key_info[i]); + if (i == primary_key) { + share->_key_descriptors[i]._is_unique = true; + share->_key_descriptors[i]._name = tokudb::memory::strdup("primary", 0); + } else { + share->_key_descriptors[i]._is_unique = false; + share->_key_descriptors[i]._name = + tokudb::memory::strdup(table_share->key_info[i].name, 0); + } + if (table_share->key_info[i].flags & HA_NOSAME) { + share->_key_descriptors[i]._is_unique = true; share->has_unique_keys = true; } if (i != primary_key) { - error = open_secondary_dictionary( - &share->key_file[i], - &table_share->key_info[i], - name, - mode == O_RDONLY, - txn - ); + error = + open_secondary_dictionary( + &share->key_file[i], + &table_share->key_info[i], + name, + mode == O_RDONLY, + txn); if (error) { goto exit; } } } - share->replace_into_fast = can_replace_into_be_fast( - table_share, - &share->kc_info, - primary_key - ); - + share->replace_into_fast = + can_replace_into_be_fast( + table_share, + &share->kc_info, + primary_key); + share->pk_has_string = false; if (!hidden_primary_key) { // @@ -1640,9 +1754,8 @@ int ha_tokudb::initialize_share(const char* name, int mode) { // estimate_num_rows should not fail under normal conditions // if (error == 0) { - share->rows = num_rows; - } - else { + share->set_row_count(num_rows, true); + } else { goto exit; } // @@ -1655,8 +1768,7 @@ int ha_tokudb::initialize_share(const char* name, int mode) { if (may_table_be_empty(txn)) { share->try_table_lock = true; - } - else { + } else { share->try_table_lock = false; } @@ -1665,12 +1777,22 @@ int ha_tokudb::initialize_share(const char* name, int mode) { init_hidden_prim_key_info(txn); // initialize cardinality info from the status dictionary - share->n_rec_per_key = tokudb::compute_total_key_parts(table_share); - share->rec_per_key = (uint64_t *) tokudb_my_realloc(share->rec_per_key, share->n_rec_per_key * sizeof (uint64_t), MYF(MY_FAE + MY_ALLOW_ZERO_PTR)); - error = tokudb::get_card_from_status(share->status_block, txn, share->n_rec_per_key, share->rec_per_key); - if (error) { - for (uint i = 0; i < share->n_rec_per_key; i++) - share->rec_per_key[i] = 0; + { + uint32_t rec_per_keys = tokudb::compute_total_key_parts(table_share); + uint64_t* rec_per_key = + (uint64_t*)tokudb::memory::malloc( + rec_per_keys * sizeof(uint64_t), + MYF(MY_FAE)); + error = + tokudb::get_card_from_status( + share->status_block, + txn, + rec_per_keys, + rec_per_key); + if (error) { + memset(rec_per_key, 0, sizeof(ulonglong) * rec_per_keys); + } + share->init_cardinality_counts(rec_per_keys, rec_per_key); } error = 0; @@ -1720,7 +1842,8 @@ int ha_tokudb::open(const char *name, int mode, uint test_if_locked) { // the "+ 1" is for the first byte that states +/- infinity // multiply everything by 2 to account for clustered keys having a key and primary key together max_key_length = 2*(table_share->max_key_length + MAX_REF_PARTS * 3 + sizeof(uchar)); - alloc_ptr = tokudb_my_multi_malloc(MYF(MY_WME), + alloc_ptr = tokudb::memory::multi_malloc( + MYF(MY_WME), &key_buff, max_key_length, &key_buff2, max_key_length, &key_buff3, max_key_length, @@ -1730,81 +1853,80 @@ int ha_tokudb::open(const char *name, int mode, uint test_if_locked) { &primary_key_buff, (hidden_primary_key ? 0 : max_key_length), &fixed_cols_for_query, table_share->fields*sizeof(uint32_t), &var_cols_for_query, table_share->fields*sizeof(uint32_t), - NullS - ); + NullS); if (alloc_ptr == NULL) { ret_val = 1; goto exit; } - size_range_query_buff = get_tokudb_read_buf_size(thd); - range_query_buff = (uchar *)tokudb_my_malloc(size_range_query_buff, MYF(MY_WME)); + size_range_query_buff = tokudb::sysvars::read_buf_size(thd); + range_query_buff = + (uchar*)tokudb::memory::malloc(size_range_query_buff, MYF(MY_WME)); if (range_query_buff == NULL) { ret_val = 1; goto exit; } - alloced_rec_buff_length = table_share->rec_buff_length + table_share->fields; - rec_buff = (uchar *) tokudb_my_malloc(alloced_rec_buff_length, MYF(MY_WME)); + alloced_rec_buff_length = table_share->rec_buff_length + + table_share->fields; + rec_buff = (uchar *) tokudb::memory::malloc( + alloced_rec_buff_length, + MYF(MY_WME)); if (rec_buff == NULL) { ret_val = 1; goto exit; } alloced_update_rec_buff_length = alloced_rec_buff_length; - rec_update_buff = (uchar *) tokudb_my_malloc(alloced_update_rec_buff_length, MYF(MY_WME)); + rec_update_buff = (uchar*)tokudb::memory::malloc( + alloced_update_rec_buff_length, + MYF(MY_WME)); if (rec_update_buff == NULL) { ret_val = 1; goto exit; } // lookup or create share - tokudb_pthread_mutex_lock(&tokudb_mutex); - share = get_share(name, table_share); - assert(share); - - thr_lock_data_init(&share->lock, &lock, NULL); + share = TOKUDB_SHARE::get_share(name, table_share, &lock, true); + assert_always(share); - tokudb_pthread_mutex_lock(&share->mutex); - tokudb_pthread_mutex_unlock(&tokudb_mutex); - share->use_count++; - while (share->m_state == TOKUDB_SHARE::OPENING || share->m_state == TOKUDB_SHARE::CLOSING) { - tokudb_pthread_cond_wait(&share->m_openclose_cond, &share->mutex); - } - if (share->m_state == TOKUDB_SHARE::CLOSED) { - share->m_state = TOKUDB_SHARE::OPENING; - tokudb_pthread_mutex_unlock(&share->mutex); + if (share->state() != TOKUDB_SHARE::OPENED) { + // means we're responsible for the transition to OPENED, ERROR or CLOSED ret_val = allocate_key_and_col_info(table_share, &share->kc_info); if (ret_val == 0) { ret_val = initialize_share(name, mode); } - tokudb_pthread_mutex_lock(&share->mutex); if (ret_val == 0) { - share->m_state = TOKUDB_SHARE::OPENED; + share->set_state(TOKUDB_SHARE::OPENED); } else { - share->m_state = TOKUDB_SHARE::ERROR; - share->m_error = ret_val; + share->set_state(TOKUDB_SHARE::ERROR); } - tokudb_pthread_cond_broadcast(&share->m_openclose_cond); + share->unlock(); + } else { + // got an already OPENED instance + share->unlock(); } - if (share->m_state == TOKUDB_SHARE::ERROR) { - ret_val = share->m_error; - tokudb_pthread_mutex_unlock(&share->mutex); - free_share(share); + + if (share->state() == TOKUDB_SHARE::ERROR) { + share->release(); goto exit; - } else { - assert(share->m_state == TOKUDB_SHARE::OPENED); - tokudb_pthread_mutex_unlock(&share->mutex); } + assert_always(share->state() == TOKUDB_SHARE::OPENED); + ref_length = share->ref_length; // If second open - - if (tokudb_debug & TOKUDB_DEBUG_OPEN) { - TOKUDB_HANDLER_TRACE("tokudbopen:%p:share=%p:file=%p:table=%p:table->s=%p:%d", - this, share, share->file, table, table->s, share->use_count); - } + + TOKUDB_HANDLER_TRACE_FOR_FLAGS( + TOKUDB_DEBUG_OPEN, + "tokudbopen:%p:share=%p:file=%p:table=%p:table->s=%p:%d", + this, + share, + share->file, + table, + table->s, + share->use_count()); key_read = false; stats.block_size = 1<<20; // QQQ Tokudb DB block size @@ -1813,13 +1935,13 @@ int ha_tokudb::open(const char *name, int mode, uint test_if_locked) { exit: if (ret_val) { - tokudb_my_free(range_query_buff); + tokudb::memory::free(range_query_buff); range_query_buff = NULL; - tokudb_my_free(alloc_ptr); + tokudb::memory::free(alloc_ptr); alloc_ptr = NULL; - tokudb_my_free(rec_buff); + tokudb::memory::free(rec_buff); rec_buff = NULL; - tokudb_my_free(rec_update_buff); + tokudb::memory::free(rec_update_buff); rec_update_buff = NULL; if (error) { @@ -1982,7 +2104,7 @@ int ha_tokudb::write_frm_data(DB* db, DB_TXN* txn, const char* frm_name) { error = 0; cleanup: - tokudb_my_free(frm_data); + tokudb::memory::free(frm_data); TOKUDB_HANDLER_DBUG_RETURN(error); } @@ -1993,8 +2115,8 @@ int ha_tokudb::remove_frm_data(DB *db, DB_TXN *txn) { static int smart_dbt_callback_verify_frm (DBT const *key, DBT const *row, void *context) { DBT* stored_frm = (DBT *)context; stored_frm->size = row->size; - stored_frm->data = (uchar *)tokudb_my_malloc(row->size, MYF(MY_WME)); - assert(stored_frm->data); + stored_frm->data = (uchar *)tokudb::memory::malloc(row->size, MYF(MY_WME)); + assert_always(stored_frm->data); memcpy(stored_frm->data, row->data, row->size); return 0; } @@ -2046,8 +2168,8 @@ int ha_tokudb::verify_frm_data(const char* frm_name, DB_TXN* txn) { error = 0; cleanup: - tokudb_my_free(mysql_frm_data); - tokudb_my_free(stored_frm.data); + tokudb::memory::free(mysql_frm_data); + tokudb::memory::free(stored_frm.data); TOKUDB_HANDLER_DBUG_RETURN(error); } @@ -2083,7 +2205,7 @@ int ha_tokudb::write_auto_inc_create(DB* db, ulonglong val, DB_TXN* txn){ // // Closes a handle to a table. // -int ha_tokudb::close(void) { +int ha_tokudb::close() { TOKUDB_HANDLER_DBUG_ENTER(""); int r = __close(); TOKUDB_HANDLER_DBUG_RETURN(r); @@ -2091,13 +2213,12 @@ int ha_tokudb::close(void) { int ha_tokudb::__close() { TOKUDB_HANDLER_DBUG_ENTER(""); - if (tokudb_debug & TOKUDB_DEBUG_OPEN) - TOKUDB_HANDLER_TRACE("close:%p", this); - tokudb_my_free(rec_buff); - tokudb_my_free(rec_update_buff); - tokudb_my_free(blob_buff); - tokudb_my_free(alloc_ptr); - tokudb_my_free(range_query_buff); + TOKUDB_HANDLER_TRACE_FOR_FLAGS(TOKUDB_DEBUG_OPEN, "close:%p", this); + tokudb::memory::free(rec_buff); + tokudb::memory::free(rec_update_buff); + tokudb::memory::free(blob_buff); + tokudb::memory::free(alloc_ptr); + tokudb::memory::free(range_query_buff); for (uint32_t i = 0; i < sizeof(mult_key_dbt_array)/sizeof(mult_key_dbt_array[0]); i++) { toku_dbt_array_destroy(&mult_key_dbt_array[i]); } @@ -2108,7 +2229,7 @@ int ha_tokudb::__close() { rec_update_buff = NULL; alloc_ptr = NULL; ha_tokudb::reset(); - int retval = free_share(share); + int retval = share->release(); TOKUDB_HANDLER_DBUG_RETURN(retval); } @@ -2120,8 +2241,11 @@ int ha_tokudb::__close() { // bool ha_tokudb::fix_rec_buff_for_blob(ulong length) { if (!rec_buff || (length > alloced_rec_buff_length)) { - uchar *newptr; - if (!(newptr = (uchar *) tokudb_my_realloc((void *) rec_buff, length, MYF(MY_ALLOW_ZERO_PTR)))) + uchar* newptr = (uchar*)tokudb::memory::realloc( + (void*)rec_buff, + length, + MYF(MY_ALLOW_ZERO_PTR)); + if (!newptr) return 1; rec_buff = newptr; alloced_rec_buff_length = length; @@ -2137,8 +2261,11 @@ bool ha_tokudb::fix_rec_buff_for_blob(ulong length) { // bool ha_tokudb::fix_rec_update_buff_for_blob(ulong length) { if (!rec_update_buff || (length > alloced_update_rec_buff_length)) { - uchar *newptr; - if (!(newptr = (uchar *) tokudb_my_realloc((void *) rec_update_buff, length, MYF(MY_ALLOW_ZERO_PTR)))) + uchar* newptr = (uchar*)tokudb::memory::realloc( + (void*)rec_update_buff, + length, + MYF(MY_ALLOW_ZERO_PTR)); + if (!newptr) return 1; rec_update_buff= newptr; alloced_update_rec_buff_length = length; @@ -2273,9 +2400,11 @@ int ha_tokudb::unpack_blobs( // // assert that num_bytes > 0 iff share->num_blobs > 0 // - assert( !((share->kc_info.num_blobs == 0) && (num_bytes > 0)) ); + assert_always( !((share->kc_info.num_blobs == 0) && (num_bytes > 0)) ); if (num_bytes > num_blob_bytes) { - ptr = (uchar *)tokudb_my_realloc((void *)blob_buff, num_bytes, MYF(MY_ALLOW_ZERO_PTR)); + ptr = (uchar*)tokudb::memory::realloc( + (void*)blob_buff, num_bytes, + MYF(MY_ALLOW_ZERO_PTR)); if (ptr == NULL) { error = ENOMEM; goto exit; @@ -2390,8 +2519,7 @@ int ha_tokudb::unpack_row( data_end_offset = uint2korr(var_field_offset_ptr); break; default: - assert(false); - break; + assert_unreachable(); } unpack_var_field( record + field_offset(field, table), @@ -2513,7 +2641,7 @@ uint32_t ha_tokudb::place_key_into_mysql_buff( // // HOPEFULLY TEMPORARY // - assert(table->s->db_low_byte_first); + assert_always(table->s->db_low_byte_first); #endif pos = unpack_toku_key_field( record + field_offset(key_part->field, table), @@ -2585,7 +2713,7 @@ uint32_t ha_tokudb::place_key_into_dbt_buff( // // HOPEFULLY TEMPORARY // - assert(table->s->db_low_byte_first); + assert_always(table->s->db_low_byte_first); #endif // // accessing field_offset(key_part->field) instead off key_part->offset @@ -2782,7 +2910,7 @@ DBT *ha_tokudb::pack_key( offset = 1; // Data is at key_ptr+1 } #if !defined(MARIADB_BASE_VERSION) - assert(table->s->db_low_byte_first); + assert_always(table->s->db_low_byte_first); #endif buff = pack_key_toku_key_field( buff, @@ -2838,7 +2966,7 @@ DBT *ha_tokudb::pack_ext_key( for (; key_part != end && (int) key_length > 0; key_part++) { // if the SK part is part of the PK, then append it to the list. if (key_part->field->part_of_key.is_set(primary_key)) { - assert(pk_next < pk_parts); + assert_always(pk_next < pk_parts); pk_info[pk_next].key_ptr = key_ptr; pk_info[pk_next].key_part = key_part; pk_next++; @@ -2855,7 +2983,7 @@ DBT *ha_tokudb::pack_ext_key( offset = 1; // Data is at key_ptr+1 } #if !defined(MARIADB_BASE_VERSION) - assert(table->s->db_low_byte_first); + assert_always(table->s->db_low_byte_first); #endif buff = pack_key_toku_key_field( buff, @@ -2869,22 +2997,33 @@ DBT *ha_tokudb::pack_ext_key( } if (key_length > 0) { - assert(key_part == end); + assert_always(key_part == end); end = key_info->key_part + get_ext_key_parts(key_info); // pack PK in order of PK key parts - for (uint pk_index = 0; key_part != end && (int) key_length > 0 && pk_index < pk_parts; pk_index++) { + for (uint pk_index = 0; + key_part != end && (int) key_length > 0 && pk_index < pk_parts; + pk_index++) { uint i; for (i = 0; i < pk_next; i++) { - if (pk_info[i].key_part->fieldnr == pk_key_info->key_part[pk_index].fieldnr) + if (pk_info[i].key_part->fieldnr == + pk_key_info->key_part[pk_index].fieldnr) break; } if (i < pk_next) { const uchar *this_key_ptr = pk_info[i].key_ptr; KEY_PART_INFO *this_key_part = pk_info[i].key_part; - buff = pack_key_toku_key_field(buff, (uchar *) this_key_ptr, this_key_part->field, this_key_part->length); + buff = pack_key_toku_key_field( + buff, + (uchar*)this_key_ptr, + this_key_part->field, + this_key_part->length); } else { - buff = pack_key_toku_key_field(buff, (uchar *) key_ptr, key_part->field, key_part->length); + buff = pack_key_toku_key_field( + buff, + (uchar*)key_ptr, + key_part->field, + key_part->length); key_ptr += key_part->store_length; key_length -= key_part->store_length; key_part++; @@ -2907,18 +3046,22 @@ void ha_tokudb::init_hidden_prim_key_info(DB_TXN *txn) { if (!(share->status & STATUS_PRIMARY_KEY_INIT)) { int error = 0; DBC* c = NULL; - error = share->key_file[primary_key]->cursor(share->key_file[primary_key], txn, &c, 0); - assert(error == 0); + error = share->key_file[primary_key]->cursor( + share->key_file[primary_key], + txn, + &c, + 0); + assert_always(error == 0); DBT key,val; memset(&key, 0, sizeof(key)); memset(&val, 0, sizeof(val)); error = c->c_get(c, &key, &val, DB_LAST); if (error == 0) { - assert(key.size == TOKUDB_HIDDEN_PRIMARY_KEY_LENGTH); + assert_always(key.size == TOKUDB_HIDDEN_PRIMARY_KEY_LENGTH); share->auto_ident = hpk_char_to_num((uchar *)key.data); } error = c->c_close(c); - assert(error == 0); + assert_always(error == 0); share->status |= STATUS_PRIMARY_KEY_INIT; } TOKUDB_HANDLER_DBUG_VOID_RETURN; @@ -2939,11 +3082,11 @@ int ha_tokudb::get_status(DB_TXN* txn) { // open status.tokudb // if (!share->status_block) { - error = open_status_dictionary( - &share->status_block, - share->table_name, - txn - ); + error = + open_status_dictionary( + &share->status_block, + share->full_table_name(), + txn); if (error) { goto cleanup; } @@ -2958,7 +3101,7 @@ int ha_tokudb::get_status(DB_TXN* txn) { key.size = sizeof(curr_key); value.flags = DB_DBT_USERMEM; - assert(share->status_block); + assert_always(share->status_block); // // get version // @@ -3048,7 +3191,7 @@ cleanup: */ ha_rows ha_tokudb::estimate_rows_upper_bound() { TOKUDB_HANDLER_DBUG_ENTER(""); - DBUG_RETURN(share->rows + HA_TOKUDB_EXTRA_ROWS); + DBUG_RETURN(share->row_count() + HA_TOKUDB_EXTRA_ROWS); } // @@ -3112,8 +3255,8 @@ bool ha_tokudb::may_table_be_empty(DB_TXN *txn) { DBC* tmp_cursor = NULL; DB_TXN* tmp_txn = NULL; - const int empty_scan = THDVAR(ha_thd(), empty_scan); - if (empty_scan == TOKUDB_EMPTY_SCAN_DISABLED) + const int empty_scan = tokudb::sysvars::empty_scan(ha_thd()); + if (empty_scan == tokudb::sysvars::TOKUDB_EMPTY_SCAN_DISABLED) goto cleanup; if (txn == NULL) { @@ -3128,7 +3271,7 @@ bool ha_tokudb::may_table_be_empty(DB_TXN *txn) { if (error) goto cleanup; tmp_cursor->c_set_check_interrupt_callback(tmp_cursor, tokudb_killed_thd_callback, ha_thd()); - if (empty_scan == TOKUDB_EMPTY_SCAN_LR) + if (empty_scan == tokudb::sysvars::TOKUDB_EMPTY_SCAN_LR) error = tmp_cursor->c_getf_next(tmp_cursor, 0, smart_dbt_do_nothing, NULL); else error = tmp_cursor->c_getf_prev(tmp_cursor, 0, smart_dbt_do_nothing, NULL); @@ -3142,7 +3285,7 @@ bool ha_tokudb::may_table_be_empty(DB_TXN *txn) { cleanup: if (tmp_cursor) { int r = tmp_cursor->c_close(tmp_cursor); - assert(r == 0); + assert_always(r == 0); tmp_cursor = NULL; } if (tmp_txn) { @@ -3165,22 +3308,23 @@ void ha_tokudb::start_bulk_insert(ha_rows rows) { ai_metadata_update_required = false; abort_loader = false; - rw_rdlock(&share->num_DBs_lock); + share->_num_DBs_lock.lock_read(); uint curr_num_DBs = table->s->keys + tokudb_test(hidden_primary_key); num_DBs_locked_in_bulk = true; lock_count = 0; if ((rows == 0 || rows > 1) && share->try_table_lock) { - if (get_prelock_empty(thd) && may_table_be_empty(transaction) && transaction != NULL) { + if (tokudb::sysvars::prelock_empty(thd) && + may_table_be_empty(transaction) && + transaction != NULL) { if (using_ignore || is_insert_ignore(thd) || thd->lex->duplicates != DUP_ERROR) { acquire_table_lock(transaction, lock_write); - } - else { + } else { mult_dbt_flags[primary_key] = 0; if (!thd_test_options(thd, OPTION_RELAXED_UNIQUE_CHECKS) && !hidden_primary_key) { mult_put_flags[primary_key] = DB_NOOVERWRITE; } - uint32_t loader_flags = (get_load_save_space(thd)) ? + uint32_t loader_flags = (tokudb::sysvars::load_save_space(thd)) ? LOADER_COMPRESS_INTERMEDIATES : 0; int error = db_env->create_loader( @@ -3195,7 +3339,7 @@ void ha_tokudb::start_bulk_insert(ha_rows rows) { loader_flags ); if (error) { - assert(loader == NULL); + assert_always(loader == NULL); goto exit_try_table_lock; } @@ -3203,18 +3347,18 @@ void ha_tokudb::start_bulk_insert(ha_rows rows) { lc.ha = this; error = loader->set_poll_function(loader, loader_poll_fun, &lc); - assert(!error); + assert_always(!error); error = loader->set_error_callback(loader, loader_dup_fun, &lc); - assert(!error); + assert_always(!error); trx->stmt_progress.using_loader = true; } } exit_try_table_lock: - tokudb_pthread_mutex_lock(&share->mutex); + share->lock(); share->try_table_lock = false; - tokudb_pthread_mutex_unlock(&share->mutex); + share->unlock(); } TOKUDB_HANDLER_DBUG_VOID_RETURN; } @@ -3231,9 +3375,9 @@ int ha_tokudb::end_bulk_insert(bool abort) { tokudb_trx_data* trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton); bool using_loader = (loader != NULL); if (ai_metadata_update_required) { - tokudb_pthread_mutex_lock(&share->mutex); + share->lock(); error = update_max_auto_inc(share->status_block, share->last_auto_increment); - tokudb_pthread_mutex_unlock(&share->mutex); + share->unlock(); if (error) { goto cleanup; } } delay_updating_ai_metadata = false; @@ -3284,7 +3428,7 @@ int ha_tokudb::end_bulk_insert(bool abort) { cleanup: if (num_DBs_locked_in_bulk) { - rw_unlock(&share->num_DBs_lock); + share->_num_DBs_lock.unlock(); } num_DBs_locked_in_bulk = false; lock_count = 0; @@ -3379,10 +3523,10 @@ int ha_tokudb::is_index_unique(bool* is_unique, DB_TXN* txn, DB* db, KEY* key_in cnt++; if ((cnt % 10000) == 0) { sprintf( - status_msg, - "Verifying index uniqueness: Checked %llu of %llu rows in key-%s.", - (long long unsigned) cnt, - share->rows, + status_msg, + "Verifying index uniqueness: Checked %llu of %llu rows in key-%s.", + (long long unsigned) cnt, + share->row_count(), key_info->name); thd_proc_info(thd, status_msg); if (thd_killed(thd)) { @@ -3466,7 +3610,7 @@ int ha_tokudb::is_val_unique(bool* is_unique, uchar* record, KEY* key_info, uint cleanup: if (tmp_cursor) { int r = tmp_cursor->c_close(tmp_cursor); - assert(r==0); + assert_always(r==0); tmp_cursor = NULL; } return error; @@ -3474,21 +3618,25 @@ cleanup: static void maybe_do_unique_checks_delay(THD *thd) { if (thd->slave_thread) { - uint64_t delay_ms = THDVAR(thd, rpl_unique_checks_delay); + uint64_t delay_ms = tokudb::sysvars::rpl_unique_checks_delay(thd); if (delay_ms) usleep(delay_ms * 1000); } } static bool need_read_only(THD *thd) { - return opt_readonly || !THDVAR(thd, rpl_check_readonly); -} + return opt_readonly || !tokudb::sysvars::rpl_check_readonly(thd); +} static bool do_unique_checks(THD *thd, bool do_rpl_event) { - if (do_rpl_event && thd->slave_thread && need_read_only(thd) && !THDVAR(thd, rpl_unique_checks)) + if (do_rpl_event && + thd->slave_thread && + need_read_only(thd) && + !tokudb::sysvars::rpl_unique_checks(thd)) { return false; - else + } else { return !thd_test_options(thd, OPTION_RELAXED_UNIQUE_CHECKS); + } } int ha_tokudb::do_uniqueness_checks(uchar* record, DB_TXN* txn, THD* thd) { @@ -3551,10 +3699,10 @@ void ha_tokudb::test_row_packing(uchar* record, DBT* pk_key, DBT* pk_val) { // //use for testing the packing of keys // - tmp_pk_key_data = (uchar *)tokudb_my_malloc(pk_key->size, MYF(MY_WME)); - assert(tmp_pk_key_data); - tmp_pk_val_data = (uchar *)tokudb_my_malloc(pk_val->size, MYF(MY_WME)); - assert(tmp_pk_val_data); + tmp_pk_key_data = (uchar*)tokudb::memory::malloc(pk_key->size, MYF(MY_WME)); + assert_always(tmp_pk_key_data); + tmp_pk_val_data = (uchar*)tokudb::memory::malloc(pk_val->size, MYF(MY_WME)); + assert_always(tmp_pk_val_data); memcpy(tmp_pk_key_data, pk_key->data, pk_key->size); memcpy(tmp_pk_val_data, pk_val->data, pk_val->size); tmp_pk_key.data = tmp_pk_key_data; @@ -3587,19 +3735,21 @@ void ha_tokudb::test_row_packing(uchar* record, DBT* pk_key, DBT* pk_val) { &tmp_pk_key, &tmp_pk_val ); - assert(tmp_num_bytes == key.size); + assert_always(tmp_num_bytes == key.size); cmp = memcmp(key_buff3,key_buff2,tmp_num_bytes); - assert(cmp == 0); + assert_always(cmp == 0); // // test key packing of clustering keys // if (key_is_clustering(&table->key_info[keynr])) { error = pack_row(&row, (const uchar *) record, keynr); - assert(error == 0); + assert_always(error == 0); uchar* tmp_buff = NULL; - tmp_buff = (uchar *)tokudb_my_malloc(alloced_rec_buff_length,MYF(MY_WME)); - assert(tmp_buff); + tmp_buff = (uchar*)tokudb::memory::malloc( + alloced_rec_buff_length, + MYF(MY_WME)); + assert_always(tmp_buff); row_desc = (uchar *)share->key_file[keynr]->descriptor->dbt.data; row_desc += (*(uint32_t *)row_desc); row_desc += (*(uint32_t *)row_desc); @@ -3611,10 +3761,10 @@ void ha_tokudb::test_row_packing(uchar* record, DBT* pk_key, DBT* pk_val) { desc_size, &tmp_pk_val ); - assert(tmp_num_bytes == row.size); + assert_always(tmp_num_bytes == row.size); cmp = memcmp(tmp_buff,rec_buff,tmp_num_bytes); - assert(cmp == 0); - tokudb_my_free(tmp_buff); + assert_always(cmp == 0); + tokudb::memory::free(tmp_buff); } } @@ -3622,12 +3772,12 @@ void ha_tokudb::test_row_packing(uchar* record, DBT* pk_key, DBT* pk_val) { // copy stuff back out // error = pack_row(pk_val, (const uchar *) record, primary_key); - assert(pk_val->size == tmp_pk_val.size); + assert_always(pk_val->size == tmp_pk_val.size); cmp = memcmp(pk_val->data, tmp_pk_val_data, pk_val->size); - assert( cmp == 0); + assert_always( cmp == 0); - tokudb_my_free(tmp_pk_key_data); - tokudb_my_free(tmp_pk_val_data); + tokudb::memory::free(tmp_pk_key_data); + tokudb::memory::free(tmp_pk_val_data); } // set the put flags for the main dictionary @@ -3674,7 +3824,7 @@ void ha_tokudb::set_main_dict_put_flags(THD* thd, bool opt_eligible, uint32_t* p int ha_tokudb::insert_row_to_main_dictionary(uchar* record, DBT* pk_key, DBT* pk_val, DB_TXN* txn) { int error = 0; uint curr_num_DBs = table->s->keys + tokudb_test(hidden_primary_key); - assert(curr_num_DBs == 1); + assert_always(curr_num_DBs == 1); uint32_t put_flags = mult_put_flags[primary_key]; THD *thd = ha_thd(); @@ -3806,33 +3956,36 @@ int ha_tokudb::write_row(uchar * record) { // of the auto inc field. // if (share->has_auto_inc && record == table->record[0]) { - tokudb_pthread_mutex_lock(&share->mutex); + share->lock(); ulonglong curr_auto_inc = retrieve_auto_increment( - table->field[share->ai_field_index]->key_type(), field_offset(table->field[share->ai_field_index], table), record); + table->field[share->ai_field_index]->key_type(), + field_offset(table->field[share->ai_field_index], table), + record); if (curr_auto_inc > share->last_auto_increment) { share->last_auto_increment = curr_auto_inc; if (delay_updating_ai_metadata) { ai_metadata_update_required = true; - } - else { - update_max_auto_inc(share->status_block, share->last_auto_increment); + } else { + update_max_auto_inc( + share->status_block, + share->last_auto_increment); } } - tokudb_pthread_mutex_unlock(&share->mutex); + share->unlock(); } // // grab reader lock on numDBs_lock // if (!num_DBs_locked_in_bulk) { - rw_rdlock(&share->num_DBs_lock); + share->_num_DBs_lock.lock_read(); num_DBs_locked = true; } else { lock_count++; if (lock_count >= 2000) { - rw_unlock(&share->num_DBs_lock); - rw_rdlock(&share->num_DBs_lock); + share->_num_DBs_lock.unlock(); + share->_num_DBs_lock.lock_read(); lock_count = 0; } } @@ -3862,10 +4015,8 @@ int ha_tokudb::write_row(uchar * record) { } } txn = create_sub_trans ? sub_trans : transaction; - if (tokudb_debug & TOKUDB_DEBUG_TXN) { - TOKUDB_HANDLER_TRACE("txn %p", txn); - } - if (tokudb_debug & TOKUDB_DEBUG_CHECK_KEY) { + TOKUDB_HANDLER_TRACE_FOR_FLAGS(TOKUDB_DEBUG_TXN, "txn %p", txn); + if (TOKUDB_UNLIKELY(TOKUDB_DEBUG_FLAGS(TOKUDB_DEBUG_CHECK_KEY))) { test_row_packing(record,&prim_key,&row); } if (loader) { @@ -3874,8 +4025,7 @@ int ha_tokudb::write_row(uchar * record) { abort_loader = true; goto cleanup; } - } - else { + } else { error = do_uniqueness_checks(record, txn, thd); if (error) { // for #4633 @@ -3888,8 +4038,7 @@ int ha_tokudb::write_row(uchar * record) { // was found and this is a duplicate key, // so we set last_dup_key last_dup_key = primary_key; - } - else if (r != DB_NOTFOUND) { + } else if (r != DB_NOTFOUND) { // if some other error is returned, return that to the user. error = r; } @@ -3899,8 +4048,7 @@ int ha_tokudb::write_row(uchar * record) { if (curr_num_DBs == 1) { error = insert_row_to_main_dictionary(record,&prim_key, &row, txn); if (error) { goto cleanup; } - } - else { + } else { error = insert_rows_to_dictionaries_mult(&prim_key, &row, txn, thd); if (error) { goto cleanup; } } @@ -3918,7 +4066,7 @@ int ha_tokudb::write_row(uchar * record) { } cleanup: if (num_DBs_locked) { - rw_unlock(&share->num_DBs_lock); + share->_num_DBs_lock.unlock(); } if (error == DB_KEYEXIST) { error = HA_ERR_FOUND_DUPP_KEY; @@ -3989,7 +4137,7 @@ int ha_tokudb::update_row(const uchar * old_row, uchar * new_row) { // of the auto inc field. // if (share->has_auto_inc && new_row == table->record[0]) { - tokudb_pthread_mutex_lock(&share->mutex); + share->lock(); ulonglong curr_auto_inc = retrieve_auto_increment( table->field[share->ai_field_index]->key_type(), field_offset(table->field[share->ai_field_index], table), @@ -4001,7 +4149,7 @@ int ha_tokudb::update_row(const uchar * old_row, uchar * new_row) { share->last_auto_increment = curr_auto_inc; } } - tokudb_pthread_mutex_unlock(&share->mutex); + share->unlock(); } // @@ -4009,7 +4157,7 @@ int ha_tokudb::update_row(const uchar * old_row, uchar * new_row) { // bool num_DBs_locked = false; if (!num_DBs_locked_in_bulk) { - rw_rdlock(&share->num_DBs_lock); + share->_num_DBs_lock.lock_read(); num_DBs_locked = true; } curr_num_DBs = share->num_DBs; @@ -4100,6 +4248,7 @@ int ha_tokudb::update_row(const uchar * old_row, uchar * new_row) { last_dup_key = primary_key; } else if (!error) { + updated_rows++; trx->stmt_progress.updated++; track_progress(thd); } @@ -4107,7 +4256,7 @@ int ha_tokudb::update_row(const uchar * old_row, uchar * new_row) { cleanup: if (num_DBs_locked) { - rw_unlock(&share->num_DBs_lock); + share->_num_DBs_lock.unlock(); } if (error == DB_KEYEXIST) { error = HA_ERR_FOUND_DUPP_KEY; @@ -4150,7 +4299,7 @@ int ha_tokudb::delete_row(const uchar * record) { // bool num_DBs_locked = false; if (!num_DBs_locked_in_bulk) { - rw_rdlock(&share->num_DBs_lock); + share->_num_DBs_lock.lock_read(); num_DBs_locked = true; } curr_num_DBs = share->num_DBs; @@ -4166,33 +4315,36 @@ int ha_tokudb::delete_row(const uchar * record) { goto cleanup; } - if (tokudb_debug & TOKUDB_DEBUG_TXN) { - TOKUDB_HANDLER_TRACE("all %p stmt %p sub_sp_level %p transaction %p", trx->all, trx->stmt, trx->sub_sp_level, transaction); - } + TOKUDB_HANDLER_TRACE_FOR_FLAGS( + TOKUDB_DEBUG_TXN, + "all %p stmt %p sub_sp_level %p transaction %p", + trx->all, + trx->stmt, + trx->sub_sp_level, + transaction); - error = db_env->del_multiple( - db_env, - share->key_file[primary_key], - transaction, - &prim_key, - &row, - curr_num_DBs, - share->key_file, - mult_key_dbt_array, - mult_del_flags - ); + error = + db_env->del_multiple( + db_env, + share->key_file[primary_key], + transaction, + &prim_key, + &row, + curr_num_DBs, + share->key_file, + mult_key_dbt_array, + mult_del_flags); if (error) { DBUG_PRINT("error", ("Got error %d", error)); - } - else { + } else { deleted_rows++; trx->stmt_progress.deleted++; track_progress(thd); } cleanup: if (num_DBs_locked) { - rw_unlock(&share->num_DBs_lock); + share->_num_DBs_lock.unlock(); } TOKUDB_HANDLER_DBUG_RETURN(error); } @@ -4309,7 +4461,7 @@ static bool tokudb_do_bulk_fetch(THD *thd) { case SQLCOM_INSERT_SELECT: case SQLCOM_REPLACE_SELECT: case SQLCOM_DELETE: - return THDVAR(thd, bulk_fetch) != 0; + return tokudb::sysvars::bulk_fetch(thd) != 0; default: return false; } @@ -4361,7 +4513,7 @@ cleanup: // if (cursor) { int r = cursor->c_close(cursor); - assert(r==0); + assert_always(r==0); cursor = NULL; remove_from_trx_handler_list(); } @@ -4404,7 +4556,7 @@ int ha_tokudb::index_init(uint keynr, bool sorted) { if (cursor) { DBUG_PRINT("note", ("Closing active cursor")); int r = cursor->c_close(cursor); - assert(r==0); + assert_always(r==0); remove_from_trx_handler_list(); } active_index = keynr; @@ -4430,10 +4582,12 @@ int ha_tokudb::index_init(uint keynr, bool sorted) { if (use_write_locks) { cursor_flags |= DB_RMW; } - if (get_disable_prefetching(thd)) { + if (tokudb::sysvars::disable_prefetching(thd)) { cursor_flags |= DBC_DISABLE_PREFETCHING; } - if ((error = share->key_file[keynr]->cursor(share->key_file[keynr], transaction, &cursor, cursor_flags))) { + if ((error = share->key_file[keynr]->cursor(share->key_file[keynr], + transaction, &cursor, + cursor_flags))) { if (error == TOKUDB_MVCC_DICTIONARY_TOO_NEW) { error = HA_ERR_TABLE_DEF_CHANGED; my_error(ER_TABLE_DEF_CHANGED, MYF(0)); @@ -4478,7 +4632,7 @@ int ha_tokudb::index_end() { if (cursor) { DBUG_PRINT("enter", ("table: '%s'", table_share->table_name.str)); int r = cursor->c_close(cursor); - assert(r==0); + assert_always(r==0); cursor = NULL; remove_from_trx_handler_list(); last_cursor_error = 0; @@ -4546,7 +4700,7 @@ void ha_tokudb::extract_hidden_primary_key(uint keynr, DBT const *found_key) { int ha_tokudb::read_row_callback (uchar * buf, uint keynr, DBT const *row, DBT const *found_key) { - assert(keynr == primary_key); + assert_always(keynr == primary_key); return unpack_row(buf, row,found_key, keynr); } @@ -4703,7 +4857,7 @@ cleanup: // -// According to InnoDB handlerton: Positions an index cursor to the index +// According to InnoDB handlerton: Positions an index cursor to the index // specified in keynr. Fetches the row if any // Parameters: // [out] buf - buffer for the returned row @@ -4719,10 +4873,20 @@ cleanup: // TODO: investigate this for correctness // error otherwise // -int ha_tokudb::index_read(uchar * buf, const uchar * key, uint key_len, enum ha_rkey_function find_flag) { - TOKUDB_HANDLER_DBUG_ENTER("key %p %u:%2.2x find=%u", key, key_len, key ? key[0] : 0, find_flag); +int ha_tokudb::index_read( + uchar* buf, + const uchar* key, + uint key_len, + enum ha_rkey_function find_flag) { + + TOKUDB_HANDLER_DBUG_ENTER( + "key %p %u:%2.2x find=%u", + key, + key_len, + key ? key[0] : 0, + find_flag); invalidate_bulk_fetch(); - if (tokudb_debug & TOKUDB_DEBUG_INDEX_KEY) { + if (TOKUDB_UNLIKELY(TOKUDB_DEBUG_FLAGS(TOKUDB_DEBUG_INDEX_KEY))) { TOKUDB_DBUG_DUMP("mysql key=", key, key_len); } DBT row; @@ -4730,14 +4894,17 @@ int ha_tokudb::index_read(uchar * buf, const uchar * key, uint key_len, enum ha_ int error = 0; uint32_t flags = 0; THD* thd = ha_thd(); - tokudb_trx_data* trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton);; + tokudb_trx_data* trx = (tokudb_trx_data*)thd_get_ha_data(thd, tokudb_hton); struct smart_dbt_info info; struct index_read_info ir_info; HANDLE_INVALID_CURSOR(); - // if we locked a non-null key range and we now have a null key, then remove the bounds from the cursor - if (range_lock_grabbed && !range_lock_grabbed_null && index_key_is_null(table, tokudb_active_index, key, key_len)) { + // if we locked a non-null key range and we now have a null key, then + // remove the bounds from the cursor + if (range_lock_grabbed && + !range_lock_grabbed_null && + index_key_is_null(table, tokudb_active_index, key, key_len)) { range_lock_grabbed = range_lock_grabbed_null = false; cursor->c_remove_restriction(cursor); } @@ -4758,7 +4925,7 @@ int ha_tokudb::index_read(uchar * buf, const uchar * key, uint key_len, enum ha_ pack_key(&lookup_key, tokudb_active_index, key_buff3, key, key_len, COL_NEG_INF); DBT lookup_bound; pack_key(&lookup_bound, tokudb_active_index, key_buff4, key, key_len, COL_POS_INF); - if (tokudb_debug & TOKUDB_DEBUG_INDEX_KEY) { + if (TOKUDB_UNLIKELY(TOKUDB_DEBUG_FLAGS(TOKUDB_DEBUG_INDEX_KEY))) { TOKUDB_DBUG_DUMP("tokudb key=", lookup_key.data, lookup_key.size); } ir_info.orig_key = &lookup_key; @@ -4815,8 +4982,8 @@ int ha_tokudb::index_read(uchar * buf, const uchar * key, uint key_len, enum ha_ if (!error && !key_read && tokudb_active_index != primary_key && !key_is_clustering(&table->key_info[tokudb_active_index])) { error = read_full_row(buf); } - - if (error && (tokudb_debug & TOKUDB_DEBUG_ERROR)) { + + if (TOKUDB_UNLIKELY(error && TOKUDB_DEBUG_FLAGS(TOKUDB_DEBUG_ERROR))) { TOKUDB_HANDLER_TRACE("error:%d:%d", error, find_flag); } trx->stmt_progress.queried++; @@ -4845,7 +5012,7 @@ int ha_tokudb::read_data_from_range_query_buff(uchar* buf, bool need_val, bool d // if this is a covering index, this is all we need if (do_key_read) { - assert(!need_val); + assert_always(!need_val); extract_hidden_primary_key(tokudb_active_index, &curr_key); read_key_only(buf, tokudb_active_index, &curr_key); error = 0; @@ -4979,7 +5146,7 @@ int ha_tokudb::fill_range_query_buf( // uint32_t size_remaining = size_range_query_buff - bytes_used_in_range_query_buff; uint32_t size_needed; - uint32_t user_defined_size = get_tokudb_read_buf_size(thd); + uint32_t user_defined_size = tokudb::sysvars::read_buf_size(thd); uchar* curr_pos = NULL; if (key_to_compare) { @@ -5036,7 +5203,7 @@ int ha_tokudb::fill_range_query_buf( size_needed = sizeof(uint32_t) + key->size; } if (size_remaining < size_needed) { - range_query_buff = (uchar *)tokudb_my_realloc( + range_query_buff = (uchar *)tokudb::memory::realloc( (void *)range_query_buff, bytes_used_in_range_query_buff+size_needed, MYF(MY_WME) @@ -5136,7 +5303,7 @@ int ha_tokudb::fill_range_query_buf( } bytes_used_in_range_query_buff = curr_pos - range_query_buff; - assert(bytes_used_in_range_query_buff <= size_range_query_buff); + assert_always(bytes_used_in_range_query_buff <= size_range_query_buff); // // now determine if we should continue with the bulk fetch @@ -5153,7 +5320,7 @@ int ha_tokudb::fill_range_query_buf( // row fetch upper bound. if (bulk_fetch_iteration < HA_TOKU_BULK_FETCH_ITERATION_MAX) { uint64_t row_fetch_upper_bound = 1LLU << bulk_fetch_iteration; - assert(row_fetch_upper_bound > 0); + assert_always(row_fetch_upper_bound > 0); if (rows_fetched_using_bulk_fetch >= row_fetch_upper_bound) { error = 0; goto cleanup; @@ -5509,40 +5676,69 @@ int ha_tokudb::rnd_next(uchar * buf) { void ha_tokudb::track_progress(THD* thd) { tokudb_trx_data* trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton); if (trx) { - ulonglong num_written = trx->stmt_progress.inserted + trx->stmt_progress.updated + trx->stmt_progress.deleted; + ulonglong num_written = trx->stmt_progress.inserted + + trx->stmt_progress.updated + + trx->stmt_progress.deleted; bool update_status = - (trx->stmt_progress.queried && tokudb_read_status_frequency && (trx->stmt_progress.queried % tokudb_read_status_frequency) == 0) || - (num_written && tokudb_write_status_frequency && (num_written % tokudb_write_status_frequency) == 0); + (trx->stmt_progress.queried && + tokudb::sysvars::read_status_frequency && + (trx->stmt_progress.queried % + tokudb::sysvars::read_status_frequency) == 0) || + (num_written && tokudb::sysvars::write_status_frequency && + (num_written % tokudb::sysvars::write_status_frequency) == 0); if (update_status) { char *next_status = write_status_msg; bool first = true; int r; if (trx->stmt_progress.queried) { - r = sprintf(next_status, "Queried about %llu row%s", trx->stmt_progress.queried, trx->stmt_progress.queried == 1 ? "" : "s"); - assert(r >= 0); + r = sprintf( + next_status, + "Queried about %llu row%s", + trx->stmt_progress.queried, + trx->stmt_progress.queried == 1 ? "" : "s"); + assert_always(r >= 0); next_status += r; first = false; } if (trx->stmt_progress.inserted) { if (trx->stmt_progress.using_loader) { - r = sprintf(next_status, "%sFetched about %llu row%s, loading data still remains", first ? "" : ", ", trx->stmt_progress.inserted, trx->stmt_progress.inserted == 1 ? "" : "s"); - } - else { - r = sprintf(next_status, "%sInserted about %llu row%s", first ? "" : ", ", trx->stmt_progress.inserted, trx->stmt_progress.inserted == 1 ? "" : "s"); + r = sprintf( + next_status, + "%sFetched about %llu row%s, loading data still remains", + first ? "" : ", ", + trx->stmt_progress.inserted, + trx->stmt_progress.inserted == 1 ? "" : "s"); + } else { + r = sprintf( + next_status, + "%sInserted about %llu row%s", + first ? "" : ", ", + trx->stmt_progress.inserted, + trx->stmt_progress.inserted == 1 ? "" : "s"); } - assert(r >= 0); + assert_always(r >= 0); next_status += r; first = false; } if (trx->stmt_progress.updated) { - r = sprintf(next_status, "%sUpdated about %llu row%s", first ? "" : ", ", trx->stmt_progress.updated, trx->stmt_progress.updated == 1 ? "" : "s"); - assert(r >= 0); + r = sprintf( + next_status, + "%sUpdated about %llu row%s", + first ? "" : ", ", + trx->stmt_progress.updated, + trx->stmt_progress.updated == 1 ? "" : "s"); + assert_always(r >= 0); next_status += r; first = false; } if (trx->stmt_progress.deleted) { - r = sprintf(next_status, "%sDeleted about %llu row%s", first ? "" : ", ", trx->stmt_progress.deleted, trx->stmt_progress.deleted == 1 ? "" : "s"); - assert(r >= 0); + r = sprintf( + next_status, + "%sDeleted about %llu row%s", + first ? "" : ", ", + trx->stmt_progress.deleted, + trx->stmt_progress.deleted == 1 ? "" : "s"); + assert_always(r >= 0); next_status += r; first = false; } @@ -5583,7 +5779,7 @@ int ha_tokudb::rnd_pos(uchar * buf, uchar * pos) { // test rpl slave by inducing a delay before the point query THD *thd = ha_thd(); if (thd->slave_thread && (in_rpl_delete_rows || in_rpl_update_rows)) { - uint64_t delay_ms = THDVAR(thd, rpl_lookup_rows_delay); + uint64_t delay_ms = tokudb::sysvars::rpl_lookup_rows_delay(thd); if (delay_ms) usleep(delay_ms * 1000); } @@ -5664,7 +5860,7 @@ int ha_tokudb::prelock_range(const key_range *start_key, const key_range *end_ke // if (cursor) { int r = cursor->c_close(cursor); - assert(r==0); + assert_always(r==0); cursor = NULL; remove_from_trx_handler_list(); } @@ -5784,7 +5980,7 @@ int ha_tokudb::info(uint flag) { DB_TXN* txn = NULL; if (flag & HA_STATUS_VARIABLE) { // Just to get optimizations right - stats.records = share->rows + share->rows_from_locked_table; + stats.records = share->row_count() + share->rows_from_locked_table; if (stats.records == 0) { stats.records++; } @@ -5795,30 +5991,35 @@ int ha_tokudb::info(uint flag) { memset(&frag_info, 0, sizeof frag_info); error = txn_begin(db_env, NULL, &txn, DB_READ_UNCOMMITTED, ha_thd()); - if (error) { goto cleanup; } + if (error) { + goto cleanup; + } // we should always have a primary key - assert(share->file != NULL); + assert_always(share->file != NULL); error = estimate_num_rows(share->file,&num_rows, txn); if (error == 0) { - share->rows = num_rows; + share->set_row_count(num_rows, false); stats.records = num_rows; if (stats.records == 0) { stats.records++; } - } - else { + } else { goto cleanup; } error = share->file->get_fragmentation(share->file, &frag_info); - if (error) { goto cleanup; } + if (error) { + goto cleanup; + } stats.delete_length = frag_info.unused_bytes; DB_BTREE_STAT64 dict_stats; error = share->file->stat64(share->file, txn, &dict_stats); - if (error) { goto cleanup; } - + if (error) { + goto cleanup; + } + stats.create_time = dict_stats.bt_create_time_sec; stats.update_time = dict_stats.bt_modify_time_sec; stats.check_time = dict_stats.bt_verify_time_sec; @@ -5828,18 +6029,24 @@ int ha_tokudb::info(uint flag) { // in this case, we have a hidden primary key, do not // want to report space taken up by the hidden primary key to the user // - uint64_t hpk_space = TOKUDB_HIDDEN_PRIMARY_KEY_LENGTH*dict_stats.bt_ndata; - stats.data_file_length = (hpk_space > stats.data_file_length) ? 0 : stats.data_file_length - hpk_space; - } - else { + uint64_t hpk_space = + TOKUDB_HIDDEN_PRIMARY_KEY_LENGTH * dict_stats.bt_ndata; + stats.data_file_length = + (hpk_space > stats.data_file_length) ? + 0 : stats.data_file_length - hpk_space; + } else { // // one infinity byte per key needs to be subtracted // uint64_t inf_byte_space = dict_stats.bt_ndata; - stats.data_file_length = (inf_byte_space > stats.data_file_length) ? 0 : stats.data_file_length - inf_byte_space; + stats.data_file_length = + (inf_byte_space > stats.data_file_length) ? + 0 : stats.data_file_length - inf_byte_space; } - stats.mean_rec_length = stats.records ? (ulong)(stats.data_file_length/stats.records) : 0; + stats.mean_rec_length = + stats.records ? + (ulong)(stats.data_file_length/stats.records) : 0; stats.index_file_length = 0; // curr_num_DBs is the number of keys we have, according // to the mysql layer. if drop index is running concurrently @@ -5858,37 +6065,42 @@ int ha_tokudb::info(uint flag) { if (i == primary_key || share->key_file[i] == NULL) { continue; } - error = share->key_file[i]->stat64( - share->key_file[i], - txn, - &dict_stats - ); - if (error) { goto cleanup; } + error = + share->key_file[i]->stat64( + share->key_file[i], + txn, + &dict_stats); + if (error) { + goto cleanup; + } stats.index_file_length += dict_stats.bt_dsize; - error = share->file->get_fragmentation( - share->file, - &frag_info - ); - if (error) { goto cleanup; } + error = + share->file->get_fragmentation( + share->file, + &frag_info); + if (error) { + goto cleanup; + } stats.delete_length += frag_info.unused_bytes; } } } if ((flag & HA_STATUS_CONST)) { - stats.max_data_file_length= 9223372036854775807ULL; - tokudb::set_card_in_key_info(table, share->n_rec_per_key, share->rec_per_key); + stats.max_data_file_length = 9223372036854775807ULL; + share->set_cardinality_counts_in_table(table); } /* Don't return key if we got an error for the internal primary key */ if (flag & HA_STATUS_ERRKEY && last_dup_key < table_share->keys) { errkey = last_dup_key; - } + } - if (flag & HA_STATUS_AUTO && table->found_next_number_field) { - THD *thd= table->in_use; - struct system_variables *variables= &thd->variables; - stats.auto_increment_value = share->last_auto_increment + variables->auto_increment_increment; + if (flag & HA_STATUS_AUTO && table->found_next_number_field) { + THD* thd = table->in_use; + struct system_variables* variables = &thd->variables; + stats.auto_increment_value = + share->last_auto_increment + variables->auto_increment_increment; } error = 0; cleanup: @@ -5935,7 +6147,7 @@ int ha_tokudb::extra(enum ha_extra_function operation) { TOKUDB_HANDLER_DBUG_RETURN(0); } -int ha_tokudb::reset(void) { +int ha_tokudb::reset() { TOKUDB_HANDLER_DBUG_ENTER(""); key_read = false; using_ignore = false; @@ -5959,14 +6171,13 @@ int ha_tokudb::acquire_table_lock (DB_TXN* trans, TABLE_LOCK_TYPE lt) { TOKUDB_HANDLER_DBUG_ENTER("%p %s", trans, lt == lock_read ? "r" : "w"); int error = ENOSYS; if (!num_DBs_locked_in_bulk) { - rw_rdlock(&share->num_DBs_lock); + share->_num_DBs_lock.lock_read(); } uint curr_num_DBs = share->num_DBs; if (lt == lock_read) { error = 0; goto cleanup; - } - else if (lt == lock_write) { + } else if (lt == lock_write) { for (uint i = 0; i < curr_num_DBs; i++) { DB* db = share->key_file[i]; error = db->pre_acquire_table_lock(db, trans); @@ -5974,11 +6185,9 @@ int ha_tokudb::acquire_table_lock (DB_TXN* trans, TABLE_LOCK_TYPE lt) { TOKUDB_HANDLER_TRACE("%d db=%p trans=%p", i, db, trans); if (error) break; } - if (tokudb_debug & TOKUDB_DEBUG_LOCK) - TOKUDB_HANDLER_TRACE("error=%d", error); + TOKUDB_HANDLER_TRACE_FOR_FLAGS(TOKUDB_DEBUG_LOCK, "error=%d", error); if (error) goto cleanup; - } - else { + } else { error = ENOSYS; goto cleanup; } @@ -5986,7 +6195,7 @@ int ha_tokudb::acquire_table_lock (DB_TXN* trans, TABLE_LOCK_TYPE lt) { error = 0; cleanup: if (!num_DBs_locked_in_bulk) { - rw_unlock(&share->num_DBs_lock); + share->_num_DBs_lock.unlock(); } TOKUDB_HANDLER_DBUG_RETURN(error); } @@ -6017,17 +6226,19 @@ int ha_tokudb::create_txn(THD* thd, tokudb_trx_data* trx) { if ((error = txn_begin(db_env, NULL, &trx->all, txn_begin_flags, thd))) { goto cleanup; } - if (tokudb_debug & TOKUDB_DEBUG_TXN) { - TOKUDB_HANDLER_TRACE("created master %p", trx->all); - } + TOKUDB_HANDLER_TRACE_FOR_FLAGS( + TOKUDB_DEBUG_TXN, + "created master %p", + trx->all); trx->sp_level = trx->all; trans_register_ha(thd, true, tokudb_hton); } DBUG_PRINT("trans", ("starting transaction stmt")); if (trx->stmt) { - if (tokudb_debug & TOKUDB_DEBUG_TXN) { - TOKUDB_HANDLER_TRACE("warning:stmt=%p", trx->stmt); - } + TOKUDB_HANDLER_TRACE_FOR_FLAGS( + TOKUDB_DEBUG_TXN, + "warning:stmt=%p", + trx->stmt); } uint32_t txn_begin_flags; if (trx->all == NULL) { @@ -6042,21 +6253,25 @@ int ha_tokudb::create_txn(THD* thd, tokudb_trx_data* trx) { if (txn_begin_flags == 0 && is_autocommit && thd_sql_command(thd) == SQLCOM_SELECT) { txn_begin_flags = DB_TXN_SNAPSHOT; } - if (is_autocommit && thd_sql_command(thd) == SQLCOM_SELECT && !thd->in_sub_stmt && lock.type <= TL_READ_NO_INSERT && !thd->lex->uses_stored_routines()) { + if (is_autocommit && thd_sql_command(thd) == SQLCOM_SELECT && + !thd->in_sub_stmt && lock.type <= TL_READ_NO_INSERT && + !thd->lex->uses_stored_routines()) { txn_begin_flags |= DB_TXN_READ_ONLY; } - } - else { + } else { txn_begin_flags = DB_INHERIT_ISOLATION; } - if ((error = txn_begin(db_env, trx->sp_level, &trx->stmt, txn_begin_flags, thd))) { + error = txn_begin(db_env, trx->sp_level, &trx->stmt, txn_begin_flags, thd); + if (error) { /* We leave the possible master transaction open */ goto cleanup; } trx->sub_sp_level = trx->stmt; - if (tokudb_debug & TOKUDB_DEBUG_TXN) { - TOKUDB_HANDLER_TRACE("created stmt %p sp_level %p", trx->sp_level, trx->stmt); - } + TOKUDB_HANDLER_TRACE_FOR_FLAGS( + TOKUDB_DEBUG_TXN, + "created stmt %p sp_level %p", + trx->sp_level, + trx->stmt); reset_stmt_progress(&trx->stmt_progress); trans_register_ha(thd, false, tokudb_hton); cleanup: @@ -6087,26 +6302,40 @@ static const char *lock_type_str(int lock_type) { // error otherwise // int ha_tokudb::external_lock(THD * thd, int lock_type) { - TOKUDB_HANDLER_DBUG_ENTER("cmd %d lock %d %s %s", thd_sql_command(thd), lock_type, lock_type_str(lock_type), share->table_name); - if (!(tokudb_debug & TOKUDB_DEBUG_ENTER) && (tokudb_debug & TOKUDB_DEBUG_LOCK)) { - TOKUDB_HANDLER_TRACE("cmd %d lock %d %s %s", thd_sql_command(thd), lock_type, lock_type_str(lock_type), share->table_name); - } - if (tokudb_debug & TOKUDB_DEBUG_LOCK) { - TOKUDB_HANDLER_TRACE("q %s", thd->query()); - } + TOKUDB_HANDLER_DBUG_ENTER( + "cmd %d lock %d %s %s", + thd_sql_command(thd), + lock_type, + lock_type_str(lock_type), + share->full_table_name()); + if (TOKUDB_UNLIKELY(!TOKUDB_DEBUG_FLAGS(TOKUDB_DEBUG_ENTER) && + TOKUDB_DEBUG_FLAGS(TOKUDB_DEBUG_LOCK))) { + TOKUDB_HANDLER_TRACE( + "cmd %d lock %d %s %s", + thd_sql_command(thd), + lock_type, + lock_type_str(lock_type), + share->full_table_name()); + } + TOKUDB_HANDLER_TRACE_FOR_FLAGS(TOKUDB_DEBUG_LOCK, "q %s", thd->query()); int error = 0; - tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton); + tokudb_trx_data* trx = (tokudb_trx_data*)thd_get_ha_data(thd, tokudb_hton); if (!trx) { error = create_tokudb_trx_data_instance(&trx); if (error) { goto cleanup; } thd_set_ha_data(thd, tokudb_hton, trx); } - if (tokudb_debug & TOKUDB_DEBUG_TXN) { - TOKUDB_HANDLER_TRACE("trx %p %p %p %p %u %u", trx->all, trx->stmt, trx->sp_level, trx->sub_sp_level, - trx->tokudb_lock_count, trx->create_lock_count); - } + TOKUDB_HANDLER_TRACE_FOR_FLAGS( + TOKUDB_DEBUG_TXN, + "trx %p %p %p %p %u %u", + trx->all, + trx->stmt, + trx->sp_level, + trx->sub_sp_level, + trx->tokudb_lock_count, + trx->create_lock_count); if (trx->all == NULL) { trx->sp_level = NULL; @@ -6126,19 +6355,11 @@ int ha_tokudb::external_lock(THD * thd, int lock_type) { } transaction = trx->sub_sp_level; trx->tokudb_lock_count++; - } - else { - tokudb_pthread_mutex_lock(&share->mutex); - // hate dealing with comparison of signed vs unsigned, so doing this - if (deleted_rows > added_rows && share->rows < (deleted_rows - added_rows)) { - share->rows = 0; - } - else { - share->rows += (added_rows - deleted_rows); - } - tokudb_pthread_mutex_unlock(&share->mutex); + } else { + share->update_row_count(thd, added_rows, deleted_rows, updated_rows); added_rows = 0; deleted_rows = 0; + updated_rows = 0; share->rows_from_locked_table = 0; if (trx->tokudb_lock_count > 0) { if (--trx->tokudb_lock_count <= trx->create_lock_count) { @@ -6160,8 +6381,7 @@ int ha_tokudb::external_lock(THD * thd, int lock_type) { } } cleanup: - if (tokudb_debug & TOKUDB_DEBUG_LOCK) - TOKUDB_HANDLER_TRACE("error=%d", error); + TOKUDB_HANDLER_TRACE_FOR_FLAGS(TOKUDB_DEBUG_LOCK, "error=%d", error); TOKUDB_HANDLER_DBUG_RETURN(error); } @@ -6170,24 +6390,32 @@ cleanup: TABLE LOCK is done. Under LOCK TABLES, each used tables will force a call to start_stmt. */ -int ha_tokudb::start_stmt(THD * thd, thr_lock_type lock_type) { - TOKUDB_HANDLER_DBUG_ENTER("cmd %d lock %d %s", thd_sql_command(thd), lock_type, share->table_name); - if (tokudb_debug & TOKUDB_DEBUG_LOCK) { - TOKUDB_HANDLER_TRACE("q %s", thd->query()); - } +int ha_tokudb::start_stmt(THD* thd, thr_lock_type lock_type) { + TOKUDB_HANDLER_DBUG_ENTER( + "cmd %d lock %d %s", + thd_sql_command(thd), + lock_type, + share->full_table_name()); + + TOKUDB_HANDLER_TRACE_FOR_FLAGS(TOKUDB_DEBUG_LOCK, "q %s", thd->query()); int error = 0; - tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton); + tokudb_trx_data* trx = (tokudb_trx_data*)thd_get_ha_data(thd, tokudb_hton); if (!trx) { error = create_tokudb_trx_data_instance(&trx); if (error) { goto cleanup; } thd_set_ha_data(thd, tokudb_hton, trx); } - if (tokudb_debug & TOKUDB_DEBUG_TXN) { - TOKUDB_HANDLER_TRACE("trx %p %p %p %p %u %u", trx->all, trx->stmt, trx->sp_level, trx->sub_sp_level, - trx->tokudb_lock_count, trx->create_lock_count); - } + TOKUDB_HANDLER_TRACE_FOR_FLAGS( + TOKUDB_DEBUG_TXN, + "trx %p %p %p %p %u %u", + trx->all, + trx->stmt, + trx->sp_level, + trx->sub_sp_level, + trx->tokudb_lock_count, + trx->create_lock_count); /* note that trx->stmt may have been already initialized as start_stmt() @@ -6200,11 +6428,11 @@ int ha_tokudb::start_stmt(THD * thd, thr_lock_type lock_type) { goto cleanup; } trx->create_lock_count = trx->tokudb_lock_count; - } - else { - if (tokudb_debug & TOKUDB_DEBUG_TXN) { - TOKUDB_HANDLER_TRACE("trx->stmt %p already existed", trx->stmt); - } + } else { + TOKUDB_HANDLER_TRACE_FOR_FLAGS( + TOKUDB_DEBUG_TXN, + "trx->stmt %p already existed", + trx->stmt); } if (added_rows > deleted_rows) { share->rows_from_locked_table = added_rows - deleted_rows; @@ -6278,27 +6506,40 @@ uint32_t ha_tokudb::get_cursor_isolation_flags(enum thr_lock_type lock_type, THD time). In the future we will probably try to remove this. */ -THR_LOCK_DATA **ha_tokudb::store_lock(THD * thd, THR_LOCK_DATA ** to, enum thr_lock_type lock_type) { - TOKUDB_HANDLER_DBUG_ENTER("lock_type=%d cmd=%d", lock_type, thd_sql_command(thd)); - if (tokudb_debug & TOKUDB_DEBUG_LOCK) { - TOKUDB_HANDLER_TRACE("lock_type=%d cmd=%d", lock_type, thd_sql_command(thd)); - } +THR_LOCK_DATA* *ha_tokudb::store_lock( + THD* thd, + THR_LOCK_DATA** to, + enum thr_lock_type lock_type) { + + TOKUDB_HANDLER_DBUG_ENTER( + "lock_type=%d cmd=%d", + lock_type, + thd_sql_command(thd)); + TOKUDB_HANDLER_TRACE_FOR_FLAGS( + TOKUDB_DEBUG_LOCK, + "lock_type=%d cmd=%d", + lock_type, + thd_sql_command(thd)); if (lock_type != TL_IGNORE && lock.type == TL_UNLOCK) { enum_sql_command sql_command = (enum_sql_command) thd_sql_command(thd); if (!thd->in_lock_tables) { - if (sql_command == SQLCOM_CREATE_INDEX && get_create_index_online(thd)) { + if (sql_command == SQLCOM_CREATE_INDEX && + tokudb::sysvars::create_index_online(thd)) { // hot indexing - rw_rdlock(&share->num_DBs_lock); + share->_num_DBs_lock.lock_read(); if (share->num_DBs == (table->s->keys + tokudb_test(hidden_primary_key))) { lock_type = TL_WRITE_ALLOW_WRITE; } - rw_unlock(&share->num_DBs_lock); - } else if ((lock_type >= TL_WRITE_CONCURRENT_INSERT && lock_type <= TL_WRITE) && - sql_command != SQLCOM_TRUNCATE && !thd_tablespace_op(thd)) { + share->_num_DBs_lock.unlock(); + } else if ((lock_type >= TL_WRITE_CONCURRENT_INSERT && + lock_type <= TL_WRITE) && + sql_command != SQLCOM_TRUNCATE && + !thd_tablespace_op(thd)) { // allow concurrent writes lock_type = TL_WRITE_ALLOW_WRITE; - } else if (sql_command == SQLCOM_OPTIMIZE && lock_type == TL_READ_NO_INSERT) { + } else if (sql_command == SQLCOM_OPTIMIZE && + lock_type == TL_READ_NO_INSERT) { // hot optimize table lock_type = TL_READ; } @@ -6306,88 +6547,130 @@ THR_LOCK_DATA **ha_tokudb::store_lock(THD * thd, THR_LOCK_DATA ** to, enum thr_l lock.type = lock_type; } *to++ = &lock; - if (tokudb_debug & TOKUDB_DEBUG_LOCK) - TOKUDB_HANDLER_TRACE("lock_type=%d", lock_type); - DBUG_RETURN(to); + TOKUDB_HANDLER_TRACE_FOR_FLAGS( + TOKUDB_DEBUG_LOCK, + "lock_type=%d", + lock_type); + TOKUDB_HANDLER_DBUG_RETURN_PTR(to); } -static toku_compression_method get_compression_method(DB *file) { +static toku_compression_method get_compression_method(DB* file) { enum toku_compression_method method; int r = file->get_compression_method(file, &method); - assert(r == 0); + assert_always(r == 0); return method; } #if TOKU_INCLUDE_ROW_TYPE_COMPRESSION -enum row_type ha_tokudb::get_row_type(void) const { +enum row_type ha_tokudb::get_row_type() const { toku_compression_method compression_method = get_compression_method(share->file); return toku_compression_method_to_row_type(compression_method); } #endif static int create_sub_table( - const char *table_name, - DBT* row_descriptor, - DB_TXN* txn, - uint32_t block_size, + const char* table_name, + DBT* row_descriptor, + DB_TXN* txn, + uint32_t block_size, uint32_t read_block_size, toku_compression_method compression_method, bool is_hot_index, - uint32_t fanout - ) -{ + uint32_t fanout) { + TOKUDB_DBUG_ENTER(""); int error; DB *file = NULL; uint32_t create_flags; - - + + error = db_create(&file, db_env, 0); if (error) { DBUG_PRINT("error", ("Got error: %d when creating table", error)); my_errno = error; goto exit; } - + if (block_size != 0) { error = file->set_pagesize(file, block_size); if (error != 0) { - DBUG_PRINT("error", ("Got error: %d when setting block size %u for table '%s'", error, block_size, table_name)); + DBUG_PRINT( + "error", + ("Got error: %d when setting block size %u for table '%s'", + error, + block_size, + table_name)); goto exit; } } if (read_block_size != 0) { error = file->set_readpagesize(file, read_block_size); if (error != 0) { - DBUG_PRINT("error", ("Got error: %d when setting read block size %u for table '%s'", error, read_block_size, table_name)); + DBUG_PRINT( + "error", + ("Got error: %d when setting read block size %u for table '%s'", + error, + read_block_size, + table_name)); goto exit; } } if (fanout != 0) { error = file->set_fanout(file, fanout); if (error != 0) { - DBUG_PRINT("error", ("Got error: %d when setting fanout %u for table '%s'", - error, fanout, table_name)); + DBUG_PRINT( + "error", + ("Got error: %d when setting fanout %u for table '%s'", + error, + fanout, + table_name)); goto exit; } } error = file->set_compression_method(file, compression_method); if (error != 0) { - DBUG_PRINT("error", ("Got error: %d when setting compression type %u for table '%s'", error, compression_method, table_name)); + DBUG_PRINT( + "error", + ("Got error: %d when setting compression type %u for table '%s'", + error, + compression_method, + table_name)); goto exit; } - create_flags = DB_THREAD | DB_CREATE | DB_EXCL | (is_hot_index ? DB_IS_HOT_INDEX : 0); - error = file->open(file, txn, table_name, NULL, DB_BTREE, create_flags, my_umask); + create_flags = + DB_THREAD | DB_CREATE | DB_EXCL | (is_hot_index ? DB_IS_HOT_INDEX : 0); + error = + file->open( + file, + txn, + table_name, + NULL, + DB_BTREE, + create_flags, + my_umask); if (error) { - DBUG_PRINT("error", ("Got error: %d when opening table '%s'", error, table_name)); + DBUG_PRINT( + "error", + ("Got error: %d when opening table '%s'", error, table_name)); goto exit; } - error = file->change_descriptor(file, txn, row_descriptor, (is_hot_index ? DB_IS_HOT_INDEX | DB_UPDATE_CMP_DESCRIPTOR : DB_UPDATE_CMP_DESCRIPTOR)); + error = + file->change_descriptor( + file, + txn, + row_descriptor, + (is_hot_index ? DB_IS_HOT_INDEX | + DB_UPDATE_CMP_DESCRIPTOR : + DB_UPDATE_CMP_DESCRIPTOR)); if (error) { - DBUG_PRINT("error", ("Got error: %d when setting row descriptor for table '%s'", error, table_name)); + DBUG_PRINT( + "error", + ("Got error: %d when setting row descriptor for table '%s'", + error, + table_name)); goto exit; } @@ -6395,7 +6678,7 @@ static int create_sub_table( exit: if (file) { int r = file->close(file, 0); - assert(r==0); + assert_always(r==0); } TOKUDB_DBUG_RETURN(error); } @@ -6413,7 +6696,8 @@ void ha_tokudb::update_create_info(HA_CREATE_INFO* create_info) { // show create table asks us to update this create_info, this makes it // so we'll always show what compression type we're using create_info->row_type = get_row_type(); - if (create_info->row_type == ROW_TYPE_TOKU_ZLIB && THDVAR(ha_thd(), hide_default_row_format) != 0) { + if (create_info->row_type == ROW_TYPE_TOKU_ZLIB && + tokudb::sysvars::hide_default_row_format(ha_thd()) != 0) { create_info->row_type = ROW_TYPE_DEFAULT; } } @@ -6477,27 +6761,43 @@ int ha_tokudb::write_key_name_to_status(DB* status_block, char* key_name, DB_TXN } // -// some tracing moved out of ha_tokudb::create, because ::create was getting cluttered +// some tracing moved out of ha_tokudb::create, because ::create was +// getting cluttered // void ha_tokudb::trace_create_table_info(const char *name, TABLE * form) { uint i; // // tracing information about what type of table we are creating // - if (tokudb_debug & TOKUDB_DEBUG_OPEN) { + if (TOKUDB_UNLIKELY(TOKUDB_DEBUG_FLAGS(TOKUDB_DEBUG_OPEN))) { for (i = 0; i < form->s->fields; i++) { Field *field = form->s->field[i]; - TOKUDB_HANDLER_TRACE("field:%d:%s:type=%d:flags=%x", i, field->field_name, field->type(), field->flags); + TOKUDB_HANDLER_TRACE( + "field:%d:%s:type=%d:flags=%x", + i, + field->field_name, + field->type(), + field->flags); } for (i = 0; i < form->s->keys; i++) { KEY *key = &form->s->key_info[i]; - TOKUDB_HANDLER_TRACE("key:%d:%s:%d", i, key->name, get_key_parts(key)); + TOKUDB_HANDLER_TRACE( + "key:%d:%s:%d", + i, + key->name, + get_key_parts(key)); uint p; for (p = 0; p < get_key_parts(key); p++) { - KEY_PART_INFO *key_part = &key->key_part[p]; - Field *field = key_part->field; - TOKUDB_HANDLER_TRACE("key:%d:%d:length=%d:%s:type=%d:flags=%x", - i, p, key_part->length, field->field_name, field->type(), field->flags); + KEY_PART_INFO* key_part = &key->key_part[p]; + Field* field = key_part->field; + TOKUDB_HANDLER_TRACE( + "key:%d:%d:length=%d:%s:type=%d:flags=%x", + i, + p, + key_part->length, + field->field_name, + field->type(), + field->flags); } } } @@ -6505,9 +6805,12 @@ void ha_tokudb::trace_create_table_info(const char *name, TABLE * form) { static uint32_t get_max_desc_size(KEY_AND_COL_INFO* kc_info, TABLE* form) { uint32_t max_row_desc_buff_size; - max_row_desc_buff_size = 2*(form->s->fields * 6)+10; // upper bound of key comparison descriptor - max_row_desc_buff_size += get_max_secondary_key_pack_desc_size(kc_info); // upper bound for sec. key part - max_row_desc_buff_size += get_max_clustering_val_pack_desc_size(form->s); // upper bound for clustering val part + // upper bound of key comparison descriptor + max_row_desc_buff_size = 2*(form->s->fields * 6)+10; + // upper bound for sec. key part + max_row_desc_buff_size += get_max_secondary_key_pack_desc_size(kc_info); + // upper bound for clustering val part + max_row_desc_buff_size += get_max_clustering_val_pack_desc_size(form->s); return max_row_desc_buff_size; } @@ -6519,9 +6822,8 @@ static uint32_t create_secondary_key_descriptor( TABLE* form, uint primary_key, uint32_t keynr, - KEY_AND_COL_INFO* kc_info - ) -{ + KEY_AND_COL_INFO* kc_info) { + uchar* ptr = NULL; ptr = buf; @@ -6560,23 +6862,25 @@ static uint32_t create_secondary_key_descriptor( // creates dictionary for secondary index, with key description key_info, all using txn // int ha_tokudb::create_secondary_dictionary( - const char* name, TABLE* form, - KEY* key_info, - DB_TXN* txn, - KEY_AND_COL_INFO* kc_info, + const char* name, + TABLE* form, + KEY* key_info, + DB_TXN* txn, + KEY_AND_COL_INFO* kc_info, uint32_t keynr, bool is_hot_index, - toku_compression_method compression_method - ) -{ + toku_compression_method compression_method) { + int error; DBT row_descriptor; uchar* row_desc_buff = NULL; char* newname = NULL; + size_t newname_len = 0; KEY* prim_key = NULL; char dict_name[MAX_DICT_NAME_LEN]; uint32_t max_row_desc_buff_size; - uint hpk= (form->s->primary_key >= MAX_KEY) ? TOKUDB_HIDDEN_PRIMARY_KEY_LENGTH : 0; + uint hpk= (form->s->primary_key >= MAX_KEY) ? + TOKUDB_HIDDEN_PRIMARY_KEY_LENGTH : 0; uint32_t block_size; uint32_t read_block_size; uint32_t fanout; @@ -6586,14 +6890,23 @@ int ha_tokudb::create_secondary_dictionary( max_row_desc_buff_size = get_max_desc_size(kc_info,form); - row_desc_buff = (uchar *)tokudb_my_malloc(max_row_desc_buff_size, MYF(MY_WME)); - if (row_desc_buff == NULL){ error = ENOMEM; goto cleanup;} + row_desc_buff = (uchar*)tokudb::memory::malloc( + max_row_desc_buff_size, + MYF(MY_WME)); + if (row_desc_buff == NULL) { + error = ENOMEM; + goto cleanup; + } - newname = (char *)tokudb_my_malloc(get_max_dict_name_path_length(name),MYF(MY_WME)); - if (newname == NULL){ error = ENOMEM; goto cleanup;} + newname_len = get_max_dict_name_path_length(name); + newname = (char*)tokudb::memory::malloc(newname_len, MYF(MY_WME)); + if (newname == NULL) { + error = ENOMEM; + goto cleanup; + } sprintf(dict_name, "key-%s", key_info->name); - make_name(newname, name, dict_name); + make_name(newname, newname_len, name, dict_name); prim_key = (hpk) ? NULL : &form->s->key_info[primary_key]; @@ -6612,20 +6925,25 @@ int ha_tokudb::create_secondary_dictionary( form, primary_key, keynr, - kc_info - ); - assert(row_descriptor.size <= max_row_desc_buff_size); + kc_info); + assert_always(row_descriptor.size <= max_row_desc_buff_size); - block_size = get_tokudb_block_size(thd); - read_block_size = get_tokudb_read_block_size(thd); - fanout = get_tokudb_fanout(thd); + block_size = tokudb::sysvars::block_size(thd); + read_block_size = tokudb::sysvars::read_block_size(thd); + fanout = tokudb::sysvars::fanout(thd); - error = create_sub_table(newname, &row_descriptor, txn, block_size, - read_block_size, compression_method, is_hot_index, - fanout); + error = create_sub_table( + newname, + &row_descriptor, + txn, + block_size, + read_block_size, + compression_method, + is_hot_index, + fanout); cleanup: - tokudb_my_free(newname); - tokudb_my_free(row_desc_buff); + tokudb::memory::free(newname); + tokudb::memory::free(row_desc_buff); return error; } @@ -6636,21 +6954,17 @@ static uint32_t create_main_key_descriptor( uint hpk, uint primary_key, TABLE* form, - KEY_AND_COL_INFO* kc_info - ) -{ + KEY_AND_COL_INFO* kc_info) { + uchar* ptr = buf; ptr += create_toku_key_descriptor( ptr, hpk, prim_key, false, - NULL - ); + NULL); - ptr += create_toku_main_key_pack_descriptor( - ptr - ); + ptr += create_toku_main_key_pack_descriptor(ptr); ptr += create_toku_clustering_val_pack_descriptor( ptr, @@ -6658,8 +6972,7 @@ static uint32_t create_main_key_descriptor( form->s, kc_info, primary_key, - false - ); + false); return ptr - buf; } @@ -6667,14 +6980,21 @@ static uint32_t create_main_key_descriptor( // create and close the main dictionarr with name of "name" using table form, all within // transaction txn. // -int ha_tokudb::create_main_dictionary(const char* name, TABLE* form, DB_TXN* txn, KEY_AND_COL_INFO* kc_info, toku_compression_method compression_method) { +int ha_tokudb::create_main_dictionary( + const char* name, + TABLE* form, + DB_TXN* txn, + KEY_AND_COL_INFO* kc_info, + toku_compression_method compression_method) { + int error; DBT row_descriptor; uchar* row_desc_buff = NULL; char* newname = NULL; + size_t newname_len = 0; KEY* prim_key = NULL; uint32_t max_row_desc_buff_size; - uint hpk= (form->s->primary_key >= MAX_KEY) ? TOKUDB_HIDDEN_PRIMARY_KEY_LENGTH : 0; + uint hpk = (form->s->primary_key >= MAX_KEY) ? TOKUDB_HIDDEN_PRIMARY_KEY_LENGTH : 0; uint32_t block_size; uint32_t read_block_size; uint32_t fanout; @@ -6683,13 +7003,22 @@ int ha_tokudb::create_main_dictionary(const char* name, TABLE* form, DB_TXN* txn memset(&row_descriptor, 0, sizeof(row_descriptor)); max_row_desc_buff_size = get_max_desc_size(kc_info, form); - row_desc_buff = (uchar *)tokudb_my_malloc(max_row_desc_buff_size, MYF(MY_WME)); - if (row_desc_buff == NULL){ error = ENOMEM; goto cleanup;} + row_desc_buff = (uchar*)tokudb::memory::malloc( + max_row_desc_buff_size, + MYF(MY_WME)); + if (row_desc_buff == NULL) { + error = ENOMEM; + goto cleanup; + } - newname = (char *)tokudb_my_malloc(get_max_dict_name_path_length(name),MYF(MY_WME)); - if (newname == NULL){ error = ENOMEM; goto cleanup;} + newname_len = get_max_dict_name_path_length(name); + newname = (char*)tokudb::memory::malloc(newname_len, MYF(MY_WME)); + if (newname == NULL) { + error = ENOMEM; + goto cleanup; + } - make_name(newname, name, "main"); + make_name(newname, newname_len, name, "main"); prim_key = (hpk) ? NULL : &form->s->key_info[primary_key]; @@ -6706,21 +7035,26 @@ int ha_tokudb::create_main_dictionary(const char* name, TABLE* form, DB_TXN* txn hpk, primary_key, form, - kc_info - ); - assert(row_descriptor.size <= max_row_desc_buff_size); + kc_info); + assert_always(row_descriptor.size <= max_row_desc_buff_size); - block_size = get_tokudb_block_size(thd); - read_block_size = get_tokudb_read_block_size(thd); - fanout = get_tokudb_fanout(thd); + block_size = tokudb::sysvars::block_size(thd); + read_block_size = tokudb::sysvars::read_block_size(thd); + fanout = tokudb::sysvars::fanout(thd); /* Create the main table that will hold the real rows */ - error = create_sub_table(newname, &row_descriptor, txn, block_size, - read_block_size, compression_method, false, - fanout); + error = create_sub_table( + newname, + &row_descriptor, + txn, + block_size, + read_block_size, + compression_method, + false, + fanout); cleanup: - tokudb_my_free(newname); - tokudb_my_free(row_desc_buff); + tokudb::memory::free(newname); + tokudb::memory::free(row_desc_buff); return error; } @@ -6734,7 +7068,11 @@ cleanup: // 0 on success // error otherwise // -int ha_tokudb::create(const char *name, TABLE * form, HA_CREATE_INFO * create_info) { +int ha_tokudb::create( + const char* name, + TABLE* form, + HA_CREATE_INFO* create_info) { + TOKUDB_HANDLER_DBUG_ENTER("%s", name); int error; @@ -6744,6 +7082,7 @@ int ha_tokudb::create(const char *name, TABLE * form, HA_CREATE_INFO * create_in DB_TXN* txn = NULL; bool do_commit = false; char* newname = NULL; + size_t newname_len = 0; KEY_AND_COL_INFO kc_info; tokudb_trx_data *trx = NULL; THD* thd = ha_thd(); @@ -6759,15 +7098,18 @@ int ha_tokudb::create(const char *name, TABLE * form, HA_CREATE_INFO * create_in #endif #if TOKU_INCLUDE_OPTION_STRUCTS - const srv_row_format_t row_format = (srv_row_format_t) form->s->option_struct->row_format; + const tokudb::sysvars::format_t row_format = + (tokudb::sysvars::row_format_t)form->s->option_struct->row_format; #else - const srv_row_format_t row_format = (create_info->used_fields & HA_CREATE_USED_ROW_FORMAT) + const tokudb::sysvars::row_format_t row_format = + (create_info->used_fields & HA_CREATE_USED_ROW_FORMAT) ? row_type_to_row_format(create_info->row_type) - : get_row_format(thd); + : tokudb::sysvars::row_format(thd); #endif - const toku_compression_method compression_method = row_format_to_toku_compression_method(row_format); + const toku_compression_method compression_method = + row_format_to_toku_compression_method(row_format); - bool create_from_engine= (create_info->table_options & HA_OPTION_CREATE_FROM_ENGINE); + bool create_from_engine = (create_info->table_options & HA_OPTION_CREATE_FROM_ENGINE); if (create_from_engine) { // table already exists, nothing to do error = 0; @@ -6792,17 +7134,23 @@ int ha_tokudb::create(const char *name, TABLE * form, HA_CREATE_INFO * create_in } } - newname = (char *)tokudb_my_malloc(get_max_dict_name_path_length(name),MYF(MY_WME)); - if (newname == NULL){ error = ENOMEM; goto cleanup;} + newname_len = get_max_dict_name_path_length(name); + newname = (char*)tokudb::memory::malloc(newname_len, MYF(MY_WME)); + if (newname == NULL) { + error = ENOMEM; + goto cleanup; + } trx = (tokudb_trx_data *) thd_get_ha_data(ha_thd(), tokudb_hton); - if (trx && trx->sub_sp_level && thd_sql_command(thd) == SQLCOM_CREATE_TABLE) { + if (trx && trx->sub_sp_level && + thd_sql_command(thd) == SQLCOM_CREATE_TABLE) { txn = trx->sub_sp_level; - } - else { + } else { do_commit = true; error = txn_begin(db_env, 0, &txn, 0, thd); - if (error) { goto cleanup; } + if (error) { + goto cleanup; + } } primary_key = form->s->primary_key; @@ -6815,45 +7163,76 @@ int ha_tokudb::create(const char *name, TABLE * form, HA_CREATE_INFO * create_in trace_create_table_info(name,form); /* Create status.tokudb and save relevant metadata */ - make_name(newname, name, "status"); + make_name(newname, newname_len, name, "status"); - error = tokudb::create_status(db_env, &status_block, newname, txn); + error = tokudb::metadata::create(db_env, &status_block, newname, txn); if (error) { goto cleanup; } version = HA_TOKU_VERSION; - error = write_to_status(status_block, hatoku_new_version,&version,sizeof(version), txn); - if (error) { goto cleanup; } + error = write_to_status( + status_block, + hatoku_new_version, + &version, + sizeof(version), + txn); + if (error) { + goto cleanup; + } capabilities = HA_TOKU_CAP; - error = write_to_status(status_block, hatoku_capabilities,&capabilities,sizeof(capabilities), txn); - if (error) { goto cleanup; } + error = write_to_status( + status_block, + hatoku_capabilities, + &capabilities, + sizeof(capabilities), + txn); + if (error) { + goto cleanup; + } - error = write_auto_inc_create(status_block, create_info->auto_increment_value, txn); - if (error) { goto cleanup; } + error = write_auto_inc_create( + status_block, + create_info->auto_increment_value, + txn); + if (error) { + goto cleanup; + } #if WITH_PARTITION_STORAGE_ENGINE if (TOKU_PARTITION_WRITE_FRM_DATA || form->part_info == NULL) { error = write_frm_data(status_block, txn, form->s->path.str); - if (error) { goto cleanup; } + if (error) { + goto cleanup; + } } #else error = write_frm_data(status_block, txn, form->s->path.str); - if (error) { goto cleanup; } + if (error) { + goto cleanup; + } #endif error = allocate_key_and_col_info(form->s, &kc_info); - if (error) { goto cleanup; } + if (error) { + goto cleanup; + } error = initialize_key_and_col_info( - form->s, + form->s, form, &kc_info, hidden_primary_key, - primary_key - ); - if (error) { goto cleanup; } + primary_key); + if (error) { + goto cleanup; + } - error = create_main_dictionary(name, form, txn, &kc_info, compression_method); + error = create_main_dictionary( + name, + form, + txn, + &kc_info, + compression_method); if (error) { goto cleanup; } @@ -6861,32 +7240,44 @@ int ha_tokudb::create(const char *name, TABLE * form, HA_CREATE_INFO * create_in for (uint i = 0; i < form->s->keys; i++) { if (i != primary_key) { - error = create_secondary_dictionary(name, form, &form->key_info[i], txn, &kc_info, i, false, compression_method); + error = create_secondary_dictionary( + name, + form, + &form->key_info[i], + txn, + &kc_info, + i, + false, + compression_method); if (error) { goto cleanup; } - error = write_key_name_to_status(status_block, form->s->key_info[i].name, txn); - if (error) { goto cleanup; } + error = write_key_name_to_status( + status_block, + form->s->key_info[i].name, + txn); + if (error) { + goto cleanup; + } } } error = 0; cleanup: if (status_block != NULL) { - int r = tokudb::close_status(&status_block); - assert(r==0); + int r = tokudb::metadata::close(&status_block); + assert_always(r==0); } free_key_and_col_info(&kc_info); if (do_commit && txn) { if (error) { abort_txn(txn); - } - else { + } else { commit_txn(txn,0); } } - tokudb_my_free(newname); + tokudb::memory::free(newname); TOKUDB_HANDLER_DBUG_RETURN(error); } @@ -6909,27 +7300,36 @@ int ha_tokudb::discard_or_import_tablespace(my_bool discard) { // is_key specifies if it is a secondary index (and hence a "key-" needs to be prepended) or // if it is not a secondary index // -int ha_tokudb::delete_or_rename_dictionary( const char* from_name, const char* to_name, const char* secondary_name, bool is_key, DB_TXN* txn, bool is_delete) { +int ha_tokudb::delete_or_rename_dictionary( + const char* from_name, + const char* to_name, + const char* secondary_name, + bool is_key, + DB_TXN* txn, + bool is_delete) { + int error; char dict_name[MAX_DICT_NAME_LEN]; char* new_from_name = NULL; + size_t new_from_name_len = 0; char* new_to_name = NULL; - assert(txn); + size_t new_to_name_len = 0; + assert_always(txn); - new_from_name = (char *)tokudb_my_malloc( - get_max_dict_name_path_length(from_name), - MYF(MY_WME) - ); + new_from_name_len = get_max_dict_name_path_length(from_name); + new_from_name = (char*)tokudb::memory::malloc( + new_from_name_len, + MYF(MY_WME)); if (new_from_name == NULL) { error = ENOMEM; goto cleanup; } if (!is_delete) { - assert(to_name); - new_to_name = (char *)tokudb_my_malloc( - get_max_dict_name_path_length(to_name), - MYF(MY_WME) - ); + assert_always(to_name); + new_to_name_len = get_max_dict_name_path_length(to_name); + new_to_name = (char*)tokudb::memory::malloc( + new_to_name_len, + MYF(MY_WME)); if (new_to_name == NULL) { error = ENOMEM; goto cleanup; @@ -6938,32 +7338,37 @@ int ha_tokudb::delete_or_rename_dictionary( const char* from_name, const char* t if (is_key) { sprintf(dict_name, "key-%s", secondary_name); - make_name(new_from_name, from_name, dict_name); - } - else { - make_name(new_from_name, from_name, secondary_name); + make_name(new_from_name, new_from_name_len, from_name, dict_name); + } else { + make_name(new_from_name, new_from_name_len, from_name, secondary_name); } if (!is_delete) { if (is_key) { sprintf(dict_name, "key-%s", secondary_name); - make_name(new_to_name, to_name, dict_name); - } - else { - make_name(new_to_name, to_name, secondary_name); + make_name(new_to_name, new_to_name_len, to_name, dict_name); + } else { + make_name(new_to_name, new_to_name_len, to_name, secondary_name); } } if (is_delete) { error = db_env->dbremove(db_env, txn, new_from_name, NULL, 0); + } else { + error = db_env->dbrename( + db_env, + txn, + new_from_name, + NULL, + new_to_name, + 0); } - else { - error = db_env->dbrename(db_env, txn, new_from_name, NULL, new_to_name, 0); + if (error) { + goto cleanup; } - if (error) { goto cleanup; } cleanup: - tokudb_my_free(new_from_name); - tokudb_my_free(new_to_name); + tokudb::memory::free(new_from_name); + tokudb::memory::free(new_to_name); return error; } @@ -7029,12 +7434,12 @@ int ha_tokudb::delete_or_rename_table (const char* from_name, const char* to_nam if (error) { goto cleanup; } error = status_cursor->c_close(status_cursor); - assert(error==0); + assert_always(error==0); status_cursor = NULL; if (error) { goto cleanup; } error = status_db->close(status_db, 0); - assert(error == 0); + assert_always(error == 0); status_db = NULL; // @@ -7047,11 +7452,11 @@ int ha_tokudb::delete_or_rename_table (const char* from_name, const char* to_nam cleanup: if (status_cursor) { int r = status_cursor->c_close(status_cursor); - assert(r==0); + assert_always(r==0); } if (status_db) { int r = status_db->close(status_db, 0); - assert(r==0); + assert_always(r==0); } if (txn) { if (error) { @@ -7075,12 +7480,25 @@ cleanup: // int ha_tokudb::delete_table(const char *name) { TOKUDB_HANDLER_DBUG_ENTER("%s", name); + TOKUDB_SHARE* share = TOKUDB_SHARE::get_share(name, NULL, NULL, false); + if (share) { + share->unlock(); + share->release(); + // this should be enough to handle locking as the higher level MDL + // on this table should prevent any new analyze tasks. + share->cancel_background_jobs(); + TOKUDB_SHARE::drop_share(share); + } + int error; error = delete_or_rename_table(name, NULL, true); - if (error == DB_LOCK_NOTGRANTED && ((tokudb_debug & TOKUDB_DEBUG_HIDE_DDL_LOCK_ERRORS) == 0)) { - sql_print_error("Could not delete table %s because \ -another transaction has accessed the table. \ -To drop the table, make sure no transactions touch the table.", name); + if (TOKUDB_LIKELY(TOKUDB_DEBUG_FLAGS(TOKUDB_DEBUG_HIDE_DDL_LOCK_ERRORS) == 0) && + error == DB_LOCK_NOTGRANTED) { + sql_print_error( + "Could not delete table %s because another transaction has " + "accessed the table. To drop the table, make sure no " + "transactions touch the table.", + name); } TOKUDB_HANDLER_DBUG_RETURN(error); } @@ -7097,12 +7515,25 @@ To drop the table, make sure no transactions touch the table.", name); // int ha_tokudb::rename_table(const char *from, const char *to) { TOKUDB_HANDLER_DBUG_ENTER("%s %s", from, to); + TOKUDB_SHARE* share = TOKUDB_SHARE::get_share(from, NULL, NULL, false); + if (share) { + share->unlock(); + share->release(); + // this should be enough to handle locking as the higher level MDL + // on this table should prevent any new analyze tasks. + share->cancel_background_jobs(); + TOKUDB_SHARE::drop_share(share); + } int error; error = delete_or_rename_table(from, to, false); - if (error == DB_LOCK_NOTGRANTED && ((tokudb_debug & TOKUDB_DEBUG_HIDE_DDL_LOCK_ERRORS) == 0)) { - sql_print_error("Could not rename table from %s to %s because \ -another transaction has accessed the table. \ -To rename the table, make sure no transactions touch the table.", from, to); + if (TOKUDB_LIKELY(TOKUDB_DEBUG_FLAGS(TOKUDB_DEBUG_HIDE_DDL_LOCK_ERRORS) == 0) && + error == DB_LOCK_NOTGRANTED) { + sql_print_error( + "Could not rename table from %s to %s because another transaction " + "has accessed the table. To rename the table, make sure no " + "transactions touch the table.", + from, + to); } TOKUDB_HANDLER_DBUG_RETURN(error); } @@ -7117,9 +7548,11 @@ To rename the table, make sure no transactions touch the table.", from, to); double ha_tokudb::scan_time() { TOKUDB_HANDLER_DBUG_ENTER(""); double ret_val = (double)stats.records / 3; - if (tokudb_debug & TOKUDB_DEBUG_RETURN) { - TOKUDB_HANDLER_TRACE("return %" PRIu64 " %f", (uint64_t) stats.records, ret_val); - } + TOKUDB_HANDLER_TRACE_FOR_FLAGS( + TOKUDB_DEBUG_RETURN, + "return %" PRIu64 " %f", + (uint64_t)stats.records, + ret_val); DBUG_RETURN(ret_val); } @@ -7143,10 +7576,7 @@ double ha_tokudb::keyread_time(uint index, uint ranges, ha_rows rows) (table->key_info[index].key_length + ref_length) + 1); ret_val = (rows + keys_per_block - 1)/ keys_per_block; - if (tokudb_debug & TOKUDB_DEBUG_RETURN) { - TOKUDB_HANDLER_TRACE("return %f", ret_val); - } - DBUG_RETURN(ret_val); + TOKUDB_HANDLER_DBUG_RETURN_DOUBLE(ret_val); } // @@ -7208,19 +7638,13 @@ double ha_tokudb::read_time( ret_val = is_clustering ? ret_val + 0.00001 : ret_val; cleanup: - if (tokudb_debug & TOKUDB_DEBUG_RETURN) { - TOKUDB_HANDLER_TRACE("return %f", ret_val); - } - DBUG_RETURN(ret_val); + TOKUDB_HANDLER_DBUG_RETURN_DOUBLE(ret_val); } double ha_tokudb::index_only_read_time(uint keynr, double records) { TOKUDB_HANDLER_DBUG_ENTER("%u %f", keynr, records); double ret_val = keyread_time(keynr, 1, (ha_rows)records); - if (tokudb_debug & TOKUDB_DEBUG_RETURN) { - TOKUDB_HANDLER_TRACE("return %f", ret_val); - } - DBUG_RETURN(ret_val); + TOKUDB_HANDLER_DBUG_RETURN_DOUBLE(ret_val); } // @@ -7293,9 +7717,11 @@ ha_rows ha_tokudb::records_in_range(uint keynr, key_range* start_key, key_range* ret_val = (ha_rows) (rows <= 1 ? 1 : rows); cleanup: - if (tokudb_debug & TOKUDB_DEBUG_RETURN) { - TOKUDB_HANDLER_TRACE("return %" PRIu64 " %" PRIu64, (uint64_t) ret_val, rows); - } + TOKUDB_HANDLER_TRACE_FOR_FLAGS( + TOKUDB_DEBUG_RETURN, + "return %" PRIu64 " %" PRIu64, + (uint64_t)ret_val, + rows); DBUG_RETURN(ret_val); } @@ -7351,24 +7777,30 @@ void ha_tokudb::init_auto_increment() { commit_txn(txn, 0); } - if (tokudb_debug & TOKUDB_DEBUG_AUTO_INCREMENT) { - TOKUDB_HANDLER_TRACE("init auto increment:%lld", share->last_auto_increment); - } + TOKUDB_HANDLER_TRACE_FOR_FLAGS( + TOKUDB_DEBUG_AUTO_INCREMENT, + "init auto increment:%lld", + share->last_auto_increment); } -void ha_tokudb::get_auto_increment(ulonglong offset, ulonglong increment, ulonglong nb_desired_values, ulonglong * first_value, ulonglong * nb_reserved_values) { +void ha_tokudb::get_auto_increment( + ulonglong offset, + ulonglong increment, + ulonglong nb_desired_values, + ulonglong* first_value, + ulonglong* nb_reserved_values) { + TOKUDB_HANDLER_DBUG_ENTER(""); ulonglong nr; bool over; - tokudb_pthread_mutex_lock(&share->mutex); + share->lock(); if (share->auto_inc_create_value > share->last_auto_increment) { nr = share->auto_inc_create_value; over = false; share->last_auto_increment = share->auto_inc_create_value; - } - else { + } else { nr = share->last_auto_increment + increment; over = nr < share->last_auto_increment; if (over) @@ -7378,19 +7810,23 @@ void ha_tokudb::get_auto_increment(ulonglong offset, ulonglong increment, ulongl share->last_auto_increment = nr + (nb_desired_values - 1)*increment; if (delay_updating_ai_metadata) { ai_metadata_update_required = true; - } - else { - update_max_auto_inc(share->status_block, share->last_auto_increment); - } - } - - if (tokudb_debug & TOKUDB_DEBUG_AUTO_INCREMENT) { - TOKUDB_HANDLER_TRACE("get_auto_increment(%lld,%lld,%lld):got:%lld:%lld", - offset, increment, nb_desired_values, nr, nb_desired_values); - } + } else { + update_max_auto_inc( + share->status_block, + share->last_auto_increment); + } + } + TOKUDB_HANDLER_TRACE_FOR_FLAGS( + TOKUDB_DEBUG_AUTO_INCREMENT, + "get_auto_increment(%lld,%lld,%lld): got:%lld:%lld", + offset, + increment, + nb_desired_values, + nr, + nb_desired_values); *first_value = nr; *nb_reserved_values = nb_desired_values; - tokudb_pthread_mutex_unlock(&share->mutex); + share->unlock(); TOKUDB_HANDLER_DBUG_VOID_RETURN; } @@ -7419,16 +7855,15 @@ bool ha_tokudb::is_auto_inc_singleton(){ // 0 on success, error otherwise // int ha_tokudb::tokudb_add_index( - TABLE *table_arg, - KEY *key_info, - uint num_of_keys, - DB_TXN* txn, + TABLE* table_arg, + KEY* key_info, + uint num_of_keys, + DB_TXN* txn, bool* inc_num_DBs, - bool* modified_DBs - ) -{ + bool* modified_DBs) { + TOKUDB_HANDLER_DBUG_ENTER(""); - assert(txn); + assert_always(txn); int error; uint curr_index = 0; @@ -7438,7 +7873,7 @@ int ha_tokudb::tokudb_add_index( THD* thd = ha_thd(); DB_LOADER* loader = NULL; DB_INDEXER* indexer = NULL; - bool loader_save_space = get_load_save_space(thd); + bool loader_save_space = tokudb::sysvars::load_save_space(thd); bool use_hot_index = (lock.type == TL_WRITE_ALLOW_WRITE); uint32_t loader_flags = loader_save_space ? LOADER_COMPRESS_INTERMEDIATES : 0; uint32_t indexer_flags = 0; @@ -7468,14 +7903,17 @@ int ha_tokudb::tokudb_add_index( // // get the row type to use for the indexes we're adding // - toku_compression_method compression_method = get_compression_method(share->file); + toku_compression_method compression_method = + get_compression_method(share->file); // // status message to be shown in "show process list" // const char *orig_proc_info = tokudb_thd_get_proc_info(thd); - char status_msg[MAX_ALIAS_NAME + 200]; //buffer of 200 should be a good upper bound. - ulonglong num_processed = 0; //variable that stores number of elements inserted thus far + // buffer of 200 should be a good upper bound. + char status_msg[MAX_ALIAS_NAME + 200]; + // variable that stores number of elements inserted thus far + ulonglong num_processed = 0; thd_proc_info(thd, "Adding indexes"); // @@ -7500,13 +7938,15 @@ int ha_tokudb::tokudb_add_index( } } - rw_wrlock(&share->num_DBs_lock); + share->_num_DBs_lock.lock_write(); rw_lock_taken = true; // // open all the DB files and set the appropriate variables in share // they go to the end of share->key_file // - creating_hot_index = use_hot_index && num_of_keys == 1 && (key_info[0].flags & HA_NOSAME) == 0; + creating_hot_index = + use_hot_index && num_of_keys == 1 && + (key_info[0].flags & HA_NOSAME) == 0; if (use_hot_index && (share->num_DBs > curr_num_DBs)) { // // already have hot index in progress, get out @@ -7522,35 +7962,47 @@ int ha_tokudb::tokudb_add_index( &share->kc_info.key_filters[curr_index], &key_info[i], table_arg, - false - ); + false); if (!hidden_primary_key) { set_key_filter( &share->kc_info.key_filters[curr_index], &table_arg->key_info[primary_key], table_arg, - false - ); + false); } - error = initialize_col_pack_info(&share->kc_info,table_arg->s,curr_index); + error = initialize_col_pack_info( + &share->kc_info, + table_arg->s, + curr_index); if (error) { goto cleanup; } } - error = create_secondary_dictionary(share->table_name, table_arg, &key_info[i], txn, &share->kc_info, curr_index, creating_hot_index, compression_method); - if (error) { goto cleanup; } + error = create_secondary_dictionary( + share->full_table_name(), + table_arg, + &key_info[i], + txn, + &share->kc_info, + curr_index, + creating_hot_index, + compression_method); + if (error) { + goto cleanup; + } error = open_secondary_dictionary( - &share->key_file[curr_index], + &share->key_file[curr_index], &key_info[i], - share->table_name, + share->full_table_name(), false, - txn - ); - if (error) { goto cleanup; } + txn); + if (error) { + goto cleanup; + } } if (creating_hot_index) { @@ -7564,17 +8016,22 @@ int ha_tokudb::tokudb_add_index( num_of_keys, &share->key_file[curr_num_DBs], mult_db_flags, - indexer_flags - ); - if (error) { goto cleanup; } + indexer_flags); + if (error) { + goto cleanup; + } error = indexer->set_poll_function(indexer, ai_poll_fun, &lc); - if (error) { goto cleanup; } + if (error) { + goto cleanup; + } error = indexer->set_error_callback(indexer, loader_ai_err_fun, &lc); - if (error) { goto cleanup; } + if (error) { + goto cleanup; + } - rw_unlock(&share->num_DBs_lock); + share->_num_DBs_lock.unlock(); rw_lock_taken = false; #ifdef HA_TOKUDB_HAS_THD_PROGRESS @@ -7585,17 +8042,20 @@ int ha_tokudb::tokudb_add_index( error = indexer->build(indexer); - if (error) { goto cleanup; } + if (error) { + goto cleanup; + } - rw_wrlock(&share->num_DBs_lock); + share->_num_DBs_lock.lock_write(); error = indexer->close(indexer); - rw_unlock(&share->num_DBs_lock); - if (error) { goto cleanup; } + share->_num_DBs_lock.unlock(); + if (error) { + goto cleanup; + } indexer = NULL; - } - else { + } else { DBUG_ASSERT(table->mdl_ticket->get_type() >= MDL_SHARED_NO_WRITE); - rw_unlock(&share->num_DBs_lock); + share->_num_DBs_lock.unlock(); rw_lock_taken = false; prelocked_right_range_size = 0; prelocked_left_range_size = 0; @@ -7608,27 +8068,37 @@ int ha_tokudb::tokudb_add_index( bf_info.key_to_compare = NULL; error = db_env->create_loader( - db_env, - txn, - &loader, + db_env, + txn, + &loader, NULL, // no src_db needed - num_of_keys, - &share->key_file[curr_num_DBs], + num_of_keys, + &share->key_file[curr_num_DBs], mult_put_flags, mult_dbt_flags, - loader_flags - ); - if (error) { goto cleanup; } + loader_flags); + if (error) { + goto cleanup; + } error = loader->set_poll_function(loader, loader_poll_fun, &lc); - if (error) { goto cleanup; } + if (error) { + goto cleanup; + } error = loader->set_error_callback(loader, loader_ai_err_fun, &lc); - if (error) { goto cleanup; } + if (error) { + goto cleanup; + } // // scan primary table, create each secondary key, add to each DB // - if ((error = share->file->cursor(share->file, txn, &tmp_cursor, DB_SERIALIZABLE))) { + error = share->file->cursor( + share->file, + txn, + &tmp_cursor, + DB_SERIALIZABLE); + if (error) { tmp_cursor = NULL; // Safety goto cleanup; } @@ -7643,16 +8113,21 @@ int ha_tokudb::tokudb_add_index( share->file->dbt_neg_infty(), share->file->dbt_pos_infty(), true, - 0 - ); - if (error) { goto cleanup; } + 0); + if (error) { + goto cleanup; + } // set the bulk fetch iteration to its max so that adding an // index fills the bulk fetch buffer every time. we do not // want it to grow exponentially fast. rows_fetched_using_bulk_fetch = 0; bulk_fetch_iteration = HA_TOKU_BULK_FETCH_ITERATION_MAX; - cursor_ret_val = tmp_cursor->c_getf_next(tmp_cursor, DB_PRELOCKED,smart_dbt_bf_callback, &bf_info); + cursor_ret_val = tmp_cursor->c_getf_next( + tmp_cursor, + DB_PRELOCKED, + smart_dbt_bf_callback, + &bf_info); #ifdef HA_TOKUDB_HAS_THD_PROGRESS // initialize a two phase progress report. @@ -7660,21 +8135,30 @@ int ha_tokudb::tokudb_add_index( thd_progress_init(thd, 2); #endif - while (cursor_ret_val != DB_NOTFOUND || ((bytes_used_in_range_query_buff - curr_range_query_buff_offset) > 0)) { - if ((bytes_used_in_range_query_buff - curr_range_query_buff_offset) == 0) { + while (cursor_ret_val != DB_NOTFOUND || + ((bytes_used_in_range_query_buff - + curr_range_query_buff_offset) > 0)) { + if ((bytes_used_in_range_query_buff - + curr_range_query_buff_offset) == 0) { invalidate_bulk_fetch(); // reset the buffers - cursor_ret_val = tmp_cursor->c_getf_next(tmp_cursor, DB_PRELOCKED, smart_dbt_bf_callback, &bf_info); + cursor_ret_val = tmp_cursor->c_getf_next( + tmp_cursor, + DB_PRELOCKED, + smart_dbt_bf_callback, + &bf_info); if (cursor_ret_val != DB_NOTFOUND && cursor_ret_val != 0) { error = cursor_ret_val; goto cleanup; } } - // do this check in case the the c_getf_next did not put anything into the buffer because - // there was no more data - if ((bytes_used_in_range_query_buff - curr_range_query_buff_offset) == 0) { + // do this check in case the the c_getf_next did not put anything + // into the buffer because there was no more data + if ((bytes_used_in_range_query_buff - + curr_range_query_buff_offset) == 0) { break; } - // at this point, we know the range query buffer has at least one key/val pair + // at this point, we know the range query buffer has at least one + // key/val pair uchar* curr_pos = range_query_buff+curr_range_query_buff_offset; uint32_t key_size = *(uint32_t *)curr_pos; @@ -7694,17 +8178,26 @@ int ha_tokudb::tokudb_add_index( curr_range_query_buff_offset = curr_pos - range_query_buff; error = loader->put(loader, &curr_pk_key, &curr_pk_val); - if (error) { goto cleanup; } + if (error) { + goto cleanup; + } num_processed++; if ((num_processed % 1000) == 0) { - sprintf(status_msg, "Adding indexes: Fetched %llu of about %llu rows, loading of data still remains.", - num_processed, (long long unsigned) share->rows); + sprintf( + status_msg, + "Adding indexes: Fetched %llu of about %llu rows, loading " + "of data still remains.", + num_processed, + (long long unsigned)share->row_count()); thd_proc_info(thd, status_msg); #ifdef HA_TOKUDB_HAS_THD_PROGRESS - thd_progress_report(thd, num_processed, (long long unsigned) share->rows); + thd_progress_report( + thd, + num_processed, + (long long unsigned)share->rows); #endif if (thd_killed(thd)) { @@ -7714,7 +8207,7 @@ int ha_tokudb::tokudb_add_index( } } error = tmp_cursor->c_close(tmp_cursor); - assert(error==0); + assert_always(error==0); tmp_cursor = NULL; #ifdef HA_TOKUDB_HAS_THD_PROGRESS @@ -7732,9 +8225,14 @@ int ha_tokudb::tokudb_add_index( for (uint i = 0; i < num_of_keys; i++, curr_index++) { if (key_info[i].flags & HA_NOSAME) { bool is_unique; - error = is_index_unique(&is_unique, txn, share->key_file[curr_index], &key_info[i], - creating_hot_index ? 0 : DB_PRELOCKED_WRITE); - if (error) goto cleanup; + error = is_index_unique( + &is_unique, + txn, + share->key_file[curr_index], + &key_info[i], + creating_hot_index ? 0 : DB_PRELOCKED_WRITE); + if (error) + goto cleanup; if (!is_unique) { error = HA_ERR_FOUND_DUPP_KEY; last_dup_key = i; @@ -7743,22 +8241,20 @@ int ha_tokudb::tokudb_add_index( } } + share->lock(); // // We have an accurate row count, might as well update share->rows // if(!creating_hot_index) { - tokudb_pthread_mutex_lock(&share->mutex); - share->rows = num_processed; - tokudb_pthread_mutex_unlock(&share->mutex); + share->set_row_count(num_processed, true); } // // now write stuff to status.tokudb // - tokudb_pthread_mutex_lock(&share->mutex); for (uint i = 0; i < num_of_keys; i++) { write_key_name_to_status(share->status_block, key_info[i].name, txn); } - tokudb_pthread_mutex_unlock(&share->mutex); + share->unlock(); error = 0; cleanup: @@ -7766,12 +8262,12 @@ cleanup: thd_progress_end(thd); #endif if (rw_lock_taken) { - rw_unlock(&share->num_DBs_lock); + share->_num_DBs_lock.unlock(); rw_lock_taken = false; } if (tmp_cursor) { int r = tmp_cursor->c_close(tmp_cursor); - assert(r==0); + assert_always(r==0); tmp_cursor = NULL; } if (loader != NULL) { @@ -7782,14 +8278,17 @@ cleanup: if (indexer != NULL) { sprintf(status_msg, "aborting creation of indexes."); thd_proc_info(thd, status_msg); - rw_wrlock(&share->num_DBs_lock); + share->_num_DBs_lock.lock_write(); indexer->abort(indexer); - rw_unlock(&share->num_DBs_lock); + share->_num_DBs_lock.unlock(); } - if (error == DB_LOCK_NOTGRANTED && ((tokudb_debug & TOKUDB_DEBUG_HIDE_DDL_LOCK_ERRORS) == 0)) { - sql_print_error("Could not add indexes to table %s because \ -another transaction has accessed the table. \ -To add indexes, make sure no transactions touch the table.", share->table_name); + if (TOKUDB_LIKELY(TOKUDB_DEBUG_FLAGS(TOKUDB_DEBUG_HIDE_DDL_LOCK_ERRORS) == 0) && + error == DB_LOCK_NOTGRANTED) { + sql_print_error( + "Could not add indexes to table %s because another transaction has " + "accessed the table. To add indexes, make sure no transactions " + "touch the table.", + share->full_table_name()); } thd_proc_info(thd, orig_proc_info); TOKUDB_HANDLER_DBUG_RETURN(error ? error : loader_error); @@ -7799,7 +8298,12 @@ To add indexes, make sure no transactions touch the table.", share->table_name); // Internal function called by ha_tokudb::add_index and ha_tokudb::alter_table_phase2 // Closes added indexes in case of error in error path of add_index and alter_table_phase2 // -void ha_tokudb::restore_add_index(TABLE* table_arg, uint num_of_keys, bool incremented_numDBs, bool modified_DBs) { +void ha_tokudb::restore_add_index( + TABLE* table_arg, + uint num_of_keys, + bool incremented_numDBs, + bool modified_DBs) { + uint curr_num_DBs = table_arg->s->keys + tokudb_test(hidden_primary_key); uint curr_index = 0; @@ -7808,7 +8312,7 @@ void ha_tokudb::restore_add_index(TABLE* table_arg, uint num_of_keys, bool incre // so that there is not a window // if (incremented_numDBs) { - rw_wrlock(&share->num_DBs_lock); + share->_num_DBs_lock.lock_write(); share->num_DBs--; } if (modified_DBs) { @@ -7821,15 +8325,14 @@ void ha_tokudb::restore_add_index(TABLE* table_arg, uint num_of_keys, bool incre if (share->key_file[curr_index]) { int r = share->key_file[curr_index]->close( share->key_file[curr_index], - 0 - ); - assert(r==0); + 0); + assert_always(r==0); share->key_file[curr_index] = NULL; } } } if (incremented_numDBs) { - rw_unlock(&share->num_DBs_lock); + share->_num_DBs_lock.unlock(); } } @@ -7837,14 +8340,22 @@ void ha_tokudb::restore_add_index(TABLE* table_arg, uint num_of_keys, bool incre // Internal function called by ha_tokudb::prepare_drop_index and ha_tokudb::alter_table_phase2 // With a transaction, drops dictionaries associated with indexes in key_num // -int ha_tokudb::drop_indexes(TABLE *table_arg, uint *key_num, uint num_of_keys, KEY *key_info, DB_TXN* txn) { +int ha_tokudb::drop_indexes( + TABLE* table_arg, + uint* key_num, + uint num_of_keys, + KEY* key_info, + DB_TXN* txn) { + TOKUDB_HANDLER_DBUG_ENTER(""); - assert(txn); + assert_always(txn); int error = 0; for (uint i = 0; i < num_of_keys; i++) { uint curr_index = key_num[i]; - error = share->key_file[curr_index]->pre_acquire_fileops_lock(share->key_file[curr_index],txn); + error = share->key_file[curr_index]->pre_acquire_fileops_lock( + share->key_file[curr_index], + txn); if (error != 0) { goto cleanup; } @@ -7852,30 +8363,51 @@ int ha_tokudb::drop_indexes(TABLE *table_arg, uint *key_num, uint num_of_keys, K for (uint i = 0; i < num_of_keys; i++) { uint curr_index = key_num[i]; int r = share->key_file[curr_index]->close(share->key_file[curr_index],0); - assert(r==0); + assert_always(r==0); share->key_file[curr_index] = NULL; - error = remove_key_name_from_status(share->status_block, key_info[curr_index].name, txn); - if (error) { goto cleanup; } + error = remove_key_name_from_status( + share->status_block, + key_info[curr_index].name, + txn); + if (error) { + goto cleanup; + } - error = delete_or_rename_dictionary(share->table_name, NULL, key_info[curr_index].name, true, txn, true); - if (error) { goto cleanup; } + error = delete_or_rename_dictionary( + share->full_table_name(), + NULL, + key_info[curr_index].name, + true, + txn, + true); + if (error) { + goto cleanup; + } } cleanup: - if (error == DB_LOCK_NOTGRANTED && ((tokudb_debug & TOKUDB_DEBUG_HIDE_DDL_LOCK_ERRORS) == 0)) { - sql_print_error("Could not drop indexes from table %s because \ -another transaction has accessed the table. \ -To drop indexes, make sure no transactions touch the table.", share->table_name); + if (TOKUDB_LIKELY(TOKUDB_DEBUG_FLAGS(TOKUDB_DEBUG_HIDE_DDL_LOCK_ERRORS) == 0) && + error == DB_LOCK_NOTGRANTED) { + sql_print_error( + "Could not drop indexes from table %s because another transaction " + "has accessed the table. To drop indexes, make sure no " + "transactions touch the table.", + share->full_table_name()); } TOKUDB_HANDLER_DBUG_RETURN(error); } // -// Internal function called by ha_tokudb::prepare_drop_index and ha_tokudb::alter_table_phase2 -// Restores dropped indexes in case of error in error path of prepare_drop_index and alter_table_phase2 +// Internal function called by ha_tokudb::prepare_drop_index and +// ha_tokudb::alter_table_phase2 +// Restores dropped indexes in case of error in error path of +// prepare_drop_index and alter_table_phase2 // -void ha_tokudb::restore_drop_indexes(TABLE *table_arg, uint *key_num, uint num_of_keys) { +void ha_tokudb::restore_drop_indexes( + TABLE* table_arg, + uint* key_num, + uint num_of_keys) { // // reopen closed dictionaries @@ -7885,13 +8417,12 @@ void ha_tokudb::restore_drop_indexes(TABLE *table_arg, uint *key_num, uint num_o uint curr_index = key_num[i]; if (share->key_file[curr_index] == NULL) { r = open_secondary_dictionary( - &share->key_file[curr_index], + &share->key_file[curr_index], &table_share->key_info[curr_index], - share->table_name, - false, // - NULL - ); - assert(!r); + share->full_table_name(), + false, + NULL); + assert_always(!r); } } } @@ -7937,56 +8468,65 @@ void ha_tokudb::print_error(int error, myf errflag) { // does so by deleting and then recreating the dictionary in the context // of a transaction // -int ha_tokudb::truncate_dictionary( uint keynr, DB_TXN* txn ) { +int ha_tokudb::truncate_dictionary(uint keynr, DB_TXN* txn) { int error; bool is_pk = (keynr == primary_key); - toku_compression_method compression_method = get_compression_method(share->key_file[keynr]); + toku_compression_method compression_method = + get_compression_method(share->key_file[keynr]); error = share->key_file[keynr]->close(share->key_file[keynr], 0); - assert(error == 0); + assert_always(error == 0); share->key_file[keynr] = NULL; - if (is_pk) { share->file = NULL; } + if (is_pk) { + share->file = NULL; + } if (is_pk) { error = delete_or_rename_dictionary( - share->table_name, + share->full_table_name(), NULL, - "main", + "main", false, //is_key txn, - true // is a delete - ); - if (error) { goto cleanup; } - } - else { + true); // is a delete + if (error) { + goto cleanup; + } + } else { error = delete_or_rename_dictionary( - share->table_name, + share->full_table_name(), NULL, - table_share->key_info[keynr].name, + table_share->key_info[keynr].name, true, //is_key txn, - true // is a delete - ); - if (error) { goto cleanup; } + true); // is a delete + if (error) { + goto cleanup; + } } if (is_pk) { - error = create_main_dictionary(share->table_name, table, txn, &share->kc_info, compression_method); - } - else { + error = create_main_dictionary( + share->full_table_name(), + table, + txn, + &share->kc_info, + compression_method); + } else { error = create_secondary_dictionary( - share->table_name, - table, - &table_share->key_info[keynr], + share->full_table_name(), + table, + &table_share->key_info[keynr], txn, &share->kc_info, keynr, false, - compression_method - ); + compression_method); + } + if (error) { + goto cleanup; } - if (error) { goto cleanup; } cleanup: return error; @@ -8026,39 +8566,44 @@ int ha_tokudb::delete_all_rows_internal() { DB_TXN* txn = NULL; error = txn_begin(db_env, 0, &txn, 0, ha_thd()); - if (error) { goto cleanup; } + if (error) { + goto cleanup; + } curr_num_DBs = table->s->keys + tokudb_test(hidden_primary_key); for (uint i = 0; i < curr_num_DBs; i++) { error = share->key_file[i]->pre_acquire_fileops_lock( - share->key_file[i], - txn - ); - if (error) { goto cleanup; } + share->key_file[i], + txn); + if (error) { + goto cleanup; + } error = share->key_file[i]->pre_acquire_table_lock( - share->key_file[i], - txn - ); - if (error) { goto cleanup; } + share->key_file[i], + txn); + if (error) { + goto cleanup; + } } for (uint i = 0; i < curr_num_DBs; i++) { error = truncate_dictionary(i, txn); - if (error) { goto cleanup; } + if (error) { + goto cleanup; + } } // zap the row count if (error == 0) { - share->rows = 0; - // update auto increment - share->last_auto_increment = 0; - // calling write_to_status directly because we need to use txn - write_to_status( - share->status_block, + share->set_row_count(0, false); + // update auto increment + share->last_auto_increment = 0; + // calling write_to_status directly because we need to use txn + write_to_status( + share->status_block, hatoku_max_ai, - &share->last_auto_increment, - sizeof(share->last_auto_increment), - txn - ); + &share->last_auto_increment, + sizeof(share->last_auto_increment), + txn); } share->try_table_lock = true; @@ -8066,16 +8611,19 @@ cleanup: if (txn) { if (error) { abort_txn(txn); - } - else { + } else { commit_txn(txn,0); } } - if (error == DB_LOCK_NOTGRANTED && ((tokudb_debug & TOKUDB_DEBUG_HIDE_DDL_LOCK_ERRORS) == 0)) { - sql_print_error("Could not truncate table %s because another transaction has accessed the \ - table. To truncate the table, make sure no transactions touch the table.", - share->table_name); + if (TOKUDB_LIKELY(TOKUDB_DEBUG_FLAGS( + TOKUDB_DEBUG_HIDE_DDL_LOCK_ERRORS) == 0) && + error == DB_LOCK_NOTGRANTED) { + sql_print_error( + "Could not truncate table %s because another transaction has " + "accessed the table. To truncate the table, make sure no " + "transactions touch the table.", + share->full_table_name()); } // // regardless of errors, need to reopen the DB's @@ -8085,21 +8633,18 @@ cleanup: if (share->key_file[i] == NULL) { if (i != primary_key) { r = open_secondary_dictionary( - &share->key_file[i], - &table_share->key_info[i], - share->table_name, - false, // - NULL - ); - assert(!r); - } - else { + &share->key_file[i], + &table_share->key_info[i], + share->full_table_name(), + false, + NULL); + assert_always(!r); + } else { r = open_main_dictionary( - share->table_name, - false, - NULL - ); - assert(!r); + share->full_table_name(), + false, + NULL); + assert_always(!r); } } } @@ -8111,7 +8656,7 @@ void ha_tokudb::set_loader_error(int err) { } void ha_tokudb::set_dup_value_for_pk(DBT* key) { - assert(!hidden_primary_key); + assert_always(!hidden_primary_key); unpack_key(table->record[0],key,primary_key); last_dup_key = primary_key; } @@ -8147,18 +8692,20 @@ Item* ha_tokudb::idx_cond_push(uint keyno_arg, Item* idx_cond_arg) { void ha_tokudb::cleanup_txn(DB_TXN *txn) { if (transaction == txn && cursor) { int r = cursor->c_close(cursor); - assert(r == 0); + assert_always(r == 0); cursor = NULL; } } void ha_tokudb::add_to_trx_handler_list() { - tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(ha_thd(), tokudb_hton); + tokudb_trx_data* trx = + (tokudb_trx_data*)thd_get_ha_data(ha_thd(), tokudb_hton); trx->handlers = list_add(trx->handlers, &trx_handler_list); } void ha_tokudb::remove_from_trx_handler_list() { - tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(ha_thd(), tokudb_hton); + tokudb_trx_data* trx = + (tokudb_trx_data*)thd_get_ha_data(ha_thd(), tokudb_hton); trx->handlers = list_delete(trx->handlers, &trx_handler_list); } @@ -8190,7 +8737,7 @@ bool ha_tokudb::rpl_lookup_rows() { if (!in_rpl_delete_rows && !in_rpl_update_rows) return true; else - return THDVAR(ha_thd(), rpl_lookup_rows); + return tokudb::sysvars::rpl_lookup_rows(ha_thd()); } // table admin diff --git a/storage/tokudb/ha_tokudb.h b/storage/tokudb/ha_tokudb.h index e263cabb0d1..8783836fbf0 100644 --- a/storage/tokudb/ha_tokudb.h +++ b/storage/tokudb/ha_tokudb.h @@ -23,11 +23,12 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." -#if !defined(HA_TOKUDB_H) -#define HA_TOKUDB_H +#ifndef _HA_TOKUDB_H +#define _HA_TOKUDB_H -#include <db.h> +#include "hatoku_hton.h" #include "hatoku_cmp.h" +#include "tokudb_background.h" #define HA_TOKU_ORIG_VERSION 4 #define HA_TOKU_VERSION 4 @@ -45,84 +46,359 @@ typedef struct loader_context { } *LOADER_CONTEXT; // -// This object stores table information that is to be shared +// This class stores table information that is to be shared // among all ha_tokudb objects. -// There is one instance per table, shared among threads. +// There is one instance per table, shared among handlers. // Some of the variables here are the DB* pointers to indexes, // and auto increment information. // +// When the last user releases it's reference on the share, +// it closes all of its database handles and releases all info +// The share instance stays around though so some data can be transiently +// kept across open-close-open-close cycles. These data will be explicitly +// noted below. +// class TOKUDB_SHARE { public: - void init(void); - void destroy(void); + enum share_state_t { + CLOSED, + OPENED, + ERROR + }; + + // one time, start up init + static void static_init(); + + // one time, shutdown destroy + static void static_destroy(); + + // retuns a locked, properly reference counted share + // callers must check to ensure share is in correct state for callers use + // and unlock the share. + // if create_new is set, a new "CLOSED" share will be created if one + // doesn't exist, otherwise will return NULL if an existing is not found. + static TOKUDB_SHARE* get_share( + const char* table_name, + TABLE_SHARE* table_share, + THR_LOCK_DATA* data, + bool create_new); + + // removes a share entirely from the pool, call to rename/deleta a table + // caller must hold ddl_mutex on this share and the share MUST have + // exactly 0 _use_count + static void drop_share(TOKUDB_SHARE* share); + + void* operator new(size_t sz); + void operator delete(void* p); + + TOKUDB_SHARE(); + + // increases the ref count and waits for any currently executing state + // transition to complete + // returns current state and leaves share locked + // callers must check to ensure share is in correct state for callers use + // and unlock the share. + share_state_t addref(); + + // decreases the ref count and potentially closes the share + // caller must not have ownership of mutex, will lock and release + int release(); + + // returns the current use count + // no locking requirements + inline int use_count() const; + + // locks the share + inline void lock() const; + + // unlocks the share + inline void unlock() const; + + // returns the current state of the share + // no locking requirements + inline share_state_t state() const; + + // sets the state of the share + // caller must hold mutex on this share + inline void set_state(share_state_t state); + + // returns the full MySQL table name of the table ex: + // ./database/table + // no locking requirements + inline const char* full_table_name() const; + + // returns the strlen of the full table name + // no locking requirements + inline uint full_table_name_length() const; + + // returns the parsed database name this table resides in + // no locking requirements + inline const char* database_name() const; + + // returns the strlen of the database name + // no locking requirements + inline uint database_name_length() const; + + // returns the parsed table name of this table + // no locking requirements + inline const char* table_name() const; + + // returns the strlen of the the table name + // no locking requirements + inline uint table_name_length() const; + + // sets the estimated number of rows in the table + // should be called only during share initialization and info call + // caller must hold mutex on this share unless specified by 'locked' + inline void set_row_count(uint64_t rows, bool locked); + + // updates tracked row count and ongoing table change delta tracking + // called from any ha_tokudb operation that inserts/modifies/deletes rows + // may spawn background analysis if enabled, allowed and threshold hit + // caller must not have ownership of mutex, will lock and release + void update_row_count( + THD* thd, + uint64_t added, + uint64_t deleted, + uint64_t updated); + + // returns the current row count estimate + // no locking requirements + inline ha_rows row_count() const; + + // initializes cardinality statistics, takes ownership of incoming buffer + // caller must hold mutex on this share + inline void init_cardinality_counts( + uint32_t rec_per_keys, + uint64_t* rec_per_key); + + // update the cardinality statistics. number of records must match + // caller must hold mutex on this share + inline void update_cardinality_counts( + uint32_t rec_per_keys, + const uint64_t* rec_per_key); + + // disallow any auto analysis from taking place + // caller must hold mutex on this share + inline void disallow_auto_analysis(); + + // allow any auto analysis to take place + // pass in true for 'reset_deltas' to reset delta counting to 0 + // caller must hold mutex on this share + inline void allow_auto_analysis(bool reset_deltas); + + // cancels all background jobs for this share + // no locking requirements + inline void cancel_background_jobs() const; + + // copies cardinality statistics into TABLE counter set + // caller must not have ownership of mutex, will lock and release + void set_cardinality_counts_in_table(TABLE* table); + + // performs table analysis on underlying indices and produces estimated + // cardinality statistics. + // on success updates cardinality counts in status database and this share + // MUST pass a valid THD to access session variables. + // MAY pass txn. If txn is passed, assumes an explicit user scheduled + // ANALYZE and not an auto ANALYZE resulting from delta threshold + // uses session variables: + // tokudb_analyze_in_background, tokudb_analyze_throttle, + // tokudb_analyze_time, and tokudb_analyze_delete_fraction + // caller must hold mutex on this share + int analyze_standard(THD* thd, DB_TXN* txn); + + // performs table scan and updates the internal FT logical row count value + // on success also updates share row count estimate. + // MUST pass a valid THD to access session variables. + // MAY pass txn. If txn is passed, assumes an explicit user scheduled + // uses session variables: + // tokudb_analyze_in_background, and tokudb_analyze_throttle + // caller must not have ownership of mutex, will lock and release + int analyze_recount_rows(THD* thd, DB_TXN* txn); public: - char *table_name; - uint table_name_length, use_count; - pthread_mutex_t mutex; - THR_LOCK lock; - + //********************************* + // Destroyed and recreated on open-close-open ulonglong auto_ident; ulonglong last_auto_increment, auto_inc_create_value; - // - // estimate on number of rows in table - // - ha_rows rows; - // + // estimate on number of rows added in the process of a locked tables // this is so we can better estimate row count during a lock table - // ha_rows rows_from_locked_table; - DB *status_block; - // + DB* status_block; + // DB that is indexed on the primary key - // - DB *file; - // + DB* file; + // array of all DB's that make up table, includes DB that // is indexed on the primary key, add 1 in case primary // key is hidden - // - DB *key_file[MAX_KEY +1]; - rw_lock_t key_file_lock; + DB* key_file[MAX_KEY + 1]; uint status, version, capabilities; uint ref_length; - // + // whether table has an auto increment column - // bool has_auto_inc; - // + // index of auto increment column in table->field, if auto_inc exists - // uint ai_field_index; - // + // whether the primary key has a string - // bool pk_has_string; KEY_AND_COL_INFO kc_info; - - // + + // key info copied from TABLE_SHARE, used by background jobs that have no + // access to a handler instance + uint _keys; + uint _max_key_parts; + struct key_descriptor_t { + uint _parts; + bool _is_unique; + char* _name; + }; + key_descriptor_t* _key_descriptors; + // we want the following optimization for bulk loads, if the table is empty, // attempt to grab a table lock. emptiness check can be expensive, // so we try it once for a table. After that, we keep this variable around - // to tell us to not try it again. - // - bool try_table_lock; + // to tell us to not try it again. + bool try_table_lock; bool has_unique_keys; bool replace_into_fast; - rw_lock_t num_DBs_lock; + tokudb::thread::rwlock_t _num_DBs_lock; uint32_t num_DBs; - pthread_cond_t m_openclose_cond; - enum { CLOSED, OPENING, OPENED, CLOSING, ERROR } m_state; - int m_error; - int m_initialize_count; +private: + static HASH _open_tables; + static tokudb::thread::mutex_t _open_tables_mutex; + + static uchar* hash_get_key( + TOKUDB_SHARE* share, + size_t* length, + TOKUDB_UNUSED(my_bool not_used)); + + static void hash_free_element(TOKUDB_SHARE* share); - uint n_rec_per_key; - uint64_t *rec_per_key; + //********************************* + // Spans open-close-open + mutable tokudb::thread::mutex_t _mutex; + mutable tokudb::thread::mutex_t _ddl_mutex; + uint _use_count; + + share_state_t _state; + + ulonglong _row_delta_activity; + bool _allow_auto_analysis; + + String _full_table_name; + String _database_name; + String _table_name; + + //********************************* + // Destroyed and recreated on open-close-open + THR_LOCK _thr_lock; + + // estimate on number of rows in table + ha_rows _rows; + + // cardinality counts + uint32_t _rec_per_keys; + uint64_t* _rec_per_key; + bool _card_changed; + + void init(const char* table_name); + void destroy(); }; +inline int TOKUDB_SHARE::use_count() const { + return _use_count; +} +inline void TOKUDB_SHARE::lock() const { + _mutex.lock(); +} +inline void TOKUDB_SHARE::unlock() const { + _mutex.unlock(); +} +inline TOKUDB_SHARE::share_state_t TOKUDB_SHARE::state() const { + return _state; +} +inline void TOKUDB_SHARE::set_state(TOKUDB_SHARE::share_state_t state) { + assert_debug(_mutex.is_owned_by_me()); + _state = state; +} +inline const char* TOKUDB_SHARE::full_table_name() const { + return _full_table_name.ptr(); +} +inline uint TOKUDB_SHARE::full_table_name_length() const { + return _full_table_name.length(); +} +inline const char* TOKUDB_SHARE::database_name() const { + return _database_name.ptr(); +} +inline uint TOKUDB_SHARE::database_name_length() const { + return _database_name.length(); +} +inline const char* TOKUDB_SHARE::table_name() const { + return _table_name.ptr(); +} +inline uint TOKUDB_SHARE::table_name_length() const { + return _table_name.length(); +} +inline void TOKUDB_SHARE::set_row_count(uint64_t rows, bool locked) { + if (!locked) { + lock(); + } else { + assert_debug(_mutex.is_owned_by_me()); + } + if (_rows && rows == 0) + _row_delta_activity = 0; + + _rows = rows; + if (!locked) { + unlock(); + } +} +inline ha_rows TOKUDB_SHARE::row_count() const { + return _rows; +} +inline void TOKUDB_SHARE::init_cardinality_counts( + uint32_t rec_per_keys, + uint64_t* rec_per_key) { + + assert_debug(_mutex.is_owned_by_me()); + // can not change number of keys live + assert_always(_rec_per_key == NULL && _rec_per_keys == 0); + _rec_per_keys = rec_per_keys; + _rec_per_key = rec_per_key; + _card_changed = true; +} +inline void TOKUDB_SHARE::update_cardinality_counts( + uint32_t rec_per_keys, + const uint64_t* rec_per_key) { + + assert_debug(_mutex.is_owned_by_me()); + // can not change number of keys live + assert_always(rec_per_keys == _rec_per_keys); + assert_always(rec_per_key != NULL); + memcpy(_rec_per_key, rec_per_key, _rec_per_keys * sizeof(uint64_t)); + _card_changed = true; +} +inline void TOKUDB_SHARE::disallow_auto_analysis() { + assert_debug(_mutex.is_owned_by_me()); + _allow_auto_analysis = false; +} +inline void TOKUDB_SHARE::allow_auto_analysis(bool reset_deltas) { + assert_debug(_mutex.is_owned_by_me()); + _allow_auto_analysis = true; + if (reset_deltas) + _row_delta_activity = 0; +} +inline void TOKUDB_SHARE::cancel_background_jobs() const { + tokudb::background::_job_manager->cancel_job(full_table_name()); +} + + typedef struct st_filter_key_part_info { uint offset; @@ -278,6 +554,7 @@ private: // ulonglong added_rows; ulonglong deleted_rows; + ulonglong updated_rows; uint last_dup_key; @@ -438,7 +715,7 @@ public: // Returns a bit mask of capabilities of storage engine. Capabilities // defined in sql/handler.h // - ulonglong table_flags(void) const; + ulonglong table_flags() const; ulong index_flags(uint inx, uint part, bool all_parts) const; @@ -482,7 +759,7 @@ public: double index_only_read_time(uint keynr, double records); int open(const char *name, int mode, uint test_if_locked); - int close(void); + int close(); void update_create_info(HA_CREATE_INFO* create_info); int create(const char *name, TABLE * form, HA_CREATE_INFO * create_info); int delete_table(const char *name); @@ -528,7 +805,7 @@ public: void position(const uchar * record); int info(uint); int extra(enum ha_extra_function operation); - int reset(void); + int reset(); int external_lock(THD * thd, int lock_type); int start_stmt(THD * thd, thr_lock_type lock_type); @@ -540,12 +817,17 @@ public: int get_status(DB_TXN* trans); void init_hidden_prim_key_info(DB_TXN *txn); inline void get_auto_primary_key(uchar * to) { - tokudb_pthread_mutex_lock(&share->mutex); + share->lock(); share->auto_ident++; hpk_num_to_char(to, share->auto_ident); - tokudb_pthread_mutex_unlock(&share->mutex); + share->unlock(); } - virtual void get_auto_increment(ulonglong offset, ulonglong increment, ulonglong nb_desired_values, ulonglong * first_value, ulonglong * nb_reserved_values); + virtual void get_auto_increment( + ulonglong offset, + ulonglong increment, + ulonglong nb_desired_values, + ulonglong* first_value, + ulonglong* nb_reserved_values); bool is_optimize_blocking(); bool is_auto_inc_singleton(); void print_error(int error, myf errflag); @@ -772,5 +1054,5 @@ static inline bool key_is_clustering(const KEY *key) { } #endif -#endif +#endif // _HA_TOKUDB_H diff --git a/storage/tokudb/ha_tokudb_admin.cc b/storage/tokudb/ha_tokudb_admin.cc index 996ce49d85f..7fa17945f15 100644 --- a/storage/tokudb/ha_tokudb_admin.cc +++ b/storage/tokudb/ha_tokudb_admin.cc @@ -23,132 +23,800 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." +#include "tokudb_sysvars.h" #include "toku_time.h" -struct analyze_progress_extra { - THD *thd; - TOKUDB_SHARE *share; - TABLE_SHARE *table_share; - uint key_i; - const char *key_name; - time_t t_start; - char *write_status_msg; +namespace tokudb { +namespace analyze { + +class recount_rows_t : public tokudb::background::job_manager_t::job_t { +public: + void* operator new(size_t sz); + void operator delete(void* p); + + recount_rows_t( + bool user_schedued, + THD* thd, + TOKUDB_SHARE* share, + DB_TXN* txn); + + virtual ~recount_rows_t(); + + virtual const char* key(); + + virtual void status( + char* database, + char* table, + char* type, + char* params, + char* status); + +protected: + virtual void on_run(); + + virtual void on_destroy(); + +private: + // to be provided by the initiator of recount rows + THD* _thd; + TOKUDB_SHARE* _share; + DB_TXN* _txn; + ulonglong _throttle; + + // for recount rows status reporting + int _result; + ulonglong _recount_start; // in microseconds + ulonglong _total_elapsed_time; // in microseconds + + bool _local_txn; + ulonglong _rows; + ulonglong _deleted_rows; + ulonglong _ticks; + + static int analyze_recount_rows_progress( + uint64_t count, + uint64_t deleted, + void* extra); + int analyze_recount_rows_progress(uint64_t count, uint64_t deleted); + void get_analyze_status(char*); }; -static int analyze_progress(void *v_extra, uint64_t rows) { - struct analyze_progress_extra *extra = (struct analyze_progress_extra *) v_extra; - THD *thd = extra->thd; - if (thd_killed(thd)) - return ER_ABORTING_CONNECTION; +void* recount_rows_t::operator new(size_t sz) { + return tokudb::memory::malloc(sz, MYF(MY_WME|MY_ZEROFILL|MY_FAE)); +} +void recount_rows_t::operator delete(void* p) { + tokudb::memory::free(p); +} +recount_rows_t::recount_rows_t( + bool user_scheduled, + THD* thd, + TOKUDB_SHARE* share, + DB_TXN* txn) : + tokudb::background::job_manager_t::job_t(user_scheduled), + _share(share), + _result(HA_ADMIN_OK), + _recount_start(0), + _total_elapsed_time(0), + _local_txn(false), + _rows(0), + _deleted_rows(0), + _ticks(0) { + + assert_debug(thd != NULL); + assert_debug(share != NULL); + + if (tokudb::sysvars::analyze_in_background(thd)) { + _thd = NULL; + _txn = NULL; + } else { + _thd = thd; + _txn = txn; + } + + _throttle = tokudb::sysvars::analyze_throttle(thd); +} +recount_rows_t::~recount_rows_t() { +} +void recount_rows_t::on_run() { + _recount_start = tokudb::time::microsec(); + _total_elapsed_time = 0; + + if (_txn == NULL) { + _result = db_env->txn_begin(db_env, NULL, &_txn, DB_READ_UNCOMMITTED); + + if (_result != 0) { + _txn = NULL; + _result = HA_ADMIN_FAILED; + goto error; + } + _local_txn = true; + } else { + _local_txn = false; + } + + _result = + _share->file->recount_rows( + _share->file, + analyze_recount_rows_progress, + this); + + if (_result != 0) { + if (_local_txn) { + _txn->abort(_txn); + _txn = NULL; + } + _result = HA_ADMIN_FAILED; + goto error; + } + + DB_BTREE_STAT64 dict_stats; + _result = _share->file->stat64(_share->file, _txn, &dict_stats); + if (_result == 0) { + _share->set_row_count(dict_stats.bt_ndata, false); + } + if (_result != 0) + _result = HA_ADMIN_FAILED; + + if (_local_txn) { + if (_result == HA_ADMIN_OK) { + _txn->commit(_txn, 0); + } else { + _txn->abort(_txn); + } + _txn = NULL; + } + + sql_print_information( + "tokudb analyze recount rows %d counted %lld", + _result, + _share->row_count()); +error: + return; +} +void recount_rows_t::on_destroy() { + _share->release(); +} +const char* recount_rows_t::key() { + return _share->full_table_name(); +} +void recount_rows_t::status( + char* database, + char* table, + char* type, + char* params, + char* status) { + + strcpy(database, _share->database_name()); + strcpy(table, _share->table_name()); + strcpy(type, "TOKUDB_ANALYZE_MODE_RECOUNT_ROWS"); + sprintf(params, "TOKUDB_ANALYZE_THROTTLE=%llu;", _throttle); + get_analyze_status(status); +} +int recount_rows_t::analyze_recount_rows_progress( + uint64_t count, + uint64_t deleted, + void* extra) { + + recount_rows_t* context = (recount_rows_t*)extra; + return context->analyze_recount_rows_progress(count, deleted); +} +int recount_rows_t::analyze_recount_rows_progress( + uint64_t count, + uint64_t deleted) { + + _rows = count; + _deleted_rows += deleted; + deleted > 0 ? _ticks += deleted : _ticks++; + + if (_ticks > 1000) { + _ticks = 0; + uint64_t now = tokudb::time::microsec(); + _total_elapsed_time = now - _recount_start; + if ((_thd && thd_killed(_thd)) || cancelled()) { + // client killed + return ER_ABORTING_CONNECTION; + } + + // report + if (_thd) { + char status[256]; + get_analyze_status(status); + thd_proc_info(_thd, status); + } + + // throttle + // given the throttle value, lets calculate the maximum number of rows + // we should have seen so far in a .1 sec resolution + if (_throttle > 0) { + uint64_t estimated_rows = _total_elapsed_time / 100000; + estimated_rows = estimated_rows * (_throttle / 10); + if (_rows + _deleted_rows > estimated_rows) { + // sleep for 1/10 of a second + tokudb::time::sleep_microsec(100000); + } + } + } + return 0; +} +void recount_rows_t::get_analyze_status(char* msg) { + sprintf( + msg, + "recount_rows %s.%s counted %llu rows and %llu deleted in %llu " + "seconds.", + _share->database_name(), + _share->table_name(), + _rows, + _deleted_rows, + _total_elapsed_time / tokudb::time::MICROSECONDS); +} + + +class standard_t : public tokudb::background::job_manager_t::job_t { +public: + void* operator new(size_t sz); + void operator delete(void* p); + + standard_t(bool user_scheduled, THD* thd, TOKUDB_SHARE* share, DB_TXN* txn); + + virtual ~standard_t(); + + virtual const char* key(void); + + virtual void status( + char* database, + char* table, + char* type, + char* params, + char* status); + +protected: + virtual void on_run(); + + virtual void on_destroy(); + +private: + // to be provided by initiator of analyze + THD* _thd; + TOKUDB_SHARE* _share; + DB_TXN* _txn; + ulonglong _throttle; // in microseconds + ulonglong _time_limit; // in microseconds + double _delete_fraction; + + // for analyze status reporting, may also use other state + int _result; + ulonglong _analyze_start; // in microseconds + ulonglong _total_elapsed_time; // in microseconds + + // for analyze internal use, pretty much these are per-key/index + ulonglong _current_key; + bool _local_txn; + ulonglong _half_time; + ulonglong _half_rows; + ulonglong _rows; + ulonglong _deleted_rows; + ulonglong _ticks; + ulonglong _analyze_key_start; // in microseconds + ulonglong _key_elapsed_time; // in microseconds + uint _scan_direction; + + static bool analyze_standard_cursor_callback( + void* extra, + uint64_t deleted_rows); + bool analyze_standard_cursor_callback(uint64_t deleted_rows); + + void get_analyze_status(char*); + int analyze_key_progress(); + int analyze_key(uint64_t* rec_per_key_part); +}; + +void* standard_t::operator new(size_t sz) { + return tokudb::memory::malloc(sz, MYF(MY_WME|MY_ZEROFILL|MY_FAE)); +} +void standard_t::operator delete(void* p) { + tokudb::memory::free(p); +} +standard_t::standard_t( + bool user_scheduled, + THD* thd, + TOKUDB_SHARE* share, + DB_TXN* txn) : + tokudb::background::job_manager_t::job_t(user_scheduled), + _share(share), + _result(HA_ADMIN_OK), + _analyze_start(0), + _total_elapsed_time(0), + _current_key(0), + _local_txn(false), + _half_time(0), + _half_rows(0), + _rows(0), + _deleted_rows(0), + _ticks(0), + _analyze_key_start(0), + _key_elapsed_time(0), + _scan_direction(0) { + + assert_debug(thd != NULL); + assert_debug(share != NULL); + + if (tokudb::sysvars::analyze_in_background(thd)) { + _thd = NULL; + _txn = NULL; + } else { + _thd = thd; + _txn = txn; + } + _throttle = tokudb::sysvars::analyze_throttle(thd); + _time_limit = + tokudb::sysvars::analyze_time(thd) * tokudb::time::MICROSECONDS; + _delete_fraction = tokudb::sysvars::analyze_delete_fraction(thd); +} +standard_t::~standard_t() { +} +void standard_t::on_run() { + DB_BTREE_STAT64 stat64; + uint64_t rec_per_key_part[_share->_max_key_parts]; + uint64_t total_key_parts = 0; + _analyze_start = tokudb::time::microsec(); + _half_time = _time_limit > 0 ? _time_limit/2 : 0; + + if (_txn == NULL) { + _result = db_env->txn_begin(db_env, NULL, &_txn, DB_READ_UNCOMMITTED); + + if (_result != 0) { + _txn = NULL; + _result = HA_ADMIN_FAILED; + goto error; + } + _local_txn = true; + } else { + _local_txn = false; + } + + _result = _share->key_file[0]->stat64(_share->key_file[0], _txn, &stat64); + if (_result != 0) { + _result = HA_ADMIN_FAILED; + goto cleanup; + } + _half_rows = stat64.bt_ndata / 2; + + for (ulonglong current_key = 0; + _result == HA_ADMIN_OK && current_key < _share->_keys; + current_key++) { + + _current_key = current_key; + _rows = _deleted_rows = _ticks = 0; + _result = analyze_key(&rec_per_key_part[total_key_parts]); + + if ((_result != 0 && _result != ETIME) || + (_result != 0 && _rows == 0 && _deleted_rows > 0)) { + _result = HA_ADMIN_FAILED; + } + if (_thd && (_result == HA_ADMIN_FAILED || + (double)_deleted_rows > + _delete_fraction * (_rows + _deleted_rows))) { + + char name[256]; int namelen; + namelen = + snprintf( + name, + sizeof(name), + "%s.%s.%s", + _share->database_name(), + _share->table_name(), + _share->_key_descriptors[_current_key]._name); + _thd->protocol->prepare_for_resend(); + _thd->protocol->store(name, namelen, system_charset_info); + _thd->protocol->store("analyze", 7, system_charset_info); + _thd->protocol->store("info", 4, system_charset_info); + char rowmsg[256]; + int rowmsglen; + rowmsglen = + snprintf( + rowmsg, + sizeof(rowmsg), + "rows processed %llu rows deleted %llu", + _rows, + _deleted_rows); + _thd->protocol->store(rowmsg, rowmsglen, system_charset_info); + _thd->protocol->write(); + + sql_print_information( + "tokudb analyze on %.*s %.*s", + namelen, + name, + rowmsglen, + rowmsg); + } + + total_key_parts += _share->_key_descriptors[_current_key]._parts; + } + if (_result == HA_ADMIN_OK) { + int error = + tokudb::set_card_in_status( + _share->status_block, + _txn, + total_key_parts, + rec_per_key_part); + if (error) + _result = HA_ADMIN_FAILED; + + _share->lock(); + _share->update_cardinality_counts(total_key_parts, rec_per_key_part); + _share->allow_auto_analysis(true); + _share->unlock(); + } + +cleanup: + if (_local_txn) { + if (_result == HA_ADMIN_OK) { + _txn->commit(_txn, 0); + } else { + _txn->abort(_txn); + } + _txn = NULL; + } + +error: + return; + +} +void standard_t::on_destroy() { + _share->lock(); + _share->allow_auto_analysis(false); + _share->unlock(); + _share->release(); +} +const char* standard_t::key() { + return _share->full_table_name(); +} +void standard_t::status( + char* database, + char* table, + char* type, + char* params, + char* status) { + + strcpy(database, _share->database_name()); + strcpy(table, _share->table_name()); + strcpy(type, "TOKUDB_ANALYZE_MODE_STANDARD"); + sprintf( + params, + "TOKUDB_ANALYZE_DELETE_FRACTION=%f; " + "TOKUDB_ANALYZE_TIME=%llu; TOKUDB_ANALYZE_THROTTLE=%llu;", + _delete_fraction, + _time_limit / tokudb::time::MICROSECONDS, + _throttle); + get_analyze_status(status); +} +bool standard_t::analyze_standard_cursor_callback( + void* extra, + uint64_t deleted_rows) { + standard_t* context = (standard_t*)extra; + return context->analyze_standard_cursor_callback(deleted_rows); +} +bool standard_t::analyze_standard_cursor_callback(uint64_t deleted_rows) { + _deleted_rows += deleted_rows; + _ticks += deleted_rows; + return analyze_key_progress() != 0; +} +void standard_t::get_analyze_status(char* msg) { + static const char* scan_direction_str[] = { + "not scanning", + "scanning forward", + "scanning backward", + "scan unknown" + }; + + const char* scan_direction = NULL; + switch (_scan_direction) { + case 0: scan_direction = scan_direction_str[0]; break; + case DB_NEXT: scan_direction = scan_direction_str[1]; break; + case DB_PREV: scan_direction = scan_direction_str[2]; break; + default: scan_direction = scan_direction_str[3]; break; + } - time_t t_now = time(0); - time_t t_limit = THDVAR(thd, analyze_time); - time_t t_start = extra->t_start; - if (t_limit > 0 && t_now - t_start > t_limit) - return ETIME; float progress_rows = 0.0; - TOKUDB_SHARE *share = extra->share; - if (share->rows > 0) - progress_rows = (float) rows / (float) share->rows; + if (_share->row_count() > 0) + progress_rows = (float) _rows / (float) _share->row_count(); float progress_time = 0.0; - if (t_limit > 0) - progress_time = (float) (t_now - t_start) / (float) t_limit; - char *write_status_msg = extra->write_status_msg; - TABLE_SHARE *table_share = extra->table_share; - sprintf(write_status_msg, "%.*s.%.*s.%s %u of %u %.lf%% rows %.lf%% time", - (int) table_share->db.length, table_share->db.str, - (int) table_share->table_name.length, table_share->table_name.str, - extra->key_name, extra->key_i, table_share->keys, progress_rows * 100.0, progress_time * 100.0); - thd_proc_info(thd, write_status_msg); + if (_time_limit > 0) + progress_time = (float) _key_elapsed_time / (float) _time_limit; + sprintf( + msg, + "analyze table standard %s.%s.%s %llu of %u %.lf%% rows %.lf%% time, " + "%s", + _share->database_name(), + _share->table_name(), + _share->_key_descriptors[_current_key]._name, + _current_key, + _share->_keys, + progress_rows * 100.0, + progress_time * 100.0, + scan_direction); +} +int standard_t::analyze_key_progress(void) { + if (_ticks > 1000) { + _ticks = 0; + uint64_t now = tokudb::time::microsec(); + _total_elapsed_time = now - _analyze_start; + _key_elapsed_time = now - _analyze_key_start; + if ((_thd && thd_killed(_thd)) || cancelled()) { + // client killed + return ER_ABORTING_CONNECTION; + } else if(_time_limit > 0 && + (uint64_t)_key_elapsed_time > _time_limit) { + // time limit reached + return ETIME; + } + + // report + if (_thd) { + char status[256]; + get_analyze_status(status); + thd_proc_info(_thd, status); + } + + // throttle + // given the throttle value, lets calculate the maximum number of rows + // we should have seen so far in a .1 sec resolution + if (_throttle > 0) { + uint64_t estimated_rows = _key_elapsed_time / 100000; + estimated_rows = estimated_rows * (_throttle / 10); + if (_rows + _deleted_rows > estimated_rows) { + // sleep for 1/10 of a second + tokudb::time::sleep_microsec(100000); + } + } + } return 0; } +int standard_t::analyze_key(uint64_t* rec_per_key_part) { + int error = 0; + DB* db = _share->key_file[_current_key]; + uint64_t num_key_parts = _share->_key_descriptors[_current_key]._parts; + uint64_t unique_rows[num_key_parts]; + bool is_unique = _share->_key_descriptors[_current_key]._is_unique; + DBC* cursor = NULL; + int close_error = 0; + DBT key, prev_key; + bool copy_key = false; + + _analyze_key_start = tokudb::time::microsec(); + _key_elapsed_time = 0; + _scan_direction = DB_NEXT; + + if (is_unique && num_key_parts == 1) { + // don't compute for unique keys with a single part. we already know + // the answer. + _rows = unique_rows[0] = 1; + goto done; + } + + for (uint64_t i = 0; i < num_key_parts; i++) + unique_rows[i] = 1; + + // stop looking when the entire dictionary was analyzed, or a + // cap on execution time was reached, or the analyze was killed. + while (1) { + if (cursor == NULL) { + error = db->cursor(db, _txn, &cursor, 0); + if (error != 0) + goto done; + + cursor->c_set_check_interrupt_callback( + cursor, + analyze_standard_cursor_callback, + this); + + memset(&key, 0, sizeof(DBT)); key.flags = DB_DBT_REALLOC; + memset(&prev_key, 0, sizeof(DBT)); prev_key.flags = DB_DBT_REALLOC; + copy_key = true; + } + + error = cursor->c_get(cursor, &key, 0, _scan_direction); + if (error != 0) { + if (error == DB_NOTFOUND || error == TOKUDB_INTERRUPTED) + error = 0; // not an error + break; + } else if (cancelled()) { + error = ER_ABORTING_CONNECTION; + break; + } + + _rows++; + _ticks++; + + // if copy_key is false at this pont, we have some value sitting in + // prev_key that we can compare to + // if the comparison reveals a unique key, we must set copy_key to true + // so the code following can copy he current key into prev_key for the + // next iteration + if (copy_key == false) { + // compare this key with the previous key. ignore + // appended PK for SK's. + // TODO if a prefix is different, then all larger keys + // that include the prefix are also different. + // TODO if we are comparing the entire primary key or + // the entire unique secondary key, then the cardinality + // must be 1, so we can avoid computing it. + for (uint64_t i = 0; i < num_key_parts; i++) { + int cmp = tokudb_cmp_dbt_key_parts(db, &prev_key, &key, i+1); + if (cmp != 0) { + unique_rows[i]++; + copy_key = true; + } + } + } + + // prev_key = key or prev_key is NULL + if (copy_key) { + prev_key.data = + tokudb::memory::realloc( + prev_key.data, + key.size, + MYF(MY_WME|MY_ZEROFILL|MY_FAE)); + assert_always(prev_key.data); + prev_key.size = key.size; + memcpy(prev_key.data, key.data, prev_key.size); + copy_key = false; + } + + error = analyze_key_progress(); + if (error == ETIME) { + error = 0; + break; + } else if (error) { + break; + } + + // if we have a time limit, are scanning forward and have exceed the + // _half_time and not passed the _half_rows number of the rows in the + // index: clean up the keys, close the cursor and reverse direction. + if (TOKUDB_UNLIKELY(_half_time > 0 && + _scan_direction == DB_NEXT && + _key_elapsed_time >= _half_time && + _rows < _half_rows)) { + + tokudb::memory::free(key.data); key.data = NULL; + tokudb::memory::free(prev_key.data); prev_key.data = NULL; + close_error = cursor->c_close(cursor); + assert_always(close_error == 0); + cursor = NULL; + _scan_direction = DB_PREV; + } + } + // cleanup + if (key.data) tokudb::memory::free(key.data); + if (prev_key.data) tokudb::memory::free(prev_key.data); + if (cursor) close_error = cursor->c_close(cursor); + assert_always(close_error == 0); + +done: + // return cardinality + for (uint64_t i = 0; i < num_key_parts; i++) { + rec_per_key_part[i] = _rows / unique_rows[i]; + } + return error; +} + +} // namespace analyze +} // namespace tokudb + int ha_tokudb::analyze(THD *thd, HA_CHECK_OPT *check_opt) { - TOKUDB_HANDLER_DBUG_ENTER("%s", share->table_name); + TOKUDB_HANDLER_DBUG_ENTER("%s", share->table_name()); + int result = HA_ADMIN_OK; + tokudb::sysvars::analyze_mode_t mode = tokudb::sysvars::analyze_mode(thd); + + switch (mode) { + case tokudb::sysvars::TOKUDB_ANALYZE_RECOUNT_ROWS: + result = share->analyze_recount_rows(thd, transaction); + break; + case tokudb::sysvars::TOKUDB_ANALYZE_STANDARD: + share->lock(); + result = share->analyze_standard(thd, transaction); + share->unlock(); + break; + case tokudb::sysvars::TOKUDB_ANALYZE_CANCEL: + share->cancel_background_jobs(); + break; + default: + break; // no-op + } + TOKUDB_HANDLER_DBUG_RETURN(result); +} + +int TOKUDB_SHARE::analyze_recount_rows(THD* thd,DB_TXN* txn) { + TOKUDB_HANDLER_DBUG_ENTER("%s", table_name()); + + assert_always(thd != NULL); + const char *orig_proc_info = tokudb_thd_get_proc_info(thd); - uint64_t rec_per_key[table_share->key_parts]; + int result = HA_ADMIN_OK; + + tokudb::analyze::recount_rows_t* job + = new tokudb::analyze::recount_rows_t(true, thd, this, txn); + assert_always(job != NULL); + + // job->destroy will drop the ref + addref(); + unlock(); + + bool ret = tokudb::background::_job_manager-> + run_job(job, tokudb::sysvars::analyze_in_background(thd)); + + if (!ret) { + job->destroy(); + delete job; + result = HA_ADMIN_FAILED; + } + + thd_proc_info(thd, orig_proc_info); + + TOKUDB_HANDLER_DBUG_RETURN(result); +} + +// on entry, if txn is !NULL, it is a user session invoking ANALYZE directly +// and no lock will be held on 'this', else if txn is NULL it is an auto and +// 'this' will be locked. +int TOKUDB_SHARE::analyze_standard(THD* thd, DB_TXN* txn) { + TOKUDB_HANDLER_DBUG_ENTER("%s", table_name()); + + assert_always(thd != NULL); + assert_debug(_mutex.is_owned_by_me() == true); + int result = HA_ADMIN_OK; // stub out analyze if optimize is remapped to alter recreate + analyze - if (thd_sql_command(thd) != SQLCOM_ANALYZE && thd_sql_command(thd) != SQLCOM_ALTER_TABLE) { + // when not auto analyze + if (txn && + thd_sql_command(thd) != SQLCOM_ANALYZE && + thd_sql_command(thd) != SQLCOM_ALTER_TABLE) { TOKUDB_HANDLER_DBUG_RETURN(result); } - DB_TXN *txn = transaction; - if (!txn) { + const char *orig_proc_info = tokudb_thd_get_proc_info(thd); + + tokudb::analyze::standard_t* job + = new tokudb::analyze::standard_t(txn == NULL ? false : true, thd, + this, txn); + assert_always(job != NULL); + + // akin to calling addref, but we know, right here, right now, everything + // in the share is set up, files open, etc... + // job->destroy will drop the ref + _use_count++; + + // don't want any autos kicking off while we are analyzing + disallow_auto_analysis(); + + unlock(); + + bool ret = + tokudb::background::_job_manager->run_job( + job, + tokudb::sysvars::analyze_in_background(thd)); + + if (!ret) { + job->destroy(); + delete job; result = HA_ADMIN_FAILED; } - uint total_key_parts = 0; - if (result == HA_ADMIN_OK) { - // compute cardinality for each key - for (uint i = 0; result == HA_ADMIN_OK && i < table_share->keys; i++) { - KEY *key_info = &table_share->key_info[i]; - uint64_t num_key_parts = get_key_parts(key_info); - const char *key_name = i == primary_key ? "primary" : key_info->name; - struct analyze_progress_extra analyze_progress_extra = { - thd, share, table_share, i, key_name, time(0), write_status_msg - }; - bool is_unique = false; - if (i == primary_key || (key_info->flags & HA_NOSAME)) - is_unique = true; - uint64_t rows = 0; - uint64_t deleted_rows = 0; - int error = tokudb::analyze_card(share->key_file[i], txn, is_unique, num_key_parts, &rec_per_key[total_key_parts], - tokudb_cmp_dbt_key_parts, analyze_progress, &analyze_progress_extra, - &rows, &deleted_rows); - sql_print_information("tokudb analyze %d %" PRIu64 " %" PRIu64, error, rows, deleted_rows); - if (error != 0 && error != ETIME) { - result = HA_ADMIN_FAILED; - } - if (error != 0 && rows == 0 && deleted_rows > 0) { - result = HA_ADMIN_FAILED; - } - double f = THDVAR(thd, analyze_delete_fraction); - if (result == HA_ADMIN_FAILED || (double) deleted_rows > f * (rows + deleted_rows)) { - char name[256]; int namelen; - namelen = snprintf(name, sizeof name, "%.*s.%.*s.%s", - (int) table_share->db.length, table_share->db.str, - (int) table_share->table_name.length, table_share->table_name.str, - key_name); - thd->protocol->prepare_for_resend(); - thd->protocol->store(name, namelen, system_charset_info); - thd->protocol->store("analyze", 7, system_charset_info); - thd->protocol->store("info", 4, system_charset_info); - char rowmsg[256]; int rowmsglen; - rowmsglen = snprintf(rowmsg, sizeof rowmsg, "rows processed %" PRIu64 " rows deleted %" PRIu64, rows, deleted_rows); - thd->protocol->store(rowmsg, rowmsglen, system_charset_info); - thd->protocol->write(); - - sql_print_information("tokudb analyze on %.*s %.*s", - namelen, name, rowmsglen, rowmsg); - } - if (tokudb_debug & TOKUDB_DEBUG_ANALYZE) { - char name[256]; int namelen; - namelen = snprintf(name, sizeof name, "%.*s.%.*s.%s", - (int) table_share->db.length, table_share->db.str, - (int) table_share->table_name.length, table_share->table_name.str, - key_name); - TOKUDB_HANDLER_TRACE("%.*s rows %" PRIu64 " deleted %" PRIu64, - namelen, name, rows, deleted_rows); - for (uint j = 0; j < num_key_parts; j++) - TOKUDB_HANDLER_TRACE("%lu", rec_per_key[total_key_parts+j]); - } - total_key_parts += num_key_parts; - } - } - if (result == HA_ADMIN_OK) { - int error = tokudb::set_card_in_status(share->status_block, txn, total_key_parts, rec_per_key); - if (error) - result = HA_ADMIN_FAILED; - } + + lock(); + thd_proc_info(thd, orig_proc_info); + TOKUDB_HANDLER_DBUG_RETURN(result); } + typedef struct hot_optimize_context { - THD *thd; + THD* thd; char* write_status_msg; - ha_tokudb *ha; + ha_tokudb* ha; uint progress_stage; uint current_table; uint num_tables; @@ -160,11 +828,18 @@ typedef struct hot_optimize_context { static int hot_optimize_progress_fun(void *extra, float progress) { HOT_OPTIMIZE_CONTEXT context = (HOT_OPTIMIZE_CONTEXT)extra; if (thd_killed(context->thd)) { - sprintf(context->write_status_msg, "The process has been killed, aborting hot optimize."); + sprintf( + context->write_status_msg, + "The process has been killed, aborting hot optimize."); return ER_ABORTING_CONNECTION; } float percentage = progress * 100; - sprintf(context->write_status_msg, "Optimization of index %u of %u about %.lf%% done", context->current_table + 1, context->num_tables, percentage); + sprintf( + context->write_status_msg, + "Optimization of index %u of %u about %.lf%% done", + context->current_table + 1, + context->num_tables, + percentage); thd_proc_info(context->thd, context->write_status_msg); #ifdef HA_TOKUDB_HAS_THD_PROGRESS if (context->progress_stage < context->current_table) { @@ -193,10 +868,10 @@ static int hot_optimize_progress_fun(void *extra, float progress) { } // flatten all DB's in this table, to do so, peform hot optimize on each db -int ha_tokudb::do_optimize(THD *thd) { - TOKUDB_HANDLER_DBUG_ENTER("%s", share->table_name); +int ha_tokudb::do_optimize(THD* thd) { + TOKUDB_HANDLER_DBUG_ENTER("%s", share->table_name()); int error = 0; - const char *orig_proc_info = tokudb_thd_get_proc_info(thd); + const char* orig_proc_info = tokudb_thd_get_proc_info(thd); uint curr_num_DBs = table->s->keys + tokudb_test(hidden_primary_key); #ifdef HA_TOKUDB_HAS_THD_PROGRESS @@ -207,10 +882,15 @@ int ha_tokudb::do_optimize(THD *thd) { // for each DB, run optimize and hot_optimize for (uint i = 0; i < curr_num_DBs; i++) { - // only optimize the index if it matches the optimize_index_name session variable - const char *optimize_index_name = THDVAR(thd, optimize_index_name); + // only optimize the index if it matches the optimize_index_name + // session variable + const char* optimize_index_name = + tokudb::sysvars::optimize_index_name(thd); if (optimize_index_name) { - const char *this_index_name = i >= table_share->keys ? "primary" : table_share->key_info[i].name; + const char* this_index_name = + i >= table_share->keys ? + "primary" : + table_share->key_info[i].name; if (strcasecmp(optimize_index_name, this_index_name) != 0) { continue; } @@ -229,11 +909,18 @@ int ha_tokudb::do_optimize(THD *thd) { hc.ha = this; hc.current_table = i; hc.num_tables = curr_num_DBs; - hc.progress_limit = THDVAR(thd, optimize_index_fraction); + hc.progress_limit = tokudb::sysvars::optimize_index_fraction(thd); hc.progress_last_time = toku_current_time_microsec(); - hc.throttle = THDVAR(thd, optimize_throttle); + hc.throttle = tokudb::sysvars::optimize_throttle(thd); uint64_t loops_run; - error = db->hot_optimize(db, NULL, NULL, hot_optimize_progress_fun, &hc, &loops_run); + error = + db->hot_optimize( + db, + NULL, + NULL, + hot_optimize_progress_fun, + &hc, + &loops_run); if (error) { goto cleanup; } @@ -248,8 +935,8 @@ cleanup: TOKUDB_HANDLER_DBUG_RETURN(error); } -int ha_tokudb::optimize(THD *thd, HA_CHECK_OPT *check_opt) { - TOKUDB_HANDLER_DBUG_ENTER("%s", share->table_name); +int ha_tokudb::optimize(THD* thd, HA_CHECK_OPT* check_opt) { + TOKUDB_HANDLER_DBUG_ENTER("%s", share->table_name()); int error; #if TOKU_OPTIMIZE_WITH_RECREATE error = HA_ADMIN_TRY_ALTER; @@ -260,23 +947,30 @@ int ha_tokudb::optimize(THD *thd, HA_CHECK_OPT *check_opt) { } struct check_context { - THD *thd; + THD* thd; }; -static int ha_tokudb_check_progress(void *extra, float progress) { - struct check_context *context = (struct check_context *) extra; +static int ha_tokudb_check_progress(void* extra, float progress) { + struct check_context* context = (struct check_context*)extra; int result = 0; if (thd_killed(context->thd)) result = ER_ABORTING_CONNECTION; return result; } -static void ha_tokudb_check_info(THD *thd, TABLE *table, const char *msg) { +static void ha_tokudb_check_info(THD* thd, TABLE* table, const char* msg) { if (thd->vio_ok()) { - char tablename[table->s->db.length + 1 + table->s->table_name.length + 1]; - snprintf(tablename, sizeof tablename, "%.*s.%.*s", - (int) table->s->db.length, table->s->db.str, - (int) table->s->table_name.length, table->s->table_name.str); + char tablename[ + table->s->db.length + 1 + + table->s->table_name.length + 1]; + snprintf( + tablename, + sizeof(tablename), + "%.*s.%.*s", + (int)table->s->db.length, + table->s->db.str, + (int)table->s->table_name.length, + table->s->table_name.str); thd->protocol->prepare_for_resend(); thd->protocol->store(tablename, strlen(tablename), system_charset_info); thd->protocol->store("check", 5, system_charset_info); @@ -286,9 +980,9 @@ static void ha_tokudb_check_info(THD *thd, TABLE *table, const char *msg) { } } -int ha_tokudb::check(THD *thd, HA_CHECK_OPT *check_opt) { - TOKUDB_HANDLER_DBUG_ENTER("%s", share->table_name); - const char *orig_proc_info = tokudb_thd_get_proc_info(thd); +int ha_tokudb::check(THD* thd, HA_CHECK_OPT* check_opt) { + TOKUDB_HANDLER_DBUG_ENTER("%s", share->table_name()); + const char* orig_proc_info = tokudb_thd_get_proc_info(thd); int result = HA_ADMIN_OK; int r; @@ -305,38 +999,72 @@ int ha_tokudb::check(THD *thd, HA_CHECK_OPT *check_opt) { result = HA_ADMIN_INTERNAL_ERROR; if (result == HA_ADMIN_OK) { uint32_t num_DBs = table_share->keys + tokudb_test(hidden_primary_key); - snprintf(write_status_msg, sizeof write_status_msg, "%s primary=%d num=%d", share->table_name, primary_key, num_DBs); - if (tokudb_debug & TOKUDB_DEBUG_CHECK) { + snprintf( + write_status_msg, + sizeof(write_status_msg), + "%s primary=%d num=%d", + share->table_name(), + primary_key, + num_DBs); + if (TOKUDB_UNLIKELY(TOKUDB_DEBUG_FLAGS(TOKUDB_DEBUG_CHECK))) { ha_tokudb_check_info(thd, table, write_status_msg); time_t now = time(0); char timebuf[32]; - TOKUDB_HANDLER_TRACE("%.24s %s", ctime_r(&now, timebuf), write_status_msg); + TOKUDB_HANDLER_TRACE( + "%.24s %s", + ctime_r(&now, timebuf), + write_status_msg); } for (uint i = 0; i < num_DBs; i++) { DB *db = share->key_file[i]; - const char *kname = i == primary_key ? "primary" : table_share->key_info[i].name; - snprintf(write_status_msg, sizeof write_status_msg, "%s key=%s %u", share->table_name, kname, i); + const char* kname = + i == primary_key ? "primary" : table_share->key_info[i].name; + snprintf( + write_status_msg, + sizeof(write_status_msg), + "%s key=%s %u", + share->table_name(), + kname, + i); thd_proc_info(thd, write_status_msg); - if (tokudb_debug & TOKUDB_DEBUG_CHECK) { + if (TOKUDB_UNLIKELY(TOKUDB_DEBUG_FLAGS(TOKUDB_DEBUG_CHECK))) { ha_tokudb_check_info(thd, table, write_status_msg); time_t now = time(0); char timebuf[32]; - TOKUDB_HANDLER_TRACE("%.24s %s", ctime_r(&now, timebuf), write_status_msg); + TOKUDB_HANDLER_TRACE( + "%.24s %s", + ctime_r(&now, timebuf), + write_status_msg); } struct check_context check_context = { thd }; - r = db->verify_with_progress(db, ha_tokudb_check_progress, &check_context, (tokudb_debug & TOKUDB_DEBUG_CHECK) != 0, keep_going); + r = db->verify_with_progress( + db, + ha_tokudb_check_progress, + &check_context, + (tokudb::sysvars::debug & TOKUDB_DEBUG_CHECK) != 0, + keep_going); if (r != 0) { char msg[32 + strlen(kname)]; sprintf(msg, "Corrupt %s", kname); ha_tokudb_check_info(thd, table, msg); } - snprintf(write_status_msg, sizeof write_status_msg, "%s key=%s %u result=%d", share->table_name, kname, i, r); + snprintf( + write_status_msg, + sizeof(write_status_msg), + "%s key=%s %u result=%d", + share->full_table_name(), + kname, + i, + r); thd_proc_info(thd, write_status_msg); - if (tokudb_debug & TOKUDB_DEBUG_CHECK) { + if (TOKUDB_UNLIKELY(TOKUDB_DEBUG_FLAGS(TOKUDB_DEBUG_CHECK))) { ha_tokudb_check_info(thd, table, write_status_msg); time_t now = time(0); char timebuf[32]; - TOKUDB_HANDLER_TRACE("%.24s %s", ctime_r(&now, timebuf), write_status_msg); + TOKUDB_HANDLER_TRACE( + "%.24s %s", + ctime_r(&now, timebuf), + write_status_msg); } if (result == HA_ADMIN_OK && r != 0) { result = HA_ADMIN_CORRUPT; diff --git a/storage/tokudb/ha_tokudb_alter_56.cc b/storage/tokudb/ha_tokudb_alter_56.cc index 724c588ed0b..d7b3bcb802a 100644 --- a/storage/tokudb/ha_tokudb_alter_56.cc +++ b/storage/tokudb/ha_tokudb_alter_56.cc @@ -67,7 +67,7 @@ public: } public: ulong handler_flags; - DB_TXN *alter_txn; + DB_TXN* alter_txn; bool add_index_changed; bool incremented_num_DBs, modified_DBs; bool drop_index_changed; @@ -79,81 +79,110 @@ public: bool expand_blob_update_needed; bool optimize_needed; Dynamic_array<uint> changed_fields; - KEY_AND_COL_INFO *table_kc_info; - KEY_AND_COL_INFO *altered_table_kc_info; + KEY_AND_COL_INFO* table_kc_info; + KEY_AND_COL_INFO* altered_table_kc_info; KEY_AND_COL_INFO altered_table_kc_info_base; }; // Debug function to print out an alter table operation -void ha_tokudb::print_alter_info(TABLE *altered_table, Alter_inplace_info *ha_alter_info) { - printf("***are keys of two tables same? %d\n", tables_have_same_keys(table, altered_table, false, false)); +void ha_tokudb::print_alter_info( + TABLE* altered_table, + Alter_inplace_info* ha_alter_info) { + + TOKUDB_TRACE( + "***are keys of two tables same? %d", + tables_have_same_keys(table, altered_table, false, false)); if (ha_alter_info->handler_flags) { - printf("***alter flags set ***\n"); + TOKUDB_TRACE("***alter flags set ***"); for (int i = 0; i < 32; i++) { if (ha_alter_info->handler_flags & (1 << i)) - printf("%d\n", i); + TOKUDB_TRACE("%d", i); } } - // everyone calculates data by doing some default_values - record[0], but I do not see why - // that is necessary - printf("******\n"); - printf("***orig table***\n"); + // everyone calculates data by doing some default_values - record[0], but + // I do not see why that is necessary + TOKUDB_TRACE("******"); + TOKUDB_TRACE("***orig table***"); for (uint i = 0; i < table->s->fields; i++) { // // make sure to use table->field, and NOT table->s->field // Field* curr_field = table->field[i]; uint null_offset = get_null_offset(table, curr_field); - printf( - "name: %s, types: %u %u, nullable: %d, null_offset: %d, is_null_field: %d, is_null %d, pack_length %u\n", - curr_field->field_name, - curr_field->real_type(), mysql_to_toku_type(curr_field), - curr_field->null_bit, - null_offset, - curr_field->real_maybe_null(), - curr_field->real_maybe_null() ? table->s->default_values[null_offset] & curr_field->null_bit : 0xffffffff, - curr_field->pack_length() - ); + TOKUDB_TRACE( + "name: %s, types: %u %u, nullable: %d, null_offset: %d, is_null_field: " + "%d, is_null %d, pack_length %u", + curr_field->field_name, + curr_field->real_type(), + mysql_to_toku_type(curr_field), + curr_field->null_bit, + null_offset, + curr_field->real_maybe_null(), + curr_field->real_maybe_null() ? + table->s->default_values[null_offset] & curr_field->null_bit : + 0xffffffff, + curr_field->pack_length()); } - printf("******\n"); - printf("***altered table***\n"); + TOKUDB_TRACE("******"); + TOKUDB_TRACE("***altered table***"); for (uint i = 0; i < altered_table->s->fields; i++) { Field* curr_field = altered_table->field[i]; uint null_offset = get_null_offset(altered_table, curr_field); - printf( - "name: %s, types: %u %u, nullable: %d, null_offset: %d, is_null_field: %d, is_null %d, pack_length %u\n", - curr_field->field_name, - curr_field->real_type(), mysql_to_toku_type(curr_field), - curr_field->null_bit, - null_offset, - curr_field->real_maybe_null(), - curr_field->real_maybe_null() ? altered_table->s->default_values[null_offset] & curr_field->null_bit : 0xffffffff, - curr_field->pack_length() - ); + TOKUDB_TRACE( + "name: %s, types: %u %u, nullable: %d, null_offset: %d, " + "is_null_field: %d, is_null %d, pack_length %u", + curr_field->field_name, + curr_field->real_type(), + mysql_to_toku_type(curr_field), + curr_field->null_bit, + null_offset, + curr_field->real_maybe_null(), + curr_field->real_maybe_null() ? + altered_table->s->default_values[null_offset] & + curr_field->null_bit : 0xffffffff, + curr_field->pack_length()); } - printf("******\n"); + TOKUDB_TRACE("******"); } -// Given two tables with equal number of fields, find all of the fields with different types -// and return the indexes of the different fields in the changed_fields array. This function ignores field -// name differences. -static int find_changed_fields(TABLE *table_a, TABLE *table_b, Dynamic_array<uint> &changed_fields) { +// Given two tables with equal number of fields, find all of the fields with +// different types and return the indexes of the different fields in the +// changed_fields array. This function ignores field name differences. +static int find_changed_fields( + TABLE* table_a, + TABLE* table_b, + Dynamic_array<uint>& changed_fields) { + for (uint i = 0; i < table_a->s->fields; i++) { - Field *field_a = table_a->field[i]; - Field *field_b = table_b->field[i]; + Field* field_a = table_a->field[i]; + Field* field_b = table_b->field[i]; if (!fields_are_same_type(field_a, field_b)) changed_fields.append(i); } return changed_fields.elements(); } -static bool change_length_is_supported(TABLE *table, TABLE *altered_table, Alter_inplace_info *ha_alter_info, tokudb_alter_ctx *ctx); - -static bool change_type_is_supported(TABLE *table, TABLE *altered_table, Alter_inplace_info *ha_alter_info, tokudb_alter_ctx *ctx); +static bool change_length_is_supported( + TABLE* table, + TABLE* altered_table, + Alter_inplace_info* ha_alter_info, + tokudb_alter_ctx* ctx); + +static bool change_type_is_supported( + TABLE* table, + TABLE* altered_table, + Alter_inplace_info* ha_alter_info, + tokudb_alter_ctx* ctx); + +// The ha_alter_info->handler_flags can not be trusted. +// This function maps the bogus handler flags to something we like. +static ulong fix_handler_flags( + THD* thd, + TABLE* table, + TABLE* altered_table, + Alter_inplace_info* ha_alter_info) { -// The ha_alter_info->handler_flags can not be trusted. This function maps the bogus handler flags to something we like. -static ulong fix_handler_flags(THD *thd, TABLE *table, TABLE *altered_table, Alter_inplace_info *ha_alter_info) { ulong handler_flags = ha_alter_info->handler_flags; #if 100000 <= MYSQL_VERSION_ID && MYSQL_VERSION_ID <= 100099 @@ -162,23 +191,33 @@ static ulong fix_handler_flags(THD *thd, TABLE *table, TABLE *altered_table, Alt #endif // workaround for fill_alter_inplace_info bug (#5193) - // the function erroneously sets the ADD_INDEX and DROP_INDEX flags for a column addition that does not - // change the keys. the following code turns the ADD_INDEX and DROP_INDEX flags so that we can do hot - // column addition later. - if (handler_flags & (Alter_inplace_info::ADD_COLUMN + Alter_inplace_info::DROP_COLUMN)) { - if (handler_flags & (Alter_inplace_info::ADD_INDEX + Alter_inplace_info::DROP_INDEX)) { - if (tables_have_same_keys(table, altered_table, THDVAR(thd, alter_print_error) != 0, false)) { - handler_flags &= ~(Alter_inplace_info::ADD_INDEX + Alter_inplace_info::DROP_INDEX); + // the function erroneously sets the ADD_INDEX and DROP_INDEX flags for a + // column addition that does not change the keys. + // the following code turns the ADD_INDEX and DROP_INDEX flags so that + // we can do hot column addition later. + if (handler_flags & + (Alter_inplace_info::ADD_COLUMN + Alter_inplace_info::DROP_COLUMN)) { + if (handler_flags & + (Alter_inplace_info::ADD_INDEX + Alter_inplace_info::DROP_INDEX)) { + if (tables_have_same_keys( + table, + altered_table, + tokudb::sysvars::alter_print_error(thd) != 0, false)) { + handler_flags &= + ~(Alter_inplace_info::ADD_INDEX + + Alter_inplace_info::DROP_INDEX); } } } - // always allow rename table + any other operation, so turn off the rename flag + // always allow rename table + any other operation, so turn off the + // rename flag if (handler_flags & Alter_inplace_info::TOKU_ALTER_RENAME) { handler_flags &= ~Alter_inplace_info::TOKU_ALTER_RENAME; } - // ALTER_COLUMN_TYPE may be set when no columns have been changed, so turn off the flag + // ALTER_COLUMN_TYPE may be set when no columns have been changed, + // so turn off the flag if (handler_flags & Alter_inplace_info::ALTER_COLUMN_TYPE) { if (all_fields_are_same_type(table, altered_table)) { handler_flags &= ~Alter_inplace_info::ALTER_COLUMN_TYPE; @@ -191,9 +230,10 @@ static ulong fix_handler_flags(THD *thd, TABLE *table, TABLE *altered_table, Alt // Require that there is no intersection of add and drop names. static bool is_disjoint_add_drop(Alter_inplace_info *ha_alter_info) { for (uint d = 0; d < ha_alter_info->index_drop_count; d++) { - KEY *drop_key = ha_alter_info->index_drop_buffer[d]; + KEY* drop_key = ha_alter_info->index_drop_buffer[d]; for (uint a = 0; a < ha_alter_info->index_add_count; a++) { - KEY *add_key = &ha_alter_info->key_info_buffer[ha_alter_info->index_add_buffer[a]]; + KEY* add_key = + &ha_alter_info->key_info_buffer[ha_alter_info->index_add_buffer[a]]; if (strcmp(drop_key->name, add_key->name) == 0) { return false; } @@ -202,196 +242,294 @@ static bool is_disjoint_add_drop(Alter_inplace_info *ha_alter_info) { return true; } -// Return true if some bit in mask is set and no bit in ~mask is set, otherwise return false. +// Return true if some bit in mask is set and no bit in ~mask is set, +// otherwise return false. static bool only_flags(ulong bits, ulong mask) { return (bits & mask) != 0 && (bits & ~mask) == 0; } -// Check if an alter table operation on this table and described by the alter table parameters is supported inplace -// and if so, what type of locking is needed to execute it. -// return values: +// Check if an alter table operation on this table and described by the alter +// table parameters is supported inplace and if so, what type of locking is +// needed to execute it. return values: + +// HA_ALTER_INPLACE_NOT_SUPPORTED: alter operation is not supported as an +// inplace operation, a table copy is required -// HA_ALTER_INPLACE_NOT_SUPPORTED: alter operation is not supported as an inplace operation, a table copy is required // HA_ALTER_ERROR: the alter table operation should fail // HA_ALTER_INPLACE_EXCLUSIVE_LOCK: prepare and alter runs with MDL X -// HA_ALTER_INPLACE_SHARED_LOCK_AFTER_PREPARE: prepare runs with MDL X, alter runs with MDL SNW -// HA_ALTER_INPLACE_SHARED_LOCK: prepare and alter methods called with MDL SNW, concurrent reads, no writes +// HA_ALTER_INPLACE_SHARED_LOCK_AFTER_PREPARE: prepare runs with MDL X, +// alter runs with MDL SNW + +// HA_ALTER_INPLACE_SHARED_LOCK: prepare and alter methods called with MDL SNW, +// concurrent reads, no writes + +// HA_ALTER_INPLACE_NO_LOCK_AFTER_PREPARE: prepare runs with MDL X, +// alter runs with MDL SW + +// HA_ALTER_INPLACE_NO_LOCK: prepare and alter methods called with MDL SW, +// concurrent reads, writes. +// must set WRITE_ALLOW_WRITE lock type in the external lock method to avoid +// deadlocks with the MDL lock and the table lock +enum_alter_inplace_result ha_tokudb::check_if_supported_inplace_alter( + TABLE* altered_table, + Alter_inplace_info* ha_alter_info) { -// HA_ALTER_INPLACE_NO_LOCK_AFTER_PREPARE: prepare runs with MDL X, alter runs with MDL SW -// HA_ALTER_INPLACE_NO_LOCK: prepare and alter methods called with MDL SW, concurrent reads, writes. -// must set WRITE_ALLOW_WRITE lock type in the external lock method to avoid deadlocks -// with the MDL lock and the table lock -enum_alter_inplace_result ha_tokudb::check_if_supported_inplace_alter(TABLE *altered_table, Alter_inplace_info *ha_alter_info) { TOKUDB_HANDLER_DBUG_ENTER(""); - if (tokudb_debug & TOKUDB_DEBUG_ALTER_TABLE) { + if (TOKUDB_UNLIKELY(TOKUDB_DEBUG_FLAGS(TOKUDB_DEBUG_ALTER_TABLE))) { print_alter_info(altered_table, ha_alter_info); } - enum_alter_inplace_result result = HA_ALTER_INPLACE_NOT_SUPPORTED; // default is NOT inplace - THD *thd = ha_thd(); + // default is NOT inplace + enum_alter_inplace_result result = HA_ALTER_INPLACE_NOT_SUPPORTED; + THD* thd = ha_thd(); // setup context - tokudb_alter_ctx *ctx = new tokudb_alter_ctx; + tokudb_alter_ctx* ctx = new tokudb_alter_ctx; ha_alter_info->handler_ctx = ctx; - ctx->handler_flags = fix_handler_flags(thd, table, altered_table, ha_alter_info); + ctx->handler_flags = + fix_handler_flags(thd, table, altered_table, ha_alter_info); ctx->table_kc_info = &share->kc_info; ctx->altered_table_kc_info = &ctx->altered_table_kc_info_base; memset(ctx->altered_table_kc_info, 0, sizeof (KEY_AND_COL_INFO)); - if (get_disable_hot_alter(thd)) { + if (tokudb::sysvars::disable_hot_alter(thd)) { ; // do nothing - } else - // add or drop index - if (only_flags(ctx->handler_flags, Alter_inplace_info::DROP_INDEX + Alter_inplace_info::DROP_UNIQUE_INDEX + - Alter_inplace_info::ADD_INDEX + Alter_inplace_info::ADD_UNIQUE_INDEX)) { + } else if (only_flags( + ctx->handler_flags, + Alter_inplace_info::DROP_INDEX + + Alter_inplace_info::DROP_UNIQUE_INDEX + + Alter_inplace_info::ADD_INDEX + + Alter_inplace_info::ADD_UNIQUE_INDEX)) { + // add or drop index if (table->s->null_bytes == altered_table->s->null_bytes && - (ha_alter_info->index_add_count > 0 || ha_alter_info->index_drop_count > 0) && - !tables_have_same_keys(table, altered_table, THDVAR(thd, alter_print_error) != 0, false) && + (ha_alter_info->index_add_count > 0 || + ha_alter_info->index_drop_count > 0) && + !tables_have_same_keys( + table, + altered_table, + tokudb::sysvars::alter_print_error(thd) != 0, false) && is_disjoint_add_drop(ha_alter_info)) { - if (ctx->handler_flags & (Alter_inplace_info::DROP_INDEX + Alter_inplace_info::DROP_UNIQUE_INDEX)) { - // the fractal tree can not handle dropping an index concurrent with querying with the index. + if (ctx->handler_flags & + (Alter_inplace_info::DROP_INDEX + + Alter_inplace_info::DROP_UNIQUE_INDEX)) { + // the fractal tree can not handle dropping an index concurrent + // with querying with the index. // we grab an exclusive MDL for the drop index. result = HA_ALTER_INPLACE_EXCLUSIVE_LOCK; } else { result = HA_ALTER_INPLACE_SHARED_LOCK_AFTER_PREPARE; - // someday, allow multiple hot indexes via alter table add key. don't forget to change the store_lock function. - // for now, hot indexing is only supported via session variable with the create index sql command - if (ha_alter_info->index_add_count == 1 && ha_alter_info->index_drop_count == 0 && // only one add or drop - ctx->handler_flags == Alter_inplace_info::ADD_INDEX && // must be add index not add unique index - thd_sql_command(thd) == SQLCOM_CREATE_INDEX && // must be a create index command - get_create_index_online(thd)) { // must be enabled - // external_lock set WRITE_ALLOW_WRITE which allows writes concurrent with the index creation + // someday, allow multiple hot indexes via alter table add key. + // don't forget to change the store_lock function. + // for now, hot indexing is only supported via session variable + // with the create index sql command + if (ha_alter_info->index_add_count == 1 && + // only one add or drop + ha_alter_info->index_drop_count == 0 && + // must be add index not add unique index + ctx->handler_flags == Alter_inplace_info::ADD_INDEX && + // must be a create index command + thd_sql_command(thd) == SQLCOM_CREATE_INDEX && + // must be enabled + tokudb::sysvars::create_index_online(thd)) { + // external_lock set WRITE_ALLOW_WRITE which allows writes + // concurrent with the index creation result = HA_ALTER_INPLACE_NO_LOCK_AFTER_PREPARE; } } } - } else - // column default - if (only_flags(ctx->handler_flags, Alter_inplace_info::ALTER_COLUMN_DEFAULT)) { + } else if (only_flags( + ctx->handler_flags, + Alter_inplace_info::ALTER_COLUMN_DEFAULT)) { + // column default if (table->s->null_bytes == altered_table->s->null_bytes) result = HA_ALTER_INPLACE_EXCLUSIVE_LOCK; - } else - // column rename - if (ctx->handler_flags & Alter_inplace_info::ALTER_COLUMN_NAME && - only_flags(ctx->handler_flags, Alter_inplace_info::ALTER_COLUMN_NAME + Alter_inplace_info::ALTER_COLUMN_DEFAULT)) { - // we have identified a possible column rename, + } else if (ctx->handler_flags & Alter_inplace_info::ALTER_COLUMN_NAME && + only_flags( + ctx->handler_flags, + Alter_inplace_info::ALTER_COLUMN_NAME + + Alter_inplace_info::ALTER_COLUMN_DEFAULT)) { + // column rename + // we have identified a possible column rename, // but let's do some more checks - + // we will only allow an hcr if there are no changes // in column positions (ALTER_COLUMN_ORDER is not set) - + // now need to verify that one and only one column // has changed only its name. If we find anything to // the contrary, we don't allow it, also check indexes if (table->s->null_bytes == altered_table->s->null_bytes) { - bool cr_supported = column_rename_supported(table, altered_table, (ctx->handler_flags & Alter_inplace_info::ALTER_COLUMN_ORDER) != 0); + bool cr_supported = + column_rename_supported( + table, + altered_table, + (ctx->handler_flags & + Alter_inplace_info::ALTER_COLUMN_ORDER) != 0); if (cr_supported) result = HA_ALTER_INPLACE_EXCLUSIVE_LOCK; } - } else - // add column - if (ctx->handler_flags & Alter_inplace_info::ADD_COLUMN && - only_flags(ctx->handler_flags, Alter_inplace_info::ADD_COLUMN + Alter_inplace_info::ALTER_COLUMN_ORDER) && - setup_kc_info(altered_table, ctx->altered_table_kc_info) == 0) { - + } else if (ctx->handler_flags & Alter_inplace_info::ADD_COLUMN && + only_flags( + ctx->handler_flags, + Alter_inplace_info::ADD_COLUMN + + Alter_inplace_info::ALTER_COLUMN_ORDER) && + setup_kc_info(altered_table, ctx->altered_table_kc_info) == 0) { + + // add column uint32_t added_columns[altered_table->s->fields]; uint32_t num_added_columns = 0; - int r = find_changed_columns(added_columns, &num_added_columns, table, altered_table); + int r = + find_changed_columns( + added_columns, + &num_added_columns, + table, + altered_table); if (r == 0) { - if (tokudb_debug & TOKUDB_DEBUG_ALTER_TABLE) { + if (TOKUDB_UNLIKELY(TOKUDB_DEBUG_FLAGS(TOKUDB_DEBUG_ALTER_TABLE))) { for (uint32_t i = 0; i < num_added_columns; i++) { uint32_t curr_added_index = added_columns[i]; - Field* curr_added_field = altered_table->field[curr_added_index]; - printf("Added column: index %d, name %s\n", curr_added_index, curr_added_field->field_name); + Field* curr_added_field = + altered_table->field[curr_added_index]; + TOKUDB_TRACE( + "Added column: index %d, name %s", + curr_added_index, + curr_added_field->field_name); } } result = HA_ALTER_INPLACE_EXCLUSIVE_LOCK; } - } else - // drop column - if (ctx->handler_flags & Alter_inplace_info::DROP_COLUMN && - only_flags(ctx->handler_flags, Alter_inplace_info::DROP_COLUMN + Alter_inplace_info::ALTER_COLUMN_ORDER) && - setup_kc_info(altered_table, ctx->altered_table_kc_info) == 0) { - + } else if (ctx->handler_flags & Alter_inplace_info::DROP_COLUMN && + only_flags( + ctx->handler_flags, + Alter_inplace_info::DROP_COLUMN + + Alter_inplace_info::ALTER_COLUMN_ORDER) && + setup_kc_info(altered_table, ctx->altered_table_kc_info) == 0) { + + // drop column uint32_t dropped_columns[table->s->fields]; uint32_t num_dropped_columns = 0; - int r = find_changed_columns(dropped_columns, &num_dropped_columns, altered_table, table); + int r = + find_changed_columns( + dropped_columns, + &num_dropped_columns, + altered_table, + table); if (r == 0) { - if (tokudb_debug & TOKUDB_DEBUG_ALTER_TABLE) { + if (TOKUDB_UNLIKELY(TOKUDB_DEBUG_FLAGS(TOKUDB_DEBUG_ALTER_TABLE))) { for (uint32_t i = 0; i < num_dropped_columns; i++) { uint32_t curr_dropped_index = dropped_columns[i]; Field* curr_dropped_field = table->field[curr_dropped_index]; - printf("Dropped column: index %d, name %s\n", curr_dropped_index, curr_dropped_field->field_name); + TOKUDB_TRACE( + "Dropped column: index %d, name %s", + curr_dropped_index, + curr_dropped_field->field_name); } } result = HA_ALTER_INPLACE_EXCLUSIVE_LOCK; } - } else - // change column length - if ((ctx->handler_flags & Alter_inplace_info::ALTER_COLUMN_EQUAL_PACK_LENGTH) && - only_flags(ctx->handler_flags, Alter_inplace_info::ALTER_COLUMN_EQUAL_PACK_LENGTH + Alter_inplace_info::ALTER_COLUMN_DEFAULT) && - table->s->fields == altered_table->s->fields && - find_changed_fields(table, altered_table, ctx->changed_fields) > 0 && - setup_kc_info(altered_table, ctx->altered_table_kc_info) == 0) { - - if (change_length_is_supported(table, altered_table, ha_alter_info, ctx)) { + } else if ((ctx->handler_flags & + Alter_inplace_info::ALTER_COLUMN_EQUAL_PACK_LENGTH) && + only_flags( + ctx->handler_flags, + Alter_inplace_info::ALTER_COLUMN_EQUAL_PACK_LENGTH + + Alter_inplace_info::ALTER_COLUMN_DEFAULT) && + table->s->fields == altered_table->s->fields && + find_changed_fields( + table, + altered_table, + ctx->changed_fields) > 0 && + setup_kc_info(altered_table, ctx->altered_table_kc_info) == 0) { + + // change column length + if (change_length_is_supported( + table, + altered_table, + ha_alter_info, ctx)) { result = HA_ALTER_INPLACE_EXCLUSIVE_LOCK; } - } else - // change column type - if ((ctx->handler_flags & Alter_inplace_info::ALTER_COLUMN_TYPE) && - only_flags(ctx->handler_flags, Alter_inplace_info::ALTER_COLUMN_TYPE + Alter_inplace_info::ALTER_COLUMN_DEFAULT) && - table->s->fields == altered_table->s->fields && - find_changed_fields(table, altered_table, ctx->changed_fields) > 0 && - setup_kc_info(altered_table, ctx->altered_table_kc_info) == 0) { - - if (change_type_is_supported(table, altered_table, ha_alter_info, ctx)) { + } else if ((ctx->handler_flags & Alter_inplace_info::ALTER_COLUMN_TYPE) && + only_flags( + ctx->handler_flags, + Alter_inplace_info::ALTER_COLUMN_TYPE + + Alter_inplace_info::ALTER_COLUMN_DEFAULT) && + table->s->fields == altered_table->s->fields && + find_changed_fields( + table, + altered_table, + ctx->changed_fields) > 0 && + setup_kc_info(altered_table, ctx->altered_table_kc_info) == 0) { + + // change column type + if (change_type_is_supported( + table, + altered_table, + ha_alter_info, ctx)) { result = HA_ALTER_INPLACE_EXCLUSIVE_LOCK; } - } else - if (only_flags(ctx->handler_flags, Alter_inplace_info::CHANGE_CREATE_OPTION)) { - HA_CREATE_INFO *create_info = ha_alter_info->create_info; + } else if (only_flags( + ctx->handler_flags, + Alter_inplace_info::CHANGE_CREATE_OPTION)) { + + HA_CREATE_INFO* create_info = ha_alter_info->create_info; #if TOKU_INCLUDE_OPTION_STRUCTS // set the USED_ROW_FORMAT flag for use later in this file for changes in the table's // compression - if (create_info->option_struct->row_format != table_share->option_struct->row_format) + if (create_info->option_struct->row_format != + table_share->option_struct->row_format) create_info->used_fields |= HA_CREATE_USED_ROW_FORMAT; #endif // alter auto_increment if (only_flags(create_info->used_fields, HA_CREATE_USED_AUTO)) { // do a sanity check that the table is what we think it is - if (tables_have_same_keys_and_columns(table, altered_table, THDVAR(thd, alter_print_error) != 0)) { + if (tables_have_same_keys_and_columns( + table, + altered_table, + tokudb::sysvars::alter_print_error(thd) != 0)) { result = HA_ALTER_INPLACE_EXCLUSIVE_LOCK; } - } - // alter row_format - else if (only_flags(create_info->used_fields, HA_CREATE_USED_ROW_FORMAT)) { + } else if (only_flags( + create_info->used_fields, + HA_CREATE_USED_ROW_FORMAT)) { + // alter row_format // do a sanity check that the table is what we think it is - if (tables_have_same_keys_and_columns(table, altered_table, THDVAR(thd, alter_print_error) != 0)) { + if (tables_have_same_keys_and_columns( + table, + altered_table, + tokudb::sysvars::alter_print_error(thd) != 0)) { result = HA_ALTER_INPLACE_EXCLUSIVE_LOCK; } } - } + } #if TOKU_OPTIMIZE_WITH_RECREATE - else if (only_flags(ctx->handler_flags, Alter_inplace_info::RECREATE_TABLE + Alter_inplace_info::ALTER_COLUMN_DEFAULT)) { + else if (only_flags( + ctx->handler_flags, + Alter_inplace_info::RECREATE_TABLE + + Alter_inplace_info::ALTER_COLUMN_DEFAULT)) { ctx->optimize_needed = true; result = HA_ALTER_INPLACE_NO_LOCK_AFTER_PREPARE; } #endif - if (result != HA_ALTER_INPLACE_NOT_SUPPORTED && table->s->null_bytes != altered_table->s->null_bytes && - (tokudb_debug & TOKUDB_DEBUG_ALTER_TABLE)) { + if (TOKUDB_UNLIKELY(TOKUDB_DEBUG_FLAGS(TOKUDB_DEBUG_ALTER_TABLE)) && + result != HA_ALTER_INPLACE_NOT_SUPPORTED && + table->s->null_bytes != altered_table->s->null_bytes) { + TOKUDB_HANDLER_TRACE("q %s", thd->query()); - TOKUDB_HANDLER_TRACE("null bytes %u -> %u", table->s->null_bytes, altered_table->s->null_bytes); + TOKUDB_HANDLER_TRACE( + "null bytes %u -> %u", + table->s->null_bytes, + altered_table->s->null_bytes); } - // turn a not supported result into an error if the slow alter table (copy) is disabled - if (result == HA_ALTER_INPLACE_NOT_SUPPORTED && get_disable_slow_alter(thd)) { + // turn a not supported result into an error if the slow alter table + // (copy) is disabled + if (result == HA_ALTER_INPLACE_NOT_SUPPORTED && + tokudb::sysvars::disable_slow_alter(thd)) { print_error(HA_ERR_UNSUPPORTED, MYF(0)); result = HA_ALTER_ERROR; } @@ -400,46 +538,77 @@ enum_alter_inplace_result ha_tokudb::check_if_supported_inplace_alter(TABLE *alt } // Prepare for the alter operations -bool ha_tokudb::prepare_inplace_alter_table(TABLE *altered_table, Alter_inplace_info *ha_alter_info) { +bool ha_tokudb::prepare_inplace_alter_table( + TABLE* altered_table, + Alter_inplace_info* ha_alter_info) { + TOKUDB_HANDLER_DBUG_ENTER(""); - tokudb_alter_ctx *ctx = static_cast<tokudb_alter_ctx *>(ha_alter_info->handler_ctx); - assert(transaction); // transaction must exist after table is locked + tokudb_alter_ctx* ctx = + static_cast<tokudb_alter_ctx*>(ha_alter_info->handler_ctx); + assert_always(transaction); // transaction must exist after table is locked ctx->alter_txn = transaction; bool result = false; // success DBUG_RETURN(result); } // Execute the alter operations. -bool ha_tokudb::inplace_alter_table(TABLE *altered_table, Alter_inplace_info *ha_alter_info) { +bool ha_tokudb::inplace_alter_table( + TABLE* altered_table, + Alter_inplace_info* ha_alter_info) { + TOKUDB_HANDLER_DBUG_ENTER(""); int error = 0; - tokudb_alter_ctx *ctx = static_cast<tokudb_alter_ctx *>(ha_alter_info->handler_ctx); - HA_CREATE_INFO *create_info = ha_alter_info->create_info; - - if (error == 0 && (ctx->handler_flags & (Alter_inplace_info::DROP_INDEX + Alter_inplace_info::DROP_UNIQUE_INDEX))) { + tokudb_alter_ctx* ctx = + static_cast<tokudb_alter_ctx*>(ha_alter_info->handler_ctx); + HA_CREATE_INFO* create_info = ha_alter_info->create_info; + + // this should be enough to handle locking as the higher level MDL + // on this table should prevent any new analyze tasks. + share->cancel_background_jobs(); + + if (error == 0 && + (ctx->handler_flags & + (Alter_inplace_info::DROP_INDEX + + Alter_inplace_info::DROP_UNIQUE_INDEX))) { error = alter_table_drop_index(altered_table, ha_alter_info); } - if (error == 0 && (ctx->handler_flags & (Alter_inplace_info::ADD_INDEX + Alter_inplace_info::ADD_UNIQUE_INDEX))) { + if (error == 0 && + (ctx->handler_flags & + (Alter_inplace_info::ADD_INDEX + + Alter_inplace_info::ADD_UNIQUE_INDEX))) { error = alter_table_add_index(altered_table, ha_alter_info); } - if (error == 0 && (ctx->handler_flags & (Alter_inplace_info::ADD_COLUMN + Alter_inplace_info::DROP_COLUMN))) { + if (error == 0 && + (ctx->handler_flags & + (Alter_inplace_info::ADD_COLUMN + + Alter_inplace_info::DROP_COLUMN))) { error = alter_table_add_or_drop_column(altered_table, ha_alter_info); } - if (error == 0 && (ctx->handler_flags & Alter_inplace_info::CHANGE_CREATE_OPTION) && (create_info->used_fields & HA_CREATE_USED_AUTO)) { - error = write_auto_inc_create(share->status_block, create_info->auto_increment_value, ctx->alter_txn); + if (error == 0 && + (ctx->handler_flags & Alter_inplace_info::CHANGE_CREATE_OPTION) && + (create_info->used_fields & HA_CREATE_USED_AUTO)) { + error = write_auto_inc_create( + share->status_block, + create_info->auto_increment_value, + ctx->alter_txn); } - if (error == 0 && (ctx->handler_flags & Alter_inplace_info::CHANGE_CREATE_OPTION) && (create_info->used_fields & HA_CREATE_USED_ROW_FORMAT)) { + if (error == 0 && + (ctx->handler_flags & Alter_inplace_info::CHANGE_CREATE_OPTION) && + (create_info->used_fields & HA_CREATE_USED_ROW_FORMAT)) { // Get the current compression DB *db = share->key_file[0]; error = db->get_compression_method(db, &ctx->orig_compression_method); - assert(error == 0); + assert_always(error == 0); // Set the new compression #if TOKU_INCLUDE_OPTION_STRUCTS - toku_compression_method method = row_format_to_toku_compression_method((srv_row_format_t) create_info->option_struct->row_format); + toku_compression_method method = + row_format_to_toku_compression_method( + (tokudb::sysvars::row_format_t)create_info->option_struct->row_format); #else - toku_compression_method method = row_type_to_toku_compression_method(create_info->row_type); + toku_compression_method method = + row_type_to_toku_compression_method(create_info->row_type); #endif uint32_t curr_num_DBs = table->s->keys + tokudb_test(hidden_primary_key); for (uint32_t i = 0; i < curr_num_DBs; i++) { @@ -457,13 +626,19 @@ bool ha_tokudb::inplace_alter_table(TABLE *altered_table, Alter_inplace_info *ha error = alter_table_expand_columns(altered_table, ha_alter_info); if (error == 0 && ctx->expand_varchar_update_needed) - error = alter_table_expand_varchar_offsets(altered_table, ha_alter_info); + error = alter_table_expand_varchar_offsets( + altered_table, + ha_alter_info); if (error == 0 && ctx->expand_blob_update_needed) error = alter_table_expand_blobs(altered_table, ha_alter_info); if (error == 0 && ctx->reset_card) { - error = tokudb::alter_card(share->status_block, ctx->alter_txn, table->s, altered_table->s); + error = tokudb::alter_card( + share->status_block, + ctx->alter_txn, + table->s, + altered_table->s); } if (error == 0 && ctx->optimize_needed) { error = do_optimize(ha_thd()); @@ -472,11 +647,15 @@ bool ha_tokudb::inplace_alter_table(TABLE *altered_table, Alter_inplace_info *ha #if (50600 <= MYSQL_VERSION_ID && MYSQL_VERSION_ID <= 50699) || \ (50700 <= MYSQL_VERSION_ID && MYSQL_VERSION_ID <= 50799) #if WITH_PARTITION_STORAGE_ENGINE - if (error == 0 && (TOKU_PARTITION_WRITE_FRM_DATA || altered_table->part_info == NULL)) { + if (error == 0 && + (TOKU_PARTITION_WRITE_FRM_DATA || altered_table->part_info == NULL)) { #else if (error == 0) { #endif - error = write_frm_data(share->status_block, ctx->alter_txn, altered_table->s->path.str); + error = write_frm_data( + share->status_block, + ctx->alter_txn, + altered_table->s->path.str); } #endif @@ -489,20 +668,34 @@ bool ha_tokudb::inplace_alter_table(TABLE *altered_table, Alter_inplace_info *ha DBUG_RETURN(result); } -int ha_tokudb::alter_table_add_index(TABLE *altered_table, Alter_inplace_info *ha_alter_info) { +int ha_tokudb::alter_table_add_index( + TABLE* altered_table, + Alter_inplace_info* ha_alter_info) { // sort keys in add index order - KEY *key_info = (KEY*) tokudb_my_malloc(sizeof (KEY) * ha_alter_info->index_add_count, MYF(MY_WME)); + KEY* key_info = (KEY*)tokudb::memory::malloc( + sizeof(KEY) * ha_alter_info->index_add_count, + MYF(MY_WME)); for (uint i = 0; i < ha_alter_info->index_add_count; i++) { KEY *key = &key_info[i]; *key = ha_alter_info->key_info_buffer[ha_alter_info->index_add_buffer[i]]; - for (KEY_PART_INFO *key_part= key->key_part; key_part < key->key_part + get_key_parts(key); key_part++) + for (KEY_PART_INFO* key_part = key->key_part; + key_part < key->key_part + get_key_parts(key); + key_part++) { key_part->field = table->field[key_part->fieldnr]; + } } - tokudb_alter_ctx *ctx = static_cast<tokudb_alter_ctx *>(ha_alter_info->handler_ctx); + tokudb_alter_ctx* ctx = + static_cast<tokudb_alter_ctx*>(ha_alter_info->handler_ctx); ctx->add_index_changed = true; - int error = tokudb_add_index(table, key_info, ha_alter_info->index_add_count, ctx->alter_txn, &ctx->incremented_num_DBs, &ctx->modified_DBs); + int error = tokudb_add_index( + table, + key_info, + ha_alter_info->index_add_count, + ctx->alter_txn, + &ctx->incremented_num_DBs, + &ctx->modified_DBs); if (error == HA_ERR_FOUND_DUPP_KEY) { // hack for now, in case of duplicate key error, // because at the moment we cannot display the right key @@ -511,7 +704,7 @@ int ha_tokudb::alter_table_add_index(TABLE *altered_table, Alter_inplace_info *h last_dup_key = MAX_KEY; } - tokudb_my_free(key_info); + tokudb::memory::free(key_info); if (error == 0) ctx->reset_card = true; @@ -519,7 +712,11 @@ int ha_tokudb::alter_table_add_index(TABLE *altered_table, Alter_inplace_info *h return error; } -static bool find_index_of_key(const char *key_name, TABLE *table, uint *index_offset_ptr) { +static bool find_index_of_key( + const char* key_name, + TABLE* table, + uint* index_offset_ptr) { + for (uint i = 0; i < table->s->keys; i++) { if (strcmp(key_name, table->key_info[i].name) == 0) { *index_offset_ptr = i; @@ -529,7 +726,12 @@ static bool find_index_of_key(const char *key_name, TABLE *table, uint *index_of return false; } -static bool find_index_of_key(const char *key_name, KEY *key_info, uint key_count, uint *index_offset_ptr) { +static bool find_index_of_key( + const char* key_name, + KEY* key_info, + uint key_count, + uint* index_offset_ptr) { + for (uint i = 0; i < key_count; i++) { if (strcmp(key_name, key_info[i].name) == 0) { *index_offset_ptr = i; @@ -539,26 +741,42 @@ static bool find_index_of_key(const char *key_name, KEY *key_info, uint key_coun return false; } -int ha_tokudb::alter_table_drop_index(TABLE *altered_table, Alter_inplace_info *ha_alter_info) { +int ha_tokudb::alter_table_drop_index( + TABLE* altered_table, + Alter_inplace_info* ha_alter_info) { + KEY *key_info = table->key_info; // translate key names to indexes into the key_info array uint index_drop_offsets[ha_alter_info->index_drop_count]; for (uint i = 0; i < ha_alter_info->index_drop_count; i++) { bool found; - found = find_index_of_key(ha_alter_info->index_drop_buffer[i]->name, table, &index_drop_offsets[i]); + found = find_index_of_key( + ha_alter_info->index_drop_buffer[i]->name, + table, + &index_drop_offsets[i]); if (!found) { // undo of add key in partition engine - found = find_index_of_key(ha_alter_info->index_drop_buffer[i]->name, ha_alter_info->key_info_buffer, ha_alter_info->key_count, &index_drop_offsets[i]); - assert(found); + found = find_index_of_key( + ha_alter_info->index_drop_buffer[i]->name, + ha_alter_info->key_info_buffer, + ha_alter_info->key_count, + &index_drop_offsets[i]); + assert_always(found); key_info = ha_alter_info->key_info_buffer; } } // drop indexes - tokudb_alter_ctx *ctx = static_cast<tokudb_alter_ctx *>(ha_alter_info->handler_ctx); + tokudb_alter_ctx* ctx = + static_cast<tokudb_alter_ctx*>(ha_alter_info->handler_ctx); ctx->drop_index_changed = true; - int error = drop_indexes(table, index_drop_offsets, ha_alter_info->index_drop_count, key_info, ctx->alter_txn); + int error = drop_indexes( + table, + index_drop_offsets, + ha_alter_info->index_drop_count, + key_info, + ctx->alter_txn); if (error == 0) ctx->reset_card = true; @@ -566,93 +784,122 @@ int ha_tokudb::alter_table_drop_index(TABLE *altered_table, Alter_inplace_info * return error; } -int ha_tokudb::alter_table_add_or_drop_column(TABLE *altered_table, Alter_inplace_info *ha_alter_info) { - tokudb_alter_ctx *ctx = static_cast<tokudb_alter_ctx *>(ha_alter_info->handler_ctx); +int ha_tokudb::alter_table_add_or_drop_column( + TABLE* altered_table, + Alter_inplace_info* ha_alter_info) { + + tokudb_alter_ctx* ctx = + static_cast<tokudb_alter_ctx*>(ha_alter_info->handler_ctx); int error; uchar *column_extra = NULL; uint32_t max_column_extra_size; uint32_t num_column_extra; uint32_t num_columns = 0; uint32_t curr_num_DBs = table->s->keys + tokudb_test(hidden_primary_key); - - uint32_t columns[table->s->fields + altered_table->s->fields]; // set size such that we know it is big enough for both cases + // set size such that we know it is big enough for both cases + uint32_t columns[table->s->fields + altered_table->s->fields]; memset(columns, 0, sizeof(columns)); // generate the array of columns if (ha_alter_info->handler_flags & Alter_inplace_info::DROP_COLUMN) { find_changed_columns( - columns, - &num_columns, - altered_table, - table - ); - } else - if (ha_alter_info->handler_flags & Alter_inplace_info::ADD_COLUMN) { + columns, + &num_columns, + altered_table, + table); + } else if (ha_alter_info->handler_flags & Alter_inplace_info::ADD_COLUMN) { find_changed_columns( - columns, - &num_columns, - table, - altered_table - ); - } else - assert(0); + columns, + &num_columns, + table, + altered_table); + } else { + assert_unreachable(); + } max_column_extra_size = - STATIC_ROW_MUTATOR_SIZE + //max static row_mutator - 4 + num_columns*(1+1+4+1+1+4) + altered_table->s->reclength + // max dynamic row_mutator - (4 + share->kc_info.num_blobs) + // max static blob size - (num_columns*(1+4+1+4)); // max dynamic blob size - column_extra = (uchar *)tokudb_my_malloc(max_column_extra_size, MYF(MY_WME)); - if (column_extra == NULL) { error = ENOMEM; goto cleanup; } + // max static row_mutator + STATIC_ROW_MUTATOR_SIZE + + // max dynamic row_mutator + 4 + num_columns*(1+1+4+1+1+4) + altered_table->s->reclength + + // max static blob size + (4 + share->kc_info.num_blobs) + + // max dynamic blob size + (num_columns*(1+4+1+4)); + column_extra = (uchar*)tokudb::memory::malloc( + max_column_extra_size, + MYF(MY_WME)); + if (column_extra == NULL) { + error = ENOMEM; + goto cleanup; + } for (uint32_t i = 0; i < curr_num_DBs; i++) { // change to a new descriptor DBT row_descriptor; memset(&row_descriptor, 0, sizeof row_descriptor); - error = new_row_descriptor(table, altered_table, ha_alter_info, i, &row_descriptor); + error = new_row_descriptor( + table, + altered_table, + ha_alter_info, + i, + &row_descriptor); if (error) goto cleanup; - error = share->key_file[i]->change_descriptor(share->key_file[i], ctx->alter_txn, &row_descriptor, 0); - tokudb_my_free(row_descriptor.data); + error = share->key_file[i]->change_descriptor( + share->key_file[i], + ctx->alter_txn, + &row_descriptor, + 0); + tokudb::memory::free(row_descriptor.data); if (error) goto cleanup; if (i == primary_key || key_is_clustering(&table_share->key_info[i])) { num_column_extra = fill_row_mutator( - column_extra, - columns, - num_columns, - altered_table, - ctx->altered_table_kc_info, - i, - (ha_alter_info->handler_flags & Alter_inplace_info::ADD_COLUMN) != 0 // true if adding columns, otherwise is a drop - ); + column_extra, + columns, + num_columns, + altered_table, + ctx->altered_table_kc_info, + i, + // true if adding columns, otherwise is a drop + (ha_alter_info->handler_flags & + Alter_inplace_info::ADD_COLUMN) != 0); DBT column_dbt; memset(&column_dbt, 0, sizeof column_dbt); column_dbt.data = column_extra; column_dbt.size = num_column_extra; DBUG_ASSERT(num_column_extra <= max_column_extra_size); error = share->key_file[i]->update_broadcast( - share->key_file[i], - ctx->alter_txn, - &column_dbt, - DB_IS_RESETTING_OP - ); - if (error) { goto cleanup; } + share->key_file[i], + ctx->alter_txn, + &column_dbt, + DB_IS_RESETTING_OP); + if (error) { + goto cleanup; + } } } error = 0; cleanup: - tokudb_my_free(column_extra); + tokudb::memory::free(column_extra); return error; } // Commit or abort the alter operations. -// If commit then write the new frm data to the status using the alter transaction. -// If abort then abort the alter transaction and try to rollback the non-transactional changes. -bool ha_tokudb::commit_inplace_alter_table(TABLE *altered_table, Alter_inplace_info *ha_alter_info, bool commit) { +// If commit then write the new frm data to the status using the alter +// transaction. +// If abort then abort the alter transaction and try to rollback the +// non-transactional changes. +bool ha_tokudb::commit_inplace_alter_table( + TABLE* altered_table, + Alter_inplace_info* ha_alter_info, + bool commit) { + TOKUDB_HANDLER_DBUG_ENTER(""); - tokudb_alter_ctx *ctx = static_cast<tokudb_alter_ctx *>(ha_alter_info->handler_ctx); + tokudb_alter_ctx* ctx = + static_cast<tokudb_alter_ctx*>(ha_alter_info->handler_ctx); bool result = false; // success THD *thd = ha_thd(); @@ -671,7 +918,10 @@ bool ha_tokudb::commit_inplace_alter_table(TABLE *altered_table, Alter_inplace_i #else if (true) { #endif - int error = write_frm_data(share->status_block, ctx->alter_txn, altered_table->s->path.str); + int error = write_frm_data( + share->status_block, + ctx->alter_txn, + altered_table->s->path.str); if (error) { commit = false; result = true; @@ -683,42 +933,55 @@ bool ha_tokudb::commit_inplace_alter_table(TABLE *altered_table, Alter_inplace_i if (!commit) { if (table->mdl_ticket->get_type() != MDL_EXCLUSIVE && - (ctx->add_index_changed || ctx->drop_index_changed || ctx->compression_changed)) { + (ctx->add_index_changed || ctx->drop_index_changed || + ctx->compression_changed)) { // get exclusive lock no matter what #if defined(MARIADB_BASE_VERSION) killed_state saved_killed_state = thd->killed; thd->killed = NOT_KILLED; - for (volatile uint i = 0; wait_while_table_is_used(thd, table, HA_EXTRA_NOT_USED); i++) { + for (volatile uint i = 0; + wait_while_table_is_used(thd, table, HA_EXTRA_NOT_USED); + i++) { if (thd->killed != NOT_KILLED) thd->killed = NOT_KILLED; sleep(1); } - assert(table->mdl_ticket->get_type() == MDL_EXCLUSIVE); + assert_always(table->mdl_ticket->get_type() == MDL_EXCLUSIVE); if (thd->killed == NOT_KILLED) thd->killed = saved_killed_state; #else THD::killed_state saved_killed_state = thd->killed; thd->killed = THD::NOT_KILLED; - // MySQL does not handle HA_EXTRA_NOT_USED so we use HA_EXTRA_PREPARE_FOR_RENAME since it is passed through + // MySQL does not handle HA_EXTRA_NOT_USED so we use + // HA_EXTRA_PREPARE_FOR_RENAME since it is passed through // the partition storage engine and is treated as a NOP by tokudb - for (volatile uint i = 0; wait_while_table_is_used(thd, table, HA_EXTRA_PREPARE_FOR_RENAME); i++) { + for (volatile uint i = 0; + wait_while_table_is_used( + thd, + table, + HA_EXTRA_PREPARE_FOR_RENAME); + i++) { if (thd->killed != THD::NOT_KILLED) thd->killed = THD::NOT_KILLED; sleep(1); } - assert(table->mdl_ticket->get_type() == MDL_EXCLUSIVE); + assert_always(table->mdl_ticket->get_type() == MDL_EXCLUSIVE); if (thd->killed == THD::NOT_KILLED) thd->killed = saved_killed_state; #endif } - // abort the alter transaction NOW so that any alters are rolled back. this allows the following restores to work. - tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton); - assert(ctx->alter_txn == trx->stmt); - assert(trx->tokudb_lock_count > 0); - // for partitioned tables, we use a single transaction to do all of the partition changes. the tokudb_lock_count - // is a reference count for each of the handlers to the same transaction. obviously, we want to only abort once. + // abort the alter transaction NOW so that any alters are rolled back. + // this allows the following restores to work. + tokudb_trx_data* trx = + (tokudb_trx_data*)thd_get_ha_data(thd, tokudb_hton); + assert_always(ctx->alter_txn == trx->stmt); + assert_always(trx->tokudb_lock_count > 0); + // for partitioned tables, we use a single transaction to do all of the + // partition changes. the tokudb_lock_count is a reference count for + // each of the handlers to the same transaction. obviously, we want + // to only abort once. if (trx->tokudb_lock_count > 0) { if (--trx->tokudb_lock_count <= trx->create_lock_count) { trx->create_lock_count = 0; @@ -731,82 +994,125 @@ bool ha_tokudb::commit_inplace_alter_table(TABLE *altered_table, Alter_inplace_i } if (ctx->add_index_changed) { - restore_add_index(table, ha_alter_info->index_add_count, ctx->incremented_num_DBs, ctx->modified_DBs); + restore_add_index( + table, + ha_alter_info->index_add_count, + ctx->incremented_num_DBs, + ctx->modified_DBs); } if (ctx->drop_index_changed) { // translate key names to indexes into the key_info array uint index_drop_offsets[ha_alter_info->index_drop_count]; for (uint i = 0; i < ha_alter_info->index_drop_count; i++) { - bool found = find_index_of_key(ha_alter_info->index_drop_buffer[i]->name, table, &index_drop_offsets[i]); - assert(found); + bool found = find_index_of_key( + ha_alter_info->index_drop_buffer[i]->name, + table, + &index_drop_offsets[i]); + assert_always(found); } - restore_drop_indexes(table, index_drop_offsets, ha_alter_info->index_drop_count); + restore_drop_indexes( + table, + index_drop_offsets, + ha_alter_info->index_drop_count); } if (ctx->compression_changed) { - uint32_t curr_num_DBs = table->s->keys + tokudb_test(hidden_primary_key); + uint32_t curr_num_DBs = + table->s->keys + tokudb_test(hidden_primary_key); for (uint32_t i = 0; i < curr_num_DBs; i++) { DB *db = share->key_file[i]; - int error = db->change_compression_method(db, ctx->orig_compression_method); - assert(error == 0); + int error = db->change_compression_method( + db, + ctx->orig_compression_method); + assert_always(error == 0); } } } - DBUG_RETURN(result); } // Setup the altered table's key and col info. -int ha_tokudb::setup_kc_info(TABLE *altered_table, KEY_AND_COL_INFO *altered_kc_info) { +int ha_tokudb::setup_kc_info( + TABLE* altered_table, + KEY_AND_COL_INFO* altered_kc_info) { + int error = allocate_key_and_col_info(altered_table->s, altered_kc_info); if (error == 0) - error = initialize_key_and_col_info(altered_table->s, altered_table, altered_kc_info, hidden_primary_key, primary_key); + error = initialize_key_and_col_info( + altered_table->s, + altered_table, + altered_kc_info, + hidden_primary_key, + primary_key); return error; } // Expand the variable length fields offsets from 1 to 2 bytes. -int ha_tokudb::alter_table_expand_varchar_offsets(TABLE *altered_table, Alter_inplace_info *ha_alter_info) { +int ha_tokudb::alter_table_expand_varchar_offsets( + TABLE* altered_table, + Alter_inplace_info* ha_alter_info) { + int error = 0; - tokudb_alter_ctx *ctx = static_cast<tokudb_alter_ctx *>(ha_alter_info->handler_ctx); + tokudb_alter_ctx* ctx = + static_cast<tokudb_alter_ctx*>(ha_alter_info->handler_ctx); uint32_t curr_num_DBs = table->s->keys + tokudb_test(hidden_primary_key); for (uint32_t i = 0; i < curr_num_DBs; i++) { // change to a new descriptor DBT row_descriptor; memset(&row_descriptor, 0, sizeof row_descriptor); - error = new_row_descriptor(table, altered_table, ha_alter_info, i, &row_descriptor); + error = new_row_descriptor( + table, + altered_table, + ha_alter_info, + i, + &row_descriptor); if (error) break; - error = share->key_file[i]->change_descriptor(share->key_file[i], ctx->alter_txn, &row_descriptor, 0); - tokudb_my_free(row_descriptor.data); + error = share->key_file[i]->change_descriptor( + share->key_file[i], + ctx->alter_txn, + &row_descriptor, + 0); + tokudb::memory::free(row_descriptor.data); if (error) break; - // for all trees that have values, make an update variable offsets message and broadcast it into the tree + // for all trees that have values, make an update variable offsets + // message and broadcast it into the tree if (i == primary_key || key_is_clustering(&table_share->key_info[i])) { - uint32_t offset_start = table_share->null_bytes + share->kc_info.mcp_info[i].fixed_field_size; - uint32_t offset_end = offset_start + share->kc_info.mcp_info[i].len_of_offsets; + uint32_t offset_start = + table_share->null_bytes + + share->kc_info.mcp_info[i].fixed_field_size; + uint32_t offset_end = + offset_start + + share->kc_info.mcp_info[i].len_of_offsets; uint32_t number_of_offsets = offset_end - offset_start; // make the expand variable offsets message DBT expand; memset(&expand, 0, sizeof expand); - expand.size = sizeof (uchar) + sizeof offset_start + sizeof offset_end; - expand.data = tokudb_my_malloc(expand.size, MYF(MY_WME)); + expand.size = + sizeof(uchar) + sizeof(offset_start) + sizeof(offset_end); + expand.data = tokudb::memory::malloc(expand.size, MYF(MY_WME)); if (!expand.data) { error = ENOMEM; break; } - uchar *expand_ptr = (uchar *)expand.data; + uchar* expand_ptr = (uchar*)expand.data; expand_ptr[0] = UPDATE_OP_EXPAND_VARIABLE_OFFSETS; - expand_ptr += sizeof (uchar); + expand_ptr += sizeof(uchar); - memcpy(expand_ptr, &number_of_offsets, sizeof number_of_offsets); - expand_ptr += sizeof number_of_offsets; + memcpy(expand_ptr, &number_of_offsets, sizeof(number_of_offsets)); + expand_ptr += sizeof(number_of_offsets); - memcpy(expand_ptr, &offset_start, sizeof offset_start); - expand_ptr += sizeof offset_start; + memcpy(expand_ptr, &offset_start, sizeof(offset_start)); + expand_ptr += sizeof(offset_start); // and broadcast it into the tree - error = share->key_file[i]->update_broadcast(share->key_file[i], ctx->alter_txn, &expand, DB_IS_RESETTING_OP); - tokudb_my_free(expand.data); + error = share->key_file[i]->update_broadcast( + share->key_file[i], + ctx->alter_txn, + &expand, + DB_IS_RESETTING_OP); + tokudb::memory::free(expand.data); if (error) break; } @@ -834,30 +1140,49 @@ static bool field_in_key_of_table(TABLE *table, Field *field) { return false; } -// Return true if all changed varchar/varbinary field lengths can be changed inplace, otherwise return false -static bool change_varchar_length_is_supported(Field *old_field, Field *new_field, TABLE *table, TABLE *altered_table, Alter_inplace_info *ha_alter_info, tokudb_alter_ctx *ctx) { +// Return true if all changed varchar/varbinary field lengths can be changed +// inplace, otherwise return false +static bool change_varchar_length_is_supported( + Field* old_field, + Field* new_field, + TABLE* table, + TABLE* altered_table, + Alter_inplace_info* ha_alter_info, + tokudb_alter_ctx* ctx) { + if (old_field->real_type() != MYSQL_TYPE_VARCHAR || new_field->real_type() != MYSQL_TYPE_VARCHAR || old_field->binary() != new_field->binary() || old_field->charset()->number != new_field->charset()->number || old_field->field_length > new_field->field_length) return false; - if (ctx->table_kc_info->num_offset_bytes > ctx->altered_table_kc_info->num_offset_bytes) + if (ctx->table_kc_info->num_offset_bytes > + ctx->altered_table_kc_info->num_offset_bytes) return false; // shrink is not supported - if (ctx->table_kc_info->num_offset_bytes < ctx->altered_table_kc_info->num_offset_bytes) - ctx->expand_varchar_update_needed = true; // sum of varchar lengths changed from 1 to 2 + if (ctx->table_kc_info->num_offset_bytes < + ctx->altered_table_kc_info->num_offset_bytes) + // sum of varchar lengths changed from 1 to 2 + ctx->expand_varchar_update_needed = true; return true; } -// Return true if all changed field lengths can be changed inplace, otherwise return false -static bool change_length_is_supported(TABLE *table, TABLE *altered_table, Alter_inplace_info *ha_alter_info, tokudb_alter_ctx *ctx) { +// Return true if all changed field lengths can be changed inplace, otherwise +// return false +static bool change_length_is_supported( + TABLE* table, + TABLE* altered_table, + Alter_inplace_info* ha_alter_info, + tokudb_alter_ctx* ctx) { + if (table->s->fields != altered_table->s->fields) return false; if (table->s->null_bytes != altered_table->s->null_bytes) return false; if (ctx->changed_fields.elements() > 1) return false; // only support one field change - for (DYNAMIC_ARRAY_ELEMENTS_TYPE ai = 0; ai < ctx->changed_fields.elements(); ai++) { + for (DYNAMIC_ARRAY_ELEMENTS_TYPE ai = 0; + ai < ctx->changed_fields.elements(); + ai++) { uint i = ctx->changed_fields.at(ai); Field *old_field = table->field[i]; Field *new_field = altered_table->field[i]; @@ -865,9 +1190,16 @@ static bool change_length_is_supported(TABLE *table, TABLE *altered_table, Alter return false; // no type conversions if (old_field->real_type() != MYSQL_TYPE_VARCHAR) return false; // only varchar - if (field_in_key_of_table(table, old_field) || field_in_key_of_table(altered_table, new_field)) + if (field_in_key_of_table(table, old_field) || + field_in_key_of_table(altered_table, new_field)) return false; // not in any key - if (!change_varchar_length_is_supported(old_field, new_field, table, altered_table, ha_alter_info, ctx)) + if (!change_varchar_length_is_supported( + old_field, + new_field, + table, + altered_table, + ha_alter_info, + ctx)) return false; } @@ -886,13 +1218,23 @@ static bool is_sorted(Dynamic_array<uint> &a) { return r; } -int ha_tokudb::alter_table_expand_columns(TABLE *altered_table, Alter_inplace_info *ha_alter_info) { +int ha_tokudb::alter_table_expand_columns( + TABLE* altered_table, + Alter_inplace_info* ha_alter_info) { + int error = 0; - tokudb_alter_ctx *ctx = static_cast<tokudb_alter_ctx *>(ha_alter_info->handler_ctx); - assert(is_sorted(ctx->changed_fields)); // since we build the changed_fields array in field order, it must be sorted - for (DYNAMIC_ARRAY_ELEMENTS_TYPE ai = 0; error == 0 && ai < ctx->changed_fields.elements(); ai++) { + tokudb_alter_ctx* ctx = + static_cast<tokudb_alter_ctx*>(ha_alter_info->handler_ctx); + // since we build the changed_fields array in field order, it must be sorted + assert_always(is_sorted(ctx->changed_fields)); + for (DYNAMIC_ARRAY_ELEMENTS_TYPE ai = 0; + error == 0 && ai < ctx->changed_fields.elements(); + ai++) { uint expand_field_num = ctx->changed_fields.at(ai); - error = alter_table_expand_one_column(altered_table, ha_alter_info, expand_field_num); + error = alter_table_expand_one_column( + altered_table, + ha_alter_info, + expand_field_num); } return error; @@ -903,10 +1245,15 @@ static bool is_unsigned(Field *f) { return (f->flags & UNSIGNED_FLAG) != 0; } -// Return the starting offset in the value for a particular index (selected by idx) of a -// particular field (selected by expand_field_num) +// Return the starting offset in the value for a particular index (selected by +// idx) of a particular field (selected by expand_field_num) // TODO: replace this? -static uint32_t alter_table_field_offset(uint32_t null_bytes, KEY_AND_COL_INFO *kc_info, int idx, int expand_field_num) { +static uint32_t alter_table_field_offset( + uint32_t null_bytes, + KEY_AND_COL_INFO* kc_info, + int idx, + int expand_field_num) { + uint32_t offset = null_bytes; for (int i = 0; i < expand_field_num; i++) { if (bitmap_is_set(&kc_info->key_filters[idx], i)) // skip key fields @@ -917,21 +1264,26 @@ static uint32_t alter_table_field_offset(uint32_t null_bytes, KEY_AND_COL_INFO * } // Send an expand message into all clustered indexes including the primary -int ha_tokudb::alter_table_expand_one_column(TABLE *altered_table, Alter_inplace_info *ha_alter_info, int expand_field_num) { +int ha_tokudb::alter_table_expand_one_column( + TABLE* altered_table, + Alter_inplace_info* ha_alter_info, + int expand_field_num) { + int error = 0; - tokudb_alter_ctx *ctx = static_cast<tokudb_alter_ctx *>(ha_alter_info->handler_ctx); + tokudb_alter_ctx* ctx = + static_cast<tokudb_alter_ctx*>(ha_alter_info->handler_ctx); Field *old_field = table->field[expand_field_num]; TOKU_TYPE old_field_type = mysql_to_toku_type(old_field); Field *new_field = altered_table->field[expand_field_num]; TOKU_TYPE new_field_type = mysql_to_toku_type(new_field); - assert(old_field_type == new_field_type); + assert_always(old_field_type == new_field_type); uchar operation; uchar pad_char; switch (old_field_type) { case toku_type_int: - assert(is_unsigned(old_field) == is_unsigned(new_field)); + assert_always(is_unsigned(old_field) == is_unsigned(new_field)); if (is_unsigned(old_field)) operation = UPDATE_OP_EXPAND_UINT; else @@ -947,38 +1299,61 @@ int ha_tokudb::alter_table_expand_one_column(TABLE *altered_table, Alter_inplace pad_char = 0; break; default: - assert(0); + assert_unreachable(); } uint32_t curr_num_DBs = table->s->keys + tokudb_test(hidden_primary_key); for (uint32_t i = 0; i < curr_num_DBs; i++) { // change to a new descriptor DBT row_descriptor; memset(&row_descriptor, 0, sizeof row_descriptor); - error = new_row_descriptor(table, altered_table, ha_alter_info, i, &row_descriptor); + error = new_row_descriptor( + table, + altered_table, + ha_alter_info, + i, + &row_descriptor); if (error) break; - error = share->key_file[i]->change_descriptor(share->key_file[i], ctx->alter_txn, &row_descriptor, 0); - tokudb_my_free(row_descriptor.data); + error = share->key_file[i]->change_descriptor( + share->key_file[i], + ctx->alter_txn, + &row_descriptor, + 0); + tokudb::memory::free(row_descriptor.data); if (error) break; - // for all trees that have values, make an expand update message and broadcast it into the tree + // for all trees that have values, make an expand update message and + // broadcast it into the tree if (i == primary_key || key_is_clustering(&table_share->key_info[i])) { - uint32_t old_offset = alter_table_field_offset(table_share->null_bytes, ctx->table_kc_info, i, expand_field_num); - uint32_t new_offset = alter_table_field_offset(table_share->null_bytes, ctx->altered_table_kc_info, i, expand_field_num); - assert(old_offset <= new_offset); - - uint32_t old_length = ctx->table_kc_info->field_lengths[expand_field_num]; - assert(old_length == old_field->pack_length()); - - uint32_t new_length = ctx->altered_table_kc_info->field_lengths[expand_field_num]; - assert(new_length == new_field->pack_length()); - - DBT expand; memset(&expand, 0, sizeof expand); - expand.size = sizeof operation + sizeof new_offset + sizeof old_length + sizeof new_length; - if (operation == UPDATE_OP_EXPAND_CHAR || operation == UPDATE_OP_EXPAND_BINARY) - expand.size += sizeof pad_char; - expand.data = tokudb_my_malloc(expand.size, MYF(MY_WME)); + uint32_t old_offset = alter_table_field_offset( + table_share->null_bytes, + ctx->table_kc_info, + i, + expand_field_num); + uint32_t new_offset = alter_table_field_offset( + table_share->null_bytes, + ctx->altered_table_kc_info, + i, + expand_field_num); + assert_always(old_offset <= new_offset); + + uint32_t old_length = + ctx->table_kc_info->field_lengths[expand_field_num]; + assert_always(old_length == old_field->pack_length()); + + uint32_t new_length = + ctx->altered_table_kc_info->field_lengths[expand_field_num]; + assert_always(new_length == new_field->pack_length()); + + DBT expand; memset(&expand, 0, sizeof(expand)); + expand.size = + sizeof(operation) + sizeof(new_offset) + + sizeof(old_length) + sizeof(new_length); + if (operation == UPDATE_OP_EXPAND_CHAR || + operation == UPDATE_OP_EXPAND_BINARY) + expand.size += sizeof(pad_char); + expand.data = tokudb::memory::malloc(expand.size, MYF(MY_WME)); if (!expand.data) { error = ENOMEM; break; @@ -987,27 +1362,34 @@ int ha_tokudb::alter_table_expand_one_column(TABLE *altered_table, Alter_inplace expand_ptr[0] = operation; expand_ptr += sizeof operation; - // for the first altered field, old_offset == new_offset. for the subsequent altered fields, the new_offset - // should be used as it includes the length changes from the previous altered fields. - memcpy(expand_ptr, &new_offset, sizeof new_offset); - expand_ptr += sizeof new_offset; + // for the first altered field, old_offset == new_offset. + // for the subsequent altered fields, the new_offset + // should be used as it includes the length changes from the + // previous altered fields. + memcpy(expand_ptr, &new_offset, sizeof(new_offset)); + expand_ptr += sizeof(new_offset); - memcpy(expand_ptr, &old_length, sizeof old_length); - expand_ptr += sizeof old_length; + memcpy(expand_ptr, &old_length, sizeof(old_length)); + expand_ptr += sizeof(old_length); - memcpy(expand_ptr, &new_length, sizeof new_length); - expand_ptr += sizeof new_length; + memcpy(expand_ptr, &new_length, sizeof(new_length)); + expand_ptr += sizeof(new_length); - if (operation == UPDATE_OP_EXPAND_CHAR || operation == UPDATE_OP_EXPAND_BINARY) { - memcpy(expand_ptr, &pad_char, sizeof pad_char); - expand_ptr += sizeof pad_char; + if (operation == UPDATE_OP_EXPAND_CHAR || + operation == UPDATE_OP_EXPAND_BINARY) { + memcpy(expand_ptr, &pad_char, sizeof(pad_char)); + expand_ptr += sizeof(pad_char); } - assert(expand_ptr == (uchar *)expand.data + expand.size); + assert_always(expand_ptr == (uchar*)expand.data + expand.size); // and broadcast it into the tree - error = share->key_file[i]->update_broadcast(share->key_file[i], ctx->alter_txn, &expand, DB_IS_RESETTING_OP); - tokudb_my_free(expand.data); + error = share->key_file[i]->update_broadcast( + share->key_file[i], + ctx->alter_txn, + &expand, + DB_IS_RESETTING_OP); + tokudb::memory::free(expand.data); if (error) break; } @@ -1016,52 +1398,85 @@ int ha_tokudb::alter_table_expand_one_column(TABLE *altered_table, Alter_inplace return error; } -static void marshall_blob_lengths(tokudb::buffer &b, uint32_t n, TABLE *table, KEY_AND_COL_INFO *kc_info) { +static void marshall_blob_lengths( + tokudb::buffer& b, + uint32_t n, + TABLE* table, + KEY_AND_COL_INFO* kc_info) { + for (uint i = 0; i < n; i++) { uint blob_field_index = kc_info->blob_fields[i]; - assert(blob_field_index < table->s->fields); - uint8_t blob_field_length = table->s->field[blob_field_index]->row_pack_length(); + assert_always(blob_field_index < table->s->fields); + uint8_t blob_field_length = + table->s->field[blob_field_index]->row_pack_length(); b.append(&blob_field_length, sizeof blob_field_length); } } -int ha_tokudb::alter_table_expand_blobs(TABLE *altered_table, Alter_inplace_info *ha_alter_info) { +int ha_tokudb::alter_table_expand_blobs( + TABLE* altered_table, + Alter_inplace_info* ha_alter_info) { + int error = 0; - tokudb_alter_ctx *ctx = static_cast<tokudb_alter_ctx *>(ha_alter_info->handler_ctx); + tokudb_alter_ctx* ctx = + static_cast<tokudb_alter_ctx*>(ha_alter_info->handler_ctx); uint32_t curr_num_DBs = table->s->keys + tokudb_test(hidden_primary_key); for (uint32_t i = 0; i < curr_num_DBs; i++) { // change to a new descriptor DBT row_descriptor; memset(&row_descriptor, 0, sizeof row_descriptor); - error = new_row_descriptor(table, altered_table, ha_alter_info, i, &row_descriptor); + error = new_row_descriptor( + table, + altered_table, + ha_alter_info, + i, + &row_descriptor); if (error) break; - error = share->key_file[i]->change_descriptor(share->key_file[i], ctx->alter_txn, &row_descriptor, 0); - tokudb_my_free(row_descriptor.data); + error = share->key_file[i]->change_descriptor( + share->key_file[i], + ctx->alter_txn, + &row_descriptor, + 0); + tokudb::memory::free(row_descriptor.data); if (error) break; - // for all trees that have values, make an update blobs message and broadcast it into the tree + // for all trees that have values, make an update blobs message and + // broadcast it into the tree if (i == primary_key || key_is_clustering(&table_share->key_info[i])) { tokudb::buffer b; uint8_t op = UPDATE_OP_EXPAND_BLOB; b.append(&op, sizeof op); - b.append_ui<uint32_t>(table->s->null_bytes + ctx->table_kc_info->mcp_info[i].fixed_field_size); - uint32_t var_offset_bytes = ctx->table_kc_info->mcp_info[i].len_of_offsets; + b.append_ui<uint32_t>( + table->s->null_bytes + + ctx->table_kc_info->mcp_info[i].fixed_field_size); + uint32_t var_offset_bytes = + ctx->table_kc_info->mcp_info[i].len_of_offsets; b.append_ui<uint32_t>(var_offset_bytes); - b.append_ui<uint32_t>(var_offset_bytes == 0 ? 0 : ctx->table_kc_info->num_offset_bytes); + b.append_ui<uint32_t>( + var_offset_bytes == 0 ? 0 : + ctx->table_kc_info->num_offset_bytes); // add blobs info uint32_t num_blobs = ctx->table_kc_info->num_blobs; b.append_ui<uint32_t>(num_blobs); marshall_blob_lengths(b, num_blobs, table, ctx->table_kc_info); - marshall_blob_lengths(b, num_blobs, altered_table, ctx->altered_table_kc_info); + marshall_blob_lengths( + b, + num_blobs, + altered_table, + ctx->altered_table_kc_info); // and broadcast it into the tree DBT expand; memset(&expand, 0, sizeof expand); expand.data = b.data(); expand.size = b.size(); - error = share->key_file[i]->update_broadcast(share->key_file[i], ctx->alter_txn, &expand, DB_IS_RESETTING_OP); + error = share->key_file[i]->update_broadcast( + share->key_file[i], + ctx->alter_txn, + &expand, + DB_IS_RESETTING_OP); if (error) break; } @@ -1071,7 +1486,13 @@ int ha_tokudb::alter_table_expand_blobs(TABLE *altered_table, Alter_inplace_info } // Return true if two fixed length fields can be changed inplace -static bool change_fixed_length_is_supported(TABLE *table, TABLE *altered_table, Field *old_field, Field *new_field, tokudb_alter_ctx *ctx) { +static bool change_fixed_length_is_supported( + TABLE* table, + TABLE* altered_table, + Field* old_field, + Field* new_field, + tokudb_alter_ctx* ctx) { + // no change in size is supported if (old_field->pack_length() == new_field->pack_length()) return true; @@ -1082,9 +1503,16 @@ static bool change_fixed_length_is_supported(TABLE *table, TABLE *altered_table, return true; } -static bool change_blob_length_is_supported(TABLE *table, TABLE *altered_table, Field *old_field, Field *new_field, tokudb_alter_ctx *ctx) { +static bool change_blob_length_is_supported( + TABLE* table, + TABLE* altered_table, + Field* old_field, + Field* new_field, + tokudb_alter_ctx* ctx) { + // blob -> longer or equal length blob - if (old_field->binary() && new_field->binary() && old_field->pack_length() <= new_field->pack_length()) { + if (old_field->binary() && new_field->binary() && + old_field->pack_length() <= new_field->pack_length()) { ctx->expand_blob_update_needed = true; return true; } @@ -1113,13 +1541,26 @@ static bool is_int_type(enum_field_types t) { } // Return true if two field types can be changed inplace -static bool change_field_type_is_supported(Field *old_field, Field *new_field, TABLE *table, TABLE *altered_table, Alter_inplace_info *ha_alter_info, tokudb_alter_ctx *ctx) { +static bool change_field_type_is_supported( + Field* old_field, + Field* new_field, + TABLE* table, + TABLE* altered_table, + Alter_inplace_info* ha_alter_info, + tokudb_alter_ctx* ctx) { + enum_field_types old_type = old_field->real_type(); enum_field_types new_type = new_field->real_type(); if (is_int_type(old_type)) { // int and unsigned int expansion - if (is_int_type(new_type) && is_unsigned(old_field) == is_unsigned(new_field)) - return change_fixed_length_is_supported(table, altered_table, old_field, new_field, ctx); + if (is_int_type(new_type) && + is_unsigned(old_field) == is_unsigned(new_field)) + return change_fixed_length_is_supported( + table, + altered_table, + old_field, + new_field, + ctx); else return false; } else if (old_type == MYSQL_TYPE_STRING) { @@ -1127,67 +1568,112 @@ static bool change_field_type_is_supported(Field *old_field, Field *new_field, T if (new_type == MYSQL_TYPE_STRING && old_field->binary() == new_field->binary() && old_field->charset()->number == new_field->charset()->number) - return change_fixed_length_is_supported(table, altered_table, old_field, new_field, ctx); + return change_fixed_length_is_supported( + table, + altered_table, + old_field, + new_field, + ctx); else return false; } else if (old_type == MYSQL_TYPE_VARCHAR) { - // varchar(X) -> varchar(Y) and varbinary(X) -> varbinary(Y) expansion where X < 256 <= Y - // the ALTER_COLUMN_TYPE handler flag is set for these cases - return change_varchar_length_is_supported(old_field, new_field, table, altered_table, ha_alter_info, ctx); + // varchar(X) -> varchar(Y) and varbinary(X) -> varbinary(Y) expansion + // where X < 256 <= Y the ALTER_COLUMN_TYPE handler flag is set for + // these cases + return change_varchar_length_is_supported( + old_field, + new_field, + table, + altered_table, + ha_alter_info, + ctx); } else if (old_type == MYSQL_TYPE_BLOB && new_type == MYSQL_TYPE_BLOB) { - return change_blob_length_is_supported(table, altered_table, old_field, new_field, ctx); + return change_blob_length_is_supported( + table, + altered_table, + old_field, + new_field, + ctx); } else return false; } // Return true if all changed field types can be changed inplace -static bool change_type_is_supported(TABLE *table, TABLE *altered_table, Alter_inplace_info *ha_alter_info, tokudb_alter_ctx *ctx) { +static bool change_type_is_supported( + TABLE* table, + TABLE* altered_table, + Alter_inplace_info* ha_alter_info, + tokudb_alter_ctx* ctx) { + if (table->s->null_bytes != altered_table->s->null_bytes) return false; if (table->s->fields != altered_table->s->fields) return false; if (ctx->changed_fields.elements() > 1) return false; // only support one field change - for (DYNAMIC_ARRAY_ELEMENTS_TYPE ai = 0; ai < ctx->changed_fields.elements(); ai++) { + for (DYNAMIC_ARRAY_ELEMENTS_TYPE ai = 0; + ai < ctx->changed_fields.elements(); + ai++) { uint i = ctx->changed_fields.at(ai); Field *old_field = table->field[i]; Field *new_field = altered_table->field[i]; - if (field_in_key_of_table(table, old_field) || field_in_key_of_table(altered_table, new_field)) + if (field_in_key_of_table(table, old_field) || + field_in_key_of_table(altered_table, new_field)) return false; - if (!change_field_type_is_supported(old_field, new_field, table, altered_table, ha_alter_info, ctx)) + if (!change_field_type_is_supported( + old_field, + new_field, + table, + altered_table, + ha_alter_info, + ctx)) return false; } return true; } -// Allocate and initialize a new descriptor for a dictionary in the altered table identified with idx. +// Allocate and initialize a new descriptor for a dictionary in the altered +// table identified with idx. // Return the new descriptor in the row_descriptor DBT. // Return non-zero on error. -int ha_tokudb::new_row_descriptor(TABLE *table, TABLE *altered_table, Alter_inplace_info *ha_alter_info, uint32_t idx, DBT *row_descriptor) { +int ha_tokudb::new_row_descriptor( + TABLE* table, + TABLE* altered_table, + Alter_inplace_info* ha_alter_info, + uint32_t idx, + DBT* row_descriptor) { + int error = 0; - tokudb_alter_ctx *ctx = static_cast<tokudb_alter_ctx *>(ha_alter_info->handler_ctx); - row_descriptor->size = get_max_desc_size(ctx->altered_table_kc_info, altered_table); - row_descriptor->data = (uchar *) tokudb_my_malloc(row_descriptor->size, MYF(MY_WME)); + tokudb_alter_ctx* ctx = + static_cast<tokudb_alter_ctx*>(ha_alter_info->handler_ctx); + row_descriptor->size = + get_max_desc_size(ctx->altered_table_kc_info, altered_table); + row_descriptor->data = + (uchar*)tokudb::memory::malloc(row_descriptor->size, MYF(MY_WME)); if (row_descriptor->data == NULL) { error = ENOMEM; } else { - KEY* prim_key = hidden_primary_key ? NULL : &altered_table->s->key_info[primary_key]; + KEY* prim_key = + hidden_primary_key ? NULL : + &altered_table->s->key_info[primary_key]; if (idx == primary_key) { - row_descriptor->size = create_main_key_descriptor((uchar *)row_descriptor->data, - prim_key, - hidden_primary_key, - primary_key, - altered_table, - ctx->altered_table_kc_info); + row_descriptor->size = create_main_key_descriptor( + (uchar*)row_descriptor->data, + prim_key, + hidden_primary_key, + primary_key, + altered_table, + ctx->altered_table_kc_info); } else { - row_descriptor->size = create_secondary_key_descriptor((uchar *)row_descriptor->data, - &altered_table->key_info[idx], - prim_key, - hidden_primary_key, - altered_table, - primary_key, - idx, - ctx->altered_table_kc_info); + row_descriptor->size = create_secondary_key_descriptor( + (uchar*)row_descriptor->data, + &altered_table->key_info[idx], + prim_key, + hidden_primary_key, + altered_table, + primary_key, + idx, + ctx->altered_table_kc_info); } error = 0; } diff --git a/storage/tokudb/ha_tokudb_alter_common.cc b/storage/tokudb/ha_tokudb_alter_common.cc index b2c2a2b0252..c58f57b4da7 100644 --- a/storage/tokudb/ha_tokudb_alter_common.cc +++ b/storage/tokudb/ha_tokudb_alter_common.cc @@ -26,8 +26,18 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #if !defined(TOKUDB_ALTER_COMMON) #define TOKUDB_ALTER_COMMON -static bool tables_have_same_keys(TABLE* table, TABLE* altered_table, bool print_error, bool check_field_index) __attribute__((unused)); -static bool tables_have_same_keys(TABLE* table, TABLE* altered_table, bool print_error, bool check_field_index) { +TOKUDB_UNUSED(static bool tables_have_same_keys( + TABLE* table, + TABLE* altered_table, + bool print_error, + bool check_field_index)); + +static bool tables_have_same_keys( + TABLE* table, + TABLE* altered_table, + bool print_error, + bool check_field_index) { + bool retval; if (table->s->keys != altered_table->s->keys) { if (print_error) { @@ -39,10 +49,9 @@ static bool tables_have_same_keys(TABLE* table, TABLE* altered_table, bool print if (table->s->primary_key != altered_table->s->primary_key) { if (print_error) { sql_print_error( - "Tables have different primary keys, %d %d", + "Tables have different primary keys, %d %d", table->s->primary_key, - altered_table->s->primary_key - ); + altered_table->s->primary_key); } retval = false; goto cleanup; @@ -53,33 +62,32 @@ static bool tables_have_same_keys(TABLE* table, TABLE* altered_table, bool print if (strcmp(curr_orig_key->name, curr_altered_key->name)) { if (print_error) { sql_print_error( - "key %d has different name, %s %s", - i, + "key %d has different name, %s %s", + i, curr_orig_key->name, - curr_altered_key->name - ); + curr_altered_key->name); } retval = false; goto cleanup; } - if (key_is_clustering(curr_orig_key) != key_is_clustering(curr_altered_key)) { + if (key_is_clustering(curr_orig_key) != + key_is_clustering(curr_altered_key)) { if (print_error) { sql_print_error( "keys disagree on if they are clustering, %d, %d", get_key_parts(curr_orig_key), - get_key_parts(curr_altered_key) - ); + get_key_parts(curr_altered_key)); } retval = false; goto cleanup; } - if (((curr_orig_key->flags & HA_NOSAME) == 0) != ((curr_altered_key->flags & HA_NOSAME) == 0)) { + if (((curr_orig_key->flags & HA_NOSAME) == 0) != + ((curr_altered_key->flags & HA_NOSAME) == 0)) { if (print_error) { sql_print_error( "keys disagree on if they are unique, %d, %d", get_key_parts(curr_orig_key), - get_key_parts(curr_altered_key) - ); + get_key_parts(curr_altered_key)); } retval = false; goto cleanup; @@ -89,8 +97,7 @@ static bool tables_have_same_keys(TABLE* table, TABLE* altered_table, bool print sql_print_error( "keys have different number of parts, %d, %d", get_key_parts(curr_orig_key), - get_key_parts(curr_altered_key) - ); + get_key_parts(curr_altered_key)); } retval = false; goto cleanup; @@ -106,10 +113,9 @@ static bool tables_have_same_keys(TABLE* table, TABLE* altered_table, bool print if (curr_orig_part->length != curr_altered_part->length) { if (print_error) { sql_print_error( - "Key %s has different length at index %d", - curr_orig_key->name, - j - ); + "Key %s has different length at index %d", + curr_orig_key->name, + j); } retval = false; goto cleanup; @@ -123,10 +129,9 @@ static bool tables_have_same_keys(TABLE* table, TABLE* altered_table, bool print if (!are_fields_same) { if (print_error) { sql_print_error( - "Key %s has different field at index %d", - curr_orig_key->name, - j - ); + "Key %s has different field at index %d", + curr_orig_key->name, + j); } retval = false; goto cleanup; @@ -143,7 +148,8 @@ cleanup: // to evaluate whether a field is NULL or not. This value is a power of 2, from // 2^0 to 2^7. We return the position of the bit within the byte, which is // lg null_bit -static inline uint32_t get_null_bit_position(uint32_t null_bit) __attribute__((unused)); +TOKUDB_UNUSED(static inline uint32_t get_null_bit_position( + uint32_t null_bit)); static inline uint32_t get_null_bit_position(uint32_t null_bit) { uint32_t retval = 0; switch(null_bit) { @@ -170,23 +176,28 @@ static inline uint32_t get_null_bit_position(uint32_t null_bit) { break; case (128): retval = 7; - break; + break; default: - assert(false); + assert_unreachable(); } return retval; } // returns the index of the null bit of field. -static inline uint32_t get_overall_null_bit_position(TABLE* table, Field* field) __attribute__((unused)); -static inline uint32_t get_overall_null_bit_position(TABLE* table, Field* field) { +TOKUDB_UNUSED(static inline uint32_t get_overall_null_bit_position( + TABLE* table, + Field* field)); +static inline uint32_t get_overall_null_bit_position( + TABLE* table, + Field* field) { + uint32_t offset = get_null_offset(table, field); uint32_t null_bit = field->null_bit; return offset*8 + get_null_bit_position(null_bit); } // not static since 51 uses this and 56 does not -static bool are_null_bits_in_order(TABLE* table) __attribute__((unused)); +TOKUDB_UNUSED(static bool are_null_bits_in_order(TABLE* table)); static bool are_null_bits_in_order(TABLE* table) { uint32_t curr_null_pos = 0; bool first = true; @@ -195,10 +206,8 @@ static bool are_null_bits_in_order(TABLE* table) { Field* curr_field = table->field[i]; bool nullable = (curr_field->null_bit != 0); if (nullable) { - uint32_t pos = get_overall_null_bit_position( - table, - curr_field - ); + uint32_t pos = + get_overall_null_bit_position(table, curr_field); if (!first && pos != curr_null_pos+1){ retval = false; break; @@ -210,34 +219,38 @@ static bool are_null_bits_in_order(TABLE* table) { return retval; } -static uint32_t get_first_null_bit_pos(TABLE* table) __attribute__((unused)); +TOKUDB_UNUSED(static uint32_t get_first_null_bit_pos(TABLE* table)); static uint32_t get_first_null_bit_pos(TABLE* table) { uint32_t table_pos = 0; for (uint i = 0; i < table->s->fields; i++) { Field* curr_field = table->field[i]; bool nullable = (curr_field->null_bit != 0); if (nullable) { - table_pos = get_overall_null_bit_position( - table, - curr_field - ); + table_pos = + get_overall_null_bit_position(table, curr_field); break; } } return table_pos; } -static bool is_column_default_null(TABLE* src_table, uint32_t field_index) __attribute__((unused)); -static bool is_column_default_null(TABLE* src_table, uint32_t field_index) { +TOKUDB_UNUSED(static bool is_column_default_null( + TABLE* src_table, + uint32_t field_index)); +static bool is_column_default_null( + TABLE* src_table, + uint32_t field_index) { + Field* curr_field = src_table->field[field_index]; bool is_null_default = false; bool nullable = curr_field->null_bit != 0; if (nullable) { - uint32_t null_bit_position = get_overall_null_bit_position(src_table, curr_field); - is_null_default = is_overall_null_position_set( - src_table->s->default_values, - null_bit_position - ); + uint32_t null_bit_position = + get_overall_null_bit_position(src_table, curr_field); + is_null_default = + is_overall_null_position_set( + src_table->s->default_values, + null_bit_position); } return is_null_default; } @@ -248,9 +261,8 @@ static uint32_t fill_static_row_mutator( TABLE* altered_table, KEY_AND_COL_INFO* orig_kc_info, KEY_AND_COL_INFO* altered_kc_info, - uint32_t keynr - ) -{ + uint32_t keynr) { + // // start packing extra // @@ -258,25 +270,28 @@ static uint32_t fill_static_row_mutator( // says what the operation is pos[0] = UP_COL_ADD_OR_DROP; pos++; - + // // null byte information // memcpy(pos, &orig_table->s->null_bytes, sizeof(orig_table->s->null_bytes)); pos += sizeof(orig_table->s->null_bytes); - memcpy(pos, &altered_table->s->null_bytes, sizeof(orig_table->s->null_bytes)); + memcpy( + pos, + &altered_table->s->null_bytes, + sizeof(orig_table->s->null_bytes)); pos += sizeof(altered_table->s->null_bytes); - + // // num_offset_bytes // - assert(orig_kc_info->num_offset_bytes <= 2); + assert_always(orig_kc_info->num_offset_bytes <= 2); pos[0] = orig_kc_info->num_offset_bytes; pos++; - assert(altered_kc_info->num_offset_bytes <= 2); + assert_always(altered_kc_info->num_offset_bytes <= 2); pos[0] = altered_kc_info->num_offset_bytes; pos++; - + // // size of fixed fields // @@ -286,7 +301,7 @@ static uint32_t fill_static_row_mutator( fixed_field_size = altered_kc_info->mcp_info[keynr].fixed_field_size; memcpy(pos, &fixed_field_size, sizeof(fixed_field_size)); pos += sizeof(fixed_field_size); - + // // length of offsets // @@ -304,7 +319,7 @@ static uint32_t fill_static_row_mutator( memcpy(pos, &altered_start_null_pos, sizeof(altered_start_null_pos)); pos += sizeof(altered_start_null_pos); - assert((pos-buf) == STATIC_ROW_MUTATOR_SIZE); + assert_always((pos-buf) == STATIC_ROW_MUTATOR_SIZE); return pos - buf; } @@ -316,9 +331,8 @@ static uint32_t fill_dynamic_row_mutator( KEY_AND_COL_INFO* src_kc_info, uint32_t keynr, bool is_add, - bool* out_has_blobs - ) -{ + bool* out_has_blobs) { + uchar* pos = buf; bool has_blobs = false; uint32_t cols = num_columns; @@ -327,7 +341,7 @@ static uint32_t fill_dynamic_row_mutator( for (uint32_t i = 0; i < num_columns; i++) { uint32_t curr_index = columns[i]; Field* curr_field = src_table->field[curr_index]; - + pos[0] = is_add ? COL_ADD : COL_DROP; pos++; // @@ -338,22 +352,22 @@ static uint32_t fill_dynamic_row_mutator( if (!nullable) { pos[0] = 0; pos++; - } - else { + } else { pos[0] = 1; pos++; // write position of null byte that is to be removed - uint32_t null_bit_position = get_overall_null_bit_position(src_table, curr_field); + uint32_t null_bit_position = + get_overall_null_bit_position(src_table, curr_field); memcpy(pos, &null_bit_position, sizeof(null_bit_position)); pos += sizeof(null_bit_position); // // if adding a column, write the value of the default null_bit // if (is_add) { - is_null_default = is_overall_null_position_set( - src_table->s->default_values, - null_bit_position - ); + is_null_default = + is_overall_null_position_set( + src_table->s->default_values, + null_bit_position); pos[0] = is_null_default ? 1 : 0; pos++; } @@ -364,7 +378,8 @@ static uint32_t fill_dynamic_row_mutator( pos[0] = COL_FIXED; pos++; //store the offset - uint32_t fixed_field_offset = src_kc_info->cp_info[keynr][curr_index].col_pack_val; + uint32_t fixed_field_offset = + src_kc_info->cp_info[keynr][curr_index].col_pack_val; memcpy(pos, &fixed_field_offset, sizeof(fixed_field_offset)); pos += sizeof(fixed_field_offset); //store the number of bytes @@ -374,38 +389,35 @@ static uint32_t fill_dynamic_row_mutator( if (is_add && !is_null_default) { uint curr_field_offset = field_offset(curr_field, src_table); memcpy( - pos, - src_table->s->default_values + curr_field_offset, - num_bytes - ); + pos, + src_table->s->default_values + curr_field_offset, + num_bytes); pos += num_bytes; } - } - else if (is_variable_field(src_kc_info, curr_index)) { + } else if (is_variable_field(src_kc_info, curr_index)) { pos[0] = COL_VAR; pos++; //store the index of the variable column - uint32_t var_field_index = src_kc_info->cp_info[keynr][curr_index].col_pack_val; + uint32_t var_field_index = + src_kc_info->cp_info[keynr][curr_index].col_pack_val; memcpy(pos, &var_field_index, sizeof(var_field_index)); pos += sizeof(var_field_index); if (is_add && !is_null_default) { uint curr_field_offset = field_offset(curr_field, src_table); uint32_t len_bytes = src_kc_info->length_bytes[curr_index]; - uint32_t data_length = get_var_data_length( - src_table->s->default_values + curr_field_offset, - len_bytes - ); + uint32_t data_length = + get_var_data_length( + src_table->s->default_values + curr_field_offset, + len_bytes); memcpy(pos, &data_length, sizeof(data_length)); pos += sizeof(data_length); memcpy( pos, src_table->s->default_values + curr_field_offset + len_bytes, - data_length - ); + data_length); pos += data_length; } - } - else { + } else { pos[0] = COL_BLOB; pos++; has_blobs = true; @@ -418,9 +430,8 @@ static uint32_t fill_dynamic_row_mutator( static uint32_t fill_static_blob_row_mutator( uchar* buf, TABLE* src_table, - KEY_AND_COL_INFO* src_kc_info - ) -{ + KEY_AND_COL_INFO* src_kc_info) { + uchar* pos = buf; // copy number of blobs memcpy(pos, &src_kc_info->num_blobs, sizeof(src_kc_info->num_blobs)); @@ -430,11 +441,11 @@ static uint32_t fill_static_blob_row_mutator( uint32_t curr_field_index = src_kc_info->blob_fields[i]; Field* field = src_table->field[curr_field_index]; uint32_t len_bytes = field->row_pack_length(); - assert(len_bytes <= 4); + assert_always(len_bytes <= 4); pos[0] = len_bytes; pos++; } - + return pos-buf; } @@ -444,9 +455,8 @@ static uint32_t fill_dynamic_blob_row_mutator( uint32_t num_columns, TABLE* src_table, KEY_AND_COL_INFO* src_kc_info, - bool is_add - ) -{ + bool is_add) { + uchar* pos = buf; for (uint32_t i = 0; i < num_columns; i++) { uint32_t curr_field_index = columns[i]; @@ -461,19 +471,19 @@ static uint32_t fill_dynamic_blob_row_mutator( } } // assert we found blob in list - assert(blob_index < src_kc_info->num_blobs); + assert_always(blob_index < src_kc_info->num_blobs); pos[0] = is_add ? COL_ADD : COL_DROP; pos++; memcpy(pos, &blob_index, sizeof(blob_index)); pos += sizeof(blob_index); if (is_add) { uint32_t len_bytes = curr_field->row_pack_length(); - assert(len_bytes <= 4); + assert_always(len_bytes <= 4); pos[0] = len_bytes; pos++; - // create a zero length blob field that can be directly copied in - // for now, in MySQL, we can only have blob fields + // create a zero length blob field that can be directly copied + // in for now, in MySQL, we can only have blob fields // that have no default value memset(pos, 0, len_bytes); pos += len_bytes; @@ -487,93 +497,86 @@ static uint32_t fill_dynamic_blob_row_mutator( // TODO: namely, when do we get stuff from share->kc_info and when we get // TODO: it from altered_kc_info, and when is keynr associated with the right thing uint32_t ha_tokudb::fill_row_mutator( - uchar* buf, - uint32_t* columns, + uchar* buf, + uint32_t* columns, uint32_t num_columns, TABLE* altered_table, KEY_AND_COL_INFO* altered_kc_info, uint32_t keynr, - bool is_add - ) -{ - if (tokudb_debug & TOKUDB_DEBUG_ALTER_TABLE) { - printf("*****some info:*************\n"); - printf( - "old things: num_null_bytes %d, num_offset_bytes %d, fixed_field_size %d, fixed_field_size %d\n", + bool is_add) { + + if (TOKUDB_UNLIKELY(TOKUDB_DEBUG_FLAGS(TOKUDB_DEBUG_ALTER_TABLE))) { + TOKUDB_HANDLER_TRACE("*****some info:*************"); + TOKUDB_HANDLER_TRACE( + "old things: num_null_bytes %d, num_offset_bytes %d, " + "fixed_field_size %d, fixed_field_size %d", table->s->null_bytes, share->kc_info.num_offset_bytes, share->kc_info.mcp_info[keynr].fixed_field_size, - share->kc_info.mcp_info[keynr].len_of_offsets - ); - printf( - "new things: num_null_bytes %d, num_offset_bytes %d, fixed_field_size %d, fixed_field_size %d\n", + share->kc_info.mcp_info[keynr].len_of_offsets); + TOKUDB_HANDLER_TRACE( + "new things: num_null_bytes %d, num_offset_bytes %d, " + "fixed_field_size %d, fixed_field_size %d", altered_table->s->null_bytes, altered_kc_info->num_offset_bytes, altered_kc_info->mcp_info[keynr].fixed_field_size, - altered_kc_info->mcp_info[keynr].len_of_offsets - ); - printf("****************************\n"); + altered_kc_info->mcp_info[keynr].len_of_offsets); + TOKUDB_HANDLER_TRACE("****************************"); } uchar* pos = buf; bool has_blobs = false; - pos += fill_static_row_mutator( - pos, - table, - altered_table, - &share->kc_info, - altered_kc_info, - keynr - ); - - if (is_add) { - pos += fill_dynamic_row_mutator( - pos, - columns, - num_columns, - altered_table, - altered_kc_info, - keynr, - is_add, - &has_blobs - ); - } - else { - pos += fill_dynamic_row_mutator( + pos += + fill_static_row_mutator( pos, - columns, - num_columns, table, + altered_table, &share->kc_info, - keynr, - is_add, - &has_blobs - ); - } - if (has_blobs) { - pos += fill_static_blob_row_mutator( - pos, - table, - &share->kc_info - ); - if (is_add) { - pos += fill_dynamic_blob_row_mutator( + altered_kc_info, + keynr); + + if (is_add) { + pos += + fill_dynamic_row_mutator( pos, columns, num_columns, altered_table, altered_kc_info, - is_add - ); - } - else { - pos += fill_dynamic_blob_row_mutator( + keynr, + is_add, + &has_blobs); + } else { + pos += + fill_dynamic_row_mutator( pos, columns, num_columns, table, &share->kc_info, - is_add - ); + keynr, + is_add, + &has_blobs); + } + if (has_blobs) { + pos += fill_static_blob_row_mutator(pos, table, &share->kc_info); + if (is_add) { + pos += + fill_dynamic_blob_row_mutator( + pos, + columns, + num_columns, + altered_table, + altered_kc_info, + is_add); + } else { + pos += + fill_dynamic_blob_row_mutator( + pos, + columns, + num_columns, + table, + &share->kc_info, + is_add); } } return pos-buf; @@ -583,16 +586,23 @@ static bool all_fields_are_same_type(TABLE *table_a, TABLE *table_b) { if (table_a->s->fields != table_b->s->fields) return false; for (uint i = 0; i < table_a->s->fields; i++) { - Field *field_a = table_a->field[i]; - Field *field_b = table_b->field[i]; + Field* field_a = table_a->field[i]; + Field* field_b = table_b->field[i]; if (!fields_are_same_type(field_a, field_b)) return false; } return true; } -static bool column_rename_supported(TABLE* orig_table, TABLE* new_table, bool alter_column_order) __attribute__((unused)); -static bool column_rename_supported(TABLE* orig_table, TABLE* new_table, bool alter_column_order) { +TOKUDB_UNUSED(static bool column_rename_supported( + TABLE* orig_table, + TABLE* new_table, + bool alter_column_order)); +static bool column_rename_supported( + TABLE* orig_table, + TABLE* new_table, + bool alter_column_order) { + bool retval = false; bool keys_same_for_cr; uint num_fields_with_different_names = 0; @@ -622,20 +632,20 @@ static bool column_rename_supported(TABLE* orig_table, TABLE* new_table, bool al retval = false; goto cleanup; } - assert(field_with_different_name < orig_table->s->fields); + assert_always(field_with_different_name < orig_table->s->fields); // // at this point, we have verified that the two tables have - // the same field types and with ONLY one field with a different name. + // the same field types and with ONLY one field with a different name. // We have also identified the field with the different name // // Now we need to check the indexes // - keys_same_for_cr = tables_have_same_keys( - orig_table, - new_table, - false, - true - ); + keys_same_for_cr = + tables_have_same_keys( + orig_table, + new_table, + false, + true); if (!keys_same_for_cr) { retval = false; goto cleanup; @@ -645,12 +655,21 @@ cleanup: return retval; } -static int find_changed_columns(uint32_t* changed_columns, uint32_t* num_changed_columns, TABLE* smaller_table, TABLE* bigger_table) __attribute__((unused)); -static int find_changed_columns(uint32_t* changed_columns, uint32_t* num_changed_columns, TABLE* smaller_table, TABLE* bigger_table) { +TOKUDB_UNUSED(static int find_changed_columns( + uint32_t* changed_columns, + uint32_t* num_changed_columns, + TABLE* smaller_table, + TABLE* bigger_table)); +static int find_changed_columns( + uint32_t* changed_columns, + uint32_t* num_changed_columns, + TABLE* smaller_table, + TABLE* bigger_table) { + int retval; uint curr_new_col_index = 0; uint32_t curr_num_changed_columns=0; - assert(bigger_table->s->fields > smaller_table->s->fields); + assert_always(bigger_table->s->fields > smaller_table->s->fields); for (uint i = 0; i < smaller_table->s->fields; i++, curr_new_col_index++) { if (curr_new_col_index >= bigger_table->s->fields) { sql_print_error("error in determining changed columns"); @@ -670,15 +689,15 @@ static int find_changed_columns(uint32_t* changed_columns, uint32_t* num_changed goto cleanup; } } - // at this point, curr_field_in_orig and curr_field_in_new should be the same, let's verify - // make sure the two fields that have the same name are ok + // at this point, curr_field_in_orig and curr_field_in_new should be + // the same, let's verify make sure the two fields that have the same + // name are ok if (!are_two_fields_same(curr_field_in_orig, curr_field_in_new)) { sql_print_error( - "Two fields that were supposedly the same are not: \ - %s in original, %s in new", + "Two fields that were supposedly the same are not: %s in " + "original, %s in new", curr_field_in_orig->field_name, - curr_field_in_new->field_name - ); + curr_field_in_new->field_name); retval = 1; goto cleanup; } @@ -693,17 +712,23 @@ cleanup: return retval; } -static bool tables_have_same_keys_and_columns(TABLE* first_table, TABLE* second_table, bool print_error) __attribute__((unused)); -static bool tables_have_same_keys_and_columns(TABLE* first_table, TABLE* second_table, bool print_error) { +TOKUDB_UNUSED(static bool tables_have_same_keys_and_columns( + TABLE* first_table, + TABLE* second_table, + bool print_error)); +static bool tables_have_same_keys_and_columns( + TABLE* first_table, + TABLE* second_table, + bool print_error) { + bool retval; if (first_table->s->null_bytes != second_table->s->null_bytes) { retval = false; if (print_error) { sql_print_error( - "tables have different number of null bytes, %d, %d", - first_table->s->null_bytes, - second_table->s->null_bytes - ); + "tables have different number of null bytes, %d, %d", + first_table->s->null_bytes, + second_table->s->null_bytes); } goto exit; } @@ -711,10 +736,9 @@ static bool tables_have_same_keys_and_columns(TABLE* first_table, TABLE* second_ retval = false; if (print_error) { sql_print_error( - "tables have different number of fields, %d, %d", - first_table->s->fields, - second_table->s->fields - ); + "tables have different number of fields, %d, %d", + first_table->s->fields, + second_table->s->fields); } goto exit; } @@ -724,9 +748,8 @@ static bool tables_have_same_keys_and_columns(TABLE* first_table, TABLE* second_ if (!are_two_fields_same(a,b)) { retval = false; sql_print_error( - "tables have different fields at position %d", - i - ); + "tables have different fields at position %d", + i); goto exit; } } @@ -741,21 +764,29 @@ exit: } #if TOKU_INCLUDE_WRITE_FRM_DATA -// write the new frm data to the status dictionary using the alter table transaction -int ha_tokudb::write_frm_data(const uchar *frm_data, size_t frm_len) { +// write the new frm data to the status dictionary using the alter table +// transaction +int ha_tokudb::write_frm_data(const uchar* frm_data, size_t frm_len) { TOKUDB_DBUG_ENTER("write_frm_data"); int error = 0; if (TOKU_PARTITION_WRITE_FRM_DATA || table->part_info == NULL) { // write frmdata to status - THD *thd = ha_thd(); - tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton); - assert(trx); - DB_TXN *txn = trx->stmt; // use alter table transaction - assert(txn); - error = write_to_status(share->status_block, hatoku_frm_data, (void *)frm_data, (uint)frm_len, txn); + THD* thd = ha_thd(); + tokudb_trx_data* trx = + (tokudb_trx_data*)thd_get_ha_data(thd, tokudb_hton); + assert_always(trx); + DB_TXN* txn = trx->stmt; // use alter table transaction + assert_always(txn); + error = + write_to_status( + share->status_block, + hatoku_frm_data, + (void*)frm_data, + (uint)frm_len, + txn); } - + TOKUDB_DBUG_RETURN(error); } #endif diff --git a/storage/tokudb/ha_tokudb_update.cc b/storage/tokudb/ha_tokudb_update.cc index 1e2d6c0cdbf..fabd1a82d0c 100644 --- a/storage/tokudb/ha_tokudb_update.cc +++ b/storage/tokudb/ha_tokudb_update.cc @@ -53,14 +53,19 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. // Replace field_offset // Debug function to dump an Item -static void dump_item(Item *item) { +static void dump_item(Item* item) { fprintf(stderr, "%u", item->type()); switch (item->type()) { case Item::FUNC_ITEM: { - Item_func *func = static_cast<Item_func*>(item); + Item_func* func = static_cast<Item_func*>(item); uint n = func->argument_count(); - Item **arguments = func->arguments(); - fprintf(stderr, ":func=%u,%s,%u(", func->functype(), func->func_name(), n); + Item** arguments = func->arguments(); + fprintf( + stderr, + ":func=%u,%s,%u(", + func->functype(), + func->func_name(), + n); for (uint i = 0; i < n ; i++) { dump_item(arguments[i]); if (i < n-1) @@ -70,25 +75,30 @@ static void dump_item(Item *item) { break; } case Item::INT_ITEM: { - Item_int *int_item = static_cast<Item_int*>(item); + Item_int* int_item = static_cast<Item_int*>(item); fprintf(stderr, ":int=%lld", int_item->val_int()); break; } case Item::STRING_ITEM: { - Item_string *str_item = static_cast<Item_string*>(item); + Item_string* str_item = static_cast<Item_string*>(item); fprintf(stderr, ":str=%s", str_item->val_str(NULL)->c_ptr()); break; } case Item::FIELD_ITEM: { - Item_field *field_item = static_cast<Item_field*>(item); - fprintf(stderr, ":field=%s.%s.%s", field_item->db_name, field_item->table_name, field_item->field_name); + Item_field* field_item = static_cast<Item_field*>(item); + fprintf( + stderr, + ":field=%s.%s.%s", + field_item->db_name, + field_item->table_name, + field_item->field_name); break; } case Item::COND_ITEM: { - Item_cond *cond_item = static_cast<Item_cond*>(item); + Item_cond* cond_item = static_cast<Item_cond*>(item); fprintf(stderr, ":cond=%s(\n", cond_item->func_name()); List_iterator<Item> li(*cond_item->argument_list()); - Item *list_item; + Item* list_item; while ((list_item = li++)) { dump_item(list_item); fprintf(stderr, "\n"); @@ -97,7 +107,7 @@ static void dump_item(Item *item) { break; } case Item::INSERT_VALUE_ITEM: { - Item_insert_value *value_item = static_cast<Item_insert_value*>(item); + Item_insert_value* value_item = static_cast<Item_insert_value*>(item); fprintf(stderr, ":insert_value"); dump_item(value_item->arg); break; @@ -109,10 +119,10 @@ static void dump_item(Item *item) { } // Debug function to dump an Item list -static void dump_item_list(const char *h, List<Item> &l) { +static void dump_item_list(const char* h, List<Item> &l) { fprintf(stderr, "%s elements=%u\n", h, l.elements); List_iterator<Item> li(l); - Item *item; + Item* item; while ((item = li++) != NULL) { dump_item(item); fprintf(stderr, "\n"); @@ -120,10 +130,10 @@ static void dump_item_list(const char *h, List<Item> &l) { } // Find a Field by its Item name -static Field *find_field_by_name(TABLE *table, Item *item) { +static Field* find_field_by_name(TABLE* table, Item* item) { if (item->type() != Item::FIELD_ITEM) return NULL; - Item_field *field_item = static_cast<Item_field*>(item); + Item_field* field_item = static_cast<Item_field*>(item); #if 0 if (strcmp(table->s->db.str, field_item->db_name) != 0 || strcmp(table->s->table_name.str, field_item->table_name) != 0) @@ -146,7 +156,12 @@ static Field *find_field_by_name(TABLE *table, Item *item) { // Return the starting offset in the value for a particular index (selected by idx) of a // particular field (selected by expand_field_num). // This only works for fixed length fields -static uint32_t fixed_field_offset(uint32_t null_bytes, KEY_AND_COL_INFO *kc_info, uint idx, uint expand_field_num) { +static uint32_t fixed_field_offset( + uint32_t null_bytes, + KEY_AND_COL_INFO* kc_info, + uint idx, + uint expand_field_num) { + uint32_t offset = null_bytes; for (uint i = 0; i < expand_field_num; i++) { if (bitmap_is_set(&kc_info->key_filters[idx], i)) @@ -156,8 +171,13 @@ static uint32_t fixed_field_offset(uint32_t null_bytes, KEY_AND_COL_INFO *kc_inf return offset; } -static uint32_t var_field_index(TABLE *table, KEY_AND_COL_INFO *kc_info, uint idx, uint field_num) { - assert(field_num < table->s->fields); +static uint32_t var_field_index( + TABLE* table, + KEY_AND_COL_INFO* kc_info, + uint idx, + uint field_num) { + + assert_always(field_num < table->s->fields); uint v_index = 0; for (uint i = 0; i < table->s->fields; i++) { if (bitmap_is_set(&kc_info->key_filters[idx], i)) @@ -171,26 +191,37 @@ static uint32_t var_field_index(TABLE *table, KEY_AND_COL_INFO *kc_info, uint id return v_index; } -static uint32_t blob_field_index(TABLE *table, KEY_AND_COL_INFO *kc_info, uint idx, uint field_num) { - assert(field_num < table->s->fields); +static uint32_t blob_field_index( + TABLE* table, + KEY_AND_COL_INFO* kc_info, + uint idx, + uint field_num) { + + assert_always(field_num < table->s->fields); uint b_index; for (b_index = 0; b_index < kc_info->num_blobs; b_index++) { if (kc_info->blob_fields[b_index] == field_num) break; } - assert(b_index < kc_info->num_blobs); + assert_always(b_index < kc_info->num_blobs); return b_index; } // Determine if an update operation can be offloaded to the storage engine. -// The update operation consists of a list of update expressions (fields[i] = values[i]), and a list -// of where conditions (conds). The function returns 0 if the update is handled in the storage engine. +// The update operation consists of a list of update expressions +// (fields[i] = values[i]), and a list of where conditions (conds). +// The function returns 0 if the update is handled in the storage engine. // Otherwise, an error is returned. -int ha_tokudb::fast_update(THD *thd, List<Item> &update_fields, List<Item> &update_values, Item *conds) { +int ha_tokudb::fast_update( + THD* thd, + List<Item>& update_fields, + List<Item>& update_values, + Item* conds) { + TOKUDB_HANDLER_DBUG_ENTER(""); int error = 0; - if (tokudb_debug & TOKUDB_DEBUG_UPSERT) { + if (TOKUDB_UNLIKELY(TOKUDB_DEBUG_FLAGS(TOKUDB_DEBUG_UPSERT))) { dump_item_list("fields", update_fields); dump_item_list("values", update_values); if (conds) { @@ -198,7 +229,8 @@ int ha_tokudb::fast_update(THD *thd, List<Item> &update_fields, List<Item> &upda } } - if (update_fields.elements < 1 || update_fields.elements != update_values.elements) { + if (update_fields.elements < 1 || + update_fields.elements != update_values.elements) { error = ENOTSUP; // something is fishy with the parameters goto return_error; } @@ -208,14 +240,18 @@ int ha_tokudb::fast_update(THD *thd, List<Item> &update_fields, List<Item> &upda goto check_error; } - error = send_update_message(update_fields, update_values, conds, transaction); + error = send_update_message( + update_fields, + update_values, + conds, + transaction); if (error != 0) { goto check_error; } check_error: if (error != 0) { - if (THDVAR(thd, disable_slow_update) != 0) + if (tokudb::sysvars::disable_slow_update(thd) != 0) error = HA_ERR_UNSUPPORTED; if (error != ENOTSUP) print_error(error, MYF(0)); @@ -225,18 +261,20 @@ return_error: TOKUDB_HANDLER_DBUG_RETURN(error); } -// Return true if an expression is a simple int expression or a simple function of +- int expression. -static bool check_int_result(Item *item) { +// Return true if an expression is a simple int expression or a simple function +// of +- int expression. +static bool check_int_result(Item* item) { Item::Type t = item->type(); if (t == Item::INT_ITEM) { return true; } else if (t == Item::FUNC_ITEM) { - Item_func *item_func = static_cast<Item_func*>(item); - if (strcmp(item_func->func_name(), "+") != 0 && strcmp(item_func->func_name(), "-") != 0) + Item_func* item_func = static_cast<Item_func*>(item); + if (strcmp(item_func->func_name(), "+") != 0 && + strcmp(item_func->func_name(), "-") != 0) return false; if (item_func->argument_count() != 1) return false; - Item **arguments = item_func->arguments(); + Item** arguments = item_func->arguments(); if (arguments[0]->type() != Item::INT_ITEM) return false; return true; @@ -245,36 +283,43 @@ static bool check_int_result(Item *item) { } // check that an item is an insert value item with the same field name -static bool check_insert_value(Item *item, const char *field_name) { +static bool check_insert_value(Item* item, const char* field_name) { if (item->type() != Item::INSERT_VALUE_ITEM) return false; - Item_insert_value *value_item = static_cast<Item_insert_value*>(item); + Item_insert_value* value_item = static_cast<Item_insert_value*>(item); if (value_item->arg->type() != Item::FIELD_ITEM) return false; - Item_field *arg = static_cast<Item_field*>(value_item->arg); + Item_field* arg = static_cast<Item_field*>(value_item->arg); if (strcmp(field_name, arg->field_name) != 0) return false; return true; } // Return true if an expression looks like field_name op constant. -static bool check_x_op_constant(const char *field_name, Item *item, const char *op, Item **item_constant, bool allow_insert_value) { +static bool check_x_op_constant( + const char* field_name, + Item* item, + const char* op, + Item** item_constant, + bool allow_insert_value) { + if (item->type() != Item::FUNC_ITEM) return false; - Item_func *item_func = static_cast<Item_func*>(item); + Item_func* item_func = static_cast<Item_func*>(item); if (strcmp(item_func->func_name(), op) != 0) return false; - Item **arguments = item_func->arguments(); + Item** arguments = item_func->arguments(); uint n = item_func->argument_count(); if (n != 2) return false; if (arguments[0]->type() != Item::FIELD_ITEM) return false; - Item_field *arg0 = static_cast<Item_field*>(arguments[0]); + Item_field* arg0 = static_cast<Item_field*>(arguments[0]); if (strcmp(field_name, arg0->field_name) != 0) return false; if (!check_int_result(arguments[1])) - if (!(allow_insert_value && check_insert_value(arguments[1], field_name))) + if (!(allow_insert_value && + check_insert_value(arguments[1], field_name))) return false; *item_constant = arguments[1]; return true; @@ -282,33 +327,35 @@ static bool check_x_op_constant(const char *field_name, Item *item, const char * // Return true if an expression looks like field_name = constant static bool check_x_equal_0(const char *field_name, Item *item) { - Item *item_constant; + Item* item_constant; if (!check_x_op_constant(field_name, item, "=", &item_constant, false)) return false; - if (item_constant->type() != Item::INT_ITEM || item_constant->val_int() != 0) + if (item_constant->type() != Item::INT_ITEM || + item_constant->val_int() != 0) return false; return true; } // Return true if an expression looks like fieldname - 1 -static bool check_x_minus_1(const char *field_name, Item *item) { - Item *item_constant; +static bool check_x_minus_1(const char* field_name, Item* item) { + Item* item_constant; if (!check_x_op_constant(field_name, item, "-", &item_constant, false)) return false; - if (item_constant->type() != Item::INT_ITEM || item_constant->val_int() != 1) + if (item_constant->type() != Item::INT_ITEM || + item_constant->val_int() != 1) return false; return true; } // Return true if an expression looks like if(fieldname=0, 0, fieldname-1) and // the field named by fieldname is an unsigned int. -static bool check_decr_floor_expression(Field *lhs_field, Item *item) { +static bool check_decr_floor_expression(Field* lhs_field, Item* item) { if (item->type() != Item::FUNC_ITEM) return false; - Item_func *item_func = static_cast<Item_func*>(item); + Item_func* item_func = static_cast<Item_func*>(item); if (strcmp(item_func->func_name(), "if") != 0) return false; - Item **arguments = item_func->arguments(); + Item** arguments = item_func->arguments(); uint n = item_func->argument_count(); if (n != 3) return false; @@ -324,8 +371,13 @@ static bool check_decr_floor_expression(Field *lhs_field, Item *item) { } // Check if lhs = rhs expression is simple. Return true if it is. -static bool check_update_expression(Item *lhs_item, Item *rhs_item, TABLE *table, bool allow_insert_value) { - Field *lhs_field = find_field_by_name(table, lhs_item); +static bool check_update_expression( + Item* lhs_item, + Item* rhs_item, + TABLE* table, + bool allow_insert_value) { + + Field* lhs_field = find_field_by_name(table, lhs_item); if (lhs_field == NULL) return false; if (!lhs_field->part_of_key.is_clear_all()) @@ -340,16 +392,26 @@ static bool check_update_expression(Item *lhs_item, Item *rhs_item, TABLE *table case MYSQL_TYPE_LONGLONG: if (check_int_result(rhs_item)) return true; - Item *item_constant; - if (check_x_op_constant(lhs_field->field_name, rhs_item, "+", &item_constant, allow_insert_value)) + Item* item_constant; + if (check_x_op_constant( + lhs_field->field_name, + rhs_item, + "+", + &item_constant, + allow_insert_value)) return true; - if (check_x_op_constant(lhs_field->field_name, rhs_item, "-", &item_constant, allow_insert_value)) + if (check_x_op_constant( + lhs_field->field_name, + rhs_item, + "-", + &item_constant, + allow_insert_value)) return true; if (check_decr_floor_expression(lhs_field, rhs_item)) return true; break; case MYSQL_TYPE_STRING: - if (rhs_type == Item::INT_ITEM || rhs_type == Item::STRING_ITEM) + if (rhs_type == Item::INT_ITEM || rhs_type == Item::STRING_ITEM) return true; break; case MYSQL_TYPE_VARCHAR: @@ -364,26 +426,35 @@ static bool check_update_expression(Item *lhs_item, Item *rhs_item, TABLE *table } // Check that all update expressions are simple. Return true if they are. -static bool check_all_update_expressions(List<Item> &fields, List<Item> &values, TABLE *table, bool allow_insert_value) { +static bool check_all_update_expressions( + List<Item>& fields, + List<Item>& values, + TABLE* table, + bool allow_insert_value) { + List_iterator<Item> lhs_i(fields); List_iterator<Item> rhs_i(values); while (1) { - Item *lhs_item = lhs_i++; + Item* lhs_item = lhs_i++; if (lhs_item == NULL) break; - Item *rhs_item = rhs_i++; - assert(rhs_item != NULL); - if (!check_update_expression(lhs_item, rhs_item, table, allow_insert_value)) + Item* rhs_item = rhs_i++; + assert_always(rhs_item != NULL); + if (!check_update_expression( + lhs_item, + rhs_item, + table, + allow_insert_value)) return false; } return true; } -static bool full_field_in_key(TABLE *table, Field *field) { - assert(table->s->primary_key < table->s->keys); - KEY *key = &table->s->key_info[table->s->primary_key]; +static bool full_field_in_key(TABLE* table, Field* field) { + assert_always(table->s->primary_key < table->s->keys); + KEY* key = &table->s->key_info[table->s->primary_key]; for (uint i = 0; i < get_key_parts(key); i++) { - KEY_PART_INFO *key_part = &key->key_part[i]; + KEY_PART_INFO* key_part = &key->key_part[i]; if (strcmp(field->field_name, key_part->field->field_name) == 0) { return key_part->length == field->field_length; } @@ -391,19 +462,24 @@ static bool full_field_in_key(TABLE *table, Field *field) { return false; } -// Check that an expression looks like fieldname = constant, fieldname is part of the -// primary key, and the named field is an int, char or varchar type. Return true if it does. -static bool check_pk_field_equal_constant(Item *item, TABLE *table, MY_BITMAP *pk_fields) { +// Check that an expression looks like fieldname = constant, fieldname is part +// of the primary key, and the named field is an int, char or varchar type. +// Return true if it does. +static bool check_pk_field_equal_constant( + Item* item, + TABLE* table, + MY_BITMAP* pk_fields) { + if (item->type() != Item::FUNC_ITEM) return false; - Item_func *func = static_cast<Item_func*>(item); + Item_func* func = static_cast<Item_func*>(item); if (strcmp(func->func_name(), "=") != 0) return false; uint n = func->argument_count(); if (n != 2) return false; - Item **arguments = func->arguments(); - Field *field = find_field_by_name(table, arguments[0]); + Item** arguments = func->arguments(); + Field* field = find_field_by_name(table, arguments[0]); if (field == NULL) return false; if (!bitmap_test_and_clear(pk_fields, field->field_index)) @@ -414,19 +490,21 @@ static bool check_pk_field_equal_constant(Item *item, TABLE *table, MY_BITMAP *p case MYSQL_TYPE_INT24: case MYSQL_TYPE_LONG: case MYSQL_TYPE_LONGLONG: - return arguments[1]->type() == Item::INT_ITEM || arguments[1]->type() == Item::STRING_ITEM; + return arguments[1]->type() == Item::INT_ITEM || + arguments[1]->type() == Item::STRING_ITEM; case MYSQL_TYPE_STRING: case MYSQL_TYPE_VARCHAR: return full_field_in_key(table, field) && - (arguments[1]->type() == Item::INT_ITEM || arguments[1]->type() == Item::STRING_ITEM); + (arguments[1]->type() == Item::INT_ITEM || + arguments[1]->type() == Item::STRING_ITEM); default: return false; } } -// Check that the where condition covers all of the primary key components with fieldname = constant -// expressions. Return true if it does. -static bool check_point_update(Item *conds, TABLE *table) { +// Check that the where condition covers all of the primary key components +// with fieldname = constant expressions. Return true if it does. +static bool check_point_update(Item* conds, TABLE* table) { bool result = false; if (conds == NULL) @@ -435,8 +513,8 @@ static bool check_point_update(Item *conds, TABLE *table) { if (table->s->primary_key >= table->s->keys) return false; // no primary key defined - // use a bitmap of the primary key fields to keep track of those fields that are covered - // by the where conditions + // use a bitmap of the primary key fields to keep track of those fields + // that are covered by the where conditions MY_BITMAP pk_fields; if (bitmap_init(&pk_fields, NULL, table->s->fields, FALSE)) // 1 -> failure return false; @@ -449,14 +527,17 @@ static bool check_point_update(Item *conds, TABLE *table) { result = check_pk_field_equal_constant(conds, table, &pk_fields); break; case Item::COND_ITEM: { - Item_cond *cond_item = static_cast<Item_cond*>(conds); + Item_cond* cond_item = static_cast<Item_cond*>(conds); if (strcmp(cond_item->func_name(), "and") != 0) break; List_iterator<Item> li(*cond_item->argument_list()); - Item *list_item; + Item* list_item; result = true; while (result == true && (list_item = li++)) { - result = check_pk_field_equal_constant(list_item, table, &pk_fields); + result = check_pk_field_equal_constant( + list_item, + table, + &pk_fields); } break; } @@ -474,13 +555,14 @@ static bool check_point_update(Item *conds, TABLE *table) { // Precompute this when the table is opened. static bool clustering_keys_exist(TABLE *table) { for (uint keynr = 0; keynr < table->s->keys; keynr++) { - if (keynr != table->s->primary_key && key_is_clustering(&table->s->key_info[keynr])) + if (keynr != table->s->primary_key && + key_is_clustering(&table->s->key_info[keynr])) return true; } return false; } -static bool is_strict_mode(THD *thd) { +static bool is_strict_mode(THD* thd) { #if 50600 <= MYSQL_VERSION_ID && MYSQL_VERSION_ID <= 50699 return thd->is_strict_mode(); #else @@ -488,8 +570,14 @@ static bool is_strict_mode(THD *thd) { #endif } -// Check if an update operation can be handled by this storage engine. Return true if it can. -bool ha_tokudb::check_fast_update(THD *thd, List<Item> &fields, List<Item> &values, Item *conds) { +// Check if an update operation can be handled by this storage engine. +// Return true if it can. +bool ha_tokudb::check_fast_update( + THD* thd, + List<Item>& fields, + List<Item>& values, + Item* conds) { + if (!transaction) return false; @@ -503,10 +591,12 @@ bool ha_tokudb::check_fast_update(THD *thd, List<Item> &fields, List<Item> &valu // no binlog if (mysql_bin_log.is_open() && - !(thd->variables.binlog_format == BINLOG_FORMAT_STMT || thd->variables.binlog_format == BINLOG_FORMAT_MIXED)) + !(thd->variables.binlog_format == BINLOG_FORMAT_STMT || + thd->variables.binlog_format == BINLOG_FORMAT_MIXED)) return false; - // no clustering keys (need to broadcast an increment into the clustering keys since we are selecting with the primary key) + // no clustering keys (need to broadcast an increment into the clustering + // keys since we are selecting with the primary key) if (clustering_keys_exist(table)) return false; @@ -519,22 +609,34 @@ bool ha_tokudb::check_fast_update(THD *thd, List<Item> &fields, List<Item> &valu return true; } -static void marshall_varchar_descriptor(tokudb::buffer &b, TABLE *table, KEY_AND_COL_INFO *kc_info, uint key_num) { +static void marshall_varchar_descriptor( + tokudb::buffer& b, + TABLE* table, + KEY_AND_COL_INFO* kc_info, + uint key_num) { + b.append_ui<uint32_t>('v'); - b.append_ui<uint32_t>(table->s->null_bytes + kc_info->mcp_info[key_num].fixed_field_size); + b.append_ui<uint32_t>( + table->s->null_bytes + kc_info->mcp_info[key_num].fixed_field_size); uint32_t var_offset_bytes = kc_info->mcp_info[key_num].len_of_offsets; b.append_ui<uint32_t>(var_offset_bytes); - b.append_ui<uint32_t>(var_offset_bytes == 0 ? 0 : kc_info->num_offset_bytes); + b.append_ui<uint32_t>( + var_offset_bytes == 0 ? 0 : kc_info->num_offset_bytes); } -static void marshall_blobs_descriptor(tokudb::buffer &b, TABLE *table, KEY_AND_COL_INFO *kc_info) { +static void marshall_blobs_descriptor( + tokudb::buffer& b, + TABLE* table, + KEY_AND_COL_INFO* kc_info) { + b.append_ui<uint32_t>('b'); uint32_t n = kc_info->num_blobs; b.append_ui<uint32_t>(n); for (uint i = 0; i < n; i++) { uint blob_field_index = kc_info->blob_fields[i]; - assert(blob_field_index < table->s->fields); - uint8_t blob_field_length = table->s->field[blob_field_index]->row_pack_length(); + assert_always(blob_field_index < table->s->fields); + uint8_t blob_field_length = + table->s->field[blob_field_index]->row_pack_length(); b.append(&blob_field_length, sizeof blob_field_length); } } @@ -542,30 +644,37 @@ static void marshall_blobs_descriptor(tokudb::buffer &b, TABLE *table, KEY_AND_C static inline uint32_t get_null_bit_position(uint32_t null_bit); // evaluate the int value of an item -static longlong item_val_int(Item *item) { +static longlong item_val_int(Item* item) { Item::Type t = item->type(); if (t == Item::INSERT_VALUE_ITEM) { - Item_insert_value *value_item = static_cast<Item_insert_value*>(item); + Item_insert_value* value_item = static_cast<Item_insert_value*>(item); return value_item->arg->val_int(); } else return item->val_int(); } // Marshall update operations to a buffer. -static void marshall_update(tokudb::buffer &b, Item *lhs_item, Item *rhs_item, TABLE *table, TOKUDB_SHARE *share) { +static void marshall_update( + tokudb::buffer& b, + Item* lhs_item, + Item* rhs_item, + TABLE* table, + TOKUDB_SHARE* share) { + // figure out the update operation type (again) - Field *lhs_field = find_field_by_name(table, lhs_item); - assert(lhs_field); // we found it before, so this should work + Field* lhs_field = find_field_by_name(table, lhs_item); + assert_always(lhs_field); // we found it before, so this should work // compute the update info uint32_t field_type; uint32_t field_null_num = 0; if (lhs_field->real_maybe_null()) { uint32_t field_num = lhs_field->field_index; - field_null_num = ((field_num/8)*8 + get_null_bit_position(lhs_field->null_bit)) + 1; + field_null_num = + ((field_num/8)*8 + get_null_bit_position(lhs_field->null_bit)) + 1; } uint32_t offset; - void *v_ptr = NULL; + void* v_ptr = NULL; uint32_t v_length; uint32_t update_operation; longlong v_ll; @@ -577,9 +686,14 @@ static void marshall_update(tokudb::buffer &b, Item *lhs_item, Item *rhs_item, T case MYSQL_TYPE_INT24: case MYSQL_TYPE_LONG: case MYSQL_TYPE_LONGLONG: { - Field_num *lhs_num = static_cast<Field_num*>(lhs_field); + Field_num* lhs_num = static_cast<Field_num*>(lhs_field); field_type = lhs_num->unsigned_flag ? UPDATE_TYPE_UINT : UPDATE_TYPE_INT; - offset = fixed_field_offset(table->s->null_bytes, &share->kc_info, table->s->primary_key, lhs_field->field_index); + offset = + fixed_field_offset( + table->s->null_bytes, + &share->kc_info, + table->s->primary_key, + lhs_field->field_index); switch (rhs_item->type()) { case Item::INT_ITEM: { update_operation = '='; @@ -589,10 +703,12 @@ static void marshall_update(tokudb::buffer &b, Item *lhs_item, Item *rhs_item, T break; } case Item::FUNC_ITEM: { - Item_func *rhs_func = static_cast<Item_func*>(rhs_item); - Item **arguments = rhs_func->arguments(); + Item_func* rhs_func = static_cast<Item_func*>(rhs_item); + Item** arguments = rhs_func->arguments(); + // we only support one if function for now, and it is a + // decrement with floor. if (strcmp(rhs_func->func_name(), "if") == 0) { - update_operation = '-'; // we only support one if function for now, and it is a decrement with floor. + update_operation = '-'; v_ll = 1; } else if (rhs_func->argument_count() == 1) { update_operation = '='; @@ -606,14 +722,20 @@ static void marshall_update(tokudb::buffer &b, Item *lhs_item, Item *rhs_item, T break; } default: - assert(0); + assert_unreachable(); } break; } case MYSQL_TYPE_STRING: { update_operation = '='; - field_type = lhs_field->binary() ? UPDATE_TYPE_BINARY : UPDATE_TYPE_CHAR; - offset = fixed_field_offset(table->s->null_bytes, &share->kc_info, table->s->primary_key, lhs_field->field_index); + field_type = + lhs_field->binary() ? UPDATE_TYPE_BINARY : UPDATE_TYPE_CHAR; + offset = + fixed_field_offset( + table->s->null_bytes, + &share->kc_info, + table->s->primary_key, + lhs_field->field_index); v_str = *rhs_item->val_str(&v_str); v_length = v_str.length(); if (v_length >= lhs_field->pack_length()) { @@ -621,7 +743,8 @@ static void marshall_update(tokudb::buffer &b, Item *lhs_item, Item *rhs_item, T v_str.length(v_length); // truncate } else { v_length = lhs_field->pack_length(); - uchar pad_char = lhs_field->binary() ? 0 : lhs_field->charset()->pad_char; + uchar pad_char = + lhs_field->binary() ? 0 : lhs_field->charset()->pad_char; v_str.fill(lhs_field->pack_length(), pad_char); // pad } v_ptr = v_str.c_ptr(); @@ -629,8 +752,14 @@ static void marshall_update(tokudb::buffer &b, Item *lhs_item, Item *rhs_item, T } case MYSQL_TYPE_VARCHAR: { update_operation = '='; - field_type = lhs_field->binary() ? UPDATE_TYPE_VARBINARY : UPDATE_TYPE_VARCHAR; - offset = var_field_index(table, &share->kc_info, table->s->primary_key, lhs_field->field_index); + field_type = + lhs_field->binary() ? UPDATE_TYPE_VARBINARY : UPDATE_TYPE_VARCHAR; + offset = + var_field_index( + table, + &share->kc_info, + table->s->primary_key, + lhs_field->field_index); v_str = *rhs_item->val_str(&v_str); v_length = v_str.length(); if (v_length >= lhs_field->row_pack_length()) { @@ -643,7 +772,12 @@ static void marshall_update(tokudb::buffer &b, Item *lhs_item, Item *rhs_item, T case MYSQL_TYPE_BLOB: { update_operation = '='; field_type = lhs_field->binary() ? UPDATE_TYPE_BLOB : UPDATE_TYPE_TEXT; - offset = blob_field_index(table, &share->kc_info, table->s->primary_key, lhs_field->field_index); + offset = + blob_field_index( + table, + &share->kc_info, + table->s->primary_key, + lhs_field->field_index); v_str = *rhs_item->val_str(&v_str); v_length = v_str.length(); if (v_length >= lhs_field->max_data_length()) { @@ -654,7 +788,7 @@ static void marshall_update(tokudb::buffer &b, Item *lhs_item, Item *rhs_item, T break; } default: - assert(0); + assert_unreachable(); } // marshall the update fields into the buffer @@ -667,14 +801,14 @@ static void marshall_update(tokudb::buffer &b, Item *lhs_item, Item *rhs_item, T } // Save an item's value into the appropriate field. Return 0 if successful. -static int save_in_field(Item *item, TABLE *table) { - assert(item->type() == Item::FUNC_ITEM); +static int save_in_field(Item* item, TABLE* table) { + assert_always(item->type() == Item::FUNC_ITEM); Item_func *func = static_cast<Item_func*>(item); - assert(strcmp(func->func_name(), "=") == 0); + assert_always(strcmp(func->func_name(), "=") == 0); uint n = func->argument_count(); - assert(n == 2); + assert_always(n == 2); Item **arguments = func->arguments(); - assert(arguments[0]->type() == Item::FIELD_ITEM); + assert_always(arguments[0]->type() == Item::FIELD_ITEM); Item_field *field_item = static_cast<Item_field*>(arguments[0]); my_bitmap_map *old_map = dbug_tmp_use_all_columns(table, table->write_set); int error = arguments[1]->save_in_field(field_item->field, 0); @@ -682,7 +816,11 @@ static int save_in_field(Item *item, TABLE *table) { return error; } -static void count_update_types(Field *lhs_field, uint *num_varchars, uint *num_blobs) { +static void count_update_types( + Field* lhs_field, + uint* num_varchars, + uint* num_blobs) { + switch (lhs_field->type()) { case MYSQL_TYPE_VARCHAR: *num_varchars += 1; @@ -695,8 +833,14 @@ static void count_update_types(Field *lhs_field, uint *num_varchars, uint *num_b } } -// Generate an update message for an update operation and send it into the primary tree. Return 0 if successful. -int ha_tokudb::send_update_message(List<Item> &update_fields, List<Item> &update_values, Item *conds, DB_TXN *txn) { +// Generate an update message for an update operation and send it into the +// primary tree. Return 0 if successful. +int ha_tokudb::send_update_message( + List<Item>& update_fields, + List<Item>& update_values, + Item* conds, + DB_TXN* txn) { + int error; // Save the primary key from the where conditions @@ -704,26 +848,32 @@ int ha_tokudb::send_update_message(List<Item> &update_fields, List<Item> &update if (t == Item::FUNC_ITEM) { error = save_in_field(conds, table); } else if (t == Item::COND_ITEM) { - Item_cond *cond_item = static_cast<Item_cond*>(conds); + Item_cond* cond_item = static_cast<Item_cond*>(conds); List_iterator<Item> li(*cond_item->argument_list()); - Item *list_item; + Item* list_item; error = 0; while (error == 0 && (list_item = li++)) { error = save_in_field(list_item, table); } - } else - assert(0); + } else { + assert_unreachable(); + } if (error) return error; // put the primary key into key_buff and wrap it with key_dbt DBT key_dbt; bool has_null; - create_dbt_key_from_table(&key_dbt, primary_key, key_buff, table->record[0], &has_null); - + create_dbt_key_from_table( + &key_dbt, + primary_key, + key_buff, + table->record[0], + &has_null); + // construct the update message tokudb::buffer update_message; - + uint8_t op = UPDATE_OP_UPDATE_2; update_message.append(&op, sizeof op); @@ -731,12 +881,12 @@ int ha_tokudb::send_update_message(List<Item> &update_fields, List<Item> &update uint num_varchars = 0, num_blobs = 0; if (1) { List_iterator<Item> lhs_i(update_fields); - Item *lhs_item; + Item* lhs_item; while ((lhs_item = lhs_i++)) { if (lhs_item == NULL) break; - Field *lhs_field = find_field_by_name(table, lhs_item); - assert(lhs_field); // we found it before, so this should work + Field* lhs_field = find_field_by_name(table, lhs_item); + assert_always(lhs_field); // we found it before, so this should work count_update_types(lhs_field, &num_varchars, &num_blobs); } if (num_varchars > 0 || num_blobs > 0) @@ -747,56 +897,75 @@ int ha_tokudb::send_update_message(List<Item> &update_fields, List<Item> &update // append the updates update_message.append_ui<uint32_t>(num_updates); - + if (num_varchars > 0 || num_blobs > 0) - marshall_varchar_descriptor(update_message, table, &share->kc_info, table->s->primary_key); + marshall_varchar_descriptor( + update_message, + table, + &share->kc_info, + table->s->primary_key); if (num_blobs > 0) marshall_blobs_descriptor(update_message, table, &share->kc_info); List_iterator<Item> lhs_i(update_fields); List_iterator<Item> rhs_i(update_values); while (error == 0) { - Item *lhs_item = lhs_i++; + Item* lhs_item = lhs_i++; if (lhs_item == NULL) break; - Item *rhs_item = rhs_i++; - assert(rhs_item != NULL); + Item* rhs_item = rhs_i++; + assert_always(rhs_item != NULL); marshall_update(update_message, lhs_item, rhs_item, table, share); } - rw_rdlock(&share->num_DBs_lock); + share->_num_DBs_lock.lock_read(); - if (share->num_DBs > table->s->keys + tokudb_test(hidden_primary_key)) { // hot index in progress + // hot index in progress + if (share->num_DBs > table->s->keys + tokudb_test(hidden_primary_key)) { error = ENOTSUP; // run on the slow path } else { - // send the update message + // send the update message DBT update_dbt; memset(&update_dbt, 0, sizeof update_dbt); update_dbt.data = update_message.data(); update_dbt.size = update_message.size(); - error = share->key_file[primary_key]->update(share->key_file[primary_key], txn, &key_dbt, &update_dbt, 0); + error = + share->key_file[primary_key]->update( + share->key_file[primary_key], + txn, + &key_dbt, + &update_dbt, + 0); } - rw_unlock(&share->num_DBs_lock); - + share->_num_DBs_lock.unlock(); + return error; } // Determine if an upsert operation can be offloaded to the storage engine. -// An upsert consists of a row and a list of update expressions (update_fields[i] = update_values[i]). -// The function returns 0 if the upsert is handled in the storage engine. Otherwise, an error code is returned. -int ha_tokudb::upsert(THD *thd, List<Item> &update_fields, List<Item> &update_values) { +// An upsert consists of a row and a list of update expressions +// (update_fields[i] = update_values[i]). +// The function returns 0 if the upsert is handled in the storage engine. +// Otherwise, an error code is returned. +int ha_tokudb::upsert( + THD* thd, + List<Item>& update_fields, + List<Item>& update_values) { + TOKUDB_HANDLER_DBUG_ENTER(""); int error = 0; - if (tokudb_debug & TOKUDB_DEBUG_UPSERT) { + if (TOKUDB_UNLIKELY(TOKUDB_DEBUG_FLAGS(TOKUDB_DEBUG_UPSERT))) { fprintf(stderr, "upsert\n"); dump_item_list("update_fields", update_fields); dump_item_list("update_values", update_values); } - if (update_fields.elements < 1 || update_fields.elements != update_values.elements) { - error = ENOTSUP; // not an upsert or something is fishy with the parameters + // not an upsert or something is fishy with the parameters + if (update_fields.elements < 1 || + update_fields.elements != update_values.elements) { + error = ENOTSUP; goto return_error; } @@ -812,7 +981,7 @@ int ha_tokudb::upsert(THD *thd, List<Item> &update_fields, List<Item> &update_va check_error: if (error != 0) { - if (THDVAR(thd, disable_slow_upsert) != 0) + if (tokudb::sysvars::disable_slow_upsert(thd) != 0) error = HA_ERR_UNSUPPORTED; if (error != ENOTSUP) print_error(error, MYF(0)); @@ -822,8 +991,13 @@ return_error: TOKUDB_HANDLER_DBUG_RETURN(error); } -// Check if an upsert can be handled by this storage engine. Return trus if it can. -bool ha_tokudb::check_upsert(THD *thd, List<Item> &update_fields, List<Item> &update_values) { +// Check if an upsert can be handled by this storage engine. +// Return true if it can. +bool ha_tokudb::check_upsert( + THD* thd, + List<Item>& update_fields, + List<Item>& update_values) { + if (!transaction) return false; @@ -845,23 +1019,38 @@ bool ha_tokudb::check_upsert(THD *thd, List<Item> &update_fields, List<Item> &up // no binlog if (mysql_bin_log.is_open() && - !(thd->variables.binlog_format == BINLOG_FORMAT_STMT || thd->variables.binlog_format == BINLOG_FORMAT_MIXED)) + !(thd->variables.binlog_format == BINLOG_FORMAT_STMT || + thd->variables.binlog_format == BINLOG_FORMAT_MIXED)) return false; - if (!check_all_update_expressions(update_fields, update_values, table, true)) + if (!check_all_update_expressions( + update_fields, + update_values, + table, + true)) return false; return true; } -// Generate an upsert message and send it into the primary tree. Return 0 if successful. -int ha_tokudb::send_upsert_message(THD *thd, List<Item> &update_fields, List<Item> &update_values, DB_TXN *txn) { +// Generate an upsert message and send it into the primary tree. +// Return 0 if successful. +int ha_tokudb::send_upsert_message( + THD* thd, + List<Item>& update_fields, + List<Item>& update_values, + DB_TXN* txn) { int error = 0; // generate primary key DBT key_dbt; bool has_null; - create_dbt_key_from_table(&key_dbt, primary_key, primary_key_buff, table->record[0], &has_null); + create_dbt_key_from_table( + &key_dbt, + primary_key, + primary_key_buff, + table->record[0], + &has_null); // generate packed row DBT row; @@ -883,12 +1072,12 @@ int ha_tokudb::send_upsert_message(THD *thd, List<Item> &update_fields, List<Ite uint num_varchars = 0, num_blobs = 0; if (1) { List_iterator<Item> lhs_i(update_fields); - Item *lhs_item; + Item* lhs_item; while ((lhs_item = lhs_i++)) { if (lhs_item == NULL) break; - Field *lhs_field = find_field_by_name(table, lhs_item); - assert(lhs_field); // we found it before, so this should work + Field* lhs_field = find_field_by_name(table, lhs_item); + assert_always(lhs_field); // we found it before, so this should work count_update_types(lhs_field, &num_varchars, &num_blobs); } if (num_varchars > 0 || num_blobs > 0) @@ -901,35 +1090,44 @@ int ha_tokudb::send_upsert_message(THD *thd, List<Item> &update_fields, List<Ite update_message.append_ui<uint32_t>(num_updates); if (num_varchars > 0 || num_blobs > 0) - marshall_varchar_descriptor(update_message, table, &share->kc_info, table->s->primary_key); + marshall_varchar_descriptor( + update_message, + table, &share->kc_info, + table->s->primary_key); if (num_blobs > 0) marshall_blobs_descriptor(update_message, table, &share->kc_info); List_iterator<Item> lhs_i(update_fields); List_iterator<Item> rhs_i(update_values); while (1) { - Item *lhs_item = lhs_i++; + Item* lhs_item = lhs_i++; if (lhs_item == NULL) break; - Item *rhs_item = rhs_i++; - if (rhs_item == NULL) - assert(0); // can not happen + Item* rhs_item = rhs_i++; + assert_always(rhs_item != NULL); marshall_update(update_message, lhs_item, rhs_item, table, share); } - rw_rdlock(&share->num_DBs_lock); + share->_num_DBs_lock.lock_read(); - if (share->num_DBs > table->s->keys + tokudb_test(hidden_primary_key)) { // hot index in progress + // hot index in progress + if (share->num_DBs > table->s->keys + tokudb_test(hidden_primary_key)) { error = ENOTSUP; // run on the slow path } else { // send the upsert message DBT update_dbt; memset(&update_dbt, 0, sizeof update_dbt); update_dbt.data = update_message.data(); update_dbt.size = update_message.size(); - error = share->key_file[primary_key]->update(share->key_file[primary_key], txn, &key_dbt, &update_dbt, 0); + error = + share->key_file[primary_key]->update( + share->key_file[primary_key], + txn, + &key_dbt, + &update_dbt, + 0); } - rw_unlock(&share->num_DBs_lock); + share->_num_DBs_lock.unlock(); return error; } diff --git a/storage/tokudb/hatoku_cmp.cc b/storage/tokudb/hatoku_cmp.cc index 9892c6f5eed..a5e0874505a 100644 --- a/storage/tokudb/hatoku_cmp.cc +++ b/storage/tokudb/hatoku_cmp.cc @@ -104,8 +104,7 @@ static void get_var_field_info( data_end_offset = uint2korr(var_field_offset_ptr + 2*var_field_index); break; default: - assert(false); - break; + assert_unreachable(); } if (var_field_index) { @@ -117,8 +116,7 @@ static void get_var_field_info( data_start_offset = uint2korr(var_field_offset_ptr + 2*(var_field_index-1)); break; default: - assert(false); - break; + assert_unreachable(); } } else { @@ -126,7 +124,7 @@ static void get_var_field_info( } *start_offset = data_start_offset; - assert(data_end_offset >= data_start_offset); + assert_always(data_end_offset >= data_start_offset); *field_len = data_end_offset - data_start_offset; } @@ -153,8 +151,7 @@ static void get_blob_field_info( data_end_offset = uint2korr(var_field_data_ptr - 2); break; default: - assert(false); - break; + assert_unreachable(); } } else { @@ -245,7 +242,7 @@ static TOKU_TYPE mysql_to_toku_type (Field* field) { case MYSQL_TYPE_DECIMAL: case MYSQL_TYPE_VAR_STRING: case MYSQL_TYPE_NULL: - assert(false); + assert_unreachable(); } exit: return ret_val; @@ -312,7 +309,7 @@ static inline uchar* pack_toku_int (uchar* to_tokudb, uchar* from_mysql, uint32_ memcpy(to_tokudb, from_mysql, 8); break; default: - assert(false); + assert_unreachable(); } return to_tokudb+num_bytes; } @@ -338,7 +335,7 @@ static inline uchar* unpack_toku_int(uchar* to_mysql, uchar* from_tokudb, uint32 memcpy(to_mysql, from_tokudb, 8); break; default: - assert(false); + assert_unreachable(); } return from_tokudb+num_bytes; } @@ -390,7 +387,7 @@ static inline int cmp_toku_int (uchar* a_buf, uchar* b_buf, bool is_unsigned, ui ret_val = 0; goto exit; default: - assert(false); + assert_unreachable(); } } // @@ -438,13 +435,13 @@ static inline int cmp_toku_int (uchar* a_buf, uchar* b_buf, bool is_unsigned, ui ret_val = 0; goto exit; default: - assert(false); + assert_unreachable(); } } // // if this is hit, indicates bug in writing of this function // - assert(false); + assert_unreachable(); exit: return ret_val; } @@ -653,7 +650,7 @@ static inline uchar* unpack_toku_varbinary( int4store(to_mysql, length); break; default: - assert(false); + assert_unreachable(); } // // copy the binary data @@ -779,7 +776,7 @@ static inline uchar* unpack_toku_blob( int4store(to_mysql, length); break; default: - assert(false); + assert_unreachable(); } // // copy the binary data @@ -957,7 +954,9 @@ static inline int tokudb_compare_two_hidden_keys( const void* saved_key_data, const uint32_t saved_key_size ) { - assert( (new_key_size >= TOKUDB_HIDDEN_PRIMARY_KEY_LENGTH) && (saved_key_size >= TOKUDB_HIDDEN_PRIMARY_KEY_LENGTH) ); + assert_always( + (new_key_size >= TOKUDB_HIDDEN_PRIMARY_KEY_LENGTH) && + (saved_key_size >= TOKUDB_HIDDEN_PRIMARY_KEY_LENGTH)); ulonglong a = hpk_char_to_num((uchar *) new_key_data); ulonglong b = hpk_char_to_num((uchar *) saved_key_data); return a < b ? -1 : (a > b ? 1 : 0); @@ -997,8 +996,7 @@ static uint32_t skip_field_in_descriptor(uchar* row_desc) { row_desc_pos += sizeof(uint32_t); break; default: - assert(false); - break; + assert_unreachable(); } return (uint32_t)(row_desc_pos - row_desc); } @@ -1026,7 +1024,7 @@ static int create_toku_key_descriptor_for_key(KEY* key, uchar* buf) { // The second byte for each field is the type // TOKU_TYPE type = mysql_to_toku_type(field); - assert (type < 256); + assert_always((int)type < 256); *pos = (uchar)(type & 255); pos++; @@ -1041,7 +1039,7 @@ static int create_toku_key_descriptor_for_key(KEY* key, uchar* buf) { // case (toku_type_int): num_bytes_in_field = field->pack_length(); - assert (num_bytes_in_field < 256); + assert_always (num_bytes_in_field < 256); *pos = (uchar)(num_bytes_in_field & 255); pos++; *pos = (field->flags & UNSIGNED_FLAG) ? 1 : 0; @@ -1059,7 +1057,7 @@ static int create_toku_key_descriptor_for_key(KEY* key, uchar* buf) { case (toku_type_fixbinary): num_bytes_in_field = field->pack_length(); set_if_smaller(num_bytes_in_field, key->key_part[i].length); - assert(num_bytes_in_field < 256); + assert_always(num_bytes_in_field < 256); pos[0] = (uchar)(num_bytes_in_field & 255); pos++; break; @@ -1087,8 +1085,7 @@ static int create_toku_key_descriptor_for_key(KEY* key, uchar* buf) { pos += 4; break; default: - assert(false); - + assert_unreachable(); } } return pos - buf; @@ -1277,8 +1274,7 @@ static inline int compare_toku_field( *read_string = true; break; default: - assert(false); - break; + assert_unreachable(); } *row_desc_bytes_read = row_desc_pos - row_desc; @@ -1301,7 +1297,7 @@ static uchar* pack_toku_key_field( TOKU_TYPE toku_type = mysql_to_toku_type(field); switch(toku_type) { case (toku_type_int): - assert(key_part_length == field->pack_length()); + assert_always(key_part_length == field->pack_length()); new_pos = pack_toku_int( to_tokudb, from_mysql, @@ -1309,13 +1305,13 @@ static uchar* pack_toku_key_field( ); goto exit; case (toku_type_double): - assert(field->pack_length() == sizeof(double)); - assert(key_part_length == sizeof(double)); + assert_always(field->pack_length() == sizeof(double)); + assert_always(key_part_length == sizeof(double)); new_pos = pack_toku_double(to_tokudb, from_mysql); goto exit; case (toku_type_float): - assert(field->pack_length() == sizeof(float)); - assert(key_part_length == sizeof(float)); + assert_always(field->pack_length() == sizeof(float)); + assert_always(key_part_length == sizeof(float)); new_pos = pack_toku_float(to_tokudb, from_mysql); goto exit; case (toku_type_fixbinary): @@ -1368,9 +1364,9 @@ static uchar* pack_toku_key_field( ); goto exit; default: - assert(false); + assert_unreachable(); } - assert(false); + assert_unreachable(); exit: return new_pos; } @@ -1419,10 +1415,10 @@ static uchar* pack_key_toku_key_field( ); goto exit; default: - assert(false); + assert_unreachable(); } - assert(false); + assert_unreachable(); exit: return new_pos; } @@ -1432,16 +1428,15 @@ uchar* unpack_toku_key_field( uchar* to_mysql, uchar* from_tokudb, Field* field, - uint32_t key_part_length - ) -{ + uint32_t key_part_length) { + uchar* new_pos = NULL; uint32_t num_bytes = 0; uint32_t num_bytes_copied; TOKU_TYPE toku_type = mysql_to_toku_type(field); switch(toku_type) { case (toku_type_int): - assert(key_part_length == field->pack_length()); + assert_always(key_part_length == field->pack_length()); new_pos = unpack_toku_int( to_mysql, from_tokudb, @@ -1449,13 +1444,13 @@ uchar* unpack_toku_key_field( ); goto exit; case (toku_type_double): - assert(field->pack_length() == sizeof(double)); - assert(key_part_length == sizeof(double)); + assert_always(field->pack_length() == sizeof(double)); + assert_always(key_part_length == sizeof(double)); new_pos = unpack_toku_double(to_mysql, from_tokudb); goto exit; case (toku_type_float): - assert(field->pack_length() == sizeof(float)); - assert(key_part_length == sizeof(float)); + assert_always(field->pack_length() == sizeof(float)); + assert_always(key_part_length == sizeof(float)); new_pos = unpack_toku_float(to_mysql, from_tokudb); goto exit; case (toku_type_fixbinary): @@ -1464,8 +1459,7 @@ uchar* unpack_toku_key_field( new_pos = unpack_toku_binary( to_mysql, from_tokudb, - num_bytes - ); + num_bytes); goto exit; case (toku_type_fixstring): num_bytes = field->pack_length(); @@ -1473,11 +1467,15 @@ uchar* unpack_toku_key_field( to_mysql, from_tokudb, get_length_bytes_from_max(key_part_length), - 0 - ); - num_bytes_copied = new_pos - (from_tokudb + get_length_bytes_from_max(key_part_length)); - assert(num_bytes_copied <= num_bytes); - memset(to_mysql+num_bytes_copied, field->charset()->pad_char, num_bytes - num_bytes_copied); + 0); + num_bytes_copied = + new_pos - + (from_tokudb + get_length_bytes_from_max(key_part_length)); + assert_always(num_bytes_copied <= num_bytes); + memset( + to_mysql + num_bytes_copied, + field->charset()->pad_char, + num_bytes - num_bytes_copied); goto exit; case (toku_type_varbinary): case (toku_type_varstring): @@ -1485,21 +1483,20 @@ uchar* unpack_toku_key_field( to_mysql, from_tokudb, get_length_bytes_from_max(key_part_length), - ((Field_varstring *)field)->length_bytes - ); + ((Field_varstring*)field)->length_bytes); goto exit; case (toku_type_blob): new_pos = unpack_toku_blob( to_mysql, from_tokudb, get_length_bytes_from_max(key_part_length), - ((Field_blob *)field)->row_pack_length() //only calling this because packlength is returned - ); + //only calling this because packlength is returned + ((Field_blob *)field)->row_pack_length()); goto exit; default: - assert(false); + assert_unreachable(); } - assert(false); + assert_unreachable(); exit: return new_pos; } @@ -1513,9 +1510,8 @@ static int tokudb_compare_two_keys( const void* row_desc, const uint32_t row_desc_size, bool cmp_prefix, - bool* read_string - ) -{ + bool* read_string) { + int ret_val = 0; int8_t new_key_inf_val = COL_NEG_INF; int8_t saved_key_inf_val = COL_NEG_INF; @@ -1538,11 +1534,9 @@ static int tokudb_compare_two_keys( } row_desc_ptr++; - while ( (uint32_t)(new_key_ptr - (uchar *)new_key_data) < new_key_size && - (uint32_t)(saved_key_ptr - (uchar *)saved_key_data) < saved_key_size && - (uint32_t)(row_desc_ptr - (uchar *)row_desc) < row_desc_size - ) - { + while ((uint32_t)(new_key_ptr - (uchar*)new_key_data) < new_key_size && + (uint32_t)(saved_key_ptr - (uchar*)saved_key_data) < saved_key_size && + (uint32_t)(row_desc_ptr - (uchar*)row_desc) < row_desc_size) { uint32_t new_key_field_length; uint32_t saved_key_field_length; uint32_t row_desc_field_length; @@ -1583,8 +1577,7 @@ static int tokudb_compare_two_keys( &new_key_field_length, &saved_key_field_length, &row_desc_field_length, - read_string - ); + read_string); new_key_ptr += new_key_field_length; saved_key_ptr += saved_key_field_length; row_desc_ptr += row_desc_field_length; @@ -1592,35 +1585,30 @@ static int tokudb_compare_two_keys( goto exit; } - assert((uint32_t)(new_key_ptr - (uchar *)new_key_data) <= new_key_size); - assert((uint32_t)(saved_key_ptr - (uchar *)saved_key_data) <= saved_key_size); - assert((uint32_t)(row_desc_ptr - (uchar *)row_desc) <= row_desc_size); - } - new_key_bytes_left = new_key_size - ((uint32_t)(new_key_ptr - (uchar *)new_key_data)); - saved_key_bytes_left = saved_key_size - ((uint32_t)(saved_key_ptr - (uchar *)saved_key_data)); + assert_always( + (uint32_t)(new_key_ptr - (uchar*)new_key_data) <= new_key_size); + assert_always( + (uint32_t)(saved_key_ptr - (uchar*)saved_key_data) <= saved_key_size); + assert_always( + (uint32_t)(row_desc_ptr - (uchar*)row_desc) <= row_desc_size); + } + new_key_bytes_left = + new_key_size - ((uint32_t)(new_key_ptr - (uchar*)new_key_data)); + saved_key_bytes_left = + saved_key_size - ((uint32_t)(saved_key_ptr - (uchar*)saved_key_data)); if (cmp_prefix) { ret_val = 0; - } - // - // in this case, read both keys to completion, now read infinity byte - // - else if (new_key_bytes_left== 0 && saved_key_bytes_left== 0) { + } else if (new_key_bytes_left== 0 && saved_key_bytes_left== 0) { + // in this case, read both keys to completion, now read infinity byte ret_val = new_key_inf_val - saved_key_inf_val; - } - // - // at this point, one SHOULD be 0 - // - else if (new_key_bytes_left == 0 && saved_key_bytes_left > 0) { + } else if (new_key_bytes_left == 0 && saved_key_bytes_left > 0) { + // at this point, one SHOULD be 0 ret_val = (new_key_inf_val == COL_POS_INF ) ? 1 : -1; - } - else if (new_key_bytes_left > 0 && saved_key_bytes_left == 0) { + } else if (new_key_bytes_left > 0 && saved_key_bytes_left == 0) { ret_val = (saved_key_inf_val == COL_POS_INF ) ? -1 : 1; - } - // - // this should never happen, perhaps we should assert(false) - // - else { - assert(false); + } else { + // this should never happen, perhaps we should assert(false) + assert_unreachable(); ret_val = new_key_bytes_left - saved_key_bytes_left; } exit: @@ -1765,9 +1753,9 @@ static int tokudb_compare_two_key_parts( goto exit; } - assert((uint32_t)(new_key_ptr - (uchar *)new_key_data) <= new_key_size); - assert((uint32_t)(saved_key_ptr - (uchar *)saved_key_data) <= saved_key_size); - assert((uint32_t)(row_desc_ptr - (uchar *)row_desc) <= row_desc_size); + assert_always((uint32_t)(new_key_ptr - (uchar *)new_key_data) <= new_key_size); + assert_always((uint32_t)(saved_key_ptr - (uchar *)saved_key_data) <= saved_key_size); + assert_always((uint32_t)(row_desc_ptr - (uchar *)row_desc) <= row_desc_size); } ret_val = 0; @@ -1776,7 +1764,7 @@ exit: } static int tokudb_cmp_dbt_key_parts(DB *file, const DBT *keya, const DBT *keyb, uint max_parts) { - assert(file->cmp_descriptor->dbt.size); + assert_always(file->cmp_descriptor->dbt.size); return tokudb_compare_two_key_parts( keya->data, keya->size, @@ -1847,7 +1835,7 @@ static uint32_t pack_desc_pk_info(uchar* buf, KEY_AND_COL_INFO* kc_info, TABLE_S case (toku_type_float): pos[0] = COL_FIX_FIELD; pos++; - assert(kc_info->field_lengths[field_index] < 256); + assert_always(kc_info->field_lengths[field_index] < 256); pos[0] = kc_info->field_lengths[field_index]; pos++; break; @@ -1856,7 +1844,7 @@ static uint32_t pack_desc_pk_info(uchar* buf, KEY_AND_COL_INFO* kc_info, TABLE_S pos++; field_length = field->pack_length(); set_if_smaller(key_part_length, field_length); - assert(key_part_length < 256); + assert_always(key_part_length < 256); pos[0] = (uchar)key_part_length; pos++; break; @@ -1871,7 +1859,7 @@ static uint32_t pack_desc_pk_info(uchar* buf, KEY_AND_COL_INFO* kc_info, TABLE_S pos++; break; default: - assert(false); + assert_unreachable(); } return pos - buf; @@ -1908,7 +1896,7 @@ static uint32_t pack_desc_pk_offset_info( } offset += pk_info[2*i + 1]; } - assert(found_col_in_pk); + assert_always(found_col_in_pk); if (is_constant_offset) { pos[0] = COL_FIX_PK_OFFSET; pos++; @@ -1966,10 +1954,10 @@ static uint32_t pack_desc_offset_info(uchar* buf, KEY_AND_COL_INFO* kc_info, uin break; } } - assert(found_index); + assert_always(found_index); break; default: - assert(false); + assert_unreachable(); } return pos - buf; @@ -2004,7 +1992,7 @@ static uint32_t pack_desc_key_length_info(uchar* buf, KEY_AND_COL_INFO* kc_info, pos += sizeof(key_part_length); break; default: - assert(false); + assert_unreachable(); } return pos - buf; @@ -2041,7 +2029,7 @@ static uint32_t pack_desc_char_info(uchar* buf, KEY_AND_COL_INFO* kc_info, TABLE pos += 4; break; default: - assert(false); + assert_unreachable(); } return pos - buf; @@ -2151,7 +2139,7 @@ static uint32_t create_toku_clustering_val_pack_descriptor ( bool col_filtered = bitmap_is_set(&kc_info->key_filters[keynr],i); bool col_filtered_in_pk = bitmap_is_set(&kc_info->key_filters[pk_index],i); if (col_filtered_in_pk) { - assert(col_filtered); + assert_always(col_filtered); } } @@ -2321,7 +2309,7 @@ static uint32_t pack_clustering_val_from_desc( memcpy(&end, desc_pos, sizeof(end)); desc_pos += sizeof(end); - assert (start <= end); + assert_always (start <= end); if (curr == CK_FIX_RANGE) { length = end - start; @@ -2367,24 +2355,21 @@ static uint32_t pack_clustering_val_from_desc( offset_diffs = (end_data_offset + end_data_size) - (uint32_t)(var_dest_data_ptr - orig_var_dest_data_ptr); for (uint32_t i = start; i <= end; i++) { if ( num_offset_bytes == 1 ) { - assert(offset_diffs < 256); + assert_always(offset_diffs < 256); var_dest_offset_ptr[0] = var_src_offset_ptr[i] - (uchar)offset_diffs; var_dest_offset_ptr++; - } - else if ( num_offset_bytes == 2 ) { + } else if ( num_offset_bytes == 2 ) { uint32_t tmp = uint2korr(var_src_offset_ptr + 2*i); uint32_t new_offset = tmp - offset_diffs; - assert(new_offset < 1<<16); + assert_always(new_offset < 1<<16); int2store(var_dest_offset_ptr,new_offset); var_dest_offset_ptr += 2; - } - else { - assert(false); + } else { + assert_unreachable(); } } - } - else { - assert(false); + } else { + assert_unreachable(); } } // @@ -2518,7 +2503,7 @@ static uint32_t create_toku_secondary_key_pack_descriptor ( // // store number of parts // - assert(get_key_parts(prim_key) < 128); + assert_always(get_key_parts(prim_key) < 128); pos[0] = 2 * get_key_parts(prim_key); pos++; // @@ -2540,7 +2525,7 @@ static uint32_t create_toku_secondary_key_pack_descriptor ( // // asserting that we moved forward as much as we think we have // - assert(tmp - pos == (2 * get_key_parts(prim_key))); + assert_always(tmp - pos == (2 * get_key_parts(prim_key))); pos = tmp; } @@ -2551,7 +2536,7 @@ static uint32_t create_toku_secondary_key_pack_descriptor ( bool is_col_in_pk = false; if (bitmap_is_set(&kc_info->key_filters[pk_index],field_index)) { - assert(!has_hpk && prim_key != NULL); + assert_always(!has_hpk && prim_key != NULL); is_col_in_pk = true; } else { @@ -2566,7 +2551,7 @@ static uint32_t create_toku_secondary_key_pack_descriptor ( // assert that columns in pk do not have a null bit // because in MySQL, pk columns cannot be null // - assert(!field->null_bit); + assert_always(!field->null_bit); } if (field->null_bit) { @@ -2668,7 +2653,7 @@ static uint32_t max_key_size_from_desc( // skip byte that states if main dictionary bool is_main_dictionary = desc_pos[0]; desc_pos++; - assert(!is_main_dictionary); + assert_always(!is_main_dictionary); // skip hpk byte desc_pos++; @@ -2731,7 +2716,7 @@ static uint32_t max_key_size_from_desc( desc_pos += sizeof(charset_num); } else { - assert(has_charset == COL_HAS_NO_CHARSET); + assert_always(has_charset == COL_HAS_NO_CHARSET); } } return max_size; @@ -2742,9 +2727,8 @@ static uint32_t pack_key_from_desc( void* row_desc, uint32_t row_desc_size, const DBT* pk_key, - const DBT* pk_val - ) -{ + const DBT* pk_val) { + MULTI_COL_PACK_INFO mcp_info; uint32_t num_null_bytes; uint32_t num_blobs; @@ -2762,7 +2746,7 @@ static uint32_t pack_key_from_desc( bool is_main_dictionary = desc_pos[0]; desc_pos++; - assert(!is_main_dictionary); + assert_always(!is_main_dictionary); // // get the constant info out of descriptor @@ -2810,7 +2794,7 @@ static uint32_t pack_key_from_desc( fixed_field_ptr = null_bytes_ptr + num_null_bytes; var_field_offset_ptr = fixed_field_ptr + mcp_info.fixed_field_size; var_field_data_ptr = var_field_offset_ptr + mcp_info.len_of_offsets; - while ( (uint32_t)(desc_pos - (uchar *)row_desc) < row_desc_size) { + while ((uint32_t)(desc_pos - (uchar*)row_desc) < row_desc_size) { uchar col_fix_val; uchar has_charset; uint32_t col_pack_val = 0; @@ -2834,8 +2818,7 @@ static uint32_t pack_key_from_desc( packed_key_pos++; desc_pos += skip_key_in_desc(desc_pos); continue; - } - else { + } else { packed_key_pos[0] = NONNULL_COL_VAL; packed_key_pos++; } @@ -2859,42 +2842,46 @@ static uint32_t pack_key_from_desc( if (has_charset == COL_HAS_CHARSET) { memcpy(&charset_num, desc_pos, sizeof(charset_num)); desc_pos += sizeof(charset_num); - } - else { - assert(has_charset == COL_HAS_NO_CHARSET); + } else { + assert_always(has_charset == COL_HAS_NO_CHARSET); } // // case where column is in pk val // - if (col_fix_val == COL_FIX_FIELD || col_fix_val == COL_VAR_FIELD || col_fix_val == COL_BLOB_FIELD) { - if (col_fix_val == COL_FIX_FIELD && has_charset == COL_HAS_NO_CHARSET) { - memcpy(packed_key_pos, &fixed_field_ptr[col_pack_val], key_length); + if (col_fix_val == COL_FIX_FIELD || + col_fix_val == COL_VAR_FIELD || + col_fix_val == COL_BLOB_FIELD) { + if (col_fix_val == COL_FIX_FIELD && + has_charset == COL_HAS_NO_CHARSET) { + memcpy( + packed_key_pos, + &fixed_field_ptr[col_pack_val], + key_length); packed_key_pos += key_length; - } - else if (col_fix_val == COL_VAR_FIELD && has_charset == COL_HAS_NO_CHARSET) { + } else if (col_fix_val == COL_VAR_FIELD && + has_charset == COL_HAS_NO_CHARSET) { uint32_t data_start_offset = 0; uint32_t data_size = 0; get_var_field_info( - &data_size, - &data_start_offset, - col_pack_val, - var_field_offset_ptr, - num_offset_bytes - ); + &data_size, + &data_start_offset, + col_pack_val, + var_field_offset_ptr, + num_offset_bytes); // // length of this field in this row is data_size // data is located beginning at var_field_data_ptr + data_start_offset // packed_key_pos = pack_toku_varbinary_from_desc( - packed_key_pos, - var_field_data_ptr + data_start_offset, - key_length, //number of bytes to use to encode the length in to_tokudb - data_size //length of field - ); - } - else { + packed_key_pos, + var_field_data_ptr + data_start_offset, + //number of bytes to use to encode the length in to_tokudb + key_length, + //length of field + data_size); + } else { const uchar* data_start = NULL; uint32_t data_start_offset = 0; uint32_t data_size = 0; @@ -2903,76 +2890,59 @@ static uint32_t pack_key_from_desc( data_start_offset = col_pack_val; data_size = key_length; data_start = fixed_field_ptr + data_start_offset; - } - else if (col_fix_val == COL_VAR_FIELD){ + } else if (col_fix_val == COL_VAR_FIELD){ get_var_field_info( - &data_size, - &data_start_offset, - col_pack_val, - var_field_offset_ptr, - num_offset_bytes - ); + &data_size, + &data_start_offset, + col_pack_val, + var_field_offset_ptr, + num_offset_bytes); data_start = var_field_data_ptr + data_start_offset; - } - else if (col_fix_val == COL_BLOB_FIELD) { + } else if (col_fix_val == COL_BLOB_FIELD) { uint32_t blob_index = col_pack_val; uint32_t blob_offset; const uchar* blob_ptr = NULL; uint32_t field_len; uint32_t field_len_bytes = blob_lengths[blob_index]; get_blob_field_info( - &blob_offset, + &blob_offset, mcp_info.len_of_offsets, - var_field_data_ptr, - num_offset_bytes - ); + var_field_data_ptr, + num_offset_bytes); blob_ptr = var_field_data_ptr + blob_offset; - assert(num_blobs > 0); - // - // skip over other blobs to get to the one we want to make a key out of - // + assert_always(num_blobs > 0); + + // skip over other blobs to get to the one we want to + // make a key out of for (uint32_t i = 0; i < blob_index; i++) { blob_ptr = unpack_toku_field_blob( NULL, blob_ptr, blob_lengths[i], - true - ); + true); } - // - // at this point, blob_ptr is pointing to the blob we want to make a key from - // + // at this point, blob_ptr is pointing to the blob we + // want to make a key from field_len = get_blob_field_len(blob_ptr, field_len_bytes); - // // now we set the variables to make the key - // data_start = blob_ptr + field_len_bytes; data_size = field_len; - - - } - else { - assert(false); + } else { + assert_unreachable(); } - packed_key_pos = pack_toku_varstring_from_desc( - packed_key_pos, + packed_key_pos = pack_toku_varstring_from_desc(packed_key_pos, data_start, key_length, data_size, - charset_num - ); + charset_num); } - } - // - // case where column is in pk key - // - else { + } else { + // case where column is in pk key if (col_fix_val == COL_FIX_PK_OFFSET) { memcpy(packed_key_pos, &pk_data_ptr[col_pack_val], key_length); packed_key_pos += key_length; - } - else if (col_fix_val == COL_VAR_PK_OFFSET) { + } else if (col_fix_val == COL_VAR_PK_OFFSET) { uchar* tmp_pk_data_ptr = pk_data_ptr; uint32_t index_in_pk = col_pack_val; // @@ -2981,25 +2951,21 @@ static uint32_t pack_key_from_desc( for (uint32_t i = 0; i < index_in_pk; i++) { if (pk_info[2*i] == COL_FIX_FIELD) { tmp_pk_data_ptr += pk_info[2*i + 1]; - } - else if (pk_info[2*i] == COL_VAR_FIELD) { + } else if (pk_info[2*i] == COL_VAR_FIELD) { uint32_t len_bytes = pk_info[2*i + 1]; uint32_t len; if (len_bytes == 1) { len = tmp_pk_data_ptr[0]; tmp_pk_data_ptr++; - } - else if (len_bytes == 2) { + } else if (len_bytes == 2) { len = uint2korr(tmp_pk_data_ptr); tmp_pk_data_ptr += 2; - } - else { - assert(false); + } else { + assert_unreachable(); } tmp_pk_data_ptr += len; - } - else { - assert(false); + } else { + assert_unreachable(); } } // @@ -3009,21 +2975,18 @@ static uint32_t pack_key_from_desc( if (is_fix_field == COL_FIX_FIELD) { memcpy(packed_key_pos, tmp_pk_data_ptr, key_length); packed_key_pos += key_length; - } - else if (is_fix_field == COL_VAR_FIELD) { + } else if (is_fix_field == COL_VAR_FIELD) { const uchar* data_start = NULL; uint32_t data_size = 0; uint32_t len_bytes = pk_info[2*index_in_pk + 1]; if (len_bytes == 1) { data_size = tmp_pk_data_ptr[0]; tmp_pk_data_ptr++; - } - else if (len_bytes == 2) { + } else if (len_bytes == 2) { data_size = uint2korr(tmp_pk_data_ptr); tmp_pk_data_ptr += 2; - } - else { - assert(false); + } else { + assert_unreachable(); } data_start = tmp_pk_data_ptr; @@ -3033,32 +2996,26 @@ static uint32_t pack_key_from_desc( data_start, key_length, data_size, - charset_num - ); - } - else if (has_charset == COL_HAS_NO_CHARSET) { + charset_num); + } else if (has_charset == COL_HAS_NO_CHARSET) { packed_key_pos = pack_toku_varbinary_from_desc( - packed_key_pos, - data_start, + packed_key_pos, + data_start, key_length, - data_size //length of field - ); - } - else { - assert(false); + data_size); + } else { + assert_unreachable(); } + } else { + assert_unreachable(); } - else { - assert(false); - } - } - else { - assert(false); + } else { + assert_unreachable(); } } } - assert( (uint32_t)(desc_pos - (uchar *)row_desc) == row_desc_size); + assert_always( (uint32_t)(desc_pos - (uchar *)row_desc) == row_desc_size); // // now append the primary key to the end of the key @@ -3066,13 +3023,12 @@ static uint32_t pack_key_from_desc( if (hpk) { memcpy(packed_key_pos, pk_key->data, pk_key->size); packed_key_pos += pk_key->size; - } - else { + } else { memcpy(packed_key_pos, (uchar *)pk_key->data + 1, pk_key->size - 1); packed_key_pos += (pk_key->size - 1); } - return (uint32_t)(packed_key_pos - buf); // + return (uint32_t)(packed_key_pos - buf); } static bool fields_have_same_name(Field* a, Field* b) { @@ -3249,7 +3205,7 @@ static bool fields_are_same_type(Field* a, Field* b) { case MYSQL_TYPE_DECIMAL: case MYSQL_TYPE_VAR_STRING: case MYSQL_TYPE_NULL: - assert(false); + assert_unreachable(); } cleanup: diff --git a/storage/tokudb/hatoku_cmp.h b/storage/tokudb/hatoku_cmp.h index 9a5358fc9af..34b3cfbe1f8 100644 --- a/storage/tokudb/hatoku_cmp.h +++ b/storage/tokudb/hatoku_cmp.h @@ -26,9 +26,9 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #ifndef _HATOKU_CMP #define _HATOKU_CMP -#include "stdint.h" +#include "hatoku_defines.h" +#include "tokudb_debug.h" -#include <db.h> // // A MySQL row is encoded in TokuDB, as follows: @@ -180,7 +180,7 @@ static inline uint32_t get_blob_field_len(const uchar* from_tokudb, uint32_t len length = uint4korr(from_tokudb); break; default: - assert(false); + assert_unreachable(); } return length; } diff --git a/storage/tokudb/hatoku_defines.h b/storage/tokudb/hatoku_defines.h index 225d11068ec..27b4493c804 100644 --- a/storage/tokudb/hatoku_defines.h +++ b/storage/tokudb/hatoku_defines.h @@ -23,8 +23,49 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." -#ifndef _TOKUDB_CONFIG_H -#define _TOKUDB_CONFIG_H +#ifndef _HATOKU_DEFINES_H +#define _HATOKU_DEFINES_H + +#include <my_config.h> +#define MYSQL_SERVER 1 +#include "mysql_version.h" +#include "sql_table.h" +#include "handler.h" +#include "table.h" +#include "log.h" +#include "sql_class.h" +#include "sql_show.h" +#include "discover.h" + +#if (50600 <= MYSQL_VERSION_ID && MYSQL_VERSION_ID <= 50699) || (50700 <= MYSQL_VERSION_ID && MYSQL_VERSION_ID <= 50799) +#include <binlog.h> +#endif + +#undef PACKAGE +#undef VERSION +#undef HAVE_DTRACE +#undef _DTRACE_VERSION + +/* We define DTRACE after mysql_priv.h in case it disabled dtrace in the main server */ +#ifdef HAVE_DTRACE +#define _DTRACE_VERSION 1 +#else +#endif + +#include <mysql/plugin.h> + +#include <ctype.h> +#include <stdint.h> +#define __STDC_FORMAT_MACROS +#include <inttypes.h> +#if defined(_WIN32) +#include "misc.h" +#endif + +#include "db.h" +#include "toku_os.h" +#include "toku_time.h" +#include "partitioned_counter.h" #ifdef USE_PRAGMA_INTERFACE #pragma interface /* gcc class implementation */ @@ -161,326 +202,51 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. /* Bits for share->status */ #define STATUS_PRIMARY_KEY_INIT 0x1 -#endif // _TOKUDB_CONFIG_H - -#ifndef _TOKUDB_DEBUG_H -#define _TOKUDB_DEBUG_H - -#define TOKU_INCLUDE_BACKTRACE 0 -#if TOKU_INCLUDE_BACKTRACE -static void tokudb_backtrace(void); -#endif - -extern ulong tokudb_debug; - -// tokudb debug tracing -#define TOKUDB_DEBUG_INIT 1 -#define TOKUDB_DEBUG_OPEN 2 -#define TOKUDB_DEBUG_ENTER 4 -#define TOKUDB_DEBUG_RETURN 8 -#define TOKUDB_DEBUG_ERROR 16 -#define TOKUDB_DEBUG_TXN 32 -#define TOKUDB_DEBUG_AUTO_INCREMENT 64 -#define TOKUDB_DEBUG_INDEX_KEY 128 -#define TOKUDB_DEBUG_LOCK 256 -#define TOKUDB_DEBUG_CHECK_KEY 1024 -#define TOKUDB_DEBUG_HIDE_DDL_LOCK_ERRORS 2048 -#define TOKUDB_DEBUG_ALTER_TABLE 4096 -#define TOKUDB_DEBUG_UPSERT 8192 -#define TOKUDB_DEBUG_CHECK (1<<14) -#define TOKUDB_DEBUG_ANALYZE (1<<15) - -#define TOKUDB_TRACE(f, ...) { \ - fprintf(stderr, "%u %s:%u %s " f "\n", my_tid(), __FILE__, __LINE__, __FUNCTION__, ##__VA_ARGS__); \ -} - -static inline unsigned int my_tid() { - return (unsigned int)toku_os_gettid(); -} - -#define TOKUDB_DBUG_ENTER(f, ...) { \ - if (tokudb_debug & TOKUDB_DEBUG_ENTER) { \ - TOKUDB_TRACE(f, ##__VA_ARGS__); \ - } \ -} \ - DBUG_ENTER(__FUNCTION__); - -#define TOKUDB_DBUG_RETURN(r) { \ - int rr = (r); \ - if ((tokudb_debug & TOKUDB_DEBUG_RETURN) || (rr != 0 && (tokudb_debug & TOKUDB_DEBUG_ERROR))) { \ - TOKUDB_TRACE("return %d", rr); \ - } \ - DBUG_RETURN(rr); \ -} - -#define TOKUDB_HANDLER_TRACE(f, ...) \ - fprintf(stderr, "%u %p %s:%u ha_tokudb::%s " f "\n", my_tid(), this, __FILE__, __LINE__, __FUNCTION__, ##__VA_ARGS__); - -#define TOKUDB_HANDLER_DBUG_ENTER(f, ...) { \ - if (tokudb_debug & TOKUDB_DEBUG_ENTER) { \ - TOKUDB_HANDLER_TRACE(f, ##__VA_ARGS__); \ - } \ -} \ - DBUG_ENTER(__FUNCTION__); - -#define TOKUDB_HANDLER_DBUG_RETURN(r) { \ - int rr = (r); \ - if ((tokudb_debug & TOKUDB_DEBUG_RETURN) || (rr != 0 && (tokudb_debug & TOKUDB_DEBUG_ERROR))) { \ - TOKUDB_HANDLER_TRACE("return %d", rr); \ - } \ - DBUG_RETURN(rr); \ -} - -#define TOKUDB_HANDLER_DBUG_VOID_RETURN { \ - if (tokudb_debug & TOKUDB_DEBUG_RETURN) { \ - TOKUDB_HANDLER_TRACE("return"); \ - } \ - DBUG_VOID_RETURN; \ -} - -#define TOKUDB_DBUG_DUMP(s, p, len) \ -{ \ - TOKUDB_TRACE("%s", s); \ - uint i; \ - for (i=0; i<len; i++) { \ - fprintf(stderr, "%2.2x", ((uchar*)p)[i]); \ - } \ - fprintf(stderr, "\n"); \ -} - -/* The purpose of this file is to define assert() for use by the handlerton. - * The intention is for a failed handlerton assert to invoke a failed assert - * in the fractal tree layer, which dumps engine status to the error log. - */ - -void toku_hton_assert_fail(const char*/*expr_as_string*/,const char */*fun*/,const char*/*file*/,int/*line*/, int/*errno*/) __attribute__((__visibility__("default"))) __attribute__((__noreturn__)); - -#undef assert -#define assert(expr) ((expr) ? (void)0 : toku_hton_assert_fail(#expr, __FUNCTION__, __FILE__, __LINE__, errno)) - -#endif // _TOKUDB_DEBUG_H - -#ifndef _TOKUDB_TXN_H -#define _TOKUDB_TXN_H - -typedef enum { - hatoku_iso_not_set = 0, - hatoku_iso_read_uncommitted, - hatoku_iso_read_committed, - hatoku_iso_repeatable_read, - hatoku_iso_serializable -} HA_TOKU_ISO_LEVEL; - - - -typedef struct st_tokudb_stmt_progress { - ulonglong inserted; - ulonglong updated; - ulonglong deleted; - ulonglong queried; - bool using_loader; -} tokudb_stmt_progress; - - -typedef struct st_tokudb_trx_data { - DB_TXN *all; - DB_TXN *stmt; - DB_TXN *sp_level; - DB_TXN *sub_sp_level; - uint tokudb_lock_count; - uint create_lock_count; - tokudb_stmt_progress stmt_progress; - bool checkpoint_lock_taken; - LIST *handlers; -} tokudb_trx_data; - -extern char *tokudb_data_dir; -extern const char *ha_tokudb_ext; - -static inline void reset_stmt_progress (tokudb_stmt_progress* val) { - val->deleted = 0; - val->inserted = 0; - val->updated = 0; - val->queried = 0; -} - -static inline int get_name_length(const char *name) { - int n = 0; - const char *newname = name; - n += strlen(newname); - n += strlen(ha_tokudb_ext); - return n; -} - -// -// returns maximum length of path to a dictionary -// -static inline int get_max_dict_name_path_length(const char *tablename) { - int n = 0; - n += get_name_length(tablename); - n += 1; //for the '-' - n += MAX_DICT_NAME_LEN; - return n; -} - -static inline void make_name(char *newname, const char *tablename, const char *dictname) { - const char *newtablename = tablename; - char *nn = newname; - assert(tablename); - assert(dictname); - nn += sprintf(nn, "%s", newtablename); - nn += sprintf(nn, "-%s", dictname); -} - -static inline int txn_begin(DB_ENV *env, DB_TXN *parent, DB_TXN **txn, uint32_t flags, THD *thd) { - *txn = NULL; - int r = env->txn_begin(env, parent, txn, flags); - if (r == 0 && thd) { - DB_TXN *this_txn = *txn; - this_txn->set_client_id(this_txn, thd_get_thread_id(thd)); - } - if ((tokudb_debug & TOKUDB_DEBUG_TXN)) { - TOKUDB_TRACE("begin txn %p %p %u r=%d", parent, *txn, flags, r); - } - return r; -} - -static inline void commit_txn(DB_TXN* txn, uint32_t flags) { - if (tokudb_debug & TOKUDB_DEBUG_TXN) - TOKUDB_TRACE("commit txn %p", txn); - int r = txn->commit(txn, flags); - if (r != 0) { - sql_print_error("tried committing transaction %p and got error code %d", txn, r); - } - assert(r == 0); -} - -static inline void abort_txn(DB_TXN* txn) { - if (tokudb_debug & TOKUDB_DEBUG_TXN) - TOKUDB_TRACE("abort txn %p", txn); - int r = txn->abort(txn); - if (r != 0) { - sql_print_error("tried aborting transaction %p and got error code %d", txn, r); - } - assert(r == 0); -} - -#endif // _TOKUDB_TXN_H - -#ifndef _TOKUDB_PORTABILITY_H -#define _TOKUDB_PORTABILITY_H - -static inline void *tokudb_my_malloc(size_t s, myf flags) { -#if 50700 <= MYSQL_VERSION_ID && MYSQL_VERSION_ID <= 50799 - return my_malloc(0, s, flags); +#if defined(TOKUDB_VERSION_MAJOR) && defined(TOKUDB_VERSION_MINOR) +#define TOKUDB_PLUGIN_VERSION ((TOKUDB_VERSION_MAJOR << 8) + TOKUDB_VERSION_MINOR) #else - return my_malloc(s, flags); +#define TOKUDB_PLUGIN_VERSION 0 #endif -} -static inline void *tokudb_my_realloc(void *p, size_t s, myf flags) { - if (s == 0) - return p; -#if 50700 <= MYSQL_VERSION_ID && MYSQL_VERSION_ID <= 50799 - return my_realloc(0, p, s, flags); -#else - return my_realloc(p, s, flags); -#endif -} +// Branch prediction macros. +// If supported by the compiler, will hint in instruction caching for likely +// branching. Should only be used where there is a very good idea of the correct +// branch heuristics as determined by profiling. Mostly copied from InnoDB. +// Use: +// "if (TOKUDB_LIKELY(x))" where the chances of "x" evaluating true are higher +// "if (TOKUDB_UNLIKELY(x))" where the chances of "x" evaluating false are higher +#if defined(__GNUC__) && (__GNUC__ > 2) && ! defined(__INTEL_COMPILER) -static inline void tokudb_my_free(void *ptr) { - if (ptr) - my_free(ptr); -} +// Tell the compiler that 'expr' probably evaluates to 'constant'. +#define TOKUDB_EXPECT(expr,constant) __builtin_expect(expr, constant) -static inline char *tokudb_my_strdup(const char *p, myf flags) { -#if 50700 <= MYSQL_VERSION_ID && MYSQL_VERSION_ID <= 50799 - return my_strdup(0, p, flags); #else - return my_strdup(p, flags); -#endif -} - -static inline void* tokudb_my_multi_malloc(myf myFlags, ...) { - va_list args; - char **ptr,*start,*res; - size_t tot_length,length; - - va_start(args,myFlags); - tot_length=0; - while ((ptr=va_arg(args, char **))) { - length=va_arg(args,uint); - tot_length+=ALIGN_SIZE(length); - } - va_end(args); - - if (!(start=(char *) tokudb_my_malloc(tot_length,myFlags))) { - return 0; - } - - va_start(args,myFlags); - res=start; - while ((ptr=va_arg(args, char **))) { - *ptr=res; - length=va_arg(args,uint); - res+=ALIGN_SIZE(length); - } - va_end(args); - return start; -} - -static inline void tokudb_pthread_mutex_init(pthread_mutex_t *mutex, const pthread_mutexattr_t *attr) { - int r = pthread_mutex_init(mutex, attr); - assert(r == 0); -} - -static inline void tokudb_pthread_mutex_destroy(pthread_mutex_t *mutex) { - int r = pthread_mutex_destroy(mutex); - assert(r == 0); -} -static inline void tokudb_pthread_mutex_lock(pthread_mutex_t *mutex) { - int r = pthread_mutex_lock(mutex); - assert(r == 0); -} +#error "No TokuDB branch prediction operations in use!" +#define TOKUDB_EXPECT(expr,constant) (expr) -static inline void tokudb_pthread_mutex_unlock(pthread_mutex_t *mutex) { - int r = pthread_mutex_unlock(mutex); - assert(r == 0); -} +#endif // defined(__GNUC__) && (__GNUC__ > 2) && ! defined(__INTEL_COMPILER) -static inline void tokudb_pthread_cond_init(pthread_cond_t *cond, const pthread_condattr_t *attr) { - int r = pthread_cond_init(cond, attr); - assert(r == 0); -} +// Tell the compiler that cond is likely to hold +#define TOKUDB_LIKELY(cond) TOKUDB_EXPECT(cond, 1) -static inline void tokudb_pthread_cond_destroy(pthread_cond_t *cond) { - int r = pthread_cond_destroy(cond); - assert(r == 0); -} - -static inline void tokudb_pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex) { - int r = pthread_cond_wait(cond, mutex); - assert(r == 0); -} - -static inline void tokudb_pthread_cond_broadcast(pthread_cond_t *cond) { - int r = pthread_cond_broadcast(cond); - assert(r == 0); -} +// Tell the compiler that cond is unlikely to hold +#define TOKUDB_UNLIKELY(cond) TOKUDB_EXPECT(cond, 0) +// Tell the compiler that the function/argument is unused +#define TOKUDB_UNUSED(_uu) _uu __attribute__((unused)) // mysql 5.6.15 removed the test macro, so we define our own #define tokudb_test(e) ((e) ? 1 : 0) -static const char *tokudb_thd_get_proc_info(THD *thd) { +inline const char* tokudb_thd_get_proc_info(const THD *thd) { return thd->proc_info; } // uint3korr reads 4 bytes and valgrind reports an error, so we use this function instead -static uint tokudb_uint3korr(const uchar *a) { +inline uint tokudb_uint3korr(const uchar *a) { uchar b[4] = {}; memcpy(b, a, 3); return uint3korr(b); } -#endif // _TOKUDB_PORTABILITY_H +#endif // _HATOKU_DEFINES_H diff --git a/storage/tokudb/hatoku_hton.cc b/storage/tokudb/hatoku_hton.cc index 959490b3160..086280a76df 100644 --- a/storage/tokudb/hatoku_hton.cc +++ b/storage/tokudb/hatoku_hton.cc @@ -24,35 +24,7 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." -#define MYSQL_SERVER 1 -#include "hatoku_defines.h" -#include <db.h> -#include <ctype.h> - -#include "stdint.h" -#if defined(_WIN32) -#include "misc.h" -#endif -#define __STDC_FORMAT_MACROS -#include <inttypes.h> -#include "toku_os.h" -#include "toku_time.h" -#include "partitioned_counter.h" - -/* We define DTRACE after mysql_priv.h in case it disabled dtrace in the main server */ -#ifdef HAVE_DTRACE -#define _DTRACE_VERSION 1 -#else -#endif - -#include <mysql/plugin.h> #include "hatoku_hton.h" -#include "ha_tokudb.h" - -#undef PACKAGE -#undef VERSION -#undef HAVE_DTRACE -#undef _DTRACE_VERSION #define TOKU_METADB_NAME "tokudb_meta" @@ -74,63 +46,96 @@ ha_create_table_option tokudb_index_options[] = { }; #endif -static uchar *tokudb_get_key(TOKUDB_SHARE * share, size_t * length, my_bool not_used __attribute__ ((unused))) { - *length = share->table_name_length; - return (uchar *) share->table_name; -} - -static handler *tokudb_create_handler(handlerton * hton, TABLE_SHARE * table, MEM_ROOT * mem_root); +static handler* tokudb_create_handler( + handlerton* hton, + TABLE_SHARE* table, + MEM_ROOT* mem_root); - -static void tokudb_print_error(const DB_ENV * db_env, const char *db_errpfx, const char *buffer); +static void tokudb_print_error( + const DB_ENV* db_env, + const char* db_errpfx, + const char* buffer); static void tokudb_cleanup_log_files(void); -static int tokudb_end(handlerton * hton, ha_panic_function type); -static bool tokudb_flush_logs(handlerton * hton); -static bool tokudb_show_status(handlerton * hton, THD * thd, stat_print_fn * print, enum ha_stat_type); +static int tokudb_end(handlerton* hton, ha_panic_function type); +static bool tokudb_flush_logs(handlerton* hton); +static bool tokudb_show_status( + handlerton* hton, + THD* thd, + stat_print_fn* print, + enum ha_stat_type); #if TOKU_INCLUDE_HANDLERTON_HANDLE_FATAL_SIGNAL -static void tokudb_handle_fatal_signal(handlerton *hton, THD *thd, int sig); +static void tokudb_handle_fatal_signal(handlerton* hton, THD* thd, int sig); #endif -static int tokudb_close_connection(handlerton * hton, THD * thd); -static int tokudb_commit(handlerton * hton, THD * thd, bool all); -static int tokudb_rollback(handlerton * hton, THD * thd, bool all); +static int tokudb_close_connection(handlerton* hton, THD* thd); +static int tokudb_commit(handlerton* hton, THD* thd, bool all); +static int tokudb_rollback(handlerton* hton, THD* thd, bool all); #if TOKU_INCLUDE_XA static int tokudb_xa_prepare(handlerton* hton, THD* thd, bool all); -static int tokudb_xa_recover(handlerton* hton, XID* xid_list, uint len); +static int tokudb_xa_recover(handlerton* hton, XID* xid_list, uint len); static int tokudb_commit_by_xid(handlerton* hton, XID* xid); -static int tokudb_rollback_by_xid(handlerton* hton, XID* xid); +static int tokudb_rollback_by_xid(handlerton* hton, XID* xid); #endif -static int tokudb_rollback_to_savepoint(handlerton * hton, THD * thd, void *savepoint); -static int tokudb_savepoint(handlerton * hton, THD * thd, void *savepoint); -static int tokudb_release_savepoint(handlerton * hton, THD * thd, void *savepoint); +static int tokudb_rollback_to_savepoint( + handlerton* hton, + THD* thd, + void* savepoint); +static int tokudb_savepoint(handlerton* hton, THD* thd, void* savepoint); +static int tokudb_release_savepoint( + handlerton* hton, + THD* thd, + void* savepoint); #if 100000 <= MYSQL_VERSION_ID && MYSQL_VERSION_ID <= 100099 static int tokudb_discover_table(handlerton *hton, THD* thd, TABLE_SHARE *ts); -static int tokudb_discover_table_existence(handlerton *hton, const char *db, const char *name); +static int tokudb_discover_table_existence( + handlerton* hton, + const char* db, + const char* name); #endif -static int tokudb_discover(handlerton *hton, THD* thd, const char *db, const char *name, uchar **frmblob, size_t *frmlen); -static int tokudb_discover2(handlerton *hton, THD* thd, const char *db, const char *name, bool translate_name, uchar **frmblob, size_t *frmlen); -static int tokudb_discover3(handlerton *hton, THD* thd, const char *db, const char *name, char *path, uchar **frmblob, size_t *frmlen); -handlerton *tokudb_hton; - -const char *ha_tokudb_ext = ".tokudb"; -char *tokudb_data_dir; -ulong tokudb_debug; -DB_ENV *db_env; -HASH tokudb_open_tables; -pthread_mutex_t tokudb_mutex; +static int tokudb_discover( + handlerton* hton, + THD* thd, + const char* db, + const char* name, + uchar** frmblob, + size_t* frmlen); +static int tokudb_discover2( + handlerton* hton, + THD* thd, + const char* db, + const char* name, + bool translate_name, + uchar** frmblob, + size_t* frmlen); +static int tokudb_discover3( + handlerton* hton, + THD* thd, + const char* db, + const char* name, + char* path, + uchar** frmblob, + size_t* frmlen); +handlerton* tokudb_hton; + +const char* ha_tokudb_ext = ".tokudb"; +DB_ENV* db_env; #if TOKU_THDVAR_MEMALLOC_BUG -static pthread_mutex_t tokudb_map_mutex; +static tokudb::thread::mutex_t tokudb_map_mutex; static TREE tokudb_map; struct tokudb_map_pair { - THD *thd; + THD* thd; char *last_lock_timeout; }; #if 50500 <= MYSQL_VERSION_ID && MYSQL_VERSION_ID <= 50599 static int tokudb_map_pair_cmp(void *custom_arg, const void *a, const void *b) { #else -static int tokudb_map_pair_cmp(const void *custom_arg, const void *a, const void *b) { +static int tokudb_map_pair_cmp( + const void* custom_arg, + const void* a, + const void* b) { #endif + const struct tokudb_map_pair *a_key = (const struct tokudb_map_pair *) a; const struct tokudb_map_pair *b_key = (const struct tokudb_map_pair *) b; if (a_key->thd < b_key->thd) @@ -142,30 +147,41 @@ static int tokudb_map_pair_cmp(const void *custom_arg, const void *a, const void }; #endif -#if TOKU_INCLUDE_HANDLERTON_HANDLE_FATAL_SIGNAL -static my_bool tokudb_gdb_on_fatal; -static char *tokudb_gdb_path; -#endif - static PARTITIONED_COUNTER tokudb_primary_key_bytes_inserted; void toku_hton_update_primary_key_bytes_inserted(uint64_t row_size) { increment_partitioned_counter(tokudb_primary_key_bytes_inserted, row_size); } -static void tokudb_lock_timeout_callback(DB *db, uint64_t requesting_txnid, const DBT *left_key, const DBT *right_key, uint64_t blocking_txnid); -static ulong tokudb_cleaner_period; -static ulong tokudb_cleaner_iterations; +static void tokudb_lock_timeout_callback( + DB* db, + uint64_t requesting_txnid, + const DBT* left_key, + const DBT* right_key, + uint64_t blocking_txnid); #define ASSERT_MSGLEN 1024 -void toku_hton_assert_fail(const char* expr_as_string, const char * fun, const char * file, int line, int caller_errno) { +void toku_hton_assert_fail( + const char* expr_as_string, + const char* fun, + const char* file, + int line, + int caller_errno) { + char msg[ASSERT_MSGLEN]; if (db_env) { snprintf(msg, ASSERT_MSGLEN, "Handlerton: %s ", expr_as_string); db_env->crash(db_env, msg, fun, file, line,caller_errno); - } - else { - snprintf(msg, ASSERT_MSGLEN, "Handlerton assertion failed, no env, %s, %d, %s, %s (errno=%d)\n", file, line, fun, expr_as_string, caller_errno); + } else { + snprintf( + msg, + ASSERT_MSGLEN, + "Handlerton assertion failed, no env, %s, %d, %s, %s (errno=%d)\n", + file, + line, + fun, + expr_as_string, + caller_errno); perror(msg); fflush(stderr); } @@ -184,36 +200,11 @@ static uint32_t tokudb_env_flags = 0; // static uint32_t tokudb_lock_type = DB_LOCK_DEFAULT; // static ulong tokudb_log_buffer_size = 0; // static ulong tokudb_log_file_size = 0; -static my_bool tokudb_directio = FALSE; -static my_bool tokudb_compress_buffers_before_eviction = TRUE; -static my_bool tokudb_checkpoint_on_flush_logs = FALSE; -static ulonglong tokudb_cache_size = 0; -static uint32_t tokudb_client_pool_threads = 0; -static uint32_t tokudb_cachetable_pool_threads = 0; -static uint32_t tokudb_checkpoint_pool_threads = 0; -static ulonglong tokudb_max_lock_memory = 0; -static my_bool tokudb_enable_partial_eviction = TRUE; -static char *tokudb_home; -static char *tokudb_tmp_dir; -static char *tokudb_log_dir; +static char* tokudb_home; // static long tokudb_lock_scan_time = 0; // static ulong tokudb_region_size = 0; // static ulong tokudb_cache_parts = 1; -const char *tokudb_hton_name = "TokuDB"; -static uint32_t tokudb_checkpointing_period; -static uint32_t tokudb_fsync_log_period; -uint32_t tokudb_write_status_frequency; -uint32_t tokudb_read_status_frequency; - -#ifdef TOKUDB_VERSION -#define tokudb_stringify_2(x) #x -#define tokudb_stringify(x) tokudb_stringify_2(x) -#define TOKUDB_VERSION_STR tokudb_stringify(TOKUDB_VERSION) -#else -#define TOKUDB_VERSION_STR NULL -#endif -char *tokudb_version = (char *) TOKUDB_VERSION_STR; -static int tokudb_fs_reserve_percent; // file system reserve as a percentage of total disk space +const char* tokudb_hton_name = "TokuDB"; #if defined(_WIN32) extern "C" { @@ -226,18 +217,8 @@ extern "C" { // Since we don't have static initializers for the opaque rwlock type, // use constructor and destructor functions to create and destroy // the lock before and after main(), respectively. -static int tokudb_hton_initialized; -static rw_lock_t tokudb_hton_initialized_lock; - -static void create_tokudb_hton_intialized_lock(void) __attribute__((constructor)); -static void create_tokudb_hton_intialized_lock(void) { - my_rwlock_init(&tokudb_hton_initialized_lock, 0); -} - -static void destroy_tokudb_hton_initialized_lock(void) __attribute__((destructor)); -static void destroy_tokudb_hton_initialized_lock(void) { - rwlock_destroy(&tokudb_hton_initialized_lock); -} +int tokudb_hton_initialized; +tokudb::thread::rwlock_t tokudb_hton_initialized_lock; static SHOW_VAR *toku_global_status_variables = NULL; static uint64_t toku_global_status_max_rows; @@ -266,7 +247,10 @@ static void handle_ydb_error(int error) { sql_print_error("************************************************************"); break; case TOKUDB_UPGRADE_FAILURE: - sql_print_error("%s upgrade failed. A clean shutdown of the previous version is required.", tokudb_hton_name); + sql_print_error( + "%s upgrade failed. A clean shutdown of the previous version is " + "required.", + tokudb_hton_name); break; default: sql_print_error("%s unknown error %d", tokudb_hton_name, error); @@ -289,27 +273,51 @@ static int tokudb_init_func(void *p) { int r; // 3938: lock the handlerton's initialized status flag for writing - r = rw_wrlock(&tokudb_hton_initialized_lock); - assert(r == 0); + tokudb_hton_initialized_lock.lock_write(); db_env = NULL; tokudb_hton = (handlerton *) p; #if TOKUDB_CHECK_JEMALLOC - if (tokudb_check_jemalloc && dlsym(RTLD_DEFAULT, "mallctl") == NULL) { - sql_print_error("%s is not initialized because jemalloc is not loaded", tokudb_hton_name); - goto error; + if (tokudb::sysvars::check_jemalloc) { + typedef int (*mallctl_type)( + const char*, + void*, + size_t*, + void*, + size_t); + mallctl_type mallctl_func; + mallctl_func= (mallctl_type)dlsym(RTLD_DEFAULT, "mallctl"); + if (!mallctl_func) { + sql_print_error( + "%s is not initialized because jemalloc is not loaded", + tokudb_hton_name); + goto error; + } + char *ver; + size_t len = sizeof(ver); + mallctl_func("version", &ver, &len, NULL, 0); + /* jemalloc 2.2.5 crashes mysql-test */ + if (strcmp(ver, "2.3.") < 0) { + sql_print_error( + "%s is not initialized because jemalloc is older than 2.3.0", + tokudb_hton_name); + goto error; + } } #endif r = tokudb_set_product_name(); if (r) { - sql_print_error("%s can not set product name error %d", tokudb_hton_name, r); + sql_print_error( + "%s can not set product name error %d", + tokudb_hton_name, + r); goto error; } - tokudb_pthread_mutex_init(&tokudb_mutex, MY_MUTEX_INIT_FAST); - (void) my_hash_init(&tokudb_open_tables, table_alias_charset, 32, 0, 0, (my_hash_get_key) tokudb_get_key, 0, 0); + TOKUDB_SHARE::static_init(); + tokudb::background::initialize(); tokudb_hton->state = SHOW_OPTION_YES; // tokudb_hton->flags= HTON_CAN_RECREATE; // QQQ this came from skeleton @@ -386,8 +394,11 @@ static int tokudb_init_func(void *p) { DBUG_PRINT("info", ("tokudb_env_flags: 0x%x\n", tokudb_env_flags)); r = db_env->set_flags(db_env, tokudb_env_flags, 1); if (r) { // QQQ - if (tokudb_debug & TOKUDB_DEBUG_INIT) - TOKUDB_TRACE("WARNING: flags=%x r=%d", tokudb_env_flags, r); + TOKUDB_TRACE_FOR_FLAGS( + TOKUDB_DEBUG_INIT, + "WARNING: flags=%x r=%d", + tokudb_env_flags, + r); // goto error; } @@ -405,8 +416,8 @@ static int tokudb_init_func(void *p) { } { - char *tmp_dir = tokudb_tmp_dir; - char *data_dir = tokudb_data_dir; + char* tmp_dir = tokudb::sysvars::tmp_dir; + char* data_dir = tokudb::sysvars::data_dir; if (data_dir == 0) { data_dir = mysql_data_home; } @@ -419,36 +430,48 @@ static int tokudb_init_func(void *p) { db_env->set_tmp_dir(db_env, tmp_dir); } - if (tokudb_log_dir) { - DBUG_PRINT("info", ("tokudb_log_dir: %s\n", tokudb_log_dir)); - db_env->set_lg_dir(db_env, tokudb_log_dir); + if (tokudb::sysvars::log_dir) { + DBUG_PRINT("info", ("tokudb_log_dir: %s\n", tokudb::sysvars::log_dir)); + db_env->set_lg_dir(db_env, tokudb::sysvars::log_dir); } - // config the cache table size to min(1/2 of physical memory, 1/8 of the process address space) - if (tokudb_cache_size == 0) { + // config the cache table size to min(1/2 of physical memory, 1/8 of the + // process address space) + if (tokudb::sysvars::cache_size == 0) { uint64_t physmem, maxdata; physmem = toku_os_get_phys_memory_size(); - tokudb_cache_size = physmem / 2; + tokudb::sysvars::cache_size = physmem / 2; r = toku_os_get_max_process_data_size(&maxdata); if (r == 0) { - if (tokudb_cache_size > maxdata / 8) - tokudb_cache_size = maxdata / 8; + if (tokudb::sysvars::cache_size > maxdata / 8) + tokudb::sysvars::cache_size = maxdata / 8; } } - if (tokudb_cache_size) { - DBUG_PRINT("info", ("tokudb_cache_size: %lld\n", tokudb_cache_size)); - r = db_env->set_cachesize(db_env, (uint32_t)(tokudb_cache_size >> 30), (uint32_t)(tokudb_cache_size % (1024L * 1024L * 1024L)), 1); + if (tokudb::sysvars::cache_size) { + DBUG_PRINT( + "info", + ("tokudb_cache_size: %lld\n", tokudb::sysvars::cache_size)); + r = db_env->set_cachesize( + db_env, + (uint32_t)(tokudb::sysvars::cache_size >> 30), + (uint32_t)(tokudb::sysvars::cache_size % + (1024L * 1024L * 1024L)), 1); if (r) { DBUG_PRINT("info", ("set_cachesize %d\n", r)); goto error; } } - if (tokudb_max_lock_memory == 0) { - tokudb_max_lock_memory = tokudb_cache_size/8; - } - if (tokudb_max_lock_memory) { - DBUG_PRINT("info", ("tokudb_max_lock_memory: %lld\n", tokudb_max_lock_memory)); - r = db_env->set_lk_max_memory(db_env, tokudb_max_lock_memory); + if (tokudb::sysvars::max_lock_memory == 0) { + tokudb::sysvars::max_lock_memory = tokudb::sysvars::cache_size/8; + } + if (tokudb::sysvars::max_lock_memory) { + DBUG_PRINT( + "info", + ("tokudb_max_lock_memory: %lld\n", + tokudb::sysvars::max_lock_memory)); + r = db_env->set_lk_max_memory( + db_env, + tokudb::sysvars::max_lock_memory); if (r) { DBUG_PRINT("info", ("set_lk_max_memory %d\n", r)); goto error; @@ -457,51 +480,73 @@ static int tokudb_init_func(void *p) { uint32_t gbytes, bytes; int parts; r = db_env->get_cachesize(db_env, &gbytes, &bytes, &parts); - if (tokudb_debug & TOKUDB_DEBUG_INIT) - TOKUDB_TRACE("tokudb_cache_size=%lld r=%d", ((unsigned long long) gbytes << 30) + bytes, r); + TOKUDB_TRACE_FOR_FLAGS( + TOKUDB_DEBUG_INIT, + "tokudb_cache_size=%lld r=%d", + ((unsigned long long) gbytes << 30) + bytes, + r); - r = db_env->set_client_pool_threads(db_env, tokudb_client_pool_threads); + r = db_env->set_client_pool_threads( + db_env, + tokudb::sysvars::client_pool_threads); if (r) { DBUG_PRINT("info", ("set_client_pool_threads %d\n", r)); goto error; } - r = db_env->set_cachetable_pool_threads(db_env, tokudb_cachetable_pool_threads); + r = db_env->set_cachetable_pool_threads( + db_env, + tokudb::sysvars::cachetable_pool_threads); if (r) { DBUG_PRINT("info", ("set_cachetable_pool_threads %d\n", r)); goto error; } - r = db_env->set_checkpoint_pool_threads(db_env, tokudb_checkpoint_pool_threads); + r = db_env->set_checkpoint_pool_threads( + db_env, + tokudb::sysvars::checkpoint_pool_threads); if (r) { DBUG_PRINT("info", ("set_checkpoint_pool_threads %d\n", r)); goto error; } if (db_env->set_redzone) { - r = db_env->set_redzone(db_env, tokudb_fs_reserve_percent); - if (tokudb_debug & TOKUDB_DEBUG_INIT) - TOKUDB_TRACE("set_redzone r=%d", r); + r = db_env->set_redzone(db_env, tokudb::sysvars::fs_reserve_percent); + TOKUDB_TRACE_FOR_FLAGS(TOKUDB_DEBUG_INIT, "set_redzone r=%d", r); } + TOKUDB_TRACE_FOR_FLAGS( + TOKUDB_DEBUG_INIT, + "env open:flags=%x", + tokudb_init_flags); + + r = db_env->set_generate_row_callback_for_put(db_env, generate_row_for_put); + assert_always(r == 0); - if (tokudb_debug & TOKUDB_DEBUG_INIT) - TOKUDB_TRACE("env open:flags=%x", tokudb_init_flags); + r = db_env->set_generate_row_callback_for_del(db_env, generate_row_for_del); + assert_always(r == 0); - r = db_env->set_generate_row_callback_for_put(db_env,generate_row_for_put); - assert(r == 0); - r = db_env->set_generate_row_callback_for_del(db_env,generate_row_for_del); - assert(r == 0); db_env->set_update(db_env, tokudb_update_fun); - db_env_set_direct_io(tokudb_directio == TRUE); - db_env_set_compress_buffers_before_eviction(tokudb_compress_buffers_before_eviction == TRUE); - db_env->change_fsync_log_period(db_env, tokudb_fsync_log_period); + + db_env_set_direct_io(tokudb::sysvars::directio == TRUE); + + db_env_set_compress_buffers_before_eviction( + tokudb::sysvars::compress_buffers_before_eviction == TRUE); + + db_env->change_fsync_log_period(db_env, tokudb::sysvars::fsync_log_period); + db_env->set_lock_timeout_callback(db_env, tokudb_lock_timeout_callback); - db_env->set_loader_memory_size(db_env, tokudb_get_loader_memory_size_callback); - r = db_env->open(db_env, tokudb_home, tokudb_init_flags, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH); + db_env->set_loader_memory_size( + db_env, + tokudb_get_loader_memory_size_callback); + + r = db_env->open( + db_env, + tokudb_home, + tokudb_init_flags, + S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH); - if (tokudb_debug & TOKUDB_DEBUG_INIT) - TOKUDB_TRACE("env opened:return=%d", r); + TOKUDB_TRACE_FOR_FLAGS(TOKUDB_DEBUG_INIT, "env opened:return=%d", r); if (r) { DBUG_PRINT("info", ("env->open %d", r)); @@ -509,72 +554,106 @@ static int tokudb_init_func(void *p) { goto error; } - r = db_env->checkpointing_set_period(db_env, tokudb_checkpointing_period); - assert(r == 0); - r = db_env->cleaner_set_period(db_env, tokudb_cleaner_period); - assert(r == 0); - r = db_env->cleaner_set_iterations(db_env, tokudb_cleaner_iterations); - assert(r == 0); + r = db_env->checkpointing_set_period( + db_env, + tokudb::sysvars::checkpointing_period); + assert_always(r == 0); - r = db_env->set_lock_timeout(db_env, DEFAULT_TOKUDB_LOCK_TIMEOUT, tokudb_get_lock_wait_time_callback); - assert(r == 0); + r = db_env->cleaner_set_period(db_env, tokudb::sysvars::cleaner_period); + assert_always(r == 0); - r = db_env->evictor_set_enable_partial_eviction(db_env, - tokudb_enable_partial_eviction); - assert(r == 0); + r = db_env->cleaner_set_iterations( + db_env, + tokudb::sysvars::cleaner_iterations); + assert_always(r == 0); - db_env->set_killed_callback(db_env, DEFAULT_TOKUDB_KILLED_TIME, tokudb_get_killed_time_callback, tokudb_killed_callback); + r = db_env->set_lock_timeout( + db_env, + DEFAULT_TOKUDB_LOCK_TIMEOUT, + tokudb_get_lock_wait_time_callback); + assert_always(r == 0); - r = db_env->get_engine_status_num_rows (db_env, &toku_global_status_max_rows); - assert(r == 0); + r = db_env->evictor_set_enable_partial_eviction( + db_env, + tokudb::sysvars::enable_partial_eviction); + assert_always(r == 0); + + db_env->set_killed_callback( + db_env, + DEFAULT_TOKUDB_KILLED_TIME, + tokudb_get_killed_time_callback, + tokudb_killed_callback); + + r = db_env->get_engine_status_num_rows( + db_env, + &toku_global_status_max_rows); + assert_always(r == 0); { - const myf mem_flags = MY_FAE|MY_WME|MY_ZEROFILL|MY_ALLOW_ZERO_PTR|MY_FREE_ON_ERROR; - toku_global_status_variables = (SHOW_VAR*)tokudb_my_malloc(sizeof(*toku_global_status_variables)*toku_global_status_max_rows, mem_flags); - toku_global_status_rows = (TOKU_ENGINE_STATUS_ROW_S*)tokudb_my_malloc(sizeof(*toku_global_status_rows)*toku_global_status_max_rows, mem_flags); + const myf mem_flags = + MY_FAE|MY_WME|MY_ZEROFILL|MY_ALLOW_ZERO_PTR|MY_FREE_ON_ERROR; + toku_global_status_variables = + (SHOW_VAR*)tokudb::memory::malloc( + sizeof(*toku_global_status_variables) * + toku_global_status_max_rows, + mem_flags); + toku_global_status_rows = + (TOKU_ENGINE_STATUS_ROW_S*)tokudb::memory::malloc( + sizeof(*toku_global_status_rows)* + toku_global_status_max_rows, + mem_flags); } tokudb_primary_key_bytes_inserted = create_partitioned_counter(); #if TOKU_THDVAR_MEMALLOC_BUG - tokudb_pthread_mutex_init(&tokudb_map_mutex, MY_MUTEX_INIT_FAST); init_tree(&tokudb_map, 0, 0, 0, tokudb_map_pair_cmp, true, NULL, NULL); #endif + if (tokudb::sysvars::strip_frm_data) { + r = tokudb::metadata::strip_frm_data(db_env); + if (r) { + DBUG_PRINT("info", ("env->open %d", r)); + handle_ydb_error(r); + goto error; + } + } + //3938: succeeded, set the init status flag and unlock tokudb_hton_initialized = 1; - rw_unlock(&tokudb_hton_initialized_lock); + tokudb_hton_initialized_lock.unlock(); DBUG_RETURN(false); error: if (db_env) { int rr= db_env->close(db_env, 0); - assert(rr==0); + assert_always(rr==0); db_env = 0; } // 3938: failed to initialized, drop the flag and lock tokudb_hton_initialized = 0; - rw_unlock(&tokudb_hton_initialized_lock); + tokudb_hton_initialized_lock.unlock(); DBUG_RETURN(true); } -static int tokudb_done_func(void *p) { +static int tokudb_done_func(void* p) { TOKUDB_DBUG_ENTER(""); - tokudb_my_free(toku_global_status_variables); + tokudb::memory::free(toku_global_status_variables); toku_global_status_variables = NULL; - tokudb_my_free(toku_global_status_rows); + tokudb::memory::free(toku_global_status_rows); toku_global_status_rows = NULL; - my_hash_free(&tokudb_open_tables); - tokudb_pthread_mutex_destroy(&tokudb_mutex); TOKUDB_DBUG_RETURN(0); } -static handler *tokudb_create_handler(handlerton * hton, TABLE_SHARE * table, MEM_ROOT * mem_root) { +static handler* tokudb_create_handler( + handlerton* hton, + TABLE_SHARE* table, + MEM_ROOT* mem_root) { return new(mem_root) ha_tokudb(hton, table); } -int tokudb_end(handlerton * hton, ha_panic_function type) { +int tokudb_end(handlerton* hton, ha_panic_function type) { TOKUDB_DBUG_ENTER(""); int error = 0; @@ -582,41 +661,57 @@ int tokudb_end(handlerton * hton, ha_panic_function type) { // initialized. grab a writer lock for the duration of the // call, so we can drop the flag and destroy the mutexes // in isolation. - rw_wrlock(&tokudb_hton_initialized_lock); - assert(tokudb_hton_initialized); + tokudb_hton_initialized_lock.lock_write(); + assert_always(tokudb_hton_initialized); + + tokudb::background::destroy(); + TOKUDB_SHARE::static_destroy(); if (db_env) { if (tokudb_init_flags & DB_INIT_LOG) tokudb_cleanup_log_files(); - long total_prepared = 0; // count the total number of prepared txn's that we discard + + // count the total number of prepared txn's that we discard + long total_prepared = 0; #if TOKU_INCLUDE_XA while (1) { // get xid's const long n_xid = 1; TOKU_XA_XID xids[n_xid]; long n_prepared = 0; - error = db_env->txn_xa_recover(db_env, xids, n_xid, &n_prepared, total_prepared == 0 ? DB_FIRST : DB_NEXT); - assert(error == 0); + error = db_env->txn_xa_recover( + db_env, + xids, + n_xid, + &n_prepared, + total_prepared == 0 ? DB_FIRST : DB_NEXT); + assert_always(error == 0); if (n_prepared == 0) break; // discard xid's for (long i = 0; i < n_xid; i++) { DB_TXN *txn = NULL; error = db_env->get_txn_from_xid(db_env, &xids[i], &txn); - assert(error == 0); + assert_always(error == 0); error = txn->discard(txn, 0); - assert(error == 0); + assert_always(error == 0); } total_prepared += n_prepared; } #endif - error = db_env->close(db_env, total_prepared > 0 ? TOKUFT_DIRTY_SHUTDOWN : 0); + error = db_env->close( + db_env, + total_prepared > 0 ? TOKUFT_DIRTY_SHUTDOWN : 0); #if TOKU_INCLUDE_XA if (error != 0 && total_prepared > 0) { - sql_print_error("%s: %ld prepared txns still live, please shutdown, error %d", tokudb_hton_name, total_prepared, error); + sql_print_error( + "%s: %ld prepared txns still live, please shutdown, error %d", + tokudb_hton_name, + total_prepared, + error); } else #endif - assert(error == 0); + assert_always(error == 0); db_env = NULL; } @@ -626,33 +721,34 @@ int tokudb_end(handlerton * hton, ha_panic_function type) { } #if TOKU_THDVAR_MEMALLOC_BUG - tokudb_pthread_mutex_destroy(&tokudb_map_mutex); delete_tree(&tokudb_map); #endif // 3938: drop the initialized flag and unlock tokudb_hton_initialized = 0; - rw_unlock(&tokudb_hton_initialized_lock); + tokudb_hton_initialized_lock.unlock(); TOKUDB_DBUG_RETURN(error); } -static int tokudb_close_connection(handlerton * hton, THD * thd) { +static int tokudb_close_connection(handlerton* hton, THD* thd) { int error = 0; - tokudb_trx_data* trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton); + tokudb_trx_data* trx = (tokudb_trx_data*)thd_get_ha_data(thd, tokudb_hton); if (trx && trx->checkpoint_lock_taken) { error = db_env->checkpointing_resume(db_env); } - tokudb_my_free(trx); + tokudb::memory::free(trx); #if TOKU_THDVAR_MEMALLOC_BUG - tokudb_pthread_mutex_lock(&tokudb_map_mutex); + tokudb_map_mutex.lock(); struct tokudb_map_pair key = { thd, NULL }; - struct tokudb_map_pair *found_key = (struct tokudb_map_pair *) tree_search(&tokudb_map, &key, NULL); + struct tokudb_map_pair* found_key = + (struct tokudb_map_pair*) tree_search(&tokudb_map, &key, NULL); + if (found_key) { - tokudb_my_free(found_key->last_lock_timeout); - tree_delete(&tokudb_map, found_key, sizeof *found_key, NULL); + tokudb::memory::free(found_key->last_lock_timeout); + tree_delete(&tokudb_map, found_key, sizeof(*found_key), NULL); } - tokudb_pthread_mutex_unlock(&tokudb_map_mutex); + tokudb_map_mutex.unlock(); #endif return error; } @@ -662,7 +758,7 @@ bool tokudb_flush_logs(handlerton * hton) { int error; bool result = 0; - if (tokudb_checkpoint_on_flush_logs) { + if (tokudb::sysvars::checkpoint_on_flush_logs) { // // take the checkpoint // @@ -675,7 +771,7 @@ bool tokudb_flush_logs(handlerton * hton) { } else { error = db_env->log_flush(db_env, NULL); - assert(error == 0); + assert_always(error == 0); } result = 0; @@ -691,14 +787,14 @@ typedef struct txn_progress_info { static void txn_progress_func(TOKU_TXN_PROGRESS progress, void* extra) { TXN_PROGRESS_INFO progress_info = (TXN_PROGRESS_INFO)extra; - int r = sprintf(progress_info->status, - "%sprocessing %s of transaction, %" PRId64 " out of %" PRId64, - progress->stalled_on_checkpoint ? "Writing committed changes to disk, " : "", - progress->is_commit ? "commit" : "abort", - progress->entries_processed, - progress->entries_total - ); - assert(r >= 0); + int r = sprintf( + progress_info->status, + "%sprocessing %s of transaction, %" PRId64 " out of %" PRId64, + progress->stalled_on_checkpoint ? "Writing committed changes to disk, " : "", + progress->is_commit ? "commit" : "abort", + progress->entries_processed, + progress->entries_total); + assert_always(r >= 0); thd_proc_info(progress_info->thd, progress_info->status); } @@ -708,9 +804,13 @@ static void commit_txn_with_progress(DB_TXN* txn, uint32_t flags, THD* thd) { info.thd = thd; int r = txn->commit_with_progress(txn, flags, txn_progress_func, &info); if (r != 0) { - sql_print_error("%s: tried committing transaction %p and got error code %d", tokudb_hton_name, txn, r); + sql_print_error( + "%s: tried committing transaction %p and got error code %d", + tokudb_hton_name, + txn, + r); } - assert(r == 0); + assert_always(r == 0); thd_proc_info(thd, orig_proc_info); } @@ -720,9 +820,13 @@ static void abort_txn_with_progress(DB_TXN* txn, THD* thd) { info.thd = thd; int r = txn->abort_with_progress(txn, txn_progress_func, &info); if (r != 0) { - sql_print_error("%s: tried aborting transaction %p and got error code %d", tokudb_hton_name, txn, r); + sql_print_error( + "%s: tried aborting transaction %p and got error code %d", + tokudb_hton_name, + txn, + r); } - assert(r == 0); + assert_always(r == 0); thd_proc_info(thd, orig_proc_info); } @@ -736,11 +840,12 @@ static void tokudb_cleanup_handlers(tokudb_trx_data *trx, DB_TXN *txn) { } #if MYSQL_VERSION_ID >= 50600 -extern "C" enum durability_properties thd_get_durability_property(const MYSQL_THD thd); +extern "C" enum durability_properties thd_get_durability_property( + const MYSQL_THD thd); #endif // Determine if an fsync is used when a transaction is committed. -static bool tokudb_sync_on_commit(THD *thd, tokudb_trx_data *trx, DB_TXN *txn) { +static bool tokudb_sync_on_commit(THD* thd, tokudb_trx_data* trx, DB_TXN* txn) { #if MYSQL_VERSION_ID >= 50600 // Check the client durability property which is set during 2PC if (thd_get_durability_property(thd) == HA_IGNORE_DURABILITY) @@ -751,9 +856,9 @@ static bool tokudb_sync_on_commit(THD *thd, tokudb_trx_data *trx, DB_TXN *txn) { if (txn->is_prepared(txn) && mysql_bin_log.is_open()) return false; #endif - if (tokudb_fsync_log_period > 0) + if (tokudb::sysvars::fsync_log_period > 0) return false; - return THDVAR(thd, commit_sync) != 0; + return tokudb::sysvars::commit_sync(thd) != 0; } static int tokudb_commit(handlerton * hton, THD * thd, bool all) { @@ -763,10 +868,14 @@ static int tokudb_commit(handlerton * hton, THD * thd, bool all) { DB_TXN **txn = all ? &trx->all : &trx->stmt; DB_TXN *this_txn = *txn; if (this_txn) { - uint32_t syncflag = tokudb_sync_on_commit(thd, trx, this_txn) ? 0 : DB_TXN_NOSYNC; - if (tokudb_debug & TOKUDB_DEBUG_TXN) { - TOKUDB_TRACE("commit trx %u txn %p syncflag %u", all, this_txn, syncflag); - } + uint32_t syncflag = + tokudb_sync_on_commit(thd, trx, this_txn) ? 0 : DB_TXN_NOSYNC; + TOKUDB_TRACE_FOR_FLAGS( + TOKUDB_DEBUG_TXN, + "commit trx %u txn %p syncflag %u", + all, + this_txn, + syncflag); // test hook to induce a crash on a debug build DBUG_EXECUTE_IF("tokudb_crash_commit_before", DBUG_SUICIDE();); tokudb_cleanup_handlers(trx, this_txn); @@ -778,9 +887,8 @@ static int tokudb_commit(handlerton * hton, THD * thd, bool all) { if (this_txn == trx->sp_level || trx->all == NULL) { trx->sp_level = NULL; } - } - else if (tokudb_debug & TOKUDB_DEBUG_TXN) { - TOKUDB_TRACE("nothing to commit %d", all); + } else { + TOKUDB_TRACE_FOR_FLAGS(TOKUDB_DEBUG_TXN, "nothing to commit %d", all); } reset_stmt_progress(&trx->stmt_progress); TOKUDB_DBUG_RETURN(0); @@ -793,9 +901,11 @@ static int tokudb_rollback(handlerton * hton, THD * thd, bool all) { DB_TXN **txn = all ? &trx->all : &trx->stmt; DB_TXN *this_txn = *txn; if (this_txn) { - if (tokudb_debug & TOKUDB_DEBUG_TXN) { - TOKUDB_TRACE("rollback %u txn %p", all, this_txn); - } + TOKUDB_TRACE_FOR_FLAGS( + TOKUDB_DEBUG_TXN, + "rollback %u txn %p", + all, + this_txn); tokudb_cleanup_handlers(trx, this_txn); abort_txn_with_progress(this_txn, thd); *txn = NULL; @@ -803,11 +913,8 @@ static int tokudb_rollback(handlerton * hton, THD * thd, bool all) { if (this_txn == trx->sp_level || trx->all == NULL) { trx->sp_level = NULL; } - } - else { - if (tokudb_debug & TOKUDB_DEBUG_TXN) { - TOKUDB_TRACE("abort0"); - } + } else { + TOKUDB_TRACE_FOR_FLAGS(TOKUDB_DEBUG_TXN, "abort0"); } reset_stmt_progress(&trx->stmt_progress); TOKUDB_DBUG_RETURN(0); @@ -816,7 +923,7 @@ static int tokudb_rollback(handlerton * hton, THD * thd, bool all) { #if TOKU_INCLUDE_XA static bool tokudb_sync_on_prepare(void) { // skip sync of log if fsync log period > 0 - if (tokudb_fsync_log_period > 0) + if (tokudb::sysvars::fsync_log_period > 0) return false; else return true; @@ -827,7 +934,7 @@ static int tokudb_xa_prepare(handlerton* hton, THD* thd, bool all) { int r = 0; // if tokudb_support_xa is disable, just return - if (!THDVAR(thd, support_xa)) { + if (!tokudb::sysvars::support_xa(thd)) { TOKUDB_DBUG_RETURN(r); } @@ -836,9 +943,11 @@ static int tokudb_xa_prepare(handlerton* hton, THD* thd, bool all) { DB_TXN* txn = all ? trx->all : trx->stmt; if (txn) { uint32_t syncflag = tokudb_sync_on_prepare() ? 0 : DB_TXN_NOSYNC; - if (tokudb_debug & TOKUDB_DEBUG_TXN) { - TOKUDB_TRACE("doing txn prepare:%d:%p", all, txn); - } + TOKUDB_TRACE_FOR_FLAGS( + TOKUDB_DEBUG_TXN, + "doing txn prepare:%d:%p", + all, + txn); // a TOKU_XA_XID is identical to a MYSQL_XID TOKU_XA_XID thd_xid; thd_get_xid(thd, (MYSQL_XID*) &thd_xid); @@ -847,9 +956,8 @@ static int tokudb_xa_prepare(handlerton* hton, THD* thd, bool all) { r = txn->xa_prepare(txn, &thd_xid, syncflag); // test hook to induce a crash on a debug build DBUG_EXECUTE_IF("tokudb_crash_prepare_after", DBUG_SUICIDE();); - } - else if (tokudb_debug & TOKUDB_DEBUG_TXN) { - TOKUDB_TRACE("nothing to prepare %d", all); + } else { + TOKUDB_TRACE_FOR_FLAGS(TOKUDB_DEBUG_TXN, "nothing to prepare %d", all); } TOKUDB_DBUG_RETURN(r); } @@ -866,9 +974,8 @@ static int tokudb_xa_recover(handlerton* hton, XID* xid_list, uint len) { (TOKU_XA_XID*)xid_list, len, &num_returned, - DB_NEXT - ); - assert(r == 0); + DB_NEXT); + assert_always(r == 0); TOKUDB_DBUG_RETURN((int)num_returned); } @@ -914,43 +1021,55 @@ static int tokudb_savepoint(handlerton * hton, THD * thd, void *savepoint) { SP_INFO save_info = (SP_INFO)savepoint; tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(thd, hton); if (thd->in_sub_stmt) { - assert(trx->stmt); - error = txn_begin(db_env, trx->sub_sp_level, &(save_info->txn), DB_INHERIT_ISOLATION, thd); + assert_always(trx->stmt); + error = txn_begin( + db_env, + trx->sub_sp_level, + &(save_info->txn), + DB_INHERIT_ISOLATION, + thd); if (error) { goto cleanup; } trx->sub_sp_level = save_info->txn; save_info->in_sub_stmt = true; - } - else { - error = txn_begin(db_env, trx->sp_level, &(save_info->txn), DB_INHERIT_ISOLATION, thd); + } else { + error = txn_begin( + db_env, + trx->sp_level, + &(save_info->txn), + DB_INHERIT_ISOLATION, + thd); if (error) { goto cleanup; } trx->sp_level = save_info->txn; save_info->in_sub_stmt = false; } - if (tokudb_debug & TOKUDB_DEBUG_TXN) { - TOKUDB_TRACE("begin txn %p", save_info->txn); - } + TOKUDB_TRACE_FOR_FLAGS(TOKUDB_DEBUG_TXN, "begin txn %p", save_info->txn); save_info->trx = trx; error = 0; cleanup: TOKUDB_DBUG_RETURN(error); } -static int tokudb_rollback_to_savepoint(handlerton * hton, THD * thd, void *savepoint) { +static int tokudb_rollback_to_savepoint( + handlerton* hton, + THD* thd, + void* savepoint) { + TOKUDB_DBUG_ENTER("%p", savepoint); int error; SP_INFO save_info = (SP_INFO)savepoint; DB_TXN* parent = NULL; DB_TXN* txn_to_rollback = save_info->txn; - tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(thd, hton); + tokudb_trx_data* trx = (tokudb_trx_data*)thd_get_ha_data(thd, hton); parent = txn_to_rollback->parent; - if (tokudb_debug & TOKUDB_DEBUG_TXN) { - TOKUDB_TRACE("rollback txn %p", txn_to_rollback); - } + TOKUDB_TRACE_FOR_FLAGS( + TOKUDB_DEBUG_TXN, + "rollback txn %p", + txn_to_rollback); if (!(error = txn_to_rollback->abort(txn_to_rollback))) { if (save_info->in_sub_stmt) { trx->sub_sp_level = parent; @@ -963,7 +1082,11 @@ static int tokudb_rollback_to_savepoint(handlerton * hton, THD * thd, void *save TOKUDB_DBUG_RETURN(error); } -static int tokudb_release_savepoint(handlerton * hton, THD * thd, void *savepoint) { +static int tokudb_release_savepoint( + handlerton* hton, + THD* thd, + void* savepoint) { + TOKUDB_DBUG_ENTER("%p", savepoint); int error = 0; SP_INFO save_info = (SP_INFO)savepoint; @@ -972,9 +1095,7 @@ static int tokudb_release_savepoint(handlerton * hton, THD * thd, void *savepoin tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(thd, hton); parent = txn_to_commit->parent; - if (tokudb_debug & TOKUDB_DEBUG_TXN) { - TOKUDB_TRACE("commit txn %p", txn_to_commit); - } + TOKUDB_TRACE_FOR_FLAGS(TOKUDB_DEBUG_TXN, "commit txn %p", txn_to_commit); DB_TXN *child = txn_to_commit->get_child(txn_to_commit); if (child == NULL && !(error = txn_to_commit->commit(txn_to_commit, 0))) { if (save_info->in_sub_stmt) { @@ -992,8 +1113,14 @@ static int tokudb_release_savepoint(handlerton * hton, THD * thd, void *savepoin static int tokudb_discover_table(handlerton *hton, THD* thd, TABLE_SHARE *ts) { uchar *frmblob = 0; size_t frmlen; - int res= tokudb_discover3(hton, thd, ts->db.str, ts->table_name.str, - ts->normalized_path.str, &frmblob, &frmlen); + int res= tokudb_discover3( + hton, + thd, + ts->db.str, + ts->table_name.str, + ts->normalized_path.str, + &frmblob, + &frmlen); if (!res) res= ts->init_from_binary_frm_image(thd, true, frmblob, frmlen); @@ -1002,7 +1129,11 @@ static int tokudb_discover_table(handlerton *hton, THD* thd, TABLE_SHARE *ts) { return res == ENOENT ? HA_ERR_NO_SUCH_TABLE : res; } -static int tokudb_discover_table_existence(handlerton *hton, const char *db, const char *name) { +static int tokudb_discover_table_existence( + handlerton* hton, + const char* db, + const char* name) { + uchar *frmblob = 0; size_t frmlen; int res= tokudb_discover(hton, current_thd, db, name, &frmblob, &frmlen); @@ -1011,19 +1142,46 @@ static int tokudb_discover_table_existence(handlerton *hton, const char *db, con } #endif -static int tokudb_discover(handlerton *hton, THD* thd, const char *db, const char *name, uchar **frmblob, size_t *frmlen) { +static int tokudb_discover( + handlerton* hton, + THD* thd, + const char* db, + const char* name, + uchar** frmblob, + size_t* frmlen) { + return tokudb_discover2(hton, thd, db, name, true, frmblob, frmlen); } -static int tokudb_discover2(handlerton *hton, THD* thd, const char *db, const char *name, bool translate_name, - uchar **frmblob, size_t *frmlen) { +static int tokudb_discover2( + handlerton* hton, + THD* thd, + const char* db, + const char* name, + bool translate_name, + uchar** frmblob, + size_t*frmlen) { + char path[FN_REFLEN + 1]; - build_table_filename(path, sizeof(path) - 1, db, name, "", translate_name ? 0 : FN_IS_TMP); + build_table_filename( + path, + sizeof(path) - 1, + db, + name, + "", + translate_name ? 0 : FN_IS_TMP); return tokudb_discover3(hton, thd, db, name, path, frmblob, frmlen); } -static int tokudb_discover3(handlerton *hton, THD* thd, const char *db, const char *name, char *path, - uchar **frmblob, size_t *frmlen) { +static int tokudb_discover3( + handlerton* hton, + THD* thd, + const char* db, + const char* name, + char* path, + uchar** frmblob, + size_t* frmlen) { + TOKUDB_DBUG_ENTER("%s %s %s", db, name, path); int error; DB* status_db = NULL; @@ -1034,8 +1192,10 @@ static int tokudb_discover3(handlerton *hton, THD* thd, const char *db, const ch bool do_commit = false; #if 100000 <= MYSQL_VERSION_ID && MYSQL_VERSION_ID <= 100099 - tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton); - if (thd_sql_command(thd) == SQLCOM_CREATE_TABLE && trx && trx->sub_sp_level) { + tokudb_trx_data* trx = (tokudb_trx_data*)thd_get_ha_data(thd, tokudb_hton); + if (thd_sql_command(thd) == SQLCOM_CREATE_TABLE && + trx && + trx->sub_sp_level) { do_commit = false; txn = trx->sub_sp_level; } else { @@ -1061,8 +1221,7 @@ static int tokudb_discover3(handlerton *hton, THD* thd, const char *db, const ch 0, &key, smart_dbt_callback_verify_frm, - &value - ); + &value); if (error) { goto cleanup; } @@ -1078,19 +1237,23 @@ cleanup: if (do_commit && txn) { commit_txn(txn, 0); } - TOKUDB_DBUG_RETURN(error); + TOKUDB_DBUG_RETURN(error); } -#define STATPRINT(legend, val) if (legend != NULL && val != NULL) stat_print(thd, \ - tokudb_hton_name, \ - strlen(tokudb_hton_name), \ - legend, \ - strlen(legend), \ - val, \ - strlen(val)) +#define STATPRINT(legend, val) if (legend != NULL && val != NULL) \ + stat_print(thd, \ + tokudb_hton_name, \ + strlen(tokudb_hton_name), \ + legend, \ + strlen(legend), \ + val, \ + strlen(val)) -extern sys_var *intern_find_sys_var(const char *str, uint length, bool no_error); +extern sys_var* intern_find_sys_var( + const char* str, + uint length, + bool no_error); static bool tokudb_show_engine_status(THD * thd, stat_print_fn * stat_print) { TOKUDB_DBUG_ENTER(""); @@ -1106,14 +1269,29 @@ static bool tokudb_show_engine_status(THD * thd, stat_print_fn * stat_print) { #if MYSQL_VERSION_ID < 50500 { - sys_var * version = intern_find_sys_var("version", 0, false); - snprintf(buf, bufsiz, "%s", version->value_ptr(thd, (enum_var_type)0, (LEX_STRING*)NULL)); + sys_var* version = intern_find_sys_var("version", 0, false); + snprintf( + buf, + bufsiz, + "%s", + version->value_ptr(thd, + (enum_var_type)0, + (LEX_STRING*)NULL)); STATPRINT("Version", buf); } #endif error = db_env->get_engine_status_num_rows (db_env, &max_rows); TOKU_ENGINE_STATUS_ROW_S mystat[max_rows]; - error = db_env->get_engine_status (db_env, mystat, max_rows, &num_rows, &redzone_state, &panic, panic_string, panic_string_len, TOKU_ENGINE_STATUS); + error = db_env->get_engine_status( + db_env, + mystat, + max_rows, + &num_rows, + &redzone_state, + &panic, + panic_string, + panic_string_len, + TOKU_ENGINE_STATUS); if (strlen(panic_string)) { STATPRINT("Environment panic string", panic_string); @@ -1125,20 +1303,35 @@ static bool tokudb_show_engine_status(THD * thd, stat_print_fn * stat_print) { } if(redzone_state == FS_BLOCKED) { - STATPRINT("*** URGENT WARNING ***", "FILE SYSTEM IS COMPLETELY FULL"); + STATPRINT( + "*** URGENT WARNING ***", "FILE SYSTEM IS COMPLETELY FULL"); snprintf(buf, bufsiz, "FILE SYSTEM IS COMPLETELY FULL"); - } - else if (redzone_state == FS_GREEN) { - snprintf(buf, bufsiz, "more than %d percent of total file system space", 2*tokudb_fs_reserve_percent); - } - else if (redzone_state == FS_YELLOW) { - snprintf(buf, bufsiz, "*** WARNING *** FILE SYSTEM IS GETTING FULL (less than %d percent free)", 2*tokudb_fs_reserve_percent); - } - else if (redzone_state == FS_RED){ - snprintf(buf, bufsiz, "*** WARNING *** FILE SYSTEM IS GETTING VERY FULL (less than %d percent free): INSERTS ARE PROHIBITED", tokudb_fs_reserve_percent); - } - else { - snprintf(buf, bufsiz, "information unavailable, unknown redzone state %d", redzone_state); + } else if (redzone_state == FS_GREEN) { + snprintf( + buf, + bufsiz, + "more than %d percent of total file system space", + 2 * tokudb::sysvars::fs_reserve_percent); + } else if (redzone_state == FS_YELLOW) { + snprintf( + buf, + bufsiz, + "*** WARNING *** FILE SYSTEM IS GETTING FULL (less than %d " + "percent free)", + 2 * tokudb::sysvars::fs_reserve_percent); + } else if (redzone_state == FS_RED){ + snprintf( + buf, + bufsiz, + "*** WARNING *** FILE SYSTEM IS GETTING VERY FULL (less than " + "%d percent free): INSERTS ARE PROHIBITED", + tokudb::sysvars::fs_reserve_percent); + } else { + snprintf( + buf, + bufsiz, + "information unavailable, unknown redzone state %d", + redzone_state); } STATPRINT ("disk free space", buf); @@ -1165,7 +1358,8 @@ static bool tokudb_show_engine_status(THD * thd, stat_print_fn * stat_print) { break; } case PARCOUNT: { - uint64_t v = read_partitioned_counter(mystat[row].value.parcount); + uint64_t v = read_partitioned_counter( + mystat[row].value.parcount); snprintf(buf, bufsiz, "%" PRIu64, v); break; } @@ -1173,12 +1367,17 @@ static bool tokudb_show_engine_status(THD * thd, stat_print_fn * stat_print) { snprintf(buf, bufsiz, "%.6f", mystat[row].value.dnum); break; default: - snprintf(buf, bufsiz, "UNKNOWN STATUS TYPE: %d", mystat[row].type); - break; + snprintf( + buf, + bufsiz, + "UNKNOWN STATUS TYPE: %d", + mystat[row].type); + break; } STATPRINT(mystat[row].legend, buf); } - uint64_t bytes_inserted = read_partitioned_counter(tokudb_primary_key_bytes_inserted); + uint64_t bytes_inserted = read_partitioned_counter( + tokudb_primary_key_bytes_inserted); snprintf(buf, bufsiz, "%" PRIu64, bytes_inserted); STATPRINT("handlerton: primary key bytes inserted", buf); } @@ -1186,16 +1385,16 @@ static bool tokudb_show_engine_status(THD * thd, stat_print_fn * stat_print) { TOKUDB_DBUG_RETURN(error); } -static void tokudb_checkpoint_lock(THD * thd) { +void tokudb_checkpoint_lock(THD * thd) { int error; const char *old_proc_info; - tokudb_trx_data* trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton); + tokudb_trx_data* trx = (tokudb_trx_data*)thd_get_ha_data(thd, tokudb_hton); if (!trx) { error = create_tokudb_trx_data_instance(&trx); // // can only fail due to memory allocation, so ok to assert // - assert(!error); + assert_always(!error); thd_set_ha_data(thd, tokudb_hton, trx); } @@ -1209,7 +1408,7 @@ static void tokudb_checkpoint_lock(THD * thd) { old_proc_info = tokudb_thd_get_proc_info(thd); thd_proc_info(thd, "Trying to grab checkpointing lock."); error = db_env->checkpointing_postpone(db_env); - assert(!error); + assert_always(!error); thd_proc_info(thd, old_proc_info); trx->checkpoint_lock_taken = true; @@ -1217,10 +1416,10 @@ cleanup: return; } -static void tokudb_checkpoint_unlock(THD * thd) { +void tokudb_checkpoint_unlock(THD * thd) { int error; const char *old_proc_info; - tokudb_trx_data* trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton); + tokudb_trx_data* trx = (tokudb_trx_data*)thd_get_ha_data(thd, tokudb_hton); if (!trx) { error = 0; goto cleanup; @@ -1235,7 +1434,7 @@ static void tokudb_checkpoint_unlock(THD * thd) { old_proc_info = tokudb_thd_get_proc_info(thd); thd_proc_info(thd, "Trying to release checkpointing lock."); error = db_env->checkpointing_resume(db_env); - assert(!error); + assert_always(!error); thd_proc_info(thd, old_proc_info); trx->checkpoint_lock_taken = false; @@ -1244,7 +1443,12 @@ cleanup: return; } -static bool tokudb_show_status(handlerton * hton, THD * thd, stat_print_fn * stat_print, enum ha_stat_type stat_type) { +static bool tokudb_show_status( + handlerton* hton, + THD* thd, + stat_print_fn* stat_print, + enum ha_stat_type stat_type) { + switch (stat_type) { case HA_ENGINE_STATUS: return tokudb_show_engine_status(thd, stat_print); @@ -1256,14 +1460,21 @@ static bool tokudb_show_status(handlerton * hton, THD * thd, stat_print_fn * sta } #if TOKU_INCLUDE_HANDLERTON_HANDLE_FATAL_SIGNAL -static void tokudb_handle_fatal_signal(handlerton *hton __attribute__ ((__unused__)), THD *thd __attribute__ ((__unused__)), int sig) { +static void tokudb_handle_fatal_signal( + TOKUDB_UNUSED(handlerton* hton), + TOKUDB_UNUSD(THD* thd), + int sig) { + if (tokudb_gdb_on_fatal) { db_env_try_gdb_stack_trace(tokudb_gdb_path); } } #endif -static void tokudb_print_error(const DB_ENV * db_env, const char *db_errpfx, const char *buffer) { +static void tokudb_print_error( + const DB_ENV* db_env, + const char* db_errpfx, + const char* buffer) { sql_print_error("%s: %s", db_errpfx, buffer); } @@ -1285,7 +1496,7 @@ static void tokudb_cleanup_log_files(void) { char **np; for (np = names; *np; ++np) { #if 1 - if (tokudb_debug) + if (TOKUDB_UNLIKELY(tokudb::sysvars::debug)) TOKUDB_TRACE("cleanup:%s", *np); #else my_delete(*np, MYF(MY_WME)); @@ -1298,246 +1509,13 @@ static void tokudb_cleanup_log_files(void) { DBUG_VOID_RETURN; } -// options flags -// PLUGIN_VAR_THDLOCAL Variable is per-connection -// PLUGIN_VAR_READONLY Server variable is read only -// PLUGIN_VAR_NOSYSVAR Not a server variable -// PLUGIN_VAR_NOCMDOPT Not a command line option -// PLUGIN_VAR_NOCMDARG No argument for cmd line -// PLUGIN_VAR_RQCMDARG Argument required for cmd line -// PLUGIN_VAR_OPCMDARG Argument optional for cmd line -// PLUGIN_VAR_MEMALLOC String needs memory allocated - - -// system variables -static void tokudb_cleaner_period_update(THD* thd, - struct st_mysql_sys_var* sys_var, - void* var, const void * save) { - ulong * cleaner_period = (ulong *) var; - *cleaner_period = *(const ulonglong *) save; - int r = db_env->cleaner_set_period(db_env, *cleaner_period); - assert(r == 0); -} - -#define DEFAULT_CLEANER_PERIOD 1 - -static MYSQL_SYSVAR_ULONG(cleaner_period, tokudb_cleaner_period, - 0, "TokuDB cleaner_period", - NULL, tokudb_cleaner_period_update, DEFAULT_CLEANER_PERIOD, - 0, ~0UL, 0); - -static void tokudb_cleaner_iterations_update(THD* thd, - struct st_mysql_sys_var* sys_var, - void* var, const void* save) { - ulong * cleaner_iterations = (ulong *) var; - *cleaner_iterations = *(const ulonglong *) save; - int r = db_env->cleaner_set_iterations(db_env, *cleaner_iterations); - assert(r == 0); -} - -#define DEFAULT_CLEANER_ITERATIONS 5 - -static MYSQL_SYSVAR_ULONG(cleaner_iterations, tokudb_cleaner_iterations, - 0, "TokuDB cleaner_iterations", - NULL, tokudb_cleaner_iterations_update, DEFAULT_CLEANER_ITERATIONS, - 0, ~0UL, 0); - -static void tokudb_checkpointing_period_update(THD* thd, - struct st_mysql_sys_var* sys_var, - void* var, const void* save) { - uint * checkpointing_period = (uint *) var; - *checkpointing_period = *(const ulonglong *) save; - int r = db_env->checkpointing_set_period(db_env, *checkpointing_period); - assert(r == 0); -} - -static MYSQL_SYSVAR_UINT(checkpointing_period, tokudb_checkpointing_period, - 0, "TokuDB Checkpointing period", - NULL, tokudb_checkpointing_period_update, 60, - 0, ~0U, 0); - -static MYSQL_SYSVAR_BOOL(directio, tokudb_directio, - PLUGIN_VAR_READONLY, "TokuDB Enable Direct I/O ", - NULL, NULL, FALSE); - -static MYSQL_SYSVAR_BOOL(compress_buffers_before_eviction, - tokudb_compress_buffers_before_eviction, - PLUGIN_VAR_READONLY, - "TokuDB Enable buffer compression before partial eviction", - NULL, NULL, TRUE); - -static MYSQL_SYSVAR_BOOL(checkpoint_on_flush_logs, tokudb_checkpoint_on_flush_logs, - 0, "TokuDB Checkpoint on Flush Logs ", - NULL, NULL, FALSE); - -static MYSQL_SYSVAR_ULONGLONG(cache_size, tokudb_cache_size, - PLUGIN_VAR_READONLY, "TokuDB cache table size", - NULL, NULL, 0, - 0, ~0ULL, 0); - -static MYSQL_SYSVAR_ULONGLONG(max_lock_memory, tokudb_max_lock_memory, - PLUGIN_VAR_READONLY, "TokuDB max memory for locks", - NULL, NULL, 0, - 0, ~0ULL, 0); - -static MYSQL_SYSVAR_UINT(client_pool_threads, tokudb_client_pool_threads, - PLUGIN_VAR_READONLY, "TokuDB client ops thread pool size", NULL, NULL, 0, - 0, 1024, 0); - -static MYSQL_SYSVAR_UINT(cachetable_pool_threads, tokudb_cachetable_pool_threads, - PLUGIN_VAR_READONLY, "TokuDB cachetable ops thread pool size", NULL, NULL, 0, - 0, 1024, 0); - -static MYSQL_SYSVAR_UINT(checkpoint_pool_threads, tokudb_checkpoint_pool_threads, - PLUGIN_VAR_READONLY, "TokuDB checkpoint ops thread pool size", NULL, NULL, 0, - 0, 1024, 0); - -static void tokudb_enable_partial_eviction_update(THD* thd, - struct st_mysql_sys_var* sys_var, - void* var, const void* save) { - my_bool * enable_partial_eviction = (my_bool *) var; - *enable_partial_eviction = *(const my_bool *) save; - int r = db_env->evictor_set_enable_partial_eviction(db_env, *enable_partial_eviction); - assert(r == 0); -} - -static MYSQL_SYSVAR_BOOL(enable_partial_eviction, tokudb_enable_partial_eviction, - 0, "TokuDB enable partial node eviction", - NULL, tokudb_enable_partial_eviction_update, TRUE); - -static MYSQL_SYSVAR_ULONG(debug, tokudb_debug, - 0, "TokuDB Debug", - NULL, NULL, 0, - 0, ~0UL, 0); - -static MYSQL_SYSVAR_STR(log_dir, tokudb_log_dir, - PLUGIN_VAR_READONLY, "TokuDB Log Directory", - NULL, NULL, NULL); - -static MYSQL_SYSVAR_STR(data_dir, tokudb_data_dir, - PLUGIN_VAR_READONLY, "TokuDB Data Directory", - NULL, NULL, NULL); - -static MYSQL_SYSVAR_STR(version, tokudb_version, - PLUGIN_VAR_READONLY, "TokuDB Version", - NULL, NULL, NULL); - -static MYSQL_SYSVAR_UINT(write_status_frequency, tokudb_write_status_frequency, - 0, "TokuDB frequency that show processlist updates status of writes", - NULL, NULL, 1000, - 0, ~0U, 0); - -static MYSQL_SYSVAR_UINT(read_status_frequency, tokudb_read_status_frequency, - 0, "TokuDB frequency that show processlist updates status of reads", - NULL, NULL, 10000, - 0, ~0U, 0); - -static MYSQL_SYSVAR_INT(fs_reserve_percent, tokudb_fs_reserve_percent, - PLUGIN_VAR_READONLY, "TokuDB file system space reserve (percent free required)", - NULL, NULL, 5, - 0, 100, 0); - -static MYSQL_SYSVAR_STR(tmp_dir, tokudb_tmp_dir, - PLUGIN_VAR_READONLY, "Tokudb Tmp Dir", - NULL, NULL, NULL); - -#if TOKU_INCLUDE_HANDLERTON_HANDLE_FATAL_SIGNAL -static MYSQL_SYSVAR_STR(gdb_path, tokudb_gdb_path, - PLUGIN_VAR_READONLY|PLUGIN_VAR_RQCMDARG, "TokuDB path to gdb for extra debug info on fatal signal", - NULL, NULL, "/usr/bin/gdb"); - -static MYSQL_SYSVAR_BOOL(gdb_on_fatal, tokudb_gdb_on_fatal, - 0, "TokuDB enable gdb debug info on fatal signal", - NULL, NULL, true); -#endif - -static void tokudb_fsync_log_period_update(THD* thd, - struct st_mysql_sys_var* sys_var, - void* var, const void* save) { - uint32 *period = (uint32 *) var; - *period = *(const ulonglong *) save; - db_env->change_fsync_log_period(db_env, *period); -} - -static MYSQL_SYSVAR_UINT(fsync_log_period, tokudb_fsync_log_period, - 0, "TokuDB fsync log period", - NULL, tokudb_fsync_log_period_update, 0, - 0, ~0U, 0); - -static struct st_mysql_sys_var *tokudb_system_variables[] = { - MYSQL_SYSVAR(cache_size), - MYSQL_SYSVAR(client_pool_threads), - MYSQL_SYSVAR(cachetable_pool_threads), - MYSQL_SYSVAR(checkpoint_pool_threads), - MYSQL_SYSVAR(max_lock_memory), - MYSQL_SYSVAR(enable_partial_eviction), - MYSQL_SYSVAR(data_dir), - MYSQL_SYSVAR(log_dir), - MYSQL_SYSVAR(debug), - MYSQL_SYSVAR(commit_sync), - MYSQL_SYSVAR(lock_timeout), - MYSQL_SYSVAR(cleaner_period), - MYSQL_SYSVAR(cleaner_iterations), - MYSQL_SYSVAR(pk_insert_mode), - MYSQL_SYSVAR(load_save_space), - MYSQL_SYSVAR(disable_slow_alter), - MYSQL_SYSVAR(disable_hot_alter), - MYSQL_SYSVAR(alter_print_error), - MYSQL_SYSVAR(create_index_online), - MYSQL_SYSVAR(disable_prefetching), - MYSQL_SYSVAR(version), - MYSQL_SYSVAR(checkpointing_period), - MYSQL_SYSVAR(prelock_empty), - MYSQL_SYSVAR(checkpoint_lock), - MYSQL_SYSVAR(write_status_frequency), - MYSQL_SYSVAR(read_status_frequency), - MYSQL_SYSVAR(fs_reserve_percent), - MYSQL_SYSVAR(tmp_dir), - MYSQL_SYSVAR(block_size), - MYSQL_SYSVAR(read_block_size), - MYSQL_SYSVAR(read_buf_size), - MYSQL_SYSVAR(fanout), - MYSQL_SYSVAR(row_format), - MYSQL_SYSVAR(directio), - MYSQL_SYSVAR(compress_buffers_before_eviction), - MYSQL_SYSVAR(checkpoint_on_flush_logs), -#if TOKU_INCLUDE_UPSERT - MYSQL_SYSVAR(disable_slow_update), - MYSQL_SYSVAR(disable_slow_upsert), -#endif - MYSQL_SYSVAR(analyze_time), - MYSQL_SYSVAR(analyze_delete_fraction), - MYSQL_SYSVAR(fsync_log_period), -#if TOKU_INCLUDE_HANDLERTON_HANDLE_FATAL_SIGNAL - MYSQL_SYSVAR(gdb_path), - MYSQL_SYSVAR(gdb_on_fatal), -#endif - MYSQL_SYSVAR(last_lock_timeout), - MYSQL_SYSVAR(lock_timeout_debug), - MYSQL_SYSVAR(loader_memory_size), - MYSQL_SYSVAR(hide_default_row_format), - MYSQL_SYSVAR(killed_time), - MYSQL_SYSVAR(empty_scan), -#if TOKUDB_CHECK_JEMALLOC - MYSQL_SYSVAR(check_jemalloc), -#endif - MYSQL_SYSVAR(bulk_fetch), -#if TOKU_INCLUDE_XA - MYSQL_SYSVAR(support_xa), -#endif - MYSQL_SYSVAR(rpl_unique_checks), - MYSQL_SYSVAR(rpl_unique_checks_delay), - MYSQL_SYSVAR(rpl_lookup_rows), - MYSQL_SYSVAR(rpl_lookup_rows_delay), - MYSQL_SYSVAR(rpl_check_readonly), - MYSQL_SYSVAR(optimize_index_name), - MYSQL_SYSVAR(optimize_index_fraction), - MYSQL_SYSVAR(optimize_throttle), - NULL -}; - // Split ./database/table-dictionary into database, table and dictionary strings -static void tokudb_split_dname(const char *dname, String &database_name, String &table_name, String &dictionary_name) { +void tokudb_split_dname( + const char* dname, + String& database_name, + String& table_name, + String& dictionary_name) { + const char *splitter = strchr(dname, '/'); if (splitter) { const char *database_ptr = splitter+1; @@ -1550,482 +1528,18 @@ static void tokudb_split_dname(const char *dname, String &database_name, String table_name.append(table_ptr, dictionary_ptr - table_ptr); dictionary_ptr += 1; dictionary_name.append(dictionary_ptr); + } else { + table_name.append(table_ptr); } - } - } -} - -struct st_mysql_storage_engine tokudb_storage_engine = { MYSQL_HANDLERTON_INTERFACE_VERSION }; - -static struct st_mysql_information_schema tokudb_file_map_information_schema = { MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION }; - -static ST_FIELD_INFO tokudb_file_map_field_info[] = { - {"dictionary_name", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, - {"internal_file_name", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, - {"table_schema", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, - {"table_name", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, - {"table_dictionary_name", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, - {NULL, 0, MYSQL_TYPE_NULL, 0, 0, NULL, SKIP_OPEN_TABLE} -}; - -static int tokudb_file_map(TABLE *table, THD *thd) { - int error; - DB_TXN* txn = NULL; - DBC* tmp_cursor = NULL; - DBT curr_key; - DBT curr_val; - memset(&curr_key, 0, sizeof curr_key); - memset(&curr_val, 0, sizeof curr_val); - error = txn_begin(db_env, 0, &txn, DB_READ_UNCOMMITTED, thd); - if (error) { - goto cleanup; - } - error = db_env->get_cursor_for_directory(db_env, txn, &tmp_cursor); - if (error) { - goto cleanup; - } - while (error == 0) { - error = tmp_cursor->c_get(tmp_cursor, &curr_key, &curr_val, DB_NEXT); - if (!error) { - // We store the NULL terminator in the directory so it's included in the size. - // See #5789 - // Recalculate and check just to be safe. - const char *dname = (const char *) curr_key.data; - size_t dname_len = strlen(dname); - assert(dname_len == curr_key.size - 1); - table->field[0]->store(dname, dname_len, system_charset_info); - - const char *iname = (const char *) curr_val.data; - size_t iname_len = strlen(iname); - assert(iname_len == curr_val.size - 1); - table->field[1]->store(iname, iname_len, system_charset_info); - - // split the dname - String database_name, table_name, dictionary_name; - tokudb_split_dname(dname, database_name, table_name, dictionary_name); - table->field[2]->store(database_name.c_ptr(), database_name.length(), system_charset_info); - table->field[3]->store(table_name.c_ptr(), table_name.length(), system_charset_info); - table->field[4]->store(dictionary_name.c_ptr(), dictionary_name.length(), system_charset_info); - - error = schema_table_store_record(thd, table); - } - if (!error && thd_killed(thd)) - error = ER_QUERY_INTERRUPTED; - } - if (error == DB_NOTFOUND) { - error = 0; - } -cleanup: - if (tmp_cursor) { - int r = tmp_cursor->c_close(tmp_cursor); - assert(r == 0); - } - if (txn) { - commit_txn(txn, 0); - } - return error; -} - -#if MYSQL_VERSION_ID >= 50600 -static int tokudb_file_map_fill_table(THD *thd, TABLE_LIST *tables, Item *cond) { -#else -static int tokudb_file_map_fill_table(THD *thd, TABLE_LIST *tables, COND *cond) { -#endif - TOKUDB_DBUG_ENTER(""); - int error; - TABLE *table = tables->table; - - rw_rdlock(&tokudb_hton_initialized_lock); - - if (!tokudb_hton_initialized) { - error = ER_PLUGIN_IS_NOT_LOADED; - my_error(error, MYF(0), tokudb_hton_name); - } else { - error = tokudb_file_map(table, thd); - if (error) - my_error(ER_GET_ERRNO, MYF(0), error, tokudb_hton_name); - } - - rw_unlock(&tokudb_hton_initialized_lock); - TOKUDB_DBUG_RETURN(error); -} - -static int tokudb_file_map_init(void *p) { - ST_SCHEMA_TABLE *schema = (ST_SCHEMA_TABLE *) p; - schema->fields_info = tokudb_file_map_field_info; - schema->fill_table = tokudb_file_map_fill_table; - return 0; -} - -static int tokudb_file_map_done(void *p) { - return 0; -} - -static struct st_mysql_information_schema tokudb_fractal_tree_info_information_schema = { MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION }; - -static ST_FIELD_INFO tokudb_fractal_tree_info_field_info[] = { - {"dictionary_name", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, - {"internal_file_name", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, - {"bt_num_blocks_allocated", 0, MYSQL_TYPE_LONGLONG, 0, 0, NULL, SKIP_OPEN_TABLE }, - {"bt_num_blocks_in_use", 0, MYSQL_TYPE_LONGLONG, 0, 0, NULL, SKIP_OPEN_TABLE }, - {"bt_size_allocated", 0, MYSQL_TYPE_LONGLONG, 0, 0, NULL, SKIP_OPEN_TABLE }, - {"bt_size_in_use", 0, MYSQL_TYPE_LONGLONG, 0, 0, NULL, SKIP_OPEN_TABLE }, - {"table_schema", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, - {"table_name", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, - {"table_dictionary_name", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, - {NULL, 0, MYSQL_TYPE_NULL, 0, 0, NULL, SKIP_OPEN_TABLE} -}; - -static int tokudb_report_fractal_tree_info_for_db(const DBT *dname, const DBT *iname, TABLE *table, THD *thd) { - int error; - uint64_t bt_num_blocks_allocated; - uint64_t bt_num_blocks_in_use; - uint64_t bt_size_allocated; - uint64_t bt_size_in_use; - - DB *db = NULL; - error = db_create(&db, db_env, 0); - if (error) { - goto exit; - } - error = db->open(db, NULL, (char *)dname->data, NULL, DB_BTREE, 0, 0666); - if (error) { - goto exit; - } - error = db->get_fractal_tree_info64(db, - &bt_num_blocks_allocated, &bt_num_blocks_in_use, - &bt_size_allocated, &bt_size_in_use); - if (error) { - goto exit; - } - - // We store the NULL terminator in the directory so it's included in the size. - // See #5789 - // Recalculate and check just to be safe. - { - size_t dname_len = strlen((const char *)dname->data); - assert(dname_len == dname->size - 1); - table->field[0]->store((char *)dname->data, dname_len, system_charset_info); - size_t iname_len = strlen((const char *)iname->data); - assert(iname_len == iname->size - 1); - table->field[1]->store((char *)iname->data, iname_len, system_charset_info); - } - table->field[2]->store(bt_num_blocks_allocated, false); - table->field[3]->store(bt_num_blocks_in_use, false); - table->field[4]->store(bt_size_allocated, false); - table->field[5]->store(bt_size_in_use, false); - - // split the dname - { - String database_name, table_name, dictionary_name; - tokudb_split_dname((const char *)dname->data, database_name, table_name, dictionary_name); - table->field[6]->store(database_name.c_ptr(), database_name.length(), system_charset_info); - table->field[7]->store(table_name.c_ptr(), table_name.length(), system_charset_info); - table->field[8]->store(dictionary_name.c_ptr(), dictionary_name.length(), system_charset_info); - } - error = schema_table_store_record(thd, table); - -exit: - if (db) { - int close_error = db->close(db, 0); - if (error == 0) - error = close_error; - } - return error; -} - -static int tokudb_fractal_tree_info(TABLE *table, THD *thd) { - int error; - DB_TXN* txn = NULL; - DBC* tmp_cursor = NULL; - DBT curr_key; - DBT curr_val; - memset(&curr_key, 0, sizeof curr_key); - memset(&curr_val, 0, sizeof curr_val); - error = txn_begin(db_env, 0, &txn, DB_READ_UNCOMMITTED, thd); - if (error) { - goto cleanup; - } - error = db_env->get_cursor_for_directory(db_env, txn, &tmp_cursor); - if (error) { - goto cleanup; - } - while (error == 0) { - error = tmp_cursor->c_get(tmp_cursor, &curr_key, &curr_val, DB_NEXT); - if (!error) { - error = tokudb_report_fractal_tree_info_for_db(&curr_key, &curr_val, table, thd); - if (error) - error = 0; // ignore read uncommitted errors - } - if (!error && thd_killed(thd)) - error = ER_QUERY_INTERRUPTED; - } - if (error == DB_NOTFOUND) { - error = 0; - } -cleanup: - if (tmp_cursor) { - int r = tmp_cursor->c_close(tmp_cursor); - assert(r == 0); - } - if (txn) { - commit_txn(txn, 0); - } - return error; -} - -#if MYSQL_VERSION_ID >= 50600 -static int tokudb_fractal_tree_info_fill_table(THD *thd, TABLE_LIST *tables, Item *cond) { -#else -static int tokudb_fractal_tree_info_fill_table(THD *thd, TABLE_LIST *tables, COND *cond) { -#endif - TOKUDB_DBUG_ENTER(""); - int error; - TABLE *table = tables->table; - - // 3938: Get a read lock on the status flag, since we must - // read it before safely proceeding - rw_rdlock(&tokudb_hton_initialized_lock); - - if (!tokudb_hton_initialized) { - error = ER_PLUGIN_IS_NOT_LOADED; - my_error(error, MYF(0), tokudb_hton_name); - } else { - error = tokudb_fractal_tree_info(table, thd); - if (error) - my_error(ER_GET_ERRNO, MYF(0), error, tokudb_hton_name); - } - - //3938: unlock the status flag lock - rw_unlock(&tokudb_hton_initialized_lock); - TOKUDB_DBUG_RETURN(error); -} - -static int tokudb_fractal_tree_info_init(void *p) { - ST_SCHEMA_TABLE *schema = (ST_SCHEMA_TABLE *) p; - schema->fields_info = tokudb_fractal_tree_info_field_info; - schema->fill_table = tokudb_fractal_tree_info_fill_table; - return 0; -} - -static int tokudb_fractal_tree_info_done(void *p) { - return 0; -} - -static struct st_mysql_information_schema tokudb_fractal_tree_block_map_information_schema = { MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION }; - -static ST_FIELD_INFO tokudb_fractal_tree_block_map_field_info[] = { - {"dictionary_name", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, - {"internal_file_name", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, - {"checkpoint_count", 0, MYSQL_TYPE_LONGLONG, 0, 0, NULL, SKIP_OPEN_TABLE }, - {"blocknum", 0, MYSQL_TYPE_LONGLONG, 0, 0, NULL, SKIP_OPEN_TABLE }, - {"offset", 0, MYSQL_TYPE_LONGLONG, 0, MY_I_S_MAYBE_NULL, NULL, SKIP_OPEN_TABLE }, - {"size", 0, MYSQL_TYPE_LONGLONG, 0, MY_I_S_MAYBE_NULL, NULL, SKIP_OPEN_TABLE }, - {"table_schema", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, - {"table_name", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, - {"table_dictionary_name", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, - {NULL, 0, MYSQL_TYPE_NULL, 0, 0, NULL, SKIP_OPEN_TABLE} -}; - -struct tokudb_report_fractal_tree_block_map_iterator_extra { - int64_t num_rows; - int64_t i; - uint64_t *checkpoint_counts; - int64_t *blocknums; - int64_t *diskoffs; - int64_t *sizes; -}; - -// This iterator is called while holding the blocktable lock. We should be as quick as possible. -// We don't want to do one call to get the number of rows, release the blocktable lock, and then do another call to get all the rows because the number of rows may change if we don't hold the lock. -// As a compromise, we'll do some mallocs inside the lock on the first call, but everything else should be fast. -static int tokudb_report_fractal_tree_block_map_iterator(uint64_t checkpoint_count, - int64_t num_rows, - int64_t blocknum, - int64_t diskoff, - int64_t size, - void *iter_extra) { - struct tokudb_report_fractal_tree_block_map_iterator_extra *e = static_cast<struct tokudb_report_fractal_tree_block_map_iterator_extra *>(iter_extra); - - assert(num_rows > 0); - if (e->num_rows == 0) { - e->checkpoint_counts = (uint64_t *) tokudb_my_malloc(num_rows * (sizeof *e->checkpoint_counts), MYF(MY_WME|MY_ZEROFILL|MY_FAE)); - e->blocknums = (int64_t *) tokudb_my_malloc(num_rows * (sizeof *e->blocknums), MYF(MY_WME|MY_ZEROFILL|MY_FAE)); - e->diskoffs = (int64_t *) tokudb_my_malloc(num_rows * (sizeof *e->diskoffs), MYF(MY_WME|MY_ZEROFILL|MY_FAE)); - e->sizes = (int64_t *) tokudb_my_malloc(num_rows * (sizeof *e->sizes), MYF(MY_WME|MY_ZEROFILL|MY_FAE)); - e->num_rows = num_rows; - } - - e->checkpoint_counts[e->i] = checkpoint_count; - e->blocknums[e->i] = blocknum; - e->diskoffs[e->i] = diskoff; - e->sizes[e->i] = size; - ++(e->i); - - return 0; -} - -static int tokudb_report_fractal_tree_block_map_for_db(const DBT *dname, const DBT *iname, TABLE *table, THD *thd) { - int error; - DB *db; - struct tokudb_report_fractal_tree_block_map_iterator_extra e = {}; // avoid struct initializers so that we can compile with older gcc versions - - error = db_create(&db, db_env, 0); - if (error) { - goto exit; - } - error = db->open(db, NULL, (char *)dname->data, NULL, DB_BTREE, 0, 0666); - if (error) { - goto exit; - } - error = db->iterate_fractal_tree_block_map(db, tokudb_report_fractal_tree_block_map_iterator, &e); - { - int close_error = db->close(db, 0); - if (!error) { - error = close_error; - } - } - if (error) { - goto exit; - } - - // If not, we should have gotten an error and skipped this section of code - assert(e.i == e.num_rows); - for (int64_t i = 0; error == 0 && i < e.num_rows; ++i) { - // We store the NULL terminator in the directory so it's included in the size. - // See #5789 - // Recalculate and check just to be safe. - size_t dname_len = strlen((const char *)dname->data); - assert(dname_len == dname->size - 1); - table->field[0]->store((char *)dname->data, dname_len, system_charset_info); - - size_t iname_len = strlen((const char *)iname->data); - assert(iname_len == iname->size - 1); - table->field[1]->store((char *)iname->data, iname_len, system_charset_info); - - table->field[2]->store(e.checkpoint_counts[i], false); - table->field[3]->store(e.blocknums[i], false); - static const int64_t freelist_null = -1; - static const int64_t diskoff_unused = -2; - if (e.diskoffs[i] == diskoff_unused || e.diskoffs[i] == freelist_null) { - table->field[4]->set_null(); } else { - table->field[4]->set_notnull(); - table->field[4]->store(e.diskoffs[i], false); + database_name.append(database_ptr); } - static const int64_t size_is_free = -1; - if (e.sizes[i] == size_is_free) { - table->field[5]->set_null(); - } else { - table->field[5]->set_notnull(); - table->field[5]->store(e.sizes[i], false); - } - - // split the dname - String database_name, table_name, dictionary_name; - tokudb_split_dname((const char *)dname->data, database_name, table_name,dictionary_name); - table->field[6]->store(database_name.c_ptr(), database_name.length(), system_charset_info); - table->field[7]->store(table_name.c_ptr(), table_name.length(), system_charset_info); - table->field[8]->store(dictionary_name.c_ptr(), dictionary_name.length(), system_charset_info); - - error = schema_table_store_record(thd, table); - } - -exit: - if (e.checkpoint_counts != NULL) { - tokudb_my_free(e.checkpoint_counts); - e.checkpoint_counts = NULL; - } - if (e.blocknums != NULL) { - tokudb_my_free(e.blocknums); - e.blocknums = NULL; - } - if (e.diskoffs != NULL) { - tokudb_my_free(e.diskoffs); - e.diskoffs = NULL; } - if (e.sizes != NULL) { - tokudb_my_free(e.sizes); - e.sizes = NULL; - } - return error; -} - -static int tokudb_fractal_tree_block_map(TABLE *table, THD *thd) { - int error; - DB_TXN* txn = NULL; - DBC* tmp_cursor = NULL; - DBT curr_key; - DBT curr_val; - memset(&curr_key, 0, sizeof curr_key); - memset(&curr_val, 0, sizeof curr_val); - error = txn_begin(db_env, 0, &txn, DB_READ_UNCOMMITTED, thd); - if (error) { - goto cleanup; - } - error = db_env->get_cursor_for_directory(db_env, txn, &tmp_cursor); - if (error) { - goto cleanup; - } - while (error == 0) { - error = tmp_cursor->c_get(tmp_cursor, &curr_key, &curr_val, DB_NEXT); - if (!error) { - error = tokudb_report_fractal_tree_block_map_for_db(&curr_key, &curr_val, table, thd); - } - if (!error && thd_killed(thd)) - error = ER_QUERY_INTERRUPTED; - } - if (error == DB_NOTFOUND) { - error = 0; - } -cleanup: - if (tmp_cursor) { - int r = tmp_cursor->c_close(tmp_cursor); - assert(r == 0); - } - if (txn) { - commit_txn(txn, 0); - } - return error; -} - -#if MYSQL_VERSION_ID >= 50600 -static int tokudb_fractal_tree_block_map_fill_table(THD *thd, TABLE_LIST *tables, Item *cond) { -#else -static int tokudb_fractal_tree_block_map_fill_table(THD *thd, TABLE_LIST *tables, COND *cond) { -#endif - TOKUDB_DBUG_ENTER(""); - int error; - TABLE *table = tables->table; - - // 3938: Get a read lock on the status flag, since we must - // read it before safely proceeding - rw_rdlock(&tokudb_hton_initialized_lock); - - if (!tokudb_hton_initialized) { - error = ER_PLUGIN_IS_NOT_LOADED; - my_error(error, MYF(0), tokudb_hton_name); - } else { - error = tokudb_fractal_tree_block_map(table, thd); - if (error) - my_error(ER_GET_ERRNO, MYF(0), error, tokudb_hton_name); - } - - //3938: unlock the status flag lock - rw_unlock(&tokudb_hton_initialized_lock); - TOKUDB_DBUG_RETURN(error); -} - -static int tokudb_fractal_tree_block_map_init(void *p) { - ST_SCHEMA_TABLE *schema = (ST_SCHEMA_TABLE *) p; - schema->fields_info = tokudb_fractal_tree_block_map_field_info; - schema->fill_table = tokudb_fractal_tree_block_map_fill_table; - return 0; } -static int tokudb_fractal_tree_block_map_done(void *p) { - return 0; -} +struct st_mysql_storage_engine tokudb_storage_engine = { + MYSQL_HANDLERTON_INTERFACE_VERSION +}; #if TOKU_INCLUDE_LOCK_TIMEOUT_QUERY_STRING struct tokudb_search_txn_extra { @@ -2034,10 +1548,16 @@ struct tokudb_search_txn_extra { uint64_t match_client_id; }; -static int tokudb_search_txn_callback(DB_TXN *txn, iterate_row_locks_callback iterate_locks, void *locks_extra, void *extra) { +static int tokudb_search_txn_callback( + DB_TXN* txn, + iterate_row_locks_callback iterate_locks, + void* locks_extra, + void* extra) { + uint64_t txn_id = txn->id64(txn); uint64_t client_id = txn->get_client_id(txn); - struct tokudb_search_txn_extra *e = reinterpret_cast<struct tokudb_search_txn_extra *>(extra); + struct tokudb_search_txn_extra* e = + reinterpret_cast<struct tokudb_search_txn_extra*>(extra); if (e->match_txn_id == txn_id) { e->match_found = true; e->match_client_id = client_id; @@ -2046,9 +1566,17 @@ static int tokudb_search_txn_callback(DB_TXN *txn, iterate_row_locks_callback it return 0; } -static bool tokudb_txn_id_to_client_id(THD *thd, uint64_t blocking_txnid, uint64_t *blocking_client_id) { - struct tokudb_search_txn_extra e = { false, blocking_txnid, 0}; - (void) db_env->iterate_live_transactions(db_env, tokudb_search_txn_callback, &e); +static bool tokudb_txn_id_to_client_id( + THD* thd, + uint64_t blocking_txnid, + uint64_t* blocking_client_id) { + + struct tokudb_search_txn_extra e = { + false, + blocking_txnid, + 0 + }; + db_env->iterate_live_transactions(db_env, tokudb_search_txn_callback, &e); if (e.match_found) { *blocking_client_id = e.match_client_id; } @@ -2056,14 +1584,20 @@ static bool tokudb_txn_id_to_client_id(THD *thd, uint64_t blocking_txnid, uint64 } #endif -static void tokudb_pretty_key(const DB *db, const DBT *key, const char *default_key, String *out) { +static void tokudb_pretty_key( + const DB* db, + const DBT* key, + const char* default_key, + String* out) { + if (key->data == NULL) { out->append(default_key); } else { bool do_hexdump = true; if (do_hexdump) { // hexdump the key - const unsigned char *data = reinterpret_cast<const unsigned char *>(key->data); + const unsigned char* data = + reinterpret_cast<const unsigned char*>(key->data); for (size_t i = 0; i < key->size; i++) { char str[3]; snprintf(str, sizeof str, "%2.2x", data[i]); @@ -2073,15 +1607,15 @@ static void tokudb_pretty_key(const DB *db, const DBT *key, const char *default_ } } -static void tokudb_pretty_left_key(const DB *db, const DBT *key, String *out) { +void tokudb_pretty_left_key(const DB* db, const DBT* key, String* out) { tokudb_pretty_key(db, key, "-infinity", out); } -static void tokudb_pretty_right_key(const DB *db, const DBT *key, String *out) { +void tokudb_pretty_right_key(const DB* db, const DBT* key, String* out) { tokudb_pretty_key(db, key, "+infinity", out); } -static const char *tokudb_get_index_name(DB *db) { +const char* tokudb_get_index_name(DB* db) { if (db != NULL) { return db->get_dname(db); } else { @@ -2090,17 +1624,24 @@ static const char *tokudb_get_index_name(DB *db) { } static int tokudb_equal_key(const DBT *left_key, const DBT *right_key) { - if (left_key->data == NULL || right_key->data == NULL || left_key->size != right_key->size) + if (left_key->data == NULL || right_key->data == NULL || + left_key->size != right_key->size) return 0; else return memcmp(left_key->data, right_key->data, left_key->size) == 0; } -static void tokudb_lock_timeout_callback(DB *db, uint64_t requesting_txnid, const DBT *left_key, const DBT *right_key, uint64_t blocking_txnid) { - THD *thd = current_thd; +static void tokudb_lock_timeout_callback( + DB* db, + uint64_t requesting_txnid, + const DBT* left_key, + const DBT* right_key, + uint64_t blocking_txnid) { + + THD* thd = current_thd; if (!thd) return; - ulong lock_timeout_debug = THDVAR(thd, lock_timeout_debug); + ulong lock_timeout_debug = tokudb::sysvars::lock_timeout_debug(thd); if (lock_timeout_debug != 0) { // generate a JSON document with the lock timeout info String log_str; @@ -2109,7 +1650,9 @@ static void tokudb_lock_timeout_callback(DB *db, uint64_t requesting_txnid, cons log_str.append("\"mysql_thread_id\":"); log_str.append_ulonglong(mysql_thread_id); log_str.append(", \"dbname\":"); - log_str.append("\""); log_str.append(tokudb_get_index_name(db)); log_str.append("\""); + log_str.append("\""); + log_str.append(tokudb_get_index_name(db)); + log_str.append("\""); log_str.append(", \"requesting_txnid\":"); log_str.append_ulonglong(requesting_txnid); log_str.append(", \"blocking_txnid\":"); @@ -2118,44 +1661,71 @@ static void tokudb_lock_timeout_callback(DB *db, uint64_t requesting_txnid, cons String key_str; tokudb_pretty_key(db, left_key, "?", &key_str); log_str.append(", \"key\":"); - log_str.append("\""); log_str.append(key_str); log_str.append("\""); + log_str.append("\""); + log_str.append(key_str); + log_str.append("\""); } else { String left_str; tokudb_pretty_left_key(db, left_key, &left_str); log_str.append(", \"key_left\":"); - log_str.append("\""); log_str.append(left_str); log_str.append("\""); + log_str.append("\""); + log_str.append(left_str); + log_str.append("\""); String right_str; tokudb_pretty_right_key(db, right_key, &right_str); log_str.append(", \"key_right\":"); - log_str.append("\""); log_str.append(right_str); log_str.append("\""); + log_str.append("\""); + log_str.append(right_str); + log_str.append("\""); } log_str.append("}"); // set last_lock_timeout if (lock_timeout_debug & 1) { - char *old_lock_timeout = THDVAR(thd, last_lock_timeout); - char *new_lock_timeout = tokudb_my_strdup(log_str.c_ptr(), MY_FAE); - THDVAR(thd, last_lock_timeout) = new_lock_timeout; - tokudb_my_free(old_lock_timeout); + char* old_lock_timeout = tokudb::sysvars::last_lock_timeout(thd); + char* new_lock_timeout = + tokudb::memory::strdup(log_str.c_ptr(), MY_FAE); + tokudb::sysvars::set_last_lock_timeout(thd, new_lock_timeout); #if TOKU_THDVAR_MEMALLOC_BUG - tokudb_pthread_mutex_lock(&tokudb_map_mutex); + tokudb_map_mutex.lock(); struct tokudb_map_pair old_key = { thd, old_lock_timeout }; tree_delete(&tokudb_map, &old_key, sizeof old_key, NULL); struct tokudb_map_pair new_key = { thd, new_lock_timeout }; tree_insert(&tokudb_map, &new_key, sizeof new_key, NULL); - tokudb_pthread_mutex_unlock(&tokudb_map_mutex); + tokudb_map_mutex.unlock(); #endif + tokudb::memory::free(old_lock_timeout); } // dump to stderr if (lock_timeout_debug & 2) { - sql_print_error("%s: lock timeout %s", tokudb_hton_name, log_str.c_ptr()); + sql_print_error( + "%s: lock timeout %s", + tokudb_hton_name, + log_str.c_ptr()); LEX_STRING *qs = thd_query_string(thd); - sql_print_error("%s: requesting_thread_id:%" PRIu64 " q:%.*s", tokudb_hton_name, mysql_thread_id, (int) qs->length, qs->str); + sql_print_error( + "%s: requesting_thread_id:%" PRIu64 " q:%.*s", + tokudb_hton_name, + mysql_thread_id, + (int)qs->length, + qs->str); #if TOKU_INCLUDE_LOCK_TIMEOUT_QUERY_STRING uint64_t blocking_thread_id = 0; - if (tokudb_txn_id_to_client_id(thd, blocking_txnid, &blocking_thread_id)) { + if (tokudb_txn_id_to_client_id( + thd, + blocking_txnid, + &blocking_thread_id)) { + String blocking_qs; - if (get_thread_query_string(blocking_thread_id, blocking_qs) == 0) { - sql_print_error("%s: blocking_thread_id:%" PRIu64 " q:%.*s", tokudb_hton_name, blocking_thread_id, blocking_qs.length(), blocking_qs.c_ptr()); + if (get_thread_query_string( + blocking_thread_id, + blocking_qs) == 0) { + + sql_print_error( + "%s: blocking_thread_id:%" PRIu64 " q:%.*s", + tokudb_hton_name, + blocking_thread_id, + blocking_qs.length(), + blocking_qs.c_ptr()); } } #endif @@ -2163,254 +1733,9 @@ static void tokudb_lock_timeout_callback(DB *db, uint64_t requesting_txnid, cons } } -static struct st_mysql_information_schema tokudb_trx_information_schema = { MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION }; - -static ST_FIELD_INFO tokudb_trx_field_info[] = { - {"trx_id", 0, MYSQL_TYPE_LONGLONG, 0, 0, NULL, SKIP_OPEN_TABLE }, - {"trx_mysql_thread_id", 0, MYSQL_TYPE_LONGLONG, 0, 0, NULL, SKIP_OPEN_TABLE }, - {"trx_time", 0, MYSQL_TYPE_LONGLONG, 0, 0, NULL, SKIP_OPEN_TABLE }, - {NULL, 0, MYSQL_TYPE_NULL, 0, 0, NULL, SKIP_OPEN_TABLE} -}; - -struct tokudb_trx_extra { - THD *thd; - TABLE *table; -}; - -static int tokudb_trx_callback(DB_TXN *txn, iterate_row_locks_callback iterate_locks, void *locks_extra, void *extra) { - uint64_t txn_id = txn->id64(txn); - uint64_t client_id = txn->get_client_id(txn); - uint64_t start_time = txn->get_start_time(txn); - struct tokudb_trx_extra *e = reinterpret_cast<struct tokudb_trx_extra *>(extra); - THD *thd = e->thd; - TABLE *table = e->table; - table->field[0]->store(txn_id, false); - table->field[1]->store(client_id, false); - uint64_t tnow = (uint64_t) time(NULL); - table->field[2]->store(tnow >= start_time ? tnow - start_time : 0, false); - int error = schema_table_store_record(thd, table); - if (!error && thd_killed(thd)) - error = ER_QUERY_INTERRUPTED; - return error; -} - -#if MYSQL_VERSION_ID >= 50600 -static int tokudb_trx_fill_table(THD *thd, TABLE_LIST *tables, Item *cond) { -#else -static int tokudb_trx_fill_table(THD *thd, TABLE_LIST *tables, COND *cond) { -#endif - TOKUDB_DBUG_ENTER(""); - int error; - - rw_rdlock(&tokudb_hton_initialized_lock); - - if (!tokudb_hton_initialized) { - error = ER_PLUGIN_IS_NOT_LOADED; - my_error(error, MYF(0), tokudb_hton_name); - } else { - struct tokudb_trx_extra e = { thd, tables->table }; - error = db_env->iterate_live_transactions(db_env, tokudb_trx_callback, &e); - if (error) - my_error(ER_GET_ERRNO, MYF(0), error, tokudb_hton_name); - } - - rw_unlock(&tokudb_hton_initialized_lock); - TOKUDB_DBUG_RETURN(error); -} - -static int tokudb_trx_init(void *p) { - ST_SCHEMA_TABLE *schema = (ST_SCHEMA_TABLE *) p; - schema->fields_info = tokudb_trx_field_info; - schema->fill_table = tokudb_trx_fill_table; - return 0; -} - -static int tokudb_trx_done(void *p) { - return 0; -} - -static struct st_mysql_information_schema tokudb_lock_waits_information_schema = { MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION }; - -static ST_FIELD_INFO tokudb_lock_waits_field_info[] = { - {"requesting_trx_id", 0, MYSQL_TYPE_LONGLONG, 0, 0, NULL, SKIP_OPEN_TABLE }, - {"blocking_trx_id", 0, MYSQL_TYPE_LONGLONG, 0, 0, NULL, SKIP_OPEN_TABLE }, - {"lock_waits_dname", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, - {"lock_waits_key_left", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, - {"lock_waits_key_right", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, - {"lock_waits_start_time", 0, MYSQL_TYPE_LONGLONG, 0, 0, NULL, SKIP_OPEN_TABLE }, - {"lock_waits_table_schema", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, - {"lock_waits_table_name", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, - {"lock_waits_table_dictionary_name", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, - {NULL, 0, MYSQL_TYPE_NULL, 0, 0, NULL, SKIP_OPEN_TABLE} -}; - -struct tokudb_lock_waits_extra { - THD *thd; - TABLE *table; -}; - -static int tokudb_lock_waits_callback(DB *db, uint64_t requesting_txnid, const DBT *left_key, const DBT *right_key, - uint64_t blocking_txnid, uint64_t start_time, void *extra) { - struct tokudb_lock_waits_extra *e = reinterpret_cast<struct tokudb_lock_waits_extra *>(extra); - THD *thd = e->thd; - TABLE *table = e->table; - table->field[0]->store(requesting_txnid, false); - table->field[1]->store(blocking_txnid, false); - const char *dname = tokudb_get_index_name(db); - size_t dname_length = strlen(dname); - table->field[2]->store(dname, dname_length, system_charset_info); - String left_str; - tokudb_pretty_left_key(db, left_key, &left_str); - table->field[3]->store(left_str.ptr(), left_str.length(), system_charset_info); - String right_str; - tokudb_pretty_right_key(db, right_key, &right_str); - table->field[4]->store(right_str.ptr(), right_str.length(), system_charset_info); - table->field[5]->store(start_time, false); - - String database_name, table_name, dictionary_name; - tokudb_split_dname(dname, database_name, table_name, dictionary_name); - table->field[6]->store(database_name.c_ptr(), database_name.length(), system_charset_info); - table->field[7]->store(table_name.c_ptr(), table_name.length(), system_charset_info); - table->field[8]->store(dictionary_name.c_ptr(), dictionary_name.length(), system_charset_info); - - int error = schema_table_store_record(thd, table); - - if (!error && thd_killed(thd)) - error = ER_QUERY_INTERRUPTED; - - return error; -} - -#if MYSQL_VERSION_ID >= 50600 -static int tokudb_lock_waits_fill_table(THD *thd, TABLE_LIST *tables, Item *cond) { -#else -static int tokudb_lock_waits_fill_table(THD *thd, TABLE_LIST *tables, COND *cond) { -#endif - TOKUDB_DBUG_ENTER(""); - int error; - - rw_rdlock(&tokudb_hton_initialized_lock); - - if (!tokudb_hton_initialized) { - error = ER_PLUGIN_IS_NOT_LOADED; - my_error(error, MYF(0), tokudb_hton_name); - } else { - struct tokudb_lock_waits_extra e = { thd, tables->table }; - error = db_env->iterate_pending_lock_requests(db_env, tokudb_lock_waits_callback, &e); - if (error) - my_error(ER_GET_ERRNO, MYF(0), error, tokudb_hton_name); - } - - rw_unlock(&tokudb_hton_initialized_lock); - TOKUDB_DBUG_RETURN(error); -} - -static int tokudb_lock_waits_init(void *p) { - ST_SCHEMA_TABLE *schema = (ST_SCHEMA_TABLE *) p; - schema->fields_info = tokudb_lock_waits_field_info; - schema->fill_table = tokudb_lock_waits_fill_table; - return 0; -} - -static int tokudb_lock_waits_done(void *p) { - return 0; -} - -static struct st_mysql_information_schema tokudb_locks_information_schema = { MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION }; - -static ST_FIELD_INFO tokudb_locks_field_info[] = { - {"locks_trx_id", 0, MYSQL_TYPE_LONGLONG, 0, 0, NULL, SKIP_OPEN_TABLE }, - {"locks_mysql_thread_id", 0, MYSQL_TYPE_LONGLONG, 0, 0, NULL, SKIP_OPEN_TABLE }, - {"locks_dname", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, - {"locks_key_left", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, - {"locks_key_right", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, - {"locks_table_schema", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, - {"locks_table_name", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, - {"locks_table_dictionary_name", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, - {NULL, 0, MYSQL_TYPE_NULL, 0, 0, NULL, SKIP_OPEN_TABLE} -}; - -struct tokudb_locks_extra { - THD *thd; - TABLE *table; -}; - -static int tokudb_locks_callback(DB_TXN *txn, iterate_row_locks_callback iterate_locks, void *locks_extra, void *extra) { - uint64_t txn_id = txn->id64(txn); - uint64_t client_id = txn->get_client_id(txn); - struct tokudb_locks_extra *e = reinterpret_cast<struct tokudb_locks_extra *>(extra); - THD *thd = e->thd; - TABLE *table = e->table; - int error = 0; - DB *db; - DBT left_key, right_key; - while (error == 0 && iterate_locks(&db, &left_key, &right_key, locks_extra) == 0) { - table->field[0]->store(txn_id, false); - table->field[1]->store(client_id, false); - - const char *dname = tokudb_get_index_name(db); - size_t dname_length = strlen(dname); - table->field[2]->store(dname, dname_length, system_charset_info); - - String left_str; - tokudb_pretty_left_key(db, &left_key, &left_str); - table->field[3]->store(left_str.ptr(), left_str.length(), system_charset_info); - - String right_str; - tokudb_pretty_right_key(db, &right_key, &right_str); - table->field[4]->store(right_str.ptr(), right_str.length(), system_charset_info); - - String database_name, table_name, dictionary_name; - tokudb_split_dname(dname, database_name, table_name, dictionary_name); - table->field[5]->store(database_name.c_ptr(), database_name.length(), system_charset_info); - table->field[6]->store(table_name.c_ptr(), table_name.length(), system_charset_info); - table->field[7]->store(dictionary_name.c_ptr(), dictionary_name.length(), system_charset_info); - - error = schema_table_store_record(thd, table); - - if (!error && thd_killed(thd)) - error = ER_QUERY_INTERRUPTED; - } - return error; -} - -#if MYSQL_VERSION_ID >= 50600 -static int tokudb_locks_fill_table(THD *thd, TABLE_LIST *tables, Item *cond) { -#else -static int tokudb_locks_fill_table(THD *thd, TABLE_LIST *tables, COND *cond) { -#endif - TOKUDB_DBUG_ENTER(""); - int error; - - rw_rdlock(&tokudb_hton_initialized_lock); - - if (!tokudb_hton_initialized) { - error = ER_PLUGIN_IS_NOT_LOADED; - my_error(error, MYF(0), tokudb_hton_name); - } else { - struct tokudb_locks_extra e = { thd, tables->table }; - error = db_env->iterate_live_transactions(db_env, tokudb_locks_callback, &e); - if (error) - my_error(ER_GET_ERRNO, MYF(0), error, tokudb_hton_name); - } - - rw_unlock(&tokudb_hton_initialized_lock); - TOKUDB_DBUG_RETURN(error); -} - -static int tokudb_locks_init(void *p) { - ST_SCHEMA_TABLE *schema = (ST_SCHEMA_TABLE *) p; - schema->fields_info = tokudb_locks_field_info; - schema->fill_table = tokudb_locks_fill_table; - return 0; -} - -static int tokudb_locks_done(void *p) { - return 0; -} - // Retrieves variables for information_schema.global_status. -// Names (columnname) are automatically converted to upper case, and prefixed with "TOKUDB_" +// Names (columnname) are automatically converted to upper case, +// and prefixed with "TOKUDB_" static int show_tokudb_vars(THD *thd, SHOW_VAR *var, char *buff) { TOKUDB_DBUG_ENTER(""); @@ -2421,13 +1746,23 @@ static int show_tokudb_vars(THD *thd, SHOW_VAR *var, char *buff) { fs_redzone_state redzone_state; uint64_t num_rows; - error = db_env->get_engine_status (db_env, toku_global_status_rows, toku_global_status_max_rows, &num_rows, &redzone_state, &panic, panic_string, panic_string_len, TOKU_GLOBAL_STATUS); + error = db_env->get_engine_status( + db_env, + toku_global_status_rows, + toku_global_status_max_rows, + &num_rows, + &redzone_state, + &panic, + panic_string, + panic_string_len, + TOKU_GLOBAL_STATUS); //TODO: Maybe do something with the panic output? if (error == 0) { - assert(num_rows <= toku_global_status_max_rows); + assert_always(num_rows <= toku_global_status_max_rows); //TODO: Maybe enable some of the items here: (copied from engine status - //TODO: (optionally) add redzone state, panic, panic string, etc. Right now it's being ignored. + //TODO: (optionally) add redzone state, panic, panic string, etc. + //Right now it's being ignored. for (uint64_t row = 0; row < num_rows; row++) { SHOW_VAR &status_var = toku_global_status_variables[row]; @@ -2449,7 +1784,11 @@ static int show_tokudb_vars(THD *thd, SHOW_VAR *var, char *buff) { time_t t = status_row.value.num; char tbuf[26]; // Reuse the memory in status_row. (It belongs to us). - snprintf(status_row.value.datebuf, sizeof(status_row.value.datebuf), "%.24s", ctime_r(&t, tbuf)); + snprintf( + status_row.value.datebuf, + sizeof(status_row.value.datebuf), + "%.24s", + ctime_r(&t, tbuf)); status_var.value = (char*)&status_row.value.datebuf[0]; break; } @@ -2475,7 +1814,11 @@ static int show_tokudb_vars(THD *thd, SHOW_VAR *var, char *buff) { status_var.type = SHOW_CHAR; // Reuse the memory in status_row.datebuf. (It belongs to us). // UNKNOWN TYPE: %d fits in 26 bytes (sizeof datebuf) for any integer. - snprintf(status_row.value.datebuf, sizeof(status_row.value.datebuf), "UNKNOWN TYPE: %d", status_row.type); + snprintf( + status_row.value.datebuf, + sizeof(status_row.value.datebuf), + "UNKNOWN TYPE: %d", + status_row.type); status_var.value = (char*)&status_row.value.datebuf[0]; break; } @@ -2507,157 +1850,38 @@ static void tokudb_backtrace(void) { } #endif -#if defined(TOKUDB_VERSION_MAJOR) && defined(TOKUDB_VERSION_MINOR) -#define TOKUDB_PLUGIN_VERSION ((TOKUDB_VERSION_MAJOR << 8) + TOKUDB_VERSION_MINOR) -#else -#define TOKUDB_PLUGIN_VERSION 0 -#endif - #ifdef MARIA_PLUGIN_INTERFACE_VERSION maria_declare_plugin(tokudb) #else mysql_declare_plugin(tokudb) #endif -{ - MYSQL_STORAGE_ENGINE_PLUGIN, - &tokudb_storage_engine, - tokudb_hton_name, - "Percona", - "Percona TokuDB Storage Engine with Fractal Tree(tm) Technology", - PLUGIN_LICENSE_GPL, - tokudb_init_func, /* plugin init */ - tokudb_done_func, /* plugin deinit */ - TOKUDB_PLUGIN_VERSION, - toku_global_status_variables_export, /* status variables */ - tokudb_system_variables, /* system variables */ -#ifdef MARIA_PLUGIN_INTERFACE_VERSION - tokudb_version, - MariaDB_PLUGIN_MATURITY_STABLE /* maturity */ -#else - NULL, /* config options */ - 0, /* flags */ -#endif -}, -{ - MYSQL_INFORMATION_SCHEMA_PLUGIN, - &tokudb_trx_information_schema, - "TokuDB_trx", - "Percona", - "Percona TokuDB Storage Engine with Fractal Tree(tm) Technology", - PLUGIN_LICENSE_GPL, - tokudb_trx_init, /* plugin init */ - tokudb_trx_done, /* plugin deinit */ - TOKUDB_PLUGIN_VERSION, - NULL, /* status variables */ - NULL, /* system variables */ -#ifdef MARIA_PLUGIN_INTERFACE_VERSION - tokudb_version, - MariaDB_PLUGIN_MATURITY_STABLE /* maturity */ -#else - NULL, /* config options */ - 0, /* flags */ -#endif -}, -{ - MYSQL_INFORMATION_SCHEMA_PLUGIN, - &tokudb_lock_waits_information_schema, - "TokuDB_lock_waits", - "Percona", - "Percona TokuDB Storage Engine with Fractal Tree(tm) Technology", - PLUGIN_LICENSE_GPL, - tokudb_lock_waits_init, /* plugin init */ - tokudb_lock_waits_done, /* plugin deinit */ - TOKUDB_PLUGIN_VERSION, - NULL, /* status variables */ - NULL, /* system variables */ -#ifdef MARIA_PLUGIN_INTERFACE_VERSION - tokudb_version, - MariaDB_PLUGIN_MATURITY_STABLE /* maturity */ -#else - NULL, /* config options */ - 0, /* flags */ -#endif -}, -{ - MYSQL_INFORMATION_SCHEMA_PLUGIN, - &tokudb_locks_information_schema, - "TokuDB_locks", - "Percona", - "Percona TokuDB Storage Engine with Fractal Tree(tm) Technology", - PLUGIN_LICENSE_GPL, - tokudb_locks_init, /* plugin init */ - tokudb_locks_done, /* plugin deinit */ - TOKUDB_PLUGIN_VERSION, - NULL, /* status variables */ - NULL, /* system variables */ -#ifdef MARIA_PLUGIN_INTERFACE_VERSION - tokudb_version, - MariaDB_PLUGIN_MATURITY_STABLE /* maturity */ -#else - NULL, /* config options */ - 0, /* flags */ -#endif -}, -{ - MYSQL_INFORMATION_SCHEMA_PLUGIN, - &tokudb_file_map_information_schema, - "TokuDB_file_map", - "Percona", - "Percona TokuDB Storage Engine with Fractal Tree(tm) Technology", - PLUGIN_LICENSE_GPL, - tokudb_file_map_init, /* plugin init */ - tokudb_file_map_done, /* plugin deinit */ - TOKUDB_PLUGIN_VERSION, - NULL, /* status variables */ - NULL, /* system variables */ -#ifdef MARIA_PLUGIN_INTERFACE_VERSION - tokudb_version, - MariaDB_PLUGIN_MATURITY_STABLE /* maturity */ -#else - NULL, /* config options */ - 0, /* flags */ -#endif -}, -{ - MYSQL_INFORMATION_SCHEMA_PLUGIN, - &tokudb_fractal_tree_info_information_schema, - "TokuDB_fractal_tree_info", - "Percona", - "Percona TokuDB Storage Engine with Fractal Tree(tm) Technology", - PLUGIN_LICENSE_GPL, - tokudb_fractal_tree_info_init, /* plugin init */ - tokudb_fractal_tree_info_done, /* plugin deinit */ - TOKUDB_PLUGIN_VERSION, - NULL, /* status variables */ - NULL, /* system variables */ -#ifdef MARIA_PLUGIN_INTERFACE_VERSION - tokudb_version, - MariaDB_PLUGIN_MATURITY_STABLE /* maturity */ -#else - NULL, /* config options */ - 0, /* flags */ -#endif -}, -{ - MYSQL_INFORMATION_SCHEMA_PLUGIN, - &tokudb_fractal_tree_block_map_information_schema, - "TokuDB_fractal_tree_block_map", - "Percona", - "Percona TokuDB Storage Engine with Fractal Tree(tm) Technology", - PLUGIN_LICENSE_GPL, - tokudb_fractal_tree_block_map_init, /* plugin init */ - tokudb_fractal_tree_block_map_done, /* plugin deinit */ - TOKUDB_PLUGIN_VERSION, - NULL, /* status variables */ - NULL, /* system variables */ + { + MYSQL_STORAGE_ENGINE_PLUGIN, + &tokudb_storage_engine, + tokudb_hton_name, + "Percona", + "Percona TokuDB Storage Engine with Fractal Tree(tm) Technology", + PLUGIN_LICENSE_GPL, + tokudb_init_func, /* plugin init */ + tokudb_done_func, /* plugin deinit */ + TOKUDB_PLUGIN_VERSION, + toku_global_status_variables_export, /* status variables */ + tokudb::sysvars::system_variables, /* system variables */ #ifdef MARIA_PLUGIN_INTERFACE_VERSION - tokudb_version, - MariaDB_PLUGIN_MATURITY_STABLE /* maturity */ + tokudb::sysvars::version, + MariaDB_PLUGIN_MATURITY_STABLE /* maturity */ #else - NULL, /* config options */ - 0, /* flags */ + NULL, /* config options */ + 0, /* flags */ #endif -} + }, + tokudb::information_schema::trx, + tokudb::information_schema::lock_waits, + tokudb::information_schema::locks, + tokudb::information_schema::file_map, + tokudb::information_schema::fractal_tree_info, + tokudb::information_schema::fractal_tree_block_map, + tokudb::information_schema::background_job_status #ifdef MARIA_PLUGIN_INTERFACE_VERSION maria_declare_plugin_end; #else diff --git a/storage/tokudb/hatoku_hton.h b/storage/tokudb/hatoku_hton.h index 6b2cbbe2383..37c61be849d 100644 --- a/storage/tokudb/hatoku_hton.h +++ b/storage/tokudb/hatoku_hton.h @@ -26,492 +26,187 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #ifndef _HATOKU_HTON_H #define _HATOKU_HTON_H -#include "db.h" +#include "hatoku_defines.h" +#include "tokudb_debug.h" +#include "tokudb_information_schema.h" +#include "tokudb_memory.h" +#include "tokudb_thread.h" +#include "tokudb_time.h" +#include "tokudb_txn.h" +#include "tokudb_sysvars.h" -extern handlerton *tokudb_hton; +extern handlerton* tokudb_hton; -extern DB_ENV *db_env; +extern DB_ENV* db_env; -enum srv_row_format_enum { - SRV_ROW_FORMAT_UNCOMPRESSED = 0, - SRV_ROW_FORMAT_ZLIB = 1, - SRV_ROW_FORMAT_SNAPPY = 2, - SRV_ROW_FORMAT_QUICKLZ = 3, - SRV_ROW_FORMAT_LZMA = 4, - SRV_ROW_FORMAT_FAST = 5, - SRV_ROW_FORMAT_SMALL = 6, - SRV_ROW_FORMAT_DEFAULT = 7 -}; -typedef enum srv_row_format_enum srv_row_format_t; +inline tokudb::sysvars::row_format_t toku_compression_method_to_row_format( + toku_compression_method method) { -static inline srv_row_format_t toku_compression_method_to_row_format(toku_compression_method method) { switch (method) { case TOKU_NO_COMPRESSION: - return SRV_ROW_FORMAT_UNCOMPRESSED; + return tokudb::sysvars::SRV_ROW_FORMAT_UNCOMPRESSED; case TOKU_ZLIB_WITHOUT_CHECKSUM_METHOD: case TOKU_ZLIB_METHOD: - return SRV_ROW_FORMAT_ZLIB; + return tokudb::sysvars::SRV_ROW_FORMAT_ZLIB; case TOKU_SNAPPY_METHOD: - return SRV_ROW_FORMAT_SNAPPY; + return tokudb::sysvars::SRV_ROW_FORMAT_SNAPPY; case TOKU_QUICKLZ_METHOD: - return SRV_ROW_FORMAT_QUICKLZ; + return tokudb::sysvars::SRV_ROW_FORMAT_QUICKLZ; case TOKU_LZMA_METHOD: - return SRV_ROW_FORMAT_LZMA; + return tokudb::sysvars::SRV_ROW_FORMAT_LZMA; case TOKU_DEFAULT_COMPRESSION_METHOD: - return SRV_ROW_FORMAT_DEFAULT; + return tokudb::sysvars::SRV_ROW_FORMAT_DEFAULT; case TOKU_FAST_COMPRESSION_METHOD: - return SRV_ROW_FORMAT_FAST; + return tokudb::sysvars::SRV_ROW_FORMAT_FAST; case TOKU_SMALL_COMPRESSION_METHOD: - return SRV_ROW_FORMAT_SMALL; + return tokudb::sysvars::SRV_ROW_FORMAT_SMALL; default: - assert(0); + assert_unreachable(); } } -static inline toku_compression_method row_format_to_toku_compression_method(srv_row_format_t row_format) { +inline toku_compression_method row_format_to_toku_compression_method( + tokudb::sysvars::row_format_t row_format) { + switch (row_format) { - case SRV_ROW_FORMAT_UNCOMPRESSED: + case tokudb::sysvars::SRV_ROW_FORMAT_UNCOMPRESSED: return TOKU_NO_COMPRESSION; - case SRV_ROW_FORMAT_QUICKLZ: - case SRV_ROW_FORMAT_FAST: + case tokudb::sysvars::SRV_ROW_FORMAT_QUICKLZ: + case tokudb::sysvars::SRV_ROW_FORMAT_FAST: return TOKU_QUICKLZ_METHOD; - case SRV_ROW_FORMAT_SNAPPY: + case tokudb::sysvars::SRV_ROW_FORMAT_SNAPPY: return TOKU_SNAPPY_METHOD; - case SRV_ROW_FORMAT_ZLIB: - case SRV_ROW_FORMAT_DEFAULT: + case tokudb::sysvars::SRV_ROW_FORMAT_ZLIB: + case tokudb::sysvars::SRV_ROW_FORMAT_DEFAULT: return TOKU_ZLIB_WITHOUT_CHECKSUM_METHOD; - case SRV_ROW_FORMAT_LZMA: - case SRV_ROW_FORMAT_SMALL: + case tokudb::sysvars::SRV_ROW_FORMAT_LZMA: + case tokudb::sysvars::SRV_ROW_FORMAT_SMALL: return TOKU_LZMA_METHOD; default: - assert(0); + assert_unreachable(); } } -static inline enum row_type row_format_to_row_type(srv_row_format_t row_format) { +inline enum row_type row_format_to_row_type( + tokudb::sysvars::row_format_t row_format) { #if TOKU_INCLUDE_ROW_TYPE_COMPRESSION switch (row_format) { - case SRV_ROW_FORMAT_UNCOMPRESSED: + case tokudb::sysvars::SRV_ROW_FORMAT_UNCOMPRESSED: return ROW_TYPE_TOKU_UNCOMPRESSED; - case SRV_ROW_FORMAT_ZLIB: + case tokudb::sysvars::SRV_ROW_FORMAT_ZLIB: return ROW_TYPE_TOKU_ZLIB; - case SRV_ROW_FORMAT_SNAPPY: + case tokudb::sysvars::SRV_ROW_FORMAT_SNAPPY: return ROW_TYPE_TOKU_SNAPPY; - case SRV_ROW_FORMAT_QUICKLZ: + case tokudb::sysvars::SRV_ROW_FORMAT_QUICKLZ: return ROW_TYPE_TOKU_QUICKLZ; - case SRV_ROW_FORMAT_LZMA: + case tokudb::sysvars::SRV_ROW_FORMAT_LZMA: return ROW_TYPE_TOKU_LZMA; - case SRV_ROW_FORMAT_SMALL: + case tokudb::sysvars::SRV_ROW_FORMAT_SMALL: return ROW_TYPE_TOKU_SMALL; - case SRV_ROW_FORMAT_FAST: + case tokudb::sysvars::SRV_ROW_FORMAT_FAST: return ROW_TYPE_TOKU_FAST; - case SRV_ROW_FORMAT_DEFAULT: + case tokudb::sysvars::SRV_ROW_FORMAT_DEFAULT: return ROW_TYPE_DEFAULT; } #endif return ROW_TYPE_DEFAULT; } -static inline srv_row_format_t row_type_to_row_format(enum row_type type) { +inline tokudb::sysvars::row_format_t row_type_to_row_format( + enum row_type type) { #if TOKU_INCLUDE_ROW_TYPE_COMPRESSION switch (type) { case ROW_TYPE_TOKU_UNCOMPRESSED: - return SRV_ROW_FORMAT_UNCOMPRESSED; + return tokudb::sysvars::SRV_ROW_FORMAT_UNCOMPRESSED; case ROW_TYPE_TOKU_ZLIB: - return SRV_ROW_FORMAT_ZLIB; + return tokudb::sysvars::SRV_ROW_FORMAT_ZLIB; case ROW_TYPE_TOKU_SNAPPY: - return SRV_ROW_FORMAT_SNAPPY; + return tokudb::sysvars::SRV_ROW_FORMAT_SNAPPY; case ROW_TYPE_TOKU_QUICKLZ: - return SRV_ROW_FORMAT_QUICKLZ; + return tokudb::sysvars::SRV_ROW_FORMAT_QUICKLZ; case ROW_TYPE_TOKU_LZMA: - return SRV_ROW_FORMAT_LZMA; + return tokudb::sysvars::SRV_ROW_FORMAT_LZMA; case ROW_TYPE_TOKU_SMALL: - return SRV_ROW_FORMAT_SMALL; + return tokudb::sysvars::SRV_ROW_FORMAT_SMALL; case ROW_TYPE_TOKU_FAST: - return SRV_ROW_FORMAT_FAST; + return tokudb::sysvars::SRV_ROW_FORMAT_FAST; case ROW_TYPE_DEFAULT: - return SRV_ROW_FORMAT_DEFAULT; + return tokudb::sysvars::SRV_ROW_FORMAT_DEFAULT; default: - return SRV_ROW_FORMAT_DEFAULT; + return tokudb::sysvars::SRV_ROW_FORMAT_DEFAULT; } #endif - return SRV_ROW_FORMAT_DEFAULT; -} - -static inline enum row_type toku_compression_method_to_row_type(toku_compression_method method) { - return row_format_to_row_type(toku_compression_method_to_row_format(method)); -} - -static inline toku_compression_method row_type_to_toku_compression_method(enum row_type type) { - return row_format_to_toku_compression_method(row_type_to_row_format(type)); -} - -// thread variables - -static MYSQL_THDVAR_BOOL(commit_sync, - PLUGIN_VAR_THDLOCAL, - "sync on txn commit", - /* check */ NULL, - /* update */ NULL, - /* default*/ true -); - -static MYSQL_THDVAR_UINT(pk_insert_mode, - 0, - "set the primary key insert mode", - NULL, - NULL, - 1, // default - 0, // min? - 2, // max - 1 // blocksize -); - -static uint get_pk_insert_mode(THD* thd) { - return THDVAR(thd, pk_insert_mode); -} - -static MYSQL_THDVAR_BOOL(load_save_space, - 0, - "compress intermediate bulk loader files to save space", - NULL, - NULL, - true -); - -static bool get_load_save_space(THD* thd) { - return (THDVAR(thd, load_save_space) != 0); -} - -static MYSQL_THDVAR_BOOL(disable_slow_alter, - 0, - "if on, alter tables that require copy are disabled", - NULL, - NULL, - false -); - -static bool get_disable_slow_alter(THD* thd) { - return (THDVAR(thd, disable_slow_alter) != 0); -} - -static MYSQL_THDVAR_BOOL(disable_hot_alter, - 0, - "if on, hot alter table is disabled", - NULL, - NULL, - false -); - -static bool get_disable_hot_alter(THD* thd) { - return THDVAR(thd, disable_hot_alter) != 0; -} - -static MYSQL_THDVAR_BOOL(create_index_online, - 0, - "if on, create index done online", - NULL, - NULL, - true -); - -static bool get_create_index_online(THD* thd) { - return (THDVAR(thd, create_index_online) != 0); -} - -static MYSQL_THDVAR_BOOL(alter_print_error, - 0, - "Print errors for alter table operations", - NULL, - NULL, - false -); - -static MYSQL_THDVAR_BOOL(disable_prefetching, - 0, - "if on, prefetching disabled", - NULL, - NULL, - false -); - -static bool get_disable_prefetching(THD* thd) { - return (THDVAR(thd, disable_prefetching) != 0); -} - -static MYSQL_THDVAR_BOOL(prelock_empty, - 0, - "Tokudb Prelock Empty Table", - NULL, - NULL, - true -); - -static bool get_prelock_empty(THD* thd) { - return (THDVAR(thd, prelock_empty) != 0); -} - -static MYSQL_THDVAR_UINT(block_size, - 0, - "fractal tree block size", - NULL, - NULL, - 4<<20, // default - 4096, // min - ~0U, // max - 1 // blocksize??? -); - -static uint get_tokudb_block_size(THD* thd) { - return THDVAR(thd, block_size); -} - -static MYSQL_THDVAR_UINT(read_block_size, - 0, - "fractal tree read block size", - NULL, - NULL, - 64*1024, // default - 4096, // min - ~0U, // max - 1 // blocksize??? -); - -static uint get_tokudb_read_block_size(THD* thd) { - return THDVAR(thd, read_block_size); + return tokudb::sysvars::SRV_ROW_FORMAT_DEFAULT; } -static MYSQL_THDVAR_UINT(read_buf_size, - 0, - "fractal tree read block size", //TODO: Is this a typo? - NULL, - NULL, - 128*1024, // default - 0, // min - 1*1024*1024, // max - 1 // blocksize??? -); - -static uint get_tokudb_read_buf_size(THD* thd) { - return THDVAR(thd, read_buf_size); -} - -#if TOKU_INCLUDE_UPSERT -static MYSQL_THDVAR_BOOL(disable_slow_update, - PLUGIN_VAR_THDLOCAL, - "disable slow update", - NULL, // check - NULL, // update - false // default -); - -static MYSQL_THDVAR_BOOL(disable_slow_upsert, - PLUGIN_VAR_THDLOCAL, - "disable slow upsert", - NULL, // check - NULL, // update - false // default -); -#endif +inline enum row_type toku_compression_method_to_row_type( + toku_compression_method method) { -static MYSQL_THDVAR_UINT(fanout, - 0, - "fractal tree fanout", - NULL, - NULL, - 16, // default - 2, // min - 16*1024, // max - 1 // blocksize??? -); - -static uint get_tokudb_fanout(THD* thd) { - return THDVAR(thd, fanout); + return row_format_to_row_type( + toku_compression_method_to_row_format(method)); } -static MYSQL_THDVAR_UINT(analyze_time, 0, "analyze time (seconds)", NULL /*check*/, NULL /*update*/, 5 /*default*/, 0 /*min*/, ~0U /*max*/, 1 /*blocksize*/); - -static MYSQL_THDVAR_DOUBLE(analyze_delete_fraction, 0, "fraction of rows allowed to be deleted", NULL /*check*/, NULL /*update*/, 1.0 /*def*/, 0 /*min*/, 1.0 /*max*/, 1); +inline toku_compression_method row_type_to_toku_compression_method( + enum row_type type) { -static void tokudb_checkpoint_lock(THD * thd); -static void tokudb_checkpoint_unlock(THD * thd); - -static void tokudb_checkpoint_lock_update( - THD* thd, - struct st_mysql_sys_var* var, - void* var_ptr, - const void* save) -{ - my_bool* val = (my_bool *) var_ptr; - *val= *(my_bool *) save ? true : false; - if (*val) { - tokudb_checkpoint_lock(thd); - } - else { - tokudb_checkpoint_unlock(thd); - } -} - -static MYSQL_THDVAR_BOOL(checkpoint_lock, - 0, - "Tokudb Checkpoint Lock", - NULL, - tokudb_checkpoint_lock_update, - false -); - -static const char *tokudb_row_format_names[] = { - "tokudb_uncompressed", - "tokudb_zlib", - "tokudb_snappy", - "tokudb_quicklz", - "tokudb_lzma", - "tokudb_fast", - "tokudb_small", - "tokudb_default", - NullS -}; - -static TYPELIB tokudb_row_format_typelib = { - array_elements(tokudb_row_format_names) - 1, - "tokudb_row_format_typelib", - tokudb_row_format_names, - NULL -}; - -static MYSQL_THDVAR_ENUM(row_format, PLUGIN_VAR_OPCMDARG, - "Specifies the compression method for a table during this session. " - "Possible values are TOKUDB_UNCOMPRESSED, TOKUDB_ZLIB, TOKUDB_SNAPPY, " - "TOKUDB_QUICKLZ, TOKUDB_LZMA, TOKUDB_FAST, TOKUDB_SMALL and TOKUDB_DEFAULT", - NULL, NULL, SRV_ROW_FORMAT_ZLIB, &tokudb_row_format_typelib); - -static inline srv_row_format_t get_row_format(THD *thd) { - return (srv_row_format_t) THDVAR(thd, row_format); + return row_format_to_toku_compression_method(row_type_to_row_format(type)); } -static MYSQL_THDVAR_UINT(lock_timeout_debug, 0, "TokuDB lock timeout debug", NULL /*check*/, NULL /*update*/, 1 /*default*/, 0 /*min*/, ~0U /*max*/, 1); - -static MYSQL_THDVAR_STR(last_lock_timeout, PLUGIN_VAR_MEMALLOC, "last TokuDB lock timeout", NULL /*check*/, NULL /*update*/, NULL /*default*/); - -static MYSQL_THDVAR_BOOL(hide_default_row_format, 0, "hide the default row format", NULL /*check*/, NULL /*update*/, true); - -static const uint64_t DEFAULT_TOKUDB_LOCK_TIMEOUT = 4000; /*milliseconds*/ +void tokudb_checkpoint_lock(THD * thd); +void tokudb_checkpoint_unlock(THD * thd); -static MYSQL_THDVAR_ULONGLONG(lock_timeout, 0, "TokuDB lock timeout", NULL, NULL, DEFAULT_TOKUDB_LOCK_TIMEOUT, 0 /*min*/, ~0ULL /*max*/, 1 /*blocksize*/); - -static uint64_t tokudb_get_lock_wait_time_callback(uint64_t default_wait_time) { +inline uint64_t tokudb_get_lock_wait_time_callback(uint64_t default_wait_time) { THD *thd = current_thd; - uint64_t wait_time = THDVAR(thd, lock_timeout); - return wait_time; + return tokudb::sysvars::lock_timeout(thd); } -static MYSQL_THDVAR_ULONGLONG(loader_memory_size, - 0, - "TokuDB loader memory size", - NULL, - NULL, - 100*1000*1000, /*default*/ - 0, /*min*/ - ~0ULL, /*max*/ - 1 /*blocksize*/ -); - -static uint64_t tokudb_get_loader_memory_size_callback(void) { +inline uint64_t tokudb_get_loader_memory_size_callback(void) { THD *thd = current_thd; - uint64_t memory_size = THDVAR(thd, loader_memory_size); - return memory_size; + return tokudb::sysvars::loader_memory_size(thd); } -static const uint64_t DEFAULT_TOKUDB_KILLED_TIME = 4000; - -static MYSQL_THDVAR_ULONGLONG(killed_time, 0, "TokuDB killed time", NULL, NULL, DEFAULT_TOKUDB_KILLED_TIME, 0 /*min*/, ~0ULL /*max*/, 1 /*blocksize*/); - -static uint64_t tokudb_get_killed_time_callback(uint64_t default_killed_time) { +inline uint64_t tokudb_get_killed_time_callback(uint64_t default_killed_time) { THD *thd = current_thd; - uint64_t killed_time = THDVAR(thd, killed_time); - return killed_time; + return tokudb::sysvars::killed_time(thd); } -static int tokudb_killed_callback(void) { +inline int tokudb_killed_callback(void) { THD *thd = current_thd; return thd_killed(thd); } -static bool tokudb_killed_thd_callback(void *extra, uint64_t deleted_rows) { +inline bool tokudb_killed_thd_callback(void *extra, uint64_t deleted_rows) { THD *thd = static_cast<THD *>(extra); return thd_killed(thd) != 0; } -enum { - TOKUDB_EMPTY_SCAN_DISABLED = 0, - TOKUDB_EMPTY_SCAN_LR = 1, - TOKUDB_EMPTY_SCAN_RL = 2, -}; - -static const char *tokudb_empty_scan_names[] = { - "disabled", - "lr", - "rl", - NullS -}; - -static TYPELIB tokudb_empty_scan_typelib = { - array_elements(tokudb_empty_scan_names) - 1, - "tokudb_empty_scan_typelib", - tokudb_empty_scan_names, - NULL -}; - -static MYSQL_THDVAR_ENUM(empty_scan, PLUGIN_VAR_OPCMDARG, - "TokuDB algorithm to check if the table is empty when opened. ", - NULL, NULL, TOKUDB_EMPTY_SCAN_RL, &tokudb_empty_scan_typelib -); - -#if TOKUDB_CHECK_JEMALLOC -static uint tokudb_check_jemalloc; -static MYSQL_SYSVAR_UINT(check_jemalloc, tokudb_check_jemalloc, 0, "Check if jemalloc is linked", - NULL, NULL, 1, 0, 1, 0); -#endif - -static MYSQL_THDVAR_BOOL(bulk_fetch, PLUGIN_VAR_THDLOCAL, "enable bulk fetch", - NULL /*check*/, NULL /*update*/, true /*default*/); - -#if TOKU_INCLUDE_XA -static MYSQL_THDVAR_BOOL(support_xa, - PLUGIN_VAR_OPCMDARG, - "Enable TokuDB support for the XA two-phase commit", - NULL, // check - NULL, // update - true // default -); -#endif - -static MYSQL_THDVAR_BOOL(rpl_unique_checks, PLUGIN_VAR_THDLOCAL, "enable unique checks on replication slave", - NULL /*check*/, NULL /*update*/, true /*default*/); - -static MYSQL_THDVAR_ULONGLONG(rpl_unique_checks_delay, PLUGIN_VAR_THDLOCAL, "time in milliseconds to add to unique checks test on replication slave", - NULL, NULL, 0 /*default*/, 0 /*min*/, ~0ULL /*max*/, 1 /*blocksize*/); - -static MYSQL_THDVAR_BOOL(rpl_lookup_rows, PLUGIN_VAR_THDLOCAL, "lookup a row on rpl slave", - NULL /*check*/, NULL /*update*/, true /*default*/); - -static MYSQL_THDVAR_ULONGLONG(rpl_lookup_rows_delay, PLUGIN_VAR_THDLOCAL, "time in milliseconds to add to lookups on replication slave", - NULL, NULL, 0 /*default*/, 0 /*min*/, ~0ULL /*max*/, 1 /*blocksize*/); - -static MYSQL_THDVAR_BOOL(rpl_check_readonly, PLUGIN_VAR_THDLOCAL, "check if the slave is read only", - NULL /*check*/, NULL /*update*/, true /*default*/); - -static MYSQL_THDVAR_STR(optimize_index_name, PLUGIN_VAR_THDLOCAL + PLUGIN_VAR_MEMALLOC, "optimize index name (default all indexes)", NULL /*check*/, NULL /*update*/, NULL /*default*/); - -static MYSQL_THDVAR_DOUBLE(optimize_index_fraction, 0, "optimize index fraction (default 1.0 all)", NULL /*check*/, NULL /*update*/, 1.0 /*def*/, 0 /*min*/, 1.0 /*max*/, 1); - -static MYSQL_THDVAR_ULONGLONG(optimize_throttle, 0, "optimize throttle (default no throttle)", NULL /*check*/, NULL /*update*/, 0 /*def*/, 0 /*min*/, ~0ULL /*max*/, 1); extern HASH tokudb_open_tables; -extern pthread_mutex_t tokudb_mutex; -extern uint32_t tokudb_write_status_frequency; -extern uint32_t tokudb_read_status_frequency; +extern tokudb::thread::mutex_t tokudb_mutex; +extern const char* tokudb_hton_name; +extern int tokudb_hton_initialized; +extern tokudb::thread::rwlock_t tokudb_hton_initialized_lock; void toku_hton_update_primary_key_bytes_inserted(uint64_t row_size); +void tokudb_split_dname( + const char* dname, + String& database_name, + String& table_name, + String& dictionary_name); + +void tokudb_pretty_left_key(const DB* db, const DBT* key, String* out); +void tokudb_pretty_right_key(const DB* db, const DBT* key, String* out); +const char *tokudb_get_index_name(DB* db); + +inline uint get_key_parts(const KEY *key) { +#if (50609 <= MYSQL_VERSION_ID && MYSQL_VERSION_ID <= 50699) || \ + (50700 <= MYSQL_VERSION_ID && MYSQL_VERSION_ID <= 50799) || \ + (100009 <= MYSQL_VERSION_ID && MYSQL_VERSION_ID <= 100099) + return key->user_defined_key_parts; +#else + return key->key_parts; +#endif +} + #endif //#ifdef _HATOKU_HTON diff --git a/storage/tokudb/tokudb_background.cc b/storage/tokudb/tokudb_background.cc new file mode 100644 index 00000000000..d8ef54a5972 --- /dev/null +++ b/storage/tokudb/tokudb_background.cc @@ -0,0 +1,253 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +/* -*- mode: C; c-basic-offset: 4 -*- */ +#ident "$Id$" +/*====== +This file is part of TokuDB + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + TokuDBis is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + TokuDB is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TokuDB. If not, see <http://www.gnu.org/licenses/>. + +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#include "tokudb_background.h" +#include "tokudb_sysvars.h" + +namespace tokudb { +namespace background { + + +std::atomic<uint64_t> job_manager_t::job_t::_next_id(1); + +job_manager_t::job_t::job_t(bool user_scheduled) : + _running(false), + _cancelled(false), + _id(_next_id++), + _user_scheduled(user_scheduled), + _scheduled_time(::time(0)), + _started_time(0) { +} +job_manager_t::job_t::~job_t() { +} +void* job_manager_t::operator new(size_t sz) { + return tokudb::memory::malloc(sz, MYF(MY_WME|MY_ZEROFILL|MY_FAE)); +} +void job_manager_t::operator delete(void* p) { + tokudb::memory::free(p); +} +job_manager_t::job_manager_t() : + _sem(0, 65535), + _shutdown(false) { +} +job_manager_t::~job_manager_t() { +} +void job_manager_t::initialize() { + int r = _thread.start(thread_func, this); + assert_always(r == 0); +} +void job_manager_t::destroy() { + assert_always(!_shutdown); + assert_always(_foreground_jobs.size() == 0); + _shutdown = true; + _sem.set_interrupt(); + + while (_background_jobs.size()) { + _mutex.lock(); + job_t* job = _background_jobs.front(); + cancel(job); + _background_jobs.pop_front(); + delete job; + _mutex.unlock(); + } + + void* result; + int r = _thread.join(&result); + assert_always(r == 0); +} +bool job_manager_t::run_job(job_t* newjob, bool background) { + bool ret = false; + const char* jobkey = newjob->key(); + + _mutex.lock(); + assert_always(!_shutdown); + + for (jobs_t::iterator it = _background_jobs.begin(); + it != _background_jobs.end(); + it++) { + job_t* job = *it; + if (!job->cancelled() && strcmp(job->key(), jobkey) == 0) { + // if this is a foreground job being run and + // there is an existing background job of the same type + // and it is not running yet, we can cancel the background job + // and just run this one in the foreground, might have different + // params, but that is up to the user to figure out. + if (!background && !job->running()) { + job->cancel(); + } else { + // can't schedule or run another job on the same key + goto cleanup; + } + } + } + for (jobs_t::iterator it = _foreground_jobs.begin(); + it != _foreground_jobs.end(); + it++) { + job_t* job = *it; + if (strcmp(job->key(), jobkey) == 0) { + // can't schedule or run another job on the same key + // as an existing foreground job + goto cleanup; + } + } + + if (background) { + _background_jobs.push_back(newjob); + _sem.signal(); + ret = true; + } else { + _foreground_jobs.push_back(newjob); + + run(newjob); + + for (jobs_t::iterator it = _foreground_jobs.begin(); + it != _foreground_jobs.end(); + it++) { + job_t* job = *it; + if (job == newjob) { + _foreground_jobs.erase(it); + delete job; + break; + } + } + ret = true; + } + +cleanup: + _mutex.unlock(); + return ret; +} +bool job_manager_t::cancel_job(const char* key) { + bool ret = false; + _mutex.lock(); + + for (jobs_t::iterator it = _background_jobs.begin(); + it != _background_jobs.end(); it++) { + job_t* job = *it; + + if (!job->cancelled() && + strcmp(job->key(), key) == 0) { + + cancel(job); + + ret = true; + } + } + + _mutex.unlock(); + return ret; +} +void job_manager_t::iterate_jobs(pfn_iterate_t callback, void* extra) const { + + char database[256], table[256], type[256], params[256], status[256]; + + _mutex.lock(); + + for (jobs_t::const_iterator it = _background_jobs.begin(); + it != _background_jobs.end(); + it++) { + job_t* job = *it; + if (!job->cancelled()) { + database[0] = table[0] = type[0] = params[0] = status[0] = '\0'; + job->status(database, table, type, params, status); + callback( + job->id(), + database, + table, + type, + params, + status, + job->user_scheduled(), + job->scheduled_time(), + job->started_time(), + extra); + } + } + + _mutex.unlock(); +} +void* job_manager_t::thread_func(void* v) { + return ((tokudb::background::job_manager_t*)v)->real_thread_func(); +} +void* job_manager_t::real_thread_func() { + while (_shutdown == false) { + tokudb::thread::semaphore_t::E_WAIT res = _sem.wait(); + if (res == tokudb::thread::semaphore_t::E_INTERRUPTED || _shutdown) { + break; + } else if (res == tokudb::thread::semaphore_t::E_SIGNALLED) { +#if TOKUDB_DEBUG + if (TOKUDB_UNLIKELY( + tokudb::sysvars::debug_pause_background_job_manager)) { + _sem.signal(); + tokudb::time::sleep_microsec(250000); + continue; + } +#endif // TOKUDB_DEBUG + + _mutex.lock(); + assert_debug(_background_jobs.size() > 0); + job_t* job = _background_jobs.front(); + run(job); + _background_jobs.pop_front(); + _mutex.unlock(); + delete job; + } + } + return NULL; +} +void job_manager_t::run(job_t* job) { + assert_debug(_mutex.is_owned_by_me()); + if (!job->cancelled()) { + _mutex.unlock(); + // do job + job->run(); + // done job + _mutex.lock(); + } + if (!job->cancelled()) { + job->destroy(); + } +} +void job_manager_t::cancel(job_t* job) { + assert_debug(_mutex.is_owned_by_me()); + job->cancel(); +} +job_manager_t* _job_manager = NULL; + +bool initialize() { + assert_always(_job_manager == NULL); + _job_manager = new job_manager_t; + _job_manager->initialize(); + return true; +} +bool destroy() { + _job_manager->destroy(); + delete _job_manager; + _job_manager = NULL; + return true; +} +} // namespace background +} // namespace tokudb diff --git a/storage/tokudb/tokudb_background.h b/storage/tokudb/tokudb_background.h new file mode 100644 index 00000000000..3786701fd0f --- /dev/null +++ b/storage/tokudb/tokudb_background.h @@ -0,0 +1,212 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +#ident "$Id$" +/*====== +This file is part of TokuDB + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + TokuDBis is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + TokuDB is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TokuDB. If not, see <http://www.gnu.org/licenses/>. + +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#ifndef _TOKUDB_BACKGROUND_H +#define _TOKUDB_BACKGROUND_H + +#include "hatoku_hton.h" +#include <atomic> +#include <list> + +namespace tokudb { +namespace background { + +class job_manager_t { +public: + class job_t { + public: + // NO default constructor + job_t() = delete; + + job_t(bool user_scheduled); + + virtual ~job_t(); + + // method that runs the job + inline void run(); + + // method that tells the job to cancel ASAP + inline void cancel(); + + // method that tells the job to clean up/free resources on cancel + // or completion + inline void destroy(); + + // method that returns a 'key' string for finding a specific job + // (or jobs) usually used to find jobs to cancel + virtual const char* key() = 0; + + // method to get info for information schema, 255 chars per buffer + virtual void status( + char* database, + char* table, + char* type, + char* params, + char* status) = 0; + + inline bool running() const; + + inline bool cancelled() const; + + inline uint64_t id() const; + + inline bool user_scheduled() const; + + inline time_t scheduled_time() const; + + inline time_t started_time() const; + + protected: + // derived classed implement this to actually run their job + virtual void on_run() {}; + + // derived classes implement this to cancel their job + virtual void on_cancel() {}; + + // derived classes implement this to clean up/free resources + virtual void on_destroy() {}; + + private: + static std::atomic<uint64_t> _next_id; + std::atomic<bool> _running; + std::atomic<bool> _cancelled; + uint64_t _id; + bool _user_scheduled; + time_t _scheduled_time; + time_t _started_time; + }; + + // pfn for iterate callback + typedef void (*pfn_iterate_t)( + uint64_t, + const char*, + const char*, + const char*, + const char*, + const char*, + bool, + time_t, + time_t, + void*); + +public: + void* operator new(size_t sz); + void operator delete(void* p); + + job_manager_t(); + + ~job_manager_t(); + + // creates/initializes a singleton bjm + void initialize(); + + // destroys a bjm singleton + // cancels all jobs abd frees all resources + void destroy(); + + // schedules or runs a job depending on the 'background' value + // job specifics all depend on the implementation od 'job' + // background jobs will be executed in a FIFO fashion + // two jobs with the same key can not run concurrently + // if a foreground job is attempted, any currently scheduled or running + // background jobs will be cancelled first + // if another foreground job is already running, a new foreground job with + // the same key will be rejected + bool run_job(job_t* newjob, bool background); + + // cancels any background job with a matching key + bool cancel_job(const char* key); + + // iterates currently pending and running background jobs, calling + // 'callback' with the 'extra' data provided and the original 'extra' + // data passed when the job was scheduled + void iterate_jobs(pfn_iterate_t callback, void* extra) const; + +private: + static void* thread_func(void* v); + + void* real_thread_func(); + + // _mutex MUST be held on entry, will release and reaquire on exit + void run(job_t* job); + + // _mutex MUST be held on entry + void cancel(job_t* job); +private: + typedef std::list<job_t*> jobs_t; + + mutable tokudb::thread::mutex_t _mutex; + mutable tokudb::thread::semaphore_t _sem; + mutable tokudb::thread::thread_t _thread; + jobs_t _background_jobs; + jobs_t _foreground_jobs; + std::atomic<bool> _shutdown; +}; + +extern job_manager_t* _job_manager; + +bool initialize(); +bool destroy(); + +inline void job_manager_t::job_t::run() { + if (!_cancelled) { + _running = true; + _started_time = ::time(0); + on_run(); + _running = false; + } +} +inline void job_manager_t::job_t::cancel() { + _cancelled = true; + if (_running) + on_cancel(); + while (_running) tokudb::time::sleep_microsec(500000); + destroy(); +} +void job_manager_t::job_t::destroy() { + on_destroy(); +} +inline bool job_manager_t::job_t::running() const { + return _running; +} +inline bool job_manager_t::job_t::cancelled() const { + return _cancelled; +} +inline uint64_t job_manager_t::job_t::id() const { + return _id; +} +inline bool job_manager_t::job_t::user_scheduled() const { + return _user_scheduled; +} +inline time_t job_manager_t::job_t::scheduled_time() const { + return _scheduled_time; +} +inline time_t job_manager_t::job_t::started_time() const { + return _started_time; +} +} // namespace background +} // namespace tokudb + +#endif // _TOKUDB_BACKGROUND_H diff --git a/storage/tokudb/tokudb_buffer.h b/storage/tokudb/tokudb_buffer.h index 1604ea61e2d..022f1b49643 100644 --- a/storage/tokudb/tokudb_buffer.h +++ b/storage/tokudb/tokudb_buffer.h @@ -23,38 +23,54 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. #ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." -#if !defined(_TOKUDB_BUFFER_H) +#ifndef _TOKUDB_BUFFER_H #define _TOKUDB_BUFFER_H +#include "hatoku_defines.h" +#include "tokudb_debug.h" +#include "tokudb_thread.h" #include "tokudb_vlq.h" namespace tokudb { -// A Buffer manages a contiguous chunk of memory and supports appending new data to the end of the buffer, and -// consuming chunks from the beginning of the buffer. The buffer will reallocate memory when appending -// new data to a full buffer. +// A Buffer manages a contiguous chunk of memory and supports appending new +// data to the end of the buffer, and consuming chunks from the beginning of +// the buffer. The buffer will reallocate memory when appending new data to +// a full buffer. class buffer { public: - buffer(void *the_data, size_t s, size_t l) : m_data(the_data), m_size(s), m_limit(l), m_is_static(true) { + inline buffer( + void* the_data, + size_t s, + size_t l) : + m_data(the_data), + m_size(s), + m_limit(l), + m_is_static(true) { } - buffer() : m_data(NULL), m_size(0), m_limit(0), m_is_static(false) { + inline buffer() : + m_data(NULL), + m_size(0), + m_limit(0), + m_is_static(false) { } virtual ~buffer() { if (!m_is_static) free(m_data); } - // Return a pointer to the end of the buffer suitable for appending a fixed number of bytes. - void *append_ptr(size_t s) { + // Return a pointer to the end of the buffer suitable for appending a + // fixed number of bytes. + void* append_ptr(size_t s) { maybe_realloc(s); - void *p = (char *) m_data + m_size; + void* p = (char*)m_data + m_size; m_size += s; return p; } // Append bytes to the buffer - void append(void *p, size_t s) { + void append(void* p, size_t s) { memcpy(append_ptr(s), p, s); } @@ -68,63 +84,70 @@ public: return s; } - // Return a pointer to the next location in the buffer where bytes are consumed from. - void *consume_ptr(size_t s) { + // Return a pointer to the next location in the buffer where bytes are + // consumed from. + void* consume_ptr(size_t s) { if (m_size + s > m_limit) return NULL; - void *p = (char *) m_data + m_size; + void* p = (char*)m_data + m_size; m_size += s; return p; } // Consume bytes from the buffer. - void consume(void *p, size_t s) { + void consume(void* p, size_t s) { memcpy(p, consume_ptr(s), s); } // Consume an unsigned int from the buffer. - // Returns 0 if the unsigned int could not be decoded, probably because the buffer is too short. - // Otherwise return the number of bytes consumed, and stuffs the decoded number in *p. - template<class T> size_t consume_ui(T *p) { - size_t s = tokudb::vlq_decode_ui<T>(p, (char *) m_data + m_size, m_limit - m_size); + // Returns 0 if the unsigned int could not be decoded, probably because + // the buffer is too short. + // Otherwise return the number of bytes consumed, and stuffs the decoded + // number in *p. + template<class T> size_t consume_ui(T* p) { + size_t s = tokudb::vlq_decode_ui<T>( + p, + (char*)m_data + m_size, + m_limit - m_size); m_size += s; return s; } // Write p_length bytes at an offset in the buffer - void write(void *p, size_t p_length, size_t offset) { - assert(offset + p_length <= m_size); - memcpy((char *)m_data + offset, p, p_length); + void write(void* p, size_t p_length, size_t offset) { + assert_always(offset + p_length <= m_size); + memcpy((char*)m_data + offset, p, p_length); } // Read p_length bytes at an offset in the buffer - void read(void *p, size_t p_length, size_t offset) { - assert(offset + p_length <= m_size); - memcpy(p, (char *)m_data + offset, p_length); + void read(void* p, size_t p_length, size_t offset) { + assert_always(offset + p_length <= m_size); + memcpy(p, (char*)m_data + offset, p_length); } - // Replace a field in the buffer with new data. If the new data size is different, then readjust the - // size of the buffer and move things around. - void replace(size_t offset, size_t old_s, void *new_p, size_t new_s) { - assert(offset + old_s <= m_size); + // Replace a field in the buffer with new data. If the new data size is + // different, then readjust the size of the buffer and move things around. + void replace(size_t offset, size_t old_s, void* new_p, size_t new_s) { + assert_always(offset + old_s <= m_size); if (new_s > old_s) maybe_realloc(new_s - old_s); - char *data_offset = (char *) m_data + offset; + char* data_offset = (char*)m_data + offset; if (new_s != old_s) { size_t n = m_size - (offset + old_s); - assert(offset + new_s + n <= m_limit && offset + old_s + n <= m_limit); + assert_always( + offset + new_s + n <= m_limit && offset + old_s + n <= m_limit); memmove(data_offset + new_s, data_offset + old_s, n); if (new_s > old_s) m_size += new_s - old_s; else m_size -= old_s - new_s; - assert(m_size <= m_limit); + assert_always(m_size <= m_limit); } memcpy(data_offset, new_p, new_s); } // Return a pointer to the data in the buffer - void *data() const { + void* data() const { return m_data; } @@ -145,15 +168,15 @@ private: size_t new_limit = m_limit * 2; if (new_limit < m_size + s) new_limit = m_size + s; - assert(!m_is_static); + assert_always(!m_is_static); void *new_data = realloc(m_data, new_limit); - assert(new_data != NULL); + assert_always(new_data != NULL); m_data = new_data; m_limit = new_limit; } } private: - void *m_data; + void* m_data; size_t m_size; size_t m_limit; bool m_is_static; diff --git a/storage/tokudb/tokudb_card.h b/storage/tokudb/tokudb_card.h index 04eac731aeb..fdf18d4ab12 100644 --- a/storage/tokudb/tokudb_card.h +++ b/storage/tokudb/tokudb_card.h @@ -32,43 +32,49 @@ namespace tokudb { return total_key_parts; } - // Set the key_info cardinality counters for the table. - void set_card_in_key_info(TABLE *table, uint rec_per_keys, uint64_t rec_per_key[]) { - uint next_key_part = 0; - for (uint i = 0; i < table->s->keys; i++) { - bool is_unique_key = (i == table->s->primary_key) || (table->key_info[i].flags & HA_NOSAME); - uint num_key_parts = get_key_parts(&table->key_info[i]); - for (uint j = 0; j < num_key_parts; j++) { - assert(next_key_part < rec_per_keys); - ulong val = rec_per_key[next_key_part++]; - if (is_unique_key && j == num_key_parts-1) - val = 1; - table->key_info[i].rec_per_key[j] = val; - } - } - } - // Put the cardinality counters into the status dictionary. - int set_card_in_status(DB *status_db, DB_TXN *txn, uint rec_per_keys, uint64_t rec_per_key[]) { + int set_card_in_status( + DB* status_db, + DB_TXN* txn, + uint rec_per_keys, + const uint64_t rec_per_key[]) { + // encode cardinality into the buffer tokudb::buffer b; size_t s; s = b.append_ui<uint32_t>(rec_per_keys); - assert(s > 0); + assert_always(s > 0); for (uint i = 0; i < rec_per_keys; i++) { s = b.append_ui<uint64_t>(rec_per_key[i]); - assert(s > 0); + assert_always(s > 0); } // write cardinality to status - int error = write_to_status(status_db, hatoku_cardinality, b.data(), b.size(), txn); + int error = + tokudb::metadata::write( + status_db, + hatoku_cardinality, + b.data(), + b.size(), + txn); return error; } // Get the cardinality counters from the status dictionary. - int get_card_from_status(DB *status_db, DB_TXN *txn, uint rec_per_keys, uint64_t rec_per_key[]) { + int get_card_from_status( + DB* status_db, + DB_TXN* txn, + uint rec_per_keys, + uint64_t rec_per_key[]) { + // read cardinality from status - void *buf = 0; size_t buf_size = 0; - int error = get_status_realloc(status_db, txn, hatoku_cardinality, &buf, &buf_size); + void* buf = 0; size_t buf_size = 0; + int error = + tokudb::metadata::read_realloc( + status_db, + txn, + hatoku_cardinality, + &buf, + &buf_size); if (error == 0) { // decode cardinality from the buffer tokudb::buffer b(buf, 0, buf_size); @@ -93,12 +99,17 @@ namespace tokudb { } // Delete the cardinality counters from the status dictionary. - int delete_card_from_status(DB *status_db, DB_TXN *txn) { - int error = remove_from_status(status_db, hatoku_cardinality, txn); + int delete_card_from_status(DB* status_db, DB_TXN* txn) { + int error = + tokudb::metadata::remove(status_db, hatoku_cardinality, txn); return error; } - bool find_index_of_key(const char *key_name, TABLE_SHARE *table_share, uint *index_offset_ptr) { + bool find_index_of_key( + const char* key_name, + TABLE_SHARE* table_share, + uint* index_offset_ptr) { + for (uint i = 0; i < table_share->keys; i++) { if (strcmp(key_name, table_share->key_info[i].name) == 0) { *index_offset_ptr = i; @@ -113,16 +124,30 @@ namespace tokudb { dest[i] = src[i]; } - // Altered table cardinality = select cardinality data from current table cardinality for keys that exist + // Altered table cardinality = select cardinality data from current table + // cardinality for keys that exist // in the altered table and the current table. - int alter_card(DB *status_db, DB_TXN *txn, TABLE_SHARE *table_share, TABLE_SHARE *altered_table_share) { + int alter_card( + DB* status_db, + DB_TXN *txn, + TABLE_SHARE* table_share, + TABLE_SHARE* altered_table_share) { + int error; // read existing cardinality data from status - uint table_total_key_parts = tokudb::compute_total_key_parts(table_share); + uint table_total_key_parts = + tokudb::compute_total_key_parts(table_share); + uint64_t rec_per_key[table_total_key_parts]; - error = get_card_from_status(status_db, txn, table_total_key_parts, rec_per_key); + error = + get_card_from_status( + status_db, + txn, + table_total_key_parts, + rec_per_key); // set altered records per key to unknown - uint altered_table_total_key_parts = tokudb::compute_total_key_parts(altered_table_share); + uint altered_table_total_key_parts = + tokudb::compute_total_key_parts(altered_table_share); uint64_t altered_rec_per_key[altered_table_total_key_parts]; for (uint i = 0; i < altered_table_total_key_parts; i++) altered_rec_per_key[i] = 0; @@ -139,113 +164,28 @@ namespace tokudb { for (uint i = 0; error == 0 && i < altered_table_share->keys; i++) { uint ith_key_parts = get_key_parts(&altered_table_share->key_info[i]); uint orig_key_index; - if (find_index_of_key(altered_table_share->key_info[i].name, table_share, &orig_key_index)) { - copy_card(&altered_rec_per_key[next_key_parts], &rec_per_key[orig_key_offset[orig_key_index]], ith_key_parts); + if (find_index_of_key( + altered_table_share->key_info[i].name, + table_share, + &orig_key_index)) { + copy_card( + &altered_rec_per_key[next_key_parts], + &rec_per_key[orig_key_offset[orig_key_index]], + ith_key_parts); } next_key_parts += ith_key_parts; } } - if (error == 0) - error = set_card_in_status(status_db, txn, altered_table_total_key_parts, altered_rec_per_key); - else - error = delete_card_from_status(status_db, txn); - return error; - } - - struct analyze_card_cursor_callback_extra { - int (*analyze_progress)(void *extra, uint64_t rows); - void *analyze_extra; - uint64_t *rows; - uint64_t *deleted_rows; - }; - - bool analyze_card_cursor_callback(void *extra, uint64_t deleted_rows) { - analyze_card_cursor_callback_extra *a_extra = static_cast<analyze_card_cursor_callback_extra *>(extra); - *a_extra->deleted_rows += deleted_rows; - int r = a_extra->analyze_progress(a_extra->analyze_extra, *a_extra->rows); - sql_print_information("tokudb analyze_card_cursor_callback %u %" PRIu64 " %" PRIu64, r, *a_extra->deleted_rows, deleted_rows); - return r != 0; - } - - // Compute records per key for all key parts of the ith key of the table. - // For each key part, put records per key part in *rec_per_key_part[key_part_index]. - // Returns 0 if success, otherwise an error number. - // TODO statistical dives into the FT - int analyze_card(DB *db, DB_TXN *txn, bool is_unique, uint64_t num_key_parts, uint64_t *rec_per_key_part, - int (*key_compare)(DB *, const DBT *, const DBT *, uint), - int (*analyze_progress)(void *extra, uint64_t rows), void *progress_extra, - uint64_t *return_rows, uint64_t *return_deleted_rows) { - int error = 0; - uint64_t rows = 0; - uint64_t deleted_rows = 0; - uint64_t unique_rows[num_key_parts]; - if (is_unique && num_key_parts == 1) { - // dont compute for unique keys with a single part. we already know the answer. - rows = unique_rows[0] = 1; + if (error == 0) { + error = + set_card_in_status( + status_db, + txn, + altered_table_total_key_parts, + altered_rec_per_key); } else { - DBC *cursor = NULL; - error = db->cursor(db, txn, &cursor, 0); - if (error == 0) { - analyze_card_cursor_callback_extra e = { analyze_progress, progress_extra, &rows, &deleted_rows }; - cursor->c_set_check_interrupt_callback(cursor, analyze_card_cursor_callback, &e); - for (uint64_t i = 0; i < num_key_parts; i++) - unique_rows[i] = 1; - // stop looking when the entire dictionary was analyzed, or a cap on execution time was reached, or the analyze was killed. - DBT key = {}; key.flags = DB_DBT_REALLOC; - DBT prev_key = {}; prev_key.flags = DB_DBT_REALLOC; - while (1) { - error = cursor->c_get(cursor, &key, 0, DB_NEXT); - if (error != 0) { - if (error == DB_NOTFOUND || error == TOKUDB_INTERRUPTED) - error = 0; // not an error - break; - } - rows++; - // first row is a unique row, otherwise compare with the previous key - bool copy_key = false; - if (rows == 1) { - copy_key = true; - } else { - // compare this key with the previous key. ignore appended PK for SK's. - // TODO if a prefix is different, then all larger keys that include the prefix are also different. - // TODO if we are comparing the entire primary key or the entire unique secondary key, then the cardinality must be 1, - // so we can avoid computing it. - for (uint64_t i = 0; i < num_key_parts; i++) { - int cmp = key_compare(db, &prev_key, &key, i+1); - if (cmp != 0) { - unique_rows[i]++; - copy_key = true; - } - } - } - // prev_key = key - if (copy_key) { - prev_key.data = realloc(prev_key.data, key.size); - assert(prev_key.data); - prev_key.size = key.size; - memcpy(prev_key.data, key.data, prev_key.size); - } - // check for limit - if (analyze_progress && (rows % 1000) == 0) { - error = analyze_progress(progress_extra, rows); - if (error) - break; - } - } - // cleanup - free(key.data); - free(prev_key.data); - int close_error = cursor->c_close(cursor); - assert(close_error == 0); - } + error = delete_card_from_status(status_db, txn); } - // return cardinality - if (return_rows) - *return_rows = rows; - if (return_deleted_rows) - *return_deleted_rows = deleted_rows; - for (uint64_t i = 0; i < num_key_parts; i++) - rec_per_key_part[i] = rows / unique_rows[i]; return error; } } diff --git a/storage/tokudb/tokudb_debug.h b/storage/tokudb/tokudb_debug.h new file mode 100644 index 00000000000..db66cab050c --- /dev/null +++ b/storage/tokudb/tokudb_debug.h @@ -0,0 +1,171 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +/* -*- mode: C; c-basic-offset: 4 -*- */ +#ident "$Id$" +/*====== +This file is part of TokuDB + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + TokuDBis is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + TokuDB is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TokuDB. If not, see <http://www.gnu.org/licenses/>. + +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#ifndef _TOKUDB_DEBUG_H +#define _TOKUDB_DEBUG_H + +#include "hatoku_defines.h" + +#define TOKU_INCLUDE_BACKTRACE 0 +#if TOKU_INCLUDE_BACKTRACE +static void tokudb_backtrace(void); +#endif + +// tokudb debug tracing for tokudb_debug declared in tokudb_sysvars.h/.cc +#define TOKUDB_DEBUG_INIT (1<<0) +#define TOKUDB_DEBUG_OPEN (1<<1) +#define TOKUDB_DEBUG_ENTER (1<<2) +#define TOKUDB_DEBUG_RETURN (1<<3) +#define TOKUDB_DEBUG_ERROR (1<<4) +#define TOKUDB_DEBUG_TXN (1<<5) +#define TOKUDB_DEBUG_AUTO_INCREMENT (1<<6) +#define TOKUDB_DEBUG_INDEX_KEY (1<<7) +#define TOKUDB_DEBUG_LOCK (1<<8) +#define TOKUDB_DEBUG_CHECK_KEY (1<<9) +#define TOKUDB_DEBUG_HIDE_DDL_LOCK_ERRORS (1<<10) +#define TOKUDB_DEBUG_ALTER_TABLE (1<<11) +#define TOKUDB_DEBUG_UPSERT (1<<12) +#define TOKUDB_DEBUG_CHECK (1<<13) +#define TOKUDB_DEBUG_ANALYZE (1<<14) + +#define TOKUDB_TRACE(_fmt, ...) { \ + fprintf(stderr, "%u %s:%u %s " _fmt "\n", tokudb::thread::my_tid(), \ + __FILE__, __LINE__, __FUNCTION__, ##__VA_ARGS__); \ +} + +#define TOKUDB_DEBUG_FLAGS(_flags) \ + (tokudb::sysvars::debug & _flags) + +#define TOKUDB_TRACE_FOR_FLAGS(_flags, _fmt, ...) { \ + if (TOKUDB_UNLIKELY(TOKUDB_DEBUG_FLAGS(_flags))) { \ + TOKUDB_TRACE(_fmt, ##__VA_ARGS__); \ + } \ +} + +#define TOKUDB_DBUG_ENTER(_fmt, ...) { \ + if (TOKUDB_UNLIKELY(tokudb::sysvars::debug & TOKUDB_DEBUG_ENTER)) { \ + TOKUDB_TRACE(_fmt, ##__VA_ARGS__); \ + } \ +} \ + DBUG_ENTER(__FUNCTION__); + +#define TOKUDB_DBUG_RETURN(r) { \ + int rr = (r); \ + if (TOKUDB_UNLIKELY((tokudb::sysvars::debug & TOKUDB_DEBUG_RETURN) || \ + (rr != 0 && (tokudb::sysvars::debug & TOKUDB_DEBUG_ERROR)))) { \ + TOKUDB_TRACE("return %d", rr); \ + } \ + DBUG_RETURN(rr); \ +} + +#define TOKUDB_HANDLER_TRACE(_fmt, ...) \ + fprintf(stderr, "%u %p %s:%u ha_tokudb::%s " _fmt "\n", \ + tokudb::thread::my_tid(), this, __FILE__, __LINE__, \ + __FUNCTION__, ##__VA_ARGS__); + +#define TOKUDB_HANDLER_TRACE_FOR_FLAGS(_flags, _fmt, ...) { \ + if (TOKUDB_UNLIKELY(TOKUDB_DEBUG_FLAGS(_flags))) { \ + TOKUDB_HANDLER_TRACE(_fmt, ##__VA_ARGS__); \ + } \ +} + + +#define TOKUDB_HANDLER_DBUG_ENTER(_fmt, ...) { \ + if (TOKUDB_UNLIKELY(tokudb::sysvars::debug & TOKUDB_DEBUG_ENTER)) { \ + TOKUDB_HANDLER_TRACE(_fmt, ##__VA_ARGS__); \ + } \ +} \ + DBUG_ENTER(__FUNCTION__); + +#define TOKUDB_HANDLER_DBUG_RETURN(r) { \ + int rr = (r); \ + if (TOKUDB_UNLIKELY((tokudb::sysvars::debug & TOKUDB_DEBUG_RETURN) || \ + (rr != 0 && (tokudb::sysvars::debug & TOKUDB_DEBUG_ERROR)))) { \ + TOKUDB_HANDLER_TRACE("return %d", rr); \ + } \ + DBUG_RETURN(rr); \ +} + +#define TOKUDB_HANDLER_DBUG_RETURN_DOUBLE(r) { \ + double rr = (r); \ + if (TOKUDB_UNLIKELY(tokudb::sysvars::debug & TOKUDB_DEBUG_RETURN)) { \ + TOKUDB_HANDLER_TRACE("return %f", rr); \ + } \ + DBUG_RETURN(rr); \ +} + +#define TOKUDB_HANDLER_DBUG_RETURN_PTR(r) { \ + if (TOKUDB_UNLIKELY(tokudb::sysvars::debug & TOKUDB_DEBUG_RETURN)) { \ + TOKUDB_HANDLER_TRACE("return 0x%p", r); \ + } \ + DBUG_RETURN(r); \ +} + + +#define TOKUDB_HANDLER_DBUG_VOID_RETURN { \ + if (TOKUDB_UNLIKELY(tokudb::sysvars::debug & TOKUDB_DEBUG_RETURN)) { \ + TOKUDB_HANDLER_TRACE("return"); \ + } \ + DBUG_VOID_RETURN; \ +} + +#define TOKUDB_DBUG_DUMP(s, p, len) \ +{ \ + TOKUDB_TRACE("%s", s); \ + uint i; \ + for (i=0; i<len; i++) { \ + fprintf(stderr, "%2.2x", ((uchar*)p)[i]); \ + } \ + fprintf(stderr, "\n"); \ +} + +// The intention is for a failed handlerton assert to invoke a failed assert +// in the fractal tree layer, which dumps engine status to the error log. +void toku_hton_assert_fail( + const char* /*expr_as_string*/, + const char* /*fun*/, + const char* /*file*/, + int /*line*/, + int /*errno*/) + __attribute__((__visibility__("default"))) + __attribute__((__noreturn__)); + +#define assert_always(expr) ((expr) ? (void)0 : \ + toku_hton_assert_fail(#expr, __FUNCTION__, __FILE__, __LINE__, errno)) + +#undef assert +#define assert(expr) assert_always(expr) + +#ifdef TOKUDB_DEBUG + #define assert_debug(expr) ((expr) ? (void)0 : \ + toku_hton_assert_fail(#expr, __FUNCTION__, __FILE__, __LINE__, errno)) +#else + #define assert_debug(expr) (void)0 +#endif // TOKUDB_DEBUG + +#define assert_unreachable __builtin_unreachable + +#endif // _TOKUDB_DEBUG_H diff --git a/storage/tokudb/tokudb_information_schema.cc b/storage/tokudb/tokudb_information_schema.cc new file mode 100644 index 00000000000..7ea3b8abbdd --- /dev/null +++ b/storage/tokudb/tokudb_information_schema.cc @@ -0,0 +1,1208 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +/* -*- mode: C; c-basic-offset: 4 -*- */ +#ident "$Id$" +/*====== +This file is part of TokuDB + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + TokuDBis is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + TokuDB is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TokuDB. If not, see <http://www.gnu.org/licenses/>. + +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#include "hatoku_hton.h" +#include "tokudb_information_schema.h" +#include "sql_time.h" +#include "tokudb_background.h" + + +namespace tokudb { +namespace information_schema { + +static void field_store_time_t(Field* field, time_t time) { + MYSQL_TIME my_time; + struct tm tm_time; + + if (time) { + localtime_r(&time, &tm_time); + localtime_to_TIME(&my_time, &tm_time); + my_time.time_type = MYSQL_TIMESTAMP_DATETIME; + field->store_time(&my_time, MYSQL_TIMESTAMP_DATETIME); + field->set_notnull(); + } else { + memset(&my_time, 0, sizeof(my_time)); + field->store_time(&my_time, MYSQL_TIMESTAMP_DATETIME); + field->set_null(); + } +} + +st_mysql_information_schema trx_information_schema = { + MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION +}; + +ST_FIELD_INFO trx_field_info[] = { + {"trx_id", 0, MYSQL_TYPE_LONGLONG, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"trx_mysql_thread_id", 0, MYSQL_TYPE_LONGLONG, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"trx_time", 0, MYSQL_TYPE_LONGLONG, 0, 0, NULL, SKIP_OPEN_TABLE }, + {NULL, 0, MYSQL_TYPE_NULL, 0, 0, NULL, SKIP_OPEN_TABLE} +}; + +struct trx_extra_t { + THD *thd; + TABLE *table; +}; + +int trx_callback( + DB_TXN* txn, + iterate_row_locks_callback iterate_locks, + void* locks_extra, + void *extra) { + + uint64_t txn_id = txn->id64(txn); + uint64_t client_id = txn->get_client_id(txn); + uint64_t start_time = txn->get_start_time(txn); + trx_extra_t* e = reinterpret_cast<struct trx_extra_t*>(extra); + THD* thd = e->thd; + TABLE* table = e->table; + table->field[0]->store(txn_id, false); + table->field[1]->store(client_id, false); + uint64_t tnow = (uint64_t) ::time(NULL); + table->field[2]->store(tnow >= start_time ? tnow - start_time : 0, false); + int error = schema_table_store_record(thd, table); + if (!error && thd_killed(thd)) + error = ER_QUERY_INTERRUPTED; + return error; +} + +#if MYSQL_VERSION_ID >= 50600 +int trx_fill_table(THD* thd, TABLE_LIST* tables, Item* cond) { +#else +int trx_fill_table(THD* thd, TABLE_LIST* tables, COND* cond) { +#endif + TOKUDB_DBUG_ENTER(""); + int error; + + tokudb_hton_initialized_lock.lock_read(); + + if (!tokudb_hton_initialized) { + error = ER_PLUGIN_IS_NOT_LOADED; + my_error(error, MYF(0), tokudb_hton_name); + } else { + trx_extra_t e = { thd, tables->table }; + error = db_env->iterate_live_transactions(db_env, trx_callback, &e); + if (error) + my_error(ER_GET_ERRNO, MYF(0), error, tokudb_hton_name); + } + + tokudb_hton_initialized_lock.unlock(); + TOKUDB_DBUG_RETURN(error); +} + +int trx_init(void* p) { + ST_SCHEMA_TABLE *schema = (ST_SCHEMA_TABLE*) p; + schema->fields_info = trx_field_info; + schema->fill_table = trx_fill_table; + return 0; +} + +int trx_done(void* p) { + return 0; +} + +st_mysql_plugin trx = { + MYSQL_INFORMATION_SCHEMA_PLUGIN, + &trx_information_schema, + "TokuDB_trx", + "Percona", + "Percona TokuDB Storage Engine with Fractal Tree(tm) Technology", + PLUGIN_LICENSE_GPL, + trx_init, /* plugin init */ + trx_done, /* plugin deinit */ + TOKUDB_PLUGIN_VERSION, + NULL, /* status variables */ + NULL, /* system variables */ +#ifdef MARIA_PLUGIN_INTERFACE_VERSION + tokudb::sysvars::version, + MariaDB_PLUGIN_MATURITY_STABLE /* maturity */ +#else + NULL, /* config options */ + 0, /* flags */ +#endif +}; + + + +st_mysql_information_schema lock_waits_information_schema = { + MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION +}; + +ST_FIELD_INFO lock_waits_field_info[] = { + {"requesting_trx_id", 0, MYSQL_TYPE_LONGLONG, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"blocking_trx_id", 0, MYSQL_TYPE_LONGLONG, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"lock_waits_dname", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"lock_waits_key_left", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"lock_waits_key_right", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"lock_waits_start_time", 0, MYSQL_TYPE_LONGLONG, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"lock_waits_table_schema", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"lock_waits_table_name", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"lock_waits_table_dictionary_name", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, + {NULL, 0, MYSQL_TYPE_NULL, 0, 0, NULL, SKIP_OPEN_TABLE} +}; + +struct lock_waits_extra_t { + THD* thd; + TABLE* table; +}; + +int lock_waits_callback( + DB* db, + uint64_t requesting_txnid, + const DBT* left_key, + const DBT* right_key, + uint64_t blocking_txnid, + uint64_t start_time, + void *extra) { + + lock_waits_extra_t* e = + reinterpret_cast<struct lock_waits_extra_t*>(extra); + THD* thd = e->thd; + TABLE* table = e->table; + table->field[0]->store(requesting_txnid, false); + table->field[1]->store(blocking_txnid, false); + const char* dname = tokudb_get_index_name(db); + size_t dname_length = strlen(dname); + table->field[2]->store(dname, dname_length, system_charset_info); + String left_str; + tokudb_pretty_left_key(db, left_key, &left_str); + table->field[3]->store( + left_str.ptr(), + left_str.length(), + system_charset_info); + String right_str; + tokudb_pretty_right_key(db, right_key, &right_str); + table->field[4]->store( + right_str.ptr(), + right_str.length(), + system_charset_info); + table->field[5]->store(start_time, false); + + String database_name, table_name, dictionary_name; + tokudb_split_dname(dname, database_name, table_name, dictionary_name); + table->field[6]->store( + database_name.c_ptr(), + database_name.length(), + system_charset_info); + table->field[7]->store( + table_name.c_ptr(), + table_name.length(), + system_charset_info); + table->field[8]->store( + dictionary_name.c_ptr(), + dictionary_name.length(), + system_charset_info); + + int error = schema_table_store_record(thd, table); + + if (!error && thd_killed(thd)) + error = ER_QUERY_INTERRUPTED; + + return error; +} + +#if MYSQL_VERSION_ID >= 50600 +int lock_waits_fill_table(THD* thd, TABLE_LIST* tables, Item* cond) { +#else +int lock_waits_fill_table(THD* thd, TABLE_LIST* tables, COND* cond) { +#endif + TOKUDB_DBUG_ENTER(""); + int error; + + tokudb_hton_initialized_lock.lock_read(); + + if (!tokudb_hton_initialized) { + error = ER_PLUGIN_IS_NOT_LOADED; + my_error(error, MYF(0), tokudb_hton_name); + } else { + lock_waits_extra_t e = { thd, tables->table }; + error = db_env->iterate_pending_lock_requests( + db_env, + lock_waits_callback, + &e); + if (error) + my_error(ER_GET_ERRNO, MYF(0), error, tokudb_hton_name); + } + + tokudb_hton_initialized_lock.unlock(); + TOKUDB_DBUG_RETURN(error); +} + +int lock_waits_init(void* p) { + ST_SCHEMA_TABLE* schema = (ST_SCHEMA_TABLE*)p; + schema->fields_info = lock_waits_field_info; + schema->fill_table = lock_waits_fill_table; + return 0; +} + +int lock_waits_done(void *p) { + return 0; +} + +st_mysql_plugin lock_waits = { + MYSQL_INFORMATION_SCHEMA_PLUGIN, + &lock_waits_information_schema, + "TokuDB_lock_waits", + "Percona", + "Percona TokuDB Storage Engine with Fractal Tree(tm) Technology", + PLUGIN_LICENSE_GPL, + lock_waits_init, /* plugin init */ + lock_waits_done, /* plugin deinit */ + TOKUDB_PLUGIN_VERSION, + NULL, /* status variables */ + NULL, /* system variables */ +#ifdef MARIA_PLUGIN_INTERFACE_VERSION + tokudb::sysvars::version, + MariaDB_PLUGIN_MATURITY_STABLE /* maturity */ +#else + NULL, /* config options */ + 0, /* flags */ +#endif +}; + + + +st_mysql_information_schema locks_information_schema = { + MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION +}; + + ST_FIELD_INFO locks_field_info[] = { + {"locks_trx_id", 0, MYSQL_TYPE_LONGLONG, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"locks_mysql_thread_id", 0, MYSQL_TYPE_LONGLONG, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"locks_dname", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"locks_key_left", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"locks_key_right", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"locks_table_schema", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"locks_table_name", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"locks_table_dictionary_name", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, + {NULL, 0, MYSQL_TYPE_NULL, 0, 0, NULL, SKIP_OPEN_TABLE} +}; + +struct locks_extra_t { + THD* thd; + TABLE* table; +}; + +int locks_callback( + DB_TXN* txn, + iterate_row_locks_callback iterate_locks, + void* locks_extra, + void* extra) { + + uint64_t txn_id = txn->id64(txn); + uint64_t client_id = txn->get_client_id(txn); + locks_extra_t* e = reinterpret_cast<struct locks_extra_t*>(extra); + THD* thd = e->thd; + TABLE* table = e->table; + int error = 0; + DB* db; + DBT left_key, right_key; + while (error == 0 && + iterate_locks(&db, &left_key, &right_key, locks_extra) == 0) { + table->field[0]->store(txn_id, false); + table->field[1]->store(client_id, false); + + const char* dname = tokudb_get_index_name(db); + size_t dname_length = strlen(dname); + table->field[2]->store(dname, dname_length, system_charset_info); + + String left_str; + tokudb_pretty_left_key(db, &left_key, &left_str); + table->field[3]->store( + left_str.ptr(), + left_str.length(), + system_charset_info); + + String right_str; + tokudb_pretty_right_key(db, &right_key, &right_str); + table->field[4]->store( + right_str.ptr(), + right_str.length(), + system_charset_info); + + String database_name, table_name, dictionary_name; + tokudb_split_dname(dname, database_name, table_name, dictionary_name); + table->field[5]->store( + database_name.c_ptr(), + database_name.length(), + system_charset_info); + table->field[6]->store( + table_name.c_ptr(), + table_name.length(), + system_charset_info); + table->field[7]->store( + dictionary_name.c_ptr(), + dictionary_name.length(), + system_charset_info); + + error = schema_table_store_record(thd, table); + + if (!error && thd_killed(thd)) + error = ER_QUERY_INTERRUPTED; + } + return error; +} + +#if MYSQL_VERSION_ID >= 50600 +int locks_fill_table(THD* thd, TABLE_LIST* tables, Item* cond) { +#else +int locks_fill_table(THD* thd, TABLE_LIST* tables, COND* cond) { +#endif + TOKUDB_DBUG_ENTER(""); + int error; + + tokudb_hton_initialized_lock.lock_read(); + + if (!tokudb_hton_initialized) { + error = ER_PLUGIN_IS_NOT_LOADED; + my_error(error, MYF(0), tokudb_hton_name); + } else { + locks_extra_t e = { thd, tables->table }; + error = db_env->iterate_live_transactions(db_env, locks_callback, &e); + if (error) + my_error(ER_GET_ERRNO, MYF(0), error, tokudb_hton_name); + } + + tokudb_hton_initialized_lock.unlock(); + TOKUDB_DBUG_RETURN(error); +} + +int locks_init(void* p) { + ST_SCHEMA_TABLE* schema = (ST_SCHEMA_TABLE*)p; + schema->fields_info = locks_field_info; + schema->fill_table = locks_fill_table; + return 0; +} + +int locks_done(void* p) { + return 0; +} + +st_mysql_plugin locks = { + MYSQL_INFORMATION_SCHEMA_PLUGIN, + &locks_information_schema, + "TokuDB_locks", + "Percona", + "Percona TokuDB Storage Engine with Fractal Tree(tm) Technology", + PLUGIN_LICENSE_GPL, + locks_init, /* plugin init */ + locks_done, /* plugin deinit */ + TOKUDB_PLUGIN_VERSION, + NULL, /* status variables */ + NULL, /* system variables */ +#ifdef MARIA_PLUGIN_INTERFACE_VERSION + tokudb::sysvars::version, + MariaDB_PLUGIN_MATURITY_STABLE /* maturity */ +#else + NULL, /* config options */ + 0, /* flags */ +#endif +}; + + + +st_mysql_information_schema file_map_information_schema = { + MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION +}; + +ST_FIELD_INFO file_map_field_info[] = { + {"dictionary_name", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"internal_file_name", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"table_schema", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"table_name", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"table_dictionary_name", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, + {NULL, 0, MYSQL_TYPE_NULL, 0, 0, NULL, SKIP_OPEN_TABLE} +}; + +int report_file_map(TABLE* table, THD* thd) { + int error; + DB_TXN* txn = NULL; + DBC* tmp_cursor = NULL; + DBT curr_key; + DBT curr_val; + memset(&curr_key, 0, sizeof curr_key); + memset(&curr_val, 0, sizeof curr_val); + error = txn_begin(db_env, 0, &txn, DB_READ_UNCOMMITTED, thd); + if (error) { + goto cleanup; + } + error = db_env->get_cursor_for_directory(db_env, txn, &tmp_cursor); + if (error) { + goto cleanup; + } + while (error == 0) { + error = tmp_cursor->c_get(tmp_cursor, &curr_key, &curr_val, DB_NEXT); + if (!error) { + // We store the NULL terminator in the directory so it's included + // in the size. + // See #5789 + // Recalculate and check just to be safe. + const char *dname = (const char *) curr_key.data; + size_t dname_len = strlen(dname); + assert(dname_len == curr_key.size - 1); + table->field[0]->store(dname, dname_len, system_charset_info); + + const char *iname = (const char *) curr_val.data; + size_t iname_len = strlen(iname); + assert(iname_len == curr_val.size - 1); + table->field[1]->store(iname, iname_len, system_charset_info); + + // split the dname + String database_name, table_name, dictionary_name; + tokudb_split_dname( + dname, + database_name, + table_name, + dictionary_name); + table->field[2]->store( + database_name.c_ptr(), + database_name.length(), + system_charset_info); + table->field[3]->store( + table_name.c_ptr(), + table_name.length(), + system_charset_info); + table->field[4]->store( + dictionary_name.c_ptr(), + dictionary_name.length(), + system_charset_info); + + error = schema_table_store_record(thd, table); + } + if (!error && thd_killed(thd)) + error = ER_QUERY_INTERRUPTED; + } + if (error == DB_NOTFOUND) { + error = 0; + } +cleanup: + if (tmp_cursor) { + int r = tmp_cursor->c_close(tmp_cursor); + assert(r == 0); + } + if (txn) { + commit_txn(txn, 0); + } + return error; +} + +#if MYSQL_VERSION_ID >= 50600 +int file_map_fill_table(THD* thd, TABLE_LIST* tables, Item* cond) { +#else +int file_map_fill_table(THD* thd, TABLE_LIST* tables, COND* cond) { +#endif + TOKUDB_DBUG_ENTER(""); + int error; + TABLE* table = tables->table; + + tokudb_hton_initialized_lock.lock_read(); + + if (!tokudb_hton_initialized) { + error = ER_PLUGIN_IS_NOT_LOADED; + my_error(error, MYF(0), tokudb_hton_name); + } else { + error = report_file_map(table, thd); + if (error) + my_error(ER_GET_ERRNO, MYF(0), error, tokudb_hton_name); + } + + tokudb_hton_initialized_lock.unlock(); + TOKUDB_DBUG_RETURN(error); +} + +int file_map_init(void* p) { + ST_SCHEMA_TABLE* schema = (ST_SCHEMA_TABLE*)p; + schema->fields_info = file_map_field_info; + schema->fill_table = file_map_fill_table; + return 0; +} + +int file_map_done(void* p) { + return 0; +} + +st_mysql_plugin file_map = { + MYSQL_INFORMATION_SCHEMA_PLUGIN, + &file_map_information_schema, + "TokuDB_file_map", + "Percona", + "Percona TokuDB Storage Engine with Fractal Tree(tm) Technology", + PLUGIN_LICENSE_GPL, + file_map_init, /* plugin init */ + file_map_done, /* plugin deinit */ + TOKUDB_PLUGIN_VERSION, + NULL, /* status variables */ + NULL, /* system variables */ +#ifdef MARIA_PLUGIN_INTERFACE_VERSION + tokudb::sysvars::version, + MariaDB_PLUGIN_MATURITY_STABLE /* maturity */ +#else + NULL, /* config options */ + 0, /* flags */ +#endif +}; + + + +st_mysql_information_schema fractal_tree_info_information_schema = { + MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION +}; + +ST_FIELD_INFO fractal_tree_info_field_info[] = { + {"dictionary_name", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"internal_file_name", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"bt_num_blocks_allocated", 0, MYSQL_TYPE_LONGLONG, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"bt_num_blocks_in_use", 0, MYSQL_TYPE_LONGLONG, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"bt_size_allocated", 0, MYSQL_TYPE_LONGLONG, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"bt_size_in_use", 0, MYSQL_TYPE_LONGLONG, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"table_schema", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"table_name", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"table_dictionary_name", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, + {NULL, 0, MYSQL_TYPE_NULL, 0, 0, NULL, SKIP_OPEN_TABLE} +}; + +int report_fractal_tree_info_for_db( + const DBT* dname, + const DBT* iname, + TABLE* table, + THD* thd) { + + int error; + uint64_t bt_num_blocks_allocated; + uint64_t bt_num_blocks_in_use; + uint64_t bt_size_allocated; + uint64_t bt_size_in_use; + + DB *db = NULL; + error = db_create(&db, db_env, 0); + if (error) { + goto exit; + } + error = db->open(db, NULL, (char *)dname->data, NULL, DB_BTREE, 0, 0666); + if (error) { + goto exit; + } + error = db->get_fractal_tree_info64( + db, + &bt_num_blocks_allocated, + &bt_num_blocks_in_use, + &bt_size_allocated, + &bt_size_in_use); + if (error) { + goto exit; + } + + // We store the NULL terminator in the directory so it's included in the + // size. + // See #5789 + // Recalculate and check just to be safe. + { + size_t dname_len = strlen((const char*)dname->data); + assert(dname_len == dname->size - 1); + table->field[0]->store( + (char*)dname->data, + dname_len, + system_charset_info); + size_t iname_len = strlen((const char*)iname->data); + assert(iname_len == iname->size - 1); + table->field[1]->store( + (char*)iname->data, + iname_len, + system_charset_info); + } + table->field[2]->store(bt_num_blocks_allocated, false); + table->field[3]->store(bt_num_blocks_in_use, false); + table->field[4]->store(bt_size_allocated, false); + table->field[5]->store(bt_size_in_use, false); + + // split the dname + { + String database_name, table_name, dictionary_name; + tokudb_split_dname( + (const char*)dname->data, + database_name, + table_name, + dictionary_name); + table->field[6]->store( + database_name.c_ptr(), + database_name.length(), + system_charset_info); + table->field[7]->store( + table_name.c_ptr(), + table_name.length(), + system_charset_info); + table->field[8]->store( + dictionary_name.c_ptr(), + dictionary_name.length(), + system_charset_info); + } + error = schema_table_store_record(thd, table); + +exit: + if (db) { + int close_error = db->close(db, 0); + if (error == 0) + error = close_error; + } + return error; +} + +int report_fractal_tree_info(TABLE* table, THD* thd) { + int error; + DB_TXN* txn = NULL; + DBC* tmp_cursor = NULL; + DBT curr_key; + DBT curr_val; + memset(&curr_key, 0, sizeof curr_key); + memset(&curr_val, 0, sizeof curr_val); + error = txn_begin(db_env, 0, &txn, DB_READ_UNCOMMITTED, thd); + if (error) { + goto cleanup; + } + error = db_env->get_cursor_for_directory(db_env, txn, &tmp_cursor); + if (error) { + goto cleanup; + } + while (error == 0) { + error = tmp_cursor->c_get(tmp_cursor, &curr_key, &curr_val, DB_NEXT); + if (!error) { + error = report_fractal_tree_info_for_db( + &curr_key, + &curr_val, + table, + thd); + if (error) + error = 0; // ignore read uncommitted errors + } + if (!error && thd_killed(thd)) + error = ER_QUERY_INTERRUPTED; + } + if (error == DB_NOTFOUND) { + error = 0; + } +cleanup: + if (tmp_cursor) { + int r = tmp_cursor->c_close(tmp_cursor); + assert(r == 0); + } + if (txn) { + commit_txn(txn, 0); + } + return error; +} + +#if MYSQL_VERSION_ID >= 50600 +int fractal_tree_info_fill_table(THD* thd, TABLE_LIST* tables, Item* cond) { +#else +int fractal_tree_info_fill_table(THD* thd, TABLE_LIST* tables, COND* cond) { +#endif + TOKUDB_DBUG_ENTER(""); + int error; + TABLE* table = tables->table; + + // 3938: Get a read lock on the status flag, since we must + // read it before safely proceeding + tokudb_hton_initialized_lock.lock_read(); + + if (!tokudb_hton_initialized) { + error = ER_PLUGIN_IS_NOT_LOADED; + my_error(error, MYF(0), tokudb_hton_name); + } else { + error = report_fractal_tree_info(table, thd); + if (error) + my_error(ER_GET_ERRNO, MYF(0), error, tokudb_hton_name); + } + + //3938: unlock the status flag lock + tokudb_hton_initialized_lock.unlock(); + TOKUDB_DBUG_RETURN(error); +} + +int fractal_tree_info_init(void* p) { + ST_SCHEMA_TABLE* schema = (ST_SCHEMA_TABLE*)p; + schema->fields_info = fractal_tree_info_field_info; + schema->fill_table = fractal_tree_info_fill_table; + return 0; +} + +int fractal_tree_info_done(void* p) { + return 0; +} + +st_mysql_plugin fractal_tree_info = { + MYSQL_INFORMATION_SCHEMA_PLUGIN, + &fractal_tree_info_information_schema, + "TokuDB_fractal_tree_info", + "Percona", + "Percona TokuDB Storage Engine with Fractal Tree(tm) Technology", + PLUGIN_LICENSE_GPL, + fractal_tree_info_init, /* plugin init */ + fractal_tree_info_done, /* plugin deinit */ + TOKUDB_PLUGIN_VERSION, + NULL, /* status variables */ + NULL, /* system variables */ +#ifdef MARIA_PLUGIN_INTERFACE_VERSION + tokudb::sysvars::version, + MariaDB_PLUGIN_MATURITY_STABLE /* maturity */ +#else + NULL, /* config options */ + 0, /* flags */ +#endif +}; + + + +st_mysql_information_schema fractal_tree_block_map_information_schema = { + MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION +}; + +ST_FIELD_INFO fractal_tree_block_map_field_info[] = { + {"dictionary_name", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"internal_file_name", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"checkpoint_count", 0, MYSQL_TYPE_LONGLONG, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"blocknum", 0, MYSQL_TYPE_LONGLONG, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"offset", 0, MYSQL_TYPE_LONGLONG, 0, MY_I_S_MAYBE_NULL, NULL, SKIP_OPEN_TABLE }, + {"size", 0, MYSQL_TYPE_LONGLONG, 0, MY_I_S_MAYBE_NULL, NULL, SKIP_OPEN_TABLE }, + {"table_schema", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"table_name", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"table_dictionary_name", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, + {NULL, 0, MYSQL_TYPE_NULL, 0, 0, NULL, SKIP_OPEN_TABLE} +}; + +struct report_fractal_tree_block_map_iterator_extra_t { + int64_t num_rows; + int64_t i; + uint64_t* checkpoint_counts; + int64_t* blocknums; + int64_t* diskoffs; + int64_t* sizes; +}; + +// This iterator is called while holding the blocktable lock. +// We should be as quick as possible. +// We don't want to do one call to get the number of rows, release the +// blocktable lock, and then do another call to get all the rows because +// the number of rows may change if we don't hold the lock. +// As a compromise, we'll do some mallocs inside the lock on the first call, +// but everything else should be fast. +int report_fractal_tree_block_map_iterator( + uint64_t checkpoint_count, + int64_t num_rows, + int64_t blocknum, + int64_t diskoff, + int64_t size, + void* iter_extra) { + + struct report_fractal_tree_block_map_iterator_extra_t* e = + static_cast<struct report_fractal_tree_block_map_iterator_extra_t*>(iter_extra); + + assert(num_rows > 0); + if (e->num_rows == 0) { + e->checkpoint_counts = + (uint64_t*)tokudb::memory::malloc( + num_rows * (sizeof *e->checkpoint_counts), + MYF(MY_WME|MY_ZEROFILL|MY_FAE)); + e->blocknums = + (int64_t*)tokudb::memory::malloc( + num_rows * (sizeof *e->blocknums), + MYF(MY_WME|MY_ZEROFILL|MY_FAE)); + e->diskoffs = + (int64_t*)tokudb::memory::malloc( + num_rows * (sizeof *e->diskoffs), + MYF(MY_WME|MY_ZEROFILL|MY_FAE)); + e->sizes = + (int64_t*)tokudb::memory::malloc( + num_rows * (sizeof *e->sizes), + MYF(MY_WME|MY_ZEROFILL|MY_FAE)); + e->num_rows = num_rows; + } + + e->checkpoint_counts[e->i] = checkpoint_count; + e->blocknums[e->i] = blocknum; + e->diskoffs[e->i] = diskoff; + e->sizes[e->i] = size; + ++(e->i); + + return 0; +} + +int report_fractal_tree_block_map_for_db( + const DBT* dname, + const DBT* iname, + TABLE* table, + THD* thd) { + + int error; + DB* db; + // avoid struct initializers so that we can compile with older gcc versions + report_fractal_tree_block_map_iterator_extra_t e = {}; + + error = db_create(&db, db_env, 0); + if (error) { + goto exit; + } + error = db->open(db, NULL, (char *)dname->data, NULL, DB_BTREE, 0, 0666); + if (error) { + goto exit; + } + error = db->iterate_fractal_tree_block_map( + db, + report_fractal_tree_block_map_iterator, + &e); + { + int close_error = db->close(db, 0); + if (!error) { + error = close_error; + } + } + if (error) { + goto exit; + } + + // If not, we should have gotten an error and skipped this section of code + assert(e.i == e.num_rows); + for (int64_t i = 0; error == 0 && i < e.num_rows; ++i) { + // We store the NULL terminator in the directory so it's included in the size. + // See #5789 + // Recalculate and check just to be safe. + size_t dname_len = strlen((const char*)dname->data); + assert(dname_len == dname->size - 1); + table->field[0]->store( + (char*)dname->data, + dname_len, + system_charset_info); + + size_t iname_len = strlen((const char*)iname->data); + assert(iname_len == iname->size - 1); + table->field[1]->store( + (char*)iname->data, + iname_len, + system_charset_info); + + table->field[2]->store(e.checkpoint_counts[i], false); + table->field[3]->store(e.blocknums[i], false); + static const int64_t freelist_null = -1; + static const int64_t diskoff_unused = -2; + if (e.diskoffs[i] == diskoff_unused || e.diskoffs[i] == freelist_null) { + table->field[4]->set_null(); + } else { + table->field[4]->set_notnull(); + table->field[4]->store(e.diskoffs[i], false); + } + static const int64_t size_is_free = -1; + if (e.sizes[i] == size_is_free) { + table->field[5]->set_null(); + } else { + table->field[5]->set_notnull(); + table->field[5]->store(e.sizes[i], false); + } + + // split the dname + String database_name, table_name, dictionary_name; + tokudb_split_dname( + (const char*)dname->data, + database_name, + table_name, + dictionary_name); + table->field[6]->store( + database_name.c_ptr(), + database_name.length(), + system_charset_info); + table->field[7]->store( + table_name.c_ptr(), + table_name.length(), + system_charset_info); + table->field[8]->store( + dictionary_name.c_ptr(), + dictionary_name.length(), + system_charset_info); + + error = schema_table_store_record(thd, table); + } + +exit: + if (e.checkpoint_counts != NULL) { + tokudb::memory::free(e.checkpoint_counts); + e.checkpoint_counts = NULL; + } + if (e.blocknums != NULL) { + tokudb::memory::free(e.blocknums); + e.blocknums = NULL; + } + if (e.diskoffs != NULL) { + tokudb::memory::free(e.diskoffs); + e.diskoffs = NULL; + } + if (e.sizes != NULL) { + tokudb::memory::free(e.sizes); + e.sizes = NULL; + } + return error; +} + +int report_fractal_tree_block_map(TABLE* table, THD* thd) { + int error; + DB_TXN* txn = NULL; + DBC* tmp_cursor = NULL; + DBT curr_key; + DBT curr_val; + memset(&curr_key, 0, sizeof curr_key); + memset(&curr_val, 0, sizeof curr_val); + error = txn_begin(db_env, 0, &txn, DB_READ_UNCOMMITTED, thd); + if (error) { + goto cleanup; + } + error = db_env->get_cursor_for_directory(db_env, txn, &tmp_cursor); + if (error) { + goto cleanup; + } + while (error == 0) { + error = tmp_cursor->c_get(tmp_cursor, &curr_key, &curr_val, DB_NEXT); + if (!error) { + error = report_fractal_tree_block_map_for_db( + &curr_key, + &curr_val, + table, + thd); + } + if (!error && thd_killed(thd)) + error = ER_QUERY_INTERRUPTED; + } + if (error == DB_NOTFOUND) { + error = 0; + } +cleanup: + if (tmp_cursor) { + int r = tmp_cursor->c_close(tmp_cursor); + assert(r == 0); + } + if (txn) { + commit_txn(txn, 0); + } + return error; +} + +#if MYSQL_VERSION_ID >= 50600 +int fractal_tree_block_map_fill_table( + THD* thd, + TABLE_LIST* tables, + Item* cond) { +#else +int fractal_tree_block_map_fill_table( + THD* thd, + TABLE_LIST* tables, + COND* cond) { +#endif + TOKUDB_DBUG_ENTER(""); + int error; + TABLE* table = tables->table; + + // 3938: Get a read lock on the status flag, since we must + // read it before safely proceeding + tokudb_hton_initialized_lock.lock_read(); + + if (!tokudb_hton_initialized) { + error = ER_PLUGIN_IS_NOT_LOADED; + my_error(error, MYF(0), tokudb_hton_name); + } else { + error = report_fractal_tree_block_map(table, thd); + if (error) + my_error(ER_GET_ERRNO, MYF(0), error, tokudb_hton_name); + } + + //3938: unlock the status flag lock + tokudb_hton_initialized_lock.unlock(); + TOKUDB_DBUG_RETURN(error); +} + +int fractal_tree_block_map_init(void* p) { + ST_SCHEMA_TABLE* schema = (ST_SCHEMA_TABLE*)p; + schema->fields_info = fractal_tree_block_map_field_info; + schema->fill_table = fractal_tree_block_map_fill_table; + return 0; +} + +int fractal_tree_block_map_done(void *p) { + return 0; +} + +st_mysql_plugin fractal_tree_block_map = { + MYSQL_INFORMATION_SCHEMA_PLUGIN, + &fractal_tree_block_map_information_schema, + "TokuDB_fractal_tree_block_map", + "Percona", + "Percona TokuDB Storage Engine with Fractal Tree(tm) Technology", + PLUGIN_LICENSE_GPL, + fractal_tree_block_map_init, /* plugin init */ + fractal_tree_block_map_done, /* plugin deinit */ + TOKUDB_PLUGIN_VERSION, + NULL, /* status variables */ + NULL, /* system variables */ +#ifdef MARIA_PLUGIN_INTERFACE_VERSION + tokudb::sysvars::version, + MariaDB_PLUGIN_MATURITY_STABLE /* maturity */ +#else + NULL, /* config options */ + 0, /* flags */ +#endif +}; + + +st_mysql_information_schema background_job_status_information_schema = { + MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION +}; + +ST_FIELD_INFO background_job_status_field_info[] = { + {"id", 0, MYSQL_TYPE_LONGLONG, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"database_name", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"table_name", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"job_type", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"job_params", 256, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"scheduler", 32, MYSQL_TYPE_STRING, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"scheduled_time", 0, MYSQL_TYPE_DATETIME, 0, 0, NULL, SKIP_OPEN_TABLE }, + {"started_time", 0, MYSQL_TYPE_DATETIME, 0, MY_I_S_MAYBE_NULL, NULL, SKIP_OPEN_TABLE }, + {"status", 256, MYSQL_TYPE_STRING, 0, MY_I_S_MAYBE_NULL, SKIP_OPEN_TABLE }, + {NULL, 0, MYSQL_TYPE_NULL, 0, 0, NULL, SKIP_OPEN_TABLE} +}; + +struct background_job_status_extra { + THD* thd; + TABLE* table; +}; + +void background_job_status_callback( + uint64_t id, + const char* database_name, + const char* table_name, + const char* type, + const char* params, + const char* status, + bool user_scheduled, + time_t scheduled_time, + time_t started_time, + void* extra) { + + background_job_status_extra* e = + reinterpret_cast<background_job_status_extra*>(extra); + + THD* thd = e->thd; + TABLE* table = e->table; + + table->field[0]->store(id, false); + table->field[1]->store( + database_name, + strlen(database_name), + system_charset_info); + table->field[2]->store(table_name, strlen(table_name), system_charset_info); + table->field[3]->store(type, strlen(type), system_charset_info); + table->field[4]->store(params, strlen(params), system_charset_info); + if (user_scheduled) + table->field[5]->store("USER", sizeof("USER"), system_charset_info); + else + table->field[5]->store("AUTO", sizeof("AUTO"), system_charset_info); + + field_store_time_t(table->field[6], scheduled_time); + field_store_time_t(table->field[7], started_time); + if (status[0] != '\0') { + table->field[8]->store(status, strlen(status), system_charset_info); + table->field[8]->set_notnull(); + } else { + table->field[8]->store(NULL, 0, system_charset_info); + table->field[8]->set_null(); + } + + schema_table_store_record(thd, table); +} + +int report_background_job_status(TABLE *table, THD *thd) { + int error = 0; + background_job_status_extra extra = { + thd, + table + }; + tokudb::background::_job_manager->iterate_jobs( + background_job_status_callback, + &extra); + return error; +} + +#if MYSQL_VERSION_ID >= 50600 +int background_job_status_fill_table(THD *thd, TABLE_LIST *tables, Item *cond) { +#else +int background_job_status_fill_table(THD *thd, TABLE_LIST *tables, COND *cond) { +#endif + TOKUDB_DBUG_ENTER(""); + int error; + TABLE* table = tables->table; + + tokudb_hton_initialized_lock.lock_read(); + + if (!tokudb_hton_initialized) { + error = ER_PLUGIN_IS_NOT_LOADED; + my_error(error, MYF(0), tokudb_hton_name); + } else { + error = report_background_job_status(table, thd); + if (error) + my_error(ER_GET_ERRNO, MYF(0), error, tokudb_hton_name); + } + + tokudb_hton_initialized_lock.unlock(); + TOKUDB_DBUG_RETURN(error); +} + +int background_job_status_init(void* p) { + ST_SCHEMA_TABLE* schema = (ST_SCHEMA_TABLE*)p; + schema->fields_info = background_job_status_field_info; + schema->fill_table = background_job_status_fill_table; + return 0; +} + +int background_job_status_done(void* p) { + return 0; +} + +st_mysql_plugin background_job_status = { + MYSQL_INFORMATION_SCHEMA_PLUGIN, + &background_job_status_information_schema, + "TokuDB_background_job_status", + "Percona", + "Percona TokuDB Storage Engine with Fractal Tree(tm) Technology", + PLUGIN_LICENSE_GPL, + background_job_status_init, /* plugin init */ + background_job_status_done, /* plugin deinit */ + TOKUDB_PLUGIN_VERSION, + NULL, /* status variables */ + NULL, /* system variables */ +#ifdef MARIA_PLUGIN_INTERFACE_VERSION + tokudb::sysvars::version, + MariaDB_PLUGIN_MATURITY_STABLE /* maturity */ +#else + NULL, /* config options */ + 0, /* flags */ +#endif +}; + +} // namespace information_schema +} // namespace tokudb diff --git a/storage/tokudb/tokudb_information_schema.h b/storage/tokudb/tokudb_information_schema.h new file mode 100644 index 00000000000..7f42d68ef59 --- /dev/null +++ b/storage/tokudb/tokudb_information_schema.h @@ -0,0 +1,46 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +/* -*- mode: C; c-basic-offset: 4 -*- */ +#ident "$Id$" +/*====== +This file is part of TokuDB + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + TokuDBis is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + TokuDB is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TokuDB. If not, see <http://www.gnu.org/licenses/>. + +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#ifndef _TOKUDB_INFORMATION_SCHEMA_H +#define _TOKUDB_INFORMATION_SCHEMA_H + +#include "hatoku_defines.h" + +namespace tokudb { +namespace information_schema { + +extern st_mysql_plugin trx; +extern st_mysql_plugin lock_waits; +extern st_mysql_plugin locks; +extern st_mysql_plugin file_map; +extern st_mysql_plugin fractal_tree_info; +extern st_mysql_plugin fractal_tree_block_map; +extern st_mysql_plugin background_job_status; + +} // namespace information_schema +} // namespace tokudb + +#endif // _TOKUDB_INFORMATION_SCHEMA_H diff --git a/storage/tokudb/tokudb_math.h b/storage/tokudb/tokudb_math.h index c49af2a6a21..0338bf3871e 100644 --- a/storage/tokudb/tokudb_math.h +++ b/storage/tokudb/tokudb_math.h @@ -32,30 +32,34 @@ namespace tokudb { // Overflow detection adapted from "Hackers Delight", Henry S. Warren // Return a bit mask for bits 0 .. length_bits-1 -static uint64_t uint_mask(uint length_bits) __attribute__((unused)); +TOKUDB_UNUSED(static uint64_t uint_mask(uint length_bits)); static uint64_t uint_mask(uint length_bits) { return length_bits == 64 ? ~0ULL : (1ULL<<length_bits)-1; } // Return the highest unsigned int with a given number of bits -static uint64_t uint_high_endpoint(uint length_bits) __attribute__((unused)); +TOKUDB_UNUSED(static uint64_t uint_high_endpoint(uint length_bits)); static uint64_t uint_high_endpoint(uint length_bits) { return uint_mask(length_bits); } // Return the lowest unsigned int with a given number of bits -static uint64_t uint_low_endpoint(uint length_bits) __attribute__((unused)); -static uint64_t uint_low_endpoint(uint length_bits __attribute__((unused))) { +TOKUDB_UNUSED(static uint64_t uint_low_endpoint(uint length_bits)); +static uint64_t uint_low_endpoint(TOKUDB_UNUSED(uint length_bits)) { return 0; } // Add two unsigned integers with max maximum value. // If there is an overflow then set the sum to the max. // Return the sum and the overflow. -static uint64_t uint_add(uint64_t x, uint64_t y, uint length_bits, bool *over) __attribute__((unused)); +TOKUDB_UNUSED(static uint64_t uint_add( + uint64_t x, + uint64_t y, + uint length_bits, + bool* over)); static uint64_t uint_add(uint64_t x, uint64_t y, uint length_bits, bool *over) { uint64_t mask = uint_mask(length_bits); - assert((x & ~mask) == 0 && (y & ~mask) == 0); + assert_always((x & ~mask) == 0 && (y & ~mask) == 0); uint64_t s = (x + y) & mask; *over = s < x; // check for overflow return s; @@ -64,30 +68,34 @@ static uint64_t uint_add(uint64_t x, uint64_t y, uint length_bits, bool *over) { // Subtract two unsigned ints with max maximum value. // If there is an over then set the difference to 0. // Return the difference and the overflow. -static uint64_t uint_sub(uint64_t x, uint64_t y, uint length_bits, bool *over) __attribute__((unused)); +TOKUDB_UNUSED(static uint64_t uint_sub( + uint64_t x, + uint64_t y, + uint length_bits, + bool* over)); static uint64_t uint_sub(uint64_t x, uint64_t y, uint length_bits, bool *over) { uint64_t mask = uint_mask(length_bits); - assert((x & ~mask) == 0 && (y & ~mask) == 0); + assert_always((x & ~mask) == 0 && (y & ~mask) == 0); uint64_t s = (x - y) & mask; *over = s > x; // check for overflow return s; } // Return the highest int with a given number of bits -static int64_t int_high_endpoint(uint length_bits) __attribute__((unused)); +TOKUDB_UNUSED(static int64_t int_high_endpoint(uint length_bits)); static int64_t int_high_endpoint(uint length_bits) { return (1ULL<<(length_bits-1))-1; } // Return the lowest int with a given number of bits -static int64_t int_low_endpoint(uint length_bits) __attribute__((unused)); +TOKUDB_UNUSED(static int64_t int_low_endpoint(uint length_bits)); static int64_t int_low_endpoint(uint length_bits) { int64_t mask = uint_mask(length_bits); return (1ULL<<(length_bits-1)) | ~mask; } // Sign extend to 64 bits an int with a given number of bits -static int64_t int_sign_extend(int64_t n, uint length_bits) __attribute__((unused)); +TOKUDB_UNUSED(static int64_t int_sign_extend(int64_t n, uint length_bits)); static int64_t int_sign_extend(int64_t n, uint length_bits) { if (n & (1ULL<<(length_bits-1))) n |= ~uint_mask(length_bits); @@ -99,7 +107,11 @@ static int64_t int_sign_extend(int64_t n, uint length_bits) { // depending on the sign bit. // Sign extend to 64 bits. // Return the sum and the overflow. -static int64_t int_add(int64_t x, int64_t y, uint length_bits, bool *over) __attribute__((unused)); +TOKUDB_UNUSED(static int64_t int_add( + int64_t x, + int64_t y, + uint length_bits, + bool* over)); static int64_t int_add(int64_t x, int64_t y, uint length_bits, bool *over) { int64_t mask = uint_mask(length_bits); int64_t n = (x + y) & mask; @@ -114,7 +126,11 @@ static int64_t int_add(int64_t x, int64_t y, uint length_bits, bool *over) { // depending on the sign bit. // Sign extend to 64 bits. // Return the sum and the overflow. -static int64_t int_sub(int64_t x, int64_t y, uint length_bits, bool *over) __attribute__((unused)); +TOKUDB_UNUSED(static int64_t int_sub( + int64_t x, + int64_t y, + uint length_bits, + bool* over)); static int64_t int_sub(int64_t x, int64_t y, uint length_bits, bool *over) { int64_t mask = uint_mask(length_bits); int64_t n = (x - y) & mask; diff --git a/storage/tokudb/tokudb_memory.h b/storage/tokudb/tokudb_memory.h new file mode 100644 index 00000000000..26587de6062 --- /dev/null +++ b/storage/tokudb/tokudb_memory.h @@ -0,0 +1,102 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +/* -*- mode: C; c-basic-offset: 4 -*- */ +#ident "$Id$" +/*====== +This file is part of TokuDB + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + TokuDBis is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + TokuDB is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TokuDB. If not, see <http://www.gnu.org/licenses/>. + +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#ifndef _TOKUDB_MEMORY_H +#define _TOKUDB_MEMORY_H + +#include "hatoku_defines.h" + +namespace tokudb { +namespace memory { + +void* malloc(size_t s, myf flags); +void* realloc(void* p, size_t s, myf flags); +void free(void* ptr); +char* strdup(const char* p, myf flags); +void* multi_malloc(myf myFlags, ...); + + +inline void* malloc(size_t s, myf flags) { +#if 50700 <= MYSQL_VERSION_ID && MYSQL_VERSION_ID <= 50799 + return ::my_malloc(0, s, flags); +#else + return ::my_malloc(s, flags); +#endif +} +inline void* realloc(void* p, size_t s, myf flags) { + if (s == 0) + return p; +#if 50700 <= MYSQL_VERSION_ID && MYSQL_VERSION_ID <= 50799 + return ::my_realloc(0, p, s, flags); +#else + return ::my_realloc(p, s, flags); +#endif +} +inline void free(void* ptr) { + if (ptr) + ::my_free(ptr); +} +inline char* strdup(const char* p, myf flags) { +#if 50700 <= MYSQL_VERSION_ID && MYSQL_VERSION_ID <= 50799 + return ::my_strdup(0, p, flags); +#else + return ::my_strdup(p, flags); +#endif +} +inline void* multi_malloc(myf myFlags, ...) { + va_list args; + char** ptr; + char* start; + char* res; + size_t tot_length,length; + + va_start(args,myFlags); + tot_length = 0; + while ((ptr = va_arg(args, char**))) { + length = va_arg(args, uint); + tot_length += ALIGN_SIZE(length); + } + va_end(args); + + if (!(start = (char*)malloc(tot_length, myFlags))) { + return 0; + } + + va_start(args, myFlags); + res = start; + while ((ptr = va_arg(args, char**))) { + *ptr = res; + length = va_arg(args,uint); + res += ALIGN_SIZE(length); + } + va_end(args); + return start; +} + +} // namespace thread +} // namespace tokudb + +#endif // _TOKUDB_MEMORY_H diff --git a/storage/tokudb/tokudb_status.h b/storage/tokudb/tokudb_status.h index 4600e04596f..5cca54e52c9 100644 --- a/storage/tokudb/tokudb_status.h +++ b/storage/tokudb/tokudb_status.h @@ -43,118 +43,374 @@ typedef ulonglong HA_METADATA_KEY; #define status_dict_pagesize 1024 namespace tokudb { +namespace metadata { - // get the value for a given key in the status dictionary. copy the value to the supplied buffer. - // returns 0 if successful. - int get_status(DB *status_db, DB_TXN *txn, HA_METADATA_KEY k, void *p, size_t s, size_t *sp) { - DBT key = {}; key.data = &k; key.size = sizeof k; - DBT val = {}; val.data = p; val.ulen = (uint32_t) s; val.flags = DB_DBT_USERMEM; - int error = status_db->get(status_db, txn, &key, &val, 0); - if (error == 0) { - *sp = val.size; - } - return error; - } +// get the value for a given key in the status dictionary. +// copy the value to the supplied buffer. +// returns 0 if successful. +int read( + DB* status_db, + DB_TXN* txn, + HA_METADATA_KEY k, + void* p, + size_t s, + size_t* sp) { - // get the value for a given key in the status dictionary. put the value in a realloced buffer. - // returns 0 if successful. - int get_status_realloc(DB *status_db, DB_TXN *txn, HA_METADATA_KEY k, void **pp, size_t *sp) { - DBT key = {}; key.data = &k; key.size = sizeof k; - DBT val = {}; val.data = *pp; val.size = (uint32_t) *sp; val.flags = DB_DBT_REALLOC; - int error = status_db->get(status_db, txn, &key, &val, 0); - if (error == 0) { - *pp = val.data; - *sp = val.size; - } - return error; + DBT key = {}; + key.data = &k; + key.size = sizeof(k); + DBT val = {}; + val.data = p; + val.ulen = (uint32_t)s; + val.flags = DB_DBT_USERMEM; + int error = status_db->get(status_db, txn, &key, &val, 0); + if (error == 0) { + *sp = val.size; } + return error; +} - // write a key value pair into the status dictionary, overwriting the previous value if any. - // auto create a txn if necessary. - // returns 0 if successful. - int write_metadata(DB *status_db, void *key_data, uint key_size, void* val_data, uint val_size, DB_TXN *txn) { - DBT key = {}; key.data = key_data; key.size = key_size; - DBT value = {}; value.data = val_data; value.size = val_size; - int error = status_db->put(status_db, txn, &key, &value, 0); - return error; - } +// get the value for a given key in the status dictionary. +// put the value in a realloced buffer. +// returns 0 if successful. +int read_realloc( + DB* status_db, + DB_TXN* txn, + HA_METADATA_KEY k, + void** pp, + size_t* sp) { - // write a key value pair into the status dictionary, overwriting the previous value if any. - // the key must be a HA_METADATA_KEY. - // returns 0 if successful. - int write_to_status(DB *status_db, HA_METADATA_KEY curr_key_data, void *val, size_t val_size, DB_TXN *txn) { - return write_metadata(status_db, &curr_key_data, sizeof curr_key_data, val, val_size, txn); + DBT key = {}; + key.data = &k; + key.size = sizeof(k); + DBT val = {}; + val.data = *pp; + val.size = (uint32_t)*sp; + val.flags = DB_DBT_REALLOC; + int error = status_db->get(status_db, txn, &key, &val, 0); + if (error == 0) { + *pp = val.data; + *sp = val.size; } + return error; +} - // remove a key from the status dictionary. - // auto create a txn if necessary. - // returns 0 if successful. - int remove_metadata(DB *status_db, void *key_data, uint key_size, DB_TXN *txn) { - DBT key = {}; key.data = key_data; key.size = key_size; - int error = status_db->del(status_db, txn, &key, DB_DELETE_ANY); - return error; - } +// write a key value pair into the status dictionary, +// overwriting the previous value if any. +// auto create a txn if necessary. +// returns 0 if successful. +int write_low( + DB* status_db, + void* key_data, + uint key_size, + void* val_data, + uint val_size, + DB_TXN *txn) { - // remove a key from the status dictionary. - // the key must be a HA_METADATA_KEY - // returns 0 if successful. - int remove_from_status(DB *status_db, HA_METADATA_KEY curr_key_data, DB_TXN *txn) { - return remove_metadata(status_db, &curr_key_data, sizeof curr_key_data, txn); + DBT key = {}; + key.data = key_data; + key.size = key_size; + DBT value = {}; + value.data = val_data; + value.size = val_size; + int error = status_db->put(status_db, txn, &key, &value, 0); + return error; +} + +// write a key value pair into the status dictionary, +// overwriting the previous value if any. +// the key must be a HA_METADATA_KEY. +// returns 0 if successful. +int write( + DB* status_db, + HA_METADATA_KEY curr_key_data, + void* val, + size_t val_size, + DB_TXN* txn) { + + return + tokudb::metadata::write_low( + status_db, + &curr_key_data, + sizeof(curr_key_data), + val, + val_size, + txn); +} + +// remove a key from the status dictionary. +// auto create a txn if necessary. +// returns 0 if successful. +int remove_low( + DB* status_db, + void* key_data, + uint key_size, + DB_TXN* txn) { + + DBT key = {}; + key.data = key_data; + key.size = key_size; + int error = status_db->del(status_db, txn, &key, DB_DELETE_ANY); + return error; +} + +// remove a key from the status dictionary. +// the key must be a HA_METADATA_KEY +// returns 0 if successful. +int remove( + DB* status_db, + HA_METADATA_KEY curr_key_data, + DB_TXN* txn) { + return + tokudb::metadata::remove_low( + status_db, + &curr_key_data, + sizeof(curr_key_data), + txn); +} + +int close(DB** status_db_ptr) { + int error = 0; + DB* status_db = *status_db_ptr; + if (status_db) { + error = status_db->close(status_db, 0); + if (error == 0) + *status_db_ptr = NULL; } + return error; +} - int close_status(DB **status_db_ptr) { - int error = 0; - DB *status_db = *status_db_ptr; - if (status_db) { - error = status_db->close(status_db, 0); - if (error == 0) - *status_db_ptr = NULL; - } - return error; +int create( + DB_ENV* env, + DB** status_db_ptr, + const char* name, + DB_TXN* txn) { + + int error; + DB *status_db = NULL; + + error = db_create(&status_db, env, 0); + if (error == 0) { + error = status_db->set_pagesize(status_db, status_dict_pagesize); } + if (error == 0) { + error = + status_db->open( + status_db, + txn, + name, + NULL, + DB_BTREE, DB_CREATE | DB_EXCL, + 0); + } + if (error == 0) { + *status_db_ptr = status_db; + } else { + int r = tokudb::metadata::close(&status_db); + assert_always(r == 0); + } + return error; +} - int create_status(DB_ENV *env, DB **status_db_ptr, const char *name, DB_TXN *txn) { - int error; - DB *status_db = NULL; +int open( + DB_ENV* env, + DB** status_db_ptr, + const char* name, + DB_TXN* txn) { - error = db_create(&status_db, env, 0); - if (error == 0) { - error = status_db->set_pagesize(status_db, status_dict_pagesize); - } - if (error == 0) { - error = status_db->open(status_db, txn, name, NULL, DB_BTREE, DB_CREATE | DB_EXCL, 0); - } - if (error == 0) { - *status_db_ptr = status_db; - } else { - int r = close_status(&status_db); - assert(r == 0); + int error = 0; + DB* status_db = NULL; + error = db_create(&status_db, env, 0); + if (error == 0) { + error = + status_db->open( + status_db, + txn, + name, + NULL, + DB_BTREE, + DB_THREAD, + 0); + } + if (error == 0) { + uint32_t pagesize = 0; + error = status_db->get_pagesize(status_db, &pagesize); + if (error == 0 && pagesize > status_dict_pagesize) { + error = + status_db->change_pagesize(status_db, status_dict_pagesize); } - return error; } + if (error == 0) { + *status_db_ptr = status_db; + } else { + int r = tokudb::metadata::close(&status_db); + assert_always(r == 0); + } + return error; +} - int open_status(DB_ENV *env, DB **status_db_ptr, const char *name, DB_TXN *txn) { - int error = 0; - DB *status_db = NULL; - error = db_create(&status_db, env, 0); - if (error == 0) { - error = status_db->open(status_db, txn, name, NULL, DB_BTREE, DB_THREAD, 0); - } - if (error == 0) { - uint32_t pagesize = 0; - error = status_db->get_pagesize(status_db, &pagesize); - if (error == 0 && pagesize > status_dict_pagesize) { - error = status_db->change_pagesize(status_db, status_dict_pagesize); +int strip_frm_data(DB_ENV* env) { + int r; + DB_TXN* txn = NULL; + + fprintf(stderr, "TokuDB strip_frm_data : Beginning stripping process.\n"); + + r = db_env->txn_begin(env, NULL, &txn, 0); + assert_always(r == 0); + + DBC* c = NULL; + r = env->get_cursor_for_directory(env, txn, &c); + assert_always(r == 0); + + DBT key = { }; + key.flags = DB_DBT_REALLOC; + + DBT val = { }; + val.flags = DB_DBT_REALLOC; + while (1) { + r = c->c_get(c, &key, &val, DB_NEXT); + if (r == DB_NOTFOUND) + break; + const char* dname = (const char*) key.data; + const char* iname = (const char*) val.data; + assert_always(r == 0); + + if (strstr(iname, "_status_")) { + fprintf( + stderr, + "TokuDB strip_frm_data : stripping from dname=%s iname=%s\n", + dname, + iname); + + DB* status_db; + r = tokudb::metadata::open(db_env, &status_db, dname, txn); + if (r != 0) { + fprintf( + stderr, + "TokuDB strip_frm_data : unable to open status file %s, " + "error = %d\n", + dname, + r); + continue; } + + // GOL : this is a godawful hack. The inventors of this did not + // think it would be a good idea to use some kind of magic + // identifier k/v pair so that you can in fact tell a proper status + // file from any other file that might have the string _status_ in + // it. Out in ha_tokudb::create, when the status file is initially + // created, it is immediately populated with: + // uint hatoku_new_version=HA_TOKU_VERSION=4 and + // uint hatoku_capabilities=HA_TOKU_CAP=0 + // Since I can't count on the fact that these values are/were + // _always_ 4 and 0, I can count on the fact that they _must_ be + // there and the _must_ be sizeof(uint). That will at least give us + // a much better idea that these are in fact status files. + void* p = NULL; + size_t sz; + r = + tokudb::metadata::read_realloc( + status_db, + txn, + hatoku_new_version, + &p, + &sz); + if (r != 0) { + fprintf( + stderr, + "TokuDB strip_frm_data : does not look like a real TokuDB " + "status file, new_verion is missing, leaving alone %s \n", + dname); + + r = tokudb::metadata::close(&status_db); + assert_always(r == 0); + continue; + } else if (sz != sizeof(uint)) { + fprintf( + stderr, + "TokuDB strip_frm_data : does not look like a real TokuDB " + "status file, new_verion is the wrong size, " + "leaving alone %s \n", + dname); + + tokudb::memory::free(p); + r = tokudb::metadata::close(&status_db); + assert_always(r == 0); + continue; + } + tokudb::memory::free(p); + p = NULL; + + r = + tokudb::metadata::read_realloc( + status_db, + txn, + hatoku_capabilities, + &p, + &sz); + if (r != 0) { + fprintf( + stderr, + "TokuDB strip_frm_data : does not look like a real TokuDB " + "status file, capabilities is missing, leaving alone %s \n", + dname); + + r = tokudb::metadata::close(&status_db); + assert_always(r == 0); + continue; + } else if (sz != sizeof(uint)) { + fprintf( + stderr, + "TokuDB strip_frm_data : does not look like a real TokuDB " + "status file, capabilities is the wrong size, " + "leaving alone %s \n", + dname); + + tokudb::memory::free(p); + r = tokudb::metadata::close(&status_db); + assert_always(r == 0); + continue; + } + tokudb::memory::free(p); + + // OK, st this point, it is probably a status file, not 100% but + // it looks like it :( + r = tokudb::metadata::remove(status_db, hatoku_frm_data, txn); + if (r != 0) { + fprintf( + stderr, + "TokuDB strip_frm_data : unable to find/strip frm data " + "from status file %s, error = %d\n", + dname, + r); + } + + r = tokudb::metadata::close(&status_db); + assert_always(r == 0); } - if (error == 0) { - *status_db_ptr = status_db; - } else { - int r = close_status(&status_db); - assert(r == 0); - } - return error; } + tokudb::memory::free(key.data); + tokudb::memory::free(val.data); + + fprintf( + stderr, + "TokuDB strip_frm_data : Stripping process complete, beginning " + "commit, this may take some time.\n"); + + r = c->c_close(c); + assert_always(r == 0); + + r = txn->commit(txn, 0); + assert_always(r == 0); + + fprintf( + stderr, + "TokuDB strip_frm_data : Commit complete, resuming server init " + "process."); + + return 0; } +} // namespace metadata +} // namespace tokudb #endif diff --git a/storage/tokudb/tokudb_sysvars.cc b/storage/tokudb/tokudb_sysvars.cc new file mode 100644 index 00000000000..168fb0cc647 --- /dev/null +++ b/storage/tokudb/tokudb_sysvars.cc @@ -0,0 +1,1099 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +/* -*- mode: C; c-basic-offset: 4 -*- */ +#ident "$Id$" +/*====== +This file is part of TokuDB + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + TokuDBis is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + TokuDB is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TokuDB. If not, see <http://www.gnu.org/licenses/>. + +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#include "hatoku_hton.h" + +namespace tokudb { +namespace sysvars { + +//****************************************************************************** +// global variables +//****************************************************************************** +#ifdef TOKUDB_VERSION +#define tokudb_stringify_2(x) #x +#define tokudb_stringify(x) tokudb_stringify_2(x) +#define TOKUDB_VERSION_STR tokudb_stringify(TOKUDB_VERSION) +#else +#define TOKUDB_VERSION_STR NULL +#endif + +ulonglong cache_size = 0; +uint cachetable_pool_threads = 0; +int cardinality_scale_percent = 0; +my_bool checkpoint_on_flush_logs = FALSE; +uint checkpoint_pool_threads = 0; +uint checkpointing_period = 0; +ulong cleaner_iterations = 0; +ulong cleaner_period = 0; +uint client_pool_threads = 0; +my_bool compress_buffers_before_eviction = TRUE; +char* data_dir = NULL; +ulong debug = 0; +#if TOKUDB_DEBUG +// used to control background job manager +my_bool debug_pause_background_job_manager = FALSE; +#endif +my_bool directio = FALSE; +my_bool enable_partial_eviction = TRUE; +int fs_reserve_percent = 0; +uint fsync_log_period = 0; +char* log_dir = NULL; +ulonglong max_lock_memory = 0; +uint read_status_frequency = 0; +my_bool strip_frm_data = FALSE; +char* tmp_dir = NULL; +uint write_status_frequency = 0; +char* version = (char*) TOKUDB_VERSION_STR; + +// file system reserve as a percentage of total disk space +#if TOKU_INCLUDE_HANDLERTON_HANDLE_FATAL_SIGNAL +char* gdb_path = NULL; +my_bool gdb_on_fatal = FALSE; +#endif + +#if TOKUDB_CHECK_JEMALLOC +uint check_jemalloc = 0; +#endif + +static MYSQL_SYSVAR_ULONGLONG( + cache_size, + cache_size, + PLUGIN_VAR_READONLY, + "cache table size", + NULL, + NULL, + 0, + 0, + ~0ULL, + 0); + +static MYSQL_SYSVAR_UINT( + cachetable_pool_threads, + cachetable_pool_threads, + PLUGIN_VAR_READONLY, + "cachetable ops thread pool size", + NULL, + NULL, + 0, + 0, + 1024, + 0); + +static MYSQL_SYSVAR_INT( + cardinality_scale_percent, + cardinality_scale_percent, + 0, + "index cardinality scale percentage", + NULL, + NULL, + 50, + 0, + 100, + 0); + +static MYSQL_SYSVAR_BOOL( + checkpoint_on_flush_logs, + checkpoint_on_flush_logs, + 0, + "checkpoint on flush logs", + NULL, + NULL, + FALSE); + +static MYSQL_SYSVAR_UINT( + checkpoint_pool_threads, + checkpoint_pool_threads, + PLUGIN_VAR_READONLY, + "checkpoint ops thread pool size", + NULL, + NULL, + 0, + 0, + 1024, + 0); + +static void checkpointing_period_update( + THD* thd, + st_mysql_sys_var* sys_var, + void* var, + const void* save) { + + uint* cp = (uint*)var; + *cp = *(const uint*)save; + int r = db_env->checkpointing_set_period(db_env, *cp); + assert(r == 0); +} + +static MYSQL_SYSVAR_UINT( + checkpointing_period, + checkpointing_period, + 0, + "checkpointing period", + NULL, + checkpointing_period_update, + 60, + 0, + ~0U, + 0); + +static void cleaner_iterations_update( + THD* thd, + st_mysql_sys_var* sys_var, + void* var, + const void* save) { + + ulong* ci = (ulong*)var; + *ci = *(const ulong*)save; + int r = db_env->cleaner_set_iterations(db_env, *ci); + assert(r == 0); +} + +static MYSQL_SYSVAR_ULONG( + cleaner_iterations, + cleaner_iterations, + 0, + "cleaner_iterations", + NULL, + cleaner_iterations_update, + DEFAULT_TOKUDB_CLEANER_ITERATIONS, + 0, + ~0UL, + 0); + +static void cleaner_period_update( + THD* thd, + st_mysql_sys_var* sys_var, + void* var, + const void * save) { + + ulong* cp = (ulong*)var; + *cp = *(const ulong*)save; + int r = db_env->cleaner_set_period(db_env, *cp); + assert(r == 0); +} + +static MYSQL_SYSVAR_ULONG( + cleaner_period, + cleaner_period, + 0, + "cleaner_period", + NULL, + cleaner_period_update, + DEFAULT_TOKUDB_CLEANER_PERIOD, + 0, + ~0UL, + 0); + +static MYSQL_SYSVAR_UINT( + client_pool_threads, + client_pool_threads, + PLUGIN_VAR_READONLY, + "client ops thread pool size", + NULL, + NULL, + 0, + 0, + 1024, + 0); + +static MYSQL_SYSVAR_BOOL( + compress_buffers_before_eviction, + compress_buffers_before_eviction, + PLUGIN_VAR_READONLY, + "enable in-memory buffer compression before partial eviction", + NULL, + NULL, + TRUE); + +static MYSQL_SYSVAR_STR( + data_dir, + data_dir, + PLUGIN_VAR_READONLY, + "data directory", + NULL, + NULL, + NULL); + +static MYSQL_SYSVAR_ULONG( + debug, + debug, + 0, + "plugin debug mask", + NULL, + NULL, + 0, + 0, + ~0UL, + 0); + +#if TOKUDB_DEBUG +static MYSQL_SYSVAR_BOOL( + debug_pause_background_job_manager, + debug_pause_background_job_manager, + 0, + "debug : pause the background job manager", + NULL, + NULL, + FALSE); +#endif // TOKUDB_DEBUG + +static MYSQL_SYSVAR_BOOL( + directio, + directio, + PLUGIN_VAR_READONLY, "enable direct i/o ", + NULL, + NULL, + FALSE); + +static void enable_partial_eviction_update( + THD* thd, + st_mysql_sys_var* sys_var, + void* var, + const void* save) { + + my_bool* epe = (my_bool*)var; + *epe = *(const my_bool*)save; + int r = db_env->evictor_set_enable_partial_eviction(db_env, *epe); + assert(r == 0); +} + +static MYSQL_SYSVAR_BOOL( + enable_partial_eviction, + enable_partial_eviction, + 0, + "enable partial node eviction", + NULL, + enable_partial_eviction_update, + TRUE); + +static MYSQL_SYSVAR_INT( + fs_reserve_percent, + fs_reserve_percent, + PLUGIN_VAR_READONLY, + "file system space reserve (percent free required)", + NULL, + NULL, + 5, + 0, + 100, + 0); + +static void fsync_log_period_update( + THD* thd, + st_mysql_sys_var* sys_var, + void* var, + const void* save) { + + uint* flp = (uint*)var; + *flp = *(const uint*)save; + db_env->change_fsync_log_period(db_env, *flp); +} + +static MYSQL_SYSVAR_UINT( + fsync_log_period, + fsync_log_period, + 0, + "fsync log period", + NULL, + fsync_log_period_update, + 0, + 0, + ~0U, + 0); + +static MYSQL_SYSVAR_STR( + log_dir, + log_dir, + PLUGIN_VAR_READONLY, + "log directory", + NULL, + NULL, + NULL); + +static MYSQL_SYSVAR_ULONGLONG( + max_lock_memory, + max_lock_memory, + PLUGIN_VAR_READONLY, + "max memory for locks", + NULL, + NULL, + 0, + 0, + ~0ULL, + 0); + +static MYSQL_SYSVAR_UINT( + read_status_frequency, + read_status_frequency, + 0, + "frequency that show processlist updates status of reads", + NULL, + NULL, + 10000, + 0, + ~0U, + 0); + +static MYSQL_SYSVAR_BOOL( + strip_frm_data, + strip_frm_data, + PLUGIN_VAR_READONLY, + "strip .frm data from metadata file(s)", + NULL, + NULL, + FALSE); + +static MYSQL_SYSVAR_STR( + tmp_dir, + tmp_dir, + PLUGIN_VAR_READONLY, + "directory to use for temporary files", + NULL, + NULL, + NULL); + +static MYSQL_SYSVAR_STR( + version, + version, + PLUGIN_VAR_READONLY, + "plugin version", + NULL, + NULL, + NULL); + +static MYSQL_SYSVAR_UINT( + write_status_frequency, + write_status_frequency, + 0, + "frequency that show processlist updates status of writes", + NULL, + NULL, + 1000, + 0, + ~0U, + 0); + +#if TOKU_INCLUDE_HANDLERTON_HANDLE_FATAL_SIGNAL +static MYSQL_SYSVAR_STR( + gdb_path, + gdb_path, + PLUGIN_VAR_READONLY|PLUGIN_VAR_RQCMDARG, + "path to gdb for extra debug info on fatal signal", + NULL, + NULL, + "/usr/bin/gdb"); + +static MYSQL_SYSVAR_BOOL( + gdb_on_fatal, + gdb_on_fatal, + 0, + "enable gdb debug info on fatal signal", + NULL, + NULL, + true); +#endif + +#if TOKUDB_CHECK_JEMALLOC +static MYSQL_SYSVAR_UINT( + check_jemalloc, + check_jemalloc, + 0, + "check if jemalloc is linked", + NULL, + NULL, + 1, + 0, + 1, + 0); +#endif + + +//****************************************************************************** +// session variables +//****************************************************************************** +static MYSQL_THDVAR_BOOL( + alter_print_error, + 0, + "print errors for alter table operations", + NULL, + NULL, + false); + +static MYSQL_THDVAR_DOUBLE( + analyze_delete_fraction, + 0, + "fraction of rows allowed to be deleted", + NULL, + NULL, + 1.0, + 0, + 1.0, + 1); + +static MYSQL_THDVAR_BOOL( + analyze_in_background, + 0, + "dispatch ANALYZE TABLE to background job.", + NULL, + NULL, + false); + +const char* srv_analyze_mode_names[] = { + "TOKUDB_ANALYZE_STANDARD", + "TOKUDB_ANALYZE_RECOUNT_ROWS", + "TOKUDB_ANALYZE_CANCEL", + NullS +}; + +static TYPELIB tokudb_analyze_mode_typelib = { + array_elements(srv_analyze_mode_names) - 1, + "tokudb_analyze_mode_typelib", + srv_analyze_mode_names, + NULL +}; + +static MYSQL_THDVAR_ENUM(analyze_mode, + PLUGIN_VAR_RQCMDARG, + "Controls the function of ANALYZE TABLE. Possible values are: " + "TOKUDB_ANALYZE_STANDARD perform standard table analysis (default); " + "TOKUDB_ANALYZE_RECOUNT_ROWS perform logical recount of table rows;" + "TOKUDB_ANALYZE_CANCEL terminate and cancel all scheduled background jobs " + "for a table", + NULL, + NULL, + TOKUDB_ANALYZE_STANDARD, + &tokudb_analyze_mode_typelib); + +static MYSQL_THDVAR_ULONGLONG( + analyze_throttle, + 0, + "analyze throttle (keys)", + NULL, + NULL, + 0, + 0, + ~0U, + 1); + +static MYSQL_THDVAR_UINT( + analyze_time, + 0, + "analyze time (seconds)", + NULL, + NULL, + 5, + 0, + ~0U, + 1); + +static MYSQL_THDVAR_ULONGLONG( + auto_analyze, + 0, + "auto analyze threshold (percent)", + NULL, + NULL, + 0, + 0, + ~0U, + 1); + +static MYSQL_THDVAR_UINT( + block_size, + 0, + "fractal tree block size", + NULL, + NULL, + 4<<20, + 4096, + ~0U, + 1); + +static MYSQL_THDVAR_BOOL( + bulk_fetch, + PLUGIN_VAR_THDLOCAL, + "enable bulk fetch", + NULL, + NULL, + true); + +static void checkpoint_lock_update( + THD* thd, + st_mysql_sys_var* var, + void* var_ptr, + const void* save) { + + my_bool* val = (my_bool*)var_ptr; + *val= *(my_bool*)save ? true : false; + if (*val) { + tokudb_checkpoint_lock(thd); + } else { + tokudb_checkpoint_unlock(thd); + } +} + +static MYSQL_THDVAR_BOOL( + checkpoint_lock, + 0, + "checkpoint lock", + NULL, + checkpoint_lock_update, + false); + +static MYSQL_THDVAR_BOOL( + commit_sync, + PLUGIN_VAR_THDLOCAL, + "sync on txn commit", + NULL, + NULL, + true); + +static MYSQL_THDVAR_BOOL( + create_index_online, + 0, + "if on, create index done online", + NULL, + NULL, + true); + +static MYSQL_THDVAR_BOOL( + disable_hot_alter, + 0, + "if on, hot alter table is disabled", + NULL, + NULL, + false); + +static MYSQL_THDVAR_BOOL( + disable_prefetching, + 0, + "if on, prefetching disabled", + NULL, + NULL, + false); + +static MYSQL_THDVAR_BOOL( + disable_slow_alter, + 0, + "if on, alter tables that require copy are disabled", + NULL, + NULL, + false); + +static const char *tokudb_empty_scan_names[] = { + "disabled", + "lr", + "rl", + NullS +}; + +static TYPELIB tokudb_empty_scan_typelib = { + array_elements(tokudb_empty_scan_names) - 1, + "tokudb_empty_scan_typelib", + tokudb_empty_scan_names, + NULL +}; + +static MYSQL_THDVAR_ENUM( + empty_scan, + PLUGIN_VAR_OPCMDARG, + "algorithm to check if the table is empty when opened", + NULL, + NULL, + TOKUDB_EMPTY_SCAN_RL, + &tokudb_empty_scan_typelib); + +static MYSQL_THDVAR_UINT( + fanout, + 0, + "fractal tree fanout", + NULL, + NULL, + 16, + 2, + 16*1024, + 1); + +static MYSQL_THDVAR_BOOL( + hide_default_row_format, + 0, + "hide the default row format", + NULL, + NULL, + true); + +static MYSQL_THDVAR_ULONGLONG( + killed_time, + 0, + "killed time", + NULL, + NULL, + DEFAULT_TOKUDB_KILLED_TIME, + 0, + ~0ULL, + 1); + +static MYSQL_THDVAR_STR( + last_lock_timeout, + PLUGIN_VAR_MEMALLOC, + "last lock timeout", + NULL, + NULL, + NULL); + +static MYSQL_THDVAR_BOOL( + load_save_space, + 0, + "compress intermediate bulk loader files to save space", + NULL, + NULL, + true); + +static MYSQL_THDVAR_ULONGLONG( + loader_memory_size, + 0, + "loader memory size", + NULL, + NULL, + 100*1000*1000, + 0, + ~0ULL, + 1); + +static MYSQL_THDVAR_ULONGLONG( + lock_timeout, + 0, + "lock timeout", + NULL, + NULL, + DEFAULT_TOKUDB_LOCK_TIMEOUT, + 0, + ~0ULL, + 1); + +static MYSQL_THDVAR_UINT( + lock_timeout_debug, + 0, + "lock timeout debug", + NULL, + NULL, + 1, + 0, + ~0U, + 1); + +static MYSQL_THDVAR_DOUBLE( + optimize_index_fraction, + 0, + "optimize index fraction (default 1.0 all)", + NULL, + NULL, + 1.0, + 0, + 1.0, + 1); + +static MYSQL_THDVAR_STR( + optimize_index_name, + PLUGIN_VAR_THDLOCAL + PLUGIN_VAR_MEMALLOC, + "optimize index name (default all indexes)", + NULL, + NULL, + NULL); + +static MYSQL_THDVAR_ULONGLONG( + optimize_throttle, + 0, + "optimize throttle (default no throttle)", + NULL, + NULL, + 0, + 0, + ~0ULL, + 1); + +static MYSQL_THDVAR_UINT( + pk_insert_mode, + 0, + "set the primary key insert mode", + NULL, + NULL, + 1, + 0, + 2, + 1); + +static MYSQL_THDVAR_BOOL( + prelock_empty, + 0, + "prelock empty table", + NULL, + NULL, + true); + +static MYSQL_THDVAR_UINT( + read_block_size, + 0, + "fractal tree read block size", + NULL, + NULL, + 64*1024, + 4096, + ~0U, + 1); + +static MYSQL_THDVAR_UINT( + read_buf_size, + 0, + "range query read buffer size", + NULL, + NULL, + 128*1024, + 0, + 1*1024*1024, + 1); + +static const char *tokudb_row_format_names[] = { + "tokudb_uncompressed", + "tokudb_zlib", + "tokudb_snappy", + "tokudb_quicklz", + "tokudb_lzma", + "tokudb_fast", + "tokudb_small", + "tokudb_default", + NullS +}; + +static TYPELIB tokudb_row_format_typelib = { + array_elements(tokudb_row_format_names) - 1, + "tokudb_row_format_typelib", + tokudb_row_format_names, + NULL +}; + +static MYSQL_THDVAR_ENUM( + row_format, + PLUGIN_VAR_OPCMDARG, + "Specifies the compression method for a table created during this session. " + "Possible values are TOKUDB_UNCOMPRESSED, TOKUDB_ZLIB, TOKUDB_SNAPPY, " + "TOKUDB_QUICKLZ, TOKUDB_LZMA, TOKUDB_FAST, TOKUDB_SMALL and TOKUDB_DEFAULT", + NULL, + NULL, + SRV_ROW_FORMAT_ZLIB, + &tokudb_row_format_typelib); + +static MYSQL_THDVAR_BOOL( + rpl_check_readonly, + PLUGIN_VAR_THDLOCAL, + "check if the slave is read only", + NULL, + NULL, + true); + +static MYSQL_THDVAR_BOOL( + rpl_lookup_rows, + PLUGIN_VAR_THDLOCAL, + "lookup a row on rpl slave", + NULL, + NULL, + true); + +static MYSQL_THDVAR_ULONGLONG( + rpl_lookup_rows_delay, + PLUGIN_VAR_THDLOCAL, + "time in milliseconds to add to lookups on replication slave", + NULL, + NULL, + 0, + 0, + ~0ULL, + 1); + +static MYSQL_THDVAR_BOOL( + rpl_unique_checks, + PLUGIN_VAR_THDLOCAL, + "enable unique checks on replication slave", + NULL, + NULL, + true); + +static MYSQL_THDVAR_ULONGLONG( + rpl_unique_checks_delay, + PLUGIN_VAR_THDLOCAL, + "time in milliseconds to add to unique checks test on replication slave", + NULL, + NULL, + 0, + 0, + ~0ULL, + 1); + +#if TOKU_INCLUDE_UPSERT +static MYSQL_THDVAR_BOOL( + disable_slow_update, + PLUGIN_VAR_THDLOCAL, + "disable slow update", + NULL, + NULL, + false); + +static MYSQL_THDVAR_BOOL( + disable_slow_upsert, + PLUGIN_VAR_THDLOCAL, + "disable slow upsert", + NULL, + NULL, + false); +#endif + +#if TOKU_INCLUDE_XA +static MYSQL_THDVAR_BOOL( + support_xa, + PLUGIN_VAR_OPCMDARG, + "Enable TokuDB support for the XA two-phase commit", + NULL, + NULL, + true); +#endif + + + +//****************************************************************************** +// all system variables +//****************************************************************************** +st_mysql_sys_var* system_variables[] = { + // global vars + MYSQL_SYSVAR(cache_size), + MYSQL_SYSVAR(checkpoint_on_flush_logs), + MYSQL_SYSVAR(cachetable_pool_threads), + MYSQL_SYSVAR(cardinality_scale_percent), + MYSQL_SYSVAR(checkpoint_pool_threads), + MYSQL_SYSVAR(checkpointing_period), + MYSQL_SYSVAR(cleaner_iterations), + MYSQL_SYSVAR(cleaner_period), + MYSQL_SYSVAR(client_pool_threads), + MYSQL_SYSVAR(compress_buffers_before_eviction), + MYSQL_SYSVAR(data_dir), + MYSQL_SYSVAR(debug), + MYSQL_SYSVAR(directio), + MYSQL_SYSVAR(enable_partial_eviction), + MYSQL_SYSVAR(fs_reserve_percent), + MYSQL_SYSVAR(fsync_log_period), + MYSQL_SYSVAR(log_dir), + MYSQL_SYSVAR(max_lock_memory), + MYSQL_SYSVAR(read_status_frequency), + MYSQL_SYSVAR(strip_frm_data), + MYSQL_SYSVAR(tmp_dir), + MYSQL_SYSVAR(version), + MYSQL_SYSVAR(write_status_frequency), + +#if TOKU_INCLUDE_HANDLERTON_HANDLE_FATAL_SIGNAL + MYSQL_SYSVAR(gdb_path), + MYSQL_SYSVAR(gdb_on_fatal), +#endif + +#if TOKUDB_CHECK_JEMALLOC + MYSQL_SYSVAR(check_jemalloc), +#endif + + // session vars + MYSQL_SYSVAR(alter_print_error), + MYSQL_SYSVAR(analyze_delete_fraction), + MYSQL_SYSVAR(analyze_in_background), + MYSQL_SYSVAR(analyze_mode), + MYSQL_SYSVAR(analyze_throttle), + MYSQL_SYSVAR(analyze_time), + MYSQL_SYSVAR(auto_analyze), + MYSQL_SYSVAR(block_size), + MYSQL_SYSVAR(bulk_fetch), + MYSQL_SYSVAR(checkpoint_lock), + MYSQL_SYSVAR(commit_sync), + MYSQL_SYSVAR(create_index_online), + MYSQL_SYSVAR(disable_hot_alter), + MYSQL_SYSVAR(disable_prefetching), + MYSQL_SYSVAR(disable_slow_alter), + MYSQL_SYSVAR(empty_scan), + MYSQL_SYSVAR(fanout), + MYSQL_SYSVAR(hide_default_row_format), + MYSQL_SYSVAR(killed_time), + MYSQL_SYSVAR(last_lock_timeout), + MYSQL_SYSVAR(load_save_space), + MYSQL_SYSVAR(loader_memory_size), + MYSQL_SYSVAR(lock_timeout), + MYSQL_SYSVAR(lock_timeout_debug), + MYSQL_SYSVAR(optimize_index_fraction), + MYSQL_SYSVAR(optimize_index_name), + MYSQL_SYSVAR(optimize_throttle), + MYSQL_SYSVAR(pk_insert_mode), + MYSQL_SYSVAR(prelock_empty), + MYSQL_SYSVAR(read_block_size), + MYSQL_SYSVAR(read_buf_size), + MYSQL_SYSVAR(row_format), + MYSQL_SYSVAR(rpl_check_readonly), + MYSQL_SYSVAR(rpl_lookup_rows), + MYSQL_SYSVAR(rpl_lookup_rows_delay), + MYSQL_SYSVAR(rpl_unique_checks), + MYSQL_SYSVAR(rpl_unique_checks_delay), + +#if TOKU_INCLUDE_UPSERT + MYSQL_SYSVAR(disable_slow_update), + MYSQL_SYSVAR(disable_slow_upsert), +#endif + +#if TOKU_INCLUDE_XA + MYSQL_SYSVAR(support_xa), +#endif + +#if TOKUDB_DEBUG + MYSQL_SYSVAR(debug_pause_background_job_manager), +#endif // TOKUDB_DEBUG + + NULL +}; + +my_bool alter_print_error(THD* thd) { + return (THDVAR(thd, alter_print_error) != 0); +} +double analyze_delete_fraction(THD* thd) { + return THDVAR(thd, analyze_delete_fraction); +} +my_bool analyze_in_background(THD* thd) { + return (THDVAR(thd, analyze_in_background) != 0); +} +analyze_mode_t analyze_mode(THD* thd) { + return (analyze_mode_t ) THDVAR(thd, analyze_mode); +} +ulonglong analyze_throttle(THD* thd) { + return THDVAR(thd, analyze_throttle); +} +ulonglong analyze_time(THD* thd) { + return THDVAR(thd, analyze_time); +} +ulonglong auto_analyze(THD* thd) { + return THDVAR(thd, auto_analyze); +} +my_bool bulk_fetch(THD* thd) { + return (THDVAR(thd, bulk_fetch) != 0); +} +uint block_size(THD* thd) { + return THDVAR(thd, block_size); +} +my_bool commit_sync(THD* thd) { + return (THDVAR(thd, commit_sync) != 0); +} +my_bool create_index_online(THD* thd) { + return (THDVAR(thd, create_index_online) != 0); +} +my_bool disable_hot_alter(THD* thd) { + return (THDVAR(thd, disable_hot_alter) != 0); +} +my_bool disable_prefetching(THD* thd) { + return (THDVAR(thd, disable_prefetching) != 0); +} +my_bool disable_slow_alter(THD* thd) { + return (THDVAR(thd, disable_slow_alter) != 0); +} +my_bool disable_slow_update(THD* thd) { + return (THDVAR(thd, disable_slow_update) != 0); +} +my_bool disable_slow_upsert(THD* thd) { + return (THDVAR(thd, disable_slow_upsert) != 0); +} +empty_scan_mode_t empty_scan(THD* thd) { + return (empty_scan_mode_t)THDVAR(thd, empty_scan); +} +uint fanout(THD* thd) { + return THDVAR(thd, fanout); +} +my_bool hide_default_row_format(THD* thd) { + return (THDVAR(thd, hide_default_row_format) != 0); +} +ulonglong killed_time(THD* thd) { + return THDVAR(thd, killed_time); +} +char* last_lock_timeout(THD* thd) { + return THDVAR(thd, last_lock_timeout); +} +void set_last_lock_timeout(THD* thd, char* last) { + THDVAR(thd, last_lock_timeout) = last; +} +my_bool load_save_space(THD* thd) { + return (THDVAR(thd, load_save_space) != 0); +} +ulonglong loader_memory_size(THD* thd) { + return THDVAR(thd, loader_memory_size); +} +ulonglong lock_timeout(THD* thd) { + return THDVAR(thd, lock_timeout); +} +uint lock_timeout_debug(THD* thd) { + return THDVAR(thd, lock_timeout_debug); +} +double optimize_index_fraction(THD* thd) { + return THDVAR(thd, optimize_index_fraction); +} +const char* optimize_index_name(THD* thd) { + return THDVAR(thd, optimize_index_name); +} +ulonglong optimize_throttle(THD* thd) { + return THDVAR(thd, optimize_throttle); +} +uint pk_insert_mode(THD* thd) { + return THDVAR(thd, pk_insert_mode); +} +my_bool prelock_empty(THD* thd) { + return (THDVAR(thd, prelock_empty) != 0); +} +uint read_block_size(THD* thd) { + return THDVAR(thd, read_block_size); +} +uint read_buf_size(THD* thd) { + return THDVAR(thd, read_buf_size); +} +row_format_t row_format(THD *thd) { + return (row_format_t) THDVAR(thd, row_format); +} +my_bool rpl_check_readonly(THD* thd) { + return (THDVAR(thd, rpl_check_readonly) != 0); +} +my_bool rpl_lookup_rows(THD* thd) { + return (THDVAR(thd, rpl_lookup_rows) != 0); +} +ulonglong rpl_lookup_rows_delay(THD* thd) { + return THDVAR(thd, rpl_lookup_rows_delay); +} +my_bool rpl_unique_checks(THD* thd) { + return (THDVAR(thd, rpl_unique_checks) != 0); +} +ulonglong rpl_unique_checks_delay(THD* thd) { + return THDVAR(thd, rpl_unique_checks_delay); +} +my_bool support_xa(THD* thd) { + return (THDVAR(thd, support_xa) != 0); +} + +} // namespace sysvars +} // namespace tokudb diff --git a/storage/tokudb/tokudb_sysvars.h b/storage/tokudb/tokudb_sysvars.h new file mode 100644 index 00000000000..a7490c7b473 --- /dev/null +++ b/storage/tokudb/tokudb_sysvars.h @@ -0,0 +1,147 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +#ident "$Id$" +/*====== +This file is part of TokuDB + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + TokuDBis is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + TokuDB is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TokuDB. If not, see <http://www.gnu.org/licenses/>. + +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#ifndef _TOKUDB_SYSVARS_H +#define _TOKUDB_SYSVARS_H + +namespace tokudb { +namespace sysvars { + +enum analyze_mode_t { + TOKUDB_ANALYZE_STANDARD = 0, + TOKUDB_ANALYZE_RECOUNT_ROWS = 1, + TOKUDB_ANALYZE_CANCEL = 2 +}; + +enum empty_scan_mode_t { + TOKUDB_EMPTY_SCAN_DISABLED = 0, + TOKUDB_EMPTY_SCAN_LR = 1, + TOKUDB_EMPTY_SCAN_RL = 2, +}; + +enum row_format_t { + SRV_ROW_FORMAT_UNCOMPRESSED = 0, + SRV_ROW_FORMAT_ZLIB = 1, + SRV_ROW_FORMAT_SNAPPY = 2, + SRV_ROW_FORMAT_QUICKLZ = 3, + SRV_ROW_FORMAT_LZMA = 4, + SRV_ROW_FORMAT_FAST = 5, + SRV_ROW_FORMAT_SMALL = 6, + SRV_ROW_FORMAT_DEFAULT = 7 +}; + +#define DEFAULT_TOKUDB_CLEANER_ITERATIONS 5 +#define DEFAULT_TOKUDB_CLEANER_PERIOD 1 +#define DEFAULT_TOKUDB_KILLED_TIME 4000 // milliseconds +#define DEFAULT_TOKUDB_LOCK_TIMEOUT 4000 // milliseconds + + +// globals +extern ulonglong cache_size; +extern uint cachetable_pool_threads; +extern int cardinality_scale_percent; +extern my_bool checkpoint_on_flush_logs; +extern uint checkpoint_pool_threads; +extern uint checkpointing_period; +extern ulong cleaner_iterations; +extern ulong cleaner_period; +extern uint client_pool_threads; +extern my_bool compress_buffers_before_eviction; +extern char* data_dir; +extern ulong debug; +extern my_bool directio; +extern my_bool enable_partial_eviction; +extern int fs_reserve_percent; +extern uint fsync_log_period; +extern char* log_dir; +extern ulonglong max_lock_memory; +extern uint read_status_frequency; +extern my_bool strip_frm_data; +extern char* tmp_dir; +extern uint write_status_frequency; +extern char* version; + +#if TOKU_INCLUDE_HANDLERTON_HANDLE_FATAL_SIGNAL +extern char* gdb_path; +extern my_bool gdb_on_fatal; +#endif + +#if TOKUDB_CHECK_JEMALLOC +extern uint check_jemalloc; +#endif + +#if TOKUDB_DEBUG +// used to control background job manager +extern my_bool debug_pause_background_job_manager; +#endif // TOKUDB_DEBUG + +// session/thread +my_bool alter_print_error(THD* thd); +double analyze_delete_fraction(THD* thd); +my_bool analyze_in_background(THD* thd); +analyze_mode_t analyze_mode(THD* thd); +ulonglong analyze_throttle(THD* thd); +ulonglong analyze_time(THD* thd); +ulonglong auto_analyze(THD* thd); +uint block_size(THD* thd); +my_bool bulk_fetch(THD* thd); +my_bool commit_sync(THD* thd); +my_bool create_index_online(THD* thd); +my_bool disable_hot_alter(THD* thd); +my_bool disable_prefetching(THD* thd); +my_bool disable_slow_alter(THD* thd); +my_bool disable_slow_update(THD* thd); +my_bool disable_slow_upsert(THD* thd); +empty_scan_mode_t empty_scan(THD* thd); +uint fanout(THD* thd); +my_bool hide_default_row_format(THD* thd); +ulonglong killed_time(THD* thd); +my_bool load_save_space(THD* thd); +char* last_lock_timeout(THD* thd); +void set_last_lock_timeout(THD* thd, char* last); +ulonglong loader_memory_size(THD* thd); +ulonglong lock_timeout(THD* thd); +uint lock_timeout_debug(THD* thd); +double optimize_index_fraction(THD* thd); +const char* optimize_index_name(THD* thd); +ulonglong optimize_throttle(THD* thd); +uint pk_insert_mode(THD* thd); +my_bool prelock_empty(THD* thd); +uint read_block_size(THD* thd); +uint read_buf_size(THD* thd); +row_format_t row_format(THD *thd); +my_bool rpl_check_readonly(THD* thd); +my_bool rpl_lookup_rows(THD* thd); +ulonglong rpl_lookup_rows_delay(THD* thd); +my_bool rpl_unique_checks(THD* thd); +ulonglong rpl_unique_checks_delay(THD* thd); +my_bool support_xa(THD* thd); + +extern st_mysql_sys_var* system_variables[]; + +} // namespace sysvars +} // namespace tokudb + +#endif // _TOKUDB_SYSVARS_H diff --git a/storage/tokudb/tokudb_thread.cc b/storage/tokudb/tokudb_thread.cc new file mode 100644 index 00000000000..f27e803a065 --- /dev/null +++ b/storage/tokudb/tokudb_thread.cc @@ -0,0 +1,35 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +/* -*- mode: C; c-basic-offset: 4 -*- */ +#ident "$Id$" +/*====== +This file is part of TokuDB + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + TokuDBis is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + TokuDB is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TokuDB. If not, see <http://www.gnu.org/licenses/>. + +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#include "tokudb_thread.h" + +namespace tokudb { +namespace thread { + +pthread_t mutex_t::_null_owner = 0; + +} // namespace thread +} // namespace tokudb diff --git a/storage/tokudb/tokudb_thread.h b/storage/tokudb/tokudb_thread.h new file mode 100644 index 00000000000..ab1633a16ca --- /dev/null +++ b/storage/tokudb/tokudb_thread.h @@ -0,0 +1,548 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +/* -*- mode: C; c-basic-offset: 4 -*- */ +#ident "$Id$" +/*====== +This file is part of TokuDB + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + TokuDBis is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + TokuDB is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TokuDB. If not, see <http://www.gnu.org/licenses/>. + +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#ifndef _TOKUDB_SYNC_H +#define _TOKUDB_SYNC_H + +#include "hatoku_defines.h" +#include "tokudb_debug.h" +#include "tokudb_time.h" + +namespace tokudb { +namespace thread { + +uint my_tid(void); + +// Your basic mutex +class mutex_t { +public: + mutex_t(void); + ~mutex_t(void); + + void lock(void); + int lock(ulonglong microseconds); + void unlock(void); +#ifdef TOKUDB_DEBUG + bool is_owned_by_me(void) const; +#endif +private: + static pthread_t _null_owner; + pthread_mutex_t _mutex; +#ifdef TOKUDB_DEBUG + uint _owners; + pthread_t _owner; +#endif +}; + +// Simple read write lock +class rwlock_t { +public: + rwlock_t(void); + ~rwlock_t(void); + + void lock_read(void); + int lock_read(ulonglong microseconds); + void lock_write(void); + int lock_write(ulonglong microseconds); + void unlock(void); + +private: + rwlock_t(const rwlock_t&); + rwlock_t& operator=(const rwlock_t&); + + pthread_rwlock_t _rwlock; +}; + +// Simple event signal/wait class +class event_t { +public: + // create_signalled - create the event in a signalled state + // manual_reset - create an event that must be manually reset + // after signaling + event_t( + bool create_signalled = false, + bool manual_reset = false); + ~event_t(void); + + // wait for the event to become signalled + void wait(void); + int wait(ulonglong microseconds); + + // signal the event + void signal(void); + + // pulse the event (signal and free exactly one waiter) + void pulse(void); + + // is the event currently signalled + bool signalled(void); + + // unsignal/clear the event + void reset(void); + +private: + event_t(const event_t&); + event_t& operator=(const event_t&); + + pthread_mutex_t _mutex; + pthread_cond_t _cond; + bool _signalled; + bool _pulsed; + bool _manual_reset; +}; + +// Semaphore signal/wait class +class semaphore_t { +public: + // initial_count - the initial signal count of the semaphore + // max_count - the maximum signal count for the semaphore. + semaphore_t(int initial_count, int max_count); + ~semaphore_t(void); + + enum E_WAIT { + E_SIGNALLED = 0, + E_INTERRUPTED = 1, + E_TIMEDOUT = 2 + }; + + // wait for the semaphore to become signalled + E_WAIT wait(void); + E_WAIT wait(ulonglong microseconds); + + // signal the semaphore to increase the count + // return true if signalled, false if ignored due to count + bool signal(void); + + // what is the semaphore signal count + int signalled(void); + + // unsignal a signalled semaphore + void reset(void); + + // set to interrupt any waiters, as long as is set, + // waiters will return immediately with E_INTERRUPTED. + // the semaphore signal count and tracking will continue + // accepting signals and leave the signalled state intact + void set_interrupt(void); + void clear_interrupt(void); + +private: + semaphore_t(const semaphore_t&); + semaphore_t& operator=(const semaphore_t&); + + pthread_mutex_t _mutex; + pthread_cond_t _cond; + bool _interrupted; + int _signalled; + int _initial_count; + int _max_count; +}; + +// Thread class +class thread_t { +public: + thread_t(void); + ~thread_t(void); + + int start(void* (*pfn)(void*), void* arg); + int join(void** value_ptr); + int detach(void); + +private: + pthread_t _thread; +}; + + +inline uint my_tid(void) { + return (uint)toku_os_gettid(); +} + + +inline mutex_t::mutex_t(void) { + #ifdef TOKUDB_DEBUG + _owners = 0; + _owner = _null_owner; + #endif + int r = pthread_mutex_init(&_mutex, MY_MUTEX_INIT_FAST); + assert_debug(r == 0); +} +inline mutex_t::~mutex_t(void) { + #ifdef TOKUDB_DEBUG + assert_debug(_owners == 0); + #endif + int r = pthread_mutex_destroy(&_mutex); + assert_debug(r == 0); +} +inline void mutex_t::lock(void) { + assert_debug(is_owned_by_me() == false); + int r = pthread_mutex_lock(&_mutex); + assert_debug(r == 0); + #ifdef TOKUDB_DEBUG + _owners++; + _owner = pthread_self(); + #endif +} +inline int mutex_t::lock(ulonglong microseconds) { + assert_debug(is_owned_by_me() == false); + timespec waittime = time::offset_timespec(microseconds); + int r = pthread_mutex_timedlock(&_mutex, &waittime); + #ifdef TOKUDB_DEBUG + if (r == 0) { + _owners++; + _owner = pthread_self(); + } + #endif + assert_debug(r == 0 || r == ETIMEDOUT); + return r; +} +inline void mutex_t::unlock(void) { + #ifdef TOKUDB_DEBUG + assert_debug(_owners > 0); + assert_debug(is_owned_by_me()); + _owners--; + _owner = _null_owner; + #endif + int r = pthread_mutex_unlock(&_mutex); + assert_debug(r == 0); +} +#ifdef TOKUDB_DEBUG +inline bool mutex_t::is_owned_by_me(void) const { + return pthread_equal(pthread_self(), _owner) != 0 ? true : false; +} +#endif + + +inline rwlock_t::rwlock_t(void) { + int r = pthread_rwlock_init(&_rwlock, NULL); + assert_debug(r == 0); +} +inline rwlock_t::~rwlock_t(void) { + int r = pthread_rwlock_destroy(&_rwlock); + assert_debug(r == 0); +} +inline void rwlock_t::lock_read(void) { + int r; + while ((r = pthread_rwlock_rdlock(&_rwlock)) != 0) { + if (r == EBUSY || r == EAGAIN) { + time::sleep_microsec(1000); + continue; + } + break; + } + assert_debug(r == 0); +} +inline int rwlock_t::lock_read(ulonglong microseconds) { + int r; + timespec waittime = time::offset_timespec(microseconds); + while ((r = pthread_rwlock_timedrdlock(&_rwlock, &waittime)) != 0) { + if (r == EBUSY || r == EAGAIN) { + time::sleep_microsec(1000); + continue; + } else if (r == ETIMEDOUT) { + return ETIMEDOUT; + } + break; + } + assert_debug(r == 0); + return r; +} +inline void rwlock_t::lock_write(void) { + int r; + while ((r = pthread_rwlock_wrlock(&_rwlock)) != 0) { + if (r == EBUSY || r == EAGAIN) { + time::sleep_microsec(1000); + continue; + } + break; + } + assert_debug(r == 0); +} +inline int rwlock_t::lock_write(ulonglong microseconds) { + int r; + timespec waittime = time::offset_timespec(microseconds); + while ((r = pthread_rwlock_timedwrlock(&_rwlock, &waittime)) != 0) { + if (r == EBUSY || r == EAGAIN) { + time::sleep_microsec(1000); + continue; + } else if (r == ETIMEDOUT) { + return ETIMEDOUT; + } + break; + } + assert_debug(r == 0); + return r; +} +inline void rwlock_t::unlock(void) { + int r = pthread_rwlock_unlock(&_rwlock); + assert_debug(r == 0); +} +inline rwlock_t::rwlock_t(const rwlock_t&) { +} +inline rwlock_t& rwlock_t::operator=(const rwlock_t&) { + return *this; +} + + +inline event_t::event_t(bool create_signalled, bool manual_reset) : + _manual_reset(manual_reset) { + + int r = pthread_mutex_init(&_mutex, NULL); + assert_debug(r == 0); + r = pthread_cond_init(&_cond, NULL); + assert_debug(r == 0); + if (create_signalled) { + _signalled = true; + } else { + _signalled = false; + } + _pulsed = false; +} +inline event_t::~event_t(void) { + int r = pthread_mutex_destroy(&_mutex); + assert_debug(r == 0); + r = pthread_cond_destroy(&_cond); + assert_debug(r == 0); +} +inline void event_t::wait(void) { + int r = pthread_mutex_lock(&_mutex); + assert_debug(r == 0); + while (_signalled == false && _pulsed == false) { + r = pthread_cond_wait(&_cond, &_mutex); + assert_debug(r == 0); + } + if (_manual_reset == false) + _signalled = false; + if (_pulsed) + _pulsed = false; + r = pthread_mutex_unlock(&_mutex); + assert_debug(r == 0); + return; +} +inline int event_t::wait(ulonglong microseconds) { + timespec waittime = time::offset_timespec(microseconds); + int r = pthread_mutex_timedlock(&_mutex, &waittime); + if (r == ETIMEDOUT) return ETIMEDOUT; + assert_debug(r == 0); + while (_signalled == false && _pulsed == false) { + r = pthread_cond_timedwait(&_cond, &_mutex, &waittime); + if (r == ETIMEDOUT) { + r = pthread_mutex_unlock(&_mutex); + assert_debug(r == 0); + return ETIMEDOUT; + } + assert_debug(r == 0); + } + if (_manual_reset == false) + _signalled = false; + if (_pulsed) + _pulsed = false; + r = pthread_mutex_unlock(&_mutex); + assert_debug(r == 0); + return 0; +} +inline void event_t::signal(void) { + int r = pthread_mutex_lock(&_mutex); + assert_debug(r == 0); + _signalled = true; + if (_manual_reset) { + r = pthread_cond_broadcast(&_cond); + assert_debug(r == 0); + } else { + r = pthread_cond_signal(&_cond); + assert_debug(r == 0); + } + r = pthread_mutex_unlock(&_mutex); + assert_debug(r == 0); +} +inline void event_t::pulse(void) { + int r = pthread_mutex_lock(&_mutex); + assert_debug(r == 0); + _pulsed = true; + r = pthread_cond_signal(&_cond); + assert_debug(r == 0); + r = pthread_mutex_unlock(&_mutex); + assert_debug(r == 0); +} +inline bool event_t::signalled(void) { + bool ret = false; + int r = pthread_mutex_lock(&_mutex); + assert_debug(r == 0); + ret = _signalled; + r = pthread_mutex_unlock(&_mutex); + assert_debug(r == 0); + return ret; +} +inline void event_t::reset(void) { + int r = pthread_mutex_lock(&_mutex); + assert_debug(r == 0); + _signalled = false; + _pulsed = false; + r = pthread_mutex_unlock(&_mutex); + assert_debug(r == 0); + return; +} +inline event_t::event_t(const event_t&) { +} +inline event_t& event_t::operator=(const event_t&) { + return *this; +} + + +inline semaphore_t::semaphore_t( + int initial_count, + int max_count) : + _interrupted(false), + _initial_count(initial_count), + _max_count(max_count) { + + int r = pthread_mutex_init(&_mutex, NULL); + assert_debug(r == 0); + r = pthread_cond_init(&_cond, NULL); + assert_debug(r == 0); + _signalled = _initial_count; +} +inline semaphore_t::~semaphore_t(void) { + int r = pthread_mutex_destroy(&_mutex); + assert_debug(r == 0); + r = pthread_cond_destroy(&_cond); + assert_debug(r == 0); +} +inline semaphore_t::E_WAIT semaphore_t::wait(void) { + E_WAIT ret; + int r = pthread_mutex_lock(&_mutex); + assert_debug(r == 0); + while (_signalled == 0 && _interrupted == false) { + r = pthread_cond_wait(&_cond, &_mutex); + assert_debug(r == 0); + } + if (_interrupted) { + ret = E_INTERRUPTED; + } else { + _signalled--; + ret = E_SIGNALLED; + } + r = pthread_mutex_unlock(&_mutex); + assert_debug(r == 0); + return ret; +} +inline semaphore_t::E_WAIT semaphore_t::wait(ulonglong microseconds) { + E_WAIT ret; + timespec waittime = time::offset_timespec(microseconds); + int r = pthread_mutex_timedlock(&_mutex, &waittime); + if (r == ETIMEDOUT) return E_TIMEDOUT; + assert_debug(r == 0); + while (_signalled == 0 && _interrupted == false) { + r = pthread_cond_timedwait(&_cond, &_mutex, &waittime); + if (r == ETIMEDOUT) { + r = pthread_mutex_unlock(&_mutex); + assert_debug(r == 0); + return E_TIMEDOUT; + } + assert_debug(r == 0); + } + if (_interrupted) { + ret = E_INTERRUPTED; + } else { + _signalled--; + ret = E_SIGNALLED; + } + r = pthread_mutex_unlock(&_mutex); + assert_debug(r == 0); + return ret; +} +inline bool semaphore_t::signal(void) { + bool ret = false; + int r = pthread_mutex_lock(&_mutex); + assert_debug(r == 0); + if (_signalled < _max_count) { + _signalled++; + ret = true; + } + r = pthread_cond_signal(&_cond); + assert_debug(r == 0); + r = pthread_mutex_unlock(&_mutex); + assert_debug(r == 0); + return ret; +} +inline int semaphore_t::signalled(void) { + int ret = 0; + int r = pthread_mutex_lock(&_mutex); + assert_debug(r == 0); + ret = _signalled; + r = pthread_mutex_unlock(&_mutex); + assert_debug(r == 0); + return ret; +} +inline void semaphore_t::reset(void) { + int r = pthread_mutex_lock(&_mutex); + assert_debug(r == 0); + _signalled = 0; + r = pthread_mutex_unlock(&_mutex); + assert_debug(r == 0); + return; +} +inline void semaphore_t::set_interrupt(void) { + int r = pthread_mutex_lock(&_mutex); + assert_debug(r == 0); + _interrupted = true; + r = pthread_cond_broadcast(&_cond); + assert_debug(r == 0); + r = pthread_mutex_unlock(&_mutex); + assert_debug(r == 0); +} +inline void semaphore_t::clear_interrupt(void) { + int r = pthread_mutex_lock(&_mutex); + assert_debug(r == 0); + _interrupted = false; + r = pthread_mutex_unlock(&_mutex); + assert_debug(r == 0); +} +inline semaphore_t::semaphore_t(const semaphore_t&) { +} +inline semaphore_t& semaphore_t::operator=(const semaphore_t&) { + return *this; +} + + +inline thread_t::thread_t(void) : _thread(0) { +} +inline thread_t::~thread_t(void) { +} +inline int thread_t::start(void*(*pfn)(void*), void* arg) { + return pthread_create(&_thread, NULL, pfn, arg); +} +inline int thread_t::join(void** value_ptr) { + return pthread_join(_thread, value_ptr); +} +inline int thread_t::detach(void) { + return pthread_detach(_thread); +} + +} // namespace thread +} // namespace tokudb + + +#endif // _TOKUDB_SYNC_H diff --git a/storage/tokudb/tokudb_time.h b/storage/tokudb/tokudb_time.h new file mode 100644 index 00000000000..12baa0de24d --- /dev/null +++ b/storage/tokudb/tokudb_time.h @@ -0,0 +1,73 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +/* -*- mode: C; c-basic-offset: 4 -*- */ +#ident "$Id$" +/*====== +This file is part of TokuDB + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + TokuDBis is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + TokuDB is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TokuDB. If not, see <http://www.gnu.org/licenses/>. + +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#ifndef _TOKUDB_TIME_H +#define _TOKUDB_TIME_H + +#include "hatoku_defines.h" + +namespace tokudb { +namespace time { + +static const ulonglong MILLISECONDS = 1000; +static const ulonglong MICROSECONDS = 1000000; +static const ulonglong NANOSECONDS = 1000000000; + +// gets curent time of day in microseconds +ulonglong microsec(void); + +// gets a timespec in the future based on the current time and an offset forward +timespec offset_timespec(ulonglong offset); + +// sleep microseconds +void sleep_microsec(ulong tm); + + + +inline ulonglong microsec(void) { + timeval t; + gettimeofday(&t, NULL); + return t.tv_sec * (1UL * 1000 * 1000) + t.tv_usec; +} +inline timespec offset_timespec(ulonglong offset) { + timespec ret; + ulonglong tm = offset + microsec(); + ret.tv_sec = tm / MICROSECONDS; + ret.tv_nsec = (tm % MICROSECONDS) * 1000; + return ret; +} +inline void sleep_microsec(ulong tm) { + timeval t; + t.tv_sec = tm / MICROSECONDS; + t.tv_usec = tm % MICROSECONDS; + + select(0, NULL, NULL, NULL, &t); +} + +} // namespace time +} // namespace tokudb + +#endif // _TOKUDB_TIME_H diff --git a/storage/tokudb/tokudb_txn.h b/storage/tokudb/tokudb_txn.h new file mode 100644 index 00000000000..67bf591d088 --- /dev/null +++ b/storage/tokudb/tokudb_txn.h @@ -0,0 +1,155 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +/* -*- mode: C; c-basic-offset: 4 -*- */ +#ident "$Id$" +/*====== +This file is part of TokuDB + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + TokuDBis is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + TokuDB is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TokuDB. If not, see <http://www.gnu.org/licenses/>. + +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#ifndef _TOKUDB_TXN_H +#define _TOKUDB_TXN_H + +#include "hatoku_defines.h" +#include "tokudb_debug.h" +#include "tokudb_sysvars.h" + +typedef enum { + hatoku_iso_not_set = 0, + hatoku_iso_read_uncommitted, + hatoku_iso_read_committed, + hatoku_iso_repeatable_read, + hatoku_iso_serializable +} HA_TOKU_ISO_LEVEL; + +typedef struct st_tokudb_stmt_progress { + ulonglong inserted; + ulonglong updated; + ulonglong deleted; + ulonglong queried; + bool using_loader; +} tokudb_stmt_progress; + +typedef struct st_tokudb_trx_data { + DB_TXN* all; + DB_TXN* stmt; + DB_TXN* sp_level; + DB_TXN* sub_sp_level; + uint tokudb_lock_count; + uint create_lock_count; + tokudb_stmt_progress stmt_progress; + bool checkpoint_lock_taken; + LIST* handlers; +} tokudb_trx_data; + +extern char* tokudb_data_dir; +extern const char* ha_tokudb_ext; + +inline void reset_stmt_progress(tokudb_stmt_progress* val) { + val->deleted = 0; + val->inserted = 0; + val->updated = 0; + val->queried = 0; +} + +inline int get_name_length(const char* name) { + int n = 0; + const char* newname = name; + n += strlen(newname); + n += strlen(ha_tokudb_ext); + return n; +} + +// +// returns maximum length of path to a dictionary +// +inline int get_max_dict_name_path_length(const char* tablename) { + int n = 0; + n += get_name_length(tablename); + n += 1; //for the '-' + n += MAX_DICT_NAME_LEN; + return n; +} + +inline void make_name( + char* newname, + size_t newname_len, + const char* tablename, + const char* dictname) { + + assert_always(tablename); + assert_always(dictname); + size_t real_size = snprintf( + newname, + newname_len, + "%s-%s", + tablename, + dictname); + assert_always(real_size < newname_len); +} + +inline int txn_begin( + DB_ENV* env, + DB_TXN* parent, + DB_TXN** txn, + uint32_t flags, + THD* thd) { + + *txn = NULL; + int r = env->txn_begin(env, parent, txn, flags); + if (r == 0 && thd) { + DB_TXN* this_txn = *txn; + this_txn->set_client_id(this_txn, thd_get_thread_id(thd)); + } + TOKUDB_TRACE_FOR_FLAGS( + TOKUDB_DEBUG_TXN, + "begin txn %p %p %u r=%d", + parent, + *txn, + flags, + r); + return r; +} + +inline void commit_txn(DB_TXN* txn, uint32_t flags) { + TOKUDB_TRACE_FOR_FLAGS(TOKUDB_DEBUG_TXN, "commit txn %p", txn); + int r = txn->commit(txn, flags); + if (r != 0) { + sql_print_error( + "tried committing transaction %p and got error code %d", + txn, + r); + } + assert_always(r == 0); +} + +inline void abort_txn(DB_TXN* txn) { + TOKUDB_TRACE_FOR_FLAGS(TOKUDB_DEBUG_TXN, "abort txn %p", txn); + int r = txn->abort(txn); + if (r != 0) { + sql_print_error( + "tried aborting transaction %p and got error code %d", + txn, + r); + } + assert_always(r == 0); +} + +#endif // _TOKUDB_TXN_H diff --git a/storage/tokudb/tokudb_update_fun.cc b/storage/tokudb/tokudb_update_fun.cc index 4647c91880b..05149d17317 100644 --- a/storage/tokudb/tokudb_update_fun.cc +++ b/storage/tokudb/tokudb_update_fun.cc @@ -105,7 +105,8 @@ enum { // at most, 4 0's // So, upperbound is num_blobs(1+4+1+4) = num_columns*10 -// The expand varchar offsets message is used to expand the size of an offset from 1 to 2 bytes. Not VLQ coded. +// The expand varchar offsets message is used to expand the size of an offset +// from 1 to 2 bytes. Not VLQ coded. // uint8 operation = UPDATE_OP_EXPAND_VARIABLE_OFFSETS // uint32 number of offsets // uint32 starting offset of the variable length field offsets @@ -132,8 +133,8 @@ enum { // uint8 old lengths[N] // uint8 new lengths[N] -// Update and Upsert version 1 messages. Not VLQ coded. Not used anymore, but may be in the -// fractal tree from a previous build. +// Update and Upsert version 1 messages. Not VLQ coded. Not used anymore, but +// may be in the fractal tree from a previous build. // // Field descriptor: // Operations: @@ -143,8 +144,10 @@ enum { // x = x - k // field type 4 see field types above // unused 4 unused -// field null num 4 bit 31 is 1 if the field is nullible and the remaining bits contain the null bit number -// field offset 4 for fixed fields, this is the offset from begining of the row of the field +// field null num 4 bit 31 is 1 if the field is nullible and the +// remaining bits contain the null bit number +// field offset 4 for fixed fields, this is the offset from +// begining of the row of the field // value: // value length 4 == N, length of the value // value N value to add or subtract @@ -234,7 +237,11 @@ static inline bool is_overall_null_position_set(uchar* data, uint32_t pos) { // // sets the bit at index pos in data to 1 if is_null, 0 otherwise // -static inline void set_overall_null_position(uchar* data, uint32_t pos, bool is_null) { +static inline void set_overall_null_position( + uchar* data, + uint32_t pos, + bool is_null) { + uint32_t offset = pos/8; uchar remainder = pos%8; uchar null_bit = 1<<remainder; @@ -251,9 +258,7 @@ static inline void copy_null_bits( uint32_t start_new_pos, uint32_t num_bits, uchar* old_null_bytes, - uchar* new_null_bytes - ) -{ + uchar* new_null_bytes) { for (uint32_t i = 0; i < num_bits; i++) { uint32_t curr_old_pos = i + start_old_pos; uint32_t curr_new_pos = i + start_new_pos; @@ -268,19 +273,27 @@ static inline void copy_null_bits( } static inline void copy_var_fields( - uint32_t start_old_num_var_field, //index of var fields that we should start writing - uint32_t num_var_fields, // number of var fields to copy - uchar* old_var_field_offset_ptr, //static ptr to where offset bytes begin in old row - uchar old_num_offset_bytes, //number of offset bytes used in old row - uchar* start_new_var_field_data_ptr, // where the new var data should be written - uchar* start_new_var_field_offset_ptr, // where the new var offsets should be written - uchar* new_var_field_data_ptr, // pointer to beginning of var fields in new row - uchar* old_var_field_data_ptr, // pointer to beginning of var fields in old row - uint32_t new_num_offset_bytes, // number of offset bytes used in new row + //index of var fields that we should start writing + uint32_t start_old_num_var_field, + // number of var fields to copy + uint32_t num_var_fields, + //static ptr to where offset bytes begin in old row + uchar* old_var_field_offset_ptr, + //number of offset bytes used in old row + uchar old_num_offset_bytes, + // where the new var data should be written + uchar* start_new_var_field_data_ptr, + // where the new var offsets should be written + uchar* start_new_var_field_offset_ptr, + // pointer to beginning of var fields in new row + uchar* new_var_field_data_ptr, + // pointer to beginning of var fields in old row + uchar* old_var_field_data_ptr, + // number of offset bytes used in new row + uint32_t new_num_offset_bytes, uint32_t* num_data_bytes_written, - uint32_t* num_offset_bytes_written - ) -{ + uint32_t* num_offset_bytes_written) { + uchar* curr_new_var_field_data_ptr = start_new_var_field_data_ptr; uchar* curr_new_var_field_offset_ptr = start_new_var_field_offset_ptr; for (uint32_t i = 0; i < num_var_fields; i++) { @@ -290,12 +303,11 @@ static inline void copy_var_fields( uchar* data_to_copy = NULL; // get the length and pointer to data that needs to be copied get_var_field_info( - &field_len, - &start_read_offset, - curr_old, - old_var_field_offset_ptr, - old_num_offset_bytes - ); + &field_len, + &start_read_offset, + curr_old, + old_var_field_offset_ptr, + old_num_offset_bytes); data_to_copy = old_var_field_data_ptr + start_read_offset; // now need to copy field_len bytes starting from data_to_copy curr_new_var_field_data_ptr = write_var_field( @@ -304,15 +316,22 @@ static inline void copy_var_fields( new_var_field_data_ptr, data_to_copy, field_len, - new_num_offset_bytes - ); + new_num_offset_bytes); curr_new_var_field_offset_ptr += new_num_offset_bytes; } - *num_data_bytes_written = (uint32_t)(curr_new_var_field_data_ptr - start_new_var_field_data_ptr); - *num_offset_bytes_written = (uint32_t)(curr_new_var_field_offset_ptr - start_new_var_field_offset_ptr); + *num_data_bytes_written = + (uint32_t)(curr_new_var_field_data_ptr - start_new_var_field_data_ptr); + *num_offset_bytes_written = + (uint32_t)(curr_new_var_field_offset_ptr - + start_new_var_field_offset_ptr); } -static inline uint32_t copy_toku_blob(uchar* to_ptr, uchar* from_ptr, uint32_t len_bytes, bool skip) { +static inline uint32_t copy_toku_blob( + uchar* to_ptr, + uchar* from_ptr, + uint32_t len_bytes, + bool skip) { + uint32_t length = 0; if (!skip) { memcpy(to_ptr, from_ptr, len_bytes); @@ -326,13 +345,12 @@ static inline uint32_t copy_toku_blob(uchar* to_ptr, uchar* from_ptr, uint32_t l static int tokudb_hcad_update_fun( DB* db, - const DBT *key, - const DBT *old_val, - const DBT *extra, - void (*set_val)(const DBT *new_val, void *set_extra), - void *set_extra - ) -{ + const DBT* key, + const DBT* old_val, + const DBT* extra, + void (*set_val)(const DBT* new_val, void* set_extra), + void* set_extra) { + uint32_t max_num_bytes; uint32_t num_columns; DBT new_val; @@ -397,7 +415,7 @@ static int tokudb_hcad_update_fun( operation = extra_pos[0]; extra_pos++; - assert(operation == UP_COL_ADD_OR_DROP); + assert_always(operation == UP_COL_ADD_OR_DROP); memcpy(&old_num_null_bytes, extra_pos, sizeof(uint32_t)); extra_pos += sizeof(uint32_t); @@ -419,12 +437,15 @@ static int tokudb_hcad_update_fun( memcpy(&new_len_of_offsets, extra_pos, sizeof(uint32_t)); extra_pos += sizeof(uint32_t); - max_num_bytes = old_val->size + extra->size + new_len_of_offsets + new_fixed_field_size; - new_val_data = (uchar *)tokudb_my_malloc( - max_num_bytes, - MYF(MY_FAE) - ); - if (new_val_data == NULL) { goto cleanup; } + max_num_bytes = + old_val->size + extra->size + new_len_of_offsets + new_fixed_field_size; + new_val_data = (uchar *)tokudb::memory::malloc( + max_num_bytes, + MYF(MY_FAE)); + if (new_val_data == NULL) { + error = ENOMEM; + goto cleanup; + } old_fixed_field_ptr = (uchar *) old_val->data; old_fixed_field_ptr += old_num_null_bytes; @@ -465,7 +486,7 @@ static int tokudb_hcad_update_fun( bool is_null_default = false; extra_pos++; - assert(op_type == COL_DROP || op_type == COL_ADD); + assert_always(op_type == COL_DROP || op_type == COL_ADD); bool nullable = (extra_pos[0] != 0); extra_pos++; if (nullable) { @@ -474,11 +495,10 @@ static int tokudb_hcad_update_fun( extra_pos += sizeof(uint32_t); uint32_t num_bits; if (op_type == COL_DROP) { - assert(curr_old_null_pos <= null_bit_position); + assert_always(curr_old_null_pos <= null_bit_position); num_bits = null_bit_position - curr_old_null_pos; - } - else { - assert(curr_new_null_pos <= null_bit_position); + } else { + assert_always(curr_new_null_pos <= null_bit_position); num_bits = null_bit_position - curr_new_null_pos; } copy_null_bits( @@ -486,22 +506,19 @@ static int tokudb_hcad_update_fun( curr_new_null_pos, num_bits, old_null_bytes, - new_null_bytes - ); + new_null_bytes); // update the positions curr_new_null_pos += num_bits; curr_old_null_pos += num_bits; if (op_type == COL_DROP) { curr_old_null_pos++; // account for dropped column - } - else { + } else { is_null_default = (extra_pos[0] != 0); extra_pos++; set_overall_null_position( new_null_bytes, null_bit_position, - is_null_default - ); + is_null_default); curr_new_null_pos++; //account for added column } } @@ -518,47 +535,44 @@ static int tokudb_hcad_update_fun( if (op_type == COL_DROP) { num_bytes_to_copy = col_offset - curr_old_fixed_offset; - } - else { + } else { num_bytes_to_copy = col_offset - curr_new_fixed_offset; } memcpy( new_fixed_field_ptr + curr_new_fixed_offset, - old_fixed_field_ptr + curr_old_fixed_offset, - num_bytes_to_copy - ); + old_fixed_field_ptr + curr_old_fixed_offset, + num_bytes_to_copy); curr_old_fixed_offset += num_bytes_to_copy; curr_new_fixed_offset += num_bytes_to_copy; if (op_type == COL_DROP) { - // move old_fixed_offset val to skip OVER column that is being dropped + // move old_fixed_offset val to skip OVER column that is + // being dropped curr_old_fixed_offset += col_size; - } - else { + } else { if (is_null_default) { // copy zeroes - memset(new_fixed_field_ptr + curr_new_fixed_offset, 0, col_size); - } - else { + memset( + new_fixed_field_ptr + curr_new_fixed_offset, + 0, + col_size); + } else { // copy data from extra_pos into new row memcpy( new_fixed_field_ptr + curr_new_fixed_offset, extra_pos, - col_size - ); + col_size); extra_pos += col_size; } curr_new_fixed_offset += col_size; } - } - else if (col_type == COL_VAR) { + } else if (col_type == COL_VAR) { uint32_t var_col_index; memcpy(&var_col_index, extra_pos, sizeof(uint32_t)); extra_pos += sizeof(uint32_t); if (op_type == COL_DROP) { num_var_fields_to_copy = var_col_index - curr_old_num_var_field; - } - else { + } else { num_var_fields_to_copy = var_col_index - curr_new_num_var_field; } copy_var_fields( @@ -568,20 +582,21 @@ static int tokudb_hcad_update_fun( old_num_offset_bytes, curr_new_var_field_data_ptr, curr_new_var_field_offset_ptr, - new_var_field_data_ptr, // pointer to beginning of var fields in new row - old_var_field_data_ptr, // pointer to beginning of var fields in old row - new_num_offset_bytes, // number of offset bytes used in new row + // pointer to beginning of var fields in new row + new_var_field_data_ptr, + // pointer to beginning of var fields in old row + old_var_field_data_ptr, + // number of offset bytes used in new row + new_num_offset_bytes, &num_data_bytes_written, - &num_offset_bytes_written - ); + &num_offset_bytes_written); curr_new_var_field_data_ptr += num_data_bytes_written; curr_new_var_field_offset_ptr += num_offset_bytes_written; curr_new_num_var_field += num_var_fields_to_copy; curr_old_num_var_field += num_var_fields_to_copy; if (op_type == COL_DROP) { curr_old_num_var_field++; // skip over dropped field - } - else { + } else { if (is_null_default) { curr_new_var_field_data_ptr = write_var_field( curr_new_var_field_offset_ptr, @@ -589,11 +604,9 @@ static int tokudb_hcad_update_fun( new_var_field_data_ptr, NULL, //copying no data 0, //copying 0 bytes - new_num_offset_bytes - ); + new_num_offset_bytes); curr_new_var_field_offset_ptr += new_num_offset_bytes; - } - else { + } else { uint32_t data_length; memcpy(&data_length, extra_pos, sizeof(data_length)); extra_pos += sizeof(data_length); @@ -603,20 +616,17 @@ static int tokudb_hcad_update_fun( new_var_field_data_ptr, extra_pos, //copying data from mutator data_length, //copying data_length bytes - new_num_offset_bytes - ); + new_num_offset_bytes); extra_pos += data_length; curr_new_var_field_offset_ptr += new_num_offset_bytes; } curr_new_num_var_field++; //account for added column } - } - else if (col_type == COL_BLOB) { + } else if (col_type == COL_BLOB) { // handle blob data later continue; - } - else { - assert(false); + } else { + assert_unreachable(); } } // finish copying the null stuff @@ -629,19 +639,17 @@ static int tokudb_hcad_update_fun( curr_new_null_pos, overall_null_bits_left, old_null_bytes, - new_null_bytes - ); + new_null_bytes); // finish copying fixed field stuff num_bytes_left = old_fixed_field_size - curr_old_fixed_offset; memcpy( new_fixed_field_ptr + curr_new_fixed_offset, old_fixed_field_ptr + curr_old_fixed_offset, - num_bytes_left - ); + num_bytes_left); curr_old_fixed_offset += num_bytes_left; curr_new_fixed_offset += num_bytes_left; // sanity check - assert(curr_new_fixed_offset == new_fixed_field_size); + assert_always(curr_new_fixed_offset == new_fixed_field_size); // finish copying var field stuff num_var_fields_to_copy = old_num_var_fields - curr_old_num_var_field; @@ -652,33 +660,34 @@ static int tokudb_hcad_update_fun( old_num_offset_bytes, curr_new_var_field_data_ptr, curr_new_var_field_offset_ptr, - new_var_field_data_ptr, // pointer to beginning of var fields in new row - old_var_field_data_ptr, // pointer to beginning of var fields in old row - new_num_offset_bytes, // number of offset bytes used in new row + // pointer to beginning of var fields in new row + new_var_field_data_ptr, + // pointer to beginning of var fields in old row + old_var_field_data_ptr, + // number of offset bytes used in new row + new_num_offset_bytes, &num_data_bytes_written, - &num_offset_bytes_written - ); + &num_offset_bytes_written); curr_new_var_field_offset_ptr += num_offset_bytes_written; curr_new_var_field_data_ptr += num_data_bytes_written; // sanity check - assert(curr_new_var_field_offset_ptr == new_var_field_data_ptr); + assert_always(curr_new_var_field_offset_ptr == new_var_field_data_ptr); // start handling blobs get_blob_field_info( &start_blob_offset, old_len_of_offsets, old_var_field_data_ptr, - old_num_offset_bytes - ); + old_num_offset_bytes); start_blob_ptr = old_var_field_data_ptr + start_blob_offset; - // if nothing else in extra, then there are no blobs to add or drop, so can copy blobs straight + // if nothing else in extra, then there are no blobs to add or drop, so + // can copy blobs straight if ((extra_pos - extra_pos_start) == extra->size) { num_blob_bytes = old_val->size - (start_blob_ptr - old_null_bytes); memcpy(curr_new_var_field_data_ptr, start_blob_ptr, num_blob_bytes); curr_new_var_field_data_ptr += num_blob_bytes; - } - // else, there is blob information to process - else { + } else { + // else, there is blob information to process uchar* len_bytes = NULL; uint32_t curr_old_blob = 0; uint32_t curr_new_blob = 0; @@ -696,11 +705,10 @@ static int tokudb_hcad_update_fun( uint32_t blob_index; memcpy(&blob_index, extra_pos, sizeof(blob_index)); extra_pos += sizeof(blob_index); - assert (op_type == COL_DROP || op_type == COL_ADD); + assert_always (op_type == COL_DROP || op_type == COL_ADD); if (op_type == COL_DROP) { num_blobs_to_copy = blob_index - curr_old_blob; - } - else { + } else { num_blobs_to_copy = blob_index - curr_new_blob; } for (uint32_t i = 0; i < num_blobs_to_copy; i++) { @@ -708,8 +716,7 @@ static int tokudb_hcad_update_fun( curr_new_var_field_data_ptr, curr_old_blob_ptr, len_bytes[curr_old_blob + i], - false - ); + false); curr_old_blob_ptr += num_bytes_written; curr_new_var_field_data_ptr += num_bytes_written; } @@ -721,12 +728,10 @@ static int tokudb_hcad_update_fun( NULL, curr_old_blob_ptr, len_bytes[curr_old_blob], - true - ); + true); curr_old_blob++; curr_old_blob_ptr += num_bytes; - } - else { + } else { // copy new data uint32_t new_len_bytes = extra_pos[0]; extra_pos++; @@ -734,8 +739,7 @@ static int tokudb_hcad_update_fun( curr_new_var_field_data_ptr, extra_pos, new_len_bytes, - false - ); + false); curr_new_blob++; curr_new_var_field_data_ptr += num_bytes; extra_pos += num_bytes; @@ -751,27 +755,27 @@ static int tokudb_hcad_update_fun( error = 0; cleanup: - tokudb_my_free(new_val_data); + tokudb::memory::free(new_val_data); return error; } -// Expand the variable offset array in the old row given the update mesage in the extra. +// Expand the variable offset array in the old row given the update mesage +// in the extra. static int tokudb_expand_variable_offsets( DB* db, - const DBT *key, - const DBT *old_val, - const DBT *extra, - void (*set_val)(const DBT *new_val, void *set_extra), - void *set_extra - ) -{ + const DBT* key, + const DBT* old_val, + const DBT* extra, + void (*set_val)(const DBT* new_val, void* set_extra), + void* set_extra) { + int error = 0; tokudb::buffer extra_val(extra->data, 0, extra->size); // decode the operation uint8_t operation; extra_val.consume(&operation, sizeof operation); - assert(operation == UPDATE_OP_EXPAND_VARIABLE_OFFSETS); + assert_always(operation == UPDATE_OP_EXPAND_VARIABLE_OFFSETS); // decode number of offsets uint32_t number_of_offsets; @@ -781,18 +785,20 @@ static int tokudb_expand_variable_offsets( uint32_t offset_start; extra_val.consume(&offset_start, sizeof offset_start); - assert(extra_val.size() == extra_val.limit()); + assert_always(extra_val.size() == extra_val.limit()); DBT new_val; memset(&new_val, 0, sizeof new_val); if (old_val != NULL) { - assert(offset_start + number_of_offsets <= old_val->size); + assert_always(offset_start + number_of_offsets <= old_val->size); // compute the new val from the old val - uchar *old_val_ptr = (uchar *)old_val->data; + uchar* old_val_ptr = (uchar*)old_val->data; // allocate space for the new val's data - uchar *new_val_ptr = (uchar *)tokudb_my_malloc(number_of_offsets + old_val->size, MYF(MY_FAE)); + uchar* new_val_ptr = (uchar*)tokudb::memory::malloc( + number_of_offsets + old_val->size, + MYF(MY_FAE)); if (!new_val_ptr) { error = ENOMEM; goto cleanup; @@ -819,8 +825,8 @@ static int tokudb_expand_variable_offsets( old_val_ptr += n; new_val.size = new_val_ptr - (uchar *)new_val.data; - assert(new_val_ptr == (uchar *)new_val.data + new_val.size); - assert(old_val_ptr == (uchar *)old_val->data + old_val->size); + assert_always(new_val_ptr == (uchar *)new_val.data + new_val.size); + assert_always(old_val_ptr == (uchar *)old_val->data + old_val->size); // set the new val set_val(&new_val, set_extra); @@ -829,46 +835,50 @@ static int tokudb_expand_variable_offsets( error = 0; cleanup: - tokudb_my_free(new_val.data); + tokudb::memory::free(new_val.data); return error; } // Expand an int field in a old row given the expand message in the extra. static int tokudb_expand_int_field( DB* db, - const DBT *key, - const DBT *old_val, - const DBT *extra, - void (*set_val)(const DBT *new_val, void *set_extra), - void *set_extra - ) -{ + const DBT* key, + const DBT* old_val, + const DBT* extra, + void (*set_val)(const DBT* new_val, void* set_extra), + void* set_extra) { + int error = 0; tokudb::buffer extra_val(extra->data, 0, extra->size); uint8_t operation; extra_val.consume(&operation, sizeof operation); - assert(operation == UPDATE_OP_EXPAND_INT || operation == UPDATE_OP_EXPAND_UINT); + assert_always( + operation == UPDATE_OP_EXPAND_INT || + operation == UPDATE_OP_EXPAND_UINT); uint32_t the_offset; extra_val.consume(&the_offset, sizeof the_offset); uint32_t old_length; extra_val.consume(&old_length, sizeof old_length); uint32_t new_length; extra_val.consume(&new_length, sizeof new_length); - assert(extra_val.size() == extra_val.limit()); + assert_always(extra_val.size() == extra_val.limit()); - assert(new_length >= old_length); // expand only + assert_always(new_length >= old_length); // expand only DBT new_val; memset(&new_val, 0, sizeof new_val); if (old_val != NULL) { - assert(the_offset + old_length <= old_val->size); // old field within the old val + // old field within the old val + assert_always(the_offset + old_length <= old_val->size); // compute the new val from the old val - uchar *old_val_ptr = (uchar *)old_val->data; + uchar* old_val_ptr = (uchar*)old_val->data; // allocate space for the new val's data - uchar *new_val_ptr = (uchar *)tokudb_my_malloc(old_val->size + (new_length - old_length), MYF(MY_FAE)); + uchar* new_val_ptr = (uchar*)tokudb::memory::malloc( + old_val->size + (new_length - old_length), + MYF(MY_FAE)); if (!new_val_ptr) { error = ENOMEM; goto cleanup; @@ -882,21 +892,27 @@ static int tokudb_expand_int_field( switch (operation) { case UPDATE_OP_EXPAND_INT: - // fill the entire new value with ones or zeros depending on the sign bit - // the encoding is little endian - memset(new_val_ptr, (old_val_ptr[old_length-1] & 0x80) ? 0xff : 0x00, new_length); - memcpy(new_val_ptr, old_val_ptr, old_length); // overlay the low bytes of the new value with the old value + // fill the entire new value with ones or zeros depending on the + // sign bit the encoding is little endian + memset( + new_val_ptr, + (old_val_ptr[old_length-1] & 0x80) ? 0xff : 0x00, + new_length); + // overlay the low bytes of the new value with the old value + memcpy(new_val_ptr, old_val_ptr, old_length); new_val_ptr += new_length; old_val_ptr += old_length; break; case UPDATE_OP_EXPAND_UINT: - memset(new_val_ptr, 0, new_length); // fill the entire new value with zeros - memcpy(new_val_ptr, old_val_ptr, old_length); // overlay the low bytes of the new value with the old value + // fill the entire new value with zeros + memset(new_val_ptr, 0, new_length); + // overlay the low bytes of the new value with the old value + memcpy(new_val_ptr, old_val_ptr, old_length); new_val_ptr += new_length; old_val_ptr += old_length; break; default: - assert(0); + assert_unreachable(); } // copy the rest @@ -906,8 +922,8 @@ static int tokudb_expand_int_field( old_val_ptr += n; new_val.size = new_val_ptr - (uchar *)new_val.data; - assert(new_val_ptr == (uchar *)new_val.data + new_val.size); - assert(old_val_ptr == (uchar *)old_val->data + old_val->size); + assert_always(new_val_ptr == (uchar *)new_val.data + new_val.size); + assert_always(old_val_ptr == (uchar *)old_val->data + old_val->size); // set the new val set_val(&new_val, set_extra); @@ -916,26 +932,27 @@ static int tokudb_expand_int_field( error = 0; cleanup: - tokudb_my_free(new_val.data); + tokudb::memory::free(new_val.data); return error; } // Expand a char field in a old row given the expand message in the extra. static int tokudb_expand_char_field( DB* db, - const DBT *key, - const DBT *old_val, - const DBT *extra, - void (*set_val)(const DBT *new_val, void *set_extra), - void *set_extra - ) -{ + const DBT* key, + const DBT* old_val, + const DBT* extra, + void (*set_val)(const DBT* new_val, void* set_extra), + void* set_extra) { + int error = 0; tokudb::buffer extra_val(extra->data, 0, extra->size); uint8_t operation; extra_val.consume(&operation, sizeof operation); - assert(operation == UPDATE_OP_EXPAND_CHAR || operation == UPDATE_OP_EXPAND_BINARY); + assert_always( + operation == UPDATE_OP_EXPAND_CHAR || + operation == UPDATE_OP_EXPAND_BINARY); uint32_t the_offset; extra_val.consume(&the_offset, sizeof the_offset); uint32_t old_length; @@ -944,20 +961,23 @@ static int tokudb_expand_char_field( extra_val.consume(&new_length, sizeof new_length); uchar pad_char; extra_val.consume(&pad_char, sizeof pad_char); - assert(extra_val.size() == extra_val.limit()); + assert_always(extra_val.size() == extra_val.limit()); - assert(new_length >= old_length); // expand only + assert_always(new_length >= old_length); // expand only DBT new_val; memset(&new_val, 0, sizeof new_val); if (old_val != NULL) { - assert(the_offset + old_length <= old_val->size); // old field within the old val + // old field within the old val + assert_always(the_offset + old_length <= old_val->size); // compute the new val from the old val - uchar *old_val_ptr = (uchar *)old_val->data; + uchar* old_val_ptr = (uchar*)old_val->data; // allocate space for the new val's data - uchar *new_val_ptr = (uchar *)tokudb_my_malloc(old_val->size + (new_length - old_length), MYF(MY_FAE)); + uchar* new_val_ptr = (uchar*)tokudb::memory::malloc( + old_val->size + (new_length - old_length), + MYF(MY_FAE)); if (!new_val_ptr) { error = ENOMEM; goto cleanup; @@ -972,13 +992,15 @@ static int tokudb_expand_char_field( switch (operation) { case UPDATE_OP_EXPAND_CHAR: case UPDATE_OP_EXPAND_BINARY: - memset(new_val_ptr, pad_char, new_length); // fill the entire new value with the pad char - memcpy(new_val_ptr, old_val_ptr, old_length); // overlay the low bytes of the new value with the old value + // fill the entire new value with the pad char + memset(new_val_ptr, pad_char, new_length); + // overlay the low bytes of the new value with the old value + memcpy(new_val_ptr, old_val_ptr, old_length); new_val_ptr += new_length; old_val_ptr += old_length; break; default: - assert(0); + assert_unreachable(); } // copy the rest @@ -988,8 +1010,8 @@ static int tokudb_expand_char_field( old_val_ptr += n; new_val.size = new_val_ptr - (uchar *)new_val.data; - assert(new_val_ptr == (uchar *)new_val.data + new_val.size); - assert(old_val_ptr == (uchar *)old_val->data + old_val->size); + assert_always(new_val_ptr == (uchar *)new_val.data + new_val.size); + assert_always(old_val_ptr == (uchar *)old_val->data + old_val->size); // set the new val set_val(&new_val, set_extra); @@ -998,7 +1020,7 @@ static int tokudb_expand_char_field( error = 0; cleanup: - tokudb_my_free(new_val.data); + tokudb::memory::free(new_val.data); return error; } @@ -1006,17 +1028,25 @@ namespace tokudb { class var_fields { public: - var_fields() { + inline var_fields() { } - void init_var_fields(uint32_t var_offset, uint32_t offset_bytes, uint32_t bytes_per_offset, tokudb::buffer *val_buffer) { - assert(bytes_per_offset == 0 || bytes_per_offset == 1 || bytes_per_offset == 2); + inline void init_var_fields( + uint32_t var_offset, + uint32_t offset_bytes, + uint32_t bytes_per_offset, + tokudb::buffer* val_buffer) { + + assert_always( + bytes_per_offset == 0 || + bytes_per_offset == 1 || + bytes_per_offset == 2); m_var_offset = var_offset; m_val_offset = m_var_offset + offset_bytes; m_bytes_per_offset = bytes_per_offset; if (bytes_per_offset > 0) { m_num_fields = offset_bytes / bytes_per_offset; } else { - assert(offset_bytes == 0); + assert_always(offset_bytes == 0); m_num_fields = 0; } m_val_buffer = val_buffer; @@ -1025,7 +1055,10 @@ public: uint32_t value_length(uint32_t var_index); void update_offsets(uint32_t var_index, uint32_t old_s, uint32_t new_s); uint32_t end_offset(); - void replace(uint32_t var_index, void *new_val_ptr, uint32_t new_val_length); + void replace( + uint32_t var_index, + void* new_val_ptr, + uint32_t new_val_length); private: uint32_t read_offset(uint32_t var_index); void write_offset(uint32_t var_index, uint32_t v); @@ -1034,24 +1067,28 @@ private: uint32_t m_val_offset; uint32_t m_bytes_per_offset; uint32_t m_num_fields; - tokudb::buffer *m_val_buffer; + tokudb::buffer* m_val_buffer; }; // Return the ith variable length offset uint32_t var_fields::read_offset(uint32_t var_index) { uint32_t offset = 0; - m_val_buffer->read(&offset, m_bytes_per_offset, m_var_offset + var_index * m_bytes_per_offset); + m_val_buffer->read( + &offset, m_bytes_per_offset, m_var_offset + var_index * m_bytes_per_offset); return offset; } // Write the ith variable length offset with a new offset. void var_fields::write_offset(uint32_t var_index, uint32_t new_offset) { - m_val_buffer->write(&new_offset, m_bytes_per_offset, m_var_offset + var_index * m_bytes_per_offset); + m_val_buffer->write( + &new_offset, + m_bytes_per_offset, + m_var_offset + var_index * m_bytes_per_offset); } // Return the offset of the ith variable length field uint32_t var_fields::value_offset(uint32_t var_index) { - assert(var_index < m_num_fields); + assert_always(var_index < m_num_fields); if (var_index == 0) return m_val_offset; else @@ -1060,16 +1097,21 @@ uint32_t var_fields::value_offset(uint32_t var_index) { // Return the length of the ith variable length field uint32_t var_fields::value_length(uint32_t var_index) { - assert(var_index < m_num_fields); + assert_always(var_index < m_num_fields); if (var_index == 0) return read_offset(0); else return read_offset(var_index) - read_offset(var_index-1); } -// The length of the ith variable length fields changed. Update all of the subsequent offsets. -void var_fields::update_offsets(uint32_t var_index, uint32_t old_s, uint32_t new_s) { - assert(var_index < m_num_fields); +// The length of the ith variable length fields changed. +// Update all of the subsequent offsets. +void var_fields::update_offsets( + uint32_t var_index, + uint32_t old_s, + uint32_t new_s) { + + assert_always(var_index < m_num_fields); if (old_s == new_s) return; for (uint i = var_index; i < m_num_fields; i++) { @@ -1088,7 +1130,11 @@ uint32_t var_fields::end_offset() { return m_val_offset + read_offset(m_num_fields-1); } -void var_fields::replace(uint32_t var_index, void *new_val_ptr, uint32_t new_val_length) { +void var_fields::replace( + uint32_t var_index, + void* new_val_ptr, + uint32_t new_val_length) { + // replace the new val with the extra val uint32_t the_offset = value_offset(var_index); uint32_t old_s = value_length(var_index); @@ -1103,15 +1149,23 @@ class blob_fields { public: blob_fields() { } - void init_blob_fields(uint32_t num_blobs, const uint8_t *blob_lengths, tokudb::buffer *val_buffer) { - m_num_blobs = num_blobs; m_blob_lengths = blob_lengths; m_val_buffer = val_buffer; + void init_blob_fields( + uint32_t num_blobs, + const uint8_t* blob_lengths, + tokudb::buffer* val_buffer) { + m_num_blobs = num_blobs; + m_blob_lengths = blob_lengths; + m_val_buffer = val_buffer; } void start_blobs(uint32_t offset) { m_blob_offset = offset; } void replace(uint32_t blob_index, uint32_t length, void *p); - void expand_length(uint32_t blob_index, uint8_t old_length_length, uint8_t new_length_length); + void expand_length( + uint32_t blob_index, + uint8_t old_length_length, + uint8_t new_length_length); private: uint32_t read_length(uint32_t offset, size_t size); void write_length(uint32_t offset, size_t size, uint32_t new_length); @@ -1129,12 +1183,15 @@ uint32_t blob_fields::read_length(uint32_t offset, size_t blob_length) { return length; } -void blob_fields::write_length(uint32_t offset, size_t size, uint32_t new_length) { +void blob_fields::write_length( + uint32_t offset, + size_t size, + uint32_t new_length) { m_val_buffer->write(&new_length, size, offset); } uint32_t blob_fields::blob_offset(uint32_t blob_index) { - assert(blob_index < m_num_blobs); + assert_always(blob_index < m_num_blobs); uint32_t offset = m_blob_offset; for (uint i = 0; i < blob_index; i++) { uint32_t blob_length = m_blob_lengths[i]; @@ -1144,8 +1201,12 @@ uint32_t blob_fields::blob_offset(uint32_t blob_index) { return offset; } -void blob_fields::replace(uint32_t blob_index, uint32_t new_length, void *new_value) { - assert(blob_index < m_num_blobs); +void blob_fields::replace( + uint32_t blob_index, + uint32_t new_length, + void* new_value) { + + assert_always(blob_index < m_num_blobs); // compute the ith blob offset uint32_t offset = blob_offset(blob_index); @@ -1155,15 +1216,23 @@ void blob_fields::replace(uint32_t blob_index, uint32_t new_length, void *new_va uint32_t old_length = read_length(offset, blob_length); // replace the data - m_val_buffer->replace(offset + blob_length, old_length, new_value, new_length); + m_val_buffer->replace( + offset + blob_length, + old_length, + new_value, + new_length); // write the new length write_length(offset, blob_length, new_length); } -void blob_fields::expand_length(uint32_t blob_index, uint8_t old_length_length, uint8_t new_length_length) { - assert(blob_index < m_num_blobs); - assert(old_length_length == m_blob_lengths[blob_index]); +void blob_fields::expand_length( + uint32_t blob_index, + uint8_t old_length_length, + uint8_t new_length_length) { + + assert_always(blob_index < m_num_blobs); + assert_always(old_length_length == m_blob_lengths[blob_index]); // compute the ith blob offset uint32_t offset = blob_offset(blob_index); @@ -1172,7 +1241,11 @@ void blob_fields::expand_length(uint32_t blob_index, uint8_t old_length_length, uint32_t blob_length = read_length(offset, old_length_length); // expand the length - m_val_buffer->replace(offset, old_length_length, &blob_length, new_length_length); + m_val_buffer->replace( + offset, + old_length_length, + &blob_length, + new_length_length); } class value_map { @@ -1180,8 +1253,16 @@ public: value_map(tokudb::buffer *val_buffer) : m_val_buffer(val_buffer) { } - void init_var_fields(uint32_t var_offset, uint32_t offset_bytes, uint32_t bytes_per_offset) { - m_var_fields.init_var_fields(var_offset, offset_bytes, bytes_per_offset, m_val_buffer); + void init_var_fields( + uint32_t var_offset, + uint32_t offset_bytes, + uint32_t bytes_per_offset) { + + m_var_fields.init_var_fields( + var_offset, + offset_bytes, + bytes_per_offset, + m_val_buffer); } void init_blob_fields(uint32_t num_blobs, const uint8_t *blob_lengths) { @@ -1189,31 +1270,63 @@ public: } // Replace the value of a fixed length field - void replace_fixed(uint32_t the_offset, uint32_t field_null_num, void *new_val_ptr, uint32_t new_val_length) { - m_val_buffer->replace(the_offset, new_val_length, new_val_ptr, new_val_length); + void replace_fixed( + uint32_t the_offset, + uint32_t field_null_num, + void* new_val_ptr, + uint32_t new_val_length) { + + m_val_buffer->replace( + the_offset, + new_val_length, + new_val_ptr, + new_val_length); maybe_clear_null(field_null_num); } // Replace the value of a variable length field - void replace_varchar(uint32_t var_index, uint32_t field_null_num, void *new_val_ptr, uint32_t new_val_length) { + void replace_varchar( + uint32_t var_index, + uint32_t field_null_num, + void* new_val_ptr, + uint32_t new_val_length) { + m_var_fields.replace(var_index, new_val_ptr, new_val_length); maybe_clear_null(field_null_num); } // Replace the value of a blob field - void replace_blob(uint32_t blob_index, uint32_t field_null_num, void *new_val_ptr, uint32_t new_val_length) { + void replace_blob( + uint32_t blob_index, + uint32_t field_null_num, + void* new_val_ptr, + uint32_t new_val_length) { + m_blob_fields.start_blobs(m_var_fields.end_offset()); m_blob_fields.replace(blob_index, new_val_length, new_val_ptr); maybe_clear_null(field_null_num); } - void expand_blob_lengths(uint32_t num_blob, const uint8_t *old_length, const uint8_t *new_length); - - void int_op(uint32_t operation, uint32_t the_offset, uint32_t length, uint32_t field_null_num, - tokudb::buffer &old_val, void *extra_val); - - void uint_op(uint32_t operation, uint32_t the_offset, uint32_t length, uint32_t field_null_num, - tokudb::buffer &old_val, void *extra_val); + void expand_blob_lengths( + uint32_t num_blob, + const uint8_t* old_length, + const uint8_t* new_length); + + void int_op( + uint32_t operation, + uint32_t the_offset, + uint32_t length, + uint32_t field_null_num, + tokudb::buffer& old_val, + void* extra_val); + + void uint_op( + uint32_t operation, + uint32_t the_offset, + uint32_t length, + uint32_t field_null_num, + tokudb::buffer& old_val, + void* extra_val); private: bool is_null(uint32_t null_num, uchar *null_bytes) { @@ -1234,7 +1347,10 @@ private: null_num &= ~(1<<31); else null_num -= 1; - set_overall_null_position((uchar *) m_val_buffer->data(), null_num, false); + set_overall_null_position( + (uchar*)m_val_buffer->data(), + null_num, + false); } } @@ -1245,11 +1361,19 @@ private: }; // Update an int field: signed newval@offset = old_val@offset OP extra_val -void value_map::int_op(uint32_t operation, uint32_t the_offset, uint32_t length, uint32_t field_null_num, - tokudb::buffer &old_val, void *extra_val) { - assert(the_offset + length <= m_val_buffer->size()); - assert(the_offset + length <= old_val.size()); - assert(length == 1 || length == 2 || length == 3 || length == 4 || length == 8); +void value_map::int_op( + uint32_t operation, + uint32_t the_offset, + uint32_t length, + uint32_t field_null_num, + tokudb::buffer &old_val, + void* extra_val) { + + assert_always(the_offset + length <= m_val_buffer->size()); + assert_always(the_offset + length <= old_val.size()); + assert_always( + length == 1 || length == 2 || length == 3 || + length == 4 || length == 8); uchar *old_val_ptr = (uchar *) old_val.data(); bool field_is_null = is_null(field_null_num, old_val_ptr); @@ -1287,16 +1411,24 @@ void value_map::int_op(uint32_t operation, uint32_t the_offset, uint32_t length, } break; default: - assert(0); + assert_unreachable(); } } // Update an unsigned field: unsigned newval@offset = old_val@offset OP extra_val -void value_map::uint_op(uint32_t operation, uint32_t the_offset, uint32_t length, uint32_t field_null_num, - tokudb::buffer &old_val, void *extra_val) { - assert(the_offset + length <= m_val_buffer->size()); - assert(the_offset + length <= old_val.size()); - assert(length == 1 || length == 2 || length == 3 || length == 4 || length == 8); +void value_map::uint_op( + uint32_t operation, + uint32_t the_offset, + uint32_t length, + uint32_t field_null_num, + tokudb::buffer& old_val, + void* extra_val) { + + assert_always(the_offset + length <= m_val_buffer->size()); + assert_always(the_offset + length <= old_val.size()); + assert_always( + length == 1 || length == 2 || length == 3 || + length == 4 || length == 8); uchar *old_val_ptr = (uchar *) old_val.data(); bool field_is_null = is_null(field_null_num, old_val_ptr); @@ -1326,16 +1458,23 @@ void value_map::uint_op(uint32_t operation, uint32_t the_offset, uint32_t length } break; default: - assert(0); + assert_unreachable(); } } -void value_map::expand_blob_lengths(uint32_t num_blob, const uint8_t *old_length, const uint8_t *new_length) { +void value_map::expand_blob_lengths( + uint32_t num_blob, + const uint8_t* old_length, + const uint8_t* new_length) { + uint8_t current_length[num_blob]; memcpy(current_length, old_length, num_blob); for (uint32_t i = 0; i < num_blob; i++) { if (new_length[i] > current_length[i]) { - m_blob_fields.init_blob_fields(num_blob, current_length, m_val_buffer); + m_blob_fields.init_blob_fields( + num_blob, + current_length, + m_val_buffer); m_blob_fields.start_blobs(m_var_fields.end_offset()); m_blob_fields.expand_length(i, current_length[i], new_length[i]); current_length[i] = new_length[i]; @@ -1348,30 +1487,29 @@ void value_map::expand_blob_lengths(uint32_t num_blob, const uint8_t *old_length static uint32_t consume_uint32(tokudb::buffer &b) { uint32_t n; size_t s = b.consume_ui<uint32_t>(&n); - assert(s > 0); + assert_always(s > 0); return n; } static uint8_t *consume_uint8_array(tokudb::buffer &b, uint32_t array_size) { uint8_t *p = (uint8_t *) b.consume_ptr(array_size); - assert(p); + assert_always(p); return p; } static int tokudb_expand_blobs( DB* db, - const DBT *key_dbt, - const DBT *old_val_dbt, - const DBT *extra, - void (*set_val)(const DBT *new_val_dbt, void *set_extra), - void *set_extra - ) -{ + const DBT* key_dbt, + const DBT* old_val_dbt, + const DBT* extra, + void (*set_val)(const DBT* new_val_dbt, void* set_extra), + void* set_extra) { + tokudb::buffer extra_val(extra->data, 0, extra->size); uint8_t operation; extra_val.consume(&operation, sizeof operation); - assert(operation == UPDATE_OP_EXPAND_BLOB); + assert_always(operation == UPDATE_OP_EXPAND_BLOB); if (old_val_dbt != NULL) { // new val = old val @@ -1384,13 +1522,18 @@ static int tokudb_expand_blobs( uint32_t var_field_offset = consume_uint32(extra_val); uint32_t var_offset_bytes = consume_uint32(extra_val); uint32_t bytes_per_offset = consume_uint32(extra_val); - vd.init_var_fields(var_field_offset, var_offset_bytes, bytes_per_offset); + vd.init_var_fields( + var_field_offset, + var_offset_bytes, + bytes_per_offset); // decode blob info uint32_t num_blob = consume_uint32(extra_val); - const uint8_t *old_blob_length = consume_uint8_array(extra_val, num_blob); - const uint8_t *new_blob_length = consume_uint8_array(extra_val, num_blob); - assert(extra_val.size() == extra_val.limit()); + const uint8_t* old_blob_length = + consume_uint8_array(extra_val, num_blob); + const uint8_t* new_blob_length = + consume_uint8_array(extra_val, num_blob); + assert_always(extra_val.size() == extra_val.limit()); // expand blob lengths vd.expand_blob_lengths(num_blob, old_blob_length, new_blob_length); @@ -1404,8 +1547,14 @@ static int tokudb_expand_blobs( return 0; } -// Decode and apply a sequence of update operations defined in the extra to the old value and put the result in the new value. -static void apply_1_updates(tokudb::value_map &vd, tokudb::buffer &new_val, tokudb::buffer &old_val, tokudb::buffer &extra_val) { +// Decode and apply a sequence of update operations defined in the extra to +// the old value and put the result in the new value. +static void apply_1_updates( + tokudb::value_map& vd, + tokudb::buffer& new_val, + tokudb::buffer& old_val, + tokudb::buffer& extra_val) { + uint32_t num_updates; extra_val.consume(&num_updates, sizeof num_updates); for ( ; num_updates > 0; num_updates--) { @@ -1428,46 +1577,70 @@ static void apply_1_updates(tokudb::value_map &vd, tokudb::buffer &new_val, toku switch (field_type) { case UPDATE_TYPE_INT: if (update_operation == '=') - vd.replace_fixed(the_offset, field_null_num, extra_val_ptr, extra_val_length); + vd.replace_fixed( + the_offset, + field_null_num, + extra_val_ptr, + extra_val_length); else - vd.int_op(update_operation, the_offset, extra_val_length, field_null_num, old_val, extra_val_ptr); + vd.int_op( + update_operation, + the_offset, + extra_val_length, + field_null_num, + old_val, + extra_val_ptr); break; case UPDATE_TYPE_UINT: if (update_operation == '=') - vd.replace_fixed(the_offset, field_null_num, extra_val_ptr, extra_val_length); + vd.replace_fixed( + the_offset, + field_null_num, + extra_val_ptr, + extra_val_length); else - vd.uint_op(update_operation, the_offset, extra_val_length, field_null_num, old_val, extra_val_ptr); + vd.uint_op( + update_operation, + the_offset, + extra_val_length, + field_null_num, + old_val, + extra_val_ptr); break; case UPDATE_TYPE_CHAR: case UPDATE_TYPE_BINARY: if (update_operation == '=') - vd.replace_fixed(the_offset, field_null_num, extra_val_ptr, extra_val_length); + vd.replace_fixed( + the_offset, + field_null_num, + extra_val_ptr, + extra_val_length); else - assert(0); + assert_unreachable(); break; default: - assert(0); + assert_unreachable(); break; } } - assert(extra_val.size() == extra_val.limit()); + assert_always(extra_val.size() == extra_val.limit()); } -// Simple update handler. Decode the update message, apply the update operations to the old value, and set the new value. +// Simple update handler. Decode the update message, apply the update operations +// to the old value, and set the new value. static int tokudb_update_1_fun( DB* db, - const DBT *key_dbt, - const DBT *old_val_dbt, - const DBT *extra, - void (*set_val)(const DBT *new_val_dbt, void *set_extra), - void *set_extra - ) -{ + const DBT* key_dbt, + const DBT* old_val_dbt, + const DBT* extra, + void (*set_val)(const DBT* new_val_dbt, void* set_extra), + void* set_extra) { + tokudb::buffer extra_val(extra->data, 0, extra->size); uint8_t operation; extra_val.consume(&operation, sizeof operation); - assert(operation == UPDATE_OP_UPDATE_1); + assert_always(operation == UPDATE_OP_UPDATE_1); if (old_val_dbt != NULL) { // get the simple descriptor @@ -1480,14 +1653,20 @@ static int tokudb_update_1_fun( uint32_t m_bytes_per_offset; extra_val.consume(&m_bytes_per_offset, sizeof m_bytes_per_offset); - tokudb::buffer old_val(old_val_dbt->data, old_val_dbt->size, old_val_dbt->size); + tokudb::buffer old_val( + old_val_dbt->data, + old_val_dbt->size, + old_val_dbt->size); // new val = old val tokudb::buffer new_val; new_val.append(old_val_dbt->data, old_val_dbt->size); tokudb::value_map vd(&new_val); - vd.init_var_fields(m_var_field_offset, m_var_offset_bytes, m_bytes_per_offset); + vd.init_var_fields( + m_var_field_offset, + m_var_offset_bytes, + m_bytes_per_offset); // apply updates to new val apply_1_updates(vd, new_val, old_val, extra_val); @@ -1502,22 +1681,23 @@ static int tokudb_update_1_fun( return 0; } -// Simple upsert handler. Decode the upsert message. If the key does not exist, then insert a new value from the extra. -// Otherwise, apply the update operations to the old value, and then set the new value. +// Simple upsert handler. Decode the upsert message. If the key does not exist, +// then insert a new value from the extra. +// Otherwise, apply the update operations to the old value, and then set the +// new value. static int tokudb_upsert_1_fun( DB* db, - const DBT *key_dbt, - const DBT *old_val_dbt, - const DBT *extra, - void (*set_val)(const DBT *new_val_dbt, void *set_extra), - void *set_extra - ) -{ + const DBT* key_dbt, + const DBT* old_val_dbt, + const DBT* extra, + void (*set_val)(const DBT* new_val_dbt, void* set_extra), + void* set_extra) { + tokudb::buffer extra_val(extra->data, 0, extra->size); uint8_t operation; extra_val.consume(&operation, sizeof operation); - assert(operation == UPDATE_OP_UPSERT_1); + assert_always(operation == UPDATE_OP_UPSERT_1); uint32_t insert_length; extra_val.consume(&insert_length, sizeof insert_length); @@ -1540,14 +1720,20 @@ static int tokudb_upsert_1_fun( uint32_t m_bytes_per_offset; extra_val.consume(&m_bytes_per_offset, sizeof m_bytes_per_offset); - tokudb::buffer old_val(old_val_dbt->data, old_val_dbt->size, old_val_dbt->size); + tokudb::buffer old_val( + old_val_dbt->data, + old_val_dbt->size, + old_val_dbt->size); // new val = old val tokudb::buffer new_val; new_val.append(old_val_dbt->data, old_val_dbt->size); tokudb::value_map vd(&new_val); - vd.init_var_fields(m_var_field_offset, m_var_offset_bytes, m_bytes_per_offset); + vd.init_var_fields( + m_var_field_offset, + m_var_offset_bytes, + m_bytes_per_offset); // apply updates to new val apply_1_updates(vd, new_val, old_val, extra_val); @@ -1562,8 +1748,14 @@ static int tokudb_upsert_1_fun( return 0; } -// Decode and apply a sequence of update operations defined in the extra to the old value and put the result in the new value. -static void apply_2_updates(tokudb::value_map &vd, tokudb::buffer &new_val, tokudb::buffer &old_val, tokudb::buffer &extra_val) { +// Decode and apply a sequence of update operations defined in the extra to the +// old value and put the result in the new value. +static void apply_2_updates( + tokudb::value_map& vd, + tokudb::buffer& new_val, + tokudb::buffer& old_val, + tokudb::buffer& extra_val) { + uint32_t num_updates = consume_uint32(extra_val); for (uint32_t i = 0; i < num_updates; i++) { uint32_t update_operation = consume_uint32(extra_val); @@ -1571,79 +1763,118 @@ static void apply_2_updates(tokudb::value_map &vd, tokudb::buffer &new_val, toku uint32_t var_field_offset = consume_uint32(extra_val); uint32_t var_offset_bytes = consume_uint32(extra_val); uint32_t bytes_per_offset = consume_uint32(extra_val); - vd.init_var_fields(var_field_offset, var_offset_bytes, bytes_per_offset); + vd.init_var_fields( + var_field_offset, + var_offset_bytes, + bytes_per_offset); } else if (update_operation == 'b') { uint32_t num_blobs = consume_uint32(extra_val); - const uint8_t *blob_lengths = consume_uint8_array(extra_val, num_blobs); + const uint8_t* blob_lengths = + consume_uint8_array(extra_val, num_blobs); vd.init_blob_fields(num_blobs, blob_lengths); } else { uint32_t field_type = consume_uint32(extra_val); uint32_t field_null_num = consume_uint32(extra_val); uint32_t the_offset = consume_uint32(extra_val); uint32_t extra_val_length = consume_uint32(extra_val); - void *extra_val_ptr = extra_val.consume_ptr(extra_val_length); assert(extra_val_ptr); + void* extra_val_ptr = extra_val.consume_ptr(extra_val_length); + assert_always(extra_val_ptr); switch (field_type) { case UPDATE_TYPE_INT: if (update_operation == '=') - vd.replace_fixed(the_offset, field_null_num, extra_val_ptr, extra_val_length); + vd.replace_fixed( + the_offset, + field_null_num, + extra_val_ptr, + extra_val_length); else - vd.int_op(update_operation, the_offset, extra_val_length, field_null_num, old_val, extra_val_ptr); + vd.int_op( + update_operation, + the_offset, + extra_val_length, + field_null_num, + old_val, + extra_val_ptr); break; case UPDATE_TYPE_UINT: if (update_operation == '=') - vd.replace_fixed(the_offset, field_null_num, extra_val_ptr, extra_val_length); + vd.replace_fixed( + the_offset, + field_null_num, + extra_val_ptr, + extra_val_length); else - vd.uint_op(update_operation, the_offset, extra_val_length, field_null_num, old_val, extra_val_ptr); + vd.uint_op( + update_operation, + the_offset, + extra_val_length, + field_null_num, + old_val, + extra_val_ptr); break; case UPDATE_TYPE_CHAR: case UPDATE_TYPE_BINARY: if (update_operation == '=') - vd.replace_fixed(the_offset, field_null_num, extra_val_ptr, extra_val_length); + vd.replace_fixed( + the_offset, + field_null_num, + extra_val_ptr, + extra_val_length); else - assert(0); + assert_unreachable(); break; case UPDATE_TYPE_VARBINARY: case UPDATE_TYPE_VARCHAR: if (update_operation == '=') - vd.replace_varchar(the_offset, field_null_num, extra_val_ptr, extra_val_length); + vd.replace_varchar( + the_offset, + field_null_num, + extra_val_ptr, + extra_val_length); else - assert(0); + assert_unreachable(); break; case UPDATE_TYPE_TEXT: case UPDATE_TYPE_BLOB: if (update_operation == '=') - vd.replace_blob(the_offset, field_null_num, extra_val_ptr, extra_val_length); + vd.replace_blob( + the_offset, + field_null_num, + extra_val_ptr, + extra_val_length); else - assert(0); + assert_unreachable(); break; default: - assert(0); - break; + assert_unreachable(); } } } - assert(extra_val.size() == extra_val.limit()); + assert_always(extra_val.size() == extra_val.limit()); } -// Simple update handler. Decode the update message, apply the update operations to the old value, and set the new value. +// Simple update handler. Decode the update message, apply the update +// operations to the old value, and set the new value. static int tokudb_update_2_fun( DB* db, - const DBT *key_dbt, - const DBT *old_val_dbt, - const DBT *extra, - void (*set_val)(const DBT *new_val_dbt, void *set_extra), - void *set_extra - ) -{ + const DBT* key_dbt, + const DBT* old_val_dbt, + const DBT* extra, + void (*set_val)(const DBT* new_val_dbt, void* set_extra), + void* set_extra) { + tokudb::buffer extra_val(extra->data, 0, extra->size); uint8_t op; extra_val.consume(&op, sizeof op); - assert(op == UPDATE_OP_UPDATE_2); + assert_always(op == UPDATE_OP_UPDATE_2); if (old_val_dbt != NULL) { - tokudb::buffer old_val(old_val_dbt->data, old_val_dbt->size, old_val_dbt->size); + tokudb::buffer old_val( + old_val_dbt->data, + old_val_dbt->size, + old_val_dbt->size); // new val = old val tokudb::buffer new_val; @@ -1664,26 +1895,28 @@ static int tokudb_update_2_fun( return 0; } -// Simple upsert handler. Decode the upsert message. If the key does not exist, then insert a new value from the extra. -// Otherwise, apply the update operations to the old value, and then set the new value. +// Simple upsert handler. Decode the upsert message. If the key does not exist, +// then insert a new value from the extra. +// Otherwise, apply the update operations to the old value, and then set the +// new value. static int tokudb_upsert_2_fun( DB* db, - const DBT *key_dbt, - const DBT *old_val_dbt, - const DBT *extra, - void (*set_val)(const DBT *new_val_dbt, void *set_extra), - void *set_extra - ) -{ + const DBT* key_dbt, + const DBT* old_val_dbt, + const DBT* extra, + void (*set_val)(const DBT* new_val_dbt, void* set_extra), + void* set_extra) { + tokudb::buffer extra_val(extra->data, 0, extra->size); uint8_t op; extra_val.consume(&op, sizeof op); - assert(op == UPDATE_OP_UPSERT_2); + assert_always(op == UPDATE_OP_UPSERT_2); uint32_t insert_length = consume_uint32(extra_val); - assert(insert_length < extra_val.limit()); - void *insert_row = extra_val.consume_ptr(insert_length); assert(insert_row); + assert_always(insert_length < extra_val.limit()); + void* insert_row = extra_val.consume_ptr(insert_length); + assert_always(insert_row); if (old_val_dbt == NULL) { // insert a new row @@ -1692,7 +1925,10 @@ static int tokudb_upsert_2_fun( new_val_dbt.data = insert_row; set_val(&new_val_dbt, set_extra); } else { - tokudb::buffer old_val(old_val_dbt->data, old_val_dbt->size, old_val_dbt->size); + tokudb::buffer old_val( + old_val_dbt->data, + old_val_dbt->size, + old_val_dbt->size); // new val = old val tokudb::buffer new_val; @@ -1713,56 +1949,107 @@ static int tokudb_upsert_2_fun( return 0; } -// This function is the update callback function that is registered with the YDB environment. -// It uses the first byte in the update message to identify the update message type and call -// the handler for that message. +// This function is the update callback function that is registered with the +// YDB environment. It uses the first byte in the update message to identify +// the update message type and call the handler for that message. int tokudb_update_fun( DB* db, - const DBT *key, - const DBT *old_val, - const DBT *extra, - void (*set_val)(const DBT *new_val, void *set_extra), - void *set_extra - ) -{ - assert(extra->size > 0); - uint8_t *extra_pos = (uchar *)extra->data; + const DBT* key, + const DBT* old_val, + const DBT* extra, + void (*set_val)(const DBT* new_val, void* set_extra), + void* set_extra) { + + assert_always(extra->size > 0); + uint8_t* extra_pos = (uchar*)extra->data; uint8_t operation = extra_pos[0]; int error; switch (operation) { case UPDATE_OP_COL_ADD_OR_DROP: - error = tokudb_hcad_update_fun(db, key, old_val, extra, set_val, set_extra); + error = tokudb_hcad_update_fun( + db, + key, + old_val, + extra, + set_val, + set_extra); break; case UPDATE_OP_EXPAND_VARIABLE_OFFSETS: - error = tokudb_expand_variable_offsets(db, key, old_val, extra, set_val, set_extra); + error = tokudb_expand_variable_offsets( + db, + key, + old_val, + extra, + set_val, + set_extra); break; case UPDATE_OP_EXPAND_INT: case UPDATE_OP_EXPAND_UINT: - error = tokudb_expand_int_field(db, key, old_val, extra, set_val, set_extra); + error = tokudb_expand_int_field( + db, + key, + old_val, + extra, + set_val, + set_extra); break; case UPDATE_OP_EXPAND_CHAR: case UPDATE_OP_EXPAND_BINARY: - error = tokudb_expand_char_field(db, key, old_val, extra, set_val, set_extra); + error = tokudb_expand_char_field( + db, + key, + old_val, + extra, + set_val, + set_extra); break; case UPDATE_OP_EXPAND_BLOB: - error = tokudb_expand_blobs(db, key, old_val, extra, set_val, set_extra); + error = tokudb_expand_blobs( + db, + key, + old_val, + extra, + set_val, + set_extra); break; case UPDATE_OP_UPDATE_1: - error = tokudb_update_1_fun(db, key, old_val, extra, set_val, set_extra); + error = tokudb_update_1_fun( + db, + key, + old_val, + extra, + set_val, + set_extra); break; case UPDATE_OP_UPSERT_1: - error = tokudb_upsert_1_fun(db, key, old_val, extra, set_val, set_extra); + error = tokudb_upsert_1_fun( + db, + key, + old_val, + extra, + set_val, + set_extra); break; case UPDATE_OP_UPDATE_2: - error = tokudb_update_2_fun(db, key, old_val, extra, set_val, set_extra); + error = tokudb_update_2_fun( + db, + key, + old_val, + extra, + set_val, + set_extra); break; case UPDATE_OP_UPSERT_2: - error = tokudb_upsert_2_fun(db, key, old_val, extra, set_val, set_extra); + error = tokudb_upsert_2_fun( + db, + key, + old_val, + extra, + set_val, + set_extra); break; default: - assert(0); - error = EINVAL; - break; + assert_unreachable(); } return error; } |