diff options
48 files changed, 2697 insertions, 476 deletions
diff --git a/client/mysqltest.c b/client/mysqltest.c index 4991e565594..9240e10ce03 100644 --- a/client/mysqltest.c +++ b/client/mysqltest.c @@ -593,7 +593,7 @@ static void abort_not_supported_test() printf("skipped\n"); free_used_memory(); my_end(MY_CHECK_ERROR); - exit(2); + exit(62); } static void verbose_msg(const char* fmt, ...) @@ -2729,6 +2729,8 @@ static int run_query_normal(MYSQL* mysql, struct st_query* q, int flags) if (!disable_result_log) { + ulong affected_rows; /* Ok to be undef if 'disable_info' is set */ + if (res) { MYSQL_FIELD *field= mysql_fetch_fields(res); @@ -2751,6 +2753,13 @@ static int run_query_normal(MYSQL* mysql, struct st_query* q, int flags) } /* + Need to call mysql_affected_rows() before the new + query to find the warnings + */ + if (!disable_info) + affected_rows= (ulong)mysql_affected_rows(mysql); + + /* Add all warnings to the result. We can't do this if we are in the middle of processing results from multi-statement, because this will break protocol. @@ -2777,7 +2786,7 @@ static int run_query_normal(MYSQL* mysql, struct st_query* q, int flags) if (!disable_info) { char buf[40]; - sprintf(buf,"affected rows: %lu\n",(ulong) mysql_affected_rows(mysql)); + sprintf(buf,"affected rows: %lu\n", affected_rows); dynstr_append(ds, buf); if (mysql_info(mysql)) { diff --git a/configure.in b/configure.in index cd4f011e1ce..3e116d608f7 100644 --- a/configure.in +++ b/configure.in @@ -1374,20 +1374,20 @@ then AC_MSG_CHECKING("for gcc") if expr "$CC" : ".*gcc.*" then - CC="$CC -pthread -DUNIXWARE_7 -DHAVE_BROKEN_RWLOCK"; - CXX="$CXX -pthread -DUNIXWARE_7 -DHAVE_BROKEN_RWLOCK"; + CC="$CC -pthread -DUNIXWARE_7 -DHAVE_BROKEN_RWLOCK" + CXX="$CXX -pthread -DUNIXWARE_7 -DHAVE_BROKEN_RWLOCK" else - CC="$CC -Kthread -DUNIXWARE_7 -DHAVE_BROKEN_RWLOCK"; - CXX="$CXX -Kthread -DUNIXWARE_7 -DHAVE_BROKEN_RWLOCK"; + CC="$CC -Kthread -DUNIXWARE_7 -DHAVE_BROKEN_RWLOCK" + CXX="$CXX -Kthread -DUNIXWARE_7 -DHAVE_BROKEN_RWLOCK" fi else - { AC_MSG_ERROR([configure: error: Can't find thread libs on SCO UnixWare7. See the Installation chapter in the Reference Manual.]) }; + AC_MSG_ERROR([configure: error: Can't find thread libs on SCO UnixWare7. See the Installation chapter in the Reference Manual.]) fi else AC_MSG_RESULT("no") fi else - AC_MSG_ERROR([On SCO UNIX MySQL requires that the FSUThreads package is installed. See the Installation chapter in the Reference Manual.]); + AC_MSG_ERROR([On SCO UNIX MySQL requires that the FSUThreads package is installed. See the Installation chapter in the Reference Manual.]) fi else AC_MSG_RESULT("no") @@ -1419,15 +1419,15 @@ then AC_MSG_CHECKING("for gcc") if expr "$CC" : ".*gcc.*" then - CC="$CC -pthread -DUNIXWARE_7 -DHAVE_BROKEN_RWLOCK"; - CXX="$CXX -pthread -DUNIXWARE_7 -DHAVE_BROKEN_RWLOCK"; + CC="$CC -pthread -DUNIXWARE_7 -DHAVE_BROKEN_RWLOCK" + CXX="$CXX -pthread -DUNIXWARE_7 -DHAVE_BROKEN_RWLOCK" else - CC="$CC -Kthread -DUNIXWARE_7 -DHAVE_BROKEN_RWLOCK"; - CXX="$CXX -Kthread -DUNIXWARE_7 -DHAVE_BROKEN_RWLOCK"; + CC="$CC -Kthread -DUNIXWARE_7 -DHAVE_BROKEN_RWLOCK" + CXX="$CXX -Kthread -DUNIXWARE_7 -DHAVE_BROKEN_RWLOCK" fi AC_MSG_RESULT("yes") else - { AC_MSG_ERROR([configure: error: Can't find thread libs on SCO UnixWare7. See the Installation chapter in the Reference Manual.]) }; + AC_MSG_ERROR([configure: error: Can't find thread libs on SCO UnixWare7. See the Installation chapter in the Reference Manual.]) fi else AC_MSG_RESULT("no") @@ -1462,15 +1462,15 @@ then AC_MSG_CHECKING("for gcc") if expr "$CC" : ".*gcc.*" then - CC="$CC -pthread -DUNIXWARE_7 -DHAVE_BROKEN_RWLOCK"; - CXX="$CXX -pthread -DUNIXWARE_7 -DHAVE_BROKEN_RWLOCK"; + CC="$CC -pthread -DUNIXWARE_7 -DHAVE_BROKEN_RWLOCK" + CXX="$CXX -pthread -DUNIXWARE_7 -DHAVE_BROKEN_RWLOCK" else - CC="$CC -Kthread -DUNIXWARE_7 -DHAVE_BROKEN_RWLOCK"; - CXX="$CXX -Kthread -DUNIXWARE_7 -DHAVE_BROKEN_RWLOCK"; + CC="$CC -Kthread -DUNIXWARE_7 -DHAVE_BROKEN_RWLOCK" + CXX="$CXX -Kthread -DUNIXWARE_7 -DHAVE_BROKEN_RWLOCK" fi AC_MSG_RESULT("yes") else - { AC_MSG_ERROR([configure: error: Can't find thread libs on Caldera OpenUNIX 8. See the Installation chapter in the Reference Manual.]) }; + AC_MSG_ERROR([configure: error: Can't find thread libs on Caldera OpenUNIX 8. See the Installation chapter in the Reference Manual.]) fi else AC_MSG_RESULT("no") diff --git a/mysql-test/lib/init_db.sql b/mysql-test/lib/init_db.sql index 97fa6fb955c..18699497b64 100644 --- a/mysql-test/lib/init_db.sql +++ b/mysql-test/lib/init_db.sql @@ -1,4 +1,5 @@ -USE mysql; +use mysql; +set table_type=myisam; CREATE TABLE db ( Host char(60) binary DEFAULT '' NOT NULL, @@ -206,7 +207,7 @@ INSERT INTO time_zone_name (Name, Time_Zone_id) VALUES CREATE TABLE time_zone ( Time_zone_id int unsigned NOT NULL auto_increment, - Use_leap_seconds enum('Y','N') DEFAULT 'N' NOT NULL, + Use_leap_seconds enum('Y','N') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL, PRIMARY KEY TzId (Time_zone_id) ) engine=MyISAM CHARACTER SET utf8 diff --git a/mysql-test/lib/mtr_process.pl b/mysql-test/lib/mtr_process.pl index 8aefc235d72..78758e54aa4 100644 --- a/mysql-test/lib/mtr_process.pl +++ b/mysql-test/lib/mtr_process.pl @@ -344,7 +344,7 @@ sub mtr_kill_leftovers () { # We scan the "var/run/" directory for other process id's to kill # FIXME $path_run_dir or something - my $rundir= "$::glob_mysql_test_dir/var/run"; + my $rundir= "$::opt_vardir/run"; if ( -d $rundir ) { diff --git a/mysql-test/lib/mtr_report.pl b/mysql-test/lib/mtr_report.pl index fd665b154b8..cb41549422f 100644 --- a/mysql-test/lib/mtr_report.pl +++ b/mysql-test/lib/mtr_report.pl @@ -89,9 +89,10 @@ sub mtr_report_test_passed ($) { my $tinfo= shift; my $timer= ""; - if ( $::opt_timer and -f "$::glob_mysql_test_dir/var/log/timer" ) + if ( $::opt_timer and -f "$::opt_vardir/log/timer" ) { - $timer= mtr_fromfile("$::glob_mysql_test_dir/var/log/timer"); + $timer= mtr_fromfile("$::opt_vardir/log/timer"); + $::glob_tot_real_time += $timer; $timer= sprintf "%12s", $timer; } $tinfo->{'result'}= 'MTR_RES_PASSED'; @@ -177,8 +178,8 @@ sub mtr_report_stats ($) { # Report if there was any fatal warnings/errors in the log files # - unlink("$::glob_mysql_test_dir/var/log/warnings"); - unlink("$::glob_mysql_test_dir/var/log/warnings.tmp"); + unlink("$::opt_vardir/log/warnings"); + unlink("$::opt_vardir/log/warnings.tmp"); # Remove some non fatal warnings from the log files # FIXME what is going on ????? ;-) @@ -196,11 +197,11 @@ sub mtr_report_stats ($) { # found_error=1 # } # done -# unlink("$::glob_mysql_test_dir/var/log/warnings.tmp"); +# unlink("$::opt_vardir/log/warnings.tmp"); # if ( $found_error= "1" ) # { # print "WARNING: Got errors/warnings while running tests. Please examine\n" -# print "$::glob_mysql_test_dir/var/log/warnings for details.\n" +# print "$::opt_vardir/log/warnings for details.\n" # } # } } diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index ae4136b5494..52ffff088b5 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -84,7 +84,7 @@ use Sys::Hostname; #use Carp; use IO::Socket; use IO::Socket::INET; -#use Data::Dumper; +use Data::Dumper; use strict; #use diagnostics; @@ -177,6 +177,7 @@ our $path_timefile; our $path_manager_log; # Used by mysqldadmin our $path_slave_load_tmpdir; # What is this?! our $path_my_basedir; +our $opt_vardir; # A path but set directly on cmd line our $opt_tmpdir; # A path but set directly on cmd line our $opt_usage; @@ -447,8 +448,6 @@ sub initial_setup () { $glob_basedir= dirname($glob_mysql_test_dir); $glob_mysql_bench_dir= "$glob_basedir/mysql-bench"; # FIXME make configurable - $path_timefile= "$glob_mysql_test_dir/var/log/mysqltest-time"; - # needs to be same length to test logging (FIXME what???) $path_slave_load_tmpdir= "../../var/tmp"; @@ -469,11 +468,6 @@ sub command_line_setup () { # These are defaults for things that are set on the command line $opt_suite= "main"; # Special default suite - $opt_tmpdir= "$glob_mysql_test_dir/var/tmp"; - # FIXME maybe unneded? - $path_manager_log= "$glob_mysql_test_dir/var/log/manager.log"; - $opt_current_test= "$glob_mysql_test_dir/var/log/current_test"; - my $opt_master_myport= 9306; my $opt_slave_myport= 9308; $opt_ndbcluster_port= 9350; @@ -549,6 +543,7 @@ sub command_line_setup () { 'unified-diff|udiff' => \$opt_udiff, 'user-test=s' => \$opt_user_test, 'user=s' => \$opt_user, + 'vardir=s' => \$opt_vardir, 'verbose' => \$opt_verbose, 'wait-timeout=i' => \$opt_wait_timeout, 'warnings|log-warnings' => \$opt_warnings, @@ -564,51 +559,35 @@ sub command_line_setup () { @opt_cases= @ARGV; - # Put this into a hash, will be a C struct - - $master->[0]->{'path_myddir'}= "$glob_mysql_test_dir/var/master-data"; - $master->[0]->{'path_myerr'}= "$glob_mysql_test_dir/var/log/master.err"; - $master->[0]->{'path_mylog'}= "$glob_mysql_test_dir/var/log/master.log"; - $master->[0]->{'path_mypid'}= "$glob_mysql_test_dir/var/run/master.pid"; - $master->[0]->{'path_mysock'}= "$opt_tmpdir/master.sock"; - $master->[0]->{'path_myport'}= $opt_master_myport; - $master->[0]->{'start_timeout'}= 400; # enough time create innodb tables - - $master->[0]->{'ndbcluster'}= 1; # ndbcluster not started + # -------------------------------------------------------------------------- + # Set the "var/" directory, as it is the base for everything else + # -------------------------------------------------------------------------- - $master->[1]->{'path_myddir'}= "$glob_mysql_test_dir/var/master1-data"; - $master->[1]->{'path_myerr'}= "$glob_mysql_test_dir/var/log/master1.err"; - $master->[1]->{'path_mylog'}= "$glob_mysql_test_dir/var/log/master1.log"; - $master->[1]->{'path_mypid'}= "$glob_mysql_test_dir/var/run/master1.pid"; - $master->[1]->{'path_mysock'}= "$opt_tmpdir/master1.sock"; - $master->[1]->{'path_myport'}= $opt_master_myport + 1; - $master->[1]->{'start_timeout'}= 400; # enough time create innodb tables + if ( ! $opt_vardir ) + { + $opt_vardir= "$glob_mysql_test_dir/var"; + } - $slave->[0]->{'path_myddir'}= "$glob_mysql_test_dir/var/slave-data"; - $slave->[0]->{'path_myerr'}= "$glob_mysql_test_dir/var/log/slave.err"; - $slave->[0]->{'path_mylog'}= "$glob_mysql_test_dir/var/log/slave.log"; - $slave->[0]->{'path_mypid'}= "$glob_mysql_test_dir/var/run/slave.pid"; - $slave->[0]->{'path_mysock'}= "$opt_tmpdir/slave.sock"; - $slave->[0]->{'path_myport'}= $opt_slave_myport; - $slave->[0]->{'start_timeout'}= 400; + if ( $opt_vardir !~ m,^/, ) + { + # Make absolute path, relative test dir + $opt_vardir= "$glob_mysql_test_dir/$opt_vardir"; + } - $slave->[1]->{'path_myddir'}= "$glob_mysql_test_dir/var/slave1-data"; - $slave->[1]->{'path_myerr'}= "$glob_mysql_test_dir/var/log/slave1.err"; - $slave->[1]->{'path_mylog'}= "$glob_mysql_test_dir/var/log/slave1.log"; - $slave->[1]->{'path_mypid'}= "$glob_mysql_test_dir/var/run/slave1.pid"; - $slave->[1]->{'path_mysock'}= "$opt_tmpdir/slave1.sock"; - $slave->[1]->{'path_myport'}= $opt_slave_myport + 1; - $slave->[1]->{'start_timeout'}= 300; + # -------------------------------------------------------------------------- + # If not set, set these to defaults + # -------------------------------------------------------------------------- - $slave->[2]->{'path_myddir'}= "$glob_mysql_test_dir/var/slave2-data"; - $slave->[2]->{'path_myerr'}= "$glob_mysql_test_dir/var/log/slave2.err"; - $slave->[2]->{'path_mylog'}= "$glob_mysql_test_dir/var/log/slave2.log"; - $slave->[2]->{'path_mypid'}= "$glob_mysql_test_dir/var/run/slave2.pid"; - $slave->[2]->{'path_mysock'}= "$opt_tmpdir/slave2.sock"; - $slave->[2]->{'path_myport'}= $opt_slave_myport + 2; - $slave->[2]->{'start_timeout'}= 300; + $opt_tmpdir= "$opt_vardir/tmp" unless $opt_tmpdir; + # FIXME maybe not needed? + $path_manager_log= "$opt_vardir/log/manager.log" + unless $path_manager_log; + $opt_current_test= "$opt_vardir/log/current_test" + unless $opt_current_test; + # -------------------------------------------------------------------------- # Do sanity checks of command line arguments + # -------------------------------------------------------------------------- if ( $opt_extern and $opt_local ) { @@ -621,13 +600,6 @@ sub command_line_setup () { $opt_socket= "/tmp/mysql.sock"; # FIXME } - if ( $opt_extern ) - { - $glob_use_running_server= 1; - $opt_skip_rpl= 1; # We don't run rpl test cases - $master->[0]->{'path_mysock'}= $opt_socket; - } - # -------------------------------------------------------------------------- # Look at the command line options and set script flags # -------------------------------------------------------------------------- @@ -746,6 +718,58 @@ sub command_line_setup () { } } + # Put this into a hash, will be a C struct + + $master->[0]->{'path_myddir'}= "$opt_vardir/master-data"; + $master->[0]->{'path_myerr'}= "$opt_vardir/log/master.err"; + $master->[0]->{'path_mylog'}= "$opt_vardir/log/master.log"; + $master->[0]->{'path_mypid'}= "$opt_vardir/run/master.pid"; + $master->[0]->{'path_mysock'}= "$opt_tmpdir/master.sock"; + $master->[0]->{'path_myport'}= $opt_master_myport; + $master->[0]->{'start_timeout'}= 400; # enough time create innodb tables + + $master->[0]->{'ndbcluster'}= 1; # ndbcluster not started + + $master->[1]->{'path_myddir'}= "$opt_vardir/master1-data"; + $master->[1]->{'path_myerr'}= "$opt_vardir/log/master1.err"; + $master->[1]->{'path_mylog'}= "$opt_vardir/log/master1.log"; + $master->[1]->{'path_mypid'}= "$opt_vardir/run/master1.pid"; + $master->[1]->{'path_mysock'}= "$opt_tmpdir/master1.sock"; + $master->[1]->{'path_myport'}= $opt_master_myport + 1; + $master->[1]->{'start_timeout'}= 400; # enough time create innodb tables + + $slave->[0]->{'path_myddir'}= "$opt_vardir/slave-data"; + $slave->[0]->{'path_myerr'}= "$opt_vardir/log/slave.err"; + $slave->[0]->{'path_mylog'}= "$opt_vardir/log/slave.log"; + $slave->[0]->{'path_mypid'}= "$opt_vardir/run/slave.pid"; + $slave->[0]->{'path_mysock'}= "$opt_tmpdir/slave.sock"; + $slave->[0]->{'path_myport'}= $opt_slave_myport; + $slave->[0]->{'start_timeout'}= 400; + + $slave->[1]->{'path_myddir'}= "$opt_vardir/slave1-data"; + $slave->[1]->{'path_myerr'}= "$opt_vardir/log/slave1.err"; + $slave->[1]->{'path_mylog'}= "$opt_vardir/log/slave1.log"; + $slave->[1]->{'path_mypid'}= "$opt_vardir/run/slave1.pid"; + $slave->[1]->{'path_mysock'}= "$opt_tmpdir/slave1.sock"; + $slave->[1]->{'path_myport'}= $opt_slave_myport + 1; + $slave->[1]->{'start_timeout'}= 300; + + $slave->[2]->{'path_myddir'}= "$opt_vardir/slave2-data"; + $slave->[2]->{'path_myerr'}= "$opt_vardir/log/slave2.err"; + $slave->[2]->{'path_mylog'}= "$opt_vardir/log/slave2.log"; + $slave->[2]->{'path_mypid'}= "$opt_vardir/run/slave2.pid"; + $slave->[2]->{'path_mysock'}= "$opt_tmpdir/slave2.sock"; + $slave->[2]->{'path_myport'}= $opt_slave_myport + 2; + $slave->[2]->{'start_timeout'}= 300; + + if ( $opt_extern ) + { + $glob_use_running_server= 1; + $opt_skip_rpl= 1; # We don't run rpl test cases + $master->[0]->{'path_mysock'}= $opt_socket; + } + + $path_timefile= "$opt_vardir/log/mysqltest-time"; } @@ -892,8 +916,8 @@ sub executable_setup () { } $path_ndb_backup_dir= - "$glob_mysql_test_dir/var/ndbcluster-$opt_ndbcluster_port"; - $file_ndb_testrun_log= "$glob_mysql_test_dir/var/log/ndb_testrun.log"; + "$opt_vardir/ndbcluster-$opt_ndbcluster_port"; + $file_ndb_testrun_log= "$opt_vardir/log/ndb_testrun.log"; } @@ -981,7 +1005,7 @@ sub kill_running_server () { # leftovers from previous runs. mtr_report("Killing Possible Leftover Processes"); - mkpath("$glob_mysql_test_dir/var/log"); # Needed for mysqladmin log + mkpath("$opt_vardir/log"); # Needed for mysqladmin log mtr_kill_leftovers(); ndbcluster_stop(); @@ -995,15 +1019,20 @@ sub kill_and_cleanup () { mtr_report("Removing Stale Files"); - rmtree("$glob_mysql_test_dir/var/log"); - rmtree("$glob_mysql_test_dir/var/ndbcluster-$opt_ndbcluster_port"); - rmtree("$glob_mysql_test_dir/var/run"); - rmtree("$glob_mysql_test_dir/var/tmp"); + if ( -l $opt_vardir and ! unlink($opt_vardir) ) + { + mtr_error("Can't remove soft link \"$opt_vardir\""); + } + + rmtree("$opt_vardir/log"); + rmtree("$opt_vardir/ndbcluster-$opt_ndbcluster_port"); + rmtree("$opt_vardir/run"); + rmtree("$opt_vardir/tmp"); - mkpath("$glob_mysql_test_dir/var/log"); - mkpath("$glob_mysql_test_dir/var/run"); - mkpath("$glob_mysql_test_dir/var/tmp"); - mkpath($opt_tmpdir); + mkpath("$opt_vardir/log"); + mkpath("$opt_vardir/run"); + mkpath("$opt_vardir/tmp"); + mkpath($opt_tmpdir) if $opt_tmpdir ne "$opt_vardir/tmp"; # FIXME do we really need to create these all, or are they # created for us when tables are created? @@ -1027,6 +1056,16 @@ sub kill_and_cleanup () { rmtree("$slave->[2]->{'path_myddir'}"); mkpath("$slave->[2]->{'path_myddir'}/mysql"); mkpath("$slave->[2]->{'path_myddir'}/test"); + + # To make some old test cases work, we create a soft + # link from the old "var" location to the new one + + if ( ! $glob_win32 and $opt_vardir ne "$glob_mysql_test_dir/var" ) + { + # FIXME why bother with the above, why not always remove all of var?! + rmtree("$glob_mysql_test_dir/var"); # Clean old var, FIXME or rename it?! + symlink($opt_vardir, "$glob_mysql_test_dir/var"); + } } @@ -1048,7 +1087,7 @@ sub ndbcluster_install () { my $ndbcluster_opts= $opt_bench ? "" : "--small"; if ( mtr_run("$glob_mysql_test_dir/ndb/ndbcluster", ["--port=$opt_ndbcluster_port", - "--data-dir=$glob_mysql_test_dir/var", + "--data-dir=$opt_vardir", $ndbcluster_opts, "--initial"], "", "", "", "") ) @@ -1072,7 +1111,7 @@ sub ndbcluster_start () { # FIXME, we want to _append_ output to file $file_ndb_testrun_log instead of /dev/null if ( mtr_run("$glob_mysql_test_dir/ndb/ndbcluster", ["--port=$opt_ndbcluster_port", - "--data-dir=$glob_mysql_test_dir/var"], + "--data-dir=$opt_vardir"], "", "/dev/null", "", "") ) { mtr_error("Error ndbcluster_start"); @@ -1091,7 +1130,7 @@ sub ndbcluster_stop () { # FIXME, we want to _append_ output to file $file_ndb_testrun_log instead of /dev/null mtr_run("$glob_mysql_test_dir/ndb/ndbcluster", ["--port=$opt_ndbcluster_port", - "--data-dir=$glob_mysql_test_dir/var", + "--data-dir=$opt_vardir", "--stop"], "", "/dev/null", "", ""); @@ -1494,15 +1533,15 @@ sub run_testcase ($) { { mtr_report_test_passed($tinfo); } - elsif ( $res == 2 ) + elsif ( $res == 62 ) { # Testcase itself tell us to skip this one mtr_report_test_skipped($tinfo); } else { - # Test case failed - if ( $res > 2 ) + # Test case failed, if in control mysqltest returns 1 + if ( $res != 1 ) { mtr_tofile($path_timefile, "mysqltest returned unexpected code $res, " . @@ -1563,17 +1602,17 @@ sub do_before_start_master ($$) { $tname ne "rpl_crash_binlog_ib_3b") { # FIXME we really want separate dir for binlogs - foreach my $bin ( glob("$glob_mysql_test_dir/var/log/master*-bin.*") ) + foreach my $bin ( glob("$opt_vardir/log/master*-bin.*") ) { unlink($bin); } } # Remove old master.info and relay-log.info files - unlink("$glob_mysql_test_dir/var/master-data/master.info"); - unlink("$glob_mysql_test_dir/var/master-data/relay-log.info"); - unlink("$glob_mysql_test_dir/var/master1-data/master.info"); - unlink("$glob_mysql_test_dir/var/master1-data/relay-log.info"); + unlink("$opt_vardir/master-data/master.info"); + unlink("$opt_vardir/master-data/relay-log.info"); + unlink("$opt_vardir/master1-data/master.info"); + unlink("$opt_vardir/master1-data/relay-log.info"); # Run master initialization shell script if one exists if ( $init_script ) @@ -1600,13 +1639,13 @@ sub do_before_start_slave ($$) { $tname ne "rpl_crash_binlog_ib_3b" ) { # FIXME we really want separate dir for binlogs - foreach my $bin ( glob("$glob_mysql_test_dir/var/log/slave*-bin.*") ) + foreach my $bin ( glob("$opt_vardir/log/slave*-bin.*") ) { unlink($bin); } # FIXME really master?! - unlink("$glob_mysql_test_dir/var/slave-data/master.info"); - unlink("$glob_mysql_test_dir/var/slave-data/relay-log.info"); + unlink("$opt_vardir/slave-data/master.info"); + unlink("$opt_vardir/slave-data/relay-log.info"); } # Run slave initialization shell script if one exists @@ -1620,8 +1659,8 @@ sub do_before_start_slave ($$) { } } - `rm -f $glob_mysql_test_dir/var/slave-data/log.*`; -# unlink("$glob_mysql_test_dir/var/slave-data/log.*"); + `rm -f $opt_vardir/slave-data/log.*`; +# unlink("$opt_vardir/slave-data/log.*"); } sub mysqld_arguments ($$$$$) { @@ -1667,8 +1706,7 @@ sub mysqld_arguments ($$$$$) { if ( $type eq 'master' ) { - mtr_add_arg($args, "%s--log-bin=%s/var/log/master-bin", $prefix, - $glob_mysql_test_dir); + mtr_add_arg($args, "%s--log-bin=%s/log/master-bin", $prefix, $opt_vardir); mtr_add_arg($args, "%s--pid-file=%s", $prefix, $master->[$idx]->{'path_mypid'}); mtr_add_arg($args, "%s--port=%d", $prefix, @@ -1692,8 +1730,8 @@ sub mysqld_arguments ($$$$$) { # FIXME slave get this option twice?! mtr_add_arg($args, "%s--exit-info=256", $prefix); mtr_add_arg($args, "%s--init-rpl-role=slave", $prefix); - mtr_add_arg($args, "%s--log-bin=%s/var/log/slave%s-bin", $prefix, - $glob_mysql_test_dir, $sidx); # FIXME use own dir for binlogs + mtr_add_arg($args, "%s--log-bin=%s/log/slave%s-bin", $prefix, + $opt_vardir, $sidx); # FIXME use own dir for binlogs mtr_add_arg($args, "%s--log-slave-updates", $prefix); # FIXME option duplicated for slave mtr_add_arg($args, "%s--log=%s", $prefix, @@ -1703,8 +1741,8 @@ sub mysqld_arguments ($$$$$) { $slave->[$idx]->{'path_mypid'}); mtr_add_arg($args, "%s--port=%d", $prefix, $slave->[$idx]->{'path_myport'}); - mtr_add_arg($args, "%s--relay-log=%s/var/log/slave%s-relay-bin", $prefix, - $glob_mysql_test_dir, $sidx); + mtr_add_arg($args, "%s--relay-log=%s/log/slave%s-relay-bin", $prefix, + $opt_vardir, $sidx); mtr_add_arg($args, "%s--report-host=127.0.0.1", $prefix); mtr_add_arg($args, "%s--report-port=%d", $prefix, $slave->[$idx]->{'path_myport'}); @@ -1742,13 +1780,13 @@ sub mysqld_arguments ($$$$$) { { if ( $type eq 'master' ) { - mtr_add_arg($args, "%s--debug=d:t:i:A,%s/var/log/master%s.trace", - $prefix, $glob_mysql_test_dir, $sidx); + mtr_add_arg($args, "%s--debug=d:t:i:A,%s/log/master%s.trace", + $prefix, $opt_vardir, $sidx); } if ( $type eq 'slave' ) { - mtr_add_arg($args, "%s--debug=d:t:i:A,%s/var/log/slave%s.trace", - $prefix, $glob_mysql_test_dir, $sidx); + mtr_add_arg($args, "%s--debug=d:t:i:A,%s/log/slave%s.trace", + $prefix, $opt_vardir, $sidx); } } @@ -2000,7 +2038,7 @@ sub run_mysqltest ($$) { if ( $opt_debug ) { $cmdline_mysqldump .= - " --debug=d:t:A,$glob_mysql_test_dir/var/log/mysqldump.trace"; + " --debug=d:t:A,$opt_vardir/log/mysqldump.trace"; } my $cmdline_mysqlbinlog= @@ -2009,7 +2047,7 @@ sub run_mysqltest ($$) { if ( $opt_debug ) { $cmdline_mysqlbinlog .= - " --debug=d:t:A,$glob_mysql_test_dir/var/log/mysqlbinlog.trace"; + " --debug=d:t:A,$opt_vardir/log/mysqlbinlog.trace"; } my $cmdline_mysql= @@ -2072,13 +2110,13 @@ sub run_mysqltest ($$) { { $exe= "strace"; # FIXME there are ktrace, .... mtr_add_arg($args, "-o"); - mtr_add_arg($args, "%s/var/log/mysqltest.strace", $glob_mysql_test_dir); + mtr_add_arg($args, "%s/log/mysqltest.strace", $opt_vardir); mtr_add_arg($args, "$exe_mysqltest"); } if ( $opt_timer ) { - mtr_add_arg($args, "--timer-file=%s/var/log/timer", $glob_mysql_test_dir); + mtr_add_arg($args, "--timer-file=%s/log/timer", $opt_vardir); } if ( $opt_big_test ) @@ -2103,8 +2141,7 @@ sub run_mysqltest ($$) { if ( $opt_debug ) { - mtr_add_arg($args, "--debug=d:t:A,%s/var/log/mysqltest.trace", - $glob_mysql_test_dir); + mtr_add_arg($args, "--debug=d:t:A,%s/log/mysqltest.trace", $opt_vardir); } if ( $opt_with_openssl ) @@ -2126,7 +2163,7 @@ sub run_mysqltest ($$) { mysqld_arguments($args,'master',0,$tinfo->{'master_opt'},[]); } - return mtr_run_test($exe_mysqltest,$args,$tinfo->{'path'},"",$path_timefile,""); + return mtr_run_test($exe,$args,$tinfo->{'path'},"",$path_timefile,""); } ############################################################################## diff --git a/mysql-test/mysql-test-run.sh b/mysql-test/mysql-test-run.sh index 1a648f98b1b..967d6e6408e 100644 --- a/mysql-test/mysql-test-run.sh +++ b/mysql-test/mysql-test-run.sh @@ -586,10 +586,7 @@ if [ x$SOURCE_DIST = x1 ] ; then NDB_MGM="$BASEDIR/ndb/src/mgmclient/ndb_mgm" if [ -n "$USE_PURIFY" ] ; then - PSUP="$MYSQL_TEST_DIR/purify.suppress" - echo "suppress UMR rw_read_held; mi_open; ha_myisam::open64; handler::ha_open; openfrm" > $PSUP - echo "suppress UMR my_end; main" >> $PSUP - echo "suppress UMR _doprnt; fprintf; my_end; main" >> $PSUP + PSUP="$MYSQL_TEST_DIR/suppress.purify" PURIFYOPTIONS="-windows=no -log-file=%v.purifylog -append-logfile -add-suppression-files=$PSUP" if [ -f "${MYSQL_TEST}-purify" ] ; then MYSQL_TEST="${MYSQL_TEST}-purify" @@ -1705,11 +1702,11 @@ run_testcase () $ECHO "$RES$RES_SPACE [ pass ] $TIMER" else # why the following ``if'' ? That is why res==1 is special ? - if [ $res = 2 ]; then + if [ $res = 62 ]; then skip_inc $ECHO "$RES$RES_SPACE [ skipped ]" else - if [ $res -gt 2 ]; then + if [ $res -ne 1 ]; then $ECHO "mysqltest returned unexpected code $res, it has probably crashed" >> $TIMEFILE fi total_inc diff --git a/mysql-test/suppress.purify b/mysql-test/suppress.purify new file mode 100644 index 00000000000..58553130c51 --- /dev/null +++ b/mysql-test/suppress.purify @@ -0,0 +1,10 @@ +suppress UMR rw_read_held; mi_open; ha_myisam::open64; handler::ha_open; openfrm +suppress UMR my_end; main +suppress UMR _doprnt; fprintf; my_end; main +suppress umr rw_read_held; mi_open +suppress umr _putmsg; putmsg; _tx_sndudata +suppress umr rw_read_held; Query_cache::store_query(THD*,st_table_list*); mysql_execute_command(THD*) +suppress sig ...; _select; select; handle_connections_sockets; main; _start +suppress sig ...; read; vio_read; my_real_read(st_net*,unsigned long*); my_net_read; do_command(THD*) +suppress sig ...; read; vio_read; my_real_read(st_net*,unsigned long*); my_net_read; net_safe_read +suppress sig ...; write; vio_write; net_real_write; net_write_buff(st_net*,const char*,unsigned long); my_net_write diff --git a/mysys/default.c b/mysys/default.c index a680915b6d5..ba1a6c1513b 100644 --- a/mysys/default.c +++ b/mysys/default.c @@ -117,11 +117,10 @@ int my_correct_defaults_file(const char *file_location, const char *option, char *file_buffer; uint position= 0; int is_found= FALSE; + DBUG_ENTER("my_correct_defaults_file"); optlen= strlen(option); - DBUG_ENTER("my_correct_file"); - if (!(cnf_file= my_fopen(file_location, O_RDWR, MYF(0)))) goto err_fopen; diff --git a/ndb/include/debugger/SignalLoggerManager.hpp b/ndb/include/debugger/SignalLoggerManager.hpp index 742bf7d294e..d212329bf78 100644 --- a/ndb/include/debugger/SignalLoggerManager.hpp +++ b/ndb/include/debugger/SignalLoggerManager.hpp @@ -87,7 +87,7 @@ public: /** * Generic messages in the signal log */ - void log(BlockNumber bno, const char * msg); + void log(BlockNumber bno, const char * msg, ...); /** * LogModes diff --git a/ndb/include/kernel/ndb_limits.h b/ndb/include/kernel/ndb_limits.h index 2646b54fa02..d9d28398be9 100644 --- a/ndb/include/kernel/ndb_limits.h +++ b/ndb/include/kernel/ndb_limits.h @@ -50,7 +50,7 @@ **/ #define MAX_TUPLES_PER_PAGE 8191 #define MAX_TUPLES_BITS 13 /* 13 bits = 8191 tuples per page */ -#define MAX_TABLES 1600 +#define MAX_TABLES 20320 /* SchemaFile.hpp */ #define MAX_TAB_NAME_SIZE 128 #define MAX_ATTR_NAME_SIZE 32 #define MAX_ATTR_DEFAULT_VALUE_SIZE 128 diff --git a/ndb/include/kernel/signaldata/AccScan.hpp b/ndb/include/kernel/signaldata/AccScan.hpp index d1ca4424d1a..d94d4da8cca 100644 --- a/ndb/include/kernel/signaldata/AccScan.hpp +++ b/ndb/include/kernel/signaldata/AccScan.hpp @@ -34,6 +34,7 @@ class AccScanReq { */ friend class Dbacc; friend class Dbtux; + friend class Dbtup; public: STATIC_CONST( SignalLength = 8 ); @@ -120,6 +121,7 @@ class AccScanConf { */ friend class Dbacc; friend class Dbtux; + friend class Dbtup; /** * Reciver(s) @@ -148,6 +150,7 @@ private: class AccCheckScan { friend class Dbacc; friend class Dbtux; + friend class Dbtup; friend class Dblqh; enum { ZCHECK_LCP_STOP = 0, diff --git a/ndb/include/kernel/signaldata/NextScan.hpp b/ndb/include/kernel/signaldata/NextScan.hpp index 3a1882f94e8..a502a89108c 100644 --- a/ndb/include/kernel/signaldata/NextScan.hpp +++ b/ndb/include/kernel/signaldata/NextScan.hpp @@ -23,6 +23,7 @@ class NextScanReq { friend class Dblqh; friend class Dbacc; friend class Dbtux; + friend class Dbtup; public: // two sets of defs picked from lqh/acc enum ScanFlag { @@ -50,6 +51,7 @@ private: class NextScanConf { friend class Dbacc; friend class Dbtux; + friend class Dbtup; friend class Dblqh; public: // length is less if no keyinfo or no next result diff --git a/ndb/include/kernel/signaldata/ScanFrag.hpp b/ndb/include/kernel/signaldata/ScanFrag.hpp index e4b774f1416..f21a3eef7ac 100644 --- a/ndb/include/kernel/signaldata/ScanFrag.hpp +++ b/ndb/include/kernel/signaldata/ScanFrag.hpp @@ -57,6 +57,7 @@ public: static Uint32 getReadCommittedFlag(const Uint32 & requestInfo); static Uint32 getRangeScanFlag(const Uint32 & requestInfo); static Uint32 getDescendingFlag(const Uint32 & requestInfo); + static Uint32 getTupScanFlag(const Uint32 & requestInfo); static Uint32 getAttrLen(const Uint32 & requestInfo); static Uint32 getScanPrio(const Uint32 & requestInfo); @@ -66,6 +67,7 @@ public: static void setReadCommittedFlag(Uint32 & requestInfo, Uint32 readCommitted); static void setRangeScanFlag(Uint32 & requestInfo, Uint32 rangeScan); static void setDescendingFlag(Uint32 & requestInfo, Uint32 descending); + static void setTupScanFlag(Uint32 & requestInfo, Uint32 tupScan); static void setAttrLen(Uint32 & requestInfo, Uint32 attrLen); static void setScanPrio(Uint32& requestInfo, Uint32 prio); }; @@ -200,11 +202,12 @@ public: * r = read committed - 1 Bit 9 * x = range scan - 1 Bit 6 * z = descending - 1 Bit 10 + * t = tup scan -1 Bit 11 (implies x=z=0) * p = Scan prio - 4 Bits (12-15) -> max 15 * * 1111111111222222222233 * 01234567890123456789012345678901 - * lxhkrz ppppaaaaaaaaaaaaaaaa + * lxhkrztppppaaaaaaaaaaaaaaaa */ #define SF_LOCK_MODE_SHIFT (5) #define SF_LOCK_MODE_MASK (1) @@ -214,6 +217,7 @@ public: #define SF_READ_COMMITTED_SHIFT (9) #define SF_RANGE_SCAN_SHIFT (6) #define SF_DESCENDING_SHIFT (10) +#define SF_TUP_SCAN_SHIFT (11) #define SF_ATTR_LEN_SHIFT (16) #define SF_ATTR_LEN_MASK (65535) @@ -253,6 +257,12 @@ ScanFragReq::getDescendingFlag(const Uint32 & requestInfo){ inline Uint32 +ScanFragReq::getTupScanFlag(const Uint32 & requestInfo){ + return (requestInfo >> SF_TUP_SCAN_SHIFT) & 1; +} + +inline +Uint32 ScanFragReq::getReadCommittedFlag(const Uint32 & requestInfo){ return (requestInfo >> SF_READ_COMMITTED_SHIFT) & 1; } @@ -320,6 +330,13 @@ ScanFragReq::setDescendingFlag(UintR & requestInfo, UintR val){ inline void +ScanFragReq::setTupScanFlag(UintR & requestInfo, UintR val){ + ASSERT_BOOL(val, "ScanFragReq::setTupScanFlag"); + requestInfo |= (val << SF_TUP_SCAN_SHIFT); +} + +inline +void ScanFragReq::setAttrLen(UintR & requestInfo, UintR val){ ASSERT_MAX(val, SF_ATTR_LEN_MASK, "ScanFragReq::setAttrLen"); requestInfo |= (val << SF_ATTR_LEN_SHIFT); diff --git a/ndb/include/kernel/signaldata/ScanTab.hpp b/ndb/include/kernel/signaldata/ScanTab.hpp index 894f973145c..8cb282270ff 100644 --- a/ndb/include/kernel/signaldata/ScanTab.hpp +++ b/ndb/include/kernel/signaldata/ScanTab.hpp @@ -81,6 +81,7 @@ private: static Uint8 getReadCommittedFlag(const UintR & requestInfo); static Uint8 getRangeScanFlag(const UintR & requestInfo); static Uint8 getDescendingFlag(const UintR & requestInfo); + static Uint8 getTupScanFlag(const UintR & requestInfo); static Uint8 getKeyinfoFlag(const UintR & requestInfo); static Uint16 getScanBatch(const UintR & requestInfo); static Uint8 getDistributionKeyFlag(const UintR & requestInfo); @@ -95,6 +96,7 @@ private: static void setReadCommittedFlag(UintR & requestInfo, Uint32 flag); static void setRangeScanFlag(UintR & requestInfo, Uint32 flag); static void setDescendingFlag(UintR & requestInfo, Uint32 flag); + static void setTupScanFlag(UintR & requestInfo, Uint32 flag); static void setKeyinfoFlag(UintR & requestInfo, Uint32 flag); static void setScanBatch(Uint32& requestInfo, Uint32 sz); static void setDistributionKeyFlag(Uint32& requestInfo, Uint32 flag); @@ -108,6 +110,7 @@ private: h = Hold lock mode - 1 Bit 10 c = Read Committed - 1 Bit 11 k = Keyinfo - 1 Bit 12 + t = Tup scan - 1 Bit 13 z = Descending (TUX) - 1 Bit 14 x = Range Scan (TUX) - 1 Bit 15 b = Scan batch - 10 Bit 16-25 (max 1023) @@ -115,7 +118,7 @@ private: 1111111111222222222233 01234567890123456789012345678901 - ppppppppl hck zxbbbbbbbbbb + ppppppppl hcktzxbbbbbbbbbb */ #define PARALLELL_SHIFT (0) @@ -139,6 +142,9 @@ private: #define DESCENDING_SHIFT (14) #define DESCENDING_MASK (1) +#define TUP_SCAN_SHIFT (13) +#define TUP_SCAN_MASK (1) + #define SCAN_BATCH_SHIFT (16) #define SCAN_BATCH_MASK (1023) @@ -181,6 +187,12 @@ ScanTabReq::getDescendingFlag(const UintR & requestInfo){ } inline +Uint8 +ScanTabReq::getTupScanFlag(const UintR & requestInfo){ + return (Uint8)((requestInfo >> TUP_SCAN_SHIFT) & TUP_SCAN_MASK); +} + +inline Uint16 ScanTabReq::getScanBatch(const Uint32 & requestInfo){ return (Uint16)((requestInfo >> SCAN_BATCH_SHIFT) & SCAN_BATCH_MASK); @@ -235,6 +247,13 @@ ScanTabReq::setDescendingFlag(UintR & requestInfo, Uint32 flag){ } inline +void +ScanTabReq::setTupScanFlag(UintR & requestInfo, Uint32 flag){ + ASSERT_BOOL(flag, "ScanTabReq::setTupScanFlag"); + requestInfo |= (flag << TUP_SCAN_SHIFT); +} + +inline void ScanTabReq::setScanBatch(Uint32 & requestInfo, Uint32 flag){ ASSERT_MAX(flag, SCAN_BATCH_MASK, "ScanTabReq::setScanBatch"); diff --git a/ndb/include/kernel/signaldata/TcCommit.hpp b/ndb/include/kernel/signaldata/TcCommit.hpp index 9499b20ada3..dcbca0cb6f2 100644 --- a/ndb/include/kernel/signaldata/TcCommit.hpp +++ b/ndb/include/kernel/signaldata/TcCommit.hpp @@ -36,7 +36,7 @@ class TcCommitConf { friend class NdbTransaction; public: - STATIC_CONST( SignalLength = 3 ); + STATIC_CONST( SignalLength = 4 ); private: /** @@ -49,6 +49,7 @@ private: Uint32 transId1; Uint32 transId2; + Uint32 gci; }; class TcCommitRef { diff --git a/ndb/include/ndbapi/NdbIndexScanOperation.hpp b/ndb/include/ndbapi/NdbIndexScanOperation.hpp index b91a2a90d11..0a31f228921 100644 --- a/ndb/include/ndbapi/NdbIndexScanOperation.hpp +++ b/ndb/include/ndbapi/NdbIndexScanOperation.hpp @@ -37,6 +37,17 @@ public: * readTuples using ordered index * * @param lock_mode Lock mode + * @param scan_flags see @ref ScanFlag + * @param parallel No of fragments to scan in parallel (0=max) + */ + virtual int readTuples(LockMode lock_mode = LM_Read, + Uint32 scan_flags = 0, Uint32 parallel = 0); + +#ifndef DOXYGEN_SHOULD_SKIP_INTERNAL + /** + * readTuples using ordered index + * + * @param lock_mode Lock mode * @param batch No of rows to fetch from each fragment at a time * @param parallel No of fragments to scan in parallel * @param order_by Order result set in index order @@ -45,13 +56,20 @@ public: * @returns 0 for success and -1 for failure * @see NdbScanOperation::readTuples */ - int readTuples(LockMode lock_mode = LM_Read, - Uint32 batch = 0, - Uint32 parallel = 0, - bool order_by = false, - bool order_desc = false, - bool read_range_no = false); - + inline int readTuples(LockMode lock_mode, + Uint32 batch, + Uint32 parallel, + bool order_by, + bool order_desc = false, + bool read_range_no = false) { + Uint32 scan_flags = + (SF_OrderBy & -(Int32)order_by) | + (SF_Descending & -(Int32)order_desc) | + (SF_ReadRangeNo & -(Int32)read_range_no); + return readTuples(lock_mode, scan_flags, parallel); + } +#endif + /** * Type of ordered index key bound. The values (0-4) will not change * and can be used explicitly (e.g. they could be computed). diff --git a/ndb/include/ndbapi/NdbScanOperation.hpp b/ndb/include/ndbapi/NdbScanOperation.hpp index 1c9649195d2..bf8f362cefc 100644 --- a/ndb/include/ndbapi/NdbScanOperation.hpp +++ b/ndb/include/ndbapi/NdbScanOperation.hpp @@ -37,6 +37,29 @@ class NdbScanOperation : public NdbOperation { public: /** + * Scan flags. OR-ed together and passed as second argument to + * readTuples. + */ + enum ScanFlag { + SF_TupScan = (1 << 16), // scan TUP - only LM_CommittedRead + SF_OrderBy = (1 << 24), // index scan in order + SF_Descending = (2 << 24), // index scan in descending order + SF_ReadRangeNo = (4 << 24) // enable @ref get_range_no + }; + + /** + * readTuples + * + * @param lock_mode Lock mode + * @param scan_flags see @ref ScanFlag + * @param parallel No of fragments to scan in parallel (0=max) + */ + virtual + int readTuples(LockMode lock_mode = LM_Read, + Uint32 scan_flags = 0, Uint32 parallel = 0); + +#ifndef DOXYGEN_SHOULD_SKIP_DEPRECATED + /** * readTuples * * @param lock_mode Lock mode @@ -44,10 +67,11 @@ public: * @param parallel No of fragments to scan in parallell * @note specifying 0 for batch and parallall means max performance */ +#ifdef ndb_readtuples_impossible_overload int readTuples(LockMode lock_mode = LM_Read, Uint32 batch = 0, Uint32 parallel = 0); +#endif -#ifndef DOXYGEN_SHOULD_SKIP_DEPRECATED inline int readTuples(int parallell){ return readTuples(LM_Read, 0, parallell); } diff --git a/ndb/src/common/debugger/SignalLoggerManager.cpp b/ndb/src/common/debugger/SignalLoggerManager.cpp index d642ed09a68..1f0bd8974e7 100644 --- a/ndb/src/common/debugger/SignalLoggerManager.cpp +++ b/ndb/src/common/debugger/SignalLoggerManager.cpp @@ -383,7 +383,7 @@ SignalLoggerManager::sendSignalWithDelay(Uint32 delayInMilliSeconds, * Generic messages in the signal log */ void -SignalLoggerManager::log(BlockNumber bno, const char * msg) +SignalLoggerManager::log(BlockNumber bno, const char * msg, ...) { // Normalise blocknumber for use in logModes array const BlockNumber bno2 = bno - MIN_BLOCK_NO; @@ -391,7 +391,12 @@ SignalLoggerManager::log(BlockNumber bno, const char * msg) if(outputStream != 0 && logModes[bno2] != LogOff){ - fprintf(outputStream, "%s: %s\n", getBlockName(bno, "API"), msg); + va_list ap; + va_start(ap, msg); + fprintf(outputStream, "%s: ", getBlockName(bno, "API")); + vfprintf(outputStream, msg, ap); + fprintf(outputStream, "\n", msg); + va_end(ap); } } diff --git a/ndb/src/common/debugger/signaldata/ScanTab.cpp b/ndb/src/common/debugger/signaldata/ScanTab.cpp index e9c5ba6cc52..d78beb4740a 100644 --- a/ndb/src/common/debugger/signaldata/ScanTab.cpp +++ b/ndb/src/common/debugger/signaldata/ScanTab.cpp @@ -30,7 +30,7 @@ printSCANTABREQ(FILE * output, const Uint32 * theData, Uint32 len, Uint16 receiv fprintf(output, " apiConnectPtr: H\'%.8x", sig->apiConnectPtr); fprintf(output, " requestInfo: H\'%.8x:\n", requestInfo); - fprintf(output, " Parallellism: %u, Batch: %u LockMode: %u Keyinfo: %u Holdlock: %u RangeScan: %u Descending: %u ReadCommitted: %u\n DistributionKeyFlag: %u", + fprintf(output, " Parallellism: %u Batch: %u LockMode: %u Keyinfo: %u Holdlock: %u RangeScan: %u Descending: %u TupScan: %u\n ReadCommitted: %u DistributionKeyFlag: %u", sig->getParallelism(requestInfo), sig->getScanBatch(requestInfo), sig->getLockMode(requestInfo), @@ -38,6 +38,7 @@ printSCANTABREQ(FILE * output, const Uint32 * theData, Uint32 len, Uint16 receiv sig->getHoldLockFlag(requestInfo), sig->getRangeScanFlag(requestInfo), sig->getDescendingFlag(requestInfo), + sig->getTupScanFlag(requestInfo), sig->getReadCommittedFlag(requestInfo), sig->getDistributionKeyFlag(requestInfo)); diff --git a/ndb/src/cw/cpcd/APIService.cpp b/ndb/src/cw/cpcd/APIService.cpp index b009f0c0fc4..e7a2092c15d 100644 --- a/ndb/src/cw/cpcd/APIService.cpp +++ b/ndb/src/cw/cpcd/APIService.cpp @@ -136,6 +136,8 @@ ParserRow<CPCDAPISession> commands[] = CPCD_ARG("id", Int, Mandatory, "Id of process"), CPCD_CMD("list processes", &CPCDAPISession::listProcesses, ""), + + CPCD_CMD("show version", &CPCDAPISession::showVersion, ""), CPCD_END() }; @@ -359,6 +361,7 @@ CPCDAPISession::listProcesses(Parser_t::Context & /* unused */, m_output->println("stdout: %s", p->m_stdout.c_str()); m_output->println("stderr: %s", p->m_stderr.c_str()); m_output->println("ulimit: %s", p->m_ulimit.c_str()); + m_output->println("shutdown: %s", p->m_shutdown_options.c_str()); switch(p->m_status){ case STOPPED: m_output->println("status: stopped"); @@ -384,4 +387,16 @@ CPCDAPISession::listProcesses(Parser_t::Context & /* unused */, m_cpcd.m_processes.unlock(); } +void +CPCDAPISession::showVersion(Parser_t::Context & /* unused */, + const class Properties & args){ + Uint32 id; + CPCD::RequestStatus rs; + + m_output->println("show version"); + m_output->println("compile time: %s %s", __DATE__, __TIME__); + + m_output->println(""); +} + template class Vector<ParserRow<CPCDAPISession> const*>; diff --git a/ndb/src/cw/cpcd/APIService.hpp b/ndb/src/cw/cpcd/APIService.hpp index ef988785f89..3586d64187e 100644 --- a/ndb/src/cw/cpcd/APIService.hpp +++ b/ndb/src/cw/cpcd/APIService.hpp @@ -49,6 +49,7 @@ public: void stopProcess(Parser_t::Context & ctx, const class Properties & args); void showProcess(Parser_t::Context & ctx, const class Properties & args); void listProcesses(Parser_t::Context & ctx, const class Properties & args); + void showVersion(Parser_t::Context & ctx, const class Properties & args); }; class CPCDAPIService : public SocketServer::Service { diff --git a/ndb/src/kernel/blocks/dbacc/DbaccInit.cpp b/ndb/src/kernel/blocks/dbacc/DbaccInit.cpp index 94782e13e00..90839163a72 100644 --- a/ndb/src/kernel/blocks/dbacc/DbaccInit.cpp +++ b/ndb/src/kernel/blocks/dbacc/DbaccInit.cpp @@ -204,6 +204,80 @@ Dbacc::Dbacc(const class Configuration & conf): addRecSignal(GSN_SET_VAR_REQ, &Dbacc::execSET_VAR_REQ); initData(); + +#ifdef VM_TRACE + { + void* tmp[] = { &expDirRangePtr, + &gnsDirRangePtr, + &newDirRangePtr, + &rdDirRangePtr, + &nciOverflowrangeptr, + &expDirptr, + &rdDirptr, + &sdDirptr, + &nciOverflowDirptr, + &fragrecptr, + &fsConnectptr, + &fsOpptr, + &lcpConnectptr, + &operationRecPtr, + &idrOperationRecPtr, + ©InOperPtr, + ©OperPtr, + &mlpqOperPtr, + &queOperPtr, + &readWriteOpPtr, + &iopOverflowRecPtr, + &tfoOverflowRecPtr, + &porOverflowRecPtr, + &priOverflowRecPtr, + &rorOverflowRecPtr, + &sorOverflowRecPtr, + &troOverflowRecPtr, + &ancPageptr, + &colPageptr, + &ccoPageptr, + &datapageptr, + &delPageptr, + &excPageptr, + &expPageptr, + &gdiPageptr, + &gePageptr, + &gflPageptr, + &idrPageptr, + &ilcPageptr, + &inpPageptr, + &iopPageptr, + &lastPageptr, + &lastPrevpageptr, + &lcnPageptr, + &lcnCopyPageptr, + &lupPageptr, + &priPageptr, + &pwiPageptr, + &ciPageidptr, + &gsePageidptr, + &isoPageptr, + &nciPageidptr, + &rsbPageidptr, + &rscPageidptr, + &slPageidptr, + &sscPageidptr, + &rlPageptr, + &rlpPageptr, + &ropPageptr, + &rpPageptr, + &slPageptr, + &spPageptr, + &rootfragrecptr, + &scanPtr, + &srVersionPtr, + &tabptr, + &undopageptr + }; + init_globals_list(tmp, sizeof(tmp)/sizeof(tmp[0])); + } +#endif }//Dbacc::Dbacc() Dbacc::~Dbacc() diff --git a/ndb/src/kernel/blocks/dbdict/Dbdict.cpp b/ndb/src/kernel/blocks/dbdict/Dbdict.cpp index b9877dae46c..d51f9537154 100644 --- a/ndb/src/kernel/blocks/dbdict/Dbdict.cpp +++ b/ndb/src/kernel/blocks/dbdict/Dbdict.cpp @@ -634,7 +634,7 @@ void Dbdict::writeTableFile(Signal* signal, Uint32 filePtr, Uint32 fsConPtr) FsReadWriteReq::setSyncFlag(fsRWReq->operationFlag, 1); FsReadWriteReq::setFormatFlag(fsRWReq->operationFlag, FsReadWriteReq::fsFormatArrayOfPages); - fsRWReq->varIndex = ZALLOCATE; + fsRWReq->varIndex = ZBAT_TABLE_FILE; fsRWReq->numberOfPages = c_writeTableRecord.noOfPages; fsRWReq->data.arrayOfPages.varIndex = c_writeTableRecord.pageId; fsRWReq->data.arrayOfPages.fileOffset = 0; // Write to file page 0 @@ -711,7 +711,7 @@ void Dbdict::readTableFile(Signal* signal, Uint32 filePtr, Uint32 fsConPtr) FsReadWriteReq::setSyncFlag(fsRWReq->operationFlag, 0); FsReadWriteReq::setFormatFlag(fsRWReq->operationFlag, FsReadWriteReq::fsFormatArrayOfPages); - fsRWReq->varIndex = ZALLOCATE; + fsRWReq->varIndex = ZBAT_TABLE_FILE; fsRWReq->numberOfPages = c_readTableRecord.noOfPages; fsRWReq->data.arrayOfPages.varIndex = c_readTableRecord.pageId; fsRWReq->data.arrayOfPages.fileOffset = 0; // Write to file page 0 @@ -777,11 +777,9 @@ Dbdict::updateSchemaState(Signal* signal, Uint32 tableId, SchemaFile::TableEntry* te, Callback* callback){ jam(); - PageRecordPtr pagePtr; - c_pageRecordArray.getPtr(pagePtr, c_schemaRecord.schemaPage); - ndbrequire(tableId < c_tableRecordPool.getSize()); - SchemaFile::TableEntry * tableEntry = getTableEntry(pagePtr.p, tableId); + XSchemaFile * xsf = &c_schemaFile[c_schemaRecord.schemaPage != 0]; + SchemaFile::TableEntry * tableEntry = getTableEntry(xsf, tableId); SchemaFile::TableState newState = (SchemaFile::TableState)te->m_tableState; @@ -828,12 +826,15 @@ Dbdict::updateSchemaState(Signal* signal, Uint32 tableId, ndbrequire(ok); * tableEntry = * te; - computeChecksum((SchemaFile*)pagePtr.p); + computeChecksum(xsf, tableId / NDB_SF_PAGE_ENTRIES); ndbrequire(c_writeSchemaRecord.inUse == false); c_writeSchemaRecord.inUse = true; c_writeSchemaRecord.pageId = c_schemaRecord.schemaPage; + c_writeSchemaRecord.newFile = false; + c_writeSchemaRecord.firstPage = tableId / NDB_SF_PAGE_ENTRIES; + c_writeSchemaRecord.noOfPages = 1; c_writeSchemaRecord.m_callback = * callback; startWriteSchemaFile(signal); @@ -844,14 +845,15 @@ void Dbdict::startWriteSchemaFile(Signal* signal) FsConnectRecordPtr fsPtr; c_fsConnectRecordPool.getPtr(fsPtr, getFsConnRecord()); fsPtr.p->fsState = FsConnectRecord::OPEN_WRITE_SCHEMA; - openSchemaFile(signal, 0, fsPtr.i, true); + openSchemaFile(signal, 0, fsPtr.i, true, c_writeSchemaRecord.newFile); c_writeSchemaRecord.noOfSchemaFilesHandled = 0; }//Dbdict::startWriteSchemaFile() void Dbdict::openSchemaFile(Signal* signal, Uint32 fileNo, Uint32 fsConPtr, - bool writeFlag) + bool writeFlag, + bool newFile) { FsOpenReq * const fsOpenReq = (FsOpenReq *)&signal->theData[0]; fsOpenReq->userReference = reference(); @@ -860,9 +862,11 @@ void Dbdict::openSchemaFile(Signal* signal, jam(); fsOpenReq->fileFlags = FsOpenReq::OM_WRITEONLY | - FsOpenReq::OM_TRUNCATE | - FsOpenReq::OM_CREATE | FsOpenReq::OM_SYNC; + if (newFile) + fsOpenReq->fileFlags |= + FsOpenReq::OM_TRUNCATE | + FsOpenReq::OM_CREATE; } else { jam(); fsOpenReq->fileFlags = FsOpenReq::OM_READONLY; @@ -887,6 +891,12 @@ void Dbdict::writeSchemaFile(Signal* signal, Uint32 filePtr, Uint32 fsConPtr) { FsReadWriteReq * const fsRWReq = (FsReadWriteReq *)&signal->theData[0]; + // check write record + WriteSchemaRecord & wr = c_writeSchemaRecord; + ndbrequire(wr.pageId == (wr.pageId != 0) * NDB_SF_MAX_PAGES); + ndbrequire(wr.noOfPages != 0); + ndbrequire(wr.firstPage + wr.noOfPages <= NDB_SF_MAX_PAGES); + fsRWReq->filePointer = filePtr; fsRWReq->userReference = reference(); fsRWReq->userPointer = fsConPtr; @@ -894,11 +904,11 @@ void Dbdict::writeSchemaFile(Signal* signal, Uint32 filePtr, Uint32 fsConPtr) FsReadWriteReq::setSyncFlag(fsRWReq->operationFlag, 1); FsReadWriteReq::setFormatFlag(fsRWReq->operationFlag, FsReadWriteReq::fsFormatArrayOfPages); - fsRWReq->varIndex = ZALLOCATE; - fsRWReq->numberOfPages = 1; -// Write from memory page - fsRWReq->data.arrayOfPages.varIndex = c_writeSchemaRecord.pageId; - fsRWReq->data.arrayOfPages.fileOffset = 0; // Write to file page 0 + fsRWReq->varIndex = ZBAT_SCHEMA_FILE; + fsRWReq->numberOfPages = wr.noOfPages; + // Write from memory page + fsRWReq->data.arrayOfPages.varIndex = wr.pageId + wr.firstPage; + fsRWReq->data.arrayOfPages.fileOffset = wr.firstPage; sendSignal(NDBFS_REF, GSN_FSWRITEREQ, signal, 8, JBA); }//writeSchemaFile() @@ -928,7 +938,7 @@ void Dbdict::closeWriteSchemaConf(Signal* signal, if (c_writeSchemaRecord.noOfSchemaFilesHandled < 2) { jam(); fsPtr.p->fsState = FsConnectRecord::OPEN_WRITE_SCHEMA; - openSchemaFile(signal, 1, fsPtr.i, true); + openSchemaFile(signal, 1, fsPtr.i, true, c_writeSchemaRecord.newFile); return; } ndbrequire(c_writeSchemaRecord.noOfSchemaFilesHandled == 2); @@ -946,20 +956,26 @@ void Dbdict::startReadSchemaFile(Signal* signal) FsConnectRecordPtr fsPtr; c_fsConnectRecordPool.getPtr(fsPtr, getFsConnRecord()); fsPtr.p->fsState = FsConnectRecord::OPEN_READ_SCHEMA1; - openSchemaFile(signal, 0, fsPtr.i, false); + openSchemaFile(signal, 0, fsPtr.i, false, false); }//Dbdict::startReadSchemaFile() void Dbdict::openReadSchemaRef(Signal* signal, FsConnectRecordPtr fsPtr) { fsPtr.p->fsState = FsConnectRecord::OPEN_READ_SCHEMA2; - openSchemaFile(signal, 1, fsPtr.i, false); + openSchemaFile(signal, 1, fsPtr.i, false, false); }//Dbdict::openReadSchemaRef() void Dbdict::readSchemaFile(Signal* signal, Uint32 filePtr, Uint32 fsConPtr) { FsReadWriteReq * const fsRWReq = (FsReadWriteReq *)&signal->theData[0]; + // check read record + ReadSchemaRecord & rr = c_readSchemaRecord; + ndbrequire(rr.pageId == (rr.pageId != 0) * NDB_SF_MAX_PAGES); + ndbrequire(rr.noOfPages != 0); + ndbrequire(rr.firstPage + rr.noOfPages <= NDB_SF_MAX_PAGES); + fsRWReq->filePointer = filePtr; fsRWReq->userReference = reference(); fsRWReq->userPointer = fsConPtr; @@ -967,10 +983,10 @@ void Dbdict::readSchemaFile(Signal* signal, Uint32 filePtr, Uint32 fsConPtr) FsReadWriteReq::setSyncFlag(fsRWReq->operationFlag, 0); FsReadWriteReq::setFormatFlag(fsRWReq->operationFlag, FsReadWriteReq::fsFormatArrayOfPages); - fsRWReq->varIndex = ZALLOCATE; - fsRWReq->numberOfPages = 1; - fsRWReq->data.arrayOfPages.varIndex = c_readSchemaRecord.pageId; - fsRWReq->data.arrayOfPages.fileOffset = 0; + fsRWReq->varIndex = ZBAT_SCHEMA_FILE; + fsRWReq->numberOfPages = rr.noOfPages; + fsRWReq->data.arrayOfPages.varIndex = rr.pageId + rr.firstPage; + fsRWReq->data.arrayOfPages.fileOffset = rr.firstPage; sendSignal(NDBFS_REF, GSN_FSREADREQ, signal, 8, JBA); }//readSchemaFile() @@ -988,20 +1004,61 @@ void Dbdict::readSchemaConf(Signal* signal, jam(); crashInd = true; }//if - PageRecordPtr tmpPagePtr; - c_pageRecordArray.getPtr(tmpPagePtr, c_readSchemaRecord.pageId); - Uint32 sz = ZSIZE_OF_PAGES_IN_WORDS; - Uint32 chk = computeChecksum((const Uint32*)tmpPagePtr.p, sz); + ReadSchemaRecord & rr = c_readSchemaRecord; + XSchemaFile * xsf = &c_schemaFile[rr.pageId != 0]; - ndbrequire((chk == 0) || !crashInd); + if (rr.schemaReadState == ReadSchemaRecord::INITIAL_READ_HEAD) { + jam(); + ndbrequire(rr.firstPage == 0); + SchemaFile * sf = &xsf->schemaPage[0]; + Uint32 noOfPages; + if (sf->NdbVersion < NDB_SF_VERSION_5_0_6) { + jam(); + const Uint32 pageSize_old = 32 * 1024; + noOfPages = pageSize_old / NDB_SF_PAGE_SIZE - 1; + } else { + noOfPages = sf->FileSize / NDB_SF_PAGE_SIZE - 1; + } + rr.schemaReadState = ReadSchemaRecord::INITIAL_READ; + if (noOfPages != 0) { + rr.firstPage = 1; + rr.noOfPages = noOfPages; + readSchemaFile(signal, fsPtr.p->filePtr, fsPtr.i); + return; + } + } + + SchemaFile * sf0 = &xsf->schemaPage[0]; + xsf->noOfPages = sf0->FileSize / NDB_SF_PAGE_SIZE; - if (chk != 0){ + if (sf0->NdbVersion < NDB_SF_VERSION_5_0_6 && + ! convertSchemaFileTo_5_0_6(xsf)) { jam(); + ndbrequire(! crashInd); ndbrequire(fsPtr.p->fsState == FsConnectRecord::READ_SCHEMA1); readSchemaRef(signal, fsPtr); return; - }//if + } + + for (Uint32 n = 0; n < xsf->noOfPages; n++) { + SchemaFile * sf = &xsf->schemaPage[n]; + bool ok = + memcmp(sf->Magic, NDB_SF_MAGIC, sizeof(sf->Magic)) == 0 && + sf->FileSize != 0 && + sf->FileSize % NDB_SF_PAGE_SIZE == 0 && + sf->FileSize == sf0->FileSize && + sf->PageNumber == n && + computeChecksum((Uint32*)sf, NDB_SF_PAGE_SIZE_IN_WORDS) == 0; + ndbrequire(ok || !crashInd); + if (! ok) { + jam(); + ndbrequire(fsPtr.p->fsState == FsConnectRecord::READ_SCHEMA1); + readSchemaRef(signal, fsPtr); + return; + } + } + fsPtr.p->fsState = FsConnectRecord::CLOSE_READ_SCHEMA; closeFile(signal, fsPtr.p->filePtr, fsPtr.i); return; @@ -1011,7 +1068,7 @@ void Dbdict::readSchemaRef(Signal* signal, FsConnectRecordPtr fsPtr) { fsPtr.p->fsState = FsConnectRecord::OPEN_READ_SCHEMA2; - openSchemaFile(signal, 1, fsPtr.i, false); + openSchemaFile(signal, 1, fsPtr.i, false, false); return; }//Dbdict::readSchemaRef() @@ -1025,7 +1082,27 @@ void Dbdict::closeReadSchemaConf(Signal* signal, switch(state) { case ReadSchemaRecord::INITIAL_READ : jam(); - sendNDB_STTORRY(signal); + { + // write back both copies + + ndbrequire(c_writeSchemaRecord.inUse == false); + XSchemaFile * xsf = &c_schemaFile[c_schemaRecord.oldSchemaPage != 0 ]; + Uint32 noOfPages = + (c_tableRecordPool.getSize() + NDB_SF_PAGE_ENTRIES - 1) / + NDB_SF_PAGE_ENTRIES; + resizeSchemaFile(xsf, noOfPages); + + c_writeSchemaRecord.inUse = true; + c_writeSchemaRecord.pageId = c_schemaRecord.oldSchemaPage; + c_writeSchemaRecord.newFile = true; + c_writeSchemaRecord.firstPage = 0; + c_writeSchemaRecord.noOfPages = xsf->noOfPages; + + c_writeSchemaRecord.m_callback.m_callbackFunction = + safe_cast(&Dbdict::initSchemaFile_conf); + + startWriteSchemaFile(signal); + } break; default : @@ -1035,6 +1112,54 @@ void Dbdict::closeReadSchemaConf(Signal* signal, }//switch }//Dbdict::closeReadSchemaConf() +bool +Dbdict::convertSchemaFileTo_5_0_6(XSchemaFile * xsf) +{ + const Uint32 pageSize_old = 32 * 1024; + Uint32 page_old[pageSize_old >> 2]; + SchemaFile * sf_old = (SchemaFile *)page_old; + + if (xsf->noOfPages * NDB_SF_PAGE_SIZE != pageSize_old) + return false; + SchemaFile * sf0 = &xsf->schemaPage[0]; + memcpy(sf_old, sf0, pageSize_old); + + // init max number new pages needed + xsf->noOfPages = (sf_old->NoOfTableEntries + NDB_SF_PAGE_ENTRIES - 1) / + NDB_SF_PAGE_ENTRIES; + initSchemaFile(xsf, 0, xsf->noOfPages, true); + + Uint32 noOfPages = 1; + Uint32 n, i, j; + for (n = 0; n < xsf->noOfPages; n++) { + jam(); + for (i = 0; i < NDB_SF_PAGE_ENTRIES; i++) { + j = n * NDB_SF_PAGE_ENTRIES + i; + if (j >= sf_old->NoOfTableEntries) + continue; + const SchemaFile::TableEntry_old & te_old = sf_old->TableEntries_old[j]; + if (te_old.m_tableState == SchemaFile::INIT || + te_old.m_tableState == SchemaFile::DROP_TABLE_COMMITTED || + te_old.m_noOfPages == 0) + continue; + SchemaFile * sf = &xsf->schemaPage[n]; + SchemaFile::TableEntry & te = sf->TableEntries[i]; + te.m_tableState = te_old.m_tableState; + te.m_tableVersion = te_old.m_tableVersion; + te.m_tableType = te_old.m_tableType; + te.m_info_words = te_old.m_noOfPages * ZSIZE_OF_PAGES_IN_WORDS - + ZPAGE_HEADER_SIZE; + te.m_gcp = te_old.m_gcp; + if (noOfPages < n) + noOfPages = n; + } + } + xsf->noOfPages = noOfPages; + initSchemaFile(xsf, 0, xsf->noOfPages, false); + + return true; +} + /* **************************************************************** */ /* ---------------------------------------------------------------- */ /* MODULE: INITIALISATION MODULE ------------------------- */ @@ -1306,6 +1431,7 @@ void Dbdict::initRetrieveRecord(Signal* signal, Uint32 i, Uint32 returnCode) void Dbdict::initSchemaRecord() { c_schemaRecord.schemaPage = RNIL; + c_schemaRecord.oldSchemaPage = RNIL; }//Dbdict::initSchemaRecord() void Dbdict::initRestartRecord() @@ -1327,10 +1453,10 @@ void Dbdict::initNodeRecords() void Dbdict::initPageRecords() { - c_schemaRecord.schemaPage = ZMAX_PAGES_OF_TABLE_DEFINITION; - c_schemaRecord.oldSchemaPage = ZMAX_PAGES_OF_TABLE_DEFINITION + 1; - c_retrieveRecord.retrievePage = ZMAX_PAGES_OF_TABLE_DEFINITION + 2; - ndbrequire(ZNUMBER_OF_PAGES >= (2 * ZMAX_PAGES_OF_TABLE_DEFINITION + 2)); + c_retrieveRecord.retrievePage = ZMAX_PAGES_OF_TABLE_DEFINITION; + ndbrequire(ZNUMBER_OF_PAGES >= (ZMAX_PAGES_OF_TABLE_DEFINITION + 1)); + c_schemaRecord.schemaPage = 0; + c_schemaRecord.oldSchemaPage = NDB_SF_MAX_PAGES; }//Dbdict::initPageRecords() void Dbdict::initTableRecords() @@ -1598,6 +1724,7 @@ void Dbdict::execREAD_CONFIG_REQ(Signal* signal) c_fsConnectRecordPool.setSize(ZFS_CONNECT_SIZE); c_nodes.setSize(MAX_NODES); c_pageRecordArray.setSize(ZNUMBER_OF_PAGES); + c_schemaPageRecordArray.setSize(2 * NDB_SF_MAX_PAGES); c_tableRecordPool.setSize(tablerecSize); c_tableRecordHash.setSize(tablerecSize); c_triggerRecordPool.setSize(c_maxNoOfTriggers); @@ -1616,12 +1743,23 @@ void Dbdict::execREAD_CONFIG_REQ(Signal* signal) c_opCreateTrigger.setSize(8); c_opDropTrigger.setSize(8); c_opAlterTrigger.setSize(8); + + // Initialize schema file copies + c_schemaFile[0].schemaPage = + (SchemaFile*)c_schemaPageRecordArray.getPtr(0 * NDB_SF_MAX_PAGES); + c_schemaFile[0].noOfPages = 0; + c_schemaFile[1].schemaPage = + (SchemaFile*)c_schemaPageRecordArray.getPtr(1 * NDB_SF_MAX_PAGES); + c_schemaFile[1].noOfPages = 0; // Initialize BAT for interface to file system - PageRecordPtr pageRecPtr; - c_pageRecordArray.getPtr(pageRecPtr, 0); NewVARIABLE* bat = allocateBat(2); - bat[1].WA = &pageRecPtr.p->word[0]; + bat[0].WA = &c_schemaPageRecordArray.getPtr(0)->word[0]; + bat[0].nrr = 2 * NDB_SF_MAX_PAGES; + bat[0].ClusterSize = NDB_SF_PAGE_SIZE; + bat[0].bits.q = NDB_SF_PAGE_SIZE_IN_WORDS_LOG2; + bat[0].bits.v = 5; // 32 bits per element + bat[1].WA = &c_pageRecordArray.getPtr(0)->word[0]; bat[1].nrr = ZNUMBER_OF_PAGES; bat[1].ClusterSize = ZSIZE_OF_PAGES_IN_WORDS * 4; bat[1].bits.q = ZLOG_SIZE_OF_PAGES_IN_WORDS; // 2**13 = 8192 elements @@ -1766,16 +1904,23 @@ void Dbdict::execHOT_SPAREREP(Signal* signal) void Dbdict::initSchemaFile(Signal* signal) { - PageRecordPtr pagePtr; - c_pageRecordArray.getPtr(pagePtr, c_schemaRecord.schemaPage); - SchemaFile * schemaFile = (SchemaFile *)pagePtr.p; - initSchemaFile(schemaFile, 4 * ZSIZE_OF_PAGES_IN_WORDS); + XSchemaFile * xsf = &c_schemaFile[c_schemaRecord.schemaPage != 0]; + xsf->noOfPages = (c_tableRecordPool.getSize() + NDB_SF_PAGE_ENTRIES - 1) + / NDB_SF_PAGE_ENTRIES; + initSchemaFile(xsf, 0, xsf->noOfPages, true); + // init alt copy too for INR + XSchemaFile * oldxsf = &c_schemaFile[c_schemaRecord.oldSchemaPage != 0]; + oldxsf->noOfPages = xsf->noOfPages; + memcpy(&oldxsf->schemaPage[0], &xsf->schemaPage[0], xsf->schemaPage[0].FileSize); if (c_initialStart || c_initialNodeRestart) { jam(); ndbrequire(c_writeSchemaRecord.inUse == false); c_writeSchemaRecord.inUse = true; c_writeSchemaRecord.pageId = c_schemaRecord.schemaPage; + c_writeSchemaRecord.newFile = true; + c_writeSchemaRecord.firstPage = 0; + c_writeSchemaRecord.noOfPages = xsf->noOfPages; c_writeSchemaRecord.m_callback.m_callbackFunction = safe_cast(&Dbdict::initSchemaFile_conf); @@ -1785,7 +1930,9 @@ void Dbdict::initSchemaFile(Signal* signal) jam(); ndbrequire(c_readSchemaRecord.schemaReadState == ReadSchemaRecord::IDLE); c_readSchemaRecord.pageId = c_schemaRecord.oldSchemaPage; - c_readSchemaRecord.schemaReadState = ReadSchemaRecord::INITIAL_READ; + c_readSchemaRecord.firstPage = 0; + c_readSchemaRecord.noOfPages = 1; + c_readSchemaRecord.schemaReadState = ReadSchemaRecord::INITIAL_READ_HEAD; startReadSchemaFile(signal); } else { ndbrequire(false); @@ -1924,7 +2071,7 @@ void Dbdict::execDICTSTARTREQ(Signal* signal) safe_cast(&Dbdict::masterRestart_checkSchemaStatusComplete); c_restartRecord.activeTable = 0; - c_schemaRecord.schemaPage = c_schemaRecord.oldSchemaPage; + c_schemaRecord.schemaPage = c_schemaRecord.oldSchemaPage; // ugly checkSchemaStatus(signal); }//execDICTSTARTREQ() @@ -1933,15 +2080,13 @@ Dbdict::masterRestart_checkSchemaStatusComplete(Signal* signal, Uint32 callbackData, Uint32 returnCode){ - c_schemaRecord.schemaPage = ZMAX_PAGES_OF_TABLE_DEFINITION; + c_schemaRecord.schemaPage = 0; // ugly + XSchemaFile * oldxsf = &c_schemaFile[c_schemaRecord.oldSchemaPage != 0]; + ndbrequire(oldxsf->noOfPages != 0); LinearSectionPtr ptr[3]; - - PageRecordPtr pagePtr; - c_pageRecordArray.getPtr(pagePtr, c_schemaRecord.oldSchemaPage); - - ptr[0].p = &pagePtr.p->word[0]; - ptr[0].sz = ZSIZE_OF_PAGES_IN_WORDS; + ptr[0].p = (Uint32*)&oldxsf->schemaPage[0]; + ptr[0].sz = oldxsf->noOfPages * NDB_SF_PAGE_SIZE_IN_WORDS; c_sendSchemaRecord.m_SCHEMAINFO_Counter = c_aliveNodes; NodeReceiverGroup rg(DBDICT, c_aliveNodes); @@ -1957,10 +2102,10 @@ Dbdict::masterRestart_checkSchemaStatusComplete(Signal* signal, 1, c); - PageRecordPtr newPagePtr; - c_pageRecordArray.getPtr(newPagePtr, c_schemaRecord.schemaPage); - memcpy(&newPagePtr.p->word[0], &pagePtr.p->word[0], - 4 * ZSIZE_OF_PAGES_IN_WORDS); + XSchemaFile * newxsf = &c_schemaFile[c_schemaRecord.schemaPage != 0]; + newxsf->noOfPages = oldxsf->noOfPages; + memcpy(&newxsf->schemaPage[0], &oldxsf->schemaPage[0], + oldxsf->noOfPages * NDB_SF_PAGE_SIZE); signal->theData[0] = getOwnNodeId(); sendSignal(reference(), GSN_SCHEMA_INFOCONF, signal, 1, JBB); @@ -1977,11 +2122,11 @@ Dbdict::execGET_SCHEMA_INFOREQ(Signal* signal){ LinearSectionPtr ptr[3]; - PageRecordPtr pagePtr; - c_pageRecordArray.getPtr(pagePtr, c_schemaRecord.schemaPage); + XSchemaFile * xsf = &c_schemaFile[c_schemaRecord.schemaPage != 0]; + ndbrequire(xsf->noOfPages != 0); - ptr[0].p = &pagePtr.p->word[0]; - ptr[0].sz = ZSIZE_OF_PAGES_IN_WORDS; + ptr[0].p = (Uint32*)&xsf->schemaPage[0]; + ptr[0].sz = xsf->noOfPages * NDB_SF_PAGE_SIZE_IN_WORDS; Callback c = { safe_cast(&Dbdict::sendSchemaComplete), 0 }; sendFragmentedSignal(ref, @@ -2023,12 +2168,22 @@ void Dbdict::execSCHEMA_INFO(Signal* signal) SegmentedSectionPtr schemaDataPtr; signal->getSection(schemaDataPtr, 0); - PageRecordPtr pagePtr; - c_pageRecordArray.getPtr(pagePtr, c_schemaRecord.schemaPage); - copy(&pagePtr.p->word[0], schemaDataPtr); + XSchemaFile * xsf = &c_schemaFile[c_schemaRecord.schemaPage != 0]; + ndbrequire(schemaDataPtr.sz % NDB_SF_PAGE_SIZE_IN_WORDS == 0); + xsf->noOfPages = schemaDataPtr.sz / NDB_SF_PAGE_SIZE_IN_WORDS; + copy((Uint32*)&xsf->schemaPage[0], schemaDataPtr); releaseSections(signal); + + SchemaFile * sf0 = &xsf->schemaPage[0]; + if (sf0->NdbVersion < NDB_SF_VERSION_5_0_6) { + bool ok = convertSchemaFileTo_5_0_6(xsf); + ndbrequire(ok); + } - validateChecksum((SchemaFile*)pagePtr.p); + validateChecksum(xsf); + + XSchemaFile * oldxsf = &c_schemaFile[c_schemaRecord.oldSchemaPage != 0]; + resizeSchemaFile(xsf, oldxsf->noOfPages); ndbrequire(signal->getSendersBlockRef() != reference()); @@ -2053,7 +2208,11 @@ Dbdict::restart_checkSchemaStatusComplete(Signal * signal, ndbrequire(c_writeSchemaRecord.inUse == false); c_writeSchemaRecord.inUse = true; + XSchemaFile * xsf = &c_schemaFile[c_schemaRecord.schemaPage != 0]; c_writeSchemaRecord.pageId = c_schemaRecord.schemaPage; + c_writeSchemaRecord.newFile = true; + c_writeSchemaRecord.firstPage = 0; + c_writeSchemaRecord.noOfPages = xsf->noOfPages; c_writeSchemaRecord.m_callback.m_callbackData = 0; c_writeSchemaRecord.m_callback.m_callbackFunction = safe_cast(&Dbdict::restart_writeSchemaConf); @@ -2102,20 +2261,18 @@ void Dbdict::execSCHEMA_INFOCONF(Signal* signal) void Dbdict::checkSchemaStatus(Signal* signal) { - PageRecordPtr pagePtr; - c_pageRecordArray.getPtr(pagePtr, c_schemaRecord.schemaPage); - - PageRecordPtr oldPagePtr; - c_pageRecordArray.getPtr(oldPagePtr, c_schemaRecord.oldSchemaPage); + XSchemaFile * newxsf = &c_schemaFile[c_schemaRecord.schemaPage != 0]; + XSchemaFile * oldxsf = &c_schemaFile[c_schemaRecord.oldSchemaPage != 0]; + ndbrequire(newxsf->noOfPages == oldxsf->noOfPages); + const Uint32 noOfEntries = newxsf->noOfPages * NDB_SF_PAGE_ENTRIES; - for (; c_restartRecord.activeTable < MAX_TABLES; + for (; c_restartRecord.activeTable < noOfEntries; c_restartRecord.activeTable++) { jam(); Uint32 tableId = c_restartRecord.activeTable; - SchemaFile::TableEntry *newEntry = getTableEntry(pagePtr.p, tableId); - SchemaFile::TableEntry *oldEntry = getTableEntry(oldPagePtr.p, tableId, - true); + SchemaFile::TableEntry *newEntry = getTableEntry(newxsf, tableId); + SchemaFile::TableEntry *oldEntry = getTableEntry(oldxsf, tableId); SchemaFile::TableState schemaState = (SchemaFile::TableState)newEntry->m_tableState; SchemaFile::TableState oldSchemaState = @@ -2349,7 +2506,8 @@ Dbdict::restartCreateTab(Signal* signal, Uint32 tableId, if(file && !ERROR_INSERTED(6002)){ jam(); - c_readTableRecord.noOfPages = te->m_noOfPages; + c_readTableRecord.noOfPages = + DIV(te->m_info_words + ZPAGE_HEADER_SIZE, ZSIZE_OF_PAGES_IN_WORDS); c_readTableRecord.pageId = 0; c_readTableRecord.m_callback.m_callbackData = createTabPtr.p->key; c_readTableRecord.m_callback.m_callbackFunction = @@ -3137,8 +3295,8 @@ Dbdict::execALTER_TAB_REQ(Signal * signal) tabEntry.m_tableType = tablePtr.p->tableType; tabEntry.m_tableState = SchemaFile::ALTER_TABLE_COMMITTED; tabEntry.m_gcp = gci; - tabEntry.m_noOfPages = - DIV(tabInfoPtr.sz + ZPAGE_HEADER_SIZE, ZSIZE_OF_PAGES_IN_WORDS); + tabEntry.m_info_words = tabInfoPtr.sz; + memset(tabEntry.m_unused, 0, sizeof(tabEntry.m_unused)); Callback callback; callback.m_callbackData = senderData; @@ -3652,9 +3810,8 @@ Dbdict::execCREATE_FRAGMENTATION_CONF(Signal* signal){ /** * Update table version */ - PageRecordPtr pagePtr; - c_pageRecordArray.getPtr(pagePtr, c_schemaRecord.schemaPage); - SchemaFile::TableEntry * tabEntry = getTableEntry(pagePtr.p, tabPtr.i); + XSchemaFile * xsf = &c_schemaFile[c_schemaRecord.schemaPage != 0]; + SchemaFile::TableEntry * tabEntry = getTableEntry(xsf, tabPtr.i); tabPtr.p->tableVersion = tabEntry->m_tableVersion + 1; @@ -3960,8 +4117,8 @@ Dbdict::createTab_prepare(Signal* signal, CreateTabReq * req){ tabEntry.m_tableType = tabPtr.p->tableType; tabEntry.m_tableState = SchemaFile::ADD_STARTED; tabEntry.m_gcp = gci; - tabEntry.m_noOfPages = - DIV(tabInfoPtr.sz + ZPAGE_HEADER_SIZE, ZSIZE_OF_PAGES_IN_WORDS); + tabEntry.m_info_words = tabInfoPtr.sz; + memset(tabEntry.m_unused, 0, sizeof(tabEntry.m_unused)); Callback callback; callback.m_callbackData = createTabPtr.p->key; @@ -4418,8 +4575,8 @@ Dbdict::createTab_commit(Signal * signal, CreateTabReq * req){ tabEntry.m_tableType = tabPtr.p->tableType; tabEntry.m_tableState = SchemaFile::TABLE_ADD_COMMITTED; tabEntry.m_gcp = tabPtr.p->gciTableCreated; - tabEntry.m_noOfPages = - DIV(tabPtr.p->packedSize + ZPAGE_HEADER_SIZE, ZSIZE_OF_PAGES_IN_WORDS); + tabEntry.m_info_words = tabPtr.p->packedSize; + memset(tabEntry.m_unused, 0, sizeof(tabEntry.m_unused)); Callback callback; callback.m_callbackData = createTabPtr.p->key; @@ -4520,10 +4677,9 @@ Dbdict::createTab_dropComplete(Signal* signal, c_tableRecordPool.getPtr(tabPtr, createTabPtr.p->m_tablePtrI); releaseTableObject(tabPtr.i); - PageRecordPtr pagePtr; - c_pageRecordArray.getPtr(pagePtr, c_schemaRecord.schemaPage); - SchemaFile::TableEntry * tableEntry = getTableEntry(pagePtr.p, tabPtr.i); + XSchemaFile * xsf = &c_schemaFile[c_schemaRecord.schemaPage != 0]; + SchemaFile::TableEntry * tableEntry = getTableEntry(xsf, tabPtr.i); tableEntry->m_tableState = SchemaFile::DROP_TABLE_COMMITTED; //@todo check error @@ -5355,21 +5511,22 @@ Dbdict::execPREP_DROP_TAB_REQ(Signal* signal){ /** * Modify schema */ - PageRecordPtr pagePtr; - c_pageRecordArray.getPtr(pagePtr, c_schemaRecord.schemaPage); - - SchemaFile::TableEntry * tableEntry = getTableEntry(pagePtr.p, tablePtr.i); + XSchemaFile * xsf = &c_schemaFile[c_schemaRecord.schemaPage != 0]; + SchemaFile::TableEntry * tableEntry = getTableEntry(xsf, tablePtr.i); SchemaFile::TableState tabState = (SchemaFile::TableState)tableEntry->m_tableState; ndbrequire(tabState == SchemaFile::TABLE_ADD_COMMITTED || tabState == SchemaFile::ALTER_TABLE_COMMITTED); tableEntry->m_tableState = SchemaFile::DROP_TABLE_STARTED; - computeChecksum((SchemaFile*)pagePtr.p); + computeChecksum(xsf, tablePtr.i / NDB_SF_PAGE_ENTRIES); ndbrequire(c_writeSchemaRecord.inUse == false); c_writeSchemaRecord.inUse = true; c_writeSchemaRecord.pageId = c_schemaRecord.schemaPage; + c_writeSchemaRecord.newFile = false; + c_writeSchemaRecord.firstPage = tablePtr.i / NDB_SF_PAGE_ENTRIES; + c_writeSchemaRecord.noOfPages = 1; c_writeSchemaRecord.m_callback.m_callbackData = dropTabPtr.p->key; c_writeSchemaRecord.m_callback.m_callbackFunction = safe_cast(&Dbdict::prepDropTab_writeSchemaConf); @@ -5530,20 +5687,20 @@ Dbdict::dropTab_complete(Signal* signal, /** * Write to schema file */ - PageRecordPtr pagePtr; - c_pageRecordArray.getPtr(pagePtr, c_schemaRecord.schemaPage); - - SchemaFile::TableEntry * tableEntry = getTableEntry(pagePtr.p, tableId); + XSchemaFile * xsf = &c_schemaFile[c_schemaRecord.schemaPage != 0]; + SchemaFile::TableEntry * tableEntry = getTableEntry(xsf, tableId); SchemaFile::TableState tabState = (SchemaFile::TableState)tableEntry->m_tableState; ndbrequire(tabState == SchemaFile::DROP_TABLE_STARTED); tableEntry->m_tableState = SchemaFile::DROP_TABLE_COMMITTED; - computeChecksum((SchemaFile*)pagePtr.p); + computeChecksum(xsf, tableId / NDB_SF_PAGE_ENTRIES); ndbrequire(c_writeSchemaRecord.inUse == false); c_writeSchemaRecord.inUse = true; c_writeSchemaRecord.pageId = c_schemaRecord.schemaPage; + c_writeSchemaRecord.firstPage = tableId / NDB_SF_PAGE_ENTRIES; + c_writeSchemaRecord.noOfPages = 1; c_writeSchemaRecord.m_callback.m_callbackData = dropTabPtr.p->key; c_writeSchemaRecord.m_callback.m_callbackFunction = safe_cast(&Dbdict::dropTab_writeSchemaConf); @@ -11755,36 +11912,75 @@ Dbdict::getIndexAttrMask(TableRecordPtr indexPtr, AttributeMask& mask) /* **************************************************************** */ void -Dbdict::initSchemaFile(SchemaFile * sf, Uint32 fileSz){ - memcpy(sf->Magic, "NDBSCHMA", sizeof(sf->Magic)); - sf->ByteOrder = 0x12345678; - sf->NdbVersion = NDB_VERSION; - sf->FileSize = fileSz; - sf->CheckSum = 0; - - Uint32 headSz = (sizeof(SchemaFile)-sizeof(SchemaFile::TableEntry)); - Uint32 noEntries = (fileSz - headSz) / sizeof(SchemaFile::TableEntry); - Uint32 slack = (fileSz - headSz) - noEntries * sizeof(SchemaFile::TableEntry); - - ndbrequire(noEntries > MAX_TABLES); +Dbdict::initSchemaFile(XSchemaFile * xsf, Uint32 firstPage, Uint32 lastPage, + bool initEntries) +{ + ndbrequire(lastPage <= xsf->noOfPages); + for (Uint32 n = firstPage; n < lastPage; n++) { + SchemaFile * sf = &xsf->schemaPage[n]; + if (initEntries) + memset(sf, 0, NDB_SF_PAGE_SIZE); + + Uint32 ndb_version = NDB_VERSION; + if (ndb_version < NDB_SF_VERSION_5_0_6) + ndb_version = NDB_SF_VERSION_5_0_6; + + memcpy(sf->Magic, NDB_SF_MAGIC, sizeof(sf->Magic)); + sf->ByteOrder = 0x12345678; + sf->NdbVersion = ndb_version; + sf->FileSize = xsf->noOfPages * NDB_SF_PAGE_SIZE; + sf->PageNumber = n; + sf->CheckSum = 0; + sf->NoOfTableEntries = NDB_SF_PAGE_ENTRIES; - sf->NoOfTableEntries = noEntries; - memset(sf->TableEntries, 0, noEntries*sizeof(SchemaFile::TableEntry)); - memset(&(sf->TableEntries[noEntries]), 0, slack); - computeChecksum(sf); + computeChecksum(xsf, n); + } +} + +void +Dbdict::resizeSchemaFile(XSchemaFile * xsf, Uint32 noOfPages) +{ + ndbrequire(noOfPages <= NDB_SF_MAX_PAGES); + if (xsf->noOfPages < noOfPages) { + jam(); + Uint32 firstPage = xsf->noOfPages; + xsf->noOfPages = noOfPages; + initSchemaFile(xsf, 0, firstPage, false); + initSchemaFile(xsf, firstPage, xsf->noOfPages, true); + } + if (xsf->noOfPages > noOfPages) { + jam(); + Uint32 tableId = noOfPages * NDB_SF_PAGE_ENTRIES; + while (tableId < xsf->noOfPages * NDB_SF_PAGE_ENTRIES) { + SchemaFile::TableEntry * te = getTableEntry(xsf, tableId); + if (te->m_tableState != SchemaFile::INIT && + te->m_tableState != SchemaFile::DROP_TABLE_COMMITTED) { + ndbrequire(false); + } + tableId++; + } + xsf->noOfPages = noOfPages; + initSchemaFile(xsf, 0, xsf->noOfPages, false); + } } void -Dbdict::computeChecksum(SchemaFile * sf){ +Dbdict::computeChecksum(XSchemaFile * xsf, Uint32 pageNo){ + SchemaFile * sf = &xsf->schemaPage[pageNo]; sf->CheckSum = 0; - sf->CheckSum = computeChecksum((const Uint32*)sf, sf->FileSize/4); + sf->CheckSum = computeChecksum((Uint32*)sf, NDB_SF_PAGE_SIZE_IN_WORDS); } bool -Dbdict::validateChecksum(const SchemaFile * sf){ +Dbdict::validateChecksum(const XSchemaFile * xsf){ - Uint32 c = computeChecksum((const Uint32*)sf, sf->FileSize/4); - return c == 0; + for (Uint32 n = 0; n < xsf->noOfPages; n++) { + SchemaFile * sf = &xsf->schemaPage[n]; + Uint32 c = computeChecksum((Uint32*)sf, NDB_SF_PAGE_SIZE_IN_WORDS); + if ( c != 0) + return false; + } + return true; } Uint32 @@ -11796,11 +11992,14 @@ Dbdict::computeChecksum(const Uint32 * src, Uint32 len){ } SchemaFile::TableEntry * -Dbdict::getTableEntry(void * p, Uint32 tableId, bool allowTooBig){ - SchemaFile * sf = (SchemaFile*)p; - - ndbrequire(allowTooBig || tableId < sf->NoOfTableEntries); - return &sf->TableEntries[tableId]; +Dbdict::getTableEntry(XSchemaFile * xsf, Uint32 tableId) +{ + Uint32 n = tableId / NDB_SF_PAGE_ENTRIES; + Uint32 i = tableId % NDB_SF_PAGE_ENTRIES; + ndbrequire(n < xsf->noOfPages); + + SchemaFile * sf = &xsf->schemaPage[n]; + return &sf->TableEntries[i]; } // global metadata support diff --git a/ndb/src/kernel/blocks/dbdict/Dbdict.hpp b/ndb/src/kernel/blocks/dbdict/Dbdict.hpp index 73fbdcc8e16..68bb9b628d4 100644 --- a/ndb/src/kernel/blocks/dbdict/Dbdict.hpp +++ b/ndb/src/kernel/blocks/dbdict/Dbdict.hpp @@ -78,7 +78,8 @@ /*--------------------------------------------------------------*/ // Page constants /*--------------------------------------------------------------*/ -#define ZALLOCATE 1 //Variable number of page for NDBFS +#define ZBAT_SCHEMA_FILE 0 //Variable number of page for NDBFS +#define ZBAT_TABLE_FILE 1 //Variable number of page for NDBFS #define ZPAGE_HEADER_SIZE 32 #define ZPOS_PAGE_SIZE 16 #define ZPOS_CHECKSUM 17 @@ -92,7 +93,7 @@ #define ZSIZE_OF_PAGES_IN_WORDS 8192 #define ZLOG_SIZE_OF_PAGES_IN_WORDS 13 #define ZMAX_PAGES_OF_TABLE_DEFINITION 8 -#define ZNUMBER_OF_PAGES (2 * ZMAX_PAGES_OF_TABLE_DEFINITION + 2) +#define ZNUMBER_OF_PAGES (ZMAX_PAGES_OF_TABLE_DEFINITION + 1) #define ZNO_OF_FRAGRECORD 5 /*--------------------------------------------------------------*/ @@ -429,6 +430,12 @@ public: typedef Ptr<PageRecord> PageRecordPtr; CArray<PageRecord> c_pageRecordArray; + struct SchemaPageRecord { + Uint32 word[NDB_SF_PAGE_SIZE_IN_WORDS]; + }; + + CArray<SchemaPageRecord> c_schemaPageRecordArray; + /** * A page for create index table signal. */ @@ -655,16 +662,20 @@ private: struct ReadSchemaRecord { /** Page Id of schema page */ Uint32 pageId; + /** First page to read */ + Uint32 firstPage; + /** Number of pages to read */ + Uint32 noOfPages; /** State, indicates from where it was called */ enum SchemaReadState { IDLE = 0, - INITIAL_READ = 1 + INITIAL_READ_HEAD = 1, + INITIAL_READ = 2 }; SchemaReadState schemaReadState; }; ReadSchemaRecord c_readSchemaRecord; -private: /** * This record stores all the state needed * when a schema file is being written to disk @@ -672,6 +683,12 @@ private: struct WriteSchemaRecord { /** Page Id of schema page */ Uint32 pageId; + /** Rewrite entire file */ + Uint32 newFile; + /** First page to write */ + Uint32 firstPage; + /** Number of pages to write */ + Uint32 noOfPages; /** Schema Files Handled, local state variable */ Uint32 noOfSchemaFilesHandled; @@ -752,21 +769,33 @@ private: * Word 4: Currently zero ****************************************************************************/ struct SchemaRecord { - /** Schema page */ + /** Schema file first page (0) */ Uint32 schemaPage; - /** Old Schema page (used at node restart) */ + /** Old Schema file first page (used at node restart) */ Uint32 oldSchemaPage; Callback m_callback; }; SchemaRecord c_schemaRecord; - void initSchemaFile(SchemaFile *, Uint32 sz); - void computeChecksum(SchemaFile *); - bool validateChecksum(const SchemaFile *); - SchemaFile::TableEntry * getTableEntry(void * buf, Uint32 tableId, - bool allowTooBig = false); + /* + * Schema file, list of schema pages. Use an array until a pool + * exists and NDBFS interface can use it. + */ + struct XSchemaFile { + SchemaFile* schemaPage; + Uint32 noOfPages; + }; + // 0-normal 1-old + XSchemaFile c_schemaFile[2]; + + void initSchemaFile(XSchemaFile *, Uint32 firstPage, Uint32 lastPage, + bool initEntries); + void resizeSchemaFile(XSchemaFile * xsf, Uint32 noOfPages); + void computeChecksum(XSchemaFile *, Uint32 pageNo); + bool validateChecksum(const XSchemaFile *); + SchemaFile::TableEntry * getTableEntry(XSchemaFile *, Uint32 tableId); Uint32 computeChecksum(const Uint32 * src, Uint32 len); @@ -1631,7 +1660,8 @@ private: void openSchemaFile(Signal* signal, Uint32 fileNo, Uint32 fsPtr, - bool writeFlag); + bool writeFlag, + bool newFile); void writeSchemaFile(Signal* signal, Uint32 filePtr, Uint32 fsPtr); void writeSchemaConf(Signal* signal, FsConnectRecordPtr fsPtr); @@ -1673,6 +1703,7 @@ private: void readSchemaRef(Signal* signal, FsConnectRecordPtr fsPtr); void closeReadSchemaConf(Signal* signal, FsConnectRecordPtr fsPtr); + bool convertSchemaFileTo_5_0_6(XSchemaFile*); /* ------------------------------------------------------------ */ // Get table definitions diff --git a/ndb/src/kernel/blocks/dbdict/SchemaFile.hpp b/ndb/src/kernel/blocks/dbdict/SchemaFile.hpp index 7c3223d3d14..0226991a073 100644 --- a/ndb/src/kernel/blocks/dbdict/SchemaFile.hpp +++ b/ndb/src/kernel/blocks/dbdict/SchemaFile.hpp @@ -18,16 +18,35 @@ #define DBDICT_SCHEMA_FILE_HPP #include <ndb_types.h> +#include <ndb_version.h> #include <string.h> +#define NDB_SF_MAGIC "NDBSCHMA" + +// page size 4k +#define NDB_SF_PAGE_SIZE_IN_WORDS_LOG2 10 +#define NDB_SF_PAGE_SIZE_IN_WORDS (1 << NDB_SF_PAGE_SIZE_IN_WORDS_LOG2) +#define NDB_SF_PAGE_SIZE (NDB_SF_PAGE_SIZE_IN_WORDS << 2) + +// 4k = (1 + 127) * 32 +#define NDB_SF_PAGE_ENTRIES 127 + +// 160 pages = 20320 objects +#define NDB_SF_MAX_PAGES 160 + +// versions where format changed +#define NDB_SF_VERSION_5_0_6 MAKE_VERSION(5, 0, 6) + +// One page in schema file. struct SchemaFile { + // header size 32 bytes char Magic[8]; Uint32 ByteOrder; Uint32 NdbVersion; Uint32 FileSize; // In bytes - Uint32 Unused; - - Uint32 CheckSum; + Uint32 PageNumber; + Uint32 CheckSum; // Of this page + Uint32 NoOfTableEntries; // On this page (NDB_SF_PAGE_ENTRIES) enum TableState { INIT = 0, @@ -38,20 +57,33 @@ struct SchemaFile { ALTER_TABLE_COMMITTED = 5 }; + // entry size 32 bytes struct TableEntry { Uint32 m_tableState; Uint32 m_tableVersion; Uint32 m_tableType; - Uint32 m_noOfPages; + Uint32 m_info_words; Uint32 m_gcp; + Uint32 m_unused[3]; bool operator==(const TableEntry& o) const { return memcmp(this, &o, sizeof(* this))== 0; } }; + + // pre-5.0.6 + struct TableEntry_old { + Uint32 m_tableState; + Uint32 m_tableVersion; + Uint32 m_tableType; + Uint32 m_noOfPages; + Uint32 m_gcp; + }; - Uint32 NoOfTableEntries; - TableEntry TableEntries[1]; + union { + TableEntry TableEntries[NDB_SF_PAGE_ENTRIES]; + TableEntry_old TableEntries_old[1]; + }; }; #endif diff --git a/ndb/src/kernel/blocks/dbdict/printSchemaFile.cpp b/ndb/src/kernel/blocks/dbdict/printSchemaFile.cpp index 0ba52878b7c..b9b144cd977 100644 --- a/ndb/src/kernel/blocks/dbdict/printSchemaFile.cpp +++ b/ndb/src/kernel/blocks/dbdict/printSchemaFile.cpp @@ -1,7 +1,10 @@ #if 0 make -f Makefile -f - printSchemaFile <<'_eof_' -printSchemaFile: printSchemaFile.cpp +printSchemaFile: printSchemaFile.cpp SchemaFile.hpp $(CXXCOMPILE) -o $@ $@.cpp -L../../../common/util/.libs -lgeneral +ifneq ($(MYSQL_HOME),) + ln -sf `pwd`/$@ $(MYSQL_HOME)/bin/$@ +endif _eof_ exit $? #endif @@ -24,19 +27,28 @@ exit $? #include <ndb_global.h> +#include <ndb_version.h> #include <NdbMain.h> #include <NdbOut.hpp> #include <SchemaFile.hpp> -void -usage(const char * prg){ - ndbout << "Usage " << prg - << " P0.SchemaLog" << endl; +static const char* progname = 0; +static bool allflag = false; +static bool checkonly = false; +static int xitcode = 0; + +static void +usage() +{ + ndbout << "Usage " << progname + << " [-ac]" + << " P0.SchemaLog" << endl; } -void -fill(const char * buf, int mod){ +static void +fill(const char * buf, int mod) +{ int len = strlen(buf)+1; ndbout << buf << " "; while((len % mod) != 0){ @@ -45,19 +57,34 @@ fill(const char * buf, int mod){ } } -void -print(const char * filename, const SchemaFile * file){ - ndbout << "----- Schemafile: " << filename << " -----" << endl; - ndbout_c("Magic: %.*s ByteOrder: %.8x NdbVersion: %d FileSize: %d", - sizeof(file->Magic), file->Magic, - file->ByteOrder, - file->NdbVersion, - file->FileSize); - - for(Uint32 i = 0; i<file->NoOfTableEntries; i++){ - SchemaFile::TableEntry te = file->TableEntries[i]; - if(te.m_tableState != SchemaFile::INIT){ - ndbout << "Table " << i << ": State = " << te.m_tableState +static void +print_head(const char * filename, const SchemaFile * sf) +{ + if (! checkonly) { + ndbout << "----- Schemafile: " << filename << " -----" << endl; + ndbout_c("Magic: %.*s ByteOrder: %.8x NdbVersion: %d.%d.%d FileSize: %d", + sizeof(sf->Magic), + sf->Magic, + sf->ByteOrder, + sf->NdbVersion >> 16, + (sf->NdbVersion >> 8) & 0xFF, + sf->NdbVersion & 0xFF, + sf->FileSize); + } +} + +static void +print_old(const char * filename, const SchemaFile * sf) +{ + print_head(filename, sf); + + for (Uint32 i = 0; i < sf->NoOfTableEntries; i++) { + SchemaFile::TableEntry_old te = sf->TableEntries_old[i]; + if (allflag || + (te.m_tableState != SchemaFile::INIT && + te.m_tableState != SchemaFile::DROP_TABLE_COMMITTED)) { + ndbout << "Table " << i << ":" + << " State = " << te.m_tableState << " version = " << te.m_tableVersion << " type = " << te.m_tableType << " noOfPages = " << te.m_noOfPages @@ -66,47 +93,114 @@ print(const char * filename, const SchemaFile * file){ } } -NDB_COMMAND(printSchemafile, - "printSchemafile", "printSchemafile", "Prints a schemafile", 16384){ - if(argc < 2){ - usage(argv[0]); - return 0; - } +static void +print(const char * filename, const SchemaFile * xsf, Uint32 sz) +{ + int retcode = 0; - const char * filename = argv[1]; + print_head(filename, xsf); - struct stat sbuf; - const int res = stat(filename, &sbuf); - if(res != 0){ - ndbout << "Could not find file: \"" << filename << "\"" << endl; - return 0; + assert(sizeof(SchemaFile) == NDB_SF_PAGE_SIZE); + if (xsf->FileSize != sz || xsf->FileSize % NDB_SF_PAGE_SIZE != 0) { + ndbout << "***** invalid FileSize " << xsf->FileSize << endl; + retcode = 1; } - const Uint32 bytes = sbuf.st_size; - - Uint32 * buf = new Uint32[bytes/4+1]; - - FILE * f = fopen(filename, "rb"); - if(f == 0){ - ndbout << "Failed to open file" << endl; - delete [] buf; - return 0; + Uint32 noOfPages = xsf->FileSize / NDB_SF_PAGE_SIZE; + for (Uint32 n = 0; n < noOfPages; n++) { + if (! checkonly) { + ndbout << "----- Page: " << n << " (" << noOfPages << ") -----" << endl; + } + const SchemaFile * sf = &xsf[n]; + if (sf->FileSize != xsf->FileSize) { + ndbout << "***** page " << n << " FileSize changed to " << sf->FileSize << "!=" << xsf->FileSize << endl; + retcode = 1; + } + Uint32 cs = 0; + for (Uint32 j = 0; j < NDB_SF_PAGE_SIZE_IN_WORDS; j++) + cs ^= ((const Uint32*)sf)[j]; + if (cs != 0) { + ndbout << "***** page " << n << " invalid CheckSum" << endl; + retcode = 1; + } + if (sf->NoOfTableEntries != NDB_SF_PAGE_ENTRIES) { + ndbout << "***** page " << n << " invalid NoOfTableEntries " << sf->NoOfTableEntries << endl; + retcode = 1; + } + for (Uint32 i = 0; i < NDB_SF_PAGE_ENTRIES; i++) { + SchemaFile::TableEntry te = sf->TableEntries[i]; + Uint32 j = n * NDB_SF_PAGE_ENTRIES + i; + if (allflag || + (te.m_tableState != SchemaFile::INIT && + te.m_tableState != SchemaFile::DROP_TABLE_COMMITTED)) { + if (! checkonly) + ndbout << "Table " << j << ":" + << " State = " << te.m_tableState + << " version = " << te.m_tableVersion + << " type = " << te.m_tableType + << " noOfWords = " << te.m_info_words + << " gcp: " << te.m_gcp << endl; + } + if (te.m_unused[0] != 0 || te.m_unused[1] != 0 || te.m_unused[2] != 0) { + ndbout << "***** entry " << j << " garbage in m_unused[3]" << endl; + retcode = 1; + } + } } - Uint32 sz = fread(buf, 1, bytes, f); - fclose(f); - if(sz != bytes){ - ndbout << "Failure while reading file" << endl; - delete [] buf; - return 0; + + if (retcode != 0) + xitcode = 1; + else if (checkonly) + ndbout << "ok: " << filename << endl; +} + +NDB_COMMAND(printSchemafile, + "printSchemafile", "printSchemafile", "Prints a schemafile", 16384) +{ + progname = argv[0]; + + while (argv[1][0] == '-') { + if (strchr(argv[1], 'a') != 0) + allflag = true; + if (strchr(argv[1], 'c') != 0) + checkonly = true; + argc--, argv++; } - - print(filename, (SchemaFile *)&buf[0]); - Uint32 chk = 0, i; - for (i = 0; i < bytes/4; i++) - chk ^= buf[i]; - if (chk != 0) - ndbout << "Invalid checksum!" << endl; + while (argc > 1) { + const char * filename = argv[1]; + argc--, argv++; + + struct stat sbuf; + const int res = stat(filename, &sbuf); + if (res != 0) { + ndbout << "Could not find file: \"" << filename << "\"" << endl; + return 1; + } + const Uint32 bytes = sbuf.st_size; + + Uint32 * buf = new Uint32[bytes/4+1]; + + FILE * f = fopen(filename, "rb"); + if (f == 0) { + ndbout << "Failed to open file" << endl; + delete [] buf; + return 1; + } + Uint32 sz = fread(buf, 1, bytes, f); + fclose(f); + if (sz != bytes) { + ndbout << "Failure while reading file" << endl; + delete [] buf; + return 1; + } + + SchemaFile* sf = (SchemaFile *)&buf[0]; + if (sf->NdbVersion < NDB_SF_VERSION_5_0_6) + print_old(filename, sf); + else + print(filename, sf, sz); + delete [] buf; + } - delete [] buf; - return 0; + return xitcode; } diff --git a/ndb/src/kernel/blocks/dblqh/Dblqh.hpp b/ndb/src/kernel/blocks/dblqh/Dblqh.hpp index f5d2dbc0a6c..fa7e8667e27 100644 --- a/ndb/src/kernel/blocks/dblqh/Dblqh.hpp +++ b/ndb/src/kernel/blocks/dblqh/Dblqh.hpp @@ -565,6 +565,9 @@ public: NodeId scanNodeId; Uint16 scanReleaseCounter; Uint16 scanNumber; + + // scan source block ACC TUX TUP + BlockReference scanBlockref; Uint8 scanCompletedStatus; Uint8 scanFlag; @@ -573,6 +576,7 @@ public: Uint8 readCommitted; Uint8 rangeScan; Uint8 descending; + Uint8 tupScan; Uint8 scanTcWaiting; Uint8 scanKeyinfoFlag; Uint8 m_last_row; diff --git a/ndb/src/kernel/blocks/dblqh/DblqhMain.cpp b/ndb/src/kernel/blocks/dblqh/DblqhMain.cpp index 50ee4c4b06e..138a2526804 100644 --- a/ndb/src/kernel/blocks/dblqh/DblqhMain.cpp +++ b/ndb/src/kernel/blocks/dblqh/DblqhMain.cpp @@ -7176,10 +7176,7 @@ void Dblqh::continueScanReleaseAfterBlockedLab(Signal* signal) scanptr.p->scanReleaseCounter -1, false); signal->theData[2] = NextScanReq::ZSCAN_COMMIT; - if (! scanptr.p->rangeScan) - sendSignal(tcConnectptr.p->tcAccBlockref, GSN_NEXT_SCANREQ, signal, 3, JBB); - else - sendSignal(tcConnectptr.p->tcTuxBlockref, GSN_NEXT_SCANREQ, signal, 3, JBB); + sendSignal(scanptr.p->scanBlockref, GSN_NEXT_SCANREQ, signal, 3, JBB); }//Dblqh::continueScanReleaseAfterBlockedLab() /* ------------------------------------------------------------------------- @@ -7492,6 +7489,7 @@ void Dblqh::execSCAN_FRAGREQ(Signal* signal) const Uint32 scanLockMode = ScanFragReq::getLockMode(reqinfo); const Uint8 keyinfo = ScanFragReq::getKeyinfoFlag(reqinfo); const Uint8 rangeScan = ScanFragReq::getRangeScanFlag(reqinfo); + const Uint8 tupScan = ScanFragReq::getTupScanFlag(reqinfo); ptrCheckGuard(tabptr, ctabrecFileSize, tablerec); if(tabptr.p->tableStatus != Tablerec::TABLE_DEFINED){ @@ -7641,13 +7639,8 @@ void Dblqh::continueAfterReceivingAllAiLab(Signal* signal) req->transId1 = tcConnectptr.p->transid[0]; req->transId2 = tcConnectptr.p->transid[1]; req->savePointId = tcConnectptr.p->savePointId; - // always use if-stmt to switch (instead of setting a "scan block ref") - if (! scanptr.p->rangeScan) - sendSignal(tcConnectptr.p->tcAccBlockref, GSN_ACC_SCANREQ, signal, - AccScanReq::SignalLength, JBB); - else - sendSignal(tcConnectptr.p->tcTuxBlockref, GSN_ACC_SCANREQ, signal, - AccScanReq::SignalLength, JBB); + sendSignal(scanptr.p->scanBlockref, GSN_ACC_SCANREQ, signal, + AccScanReq::SignalLength, JBB); }//Dblqh::continueAfterReceivingAllAiLab() void Dblqh::scanAttrinfoLab(Signal* signal, Uint32* dataPtr, Uint32 length) @@ -8002,10 +7995,7 @@ void Dblqh::continueFirstScanAfterBlockedLab(Signal* signal) signal->theData[0] = scanptr.p->scanAccPtr; signal->theData[1] = RNIL; signal->theData[2] = NextScanReq::ZSCAN_NEXT; - if (! scanptr.p->rangeScan) - sendSignal(tcConnectptr.p->tcAccBlockref, GSN_NEXT_SCANREQ, signal, 3, JBB); - else - sendSignal(tcConnectptr.p->tcTuxBlockref, GSN_NEXT_SCANREQ, signal, 3, JBB); + sendSignal(scanptr.p->scanBlockref, GSN_NEXT_SCANREQ, signal, 3, JBB); return; }//Dblqh::continueFirstScanAfterBlockedLab() @@ -8075,10 +8065,8 @@ void Dblqh::continueAfterCheckLcpStopBlocked(Signal* signal) c_scanRecordPool.getPtr(scanptr); signal->theData[0] = scanptr.p->scanAccPtr; signal->theData[1] = AccCheckScan::ZNOT_CHECK_LCP_STOP; - if (! scanptr.p->rangeScan) - EXECUTE_DIRECT(DBACC, GSN_ACC_CHECK_SCAN, signal, 2); - else - EXECUTE_DIRECT(DBTUX, GSN_ACC_CHECK_SCAN, signal, 2); + EXECUTE_DIRECT(refToBlock(scanptr.p->scanBlockref), GSN_ACC_CHECK_SCAN, + signal, 2); }//Dblqh::continueAfterCheckLcpStopBlocked() /* ------------------------------------------------------------------------- @@ -8168,12 +8156,8 @@ void Dblqh::nextScanConfScanLab(Signal* signal) signal->theData[0] = scanptr.p->scanAccPtr; signal->theData[1] = AccCheckScan::ZCHECK_LCP_STOP; - if (! scanptr.p->rangeScan) - sendSignal(tcConnectptr.p->tcAccBlockref, - GSN_ACC_CHECK_SCAN, signal, 2, JBB); - else - sendSignal(tcConnectptr.p->tcTuxBlockref, - GSN_ACC_CHECK_SCAN, signal, 2, JBB); + sendSignal(scanptr.p->scanBlockref, + GSN_ACC_CHECK_SCAN, signal, 2, JBB); return; }//if jam(); @@ -8416,10 +8400,7 @@ void Dblqh::continueScanAfterBlockedLab(Signal* signal) signal->theData[0] = scanptr.p->scanAccPtr; signal->theData[1] = accOpPtr; signal->theData[2] = scanptr.p->scanFlag; - if (! scanptr.p->rangeScan) - sendSignal(tcConnectptr.p->tcAccBlockref, GSN_NEXT_SCANREQ, signal, 3,JBB); - else - sendSignal(tcConnectptr.p->tcTuxBlockref, GSN_NEXT_SCANREQ, signal, 3,JBB); + sendSignal(scanptr.p->scanBlockref, GSN_NEXT_SCANREQ, signal, 3, JBB); }//Dblqh::continueScanAfterBlockedLab() /* ------------------------------------------------------------------------- @@ -8541,10 +8522,7 @@ void Dblqh::continueCloseScanAfterBlockedLab(Signal* signal) signal->theData[0] = scanptr.p->scanAccPtr; signal->theData[1] = RNIL; signal->theData[2] = NextScanReq::ZSCAN_CLOSE; - if (! scanptr.p->rangeScan) - sendSignal(tcConnectptr.p->tcAccBlockref, GSN_NEXT_SCANREQ, signal, 3, JBB); - else - sendSignal(tcConnectptr.p->tcTuxBlockref, GSN_NEXT_SCANREQ, signal, 3, JBB); + sendSignal(scanptr.p->scanBlockref, GSN_NEXT_SCANREQ, signal, 3, JBB); }//Dblqh::continueCloseScanAfterBlockedLab() /* ------------------------------------------------------------------------- @@ -8628,8 +8606,9 @@ Uint32 Dblqh::initScanrec(const ScanFragReq* scanFragReq) const Uint32 scanLockHold = ScanFragReq::getHoldLockFlag(reqinfo); const Uint32 keyinfo = ScanFragReq::getKeyinfoFlag(reqinfo); const Uint32 readCommitted = ScanFragReq::getReadCommittedFlag(reqinfo); - const Uint32 idx = ScanFragReq::getRangeScanFlag(reqinfo); + const Uint32 rangeScan = ScanFragReq::getRangeScanFlag(reqinfo); const Uint32 descending = ScanFragReq::getDescendingFlag(reqinfo); + const Uint32 tupScan = ScanFragReq::getTupScanFlag(reqinfo); const Uint32 attrLen = ScanFragReq::getAttrLen(reqinfo); const Uint32 scanPrio = ScanFragReq::getScanPrio(reqinfo); @@ -8647,11 +8626,19 @@ Uint32 Dblqh::initScanrec(const ScanFragReq* scanFragReq) scanptr.p->m_max_batch_size_rows = max_rows; scanptr.p->m_max_batch_size_bytes = max_bytes; + if (! rangeScan && ! tupScan) + scanptr.p->scanBlockref = tcConnectptr.p->tcAccBlockref; + else if (! tupScan) + scanptr.p->scanBlockref = tcConnectptr.p->tcTuxBlockref; + else + scanptr.p->scanBlockref = tcConnectptr.p->tcTupBlockref; + scanptr.p->scanErrorCounter = 0; scanptr.p->scanLockMode = scanLockMode; scanptr.p->readCommitted = readCommitted; - scanptr.p->rangeScan = idx; + scanptr.p->rangeScan = rangeScan; scanptr.p->descending = descending; + scanptr.p->tupScan = tupScan; scanptr.p->scanState = ScanRecord::SCAN_FREE; scanptr.p->scanFlag = ZFALSE; scanptr.p->scanLocalref[0] = 0; @@ -8683,8 +8670,8 @@ Uint32 Dblqh::initScanrec(const ScanFragReq* scanFragReq) * !idx uses 1 - (MAX_PARALLEL_SCANS_PER_FRAG - 1) = 1-11 * idx uses from MAX_PARALLEL_SCANS_PER_FRAG - MAX = 12-42) */ - Uint32 start = (idx ? MAX_PARALLEL_SCANS_PER_FRAG : 1 ); - Uint32 stop = (idx ? MAX_PARALLEL_INDEX_SCANS_PER_FRAG : MAX_PARALLEL_SCANS_PER_FRAG - 1); + Uint32 start = (rangeScan || tupScan ? MAX_PARALLEL_SCANS_PER_FRAG : 1 ); + Uint32 stop = (rangeScan || tupScan ? MAX_PARALLEL_INDEX_SCANS_PER_FRAG : MAX_PARALLEL_SCANS_PER_FRAG - 1); stop += start; Uint32 free = tFragPtr.p->m_scanNumberMask.find(start); @@ -9111,6 +9098,7 @@ void Dblqh::execCOPY_FRAGREQ(Signal* signal) /* ------------------------------------------------------------------------- */ scanptr.p->m_max_batch_size_rows = 0; scanptr.p->rangeScan = 0; + scanptr.p->tupScan = 0; seizeTcrec(); /** @@ -9129,6 +9117,7 @@ void Dblqh::execCOPY_FRAGREQ(Signal* signal) scanptr.p->scanKeyinfoFlag = 0; // Don't put into hash scanptr.p->fragPtrI = fragptr.i; fragptr.p->m_scanNumberMask.clear(NR_ScanNo); + scanptr.p->scanBlockref = DBACC_REF; initScanTc(signal, 0, diff --git a/ndb/src/kernel/blocks/dbtc/DbtcMain.cpp b/ndb/src/kernel/blocks/dbtc/DbtcMain.cpp index 38d514047b8..16554955646 100644 --- a/ndb/src/kernel/blocks/dbtc/DbtcMain.cpp +++ b/ndb/src/kernel/blocks/dbtc/DbtcMain.cpp @@ -4687,8 +4687,9 @@ void Dbtc::sendApiCommit(Signal* signal) } commitConf->transId1 = regApiPtr->transid[0]; commitConf->transId2 = regApiPtr->transid[1]; - - sendSignal(regApiPtr->ndbapiBlockref, GSN_TC_COMMITCONF, signal, 3, JBB); + commitConf->gci = regApiPtr->globalcheckpointid; + sendSignal(regApiPtr->ndbapiBlockref, GSN_TC_COMMITCONF, signal, + TcCommitConf::SignalLength, JBB); } else if (regApiPtr->returnsignal == RS_NO_RETURN) { jam(); } else { @@ -5381,8 +5382,9 @@ void Dbtc::execTC_COMMITREQ(Signal* signal) commitConf->apiConnectPtr = apiConnectPtr; commitConf->transId1 = transId1; commitConf->transId2 = transId2; - - sendSignal(apiBlockRef, GSN_TC_COMMITCONF, signal, 3, JBB); + commitConf->gci = 0; + sendSignal(apiBlockRef, GSN_TC_COMMITCONF, signal, + TcCommitConf::SignalLength, JBB); regApiPtr->returnsignal = RS_NO_RETURN; releaseAbortResources(signal); @@ -8839,6 +8841,7 @@ void Dbtc::initScanrec(ScanRecordPtr scanptr, ScanFragReq::setReadCommittedFlag(tmp,ScanTabReq::getReadCommittedFlag(ri)); ScanFragReq::setRangeScanFlag(tmp, ScanTabReq::getRangeScanFlag(ri)); ScanFragReq::setDescendingFlag(tmp, ScanTabReq::getDescendingFlag(ri)); + ScanFragReq::setTupScanFlag(tmp, ScanTabReq::getTupScanFlag(ri)); ScanFragReq::setAttrLen(tmp, scanTabReq->attrLenKeyLen & 0xFFFF); scanptr.p->scanRequestInfo = tmp; diff --git a/ndb/src/kernel/blocks/dbtup/Dbtup.hpp b/ndb/src/kernel/blocks/dbtup/Dbtup.hpp index a0103f56add..f985e44d307 100644 --- a/ndb/src/kernel/blocks/dbtup/Dbtup.hpp +++ b/ndb/src/kernel/blocks/dbtup/Dbtup.hpp @@ -64,6 +64,7 @@ // DbtupSystemRestart.cpp 26000 // DbtupIndex.cpp 28000 // DbtupDebug.cpp 30000 +// DbtupScan.cpp 32000 //------------------------------------------------------------------ /* @@ -511,6 +512,49 @@ struct Fragoperrec { }; typedef Ptr<Fragoperrec> FragoperrecPtr; + // Position for use by scan + struct PagePos { + Uint32 m_fragId; // "base" fragment id + Uint32 m_fragBit; // two fragments in 5.0 + Uint32 m_pageId; + Uint32 m_tupleNo; + bool m_match; + }; + + // Tup scan op (compare Dbtux::ScanOp) + struct ScanOp { + enum { + Undef = 0, + First = 1, // before first entry + Locked = 4, // at current entry (no lock needed) + Next = 5, // looking for next extry + Last = 6, // after last entry + Invalid = 9 // cannot return REF to LQH currently + }; + Uint16 m_state; + Uint16 m_lockwait; // unused + Uint32 m_userPtr; // scanptr.i in LQH + Uint32 m_userRef; + Uint32 m_tableId; + Uint32 m_fragId; // "base" fragment id + Uint32 m_fragPtrI[2]; + Uint32 m_transId1; + Uint32 m_transId2; + PagePos m_scanPos; + union { + Uint32 nextPool; + Uint32 nextList; + }; + Uint32 prevList; + }; + typedef Ptr<ScanOp> ScanOpPtr; + ArrayPool<ScanOp> c_scanOpPool; + + void scanFirst(Signal* signal, ScanOpPtr scanPtr); + void scanNext(Signal* signal, ScanOpPtr scanPtr); + void scanClose(Signal* signal, ScanOpPtr scanPtr); + void releaseScanOp(ScanOpPtr& scanPtr); + struct Fragrecord { Uint32 nextStartRange; Uint32 currentPageRange; @@ -532,6 +576,9 @@ struct Fragrecord { Uint32 fragTableId; Uint32 fragmentId; Uint32 nextfreefrag; + + DLList<ScanOp> m_scanList; + Fragrecord(ArrayPool<ScanOp> & scanOpPool) : m_scanList(scanOpPool) {} }; typedef Ptr<Fragrecord> FragrecordPtr; @@ -1084,6 +1131,11 @@ private: void buildIndex(Signal* signal, Uint32 buildPtrI); void buildIndexReply(Signal* signal, const BuildIndexRec* buildRec); + // Tup scan + void execACC_SCANREQ(Signal* signal); + void execNEXT_SCANREQ(Signal* signal); + void execACC_CHECK_SCAN(Signal* signal); + //------------------------------------------------------------------ //------------------------------------------------------------------ // Methods to handle execution of TUPKEYREQ + ATTRINFO. diff --git a/ndb/src/kernel/blocks/dbtup/DbtupGen.cpp b/ndb/src/kernel/blocks/dbtup/DbtupGen.cpp index 0d7430e662d..03f02dd0b92 100644 --- a/ndb/src/kernel/blocks/dbtup/DbtupGen.cpp +++ b/ndb/src/kernel/blocks/dbtup/DbtupGen.cpp @@ -143,6 +143,11 @@ Dbtup::Dbtup(const class Configuration & conf) // Ordered index related addRecSignal(GSN_BUILDINDXREQ, &Dbtup::execBUILDINDXREQ); + // Tup scan + addRecSignal(GSN_ACC_SCANREQ, &Dbtup::execACC_SCANREQ); + addRecSignal(GSN_NEXT_SCANREQ, &Dbtup::execNEXT_SCANREQ); + addRecSignal(GSN_ACC_CHECK_SCAN, &Dbtup::execACC_CHECK_SCAN); + initData(); }//Dbtup::Dbtup() @@ -652,6 +657,10 @@ void Dbtup::execREAD_CONFIG_REQ(Signal* signal) c_buildIndexPool.setSize(c_noOfBuildIndexRec); c_triggerPool.setSize(noOfTriggers); + Uint32 nScanOp; // use TUX config for now + ndbrequire(!ndb_mgm_get_int_parameter(p, CFG_TUX_SCAN_OP, &nScanOp)); + c_scanOpPool.setSize(nScanOp); + initRecords(); czero = 0; cminusOne = czero - 1; @@ -672,6 +681,8 @@ void Dbtup::execREAD_CONFIG_REQ(Signal* signal) void Dbtup::initRecords() { + unsigned i; + // Records with dynamic sizes attrbufrec = (Attrbufrec*)allocRecord("Attrbufrec", sizeof(Attrbufrec), @@ -693,6 +704,11 @@ void Dbtup::initRecords() fragrecord = (Fragrecord*)allocRecord("Fragrecord", sizeof(Fragrecord), cnoOfFragrec); + + for (i = 0; i<cnoOfFragrec; i++) { + void * p = &fragrecord[i]; + new (p) Fragrecord(c_scanOpPool); + } hostBuffer = (HostBuffer*)allocRecord("HostBuffer", sizeof(HostBuffer), @@ -730,7 +746,7 @@ void Dbtup::initRecords() sizeof(Tablerec), cnoOfTablerec); - for(unsigned i = 0; i<cnoOfTablerec; i++) { + for (i = 0; i<cnoOfTablerec; i++) { void * p = &tablerec[i]; new (p) Tablerec(c_triggerPool); } diff --git a/ndb/src/kernel/blocks/dbtup/DbtupScan.cpp b/ndb/src/kernel/blocks/dbtup/DbtupScan.cpp new file mode 100644 index 00000000000..396404faa8c --- /dev/null +++ b/ndb/src/kernel/blocks/dbtup/DbtupScan.cpp @@ -0,0 +1,315 @@ +/* Copyright (C) 2003 MySQL AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +#define DBTUP_C +#include "Dbtup.hpp" +#include <signaldata/AccScan.hpp> +#include <signaldata/NextScan.hpp> + +#undef jam +#undef jamEntry +#define jam() { jamLine(32000 + __LINE__); } +#define jamEntry() { jamEntryLine(32000 + __LINE__); } + +void +Dbtup::execACC_SCANREQ(Signal* signal) +{ + jamEntry(); + const AccScanReq reqCopy = *(const AccScanReq*)signal->getDataPtr(); + const AccScanReq* const req = &reqCopy; + ScanOpPtr scanPtr; + scanPtr.i = RNIL; + do { + // find table and fragments + TablerecPtr tablePtr; + tablePtr.i = req->tableId; + ptrCheckGuard(tablePtr, cnoOfTablerec, tablerec); + FragrecordPtr fragPtr[2]; + Uint32 fragId = req->fragmentNo << 1; + fragPtr[0].i = fragPtr[1].i = RNIL; + getFragmentrec(fragPtr[0], fragId | 0, tablePtr.p); + getFragmentrec(fragPtr[1], fragId | 1, tablePtr.p); + ndbrequire(fragPtr[0].i != RNIL && fragPtr[1].i != RNIL); + Fragrecord& frag = *fragPtr[0].p; + // seize from pool and link to per-fragment list + if (! frag.m_scanList.seize(scanPtr)) { + jam(); + break; + } + new (scanPtr.p) ScanOp(); + ScanOp& scan = *scanPtr.p; + scan.m_state = ScanOp::First; + scan.m_userPtr = req->senderData; + scan.m_userRef = req->senderRef; + scan.m_tableId = tablePtr.i; + scan.m_fragId = frag.fragmentId; + scan.m_fragPtrI[0] = fragPtr[0].i; + scan.m_fragPtrI[1] = fragPtr[1].i; + scan.m_transId1 = req->transId1; + scan.m_transId2 = req->transId2; + // conf + AccScanConf* const conf = (AccScanConf*)signal->getDataPtrSend(); + conf->scanPtr = req->senderData; + conf->accPtr = scanPtr.i; + conf->flag = AccScanConf::ZNOT_EMPTY_FRAGMENT; + sendSignal(req->senderRef, GSN_ACC_SCANCONF, signal, + AccScanConf::SignalLength, JBB); + return; + } while (0); + if (scanPtr.i != RNIL) { + jam(); + releaseScanOp(scanPtr); + } + // LQH does not handle REF + signal->theData[0] = 0x313; + sendSignal(req->senderRef, GSN_ACC_SCANREF, signal, 1, JBB); +} + +void +Dbtup::execNEXT_SCANREQ(Signal* signal) +{ + jamEntry(); + const NextScanReq reqCopy = *(const NextScanReq*)signal->getDataPtr(); + const NextScanReq* const req = &reqCopy; + ScanOpPtr scanPtr; + c_scanOpPool.getPtr(scanPtr, req->accPtr); + ScanOp& scan = *scanPtr.p; + FragrecordPtr fragPtr; + fragPtr.i = scan.m_fragPtrI[0]; + ptrCheckGuard(fragPtr, cnoOfFragrec, fragrecord); + Fragrecord& frag = *fragPtr.p; + switch (req->scanFlag) { + case NextScanReq::ZSCAN_NEXT: + jam(); + break; + case NextScanReq::ZSCAN_NEXT_COMMIT: + jam(); + break; + case NextScanReq::ZSCAN_COMMIT: + jam(); + { + NextScanConf* const conf = (NextScanConf*)signal->getDataPtrSend(); + conf->scanPtr = scan.m_userPtr; + unsigned signalLength = 1; + sendSignal(scanPtr.p->m_userRef, GSN_NEXT_SCANCONF, + signal, signalLength, JBB); + return; + } + break; + case NextScanReq::ZSCAN_CLOSE: + jam(); + scanClose(signal, scanPtr); + return; + case NextScanReq::ZSCAN_NEXT_ABORT: + jam(); + default: + jam(); + ndbrequire(false); + break; + } + // start looking for next scan result + AccCheckScan* checkReq = (AccCheckScan*)signal->getDataPtrSend(); + checkReq->accPtr = scanPtr.i; + checkReq->checkLcpStop = AccCheckScan::ZNOT_CHECK_LCP_STOP; + EXECUTE_DIRECT(DBTUP, GSN_ACC_CHECK_SCAN, signal, AccCheckScan::SignalLength); + jamEntry(); +} + +void +Dbtup::execACC_CHECK_SCAN(Signal* signal) +{ + jamEntry(); + const AccCheckScan reqCopy = *(const AccCheckScan*)signal->getDataPtr(); + const AccCheckScan* const req = &reqCopy; + ScanOpPtr scanPtr; + c_scanOpPool.getPtr(scanPtr, req->accPtr); + ScanOp& scan = *scanPtr.p; + FragrecordPtr fragPtr; + fragPtr.i = scan.m_fragPtrI[0]; + ptrCheckGuard(fragPtr, cnoOfFragrec, fragrecord); + Fragrecord& frag = *fragPtr.p; + if (req->checkLcpStop == AccCheckScan::ZCHECK_LCP_STOP) { + jam(); + signal->theData[0] = scan.m_userPtr; + signal->theData[1] = true; + EXECUTE_DIRECT(DBLQH, GSN_CHECK_LCP_STOP, signal, 2); + jamEntry(); + return; + } + if (scan.m_state == ScanOp::First) { + jam(); + scanFirst(signal, scanPtr); + } + if (scan.m_state == ScanOp::Next) { + jam(); + scanNext(signal, scanPtr); + } + if (scan.m_state == ScanOp::Locked) { + jam(); + const PagePos& pos = scan.m_scanPos; + NextScanConf* const conf = (NextScanConf*)signal->getDataPtrSend(); + conf->scanPtr = scan.m_userPtr; + conf->accOperationPtr = (Uint32)-1; // no lock returned + conf->fragId = frag.fragmentId | pos.m_fragBit; + conf->localKey[0] = (pos.m_pageId << MAX_TUPLES_BITS) | + (pos.m_tupleNo << 1); + conf->localKey[1] = 0; + conf->localKeyLength = 1; + unsigned signalLength = 6; + Uint32 blockNo = refToBlock(scan.m_userRef); + EXECUTE_DIRECT(blockNo, GSN_NEXT_SCANCONF, signal, signalLength); + jamEntry(); + // next time look for next entry + scan.m_state = ScanOp::Next; + return; + } + if (scan.m_state == ScanOp::Last || + scan.m_state == ScanOp::Invalid) { + jam(); + NextScanConf* const conf = (NextScanConf*)signal->getDataPtrSend(); + conf->scanPtr = scan.m_userPtr; + conf->accOperationPtr = RNIL; + conf->fragId = RNIL; + unsigned signalLength = 3; + sendSignal(scanPtr.p->m_userRef, GSN_NEXT_SCANCONF, + signal, signalLength, JBB); + return; + } + ndbrequire(false); +} + +void +Dbtup::scanFirst(Signal* signal, ScanOpPtr scanPtr) +{ + ScanOp& scan = *scanPtr.p; + // set to first fragment, first page, first tuple + PagePos& pos = scan.m_scanPos; + pos.m_fragId = scan.m_fragId; + pos.m_fragBit = 0; + pos.m_pageId = 0; + pos.m_tupleNo = 0; + // just before + pos.m_match = false; + // let scanNext() do the work + scan.m_state = ScanOp::Next; +} + +// TODO optimize this + index build +void +Dbtup::scanNext(Signal* signal, ScanOpPtr scanPtr) +{ + ScanOp& scan = *scanPtr.p; + PagePos& pos = scan.m_scanPos; + TablerecPtr tablePtr; + tablePtr.i = scan.m_tableId; + ptrCheckGuard(tablePtr, cnoOfTablerec, tablerec); + while (true) { + // TODO time-slice here after X loops + jam(); + // get fragment + if (pos.m_fragBit == 2) { + jam(); + scan.m_state = ScanOp::Last; + break; + } + ndbrequire(pos.m_fragBit <= 1); + FragrecordPtr fragPtr; + fragPtr.i = scan.m_fragPtrI[pos.m_fragBit]; + ptrCheckGuard(fragPtr, cnoOfFragrec, fragrecord); + Fragrecord& frag = *fragPtr.p; + // get page + PagePtr pagePtr; + if (pos.m_pageId >= frag.noOfPages) { + jam(); + pos.m_fragBit++; + pos.m_pageId = 0; + pos.m_tupleNo = 0; + pos.m_match = false; + continue; + } + Uint32 realPageId = getRealpid(fragPtr.p, pos.m_pageId); + pagePtr.i = realPageId; + ptrCheckGuard(pagePtr, cnoOfPage, page); + const Uint32 pageState = pagePtr.p->pageWord[ZPAGE_STATE_POS]; + if (pageState != ZTH_MM_FREE && + pageState != ZTH_MM_FULL) { + jam(); + pos.m_pageId++; + pos.m_tupleNo = 0; + pos.m_match = false; + continue; + } + // get next tuple + if (pos.m_match) + pos.m_tupleNo++; + pos.m_match = true; + const Uint32 tupheadsize = tablePtr.p->tupheadsize; + Uint32 pageOffset = ZPAGE_HEADER_SIZE + pos.m_tupleNo * tupheadsize; + if (pageOffset + tupheadsize > ZWORDS_ON_PAGE) { + jam(); + pos.m_pageId++; + pos.m_tupleNo = 0; + pos.m_match = false; + continue; + } + // skip over free tuple + bool isFree = false; + if (pageState == ZTH_MM_FREE) { + jam(); + if ((pagePtr.p->pageWord[pageOffset] >> 16) == tupheadsize) { + Uint32 nextTuple = pagePtr.p->pageWord[ZFREELIST_HEADER_POS] >> 16; + while (nextTuple != 0) { + jam(); + if (nextTuple == pageOffset) { + jam(); + isFree = true; + break; + } + nextTuple = pagePtr.p->pageWord[nextTuple] & 0xffff; + } + } + } + if (isFree) { + jam(); + continue; + } + // TODO check for operation and return latest in own tx + scan.m_state = ScanOp::Locked; + break; + } +} + +void +Dbtup::scanClose(Signal* signal, ScanOpPtr scanPtr) +{ + NextScanConf* const conf = (NextScanConf*)signal->getDataPtrSend(); + conf->scanPtr = scanPtr.p->m_userPtr; + conf->accOperationPtr = RNIL; + conf->fragId = RNIL; + unsigned signalLength = 3; + sendSignal(scanPtr.p->m_userRef, GSN_NEXT_SCANCONF, + signal, signalLength, JBB); + releaseScanOp(scanPtr); +} + +void +Dbtup::releaseScanOp(ScanOpPtr& scanPtr) +{ + FragrecordPtr fragPtr; + fragPtr.i = scanPtr.p->m_fragPtrI[0]; + ptrCheckGuard(fragPtr, cnoOfFragrec, fragrecord); + fragPtr.p->m_scanList.release(scanPtr); +} diff --git a/ndb/src/kernel/blocks/dbtup/Makefile.am b/ndb/src/kernel/blocks/dbtup/Makefile.am index e51410e6be3..2d14ad41025 100644 --- a/ndb/src/kernel/blocks/dbtup/Makefile.am +++ b/ndb/src/kernel/blocks/dbtup/Makefile.am @@ -18,6 +18,7 @@ libdbtup_a_SOURCES = \ DbtupGen.cpp \ DbtupSystemRestart.cpp \ DbtupIndex.cpp \ + DbtupScan.cpp \ DbtupDebug.cpp include $(top_srcdir)/ndb/config/common.mk.am diff --git a/ndb/src/kernel/vm/Makefile.am b/ndb/src/kernel/vm/Makefile.am index 0dce9285ae3..d9e57ce9dd6 100644 --- a/ndb/src/kernel/vm/Makefile.am +++ b/ndb/src/kernel/vm/Makefile.am @@ -18,7 +18,8 @@ libkernel_a_SOURCES = \ SimplePropertiesSection.cpp \ SectionReader.cpp \ MetaData.cpp \ - Mutex.cpp SafeCounter.cpp + Mutex.cpp SafeCounter.cpp \ + SuperPool.cpp INCLUDES_LOC = -I$(top_srcdir)/ndb/src/mgmapi diff --git a/ndb/src/kernel/vm/SuperPool.cpp b/ndb/src/kernel/vm/SuperPool.cpp new file mode 100644 index 00000000000..65e5dd99629 --- /dev/null +++ b/ndb/src/kernel/vm/SuperPool.cpp @@ -0,0 +1,442 @@ +/* Copyright (C) 2003 MySQL AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +#include <ndb_global.h> +#include "SuperPool.hpp" + +// SuperPool + +SuperPool::SuperPool(Uint32 pageSize, Uint32 pageBits) : + m_pageSize(SP_ALIGN_SIZE(pageSize, SP_ALIGN)), + m_pageBits(pageBits), + m_memRoot(0), + m_pageEnt(0), + m_typeCheck(0), + m_typeSeq(0), + m_pageList(), + m_totalSize(0), + m_initSize(0), + m_incrSize(0), + m_maxSize(0) +{ + assert(5 <= pageBits <= 30); +} + +bool +SuperPool::init() +{ + return true; +} + +SuperPool::~SuperPool() +{ +} + +SuperPool::PageEnt::PageEnt() : + m_pageType(0), + m_freeRecI(RNIL), + m_useCount(0), + m_nextPageI(RNIL), + m_prevPageI(RNIL) +{ +} + +SuperPool::PageList::PageList() : + m_headPageI(RNIL), + m_tailPageI(RNIL), + m_pageCount(0) +{ +} + +SuperPool::PageList::PageList(PtrI pageI) : + m_headPageI(pageI), + m_tailPageI(pageI), + m_pageCount(1) +{ +} + +SuperPool::RecInfo::RecInfo(Uint32 recType, Uint32 recSize) : + m_recType(recType), + m_recSize(recSize), + m_maxUseCount(0), + m_currPageI(RNIL), + m_currFreeRecI(RNIL), + m_currUseCount(0), + m_totalUseCount(0), + m_totalRecCount(0), + m_freeList(), + m_activeList(), + m_fullList() +{ +} + +SuperPool::PtrI +SuperPool::getPageI(void* pageP) +{ + const Uint32 pageSize = m_pageSize; + const Uint32 pageBits = m_pageBits; + const Uint32 recBits = 32 - pageBits; + void* const memRoot = m_memRoot; + assert(pageP == SP_ALIGN_PTR(pageP, memRoot, pageSize)); + my_ptrdiff_t ipL = ((Uint8*)pageP - (Uint8*)memRoot) / pageSize; + Int32 ip = (Int32)ipL; + Int32 lim = 1 << (pageBits - 1); + assert(ip == ipL && -lim <= ip && ip < lim && ip != -1); + PtrI pageI = ip << recBits; + assert(pageP == getPageP(pageI)); + return pageI; +} + +void +SuperPool::movePages(PageList& pl1, PageList& pl2) +{ + const Uint32 recBits = 32 - m_pageBits; + if (pl1.m_pageCount != 0) { + if (pl2.m_pageCount != 0) { + PtrI pageI1 = pl1.m_tailPageI; + PtrI pageI2 = pl2.m_headPageI; + PageEnt& pe1 = getPageEnt(pageI1); + PageEnt& pe2 = getPageEnt(pageI2); + pe1.m_nextPageI = pageI2; + pe2.m_prevPageI = pageI1; + pl1.m_pageCount += pl2.m_pageCount; + } + } else { + pl1 = pl2; + } + pl2.m_headPageI = pl2.m_tailPageI = RNIL; + pl2.m_pageCount = 0; +} + +void +SuperPool::addHeadPage(PageList& pl, PtrI pageI) +{ + PageList pl2(pageI); + movePages(pl2, pl); + pl = pl2; +} + +void +SuperPool::addTailPage(PageList& pl, PtrI pageI) +{ + PageList pl2(pageI); + movePages(pl, pl2); +} + +void +SuperPool::removePage(PageList& pl, PtrI pageI) +{ + PageEnt& pe = getPageEnt(pageI); + PtrI pageI1 = pe.m_prevPageI; + PtrI pageI2 = pe.m_nextPageI; + if (pageI1 != RNIL) { + PageEnt& pe1 = getPageEnt(pageI1); + pe1.m_nextPageI = pageI2; + if (pageI2 != RNIL) { + PageEnt& pe2 = getPageEnt(pageI2); + pe2.m_prevPageI = pageI1; + } else { + pl.m_tailPageI = pageI1; + } + } else { + if (pageI2 != RNIL) { + PageEnt& pe2 = getPageEnt(pageI2); + pe2.m_prevPageI = pageI1; + pl.m_headPageI = pageI2; + } else { + pl.m_headPageI = pl.m_tailPageI = RNIL; + } + } + pe.m_prevPageI = pe.m_nextPageI = RNIL; + assert(pl.m_pageCount != 0); + pl.m_pageCount--; +} + +void +SuperPool::setCurrPage(RecInfo& ri, PtrI newPageI) +{ + PtrI oldPageI = ri.m_currPageI; + if (oldPageI != RNIL) { + // copy from cache + PageEnt& pe = getPageEnt(oldPageI); + pe.m_freeRecI = ri.m_currFreeRecI; + pe.m_useCount = ri.m_currUseCount; + // add to right list according to "pp2" policy + if (pe.m_useCount == 0) { + pe.m_pageType = 0; + addHeadPage(m_pageList, oldPageI); + ri.m_totalRecCount -= ri.m_maxUseCount; + } else if (pe.m_useCount < ri.m_maxUseCount) { + addHeadPage(ri.m_activeList, oldPageI); + } else { + addHeadPage(ri.m_fullList, oldPageI); + } + } + if (newPageI != RNIL) { + PageEnt& pe = getPageEnt(newPageI); + // copy to cache + ri.m_currPageI = newPageI; + ri.m_currFreeRecI = pe.m_freeRecI; + ri.m_currUseCount = pe.m_useCount; + // remove from right list + if (pe.m_useCount == 0) { + removePage(ri.m_freeList, newPageI); + } else if (pe.m_useCount < ri.m_maxUseCount) { + removePage(ri.m_activeList, newPageI); + } else { + removePage(ri.m_fullList, newPageI); + } + } else { + ri.m_currPageI = RNIL; + ri.m_currFreeRecI = RNIL; + ri.m_currUseCount = 0; + } +} + +bool +SuperPool::getAvailPage(RecInfo& ri) +{ + PtrI pageI; + if ((pageI = ri.m_activeList.m_headPageI) != RNIL || + (pageI = ri.m_freeList.m_headPageI) != RNIL || + (pageI = getFreePage(ri)) != RNIL) { + setCurrPage(ri, pageI); + return true; + } + return false; +} + +SuperPool::PtrI +SuperPool::getFreePage(RecInfo& ri) +{ + PtrI pageI; + if (m_pageList.m_pageCount != 0) { + pageI = m_pageList.m_headPageI; + removePage(m_pageList, pageI); + } else { + pageI = getNewPage(); + if (pageI == RNIL) + return RNIL; + } + void* pageP = getPageP(pageI); + // set up free record list + Uint32 maxUseCount = ri.m_maxUseCount; + Uint32 recSize = ri.m_recSize; + void* recP = (Uint8*)pageP; + Uint32 irNext = 1; + while (irNext < maxUseCount) { + *(Uint32*)recP = pageI | irNext; + recP = (Uint8*)recP + recSize; + irNext++; + } + *(Uint32*)recP = RNIL; + // add to total record count + ri.m_totalRecCount += maxUseCount; + // set up new page entry + PageEnt& pe = getPageEnt(pageI); + new (&pe) PageEnt(); + pe.m_pageType = ri.m_recType; + pe.m_freeRecI = pageI | 0; + pe.m_useCount = 0; + // set type check bits + setCheckBits(pageI, ri.m_recType); + // add to record pool free list + addHeadPage(ri.m_freeList, pageI); + return pageI; +} + +void +SuperPool::setSizes(size_t initSize, size_t incrSize, size_t maxSize) +{ + const Uint32 pageSize = m_pageSize; + m_initSize = SP_ALIGN_SIZE(initSize, pageSize); + m_incrSize = SP_ALIGN_SIZE(incrSize, pageSize); + m_maxSize = SP_ALIGN_SIZE(maxSize, pageSize); +} + +void +SuperPool::verify(RecInfo& ri) +{ + PageList* plList[3] = { &ri.m_freeList, &ri.m_activeList, &ri.m_fullList }; + for (int i = 0; i < 3; i++) { + PageList& pl = *plList[i]; + unsigned count = 0; + PtrI pageI = pl.m_headPageI; + while (pageI != RNIL) { + PageEnt& pe = getPageEnt(pageI); + PtrI pageI1 = pe.m_prevPageI; + PtrI pageI2 = pe.m_nextPageI; + if (count == 0) { + assert(pageI1 == RNIL); + } else { + assert(pageI1 != RNIL); + PageEnt& pe1 = getPageEnt(pageI1); + assert(pe1.m_nextPageI == pageI); + if (pageI2 != RNIL) { + PageEnt& pe2 = getPageEnt(pageI2); + assert(pe2.m_prevPageI == pageI); + } + } + pageI = pageI2; + count++; + } + assert(pl.m_pageCount == count); + } +} + +// HeapPool + +HeapPool::HeapPool(Uint32 pageSize, Uint32 pageBits) : + SuperPool(pageSize, pageBits), + m_areaHead(), + m_currArea(&m_areaHead), + m_lastArea(&m_areaHead), + m_mallocPart(4) +{ +} + +bool +HeapPool::init() +{ + const Uint32 pageBits = m_pageBits; + if (! SuperPool::init()) + return false;; + // allocate page entry array + Uint32 peBytes = (1 << pageBits) * sizeof(PageEnt); + m_pageEnt = static_cast<PageEnt*>(malloc(peBytes)); + if (m_pageEnt == 0) + return false; + memset(m_pageEnt, 0, peBytes); + // allocate type check array + Uint32 tcWords = 1 << (pageBits - (5 - SP_CHECK_LOG2)); + m_typeCheck = static_cast<Uint32*>(malloc(tcWords << 2)); + if (m_typeCheck == 0) + return false; + memset(m_typeCheck, 0, tcWords << 2); + // allocate initial data + assert(m_totalSize == 0); + if (! allocMoreData(m_initSize)) + return false; + return true; +} + +HeapPool::~HeapPool() +{ + free(m_pageEnt); + free(m_typeCheck); + Area* ap; + while ((ap = m_areaHead.m_nextArea) != 0) { + m_areaHead.m_nextArea = ap->m_nextArea; + free(ap->m_memory); + free(ap); + } +} + +HeapPool::Area::Area() : + m_nextArea(0), + m_firstPageI(RNIL), + m_currPage(0), + m_numPages(0), + m_memory(0) +{ +} + +SuperPool::PtrI +HeapPool::getNewPage() +{ + const Uint32 pageSize = m_pageSize; + const Uint32 pageBits = m_pageBits; + const Uint32 recBits= 32 - pageBits; + Area* ap = m_currArea; + if (ap->m_currPage == ap->m_numPages) { + // area is used up + if (ap->m_nextArea == 0) { + // todo dynamic increase + assert(m_incrSize == 0); + return RNIL; + } + ap = m_currArea = ap->m_nextArea; + } + assert(ap->m_currPage < ap->m_numPages); + PtrI pageI = ap->m_firstPageI; + Int32 ip = (Int32)pageI >> recBits; + ip += ap->m_currPage; + pageI = ip << recBits; + ap->m_currPage++; + return pageI; +} + +bool +HeapPool::allocMoreData(size_t size) +{ + const Uint32 pageSize = m_pageSize; + const Uint32 pageBits = m_pageBits; + const Uint32 recBits = 32 - pageBits; + const Uint32 incrSize = m_incrSize; + const Uint32 incrPages = incrSize / pageSize; + const Uint32 mallocPart = m_mallocPart; + size = SP_ALIGN_SIZE(size, pageSize); + if (incrSize != 0) + size = SP_ALIGN_SIZE(size, incrSize); + Uint32 needPages = size / pageSize; + while (needPages != 0) { + Uint32 wantPages = needPages; + if (incrPages != 0 && wantPages > incrPages) + wantPages = incrPages; + Uint32 tryPages = 0; + void* p1 = 0; + for (Uint32 i = mallocPart; i > 0 && p1 == 0; i--) { + // one page is usually wasted due to alignment to memory root + tryPages = ((wantPages + 1) * i) / mallocPart; + if (tryPages < 2) + break; + p1 = malloc(pageSize * tryPages); + } + if (p1 == 0) + return false; + if (m_memRoot == 0) { + // set memory root at first "big" alloc + // assume malloc header makes later ip = -1 impossible + m_memRoot = p1; + } + void* p2 = SP_ALIGN_PTR(p1, m_memRoot, pageSize); + Uint32 numPages = tryPages - (p1 != p2); + my_ptrdiff_t ipL = ((Uint8*)p2 - (Uint8*)m_memRoot) / pageSize; + Int32 ip = (Int32)ipL; + Int32 lim = 1 << (pageBits - 1); + if (! (ip == ipL && -lim <= ip && ip + numPages < lim)) { + free(p1); + return false; + } + assert(ip != -1); + PtrI pageI = ip << recBits; + needPages = (needPages >= numPages ? needPages - numPages : 0); + m_totalSize += numPages * pageSize; + // allocate new area + Area* ap = static_cast<Area*>(malloc(sizeof(Area))); + if (ap == 0) { + free(p1); + return false; + } + new (ap) Area(); + ap->m_firstPageI = pageI; + ap->m_numPages = numPages; + ap->m_memory = p1; + m_lastArea->m_nextArea = ap; + m_lastArea = ap; + } + return true; +} diff --git a/ndb/src/kernel/vm/SuperPool.hpp b/ndb/src/kernel/vm/SuperPool.hpp new file mode 100644 index 00000000000..157c75aa0d5 --- /dev/null +++ b/ndb/src/kernel/vm/SuperPool.hpp @@ -0,0 +1,561 @@ +/* Copyright (C) 2003 MySQL AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +#ifndef SUPER_POOL_HPP +#define SUPER_POOL_HPP + +#include <ndb_global.h> + +#include <pc.hpp> +#include <ErrorReporter.hpp> + +#define NDB_SP_VERIFY_LEVEL 1 + +/* + * SuperPool - super pool for record pools (abstract class) + * + * Documents SuperPool and RecordPool<T>. + * + * GENERAL + * + * A "super pool" is a shared pool of pages of fixed size. A "record + * pool" is a pool of records of fixed size. One super pool instance is + * used by any number of record pools to allocate their memory. + * A special case is a "page pool" where a record is a simple page, + * possibly smaller than super pool page. + * + * A record pool allocates memory in pages. Thus each used page is + * associated with one record pool and one record type. The records on + * a page form an array starting at start of page. Thus each record has + * an index within the page. Any last partial record which does not fit + * on the page is disregarded. + * + * I-VALUE + * + * The old "i-p" principle is kept. A reference to a super pool page or + * record is stored as an "i-value" from which the record pointer "p" is + * computed. In super pool the i-value is a Uint32 with two parts: + * + * - "ip" index of page within super pool (high pageBits) + * - "ir" index of record within page (low recBits) + * + * The translation between "ip" and page address is described in next + * section. Once page address is known, the record address is found + * from "ir" in the obvious way. + * + * The main advantage with i-value is that it can be verified. The + * level of verification depends on compile type (release, debug). + * + * - "v0" minimal sanity check + * - "v1" check record type matches page type, see below + * - "v2" check record is in use (not yet implemented) + * + * Another advantage of a 32-bit i-value is that it extends the space of + * 32-bit addressable records on a 64-bit platform. + * + * RNIL is 0xffffff00 and indicates NULL i-value. To avoid hitting RNIL + * it is required that pageBits <= 30 and that the maximum value of the + * range (2^pageBits-1) is not used. + * + * MEMORY ROOT + * + * This super pool requires a "memory root" i.e. a memory address such + * that the index of a page "ip" satisfies + * + * page address = memory root + (signed)ip * page size + * + * This is possible on most platforms, provided that the memory root and + * all pages are either on the heap or on the stack, in order to keep + * the size of "ip" reasonably small. + * + * The cast (signed)ip is done as integer of pageBits bits. "ip" has + * same sign bit as i-value "i" so (signed)ip = (Int32)i >> recBits. + * The RNIL restriction can be expressed as (signed)ip != -1. + * + * PAGE ENTRIES + * + * Each super pool page has a "page entry". It contains: + * + * - page type + * - i-value of first free record on page + * - page use count, to see if page can be freed + * - pointers (as i-values) to next and previous page in list + * + * Page entry cannot be stored on the page itself since this prevents + * aligning pages to OS block size and the use of BATs (don't ask) for + * page pools in NDB. For now the implementation provides an array of + * page entries with place for all (2^pageBits) entries. + * + * PAGE TYPE + * + * Page type is (in principle) unique to the record pool using the super + * pool. It is assigned in record pool constructor. Page type zero + * means that the page is free i.e. not allocated to a record pool. + * + * Each "i-p" conversion checks ("v1") that the record belongs to same + * pool as the page. This check is much more common than page or record + * allocation. To make it cache effective, there is a separate array of + * reduced "type bits" (computed from real type). + * + * FREE LISTS + * + * A record is either used or on the free list of the record pool. + * A page has a use count i.e. number of used records. When use count + * drops to zero the page can be returned to the super pool. This is + * not necessarily done at once, or ever. + * + * To make freeing pages feasible, the record pool free list has two + * levels. There are available pages (some free) and a singly linked + * free list within the page. A page allocated to record pool is on one + * of 4 lists: + * + * - free page list (all free, available) + * - active page list (some free, some used, available) + * - full page list (none free) + * - current page (list of 1), see below + * + * Some usage types (temporary pools) may never free records. They pay + * a small penalty for the extra overhead. + * + * RECORD POOL + * + * A pool of records which allocates its memory from a super pool + * instance specified in the constructor. There are 3 basic operations: + * + * - getPtr - translate i-value to pointer-to-record p + * - seize - allocate record + * - release - free record + * + * CURRENT PAGE + * + * getPtr is a fast computation which does not touch the page. For + * seize and release there is an optimization: + * + * Define "current page" as page of latest seize or release. Its page + * entry is cached under record pool instance. The page is removed from + * its normal list. Seize and release on current page are fast and + * avoid touching the page. The current page is used until + * + * - seize and current page is full + * - release and the page is not current page + * + * Then the real page entry is updated and the page is added to the + * appropriate list, and a new page is made current. + * + * PAGE POLICY + * + * Allocating new page to record pool is expensive. Therefore record + * pool should not always return empty pages to super pool. There are + * two trivial policies, each with problems: + * + * - "pp1" never return empty page to super pool + * - "pp2" always return empty page to super pool + * + * This implementation uses "pp2" for now. A real policy is implemented + * in next version. + * + * OPEN ISSUES AND LIMITATIONS + * + * - smarter (virtual) placement of check bits & page entries + * - should getPtr etc be inlined? (too much code) + * - real page policy + * - other implementations (only HeapPool is done) + * - super pool list of all record pools, for statistics etc + * - access by multiple threads is not supported + */ + +// align size +#define SP_ALIGN_SIZE(sz, al) \ + (((sz) + (al) - 1) & ~((al) - 1)) + +// align pointer relative to base +#define SP_ALIGN_PTR(p, base, al) \ + (void*)((Uint8*)(base) + SP_ALIGN_SIZE((Uint8*)(p) - (Uint8*)(base), (al))) + +class SuperPool { +public: + // Type of i-value, used to reference both pages and records. Page + // index "ip" occupies the high bits. The i-value of a page is same + // as i-value of record 0 on the page. + typedef Uint32 PtrI; + + // Size and address alignment given as number of bytes (power of 2). + STATIC_CONST( SP_ALIGN = 8 ); + + // Page entry. Current|y allocated as array of (2^pageBits). + struct PageEnt { + PageEnt(); + Uint32 m_pageType; + Uint32 m_freeRecI; + Uint32 m_useCount; + PtrI m_nextPageI; + PtrI m_prevPageI; + }; + + // Number of bits for cache effective type check given as log of 2. + // Example: 2 means 4 bits and uses 32k for 2g of 32k pages. + STATIC_CONST( SP_CHECK_LOG2 = 2 ); + + // Doubly-linked list of pages. There is one free list in super pool + // and free, active, full list in each record pool. + struct PageList { + PageList(); + PageList(PtrI pageI); + PtrI m_headPageI; + PtrI m_tailPageI; + Uint32 m_pageCount; + }; + + // Record pool information. Each record pool instance contains one. + struct RecInfo { + RecInfo(Uint32 recType, Uint32 recSize); + const Uint32 m_recType; + const Uint32 m_recSize; + Uint32 m_maxUseCount; // could be computed + Uint32 m_currPageI; // current page + Uint32 m_currFreeRecI; + Uint32 m_currUseCount; + Uint32 m_totalUseCount; // total per pool + Uint32 m_totalRecCount; + PageList m_freeList; + PageList m_activeList; + PageList m_fullList; + }; + + // Constructor. Gives page size in bytes (excluding page header) and + // number of bits to use for page index "ip" in i-value. + SuperPool(Uint32 pageSize, Uint32 pageBits); + + // Initialize. Must be called after setting sizes or other parameters + // and before the pool is used. + virtual bool init(); + + // Destructor. + virtual ~SuperPool() = 0; + + // Translate i-value to page entry. + PageEnt& getPageEnt(PtrI pageI); + + // Translate i-value to page address. + void* getPageP(PtrI pageI); + + // Translate page address to i-value (unused). + PtrI getPageI(void* pageP); + + // Given type, return non-zero reduced type check bits. + Uint32 makeCheckBits(Uint32 type); + + // Get type check bits from type check array. + Uint32 getCheckBits(PtrI pageI); + + // Set type check bits in type check array. + void setCheckBits(PtrI pageI, Uint32 type); + + // Translate i-value to record address. + void* getRecP(PtrI recI, RecInfo& ri); + + // Move all pages from second list to end of first list. + void movePages(PageList& pl1, PageList& pl2); + + // Add page to beginning of page list. + void addHeadPage(PageList& pl, PtrI pageI); + + // Add page to end of page list. + void addTailPage(PageList& pl, PtrI pageI); + + // Remove any page from page list. + void removePage(PageList& pl, PtrI pageI); + + // Set current page. Previous current page is updated and added to + // appropriate list. + void setCurrPage(RecInfo& ri, PtrI pageI); + + // Get page with some free records and make it current. Takes head of + // active or free list, or else gets free page from super pool. + bool getAvailPage(RecInfo& ri); + + // Get free page from super pool and add it to record pool free list. + // This is an expensive subroutine of getAvailPage(). + PtrI getFreePage(RecInfo& ri); + + // Get new free page from the implementation. + virtual PtrI getNewPage() = 0; + + // Set 3 size parameters, rounded to page size. If called before + // init() then init() allocates the initial size. + void setSizes(size_t initSize = 0, size_t incrSize = 0, size_t maxSize = 0); + + const Uint32 m_pageSize; + const Uint32 m_pageBits; + // implementation must set up these pointers + void* m_memRoot; + PageEnt* m_pageEnt; + Uint32* m_typeCheck; + Uint32 m_typeSeq; + PageList m_pageList; + size_t m_totalSize; + size_t m_initSize; + size_t m_incrSize; + size_t m_maxSize; + + // Debugging. + void verify(RecInfo& ri); +}; + +inline SuperPool::PageEnt& +SuperPool::getPageEnt(PtrI pageI) +{ + Uint32 ip = pageI >> (32 - m_pageBits); + return m_pageEnt[ip]; +} + +inline void* +SuperPool::getPageP(PtrI ptrI) +{ + Int32 ip = (Int32)ptrI >> (32 - m_pageBits); + my_ptrdiff_t sz = m_pageSize; + void* pageP = (Uint8*)m_memRoot + ip * sz; + return pageP; +} + +inline Uint32 +SuperPool::makeCheckBits(Uint32 type) +{ + Uint32 shift = 1 << SP_CHECK_LOG2; + Uint32 mask = (1 << shift) - 1; + return 1 + type % mask; +} + +inline Uint32 +SuperPool::getCheckBits(PtrI pageI) +{ + Uint32 ip = pageI >> (32 - m_pageBits); + Uint32 xp = ip >> (5 - SP_CHECK_LOG2); + Uint32 yp = ip & (1 << (5 - SP_CHECK_LOG2)) - 1; + Uint32& w = m_typeCheck[xp]; + Uint32 shift = 1 << SP_CHECK_LOG2; + Uint32 mask = (1 << shift) - 1; + // get + Uint32 bits = (w >> yp * shift) & mask; + return bits; +} + +inline void +SuperPool::setCheckBits(PtrI pageI, Uint32 type) +{ + Uint32 ip = pageI >> (32 - m_pageBits); + Uint32 xp = ip >> (5 - SP_CHECK_LOG2); + Uint32 yp = ip & (1 << (5 - SP_CHECK_LOG2)) - 1; + Uint32& w = m_typeCheck[xp]; + Uint32 shift = 1 << SP_CHECK_LOG2; + Uint32 mask = (1 << shift) - 1; + // set + Uint32 bits = makeCheckBits(type); + w &= ~(mask << yp * shift); + w |= (bits << yp * shift); +} + +inline void* +SuperPool::getRecP(PtrI ptrI, RecInfo& ri) +{ + const Uint32 recMask = (1 << (32 - m_pageBits)) - 1; + PtrI pageI = ptrI & ~recMask; +#if NDB_SP_VERIFY_LEVEL >= 1 + Uint32 bits1 = getCheckBits(pageI); + Uint32 bits2 = makeCheckBits(ri.m_recType); + assert(bits1 == bits2); +#endif + void* pageP = getPageP(pageI); + Uint32 ir = ptrI & recMask; + void* recP = (Uint8*)pageP + ir * ri.m_recSize; + return recP; +} + +/* + * HeapPool - SuperPool on heap (concrete class) + * + * A super pool based on malloc with memory root on the heap. This + * pool type has 2 realistic uses: + * + * - a small pool with only initial malloc and pageBits set to match + * - the big pool from which all heap allocations are done + * + * A "smart" malloc may break "ip" limit by using different VM areas for + * different sized requests. For this reason malloc is done in units of + * increment size if possible. Memory root is set to start of first + * malloc. + */ + +class HeapPool : public SuperPool { +public: + // Describes malloc area. The areas are kept in singly linked list. + // There is a list head and pointers to current and last area. + struct Area { + Area(); + Area* m_nextArea; + PtrI m_firstPageI; + Uint32 m_currPage; + Uint32 m_numPages; + void* m_memory; + }; + + // Constructor. + HeapPool(Uint32 pageSize, Uint32 pageBits); + + // Initialize. + virtual bool init(); + + // Destructor. + virtual ~HeapPool(); + + // Use malloc to allocate more. + bool allocMoreData(size_t size); + + // Get new page from current area. + virtual PtrI getNewPage(); + + // List of malloc areas. + Area m_areaHead; + Area* m_currArea; + Area* m_lastArea; + + // Fraction of malloc size to try if cannot get all in one. + Uint32 m_mallocPart; +}; + +/* + * RecordPool - record pool using one super pool instance (template) + * + * Documented under SuperPool. Satisfies ArrayPool interface. + */ + +template <class T> +class RecordPool { +public: + // Constructor. + RecordPool(SuperPool& superPool); + + // Destructor. + ~RecordPool(); + + // Update pointer ptr.p according to i-value ptr.i. + void getPtr(Ptr<T>& ptr); + + // Allocate record from the pool. + bool seize(Ptr<T>& ptr); + + // Return record to the pool. + void release(Ptr<T>& ptr); + + // todo variants of basic methods + + // Return all pages to super pool. The force flag is required if + // there are any used records. + void free(bool force); + + SuperPool& m_superPool; + SuperPool::RecInfo m_recInfo; +}; + +template <class T> +inline +RecordPool<T>::RecordPool(SuperPool& superPool) : + m_superPool(superPool), + m_recInfo(1 + superPool.m_typeSeq++, sizeof(T)) +{ + SuperPool::RecInfo& ri = m_recInfo; + assert(sizeof(T) == SP_ALIGN_SIZE(sizeof(T), sizeof(Uint32))); + Uint32 maxUseCount = superPool.m_pageSize / sizeof(T); + Uint32 sizeLimit = 1 << (32 - superPool.m_pageBits); + if (maxUseCount >= sizeLimit) + maxUseCount = sizeLimit; + ri.m_maxUseCount = maxUseCount; +} + +template <class T> +inline +RecordPool<T>::~RecordPool() +{ + free(true); +} + +template <class T> +inline void +RecordPool<T>::getPtr(Ptr<T>& ptr) +{ + void* recP = m_superPool.getRecP(ptr.i, m_recInfo); + ptr.p = static_cast<T*>(recP); +} + +template <class T> +inline bool +RecordPool<T>::seize(Ptr<T>& ptr) +{ + SuperPool& sp = m_superPool; + SuperPool::RecInfo& ri = m_recInfo; + if (ri.m_currFreeRecI != RNIL || sp.getAvailPage(ri)) { + SuperPool::PtrI recI = ri.m_currFreeRecI; + void* recP = sp.getRecP(recI, ri); + ri.m_currFreeRecI = *(Uint32*)recP; + Uint32 useCount = ri.m_currUseCount; + assert(useCount < ri.m_maxUseCount); + ri.m_currUseCount = useCount + 1; + ri.m_totalUseCount++; + ptr.i = recI; + ptr.p = static_cast<T*>(recP); + return true; + } + return false; +} + +template <class T> +inline void +RecordPool<T>::release(Ptr<T>& ptr) +{ + SuperPool& sp = m_superPool; + SuperPool::RecInfo& ri = m_recInfo; + const Uint32 recMask = (1 << (32 - sp.m_pageBits)) - 1; + SuperPool::PtrI recI = ptr.i; + SuperPool::PtrI pageI = recI & ~recMask; + if (pageI != ri.m_currPageI) { + sp.setCurrPage(ri, pageI); + } + void* recP = sp.getRecP(recI, ri); + *(Uint32*)recP = ri.m_currFreeRecI; + ri.m_currFreeRecI = recI; + Uint32 useCount = ri.m_currUseCount; + assert(useCount != 0); + ri.m_currUseCount = useCount - 1; + ri.m_totalUseCount--; + ptr.i = RNIL; + ptr.p = 0; +} + +template <class T> +inline void +RecordPool<T>::free(bool force) +{ + SuperPool& sp = m_superPool; + SuperPool::RecInfo& ri = m_recInfo; + sp.setCurrPage(ri, RNIL); + assert(force || ri.m_totalUseCount == 0); + sp.movePages(sp.m_pageList, ri.m_freeList); + sp.movePages(sp.m_pageList, ri.m_activeList); + sp.movePages(sp.m_pageList, ri.m_fullList); + ri.m_totalRecCount = 0; +} + +#endif diff --git a/ndb/src/kernel/vm/testSuperPool.cpp b/ndb/src/kernel/vm/testSuperPool.cpp new file mode 100644 index 00000000000..194b3a43fa0 --- /dev/null +++ b/ndb/src/kernel/vm/testSuperPool.cpp @@ -0,0 +1,220 @@ +#if 0 +make -f Makefile -f - testSuperPool <<'_eof_' +testSuperPool: testSuperPool.cpp libkernel.a + $(CXXCOMPILE) -o $@ $@.cpp libkernel.a -L../../common/util/.libs -lgeneral +_eof_ +exit $? +#endif + +/* Copyright (C) 2003 MySQL AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +#include "SuperPool.hpp" +#include <NdbOut.hpp> + +template <Uint32 sz> +struct A { + Uint32 a[sz]; + void fill() { + Uint32 c = 0; + for (Uint32 i = 0; i + 1 < sz; i++) { + a[i] = random(); + c = (c << 1) ^ a[i]; + } + a[sz - 1] = c; + } + void check() { + Uint32 c = 0; + for (Uint32 i = 0; i + 1 < sz; i++) { + c = (c << 1) ^ a[i]; + } + assert(a[sz - 1] == c); + } +}; + +static Uint32 +urandom(Uint32 n) +{ + return (Uint32)random() % n; +} + +static Uint32 +random_coprime(Uint32 n) +{ + Uint32 prime[] = { 101, 211, 307, 401, 503, 601, 701, 809, 907 }; + Uint32 count = sizeof(prime) / sizeof(prime[0]); + while (1) { + Uint32 i = urandom(count); + if (n % prime[i] != 0) + return prime[i]; + } +} + +static int +cmpPtrI(const void* a, const void* b) +{ + Ptr<const void> u = *(Ptr<const void>*)a; + Ptr<const void> v = *(Ptr<const void>*)b; + return u.i < v.i ? -1 : u.i > v.i ? +1 : 0; +} + +static int +cmpPtrP(const void* a, const void* b) +{ + Ptr<const void> u = *(Ptr<const void>*)a; + Ptr<const void> v = *(Ptr<const void>*)b; + return u.p < v.p ? -1 : u.p > v.p ? +1 : 0; +} + +static Uint32 loopcount = 3; + +template <Uint32 sz> +void +sp_test(SuperPool& sp) +{ + typedef A<sz> T; + RecordPool<T> rp(sp); + SuperPool::RecInfo& ri = rp.m_recInfo; + Uint32 pageCount = sp.m_totalSize / sp.m_pageSize; + Uint32 perPage = rp.m_recInfo.m_maxUseCount; + Uint32 perPool = perPage * pageCount; + ndbout << "pages=" << pageCount << " perpage=" << perPage << " perpool=" << perPool << endl; + Ptr<T>* ptrList = new Ptr<T> [perPool]; + memset(ptrList, 0x1f, perPool * sizeof(Ptr<T>)); + Uint32 loop; + for (loop = 0; loop < loopcount; loop++) { + ndbout << "loop " << loop << endl; + Uint32 i, j; + // seize all + ndbout << "seize all" << endl; + for (i = 0; i < perPool + 1; i++) { + j = i; + sp.verify(ri); + Ptr<T> ptr1 = { 0, RNIL }; + if (! rp.seize(ptr1)) + break; + // write value + ptr1.p->fill(); + ptr1.p->check(); + // verify getPtr + Ptr<T> ptr2 = { 0, ptr1.i }; + rp.getPtr(ptr2); + assert(ptr1.i == ptr2.i && ptr1.p == ptr2.p); + // save + ptrList[j] = ptr1; + } + assert(i == perPool); + assert(ri.m_totalUseCount == perPool && ri.m_totalRecCount == perPool); + sp.verify(ri); + // check duplicates + { + Ptr<T>* ptrList2 = new Ptr<T> [perPool]; + memcpy(ptrList2, ptrList, perPool * sizeof(Ptr<T>)); + qsort(ptrList2, perPool, sizeof(Ptr<T>), cmpPtrI); + for (i = 1; i < perPool; i++) + assert(ptrList2[i - 1].i != ptrList2[i].i); + qsort(ptrList2, perPool, sizeof(Ptr<T>), cmpPtrP); + for (i = 1; i < perPool; i++) + assert(ptrList2[i - 1].p != ptrList2[i].p); + delete [] ptrList2; + } + // release all in various orders + ndbout << "release all" << endl; + Uint32 coprime = random_coprime(perPool); + for (i = 0; i < perPool; i++) { + sp.verify(ri); + switch (loop % 3) { + case 0: // ascending + j = i; + break; + case 1: // descending + j = perPool - 1 - i; + break; + case 2: // pseudo-random + j = (coprime * i) % perPool; + break; + } + Ptr<T>& ptr = ptrList[j]; + assert(ptr.i != RNIL && ptr.p != 0); + ptr.p->check(); + rp.release(ptr); + assert(ptr.i == RNIL && ptr.p == 0); + } + sp.setCurrPage(ri, RNIL); + assert(ri.m_totalUseCount == 0 && ri.m_totalRecCount == 0); + sp.verify(ri); + // seize/release at random + ndbout << "seize/release at random" << endl; + for (i = 0; i < loopcount * perPool; i++) { + j = urandom(perPool); + Ptr<T>& ptr = ptrList[j]; + if (ptr.i == RNIL) { + rp.seize(ptr); + ptr.p->fill(); + } else { + ptr.p->check(); + rp.release(ptr); + } + } + ndbout << "used " << ri.m_totalUseCount << endl; + sp.verify(ri); + // release all + ndbout << "release all" << endl; + for (i = 0; i < perPool; i++) { + j = i; + Ptr<T>& ptr = ptrList[j]; + if (ptr.i != RNIL) { + ptr.p->check(); + rp.release(ptr); + } + } + sp.setCurrPage(ri, RNIL); + assert(ri.m_totalUseCount == 0 && ri.m_totalRecCount == 0); + sp.verify(ri); + } + // done + delete [] ptrList; +} + +static Uint32 pageCount = 99; +static Uint32 pageSize = 32768; +static Uint32 pageBits = 15; + +const Uint32 sz1 = 3, sz2 = 4, sz3 = 53, sz4 = 424, sz5 = 5353; + +template void sp_test<sz1>(SuperPool& sp); +template void sp_test<sz2>(SuperPool& sp); +template void sp_test<sz3>(SuperPool& sp); +template void sp_test<sz4>(SuperPool& sp); +template void sp_test<sz5>(SuperPool& sp); + +int +main() +{ + HeapPool sp(pageSize, pageBits); + sp.setSizes(pageCount * pageSize); + if (! sp.init()) + assert(false); + Uint16 s = (Uint16)getpid(); + srandom(s); + ndbout << "rand " << s << endl; + sp_test<sz1>(sp); + sp_test<sz2>(sp); + sp_test<sz3>(sp); + sp_test<sz4>(sp); + sp_test<sz5>(sp); + return 0; +} diff --git a/ndb/src/ndbapi/NdbDictionaryImpl.cpp b/ndb/src/ndbapi/NdbDictionaryImpl.cpp index 96c8f6020e5..ba1dc3fc6b3 100644 --- a/ndb/src/ndbapi/NdbDictionaryImpl.cpp +++ b/ndb/src/ndbapi/NdbDictionaryImpl.cpp @@ -208,11 +208,13 @@ NdbColumnImpl::equal(const NdbColumnImpl& col) const if(m_nullable != col.m_nullable){ DBUG_RETURN(false); } +#ifdef ndb_dictionary_dkey_fixed if(m_pk){ if(m_distributionKey != col.m_distributionKey){ DBUG_RETURN(false); } } +#endif if (m_precision != col.m_precision || m_scale != col.m_scale || m_length != col.m_length || diff --git a/ndb/src/ndbapi/NdbScanOperation.cpp b/ndb/src/ndbapi/NdbScanOperation.cpp index 5980f2588bc..6898639e059 100644 --- a/ndb/src/ndbapi/NdbScanOperation.cpp +++ b/ndb/src/ndbapi/NdbScanOperation.cpp @@ -113,7 +113,7 @@ NdbScanOperation::init(const NdbTableImpl* tab, NdbTransaction* myConnection) int NdbScanOperation::readTuples(NdbScanOperation::LockMode lm, - Uint32 batch, + Uint32 scan_flags, Uint32 parallel) { m_ordered = m_descending = false; @@ -159,7 +159,7 @@ NdbScanOperation::readTuples(NdbScanOperation::LockMode lm, m_keyInfo = lockExcl ? 1 : 0; - bool range = false; + bool rangeScan = false; if (m_accessTable->m_indexType == NdbDictionary::Index::OrderedIndex) { if (m_currentTable == m_accessTable){ @@ -172,8 +172,12 @@ NdbScanOperation::readTuples(NdbScanOperation::LockMode lm, // Modify operation state theStatus = GetValue; theOperationType = OpenRangeScanRequest; - range = true; + rangeScan = true; } + + bool tupScan = (scan_flags & SF_TupScan); + if (tupScan && rangeScan) + tupScan = false; theParallelism = parallel; @@ -202,7 +206,8 @@ NdbScanOperation::readTuples(NdbScanOperation::LockMode lm, ScanTabReq::setLockMode(reqInfo, lockExcl); ScanTabReq::setHoldLockFlag(reqInfo, lockHoldMode); ScanTabReq::setReadCommittedFlag(reqInfo, readCommitted); - ScanTabReq::setRangeScanFlag(reqInfo, range); + ScanTabReq::setRangeScanFlag(reqInfo, rangeScan); + ScanTabReq::setTupScanFlag(reqInfo, tupScan); req->requestInfo = reqInfo; Uint64 transId = theNdbCon->getTransactionId(); @@ -1191,12 +1196,14 @@ error: int NdbIndexScanOperation::readTuples(LockMode lm, - Uint32 batch, - Uint32 parallel, - bool order_by, - bool order_desc, - bool read_range_no){ - int res = NdbScanOperation::readTuples(lm, batch, 0); + Uint32 scan_flags, + Uint32 parallel) +{ + const bool order_by = scan_flags & SF_OrderBy; + const bool order_desc = scan_flags & SF_Descending; + const bool read_range_no = scan_flags & SF_ReadRangeNo; + + int res = NdbScanOperation::readTuples(lm, scan_flags, 0); if(!res && read_range_no) { m_read_range_no = 1; diff --git a/ndb/src/ndbapi/NdbTransaction.cpp b/ndb/src/ndbapi/NdbTransaction.cpp index e32a7ddf70a..d0d664c9a3c 100644 --- a/ndb/src/ndbapi/NdbTransaction.cpp +++ b/ndb/src/ndbapi/NdbTransaction.cpp @@ -1521,6 +1521,7 @@ NdbTransaction::receiveTC_COMMITCONF(const TcCommitConf * commitConf) if(checkState_TransId(&commitConf->transId1)){ theCommitStatus = Committed; theCompletionStatus = CompletedSuccess; + theGlobalCheckpointId = commitConf->gci; return 0; } else { #ifdef NDB_NO_DROPPED_SIGNAL diff --git a/ndb/test/include/HugoTransactions.hpp b/ndb/test/include/HugoTransactions.hpp index 83493bcc2a4..5795bbc94c9 100644 --- a/ndb/test/include/HugoTransactions.hpp +++ b/ndb/test/include/HugoTransactions.hpp @@ -42,7 +42,8 @@ public: int records, int abort = 0, int parallelism = 0, - NdbOperation::LockMode = NdbOperation::LM_Read); + NdbOperation::LockMode = NdbOperation::LM_Read, + int scan_flags = 0); int scanReadRecords(Ndb*, const NdbDictionary::Index*, @@ -50,7 +51,7 @@ public: int abort = 0, int parallelism = 0, NdbOperation::LockMode = NdbOperation::LM_Read, - bool sorted = false); + int scan_flags = 0); int pkReadRecords(Ndb*, int records, diff --git a/ndb/test/ndbapi/testDict.cpp b/ndb/test/ndbapi/testDict.cpp index 5240735dcc6..aca8b299b49 100644 --- a/ndb/test/ndbapi/testDict.cpp +++ b/ndb/test/ndbapi/testDict.cpp @@ -428,103 +428,99 @@ int runUseTableUntilStopped(NDBT_Context* ctx, NDBT_Step* step){ } -int runCreateMaxTables(NDBT_Context* ctx, NDBT_Step* step){ - int failures = 0; +int +runCreateMaxTables(NDBT_Context* ctx, NDBT_Step* step) +{ char tabName[256]; int numTables = ctx->getProperty("tables", 1000); Ndb* pNdb = GETNDB(step); - - for (int i = 0; i < numTables && failures < 5; i++){ + NdbDictionary::Dictionary* pDic = pNdb->getDictionary(); + int i = 0; + for (i = 0; i < numTables; i++) { BaseString::snprintf(tabName, 256, "MAXTAB%d", i); - - if (pNdb->waitUntilReady(30) != 0){ + if (pNdb->waitUntilReady(30) != 0) { // Db is not ready, return with failure return NDBT_FAILED; } - const NdbDictionary::Table* pTab = ctx->getTab(); - ndbout << "|- " << tabName << endl; - + //ndbout << "|- " << tabName << endl; // Set new name for T1 NdbDictionary::Table newTab(* pTab); newTab.setName(tabName); - + // Drop any old (or try to) + (void)pDic->dropTable(newTab.getName()); // Try to create table in db - if (newTab.createTableInDb(pNdb) != 0){ - ndbout << tabName << " coult not be created"<< endl; - failures++; - continue; + if (newTab.createTableInDb(pNdb) != 0) { + ndbout << tabName << " could not be created: " + << pDic->getNdbError() << endl; + if (pDic->getNdbError().code == 707 || + pDic->getNdbError().code == 708 || + pDic->getNdbError().code == 826 || + pDic->getNdbError().code == 827) + break; + return NDBT_FAILED; } - // Verify that table exists in db const NdbDictionary::Table* pTab3 = NDBT_Table::discoverTableFromDb(pNdb, tabName) ; if (pTab3 == NULL){ - ndbout << tabName << " was not found in DB"<< endl; - failures++; - continue; + ndbout << tabName << " was not found in DB: " + << pDic->getNdbError() << endl; + return NDBT_FAILED; } - - if (pTab->equal(*pTab3) == false){ - ndbout << "It was not equal" << endl; - failures++; + if (! newTab.equal(*pTab3)) { + ndbout << "It was not equal" << endl; abort(); + return NDBT_FAILED; } - - int records = 1000; + int records = ctx->getNumRecords(); HugoTransactions hugoTrans(*pTab3); - if (hugoTrans.loadTable(pNdb, records) != 0){ + if (hugoTrans.loadTable(pNdb, records) != 0) { ndbout << "It can NOT be loaded" << endl; - } else{ - ndbout << "It can be loaded" << endl; - - UtilTransactions utilTrans(*pTab3); - if (utilTrans.clearTable(pNdb, records, 64) != 0){ - ndbout << "It can NOT be cleared" << endl; - } else{ - ndbout << "It can be cleared" << endl; - } + return NDBT_FAILED; + } + UtilTransactions utilTrans(*pTab3); + if (utilTrans.clearTable(pNdb, records, 64) != 0) { + ndbout << "It can NOT be cleared" << endl; + return NDBT_FAILED; } - } - if (pNdb->waitUntilReady(30) != 0){ + if (pNdb->waitUntilReady(30) != 0) { // Db is not ready, return with failure return NDBT_FAILED; } + ctx->setProperty("maxtables", i); // HURRAAA! return NDBT_OK; } -int runDropMaxTables(NDBT_Context* ctx, NDBT_Step* step){ - int result = NDBT_OK; +int runDropMaxTables(NDBT_Context* ctx, NDBT_Step* step) +{ char tabName[256]; - int numTables = ctx->getProperty("tables", 1000); + int numTables = ctx->getProperty("maxtables", (Uint32)0); Ndb* pNdb = GETNDB(step); - - for (int i = 0; i < numTables; i++){ + NdbDictionary::Dictionary* pDic = pNdb->getDictionary(); + for (int i = 0; i < numTables; i++) { BaseString::snprintf(tabName, 256, "MAXTAB%d", i); - - if (pNdb->waitUntilReady(30) != 0){ + if (pNdb->waitUntilReady(30) != 0) { // Db is not ready, return with failure return NDBT_FAILED; } - // Verify that table exists in db const NdbDictionary::Table* pTab3 = NDBT_Table::discoverTableFromDb(pNdb, tabName) ; - if (pTab3 == NULL){ - ndbout << tabName << " was not found in DB"<< endl; - continue; + if (pTab3 == NULL) { + ndbout << tabName << " was not found in DB: " + << pDic->getNdbError() << endl; + return NDBT_FAILED; } - - // Try to drop table in db - if (pNdb->getDictionary()->dropTable(pTab3->getName()) != 0){ - ndbout << tabName << " coult not be dropped"<< endl; - result = NDBT_FAILED; + if (pDic->dropTable(pTab3->getName()) != 0) { + ndbout << tabName << " could not be dropped: " + << pDic->getNdbError() << endl; + return NDBT_FAILED; } - } - return result; + return NDBT_OK; } int runTestFragmentTypes(NDBT_Context* ctx, NDBT_Step* step){ @@ -1622,7 +1618,7 @@ TESTCASE("CreateMaxTables", "Create tables until db says that it can't create any more\n"){ TC_PROPERTY("tables", 1000); INITIALIZER(runCreateMaxTables); - FINALIZER(runDropMaxTables); + INITIALIZER(runDropMaxTables); } TESTCASE("PkSizes", "Create tables with all different primary key sizes.\n"\ diff --git a/ndb/test/ndbapi/testOIBasic.cpp b/ndb/test/ndbapi/testOIBasic.cpp index c7c9f417d1a..b6fc2e29bf5 100644 --- a/ndb/test/ndbapi/testOIBasic.cpp +++ b/ndb/test/ndbapi/testOIBasic.cpp @@ -55,7 +55,6 @@ struct Opt { unsigned m_pctnull; unsigned m_rows; unsigned m_samples; - unsigned m_scanbat; unsigned m_scanpar; unsigned m_scanstop; int m_seed; @@ -83,7 +82,6 @@ struct Opt { m_pctnull(10), m_rows(1000), m_samples(0), - m_scanbat(0), m_scanpar(0), m_scanstop(0), m_seed(-1), @@ -121,7 +119,6 @@ printhelp() << " -pctnull N pct NULL values in nullable column [" << d.m_pctnull << "]" << endl << " -rows N rows per thread [" << d.m_rows << "]" << endl << " -samples N samples for some timings (0=all) [" << d.m_samples << "]" << endl - << " -scanbat N scan batch per fragment (ignored by ndb api) [" << d.m_scanbat << "]" << endl << " -scanpar N scan parallelism [" << d.m_scanpar << "]" << endl << " -seed N srandom seed 0=loop number -1=random [" << d.m_seed << "]" << endl << " -subloop N subtest loop count [" << d.m_subloop << "]" << endl @@ -272,7 +269,8 @@ struct Par : public Opt { // abort percentabge unsigned m_abortpct; NdbOperation::LockMode m_lockmode; - // ordered range scan + // scan options + bool m_tupscan; bool m_ordered; bool m_descending; // timer location @@ -295,6 +293,7 @@ struct Par : public Opt { m_deadlock(false), m_abortpct(0), m_lockmode(NdbOperation::LM_Read), + m_tupscan(false), m_ordered(false), m_descending(false) { } @@ -1390,7 +1389,10 @@ int Con::readTuples(Par par) { assert(m_tx != 0 && m_scanop != 0); - CHKCON(m_scanop->readTuples(par.m_lockmode, par.m_scanbat, par.m_scanpar) == 0, *this); + int scan_flags = 0; + if (par.m_tupscan) + scan_flags |= NdbScanOperation::SF_TupScan; + CHKCON(m_scanop->readTuples(par.m_lockmode, scan_flags, par.m_scanpar) == 0, *this); return 0; } @@ -1398,7 +1400,7 @@ int Con::readIndexTuples(Par par) { assert(m_tx != 0 && m_indexscanop != 0); - CHKCON(m_indexscanop->readTuples(par.m_lockmode, par.m_scanbat, par.m_scanpar, par.m_ordered, par.m_descending) == 0, *this); + CHKCON(m_indexscanop->readTuples(par.m_lockmode, 0, par.m_scanpar, par.m_ordered, par.m_descending) == 0, *this); return 0; } @@ -3608,7 +3610,7 @@ scanreadtable(Par par) const Set& set = par.set(); // expected const Set& set1 = set; - LL3("scanread " << tab.m_name << " lockmode=" << par.m_lockmode << " expect=" << set1.count() << " verify=" << par.m_verify); + LL3("scanread " << tab.m_name << " lockmode=" << par.m_lockmode << " tupscan=" << par.m_tupscan << " expect=" << set1.count() << " verify=" << par.m_verify); Set set2(tab, set.m_rows); CHK(con.startTransaction() == 0); CHK(con.getNdbScanOperation(tab) == 0); @@ -4172,6 +4174,9 @@ readverifyfull(Par par) if (par.m_no == 0) { // thread 0 scans table CHK(scanreadtable(par) == 0); + // once more via tup scan + par.m_tupscan = true; + CHK(scanreadtable(par) == 0); } // each thread scans different indexes for (unsigned i = 0; i < tab.m_itabs; i++) { @@ -5036,12 +5041,6 @@ NDB_COMMAND(testOIBasic, "testOIBasic", "testOIBasic", "testOIBasic", 65535) continue; } } - if (strcmp(arg, "-scanbat") == 0) { - if (++argv, --argc > 0) { - g_opt.m_scanbat = atoi(argv[0]); - continue; - } - } if (strcmp(arg, "-scanpar") == 0) { if (++argv, --argc > 0) { g_opt.m_scanpar = atoi(argv[0]); diff --git a/ndb/test/ndbapi/testScan.cpp b/ndb/test/ndbapi/testScan.cpp index 3b52778a013..2802f1c950e 100644 --- a/ndb/test/ndbapi/testScan.cpp +++ b/ndb/test/ndbapi/testScan.cpp @@ -316,11 +316,16 @@ int runScanReadIndex(NDBT_Context* ctx, NDBT_Step* step){ while (pIdx && i<loops && !ctx->isTestStopped()) { g_info << i << ": "; bool sort = (rand() % 100) > 50 ? true : false; + bool desc = (rand() % 100) > 50 ? true : false; + desc = false; // random causes too many deadlocks + int scan_flags = + (NdbScanOperation::SF_OrderBy & -(int)sort) | + (NdbScanOperation::SF_Descending & -(int)desc); NdbOperation::LockMode lm = (NdbOperation::LockMode)(rand() % 3); if (hugoTrans.scanReadRecords(GETNDB(step), pIdx, records, abort, parallelism, lm, - sort) != 0){ + scan_flags) != 0){ return NDBT_FAILED; } i++; @@ -333,6 +338,8 @@ int runScanReadCommitted(NDBT_Context* ctx, NDBT_Step* step){ int records = ctx->getNumRecords(); int parallelism = ctx->getProperty("Parallelism", 240); int abort = ctx->getProperty("AbortProb", 5); + bool tupScan = ctx->getProperty("TupScan"); + int scan_flags = (NdbScanOperation::SF_TupScan & -(int)tupScan); int i = 0; HugoTransactions hugoTrans(*ctx->getTab()); @@ -340,7 +347,8 @@ int runScanReadCommitted(NDBT_Context* ctx, NDBT_Step* step){ g_info << i << ": "; if (hugoTrans.scanReadRecords(GETNDB(step), records, abort, parallelism, - NdbOperation::LM_CommittedRead) != 0){ + NdbOperation::LM_CommittedRead, + scan_flags) != 0){ return NDBT_FAILED; } i++; @@ -1150,6 +1158,18 @@ TESTCASE("ScanReadCommitted240", "downgraded to the maximum parallelism value for the current config)"){ INITIALIZER(runLoadTable); TC_PROPERTY("Parallelism", 240); + TC_PROPERTY("TupScan", (Uint32)0); + STEP(runScanReadCommitted); + FINALIZER(runClearTable); +} +TESTCASE("ScanTupReadCommitted240", + "Verify scan requirement: It should be possible to scan read committed with "\ + "parallelism, test with parallelism 240(240 would automatically be "\ + "downgraded to the maximum parallelism value for the current config). "\ + "Scans TUP pages directly without using ACC."){ + INITIALIZER(runLoadTable); + TC_PROPERTY("Parallelism", 240); + TC_PROPERTY("TupScan", 1); STEP(runScanReadCommitted); FINALIZER(runClearTable); } diff --git a/ndb/test/ndbapi/testScanPerf.cpp b/ndb/test/ndbapi/testScanPerf.cpp index 8ac81297ac3..a730136c3af 100644 --- a/ndb/test/ndbapi/testScanPerf.cpp +++ b/ndb/test/ndbapi/testScanPerf.cpp @@ -216,7 +216,7 @@ run_scan(){ } int par = g_paramters[P_PARRA].value; - int bat = g_paramters[P_BATCH].value; + int bat = 0; // g_paramters[P_BATCH].value; NdbScanOperation::LockMode lm; switch(g_paramters[P_LOCK].value){ case 0: diff --git a/ndb/test/src/HugoOperations.cpp b/ndb/test/src/HugoOperations.cpp index 3dcbe3d5ffd..f9a09eddd1f 100644 --- a/ndb/test/src/HugoOperations.cpp +++ b/ndb/test/src/HugoOperations.cpp @@ -613,7 +613,7 @@ HugoOperations::scanReadRecords(Ndb* pNdb, NdbScanOperation::LockMode lm, if(!pOp) return -1; - if(pOp->readTuples(lm, 1, 1)){ + if(pOp->readTuples(lm, 0, 1)){ return -1; } diff --git a/ndb/test/src/HugoTransactions.cpp b/ndb/test/src/HugoTransactions.cpp index bfe7ea72394..3260b921985 100644 --- a/ndb/test/src/HugoTransactions.cpp +++ b/ndb/test/src/HugoTransactions.cpp @@ -35,7 +35,8 @@ HugoTransactions::scanReadRecords(Ndb* pNdb, int records, int abortPercent, int parallelism, - NdbOperation::LockMode lm) + NdbOperation::LockMode lm, + int scan_flags) { int retryAttempt = 0; @@ -72,7 +73,7 @@ HugoTransactions::scanReadRecords(Ndb* pNdb, return NDBT_FAILED; } - if( pOp ->readTuples(lm, 0, parallelism) ) { + if( pOp ->readTuples(lm, scan_flags, parallelism) ) { ERR(pTrans->getNdbError()); closeTransaction(pNdb); return NDBT_FAILED; @@ -187,7 +188,7 @@ HugoTransactions::scanReadRecords(Ndb* pNdb, int abortPercent, int parallelism, NdbOperation::LockMode lm, - bool sorted) + int scan_flags) { int retryAttempt = 0; @@ -224,7 +225,7 @@ HugoTransactions::scanReadRecords(Ndb* pNdb, return NDBT_FAILED; } - if( pOp ->readTuples(lm, 0, parallelism, sorted) ) { + if( pOp ->readTuples(lm, scan_flags, parallelism) ) { ERR(pTrans->getNdbError()); closeTransaction(pNdb); return NDBT_FAILED; |