summaryrefslogtreecommitdiff
path: root/storage/ndb/test/ndbnet/testMNF.run
diff options
context:
space:
mode:
Diffstat (limited to 'storage/ndb/test/ndbnet/testMNF.run')
-rw-r--r--storage/ndb/test/ndbnet/testMNF.run277
1 files changed, 277 insertions, 0 deletions
diff --git a/storage/ndb/test/ndbnet/testMNF.run b/storage/ndb/test/ndbnet/testMNF.run
new file mode 100644
index 00000000000..df226cd3359
--- /dev/null
+++ b/storage/ndb/test/ndbnet/testMNF.run
@@ -0,0 +1,277 @@
+#
+# file : test/ndbnet/testError.run
+# usage: perl testError.run
+#
+# you need to have $NDB_TOP/lib/perl5 on search path $PERL5LIB
+# or else write perl -I$NDB_TOP/lib/perl5 test1.run
+#
+# The database is specified by the $NDB_DATABASE environment variable
+#
+# method names and argument style will change slightly.
+#
+#
+
+use strict;
+use NDB::Run;
+
+my $env = NDB::Run->getenv;
+my $log = $env->getlog;
+$log->setpart(time => 1, line => 0);
+$log->setprio("info");
+
+my $database = $ENV{NDB_DATABASE};
+my $api_cmd = $ENV{API_CMD};
+my $api_cmd2 = $ENV{API_CMD2};
+$log->put("start test database=$database");
+$env->init or $log->push("init failed")->fatal;
+
+my $db = $env->getdb($database) or $log->push->fatal;
+my $mgm = $db->getnode(1) or $log->push->fatal;
+my $api = $db->getnode(6) or $log->push->fatal;
+my $api2 = $db->getnode(7) or $log->push->fatal;
+
+my @dbnode = (); # array of db nodes indexed 2..5
+for my $i (2..5) {
+ $dbnode[$i] = $db->getnode($i) or $log->push->fatal;
+}
+
+# list of db nodes and errors to insert
+my @errors = ( # array of array refs
+ [ 3, 4, 8030, 8031 ],
+ [ 2, 5, 8027, 8030 ],
+ [ 2, 4, 5000, 5000 ],
+ [ 2, 5, 5000, 5000 ],
+ [ 3, 4, 5000, 7008 ],
+ [ 3, 5, 5000, 7008 ],
+ [ 2, 4, 5000, 7007 ],
+ [ 3, 4, 5000, 7007 ],
+ [ 3, 5, 5000, 7006 ],
+ [ 2, 5, 5000, 7006 ],
+ [ 2, 4, 5000, 7005 ],
+ [ 3, 5, 5000, 7005 ],
+ [ 3, 4, 7005, 7005 ],
+ [ 2, 5, 7005, 7005 ],
+ [ 3, 4, 7005, 7007 ],
+ [ 3, 5, 7005, 7007 ],
+ [ 2, 5, 7005, 7008 ],
+ [ 2, 4, 7005, 7008 ],
+ [ 3, 5, 7006, 7006 ],
+ [ 3, 4, 7006, 7006 ],
+ [ 2, 4, 7006, 7007 ],
+ [ 2, 5, 7006, 7007 ],
+ [ 2, 5, 7006, 7008 ],
+ [ 2, 4, 7006, 7008 ],
+ [ 3, 4, 7007, 7007 ],
+ [ 3, 5, 7007, 7007 ],
+ [ 3, 5, 7007, 7008 ],
+ [ 3, 4, 7007, 7008 ],
+ [ 2, 4, 7008, 7008 ],
+ [ 2, 5, 7008, 7008 ],
+ [ 2, 5, 7008, 7008 ],
+ [ 2, 5, 7008, 7008 ],
+ [ 2, 5, 8000, 8000 ],
+ [ 2, 4, 8000, 8000 ],
+ [ 3, 5, 8000, 8001 ],
+ [ 3, 4, 8000, 8001 ],
+ [ 2, 5, 8000, 5001 ],
+ [ 3, 5, 8000, 5001 ],
+ [ 3, 4, 8001, 5001 ],
+ [ 2, 4, 8001, 5001 ],
+ [ 2, 5, 8002, 8029 ],
+ [ 3, 4, 8030, 8031 ],
+ [ 3, 5, 8020, 8021 ],
+ [ 2, 4, 8022, 9999 ],
+ [ 2, 4, 8023, 8025 ],
+ [ 2, 5, 8027, 8030 ],
+ [ 3, 4, 8002, 8002 ],
+ [ 3, 5, 8029, 8030 ],
+ [ 3, 5, 8031, 8031 ],
+ [ 3, 4, 8020, 8020 ],
+ [ 2, 4, 8021, 8021 ],
+ [ 2, 5, 8022, 8022 ],
+ [ 3, 4, 8023, 8023 ],
+ [ 3, 5, 8025, 8025 ],
+ [ 2, 4, 8027, 8027 ],
+ [ 2, 5, 8027, 8027 ],
+ [ 3, 4, 8023, 9999 ],
+ [ 3, 5, 8025, 9998 ],
+ [ 2, 4, 8027, 9999 ],
+ [ 2, 5, 8027, 9998 ],
+ [ 3, 4, 8000, 9999 ],
+ [ 3, 5, 8001, 9998 ],
+ [ 2, 4, 5001, 9999 ],
+ [ 2, 5, 5000, 9999 ],
+ [ 3, 4, 7005, 9999 ],
+ [ 3, 5, 7006, 9999 ],
+ [ 2, 4, 7007, 9999 ],
+ [ 2, 5, 7008, 9999 ],
+ [ 2, 4, 9998, 9998 ],
+ [ 3, 5, 9998, 9998 ],
+ [ 2, 4, 9998, 9998 ],
+ [ 2, 4, 9998, 9998 ],
+ [ 3, 5, 9998, 9998 ],
+ [ 2, 4, 9999, 9999 ],
+ [ 2, 4, 9998, 9998 ],
+ [ 3, 5, 9999, 9999 ],
+ [ 2, 4, 9999, 9999 ],
+ [ 2, 4, 9999, 9999 ],
+ [ 3, 5, 9999, 9999 ],
+);
+
+$db->kill;
+$db->start({init_rm=>1}) or $log->push->fatal;
+sleep 10;
+
+# should be option (or default) in $db->start
+sub wait_until_started {
+ my $local_cnt = 60;
+ while (--$local_cnt > 0) {
+ sleep 5;
+ my $ret = $mgm->write("all status", { wait => 3 });
+ $ret or $log->fatal;
+ my $output = $ret->{output};
+ if ($output =~ /((.|\n)*\bstarted\b(.|\n)*){4}/i) {
+ $log->put("*** db is started ***")->info;
+ return;
+ }
+ if ($output =~ /((.|\n)*\bno.contact\b(.|\n)*){4}/i) {
+ print "NDBT_ProgramExit: 1 - test error\n";
+ $log->put("*** db is dead ***")->fatal;
+ }
+ }
+ print "NDBT_ProgramExit: 1 - test error\n";
+ $log->put("*** node recovery failed ***")->fatal;
+}
+
+sub getdbstatus {
+ my $ret = $mgm->write("all status", { wait => 3 });
+ $ret or return undef;
+ my $output = $ret->{output};
+ my @status = (); # indexed 2..5
+ for my $i (2..5) {
+ my $s = "Unknown";
+ if ($output =~ /Node\s*$i\s*:\s*(\w+)/i) {
+ $s = $1;
+ }#if
+ $status[$i] = lc $s;
+ }#for
+ return \@status;
+}
+
+# count elapsed time
+my $lasttime;
+sub settime { $lasttime = time };
+sub gettime { return time - $lasttime };
+
+wait_until_started();
+$api->start({run=>$api_cmd});
+sleep 15;
+$api2->start({run=>$api_cmd2});
+sleep 15;
+
+# loop over error inserts
+for my $x (0..1) {
+for my $e (@errors) {
+for my $z (0..3) {
+ my $i1 = $e->[0]; # db node number 2..5
+ my $i2 = $e->[1]; # db node number 2..5
+ my $c1 = $e->[2]; # error code
+ my $c2 = $e->[3]; # error code
+ my $dbnode1 = $dbnode[$i1];
+ my $dbnode2 = $dbnode[$i2];
+ my $kill_no = 9998;
+ my @survivor = ();
+ my $survivor_count = 0;
+
+ $log->put("insert error $c1 on node $i1")->push($dbnode1)->notice;
+ $log->put("and insert error $c2 on node $i2")->push($dbnode2)->notice;
+
+ for my $node (2..5) {
+ if (($node ne $i1) && ($node ne $i2)) {
+ $survivor[$survivor_count] = $node;
+ $survivor_count++;
+ }#if
+ }#for
+ # insert error
+ if ($c1 eq $kill_no) {
+ $dbnode1->kill
+ or $log->put("Kill 1 failed")->fatal;
+ } else {
+ $mgm->write("$i1 error $c1")
+ or $log->put("insert error 1 fault")->fatal;
+ }#if
+ if ($c2 eq $kill_no) {
+ $dbnode2->kill
+ or $log->put("Kill 2 failed")->fatal;
+ } else {
+ $mgm->write("$i2 error $c2")
+ or $log->put("insert error 2 fault")->fatal;
+ }#if
+
+ # after a few seconds check that node is dead
+ settime();
+ loop: {
+ gettime() <= 300
+ or $log->put("db node $i1 or $i2 refuses to die")->fatal;
+ my $status = getdbstatus()
+ or $log->put("getdbstatus error")->fatal;
+
+ if (($status->[$survivor[0]] eq 'no') ||
+ ($status->[$survivor[1]] eq 'no')) {
+ print "NDBT_ProgramExit: 1 - test error\n";
+ $db->kill;
+ $log->put("*** db is dead ***")->fatal;
+ }#if
+ if (($status->[$i1] ne 'no') || ($status->[$i2] ne 'no')) { # ...contact
+ sleep 2;
+ redo loop;
+ }#if
+ }#loop
+
+ # have to even check the process is gone
+ sleep 5;
+ if ($dbnode1->stat ne "down") {
+ $log->put("ndb did not die, kill it")->push($dbnode1)->warn;
+ $dbnode1->kill
+ or $log->put("Kill 3 failed")->fatal;
+ }#if
+ if ($dbnode2->stat ne "down") {
+ $log->put("ndb did not die, kill it")->push($dbnode2)->warn;
+ $dbnode2->kill
+ or $log->put("Kill 4 failed")->fatal;
+ }#if
+
+ $log->put("node $i1 and node $i2 is dead")->notice;
+
+ # start the failed nodes
+ $dbnode1->start
+ or $log->put("Start ndb node 1 failed")->fatal;
+ $dbnode2->start
+ or $log->put("Start ndb node 2 failed")->fatal;
+ wait_until_started();
+ $log->put("node $i1 and node $i2 is up again")->notice;
+
+ # check test pgm is running
+ my $stat = $api->stat
+ or $log->put("api->stat failed")->fatal;
+ if ($stat ne "up") {
+ print "NDBT_ProgramExit: 1 - test error\n";
+ sleep 15;
+ $db->kill;
+ $log->put("flexBench has crashed")->fatal;
+ }#if
+ my $stat = $api2->stat
+ or $log->put("api2->stat failed")->fatal;
+ if ($stat ne "up") {
+ print "NDBT_ProgramExit: 1 - test error\n";
+ sleep 15;
+ $db->kill;
+ $log->put("flexBench2 has crashed")->fatal;
+ }#if
+}#for
+}#for
+}#for
+print "NDBT_ProgramExit: 0 - test ready\n";
+$db->kill;
+
+# vim: set sw=4:
View Lorry Controller Status