Thu Jul 24 21:42:52 UTC 2014 Phil Mesnier <mesnier_p@ociweb.com>

        * orbsvcs/ImplRepo_Service/ImR_Locator_i.h:
        * orbsvcs/ImplRepo_Service/ImR_Locator_i.cpp:
        * orbsvcs/ImplRepo_Service/Shared_Backing_Store.cpp:
          Address the scenario where a server IOR is picked up on
          a refresh from the persistent store to ensure the timeout
          policy is applied. Also cleaned up previous commit's
          temporary fix, it is the right thing to do.

        * orbsvcs/tests/ImplRepo/double_start/Test_i.cpp:
        * orbsvcs/tests/ImplRepo/double_start/client.cpp:
        * orbsvcs/tests/ImplRepo/double_start/double_start.mpc:
        * orbsvcs/tests/ImplRepo/double_start/run_test.pl:
        * orbsvcs/tests/ImplRepo/double_start/server.cpp:
        * bin/tao_other_tests.lst:
          Added a sub-test for the case where the target server is hung
          and the primary ImR locator is off line. Under some conditions
          this can lead to server duplication. Solution TBD.

5 files changed, 244 insertions, 91 deletions

diff --git a/TAO/orbsvcs/tests/ImplRepo/double_start/Test_i.cpp b/TAO/orbsvcs/tests/ImplRepo/double_start/Test_i.cpp
index 94d5b370275..dcea81bada0 100644
--- a/TAO/orbsvcs/tests/ImplRepo/double_start/Test_i.cpp
+++ b/TAO/orbsvcs/tests/ImplRepo/double_start/Test_i.cpp
@@ -49,8 +49,8 @@ Test_i::trigger (void)
 {
   if (this->armed_)
     {
-      ACE_DEBUG ((LM_DEBUG, "(%P) BOOM!\n"));
+      ACE_DEBUG ((LM_DEBUG, ACE_TEXT ("(%P) BOOM!\n")));
       ACE_OS::exit (1);
     }
-  ACE_DEBUG ((LM_DEBUG, "(%P) Click! not armed, returning\n"));
+  ACE_DEBUG ((LM_DEBUG, ACE_TEXT ("(%P) Click! not armed, returning\n")));
 }
diff --git a/TAO/orbsvcs/tests/ImplRepo/double_start/client.cpp b/TAO/orbsvcs/tests/ImplRepo/double_start/client.cpp
index fb8ab5050d1..0891c6b0487 100644
--- a/TAO/orbsvcs/tests/ImplRepo/double_start/client.cpp
+++ b/TAO/orbsvcs/tests/ImplRepo/double_start/client.cpp
@@ -4,16 +4,19 @@
 #include <iostream>
 #include "ace/Get_Opt.h"
 #include "ace/OS_NS_unistd.h"
+#include "orbsvcs/Time_Utilities.h"
+#include "tao/Messaging/Messaging.h"
+#include "tao/AnyTypeCode/Any.h"
 
 int request_delay_secs = 0;
 bool server_abort = false;
-const ACE_TCHAR *ior = ACE_TEXT("");
+const ACE_TCHAR *ior = ACE_TEXT ("");
 CORBA::ORB_var orb;
 
 int
 parse_args (int argc, ACE_TCHAR *argv[])
 {
-  ACE_Get_Opt get_opts (argc, argv, ACE_TEXT("k:d:a"));
+  ACE_Get_Opt get_opts (argc, argv, ACE_TEXT ("k:d:a"));
   int c;
 
   while ((c = get_opts ()) != -1)
@@ -34,10 +37,10 @@ parse_args (int argc, ACE_TCHAR *argv[])
       case '?':
       default:
         ACE_ERROR_RETURN ((LM_ERROR,
-                           "usage:  %s "
-                           "-d <request delay in seconds> "
-                           "-a [abort server] "
-                           "\n",
+                           ACE_TEXT ("usage:  %s ")
+                           ACE_TEXT ("-d <request delay in seconds> ")
+                           ACE_TEXT ("-a [abort server] ")
+                           ACE_TEXT ("\n"),
                            argv [0]),
                           -1);
       }
@@ -45,6 +48,43 @@ parse_args (int argc, ACE_TCHAR *argv[])
   return 0;
 }
 
+CORBA::Object_ptr
+set_timeout_policy (CORBA::Object_ptr obj, const ACE_Time_Value& to)
+{
+  CORBA::Object_var ret;
+
+  try
+    {
+      TimeBase::TimeT timeout;
+      ORBSVCS_Time::Time_Value_to_TimeT (timeout, to);
+      CORBA::Any tmp;
+      tmp <<= timeout;
+
+      CORBA::PolicyList policies (1);
+      policies.length (1);
+      policies[0] = orb->create_policy (Messaging::RELATIVE_RT_TIMEOUT_POLICY_TYPE,
+                                        tmp);
+
+      ret = obj->_set_policy_overrides (policies, CORBA::ADD_OVERRIDE);
+
+      policies[0]->destroy ();
+
+      if (CORBA::is_nil (ret.in ()))
+        {
+          ACE_DEBUG ((LM_DEBUG,
+                      ACE_TEXT ("Unable to set timeout policy.\n")));
+          ret = CORBA::Object::_duplicate (obj);
+        }
+    }
+  catch (CORBA::Exception& ex)
+    {
+      ex._tao_print_exception (ACE_TEXT ("set_timeout_policy ()"));
+      ret = CORBA::Object::_duplicate (obj);
+    }
+
+  return ret._retn ();
+}
+
 void
 do_number_test (void)
 {
@@ -61,7 +101,7 @@ do_number_test (void)
     {
       CORBA::Short n = test->get_server_num (request_delay_secs);
       ACE_DEBUG ((LM_DEBUG,
-                  "Client received reply from server %d\n",
+                  ACE_TEXT ("Client received reply from server %d\n"),
                   n));
     }
 }
@@ -70,22 +110,41 @@ void
 do_restart_test (void)
 {
   CORBA::Object_var obj = orb->string_to_object (ior);
-  ACE_ASSERT (!CORBA::is_nil(obj.in()));
-  Test_var test = Test::_narrow( obj.in() );
-  ACE_ASSERT (!CORBA::is_nil(test.in()));
+  ACE_ASSERT (!CORBA::is_nil(obj.in ()));
+  obj = set_timeout_policy (obj.in (), ACE_Time_Value (5,0));
+  Test_var test = Test::_narrow( obj.in () );
+  ACE_ASSERT (!CORBA::is_nil(test.in ()));
+
+  try
+    {
+      test->arm ();
+    }
+  catch (const CORBA::Exception& ex)
+    {
+      ACE_DEBUG ((LM_DEBUG,
+                  ACE_TEXT ("client caught %C during arm\n"),
+                  ex._name ()));
+      return;
+    }
 
-  test->arm ();
+  ACE_DEBUG ((LM_DEBUG,
+ACE_TEXT ("client sleeping %d seconds\n"),
+request_delay_secs));
+  ACE_OS::sleep (request_delay_secs);
   try
     {
       test->trigger ();
-      ACE_DEBUG ((LM_DEBUG, "client trigger completed\n"));
+      ACE_DEBUG ((LM_DEBUG,
+ACE_TEXT ("client trigger completed\n")));
       return;
     }
   catch (const CORBA::Exception& ex)
     {
-      ex._tao_print_exception ("trigger:");
+      ACE_DEBUG ((LM_DEBUG,
+ACE_TEXT ("client caught %C during first trigger\n"),
+                  ex._name ()));
     }
-  ACE_DEBUG ((LM_DEBUG, "client second trigger\n"));
+  ACE_DEBUG ((LM_DEBUG, ACE_TEXT ("client second trigger\n")));
   test->trigger ();
 }
 
@@ -93,14 +152,12 @@ int
 ACE_TMAIN (int argc, ACE_TCHAR *argv[])
 {
   try {
-    ACE_DEBUG ((LM_DEBUG, "client started, argc = %d\n", argc));
     orb = CORBA::ORB_init( argc, argv );
 
-    ACE_DEBUG ((LM_DEBUG, "client calling parse_ags, argc = %d\n", argc));
     if (parse_args (argc, argv) != 0)
       return 1;
 
-    ACE_DEBUG ((LM_DEBUG, "client ior = %s\n", ior));
+    ACE_DEBUG ((LM_DEBUG, ACE_TEXT ("client ior = %s\n"), ior));
     if (ACE_OS::strlen (ior) == 0)
       {
         do_number_test ();
@@ -113,7 +170,7 @@ ACE_TMAIN (int argc, ACE_TCHAR *argv[])
 
   }
   catch(const CORBA::Exception& ex) {
-    ex._tao_print_exception ("client:");
+    ex._tao_print_exception (ACE_TEXT ("client:"));
   }
 
   return -1;
diff --git a/TAO/orbsvcs/tests/ImplRepo/double_start/double_start.mpc b/TAO/orbsvcs/tests/ImplRepo/double_start/double_start.mpc
index 5ac2135b6b5..4ac24bfb52a 100644
--- a/TAO/orbsvcs/tests/ImplRepo/double_start/double_start.mpc
+++ b/TAO/orbsvcs/tests/ImplRepo/double_start/double_start.mpc
@@ -22,7 +22,7 @@ project(*server): portableserver, orbsvcsexe, avoids_minimum_corba, iortable, im
   }
 }
 
-project(*client): taoclient, anytypecode, avoids_minimum_corba, threads {
+project(*client): taoclient, orbsvcslib, messaging, pi, avoids_minimum_corba, threads {
   after += *idl
   exename = client
   Source_Files {
diff --git a/TAO/orbsvcs/tests/ImplRepo/double_start/run_test.pl b/TAO/orbsvcs/tests/ImplRepo/double_start/run_test.pl
index 409abedc06e..4255d70ee0c 100755
--- a/TAO/orbsvcs/tests/ImplRepo/double_start/run_test.pl
+++ b/TAO/orbsvcs/tests/ImplRepo/double_start/run_test.pl
@@ -12,6 +12,8 @@ use PerlACE::TestTarget;
 $status = 0;
 $debuglevel = 0;
 $cltdbg = 1;
+$kill = 0;
+$server_pid = 0;
 
 if ($#ARGV >= 0) {
     for (my $i = 0; $i <= $#ARGV; $i++) {
@@ -22,6 +24,9 @@ if ($#ARGV >= 0) {
             $i++;
 	    $cltdbg = $ARGV[$i];
 	}
+        elsif ($ARGV[$i] eq '-kill') {
+            $kill = 1;
+        }
 	else {
             usage();
 	    exit 1;
@@ -185,19 +190,46 @@ sub kill_imr
     return 1;
 }
 
+sub kill_primary
+{
+    print "Killing primary ImR\n";
+    $IMR->Kill (); $IMR->TimedWait (1);
+}
+
+sub get_server_pid
+{
+    my $pid = 0;
+    open (FILE, "server.pid") or die "Can't open server.pid: $!";
+    while (<FILE>) {
+      $pid = $_;
+    }
+    close FILE;
+    return $pid;
+}
+
+sub signal_server
+{
+    my $sig = shift;
+    $server_pid = get_server_pid () if ($server_pid == 0);
+    print "signal $sig to server $server_pid\n";
+    kill ($sig, $server_pid);
+}
+
 sub start_imr
 {
+    my $all = shift;
     my $debugbase = "-ORBDebugLevel $debuglevel " .
                     "-ORBVerboseLogging 1 -ORBLogFile ";
-    my $actargs = "-l -o $act_actiorfile $act_initref";
+    my $actargs = "-l -o $act_actiorfile $act_initref -ORBListenEndpoints iiop://127.0.0.1:
+";
 
-    my $imrargs = " -d $debuglevel -v 1000 " .
+    my $imrargs = " -d $debuglevel -i -v 1000 " .
         "--directory . --primary " .
-        "-ORBListenEndpoints iiop://localhost:$port";
+        "-ORBListenEndpoints iiop://127.0.0.1:$port";
 
-    my $rimrargs = " -d $debuglevel -v 1000 -o $imr_imriorfile " .
+    my $rimrargs = " -d $debuglevel -i -v 1000 -o $imr_imriorfile " .
         "--directory . --backup " .
-        "-ORBListenEndpoints iiop://localhost:$rport";
+        "-ORBListenEndpoints iiop://127.0.0.1:$rport";
 
     if ($debuglevel > 0) {
         $imrargs .= " $debugbase $imrlogfile";
@@ -225,51 +257,53 @@ sub start_imr
         return 1;
     }
 
-    $IMR_status = $RIMR->Spawn ();
-    if ($IMR_status != 0) {
-        print STDERR "ERROR: replica ImplRepo Service returned $IMR_status\n";
-        $IMR->Kill (); $IMR->TimedWait (1);
-        return 1;
-    }
-    if ($imr->WaitForFileTimed ($imriorfile, $imr->ProcessStartWaitInterval()) == -1) {
-        print STDERR "ERROR: cannot find file <$imr_imriorfile>\n";
-        $IMR->Kill (); $IMR->TimedWait (1);
-        $RIMR->Kill (); $RIMR->TimedWait (1);
-        return 1;
-    }
+    if ($all == 1) {
+        $IMR_status = $RIMR->Spawn ();
+        if ($IMR_status != 0) {
+            print STDERR "ERROR: replica ImplRepo Service returned $IMR_status\n";
+            $IMR->Kill (); $IMR->TimedWait (1);
+            return 1;
+        }
+        if ($imr->WaitForFileTimed ($imriorfile, $imr->ProcessStartWaitInterval()) == -1) {
+            print STDERR "ERROR: cannot find file <$imr_imriorfile>\n";
+            $IMR->Kill (); $IMR->TimedWait (1);
+            $RIMR->Kill (); $RIMR->TimedWait (1);
+            return 1;
+        }
 
-    if ($imr->GetFile ($imriorfile) == -1) {
-        print STDERR "ERROR: cannot retrieve file <$imr_imriorfile>\n";
-        $IMR->Kill (); $IMR->TimedWait (1);
-        $RIMR->Kill (); $RIMR->TimedWait (1);
-        return 1;
-    }
-    if ($act->PutFile ($imriorfile) == -1) {
-        print STDERR "ERROR: cannot set file <$act_imriorfile>\n";
-        $IMR->Kill (); $IMR->TimedWait (1);
-        $RIMR->Kill (); $RIMR->TimedWait (1);
-        return 1;
-    }
-    if ($ti->PutFile ($imriorfile) == -1) {
-        print STDERR "ERROR: cannot set file <$ti_imriorfile>\n";
-        $IMR->Kill (); $IMR->TimedWait (1);
-        $RIMR->Kill (); $RIMR->TimedWait (1);
-        return 1;
-    }
-    if ($srv->PutFile ($imriorfile) == -1) {
-        print STDERR "ERROR: cannot set file <$srv_imriorfile>\n";
-        $IMR->Kill (); $IMR->TimedWait (1);
-        $RIMR->Kill (); $RIMR->TimedWait (1);
-        return 1;
-    }
+        if ($imr->GetFile ($imriorfile) == -1) {
+            print STDERR "ERROR: cannot retrieve file <$imr_imriorfile>\n";
+            $IMR->Kill (); $IMR->TimedWait (1);
+            $RIMR->Kill (); $RIMR->TimedWait (1);
+            return 1;
+        }
+        if ($act->PutFile ($imriorfile) == -1) {
+            print STDERR "ERROR: cannot set file <$act_imriorfile>\n";
+            $IMR->Kill (); $IMR->TimedWait (1);
+            $RIMR->Kill (); $RIMR->TimedWait (1);
+            return 1;
+        }
+        if ($ti->PutFile ($imriorfile) == -1) {
+            print STDERR "ERROR: cannot set file <$ti_imriorfile>\n";
+            $IMR->Kill (); $IMR->TimedWait (1);
+            $RIMR->Kill (); $RIMR->TimedWait (1);
+            return 1;
+        }
+        if ($srv->PutFile ($imriorfile) == -1) {
+            print STDERR "ERROR: cannot set file <$srv_imriorfile>\n";
+            $IMR->Kill (); $IMR->TimedWait (1);
+            $RIMR->Kill (); $RIMR->TimedWait (1);
+            return 1;
+        }
 
-    $ACT_status = $ACT->Spawn ();
-    if ($ACT_status != 0) {
-        print STDERR "ERROR: ImR Activator returned $ACT_status\n";
-        return 1;
-    }
-    if ($act->WaitForFileTimed ($actiorfile,$act->ProcessStartWaitInterval()) == -1) {
-        return kill_imr ("cannot find file <$act_imriorfile>");
+        $ACT_status = $ACT->Spawn ();
+        if ($ACT_status != 0) {
+            print STDERR "ERROR: ImR Activator returned $ACT_status\n";
+            return 1;
+        }
+        if ($act->WaitForFileTimed ($actiorfile,$act->ProcessStartWaitInterval()) == -1) {
+            return kill_imr ("cannot find file <$act_imriorfile>");
+        }
     }
 }
 
@@ -290,7 +324,7 @@ sub run_client
     print "running client $args\n";
 
     $CLT->Arguments ($args);
-    if ($CLT->SpawnWaitKill ($clt->ProcessStartWaitInterval()) == -1) {
+    if ($CLT->SpawnWaitKill ($clt->ProcessStartWaitInterval() + 120) == -1) {
         print STDERR "ERROR: client failed\n";
         return 1;
     }
@@ -298,16 +332,11 @@ sub run_client
 
 sub do_ti_command
 {
-    my $delstat = shift;
     my $cmd = shift;
     my $cmdargs1 = shift;
     my $cmdargs2 = shift;
 
     my $obj_name = $objprefix . "0";
-    if ($delstat != 0) {
-        my $status_file_name = $obj_name . ".status";
-        $srv->DeleteFile ($status_file_name);
-    }
     my $cmdargs = $cmdargs1;
     $cmdargs .= "0 $cmdargs2" if (length ($cmdargs2) > 0);
     print "invoking ti cmd $cmd $obj_name $cmdargs\n" if ($debuglevel > 0);
@@ -343,26 +372,85 @@ sub list_active_servers
     return $active_servers;
 }
 
-sub servers_list_test
+sub kill_primary_test
+{
+    print "Running double server start test killing the primary ImR.\n";
+
+    my $result = 0;
+    my $start_time = time();
+
+    if (start_imr (1) != 0) {
+        return 1;
+    }
+
+    my $cmdline = $server_cmd . " -o $srviorfile -ORBUseIMR 1 -n 0 $act_initref "
+        . "-ORBListenEndpoints iiop://127.0.0.1:";
+    $cmdline = "./restart.sh -e \\\"$cmdline\\\" -p 0 -r 0 -s $objprefix" . "0";
+
+    if (do_ti_command ("add", "-c \"$cmdline\"") != 0) {
+        return 1;
+    }
+
+    if (do_ti_command ("start") != 0) {
+        return 1;
+    }
+
+    list_active_servers ("-v");
+
+    signal_server ("STOP");
+    kill_primary ();
+    sleep 2;
+    start_imr (0);
+    sleep 2;
+    print "starting client\n";
+
+    if (run_client () != 0) {
+        return 1;
+    }
+
+    signal_server ("CONT");
+
+    my $final_pid = get_server_pid ();
+    if ($final_pid != $server_pid) {
+        print "first server pid was $server_pid, but now there is $final_pid\n";
+        $server_pid = $final_pid;
+        signal_server ("TERM");
+        $status = 1;
+    }
+
+    if (do_ti_command ("shutdown") != 0) {
+        return 1;
+    }
+
+    kill_imr ("");
+
+    my $test_time = time() - $start_time;
+
+    print "\nFinished. The test took $test_time seconds.\n";
+
+    return $status;
+}
+
+sub double_server_test
 {
     print "Running double server start test.\n";
 
     my $result = 0;
     my $start_time = time();
 
-    if (start_imr () != 0) {
+    if (start_imr (1) != 0) {
         return 1;
     }
 
     my $cmdline = $server_cmd . " -o $srviorfile -ORBUseIMR 1 -n 0 $act_initref "
-        . "-ORBListenEndpoints iiop://localhost:";
+        . "-ORBListenEndpoints iiop://127.0.0.1:";
     $cmdline = "./restart.sh -e \\\"$cmdline\\\" -p 0 -r 0 -s $objprefix" . "0";
 
-    if (do_ti_command (0, "add", "-c \"$cmdline\"") != 0) {
+    if (do_ti_command ("add", "-c \"$cmdline\"") != 0) {
         return 1;
     }
 
-    if (do_ti_command (0, "start") != 0) {
+    if (do_ti_command ("start") != 0) {
         return 1;
     }
 
@@ -372,7 +460,7 @@ sub servers_list_test
         return 1;
     }
 
-    if (do_ti_command (1, "shutdown") != 0) {
+    if (do_ti_command ("shutdown") != 0) {
         return 1;
     }
 
@@ -396,6 +484,12 @@ sub usage() {
 
 delete_files (1);
 
-my $ret = servers_list_test();
+$ret = 0;
+if ($kill) {
+    $ret = kill_primary_test ();
+}
+else {
+    $ret = double_server_test ();
+}
 
 exit $ret;
diff --git a/TAO/orbsvcs/tests/ImplRepo/double_start/server.cpp b/TAO/orbsvcs/tests/ImplRepo/double_start/server.cpp
index 0b2e0929f46..964de4acb3f 100644
--- a/TAO/orbsvcs/tests/ImplRepo/double_start/server.cpp
+++ b/TAO/orbsvcs/tests/ImplRepo/double_start/server.cpp
@@ -74,9 +74,9 @@ ACE_TMAIN (int argc, ACE_TCHAR *argv[])
           break;
         case '?':
           ACE_DEBUG ((LM_DEBUG,
-                      "usage: %s "
-                      "-d <seconds to delay before initializing POA> "
-                      "-n Number of the server\n",
+                      ACE_TEXT ("usage: %s ")
+                      ACE_TEXT ("-d <seconds to delay before initializing POA> ")
+                      ACE_TEXT ("-n Number of the server\n"),
                       argv[0]));
           return 1;
           break;
@@ -147,13 +147,13 @@ ACE_TMAIN (int argc, ACE_TCHAR *argv[])
     mgr->activate();
 
     ACE_DEBUG ((LM_DEBUG,
-      "Started Server %s \n",
-      poa_name.c_str()));
+                ACE_TEXT ("Started Server <%C>\n"),
+                poa_name.c_str()));
 
+    ACE_CString pid_file = "server.pid";
     {
-      ACE_CString status_file = poa_name + ACE_CString(".status");
-      ofstream out(status_file.c_str ());
-      out << "started" << endl;
+      ofstream out(pid_file.c_str ());
+      out << ACE_OS::getpid () << endl;
     }
 
     orb->run();
@@ -161,9 +161,11 @@ ACE_TMAIN (int argc, ACE_TCHAR *argv[])
     root_poa->destroy(1,1);
     orb->destroy();
 
+    ACE_OS::unlink (pid_file.c_str ());
+
   }
   catch(const CORBA::Exception& ex) {
-    ex._tao_print_exception ("Server main()");
+    ex._tao_print_exception (ACE_TEXT ("Server main()"));
     return 1;
   }