Added backup restart features for Implementation Repository and FT_Naming_Server.

9 files changed, 477 insertions, 38 deletions

diff --git a/TAO/ChangeLog b/TAO/ChangeLog
index 05e5cc81726..6f0e4b6ecef 100644
--- a/TAO/ChangeLog
+++ b/TAO/ChangeLog
@@ -1,3 +1,35 @@
+Thu Mar 14 15:54:03 UTC 2013  Kevin Stanley  <stanleyk@ociweb.com>
+
+        * orbsvcs/tests/ImplRepo/run_test.pl:
+        * bin/tao_other_tests.lst:
+
+          Added test for backup_restart feature for ImR.
+
+        * orbsvcs/ImplRepo_Service/Locator_Repository.h:
+        * orbsvcs/ImplRepo_Service/Locator_Repository.cpp:
+
+          Added method to support recovering the ior from the combined ior
+          file when restarting the backup.
+
+        * orbsvcs/ImplRepo_Service/README:
+
+          Updated documentation to indicate that backup can now be started
+          without the primary running when a restart is occuring.
+
+        * orbsvcs/ImplRepo_Service/Shared_Backing_Store.cpp:
+
+          Implemented feature for restarting backup after a replicated pair
+          has been successfully started in the past.
+
+        * orbsvcs/tests/FT_Naming/FaultTolerant/run_backup_restart_test.pl:
+
+          Added test for restart of FT_Naming_Server backup after the FT
+          pair has successfully started in the past.
+
+        * tao/ImR_Client/ImR_Client.cpp:
+
+          Fixed memory leak. Clean up list_buffer_ array in destructor.
+
 Thu Mar 14 08:39:00 UTC 2013  Johnny Willemsen  <jwillemsen@remedy.nl>
 
         * NEWS:
diff --git a/TAO/bin/tao_other_tests.lst b/TAO/bin/tao_other_tests.lst
index 642fb9d0d5c..4926eb0cfce 100644
--- a/TAO/bin/tao_other_tests.lst
+++ b/TAO/bin/tao_other_tests.lst
@@ -120,6 +120,7 @@ TAO/orbsvcs/tests/ImplRepo/run_test.pl perclient: !MINIMUM !CORBA_E_COMPACT !COR
 TAO/orbsvcs/tests/ImplRepo/run_test.pl persistent_ir_hash: !MINIMUM !CORBA_E_COMPACT !CORBA_E_MICRO !WCHAR !ACE_FOR_TAO
 TAO/orbsvcs/tests/ImplRepo/run_test.pl persistent_ir_shared: !MINIMUM !CORBA_E_COMPACT !CORBA_E_MICRO !WCHAR !ACE_FOR_TAO !Win32
 TAO/orbsvcs/tests/ImplRepo/run_test.pl failover -replica: !MINIMUM !CORBA_E_COMPACT !CORBA_E_MICRO !WCHAR !ACE_FOR_TAO !Win32
+TAO/orbsvcs/tests/ImplRepo/run_test.pl backup_restart -replica: !MINIMUM !CORBA_E_COMPACT !CORBA_E_MICRO !WCHAR !ACE_FOR_TAO !Win32
 TAO/orbsvcs/tests/ImplRepo/run_test.pl persistent_ft -replica: !MINIMUM !CORBA_E_COMPACT !CORBA_E_MICRO !WCHAR !ACE_FOR_TAO !Win32
 TAO/orbsvcs/tests/ImplRepo/run_test.pl persistent_listingcorrupt -replica: !MINIMUM !CORBA_E_COMPACT !CORBA_E_MICRO !WCHAR !ACE_FOR_TAO !Win32
 TAO/orbsvcs/tests/ImplRepo/run_test.pl persistent_activatorcorrupt -replica: !MINIMUM !CORBA_E_COMPACT !CORBA_E_MICRO !WCHAR !ACE_FOR_TAO !Win32
diff --git a/TAO/orbsvcs/ImplRepo_Service/Locator_Repository.cpp b/TAO/orbsvcs/ImplRepo_Service/Locator_Repository.cpp
index 9dcc0d4383b..29630185a29 100644
--- a/TAO/orbsvcs/ImplRepo_Service/Locator_Repository.cpp
+++ b/TAO/orbsvcs/ImplRepo_Service/Locator_Repository.cpp
@@ -67,8 +67,8 @@ Locator_Repository::report_ior (PortableServer::POA_ptr )
   IORTable::Table_var ior_table = IORTable::Table::_narrow (obj.in ());
   ACE_ASSERT (! CORBA::is_nil (ior_table.in ()));
 
-  ior_table->bind ("ImplRepoService", this->imr_ior_.in());
-  ior_table->bind ("ImR", this->imr_ior_.in());
+  ior_table->rebind ("ImplRepoService", this->imr_ior_.in());
+  ior_table->rebind ("ImR", this->imr_ior_.in());
 
   // Set up multicast support (if enabled)
   if (this->opts_.multicast ())
@@ -121,6 +121,70 @@ Locator_Repository::report_ior (PortableServer::POA_ptr )
 }
 
 int
+Locator_Repository::recover_ior (void)
+{
+  if (this->registered_)
+    {
+      ACE_ERROR_RETURN ((LM_ERROR,
+         ACE_TEXT ("ERROR: Repository already registered IOR. ")
+         ACE_TEXT ("recover_ior should not be called.\n")), -1);
+    }
+
+  if (this->opts_.debug () > 0)
+    {
+      ACE_DEBUG ((LM_INFO, ACE_TEXT ("recover_ior <%C>\n"),
+                  this->opts_.ior_filename ().c_str()));
+    }
+
+  // Load the IOR from the specified file if it is available.
+  const ACE_TString& combined_ior_file = this->opts_.ior_filename ();
+
+  // Check if the file exists. If not, then return 1 indicating
+  // we cannot recover our state.
+  if (ACE_OS::access (combined_ior_file.c_str (), F_OK) != 0)
+    return -1;
+
+  try {
+    ACE_TString combined_ior = "file://" + combined_ior_file;
+
+    CORBA::Object_var combined_obj =
+      this->orb_->string_to_object (combined_ior.c_str());
+
+    if (!CORBA::is_nil (combined_obj.in ()))
+      {
+        // Convert the object back into an IOR string to store in the
+        // imr_ior_ attribute.
+        this->imr_ior_ = this->orb_->object_to_string (combined_obj.in ());
+      }
+  }
+  catch (const CORBA::Exception& ex)
+    {
+      ex._tao_print_exception ("Attempting to read combined_ior for ImR_Locator\n");
+      return -1;
+    }
+
+  // Register the ImR for use with INS
+  CORBA::Object_var obj = this->orb_->resolve_initial_references ("IORTable");
+  IORTable::Table_var ior_table = IORTable::Table::_narrow (obj.in ());
+  ACE_ASSERT (! CORBA::is_nil (ior_table.in ()));
+
+  ior_table->bind ("ImplRepoService", this->imr_ior_.in());
+  ior_table->bind ("ImR", this->imr_ior_.in());
+
+  // Set up multicast support (if enabled)
+  if (this->opts_.multicast ())
+    {
+      ACE_Reactor* reactor = this->orb_->orb_core ()->reactor ();
+      if (this->setup_multicast (reactor, this->imr_ior_.in ()) != 0)
+        return -1;
+    }
+
+  registered_ = true;
+
+  return 0;
+}
+
+int
 Locator_Repository::setup_multicast (ACE_Reactor* reactor, const char* ior)
 {
   ACE_ASSERT (reactor != 0);
diff --git a/TAO/orbsvcs/ImplRepo_Service/Locator_Repository.h b/TAO/orbsvcs/ImplRepo_Service/Locator_Repository.h
index 187debe8335..38759b90faa 100644
--- a/TAO/orbsvcs/ImplRepo_Service/Locator_Repository.h
+++ b/TAO/orbsvcs/ImplRepo_Service/Locator_Repository.h
@@ -146,6 +146,9 @@ protected:
   /// report the ImR Locator's IOR
   virtual int report_ior(PortableServer::POA_ptr imr_poa);
 
+  /// recover the ImR Locator's IOR from the persisted file
+  virtual int recover_ior(void);
+
   int setup_multicast (ACE_Reactor* reactor, const char* imr_ior);
   void teardown_multicast();
 
diff --git a/TAO/orbsvcs/ImplRepo_Service/README b/TAO/orbsvcs/ImplRepo_Service/README
index c5465941cc1..20f5a08f359 100644
--- a/TAO/orbsvcs/ImplRepo_Service/README
+++ b/TAO/orbsvcs/ImplRepo_Service/README
@@ -238,9 +238,11 @@ profile to not be available in the ior.
 
 At startup the primary must be started first, then the backup. When
 a single ImR_Locator goes down, it can be restarted at anytime
-(using the same ORBEndPoints). If they are both shutdown then Fault Tolerant
-ImplRepo_Service will be down until both the primary and backup are restarted
-in that order.. The previous ior file will only remain valid if the ORBEndPoint
+(using the same ORBEndPoints). If they are both shutdown after a successful
+startup of the redundant pair, then either the primary or backup can be
+restarted first. With only a single server running, there is no fault
+tolerance support, but the original fault tolerant ior which was created
+by the backup with the -o option can still be used as long as the ORBEndPoint
 list remains the same for both instances.
 
 As long as both the primary and backup ImR_Locators are not shutdown at the
@@ -276,7 +278,7 @@ same time the Fault Tolerant ImplRepo_Service will always be available.
 
 @subsection activator ImR_Activator
 
-  ImR_Activators, as you might have guessed, do the real work of
+ImR_Activators, as you might have guessed, do the real work of
 activating servers or shutting them down and maintaining the information
 about servers related to them. Only one instance of an ImR_Activator
 can be run on one host. The ImR_Activator is not exposed at all to the
diff --git a/TAO/orbsvcs/ImplRepo_Service/Shared_Backing_Store.cpp b/TAO/orbsvcs/ImplRepo_Service/Shared_Backing_Store.cpp
index fb03f8832f5..2b8dbf523f4 100644
--- a/TAO/orbsvcs/ImplRepo_Service/Shared_Backing_Store.cpp
+++ b/TAO/orbsvcs/ImplRepo_Service/Shared_Backing_Store.cpp
@@ -614,47 +614,87 @@ Shared_Backing_Store::repo_mode() const
 int
 Shared_Backing_Store::connect_replicas (Replica_ptr this_replica)
 {
-  const ACE_TString& replica_ior = replica_ior_filename(true);
+  const ACE_TString& replica_ior_file = replica_ior_filename(true);
   if (this->opts_.debug() > 1)
     {
       ACE_DEBUG((LM_INFO,
-        ACE_TEXT("Resolving ImR replica %s\n"), replica_ior.c_str()));
+        ACE_TEXT("Resolving ImR replica %s\n"), replica_ior_file.c_str()));
     }
 
-  CORBA::Object_var obj =
-    this->orb_->string_to_object (replica_ior.c_str());
+  // Determine if the peer has started previously by checking if the
+  // ior file for the replica is there.
+  int peer_started_previously = 0;
+  if (ACE_OS::access (replica_ior_file.c_str (), F_OK) == 0)
+    {
+      peer_started_previously = 1;
+    }
+  else
+    this->peer_replica_ =
+      ImplementationRepository::UpdatePushNotification::_nil();
 
-  if (!CORBA::is_nil (obj.in ()))
+  if (peer_started_previously)
     {
-      bool non_exist = true;
-      try
-        {
-          this->peer_replica_ = ImplementationRepository::
-            UpdatePushNotification::_narrow (obj.in());
-          non_exist = (this->peer_replica_->_non_existent() == 1);
-        }
-      catch (const CORBA::Exception& )
-        {
-          // let error be handled below
-        }
+      ACE_TString replica_ior = "file://" + replica_ior_file;
+      CORBA::Object_var obj =
+        this->orb_->string_to_object (replica_ior.c_str());
 
-      if (non_exist)
+      if (!CORBA::is_nil (obj.in ()))
         {
-          this->peer_replica_ =
-            ImplementationRepository::UpdatePushNotification::_nil();
+          bool non_exist = true;
+          try
+            {
+              this->peer_replica_ = ImplementationRepository::
+                UpdatePushNotification::_narrow (obj.in());
+              non_exist = (this->peer_replica_->_non_existent() == 1);
+            }
+          catch (const CORBA::Exception& )
+            {
+              // let error be handled below
+            }
+
+          if (non_exist)
+            {
+              this->peer_replica_ =
+                ImplementationRepository::UpdatePushNotification::_nil();
+            }
         }
     }
 
+  // Check if a peer IOR is defined
   if (CORBA::is_nil (this->peer_replica_.in()))
     {
       if (this->imr_type_ == Options::BACKUP_IMR)
-        {
-          ACE_ERROR_RETURN ((LM_ERROR,
-            ACE_TEXT("Error: No primary ImR replica is running <%s>\n"),
-            replica_ior.c_str()), -1);
+        { // We are a backup IMR Locator
+
+          // If the primary has started at some point in the past, but is
+          // not available right now, then we will assume that we are in
+          // a restart situation where the backup is being started while
+          // the primary is still down. This implies that a successful
+          // start of the replication pair has been made in the past and
+          // we can use the combined ior from the previous run.
+          if (peer_started_previously)
+            {
+              // Verify that we recovered the IOR successfully. If we did not
+              // then fail startup of the backup IMR Locator.
+              if (this->recover_ior () == -1)
+                ACE_ERROR_RETURN ((LM_ERROR,
+                                   ACE_TEXT("Error: Unable to retrieve IOR from combined IOR ")
+                                   ACE_TEXT ("file: %C\n"),
+                                   replica_ior_file.c_str()),
+                                  -1);
+            }
+          else
+            { // There has been a startup error. The backup can only be started
+              // after the primary has been successfully started.
+              ACE_ERROR_RETURN ((LM_ERROR,
+                               ACE_TEXT("Error: Primary has not been started previously.\n ")
+                               ACE_TEXT ("file: %C\n"),
+                               replica_ior_file.c_str()),
+                              -1);
+            }
         }
 
-      // no connection currently, just wait for backup
+      // For either primary or backup - no connection currently, just wait for peer to start
       return 0;
     }
 
@@ -674,7 +714,7 @@ Shared_Backing_Store::connect_replicas (Replica_ptr this_replica)
     {
       ACE_ERROR_RETURN ((LM_ERROR,
         ACE_TEXT("Error: obj key <%s> is an invalid ImR replica because %s\n"),
-        replica_ior.c_str(), ip.reason.in()), -1);
+        replica_ior_file.c_str(), ip.reason.in()), -1);
     }
 
   if (opts_.debug() > 9)
@@ -1240,11 +1280,6 @@ Shared_Backing_Store::replica_ior_filename(bool peer_ior_file) const
     }
   ACE_CString ior =
     this->filename_ + IMR_REPLICA[desired_type] + ACE_TEXT(".ior");
-  if (peer_ior_file)
-    {
-      // the peer ior file needs the file prefix
-      ior = "file://" + ior;
-    }
 
   return ior;
 }
diff --git a/TAO/orbsvcs/tests/FT_Naming/FaultTolerant/run_backup_restart_test.pl b/TAO/orbsvcs/tests/FT_Naming/FaultTolerant/run_backup_restart_test.pl
index ece8e711428..70ffe60cc3c 100755
--- a/TAO/orbsvcs/tests/FT_Naming/FaultTolerant/run_backup_restart_test.pl
+++ b/TAO/orbsvcs/tests/FT_Naming/FaultTolerant/run_backup_restart_test.pl
@@ -381,9 +381,10 @@ sub backup_restart_test()
         $status = 1;
     }
 
+    print_msg("INFO: killing backup server");
     $server_status = $NS2->TerminateWaitKill ($server->ProcessStopWaitInterval());
     if ($server_status != 0) {
-        print STDERR "ERROR: server 2 returned $server_status\n";
+        print STDERR "ERROR: Backup Naming Server shutdown returned $server_status\n";
         $status = 1;
     }
 
diff --git a/TAO/orbsvcs/tests/ImplRepo/run_test.pl b/TAO/orbsvcs/tests/ImplRepo/run_test.pl
index 27f97aa3749..f57a9056a46 100755
--- a/TAO/orbsvcs/tests/ImplRepo/run_test.pl
+++ b/TAO/orbsvcs/tests/ImplRepo/run_test.pl
@@ -416,6 +416,32 @@ sub shutdown_servers_using_tao_imr
 }
 
 ###############################################################################
+# Use the TAO IMR through the provided IOR
+
+sub shutdown_servers_using_ior
+{
+    my $provided_ior = shift;
+    my $target = shift;
+
+    for ($index = 0; $index < $num_srvr; ++$index) {
+        print "\n\nshutting down server $index using tao_imr\n";
+        $TI->Arguments ("-ORBInitRef ImplRepoService" .
+            "=file://$provided_ior $refstyle " .
+            "shutdown $a_srv_name[$index]");
+        my $TI_status = $TI->SpawnWaitKill (
+            $target->ProcessStartWaitInterval());
+        if ($TI_status != 0) {
+            print STDERR "ERROR: tao_imr ($index) returned $TI_status\n";
+            kill_act();
+            kill_imr();
+            return 1;
+        }
+        print "\n\nshut down server $index using tao_imr\n";
+    }
+    return 0;
+}
+
+###############################################################################
 
 sub wait_for_imr
 {
@@ -2024,6 +2050,8 @@ sub failover_test
     $replica_imr->DeleteFile ($replica_imriorfile);
     $replica_imr->DeleteFile ($backupiorfile);
 
+    cleanup_replication ();
+
     print "\n\nstarting primary tao_imr_locator\n";
     $repo{IMR}->Arguments ("-d $test_debug_level -o $repo{imr_imriorfile} " .
         "$repo{imr_endpoint_flag} $imr_refstyle $repo{imr_backing_store_flag} ");
@@ -2086,6 +2114,7 @@ sub failover_test
     }
     print "started tao_imr_activator\n";
 
+    print "Adding servers\n";
     if (add_servers (\@repo_for_srvr, \@imr_airplaneiorfile) != 0) {
         return 1;
     }
@@ -2286,6 +2315,271 @@ sub failover_test
 }
 
 ###############################################################################
+# Start the primary then the backup. Kill both the primary and the backup and
+# restart the backup and verify it works.
+
+sub backup_restart_test
+{
+    if (!$replica) {
+      # The failover test needs the -replica flag
+      return 0;
+    }
+
+    my $status = 0;
+
+    my $imr_port = 10001 + $imr->RandomPort ();
+    my $replica_imr_port = $imr_port + 10;
+
+    my %repo;
+    setup_repo(\%repo, $imr, $IMR, $imriorfile, $act, $ACT, $actiorfile, $ti,
+      $TI, $imr_port, "--primary", $backupiorfile);
+
+    my %backup_repo;
+    setup_repo(\%backup_repo, $replica_imr, $replica_IMR, $imriorfile,
+      $act, $ACT, $actiorfile, $ti, $TI,
+      $replica_imr_port, "--backup", $primaryiorfile);
+
+    my @repo_for_srvr;
+    my @a_srv_imriorfile;
+    my @imr_airplaneiorfile;
+    my @a_srv_airplaneiorfile;
+    for ($index = 0; $index < $num_srvr; ++$index) {
+        if ($index == ($num_srvr - 1)) {
+            push(@repo_for_srvr, \%backup_repo);
+        } else {
+            push(@repo_for_srvr, \%repo);
+        }
+
+        push(@a_srv_imriorfile, $a_srv[$index]->LocalFile ($repo_for_srvr[$index]->{imriorfile}));
+        push(@imr_airplaneiorfile, $repo_for_srvr[$index]->{imr}->LocalFile ($airplaneiorfile[$index]));
+        push(@a_srv_airplaneiorfile, $a_srv[$index]->LocalFile ($airplaneiorfile[$index]));
+        $a_srv[$index]->DeleteFile ($repo_for_srvr[$index]->{imriorfile});
+        $a_srv[$index]->DeleteFile ($airplaneiorfile[$index]);
+        $a_cli[$index]->DeleteFile ($airplaneiorfile[$index]);
+        $repo_for_srvr[$index]->{imr}->DeleteFile ($airplaneiorfile[$index]);
+    }
+    $imr->DeleteFile ($imriorfile);
+    $imr->DeleteFile ($primaryiorfile);
+    $act->DeleteFile ($imriorfile);
+    $ti->DeleteFile ($imriorfile);
+    $act->DeleteFile ($actiorfile);
+    $replica_imr->DeleteFile ($replica_imriorfile);
+    $replica_imr->DeleteFile ($backupiorfile);
+
+    cleanup_replication ();
+
+    print "\n\nstarting primary tao_imr_locator\n";
+    print STDERR "tao_imr_locator -d $test_debug_level -o $repo{imr_imriorfile} $repo{imr_endpoint_flag} $imr_refstyle $repo{imr_backing_store_flag} \n";
+
+    $repo{IMR}->Arguments ("-d $test_debug_level -o $repo{imr_imriorfile} " .
+        "$repo{imr_endpoint_flag} $imr_refstyle $repo{imr_backing_store_flag} ");
+    my $IMR_status = $repo{IMR}->Spawn ();
+    if ($IMR_status != 0) {
+        print STDERR "ERROR: ImR Service returned $IMR_status\n";
+        return 1;
+    }
+
+    if (wait_for_imr(\%backup_repo, "replicaiorfile")) {
+        return 1;
+    }
+
+    print "\n\nstarting backup tao_imr_locator\n";
+    print STDERR "-d $test_debug_level -o $backup_repo{imriorfile} $imr_refstyle $backup_repo{imr_endpoint_flag} $backup_repo{imr_backing_store_flag}\n";
+
+    $backup_repo{IMR}->Arguments ("-d $test_debug_level -o " .
+        "$backup_repo{imriorfile} $imr_refstyle " .
+        "$backup_repo{imr_endpoint_flag} " .
+        "$backup_repo{imr_backing_store_flag}");
+    my $replica_IMR_status = $backup_repo{IMR}->Spawn ();
+    if ($replica_IMR_status != 0) {
+        print STDERR "ERROR: ImR Service replica returned $replica_IMR_status\n";
+        return 1;
+    }
+    if (wait_for_imr(\%repo, "replicaiorfile")) {
+        return 1;
+    }
+    print "started backup tao_imr_locator\n";
+
+    if (wait_for_imr(\%repo)) {
+        return 1;
+    }
+    print "started primary tao_imr_locator\n";
+
+    for ($index = 0; $index < $num_srvr; ++$index) {
+        if ($a_srv[$index]->PutFile ($repo_for_srvr[$index]->{imriorfile}) == -1) {
+            print STDERR "ERROR: cannot set file <$a_srv_imriorfile[$index]>\n";
+            kill_imr();
+            return 1;
+        }
+    }
+
+    print "\n\nstarting tao_imr_activator\n";
+    $repo{ACT}->Arguments ("-d $test_debug_level -o $repo{act_actiorfile} " .
+        "-ORBInitRef ImplRepoService=file://$repo{act_imriorfile} $refstyle " .
+        $repo{act_explicit_flag});
+    my $ACT_status = $repo{ACT}->Spawn ();
+    if ($ACT_status != 0) {
+        print STDERR "ERROR: ImR Activator returned $ACT_status\n";
+        return 1;
+    }
+
+    if ($repo{act}->WaitForFileTimed (
+                  $repo{actiorfile},
+                  $repo{act}->ProcessStartWaitInterval()) == -1) {
+        print STDERR "ERROR: cannot find file <" .
+            $repo{act_actiorfile} . ">\n";
+        kill_act();
+        kill_imr();
+        return 1;
+    }
+    print "started tao_imr_activator\n";
+
+    print "Adding servers\n";
+    if (add_servers (\@repo_for_srvr, \@imr_airplaneiorfile) != 0) {
+        return 1;
+    }
+
+    if (add_servers_again (\@repo_for_srvr, \@imr_airplaneiorfile) != 0) {
+        return 1;
+    }
+
+   for ($index = 0; $index < $num_srvr; ++$index) {
+        print "\n\nstarting server $index using tao_imr\n";
+        $repo_for_srvr[$index]->{TI}->Arguments ("-ORBInitRef ImplRepoService" .
+            "=file://$repo_for_srvr[$index]->{ti_imriorfile} $refstyle " .
+            "start $a_srv_name[$index]");
+        my $TI_status = $repo_for_srvr[$index]->{TI}->SpawnWaitKill (
+            $repo_for_srvr[$index]->{ti}->ProcessStartWaitInterval());
+        if ($TI_status != 0) {
+            print STDERR "ERROR: tao_imr ($index) returned $TI_status\n";
+            kill_act();
+            kill_imr();
+            return 1;
+        }
+        print "started server $index using tao_imr\n";
+    }
+
+    for ($index = 0; $index < $num_srvr; ++$index) {
+        if ($a_srv[$index]->WaitForFileTimed (
+                  $airplaneiorfile[$index],
+                  $a_srv[$index]->ProcessStartWaitInterval()) == -1) {
+            print STDERR
+              "ERROR: cannot find file <$a_srv_airplaneiorfile[$index]>\n";
+            kill_then_timed_wait(\@A_SRV, 1);
+            $ACT->Kill (); $ACT->TimedWait (1);
+            $IMR->Kill (); $IMR->TimedWait (1);
+            return 1;
+        }
+        if ($a_srv[$index]->GetFile ($airplaneiorfile[$index]) == -1) {
+            print STDERR
+              "ERROR: cannot retrieve file <$a_srv_airplaneiorfile[$index]>\n";
+            kill_then_timed_wait(\@A_SRV, 1);
+            $ACT->Kill (); $ACT->TimedWait (1);
+            $IMR->Kill (); $IMR->TimedWait (1);
+            return 1;
+        }
+        if ($a_cli[$index]->PutFile ($airplaneiorfile[$index]) == -1) {
+            print STDERR "ERROR: cannot set file <$a_cli_airplaneiorfile[$index]>\n";
+            kill_then_timed_wait(\@A_SRV, 1);
+            $ACT->Kill (); $ACT->TimedWait (1);
+            $IMR->Kill (); $IMR->TimedWait (1);
+            return 1;
+        }
+    }
+
+    if (start_clients() != 0) {
+        return 1;
+    }
+
+    if (stop_clients() != 0) {
+        return 1;
+    }
+
+    if (shutdown_servers_using_ior($imriorfile, $ti) != 0) {
+        return 1;
+    }
+
+    if (start_clients() != 0) {
+        return 1;
+    }
+
+    if (stop_clients() != 0) {
+        return 1;
+    }
+
+    print "\n\nkilling the primary tao_imr_locator\n";
+    $IMR->Kill(); $IMR->TimedWait();
+    print "killed the primary tao_imr_locator\n";
+
+
+    # Kill the backup and then restart it
+    print "\n\nkilling the backup tao_imr_locator\n";
+    $replica_IMR->Kill(); $replica_IMR->TimedWait();
+    print "killed the backup tao_imr_locator\n";
+
+    print "\n\nstarting backup tao_imr_locator\n";
+    print "tao_imr_locator -d $test_debug_level -o " .
+        "$backup_repo{imriorfile} $imr_refstyle " .
+        "$backup_repo{imr_endpoint_flag} " .
+        "$backup_repo{imr_backing_store_flag} \n";
+    $backup_repo{IMR}->Arguments ("-d $test_debug_level -o " .
+        "$backup_repo{imriorfile} $imr_refstyle " .
+        "$backup_repo{imr_endpoint_flag} " .
+        "$backup_repo{imr_backing_store_flag}");
+    $replica_IMR_status = $backup_repo{IMR}->Spawn ();
+    if ($replica_IMR_status != 0) {
+        print STDERR "ERROR: ImR Service replica returned $replica_IMR_status\n";
+        return 1;
+    }
+    if (wait_for_imr(\%repo, "replicaiorfile")) {
+        return 1;
+    }
+    print "started backup tao_imr_locator again\n";
+
+    # Wait for the backup to restart.
+    sleep (10);
+
+    if (start_clients() != 0) {
+        return 1;
+    }
+
+    if (stop_clients() != 0) {
+        return 1;
+    }
+
+    print STDERR "Shuting down servers after backup restarted\n";
+    if (shutdown_servers_using_ior($imriorfile, $ti) != 0) {
+        return 1;
+    }
+
+    print "\n\nkilling the backup tao_imr_locator\n";
+    $replica_IMR->Kill(); $replica_IMR->TimedWait();
+    print "killed the backup tao_imr_locator\n";
+
+    # kill the activator
+    $ACT->Kill(); $ACT->TimedWait();
+
+    cleanup_replication ();
+
+    # clean up IOR files
+    for ($index = 0; $index < $num_srvr; ++$index) {
+        $a_srv[$index]->DeleteFile ($repo_for_srvr[$index]->{imriorfile});
+        $a_srv[$index]->DeleteFile ($airplaneiorfile[$index]);
+        $a_cli[$index]->DeleteFile ($airplaneiorfile[$index]);
+        $repo_for_srvr[$index]->{imr}->DeleteFile ($airplaneiorfile[$index]);
+    }
+    $imr->DeleteFile ($imriorfile);
+    $imr->DeleteFile ($primaryiorfile);
+    $act->DeleteFile ($imriorfile);
+    $ti->DeleteFile ($imriorfile);
+    $act->DeleteFile ($actiorfile);
+    $replica_imr->DeleteFile ($replica_imriorfile);
+    $replica_imr->DeleteFile ($backupiorfile);
+
+    return $status;
+}
+
+###############################################################################
 
 sub persistent_ft_test
 {
@@ -2860,7 +3154,9 @@ my $test_num = 0;
 
 my @tests = ("airplane", "airplane_ir", "nestea", "nestea_ir",
              "both_ir", "persistent_ir", "persistent_ir_hash",
-             "persistent_ir_shared", "persistent_ft", "failover");
+             "persistent_ir_shared", "persistent_ft", "failover",
+             "backup_restart");
+
 my @nt_tests = ("nt_service_ir", "persistent_ir_registry");
 
 my $i;
@@ -2949,6 +3245,9 @@ for ($i = 0; $i <= $#ARGV; $i++) {
     elsif ($ARGV[$i] eq "failover") {
         $ret = failover_test ();
     }
+    elsif ($ARGV[$i] eq "backup_restart") {
+        $ret = backup_restart_test ();
+    }
     elsif ($ARGV[$i] eq "perclient") {
         $ret = perclient();
     }
diff --git a/TAO/tao/ImR_Client/ImR_Client.cpp b/TAO/tao/ImR_Client/ImR_Client.cpp
index fffcfef6398..66c83596235 100644
--- a/TAO/tao/ImR_Client/ImR_Client.cpp
+++ b/TAO/tao/ImR_Client/ImR_Client.cpp
@@ -91,6 +91,8 @@ namespace
     {
     }
 
+    ~ImRifyProfiles () { delete [] list_buffer_; }
+
     CORBA::Object_ptr combined_ior ()
     {
       const CORBA::ULong pcount = base_profiles_.profile_count ();