Thu Oct 23 21:43:09 UTC 2014 Phil Mesnier <mesnier_p@ociweb.com>

        * orbsvcs/ImplRepo_Service/AsyncAccessManager.h:
        * orbsvcs/ImplRepo_Service/AsyncAccessManager.cpp:
        * orbsvcs/ImplRepo_Service/LiveCheck.cpp:
          Ensure that an AAM instance that is updated by the peer
          Locator is properly finalized if the new state is final.

        * bin/tao_other_tests.lst:
        * orbsvcs/tests/ImplRepo/manual_start:
          A test verifying the above fix.

        * tao/Invocation_Retry_State.cpp:
        * tao/Invocation_Utils.h:
        * tao/Synch_Invocation.cpp:
          Refactor the retry decision code to be more comprehendable.

16 files changed, 882 insertions, 58 deletions

diff --git a/TAO/ChangeLog b/TAO/ChangeLog
index 56246ac578a..7e3bee67cea 100644
--- a/TAO/ChangeLog
+++ b/TAO/ChangeLog
@@ -1,3 +1,20 @@
+Thu Oct 23 21:43:09 UTC 2014  Phil Mesnier  <mesnier_p@ociweb.com>
+
+        * orbsvcs/ImplRepo_Service/AsyncAccessManager.h:
+        * orbsvcs/ImplRepo_Service/AsyncAccessManager.cpp:
+        * orbsvcs/ImplRepo_Service/LiveCheck.cpp:
+          Ensure that an AAM instance that is updated by the peer
+          Locator is properly finalized if the new state is final.
+
+        * bin/tao_other_tests.lst:
+        * orbsvcs/tests/ImplRepo/manual_start:
+          A test verifying the above fix.
+
+        * tao/Invocation_Retry_State.cpp:
+        * tao/Invocation_Utils.h:
+        * tao/Synch_Invocation.cpp:
+          Refactor the retry decision code to be more comprehendable.
+
 Tue Oct 21 16:03:14 UTC 2014  Phil Mesnier  <mesnier_p@ociweb.com>
 
         * orbsvcs/ImplRepo_Service/ImR_Locator_i.h:
diff --git a/TAO/bin/tao_other_tests.lst b/TAO/bin/tao_other_tests.lst
index e9bb26ae350..8eae2e5c65b 100644
--- a/TAO/bin/tao_other_tests.lst
+++ b/TAO/bin/tao_other_tests.lst
@@ -133,6 +133,7 @@ TAO/orbsvcs/tests/ImplRepo/kill_server/run_test.pl: !MINIMUM !CORBA_E_COMPACT !C
 TAO/orbsvcs/tests/ImplRepo/kill_slow_server/run_test.pl: !MINIMUM !CORBA_E_COMPACT !CORBA_E_MICRO !WCHAR !LynxOS !ACE_FOR_TAO
 TAO/orbsvcs/tests/ImplRepo/oneway/run_test.pl: !MINIMUM !CORBA_E_COMPACT !CORBA_E_MICRO !WCHAR !LynxOS !ACE_FOR_TAO
 TAO/orbsvcs/tests/ImplRepo/locked/run_test.pl: !MINIMUM !CORBA_E_COMPACT !CORBA_E_MICRO !WCHAR !LynxOS !ACE_FOR_TAO
+TAO/orbsvcs/tests/ImplRepo/manual_start/run_test.pl: !MINIMUM !CORBA_E_COMPACT !CORBA_E_MICRO !WCHAR !LynxOS !ACE_FOR_TAO !OpenVMS
 TAO/orbsvcs/tests/ImplRepo/scale/run_test.pl -servers 5 -objects 5: !MINIMUM !CORBA_E_COMPACT !CORBA_E_MICRO !WCHAR !ACE_FOR_TAO !LynxOS
 TAO/orbsvcs/tests/ImplRepo/scale_clients/run_test.pl: !MINIMUM !CORBA_E_COMPACT !CORBA_E_MICRO !WCHAR !ACE_FOR_TAO !LynxOS
 TAO/orbsvcs/tests/ImplRepo/servers_list/run_test.pl: !ST !MINIMUM !CORBA_E_COMPACT !CORBA_E_MICRO !WCHAR !ACE_FOR_TAO !LynxOS
diff --git a/TAO/orbsvcs/ImplRepo_Service/AsyncAccessManager.cpp b/TAO/orbsvcs/ImplRepo_Service/AsyncAccessManager.cpp
index 9b3c9bf34f2..81ec3587b44 100644
--- a/TAO/orbsvcs/ImplRepo_Service/AsyncAccessManager.cpp
+++ b/TAO/orbsvcs/ImplRepo_Service/AsyncAccessManager.cpp
@@ -27,13 +27,19 @@ AsyncAccessManager::AsyncAccessManager (UpdateableServerInfo &info,
   if (ImR_Locator_i::debug () > 4)
     {
       ORBSVCS_DEBUG ((LM_DEBUG,
-                      ACE_TEXT ("(%P|%t) AsyncAccessManager::ctor server = %s\n"),
-                      info->ping_id ()));
+                      ACE_TEXT ("(%P|%t) AsyncAccessManager(%@)::ctor server = %s\n"),
+                      this, info->ping_id ()));
     }
 }
 
 AsyncAccessManager::~AsyncAccessManager (void)
 {
+  if (ImR_Locator_i::debug () > 4)
+    {
+      ORBSVCS_DEBUG ((LM_DEBUG,
+                      ACE_TEXT ("(%P|%t) AsyncAccessManager(%@)::dtor server = %s\n"),
+                      this, info_->ping_id ()));
+    }
 }
 
 void
@@ -58,7 +64,8 @@ AsyncAccessManager::add_interest (ImR_ResponseHandler *rh)
   if (ImR_Locator_i::debug () > 4)
     {
       ORBSVCS_DEBUG ((LM_DEBUG,
-                      ACE_TEXT ("(%P|%t) AsyncAccessManager::add_interest status = %s\n"),
+                      ACE_TEXT ("(%P|%t) AsyncAccessManager(%@)::add_interest status = %s\n"),
+                      this,
                       status_name (this->status_)));
     }
 
@@ -122,19 +129,32 @@ AsyncAccessManager::remote_state (ImplementationRepository::AAM_Status state)
   this->status (state);
   if (AsyncAccessManager::is_final (state))
     {
-      this->notify_waiters ();
+      this->final_state (false);
     }
 }
 
 void
-AsyncAccessManager::final_state (void)
+AsyncAccessManager::final_state (bool active)
 {
-  this->info_.update_repo ();
+  if (active)
+    {
+      this->info_.update_repo ();
+    }
   this->notify_waiters ();
-  this->info_.notify_remote_access (this->status_);
+  if (active)
+    {
+      this->info_.notify_remote_access (this->status_);
+    }
   if (this->info_->is_mode (ImplementationRepository::PER_CLIENT) ||
       this->status_ != ImplementationRepository::AAM_SERVER_READY)
     {
+      if (ImR_Locator_i::debug () > 5)
+        {
+          ORBSVCS_DEBUG ((LM_DEBUG,
+                          ACE_TEXT ("(%P|%t) AsyncAccessManager(%@)::final_state ")
+                          ACE_TEXT ("removing this from map, server = %s\n"),
+                          this, info_->ping_id ()));
+        }
       AsyncAccessManager_ptr aam (this);
       this->locator_.remove_aam (aam);
       aam._retn(); // release w/o decrementing since table held last reference.
diff --git a/TAO/orbsvcs/ImplRepo_Service/AsyncAccessManager.h b/TAO/orbsvcs/ImplRepo_Service/AsyncAccessManager.h
index 20231db4d63..4e75920e8c5 100644
--- a/TAO/orbsvcs/ImplRepo_Service/AsyncAccessManager.h
+++ b/TAO/orbsvcs/ImplRepo_Service/AsyncAccessManager.h
@@ -80,7 +80,7 @@ class Locator_Export AsyncAccessManager
   static bool is_final (ImplementationRepository::AAM_Status s);
 
  private:
-  void final_state (void);
+  void final_state (bool active = true);
   void notify_waiters (void);
   void status (ImplementationRepository::AAM_Status s);
   void update_status (ImplementationRepository::AAM_Status s);
diff --git a/TAO/orbsvcs/ImplRepo_Service/LiveCheck.cpp b/TAO/orbsvcs/ImplRepo_Service/LiveCheck.cpp
index 90de50068b9..f6122cdc77c 100644
--- a/TAO/orbsvcs/ImplRepo_Service/LiveCheck.cpp
+++ b/TAO/orbsvcs/ImplRepo_Service/LiveCheck.cpp
@@ -34,6 +34,12 @@ LiveListener::_add_ref (void)
 {
   ACE_GUARD_RETURN (TAO_SYNCH_MUTEX, mon, this->lock_, 0);
   ++this->refcount_;
+  if (ImR_Locator_i::debug () > 5)
+    {
+      ORBSVCS_DEBUG ((LM_DEBUG,
+                      ACE_TEXT ("LiveListener::add_ref, %s, count = %d\n"),
+                      server_.c_str(), refcount_));
+    }
   return this;
 }
 
@@ -44,6 +50,12 @@ LiveListener::_remove_ref (void)
   {
     ACE_GUARD (TAO_SYNCH_MUTEX, mon, this->lock_);
     count = --this->refcount_;
+    if (ImR_Locator_i::debug () > 5)
+      {
+        ORBSVCS_DEBUG ((LM_DEBUG,
+                        ACE_TEXT  ("LiveListener::remove_ref, %s, count = %d\n"),
+                        server_.c_str(), count));
+      }
   }
   if (count == 0)
     {
diff --git a/TAO/orbsvcs/tests/ImplRepo/manual_start/README b/TAO/orbsvcs/tests/ImplRepo/manual_start/README
new file mode 100644
index 00000000000..e4ff234c4c2
--- /dev/null
+++ b/TAO/orbsvcs/tests/ImplRepo/manual_start/README
@@ -0,0 +1,9 @@
+$Id$
+
+This test demonstrates that a client attempting to contact a stopped MANUAL server will
+not lock up the ImR. The test starts primary and backup Locators, then registers and
+starts a server with MANUAL activation mode. The server connects to the locator then
+shuts down. Next a client is started that attempts to contact the server via the locator.
+Since this is a disallowed implicit activation, a TRANSIENT exception is raised. The
+client tries again via the backup IMR. At this point the IMR would hang due to a retained
+server accessor in an inactive but final state. The fixed IMR continues to operate normally.
diff --git a/TAO/orbsvcs/tests/ImplRepo/manual_start/client.cpp b/TAO/orbsvcs/tests/ImplRepo/manual_start/client.cpp
new file mode 100644
index 00000000000..2c68f11cce2
--- /dev/null
+++ b/TAO/orbsvcs/tests/ImplRepo/manual_start/client.cpp
@@ -0,0 +1,77 @@
+// $Id$
+
+#include "testC.h"
+#include <iostream>
+#include "ace/Get_Opt.h"
+#include "ace/OS_NS_unistd.h"
+
+const ACE_TCHAR *ior = ACE_TEXT ("file://test.ior");
+CORBA::ORB_var orb;
+
+int
+parse_args (int argc, ACE_TCHAR *argv[])
+{
+  ACE_Get_Opt get_opts (argc, argv, ACE_TEXT ("k:"));
+  int c;
+
+  while ((c = get_opts ()) != -1)
+    {
+      switch (c)
+        {
+        case 'k':
+          ior = get_opts.opt_arg ();
+          break;
+        default:
+          ACE_ERROR_RETURN ((LM_ERROR,
+                             ACE_TEXT ("usage:  %s ")
+                             ACE_TEXT ("-k <ior> ")
+                             ACE_TEXT ("\n"),
+                             argv [0]),
+                            -1);
+        }
+    }
+  // Indicates successful parsing of the command line
+  return 0;
+}
+
+void
+do_test (void)
+{
+  CORBA::Object_var obj = orb->string_to_object (ior);
+  Test_var test = Test::_narrow( obj.in () );
+  ACE_ASSERT (!CORBA::is_nil(test.in ()));
+
+  try
+    {
+      test->contact ();
+      ACE_DEBUG ((LM_DEBUG,
+                  ACE_TEXT ("client contact completed\n")));
+      return;
+    }
+  catch (const CORBA::Exception& ex)
+    {
+      ACE_DEBUG ((LM_DEBUG,
+                  ACE_TEXT ("client caught %C during contact\n"),
+                  ex._name ()));
+    }
+}
+
+int
+ACE_TMAIN (int argc, ACE_TCHAR *argv[])
+{
+  try {
+    orb = CORBA::ORB_init( argc, argv );
+
+    if (parse_args (argc, argv) != 0)
+      return 1;
+
+    do_test ();
+    return 0;
+
+  }
+  catch(const CORBA::Exception& ex) {
+    ex._tao_print_exception (ACE_TEXT ("client:"));
+  }
+
+  return -1;
+}
diff --git a/TAO/orbsvcs/tests/ImplRepo/manual_start/manual_start.mpc b/TAO/orbsvcs/tests/ImplRepo/manual_start/manual_start.mpc
new file mode 100644
index 00000000000..a178cdbfe1f
--- /dev/null
+++ b/TAO/orbsvcs/tests/ImplRepo/manual_start/manual_start.mpc
@@ -0,0 +1,33 @@
+// $Id$
+
+project(*idl): taoidldefaults {
+  IDL_Files {
+    test.idl
+  }
+
+  custom_only = 1
+}
+
+project(*server): portableserver, orbsvcsexe, avoids_minimum_corba, imr_client, avoids_corba_e_micro, threads {
+  after += *idl
+  exename = server
+  IDL_Files {
+  }
+  Source_Files {
+    test_i.cpp
+    server.cpp
+    testC.cpp
+    testS.cpp
+  }
+}
+
+project(*client): taoclient, avoids_minimum_corba, threads {
+  after += *idl
+  exename = client
+  Source_Files {
+    client.cpp
+    testC.cpp
+  }
+  IDL_Files {
+  }
+}
diff --git a/TAO/orbsvcs/tests/ImplRepo/manual_start/run_test.pl b/TAO/orbsvcs/tests/ImplRepo/manual_start/run_test.pl
new file mode 100755
index 00000000000..5c52465fe0e
--- /dev/null
+++ b/TAO/orbsvcs/tests/ImplRepo/manual_start/run_test.pl
@@ -0,0 +1,469 @@
+eval '(exit $?0)' && eval 'exec perl -S $0 ${1+"$@"}'
+    & eval 'exec perl -S $0 $argv:q'
+    if 0;
+
+# $Id$
+# -*- perl -*-
+#
+# 1. Register server as MANUAL start
+# 2. Start server via tao_imr, it writes IOR and exits
+# 3. Run client using IOR which should fail
+# 4. Update server as NORMAL start
+# 5. Start server via tao_imr which should succeed
+#
+###############################################################################
+use lib "$ENV{ACE_ROOT}/bin";
+use PerlACE::TestTarget;
+
+$status = 0;
+$debuglevel = 0;
+$cltdbg = 1;
+$cltpause = 5;
+$kill = 0;
+$server_pid = 0;
+
+if ($#ARGV >= 0) {
+    for (my $i = 0; $i <= $#ARGV; $i++) {
+	if ($ARGV[$i] eq '-debug') {
+	    $debuglevel = 10;
+	}
+	elsif ($ARGV[$i] eq '-cd') {
+            $i++;
+	    $cltdbg = $ARGV[$i];
+	}
+        elsif ($ARGV[$i] eq '-kill') {
+            $kill = 1;
+            $cltpause = 5;
+        }
+	else {
+            usage();
+	    exit 1;
+	}
+    }
+}
+
+my $tgt_num = 0;
+my $imr = PerlACE::TestTarget::create_target (++$tgt_num) || die "Create target $tgt_num failed\n";
+my $replica_imr = PerlACE::TestTarget::create_target (++$tgt_num) || die "Create target $tgt_num failed\n";
+my $act = PerlACE::TestTarget::create_target (++$tgt_num) || die "Create target $tgt_num failed\n";
+my $ti  = PerlACE::TestTarget::create_target (++$tgt_num) || die "Create target $tgt_num failed\n";
+my $srv = PerlACE::TestTarget::create_target (++$tgt_num) || die "Create target $tgt_num failed\n";
+my $clt = PerlACE::TestTarget::create_target (++$tgt_num) || die "Create target $tgt_num failed\n";
+
+my $port = 10001 + $imr->RandomPort ();
+my $rport = $port + 1;
+
+my $imriorfile = "imr_locator.ior";
+my $actiorfile = "imr_activator.ior";
+my $primaryiorfile = "ImR_ReplicaPrimary.ior";
+my $backupiorfile = "ImR_ReplicaBackup.ior";
+my $srviorfile = "test.ior";
+my $persistfile = "persist.xml";
+
+my $imr_imriorfile = $imr->LocalFile ($imriorfile);
+my $imr_persistfile = $imr->LocalFile ($persistfile);
+my $act_imriorfile = $act->LocalFile ($imriorfile);
+my $ti_imriorfile = $ti->LocalFile ($imriorfile);
+my $srv_imriorfile = $srv->LocalFile ($imriorfile);
+my $act_actiorfile = $act->LocalFile ($actiorfile);
+my $srv_srviorfile = $srv->LocalFile ($srviorfile);
+my $clt_srviorfile = $clt->LocalFile ($srviorfile);
+
+my $ti_initref = "-ORBInitRef ImplRepoService=file://$ti_imriorfile";
+my $act_initref = "-ORBInitRef ImplRepoService=file://$act_imriorfile";
+
+$IMR = $imr->CreateProcess ("$ENV{TAO_ROOT}/orbsvcs/ImplRepo_Service/tao_imr_locator");
+$RIMR = $replica_imr->CreateProcess ("$ENV{TAO_ROOT}/orbsvcs/ImplRepo_Service/tao_imr_locator");
+$ACT = $act->CreateProcess ("$ENV{TAO_ROOT}/orbsvcs/ImplRepo_Service/tao_imr_activator");
+$TI = $ti->CreateProcess ("$ENV{ACE_ROOT}/bin/tao_imr");
+$SRV = $srv->CreateProcess ("server");
+$CLT = $clt->CreateProcess ("client");
+
+my $server_cmd = $act->LocalFile ($SRV->Executable());
+
+my $imrlogfile = "imr.log";
+my $rimrlogfile = "replica_imr.log";
+my $actlogfile = "act.log";
+my $cltlogfile = "client.log";
+my $srvlogfile = "server.log";
+
+my $imr_imrlogfile = $imr->LocalFile ($imrlogfile);
+my $act_actlogfile = $act->LocalFile ($actlogfile);
+
+my $stdout_file      = "test.out";
+my $stderr_file      = "test.err";
+my $ti_stdout_file = $ti->LocalFile ($stdout_file);
+my $ti_stderr_file = $ti->LocalFile ($stderr_file);
+
+sub delete_files
+{
+    my $logs_too = shift;
+    if ($logs_too == 1) {
+        $imr->DeleteFile ($imrlogfile);
+        $replica_imr->DeleteFile ($rimrlogfile);
+        $act->DeleteFile ($actlogfile);
+        $clt->DeleteFile ($cltlogfile);
+        $srv->DeleteFile ($srvlogfile);
+    }
+    $imr->DeleteFile ($imriorfile);
+    $imr->DeleteFile ($persistfile);
+    $replica_imr->DeleteFile ($replica_imrlogfile);
+    $act->DeleteFile ($imriorfile);
+    $ti->DeleteFile ($imriorfile);
+    $ti->DeleteFile ($replica_imriorfile);
+    $srv->DeleteFile ($imriorfile);
+    $act->DeleteFile ($actiorfile);
+
+    $ti->DeleteFile ($stdout_file);
+    $ti->DeleteFile ($stderr_file);
+
+    cleanup_replication ('.');
+}
+
+# Clean up after exit call
+END
+{
+    delete_files (0);
+}
+
+sub cleanup_replication
+{
+    my $dir = shift;
+    if (!defined($dir)) {
+        $dir = ".";
+    }
+
+    my $listings = "$dir/imr_listing.xml";
+    my $fnd = 0;
+    if (open FILE, "<$listings") {
+        while (<FILE>) {
+            if ($_ =~ /fname="([^"]+)"?/) {
+                $fnd = 1;
+                my $file = "$dir/$1";
+                print "deleting $file\n" if ($debuglevel > 0);
+                $imr->DeleteFile ($file);
+                $imr->DeleteFile ($file . ".bak");
+            }
+        }
+         close FILE;
+    }
+
+#   If the primary listings file has been corrupt then perform the
+#   deletions from the backup file.
+
+    if (!$fnd) {
+       if (open FILE, "<$listings" . ".bak") {
+           while (<FILE>) {
+               if ($_ =~ /fname="([^"]+)"?/) {
+                   my $file = "$dir/$1";
+                   print "deleting $file\n" if ($debuglevel > 0);
+                   $imr->DeleteFile ($file);
+                   $imr->DeleteFile ($file . ".bak");
+               }
+           }
+            close FILE;
+       }
+    }
+    print "deleting $listings\n" if ($debuglevel > 0);
+    $imr->DeleteFile ("$listings");
+    $imr->DeleteFile ("$listings" . ".bak");
+    $imr->DeleteFile ("$dir/$primaryiorfile");
+    $imr->DeleteFile ("$dir/$backupiorfile");
+}
+
+sub redirect_output
+{
+    open(OLDOUT, ">&", \*STDOUT) or die "Can't dup STDOUT: $!";
+    open(OLDERR, ">&", \*STDERR) or die "Can't dup STDERR: $!";
+    open STDERR, '>', $ti_stderr_file;
+    open STDOUT, '>', $ti_stdout_file;
+}
+
+sub restore_output
+{
+    open(STDERR, ">&OLDERR") or die "Can't dup OLDERR: $!";
+    open(STDOUT, ">&OLDOUT") or die "Can't dup OLDOUT: $!";
+}
+
+sub kill_imr
+{
+    my $msg = shift;
+    print STDERR "ERROR: $msg\n" if (length ($msg) > 0);
+    $ACT->Kill (); $ACT->TimedWait (1);
+    $IMR->Kill (); $IMR->TimedWait (1);
+    $RIMR->Kill (); $RIMR->TimedWait (1);
+    return 1;
+}
+
+sub kill_primary
+{
+    print "Killing primary ImR\n";
+    $IMR->Kill (); $IMR->TimedWait (1);
+}
+
+sub get_server_pid
+{
+    my $pid = 0;
+    open (FILE, "server.pid") or die "Can't open server.pid: $!";
+    while (<FILE>) {
+        chomp;
+        $pid = $_;
+        $server_pid = $pid if ($server_pid == 0);
+    }
+    close FILE;
+    return $pid;
+}
+
+sub signal_server
+{
+    my $sig = shift;
+    print "signal $sig to server $server_pid\n";
+    kill ($sig, $server_pid);
+}
+
+sub start_imr
+{
+    my $all = shift;
+    my $debugbase = "-ORBDebugLevel $debuglevel " .
+                    "-ORBVerboseLogging 1 -ORBLogFile ";
+    my $actargs = "-d $debuglevel -l -o $act_actiorfile $act_initref -ORBListenEndpoints iiop://127.0.0.1:";
+
+    my $imrargs = " -d $debuglevel -i -v 1000 " .
+        "--directory . --primary " .
+        "-ORBListenEndpoints iiop://127.0.0.1:$port";
+
+    my $rimrargs = " -d $debuglevel -i -v 1000 -o $imr_imriorfile " .
+        "--directory . --backup " .
+        "-ORBListenEndpoints iiop://127.0.0.1:$rport";
+
+    if ($debuglevel > 0) {
+        $imrargs .= " $debugbase $imrlogfile";
+        $rimrargs .= " $debugbase $rimrlogfile";
+        $actargs .= " $debugbase $actlogfile";
+    }
+
+    print "imr args = \"$imrargs\"\n" if ($debuglevel > 0);
+    print "replica imr args = \"$rimrargs\"\n" if ($debuglevel > 0);
+    print "act args = \"$actargs\"\n" if ($debuglevel > 0);
+
+    $IMR->Arguments ($imrargs);
+    $RIMR->Arguments ($rimrargs);
+    $ACT->Arguments ($actargs);
+
+    ##### Start ImplRepo #####
+    $IMR_status = $IMR->Spawn ();
+    if ($IMR_status != 0) {
+        print STDERR "ERROR: ImplRepo Service returned $IMR_status\n";
+        return 1;
+    }
+    if ($imr->WaitForFileTimed ($primaryiorfile, $imr->ProcessStartWaitInterval()) == -1) {
+        print STDERR "ERROR: cannot find file <$primaryiorfile>\n";
+        $IMR->Kill (); $IMR->TimedWait (1);
+        return 1;
+    }
+
+    if ($all == 1) {
+        $IMR_status = $RIMR->Spawn ();
+        if ($IMR_status != 0) {
+            print STDERR "ERROR: replica ImplRepo Service returned $IMR_status\n";
+            $IMR->Kill (); $IMR->TimedWait (1);
+            return 1;
+        }
+        if ($imr->WaitForFileTimed ($imriorfile, $imr->ProcessStartWaitInterval()) == -1) {
+            print STDERR "ERROR: cannot find file <$imr_imriorfile>\n";
+            $IMR->Kill (); $IMR->TimedWait (1);
+            $RIMR->Kill (); $RIMR->TimedWait (1);
+            return 1;
+        }
+
+        if ($imr->GetFile ($imriorfile) == -1) {
+            print STDERR "ERROR: cannot retrieve file <$imr_imriorfile>\n";
+            $IMR->Kill (); $IMR->TimedWait (1);
+            return 1;
+        }
+        if ($act->PutFile ($imriorfile) == -1) {
+            print STDERR "ERROR: cannot set file <$act_imriorfile>\n";
+            $IMR->Kill (); $IMR->TimedWait (1);
+            return 1;
+        }
+        if ($ti->PutFile ($imriorfile) == -1) {
+            print STDERR "ERROR: cannot set file <$ti_imriorfile>\n";
+            $IMR->Kill (); $IMR->TimedWait (1);
+            return 1;
+        }
+        if ($srv->PutFile ($imriorfile) == -1) {
+            print STDERR "ERROR: cannot set file <$srv_imriorfile>\n";
+            $IMR->Kill (); $IMR->TimedWait (1);
+            return 1;
+        }
+
+        $ACT_status = $ACT->Spawn ();
+        if ($ACT_status != 0) {
+            print STDERR "ERROR: ImR Activator returned $ACT_status\n";
+            return 1;
+        }
+        if ($act->WaitForFileTimed ($actiorfile,$act->ProcessStartWaitInterval()) == -1) {
+            return kill_imr ("cannot find file <$act_imriorfile>");
+        }
+    }
+}
+
+sub launch_client
+{
+    if ($srv->GetFile ($srviorfile) == -1) {
+        print STDERR "ERROR: cannot retrieve file <$srv_srviorfile>\n";
+        return 1;
+    }
+    if ($clt->PutFile ($srviorfile) == -1) {
+        print STDERR "ERROR: cannot set file <$clt_srviorfile>\n";
+        return 1;
+    }
+
+    my $args = "-k file://$srviorfile";
+    $args .= " -ORBDebuglevel $cltdbg -ORBVerboseLogging 1 -ORBLogFile $cltlogfile" if ($debuglevel > 0);
+
+    print "running client $args\n";
+
+    $CLT->Arguments ($args);
+    if ($CLT->Spawn () == -1) {
+        print STDERR "ERROR: client failed\n";
+        return 1;
+    }
+    return 0;
+}
+
+sub run_client
+{
+    if (launch_client () == 0) {
+        if ($CLT->WaitKill ($clt->ProcessStartWaitInterval() + 120) == -1) {
+            print STDERR "ERROR: client failed\n";
+            return 1;
+        }
+    }
+}
+
+sub do_ti_command
+{
+    my $cmd = shift;
+    my $cmdargs = shift;
+
+    my $obj_name = "manual_test";
+    print "invoking ti cmd $cmd $obj_name $cmdargs\n" if ($debuglevel > 0);
+    $TI->Arguments ("$ti_initref $cmd $obj_name $cmdargs");
+    $TI_status = $TI->SpawnWaitKill ($ti->ProcessStartWaitInterval());
+    if ($TI_status != 0 && $TI_status != 4) {
+        return kill_imr ("tao_imr $cmd $obj_name returned $TI_status");
+    }
+}
+
+sub list_active_servers
+{
+    my $list_options = shift;
+    my $start_time = time();
+    $TI->Arguments ("$ti_initref list $list_options");
+    # Redirect output so we can count number of lines in output
+    redirect_output();
+    $TI_status = $TI->SpawnWaitKill ($ti->ProcessStartWaitInterval());
+    my $list_time = time() - $start_time;
+    restore_output();
+    if ($TI_status != 0) {
+        kill_imr ("tao_imr list returned $TI_status");
+        return -1;
+    }
+    open (FILE, $stderr_file) or die "Can't open $stderr_file: $!";
+    $active_servers = 0;
+    while (<FILE>) {
+	print STDERR $_;
+	$active_servers++;
+    }
+    close FILE;
+    print STDERR "List took $list_time seconds.\n";
+    return $active_servers;
+}
+
+sub manual_test
+{
+    print "Running manual start test.\n";
+    my $result = 0;
+    my $start_time = time();
+
+    if (start_imr (1) != 0) {
+        return 1;
+    }
+
+    my $cmdline = $server_cmd . " -o $srviorfile -ORBUseIMR 1 $act_initref "
+#        . "-ORBDebuglevel 10 -ORBVerboseLogging 1 -ORBLogFile srv.log "
+        . "-ORBListenEndpoints iiop://127.0.0.1:";
+
+    if (do_ti_command ("add", "-a MANUAL -c \"$cmdline\"") != 0) {
+        return 1;
+    }
+
+    if (do_ti_command ("start") != 0) {
+        return 1;
+    }
+
+#    print STDERR "=== kill ImR Locator\n";
+#    $IMR_status = $IMR->TerminateWaitKill ($imr->ProcessStopWaitInterval());
+#    $imr->DeleteFile ($imriorfile);
+
+    sleep 2;
+
+#    print STDERR "=== restart ImR Locator\n";
+#    $IMR_status = $IMR->Spawn ();
+#    if ($IMR_status != 0) {
+#        print STDERR "ERROR: ImplRepo Service returned $IMR_status\n";
+#        exit 1;
+#    }
+#    if ($imr->WaitForFileTimed ($imriorfile,$imr->ProcessStartWaitInterval()) == -1) {
+#        print STDERR "ERROR: cannot find file <$imr_imriorfile>\n";
+#        $IMR->Kill (); $IMR->TimedWait (1);
+#        exit 1;
+#    }
+
+
+    print "starting client\n";
+
+    if (launch_client () != 0) {
+        return 1;
+    }
+    print "******waiting for client exit\n";
+    if ($CLT->WaitKill ($clt->ProcessStartWaitInterval() + 120) == -1) {
+        print STDERR "ERROR: client failed\n";
+        return 1;
+    }
+    print "******client done\n";
+
+    if (do_ti_command ("update", "-a NORMAL -c \"$cmdline\"") != 0) {
+        return 1;
+    }
+
+    if (do_ti_command ("start") != 0) {
+        return 1;
+    }
+
+    sleep 2;
+    kill_imr ("");
+
+    my $test_time = time() - $start_time;
+
+    print "\nFinished. The test took $test_time seconds.\n";
+
+    return $status;
+}
+
+
+sub usage() {
+    print "Usage: run_test.pl [-debug] [-cd n]\n";
+    print "       -debug enables ImR debugging\n";
+    print "       -cd n sets client debug level to n (default 1)\n";
+}
+
+###############################################################################
+###############################################################################
+
+delete_files (1);
+
+$ret = manual_test ();
+
+exit $ret;
diff --git a/TAO/orbsvcs/tests/ImplRepo/manual_start/server.cpp b/TAO/orbsvcs/tests/ImplRepo/manual_start/server.cpp
new file mode 100755
index 00000000000..a6ece057589
--- /dev/null
+++ b/TAO/orbsvcs/tests/ImplRepo/manual_start/server.cpp
@@ -0,0 +1,103 @@
+// $Id$
+
+// server.cpp
+// This version uses the Implementation Repository.
+
+#include "test_i.h"
+#include "ace/Get_Opt.h"
+#include "ace/streams.h"
+#include "ace/OS_NS_unistd.h"
+#include "ace/OS_NS_stdio.h"
+
+PortableServer::POA_ptr
+createPOA (PortableServer::POA_ptr root_poa, const ACE_TCHAR* poa_name)
+{
+  PortableServer::LifespanPolicy_var life =
+    root_poa->create_lifespan_policy(PortableServer::PERSISTENT);
+
+  PortableServer::IdAssignmentPolicy_var assign =
+    root_poa->create_id_assignment_policy(PortableServer::USER_ID);
+
+  CORBA::PolicyList pols;
+  pols.length(2);
+  pols[0] = PortableServer::LifespanPolicy::_duplicate(life.in());
+  pols[1] = PortableServer::IdAssignmentPolicy::_duplicate(assign.in());
+
+  PortableServer::POAManager_var mgr = root_poa->the_POAManager();
+  PortableServer::POA_var poa =
+    root_poa->create_POA(ACE_TEXT_ALWAYS_CHAR (poa_name), mgr.in(), pols);
+
+  life->destroy();
+  assign->destroy();
+
+  return poa._retn();
+}
+
+int
+ACE_TMAIN (int argc, ACE_TCHAR *argv[])
+{
+  try
+    {
+      CORBA::ORB_var orb = CORBA::ORB_init(argc, argv);
+
+      const ACE_TCHAR *ior_file_name = ACE_TEXT ("test.ior");
+      const ACE_TCHAR *poa_name = ACE_TEXT ("manual_test");
+      ACE_Get_Opt get_opts (argc, argv, ACE_TEXT ("o:p:"));
+      int c;
+
+      while ((c = get_opts ()) != -1)
+        {
+          switch (c)
+            {
+            case 'o':
+              ior_file_name = get_opts.opt_arg ();
+              break;
+            case 'p':
+              poa_name = get_opts.opt_arg ();
+              break;
+            default:
+              return -1;
+            }
+        }
+
+      CORBA::Object_var obj = orb->resolve_initial_references ("RootPOA");
+      PortableServer::POA_var root_poa = PortableServer::POA::_narrow (obj.in ());
+
+      PortableServer::POAManager_var mgr = root_poa->the_POAManager ();
+      PortableServer::POA_var test_poa = createPOA (root_poa.in(), poa_name );
+
+    Test_i *impl = new Test_i ();
+    PortableServer::Servant_var<Test_i> test_servant = impl;
+
+    PortableServer::ObjectId_var object_id =
+      PortableServer::string_to_ObjectId ("test_object");
+
+    test_poa->activate_object_with_id (object_id.in(), test_servant.in());
+
+    obj = test_poa->id_to_reference (object_id.in ());
+    CORBA::String_var test_ior = orb->object_to_string (obj.in ());
+    FILE *f = ACE_OS::fopen (ior_file_name, ACE_TEXT ("w"));
+    ACE_OS::fprintf (f, "%s", test_ior.in ());
+    ACE_OS::fclose (f);
+
+    mgr->activate();
+
+    ACE_DEBUG ((LM_DEBUG,
+                ACE_TEXT ("(%P|%t) Started Server <%s>\n"),
+                poa_name));
+
+    ACE_Time_Value tv (1,0);
+    orb->run (tv);
+    root_poa->destroy(1,1);
+    orb->destroy();
+    ACE_DEBUG ((LM_DEBUG,
+                ACE_TEXT ("(%P|%t) Server <%s> orb destroyed\n"),
+                poa_name));
+  }
+  catch(const CORBA::Exception& ex) {
+    ex._tao_print_exception (ACE_TEXT ("Server main()"));
+    return 1;
+  }
+
+  return 0;
+}
diff --git a/TAO/orbsvcs/tests/ImplRepo/manual_start/test.idl b/TAO/orbsvcs/tests/ImplRepo/manual_start/test.idl
new file mode 100755
index 00000000000..a51ce7712f4
--- /dev/null
+++ b/TAO/orbsvcs/tests/ImplRepo/manual_start/test.idl
@@ -0,0 +1,6 @@
+// $Id$
+
+interface Test
+{
+  void contact ();
+};
diff --git a/TAO/orbsvcs/tests/ImplRepo/manual_start/test_i.cpp b/TAO/orbsvcs/tests/ImplRepo/manual_start/test_i.cpp
new file mode 100755
index 00000000000..5828cb62c5e
--- /dev/null
+++ b/TAO/orbsvcs/tests/ImplRepo/manual_start/test_i.cpp
@@ -0,0 +1,19 @@
+/* -*- C++ -*- $Id$ */
+
+#include "test_i.h"
+#include "ace/Log_Msg.h"
+
+Test_i::Test_i (void)
+{
+}
+
+Test_i::~Test_i ()
+{
+}
+
+void
+Test_i::contact (void)
+{
+  ACE_DEBUG ((LM_DEBUG,
+              ACE_TEXT ("(%P|%t) Server contacted\n")));
+}
diff --git a/TAO/orbsvcs/tests/ImplRepo/manual_start/test_i.h b/TAO/orbsvcs/tests/ImplRepo/manual_start/test_i.h
new file mode 100755
index 00000000000..542a74f6946
--- /dev/null
+++ b/TAO/orbsvcs/tests/ImplRepo/manual_start/test_i.h
@@ -0,0 +1,26 @@
+/* -*- C++ -*- $Id$ */
+
+#ifndef TEST_I_H_
+#define TEST_I_H_
+
+#include "testS.h"
+
+#if !defined (ACE_LACKS_PRAGMA_ONCE)
+#pragma once
+#endif /* ACE_LACKS_PRAGMA_ONCE */
+
+class  Test_i : public virtual POA_Test
+{
+public:
+  //Constructor
+  Test_i (void);
+
+  //Destructor
+  virtual ~Test_i (void);
+
+  virtual void contact (void);
+
+private:
+};
+
+#endif /* TEST_I_H_ */
diff --git a/TAO/tao/Invocation_Retry_State.cpp b/TAO/tao/Invocation_Retry_State.cpp
index 016060495c5..179572943f4 100644
--- a/TAO/tao/Invocation_Retry_State.cpp
+++ b/TAO/tao/Invocation_Retry_State.cpp
@@ -15,14 +15,16 @@ namespace
                          TAO::Invocation_Retry_Params &client_factory_params,
                          TAO::Invocation_Retry_Params &result)
   {
-    if (command_line_params.forward_on_exception_limit_[ex] !=
-        result.forward_on_exception_limit_[ex])
-      result.forward_on_exception_limit_[ex] =
-        command_line_params.forward_on_exception_limit_[ex];
-    else if (client_factory_params.forward_on_exception_limit_[ex] !=
-             result.forward_on_exception_limit_[ex])
-      result.forward_on_exception_limit_[ex] =
-        client_factory_params.forward_on_exception_limit_[ex];
+#define FOEL forward_on_exception_limit_
+    if (command_line_params.FOEL[ex] != result.FOEL[ex])
+      {
+        result.FOEL[ex] = command_line_params.FOEL[ex];
+      }
+    else if (client_factory_params.FOEL[ex] != result.FOEL[ex])
+      {
+        result.FOEL[ex] = client_factory_params.FOEL[ex];
+      }
+#undef FOEL
   }
 
   /// Calculate the retry parameters by giving a command line parameter
diff --git a/TAO/tao/Invocation_Utils.h b/TAO/tao/Invocation_Utils.h
index f17df10c510..fde5751d878 100644
--- a/TAO/tao/Invocation_Utils.h
+++ b/TAO/tao/Invocation_Utils.h
@@ -78,6 +78,8 @@ namespace TAO
       FOE_COMM_FAILURE     = 0x2,
       FOE_TRANSIENT        = 0x4,
       FOE_INV_OBJREF       = 0x8,
+      FOE_OBJ_ADAPTER      = 0x10,
+      FOE_NO_RESPONSE      = 0x20,
       FOE_ALL              = 0xFFFFFFFF
     };
 }
diff --git a/TAO/tao/Synch_Invocation.cpp b/TAO/tao/Synch_Invocation.cpp
index 8c2a3271a79..87c593f2cc6 100644
--- a/TAO/tao/Synch_Invocation.cpp
+++ b/TAO/tao/Synch_Invocation.cpp
@@ -32,6 +32,45 @@
 
 TAO_BEGIN_VERSIONED_NAMESPACE_DECL
 
+namespace
+{
+  int
+  excep_for_type (const char *tid)
+  {
+    if (ACE_OS_String::strcmp (tid, "IDL:omg.org/CORBA/TRANSIENT:1.0") == 0)
+      {
+        return TAO::FOE_TRANSIENT;
+      }
+    else if (ACE_OS_String::strcmp (tid,
+                                    "IDL:omg.org/CORBA/COMM_FAILURE:1.0") == 0)
+      {
+        return TAO::FOE_COMM_FAILURE;
+      }
+    else if (ACE_OS_String::strcmp (tid,
+                                    "IDL:omg.org/CORBA/OBJECT_NOT_EXIST:1.0") == 0)
+      {
+        return TAO::FOE_OBJECT_NOT_EXIST;
+      }
+    else if (ACE_OS_String::strcmp (tid,
+                                    "IDL:omg.org/CORBA/INV_OBJREF:1.0") == 0)
+      {
+        return TAO::FOE_INV_OBJREF;
+      }
+    else if (ACE_OS_String::strcmp (tid,
+                                    "IDL:omg.org/CORBA/OBJ_ADAPTER:1.0") == 0)
+      {
+        return TAO::FOE_OBJ_ADAPTER;
+      }
+    else if (ACE_OS_String::strcmp (tid,
+                                    "IDL:omg.org/CORBA/NO_RESPONSE:1.0") == 0)
+      {
+        return TAO::FOE_NO_RESPONSE;
+      }
+
+    return 0;
+  }
+}
+
 namespace TAO
 {
   Synch_Twoway_Invocation::Synch_Twoway_Invocation (
@@ -584,19 +623,7 @@ namespace TAO
         this->retry_state_->forward_on_exception_limit_used () &&
         (CORBA::CompletionStatus) completion == CORBA::COMPLETED_NO)
       {
-        if ((ACE_OS_String::strcmp (type_id.in (),
-                                   "IDL:omg.org/CORBA/TRANSIENT:1.0") == 0 &&
-             this->retry_state_->forward_on_exception_increment (TAO::FOE_TRANSIENT)) ||
-            (ACE_OS_String::strcmp (type_id.in (),
-                                   "IDL:omg.org/CORBA/COMM_FAILURE:1.0") == 0 &&
-             this->retry_state_->forward_on_exception_increment (TAO::FOE_COMM_FAILURE)) ||
-            (ACE_OS_String::strcmp (type_id.in (),
-                                   "IDL:omg.org/CORBA/OBJECT_NOT_EXIST:1.0") == 0 &&
-             this->retry_state_->forward_on_exception_increment (TAO::FOE_OBJECT_NOT_EXIST)) ||
-            (ACE_OS_String::strcmp (type_id.in (),
-                                   "IDL:omg.org/CORBA/INV_OBJREF:1.0") == 0 &&
-             this->retry_state_->forward_on_exception_increment (TAO::FOE_INV_OBJREF))
-            )
+        if (this->retry_state_->forward_on_exception_increment (excep_for_type (type_id.in ())))
           {
             retry_on_exception = true;
             this->retry_state_->sleep_at_starting_profile (*this->stub ());
@@ -605,38 +632,39 @@ namespace TAO
     else
       {
         int foe_kind = orb_params->forward_once_exception();
+        int ex_id = excep_for_type (type_id.in ());
 
-        retry_on_exception =
-          (CORBA::CompletionStatus) completion != CORBA::COMPLETED_YES
-          && (((foe_kind & TAO::FOE_TRANSIENT) == 0
-               && ACE_OS_String::strcmp (type_id.in (),
-                                         "IDL:omg.org/CORBA/TRANSIENT:1.0") == 0) ||
-              ACE_OS_String::strcmp (type_id.in (),
-                                     "IDL:omg.org/CORBA/OBJ_ADAPTER:1.0") == 0 ||
-              ACE_OS_String::strcmp (type_id.in (),
-                                     "IDL:omg.org/CORBA/NO_RESPONSE:1.0") == 0 ||
-              ((foe_kind & TAO::FOE_COMM_FAILURE) == 0
-               && ACE_OS_String::strcmp (type_id.in (),
-                                         "IDL:omg.org/CORBA/COMM_FAILURE:1.0") == 0) ||
-              (orb_params->forward_invocation_on_object_not_exist ()
-               && ACE_OS_String::strcmp (type_id.in (),
-                                         "IDL:omg.org/CORBA/OBJECT_NOT_EXIST:1.0") == 0) ||
-              (do_forward = ! this->stub ()->forwarded_on_exception ()
-               && ((((foe_kind & TAO::FOE_OBJECT_NOT_EXIST) == TAO::FOE_OBJECT_NOT_EXIST)
-                    && (ACE_OS_String::strcmp (type_id.in (),
-                                               "IDL:omg.org/CORBA/OBJECT_NOT_EXIST:1.0") == 0)) ||
-                   (((foe_kind & TAO::FOE_COMM_FAILURE) == TAO::FOE_COMM_FAILURE)
-                    && (ACE_OS_String::strcmp (type_id.in (),
-                                               "IDL:omg.org/CORBA/COMM_FAILURE:1.0") == 0)) ||
-                   (((foe_kind & TAO::FOE_TRANSIENT) == TAO::FOE_TRANSIENT)
-                    && (ACE_OS_String::strcmp (type_id.in (),
-                                               "IDL:omg.org/CORBA/TRANSIENT:1.0") == 0)) ||
-                   (((foe_kind & TAO::FOE_INV_OBJREF) == TAO::FOE_INV_OBJREF)
-                    && (ACE_OS_String::strcmp (type_id.in (),
-                                               "IDL:omg.org/CORBA/INV_OBJREF:1.0") == 0)))));
+        // this logic is a little confusing but prior to  Jul 24 2009, TRANSIENT,
+        // OBJ_ADAPTER, NO_RESPONSE, and COMM_FAILURE were always retried if possible.
+        // Later, the ForwardOnceOn* were added, which reverts to default behavior
+        // when not set.
+        if ((CORBA::CompletionStatus) completion != CORBA::COMPLETED_YES)
+          {
+            switch (ex_id)
+              {
+              case TAO::FOE_TRANSIENT:
+              case TAO::FOE_COMM_FAILURE:
+                retry_on_exception = (foe_kind & ex_id) == 0;
+                break;
+              case TAO::FOE_OBJ_ADAPTER:
+              case TAO::FOE_NO_RESPONSE:
+                retry_on_exception = true;
+                break;
+              case TAO::FOE_OBJECT_NOT_EXIST:
+                retry_on_exception = orb_params->forward_invocation_on_object_not_exist ();
+                break;
+              default:
+                break;
+              }
+            if (!retry_on_exception)
+              {
+                do_forward = !this->stub ()->forwarded_on_exception () &&
+                  ((foe_kind & ex_id) == ex_id);
+              }
+          }
       }
 
-    if (retry_on_exception)
+    if (retry_on_exception || do_forward)
       {
         // If we are here then possibly we'll need a restart.
         mon.set_status (TAO_INVOKE_RESTART);