ChangeLogTag:Wed Dec 10 16:39:25 2003 Ossama Othman <ossama@dre.vanderbilt.edu>

9 files changed, 326 insertions, 116 deletions

diff --git a/TAO/ChangeLog b/TAO/ChangeLog
index d226057a353..4895e7bd2a8 100644
--- a/TAO/ChangeLog
+++ b/TAO/ChangeLog
@@ -1,3 +1,63 @@
+Wed Dec 10 16:39:25 2003  Ossama Othman  <ossama@dre.vanderbilt.edu>
+
+	* tao/Synch_Invocation.cpp (handle_system_exception):
+
+	  Corrected repository IDs for non-fatal system exceptions being
+	  detected (TRANSIENT, OBJ_ADAPTER, NO_RESPONSE).  The
+	  "IDL:omg.org/CORBA/" prefix and ":1.0" suffix were missing, thus
+	  preventing code that should have been run in the presence of
+	  these exceptions from being run.
+
+	  Attempt to try another profile in the non-fatal exception case.
+	  This fixes a problem where valid profiles were not retried when
+	  they should have been.
+
+	* orbsvcs/orbsvcs/LoadBalancing/LB_LeastLoaded.h
+	  (LL_DEFAULT_LOAD_PERCENT_DIFF_CUTOFF):
+
+	  New constant used in thundering herd fix described below.
+
+	* orbsvcs/orbsvcs/LoadBalancing/LB_LeastLoaded.cpp (get_location):
+
+	  Fixed problem where least loaded object group member was not
+	  always selected, thus reducing scalability.
+
+	  If two locations have loads that differ by a "small" amount
+	  (i.e. LL_DEFAULT_LOAD_PERCENT_DIFF_CUTOFF), choose one of them
+	  at random.  This helps alleviate a thundering herd phenomenon
+	  that could occur when an object group member becomes available
+	  for request handling.  Thanks to Carlos, Marina and Jody at ATD
+	  for coming up with this solution to this form of the thundering
+	  herd problem.
+
+	* orbsvcs/orbsvcs/LoadBalancing/LB_ServerRequestInterceptor.cpp
+	  (receive_request_service_contexts):
+
+	  Always allow LoadMonitor and LoadAlert operations to proceed
+	  regardless of the alert condition.  Addresses a problem where it
+	  was not possible to disable alert states and query loads from a
+	  LoadMonitor collocated with a server once a load alert was
+	  enabled.
+
+	* orbsvcs/performance-tests/LoadBalancing/ORBInitializer.h:
+
+	  Improved documentation of some class members.
+	
+	* orbsvcs/performance-tests/LoadBalancing/ORBInitializer.cpp:
+
+	  Cosmetic change.
+
+	* orbsvcs/performance-tests/LoadBalancing/RPS_Monitor.cpp (loads):
+
+	  If the elapsed-time is zero, set the load to zero to prevent a
+	  division-by-zero error.
+
+	* orbsvcs/performance-tests/LoadBalancing/Roundtrip.cpp
+	  (test_method):
+
+	  Commented out a bunch of code that should not be used when
+	  performing benchmarks.
+
 Wed Dec 10 13:58:41 2003  Chris Cleeland  <cleeland_c@ociweb.com>
 
 	* tao/default_client.cpp (parse_args): Corrected erroneous option
@@ -7,26 +67,26 @@ Wed Dec 10 13:58:41 2003  Chris Cleeland  <cleeland_c@ociweb.com>
 	* tao/IIOP_Transport.cpp (recv): Corrected method name printed in
 	  debug message.
 
-	* tao/GIOP_Message_Base.cpp: 
-        * tao/GIOP_Message_Base.h: 
-        * tao/GIOP_Message_Base.i: 
-        * tao/GIOP_Message_Generator_Parser_Impl.inl: 
-        * tao/GIOP_Message_Lite.cpp: 
-        * tao/GIOP_Message_Lite.h: 
-        * tao/GIOP_Message_State.cpp: 
-        * tao/GIOP_Message_State.h: 
-        * tao/GIOP_Message_State.inl: 
-        * tao/Incoming_Message_Queue.cpp: 
-        * tao/Incoming_Message_Queue.h: 
-        * tao/Incoming_Message_Queue.inl: 
-        * tao/Pluggable_Messaging.h: 
-        * tao/Transport.cpp: 
-        * tao/Transport.h: 
+	* tao/GIOP_Message_Base.cpp:
+        * tao/GIOP_Message_Base.h:
+        * tao/GIOP_Message_Base.i:
+        * tao/GIOP_Message_Generator_Parser_Impl.inl:
+        * tao/GIOP_Message_Lite.cpp:
+        * tao/GIOP_Message_Lite.h:
+        * tao/GIOP_Message_State.cpp:
+        * tao/GIOP_Message_State.h:
+        * tao/GIOP_Message_State.inl:
+        * tao/Incoming_Message_Queue.cpp:
+        * tao/Incoming_Message_Queue.h:
+        * tao/Incoming_Message_Queue.inl:
+        * tao/Pluggable_Messaging.h:
+        * tao/Transport.cpp:
+        * tao/Transport.h:
         * tao/orbconf.h:
 	* tao/Strategies/DIOP_Transport.cpp
 	* tao/Strategies/SHMIOP_Transport.cpp
 	* tao/Strategies/SHMIOP_Transport.h
-	
+
 	  Integrated aggregate fix to infamous Parse Magic Bytes (PMB)
  	  problem.  The fix was originally developed in OCI's repository
  	  and migrated here.  An overall description of the problem and
@@ -108,7 +168,7 @@ Tue Mar 18 14:57:07 2003  Chris Cleeland  <cleeland_c@ociweb.com>
 	* tao/RTCORBA/RT_ORBInitializer.h:
 	* tao/RTCORBA/RT_ORB_Loader.cpp:
 	* tao/RTCORBA/RT_Protocols_Hooks.cpp:
-	
+
 	  Merged in changes from TAO 1.3.1.  The actual tag for the merge
 	  was pmb_branch_mainline_mergeout_1.  The command was
 
@@ -191,7 +251,7 @@ Mon Mar 17 11:09:00 2003  Chris Cleeland  <cleeland_c@ociweb.com>
 	  the queue.  Also, removed old code leftover from when the
 	  incoming queue could have partial GIOP messages in it.  Finally,
 	  insured that the return value matched previous requirements.
-	  
+
 
 Thu Mar  6 15:27:14 2003  Chris Cleeland  <cleeland_c@ociweb.com>
 
@@ -264,7 +324,7 @@ Wed Feb 26 21:44:34 2003  Chris Cleeland  <cleeland_c@ociweb.com>
 	  Make message_type() static.
 
 	* TAO/tao/GIOP_Message_Generator_Parser_Impl.inl:
-	
+
 	  Reimplement check_revision() to be exact about which GIOP
 	  revisions are okay.  This is different from before where it
 	  loosely assumed that everything was valid unless it want higher
@@ -459,9 +519,9 @@ Wed Feb 26 21:44:34 2003  Chris Cleeland  <cleeland_c@ociweb.com>
 	* TAO/tests/InterOp-Naming/INS_test_client.cpp:
 
           Changes that don't appear to be related in any way to PMB...
-  
-----------------------------------------  
-    
+
+----------------------------------------
+
 Wed Dec 10 15:12:34 UTC 2003  Johnny Willemsen  <jwillemsen@remedy.nl>
 
         * tests/Smart_Proxies/Collocation/Smart_Proxy_Impl.cpp:
@@ -703,10 +763,17 @@ Mon Dec  8 19:33:03 2003  Balachandran Natarajan  <bala@dre.vanderbilt.edu>
 
         * tao/Synch_Invocation.cpp:
 
+<<<<<<< ChangeLog
+	  Fixed a minor problem with reinvocation. We were not handling
+	  errors properly, more precisely the restart errors if they show
+	  up during wait_for_reply (). Thanks to Milan Cvetkovic for
+	  motivating this change.
+=======
           Fixed a minor problem with reinvocation. We were not handling
           errors properly, more precisely the retsrat errors if they show
           up during wait_for_reply (). Thanks to Milan Cvetkovic for
           motivating this change.
+>>>>>>> 1.3209
 
         * tao/Messaging/Asynch_Invocation.cpp:
 
diff --git a/TAO/orbsvcs/orbsvcs/LoadBalancing/LB_LeastLoaded.cpp b/TAO/orbsvcs/orbsvcs/LoadBalancing/LB_LeastLoaded.cpp
index 37f458dd722..9252ffaf438 100644
--- a/TAO/orbsvcs/orbsvcs/LoadBalancing/LB_LeastLoaded.cpp
+++ b/TAO/orbsvcs/orbsvcs/LoadBalancing/LB_LeastLoaded.cpp
@@ -1,5 +1,3 @@
-// -*- C++ -*-
-
 #include "LB_LeastLoaded.h"
 #include "LB_LoadMap.h"
 #include "LB_Random.h"
@@ -211,6 +209,10 @@ TAO_LB_LeastLoaded::next_member (
 
   if (found_location)
     {
+//       ACE_DEBUG ((LM_DEBUG,
+//                   "RETURNING REFERENCE FOR LOCATION \"%s\"\n",
+//                   location[0].id.in ()));
+
       return load_manager->get_member_ref (object_group,
                                            location
                                            ACE_ENV_ARG_PARAMETER);
@@ -340,7 +342,7 @@ TAO_LB_LeastLoaded::get_location (
   ACE_ENV_ARG_DECL)
 {
   CORBA::Float min_load = FLT_MAX;  // Start out with the largest
-                                    // possible.
+                                    // positive value.
 
   CORBA::ULong location_index = 0;
   CORBA::Boolean found_location = 0;
@@ -374,14 +376,16 @@ TAO_LB_LeastLoaded::get_location (
 
 //           ACE_DEBUG ((LM_DEBUG,
 //                       "LOC = %u"
-//                       "\tCOND = %d"
+//                       "\tC = %d"
 //                       "\treject = %f"
-//                       "\tload = %f\n",
+//                       "\tload = %f\n"
+//                       "\tmin_load = %f\n",
 //                       i,
 //                       (this->reject_threshold_ == 0
 //                        || load.value < this->reject_threshold_),
 //                       this->reject_threshold_,
-//                       load.value));
+//                       load.value,
+//                       min_load));
 
           if ((this->reject_threshold_ == 0
                || load.value < this->reject_threshold_)
@@ -391,10 +395,94 @@ TAO_LB_LeastLoaded::get_location (
 //                           "**** LOAD == %f\n",
 //                           load.value));
 
-              min_load = load.value;
-              location_index = i;
-              found_location = 1;
+              if (i > 0 && load.value != 0)
+                {
+                  /*
+                    percent difference =
+                      (min_load - load.value) / load.value
+                      == (min_load / load.value) - 1
+
+                      The latter form is used to avoid a potential
+                      arithmetic overflow problem, such as when
+                      (min_load - load.value) > FLT_MAX, assuming that
+                      either load.value is negative and min_load is
+                      positive, or vice versa.
+                  */
+                  const CORBA::Float percent_diff =
+                    (min_load / load.value) - 1;
+
+                  /*
+                    A "thundering herd" phenomenon may occur when
+                    location loads are basically the same (e.g. only
+                    differ by a very small amount), where one object
+                    group member ends up receiving the majority of
+                    requests from different clients.  In order to
+                    prevent a single object group member from
+                    receiving such request bursts, one of two equally
+                    loaded locations is chosen at random.  Thanks to
+                    Carlos, Marina and Jody at ATD for coming up with
+                    this solution to this form of the thundering herd
+                    problem.
+
+                    See the documentation for
+                    TAO_LB::LL_DEFAULT_LOAD_PERCENT_DIFF_CUTOFF in
+                    LB_LeastLoaded.h for additional information.
+                  */
+                  if (percent_diff <= TAO_LB::LL_DEFAULT_LOAD_PERCENT_DIFF_CUTOFF)
+                    {
+                      // Prevent integer arithmetic overflow.
+                      const CORBA::Float NUM_MEMBERS = 2;
+
+                      // n == 0:  Use previously selected location.
+                      // n == 1:  Use current location.
+                      const CORBA::ULong n =
+                        ACE_static_cast (CORBA::ULong,
+                                         NUM_MEMBERS * ACE_OS::rand ()
+                                         / (RAND_MAX + 1.0));
+
+                      ACE_ASSERT (n == 0 || n == 1);
+
+                      if (n == 1)
+                        {
+                          min_load = load.value;
+                          location_index = i;
+                          found_location = 1;
+
+//                           ACE_DEBUG ((LM_DEBUG,
+//                                       "** NEW MIN_LOAD == %f\n",
+//                                       min_load));
+                        }
+
+//                       if (n == 0)
+//                         ACE_DEBUG ((LM_DEBUG, "^^^^^ PREVIOUS LOCATION\n"));
+//                       else
+//                         ACE_DEBUG ((LM_DEBUG, "^^^^^ CURRENT LOCATION\n"));
+
+                    }
+                  else
+                    {
+                      min_load = load.value;
+                      location_index = i;
+                      found_location = 1;
+
+//                       ACE_DEBUG ((LM_DEBUG,
+//                                   "***** NEW MIN_LOAD == %f\n",
+//                                   min_load));
+                    }
+                }
+              else
+                {
+                  min_load = load.value;
+                  location_index = i;
+                  found_location = 1;
+
+//                   ACE_DEBUG ((LM_DEBUG,
+//                               "NEW MIN_LOAD == %f\n",
+//                               min_load));
+                }
             }
+
+          // ACE_DEBUG ((LM_DEBUG, "NEW MIN_LOAD == %f\n", min_load));
         }
       ACE_CATCH (CosLoadBalancing::LocationNotFound, ex)
         {
@@ -407,7 +495,7 @@ TAO_LB_LeastLoaded::get_location (
 
 //   ACE_DEBUG ((LM_DEBUG,
 //               "FOUND_LOAD     == %u\n"
-//               "FOUND_LOCATION = %u\n",
+//               "FOUND_LOCATION == %u\n",
 //               found_load,
 //               found_location));
 
@@ -420,6 +508,8 @@ TAO_LB_LeastLoaded::get_location (
         location = locations[location_index];
       else if (this->reject_threshold_ != 0)
         ACE_THROW_RETURN (CORBA::TRANSIENT (), 0);
+
+//       ACE_DEBUG ((LM_DEBUG, "LOCATION ID == %s\n", location[0].id.in ()));
     }
 
 //   ACE_DEBUG ((LM_DEBUG, "LOCATED = %u\n", location_index));
diff --git a/TAO/orbsvcs/orbsvcs/LoadBalancing/LB_LeastLoaded.h b/TAO/orbsvcs/orbsvcs/LoadBalancing/LB_LeastLoaded.h
index 34003d356c5..e11f443c091 100644
--- a/TAO/orbsvcs/orbsvcs/LoadBalancing/LB_LeastLoaded.h
+++ b/TAO/orbsvcs/orbsvcs/LoadBalancing/LB_LeastLoaded.h
@@ -33,11 +33,33 @@ namespace TAO_LB
    *
    * Defaults defined by the Load Balancing specification.
    */
+  //@{
   const CORBA::Float LL_DEFAULT_CRITICAL_THRESHOLD = 0;
   const CORBA::Float LL_DEFAULT_REJECT_THRESHOLD = 0;
   const CORBA::Float LL_DEFAULT_TOLERANCE = 1;
   const CORBA::Float LL_DEFAULT_DAMPENING = 0;
   const CORBA::Float LL_DEFAULT_PER_BALANCE_LOAD = 0;
+  //@}
+
+  /**
+   * @name TAO-specific LeastLoaded strategy parameters.
+   *
+   * Parameters internal to TAO's LeastLoaded strategy
+   * implementation.
+   */
+  //@{
+  /// Percentage difference between two load values that determines
+  /// whether the loads are considered equivalent.
+  /**
+   * If the percent difference between two loads, i.e.:
+   *   (Old Load - New Load) / New Load
+   * is less than or equal to this value, the two loads will be
+   * considered equivalent.  In such a case, an object group member
+   * residing at the location corresponding to one of the two loads
+   * will be selected at random.
+   */
+  const CORBA::Float LL_DEFAULT_LOAD_PERCENT_DIFF_CUTOFF = 0.01;  // 1%
+  //@}
 }
 
 /**
diff --git a/TAO/orbsvcs/orbsvcs/LoadBalancing/LB_ServerRequestInterceptor.cpp b/TAO/orbsvcs/orbsvcs/LoadBalancing/LB_ServerRequestInterceptor.cpp
index c8359eb57eb..a4536839dbd 100644
--- a/TAO/orbsvcs/orbsvcs/LoadBalancing/LB_ServerRequestInterceptor.cpp
+++ b/TAO/orbsvcs/orbsvcs/LoadBalancing/LB_ServerRequestInterceptor.cpp
@@ -34,13 +34,21 @@ TAO_LB_ServerRequestInterceptor::destroy (ACE_ENV_SINGLE_ARG_DECL_NOT_USED)
 
 void
 TAO_LB_ServerRequestInterceptor::receive_request_service_contexts (
-    PortableInterceptor::ServerRequestInfo_ptr /* ri */
+    PortableInterceptor::ServerRequestInfo_ptr ri
     ACE_ENV_ARG_DECL)
   ACE_THROW_SPEC ((CORBA::SystemException,
                    PortableInterceptor::ForwardRequest))
 {
   if (this->load_alert_.alerted ())
     {
+      CORBA::String_var op = ri->operation (ACE_ENV_SINGLE_ARG_PARAMETER);
+      ACE_CHECK;
+
+      if (ACE_OS::strcmp (op.in (), "_get_loads") == 0        // LoadMonitor
+          || ACE_OS::strcmp (op.in (), "disable_alert") == 0  // LoadAlert
+          || ACE_OS::strcmp (op.in (), "enable_alert") == 0)  // LoadAlert
+        return;  // Do not redirect.
+
 #if 0
       ACE_TRY
         {
diff --git a/TAO/orbsvcs/performance-tests/LoadBalancing/ORBInitializer.cpp b/TAO/orbsvcs/performance-tests/LoadBalancing/ORBInitializer.cpp
index a52eb694474..c87697bb7d1 100644
--- a/TAO/orbsvcs/performance-tests/LoadBalancing/ORBInitializer.cpp
+++ b/TAO/orbsvcs/performance-tests/LoadBalancing/ORBInitializer.cpp
@@ -33,7 +33,6 @@ ORBInitializer::post_init (
     ACE_ENV_ARG_DECL)
   ACE_THROW_SPEC ((CORBA::SystemException))
 {
-
   ACE_NEW_THROW_EX (this->interceptor_,
                     ServerRequestInterceptor,
                     CORBA::NO_MEMORY (
diff --git a/TAO/orbsvcs/performance-tests/LoadBalancing/ORBInitializer.h b/TAO/orbsvcs/performance-tests/LoadBalancing/ORBInitializer.h
index 4f5dfe3564d..d089c75ee9e 100644
--- a/TAO/orbsvcs/performance-tests/LoadBalancing/ORBInitializer.h
+++ b/TAO/orbsvcs/performance-tests/LoadBalancing/ORBInitializer.h
@@ -84,6 +84,7 @@ private:
    */
   TAO_LB_LoadAlert load_alert_;
 
+  /// Interceptor that counts requests.
   ServerRequestInterceptor * interceptor_;
 
 };
diff --git a/TAO/orbsvcs/performance-tests/LoadBalancing/RPS_Monitor.cpp b/TAO/orbsvcs/performance-tests/LoadBalancing/RPS_Monitor.cpp
index 3b3c49a505e..945086ea8ea 100644
--- a/TAO/orbsvcs/performance-tests/LoadBalancing/RPS_Monitor.cpp
+++ b/TAO/orbsvcs/performance-tests/LoadBalancing/RPS_Monitor.cpp
@@ -78,7 +78,11 @@ RPS_Monitor::loads (ACE_ENV_SINGLE_ARG_DECL)
   load_list->length (1);
 
   load_list[0].id = CosLoadBalancing::RequestsPerSecond;
-  load_list[0].value = request_count / elapsed_time.msec () * 1000;
+
+  if (elapsed_time == ACE_Time_Value::zero)
+    load_list[0].value = 0;
+  else
+    load_list[0].value = request_count / elapsed_time.msec () * 1000;
 
   // Strictly for debugging or
   ACE_DEBUG ((LM_DEBUG, "%f\n", load_list[0].value));
diff --git a/TAO/orbsvcs/performance-tests/LoadBalancing/Roundtrip.cpp b/TAO/orbsvcs/performance-tests/LoadBalancing/Roundtrip.cpp
index c4203be4c46..1292cd5de50 100644
--- a/TAO/orbsvcs/performance-tests/LoadBalancing/Roundtrip.cpp
+++ b/TAO/orbsvcs/performance-tests/LoadBalancing/Roundtrip.cpp
@@ -16,20 +16,20 @@ Roundtrip::Roundtrip (CORBA::ORB_ptr orb)
 
 Test::Timestamp
 Roundtrip::test_method (Test::Timestamp send_time,
-                        Test::number cl_number,
-                        Test::number it_number
+                        Test::number /* cl_number */,
+                        Test::number /* it_number */
                         ACE_ENV_ARG_DECL_NOT_USED)
   ACE_THROW_SPEC ((CORBA::SystemException))
 {
-  if (it_number < 100)
-    {
-      printf("Client number is %d\n", cl_number);
-    }
+//   if (it_number < 100)
+//     {
+//       printf("Client number is %d\n", cl_number);
+//     }
 
-  if ((it_number % 2) == 0)
-    {
-      (void) ACE::is_prime (20000, 2, 10000);
-    }
+//   if ((it_number % 2) == 0)
+//     {
+//       (void) ACE::is_prime (20000, 2, 10000);
+//     }
 
   return send_time;
 }
diff --git a/TAO/tao/Synch_Invocation.cpp b/TAO/tao/Synch_Invocation.cpp
index fcfd76c3340..ab16a8f4715 100644
--- a/TAO/tao/Synch_Invocation.cpp
+++ b/TAO/tao/Synch_Invocation.cpp
@@ -1,4 +1,4 @@
-//$Id$
+// $Id$
 
 #include "Synch_Invocation.h"
 #include "Profile_Transport_Resolver.h"
@@ -21,16 +21,19 @@
 # include "Synch_Invocation.inl"
 #endif /* __ACE_INLINE__ */
 
+
 ACE_RCSID (tao,
            Synch_Invocation,
            "$Id$")
 
+
 namespace TAO
 {
-  Synch_Twoway_Invocation::Synch_Twoway_Invocation (CORBA::Object_ptr otarget,
-                                                    Profile_Transport_Resolver &resolver,
-                                                    TAO_Operation_Details &detail,
-                                                    bool response_expected)
+  Synch_Twoway_Invocation::Synch_Twoway_Invocation (
+    CORBA::Object_ptr otarget,
+    Profile_Transport_Resolver &resolver,
+    TAO_Operation_Details &detail,
+    bool response_expected)
     : Remote_Invocation (otarget,
                          resolver,
                          detail,
@@ -103,11 +106,10 @@ namespace TAO
 
         countdown.update ();
 
-        s =
-          this->send_message (cdr,
-                              TAO_Transport::TAO_TWOWAY_REQUEST,
-                              max_wait_time
-                              ACE_ENV_ARG_PARAMETER);
+        s = this->send_message (cdr,
+                                TAO_Transport::TAO_TWOWAY_REQUEST,
+                                max_wait_time
+                                ACE_ENV_ARG_PARAMETER);
         ACE_TRY_CHECK;
 
 #if TAO_HAS_INTERCEPTORS == 1
@@ -117,7 +119,7 @@ namespace TAO
         // before we leave.
         if (s == TAO_INVOKE_RESTART)
           {
-            Invocation_Status tmp =
+            const Invocation_Status tmp =
               this->receive_other_interception (ACE_ENV_SINGLE_ARG_PARAMETER);
             ACE_TRY_CHECK;
 
@@ -214,7 +216,7 @@ namespace TAO
     ACE_CATCHANY
       {
 #if TAO_HAS_INTERCEPTORS == 1
-        PortableInterceptor::ReplyStatus status =
+        const PortableInterceptor::ReplyStatus status =
           this->handle_any_exception (&ACE_ANY_EXCEPTION
                                       ACE_ENV_ARG_PARAMETER);
         ACE_TRY_CHECK;
@@ -223,7 +225,7 @@ namespace TAO
             status == PortableInterceptor::TRANSPORT_RETRY)
           s = TAO_INVOKE_RESTART;
         else if (status == PortableInterceptor::SYSTEM_EXCEPTION
-            || status == PortableInterceptor::USER_EXCEPTION)
+                 || status == PortableInterceptor::USER_EXCEPTION)
 #endif /*TAO_HAS_INTERCEPTORS*/
           ACE_RE_THROW;
       }
@@ -232,7 +234,7 @@ namespace TAO
     ACE_CATCHALL
       {
 #if TAO_HAS_INTERCEPTORS == 1
-        PortableInterceptor::ReplyStatus st =
+        const PortableInterceptor::ReplyStatus st =
           this->handle_all_exception (ACE_ENV_SINGLE_ARG_PARAMETER);
         ACE_TRY_CHECK;
 
@@ -266,12 +268,12 @@ namespace TAO
      * exception. Success alone is returned through the return value.
      */
 
-    int reply_error =
+    const int reply_error =
       this->resolver_.transport ()->wait_strategy ()->wait (max_wait_time,
                                                             rd);
     if (TAO_debug_level > 0 && max_wait_time != 0)
       {
-        CORBA::ULong msecs = max_wait_time->msec ();
+        const CORBA::ULong msecs = max_wait_time->msec ();
 
         ACE_DEBUG ((LM_DEBUG,
                     "TAO (%P|%t) - Synch_Twoway_Invocation::wait_for_reply, "
@@ -298,24 +300,26 @@ namespace TAO
         if (errno == ETIME)
           {
             // If the unbind succeeds then thrown an exception to the
-            // application, else just collect the reply and dispatch that to the
-            // application.
-            // NOTE: A fragile synchronization is provided when using the Muxed
-            // Transport strategy. We could infact be a follower thread getting
-            // timedout in the LF whereas the dispatching thread could be
-            // on the reply_dispatcher that we created. This would lead bad
-            // crashes. To get around that, the call to unbind_dispatcher ()
-            // will wait on the lock on the Muxed_Transport_Strategy if
+            // application, else just collect the reply and dispatch
+            // that to the application.
+            //
+            // NOTE: A fragile synchronization is provided when using
+            // the Muxed Transport strategy. We could infact be a
+            // follower thread getting timedout in the LF whereas the
+            // dispatching thread could be on the reply_dispatcher
+            // that we created. This would lead bad crashes. To get
+            // around that, the call to unbind_dispatcher () will wait
+            // on the lock on the Muxed_Transport_Strategy if
             // dispatching has started. This is fragile.
             if (bd.unbind_dispatcher () == 0)
               {
                 // Just a timeout with completed_maybe, don't close
                 // the connection or  anything
                 ACE_THROW_RETURN (CORBA::TIMEOUT (
-                    CORBA::SystemException::_tao_minor_code (
-                        TAO_TIMEOUT_RECV_MINOR_CODE,
-                        errno),
-                    CORBA::COMPLETED_MAYBE),
+                                    CORBA::SystemException::_tao_minor_code (
+                                      TAO_TIMEOUT_RECV_MINOR_CODE,
+                                      errno),
+                                    CORBA::COMPLETED_MAYBE),
                                   TAO_INVOKE_FAILURE);
               }
           }
@@ -325,7 +329,8 @@ namespace TAO
             this->resolver_.transport ()->close_connection ();
             this->resolver_.stub ()->reset_profiles ();
 
-            return this->orb_core ()->service_raise_comm_failure (
+            return
+              this->orb_core ()->service_raise_comm_failure (
                 this->details_.request_service_context ().service_info (),
                 this->resolver_.profile ()
                 ACE_ENV_ARG_PARAMETER);
@@ -518,8 +523,7 @@ namespace TAO
                                                     ACE_ENV_ARG_DECL)
     ACE_THROW_SPEC ((CORBA::SystemException))
   {
-    Reply_Guard mon (this,
-                     TAO_INVOKE_FAILURE);
+    Reply_Guard mon (this, TAO_INVOKE_FAILURE);
 
     if (TAO_debug_level > 3)
       ACE_DEBUG ((LM_DEBUG,
@@ -548,33 +552,51 @@ namespace TAO
                           TAO_INVOKE_FAILURE);
       }
 
-    {
-      // Start the special case for FTCORBA.
-      /**
-       * There has been a unanimous view that this is not the right way
-       * to do things. But a need to be compliant is forcing us into
-       * this.
-       */
-      if (((ACE_OS_String::strcmp (type_id.in (),
-                                  "TRANSIENT") == 0) ||
-           (ACE_OS_String::strcmp (type_id.in (),
-                                  "OBJ_ADAPTER") == 0) ||
-           (ACE_OS_String::strcmp (type_id.in (),
-                                  "NO_RESPONSE") == 0)) &&
-          (CORBA::CompletionStatus) completion != CORBA::COMPLETED_YES)
+    // Special handling for non-fatal system exceptions.
+    //
+    // Note that we are careful to retain "at most once" semantics.
+    if ((ACE_OS_String::strcmp (type_id.in (),
+                                "IDL:omg.org/CORBA/TRANSIENT:1.0")   == 0 ||
+         ACE_OS_String::strcmp (type_id.in (),
+                                "IDL:omg.org/CORBA/OBJ_ADAPTER:1.0") == 0 ||
+         ACE_OS_String::strcmp (type_id.in (),
+                                "IDL:omg.org/CORBA/NO_RESPONSE:1.0") == 0) &&
+        (CORBA::CompletionStatus) completion != CORBA::COMPLETED_YES)
+      {
         {
-         Invocation_Status s =
-           this->orb_core ()->service_raise_transient_failure (
-             this->details_.request_service_context ().service_info (),
-             this->resolver_.profile ()
-             ACE_ENV_ARG_PARAMETER);
-         ACE_CHECK_RETURN (TAO_INVOKE_FAILURE);
-
-         if (s == TAO_INVOKE_RESTART)
-           return s;
-         // else fall through and raise an exception.
+          // Start the special case for FTCORBA.
+          /**
+           * There has been a unanimous view that this is not the
+           * right way to do things. But a need to be compliant is
+           * forcing us into this.
+           */
+          const Invocation_Status s =
+            this->orb_core ()->service_raise_transient_failure (
+              this->details_.request_service_context ().service_info (),
+              this->resolver_.profile ()
+              ACE_ENV_ARG_PARAMETER);
+          ACE_CHECK_RETURN (TAO_INVOKE_FAILURE);
+
+          if (s == TAO_INVOKE_RESTART)
+            return s;
         }
-    }
+
+        // Attempt profile retry.
+        /**
+         * @note A location forwarding loop may occur where a client
+         *       is bounced from the original target to the forwarded
+         *       target and back if the application is not equipped to
+         *       handle retries of previously called targets.  TAO may
+         *       be able to help in this case but it ultimately ends
+         *       up being an application issue.
+         */
+        if (this->resolver_.stub ()->next_profile_retry ())
+          {
+            return TAO_INVOKE_RESTART;
+          }
+
+        // Fall through and raise an exception.
+      }
 
     CORBA::SystemException *ex =
       TAO_Exceptions::create_system_exception (type_id.in ()
@@ -591,15 +613,16 @@ namespace TAO
                         TAO_INVOKE_FAILURE);
       }
 
-    ex->minor (minor);
-    ex->completed (CORBA::CompletionStatus (completion));
-
 #if defined (TAO_HAS_EXCEPTIONS)
     // Without this, the call to create_system_exception() above
     // causes a memory leak. On platforms without native exceptions,
     // the CORBA::Environment class manages the memory.
     auto_ptr<CORBA::SystemException> safety (ex);
 #endif
+
+    ex->minor (minor);
+    ex->completed (CORBA::CompletionStatus (completion));
+
     if (TAO_debug_level > 4)
       ACE_DEBUG ((LM_DEBUG,
                   "TAO (%P|%t) - Synch_Twoway_Invocation::"
@@ -612,19 +635,16 @@ namespace TAO
 
     return TAO_INVOKE_SYSTEM_EXCEPTION;
   }
-/*================================================================================*/
-
-  Synch_Oneway_Invocation::Synch_Oneway_Invocation (CORBA::Object_ptr otarget,
-                                                    Profile_Transport_Resolver &r,
-                                                    TAO_Operation_Details &d)
-    : Synch_Twoway_Invocation (otarget,
-                               r,
-                               d,
-                               false)
-  {
-  }
 
+  // =========================================================================
 
+  Synch_Oneway_Invocation::Synch_Oneway_Invocation (
+    CORBA::Object_ptr otarget,
+    Profile_Transport_Resolver &r,
+    TAO_Operation_Details &d)
+    : Synch_Twoway_Invocation (otarget, r, d, false)
+  {
+  }
 
   Invocation_Status
   Synch_Oneway_Invocation::remote_oneway (ACE_Time_Value *max_wait_time
@@ -638,7 +658,6 @@ namespace TAO
 
     if (response_flags == CORBA::Octet (Messaging::SYNC_WITH_SERVER) ||
         response_flags == CORBA::Octet (Messaging::SYNC_WITH_TARGET))
-
       return Synch_Twoway_Invocation::remote_twoway (max_wait_time
                                                      ACE_ENV_ARG_PARAMETER);