ChangeLogTag: Wed Aug 1 15:54:01 UTC 2007 Dale Wilson <wilsond@ociweb.com>

21 files changed, 791 insertions, 472 deletions

diff --git a/TAO/ChangeLog b/TAO/ChangeLog
index 64d09c84c24..b7945196df4 100644
--- a/TAO/ChangeLog
+++ b/TAO/ChangeLog
@@ -1,3 +1,113 @@
+Wed Aug  1 15:54:01 UTC 2007  Dale Wilson  <wilsond@ociweb.com>
+          Corrections for bugs #2934 and #2935: connection problems in
+          the transport cache.
+
+          Briefly: the change involves adding the transport to the
+          cache at the time the connection is initiated rather than
+          waiting until the connection is complete.   This avoids the
+          situation where *way* too many connections are started in a
+          nested upcall and/or multithreading situation because none
+          of the connection attempts know the others are already in
+          progress.
+          At the same time it eliminates the wait-for-condition code
+          technique for honoring the -MaxMuxedConnections. The old
+          technique is unsafe and ineffective.
+
+        * tao/Cache_Entries.h:
+        * tao/Cache_Entries.inl:
+          Declare new cache entry status:  ENTRY_CONNECTING
+          Make the recycle_state method const (it's poorly named, but I'm not
+          fixing that now.)
+
+        * tao/IIOP_Connector.cpp:
+          Use RAII to manage tlist (list of pending connections) during parallel
+          connection.
+          Update the cache (rather than adding a new entry) when connection is
+          complete.
+          Change the way the transport is registered with the reactor so that it
+          is always registered before necessary.  Otherwise there was a race
+          condition when thread A tried to use a newly established connection
+          before thread B had "quite" finished preparing it for use.
+
+        * tao/Transport.h:
+          Declare new method: register_if_necessary to consolodate the various
+          attempts to register the transport with the reactor.
+
+        * tao/Transport.cpp:
+          Implement and use register_if_necessary
+          Update transport cache status in the post_connect method
+
+        * tao/Transport_Cache_Manager.h:
+        * tao/Transport_Cache_Manager.inl:
+        * tao/Transport_Cache_Manager.cpp:
+          Change find_transport to return one of the following statuses:
+            CACHE_FOUND_NONE
+            CACHE_FOUND_CONNECTING
+            CACHE_FOUND_BUSY
+            CACHE_FOUND_AVAILABLE
+          The more than one status applies, the last one in the above list
+          overrides (i.e. if there's one available, never mind the other stuff.)
+          Add an additional argument to find_transport to return the count of
+          busy transport's found when none were available.  This helps the
+          Transport_Connector to honor -MaxMuxedConnections.
+          Remove the attempt to open a new connection from the cache.  It belongs
+          in the Transport_Connector.
+          Improve support for updating cache entry status.
+
+        * tao/Transport_Connector.h:
+        * tao/Transport_Connector.cpp:
+          Eliminate TransportCleanupGuard.  It was ill conceived.
+          Supply new argument to and accept new status returns from find_transport.
+          New wait_for_transport method allows waiting for a connection to complete
+          even if it was started elsewhere.
+          The TAO_Connector::connect method has been reorganized.  It now consists
+          of a while loop that retries the search for the transport in the cache
+          until success, timeout, error, or a return because this is a non-blocking attempt.
+
+          The transport found by this method *ALWAYS* comes from the cache.   There
+          was code that attempted to short-circuit this for newly established connections
+          but it led to quite a few subtle bugs.  The new approach shouldn't be any
+          slower than the old one, but if is, there is room for improvment in the
+          cache now that it doesn't have to handle all the special cases.   (i.e.
+          consider long and carefully before trying to improve performance by reintroducing
+          the short-circuit-on-new-connection code.)
+
+        * orbsvcs/orbsvcs/SSLIOP/IIOP_SSL_Connector.cpp:
+        * orbsvcs/orbsvcs/SSLIOP/SSLIOP_Connector.cpp:
+        * tao/Profile_Transport_Resolver.cpp:
+        * tao/Strategies/UIOP_Connector.cpp:
+        * tests/Bug_1476_Test/client.cpp:
+          Supply additional argument to wait_for_connection_completion
+          Supply new argument to Transport_Cache_Manager::find_transport
+          Recognize new return status from Transport_Cache_Manager::find_transport
+          [no functional change]
+
+        * tao/IIOP_Transport.cpp:
+          Log status code on error.
+
+        * tao/Leader_Follower.cpp:
+          Ignore null tranport pointer (effects logging only)
+
+        * tests/AMH_Oneway/client.cpp:
+          Add a sleep before process exit.  To quote the internal documentation:
+           // The following sleep is a workaround for a defect in the Windows
+           // implementation of sockets (Win XP)
+           // The when this client exits after writing to a localhost socket
+           // Windows discards any data that has not been read by the server.
+           // The sleep gives the server time to catch up.  num_calls/2 gives
+           // it half a second per request which *really* should be overkill, but
+           // it also means the client will terminate before the server actually
+           // handles the requests (a good thing).
+           // I'm still trying to decide whether this should be a bugzilla entry.
+           // wilsond@ociweb.com
+
+        * tests/AMH_Oneway/server.cpp:
+          Modified to honor -ORB arguments on command line [so I could diagnose the silly
+          windows/localhost problem]
+
+        * performance-tests/Latency/DII/Test.idl:
+          Make the shutdown method a one-way to eliminate a very rare race condition while
+          shutting down.
 Wed Aug  1 11:44:24 UTC 2007  Chad Elliott  <elliott_c@ociweb.com>
 
         * configure.ac:
diff --git a/TAO/orbsvcs/orbsvcs/SSLIOP/IIOP_SSL_Connector.cpp b/TAO/orbsvcs/orbsvcs/SSLIOP/IIOP_SSL_Connector.cpp
index e0aa736b6de..9175fc8d0ba 100644
--- a/TAO/orbsvcs/orbsvcs/SSLIOP/IIOP_SSL_Connector.cpp
+++ b/TAO/orbsvcs/orbsvcs/SSLIOP/IIOP_SSL_Connector.cpp
@@ -176,6 +176,7 @@ TAO::IIOP_SSL_Connector::make_connection (
           // get a connected transport or not. In case of non block we get
           // a connected or not connected transport
           if (!this->wait_for_connection_completion (r,
+                                                     desc,
                                                      transport,
                                                      max_wait_time))
             {
diff --git a/TAO/orbsvcs/orbsvcs/SSLIOP/SSLIOP_Connector.cpp b/TAO/orbsvcs/orbsvcs/SSLIOP/SSLIOP_Connector.cpp
index b8f4854b3e1..ec1e87b32be 100644
--- a/TAO/orbsvcs/orbsvcs/SSLIOP/SSLIOP_Connector.cpp
+++ b/TAO/orbsvcs/orbsvcs/SSLIOP/SSLIOP_Connector.cpp
@@ -480,9 +480,12 @@ TAO::SSLIOP::Connector::ssliop_connect (
     }
 
   // Check the Cache first for connections
+  unsigned int busy_count = 0; //not used
   if (this->orb_core ()->lane_resources ().transport_cache ().find_transport (
         desc,
-        transport) == 0)
+        transport,
+        busy_count)
+        == TAO::Transport_Cache_Manager::CACHE_FOUND_AVAILABLE)
     {
       if (TAO_debug_level > 2)
         ACE_DEBUG ((LM_DEBUG,
@@ -494,6 +497,7 @@ TAO::SSLIOP::Connector::ssliop_connect (
       if (!transport->is_connected())
         {
           if (!this->wait_for_connection_completion (resolver,
+                                                     *desc,
                                                      transport,
                                                      max_wait_time))
             {
@@ -638,6 +642,7 @@ TAO::SSLIOP::Connector::ssliop_connect (
               // get a connected transport or not. In case of non block we get
               // a connected or not connected transport
               if (!this->wait_for_connection_completion (resolver,
+                                                         *desc,
                                                          transport,
                                                          max_wait_time))
                 {
diff --git a/TAO/performance-tests/Latency/DII/Test.idl b/TAO/performance-tests/Latency/DII/Test.idl
index 3f26904a3f0..c41a5194042 100644
--- a/TAO/performance-tests/Latency/DII/Test.idl
+++ b/TAO/performance-tests/Latency/DII/Test.idl
@@ -21,6 +21,6 @@ module Test
     Timestamp test_method (in Timestamp send_time);
 
     /// Shutdown the ORB
-    void shutdown ();
+    oneway void shutdown ();
   };
 };
diff --git a/TAO/tao/Cache_Entries.h b/TAO/tao/Cache_Entries.h
index 993cc36de11..e085b3916c0 100644
--- a/TAO/tao/Cache_Entries.h
+++ b/TAO/tao/Cache_Entries.h
@@ -53,6 +53,9 @@ namespace TAO
       /// Closed.
       ENTRY_CLOSED,
 
+      /// Connection in process, but not complete
+      ENTRY_CONNECTING,
+
       /// Unknown state.
       ENTRY_UNKNOWN
     };
@@ -102,7 +105,7 @@ namespace TAO
     void recycle_state (Cache_Entries_State new_state);
 
     /// Get <recycle_state>.
-    Cache_Entries_State recycle_state (void);
+    Cache_Entries_State recycle_state (void) const;
 
     /// Relinquish ownership of the TAO_Transport object associated with
     /// this Cache_IntId.
diff --git a/TAO/tao/Cache_Entries.inl b/TAO/tao/Cache_Entries.inl
index 611621c34e1..41d04e882aa 100644
--- a/TAO/tao/Cache_Entries.inl
+++ b/TAO/tao/Cache_Entries.inl
@@ -52,7 +52,7 @@ namespace TAO
   }
 
   ACE_INLINE Cache_Entries_State
-  Cache_IntId::recycle_state (void)
+  Cache_IntId::recycle_state (void) const
   {
     return this->recycle_state_;
   }
diff --git a/TAO/tao/IIOP_Connector.cpp b/TAO/tao/IIOP_Connector.cpp
index 00f354afa23..f7fdbcb02fd 100644
--- a/TAO/tao/IIOP_Connector.cpp
+++ b/TAO/tao/IIOP_Connector.cpp
@@ -382,6 +382,37 @@ TAO_IIOP_Connector::begin_connection (TAO_IIOP_Connection_Handler *&svc_handler,
   return result;
 }
 
+namespace
+{
+  /// RAII holder for tlist
+  class TList_Holder
+  {
+  public:
+    TList_Holder (size_t count)
+      : tlist_(0)
+    {
+      ACE_NEW (tlist_, TAO_Transport*[count]);
+    }
+    ~TList_Holder()
+    {
+      delete [] tlist_;
+    }
+    operator TAO_Transport **()
+    {
+      return tlist_;
+    }
+#if 0 // we don't need const access anyway (avoid silly MAC compiler warning)
+    operator TAO_Transport ** const() const
+    {
+      return tlist_;
+    }
+#endif // 0 // we don't need const access anyway
+
+  private:
+    TAO_Transport **tlist_;
+  };
+}
+
 TAO_Transport *
 TAO_IIOP_Connector::complete_connection (int result,
                                          TAO_Transport_Descriptor_Interface &desc,
@@ -395,9 +426,9 @@ TAO_IIOP_Connector::complete_connection (int result,
   // Make sure that we always do a remove_reference for every member
   // of the list
   TAO_IIOP_Connection_Handler_Array_Guard svc_handler_auto_ptr (sh_list,count);
+  TList_Holder tlist(count);
+
   TAO_Transport *transport  = 0;
-  TAO_Transport **tlist = 0;
-  ACE_NEW_RETURN (tlist,TAO_Transport*[count],0);
 
   //  populate the transport list
   for (unsigned i = 0; i < count; i++)
@@ -415,7 +446,9 @@ TAO_IIOP_Connector::complete_connection (int result,
       if (count == 1)
         {
           transport = tlist[0];
+          desc.reset_endpoint(ep_list[0]);
           if (!this->wait_for_connection_completion (r,
+                                                     desc,
                                                      transport,
                                                      timeout))
             {
@@ -461,8 +494,6 @@ TAO_IIOP_Connector::complete_connection (int result,
   ACE_OS::nanosleep (&ts);
 #endif // INDUCE_BUG_2654_A
 
-  int retval = -1;
-
   // At this point, the connection has been successfully created
   // connected or not connected, but we have a connection.
   TAO_IIOP_Connection_Handler *svc_handler = 0;
@@ -470,33 +501,19 @@ TAO_IIOP_Connector::complete_connection (int result,
 
   if (transport != 0)
     {
-      if (count == 1)
-        {
-          svc_handler = sh_list[0];
-          iiop_endpoint = ep_list[0];
-        }
-      else
+      for (unsigned i = 0; i < count; i++)
         {
-          for (unsigned i = 0; i < count; i++)
+          if (transport == tlist[i])
             {
-              if (transport == tlist[i])
-                {
-                  svc_handler = sh_list[i];
-                  iiop_endpoint = ep_list[i];
-                  break;
-                }
+              svc_handler = sh_list[i];
+              iiop_endpoint = ep_list[i];
+              break;
             }
         }
     }
-
-
-  // Done with the transport list. It was a temporary that did not
-  // affect the reference count.
-  delete [] tlist;
-
-  // In case of errors transport is zero
-  if (transport == 0)
+  else
     {
+      // In case of errors transport is zero
       // Give users a clue to the problem.
       if (TAO_debug_level > 3)
         {
@@ -559,9 +576,9 @@ TAO_IIOP_Connector::complete_connection (int result,
   struct timespec ts = udelay;
   ACE_OS::nanosleep (&ts);
 #endif // INDUCE_BUG_2654_C
+  int retval = 0;
 
-
-  // Add the handler to Cache
+  // Update the cache to show this in idle state
   if (count == 1 || desc.reset_endpoint(iiop_endpoint))
     {
       retval = this->orb_core ()->
@@ -610,25 +627,8 @@ TAO_IIOP_Connector::complete_connection (int result,
   ACE_OS::nanosleep (&ts);
 #endif // INDUCE_BUG_2654_D
 
-  if (transport->is_connected () &&
-      transport->wait_strategy ()->register_handler () != 0)
+  if (!transport->register_if_necessary ())
     {
-      // Registration failures.
-
-      // Purge from the connection cache, if we are not in the cache, this
-      // just does nothing.
-      (void) transport->purge_entry ();
-
-      // Close the handler.
-      (void) transport->close_connection ();
-
-      if (TAO_debug_level > 0)
-        ACE_ERROR ((LM_ERROR,
-                    ACE_TEXT ("TAO (%P|%t) IIOP_Connector [%d]::make_connection, ")
-                    ACE_TEXT ("could not register the transport ")
-                    ACE_TEXT ("in the reactor.\n"),
-                    transport->id ()));
-
       return 0;
     }
 
diff --git a/TAO/tao/IIOP_Transport.cpp b/TAO/tao/IIOP_Transport.cpp
index ccb7a2c4ca1..1ebe2fd698e 100644
--- a/TAO/tao/IIOP_Transport.cpp
+++ b/TAO/tao/IIOP_Transport.cpp
@@ -192,8 +192,9 @@ TAO_IIOP_Transport::recv (char *buf,
 
       ACE_DEBUG ((LM_DEBUG,
                   ACE_TEXT ("TAO (%P|%t) - IIOP_Transport[%d]::recv, ")
-                  ACE_TEXT ("read failure - %m\n"),
-                  this->id ()));
+                  ACE_TEXT ("read failure - %m errno %d\n"),
+                  this->id (),
+                  errno));
     }
 
   // Error handling
diff --git a/TAO/tao/Leader_Follower.cpp b/TAO/tao/Leader_Follower.cpp
index d440d75328b..1a158ad16d8 100644
--- a/TAO/tao/Leader_Follower.cpp
+++ b/TAO/tao/Leader_Follower.cpp
@@ -212,7 +212,7 @@ TAO_Leader_Follower::wait_for_event (TAO_LF_Event *event,
   // in PathFinder
   size_t t_id = 0;
 
-  if (TAO_debug_level)
+  if (TAO_debug_level && transport != 0)
     {
       t_id = transport->id ();
     }
diff --git a/TAO/tao/Profile_Transport_Resolver.cpp b/TAO/tao/Profile_Transport_Resolver.cpp
index 0a60929e532..9d1a7e1d829 100644
--- a/TAO/tao/Profile_Transport_Resolver.cpp
+++ b/TAO/tao/Profile_Transport_Resolver.cpp
@@ -249,10 +249,12 @@ namespace TAO
       this->profile_->orb_core()->lane_resources ().transport_cache();
 
     // the cache increments the reference count on the transport if
-    // the find is successful. Cache.find_transport returns 0 on success.
-    // but we want to return a "boolean" of 0 for failure, 1 for success.
+    // the find is successful. We want to return a "boolean" of 0 for
+    // failure, 1 for success.
+    unsigned int busy_count;
     TAO_Transport* tmp = this->transport_.get ();
-    if (cache.find_transport(desc, tmp) != 0)
+    if (cache.find_transport(desc, tmp, busy_count)
+        != Transport_Cache_Manager::CACHE_FOUND_AVAILABLE)
       return 0;
 
     this->transport_.set (tmp);
diff --git a/TAO/tao/Strategies/UIOP_Connector.cpp b/TAO/tao/Strategies/UIOP_Connector.cpp
index 2af80adeeb9..659bf8e2a1c 100644
--- a/TAO/tao/Strategies/UIOP_Connector.cpp
+++ b/TAO/tao/Strategies/UIOP_Connector.cpp
@@ -202,6 +202,7 @@ TAO_UIOP_Connector::make_connection (TAO::Profile_Transport_Resolver *r,
           // get a connected transport or not. In case of non block we get
           // a connected or not connected transport
           if (!this->wait_for_connection_completion (r,
+                                                     desc,
                                                      transport,
                                                      max_wait_time))
             {
diff --git a/TAO/tao/Transport.cpp b/TAO/tao/Transport.cpp
index 6e0d4ffdbcb..b0c34264667 100644
--- a/TAO/tao/Transport.cpp
+++ b/TAO/tao/Transport.cpp
@@ -308,6 +308,33 @@ TAO_Transport::post_connect_hook (void)
   return true;
 }
 
+bool
+TAO_Transport::register_if_necessary (void)
+{
+  if (this->is_connected () &&
+      ! this->wait_strategy ()->is_registered () &&
+      this->wait_strategy ()->register_handler () != 0)
+    {
+      // Registration failures.
+
+      // Purge from the connection cache, if we are not in the cache, this
+      // just does nothing.
+      (void) this->purge_entry ();
+
+      // Close the handler.
+      (void) this->close_connection ();
+
+      if (TAO_debug_level > 0)
+        ACE_ERROR ((LM_ERROR,
+                    ACE_TEXT ("(%P|%t) IIOP_Connector [%d]::register_if_necessary, ")
+                    ACE_TEXT ("could not register the transport ")
+                    ACE_TEXT ("in the reactor.\n"),
+                    this->id ()));
+      return false;
+    }
+  return true;
+}
+
 void
 TAO_Transport::close_connection (void)
 {
@@ -2499,19 +2526,10 @@ TAO_Transport::post_open (size_t id)
 {
   this->id_ = id;
 
-  {
-    ACE_GUARD_RETURN (ACE_Lock,
-                      ace_mon,
-                      *this->handler_lock_,
-                      false);
-    this->is_connected_ = true;
-  }
-
   // When we have data in our outgoing queue schedule ourselves
   // for output
-  if (this->queue_is_empty_i ())
-    return true;
-
+  if (!this->queue_is_empty_i ())
+    {
   // If the wait strategy wants us to be registered with the reactor
   // then we do so. If registeration is required and it succeeds,
   // #REFCOUNT# becomes two.
@@ -2528,14 +2546,28 @@ TAO_Transport::post_open (size_t id)
 
       if (TAO_debug_level > 0)
         ACE_ERROR ((LM_ERROR,
-           ACE_TEXT ("TAO (%P|%t) - Transport[%d]::post_connect , ")
+               ACE_TEXT ("TAO (%P|%t) - Transport[%d]::post_open , ")
            ACE_TEXT ("could not register the transport ")
            ACE_TEXT ("in the reactor.\n"),
            this->id ()));
 
       return false;
     }
+    }
 
+  this->is_connected_ = true;
+  // update transport cache to make this entry available
+  if (this->cache_map_entry_ != 0)
+    {
+      ACE_GUARD_RETURN (ACE_Lock,
+                          ace_mon,
+                          *this->handler_lock_,
+                          false);
+      this->transport_cache_manager ().cache_transport (
+        this->cache_map_entry_->ext_id_.property (),
+        this,
+        TAO::ENTRY_IDLE_BUT_NOT_PURGABLE);
+    }
   return true;
 }
 
diff --git a/TAO/tao/Transport.h b/TAO/tao/Transport.h
index b848018724c..c787bec6b88 100644
--- a/TAO/tao/Transport.h
+++ b/TAO/tao/Transport.h
@@ -328,6 +328,10 @@ public:
    */
   bool queue_is_empty (void);
 
+  /// Register with the reactor via the wait strategy
+  bool register_if_necessary (void);
+
+
   /// Added event handler to the handlers set.
   /**
    * Called by the cache when the cache is closing.
diff --git a/TAO/tao/Transport_Cache_Manager.cpp b/TAO/tao/Transport_Cache_Manager.cpp
index de734fc9975..ca19193907a 100644
--- a/TAO/tao/Transport_Cache_Manager.cpp
+++ b/TAO/tao/Transport_Cache_Manager.cpp
@@ -28,26 +28,15 @@ namespace TAO
     : percent_ (orb_core.resource_factory ()->purge_percentage ())
     , purging_strategy_ (orb_core.resource_factory ()->create_purging_strategy ())
     , cache_map_ (orb_core.resource_factory ()->cache_maximum ())
-    , condition_ (0)
     , cache_lock_ (0)
-    , muxed_number_ (orb_core.resource_factory ()->max_muxed_connections ())
-    , no_waiting_threads_ (0)
-    , last_entry_returned_ (0)
   {
     if (orb_core.resource_factory ()->locked_transport_cache ())
       {
-        ACE_NEW (this->condition_,
-                 TAO_Condition <TAO_SYNCH_MUTEX>);
-
         ACE_NEW (this->cache_lock_,
-                 ACE_Lock_Adapter <TAO_SYNCH_MUTEX> (*this->condition_->mutex ()));
+                 ACE_Lock_Adapter <TAO_SYNCH_MUTEX> (this->cache_map_mutex_));
       }
     else
       {
-        /// If the cache is not going to be locked then dont create a
-        /// condition variable. Make the <muxed_number_> to 0, else a
-        /// single thread could get into waiting mode
-        this->muxed_number_ = 0;
         ACE_NEW (this->cache_lock_,
                  ACE_Lock_Adapter<ACE_SYNCH_NULL_MUTEX>);
       }
@@ -55,12 +44,6 @@ namespace TAO
 
   Transport_Cache_Manager::~Transport_Cache_Manager (void)
   {
-    // Wakeup all the waiting threads threads before we shutdown stuff
-    if (this->no_waiting_threads_)
-      {
-        this->condition_->broadcast ();
-      }
-
     // Delete the lock that we have
     if (this->cache_lock_)
       {
@@ -75,12 +58,6 @@ namespace TAO
         this->purging_strategy_ = 0;
       }
 
-    // Delete the condition variable
-    if (this->condition_)
-      {
-        delete this->condition_;
-        this->condition_ = 0;
-      }
   }
 
 
@@ -92,10 +69,12 @@ namespace TAO
        {
          ACE_DEBUG ((LM_DEBUG,
                     "TAO (%P|%t) - Transport_Cache_Manager::bind_i, "
-                    "0x%x -> 0x%x Transport[%d]\n",
+                    "0x%x {%d:%d} -> 0x%x Transport[%d]\n",
                      ext_id.property (),
-                    int_id.transport (),
-                    int_id.transport ()->id ()));
+                     ext_id.hash (),
+                     ext_id.index (),
+                     int_id.transport (),
+                     int_id.transport ()->id ()));
        }
 
     // Get the entry too
@@ -116,23 +95,40 @@ namespace TAO
       }
     else if (retval == 1)
       {
-        if (TAO_debug_level > 4)
+        // if this is already in the cache, just ignore the request
+        // this happens because some protocols bind their transport early
+        // to avoid duplication simultaneous connection attempts
+        if (entry != 0 && entry->int_id_.transport () == int_id.transport ())
           {
-            ACE_DEBUG ((LM_DEBUG,
-                        "TAO (%P|%t) - Transport_Cache_Manager::bind_i, "
-                        "unable to bind in the first attempt. "
-                        "Trying with a new index\n"));
+            // rebind this entry to update cache status
+            retval = this->cache_map_.rebind (ext_id,
+                                        int_id,
+                                        entry);
+            if(retval == 1) // 1 from rebind means replaced
+              {
+                retval = 0;
+              }
           }
+        else
+        {
+          if (TAO_debug_level > 4)
+            {
+              ACE_DEBUG ((LM_DEBUG,
+                          "TAO (%P|%t) - Transport_Cache_Manager::bind_i, "
+                          "unable to bind in the first attempt. "
+                          "Trying with a new index\n"));
+            }
 
-        // There was an entry like this before, so let us do some
-        // minor adjustments and rebind
-        retval = this->get_last_index_bind (ext_id,
-                                            int_id,
-                                            entry);
-        if (retval == 0)
-          {
-            int_id.transport ()->cache_map_entry (entry);
-          }
+          // There was an entry like this before, so let us do some
+          // minor adjustments and rebind
+          retval = this->get_last_index_bind (ext_id,
+                                              int_id,
+                                              entry);
+          if (retval == 0)
+            {
+              int_id.transport ()->cache_map_entry (entry);
+            }
+        }
       }
 
     if (TAO_debug_level > 5 && retval != 0)
@@ -152,70 +148,73 @@ namespace TAO
     return retval;
   }
 
-  int
+  Transport_Cache_Manager::Find_Result
   Transport_Cache_Manager::find_transport (
     TAO_Transport_Descriptor_Interface *prop,
-    TAO_Transport *&transport)
+    TAO_Transport *&transport,
+    unsigned int &busy_count)
   {
     if (prop == 0)
       {
         transport = 0;
-        return -1;
+        return CACHE_FOUND_NONE;
       }
 
     // Compose the ExternId
     Cache_ExtId ext_id (prop);
     Cache_IntId int_id;
 
-    int const retval = this->find (ext_id,
-                                   int_id);
-    if (retval == 0)
+    Transport_Cache_Manager::Find_Result find_result = this->find (ext_id,
+                                   int_id, busy_count);
+    if (find_result != CACHE_FOUND_NONE)
       {
         transport = int_id.relinquish_transport ();
-
-        if (transport->wait_strategy ()->non_blocking () == 0 &&
-            transport->orb_core ()->client_factory ()->use_cleanup_options ())
+        if (find_result == CACHE_FOUND_AVAILABLE)
           {
-            ACE_Event_Handler * const eh =
-              transport->event_handler_i ();
+            if (transport->wait_strategy ()->non_blocking () == 0 &&
+                transport->orb_core ()->client_factory ()->use_cleanup_options ())
+              {
+                ACE_Event_Handler * const eh =
+                  transport->event_handler_i ();
 
-            ACE_Reactor * const r =
-              transport->orb_core ()->reactor ();
+                ACE_Reactor * const r =
+                  transport->orb_core ()->reactor ();
 
-            if (eh &&
-                r->remove_handler (eh,
-                                   ACE_Event_Handler::READ_MASK |
-                                   ACE_Event_Handler::DONT_CALL) == -1)
-              {
-                if (TAO_debug_level > 0)
-                  ACE_ERROR ((LM_ERROR,
-                              ACE_TEXT ("TAO (%P|%t) - TAO_Transport_Cache_Manager[%d]")
-                              ACE_TEXT ("::find_transport, remove_handler failed \n"),
-                              transport->id ()));
-              }
-            else
-              {
-                transport->wait_strategy ()->is_registered (false);
+                if (eh &&
+                    r->remove_handler (eh,
+                                       ACE_Event_Handler::READ_MASK |
+                                       ACE_Event_Handler::DONT_CALL) == -1)
+                  {
+                    if (TAO_debug_level > 0)
+                      ACE_ERROR ((LM_ERROR,
+                                  ACE_TEXT ("TAO (%P|%t) - TAO_Transport_Cache_Manager[%d]")
+                                  ACE_TEXT ("::find_transport, remove_handler failed \n"),
+                                  transport->id ()));
+                  }
+                else
+                  {
+                    transport->wait_strategy ()->is_registered (false);
+                  }
               }
           }
       }
-
-    return retval;
+    return find_result;
   }
 
-  int
+  Transport_Cache_Manager::Find_Result
   Transport_Cache_Manager::find (const Cache_ExtId &key,
-                                 Cache_IntId &value)
+                                 Cache_IntId &value,
+                                 unsigned int &busy_count)
   {
     ACE_MT (ACE_GUARD_RETURN  (ACE_Lock,
                                guard,
                                *this->cache_lock_,
-                               -1));
+                               Transport_Cache_Manager::CACHE_FOUND_NONE));
 
-    int const status =  this->find_i (key,
-                                      value);
+    Transport_Cache_Manager::Find_Result status =  this->find_i (key,
+                                      value, busy_count);
 
-    if (status == 0)
+    if (status != CACHE_FOUND_NONE)
       {
         // Update the purging strategy information while we
         // are holding our lock
@@ -225,37 +224,35 @@ namespace TAO
     return status;
   }
 
-  int
+  Transport_Cache_Manager::Find_Result
   Transport_Cache_Manager::find_i (const Cache_ExtId &key,
-                                   Cache_IntId &value)
+                                   Cache_IntId &value,
+                                   unsigned int & busy_count)
   {
     HASH_MAP_ENTRY *entry = 0;
+    busy_count = 0;
 
     // Get the entry from the Hash Map
-    int retval = 0;
+    Transport_Cache_Manager::Find_Result found = CACHE_FOUND_NONE;
 
     // Make a temporary object. It does not do a copy.
     Cache_ExtId tmp_key (key.property ());
+    int cache_status = 0;
 
-    while (retval == 0)
+    // loop until we find a usable transport, or until we've checked
+    // all cached entries for this endpoint
+    while (found != CACHE_FOUND_AVAILABLE && cache_status == 0)
       {
-        // Wait for a connection..
-        this->wait_for_connection (tmp_key);
-
-        // Look for an entry in the map
-        retval = this->cache_map_.find (tmp_key,
+        entry = 0;
+        cache_status  = this->cache_map_.find (tmp_key,
                                         entry);
-
-        // We have an entry in the map, check whether it is idle.
-        if (entry)
+        if (cache_status == 0 && entry)
           {
-            CORBA::Boolean idle =
-              this->is_entry_idle (entry);
-
-            if (idle)
+            if (this->is_entry_available (*entry))
               {
                 // Successfully found a TAO_Transport.
 
+                found = CACHE_FOUND_AVAILABLE;
                 entry->int_id_.recycle_state (ENTRY_BUSY);
 
                 // NOTE: This assignment operator indirectly incurs two
@@ -267,36 +264,71 @@ namespace TAO
                   {
                     ACE_DEBUG ((LM_DEBUG,
                                 ACE_TEXT("TAO (%P|%t) - Transport_Cache_Manager::find_i, ")
-                                ACE_TEXT("at index %d (Transport[%d]) - idle\n"),
+                                ACE_TEXT("{%d:%d} (Transport %x[%d]) - idle\n"),
+                                entry->ext_id_.hash (),
                                 entry->ext_id_.index (),
+                                entry->int_id_.transport (),
                                 entry->int_id_.transport ()->id ()));
                   }
-
-                return 0;
               }
-            else if (TAO_debug_level > 6)
+            else if (this->is_entry_connecting (*entry))
+              {
+                if (TAO_debug_level > 4)
                   {
                     ACE_DEBUG ((LM_DEBUG,
                                 ACE_TEXT("TAO (%P|%t) - Transport_Cache_Manager::find_i, ")
-                                ACE_TEXT("at index %d (Transport[%d]) - not idle\n"),
+                                ACE_TEXT("{%d:%d} (Transport %x[%d]) - connecting\n"),
+                                entry->ext_id_.hash (),
                                 entry->ext_id_.index (),
+                                entry->int_id_.transport (),
                                 entry->int_id_.transport ()->id ()));
                   }
+                // if this is the first interesting entry
+                if (found != CACHE_FOUND_CONNECTING)
+                  {
+                    // NOTE: This assignment operator indirectly incurs two
+                    //       lock operations since it duplicates and releases
+                    //       TAO_Transport objects.
+                    value = entry->int_id_;
+                    found = CACHE_FOUND_CONNECTING;
+                  }
+              }
+            else
+              {
+                // if this is the first busy entry
+                if (found == CACHE_FOUND_NONE && busy_count == 0)
+                  {
+                    value = entry->int_id_;
+                    found = CACHE_FOUND_BUSY;
+                  }
+                busy_count += 1;
+                if (TAO_debug_level > 6)
+                  {
+                    ACE_DEBUG ((LM_DEBUG,
+                                ACE_TEXT("TAO (%P|%t) - Transport_Cache_Manager::find_i, ")
+                                ACE_TEXT("{%d:%d} (Transport %x[%d]) - busy\n"),
+                                entry->ext_id_.hash (),
+                                entry->ext_id_.index (),
+                                entry->int_id_.transport (),
+                                entry->int_id_.transport ()->id ()));
+                  }
+              }
           }
 
         // Bump the index up
         tmp_key.incr_index ();
       }
 
-    // If we are here then it is an error
-    if (TAO_debug_level > 4 && retval != 0)
+    if (TAO_debug_level > 4 && found != CACHE_FOUND_AVAILABLE)
       {
         ACE_ERROR ((LM_ERROR,
                     "TAO (%P|%t) - Transport_Cache_Manager::find_i, "
-                    "no idle transport is available\n"));
+                    "no idle transport is available for {%d}\n",
+                    tmp_key.hash ()
+                    ));
       }
 
-    return retval;
+    return found;
   }
 
   int
@@ -307,16 +339,6 @@ namespace TAO
 
     entry->int_id_.recycle_state (ENTRY_IDLE_AND_PURGABLE);
 
-    // Does any one need waking?
-    if (this->no_waiting_threads_)
-      {
-        // We returned this entry to the map
-        this->last_entry_returned_ = &entry->ext_id_;
-
-        // Wake up a thread
-        this->condition_->signal ();
-      }
-
     return 0;
   }
 
@@ -443,24 +465,54 @@ namespace TAO
 
 
   bool
-  Transport_Cache_Manager::is_entry_idle (HASH_MAP_ENTRY *&entry)
+  Transport_Cache_Manager::is_entry_available (const HASH_MAP_ENTRY &entry)
   {
     Cache_Entries_State entry_state =
-      entry->int_id_.recycle_state ();
+      entry.int_id_.recycle_state ();
+    bool result = (entry_state == ENTRY_IDLE_AND_PURGABLE ||
+        entry_state == ENTRY_IDLE_BUT_NOT_PURGABLE);
+
+    if (result && entry.int_id_.transport () != 0)
+    {
+      // if it's not connected, it's not available
+      result = entry.int_id_.transport ()->is_connected();
+    }
 
     if (TAO_debug_level)
       {
         ACE_DEBUG ((LM_DEBUG,
-                    ACE_TEXT("TAO (%P|%t) - Transport_Cache_Manager::is_entry_idle, ")
+          ACE_TEXT("TAO (%P|%t) - Transport_Cache_Manager::is_entry_available: %c ")
+          ACE_TEXT("state is [%d]\n"),
+          (result?'T':'F'),
+          entry_state));
+      }
+
+    return result;
+  }
+
+  bool
+  Transport_Cache_Manager::is_entry_connecting (const HASH_MAP_ENTRY &entry)
+  {
+    Cache_Entries_State entry_state =
+      entry.int_id_.recycle_state ();
+    bool result = (entry_state == ENTRY_CONNECTING);
+    if (!result && entry.int_id_.transport () != 0)
+    {
+      // if we're not connected, that counts, too.
+      // Can this happen?  Not sure <wilsond@ociweb.com>
+      result = !entry.int_id_.transport ()->is_connected();
+    }
+
+    if (TAO_debug_level)
+      {
+        ACE_DEBUG ((LM_DEBUG,
+          ACE_TEXT("TAO (%P|%t) - Transport_Cache_Manager::is_entry_connecting: %c ")
                     ACE_TEXT("state is [%d]\n"),
+                    (result?'T':'F'),
                     entry_state));
       }
 
-    if (entry_state == ENTRY_IDLE_AND_PURGABLE ||
-        entry_state == ENTRY_IDLE_BUT_NOT_PURGABLE)
-      return true;
-
-    return false;
+    return result;
   }
 
 #if !defined (ACE_LACKS_QSORT)
@@ -515,7 +567,7 @@ namespace TAO
 
           for (int i = 0; count < amount && i < sorted_size; ++i)
             {
-              if (this->is_entry_idle (sorted_set[i]))
+              if (this->is_entry_available (*sorted_set[i]))
                 {
                   sorted_set[i]->int_id_.recycle_state (ENTRY_BUSY);
 
@@ -642,82 +694,6 @@ namespace TAO
     return current_size;
   }
 
-
-  int
-  Transport_Cache_Manager::wait_for_connection (Cache_ExtId &extid)
-  {
-    if (this->muxed_number_ && this->muxed_number_ == extid.index ())
-      {
-        // If we have a limit on the number of muxed connections for
-        // a particular endpoint just wait to get the connection
-        ++this->no_waiting_threads_;
-
-        if (TAO_debug_level > 2)
-          {
-            ACE_DEBUG ((LM_DEBUG,
-                        ACE_TEXT("TAO (%P|%t) - Transport_Cache_Manager")
-                        ACE_TEXT("::wait_for_connection, ")
-                        ACE_TEXT("entering wait loop\n")));
-          }
-
-        int ready_togo = 0;
-
-        while (ready_togo == 0)
-          {
-            this->condition_->wait ();
-
-            // Check whether we are waiting for this connection
-            ready_togo = this->is_wakeup_useful (extid);
-          }
-
-        if (TAO_debug_level > 2)
-          {
-            ACE_DEBUG ((LM_DEBUG,
-                        ACE_TEXT("TAO (%P|%t) - Transport_Cache_Manager::wait_for_connection, ")
-                        ACE_TEXT("left wait loop\n")));
-          }
-
-        // We are not waiting anymore
-        --this->no_waiting_threads_;
-      }
-
-    return 0;
-  }
-
-  int
-  Transport_Cache_Manager::is_wakeup_useful (Cache_ExtId &extid)
-  {
-    // Get the underlying property that we are looking for
-    TAO_Transport_Descriptor_Interface *prop = extid.property ();
-
-    // Just check the underlying property for equivalence. If the last
-    // connection that was returned  had the same property just return
-    // 1.
-    if (this->last_entry_returned_ &&
-        prop->is_equivalent (this->last_entry_returned_->property ()))
-      {
-        // Set the index to be right so that we can pick teh connection
-        // right away..
-        extid.index (this->last_entry_returned_->index ());
-
-        // There is no more use for it ...
-        this->last_entry_returned_ = 0;
-
-        return 1;
-      }
-
-    // If there  is an entry that was returned and if there are more
-    // threads just wake up the peer to check for the returned
-    // connection.
-    if (this->last_entry_returned_ &&
-        this->no_waiting_threads_ > 1)
-      {
-        this->condition_->signal ();
-      }
-
-    return 0;
-  }
-
 }
 
 TAO_END_VERSIONED_NAMESPACE_DECL
diff --git a/TAO/tao/Transport_Cache_Manager.h b/TAO/tao/Transport_Cache_Manager.h
index 37a3643fe3e..dde61213823 100644
--- a/TAO/tao/Transport_Cache_Manager.h
+++ b/TAO/tao/Transport_Cache_Manager.h
@@ -15,6 +15,7 @@
 
 #include /**/ "ace/pre.h"
 #include "ace/Null_Mutex.h"
+#include "ace/Thread_Mutex.h"
 
 #if !defined (ACE_LACKS_PRAGMA_ONCE)
 #define  ACE_LACKS_PRAGMA_ONCE
@@ -60,6 +61,15 @@ namespace TAO
   class TAO_Export Transport_Cache_Manager
   {
   public:
+    // results of a find
+    enum Find_Result
+    {
+      CACHE_FOUND_NONE,
+      CACHE_FOUND_CONNECTING,
+      CACHE_FOUND_BUSY,
+      CACHE_FOUND_AVAILABLE
+    };
+
     // Some useful typedef's
     typedef ACE_Hash_Map_Manager_Ex <Cache_ExtId,
                                      Cache_IntId,
@@ -86,22 +96,26 @@ namespace TAO
     /// Add the transport to the cache.
     /**
      * The transport has the property definition based on which caching
-     * can be done. This method marks the transport
-     * <CODE>ACE_RECYCLABLE_BUSY </CODE> which helps the threads
-     * opening up connections to use the  transport immediately.
+     * can be done. This method sets the cache entry status.  By
+     * default the status is set to <CODE>ENTRY_IDLE_BUT_NOT_PURGABLE</CODE>
      */
     int cache_transport (TAO_Transport_Descriptor_Interface *prop,
-                         TAO_Transport *transport);
-
-    /// Similar to the one above, but the transport is left in <CODE>
-    /// ENTRY_IDLE_AND_PURGABLE</CODE> state.
+                         TAO_Transport *transport,
+                         Cache_Entries_State state = ENTRY_IDLE_BUT_NOT_PURGABLE);
+
+    /// this is just a shortcut for cache_transport with a third argument of
+    /// <CODE>ENTRY_IDLE_AND_PURGABLE</CODE>
+    /// @@TODO: it should be replaced with a direct call to cache_transport,
+    /// but that would require changes to all *_Connection_Handler so I'm
+    /// deferring this for now. (wilsond@ociweb.com)
     int cache_idle_transport (TAO_Transport_Descriptor_Interface *prop,
-                              TAO_Transport *transport);
+                         TAO_Transport *transport);
 
     /// Check the Transport Cache to check whether the connection exists
     /// in the Cache and return the connection
-    int find_transport (TAO_Transport_Descriptor_Interface *prop,
-                        TAO_Transport *&transport);
+    Find_Result find_transport (TAO_Transport_Descriptor_Interface *prop,
+                        TAO_Transport *&transport,
+                        unsigned int & busy_count);
 
     /// Remove entries from the cache depending upon the strategy.
     int purge (void);
@@ -152,8 +166,9 @@ namespace TAO
 
     /// Lookup entry<key,value> in the cache. Grabs the lock and calls the
     /// implementation function find_i.
-    int find (const Cache_ExtId &key,
-              Cache_IntId &value);
+    Find_Result find (const Cache_ExtId &key,
+              Cache_IntId &value,
+              unsigned int & busy_count);
 
     /**
      * Non-Locking version and actual implementation of bind ()
@@ -172,8 +187,9 @@ namespace TAO
      * Hash_Map_Manager. If the find succeeds, it calls the
      * get_idle_transport ().
      */
-    int find_i (const Cache_ExtId &key,
-                Cache_IntId &value);
+    Find_Result find_i (const Cache_ExtId &key,
+                Cache_IntId &value,
+                unsigned int & busy_count);
 
     /// Non-locking version and actual implementation of make_idle ().
     int make_idle_i (HASH_MAP_ENTRY *&entry);
@@ -201,11 +217,14 @@ namespace TAO
                              HASH_MAP_ENTRY *&entry);
 
   /**
-   * Tries to find if the <int_id_> in entry is idle for use. If it is
-   * idle it is immediately markes as busy and returns a value of
-   * 1, else it returns a value of 0
+   * Tries to find if the <int_id_> in entry is available for use.
+   */
+    bool is_entry_available (const HASH_MAP_ENTRY &entry);
+
+  /**
+   * Tries to find if the <int_id_> in entry is connect pending
    */
-    bool is_entry_idle (HASH_MAP_ENTRY *&entry);
+    bool is_entry_connecting (const HASH_MAP_ENTRY &entry);
 
 #if !defined(ACE_LACKS_QSORT)
     /// Used by qsort
@@ -221,16 +240,6 @@ namespace TAO
     /// a sorted order.
     int fill_set_i (DESCRIPTOR_SET& sorted_set);
 
-    /// Wait for connections if we have reached the limit on the number
-    /// of muxed connections. If not (ie. if we dont use a muxed
-    /// connection or if we have not reached the limit) this just
-    /// behaves as a no-op. <extid> has all the information about the
-    /// connection that is being searched.
-    int wait_for_connection (Cache_ExtId &extid);
-
-    /// Is the wakeup useful todo some work?
-    int is_wakeup_useful (Cache_ExtId &extid);
-
     /// Non-locking version of blockable_client_transports ().
     bool blockable_client_transports_i (Connection_Handler_Set &handlers);
 
@@ -244,23 +253,10 @@ namespace TAO
     /// The hash map that has the connections
     HASH_MAP cache_map_;
 
-    /// The condition variable
-    CONDITION *condition_;
+    TAO_SYNCH_MUTEX cache_map_mutex_;
 
     /// The lock that is used by the cache map
     ACE_Lock *cache_lock_;
-
-    /// Number of allowed muxed connections
-    CORBA::ULong muxed_number_;
-
-    /// Number of threads waiting for connections
-    int no_waiting_threads_;
-
-    /// This is for optimization purposes. In a situation where number
-    /// of threads are waiting for connections, the last connection that
-    /// is put back is cached here. This should prevent all th threads
-    /// trying to search for their required entry.
-    Cache_ExtId *last_entry_returned_;
   };
 
 }
diff --git a/TAO/tao/Transport_Cache_Manager.inl b/TAO/tao/Transport_Cache_Manager.inl
index ed53a1a4f90..1d35224b776 100644
--- a/TAO/tao/Transport_Cache_Manager.inl
+++ b/TAO/tao/Transport_Cache_Manager.inl
@@ -23,7 +23,8 @@ namespace TAO
   ACE_INLINE int
   Transport_Cache_Manager::cache_transport (
     TAO_Transport_Descriptor_Interface *prop,
-    TAO_Transport *transport)
+    TAO_Transport *transport,
+    Cache_Entries_State state/* = ENTRY_IDLE_BUT_NOT_PURGABLE*/)
   {
     // Compose the ExternId & Intid
     Cache_ExtId ext_id (prop);
@@ -37,7 +38,7 @@ namespace TAO
                                 -1));
 
       // Do as the semantics of this method dictates
-      int_id.recycle_state (ENTRY_BUSY);
+      int_id.recycle_state (state);
 
       retval = this->bind_i (ext_id,
                              int_id);
@@ -51,24 +52,7 @@ namespace TAO
     TAO_Transport_Descriptor_Interface *prop,
     TAO_Transport *transport)
   {
-    // Compose the ExternId & Intid
-    Cache_ExtId ext_id (prop);
-    Cache_IntId int_id (transport);
-
-    int retval = 0;
-    {
-      ACE_MT (ACE_GUARD_RETURN (ACE_Lock,
-                                guard,
-                                *this->cache_lock_,
-                                -1));
-
-      // Do as the semantics of this method dictates
-      int_id.recycle_state (ENTRY_IDLE_AND_PURGABLE);
-      retval = this->bind_i (ext_id,
-                             int_id);
-    }
-
-    return retval;
+    return cache_transport(prop, transport, ENTRY_IDLE_AND_PURGABLE);
   }
 
   ACE_INLINE int
diff --git a/TAO/tao/Transport_Connector.cpp b/TAO/tao/Transport_Connector.cpp
index a02d8c6070f..46fade68adf 100644
--- a/TAO/tao/Transport_Connector.cpp
+++ b/TAO/tao/Transport_Connector.cpp
@@ -27,47 +27,6 @@ ACE_RCSID (tao,
            Connector,
            "$Id$")
 
-namespace
-{
-  class TransportCleanupGuard
-  {
-  public:
-
-    TransportCleanupGuard (TAO_Transport *tp)
-      : tp_ (tp)
-      , awake_ (true)
-    {
-    }
-
-    ~TransportCleanupGuard ()
-    {
-      if (this->awake_ && this->tp_)
-        {
-          // Purge from the connection cache.  If we are not in the
-          // cache, this does nothing.
-          this->tp_->purge_entry ();
-
-          // Close the handler.
-          this->tp_->close_connection ();
-
-          this->tp_->remove_reference ();
-        }
-    }
-
-    /// Turn off the guard.
-    void down ()
-    {
-      this->awake_ = false;
-    }
-
-  private:
-
-    TAO_Transport * const tp_;
-    bool awake_;
-
-  };
-}
-
 TAO_BEGIN_VERSIONED_NAMESPACE_DECL
 
 // Connector
@@ -322,8 +281,10 @@ TAO_Connector::parallel_connect (TAO::Profile_Transport_Resolver *r,
        ep = ep->next_filtered(this->orb_core(),root_ep))
     {
       TAO_Base_Transport_Property desc2(ep,0);
+      unsigned int busy_count = 0;
       if (tcm.find_transport (&desc2,
-                              base_transport) == 0)
+                              base_transport, busy_count)
+              == TAO::Transport_Cache_Manager::CACHE_FOUND_AVAILABLE )
         {
           if (TAO_debug_level)
             {
@@ -347,123 +308,291 @@ TAO_Connector::parallel_connect (TAO::Profile_Transport_Resolver *r,
   return this->make_parallel_connection (r,*desc,timeout);
 }
 
-TAO_Transport*
-TAO_Connector::connect (TAO::Profile_Transport_Resolver *r,
-                        TAO_Transport_Descriptor_Interface *desc,
-                        ACE_Time_Value *timeout)
+bool
+TAO_Connector::wait_for_transport(
+  TAO::Profile_Transport_Resolver *r,
+  TAO_Transport *transport,
+  ACE_Time_Value * timeout,
+  bool force_wait)
 {
-  if (desc == 0 ||
-      (this->set_validate_endpoint (desc->endpoint ()) == -1))
-    return 0;
-
-  TAO_Transport *base_transport = 0;
-
-  TAO::Transport_Cache_Manager &tcm =
-    this->orb_core ()->lane_resources ().transport_cache ();
-
-  // Check the Cache first for connections
-  // If transport found, reference count is incremented on assignment
-  // @@todo: We need to send the timeout value to the cache registry
-  // too. That should be the next step!
-  if (tcm.find_transport (desc,
-                          base_transport) != 0)
+  if (transport->connection_handler ()->is_timeout ())
     {
-      // @@TODO: This is not the right place for this!
-      // Purge connections (if necessary)
-      tcm.purge ();
+      if (TAO_debug_level > 2)
+        {
+          ACE_DEBUG ((LM_DEBUG,
+            ACE_TEXT("TAO (%P|%t) - TAO_Connector::wait_for_transport, ")
+            ACE_TEXT("transport [%d], Connection Timed out.\n"),
+            transport->id () ));
+        }
+    }
+  else if (transport->connection_handler()->is_closed ())
+    {
+      if (TAO_debug_level > 2)
+        {
+          ACE_DEBUG ((LM_DEBUG,
+            ACE_TEXT("TAO (%P|%t) - TAO_Connector::wait_for_transport, ")
+            ACE_TEXT("transport [%d], Connection failed. (%d)\n"),
+            transport->id (),
+            errno));
+        }
 
-      TAO_Transport* t = this->make_connection (r, *desc, timeout);
+      // purge from the connection cache.  If we are not in the
+      // cache, this does nothing.
+      transport->purge_entry ();
 
-      if (t == 0)
-        return t;
+      // Close the handler.
+      transport->close_connection ();
 
-      t->opened_as (TAO::TAO_CLIENT_ROLE);
+    }
 
-      if (TAO_debug_level > 4)
+  if (transport->connection_handler ()->is_open ())
+    {
+      if (TAO_debug_level > 5)
         {
           ACE_DEBUG ((LM_DEBUG,
-                      ACE_TEXT("TAO (%P|%t) - Transport_Connector::connect, ")
-                      ACE_TEXT("opening Transport[%d] in TAO_CLIENT_ROLE\n"),
-                      t->id ()));
-        }
-
-      // Call post connect hook. If the post_connect_hook () returns
-      // false, just purge the entry.
-      if (!t->post_connect_hook ())
-        {
-          (void) t->purge_entry ();
-
-          // Call connect again
-          return this->connect (r, desc, timeout);
+            ACE_TEXT("TAO (%P|%t) - TAO_Connector::wait_for_transport, ")
+            ACE_TEXT("transport [%d], connection is open: no wait.\n"),
+            transport->id () ));
         }
 
-      return t;
+      return true;
     }
+  else if (force_wait || r->blocked_connect ())
+  {
+    if (TAO_debug_level > 2)
+      {
+        ACE_DEBUG ((LM_DEBUG,
+          ACE_TEXT("TAO (%P|%t) - TAO_Connector::wait_for_transport, ")
+          ACE_TEXT(" waiting on transport [%d]\n"),
+          transport->id () ));
+      }
+    int result = this->active_connect_strategy_->wait (transport, timeout);
+    if (result == -1 && errno == ETIME)
+      {
+        if (TAO_debug_level > 2)
+          {
+            ACE_DEBUG ((LM_DEBUG,
+            ACE_TEXT("TAO (%P|%t) - TAO_Connector::wait_for_transport, ")
+            ACE_TEXT(" timeout while waiting on transport [%d]\n"),
+            transport->id () ));
+          }
+      }
+    else if(result == -1)
+      {
+        if (TAO_debug_level > 2)
+          {
+            static int complain10times = 10;
+            if (complain10times > 0)
+              {
+                complain10times -= 1;
+                ACE_DEBUG ((LM_DEBUG,
+                  ACE_TEXT("TAO (%P|%t) - TAO_Connector::wait_for_transport, ")
+                  ACE_TEXT(" unknown error waiting on transport [%d] (%d)\n"),
+                  transport->id (),
+                  errno));
+              }
+          }
+        // purge from the connection cache.  If we are not in the
+        // cache, this does nothing.
+        transport->purge_entry ();
 
-  if (TAO_debug_level > 4)
-    {
-      TAO::Connection_Role cr = base_transport->opened_as ();
+        // Close the handler.
+        transport->close_connection ();
 
+      }
+    else
+      {
+        if (TAO_debug_level > 5)
+          {
+            ACE_DEBUG ((LM_DEBUG,
+              ACE_TEXT("TAO (%P|%t) - TAO_Connector::wait_for_transport, ")
+              ACE_TEXT("transport [%d], wait completed ok.\n"),
+              transport->id () ));
+          }
+        return true;
+      }
+  }
+  else
+  {
+    if (TAO_debug_level > 2)
+    {
       ACE_DEBUG ((LM_DEBUG,
-                  "TAO (%P|%t) - Transport_Connector::connect, "
-                  "got an existing %s Transport[%d] in role %s\n",
-                  base_transport->is_connected () ? "connected" : "unconnected",
-                  base_transport->id (),
-                  cr == TAO::TAO_SERVER_ROLE ? "TAO_SERVER_ROLE" :
-                  cr == TAO::TAO_CLIENT_ROLE ? "TAO_CLIENT_ROLE" :
-                  "TAO_UNSPECIFIED_ROLE" ));
+        ACE_TEXT("TAO (%P|%t) - TAO_Connector::wait_for_transport, ")
+        ACE_TEXT(" Connection not complete [%d]\n"),
+        transport->id () ));
     }
+    transport->connection_handler ()->reset_state(
+      TAO_LF_Event::LFS_CONNECTION_WAIT);
 
-  // If connected return.
-  if (base_transport->is_connected ())
-    return base_transport;
+    return true;
+  }
 
-  // It it possible to get a transport from the cache that is not
-  // connected? If not, then the following code is bogus. We cannot
-  // wait for a connection to complete on a transport in the cache.
-  //
-  // (mesnier_p@ociweb.com) It is indeed possible to reach this point.
-  // The AMI_Buffering test does. When using non-blocking connects and
-  // the first request(s) are asynch and may be queued, the connection
-  // establishment may not be completed by the time the invocation is
-  // done with it. In that case it is up to a subsequent invocation to
-  // handle the connection completion.
-
-  TransportCleanupGuard tg(base_transport);
-  if (!this->wait_for_connection_completion (r, base_transport, timeout))
-    {
-      if (TAO_debug_level > 2)
+  return false;
+}
+
+TAO_Transport*
+TAO_Connector::connect (TAO::Profile_Transport_Resolver *r,
+                        TAO_Transport_Descriptor_Interface *desc,
+                        ACE_Time_Value *timeout
+                        )
+{
+  if (desc == 0 ||
+      (this->set_validate_endpoint (desc->endpoint ()) == -1))
+    return 0;
+
+  TAO::Transport_Cache_Manager &tcm =
+    this->orb_core ()->lane_resources ().transport_cache ();
+  // Stay in this loop until we find:
+  // a usable connection, or a timeout happens
+  while (true)
+    {
+      // Find a connection in the cache
+      // If transport found, reference count is incremented on assignment
+      TAO_Transport *base_transport = 0;
+      unsigned int busy_count = 0;
+      TAO::Transport_Cache_Manager::Find_Result found =
+          tcm.find_transport (desc,
+                              base_transport,
+                              busy_count);
+
+      if (found == TAO::Transport_Cache_Manager::CACHE_FOUND_AVAILABLE )
         {
-          ACE_ERROR ((LM_ERROR,
-                      "TAO (%P|%t) - Transport_Connector::"
-                      "connect, "
-                      "wait for completion failed\n"));
+          // one last check before using the cached connection
+          if (base_transport->connection_handler()->error_detected ())
+            {
+              if (TAO_debug_level > 0)
+                {
+                  ACE_DEBUG ((LM_DEBUG,
+                    ACE_TEXT("TAO (%P|%t) Transport_Connector::connect")
+                    ACE_TEXT(" error in transport from cache\n")));
+                }
+              (void) base_transport->close_connection ();
+              (void) base_transport->purge_entry ();
+              base_transport = 0;
+            }
+          else
+            {
+              if (TAO_debug_level > 4)
+                {
+                  TAO::Connection_Role cr = base_transport->opened_as ();
+
+                  ACE_DEBUG ((LM_DEBUG,
+                              "TAO (%P|%t) - Transport_Connector::connect, "
+                              "got an existing %s Transport[%d] in role %s\n",
+                              base_transport->is_connected () ? "connected" : "unconnected",
+                              base_transport->id (),
+                              cr == TAO::TAO_SERVER_ROLE ? "TAO_SERVER_ROLE" :
+                              cr == TAO::TAO_CLIENT_ROLE ? "TAO_CLIENT_ROLE" :
+                              "TAO_UNSPECIFIED_ROLE" ));
+                }
+                return base_transport;
+            }
         }
-      return 0;
-    }
+      else if (found == TAO::Transport_Cache_Manager::CACHE_FOUND_CONNECTING )
+        {
+          if (r->blocked_connect ())
+            {
 
-  if (base_transport->is_connected () &&
-      base_transport->wait_strategy ()->register_handler () == -1)
-    {
-      // Registration failures.
-      if (TAO_debug_level > 0)
+              if (TAO_debug_level > 4)
+                {
+                  ACE_DEBUG ((LM_DEBUG,
+                              ACE_TEXT("TAO (%P|%t) - Transport_Connector::waiting")
+                              ACE_TEXT(" for connection on transport [%d]\n"),
+                              base_transport->id ()));
+                }
+
+                // If wait_for_transport returns no errors, the base_transport
+                // points to the connection we wait for.
+                if (this->wait_for_transport (r, base_transport, timeout, false))
+                  {
+                    // be sure this transport is registered with the reactor
+                    // before using it.
+                    if (!base_transport->register_if_necessary ())
+                      {
+                          base_transport->remove_reference ();
+                          return 0;
+                      }
+                  }
+                // In either success or failure cases of wait_for_transport, the
+                // ref counter in corresponding to the ref counter added by
+                // find_transport is decremented.
+                base_transport->remove_reference ();
+            }
+          else
+            {
+              if (TAO_debug_level > 4)
+                {
+                  ACE_DEBUG ((LM_DEBUG,
+                    ACE_TEXT("TAO (%P|%t) - Transport_Connector::non-blocking: returning unconnected transport [%d]\n"),
+                    base_transport->id ()));
+                }
+
+              // return the transport in it's current, unconnected state
+              return base_transport;
+            }
+        }
+      else
         {
-          ACE_ERROR ((LM_ERROR,
-                      "TAO (%P|%t) - Transport_Connector [%d]::connect, "
-                      "could not register the transport "
-                      "in the reactor.\n",
-                      base_transport->id ()));
+          // @@TODO: This is not the right place for this!
+          // Purge connections (if necessary)
+          tcm.purge ();
+
+          bool make_new_connection =
+            (found == TAO::Transport_Cache_Manager::CACHE_FOUND_NONE) ||
+            (found == TAO::Transport_Cache_Manager::CACHE_FOUND_BUSY
+                && this->new_connection_is_ok (busy_count));
+
+          if (make_new_connection)
+            {
+              TAO_Transport* t = this->make_connection (r, *desc, timeout);
+              if (t == 0)
+                {
+                  return t;
+                }
+
+              // Should this code be moved?  If so, where to?
+              t->opened_as (TAO::TAO_CLIENT_ROLE);
+
+              if (TAO_debug_level > 4)
+                {
+                  ACE_DEBUG ((LM_DEBUG,
+                              ACE_TEXT("TAO (%P|%t) - Transport_Connector::connect, ")
+                              ACE_TEXT("opening Transport[%d] in TAO_CLIENT_ROLE\n"),
+                              t->id ()));
+                }
+
+              // Call post connect hook. If the post_connect_hook () returns
+              // false, just purge the entry.
+              if (!t->post_connect_hook ())
+                {
+                  if (TAO_debug_level > 4)
+                    {
+                      ACE_DEBUG ((LM_DEBUG,
+                                  ACE_TEXT("TAO (%P|%t) - Post_connect_hook failed.  ")
+                                  ACE_TEXT("Purging transport[%d]\n"),
+                                  t->id ()));
+                    }
+                  (void) t->purge_entry ();
+                }
+              // The new transport is in the cache.  We'll pick it up from there
+              // next time thru this loop (using it from here causes more problems
+              // than it fixes due to the changes that allow a new connection to be
+              // re-used by a nested upcall before we get back here.)
+// temporarily reverted              t->remove_reference ();
+            }
+          else // not making new connection
+            {
+               (void) this->wait_for_transport (r, base_transport, timeout, true);
+                base_transport->remove_reference ();
+            }
         }
-      return 0;
     }
-
-  tg.down ();
-  return base_transport;
 }
 
 bool
 TAO_Connector::wait_for_connection_completion (
     TAO::Profile_Transport_Resolver *r,
+    TAO_Transport_Descriptor_Interface & desc,
     TAO_Transport *&transport,
     ACE_Time_Value *timeout)
 {
@@ -516,6 +645,18 @@ TAO_Connector::wait_for_connection_completion (
     }
   else
     {
+      if (TAO_debug_level > 4)
+        {
+            ACE_DEBUG ((LM_DEBUG,
+                        "TAO (%P|%t) - Transport_Connector::"
+                        "caching connection before wait_for_connection_completion "
+                        "%d = [%d]\n",
+                        desc.hash(),
+                        transport->id ()));
+        }
+      TAO::Transport_Cache_Manager &tcm =
+        this->orb_core ()->lane_resources ().transport_cache ();
+      tcm.cache_transport(&desc, transport, TAO::ENTRY_CONNECTING);
       if (TAO_debug_level > 2)
         {
             ACE_DEBUG ((LM_DEBUG,
@@ -525,6 +666,7 @@ TAO_Connector::wait_for_connection_completion (
                         "[%d]\n",
                         transport->id ()));
         }
+
       result = this->active_connect_strategy_->wait (transport, timeout);
 
       if (TAO_debug_level > 2)
@@ -721,6 +863,18 @@ TAO_Connector::create_connect_strategy (void)
   return 0;
 }
 
+bool
+TAO_Connector::new_connection_is_ok (unsigned int busy_count)
+{
+  if (orb_core_ == 0)
+    return true;
+
+  unsigned int mux_limit = orb_core_->resource_factory ()
+    ->max_muxed_connections ();
+
+  return mux_limit == 0 || busy_count < mux_limit;
+}
+
 int
 TAO_Connector::check_connection_closure (
   TAO_Connection_Handler *connection_handler)
diff --git a/TAO/tao/Transport_Connector.h b/TAO/tao/Transport_Connector.h
index ca9029f86ab..29d50a6d705 100644
--- a/TAO/tao/Transport_Connector.h
+++ b/TAO/tao/Transport_Connector.h
@@ -17,6 +17,7 @@
 #define TAO_CONNECTOR_H
 
 #include /**/ "ace/pre.h"
+//#include "ace/CORBA_macros.h"
 #include "tao/Basic_Types.h"
 
 #if !defined (ACE_LACKS_PRAGMA_ONCE)
@@ -172,6 +173,7 @@ protected:
    */
   virtual bool wait_for_connection_completion(
       TAO::Profile_Transport_Resolver *r,
+      TAO_Transport_Descriptor_Interface &desc,
       TAO_Transport *&transport,
       ACE_Time_Value *timeout);
 
@@ -186,6 +188,19 @@ protected:
       TAO_LF_Multi_Event *mev,
       ACE_Time_Value *timeout);
 
+  /// See if a new connection is allowed
+  bool new_connection_is_ok (unsigned int busy_count);
+
+  /// Wait for a transport to be connected
+  /// Note: no longer changes transport reference count
+  /// @ returns true if wait was uneventful
+  /// @         false if error occurred during wait
+  bool wait_for_transport(
+    TAO::Profile_Transport_Resolver *r,
+    TAO_Transport *base_transport,
+    ACE_Time_Value * timeout,
+    bool force_wait);
+
   /// Set the ORB Core pointer
   void orb_core (TAO_ORB_Core *orb_core);
 
diff --git a/TAO/tests/AMH_Oneway/client.cpp b/TAO/tests/AMH_Oneway/client.cpp
index 52fe2036819..fc77aa5f1a1 100644
--- a/TAO/tests/AMH_Oneway/client.cpp
+++ b/TAO/tests/AMH_Oneway/client.cpp
@@ -2,6 +2,7 @@
 
 #include "ace/Get_Opt.h"
 #include "TestC.h"
+#include "ace/OS_NS_unistd.h"
 
 const char *ior = "file://test.ior";
 int num_calls = 10;
@@ -60,6 +61,17 @@ ACE_TMAIN(int argc, ACE_TCHAR *argv[])
           ACE_DEBUG ((LM_DEBUG, "Sent call # %d \n", i));
         }
       orb->destroy();
+      // The following sleep is a workaround for a defect in the Windows
+      // implementation of sockets (Win XP)
+      // The when this client exits after writing to a localhost socket
+      // Windows discards any data that has not been read by the server.
+      // The sleep gives the server time to catch up.  num_calls/2 gives
+      // it half a second per request which *really* should be overkill, but
+      // it also means the client will terminate before the server actually
+      // handles the requests (a good thing).
+      // I'm still trying to decide whether this should be a bugzilla entry.
+      // wilsond@ociweb.com
+      ACE_OS::sleep(num_calls/2);
     }
   catch (const CORBA::Exception& ex)
     {
diff --git a/TAO/tests/AMH_Oneway/server.cpp b/TAO/tests/AMH_Oneway/server.cpp
index 7492e455f8a..f0623094d0b 100644
--- a/TAO/tests/AMH_Oneway/server.cpp
+++ b/TAO/tests/AMH_Oneway/server.cpp
@@ -251,13 +251,13 @@ ST_AMH_Server::write_ior_to_file (CORBA::String_var ior)
 int
 ACE_TMAIN(int argc, ACE_TCHAR *argv[])
 {
-  if (parse_args (argc, argv) != 0)
-    return 1;
-
   ST_AMH_Server amh_server (&argc, argv);
 
   amh_server.start_orb_and_poa ();
 
+  if (parse_args (argc, argv) != 0)
+    return 1;
+
   ST_AMH_Servant servant (amh_server.orb ());
 
   amh_server.register_servant (&servant);
diff --git a/TAO/tests/Bug_1476_Test/client.cpp b/TAO/tests/Bug_1476_Test/client.cpp
index 4dab919d4fe..0ddc58c4793 100644
--- a/TAO/tests/Bug_1476_Test/client.cpp
+++ b/TAO/tests/Bug_1476_Test/client.cpp
@@ -183,21 +183,44 @@ ACE_TMAIN(int argc, ACE_TCHAR *argv[])
         1);
       }
 
-      if (tcm.find_transport (&desc, transport) != 0) {
-        ACE_ERROR_RETURN ((LM_ERROR,
-         "The cache should have returned the just created transport.\n"),
-        1);
+      unsigned int busy_count = 0;
+      TAO::Transport_Cache_Manager::Find_Result find_result =
+        tcm.find_transport (&desc, transport, busy_count);
+
+      switch (find_result){
+        case TAO::Transport_Cache_Manager::CACHE_FOUND_NONE:
+          {
+            ACE_ERROR_RETURN ((LM_ERROR,
+              ACE_TEXT("Expected to find a transport in the cache.\n")
+              ),1);
+          }
+        case TAO::Transport_Cache_Manager::CACHE_FOUND_CONNECTING:
+          {
+            ACE_DEBUG (( LM_DEBUG,
+              ACE_TEXT ("Transport Cache contains connecting entry as expected.\n")
+              ));
+            break; // that's what we expected
+          }
+        case TAO::Transport_Cache_Manager::CACHE_FOUND_BUSY:
+          {
+            ACE_ERROR_RETURN ((LM_ERROR,
+              ACE_TEXT("Cached Transport is busy.  Should not happen because there's no server.\n")
+              ),1);
+          }
+        case TAO::Transport_Cache_Manager::CACHE_FOUND_AVAILABLE:
+          {
+            ACE_ERROR_RETURN ((LM_ERROR,
+              ACE_TEXT("Cached Transport is available.  Should not happen because there's no server.\n")
+              ),1);
+          }
+        default:
+          {
+            ACE_ERROR_RETURN ((LM_ERROR,
+              ACE_TEXT("Transport_Cache::find returned unknown status.\n")
+              ),1);
+          }
       }
 
-      if (transport->is_connected()) {
-        ACE_ERROR_RETURN ((LM_ERROR,
-         "This cannot happen. No server to connect to."),
-        1);
-      }
-
-      if (transport != 0) {
-        ACE_DEBUG ((LM_DEBUG, "Transport is %s\n", (transport->is_connected()?"connected":"not connected")));
-      }
 
       orb->destroy ();
     }